diff --git a/adam.log b/adam.log deleted file mode 100644 index 31b366d..0000000 --- a/adam.log +++ /dev/null @@ -1,296337 +0,0 @@ -2016-05-18 19:24:44 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-18 19:24:46 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-18 19:24:47 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-18 19:24:47 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-18 19:24:47 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-18 19:24:50 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-18 19:24:51 INFO Remoting:74 - Starting remoting -2016-05-18 19:24:52 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.142:64373] -2016-05-18 19:24:52 INFO Utils:59 - Successfully started service 'sparkDriver' on port 64373. -2016-05-18 19:24:52 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-18 19:24:52 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-18 19:24:52 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-5d9062c6-b12d-43e1-a859-e5206c690e18 -2016-05-18 19:24:53 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-18 19:24:53 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-876ce1ec-8fec-42be-b0f9-86d1c5bd550f\httpd-dc3677b8-33f5-4293-816e-ab3967aba583 -2016-05-18 19:24:53 INFO HttpServer:59 - Starting HTTP Server -2016-05-18 19:24:53 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-18 19:24:53 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:64381 -2016-05-18 19:24:53 INFO Utils:59 - Successfully started service 'HTTP file server' on port 64381. -2016-05-18 19:24:53 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-18 19:24:53 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-18 19:24:53 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-18 19:24:53 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-18 19:24:54 INFO SparkUI:59 - Started SparkUI at http://211.86.159.142:4040 -2016-05-18 19:24:54 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-18 19:24:54 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-18 19:24:55 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 64400. -2016-05-18 19:24:55 INFO NettyBlockTransferService:59 - Server created on 64400 -2016-05-18 19:24:55 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-18 19:24:55 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:64400 with 730.6 MB RAM, BlockManagerId(driver, localhost, 64400) -2016-05-18 19:24:55 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-18 19:24:57 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-18 19:24:57 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-18 19:24:57 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-18 19:24:57 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-18 19:24:57 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:64400 (size: 13.9 KB, free: 730.6 MB) -2016-05-18 19:24:57 INFO SparkContext:59 - Created broadcast 0 from textFile at ALSFromSpark.scala:30 -2016-05-18 19:24:59 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f8e%20, but we couldn't find any external IP address! -2016-05-18 19:25:02 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-18 19:25:02 INFO SparkContext:59 - Starting job: count at ALS.scala:550 -2016-05-18 19:25:03 INFO DAGScheduler:59 - Registering RDD 4 (mapPartitions at ALS.scala:791) -2016-05-18 19:25:03 INFO DAGScheduler:59 - Registering RDD 7 (map at ALS.scala:1034) -2016-05-18 19:25:03 INFO DAGScheduler:59 - Got job 0 (count at ALS.scala:550) with 1 output partitions -2016-05-18 19:25:03 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at ALS.scala:550) -2016-05-18 19:25:03 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1) -2016-05-18 19:25:03 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1) -2016-05-18 19:25:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[4] at mapPartitions at ALS.scala:791), which has no missing parents -2016-05-18 19:25:03 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=171580, maxMem=766075207 -2016-05-18 19:25:03 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-18 19:25:03 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=177108, maxMem=766075207 -2016-05-18 19:25:03 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-18 19:25:03 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:64400 (size: 2.9 KB, free: 730.6 MB) -2016-05-18 19:25:03 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:03 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[4] at mapPartitions at ALS.scala:791) -2016-05-18 19:25:03 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-18 19:25:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2161 bytes) -2016-05-18 19:25:03 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-18 19:25:03 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/test.data:0+127 -2016-05-18 19:25:03 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-18 19:25:03 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-18 19:25:03 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-18 19:25:03 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-18 19:25:03 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-18 19:25:03 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2253 bytes result sent to driver -2016-05-18 19:25:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 610 ms on localhost (1/1) -2016-05-18 19:25:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-18 19:25:04 INFO DAGScheduler:59 - ShuffleMapStage 0 (mapPartitions at ALS.scala:791) finished in 0.694 s -2016-05-18 19:25:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:04 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:04 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 1, ResultStage 2) -2016-05-18 19:25:04 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:04 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 1: List() -2016-05-18 19:25:04 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List(ShuffleMapStage 1) -2016-05-18 19:25:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1 (MapPartitionsRDD[7] at map at ALS.scala:1034), which is now runnable -2016-05-18 19:25:04 INFO MemoryStore:59 - ensureFreeSpace(6408) called with curMem=180071, maxMem=766075207 -2016-05-18 19:25:04 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 6.3 KB, free 730.4 MB) -2016-05-18 19:25:04 INFO MemoryStore:59 - ensureFreeSpace(3293) called with curMem=186479, maxMem=766075207 -2016-05-18 19:25:04 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.4 MB) -2016-05-18 19:25:04 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:64400 (size: 3.2 KB, free: 730.6 MB) -2016-05-18 19:25:04 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:04 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[7] at map at ALS.scala:1034) -2016-05-18 19:25:04 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-18 19:25:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-18 19:25:04 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-18 19:25:04 INFO CacheManager:59 - Partition rdd_6_0 not found, computing it -2016-05-18 19:25:04 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-18 19:25:04 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:64400 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-18 19:25:04 INFO MemoryStore:59 - ensureFreeSpace(344) called with curMem=181281, maxMem=766075207 -2016-05-18 19:25:04 INFO MemoryStore:59 - Block rdd_6_0 stored as values in memory (estimated size 344.0 B, free 730.4 MB) -2016-05-18 19:25:04 INFO BlockManagerInfo:59 - Added rdd_6_0 in memory on localhost:64400 (size: 344.0 B, free: 730.6 MB) -2016-05-18 19:25:04 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 1954 bytes result sent to driver -2016-05-18 19:25:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 316 ms on localhost (1/1) -2016-05-18 19:25:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-18 19:25:04 INFO DAGScheduler:59 - ShuffleMapStage 1 (map at ALS.scala:1034) finished in 0.319 s -2016-05-18 19:25:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:04 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-05-18 19:25:04 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:04 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List() -2016-05-18 19:25:04 INFO DAGScheduler:59 - Submitting ResultStage 2 (userOutBlocks MapPartitionsRDD[10] at mapValues at ALS.scala:1071), which is now runnable -2016-05-18 19:25:04 INFO MemoryStore:59 - ensureFreeSpace(6992) called with curMem=181625, maxMem=766075207 -2016-05-18 19:25:04 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 6.8 KB, free 730.4 MB) -2016-05-18 19:25:04 INFO MemoryStore:59 - ensureFreeSpace(3457) called with curMem=188617, maxMem=766075207 -2016-05-18 19:25:04 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-18 19:25:04 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:64400 (size: 3.4 KB, free: 730.6 MB) -2016-05-18 19:25:04 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:04 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (userOutBlocks MapPartitionsRDD[10] at mapValues at ALS.scala:1071) -2016-05-18 19:25:04 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-18 19:25:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-18 19:25:04 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-05-18 19:25:04 INFO CacheManager:59 - Partition rdd_10_0 not found, computing it -2016-05-18 19:25:04 INFO CacheManager:59 - Partition rdd_9_0 not found, computing it -2016-05-18 19:25:04 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-18 19:25:04 INFO MemoryStore:59 - ensureFreeSpace(328) called with curMem=192074, maxMem=766075207 -2016-05-18 19:25:04 INFO MemoryStore:59 - Block rdd_9_0 stored as values in memory (estimated size 328.0 B, free 730.4 MB) -2016-05-18 19:25:04 INFO BlockManagerInfo:59 - Added rdd_9_0 in memory on localhost:64400 (size: 328.0 B, free: 730.6 MB) -2016-05-18 19:25:04 INFO MemoryStore:59 - ensureFreeSpace(120) called with curMem=192402, maxMem=766075207 -2016-05-18 19:25:04 INFO MemoryStore:59 - Block rdd_10_0 stored as values in memory (estimated size 120.0 B, free 730.4 MB) -2016-05-18 19:25:04 INFO BlockManagerInfo:59 - Added rdd_10_0 in memory on localhost:64400 (size: 120.0 B, free: 730.6 MB) -2016-05-18 19:25:04 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 1838 bytes result sent to driver -2016-05-18 19:25:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 57 ms on localhost (1/1) -2016-05-18 19:25:04 INFO DAGScheduler:59 - ResultStage 2 (count at ALS.scala:550) finished in 0.057 s -2016-05-18 19:25:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-18 19:25:04 INFO DAGScheduler:59 - Job 0 finished: count at ALS.scala:550, took 1.583657 s -2016-05-18 19:25:04 INFO SparkContext:59 - Starting job: count at ALS.scala:558 -2016-05-18 19:25:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 143 bytes -2016-05-18 19:25:04 INFO DAGScheduler:59 - Registering RDD 12 (map at ALS.scala:1034) -2016-05-18 19:25:04 INFO DAGScheduler:59 - Got job 1 (count at ALS.scala:558) with 1 output partitions -2016-05-18 19:25:04 INFO DAGScheduler:59 - Final stage: ResultStage 5(count at ALS.scala:558) -2016-05-18 19:25:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-18 19:25:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-18 19:25:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[12] at map at ALS.scala:1034), which has no missing parents -2016-05-18 19:25:04 INFO MemoryStore:59 - ensureFreeSpace(6568) called with curMem=192522, maxMem=766075207 -2016-05-18 19:25:04 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 6.4 KB, free 730.4 MB) -2016-05-18 19:25:04 INFO MemoryStore:59 - ensureFreeSpace(3326) called with curMem=199090, maxMem=766075207 -2016-05-18 19:25:04 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.4 MB) -2016-05-18 19:25:04 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:64400 (size: 3.2 KB, free: 730.6 MB) -2016-05-18 19:25:04 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:04 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[12] at map at ALS.scala:1034) -2016-05-18 19:25:04 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-05-18 19:25:04 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:64400 in memory (size: 3.4 KB, free: 730.6 MB) -2016-05-18 19:25:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 3, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-18 19:25:04 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 3) -2016-05-18 19:25:04 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-18 19:25:04 INFO BlockManager:59 - Found block rdd_6_0 locally -2016-05-18 19:25:04 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 3). 2253 bytes result sent to driver -2016-05-18 19:25:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 3) in 27 ms on localhost (1/1) -2016-05-18 19:25:04 INFO DAGScheduler:59 - ShuffleMapStage 4 (map at ALS.scala:1034) finished in 0.028 s -2016-05-18 19:25:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-18 19:25:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:04 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-18 19:25:04 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:04 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-18 19:25:04 INFO DAGScheduler:59 - Submitting ResultStage 5 (itemOutBlocks MapPartitionsRDD[15] at mapValues at ALS.scala:1071), which is now runnable -2016-05-18 19:25:04 INFO MemoryStore:59 - ensureFreeSpace(7144) called with curMem=191967, maxMem=766075207 -2016-05-18 19:25:04 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 7.0 KB, free 730.4 MB) -2016-05-18 19:25:04 INFO MemoryStore:59 - ensureFreeSpace(3497) called with curMem=199111, maxMem=766075207 -2016-05-18 19:25:04 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-18 19:25:04 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:64400 (size: 3.4 KB, free: 730.6 MB) -2016-05-18 19:25:04 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:04 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 5 (itemOutBlocks MapPartitionsRDD[15] at mapValues at ALS.scala:1071) -2016-05-18 19:25:04 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 1 tasks -2016-05-18 19:25:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 4, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-18 19:25:04 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 4) -2016-05-18 19:25:04 INFO CacheManager:59 - Partition rdd_15_0 not found, computing it -2016-05-18 19:25:04 INFO CacheManager:59 - Partition rdd_14_0 not found, computing it -2016-05-18 19:25:04 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-18 19:25:04 INFO MemoryStore:59 - ensureFreeSpace(328) called with curMem=202608, maxMem=766075207 -2016-05-18 19:25:04 INFO MemoryStore:59 - Block rdd_14_0 stored as values in memory (estimated size 328.0 B, free 730.4 MB) -2016-05-18 19:25:04 INFO BlockManagerInfo:59 - Added rdd_14_0 in memory on localhost:64400 (size: 328.0 B, free: 730.6 MB) -2016-05-18 19:25:04 INFO MemoryStore:59 - ensureFreeSpace(120) called with curMem=202936, maxMem=766075207 -2016-05-18 19:25:04 INFO MemoryStore:59 - Block rdd_15_0 stored as values in memory (estimated size 120.0 B, free 730.4 MB) -2016-05-18 19:25:04 INFO BlockManagerInfo:59 - Added rdd_15_0 in memory on localhost:64400 (size: 120.0 B, free: 730.6 MB) -2016-05-18 19:25:04 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 4). 1838 bytes result sent to driver -2016-05-18 19:25:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 4) in 24 ms on localhost (1/1) -2016-05-18 19:25:04 INFO DAGScheduler:59 - ResultStage 5 (count at ALS.scala:558) finished in 0.025 s -2016-05-18 19:25:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-18 19:25:04 INFO DAGScheduler:59 - Job 1 finished: count at ALS.scala:558, took 0.106330 s -2016-05-18 19:25:04 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:64400 in memory (size: 3.4 KB, free: 730.6 MB) -2016-05-18 19:25:04 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-18 19:25:04 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:64400 in memory (size: 3.2 KB, free: 730.6 MB) -2016-05-18 19:25:05 INFO SparkContext:59 - Starting job: count at ALS.scala:263 -2016-05-18 19:25:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 143 bytes -2016-05-18 19:25:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 143 bytes -2016-05-18 19:25:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 143 bytes -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 16 (map at ALS.scala:706) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 21 (flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 30 (flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 39 (flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 48 (flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 57 (flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 66 (flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 75 (flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 84 (flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 93 (flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 102 (flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 111 (flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 120 (flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 129 (flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 138 (flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 147 (flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 156 (flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 165 (flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 174 (flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 183 (flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Registering RDD 192 (flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Got job 2 (count at ALS.scala:263) with 1 output partitions -2016-05-18 19:25:05 INFO DAGScheduler:59 - Final stage: ResultStage 31(count at ALS.scala:263) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30, ShuffleMapStage 9) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 30) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 10 (MapPartitionsRDD[16] at map at ALS.scala:706), which has no missing parents -2016-05-18 19:25:05 INFO MemoryStore:59 - ensureFreeSpace(6912) called with curMem=182521, maxMem=766075207 -2016-05-18 19:25:05 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 6.8 KB, free 730.4 MB) -2016-05-18 19:25:05 INFO MemoryStore:59 - ensureFreeSpace(3461) called with curMem=189433, maxMem=766075207 -2016-05-18 19:25:05 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-18 19:25:05 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:64400 (size: 3.4 KB, free: 730.6 MB) -2016-05-18 19:25:05 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:05 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 10 (MapPartitionsRDD[16] at map at ALS.scala:706) -2016-05-18 19:25:05 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-18 19:25:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 5, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-18 19:25:05 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 5) -2016-05-18 19:25:05 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-18 19:25:05 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-18 19:25:05 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-18 19:25:05 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 5). 2253 bytes result sent to driver -2016-05-18 19:25:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 5) in 44 ms on localhost (1/1) -2016-05-18 19:25:05 INFO DAGScheduler:59 - ShuffleMapStage 10 (map at ALS.scala:706) finished in 0.026 s -2016-05-18 19:25:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-18 19:25:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:05 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:05 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 15, ShuffleMapStage 30, ShuffleMapStage 16, ResultStage 31, ShuffleMapStage 17, ShuffleMapStage 24, ShuffleMapStage 18, ShuffleMapStage 25, ShuffleMapStage 11, ShuffleMapStage 26, ShuffleMapStage 12, ShuffleMapStage 27, ShuffleMapStage 19, ShuffleMapStage 13, ShuffleMapStage 20, ShuffleMapStage 21, ShuffleMapStage 28, ShuffleMapStage 22, ShuffleMapStage 14, ShuffleMapStage 29, ShuffleMapStage 23) -2016-05-18 19:25:05 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 15: List(ShuffleMapStage 14) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List(ShuffleMapStage 29) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 16: List(ShuffleMapStage 15) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 17: List(ShuffleMapStage 16) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 24: List(ShuffleMapStage 23) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 18: List(ShuffleMapStage 17) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 25: List(ShuffleMapStage 24) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 11: List() -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 26: List(ShuffleMapStage 25) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 12: List(ShuffleMapStage 11) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 27: List(ShuffleMapStage 26) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 19: List(ShuffleMapStage 18) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 13: List(ShuffleMapStage 12) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 20: List(ShuffleMapStage 19) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 21: List(ShuffleMapStage 20) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 28: List(ShuffleMapStage 27) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 22: List(ShuffleMapStage 21) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 14: List(ShuffleMapStage 13) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 29: List(ShuffleMapStage 28) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 23: List(ShuffleMapStage 22) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 11 (MapPartitionsRDD[21] at flatMap at ALS.scala:1124), which is now runnable -2016-05-18 19:25:05 INFO MemoryStore:59 - ensureFreeSpace(7904) called with curMem=192894, maxMem=766075207 -2016-05-18 19:25:05 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 7.7 KB, free 730.4 MB) -2016-05-18 19:25:05 INFO MemoryStore:59 - ensureFreeSpace(3663) called with curMem=200798, maxMem=766075207 -2016-05-18 19:25:05 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-18 19:25:05 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:64400 (size: 3.6 KB, free: 730.6 MB) -2016-05-18 19:25:05 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:05 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 11 (MapPartitionsRDD[21] at flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 1 tasks -2016-05-18 19:25:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 6, localhost, PROCESS_LOCAL, 2117 bytes) -2016-05-18 19:25:05 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 6) -2016-05-18 19:25:05 INFO BlockManager:59 - Found block rdd_10_0 locally -2016-05-18 19:25:05 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-18 19:25:05 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 6). 2469 bytes result sent to driver -2016-05-18 19:25:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 6) in 49 ms on localhost (1/1) -2016-05-18 19:25:05 INFO DAGScheduler:59 - ShuffleMapStage 11 (flatMap at ALS.scala:1124) finished in 0.051 s -2016-05-18 19:25:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-18 19:25:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:05 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:05 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 15, ShuffleMapStage 30, ShuffleMapStage 16, ResultStage 31, ShuffleMapStage 17, ShuffleMapStage 24, ShuffleMapStage 18, ShuffleMapStage 25, ShuffleMapStage 26, ShuffleMapStage 12, ShuffleMapStage 27, ShuffleMapStage 19, ShuffleMapStage 13, ShuffleMapStage 20, ShuffleMapStage 21, ShuffleMapStage 28, ShuffleMapStage 22, ShuffleMapStage 14, ShuffleMapStage 29, ShuffleMapStage 23) -2016-05-18 19:25:05 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 15: List(ShuffleMapStage 14) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List(ShuffleMapStage 29) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 16: List(ShuffleMapStage 15) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 17: List(ShuffleMapStage 16) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 24: List(ShuffleMapStage 23) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 18: List(ShuffleMapStage 17) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 25: List(ShuffleMapStage 24) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 26: List(ShuffleMapStage 25) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 12: List() -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 27: List(ShuffleMapStage 26) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 19: List(ShuffleMapStage 18) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 13: List(ShuffleMapStage 12) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 20: List(ShuffleMapStage 19) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 21: List(ShuffleMapStage 20) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 28: List(ShuffleMapStage 27) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 22: List(ShuffleMapStage 21) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 14: List(ShuffleMapStage 13) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 29: List(ShuffleMapStage 28) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 23: List(ShuffleMapStage 22) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (MapPartitionsRDD[30] at flatMap at ALS.scala:1124), which is now runnable -2016-05-18 19:25:05 INFO MemoryStore:59 - ensureFreeSpace(10080) called with curMem=204461, maxMem=766075207 -2016-05-18 19:25:05 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 9.8 KB, free 730.4 MB) -2016-05-18 19:25:05 INFO MemoryStore:59 - ensureFreeSpace(4379) called with curMem=214541, maxMem=766075207 -2016-05-18 19:25:05 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.4 MB) -2016-05-18 19:25:05 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:64400 (size: 4.3 KB, free: 730.6 MB) -2016-05-18 19:25:05 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:05 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 12 (MapPartitionsRDD[30] at flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-18 19:25:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 7, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-18 19:25:05 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 7) -2016-05-18 19:25:05 INFO BlockManager:59 - Found block rdd_15_0 locally -2016-05-18 19:25:05 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-05-18 19:25:05 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-18 19:25:05 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-05-18 19:25:05 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-05-18 19:25:05 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 7). 2469 bytes result sent to driver -2016-05-18 19:25:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 7) in 115 ms on localhost (1/1) -2016-05-18 19:25:05 INFO DAGScheduler:59 - ShuffleMapStage 12 (flatMap at ALS.scala:1124) finished in 0.116 s -2016-05-18 19:25:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-18 19:25:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:05 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:05 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 15, ShuffleMapStage 30, ShuffleMapStage 16, ResultStage 31, ShuffleMapStage 17, ShuffleMapStage 24, ShuffleMapStage 18, ShuffleMapStage 25, ShuffleMapStage 26, ShuffleMapStage 27, ShuffleMapStage 19, ShuffleMapStage 13, ShuffleMapStage 20, ShuffleMapStage 21, ShuffleMapStage 28, ShuffleMapStage 22, ShuffleMapStage 14, ShuffleMapStage 29, ShuffleMapStage 23) -2016-05-18 19:25:05 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 15: List(ShuffleMapStage 14) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List(ShuffleMapStage 29) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 16: List(ShuffleMapStage 15) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 17: List(ShuffleMapStage 16) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 24: List(ShuffleMapStage 23) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 18: List(ShuffleMapStage 17) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 25: List(ShuffleMapStage 24) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 26: List(ShuffleMapStage 25) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 27: List(ShuffleMapStage 26) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 19: List(ShuffleMapStage 18) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 13: List() -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 20: List(ShuffleMapStage 19) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 21: List(ShuffleMapStage 20) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 28: List(ShuffleMapStage 27) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 22: List(ShuffleMapStage 21) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 14: List(ShuffleMapStage 13) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 29: List(ShuffleMapStage 28) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 23: List(ShuffleMapStage 22) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 13 (MapPartitionsRDD[39] at flatMap at ALS.scala:1124), which is now runnable -2016-05-18 19:25:05 INFO MemoryStore:59 - ensureFreeSpace(10944) called with curMem=218920, maxMem=766075207 -2016-05-18 19:25:05 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 10.7 KB, free 730.4 MB) -2016-05-18 19:25:05 INFO MemoryStore:59 - ensureFreeSpace(4599) called with curMem=229864, maxMem=766075207 -2016-05-18 19:25:05 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.4 MB) -2016-05-18 19:25:05 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:64400 (size: 4.5 KB, free: 730.6 MB) -2016-05-18 19:25:05 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:05 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 13 (MapPartitionsRDD[39] at flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 1 tasks -2016-05-18 19:25:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 8, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-18 19:25:05 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 8) -2016-05-18 19:25:05 INFO BlockManager:59 - Found block rdd_10_0 locally -2016-05-18 19:25:05 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-18 19:25:05 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-18 19:25:05 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 8). 2469 bytes result sent to driver -2016-05-18 19:25:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 8) in 27 ms on localhost (1/1) -2016-05-18 19:25:05 INFO DAGScheduler:59 - ShuffleMapStage 13 (flatMap at ALS.scala:1124) finished in 0.029 s -2016-05-18 19:25:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-18 19:25:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:05 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:05 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 15, ShuffleMapStage 30, ShuffleMapStage 16, ResultStage 31, ShuffleMapStage 17, ShuffleMapStage 24, ShuffleMapStage 18, ShuffleMapStage 25, ShuffleMapStage 26, ShuffleMapStage 27, ShuffleMapStage 19, ShuffleMapStage 20, ShuffleMapStage 21, ShuffleMapStage 28, ShuffleMapStage 22, ShuffleMapStage 14, ShuffleMapStage 29, ShuffleMapStage 23) -2016-05-18 19:25:05 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 15: List(ShuffleMapStage 14) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List(ShuffleMapStage 29) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 16: List(ShuffleMapStage 15) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 17: List(ShuffleMapStage 16) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 24: List(ShuffleMapStage 23) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 18: List(ShuffleMapStage 17) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 25: List(ShuffleMapStage 24) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 26: List(ShuffleMapStage 25) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 27: List(ShuffleMapStage 26) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 19: List(ShuffleMapStage 18) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 20: List(ShuffleMapStage 19) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 21: List(ShuffleMapStage 20) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 28: List(ShuffleMapStage 27) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 22: List(ShuffleMapStage 21) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 14: List() -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 29: List(ShuffleMapStage 28) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 23: List(ShuffleMapStage 22) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 14 (MapPartitionsRDD[48] at flatMap at ALS.scala:1124), which is now runnable -2016-05-18 19:25:05 INFO MemoryStore:59 - ensureFreeSpace(11792) called with curMem=234463, maxMem=766075207 -2016-05-18 19:25:05 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 11.5 KB, free 730.4 MB) -2016-05-18 19:25:05 INFO MemoryStore:59 - ensureFreeSpace(4765) called with curMem=246255, maxMem=766075207 -2016-05-18 19:25:05 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 4.7 KB, free 730.3 MB) -2016-05-18 19:25:05 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:64400 (size: 4.7 KB, free: 730.5 MB) -2016-05-18 19:25:05 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:05 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 14 (MapPartitionsRDD[48] at flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 1 tasks -2016-05-18 19:25:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 9, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-18 19:25:05 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 9) -2016-05-18 19:25:05 INFO BlockManager:59 - Found block rdd_15_0 locally -2016-05-18 19:25:05 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-05-18 19:25:05 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-18 19:25:05 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 9). 2469 bytes result sent to driver -2016-05-18 19:25:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 9) in 31 ms on localhost (1/1) -2016-05-18 19:25:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-18 19:25:05 INFO DAGScheduler:59 - ShuffleMapStage 14 (flatMap at ALS.scala:1124) finished in 0.032 s -2016-05-18 19:25:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:05 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:05 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 15, ShuffleMapStage 30, ShuffleMapStage 16, ResultStage 31, ShuffleMapStage 17, ShuffleMapStage 24, ShuffleMapStage 18, ShuffleMapStage 25, ShuffleMapStage 26, ShuffleMapStage 27, ShuffleMapStage 19, ShuffleMapStage 20, ShuffleMapStage 21, ShuffleMapStage 28, ShuffleMapStage 22, ShuffleMapStage 29, ShuffleMapStage 23) -2016-05-18 19:25:05 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 15: List() -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List(ShuffleMapStage 29) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 16: List(ShuffleMapStage 15) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 17: List(ShuffleMapStage 16) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 24: List(ShuffleMapStage 23) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 18: List(ShuffleMapStage 17) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 25: List(ShuffleMapStage 24) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 26: List(ShuffleMapStage 25) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 27: List(ShuffleMapStage 26) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 19: List(ShuffleMapStage 18) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 20: List(ShuffleMapStage 19) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 21: List(ShuffleMapStage 20) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 28: List(ShuffleMapStage 27) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 22: List(ShuffleMapStage 21) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 29: List(ShuffleMapStage 28) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 23: List(ShuffleMapStage 22) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 15 (MapPartitionsRDD[57] at flatMap at ALS.scala:1124), which is now runnable -2016-05-18 19:25:05 INFO MemoryStore:59 - ensureFreeSpace(12640) called with curMem=251020, maxMem=766075207 -2016-05-18 19:25:05 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 12.3 KB, free 730.3 MB) -2016-05-18 19:25:05 INFO MemoryStore:59 - ensureFreeSpace(4972) called with curMem=263660, maxMem=766075207 -2016-05-18 19:25:05 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 4.9 KB, free 730.3 MB) -2016-05-18 19:25:05 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:64400 (size: 4.9 KB, free: 730.5 MB) -2016-05-18 19:25:05 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:05 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 15 (MapPartitionsRDD[57] at flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 1 tasks -2016-05-18 19:25:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 10, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-18 19:25:05 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 10) -2016-05-18 19:25:05 INFO BlockManager:59 - Found block rdd_10_0 locally -2016-05-18 19:25:05 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-18 19:25:05 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-18 19:25:05 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 10). 2469 bytes result sent to driver -2016-05-18 19:25:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 10) in 26 ms on localhost (1/1) -2016-05-18 19:25:05 INFO DAGScheduler:59 - ShuffleMapStage 15 (flatMap at ALS.scala:1124) finished in 0.026 s -2016-05-18 19:25:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-18 19:25:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:05 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:05 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 30, ShuffleMapStage 16, ResultStage 31, ShuffleMapStage 17, ShuffleMapStage 24, ShuffleMapStage 18, ShuffleMapStage 25, ShuffleMapStage 26, ShuffleMapStage 27, ShuffleMapStage 19, ShuffleMapStage 20, ShuffleMapStage 21, ShuffleMapStage 28, ShuffleMapStage 22, ShuffleMapStage 29, ShuffleMapStage 23) -2016-05-18 19:25:05 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List(ShuffleMapStage 29) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 16: List() -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 17: List(ShuffleMapStage 16) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 24: List(ShuffleMapStage 23) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 18: List(ShuffleMapStage 17) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 25: List(ShuffleMapStage 24) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 26: List(ShuffleMapStage 25) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 27: List(ShuffleMapStage 26) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 19: List(ShuffleMapStage 18) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 20: List(ShuffleMapStage 19) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 21: List(ShuffleMapStage 20) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 28: List(ShuffleMapStage 27) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 22: List(ShuffleMapStage 21) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 29: List(ShuffleMapStage 28) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 23: List(ShuffleMapStage 22) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 16 (MapPartitionsRDD[66] at flatMap at ALS.scala:1124), which is now runnable -2016-05-18 19:25:05 INFO MemoryStore:59 - ensureFreeSpace(13488) called with curMem=268632, maxMem=766075207 -2016-05-18 19:25:05 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 13.2 KB, free 730.3 MB) -2016-05-18 19:25:05 INFO MemoryStore:59 - ensureFreeSpace(5118) called with curMem=282120, maxMem=766075207 -2016-05-18 19:25:05 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 5.0 KB, free 730.3 MB) -2016-05-18 19:25:05 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:64400 (size: 5.0 KB, free: 730.5 MB) -2016-05-18 19:25:05 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:05 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:64400 in memory (size: 4.7 KB, free: 730.5 MB) -2016-05-18 19:25:05 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 16 (MapPartitionsRDD[66] at flatMap at ALS.scala:1124) -2016-05-18 19:25:05 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 1 tasks -2016-05-18 19:25:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 11, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-18 19:25:05 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 11) -2016-05-18 19:25:05 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:64400 in memory (size: 4.5 KB, free: 730.5 MB) -2016-05-18 19:25:05 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:64400 in memory (size: 4.3 KB, free: 730.6 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:64400 in memory (size: 3.6 KB, free: 730.6 MB) -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_15_0 locally -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-18 19:25:06 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 11). 2469 bytes result sent to driver -2016-05-18 19:25:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 11) in 30 ms on localhost (1/1) -2016-05-18 19:25:06 INFO DAGScheduler:59 - ShuffleMapStage 16 (flatMap at ALS.scala:1124) finished in 0.030 s -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-18 19:25:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:06 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 30, ResultStage 31, ShuffleMapStage 17, ShuffleMapStage 24, ShuffleMapStage 18, ShuffleMapStage 25, ShuffleMapStage 26, ShuffleMapStage 27, ShuffleMapStage 19, ShuffleMapStage 20, ShuffleMapStage 21, ShuffleMapStage 28, ShuffleMapStage 22, ShuffleMapStage 29, ShuffleMapStage 23) -2016-05-18 19:25:06 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List(ShuffleMapStage 29) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 17: List() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 24: List(ShuffleMapStage 23) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 18: List(ShuffleMapStage 17) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 25: List(ShuffleMapStage 24) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 26: List(ShuffleMapStage 25) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 27: List(ShuffleMapStage 26) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 19: List(ShuffleMapStage 18) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 20: List(ShuffleMapStage 19) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 21: List(ShuffleMapStage 20) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 28: List(ShuffleMapStage 27) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 22: List(ShuffleMapStage 21) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 29: List(ShuffleMapStage 28) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 23: List(ShuffleMapStage 22) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 17 (MapPartitionsRDD[75] at flatMap at ALS.scala:1124), which is now runnable -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(14336) called with curMem=229112, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 14.0 KB, free 730.4 MB) -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(5285) called with curMem=243448, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 5.2 KB, free 730.3 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:64400 (size: 5.2 KB, free: 730.6 MB) -2016-05-18 19:25:06 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 17 (MapPartitionsRDD[75] at flatMap at ALS.scala:1124) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 1 tasks -2016-05-18 19:25:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 12, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-18 19:25:06 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 12) -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_10_0 locally -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-18 19:25:06 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 12). 2469 bytes result sent to driver -2016-05-18 19:25:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 12) in 38 ms on localhost (1/1) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-18 19:25:06 INFO DAGScheduler:59 - ShuffleMapStage 17 (flatMap at ALS.scala:1124) finished in 0.038 s -2016-05-18 19:25:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:06 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 30, ResultStage 31, ShuffleMapStage 24, ShuffleMapStage 18, ShuffleMapStage 25, ShuffleMapStage 26, ShuffleMapStage 27, ShuffleMapStage 19, ShuffleMapStage 20, ShuffleMapStage 21, ShuffleMapStage 28, ShuffleMapStage 22, ShuffleMapStage 29, ShuffleMapStage 23) -2016-05-18 19:25:06 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List(ShuffleMapStage 29) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 24: List(ShuffleMapStage 23) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 18: List() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 25: List(ShuffleMapStage 24) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 26: List(ShuffleMapStage 25) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 27: List(ShuffleMapStage 26) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 19: List(ShuffleMapStage 18) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 20: List(ShuffleMapStage 19) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 21: List(ShuffleMapStage 20) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 28: List(ShuffleMapStage 27) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 22: List(ShuffleMapStage 21) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 29: List(ShuffleMapStage 28) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 23: List(ShuffleMapStage 22) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 18 (MapPartitionsRDD[84] at flatMap at ALS.scala:1124), which is now runnable -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(15184) called with curMem=248733, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 14.8 KB, free 730.3 MB) -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(5460) called with curMem=263917, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 5.3 KB, free 730.3 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:64400 (size: 5.3 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 18 (MapPartitionsRDD[84] at flatMap at ALS.scala:1124) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 1 tasks -2016-05-18 19:25:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 13, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-18 19:25:06 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 13) -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_15_0 locally -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-18 19:25:06 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 13). 2469 bytes result sent to driver -2016-05-18 19:25:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 13) in 48 ms on localhost (1/1) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-18 19:25:06 INFO DAGScheduler:59 - ShuffleMapStage 18 (flatMap at ALS.scala:1124) finished in 0.050 s -2016-05-18 19:25:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:06 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 30, ResultStage 31, ShuffleMapStage 24, ShuffleMapStage 25, ShuffleMapStage 26, ShuffleMapStage 27, ShuffleMapStage 19, ShuffleMapStage 20, ShuffleMapStage 21, ShuffleMapStage 28, ShuffleMapStage 22, ShuffleMapStage 29, ShuffleMapStage 23) -2016-05-18 19:25:06 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List(ShuffleMapStage 29) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 24: List(ShuffleMapStage 23) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 25: List(ShuffleMapStage 24) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 26: List(ShuffleMapStage 25) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 27: List(ShuffleMapStage 26) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 19: List() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 20: List(ShuffleMapStage 19) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 21: List(ShuffleMapStage 20) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 28: List(ShuffleMapStage 27) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 22: List(ShuffleMapStage 21) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 29: List(ShuffleMapStage 28) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 23: List(ShuffleMapStage 22) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 19 (MapPartitionsRDD[93] at flatMap at ALS.scala:1124), which is now runnable -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(16032) called with curMem=269377, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 15.7 KB, free 730.3 MB) -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(5653) called with curMem=285409, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 5.5 KB, free 730.3 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:64400 (size: 5.5 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 19 (MapPartitionsRDD[93] at flatMap at ALS.scala:1124) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-18 19:25:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 14, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-18 19:25:06 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 14) -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_10_0 locally -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-18 19:25:06 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 14). 2469 bytes result sent to driver -2016-05-18 19:25:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 14) in 28 ms on localhost (1/1) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-18 19:25:06 INFO DAGScheduler:59 - ShuffleMapStage 19 (flatMap at ALS.scala:1124) finished in 0.031 s -2016-05-18 19:25:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:06 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 30, ResultStage 31, ShuffleMapStage 24, ShuffleMapStage 25, ShuffleMapStage 26, ShuffleMapStage 27, ShuffleMapStage 20, ShuffleMapStage 21, ShuffleMapStage 28, ShuffleMapStage 22, ShuffleMapStage 29, ShuffleMapStage 23) -2016-05-18 19:25:06 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List(ShuffleMapStage 29) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 24: List(ShuffleMapStage 23) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 25: List(ShuffleMapStage 24) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 26: List(ShuffleMapStage 25) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 27: List(ShuffleMapStage 26) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 20: List() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 21: List(ShuffleMapStage 20) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 28: List(ShuffleMapStage 27) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 22: List(ShuffleMapStage 21) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 29: List(ShuffleMapStage 28) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 23: List(ShuffleMapStage 22) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 20 (MapPartitionsRDD[102] at flatMap at ALS.scala:1124), which is now runnable -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(16880) called with curMem=291062, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 16.5 KB, free 730.3 MB) -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(5814) called with curMem=307942, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 5.7 KB, free 730.3 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:64400 (size: 5.7 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 20 (MapPartitionsRDD[102] at flatMap at ALS.scala:1124) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 1 tasks -2016-05-18 19:25:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 15, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-18 19:25:06 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 15) -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_15_0 locally -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-18 19:25:06 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 15). 2469 bytes result sent to driver -2016-05-18 19:25:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 15) in 33 ms on localhost (1/1) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-18 19:25:06 INFO DAGScheduler:59 - ShuffleMapStage 20 (flatMap at ALS.scala:1124) finished in 0.034 s -2016-05-18 19:25:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:06 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 30, ResultStage 31, ShuffleMapStage 24, ShuffleMapStage 25, ShuffleMapStage 26, ShuffleMapStage 27, ShuffleMapStage 21, ShuffleMapStage 28, ShuffleMapStage 22, ShuffleMapStage 29, ShuffleMapStage 23) -2016-05-18 19:25:06 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List(ShuffleMapStage 29) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 24: List(ShuffleMapStage 23) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 25: List(ShuffleMapStage 24) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 26: List(ShuffleMapStage 25) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 27: List(ShuffleMapStage 26) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 21: List() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 28: List(ShuffleMapStage 27) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 22: List(ShuffleMapStage 21) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 29: List(ShuffleMapStage 28) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 23: List(ShuffleMapStage 22) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 21 (MapPartitionsRDD[111] at flatMap at ALS.scala:1124), which is now runnable -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(17728) called with curMem=313756, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 17.3 KB, free 730.3 MB) -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(5985) called with curMem=331484, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 5.8 KB, free 730.3 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:64400 (size: 5.8 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:64400 in memory (size: 5.5 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 21 (MapPartitionsRDD[111] at flatMap at ALS.scala:1124) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 1 tasks -2016-05-18 19:25:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 16, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-18 19:25:06 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 16) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:64400 in memory (size: 5.3 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_10_0 locally -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:64400 in memory (size: 5.2 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 16). 2469 bytes result sent to driver -2016-05-18 19:25:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 16) in 34 ms on localhost (1/1) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-18 19:25:06 INFO DAGScheduler:59 - ShuffleMapStage 21 (flatMap at ALS.scala:1124) finished in 0.035 s -2016-05-18 19:25:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:06 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 30, ResultStage 31, ShuffleMapStage 24, ShuffleMapStage 25, ShuffleMapStage 26, ShuffleMapStage 27, ShuffleMapStage 28, ShuffleMapStage 22, ShuffleMapStage 29, ShuffleMapStage 23) -2016-05-18 19:25:06 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List(ShuffleMapStage 29) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:64400 in memory (size: 5.0 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 24: List(ShuffleMapStage 23) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 25: List(ShuffleMapStage 24) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 26: List(ShuffleMapStage 25) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 27: List(ShuffleMapStage 26) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 28: List(ShuffleMapStage 27) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 22: List() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 29: List(ShuffleMapStage 28) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 23: List(ShuffleMapStage 22) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 22 (MapPartitionsRDD[120] at flatMap at ALS.scala:1124), which is now runnable -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(18576) called with curMem=256913, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 18.1 KB, free 730.3 MB) -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(6176) called with curMem=275489, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 6.0 KB, free 730.3 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:64400 (size: 6.0 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 22 (MapPartitionsRDD[120] at flatMap at ALS.scala:1124) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 1 tasks -2016-05-18 19:25:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 17, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-18 19:25:06 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 17) -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_15_0 locally -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-18 19:25:06 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 17). 2469 bytes result sent to driver -2016-05-18 19:25:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 17) in 23 ms on localhost (1/1) -2016-05-18 19:25:06 INFO DAGScheduler:59 - ShuffleMapStage 22 (flatMap at ALS.scala:1124) finished in 0.023 s -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-18 19:25:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:06 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 30, ResultStage 31, ShuffleMapStage 24, ShuffleMapStage 25, ShuffleMapStage 26, ShuffleMapStage 27, ShuffleMapStage 28, ShuffleMapStage 29, ShuffleMapStage 23) -2016-05-18 19:25:06 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List(ShuffleMapStage 29) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 24: List(ShuffleMapStage 23) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 25: List(ShuffleMapStage 24) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 26: List(ShuffleMapStage 25) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 27: List(ShuffleMapStage 26) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 28: List(ShuffleMapStage 27) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 29: List(ShuffleMapStage 28) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 23: List() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[129] at flatMap at ALS.scala:1124), which is now runnable -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(19424) called with curMem=281665, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 19.0 KB, free 730.3 MB) -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(6351) called with curMem=301089, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 6.2 KB, free 730.3 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:64400 (size: 6.2 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[129] at flatMap at ALS.scala:1124) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 1 tasks -2016-05-18 19:25:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 18, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-18 19:25:06 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 18) -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_10_0 locally -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-18 19:25:06 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 18). 2469 bytes result sent to driver -2016-05-18 19:25:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 18) in 31 ms on localhost (1/1) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-18 19:25:06 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at ALS.scala:1124) finished in 0.033 s -2016-05-18 19:25:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:06 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 30, ResultStage 31, ShuffleMapStage 24, ShuffleMapStage 25, ShuffleMapStage 26, ShuffleMapStage 27, ShuffleMapStage 28, ShuffleMapStage 29) -2016-05-18 19:25:06 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List(ShuffleMapStage 29) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 24: List() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 25: List(ShuffleMapStage 24) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 26: List(ShuffleMapStage 25) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 27: List(ShuffleMapStage 26) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 28: List(ShuffleMapStage 27) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 29: List(ShuffleMapStage 28) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 24 (MapPartitionsRDD[138] at flatMap at ALS.scala:1124), which is now runnable -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(20272) called with curMem=307440, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 19.8 KB, free 730.3 MB) -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(6514) called with curMem=327712, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 6.4 KB, free 730.3 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:64400 (size: 6.4 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 24 (MapPartitionsRDD[138] at flatMap at ALS.scala:1124) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 1 tasks -2016-05-18 19:25:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 19, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-18 19:25:06 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 19) -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_15_0 locally -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-18 19:25:06 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 19). 2469 bytes result sent to driver -2016-05-18 19:25:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 19) in 23 ms on localhost (1/1) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-18 19:25:06 INFO DAGScheduler:59 - ShuffleMapStage 24 (flatMap at ALS.scala:1124) finished in 0.023 s -2016-05-18 19:25:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:06 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 30, ResultStage 31, ShuffleMapStage 25, ShuffleMapStage 26, ShuffleMapStage 27, ShuffleMapStage 28, ShuffleMapStage 29) -2016-05-18 19:25:06 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List(ShuffleMapStage 29) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 25: List() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 26: List(ShuffleMapStage 25) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 27: List(ShuffleMapStage 26) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 28: List(ShuffleMapStage 27) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 29: List(ShuffleMapStage 28) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 25 (MapPartitionsRDD[147] at flatMap at ALS.scala:1124), which is now runnable -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(21120) called with curMem=334226, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 20.6 KB, free 730.2 MB) -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(6676) called with curMem=355346, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 6.5 KB, free 730.2 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:64400 (size: 6.5 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 25 (MapPartitionsRDD[147] at flatMap at ALS.scala:1124) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 1 tasks -2016-05-18 19:25:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 20, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-18 19:25:06 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 20) -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_10_0 locally -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-18 19:25:06 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 20). 2469 bytes result sent to driver -2016-05-18 19:25:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 20) in 21 ms on localhost (1/1) -2016-05-18 19:25:06 INFO DAGScheduler:59 - ShuffleMapStage 25 (flatMap at ALS.scala:1124) finished in 0.021 s -2016-05-18 19:25:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-18 19:25:06 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 30, ResultStage 31, ShuffleMapStage 26, ShuffleMapStage 27, ShuffleMapStage 28, ShuffleMapStage 29) -2016-05-18 19:25:06 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List(ShuffleMapStage 29) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 26: List() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 27: List(ShuffleMapStage 26) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 28: List(ShuffleMapStage 27) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 29: List(ShuffleMapStage 28) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[156] at flatMap at ALS.scala:1124), which is now runnable -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(21968) called with curMem=362022, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 21.5 KB, free 730.2 MB) -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(6850) called with curMem=383990, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 6.7 KB, free 730.2 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:64400 (size: 6.7 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[156] at flatMap at ALS.scala:1124) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 1 tasks -2016-05-18 19:25:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 21, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-18 19:25:06 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 21) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:64400 in memory (size: 6.4 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:64400 in memory (size: 6.2 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_15_0 locally -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:64400 in memory (size: 6.0 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 21). 2469 bytes result sent to driver -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:64400 in memory (size: 5.8 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 21) in 28 ms on localhost (1/1) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-18 19:25:06 INFO DAGScheduler:59 - ShuffleMapStage 26 (flatMap at ALS.scala:1124) finished in 0.030 s -2016-05-18 19:25:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:06 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 30, ResultStage 31, ShuffleMapStage 27, ShuffleMapStage 28, ShuffleMapStage 29) -2016-05-18 19:25:06 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List(ShuffleMapStage 29) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 27: List() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 28: List(ShuffleMapStage 27) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 29: List(ShuffleMapStage 28) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 27 (MapPartitionsRDD[165] at flatMap at ALS.scala:1124), which is now runnable -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(22816) called with curMem=289814, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 22.3 KB, free 730.3 MB) -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(7012) called with curMem=312630, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 6.8 KB, free 730.3 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:64400 (size: 6.8 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 27 (MapPartitionsRDD[165] at flatMap at ALS.scala:1124) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-18 19:25:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 22, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-18 19:25:06 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 22) -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_10_0 locally -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-18 19:25:06 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 22). 2469 bytes result sent to driver -2016-05-18 19:25:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 22) in 33 ms on localhost (1/1) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-18 19:25:06 INFO DAGScheduler:59 - ShuffleMapStage 27 (flatMap at ALS.scala:1124) finished in 0.033 s -2016-05-18 19:25:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:06 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 30, ResultStage 31, ShuffleMapStage 28, ShuffleMapStage 29) -2016-05-18 19:25:06 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List(ShuffleMapStage 29) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 28: List() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 29: List(ShuffleMapStage 28) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 28 (MapPartitionsRDD[174] at flatMap at ALS.scala:1124), which is now runnable -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(23664) called with curMem=319642, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 23.1 KB, free 730.3 MB) -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(7185) called with curMem=343306, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 7.0 KB, free 730.3 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:64400 (size: 7.0 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 28 (MapPartitionsRDD[174] at flatMap at ALS.scala:1124) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 1 tasks -2016-05-18 19:25:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 23, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-18 19:25:06 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 23) -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_15_0 locally -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-18 19:25:06 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 23). 2469 bytes result sent to driver -2016-05-18 19:25:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 23) in 37 ms on localhost (1/1) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-18 19:25:06 INFO DAGScheduler:59 - ShuffleMapStage 28 (flatMap at ALS.scala:1124) finished in 0.038 s -2016-05-18 19:25:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:06 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 30, ResultStage 31, ShuffleMapStage 29) -2016-05-18 19:25:06 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List(ShuffleMapStage 29) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 29: List() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 29 (MapPartitionsRDD[183] at flatMap at ALS.scala:1124), which is now runnable -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(24512) called with curMem=350491, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 23.9 KB, free 730.2 MB) -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(7337) called with curMem=375003, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 7.2 KB, free 730.2 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:64400 (size: 7.2 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 29 (MapPartitionsRDD[183] at flatMap at ALS.scala:1124) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 1 tasks -2016-05-18 19:25:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 24, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-18 19:25:06 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 24) -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_10_0 locally -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-18 19:25:06 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 24). 2469 bytes result sent to driver -2016-05-18 19:25:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 24) in 35 ms on localhost (1/1) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-18 19:25:06 INFO DAGScheduler:59 - ShuffleMapStage 29 (flatMap at ALS.scala:1124) finished in 0.039 s -2016-05-18 19:25:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:06 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 30, ResultStage 31) -2016-05-18 19:25:06 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 30 (MapPartitionsRDD[192] at flatMap at ALS.scala:1124), which is now runnable -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(25360) called with curMem=382340, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 24.8 KB, free 730.2 MB) -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(7505) called with curMem=407700, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 7.3 KB, free 730.2 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:64400 (size: 7.3 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 30 (MapPartitionsRDD[192] at flatMap at ALS.scala:1124) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 1 tasks -2016-05-18 19:25:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 25, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-18 19:25:06 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 25) -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_15_0 locally -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-18 19:25:06 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 25). 2469 bytes result sent to driver -2016-05-18 19:25:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 25) in 23 ms on localhost (1/1) -2016-05-18 19:25:06 INFO DAGScheduler:59 - ShuffleMapStage 30 (flatMap at ALS.scala:1124) finished in 0.025 s -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-18 19:25:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:06 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - waiting: Set(ResultStage 31) -2016-05-18 19:25:06 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting ResultStage 31 (users MapPartitionsRDD[208] at mapValues at ALS.scala:255), which is now runnable -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(26568) called with curMem=415205, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 25.9 KB, free 730.2 MB) -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(7799) called with curMem=441773, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 7.6 KB, free 730.2 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:64400 (size: 7.6 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:64400 in memory (size: 6.7 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 31 (users MapPartitionsRDD[208] at mapValues at ALS.scala:255) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 1 tasks -2016-05-18 19:25:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 26, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-18 19:25:06 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 26) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:64400 in memory (size: 7.2 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:64400 in memory (size: 7.0 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:64400 in memory (size: 6.8 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO CacheManager:59 - Partition rdd_208_0 not found, computing it -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(576) called with curMem=328228, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block rdd_208_0 stored as values in memory (estimated size 576.0 B, free 730.3 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Added rdd_208_0 in memory on localhost:64400 (size: 576.0 B, free: 730.5 MB) -2016-05-18 19:25:06 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 26). 2878 bytes result sent to driver -2016-05-18 19:25:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 26) in 22 ms on localhost (1/1) -2016-05-18 19:25:06 INFO DAGScheduler:59 - ResultStage 31 (count at ALS.scala:263) finished in 0.022 s -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-18 19:25:06 INFO DAGScheduler:59 - Job 2 finished: count at ALS.scala:263, took 1.459783 s -2016-05-18 19:25:06 INFO SparkContext:59 - Starting job: count at ALS.scala:264 -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 143 bytes -2016-05-18 19:25:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 143 bytes -2016-05-18 19:25:06 INFO DAGScheduler:59 - Got job 3 (count at ALS.scala:264) with 1 output partitions -2016-05-18 19:25:06 INFO DAGScheduler:59 - Final stage: ResultStage 56(count at ALS.scala:264) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 33, ShuffleMapStage 55) -2016-05-18 19:25:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting ResultStage 56 (products MapPartitionsRDD[209] at mapValues at ALS.scala:259), which has no missing parents -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(25720) called with curMem=328804, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 25.1 KB, free 730.2 MB) -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(7610) called with curMem=354524, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 7.4 KB, free 730.2 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:64400 (size: 7.4 KB, free: 730.5 MB) -2016-05-18 19:25:06 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:06 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 56 (products MapPartitionsRDD[209] at mapValues at ALS.scala:259) -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 1 tasks -2016-05-18 19:25:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 27, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-18 19:25:06 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 27) -2016-05-18 19:25:06 INFO CacheManager:59 - Partition rdd_209_0 not found, computing it -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-05-18 19:25:06 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-18 19:25:06 INFO MemoryStore:59 - ensureFreeSpace(576) called with curMem=362134, maxMem=766075207 -2016-05-18 19:25:06 INFO MemoryStore:59 - Block rdd_209_0 stored as values in memory (estimated size 576.0 B, free 730.2 MB) -2016-05-18 19:25:06 INFO BlockManagerInfo:59 - Added rdd_209_0 in memory on localhost:64400 (size: 576.0 B, free: 730.5 MB) -2016-05-18 19:25:06 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 27). 2878 bytes result sent to driver -2016-05-18 19:25:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 27) in 22 ms on localhost (1/1) -2016-05-18 19:25:06 INFO DAGScheduler:59 - ResultStage 56 (count at ALS.scala:264) finished in 0.021 s -2016-05-18 19:25:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-18 19:25:06 INFO DAGScheduler:59 - Job 3 finished: count at ALS.scala:264, took 0.065460 s -2016-05-18 19:25:07 INFO SparkContext:59 - Starting job: first at MatrixFactorizationModel.scala:67 -2016-05-18 19:25:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 143 bytes -2016-05-18 19:25:07 INFO DAGScheduler:59 - Got job 4 (first at MatrixFactorizationModel.scala:67) with 1 output partitions -2016-05-18 19:25:07 INFO DAGScheduler:59 - Final stage: ResultStage 82(first at MatrixFactorizationModel.scala:67) -2016-05-18 19:25:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 81, ShuffleMapStage 60) -2016-05-18 19:25:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-18 19:25:07 INFO DAGScheduler:59 - Submitting ResultStage 82 (users MapPartitionsRDD[208] at mapValues at ALS.scala:255), which has no missing parents -2016-05-18 19:25:07 INFO MemoryStore:59 - ensureFreeSpace(26736) called with curMem=362710, maxMem=766075207 -2016-05-18 19:25:07 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 26.1 KB, free 730.2 MB) -2016-05-18 19:25:07 INFO MemoryStore:59 - ensureFreeSpace(7863) called with curMem=389446, maxMem=766075207 -2016-05-18 19:25:07 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.2 MB) -2016-05-18 19:25:07 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:64400 (size: 7.7 KB, free: 730.5 MB) -2016-05-18 19:25:07 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:07 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 82 (users MapPartitionsRDD[208] at mapValues at ALS.scala:255) -2016-05-18 19:25:07 INFO TaskSchedulerImpl:59 - Adding task set 82.0 with 1 tasks -2016-05-18 19:25:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 82.0 (TID 28, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-18 19:25:07 INFO Executor:59 - Running task 0.0 in stage 82.0 (TID 28) -2016-05-18 19:25:07 INFO BlockManager:59 - Found block rdd_208_0 locally -2016-05-18 19:25:07 INFO Executor:59 - Finished task 0.0 in stage 82.0 (TID 28). 2288 bytes result sent to driver -2016-05-18 19:25:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 82.0 (TID 28) in 10 ms on localhost (1/1) -2016-05-18 19:25:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 82.0, whose tasks have all completed, from pool -2016-05-18 19:25:07 INFO DAGScheduler:59 - ResultStage 82 (first at MatrixFactorizationModel.scala:67) finished in 0.011 s -2016-05-18 19:25:07 INFO DAGScheduler:59 - Job 4 finished: first at MatrixFactorizationModel.scala:67, took 0.039141 s -2016-05-18 19:25:07 INFO SparkContext:59 - Starting job: first at MatrixFactorizationModel.scala:67 -2016-05-18 19:25:07 INFO DAGScheduler:59 - Got job 5 (first at MatrixFactorizationModel.scala:67) with 1 output partitions -2016-05-18 19:25:07 INFO DAGScheduler:59 - Final stage: ResultStage 107(first at MatrixFactorizationModel.scala:67) -2016-05-18 19:25:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 84, ShuffleMapStage 106) -2016-05-18 19:25:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-18 19:25:07 INFO DAGScheduler:59 - Submitting ResultStage 107 (products MapPartitionsRDD[209] at mapValues at ALS.scala:259), which has no missing parents -2016-05-18 19:25:07 INFO MemoryStore:59 - ensureFreeSpace(25888) called with curMem=397309, maxMem=766075207 -2016-05-18 19:25:07 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 25.3 KB, free 730.2 MB) -2016-05-18 19:25:07 INFO MemoryStore:59 - ensureFreeSpace(7675) called with curMem=423197, maxMem=766075207 -2016-05-18 19:25:07 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-05-18 19:25:07 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:64400 (size: 7.5 KB, free: 730.5 MB) -2016-05-18 19:25:07 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:07 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 107 (products MapPartitionsRDD[209] at mapValues at ALS.scala:259) -2016-05-18 19:25:07 INFO TaskSchedulerImpl:59 - Adding task set 107.0 with 1 tasks -2016-05-18 19:25:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 107.0 (TID 29, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-18 19:25:07 INFO Executor:59 - Running task 0.0 in stage 107.0 (TID 29) -2016-05-18 19:25:07 INFO BlockManager:59 - Found block rdd_209_0 locally -2016-05-18 19:25:07 INFO Executor:59 - Finished task 0.0 in stage 107.0 (TID 29). 2288 bytes result sent to driver -2016-05-18 19:25:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 107.0 (TID 29) in 9 ms on localhost (1/1) -2016-05-18 19:25:07 INFO DAGScheduler:59 - ResultStage 107 (first at MatrixFactorizationModel.scala:67) finished in 0.009 s -2016-05-18 19:25:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 107.0, whose tasks have all completed, from pool -2016-05-18 19:25:07 INFO DAGScheduler:59 - Job 5 finished: first at MatrixFactorizationModel.scala:67, took 0.042527 s -2016-05-18 19:25:07 INFO SparkContext:59 - Starting job: aggregate at MatrixFactorizationModel.scala:96 -2016-05-18 19:25:07 INFO DAGScheduler:59 - Got job 6 (aggregate at MatrixFactorizationModel.scala:96) with 1 output partitions -2016-05-18 19:25:07 INFO DAGScheduler:59 - Final stage: ResultStage 108(aggregate at MatrixFactorizationModel.scala:96) -2016-05-18 19:25:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-18 19:25:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-18 19:25:07 INFO DAGScheduler:59 - Submitting ResultStage 108 (MapPartitionsRDD[210] at map at ALSFromSpark.scala:42), which has no missing parents -2016-05-18 19:25:07 INFO MemoryStore:59 - ensureFreeSpace(4008) called with curMem=430872, maxMem=766075207 -2016-05-18 19:25:07 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-18 19:25:07 INFO MemoryStore:59 - ensureFreeSpace(2239) called with curMem=434880, maxMem=766075207 -2016-05-18 19:25:07 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-18 19:25:07 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:64400 (size: 2.2 KB, free: 730.5 MB) -2016-05-18 19:25:07 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:07 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 108 (MapPartitionsRDD[210] at map at ALSFromSpark.scala:42) -2016-05-18 19:25:07 INFO TaskSchedulerImpl:59 - Adding task set 108.0 with 1 tasks -2016-05-18 19:25:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 108.0 (TID 30, localhost, PROCESS_LOCAL, 2172 bytes) -2016-05-18 19:25:07 INFO Executor:59 - Running task 0.0 in stage 108.0 (TID 30) -2016-05-18 19:25:07 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/test.data:0+127 -2016-05-18 19:25:07 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:64400 in memory (size: 7.7 KB, free: 730.5 MB) -2016-05-18 19:25:07 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-18 19:25:07 INFO Executor:59 - Finished task 0.0 in stage 108.0 (TID 30). 2218 bytes result sent to driver -2016-05-18 19:25:07 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:64400 in memory (size: 7.4 KB, free: 730.5 MB) -2016-05-18 19:25:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 108.0 (TID 30) in 20 ms on localhost (1/1) -2016-05-18 19:25:07 INFO DAGScheduler:59 - ResultStage 108 (aggregate at MatrixFactorizationModel.scala:96) finished in 0.020 s -2016-05-18 19:25:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 108.0, whose tasks have all completed, from pool -2016-05-18 19:25:07 INFO DAGScheduler:59 - Job 6 finished: aggregate at MatrixFactorizationModel.scala:96, took 0.030829 s -2016-05-18 19:25:07 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-18 19:25:07 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:64400 in memory (size: 7.6 KB, free: 730.5 MB) -2016-05-18 19:25:07 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:64400 in memory (size: 7.5 KB, free: 730.5 MB) -2016-05-18 19:25:07 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-18 19:25:07 INFO SparkContext:59 - Starting job: mean at ALSFromSpark.scala:55 -2016-05-18 19:25:07 INFO DAGScheduler:59 - Registering RDD 221 (map at ALSFromSpark.scala:49) -2016-05-18 19:25:07 INFO DAGScheduler:59 - Registering RDD 211 (map at MatrixFactorizationModel.scala:137) -2016-05-18 19:25:07 INFO DAGScheduler:59 - Registering RDD 215 (map at MatrixFactorizationModel.scala:137) -2016-05-18 19:25:07 INFO DAGScheduler:59 - Registering RDD 220 (map at ALSFromSpark.scala:46) -2016-05-18 19:25:07 INFO DAGScheduler:59 - Got job 7 (mean at ALSFromSpark.scala:55) with 1 output partitions -2016-05-18 19:25:07 INFO DAGScheduler:59 - Final stage: ResultStage 138(mean at ALSFromSpark.scala:55) -2016-05-18 19:25:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 137, ShuffleMapStage 109) -2016-05-18 19:25:07 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 137, ShuffleMapStage 109) -2016-05-18 19:25:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 109 (MapPartitionsRDD[221] at map at ALSFromSpark.scala:49), which has no missing parents -2016-05-18 19:25:07 INFO MemoryStore:59 - ensureFreeSpace(3832) called with curMem=301260, maxMem=766075207 -2016-05-18 19:25:07 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.7 KB, free 730.3 MB) -2016-05-18 19:25:07 INFO MemoryStore:59 - ensureFreeSpace(2214) called with curMem=305092, maxMem=766075207 -2016-05-18 19:25:07 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-05-18 19:25:07 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:64400 (size: 2.2 KB, free: 730.5 MB) -2016-05-18 19:25:07 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:07 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 109 (MapPartitionsRDD[221] at map at ALSFromSpark.scala:49) -2016-05-18 19:25:07 INFO TaskSchedulerImpl:59 - Adding task set 109.0 with 1 tasks -2016-05-18 19:25:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 109.0 (TID 31, localhost, PROCESS_LOCAL, 2161 bytes) -2016-05-18 19:25:07 INFO Executor:59 - Running task 0.0 in stage 109.0 (TID 31) -2016-05-18 19:25:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 110 (MapPartitionsRDD[211] at map at MatrixFactorizationModel.scala:137), which has no missing parents -2016-05-18 19:25:07 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/test.data:0+127 -2016-05-18 19:25:07 INFO MemoryStore:59 - ensureFreeSpace(3984) called with curMem=307306, maxMem=766075207 -2016-05-18 19:25:07 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 3.9 KB, free 730.3 MB) -2016-05-18 19:25:07 INFO MemoryStore:59 - ensureFreeSpace(2259) called with curMem=311290, maxMem=766075207 -2016-05-18 19:25:07 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-05-18 19:25:07 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:64400 (size: 2.2 KB, free: 730.5 MB) -2016-05-18 19:25:07 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:07 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 110 (MapPartitionsRDD[211] at map at MatrixFactorizationModel.scala:137) -2016-05-18 19:25:07 INFO TaskSchedulerImpl:59 - Adding task set 110.0 with 1 tasks -2016-05-18 19:25:07 INFO Executor:59 - Finished task 0.0 in stage 109.0 (TID 31). 2253 bytes result sent to driver -2016-05-18 19:25:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 110.0 (TID 32, localhost, PROCESS_LOCAL, 2161 bytes) -2016-05-18 19:25:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 109.0 (TID 31) in 18 ms on localhost (1/1) -2016-05-18 19:25:07 INFO DAGScheduler:59 - ShuffleMapStage 109 (map at ALSFromSpark.scala:49) finished in 0.019 s -2016-05-18 19:25:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:07 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 110) -2016-05-18 19:25:07 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 136, ShuffleMapStage 137, ResultStage 138) -2016-05-18 19:25:07 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 109.0, whose tasks have all completed, from pool -2016-05-18 19:25:07 INFO Executor:59 - Running task 0.0 in stage 110.0 (TID 32) -2016-05-18 19:25:07 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 136: List(ShuffleMapStage 110) -2016-05-18 19:25:07 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 137: List(ShuffleMapStage 136) -2016-05-18 19:25:07 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/test.data:0+127 -2016-05-18 19:25:07 INFO DAGScheduler:59 - Missing parents for ResultStage 138: List(ShuffleMapStage 137) -2016-05-18 19:25:07 INFO Executor:59 - Finished task 0.0 in stage 110.0 (TID 32). 2253 bytes result sent to driver -2016-05-18 19:25:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 110.0 (TID 32) in 25 ms on localhost (1/1) -2016-05-18 19:25:07 INFO DAGScheduler:59 - ShuffleMapStage 110 (map at MatrixFactorizationModel.scala:137) finished in 0.029 s -2016-05-18 19:25:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:07 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:07 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 136, ShuffleMapStage 137, ResultStage 138) -2016-05-18 19:25:07 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 110.0, whose tasks have all completed, from pool -2016-05-18 19:25:07 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 136: List() -2016-05-18 19:25:07 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 137: List(ShuffleMapStage 136) -2016-05-18 19:25:07 INFO DAGScheduler:59 - Missing parents for ResultStage 138: List(ShuffleMapStage 137) -2016-05-18 19:25:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 136 (MapPartitionsRDD[215] at map at MatrixFactorizationModel.scala:137), which is now runnable -2016-05-18 19:25:07 INFO MemoryStore:59 - ensureFreeSpace(26064) called with curMem=313549, maxMem=766075207 -2016-05-18 19:25:07 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 25.5 KB, free 730.3 MB) -2016-05-18 19:25:07 INFO MemoryStore:59 - ensureFreeSpace(7684) called with curMem=339613, maxMem=766075207 -2016-05-18 19:25:07 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.3 MB) -2016-05-18 19:25:07 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:64400 (size: 7.5 KB, free: 730.5 MB) -2016-05-18 19:25:07 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:07 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 136 (MapPartitionsRDD[215] at map at MatrixFactorizationModel.scala:137) -2016-05-18 19:25:07 INFO TaskSchedulerImpl:59 - Adding task set 136.0 with 1 tasks -2016-05-18 19:25:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 136.0 (TID 33, localhost, PROCESS_LOCAL, 2236 bytes) -2016-05-18 19:25:07 INFO Executor:59 - Running task 0.0 in stage 136.0 (TID 33) -2016-05-18 19:25:07 INFO BlockManager:59 - Found block rdd_209_0 locally -2016-05-18 19:25:07 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-18 19:25:07 INFO Executor:59 - Finished task 0.0 in stage 136.0 (TID 33). 2469 bytes result sent to driver -2016-05-18 19:25:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 136.0 (TID 33) in 42 ms on localhost (1/1) -2016-05-18 19:25:07 INFO DAGScheduler:59 - ShuffleMapStage 136 (map at MatrixFactorizationModel.scala:137) finished in 0.042 s -2016-05-18 19:25:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 136.0, whose tasks have all completed, from pool -2016-05-18 19:25:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:07 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:07 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 137, ResultStage 138) -2016-05-18 19:25:07 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:07 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 137: List() -2016-05-18 19:25:07 INFO DAGScheduler:59 - Missing parents for ResultStage 138: List(ShuffleMapStage 137) -2016-05-18 19:25:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 137 (MapPartitionsRDD[220] at map at ALSFromSpark.scala:46), which is now runnable -2016-05-18 19:25:07 INFO MemoryStore:59 - ensureFreeSpace(27064) called with curMem=347297, maxMem=766075207 -2016-05-18 19:25:07 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 26.4 KB, free 730.2 MB) -2016-05-18 19:25:07 INFO MemoryStore:59 - ensureFreeSpace(7970) called with curMem=374361, maxMem=766075207 -2016-05-18 19:25:07 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 7.8 KB, free 730.2 MB) -2016-05-18 19:25:07 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:64400 (size: 7.8 KB, free: 730.5 MB) -2016-05-18 19:25:07 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:07 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 137 (MapPartitionsRDD[220] at map at ALSFromSpark.scala:46) -2016-05-18 19:25:07 INFO TaskSchedulerImpl:59 - Adding task set 137.0 with 1 tasks -2016-05-18 19:25:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 137.0 (TID 34, localhost, PROCESS_LOCAL, 2236 bytes) -2016-05-18 19:25:07 INFO Executor:59 - Running task 0.0 in stage 137.0 (TID 34) -2016-05-18 19:25:07 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-18 19:25:07 INFO BlockManager:59 - Found block rdd_208_0 locally -2016-05-18 19:25:07 INFO Executor:59 - Finished task 0.0 in stage 137.0 (TID 34). 2469 bytes result sent to driver -2016-05-18 19:25:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 137.0 (TID 34) in 20 ms on localhost (1/1) -2016-05-18 19:25:07 INFO DAGScheduler:59 - ShuffleMapStage 137 (map at ALSFromSpark.scala:46) finished in 0.020 s -2016-05-18 19:25:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 137.0, whose tasks have all completed, from pool -2016-05-18 19:25:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-18 19:25:07 INFO DAGScheduler:59 - running: Set() -2016-05-18 19:25:07 INFO DAGScheduler:59 - waiting: Set(ResultStage 138) -2016-05-18 19:25:07 INFO DAGScheduler:59 - failed: Set() -2016-05-18 19:25:07 INFO DAGScheduler:59 - Missing parents for ResultStage 138: List() -2016-05-18 19:25:07 INFO DAGScheduler:59 - Submitting ResultStage 138 (MapPartitionsRDD[226] at mean at ALSFromSpark.scala:55), which is now runnable -2016-05-18 19:25:07 INFO MemoryStore:59 - ensureFreeSpace(3504) called with curMem=382331, maxMem=766075207 -2016-05-18 19:25:07 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-18 19:25:07 INFO MemoryStore:59 - ensureFreeSpace(1884) called with curMem=385835, maxMem=766075207 -2016-05-18 19:25:07 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 1884.0 B, free 730.2 MB) -2016-05-18 19:25:07 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:64400 (size: 1884.0 B, free: 730.5 MB) -2016-05-18 19:25:07 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:07 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 138 (MapPartitionsRDD[226] at mean at ALSFromSpark.scala:55) -2016-05-18 19:25:07 INFO TaskSchedulerImpl:59 - Adding task set 138.0 with 1 tasks -2016-05-18 19:25:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 138.0 (TID 35, localhost, PROCESS_LOCAL, 1974 bytes) -2016-05-18 19:25:07 INFO Executor:59 - Running task 0.0 in stage 138.0 (TID 35) -2016-05-18 19:25:07 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-18 19:25:07 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-05-18 19:25:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-18 19:25:07 INFO Executor:59 - Finished task 0.0 in stage 138.0 (TID 35). 1330 bytes result sent to driver -2016-05-18 19:25:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 138.0 (TID 35) in 53 ms on localhost (1/1) -2016-05-18 19:25:07 INFO DAGScheduler:59 - ResultStage 138 (mean at ALSFromSpark.scala:55) finished in 0.054 s -2016-05-18 19:25:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 138.0, whose tasks have all completed, from pool -2016-05-18 19:25:07 INFO DAGScheduler:59 - Job 7 finished: mean at ALSFromSpark.scala:55, took 0.243406 s -2016-05-18 19:25:07 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:64400 in memory (size: 1884.0 B, free: 730.5 MB) -2016-05-18 19:25:07 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-18 19:25:07 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:64400 in memory (size: 7.8 KB, free: 730.5 MB) -2016-05-18 19:25:07 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-18 19:25:07 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:64400 in memory (size: 7.5 KB, free: 730.5 MB) -2016-05-18 19:25:07 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-18 19:25:07 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:64400 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-18 19:25:07 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-18 19:25:07 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:64400 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-18 19:25:07 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-18 19:25:08 INFO SparkContext:59 - Starting job: saveAsTextFile at MatrixFactorizationModel.scala:360 -2016-05-18 19:25:08 INFO DAGScheduler:59 - Got job 8 (saveAsTextFile at MatrixFactorizationModel.scala:360) with 1 output partitions -2016-05-18 19:25:08 INFO DAGScheduler:59 - Final stage: ResultStage 139(saveAsTextFile at MatrixFactorizationModel.scala:360) -2016-05-18 19:25:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-18 19:25:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-18 19:25:08 INFO DAGScheduler:59 - Submitting ResultStage 139 (MapPartitionsRDD[228] at saveAsTextFile at MatrixFactorizationModel.scala:360), which has no missing parents -2016-05-18 19:25:09 INFO MemoryStore:59 - ensureFreeSpace(126944) called with curMem=301260, maxMem=766075207 -2016-05-18 19:25:09 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 124.0 KB, free 730.2 MB) -2016-05-18 19:25:09 INFO MemoryStore:59 - ensureFreeSpace(42497) called with curMem=428204, maxMem=766075207 -2016-05-18 19:25:09 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 41.5 KB, free 730.1 MB) -2016-05-18 19:25:09 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:64400 (size: 41.5 KB, free: 730.5 MB) -2016-05-18 19:25:09 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:09 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 139 (MapPartitionsRDD[228] at saveAsTextFile at MatrixFactorizationModel.scala:360) -2016-05-18 19:25:09 INFO TaskSchedulerImpl:59 - Adding task set 139.0 with 1 tasks -2016-05-18 19:25:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 139.0 (TID 36, localhost, PROCESS_LOCAL, 2190 bytes) -2016-05-18 19:25:09 INFO Executor:59 - Running task 0.0 in stage 139.0 (TID 36) -2016-05-18 19:25:09 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605181925_0139_m_000000_36' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/als/myModelPath20160518192507629/metadata/_temporary/0/task_201605181925_0139_m_000000 -2016-05-18 19:25:10 INFO SparkHadoopMapRedUtil:59 - attempt_201605181925_0139_m_000000_36: Committed -2016-05-18 19:25:10 INFO Executor:59 - Finished task 0.0 in stage 139.0 (TID 36). 1864 bytes result sent to driver -2016-05-18 19:25:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 139.0 (TID 36) in 980 ms on localhost (1/1) -2016-05-18 19:25:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 139.0, whose tasks have all completed, from pool -2016-05-18 19:25:10 INFO DAGScheduler:59 - ResultStage 139 (saveAsTextFile at MatrixFactorizationModel.scala:360) finished in 0.982 s -2016-05-18 19:25:10 INFO DAGScheduler:59 - Job 8 finished: saveAsTextFile at MatrixFactorizationModel.scala:360, took 1.057595 s -2016-05-18 19:25:16 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-18 19:25:16 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-18 19:25:16 INFO SparkContext:59 - Starting job: parquet at MatrixFactorizationModel.scala:361 -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 143 bytes -2016-05-18 19:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 143 bytes -2016-05-18 19:25:16 INFO DAGScheduler:59 - Got job 9 (parquet at MatrixFactorizationModel.scala:361) with 1 output partitions -2016-05-18 19:25:16 INFO DAGScheduler:59 - Final stage: ResultStage 165(parquet at MatrixFactorizationModel.scala:361) -2016-05-18 19:25:16 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 143, ShuffleMapStage 164) -2016-05-18 19:25:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-18 19:25:16 INFO DAGScheduler:59 - Submitting ResultStage 165 (MapPartitionsRDD[230] at parquet at MatrixFactorizationModel.scala:361), which has no missing parents -2016-05-18 19:25:17 INFO MemoryStore:59 - ensureFreeSpace(93488) called with curMem=470701, maxMem=766075207 -2016-05-18 19:25:17 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 91.3 KB, free 730.0 MB) -2016-05-18 19:25:17 INFO MemoryStore:59 - ensureFreeSpace(31146) called with curMem=564189, maxMem=766075207 -2016-05-18 19:25:17 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 30.4 KB, free 730.0 MB) -2016-05-18 19:25:17 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:64400 (size: 30.4 KB, free: 730.5 MB) -2016-05-18 19:25:17 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 165 (MapPartitionsRDD[230] at parquet at MatrixFactorizationModel.scala:361) -2016-05-18 19:25:17 INFO TaskSchedulerImpl:59 - Adding task set 165.0 with 1 tasks -2016-05-18 19:25:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 165.0 (TID 37, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-18 19:25:17 INFO Executor:59 - Running task 0.0 in stage 165.0 (TID 37) -2016-05-18 19:25:17 INFO BlockManager:59 - Found block rdd_208_0 locally -2016-05-18 19:25:17 INFO GenerateUnsafeProjection:59 - Code generated in 715.534367 ms -2016-05-18 19:25:17 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-18 19:25:17 INFO CodecConfig:151 - Compression: GZIP -2016-05-18 19:25:17 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-18 19:25:17 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-18 19:25:17 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-18 19:25:17 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-18 19:25:17 INFO ParquetOutputFormat:151 - Validation is off -2016-05-18 19:25:17 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-18 19:25:18 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-18 19:25:18 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 416 -2016-05-18 19:25:18 INFO ColumnChunkPageWriteStore:151 - written 68B for [id] INT32: 4 values, 22B raw, 35B comp, 1 pages, encodings: [PLAIN, BIT_PACKED, RLE] -2016-05-18 19:25:18 INFO ColumnChunkPageWriteStore:151 - written 106B for [features, bag, array] DOUBLE: 40 values, 44B raw, 64B comp, 1 pages, encodings: [RLE, PLAIN_DICTIONARY], dic { 20 entries, 160B raw, 20B comp} -2016-05-18 19:25:19 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605181925_0165_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/als/myModelPath20160518192507629/data/user/_temporary/0/task_201605181925_0165_m_000000 -2016-05-18 19:25:19 INFO SparkHadoopMapRedUtil:59 - attempt_201605181925_0165_m_000000_0: Committed -2016-05-18 19:25:19 INFO Executor:59 - Finished task 0.0 in stage 165.0 (TID 37). 2189 bytes result sent to driver -2016-05-18 19:25:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 165.0 (TID 37) in 2011 ms on localhost (1/1) -2016-05-18 19:25:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 165.0, whose tasks have all completed, from pool -2016-05-18 19:25:19 INFO DAGScheduler:59 - ResultStage 165 (parquet at MatrixFactorizationModel.scala:361) finished in 2.013 s -2016-05-18 19:25:19 INFO DAGScheduler:59 - Job 9 finished: parquet at MatrixFactorizationModel.scala:361, took 2.110323 s -2016-05-18 19:25:19 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-18 19:25:19 INFO DefaultWriterContainer:59 - Job job_201605181925_0000 committed. -2016-05-18 19:25:19 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/als/myModelPath20160518192507629/data/user on driver -2016-05-18 19:25:19 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/als/myModelPath20160518192507629/data/user on driver -2016-05-18 19:25:19 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-18 19:25:19 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-18 19:25:19 INFO SparkContext:59 - Starting job: parquet at MatrixFactorizationModel.scala:362 -2016-05-18 19:25:19 INFO DAGScheduler:59 - Got job 10 (parquet at MatrixFactorizationModel.scala:362) with 1 output partitions -2016-05-18 19:25:19 INFO DAGScheduler:59 - Final stage: ResultStage 190(parquet at MatrixFactorizationModel.scala:362) -2016-05-18 19:25:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 189, ShuffleMapStage 167) -2016-05-18 19:25:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-18 19:25:19 INFO DAGScheduler:59 - Submitting ResultStage 190 (MapPartitionsRDD[233] at parquet at MatrixFactorizationModel.scala:362), which has no missing parents -2016-05-18 19:25:19 INFO MemoryStore:59 - ensureFreeSpace(92648) called with curMem=595335, maxMem=766075207 -2016-05-18 19:25:19 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 90.5 KB, free 729.9 MB) -2016-05-18 19:25:19 INFO MemoryStore:59 - ensureFreeSpace(31031) called with curMem=687983, maxMem=766075207 -2016-05-18 19:25:19 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 30.3 KB, free 729.9 MB) -2016-05-18 19:25:19 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:64400 (size: 30.3 KB, free: 730.4 MB) -2016-05-18 19:25:19 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 190 (MapPartitionsRDD[233] at parquet at MatrixFactorizationModel.scala:362) -2016-05-18 19:25:19 INFO TaskSchedulerImpl:59 - Adding task set 190.0 with 1 tasks -2016-05-18 19:25:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 190.0 (TID 38, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-18 19:25:19 INFO Executor:59 - Running task 0.0 in stage 190.0 (TID 38) -2016-05-18 19:25:19 INFO BlockManager:59 - Found block rdd_209_0 locally -2016-05-18 19:25:19 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-18 19:25:19 INFO CodecConfig:151 - Compression: GZIP -2016-05-18 19:25:19 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-18 19:25:19 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-18 19:25:19 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-18 19:25:19 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-18 19:25:19 INFO ParquetOutputFormat:151 - Validation is off -2016-05-18 19:25:19 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-18 19:25:19 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-18 19:25:19 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 416 -2016-05-18 19:25:19 INFO ColumnChunkPageWriteStore:151 - written 68B for [id] INT32: 4 values, 22B raw, 35B comp, 1 pages, encodings: [PLAIN, BIT_PACKED, RLE] -2016-05-18 19:25:19 INFO ColumnChunkPageWriteStore:151 - written 106B for [features, bag, array] DOUBLE: 40 values, 44B raw, 64B comp, 1 pages, encodings: [RLE, PLAIN_DICTIONARY], dic { 20 entries, 160B raw, 20B comp} -2016-05-18 19:25:19 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605181925_0190_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/als/myModelPath20160518192507629/data/product/_temporary/0/task_201605181925_0190_m_000000 -2016-05-18 19:25:19 INFO SparkHadoopMapRedUtil:59 - attempt_201605181925_0190_m_000000_0: Committed -2016-05-18 19:25:19 INFO Executor:59 - Finished task 0.0 in stage 190.0 (TID 38). 2189 bytes result sent to driver -2016-05-18 19:25:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 190.0 (TID 38) in 89 ms on localhost (1/1) -2016-05-18 19:25:19 INFO DAGScheduler:59 - ResultStage 190 (parquet at MatrixFactorizationModel.scala:362) finished in 0.089 s -2016-05-18 19:25:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 190.0, whose tasks have all completed, from pool -2016-05-18 19:25:19 INFO DAGScheduler:59 - Job 10 finished: parquet at MatrixFactorizationModel.scala:362, took 0.151213 s -2016-05-18 19:25:19 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-18 19:25:19 INFO DefaultWriterContainer:59 - Job job_201605181925_0000 committed. -2016-05-18 19:25:19 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/als/myModelPath20160518192507629/data/product on driver -2016-05-18 19:25:19 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/als/myModelPath20160518192507629/data/product on driver -2016-05-18 19:25:19 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=719014, maxMem=766075207 -2016-05-18 19:25:19 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 225.3 KB, free 729.7 MB) -2016-05-18 19:25:19 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:64400 in memory (size: 30.3 KB, free: 730.5 MB) -2016-05-18 19:25:19 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-18 19:25:19 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-18 19:25:19 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=826039, maxMem=766075207 -2016-05-18 19:25:19 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.8 MB) -2016-05-18 19:25:19 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:64400 (size: 19.3 KB, free: 730.4 MB) -2016-05-18 19:25:19 INFO SparkContext:59 - Created broadcast 40 from textFile at modelSaveLoad.scala:129 -2016-05-18 19:25:19 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-18 19:25:19 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-18 19:25:19 INFO DAGScheduler:59 - Got job 11 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-18 19:25:19 INFO DAGScheduler:59 - Final stage: ResultStage 191(first at modelSaveLoad.scala:129) -2016-05-18 19:25:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-18 19:25:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-18 19:25:19 INFO DAGScheduler:59 - Submitting ResultStage 191 (MapPartitionsRDD[236] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-18 19:25:19 INFO MemoryStore:59 - ensureFreeSpace(3192) called with curMem=845827, maxMem=766075207 -2016-05-18 19:25:19 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 3.1 KB, free 729.8 MB) -2016-05-18 19:25:19 INFO MemoryStore:59 - ensureFreeSpace(1877) called with curMem=849019, maxMem=766075207 -2016-05-18 19:25:19 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 1877.0 B, free 729.8 MB) -2016-05-18 19:25:19 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:64400 (size: 1877.0 B, free: 730.4 MB) -2016-05-18 19:25:19 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 191 (MapPartitionsRDD[236] at textFile at modelSaveLoad.scala:129) -2016-05-18 19:25:19 INFO TaskSchedulerImpl:59 - Adding task set 191.0 with 1 tasks -2016-05-18 19:25:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 191.0 (TID 39, localhost, PROCESS_LOCAL, 2216 bytes) -2016-05-18 19:25:19 INFO Executor:59 - Running task 0.0 in stage 191.0 (TID 39) -2016-05-18 19:25:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/als/myModelPath20160518192507629/metadata/part-00000:0+101 -2016-05-18 19:25:19 INFO Executor:59 - Finished task 0.0 in stage 191.0 (TID 39). 2147 bytes result sent to driver -2016-05-18 19:25:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 191.0 (TID 39) in 9 ms on localhost (1/1) -2016-05-18 19:25:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 191.0, whose tasks have all completed, from pool -2016-05-18 19:25:19 INFO DAGScheduler:59 - ResultStage 191 (first at modelSaveLoad.scala:129) finished in 0.009 s -2016-05-18 19:25:19 INFO DAGScheduler:59 - Job 11 finished: first at modelSaveLoad.scala:129, took 0.016623 s -2016-05-18 19:25:20 INFO MemoryStore:59 - ensureFreeSpace(230744) called with curMem=850896, maxMem=766075207 -2016-05-18 19:25:20 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 225.3 KB, free 729.6 MB) -2016-05-18 19:25:20 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1081640, maxMem=766075207 -2016-05-18 19:25:20 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.5 MB) -2016-05-18 19:25:20 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:64400 (size: 19.3 KB, free: 730.4 MB) -2016-05-18 19:25:20 INFO SparkContext:59 - Created broadcast 42 from textFile at modelSaveLoad.scala:129 -2016-05-18 19:25:20 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-18 19:25:20 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-18 19:25:20 INFO DAGScheduler:59 - Got job 12 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-18 19:25:20 INFO DAGScheduler:59 - Final stage: ResultStage 192(first at modelSaveLoad.scala:129) -2016-05-18 19:25:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-18 19:25:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-18 19:25:20 INFO DAGScheduler:59 - Submitting ResultStage 192 (MapPartitionsRDD[238] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-18 19:25:20 INFO MemoryStore:59 - ensureFreeSpace(3192) called with curMem=1101428, maxMem=766075207 -2016-05-18 19:25:20 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 3.1 KB, free 729.5 MB) -2016-05-18 19:25:20 INFO MemoryStore:59 - ensureFreeSpace(1877) called with curMem=1104620, maxMem=766075207 -2016-05-18 19:25:20 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 1877.0 B, free 729.5 MB) -2016-05-18 19:25:20 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:64400 (size: 1877.0 B, free: 730.4 MB) -2016-05-18 19:25:20 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 192 (MapPartitionsRDD[238] at textFile at modelSaveLoad.scala:129) -2016-05-18 19:25:20 INFO TaskSchedulerImpl:59 - Adding task set 192.0 with 1 tasks -2016-05-18 19:25:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 192.0 (TID 40, localhost, PROCESS_LOCAL, 2216 bytes) -2016-05-18 19:25:20 INFO Executor:59 - Running task 0.0 in stage 192.0 (TID 40) -2016-05-18 19:25:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/als/myModelPath20160518192507629/metadata/part-00000:0+101 -2016-05-18 19:25:20 INFO Executor:59 - Finished task 0.0 in stage 192.0 (TID 40). 2147 bytes result sent to driver -2016-05-18 19:25:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 192.0 (TID 40) in 19 ms on localhost (1/1) -2016-05-18 19:25:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 192.0, whose tasks have all completed, from pool -2016-05-18 19:25:20 INFO DAGScheduler:59 - ResultStage 192 (first at modelSaveLoad.scala:129) finished in 0.020 s -2016-05-18 19:25:20 INFO DAGScheduler:59 - Job 12 finished: first at modelSaveLoad.scala:129, took 0.037276 s -2016-05-18 19:25:20 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/als/myModelPath20160518192507629/data/user on driver -2016-05-18 19:25:20 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:64400 in memory (size: 1877.0 B, free: 730.4 MB) -2016-05-18 19:25:20 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-18 19:25:20 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:64400 in memory (size: 19.3 KB, free: 730.4 MB) -2016-05-18 19:25:20 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:64400 in memory (size: 1877.0 B, free: 730.4 MB) -2016-05-18 19:25:20 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-18 19:25:20 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:64400 in memory (size: 19.3 KB, free: 730.5 MB) -2016-05-18 19:25:20 INFO SparkContext:59 - Starting job: parquet at MatrixFactorizationModel.scala:372 -2016-05-18 19:25:20 INFO DAGScheduler:59 - Got job 13 (parquet at MatrixFactorizationModel.scala:372) with 1 output partitions -2016-05-18 19:25:20 INFO DAGScheduler:59 - Final stage: ResultStage 193(parquet at MatrixFactorizationModel.scala:372) -2016-05-18 19:25:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-18 19:25:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-18 19:25:20 INFO DAGScheduler:59 - Submitting ResultStage 193 (MapPartitionsRDD[240] at parquet at MatrixFactorizationModel.scala:372), which has no missing parents -2016-05-18 19:25:20 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=595335, maxMem=766075207 -2016-05-18 19:25:20 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-18 19:25:20 INFO MemoryStore:59 - ensureFreeSpace(21072) called with curMem=658335, maxMem=766075207 -2016-05-18 19:25:20 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 20.6 KB, free 729.9 MB) -2016-05-18 19:25:20 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:64400 (size: 20.6 KB, free: 730.4 MB) -2016-05-18 19:25:20 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 193 (MapPartitionsRDD[240] at parquet at MatrixFactorizationModel.scala:372) -2016-05-18 19:25:20 INFO TaskSchedulerImpl:59 - Adding task set 193.0 with 1 tasks -2016-05-18 19:25:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 193.0 (TID 41, localhost, PROCESS_LOCAL, 2263 bytes) -2016-05-18 19:25:20 INFO Executor:59 - Running task 0.0 in stage 193.0 (TID 41) -2016-05-18 19:25:20 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-18 19:25:20 INFO Executor:59 - Finished task 0.0 in stage 193.0 (TID 41). 1792 bytes result sent to driver -2016-05-18 19:25:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 193.0 (TID 41) in 83 ms on localhost (1/1) -2016-05-18 19:25:20 INFO DAGScheduler:59 - ResultStage 193 (parquet at MatrixFactorizationModel.scala:372) finished in 0.083 s -2016-05-18 19:25:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 193.0, whose tasks have all completed, from pool -2016-05-18 19:25:20 INFO DAGScheduler:59 - Job 13 finished: parquet at MatrixFactorizationModel.scala:372, took 0.114080 s -2016-05-18 19:25:20 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=679407, maxMem=766075207 -2016-05-18 19:25:20 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 86.4 KB, free 729.9 MB) -2016-05-18 19:25:20 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=767919, maxMem=766075207 -2016-05-18 19:25:20 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.8 MB) -2016-05-18 19:25:20 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:64400 (size: 19.3 KB, free: 730.4 MB) -2016-05-18 19:25:20 INFO SparkContext:59 - Created broadcast 45 from map at MatrixFactorizationModel.scala:373 -2016-05-18 19:25:20 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/als/myModelPath20160518192507629/data/product on driver -2016-05-18 19:25:20 INFO SparkContext:59 - Starting job: parquet at MatrixFactorizationModel.scala:376 -2016-05-18 19:25:20 INFO DAGScheduler:59 - Got job 14 (parquet at MatrixFactorizationModel.scala:376) with 1 output partitions -2016-05-18 19:25:20 INFO DAGScheduler:59 - Final stage: ResultStage 194(parquet at MatrixFactorizationModel.scala:376) -2016-05-18 19:25:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-18 19:25:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-18 19:25:20 INFO DAGScheduler:59 - Submitting ResultStage 194 (MapPartitionsRDD[245] at parquet at MatrixFactorizationModel.scala:376), which has no missing parents -2016-05-18 19:25:20 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=787707, maxMem=766075207 -2016-05-18 19:25:20 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 61.5 KB, free 729.8 MB) -2016-05-18 19:25:20 INFO MemoryStore:59 - ensureFreeSpace(21072) called with curMem=850707, maxMem=766075207 -2016-05-18 19:25:20 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 20.6 KB, free 729.8 MB) -2016-05-18 19:25:20 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:64400 (size: 20.6 KB, free: 730.4 MB) -2016-05-18 19:25:20 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:20 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:64400 in memory (size: 20.6 KB, free: 730.4 MB) -2016-05-18 19:25:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 194 (MapPartitionsRDD[245] at parquet at MatrixFactorizationModel.scala:376) -2016-05-18 19:25:20 INFO TaskSchedulerImpl:59 - Adding task set 194.0 with 1 tasks -2016-05-18 19:25:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 194.0 (TID 42, localhost, PROCESS_LOCAL, 2266 bytes) -2016-05-18 19:25:20 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-05-18 19:25:20 INFO Executor:59 - Running task 0.0 in stage 194.0 (TID 42) -2016-05-18 19:25:20 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-18 19:25:20 INFO Executor:59 - Finished task 0.0 in stage 194.0 (TID 42). 1792 bytes result sent to driver -2016-05-18 19:25:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 194.0 (TID 42) in 32 ms on localhost (1/1) -2016-05-18 19:25:20 INFO DAGScheduler:59 - ResultStage 194 (parquet at MatrixFactorizationModel.scala:376) finished in 0.032 s -2016-05-18 19:25:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 194.0, whose tasks have all completed, from pool -2016-05-18 19:25:20 INFO DAGScheduler:59 - Job 14 finished: parquet at MatrixFactorizationModel.scala:376, took 0.062135 s -2016-05-18 19:25:20 INFO MemoryStore:59 - ensureFreeSpace(230728) called with curMem=787707, maxMem=766075207 -2016-05-18 19:25:20 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 225.3 KB, free 729.6 MB) -2016-05-18 19:25:21 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1018435, maxMem=766075207 -2016-05-18 19:25:21 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.6 MB) -2016-05-18 19:25:21 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:64400 (size: 19.3 KB, free: 730.4 MB) -2016-05-18 19:25:21 INFO SparkContext:59 - Created broadcast 47 from map at MatrixFactorizationModel.scala:377 -2016-05-18 19:25:21 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-18 19:25:21 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/als/myModelPath20160518192507629/data/user/part-r-00000-7a21a0ad-4494-49e1-aa00-bab3f388edd5.gz.parquet -2016-05-18 19:25:21 INFO SparkContext:59 - Starting job: first at MatrixFactorizationModel.scala:67 -2016-05-18 19:25:21 INFO DAGScheduler:59 - Got job 15 (first at MatrixFactorizationModel.scala:67) with 1 output partitions -2016-05-18 19:25:21 INFO DAGScheduler:59 - Final stage: ResultStage 195(first at MatrixFactorizationModel.scala:67) -2016-05-18 19:25:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-18 19:25:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-18 19:25:21 INFO DAGScheduler:59 - Submitting ResultStage 195 (MapPartitionsRDD[243] at map at MatrixFactorizationModel.scala:373), which has no missing parents -2016-05-18 19:25:21 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=1038223, maxMem=766075207 -2016-05-18 19:25:21 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 4.5 KB, free 729.6 MB) -2016-05-18 19:25:21 INFO MemoryStore:59 - ensureFreeSpace(2692) called with curMem=1042855, maxMem=766075207 -2016-05-18 19:25:21 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.6 KB, free 729.6 MB) -2016-05-18 19:25:21 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:64400 (size: 2.6 KB, free: 730.4 MB) -2016-05-18 19:25:21 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 195 (MapPartitionsRDD[243] at map at MatrixFactorizationModel.scala:373) -2016-05-18 19:25:21 INFO TaskSchedulerImpl:59 - Adding task set 195.0 with 1 tasks -2016-05-18 19:25:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 195.0 (TID 43, localhost, PROCESS_LOCAL, 2292 bytes) -2016-05-18 19:25:21 INFO Executor:59 - Running task 0.0 in stage 195.0 (TID 43) -2016-05-18 19:25:21 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/als/myModelPath20160518192507629/data/user/part-r-00000-7a21a0ad-4494-49e1-aa00-bab3f388edd5.gz.parquet start: 0 end: 806 length: 806 hosts: []} -2016-05-18 19:25:21 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-18 19:25:21 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 id; - optional group features (LIST) { - repeated group bag { - optional double array; - } - } -} - -Catalyst form: -StructType(StructField(id,IntegerType,true), StructField(features,ArrayType(DoubleType,true),true)) - -2016-05-18 19:25:21 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 4 records. -2016-05-18 19:25:21 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-18 19:25:21 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-18 19:25:21 INFO InternalParquetRecordReader:151 - block read in memory in 28 ms. row count = 4 -2016-05-18 19:25:21 INFO Executor:59 - Finished task 0.0 in stage 195.0 (TID 43). 2288 bytes result sent to driver -2016-05-18 19:25:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 195.0 (TID 43) in 177 ms on localhost (1/1) -2016-05-18 19:25:21 INFO DAGScheduler:59 - ResultStage 195 (first at MatrixFactorizationModel.scala:67) finished in 0.177 s -2016-05-18 19:25:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 195.0, whose tasks have all completed, from pool -2016-05-18 19:25:21 INFO DAGScheduler:59 - Job 15 finished: first at MatrixFactorizationModel.scala:67, took 0.194637 s -2016-05-18 19:25:21 WARN MatrixFactorizationModel:71 - User factor does not have a partitioner. Prediction on individual records could be slow. -2016-05-18 19:25:21 WARN MatrixFactorizationModel:71 - User factor is not cached. Prediction could be slow. -2016-05-18 19:25:21 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/als/myModelPath20160518192507629/data/product/part-r-00000-d2c0be01-ecfa-48f5-b9d3-9e533b28358d.gz.parquet -2016-05-18 19:25:21 INFO SparkContext:59 - Starting job: first at MatrixFactorizationModel.scala:67 -2016-05-18 19:25:21 INFO DAGScheduler:59 - Got job 16 (first at MatrixFactorizationModel.scala:67) with 1 output partitions -2016-05-18 19:25:21 INFO DAGScheduler:59 - Final stage: ResultStage 196(first at MatrixFactorizationModel.scala:67) -2016-05-18 19:25:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-18 19:25:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-18 19:25:21 INFO DAGScheduler:59 - Submitting ResultStage 196 (MapPartitionsRDD[248] at map at MatrixFactorizationModel.scala:377), which has no missing parents -2016-05-18 19:25:21 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=1045547, maxMem=766075207 -2016-05-18 19:25:21 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 4.5 KB, free 729.6 MB) -2016-05-18 19:25:21 INFO MemoryStore:59 - ensureFreeSpace(2695) called with curMem=1050179, maxMem=766075207 -2016-05-18 19:25:21 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 2.6 KB, free 729.6 MB) -2016-05-18 19:25:21 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:64400 (size: 2.6 KB, free: 730.4 MB) -2016-05-18 19:25:21 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-05-18 19:25:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 196 (MapPartitionsRDD[248] at map at MatrixFactorizationModel.scala:377) -2016-05-18 19:25:21 INFO TaskSchedulerImpl:59 - Adding task set 196.0 with 1 tasks -2016-05-18 19:25:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 196.0 (TID 44, localhost, PROCESS_LOCAL, 2294 bytes) -2016-05-18 19:25:21 INFO Executor:59 - Running task 0.0 in stage 196.0 (TID 44) -2016-05-18 19:25:21 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/als/myModelPath20160518192507629/data/product/part-r-00000-d2c0be01-ecfa-48f5-b9d3-9e533b28358d.gz.parquet start: 0 end: 811 length: 811 hosts: []} -2016-05-18 19:25:21 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-18 19:25:21 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 id; - optional group features (LIST) { - repeated group bag { - optional double array; - } - } -} - -Catalyst form: -StructType(StructField(id,IntegerType,true), StructField(features,ArrayType(DoubleType,true),true)) - -2016-05-18 19:25:21 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 4 records. -2016-05-18 19:25:21 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-18 19:25:21 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-18 19:25:21 INFO InternalParquetRecordReader:151 - block read in memory in 1 ms. row count = 4 -2016-05-18 19:25:21 INFO Executor:59 - Finished task 0.0 in stage 196.0 (TID 44). 2288 bytes result sent to driver -2016-05-18 19:25:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 196.0 (TID 44) in 14 ms on localhost (1/1) -2016-05-18 19:25:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 196.0, whose tasks have all completed, from pool -2016-05-18 19:25:21 INFO DAGScheduler:59 - ResultStage 196 (first at MatrixFactorizationModel.scala:67) finished in 0.014 s -2016-05-18 19:25:21 INFO DAGScheduler:59 - Job 16 finished: first at MatrixFactorizationModel.scala:67, took 0.023209 s -2016-05-18 19:25:21 WARN MatrixFactorizationModel:71 - Product factor does not have a partitioner. Prediction on individual records could be slow. -2016-05-18 19:25:21 WARN MatrixFactorizationModel:71 - Product factor is not cached. Prediction could be slow. -2016-05-18 19:25:21 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-18 19:25:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-18 19:25:21 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.142:4040 -2016-05-18 19:25:21 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-18 19:25:21 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-18 19:25:21 INFO MemoryStore:59 - MemoryStore cleared -2016-05-18 19:25:21 INFO BlockManager:59 - BlockManager stopped -2016-05-18 19:25:21 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-18 19:25:21 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-18 19:25:21 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-18 19:25:21 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-18 19:25:21 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-876ce1ec-8fec-42be-b0f9-86d1c5bd550f -2016-05-23 16:40:32 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 16:40:35 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 16:40:36 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 16:40:36 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 16:40:36 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 16:40:38 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 16:40:39 INFO Remoting:74 - Starting remoting -2016-05-23 16:40:39 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@219.219.220.162:53165] -2016-05-23 16:40:39 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53165. -2016-05-23 16:40:39 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 16:40:39 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 16:40:39 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-7cbc258a-b0d8-4e60-9fb7-7db5a95e3959 -2016-05-23 16:40:40 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 16:40:40 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-c0e16cca-7bf4-42ff-98d5-c91314a0b23a\httpd-e3b6d8f5-d4ed-48bc-9128-ff30ec2e106e -2016-05-23 16:40:40 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 16:40:40 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 16:40:40 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53166 -2016-05-23 16:40:40 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53166. -2016-05-23 16:40:40 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 16:40:41 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 16:40:41 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 16:40:41 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 16:40:41 INFO SparkUI:59 - Started SparkUI at http://219.219.220.162:4040 -2016-05-23 16:40:42 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 16:40:42 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 16:40:42 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53178. -2016-05-23 16:40:42 INFO NettyBlockTransferService:59 - Server created on 53178 -2016-05-23 16:40:42 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 16:40:42 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53178 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53178) -2016-05-23 16:40:42 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 16:40:45 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 16:40:45 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 16:40:45 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 16:40:45 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 16:40:45 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53178 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 16:40:45 INFO SparkContext:59 - Created broadcast 0 from textFile at KMeansTest2.scala:20 -2016-05-23 16:40:46 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: 2002:dbdb:dca2:0:0:0:dbdb:dca2, but we couldn't find any external IP address! -2016-05-23 16:40:47 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 16:40:47 INFO SparkContext:59 - Starting job: count at KMeansTest2.scala:23 -2016-05-23 16:40:47 INFO DAGScheduler:59 - Got job 0 (count at KMeansTest2.scala:23) with 2 output partitions -2016-05-23 16:40:47 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at KMeansTest2.scala:23) -2016-05-23 16:40:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 16:40:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 16:40:47 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[1] at textFile at KMeansTest2.scala:20), which has no missing parents -2016-05-23 16:40:47 INFO MemoryStore:59 - ensureFreeSpace(3000) called with curMem=171580, maxMem=766075207 -2016-05-23 16:40:47 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-23 16:40:47 INFO MemoryStore:59 - ensureFreeSpace(1778) called with curMem=174580, maxMem=766075207 -2016-05-23 16:40:47 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1778.0 B, free 730.4 MB) -2016-05-23 16:40:47 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53178 (size: 1778.0 B, free: 730.6 MB) -2016-05-23 16:40:47 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 16:40:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at textFile at KMeansTest2.scala:20) -2016-05-23 16:40:47 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 16:40:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 16:40:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 16:40:47 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 16:40:47 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 16:40:47 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-05-23 16:40:47 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-05-23 16:40:47 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 16:40:47 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 16:40:47 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 16:40:47 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 16:40:47 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 16:40:48 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-05-23 16:40:48 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2082 bytes result sent to driver -2016-05-23 16:40:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 413 ms on localhost (1/2) -2016-05-23 16:40:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 489 ms on localhost (2/2) -2016-05-23 16:40:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 16:40:48 INFO DAGScheduler:59 - ResultStage 0 (count at KMeansTest2.scala:23) finished in 0.557 s -2016-05-23 16:40:48 INFO DAGScheduler:59 - Job 0 finished: count at KMeansTest2.scala:23, took 0.832115 s -2016-05-23 16:40:48 INFO SparkContext:59 - Starting job: foreach at KMeansTest2.scala:24 -2016-05-23 16:40:48 INFO DAGScheduler:59 - Got job 1 (foreach at KMeansTest2.scala:24) with 2 output partitions -2016-05-23 16:40:48 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at KMeansTest2.scala:24) -2016-05-23 16:40:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 16:40:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 16:40:48 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[1] at textFile at KMeansTest2.scala:20), which has no missing parents -2016-05-23 16:40:48 INFO MemoryStore:59 - ensureFreeSpace(3120) called with curMem=176358, maxMem=766075207 -2016-05-23 16:40:48 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.0 KB, free 730.4 MB) -2016-05-23 16:40:48 INFO MemoryStore:59 - ensureFreeSpace(1837) called with curMem=179478, maxMem=766075207 -2016-05-23 16:40:48 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1837.0 B, free 730.4 MB) -2016-05-23 16:40:48 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53178 (size: 1837.0 B, free: 730.6 MB) -2016-05-23 16:40:48 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 16:40:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[1] at textFile at KMeansTest2.scala:20) -2016-05-23 16:40:48 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 16:40:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 16:40:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 16:40:48 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 16:40:48 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 16:40:48 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-05-23 16:40:48 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-05-23 16:40:48 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 16:40:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 43 ms on localhost (1/2) -2016-05-23 16:40:48 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 16:40:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 65 ms on localhost (2/2) -2016-05-23 16:40:48 INFO DAGScheduler:59 - ResultStage 1 (foreach at KMeansTest2.scala:24) finished in 0.066 s -2016-05-23 16:40:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 16:40:48 INFO DAGScheduler:59 - Job 1 finished: foreach at KMeansTest2.scala:24, took 0.088805 s -2016-05-23 16:40:48 INFO SparkContext:59 - Starting job: foreach at KMeansTest2.scala:27 -2016-05-23 16:40:48 INFO DAGScheduler:59 - Got job 2 (foreach at KMeansTest2.scala:27) with 2 output partitions -2016-05-23 16:40:48 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at KMeansTest2.scala:27) -2016-05-23 16:40:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 16:40:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 16:40:48 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[2] at map at KMeansTest2.scala:26), which has no missing parents -2016-05-23 16:40:48 INFO MemoryStore:59 - ensureFreeSpace(3304) called with curMem=181315, maxMem=766075207 -2016-05-23 16:40:48 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-05-23 16:40:48 INFO MemoryStore:59 - ensureFreeSpace(1926) called with curMem=184619, maxMem=766075207 -2016-05-23 16:40:48 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1926.0 B, free 730.4 MB) -2016-05-23 16:40:48 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53178 (size: 1926.0 B, free: 730.6 MB) -2016-05-23 16:40:48 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 16:40:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[2] at map at KMeansTest2.scala:26) -2016-05-23 16:40:48 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-23 16:40:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 16:40:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 16:40:48 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-23 16:40:48 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 16:40:48 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-05-23 16:40:48 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-05-23 16:40:48 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2044 bytes result sent to driver -2016-05-23 16:40:48 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2044 bytes result sent to driver -2016-05-23 16:40:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 59 ms on localhost (1/2) -2016-05-23 16:40:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 58 ms on localhost (2/2) -2016-05-23 16:40:48 INFO DAGScheduler:59 - ResultStage 2 (foreach at KMeansTest2.scala:27) finished in 0.060 s -2016-05-23 16:40:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 16:40:48 INFO DAGScheduler:59 - Job 2 finished: foreach at KMeansTest2.scala:27, took 0.146555 s -2016-05-23 16:40:48 WARN KMeans:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-05-23 16:40:48 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-05-23 16:40:48 INFO DAGScheduler:59 - Got job 3 (takeSample at KMeans.scala:376) with 2 output partitions -2016-05-23 16:40:48 INFO DAGScheduler:59 - Final stage: ResultStage 3(takeSample at KMeans.scala:376) -2016-05-23 16:40:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 16:40:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 16:40:48 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[5] at map at KMeans.scala:210), which has no missing parents -2016-05-23 16:40:48 INFO MemoryStore:59 - ensureFreeSpace(4072) called with curMem=186545, maxMem=766075207 -2016-05-23 16:40:48 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-05-23 16:40:48 INFO MemoryStore:59 - ensureFreeSpace(2290) called with curMem=190617, maxMem=766075207 -2016-05-23 16:40:48 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-05-23 16:40:48 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53178 (size: 2.2 KB, free: 730.6 MB) -2016-05-23 16:40:48 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 16:40:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[5] at map at KMeans.scala:210) -2016-05-23 16:40:48 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 16:40:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2352 bytes) -2016-05-23 16:40:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2352 bytes) -2016-05-23 16:40:48 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-23 16:40:48 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-23 16:40:48 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-05-23 16:40:48 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-05-23 16:40:48 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53178 in memory (size: 1926.0 B, free: 730.6 MB) -2016-05-23 16:40:48 INFO CacheManager:59 - Partition rdd_3_0 not found, computing it -2016-05-23 16:40:48 INFO CacheManager:59 - Partition rdd_3_1 not found, computing it -2016-05-23 16:40:48 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-05-23 16:40:48 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-05-23 16:40:49 INFO MemoryStore:59 - ensureFreeSpace(128) called with curMem=187677, maxMem=766075207 -2016-05-23 16:40:49 INFO MemoryStore:59 - Block rdd_3_0 stored as values in memory (estimated size 128.0 B, free 730.4 MB) -2016-05-23 16:40:49 INFO BlockManagerInfo:59 - Added rdd_3_0 in memory on localhost:53178 (size: 128.0 B, free: 730.6 MB) -2016-05-23 16:40:49 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 2662 bytes result sent to driver -2016-05-23 16:40:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 270 ms on localhost (1/2) -2016-05-23 16:40:49 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-23 16:40:49 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=187805, maxMem=766075207 -2016-05-23 16:40:49 INFO MemoryStore:59 - Block rdd_3_1 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-23 16:40:49 INFO BlockManagerInfo:59 - Added rdd_3_1 in memory on localhost:53178 (size: 72.0 B, free: 730.6 MB) -2016-05-23 16:40:49 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 2662 bytes result sent to driver -2016-05-23 16:40:49 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53178 in memory (size: 1837.0 B, free: 730.6 MB) -2016-05-23 16:40:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 237 ms on localhost (2/2) -2016-05-23 16:40:49 INFO DAGScheduler:59 - ResultStage 3 (takeSample at KMeans.scala:376) finished in 0.295 s -2016-05-23 16:40:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 16:40:49 INFO DAGScheduler:59 - Job 3 finished: takeSample at KMeans.scala:376, took 0.409158 s -2016-05-23 16:40:49 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-23 16:40:49 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53178 in memory (size: 1778.0 B, free: 730.6 MB) -2016-05-23 16:40:49 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-23 16:40:49 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-05-23 16:40:49 INFO DAGScheduler:59 - Got job 4 (takeSample at KMeans.scala:376) with 2 output partitions -2016-05-23 16:40:49 INFO DAGScheduler:59 - Final stage: ResultStage 4(takeSample at KMeans.scala:376) -2016-05-23 16:40:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 16:40:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 16:40:49 INFO DAGScheduler:59 - Submitting ResultStage 4 (PartitionwiseSampledRDD[7] at takeSample at KMeans.scala:376), which has no missing parents -2016-05-23 16:40:49 INFO MemoryStore:59 - ensureFreeSpace(21464) called with curMem=178142, maxMem=766075207 -2016-05-23 16:40:49 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 21.0 KB, free 730.4 MB) -2016-05-23 16:40:49 INFO MemoryStore:59 - ensureFreeSpace(8958) called with curMem=199606, maxMem=766075207 -2016-05-23 16:40:49 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 8.7 KB, free 730.4 MB) -2016-05-23 16:40:49 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53178 (size: 8.7 KB, free: 730.6 MB) -2016-05-23 16:40:49 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 16:40:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (PartitionwiseSampledRDD[7] at takeSample at KMeans.scala:376) -2016-05-23 16:40:49 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 16:40:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2461 bytes) -2016-05-23 16:40:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2461 bytes) -2016-05-23 16:40:49 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-23 16:40:49 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-23 16:40:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-05-23 16:40:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-05-23 16:40:49 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-05-23 16:40:49 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-05-23 16:40:49 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2575 bytes result sent to driver -2016-05-23 16:40:49 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2457 bytes result sent to driver -2016-05-23 16:40:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 49 ms on localhost (1/2) -2016-05-23 16:40:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 55 ms on localhost (2/2) -2016-05-23 16:40:49 INFO DAGScheduler:59 - ResultStage 4 (takeSample at KMeans.scala:376) finished in 0.057 s -2016-05-23 16:40:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 16:40:49 INFO DAGScheduler:59 - Job 4 finished: takeSample at KMeans.scala:376, took 0.102050 s -2016-05-23 16:40:49 INFO MemoryStore:59 - ensureFreeSpace(1720) called with curMem=208564, maxMem=766075207 -2016-05-23 16:40:49 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 1720.0 B, free 730.4 MB) -2016-05-23 16:40:49 INFO MemoryStore:59 - ensureFreeSpace(586) called with curMem=210284, maxMem=766075207 -2016-05-23 16:40:49 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 586.0 B, free 730.4 MB) -2016-05-23 16:40:49 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53178 (size: 586.0 B, free: 730.6 MB) -2016-05-23 16:40:49 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53178 in memory (size: 8.7 KB, free: 730.6 MB) -2016-05-23 16:40:49 INFO SparkContext:59 - Created broadcast 6 from broadcast at KMeans.scala:394 -2016-05-23 16:40:49 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 16:40:49 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53178 in memory (size: 2.2 KB, free: 730.6 MB) -2016-05-23 16:40:49 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-05-23 16:40:49 INFO DAGScheduler:59 - Got job 5 (aggregate at KMeans.scala:402) with 2 output partitions -2016-05-23 16:40:49 INFO DAGScheduler:59 - Final stage: ResultStage 5(aggregate at KMeans.scala:402) -2016-05-23 16:40:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 16:40:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 16:40:49 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[9] at map at KMeans.scala:396), which has no missing parents -2016-05-23 16:40:49 INFO MemoryStore:59 - ensureFreeSpace(5664) called with curMem=174086, maxMem=766075207 -2016-05-23 16:40:49 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.4 MB) -2016-05-23 16:40:49 INFO MemoryStore:59 - ensureFreeSpace(2875) called with curMem=179750, maxMem=766075207 -2016-05-23 16:40:49 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-23 16:40:49 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53178 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 16:40:49 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 16:40:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[9] at map at KMeans.scala:396) -2016-05-23 16:40:49 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 16:40:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2384 bytes) -2016-05-23 16:40:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2384 bytes) -2016-05-23 16:40:49 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-05-23 16:40:49 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-05-23 16:40:49 INFO CacheManager:59 - Partition rdd_9_0 not found, computing it -2016-05-23 16:40:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-05-23 16:40:49 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-05-23 16:40:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-05-23 16:40:49 INFO CacheManager:59 - Partition rdd_9_1 not found, computing it -2016-05-23 16:40:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-05-23 16:40:49 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-05-23 16:40:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-05-23 16:40:49 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-05-23 16:40:49 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-05-23 16:40:49 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 16:40:49 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 16:40:49 INFO MemoryStore:59 - ensureFreeSpace(216) called with curMem=182625, maxMem=766075207 -2016-05-23 16:40:49 INFO MemoryStore:59 - Block rdd_9_1 stored as values in memory (estimated size 216.0 B, free 730.4 MB) -2016-05-23 16:40:49 INFO MemoryStore:59 - ensureFreeSpace(416) called with curMem=182841, maxMem=766075207 -2016-05-23 16:40:49 INFO MemoryStore:59 - Block rdd_9_0 stored as values in memory (estimated size 416.0 B, free 730.4 MB) -2016-05-23 16:40:49 INFO BlockManagerInfo:59 - Added rdd_9_1 in memory on localhost:53178 (size: 216.0 B, free: 730.6 MB) -2016-05-23 16:40:49 INFO BlockManagerInfo:59 - Added rdd_9_0 in memory on localhost:53178 (size: 416.0 B, free: 730.6 MB) -2016-05-23 16:40:49 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 2687 bytes result sent to driver -2016-05-23 16:40:49 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 2687 bytes result sent to driver -2016-05-23 16:40:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 98 ms on localhost (1/2) -2016-05-23 16:40:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 98 ms on localhost (2/2) -2016-05-23 16:40:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 16:40:49 INFO DAGScheduler:59 - ResultStage 5 (aggregate at KMeans.scala:402) finished in 0.102 s -2016-05-23 16:40:49 INFO DAGScheduler:59 - Job 5 finished: aggregate at KMeans.scala:402, took 0.125480 s -2016-05-23 16:40:49 INFO MapPartitionsRDD:59 - Removing RDD 6 from persistence list -2016-05-23 16:40:49 INFO BlockManager:59 - Removing RDD 6 -2016-05-23 16:40:49 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-05-23 16:40:49 INFO DAGScheduler:59 - Got job 6 (collect at KMeans.scala:431) with 2 output partitions -2016-05-23 16:40:49 INFO DAGScheduler:59 - Final stage: ResultStage 6(collect at KMeans.scala:431) -2016-05-23 16:40:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 16:40:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 16:40:49 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[11] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-05-23 16:40:49 INFO MemoryStore:59 - ensureFreeSpace(5824) called with curMem=183257, maxMem=766075207 -2016-05-23 16:40:49 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.7 KB, free 730.4 MB) -2016-05-23 16:40:49 INFO MemoryStore:59 - ensureFreeSpace(3014) called with curMem=189081, maxMem=766075207 -2016-05-23 16:40:49 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-23 16:40:49 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53178 (size: 2.9 KB, free: 730.6 MB) -2016-05-23 16:40:49 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 16:40:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[11] at mapPartitionsWithIndex at KMeans.scala:423) -2016-05-23 16:40:49 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 16:40:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2416 bytes) -2016-05-23 16:40:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2416 bytes) -2016-05-23 16:40:49 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-05-23 16:40:49 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 13) -2016-05-23 16:40:49 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53178 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 16:40:49 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 16:40:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-05-23 16:40:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-05-23 16:40:49 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-05-23 16:40:49 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-23 16:40:49 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-05-23 16:40:49 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-23 16:40:49 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 3259 bytes result sent to driver -2016-05-23 16:40:49 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 13). 2911 bytes result sent to driver -2016-05-23 16:40:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 48 ms on localhost (1/2) -2016-05-23 16:40:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 13) in 45 ms on localhost (2/2) -2016-05-23 16:40:49 INFO DAGScheduler:59 - ResultStage 6 (collect at KMeans.scala:431) finished in 0.051 s -2016-05-23 16:40:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 16:40:49 INFO DAGScheduler:59 - Job 6 finished: collect at KMeans.scala:431, took 0.073368 s -2016-05-23 16:40:49 INFO MemoryStore:59 - ensureFreeSpace(2560) called with curMem=183556, maxMem=766075207 -2016-05-23 16:40:49 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.5 KB, free 730.4 MB) -2016-05-23 16:40:49 INFO MemoryStore:59 - ensureFreeSpace(651) called with curMem=186116, maxMem=766075207 -2016-05-23 16:40:49 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 651.0 B, free 730.4 MB) -2016-05-23 16:40:49 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53178 (size: 651.0 B, free: 730.6 MB) -2016-05-23 16:40:49 INFO SparkContext:59 - Created broadcast 9 from broadcast at KMeans.scala:394 -2016-05-23 16:40:49 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-05-23 16:40:49 INFO DAGScheduler:59 - Got job 7 (aggregate at KMeans.scala:402) with 2 output partitions -2016-05-23 16:40:49 INFO DAGScheduler:59 - Final stage: ResultStage 7(aggregate at KMeans.scala:402) -2016-05-23 16:40:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 16:40:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 16:40:49 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[13] at map at KMeans.scala:396), which has no missing parents -2016-05-23 16:40:49 INFO MemoryStore:59 - ensureFreeSpace(5896) called with curMem=186767, maxMem=766075207 -2016-05-23 16:40:49 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.8 KB, free 730.4 MB) -2016-05-23 16:40:49 INFO MemoryStore:59 - ensureFreeSpace(2957) called with curMem=192663, maxMem=766075207 -2016-05-23 16:40:49 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-23 16:40:49 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53178 (size: 2.9 KB, free: 730.6 MB) -2016-05-23 16:40:49 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 16:40:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[13] at map at KMeans.scala:396) -2016-05-23 16:40:49 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-23 16:40:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2416 bytes) -2016-05-23 16:40:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 15, localhost, PROCESS_LOCAL, 2416 bytes) -2016-05-23 16:40:49 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 14) -2016-05-23 16:40:49 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 15) -2016-05-23 16:40:49 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-23 16:40:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-05-23 16:40:49 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-23 16:40:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-05-23 16:40:49 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-05-23 16:40:49 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-23 16:40:49 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-05-23 16:40:49 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-23 16:40:49 INFO MemoryStore:59 - ensureFreeSpace(216) called with curMem=195620, maxMem=766075207 -2016-05-23 16:40:49 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 216.0 B, free 730.4 MB) -2016-05-23 16:40:49 INFO MemoryStore:59 - ensureFreeSpace(416) called with curMem=195836, maxMem=766075207 -2016-05-23 16:40:49 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 416.0 B, free 730.4 MB) -2016-05-23 16:40:49 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:53178 (size: 216.0 B, free: 730.6 MB) -2016-05-23 16:40:49 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 15). 2687 bytes result sent to driver -2016-05-23 16:40:49 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:53178 (size: 416.0 B, free: 730.6 MB) -2016-05-23 16:40:49 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 14). 2687 bytes result sent to driver -2016-05-23 16:40:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 15) in 23 ms on localhost (1/2) -2016-05-23 16:40:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 14) in 26 ms on localhost (2/2) -2016-05-23 16:40:49 INFO DAGScheduler:59 - ResultStage 7 (aggregate at KMeans.scala:402) finished in 0.026 s -2016-05-23 16:40:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 16:40:49 INFO DAGScheduler:59 - Job 7 finished: aggregate at KMeans.scala:402, took 0.047579 s -2016-05-23 16:40:49 INFO MapPartitionsRDD:59 - Removing RDD 9 from persistence list -2016-05-23 16:40:49 INFO BlockManager:59 - Removing RDD 9 -2016-05-23 16:40:50 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-05-23 16:40:50 INFO DAGScheduler:59 - Got job 8 (collect at KMeans.scala:431) with 2 output partitions -2016-05-23 16:40:50 INFO DAGScheduler:59 - Final stage: ResultStage 8(collect at KMeans.scala:431) -2016-05-23 16:40:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 16:40:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 16:40:50 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[15] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(6056) called with curMem=195620, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.9 KB, free 730.4 MB) -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(3077) called with curMem=201676, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53178 (size: 3.0 KB, free: 730.6 MB) -2016-05-23 16:40:50 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:53178 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-23 16:40:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[15] at mapPartitionsWithIndex at KMeans.scala:423) -2016-05-23 16:40:50 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-23 16:40:50 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 16:40:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2448 bytes) -2016-05-23 16:40:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 17, localhost, PROCESS_LOCAL, 2448 bytes) -2016-05-23 16:40:50 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 16) -2016-05-23 16:40:50 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 17) -2016-05-23 16:40:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-05-23 16:40:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-23 16:40:50 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 17). 2917 bytes result sent to driver -2016-05-23 16:40:50 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 16). 3209 bytes result sent to driver -2016-05-23 16:40:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 17) in 18 ms on localhost (1/2) -2016-05-23 16:40:50 INFO DAGScheduler:59 - ResultStage 8 (collect at KMeans.scala:431) finished in 0.019 s -2016-05-23 16:40:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 16) in 19 ms on localhost (2/2) -2016-05-23 16:40:50 INFO DAGScheduler:59 - Job 8 finished: collect at KMeans.scala:431, took 0.041129 s -2016-05-23 16:40:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(2120) called with curMem=195900, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(650) called with curMem=198020, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 650.0 B, free 730.4 MB) -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53178 (size: 650.0 B, free: 730.6 MB) -2016-05-23 16:40:50 INFO SparkContext:59 - Created broadcast 12 from broadcast at KMeans.scala:394 -2016-05-23 16:40:50 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-05-23 16:40:50 INFO DAGScheduler:59 - Got job 9 (aggregate at KMeans.scala:402) with 2 output partitions -2016-05-23 16:40:50 INFO DAGScheduler:59 - Final stage: ResultStage 9(aggregate at KMeans.scala:402) -2016-05-23 16:40:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 16:40:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 16:40:50 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[17] at map at KMeans.scala:396), which has no missing parents -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(6128) called with curMem=198670, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 6.0 KB, free 730.4 MB) -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(3015) called with curMem=204798, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53178 (size: 2.9 KB, free: 730.6 MB) -2016-05-23 16:40:50 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 16:40:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[17] at map at KMeans.scala:396) -2016-05-23 16:40:50 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 16:40:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2448 bytes) -2016-05-23 16:40:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 19, localhost, PROCESS_LOCAL, 2448 bytes) -2016-05-23 16:40:50 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 18) -2016-05-23 16:40:50 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 19) -2016-05-23 16:40:50 INFO CacheManager:59 - Partition rdd_17_0 not found, computing it -2016-05-23 16:40:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-05-23 16:40:50 INFO CacheManager:59 - Partition rdd_17_1 not found, computing it -2016-05-23 16:40:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(216) called with curMem=207813, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block rdd_17_1 stored as values in memory (estimated size 216.0 B, free 730.4 MB) -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(416) called with curMem=208029, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block rdd_17_0 stored as values in memory (estimated size 416.0 B, free 730.4 MB) -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Added rdd_17_1 in memory on localhost:53178 (size: 216.0 B, free: 730.6 MB) -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Added rdd_17_0 in memory on localhost:53178 (size: 416.0 B, free: 730.6 MB) -2016-05-23 16:40:50 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 19). 2687 bytes result sent to driver -2016-05-23 16:40:50 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 18). 2687 bytes result sent to driver -2016-05-23 16:40:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 19) in 25 ms on localhost (1/2) -2016-05-23 16:40:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 18) in 27 ms on localhost (2/2) -2016-05-23 16:40:50 INFO DAGScheduler:59 - ResultStage 9 (aggregate at KMeans.scala:402) finished in 0.027 s -2016-05-23 16:40:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 16:40:50 INFO DAGScheduler:59 - Job 9 finished: aggregate at KMeans.scala:402, took 0.043861 s -2016-05-23 16:40:50 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-23 16:40:50 INFO BlockManager:59 - Removing RDD 13 -2016-05-23 16:40:50 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-23 16:40:50 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:53178 in memory (size: 3.0 KB, free: 730.6 MB) -2016-05-23 16:40:50 INFO DAGScheduler:59 - Got job 10 (collect at KMeans.scala:431) with 2 output partitions -2016-05-23 16:40:50 INFO DAGScheduler:59 - Final stage: ResultStage 10(collect at KMeans.scala:431) -2016-05-23 16:40:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 16:40:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:53178 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-23 16:40:50 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[19] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(6288) called with curMem=189537, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 6.1 KB, free 730.4 MB) -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(3110) called with curMem=195825, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53178 (size: 3.0 KB, free: 730.6 MB) -2016-05-23 16:40:50 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 16:40:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[19] at mapPartitionsWithIndex at KMeans.scala:423) -2016-05-23 16:40:50 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-23 16:40:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2480 bytes) -2016-05-23 16:40:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 21, localhost, PROCESS_LOCAL, 2480 bytes) -2016-05-23 16:40:50 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 21) -2016-05-23 16:40:50 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 20) -2016-05-23 16:40:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-05-23 16:40:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_17_0 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_17_1 locally -2016-05-23 16:40:50 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 20). 2040 bytes result sent to driver -2016-05-23 16:40:50 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 21). 2738 bytes result sent to driver -2016-05-23 16:40:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 20) in 16 ms on localhost (1/2) -2016-05-23 16:40:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 21) in 16 ms on localhost (2/2) -2016-05-23 16:40:50 INFO DAGScheduler:59 - ResultStage 10 (collect at KMeans.scala:431) finished in 0.017 s -2016-05-23 16:40:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 16:40:50 INFO DAGScheduler:59 - Job 10 finished: collect at KMeans.scala:431, took 0.033755 s -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(1200) called with curMem=198935, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1200.0 B, free 730.4 MB) -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(446) called with curMem=200135, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 446.0 B, free 730.4 MB) -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53178 (size: 446.0 B, free: 730.6 MB) -2016-05-23 16:40:50 INFO SparkContext:59 - Created broadcast 15 from broadcast at KMeans.scala:394 -2016-05-23 16:40:50 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-05-23 16:40:50 INFO DAGScheduler:59 - Got job 11 (aggregate at KMeans.scala:402) with 2 output partitions -2016-05-23 16:40:50 INFO DAGScheduler:59 - Final stage: ResultStage 11(aggregate at KMeans.scala:402) -2016-05-23 16:40:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 16:40:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 16:40:50 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[21] at map at KMeans.scala:396), which has no missing parents -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(6360) called with curMem=200581, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 6.2 KB, free 730.4 MB) -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(3075) called with curMem=206941, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53178 (size: 3.0 KB, free: 730.6 MB) -2016-05-23 16:40:50 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 16:40:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[21] at map at KMeans.scala:396) -2016-05-23 16:40:50 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-23 16:40:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2480 bytes) -2016-05-23 16:40:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 23, localhost, PROCESS_LOCAL, 2480 bytes) -2016-05-23 16:40:50 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 23) -2016-05-23 16:40:50 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 22) -2016-05-23 16:40:50 INFO CacheManager:59 - Partition rdd_21_0 not found, computing it -2016-05-23 16:40:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-05-23 16:40:50 INFO CacheManager:59 - Partition rdd_21_1 not found, computing it -2016-05-23 16:40:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_17_0 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_17_1 locally -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(216) called with curMem=210016, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block rdd_21_1 stored as values in memory (estimated size 216.0 B, free 730.4 MB) -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(416) called with curMem=210232, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block rdd_21_0 stored as values in memory (estimated size 416.0 B, free 730.4 MB) -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Added rdd_21_1 in memory on localhost:53178 (size: 216.0 B, free: 730.6 MB) -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Added rdd_21_0 in memory on localhost:53178 (size: 416.0 B, free: 730.6 MB) -2016-05-23 16:40:50 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 23). 2687 bytes result sent to driver -2016-05-23 16:40:50 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 22). 2687 bytes result sent to driver -2016-05-23 16:40:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 23) in 20 ms on localhost (1/2) -2016-05-23 16:40:50 INFO DAGScheduler:59 - ResultStage 11 (aggregate at KMeans.scala:402) finished in 0.023 s -2016-05-23 16:40:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 22) in 23 ms on localhost (2/2) -2016-05-23 16:40:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-23 16:40:50 INFO DAGScheduler:59 - Job 11 finished: aggregate at KMeans.scala:402, took 0.040063 s -2016-05-23 16:40:50 INFO MapPartitionsRDD:59 - Removing RDD 17 from persistence list -2016-05-23 16:40:50 INFO BlockManager:59 - Removing RDD 17 -2016-05-23 16:40:50 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:53178 in memory (size: 3.0 KB, free: 730.6 MB) -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:53178 in memory (size: 3.0 KB, free: 730.6 MB) -2016-05-23 16:40:50 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-23 16:40:50 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-05-23 16:40:50 INFO DAGScheduler:59 - Got job 12 (collect at KMeans.scala:431) with 2 output partitions -2016-05-23 16:40:50 INFO DAGScheduler:59 - Final stage: ResultStage 12(collect at KMeans.scala:431) -2016-05-23 16:40:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 16:40:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 16:40:50 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[23] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(6520) called with curMem=191183, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 6.4 KB, free 730.4 MB) -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(3158) called with curMem=197703, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.4 MB) -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53178 (size: 3.1 KB, free: 730.6 MB) -2016-05-23 16:40:50 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 16:40:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[23] at mapPartitionsWithIndex at KMeans.scala:423) -2016-05-23 16:40:50 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-23 16:40:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2512 bytes) -2016-05-23 16:40:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 25, localhost, PROCESS_LOCAL, 2512 bytes) -2016-05-23 16:40:50 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 24) -2016-05-23 16:40:50 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 25) -2016-05-23 16:40:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-05-23 16:40:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-05-23 16:40:50 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 25). 2040 bytes result sent to driver -2016-05-23 16:40:50 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 24). 2040 bytes result sent to driver -2016-05-23 16:40:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 25) in 14 ms on localhost (1/2) -2016-05-23 16:40:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 24) in 16 ms on localhost (2/2) -2016-05-23 16:40:50 INFO DAGScheduler:59 - ResultStage 12 (collect at KMeans.scala:431) finished in 0.016 s -2016-05-23 16:40:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-23 16:40:50 INFO DAGScheduler:59 - Job 12 finished: collect at KMeans.scala:431, took 0.032613 s -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(1120) called with curMem=200861, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 1120.0 B, free 730.4 MB) -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(247) called with curMem=201981, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 247.0 B, free 730.4 MB) -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53178 (size: 247.0 B, free: 730.6 MB) -2016-05-23 16:40:50 INFO SparkContext:59 - Created broadcast 18 from broadcast at KMeans.scala:394 -2016-05-23 16:40:50 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-05-23 16:40:50 INFO DAGScheduler:59 - Got job 13 (aggregate at KMeans.scala:402) with 2 output partitions -2016-05-23 16:40:50 INFO DAGScheduler:59 - Final stage: ResultStage 13(aggregate at KMeans.scala:402) -2016-05-23 16:40:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 16:40:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 16:40:50 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[25] at map at KMeans.scala:396), which has no missing parents -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(6600) called with curMem=202228, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 6.4 KB, free 730.4 MB) -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(3127) called with curMem=208828, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.4 MB) -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53178 (size: 3.1 KB, free: 730.6 MB) -2016-05-23 16:40:50 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 16:40:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[25] at map at KMeans.scala:396) -2016-05-23 16:40:50 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 16:40:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2512 bytes) -2016-05-23 16:40:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 27, localhost, PROCESS_LOCAL, 2512 bytes) -2016-05-23 16:40:50 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 27) -2016-05-23 16:40:50 INFO CacheManager:59 - Partition rdd_25_1 not found, computing it -2016-05-23 16:40:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(216) called with curMem=211955, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block rdd_25_1 stored as values in memory (estimated size 216.0 B, free 730.4 MB) -2016-05-23 16:40:50 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 26) -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Added rdd_25_1 in memory on localhost:53178 (size: 216.0 B, free: 730.6 MB) -2016-05-23 16:40:50 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 27). 2687 bytes result sent to driver -2016-05-23 16:40:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 27) in 27 ms on localhost (1/2) -2016-05-23 16:40:50 INFO CacheManager:59 - Partition rdd_25_0 not found, computing it -2016-05-23 16:40:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(416) called with curMem=212171, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block rdd_25_0 stored as values in memory (estimated size 416.0 B, free 730.4 MB) -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Added rdd_25_0 in memory on localhost:53178 (size: 416.0 B, free: 730.6 MB) -2016-05-23 16:40:50 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 26). 2687 bytes result sent to driver -2016-05-23 16:40:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 26) in 66 ms on localhost (2/2) -2016-05-23 16:40:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 16:40:50 INFO DAGScheduler:59 - ResultStage 13 (aggregate at KMeans.scala:402) finished in 0.067 s -2016-05-23 16:40:50 INFO DAGScheduler:59 - Job 13 finished: aggregate at KMeans.scala:402, took 0.087742 s -2016-05-23 16:40:50 INFO MapPartitionsRDD:59 - Removing RDD 21 from persistence list -2016-05-23 16:40:50 INFO BlockManager:59 - Removing RDD 21 -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:53178 in memory (size: 3.1 KB, free: 730.6 MB) -2016-05-23 16:40:50 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:53178 in memory (size: 3.1 KB, free: 730.6 MB) -2016-05-23 16:40:50 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-23 16:40:50 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-05-23 16:40:50 INFO DAGScheduler:59 - Got job 14 (collect at KMeans.scala:431) with 2 output partitions -2016-05-23 16:40:50 INFO DAGScheduler:59 - Final stage: ResultStage 14(collect at KMeans.scala:431) -2016-05-23 16:40:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 16:40:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 16:40:50 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[27] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(6760) called with curMem=192550, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 6.6 KB, free 730.4 MB) -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(3213) called with curMem=199310, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.4 MB) -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:53178 (size: 3.1 KB, free: 730.6 MB) -2016-05-23 16:40:50 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 16:40:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[27] at mapPartitionsWithIndex at KMeans.scala:423) -2016-05-23 16:40:50 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-23 16:40:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2544 bytes) -2016-05-23 16:40:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 29, localhost, PROCESS_LOCAL, 2544 bytes) -2016-05-23 16:40:50 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 29) -2016-05-23 16:40:50 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 28) -2016-05-23 16:40:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-05-23 16:40:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-05-23 16:40:50 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 29). 2040 bytes result sent to driver -2016-05-23 16:40:50 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 28). 2040 bytes result sent to driver -2016-05-23 16:40:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 29) in 18 ms on localhost (1/2) -2016-05-23 16:40:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 28) in 21 ms on localhost (2/2) -2016-05-23 16:40:50 INFO DAGScheduler:59 - ResultStage 14 (collect at KMeans.scala:431) finished in 0.021 s -2016-05-23 16:40:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-23 16:40:50 INFO DAGScheduler:59 - Job 14 finished: collect at KMeans.scala:431, took 0.042739 s -2016-05-23 16:40:50 INFO MapPartitionsRDD:59 - Removing RDD 25 from persistence list -2016-05-23 16:40:50 INFO BlockManager:59 - Removing RDD 25 -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(4240) called with curMem=201891, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 4.1 KB, free 730.4 MB) -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(829) called with curMem=206131, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 829.0 B, free 730.4 MB) -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:53178 (size: 829.0 B, free: 730.6 MB) -2016-05-23 16:40:50 INFO SparkContext:59 - Created broadcast 21 from broadcast at KMeans.scala:445 -2016-05-23 16:40:50 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:450 -2016-05-23 16:40:50 INFO DAGScheduler:59 - Registering RDD 28 (flatMap at KMeans.scala:446) -2016-05-23 16:40:50 INFO DAGScheduler:59 - Got job 15 (collectAsMap at KMeans.scala:450) with 2 output partitions -2016-05-23 16:40:50 INFO DAGScheduler:59 - Final stage: ResultStage 16(collectAsMap at KMeans.scala:450) -2016-05-23 16:40:50 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 16:40:50 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 15) -2016-05-23 16:40:50 INFO DAGScheduler:59 - Submitting ShuffleMapStage 15 (MapPartitionsRDD[28] at flatMap at KMeans.scala:446), which has no missing parents -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=206960, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 5.5 KB, free 730.4 MB) -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(3027) called with curMem=212616, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:53178 (size: 3.0 KB, free: 730.6 MB) -2016-05-23 16:40:50 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 16:40:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 15 (MapPartitionsRDD[28] at flatMap at KMeans.scala:446) -2016-05-23 16:40:50 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-23 16:40:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2341 bytes) -2016-05-23 16:40:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 31, localhost, PROCESS_LOCAL, 2341 bytes) -2016-05-23 16:40:50 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 30) -2016-05-23 16:40:50 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 31) -2016-05-23 16:40:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-05-23 16:40:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-05-23 16:40:50 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-05-23 16:40:50 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 31). 2254 bytes result sent to driver -2016-05-23 16:40:50 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 30). 2254 bytes result sent to driver -2016-05-23 16:40:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 31) in 102 ms on localhost (1/2) -2016-05-23 16:40:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 30) in 105 ms on localhost (2/2) -2016-05-23 16:40:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-23 16:40:50 INFO DAGScheduler:59 - ShuffleMapStage 15 (flatMap at KMeans.scala:446) finished in 0.106 s -2016-05-23 16:40:50 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 16:40:50 INFO DAGScheduler:59 - running: Set() -2016-05-23 16:40:50 INFO DAGScheduler:59 - waiting: Set(ResultStage 16) -2016-05-23 16:40:50 INFO DAGScheduler:59 - failed: Set() -2016-05-23 16:40:50 INFO DAGScheduler:59 - Missing parents for ResultStage 16: List() -2016-05-23 16:40:50 INFO DAGScheduler:59 - Submitting ResultStage 16 (ShuffledRDD[29] at reduceByKey at KMeans.scala:450), which is now runnable -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=215643, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 16:40:50 INFO MemoryStore:59 - ensureFreeSpace(1388) called with curMem=217955, maxMem=766075207 -2016-05-23 16:40:50 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 1388.0 B, free 730.4 MB) -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:53178 (size: 1388.0 B, free: 730.6 MB) -2016-05-23 16:40:50 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 16:40:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (ShuffledRDD[29] at reduceByKey at KMeans.scala:450) -2016-05-23 16:40:50 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 16:40:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 16:40:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 33, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 16:40:50 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 32) -2016-05-23 16:40:50 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 33) -2016-05-23 16:40:50 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 16:40:50 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 16:40:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 17 ms -2016-05-23 16:40:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 17 ms -2016-05-23 16:40:50 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 32). 2358 bytes result sent to driver -2016-05-23 16:40:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 32) in 112 ms on localhost (1/2) -2016-05-23 16:40:50 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 33). 2505 bytes result sent to driver -2016-05-23 16:40:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 33) in 113 ms on localhost (2/2) -2016-05-23 16:40:50 INFO DAGScheduler:59 - ResultStage 16 (collectAsMap at KMeans.scala:450) finished in 0.115 s -2016-05-23 16:40:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 16:40:50 INFO DAGScheduler:59 - Job 15 finished: collectAsMap at KMeans.scala:450, took 0.278953 s -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:53178 in memory (size: 1388.0 B, free: 730.6 MB) -2016-05-23 16:40:50 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:53178 in memory (size: 3.0 KB, free: 730.6 MB) -2016-05-23 16:40:50 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-23 16:40:50 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-23 16:40:50 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:53178 in memory (size: 3.1 KB, free: 730.6 MB) -2016-05-23 16:40:50 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-23 16:40:51 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-05-23 16:40:51 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-05-23 16:40:51 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-05-23 16:40:51 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-05-23 16:40:51 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-05-23 16:40:51 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-05-23 16:40:51 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-05-23 16:40:51 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-05-23 16:40:51 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-05-23 16:40:51 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-05-23 16:40:51 INFO KMeans:59 - Initialization with k-means|| took 2.528 seconds. -2016-05-23 16:40:51 INFO MemoryStore:59 - ensureFreeSpace(1920) called with curMem=196987, maxMem=766075207 -2016-05-23 16:40:51 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 1920.0 B, free 730.4 MB) -2016-05-23 16:40:51 INFO MemoryStore:59 - ensureFreeSpace(401) called with curMem=198907, maxMem=766075207 -2016-05-23 16:40:51 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 401.0 B, free 730.4 MB) -2016-05-23 16:40:51 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:53178 (size: 401.0 B, free: 730.6 MB) -2016-05-23 16:40:51 INFO SparkContext:59 - Created broadcast 24 from broadcast at KMeans.scala:276 -2016-05-23 16:40:51 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:302 -2016-05-23 16:40:51 INFO DAGScheduler:59 - Registering RDD 30 (mapPartitions at KMeans.scala:279) -2016-05-23 16:40:51 INFO DAGScheduler:59 - Got job 16 (collectAsMap at KMeans.scala:302) with 2 output partitions -2016-05-23 16:40:51 INFO DAGScheduler:59 - Final stage: ResultStage 18(collectAsMap at KMeans.scala:302) -2016-05-23 16:40:51 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 17) -2016-05-23 16:40:51 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 17) -2016-05-23 16:40:51 INFO DAGScheduler:59 - Submitting ShuffleMapStage 17 (MapPartitionsRDD[30] at mapPartitions at KMeans.scala:279), which has no missing parents -2016-05-23 16:40:51 INFO MemoryStore:59 - ensureFreeSpace(6640) called with curMem=199308, maxMem=766075207 -2016-05-23 16:40:51 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 6.5 KB, free 730.4 MB) -2016-05-23 16:40:51 INFO MemoryStore:59 - ensureFreeSpace(3455) called with curMem=205948, maxMem=766075207 -2016-05-23 16:40:51 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 16:40:51 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:53178 (size: 3.4 KB, free: 730.6 MB) -2016-05-23 16:40:51 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 16:40:51 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 17 (MapPartitionsRDD[30] at mapPartitions at KMeans.scala:279) -2016-05-23 16:40:51 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-23 16:40:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2341 bytes) -2016-05-23 16:40:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 35, localhost, PROCESS_LOCAL, 2341 bytes) -2016-05-23 16:40:51 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 35) -2016-05-23 16:40:51 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 34) -2016-05-23 16:40:51 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-05-23 16:40:51 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-05-23 16:40:51 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-05-23 16:40:51 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-05-23 16:40:51 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 34). 2533 bytes result sent to driver -2016-05-23 16:40:51 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 35). 2533 bytes result sent to driver -2016-05-23 16:40:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 34) in 51 ms on localhost (1/2) -2016-05-23 16:40:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 35) in 51 ms on localhost (2/2) -2016-05-23 16:40:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 16:40:51 INFO DAGScheduler:59 - ShuffleMapStage 17 (mapPartitions at KMeans.scala:279) finished in 0.052 s -2016-05-23 16:40:51 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 16:40:51 INFO DAGScheduler:59 - running: Set() -2016-05-23 16:40:51 INFO DAGScheduler:59 - waiting: Set(ResultStage 18) -2016-05-23 16:40:51 INFO DAGScheduler:59 - failed: Set() -2016-05-23 16:40:51 INFO DAGScheduler:59 - Missing parents for ResultStage 18: List() -2016-05-23 16:40:51 INFO DAGScheduler:59 - Submitting ResultStage 18 (ShuffledRDD[31] at reduceByKey at KMeans.scala:302), which is now runnable -2016-05-23 16:40:51 INFO MemoryStore:59 - ensureFreeSpace(2808) called with curMem=209403, maxMem=766075207 -2016-05-23 16:40:51 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 2.7 KB, free 730.4 MB) -2016-05-23 16:40:51 INFO MemoryStore:59 - ensureFreeSpace(1593) called with curMem=212211, maxMem=766075207 -2016-05-23 16:40:51 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 1593.0 B, free 730.4 MB) -2016-05-23 16:40:51 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:53178 (size: 1593.0 B, free: 730.6 MB) -2016-05-23 16:40:51 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 16:40:51 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (ShuffledRDD[31] at reduceByKey at KMeans.scala:302) -2016-05-23 16:40:51 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-23 16:40:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 16:40:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 37, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 16:40:51 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 37) -2016-05-23 16:40:51 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 36) -2016-05-23 16:40:51 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 16:40:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 16:40:51 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 16:40:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 8 ms -2016-05-23 16:40:51 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 36). 2312 bytes result sent to driver -2016-05-23 16:40:51 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 37). 2018 bytes result sent to driver -2016-05-23 16:40:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 36) in 27 ms on localhost (1/2) -2016-05-23 16:40:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 37) in 29 ms on localhost (2/2) -2016-05-23 16:40:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 16:40:51 INFO DAGScheduler:59 - ResultStage 18 (collectAsMap at KMeans.scala:302) finished in 0.031 s -2016-05-23 16:40:51 INFO DAGScheduler:59 - Job 16 finished: collectAsMap at KMeans.scala:302, took 0.108033 s -2016-05-23 16:40:51 INFO KMeans:59 - Run 0 finished in 1 iterations -2016-05-23 16:40:51 INFO KMeans:59 - Run 1 finished in 1 iterations -2016-05-23 16:40:51 INFO KMeans:59 - Run 2 finished in 1 iterations -2016-05-23 16:40:51 INFO KMeans:59 - Run 3 finished in 1 iterations -2016-05-23 16:40:51 INFO KMeans:59 - Run 4 finished in 1 iterations -2016-05-23 16:40:51 INFO KMeans:59 - Run 5 finished in 1 iterations -2016-05-23 16:40:51 INFO KMeans:59 - Run 6 finished in 1 iterations -2016-05-23 16:40:51 INFO KMeans:59 - Run 7 finished in 1 iterations -2016-05-23 16:40:51 INFO KMeans:59 - Run 8 finished in 1 iterations -2016-05-23 16:40:51 INFO KMeans:59 - Run 9 finished in 1 iterations -2016-05-23 16:40:51 INFO KMeans:59 - Iterations took 0.169 seconds. -2016-05-23 16:40:51 INFO KMeans:59 - KMeans converged in 1 iterations. -2016-05-23 16:40:51 INFO KMeans:59 - The cost for the best run is 0.11999999999994547. -2016-05-23 16:40:51 INFO MapPartitionsRDD:59 - Removing RDD 3 from persistence list -2016-05-23 16:40:51 INFO BlockManager:59 - Removing RDD 3 -2016-05-23 16:40:51 WARN KMeans:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-05-23 16:40:51 INFO MemoryStore:59 - ensureFreeSpace(232) called with curMem=213604, maxMem=766075207 -2016-05-23 16:40:51 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 232.0 B, free 730.4 MB) -2016-05-23 16:40:51 INFO MemoryStore:59 - ensureFreeSpace(395) called with curMem=213836, maxMem=766075207 -2016-05-23 16:40:51 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 395.0 B, free 730.4 MB) -2016-05-23 16:40:51 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:53178 (size: 395.0 B, free: 730.6 MB) -2016-05-23 16:40:51 INFO SparkContext:59 - Created broadcast 27 from broadcast at KMeansModel.scala:87 -2016-05-23 16:40:51 INFO SparkContext:59 - Starting job: sum at KMeansModel.scala:88 -2016-05-23 16:40:51 INFO DAGScheduler:59 - Got job 17 (sum at KMeansModel.scala:88) with 2 output partitions -2016-05-23 16:40:51 INFO DAGScheduler:59 - Final stage: ResultStage 19(sum at KMeansModel.scala:88) -2016-05-23 16:40:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 16:40:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 16:40:51 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[32] at map at KMeansModel.scala:88), which has no missing parents -2016-05-23 16:40:51 INFO MemoryStore:59 - ensureFreeSpace(3968) called with curMem=214231, maxMem=766075207 -2016-05-23 16:40:51 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-05-23 16:40:51 INFO MemoryStore:59 - ensureFreeSpace(2327) called with curMem=218199, maxMem=766075207 -2016-05-23 16:40:51 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 16:40:51 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:53178 (size: 2.3 KB, free: 730.6 MB) -2016-05-23 16:40:51 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 16:40:51 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[32] at map at KMeansModel.scala:88) -2016-05-23 16:40:51 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-23 16:40:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 16:40:51 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:53178 in memory (size: 1593.0 B, free: 730.6 MB) -2016-05-23 16:40:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 39, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 16:40:51 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 39) -2016-05-23 16:40:51 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 38) -2016-05-23 16:40:51 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-05-23 16:40:51 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-05-23 16:40:51 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 39). 2084 bytes result sent to driver -2016-05-23 16:40:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 39) in 15 ms on localhost (1/2) -2016-05-23 16:40:51 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 38). 2084 bytes result sent to driver -2016-05-23 16:40:51 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-23 16:40:51 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:53178 in memory (size: 3.4 KB, free: 730.6 MB) -2016-05-23 16:40:51 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-23 16:40:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 38) in 23 ms on localhost (2/2) -2016-05-23 16:40:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 16:40:51 INFO DAGScheduler:59 - ResultStage 19 (sum at KMeansModel.scala:88) finished in 0.025 s -2016-05-23 16:40:51 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-23 16:40:51 INFO DAGScheduler:59 - Job 17 finished: sum at KMeansModel.scala:88, took 0.045975 s -2016-05-23 16:40:51 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:53178 in memory (size: 401.0 B, free: 730.6 MB) -2016-05-23 16:40:51 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-23 16:40:51 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-23 16:40:51 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-23 16:40:51 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-23 16:40:51 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-23 16:40:51 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-23 16:40:51 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-23 16:40:51 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-23 16:40:51 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-23 16:40:51 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-23 16:40:51 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 16:40:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 16:40:51 INFO SparkUI:59 - Stopped Spark web UI at http://219.219.220.162:4040 -2016-05-23 16:40:51 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 16:40:51 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 16:40:51 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 16:40:51 INFO BlockManager:59 - BlockManager stopped -2016-05-23 16:40:51 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 16:40:51 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 16:40:51 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 16:40:51 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 16:40:51 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-c0e16cca-7bf4-42ff-98d5-c91314a0b23a -2016-05-23 16:41:42 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 16:41:44 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 16:41:44 INFO Remoting:74 - Starting remoting -2016-05-23 16:41:44 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@219.219.220.162:53203] -2016-05-23 16:41:45 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 16:41:45 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53207 -2016-05-23 16:41:45 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 16:41:45 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 16:41:45 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 16:41:48 WARN KMeans:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-05-23 16:41:49 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: 2002:dbdb:dca2:0:0:0:dbdb:dca2, but we couldn't find any external IP address! -2016-05-23 16:41:49 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 16:41:49 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 16:41:49 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 16:41:49 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 16:41:49 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 16:41:49 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 16:41:50 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 16:41:50 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 16:41:51 WARN KMeans:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-05-23 16:41:53 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605231641_0018_m_000000_18' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/kmeans_data/20160523164152082/1/_temporary/0/task_201605231641_0018_m_000000 -2016-05-23 16:41:54 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605231641_0019_m_000000_19' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/kmeans_data/20160523164152082/2/_temporary/0/task_201605231641_0019_m_000000 -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 16:41:54 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 17:51:43 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 17:51:44 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 17:51:44 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 17:51:44 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 17:51:44 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 17:51:45 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 17:51:45 INFO Remoting:74 - Starting remoting -2016-05-23 17:51:45 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:56824] -2016-05-23 17:51:45 INFO Utils:59 - Successfully started service 'sparkDriver' on port 56824. -2016-05-23 17:51:45 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 17:51:45 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 17:51:45 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-f8d0dd59-f97f-410d-8756-4b9db5044b7c -2016-05-23 17:51:45 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 17:51:45 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-7b22e3de-4477-4c27-a14b-53dedef2bef3\httpd-b049405a-2df6-4376-b586-c0a877605f23 -2016-05-23 17:51:45 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 17:51:45 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 17:51:45 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:56825 -2016-05-23 17:51:45 INFO Utils:59 - Successfully started service 'HTTP file server' on port 56825. -2016-05-23 17:51:45 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 17:51:46 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 17:51:46 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 17:51:46 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 17:51:46 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 17:51:46 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 17:51:46 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 17:51:46 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 56832. -2016-05-23 17:51:46 INFO NettyBlockTransferService:59 - Server created on 56832 -2016-05-23 17:51:46 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 17:51:46 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:56832 with 730.6 MB RAM, BlockManagerId(driver, localhost, 56832) -2016-05-23 17:51:46 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 17:51:47 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 17:51:47 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 17:51:47 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 17:51:47 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 17:51:47 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:56832 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 17:51:47 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-23 17:51:50 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 17:51:50 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 17:51:50 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-23 17:51:50 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 1 output partitions -2016-05-23 17:51:50 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-23 17:51:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 17:51:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 17:51:50 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-23 17:51:50 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-23 17:51:50 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-23 17:51:50 INFO MemoryStore:59 - ensureFreeSpace(2138) called with curMem=175476, maxMem=766075207 -2016-05-23 17:51:50 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-23 17:51:50 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:56832 (size: 2.1 KB, free: 730.6 MB) -2016-05-23 17:51:50 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 17:51:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-23 17:51:50 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-23 17:51:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 17:51:50 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 17:51:50 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-23 17:51:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/sample_libsvm_data.txt:0+104736 -2016-05-23 17:51:50 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 17:51:50 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 17:51:50 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 17:51:50 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 17:51:50 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 17:51:51 INFO MemoryStore:59 - ensureFreeSpace(170712) called with curMem=177614, maxMem=766075207 -2016-05-23 17:51:51 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 166.7 KB, free 730.3 MB) -2016-05-23 17:51:51 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:56832 (size: 166.7 KB, free: 730.4 MB) -2016-05-23 17:51:51 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-23 17:51:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 557 ms on localhost (1/1) -2016-05-23 17:51:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 17:51:51 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.585 s -2016-05-23 17:51:51 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.807988 s -2016-05-23 17:51:51 INFO SparkContext:59 - Starting job: foreach at LabeledPointLearning.scala:39 -2016-05-23 17:51:51 INFO DAGScheduler:59 - Got job 1 (foreach at LabeledPointLearning.scala:39) with 1 output partitions -2016-05-23 17:51:51 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at LabeledPointLearning.scala:39) -2016-05-23 17:51:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 17:51:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 17:51:51 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-23 17:51:51 INFO MemoryStore:59 - ensureFreeSpace(3936) called with curMem=348326, maxMem=766075207 -2016-05-23 17:51:51 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-05-23 17:51:51 INFO MemoryStore:59 - ensureFreeSpace(2183) called with curMem=352262, maxMem=766075207 -2016-05-23 17:51:51 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.2 MB) -2016-05-23 17:51:51 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:56832 (size: 2.1 KB, free: 730.4 MB) -2016-05-23 17:51:51 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 17:51:51 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-23 17:51:51 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-23 17:51:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 17:51:51 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-23 17:51:51 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-23 17:51:51 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 17:51:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 115 ms on localhost (1/1) -2016-05-23 17:51:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 17:51:51 INFO DAGScheduler:59 - ResultStage 1 (foreach at LabeledPointLearning.scala:39) finished in 0.115 s -2016-05-23 17:51:51 INFO DAGScheduler:59 - Job 1 finished: foreach at LabeledPointLearning.scala:39, took 0.148958 s -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 17:51:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 17:51:51 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 17:51:51 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 17:51:51 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 17:51:51 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 17:51:51 INFO BlockManager:59 - BlockManager stopped -2016-05-23 17:51:51 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 17:51:51 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 17:51:51 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 17:51:51 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 17:51:51 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-7b22e3de-4477-4c27-a14b-53dedef2bef3 -2016-05-23 17:51:51 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 17:51:51 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 17:53:04 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 17:53:04 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 17:53:05 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 17:53:05 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 17:53:05 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 17:53:05 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 17:53:06 INFO Remoting:74 - Starting remoting -2016-05-23 17:53:06 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:56859] -2016-05-23 17:53:06 INFO Utils:59 - Successfully started service 'sparkDriver' on port 56859. -2016-05-23 17:53:06 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 17:53:06 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 17:53:06 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-13a88eec-32dc-4017-b968-66cce3154f78 -2016-05-23 17:53:06 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 17:53:06 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-2ec96de6-47c6-43ec-ba6d-41c909d1b6e4\httpd-883b96cc-0570-42e6-b063-4395deff1a41 -2016-05-23 17:53:06 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 17:53:06 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 17:53:06 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:56860 -2016-05-23 17:53:06 INFO Utils:59 - Successfully started service 'HTTP file server' on port 56860. -2016-05-23 17:53:06 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 17:53:06 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 17:53:06 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 17:53:06 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 17:53:06 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 17:53:06 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 17:53:06 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 17:53:07 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 56867. -2016-05-23 17:53:07 INFO NettyBlockTransferService:59 - Server created on 56867 -2016-05-23 17:53:07 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 17:53:07 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:56867 with 730.6 MB RAM, BlockManagerId(driver, localhost, 56867) -2016-05-23 17:53:07 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 17:53:08 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 17:53:08 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 17:53:08 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 17:53:08 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 17:53:08 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:56867 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 17:53:08 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-23 17:53:09 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 17:53:09 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 17:53:09 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-23 17:53:09 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 1 output partitions -2016-05-23 17:53:09 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-23 17:53:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 17:53:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 17:53:09 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-23 17:53:09 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-23 17:53:09 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-23 17:53:09 INFO MemoryStore:59 - ensureFreeSpace(2140) called with curMem=175476, maxMem=766075207 -2016-05-23 17:53:09 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-23 17:53:09 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:56867 (size: 2.1 KB, free: 730.6 MB) -2016-05-23 17:53:09 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 17:53:09 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-23 17:53:09 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-23 17:53:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 17:53:09 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 17:53:09 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-23 17:53:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/sample_libsvm_data.txt:0+104736 -2016-05-23 17:53:09 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 17:53:09 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 17:53:09 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 17:53:09 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 17:53:09 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 17:53:10 INFO MemoryStore:59 - ensureFreeSpace(170712) called with curMem=177616, maxMem=766075207 -2016-05-23 17:53:10 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 166.7 KB, free 730.3 MB) -2016-05-23 17:53:10 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:56867 (size: 166.7 KB, free: 730.4 MB) -2016-05-23 17:53:10 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-23 17:53:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 488 ms on localhost (1/1) -2016-05-23 17:53:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 17:53:10 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.513 s -2016-05-23 17:53:10 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.698086 s -2016-05-23 17:53:10 INFO SparkContext:59 - Starting job: foreach at LabeledPointLoadlibSVMFile.scala:17 -2016-05-23 17:53:10 INFO DAGScheduler:59 - Got job 1 (foreach at LabeledPointLoadlibSVMFile.scala:17) with 1 output partitions -2016-05-23 17:53:10 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at LabeledPointLoadlibSVMFile.scala:17) -2016-05-23 17:53:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 17:53:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 17:53:10 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-23 17:53:10 INFO MemoryStore:59 - ensureFreeSpace(3936) called with curMem=348328, maxMem=766075207 -2016-05-23 17:53:10 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-05-23 17:53:10 INFO MemoryStore:59 - ensureFreeSpace(2185) called with curMem=352264, maxMem=766075207 -2016-05-23 17:53:10 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.2 MB) -2016-05-23 17:53:10 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:56867 (size: 2.1 KB, free: 730.4 MB) -2016-05-23 17:53:10 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 17:53:10 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-23 17:53:10 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-23 17:53:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 17:53:10 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-23 17:53:10 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-23 17:53:10 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 17:53:10 INFO DAGScheduler:59 - ResultStage 1 (foreach at LabeledPointLoadlibSVMFile.scala:17) finished in 0.110 s -2016-05-23 17:53:10 INFO DAGScheduler:59 - Job 1 finished: foreach at LabeledPointLoadlibSVMFile.scala:17, took 0.133222 s -2016-05-23 17:53:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 110 ms on localhost (1/1) -2016-05-23 17:53:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 17:53:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 17:53:10 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 17:53:10 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 17:53:10 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 17:53:10 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 17:53:10 INFO BlockManager:59 - BlockManager stopped -2016-05-23 17:53:10 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 17:53:10 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 17:53:10 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 17:53:10 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 17:53:10 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-2ec96de6-47c6-43ec-ba6d-41c909d1b6e4 -2016-05-23 17:53:10 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 17:53:10 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 18:01:36 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 18:01:37 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 18:01:37 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 18:01:37 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 18:01:37 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 18:01:38 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 18:01:38 INFO Remoting:74 - Starting remoting -2016-05-23 18:01:38 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:57237] -2016-05-23 18:01:38 INFO Utils:59 - Successfully started service 'sparkDriver' on port 57237. -2016-05-23 18:01:38 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 18:01:38 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 18:01:38 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-b6717118-802b-4fec-ab62-7306ffbcbb69 -2016-05-23 18:01:38 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 18:01:38 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-c85fdb71-c5dc-40bb-84da-e8beda42d3bd\httpd-c674a8cb-f9fa-4bb3-9d1e-b383fb31c39e -2016-05-23 18:01:38 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 18:01:38 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:01:38 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:57238 -2016-05-23 18:01:38 INFO Utils:59 - Successfully started service 'HTTP file server' on port 57238. -2016-05-23 18:01:38 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 18:01:39 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:01:39 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 18:01:39 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 18:01:39 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 18:01:39 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 18:01:39 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 18:01:39 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 57245. -2016-05-23 18:01:39 INFO NettyBlockTransferService:59 - Server created on 57245 -2016-05-23 18:01:39 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 18:01:39 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:57245 with 730.6 MB RAM, BlockManagerId(driver, localhost, 57245) -2016-05-23 18:01:39 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 18:01:40 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 18:01:40 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 18:01:40 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 18:01:40 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 18:01:40 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:57245 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 18:01:40 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-23 18:01:41 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 18:01:41 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:01:41 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-23 18:01:41 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 1 output partitions -2016-05-23 18:01:41 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-23 18:01:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:01:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:01:41 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-23 18:01:42 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-23 18:01:42 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-23 18:01:42 INFO MemoryStore:59 - ensureFreeSpace(2139) called with curMem=175476, maxMem=766075207 -2016-05-23 18:01:42 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-23 18:01:42 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:57245 (size: 2.1 KB, free: 730.6 MB) -2016-05-23 18:01:42 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:01:42 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-23 18:01:42 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-23 18:01:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 18:01:42 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 18:01:42 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-23 18:01:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/sample_libsvm_data.txt:0+104736 -2016-05-23 18:01:42 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 18:01:42 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 18:01:42 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 18:01:42 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 18:01:42 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 18:01:42 INFO MemoryStore:59 - ensureFreeSpace(170712) called with curMem=177615, maxMem=766075207 -2016-05-23 18:01:42 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 166.7 KB, free 730.3 MB) -2016-05-23 18:01:42 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:57245 (size: 166.7 KB, free: 730.4 MB) -2016-05-23 18:01:42 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-23 18:01:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 551 ms on localhost (1/1) -2016-05-23 18:01:42 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.576 s -2016-05-23 18:01:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 18:01:42 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.755451 s -2016-05-23 18:01:42 INFO SparkContext:59 - Starting job: foreach at LabeledPointLoadlibSVMFile.scala:24 -2016-05-23 18:01:42 INFO DAGScheduler:59 - Got job 1 (foreach at LabeledPointLoadlibSVMFile.scala:24) with 1 output partitions -2016-05-23 18:01:42 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at LabeledPointLoadlibSVMFile.scala:24) -2016-05-23 18:01:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:01:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:01:42 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-23 18:01:42 INFO MemoryStore:59 - ensureFreeSpace(3936) called with curMem=348327, maxMem=766075207 -2016-05-23 18:01:42 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-05-23 18:01:42 INFO MemoryStore:59 - ensureFreeSpace(2189) called with curMem=352263, maxMem=766075207 -2016-05-23 18:01:42 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.2 MB) -2016-05-23 18:01:42 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:57245 (size: 2.1 KB, free: 730.4 MB) -2016-05-23 18:01:42 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:01:42 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-23 18:01:42 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-23 18:01:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 18:01:42 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-23 18:01:42 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-23 18:01:42 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 18:01:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 125 ms on localhost (1/1) -2016-05-23 18:01:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 18:01:42 INFO DAGScheduler:59 - ResultStage 1 (foreach at LabeledPointLoadlibSVMFile.scala:24) finished in 0.126 s -2016-05-23 18:01:42 INFO DAGScheduler:59 - Job 1 finished: foreach at LabeledPointLoadlibSVMFile.scala:24, took 0.148749 s -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 18:01:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 18:01:42 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 18:01:42 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 18:01:43 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 18:01:43 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 18:01:43 INFO BlockManager:59 - BlockManager stopped -2016-05-23 18:01:43 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 18:01:43 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 18:01:43 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 18:01:43 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 18:01:43 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-c85fdb71-c5dc-40bb-84da-e8beda42d3bd -2016-05-23 18:01:43 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 18:01:43 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 18:29:29 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 18:29:29 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 18:29:30 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 18:29:30 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 18:29:30 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 18:29:30 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 18:29:30 INFO Remoting:74 - Starting remoting -2016-05-23 18:29:31 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:58315] -2016-05-23 18:29:31 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58315. -2016-05-23 18:29:31 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 18:29:31 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 18:29:31 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-c9062d0e-a3ef-4003-9448-0d9239e650a0 -2016-05-23 18:29:31 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 18:29:31 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-2aba8191-733f-418b-8c12-412063081de6\httpd-40c0718e-cbf5-4b82-81e5-9ae6a281e9bc -2016-05-23 18:29:31 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 18:29:31 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:29:31 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58316 -2016-05-23 18:29:31 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58316. -2016-05-23 18:29:31 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 18:29:31 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:29:31 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 18:29:31 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 18:29:31 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 18:29:31 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 18:29:31 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 18:29:32 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58323. -2016-05-23 18:29:32 INFO NettyBlockTransferService:59 - Server created on 58323 -2016-05-23 18:29:32 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 18:29:32 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:58323 with 730.6 MB RAM, BlockManagerId(driver, localhost, 58323) -2016-05-23 18:29:32 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 18:29:32 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 18:29:32 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 18:29:32 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 18:29:32 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 18:29:32 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:58323 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 18:29:32 INFO SparkContext:59 - Created broadcast 0 from textFile at MatrixRowLearning.scala:21 -2016-05-23 18:29:34 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 18:29:34 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:29:34 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-05-23 18:29:34 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 1 output partitions -2016-05-23 18:29:34 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-05-23 18:29:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:29:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:29:34 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24), which has no missing parents -2016-05-23 18:29:34 INFO MemoryStore:59 - ensureFreeSpace(3360) called with curMem=171580, maxMem=766075207 -2016-05-23 18:29:34 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-05-23 18:29:34 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=174940, maxMem=766075207 -2016-05-23 18:29:34 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.4 MB) -2016-05-23 18:29:34 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:58323 (size: 1935.0 B, free: 730.6 MB) -2016-05-23 18:29:34 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:29:34 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24) -2016-05-23 18:29:34 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-23 18:29:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:29:34 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 18:29:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:29:34 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 18:29:34 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 18:29:34 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 18:29:34 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 18:29:34 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 18:29:34 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-05-23 18:29:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 103 ms on localhost (1/1) -2016-05-23 18:29:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 18:29:34 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.125 s -2016-05-23 18:29:34 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.233461 s -2016-05-23 18:29:34 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 18:29:34 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 18:29:34 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-05-23 18:29:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:29:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:29:34 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24), which has no missing parents -2016-05-23 18:29:34 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=176875, maxMem=766075207 -2016-05-23 18:29:34 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 18:29:34 INFO MemoryStore:59 - ensureFreeSpace(1993) called with curMem=180403, maxMem=766075207 -2016-05-23 18:29:34 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1993.0 B, free 730.4 MB) -2016-05-23 18:29:34 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:58323 (size: 1993.0 B, free: 730.6 MB) -2016-05-23 18:29:34 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:29:34 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24) -2016-05-23 18:29:34 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-23 18:29:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:29:34 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-23 18:29:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:29:34 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2183 bytes result sent to driver -2016-05-23 18:29:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 18 ms on localhost (1/1) -2016-05-23 18:29:34 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.018 s -2016-05-23 18:29:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 18:29:34 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.090976 s -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 18:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 18:29:35 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 18:29:35 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 18:29:35 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 18:29:35 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 18:29:35 INFO BlockManager:59 - BlockManager stopped -2016-05-23 18:29:35 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 18:29:35 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 18:29:35 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 18:29:35 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 18:29:35 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-2aba8191-733f-418b-8c12-412063081de6 -2016-05-23 18:29:35 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 18:29:35 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 18:30:39 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 18:30:40 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 18:30:40 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 18:30:40 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 18:30:40 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 18:30:41 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 18:30:41 INFO Remoting:74 - Starting remoting -2016-05-23 18:30:41 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:58347] -2016-05-23 18:30:41 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58347. -2016-05-23 18:30:41 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 18:30:41 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 18:30:41 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-295e392b-9374-4173-93ac-d0049dfa12dc -2016-05-23 18:30:41 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 18:30:41 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-f4b2faa6-0f99-4274-b9ee-8808b1b9c8a9\httpd-c6c87b40-7f8a-4ee3-b0f5-cf0456cb528b -2016-05-23 18:30:41 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 18:30:41 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:30:41 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58348 -2016-05-23 18:30:41 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58348. -2016-05-23 18:30:42 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 18:30:42 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:30:42 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 18:30:42 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 18:30:42 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 18:30:42 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 18:30:42 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 18:30:42 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58355. -2016-05-23 18:30:42 INFO NettyBlockTransferService:59 - Server created on 58355 -2016-05-23 18:30:42 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 18:30:42 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:58355 with 730.6 MB RAM, BlockManagerId(driver, localhost, 58355) -2016-05-23 18:30:42 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 18:30:43 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 18:30:43 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 18:30:43 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 18:30:43 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 18:30:43 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:58355 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 18:30:43 INFO SparkContext:59 - Created broadcast 0 from textFile at MatrixRowLearning.scala:21 -2016-05-23 18:30:44 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 18:30:45 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:30:45 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-05-23 18:30:45 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 1 output partitions -2016-05-23 18:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-05-23 18:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:30:45 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24), which has no missing parents -2016-05-23 18:30:45 INFO MemoryStore:59 - ensureFreeSpace(3360) called with curMem=171580, maxMem=766075207 -2016-05-23 18:30:45 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-05-23 18:30:45 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=174940, maxMem=766075207 -2016-05-23 18:30:45 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.4 MB) -2016-05-23 18:30:45 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:58355 (size: 1935.0 B, free: 730.6 MB) -2016-05-23 18:30:45 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:30:45 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24) -2016-05-23 18:30:45 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-23 18:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:30:45 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 18:30:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:30:45 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 18:30:45 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 18:30:45 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 18:30:45 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 18:30:45 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 18:30:45 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-05-23 18:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 120 ms on localhost (1/1) -2016-05-23 18:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 18:30:45 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.154 s -2016-05-23 18:30:45 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.661570 s -2016-05-23 18:30:45 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 18:30:45 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 18:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-05-23 18:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:30:45 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24), which has no missing parents -2016-05-23 18:30:45 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=176875, maxMem=766075207 -2016-05-23 18:30:45 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 18:30:45 INFO MemoryStore:59 - ensureFreeSpace(1993) called with curMem=180403, maxMem=766075207 -2016-05-23 18:30:45 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1993.0 B, free 730.4 MB) -2016-05-23 18:30:45 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:58355 (size: 1993.0 B, free: 730.6 MB) -2016-05-23 18:30:45 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:30:45 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24) -2016-05-23 18:30:45 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-23 18:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:30:45 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-23 18:30:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:30:45 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2183 bytes result sent to driver -2016-05-23 18:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 29 ms on localhost (1/1) -2016-05-23 18:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 18:30:45 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.034 s -2016-05-23 18:30:45 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.061224 s -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 18:30:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 18:30:45 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 18:30:45 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 18:30:45 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 18:30:45 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 18:30:45 INFO BlockManager:59 - BlockManager stopped -2016-05-23 18:30:45 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 18:30:45 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 18:30:46 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 18:30:46 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 18:30:46 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-f4b2faa6-0f99-4274-b9ee-8808b1b9c8a9 -2016-05-23 18:30:46 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 18:30:46 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 18:33:34 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 18:33:35 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 18:33:35 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 18:33:35 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 18:33:35 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 18:33:36 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 18:33:36 INFO Remoting:74 - Starting remoting -2016-05-23 18:33:36 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:58391] -2016-05-23 18:33:36 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58391. -2016-05-23 18:33:36 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 18:33:36 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 18:33:36 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-b2c0db8e-c40b-425d-82b7-b56660ff6b27 -2016-05-23 18:33:36 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 18:33:37 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-e21fd1a7-8b52-4c9b-bbd5-4b50a5f4500a\httpd-b5b0a3e1-6be8-4f23-8510-cb58f1608384 -2016-05-23 18:33:37 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 18:33:37 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:33:37 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58392 -2016-05-23 18:33:37 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58392. -2016-05-23 18:33:37 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 18:33:37 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:33:37 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 18:33:37 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 18:33:37 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 18:33:37 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 18:33:37 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 18:33:37 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58399. -2016-05-23 18:33:37 INFO NettyBlockTransferService:59 - Server created on 58399 -2016-05-23 18:33:37 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 18:33:37 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:58399 with 730.6 MB RAM, BlockManagerId(driver, localhost, 58399) -2016-05-23 18:33:37 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 18:33:38 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 18:33:38 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 18:33:38 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 18:33:38 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 18:33:38 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:58399 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 18:33:38 INFO SparkContext:59 - Created broadcast 0 from textFile at MatrixRowLearning.scala:21 -2016-05-23 18:33:39 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 18:33:40 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:33:40 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-05-23 18:33:40 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 1 output partitions -2016-05-23 18:33:40 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-05-23 18:33:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:33:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:33:40 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24), which has no missing parents -2016-05-23 18:33:40 INFO MemoryStore:59 - ensureFreeSpace(3360) called with curMem=171580, maxMem=766075207 -2016-05-23 18:33:40 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-05-23 18:33:40 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=174940, maxMem=766075207 -2016-05-23 18:33:40 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.4 MB) -2016-05-23 18:33:40 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:58399 (size: 1935.0 B, free: 730.6 MB) -2016-05-23 18:33:40 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:33:40 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24) -2016-05-23 18:33:40 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-23 18:33:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:33:40 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 18:33:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:33:40 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 18:33:40 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 18:33:40 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 18:33:40 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 18:33:40 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 18:33:40 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-05-23 18:33:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 122 ms on localhost (1/1) -2016-05-23 18:33:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 18:33:40 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.148 s -2016-05-23 18:33:40 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.309376 s -2016-05-23 18:33:40 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 18:33:40 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 18:33:40 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-05-23 18:33:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:33:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:33:40 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24), which has no missing parents -2016-05-23 18:33:40 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=176875, maxMem=766075207 -2016-05-23 18:33:40 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 18:33:40 INFO MemoryStore:59 - ensureFreeSpace(1993) called with curMem=180403, maxMem=766075207 -2016-05-23 18:33:40 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1993.0 B, free 730.4 MB) -2016-05-23 18:33:40 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:58399 (size: 1993.0 B, free: 730.6 MB) -2016-05-23 18:33:40 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:33:40 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24) -2016-05-23 18:33:40 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-23 18:33:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:33:40 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-23 18:33:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:33:40 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2183 bytes result sent to driver -2016-05-23 18:33:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 31 ms on localhost (1/1) -2016-05-23 18:33:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 18:33:40 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.031 s -2016-05-23 18:33:40 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.056225 s -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 18:33:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 18:33:40 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 18:33:40 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 18:33:40 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 18:33:40 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 18:33:40 INFO BlockManager:59 - BlockManager stopped -2016-05-23 18:33:40 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 18:33:40 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 18:33:40 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 18:33:40 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 18:33:40 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-e21fd1a7-8b52-4c9b-bbd5-4b50a5f4500a -2016-05-23 18:49:53 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 18:49:53 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 18:49:54 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 18:49:54 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 18:49:54 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 18:49:55 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 18:49:55 INFO Remoting:74 - Starting remoting -2016-05-23 18:49:55 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:58738] -2016-05-23 18:49:55 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58738. -2016-05-23 18:49:55 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 18:49:55 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 18:49:55 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-b2439ec4-45b3-433a-94ef-0e85c56e1351 -2016-05-23 18:49:55 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 18:49:55 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-35ec1ca2-e0cd-4854-a5e5-61f1ef058220\httpd-0c2c57bb-f4b5-4cf6-aa80-be1c4ec68c50 -2016-05-23 18:49:55 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 18:49:55 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:49:55 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58739 -2016-05-23 18:49:55 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58739. -2016-05-23 18:49:55 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 18:49:55 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:49:55 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 18:49:55 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 18:49:55 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 18:49:56 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 18:49:56 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 18:49:56 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58746. -2016-05-23 18:49:56 INFO NettyBlockTransferService:59 - Server created on 58746 -2016-05-23 18:49:56 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 18:49:56 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:58746 with 730.6 MB RAM, BlockManagerId(driver, localhost, 58746) -2016-05-23 18:49:56 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 18:49:57 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 18:49:57 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 18:49:57 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 18:49:57 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 18:49:57 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:58746 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 18:49:57 INFO SparkContext:59 - Created broadcast 0 from textFile at MatrixRowLearning.scala:21 -2016-05-23 18:49:58 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 18:49:58 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:49:58 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-05-23 18:49:58 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 1 output partitions -2016-05-23 18:49:58 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-05-23 18:49:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:49:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:49:58 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24), which has no missing parents -2016-05-23 18:49:58 INFO MemoryStore:59 - ensureFreeSpace(3360) called with curMem=171580, maxMem=766075207 -2016-05-23 18:49:58 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-05-23 18:49:58 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=174940, maxMem=766075207 -2016-05-23 18:49:58 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.4 MB) -2016-05-23 18:49:58 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:58746 (size: 1935.0 B, free: 730.6 MB) -2016-05-23 18:49:58 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:49:58 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24) -2016-05-23 18:49:58 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-23 18:49:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:49:58 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 18:49:58 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:49:58 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 18:49:58 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 18:49:58 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 18:49:58 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 18:49:58 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 18:49:58 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-05-23 18:49:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 99 ms on localhost (1/1) -2016-05-23 18:49:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 18:49:58 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.119 s -2016-05-23 18:49:58 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.258871 s -2016-05-23 18:49:58 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 18:49:58 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 18:49:58 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-05-23 18:49:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:49:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:49:58 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24), which has no missing parents -2016-05-23 18:49:58 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=176875, maxMem=766075207 -2016-05-23 18:49:58 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 18:49:58 INFO MemoryStore:59 - ensureFreeSpace(1993) called with curMem=180403, maxMem=766075207 -2016-05-23 18:49:58 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1993.0 B, free 730.4 MB) -2016-05-23 18:49:58 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:58746 (size: 1993.0 B, free: 730.6 MB) -2016-05-23 18:49:58 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:49:58 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24) -2016-05-23 18:49:58 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-23 18:49:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:49:58 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-23 18:49:58 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:49:58 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2183 bytes result sent to driver -2016-05-23 18:49:58 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.023 s -2016-05-23 18:49:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 23 ms on localhost (1/1) -2016-05-23 18:49:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 18:49:58 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.045984 s -2016-05-23 18:49:58 INFO MemoryStore:59 - ensureFreeSpace(88528) called with curMem=182396, maxMem=766075207 -2016-05-23 18:49:58 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 86.5 KB, free 730.3 MB) -2016-05-23 18:49:59 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=270924, maxMem=766075207 -2016-05-23 18:49:59 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.3 MB) -2016-05-23 18:49:59 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:58746 (size: 19.3 KB, free: 730.6 MB) -2016-05-23 18:49:59 INFO SparkContext:59 - Created broadcast 3 from textFile at MatrixRowLearning.scala:36 -2016-05-23 18:49:59 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:49:59 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:43 -2016-05-23 18:49:59 INFO DAGScheduler:59 - Got job 2 (foreach at MatrixRowLearning.scala:43) with 1 output partitions -2016-05-23 18:49:59 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at MatrixRowLearning.scala:43) -2016-05-23 18:49:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:49:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:49:59 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[8] at map at MatrixRowLearning.scala:40), which has no missing parents -2016-05-23 18:49:59 INFO MemoryStore:59 - ensureFreeSpace(3752) called with curMem=290712, maxMem=766075207 -2016-05-23 18:49:59 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.7 KB, free 730.3 MB) -2016-05-23 18:49:59 INFO MemoryStore:59 - ensureFreeSpace(2064) called with curMem=294464, maxMem=766075207 -2016-05-23 18:49:59 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-05-23 18:49:59 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:58746 (size: 2.0 KB, free: 730.5 MB) -2016-05-23 18:49:59 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:49:59 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[8] at map at MatrixRowLearning.scala:40) -2016-05-23 18:49:59 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 18:49:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:49:59 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-05-23 18:49:59 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:49:59 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 18:49:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 47 ms on localhost (1/1) -2016-05-23 18:49:59 INFO DAGScheduler:59 - ResultStage 2 (foreach at MatrixRowLearning.scala:43) finished in 0.047 s -2016-05-23 18:49:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 18:49:59 INFO DAGScheduler:59 - Job 2 finished: foreach at MatrixRowLearning.scala:43, took 0.065257 s -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 18:49:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 18:49:59 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 18:49:59 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 18:49:59 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 18:49:59 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 18:49:59 INFO BlockManager:59 - BlockManager stopped -2016-05-23 18:49:59 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 18:49:59 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 18:49:59 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 18:49:59 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 18:49:59 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-35ec1ca2-e0cd-4854-a5e5-61f1ef058220 -2016-05-23 18:49:59 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 18:51:09 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 18:51:09 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 18:51:10 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 18:51:10 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 18:51:10 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 18:51:11 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 18:51:11 INFO Remoting:74 - Starting remoting -2016-05-23 18:51:11 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:58770] -2016-05-23 18:51:11 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58770. -2016-05-23 18:51:11 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 18:51:11 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 18:51:11 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-ae1c5bd6-ea7e-4be2-93a6-acf8e79eaa61 -2016-05-23 18:51:11 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 18:51:11 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-b8749d2c-335d-495b-ace5-dd7c53ea1ad6\httpd-8dae9bb8-9d33-45c0-b151-83b29a2922bd -2016-05-23 18:51:12 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 18:51:12 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:51:12 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58771 -2016-05-23 18:51:12 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58771. -2016-05-23 18:51:12 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 18:51:12 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:51:12 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 18:51:12 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 18:51:12 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 18:51:12 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 18:51:12 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 18:51:12 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58778. -2016-05-23 18:51:12 INFO NettyBlockTransferService:59 - Server created on 58778 -2016-05-23 18:51:12 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 18:51:12 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:58778 with 730.6 MB RAM, BlockManagerId(driver, localhost, 58778) -2016-05-23 18:51:12 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 18:51:13 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 18:51:13 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 18:51:13 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 18:51:13 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 18:51:13 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:58778 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 18:51:13 INFO SparkContext:59 - Created broadcast 0 from textFile at MatrixRowLearning.scala:21 -2016-05-23 18:51:14 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 18:51:15 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:51:15 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-05-23 18:51:15 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 1 output partitions -2016-05-23 18:51:15 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-05-23 18:51:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:51:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:51:15 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24), which has no missing parents -2016-05-23 18:51:15 INFO MemoryStore:59 - ensureFreeSpace(3360) called with curMem=171580, maxMem=766075207 -2016-05-23 18:51:15 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-05-23 18:51:15 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=174940, maxMem=766075207 -2016-05-23 18:51:15 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.4 MB) -2016-05-23 18:51:15 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:58778 (size: 1935.0 B, free: 730.6 MB) -2016-05-23 18:51:15 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:51:15 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24) -2016-05-23 18:51:15 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-23 18:51:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:51:15 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 18:51:15 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:51:15 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 18:51:15 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 18:51:15 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 18:51:15 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 18:51:15 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 18:51:15 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-05-23 18:51:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 227 ms on localhost (1/1) -2016-05-23 18:51:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 18:51:15 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.271 s -2016-05-23 18:51:15 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.487433 s -2016-05-23 18:51:15 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 18:51:15 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 18:51:15 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-05-23 18:51:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:51:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:51:15 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24), which has no missing parents -2016-05-23 18:51:15 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=176875, maxMem=766075207 -2016-05-23 18:51:15 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 18:51:15 INFO MemoryStore:59 - ensureFreeSpace(1993) called with curMem=180403, maxMem=766075207 -2016-05-23 18:51:15 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1993.0 B, free 730.4 MB) -2016-05-23 18:51:15 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:58778 (size: 1993.0 B, free: 730.6 MB) -2016-05-23 18:51:15 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:51:15 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24) -2016-05-23 18:51:15 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-23 18:51:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:51:15 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-23 18:51:15 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:51:15 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2183 bytes result sent to driver -2016-05-23 18:51:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 27 ms on localhost (1/1) -2016-05-23 18:51:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 18:51:15 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.028 s -2016-05-23 18:51:15 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.058866 s -2016-05-23 18:51:15 INFO MemoryStore:59 - ensureFreeSpace(88528) called with curMem=182396, maxMem=766075207 -2016-05-23 18:51:15 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 86.5 KB, free 730.3 MB) -2016-05-23 18:51:15 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=270924, maxMem=766075207 -2016-05-23 18:51:15 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.3 MB) -2016-05-23 18:51:15 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:58778 (size: 19.3 KB, free: 730.6 MB) -2016-05-23 18:51:15 INFO SparkContext:59 - Created broadcast 3 from textFile at MatrixRowLearning.scala:35 -2016-05-23 18:51:15 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:51:15 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:42 -2016-05-23 18:51:15 INFO DAGScheduler:59 - Got job 2 (foreach at MatrixRowLearning.scala:42) with 1 output partitions -2016-05-23 18:51:15 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at MatrixRowLearning.scala:42) -2016-05-23 18:51:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:51:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:51:15 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[8] at map at MatrixRowLearning.scala:39), which has no missing parents -2016-05-23 18:51:15 INFO MemoryStore:59 - ensureFreeSpace(3752) called with curMem=290712, maxMem=766075207 -2016-05-23 18:51:15 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.7 KB, free 730.3 MB) -2016-05-23 18:51:15 INFO MemoryStore:59 - ensureFreeSpace(2062) called with curMem=294464, maxMem=766075207 -2016-05-23 18:51:15 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-05-23 18:51:15 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:58778 (size: 2.0 KB, free: 730.5 MB) -2016-05-23 18:51:15 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:51:15 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[8] at map at MatrixRowLearning.scala:39) -2016-05-23 18:51:15 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 18:51:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:51:15 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-05-23 18:51:15 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:51:15 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 18:51:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 49 ms on localhost (1/1) -2016-05-23 18:51:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 18:51:15 INFO DAGScheduler:59 - ResultStage 2 (foreach at MatrixRowLearning.scala:42) finished in 0.052 s -2016-05-23 18:51:15 INFO DAGScheduler:59 - Job 2 finished: foreach at MatrixRowLearning.scala:42, took 0.070058 s -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 18:51:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 18:51:16 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 18:51:16 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 18:51:16 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 18:51:16 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 18:51:16 INFO BlockManager:59 - BlockManager stopped -2016-05-23 18:51:16 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 18:51:16 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 18:51:16 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 18:51:16 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 18:51:16 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-b8749d2c-335d-495b-ace5-dd7c53ea1ad6 -2016-05-23 18:51:27 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 18:51:28 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 18:51:28 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 18:51:28 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 18:51:28 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 18:51:29 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 18:51:29 INFO Remoting:74 - Starting remoting -2016-05-23 18:51:30 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:58800] -2016-05-23 18:51:30 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58800. -2016-05-23 18:51:30 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 18:51:30 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 18:51:30 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-17471c5e-eef2-46d5-b762-f5e6ee49e4d8 -2016-05-23 18:51:30 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 18:51:30 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-80cb0683-8c3f-46df-a5d4-70fd224d90ed\httpd-26bf41fe-9483-441c-ba2b-1e3bdc4f8b92 -2016-05-23 18:51:30 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 18:51:30 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:51:30 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58801 -2016-05-23 18:51:30 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58801. -2016-05-23 18:51:30 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 18:51:30 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:51:30 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 18:51:30 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 18:51:30 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 18:51:30 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 18:51:30 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 18:51:31 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58808. -2016-05-23 18:51:31 INFO NettyBlockTransferService:59 - Server created on 58808 -2016-05-23 18:51:31 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 18:51:31 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:58808 with 730.6 MB RAM, BlockManagerId(driver, localhost, 58808) -2016-05-23 18:51:31 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 18:51:31 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 18:51:31 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 18:51:31 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 18:51:31 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 18:51:31 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:58808 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 18:51:31 INFO SparkContext:59 - Created broadcast 0 from textFile at MatrixRowLearning.scala:21 -2016-05-23 18:51:33 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 18:51:33 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:51:33 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-05-23 18:51:33 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 1 output partitions -2016-05-23 18:51:33 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-05-23 18:51:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:51:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:51:33 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24), which has no missing parents -2016-05-23 18:51:33 INFO MemoryStore:59 - ensureFreeSpace(3360) called with curMem=171580, maxMem=766075207 -2016-05-23 18:51:33 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-05-23 18:51:33 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=174940, maxMem=766075207 -2016-05-23 18:51:33 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.4 MB) -2016-05-23 18:51:33 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:58808 (size: 1935.0 B, free: 730.6 MB) -2016-05-23 18:51:33 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:51:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24) -2016-05-23 18:51:33 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-23 18:51:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:51:33 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 18:51:33 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:51:33 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 18:51:33 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 18:51:33 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 18:51:33 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 18:51:33 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 18:51:33 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-05-23 18:51:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 143 ms on localhost (1/1) -2016-05-23 18:51:33 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.179 s -2016-05-23 18:51:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 18:51:33 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.358888 s -2016-05-23 18:51:33 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 18:51:33 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 18:51:33 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-05-23 18:51:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:51:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:51:33 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24), which has no missing parents -2016-05-23 18:51:33 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=176875, maxMem=766075207 -2016-05-23 18:51:33 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 18:51:33 INFO MemoryStore:59 - ensureFreeSpace(1993) called with curMem=180403, maxMem=766075207 -2016-05-23 18:51:33 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1993.0 B, free 730.4 MB) -2016-05-23 18:51:33 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:58808 (size: 1993.0 B, free: 730.6 MB) -2016-05-23 18:51:33 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:51:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24) -2016-05-23 18:51:33 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-23 18:51:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:51:33 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-23 18:51:33 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:51:33 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2183 bytes result sent to driver -2016-05-23 18:51:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 40 ms on localhost (1/1) -2016-05-23 18:51:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 18:51:33 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.050 s -2016-05-23 18:51:33 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.072478 s -2016-05-23 18:51:33 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:33 -2016-05-23 18:51:33 INFO DAGScheduler:59 - Got job 2 (foreach at MatrixRowLearning.scala:33) with 1 output partitions -2016-05-23 18:51:33 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at MatrixRowLearning.scala:33) -2016-05-23 18:51:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:51:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:51:33 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24), which has no missing parents -2016-05-23 18:51:33 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=182396, maxMem=766075207 -2016-05-23 18:51:33 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 18:51:33 INFO MemoryStore:59 - ensureFreeSpace(1973) called with curMem=185876, maxMem=766075207 -2016-05-23 18:51:33 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1973.0 B, free 730.4 MB) -2016-05-23 18:51:33 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:58808 (size: 1973.0 B, free: 730.6 MB) -2016-05-23 18:51:33 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:51:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24) -2016-05-23 18:51:33 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 18:51:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:51:33 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-05-23 18:51:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:51:34 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 18:51:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 19 ms on localhost (1/1) -2016-05-23 18:51:34 INFO DAGScheduler:59 - ResultStage 2 (foreach at MatrixRowLearning.scala:33) finished in 0.020 s -2016-05-23 18:51:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 18:51:34 INFO DAGScheduler:59 - Job 2 finished: foreach at MatrixRowLearning.scala:33, took 0.041669 s -2016-05-23 18:51:34 INFO MemoryStore:59 - ensureFreeSpace(88528) called with curMem=187849, maxMem=766075207 -2016-05-23 18:51:34 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 86.5 KB, free 730.3 MB) -2016-05-23 18:51:34 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=276377, maxMem=766075207 -2016-05-23 18:51:34 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.3 MB) -2016-05-23 18:51:34 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:58808 (size: 19.3 KB, free: 730.5 MB) -2016-05-23 18:51:34 INFO SparkContext:59 - Created broadcast 4 from textFile at MatrixRowLearning.scala:36 -2016-05-23 18:51:34 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-23 18:51:34 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:58808 in memory (size: 1935.0 B, free: 730.5 MB) -2016-05-23 18:51:34 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-23 18:51:34 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:58808 in memory (size: 1973.0 B, free: 730.6 MB) -2016-05-23 18:51:34 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:51:34 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-23 18:51:34 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:58808 in memory (size: 1993.0 B, free: 730.6 MB) -2016-05-23 18:51:34 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:43 -2016-05-23 18:51:34 INFO DAGScheduler:59 - Got job 3 (foreach at MatrixRowLearning.scala:43) with 1 output partitions -2016-05-23 18:51:34 INFO DAGScheduler:59 - Final stage: ResultStage 3(foreach at MatrixRowLearning.scala:43) -2016-05-23 18:51:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:51:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:51:34 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at map at MatrixRowLearning.scala:40), which has no missing parents -2016-05-23 18:51:34 INFO MemoryStore:59 - ensureFreeSpace(3752) called with curMem=279896, maxMem=766075207 -2016-05-23 18:51:34 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.7 KB, free 730.3 MB) -2016-05-23 18:51:34 INFO MemoryStore:59 - ensureFreeSpace(2063) called with curMem=283648, maxMem=766075207 -2016-05-23 18:51:34 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-05-23 18:51:34 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:58808 (size: 2.0 KB, free: 730.6 MB) -2016-05-23 18:51:34 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:51:34 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at map at MatrixRowLearning.scala:40) -2016-05-23 18:51:34 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-05-23 18:51:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 3, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:51:34 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 3) -2016-05-23 18:51:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:51:34 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 18:51:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 3) in 16 ms on localhost (1/1) -2016-05-23 18:51:34 INFO DAGScheduler:59 - ResultStage 3 (foreach at MatrixRowLearning.scala:43) finished in 0.016 s -2016-05-23 18:51:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 18:51:34 INFO DAGScheduler:59 - Job 3 finished: foreach at MatrixRowLearning.scala:43, took 0.032177 s -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 18:51:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 18:51:34 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 18:51:34 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 18:51:34 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 18:51:34 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 18:51:34 INFO BlockManager:59 - BlockManager stopped -2016-05-23 18:51:34 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 18:51:34 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 18:51:34 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 18:51:34 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 18:51:34 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-80cb0683-8c3f-46df-a5d4-70fd224d90ed -2016-05-23 18:51:34 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 18:51:34 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 18:54:37 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 18:54:38 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 18:54:38 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 18:54:38 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 18:54:38 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 18:54:39 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 18:54:39 INFO Remoting:74 - Starting remoting -2016-05-23 18:54:39 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:58839] -2016-05-23 18:54:39 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58839. -2016-05-23 18:54:39 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 18:54:39 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 18:54:39 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-fbb03d3a-9bc9-48ac-994e-74f0afcb3d90 -2016-05-23 18:54:39 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 18:54:39 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-41ce74c2-5986-4190-88af-d3ecb27a5b76\httpd-cde82ebe-eef8-4cb9-a531-76fa447b1382 -2016-05-23 18:54:39 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 18:54:40 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:54:40 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58840 -2016-05-23 18:54:40 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58840. -2016-05-23 18:54:40 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 18:54:40 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:54:40 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 18:54:40 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 18:54:40 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 18:54:40 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 18:54:40 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 18:54:40 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58847. -2016-05-23 18:54:40 INFO NettyBlockTransferService:59 - Server created on 58847 -2016-05-23 18:54:40 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 18:54:40 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:58847 with 730.6 MB RAM, BlockManagerId(driver, localhost, 58847) -2016-05-23 18:54:40 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 18:54:41 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 18:54:41 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 18:54:41 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 18:54:41 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 18:54:41 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:58847 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 18:54:41 INFO SparkContext:59 - Created broadcast 0 from textFile at MatrixRowLearning.scala:21 -2016-05-23 18:54:42 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 18:54:43 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:54:43 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-05-23 18:54:43 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 1 output partitions -2016-05-23 18:54:43 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-05-23 18:54:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:54:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:54:43 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24), which has no missing parents -2016-05-23 18:54:43 INFO MemoryStore:59 - ensureFreeSpace(3360) called with curMem=171580, maxMem=766075207 -2016-05-23 18:54:43 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-05-23 18:54:43 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=174940, maxMem=766075207 -2016-05-23 18:54:43 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.4 MB) -2016-05-23 18:54:43 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:58847 (size: 1935.0 B, free: 730.6 MB) -2016-05-23 18:54:43 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:54:43 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24) -2016-05-23 18:54:43 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-23 18:54:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:54:43 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 18:54:43 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:54:43 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 18:54:43 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 18:54:43 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 18:54:43 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 18:54:43 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 18:54:43 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-05-23 18:54:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 122 ms on localhost (1/1) -2016-05-23 18:54:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 18:54:43 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.148 s -2016-05-23 18:54:43 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.359961 s -2016-05-23 18:54:43 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 18:54:43 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 18:54:43 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-05-23 18:54:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:54:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:54:43 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24), which has no missing parents -2016-05-23 18:54:43 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=176875, maxMem=766075207 -2016-05-23 18:54:43 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 18:54:43 INFO MemoryStore:59 - ensureFreeSpace(1993) called with curMem=180403, maxMem=766075207 -2016-05-23 18:54:43 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1993.0 B, free 730.4 MB) -2016-05-23 18:54:43 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:58847 (size: 1993.0 B, free: 730.6 MB) -2016-05-23 18:54:43 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:54:43 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24) -2016-05-23 18:54:43 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-23 18:54:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:54:43 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-23 18:54:43 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:54:43 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2183 bytes result sent to driver -2016-05-23 18:54:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 24 ms on localhost (1/1) -2016-05-23 18:54:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 18:54:43 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.024 s -2016-05-23 18:54:43 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.048595 s -2016-05-23 18:54:43 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:33 -2016-05-23 18:54:43 INFO DAGScheduler:59 - Got job 2 (foreach at MatrixRowLearning.scala:33) with 1 output partitions -2016-05-23 18:54:43 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at MatrixRowLearning.scala:33) -2016-05-23 18:54:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:54:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:54:43 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24), which has no missing parents -2016-05-23 18:54:43 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=182396, maxMem=766075207 -2016-05-23 18:54:43 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 18:54:43 INFO MemoryStore:59 - ensureFreeSpace(1973) called with curMem=185876, maxMem=766075207 -2016-05-23 18:54:43 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1973.0 B, free 730.4 MB) -2016-05-23 18:54:43 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:58847 (size: 1973.0 B, free: 730.6 MB) -2016-05-23 18:54:43 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:54:43 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:24) -2016-05-23 18:54:43 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 18:54:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:54:43 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-05-23 18:54:43 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:54:43 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 18:54:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 26 ms on localhost (1/1) -2016-05-23 18:54:43 INFO DAGScheduler:59 - ResultStage 2 (foreach at MatrixRowLearning.scala:33) finished in 0.026 s -2016-05-23 18:54:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 18:54:43 INFO DAGScheduler:59 - Job 2 finished: foreach at MatrixRowLearning.scala:33, took 0.041506 s -2016-05-23 18:54:43 INFO MemoryStore:59 - ensureFreeSpace(88528) called with curMem=187849, maxMem=766075207 -2016-05-23 18:54:43 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 86.5 KB, free 730.3 MB) -2016-05-23 18:54:43 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=276377, maxMem=766075207 -2016-05-23 18:54:43 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.3 MB) -2016-05-23 18:54:43 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:58847 (size: 19.3 KB, free: 730.5 MB) -2016-05-23 18:54:43 INFO SparkContext:59 - Created broadcast 4 from textFile at MatrixRowLearning.scala:36 -2016-05-23 18:54:43 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:54:43 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:43 -2016-05-23 18:54:43 INFO DAGScheduler:59 - Got job 3 (foreach at MatrixRowLearning.scala:43) with 1 output partitions -2016-05-23 18:54:43 INFO DAGScheduler:59 - Final stage: ResultStage 3(foreach at MatrixRowLearning.scala:43) -2016-05-23 18:54:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:54:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:54:43 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at map at MatrixRowLearning.scala:40), which has no missing parents -2016-05-23 18:54:43 INFO MemoryStore:59 - ensureFreeSpace(3752) called with curMem=296165, maxMem=766075207 -2016-05-23 18:54:43 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.7 KB, free 730.3 MB) -2016-05-23 18:54:43 INFO MemoryStore:59 - ensureFreeSpace(2063) called with curMem=299917, maxMem=766075207 -2016-05-23 18:54:43 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-05-23 18:54:43 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:58847 (size: 2.0 KB, free: 730.5 MB) -2016-05-23 18:54:43 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:54:43 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at map at MatrixRowLearning.scala:40) -2016-05-23 18:54:43 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-05-23 18:54:43 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:58847 in memory (size: 1973.0 B, free: 730.5 MB) -2016-05-23 18:54:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 3, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:54:43 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 3) -2016-05-23 18:54:43 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:58847 in memory (size: 1993.0 B, free: 730.5 MB) -2016-05-23 18:54:43 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:54:43 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:58847 in memory (size: 1935.0 B, free: 730.6 MB) -2016-05-23 18:54:43 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 18:54:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 3) in 19 ms on localhost (1/1) -2016-05-23 18:54:43 INFO DAGScheduler:59 - ResultStage 3 (foreach at MatrixRowLearning.scala:43) finished in 0.019 s -2016-05-23 18:54:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 18:54:43 INFO DAGScheduler:59 - Job 3 finished: foreach at MatrixRowLearning.scala:43, took 0.042167 s -2016-05-23 18:54:43 INFO MemoryStore:59 - ensureFreeSpace(230744) called with curMem=285711, maxMem=766075207 -2016-05-23 18:54:43 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-23 18:54:43 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=516455, maxMem=766075207 -2016-05-23 18:54:43 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-23 18:54:43 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:58847 (size: 19.3 KB, free: 730.5 MB) -2016-05-23 18:54:43 INFO SparkContext:59 - Created broadcast 6 from textFile at MatrixRowLearning.scala:47 -2016-05-23 18:54:43 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 18:54:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 18:54:43 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 18:54:43 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 18:54:44 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 18:54:44 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 18:54:44 INFO BlockManager:59 - BlockManager stopped -2016-05-23 18:54:44 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 18:54:44 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 18:54:44 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 18:54:44 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 18:54:44 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-41ce74c2-5986-4190-88af-d3ecb27a5b76 -2016-05-23 18:55:15 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 18:55:16 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 18:55:17 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 18:55:17 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 18:55:17 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 18:55:17 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 18:55:17 INFO Remoting:74 - Starting remoting -2016-05-23 18:55:18 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:58870] -2016-05-23 18:55:18 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58870. -2016-05-23 18:55:18 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 18:55:18 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 18:55:18 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-efc3c2ab-cd39-42d6-aefc-90f020dff097 -2016-05-23 18:55:18 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 18:55:18 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-e38dc376-5fc3-4f67-b80d-57b1c1721b89\httpd-3c008292-2dd8-4b00-b52d-a7e85ae2ca2a -2016-05-23 18:55:18 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 18:55:18 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:55:18 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58871 -2016-05-23 18:55:18 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58871. -2016-05-23 18:55:18 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 18:55:18 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:55:18 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 18:55:18 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 18:55:18 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 18:55:18 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 18:55:18 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 18:55:19 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58878. -2016-05-23 18:55:19 INFO NettyBlockTransferService:59 - Server created on 58878 -2016-05-23 18:55:19 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 18:55:19 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:58878 with 730.6 MB RAM, BlockManagerId(driver, localhost, 58878) -2016-05-23 18:55:19 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 18:55:19 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 18:55:19 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 18:55:19 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 18:55:19 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 18:55:19 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:58878 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 18:55:19 INFO SparkContext:59 - Created broadcast 0 from textFile at MatrixRowLearning.scala:22 -2016-05-23 18:55:21 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 18:55:21 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:55:21 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-05-23 18:55:21 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 1 output partitions -2016-05-23 18:55:21 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-05-23 18:55:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:55:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:55:21 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25), which has no missing parents -2016-05-23 18:55:21 INFO MemoryStore:59 - ensureFreeSpace(3360) called with curMem=171580, maxMem=766075207 -2016-05-23 18:55:21 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-05-23 18:55:21 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=174940, maxMem=766075207 -2016-05-23 18:55:21 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.4 MB) -2016-05-23 18:55:21 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:58878 (size: 1935.0 B, free: 730.6 MB) -2016-05-23 18:55:21 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:55:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25) -2016-05-23 18:55:21 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-23 18:55:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:55:21 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 18:55:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:55:21 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 18:55:21 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 18:55:21 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 18:55:21 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 18:55:21 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 18:55:21 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-05-23 18:55:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 95 ms on localhost (1/1) -2016-05-23 18:55:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 18:55:21 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.114 s -2016-05-23 18:55:21 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.290467 s -2016-05-23 18:55:21 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 18:55:21 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 18:55:21 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-05-23 18:55:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:55:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:55:21 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25), which has no missing parents -2016-05-23 18:55:21 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=176875, maxMem=766075207 -2016-05-23 18:55:21 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 18:55:21 INFO MemoryStore:59 - ensureFreeSpace(1993) called with curMem=180403, maxMem=766075207 -2016-05-23 18:55:21 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1993.0 B, free 730.4 MB) -2016-05-23 18:55:21 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:58878 (size: 1993.0 B, free: 730.6 MB) -2016-05-23 18:55:21 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:55:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25) -2016-05-23 18:55:21 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-23 18:55:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:55:21 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-23 18:55:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:55:21 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2183 bytes result sent to driver -2016-05-23 18:55:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 31 ms on localhost (1/1) -2016-05-23 18:55:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 18:55:21 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.032 s -2016-05-23 18:55:21 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.061433 s -2016-05-23 18:55:21 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:34 -2016-05-23 18:55:21 INFO DAGScheduler:59 - Got job 2 (foreach at MatrixRowLearning.scala:34) with 1 output partitions -2016-05-23 18:55:21 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at MatrixRowLearning.scala:34) -2016-05-23 18:55:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:55:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:55:21 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25), which has no missing parents -2016-05-23 18:55:21 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=182396, maxMem=766075207 -2016-05-23 18:55:21 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 18:55:21 INFO MemoryStore:59 - ensureFreeSpace(1973) called with curMem=185876, maxMem=766075207 -2016-05-23 18:55:21 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1973.0 B, free 730.4 MB) -2016-05-23 18:55:21 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:58878 (size: 1973.0 B, free: 730.6 MB) -2016-05-23 18:55:21 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:55:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25) -2016-05-23 18:55:21 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 18:55:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:55:21 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-05-23 18:55:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:55:21 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 18:55:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 18 ms on localhost (1/1) -2016-05-23 18:55:21 INFO DAGScheduler:59 - ResultStage 2 (foreach at MatrixRowLearning.scala:34) finished in 0.019 s -2016-05-23 18:55:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 18:55:21 INFO DAGScheduler:59 - Job 2 finished: foreach at MatrixRowLearning.scala:34, took 0.032651 s -2016-05-23 18:55:21 INFO MemoryStore:59 - ensureFreeSpace(88528) called with curMem=187849, maxMem=766075207 -2016-05-23 18:55:21 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 86.5 KB, free 730.3 MB) -2016-05-23 18:55:21 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=276377, maxMem=766075207 -2016-05-23 18:55:21 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.3 MB) -2016-05-23 18:55:21 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:58878 (size: 19.3 KB, free: 730.5 MB) -2016-05-23 18:55:21 INFO SparkContext:59 - Created broadcast 4 from textFile at MatrixRowLearning.scala:37 -2016-05-23 18:55:21 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:55:21 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:44 -2016-05-23 18:55:21 INFO DAGScheduler:59 - Got job 3 (foreach at MatrixRowLearning.scala:44) with 1 output partitions -2016-05-23 18:55:21 INFO DAGScheduler:59 - Final stage: ResultStage 3(foreach at MatrixRowLearning.scala:44) -2016-05-23 18:55:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:55:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:55:21 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at map at MatrixRowLearning.scala:41), which has no missing parents -2016-05-23 18:55:21 INFO MemoryStore:59 - ensureFreeSpace(3752) called with curMem=294192, maxMem=766075207 -2016-05-23 18:55:21 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.7 KB, free 730.3 MB) -2016-05-23 18:55:21 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:58878 in memory (size: 1973.0 B, free: 730.6 MB) -2016-05-23 18:55:21 INFO MemoryStore:59 - ensureFreeSpace(2063) called with curMem=294464, maxMem=766075207 -2016-05-23 18:55:21 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-05-23 18:55:21 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:58878 (size: 2.0 KB, free: 730.5 MB) -2016-05-23 18:55:21 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:55:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at map at MatrixRowLearning.scala:41) -2016-05-23 18:55:21 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-05-23 18:55:21 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:58878 in memory (size: 1993.0 B, free: 730.5 MB) -2016-05-23 18:55:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 3, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:55:21 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 3) -2016-05-23 18:55:21 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:58878 in memory (size: 1935.0 B, free: 730.6 MB) -2016-05-23 18:55:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:55:21 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 18:55:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 3) in 22 ms on localhost (1/1) -2016-05-23 18:55:21 INFO DAGScheduler:59 - ResultStage 3 (foreach at MatrixRowLearning.scala:44) finished in 0.023 s -2016-05-23 18:55:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 18:55:21 INFO DAGScheduler:59 - Job 3 finished: foreach at MatrixRowLearning.scala:44, took 0.053941 s -2016-05-23 18:55:21 INFO MemoryStore:59 - ensureFreeSpace(230744) called with curMem=285711, maxMem=766075207 -2016-05-23 18:55:21 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-23 18:55:21 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=516455, maxMem=766075207 -2016-05-23 18:55:21 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-23 18:55:21 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:58878 (size: 19.3 KB, free: 730.5 MB) -2016-05-23 18:55:21 INFO SparkContext:59 - Created broadcast 6 from textFile at MatrixRowLearning.scala:48 -2016-05-23 18:55:21 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:55:21 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:54 -2016-05-23 18:55:21 INFO DAGScheduler:59 - Got job 4 (foreach at MatrixRowLearning.scala:54) with 1 output partitions -2016-05-23 18:55:21 INFO DAGScheduler:59 - Final stage: ResultStage 4(foreach at MatrixRowLearning.scala:54) -2016-05-23 18:55:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:55:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:55:21 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at map at MatrixRowLearning.scala:52), which has no missing parents -2016-05-23 18:55:21 INFO MemoryStore:59 - ensureFreeSpace(3720) called with curMem=536243, maxMem=766075207 -2016-05-23 18:55:21 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.6 KB, free 730.1 MB) -2016-05-23 18:55:21 INFO MemoryStore:59 - ensureFreeSpace(2055) called with curMem=539963, maxMem=766075207 -2016-05-23 18:55:21 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.1 MB) -2016-05-23 18:55:21 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:58878 (size: 2.0 KB, free: 730.5 MB) -2016-05-23 18:55:21 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:55:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at map at MatrixRowLearning.scala:52) -2016-05-23 18:55:21 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-05-23 18:55:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 4, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:55:21 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 4) -2016-05-23 18:55:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:55:22 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 4). 2044 bytes result sent to driver -2016-05-23 18:55:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 4) in 19 ms on localhost (1/1) -2016-05-23 18:55:22 INFO DAGScheduler:59 - ResultStage 4 (foreach at MatrixRowLearning.scala:54) finished in 0.019 s -2016-05-23 18:55:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 18:55:22 INFO DAGScheduler:59 - Job 4 finished: foreach at MatrixRowLearning.scala:54, took 0.041466 s -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 18:55:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 18:55:22 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 18:55:22 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 18:55:22 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 18:55:22 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 18:55:22 INFO BlockManager:59 - BlockManager stopped -2016-05-23 18:55:22 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 18:55:22 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 18:55:22 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 18:55:22 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 18:55:22 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-e38dc376-5fc3-4f67-b80d-57b1c1721b89 -2016-05-23 18:55:22 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 18:56:35 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 18:56:36 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 18:56:36 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 18:56:36 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 18:56:36 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 18:56:37 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 18:56:37 INFO Remoting:74 - Starting remoting -2016-05-23 18:56:38 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:58903] -2016-05-23 18:56:38 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58903. -2016-05-23 18:56:38 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 18:56:38 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 18:56:38 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-4aad0881-31d0-43b0-82d5-6b03702280c9 -2016-05-23 18:56:38 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 18:56:38 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-99cf7c4b-7385-4273-8cd3-9d6b1e30abba\httpd-d7a810c7-1b39-4713-b087-f6398c53981b -2016-05-23 18:56:38 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 18:56:38 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:56:38 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58904 -2016-05-23 18:56:38 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58904. -2016-05-23 18:56:38 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 18:56:38 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:56:38 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 18:56:38 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 18:56:38 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 18:56:38 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 18:56:38 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 18:56:38 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58911. -2016-05-23 18:56:38 INFO NettyBlockTransferService:59 - Server created on 58911 -2016-05-23 18:56:38 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 18:56:38 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:58911 with 730.6 MB RAM, BlockManagerId(driver, localhost, 58911) -2016-05-23 18:56:39 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 18:56:39 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 18:56:39 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 18:56:39 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 18:56:39 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 18:56:39 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:58911 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 18:56:39 INFO SparkContext:59 - Created broadcast 0 from textFile at MatrixRowLearning.scala:22 -2016-05-23 18:56:41 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 18:56:41 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:56:41 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-05-23 18:56:41 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 1 output partitions -2016-05-23 18:56:41 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-05-23 18:56:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:56:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:56:41 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25), which has no missing parents -2016-05-23 18:56:41 INFO MemoryStore:59 - ensureFreeSpace(3360) called with curMem=171580, maxMem=766075207 -2016-05-23 18:56:41 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-05-23 18:56:41 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=174940, maxMem=766075207 -2016-05-23 18:56:41 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.4 MB) -2016-05-23 18:56:41 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:58911 (size: 1935.0 B, free: 730.6 MB) -2016-05-23 18:56:41 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:56:41 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25) -2016-05-23 18:56:41 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-23 18:56:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:56:41 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 18:56:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:56:41 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 18:56:41 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 18:56:41 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 18:56:41 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 18:56:41 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 18:56:41 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-05-23 18:56:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 128 ms on localhost (1/1) -2016-05-23 18:56:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 18:56:41 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.169 s -2016-05-23 18:56:41 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.365878 s -2016-05-23 18:56:41 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 18:56:41 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 18:56:41 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-05-23 18:56:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:56:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:56:41 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25), which has no missing parents -2016-05-23 18:56:41 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=176875, maxMem=766075207 -2016-05-23 18:56:41 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 18:56:41 INFO MemoryStore:59 - ensureFreeSpace(1993) called with curMem=180403, maxMem=766075207 -2016-05-23 18:56:41 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1993.0 B, free 730.4 MB) -2016-05-23 18:56:41 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:58911 (size: 1993.0 B, free: 730.6 MB) -2016-05-23 18:56:41 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:56:41 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25) -2016-05-23 18:56:41 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-23 18:56:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:56:41 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-23 18:56:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:56:41 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2183 bytes result sent to driver -2016-05-23 18:56:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 23 ms on localhost (1/1) -2016-05-23 18:56:41 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.023 s -2016-05-23 18:56:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 18:56:41 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.045608 s -2016-05-23 18:56:41 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:34 -2016-05-23 18:56:41 INFO DAGScheduler:59 - Got job 2 (foreach at MatrixRowLearning.scala:34) with 1 output partitions -2016-05-23 18:56:41 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at MatrixRowLearning.scala:34) -2016-05-23 18:56:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:56:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:56:41 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25), which has no missing parents -2016-05-23 18:56:41 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=182396, maxMem=766075207 -2016-05-23 18:56:41 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 18:56:41 INFO MemoryStore:59 - ensureFreeSpace(1973) called with curMem=185876, maxMem=766075207 -2016-05-23 18:56:41 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1973.0 B, free 730.4 MB) -2016-05-23 18:56:41 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:58911 (size: 1973.0 B, free: 730.6 MB) -2016-05-23 18:56:41 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:56:41 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25) -2016-05-23 18:56:41 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 18:56:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:56:41 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-05-23 18:56:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:56:41 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 18:56:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 20 ms on localhost (1/1) -2016-05-23 18:56:41 INFO DAGScheduler:59 - ResultStage 2 (foreach at MatrixRowLearning.scala:34) finished in 0.020 s -2016-05-23 18:56:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 18:56:41 INFO DAGScheduler:59 - Job 2 finished: foreach at MatrixRowLearning.scala:34, took 0.036405 s -2016-05-23 18:56:41 INFO MemoryStore:59 - ensureFreeSpace(88528) called with curMem=187849, maxMem=766075207 -2016-05-23 18:56:41 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 86.5 KB, free 730.3 MB) -2016-05-23 18:56:41 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=276377, maxMem=766075207 -2016-05-23 18:56:41 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.3 MB) -2016-05-23 18:56:41 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:58911 (size: 19.3 KB, free: 730.5 MB) -2016-05-23 18:56:41 INFO SparkContext:59 - Created broadcast 4 from textFile at MatrixRowLearning.scala:37 -2016-05-23 18:56:41 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:56:41 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:44 -2016-05-23 18:56:41 INFO DAGScheduler:59 - Got job 3 (foreach at MatrixRowLearning.scala:44) with 1 output partitions -2016-05-23 18:56:41 INFO DAGScheduler:59 - Final stage: ResultStage 3(foreach at MatrixRowLearning.scala:44) -2016-05-23 18:56:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:56:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:56:41 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at map at MatrixRowLearning.scala:41), which has no missing parents -2016-05-23 18:56:41 INFO MemoryStore:59 - ensureFreeSpace(3752) called with curMem=296165, maxMem=766075207 -2016-05-23 18:56:41 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.7 KB, free 730.3 MB) -2016-05-23 18:56:41 INFO MemoryStore:59 - ensureFreeSpace(2063) called with curMem=299917, maxMem=766075207 -2016-05-23 18:56:41 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-05-23 18:56:41 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:58911 (size: 2.0 KB, free: 730.5 MB) -2016-05-23 18:56:41 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:56:41 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at map at MatrixRowLearning.scala:41) -2016-05-23 18:56:41 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-05-23 18:56:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 3, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:56:41 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 3) -2016-05-23 18:56:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:56:42 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 18:56:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 3) in 21 ms on localhost (1/1) -2016-05-23 18:56:42 INFO DAGScheduler:59 - ResultStage 3 (foreach at MatrixRowLearning.scala:44) finished in 0.021 s -2016-05-23 18:56:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 18:56:42 INFO DAGScheduler:59 - Job 3 finished: foreach at MatrixRowLearning.scala:44, took 0.038944 s -2016-05-23 18:56:42 INFO MemoryStore:59 - ensureFreeSpace(230744) called with curMem=301980, maxMem=766075207 -2016-05-23 18:56:42 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-23 18:56:42 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=532724, maxMem=766075207 -2016-05-23 18:56:42 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-23 18:56:42 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:58911 (size: 19.3 KB, free: 730.5 MB) -2016-05-23 18:56:42 INFO SparkContext:59 - Created broadcast 6 from textFile at MatrixRowLearning.scala:48 -2016-05-23 18:56:42 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:56:42 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:54 -2016-05-23 18:56:42 INFO DAGScheduler:59 - Got job 4 (foreach at MatrixRowLearning.scala:54) with 1 output partitions -2016-05-23 18:56:42 INFO DAGScheduler:59 - Final stage: ResultStage 4(foreach at MatrixRowLearning.scala:54) -2016-05-23 18:56:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:56:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:56:42 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at map at MatrixRowLearning.scala:52), which has no missing parents -2016-05-23 18:56:42 INFO MemoryStore:59 - ensureFreeSpace(3720) called with curMem=552512, maxMem=766075207 -2016-05-23 18:56:42 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.6 KB, free 730.1 MB) -2016-05-23 18:56:42 INFO MemoryStore:59 - ensureFreeSpace(2055) called with curMem=556232, maxMem=766075207 -2016-05-23 18:56:42 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.1 MB) -2016-05-23 18:56:42 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:58911 (size: 2.0 KB, free: 730.5 MB) -2016-05-23 18:56:42 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:56:42 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at map at MatrixRowLearning.scala:52) -2016-05-23 18:56:42 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-05-23 18:56:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 4, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:56:42 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 4) -2016-05-23 18:56:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:56:42 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 4). 2044 bytes result sent to driver -2016-05-23 18:56:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 4) in 19 ms on localhost (1/1) -2016-05-23 18:56:42 INFO DAGScheduler:59 - ResultStage 4 (foreach at MatrixRowLearning.scala:54) finished in 0.019 s -2016-05-23 18:56:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 18:56:42 INFO DAGScheduler:59 - Job 4 finished: foreach at MatrixRowLearning.scala:54, took 0.040668 s -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 18:56:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 18:56:42 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 18:56:42 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 18:56:42 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 18:56:42 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 18:56:42 INFO BlockManager:59 - BlockManager stopped -2016-05-23 18:56:42 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 18:56:42 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 18:56:42 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 18:56:42 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 18:56:42 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-99cf7c4b-7385-4273-8cd3-9d6b1e30abba -2016-05-23 18:58:39 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 18:58:40 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 18:58:41 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 18:58:41 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 18:58:41 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 18:58:41 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 18:58:42 INFO Remoting:74 - Starting remoting -2016-05-23 18:58:42 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:58994] -2016-05-23 18:58:42 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58994. -2016-05-23 18:58:42 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 18:58:42 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 18:58:42 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-a969369a-467e-4942-a271-a31b8e0d5350 -2016-05-23 18:58:42 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 18:58:42 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-6a4164d1-57cb-4daa-805f-721137970ebc\httpd-dd1df7e2-255e-4eb1-902c-126d13dcc795 -2016-05-23 18:58:42 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 18:58:42 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:58:42 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58995 -2016-05-23 18:58:42 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58995. -2016-05-23 18:58:42 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 18:58:42 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 18:58:42 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 18:58:42 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 18:58:42 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 18:58:42 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 18:58:42 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 18:58:43 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 59002. -2016-05-23 18:58:43 INFO NettyBlockTransferService:59 - Server created on 59002 -2016-05-23 18:58:43 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 18:58:43 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:59002 with 730.6 MB RAM, BlockManagerId(driver, localhost, 59002) -2016-05-23 18:58:43 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 18:58:43 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 18:58:43 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 18:58:43 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 18:58:43 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 18:58:43 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:59002 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 18:58:43 INFO SparkContext:59 - Created broadcast 0 from textFile at MatrixRowLearning.scala:22 -2016-05-23 18:58:45 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 18:58:45 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:58:45 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-05-23 18:58:45 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 1 output partitions -2016-05-23 18:58:45 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-05-23 18:58:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:58:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:58:45 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25), which has no missing parents -2016-05-23 18:58:45 INFO MemoryStore:59 - ensureFreeSpace(3360) called with curMem=171580, maxMem=766075207 -2016-05-23 18:58:45 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-05-23 18:58:45 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=174940, maxMem=766075207 -2016-05-23 18:58:45 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.4 MB) -2016-05-23 18:58:45 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:59002 (size: 1935.0 B, free: 730.6 MB) -2016-05-23 18:58:45 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:58:45 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25) -2016-05-23 18:58:45 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-23 18:58:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:58:45 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 18:58:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:58:45 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 18:58:45 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 18:58:45 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 18:58:45 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 18:58:45 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 18:58:45 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-05-23 18:58:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 100 ms on localhost (1/1) -2016-05-23 18:58:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 18:58:45 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.136 s -2016-05-23 18:58:45 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.325264 s -2016-05-23 18:58:45 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 18:58:45 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 18:58:45 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-05-23 18:58:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:58:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:58:45 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25), which has no missing parents -2016-05-23 18:58:45 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=176875, maxMem=766075207 -2016-05-23 18:58:45 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 18:58:45 INFO MemoryStore:59 - ensureFreeSpace(1993) called with curMem=180403, maxMem=766075207 -2016-05-23 18:58:45 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1993.0 B, free 730.4 MB) -2016-05-23 18:58:45 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:59002 (size: 1993.0 B, free: 730.6 MB) -2016-05-23 18:58:45 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:58:45 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25) -2016-05-23 18:58:45 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-23 18:58:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:58:45 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-23 18:58:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:58:45 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2183 bytes result sent to driver -2016-05-23 18:58:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 26 ms on localhost (1/1) -2016-05-23 18:58:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 18:58:45 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.027 s -2016-05-23 18:58:45 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.050662 s -2016-05-23 18:58:45 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:34 -2016-05-23 18:58:45 INFO DAGScheduler:59 - Got job 2 (foreach at MatrixRowLearning.scala:34) with 1 output partitions -2016-05-23 18:58:45 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at MatrixRowLearning.scala:34) -2016-05-23 18:58:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:58:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:58:45 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25), which has no missing parents -2016-05-23 18:58:45 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=182396, maxMem=766075207 -2016-05-23 18:58:45 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 18:58:45 INFO MemoryStore:59 - ensureFreeSpace(1973) called with curMem=185876, maxMem=766075207 -2016-05-23 18:58:45 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1973.0 B, free 730.4 MB) -2016-05-23 18:58:45 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:59002 (size: 1973.0 B, free: 730.6 MB) -2016-05-23 18:58:45 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:58:45 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25) -2016-05-23 18:58:45 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 18:58:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:58:45 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-05-23 18:58:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:58:45 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 18:58:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 19 ms on localhost (1/1) -2016-05-23 18:58:45 INFO DAGScheduler:59 - ResultStage 2 (foreach at MatrixRowLearning.scala:34) finished in 0.019 s -2016-05-23 18:58:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 18:58:45 INFO DAGScheduler:59 - Job 2 finished: foreach at MatrixRowLearning.scala:34, took 0.034520 s -2016-05-23 18:58:45 INFO MemoryStore:59 - ensureFreeSpace(88528) called with curMem=187849, maxMem=766075207 -2016-05-23 18:58:45 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 86.5 KB, free 730.3 MB) -2016-05-23 18:58:45 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=276377, maxMem=766075207 -2016-05-23 18:58:45 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.3 MB) -2016-05-23 18:58:45 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:59002 (size: 19.3 KB, free: 730.5 MB) -2016-05-23 18:58:45 INFO SparkContext:59 - Created broadcast 4 from textFile at MatrixRowLearning.scala:37 -2016-05-23 18:58:45 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:58:45 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:44 -2016-05-23 18:58:45 INFO DAGScheduler:59 - Got job 3 (foreach at MatrixRowLearning.scala:44) with 1 output partitions -2016-05-23 18:58:45 INFO DAGScheduler:59 - Final stage: ResultStage 3(foreach at MatrixRowLearning.scala:44) -2016-05-23 18:58:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:58:45 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:59002 in memory (size: 1973.0 B, free: 730.6 MB) -2016-05-23 18:58:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:58:45 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at map at MatrixRowLearning.scala:41), which has no missing parents -2016-05-23 18:58:45 INFO MemoryStore:59 - ensureFreeSpace(3752) called with curMem=285191, maxMem=766075207 -2016-05-23 18:58:45 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.7 KB, free 730.3 MB) -2016-05-23 18:58:45 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:59002 in memory (size: 1993.0 B, free: 730.6 MB) -2016-05-23 18:58:45 INFO MemoryStore:59 - ensureFreeSpace(2063) called with curMem=287008, maxMem=766075207 -2016-05-23 18:58:45 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-05-23 18:58:45 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:59002 in memory (size: 1935.0 B, free: 730.6 MB) -2016-05-23 18:58:45 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:59002 (size: 2.0 KB, free: 730.6 MB) -2016-05-23 18:58:45 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:58:45 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at map at MatrixRowLearning.scala:41) -2016-05-23 18:58:45 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-05-23 18:58:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 3, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:58:46 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 3) -2016-05-23 18:58:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:58:46 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 18:58:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 3) in 17 ms on localhost (1/1) -2016-05-23 18:58:46 INFO DAGScheduler:59 - ResultStage 3 (foreach at MatrixRowLearning.scala:44) finished in 0.017 s -2016-05-23 18:58:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 18:58:46 INFO DAGScheduler:59 - Job 3 finished: foreach at MatrixRowLearning.scala:44, took 0.036310 s -2016-05-23 18:58:46 INFO MemoryStore:59 - ensureFreeSpace(230744) called with curMem=285711, maxMem=766075207 -2016-05-23 18:58:46 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-23 18:58:46 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=516455, maxMem=766075207 -2016-05-23 18:58:46 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-23 18:58:46 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:59002 (size: 19.3 KB, free: 730.5 MB) -2016-05-23 18:58:46 INFO SparkContext:59 - Created broadcast 6 from textFile at MatrixRowLearning.scala:48 -2016-05-23 18:58:46 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 18:58:46 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:54 -2016-05-23 18:58:46 INFO DAGScheduler:59 - Got job 4 (foreach at MatrixRowLearning.scala:54) with 1 output partitions -2016-05-23 18:58:46 INFO DAGScheduler:59 - Final stage: ResultStage 4(foreach at MatrixRowLearning.scala:54) -2016-05-23 18:58:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:58:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:58:46 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at map at MatrixRowLearning.scala:52), which has no missing parents -2016-05-23 18:58:46 INFO MemoryStore:59 - ensureFreeSpace(3720) called with curMem=536243, maxMem=766075207 -2016-05-23 18:58:46 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.6 KB, free 730.1 MB) -2016-05-23 18:58:46 INFO MemoryStore:59 - ensureFreeSpace(2055) called with curMem=539963, maxMem=766075207 -2016-05-23 18:58:46 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.1 MB) -2016-05-23 18:58:46 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:59002 (size: 2.0 KB, free: 730.5 MB) -2016-05-23 18:58:46 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:58:46 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at map at MatrixRowLearning.scala:52) -2016-05-23 18:58:46 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-05-23 18:58:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 4, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:58:46 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 4) -2016-05-23 18:58:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:58:46 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 4). 2044 bytes result sent to driver -2016-05-23 18:58:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 4) in 17 ms on localhost (1/1) -2016-05-23 18:58:46 INFO DAGScheduler:59 - ResultStage 4 (foreach at MatrixRowLearning.scala:54) finished in 0.017 s -2016-05-23 18:58:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 18:58:46 INFO DAGScheduler:59 - Job 4 finished: foreach at MatrixRowLearning.scala:54, took 0.089022 s -2016-05-23 18:58:46 INFO SparkContext:59 - Starting job: reduce at CoordinateMatrix.scala:153 -2016-05-23 18:58:46 INFO DAGScheduler:59 - Got job 5 (reduce at CoordinateMatrix.scala:153) with 1 output partitions -2016-05-23 18:58:46 INFO DAGScheduler:59 - Final stage: ResultStage 5(reduce at CoordinateMatrix.scala:153) -2016-05-23 18:58:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:58:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:58:46 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[14] at map at CoordinateMatrix.scala:153), which has no missing parents -2016-05-23 18:58:46 INFO MemoryStore:59 - ensureFreeSpace(3872) called with curMem=542018, maxMem=766075207 -2016-05-23 18:58:46 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 3.8 KB, free 730.1 MB) -2016-05-23 18:58:46 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=545890, maxMem=766075207 -2016-05-23 18:58:46 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.1 MB) -2016-05-23 18:58:46 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:59002 (size: 2.0 KB, free: 730.5 MB) -2016-05-23 18:58:46 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:58:46 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 5 (MapPartitionsRDD[14] at map at CoordinateMatrix.scala:153) -2016-05-23 18:58:46 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 1 tasks -2016-05-23 18:58:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 5, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:58:46 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 5) -2016-05-23 18:58:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:58:46 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 5). 2209 bytes result sent to driver -2016-05-23 18:58:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 5) in 19 ms on localhost (1/1) -2016-05-23 18:58:46 INFO DAGScheduler:59 - ResultStage 5 (reduce at CoordinateMatrix.scala:153) finished in 0.020 s -2016-05-23 18:58:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 18:58:46 INFO DAGScheduler:59 - Job 5 finished: reduce at CoordinateMatrix.scala:153, took 0.037266 s -2016-05-23 18:58:46 INFO SparkContext:59 - Starting job: countApproxDistinct at MatrixRowLearning.scala:57 -2016-05-23 18:58:46 INFO DAGScheduler:59 - Got job 6 (countApproxDistinct at MatrixRowLearning.scala:57) with 1 output partitions -2016-05-23 18:58:46 INFO DAGScheduler:59 - Final stage: ResultStage 6(countApproxDistinct at MatrixRowLearning.scala:57) -2016-05-23 18:58:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 18:58:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 18:58:46 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[13] at map at MatrixRowLearning.scala:52), which has no missing parents -2016-05-23 18:58:46 INFO MemoryStore:59 - ensureFreeSpace(4880) called with curMem=547974, maxMem=766075207 -2016-05-23 18:58:46 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 18:58:46 INFO MemoryStore:59 - ensureFreeSpace(2486) called with curMem=552854, maxMem=766075207 -2016-05-23 18:58:46 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.1 MB) -2016-05-23 18:58:46 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:59002 (size: 2.4 KB, free: 730.5 MB) -2016-05-23 18:58:46 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 18:58:46 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 6 (MapPartitionsRDD[13] at map at MatrixRowLearning.scala:52) -2016-05-23 18:58:46 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 1 tasks -2016-05-23 18:58:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 6, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 18:58:46 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 6) -2016-05-23 18:58:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 18:58:46 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 6). 2463 bytes result sent to driver -2016-05-23 18:58:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 6) in 18 ms on localhost (1/1) -2016-05-23 18:58:46 INFO DAGScheduler:59 - ResultStage 6 (countApproxDistinct at MatrixRowLearning.scala:57) finished in 0.019 s -2016-05-23 18:58:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 18:58:46 INFO DAGScheduler:59 - Job 6 finished: countApproxDistinct at MatrixRowLearning.scala:57, took 0.037445 s -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 18:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 18:58:46 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 18:58:46 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 18:58:46 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 18:58:46 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 18:58:46 INFO BlockManager:59 - BlockManager stopped -2016-05-23 18:58:46 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 18:58:46 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 18:58:46 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 18:58:46 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 18:58:46 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-6a4164d1-57cb-4daa-805f-721137970ebc -2016-05-23 19:04:18 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 19:04:19 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 19:04:19 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 19:04:19 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 19:04:19 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 19:04:20 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 19:04:20 INFO Remoting:74 - Starting remoting -2016-05-23 19:04:21 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:59067] -2016-05-23 19:04:21 INFO Utils:59 - Successfully started service 'sparkDriver' on port 59067. -2016-05-23 19:04:21 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 19:04:21 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 19:04:21 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-b3ba211f-6216-4a1d-9b64-c1490111f302 -2016-05-23 19:04:21 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 19:04:21 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-72d0df9c-ec83-442c-b135-277df970f692\httpd-d50094b9-0742-4263-b29d-cbb5c2012c79 -2016-05-23 19:04:21 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 19:04:21 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:04:21 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:59068 -2016-05-23 19:04:21 INFO Utils:59 - Successfully started service 'HTTP file server' on port 59068. -2016-05-23 19:04:21 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 19:04:21 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:04:21 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 19:04:21 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 19:04:21 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 19:04:21 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 19:04:21 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 19:04:22 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 59075. -2016-05-23 19:04:22 INFO NettyBlockTransferService:59 - Server created on 59075 -2016-05-23 19:04:22 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 19:04:22 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:59075 with 730.6 MB RAM, BlockManagerId(driver, localhost, 59075) -2016-05-23 19:04:22 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 19:04:22 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 19:04:22 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 19:04:22 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 19:04:22 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 19:04:22 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:59075 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 19:04:23 INFO SparkContext:59 - Created broadcast 0 from textFile at MatrixRowLearning.scala:22 -2016-05-23 19:04:24 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 19:04:24 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:04:24 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-05-23 19:04:24 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 1 output partitions -2016-05-23 19:04:24 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-05-23 19:04:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:04:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:04:24 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25), which has no missing parents -2016-05-23 19:04:24 INFO MemoryStore:59 - ensureFreeSpace(3360) called with curMem=171580, maxMem=766075207 -2016-05-23 19:04:24 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-05-23 19:04:24 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=174940, maxMem=766075207 -2016-05-23 19:04:24 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.4 MB) -2016-05-23 19:04:24 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:59075 (size: 1935.0 B, free: 730.6 MB) -2016-05-23 19:04:24 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:04:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25) -2016-05-23 19:04:24 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-23 19:04:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 19:04:24 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 19:04:24 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 19:04:24 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 19:04:24 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 19:04:24 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 19:04:24 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 19:04:24 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 19:04:24 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-05-23 19:04:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 102 ms on localhost (1/1) -2016-05-23 19:04:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 19:04:24 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.122 s -2016-05-23 19:04:24 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.241096 s -2016-05-23 19:04:24 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 19:04:24 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 19:04:24 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-05-23 19:04:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:04:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:04:24 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25), which has no missing parents -2016-05-23 19:04:24 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=176875, maxMem=766075207 -2016-05-23 19:04:24 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 19:04:24 INFO MemoryStore:59 - ensureFreeSpace(1993) called with curMem=180403, maxMem=766075207 -2016-05-23 19:04:24 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1993.0 B, free 730.4 MB) -2016-05-23 19:04:24 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:59075 (size: 1993.0 B, free: 730.6 MB) -2016-05-23 19:04:24 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:04:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25) -2016-05-23 19:04:24 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-23 19:04:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 19:04:24 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-23 19:04:24 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 19:04:24 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2183 bytes result sent to driver -2016-05-23 19:04:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 29 ms on localhost (1/1) -2016-05-23 19:04:24 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.029 s -2016-05-23 19:04:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 19:04:24 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.059943 s -2016-05-23 19:04:24 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:34 -2016-05-23 19:04:24 INFO DAGScheduler:59 - Got job 2 (foreach at MatrixRowLearning.scala:34) with 1 output partitions -2016-05-23 19:04:24 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at MatrixRowLearning.scala:34) -2016-05-23 19:04:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:04:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:04:24 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25), which has no missing parents -2016-05-23 19:04:24 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=182396, maxMem=766075207 -2016-05-23 19:04:24 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 19:04:24 INFO MemoryStore:59 - ensureFreeSpace(1973) called with curMem=185876, maxMem=766075207 -2016-05-23 19:04:24 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1973.0 B, free 730.4 MB) -2016-05-23 19:04:25 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:59075 (size: 1973.0 B, free: 730.6 MB) -2016-05-23 19:04:25 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:04:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25) -2016-05-23 19:04:25 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 19:04:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 19:04:25 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-05-23 19:04:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 19:04:25 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 19:04:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 26 ms on localhost (1/1) -2016-05-23 19:04:25 INFO DAGScheduler:59 - ResultStage 2 (foreach at MatrixRowLearning.scala:34) finished in 0.027 s -2016-05-23 19:04:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 19:04:25 INFO DAGScheduler:59 - Job 2 finished: foreach at MatrixRowLearning.scala:34, took 0.049591 s -2016-05-23 19:04:25 INFO MemoryStore:59 - ensureFreeSpace(88528) called with curMem=187849, maxMem=766075207 -2016-05-23 19:04:25 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 86.5 KB, free 730.3 MB) -2016-05-23 19:04:25 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=276377, maxMem=766075207 -2016-05-23 19:04:25 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.3 MB) -2016-05-23 19:04:25 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:59075 (size: 19.3 KB, free: 730.5 MB) -2016-05-23 19:04:25 INFO SparkContext:59 - Created broadcast 4 from textFile at MatrixRowLearning.scala:37 -2016-05-23 19:04:25 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:04:25 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:44 -2016-05-23 19:04:25 INFO DAGScheduler:59 - Got job 3 (foreach at MatrixRowLearning.scala:44) with 1 output partitions -2016-05-23 19:04:25 INFO DAGScheduler:59 - Final stage: ResultStage 3(foreach at MatrixRowLearning.scala:44) -2016-05-23 19:04:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:04:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:04:25 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at map at MatrixRowLearning.scala:41), which has no missing parents -2016-05-23 19:04:25 INFO MemoryStore:59 - ensureFreeSpace(3752) called with curMem=296165, maxMem=766075207 -2016-05-23 19:04:25 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.7 KB, free 730.3 MB) -2016-05-23 19:04:25 INFO MemoryStore:59 - ensureFreeSpace(2063) called with curMem=299917, maxMem=766075207 -2016-05-23 19:04:25 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-05-23 19:04:25 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:59075 (size: 2.0 KB, free: 730.5 MB) -2016-05-23 19:04:25 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:04:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at map at MatrixRowLearning.scala:41) -2016-05-23 19:04:25 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-05-23 19:04:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 3, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 19:04:25 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 3) -2016-05-23 19:04:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 19:04:25 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 19:04:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 3) in 21 ms on localhost (1/1) -2016-05-23 19:04:25 INFO DAGScheduler:59 - ResultStage 3 (foreach at MatrixRowLearning.scala:44) finished in 0.021 s -2016-05-23 19:04:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 19:04:25 INFO DAGScheduler:59 - Job 3 finished: foreach at MatrixRowLearning.scala:44, took 0.041180 s -2016-05-23 19:04:25 INFO MemoryStore:59 - ensureFreeSpace(230744) called with curMem=301980, maxMem=766075207 -2016-05-23 19:04:25 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-23 19:04:25 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=532724, maxMem=766075207 -2016-05-23 19:04:25 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-23 19:04:25 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:59075 (size: 19.3 KB, free: 730.5 MB) -2016-05-23 19:04:25 INFO SparkContext:59 - Created broadcast 6 from textFile at MatrixRowLearning.scala:48 -2016-05-23 19:04:25 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:04:25 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:54 -2016-05-23 19:04:25 INFO DAGScheduler:59 - Got job 4 (foreach at MatrixRowLearning.scala:54) with 1 output partitions -2016-05-23 19:04:25 INFO DAGScheduler:59 - Final stage: ResultStage 4(foreach at MatrixRowLearning.scala:54) -2016-05-23 19:04:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:04:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:04:25 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at map at MatrixRowLearning.scala:52), which has no missing parents -2016-05-23 19:04:25 INFO MemoryStore:59 - ensureFreeSpace(3720) called with curMem=552512, maxMem=766075207 -2016-05-23 19:04:25 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.6 KB, free 730.1 MB) -2016-05-23 19:04:25 INFO MemoryStore:59 - ensureFreeSpace(2055) called with curMem=556232, maxMem=766075207 -2016-05-23 19:04:25 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.1 MB) -2016-05-23 19:04:25 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:59075 (size: 2.0 KB, free: 730.5 MB) -2016-05-23 19:04:25 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:04:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at map at MatrixRowLearning.scala:52) -2016-05-23 19:04:25 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-05-23 19:04:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 4, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 19:04:25 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 4) -2016-05-23 19:04:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 19:04:25 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 4). 2044 bytes result sent to driver -2016-05-23 19:04:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 4) in 18 ms on localhost (1/1) -2016-05-23 19:04:25 INFO DAGScheduler:59 - ResultStage 4 (foreach at MatrixRowLearning.scala:54) finished in 0.018 s -2016-05-23 19:04:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 19:04:25 INFO DAGScheduler:59 - Job 4 finished: foreach at MatrixRowLearning.scala:54, took 0.057074 s -2016-05-23 19:04:25 INFO SparkContext:59 - Starting job: reduce at CoordinateMatrix.scala:153 -2016-05-23 19:04:25 INFO DAGScheduler:59 - Got job 5 (reduce at CoordinateMatrix.scala:153) with 1 output partitions -2016-05-23 19:04:25 INFO DAGScheduler:59 - Final stage: ResultStage 5(reduce at CoordinateMatrix.scala:153) -2016-05-23 19:04:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:04:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:04:25 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[14] at map at CoordinateMatrix.scala:153), which has no missing parents -2016-05-23 19:04:25 INFO MemoryStore:59 - ensureFreeSpace(3872) called with curMem=558287, maxMem=766075207 -2016-05-23 19:04:25 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 3.8 KB, free 730.1 MB) -2016-05-23 19:04:25 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=562159, maxMem=766075207 -2016-05-23 19:04:25 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.0 MB) -2016-05-23 19:04:25 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:59075 (size: 2.0 KB, free: 730.5 MB) -2016-05-23 19:04:25 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:04:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 5 (MapPartitionsRDD[14] at map at CoordinateMatrix.scala:153) -2016-05-23 19:04:25 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 1 tasks -2016-05-23 19:04:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 5, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 19:04:25 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 5) -2016-05-23 19:04:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 19:04:25 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 5). 2209 bytes result sent to driver -2016-05-23 19:04:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 5) in 25 ms on localhost (1/1) -2016-05-23 19:04:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 19:04:25 INFO DAGScheduler:59 - ResultStage 5 (reduce at CoordinateMatrix.scala:153) finished in 0.025 s -2016-05-23 19:04:25 INFO DAGScheduler:59 - Job 5 finished: reduce at CoordinateMatrix.scala:153, took 0.046941 s -2016-05-23 19:04:25 INFO SparkContext:59 - Starting job: countApproxDistinct at MatrixRowLearning.scala:58 -2016-05-23 19:04:25 INFO DAGScheduler:59 - Got job 6 (countApproxDistinct at MatrixRowLearning.scala:58) with 1 output partitions -2016-05-23 19:04:25 INFO DAGScheduler:59 - Final stage: ResultStage 6(countApproxDistinct at MatrixRowLearning.scala:58) -2016-05-23 19:04:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:04:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:04:25 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[13] at map at MatrixRowLearning.scala:52), which has no missing parents -2016-05-23 19:04:25 INFO MemoryStore:59 - ensureFreeSpace(4880) called with curMem=564243, maxMem=766075207 -2016-05-23 19:04:25 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 4.8 KB, free 730.0 MB) -2016-05-23 19:04:25 INFO MemoryStore:59 - ensureFreeSpace(2486) called with curMem=569123, maxMem=766075207 -2016-05-23 19:04:25 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.0 MB) -2016-05-23 19:04:25 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:59075 (size: 2.4 KB, free: 730.5 MB) -2016-05-23 19:04:25 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:04:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 6 (MapPartitionsRDD[13] at map at MatrixRowLearning.scala:52) -2016-05-23 19:04:25 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 1 tasks -2016-05-23 19:04:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 6, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 19:04:25 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 6) -2016-05-23 19:04:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 19:04:25 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 6). 2463 bytes result sent to driver -2016-05-23 19:04:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 6) in 20 ms on localhost (1/1) -2016-05-23 19:04:25 INFO DAGScheduler:59 - ResultStage 6 (countApproxDistinct at MatrixRowLearning.scala:58) finished in 0.020 s -2016-05-23 19:04:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 19:04:25 INFO DAGScheduler:59 - Job 6 finished: countApproxDistinct at MatrixRowLearning.scala:58, took 0.050761 s -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 19:04:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 19:04:25 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 19:04:25 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 19:04:25 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 19:04:25 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 19:04:25 INFO BlockManager:59 - BlockManager stopped -2016-05-23 19:04:25 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 19:04:25 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 19:04:25 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 19:04:25 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 19:04:25 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-72d0df9c-ec83-442c-b135-277df970f692 -2016-05-23 19:04:25 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 19:04:25 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 19:21:20 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 19:21:21 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 19:21:21 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 19:21:21 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 19:21:21 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 19:21:22 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 19:21:22 INFO Remoting:74 - Starting remoting -2016-05-23 19:21:23 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:59432] -2016-05-23 19:21:23 INFO Utils:59 - Successfully started service 'sparkDriver' on port 59432. -2016-05-23 19:21:23 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 19:21:23 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 19:21:23 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-542b70ce-97ad-4bfe-8fd2-d69e5a2137e5 -2016-05-23 19:21:23 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 19:21:23 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-517196ca-27a2-4320-aa7d-f7e315e299b6\httpd-d8544261-7caf-4397-9bf5-f0f9407f5f96 -2016-05-23 19:21:23 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 19:21:23 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:21:23 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:59433 -2016-05-23 19:21:23 INFO Utils:59 - Successfully started service 'HTTP file server' on port 59433. -2016-05-23 19:21:23 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 19:21:23 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:21:23 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 19:21:23 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 19:21:23 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 19:21:23 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 19:21:23 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 19:21:24 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 59452. -2016-05-23 19:21:24 INFO NettyBlockTransferService:59 - Server created on 59452 -2016-05-23 19:21:24 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 19:21:24 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:59452 with 730.6 MB RAM, BlockManagerId(driver, localhost, 59452) -2016-05-23 19:21:24 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 19:21:24 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 19:21:24 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 19:21:24 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 19:21:24 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 19:21:24 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:59452 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 19:21:24 INFO SparkContext:59 - Created broadcast 0 from textFile at SummaryLearning.scala:20 -2016-05-23 19:21:27 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 19:21:27 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:21:27 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:402 -2016-05-23 19:21:27 INFO DAGScheduler:59 - Got job 0 (treeAggregate at RowMatrix.scala:402) with 2 output partitions -2016-05-23 19:21:27 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at RowMatrix.scala:402) -2016-05-23 19:21:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:21:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:21:27 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402), which has no missing parents -2016-05-23 19:21:27 INFO MemoryStore:59 - ensureFreeSpace(4464) called with curMem=171580, maxMem=766075207 -2016-05-23 19:21:27 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-23 19:21:27 INFO MemoryStore:59 - ensureFreeSpace(2374) called with curMem=176044, maxMem=766075207 -2016-05-23 19:21:27 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 19:21:27 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:59452 (size: 2.3 KB, free: 730.6 MB) -2016-05-23 19:21:27 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:21:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402) -2016-05-23 19:21:27 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 19:21:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 19:21:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 19:21:27 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 19:21:27 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 19:21:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:6+6 -2016-05-23 19:21:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+6 -2016-05-23 19:21:27 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 19:21:27 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 19:21:27 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 19:21:27 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 19:21:27 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 19:21:27 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2595 bytes result sent to driver -2016-05-23 19:21:27 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2595 bytes result sent to driver -2016-05-23 19:21:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 119 ms on localhost (1/2) -2016-05-23 19:21:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 157 ms on localhost (2/2) -2016-05-23 19:21:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 19:21:27 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at RowMatrix.scala:402) finished in 0.182 s -2016-05-23 19:21:27 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at RowMatrix.scala:402, took 0.292605 s -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 19:21:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 19:21:27 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 19:21:27 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 19:21:27 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 19:21:27 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 19:21:27 INFO BlockManager:59 - BlockManager stopped -2016-05-23 19:21:27 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 19:21:27 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 19:21:27 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 19:21:27 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 19:21:27 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-517196ca-27a2-4320-aa7d-f7e315e299b6 -2016-05-23 19:21:27 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 19:21:27 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 19:22:00 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 19:22:01 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 19:22:01 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 19:22:01 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 19:22:01 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 19:22:02 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 19:22:02 INFO Remoting:74 - Starting remoting -2016-05-23 19:22:02 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:59474] -2016-05-23 19:22:02 INFO Utils:59 - Successfully started service 'sparkDriver' on port 59474. -2016-05-23 19:22:02 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 19:22:02 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 19:22:02 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-4f816d61-dfe5-409b-b753-f959b4a523c6 -2016-05-23 19:22:02 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 19:22:02 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-e241e501-0562-4be5-914f-7820a639a8c3\httpd-d92836f1-038e-48a0-a3f0-6edbf1436028 -2016-05-23 19:22:02 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 19:22:02 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:22:02 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:59475 -2016-05-23 19:22:02 INFO Utils:59 - Successfully started service 'HTTP file server' on port 59475. -2016-05-23 19:22:02 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 19:22:02 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:22:02 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 19:22:02 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 19:22:02 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 19:22:03 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 19:22:03 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 19:22:03 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 59494. -2016-05-23 19:22:03 INFO NettyBlockTransferService:59 - Server created on 59494 -2016-05-23 19:22:03 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 19:22:03 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:59494 with 730.6 MB RAM, BlockManagerId(driver, localhost, 59494) -2016-05-23 19:22:03 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 19:22:04 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 19:22:04 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 19:22:04 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 19:22:04 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 19:22:04 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:59494 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 19:22:04 INFO SparkContext:59 - Created broadcast 0 from textFile at SummaryLearning.scala:21 -2016-05-23 19:22:05 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 19:22:05 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:22:05 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:402 -2016-05-23 19:22:05 INFO DAGScheduler:59 - Got job 0 (treeAggregate at RowMatrix.scala:402) with 3 output partitions -2016-05-23 19:22:05 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at RowMatrix.scala:402) -2016-05-23 19:22:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:22:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:22:05 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402), which has no missing parents -2016-05-23 19:22:05 INFO MemoryStore:59 - ensureFreeSpace(4464) called with curMem=171580, maxMem=766075207 -2016-05-23 19:22:05 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-23 19:22:05 INFO MemoryStore:59 - ensureFreeSpace(2372) called with curMem=176044, maxMem=766075207 -2016-05-23 19:22:05 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 19:22:05 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:59494 (size: 2.3 KB, free: 730.6 MB) -2016-05-23 19:22:05 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:22:05 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402) -2016-05-23 19:22:05 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-05-23 19:22:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:22:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:22:05 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:22:05 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 19:22:05 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-05-23 19:22:05 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 19:22:05 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 19:22:05 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 19:22:05 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 19:22:05 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 19:22:05 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 19:22:05 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 19:22:05 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 19:22:05 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 19:22:06 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2483 bytes result sent to driver -2016-05-23 19:22:06 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 2351 bytes result sent to driver -2016-05-23 19:22:06 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2483 bytes result sent to driver -2016-05-23 19:22:06 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 128 ms on localhost (1/3) -2016-05-23 19:22:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 134 ms on localhost (2/3) -2016-05-23 19:22:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 157 ms on localhost (3/3) -2016-05-23 19:22:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 19:22:06 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at RowMatrix.scala:402) finished in 0.178 s -2016-05-23 19:22:06 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at RowMatrix.scala:402, took 0.301802 s -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 19:22:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 19:22:06 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 19:22:06 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 19:22:06 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 19:22:06 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 19:22:06 INFO BlockManager:59 - BlockManager stopped -2016-05-23 19:22:06 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 19:22:06 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 19:22:06 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 19:22:06 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 19:22:06 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-e241e501-0562-4be5-914f-7820a639a8c3 -2016-05-23 19:22:06 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 19:22:51 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 19:22:51 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 19:22:52 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 19:22:52 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 19:22:52 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 19:22:53 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 19:22:53 INFO Remoting:74 - Starting remoting -2016-05-23 19:22:53 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:59521] -2016-05-23 19:22:53 INFO Utils:59 - Successfully started service 'sparkDriver' on port 59521. -2016-05-23 19:22:53 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 19:22:53 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 19:22:53 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-791a44d2-4244-4830-97cd-a0e7470004de -2016-05-23 19:22:53 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 19:22:53 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-04b9a2de-1c80-46b5-a4a6-2e2bfc7e813c\httpd-b881c717-8a39-42eb-b34b-7cbcf93f1bbf -2016-05-23 19:22:53 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 19:22:53 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:22:53 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:59522 -2016-05-23 19:22:53 INFO Utils:59 - Successfully started service 'HTTP file server' on port 59522. -2016-05-23 19:22:53 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 19:22:53 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:22:53 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 19:22:53 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 19:22:53 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 19:22:53 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 19:22:53 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 19:22:54 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 59541. -2016-05-23 19:22:54 INFO NettyBlockTransferService:59 - Server created on 59541 -2016-05-23 19:22:54 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 19:22:54 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:59541 with 730.6 MB RAM, BlockManagerId(driver, localhost, 59541) -2016-05-23 19:22:54 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 19:22:54 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 19:22:54 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 19:22:55 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 19:22:55 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 19:22:55 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:59541 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 19:22:55 INFO SparkContext:59 - Created broadcast 0 from textFile at SummaryLearning.scala:21 -2016-05-23 19:22:56 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 19:22:56 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:22:56 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:402 -2016-05-23 19:22:56 INFO DAGScheduler:59 - Got job 0 (treeAggregate at RowMatrix.scala:402) with 3 output partitions -2016-05-23 19:22:56 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at RowMatrix.scala:402) -2016-05-23 19:22:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:22:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:22:56 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402), which has no missing parents -2016-05-23 19:22:56 INFO MemoryStore:59 - ensureFreeSpace(4464) called with curMem=171580, maxMem=766075207 -2016-05-23 19:22:56 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-23 19:22:56 INFO MemoryStore:59 - ensureFreeSpace(2372) called with curMem=176044, maxMem=766075207 -2016-05-23 19:22:56 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 19:22:56 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:59541 (size: 2.3 KB, free: 730.6 MB) -2016-05-23 19:22:56 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:22:56 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402) -2016-05-23 19:22:56 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-05-23 19:22:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:22:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:22:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:22:56 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-05-23 19:22:56 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 19:22:56 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 19:22:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 19:22:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 19:22:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 19:22:56 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 19:22:56 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 19:22:56 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 19:22:56 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 19:22:56 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 19:22:56 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2483 bytes result sent to driver -2016-05-23 19:22:56 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 2351 bytes result sent to driver -2016-05-23 19:22:56 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2483 bytes result sent to driver -2016-05-23 19:22:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 98 ms on localhost (1/3) -2016-05-23 19:22:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 121 ms on localhost (2/3) -2016-05-23 19:22:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 103 ms on localhost (3/3) -2016-05-23 19:22:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 19:22:56 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at RowMatrix.scala:402) finished in 0.140 s -2016-05-23 19:22:56 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at RowMatrix.scala:402, took 0.227977 s -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 19:22:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 19:22:56 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 19:22:56 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 19:22:57 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 19:22:57 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 19:22:57 INFO BlockManager:59 - BlockManager stopped -2016-05-23 19:22:57 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 19:22:57 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 19:22:57 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 19:22:57 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 19:22:57 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-04b9a2de-1c80-46b5-a4a6-2e2bfc7e813c -2016-05-23 19:22:57 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 19:27:04 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 19:27:05 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 19:27:06 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 19:27:06 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 19:27:06 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 19:27:06 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 19:27:07 INFO Remoting:74 - Starting remoting -2016-05-23 19:27:07 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:59571] -2016-05-23 19:27:07 INFO Utils:59 - Successfully started service 'sparkDriver' on port 59571. -2016-05-23 19:27:07 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 19:27:07 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 19:27:07 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-28ed1534-f84f-42e8-b6d2-9945dfae3282 -2016-05-23 19:27:07 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 19:27:07 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-774464e9-1ae0-4de3-8636-c85d07dc9284\httpd-4c9d259b-8d2c-433f-a334-355632df7475 -2016-05-23 19:27:07 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 19:27:07 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:27:07 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:59572 -2016-05-23 19:27:07 INFO Utils:59 - Successfully started service 'HTTP file server' on port 59572. -2016-05-23 19:27:07 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 19:27:07 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:27:07 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 19:27:07 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 19:27:07 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 19:27:07 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 19:27:07 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 19:27:08 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 59591. -2016-05-23 19:27:08 INFO NettyBlockTransferService:59 - Server created on 59591 -2016-05-23 19:27:08 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 19:27:08 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:59591 with 730.6 MB RAM, BlockManagerId(driver, localhost, 59591) -2016-05-23 19:27:08 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 19:27:08 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 19:27:08 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 19:27:09 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 19:27:09 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 19:27:09 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:59591 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 19:27:09 INFO SparkContext:59 - Created broadcast 0 from textFile at SummaryLearning.scala:21 -2016-05-23 19:27:10 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 19:27:10 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:27:10 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:402 -2016-05-23 19:27:10 INFO DAGScheduler:59 - Got job 0 (treeAggregate at RowMatrix.scala:402) with 3 output partitions -2016-05-23 19:27:10 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at RowMatrix.scala:402) -2016-05-23 19:27:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:27:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:27:10 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402), which has no missing parents -2016-05-23 19:27:10 INFO MemoryStore:59 - ensureFreeSpace(4480) called with curMem=171580, maxMem=766075207 -2016-05-23 19:27:10 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-23 19:27:10 INFO MemoryStore:59 - ensureFreeSpace(2386) called with curMem=176060, maxMem=766075207 -2016-05-23 19:27:10 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 19:27:10 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:59591 (size: 2.3 KB, free: 730.6 MB) -2016-05-23 19:27:10 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:27:10 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402) -2016-05-23 19:27:10 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-05-23 19:27:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:27:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:27:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:27:10 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 19:27:10 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 19:27:10 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-05-23 19:27:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 19:27:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 19:27:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 19:27:10 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 19:27:10 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 19:27:10 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 19:27:10 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 19:27:10 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 19:27:10 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2483 bytes result sent to driver -2016-05-23 19:27:10 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2483 bytes result sent to driver -2016-05-23 19:27:10 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 2351 bytes result sent to driver -2016-05-23 19:27:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 140 ms on localhost (1/3) -2016-05-23 19:27:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 164 ms on localhost (2/3) -2016-05-23 19:27:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 142 ms on localhost (3/3) -2016-05-23 19:27:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 19:27:11 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at RowMatrix.scala:402) finished in 0.186 s -2016-05-23 19:27:11 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at RowMatrix.scala:402, took 0.281451 s -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 19:27:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 19:27:11 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 19:27:11 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 19:27:11 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 19:27:11 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 19:27:11 INFO BlockManager:59 - BlockManager stopped -2016-05-23 19:27:11 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 19:27:11 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 19:27:11 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 19:27:11 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 19:27:11 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-774464e9-1ae0-4de3-8636-c85d07dc9284 -2016-05-23 19:27:21 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 19:27:21 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 19:27:22 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 19:27:22 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 19:27:22 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 19:27:23 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 19:27:23 INFO Remoting:74 - Starting remoting -2016-05-23 19:27:23 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:59614] -2016-05-23 19:27:23 INFO Utils:59 - Successfully started service 'sparkDriver' on port 59614. -2016-05-23 19:27:23 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 19:27:23 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 19:27:23 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-1392379a-46cf-490e-819c-47bf4b76c1fe -2016-05-23 19:27:23 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 19:27:23 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-9ba8b96f-5d0e-4c72-9d5c-97e388643f63\httpd-def5e583-49e9-4672-bcd9-c6c51b84caa7 -2016-05-23 19:27:23 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 19:27:23 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:27:23 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:59615 -2016-05-23 19:27:23 INFO Utils:59 - Successfully started service 'HTTP file server' on port 59615. -2016-05-23 19:27:23 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 19:27:23 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:27:23 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 19:27:23 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 19:27:23 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 19:27:24 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 19:27:24 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 19:27:24 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 59637. -2016-05-23 19:27:24 INFO NettyBlockTransferService:59 - Server created on 59637 -2016-05-23 19:27:24 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 19:27:24 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:59637 with 730.6 MB RAM, BlockManagerId(driver, localhost, 59637) -2016-05-23 19:27:24 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 19:27:25 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 19:27:25 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 19:27:25 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 19:27:25 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 19:27:25 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:59637 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 19:27:25 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsColStatsLearning.scala:21 -2016-05-23 19:27:26 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 19:27:26 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:27:26 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:402 -2016-05-23 19:27:26 INFO DAGScheduler:59 - Got job 0 (treeAggregate at RowMatrix.scala:402) with 3 output partitions -2016-05-23 19:27:26 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at RowMatrix.scala:402) -2016-05-23 19:27:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:27:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:27:26 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402), which has no missing parents -2016-05-23 19:27:27 INFO MemoryStore:59 - ensureFreeSpace(4480) called with curMem=171580, maxMem=766075207 -2016-05-23 19:27:27 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-23 19:27:27 INFO MemoryStore:59 - ensureFreeSpace(2386) called with curMem=176060, maxMem=766075207 -2016-05-23 19:27:27 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 19:27:27 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:59637 (size: 2.3 KB, free: 730.6 MB) -2016-05-23 19:27:27 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:27:27 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402) -2016-05-23 19:27:27 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-05-23 19:27:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:27:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:27:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:27:27 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-05-23 19:27:27 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 19:27:27 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 19:27:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 19:27:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 19:27:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 19:27:27 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 19:27:27 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 19:27:27 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 19:27:27 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 19:27:27 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 19:27:27 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 2351 bytes result sent to driver -2016-05-23 19:27:27 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2483 bytes result sent to driver -2016-05-23 19:27:27 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2483 bytes result sent to driver -2016-05-23 19:27:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 118 ms on localhost (1/3) -2016-05-23 19:27:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 143 ms on localhost (2/3) -2016-05-23 19:27:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 125 ms on localhost (3/3) -2016-05-23 19:27:27 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at RowMatrix.scala:402) finished in 0.161 s -2016-05-23 19:27:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 19:27:27 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at RowMatrix.scala:402, took 0.388766 s -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 19:27:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 19:27:27 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 19:27:27 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 19:27:27 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 19:27:27 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 19:27:27 INFO BlockManager:59 - BlockManager stopped -2016-05-23 19:27:27 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 19:27:27 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 19:27:27 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 19:27:27 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 19:27:27 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-9ba8b96f-5d0e-4c72-9d5c-97e388643f63 -2016-05-23 19:27:27 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 19:33:28 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 19:33:29 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 19:33:29 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 19:33:29 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 19:33:29 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 19:33:30 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 19:33:30 INFO Remoting:74 - Starting remoting -2016-05-23 19:33:31 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:51121] -2016-05-23 19:33:31 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51121. -2016-05-23 19:33:31 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 19:33:31 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 19:33:31 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-fdcb588a-0ee2-45b4-a1b3-8529ff915bf8 -2016-05-23 19:33:31 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 19:33:31 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-16b4b7b3-8260-40fc-88c0-c37d5ba86bb8\httpd-e86411ce-0e17-47bf-beb6-4ea70290f35b -2016-05-23 19:33:31 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 19:33:31 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:33:31 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51122 -2016-05-23 19:33:31 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51122. -2016-05-23 19:33:31 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 19:33:31 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:33:31 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 19:33:31 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 19:33:31 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 19:33:31 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 19:33:31 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 19:33:32 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51141. -2016-05-23 19:33:32 INFO NettyBlockTransferService:59 - Server created on 51141 -2016-05-23 19:33:32 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 19:33:32 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51141 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51141) -2016-05-23 19:33:32 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 19:33:33 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 19:33:33 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 19:33:33 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 19:33:33 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 19:33:33 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51141 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 19:33:33 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsColStatsLearning.scala:21 -2016-05-23 19:33:34 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 19:33:34 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:33:34 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:402 -2016-05-23 19:33:34 INFO DAGScheduler:59 - Got job 0 (treeAggregate at RowMatrix.scala:402) with 3 output partitions -2016-05-23 19:33:34 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at RowMatrix.scala:402) -2016-05-23 19:33:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:33:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:33:34 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402), which has no missing parents -2016-05-23 19:33:34 INFO MemoryStore:59 - ensureFreeSpace(4480) called with curMem=171580, maxMem=766075207 -2016-05-23 19:33:34 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-23 19:33:34 INFO MemoryStore:59 - ensureFreeSpace(2386) called with curMem=176060, maxMem=766075207 -2016-05-23 19:33:34 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 19:33:34 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51141 (size: 2.3 KB, free: 730.6 MB) -2016-05-23 19:33:34 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:33:34 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402) -2016-05-23 19:33:34 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-05-23 19:33:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:33:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:33:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:33:34 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 19:33:34 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-05-23 19:33:34 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 19:33:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 19:33:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 19:33:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 19:33:34 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 19:33:34 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 19:33:34 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 19:33:34 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 19:33:34 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 19:33:34 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2483 bytes result sent to driver -2016-05-23 19:33:34 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2483 bytes result sent to driver -2016-05-23 19:33:34 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 2351 bytes result sent to driver -2016-05-23 19:33:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 104 ms on localhost (1/3) -2016-05-23 19:33:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 130 ms on localhost (2/3) -2016-05-23 19:33:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 109 ms on localhost (3/3) -2016-05-23 19:33:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 19:33:34 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at RowMatrix.scala:402) finished in 0.145 s -2016-05-23 19:33:34 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at RowMatrix.scala:402, took 0.302638 s -2016-05-23 19:33:35 INFO SparkContext:59 - Starting job: foreach at StatisticsColStatsLearning.scala:26 -2016-05-23 19:33:35 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsColStatsLearning.scala:26) with 3 output partitions -2016-05-23 19:33:35 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsColStatsLearning.scala:26) -2016-05-23 19:33:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:33:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:33:35 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24), which has no missing parents -2016-05-23 19:33:35 INFO MemoryStore:59 - ensureFreeSpace(3504) called with curMem=178446, maxMem=766075207 -2016-05-23 19:33:35 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 19:33:35 INFO MemoryStore:59 - ensureFreeSpace(1984) called with curMem=181950, maxMem=766075207 -2016-05-23 19:33:35 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1984.0 B, free 730.4 MB) -2016-05-23 19:33:35 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51141 (size: 1984.0 B, free: 730.6 MB) -2016-05-23 19:33:35 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:33:35 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24) -2016-05-23 19:33:35 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 3 tasks -2016-05-23 19:33:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:33:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:33:35 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 5, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:33:35 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 5) -2016-05-23 19:33:35 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 3) -2016-05-23 19:33:35 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 4) -2016-05-23 19:33:35 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 19:33:35 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 19:33:35 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 19:33:35 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 5). 2044 bytes result sent to driver -2016-05-23 19:33:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 5) in 22 ms on localhost (1/3) -2016-05-23 19:33:35 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 19:33:35 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 4). 2044 bytes result sent to driver -2016-05-23 19:33:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 3) in 28 ms on localhost (2/3) -2016-05-23 19:33:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 4) in 27 ms on localhost (3/3) -2016-05-23 19:33:35 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsColStatsLearning.scala:26) finished in 0.028 s -2016-05-23 19:33:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 19:33:35 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsColStatsLearning.scala:26, took 0.048361 s -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 19:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 19:33:35 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 19:33:35 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 19:33:35 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 19:33:35 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 19:33:35 INFO BlockManager:59 - BlockManager stopped -2016-05-23 19:33:35 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 19:33:35 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 19:33:35 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 19:33:35 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 19:33:35 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-16b4b7b3-8260-40fc-88c0-c37d5ba86bb8 -2016-05-23 19:33:35 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 19:33:35 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 19:38:44 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 19:38:45 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 19:38:45 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 19:38:45 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 19:38:45 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 19:38:46 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 19:38:46 INFO Remoting:74 - Starting remoting -2016-05-23 19:38:46 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:51428] -2016-05-23 19:38:46 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51428. -2016-05-23 19:38:46 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 19:38:46 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 19:38:46 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-fc12d17b-e0e3-4c5f-9272-fe6498199cd8 -2016-05-23 19:38:46 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 19:38:47 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-7d2da32f-0d38-4353-8722-0556bf6559a2\httpd-da4df36d-1004-4c81-918a-6e6e10a65f9d -2016-05-23 19:38:47 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 19:38:47 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:38:47 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51429 -2016-05-23 19:38:47 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51429. -2016-05-23 19:38:47 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 19:38:47 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:38:47 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 19:38:47 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 19:38:47 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 19:38:47 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 19:38:47 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 19:38:47 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51436. -2016-05-23 19:38:47 INFO NettyBlockTransferService:59 - Server created on 51436 -2016-05-23 19:38:47 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 19:38:47 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51436 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51436) -2016-05-23 19:38:47 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 19:38:48 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 19:38:48 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 19:38:48 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 19:38:48 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 19:38:48 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51436 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 19:38:48 INFO SparkContext:59 - Created broadcast 0 from textFile at MatrixRowLearning.scala:22 -2016-05-23 19:38:49 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 19:38:50 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:38:50 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-05-23 19:38:50 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 1 output partitions -2016-05-23 19:38:50 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-05-23 19:38:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:38:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:38:50 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25), which has no missing parents -2016-05-23 19:38:50 INFO MemoryStore:59 - ensureFreeSpace(3360) called with curMem=171580, maxMem=766075207 -2016-05-23 19:38:50 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-05-23 19:38:50 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=174940, maxMem=766075207 -2016-05-23 19:38:50 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.4 MB) -2016-05-23 19:38:50 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51436 (size: 1935.0 B, free: 730.6 MB) -2016-05-23 19:38:50 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:38:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25) -2016-05-23 19:38:50 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-23 19:38:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 19:38:50 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 19:38:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 19:38:50 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 19:38:50 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 19:38:50 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 19:38:50 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 19:38:50 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 19:38:50 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-05-23 19:38:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 101 ms on localhost (1/1) -2016-05-23 19:38:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 19:38:50 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.122 s -2016-05-23 19:38:50 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.237028 s -2016-05-23 19:38:50 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 19:38:50 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 19:38:50 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-05-23 19:38:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:38:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:38:50 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25), which has no missing parents -2016-05-23 19:38:50 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=176875, maxMem=766075207 -2016-05-23 19:38:50 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 19:38:50 INFO MemoryStore:59 - ensureFreeSpace(1993) called with curMem=180403, maxMem=766075207 -2016-05-23 19:38:50 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1993.0 B, free 730.4 MB) -2016-05-23 19:38:50 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51436 (size: 1993.0 B, free: 730.6 MB) -2016-05-23 19:38:50 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:38:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25) -2016-05-23 19:38:50 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-23 19:38:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 19:38:50 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-23 19:38:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 19:38:50 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2183 bytes result sent to driver -2016-05-23 19:38:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 22 ms on localhost (1/1) -2016-05-23 19:38:50 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.022 s -2016-05-23 19:38:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 19:38:50 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.048370 s -2016-05-23 19:38:50 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:34 -2016-05-23 19:38:50 INFO DAGScheduler:59 - Got job 2 (foreach at MatrixRowLearning.scala:34) with 1 output partitions -2016-05-23 19:38:50 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at MatrixRowLearning.scala:34) -2016-05-23 19:38:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:38:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:38:50 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25), which has no missing parents -2016-05-23 19:38:50 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=182396, maxMem=766075207 -2016-05-23 19:38:50 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 19:38:50 INFO MemoryStore:59 - ensureFreeSpace(1973) called with curMem=185876, maxMem=766075207 -2016-05-23 19:38:50 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1973.0 B, free 730.4 MB) -2016-05-23 19:38:50 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:51436 (size: 1973.0 B, free: 730.6 MB) -2016-05-23 19:38:50 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:38:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25) -2016-05-23 19:38:50 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 19:38:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 19:38:50 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-05-23 19:38:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 19:38:50 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 19:38:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 19 ms on localhost (1/1) -2016-05-23 19:38:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 19:38:50 INFO DAGScheduler:59 - ResultStage 2 (foreach at MatrixRowLearning.scala:34) finished in 0.021 s -2016-05-23 19:38:50 INFO DAGScheduler:59 - Job 2 finished: foreach at MatrixRowLearning.scala:34, took 0.034850 s -2016-05-23 19:38:50 INFO MemoryStore:59 - ensureFreeSpace(88528) called with curMem=187849, maxMem=766075207 -2016-05-23 19:38:50 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 86.5 KB, free 730.3 MB) -2016-05-23 19:38:50 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=276377, maxMem=766075207 -2016-05-23 19:38:50 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.3 MB) -2016-05-23 19:38:50 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:51436 (size: 19.3 KB, free: 730.5 MB) -2016-05-23 19:38:50 INFO SparkContext:59 - Created broadcast 4 from textFile at MatrixRowLearning.scala:37 -2016-05-23 19:38:50 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:38:50 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:44 -2016-05-23 19:38:50 INFO DAGScheduler:59 - Got job 3 (foreach at MatrixRowLearning.scala:44) with 1 output partitions -2016-05-23 19:38:50 INFO DAGScheduler:59 - Final stage: ResultStage 3(foreach at MatrixRowLearning.scala:44) -2016-05-23 19:38:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:38:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:38:50 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at map at MatrixRowLearning.scala:41), which has no missing parents -2016-05-23 19:38:50 INFO MemoryStore:59 - ensureFreeSpace(3752) called with curMem=296165, maxMem=766075207 -2016-05-23 19:38:50 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.7 KB, free 730.3 MB) -2016-05-23 19:38:50 INFO MemoryStore:59 - ensureFreeSpace(2063) called with curMem=299917, maxMem=766075207 -2016-05-23 19:38:50 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-05-23 19:38:50 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:51436 (size: 2.0 KB, free: 730.5 MB) -2016-05-23 19:38:50 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:38:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at map at MatrixRowLearning.scala:41) -2016-05-23 19:38:50 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-05-23 19:38:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 3, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 19:38:50 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 3) -2016-05-23 19:38:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 19:38:50 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 19:38:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 3) in 17 ms on localhost (1/1) -2016-05-23 19:38:50 INFO DAGScheduler:59 - ResultStage 3 (foreach at MatrixRowLearning.scala:44) finished in 0.018 s -2016-05-23 19:38:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 19:38:50 INFO DAGScheduler:59 - Job 3 finished: foreach at MatrixRowLearning.scala:44, took 0.032301 s -2016-05-23 19:38:50 INFO MemoryStore:59 - ensureFreeSpace(230744) called with curMem=301980, maxMem=766075207 -2016-05-23 19:38:50 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-23 19:38:50 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=532724, maxMem=766075207 -2016-05-23 19:38:50 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-23 19:38:50 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:51436 (size: 19.3 KB, free: 730.5 MB) -2016-05-23 19:38:50 INFO SparkContext:59 - Created broadcast 6 from textFile at MatrixRowLearning.scala:48 -2016-05-23 19:38:50 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:38:50 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:54 -2016-05-23 19:38:50 INFO DAGScheduler:59 - Got job 4 (foreach at MatrixRowLearning.scala:54) with 1 output partitions -2016-05-23 19:38:50 INFO DAGScheduler:59 - Final stage: ResultStage 4(foreach at MatrixRowLearning.scala:54) -2016-05-23 19:38:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:38:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:38:50 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at map at MatrixRowLearning.scala:52), which has no missing parents -2016-05-23 19:38:50 INFO MemoryStore:59 - ensureFreeSpace(3720) called with curMem=552512, maxMem=766075207 -2016-05-23 19:38:50 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.6 KB, free 730.1 MB) -2016-05-23 19:38:50 INFO MemoryStore:59 - ensureFreeSpace(2055) called with curMem=556232, maxMem=766075207 -2016-05-23 19:38:50 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.1 MB) -2016-05-23 19:38:50 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:51436 (size: 2.0 KB, free: 730.5 MB) -2016-05-23 19:38:50 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:38:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at map at MatrixRowLearning.scala:52) -2016-05-23 19:38:50 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-05-23 19:38:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 4, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 19:38:50 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 4) -2016-05-23 19:38:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 19:38:50 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 4). 2044 bytes result sent to driver -2016-05-23 19:38:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 4) in 18 ms on localhost (1/1) -2016-05-23 19:38:50 INFO DAGScheduler:59 - ResultStage 4 (foreach at MatrixRowLearning.scala:54) finished in 0.018 s -2016-05-23 19:38:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 19:38:50 INFO DAGScheduler:59 - Job 4 finished: foreach at MatrixRowLearning.scala:54, took 0.058618 s -2016-05-23 19:38:50 INFO SparkContext:59 - Starting job: reduce at CoordinateMatrix.scala:153 -2016-05-23 19:38:50 INFO DAGScheduler:59 - Got job 5 (reduce at CoordinateMatrix.scala:153) with 1 output partitions -2016-05-23 19:38:50 INFO DAGScheduler:59 - Final stage: ResultStage 5(reduce at CoordinateMatrix.scala:153) -2016-05-23 19:38:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:38:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:38:50 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[14] at map at CoordinateMatrix.scala:153), which has no missing parents -2016-05-23 19:38:50 INFO MemoryStore:59 - ensureFreeSpace(3872) called with curMem=558287, maxMem=766075207 -2016-05-23 19:38:50 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 3.8 KB, free 730.1 MB) -2016-05-23 19:38:50 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=562159, maxMem=766075207 -2016-05-23 19:38:50 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.0 MB) -2016-05-23 19:38:50 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:51436 (size: 2.0 KB, free: 730.5 MB) -2016-05-23 19:38:50 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:38:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 5 (MapPartitionsRDD[14] at map at CoordinateMatrix.scala:153) -2016-05-23 19:38:50 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 1 tasks -2016-05-23 19:38:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 5, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 19:38:50 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 5) -2016-05-23 19:38:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 19:38:50 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 5). 2209 bytes result sent to driver -2016-05-23 19:38:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 5) in 19 ms on localhost (1/1) -2016-05-23 19:38:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 19:38:50 INFO DAGScheduler:59 - ResultStage 5 (reduce at CoordinateMatrix.scala:153) finished in 0.019 s -2016-05-23 19:38:50 INFO DAGScheduler:59 - Job 5 finished: reduce at CoordinateMatrix.scala:153, took 0.038453 s -2016-05-23 19:38:51 INFO SparkContext:59 - Starting job: countApproxDistinct at MatrixRowLearning.scala:58 -2016-05-23 19:38:51 INFO DAGScheduler:59 - Got job 6 (countApproxDistinct at MatrixRowLearning.scala:58) with 1 output partitions -2016-05-23 19:38:51 INFO DAGScheduler:59 - Final stage: ResultStage 6(countApproxDistinct at MatrixRowLearning.scala:58) -2016-05-23 19:38:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:38:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:38:51 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[13] at map at MatrixRowLearning.scala:52), which has no missing parents -2016-05-23 19:38:51 INFO MemoryStore:59 - ensureFreeSpace(4880) called with curMem=564243, maxMem=766075207 -2016-05-23 19:38:51 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 4.8 KB, free 730.0 MB) -2016-05-23 19:38:51 INFO MemoryStore:59 - ensureFreeSpace(2486) called with curMem=569123, maxMem=766075207 -2016-05-23 19:38:51 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.0 MB) -2016-05-23 19:38:51 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:51436 (size: 2.4 KB, free: 730.5 MB) -2016-05-23 19:38:51 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:38:51 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 6 (MapPartitionsRDD[13] at map at MatrixRowLearning.scala:52) -2016-05-23 19:38:51 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 1 tasks -2016-05-23 19:38:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 6, localhost, PROCESS_LOCAL, 2182 bytes) -2016-05-23 19:38:51 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 6) -2016-05-23 19:38:51 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-05-23 19:38:51 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 6). 2463 bytes result sent to driver -2016-05-23 19:38:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 6) in 17 ms on localhost (1/1) -2016-05-23 19:38:51 INFO DAGScheduler:59 - ResultStage 6 (countApproxDistinct at MatrixRowLearning.scala:58) finished in 0.017 s -2016-05-23 19:38:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 19:38:51 INFO DAGScheduler:59 - Job 6 finished: countApproxDistinct at MatrixRowLearning.scala:58, took 0.033448 s -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 19:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 19:38:51 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 19:38:51 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 19:38:51 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 19:38:51 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 19:38:51 INFO BlockManager:59 - BlockManager stopped -2016-05-23 19:38:51 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 19:38:51 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 19:38:51 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 19:38:51 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 19:38:51 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-7d2da32f-0d38-4353-8722-0556bf6559a2 -2016-05-23 19:38:51 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 19:38:51 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 19:39:15 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 19:39:16 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 19:39:16 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 19:39:16 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 19:39:16 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 19:39:17 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 19:39:17 INFO Remoting:74 - Starting remoting -2016-05-23 19:39:18 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:51458] -2016-05-23 19:39:18 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51458. -2016-05-23 19:39:18 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 19:39:18 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 19:39:18 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-13932036-c486-425c-b35e-9112ef39415c -2016-05-23 19:39:18 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 19:39:18 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-39c5fefb-8918-45ea-86dd-ddb4f005740f\httpd-b6a3c630-c324-4755-906e-8c669a924794 -2016-05-23 19:39:18 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 19:39:18 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:39:18 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51459 -2016-05-23 19:39:18 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51459. -2016-05-23 19:39:18 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 19:39:18 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:39:18 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 19:39:18 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 19:39:18 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 19:39:18 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 19:39:18 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 19:39:18 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51478. -2016-05-23 19:39:18 INFO NettyBlockTransferService:59 - Server created on 51478 -2016-05-23 19:39:18 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 19:39:18 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51478 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51478) -2016-05-23 19:39:18 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 19:39:19 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 19:39:19 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 19:39:19 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 19:39:19 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 19:39:19 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51478 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 19:39:19 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsColStatsLearning.scala:21 -2016-05-23 19:39:20 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 19:39:21 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:39:21 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:402 -2016-05-23 19:39:21 INFO DAGScheduler:59 - Got job 0 (treeAggregate at RowMatrix.scala:402) with 3 output partitions -2016-05-23 19:39:21 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at RowMatrix.scala:402) -2016-05-23 19:39:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:39:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:39:21 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402), which has no missing parents -2016-05-23 19:39:21 INFO MemoryStore:59 - ensureFreeSpace(4480) called with curMem=171580, maxMem=766075207 -2016-05-23 19:39:21 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-23 19:39:21 INFO MemoryStore:59 - ensureFreeSpace(2386) called with curMem=176060, maxMem=766075207 -2016-05-23 19:39:21 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 19:39:21 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51478 (size: 2.3 KB, free: 730.6 MB) -2016-05-23 19:39:21 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:39:21 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402) -2016-05-23 19:39:21 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-05-23 19:39:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:39:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:39:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:39:21 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-05-23 19:39:21 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 19:39:21 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 19:39:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 19:39:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 19:39:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 19:39:21 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 19:39:21 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 19:39:21 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 19:39:21 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 19:39:21 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 19:39:21 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2483 bytes result sent to driver -2016-05-23 19:39:21 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 2351 bytes result sent to driver -2016-05-23 19:39:21 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2483 bytes result sent to driver -2016-05-23 19:39:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 96 ms on localhost (1/3) -2016-05-23 19:39:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 121 ms on localhost (2/3) -2016-05-23 19:39:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 102 ms on localhost (3/3) -2016-05-23 19:39:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 19:39:21 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at RowMatrix.scala:402) finished in 0.136 s -2016-05-23 19:39:21 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at RowMatrix.scala:402, took 0.248293 s -2016-05-23 19:39:21 INFO SparkContext:59 - Starting job: foreach at StatisticsColStatsLearning.scala:27 -2016-05-23 19:39:21 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsColStatsLearning.scala:27) with 3 output partitions -2016-05-23 19:39:21 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsColStatsLearning.scala:27) -2016-05-23 19:39:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:39:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:39:21 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24), which has no missing parents -2016-05-23 19:39:21 INFO MemoryStore:59 - ensureFreeSpace(3504) called with curMem=178446, maxMem=766075207 -2016-05-23 19:39:21 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 19:39:21 INFO MemoryStore:59 - ensureFreeSpace(1984) called with curMem=181950, maxMem=766075207 -2016-05-23 19:39:21 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1984.0 B, free 730.4 MB) -2016-05-23 19:39:21 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51478 (size: 1984.0 B, free: 730.6 MB) -2016-05-23 19:39:21 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:39:21 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24) -2016-05-23 19:39:21 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 3 tasks -2016-05-23 19:39:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:39:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:39:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 5, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:39:21 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 3) -2016-05-23 19:39:21 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 5) -2016-05-23 19:39:21 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 4) -2016-05-23 19:39:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 19:39:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 19:39:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 19:39:21 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 5). 2044 bytes result sent to driver -2016-05-23 19:39:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 5) in 19 ms on localhost (1/3) -2016-05-23 19:39:21 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 4). 2044 bytes result sent to driver -2016-05-23 19:39:21 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 19:39:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 4) in 27 ms on localhost (2/3) -2016-05-23 19:39:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 3) in 29 ms on localhost (3/3) -2016-05-23 19:39:21 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsColStatsLearning.scala:27) finished in 0.029 s -2016-05-23 19:39:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 19:39:21 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsColStatsLearning.scala:27, took 0.046375 s -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 19:39:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 19:39:21 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 19:39:21 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 19:39:21 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 19:39:21 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 19:39:21 INFO BlockManager:59 - BlockManager stopped -2016-05-23 19:39:21 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 19:39:21 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 19:39:21 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 19:39:21 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 19:39:21 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-39c5fefb-8918-45ea-86dd-ddb4f005740f -2016-05-23 19:39:41 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 19:39:41 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 19:39:42 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 19:39:42 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 19:39:42 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 19:39:42 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 19:39:43 INFO Remoting:74 - Starting remoting -2016-05-23 19:39:43 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:51499] -2016-05-23 19:39:43 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51499. -2016-05-23 19:39:43 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 19:39:43 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 19:39:43 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-91185cff-d4b3-42df-996f-6f849e4bba5c -2016-05-23 19:39:43 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 19:39:43 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-3b3dd813-63ee-4723-8f9d-a888ecc7f2ac\httpd-ce1ed952-2173-4477-9a89-ac779f881d4f -2016-05-23 19:39:43 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 19:39:43 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:39:43 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51500 -2016-05-23 19:39:43 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51500. -2016-05-23 19:39:43 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 19:39:43 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:39:43 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 19:39:43 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 19:39:43 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 19:39:43 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 19:39:43 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 19:39:44 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51519. -2016-05-23 19:39:44 INFO NettyBlockTransferService:59 - Server created on 51519 -2016-05-23 19:39:44 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 19:39:44 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51519 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51519) -2016-05-23 19:39:44 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 19:39:44 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 19:39:44 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 19:39:44 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 19:39:44 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 19:39:44 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51519 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 19:39:44 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsColStatsLearning.scala:21 -2016-05-23 19:39:46 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 19:39:46 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:39:46 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:402 -2016-05-23 19:39:46 INFO DAGScheduler:59 - Got job 0 (treeAggregate at RowMatrix.scala:402) with 3 output partitions -2016-05-23 19:39:46 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at RowMatrix.scala:402) -2016-05-23 19:39:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:39:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:39:46 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402), which has no missing parents -2016-05-23 19:39:46 INFO MemoryStore:59 - ensureFreeSpace(4480) called with curMem=171580, maxMem=766075207 -2016-05-23 19:39:46 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-23 19:39:46 INFO MemoryStore:59 - ensureFreeSpace(2386) called with curMem=176060, maxMem=766075207 -2016-05-23 19:39:46 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 19:39:46 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51519 (size: 2.3 KB, free: 730.6 MB) -2016-05-23 19:39:46 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:39:46 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402) -2016-05-23 19:39:46 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-05-23 19:39:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:39:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:39:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:39:46 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 19:39:46 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-05-23 19:39:46 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 19:39:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 19:39:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 19:39:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 19:39:46 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 19:39:46 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 19:39:46 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 19:39:46 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 19:39:46 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 19:39:46 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 2351 bytes result sent to driver -2016-05-23 19:39:46 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2483 bytes result sent to driver -2016-05-23 19:39:46 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2483 bytes result sent to driver -2016-05-23 19:39:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 94 ms on localhost (1/3) -2016-05-23 19:39:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 99 ms on localhost (2/3) -2016-05-23 19:39:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 117 ms on localhost (3/3) -2016-05-23 19:39:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 19:39:46 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at RowMatrix.scala:402) finished in 0.133 s -2016-05-23 19:39:46 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at RowMatrix.scala:402, took 0.261018 s -2016-05-23 19:39:46 INFO SparkContext:59 - Starting job: foreach at StatisticsColStatsLearning.scala:27 -2016-05-23 19:39:46 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsColStatsLearning.scala:27) with 3 output partitions -2016-05-23 19:39:46 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsColStatsLearning.scala:27) -2016-05-23 19:39:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:39:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:39:46 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24), which has no missing parents -2016-05-23 19:39:46 INFO MemoryStore:59 - ensureFreeSpace(3504) called with curMem=178446, maxMem=766075207 -2016-05-23 19:39:46 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 19:39:46 INFO MemoryStore:59 - ensureFreeSpace(1984) called with curMem=181950, maxMem=766075207 -2016-05-23 19:39:46 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1984.0 B, free 730.4 MB) -2016-05-23 19:39:46 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51519 (size: 1984.0 B, free: 730.6 MB) -2016-05-23 19:39:46 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:39:46 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24) -2016-05-23 19:39:46 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 3 tasks -2016-05-23 19:39:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:39:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:39:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 5, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:39:46 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 3) -2016-05-23 19:39:46 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 5) -2016-05-23 19:39:46 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 4) -2016-05-23 19:39:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 19:39:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 19:39:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 19:39:46 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 5). 2044 bytes result sent to driver -2016-05-23 19:39:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 5) in 24 ms on localhost (1/3) -2016-05-23 19:39:46 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 19:39:46 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 4). 2044 bytes result sent to driver -2016-05-23 19:39:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 3) in 37 ms on localhost (2/3) -2016-05-23 19:39:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 4) in 38 ms on localhost (3/3) -2016-05-23 19:39:46 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsColStatsLearning.scala:27) finished in 0.040 s -2016-05-23 19:39:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 19:39:46 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsColStatsLearning.scala:27, took 0.059238 s -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 19:39:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 19:39:47 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 19:39:47 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 19:39:47 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 19:39:47 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 19:39:47 INFO BlockManager:59 - BlockManager stopped -2016-05-23 19:39:47 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 19:39:47 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 19:39:47 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 19:39:47 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 19:39:47 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-3b3dd813-63ee-4723-8f9d-a888ecc7f2ac -2016-05-23 19:39:47 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 19:39:47 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 19:42:56 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 19:42:57 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 19:42:57 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 19:42:57 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 19:42:57 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 19:42:58 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 19:42:58 INFO Remoting:74 - Starting remoting -2016-05-23 19:42:58 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:51612] -2016-05-23 19:42:58 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51612. -2016-05-23 19:42:58 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 19:42:58 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 19:42:58 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-dc862bae-3e59-49ae-99d2-7485e6e27f23 -2016-05-23 19:42:58 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 19:42:58 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-590b63a1-56fc-4214-908f-ce303a34a46e\httpd-a3972c9c-18cc-4a17-828c-a55344da0e9a -2016-05-23 19:42:58 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 19:42:59 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:42:59 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51613 -2016-05-23 19:42:59 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51613. -2016-05-23 19:42:59 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 19:42:59 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:42:59 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 19:42:59 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 19:42:59 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 19:42:59 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 19:42:59 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 19:42:59 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51632. -2016-05-23 19:42:59 INFO NettyBlockTransferService:59 - Server created on 51632 -2016-05-23 19:42:59 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 19:42:59 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51632 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51632) -2016-05-23 19:42:59 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 19:43:00 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 19:43:00 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 19:43:00 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 19:43:00 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 19:43:00 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51632 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 19:43:00 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsColStatsLearning.scala:21 -2016-05-23 19:43:01 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 19:43:01 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:43:02 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:402 -2016-05-23 19:43:02 INFO DAGScheduler:59 - Got job 0 (treeAggregate at RowMatrix.scala:402) with 2 output partitions -2016-05-23 19:43:02 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at RowMatrix.scala:402) -2016-05-23 19:43:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:43:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:43:02 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402), which has no missing parents -2016-05-23 19:43:02 INFO MemoryStore:59 - ensureFreeSpace(4480) called with curMem=171580, maxMem=766075207 -2016-05-23 19:43:02 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-23 19:43:02 INFO MemoryStore:59 - ensureFreeSpace(2387) called with curMem=176060, maxMem=766075207 -2016-05-23 19:43:02 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 19:43:02 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51632 (size: 2.3 KB, free: 730.6 MB) -2016-05-23 19:43:02 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:43:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402) -2016-05-23 19:43:02 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 19:43:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:43:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:43:02 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 19:43:02 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 19:43:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:9+9 -2016-05-23 19:43:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+9 -2016-05-23 19:43:02 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 19:43:02 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 19:43:02 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 19:43:02 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 19:43:02 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 19:43:02 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2483 bytes result sent to driver -2016-05-23 19:43:02 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2483 bytes result sent to driver -2016-05-23 19:43:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 139 ms on localhost (1/2) -2016-05-23 19:43:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 110 ms on localhost (2/2) -2016-05-23 19:43:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 19:43:02 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at RowMatrix.scala:402) finished in 0.163 s -2016-05-23 19:43:02 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at RowMatrix.scala:402, took 0.309169 s -2016-05-23 19:43:02 INFO SparkContext:59 - Starting job: foreach at StatisticsColStatsLearning.scala:27 -2016-05-23 19:43:02 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsColStatsLearning.scala:27) with 2 output partitions -2016-05-23 19:43:02 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsColStatsLearning.scala:27) -2016-05-23 19:43:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:43:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:43:02 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24), which has no missing parents -2016-05-23 19:43:02 INFO MemoryStore:59 - ensureFreeSpace(3504) called with curMem=178447, maxMem=766075207 -2016-05-23 19:43:02 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 19:43:02 INFO MemoryStore:59 - ensureFreeSpace(1985) called with curMem=181951, maxMem=766075207 -2016-05-23 19:43:02 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1985.0 B, free 730.4 MB) -2016-05-23 19:43:02 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51632 (size: 1985.0 B, free: 730.6 MB) -2016-05-23 19:43:02 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:43:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24) -2016-05-23 19:43:02 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 19:43:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:43:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:43:02 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 19:43:02 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 19:43:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+9 -2016-05-23 19:43:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:9+9 -2016-05-23 19:43:02 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 19:43:02 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 19:43:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 29 ms on localhost (1/2) -2016-05-23 19:43:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 29 ms on localhost (2/2) -2016-05-23 19:43:02 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsColStatsLearning.scala:27) finished in 0.031 s -2016-05-23 19:43:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 19:43:02 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsColStatsLearning.scala:27, took 0.048128 s -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 19:43:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 19:43:02 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 19:43:02 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 19:43:02 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 19:43:02 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 19:43:02 INFO BlockManager:59 - BlockManager stopped -2016-05-23 19:43:02 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 19:43:02 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 19:43:02 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 19:43:02 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 19:43:02 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-590b63a1-56fc-4214-908f-ce303a34a46e -2016-05-23 19:43:02 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 19:44:11 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 19:44:12 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 19:44:12 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 19:44:12 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 19:44:12 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 19:44:13 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 19:44:13 INFO Remoting:74 - Starting remoting -2016-05-23 19:44:13 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:51662] -2016-05-23 19:44:13 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51662. -2016-05-23 19:44:13 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 19:44:13 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 19:44:13 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-25581552-a0b3-4d73-94dd-c2f67ca46363 -2016-05-23 19:44:13 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 19:44:13 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-93f9e662-ea58-419c-a007-f0370ffc887c\httpd-2f6cdaf1-ecce-4de8-a964-f509a3d890e4 -2016-05-23 19:44:13 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 19:44:13 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:44:13 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51663 -2016-05-23 19:44:13 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51663. -2016-05-23 19:44:13 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 19:44:14 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:44:14 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 19:44:14 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 19:44:14 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 19:44:14 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 19:44:14 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 19:44:14 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51682. -2016-05-23 19:44:14 INFO NettyBlockTransferService:59 - Server created on 51682 -2016-05-23 19:44:14 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 19:44:14 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51682 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51682) -2016-05-23 19:44:14 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 19:44:15 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 19:44:15 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 19:44:15 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 19:44:15 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 19:44:15 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51682 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 19:44:15 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsColStatsLearning.scala:21 -2016-05-23 19:44:16 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 19:44:17 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:44:17 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:402 -2016-05-23 19:44:17 INFO DAGScheduler:59 - Got job 0 (treeAggregate at RowMatrix.scala:402) with 2 output partitions -2016-05-23 19:44:17 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at RowMatrix.scala:402) -2016-05-23 19:44:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:44:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:44:17 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402), which has no missing parents -2016-05-23 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(4480) called with curMem=171580, maxMem=766075207 -2016-05-23 19:44:17 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-23 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(2386) called with curMem=176060, maxMem=766075207 -2016-05-23 19:44:17 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 19:44:17 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51682 (size: 2.3 KB, free: 730.6 MB) -2016-05-23 19:44:17 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:44:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402) -2016-05-23 19:44:17 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 19:44:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:44:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:44:17 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 19:44:17 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 19:44:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+10 -2016-05-23 19:44:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:10+11 -2016-05-23 19:44:17 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 19:44:17 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 19:44:17 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 19:44:17 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 19:44:17 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 19:44:17 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2483 bytes result sent to driver -2016-05-23 19:44:17 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2483 bytes result sent to driver -2016-05-23 19:44:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 122 ms on localhost (1/2) -2016-05-23 19:44:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 145 ms on localhost (2/2) -2016-05-23 19:44:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 19:44:17 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at RowMatrix.scala:402) finished in 0.171 s -2016-05-23 19:44:17 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at RowMatrix.scala:402, took 0.271141 s -2016-05-23 19:44:17 INFO SparkContext:59 - Starting job: foreach at StatisticsColStatsLearning.scala:27 -2016-05-23 19:44:17 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsColStatsLearning.scala:27) with 2 output partitions -2016-05-23 19:44:17 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsColStatsLearning.scala:27) -2016-05-23 19:44:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:44:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:44:17 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24), which has no missing parents -2016-05-23 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(3504) called with curMem=178446, maxMem=766075207 -2016-05-23 19:44:17 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(1984) called with curMem=181950, maxMem=766075207 -2016-05-23 19:44:17 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1984.0 B, free 730.4 MB) -2016-05-23 19:44:17 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51682 (size: 1984.0 B, free: 730.6 MB) -2016-05-23 19:44:17 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:44:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24) -2016-05-23 19:44:17 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 19:44:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:44:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:44:17 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 19:44:17 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 19:44:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+10 -2016-05-23 19:44:17 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 19:44:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:10+11 -2016-05-23 19:44:17 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 19:44:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 48 ms on localhost (1/2) -2016-05-23 19:44:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 53 ms on localhost (2/2) -2016-05-23 19:44:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 19:44:17 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsColStatsLearning.scala:27) finished in 0.056 s -2016-05-23 19:44:17 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsColStatsLearning.scala:27, took 0.076015 s -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 19:44:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 19:44:17 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 19:44:17 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 19:44:17 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 19:44:17 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 19:44:17 INFO BlockManager:59 - BlockManager stopped -2016-05-23 19:44:17 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 19:44:17 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 19:44:17 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 19:44:17 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 19:44:17 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-93f9e662-ea58-419c-a007-f0370ffc887c -2016-05-23 19:44:17 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 19:44:30 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 19:44:31 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 19:44:31 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 19:44:31 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 19:44:31 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 19:44:32 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 19:44:32 INFO Remoting:74 - Starting remoting -2016-05-23 19:44:32 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:51705] -2016-05-23 19:44:32 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51705. -2016-05-23 19:44:32 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 19:44:32 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 19:44:32 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-d585d5d2-0d07-4e74-ae06-d2ece29c5143 -2016-05-23 19:44:32 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 19:44:32 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-3fb6ab69-9dbc-41c0-8c57-b7b298140e77\httpd-752084d0-befa-47ac-8cd3-125469e4b47a -2016-05-23 19:44:32 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 19:44:33 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:44:33 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51706 -2016-05-23 19:44:33 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51706. -2016-05-23 19:44:33 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 19:44:33 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:44:33 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 19:44:33 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 19:44:33 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 19:44:33 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 19:44:33 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 19:44:33 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51725. -2016-05-23 19:44:33 INFO NettyBlockTransferService:59 - Server created on 51725 -2016-05-23 19:44:33 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 19:44:33 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51725 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51725) -2016-05-23 19:44:33 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 19:44:34 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 19:44:34 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 19:44:34 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 19:44:34 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 19:44:34 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51725 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 19:44:34 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsColStatsLearning.scala:21 -2016-05-23 19:44:36 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 19:44:36 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:44:36 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:402 -2016-05-23 19:44:36 INFO DAGScheduler:59 - Got job 0 (treeAggregate at RowMatrix.scala:402) with 2 output partitions -2016-05-23 19:44:36 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at RowMatrix.scala:402) -2016-05-23 19:44:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:44:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:44:36 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402), which has no missing parents -2016-05-23 19:44:36 INFO MemoryStore:59 - ensureFreeSpace(4480) called with curMem=171580, maxMem=766075207 -2016-05-23 19:44:36 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-23 19:44:36 INFO MemoryStore:59 - ensureFreeSpace(2386) called with curMem=176060, maxMem=766075207 -2016-05-23 19:44:36 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 19:44:36 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51725 (size: 2.3 KB, free: 730.6 MB) -2016-05-23 19:44:36 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:44:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402) -2016-05-23 19:44:36 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 19:44:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:44:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:44:36 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 19:44:36 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 19:44:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+10 -2016-05-23 19:44:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:10+11 -2016-05-23 19:44:36 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 19:44:36 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 19:44:36 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 19:44:36 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 19:44:36 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 19:44:36 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2483 bytes result sent to driver -2016-05-23 19:44:36 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2483 bytes result sent to driver -2016-05-23 19:44:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 140 ms on localhost (1/2) -2016-05-23 19:44:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 164 ms on localhost (2/2) -2016-05-23 19:44:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 19:44:36 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at RowMatrix.scala:402) finished in 0.182 s -2016-05-23 19:44:36 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at RowMatrix.scala:402, took 0.317664 s -2016-05-23 19:44:36 INFO SparkContext:59 - Starting job: foreach at StatisticsColStatsLearning.scala:27 -2016-05-23 19:44:36 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsColStatsLearning.scala:27) with 2 output partitions -2016-05-23 19:44:36 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsColStatsLearning.scala:27) -2016-05-23 19:44:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:44:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:44:36 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24), which has no missing parents -2016-05-23 19:44:36 INFO MemoryStore:59 - ensureFreeSpace(3504) called with curMem=178446, maxMem=766075207 -2016-05-23 19:44:36 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 19:44:36 INFO MemoryStore:59 - ensureFreeSpace(1984) called with curMem=181950, maxMem=766075207 -2016-05-23 19:44:36 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1984.0 B, free 730.4 MB) -2016-05-23 19:44:36 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51725 (size: 1984.0 B, free: 730.6 MB) -2016-05-23 19:44:36 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:44:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24) -2016-05-23 19:44:36 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 19:44:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:44:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:44:36 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 19:44:36 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 19:44:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+10 -2016-05-23 19:44:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:10+11 -2016-05-23 19:44:36 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 19:44:36 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 19:44:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 25 ms on localhost (1/2) -2016-05-23 19:44:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 24 ms on localhost (2/2) -2016-05-23 19:44:36 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsColStatsLearning.scala:27) finished in 0.026 s -2016-05-23 19:44:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 19:44:36 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsColStatsLearning.scala:27, took 0.055671 s -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 19:44:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 19:44:36 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 19:44:36 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 19:44:37 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 19:44:37 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 19:44:37 INFO BlockManager:59 - BlockManager stopped -2016-05-23 19:44:37 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 19:44:37 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 19:44:37 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 19:44:37 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 19:44:37 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-3fb6ab69-9dbc-41c0-8c57-b7b298140e77 -2016-05-23 19:44:37 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 19:44:50 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 19:44:51 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 19:44:51 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 19:44:51 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 19:44:51 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 19:44:52 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 19:44:52 INFO Remoting:74 - Starting remoting -2016-05-23 19:44:53 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:51747] -2016-05-23 19:44:53 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51747. -2016-05-23 19:44:53 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 19:44:53 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 19:44:53 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-4a49f151-d339-44d3-8b07-6ba6614751cb -2016-05-23 19:44:53 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 19:44:53 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-b7b8d338-4cf7-45ed-b227-5f501cb133b7\httpd-e271adb1-bc08-4231-b6f1-e7eab7ba2c34 -2016-05-23 19:44:53 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 19:44:53 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:44:53 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51748 -2016-05-23 19:44:53 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51748. -2016-05-23 19:44:53 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 19:44:53 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:44:53 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 19:44:53 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 19:44:53 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 19:44:53 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 19:44:53 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 19:44:54 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51767. -2016-05-23 19:44:54 INFO NettyBlockTransferService:59 - Server created on 51767 -2016-05-23 19:44:54 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 19:44:54 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51767 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51767) -2016-05-23 19:44:54 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 19:44:54 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 19:44:54 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 19:44:54 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 19:44:54 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 19:44:54 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51767 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 19:44:54 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsColStatsLearning.scala:21 -2016-05-23 19:44:56 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 19:44:56 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:44:56 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:402 -2016-05-23 19:44:56 INFO DAGScheduler:59 - Got job 0 (treeAggregate at RowMatrix.scala:402) with 2 output partitions -2016-05-23 19:44:56 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at RowMatrix.scala:402) -2016-05-23 19:44:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:44:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:44:56 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402), which has no missing parents -2016-05-23 19:44:56 INFO MemoryStore:59 - ensureFreeSpace(4480) called with curMem=171580, maxMem=766075207 -2016-05-23 19:44:56 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-23 19:44:56 INFO MemoryStore:59 - ensureFreeSpace(2386) called with curMem=176060, maxMem=766075207 -2016-05-23 19:44:56 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 19:44:56 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51767 (size: 2.3 KB, free: 730.6 MB) -2016-05-23 19:44:56 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:44:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402) -2016-05-23 19:44:56 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 19:44:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:44:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:44:56 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 19:44:56 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 19:44:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:10+11 -2016-05-23 19:44:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+10 -2016-05-23 19:44:56 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 19:44:56 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 19:44:56 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 19:44:56 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 19:44:56 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 19:44:56 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2483 bytes result sent to driver -2016-05-23 19:44:56 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2483 bytes result sent to driver -2016-05-23 19:44:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 104 ms on localhost (1/2) -2016-05-23 19:44:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 127 ms on localhost (2/2) -2016-05-23 19:44:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 19:44:56 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at RowMatrix.scala:402) finished in 0.142 s -2016-05-23 19:44:56 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at RowMatrix.scala:402, took 0.251211 s -2016-05-23 19:44:56 INFO SparkContext:59 - Starting job: foreach at StatisticsColStatsLearning.scala:27 -2016-05-23 19:44:56 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsColStatsLearning.scala:27) with 2 output partitions -2016-05-23 19:44:56 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsColStatsLearning.scala:27) -2016-05-23 19:44:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:44:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:44:56 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24), which has no missing parents -2016-05-23 19:44:56 INFO MemoryStore:59 - ensureFreeSpace(3504) called with curMem=178446, maxMem=766075207 -2016-05-23 19:44:56 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 19:44:56 INFO MemoryStore:59 - ensureFreeSpace(1984) called with curMem=181950, maxMem=766075207 -2016-05-23 19:44:56 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1984.0 B, free 730.4 MB) -2016-05-23 19:44:56 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51767 (size: 1984.0 B, free: 730.6 MB) -2016-05-23 19:44:56 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:44:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24) -2016-05-23 19:44:56 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 19:44:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:44:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:44:56 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 19:44:56 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 19:44:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:10+11 -2016-05-23 19:44:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+10 -2016-05-23 19:44:56 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 19:44:56 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 19:44:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 24 ms on localhost (1/2) -2016-05-23 19:44:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 25 ms on localhost (2/2) -2016-05-23 19:44:56 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsColStatsLearning.scala:27) finished in 0.026 s -2016-05-23 19:44:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 19:44:56 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsColStatsLearning.scala:27, took 0.043641 s -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 19:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 19:44:56 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 19:44:56 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 19:44:57 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 19:44:57 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 19:44:57 INFO BlockManager:59 - BlockManager stopped -2016-05-23 19:44:57 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 19:44:57 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 19:44:57 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 19:44:57 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 19:44:57 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-b7b8d338-4cf7-45ed-b227-5f501cb133b7 -2016-05-23 19:44:57 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 19:45:03 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 19:45:04 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 19:45:04 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 19:45:04 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 19:45:04 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 19:45:05 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 19:45:05 INFO Remoting:74 - Starting remoting -2016-05-23 19:45:05 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:51788] -2016-05-23 19:45:05 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51788. -2016-05-23 19:45:05 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 19:45:05 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 19:45:05 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-8472e0d6-f451-41c9-98a0-3b80caaab98f -2016-05-23 19:45:05 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 19:45:05 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-ab409c0c-65a2-43ee-b8c8-0c4047bef4d0\httpd-fa796dce-06a1-444b-9bdd-9839c682b618 -2016-05-23 19:45:05 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 19:45:06 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:45:06 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51789 -2016-05-23 19:45:06 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51789. -2016-05-23 19:45:06 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 19:45:06 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:45:06 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 19:45:06 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 19:45:06 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 19:45:06 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 19:45:06 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 19:45:06 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51808. -2016-05-23 19:45:06 INFO NettyBlockTransferService:59 - Server created on 51808 -2016-05-23 19:45:06 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 19:45:06 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51808 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51808) -2016-05-23 19:45:06 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 19:45:07 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 19:45:07 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 19:45:07 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 19:45:07 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 19:45:07 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51808 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 19:45:07 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsColStatsLearning.scala:21 -2016-05-23 19:45:08 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 19:45:08 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:45:09 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:402 -2016-05-23 19:45:09 INFO DAGScheduler:59 - Got job 0 (treeAggregate at RowMatrix.scala:402) with 2 output partitions -2016-05-23 19:45:09 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at RowMatrix.scala:402) -2016-05-23 19:45:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:45:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:45:09 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402), which has no missing parents -2016-05-23 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(4480) called with curMem=171580, maxMem=766075207 -2016-05-23 19:45:09 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-23 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(2386) called with curMem=176060, maxMem=766075207 -2016-05-23 19:45:09 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 19:45:09 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51808 (size: 2.3 KB, free: 730.6 MB) -2016-05-23 19:45:09 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:45:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402) -2016-05-23 19:45:09 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 19:45:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:45:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:45:09 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 19:45:09 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 19:45:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:10+11 -2016-05-23 19:45:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+10 -2016-05-23 19:45:09 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 19:45:09 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 19:45:09 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 19:45:09 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 19:45:09 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 19:45:09 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2483 bytes result sent to driver -2016-05-23 19:45:09 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2483 bytes result sent to driver -2016-05-23 19:45:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 118 ms on localhost (1/2) -2016-05-23 19:45:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 102 ms on localhost (2/2) -2016-05-23 19:45:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 19:45:09 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at RowMatrix.scala:402) finished in 0.141 s -2016-05-23 19:45:09 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at RowMatrix.scala:402, took 0.254422 s -2016-05-23 19:45:09 INFO SparkContext:59 - Starting job: foreach at StatisticsColStatsLearning.scala:27 -2016-05-23 19:45:09 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsColStatsLearning.scala:27) with 2 output partitions -2016-05-23 19:45:09 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsColStatsLearning.scala:27) -2016-05-23 19:45:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:45:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:45:09 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24), which has no missing parents -2016-05-23 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(3504) called with curMem=178446, maxMem=766075207 -2016-05-23 19:45:09 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(1984) called with curMem=181950, maxMem=766075207 -2016-05-23 19:45:09 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1984.0 B, free 730.4 MB) -2016-05-23 19:45:09 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51808 (size: 1984.0 B, free: 730.6 MB) -2016-05-23 19:45:09 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:45:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24) -2016-05-23 19:45:09 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 19:45:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:45:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:45:09 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 19:45:09 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 19:45:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+10 -2016-05-23 19:45:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:10+11 -2016-05-23 19:45:09 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 19:45:09 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 19:45:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 23 ms on localhost (1/2) -2016-05-23 19:45:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 25 ms on localhost (2/2) -2016-05-23 19:45:09 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsColStatsLearning.scala:27) finished in 0.026 s -2016-05-23 19:45:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 19:45:09 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsColStatsLearning.scala:27, took 0.042685 s -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 19:45:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 19:45:09 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 19:45:09 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 19:45:09 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 19:45:09 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 19:45:09 INFO BlockManager:59 - BlockManager stopped -2016-05-23 19:45:09 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 19:45:09 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 19:45:09 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 19:45:09 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 19:45:09 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-ab409c0c-65a2-43ee-b8c8-0c4047bef4d0 -2016-05-23 19:45:09 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 19:45:09 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 19:45:10 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-05-23 19:55:21 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 19:55:22 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 19:55:22 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 19:55:22 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 19:55:22 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 19:55:23 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 19:55:23 INFO Remoting:74 - Starting remoting -2016-05-23 19:55:23 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:51952] -2016-05-23 19:55:23 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51952. -2016-05-23 19:55:23 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 19:55:23 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 19:55:23 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-33cb3a64-8706-4f00-b343-bce65350be26 -2016-05-23 19:55:23 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 19:55:23 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-dc184e9a-78ef-46c8-901a-c3fe272dd4ab\httpd-5b9294e6-9ad4-482f-bbe5-1973530cd1c3 -2016-05-23 19:55:23 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 19:55:23 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:55:23 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51953 -2016-05-23 19:55:23 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51953. -2016-05-23 19:55:23 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 19:55:23 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:55:23 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 19:55:23 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 19:55:23 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 19:55:24 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 19:55:24 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 19:55:24 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51972. -2016-05-23 19:55:24 INFO NettyBlockTransferService:59 - Server created on 51972 -2016-05-23 19:55:24 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 19:55:24 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51972 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51972) -2016-05-23 19:55:24 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 19:55:25 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 19:55:25 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 19:55:25 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 19:55:25 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 19:55:25 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51972 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 19:55:25 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsColStatsLearning.scala:21 -2016-05-23 19:55:27 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 19:55:27 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:55:27 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:402 -2016-05-23 19:55:27 INFO DAGScheduler:59 - Got job 0 (treeAggregate at RowMatrix.scala:402) with 2 output partitions -2016-05-23 19:55:27 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at RowMatrix.scala:402) -2016-05-23 19:55:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:55:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:55:27 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402), which has no missing parents -2016-05-23 19:55:27 INFO MemoryStore:59 - ensureFreeSpace(4480) called with curMem=171580, maxMem=766075207 -2016-05-23 19:55:27 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-23 19:55:27 INFO MemoryStore:59 - ensureFreeSpace(2386) called with curMem=176060, maxMem=766075207 -2016-05-23 19:55:27 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 19:55:27 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51972 (size: 2.3 KB, free: 730.6 MB) -2016-05-23 19:55:27 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:55:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402) -2016-05-23 19:55:27 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 19:55:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:55:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:55:27 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 19:55:27 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 19:55:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+10 -2016-05-23 19:55:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:10+11 -2016-05-23 19:55:27 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 19:55:27 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 19:55:27 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 19:55:27 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 19:55:27 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 19:55:27 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2483 bytes result sent to driver -2016-05-23 19:55:27 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2483 bytes result sent to driver -2016-05-23 19:55:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 158 ms on localhost (1/2) -2016-05-23 19:55:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 139 ms on localhost (2/2) -2016-05-23 19:55:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 19:55:27 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at RowMatrix.scala:402) finished in 0.185 s -2016-05-23 19:55:27 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at RowMatrix.scala:402, took 0.301240 s -2016-05-23 19:55:27 INFO SparkContext:59 - Starting job: foreach at StatisticsColStatsLearning.scala:28 -2016-05-23 19:55:27 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsColStatsLearning.scala:28) with 2 output partitions -2016-05-23 19:55:27 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsColStatsLearning.scala:28) -2016-05-23 19:55:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:55:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:55:27 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24), which has no missing parents -2016-05-23 19:55:27 INFO MemoryStore:59 - ensureFreeSpace(3504) called with curMem=178446, maxMem=766075207 -2016-05-23 19:55:27 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 19:55:27 INFO MemoryStore:59 - ensureFreeSpace(1984) called with curMem=181950, maxMem=766075207 -2016-05-23 19:55:27 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1984.0 B, free 730.4 MB) -2016-05-23 19:55:27 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51972 (size: 1984.0 B, free: 730.6 MB) -2016-05-23 19:55:27 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:55:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24) -2016-05-23 19:55:27 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 19:55:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:55:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:55:27 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 19:55:27 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 19:55:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+10 -2016-05-23 19:55:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:10+11 -2016-05-23 19:55:27 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 19:55:27 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 19:55:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 32 ms on localhost (1/2) -2016-05-23 19:55:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 31 ms on localhost (2/2) -2016-05-23 19:55:27 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsColStatsLearning.scala:28) finished in 0.034 s -2016-05-23 19:55:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 19:55:27 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsColStatsLearning.scala:28, took 0.053568 s -2016-05-23 19:55:27 INFO SparkContext:59 - Starting job: count at StatisticsColStatsLearning.scala:29 -2016-05-23 19:55:27 INFO DAGScheduler:59 - Got job 2 (count at StatisticsColStatsLearning.scala:29) with 2 output partitions -2016-05-23 19:55:27 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at StatisticsColStatsLearning.scala:29) -2016-05-23 19:55:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:55:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:55:27 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24), which has no missing parents -2016-05-23 19:55:27 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=183934, maxMem=766075207 -2016-05-23 19:55:27 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-05-23 19:55:27 INFO MemoryStore:59 - ensureFreeSpace(1946) called with curMem=187310, maxMem=766075207 -2016-05-23 19:55:27 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1946.0 B, free 730.4 MB) -2016-05-23 19:55:27 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:51972 (size: 1946.0 B, free: 730.6 MB) -2016-05-23 19:55:27 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:55:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24) -2016-05-23 19:55:27 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-23 19:55:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:55:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:55:27 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-23 19:55:27 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 19:55:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+10 -2016-05-23 19:55:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:10+11 -2016-05-23 19:55:27 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-23 19:55:27 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2082 bytes result sent to driver -2016-05-23 19:55:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 22 ms on localhost (1/2) -2016-05-23 19:55:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 25 ms on localhost (2/2) -2016-05-23 19:55:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 19:55:27 INFO DAGScheduler:59 - ResultStage 2 (count at StatisticsColStatsLearning.scala:29) finished in 0.025 s -2016-05-23 19:55:27 INFO DAGScheduler:59 - Job 2 finished: count at StatisticsColStatsLearning.scala:29, took 0.062271 s -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 19:55:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 19:55:27 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 19:55:27 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 19:55:27 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 19:55:28 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 19:55:28 INFO BlockManager:59 - BlockManager stopped -2016-05-23 19:55:28 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 19:55:28 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 19:55:28 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 19:55:29 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 19:55:29 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-dc184e9a-78ef-46c8-901a-c3fe272dd4ab -2016-05-23 19:55:29 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 19:55:29 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 19:55:29 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-05-23 19:57:39 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 19:57:40 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 19:57:40 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 19:57:40 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 19:57:40 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 19:57:41 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 19:57:41 INFO Remoting:74 - Starting remoting -2016-05-23 19:57:42 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:52033] -2016-05-23 19:57:42 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52033. -2016-05-23 19:57:42 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 19:57:42 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 19:57:42 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-3623b065-2a13-47e0-add6-79e40c224222 -2016-05-23 19:57:42 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 19:57:42 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-14c478de-7245-4e2c-a429-f32d34e008fd\httpd-f7f14aa9-a6b0-461e-8663-773b9e59b64e -2016-05-23 19:57:42 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 19:57:42 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:57:42 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52034 -2016-05-23 19:57:42 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52034. -2016-05-23 19:57:42 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 19:57:42 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 19:57:42 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 19:57:42 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 19:57:42 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 19:57:42 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 19:57:42 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 19:57:43 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52053. -2016-05-23 19:57:43 INFO NettyBlockTransferService:59 - Server created on 52053 -2016-05-23 19:57:43 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 19:57:43 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52053 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52053) -2016-05-23 19:57:43 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 19:57:43 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 19:57:43 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 19:57:44 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 19:57:44 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 19:57:44 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52053 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 19:57:44 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsColStatsLearning.scala:21 -2016-05-23 19:57:45 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 19:57:45 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 19:57:45 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:402 -2016-05-23 19:57:45 INFO DAGScheduler:59 - Got job 0 (treeAggregate at RowMatrix.scala:402) with 3 output partitions -2016-05-23 19:57:45 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at RowMatrix.scala:402) -2016-05-23 19:57:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:57:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:57:45 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402), which has no missing parents -2016-05-23 19:57:45 INFO MemoryStore:59 - ensureFreeSpace(4480) called with curMem=171580, maxMem=766075207 -2016-05-23 19:57:45 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-23 19:57:45 INFO MemoryStore:59 - ensureFreeSpace(2386) called with curMem=176060, maxMem=766075207 -2016-05-23 19:57:45 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 19:57:45 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52053 (size: 2.3 KB, free: 730.6 MB) -2016-05-23 19:57:45 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:57:45 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402) -2016-05-23 19:57:45 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-05-23 19:57:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:57:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:57:45 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:57:45 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 19:57:45 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 19:57:45 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-05-23 19:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 19:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 19:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 19:57:45 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 19:57:45 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 19:57:45 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 19:57:45 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 19:57:45 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 19:57:45 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2483 bytes result sent to driver -2016-05-23 19:57:45 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 2351 bytes result sent to driver -2016-05-23 19:57:45 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2483 bytes result sent to driver -2016-05-23 19:57:45 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 112 ms on localhost (1/3) -2016-05-23 19:57:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 117 ms on localhost (2/3) -2016-05-23 19:57:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 138 ms on localhost (3/3) -2016-05-23 19:57:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 19:57:45 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at RowMatrix.scala:402) finished in 0.155 s -2016-05-23 19:57:45 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at RowMatrix.scala:402, took 0.312620 s -2016-05-23 19:57:45 INFO SparkContext:59 - Starting job: foreach at StatisticsColStatsLearning.scala:28 -2016-05-23 19:57:45 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsColStatsLearning.scala:28) with 3 output partitions -2016-05-23 19:57:45 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsColStatsLearning.scala:28) -2016-05-23 19:57:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:57:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:57:45 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24), which has no missing parents -2016-05-23 19:57:45 INFO MemoryStore:59 - ensureFreeSpace(3504) called with curMem=178446, maxMem=766075207 -2016-05-23 19:57:45 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 19:57:45 INFO MemoryStore:59 - ensureFreeSpace(1984) called with curMem=181950, maxMem=766075207 -2016-05-23 19:57:45 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1984.0 B, free 730.4 MB) -2016-05-23 19:57:45 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52053 (size: 1984.0 B, free: 730.6 MB) -2016-05-23 19:57:45 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:57:45 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24) -2016-05-23 19:57:45 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 3 tasks -2016-05-23 19:57:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:57:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:57:45 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 5, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:57:45 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 4) -2016-05-23 19:57:45 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 3) -2016-05-23 19:57:45 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 5) -2016-05-23 19:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 19:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 19:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 19:57:45 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 5). 2044 bytes result sent to driver -2016-05-23 19:57:45 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 5) in 21 ms on localhost (1/3) -2016-05-23 19:57:45 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 4). 2044 bytes result sent to driver -2016-05-23 19:57:45 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 19:57:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 3) in 27 ms on localhost (2/3) -2016-05-23 19:57:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 4) in 27 ms on localhost (3/3) -2016-05-23 19:57:45 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsColStatsLearning.scala:28) finished in 0.028 s -2016-05-23 19:57:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 19:57:45 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsColStatsLearning.scala:28, took 0.045790 s -2016-05-23 19:57:45 INFO SparkContext:59 - Starting job: count at StatisticsColStatsLearning.scala:29 -2016-05-23 19:57:45 INFO DAGScheduler:59 - Got job 2 (count at StatisticsColStatsLearning.scala:29) with 3 output partitions -2016-05-23 19:57:45 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at StatisticsColStatsLearning.scala:29) -2016-05-23 19:57:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 19:57:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 19:57:45 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24), which has no missing parents -2016-05-23 19:57:45 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=183934, maxMem=766075207 -2016-05-23 19:57:45 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-05-23 19:57:45 INFO MemoryStore:59 - ensureFreeSpace(1946) called with curMem=187310, maxMem=766075207 -2016-05-23 19:57:45 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1946.0 B, free 730.4 MB) -2016-05-23 19:57:45 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52053 (size: 1946.0 B, free: 730.6 MB) -2016-05-23 19:57:45 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 19:57:45 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 2 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24) -2016-05-23 19:57:45 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-05-23 19:57:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:57:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:57:45 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 8, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 19:57:45 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 6) -2016-05-23 19:57:45 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 8) -2016-05-23 19:57:45 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 7) -2016-05-23 19:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 19:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 19:57:45 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 8). 2082 bytes result sent to driver -2016-05-23 19:57:45 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 6). 2082 bytes result sent to driver -2016-05-23 19:57:45 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 8) in 19 ms on localhost (1/3) -2016-05-23 19:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 19:57:45 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 7). 2082 bytes result sent to driver -2016-05-23 19:57:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 6) in 33 ms on localhost (2/3) -2016-05-23 19:57:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 7) in 35 ms on localhost (3/3) -2016-05-23 19:57:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 19:57:46 INFO DAGScheduler:59 - ResultStage 2 (count at StatisticsColStatsLearning.scala:29) finished in 0.038 s -2016-05-23 19:57:46 INFO DAGScheduler:59 - Job 2 finished: count at StatisticsColStatsLearning.scala:29, took 0.090081 s -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 19:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 19:57:46 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 19:57:46 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 19:57:46 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 19:57:46 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 19:57:46 INFO BlockManager:59 - BlockManager stopped -2016-05-23 19:57:46 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 19:57:46 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 19:57:46 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 19:57:46 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 19:57:46 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-14c478de-7245-4e2c-a429-f32d34e008fd -2016-05-23 20:00:03 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 20:00:03 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 20:00:04 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 20:00:04 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 20:00:04 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 20:00:05 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 20:00:05 INFO Remoting:74 - Starting remoting -2016-05-23 20:00:05 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:52091] -2016-05-23 20:00:05 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52091. -2016-05-23 20:00:05 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 20:00:05 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 20:00:05 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-e7d1e21b-1fae-411f-84b6-0941a6a6870c -2016-05-23 20:00:05 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 20:00:06 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-88bf82fc-fc0b-473f-8a59-5870cbded9c1\httpd-97d0c0b2-1e15-4695-98d4-26d6286bf71c -2016-05-23 20:00:06 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 20:00:06 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:00:06 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52092 -2016-05-23 20:00:06 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52092. -2016-05-23 20:00:06 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 20:00:06 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:00:06 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 20:00:06 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 20:00:06 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 20:00:06 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 20:00:06 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 20:00:06 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52111. -2016-05-23 20:00:06 INFO NettyBlockTransferService:59 - Server created on 52111 -2016-05-23 20:00:06 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 20:00:06 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52111 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52111) -2016-05-23 20:00:06 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 20:00:07 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 20:00:07 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 20:00:07 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 20:00:07 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 20:00:07 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52111 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:00:07 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsColStatsLearning.scala:21 -2016-05-23 20:00:09 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 20:00:09 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:00:09 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:402 -2016-05-23 20:00:09 INFO DAGScheduler:59 - Got job 0 (treeAggregate at RowMatrix.scala:402) with 3 output partitions -2016-05-23 20:00:09 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at RowMatrix.scala:402) -2016-05-23 20:00:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:00:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:00:09 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402), which has no missing parents -2016-05-23 20:00:09 INFO MemoryStore:59 - ensureFreeSpace(4480) called with curMem=171580, maxMem=766075207 -2016-05-23 20:00:09 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-23 20:00:09 INFO MemoryStore:59 - ensureFreeSpace(2386) called with curMem=176060, maxMem=766075207 -2016-05-23 20:00:09 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 20:00:09 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52111 (size: 2.3 KB, free: 730.6 MB) -2016-05-23 20:00:09 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:00:09 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402) -2016-05-23 20:00:09 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-05-23 20:00:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:09 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 20:00:09 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-05-23 20:00:09 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 20:00:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 20:00:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 20:00:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 20:00:09 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 20:00:09 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 20:00:09 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 20:00:09 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 20:00:09 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 20:00:09 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 2351 bytes result sent to driver -2016-05-23 20:00:09 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2483 bytes result sent to driver -2016-05-23 20:00:09 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2483 bytes result sent to driver -2016-05-23 20:00:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 167 ms on localhost (1/3) -2016-05-23 20:00:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 217 ms on localhost (2/3) -2016-05-23 20:00:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 181 ms on localhost (3/3) -2016-05-23 20:00:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 20:00:10 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at RowMatrix.scala:402) finished in 0.253 s -2016-05-23 20:00:10 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at RowMatrix.scala:402, took 0.414578 s -2016-05-23 20:00:10 INFO SparkContext:59 - Starting job: foreach at StatisticsColStatsLearning.scala:28 -2016-05-23 20:00:10 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsColStatsLearning.scala:28) with 3 output partitions -2016-05-23 20:00:10 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsColStatsLearning.scala:28) -2016-05-23 20:00:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:00:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:00:10 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24), which has no missing parents -2016-05-23 20:00:10 INFO MemoryStore:59 - ensureFreeSpace(3504) called with curMem=178446, maxMem=766075207 -2016-05-23 20:00:10 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 20:00:10 INFO MemoryStore:59 - ensureFreeSpace(1984) called with curMem=181950, maxMem=766075207 -2016-05-23 20:00:10 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1984.0 B, free 730.4 MB) -2016-05-23 20:00:10 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52111 (size: 1984.0 B, free: 730.6 MB) -2016-05-23 20:00:10 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:00:10 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24) -2016-05-23 20:00:10 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 3 tasks -2016-05-23 20:00:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 5, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:10 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 3) -2016-05-23 20:00:10 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 5) -2016-05-23 20:00:10 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 4) -2016-05-23 20:00:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 20:00:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 20:00:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 20:00:10 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 5). 2044 bytes result sent to driver -2016-05-23 20:00:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 5) in 25 ms on localhost (1/3) -2016-05-23 20:00:10 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 4). 2044 bytes result sent to driver -2016-05-23 20:00:10 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 20:00:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 4) in 31 ms on localhost (2/3) -2016-05-23 20:00:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 3) in 33 ms on localhost (3/3) -2016-05-23 20:00:10 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsColStatsLearning.scala:28) finished in 0.033 s -2016-05-23 20:00:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 20:00:10 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsColStatsLearning.scala:28, took 0.052584 s -2016-05-23 20:00:10 INFO SparkContext:59 - Starting job: count at StatisticsColStatsLearning.scala:29 -2016-05-23 20:00:10 INFO DAGScheduler:59 - Got job 2 (count at StatisticsColStatsLearning.scala:29) with 3 output partitions -2016-05-23 20:00:10 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at StatisticsColStatsLearning.scala:29) -2016-05-23 20:00:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:00:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:00:10 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24), which has no missing parents -2016-05-23 20:00:10 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=183934, maxMem=766075207 -2016-05-23 20:00:10 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-05-23 20:00:10 INFO MemoryStore:59 - ensureFreeSpace(1946) called with curMem=187310, maxMem=766075207 -2016-05-23 20:00:10 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1946.0 B, free 730.4 MB) -2016-05-23 20:00:10 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52111 (size: 1946.0 B, free: 730.6 MB) -2016-05-23 20:00:10 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:00:10 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 2 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24) -2016-05-23 20:00:10 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-05-23 20:00:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 8, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:10 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 8) -2016-05-23 20:00:10 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 7) -2016-05-23 20:00:10 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 6) -2016-05-23 20:00:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 20:00:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 20:00:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 20:00:10 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 8). 2082 bytes result sent to driver -2016-05-23 20:00:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 8) in 22 ms on localhost (1/3) -2016-05-23 20:00:10 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 7). 2082 bytes result sent to driver -2016-05-23 20:00:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 7) in 29 ms on localhost (2/3) -2016-05-23 20:00:10 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 6). 2082 bytes result sent to driver -2016-05-23 20:00:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 6) in 35 ms on localhost (3/3) -2016-05-23 20:00:10 INFO DAGScheduler:59 - ResultStage 2 (count at StatisticsColStatsLearning.scala:29) finished in 0.036 s -2016-05-23 20:00:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 20:00:10 INFO DAGScheduler:59 - Job 2 finished: count at StatisticsColStatsLearning.scala:29, took 0.056988 s -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 20:00:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 20:00:10 WARN QueuedThreadPool:145 - 3 threads could not be stopped -2016-05-23 20:00:10 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 20:00:10 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 20:00:10 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 20:00:10 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 20:00:10 INFO BlockManager:59 - BlockManager stopped -2016-05-23 20:00:10 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 20:00:10 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 20:00:10 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 20:00:10 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 20:00:10 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-88bf82fc-fc0b-473f-8a59-5870cbded9c1 -2016-05-23 20:00:24 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 20:00:25 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 20:00:26 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 20:00:26 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 20:00:26 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 20:00:27 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 20:00:27 INFO Remoting:74 - Starting remoting -2016-05-23 20:00:27 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:52133] -2016-05-23 20:00:27 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52133. -2016-05-23 20:00:27 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 20:00:27 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 20:00:27 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-d241cfd3-3653-4547-8678-1df686bdbdfc -2016-05-23 20:00:27 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 20:00:28 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-f911f16d-7296-49c7-882b-496d7d53e4a7\httpd-55f0f70e-6f88-4e47-a976-8855b4daa6ea -2016-05-23 20:00:28 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 20:00:28 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:00:28 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52134 -2016-05-23 20:00:28 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52134. -2016-05-23 20:00:28 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 20:00:28 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:00:28 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 20:00:28 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 20:00:28 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 20:00:28 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 20:00:28 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 20:00:28 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52153. -2016-05-23 20:00:28 INFO NettyBlockTransferService:59 - Server created on 52153 -2016-05-23 20:00:28 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 20:00:28 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52153 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52153) -2016-05-23 20:00:28 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 20:00:29 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 20:00:29 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 20:00:29 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 20:00:29 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 20:00:29 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52153 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:00:29 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsColStatsLearning.scala:21 -2016-05-23 20:00:31 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 20:00:31 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:00:31 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:402 -2016-05-23 20:00:31 INFO DAGScheduler:59 - Got job 0 (treeAggregate at RowMatrix.scala:402) with 3 output partitions -2016-05-23 20:00:31 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at RowMatrix.scala:402) -2016-05-23 20:00:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:00:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:00:31 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402), which has no missing parents -2016-05-23 20:00:31 INFO MemoryStore:59 - ensureFreeSpace(4480) called with curMem=171580, maxMem=766075207 -2016-05-23 20:00:31 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-23 20:00:31 INFO MemoryStore:59 - ensureFreeSpace(2386) called with curMem=176060, maxMem=766075207 -2016-05-23 20:00:31 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 20:00:31 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52153 (size: 2.3 KB, free: 730.6 MB) -2016-05-23 20:00:31 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:00:31 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402) -2016-05-23 20:00:31 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-05-23 20:00:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:31 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-05-23 20:00:31 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 20:00:31 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 20:00:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 20:00:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 20:00:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 20:00:31 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 20:00:31 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 20:00:31 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 20:00:31 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 20:00:31 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 20:00:31 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 2351 bytes result sent to driver -2016-05-23 20:00:31 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2483 bytes result sent to driver -2016-05-23 20:00:31 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2483 bytes result sent to driver -2016-05-23 20:00:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 113 ms on localhost (1/3) -2016-05-23 20:00:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 142 ms on localhost (2/3) -2016-05-23 20:00:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 106 ms on localhost (3/3) -2016-05-23 20:00:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 20:00:31 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at RowMatrix.scala:402) finished in 0.165 s -2016-05-23 20:00:31 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at RowMatrix.scala:402, took 0.366804 s -2016-05-23 20:00:31 INFO SparkContext:59 - Starting job: foreach at StatisticsColStatsLearning.scala:28 -2016-05-23 20:00:31 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsColStatsLearning.scala:28) with 3 output partitions -2016-05-23 20:00:31 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsColStatsLearning.scala:28) -2016-05-23 20:00:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:00:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:00:31 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24), which has no missing parents -2016-05-23 20:00:31 INFO MemoryStore:59 - ensureFreeSpace(3504) called with curMem=178446, maxMem=766075207 -2016-05-23 20:00:31 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 20:00:31 INFO MemoryStore:59 - ensureFreeSpace(1984) called with curMem=181950, maxMem=766075207 -2016-05-23 20:00:31 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1984.0 B, free 730.4 MB) -2016-05-23 20:00:31 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52153 (size: 1984.0 B, free: 730.6 MB) -2016-05-23 20:00:31 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:00:31 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24) -2016-05-23 20:00:31 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 3 tasks -2016-05-23 20:00:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 5, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:31 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 3) -2016-05-23 20:00:31 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 5) -2016-05-23 20:00:31 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 4) -2016-05-23 20:00:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 20:00:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 20:00:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 20:00:31 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 5). 2044 bytes result sent to driver -2016-05-23 20:00:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 5) in 20 ms on localhost (1/3) -2016-05-23 20:00:31 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 20:00:31 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 4). 2044 bytes result sent to driver -2016-05-23 20:00:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 3) in 27 ms on localhost (2/3) -2016-05-23 20:00:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 4) in 27 ms on localhost (3/3) -2016-05-23 20:00:31 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsColStatsLearning.scala:28) finished in 0.028 s -2016-05-23 20:00:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 20:00:31 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsColStatsLearning.scala:28, took 0.043838 s -2016-05-23 20:00:31 INFO SparkContext:59 - Starting job: count at StatisticsColStatsLearning.scala:29 -2016-05-23 20:00:31 INFO DAGScheduler:59 - Got job 2 (count at StatisticsColStatsLearning.scala:29) with 3 output partitions -2016-05-23 20:00:31 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at StatisticsColStatsLearning.scala:29) -2016-05-23 20:00:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:00:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:00:31 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24), which has no missing parents -2016-05-23 20:00:31 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=183934, maxMem=766075207 -2016-05-23 20:00:31 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-05-23 20:00:31 INFO MemoryStore:59 - ensureFreeSpace(1946) called with curMem=187310, maxMem=766075207 -2016-05-23 20:00:31 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1946.0 B, free 730.4 MB) -2016-05-23 20:00:31 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52153 (size: 1946.0 B, free: 730.6 MB) -2016-05-23 20:00:31 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:00:31 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 2 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24) -2016-05-23 20:00:31 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-05-23 20:00:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 8, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:31 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 7) -2016-05-23 20:00:31 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 6) -2016-05-23 20:00:31 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 8) -2016-05-23 20:00:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 20:00:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 20:00:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 20:00:31 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 6). 2082 bytes result sent to driver -2016-05-23 20:00:31 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 7). 2082 bytes result sent to driver -2016-05-23 20:00:31 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 8). 2082 bytes result sent to driver -2016-05-23 20:00:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 7) in 27 ms on localhost (1/3) -2016-05-23 20:00:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 6) in 29 ms on localhost (2/3) -2016-05-23 20:00:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 8) in 29 ms on localhost (3/3) -2016-05-23 20:00:31 INFO DAGScheduler:59 - ResultStage 2 (count at StatisticsColStatsLearning.scala:29) finished in 0.032 s -2016-05-23 20:00:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 20:00:31 INFO DAGScheduler:59 - Job 2 finished: count at StatisticsColStatsLearning.scala:29, took 0.047756 s -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 20:00:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 20:00:31 WARN QueuedThreadPool:145 - 7 threads could not be stopped -2016-05-23 20:00:31 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 20:00:32 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 20:00:32 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 20:00:32 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 20:00:32 INFO BlockManager:59 - BlockManager stopped -2016-05-23 20:00:32 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 20:00:32 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 20:00:32 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 20:00:32 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 20:00:32 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-f911f16d-7296-49c7-882b-496d7d53e4a7 -2016-05-23 20:00:32 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 20:00:46 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 20:00:47 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 20:00:47 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 20:00:47 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 20:00:47 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 20:00:48 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 20:00:48 INFO Remoting:74 - Starting remoting -2016-05-23 20:00:48 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:52176] -2016-05-23 20:00:48 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52176. -2016-05-23 20:00:48 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 20:00:48 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 20:00:48 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-65b23775-9288-48e5-9a84-97f3fa5c599b -2016-05-23 20:00:49 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 20:00:49 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-8274dd60-9100-41bd-ace7-fb65e0bf5679\httpd-5746bf03-9c28-4d86-9108-c2a9d19745e4 -2016-05-23 20:00:49 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 20:00:49 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:00:49 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52177 -2016-05-23 20:00:49 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52177. -2016-05-23 20:00:49 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 20:00:49 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:00:49 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 20:00:49 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 20:00:49 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 20:00:49 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 20:00:49 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 20:00:49 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52196. -2016-05-23 20:00:49 INFO NettyBlockTransferService:59 - Server created on 52196 -2016-05-23 20:00:49 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 20:00:50 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52196 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52196) -2016-05-23 20:00:50 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 20:00:50 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 20:00:50 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 20:00:50 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 20:00:50 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 20:00:50 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52196 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:00:50 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsColStatsLearning.scala:21 -2016-05-23 20:00:52 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 20:00:52 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:00:52 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:402 -2016-05-23 20:00:52 INFO DAGScheduler:59 - Got job 0 (treeAggregate at RowMatrix.scala:402) with 3 output partitions -2016-05-23 20:00:52 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at RowMatrix.scala:402) -2016-05-23 20:00:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:00:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:00:52 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402), which has no missing parents -2016-05-23 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(4480) called with curMem=171580, maxMem=766075207 -2016-05-23 20:00:52 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-23 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(2386) called with curMem=176060, maxMem=766075207 -2016-05-23 20:00:52 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52196 (size: 2.3 KB, free: 730.6 MB) -2016-05-23 20:00:52 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:00:52 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:402) -2016-05-23 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-05-23 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:52 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-05-23 20:00:52 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 20:00:52 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 20:00:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 20:00:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 20:00:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 20:00:52 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 20:00:52 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 20:00:52 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 20:00:52 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 20:00:52 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 20:00:52 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 2351 bytes result sent to driver -2016-05-23 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2483 bytes result sent to driver -2016-05-23 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2483 bytes result sent to driver -2016-05-23 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 156 ms on localhost (1/3) -2016-05-23 20:00:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 133 ms on localhost (2/3) -2016-05-23 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 136 ms on localhost (3/3) -2016-05-23 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 20:00:52 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at RowMatrix.scala:402) finished in 0.191 s -2016-05-23 20:00:52 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at RowMatrix.scala:402, took 0.284262 s -2016-05-23 20:00:52 INFO SparkContext:59 - Starting job: foreach at StatisticsColStatsLearning.scala:28 -2016-05-23 20:00:52 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsColStatsLearning.scala:28) with 3 output partitions -2016-05-23 20:00:52 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsColStatsLearning.scala:28) -2016-05-23 20:00:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:00:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:00:52 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24), which has no missing parents -2016-05-23 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(3504) called with curMem=178446, maxMem=766075207 -2016-05-23 20:00:52 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(1984) called with curMem=181950, maxMem=766075207 -2016-05-23 20:00:52 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1984.0 B, free 730.4 MB) -2016-05-23 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52196 (size: 1984.0 B, free: 730.6 MB) -2016-05-23 20:00:52 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:00:52 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24) -2016-05-23 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 3 tasks -2016-05-23 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 5, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:52 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 3) -2016-05-23 20:00:52 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 4) -2016-05-23 20:00:52 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 5) -2016-05-23 20:00:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 20:00:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 20:00:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 20:00:52 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 5). 2044 bytes result sent to driver -2016-05-23 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 4). 2044 bytes result sent to driver -2016-05-23 20:00:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 5) in 34 ms on localhost (1/3) -2016-05-23 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 4) in 36 ms on localhost (2/3) -2016-05-23 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 3) in 44 ms on localhost (3/3) -2016-05-23 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 20:00:52 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsColStatsLearning.scala:28) finished in 0.044 s -2016-05-23 20:00:52 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsColStatsLearning.scala:28, took 0.066571 s -2016-05-23 20:00:52 INFO SparkContext:59 - Starting job: count at StatisticsColStatsLearning.scala:29 -2016-05-23 20:00:52 INFO DAGScheduler:59 - Got job 2 (count at StatisticsColStatsLearning.scala:29) with 3 output partitions -2016-05-23 20:00:52 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at StatisticsColStatsLearning.scala:29) -2016-05-23 20:00:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:00:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:00:52 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24), which has no missing parents -2016-05-23 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=183934, maxMem=766075207 -2016-05-23 20:00:52 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-05-23 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(1946) called with curMem=187310, maxMem=766075207 -2016-05-23 20:00:52 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1946.0 B, free 730.4 MB) -2016-05-23 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52196 (size: 1946.0 B, free: 730.6 MB) -2016-05-23 20:00:52 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:00:52 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 2 (MapPartitionsRDD[3] at map at StatisticsColStatsLearning.scala:24) -2016-05-23 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-05-23 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 8, localhost, PROCESS_LOCAL, 2178 bytes) -2016-05-23 20:00:52 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 7) -2016-05-23 20:00:52 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 8) -2016-05-23 20:00:52 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 6) -2016-05-23 20:00:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:7+7 -2016-05-23 20:00:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:0+7 -2016-05-23 20:00:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/stats.txt:14+1 -2016-05-23 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 6). 2082 bytes result sent to driver -2016-05-23 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 7). 2082 bytes result sent to driver -2016-05-23 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 6) in 21 ms on localhost (1/3) -2016-05-23 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 7) in 21 ms on localhost (2/3) -2016-05-23 20:00:52 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 8). 2082 bytes result sent to driver -2016-05-23 20:00:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 8) in 24 ms on localhost (3/3) -2016-05-23 20:00:52 INFO DAGScheduler:59 - ResultStage 2 (count at StatisticsColStatsLearning.scala:29) finished in 0.027 s -2016-05-23 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 20:00:52 INFO DAGScheduler:59 - Job 2 finished: count at StatisticsColStatsLearning.scala:29, took 0.040997 s -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 20:00:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 20:00:52 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 20:00:52 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 20:00:52 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 20:00:52 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 20:00:52 INFO BlockManager:59 - BlockManager stopped -2016-05-23 20:00:52 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 20:00:52 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 20:00:52 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 20:00:52 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 20:00:52 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-8274dd60-9100-41bd-ace7-fb65e0bf5679 -2016-05-23 20:00:53 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 20:00:53 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 20:08:08 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 20:08:09 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 20:08:09 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 20:08:09 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 20:08:09 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 20:08:10 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 20:08:10 INFO Remoting:74 - Starting remoting -2016-05-23 20:08:11 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:52419] -2016-05-23 20:08:11 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52419. -2016-05-23 20:08:11 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 20:08:11 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 20:08:11 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-92bbc018-8d95-4c63-8388-41ca6256c8f9 -2016-05-23 20:08:11 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 20:08:11 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-88c95d7e-3d87-464c-a027-c2ec3c273bbc\httpd-258b1052-6c6c-4e13-8b0c-acaa5f101702 -2016-05-23 20:08:11 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 20:08:11 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:08:11 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52420 -2016-05-23 20:08:11 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52420. -2016-05-23 20:08:11 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 20:08:11 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:08:11 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 20:08:11 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 20:08:11 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 20:08:11 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 20:08:11 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 20:08:12 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52439. -2016-05-23 20:08:12 INFO NettyBlockTransferService:59 - Server created on 52439 -2016-05-23 20:08:12 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 20:08:12 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52439 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52439) -2016-05-23 20:08:12 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 20:08:12 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 20:08:12 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 20:08:12 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 20:08:12 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 20:08:12 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52439 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:08:12 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:20 -2016-05-23 20:08:12 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 20:08:12 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 20:08:12 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 20:08:12 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 20:08:12 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52439 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:08:12 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 20:08:14 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 20:08:14 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:08:14 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:08:14 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 20:08:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 20:08:14 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 20:08:14 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 20:08:14 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 20:08:14 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 20:08:14 INFO BlockManager:59 - BlockManager stopped -2016-05-23 20:08:14 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 20:08:14 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 20:08:14 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 20:08:14 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 20:08:14 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-88c95d7e-3d87-464c-a027-c2ec3c273bbc -2016-05-23 20:08:14 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 20:09:02 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 20:09:03 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 20:09:03 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 20:09:03 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 20:09:03 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 20:09:04 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 20:09:04 INFO Remoting:74 - Starting remoting -2016-05-23 20:09:04 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:52465] -2016-05-23 20:09:04 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52465. -2016-05-23 20:09:05 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 20:09:05 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 20:09:05 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-62dafc2f-05d9-4d07-8457-7e0886ee40d1 -2016-05-23 20:09:05 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 20:09:05 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-10fd18a2-a606-4f03-8f5a-ab828e431de4\httpd-3375653c-e942-4d9b-80b3-87405426871f -2016-05-23 20:09:05 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 20:09:05 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:09:05 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52466 -2016-05-23 20:09:05 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52466. -2016-05-23 20:09:05 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 20:09:05 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:09:05 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 20:09:05 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 20:09:05 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 20:09:05 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 20:09:05 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 20:09:06 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52485. -2016-05-23 20:09:06 INFO NettyBlockTransferService:59 - Server created on 52485 -2016-05-23 20:09:06 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 20:09:06 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52485 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52485) -2016-05-23 20:09:06 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 20:09:06 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 20:09:06 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 20:09:06 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 20:09:06 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 20:09:06 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52485 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:09:06 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:20 -2016-05-23 20:09:06 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 20:09:06 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 20:09:06 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 20:09:06 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 20:09:06 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52485 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:09:06 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 20:09:08 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 20:09:08 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:09:08 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:09:08 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:09:08 INFO DAGScheduler:59 - Got job 0 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:09:08 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at RowMatrix.scala:65) -2016-05-23 20:09:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:09:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:09:08 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:09:08 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=343200, maxMem=766075207 -2016-05-23 20:09:08 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.3 MB) -2016-05-23 20:09:08 INFO MemoryStore:59 - ensureFreeSpace(2469) called with curMem=347760, maxMem=766075207 -2016-05-23 20:09:08 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.3 MB) -2016-05-23 20:09:08 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52485 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:09:08 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:09:08 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 20:09:08 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-23 20:09:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:09:08 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 20:09:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:09:08 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 20:09:08 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 20:09:08 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 20:09:08 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 20:09:08 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 20:09:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:09:08 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2175 bytes result sent to driver -2016-05-23 20:09:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 164 ms on localhost (1/1) -2016-05-23 20:09:08 INFO DAGScheduler:59 - ResultStage 0 (first at RowMatrix.scala:65) finished in 0.189 s -2016-05-23 20:09:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 20:09:08 INFO DAGScheduler:59 - Job 0 finished: first at RowMatrix.scala:65, took 0.297672 s -2016-05-23 20:09:10 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52485 in memory (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:09:10 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-23 20:09:12 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:09:12 INFO DAGScheduler:59 - Got job 1 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:09:12 INFO DAGScheduler:59 - Final stage: ResultStage 1(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:09:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:09:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:09:12 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:09:12 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=343200, maxMem=766075207 -2016-05-23 20:09:12 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.3 KB, free 730.3 MB) -2016-05-23 20:09:12 INFO MemoryStore:59 - ensureFreeSpace(2856) called with curMem=348648, maxMem=766075207 -2016-05-23 20:09:12 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-05-23 20:09:12 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52485 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:09:12 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:09:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:09:12 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 20:09:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:09:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:09:12 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-23 20:09:12 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-23 20:09:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:09:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:09:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:09:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:09:12 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2360 bytes result sent to driver -2016-05-23 20:09:12 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2360 bytes result sent to driver -2016-05-23 20:09:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 98 ms on localhost (1/2) -2016-05-23 20:09:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 100 ms on localhost (2/2) -2016-05-23 20:09:12 INFO DAGScheduler:59 - ResultStage 1 (treeAggregate at RowMatrix.scala:332) finished in 0.101 s -2016-05-23 20:09:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 20:09:12 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 20:09:12 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 20:09:12 INFO DAGScheduler:59 - Job 1 finished: treeAggregate at RowMatrix.scala:332, took 0.150341 s -2016-05-23 20:09:12 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:09:12 INFO DAGScheduler:59 - Got job 2 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:09:12 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:09:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:09:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:09:12 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:09:12 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=351504, maxMem=766075207 -2016-05-23 20:09:12 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 20:09:12 INFO MemoryStore:59 - ensureFreeSpace(2851) called with curMem=357000, maxMem=766075207 -2016-05-23 20:09:12 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:09:12 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52485 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:09:12 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:09:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:09:12 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-23 20:09:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:09:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:09:12 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-23 20:09:12 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-23 20:09:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:09:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:09:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:09:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:09:12 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2238 bytes result sent to driver -2016-05-23 20:09:12 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2238 bytes result sent to driver -2016-05-23 20:09:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 32 ms on localhost (1/2) -2016-05-23 20:09:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 34 ms on localhost (2/2) -2016-05-23 20:09:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 20:09:12 INFO DAGScheduler:59 - ResultStage 2 (treeAggregate at RowMatrix.scala:124) finished in 0.035 s -2016-05-23 20:09:12 INFO DAGScheduler:59 - Job 2 finished: treeAggregate at RowMatrix.scala:124, took 0.057171 s -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 20:09:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 20:09:13 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 20:09:13 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 20:09:13 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 20:09:14 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 20:09:14 INFO BlockManager:59 - BlockManager stopped -2016-05-23 20:09:14 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 20:09:14 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 20:09:14 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 20:09:14 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 20:09:14 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-10fd18a2-a606-4f03-8f5a-ab828e431de4 -2016-05-23 20:10:10 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 20:10:10 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 20:10:11 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 20:10:11 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 20:10:11 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 20:10:11 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 20:10:11 INFO Remoting:74 - Starting remoting -2016-05-23 20:10:12 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:52509] -2016-05-23 20:10:12 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52509. -2016-05-23 20:10:12 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 20:10:12 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 20:10:12 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-642bd92d-f93d-4a87-9276-6b970eb0c1eb -2016-05-23 20:10:12 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 20:10:12 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-eab691df-fd55-42bd-a1d2-c0fd1906765c\httpd-4a622810-09a9-4936-98b5-0fc28fcf9237 -2016-05-23 20:10:12 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 20:10:12 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:10:12 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52510 -2016-05-23 20:10:12 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52510. -2016-05-23 20:10:12 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 20:10:12 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:10:12 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 20:10:12 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 20:10:12 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 20:10:12 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 20:10:12 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 20:10:13 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52529. -2016-05-23 20:10:13 INFO NettyBlockTransferService:59 - Server created on 52529 -2016-05-23 20:10:13 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 20:10:13 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52529 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52529) -2016-05-23 20:10:13 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 20:10:13 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 20:10:13 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 20:10:13 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 20:10:13 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 20:10:13 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52529 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:10:13 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:20 -2016-05-23 20:10:14 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 20:10:14 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 20:10:14 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 20:10:14 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 20:10:14 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52529 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:10:14 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 20:10:15 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 20:10:15 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:10:15 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:10:15 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:10:15 INFO DAGScheduler:59 - Got job 0 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:10:15 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at RowMatrix.scala:65) -2016-05-23 20:10:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:10:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:10:15 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:10:15 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=343200, maxMem=766075207 -2016-05-23 20:10:15 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.3 MB) -2016-05-23 20:10:15 INFO MemoryStore:59 - ensureFreeSpace(2469) called with curMem=347760, maxMem=766075207 -2016-05-23 20:10:15 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.3 MB) -2016-05-23 20:10:15 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52529 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:10:15 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:10:15 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 20:10:15 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-23 20:10:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:10:15 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 20:10:15 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:10:15 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 20:10:15 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 20:10:15 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 20:10:15 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 20:10:15 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 20:10:15 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:10:15 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2175 bytes result sent to driver -2016-05-23 20:10:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 150 ms on localhost (1/1) -2016-05-23 20:10:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 20:10:16 INFO DAGScheduler:59 - ResultStage 0 (first at RowMatrix.scala:65) finished in 0.171 s -2016-05-23 20:10:16 INFO DAGScheduler:59 - Job 0 finished: first at RowMatrix.scala:65, took 0.294148 s -2016-05-23 20:10:16 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52529 in memory (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:10:16 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:10:16 INFO DAGScheduler:59 - Got job 1 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:10:16 INFO DAGScheduler:59 - Final stage: ResultStage 1(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:10:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:10:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:10:16 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:10:16 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=343200, maxMem=766075207 -2016-05-23 20:10:16 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.3 KB, free 730.3 MB) -2016-05-23 20:10:16 INFO MemoryStore:59 - ensureFreeSpace(2856) called with curMem=348648, maxMem=766075207 -2016-05-23 20:10:16 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-05-23 20:10:16 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52529 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:10:16 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:10:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:10:16 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 20:10:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:10:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:10:16 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-23 20:10:16 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-23 20:10:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:10:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:10:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:10:16 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2360 bytes result sent to driver -2016-05-23 20:10:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 50 ms on localhost (1/2) -2016-05-23 20:10:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:10:16 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2360 bytes result sent to driver -2016-05-23 20:10:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 62 ms on localhost (2/2) -2016-05-23 20:10:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 20:10:16 INFO DAGScheduler:59 - ResultStage 1 (treeAggregate at RowMatrix.scala:332) finished in 0.066 s -2016-05-23 20:10:16 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 20:10:16 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 20:10:16 INFO DAGScheduler:59 - Job 1 finished: treeAggregate at RowMatrix.scala:332, took 0.100681 s -2016-05-23 20:10:16 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:10:16 INFO DAGScheduler:59 - Got job 2 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:10:16 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:10:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:10:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:10:16 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:10:16 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=351504, maxMem=766075207 -2016-05-23 20:10:16 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 20:10:16 INFO MemoryStore:59 - ensureFreeSpace(2851) called with curMem=357000, maxMem=766075207 -2016-05-23 20:10:16 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:10:16 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52529 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:10:16 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:10:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:10:16 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-23 20:10:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:10:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:10:16 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-23 20:10:16 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-23 20:10:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:10:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:10:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:10:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:10:16 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2238 bytes result sent to driver -2016-05-23 20:10:16 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2238 bytes result sent to driver -2016-05-23 20:10:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 31 ms on localhost (1/2) -2016-05-23 20:10:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 32 ms on localhost (2/2) -2016-05-23 20:10:16 INFO DAGScheduler:59 - ResultStage 2 (treeAggregate at RowMatrix.scala:124) finished in 0.033 s -2016-05-23 20:10:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 20:10:16 INFO DAGScheduler:59 - Job 2 finished: treeAggregate at RowMatrix.scala:124, took 0.052534 s -2016-05-23 20:10:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 20:10:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 20:10:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 20:10:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 20:10:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 20:10:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 20:10:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 20:10:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 20:10:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 20:10:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 20:10:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 20:10:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 20:10:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 20:10:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 20:10:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 20:10:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 20:10:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 20:10:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 20:10:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 20:10:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 20:10:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 20:10:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 20:10:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 20:10:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 20:10:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 20:10:17 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 20:10:17 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 20:10:17 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 20:10:17 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 20:10:17 INFO BlockManager:59 - BlockManager stopped -2016-05-23 20:10:17 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 20:10:17 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 20:10:17 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 20:10:17 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 20:10:17 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-eab691df-fd55-42bd-a1d2-c0fd1906765c -2016-05-23 20:10:17 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 20:13:54 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 20:13:55 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 20:13:55 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 20:13:55 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 20:13:55 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 20:13:56 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 20:13:56 INFO Remoting:74 - Starting remoting -2016-05-23 20:13:56 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:52620] -2016-05-23 20:13:56 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52620. -2016-05-23 20:13:56 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 20:13:56 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 20:13:56 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-d23c2dda-9999-4494-b501-3ee4f292ad65 -2016-05-23 20:13:56 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 20:13:56 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-a1940399-f680-4238-b373-f255d5cfa0b3\httpd-a3d9f542-ac61-4405-93ab-6e283d23746c -2016-05-23 20:13:56 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 20:13:57 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:13:57 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52621 -2016-05-23 20:13:57 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52621. -2016-05-23 20:13:57 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 20:13:57 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:13:57 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 20:13:57 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 20:13:57 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 20:13:57 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 20:13:57 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 20:13:57 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52640. -2016-05-23 20:13:57 INFO NettyBlockTransferService:59 - Server created on 52640 -2016-05-23 20:13:57 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 20:13:57 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52640 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52640) -2016-05-23 20:13:57 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 20:13:58 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 20:13:58 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 20:13:58 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 20:13:58 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 20:13:58 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52640 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:13:58 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:20 -2016-05-23 20:13:58 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 20:13:58 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 20:13:58 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 20:13:58 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 20:13:58 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52640 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:13:58 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 20:14:00 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 20:14:00 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:14:00 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:27 -2016-05-23 20:14:00 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:27) with 2 output partitions -2016-05-23 20:14:00 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:27) -2016-05-23 20:14:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:14:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:14:00 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21), which has no missing parents -2016-05-23 20:14:00 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 20:14:00 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:14:00 INFO MemoryStore:59 - ensureFreeSpace(2043) called with curMem=346712, maxMem=766075207 -2016-05-23 20:14:00 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2043.0 B, free 730.3 MB) -2016-05-23 20:14:00 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52640 (size: 2043.0 B, free: 730.6 MB) -2016-05-23 20:14:00 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:14:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21) -2016-05-23 20:14:00 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 20:14:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:14:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:14:00 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 20:14:00 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 20:14:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:14:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:14:00 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 20:14:00 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 20:14:00 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 20:14:00 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 20:14:00 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 20:14:00 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 20:14:00 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 20:14:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 100 ms on localhost (1/2) -2016-05-23 20:14:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 131 ms on localhost (2/2) -2016-05-23 20:14:00 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:27) finished in 0.147 s -2016-05-23 20:14:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 20:14:00 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:27, took 0.328444 s -2016-05-23 20:14:00 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:14:00 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 20:14:00 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 20:14:00 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 20:14:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:14:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:14:00 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24), which has no missing parents -2016-05-23 20:14:00 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348755, maxMem=766075207 -2016-05-23 20:14:00 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:14:00 INFO MemoryStore:59 - ensureFreeSpace(2041) called with curMem=352267, maxMem=766075207 -2016-05-23 20:14:00 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2041.0 B, free 730.2 MB) -2016-05-23 20:14:00 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52640 (size: 2041.0 B, free: 730.6 MB) -2016-05-23 20:14:00 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:14:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24) -2016-05-23 20:14:00 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 20:14:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:14:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:14:00 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 20:14:00 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 20:14:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:14:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:14:00 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 20:14:00 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 20:14:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 29 ms on localhost (1/2) -2016-05-23 20:14:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 28 ms on localhost (2/2) -2016-05-23 20:14:00 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:29) finished in 0.030 s -2016-05-23 20:14:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 20:14:00 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:29, took 0.055931 s -2016-05-23 20:14:00 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:14:00 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:14:00 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 20:14:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:14:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:14:00 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:14:00 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354308, maxMem=766075207 -2016-05-23 20:14:00 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 20:14:00 INFO MemoryStore:59 - ensureFreeSpace(2469) called with curMem=358868, maxMem=766075207 -2016-05-23 20:14:00 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:14:00 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52640 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:14:00 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:14:00 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 20:14:00 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 20:14:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:14:00 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 20:14:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:14:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:14:00 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 20:14:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 53 ms on localhost (1/1) -2016-05-23 20:14:00 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.053 s -2016-05-23 20:14:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 20:14:00 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.081210 s -2016-05-23 20:14:01 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:14:01 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:14:01 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:14:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:14:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:14:01 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:14:01 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=361337, maxMem=766075207 -2016-05-23 20:14:01 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 20:14:01 INFO MemoryStore:59 - ensureFreeSpace(2856) called with curMem=366785, maxMem=766075207 -2016-05-23 20:14:01 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:14:01 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52640 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:14:01 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:14:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:14:01 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 20:14:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:14:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:14:01 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 20:14:01 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 20:14:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:14:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:14:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:14:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:14:01 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 20:14:01 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 20:14:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 60 ms on localhost (1/2) -2016-05-23 20:14:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 59 ms on localhost (2/2) -2016-05-23 20:14:01 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.063 s -2016-05-23 20:14:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 20:14:01 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 20:14:01 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 20:14:01 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.104006 s -2016-05-23 20:14:01 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:14:01 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:14:01 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:14:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:14:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:14:01 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:14:01 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=369641, maxMem=766075207 -2016-05-23 20:14:01 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 20:14:01 INFO MemoryStore:59 - ensureFreeSpace(2851) called with curMem=375137, maxMem=766075207 -2016-05-23 20:14:01 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:14:01 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52640 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:14:01 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:14:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:14:01 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 20:14:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:14:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:14:01 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 20:14:01 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 20:14:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:14:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:14:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:14:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:14:01 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 20:14:01 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 20:14:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 51 ms on localhost (1/2) -2016-05-23 20:14:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 50 ms on localhost (2/2) -2016-05-23 20:14:01 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.053 s -2016-05-23 20:14:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 20:14:01 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.088051 s -2016-05-23 20:14:01 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:52640 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:14:01 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 20:14:01 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52640 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:14:01 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 20:14:01 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52640 in memory (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:14:01 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-23 20:14:01 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52640 in memory (size: 2041.0 B, free: 730.6 MB) -2016-05-23 20:14:01 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-23 20:14:01 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52640 in memory (size: 2043.0 B, free: 730.6 MB) -2016-05-23 20:14:01 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-23 20:14:01 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 20:14:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 20:14:02 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 20:14:02 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 20:14:02 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 20:14:02 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 20:14:02 INFO BlockManager:59 - BlockManager stopped -2016-05-23 20:14:02 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 20:14:02 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 20:14:02 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 20:14:02 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 20:14:02 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-a1940399-f680-4238-b373-f255d5cfa0b3 -2016-05-23 20:19:18 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 20:19:19 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 20:19:19 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 20:19:19 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 20:19:19 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 20:19:20 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 20:19:20 INFO Remoting:74 - Starting remoting -2016-05-23 20:19:20 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:52714] -2016-05-23 20:19:20 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52714. -2016-05-23 20:19:20 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 20:19:20 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 20:19:20 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-8f76f9fa-cf15-48fe-9606-0bc2687e762a -2016-05-23 20:19:20 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 20:19:21 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-4ca4478e-abc4-4b25-a495-2d376fae1040\httpd-6525aa35-fe6c-462e-911a-dcab82204fc3 -2016-05-23 20:19:21 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 20:19:21 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:19:21 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52715 -2016-05-23 20:19:21 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52715. -2016-05-23 20:19:21 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 20:19:21 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:19:21 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 20:19:21 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 20:19:21 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 20:19:21 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 20:19:21 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 20:19:21 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52734. -2016-05-23 20:19:21 INFO NettyBlockTransferService:59 - Server created on 52734 -2016-05-23 20:19:21 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 20:19:21 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52734 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52734) -2016-05-23 20:19:21 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 20:19:22 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 20:19:22 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 20:19:22 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 20:19:22 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 20:19:22 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52734 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:19:22 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:20 -2016-05-23 20:19:22 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 20:19:22 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 20:19:22 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 20:19:22 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 20:19:22 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52734 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:19:22 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 20:19:24 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 20:19:24 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:19:24 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:27 -2016-05-23 20:19:24 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:27) with 2 output partitions -2016-05-23 20:19:24 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:27) -2016-05-23 20:19:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:19:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:19:24 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21), which has no missing parents -2016-05-23 20:19:24 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 20:19:24 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:19:24 INFO MemoryStore:59 - ensureFreeSpace(2041) called with curMem=346712, maxMem=766075207 -2016-05-23 20:19:24 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2041.0 B, free 730.3 MB) -2016-05-23 20:19:24 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52734 (size: 2041.0 B, free: 730.6 MB) -2016-05-23 20:19:24 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:19:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21) -2016-05-23 20:19:24 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 20:19:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:19:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:19:24 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 20:19:24 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 20:19:24 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:19:24 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:19:24 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 20:19:24 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 20:19:24 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 20:19:24 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 20:19:24 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 20:19:25 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 20:19:25 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 20:19:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 121 ms on localhost (1/2) -2016-05-23 20:19:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 101 ms on localhost (2/2) -2016-05-23 20:19:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 20:19:25 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:27) finished in 0.154 s -2016-05-23 20:19:25 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:27, took 0.576431 s -2016-05-23 20:19:25 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:19:25 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 20:19:25 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 20:19:25 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 20:19:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:19:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:19:25 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24), which has no missing parents -2016-05-23 20:19:25 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348753, maxMem=766075207 -2016-05-23 20:19:25 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:19:25 INFO MemoryStore:59 - ensureFreeSpace(2041) called with curMem=352265, maxMem=766075207 -2016-05-23 20:19:25 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2041.0 B, free 730.2 MB) -2016-05-23 20:19:25 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52734 (size: 2041.0 B, free: 730.6 MB) -2016-05-23 20:19:25 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:19:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24) -2016-05-23 20:19:25 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 20:19:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:19:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:19:25 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 20:19:25 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 20:19:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:19:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:19:25 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 20:19:25 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 20:19:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 24 ms on localhost (1/2) -2016-05-23 20:19:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 27 ms on localhost (2/2) -2016-05-23 20:19:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 20:19:25 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:29) finished in 0.025 s -2016-05-23 20:19:25 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:29, took 0.077382 s -2016-05-23 20:19:25 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:19:25 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:19:25 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 20:19:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:19:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:19:25 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:19:25 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354306, maxMem=766075207 -2016-05-23 20:19:25 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 20:19:25 INFO MemoryStore:59 - ensureFreeSpace(2463) called with curMem=358866, maxMem=766075207 -2016-05-23 20:19:25 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:19:25 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52734 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:19:25 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:19:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 20:19:25 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 20:19:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:19:25 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 20:19:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:19:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:19:25 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 20:19:25 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.038 s -2016-05-23 20:19:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 38 ms on localhost (1/1) -2016-05-23 20:19:25 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.163735 s -2016-05-23 20:19:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 20:19:25 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:19:25 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:19:25 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:19:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:19:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:19:25 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:19:25 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=361329, maxMem=766075207 -2016-05-23 20:19:25 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 20:19:25 INFO MemoryStore:59 - ensureFreeSpace(2862) called with curMem=366777, maxMem=766075207 -2016-05-23 20:19:25 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:19:25 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52734 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:19:25 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:19:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:19:25 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 20:19:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:19:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:19:25 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 20:19:25 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 20:19:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:19:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:19:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:19:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:19:25 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 20:19:25 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 20:19:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 47 ms on localhost (1/2) -2016-05-23 20:19:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 47 ms on localhost (2/2) -2016-05-23 20:19:25 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.050 s -2016-05-23 20:19:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 20:19:25 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 20:19:25 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 20:19:25 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.079962 s -2016-05-23 20:19:25 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:19:25 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:19:25 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:19:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:19:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:19:25 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:19:25 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=369639, maxMem=766075207 -2016-05-23 20:19:25 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 20:19:25 INFO MemoryStore:59 - ensureFreeSpace(2853) called with curMem=375135, maxMem=766075207 -2016-05-23 20:19:25 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:19:26 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52734 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:19:26 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:19:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:19:26 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 20:19:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:19:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:19:26 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 20:19:26 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 20:19:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:19:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:19:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:19:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:19:26 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 20:19:26 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 20:19:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 33 ms on localhost (1/2) -2016-05-23 20:19:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 33 ms on localhost (2/2) -2016-05-23 20:19:26 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.035 s -2016-05-23 20:19:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 20:19:26 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.061235 s -2016-05-23 20:19:26 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 20:19:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 20:19:26 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 20:19:26 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 20:19:26 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 20:19:26 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 20:19:26 INFO BlockManager:59 - BlockManager stopped -2016-05-23 20:19:26 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 20:19:26 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 20:19:26 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 20:19:26 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 20:19:26 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-4ca4478e-abc4-4b25-a495-2d376fae1040 -2016-05-23 20:20:21 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 20:20:22 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 20:20:22 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 20:20:22 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 20:20:22 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 20:20:23 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 20:20:23 INFO Remoting:74 - Starting remoting -2016-05-23 20:20:23 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:52757] -2016-05-23 20:20:23 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52757. -2016-05-23 20:20:23 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 20:20:23 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 20:20:23 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-f4000bcd-5221-4c4d-a260-c20ea2226071 -2016-05-23 20:20:23 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 20:20:23 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-49c8384e-9c93-459b-aeed-f9c91c22c4f4\httpd-b8f0ba96-f987-4b9a-8ec3-250c72925b57 -2016-05-23 20:20:23 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 20:20:23 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:20:23 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52758 -2016-05-23 20:20:23 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52758. -2016-05-23 20:20:23 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 20:20:23 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:20:23 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 20:20:23 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 20:20:23 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 20:20:23 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 20:20:24 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 20:20:24 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52777. -2016-05-23 20:20:24 INFO NettyBlockTransferService:59 - Server created on 52777 -2016-05-23 20:20:24 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 20:20:24 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52777 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52777) -2016-05-23 20:20:24 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 20:20:24 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 20:20:24 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 20:20:25 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 20:20:25 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 20:20:25 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52777 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:20:25 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:20 -2016-05-23 20:20:25 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 20:20:25 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 20:20:25 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 20:20:25 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 20:20:25 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52777 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:20:25 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 20:20:26 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 20:20:26 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:20:26 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:27 -2016-05-23 20:20:26 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:27) with 2 output partitions -2016-05-23 20:20:26 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:27) -2016-05-23 20:20:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:20:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:20:26 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21), which has no missing parents -2016-05-23 20:20:26 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 20:20:26 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:20:26 INFO MemoryStore:59 - ensureFreeSpace(2041) called with curMem=346712, maxMem=766075207 -2016-05-23 20:20:26 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2041.0 B, free 730.3 MB) -2016-05-23 20:20:26 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52777 (size: 2041.0 B, free: 730.6 MB) -2016-05-23 20:20:26 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21) -2016-05-23 20:20:26 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 20:20:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:20:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:20:26 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 20:20:26 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 20:20:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:20:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:20:26 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 20:20:26 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 20:20:26 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 20:20:26 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 20:20:26 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 20:20:26 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 20:20:26 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 20:20:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 96 ms on localhost (1/2) -2016-05-23 20:20:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 121 ms on localhost (2/2) -2016-05-23 20:20:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 20:20:26 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:27) finished in 0.140 s -2016-05-23 20:20:26 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:27, took 0.265685 s -2016-05-23 20:20:26 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:20:26 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 20:20:26 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 20:20:26 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 20:20:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:20:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:20:26 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24), which has no missing parents -2016-05-23 20:20:26 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348753, maxMem=766075207 -2016-05-23 20:20:26 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:20:26 INFO MemoryStore:59 - ensureFreeSpace(2041) called with curMem=352265, maxMem=766075207 -2016-05-23 20:20:26 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2041.0 B, free 730.2 MB) -2016-05-23 20:20:26 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52777 (size: 2041.0 B, free: 730.6 MB) -2016-05-23 20:20:26 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24) -2016-05-23 20:20:26 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 20:20:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:20:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:20:26 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 20:20:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:20:26 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 20:20:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:20:27 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 20:20:27 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 20:20:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 38 ms on localhost (1/2) -2016-05-23 20:20:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 37 ms on localhost (2/2) -2016-05-23 20:20:27 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:29) finished in 0.041 s -2016-05-23 20:20:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 20:20:27 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:29, took 0.062485 s -2016-05-23 20:20:27 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:20:27 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:20:27 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 20:20:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:20:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:20:27 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:20:27 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354306, maxMem=766075207 -2016-05-23 20:20:27 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 20:20:27 INFO MemoryStore:59 - ensureFreeSpace(2463) called with curMem=358866, maxMem=766075207 -2016-05-23 20:20:27 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:20:27 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52777 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:20:27 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 20:20:27 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 20:20:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:20:27 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 20:20:27 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52777 in memory (size: 2041.0 B, free: 730.6 MB) -2016-05-23 20:20:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:20:27 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52777 in memory (size: 2041.0 B, free: 730.6 MB) -2016-05-23 20:20:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:20:27 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 20:20:27 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.052 s -2016-05-23 20:20:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 40 ms on localhost (1/1) -2016-05-23 20:20:27 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.100280 s -2016-05-23 20:20:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 20:20:27 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:20:27 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:20:27 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:20:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:20:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:20:27 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:20:27 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=350223, maxMem=766075207 -2016-05-23 20:20:27 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 20:20:27 INFO MemoryStore:59 - ensureFreeSpace(2862) called with curMem=355671, maxMem=766075207 -2016-05-23 20:20:27 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:20:27 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52777 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:20:27 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:20:27 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 20:20:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:20:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:20:27 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 20:20:27 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 20:20:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:20:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:20:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:20:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:20:27 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 20:20:27 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 20:20:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 45 ms on localhost (1/2) -2016-05-23 20:20:27 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.047 s -2016-05-23 20:20:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 44 ms on localhost (2/2) -2016-05-23 20:20:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 20:20:27 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 20:20:27 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 20:20:27 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.078757 s -2016-05-23 20:20:27 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:20:27 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:20:27 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:20:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:20:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:20:27 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:20:27 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=358533, maxMem=766075207 -2016-05-23 20:20:27 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 20:20:27 INFO MemoryStore:59 - ensureFreeSpace(2853) called with curMem=364029, maxMem=766075207 -2016-05-23 20:20:27 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:20:27 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52777 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:20:27 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:20:27 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 20:20:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:20:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:20:27 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 20:20:27 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 20:20:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:20:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:20:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:20:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:20:27 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 20:20:27 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 20:20:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 35 ms on localhost (1/2) -2016-05-23 20:20:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 38 ms on localhost (2/2) -2016-05-23 20:20:27 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.039 s -2016-05-23 20:20:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 20:20:27 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.061920 s -2016-05-23 20:20:28 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:20:28 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 20:20:28 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:20:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:20:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:20:28 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:20:28 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=366882, maxMem=766075207 -2016-05-23 20:20:28 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 20:20:28 INFO MemoryStore:59 - ensureFreeSpace(2745) called with curMem=372226, maxMem=766075207 -2016-05-23 20:20:28 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 20:20:28 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52777 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 20:20:28 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:20:28 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 20:20:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:20:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:20:28 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 20:20:28 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 20:20:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:20:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:20:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:20:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:20:28 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 20:20:28 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 20:20:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 86 ms on localhost (1/2) -2016-05-23 20:20:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 89 ms on localhost (2/2) -2016-05-23 20:20:28 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.089 s -2016-05-23 20:20:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 20:20:28 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.106748 s -2016-05-23 20:20:28 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:20:28 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:20:28 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 20:20:28 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:20:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 20:20:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 20:20:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:20:28 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=374971, maxMem=766075207 -2016-05-23 20:20:28 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 20:20:28 INFO MemoryStore:59 - ensureFreeSpace(3155) called with curMem=380899, maxMem=766075207 -2016-05-23 20:20:28 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 20:20:28 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52777 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:20:28 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:20:28 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 20:20:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:20:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:20:28 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 20:20:28 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 20:20:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:20:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:20:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:20:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:20:28 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 20:20:28 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 20:20:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 227 ms on localhost (1/2) -2016-05-23 20:20:28 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.232 s -2016-05-23 20:20:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 232 ms on localhost (2/2) -2016-05-23 20:20:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 20:20:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:20:28 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:20:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 20:20:28 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:20:28 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 20:20:28 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:20:28 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=384054, maxMem=766075207 -2016-05-23 20:20:28 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:20:28 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=386558, maxMem=766075207 -2016-05-23 20:20:28 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 20:20:28 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52777 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:20:28 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:20:28 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 20:20:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:20:28 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 20:20:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:20:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 41 ms -2016-05-23 20:20:29 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 166 ms on localhost (1/1) -2016-05-23 20:20:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 20:20:29 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.168 s -2016-05-23 20:20:29 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.552923 s -2016-05-23 20:20:29 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:20:29 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:52777 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:20:29 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 20:20:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:20:29 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:52777 in memory (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:20:29 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:20:29 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 20:20:29 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:20:29 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 20:20:29 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 20:20:29 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 20:20:29 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:20:29 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:52777 in memory (size: 2.7 KB, free: 730.6 MB) -2016-05-23 20:20:29 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=366882, maxMem=766075207 -2016-05-23 20:20:29 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 20:20:29 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:20:29 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=368221, maxMem=766075207 -2016-05-23 20:20:29 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:20:29 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:52777 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:20:29 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:52777 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:20:29 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:20:29 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 20:20:29 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:20:29 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 20:20:29 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 20:20:29 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52777 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:20:29 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 20:20:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:20:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:20:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:20:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:20:29 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 20:20:29 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 53 ms on localhost (1/2) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 51 ms on localhost (2/2) -2016-05-23 20:20:29 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.055 s -2016-05-23 20:20:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 20:20:29 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:20:29 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:20:29 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 20:20:29 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:20:29 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 20:20:29 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:20:29 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=356697, maxMem=766075207 -2016-05-23 20:20:29 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 20:20:29 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=361553, maxMem=766075207 -2016-05-23 20:20:29 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:20:29 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:52777 (size: 2.5 KB, free: 730.6 MB) -2016-05-23 20:20:29 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:20:29 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:20:29 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 20:20:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:20:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:20:29 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 46 ms on localhost (1/1) -2016-05-23 20:20:29 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.046 s -2016-05-23 20:20:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 20:20:29 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.166253 s -2016-05-23 20:20:29 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:20:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:20:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 20:20:29 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:20:29 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:20:29 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 20:20:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:20:29 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:20:29 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=364084, maxMem=766075207 -2016-05-23 20:20:29 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 20:20:29 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=369740, maxMem=766075207 -2016-05-23 20:20:29 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:20:29 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:52777 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:20:29 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:20:29 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:20:29 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 20:20:29 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 20:20:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:20:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:20:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:20:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:20:29 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 29 ms on localhost (1/2) -2016-05-23 20:20:29 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 30 ms on localhost (2/2) -2016-05-23 20:20:29 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.032 s -2016-05-23 20:20:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 20:20:29 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.052262 s -2016-05-23 20:20:29 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:20:29 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:20:29 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:20:29 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 20:20:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:20:29 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:20:29 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=372648, maxMem=766075207 -2016-05-23 20:20:29 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:20:29 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=378344, maxMem=766075207 -2016-05-23 20:20:29 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:20:29 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:52777 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:20:29 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:20:29 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:20:29 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 20:20:29 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 20:20:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:20:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:20:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:20:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:20:29 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 20:20:29 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 21 ms on localhost (1/2) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 20 ms on localhost (2/2) -2016-05-23 20:20:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 20:20:29 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.023 s -2016-05-23 20:20:29 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.038825 s -2016-05-23 20:20:29 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:20:29 INFO DAGScheduler:59 - Got job 10 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:20:29 INFO DAGScheduler:59 - Final stage: ResultStage 17(first at RowMatrix.scala:65) -2016-05-23 20:20:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:20:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:20:29 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:20:29 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=381233, maxMem=766075207 -2016-05-23 20:20:29 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:20:29 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=383737, maxMem=766075207 -2016-05-23 20:20:29 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 20:20:29 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:52777 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 20:20:29 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 17 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 20:20:29 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 1 tasks -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:20:29 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 20:20:29 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 1046 bytes result sent to driver -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 18 ms on localhost (1/1) -2016-05-23 20:20:29 INFO DAGScheduler:59 - ResultStage 17 (first at RowMatrix.scala:65) finished in 0.018 s -2016-05-23 20:20:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 20:20:29 INFO DAGScheduler:59 - Job 10 finished: first at RowMatrix.scala:65, took 0.032272 s -2016-05-23 20:20:29 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:20:29 INFO DAGScheduler:59 - Got job 11 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:20:29 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:20:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:20:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:20:29 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:20:29 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=385261, maxMem=766075207 -2016-05-23 20:20:29 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:20:29 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=388661, maxMem=766075207 -2016-05-23 20:20:29 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 20:20:29 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:52777 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 20:20:29 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:20:29 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 22, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 23, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 24, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:20:29 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 23) -2016-05-23 20:20:29 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 22) -2016-05-23 20:20:29 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 24) -2016-05-23 20:20:29 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 25) -2016-05-23 20:20:29 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 22). 1231 bytes result sent to driver -2016-05-23 20:20:29 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 23). 1231 bytes result sent to driver -2016-05-23 20:20:29 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 25). 1231 bytes result sent to driver -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 22) in 16 ms on localhost (1/4) -2016-05-23 20:20:29 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 24). 1231 bytes result sent to driver -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 23) in 17 ms on localhost (2/4) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 24) in 16 ms on localhost (3/4) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 25) in 15 ms on localhost (4/4) -2016-05-23 20:20:29 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at RowMatrix.scala:332) finished in 0.019 s -2016-05-23 20:20:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 20:20:29 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at RowMatrix.scala:332, took 0.034053 s -2016-05-23 20:20:29 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:20:29 INFO DAGScheduler:59 - Got job 12 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:20:29 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:20:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:20:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:20:29 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:20:29 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=390595, maxMem=766075207 -2016-05-23 20:20:29 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 20:20:29 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=394035, maxMem=766075207 -2016-05-23 20:20:29 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 20:20:29 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:52777 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 20:20:29 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 19 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:20:29 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 4 tasks -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 26, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 27, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 19.0 (TID 28, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:20:29 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 26) -2016-05-23 20:20:29 INFO Executor:59 - Running task 3.0 in stage 19.0 (TID 29) -2016-05-23 20:20:29 INFO Executor:59 - Running task 2.0 in stage 19.0 (TID 28) -2016-05-23 20:20:29 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 26). 1109 bytes result sent to driver -2016-05-23 20:20:29 INFO Executor:59 - Finished task 3.0 in stage 19.0 (TID 29). 1109 bytes result sent to driver -2016-05-23 20:20:29 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 27) -2016-05-23 20:20:29 INFO Executor:59 - Finished task 2.0 in stage 19.0 (TID 28). 1109 bytes result sent to driver -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 19.0 (TID 28) in 15 ms on localhost (1/4) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 19.0 (TID 29) in 14 ms on localhost (2/4) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 26) in 22 ms on localhost (3/4) -2016-05-23 20:20:29 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 27). 1109 bytes result sent to driver -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 27) in 23 ms on localhost (4/4) -2016-05-23 20:20:29 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at RowMatrix.scala:124) finished in 0.026 s -2016-05-23 20:20:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 20:20:29 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at RowMatrix.scala:124, took 0.039723 s -2016-05-23 20:20:29 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:20:29 INFO DAGScheduler:59 - Got job 13 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:20:29 INFO DAGScheduler:59 - Final stage: ResultStage 20(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:20:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:20:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:20:29 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:20:29 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=395953, maxMem=766075207 -2016-05-23 20:20:29 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:20:29 INFO MemoryStore:59 - ensureFreeSpace(1860) called with curMem=399329, maxMem=766075207 -2016-05-23 20:20:29 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1860.0 B, free 730.2 MB) -2016-05-23 20:20:29 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:52777 (size: 1860.0 B, free: 730.5 MB) -2016-05-23 20:20:29 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:20:29 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:20:29 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 20:20:29 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 20:20:29 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1334 bytes result sent to driver -2016-05-23 20:20:29 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1294 bytes result sent to driver -2016-05-23 20:20:29 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 20:20:29 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1334 bytes result sent to driver -2016-05-23 20:20:29 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 20:20:29 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1294 bytes result sent to driver -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 60 ms on localhost (1/4) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 66 ms on localhost (2/4) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 69 ms on localhost (3/4) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 72 ms on localhost (4/4) -2016-05-23 20:20:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 20:20:29 INFO DAGScheduler:59 - ResultStage 20 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.073 s -2016-05-23 20:20:29 INFO DAGScheduler:59 - Job 13 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.092509 s -2016-05-23 20:20:29 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:20:29 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:20:29 INFO DAGScheduler:59 - Got job 14 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:20:29 INFO DAGScheduler:59 - Final stage: ResultStage 22(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:20:29 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 21) -2016-05-23 20:20:29 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 21) -2016-05-23 20:20:29 INFO DAGScheduler:59 - Submitting ShuffleMapStage 21 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:20:29 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=401189, maxMem=766075207 -2016-05-23 20:20:29 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 20:20:29 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=405229, maxMem=766075207 -2016-05-23 20:20:29 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:20:29 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:52777 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:20:29 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 21 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:20:29 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:20:29 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 20:20:29 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1161 bytes result sent to driver -2016-05-23 20:20:29 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 20:20:29 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 20:20:29 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 20:20:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 69 ms on localhost (1/4) -2016-05-23 20:20:29 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1161 bytes result sent to driver -2016-05-23 20:20:30 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1161 bytes result sent to driver -2016-05-23 20:20:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 138 ms on localhost (2/4) -2016-05-23 20:20:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 142 ms on localhost (3/4) -2016-05-23 20:20:30 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1161 bytes result sent to driver -2016-05-23 20:20:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 153 ms on localhost (4/4) -2016-05-23 20:20:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 20:20:30 INFO DAGScheduler:59 - ShuffleMapStage 21 (flatMap at SpearmanCorrelation.scala:49) finished in 0.159 s -2016-05-23 20:20:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:20:30 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:20:30 INFO DAGScheduler:59 - waiting: Set(ResultStage 22) -2016-05-23 20:20:30 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:20:30 INFO DAGScheduler:59 - Missing parents for ResultStage 22: List() -2016-05-23 20:20:30 INFO DAGScheduler:59 - Submitting ResultStage 22 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:20:30 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=407511, maxMem=766075207 -2016-05-23 20:20:30 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:20:30 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=410055, maxMem=766075207 -2016-05-23 20:20:30 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 20:20:30 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:52777 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 20:20:30 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:30 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 22 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:20:30 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 3 tasks -2016-05-23 20:20:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:20:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:20:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:20:30 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 20:20:30 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:20:30 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:20:30 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1203 bytes result sent to driver -2016-05-23 20:20:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 25 ms on localhost (1/3) -2016-05-23 20:20:30 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1203 bytes result sent to driver -2016-05-23 20:20:30 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1203 bytes result sent to driver -2016-05-23 20:20:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 41 ms on localhost (2/3) -2016-05-23 20:20:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 45 ms on localhost (3/3) -2016-05-23 20:20:30 INFO DAGScheduler:59 - ResultStage 22 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.046 s -2016-05-23 20:20:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 20:20:30 INFO DAGScheduler:59 - Job 14 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.325345 s -2016-05-23 20:20:30 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:20:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:20:30 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:20:30 INFO DAGScheduler:59 - Got job 15 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:20:30 INFO DAGScheduler:59 - Final stage: ResultStage 25(first at RowMatrix.scala:65) -2016-05-23 20:20:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 24) -2016-05-23 20:20:30 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 24) -2016-05-23 20:20:30 INFO DAGScheduler:59 - Submitting ShuffleMapStage 24 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:20:30 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=411623, maxMem=766075207 -2016-05-23 20:20:30 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:20:30 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=415855, maxMem=766075207 -2016-05-23 20:20:30 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 20:20:30 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:52777 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:20:30 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 24 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:20:30 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 4 tasks -2016-05-23 20:20:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 41, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:20:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 42, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:20:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 43, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:20:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 24.0 (TID 44, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:20:30 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 43) -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:20:30 INFO Executor:59 - Running task 3.0 in stage 24.0 (TID 44) -2016-05-23 20:20:30 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 41) -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:20:30 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 42) -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:20:30 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 41). 1377 bytes result sent to driver -2016-05-23 20:20:30 INFO Executor:59 - Finished task 3.0 in stage 24.0 (TID 44). 1377 bytes result sent to driver -2016-05-23 20:20:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 41) in 163 ms on localhost (1/4) -2016-05-23 20:20:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 24.0 (TID 44) in 160 ms on localhost (2/4) -2016-05-23 20:20:30 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 42). 1377 bytes result sent to driver -2016-05-23 20:20:30 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 43). 1377 bytes result sent to driver -2016-05-23 20:20:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 42) in 185 ms on localhost (3/4) -2016-05-23 20:20:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 43) in 184 ms on localhost (4/4) -2016-05-23 20:20:30 INFO DAGScheduler:59 - ShuffleMapStage 24 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.188 s -2016-05-23 20:20:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 20:20:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:20:30 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:20:30 INFO DAGScheduler:59 - waiting: Set(ResultStage 25) -2016-05-23 20:20:30 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:20:30 INFO DAGScheduler:59 - Missing parents for ResultStage 25: List() -2016-05-23 20:20:30 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:20:30 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=418161, maxMem=766075207 -2016-05-23 20:20:30 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 20:20:30 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=423057, maxMem=766075207 -2016-05-23 20:20:30 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:20:30 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:52777 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:20:30 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 25 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:20:30 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 1 tasks -2016-05-23 20:20:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 45, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:20:30 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 45) -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:20:30 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 45). 1296 bytes result sent to driver -2016-05-23 20:20:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 45) in 16 ms on localhost (1/1) -2016-05-23 20:20:30 INFO DAGScheduler:59 - ResultStage 25 (first at RowMatrix.scala:65) finished in 0.017 s -2016-05-23 20:20:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-23 20:20:30 INFO DAGScheduler:59 - Job 15 finished: first at RowMatrix.scala:65, took 0.242230 s -2016-05-23 20:20:30 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:20:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:20:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 20:20:30 INFO DAGScheduler:59 - Got job 16 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:20:30 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:20:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 27) -2016-05-23 20:20:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:20:30 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:20:30 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=425599, maxMem=766075207 -2016-05-23 20:20:30 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:20:30 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=431295, maxMem=766075207 -2016-05-23 20:20:30 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:20:30 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:52777 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:20:30 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 28 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:20:30 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 4 tasks -2016-05-23 20:20:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:20:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:20:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 28.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:20:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 28.0 (TID 49, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:20:30 INFO Executor:59 - Running task 2.0 in stage 28.0 (TID 48) -2016-05-23 20:20:30 INFO Executor:59 - Running task 3.0 in stage 28.0 (TID 49) -2016-05-23 20:20:30 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 47) -2016-05-23 20:20:30 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 46) -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:20:30 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 47). 1481 bytes result sent to driver -2016-05-23 20:20:30 INFO Executor:59 - Finished task 3.0 in stage 28.0 (TID 49). 1481 bytes result sent to driver -2016-05-23 20:20:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 47) in 34 ms on localhost (1/4) -2016-05-23 20:20:30 INFO Executor:59 - Finished task 2.0 in stage 28.0 (TID 48). 1481 bytes result sent to driver -2016-05-23 20:20:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 28.0 (TID 49) in 33 ms on localhost (2/4) -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:20:30 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 46). 1481 bytes result sent to driver -2016-05-23 20:20:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 28.0 (TID 48) in 56 ms on localhost (3/4) -2016-05-23 20:20:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 46) in 60 ms on localhost (4/4) -2016-05-23 20:20:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-23 20:20:30 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at RowMatrix.scala:332) finished in 0.063 s -2016-05-23 20:20:30 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at RowMatrix.scala:332, took 0.112918 s -2016-05-23 20:20:30 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:20:30 INFO DAGScheduler:59 - Got job 17 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:20:30 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:20:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30) -2016-05-23 20:20:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:20:30 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:20:30 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=434204, maxMem=766075207 -2016-05-23 20:20:30 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:20:30 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=439940, maxMem=766075207 -2016-05-23 20:20:30 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:20:30 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:52777 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:20:30 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:20:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 31 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:20:30 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 4 tasks -2016-05-23 20:20:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 50, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:20:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 51, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:20:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 31.0 (TID 52, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:20:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 31.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:20:30 INFO Executor:59 - Running task 2.0 in stage 31.0 (TID 52) -2016-05-23 20:20:30 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 51) -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:20:30 INFO Executor:59 - Running task 3.0 in stage 31.0 (TID 53) -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:20:30 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 51). 1359 bytes result sent to driver -2016-05-23 20:20:30 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 50) -2016-05-23 20:20:30 INFO Executor:59 - Finished task 3.0 in stage 31.0 (TID 53). 1359 bytes result sent to driver -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:20:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:20:30 INFO Executor:59 - Finished task 2.0 in stage 31.0 (TID 52). 1359 bytes result sent to driver -2016-05-23 20:20:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 51) in 40 ms on localhost (1/4) -2016-05-23 20:20:30 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 50). 1359 bytes result sent to driver -2016-05-23 20:20:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 31.0 (TID 53) in 39 ms on localhost (2/4) -2016-05-23 20:20:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 31.0 (TID 52) in 41 ms on localhost (3/4) -2016-05-23 20:20:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 50) in 46 ms on localhost (4/4) -2016-05-23 20:20:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-23 20:20:30 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at RowMatrix.scala:124) finished in 0.047 s -2016-05-23 20:20:30 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at RowMatrix.scala:124, took 0.071272 s -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 20:20:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 20:20:30 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 20:20:30 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 20:20:30 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 20:20:31 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 20:20:31 INFO BlockManager:59 - BlockManager stopped -2016-05-23 20:20:31 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 20:20:31 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 20:20:31 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 20:20:31 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 20:20:31 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-49c8384e-9c93-459b-aeed-f9c91c22c4f4 -2016-05-23 20:21:09 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 20:21:10 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 20:21:10 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 20:21:10 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 20:21:10 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 20:21:11 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 20:21:11 INFO Remoting:74 - Starting remoting -2016-05-23 20:21:11 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:52804] -2016-05-23 20:21:11 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52804. -2016-05-23 20:21:11 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 20:21:11 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 20:21:11 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-6a492c61-c222-4ebe-9029-16eb0835abf0 -2016-05-23 20:21:11 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 20:21:11 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-6f1e3050-a799-45a4-80f7-ffb3070244b5\httpd-d71a6ab3-bef3-48ba-9f89-c646df743878 -2016-05-23 20:21:11 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 20:21:11 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:21:11 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52805 -2016-05-23 20:21:11 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52805. -2016-05-23 20:21:11 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 20:21:12 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:21:12 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 20:21:12 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 20:21:12 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 20:21:12 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 20:21:12 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 20:21:12 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52824. -2016-05-23 20:21:12 INFO NettyBlockTransferService:59 - Server created on 52824 -2016-05-23 20:21:12 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 20:21:12 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52824 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52824) -2016-05-23 20:21:12 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 20:21:13 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 20:21:13 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 20:21:13 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 20:21:13 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 20:21:13 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52824 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:21:13 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:20 -2016-05-23 20:21:13 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 20:21:13 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 20:21:13 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 20:21:13 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 20:21:13 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52824 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:21:13 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 20:21:14 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 20:21:14 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:21:14 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:27 -2016-05-23 20:21:14 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:27) with 2 output partitions -2016-05-23 20:21:14 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:27) -2016-05-23 20:21:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:21:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:21:14 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21), which has no missing parents -2016-05-23 20:21:15 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 20:21:15 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:21:15 INFO MemoryStore:59 - ensureFreeSpace(2043) called with curMem=346712, maxMem=766075207 -2016-05-23 20:21:15 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2043.0 B, free 730.3 MB) -2016-05-23 20:21:15 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52824 (size: 2043.0 B, free: 730.6 MB) -2016-05-23 20:21:15 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21) -2016-05-23 20:21:15 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 20:21:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:21:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:21:15 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 20:21:15 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 20:21:15 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:21:15 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:21:15 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 20:21:15 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 20:21:15 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 20:21:15 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 20:21:15 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 20:21:15 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 20:21:15 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 20:21:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 94 ms on localhost (1/2) -2016-05-23 20:21:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 118 ms on localhost (2/2) -2016-05-23 20:21:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 20:21:15 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:27) finished in 0.135 s -2016-05-23 20:21:15 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:27, took 0.270249 s -2016-05-23 20:21:15 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:21:15 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 20:21:15 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 20:21:15 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 20:21:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:21:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:21:15 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24), which has no missing parents -2016-05-23 20:21:15 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348755, maxMem=766075207 -2016-05-23 20:21:15 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:21:15 INFO MemoryStore:59 - ensureFreeSpace(2043) called with curMem=352267, maxMem=766075207 -2016-05-23 20:21:15 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2043.0 B, free 730.2 MB) -2016-05-23 20:21:15 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52824 (size: 2043.0 B, free: 730.6 MB) -2016-05-23 20:21:15 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24) -2016-05-23 20:21:15 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 20:21:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:21:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:21:15 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 20:21:15 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 20:21:15 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:21:15 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:21:15 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 20:21:15 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 20:21:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 22 ms on localhost (1/2) -2016-05-23 20:21:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 20 ms on localhost (2/2) -2016-05-23 20:21:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 20:21:15 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:29) finished in 0.024 s -2016-05-23 20:21:15 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:29, took 0.043769 s -2016-05-23 20:21:15 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:21:15 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:21:15 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 20:21:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:21:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:21:15 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:21:15 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354310, maxMem=766075207 -2016-05-23 20:21:15 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 20:21:15 INFO MemoryStore:59 - ensureFreeSpace(2466) called with curMem=358870, maxMem=766075207 -2016-05-23 20:21:15 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:21:15 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52824 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:21:15 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:15 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 20:21:15 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 20:21:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:21:15 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 20:21:15 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:21:15 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:21:15 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 20:21:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 67 ms on localhost (1/1) -2016-05-23 20:21:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 20:21:15 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.069 s -2016-05-23 20:21:15 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.089030 s -2016-05-23 20:21:15 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52824 in memory (size: 2043.0 B, free: 730.6 MB) -2016-05-23 20:21:15 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-23 20:21:15 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52824 in memory (size: 2043.0 B, free: 730.6 MB) -2016-05-23 20:21:15 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-23 20:21:15 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:21:15 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:21:15 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:21:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:21:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:21:15 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:21:15 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=350226, maxMem=766075207 -2016-05-23 20:21:15 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 20:21:15 INFO MemoryStore:59 - ensureFreeSpace(2865) called with curMem=355674, maxMem=766075207 -2016-05-23 20:21:15 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:21:15 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52824 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:21:15 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:21:15 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 20:21:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:21:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:21:15 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 20:21:15 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 20:21:15 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:21:15 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:21:15 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:21:15 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:21:15 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 20:21:15 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 20:21:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 49 ms on localhost (1/2) -2016-05-23 20:21:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 48 ms on localhost (2/2) -2016-05-23 20:21:15 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.051 s -2016-05-23 20:21:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 20:21:15 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 20:21:15 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 20:21:15 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.083870 s -2016-05-23 20:21:15 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:21:15 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:21:15 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:21:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:21:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:21:15 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:21:15 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=358539, maxMem=766075207 -2016-05-23 20:21:16 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 20:21:16 INFO MemoryStore:59 - ensureFreeSpace(2856) called with curMem=364035, maxMem=766075207 -2016-05-23 20:21:16 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:21:16 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52824 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:21:16 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:21:16 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 20:21:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:21:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:21:16 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 20:21:16 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 20:21:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:21:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:21:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:21:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:21:16 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 20:21:16 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 20:21:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 43 ms on localhost (1/2) -2016-05-23 20:21:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 42 ms on localhost (2/2) -2016-05-23 20:21:16 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.045 s -2016-05-23 20:21:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 20:21:16 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.075921 s -2016-05-23 20:21:16 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:21:16 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 20:21:16 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:21:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:21:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:21:16 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:21:16 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=366891, maxMem=766075207 -2016-05-23 20:21:16 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 20:21:16 INFO MemoryStore:59 - ensureFreeSpace(2746) called with curMem=372235, maxMem=766075207 -2016-05-23 20:21:16 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 20:21:16 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52824 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 20:21:16 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:21:16 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 20:21:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:21:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:21:16 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 20:21:16 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 20:21:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:21:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:21:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:21:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:21:16 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 20:21:16 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 20:21:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 53 ms on localhost (1/2) -2016-05-23 20:21:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 56 ms on localhost (2/2) -2016-05-23 20:21:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 20:21:16 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.059 s -2016-05-23 20:21:16 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.123341 s -2016-05-23 20:21:16 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:21:16 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:21:16 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 20:21:16 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:21:16 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 20:21:16 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 20:21:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:21:16 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=374981, maxMem=766075207 -2016-05-23 20:21:16 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 20:21:16 INFO MemoryStore:59 - ensureFreeSpace(3158) called with curMem=380909, maxMem=766075207 -2016-05-23 20:21:16 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 20:21:16 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52824 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:21:16 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:21:16 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 20:21:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:21:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:21:16 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 20:21:16 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 20:21:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:21:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:21:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:21:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:21:16 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 20:21:16 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 20:21:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 72 ms on localhost (1/2) -2016-05-23 20:21:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 74 ms on localhost (2/2) -2016-05-23 20:21:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 20:21:16 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.076 s -2016-05-23 20:21:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:21:16 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:21:16 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 20:21:16 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:21:16 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 20:21:16 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:21:16 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=384067, maxMem=766075207 -2016-05-23 20:21:16 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:21:16 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=386571, maxMem=766075207 -2016-05-23 20:21:16 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 20:21:16 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52824 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:21:16 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:16 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:21:16 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 20:21:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:21:16 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 20:21:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:21:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-23 20:21:17 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 59 ms on localhost (1/1) -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 20:21:17 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.060 s -2016-05-23 20:21:17 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.188319 s -2016-05-23 20:21:17 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:21:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:21:17 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:21:17 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=388116, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=392308, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:21:17 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:52824 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:21:17 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:21:17 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:52824 in memory (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:21:17 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:52824 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:21:17 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 20:21:17 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 20:21:17 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 20:21:17 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 20:21:17 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:52824 in memory (size: 2.7 KB, free: 730.5 MB) -2016-05-23 20:21:17 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:21:17 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:52824 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:21:17 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 20:21:17 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52824 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:21:17 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 20:21:17 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52824 in memory (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:21:17 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-23 20:21:17 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 20:21:17 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 45 ms on localhost (1/2) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 50 ms on localhost (2/2) -2016-05-23 20:21:17 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.051 s -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 20:21:17 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:21:17 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:21:17 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 20:21:17 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:21:17 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=349674, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=354530, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:21:17 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:52824 (size: 2.5 KB, free: 730.6 MB) -2016-05-23 20:21:17 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:21:17 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:21:17 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 24 ms on localhost (1/1) -2016-05-23 20:21:17 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.024 s -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 20:21:17 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.136357 s -2016-05-23 20:21:17 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:21:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:21:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 20:21:17 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:21:17 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=357061, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=362717, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:21:17 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:52824 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:21:17 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:21:17 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 20:21:17 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:21:17 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 23 ms on localhost (1/2) -2016-05-23 20:21:17 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 24 ms on localhost (2/2) -2016-05-23 20:21:17 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.026 s -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 20:21:17 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.042430 s -2016-05-23 20:21:17 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:21:17 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=365625, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=371321, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:21:17 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:52824 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:21:17 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:21:17 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 20:21:17 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:21:17 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 22 ms on localhost (1/2) -2016-05-23 20:21:17 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 31 ms on localhost (2/2) -2016-05-23 20:21:17 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.031 s -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 20:21:17 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.052653 s -2016-05-23 20:21:17 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:43 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:43) with 4 output partitions -2016-05-23 20:21:17 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:43) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:21:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40), which has no missing parents -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=374210, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=375434, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:21:17 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:52824 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:21:17 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40) -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:21:17 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 20:21:17 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 20:21:17 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 20:21:17 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 20:21:17 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 20:21:17 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 9 ms on localhost (1/4) -2016-05-23 20:21:17 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 11 ms on localhost (2/4) -2016-05-23 20:21:17 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 25 ms on localhost (3/4) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 17 ms on localhost (4/4) -2016-05-23 20:21:17 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:43) finished in 0.027 s -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 20:21:17 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:43, took 0.043053 s -2016-05-23 20:21:17 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:45 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:45) with 4 output partitions -2016-05-23 20:21:17 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:45) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:21:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41), which has no missing parents -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=376327, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=377551, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:21:17 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:52824 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:21:17 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41) -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:21:17 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 20:21:17 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 20:21:17 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 20:21:17 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 20:21:17 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 20:21:17 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 20:21:17 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 20:21:17 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 11 ms on localhost (1/4) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 11 ms on localhost (2/4) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 13 ms on localhost (3/4) -2016-05-23 20:21:17 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:45) finished in 0.015 s -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 12 ms on localhost (4/4) -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 20:21:17 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:45, took 0.027263 s -2016-05-23 20:21:17 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:21:17 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:21:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=378444, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=380948, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 20:21:17 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:52824 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 20:21:17 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:21:17 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 20:21:17 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 9 ms on localhost (1/1) -2016-05-23 20:21:17 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.009 s -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 20:21:17 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.022032 s -2016-05-23 20:21:17 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:21:17 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:21:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=382472, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=385872, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 20:21:17 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:52824 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 20:21:17 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:21:17 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 20:21:17 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 20:21:17 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 20:21:17 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 20:21:17 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 20:21:17 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 14 ms on localhost (1/4) -2016-05-23 20:21:17 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 15 ms on localhost (2/4) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 15 ms on localhost (3/4) -2016-05-23 20:21:17 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 16 ms on localhost (4/4) -2016-05-23 20:21:17 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.021 s -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 20:21:17 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.033461 s -2016-05-23 20:21:17 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:21:17 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:21:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=387806, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=391246, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 20:21:17 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:52824 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 20:21:17 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:21:17 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 20:21:17 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 20:21:17 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 20:21:17 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 20:21:17 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 20:21:17 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 20:21:17 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 20:21:17 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 22 ms on localhost (1/4) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 20 ms on localhost (2/4) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 22 ms on localhost (3/4) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 20 ms on localhost (4/4) -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 20:21:17 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.026 s -2016-05-23 20:21:17 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.040130 s -2016-05-23 20:21:17 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:21:17 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:21:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=393164, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(1860) called with curMem=396540, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1860.0 B, free 730.2 MB) -2016-05-23 20:21:17 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:52824 (size: 1860.0 B, free: 730.5 MB) -2016-05-23 20:21:17 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:21:17 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 20:21:17 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 20:21:17 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 20:21:17 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 20:21:17 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 13 ms on localhost (1/4) -2016-05-23 20:21:17 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 20:21:17 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 18 ms on localhost (2/4) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 25 ms on localhost (3/4) -2016-05-23 20:21:17 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 20:21:17 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.029 s -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 28 ms on localhost (4/4) -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 20:21:17 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.046845 s -2016-05-23 20:21:17 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:21:17 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=398400, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(2284) called with curMem=402440, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:21:17 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:52824 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:21:17 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:21:17 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 20:21:17 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 20:21:17 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 20:21:17 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 20:21:17 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 20:21:17 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 65 ms on localhost (1/4) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 66 ms on localhost (2/4) -2016-05-23 20:21:17 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 73 ms on localhost (3/4) -2016-05-23 20:21:17 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 100 ms on localhost (4/4) -2016-05-23 20:21:17 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.100 s -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 20:21:17 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:21:17 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:21:17 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 20:21:17 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:21:17 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=404724, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=407268, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 20:21:17 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:52824 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 20:21:17 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:21:17 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 20:21:17 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 20:21:17 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:21:17 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 20:21:17 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 27 ms on localhost (1/3) -2016-05-23 20:21:17 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 29 ms on localhost (2/3) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 33 ms on localhost (3/3) -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 20:21:17 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.035 s -2016-05-23 20:21:17 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.170004 s -2016-05-23 20:21:17 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:21:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:21:17 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:21:17 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=408836, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:21:17 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=413068, maxMem=766075207 -2016-05-23 20:21:17 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 20:21:17 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:52824 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:21:17 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:21:17 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:21:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:21:17 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 20:21:17 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 20:21:17 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 20:21:17 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:21:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 8 ms -2016-05-23 20:21:17 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 20:21:17 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 20:21:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 84 ms on localhost (1/4) -2016-05-23 20:21:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 84 ms on localhost (2/4) -2016-05-23 20:21:18 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 20:21:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 106 ms on localhost (3/4) -2016-05-23 20:21:18 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 20:21:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 123 ms on localhost (4/4) -2016-05-23 20:21:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 20:21:18 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.125 s -2016-05-23 20:21:18 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:21:18 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:21:18 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 20:21:18 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:21:18 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 20:21:18 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:21:18 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=415374, maxMem=766075207 -2016-05-23 20:21:18 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 20:21:18 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=420270, maxMem=766075207 -2016-05-23 20:21:18 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:21:18 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:52824 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:21:18 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:21:18 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 20:21:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:21:18 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 20:21:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:21:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:21:18 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 20:21:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 22 ms on localhost (1/1) -2016-05-23 20:21:18 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.023 s -2016-05-23 20:21:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 20:21:18 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.187436 s -2016-05-23 20:21:18 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:21:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:21:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 20:21:18 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:21:18 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:21:18 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 20:21:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:21:18 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:21:18 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=422812, maxMem=766075207 -2016-05-23 20:21:18 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:21:18 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=428508, maxMem=766075207 -2016-05-23 20:21:18 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:21:18 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:52824 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:21:18 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:21:18 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 20:21:18 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:52824 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:21:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:21:18 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-23 20:21:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:21:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:21:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:21:18 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 20:21:18 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 20:21:18 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 20:21:18 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 20:21:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:21:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:21:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:21:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:21:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:21:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:21:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:21:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:21:18 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:52824 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:21:18 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 20:21:18 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 20:21:18 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 20:21:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 39 ms on localhost (1/4) -2016-05-23 20:21:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 41 ms on localhost (2/4) -2016-05-23 20:21:18 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-23 20:21:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 39 ms on localhost (3/4) -2016-05-23 20:21:18 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:52824 in memory (size: 1568.0 B, free: 730.5 MB) -2016-05-23 20:21:18 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-23 20:21:18 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:52824 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:21:18 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 20:21:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 51 ms on localhost (4/4) -2016-05-23 20:21:18 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.056 s -2016-05-23 20:21:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 20:21:18 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-23 20:21:18 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.088973 s -2016-05-23 20:21:18 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:52824 in memory (size: 1860.0 B, free: 730.5 MB) -2016-05-23 20:21:18 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-23 20:21:18 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:52824 in memory (size: 1918.0 B, free: 730.5 MB) -2016-05-23 20:21:18 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-23 20:21:18 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:21:18 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:52824 in memory (size: 1934.0 B, free: 730.5 MB) -2016-05-23 20:21:18 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:21:18 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:21:18 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 20:21:18 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-23 20:21:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:21:18 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:21:18 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=391077, maxMem=766075207 -2016-05-23 20:21:18 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:21:18 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=395289, maxMem=766075207 -2016-05-23 20:21:18 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:52824 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 20:21:18 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:21:18 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-23 20:21:18 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:52824 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:21:18 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:21:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:21:18 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 20:21:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:21:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:21:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:21:18 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:52824 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:21:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:21:18 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 20:21:18 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 20:21:18 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 20:21:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:21:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:21:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:21:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:21:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:21:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:21:18 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 20:21:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 25 ms on localhost (1/4) -2016-05-23 20:21:18 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 20:21:18 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 20:21:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 27 ms on localhost (2/4) -2016-05-23 20:21:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 30 ms on localhost (3/4) -2016-05-23 20:21:18 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 20:21:18 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-23 20:21:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:21:18 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:52824 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:21:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:21:18 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-23 20:21:18 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:52824 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:21:18 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 20:21:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 48 ms on localhost (4/4) -2016-05-23 20:21:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 20:21:18 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.050 s -2016-05-23 20:21:18 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.069248 s -2016-05-23 20:21:18 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-23 20:21:18 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:52824 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:21:18 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-23 20:21:18 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:52824 in memory (size: 2.5 KB, free: 730.6 MB) -2016-05-23 20:21:18 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 20:21:18 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:52824 in memory (size: 2.2 KB, free: 730.6 MB) -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 20:21:18 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 20:21:18 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-23 20:21:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 20:21:18 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-23 20:21:18 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 20:21:18 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 20:21:18 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 20:21:19 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 20:21:19 INFO BlockManager:59 - BlockManager stopped -2016-05-23 20:21:19 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 20:21:19 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 20:21:19 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 20:21:19 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 20:21:19 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-6f1e3050-a799-45a4-80f7-ffb3070244b5 -2016-05-23 20:21:19 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 20:21:19 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 20:22:33 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 20:22:34 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 20:22:34 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 20:22:34 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 20:22:34 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 20:22:35 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 20:22:35 INFO Remoting:74 - Starting remoting -2016-05-23 20:22:35 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:52852] -2016-05-23 20:22:35 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52852. -2016-05-23 20:22:35 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 20:22:35 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 20:22:35 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-d6e67c29-a0b8-429d-8fc3-d9826e9ee33c -2016-05-23 20:22:35 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 20:22:35 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-23a3bb8c-4ac6-42d4-a133-e3f1473cf69e\httpd-bcf847dd-cd57-4142-8b3a-27b40afd9439 -2016-05-23 20:22:35 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 20:22:35 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:22:35 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52853 -2016-05-23 20:22:35 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52853. -2016-05-23 20:22:35 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 20:22:35 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:22:35 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 20:22:35 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 20:22:35 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 20:22:36 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 20:22:36 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 20:22:36 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52872. -2016-05-23 20:22:36 INFO NettyBlockTransferService:59 - Server created on 52872 -2016-05-23 20:22:36 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 20:22:36 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52872 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52872) -2016-05-23 20:22:36 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 20:22:37 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 20:22:37 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 20:22:37 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 20:22:37 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 20:22:37 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52872 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:22:37 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:20 -2016-05-23 20:22:37 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 20:22:37 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 20:22:37 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 20:22:37 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 20:22:37 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52872 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:22:37 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 20:22:38 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 20:22:38 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:22:38 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:27 -2016-05-23 20:22:38 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:27) with 2 output partitions -2016-05-23 20:22:38 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:27) -2016-05-23 20:22:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:22:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:22:38 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21), which has no missing parents -2016-05-23 20:22:38 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 20:22:38 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:22:38 INFO MemoryStore:59 - ensureFreeSpace(2043) called with curMem=346712, maxMem=766075207 -2016-05-23 20:22:38 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2043.0 B, free 730.3 MB) -2016-05-23 20:22:38 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52872 (size: 2043.0 B, free: 730.6 MB) -2016-05-23 20:22:38 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21) -2016-05-23 20:22:38 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 20:22:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:22:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:22:38 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 20:22:38 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 20:22:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:22:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:22:38 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 20:22:38 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 20:22:38 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 20:22:38 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 20:22:38 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 20:22:39 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 20:22:39 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 20:22:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 93 ms on localhost (1/2) -2016-05-23 20:22:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 119 ms on localhost (2/2) -2016-05-23 20:22:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 20:22:39 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:27) finished in 0.137 s -2016-05-23 20:22:39 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:27, took 0.277449 s -2016-05-23 20:22:39 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:22:39 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 20:22:39 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 20:22:39 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 20:22:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:22:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:22:39 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24), which has no missing parents -2016-05-23 20:22:39 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348755, maxMem=766075207 -2016-05-23 20:22:39 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:22:39 INFO MemoryStore:59 - ensureFreeSpace(2043) called with curMem=352267, maxMem=766075207 -2016-05-23 20:22:39 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2043.0 B, free 730.2 MB) -2016-05-23 20:22:39 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52872 (size: 2043.0 B, free: 730.6 MB) -2016-05-23 20:22:39 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24) -2016-05-23 20:22:39 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 20:22:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:22:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:22:39 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 20:22:39 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 20:22:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:22:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:22:39 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 20:22:39 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 20:22:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 23 ms on localhost (1/2) -2016-05-23 20:22:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 23 ms on localhost (2/2) -2016-05-23 20:22:39 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:29) finished in 0.025 s -2016-05-23 20:22:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 20:22:39 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:29, took 0.044208 s -2016-05-23 20:22:39 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:22:39 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:22:39 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 20:22:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:22:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:22:39 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:22:39 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354310, maxMem=766075207 -2016-05-23 20:22:39 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 20:22:39 INFO MemoryStore:59 - ensureFreeSpace(2465) called with curMem=358870, maxMem=766075207 -2016-05-23 20:22:39 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:22:39 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52872 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:22:39 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:39 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 20:22:39 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 20:22:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:22:39 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 20:22:39 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52872 in memory (size: 2043.0 B, free: 730.6 MB) -2016-05-23 20:22:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:22:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:22:39 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 20:22:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 50 ms on localhost (1/1) -2016-05-23 20:22:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 20:22:39 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.072 s -2016-05-23 20:22:39 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.130218 s -2016-05-23 20:22:39 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52872 in memory (size: 2043.0 B, free: 730.6 MB) -2016-05-23 20:22:39 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:22:39 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:22:39 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:22:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:22:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:22:39 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:22:39 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=350225, maxMem=766075207 -2016-05-23 20:22:39 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 20:22:39 INFO MemoryStore:59 - ensureFreeSpace(2864) called with curMem=355673, maxMem=766075207 -2016-05-23 20:22:39 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:22:39 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52872 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:22:39 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:22:39 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 20:22:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:22:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:22:39 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 20:22:39 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 20:22:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:22:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:22:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:22:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:22:39 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 20:22:39 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 20:22:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 47 ms on localhost (1/2) -2016-05-23 20:22:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 48 ms on localhost (2/2) -2016-05-23 20:22:39 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.049 s -2016-05-23 20:22:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 20:22:39 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 20:22:39 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 20:22:39 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.080325 s -2016-05-23 20:22:39 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:22:39 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:22:39 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:22:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:22:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:22:39 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:22:39 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=358537, maxMem=766075207 -2016-05-23 20:22:39 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 20:22:39 INFO MemoryStore:59 - ensureFreeSpace(2855) called with curMem=364033, maxMem=766075207 -2016-05-23 20:22:39 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:22:39 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52872 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:22:39 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:22:39 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 20:22:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:22:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:22:39 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 20:22:39 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 20:22:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:22:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:22:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:22:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:22:39 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 20:22:39 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 20:22:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 36 ms on localhost (1/2) -2016-05-23 20:22:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 35 ms on localhost (2/2) -2016-05-23 20:22:39 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.038 s -2016-05-23 20:22:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 20:22:39 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.062176 s -2016-05-23 20:22:40 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:22:40 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 20:22:40 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:22:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:22:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:22:40 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=366888, maxMem=766075207 -2016-05-23 20:22:40 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(2745) called with curMem=372232, maxMem=766075207 -2016-05-23 20:22:40 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 20:22:40 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52872 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 20:22:40 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:22:40 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 20:22:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:22:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:22:40 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 20:22:40 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 20:22:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:22:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:22:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:22:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:22:40 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 20:22:40 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 20:22:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 39 ms on localhost (1/2) -2016-05-23 20:22:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 42 ms on localhost (2/2) -2016-05-23 20:22:40 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.042 s -2016-05-23 20:22:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 20:22:40 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.063189 s -2016-05-23 20:22:40 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:22:40 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:22:40 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 20:22:40 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:22:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 20:22:40 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 20:22:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=374977, maxMem=766075207 -2016-05-23 20:22:40 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(3157) called with curMem=380905, maxMem=766075207 -2016-05-23 20:22:40 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 20:22:40 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52872 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:22:40 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:22:40 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 20:22:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:22:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:22:40 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 20:22:40 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 20:22:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:22:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:22:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:22:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:22:40 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 20:22:40 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 20:22:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 73 ms on localhost (1/2) -2016-05-23 20:22:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 75 ms on localhost (2/2) -2016-05-23 20:22:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 20:22:40 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.076 s -2016-05-23 20:22:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:22:40 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:22:40 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 20:22:40 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:22:40 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 20:22:40 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=384062, maxMem=766075207 -2016-05-23 20:22:40 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=386566, maxMem=766075207 -2016-05-23 20:22:40 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 20:22:40 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52872 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:22:40 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:40 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:22:40 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 20:22:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:22:40 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 20:22:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:22:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-23 20:22:40 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 20:22:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 58 ms on localhost (1/1) -2016-05-23 20:22:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 20:22:40 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.059 s -2016-05-23 20:22:40 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.185552 s -2016-05-23 20:22:40 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:22:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:22:40 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:22:40 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:22:40 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 20:22:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 20:22:40 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 20:22:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:22:40 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:52872 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=384062, maxMem=766075207 -2016-05-23 20:22:40 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:22:40 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=388254, maxMem=766075207 -2016-05-23 20:22:40 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:22:40 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:52872 in memory (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:22:40 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 20:22:40 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:52872 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:22:40 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:22:40 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 20:22:40 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:52872 in memory (size: 2.7 KB, free: 730.5 MB) -2016-05-23 20:22:40 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 20:22:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:22:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:22:40 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 20:22:40 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 20:22:40 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:52872 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:22:40 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 20:22:40 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52872 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:22:40 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 20:22:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:22:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 20:22:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:22:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:22:40 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52872 in memory (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:22:40 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 20:22:40 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 20:22:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 43 ms on localhost (1/2) -2016-05-23 20:22:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 45 ms on localhost (2/2) -2016-05-23 20:22:40 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.046 s -2016-05-23 20:22:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 20:22:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:22:40 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:22:40 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 20:22:40 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:22:40 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 20:22:40 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=349674, maxMem=766075207 -2016-05-23 20:22:40 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=354530, maxMem=766075207 -2016-05-23 20:22:40 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:22:40 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:52872 (size: 2.5 KB, free: 730.6 MB) -2016-05-23 20:22:40 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:40 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:22:40 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 20:22:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:22:40 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 20:22:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:22:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:22:40 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 20:22:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 23 ms on localhost (1/1) -2016-05-23 20:22:40 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.023 s -2016-05-23 20:22:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 20:22:40 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.129453 s -2016-05-23 20:22:40 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:22:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:22:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 20:22:40 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:22:40 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:22:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 20:22:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:22:40 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=357061, maxMem=766075207 -2016-05-23 20:22:40 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=362717, maxMem=766075207 -2016-05-23 20:22:40 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:22:40 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:52872 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:22:40 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:22:40 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 20:22:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:22:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:22:40 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 20:22:40 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 20:22:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:22:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:22:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:22:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:22:40 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 20:22:40 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 20:22:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 22 ms on localhost (1/2) -2016-05-23 20:22:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 20 ms on localhost (2/2) -2016-05-23 20:22:40 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.023 s -2016-05-23 20:22:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 20:22:40 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.040723 s -2016-05-23 20:22:40 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:22:40 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:22:40 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:22:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 20:22:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:22:40 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=365625, maxMem=766075207 -2016-05-23 20:22:40 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=371321, maxMem=766075207 -2016-05-23 20:22:40 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:22:40 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:52872 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:22:40 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:22:40 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 20:22:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:22:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:22:40 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 20:22:40 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 20:22:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:22:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:22:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:22:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:22:40 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 20:22:40 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 20:22:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 21 ms on localhost (1/2) -2016-05-23 20:22:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 21 ms on localhost (2/2) -2016-05-23 20:22:40 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.022 s -2016-05-23 20:22:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 20:22:40 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.039496 s -2016-05-23 20:22:40 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:43 -2016-05-23 20:22:40 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:43) with 4 output partitions -2016-05-23 20:22:40 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:43) -2016-05-23 20:22:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:22:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:22:40 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40), which has no missing parents -2016-05-23 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=374210, maxMem=766075207 -2016-05-23 20:22:40 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=375434, maxMem=766075207 -2016-05-23 20:22:40 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:22:40 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:52872 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:22:40 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:40 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40) -2016-05-23 20:22:40 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 20:22:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:22:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:22:40 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:22:40 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:22:40 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 20:22:40 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 20:22:40 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 20:22:40 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 20:22:40 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 20:22:40 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 20:22:40 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 20:22:40 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 13 ms on localhost (1/4) -2016-05-23 20:22:40 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 12 ms on localhost (2/4) -2016-05-23 20:22:40 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 20:22:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 16 ms on localhost (3/4) -2016-05-23 20:22:40 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:43) finished in 0.026 s -2016-05-23 20:22:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 25 ms on localhost (4/4) -2016-05-23 20:22:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 20:22:40 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:43, took 0.042319 s -2016-05-23 20:22:40 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:45 -2016-05-23 20:22:40 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:45) with 4 output partitions -2016-05-23 20:22:40 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:45) -2016-05-23 20:22:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:22:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:22:40 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41), which has no missing parents -2016-05-23 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=376327, maxMem=766075207 -2016-05-23 20:22:40 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=377551, maxMem=766075207 -2016-05-23 20:22:40 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:22:40 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:52872 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:22:41 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41) -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:22:41 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 20:22:41 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 20:22:41 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 20:22:41 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 20:22:41 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 20:22:41 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 20:22:41 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 20:22:41 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 11 ms on localhost (1/4) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 10 ms on localhost (2/4) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 13 ms on localhost (3/4) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 15 ms on localhost (4/4) -2016-05-23 20:22:41 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:45) finished in 0.015 s -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 20:22:41 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:45, took 0.027115 s -2016-05-23 20:22:41 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:22:41 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:22:41 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 20:22:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:22:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=378444, maxMem=766075207 -2016-05-23 20:22:41 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=380948, maxMem=766075207 -2016-05-23 20:22:41 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 20:22:41 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:52872 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 20:22:41 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:22:41 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 20:22:41 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 6 ms on localhost (1/1) -2016-05-23 20:22:41 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.007 s -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 20:22:41 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.020433 s -2016-05-23 20:22:41 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:22:41 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:22:41 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:22:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:22:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=382472, maxMem=766075207 -2016-05-23 20:22:41 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=385872, maxMem=766075207 -2016-05-23 20:22:41 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 20:22:41 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:52872 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 20:22:41 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:22:41 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 20:22:41 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 20:22:41 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 20:22:41 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 20:22:41 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 20:22:41 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 16 ms on localhost (1/4) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 14 ms on localhost (2/4) -2016-05-23 20:22:41 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 20:22:41 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 20 ms on localhost (3/4) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 19 ms on localhost (4/4) -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 20:22:41 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.023 s -2016-05-23 20:22:41 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.035291 s -2016-05-23 20:22:41 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:22:41 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:22:41 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:22:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:22:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=387806, maxMem=766075207 -2016-05-23 20:22:41 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=391246, maxMem=766075207 -2016-05-23 20:22:41 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 20:22:41 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:52872 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 20:22:41 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:22:41 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 20:22:41 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 20:22:41 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 20:22:41 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 20:22:41 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 20:22:41 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 20:22:41 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 14 ms on localhost (1/4) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 12 ms on localhost (2/4) -2016-05-23 20:22:41 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 14 ms on localhost (3/4) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 17 ms on localhost (4/4) -2016-05-23 20:22:41 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.019 s -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 20:22:41 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.034554 s -2016-05-23 20:22:41 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:22:41 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:22:41 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:22:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:22:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=393164, maxMem=766075207 -2016-05-23 20:22:41 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(1860) called with curMem=396540, maxMem=766075207 -2016-05-23 20:22:41 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1860.0 B, free 730.2 MB) -2016-05-23 20:22:41 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:52872 (size: 1860.0 B, free: 730.5 MB) -2016-05-23 20:22:41 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:22:41 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 20:22:41 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 20:22:41 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 20:22:41 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 20:22:41 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 20:22:41 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 20:22:41 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 17 ms on localhost (1/4) -2016-05-23 20:22:41 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 26 ms on localhost (2/4) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 26 ms on localhost (3/4) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 25 ms on localhost (4/4) -2016-05-23 20:22:41 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.029 s -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 20:22:41 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.048693 s -2016-05-23 20:22:41 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:22:41 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:22:41 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:22:41 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:22:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 20:22:41 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=398400, maxMem=766075207 -2016-05-23 20:22:41 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(2284) called with curMem=402440, maxMem=766075207 -2016-05-23 20:22:41 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:22:41 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:52872 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:22:41 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:22:41 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 20:22:41 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 20:22:41 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 20:22:41 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 20:22:41 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 20:22:41 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 20:22:41 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 20:22:41 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 83 ms on localhost (1/4) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 87 ms on localhost (2/4) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 87 ms on localhost (3/4) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 91 ms on localhost (4/4) -2016-05-23 20:22:41 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.091 s -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 20:22:41 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:22:41 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:22:41 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 20:22:41 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:22:41 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=404724, maxMem=766075207 -2016-05-23 20:22:41 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=407268, maxMem=766075207 -2016-05-23 20:22:41 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 20:22:41 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:52872 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 20:22:41 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:22:41 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 20:22:41 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 20:22:41 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:22:41 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 20:22:41 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 23 ms on localhost (1/3) -2016-05-23 20:22:41 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 27 ms on localhost (2/3) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 25 ms on localhost (3/3) -2016-05-23 20:22:41 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.030 s -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 20:22:41 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.156782 s -2016-05-23 20:22:41 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:22:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:22:41 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:22:41 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:22:41 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 20:22:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 20:22:41 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=408836, maxMem=766075207 -2016-05-23 20:22:41 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=413068, maxMem=766075207 -2016-05-23 20:22:41 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 20:22:41 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:52872 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:22:41 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:22:41 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 20:22:41 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 20:22:41 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 20:22:41 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 10 ms -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 9 ms -2016-05-23 20:22:41 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 116 ms on localhost (1/4) -2016-05-23 20:22:41 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 119 ms on localhost (2/4) -2016-05-23 20:22:41 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 152 ms on localhost (3/4) -2016-05-23 20:22:41 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 157 ms on localhost (4/4) -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 20:22:41 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.160 s -2016-05-23 20:22:41 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:22:41 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:22:41 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 20:22:41 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:22:41 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=415374, maxMem=766075207 -2016-05-23 20:22:41 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=420270, maxMem=766075207 -2016-05-23 20:22:41 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:22:41 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:52872 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:22:41 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:22:41 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:22:41 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 18 ms on localhost (1/1) -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 20:22:41 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.020 s -2016-05-23 20:22:41 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.217544 s -2016-05-23 20:22:41 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:22:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:22:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 20:22:41 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:22:41 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:22:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 20:22:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=422812, maxMem=766075207 -2016-05-23 20:22:41 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=428508, maxMem=766075207 -2016-05-23 20:22:41 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:22:41 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:52872 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:22:41 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:22:41 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 20:22:41 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 20:22:41 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:22:41 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:22:41 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:22:41 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 20:22:41 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 20:22:41 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 50 ms on localhost (1/4) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 51 ms on localhost (2/4) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 56 ms on localhost (3/4) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 62 ms on localhost (4/4) -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 20:22:41 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.063 s -2016-05-23 20:22:41 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.084095 s -2016-05-23 20:22:41 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:22:41 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:22:41 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:22:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 20:22:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=431417, maxMem=766075207 -2016-05-23 20:22:41 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=437153, maxMem=766075207 -2016-05-23 20:22:41 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:22:41 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:52872 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:22:41 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:22:41 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:22:41 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 20:22:41 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 20:22:41 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:22:41 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 20:22:41 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 21 ms on localhost (1/4) -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:22:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:22:41 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 20:22:41 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 20:22:41 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 37 ms on localhost (2/4) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 40 ms on localhost (3/4) -2016-05-23 20:22:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 39 ms on localhost (4/4) -2016-05-23 20:22:41 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.041 s -2016-05-23 20:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 20:22:41 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.061073 s -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 20:22:41 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 20:22:41 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 20:22:41 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 20:22:42 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 20:22:42 INFO BlockManager:59 - BlockManager stopped -2016-05-23 20:22:42 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 20:22:42 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 20:22:42 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 20:22:42 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 20:22:42 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-23a3bb8c-4ac6-42d4-a133-e3f1473cf69e -2016-05-23 20:22:42 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 20:22:42 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 20:26:45 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 20:26:46 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 20:26:46 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 20:26:46 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 20:26:46 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 20:26:47 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 20:26:47 INFO Remoting:74 - Starting remoting -2016-05-23 20:26:47 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:52912] -2016-05-23 20:26:47 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52912. -2016-05-23 20:26:47 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 20:26:47 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 20:26:47 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-f3f5bcde-c4fe-4d7e-b1d8-0ad08b295b8c -2016-05-23 20:26:47 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 20:26:47 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-56ec591d-3ff3-4cd8-97c4-9d08a23c89c4\httpd-abd47ad1-1451-4a97-8b17-aa230cc42b3c -2016-05-23 20:26:47 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 20:26:47 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:26:47 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52913 -2016-05-23 20:26:47 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52913. -2016-05-23 20:26:47 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 20:26:48 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:26:48 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 20:26:48 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 20:26:48 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 20:26:48 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 20:26:48 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 20:26:49 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52933. -2016-05-23 20:26:49 INFO NettyBlockTransferService:59 - Server created on 52933 -2016-05-23 20:26:49 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 20:26:49 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52933 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52933) -2016-05-23 20:26:49 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 20:26:50 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 20:26:50 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 20:26:50 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 20:26:50 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 20:26:50 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52933 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:26:50 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:20 -2016-05-23 20:26:50 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 20:26:50 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 20:26:50 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 20:26:50 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 20:26:50 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52933 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:26:50 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 20:26:51 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 20:26:51 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:26:51 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:27 -2016-05-23 20:26:51 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:27) with 2 output partitions -2016-05-23 20:26:51 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:27) -2016-05-23 20:26:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:26:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:51 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21), which has no missing parents -2016-05-23 20:26:52 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 20:26:52 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:26:52 INFO MemoryStore:59 - ensureFreeSpace(2047) called with curMem=346712, maxMem=766075207 -2016-05-23 20:26:52 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2047.0 B, free 730.3 MB) -2016-05-23 20:26:52 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52933 (size: 2047.0 B, free: 730.6 MB) -2016-05-23 20:26:52 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21) -2016-05-23 20:26:52 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 20:26:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:26:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:26:52 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 20:26:52 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 20:26:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:26:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:26:52 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 20:26:52 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 20:26:52 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 20:26:52 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 20:26:52 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 20:26:52 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 20:26:52 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 20:26:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 140 ms on localhost (1/2) -2016-05-23 20:26:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 112 ms on localhost (2/2) -2016-05-23 20:26:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 20:26:52 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:27) finished in 0.165 s -2016-05-23 20:26:52 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:27, took 0.329024 s -2016-05-23 20:26:52 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:26:52 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 20:26:52 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 20:26:52 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 20:26:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:26:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:52 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24), which has no missing parents -2016-05-23 20:26:52 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348759, maxMem=766075207 -2016-05-23 20:26:52 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:26:52 INFO MemoryStore:59 - ensureFreeSpace(2045) called with curMem=352271, maxMem=766075207 -2016-05-23 20:26:52 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2045.0 B, free 730.2 MB) -2016-05-23 20:26:52 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52933 (size: 2045.0 B, free: 730.6 MB) -2016-05-23 20:26:52 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24) -2016-05-23 20:26:52 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 20:26:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:26:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:26:52 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 20:26:52 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 20:26:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:26:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:26:52 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 20:26:52 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 20:26:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 21 ms on localhost (1/2) -2016-05-23 20:26:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 21 ms on localhost (2/2) -2016-05-23 20:26:52 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:29) finished in 0.024 s -2016-05-23 20:26:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 20:26:52 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:29, took 0.044207 s -2016-05-23 20:26:52 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:26:52 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:26:52 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 20:26:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:26:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:52 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:26:52 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354316, maxMem=766075207 -2016-05-23 20:26:52 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 20:26:52 INFO MemoryStore:59 - ensureFreeSpace(2473) called with curMem=358876, maxMem=766075207 -2016-05-23 20:26:52 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:26:52 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52933 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:26:52 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:52 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 20:26:52 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 20:26:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:26:52 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 20:26:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:26:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:26:52 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 20:26:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 65 ms on localhost (1/1) -2016-05-23 20:26:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 20:26:52 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52933 in memory (size: 2045.0 B, free: 730.6 MB) -2016-05-23 20:26:52 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.186 s -2016-05-23 20:26:52 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.271475 s -2016-05-23 20:26:52 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52933 in memory (size: 2047.0 B, free: 730.6 MB) -2016-05-23 20:26:53 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:26:53 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:26:53 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:26:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:26:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:53 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:26:53 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=350233, maxMem=766075207 -2016-05-23 20:26:53 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 20:26:53 INFO MemoryStore:59 - ensureFreeSpace(2859) called with curMem=355681, maxMem=766075207 -2016-05-23 20:26:53 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:26:53 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52933 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:26:53 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:26:53 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 20:26:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:26:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:26:53 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 20:26:53 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 20:26:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:26:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:26:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:26:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:26:53 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 20:26:53 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 20:26:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 40 ms on localhost (1/2) -2016-05-23 20:26:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 43 ms on localhost (2/2) -2016-05-23 20:26:53 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.043 s -2016-05-23 20:26:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 20:26:53 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 20:26:53 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 20:26:53 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.077916 s -2016-05-23 20:26:53 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:26:53 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:26:53 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:26:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:26:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:53 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:26:53 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=358540, maxMem=766075207 -2016-05-23 20:26:53 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 20:26:53 INFO MemoryStore:59 - ensureFreeSpace(2850) called with curMem=364036, maxMem=766075207 -2016-05-23 20:26:53 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:26:53 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52933 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:26:53 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:26:53 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 20:26:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:26:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:26:53 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 20:26:53 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 20:26:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:26:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:26:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:26:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:26:53 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 20:26:53 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 20:26:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 35 ms on localhost (1/2) -2016-05-23 20:26:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 39 ms on localhost (2/2) -2016-05-23 20:26:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 20:26:53 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.040 s -2016-05-23 20:26:53 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.062922 s -2016-05-23 20:26:54 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:26:54 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 20:26:54 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:26:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:26:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=366886, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(2743) called with curMem=372230, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 20:26:54 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52933 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 20:26:54 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:26:54 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 20:26:54 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 20:26:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:26:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:26:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:26:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:26:54 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 20:26:54 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 37 ms on localhost (1/2) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 40 ms on localhost (2/2) -2016-05-23 20:26:54 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.040 s -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 20:26:54 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.057628 s -2016-05-23 20:26:54 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:26:54 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:26:54 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 20:26:54 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:26:54 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 20:26:54 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=374973, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(3163) called with curMem=380901, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 20:26:54 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52933 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:26:54 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:26:54 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 20:26:54 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 20:26:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:26:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:26:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:26:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:26:54 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 20:26:54 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 83 ms on localhost (1/2) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 80 ms on localhost (2/2) -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 20:26:54 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.084 s -2016-05-23 20:26:54 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:26:54 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:26:54 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 20:26:54 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:26:54 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=384064, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=386568, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 20:26:54 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52933 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:26:54 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:54 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 20:26:54 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:26:54 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-23 20:26:54 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 59 ms on localhost (1/1) -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 20:26:54 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.060 s -2016-05-23 20:26:54 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.192231 s -2016-05-23 20:26:54 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:26:54 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:26:54 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:26:54 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:26:54 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 20:26:54 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 20:26:54 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=388113, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=392305, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:26:54 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:52933 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:26:54 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 20:26:54 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:26:54 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:52933 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:26:54 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:26:54 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 20:26:54 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 20:26:54 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:52933 in memory (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:26:54 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 20:26:54 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:26:54 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:54 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:26:54 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:54 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:52933 in memory (size: 2.7 KB, free: 730.5 MB) -2016-05-23 20:26:54 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 20:26:54 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:52933 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:26:54 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52933 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:26:54 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 20:26:54 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 20:26:54 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 45 ms on localhost (1/2) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 47 ms on localhost (2/2) -2016-05-23 20:26:54 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.049 s -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 20:26:54 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:26:54 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:26:54 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 20:26:54 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:26:54 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=356707, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=361563, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:26:54 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:52933 (size: 2.5 KB, free: 730.6 MB) -2016-05-23 20:26:54 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:54 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 20:26:54 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:26:54 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:54 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 26 ms on localhost (1/1) -2016-05-23 20:26:54 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.026 s -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 20:26:54 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.136040 s -2016-05-23 20:26:54 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:26:54 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:26:54 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 20:26:54 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:26:54 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:26:54 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 20:26:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=364094, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=369750, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:26:54 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:52933 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:26:54 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:54 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 20:26:54 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 20:26:54 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:26:54 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:26:54 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:26:54 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:54 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 20:26:54 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 21 ms on localhost (1/2) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 22 ms on localhost (2/2) -2016-05-23 20:26:54 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.023 s -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 20:26:54 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.038770 s -2016-05-23 20:26:54 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:26:54 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:26:54 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:26:54 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 20:26:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=372658, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=378354, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:26:54 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:52933 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:26:54 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:54 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 20:26:54 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 20:26:54 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:26:54 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:26:54 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:26:54 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:26:54 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 20:26:54 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 22 ms on localhost (1/2) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 27 ms on localhost (2/2) -2016-05-23 20:26:54 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.027 s -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 20:26:54 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.044321 s -2016-05-23 20:26:54 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:43 -2016-05-23 20:26:54 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:43) with 4 output partitions -2016-05-23 20:26:54 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:43) -2016-05-23 20:26:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:26:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40), which has no missing parents -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=381243, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=382467, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:26:54 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:52933 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:26:54 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40) -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:26:54 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 20:26:54 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 20:26:54 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 20:26:54 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 20:26:54 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 20:26:54 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 13 ms on localhost (1/4) -2016-05-23 20:26:54 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 20:26:54 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 15 ms on localhost (2/4) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 26 ms on localhost (3/4) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 16 ms on localhost (4/4) -2016-05-23 20:26:54 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:43) finished in 0.027 s -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 20:26:54 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:43, took 0.040678 s -2016-05-23 20:26:54 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:45 -2016-05-23 20:26:54 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:45) with 4 output partitions -2016-05-23 20:26:54 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:45) -2016-05-23 20:26:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:26:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41), which has no missing parents -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=383360, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=384584, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:26:54 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:52933 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:26:54 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41) -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:26:54 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 20:26:54 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 20:26:54 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 20:26:54 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 20:26:54 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 10 ms on localhost (1/4) -2016-05-23 20:26:54 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 10 ms on localhost (2/4) -2016-05-23 20:26:54 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 20:26:54 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 14 ms on localhost (3/4) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 16 ms on localhost (4/4) -2016-05-23 20:26:54 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:45) finished in 0.018 s -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 20:26:54 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:45, took 0.027978 s -2016-05-23 20:26:54 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:26:54 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:26:54 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 20:26:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:26:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=385477, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=387981, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 20:26:54 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:52933 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 20:26:54 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:26:54 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 20:26:54 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 20:26:54 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.009 s -2016-05-23 20:26:54 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.024284 s -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 9 ms on localhost (1/1) -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 20:26:54 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:26:54 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:26:54 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:26:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:26:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=389505, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=392905, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 20:26:54 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:52933 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 20:26:54 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:26:54 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 20:26:54 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 20:26:54 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 20:26:54 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 20:26:54 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 20:26:54 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 14 ms on localhost (1/4) -2016-05-23 20:26:54 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 16 ms on localhost (2/4) -2016-05-23 20:26:54 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 15 ms on localhost (3/4) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 18 ms on localhost (4/4) -2016-05-23 20:26:54 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.021 s -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 20:26:54 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.035159 s -2016-05-23 20:26:54 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:26:54 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:26:54 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:26:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:26:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=394839, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 20:26:54 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=398279, maxMem=766075207 -2016-05-23 20:26:54 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 20:26:54 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:52933 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 20:26:54 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:54 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:26:54 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:26:54 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 20:26:54 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 20:26:54 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 20:26:54 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 20:26:54 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 20:26:54 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 20:26:54 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 13 ms on localhost (1/4) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 14 ms on localhost (2/4) -2016-05-23 20:26:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 18 ms on localhost (3/4) -2016-05-23 20:26:54 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 20:26:55 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.022 s -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 18 ms on localhost (4/4) -2016-05-23 20:26:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 20:26:55 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.036284 s -2016-05-23 20:26:55 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:26:55 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:26:55 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:26:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:26:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:55 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:26:55 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=400197, maxMem=766075207 -2016-05-23 20:26:55 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:26:55 INFO MemoryStore:59 - ensureFreeSpace(1860) called with curMem=403573, maxMem=766075207 -2016-05-23 20:26:55 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1860.0 B, free 730.2 MB) -2016-05-23 20:26:55 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:52933 (size: 1860.0 B, free: 730.5 MB) -2016-05-23 20:26:55 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:26:55 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:26:55 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 20:26:55 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 20:26:55 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 20:26:55 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 20:26:55 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 20:26:55 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 15 ms on localhost (1/4) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 19 ms on localhost (2/4) -2016-05-23 20:26:55 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 20:26:55 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 26 ms on localhost (3/4) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 24 ms on localhost (4/4) -2016-05-23 20:26:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 20:26:55 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.027 s -2016-05-23 20:26:55 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.046152 s -2016-05-23 20:26:55 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:26:55 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:26:55 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:26:55 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:26:55 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 20:26:55 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 20:26:55 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:26:55 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=405433, maxMem=766075207 -2016-05-23 20:26:55 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 20:26:55 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=409473, maxMem=766075207 -2016-05-23 20:26:55 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:26:55 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:52933 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:26:55 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:26:55 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:26:55 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 20:26:55 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 20:26:55 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 20:26:55 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 20:26:55 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 61 ms on localhost (1/4) -2016-05-23 20:26:55 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 20:26:55 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 79 ms on localhost (2/4) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 75 ms on localhost (3/4) -2016-05-23 20:26:55 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 20:26:55 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.161 s -2016-05-23 20:26:55 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:26:55 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:26:55 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 20:26:55 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:26:55 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 20:26:55 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:26:55 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=411755, maxMem=766075207 -2016-05-23 20:26:55 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:26:55 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=414299, maxMem=766075207 -2016-05-23 20:26:55 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 20:26:55 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:52933 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 20:26:55 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:55 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:26:55 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 159 ms on localhost (4/4) -2016-05-23 20:26:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:55 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:55 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 20:26:55 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:26:55 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 20:26:55 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 24 ms on localhost (1/3) -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 31 ms on localhost (2/3) -2016-05-23 20:26:55 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 47 ms on localhost (3/3) -2016-05-23 20:26:55 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.048 s -2016-05-23 20:26:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 20:26:55 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.244094 s -2016-05-23 20:26:55 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:26:55 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:26:55 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:26:55 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:26:55 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 20:26:55 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 20:26:55 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 20:26:55 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:26:55 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=415867, maxMem=766075207 -2016-05-23 20:26:55 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:26:55 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=420099, maxMem=766075207 -2016-05-23 20:26:55 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 20:26:55 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:52933 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:26:55 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:26:55 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:26:55 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 20:26:55 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 20:26:55 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 20:26:55 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:26:55 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 102 ms on localhost (1/4) -2016-05-23 20:26:55 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 106 ms on localhost (2/4) -2016-05-23 20:26:55 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 166 ms on localhost (3/4) -2016-05-23 20:26:55 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 215 ms on localhost (4/4) -2016-05-23 20:26:55 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.218 s -2016-05-23 20:26:55 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:26:55 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:26:55 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 20:26:55 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:26:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 20:26:55 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 20:26:55 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:26:55 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=422405, maxMem=766075207 -2016-05-23 20:26:55 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 20:26:55 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=427301, maxMem=766075207 -2016-05-23 20:26:55 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:26:55 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:52933 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:26:55 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:55 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:26:55 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:55 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:55 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 16 ms on localhost (1/1) -2016-05-23 20:26:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 20:26:55 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.019 s -2016-05-23 20:26:55 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.341900 s -2016-05-23 20:26:55 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:26:55 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:26:55 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 20:26:55 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:26:55 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:26:55 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 20:26:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:55 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:26:55 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=429843, maxMem=766075207 -2016-05-23 20:26:55 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:26:55 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=435539, maxMem=766075207 -2016-05-23 20:26:55 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:26:55 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:52933 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:26:55 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:26:55 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:55 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 20:26:55 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 20:26:55 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:55 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:26:55 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 29 ms on localhost (1/4) -2016-05-23 20:26:55 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 20:26:55 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 41 ms on localhost (2/4) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 42 ms on localhost (3/4) -2016-05-23 20:26:55 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 48 ms on localhost (4/4) -2016-05-23 20:26:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 20:26:55 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.051 s -2016-05-23 20:26:55 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.069127 s -2016-05-23 20:26:55 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:26:55 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:26:55 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:26:55 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 20:26:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:55 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:26:55 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=438448, maxMem=766075207 -2016-05-23 20:26:55 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:26:55 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=444184, maxMem=766075207 -2016-05-23 20:26:55 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:26:55 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:52933 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:26:55 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:26:55 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:55 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 20:26:55 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 20:26:55 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:26:55 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 20:26:55 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 20:26:55 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 20:26:55 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 40 ms on localhost (1/4) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 41 ms on localhost (2/4) -2016-05-23 20:26:55 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 42 ms on localhost (3/4) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 44 ms on localhost (4/4) -2016-05-23 20:26:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 20:26:55 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.048 s -2016-05-23 20:26:55 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.068676 s -2016-05-23 20:26:55 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:57 -2016-05-23 20:26:55 INFO DAGScheduler:59 - Got job 20 (foreach at StatisticsCorrLearning.scala:57) with 4 output partitions -2016-05-23 20:26:55 INFO DAGScheduler:59 - Final stage: ResultStage 34(foreach at StatisticsCorrLearning.scala:57) -2016-05-23 20:26:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:26:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:55 INFO DAGScheduler:59 - Submitting ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:54), which has no missing parents -2016-05-23 20:26:55 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=447076, maxMem=766075207 -2016-05-23 20:26:55 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:26:55 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=448300, maxMem=766075207 -2016-05-23 20:26:55 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:26:55 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:52933 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:26:55 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:54) -2016-05-23 20:26:55 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 62, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 63, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 64, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:26:55 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 62) -2016-05-23 20:26:55 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 64) -2016-05-23 20:26:55 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 65) -2016-05-23 20:26:55 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 63) -2016-05-23 20:26:55 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 62). 915 bytes result sent to driver -2016-05-23 20:26:55 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 63). 915 bytes result sent to driver -2016-05-23 20:26:55 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 65). 915 bytes result sent to driver -2016-05-23 20:26:55 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 64). 915 bytes result sent to driver -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 62) in 9 ms on localhost (1/4) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 63) in 9 ms on localhost (2/4) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 65) in 8 ms on localhost (3/4) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 64) in 14 ms on localhost (4/4) -2016-05-23 20:26:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-23 20:26:55 INFO DAGScheduler:59 - ResultStage 34 (foreach at StatisticsCorrLearning.scala:57) finished in 0.016 s -2016-05-23 20:26:55 INFO DAGScheduler:59 - Job 20 finished: foreach at StatisticsCorrLearning.scala:57, took 0.027868 s -2016-05-23 20:26:55 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:59 -2016-05-23 20:26:55 INFO DAGScheduler:59 - Got job 21 (foreach at StatisticsCorrLearning.scala:59) with 4 output partitions -2016-05-23 20:26:55 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at StatisticsCorrLearning.scala:59) -2016-05-23 20:26:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:26:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:55 INFO DAGScheduler:59 - Submitting ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:55), which has no missing parents -2016-05-23 20:26:55 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=449193, maxMem=766075207 -2016-05-23 20:26:55 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:26:55 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=450417, maxMem=766075207 -2016-05-23 20:26:55 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:26:55 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:52933 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:26:55 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:55) -2016-05-23 20:26:55 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 4 tasks -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 66, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:26:55 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 68) -2016-05-23 20:26:55 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 66) -2016-05-23 20:26:55 INFO Executor:59 - Running task 3.0 in stage 35.0 (TID 69) -2016-05-23 20:26:55 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 68). 915 bytes result sent to driver -2016-05-23 20:26:55 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 66). 915 bytes result sent to driver -2016-05-23 20:26:55 INFO Executor:59 - Finished task 3.0 in stage 35.0 (TID 69). 915 bytes result sent to driver -2016-05-23 20:26:55 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 67) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 68) in 8 ms on localhost (1/4) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 66) in 11 ms on localhost (2/4) -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 35.0 (TID 69) in 9 ms on localhost (3/4) -2016-05-23 20:26:55 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 67). 915 bytes result sent to driver -2016-05-23 20:26:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 67) in 13 ms on localhost (4/4) -2016-05-23 20:26:55 INFO DAGScheduler:59 - ResultStage 35 (foreach at StatisticsCorrLearning.scala:59) finished in 0.014 s -2016-05-23 20:26:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-23 20:26:55 INFO DAGScheduler:59 - Job 21 finished: foreach at StatisticsCorrLearning.scala:59, took 0.026536 s -2016-05-23 20:26:55 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:26:55 INFO DAGScheduler:59 - Got job 22 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:26:55 INFO DAGScheduler:59 - Final stage: ResultStage 36(first at RowMatrix.scala:65) -2016-05-23 20:26:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:26:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:55 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:26:55 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=451310, maxMem=766075207 -2016-05-23 20:26:55 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:26:55 INFO MemoryStore:59 - ensureFreeSpace(1522) called with curMem=453814, maxMem=766075207 -2016-05-23 20:26:56 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1522.0 B, free 730.2 MB) -2016-05-23 20:26:56 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:52933 (size: 1522.0 B, free: 730.5 MB) -2016-05-23 20:26:56 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:56 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 20:26:56 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-23 20:26:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:26:56 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 70) -2016-05-23 20:26:56 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 70). 1046 bytes result sent to driver -2016-05-23 20:26:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 70) in 4 ms on localhost (1/1) -2016-05-23 20:26:56 INFO DAGScheduler:59 - ResultStage 36 (first at RowMatrix.scala:65) finished in 0.004 s -2016-05-23 20:26:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-23 20:26:56 INFO DAGScheduler:59 - Job 22 finished: first at RowMatrix.scala:65, took 0.015531 s -2016-05-23 20:26:56 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:26:56 INFO DAGScheduler:59 - Got job 23 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:26:56 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:26:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:26:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:56 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:26:56 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=455336, maxMem=766075207 -2016-05-23 20:26:56 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 20:26:56 INFO MemoryStore:59 - ensureFreeSpace(1932) called with curMem=458736, maxMem=766075207 -2016-05-23 20:26:56 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1932.0 B, free 730.1 MB) -2016-05-23 20:26:56 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:52933 (size: 1932.0 B, free: 730.5 MB) -2016-05-23 20:26:56 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:56 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:26:56 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 4 tasks -2016-05-23 20:26:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Starting task 3.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:26:56 INFO Executor:59 - Running task 3.0 in stage 37.0 (TID 74) -2016-05-23 20:26:56 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 73) -2016-05-23 20:26:56 INFO Executor:59 - Finished task 3.0 in stage 37.0 (TID 74). 1231 bytes result sent to driver -2016-05-23 20:26:56 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 73). 1231 bytes result sent to driver -2016-05-23 20:26:56 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 72) -2016-05-23 20:26:56 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-23 20:26:56 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 72). 1231 bytes result sent to driver -2016-05-23 20:26:56 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 1231 bytes result sent to driver -2016-05-23 20:26:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 37.0 (TID 74) in 16 ms on localhost (1/4) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 73) in 18 ms on localhost (2/4) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 20 ms on localhost (3/4) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 72) in 24 ms on localhost (4/4) -2016-05-23 20:26:56 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at RowMatrix.scala:332) finished in 0.025 s -2016-05-23 20:26:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-23 20:26:56 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at RowMatrix.scala:332, took 0.035279 s -2016-05-23 20:26:56 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:26:56 INFO DAGScheduler:59 - Got job 24 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:26:56 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:26:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:26:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:56 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:26:56 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=460668, maxMem=766075207 -2016-05-23 20:26:56 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-23 20:26:56 INFO MemoryStore:59 - ensureFreeSpace(1916) called with curMem=464108, maxMem=766075207 -2016-05-23 20:26:56 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1916.0 B, free 730.1 MB) -2016-05-23 20:26:56 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:52933 (size: 1916.0 B, free: 730.5 MB) -2016-05-23 20:26:56 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:56 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:26:56 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-23 20:26:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 77, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 78, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:26:56 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 78) -2016-05-23 20:26:56 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 78). 1109 bytes result sent to driver -2016-05-23 20:26:56 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 77) -2016-05-23 20:26:56 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 77). 1109 bytes result sent to driver -2016-05-23 20:26:56 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-23 20:26:56 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-23 20:26:56 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 1109 bytes result sent to driver -2016-05-23 20:26:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 78) in 11 ms on localhost (1/4) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 77) in 14 ms on localhost (2/4) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 15 ms on localhost (3/4) -2016-05-23 20:26:56 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 1109 bytes result sent to driver -2016-05-23 20:26:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 27 ms on localhost (4/4) -2016-05-23 20:26:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-23 20:26:56 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at RowMatrix.scala:124) finished in 0.028 s -2016-05-23 20:26:56 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at RowMatrix.scala:124, took 0.042182 s -2016-05-23 20:26:56 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:26:56 INFO DAGScheduler:59 - Got job 25 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:26:56 INFO DAGScheduler:59 - Final stage: ResultStage 39(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:26:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:26:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:56 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:26:56 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=466024, maxMem=766075207 -2016-05-23 20:26:56 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 20:26:56 INFO MemoryStore:59 - ensureFreeSpace(1851) called with curMem=469400, maxMem=766075207 -2016-05-23 20:26:56 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1851.0 B, free 730.1 MB) -2016-05-23 20:26:56 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:52933 (size: 1851.0 B, free: 730.5 MB) -2016-05-23 20:26:56 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:56 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:26:56 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 4 tasks -2016-05-23 20:26:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 79, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 80, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 81, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Starting task 3.0 in stage 39.0 (TID 82, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:26:56 INFO Executor:59 - Running task 3.0 in stage 39.0 (TID 82) -2016-05-23 20:26:56 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 81) -2016-05-23 20:26:56 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 81). 1294 bytes result sent to driver -2016-05-23 20:26:56 INFO Executor:59 - Finished task 3.0 in stage 39.0 (TID 82). 1334 bytes result sent to driver -2016-05-23 20:26:56 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 79) -2016-05-23 20:26:56 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 80) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 81) in 16 ms on localhost (1/4) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 39.0 (TID 82) in 16 ms on localhost (2/4) -2016-05-23 20:26:56 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 79). 1294 bytes result sent to driver -2016-05-23 20:26:56 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 80). 1334 bytes result sent to driver -2016-05-23 20:26:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 79) in 21 ms on localhost (3/4) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 80) in 21 ms on localhost (4/4) -2016-05-23 20:26:56 INFO DAGScheduler:59 - ResultStage 39 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.022 s -2016-05-23 20:26:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-23 20:26:56 INFO DAGScheduler:59 - Job 25 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.036567 s -2016-05-23 20:26:56 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:26:56 INFO DAGScheduler:59 - Registering RDD 51 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:26:56 INFO DAGScheduler:59 - Got job 26 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:26:56 INFO DAGScheduler:59 - Final stage: ResultStage 41(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:26:56 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 40) -2016-05-23 20:26:56 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 40) -2016-05-23 20:26:56 INFO DAGScheduler:59 - Submitting ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:26:56 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=471251, maxMem=766075207 -2016-05-23 20:26:56 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-05-23 20:26:56 INFO MemoryStore:59 - ensureFreeSpace(2281) called with curMem=475291, maxMem=766075207 -2016-05-23 20:26:56 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 20:26:56 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:52933 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:26:56 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:56 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:26:56 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 4 tasks -2016-05-23 20:26:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 83, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 84, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 85, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Starting task 3.0 in stage 40.0 (TID 86, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:26:56 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 83) -2016-05-23 20:26:56 INFO Executor:59 - Running task 3.0 in stage 40.0 (TID 86) -2016-05-23 20:26:56 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 85) -2016-05-23 20:26:56 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 84) -2016-05-23 20:26:56 INFO Executor:59 - Finished task 3.0 in stage 40.0 (TID 86). 1161 bytes result sent to driver -2016-05-23 20:26:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 40.0 (TID 86) in 656 ms on localhost (1/4) -2016-05-23 20:26:56 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 83). 1161 bytes result sent to driver -2016-05-23 20:26:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 83) in 747 ms on localhost (2/4) -2016-05-23 20:26:56 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 85). 1161 bytes result sent to driver -2016-05-23 20:26:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 85) in 749 ms on localhost (3/4) -2016-05-23 20:26:56 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 84). 1161 bytes result sent to driver -2016-05-23 20:26:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 84) in 753 ms on localhost (4/4) -2016-05-23 20:26:56 INFO DAGScheduler:59 - ShuffleMapStage 40 (flatMap at SpearmanCorrelation.scala:49) finished in 0.754 s -2016-05-23 20:26:56 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:26:56 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:26:56 INFO DAGScheduler:59 - waiting: Set(ResultStage 41) -2016-05-23 20:26:56 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:26:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-23 20:26:56 INFO DAGScheduler:59 - Missing parents for ResultStage 41: List() -2016-05-23 20:26:56 INFO DAGScheduler:59 - Submitting ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:26:56 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=477572, maxMem=766075207 -2016-05-23 20:26:56 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 20:26:56 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=480116, maxMem=766075207 -2016-05-23 20:26:56 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.1 MB) -2016-05-23 20:26:56 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:52933 (size: 1567.0 B, free: 730.5 MB) -2016-05-23 20:26:56 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:56 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:26:56 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 3 tasks -2016-05-23 20:26:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 87, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:56 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 88) -2016-05-23 20:26:56 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 87) -2016-05-23 20:26:56 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:56 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:56 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 87). 1203 bytes result sent to driver -2016-05-23 20:26:56 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:56 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:56 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 89) -2016-05-23 20:26:56 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:56 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:26:56 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 88). 1203 bytes result sent to driver -2016-05-23 20:26:57 INFO Executor:59 - Finished task 2.0 in stage 41.0 (TID 89). 1203 bytes result sent to driver -2016-05-23 20:26:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 87) in 34 ms on localhost (1/3) -2016-05-23 20:26:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 88) in 35 ms on localhost (2/3) -2016-05-23 20:26:57 INFO TaskSetManager:59 - Finished task 2.0 in stage 41.0 (TID 89) in 36 ms on localhost (3/3) -2016-05-23 20:26:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-23 20:26:57 INFO DAGScheduler:59 - ResultStage 41 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.037 s -2016-05-23 20:26:57 INFO DAGScheduler:59 - Job 26 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.816717 s -2016-05-23 20:26:57 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:26:57 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 168 bytes -2016-05-23 20:26:57 INFO DAGScheduler:59 - Registering RDD 56 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:26:57 INFO DAGScheduler:59 - Got job 27 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:26:57 INFO DAGScheduler:59 - Final stage: ResultStage 44(first at RowMatrix.scala:65) -2016-05-23 20:26:57 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 43) -2016-05-23 20:26:57 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 43) -2016-05-23 20:26:57 INFO DAGScheduler:59 - Submitting ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:26:57 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=481683, maxMem=766075207 -2016-05-23 20:26:57 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 20:26:57 INFO MemoryStore:59 - ensureFreeSpace(2305) called with curMem=485915, maxMem=766075207 -2016-05-23 20:26:57 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-23 20:26:57 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:52933 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:26:57 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:57 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:26:57 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 4 tasks -2016-05-23 20:26:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 90, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:26:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 91, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:26:57 INFO TaskSetManager:59 - Starting task 2.0 in stage 43.0 (TID 92, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:26:57 INFO TaskSetManager:59 - Starting task 3.0 in stage 43.0 (TID 93, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:26:57 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 90) -2016-05-23 20:26:57 INFO Executor:59 - Running task 2.0 in stage 43.0 (TID 92) -2016-05-23 20:26:57 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 91) -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 10 ms -2016-05-23 20:26:57 INFO Executor:59 - Running task 3.0 in stage 43.0 (TID 93) -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:57 INFO Executor:59 - Finished task 3.0 in stage 43.0 (TID 93). 1377 bytes result sent to driver -2016-05-23 20:26:57 INFO TaskSetManager:59 - Finished task 3.0 in stage 43.0 (TID 93) in 251 ms on localhost (1/4) -2016-05-23 20:26:57 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 90). 1377 bytes result sent to driver -2016-05-23 20:26:57 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 91). 1377 bytes result sent to driver -2016-05-23 20:26:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 90) in 282 ms on localhost (2/4) -2016-05-23 20:26:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 91) in 282 ms on localhost (3/4) -2016-05-23 20:26:57 INFO Executor:59 - Finished task 2.0 in stage 43.0 (TID 92). 1377 bytes result sent to driver -2016-05-23 20:26:57 INFO TaskSetManager:59 - Finished task 2.0 in stage 43.0 (TID 92) in 285 ms on localhost (4/4) -2016-05-23 20:26:57 INFO DAGScheduler:59 - ShuffleMapStage 43 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.287 s -2016-05-23 20:26:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-23 20:26:57 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:26:57 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:26:57 INFO DAGScheduler:59 - waiting: Set(ResultStage 44) -2016-05-23 20:26:57 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:26:57 INFO DAGScheduler:59 - Missing parents for ResultStage 44: List() -2016-05-23 20:26:57 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:26:57 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=488220, maxMem=766075207 -2016-05-23 20:26:57 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 20:26:57 INFO MemoryStore:59 - ensureFreeSpace(2545) called with curMem=493116, maxMem=766075207 -2016-05-23 20:26:57 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 20:26:57 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:52933 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:26:57 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:57 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:26:57 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 1 tasks -2016-05-23 20:26:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 94, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:57 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 94) -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:57 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 94). 1296 bytes result sent to driver -2016-05-23 20:26:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 94) in 229 ms on localhost (1/1) -2016-05-23 20:26:57 INFO DAGScheduler:59 - ResultStage 44 (first at RowMatrix.scala:65) finished in 0.231 s -2016-05-23 20:26:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-23 20:26:57 INFO DAGScheduler:59 - Job 27 finished: first at RowMatrix.scala:65, took 0.607289 s -2016-05-23 20:26:57 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:26:57 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 168 bytes -2016-05-23 20:26:57 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 166 bytes -2016-05-23 20:26:57 INFO DAGScheduler:59 - Got job 28 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:26:57 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:26:57 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 46) -2016-05-23 20:26:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:57 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:26:57 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=495661, maxMem=766075207 -2016-05-23 20:26:57 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 20:26:57 INFO MemoryStore:59 - ensureFreeSpace(2914) called with curMem=501357, maxMem=766075207 -2016-05-23 20:26:57 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:26:57 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:52933 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:26:57 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:57 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:26:57 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 4 tasks -2016-05-23 20:26:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 95, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 96, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:57 INFO TaskSetManager:59 - Starting task 2.0 in stage 47.0 (TID 97, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:57 INFO TaskSetManager:59 - Starting task 3.0 in stage 47.0 (TID 98, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:57 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 95) -2016-05-23 20:26:57 INFO Executor:59 - Running task 2.0 in stage 47.0 (TID 97) -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:26:57 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 96) -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:57 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 95). 1481 bytes result sent to driver -2016-05-23 20:26:57 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 96). 1481 bytes result sent to driver -2016-05-23 20:26:57 INFO Executor:59 - Running task 3.0 in stage 47.0 (TID 98) -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:57 INFO Executor:59 - Finished task 2.0 in stage 47.0 (TID 97). 1481 bytes result sent to driver -2016-05-23 20:26:57 INFO Executor:59 - Finished task 3.0 in stage 47.0 (TID 98). 1481 bytes result sent to driver -2016-05-23 20:26:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 95) in 27 ms on localhost (1/4) -2016-05-23 20:26:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 96) in 26 ms on localhost (2/4) -2016-05-23 20:26:57 INFO TaskSetManager:59 - Finished task 3.0 in stage 47.0 (TID 98) in 27 ms on localhost (3/4) -2016-05-23 20:26:57 INFO TaskSetManager:59 - Finished task 2.0 in stage 47.0 (TID 97) in 28 ms on localhost (4/4) -2016-05-23 20:26:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-23 20:26:57 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at RowMatrix.scala:332) finished in 0.030 s -2016-05-23 20:26:57 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at RowMatrix.scala:332, took 0.046362 s -2016-05-23 20:26:57 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:26:57 INFO DAGScheduler:59 - Got job 29 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:26:57 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:26:57 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 49) -2016-05-23 20:26:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:26:57 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:26:57 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=504271, maxMem=766075207 -2016-05-23 20:26:57 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 20:26:57 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=510007, maxMem=766075207 -2016-05-23 20:26:57 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:26:57 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:52933 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:26:57 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:26:57 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:26:57 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 4 tasks -2016-05-23 20:26:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:57 INFO TaskSetManager:59 - Starting task 2.0 in stage 50.0 (TID 101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:57 INFO TaskSetManager:59 - Starting task 3.0 in stage 50.0 (TID 102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:26:57 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-23 20:26:57 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-23 20:26:57 INFO Executor:59 - Running task 2.0 in stage 50.0 (TID 101) -2016-05-23 20:26:57 INFO Executor:59 - Running task 3.0 in stage 50.0 (TID 102) -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:26:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:26:57 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 1359 bytes result sent to driver -2016-05-23 20:26:57 INFO Executor:59 - Finished task 2.0 in stage 50.0 (TID 101). 1359 bytes result sent to driver -2016-05-23 20:26:57 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 1359 bytes result sent to driver -2016-05-23 20:26:57 INFO Executor:59 - Finished task 3.0 in stage 50.0 (TID 102). 1359 bytes result sent to driver -2016-05-23 20:26:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 17 ms on localhost (1/4) -2016-05-23 20:26:57 INFO TaskSetManager:59 - Finished task 2.0 in stage 50.0 (TID 101) in 17 ms on localhost (2/4) -2016-05-23 20:26:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 18 ms on localhost (3/4) -2016-05-23 20:26:57 INFO TaskSetManager:59 - Finished task 3.0 in stage 50.0 (TID 102) in 19 ms on localhost (4/4) -2016-05-23 20:26:57 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at RowMatrix.scala:124) finished in 0.020 s -2016-05-23 20:26:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-23 20:26:57 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at RowMatrix.scala:124, took 0.031777 s -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 20:26:57 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 20:26:57 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:52933 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 20:26:57 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 20:26:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 20:26:57 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:52933 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:26:57 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-23 20:26:57 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:52933 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:26:57 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-23 20:26:57 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:52933 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:26:57 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-23 20:26:57 INFO ContextCleaner:59 - Cleaned shuffle 5 -2016-05-23 20:26:57 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:52933 in memory (size: 1567.0 B, free: 730.5 MB) -2016-05-23 20:26:57 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-23 20:26:57 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:52933 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:26:57 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-23 20:26:57 INFO ContextCleaner:59 - Cleaned shuffle 4 -2016-05-23 20:26:57 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:52933 in memory (size: 1851.0 B, free: 730.5 MB) -2016-05-23 20:26:57 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-23 20:26:57 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:52933 in memory (size: 1916.0 B, free: 730.5 MB) -2016-05-23 20:26:57 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-23 20:26:57 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:52933 in memory (size: 1932.0 B, free: 730.5 MB) -2016-05-23 20:26:57 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-23 20:26:57 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 20:26:57 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:52933 in memory (size: 1522.0 B, free: 730.5 MB) -2016-05-23 20:26:57 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 20:26:57 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 20:27:00 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 20:27:00 INFO BlockManager:59 - BlockManager stopped -2016-05-23 20:27:00 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 20:27:00 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 20:27:00 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 20:27:00 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 20:27:00 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-56ec591d-3ff3-4cd8-97c4-9d08a23c89c4 -2016-05-23 20:27:00 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 20:27:00 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 20:27:29 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 20:27:30 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 20:27:30 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 20:27:30 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 20:27:30 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 20:27:31 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 20:27:31 INFO Remoting:74 - Starting remoting -2016-05-23 20:27:31 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:52960] -2016-05-23 20:27:31 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52960. -2016-05-23 20:27:31 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 20:27:31 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 20:27:31 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-f6c13e32-cbc5-4645-95b4-61254c1e66ec -2016-05-23 20:27:31 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 20:27:31 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-eed62070-50dd-4b49-941e-35ecd8b951d4\httpd-fff66501-c112-4ce9-89fd-eb871cfc8b27 -2016-05-23 20:27:31 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 20:27:31 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:27:32 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52961 -2016-05-23 20:27:32 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52961. -2016-05-23 20:27:32 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 20:27:32 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:27:32 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 20:27:32 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 20:27:32 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 20:27:32 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 20:27:32 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 20:27:32 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52980. -2016-05-23 20:27:32 INFO NettyBlockTransferService:59 - Server created on 52980 -2016-05-23 20:27:32 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 20:27:32 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52980 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52980) -2016-05-23 20:27:32 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 20:27:33 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 20:27:33 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 20:27:33 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 20:27:33 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 20:27:33 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52980 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:27:33 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:20 -2016-05-23 20:27:34 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 20:27:34 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 20:27:34 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 20:27:34 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 20:27:34 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52980 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:27:34 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 20:27:35 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 20:27:35 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:27:35 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:27 -2016-05-23 20:27:35 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:27) with 2 output partitions -2016-05-23 20:27:35 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:27) -2016-05-23 20:27:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:27:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:35 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21), which has no missing parents -2016-05-23 20:27:35 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 20:27:35 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:27:35 INFO MemoryStore:59 - ensureFreeSpace(2047) called with curMem=346712, maxMem=766075207 -2016-05-23 20:27:35 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2047.0 B, free 730.3 MB) -2016-05-23 20:27:35 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52980 (size: 2047.0 B, free: 730.6 MB) -2016-05-23 20:27:35 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21) -2016-05-23 20:27:35 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 20:27:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:27:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:27:35 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 20:27:35 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 20:27:35 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:27:35 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:27:35 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 20:27:35 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 20:27:35 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 20:27:35 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 20:27:35 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 20:27:35 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 20:27:35 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 20:27:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 174 ms on localhost (1/2) -2016-05-23 20:27:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 147 ms on localhost (2/2) -2016-05-23 20:27:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 20:27:35 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:27) finished in 0.203 s -2016-05-23 20:27:35 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:27, took 0.399244 s -2016-05-23 20:27:36 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:27:36 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 20:27:36 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 20:27:36 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 20:27:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:27:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:36 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24), which has no missing parents -2016-05-23 20:27:36 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348759, maxMem=766075207 -2016-05-23 20:27:36 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:27:36 INFO MemoryStore:59 - ensureFreeSpace(2045) called with curMem=352271, maxMem=766075207 -2016-05-23 20:27:36 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2045.0 B, free 730.2 MB) -2016-05-23 20:27:36 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52980 (size: 2045.0 B, free: 730.6 MB) -2016-05-23 20:27:36 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24) -2016-05-23 20:27:36 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 20:27:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:27:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:27:36 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 20:27:36 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 20:27:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:27:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:27:36 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 20:27:36 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 20:27:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 31 ms on localhost (1/2) -2016-05-23 20:27:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 34 ms on localhost (2/2) -2016-05-23 20:27:36 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:29) finished in 0.034 s -2016-05-23 20:27:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 20:27:36 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:29, took 0.057010 s -2016-05-23 20:27:36 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:27:36 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:27:36 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 20:27:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:27:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:36 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:27:36 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354316, maxMem=766075207 -2016-05-23 20:27:36 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 20:27:36 INFO MemoryStore:59 - ensureFreeSpace(2465) called with curMem=358876, maxMem=766075207 -2016-05-23 20:27:36 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:27:36 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52980 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:27:36 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:36 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 20:27:36 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 20:27:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:27:36 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 20:27:36 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52980 in memory (size: 2045.0 B, free: 730.6 MB) -2016-05-23 20:27:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:27:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:27:36 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52980 in memory (size: 2047.0 B, free: 730.6 MB) -2016-05-23 20:27:36 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 20:27:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 54 ms on localhost (1/1) -2016-05-23 20:27:36 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.056 s -2016-05-23 20:27:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 20:27:36 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.130789 s -2016-05-23 20:27:36 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:27:36 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:27:36 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:27:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:27:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:36 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:27:36 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=350225, maxMem=766075207 -2016-05-23 20:27:36 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 20:27:36 INFO MemoryStore:59 - ensureFreeSpace(2862) called with curMem=355673, maxMem=766075207 -2016-05-23 20:27:36 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:27:36 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52980 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:27:36 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:27:36 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 20:27:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:27:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:27:36 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 20:27:36 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 20:27:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:27:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:27:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:27:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:27:37 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 20:27:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 44 ms on localhost (1/2) -2016-05-23 20:27:37 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 20:27:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 50 ms on localhost (2/2) -2016-05-23 20:27:37 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.050 s -2016-05-23 20:27:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 20:27:37 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 20:27:37 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 20:27:37 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.078604 s -2016-05-23 20:27:37 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:27:37 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:27:37 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:27:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:27:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:37 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:27:37 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=358535, maxMem=766075207 -2016-05-23 20:27:37 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 20:27:37 INFO MemoryStore:59 - ensureFreeSpace(2851) called with curMem=364031, maxMem=766075207 -2016-05-23 20:27:37 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:27:37 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52980 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:27:37 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:27:37 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 20:27:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:27:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:27:37 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 20:27:37 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 20:27:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:27:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:27:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:27:37 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 20:27:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:27:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 37 ms on localhost (1/2) -2016-05-23 20:27:37 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 20:27:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 42 ms on localhost (2/2) -2016-05-23 20:27:37 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.045 s -2016-05-23 20:27:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 20:27:37 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.068994 s -2016-05-23 20:27:37 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:27:37 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 20:27:37 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:27:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:27:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:37 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:27:37 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=366882, maxMem=766075207 -2016-05-23 20:27:37 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 20:27:37 INFO MemoryStore:59 - ensureFreeSpace(2744) called with curMem=372226, maxMem=766075207 -2016-05-23 20:27:37 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 20:27:37 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52980 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 20:27:37 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:27:37 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 20:27:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:27:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:27:37 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 20:27:37 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 20:27:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:27:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:27:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:27:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:27:37 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 20:27:37 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 20:27:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 40 ms on localhost (1/2) -2016-05-23 20:27:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 39 ms on localhost (2/2) -2016-05-23 20:27:37 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.041 s -2016-05-23 20:27:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 20:27:37 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.058092 s -2016-05-23 20:27:37 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:27:37 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:27:37 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 20:27:37 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:27:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 20:27:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 20:27:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:27:37 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=374970, maxMem=766075207 -2016-05-23 20:27:37 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 20:27:37 INFO MemoryStore:59 - ensureFreeSpace(3156) called with curMem=380898, maxMem=766075207 -2016-05-23 20:27:37 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 20:27:37 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52980 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:27:37 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:27:37 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 20:27:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:27:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:27:37 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 20:27:37 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 20:27:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:27:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:27:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:27:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:27:37 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 20:27:37 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 20:27:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 72 ms on localhost (1/2) -2016-05-23 20:27:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 75 ms on localhost (2/2) -2016-05-23 20:27:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 20:27:37 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.077 s -2016-05-23 20:27:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:27:37 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:27:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 20:27:37 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:27:37 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 20:27:37 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:27:37 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=384054, maxMem=766075207 -2016-05-23 20:27:37 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:27:37 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=386558, maxMem=766075207 -2016-05-23 20:27:37 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 20:27:37 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52980 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:27:37 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:37 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:27:37 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 20:27:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:37 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 20:27:37 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:27:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-23 20:27:37 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 20:27:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 55 ms on localhost (1/1) -2016-05-23 20:27:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 20:27:37 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.056 s -2016-05-23 20:27:37 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.183947 s -2016-05-23 20:27:37 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:27:37 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:27:37 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:27:37 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:27:37 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 20:27:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 20:27:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 20:27:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:27:37 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=388103, maxMem=766075207 -2016-05-23 20:27:37 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:27:37 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=392295, maxMem=766075207 -2016-05-23 20:27:37 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:27:37 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:52980 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:27:37 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:52980 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:27:37 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:27:37 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 20:27:37 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 20:27:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:27:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:27:37 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 20:27:37 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:52980 in memory (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:27:37 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 20:27:37 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:27:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:27:37 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:27:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:37 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 20:27:37 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:52980 in memory (size: 2.7 KB, free: 730.5 MB) -2016-05-23 20:27:37 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 20:27:37 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:52980 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:27:37 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 20:27:37 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 20:27:37 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52980 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:27:37 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 20:27:37 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 20:27:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 64 ms on localhost (1/2) -2016-05-23 20:27:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 70 ms on localhost (2/2) -2016-05-23 20:27:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 20:27:37 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.071 s -2016-05-23 20:27:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:27:37 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:27:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 20:27:37 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:27:37 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 20:27:37 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:27:37 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=356699, maxMem=766075207 -2016-05-23 20:27:37 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 20:27:37 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=361555, maxMem=766075207 -2016-05-23 20:27:37 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:27:37 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:52980 (size: 2.5 KB, free: 730.6 MB) -2016-05-23 20:27:37 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:37 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:27:37 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 20:27:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:37 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 20:27:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:27:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:38 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 30 ms on localhost (1/1) -2016-05-23 20:27:38 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.031 s -2016-05-23 20:27:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 20:27:38 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.181367 s -2016-05-23 20:27:38 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:27:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:27:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 20:27:38 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:27:38 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:27:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 20:27:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:38 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:27:38 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=364086, maxMem=766075207 -2016-05-23 20:27:38 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 20:27:38 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=369742, maxMem=766075207 -2016-05-23 20:27:38 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:27:38 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:52980 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:27:38 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:27:38 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:38 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 20:27:38 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 20:27:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:27:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:27:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:27:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:27:38 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 24 ms on localhost (1/2) -2016-05-23 20:27:38 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 27 ms on localhost (2/2) -2016-05-23 20:27:38 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.028 s -2016-05-23 20:27:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 20:27:38 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.047857 s -2016-05-23 20:27:38 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:27:38 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:27:38 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:27:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 20:27:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:38 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:27:38 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=372650, maxMem=766075207 -2016-05-23 20:27:38 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:27:38 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=378346, maxMem=766075207 -2016-05-23 20:27:38 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:27:38 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:52980 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:27:38 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:27:38 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:38 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 20:27:38 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 20:27:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:27:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:27:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-23 20:27:38 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 28 ms on localhost (1/2) -2016-05-23 20:27:38 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 36 ms on localhost (2/2) -2016-05-23 20:27:38 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.037 s -2016-05-23 20:27:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 20:27:38 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.067818 s -2016-05-23 20:27:38 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:43 -2016-05-23 20:27:38 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:43) with 4 output partitions -2016-05-23 20:27:38 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:43) -2016-05-23 20:27:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:27:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:38 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40), which has no missing parents -2016-05-23 20:27:38 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=381235, maxMem=766075207 -2016-05-23 20:27:38 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:27:38 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=382459, maxMem=766075207 -2016-05-23 20:27:38 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:27:38 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:52980 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:27:38 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:38 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40) -2016-05-23 20:27:38 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:27:38 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 20:27:38 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 20:27:38 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 20:27:38 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 11 ms on localhost (1/4) -2016-05-23 20:27:38 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 20:27:38 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 20:27:38 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 20:27:38 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 14 ms on localhost (2/4) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 17 ms on localhost (3/4) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 30 ms on localhost (4/4) -2016-05-23 20:27:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 20:27:38 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:43) finished in 0.030 s -2016-05-23 20:27:38 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:43, took 0.155523 s -2016-05-23 20:27:38 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:45 -2016-05-23 20:27:38 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:45) with 4 output partitions -2016-05-23 20:27:38 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:45) -2016-05-23 20:27:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:27:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:38 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41), which has no missing parents -2016-05-23 20:27:38 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=383352, maxMem=766075207 -2016-05-23 20:27:38 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:27:38 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=384576, maxMem=766075207 -2016-05-23 20:27:38 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:27:38 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:52980 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:27:38 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:38 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41) -2016-05-23 20:27:38 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:27:38 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 20:27:38 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 20:27:38 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 20:27:38 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 20:27:38 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 20:27:38 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 15 ms on localhost (1/4) -2016-05-23 20:27:38 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 20:27:38 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 15 ms on localhost (2/4) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 13 ms on localhost (3/4) -2016-05-23 20:27:38 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:45) finished in 0.020 s -2016-05-23 20:27:38 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:45, took 0.033162 s -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 16 ms on localhost (4/4) -2016-05-23 20:27:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 20:27:38 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:27:38 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:27:38 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 20:27:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:27:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:38 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:27:38 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=385469, maxMem=766075207 -2016-05-23 20:27:38 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:27:38 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=387973, maxMem=766075207 -2016-05-23 20:27:38 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 20:27:38 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:52980 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 20:27:38 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:38 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 20:27:38 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:27:38 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 20:27:38 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 11 ms on localhost (1/1) -2016-05-23 20:27:38 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.011 s -2016-05-23 20:27:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 20:27:38 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.052332 s -2016-05-23 20:27:38 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:27:38 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:27:38 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:27:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:27:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:38 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:27:38 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=389497, maxMem=766075207 -2016-05-23 20:27:38 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:27:38 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=392897, maxMem=766075207 -2016-05-23 20:27:38 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 20:27:38 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:52980 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 20:27:38 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:38 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:27:38 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:27:38 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 20:27:38 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 20:27:38 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 20:27:38 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 20:27:38 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 20:27:38 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 16 ms on localhost (1/4) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 19 ms on localhost (2/4) -2016-05-23 20:27:38 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 18 ms on localhost (3/4) -2016-05-23 20:27:38 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 25 ms on localhost (4/4) -2016-05-23 20:27:38 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.028 s -2016-05-23 20:27:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 20:27:38 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.043539 s -2016-05-23 20:27:38 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:27:38 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:27:38 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:27:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:27:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:38 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:27:38 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=394831, maxMem=766075207 -2016-05-23 20:27:38 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 20:27:38 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=398271, maxMem=766075207 -2016-05-23 20:27:38 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 20:27:38 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:52980 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 20:27:38 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:38 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:27:38 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:27:38 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 20:27:38 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 20:27:38 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 20:27:38 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 20:27:38 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 15 ms on localhost (1/4) -2016-05-23 20:27:38 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 20:27:38 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 20:27:38 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 25 ms on localhost (2/4) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 23 ms on localhost (3/4) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 22 ms on localhost (4/4) -2016-05-23 20:27:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 20:27:38 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.029 s -2016-05-23 20:27:38 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.046600 s -2016-05-23 20:27:38 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:27:38 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:27:38 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:27:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:27:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:38 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:27:38 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=400189, maxMem=766075207 -2016-05-23 20:27:38 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:27:38 INFO MemoryStore:59 - ensureFreeSpace(1860) called with curMem=403565, maxMem=766075207 -2016-05-23 20:27:38 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1860.0 B, free 730.2 MB) -2016-05-23 20:27:38 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:52980 (size: 1860.0 B, free: 730.5 MB) -2016-05-23 20:27:38 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:38 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:27:38 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:27:38 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 20:27:38 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 20:27:38 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 20:27:38 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 20:27:38 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 20:27:38 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 20:27:38 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 20:27:38 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 52 ms on localhost (1/4) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 48 ms on localhost (2/4) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 52 ms on localhost (3/4) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 51 ms on localhost (4/4) -2016-05-23 20:27:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 20:27:38 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.055 s -2016-05-23 20:27:38 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.182715 s -2016-05-23 20:27:38 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:27:38 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:27:38 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:27:38 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:27:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 20:27:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 20:27:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:27:38 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=405425, maxMem=766075207 -2016-05-23 20:27:38 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 20:27:38 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=409465, maxMem=766075207 -2016-05-23 20:27:38 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:27:38 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:52980 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:27:38 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:38 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:27:38 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:27:38 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:27:38 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 20:27:38 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 20:27:38 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 20:27:38 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 20:27:39 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 20:27:39 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 20:27:39 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 20:27:39 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 171 ms on localhost (1/4) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 190 ms on localhost (2/4) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 189 ms on localhost (3/4) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 195 ms on localhost (4/4) -2016-05-23 20:27:39 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.197 s -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 20:27:39 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:27:39 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:27:39 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 20:27:39 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:27:39 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 20:27:39 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:27:39 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=411747, maxMem=766075207 -2016-05-23 20:27:39 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:27:39 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=414291, maxMem=766075207 -2016-05-23 20:27:39 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 20:27:39 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:52980 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 20:27:39 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:39 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:39 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 20:27:39 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 20:27:39 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:27:39 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 20:27:39 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 56 ms on localhost (1/3) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 59 ms on localhost (2/3) -2016-05-23 20:27:39 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 68 ms on localhost (3/3) -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 20:27:39 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.071 s -2016-05-23 20:27:39 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.305649 s -2016-05-23 20:27:39 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:27:39 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:27:39 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:27:39 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:27:39 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 20:27:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 20:27:39 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 20:27:39 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:27:39 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=415859, maxMem=766075207 -2016-05-23 20:27:39 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:27:39 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=420091, maxMem=766075207 -2016-05-23 20:27:39 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 20:27:39 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:52980 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:27:39 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:39 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:27:39 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 20:27:39 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 20:27:39 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:39 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:39 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 173 ms on localhost (1/4) -2016-05-23 20:27:39 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 196 ms on localhost (2/4) -2016-05-23 20:27:39 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 203 ms on localhost (3/4) -2016-05-23 20:27:39 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 209 ms on localhost (4/4) -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 20:27:39 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.214 s -2016-05-23 20:27:39 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:27:39 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:27:39 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 20:27:39 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:27:39 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 20:27:39 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:27:39 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=422397, maxMem=766075207 -2016-05-23 20:27:39 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 20:27:39 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=427293, maxMem=766075207 -2016-05-23 20:27:39 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:27:39 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:52980 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:27:39 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:39 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:39 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:39 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 14 ms on localhost (1/1) -2016-05-23 20:27:39 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.015 s -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 20:27:39 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.268749 s -2016-05-23 20:27:39 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:27:39 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:27:39 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 20:27:39 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:27:39 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:27:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 20:27:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:39 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:27:39 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=429835, maxMem=766075207 -2016-05-23 20:27:39 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:27:39 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=435531, maxMem=766075207 -2016-05-23 20:27:39 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:27:39 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:52980 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:27:39 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:39 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:39 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 20:27:39 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 20:27:39 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:39 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:27:39 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:39 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 20:27:39 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:39 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 80 ms on localhost (1/4) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 81 ms on localhost (2/4) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 81 ms on localhost (3/4) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 84 ms on localhost (4/4) -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 20:27:39 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.086 s -2016-05-23 20:27:39 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.103038 s -2016-05-23 20:27:39 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:27:39 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:27:39 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:27:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 20:27:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:39 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:27:39 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=438440, maxMem=766075207 -2016-05-23 20:27:39 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:27:39 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=444176, maxMem=766075207 -2016-05-23 20:27:39 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:27:39 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:52980 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:27:39 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:39 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:39 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 20:27:39 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 20:27:39 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:39 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 20:27:39 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 27 ms on localhost (1/4) -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:39 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 33 ms on localhost (2/4) -2016-05-23 20:27:39 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 38 ms on localhost (3/4) -2016-05-23 20:27:39 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 42 ms on localhost (4/4) -2016-05-23 20:27:39 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.044 s -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 20:27:39 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.060193 s -2016-05-23 20:27:39 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:57 -2016-05-23 20:27:39 INFO DAGScheduler:59 - Got job 20 (foreach at StatisticsCorrLearning.scala:57) with 4 output partitions -2016-05-23 20:27:39 INFO DAGScheduler:59 - Final stage: ResultStage 34(foreach at StatisticsCorrLearning.scala:57) -2016-05-23 20:27:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:27:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:39 INFO DAGScheduler:59 - Submitting ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:54), which has no missing parents -2016-05-23 20:27:39 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=447068, maxMem=766075207 -2016-05-23 20:27:39 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:27:39 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=448292, maxMem=766075207 -2016-05-23 20:27:39 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:27:39 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:52980 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:27:39 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:39 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:54) -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 62, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 63, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 64, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:27:39 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 64) -2016-05-23 20:27:39 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 64). 915 bytes result sent to driver -2016-05-23 20:27:39 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 62) -2016-05-23 20:27:39 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 63) -2016-05-23 20:27:39 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 62). 915 bytes result sent to driver -2016-05-23 20:27:39 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 65) -2016-05-23 20:27:39 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 65). 915 bytes result sent to driver -2016-05-23 20:27:39 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 63). 915 bytes result sent to driver -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 62) in 20 ms on localhost (1/4) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 64) in 19 ms on localhost (2/4) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 65) in 20 ms on localhost (3/4) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 63) in 24 ms on localhost (4/4) -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-23 20:27:39 INFO DAGScheduler:59 - ResultStage 34 (foreach at StatisticsCorrLearning.scala:57) finished in 0.026 s -2016-05-23 20:27:39 INFO DAGScheduler:59 - Job 20 finished: foreach at StatisticsCorrLearning.scala:57, took 0.038783 s -2016-05-23 20:27:39 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:59 -2016-05-23 20:27:39 INFO DAGScheduler:59 - Got job 21 (foreach at StatisticsCorrLearning.scala:59) with 4 output partitions -2016-05-23 20:27:39 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at StatisticsCorrLearning.scala:59) -2016-05-23 20:27:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:27:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:39 INFO DAGScheduler:59 - Submitting ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:55), which has no missing parents -2016-05-23 20:27:39 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=449185, maxMem=766075207 -2016-05-23 20:27:39 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:27:39 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=450409, maxMem=766075207 -2016-05-23 20:27:39 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:27:39 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:52980 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:27:39 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:39 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:55) -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 4 tasks -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 66, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 3.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:27:39 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 68) -2016-05-23 20:27:39 INFO Executor:59 - Running task 3.0 in stage 35.0 (TID 69) -2016-05-23 20:27:39 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 68). 915 bytes result sent to driver -2016-05-23 20:27:39 INFO Executor:59 - Finished task 3.0 in stage 35.0 (TID 69). 915 bytes result sent to driver -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 68) in 7 ms on localhost (1/4) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 3.0 in stage 35.0 (TID 69) in 7 ms on localhost (2/4) -2016-05-23 20:27:39 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 66) -2016-05-23 20:27:39 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 67) -2016-05-23 20:27:39 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 66). 915 bytes result sent to driver -2016-05-23 20:27:39 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 67). 915 bytes result sent to driver -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 66) in 15 ms on localhost (3/4) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 67) in 15 ms on localhost (4/4) -2016-05-23 20:27:39 INFO DAGScheduler:59 - ResultStage 35 (foreach at StatisticsCorrLearning.scala:59) finished in 0.016 s -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-23 20:27:39 INFO DAGScheduler:59 - Job 21 finished: foreach at StatisticsCorrLearning.scala:59, took 0.043239 s -2016-05-23 20:27:39 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:27:39 INFO DAGScheduler:59 - Got job 22 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:27:39 INFO DAGScheduler:59 - Final stage: ResultStage 36(first at RowMatrix.scala:65) -2016-05-23 20:27:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:27:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:39 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:27:39 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=451302, maxMem=766075207 -2016-05-23 20:27:39 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:27:39 INFO MemoryStore:59 - ensureFreeSpace(1522) called with curMem=453806, maxMem=766075207 -2016-05-23 20:27:39 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1522.0 B, free 730.2 MB) -2016-05-23 20:27:39 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:52980 (size: 1522.0 B, free: 730.5 MB) -2016-05-23 20:27:39 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:39 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:27:39 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 70) -2016-05-23 20:27:39 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 70). 1046 bytes result sent to driver -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 70) in 6 ms on localhost (1/1) -2016-05-23 20:27:39 INFO DAGScheduler:59 - ResultStage 36 (first at RowMatrix.scala:65) finished in 0.006 s -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-23 20:27:39 INFO DAGScheduler:59 - Job 22 finished: first at RowMatrix.scala:65, took 0.021634 s -2016-05-23 20:27:39 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:27:39 INFO DAGScheduler:59 - Got job 23 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:27:39 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:27:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:27:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:39 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:27:39 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=455328, maxMem=766075207 -2016-05-23 20:27:39 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 20:27:39 INFO MemoryStore:59 - ensureFreeSpace(1932) called with curMem=458728, maxMem=766075207 -2016-05-23 20:27:39 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1932.0 B, free 730.1 MB) -2016-05-23 20:27:39 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:52980 (size: 1932.0 B, free: 730.5 MB) -2016-05-23 20:27:39 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:39 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 4 tasks -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 3.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:27:39 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 73) -2016-05-23 20:27:39 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 72) -2016-05-23 20:27:39 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-23 20:27:39 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 73). 1231 bytes result sent to driver -2016-05-23 20:27:39 INFO Executor:59 - Running task 3.0 in stage 37.0 (TID 74) -2016-05-23 20:27:39 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 1231 bytes result sent to driver -2016-05-23 20:27:39 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 72). 1231 bytes result sent to driver -2016-05-23 20:27:39 INFO Executor:59 - Finished task 3.0 in stage 37.0 (TID 74). 1231 bytes result sent to driver -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 73) in 12 ms on localhost (1/4) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 15 ms on localhost (2/4) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 72) in 15 ms on localhost (3/4) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 3.0 in stage 37.0 (TID 74) in 14 ms on localhost (4/4) -2016-05-23 20:27:39 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at RowMatrix.scala:332) finished in 0.017 s -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-23 20:27:39 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at RowMatrix.scala:332, took 0.031996 s -2016-05-23 20:27:39 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:27:39 INFO DAGScheduler:59 - Got job 24 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:27:39 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:27:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:27:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:39 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:27:39 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=460660, maxMem=766075207 -2016-05-23 20:27:39 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-23 20:27:39 INFO MemoryStore:59 - ensureFreeSpace(1916) called with curMem=464100, maxMem=766075207 -2016-05-23 20:27:39 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1916.0 B, free 730.1 MB) -2016-05-23 20:27:39 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:52980 (size: 1916.0 B, free: 730.5 MB) -2016-05-23 20:27:39 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:39 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 77, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 78, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:27:39 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-23 20:27:39 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 1109 bytes result sent to driver -2016-05-23 20:27:39 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 78) -2016-05-23 20:27:39 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 78). 1109 bytes result sent to driver -2016-05-23 20:27:39 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-23 20:27:39 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 1109 bytes result sent to driver -2016-05-23 20:27:39 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 77) -2016-05-23 20:27:39 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 77). 1109 bytes result sent to driver -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 33 ms on localhost (1/4) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 78) in 32 ms on localhost (2/4) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 45 ms on localhost (3/4) -2016-05-23 20:27:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 77) in 45 ms on localhost (4/4) -2016-05-23 20:27:39 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at RowMatrix.scala:124) finished in 0.048 s -2016-05-23 20:27:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-23 20:27:39 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at RowMatrix.scala:124, took 0.095596 s -2016-05-23 20:27:40 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:27:40 INFO DAGScheduler:59 - Got job 25 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:27:40 INFO DAGScheduler:59 - Final stage: ResultStage 39(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:27:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:27:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:40 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:27:40 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=466016, maxMem=766075207 -2016-05-23 20:27:40 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 20:27:40 INFO MemoryStore:59 - ensureFreeSpace(1851) called with curMem=469392, maxMem=766075207 -2016-05-23 20:27:40 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1851.0 B, free 730.1 MB) -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:52980 (size: 1851.0 B, free: 730.5 MB) -2016-05-23 20:27:40 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:40 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:27:40 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 4 tasks -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 79, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 80, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 81, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 3.0 in stage 39.0 (TID 82, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:27:40 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 80) -2016-05-23 20:27:40 INFO Executor:59 - Running task 3.0 in stage 39.0 (TID 82) -2016-05-23 20:27:40 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 81) -2016-05-23 20:27:40 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 79) -2016-05-23 20:27:40 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 80). 1334 bytes result sent to driver -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 80) in 24 ms on localhost (1/4) -2016-05-23 20:27:40 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 79). 1294 bytes result sent to driver -2016-05-23 20:27:40 INFO Executor:59 - Finished task 3.0 in stage 39.0 (TID 82). 1334 bytes result sent to driver -2016-05-23 20:27:40 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 81). 1294 bytes result sent to driver -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 79) in 32 ms on localhost (2/4) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 81) in 31 ms on localhost (3/4) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 3.0 in stage 39.0 (TID 82) in 31 ms on localhost (4/4) -2016-05-23 20:27:40 INFO DAGScheduler:59 - ResultStage 39 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.034 s -2016-05-23 20:27:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-23 20:27:40 INFO DAGScheduler:59 - Job 25 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.048595 s -2016-05-23 20:27:40 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:27:40 INFO DAGScheduler:59 - Registering RDD 51 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:27:40 INFO DAGScheduler:59 - Got job 26 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:27:40 INFO DAGScheduler:59 - Final stage: ResultStage 41(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:27:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 40) -2016-05-23 20:27:40 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 40) -2016-05-23 20:27:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:27:40 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=471243, maxMem=766075207 -2016-05-23 20:27:40 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-05-23 20:27:40 INFO MemoryStore:59 - ensureFreeSpace(2280) called with curMem=475283, maxMem=766075207 -2016-05-23 20:27:40 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:52980 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:27:40 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:40 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:27:40 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 4 tasks -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 83, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 84, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 85, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 3.0 in stage 40.0 (TID 86, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:27:40 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 83) -2016-05-23 20:27:40 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 84) -2016-05-23 20:27:40 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 85) -2016-05-23 20:27:40 INFO Executor:59 - Running task 3.0 in stage 40.0 (TID 86) -2016-05-23 20:27:40 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 85). 1161 bytes result sent to driver -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 85) in 56 ms on localhost (1/4) -2016-05-23 20:27:40 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 83). 1161 bytes result sent to driver -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 83) in 60 ms on localhost (2/4) -2016-05-23 20:27:40 INFO Executor:59 - Finished task 3.0 in stage 40.0 (TID 86). 1161 bytes result sent to driver -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 3.0 in stage 40.0 (TID 86) in 68 ms on localhost (3/4) -2016-05-23 20:27:40 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 84). 1161 bytes result sent to driver -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 84) in 75 ms on localhost (4/4) -2016-05-23 20:27:40 INFO DAGScheduler:59 - ShuffleMapStage 40 (flatMap at SpearmanCorrelation.scala:49) finished in 0.076 s -2016-05-23 20:27:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-23 20:27:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:27:40 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:27:40 INFO DAGScheduler:59 - waiting: Set(ResultStage 41) -2016-05-23 20:27:40 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:27:40 INFO DAGScheduler:59 - Missing parents for ResultStage 41: List() -2016-05-23 20:27:40 INFO DAGScheduler:59 - Submitting ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:27:40 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=477563, maxMem=766075207 -2016-05-23 20:27:40 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 20:27:40 INFO MemoryStore:59 - ensureFreeSpace(1566) called with curMem=480107, maxMem=766075207 -2016-05-23 20:27:40 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 1566.0 B, free 730.1 MB) -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:52980 (size: 1566.0 B, free: 730.5 MB) -2016-05-23 20:27:40 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:40 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:27:40 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 3 tasks -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 87, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:40 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 87) -2016-05-23 20:27:40 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 89) -2016-05-23 20:27:40 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 88) -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:27:40 INFO Executor:59 - Finished task 2.0 in stage 41.0 (TID 89). 1203 bytes result sent to driver -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 2.0 in stage 41.0 (TID 89) in 20 ms on localhost (1/3) -2016-05-23 20:27:40 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 88). 1203 bytes result sent to driver -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 88) in 25 ms on localhost (2/3) -2016-05-23 20:27:40 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 87). 1203 bytes result sent to driver -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 87) in 28 ms on localhost (3/3) -2016-05-23 20:27:40 INFO DAGScheduler:59 - ResultStage 41 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.029 s -2016-05-23 20:27:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-23 20:27:40 INFO DAGScheduler:59 - Job 26 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.133489 s -2016-05-23 20:27:40 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:27:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 168 bytes -2016-05-23 20:27:40 INFO DAGScheduler:59 - Registering RDD 56 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:27:40 INFO DAGScheduler:59 - Got job 27 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:27:40 INFO DAGScheduler:59 - Final stage: ResultStage 44(first at RowMatrix.scala:65) -2016-05-23 20:27:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 43) -2016-05-23 20:27:40 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 43) -2016-05-23 20:27:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:27:40 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=481673, maxMem=766075207 -2016-05-23 20:27:40 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 20:27:40 INFO MemoryStore:59 - ensureFreeSpace(2304) called with curMem=485905, maxMem=766075207 -2016-05-23 20:27:40 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:52980 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:27:40 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:40 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:27:40 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 4 tasks -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 90, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 91, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 2.0 in stage 43.0 (TID 92, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 3.0 in stage 43.0 (TID 93, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:27:40 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 90) -2016-05-23 20:27:40 INFO Executor:59 - Running task 3.0 in stage 43.0 (TID 93) -2016-05-23 20:27:40 INFO Executor:59 - Running task 2.0 in stage 43.0 (TID 92) -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:27:40 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 91) -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:27:40 INFO Executor:59 - Finished task 2.0 in stage 43.0 (TID 92). 1377 bytes result sent to driver -2016-05-23 20:27:40 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 91). 1377 bytes result sent to driver -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 2.0 in stage 43.0 (TID 92) in 205 ms on localhost (1/4) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 91) in 206 ms on localhost (2/4) -2016-05-23 20:27:40 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 90). 1377 bytes result sent to driver -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 90) in 219 ms on localhost (3/4) -2016-05-23 20:27:40 INFO Executor:59 - Finished task 3.0 in stage 43.0 (TID 93). 1377 bytes result sent to driver -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 3.0 in stage 43.0 (TID 93) in 228 ms on localhost (4/4) -2016-05-23 20:27:40 INFO DAGScheduler:59 - ShuffleMapStage 43 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.231 s -2016-05-23 20:27:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-23 20:27:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:27:40 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:27:40 INFO DAGScheduler:59 - waiting: Set(ResultStage 44) -2016-05-23 20:27:40 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:27:40 INFO DAGScheduler:59 - Missing parents for ResultStage 44: List() -2016-05-23 20:27:40 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:27:40 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=488209, maxMem=766075207 -2016-05-23 20:27:40 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 20:27:40 INFO MemoryStore:59 - ensureFreeSpace(2538) called with curMem=493105, maxMem=766075207 -2016-05-23 20:27:40 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:52980 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:27:40 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:40 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:27:40 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 1 tasks -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 94, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:40 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 94) -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:27:40 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 94). 1296 bytes result sent to driver -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 94) in 10 ms on localhost (1/1) -2016-05-23 20:27:40 INFO DAGScheduler:59 - ResultStage 44 (first at RowMatrix.scala:65) finished in 0.011 s -2016-05-23 20:27:40 INFO DAGScheduler:59 - Job 27 finished: first at RowMatrix.scala:65, took 0.270058 s -2016-05-23 20:27:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-23 20:27:40 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:27:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 168 bytes -2016-05-23 20:27:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 166 bytes -2016-05-23 20:27:40 INFO DAGScheduler:59 - Got job 28 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:27:40 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:27:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 46) -2016-05-23 20:27:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:40 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:27:40 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=495643, maxMem=766075207 -2016-05-23 20:27:40 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 20:27:40 INFO MemoryStore:59 - ensureFreeSpace(2907) called with curMem=501339, maxMem=766075207 -2016-05-23 20:27:40 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:52980 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:27:40 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:40 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:27:40 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 4 tasks -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 95, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 96, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 2.0 in stage 47.0 (TID 97, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 3.0 in stage 47.0 (TID 98, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:40 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 95) -2016-05-23 20:27:40 INFO Executor:59 - Running task 3.0 in stage 47.0 (TID 98) -2016-05-23 20:27:40 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 96) -2016-05-23 20:27:40 INFO Executor:59 - Running task 2.0 in stage 47.0 (TID 97) -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:27:40 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 95). 1481 bytes result sent to driver -2016-05-23 20:27:40 INFO Executor:59 - Finished task 3.0 in stage 47.0 (TID 98). 1481 bytes result sent to driver -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 95) in 18 ms on localhost (1/4) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 3.0 in stage 47.0 (TID 98) in 16 ms on localhost (2/4) -2016-05-23 20:27:40 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 96). 1481 bytes result sent to driver -2016-05-23 20:27:40 INFO Executor:59 - Finished task 2.0 in stage 47.0 (TID 97). 1481 bytes result sent to driver -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 2.0 in stage 47.0 (TID 97) in 20 ms on localhost (3/4) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 96) in 22 ms on localhost (4/4) -2016-05-23 20:27:40 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at RowMatrix.scala:332) finished in 0.023 s -2016-05-23 20:27:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-23 20:27:40 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at RowMatrix.scala:332, took 0.038295 s -2016-05-23 20:27:40 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:27:40 INFO DAGScheduler:59 - Got job 29 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:27:40 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:27:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 49) -2016-05-23 20:27:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:27:40 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:27:40 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=504246, maxMem=766075207 -2016-05-23 20:27:40 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 20:27:40 INFO MemoryStore:59 - ensureFreeSpace(2888) called with curMem=509982, maxMem=766075207 -2016-05-23 20:27:40 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:52980 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:27:40 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:27:40 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:27:40 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 4 tasks -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 2.0 in stage 50.0 (TID 101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Starting task 3.0 in stage 50.0 (TID 102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:27:40 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-23 20:27:40 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-23 20:27:40 INFO Executor:59 - Running task 2.0 in stage 50.0 (TID 101) -2016-05-23 20:27:40 INFO Executor:59 - Running task 3.0 in stage 50.0 (TID 102) -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:27:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:27:40 INFO Executor:59 - Finished task 2.0 in stage 50.0 (TID 101). 1359 bytes result sent to driver -2016-05-23 20:27:40 INFO Executor:59 - Finished task 3.0 in stage 50.0 (TID 102). 1359 bytes result sent to driver -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 2.0 in stage 50.0 (TID 101) in 16 ms on localhost (1/4) -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 3.0 in stage 50.0 (TID 102) in 17 ms on localhost (2/4) -2016-05-23 20:27:40 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 1359 bytes result sent to driver -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 40 ms on localhost (3/4) -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:52980 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:27:40 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 1359 bytes result sent to driver -2016-05-23 20:27:40 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-23 20:27:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 48 ms on localhost (4/4) -2016-05-23 20:27:40 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at RowMatrix.scala:124) finished in 0.048 s -2016-05-23 20:27:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-23 20:27:40 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at RowMatrix.scala:124, took 0.063000 s -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:52980 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:27:40 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:52980 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:27:40 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:52980 in memory (size: 1566.0 B, free: 730.5 MB) -2016-05-23 20:27:40 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:52980 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 20:27:40 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:52980 in memory (size: 1851.0 B, free: 730.5 MB) -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 20:27:40 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 20:27:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:52980 in memory (size: 1916.0 B, free: 730.5 MB) -2016-05-23 20:27:40 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:52980 in memory (size: 1932.0 B, free: 730.5 MB) -2016-05-23 20:27:40 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:52980 in memory (size: 1522.0 B, free: 730.5 MB) -2016-05-23 20:27:40 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:52980 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:27:40 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:52980 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:27:40 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:52980 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:27:40 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:52980 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:27:40 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:52980 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:27:40 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:52980 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:27:40 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-23 20:27:40 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:52980 in memory (size: 1568.0 B, free: 730.5 MB) -2016-05-23 20:27:40 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-23 20:27:40 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 20:27:40 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:52980 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:27:40 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 20:27:40 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 20:27:41 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 20:27:41 INFO BlockManager:59 - BlockManager stopped -2016-05-23 20:27:41 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 20:27:41 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 20:27:41 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 20:27:41 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 20:27:41 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-eed62070-50dd-4b49-941e-35ecd8b951d4 -2016-05-23 20:28:49 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 20:28:49 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 20:28:50 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 20:28:50 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 20:28:50 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 20:28:50 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 20:28:51 INFO Remoting:74 - Starting remoting -2016-05-23 20:28:51 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53013] -2016-05-23 20:28:51 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53013. -2016-05-23 20:28:51 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 20:28:51 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 20:28:51 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-c9c680cb-e0d9-436a-b1af-566f535a99ff -2016-05-23 20:28:51 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 20:28:51 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-b263c525-ecac-4cfc-a158-03f81763aeb5\httpd-fd9e13b0-1620-4d3e-a612-f09ad599eab0 -2016-05-23 20:28:51 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 20:28:51 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:28:51 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53014 -2016-05-23 20:28:51 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53014. -2016-05-23 20:28:51 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 20:28:51 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:28:51 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 20:28:51 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 20:28:51 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 20:28:51 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 20:28:51 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 20:28:52 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53033. -2016-05-23 20:28:52 INFO NettyBlockTransferService:59 - Server created on 53033 -2016-05-23 20:28:52 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 20:28:52 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53033 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53033) -2016-05-23 20:28:52 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 20:28:52 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 20:28:52 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 20:28:53 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 20:28:53 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 20:28:53 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53033 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:28:53 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:20 -2016-05-23 20:28:53 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 20:28:53 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 20:28:53 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 20:28:53 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 20:28:53 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53033 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:28:53 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 20:28:54 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 20:28:54 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:28:54 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:27 -2016-05-23 20:28:54 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:27) with 2 output partitions -2016-05-23 20:28:54 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:27) -2016-05-23 20:28:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:28:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:54 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21), which has no missing parents -2016-05-23 20:28:54 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 20:28:54 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:28:54 INFO MemoryStore:59 - ensureFreeSpace(2047) called with curMem=346712, maxMem=766075207 -2016-05-23 20:28:54 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2047.0 B, free 730.3 MB) -2016-05-23 20:28:54 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53033 (size: 2047.0 B, free: 730.6 MB) -2016-05-23 20:28:54 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21) -2016-05-23 20:28:54 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 20:28:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:28:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:28:54 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 20:28:54 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 20:28:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:28:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:28:54 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 20:28:54 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 20:28:54 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 20:28:54 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 20:28:54 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 20:28:54 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 20:28:54 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 20:28:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 118 ms on localhost (1/2) -2016-05-23 20:28:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 100 ms on localhost (2/2) -2016-05-23 20:28:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 20:28:54 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:27) finished in 0.138 s -2016-05-23 20:28:54 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:27, took 0.303131 s -2016-05-23 20:28:54 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:28:54 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 20:28:54 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 20:28:54 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 20:28:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:28:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:54 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24), which has no missing parents -2016-05-23 20:28:54 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348759, maxMem=766075207 -2016-05-23 20:28:54 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:28:54 INFO MemoryStore:59 - ensureFreeSpace(2045) called with curMem=352271, maxMem=766075207 -2016-05-23 20:28:54 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2045.0 B, free 730.2 MB) -2016-05-23 20:28:54 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53033 (size: 2045.0 B, free: 730.6 MB) -2016-05-23 20:28:54 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24) -2016-05-23 20:28:54 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 20:28:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:28:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:28:54 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 20:28:54 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 20:28:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:28:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:28:55 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 20:28:55 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 20:28:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 21 ms on localhost (1/2) -2016-05-23 20:28:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 23 ms on localhost (2/2) -2016-05-23 20:28:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 20:28:55 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:29) finished in 0.024 s -2016-05-23 20:28:55 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:29, took 0.050231 s -2016-05-23 20:28:55 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:28:55 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:28:55 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 20:28:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:28:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:55 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:28:55 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354316, maxMem=766075207 -2016-05-23 20:28:55 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 20:28:55 INFO MemoryStore:59 - ensureFreeSpace(2465) called with curMem=358876, maxMem=766075207 -2016-05-23 20:28:55 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:28:55 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53033 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:28:55 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:55 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 20:28:55 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 20:28:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:28:55 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 20:28:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:28:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:28:55 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 20:28:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 33 ms on localhost (1/1) -2016-05-23 20:28:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 20:28:55 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.035 s -2016-05-23 20:28:55 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.074000 s -2016-05-23 20:28:55 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:28:55 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:28:55 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:28:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:28:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:55 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:28:55 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=361341, maxMem=766075207 -2016-05-23 20:28:55 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 20:28:55 INFO MemoryStore:59 - ensureFreeSpace(2862) called with curMem=366789, maxMem=766075207 -2016-05-23 20:28:55 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:28:55 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53033 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:28:55 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:28:55 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 20:28:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:28:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:28:55 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 20:28:55 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 20:28:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:28:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:28:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:28:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:28:55 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 20:28:55 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 20:28:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 42 ms on localhost (1/2) -2016-05-23 20:28:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 45 ms on localhost (2/2) -2016-05-23 20:28:55 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.045 s -2016-05-23 20:28:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 20:28:55 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 20:28:55 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 20:28:55 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.074597 s -2016-05-23 20:28:55 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:28:55 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:28:55 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:28:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:28:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:55 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:28:55 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=369651, maxMem=766075207 -2016-05-23 20:28:55 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 20:28:55 INFO MemoryStore:59 - ensureFreeSpace(2851) called with curMem=375147, maxMem=766075207 -2016-05-23 20:28:55 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:28:55 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53033 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:28:55 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:28:55 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 20:28:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:28:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:28:55 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 20:28:55 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 20:28:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:28:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:28:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:28:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:28:55 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 20:28:55 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 20:28:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 37 ms on localhost (1/2) -2016-05-23 20:28:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 36 ms on localhost (2/2) -2016-05-23 20:28:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 20:28:55 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.039 s -2016-05-23 20:28:55 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.061909 s -2016-05-23 20:28:56 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:28:56 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 20:28:56 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:28:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:28:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:56 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:28:56 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=377998, maxMem=766075207 -2016-05-23 20:28:56 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 20:28:56 INFO MemoryStore:59 - ensureFreeSpace(2744) called with curMem=383342, maxMem=766075207 -2016-05-23 20:28:56 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 20:28:56 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53033 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 20:28:56 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:28:56 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 20:28:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:28:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:28:56 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 20:28:56 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 20:28:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:28:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:28:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:28:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:28:56 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 20:28:56 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 20:28:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 39 ms on localhost (1/2) -2016-05-23 20:28:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 39 ms on localhost (2/2) -2016-05-23 20:28:56 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.041 s -2016-05-23 20:28:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 20:28:56 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.058575 s -2016-05-23 20:28:56 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:28:56 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:28:56 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 20:28:56 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:28:56 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 20:28:56 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 20:28:56 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:28:56 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=386086, maxMem=766075207 -2016-05-23 20:28:56 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 20:28:56 INFO MemoryStore:59 - ensureFreeSpace(3159) called with curMem=392014, maxMem=766075207 -2016-05-23 20:28:56 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 20:28:56 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53033 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:28:56 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:28:56 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 20:28:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:28:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:28:56 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 20:28:56 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 20:28:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:28:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:28:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:28:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:28:56 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 20:28:56 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 20:28:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 481 ms on localhost (1/2) -2016-05-23 20:28:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 484 ms on localhost (2/2) -2016-05-23 20:28:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 20:28:56 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.485 s -2016-05-23 20:28:56 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:28:56 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:28:56 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 20:28:56 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:28:56 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 20:28:56 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:28:56 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=395173, maxMem=766075207 -2016-05-23 20:28:56 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:28:56 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=397677, maxMem=766075207 -2016-05-23 20:28:56 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 20:28:56 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53033 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:28:56 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:56 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:28:56 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 20:28:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:56 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 20:28:56 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:28:56 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-23 20:28:56 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 20:28:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 55 ms on localhost (1/1) -2016-05-23 20:28:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 20:28:56 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.056 s -2016-05-23 20:28:56 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.592920 s -2016-05-23 20:28:56 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:28:56 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:28:56 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:28:56 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:28:56 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 20:28:56 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 20:28:56 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 20:28:56 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:28:56 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=399222, maxMem=766075207 -2016-05-23 20:28:56 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=403414, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:28:57 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53033 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:28:57 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:28:57 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 20:28:57 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 20:28:57 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53033 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:28:57 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 20:28:57 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 20:28:57 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53033 in memory (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:28:57 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 51 ms on localhost (1/2) -2016-05-23 20:28:57 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 57 ms on localhost (2/2) -2016-05-23 20:28:57 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.058 s -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 20:28:57 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53033 in memory (size: 2.7 KB, free: 730.5 MB) -2016-05-23 20:28:57 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:28:57 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:28:57 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 20:28:57 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:28:57 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:28:57 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53033 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=381621, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 20:28:57 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=378119, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:28:57 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53033 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:28:57 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 20:28:57 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53033 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:28:57 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:57 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:28:57 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 28 ms on localhost (1/1) -2016-05-23 20:28:57 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.029 s -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 20:28:57 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.154333 s -2016-05-23 20:28:57 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:28:57 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:28:57 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 20:28:57 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:28:57 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:28:57 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 20:28:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=375202, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=380858, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:28:57 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53033 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:28:57 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:57 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 20:28:57 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:57 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 20:28:57 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 26 ms on localhost (1/2) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 24 ms on localhost (2/2) -2016-05-23 20:28:57 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.026 s -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 20:28:57 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.046514 s -2016-05-23 20:28:57 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:28:57 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:28:57 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 20:28:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=383766, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=389462, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:28:57 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53033 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:28:57 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:57 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 20:28:57 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:57 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 23 ms on localhost (1/2) -2016-05-23 20:28:57 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 27 ms on localhost (2/2) -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 20:28:57 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.030 s -2016-05-23 20:28:57 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.053295 s -2016-05-23 20:28:57 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:43 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:43) with 4 output partitions -2016-05-23 20:28:57 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:43) -2016-05-23 20:28:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:28:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40), which has no missing parents -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=392351, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=393575, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:28:57 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53033 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:28:57 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40) -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:28:57 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 20:28:57 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 20:28:57 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 20:28:57 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 20:28:57 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 20:28:57 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 20:28:57 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 12 ms on localhost (1/4) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 12 ms on localhost (2/4) -2016-05-23 20:28:57 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 26 ms on localhost (3/4) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 16 ms on localhost (4/4) -2016-05-23 20:28:57 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:43) finished in 0.027 s -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 20:28:57 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:43, took 0.041683 s -2016-05-23 20:28:57 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:45 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:45) with 4 output partitions -2016-05-23 20:28:57 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:45) -2016-05-23 20:28:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:28:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41), which has no missing parents -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=394468, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=395692, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:28:57 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53033 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:28:57 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41) -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:28:57 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 20:28:57 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 20:28:57 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 20:28:57 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 20:28:57 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 20:28:57 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 20:28:57 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 11 ms on localhost (1/4) -2016-05-23 20:28:57 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 9 ms on localhost (2/4) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 13 ms on localhost (3/4) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 12 ms on localhost (4/4) -2016-05-23 20:28:57 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:45) finished in 0.014 s -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 20:28:57 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:45, took 0.025011 s -2016-05-23 20:28:57 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:28:57 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 20:28:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:28:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=396585, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(1525) called with curMem=399089, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1525.0 B, free 730.2 MB) -2016-05-23 20:28:57 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53033 (size: 1525.0 B, free: 730.5 MB) -2016-05-23 20:28:57 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:28:57 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 20:28:57 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 8 ms on localhost (1/1) -2016-05-23 20:28:57 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.008 s -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 20:28:57 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.021510 s -2016-05-23 20:28:57 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:28:57 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:28:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:28:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=400614, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=404014, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.2 MB) -2016-05-23 20:28:57 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53033 (size: 1935.0 B, free: 730.5 MB) -2016-05-23 20:28:57 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:28:57 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 20:28:57 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 20:28:57 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 20:28:57 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 20:28:57 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 14 ms on localhost (1/4) -2016-05-23 20:28:57 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 15 ms on localhost (2/4) -2016-05-23 20:28:57 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 20:28:57 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 23 ms on localhost (3/4) -2016-05-23 20:28:57 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.025 s -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 21 ms on localhost (4/4) -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 20:28:57 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.038631 s -2016-05-23 20:28:57 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:28:57 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:28:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:28:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=405949, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(1919) called with curMem=409389, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1919.0 B, free 730.2 MB) -2016-05-23 20:28:57 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53033 (size: 1919.0 B, free: 730.5 MB) -2016-05-23 20:28:57 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:28:57 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 20:28:57 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 20:28:57 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 20:28:57 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 20:28:57 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 20:28:57 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 20:28:57 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 20:28:57 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 14 ms on localhost (1/4) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 18 ms on localhost (2/4) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 17 ms on localhost (3/4) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 18 ms on localhost (4/4) -2016-05-23 20:28:57 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.020 s -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 20:28:57 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.033895 s -2016-05-23 20:28:57 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:28:57 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:28:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:28:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=411308, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(1861) called with curMem=414684, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1861.0 B, free 730.2 MB) -2016-05-23 20:28:57 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53033 (size: 1861.0 B, free: 730.5 MB) -2016-05-23 20:28:57 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:28:57 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 20:28:57 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 20:28:57 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 20:28:57 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 20:28:57 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 20:28:57 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 14 ms on localhost (1/4) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 16 ms on localhost (2/4) -2016-05-23 20:28:57 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 20:28:57 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 23 ms on localhost (3/4) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 21 ms on localhost (4/4) -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 20:28:57 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.025 s -2016-05-23 20:28:57 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.041252 s -2016-05-23 20:28:57 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:28:57 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:28:57 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:28:57 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 20:28:57 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=416545, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(2283) called with curMem=420585, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:28:57 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:53033 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:28:57 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:28:57 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 20:28:57 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 20:28:57 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 20:28:57 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 20:28:57 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 80 ms on localhost (1/4) -2016-05-23 20:28:57 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 20:28:57 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 85 ms on localhost (2/4) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 84 ms on localhost (3/4) -2016-05-23 20:28:57 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 135 ms on localhost (4/4) -2016-05-23 20:28:57 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.140 s -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 20:28:57 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:28:57 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:28:57 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 20:28:57 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:28:57 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=422868, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=425412, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 20:28:57 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:53033 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 20:28:57 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:57 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 20:28:57 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:57 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 20:28:57 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 35 ms on localhost (1/3) -2016-05-23 20:28:57 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 20:28:57 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 47 ms on localhost (2/3) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 48 ms on localhost (3/3) -2016-05-23 20:28:57 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.050 s -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 20:28:57 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.230030 s -2016-05-23 20:28:57 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:28:57 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:28:57 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:28:57 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:28:57 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 20:28:57 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 20:28:57 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=426980, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:28:57 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=431212, maxMem=766075207 -2016-05-23 20:28:57 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 20:28:57 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:53033 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:28:57 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:57 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:28:57 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:28:57 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:28:57 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 20:28:57 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 20:28:57 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 20:28:57 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:28:57 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 92 ms on localhost (1/4) -2016-05-23 20:28:57 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 20:28:57 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 120 ms on localhost (2/4) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 20:28:58 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 212 ms on localhost (3/4) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 215 ms on localhost (4/4) -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 20:28:58 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.217 s -2016-05-23 20:28:58 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:28:58 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:28:58 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 20:28:58 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:28:58 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=433518, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=438414, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:28:58 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:53033 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:28:58 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:58 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:28:58 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 18 ms on localhost (1/1) -2016-05-23 20:28:58 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.018 s -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 20:28:58 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.271086 s -2016-05-23 20:28:58 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:28:58 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:28:58 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 20:28:58 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:28:58 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:28:58 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 20:28:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=440956, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=446652, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:28:58 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:53033 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:28:58 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:58 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 20:28:58 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 20:28:58 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:28:58 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 20:28:58 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:28:58 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 34 ms on localhost (1/4) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 34 ms on localhost (2/4) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 36 ms on localhost (3/4) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 43 ms on localhost (4/4) -2016-05-23 20:28:58 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.043 s -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 20:28:58 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.061397 s -2016-05-23 20:28:58 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:28:58 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:28:58 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:28:58 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 20:28:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=449561, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=455297, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:28:58 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:53033 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:28:58 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:58 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 20:28:58 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:58 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 20:28:58 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:58 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 26 ms on localhost (1/4) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 20:28:58 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 37 ms on localhost (2/4) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 39 ms on localhost (3/4) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 39 ms on localhost (4/4) -2016-05-23 20:28:58 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.042 s -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 20:28:58 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.062668 s -2016-05-23 20:28:58 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:57 -2016-05-23 20:28:58 INFO DAGScheduler:59 - Got job 20 (foreach at StatisticsCorrLearning.scala:57) with 4 output partitions -2016-05-23 20:28:58 INFO DAGScheduler:59 - Final stage: ResultStage 34(foreach at StatisticsCorrLearning.scala:57) -2016-05-23 20:28:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:28:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:54), which has no missing parents -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=458189, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=459413, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.1 MB) -2016-05-23 20:28:58 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:53033 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:28:58 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:54) -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 62, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 63, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 64, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:28:58 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 63) -2016-05-23 20:28:58 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 62) -2016-05-23 20:28:58 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 64) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 63). 915 bytes result sent to driver -2016-05-23 20:28:58 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 65) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 64). 915 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 63) in 8 ms on localhost (1/4) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 65). 915 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 64) in 9 ms on localhost (2/4) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 62). 915 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 65) in 12 ms on localhost (3/4) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 62) in 15 ms on localhost (4/4) -2016-05-23 20:28:58 INFO DAGScheduler:59 - ResultStage 34 (foreach at StatisticsCorrLearning.scala:57) finished in 0.015 s -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-23 20:28:58 INFO DAGScheduler:59 - Job 20 finished: foreach at StatisticsCorrLearning.scala:57, took 0.028651 s -2016-05-23 20:28:58 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:59 -2016-05-23 20:28:58 INFO DAGScheduler:59 - Got job 21 (foreach at StatisticsCorrLearning.scala:59) with 4 output partitions -2016-05-23 20:28:58 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at StatisticsCorrLearning.scala:59) -2016-05-23 20:28:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:28:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:55), which has no missing parents -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=460306, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=461530, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.1 MB) -2016-05-23 20:28:58 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:53033 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:28:58 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:55) -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 4 tasks -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 66, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 3.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:28:58 INFO Executor:59 - Running task 3.0 in stage 35.0 (TID 69) -2016-05-23 20:28:58 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 67) -2016-05-23 20:28:58 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 66) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 3.0 in stage 35.0 (TID 69). 915 bytes result sent to driver -2016-05-23 20:28:58 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 67). 915 bytes result sent to driver -2016-05-23 20:28:58 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 68) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 66). 915 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 3.0 in stage 35.0 (TID 69) in 10 ms on localhost (1/4) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 68). 915 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 67) in 12 ms on localhost (2/4) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 66) in 14 ms on localhost (3/4) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 68) in 13 ms on localhost (4/4) -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-23 20:28:58 INFO DAGScheduler:59 - ResultStage 35 (foreach at StatisticsCorrLearning.scala:59) finished in 0.015 s -2016-05-23 20:28:58 INFO DAGScheduler:59 - Job 21 finished: foreach at StatisticsCorrLearning.scala:59, took 0.028721 s -2016-05-23 20:28:58 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:28:58 INFO DAGScheduler:59 - Got job 22 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:28:58 INFO DAGScheduler:59 - Final stage: ResultStage 36(first at RowMatrix.scala:65) -2016-05-23 20:28:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:28:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=462423, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(1523) called with curMem=464927, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1523.0 B, free 730.1 MB) -2016-05-23 20:28:58 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:53033 (size: 1523.0 B, free: 730.5 MB) -2016-05-23 20:28:58 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:28:58 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 70) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 70). 1046 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 70) in 5 ms on localhost (1/1) -2016-05-23 20:28:58 INFO DAGScheduler:59 - ResultStage 36 (first at RowMatrix.scala:65) finished in 0.006 s -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-23 20:28:58 INFO DAGScheduler:59 - Job 22 finished: first at RowMatrix.scala:65, took 0.019838 s -2016-05-23 20:28:58 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:28:58 INFO DAGScheduler:59 - Got job 23 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:28:58 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:28:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:28:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=466450, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(1933) called with curMem=469850, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1933.0 B, free 730.1 MB) -2016-05-23 20:28:58 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:53033 (size: 1933.0 B, free: 730.5 MB) -2016-05-23 20:28:58 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 4 tasks -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 3.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:28:58 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 73) -2016-05-23 20:28:58 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-23 20:28:58 INFO Executor:59 - Running task 3.0 in stage 37.0 (TID 74) -2016-05-23 20:28:58 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 72) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 73). 1231 bytes result sent to driver -2016-05-23 20:28:58 INFO Executor:59 - Finished task 3.0 in stage 37.0 (TID 74). 1231 bytes result sent to driver -2016-05-23 20:28:58 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 1231 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 73) in 8 ms on localhost (1/4) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 72). 1231 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 3.0 in stage 37.0 (TID 74) in 8 ms on localhost (2/4) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 11 ms on localhost (3/4) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 72) in 11 ms on localhost (4/4) -2016-05-23 20:28:58 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at RowMatrix.scala:332) finished in 0.013 s -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-23 20:28:58 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at RowMatrix.scala:332, took 0.025145 s -2016-05-23 20:28:58 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:28:58 INFO DAGScheduler:59 - Got job 24 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:28:58 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:28:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:28:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=471783, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(1917) called with curMem=475223, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1917.0 B, free 730.1 MB) -2016-05-23 20:28:58 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:53033 (size: 1917.0 B, free: 730.5 MB) -2016-05-23 20:28:58 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 77, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 78, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:28:58 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-23 20:28:58 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 78) -2016-05-23 20:28:58 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-23 20:28:58 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 77) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 1109 bytes result sent to driver -2016-05-23 20:28:58 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 77). 1109 bytes result sent to driver -2016-05-23 20:28:58 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 78). 1109 bytes result sent to driver -2016-05-23 20:28:58 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 1109 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 11 ms on localhost (1/4) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 77) in 11 ms on localhost (2/4) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 78) in 11 ms on localhost (3/4) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 13 ms on localhost (4/4) -2016-05-23 20:28:58 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at RowMatrix.scala:124) finished in 0.014 s -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-23 20:28:58 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at RowMatrix.scala:124, took 0.030621 s -2016-05-23 20:28:58 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:28:58 INFO DAGScheduler:59 - Got job 25 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:28:58 INFO DAGScheduler:59 - Final stage: ResultStage 39(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:28:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:28:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=477140, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(1852) called with curMem=480516, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1852.0 B, free 730.1 MB) -2016-05-23 20:28:58 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:53033 (size: 1852.0 B, free: 730.5 MB) -2016-05-23 20:28:58 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 4 tasks -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 79, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 80, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 81, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 3.0 in stage 39.0 (TID 82, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:28:58 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 81) -2016-05-23 20:28:58 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 80) -2016-05-23 20:28:58 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 79) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 79). 1294 bytes result sent to driver -2016-05-23 20:28:58 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 80). 1334 bytes result sent to driver -2016-05-23 20:28:58 INFO Executor:59 - Running task 3.0 in stage 39.0 (TID 82) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 81). 1294 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 79) in 15 ms on localhost (1/4) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 81) in 13 ms on localhost (2/4) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 80) in 15 ms on localhost (3/4) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 3.0 in stage 39.0 (TID 82). 1334 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 3.0 in stage 39.0 (TID 82) in 14 ms on localhost (4/4) -2016-05-23 20:28:58 INFO DAGScheduler:59 - ResultStage 39 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.018 s -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-23 20:28:58 INFO DAGScheduler:59 - Job 25 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.031633 s -2016-05-23 20:28:58 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:28:58 INFO DAGScheduler:59 - Registering RDD 51 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:28:58 INFO DAGScheduler:59 - Got job 26 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:28:58 INFO DAGScheduler:59 - Final stage: ResultStage 41(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:28:58 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 40) -2016-05-23 20:28:58 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 40) -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=482368, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(2281) called with curMem=486408, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 20:28:58 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:53033 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:28:58 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 4 tasks -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 83, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 84, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 85, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 3.0 in stage 40.0 (TID 86, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:28:58 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 83) -2016-05-23 20:28:58 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 85) -2016-05-23 20:28:58 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 84) -2016-05-23 20:28:58 INFO Executor:59 - Running task 3.0 in stage 40.0 (TID 86) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 3.0 in stage 40.0 (TID 86). 1161 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 3.0 in stage 40.0 (TID 86) in 219 ms on localhost (1/4) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 84). 1161 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 84) in 228 ms on localhost (2/4) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 85). 1161 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 85) in 234 ms on localhost (3/4) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 83). 1161 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 83) in 245 ms on localhost (4/4) -2016-05-23 20:28:58 INFO DAGScheduler:59 - ShuffleMapStage 40 (flatMap at SpearmanCorrelation.scala:49) finished in 0.245 s -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-23 20:28:58 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:28:58 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:28:58 INFO DAGScheduler:59 - waiting: Set(ResultStage 41) -2016-05-23 20:28:58 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:28:58 INFO DAGScheduler:59 - Missing parents for ResultStage 41: List() -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=488689, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(1566) called with curMem=491233, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 1566.0 B, free 730.1 MB) -2016-05-23 20:28:58 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:53033 (size: 1566.0 B, free: 730.5 MB) -2016-05-23 20:28:58 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 3 tasks -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 87, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:58 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 87) -2016-05-23 20:28:58 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 88) -2016-05-23 20:28:58 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 89) -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:58 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 88). 1203 bytes result sent to driver -2016-05-23 20:28:58 INFO Executor:59 - Finished task 2.0 in stage 41.0 (TID 89). 1203 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 88) in 12 ms on localhost (1/3) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 2.0 in stage 41.0 (TID 89) in 13 ms on localhost (2/3) -2016-05-23 20:28:58 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 87). 1203 bytes result sent to driver -2016-05-23 20:28:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 87) in 15 ms on localhost (3/3) -2016-05-23 20:28:58 INFO DAGScheduler:59 - ResultStage 41 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.017 s -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-23 20:28:58 INFO DAGScheduler:59 - Job 26 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.288423 s -2016-05-23 20:28:58 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:28:58 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 168 bytes -2016-05-23 20:28:58 INFO DAGScheduler:59 - Registering RDD 56 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:28:58 INFO DAGScheduler:59 - Got job 27 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:28:58 INFO DAGScheduler:59 - Final stage: ResultStage 44(first at RowMatrix.scala:65) -2016-05-23 20:28:58 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 43) -2016-05-23 20:28:58 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 43) -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=492799, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 20:28:58 INFO MemoryStore:59 - ensureFreeSpace(2304) called with curMem=497031, maxMem=766075207 -2016-05-23 20:28:58 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-23 20:28:58 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:53033 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:28:58 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:58 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:28:58 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 4 tasks -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 90, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 91, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 2.0 in stage 43.0 (TID 92, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:28:58 INFO TaskSetManager:59 - Starting task 3.0 in stage 43.0 (TID 93, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:28:58 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 90) -2016-05-23 20:28:58 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 91) -2016-05-23 20:28:58 INFO Executor:59 - Running task 2.0 in stage 43.0 (TID 92) -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:28:58 INFO Executor:59 - Running task 3.0 in stage 43.0 (TID 93) -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:59 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 91). 1377 bytes result sent to driver -2016-05-23 20:28:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 91) in 155 ms on localhost (1/4) -2016-05-23 20:28:59 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 90). 1377 bytes result sent to driver -2016-05-23 20:28:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 90) in 179 ms on localhost (2/4) -2016-05-23 20:28:59 INFO Executor:59 - Finished task 3.0 in stage 43.0 (TID 93). 1377 bytes result sent to driver -2016-05-23 20:28:59 INFO Executor:59 - Finished task 2.0 in stage 43.0 (TID 92). 1377 bytes result sent to driver -2016-05-23 20:28:59 INFO TaskSetManager:59 - Finished task 3.0 in stage 43.0 (TID 93) in 292 ms on localhost (3/4) -2016-05-23 20:28:59 INFO TaskSetManager:59 - Finished task 2.0 in stage 43.0 (TID 92) in 294 ms on localhost (4/4) -2016-05-23 20:28:59 INFO DAGScheduler:59 - ShuffleMapStage 43 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.295 s -2016-05-23 20:28:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-23 20:28:59 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:28:59 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:28:59 INFO DAGScheduler:59 - waiting: Set(ResultStage 44) -2016-05-23 20:28:59 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:28:59 INFO DAGScheduler:59 - Missing parents for ResultStage 44: List() -2016-05-23 20:28:59 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:28:59 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=499335, maxMem=766075207 -2016-05-23 20:28:59 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 20:28:59 INFO MemoryStore:59 - ensureFreeSpace(2538) called with curMem=504231, maxMem=766075207 -2016-05-23 20:28:59 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:53033 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:28:59 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:59 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:28:59 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 1 tasks -2016-05-23 20:28:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 94, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:59 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 94) -2016-05-23 20:28:59 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:59 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:59 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 94). 1296 bytes result sent to driver -2016-05-23 20:28:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 94) in 9 ms on localhost (1/1) -2016-05-23 20:28:59 INFO DAGScheduler:59 - ResultStage 44 (first at RowMatrix.scala:65) finished in 0.011 s -2016-05-23 20:28:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-23 20:28:59 INFO DAGScheduler:59 - Job 27 finished: first at RowMatrix.scala:65, took 0.332696 s -2016-05-23 20:28:59 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:28:59 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 168 bytes -2016-05-23 20:28:59 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 166 bytes -2016-05-23 20:28:59 INFO DAGScheduler:59 - Got job 28 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:28:59 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:28:59 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 46) -2016-05-23 20:28:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:59 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:28:59 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=506769, maxMem=766075207 -2016-05-23 20:28:59 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 20:28:59 INFO MemoryStore:59 - ensureFreeSpace(2907) called with curMem=512465, maxMem=766075207 -2016-05-23 20:28:59 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:53033 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:28:59 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:59 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:28:59 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 4 tasks -2016-05-23 20:28:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 95, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 96, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:59 INFO TaskSetManager:59 - Starting task 2.0 in stage 47.0 (TID 97, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:59 INFO TaskSetManager:59 - Starting task 3.0 in stage 47.0 (TID 98, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:59 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 95) -2016-05-23 20:28:59 INFO Executor:59 - Running task 3.0 in stage 47.0 (TID 98) -2016-05-23 20:28:59 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 96) -2016-05-23 20:28:59 INFO Executor:59 - Running task 2.0 in stage 47.0 (TID 97) -2016-05-23 20:28:59 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:59 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:59 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:59 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:59 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:59 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:28:59 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:59 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:59 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 95). 1481 bytes result sent to driver -2016-05-23 20:28:59 INFO Executor:59 - Finished task 2.0 in stage 47.0 (TID 97). 1481 bytes result sent to driver -2016-05-23 20:28:59 INFO Executor:59 - Finished task 3.0 in stage 47.0 (TID 98). 1481 bytes result sent to driver -2016-05-23 20:28:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 95) in 19 ms on localhost (1/4) -2016-05-23 20:28:59 INFO TaskSetManager:59 - Finished task 3.0 in stage 47.0 (TID 98) in 18 ms on localhost (2/4) -2016-05-23 20:28:59 INFO TaskSetManager:59 - Finished task 2.0 in stage 47.0 (TID 97) in 20 ms on localhost (3/4) -2016-05-23 20:28:59 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 96). 1481 bytes result sent to driver -2016-05-23 20:28:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 96) in 23 ms on localhost (4/4) -2016-05-23 20:28:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-23 20:28:59 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at RowMatrix.scala:332) finished in 0.026 s -2016-05-23 20:28:59 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at RowMatrix.scala:332, took 0.041699 s -2016-05-23 20:28:59 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:28:59 INFO DAGScheduler:59 - Got job 29 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:28:59 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:28:59 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 49) -2016-05-23 20:28:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:28:59 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:28:59 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=515372, maxMem=766075207 -2016-05-23 20:28:59 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 20:28:59 INFO MemoryStore:59 - ensureFreeSpace(2888) called with curMem=521108, maxMem=766075207 -2016-05-23 20:28:59 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:53033 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:28:59 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:28:59 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:28:59 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 4 tasks -2016-05-23 20:28:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:59 INFO TaskSetManager:59 - Starting task 2.0 in stage 50.0 (TID 101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:59 INFO TaskSetManager:59 - Starting task 3.0 in stage 50.0 (TID 102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:28:59 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-23 20:28:59 INFO Executor:59 - Running task 3.0 in stage 50.0 (TID 102) -2016-05-23 20:28:59 INFO Executor:59 - Running task 2.0 in stage 50.0 (TID 101) -2016-05-23 20:28:59 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-23 20:28:59 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:59 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:59 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:28:59 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:28:59 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:59 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:59 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:28:59 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:28:59 INFO Executor:59 - Finished task 3.0 in stage 50.0 (TID 102). 1359 bytes result sent to driver -2016-05-23 20:28:59 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 1359 bytes result sent to driver -2016-05-23 20:28:59 INFO TaskSetManager:59 - Finished task 3.0 in stage 50.0 (TID 102) in 32 ms on localhost (1/4) -2016-05-23 20:28:59 INFO Executor:59 - Finished task 2.0 in stage 50.0 (TID 101). 1359 bytes result sent to driver -2016-05-23 20:28:59 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 1359 bytes result sent to driver -2016-05-23 20:28:59 INFO TaskSetManager:59 - Finished task 2.0 in stage 50.0 (TID 101) in 38 ms on localhost (2/4) -2016-05-23 20:28:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 39 ms on localhost (3/4) -2016-05-23 20:28:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 42 ms on localhost (4/4) -2016-05-23 20:28:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-23 20:28:59 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at RowMatrix.scala:124) finished in 0.044 s -2016-05-23 20:28:59 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at RowMatrix.scala:124, took 0.060284 s -2016-05-23 20:28:59 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:53033 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:28:59 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:53033 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 20:28:59 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:53033 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 20:28:59 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 20:28:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:53033 in memory (size: 1566.0 B, free: 730.5 MB) -2016-05-23 20:28:59 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:53033 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:28:59 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:53033 in memory (size: 1852.0 B, free: 730.5 MB) -2016-05-23 20:28:59 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:53033 in memory (size: 1917.0 B, free: 730.5 MB) -2016-05-23 20:28:59 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:53033 in memory (size: 1933.0 B, free: 730.5 MB) -2016-05-23 20:28:59 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:53033 in memory (size: 1523.0 B, free: 730.5 MB) -2016-05-23 20:28:59 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:53033 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:28:59 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:53033 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:28:59 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:53033 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:28:59 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:53033 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:28:59 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:53033 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:28:59 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:53033 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:28:59 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-23 20:28:59 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:53033 in memory (size: 1568.0 B, free: 730.5 MB) -2016-05-23 20:28:59 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-23 20:28:59 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:53033 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:28:59 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-23 20:28:59 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-05-23 20:28:59 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:53033 in memory (size: 1861.0 B, free: 730.5 MB) -2016-05-23 20:28:59 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 20:28:59 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 20:28:59 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 20:28:59 INFO BlockManager:59 - BlockManager stopped -2016-05-23 20:28:59 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 20:28:59 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 20:28:59 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 20:28:59 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 20:28:59 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-b263c525-ecac-4cfc-a158-03f81763aeb5 -2016-05-23 20:28:59 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 20:28:59 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 20:29:19 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 20:29:21 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 20:29:21 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 20:29:21 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 20:29:21 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 20:29:23 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 20:29:23 INFO Remoting:74 - Starting remoting -2016-05-23 20:29:23 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53055] -2016-05-23 20:29:23 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53055. -2016-05-23 20:29:23 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 20:29:23 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 20:29:23 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-a397734f-af20-4cf3-95c4-e04f0e9956bb -2016-05-23 20:29:23 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 20:29:23 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-8ecec389-faa7-409e-81d4-9d1a8ec11b95\httpd-52a27a97-e816-4d47-a8d2-73843dc4df37 -2016-05-23 20:29:23 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 20:29:23 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:29:23 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53056 -2016-05-23 20:29:23 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53056. -2016-05-23 20:29:23 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 20:29:24 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:29:24 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 20:29:24 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 20:29:24 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 20:29:24 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 20:29:24 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 20:29:25 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53075. -2016-05-23 20:29:25 INFO NettyBlockTransferService:59 - Server created on 53075 -2016-05-23 20:29:25 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 20:29:25 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53075 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53075) -2016-05-23 20:29:25 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 20:29:26 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 20:29:26 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 20:29:26 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 20:29:26 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 20:29:26 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53075 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:29:26 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:20 -2016-05-23 20:29:26 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 20:29:26 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 20:29:26 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 20:29:26 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 20:29:26 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53075 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:29:26 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 20:29:28 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 20:29:28 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:29:28 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:27 -2016-05-23 20:29:28 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:27) with 2 output partitions -2016-05-23 20:29:28 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:27) -2016-05-23 20:29:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:29:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:28 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21), which has no missing parents -2016-05-23 20:29:28 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 20:29:28 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:29:28 INFO MemoryStore:59 - ensureFreeSpace(2047) called with curMem=346712, maxMem=766075207 -2016-05-23 20:29:28 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2047.0 B, free 730.3 MB) -2016-05-23 20:29:28 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53075 (size: 2047.0 B, free: 730.6 MB) -2016-05-23 20:29:28 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21) -2016-05-23 20:29:28 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 20:29:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:29:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:29:28 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 20:29:28 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 20:29:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:29:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:29:28 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 20:29:28 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 20:29:28 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 20:29:28 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 20:29:28 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 20:29:28 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 20:29:28 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 20:29:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 288 ms on localhost (1/2) -2016-05-23 20:29:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 258 ms on localhost (2/2) -2016-05-23 20:29:28 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:27) finished in 0.348 s -2016-05-23 20:29:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 20:29:28 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:27, took 0.548226 s -2016-05-23 20:29:28 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:29:28 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 20:29:28 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 20:29:28 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 20:29:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:29:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:28 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24), which has no missing parents -2016-05-23 20:29:28 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348759, maxMem=766075207 -2016-05-23 20:29:28 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:29:28 INFO MemoryStore:59 - ensureFreeSpace(2045) called with curMem=352271, maxMem=766075207 -2016-05-23 20:29:28 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2045.0 B, free 730.2 MB) -2016-05-23 20:29:28 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53075 (size: 2045.0 B, free: 730.6 MB) -2016-05-23 20:29:28 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24) -2016-05-23 20:29:28 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 20:29:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:29:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:29:28 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 20:29:28 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 20:29:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:29:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:29:28 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 20:29:28 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 20:29:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 43 ms on localhost (1/2) -2016-05-23 20:29:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 56 ms on localhost (2/2) -2016-05-23 20:29:28 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:29) finished in 0.056 s -2016-05-23 20:29:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 20:29:28 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:29, took 0.090072 s -2016-05-23 20:29:28 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:29:28 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:29:28 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 20:29:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:29:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:28 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:29:28 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354316, maxMem=766075207 -2016-05-23 20:29:28 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 20:29:29 INFO MemoryStore:59 - ensureFreeSpace(2465) called with curMem=358876, maxMem=766075207 -2016-05-23 20:29:29 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:29:29 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53075 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:29:29 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 20:29:29 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 20:29:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:29:29 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 20:29:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:29:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:29:29 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 20:29:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 99 ms on localhost (1/1) -2016-05-23 20:29:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 20:29:29 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.100 s -2016-05-23 20:29:29 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.170529 s -2016-05-23 20:29:29 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:29:29 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:29:29 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:29:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:29:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:29 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:29:29 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=361341, maxMem=766075207 -2016-05-23 20:29:29 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 20:29:29 INFO MemoryStore:59 - ensureFreeSpace(2862) called with curMem=366789, maxMem=766075207 -2016-05-23 20:29:29 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:29:29 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53075 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:29:29 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:29:29 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 20:29:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:29:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:29:29 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 20:29:29 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 20:29:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:29:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:29:29 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 20:29:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:29:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:29:29 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 20:29:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 166 ms on localhost (1/2) -2016-05-23 20:29:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 174 ms on localhost (2/2) -2016-05-23 20:29:29 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.174 s -2016-05-23 20:29:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 20:29:29 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 20:29:29 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 20:29:29 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.211164 s -2016-05-23 20:29:30 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:29:30 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:29:30 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:29:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:29:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:30 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:29:30 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=369651, maxMem=766075207 -2016-05-23 20:29:30 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 20:29:30 INFO MemoryStore:59 - ensureFreeSpace(2851) called with curMem=375147, maxMem=766075207 -2016-05-23 20:29:30 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:29:30 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53075 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:29:30 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:29:30 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 20:29:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:29:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:29:30 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 20:29:30 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 20:29:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:29:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:29:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:29:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:29:30 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 20:29:30 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 20:29:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 40 ms on localhost (1/2) -2016-05-23 20:29:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 44 ms on localhost (2/2) -2016-05-23 20:29:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 20:29:30 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.045 s -2016-05-23 20:29:30 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.080366 s -2016-05-23 20:29:30 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:29:30 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 20:29:30 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:29:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:29:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:30 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:29:30 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=377998, maxMem=766075207 -2016-05-23 20:29:30 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 20:29:30 INFO MemoryStore:59 - ensureFreeSpace(2744) called with curMem=383342, maxMem=766075207 -2016-05-23 20:29:30 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 20:29:30 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53075 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 20:29:30 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:29:30 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 20:29:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:29:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:29:30 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 20:29:30 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 20:29:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:29:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:29:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:29:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:29:30 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 20:29:30 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 20:29:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 74 ms on localhost (1/2) -2016-05-23 20:29:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 82 ms on localhost (2/2) -2016-05-23 20:29:30 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.082 s -2016-05-23 20:29:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 20:29:30 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.103675 s -2016-05-23 20:29:30 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:29:30 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:29:30 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 20:29:30 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:29:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 20:29:30 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 20:29:30 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:29:30 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=386086, maxMem=766075207 -2016-05-23 20:29:30 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 20:29:30 INFO MemoryStore:59 - ensureFreeSpace(3156) called with curMem=392014, maxMem=766075207 -2016-05-23 20:29:30 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 20:29:30 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53075 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:29:30 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:29:30 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 20:29:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:29:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:29:30 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 20:29:30 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 20:29:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:29:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:29:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:29:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:29:30 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 20:29:30 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 20:29:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 82 ms on localhost (1/2) -2016-05-23 20:29:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 81 ms on localhost (2/2) -2016-05-23 20:29:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 20:29:30 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.084 s -2016-05-23 20:29:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:29:30 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:29:30 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 20:29:30 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:29:30 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 20:29:30 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:29:30 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=395170, maxMem=766075207 -2016-05-23 20:29:30 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:29:30 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=397674, maxMem=766075207 -2016-05-23 20:29:30 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 20:29:30 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53075 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:29:30 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:29:30 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:31 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 20:29:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:29:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-23 20:29:31 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 74 ms on localhost (1/1) -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 20:29:31 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.075 s -2016-05-23 20:29:31 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.221290 s -2016-05-23 20:29:31 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:29:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:29:31 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:29:31 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:29:31 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 20:29:31 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 20:29:31 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=399219, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=403411, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:29:31 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53075 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:29:31 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:29:31 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 20:29:31 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 20:29:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:29:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 20:29:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:29:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:31 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 42 ms on localhost (1/2) -2016-05-23 20:29:31 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 93 ms on localhost (2/2) -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 20:29:31 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.099 s -2016-05-23 20:29:31 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:29:31 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:29:31 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 20:29:31 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:29:31 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=405693, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=410549, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:29:31 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53075 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:29:31 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:31 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 20:29:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:29:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:31 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53075 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:29:31 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 20:29:31 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 37 ms on localhost (1/1) -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 20:29:31 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.039 s -2016-05-23 20:29:31 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.244069 s -2016-05-23 20:29:31 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53075 in memory (size: 2.7 KB, free: 730.5 MB) -2016-05-23 20:29:31 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 20:29:31 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53075 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:29:31 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 20:29:31 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 20:29:31 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53075 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:29:31 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:29:31 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 20:29:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:29:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 20:29:31 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53075 in memory (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:29:31 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:29:31 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:29:31 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 20:29:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=375202, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=380858, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:29:31 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53075 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:29:31 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:31 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 20:29:31 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 20:29:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:29:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:29:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:29:31 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 28 ms on localhost (1/2) -2016-05-23 20:29:31 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 33 ms on localhost (2/2) -2016-05-23 20:29:31 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.033 s -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 20:29:31 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.056854 s -2016-05-23 20:29:31 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:29:31 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:29:31 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:29:31 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 20:29:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=383766, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=389462, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:29:31 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53075 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:29:31 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:31 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 20:29:31 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 20:29:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:29:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:29:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:29:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:31 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 20:29:31 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 21 ms on localhost (1/2) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 21 ms on localhost (2/2) -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 20:29:31 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.023 s -2016-05-23 20:29:31 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.040993 s -2016-05-23 20:29:31 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:43 -2016-05-23 20:29:31 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:43) with 4 output partitions -2016-05-23 20:29:31 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:43) -2016-05-23 20:29:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:29:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40), which has no missing parents -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=392351, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=393575, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:29:31 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53075 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:29:31 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40) -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:29:31 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 20:29:31 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 20:29:31 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 20:29:31 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 20:29:31 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 20:29:31 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 20:29:31 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 10 ms on localhost (1/4) -2016-05-23 20:29:31 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 14 ms on localhost (2/4) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 24 ms on localhost (3/4) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 13 ms on localhost (4/4) -2016-05-23 20:29:31 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:43) finished in 0.024 s -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 20:29:31 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:43, took 0.040653 s -2016-05-23 20:29:31 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:45 -2016-05-23 20:29:31 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:45) with 4 output partitions -2016-05-23 20:29:31 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:45) -2016-05-23 20:29:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:29:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41), which has no missing parents -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=394468, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=395692, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:29:31 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53075 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:29:31 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41) -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:29:31 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 20:29:31 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 20:29:31 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 20:29:31 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 20:29:31 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 20:29:31 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 7 ms on localhost (1/4) -2016-05-23 20:29:31 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 20:29:31 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 13 ms on localhost (2/4) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 12 ms on localhost (3/4) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 11 ms on localhost (4/4) -2016-05-23 20:29:31 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:45) finished in 0.014 s -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 20:29:31 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:45, took 0.026595 s -2016-05-23 20:29:31 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:29:31 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:29:31 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 20:29:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:29:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=396585, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=399089, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 20:29:31 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53075 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 20:29:31 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:29:31 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 20:29:31 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 8 ms on localhost (1/1) -2016-05-23 20:29:31 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.008 s -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 20:29:31 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.020199 s -2016-05-23 20:29:31 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:29:31 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:29:31 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:29:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:29:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=400613, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=404013, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 20:29:31 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53075 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 20:29:31 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:29:31 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 20:29:31 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 20:29:31 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 20:29:31 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 20:29:31 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 20:29:31 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 9 ms on localhost (1/4) -2016-05-23 20:29:31 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 12 ms on localhost (2/4) -2016-05-23 20:29:31 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 15 ms on localhost (3/4) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 15 ms on localhost (4/4) -2016-05-23 20:29:31 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.016 s -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 20:29:31 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.032087 s -2016-05-23 20:29:31 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:29:31 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:29:31 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:29:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:29:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=405947, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=409387, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 20:29:31 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53075 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 20:29:31 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:29:31 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 20:29:31 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 20:29:31 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 20:29:31 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 20:29:31 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 20:29:31 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 20:29:31 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 20:29:31 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 25 ms on localhost (1/4) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 20 ms on localhost (2/4) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 30 ms on localhost (3/4) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 25 ms on localhost (4/4) -2016-05-23 20:29:31 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.032 s -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 20:29:31 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.052041 s -2016-05-23 20:29:31 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:29:31 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:29:31 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:29:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:29:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=411305, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(1860) called with curMem=414681, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1860.0 B, free 730.2 MB) -2016-05-23 20:29:31 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53075 (size: 1860.0 B, free: 730.5 MB) -2016-05-23 20:29:31 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:29:31 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 20:29:31 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 20:29:31 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 20:29:31 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 20:29:31 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 20:29:31 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 20:29:31 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 21 ms on localhost (1/4) -2016-05-23 20:29:31 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 26 ms on localhost (2/4) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 23 ms on localhost (3/4) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 27 ms on localhost (4/4) -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 20:29:31 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.030 s -2016-05-23 20:29:31 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.047429 s -2016-05-23 20:29:31 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:29:31 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:29:31 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:29:31 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:29:31 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 20:29:31 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=416541, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 20:29:31 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=420581, maxMem=766075207 -2016-05-23 20:29:31 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:29:31 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:53075 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:29:31 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:29:31 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:29:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:29:31 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 20:29:31 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 20:29:31 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 20:29:31 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 20:29:31 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 20:29:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 89 ms on localhost (1/4) -2016-05-23 20:29:31 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 20:29:31 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 145 ms on localhost (2/4) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 145 ms on localhost (3/4) -2016-05-23 20:29:32 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 191 ms on localhost (4/4) -2016-05-23 20:29:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 20:29:32 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.196 s -2016-05-23 20:29:32 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:29:32 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:29:32 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 20:29:32 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:29:32 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 20:29:32 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:29:32 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=422863, maxMem=766075207 -2016-05-23 20:29:32 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:29:32 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=425407, maxMem=766075207 -2016-05-23 20:29:32 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 20:29:32 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:53075 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 20:29:32 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:32 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:29:32 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:32 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:32 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 20:29:32 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:29:32 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 33 ms on localhost (1/3) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 36 ms on localhost (2/3) -2016-05-23 20:29:32 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:29:32 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 66 ms on localhost (3/3) -2016-05-23 20:29:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 20:29:32 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.067 s -2016-05-23 20:29:32 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.336869 s -2016-05-23 20:29:32 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:29:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:29:32 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:29:32 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:29:32 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 20:29:32 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 20:29:32 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 20:29:32 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:29:32 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=426975, maxMem=766075207 -2016-05-23 20:29:32 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:29:32 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=431207, maxMem=766075207 -2016-05-23 20:29:32 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 20:29:32 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:53075 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:29:32 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:29:32 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:29:32 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:32 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:29:32 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:32 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:32 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 20:29:32 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 137 ms on localhost (1/4) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 137 ms on localhost (2/4) -2016-05-23 20:29:32 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 184 ms on localhost (3/4) -2016-05-23 20:29:32 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 186 ms on localhost (4/4) -2016-05-23 20:29:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 20:29:32 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.194 s -2016-05-23 20:29:32 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:29:32 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:29:32 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 20:29:32 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:29:32 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 20:29:32 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:29:32 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=433513, maxMem=766075207 -2016-05-23 20:29:32 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 20:29:32 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=438409, maxMem=766075207 -2016-05-23 20:29:32 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:29:32 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:53075 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:29:32 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:29:32 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:32 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:32 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 121 ms on localhost (1/1) -2016-05-23 20:29:32 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.123 s -2016-05-23 20:29:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 20:29:32 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.359429 s -2016-05-23 20:29:32 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:29:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:29:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 20:29:32 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:29:32 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:29:32 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 20:29:32 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:32 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:29:32 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=440951, maxMem=766075207 -2016-05-23 20:29:32 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:29:32 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=446647, maxMem=766075207 -2016-05-23 20:29:32 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:29:32 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:53075 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:29:32 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:29:32 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:32 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 20:29:32 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 20:29:32 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:32 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:29:32 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 38 ms on localhost (1/4) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 41 ms on localhost (2/4) -2016-05-23 20:29:32 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:32 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 20:29:32 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 97 ms on localhost (3/4) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 97 ms on localhost (4/4) -2016-05-23 20:29:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 20:29:32 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.101 s -2016-05-23 20:29:32 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.119847 s -2016-05-23 20:29:32 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:29:32 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:29:32 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:29:32 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 20:29:32 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:32 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:29:32 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=449556, maxMem=766075207 -2016-05-23 20:29:32 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:29:32 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=455292, maxMem=766075207 -2016-05-23 20:29:32 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:29:32 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:53075 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:29:32 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:29:32 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:32 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 20:29:32 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:29:32 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:32 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:32 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 20:29:32 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 61 ms on localhost (1/4) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 61 ms on localhost (2/4) -2016-05-23 20:29:32 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 85 ms on localhost (3/4) -2016-05-23 20:29:32 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 90 ms on localhost (4/4) -2016-05-23 20:29:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 20:29:32 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.093 s -2016-05-23 20:29:32 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.124664 s -2016-05-23 20:29:32 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:57 -2016-05-23 20:29:32 INFO DAGScheduler:59 - Got job 20 (foreach at StatisticsCorrLearning.scala:57) with 4 output partitions -2016-05-23 20:29:32 INFO DAGScheduler:59 - Final stage: ResultStage 34(foreach at StatisticsCorrLearning.scala:57) -2016-05-23 20:29:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:29:32 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:32 INFO DAGScheduler:59 - Submitting ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:54), which has no missing parents -2016-05-23 20:29:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=458184, maxMem=766075207 -2016-05-23 20:29:32 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-05-23 20:29:32 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=459408, maxMem=766075207 -2016-05-23 20:29:32 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.1 MB) -2016-05-23 20:29:32 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:53075 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:29:32 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:54) -2016-05-23 20:29:32 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 62, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 63, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 64, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:29:32 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 65) -2016-05-23 20:29:32 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 63) -2016-05-23 20:29:32 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 65). 915 bytes result sent to driver -2016-05-23 20:29:32 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 63). 915 bytes result sent to driver -2016-05-23 20:29:32 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 64) -2016-05-23 20:29:32 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 64). 915 bytes result sent to driver -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 65) in 10 ms on localhost (1/4) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 63) in 14 ms on localhost (2/4) -2016-05-23 20:29:32 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 62) -2016-05-23 20:29:32 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 62). 915 bytes result sent to driver -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 64) in 24 ms on localhost (3/4) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 62) in 27 ms on localhost (4/4) -2016-05-23 20:29:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-23 20:29:32 INFO DAGScheduler:59 - ResultStage 34 (foreach at StatisticsCorrLearning.scala:57) finished in 0.028 s -2016-05-23 20:29:32 INFO DAGScheduler:59 - Job 20 finished: foreach at StatisticsCorrLearning.scala:57, took 0.041773 s -2016-05-23 20:29:32 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:59 -2016-05-23 20:29:32 INFO DAGScheduler:59 - Got job 21 (foreach at StatisticsCorrLearning.scala:59) with 4 output partitions -2016-05-23 20:29:32 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at StatisticsCorrLearning.scala:59) -2016-05-23 20:29:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:29:32 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:32 INFO DAGScheduler:59 - Submitting ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:55), which has no missing parents -2016-05-23 20:29:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=460301, maxMem=766075207 -2016-05-23 20:29:32 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-05-23 20:29:32 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=461525, maxMem=766075207 -2016-05-23 20:29:32 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.1 MB) -2016-05-23 20:29:32 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:53075 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:29:32 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:55) -2016-05-23 20:29:32 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 4 tasks -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 66, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:29:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:29:32 INFO Executor:59 - Running task 3.0 in stage 35.0 (TID 69) -2016-05-23 20:29:32 INFO Executor:59 - Finished task 3.0 in stage 35.0 (TID 69). 915 bytes result sent to driver -2016-05-23 20:29:32 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 67) -2016-05-23 20:29:32 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 67). 915 bytes result sent to driver -2016-05-23 20:29:32 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 68) -2016-05-23 20:29:32 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 68). 915 bytes result sent to driver -2016-05-23 20:29:33 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 66) -2016-05-23 20:29:33 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 66). 915 bytes result sent to driver -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 35.0 (TID 69) in 56 ms on localhost (1/4) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 67) in 60 ms on localhost (2/4) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 68) in 60 ms on localhost (3/4) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 66) in 64 ms on localhost (4/4) -2016-05-23 20:29:33 INFO DAGScheduler:59 - ResultStage 35 (foreach at StatisticsCorrLearning.scala:59) finished in 0.064 s -2016-05-23 20:29:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-23 20:29:33 INFO DAGScheduler:59 - Job 21 finished: foreach at StatisticsCorrLearning.scala:59, took 0.091179 s -2016-05-23 20:29:33 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:29:33 INFO DAGScheduler:59 - Got job 22 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:29:33 INFO DAGScheduler:59 - Final stage: ResultStage 36(first at RowMatrix.scala:65) -2016-05-23 20:29:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:29:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:33 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:29:33 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=462418, maxMem=766075207 -2016-05-23 20:29:33 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-05-23 20:29:33 INFO MemoryStore:59 - ensureFreeSpace(1522) called with curMem=464922, maxMem=766075207 -2016-05-23 20:29:33 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1522.0 B, free 730.1 MB) -2016-05-23 20:29:33 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:53075 (size: 1522.0 B, free: 730.5 MB) -2016-05-23 20:29:33 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 20:29:33 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:29:33 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 70) -2016-05-23 20:29:33 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 70). 1046 bytes result sent to driver -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 70) in 8 ms on localhost (1/1) -2016-05-23 20:29:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-23 20:29:33 INFO DAGScheduler:59 - ResultStage 36 (first at RowMatrix.scala:65) finished in 0.010 s -2016-05-23 20:29:33 INFO DAGScheduler:59 - Job 22 finished: first at RowMatrix.scala:65, took 0.027613 s -2016-05-23 20:29:33 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:29:33 INFO DAGScheduler:59 - Got job 23 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:29:33 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:29:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:29:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:33 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:29:33 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=466444, maxMem=766075207 -2016-05-23 20:29:33 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 20:29:33 INFO MemoryStore:59 - ensureFreeSpace(1932) called with curMem=469844, maxMem=766075207 -2016-05-23 20:29:33 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1932.0 B, free 730.1 MB) -2016-05-23 20:29:33 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:53075 (size: 1932.0 B, free: 730.5 MB) -2016-05-23 20:29:33 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:29:33 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 4 tasks -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:29:33 INFO Executor:59 - Running task 3.0 in stage 37.0 (TID 74) -2016-05-23 20:29:33 INFO Executor:59 - Finished task 3.0 in stage 37.0 (TID 74). 1231 bytes result sent to driver -2016-05-23 20:29:33 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 73) -2016-05-23 20:29:33 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 73). 1231 bytes result sent to driver -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 37.0 (TID 74) in 12 ms on localhost (1/4) -2016-05-23 20:29:33 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 72) -2016-05-23 20:29:33 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 72). 1231 bytes result sent to driver -2016-05-23 20:29:33 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-23 20:29:33 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 1231 bytes result sent to driver -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 73) in 35 ms on localhost (2/4) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 72) in 37 ms on localhost (3/4) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 40 ms on localhost (4/4) -2016-05-23 20:29:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-23 20:29:33 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at RowMatrix.scala:332) finished in 0.049 s -2016-05-23 20:29:33 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at RowMatrix.scala:332, took 0.077813 s -2016-05-23 20:29:33 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:29:33 INFO DAGScheduler:59 - Got job 24 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:29:33 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:29:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:29:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:33 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:29:33 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=471776, maxMem=766075207 -2016-05-23 20:29:33 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-23 20:29:33 INFO MemoryStore:59 - ensureFreeSpace(1916) called with curMem=475216, maxMem=766075207 -2016-05-23 20:29:33 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1916.0 B, free 730.1 MB) -2016-05-23 20:29:33 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:53075 (size: 1916.0 B, free: 730.5 MB) -2016-05-23 20:29:33 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:29:33 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 77, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 78, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:29:33 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-23 20:29:33 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 1109 bytes result sent to driver -2016-05-23 20:29:33 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-23 20:29:33 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 1109 bytes result sent to driver -2016-05-23 20:29:33 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 78) -2016-05-23 20:29:33 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 78). 1109 bytes result sent to driver -2016-05-23 20:29:33 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 77) -2016-05-23 20:29:33 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 77). 1109 bytes result sent to driver -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 77) in 106 ms on localhost (1/4) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 109 ms on localhost (2/4) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 108 ms on localhost (3/4) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 78) in 107 ms on localhost (4/4) -2016-05-23 20:29:33 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at RowMatrix.scala:124) finished in 0.124 s -2016-05-23 20:29:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-23 20:29:33 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at RowMatrix.scala:124, took 0.142491 s -2016-05-23 20:29:33 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:29:33 INFO DAGScheduler:59 - Got job 25 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:29:33 INFO DAGScheduler:59 - Final stage: ResultStage 39(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:29:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:29:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:33 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:29:33 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=477132, maxMem=766075207 -2016-05-23 20:29:33 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 20:29:33 INFO MemoryStore:59 - ensureFreeSpace(1851) called with curMem=480508, maxMem=766075207 -2016-05-23 20:29:33 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1851.0 B, free 730.1 MB) -2016-05-23 20:29:33 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:53075 (size: 1851.0 B, free: 730.5 MB) -2016-05-23 20:29:33 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:29:33 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 4 tasks -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 79, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 80, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 81, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 39.0 (TID 82, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:29:33 INFO Executor:59 - Running task 3.0 in stage 39.0 (TID 82) -2016-05-23 20:29:33 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 81) -2016-05-23 20:29:33 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 80) -2016-05-23 20:29:33 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 79) -2016-05-23 20:29:33 INFO Executor:59 - Finished task 3.0 in stage 39.0 (TID 82). 1334 bytes result sent to driver -2016-05-23 20:29:33 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 80). 1334 bytes result sent to driver -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 80) in 8 ms on localhost (1/4) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 39.0 (TID 82) in 7 ms on localhost (2/4) -2016-05-23 20:29:33 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 81). 1294 bytes result sent to driver -2016-05-23 20:29:33 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 79). 1294 bytes result sent to driver -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 79) in 10 ms on localhost (3/4) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 81) in 10 ms on localhost (4/4) -2016-05-23 20:29:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-23 20:29:33 INFO DAGScheduler:59 - ResultStage 39 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.012 s -2016-05-23 20:29:33 INFO DAGScheduler:59 - Job 25 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.048514 s -2016-05-23 20:29:33 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:29:33 INFO DAGScheduler:59 - Registering RDD 51 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:29:33 INFO DAGScheduler:59 - Got job 26 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:29:33 INFO DAGScheduler:59 - Final stage: ResultStage 41(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:29:33 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 40) -2016-05-23 20:29:33 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 40) -2016-05-23 20:29:33 INFO DAGScheduler:59 - Submitting ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:29:33 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=482359, maxMem=766075207 -2016-05-23 20:29:33 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-05-23 20:29:33 INFO MemoryStore:59 - ensureFreeSpace(2281) called with curMem=486399, maxMem=766075207 -2016-05-23 20:29:33 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 20:29:33 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:53075 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:29:33 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:29:33 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 4 tasks -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 83, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 84, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 85, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 40.0 (TID 86, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:29:33 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 85) -2016-05-23 20:29:33 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 84) -2016-05-23 20:29:33 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 83) -2016-05-23 20:29:33 INFO Executor:59 - Running task 3.0 in stage 40.0 (TID 86) -2016-05-23 20:29:33 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 84). 1161 bytes result sent to driver -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 84) in 30 ms on localhost (1/4) -2016-05-23 20:29:33 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 83). 1161 bytes result sent to driver -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 83) in 37 ms on localhost (2/4) -2016-05-23 20:29:33 INFO Executor:59 - Finished task 3.0 in stage 40.0 (TID 86). 1161 bytes result sent to driver -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 40.0 (TID 86) in 64 ms on localhost (3/4) -2016-05-23 20:29:33 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 85). 1161 bytes result sent to driver -2016-05-23 20:29:33 INFO DAGScheduler:59 - ShuffleMapStage 40 (flatMap at SpearmanCorrelation.scala:49) finished in 0.076 s -2016-05-23 20:29:33 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:29:33 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:29:33 INFO DAGScheduler:59 - waiting: Set(ResultStage 41) -2016-05-23 20:29:33 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:29:33 INFO DAGScheduler:59 - Missing parents for ResultStage 41: List() -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 85) in 74 ms on localhost (4/4) -2016-05-23 20:29:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-23 20:29:33 INFO DAGScheduler:59 - Submitting ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:29:33 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=488680, maxMem=766075207 -2016-05-23 20:29:33 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 20:29:33 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=491224, maxMem=766075207 -2016-05-23 20:29:33 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.1 MB) -2016-05-23 20:29:33 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:53075 (size: 1567.0 B, free: 730.5 MB) -2016-05-23 20:29:33 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:33 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:29:33 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 3 tasks -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 87, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:33 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 88) -2016-05-23 20:29:33 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 87) -2016-05-23 20:29:33 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 89) -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:29:33 INFO Executor:59 - Finished task 2.0 in stage 41.0 (TID 89). 1203 bytes result sent to driver -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:33 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 88). 1203 bytes result sent to driver -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 41.0 (TID 89) in 25 ms on localhost (1/3) -2016-05-23 20:29:33 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 87). 1203 bytes result sent to driver -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 88) in 27 ms on localhost (2/3) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 87) in 30 ms on localhost (3/3) -2016-05-23 20:29:33 INFO DAGScheduler:59 - ResultStage 41 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.030 s -2016-05-23 20:29:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-23 20:29:33 INFO DAGScheduler:59 - Job 26 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.132907 s -2016-05-23 20:29:33 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:29:33 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 168 bytes -2016-05-23 20:29:33 INFO DAGScheduler:59 - Registering RDD 56 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:29:33 INFO DAGScheduler:59 - Got job 27 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:29:33 INFO DAGScheduler:59 - Final stage: ResultStage 44(first at RowMatrix.scala:65) -2016-05-23 20:29:33 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 43) -2016-05-23 20:29:33 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 43) -2016-05-23 20:29:33 INFO DAGScheduler:59 - Submitting ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:29:33 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=492791, maxMem=766075207 -2016-05-23 20:29:33 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 20:29:33 INFO MemoryStore:59 - ensureFreeSpace(2305) called with curMem=497023, maxMem=766075207 -2016-05-23 20:29:33 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-23 20:29:33 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:53075 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:29:33 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:29:33 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 4 tasks -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 90, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 91, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 43.0 (TID 92, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 43.0 (TID 93, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:29:33 INFO Executor:59 - Running task 3.0 in stage 43.0 (TID 93) -2016-05-23 20:29:33 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 91) -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:29:33 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 90) -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:33 INFO Executor:59 - Running task 2.0 in stage 43.0 (TID 92) -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:33 INFO Executor:59 - Finished task 3.0 in stage 43.0 (TID 93). 1377 bytes result sent to driver -2016-05-23 20:29:33 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 91). 1377 bytes result sent to driver -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 91) in 60 ms on localhost (1/4) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 43.0 (TID 93) in 59 ms on localhost (2/4) -2016-05-23 20:29:33 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 90). 1377 bytes result sent to driver -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 90) in 70 ms on localhost (3/4) -2016-05-23 20:29:33 INFO Executor:59 - Finished task 2.0 in stage 43.0 (TID 92). 1377 bytes result sent to driver -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 43.0 (TID 92) in 99 ms on localhost (4/4) -2016-05-23 20:29:33 INFO DAGScheduler:59 - ShuffleMapStage 43 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.102 s -2016-05-23 20:29:33 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:29:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-23 20:29:33 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:29:33 INFO DAGScheduler:59 - waiting: Set(ResultStage 44) -2016-05-23 20:29:33 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:29:33 INFO DAGScheduler:59 - Missing parents for ResultStage 44: List() -2016-05-23 20:29:33 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:29:33 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=499328, maxMem=766075207 -2016-05-23 20:29:33 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 20:29:33 INFO MemoryStore:59 - ensureFreeSpace(2545) called with curMem=504224, maxMem=766075207 -2016-05-23 20:29:33 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 20:29:33 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:53075 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:29:33 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:29:33 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 1 tasks -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 94, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:33 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 94) -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:29:33 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 94). 1296 bytes result sent to driver -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 94) in 12 ms on localhost (1/1) -2016-05-23 20:29:33 INFO DAGScheduler:59 - ResultStage 44 (first at RowMatrix.scala:65) finished in 0.012 s -2016-05-23 20:29:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-23 20:29:33 INFO DAGScheduler:59 - Job 27 finished: first at RowMatrix.scala:65, took 0.151012 s -2016-05-23 20:29:33 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:29:33 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 168 bytes -2016-05-23 20:29:33 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 166 bytes -2016-05-23 20:29:33 INFO DAGScheduler:59 - Got job 28 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:29:33 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:29:33 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 46) -2016-05-23 20:29:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:33 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:29:33 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=506769, maxMem=766075207 -2016-05-23 20:29:33 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 20:29:33 INFO MemoryStore:59 - ensureFreeSpace(2914) called with curMem=512465, maxMem=766075207 -2016-05-23 20:29:33 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:29:33 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:53075 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:29:33 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:29:33 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 4 tasks -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 95, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 96, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 47.0 (TID 97, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 47.0 (TID 98, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:33 INFO Executor:59 - Running task 2.0 in stage 47.0 (TID 97) -2016-05-23 20:29:33 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 96) -2016-05-23 20:29:33 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 95) -2016-05-23 20:29:33 INFO Executor:59 - Running task 3.0 in stage 47.0 (TID 98) -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:33 INFO Executor:59 - Finished task 3.0 in stage 47.0 (TID 98). 1481 bytes result sent to driver -2016-05-23 20:29:33 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 96). 1481 bytes result sent to driver -2016-05-23 20:29:33 INFO Executor:59 - Finished task 2.0 in stage 47.0 (TID 97). 1481 bytes result sent to driver -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 47.0 (TID 98) in 24 ms on localhost (1/4) -2016-05-23 20:29:33 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 95). 1481 bytes result sent to driver -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 96) in 30 ms on localhost (2/4) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 47.0 (TID 97) in 30 ms on localhost (3/4) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 95) in 33 ms on localhost (4/4) -2016-05-23 20:29:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-23 20:29:33 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at RowMatrix.scala:332) finished in 0.034 s -2016-05-23 20:29:33 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at RowMatrix.scala:332, took 0.072881 s -2016-05-23 20:29:33 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:29:33 INFO DAGScheduler:59 - Got job 29 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:29:33 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:29:33 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 49) -2016-05-23 20:29:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:29:33 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:29:33 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=515379, maxMem=766075207 -2016-05-23 20:29:33 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 20:29:33 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=521115, maxMem=766075207 -2016-05-23 20:29:33 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:29:33 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:53075 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:29:33 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:29:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:29:33 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 4 tasks -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 50.0 (TID 101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 50.0 (TID 102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:29:33 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-23 20:29:33 INFO Executor:59 - Running task 3.0 in stage 50.0 (TID 102) -2016-05-23 20:29:33 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:29:33 INFO Executor:59 - Running task 2.0 in stage 50.0 (TID 101) -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:29:33 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 1359 bytes result sent to driver -2016-05-23 20:29:34 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-23 20:29:34 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:53075 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:29:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:29:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:29:34 INFO Executor:59 - Finished task 3.0 in stage 50.0 (TID 102). 1359 bytes result sent to driver -2016-05-23 20:29:34 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 1359 bytes result sent to driver -2016-05-23 20:29:34 INFO Executor:59 - Finished task 2.0 in stage 50.0 (TID 101). 1359 bytes result sent to driver -2016-05-23 20:29:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 213 ms on localhost (1/4) -2016-05-23 20:29:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 50.0 (TID 102) in 211 ms on localhost (2/4) -2016-05-23 20:29:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 215 ms on localhost (3/4) -2016-05-23 20:29:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 50.0 (TID 101) in 215 ms on localhost (4/4) -2016-05-23 20:29:34 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at RowMatrix.scala:124) finished in 0.217 s -2016-05-23 20:29:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-23 20:29:34 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-23 20:29:34 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at RowMatrix.scala:124, took 0.246199 s -2016-05-23 20:29:34 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:53075 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:29:34 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-23 20:29:34 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:53075 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:29:34 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-23 20:29:34 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:53075 in memory (size: 1567.0 B, free: 730.5 MB) -2016-05-23 20:29:34 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 20:29:34 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:53075 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 20:29:34 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 20:29:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 20:29:34 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:53075 in memory (size: 1851.0 B, free: 730.5 MB) -2016-05-23 20:29:34 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-23 20:29:34 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:53075 in memory (size: 1916.0 B, free: 730.5 MB) -2016-05-23 20:29:34 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-23 20:29:34 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:53075 in memory (size: 1932.0 B, free: 730.5 MB) -2016-05-23 20:29:34 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-23 20:29:34 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:53075 in memory (size: 1522.0 B, free: 730.5 MB) -2016-05-23 20:29:34 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-23 20:29:34 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:53075 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:29:34 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-23 20:29:34 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:53075 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:29:34 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-23 20:29:34 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:53075 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:29:34 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-23 20:29:34 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:53075 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:29:34 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-23 20:29:34 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:53075 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:29:34 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-23 20:29:34 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:53075 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:29:34 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-23 20:29:34 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-05-23 20:29:34 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 20:29:34 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:53075 in memory (size: 1568.0 B, free: 730.5 MB) -2016-05-23 20:29:34 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 20:29:34 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 20:29:34 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 20:29:34 INFO BlockManager:59 - BlockManager stopped -2016-05-23 20:29:34 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 20:29:34 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 20:29:34 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 20:29:34 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 20:29:34 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-8ecec389-faa7-409e-81d4-9d1a8ec11b95 -2016-05-23 20:30:34 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 20:30:35 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 20:30:35 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 20:30:35 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 20:30:35 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 20:30:37 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 20:30:37 INFO Remoting:74 - Starting remoting -2016-05-23 20:30:38 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53100] -2016-05-23 20:30:38 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53100. -2016-05-23 20:30:38 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 20:30:38 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 20:30:38 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-e1bca253-1708-4b3e-963e-50929312e17f -2016-05-23 20:30:38 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 20:30:38 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-240ae5af-ae49-4942-8828-b0ef399a4496\httpd-c1dcf0ff-f0fc-451f-85c7-4922b25da0de -2016-05-23 20:30:38 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 20:30:38 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:30:38 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53101 -2016-05-23 20:30:38 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53101. -2016-05-23 20:30:38 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 20:30:38 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:30:38 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 20:30:38 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 20:30:38 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 20:30:38 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 20:30:38 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 20:30:39 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53120. -2016-05-23 20:30:39 INFO NettyBlockTransferService:59 - Server created on 53120 -2016-05-23 20:30:39 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 20:30:39 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53120 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53120) -2016-05-23 20:30:39 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 20:30:40 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 20:30:40 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 20:30:40 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 20:30:40 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 20:30:40 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53120 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:30:40 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:20 -2016-05-23 20:30:40 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 20:30:40 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 20:30:40 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 20:30:40 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 20:30:40 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53120 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:30:40 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 20:30:42 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 20:30:42 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:30:42 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:27 -2016-05-23 20:30:42 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:27) with 2 output partitions -2016-05-23 20:30:42 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:27) -2016-05-23 20:30:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:30:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:42 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21), which has no missing parents -2016-05-23 20:30:42 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 20:30:42 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:30:42 INFO MemoryStore:59 - ensureFreeSpace(2047) called with curMem=346712, maxMem=766075207 -2016-05-23 20:30:42 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2047.0 B, free 730.3 MB) -2016-05-23 20:30:42 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53120 (size: 2047.0 B, free: 730.6 MB) -2016-05-23 20:30:42 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21) -2016-05-23 20:30:42 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 20:30:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:30:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:30:42 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 20:30:42 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 20:30:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:30:42 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 20:30:42 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 20:30:42 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 20:30:42 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 20:30:42 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 20:30:42 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 20:30:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:30:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 264 ms on localhost (1/2) -2016-05-23 20:30:43 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 20:30:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 231 ms on localhost (2/2) -2016-05-23 20:30:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 20:30:43 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:27) finished in 0.332 s -2016-05-23 20:30:43 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:27, took 0.600277 s -2016-05-23 20:30:43 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:30:43 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 20:30:43 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 20:30:43 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 20:30:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:30:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:43 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24), which has no missing parents -2016-05-23 20:30:43 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348759, maxMem=766075207 -2016-05-23 20:30:43 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:30:43 INFO MemoryStore:59 - ensureFreeSpace(2045) called with curMem=352271, maxMem=766075207 -2016-05-23 20:30:43 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2045.0 B, free 730.2 MB) -2016-05-23 20:30:43 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53120 (size: 2045.0 B, free: 730.6 MB) -2016-05-23 20:30:43 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24) -2016-05-23 20:30:43 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 20:30:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:30:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:30:43 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 20:30:43 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:30:43 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 20:30:43 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:30:43 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 20:30:43 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 20:30:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 54 ms on localhost (1/2) -2016-05-23 20:30:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 57 ms on localhost (2/2) -2016-05-23 20:30:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 20:30:43 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:29) finished in 0.059 s -2016-05-23 20:30:43 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:29, took 0.086907 s -2016-05-23 20:30:43 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:30:43 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:30:43 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 20:30:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:30:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:43 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:30:43 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354316, maxMem=766075207 -2016-05-23 20:30:43 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 20:30:43 INFO MemoryStore:59 - ensureFreeSpace(2473) called with curMem=358876, maxMem=766075207 -2016-05-23 20:30:43 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:30:43 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53120 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:30:43 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:43 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 20:30:43 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 20:30:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:30:43 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 20:30:43 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53120 in memory (size: 2045.0 B, free: 730.6 MB) -2016-05-23 20:30:43 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:30:43 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:30:43 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 20:30:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 243 ms on localhost (1/1) -2016-05-23 20:30:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 20:30:43 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.244 s -2016-05-23 20:30:43 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-23 20:30:43 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.313804 s -2016-05-23 20:30:43 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53120 in memory (size: 2047.0 B, free: 730.6 MB) -2016-05-23 20:30:43 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-23 20:30:44 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:30:44 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:30:44 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:30:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:30:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:44 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:30:44 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=350233, maxMem=766075207 -2016-05-23 20:30:44 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 20:30:44 INFO MemoryStore:59 - ensureFreeSpace(2859) called with curMem=355681, maxMem=766075207 -2016-05-23 20:30:44 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:30:44 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53120 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:30:44 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:30:44 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 20:30:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:30:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:30:44 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 20:30:44 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 20:30:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:30:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:30:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:30:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:30:44 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 20:30:44 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 20:30:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 72 ms on localhost (1/2) -2016-05-23 20:30:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 74 ms on localhost (2/2) -2016-05-23 20:30:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 20:30:44 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.077 s -2016-05-23 20:30:44 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 20:30:44 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 20:30:44 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.110369 s -2016-05-23 20:30:44 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:30:44 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:30:44 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:30:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:30:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:44 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:30:44 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=358540, maxMem=766075207 -2016-05-23 20:30:44 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 20:30:44 INFO MemoryStore:59 - ensureFreeSpace(2850) called with curMem=364036, maxMem=766075207 -2016-05-23 20:30:44 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:30:44 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53120 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:30:44 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:30:44 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 20:30:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:30:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:30:44 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 20:30:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:30:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:30:44 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 20:30:44 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 20:30:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 107 ms on localhost (1/2) -2016-05-23 20:30:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:30:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:30:44 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 20:30:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 130 ms on localhost (2/2) -2016-05-23 20:30:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 20:30:44 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.133 s -2016-05-23 20:30:44 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.180940 s -2016-05-23 20:30:45 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:30:45 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 20:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:45 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:30:45 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=366886, maxMem=766075207 -2016-05-23 20:30:45 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 20:30:45 INFO MemoryStore:59 - ensureFreeSpace(2743) called with curMem=372230, maxMem=766075207 -2016-05-23 20:30:45 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 20:30:45 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53120 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 20:30:45 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:30:45 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 20:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:30:45 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 20:30:45 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 20:30:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:30:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:30:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:30:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:30:45 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 20:30:45 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 20:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 47 ms on localhost (1/2) -2016-05-23 20:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 46 ms on localhost (2/2) -2016-05-23 20:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 20:30:45 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.048 s -2016-05-23 20:30:45 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.068584 s -2016-05-23 20:30:45 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:30:45 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:30:45 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 20:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:30:45 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 20:30:45 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 20:30:45 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:30:45 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=374973, maxMem=766075207 -2016-05-23 20:30:45 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 20:30:45 INFO MemoryStore:59 - ensureFreeSpace(3163) called with curMem=380901, maxMem=766075207 -2016-05-23 20:30:45 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 20:30:45 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53120 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:30:45 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:30:45 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 20:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:30:45 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 20:30:45 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 20:30:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:30:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:30:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:30:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:30:45 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 20:30:45 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 20:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 104 ms on localhost (1/2) -2016-05-23 20:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 100 ms on localhost (2/2) -2016-05-23 20:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 20:30:45 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.110 s -2016-05-23 20:30:45 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:30:45 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:30:45 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 20:30:45 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:30:45 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 20:30:45 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:30:45 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=384064, maxMem=766075207 -2016-05-23 20:30:45 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:30:45 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=386568, maxMem=766075207 -2016-05-23 20:30:45 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 20:30:45 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53120 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:30:45 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:45 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:30:45 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 20:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:45 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 20:30:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:30:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 8 ms -2016-05-23 20:30:45 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 20:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 90 ms on localhost (1/1) -2016-05-23 20:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 20:30:45 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.092 s -2016-05-23 20:30:45 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.271910 s -2016-05-23 20:30:45 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:30:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:30:45 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:30:45 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 20:30:45 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53120 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:30:45 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 20:30:45 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 20:30:45 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:30:45 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=384064, maxMem=766075207 -2016-05-23 20:30:45 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:30:45 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=388256, maxMem=766075207 -2016-05-23 20:30:45 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:30:45 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 20:30:45 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53120 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:30:45 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:45 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53120 in memory (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:30:45 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 20:30:45 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 20:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:30:45 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53120 in memory (size: 2.7 KB, free: 730.5 MB) -2016-05-23 20:30:45 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 20:30:45 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 20:30:45 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 20:30:45 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53120 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:30:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:30:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:30:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:30:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:45 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 20:30:45 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53120 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:30:45 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 20:30:45 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 20:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 57 ms on localhost (1/2) -2016-05-23 20:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 56 ms on localhost (2/2) -2016-05-23 20:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 20:30:45 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 20:30:45 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.059 s -2016-05-23 20:30:45 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:30:45 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:30:45 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 20:30:45 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:30:45 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 20:30:45 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:30:45 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=356707, maxMem=766075207 -2016-05-23 20:30:45 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 20:30:45 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=361563, maxMem=766075207 -2016-05-23 20:30:45 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:30:45 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53120 (size: 2.5 KB, free: 730.6 MB) -2016-05-23 20:30:45 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:45 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:30:45 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 20:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:45 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 20:30:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:30:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:45 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 20:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 28 ms on localhost (1/1) -2016-05-23 20:30:45 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.028 s -2016-05-23 20:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 20:30:45 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.176065 s -2016-05-23 20:30:45 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:30:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:30:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 20:30:45 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:30:45 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 20:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:45 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:30:45 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=364094, maxMem=766075207 -2016-05-23 20:30:45 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 20:30:45 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=369750, maxMem=766075207 -2016-05-23 20:30:45 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:30:45 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53120 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:30:45 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:30:45 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 20:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:45 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 20:30:45 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 20:30:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:30:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:30:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:30:45 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 20:30:45 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 20:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 23 ms on localhost (1/2) -2016-05-23 20:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 25 ms on localhost (2/2) -2016-05-23 20:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 20:30:45 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.026 s -2016-05-23 20:30:45 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.043880 s -2016-05-23 20:30:45 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:30:45 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:30:45 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 20:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:45 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:30:45 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=372658, maxMem=766075207 -2016-05-23 20:30:45 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:30:45 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=378354, maxMem=766075207 -2016-05-23 20:30:45 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:30:45 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53120 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:30:45 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:30:45 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 20:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:45 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 20:30:45 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 20:30:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:30:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:30:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:30:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:45 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 20:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 81 ms on localhost (1/2) -2016-05-23 20:30:45 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 20:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 88 ms on localhost (2/2) -2016-05-23 20:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 20:30:45 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.091 s -2016-05-23 20:30:45 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.117233 s -2016-05-23 20:30:45 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:43 -2016-05-23 20:30:45 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:43) with 4 output partitions -2016-05-23 20:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:43) -2016-05-23 20:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:45 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40), which has no missing parents -2016-05-23 20:30:46 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=381243, maxMem=766075207 -2016-05-23 20:30:46 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:30:46 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=382467, maxMem=766075207 -2016-05-23 20:30:46 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:30:46 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53120 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:30:46 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:46 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40) -2016-05-23 20:30:46 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 20:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:30:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:30:46 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:30:46 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 20:30:46 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 20:30:46 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 20:30:46 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 20:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 15 ms on localhost (1/4) -2016-05-23 20:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 28 ms on localhost (2/4) -2016-05-23 20:30:46 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 20:30:46 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 20:30:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 23 ms on localhost (3/4) -2016-05-23 20:30:46 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 20:30:46 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 20:30:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 41 ms on localhost (4/4) -2016-05-23 20:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 20:30:46 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:43) finished in 0.054 s -2016-05-23 20:30:46 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:43, took 0.076923 s -2016-05-23 20:30:46 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:45 -2016-05-23 20:30:46 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:45) with 4 output partitions -2016-05-23 20:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:45) -2016-05-23 20:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:46 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41), which has no missing parents -2016-05-23 20:30:46 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=383360, maxMem=766075207 -2016-05-23 20:30:46 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:30:46 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=384584, maxMem=766075207 -2016-05-23 20:30:46 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:30:46 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53120 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:30:46 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:46 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41) -2016-05-23 20:30:46 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 20:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:30:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:30:46 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:30:46 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 20:30:46 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 20:30:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 8 ms on localhost (1/4) -2016-05-23 20:30:46 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 20:30:46 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 20:30:46 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 20:30:46 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 20:30:46 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 20:30:46 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 20:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 89 ms on localhost (2/4) -2016-05-23 20:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 95 ms on localhost (3/4) -2016-05-23 20:30:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 93 ms on localhost (4/4) -2016-05-23 20:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 20:30:46 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:45) finished in 0.106 s -2016-05-23 20:30:46 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:45, took 0.224248 s -2016-05-23 20:30:46 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:30:46 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 20:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:46 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:30:46 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=385477, maxMem=766075207 -2016-05-23 20:30:46 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:30:46 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=387981, maxMem=766075207 -2016-05-23 20:30:46 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 20:30:46 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53120 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 20:30:46 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:46 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 20:30:46 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 20:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:30:46 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 20:30:46 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 20:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 54 ms on localhost (1/1) -2016-05-23 20:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 20:30:46 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.060 s -2016-05-23 20:30:46 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.102731 s -2016-05-23 20:30:46 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:30:46 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:46 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:30:46 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=389505, maxMem=766075207 -2016-05-23 20:30:46 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:30:46 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=392905, maxMem=766075207 -2016-05-23 20:30:46 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 20:30:46 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53120 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 20:30:46 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:46 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:30:46 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 20:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:30:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:30:46 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:30:46 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 20:30:46 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 20:30:46 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 20:30:46 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 20:30:46 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 20:30:46 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 20:30:46 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 20:30:46 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 20:30:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 474 ms on localhost (1/4) -2016-05-23 20:30:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 474 ms on localhost (2/4) -2016-05-23 20:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 480 ms on localhost (3/4) -2016-05-23 20:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 480 ms on localhost (4/4) -2016-05-23 20:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 20:30:47 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.574 s -2016-05-23 20:30:47 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.593963 s -2016-05-23 20:30:47 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:30:47 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:30:47 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:30:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:30:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:47 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:30:47 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=394839, maxMem=766075207 -2016-05-23 20:30:47 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 20:30:47 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=398279, maxMem=766075207 -2016-05-23 20:30:47 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 20:30:47 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53120 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 20:30:47 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:30:47 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 20:30:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:30:47 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 20:30:47 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 20:30:47 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 20:30:47 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 20:30:47 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 20:30:47 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 20:30:47 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 20:30:47 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 20:30:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 18 ms on localhost (1/4) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 25 ms on localhost (2/4) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 28 ms on localhost (3/4) -2016-05-23 20:30:47 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.032 s -2016-05-23 20:30:47 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.050177 s -2016-05-23 20:30:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 23 ms on localhost (4/4) -2016-05-23 20:30:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 20:30:47 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:30:47 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:30:47 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:30:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:30:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:47 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:30:47 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=400197, maxMem=766075207 -2016-05-23 20:30:47 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:30:47 INFO MemoryStore:59 - ensureFreeSpace(1860) called with curMem=403573, maxMem=766075207 -2016-05-23 20:30:47 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1860.0 B, free 730.2 MB) -2016-05-23 20:30:47 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53120 (size: 1860.0 B, free: 730.5 MB) -2016-05-23 20:30:47 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:30:47 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 20:30:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:30:47 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 20:30:47 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 20:30:47 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 20:30:47 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 20:30:47 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 20:30:47 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 20:30:47 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 20:30:47 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 20:30:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 41 ms on localhost (1/4) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 47 ms on localhost (2/4) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 46 ms on localhost (3/4) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 45 ms on localhost (4/4) -2016-05-23 20:30:47 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.049 s -2016-05-23 20:30:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 20:30:47 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.070507 s -2016-05-23 20:30:47 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:30:47 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:30:47 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:30:47 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:30:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 20:30:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 20:30:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:30:47 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=405433, maxMem=766075207 -2016-05-23 20:30:47 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 20:30:47 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=409473, maxMem=766075207 -2016-05-23 20:30:47 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:30:47 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:53120 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:30:47 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:30:47 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 20:30:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:30:47 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 20:30:47 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 20:30:47 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 20:30:47 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 20:30:47 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 20:30:47 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 20:30:47 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 20:30:47 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 20:30:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 148 ms on localhost (1/4) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 155 ms on localhost (2/4) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 153 ms on localhost (3/4) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 155 ms on localhost (4/4) -2016-05-23 20:30:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 20:30:47 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.173 s -2016-05-23 20:30:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:30:47 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:30:47 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 20:30:47 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:30:47 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 20:30:47 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:30:47 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=411755, maxMem=766075207 -2016-05-23 20:30:47 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:30:47 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=414299, maxMem=766075207 -2016-05-23 20:30:47 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 20:30:47 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:53120 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 20:30:47 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:47 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:30:47 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 20:30:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:47 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 20:30:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:47 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 20:30:47 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 20:30:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:47 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 20:30:47 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 76 ms on localhost (1/3) -2016-05-23 20:30:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:47 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 20:30:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 89 ms on localhost (2/3) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 93 ms on localhost (3/3) -2016-05-23 20:30:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 20:30:47 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.097 s -2016-05-23 20:30:47 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.311137 s -2016-05-23 20:30:47 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:30:47 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:30:47 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:30:47 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:30:47 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 20:30:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 20:30:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 20:30:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:30:47 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=415867, maxMem=766075207 -2016-05-23 20:30:47 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:30:47 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=420099, maxMem=766075207 -2016-05-23 20:30:47 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 20:30:47 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:53120 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:30:47 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:30:47 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 20:30:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:30:47 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 20:30:47 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 20:30:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:30:47 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 20:30:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:30:47 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 20:30:47 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 20:30:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 165 ms on localhost (1/4) -2016-05-23 20:30:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 168 ms on localhost (2/4) -2016-05-23 20:30:47 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 20:30:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:30:48 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 20:30:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 424 ms on localhost (3/4) -2016-05-23 20:30:48 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 20:30:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 577 ms on localhost (4/4) -2016-05-23 20:30:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 20:30:48 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.583 s -2016-05-23 20:30:48 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:30:48 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:30:48 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 20:30:48 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:30:48 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 20:30:48 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:30:48 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=422405, maxMem=766075207 -2016-05-23 20:30:48 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 20:30:48 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=427301, maxMem=766075207 -2016-05-23 20:30:48 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:30:48 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:53120 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:30:48 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:48 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:30:48 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 20:30:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:48 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 20:30:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:30:48 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 20:30:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 22 ms on localhost (1/1) -2016-05-23 20:30:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 20:30:48 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.023 s -2016-05-23 20:30:48 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.653958 s -2016-05-23 20:30:48 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:30:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:30:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 20:30:48 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:30:48 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:30:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 20:30:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:48 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:30:48 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=429843, maxMem=766075207 -2016-05-23 20:30:48 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:30:48 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=435539, maxMem=766075207 -2016-05-23 20:30:48 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:30:48 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:53120 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:30:48 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:30:48 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 20:30:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:48 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 20:30:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:30:48 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 20:30:48 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 20:30:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:48 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 20:30:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:48 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 20:30:48 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 20:30:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:48 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 20:30:48 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 20:30:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 152 ms on localhost (1/4) -2016-05-23 20:30:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 154 ms on localhost (2/4) -2016-05-23 20:30:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 157 ms on localhost (3/4) -2016-05-23 20:30:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 159 ms on localhost (4/4) -2016-05-23 20:30:48 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.159 s -2016-05-23 20:30:48 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.187531 s -2016-05-23 20:30:48 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:30:48 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:30:48 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:30:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 20:30:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:48 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:30:48 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=438448, maxMem=766075207 -2016-05-23 20:30:48 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:30:48 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=444184, maxMem=766075207 -2016-05-23 20:30:48 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:30:48 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:53120 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:30:48 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:30:48 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 20:30:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 20:30:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:48 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 20:30:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:48 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 20:30:48 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 20:30:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:48 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 20:30:48 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 20:30:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:48 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 20:30:48 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 20:30:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:48 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 20:30:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 73 ms on localhost (1/4) -2016-05-23 20:30:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 73 ms on localhost (2/4) -2016-05-23 20:30:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 71 ms on localhost (3/4) -2016-05-23 20:30:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 73 ms on localhost (4/4) -2016-05-23 20:30:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 20:30:48 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.077 s -2016-05-23 20:30:48 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.102835 s -2016-05-23 20:30:48 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:57 -2016-05-23 20:30:48 INFO DAGScheduler:59 - Got job 20 (foreach at StatisticsCorrLearning.scala:57) with 4 output partitions -2016-05-23 20:30:48 INFO DAGScheduler:59 - Final stage: ResultStage 34(foreach at StatisticsCorrLearning.scala:57) -2016-05-23 20:30:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:30:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:48 INFO DAGScheduler:59 - Submitting ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:54), which has no missing parents -2016-05-23 20:30:48 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=447076, maxMem=766075207 -2016-05-23 20:30:48 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:30:48 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=448300, maxMem=766075207 -2016-05-23 20:30:48 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:30:48 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:53120 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:30:48 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:54) -2016-05-23 20:30:48 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-05-23 20:30:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 62, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:30:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 63, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:30:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 64, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:30:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:30:48 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 62) -2016-05-23 20:30:48 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 62). 915 bytes result sent to driver -2016-05-23 20:30:48 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 63) -2016-05-23 20:30:48 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 63). 915 bytes result sent to driver -2016-05-23 20:30:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 62) in 32 ms on localhost (1/4) -2016-05-23 20:30:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 63) in 31 ms on localhost (2/4) -2016-05-23 20:30:48 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 64) -2016-05-23 20:30:48 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 64). 915 bytes result sent to driver -2016-05-23 20:30:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 64) in 40 ms on localhost (3/4) -2016-05-23 20:30:48 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 65) -2016-05-23 20:30:48 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 65). 915 bytes result sent to driver -2016-05-23 20:30:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 65) in 209 ms on localhost (4/4) -2016-05-23 20:30:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-23 20:30:48 INFO DAGScheduler:59 - ResultStage 34 (foreach at StatisticsCorrLearning.scala:57) finished in 0.215 s -2016-05-23 20:30:49 INFO DAGScheduler:59 - Job 20 finished: foreach at StatisticsCorrLearning.scala:57, took 0.470492 s -2016-05-23 20:30:49 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:59 -2016-05-23 20:30:49 INFO DAGScheduler:59 - Got job 21 (foreach at StatisticsCorrLearning.scala:59) with 4 output partitions -2016-05-23 20:30:49 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at StatisticsCorrLearning.scala:59) -2016-05-23 20:30:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:30:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:49 INFO DAGScheduler:59 - Submitting ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:55), which has no missing parents -2016-05-23 20:30:49 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=449193, maxMem=766075207 -2016-05-23 20:30:49 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:30:49 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=450417, maxMem=766075207 -2016-05-23 20:30:49 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:30:49 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:53120 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:30:49 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:49 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:55) -2016-05-23 20:30:49 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 4 tasks -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 66, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 3.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:30:49 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 68) -2016-05-23 20:30:49 INFO Executor:59 - Running task 3.0 in stage 35.0 (TID 69) -2016-05-23 20:30:49 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 68). 915 bytes result sent to driver -2016-05-23 20:30:49 INFO Executor:59 - Finished task 3.0 in stage 35.0 (TID 69). 915 bytes result sent to driver -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 68) in 8 ms on localhost (1/4) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 3.0 in stage 35.0 (TID 69) in 9 ms on localhost (2/4) -2016-05-23 20:30:49 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 66) -2016-05-23 20:30:49 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 66). 915 bytes result sent to driver -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 66) in 20 ms on localhost (3/4) -2016-05-23 20:30:49 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 67) -2016-05-23 20:30:49 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 67). 915 bytes result sent to driver -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 67) in 27 ms on localhost (4/4) -2016-05-23 20:30:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-23 20:30:49 INFO DAGScheduler:59 - ResultStage 35 (foreach at StatisticsCorrLearning.scala:59) finished in 0.031 s -2016-05-23 20:30:49 INFO DAGScheduler:59 - Job 21 finished: foreach at StatisticsCorrLearning.scala:59, took 0.049059 s -2016-05-23 20:30:49 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:30:49 INFO DAGScheduler:59 - Got job 22 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:30:49 INFO DAGScheduler:59 - Final stage: ResultStage 36(first at RowMatrix.scala:65) -2016-05-23 20:30:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:30:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:49 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:30:49 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=451310, maxMem=766075207 -2016-05-23 20:30:49 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:30:49 INFO MemoryStore:59 - ensureFreeSpace(1522) called with curMem=453814, maxMem=766075207 -2016-05-23 20:30:49 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1522.0 B, free 730.2 MB) -2016-05-23 20:30:49 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:53120 (size: 1522.0 B, free: 730.5 MB) -2016-05-23 20:30:49 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:49 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 20:30:49 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:30:49 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 70) -2016-05-23 20:30:49 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 70). 1046 bytes result sent to driver -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 70) in 54 ms on localhost (1/1) -2016-05-23 20:30:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-23 20:30:49 INFO DAGScheduler:59 - ResultStage 36 (first at RowMatrix.scala:65) finished in 0.056 s -2016-05-23 20:30:49 INFO DAGScheduler:59 - Job 22 finished: first at RowMatrix.scala:65, took 0.073167 s -2016-05-23 20:30:49 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:30:49 INFO DAGScheduler:59 - Got job 23 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:30:49 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:30:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:30:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:49 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:30:49 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=455336, maxMem=766075207 -2016-05-23 20:30:49 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 20:30:49 INFO MemoryStore:59 - ensureFreeSpace(1932) called with curMem=458736, maxMem=766075207 -2016-05-23 20:30:49 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1932.0 B, free 730.1 MB) -2016-05-23 20:30:49 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:53120 (size: 1932.0 B, free: 730.5 MB) -2016-05-23 20:30:49 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:49 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:30:49 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 4 tasks -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 3.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:30:49 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 73) -2016-05-23 20:30:49 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 73). 1231 bytes result sent to driver -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 73) in 18 ms on localhost (1/4) -2016-05-23 20:30:49 INFO Executor:59 - Running task 3.0 in stage 37.0 (TID 74) -2016-05-23 20:30:49 INFO Executor:59 - Finished task 3.0 in stage 37.0 (TID 74). 1231 bytes result sent to driver -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 3.0 in stage 37.0 (TID 74) in 16 ms on localhost (2/4) -2016-05-23 20:30:49 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-23 20:30:49 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 1231 bytes result sent to driver -2016-05-23 20:30:49 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 72) -2016-05-23 20:30:49 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 72). 1231 bytes result sent to driver -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 51 ms on localhost (3/4) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 72) in 46 ms on localhost (4/4) -2016-05-23 20:30:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-23 20:30:49 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at RowMatrix.scala:332) finished in 0.056 s -2016-05-23 20:30:49 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at RowMatrix.scala:332, took 0.079656 s -2016-05-23 20:30:49 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:30:49 INFO DAGScheduler:59 - Got job 24 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:30:49 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:30:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:30:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:49 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:30:49 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=460668, maxMem=766075207 -2016-05-23 20:30:49 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-23 20:30:49 INFO MemoryStore:59 - ensureFreeSpace(1916) called with curMem=464108, maxMem=766075207 -2016-05-23 20:30:49 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1916.0 B, free 730.1 MB) -2016-05-23 20:30:49 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:53120 (size: 1916.0 B, free: 730.5 MB) -2016-05-23 20:30:49 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:49 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:30:49 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 77, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 78, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:30:49 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 77) -2016-05-23 20:30:49 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 77). 1109 bytes result sent to driver -2016-05-23 20:30:49 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 78) -2016-05-23 20:30:49 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 78). 1109 bytes result sent to driver -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 77) in 13 ms on localhost (1/4) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 78) in 13 ms on localhost (2/4) -2016-05-23 20:30:49 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-23 20:30:49 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 1109 bytes result sent to driver -2016-05-23 20:30:49 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-23 20:30:49 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 1109 bytes result sent to driver -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 117 ms on localhost (3/4) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 117 ms on localhost (4/4) -2016-05-23 20:30:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-23 20:30:49 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at RowMatrix.scala:124) finished in 0.122 s -2016-05-23 20:30:49 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at RowMatrix.scala:124, took 0.174452 s -2016-05-23 20:30:49 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:30:49 INFO DAGScheduler:59 - Got job 25 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:30:49 INFO DAGScheduler:59 - Final stage: ResultStage 39(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:30:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:30:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:49 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:30:49 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=466024, maxMem=766075207 -2016-05-23 20:30:49 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 20:30:49 INFO MemoryStore:59 - ensureFreeSpace(1851) called with curMem=469400, maxMem=766075207 -2016-05-23 20:30:49 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1851.0 B, free 730.1 MB) -2016-05-23 20:30:49 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:53120 (size: 1851.0 B, free: 730.5 MB) -2016-05-23 20:30:49 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:49 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:30:49 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 4 tasks -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 79, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 80, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 81, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 3.0 in stage 39.0 (TID 82, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:30:49 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 81) -2016-05-23 20:30:49 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 81). 1294 bytes result sent to driver -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 81) in 8 ms on localhost (1/4) -2016-05-23 20:30:49 INFO Executor:59 - Running task 3.0 in stage 39.0 (TID 82) -2016-05-23 20:30:49 INFO Executor:59 - Finished task 3.0 in stage 39.0 (TID 82). 1334 bytes result sent to driver -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 3.0 in stage 39.0 (TID 82) in 13 ms on localhost (2/4) -2016-05-23 20:30:49 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 79) -2016-05-23 20:30:49 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 79). 1294 bytes result sent to driver -2016-05-23 20:30:49 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 80) -2016-05-23 20:30:49 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 80). 1334 bytes result sent to driver -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 79) in 32 ms on localhost (3/4) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 80) in 26 ms on localhost (4/4) -2016-05-23 20:30:49 INFO DAGScheduler:59 - ResultStage 39 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.034 s -2016-05-23 20:30:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-23 20:30:49 INFO DAGScheduler:59 - Job 25 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.053682 s -2016-05-23 20:30:49 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:30:49 INFO DAGScheduler:59 - Registering RDD 51 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:30:49 INFO DAGScheduler:59 - Got job 26 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:30:49 INFO DAGScheduler:59 - Final stage: ResultStage 41(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:30:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 40) -2016-05-23 20:30:49 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 40) -2016-05-23 20:30:49 INFO DAGScheduler:59 - Submitting ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:30:49 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=471251, maxMem=766075207 -2016-05-23 20:30:49 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-05-23 20:30:49 INFO MemoryStore:59 - ensureFreeSpace(2281) called with curMem=475291, maxMem=766075207 -2016-05-23 20:30:49 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 20:30:49 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:53120 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:30:49 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:49 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:30:49 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 4 tasks -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 83, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 84, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 85, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 3.0 in stage 40.0 (TID 86, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:30:49 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 85) -2016-05-23 20:30:49 INFO Executor:59 - Running task 3.0 in stage 40.0 (TID 86) -2016-05-23 20:30:49 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 85). 1161 bytes result sent to driver -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 85) in 62 ms on localhost (1/4) -2016-05-23 20:30:49 INFO Executor:59 - Finished task 3.0 in stage 40.0 (TID 86). 1161 bytes result sent to driver -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 3.0 in stage 40.0 (TID 86) in 65 ms on localhost (2/4) -2016-05-23 20:30:49 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 83) -2016-05-23 20:30:49 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 84) -2016-05-23 20:30:49 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 83). 1161 bytes result sent to driver -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 83) in 217 ms on localhost (3/4) -2016-05-23 20:30:49 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 84). 1161 bytes result sent to driver -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 84) in 281 ms on localhost (4/4) -2016-05-23 20:30:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-23 20:30:49 INFO DAGScheduler:59 - ShuffleMapStage 40 (flatMap at SpearmanCorrelation.scala:49) finished in 0.279 s -2016-05-23 20:30:49 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:30:49 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:30:49 INFO DAGScheduler:59 - waiting: Set(ResultStage 41) -2016-05-23 20:30:49 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:30:49 INFO DAGScheduler:59 - Missing parents for ResultStage 41: List() -2016-05-23 20:30:49 INFO DAGScheduler:59 - Submitting ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:30:49 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=477572, maxMem=766075207 -2016-05-23 20:30:49 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 20:30:49 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=480116, maxMem=766075207 -2016-05-23 20:30:49 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.1 MB) -2016-05-23 20:30:49 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:53120 (size: 1567.0 B, free: 730.5 MB) -2016-05-23 20:30:49 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:49 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:30:49 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 3 tasks -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 87, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:49 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 89) -2016-05-23 20:30:49 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:49 INFO Executor:59 - Finished task 2.0 in stage 41.0 (TID 89). 1203 bytes result sent to driver -2016-05-23 20:30:49 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 88) -2016-05-23 20:30:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 41.0 (TID 89) in 23 ms on localhost (1/3) -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:30:50 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 87) -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:50 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 87). 1203 bytes result sent to driver -2016-05-23 20:30:50 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 88). 1203 bytes result sent to driver -2016-05-23 20:30:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 87) in 90 ms on localhost (2/3) -2016-05-23 20:30:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 88) in 91 ms on localhost (3/3) -2016-05-23 20:30:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-23 20:30:50 INFO DAGScheduler:59 - ResultStage 41 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.093 s -2016-05-23 20:30:50 INFO DAGScheduler:59 - Job 26 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.409097 s -2016-05-23 20:30:50 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:30:50 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 168 bytes -2016-05-23 20:30:50 INFO DAGScheduler:59 - Registering RDD 56 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:30:50 INFO DAGScheduler:59 - Got job 27 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:30:50 INFO DAGScheduler:59 - Final stage: ResultStage 44(first at RowMatrix.scala:65) -2016-05-23 20:30:50 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 43) -2016-05-23 20:30:50 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 43) -2016-05-23 20:30:50 INFO DAGScheduler:59 - Submitting ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:30:50 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=481683, maxMem=766075207 -2016-05-23 20:30:50 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 20:30:50 INFO MemoryStore:59 - ensureFreeSpace(2305) called with curMem=485915, maxMem=766075207 -2016-05-23 20:30:50 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-23 20:30:50 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:53120 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:30:50 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:50 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:30:50 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 4 tasks -2016-05-23 20:30:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 90, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:30:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 91, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:30:50 INFO TaskSetManager:59 - Starting task 2.0 in stage 43.0 (TID 92, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:30:50 INFO TaskSetManager:59 - Starting task 3.0 in stage 43.0 (TID 93, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:30:50 INFO Executor:59 - Running task 2.0 in stage 43.0 (TID 92) -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:30:50 INFO Executor:59 - Finished task 2.0 in stage 43.0 (TID 92). 1377 bytes result sent to driver -2016-05-23 20:30:50 INFO Executor:59 - Running task 3.0 in stage 43.0 (TID 93) -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:30:50 INFO Executor:59 - Finished task 3.0 in stage 43.0 (TID 93). 1377 bytes result sent to driver -2016-05-23 20:30:50 INFO TaskSetManager:59 - Finished task 2.0 in stage 43.0 (TID 92) in 99 ms on localhost (1/4) -2016-05-23 20:30:50 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 90) -2016-05-23 20:30:50 INFO TaskSetManager:59 - Finished task 3.0 in stage 43.0 (TID 93) in 98 ms on localhost (2/4) -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:30:50 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 91) -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:50 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 90). 1377 bytes result sent to driver -2016-05-23 20:30:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 90) in 183 ms on localhost (3/4) -2016-05-23 20:30:50 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 91). 1377 bytes result sent to driver -2016-05-23 20:30:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 91) in 303 ms on localhost (4/4) -2016-05-23 20:30:50 INFO DAGScheduler:59 - ShuffleMapStage 43 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.306 s -2016-05-23 20:30:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-23 20:30:50 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:30:50 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:30:50 INFO DAGScheduler:59 - waiting: Set(ResultStage 44) -2016-05-23 20:30:50 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:30:50 INFO DAGScheduler:59 - Missing parents for ResultStage 44: List() -2016-05-23 20:30:50 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:30:50 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=488220, maxMem=766075207 -2016-05-23 20:30:50 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 20:30:50 INFO MemoryStore:59 - ensureFreeSpace(2545) called with curMem=493116, maxMem=766075207 -2016-05-23 20:30:50 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 20:30:50 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:53120 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:30:50 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:30:50 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 1 tasks -2016-05-23 20:30:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 94, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:50 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 94) -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:30:50 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 94). 1296 bytes result sent to driver -2016-05-23 20:30:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 94) in 15 ms on localhost (1/1) -2016-05-23 20:30:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-23 20:30:50 INFO DAGScheduler:59 - ResultStage 44 (first at RowMatrix.scala:65) finished in 0.017 s -2016-05-23 20:30:50 INFO DAGScheduler:59 - Job 27 finished: first at RowMatrix.scala:65, took 0.353853 s -2016-05-23 20:30:50 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:30:50 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 168 bytes -2016-05-23 20:30:50 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 166 bytes -2016-05-23 20:30:50 INFO DAGScheduler:59 - Got job 28 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:30:50 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:30:50 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 46) -2016-05-23 20:30:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:50 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:30:50 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=495661, maxMem=766075207 -2016-05-23 20:30:50 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 20:30:50 INFO MemoryStore:59 - ensureFreeSpace(2914) called with curMem=501357, maxMem=766075207 -2016-05-23 20:30:50 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:30:50 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:53120 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:30:50 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:50 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:30:50 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 4 tasks -2016-05-23 20:30:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 95, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 96, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:50 INFO TaskSetManager:59 - Starting task 2.0 in stage 47.0 (TID 97, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:50 INFO TaskSetManager:59 - Starting task 3.0 in stage 47.0 (TID 98, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:50 INFO Executor:59 - Running task 3.0 in stage 47.0 (TID 98) -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:50 INFO Executor:59 - Running task 2.0 in stage 47.0 (TID 97) -2016-05-23 20:30:50 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 95) -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:50 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 96) -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:50 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 96). 1481 bytes result sent to driver -2016-05-23 20:30:50 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 95). 1481 bytes result sent to driver -2016-05-23 20:30:50 INFO Executor:59 - Finished task 3.0 in stage 47.0 (TID 98). 1481 bytes result sent to driver -2016-05-23 20:30:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 96) in 35 ms on localhost (1/4) -2016-05-23 20:30:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 95) in 37 ms on localhost (2/4) -2016-05-23 20:30:50 INFO TaskSetManager:59 - Finished task 3.0 in stage 47.0 (TID 98) in 37 ms on localhost (3/4) -2016-05-23 20:30:50 INFO Executor:59 - Finished task 2.0 in stage 47.0 (TID 97). 1481 bytes result sent to driver -2016-05-23 20:30:50 INFO TaskSetManager:59 - Finished task 2.0 in stage 47.0 (TID 97) in 42 ms on localhost (4/4) -2016-05-23 20:30:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-23 20:30:50 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at RowMatrix.scala:332) finished in 0.046 s -2016-05-23 20:30:50 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at RowMatrix.scala:332, took 0.074075 s -2016-05-23 20:30:50 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:30:50 INFO DAGScheduler:59 - Got job 29 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:30:50 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:30:50 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 49) -2016-05-23 20:30:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:30:50 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:30:50 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=504271, maxMem=766075207 -2016-05-23 20:30:50 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 20:30:50 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=510007, maxMem=766075207 -2016-05-23 20:30:50 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:30:50 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:53120 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:30:50 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:30:50 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:30:50 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 4 tasks -2016-05-23 20:30:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:50 INFO TaskSetManager:59 - Starting task 2.0 in stage 50.0 (TID 101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:50 INFO TaskSetManager:59 - Starting task 3.0 in stage 50.0 (TID 102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:30:50 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-23 20:30:50 INFO Executor:59 - Running task 3.0 in stage 50.0 (TID 102) -2016-05-23 20:30:50 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-23 20:30:50 INFO Executor:59 - Running task 2.0 in stage 50.0 (TID 101) -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:30:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:30:50 INFO Executor:59 - Finished task 3.0 in stage 50.0 (TID 102). 1359 bytes result sent to driver -2016-05-23 20:30:50 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 1359 bytes result sent to driver -2016-05-23 20:30:50 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 1359 bytes result sent to driver -2016-05-23 20:30:50 INFO TaskSetManager:59 - Finished task 3.0 in stage 50.0 (TID 102) in 19 ms on localhost (1/4) -2016-05-23 20:30:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 22 ms on localhost (2/4) -2016-05-23 20:30:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 26 ms on localhost (3/4) -2016-05-23 20:30:50 INFO Executor:59 - Finished task 2.0 in stage 50.0 (TID 101). 1359 bytes result sent to driver -2016-05-23 20:30:50 INFO TaskSetManager:59 - Finished task 2.0 in stage 50.0 (TID 101) in 130 ms on localhost (4/4) -2016-05-23 20:30:50 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at RowMatrix.scala:124) finished in 0.133 s -2016-05-23 20:30:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-23 20:30:50 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at RowMatrix.scala:124, took 0.153872 s -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 20:30:50 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:53120 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 20:30:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 20:30:50 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-23 20:30:50 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:53120 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:30:50 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-23 20:30:50 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:53120 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:30:50 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-23 20:30:50 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:53120 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:30:50 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-23 20:30:50 INFO ContextCleaner:59 - Cleaned shuffle 5 -2016-05-23 20:30:50 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:53120 in memory (size: 1567.0 B, free: 730.5 MB) -2016-05-23 20:30:51 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-23 20:30:51 WARN QueuedThreadPool:145 - 7 threads could not be stopped -2016-05-23 20:30:51 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 20:30:51 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:53120 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:30:51 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 20:30:51 WARN QueuedThreadPool:145 - 4 threads could not be stopped -2016-05-23 20:30:51 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 20:30:51 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 20:30:51 INFO BlockManager:59 - BlockManager stopped -2016-05-23 20:30:51 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 20:30:51 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 20:30:51 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 20:30:51 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 20:30:51 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-240ae5af-ae49-4942-8828-b0ef399a4496 -2016-05-23 20:30:51 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 20:33:32 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 20:33:33 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 20:33:33 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 20:33:33 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 20:33:33 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 20:33:34 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 20:33:34 INFO Remoting:74 - Starting remoting -2016-05-23 20:33:34 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53158] -2016-05-23 20:33:34 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53158. -2016-05-23 20:33:34 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 20:33:34 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 20:33:34 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-36f52436-e398-484c-8f4b-78ba147f0bee -2016-05-23 20:33:34 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 20:33:34 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-ba3defed-88f9-4535-8259-486d1f84e95e\httpd-e19f0cb1-1751-4678-8998-b982bdde8ed3 -2016-05-23 20:33:34 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 20:33:34 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:33:34 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53159 -2016-05-23 20:33:34 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53159. -2016-05-23 20:33:34 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 20:33:34 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:33:35 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 20:33:35 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 20:33:35 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 20:33:35 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 20:33:35 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 20:33:35 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53178. -2016-05-23 20:33:35 INFO NettyBlockTransferService:59 - Server created on 53178 -2016-05-23 20:33:35 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 20:33:35 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53178 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53178) -2016-05-23 20:33:35 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 20:33:36 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 20:33:36 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 20:33:36 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 20:33:36 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 20:33:36 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53178 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:33:36 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:20 -2016-05-23 20:33:36 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 20:33:36 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 20:33:36 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 20:33:36 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 20:33:36 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53178 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:33:36 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 20:33:37 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 20:33:38 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:33:38 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:27 -2016-05-23 20:33:38 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:27) with 2 output partitions -2016-05-23 20:33:38 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:27) -2016-05-23 20:33:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:33:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:38 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21), which has no missing parents -2016-05-23 20:33:38 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 20:33:38 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:33:38 INFO MemoryStore:59 - ensureFreeSpace(2047) called with curMem=346712, maxMem=766075207 -2016-05-23 20:33:38 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2047.0 B, free 730.3 MB) -2016-05-23 20:33:38 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53178 (size: 2047.0 B, free: 730.6 MB) -2016-05-23 20:33:38 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21) -2016-05-23 20:33:38 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 20:33:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:33:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:33:38 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 20:33:38 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 20:33:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:33:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:33:38 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 20:33:38 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 20:33:38 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 20:33:38 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 20:33:38 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 20:33:38 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 20:33:38 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 20:33:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 113 ms on localhost (1/2) -2016-05-23 20:33:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 137 ms on localhost (2/2) -2016-05-23 20:33:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 20:33:38 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:27) finished in 0.155 s -2016-05-23 20:33:38 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:27, took 0.284966 s -2016-05-23 20:33:38 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:33:38 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 20:33:38 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 20:33:38 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 20:33:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:33:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:38 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24), which has no missing parents -2016-05-23 20:33:38 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348759, maxMem=766075207 -2016-05-23 20:33:38 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:33:38 INFO MemoryStore:59 - ensureFreeSpace(2045) called with curMem=352271, maxMem=766075207 -2016-05-23 20:33:38 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2045.0 B, free 730.2 MB) -2016-05-23 20:33:38 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53178 (size: 2045.0 B, free: 730.6 MB) -2016-05-23 20:33:38 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24) -2016-05-23 20:33:38 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 20:33:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:33:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:33:38 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 20:33:38 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 20:33:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:33:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:33:38 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 20:33:38 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 20:33:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 23 ms on localhost (1/2) -2016-05-23 20:33:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 28 ms on localhost (2/2) -2016-05-23 20:33:38 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:29) finished in 0.028 s -2016-05-23 20:33:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 20:33:38 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:29, took 0.048488 s -2016-05-23 20:33:38 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:33:38 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:33:38 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 20:33:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:33:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:38 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:33:38 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354316, maxMem=766075207 -2016-05-23 20:33:38 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 20:33:38 INFO MemoryStore:59 - ensureFreeSpace(2473) called with curMem=358876, maxMem=766075207 -2016-05-23 20:33:38 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:33:38 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53178 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:33:38 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:38 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 20:33:38 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 20:33:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:33:38 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 20:33:38 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53178 in memory (size: 2045.0 B, free: 730.6 MB) -2016-05-23 20:33:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:33:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:33:38 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53178 in memory (size: 2047.0 B, free: 730.6 MB) -2016-05-23 20:33:38 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 20:33:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 34 ms on localhost (1/1) -2016-05-23 20:33:38 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.034 s -2016-05-23 20:33:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 20:33:38 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.071057 s -2016-05-23 20:33:39 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:33:39 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:33:39 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:33:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:33:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:39 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:33:39 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=350233, maxMem=766075207 -2016-05-23 20:33:39 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 20:33:39 INFO MemoryStore:59 - ensureFreeSpace(2859) called with curMem=355681, maxMem=766075207 -2016-05-23 20:33:39 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:33:39 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53178 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:33:39 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:33:39 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 20:33:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:33:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:33:39 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 20:33:39 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 20:33:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:33:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:33:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:33:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:33:39 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 20:33:39 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 20:33:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 47 ms on localhost (1/2) -2016-05-23 20:33:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 47 ms on localhost (2/2) -2016-05-23 20:33:39 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.049 s -2016-05-23 20:33:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 20:33:39 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 20:33:39 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 20:33:39 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.080719 s -2016-05-23 20:33:39 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:33:39 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:33:39 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:33:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:33:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:39 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:33:39 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=358540, maxMem=766075207 -2016-05-23 20:33:39 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 20:33:39 INFO MemoryStore:59 - ensureFreeSpace(2850) called with curMem=364036, maxMem=766075207 -2016-05-23 20:33:39 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:33:39 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53178 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:33:39 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:33:39 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 20:33:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:33:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:33:39 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 20:33:39 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 20:33:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:33:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:33:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:33:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:33:39 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 20:33:39 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 20:33:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 35 ms on localhost (1/2) -2016-05-23 20:33:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 37 ms on localhost (2/2) -2016-05-23 20:33:39 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.037 s -2016-05-23 20:33:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 20:33:39 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.063758 s -2016-05-23 20:33:39 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:33:39 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 20:33:39 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:33:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:33:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:39 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:33:39 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=366886, maxMem=766075207 -2016-05-23 20:33:39 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 20:33:39 INFO MemoryStore:59 - ensureFreeSpace(2743) called with curMem=372230, maxMem=766075207 -2016-05-23 20:33:39 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 20:33:39 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53178 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 20:33:39 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:33:39 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 20:33:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:33:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:33:39 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 20:33:39 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 20:33:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:33:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:33:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:33:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:33:39 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 20:33:39 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 20:33:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 37 ms on localhost (1/2) -2016-05-23 20:33:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 40 ms on localhost (2/2) -2016-05-23 20:33:39 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.041 s -2016-05-23 20:33:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 20:33:39 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.058749 s -2016-05-23 20:33:39 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:33:39 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:33:39 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 20:33:39 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:33:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 20:33:39 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 20:33:39 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:33:39 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=374973, maxMem=766075207 -2016-05-23 20:33:39 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 20:33:39 INFO MemoryStore:59 - ensureFreeSpace(3163) called with curMem=380901, maxMem=766075207 -2016-05-23 20:33:39 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 20:33:39 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53178 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:33:39 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:33:39 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 20:33:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:33:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:33:39 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 20:33:39 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 20:33:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:33:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:33:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:33:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:33:39 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 20:33:39 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 20:33:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 80 ms on localhost (1/2) -2016-05-23 20:33:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 83 ms on localhost (2/2) -2016-05-23 20:33:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 20:33:39 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.084 s -2016-05-23 20:33:39 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:33:39 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:33:39 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 20:33:39 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:33:39 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 20:33:39 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:33:39 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=384064, maxMem=766075207 -2016-05-23 20:33:39 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:33:39 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=386568, maxMem=766075207 -2016-05-23 20:33:39 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 20:33:39 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53178 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:33:39 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:39 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:33:39 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 20:33:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:39 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 20:33:39 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:33:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-23 20:33:39 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 20:33:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 57 ms on localhost (1/1) -2016-05-23 20:33:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 20:33:39 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.058 s -2016-05-23 20:33:39 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.190742 s -2016-05-23 20:33:39 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53178 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:33:39 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 20:33:39 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53178 in memory (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:33:39 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 20:33:39 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:33:39 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53178 in memory (size: 2.7 KB, free: 730.6 MB) -2016-05-23 20:33:39 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 20:33:39 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53178 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:33:39 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:33:39 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 20:33:39 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53178 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:33:39 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 20:33:39 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:33:39 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:33:39 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 20:33:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 20:33:39 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 20:33:39 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:33:39 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=350233, maxMem=766075207 -2016-05-23 20:33:39 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:33:39 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=354425, maxMem=766075207 -2016-05-23 20:33:39 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:33:39 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53178 (size: 2.2 KB, free: 730.6 MB) -2016-05-23 20:33:39 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:33:39 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 20:33:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:33:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:33:39 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 20:33:39 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 20:33:39 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:33:39 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:33:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:39 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 20:33:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 35 ms on localhost (1/2) -2016-05-23 20:33:39 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 20:33:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 38 ms on localhost (2/2) -2016-05-23 20:33:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 20:33:39 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.040 s -2016-05-23 20:33:39 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:33:39 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:33:39 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 20:33:39 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:33:39 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 20:33:39 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:33:39 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=356707, maxMem=766075207 -2016-05-23 20:33:39 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=361563, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53178 (size: 2.5 KB, free: 730.6 MB) -2016-05-23 20:33:40 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:40 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:40 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 24 ms on localhost (1/1) -2016-05-23 20:33:40 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.024 s -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 20:33:40 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.102088 s -2016-05-23 20:33:40 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:33:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:33:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 20:33:40 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:33:40 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=364094, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=369750, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53178 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:33:40 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:40 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 20:33:40 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:33:40 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 19 ms on localhost (1/2) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 23 ms on localhost (2/2) -2016-05-23 20:33:40 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.023 s -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 20:33:40 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.039037 s -2016-05-23 20:33:40 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:33:40 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=372658, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=378354, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53178 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:33:40 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:40 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 20:33:40 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:40 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 22 ms on localhost (1/2) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 32 ms on localhost (2/2) -2016-05-23 20:33:40 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.032 s -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 20:33:40 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.052615 s -2016-05-23 20:33:40 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:43 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:43) with 4 output partitions -2016-05-23 20:33:40 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:43) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:33:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40), which has no missing parents -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=381243, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=382467, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:33:40 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53178 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:33:40 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:33:40 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 20:33:40 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 20:33:40 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 20:33:40 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 20:33:40 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 10 ms on localhost (1/4) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 10 ms on localhost (2/4) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 10 ms on localhost (3/4) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 20:33:40 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:43) finished in 0.023 s -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 23 ms on localhost (4/4) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 20:33:40 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:43, took 0.039082 s -2016-05-23 20:33:40 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:45 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:45) with 4 output partitions -2016-05-23 20:33:40 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:45) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:33:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41), which has no missing parents -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=383360, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=384584, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:33:40 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53178 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:33:40 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:33:40 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 20:33:40 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 20:33:40 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 20:33:40 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 20:33:40 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 20:33:40 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 20:33:40 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 13 ms on localhost (1/4) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 10 ms on localhost (2/4) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 13 ms on localhost (3/4) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 12 ms on localhost (4/4) -2016-05-23 20:33:40 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:45) finished in 0.014 s -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 20:33:40 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:45, took 0.024267 s -2016-05-23 20:33:40 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:33:40 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:33:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=385477, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=387981, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 20:33:40 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53178 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 20:33:40 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:33:40 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 20:33:40 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.009 s -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 8 ms on localhost (1/1) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 20:33:40 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.022234 s -2016-05-23 20:33:40 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:33:40 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:33:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=389505, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=392905, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 20:33:40 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53178 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 20:33:40 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:33:40 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 20:33:40 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 20:33:40 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 20:33:40 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 20:33:40 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 20:33:40 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 11 ms on localhost (1/4) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 14 ms on localhost (2/4) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 18 ms on localhost (3/4) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 18 ms on localhost (4/4) -2016-05-23 20:33:40 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.020 s -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 20:33:40 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.033205 s -2016-05-23 20:33:40 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:33:40 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:33:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=394839, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=398279, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 20:33:40 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53178 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 20:33:40 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:33:40 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 20:33:40 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 20:33:40 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 20:33:40 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 20:33:40 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 20:33:40 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 11 ms on localhost (1/4) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 16 ms on localhost (2/4) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 12 ms on localhost (3/4) -2016-05-23 20:33:40 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.019 s -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 17 ms on localhost (4/4) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 20:33:40 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.036003 s -2016-05-23 20:33:40 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:33:40 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:33:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=400197, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(1860) called with curMem=403573, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1860.0 B, free 730.2 MB) -2016-05-23 20:33:40 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53178 (size: 1860.0 B, free: 730.5 MB) -2016-05-23 20:33:40 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:33:40 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 20:33:40 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 20:33:40 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 20:33:40 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 12 ms on localhost (1/4) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 15 ms on localhost (2/4) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 25 ms on localhost (3/4) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 28 ms on localhost (4/4) -2016-05-23 20:33:40 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.029 s -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 20:33:40 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.047586 s -2016-05-23 20:33:40 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:33:40 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=405433, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=409473, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:53178 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:33:40 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:33:40 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 20:33:40 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 20:33:40 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 20:33:40 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 20:33:40 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 67 ms on localhost (1/4) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 64 ms on localhost (2/4) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 77 ms on localhost (3/4) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 87 ms on localhost (4/4) -2016-05-23 20:33:40 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.088 s -2016-05-23 20:33:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:33:40 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:33:40 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 20:33:40 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 20:33:40 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=411755, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=414299, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 20:33:40 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:53178 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 20:33:40 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:40 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 20:33:40 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:40 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 20:33:40 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 25 ms on localhost (1/3) -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:40 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 20:33:40 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 42 ms on localhost (2/3) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 43 ms on localhost (3/3) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 20:33:40 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.046 s -2016-05-23 20:33:40 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.172653 s -2016-05-23 20:33:40 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:33:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:33:40 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:33:40 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=415867, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=420099, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:53178 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:33:40 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:33:40 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 20:33:40 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:33:40 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:40 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:33:40 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 65 ms on localhost (1/4) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 84 ms on localhost (2/4) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 125 ms on localhost (3/4) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 155 ms on localhost (4/4) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 20:33:40 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.157 s -2016-05-23 20:33:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:33:40 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:33:40 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 20:33:40 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:33:40 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=422405, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=427301, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:53178 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:33:40 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:40 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:40 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 13 ms on localhost (1/1) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 20:33:40 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.013 s -2016-05-23 20:33:40 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.202035 s -2016-05-23 20:33:40 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:33:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:33:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 20:33:40 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:33:40 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=429843, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=435539, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:53178 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:33:40 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:40 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 20:33:40 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 20:33:40 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 20:33:40 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 9 ms -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:33:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 14 ms -2016-05-23 20:33:40 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 28 ms on localhost (1/4) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 38 ms on localhost (2/4) -2016-05-23 20:33:40 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 20:33:40 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 44 ms on localhost (3/4) -2016-05-23 20:33:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 47 ms on localhost (4/4) -2016-05-23 20:33:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 20:33:40 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.047 s -2016-05-23 20:33:40 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.065239 s -2016-05-23 20:33:40 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:33:40 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:33:40 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 20:33:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:40 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=438448, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=444184, maxMem=766075207 -2016-05-23 20:33:40 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:33:40 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:53178 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:33:41 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:41 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:33:41 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:33:41 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 20:33:41 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 20:33:41 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 24 ms on localhost (1/4) -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:33:41 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 3 ms -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 34 ms on localhost (2/4) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 46 ms on localhost (3/4) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 53 ms on localhost (4/4) -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 20:33:41 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.054 s -2016-05-23 20:33:41 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.072735 s -2016-05-23 20:33:41 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:57 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Got job 20 (foreach at StatisticsCorrLearning.scala:57) with 4 output partitions -2016-05-23 20:33:41 INFO DAGScheduler:59 - Final stage: ResultStage 34(foreach at StatisticsCorrLearning.scala:57) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:54), which has no missing parents -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=447076, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=448300, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:53178 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:54) -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 62, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 63, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 64, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 20:33:41 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 64) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 64). 915 bytes result sent to driver -2016-05-23 20:33:41 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 65) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 65). 915 bytes result sent to driver -2016-05-23 20:33:41 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 63) -2016-05-23 20:33:41 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 62) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 64) in 12 ms on localhost (1/4) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 65) in 12 ms on localhost (2/4) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 63). 915 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 63) in 15 ms on localhost (3/4) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 62). 915 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 62) in 18 ms on localhost (4/4) -2016-05-23 20:33:41 INFO DAGScheduler:59 - ResultStage 34 (foreach at StatisticsCorrLearning.scala:57) finished in 0.018 s -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-23 20:33:41 INFO DAGScheduler:59 - Job 20 finished: foreach at StatisticsCorrLearning.scala:57, took 0.030898 s -2016-05-23 20:33:41 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:59 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Got job 21 (foreach at StatisticsCorrLearning.scala:59) with 4 output partitions -2016-05-23 20:33:41 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at StatisticsCorrLearning.scala:59) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:55), which has no missing parents -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=449193, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=450417, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:53178 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:55) -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 4 tasks -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 66, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 3.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 20:33:41 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 68) -2016-05-23 20:33:41 INFO Executor:59 - Running task 3.0 in stage 35.0 (TID 69) -2016-05-23 20:33:41 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 67) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 68). 915 bytes result sent to driver -2016-05-23 20:33:41 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 67). 915 bytes result sent to driver -2016-05-23 20:33:41 INFO Executor:59 - Finished task 3.0 in stage 35.0 (TID 69). 915 bytes result sent to driver -2016-05-23 20:33:41 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 66) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 66). 915 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 68) in 13 ms on localhost (1/4) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 67) in 13 ms on localhost (2/4) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 66) in 17 ms on localhost (3/4) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 3.0 in stage 35.0 (TID 69) in 15 ms on localhost (4/4) -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-23 20:33:41 INFO DAGScheduler:59 - ResultStage 35 (foreach at StatisticsCorrLearning.scala:59) finished in 0.018 s -2016-05-23 20:33:41 INFO DAGScheduler:59 - Job 21 finished: foreach at StatisticsCorrLearning.scala:59, took 0.032282 s -2016-05-23 20:33:41 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Got job 22 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:33:41 INFO DAGScheduler:59 - Final stage: ResultStage 36(first at RowMatrix.scala:65) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=451310, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(1522) called with curMem=453814, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1522.0 B, free 730.2 MB) -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:53178 (size: 1522.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:33:41 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 70) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 70). 935 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 70) in 6 ms on localhost (1/1) -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-23 20:33:41 INFO DAGScheduler:59 - ResultStage 36 (first at RowMatrix.scala:65) finished in 0.007 s -2016-05-23 20:33:41 INFO DAGScheduler:59 - Job 22 finished: first at RowMatrix.scala:65, took 0.020914 s -2016-05-23 20:33:41 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Got job 23 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 20:33:41 INFO DAGScheduler:59 - Final stage: ResultStage 37(first at RowMatrix.scala:65) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=455336, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(1522) called with curMem=457840, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1522.0 B, free 730.1 MB) -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:53178 (size: 1522.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 37 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 3 tasks -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:33:41 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 72) -2016-05-23 20:33:41 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-23 20:33:41 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 73) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 1046 bytes result sent to driver -2016-05-23 20:33:41 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 73). 1046 bytes result sent to driver -2016-05-23 20:33:41 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 72). 935 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 73) in 8 ms on localhost (1/3) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 10 ms on localhost (2/3) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 72) in 10 ms on localhost (3/3) -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-23 20:33:41 INFO DAGScheduler:59 - ResultStage 37 (first at RowMatrix.scala:65) finished in 0.011 s -2016-05-23 20:33:41 INFO DAGScheduler:59 - Job 23 finished: first at RowMatrix.scala:65, took 0.024491 s -2016-05-23 20:33:41 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Got job 24 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:33:41 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=459362, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(1932) called with curMem=462762, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1932.0 B, free 730.1 MB) -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:53178 (size: 1932.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 74, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 77, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:33:41 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 75) -2016-05-23 20:33:41 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 74) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 74). 1231 bytes result sent to driver -2016-05-23 20:33:41 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 77) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 74) in 11 ms on localhost (1/4) -2016-05-23 20:33:41 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 76) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 77). 1231 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 77) in 13 ms on localhost (2/4) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 75). 1231 bytes result sent to driver -2016-05-23 20:33:41 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 76). 1231 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 75) in 17 ms on localhost (3/4) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 76) in 17 ms on localhost (4/4) -2016-05-23 20:33:41 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at RowMatrix.scala:332) finished in 0.020 s -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-23 20:33:41 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at RowMatrix.scala:332, took 0.036045 s -2016-05-23 20:33:41 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Got job 25 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:33:41 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=464694, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(1916) called with curMem=468134, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1916.0 B, free 730.1 MB) -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:53178 (size: 1916.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 39 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 4 tasks -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 79, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 80, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 3.0 in stage 39.0 (TID 81, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:33:41 INFO Executor:59 - Running task 3.0 in stage 39.0 (TID 81) -2016-05-23 20:33:41 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 80) -2016-05-23 20:33:41 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 78) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 80). 1109 bytes result sent to driver -2016-05-23 20:33:41 INFO Executor:59 - Finished task 3.0 in stage 39.0 (TID 81). 1109 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 80) in 8 ms on localhost (1/4) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 3.0 in stage 39.0 (TID 81) in 8 ms on localhost (2/4) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 78). 1109 bytes result sent to driver -2016-05-23 20:33:41 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 79) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 78) in 13 ms on localhost (3/4) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 79). 1109 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 79) in 15 ms on localhost (4/4) -2016-05-23 20:33:41 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at RowMatrix.scala:124) finished in 0.017 s -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-23 20:33:41 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at RowMatrix.scala:124, took 0.031564 s -2016-05-23 20:33:41 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Got job 26 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:33:41 INFO DAGScheduler:59 - Final stage: ResultStage 40(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=470050, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(1851) called with curMem=473426, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 1851.0 B, free 730.1 MB) -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:53178 (size: 1851.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 40 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 4 tasks -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 82, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 83, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 84, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 3.0 in stage 40.0 (TID 85, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:33:41 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 84) -2016-05-23 20:33:41 INFO Executor:59 - Running task 3.0 in stage 40.0 (TID 85) -2016-05-23 20:33:41 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 82) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 3.0 in stage 40.0 (TID 85). 1254 bytes result sent to driver -2016-05-23 20:33:41 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 82). 1104 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 3.0 in stage 40.0 (TID 85) in 7 ms on localhost (1/4) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 84). 1109 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 82) in 13 ms on localhost (2/4) -2016-05-23 20:33:41 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 83) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 84) in 11 ms on localhost (3/4) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 83). 1254 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 83) in 16 ms on localhost (4/4) -2016-05-23 20:33:41 INFO DAGScheduler:59 - ResultStage 40 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.018 s -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-23 20:33:41 INFO DAGScheduler:59 - Job 26 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.031516 s -2016-05-23 20:33:41 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Registering RDD 51 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Got job 27 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:33:41 INFO DAGScheduler:59 - Final stage: ResultStage 42(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 41) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 41) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting ShuffleMapStage 41 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=475277, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(2283) called with curMem=479317, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:53178 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:33:41 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 41 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 4 tasks -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 86, localhost, PROCESS_LOCAL, 2221 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 87, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 88, localhost, PROCESS_LOCAL, 2221 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 3.0 in stage 41.0 (TID 89, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 20:33:41 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 86) -2016-05-23 20:33:41 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 88) -2016-05-23 20:33:41 INFO Executor:59 - Running task 3.0 in stage 41.0 (TID 89) -2016-05-23 20:33:41 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 87) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 2.0 in stage 41.0 (TID 88). 1161 bytes result sent to driver -2016-05-23 20:33:41 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 86). 1161 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 86) in 16 ms on localhost (1/4) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 41.0 (TID 88) in 15 ms on localhost (2/4) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 3.0 in stage 41.0 (TID 89). 1161 bytes result sent to driver -2016-05-23 20:33:41 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 87). 1161 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 3.0 in stage 41.0 (TID 89) in 34 ms on localhost (3/4) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 87) in 36 ms on localhost (4/4) -2016-05-23 20:33:41 INFO DAGScheduler:59 - ShuffleMapStage 41 (flatMap at SpearmanCorrelation.scala:49) finished in 0.037 s -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-23 20:33:41 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:33:41 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:33:41 INFO DAGScheduler:59 - waiting: Set(ResultStage 42) -2016-05-23 20:33:41 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Missing parents for ResultStage 42: List() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting ResultStage 42 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=481600, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(1569) called with curMem=484144, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 1569.0 B, free 730.1 MB) -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:53178 (size: 1569.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 42 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 3 tasks -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 90, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 91, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 42.0 (TID 92, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:41 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 90) -2016-05-23 20:33:41 INFO Executor:59 - Running task 2.0 in stage 42.0 (TID 92) -2016-05-23 20:33:41 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 91) -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:41 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 90). 1203 bytes result sent to driver -2016-05-23 20:33:41 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 91). 1203 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 90) in 8 ms on localhost (1/3) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 2.0 in stage 42.0 (TID 92). 1203 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 91) in 9 ms on localhost (2/3) -2016-05-23 20:33:41 INFO DAGScheduler:59 - ResultStage 42 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.011 s -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 42.0 (TID 92) in 9 ms on localhost (3/3) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Job 27 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.067546 s -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-23 20:33:41 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:33:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 165 bytes -2016-05-23 20:33:41 INFO DAGScheduler:59 - Registering RDD 56 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Got job 28 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:33:41 INFO DAGScheduler:59 - Final stage: ResultStage 45(first at RowMatrix.scala:65) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 44) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 44) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting ShuffleMapStage 44 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=485713, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(2307) called with curMem=489945, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:53178 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:33:41 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 44 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 4 tasks -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 93, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 94, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 44.0 (TID 95, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 3.0 in stage 44.0 (TID 96, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:33:41 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 93) -2016-05-23 20:33:41 INFO Executor:59 - Running task 3.0 in stage 44.0 (TID 96) -2016-05-23 20:33:41 INFO Executor:59 - Running task 2.0 in stage 44.0 (TID 95) -2016-05-23 20:33:41 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 94) -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 20:33:41 INFO Executor:59 - Finished task 3.0 in stage 44.0 (TID 96). 1377 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 3.0 in stage 44.0 (TID 96) in 14 ms on localhost (1/4) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 94). 1377 bytes result sent to driver -2016-05-23 20:33:41 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 93). 1377 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 94) in 47 ms on localhost (2/4) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 93) in 51 ms on localhost (3/4) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 2.0 in stage 44.0 (TID 95). 1377 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 44.0 (TID 95) in 62 ms on localhost (4/4) -2016-05-23 20:33:41 INFO DAGScheduler:59 - ShuffleMapStage 44 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.063 s -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-23 20:33:41 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:33:41 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:33:41 INFO DAGScheduler:59 - waiting: Set(ResultStage 45) -2016-05-23 20:33:41 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Missing parents for ResultStage 45: List() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=492252, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=497148, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:53178 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:33:41 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 45 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 1 tasks -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 97, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:41 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 97) -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:41 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 97). 1185 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 97) in 8 ms on localhost (1/1) -2016-05-23 20:33:41 INFO DAGScheduler:59 - ResultStage 45 (first at RowMatrix.scala:65) finished in 0.008 s -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-23 20:33:41 INFO DAGScheduler:59 - Job 28 finished: first at RowMatrix.scala:65, took 0.092464 s -2016-05-23 20:33:41 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:33:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 165 bytes -2016-05-23 20:33:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 164 bytes -2016-05-23 20:33:41 INFO DAGScheduler:59 - Got job 29 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 20:33:41 INFO DAGScheduler:59 - Final stage: ResultStage 48(first at RowMatrix.scala:65) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 47) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which has no missing parents -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=499695, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=504591, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:53178 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:33:41 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 48 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 3 tasks -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 98, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 99, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 48.0 (TID 100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:41 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 98) -2016-05-23 20:33:41 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 99) -2016-05-23 20:33:41 INFO Executor:59 - Running task 2.0 in stage 48.0 (TID 100) -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:41 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 99). 1185 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 99) in 12 ms on localhost (1/3) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 2.0 in stage 48.0 (TID 100). 1296 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 48.0 (TID 100) in 14 ms on localhost (2/3) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 98). 1296 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 98) in 16 ms on localhost (3/3) -2016-05-23 20:33:41 INFO DAGScheduler:59 - ResultStage 48 (first at RowMatrix.scala:65) finished in 0.017 s -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-23 20:33:41 INFO DAGScheduler:59 - Job 29 finished: first at RowMatrix.scala:65, took 0.030734 s -2016-05-23 20:33:41 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Got job 30 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:33:41 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 50) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=507138, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(2916) called with curMem=512834, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:53178 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:33:41 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 51 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:53178 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 4 tasks -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 51.0 (TID 103, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 3.0 in stage 51.0 (TID 104, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:41 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-23 20:33:41 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:33:41 INFO Executor:59 - Running task 2.0 in stage 51.0 (TID 103) -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:41 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 1481 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 10 ms on localhost (1/4) -2016-05-23 20:33:41 INFO Executor:59 - Running task 3.0 in stage 51.0 (TID 104) -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:53178 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 1481 bytes result sent to driver -2016-05-23 20:33:41 INFO Executor:59 - Finished task 2.0 in stage 51.0 (TID 103). 1481 bytes result sent to driver -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 15 ms on localhost (2/4) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 51.0 (TID 103) in 14 ms on localhost (3/4) -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:53178 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-23 20:33:41 INFO Executor:59 - Finished task 3.0 in stage 51.0 (TID 104). 1481 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 3.0 in stage 51.0 (TID 104) in 19 ms on localhost (4/4) -2016-05-23 20:33:41 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at RowMatrix.scala:332) finished in 0.021 s -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-23 20:33:41 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at RowMatrix.scala:332, took 0.059274 s -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:53178 in memory (size: 1569.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:53178 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-23 20:33:41 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:53178 in memory (size: 1851.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Got job 31 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:33:41 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 53) -2016-05-23 20:33:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:53178 in memory (size: 1916.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=473306, maxMem=766075207 -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 20:33:41 INFO MemoryStore:59 - ensureFreeSpace(2898) called with curMem=477110, maxMem=766075207 -2016-05-23 20:33:41 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:53178 in memory (size: 1932.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:53178 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-23 20:33:41 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 54 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 4 tasks -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 105, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 106, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Starting task 3.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:53178 in memory (size: 1522.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 106) -2016-05-23 20:33:41 INFO Executor:59 - Running task 2.0 in stage 54.0 (TID 107) -2016-05-23 20:33:41 INFO Executor:59 - Running task 3.0 in stage 54.0 (TID 108) -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:41 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 105) -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:33:41 INFO Executor:59 - Finished task 2.0 in stage 54.0 (TID 107). 1359 bytes result sent to driver -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:33:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-23 20:33:41 INFO Executor:59 - Finished task 3.0 in stage 54.0 (TID 108). 1359 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 54.0 (TID 107) in 15 ms on localhost (1/4) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 3.0 in stage 54.0 (TID 108) in 15 ms on localhost (2/4) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 105). 1359 bytes result sent to driver -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 105) in 18 ms on localhost (3/4) -2016-05-23 20:33:41 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 106). 1359 bytes result sent to driver -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:53178 in memory (size: 1522.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 106) in 22 ms on localhost (4/4) -2016-05-23 20:33:41 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at RowMatrix.scala:124) finished in 0.023 s -2016-05-23 20:33:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-23 20:33:41 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at RowMatrix.scala:124, took 0.036852 s -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:53178 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:53178 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:53178 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:53178 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:53178 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:53178 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 20:33:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:53178 in memory (size: 1568.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:53178 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:53178 in memory (size: 1860.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:53178 in memory (size: 1918.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:53178 in memory (size: 1934.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:53178 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:53178 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:53178 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:53178 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:53178 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:53178 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-23 20:33:41 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 20:33:41 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:53178 in memory (size: 2.2 KB, free: 730.6 MB) -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-23 20:33:41 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-23 20:33:41 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 20:33:41 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 20:33:42 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 20:33:42 INFO BlockManager:59 - BlockManager stopped -2016-05-23 20:33:42 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 20:33:42 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 20:33:42 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 20:33:42 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 20:33:42 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-ba3defed-88f9-4535-8259-486d1f84e95e -2016-05-23 20:33:42 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 20:33:42 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 20:39:22 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 20:39:22 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 20:39:23 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 20:39:23 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 20:39:23 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 20:39:24 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 20:39:24 INFO Remoting:74 - Starting remoting -2016-05-23 20:39:24 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53307] -2016-05-23 20:39:24 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53307. -2016-05-23 20:39:24 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 20:39:24 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 20:39:24 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-c2020868-fab8-4327-9fed-4af337454acf -2016-05-23 20:39:24 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 20:39:24 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-b894b55b-acea-487f-a709-35378595727a\httpd-ebc70081-82a5-44b2-8cde-964ddc8cb9a5 -2016-05-23 20:39:24 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 20:39:24 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:39:24 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53308 -2016-05-23 20:39:24 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53308. -2016-05-23 20:39:24 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 20:39:25 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:39:25 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 20:39:25 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 20:39:25 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 20:39:25 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 20:39:25 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 20:39:25 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53327. -2016-05-23 20:39:25 INFO NettyBlockTransferService:59 - Server created on 53327 -2016-05-23 20:39:25 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 20:39:25 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53327 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53327) -2016-05-23 20:39:25 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 20:39:26 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 20:39:26 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 20:39:26 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 20:39:26 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 20:39:26 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53327 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:39:26 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:20 -2016-05-23 20:39:26 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 20:39:26 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 20:39:26 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 20:39:26 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 20:39:26 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53327 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:39:26 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 20:39:27 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 20:39:27 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:39:27 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:27 -2016-05-23 20:39:27 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:27) with 2 output partitions -2016-05-23 20:39:27 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:27) -2016-05-23 20:39:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:39:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:27 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21), which has no missing parents -2016-05-23 20:39:27 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 20:39:27 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:39:27 INFO MemoryStore:59 - ensureFreeSpace(2047) called with curMem=346712, maxMem=766075207 -2016-05-23 20:39:27 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2047.0 B, free 730.3 MB) -2016-05-23 20:39:27 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53327 (size: 2047.0 B, free: 730.6 MB) -2016-05-23 20:39:27 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21) -2016-05-23 20:39:27 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 20:39:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:39:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:39:27 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 20:39:27 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 20:39:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:39:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:39:28 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 20:39:28 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 20:39:28 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 20:39:28 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 20:39:28 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 20:39:28 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 20:39:28 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 20:39:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 113 ms on localhost (1/2) -2016-05-23 20:39:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 95 ms on localhost (2/2) -2016-05-23 20:39:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 20:39:28 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:27) finished in 0.132 s -2016-05-23 20:39:28 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:27, took 0.256231 s -2016-05-23 20:39:28 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:39:28 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 20:39:28 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 20:39:28 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 20:39:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:39:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:28 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24), which has no missing parents -2016-05-23 20:39:28 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348759, maxMem=766075207 -2016-05-23 20:39:28 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:39:28 INFO MemoryStore:59 - ensureFreeSpace(2045) called with curMem=352271, maxMem=766075207 -2016-05-23 20:39:28 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2045.0 B, free 730.2 MB) -2016-05-23 20:39:28 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53327 (size: 2045.0 B, free: 730.6 MB) -2016-05-23 20:39:28 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24) -2016-05-23 20:39:28 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 20:39:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:39:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:39:28 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 20:39:28 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 20:39:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:39:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:39:28 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 20:39:28 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 20:39:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 23 ms on localhost (1/2) -2016-05-23 20:39:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 26 ms on localhost (2/2) -2016-05-23 20:39:28 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:29) finished in 0.026 s -2016-05-23 20:39:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 20:39:28 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:29, took 0.046563 s -2016-05-23 20:39:28 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:39:28 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:39:28 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 20:39:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:39:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:28 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:39:28 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354316, maxMem=766075207 -2016-05-23 20:39:28 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 20:39:28 INFO MemoryStore:59 - ensureFreeSpace(2464) called with curMem=358876, maxMem=766075207 -2016-05-23 20:39:28 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:39:28 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53327 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:39:28 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 20:39:28 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 20:39:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:39:28 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 20:39:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:39:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:39:28 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 20:39:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 29 ms on localhost (1/1) -2016-05-23 20:39:28 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.030 s -2016-05-23 20:39:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 20:39:28 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.048780 s -2016-05-23 20:39:28 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:39:28 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:39:28 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:39:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:39:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:28 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:39:28 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=361340, maxMem=766075207 -2016-05-23 20:39:28 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 20:39:28 INFO MemoryStore:59 - ensureFreeSpace(2861) called with curMem=366788, maxMem=766075207 -2016-05-23 20:39:28 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:39:28 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53327 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:39:28 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:39:28 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 20:39:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:39:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:39:28 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 20:39:28 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 20:39:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:39:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:39:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:39:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:39:28 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 20:39:28 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 20:39:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 46 ms on localhost (1/2) -2016-05-23 20:39:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 50 ms on localhost (2/2) -2016-05-23 20:39:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 20:39:28 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.050 s -2016-05-23 20:39:28 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 20:39:28 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 20:39:28 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.079300 s -2016-05-23 20:39:28 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:39:28 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:39:28 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:39:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:39:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:28 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:39:28 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=369649, maxMem=766075207 -2016-05-23 20:39:28 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 20:39:28 INFO MemoryStore:59 - ensureFreeSpace(2850) called with curMem=375145, maxMem=766075207 -2016-05-23 20:39:28 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:39:28 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53327 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:39:28 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:39:28 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 20:39:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:39:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:39:28 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 20:39:28 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 20:39:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:39:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:39:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:39:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:39:28 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 20:39:28 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 20:39:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 36 ms on localhost (1/2) -2016-05-23 20:39:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 35 ms on localhost (2/2) -2016-05-23 20:39:28 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.038 s -2016-05-23 20:39:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 20:39:28 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.060835 s -2016-05-23 20:39:29 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53327 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:39:29 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 20:39:29 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53327 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:39:29 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 20:39:29 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53327 in memory (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:39:29 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-23 20:39:29 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53327 in memory (size: 2045.0 B, free: 730.6 MB) -2016-05-23 20:39:29 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-23 20:39:29 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53327 in memory (size: 2047.0 B, free: 730.6 MB) -2016-05-23 20:39:29 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-23 20:39:29 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:39:29 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 20:39:29 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:39:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:39:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:29 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:39:29 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=343200, maxMem=766075207 -2016-05-23 20:39:29 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.3 MB) -2016-05-23 20:39:29 INFO MemoryStore:59 - ensureFreeSpace(2743) called with curMem=348544, maxMem=766075207 -2016-05-23 20:39:29 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.3 MB) -2016-05-23 20:39:29 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53327 (size: 2.7 KB, free: 730.6 MB) -2016-05-23 20:39:29 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:39:29 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:39:29 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 20:39:29 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 20:39:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:39:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:39:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:39:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:39:29 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 20:39:29 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 37 ms on localhost (1/2) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 40 ms on localhost (2/2) -2016-05-23 20:39:29 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.040 s -2016-05-23 20:39:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 20:39:29 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.056865 s -2016-05-23 20:39:29 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:39:29 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:39:29 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 20:39:29 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:39:29 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 20:39:29 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 20:39:29 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:39:29 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=351287, maxMem=766075207 -2016-05-23 20:39:29 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 20:39:29 INFO MemoryStore:59 - ensureFreeSpace(3155) called with curMem=357215, maxMem=766075207 -2016-05-23 20:39:29 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 20:39:29 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53327 (size: 3.1 KB, free: 730.6 MB) -2016-05-23 20:39:29 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:39:29 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:39:29 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 20:39:29 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 20:39:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:39:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:39:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:39:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:39:29 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 20:39:29 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 62 ms on localhost (1/2) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 61 ms on localhost (2/2) -2016-05-23 20:39:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 20:39:29 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.064 s -2016-05-23 20:39:29 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:39:29 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:39:29 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 20:39:29 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:39:29 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 20:39:29 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:39:29 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=360370, maxMem=766075207 -2016-05-23 20:39:29 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:39:29 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=362874, maxMem=766075207 -2016-05-23 20:39:29 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 20:39:29 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53327 (size: 1545.0 B, free: 730.6 MB) -2016-05-23 20:39:29 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:39:29 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:29 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 20:39:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:39:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-23 20:39:29 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 57 ms on localhost (1/1) -2016-05-23 20:39:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 20:39:29 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.058 s -2016-05-23 20:39:29 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.177849 s -2016-05-23 20:39:29 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:39:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:39:29 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:39:29 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:39:29 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 20:39:29 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 20:39:29 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 20:39:29 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:39:29 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=364419, maxMem=766075207 -2016-05-23 20:39:29 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:39:29 INFO MemoryStore:59 - ensureFreeSpace(2280) called with curMem=368611, maxMem=766075207 -2016-05-23 20:39:29 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:39:29 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53327 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:39:29 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:39:29 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:39:29 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 20:39:29 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 20:39:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:39:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:39:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:39:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:39:29 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 20:39:29 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 38 ms on localhost (1/2) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 43 ms on localhost (2/2) -2016-05-23 20:39:29 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.043 s -2016-05-23 20:39:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 20:39:29 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:39:29 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:39:29 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 20:39:29 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:39:29 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 20:39:29 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:39:29 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=370891, maxMem=766075207 -2016-05-23 20:39:29 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 20:39:29 INFO MemoryStore:59 - ensureFreeSpace(2530) called with curMem=375747, maxMem=766075207 -2016-05-23 20:39:29 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:39:29 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53327 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:39:29 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:39:29 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:29 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 20:39:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:39:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:29 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 24 ms on localhost (1/1) -2016-05-23 20:39:29 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.024 s -2016-05-23 20:39:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 20:39:29 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.099549 s -2016-05-23 20:39:29 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:39:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:39:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 20:39:29 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:39:29 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:39:29 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 20:39:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:29 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:39:29 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=378277, maxMem=766075207 -2016-05-23 20:39:29 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 20:39:29 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=383933, maxMem=766075207 -2016-05-23 20:39:29 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:39:29 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53327 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:39:29 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:39:29 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:29 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 20:39:29 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 20:39:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:39:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:39:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:29 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 20:39:29 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 19 ms on localhost (1/2) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 21 ms on localhost (2/2) -2016-05-23 20:39:29 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.021 s -2016-05-23 20:39:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 20:39:29 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.037182 s -2016-05-23 20:39:29 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:39:29 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:39:29 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:39:29 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 20:39:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:29 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:39:29 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=386841, maxMem=766075207 -2016-05-23 20:39:29 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:39:29 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=392537, maxMem=766075207 -2016-05-23 20:39:29 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:39:29 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53327 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:39:29 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:39:29 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:29 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 20:39:29 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 20:39:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:39:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:39:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:39:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:29 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 20:39:29 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 22 ms on localhost (1/2) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 21 ms on localhost (2/2) -2016-05-23 20:39:29 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.023 s -2016-05-23 20:39:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 20:39:29 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.041998 s -2016-05-23 20:39:29 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:43 -2016-05-23 20:39:29 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:43) with 4 output partitions -2016-05-23 20:39:29 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:43) -2016-05-23 20:39:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:39:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:29 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40), which has no missing parents -2016-05-23 20:39:29 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=395426, maxMem=766075207 -2016-05-23 20:39:29 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:39:29 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=396650, maxMem=766075207 -2016-05-23 20:39:29 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:39:29 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53327 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:39:29 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40) -2016-05-23 20:39:29 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:39:29 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 20:39:29 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 20:39:29 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 20:39:29 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 20:39:29 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 20:39:29 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 20:39:29 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 10 ms on localhost (1/4) -2016-05-23 20:39:29 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 12 ms on localhost (2/4) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 21 ms on localhost (3/4) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 12 ms on localhost (4/4) -2016-05-23 20:39:29 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:43) finished in 0.022 s -2016-05-23 20:39:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 20:39:29 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:43, took 0.037305 s -2016-05-23 20:39:29 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:45 -2016-05-23 20:39:29 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:45) with 4 output partitions -2016-05-23 20:39:29 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:45) -2016-05-23 20:39:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:39:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:29 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41), which has no missing parents -2016-05-23 20:39:29 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=397543, maxMem=766075207 -2016-05-23 20:39:29 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:39:29 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=398767, maxMem=766075207 -2016-05-23 20:39:29 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:39:29 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53327 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:39:29 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41) -2016-05-23 20:39:29 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:39:29 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 20:39:29 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 20:39:29 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 20:39:29 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 20:39:29 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 20:39:29 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 20:39:29 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 20:39:29 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 9 ms on localhost (1/4) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 14 ms on localhost (2/4) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 11 ms on localhost (3/4) -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 14 ms on localhost (4/4) -2016-05-23 20:39:29 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:45) finished in 0.015 s -2016-05-23 20:39:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 20:39:29 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:45, took 0.026449 s -2016-05-23 20:39:29 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:39:29 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:39:29 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 20:39:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:39:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:29 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:39:29 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=399660, maxMem=766075207 -2016-05-23 20:39:29 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:39:29 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=402164, maxMem=766075207 -2016-05-23 20:39:29 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 20:39:29 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53327 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 20:39:29 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 20:39:29 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 20:39:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:39:29 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 20:39:29 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 20:39:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 7 ms on localhost (1/1) -2016-05-23 20:39:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 20:39:29 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.007 s -2016-05-23 20:39:29 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.023087 s -2016-05-23 20:39:30 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:39:30 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:39:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:39:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=403688, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=407088, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53327 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 20:39:30 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:39:30 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 20:39:30 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 20:39:30 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 20:39:30 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 20:39:30 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 20:39:30 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 10 ms on localhost (1/4) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 15 ms on localhost (2/4) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 18 ms on localhost (3/4) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 20 ms on localhost (4/4) -2016-05-23 20:39:30 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.021 s -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 20:39:30 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.040837 s -2016-05-23 20:39:30 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:39:30 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:39:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:39:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=409022, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=412462, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53327 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 20:39:30 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:53327 in memory (size: 1934.0 B, free: 730.5 MB) -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 20:39:30 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:39:30 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 20:39:30 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:53327 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 20:39:30 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 20:39:30 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 20:39:30 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 11 ms on localhost (1/4) -2016-05-23 20:39:30 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:53327 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 21 ms on localhost (2/4) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 17 ms on localhost (3/4) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 20:39:30 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 27 ms on localhost (4/4) -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 20:39:30 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.028 s -2016-05-23 20:39:30 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.067037 s -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:53327 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:39:30 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:53327 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:39:30 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:53327 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:39:30 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:53327 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:39:30 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:53327 in memory (size: 2.2 KB, free: 730.6 MB) -2016-05-23 20:39:30 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:39:30 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:39:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:39:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=369777, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(1860) called with curMem=373153, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1860.0 B, free 730.2 MB) -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53327 (size: 1860.0 B, free: 730.5 MB) -2016-05-23 20:39:30 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 20:39:30 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:39:30 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-23 20:39:30 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 20:39:30 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 22 ms on localhost (1/4) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 20:39:30 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 28 ms on localhost (2/4) -2016-05-23 20:39:30 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 20:39:30 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53327 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 41 ms on localhost (3/4) -2016-05-23 20:39:30 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 44 ms on localhost (4/4) -2016-05-23 20:39:30 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.046 s -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 20:39:30 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.068557 s -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53327 in memory (size: 3.1 KB, free: 730.6 MB) -2016-05-23 20:39:30 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 20:39:30 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53327 in memory (size: 2.7 KB, free: 730.6 MB) -2016-05-23 20:39:30 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 20:39:30 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:39:30 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:39:30 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:39:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 20:39:30 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=353794, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=357834, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:53327 (size: 2.2 KB, free: 730.6 MB) -2016-05-23 20:39:30 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:39:30 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 20:39:30 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 20:39:30 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 20:39:30 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 58 ms on localhost (1/4) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 89 ms on localhost (2/4) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 91 ms on localhost (3/4) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 97 ms on localhost (4/4) -2016-05-23 20:39:30 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.098 s -2016-05-23 20:39:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 20:39:30 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:39:30 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 20:39:30 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:39:30 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=360116, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=362660, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:53327 (size: 1568.0 B, free: 730.6 MB) -2016-05-23 20:39:30 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:30 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 20:39:30 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 20:39:30 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:30 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 26 ms on localhost (1/3) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 38 ms on localhost (2/3) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 39 ms on localhost (3/3) -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 20:39:30 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.042 s -2016-05-23 20:39:30 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.173349 s -2016-05-23 20:39:30 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:39:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:39:30 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:39:30 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:39:30 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 20:39:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 20:39:30 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=364228, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(2304) called with curMem=368460, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:53327 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:39:30 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:39:30 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 20:39:30 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:30 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 20:39:30 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:39:30 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 20:39:30 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 90 ms on localhost (1/4) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 92 ms on localhost (2/4) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 142 ms on localhost (3/4) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 152 ms on localhost (4/4) -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 20:39:30 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.155 s -2016-05-23 20:39:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:39:30 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:39:30 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 20:39:30 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:39:30 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=370764, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(2541) called with curMem=375660, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:53327 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:39:30 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:30 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:30 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 17 ms on localhost (1/1) -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 20:39:30 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.018 s -2016-05-23 20:39:30 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.205810 s -2016-05-23 20:39:30 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:39:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:39:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 20:39:30 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:39:30 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:39:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 20:39:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=378201, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=383897, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:53327 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:39:30 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:30 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 20:39:30 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 20:39:30 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:39:30 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:39:30 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 26 ms on localhost (1/4) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 34 ms on localhost (2/4) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 20:39:30 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 43 ms on localhost (3/4) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 45 ms on localhost (4/4) -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 20:39:30 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.047 s -2016-05-23 20:39:30 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.064145 s -2016-05-23 20:39:30 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:39:30 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:39:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 20:39:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=386806, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=392542, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:53327 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:39:30 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:30 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 20:39:30 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 20:39:30 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:30 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 20:39:30 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 18 ms on localhost (1/4) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 21 ms on localhost (2/4) -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:39:30 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:39:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:39:30 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 20:39:30 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 38 ms on localhost (3/4) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 42 ms on localhost (4/4) -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 20:39:30 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.043 s -2016-05-23 20:39:30 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.057389 s -2016-05-23 20:39:30 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:57 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Got job 20 (foreach at StatisticsCorrLearning.scala:57) with 4 output partitions -2016-05-23 20:39:30 INFO DAGScheduler:59 - Final stage: ResultStage 34(foreach at StatisticsCorrLearning.scala:57) -2016-05-23 20:39:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:39:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:54), which has no missing parents -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=395434, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=396658, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:53327 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:39:30 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:54) -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 62, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 63, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 64, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 20:39:30 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 65) -2016-05-23 20:39:30 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 64) -2016-05-23 20:39:30 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 62) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 64). 915 bytes result sent to driver -2016-05-23 20:39:30 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 62). 915 bytes result sent to driver -2016-05-23 20:39:30 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 65). 915 bytes result sent to driver -2016-05-23 20:39:30 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 63) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 64) in 10 ms on localhost (1/4) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 63). 915 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 62) in 13 ms on localhost (2/4) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 65) in 10 ms on localhost (3/4) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 63) in 14 ms on localhost (4/4) -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-23 20:39:30 INFO DAGScheduler:59 - ResultStage 34 (foreach at StatisticsCorrLearning.scala:57) finished in 0.017 s -2016-05-23 20:39:30 INFO DAGScheduler:59 - Job 20 finished: foreach at StatisticsCorrLearning.scala:57, took 0.030557 s -2016-05-23 20:39:30 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:59 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Got job 21 (foreach at StatisticsCorrLearning.scala:59) with 4 output partitions -2016-05-23 20:39:30 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at StatisticsCorrLearning.scala:59) -2016-05-23 20:39:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:39:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:55), which has no missing parents -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=397551, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=398775, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:53327 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:39:30 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:55) -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 4 tasks -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 66, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 20:39:30 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 66) -2016-05-23 20:39:30 INFO Executor:59 - Running task 3.0 in stage 35.0 (TID 69) -2016-05-23 20:39:30 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 67) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 3.0 in stage 35.0 (TID 69). 915 bytes result sent to driver -2016-05-23 20:39:30 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 68) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 67). 915 bytes result sent to driver -2016-05-23 20:39:30 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 68). 915 bytes result sent to driver -2016-05-23 20:39:30 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 66). 915 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 35.0 (TID 69) in 10 ms on localhost (1/4) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 67) in 13 ms on localhost (2/4) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 68) in 13 ms on localhost (3/4) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 66) in 16 ms on localhost (4/4) -2016-05-23 20:39:30 INFO DAGScheduler:59 - ResultStage 35 (foreach at StatisticsCorrLearning.scala:59) finished in 0.016 s -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-23 20:39:30 INFO DAGScheduler:59 - Job 21 finished: foreach at StatisticsCorrLearning.scala:59, took 0.028953 s -2016-05-23 20:39:30 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Got job 22 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:39:30 INFO DAGScheduler:59 - Final stage: ResultStage 36(first at RowMatrix.scala:65) -2016-05-23 20:39:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:39:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=399668, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(1522) called with curMem=402172, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1522.0 B, free 730.2 MB) -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:53327 (size: 1522.0 B, free: 730.5 MB) -2016-05-23 20:39:30 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:39:30 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 70) -2016-05-23 20:39:30 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 70). 935 bytes result sent to driver -2016-05-23 20:39:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 70) in 6 ms on localhost (1/1) -2016-05-23 20:39:30 INFO DAGScheduler:59 - ResultStage 36 (first at RowMatrix.scala:65) finished in 0.006 s -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-23 20:39:30 INFO DAGScheduler:59 - Job 22 finished: first at RowMatrix.scala:65, took 0.016388 s -2016-05-23 20:39:30 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Got job 23 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 20:39:30 INFO DAGScheduler:59 - Final stage: ResultStage 37(first at RowMatrix.scala:65) -2016-05-23 20:39:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:39:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=403694, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:39:30 INFO MemoryStore:59 - ensureFreeSpace(1522) called with curMem=406198, maxMem=766075207 -2016-05-23 20:39:30 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1522.0 B, free 730.2 MB) -2016-05-23 20:39:30 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:53327 (size: 1522.0 B, free: 730.5 MB) -2016-05-23 20:39:30 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:30 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 37 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 20:39:30 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 3 tasks -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:39:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:39:30 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-23 20:39:30 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 72) -2016-05-23 20:39:30 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 73) -2016-05-23 20:39:31 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 1046 bytes result sent to driver -2016-05-23 20:39:31 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 72). 935 bytes result sent to driver -2016-05-23 20:39:31 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 73). 1046 bytes result sent to driver -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 10 ms on localhost (1/3) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 72) in 10 ms on localhost (2/3) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 73) in 9 ms on localhost (3/3) -2016-05-23 20:39:31 INFO DAGScheduler:59 - ResultStage 37 (first at RowMatrix.scala:65) finished in 0.011 s -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-23 20:39:31 INFO DAGScheduler:59 - Job 23 finished: first at RowMatrix.scala:65, took 0.023388 s -2016-05-23 20:39:31 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:39:31 INFO DAGScheduler:59 - Got job 24 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:39:31 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:39:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:39:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:31 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:39:31 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=407720, maxMem=766075207 -2016-05-23 20:39:31 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:39:31 INFO MemoryStore:59 - ensureFreeSpace(1932) called with curMem=411120, maxMem=766075207 -2016-05-23 20:39:31 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1932.0 B, free 730.2 MB) -2016-05-23 20:39:31 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:53327 (size: 1932.0 B, free: 730.5 MB) -2016-05-23 20:39:31 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 74, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 77, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:39:31 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 74) -2016-05-23 20:39:31 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 77) -2016-05-23 20:39:31 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 76) -2016-05-23 20:39:31 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 75) -2016-05-23 20:39:31 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 77). 1231 bytes result sent to driver -2016-05-23 20:39:31 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 75). 1231 bytes result sent to driver -2016-05-23 20:39:31 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 74). 1231 bytes result sent to driver -2016-05-23 20:39:31 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 76). 1231 bytes result sent to driver -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 77) in 8 ms on localhost (1/4) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 75) in 11 ms on localhost (2/4) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 74) in 13 ms on localhost (3/4) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 76) in 12 ms on localhost (4/4) -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-23 20:39:31 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at RowMatrix.scala:332) finished in 0.014 s -2016-05-23 20:39:31 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at RowMatrix.scala:332, took 0.026233 s -2016-05-23 20:39:31 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:39:31 INFO DAGScheduler:59 - Got job 25 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:39:31 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:39:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:39:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:31 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:39:31 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=413052, maxMem=766075207 -2016-05-23 20:39:31 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 20:39:31 INFO MemoryStore:59 - ensureFreeSpace(1916) called with curMem=416492, maxMem=766075207 -2016-05-23 20:39:31 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1916.0 B, free 730.2 MB) -2016-05-23 20:39:31 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:53327 (size: 1916.0 B, free: 730.5 MB) -2016-05-23 20:39:31 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 39 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 4 tasks -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 79, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 80, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 39.0 (TID 81, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:39:31 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 78) -2016-05-23 20:39:31 INFO Executor:59 - Running task 3.0 in stage 39.0 (TID 81) -2016-05-23 20:39:31 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 79) -2016-05-23 20:39:31 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 78). 1109 bytes result sent to driver -2016-05-23 20:39:31 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 79). 1109 bytes result sent to driver -2016-05-23 20:39:31 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 80) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 78) in 9 ms on localhost (1/4) -2016-05-23 20:39:31 INFO Executor:59 - Finished task 3.0 in stage 39.0 (TID 81). 1109 bytes result sent to driver -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 79) in 10 ms on localhost (2/4) -2016-05-23 20:39:31 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 80). 1109 bytes result sent to driver -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 39.0 (TID 81) in 11 ms on localhost (3/4) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 80) in 12 ms on localhost (4/4) -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-23 20:39:31 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at RowMatrix.scala:124) finished in 0.014 s -2016-05-23 20:39:31 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at RowMatrix.scala:124, took 0.028472 s -2016-05-23 20:39:31 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:39:31 INFO DAGScheduler:59 - Got job 26 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:39:31 INFO DAGScheduler:59 - Final stage: ResultStage 40(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:39:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:39:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:31 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:39:31 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=418408, maxMem=766075207 -2016-05-23 20:39:31 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:39:31 INFO MemoryStore:59 - ensureFreeSpace(1851) called with curMem=421784, maxMem=766075207 -2016-05-23 20:39:31 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 1851.0 B, free 730.2 MB) -2016-05-23 20:39:31 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:53327 (size: 1851.0 B, free: 730.5 MB) -2016-05-23 20:39:31 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 40 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 4 tasks -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 82, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 83, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 84, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 40.0 (TID 85, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:39:31 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 82) -2016-05-23 20:39:31 INFO Executor:59 - Running task 3.0 in stage 40.0 (TID 85) -2016-05-23 20:39:31 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 84) -2016-05-23 20:39:31 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 83) -2016-05-23 20:39:31 INFO Executor:59 - Finished task 3.0 in stage 40.0 (TID 85). 1254 bytes result sent to driver -2016-05-23 20:39:31 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 84). 1109 bytes result sent to driver -2016-05-23 20:39:31 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 83). 1254 bytes result sent to driver -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 40.0 (TID 85) in 6 ms on localhost (1/4) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 84) in 8 ms on localhost (2/4) -2016-05-23 20:39:31 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 82). 1104 bytes result sent to driver -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 83) in 8 ms on localhost (3/4) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 82) in 10 ms on localhost (4/4) -2016-05-23 20:39:31 INFO DAGScheduler:59 - ResultStage 40 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.010 s -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-23 20:39:31 INFO DAGScheduler:59 - Job 26 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.021169 s -2016-05-23 20:39:31 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:39:31 INFO DAGScheduler:59 - Registering RDD 51 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:39:31 INFO DAGScheduler:59 - Got job 27 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:39:31 INFO DAGScheduler:59 - Final stage: ResultStage 42(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:39:31 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 41) -2016-05-23 20:39:31 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 41) -2016-05-23 20:39:31 INFO DAGScheduler:59 - Submitting ShuffleMapStage 41 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:39:31 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=423635, maxMem=766075207 -2016-05-23 20:39:31 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 20:39:31 INFO MemoryStore:59 - ensureFreeSpace(2284) called with curMem=427675, maxMem=766075207 -2016-05-23 20:39:31 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:39:31 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:53327 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:39:31 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 41 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 4 tasks -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 86, localhost, PROCESS_LOCAL, 2221 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 87, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 88, localhost, PROCESS_LOCAL, 2221 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 41.0 (TID 89, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 20:39:31 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 86) -2016-05-23 20:39:31 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 88) -2016-05-23 20:39:31 INFO Executor:59 - Running task 3.0 in stage 41.0 (TID 89) -2016-05-23 20:39:31 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 87) -2016-05-23 20:39:31 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 86). 1161 bytes result sent to driver -2016-05-23 20:39:31 INFO Executor:59 - Finished task 2.0 in stage 41.0 (TID 88). 1161 bytes result sent to driver -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 86) in 9 ms on localhost (1/4) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 41.0 (TID 88) in 10 ms on localhost (2/4) -2016-05-23 20:39:31 INFO Executor:59 - Finished task 3.0 in stage 41.0 (TID 89). 1161 bytes result sent to driver -2016-05-23 20:39:31 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 87). 1161 bytes result sent to driver -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 41.0 (TID 89) in 34 ms on localhost (3/4) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 87) in 35 ms on localhost (4/4) -2016-05-23 20:39:31 INFO DAGScheduler:59 - ShuffleMapStage 41 (flatMap at SpearmanCorrelation.scala:49) finished in 0.036 s -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-23 20:39:31 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:39:31 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:39:31 INFO DAGScheduler:59 - waiting: Set(ResultStage 42) -2016-05-23 20:39:31 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:39:31 INFO DAGScheduler:59 - Missing parents for ResultStage 42: List() -2016-05-23 20:39:31 INFO DAGScheduler:59 - Submitting ResultStage 42 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:39:31 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=429959, maxMem=766075207 -2016-05-23 20:39:31 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:39:31 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=432503, maxMem=766075207 -2016-05-23 20:39:31 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 20:39:31 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:53327 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 20:39:31 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:31 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 42 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 3 tasks -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 90, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 91, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 42.0 (TID 92, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:31 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 90) -2016-05-23 20:39:31 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 91) -2016-05-23 20:39:31 INFO Executor:59 - Running task 2.0 in stage 42.0 (TID 92) -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:31 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 91). 1203 bytes result sent to driver -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 91) in 8 ms on localhost (1/3) -2016-05-23 20:39:31 INFO Executor:59 - Finished task 2.0 in stage 42.0 (TID 92). 1203 bytes result sent to driver -2016-05-23 20:39:31 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 90). 1203 bytes result sent to driver -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 42.0 (TID 92) in 9 ms on localhost (2/3) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 90) in 11 ms on localhost (3/3) -2016-05-23 20:39:31 INFO DAGScheduler:59 - ResultStage 42 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.011 s -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-23 20:39:31 INFO DAGScheduler:59 - Job 27 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.069679 s -2016-05-23 20:39:31 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:39:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 165 bytes -2016-05-23 20:39:31 INFO DAGScheduler:59 - Registering RDD 56 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:39:31 INFO DAGScheduler:59 - Got job 28 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:39:31 INFO DAGScheduler:59 - Final stage: ResultStage 45(first at RowMatrix.scala:65) -2016-05-23 20:39:31 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 44) -2016-05-23 20:39:31 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 44) -2016-05-23 20:39:31 INFO DAGScheduler:59 - Submitting ShuffleMapStage 44 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:39:31 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=434071, maxMem=766075207 -2016-05-23 20:39:31 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:39:31 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=438303, maxMem=766075207 -2016-05-23 20:39:31 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 20:39:31 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:53327 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:39:31 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 44 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 4 tasks -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 93, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 94, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 44.0 (TID 95, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 44.0 (TID 96, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:39:31 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 93) -2016-05-23 20:39:31 INFO Executor:59 - Running task 3.0 in stage 44.0 (TID 96) -2016-05-23 20:39:31 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 94) -2016-05-23 20:39:31 INFO Executor:59 - Running task 2.0 in stage 44.0 (TID 95) -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:31 INFO Executor:59 - Finished task 3.0 in stage 44.0 (TID 96). 1377 bytes result sent to driver -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 44.0 (TID 96) in 13 ms on localhost (1/4) -2016-05-23 20:39:31 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 93). 1377 bytes result sent to driver -2016-05-23 20:39:31 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 94). 1377 bytes result sent to driver -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 93) in 50 ms on localhost (2/4) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 94) in 51 ms on localhost (3/4) -2016-05-23 20:39:31 INFO Executor:59 - Finished task 2.0 in stage 44.0 (TID 95). 1377 bytes result sent to driver -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 44.0 (TID 95) in 52 ms on localhost (4/4) -2016-05-23 20:39:31 INFO DAGScheduler:59 - ShuffleMapStage 44 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.054 s -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-23 20:39:31 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:39:31 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:39:31 INFO DAGScheduler:59 - waiting: Set(ResultStage 45) -2016-05-23 20:39:31 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:39:31 INFO DAGScheduler:59 - Missing parents for ResultStage 45: List() -2016-05-23 20:39:31 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:39:31 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=440609, maxMem=766075207 -2016-05-23 20:39:31 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 20:39:31 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=445505, maxMem=766075207 -2016-05-23 20:39:31 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:39:31 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:53327 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:39:31 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 45 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 1 tasks -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 97, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:31 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 97) -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:39:31 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 97). 1185 bytes result sent to driver -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 97) in 7 ms on localhost (1/1) -2016-05-23 20:39:31 INFO DAGScheduler:59 - ResultStage 45 (first at RowMatrix.scala:65) finished in 0.008 s -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-23 20:39:31 INFO DAGScheduler:59 - Job 28 finished: first at RowMatrix.scala:65, took 0.083523 s -2016-05-23 20:39:31 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:39:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 165 bytes -2016-05-23 20:39:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 164 bytes -2016-05-23 20:39:31 INFO DAGScheduler:59 - Got job 29 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 20:39:31 INFO DAGScheduler:59 - Final stage: ResultStage 48(first at RowMatrix.scala:65) -2016-05-23 20:39:31 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 47) -2016-05-23 20:39:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:31 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which has no missing parents -2016-05-23 20:39:31 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=448052, maxMem=766075207 -2016-05-23 20:39:31 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 20:39:31 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=452948, maxMem=766075207 -2016-05-23 20:39:31 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:39:31 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:53327 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:39:31 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:31 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 48 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 3 tasks -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 98, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 99, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 48.0 (TID 100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:31 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 99) -2016-05-23 20:39:31 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 98) -2016-05-23 20:39:31 INFO Executor:59 - Running task 2.0 in stage 48.0 (TID 100) -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:31 INFO Executor:59 - Finished task 2.0 in stage 48.0 (TID 100). 1296 bytes result sent to driver -2016-05-23 20:39:31 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 99). 1185 bytes result sent to driver -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 48.0 (TID 100) in 10 ms on localhost (1/3) -2016-05-23 20:39:31 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 98). 1296 bytes result sent to driver -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 99) in 11 ms on localhost (2/3) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 98) in 13 ms on localhost (3/3) -2016-05-23 20:39:31 INFO DAGScheduler:59 - ResultStage 48 (first at RowMatrix.scala:65) finished in 0.013 s -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-23 20:39:31 INFO DAGScheduler:59 - Job 29 finished: first at RowMatrix.scala:65, took 0.025981 s -2016-05-23 20:39:31 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:39:31 INFO DAGScheduler:59 - Got job 30 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:39:31 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:39:31 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 50) -2016-05-23 20:39:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:31 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:39:31 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=455495, maxMem=766075207 -2016-05-23 20:39:31 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 20:39:31 INFO MemoryStore:59 - ensureFreeSpace(2917) called with curMem=461191, maxMem=766075207 -2016-05-23 20:39:31 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:39:31 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:53327 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:39:31 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 51 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 4 tasks -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 51.0 (TID 103, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 51.0 (TID 104, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:31 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-23 20:39:31 INFO Executor:59 - Running task 3.0 in stage 51.0 (TID 104) -2016-05-23 20:39:31 INFO Executor:59 - Running task 2.0 in stage 51.0 (TID 103) -2016-05-23 20:39:31 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:39:31 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 1481 bytes result sent to driver -2016-05-23 20:39:31 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 1481 bytes result sent to driver -2016-05-23 20:39:31 INFO Executor:59 - Finished task 3.0 in stage 51.0 (TID 104). 1481 bytes result sent to driver -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 14 ms on localhost (1/4) -2016-05-23 20:39:31 INFO Executor:59 - Finished task 2.0 in stage 51.0 (TID 103). 1481 bytes result sent to driver -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 16 ms on localhost (2/4) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 51.0 (TID 104) in 15 ms on localhost (3/4) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 51.0 (TID 103) in 17 ms on localhost (4/4) -2016-05-23 20:39:31 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at RowMatrix.scala:332) finished in 0.018 s -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-23 20:39:31 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at RowMatrix.scala:332, took 0.028983 s -2016-05-23 20:39:31 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:39:31 INFO DAGScheduler:59 - Got job 31 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:39:31 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:39:31 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 53) -2016-05-23 20:39:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:39:31 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:39:31 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=464108, maxMem=766075207 -2016-05-23 20:39:31 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 20:39:31 INFO MemoryStore:59 - ensureFreeSpace(2899) called with curMem=469844, maxMem=766075207 -2016-05-23 20:39:31 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:39:31 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:53327 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:39:31 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:39:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 54 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 4 tasks -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 105, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 106, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:39:31 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 106) -2016-05-23 20:39:31 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 105) -2016-05-23 20:39:31 INFO Executor:59 - Running task 2.0 in stage 54.0 (TID 107) -2016-05-23 20:39:31 INFO Executor:59 - Running task 3.0 in stage 54.0 (TID 108) -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:39:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:39:31 INFO Executor:59 - Finished task 3.0 in stage 54.0 (TID 108). 1359 bytes result sent to driver -2016-05-23 20:39:31 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 105). 1359 bytes result sent to driver -2016-05-23 20:39:31 INFO Executor:59 - Finished task 2.0 in stage 54.0 (TID 107). 1359 bytes result sent to driver -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 54.0 (TID 108) in 12 ms on localhost (1/4) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 105) in 14 ms on localhost (2/4) -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 54.0 (TID 107) in 15 ms on localhost (3/4) -2016-05-23 20:39:31 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 106). 1359 bytes result sent to driver -2016-05-23 20:39:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 106) in 16 ms on localhost (4/4) -2016-05-23 20:39:31 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at RowMatrix.scala:124) finished in 0.017 s -2016-05-23 20:39:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-23 20:39:31 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at RowMatrix.scala:124, took 0.028012 s -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 20:39:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 20:39:31 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 20:39:31 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 20:39:31 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 20:39:31 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 20:39:31 INFO BlockManager:59 - BlockManager stopped -2016-05-23 20:39:31 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 20:39:31 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 20:39:31 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 20:39:31 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 20:39:31 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-b894b55b-acea-487f-a709-35378595727a -2016-05-23 20:39:31 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 20:41:19 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 20:41:20 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 20:41:20 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 20:41:20 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 20:41:20 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 20:41:21 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 20:41:21 INFO Remoting:74 - Starting remoting -2016-05-23 20:41:21 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53381] -2016-05-23 20:41:21 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53381. -2016-05-23 20:41:22 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 20:41:22 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 20:41:22 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-9fbaba20-0329-4f24-8d60-fa22a3c80e72 -2016-05-23 20:41:22 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 20:41:22 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-3c9d6ce7-3c11-4774-af0b-60a4c0613d83\httpd-be9e6a68-f551-4735-9d30-22ff8830816b -2016-05-23 20:41:22 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 20:41:22 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:41:22 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53382 -2016-05-23 20:41:22 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53382. -2016-05-23 20:41:22 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 20:41:22 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:41:22 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 20:41:22 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 20:41:22 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 20:41:22 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 20:41:22 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 20:41:22 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53401. -2016-05-23 20:41:22 INFO NettyBlockTransferService:59 - Server created on 53401 -2016-05-23 20:41:22 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 20:41:22 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53401 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53401) -2016-05-23 20:41:22 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 20:41:23 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 20:41:23 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 20:41:23 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 20:41:23 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 20:41:23 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53401 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:41:23 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:20 -2016-05-23 20:41:23 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 20:41:23 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 20:41:23 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 20:41:23 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 20:41:23 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53401 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:41:23 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 20:41:25 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 20:41:25 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:41:25 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:27 -2016-05-23 20:41:25 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:27) with 2 output partitions -2016-05-23 20:41:25 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:27) -2016-05-23 20:41:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:41:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:25 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21), which has no missing parents -2016-05-23 20:41:25 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 20:41:25 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:41:25 INFO MemoryStore:59 - ensureFreeSpace(2047) called with curMem=346712, maxMem=766075207 -2016-05-23 20:41:25 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2047.0 B, free 730.3 MB) -2016-05-23 20:41:25 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53401 (size: 2047.0 B, free: 730.6 MB) -2016-05-23 20:41:25 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21) -2016-05-23 20:41:25 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 20:41:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:41:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:41:25 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 20:41:25 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 20:41:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:41:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:41:25 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 20:41:25 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 20:41:25 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 20:41:25 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 20:41:25 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 20:41:25 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 20:41:25 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 20:41:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 146 ms on localhost (1/2) -2016-05-23 20:41:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 121 ms on localhost (2/2) -2016-05-23 20:41:25 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:27) finished in 0.173 s -2016-05-23 20:41:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 20:41:25 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:27, took 0.340675 s -2016-05-23 20:41:25 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:41:25 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 20:41:25 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 20:41:25 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 20:41:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:41:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:25 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24), which has no missing parents -2016-05-23 20:41:25 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348759, maxMem=766075207 -2016-05-23 20:41:25 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:41:25 INFO MemoryStore:59 - ensureFreeSpace(2045) called with curMem=352271, maxMem=766075207 -2016-05-23 20:41:25 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2045.0 B, free 730.2 MB) -2016-05-23 20:41:25 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53401 (size: 2045.0 B, free: 730.6 MB) -2016-05-23 20:41:25 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24) -2016-05-23 20:41:25 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 20:41:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:41:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:41:25 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 20:41:25 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 20:41:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:41:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:41:25 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 20:41:25 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 20:41:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 22 ms on localhost (1/2) -2016-05-23 20:41:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 25 ms on localhost (2/2) -2016-05-23 20:41:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 20:41:25 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:29) finished in 0.027 s -2016-05-23 20:41:25 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:29, took 0.048581 s -2016-05-23 20:41:25 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:41:25 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:41:25 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 20:41:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:41:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:25 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:41:25 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354316, maxMem=766075207 -2016-05-23 20:41:25 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 20:41:25 INFO MemoryStore:59 - ensureFreeSpace(2465) called with curMem=358876, maxMem=766075207 -2016-05-23 20:41:25 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:41:25 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53401 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:41:25 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 20:41:25 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 20:41:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:41:25 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 20:41:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:41:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:41:25 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53401 in memory (size: 2045.0 B, free: 730.6 MB) -2016-05-23 20:41:25 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 20:41:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 53 ms on localhost (1/1) -2016-05-23 20:41:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 20:41:25 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.055 s -2016-05-23 20:41:25 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.105940 s -2016-05-23 20:41:25 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53401 in memory (size: 2047.0 B, free: 730.6 MB) -2016-05-23 20:41:26 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:41:26 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:41:26 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:41:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:41:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:26 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:41:26 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=350225, maxMem=766075207 -2016-05-23 20:41:26 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 20:41:26 INFO MemoryStore:59 - ensureFreeSpace(2862) called with curMem=355673, maxMem=766075207 -2016-05-23 20:41:26 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:41:26 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53401 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:41:26 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:41:26 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 20:41:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:41:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:41:26 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 20:41:26 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 20:41:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:41:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:41:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:41:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:41:26 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 20:41:26 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 20:41:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 43 ms on localhost (1/2) -2016-05-23 20:41:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 47 ms on localhost (2/2) -2016-05-23 20:41:26 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.047 s -2016-05-23 20:41:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 20:41:26 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 20:41:26 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 20:41:26 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.078020 s -2016-05-23 20:41:26 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:41:26 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:41:26 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:41:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:41:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:26 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:41:26 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=358535, maxMem=766075207 -2016-05-23 20:41:26 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 20:41:26 INFO MemoryStore:59 - ensureFreeSpace(2851) called with curMem=364031, maxMem=766075207 -2016-05-23 20:41:26 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:41:26 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53401 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:41:26 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:41:26 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 20:41:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:41:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:41:26 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 20:41:26 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 20:41:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:41:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:41:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:41:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:41:26 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 20:41:26 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 20:41:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 37 ms on localhost (1/2) -2016-05-23 20:41:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 35 ms on localhost (2/2) -2016-05-23 20:41:26 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.038 s -2016-05-23 20:41:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 20:41:26 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.063133 s -2016-05-23 20:41:26 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:41:26 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 20:41:26 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:41:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:41:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:26 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:41:26 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=366882, maxMem=766075207 -2016-05-23 20:41:26 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 20:41:26 INFO MemoryStore:59 - ensureFreeSpace(2744) called with curMem=372226, maxMem=766075207 -2016-05-23 20:41:26 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 20:41:26 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53401 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 20:41:26 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:41:26 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 20:41:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:41:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:41:26 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 20:41:26 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 20:41:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:41:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:41:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:41:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:41:26 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 20:41:26 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 20:41:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 37 ms on localhost (1/2) -2016-05-23 20:41:26 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.038 s -2016-05-23 20:41:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 36 ms on localhost (2/2) -2016-05-23 20:41:26 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.058244 s -2016-05-23 20:41:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 20:41:26 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:41:26 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:41:26 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 20:41:26 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:41:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 20:41:26 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 20:41:26 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:41:26 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=374970, maxMem=766075207 -2016-05-23 20:41:26 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 20:41:26 INFO MemoryStore:59 - ensureFreeSpace(3156) called with curMem=380898, maxMem=766075207 -2016-05-23 20:41:26 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 20:41:26 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53401 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:41:26 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:41:26 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 20:41:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:41:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:41:26 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 20:41:26 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 20:41:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:41:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:41:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:41:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:41:26 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 20:41:26 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 20:41:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 72 ms on localhost (1/2) -2016-05-23 20:41:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 69 ms on localhost (2/2) -2016-05-23 20:41:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 20:41:26 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.074 s -2016-05-23 20:41:26 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:41:26 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:41:26 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 20:41:26 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:41:26 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=384054, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=386558, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 20:41:27 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53401 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:41:27 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:27 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-23 20:41:27 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 55 ms on localhost (1/1) -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 20:41:27 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.056 s -2016-05-23 20:41:27 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.176451 s -2016-05-23 20:41:27 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:41:27 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53401 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:41:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:41:27 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:41:27 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53401 in memory (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:41:27 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 20:41:27 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:41:27 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53401 in memory (size: 2.7 KB, free: 730.6 MB) -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=366882, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:41:27 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=368223, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:41:27 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53401 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:41:27 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 20:41:27 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53401 (size: 2.2 KB, free: 730.6 MB) -2016-05-23 20:41:27 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 20:41:27 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53401 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:41:27 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:41:27 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 20:41:27 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:27 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 20:41:27 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 42 ms on localhost (1/2) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 41 ms on localhost (2/2) -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 20:41:27 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.044 s -2016-05-23 20:41:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:41:27 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:41:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 20:41:27 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:41:27 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=356699, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=361555, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:41:27 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53401 (size: 2.5 KB, free: 730.6 MB) -2016-05-23 20:41:27 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:27 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:41:27 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 23 ms on localhost (1/1) -2016-05-23 20:41:27 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.023 s -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 20:41:27 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.108518 s -2016-05-23 20:41:27 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:41:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:41:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 20:41:27 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:41:27 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=364086, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=369742, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:41:27 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53401 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:41:27 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:27 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 20:41:27 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:41:27 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 20:41:27 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 19 ms on localhost (1/2) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 20 ms on localhost (2/2) -2016-05-23 20:41:27 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.020 s -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 20:41:27 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.036291 s -2016-05-23 20:41:27 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:41:27 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=372650, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=378346, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:41:27 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53401 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:41:27 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:27 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 20:41:27 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:27 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 20:41:27 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 21 ms on localhost (1/2) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 21 ms on localhost (2/2) -2016-05-23 20:41:27 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.022 s -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 20:41:27 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.042216 s -2016-05-23 20:41:27 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:43 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:43) with 4 output partitions -2016-05-23 20:41:27 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:43) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:41:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40), which has no missing parents -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=381235, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=382459, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:41:27 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53401 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:41:27 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40) -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:41:27 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 20:41:27 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 20:41:27 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 20:41:27 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 20:41:27 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 20:41:27 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 19 ms on localhost (1/4) -2016-05-23 20:41:27 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 20:41:27 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 12 ms on localhost (2/4) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 13 ms on localhost (3/4) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 16 ms on localhost (4/4) -2016-05-23 20:41:27 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:43) finished in 0.024 s -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 20:41:27 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:43, took 0.038976 s -2016-05-23 20:41:27 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:45 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:45) with 4 output partitions -2016-05-23 20:41:27 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:45) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:41:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41), which has no missing parents -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=383352, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=384576, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:41:27 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53401 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:41:27 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41) -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:41:27 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 20:41:27 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 20:41:27 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 20:41:27 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 20:41:27 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 20:41:27 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 9 ms on localhost (1/4) -2016-05-23 20:41:27 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 20:41:27 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 12 ms on localhost (2/4) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 11 ms on localhost (3/4) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 14 ms on localhost (4/4) -2016-05-23 20:41:27 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:45) finished in 0.015 s -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 20:41:27 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:45, took 0.024709 s -2016-05-23 20:41:27 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:41:27 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:41:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=385469, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=387973, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 20:41:27 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53401 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 20:41:27 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:41:27 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 20:41:27 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 20:41:27 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.008 s -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 8 ms on localhost (1/1) -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 20:41:27 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.023073 s -2016-05-23 20:41:27 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:41:27 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:41:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=389497, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=392897, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 20:41:27 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53401 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 20:41:27 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:41:27 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 20:41:27 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 20:41:27 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 20:41:27 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 20:41:27 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 20:41:27 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 20:41:27 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 13 ms on localhost (1/4) -2016-05-23 20:41:27 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 16 ms on localhost (2/4) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 16 ms on localhost (3/4) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 20 ms on localhost (4/4) -2016-05-23 20:41:27 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.021 s -2016-05-23 20:41:27 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.034372 s -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 20:41:27 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:41:27 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:41:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=394831, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=398271, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 20:41:27 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53401 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 20:41:27 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:41:27 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 20:41:27 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 20:41:27 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 20:41:27 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 20:41:27 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 20:41:27 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 20:41:27 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 12 ms on localhost (1/4) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 15 ms on localhost (2/4) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 13 ms on localhost (3/4) -2016-05-23 20:41:27 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 20:41:27 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.018 s -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 17 ms on localhost (4/4) -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 20:41:27 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.035368 s -2016-05-23 20:41:27 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:41:27 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:41:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=400189, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(1860) called with curMem=403565, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1860.0 B, free 730.2 MB) -2016-05-23 20:41:27 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53401 (size: 1860.0 B, free: 730.5 MB) -2016-05-23 20:41:27 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:41:27 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 20:41:27 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 20:41:27 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 20:41:27 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 20:41:27 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 20:41:27 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 20:41:27 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 20:41:27 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 19 ms on localhost (1/4) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 20 ms on localhost (2/4) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 25 ms on localhost (3/4) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 24 ms on localhost (4/4) -2016-05-23 20:41:27 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.026 s -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 20:41:27 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.043541 s -2016-05-23 20:41:27 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:41:27 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=405425, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=409465, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:41:27 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:53401 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:41:27 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:41:27 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 20:41:27 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 20:41:27 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 20:41:27 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 44 ms on localhost (1/4) -2016-05-23 20:41:27 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 20:41:27 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 20:41:27 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 67 ms on localhost (2/4) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 69 ms on localhost (3/4) -2016-05-23 20:41:27 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 89 ms on localhost (4/4) -2016-05-23 20:41:27 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.093 s -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 20:41:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:41:27 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:41:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 20:41:27 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:41:27 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=411747, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=414291, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 20:41:27 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:53401 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 20:41:27 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:27 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 20:41:27 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:41:27 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 20:41:27 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:41:27 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 29 ms on localhost (1/3) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 29 ms on localhost (2/3) -2016-05-23 20:41:27 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 20:41:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 33 ms on localhost (3/3) -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 20:41:27 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.038 s -2016-05-23 20:41:27 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.176139 s -2016-05-23 20:41:27 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:41:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:41:27 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:41:27 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=415859, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:41:27 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=420091, maxMem=766075207 -2016-05-23 20:41:27 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 20:41:27 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:53401 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:41:27 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:41:27 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:41:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:41:27 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 20:41:27 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 20:41:27 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 20:41:27 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:41:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:41:27 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 20:41:27 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 94 ms on localhost (1/4) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 104 ms on localhost (2/4) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 121 ms on localhost (3/4) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 129 ms on localhost (4/4) -2016-05-23 20:41:28 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.133 s -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 20:41:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:41:28 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:41:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 20:41:28 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=422397, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=427293, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:53401 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:41:28 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:28 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 17 ms on localhost (1/1) -2016-05-23 20:41:28 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.019 s -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 20:41:28 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.190776 s -2016-05-23 20:41:28 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:41:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:41:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 20:41:28 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:41:28 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=429835, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=435531, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:53401 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:41:28 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 20:41:28 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 20:41:28 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 20:41:28 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:41:28 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:28 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 40 ms on localhost (1/4) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 40 ms on localhost (2/4) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 40 ms on localhost (3/4) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 43 ms on localhost (4/4) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 20:41:28 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.044 s -2016-05-23 20:41:28 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.060165 s -2016-05-23 20:41:28 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:41:28 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=438440, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=444176, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:53401 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:41:28 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 20:41:28 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 20:41:28 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:41:28 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:28 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 32 ms on localhost (1/4) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 32 ms on localhost (2/4) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 34 ms on localhost (3/4) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 46 ms on localhost (4/4) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 20:41:28 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.049 s -2016-05-23 20:41:28 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.070508 s -2016-05-23 20:41:28 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:57 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Got job 20 (foreach at StatisticsCorrLearning.scala:57) with 4 output partitions -2016-05-23 20:41:28 INFO DAGScheduler:59 - Final stage: ResultStage 34(foreach at StatisticsCorrLearning.scala:57) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:54), which has no missing parents -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=447068, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=448292, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:53401 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:41:28 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:54) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 62, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 63, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 64, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 20:41:28 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 63) -2016-05-23 20:41:28 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 62) -2016-05-23 20:41:28 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 64) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 63). 915 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 65) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 64). 915 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 65). 915 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 63) in 10 ms on localhost (1/4) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 64) in 10 ms on localhost (2/4) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 62). 915 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 65) in 11 ms on localhost (3/4) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 62) in 16 ms on localhost (4/4) -2016-05-23 20:41:28 INFO DAGScheduler:59 - ResultStage 34 (foreach at StatisticsCorrLearning.scala:57) finished in 0.016 s -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-23 20:41:28 INFO DAGScheduler:59 - Job 20 finished: foreach at StatisticsCorrLearning.scala:57, took 0.030291 s -2016-05-23 20:41:28 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:59 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Got job 21 (foreach at StatisticsCorrLearning.scala:59) with 4 output partitions -2016-05-23 20:41:28 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at StatisticsCorrLearning.scala:59) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:55), which has no missing parents -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=449185, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=450409, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:53401 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:41:28 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:55) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 4 tasks -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 66, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 20:41:28 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 67) -2016-05-23 20:41:28 INFO Executor:59 - Running task 3.0 in stage 35.0 (TID 69) -2016-05-23 20:41:28 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 68) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 67). 915 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 66) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 68). 915 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 66). 915 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 67) in 11 ms on localhost (1/4) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 3.0 in stage 35.0 (TID 69). 915 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 68) in 11 ms on localhost (2/4) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 66) in 15 ms on localhost (3/4) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 35.0 (TID 69) in 14 ms on localhost (4/4) -2016-05-23 20:41:28 INFO DAGScheduler:59 - ResultStage 35 (foreach at StatisticsCorrLearning.scala:59) finished in 0.018 s -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-23 20:41:28 INFO DAGScheduler:59 - Job 21 finished: foreach at StatisticsCorrLearning.scala:59, took 0.029139 s -2016-05-23 20:41:28 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Got job 22 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:41:28 INFO DAGScheduler:59 - Final stage: ResultStage 36(first at RowMatrix.scala:65) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=451302, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(1522) called with curMem=453806, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1522.0 B, free 730.2 MB) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:53401 (size: 1522.0 B, free: 730.5 MB) -2016-05-23 20:41:28 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:41:28 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 70) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 70). 935 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 70) in 6 ms on localhost (1/1) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-23 20:41:28 INFO DAGScheduler:59 - ResultStage 36 (first at RowMatrix.scala:65) finished in 0.006 s -2016-05-23 20:41:28 INFO DAGScheduler:59 - Job 22 finished: first at RowMatrix.scala:65, took 0.023455 s -2016-05-23 20:41:28 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Got job 23 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 20:41:28 INFO DAGScheduler:59 - Final stage: ResultStage 37(first at RowMatrix.scala:65) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=455328, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(1522) called with curMem=457832, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1522.0 B, free 730.1 MB) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:53401 (size: 1522.0 B, free: 730.5 MB) -2016-05-23 20:41:28 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 37 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 3 tasks -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:41:28 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 73) -2016-05-23 20:41:28 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 73). 1046 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 1046 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 72) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 73) in 8 ms on localhost (1/3) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 72). 935 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 13 ms on localhost (2/3) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 72) in 13 ms on localhost (3/3) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-23 20:41:28 INFO DAGScheduler:59 - ResultStage 37 (first at RowMatrix.scala:65) finished in 0.015 s -2016-05-23 20:41:28 INFO DAGScheduler:59 - Job 23 finished: first at RowMatrix.scala:65, took 0.028367 s -2016-05-23 20:41:28 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Got job 24 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:41:28 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=459354, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(1932) called with curMem=462754, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1932.0 B, free 730.1 MB) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:53401 (size: 1932.0 B, free: 730.5 MB) -2016-05-23 20:41:28 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 74, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 77, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:41:28 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 74) -2016-05-23 20:41:28 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 75) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 74). 1231 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 75). 1231 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 76) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 74) in 16 ms on localhost (1/4) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 75) in 17 ms on localhost (2/4) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 76). 1231 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 77) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 76) in 18 ms on localhost (3/4) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 77). 1231 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 77) in 21 ms on localhost (4/4) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-23 20:41:28 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at RowMatrix.scala:332) finished in 0.024 s -2016-05-23 20:41:28 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at RowMatrix.scala:332, took 0.041562 s -2016-05-23 20:41:28 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Got job 25 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:41:28 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=464686, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(1916) called with curMem=468126, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1916.0 B, free 730.1 MB) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:53401 (size: 1916.0 B, free: 730.5 MB) -2016-05-23 20:41:28 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 39 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 4 tasks -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 79, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 80, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 39.0 (TID 81, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:41:28 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 78) -2016-05-23 20:41:28 INFO Executor:59 - Running task 3.0 in stage 39.0 (TID 81) -2016-05-23 20:41:28 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 79) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 78). 1109 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 80) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 80). 1109 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Finished task 3.0 in stage 39.0 (TID 81). 1109 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 79). 1109 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 78) in 19 ms on localhost (1/4) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 80) in 18 ms on localhost (2/4) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 39.0 (TID 81) in 18 ms on localhost (3/4) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 79) in 20 ms on localhost (4/4) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-23 20:41:28 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at RowMatrix.scala:124) finished in 0.022 s -2016-05-23 20:41:28 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at RowMatrix.scala:124, took 0.034572 s -2016-05-23 20:41:28 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Got job 26 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:41:28 INFO DAGScheduler:59 - Final stage: ResultStage 40(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=470042, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(1851) called with curMem=473418, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 1851.0 B, free 730.1 MB) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:53401 (size: 1851.0 B, free: 730.5 MB) -2016-05-23 20:41:28 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 40 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 4 tasks -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 82, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 83, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 84, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 40.0 (TID 85, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:41:28 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 83) -2016-05-23 20:41:28 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 84) -2016-05-23 20:41:28 INFO Executor:59 - Running task 3.0 in stage 40.0 (TID 85) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 84). 1109 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 82) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 3.0 in stage 40.0 (TID 85). 1254 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 84) in 9 ms on localhost (1/4) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 82). 1104 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 40.0 (TID 85) in 10 ms on localhost (2/4) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 82) in 13 ms on localhost (3/4) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 83). 1254 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 83) in 15 ms on localhost (4/4) -2016-05-23 20:41:28 INFO DAGScheduler:59 - ResultStage 40 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.016 s -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-23 20:41:28 INFO DAGScheduler:59 - Job 26 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.030053 s -2016-05-23 20:41:28 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Registering RDD 51 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Got job 27 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:41:28 INFO DAGScheduler:59 - Final stage: ResultStage 42(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 41) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 41) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 41 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=475269, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(2283) called with curMem=479309, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:53401 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:41:28 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 41 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 4 tasks -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 86, localhost, PROCESS_LOCAL, 2221 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 87, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 88, localhost, PROCESS_LOCAL, 2221 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 41.0 (TID 89, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 20:41:28 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 87) -2016-05-23 20:41:28 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 88) -2016-05-23 20:41:28 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 86) -2016-05-23 20:41:28 INFO Executor:59 - Running task 3.0 in stage 41.0 (TID 89) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 2.0 in stage 41.0 (TID 88). 1161 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 41.0 (TID 88) in 12 ms on localhost (1/4) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 86). 1161 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 87). 1161 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 86) in 40 ms on localhost (2/4) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 87) in 40 ms on localhost (3/4) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 3.0 in stage 41.0 (TID 89). 1161 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 41.0 (TID 89) in 41 ms on localhost (4/4) -2016-05-23 20:41:28 INFO DAGScheduler:59 - ShuffleMapStage 41 (flatMap at SpearmanCorrelation.scala:49) finished in 0.045 s -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-23 20:41:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:41:28 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:41:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 42) -2016-05-23 20:41:28 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Missing parents for ResultStage 42: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting ResultStage 42 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=481592, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(1569) called with curMem=484136, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 1569.0 B, free 730.1 MB) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:53401 (size: 1569.0 B, free: 730.5 MB) -2016-05-23 20:41:28 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 42 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 3 tasks -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 90, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 91, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 42.0 (TID 92, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 90) -2016-05-23 20:41:28 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 91) -2016-05-23 20:41:28 INFO Executor:59 - Running task 2.0 in stage 42.0 (TID 92) -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:28 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 90). 1203 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Finished task 2.0 in stage 42.0 (TID 92). 1203 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 90) in 18 ms on localhost (1/3) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 42.0 (TID 92) in 18 ms on localhost (2/3) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 91). 1203 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 91) in 20 ms on localhost (3/3) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-23 20:41:28 INFO DAGScheduler:59 - ResultStage 42 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.021 s -2016-05-23 20:41:28 INFO DAGScheduler:59 - Job 27 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.094324 s -2016-05-23 20:41:28 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:41:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 165 bytes -2016-05-23 20:41:28 INFO DAGScheduler:59 - Registering RDD 56 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Got job 28 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:41:28 INFO DAGScheduler:59 - Final stage: ResultStage 45(first at RowMatrix.scala:65) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 44) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 44) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 44 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=485705, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(2307) called with curMem=489937, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:53401 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:41:28 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 44 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 4 tasks -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 93, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 94, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 44.0 (TID 95, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 44.0 (TID 96, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:41:28 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 93) -2016-05-23 20:41:28 INFO Executor:59 - Running task 3.0 in stage 44.0 (TID 96) -2016-05-23 20:41:28 INFO Executor:59 - Running task 2.0 in stage 44.0 (TID 95) -2016-05-23 20:41:28 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 94) -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO Executor:59 - Finished task 3.0 in stage 44.0 (TID 96). 1377 bytes result sent to driver -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 44.0 (TID 96) in 13 ms on localhost (1/4) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 93). 1377 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 93) in 42 ms on localhost (2/4) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 94). 1377 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 94) in 60 ms on localhost (3/4) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 2.0 in stage 44.0 (TID 95). 1377 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 44.0 (TID 95) in 63 ms on localhost (4/4) -2016-05-23 20:41:28 INFO DAGScheduler:59 - ShuffleMapStage 44 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.064 s -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-23 20:41:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:41:28 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:41:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 45) -2016-05-23 20:41:28 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Missing parents for ResultStage 45: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=492244, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=497140, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:53401 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:41:28 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 45 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 1 tasks -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 97, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 97) -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:28 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 97). 1185 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 97) in 8 ms on localhost (1/1) -2016-05-23 20:41:28 INFO DAGScheduler:59 - ResultStage 45 (first at RowMatrix.scala:65) finished in 0.009 s -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-23 20:41:28 INFO DAGScheduler:59 - Job 28 finished: first at RowMatrix.scala:65, took 0.095510 s -2016-05-23 20:41:28 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:41:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 165 bytes -2016-05-23 20:41:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 164 bytes -2016-05-23 20:41:28 INFO DAGScheduler:59 - Got job 29 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 20:41:28 INFO DAGScheduler:59 - Final stage: ResultStage 48(first at RowMatrix.scala:65) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 47) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which has no missing parents -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=499687, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=504583, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:53401 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:41:28 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 48 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 3 tasks -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 98, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 99, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 48.0 (TID 100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 98) -2016-05-23 20:41:28 INFO Executor:59 - Running task 2.0 in stage 48.0 (TID 100) -2016-05-23 20:41:28 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 99) -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:41:28 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 99). 1185 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 98). 1296 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Finished task 2.0 in stage 48.0 (TID 100). 1296 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 99) in 12 ms on localhost (1/3) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 98) in 13 ms on localhost (2/3) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 48.0 (TID 100) in 13 ms on localhost (3/3) -2016-05-23 20:41:28 INFO DAGScheduler:59 - ResultStage 48 (first at RowMatrix.scala:65) finished in 0.015 s -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-23 20:41:28 INFO DAGScheduler:59 - Job 29 finished: first at RowMatrix.scala:65, took 0.025760 s -2016-05-23 20:41:28 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Got job 30 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:41:28 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 50) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=507130, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(2916) called with curMem=512826, maxMem=766075207 -2016-05-23 20:41:28 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:53401 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:41:28 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 51 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:53401 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 4 tasks -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 51.0 (TID 103, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 51.0 (TID 104, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-23 20:41:28 INFO Executor:59 - Running task 3.0 in stage 51.0 (TID 104) -2016-05-23 20:41:28 INFO Executor:59 - Running task 2.0 in stage 51.0 (TID 103) -2016-05-23 20:41:28 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:53401 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:28 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:28 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-05-23 20:41:28 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 1481 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 15 ms on localhost (1/4) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 3.0 in stage 51.0 (TID 104). 1481 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 51.0 (TID 104) in 16 ms on localhost (2/4) -2016-05-23 20:41:28 INFO Executor:59 - Finished task 2.0 in stage 51.0 (TID 103). 1481 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 51.0 (TID 103) in 20 ms on localhost (3/4) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:53401 in memory (size: 1568.0 B, free: 730.5 MB) -2016-05-23 20:41:28 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-23 20:41:28 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 1481 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 26 ms on localhost (4/4) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-23 20:41:28 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at RowMatrix.scala:332) finished in 0.026 s -2016-05-23 20:41:28 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at RowMatrix.scala:332, took 0.059308 s -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:53401 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:41:28 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-23 20:41:28 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:53401 in memory (size: 1860.0 B, free: 730.5 MB) -2016-05-23 20:41:28 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:53401 in memory (size: 1918.0 B, free: 730.5 MB) -2016-05-23 20:41:28 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:41:28 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Got job 31 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:41:28 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 53) -2016-05-23 20:41:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=478804, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:53401 in memory (size: 1934.0 B, free: 730.5 MB) -2016-05-23 20:41:28 INFO MemoryStore:59 - ensureFreeSpace(2898) called with curMem=484540, maxMem=766075207 -2016-05-23 20:41:28 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:41:28 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:53401 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:41:28 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:41:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 54 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 4 tasks -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 105, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:53401 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 106, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:41:28 INFO Executor:59 - Running task 3.0 in stage 54.0 (TID 108) -2016-05-23 20:41:28 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-23 20:41:28 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 105) -2016-05-23 20:41:28 INFO Executor:59 - Running task 2.0 in stage 54.0 (TID 107) -2016-05-23 20:41:28 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 106) -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:28 INFO Executor:59 - Finished task 3.0 in stage 54.0 (TID 108). 1359 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 106). 1359 bytes result sent to driver -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:41:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:41:28 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 105). 1359 bytes result sent to driver -2016-05-23 20:41:28 INFO Executor:59 - Finished task 2.0 in stage 54.0 (TID 107). 1359 bytes result sent to driver -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 54.0 (TID 108) in 22 ms on localhost (1/4) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 106) in 25 ms on localhost (2/4) -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:53401 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 105) in 27 ms on localhost (3/4) -2016-05-23 20:41:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 54.0 (TID 107) in 25 ms on localhost (4/4) -2016-05-23 20:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-23 20:41:28 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at RowMatrix.scala:124) finished in 0.028 s -2016-05-23 20:41:28 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at RowMatrix.scala:124, took 0.045050 s -2016-05-23 20:41:28 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:53401 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:41:28 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:53401 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:41:28 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:53401 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:41:28 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-23 20:41:28 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:53401 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:41:28 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 20:41:29 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:53401 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 20:41:29 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 20:41:29 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 20:41:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 20:41:29 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-23 20:41:29 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:53401 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:41:29 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-23 20:41:29 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:53401 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:41:29 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-23 20:41:29 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:53401 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:41:29 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-23 20:41:29 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:53401 in memory (size: 1569.0 B, free: 730.5 MB) -2016-05-23 20:41:29 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-23 20:41:29 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:53401 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:41:29 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-23 20:41:29 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:53401 in memory (size: 1851.0 B, free: 730.5 MB) -2016-05-23 20:41:29 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-23 20:41:29 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:53401 in memory (size: 1916.0 B, free: 730.5 MB) -2016-05-23 20:41:29 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-23 20:41:29 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:53401 in memory (size: 1932.0 B, free: 730.5 MB) -2016-05-23 20:41:29 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-23 20:41:29 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:53401 in memory (size: 1522.0 B, free: 730.5 MB) -2016-05-23 20:41:29 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-23 20:41:29 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:53401 in memory (size: 1522.0 B, free: 730.5 MB) -2016-05-23 20:41:29 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-23 20:41:29 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:53401 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:41:29 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-23 20:41:29 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:53401 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:41:29 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-23 20:41:29 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:53401 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:41:29 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-23 20:41:29 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 20:41:29 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:53401 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:41:29 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 20:41:29 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 20:41:29 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 20:41:29 INFO BlockManager:59 - BlockManager stopped -2016-05-23 20:41:29 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 20:41:29 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 20:41:29 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 20:41:29 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 20:41:29 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-3c9d6ce7-3c11-4774-af0b-60a4c0613d83 -2016-05-23 20:57:38 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 20:57:39 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 20:57:39 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 20:57:39 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 20:57:39 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 20:57:40 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 20:57:40 INFO Remoting:74 - Starting remoting -2016-05-23 20:57:41 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53920] -2016-05-23 20:57:41 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53920. -2016-05-23 20:57:41 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 20:57:41 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 20:57:41 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-d3a3658c-b98b-4735-a369-03e7ddc66543 -2016-05-23 20:57:41 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 20:57:41 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-69b44dd3-cb31-4902-8c9f-fb949d5cb5a6\httpd-1287fece-937d-41c8-9512-df9704a24e51 -2016-05-23 20:57:41 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 20:57:41 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:57:41 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53921 -2016-05-23 20:57:41 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53921. -2016-05-23 20:57:41 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 20:57:41 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 20:57:41 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 20:57:41 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 20:57:41 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 20:57:41 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 20:57:41 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 20:57:42 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53940. -2016-05-23 20:57:42 INFO NettyBlockTransferService:59 - Server created on 53940 -2016-05-23 20:57:42 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 20:57:42 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53940 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53940) -2016-05-23 20:57:42 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 20:57:42 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 20:57:42 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 20:57:42 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 20:57:42 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 20:57:43 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53940 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:57:43 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:20 -2016-05-23 20:57:43 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 20:57:43 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 20:57:43 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 20:57:43 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 20:57:43 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53940 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 20:57:43 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 20:57:44 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 20:57:44 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:57:44 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:27 -2016-05-23 20:57:44 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:27) with 2 output partitions -2016-05-23 20:57:44 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:27) -2016-05-23 20:57:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:57:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:44 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21), which has no missing parents -2016-05-23 20:57:44 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 20:57:44 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:57:44 INFO MemoryStore:59 - ensureFreeSpace(2043) called with curMem=346712, maxMem=766075207 -2016-05-23 20:57:44 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2043.0 B, free 730.3 MB) -2016-05-23 20:57:44 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53940 (size: 2043.0 B, free: 730.6 MB) -2016-05-23 20:57:44 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:21) -2016-05-23 20:57:44 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 20:57:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:57:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:57:44 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 20:57:44 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 20:57:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:57:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:57:44 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 20:57:44 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 20:57:44 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 20:57:44 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 20:57:44 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 20:57:44 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 20:57:44 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 20:57:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 141 ms on localhost (1/2) -2016-05-23 20:57:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 115 ms on localhost (2/2) -2016-05-23 20:57:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 20:57:44 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:27) finished in 0.173 s -2016-05-23 20:57:44 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:27, took 0.335354 s -2016-05-23 20:57:45 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 20:57:45 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 20:57:45 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 20:57:45 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 20:57:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:57:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:45 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24), which has no missing parents -2016-05-23 20:57:45 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348755, maxMem=766075207 -2016-05-23 20:57:45 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 20:57:45 INFO MemoryStore:59 - ensureFreeSpace(2043) called with curMem=352267, maxMem=766075207 -2016-05-23 20:57:45 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2043.0 B, free 730.2 MB) -2016-05-23 20:57:45 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53940 (size: 2043.0 B, free: 730.6 MB) -2016-05-23 20:57:45 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:24) -2016-05-23 20:57:45 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 20:57:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:57:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 20:57:45 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 20:57:45 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 20:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:57:45 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 20:57:45 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 20:57:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 19 ms on localhost (1/2) -2016-05-23 20:57:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 23 ms on localhost (2/2) -2016-05-23 20:57:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 20:57:45 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:29) finished in 0.024 s -2016-05-23 20:57:45 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:29, took 0.044943 s -2016-05-23 20:57:45 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:57:45 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:57:45 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 20:57:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:57:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:45 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:57:45 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354310, maxMem=766075207 -2016-05-23 20:57:45 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 20:57:45 INFO MemoryStore:59 - ensureFreeSpace(2465) called with curMem=358870, maxMem=766075207 -2016-05-23 20:57:45 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:57:45 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53940 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 20:57:45 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:45 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 20:57:45 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 20:57:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:57:45 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 20:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:57:45 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 20:57:45 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.038 s -2016-05-23 20:57:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 38 ms on localhost (1/1) -2016-05-23 20:57:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 20:57:45 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.084709 s -2016-05-23 20:57:45 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:57:45 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:57:45 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:57:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:57:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:45 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:57:45 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=361335, maxMem=766075207 -2016-05-23 20:57:45 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 20:57:45 INFO MemoryStore:59 - ensureFreeSpace(2864) called with curMem=366783, maxMem=766075207 -2016-05-23 20:57:45 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:57:45 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53940 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 20:57:45 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:57:45 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 20:57:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:57:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:57:45 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 20:57:45 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 20:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:57:45 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 20:57:45 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 20:57:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 45 ms on localhost (1/2) -2016-05-23 20:57:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 49 ms on localhost (2/2) -2016-05-23 20:57:45 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.049 s -2016-05-23 20:57:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 20:57:45 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 20:57:45 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 20:57:45 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.081138 s -2016-05-23 20:57:45 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:57:45 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:57:45 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:57:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:57:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:45 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:57:45 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=369647, maxMem=766075207 -2016-05-23 20:57:45 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 20:57:45 INFO MemoryStore:59 - ensureFreeSpace(2855) called with curMem=375143, maxMem=766075207 -2016-05-23 20:57:45 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:57:45 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53940 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:57:45 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:57:45 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 20:57:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:57:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:57:45 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 20:57:45 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 20:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:57:45 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 20:57:45 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 20:57:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 38 ms on localhost (1/2) -2016-05-23 20:57:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 38 ms on localhost (2/2) -2016-05-23 20:57:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 20:57:45 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.039 s -2016-05-23 20:57:45 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.065704 s -2016-05-23 20:57:46 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:57:46 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 20:57:46 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:57:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:57:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=377998, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(2745) called with curMem=383342, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 20:57:46 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53940 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 20:57:46 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 20:57:46 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 20:57:46 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 20:57:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:57:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:57:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:57:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:57:46 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 20:57:46 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 38 ms on localhost (1/2) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 37 ms on localhost (2/2) -2016-05-23 20:57:46 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.039 s -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 20:57:46 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.054179 s -2016-05-23 20:57:46 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:57:46 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:57:46 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 20:57:46 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:57:46 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 20:57:46 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=386087, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(3157) called with curMem=392015, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 20:57:46 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53940 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 20:57:46 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 20:57:46 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 20:57:46 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 20:57:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 20:57:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 20:57:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 20:57:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 20:57:46 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 20:57:46 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 74 ms on localhost (1/2) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 78 ms on localhost (2/2) -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 20:57:46 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.078 s -2016-05-23 20:57:46 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:57:46 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:57:46 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 20:57:46 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:57:46 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=395172, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=397676, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 20:57:46 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53940 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:57:46 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:46 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 20:57:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:57:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-23 20:57:46 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 54 ms on localhost (1/1) -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 20:57:46 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.055 s -2016-05-23 20:57:46 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.183270 s -2016-05-23 20:57:46 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:57:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:57:46 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:57:46 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:57:46 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 20:57:46 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 20:57:46 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=399221, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=403413, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:57:46 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53940 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:57:46 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 20:57:46 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:57:46 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 20:57:46 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 20:57:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:57:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:57:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 20:57:46 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53940 in memory (size: 2.7 KB, free: 730.5 MB) -2016-05-23 20:57:46 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 20:57:46 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53940 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:57:46 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 20:57:46 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53940 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:57:46 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 20:57:46 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 20:57:46 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53940 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 20:57:46 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 52 ms on localhost (1/2) -2016-05-23 20:57:46 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 20:57:46 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53940 in memory (size: 3.1 KB, free: 730.6 MB) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 55 ms on localhost (2/2) -2016-05-23 20:57:46 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.059 s -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 20:57:46 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:57:46 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:57:46 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 20:57:46 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:57:46 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=367809, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=372665, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:57:46 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53940 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:57:46 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:46 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 20:57:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:57:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:46 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 24 ms on localhost (1/1) -2016-05-23 20:57:46 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.025 s -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 20:57:46 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.145604 s -2016-05-23 20:57:46 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:57:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 20:57:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 20:57:46 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 20:57:46 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:57:46 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 20:57:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=375196, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=380852, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:57:46 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53940 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:57:46 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:46 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 20:57:46 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 20:57:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:57:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:57:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:57:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:46 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 20:57:46 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 21 ms on localhost (1/2) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 22 ms on localhost (2/2) -2016-05-23 20:57:46 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.023 s -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 20:57:46 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.038302 s -2016-05-23 20:57:46 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:57:46 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 20:57:46 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:57:46 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 20:57:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=383760, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=389456, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:57:46 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53940 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:57:46 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:46 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 20:57:46 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 20:57:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:57:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 20:57:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:57:46 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 20:57:46 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 23 ms on localhost (1/2) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 23 ms on localhost (2/2) -2016-05-23 20:57:46 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.025 s -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 20:57:46 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.045739 s -2016-05-23 20:57:46 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:43 -2016-05-23 20:57:46 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:43) with 4 output partitions -2016-05-23 20:57:46 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:43) -2016-05-23 20:57:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:57:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40), which has no missing parents -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=392345, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=393569, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:57:46 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53940 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:57:46 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:40) -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:57:46 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 20:57:46 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 20:57:46 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 20:57:46 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 20:57:46 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 20:57:46 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 20:57:46 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 20 ms on localhost (1/4) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 11 ms on localhost (2/4) -2016-05-23 20:57:46 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 13 ms on localhost (3/4) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 12 ms on localhost (4/4) -2016-05-23 20:57:46 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:43) finished in 0.025 s -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 20:57:46 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:43, took 0.038796 s -2016-05-23 20:57:46 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:45 -2016-05-23 20:57:46 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:45) with 4 output partitions -2016-05-23 20:57:46 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:45) -2016-05-23 20:57:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:57:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41), which has no missing parents -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=394462, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=395686, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 20:57:46 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53940 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:57:46 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:41) -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 20:57:46 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 20:57:46 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 20:57:46 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 20:57:46 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 20:57:46 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 20:57:46 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 20:57:46 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 11 ms on localhost (1/4) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 10 ms on localhost (2/4) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 12 ms on localhost (3/4) -2016-05-23 20:57:46 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 15 ms on localhost (4/4) -2016-05-23 20:57:46 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:45) finished in 0.015 s -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 20:57:46 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:45, took 0.027754 s -2016-05-23 20:57:46 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:57:46 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:57:46 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 20:57:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:57:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=396579, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(1525) called with curMem=399083, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1525.0 B, free 730.2 MB) -2016-05-23 20:57:46 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53940 (size: 1525.0 B, free: 730.5 MB) -2016-05-23 20:57:46 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:57:46 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 20:57:46 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 7 ms on localhost (1/1) -2016-05-23 20:57:46 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.007 s -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 20:57:46 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.018709 s -2016-05-23 20:57:46 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:57:46 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:57:46 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:57:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:57:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=400608, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:57:46 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=404008, maxMem=766075207 -2016-05-23 20:57:46 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.2 MB) -2016-05-23 20:57:46 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53940 (size: 1935.0 B, free: 730.5 MB) -2016-05-23 20:57:46 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:46 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:57:46 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 20:57:46 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 20:57:46 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 20:57:46 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 20:57:46 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 20:57:46 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 20:57:46 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 20:57:46 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 19 ms on localhost (1/4) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 17 ms on localhost (2/4) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 20 ms on localhost (3/4) -2016-05-23 20:57:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 23 ms on localhost (4/4) -2016-05-23 20:57:46 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.023 s -2016-05-23 20:57:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 20:57:46 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.036821 s -2016-05-23 20:57:47 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:57:47 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:57:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=405943, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(1919) called with curMem=409383, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1919.0 B, free 730.2 MB) -2016-05-23 20:57:47 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53940 (size: 1919.0 B, free: 730.5 MB) -2016-05-23 20:57:47 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:57:47 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 20:57:47 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 20:57:47 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 20:57:47 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 20:57:47 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 20:57:47 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 9 ms on localhost (1/4) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 13 ms on localhost (2/4) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 17 ms on localhost (3/4) -2016-05-23 20:57:47 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.019 s -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 17 ms on localhost (4/4) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 20:57:47 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.036119 s -2016-05-23 20:57:47 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:57:47 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:57:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=411302, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(1861) called with curMem=414678, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1861.0 B, free 730.2 MB) -2016-05-23 20:57:47 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53940 (size: 1861.0 B, free: 730.5 MB) -2016-05-23 20:57:47 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 20:57:47 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 20:57:47 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 20:57:47 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 20:57:47 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 13 ms on localhost (1/4) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 19 ms on localhost (2/4) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 20:57:47 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 21 ms on localhost (3/4) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 28 ms on localhost (4/4) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 20:57:47 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.033 s -2016-05-23 20:57:47 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.050355 s -2016-05-23 20:57:47 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:57:47 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:57:47 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:57:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 20:57:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=416539, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(2283) called with curMem=420579, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 20:57:47 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:53940 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:57:47 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 20:57:47 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 20:57:47 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 20:57:47 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 20:57:47 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 55 ms on localhost (1/4) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 81 ms on localhost (2/4) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 87 ms on localhost (3/4) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 91 ms on localhost (4/4) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 20:57:47 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.096 s -2016-05-23 20:57:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:57:47 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:57:47 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 20:57:47 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=422862, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=425406, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 20:57:47 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:53940 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 20:57:47 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:47 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 20:57:47 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:47 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 20:57:47 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 20 ms on localhost (1/3) -2016-05-23 20:57:47 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 21 ms on localhost (2/3) -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:47 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 43 ms on localhost (3/3) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 20:57:47 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.045 s -2016-05-23 20:57:47 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.177994 s -2016-05-23 20:57:47 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:57:47 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:57:47 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:57:47 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:57:47 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 20:57:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 20:57:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=426974, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=431206, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 20:57:47 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:53940 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:57:47 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:57:47 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 20:57:47 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 20:57:47 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 20:57:47 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:47 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 20:57:47 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 106 ms on localhost (1/4) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 112 ms on localhost (2/4) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 118 ms on localhost (3/4) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 140 ms on localhost (4/4) -2016-05-23 20:57:47 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.143 s -2016-05-23 20:57:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:57:47 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:57:47 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 20:57:47 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 20:57:47 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=433512, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=438408, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 20:57:47 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:53940 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:57:47 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:47 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:57:47 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 13 ms on localhost (1/1) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 20:57:47 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.014 s -2016-05-23 20:57:47 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.191633 s -2016-05-23 20:57:47 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:57:47 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 20:57:47 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 20:57:47 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:57:47 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:57:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 20:57:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=440950, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=446646, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 20:57:47 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:53940 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:57:47 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:47 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 20:57:47 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 20:57:47 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:47 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 20:57:47 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 28 ms on localhost (1/4) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 26 ms on localhost (2/4) -2016-05-23 20:57:47 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 38 ms on localhost (3/4) -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:47 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 51 ms on localhost (4/4) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 20:57:47 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.053 s -2016-05-23 20:57:47 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.069790 s -2016-05-23 20:57:47 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:57:47 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:57:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 20:57:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=449555, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=455291, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:57:47 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:53940 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:57:47 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:47 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 20:57:47 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 20:57:47 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:47 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:47 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 20:57:47 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 20:57:47 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 35 ms on localhost (1/4) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 35 ms on localhost (2/4) -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 43 ms on localhost (3/4) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 51 ms on localhost (4/4) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 20:57:47 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.054 s -2016-05-23 20:57:47 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.075281 s -2016-05-23 20:57:47 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:59 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Got job 20 (foreach at StatisticsCorrLearning.scala:59) with 4 output partitions -2016-05-23 20:57:47 INFO DAGScheduler:59 - Final stage: ResultStage 34(foreach at StatisticsCorrLearning.scala:59) -2016-05-23 20:57:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:56), which has no missing parents -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=458183, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=459407, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.1 MB) -2016-05-23 20:57:47 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:53940 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:57:47 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:56) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 62, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 63, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 64, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 20:57:47 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 64) -2016-05-23 20:57:47 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 63) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 64). 915 bytes result sent to driver -2016-05-23 20:57:47 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 62) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 63). 915 bytes result sent to driver -2016-05-23 20:57:47 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 62). 915 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 64) in 11 ms on localhost (1/4) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 63) in 13 ms on localhost (2/4) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 62) in 14 ms on localhost (3/4) -2016-05-23 20:57:47 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 65) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 65). 915 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 65) in 17 ms on localhost (4/4) -2016-05-23 20:57:47 INFO DAGScheduler:59 - ResultStage 34 (foreach at StatisticsCorrLearning.scala:59) finished in 0.021 s -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-23 20:57:47 INFO DAGScheduler:59 - Job 20 finished: foreach at StatisticsCorrLearning.scala:59, took 0.036646 s -2016-05-23 20:57:47 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:61 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Got job 21 (foreach at StatisticsCorrLearning.scala:61) with 4 output partitions -2016-05-23 20:57:47 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at StatisticsCorrLearning.scala:61) -2016-05-23 20:57:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:57), which has no missing parents -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=460300, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=461524, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.1 MB) -2016-05-23 20:57:47 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:53940 (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:57:47 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:57) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 4 tasks -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 66, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 20:57:47 INFO Executor:59 - Running task 3.0 in stage 35.0 (TID 69) -2016-05-23 20:57:47 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 66) -2016-05-23 20:57:47 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 68) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 3.0 in stage 35.0 (TID 69). 915 bytes result sent to driver -2016-05-23 20:57:47 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 67) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 68). 915 bytes result sent to driver -2016-05-23 20:57:47 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 67). 915 bytes result sent to driver -2016-05-23 20:57:47 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 66). 915 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 35.0 (TID 69) in 11 ms on localhost (1/4) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 67) in 13 ms on localhost (2/4) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 68) in 13 ms on localhost (3/4) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 66) in 15 ms on localhost (4/4) -2016-05-23 20:57:47 INFO DAGScheduler:59 - ResultStage 35 (foreach at StatisticsCorrLearning.scala:61) finished in 0.016 s -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-23 20:57:47 INFO DAGScheduler:59 - Job 21 finished: foreach at StatisticsCorrLearning.scala:61, took 0.028564 s -2016-05-23 20:57:47 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Got job 22 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:57:47 INFO DAGScheduler:59 - Final stage: ResultStage 36(first at RowMatrix.scala:65) -2016-05-23 20:57:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=462417, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(1523) called with curMem=464921, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1523.0 B, free 730.1 MB) -2016-05-23 20:57:47 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:53940 (size: 1523.0 B, free: 730.5 MB) -2016-05-23 20:57:47 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:57:47 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 70) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 70). 935 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 70) in 6 ms on localhost (1/1) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-23 20:57:47 INFO DAGScheduler:59 - ResultStage 36 (first at RowMatrix.scala:65) finished in 0.006 s -2016-05-23 20:57:47 INFO DAGScheduler:59 - Job 22 finished: first at RowMatrix.scala:65, took 0.018162 s -2016-05-23 20:57:47 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Got job 23 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 20:57:47 INFO DAGScheduler:59 - Final stage: ResultStage 37(first at RowMatrix.scala:65) -2016-05-23 20:57:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=466444, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(1523) called with curMem=468948, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1523.0 B, free 730.1 MB) -2016-05-23 20:57:47 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:53940 (size: 1523.0 B, free: 730.5 MB) -2016-05-23 20:57:47 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 37 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 3 tasks -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:57:47 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-23 20:57:47 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 73) -2016-05-23 20:57:47 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 72) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 1046 bytes result sent to driver -2016-05-23 20:57:47 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 73). 1046 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 7 ms on localhost (1/3) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 72). 1046 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 73) in 8 ms on localhost (2/3) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 72) in 10 ms on localhost (3/3) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-23 20:57:47 INFO DAGScheduler:59 - ResultStage 37 (first at RowMatrix.scala:65) finished in 0.012 s -2016-05-23 20:57:47 INFO DAGScheduler:59 - Job 23 finished: first at RowMatrix.scala:65, took 0.025224 s -2016-05-23 20:57:47 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Got job 24 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:57:47 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:57:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=470471, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(1933) called with curMem=473871, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1933.0 B, free 730.1 MB) -2016-05-23 20:57:47 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:53940 (size: 1933.0 B, free: 730.5 MB) -2016-05-23 20:57:47 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 74, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 77, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:57:47 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 74) -2016-05-23 20:57:47 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 77) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 74). 1231 bytes result sent to driver -2016-05-23 20:57:47 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 77). 1231 bytes result sent to driver -2016-05-23 20:57:47 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 75) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 75). 1231 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 77) in 14 ms on localhost (1/4) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 74) in 17 ms on localhost (2/4) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 75) in 18 ms on localhost (3/4) -2016-05-23 20:57:47 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 76) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 76). 1231 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 76) in 23 ms on localhost (4/4) -2016-05-23 20:57:47 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at RowMatrix.scala:332) finished in 0.025 s -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-23 20:57:47 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at RowMatrix.scala:332, took 0.040886 s -2016-05-23 20:57:47 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Got job 25 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:57:47 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:57:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=475804, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-23 20:57:47 INFO MemoryStore:59 - ensureFreeSpace(1917) called with curMem=479244, maxMem=766075207 -2016-05-23 20:57:47 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1917.0 B, free 730.1 MB) -2016-05-23 20:57:47 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:53940 (size: 1917.0 B, free: 730.5 MB) -2016-05-23 20:57:47 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 39 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 4 tasks -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 79, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 80, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 39.0 (TID 81, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:57:47 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 80) -2016-05-23 20:57:47 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 78) -2016-05-23 20:57:47 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 79) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 80). 1109 bytes result sent to driver -2016-05-23 20:57:47 INFO Executor:59 - Running task 3.0 in stage 39.0 (TID 81) -2016-05-23 20:57:47 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 79). 1109 bytes result sent to driver -2016-05-23 20:57:47 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 78). 1109 bytes result sent to driver -2016-05-23 20:57:47 INFO Executor:59 - Finished task 3.0 in stage 39.0 (TID 81). 1109 bytes result sent to driver -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 80) in 13 ms on localhost (1/4) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 79) in 16 ms on localhost (2/4) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 39.0 (TID 81) in 14 ms on localhost (3/4) -2016-05-23 20:57:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 78) in 18 ms on localhost (4/4) -2016-05-23 20:57:47 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at RowMatrix.scala:124) finished in 0.019 s -2016-05-23 20:57:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-23 20:57:47 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at RowMatrix.scala:124, took 0.034461 s -2016-05-23 20:57:48 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 20:57:48 INFO DAGScheduler:59 - Got job 26 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 20:57:48 INFO DAGScheduler:59 - Final stage: ResultStage 40(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:57:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 20:57:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:48 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 20:57:48 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=481161, maxMem=766075207 -2016-05-23 20:57:48 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 20:57:48 INFO MemoryStore:59 - ensureFreeSpace(1852) called with curMem=484537, maxMem=766075207 -2016-05-23 20:57:48 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 1852.0 B, free 730.1 MB) -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:53940 (size: 1852.0 B, free: 730.5 MB) -2016-05-23 20:57:48 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 40 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:57:48 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 4 tasks -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 82, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 83, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 84, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 40.0 (TID 85, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 20:57:48 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 83) -2016-05-23 20:57:48 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 84) -2016-05-23 20:57:48 INFO Executor:59 - Running task 3.0 in stage 40.0 (TID 85) -2016-05-23 20:57:48 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 84). 1249 bytes result sent to driver -2016-05-23 20:57:48 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 82) -2016-05-23 20:57:48 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 83). 1254 bytes result sent to driver -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 84) in 10 ms on localhost (1/4) -2016-05-23 20:57:48 INFO Executor:59 - Finished task 3.0 in stage 40.0 (TID 85). 1254 bytes result sent to driver -2016-05-23 20:57:48 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 82). 1104 bytes result sent to driver -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 40.0 (TID 85) in 13 ms on localhost (2/4) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 82) in 15 ms on localhost (3/4) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 83) in 16 ms on localhost (4/4) -2016-05-23 20:57:48 INFO DAGScheduler:59 - ResultStage 40 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.017 s -2016-05-23 20:57:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-23 20:57:48 INFO DAGScheduler:59 - Job 26 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.032784 s -2016-05-23 20:57:48 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 20:57:48 INFO DAGScheduler:59 - Registering RDD 51 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:57:48 INFO DAGScheduler:59 - Got job 27 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 20:57:48 INFO DAGScheduler:59 - Final stage: ResultStage 42(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 20:57:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 41) -2016-05-23 20:57:48 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 41) -2016-05-23 20:57:48 INFO DAGScheduler:59 - Submitting ShuffleMapStage 41 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 20:57:48 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=486389, maxMem=766075207 -2016-05-23 20:57:48 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-05-23 20:57:48 INFO MemoryStore:59 - ensureFreeSpace(2280) called with curMem=490429, maxMem=766075207 -2016-05-23 20:57:48 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:53940 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:57:48 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 41 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 20:57:48 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 4 tasks -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 86, localhost, PROCESS_LOCAL, 2221 bytes) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 87, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 88, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 41.0 (TID 89, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 20:57:48 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 86) -2016-05-23 20:57:48 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 88) -2016-05-23 20:57:48 INFO Executor:59 - Running task 3.0 in stage 41.0 (TID 89) -2016-05-23 20:57:48 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 87) -2016-05-23 20:57:48 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 86). 1161 bytes result sent to driver -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 86) in 12 ms on localhost (1/4) -2016-05-23 20:57:48 INFO Executor:59 - Finished task 2.0 in stage 41.0 (TID 88). 1161 bytes result sent to driver -2016-05-23 20:57:48 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 87). 1161 bytes result sent to driver -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 41.0 (TID 88) in 97 ms on localhost (2/4) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 87) in 98 ms on localhost (3/4) -2016-05-23 20:57:48 INFO Executor:59 - Finished task 3.0 in stage 41.0 (TID 89). 1161 bytes result sent to driver -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 41.0 (TID 89) in 133 ms on localhost (4/4) -2016-05-23 20:57:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-23 20:57:48 INFO DAGScheduler:59 - ShuffleMapStage 41 (flatMap at SpearmanCorrelation.scala:49) finished in 0.136 s -2016-05-23 20:57:48 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:57:48 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:57:48 INFO DAGScheduler:59 - waiting: Set(ResultStage 42) -2016-05-23 20:57:48 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:57:48 INFO DAGScheduler:59 - Missing parents for ResultStage 42: List() -2016-05-23 20:57:48 INFO DAGScheduler:59 - Submitting ResultStage 42 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 20:57:48 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=492709, maxMem=766075207 -2016-05-23 20:57:48 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 20:57:48 INFO MemoryStore:59 - ensureFreeSpace(1563) called with curMem=495253, maxMem=766075207 -2016-05-23 20:57:48 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 1563.0 B, free 730.1 MB) -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:53940 (size: 1563.0 B, free: 730.5 MB) -2016-05-23 20:57:48 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:48 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 42 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 20:57:48 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 3 tasks -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 90, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 91, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 42.0 (TID 92, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:48 INFO Executor:59 - Running task 2.0 in stage 42.0 (TID 92) -2016-05-23 20:57:48 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 91) -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:48 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 90) -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:57:48 INFO Executor:59 - Finished task 2.0 in stage 42.0 (TID 92). 1203 bytes result sent to driver -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 42.0 (TID 92) in 16 ms on localhost (1/3) -2016-05-23 20:57:48 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 90). 1203 bytes result sent to driver -2016-05-23 20:57:48 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 91). 1203 bytes result sent to driver -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 90) in 25 ms on localhost (2/3) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 91) in 25 ms on localhost (3/3) -2016-05-23 20:57:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-23 20:57:48 INFO DAGScheduler:59 - ResultStage 42 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.027 s -2016-05-23 20:57:48 INFO DAGScheduler:59 - Job 27 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.188009 s -2016-05-23 20:57:48 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:57:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 168 bytes -2016-05-23 20:57:48 INFO DAGScheduler:59 - Registering RDD 56 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:57:48 INFO DAGScheduler:59 - Got job 28 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 20:57:48 INFO DAGScheduler:59 - Final stage: ResultStage 45(first at RowMatrix.scala:65) -2016-05-23 20:57:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 44) -2016-05-23 20:57:48 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 44) -2016-05-23 20:57:48 INFO DAGScheduler:59 - Submitting ShuffleMapStage 44 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 20:57:48 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=496816, maxMem=766075207 -2016-05-23 20:57:48 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 20:57:48 INFO MemoryStore:59 - ensureFreeSpace(2301) called with curMem=501048, maxMem=766075207 -2016-05-23 20:57:48 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:53940 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:57:48 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 44 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 20:57:48 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 4 tasks -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 93, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 94, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 44.0 (TID 95, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 44.0 (TID 96, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 20:57:48 INFO Executor:59 - Running task 3.0 in stage 44.0 (TID 96) -2016-05-23 20:57:48 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 94) -2016-05-23 20:57:48 INFO Executor:59 - Running task 2.0 in stage 44.0 (TID 95) -2016-05-23 20:57:48 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 93) -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:57:48 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 94). 1377 bytes result sent to driver -2016-05-23 20:57:48 INFO Executor:59 - Finished task 3.0 in stage 44.0 (TID 96). 1377 bytes result sent to driver -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 94) in 52 ms on localhost (1/4) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 44.0 (TID 96) in 53 ms on localhost (2/4) -2016-05-23 20:57:48 INFO Executor:59 - Finished task 2.0 in stage 44.0 (TID 95). 1377 bytes result sent to driver -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 44.0 (TID 95) in 58 ms on localhost (3/4) -2016-05-23 20:57:48 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 93). 1377 bytes result sent to driver -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 93) in 60 ms on localhost (4/4) -2016-05-23 20:57:48 INFO DAGScheduler:59 - ShuffleMapStage 44 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.061 s -2016-05-23 20:57:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-23 20:57:48 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 20:57:48 INFO DAGScheduler:59 - running: Set() -2016-05-23 20:57:48 INFO DAGScheduler:59 - waiting: Set(ResultStage 45) -2016-05-23 20:57:48 INFO DAGScheduler:59 - failed: Set() -2016-05-23 20:57:48 INFO DAGScheduler:59 - Missing parents for ResultStage 45: List() -2016-05-23 20:57:48 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 20:57:48 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=503349, maxMem=766075207 -2016-05-23 20:57:48 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 20:57:48 INFO MemoryStore:59 - ensureFreeSpace(2541) called with curMem=508245, maxMem=766075207 -2016-05-23 20:57:48 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:53940 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:57:48 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:48 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 45 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:57:48 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 1 tasks -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 97, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:48 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 97) -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:57:48 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 97). 1185 bytes result sent to driver -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 97) in 8 ms on localhost (1/1) -2016-05-23 20:57:48 INFO DAGScheduler:59 - ResultStage 45 (first at RowMatrix.scala:65) finished in 0.009 s -2016-05-23 20:57:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-23 20:57:48 INFO DAGScheduler:59 - Job 28 finished: first at RowMatrix.scala:65, took 0.089249 s -2016-05-23 20:57:48 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 20:57:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 168 bytes -2016-05-23 20:57:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 165 bytes -2016-05-23 20:57:48 INFO DAGScheduler:59 - Got job 29 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 20:57:48 INFO DAGScheduler:59 - Final stage: ResultStage 48(first at RowMatrix.scala:65) -2016-05-23 20:57:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 47) -2016-05-23 20:57:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:48 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which has no missing parents -2016-05-23 20:57:48 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=510786, maxMem=766075207 -2016-05-23 20:57:48 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 20:57:48 INFO MemoryStore:59 - ensureFreeSpace(2541) called with curMem=515682, maxMem=766075207 -2016-05-23 20:57:48 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:53940 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:57:48 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:48 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 48 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 20:57:48 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 3 tasks -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 98, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 99, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 48.0 (TID 100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:48 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 98) -2016-05-23 20:57:48 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 99) -2016-05-23 20:57:48 INFO Executor:59 - Running task 2.0 in stage 48.0 (TID 100) -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:57:48 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 98). 1296 bytes result sent to driver -2016-05-23 20:57:48 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 99). 1296 bytes result sent to driver -2016-05-23 20:57:48 INFO Executor:59 - Finished task 2.0 in stage 48.0 (TID 100). 1296 bytes result sent to driver -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 98) in 14 ms on localhost (1/3) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 48.0 (TID 100) in 13 ms on localhost (2/3) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 99) in 14 ms on localhost (3/3) -2016-05-23 20:57:48 INFO DAGScheduler:59 - ResultStage 48 (first at RowMatrix.scala:65) finished in 0.015 s -2016-05-23 20:57:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-23 20:57:48 INFO DAGScheduler:59 - Job 29 finished: first at RowMatrix.scala:65, took 0.027917 s -2016-05-23 20:57:48 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 20:57:48 INFO DAGScheduler:59 - Got job 30 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 20:57:48 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at RowMatrix.scala:332) -2016-05-23 20:57:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 50) -2016-05-23 20:57:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:48 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 20:57:48 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=518223, maxMem=766075207 -2016-05-23 20:57:48 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 20:57:48 INFO MemoryStore:59 - ensureFreeSpace(2910) called with curMem=523919, maxMem=766075207 -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-23 20:57:48 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:53940 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:57:48 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-23 20:57:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 51 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332) -2016-05-23 20:57:48 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 4 tasks -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 51.0 (TID 103, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:53940 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 51.0 (TID 104, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:48 INFO Executor:59 - Running task 2.0 in stage 51.0 (TID 103) -2016-05-23 20:57:48 INFO Executor:59 - Running task 3.0 in stage 51.0 (TID 104) -2016-05-23 20:57:48 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-23 20:57:48 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:53940 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:57:48 INFO Executor:59 - Finished task 3.0 in stage 51.0 (TID 104). 1481 bytes result sent to driver -2016-05-23 20:57:48 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 1481 bytes result sent to driver -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-23 20:57:48 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 1481 bytes result sent to driver -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 51.0 (TID 104) in 17 ms on localhost (1/4) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 20 ms on localhost (2/4) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 21 ms on localhost (3/4) -2016-05-23 20:57:48 INFO Executor:59 - Finished task 2.0 in stage 51.0 (TID 103). 1481 bytes result sent to driver -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 51.0 (TID 103) in 21 ms on localhost (4/4) -2016-05-23 20:57:48 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at RowMatrix.scala:332) finished in 0.023 s -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:53940 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:57:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-23 20:57:48 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at RowMatrix.scala:332, took 0.054159 s -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:53940 in memory (size: 1563.0 B, free: 730.5 MB) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:53940 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-23 20:57:48 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 20:57:48 INFO DAGScheduler:59 - Got job 31 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:53940 in memory (size: 1852.0 B, free: 730.5 MB) -2016-05-23 20:57:48 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at RowMatrix.scala:124) -2016-05-23 20:57:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 53) -2016-05-23 20:57:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-23 20:57:48 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 20:57:48 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=486327, maxMem=766075207 -2016-05-23 20:57:48 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 20:57:48 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=490146, maxMem=766075207 -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:53940 in memory (size: 1917.0 B, free: 730.5 MB) -2016-05-23 20:57:48 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:53940 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:57:48 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-23 20:57:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 54 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124) -2016-05-23 20:57:48 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 4 tasks -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 105, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:53940 in memory (size: 1933.0 B, free: 730.5 MB) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 106, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 20:57:48 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 106) -2016-05-23 20:57:48 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 105) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:48 INFO Executor:59 - Running task 2.0 in stage 54.0 (TID 107) -2016-05-23 20:57:48 INFO Executor:59 - Running task 3.0 in stage 54.0 (TID 108) -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:53940 in memory (size: 1523.0 B, free: 730.5 MB) -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 20:57:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 20:57:48 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 106). 1359 bytes result sent to driver -2016-05-23 20:57:48 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 105). 1359 bytes result sent to driver -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 106) in 23 ms on localhost (1/4) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 105) in 26 ms on localhost (2/4) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-23 20:57:48 INFO Executor:59 - Finished task 2.0 in stage 54.0 (TID 107). 1359 bytes result sent to driver -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 54.0 (TID 107) in 28 ms on localhost (3/4) -2016-05-23 20:57:48 INFO Executor:59 - Finished task 3.0 in stage 54.0 (TID 108). 1359 bytes result sent to driver -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:53940 in memory (size: 1523.0 B, free: 730.5 MB) -2016-05-23 20:57:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 54.0 (TID 108) in 29 ms on localhost (4/4) -2016-05-23 20:57:48 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at RowMatrix.scala:124) finished in 0.033 s -2016-05-23 20:57:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-23 20:57:48 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at RowMatrix.scala:124, took 0.045113 s -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:53940 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:53940 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:53940 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:53940 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:53940 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 20:57:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:53940 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:53940 in memory (size: 1568.0 B, free: 730.5 MB) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:53940 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:53940 in memory (size: 1861.0 B, free: 730.5 MB) -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:53940 in memory (size: 1919.0 B, free: 730.5 MB) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:53940 in memory (size: 1935.0 B, free: 730.5 MB) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:53940 in memory (size: 1525.0 B, free: 730.5 MB) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:53940 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:53940 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:53940 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:53940 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:53940 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-23 20:57:48 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:53940 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-23 20:57:48 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-23 20:57:48 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 20:57:48 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 20:57:48 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 20:57:48 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 20:57:48 INFO BlockManager:59 - BlockManager stopped -2016-05-23 20:57:48 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 20:57:48 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 20:57:48 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 20:57:48 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 20:57:48 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-69b44dd3-cb31-4902-8c9f-fb949d5cb5a6 -2016-05-23 20:57:48 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 21:03:42 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 21:03:43 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 21:03:43 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 21:03:43 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 21:03:43 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 21:03:44 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 21:03:44 INFO Remoting:74 - Starting remoting -2016-05-23 21:03:44 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:54144] -2016-05-23 21:03:44 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54144. -2016-05-23 21:03:44 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 21:03:44 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 21:03:44 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-b53d4579-d75a-41e9-bd01-6242321afecb -2016-05-23 21:03:44 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 21:03:45 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-2c53b2b5-27aa-402b-b765-3672d9b76837\httpd-9c61ecda-b12a-48e6-b6a2-1f6503b1b1b6 -2016-05-23 21:03:45 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 21:03:45 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 21:03:45 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54145 -2016-05-23 21:03:45 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54145. -2016-05-23 21:03:45 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 21:03:45 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 21:03:45 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 21:03:45 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 21:03:45 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 21:03:45 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 21:03:45 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 21:03:45 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54169. -2016-05-23 21:03:45 INFO NettyBlockTransferService:59 - Server created on 54169 -2016-05-23 21:03:45 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 21:03:45 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54169 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54169) -2016-05-23 21:03:45 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 21:03:46 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 21:03:46 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 21:03:46 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 21:03:46 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 21:03:46 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54169 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 21:03:46 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:22 -2016-05-23 21:03:46 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 21:03:46 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 21:03:46 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 21:03:46 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 21:03:46 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54169 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 21:03:46 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:25 -2016-05-23 21:03:48 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 21:03:48 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 21:03:48 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 21:03:48 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 21:03:48 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 21:03:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:03:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:48 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:23), which has no missing parents -2016-05-23 21:03:48 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 21:03:48 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 21:03:48 INFO MemoryStore:59 - ensureFreeSpace(2043) called with curMem=346712, maxMem=766075207 -2016-05-23 21:03:48 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2043.0 B, free 730.3 MB) -2016-05-23 21:03:48 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54169 (size: 2043.0 B, free: 730.6 MB) -2016-05-23 21:03:48 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:23) -2016-05-23 21:03:48 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 21:03:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:03:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:03:48 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 21:03:48 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 21:03:48 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:03:48 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:03:48 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 21:03:48 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 21:03:48 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 21:03:48 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 21:03:48 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 21:03:48 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 21:03:48 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 21:03:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 140 ms on localhost (1/2) -2016-05-23 21:03:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 120 ms on localhost (2/2) -2016-05-23 21:03:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 21:03:48 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:29) finished in 0.164 s -2016-05-23 21:03:48 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:29, took 0.309025 s -2016-05-23 21:03:48 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 21:03:48 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:31 -2016-05-23 21:03:48 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:31) with 2 output partitions -2016-05-23 21:03:48 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:31) -2016-05-23 21:03:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:03:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:48 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:26), which has no missing parents -2016-05-23 21:03:48 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348755, maxMem=766075207 -2016-05-23 21:03:48 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 21:03:48 INFO MemoryStore:59 - ensureFreeSpace(2043) called with curMem=352267, maxMem=766075207 -2016-05-23 21:03:48 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2043.0 B, free 730.2 MB) -2016-05-23 21:03:48 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54169 (size: 2043.0 B, free: 730.6 MB) -2016-05-23 21:03:48 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:26) -2016-05-23 21:03:48 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 21:03:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:03:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:03:48 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 21:03:48 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 21:03:48 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:03:48 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:03:48 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 21:03:48 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 21:03:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 22 ms on localhost (1/2) -2016-05-23 21:03:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 28 ms on localhost (2/2) -2016-05-23 21:03:48 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:31) finished in 0.029 s -2016-05-23 21:03:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 21:03:48 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:31, took 0.049502 s -2016-05-23 21:03:48 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:03:48 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:03:48 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 21:03:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:03:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:48 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:03:48 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354310, maxMem=766075207 -2016-05-23 21:03:48 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 21:03:48 INFO MemoryStore:59 - ensureFreeSpace(2466) called with curMem=358870, maxMem=766075207 -2016-05-23 21:03:48 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:03:48 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54169 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 21:03:48 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:48 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 21:03:48 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 21:03:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:03:48 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 21:03:48 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:54169 in memory (size: 2043.0 B, free: 730.6 MB) -2016-05-23 21:03:48 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:03:48 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:03:48 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:54169 in memory (size: 2043.0 B, free: 730.6 MB) -2016-05-23 21:03:48 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 21:03:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 45 ms on localhost (1/1) -2016-05-23 21:03:48 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.054 s -2016-05-23 21:03:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 21:03:48 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.117175 s -2016-05-23 21:03:49 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:03:49 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:03:49 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:03:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:03:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:49 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:03:49 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=350226, maxMem=766075207 -2016-05-23 21:03:49 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 21:03:49 INFO MemoryStore:59 - ensureFreeSpace(2865) called with curMem=355674, maxMem=766075207 -2016-05-23 21:03:49 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:03:49 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:54169 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:03:49 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:03:49 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 21:03:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:03:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:03:49 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 21:03:49 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 21:03:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:03:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:03:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:03:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:03:49 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 21:03:49 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 21:03:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 57 ms on localhost (1/2) -2016-05-23 21:03:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 59 ms on localhost (2/2) -2016-05-23 21:03:49 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.062 s -2016-05-23 21:03:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 21:03:49 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 21:03:49 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 21:03:49 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.092282 s -2016-05-23 21:03:49 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:03:49 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:03:49 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:03:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:03:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:49 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:03:49 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=358539, maxMem=766075207 -2016-05-23 21:03:49 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 21:03:49 INFO MemoryStore:59 - ensureFreeSpace(2856) called with curMem=364035, maxMem=766075207 -2016-05-23 21:03:49 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:03:49 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:54169 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:03:49 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:03:49 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 21:03:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:03:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:03:49 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 21:03:49 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 21:03:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:03:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:03:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:03:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:03:49 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 21:03:49 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 21:03:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 29 ms on localhost (1/2) -2016-05-23 21:03:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 33 ms on localhost (2/2) -2016-05-23 21:03:49 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.033 s -2016-05-23 21:03:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 21:03:49 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.062726 s -2016-05-23 21:03:49 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:03:49 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 21:03:49 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:03:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:03:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:49 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:03:49 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=366891, maxMem=766075207 -2016-05-23 21:03:49 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 21:03:49 INFO MemoryStore:59 - ensureFreeSpace(2746) called with curMem=372235, maxMem=766075207 -2016-05-23 21:03:49 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 21:03:49 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:54169 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 21:03:49 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:03:49 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 21:03:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:03:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:03:49 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 21:03:49 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 21:03:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:03:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:03:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:03:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:03:49 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 21:03:49 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 21:03:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 36 ms on localhost (1/2) -2016-05-23 21:03:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 39 ms on localhost (2/2) -2016-05-23 21:03:49 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.039 s -2016-05-23 21:03:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 21:03:49 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.060509 s -2016-05-23 21:03:49 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:03:49 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:03:49 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 21:03:49 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:03:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 21:03:49 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 21:03:49 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:03:49 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=374981, maxMem=766075207 -2016-05-23 21:03:49 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 21:03:49 INFO MemoryStore:59 - ensureFreeSpace(3158) called with curMem=380909, maxMem=766075207 -2016-05-23 21:03:49 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 21:03:49 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:54169 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 21:03:49 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:03:49 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 21:03:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 21:03:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 21:03:49 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 21:03:49 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 21:03:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:03:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:03:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:03:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:03:50 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 64 ms on localhost (1/2) -2016-05-23 21:03:50 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 72 ms on localhost (2/2) -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 21:03:50 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.073 s -2016-05-23 21:03:50 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:03:50 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:03:50 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 21:03:50 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:03:50 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=384067, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=386571, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 21:03:50 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:54169 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 21:03:50 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:50 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-23 21:03:50 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 72 ms on localhost (1/1) -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 21:03:50 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.073 s -2016-05-23 21:03:50 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.200024 s -2016-05-23 21:03:50 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:03:50 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 21:03:50 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:03:50 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:03:50 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 21:03:50 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 21:03:50 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:03:50 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=388116, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=392308, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:03:50 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:54169 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:03:50 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:54169 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 21:03:50 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 21:03:50 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:03:50 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 21:03:50 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 21:03:50 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:54169 in memory (size: 3.1 KB, free: 730.5 MB) -2016-05-23 21:03:50 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:50 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:54169 in memory (size: 2.7 KB, free: 730.5 MB) -2016-05-23 21:03:50 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 21:03:50 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:54169 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:03:50 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:54169 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:03:50 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 21:03:50 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 43 ms on localhost (1/2) -2016-05-23 21:03:50 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 49 ms on localhost (2/2) -2016-05-23 21:03:50 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.049 s -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 21:03:50 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:03:50 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:03:50 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 21:03:50 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:03:50 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=356700, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=361556, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:03:50 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:54169 (size: 2.5 KB, free: 730.6 MB) -2016-05-23 21:03:50 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:50 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:50 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 23 ms on localhost (1/1) -2016-05-23 21:03:50 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.023 s -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 21:03:50 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.133618 s -2016-05-23 21:03:50 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:03:50 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 21:03:50 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 21:03:50 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:03:50 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:03:50 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 21:03:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=364087, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=369743, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:03:50 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:54169 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:03:50 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:50 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 21:03:50 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:03:50 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 21 ms on localhost (1/2) -2016-05-23 21:03:50 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 28 ms on localhost (2/2) -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 21:03:50 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.028 s -2016-05-23 21:03:50 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.044717 s -2016-05-23 21:03:50 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:03:50 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:03:50 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 21:03:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=372651, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=378347, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:03:50 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:54169 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:03:50 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:50 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:50 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:03:50 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 21:03:50 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 35 ms on localhost (1/2) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 34 ms on localhost (2/2) -2016-05-23 21:03:50 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.036 s -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 21:03:50 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.063570 s -2016-05-23 21:03:50 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:45 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:45) with 4 output partitions -2016-05-23 21:03:50 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:45) -2016-05-23 21:03:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:03:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:42), which has no missing parents -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=381236, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=382460, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:03:50 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:54169 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:03:50 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:42) -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:03:50 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 21:03:50 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 21:03:50 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 21:03:50 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 21:03:50 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 21:03:50 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 21:03:50 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 21:03:50 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 30 ms on localhost (1/4) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 16 ms on localhost (2/4) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 17 ms on localhost (3/4) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 23 ms on localhost (4/4) -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 21:03:50 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:45) finished in 0.034 s -2016-05-23 21:03:50 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:45, took 0.050219 s -2016-05-23 21:03:50 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:47 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:47) with 4 output partitions -2016-05-23 21:03:50 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:47) -2016-05-23 21:03:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:03:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:43), which has no missing parents -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=383353, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=384577, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:03:50 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:54169 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:03:50 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:43) -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:03:50 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 21:03:50 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 21:03:50 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 21:03:50 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 21:03:50 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 21:03:50 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 21:03:50 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 21:03:50 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 10 ms on localhost (1/4) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 11 ms on localhost (2/4) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 12 ms on localhost (3/4) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 11 ms on localhost (4/4) -2016-05-23 21:03:50 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:47) finished in 0.014 s -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 21:03:50 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:47, took 0.023803 s -2016-05-23 21:03:50 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:03:50 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 21:03:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:03:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=385470, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(1525) called with curMem=387974, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1525.0 B, free 730.2 MB) -2016-05-23 21:03:50 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:54169 (size: 1525.0 B, free: 730.5 MB) -2016-05-23 21:03:50 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:03:50 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 21:03:50 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 8 ms on localhost (1/1) -2016-05-23 21:03:50 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.008 s -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 21:03:50 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.024266 s -2016-05-23 21:03:50 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:03:50 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:03:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:03:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=389499, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=392899, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.2 MB) -2016-05-23 21:03:50 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:54169 (size: 1935.0 B, free: 730.5 MB) -2016-05-23 21:03:50 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:03:50 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 21:03:50 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 21:03:50 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 21:03:50 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 21:03:50 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 21:03:50 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 21:03:50 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 10 ms on localhost (1/4) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 16 ms on localhost (2/4) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 15 ms on localhost (3/4) -2016-05-23 21:03:50 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 22 ms on localhost (4/4) -2016-05-23 21:03:50 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.024 s -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 21:03:50 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.037010 s -2016-05-23 21:03:50 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:03:50 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:03:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:03:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=394834, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(1919) called with curMem=398274, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1919.0 B, free 730.2 MB) -2016-05-23 21:03:50 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:54169 (size: 1919.0 B, free: 730.5 MB) -2016-05-23 21:03:50 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:03:50 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 21:03:50 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 21:03:50 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 21:03:50 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 21:03:50 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 21:03:50 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 21:03:50 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 13 ms on localhost (1/4) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 10 ms on localhost (2/4) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 17 ms on localhost (3/4) -2016-05-23 21:03:50 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 15 ms on localhost (4/4) -2016-05-23 21:03:50 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.018 s -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 21:03:50 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.033457 s -2016-05-23 21:03:50 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:03:50 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:03:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:03:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=400193, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(1861) called with curMem=403569, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1861.0 B, free 730.2 MB) -2016-05-23 21:03:50 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:54169 (size: 1861.0 B, free: 730.5 MB) -2016-05-23 21:03:50 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:03:50 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 21:03:50 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 21:03:50 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 21:03:50 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 21:03:50 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 21:03:50 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 21:03:50 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 19 ms on localhost (1/4) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 24 ms on localhost (2/4) -2016-05-23 21:03:50 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 23 ms on localhost (3/4) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 25 ms on localhost (4/4) -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 21:03:50 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.028 s -2016-05-23 21:03:50 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.046367 s -2016-05-23 21:03:50 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:03:50 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:03:50 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:03:50 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 21:03:50 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=405430, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(2283) called with curMem=409470, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:03:50 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:54169 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:03:50 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 21:03:50 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 21:03:50 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 21:03:50 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 21:03:50 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 21:03:50 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 69 ms on localhost (1/4) -2016-05-23 21:03:50 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 74 ms on localhost (2/4) -2016-05-23 21:03:50 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 78 ms on localhost (3/4) -2016-05-23 21:03:50 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 86 ms on localhost (4/4) -2016-05-23 21:03:50 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.088 s -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 21:03:50 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:03:50 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:03:50 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 21:03:50 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:03:50 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=411753, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:03:50 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=414297, maxMem=766075207 -2016-05-23 21:03:50 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 21:03:50 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:54169 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 21:03:50 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:50 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:50 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 21:03:50 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 21:03:50 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:50 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 21:03:50 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 21:03:50 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 37 ms on localhost (1/3) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 35 ms on localhost (2/3) -2016-05-23 21:03:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 40 ms on localhost (3/3) -2016-05-23 21:03:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 21:03:50 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.045 s -2016-05-23 21:03:50 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.165370 s -2016-05-23 21:03:51 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:03:51 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 21:03:51 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:03:51 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=415865, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=420097, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 21:03:51 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:54169 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:03:51 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:03:51 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 21:03:51 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 21:03:51 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:51 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:51 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 21:03:51 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 87 ms on localhost (1/4) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 91 ms on localhost (2/4) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 21:03:51 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 155 ms on localhost (3/4) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 154 ms on localhost (4/4) -2016-05-23 21:03:51 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.157 s -2016-05-23 21:03:51 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:03:51 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:03:51 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 21:03:51 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:03:51 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=422403, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=427299, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:03:51 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:54169 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:03:51 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:51 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:03:51 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 17 ms on localhost (1/1) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 21:03:51 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.017 s -2016-05-23 21:03:51 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.208085 s -2016-05-23 21:03:51 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:03:51 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 21:03:51 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 21:03:51 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:03:51 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=429841, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=435537, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:03:51 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:54169 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:03:51 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:51 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 21:03:51 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 21:03:51 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:51 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 21:03:51 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 18 ms on localhost (1/4) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 20 ms on localhost (2/4) -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:51 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:03:51 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 39 ms on localhost (3/4) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 44 ms on localhost (4/4) -2016-05-23 21:03:51 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.045 s -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 21:03:51 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.060736 s -2016-05-23 21:03:51 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:03:51 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=438446, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=444182, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:03:51 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:54169 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:03:51 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:51 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 21:03:51 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 21:03:51 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:51 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:51 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 21:03:51 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 30 ms on localhost (1/4) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 35 ms on localhost (2/4) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 45 ms on localhost (3/4) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 44 ms on localhost (4/4) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 21:03:51 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.048 s -2016-05-23 21:03:51 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.070208 s -2016-05-23 21:03:51 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:61 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Got job 20 (foreach at StatisticsCorrLearning.scala:61) with 4 output partitions -2016-05-23 21:03:51 INFO DAGScheduler:59 - Final stage: ResultStage 34(foreach at StatisticsCorrLearning.scala:61) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:03:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:58), which has no missing parents -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=447074, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=448298, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:03:51 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:54169 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:03:51 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:58) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 62, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 63, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 64, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:03:51 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 62) -2016-05-23 21:03:51 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 65) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 62). 915 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 62) in 9 ms on localhost (1/4) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 65). 915 bytes result sent to driver -2016-05-23 21:03:51 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 64) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 64). 915 bytes result sent to driver -2016-05-23 21:03:51 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 63) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 63). 915 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 65) in 20 ms on localhost (2/4) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 64) in 21 ms on localhost (3/4) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 63) in 25 ms on localhost (4/4) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-23 21:03:51 INFO DAGScheduler:59 - ResultStage 34 (foreach at StatisticsCorrLearning.scala:61) finished in 0.028 s -2016-05-23 21:03:51 INFO DAGScheduler:59 - Job 20 finished: foreach at StatisticsCorrLearning.scala:61, took 0.041712 s -2016-05-23 21:03:51 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:63 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Got job 21 (foreach at StatisticsCorrLearning.scala:63) with 4 output partitions -2016-05-23 21:03:51 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at StatisticsCorrLearning.scala:63) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:03:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:59), which has no missing parents -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=449191, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=450415, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:03:51 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:54169 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:03:51 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:59) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 4 tasks -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 66, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:03:51 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 68) -2016-05-23 21:03:51 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 66) -2016-05-23 21:03:51 INFO Executor:59 - Running task 3.0 in stage 35.0 (TID 69) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 68). 915 bytes result sent to driver -2016-05-23 21:03:51 INFO Executor:59 - Finished task 3.0 in stage 35.0 (TID 69). 915 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 68) in 8 ms on localhost (1/4) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 3.0 in stage 35.0 (TID 69) in 8 ms on localhost (2/4) -2016-05-23 21:03:51 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 67) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 67). 915 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 67) in 23 ms on localhost (3/4) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 66). 915 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 66) in 33 ms on localhost (4/4) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-23 21:03:51 INFO DAGScheduler:59 - ResultStage 35 (foreach at StatisticsCorrLearning.scala:63) finished in 0.036 s -2016-05-23 21:03:51 INFO DAGScheduler:59 - Job 21 finished: foreach at StatisticsCorrLearning.scala:63, took 0.055716 s -2016-05-23 21:03:51 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Got job 22 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:03:51 INFO DAGScheduler:59 - Final stage: ResultStage 36(first at RowMatrix.scala:65) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:03:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=451308, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(1523) called with curMem=453812, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1523.0 B, free 730.2 MB) -2016-05-23 21:03:51 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:54169 (size: 1523.0 B, free: 730.5 MB) -2016-05-23 21:03:51 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:03:51 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 70) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 70). 935 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 70) in 7 ms on localhost (1/1) -2016-05-23 21:03:51 INFO DAGScheduler:59 - ResultStage 36 (first at RowMatrix.scala:65) finished in 0.008 s -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-23 21:03:51 INFO DAGScheduler:59 - Job 22 finished: first at RowMatrix.scala:65, took 0.024113 s -2016-05-23 21:03:51 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Got job 23 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 21:03:51 INFO DAGScheduler:59 - Final stage: ResultStage 37(first at RowMatrix.scala:65) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:03:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=455335, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(1523) called with curMem=457839, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1523.0 B, free 730.1 MB) -2016-05-23 21:03:51 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:54169 (size: 1523.0 B, free: 730.5 MB) -2016-05-23 21:03:51 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 37 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 3 tasks -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:03:51 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 72) -2016-05-23 21:03:51 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-23 21:03:51 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 73) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 1046 bytes result sent to driver -2016-05-23 21:03:51 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 72). 1046 bytes result sent to driver -2016-05-23 21:03:51 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 73). 1046 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 11 ms on localhost (1/3) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 73) in 11 ms on localhost (2/3) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 72) in 13 ms on localhost (3/3) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-23 21:03:51 INFO DAGScheduler:59 - ResultStage 37 (first at RowMatrix.scala:65) finished in 0.015 s -2016-05-23 21:03:51 INFO DAGScheduler:59 - Job 23 finished: first at RowMatrix.scala:65, took 0.030165 s -2016-05-23 21:03:51 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Got job 24 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:03:51 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:03:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=459362, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(1933) called with curMem=462762, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1933.0 B, free 730.1 MB) -2016-05-23 21:03:51 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:54169 (size: 1933.0 B, free: 730.5 MB) -2016-05-23 21:03:51 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 74, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 77, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:03:51 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 75) -2016-05-23 21:03:51 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 76) -2016-05-23 21:03:51 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 77) -2016-05-23 21:03:51 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 74) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 74). 1231 bytes result sent to driver -2016-05-23 21:03:51 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 77). 1231 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 74) in 9 ms on localhost (1/4) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 77) in 8 ms on localhost (2/4) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 75). 1231 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 75) in 12 ms on localhost (3/4) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 76). 1231 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 76) in 18 ms on localhost (4/4) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-23 21:03:51 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at RowMatrix.scala:332) finished in 0.020 s -2016-05-23 21:03:51 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at RowMatrix.scala:332, took 0.032153 s -2016-05-23 21:03:51 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Got job 25 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:03:51 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:03:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=464695, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(1917) called with curMem=468135, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1917.0 B, free 730.1 MB) -2016-05-23 21:03:51 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:54169 (size: 1917.0 B, free: 730.5 MB) -2016-05-23 21:03:51 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 39 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 4 tasks -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 79, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 80, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 39.0 (TID 81, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:03:51 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 79) -2016-05-23 21:03:51 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 80) -2016-05-23 21:03:51 INFO Executor:59 - Running task 3.0 in stage 39.0 (TID 81) -2016-05-23 21:03:51 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 78) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 3.0 in stage 39.0 (TID 81). 1109 bytes result sent to driver -2016-05-23 21:03:51 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 78). 1109 bytes result sent to driver -2016-05-23 21:03:51 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 79). 1109 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 3.0 in stage 39.0 (TID 81) in 11 ms on localhost (1/4) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 80). 1109 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 79) in 15 ms on localhost (2/4) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 80) in 14 ms on localhost (3/4) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 78) in 17 ms on localhost (4/4) -2016-05-23 21:03:51 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at RowMatrix.scala:124) finished in 0.017 s -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-23 21:03:51 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at RowMatrix.scala:124, took 0.032161 s -2016-05-23 21:03:51 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Got job 26 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:03:51 INFO DAGScheduler:59 - Final stage: ResultStage 40(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:03:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=470052, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(1852) called with curMem=473428, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 1852.0 B, free 730.1 MB) -2016-05-23 21:03:51 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:54169 (size: 1852.0 B, free: 730.5 MB) -2016-05-23 21:03:51 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 40 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 4 tasks -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 82, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 83, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 84, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 40.0 (TID 85, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:03:51 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 82) -2016-05-23 21:03:51 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 84) -2016-05-23 21:03:51 INFO Executor:59 - Running task 3.0 in stage 40.0 (TID 85) -2016-05-23 21:03:51 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 83) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 82). 1104 bytes result sent to driver -2016-05-23 21:03:51 INFO Executor:59 - Finished task 3.0 in stage 40.0 (TID 85). 1254 bytes result sent to driver -2016-05-23 21:03:51 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 84). 1249 bytes result sent to driver -2016-05-23 21:03:51 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 83). 1254 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 82) in 13 ms on localhost (1/4) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 3.0 in stage 40.0 (TID 85) in 11 ms on localhost (2/4) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 84) in 13 ms on localhost (3/4) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 83) in 14 ms on localhost (4/4) -2016-05-23 21:03:51 INFO DAGScheduler:59 - ResultStage 40 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.016 s -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-23 21:03:51 INFO DAGScheduler:59 - Job 26 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.031494 s -2016-05-23 21:03:51 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Registering RDD 51 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Got job 27 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:03:51 INFO DAGScheduler:59 - Final stage: ResultStage 42(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 41) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 41) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting ShuffleMapStage 41 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=475280, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(2280) called with curMem=479320, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 21:03:51 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:54169 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:03:51 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 41 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 4 tasks -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 86, localhost, PROCESS_LOCAL, 2221 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 87, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 88, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 41.0 (TID 89, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:03:51 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 88) -2016-05-23 21:03:51 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 86) -2016-05-23 21:03:51 INFO Executor:59 - Running task 3.0 in stage 41.0 (TID 89) -2016-05-23 21:03:51 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 87) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 86). 1161 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 86) in 18 ms on localhost (1/4) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 3.0 in stage 41.0 (TID 89). 1161 bytes result sent to driver -2016-05-23 21:03:51 INFO Executor:59 - Finished task 2.0 in stage 41.0 (TID 88). 1161 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 3.0 in stage 41.0 (TID 89) in 47 ms on localhost (2/4) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 41.0 (TID 88) in 49 ms on localhost (3/4) -2016-05-23 21:03:51 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 87). 1161 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 87) in 85 ms on localhost (4/4) -2016-05-23 21:03:51 INFO DAGScheduler:59 - ShuffleMapStage 41 (flatMap at SpearmanCorrelation.scala:49) finished in 0.087 s -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-23 21:03:51 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:03:51 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:03:51 INFO DAGScheduler:59 - waiting: Set(ResultStage 42) -2016-05-23 21:03:51 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:03:51 INFO DAGScheduler:59 - Missing parents for ResultStage 42: List() -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting ResultStage 42 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=481600, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(1563) called with curMem=484144, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 1563.0 B, free 730.1 MB) -2016-05-23 21:03:51 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:54169 (size: 1563.0 B, free: 730.5 MB) -2016-05-23 21:03:51 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 42 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 3 tasks -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 90, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 91, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 42.0 (TID 92, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:51 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 91) -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:51 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 91). 1203 bytes result sent to driver -2016-05-23 21:03:51 INFO Executor:59 - Running task 2.0 in stage 42.0 (TID 92) -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:51 INFO Executor:59 - Finished task 2.0 in stage 42.0 (TID 92). 1203 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 91) in 16 ms on localhost (1/3) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 42.0 (TID 92) in 17 ms on localhost (2/3) -2016-05-23 21:03:51 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 90) -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:03:51 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 90). 1203 bytes result sent to driver -2016-05-23 21:03:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 90) in 44 ms on localhost (3/3) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-23 21:03:51 INFO DAGScheduler:59 - ResultStage 42 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.044 s -2016-05-23 21:03:51 INFO DAGScheduler:59 - Job 27 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.159436 s -2016-05-23 21:03:51 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:03:51 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 168 bytes -2016-05-23 21:03:51 INFO DAGScheduler:59 - Registering RDD 56 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Got job 28 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:03:51 INFO DAGScheduler:59 - Final stage: ResultStage 45(first at RowMatrix.scala:65) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 44) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 44) -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting ShuffleMapStage 44 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=485707, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 21:03:51 INFO MemoryStore:59 - ensureFreeSpace(2301) called with curMem=489939, maxMem=766075207 -2016-05-23 21:03:51 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 21:03:51 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:54169 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:03:51 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 44 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:03:51 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 4 tasks -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 93, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 94, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 44.0 (TID 95, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:03:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 44.0 (TID 96, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:03:51 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 94) -2016-05-23 21:03:51 INFO Executor:59 - Running task 3.0 in stage 44.0 (TID 96) -2016-05-23 21:03:51 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 93) -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:51 INFO Executor:59 - Running task 2.0 in stage 44.0 (TID 95) -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:03:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:52 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 93). 1377 bytes result sent to driver -2016-05-23 21:03:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 93) in 43 ms on localhost (1/4) -2016-05-23 21:03:52 INFO Executor:59 - Finished task 3.0 in stage 44.0 (TID 96). 1377 bytes result sent to driver -2016-05-23 21:03:52 INFO TaskSetManager:59 - Finished task 3.0 in stage 44.0 (TID 96) in 41 ms on localhost (2/4) -2016-05-23 21:03:52 INFO Executor:59 - Finished task 2.0 in stage 44.0 (TID 95). 1377 bytes result sent to driver -2016-05-23 21:03:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 44.0 (TID 95) in 196 ms on localhost (3/4) -2016-05-23 21:03:52 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 94). 1377 bytes result sent to driver -2016-05-23 21:03:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 94) in 201 ms on localhost (4/4) -2016-05-23 21:03:52 INFO DAGScheduler:59 - ShuffleMapStage 44 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.202 s -2016-05-23 21:03:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-23 21:03:52 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:03:52 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:03:52 INFO DAGScheduler:59 - waiting: Set(ResultStage 45) -2016-05-23 21:03:52 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:03:52 INFO DAGScheduler:59 - Missing parents for ResultStage 45: List() -2016-05-23 21:03:52 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:03:52 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=492240, maxMem=766075207 -2016-05-23 21:03:52 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 21:03:52 INFO MemoryStore:59 - ensureFreeSpace(2541) called with curMem=497136, maxMem=766075207 -2016-05-23 21:03:52 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:54169 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:03:52 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:52 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 45 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:03:52 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 1 tasks -2016-05-23 21:03:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 97, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:52 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 97) -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:03:52 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 97). 1185 bytes result sent to driver -2016-05-23 21:03:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 97) in 8 ms on localhost (1/1) -2016-05-23 21:03:52 INFO DAGScheduler:59 - ResultStage 45 (first at RowMatrix.scala:65) finished in 0.008 s -2016-05-23 21:03:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-23 21:03:52 INFO DAGScheduler:59 - Job 28 finished: first at RowMatrix.scala:65, took 0.229938 s -2016-05-23 21:03:52 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:03:52 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 168 bytes -2016-05-23 21:03:52 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 166 bytes -2016-05-23 21:03:52 INFO DAGScheduler:59 - Got job 29 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 21:03:52 INFO DAGScheduler:59 - Final stage: ResultStage 48(first at RowMatrix.scala:65) -2016-05-23 21:03:52 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 47) -2016-05-23 21:03:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:52 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which has no missing parents -2016-05-23 21:03:52 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=499677, maxMem=766075207 -2016-05-23 21:03:52 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 21:03:52 INFO MemoryStore:59 - ensureFreeSpace(2541) called with curMem=504573, maxMem=766075207 -2016-05-23 21:03:52 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:54169 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:03:52 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:52 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 48 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:03:52 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 3 tasks -2016-05-23 21:03:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 98, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 99, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 48.0 (TID 100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:52 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 98) -2016-05-23 21:03:52 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 99) -2016-05-23 21:03:52 INFO Executor:59 - Running task 2.0 in stage 48.0 (TID 100) -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:52 INFO Executor:59 - Finished task 2.0 in stage 48.0 (TID 100). 1296 bytes result sent to driver -2016-05-23 21:03:52 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 99). 1296 bytes result sent to driver -2016-05-23 21:03:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 48.0 (TID 100) in 14 ms on localhost (1/3) -2016-05-23 21:03:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 99) in 15 ms on localhost (2/3) -2016-05-23 21:03:52 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 98). 1296 bytes result sent to driver -2016-05-23 21:03:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 98) in 17 ms on localhost (3/3) -2016-05-23 21:03:52 INFO DAGScheduler:59 - ResultStage 48 (first at RowMatrix.scala:65) finished in 0.017 s -2016-05-23 21:03:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-23 21:03:52 INFO DAGScheduler:59 - Job 29 finished: first at RowMatrix.scala:65, took 0.030738 s -2016-05-23 21:03:52 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:03:52 INFO DAGScheduler:59 - Got job 30 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:03:52 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:03:52 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 50) -2016-05-23 21:03:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:52 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:03:52 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=507114, maxMem=766075207 -2016-05-23 21:03:52 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 21:03:52 INFO MemoryStore:59 - ensureFreeSpace(2910) called with curMem=512810, maxMem=766075207 -2016-05-23 21:03:52 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:54169 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:54169 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:03:52 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:52 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 51 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:03:52 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 4 tasks -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-23 21:03:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 51.0 (TID 103, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:52 INFO TaskSetManager:59 - Starting task 3.0 in stage 51.0 (TID 104, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:52 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-23 21:03:52 INFO Executor:59 - Running task 2.0 in stage 51.0 (TID 103) -2016-05-23 21:03:52 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:54169 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:03:52 INFO Executor:59 - Running task 3.0 in stage 51.0 (TID 104) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:54169 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:03:52 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 1481 bytes result sent to driver -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:52 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 1481 bytes result sent to driver -2016-05-23 21:03:52 INFO Executor:59 - Finished task 2.0 in stage 51.0 (TID 103). 1481 bytes result sent to driver -2016-05-23 21:03:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 15 ms on localhost (1/4) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-23 21:03:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 17 ms on localhost (2/4) -2016-05-23 21:03:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 51.0 (TID 103) in 16 ms on localhost (3/4) -2016-05-23 21:03:52 INFO Executor:59 - Finished task 3.0 in stage 51.0 (TID 104). 1481 bytes result sent to driver -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:54169 in memory (size: 1563.0 B, free: 730.5 MB) -2016-05-23 21:03:52 INFO TaskSetManager:59 - Finished task 3.0 in stage 51.0 (TID 104) in 19 ms on localhost (4/4) -2016-05-23 21:03:52 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at RowMatrix.scala:332) finished in 0.021 s -2016-05-23 21:03:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-23 21:03:52 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at RowMatrix.scala:332, took 0.054727 s -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:54169 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:54169 in memory (size: 1852.0 B, free: 730.5 MB) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-23 21:03:52 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:03:52 INFO DAGScheduler:59 - Got job 31 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:03:52 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:03:52 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 53) -2016-05-23 21:03:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:54169 in memory (size: 1917.0 B, free: 730.5 MB) -2016-05-23 21:03:52 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-23 21:03:52 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=473301, maxMem=766075207 -2016-05-23 21:03:52 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:54169 in memory (size: 1933.0 B, free: 730.5 MB) -2016-05-23 21:03:52 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=477104, maxMem=766075207 -2016-05-23 21:03:52 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:54169 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:03:52 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:03:52 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 54 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:03:52 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 4 tasks -2016-05-23 21:03:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 105, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 106, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:54169 in memory (size: 1523.0 B, free: 730.5 MB) -2016-05-23 21:03:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:52 INFO TaskSetManager:59 - Starting task 3.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:03:52 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 105) -2016-05-23 21:03:52 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 106) -2016-05-23 21:03:52 INFO Executor:59 - Running task 2.0 in stage 54.0 (TID 107) -2016-05-23 21:03:52 INFO Executor:59 - Running task 3.0 in stage 54.0 (TID 108) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:54169 in memory (size: 1523.0 B, free: 730.5 MB) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-23 21:03:52 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 106). 1359 bytes result sent to driver -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:54169 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:54169 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:03:52 INFO Executor:59 - Finished task 3.0 in stage 54.0 (TID 108). 1359 bytes result sent to driver -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-23 21:03:52 INFO Executor:59 - Finished task 2.0 in stage 54.0 (TID 107). 1359 bytes result sent to driver -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:03:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 3 ms -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:54169 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:54169 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:03:52 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 105). 1359 bytes result sent to driver -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-23 21:03:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 106) in 38 ms on localhost (1/4) -2016-05-23 21:03:52 INFO TaskSetManager:59 - Finished task 3.0 in stage 54.0 (TID 108) in 37 ms on localhost (2/4) -2016-05-23 21:03:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 54.0 (TID 107) in 40 ms on localhost (3/4) -2016-05-23 21:03:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 105) in 41 ms on localhost (4/4) -2016-05-23 21:03:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-23 21:03:52 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at RowMatrix.scala:124) finished in 0.043 s -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:54169 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:03:52 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at RowMatrix.scala:124, took 0.057861 s -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:54169 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:54169 in memory (size: 1568.0 B, free: 730.5 MB) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:54169 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-05-23 21:03:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:54169 in memory (size: 1861.0 B, free: 730.5 MB) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:54169 in memory (size: 1919.0 B, free: 730.5 MB) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:54169 in memory (size: 1935.0 B, free: 730.5 MB) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:54169 in memory (size: 1525.0 B, free: 730.5 MB) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:54169 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:54169 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:54169 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:54169 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:54169 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-23 21:03:52 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:54169 in memory (size: 2.2 KB, free: 730.6 MB) -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-23 21:03:52 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-23 21:03:52 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 21:03:52 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 21:03:52 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 21:03:54 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 21:03:54 INFO BlockManager:59 - BlockManager stopped -2016-05-23 21:03:54 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 21:03:54 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 21:03:54 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 21:03:54 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 21:03:54 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 21:03:55 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-05-23 21:03:55 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 21:03:55 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-2c53b2b5-27aa-402b-b765-3672d9b76837 -2016-05-23 21:04:11 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 21:04:12 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 21:04:12 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 21:04:12 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 21:04:12 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 21:04:13 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 21:04:13 INFO Remoting:74 - Starting remoting -2016-05-23 21:04:13 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:54218] -2016-05-23 21:04:13 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54218. -2016-05-23 21:04:13 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 21:04:13 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 21:04:13 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-e5376fa9-9e5f-4f1f-9ace-f5ec7fa2dbf7 -2016-05-23 21:04:13 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 21:04:13 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-3df2577b-1a98-4d2a-ac34-b56a9143f44a\httpd-59507a10-23a8-4037-9679-57f2b602d9d0 -2016-05-23 21:04:13 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 21:04:13 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 21:04:13 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54219 -2016-05-23 21:04:13 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54219. -2016-05-23 21:04:13 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 21:04:14 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 21:04:14 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 21:04:14 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 21:04:14 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 21:04:14 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 21:04:14 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 21:04:14 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54239. -2016-05-23 21:04:14 INFO NettyBlockTransferService:59 - Server created on 54239 -2016-05-23 21:04:14 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 21:04:14 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54239 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54239) -2016-05-23 21:04:14 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 21:04:15 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 21:04:15 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 21:04:15 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 21:04:15 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 21:04:15 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54239 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 21:04:15 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:22 -2016-05-23 21:04:15 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 21:04:15 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 21:04:15 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 21:04:15 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 21:04:15 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54239 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 21:04:15 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:25 -2016-05-23 21:04:16 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 21:04:16 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 21:04:16 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 21:04:16 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 21:04:16 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 21:04:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:04:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:16 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:23), which has no missing parents -2016-05-23 21:04:17 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 21:04:17 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 21:04:17 INFO MemoryStore:59 - ensureFreeSpace(2043) called with curMem=346712, maxMem=766075207 -2016-05-23 21:04:17 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2043.0 B, free 730.3 MB) -2016-05-23 21:04:17 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54239 (size: 2043.0 B, free: 730.6 MB) -2016-05-23 21:04:17 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:23) -2016-05-23 21:04:17 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 21:04:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:04:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:04:17 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 21:04:17 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 21:04:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:04:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:04:17 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 21:04:17 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 21:04:17 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 21:04:17 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 21:04:17 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 21:04:17 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 21:04:17 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 21:04:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 136 ms on localhost (1/2) -2016-05-23 21:04:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 118 ms on localhost (2/2) -2016-05-23 21:04:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 21:04:17 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:29) finished in 0.158 s -2016-05-23 21:04:17 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:29, took 0.308407 s -2016-05-23 21:04:17 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 21:04:17 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:31 -2016-05-23 21:04:17 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:31) with 2 output partitions -2016-05-23 21:04:17 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:31) -2016-05-23 21:04:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:04:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:17 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:26), which has no missing parents -2016-05-23 21:04:17 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348755, maxMem=766075207 -2016-05-23 21:04:17 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 21:04:17 INFO MemoryStore:59 - ensureFreeSpace(2043) called with curMem=352267, maxMem=766075207 -2016-05-23 21:04:17 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2043.0 B, free 730.2 MB) -2016-05-23 21:04:17 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54239 (size: 2043.0 B, free: 730.6 MB) -2016-05-23 21:04:17 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:26) -2016-05-23 21:04:17 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 21:04:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:04:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:04:17 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 21:04:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:04:17 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 21:04:17 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 21:04:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:04:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 37 ms on localhost (1/2) -2016-05-23 21:04:17 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 21:04:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 41 ms on localhost (2/2) -2016-05-23 21:04:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 21:04:17 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:31) finished in 0.044 s -2016-05-23 21:04:17 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:31, took 0.073297 s -2016-05-23 21:04:17 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:04:17 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:04:17 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 21:04:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:04:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:17 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:04:17 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354310, maxMem=766075207 -2016-05-23 21:04:17 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 21:04:17 INFO MemoryStore:59 - ensureFreeSpace(2465) called with curMem=358870, maxMem=766075207 -2016-05-23 21:04:17 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:04:17 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54239 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 21:04:17 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 21:04:17 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 21:04:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:04:17 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 21:04:17 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:54239 in memory (size: 2043.0 B, free: 730.6 MB) -2016-05-23 21:04:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:04:17 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:54239 in memory (size: 2043.0 B, free: 730.6 MB) -2016-05-23 21:04:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:04:17 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 21:04:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 44 ms on localhost (1/1) -2016-05-23 21:04:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 21:04:17 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.046 s -2016-05-23 21:04:17 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.100629 s -2016-05-23 21:04:17 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:04:17 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:04:17 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:04:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:04:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:17 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:04:17 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=350225, maxMem=766075207 -2016-05-23 21:04:17 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 21:04:17 INFO MemoryStore:59 - ensureFreeSpace(2864) called with curMem=355673, maxMem=766075207 -2016-05-23 21:04:17 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:04:17 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:54239 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:04:17 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:04:17 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 21:04:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:04:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:04:17 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 21:04:17 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 21:04:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:04:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:04:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:04:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:04:18 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 21:04:18 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 21:04:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 46 ms on localhost (1/2) -2016-05-23 21:04:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 45 ms on localhost (2/2) -2016-05-23 21:04:18 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.048 s -2016-05-23 21:04:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 21:04:18 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 21:04:18 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 21:04:18 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.079147 s -2016-05-23 21:04:18 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:04:18 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:04:18 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:04:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:04:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:18 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:04:18 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=358537, maxMem=766075207 -2016-05-23 21:04:18 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 21:04:18 INFO MemoryStore:59 - ensureFreeSpace(2855) called with curMem=364033, maxMem=766075207 -2016-05-23 21:04:18 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:04:18 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:54239 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:04:18 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:04:18 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 21:04:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:04:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:04:18 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 21:04:18 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 21:04:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:04:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:04:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:04:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:04:18 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 21:04:18 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 21:04:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 38 ms on localhost (1/2) -2016-05-23 21:04:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 38 ms on localhost (2/2) -2016-05-23 21:04:18 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.040 s -2016-05-23 21:04:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 21:04:18 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.065369 s -2016-05-23 21:04:18 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:04:18 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 21:04:18 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:04:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:04:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:18 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:04:18 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=366888, maxMem=766075207 -2016-05-23 21:04:18 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 21:04:18 INFO MemoryStore:59 - ensureFreeSpace(2745) called with curMem=372232, maxMem=766075207 -2016-05-23 21:04:18 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 21:04:18 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:54239 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 21:04:18 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:04:18 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 21:04:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:04:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:04:18 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 21:04:18 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 21:04:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:04:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:04:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:04:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:04:18 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 21:04:18 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 21:04:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 37 ms on localhost (1/2) -2016-05-23 21:04:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 39 ms on localhost (2/2) -2016-05-23 21:04:18 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.039 s -2016-05-23 21:04:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 21:04:18 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.057610 s -2016-05-23 21:04:18 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:04:18 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:04:18 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 21:04:18 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:04:18 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 21:04:18 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 21:04:18 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:04:18 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=374977, maxMem=766075207 -2016-05-23 21:04:18 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 21:04:18 INFO MemoryStore:59 - ensureFreeSpace(3157) called with curMem=380905, maxMem=766075207 -2016-05-23 21:04:18 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 21:04:18 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:54239 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 21:04:18 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:04:18 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 21:04:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 21:04:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 21:04:18 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 21:04:18 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 21:04:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:04:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:04:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:04:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:04:18 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 21:04:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 73 ms on localhost (1/2) -2016-05-23 21:04:18 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 21:04:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 80 ms on localhost (2/2) -2016-05-23 21:04:18 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.080 s -2016-05-23 21:04:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 21:04:18 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:04:18 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:04:18 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 21:04:18 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:04:18 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 21:04:18 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:04:18 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=384062, maxMem=766075207 -2016-05-23 21:04:18 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:04:18 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=386566, maxMem=766075207 -2016-05-23 21:04:18 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 21:04:18 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:54239 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 21:04:18 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:04:18 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 21:04:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:18 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 21:04:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:04:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-23 21:04:18 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 21:04:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 66 ms on localhost (1/1) -2016-05-23 21:04:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 21:04:18 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.067 s -2016-05-23 21:04:18 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.201981 s -2016-05-23 21:04:18 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:04:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 21:04:18 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:04:18 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:04:18 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 21:04:18 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 21:04:18 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 21:04:18 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:04:18 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=388111, maxMem=766075207 -2016-05-23 21:04:18 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:04:18 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=388254, maxMem=766075207 -2016-05-23 21:04:18 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:04:18 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:54239 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 21:04:18 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:54239 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:04:18 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 21:04:18 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:04:18 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 21:04:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:04:18 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:54239 in memory (size: 3.1 KB, free: 730.5 MB) -2016-05-23 21:04:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:04:18 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 21:04:18 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 21:04:18 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 21:04:18 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:54239 in memory (size: 2.7 KB, free: 730.5 MB) -2016-05-23 21:04:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:04:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:04:18 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 21:04:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:04:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:04:18 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:54239 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:04:18 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 21:04:18 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:54239 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:04:18 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 21:04:18 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:54239 in memory (size: 2.4 KB, free: 730.6 MB) -2016-05-23 21:04:18 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 21:04:18 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 21:04:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 49 ms on localhost (1/2) -2016-05-23 21:04:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 54 ms on localhost (2/2) -2016-05-23 21:04:18 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.054 s -2016-05-23 21:04:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 21:04:18 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:04:18 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:04:18 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 21:04:18 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:04:18 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 21:04:18 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:04:18 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=349674, maxMem=766075207 -2016-05-23 21:04:18 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 21:04:18 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=354530, maxMem=766075207 -2016-05-23 21:04:18 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:04:18 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:54239 (size: 2.5 KB, free: 730.6 MB) -2016-05-23 21:04:18 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:04:18 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 21:04:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:18 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 21:04:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:04:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:04:19 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 22 ms on localhost (1/1) -2016-05-23 21:04:19 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.022 s -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 21:04:19 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.136507 s -2016-05-23 21:04:19 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:04:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 21:04:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 21:04:19 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:04:19 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=357061, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=362717, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:04:19 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:54239 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:04:19 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:19 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 21:04:19 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:04:19 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 20 ms on localhost (1/2) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 22 ms on localhost (2/2) -2016-05-23 21:04:19 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.023 s -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 21:04:19 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.038578 s -2016-05-23 21:04:19 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:04:19 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=365625, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=371321, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:04:19 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:54239 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:04:19 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:19 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 21:04:19 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:04:19 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 21:04:19 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 20 ms on localhost (1/2) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 23 ms on localhost (2/2) -2016-05-23 21:04:19 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.023 s -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 21:04:19 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.039923 s -2016-05-23 21:04:19 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:45 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:45) with 4 output partitions -2016-05-23 21:04:19 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:45) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:04:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:42), which has no missing parents -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=374210, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=375434, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:04:19 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:54239 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:04:19 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:42) -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:04:19 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 21:04:19 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 21:04:19 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 21:04:19 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 21:04:19 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 18 ms on localhost (1/4) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 13 ms on localhost (2/4) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 13 ms on localhost (3/4) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 16 ms on localhost (4/4) -2016-05-23 21:04:19 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:45) finished in 0.025 s -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 21:04:19 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:45, took 0.040142 s -2016-05-23 21:04:19 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:47 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:47) with 4 output partitions -2016-05-23 21:04:19 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:47) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:04:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:43), which has no missing parents -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=376327, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=377551, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:04:19 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:54239 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:04:19 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:43) -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:04:19 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 21:04:19 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 21:04:19 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 21:04:19 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 21:04:19 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 21:04:19 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 7 ms on localhost (1/4) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 9 ms on localhost (2/4) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 12 ms on localhost (3/4) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 13 ms on localhost (4/4) -2016-05-23 21:04:19 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:47) finished in 0.015 s -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 21:04:19 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:47, took 0.026261 s -2016-05-23 21:04:19 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:04:19 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:04:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=378444, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(1525) called with curMem=380948, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1525.0 B, free 730.2 MB) -2016-05-23 21:04:19 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:54239 (size: 1525.0 B, free: 730.5 MB) -2016-05-23 21:04:19 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:04:19 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 6 ms on localhost (1/1) -2016-05-23 21:04:19 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.007 s -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 21:04:19 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.018788 s -2016-05-23 21:04:19 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:04:19 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:04:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=382473, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=385873, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.2 MB) -2016-05-23 21:04:19 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:54239 (size: 1935.0 B, free: 730.5 MB) -2016-05-23 21:04:19 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:04:19 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 21:04:19 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 21:04:19 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 21:04:19 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 21:04:19 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 21:04:19 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 16 ms on localhost (1/4) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 15 ms on localhost (2/4) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 17 ms on localhost (3/4) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 20 ms on localhost (4/4) -2016-05-23 21:04:19 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.022 s -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 21:04:19 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.033853 s -2016-05-23 21:04:19 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:04:19 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:04:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=387808, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(1919) called with curMem=391248, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1919.0 B, free 730.2 MB) -2016-05-23 21:04:19 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:54239 (size: 1919.0 B, free: 730.5 MB) -2016-05-23 21:04:19 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:04:19 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 21:04:19 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 21:04:19 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 21:04:19 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 21:04:19 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 21:04:19 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 16 ms on localhost (1/4) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 13 ms on localhost (2/4) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 16 ms on localhost (3/4) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 19 ms on localhost (4/4) -2016-05-23 21:04:19 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.020 s -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 21:04:19 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.036901 s -2016-05-23 21:04:19 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:04:19 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:04:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=393167, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(1861) called with curMem=396543, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1861.0 B, free 730.2 MB) -2016-05-23 21:04:19 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:54239 (size: 1861.0 B, free: 730.5 MB) -2016-05-23 21:04:19 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:04:19 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 21:04:19 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 21:04:19 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 13 ms on localhost (1/4) -2016-05-23 21:04:19 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 18 ms on localhost (2/4) -2016-05-23 21:04:19 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 21:04:19 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 24 ms on localhost (3/4) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 23 ms on localhost (4/4) -2016-05-23 21:04:19 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.029 s -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 21:04:19 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.049041 s -2016-05-23 21:04:19 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:04:19 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=398404, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(2283) called with curMem=402444, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:04:19 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:54239 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:04:19 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 21:04:19 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 21:04:19 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 21:04:19 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 21:04:19 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 72 ms on localhost (1/4) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 79 ms on localhost (2/4) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 83 ms on localhost (3/4) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 88 ms on localhost (4/4) -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 21:04:19 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.089 s -2016-05-23 21:04:19 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:04:19 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:04:19 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 21:04:19 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:04:19 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=404727, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=407271, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 21:04:19 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:54239 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 21:04:19 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:19 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 21:04:19 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 21:04:19 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:19 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 21:04:19 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 25 ms on localhost (1/3) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 25 ms on localhost (2/3) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 37 ms on localhost (3/3) -2016-05-23 21:04:19 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.037 s -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 21:04:19 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.164179 s -2016-05-23 21:04:19 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:04:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 21:04:19 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:04:19 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=408839, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=413071, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 21:04:19 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:54239 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:04:19 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:04:19 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 21:04:19 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 21:04:19 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 21:04:19 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:19 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 92 ms on localhost (1/4) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 98 ms on localhost (2/4) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 21:04:19 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 142 ms on localhost (3/4) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 139 ms on localhost (4/4) -2016-05-23 21:04:19 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.143 s -2016-05-23 21:04:19 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:04:19 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:04:19 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 21:04:19 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:04:19 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=415377, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=420273, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:04:19 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:54239 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:04:19 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:19 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:19 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 16 ms on localhost (1/1) -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 21:04:19 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.017 s -2016-05-23 21:04:19 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.193081 s -2016-05-23 21:04:19 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:04:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 21:04:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 21:04:19 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:04:19 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=422815, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=428511, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:04:19 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:54239 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:04:19 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:19 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 21:04:19 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 21:04:19 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 21:04:19 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:04:19 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 21:04:19 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 19 ms on localhost (1/4) -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 21 ms on localhost (2/4) -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:19 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 37 ms on localhost (3/4) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 42 ms on localhost (4/4) -2016-05-23 21:04:19 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.042 s -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 21:04:19 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.060753 s -2016-05-23 21:04:19 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:04:19 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 21:04:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=431420, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:04:19 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=437156, maxMem=766075207 -2016-05-23 21:04:19 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:04:19 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:54239 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:04:19 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:19 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 21:04:19 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 21:04:19 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:04:19 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:19 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 21:04:19 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 34 ms on localhost (1/4) -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 33 ms on localhost (2/4) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 39 ms on localhost (3/4) -2016-05-23 21:04:19 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 21:04:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 43 ms on localhost (4/4) -2016-05-23 21:04:19 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.046 s -2016-05-23 21:04:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 21:04:19 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.068511 s -2016-05-23 21:04:20 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:61 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Got job 20 (foreach at StatisticsCorrLearning.scala:61) with 4 output partitions -2016-05-23 21:04:20 INFO DAGScheduler:59 - Final stage: ResultStage 34(foreach at StatisticsCorrLearning.scala:61) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:58), which has no missing parents -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=440048, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=441272, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:54239 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:58) -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 62, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 63, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 64, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:04:20 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 62) -2016-05-23 21:04:20 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 64) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 62). 915 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 62) in 13 ms on localhost (1/4) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 64). 915 bytes result sent to driver -2016-05-23 21:04:20 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 63) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 64) in 14 ms on localhost (2/4) -2016-05-23 21:04:20 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 65) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 63). 915 bytes result sent to driver -2016-05-23 21:04:20 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 65). 915 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 63) in 21 ms on localhost (3/4) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 65) in 20 ms on localhost (4/4) -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-23 21:04:20 INFO DAGScheduler:59 - ResultStage 34 (foreach at StatisticsCorrLearning.scala:61) finished in 0.023 s -2016-05-23 21:04:20 INFO DAGScheduler:59 - Job 20 finished: foreach at StatisticsCorrLearning.scala:61, took 0.037473 s -2016-05-23 21:04:20 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:63 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Got job 21 (foreach at StatisticsCorrLearning.scala:63) with 4 output partitions -2016-05-23 21:04:20 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at StatisticsCorrLearning.scala:63) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:59), which has no missing parents -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=442165, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=443389, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:54239 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:59) -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 4 tasks -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 66, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:04:20 INFO Executor:59 - Running task 3.0 in stage 35.0 (TID 69) -2016-05-23 21:04:20 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 66) -2016-05-23 21:04:20 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 68) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 3.0 in stage 35.0 (TID 69). 915 bytes result sent to driver -2016-05-23 21:04:20 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 68). 915 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 35.0 (TID 69) in 6 ms on localhost (1/4) -2016-05-23 21:04:20 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 67) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 66). 915 bytes result sent to driver -2016-05-23 21:04:20 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 67). 915 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 68) in 12 ms on localhost (2/4) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 67) in 14 ms on localhost (3/4) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 66) in 15 ms on localhost (4/4) -2016-05-23 21:04:20 INFO DAGScheduler:59 - ResultStage 35 (foreach at StatisticsCorrLearning.scala:63) finished in 0.016 s -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-23 21:04:20 INFO DAGScheduler:59 - Job 21 finished: foreach at StatisticsCorrLearning.scala:63, took 0.029507 s -2016-05-23 21:04:20 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Got job 22 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:04:20 INFO DAGScheduler:59 - Final stage: ResultStage 36(first at RowMatrix.scala:65) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=444282, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(1523) called with curMem=446786, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1523.0 B, free 730.2 MB) -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:54239 (size: 1523.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:04:20 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 70) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 70). 935 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 70) in 7 ms on localhost (1/1) -2016-05-23 21:04:20 INFO DAGScheduler:59 - ResultStage 36 (first at RowMatrix.scala:65) finished in 0.008 s -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-23 21:04:20 INFO DAGScheduler:59 - Job 22 finished: first at RowMatrix.scala:65, took 0.023044 s -2016-05-23 21:04:20 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Got job 23 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 21:04:20 INFO DAGScheduler:59 - Final stage: ResultStage 37(first at RowMatrix.scala:65) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=448309, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(1523) called with curMem=450813, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1523.0 B, free 730.2 MB) -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:54239 (size: 1523.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 37 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 3 tasks -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:04:20 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 73) -2016-05-23 21:04:20 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 73). 1046 bytes result sent to driver -2016-05-23 21:04:20 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 1046 bytes result sent to driver -2016-05-23 21:04:20 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 72) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 73) in 8 ms on localhost (1/3) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 11 ms on localhost (2/3) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 72). 1046 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 72) in 12 ms on localhost (3/3) -2016-05-23 21:04:20 INFO DAGScheduler:59 - ResultStage 37 (first at RowMatrix.scala:65) finished in 0.015 s -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-23 21:04:20 INFO DAGScheduler:59 - Job 23 finished: first at RowMatrix.scala:65, took 0.028787 s -2016-05-23 21:04:20 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Got job 24 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:04:20 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=452336, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(1933) called with curMem=455736, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1933.0 B, free 730.1 MB) -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:54239 (size: 1933.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 74, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 77, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:04:20 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 76) -2016-05-23 21:04:20 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 77) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 77). 1231 bytes result sent to driver -2016-05-23 21:04:20 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 76). 1231 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 77) in 13 ms on localhost (1/4) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 76) in 15 ms on localhost (2/4) -2016-05-23 21:04:20 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 74) -2016-05-23 21:04:20 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 75) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 74). 1231 bytes result sent to driver -2016-05-23 21:04:20 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 75). 1231 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 74) in 32 ms on localhost (3/4) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 75) in 32 ms on localhost (4/4) -2016-05-23 21:04:20 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at RowMatrix.scala:332) finished in 0.034 s -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-23 21:04:20 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at RowMatrix.scala:332, took 0.048404 s -2016-05-23 21:04:20 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Got job 25 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:04:20 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=457669, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(1917) called with curMem=461109, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1917.0 B, free 730.1 MB) -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:54239 (size: 1917.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 39 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 4 tasks -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 79, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 80, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 39.0 (TID 81, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:04:20 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 80) -2016-05-23 21:04:20 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 78) -2016-05-23 21:04:20 INFO Executor:59 - Running task 3.0 in stage 39.0 (TID 81) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 78). 1109 bytes result sent to driver -2016-05-23 21:04:20 INFO Executor:59 - Finished task 3.0 in stage 39.0 (TID 81). 1109 bytes result sent to driver -2016-05-23 21:04:20 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 80). 1109 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 78) in 10 ms on localhost (1/4) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 39.0 (TID 81) in 8 ms on localhost (2/4) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 80) in 10 ms on localhost (3/4) -2016-05-23 21:04:20 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 79) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 79). 1109 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 79) in 17 ms on localhost (4/4) -2016-05-23 21:04:20 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at RowMatrix.scala:124) finished in 0.018 s -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-23 21:04:20 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at RowMatrix.scala:124, took 0.032179 s -2016-05-23 21:04:20 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Got job 26 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:04:20 INFO DAGScheduler:59 - Final stage: ResultStage 40(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=463026, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(1852) called with curMem=466402, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 1852.0 B, free 730.1 MB) -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:54239 (size: 1852.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 40 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 4 tasks -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 82, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 83, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 84, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 40.0 (TID 85, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:04:20 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 84) -2016-05-23 21:04:20 INFO Executor:59 - Running task 3.0 in stage 40.0 (TID 85) -2016-05-23 21:04:20 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 82) -2016-05-23 21:04:20 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 83) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 3.0 in stage 40.0 (TID 85). 1254 bytes result sent to driver -2016-05-23 21:04:20 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 82). 1104 bytes result sent to driver -2016-05-23 21:04:20 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 84). 1249 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 40.0 (TID 85) in 7 ms on localhost (1/4) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 82) in 10 ms on localhost (2/4) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 83). 1254 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 84) in 9 ms on localhost (3/4) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 83) in 11 ms on localhost (4/4) -2016-05-23 21:04:20 INFO DAGScheduler:59 - ResultStage 40 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.012 s -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-23 21:04:20 INFO DAGScheduler:59 - Job 26 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.025358 s -2016-05-23 21:04:20 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Registering RDD 51 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Got job 27 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:04:20 INFO DAGScheduler:59 - Final stage: ResultStage 42(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 41) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 41) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting ShuffleMapStage 41 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=468254, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(2280) called with curMem=472294, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:54239 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:04:20 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 41 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 4 tasks -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 86, localhost, PROCESS_LOCAL, 2221 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 87, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 88, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 41.0 (TID 89, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:04:20 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 86) -2016-05-23 21:04:20 INFO Executor:59 - Running task 3.0 in stage 41.0 (TID 89) -2016-05-23 21:04:20 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 88) -2016-05-23 21:04:20 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 87) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 86). 1161 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 86) in 14 ms on localhost (1/4) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 87). 1161 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 87) in 47 ms on localhost (2/4) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 3.0 in stage 41.0 (TID 89). 1161 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 41.0 (TID 89) in 56 ms on localhost (3/4) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 2.0 in stage 41.0 (TID 88). 1161 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 41.0 (TID 88) in 60 ms on localhost (4/4) -2016-05-23 21:04:20 INFO DAGScheduler:59 - ShuffleMapStage 41 (flatMap at SpearmanCorrelation.scala:49) finished in 0.062 s -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-23 21:04:20 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:04:20 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:04:20 INFO DAGScheduler:59 - waiting: Set(ResultStage 42) -2016-05-23 21:04:20 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Missing parents for ResultStage 42: List() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting ResultStage 42 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=474574, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(1563) called with curMem=477118, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 1563.0 B, free 730.1 MB) -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:54239 (size: 1563.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 42 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 3 tasks -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 90, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 91, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 42.0 (TID 92, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:20 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 90) -2016-05-23 21:04:20 INFO Executor:59 - Running task 2.0 in stage 42.0 (TID 92) -2016-05-23 21:04:20 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 91) -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:20 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 90). 1203 bytes result sent to driver -2016-05-23 21:04:20 INFO Executor:59 - Finished task 2.0 in stage 42.0 (TID 92). 1203 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 90) in 12 ms on localhost (1/3) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 42.0 (TID 92) in 12 ms on localhost (2/3) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 91). 1203 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 91) in 15 ms on localhost (3/3) -2016-05-23 21:04:20 INFO DAGScheduler:59 - ResultStage 42 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.015 s -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-23 21:04:20 INFO DAGScheduler:59 - Job 27 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.102711 s -2016-05-23 21:04:20 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:04:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 168 bytes -2016-05-23 21:04:20 INFO DAGScheduler:59 - Registering RDD 56 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Got job 28 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:04:20 INFO DAGScheduler:59 - Final stage: ResultStage 45(first at RowMatrix.scala:65) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 44) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 44) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting ShuffleMapStage 44 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=478681, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(2301) called with curMem=482913, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:54239 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:04:20 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 44 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 4 tasks -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 93, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 94, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 44.0 (TID 95, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 44.0 (TID 96, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:04:20 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 93) -2016-05-23 21:04:20 INFO Executor:59 - Running task 3.0 in stage 44.0 (TID 96) -2016-05-23 21:04:20 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 94) -2016-05-23 21:04:20 INFO Executor:59 - Running task 2.0 in stage 44.0 (TID 95) -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:20 INFO Executor:59 - Finished task 2.0 in stage 44.0 (TID 95). 1377 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 44.0 (TID 95) in 54 ms on localhost (1/4) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 3.0 in stage 44.0 (TID 96). 1377 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 44.0 (TID 96) in 69 ms on localhost (2/4) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 94). 1377 bytes result sent to driver -2016-05-23 21:04:20 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 93). 1377 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 94) in 75 ms on localhost (3/4) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 93) in 76 ms on localhost (4/4) -2016-05-23 21:04:20 INFO DAGScheduler:59 - ShuffleMapStage 44 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.077 s -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-23 21:04:20 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:04:20 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:04:20 INFO DAGScheduler:59 - waiting: Set(ResultStage 45) -2016-05-23 21:04:20 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Missing parents for ResultStage 45: List() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=485214, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(2541) called with curMem=490110, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:54239 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:04:20 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 45 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 1 tasks -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 97, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:20 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 97) -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:04:20 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 97). 1185 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 97) in 8 ms on localhost (1/1) -2016-05-23 21:04:20 INFO DAGScheduler:59 - ResultStage 45 (first at RowMatrix.scala:65) finished in 0.008 s -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-23 21:04:20 INFO DAGScheduler:59 - Job 28 finished: first at RowMatrix.scala:65, took 0.105417 s -2016-05-23 21:04:20 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:04:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 168 bytes -2016-05-23 21:04:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 165 bytes -2016-05-23 21:04:20 INFO DAGScheduler:59 - Got job 29 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 21:04:20 INFO DAGScheduler:59 - Final stage: ResultStage 48(first at RowMatrix.scala:65) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 47) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which has no missing parents -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=492651, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(2541) called with curMem=497547, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:54239 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:04:20 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 48 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 3 tasks -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 98, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 99, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 48.0 (TID 100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:20 INFO Executor:59 - Running task 2.0 in stage 48.0 (TID 100) -2016-05-23 21:04:20 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 99) -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:20 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 98) -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:04:20 INFO Executor:59 - Finished task 2.0 in stage 48.0 (TID 100). 1296 bytes result sent to driver -2016-05-23 21:04:20 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 99). 1296 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 48.0 (TID 100) in 15 ms on localhost (1/3) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 99) in 18 ms on localhost (2/3) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 98). 1296 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 98) in 22 ms on localhost (3/3) -2016-05-23 21:04:20 INFO DAGScheduler:59 - ResultStage 48 (first at RowMatrix.scala:65) finished in 0.022 s -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-23 21:04:20 INFO DAGScheduler:59 - Job 29 finished: first at RowMatrix.scala:65, took 0.034746 s -2016-05-23 21:04:20 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Got job 30 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:04:20 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 50) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=500088, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(2910) called with curMem=505784, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:54239 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:04:20 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 51 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 4 tasks -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 51.0 (TID 103, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 51.0 (TID 104, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:20 INFO Executor:59 - Running task 2.0 in stage 51.0 (TID 103) -2016-05-23 21:04:20 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-23 21:04:20 INFO Executor:59 - Running task 3.0 in stage 51.0 (TID 104) -2016-05-23 21:04:20 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:54239 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:04:20 INFO Executor:59 - Finished task 2.0 in stage 51.0 (TID 103). 1481 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 51.0 (TID 103) in 13 ms on localhost (1/4) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 3.0 in stage 51.0 (TID 104). 1481 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 51.0 (TID 104) in 14 ms on localhost (2/4) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 1481 bytes result sent to driver -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-23 21:04:20 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 1481 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 20 ms on localhost (3/4) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 23 ms on localhost (4/4) -2016-05-23 21:04:20 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at RowMatrix.scala:332) finished in 0.023 s -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-23 21:04:20 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at RowMatrix.scala:332, took 0.054915 s -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:54239 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:54239 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:54239 in memory (size: 1563.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-23 21:04:20 INFO DAGScheduler:59 - Got job 31 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:04:20 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 53) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:54239 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=476860, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-23 21:04:20 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=482596, maxMem=766075207 -2016-05-23 21:04:20 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:54239 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:04:20 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:54239 in memory (size: 1852.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 54 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 4 tasks -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 105, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 106, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:04:20 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 105) -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:54239 in memory (size: 1917.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO Executor:59 - Running task 3.0 in stage 54.0 (TID 108) -2016-05-23 21:04:20 INFO Executor:59 - Running task 2.0 in stage 54.0 (TID 107) -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:04:20 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 105). 1359 bytes result sent to driver -2016-05-23 21:04:20 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 106) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 3.0 in stage 54.0 (TID 108). 1359 bytes result sent to driver -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:04:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 105) in 22 ms on localhost (1/4) -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 54.0 (TID 108) in 21 ms on localhost (2/4) -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-23 21:04:20 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 106). 1359 bytes result sent to driver -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 106) in 28 ms on localhost (3/4) -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:54239 in memory (size: 1933.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO Executor:59 - Finished task 2.0 in stage 54.0 (TID 107). 1359 bytes result sent to driver -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-23 21:04:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 54.0 (TID 107) in 31 ms on localhost (4/4) -2016-05-23 21:04:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-23 21:04:20 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at RowMatrix.scala:124) finished in 0.033 s -2016-05-23 21:04:20 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at RowMatrix.scala:124, took 0.047139 s -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:54239 in memory (size: 1523.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:54239 in memory (size: 1523.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:54239 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:54239 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:54239 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:54239 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 21:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:54239 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:54239 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:54239 in memory (size: 1568.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:54239 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:54239 in memory (size: 1861.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:54239 in memory (size: 1919.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:54239 in memory (size: 1935.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:54239 in memory (size: 1525.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:54239 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:54239 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:54239 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-23 21:04:20 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:54239 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:04:20 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-23 21:04:20 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:54239 in memory (size: 2.5 KB, free: 730.6 MB) -2016-05-23 21:04:20 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 21:04:20 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 21:04:21 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 21:04:21 INFO BlockManager:59 - BlockManager stopped -2016-05-23 21:04:21 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 21:04:21 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 21:04:21 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 21:04:21 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 21:04:21 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-3df2577b-1a98-4d2a-ac34-b56a9143f44a -2016-05-23 21:04:21 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 21:06:12 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 21:06:13 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 21:06:13 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 21:06:13 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 21:06:13 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 21:06:14 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 21:06:14 INFO Remoting:74 - Starting remoting -2016-05-23 21:06:14 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:54364] -2016-05-23 21:06:14 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54364. -2016-05-23 21:06:14 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 21:06:14 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 21:06:14 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-a7163a5e-9160-4b0c-933f-3dd1fa7a1fc0 -2016-05-23 21:06:14 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 21:06:15 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-669674e8-105a-4905-9258-921d98098ab5\httpd-8987cd1f-3f73-4ec9-a753-a5811c289c7e -2016-05-23 21:06:15 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 21:06:15 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 21:06:15 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54365 -2016-05-23 21:06:15 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54365. -2016-05-23 21:06:15 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 21:06:15 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 21:06:15 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 21:06:15 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 21:06:15 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 21:06:15 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 21:06:15 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 21:06:15 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54384. -2016-05-23 21:06:15 INFO NettyBlockTransferService:59 - Server created on 54384 -2016-05-23 21:06:15 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 21:06:15 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54384 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54384) -2016-05-23 21:06:15 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 21:06:16 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 21:06:16 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 21:06:16 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 21:06:16 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 21:06:16 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54384 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 21:06:16 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:22 -2016-05-23 21:06:16 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 21:06:16 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 21:06:16 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 21:06:16 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 21:06:16 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54384 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 21:06:16 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:25 -2016-05-23 21:06:17 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 21:06:18 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 21:06:18 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 21:06:18 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 21:06:18 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 21:06:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:06:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:18 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:23), which has no missing parents -2016-05-23 21:06:18 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 21:06:18 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 21:06:18 INFO MemoryStore:59 - ensureFreeSpace(2043) called with curMem=346712, maxMem=766075207 -2016-05-23 21:06:18 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2043.0 B, free 730.3 MB) -2016-05-23 21:06:18 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54384 (size: 2043.0 B, free: 730.6 MB) -2016-05-23 21:06:18 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:23) -2016-05-23 21:06:18 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 21:06:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:06:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:06:18 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 21:06:18 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 21:06:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:06:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:06:18 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 21:06:18 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 21:06:18 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 21:06:18 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 21:06:18 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 21:06:18 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 21:06:18 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 21:06:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 117 ms on localhost (1/2) -2016-05-23 21:06:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 97 ms on localhost (2/2) -2016-05-23 21:06:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 21:06:18 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:29) finished in 0.139 s -2016-05-23 21:06:18 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:29, took 0.305206 s -2016-05-23 21:06:18 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 21:06:18 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:31 -2016-05-23 21:06:18 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:31) with 2 output partitions -2016-05-23 21:06:18 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:31) -2016-05-23 21:06:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:06:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:18 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:26), which has no missing parents -2016-05-23 21:06:18 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348755, maxMem=766075207 -2016-05-23 21:06:18 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 21:06:18 INFO MemoryStore:59 - ensureFreeSpace(2043) called with curMem=352267, maxMem=766075207 -2016-05-23 21:06:18 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2043.0 B, free 730.2 MB) -2016-05-23 21:06:18 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54384 (size: 2043.0 B, free: 730.6 MB) -2016-05-23 21:06:18 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:26) -2016-05-23 21:06:18 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 21:06:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:06:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:06:18 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 21:06:18 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 21:06:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:06:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:06:18 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 21:06:18 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 21:06:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 22 ms on localhost (1/2) -2016-05-23 21:06:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 23 ms on localhost (2/2) -2016-05-23 21:06:18 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:31) finished in 0.025 s -2016-05-23 21:06:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 21:06:18 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:31, took 0.041979 s -2016-05-23 21:06:18 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:06:18 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:06:18 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 21:06:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:06:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:18 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:06:18 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354310, maxMem=766075207 -2016-05-23 21:06:18 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 21:06:18 INFO MemoryStore:59 - ensureFreeSpace(2465) called with curMem=358870, maxMem=766075207 -2016-05-23 21:06:18 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:06:18 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54384 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 21:06:18 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 21:06:18 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 21:06:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:06:18 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 21:06:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:06:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:06:18 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:54384 in memory (size: 2043.0 B, free: 730.6 MB) -2016-05-23 21:06:18 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 21:06:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 41 ms on localhost (1/1) -2016-05-23 21:06:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 21:06:18 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.044 s -2016-05-23 21:06:18 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.090289 s -2016-05-23 21:06:18 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:54384 in memory (size: 2043.0 B, free: 730.6 MB) -2016-05-23 21:06:19 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:06:19 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:06:19 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:06:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:06:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:19 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:06:19 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=350225, maxMem=766075207 -2016-05-23 21:06:19 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 21:06:19 INFO MemoryStore:59 - ensureFreeSpace(2864) called with curMem=355673, maxMem=766075207 -2016-05-23 21:06:19 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:06:19 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:54384 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:06:19 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:06:19 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 21:06:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:06:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:06:19 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 21:06:19 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 21:06:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:06:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:06:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:06:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:06:19 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 21:06:19 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 21:06:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 42 ms on localhost (1/2) -2016-05-23 21:06:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 45 ms on localhost (2/2) -2016-05-23 21:06:19 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.046 s -2016-05-23 21:06:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 21:06:19 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 21:06:19 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 21:06:19 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.077075 s -2016-05-23 21:06:19 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:06:19 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:06:19 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:06:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:06:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:19 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:06:19 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=358537, maxMem=766075207 -2016-05-23 21:06:19 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 21:06:19 INFO MemoryStore:59 - ensureFreeSpace(2855) called with curMem=364033, maxMem=766075207 -2016-05-23 21:06:19 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:06:19 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:54384 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:06:19 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:06:19 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 21:06:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:06:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:06:19 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 21:06:19 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 21:06:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:06:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:06:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:06:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:06:19 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 21:06:19 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 21:06:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 32 ms on localhost (1/2) -2016-05-23 21:06:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 36 ms on localhost (2/2) -2016-05-23 21:06:19 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.037 s -2016-05-23 21:06:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 21:06:19 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.063367 s -2016-05-23 21:06:19 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:06:19 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 21:06:19 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:06:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:06:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:19 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:06:19 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=366888, maxMem=766075207 -2016-05-23 21:06:19 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 21:06:19 INFO MemoryStore:59 - ensureFreeSpace(2745) called with curMem=372232, maxMem=766075207 -2016-05-23 21:06:19 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 21:06:19 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:54384 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 21:06:19 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:06:19 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 21:06:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:06:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:06:19 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 21:06:19 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 21:06:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:06:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:06:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:06:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:06:19 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 21:06:19 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 21:06:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 40 ms on localhost (1/2) -2016-05-23 21:06:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 43 ms on localhost (2/2) -2016-05-23 21:06:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 21:06:19 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.043 s -2016-05-23 21:06:19 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.061931 s -2016-05-23 21:06:19 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:06:19 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:06:19 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 21:06:19 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:06:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 21:06:19 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 21:06:19 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:06:19 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=374977, maxMem=766075207 -2016-05-23 21:06:19 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 21:06:19 INFO MemoryStore:59 - ensureFreeSpace(3157) called with curMem=380905, maxMem=766075207 -2016-05-23 21:06:19 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 21:06:19 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:54384 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 21:06:19 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:06:19 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 21:06:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 21:06:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 21:06:19 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 21:06:19 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 21:06:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:06:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:06:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:06:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:06:19 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 21:06:19 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 21:06:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 72 ms on localhost (1/2) -2016-05-23 21:06:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 70 ms on localhost (2/2) -2016-05-23 21:06:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 21:06:19 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.073 s -2016-05-23 21:06:19 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:06:19 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:06:19 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 21:06:19 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:06:19 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 21:06:19 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:06:19 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=384062, maxMem=766075207 -2016-05-23 21:06:19 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:06:19 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=386566, maxMem=766075207 -2016-05-23 21:06:19 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 21:06:19 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:54384 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 21:06:19 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:06:19 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 21:06:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:19 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 21:06:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:06:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-23 21:06:19 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 21:06:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 55 ms on localhost (1/1) -2016-05-23 21:06:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 21:06:19 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.056 s -2016-05-23 21:06:19 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.176693 s -2016-05-23 21:06:20 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:06:20 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:54384 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 21:06:20 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 21:06:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 151 bytes -2016-05-23 21:06:20 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:54384 in memory (size: 3.1 KB, free: 730.5 MB) -2016-05-23 21:06:20 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:06:20 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:06:20 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:54384 in memory (size: 2.7 KB, free: 730.6 MB) -2016-05-23 21:06:20 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=364033, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:06:20 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:54384 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:06:20 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=362729, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:06:20 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:54384 (size: 2.2 KB, free: 730.6 MB) -2016-05-23 21:06:20 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:54384 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:06:20 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 21:06:20 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:06:20 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 21:06:20 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:06:20 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 37 ms on localhost (1/2) -2016-05-23 21:06:20 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 41 ms on localhost (2/2) -2016-05-23 21:06:20 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.043 s -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 21:06:20 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:06:20 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:06:20 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 21:06:20 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=356699, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=361555, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:06:20 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:54384 (size: 2.5 KB, free: 730.6 MB) -2016-05-23 21:06:20 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:20 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:20 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 24 ms on localhost (1/1) -2016-05-23 21:06:20 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.024 s -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 21:06:20 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.104510 s -2016-05-23 21:06:20 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:06:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 151 bytes -2016-05-23 21:06:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 154 bytes -2016-05-23 21:06:20 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:06:20 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=364086, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=369742, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:06:20 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:54384 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:06:20 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:20 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 21:06:20 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:06:20 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 21:06:20 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 21 ms on localhost (1/2) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 20 ms on localhost (2/2) -2016-05-23 21:06:20 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.022 s -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 21:06:20 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.038739 s -2016-05-23 21:06:20 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:06:20 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=372650, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=378346, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:06:20 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:54384 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:06:20 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:20 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 21:06:20 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:06:20 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 21:06:20 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 21 ms on localhost (1/2) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 20 ms on localhost (2/2) -2016-05-23 21:06:20 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.022 s -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 21:06:20 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.041752 s -2016-05-23 21:06:20 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:45 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:45) with 4 output partitions -2016-05-23 21:06:20 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:45) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:42), which has no missing parents -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=381235, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=382459, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:06:20 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:54384 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:06:20 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:42) -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:06:20 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 21:06:20 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 21:06:20 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 21:06:20 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 21:06:20 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 21:06:20 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 21:06:20 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 19 ms on localhost (1/4) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 12 ms on localhost (2/4) -2016-05-23 21:06:20 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 14 ms on localhost (3/4) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 14 ms on localhost (4/4) -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 21:06:20 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:45) finished in 0.023 s -2016-05-23 21:06:20 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:45, took 0.038755 s -2016-05-23 21:06:20 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:47 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:47) with 4 output partitions -2016-05-23 21:06:20 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:47) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:43), which has no missing parents -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=383352, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=384576, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:06:20 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:54384 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:06:20 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:43) -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:06:20 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 21:06:20 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 21:06:20 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 21:06:20 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 21:06:20 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 21:06:20 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 21:06:20 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 21:06:20 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 12 ms on localhost (1/4) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 11 ms on localhost (2/4) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 13 ms on localhost (3/4) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 12 ms on localhost (4/4) -2016-05-23 21:06:20 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:47) finished in 0.015 s -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 21:06:20 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:47, took 0.025260 s -2016-05-23 21:06:20 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:06:20 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=385469, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(1525) called with curMem=387973, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1525.0 B, free 730.2 MB) -2016-05-23 21:06:20 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:54384 (size: 1525.0 B, free: 730.5 MB) -2016-05-23 21:06:20 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:06:20 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 21:06:20 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 7 ms on localhost (1/1) -2016-05-23 21:06:20 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.007 s -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 21:06:20 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.020528 s -2016-05-23 21:06:20 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:06:20 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=389498, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=392898, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.2 MB) -2016-05-23 21:06:20 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:54384 (size: 1935.0 B, free: 730.5 MB) -2016-05-23 21:06:20 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:06:20 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 21:06:20 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 21:06:20 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 21:06:20 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 21:06:20 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 21:06:20 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 21:06:20 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 12 ms on localhost (1/4) -2016-05-23 21:06:20 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 13 ms on localhost (2/4) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 16 ms on localhost (3/4) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 19 ms on localhost (4/4) -2016-05-23 21:06:20 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.019 s -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 21:06:20 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.032541 s -2016-05-23 21:06:20 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:06:20 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=394833, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(1919) called with curMem=398273, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1919.0 B, free 730.2 MB) -2016-05-23 21:06:20 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:54384 (size: 1919.0 B, free: 730.5 MB) -2016-05-23 21:06:20 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:06:20 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 21:06:20 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 21:06:20 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 21:06:20 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 21:06:20 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 21:06:20 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 21:06:20 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 12 ms on localhost (1/4) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 16 ms on localhost (2/4) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 18 ms on localhost (3/4) -2016-05-23 21:06:20 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 16 ms on localhost (4/4) -2016-05-23 21:06:20 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.021 s -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 21:06:20 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.035706 s -2016-05-23 21:06:20 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:06:20 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=400192, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(1861) called with curMem=403568, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1861.0 B, free 730.2 MB) -2016-05-23 21:06:20 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:54384 (size: 1861.0 B, free: 730.5 MB) -2016-05-23 21:06:20 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:06:20 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 21:06:20 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 21:06:20 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 21:06:20 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 15 ms on localhost (1/4) -2016-05-23 21:06:20 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 21:06:20 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 21:06:20 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 21:06:20 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 22 ms on localhost (2/4) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 22 ms on localhost (3/4) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 25 ms on localhost (4/4) -2016-05-23 21:06:20 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.027 s -2016-05-23 21:06:20 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.044910 s -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 21:06:20 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:06:20 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=405429, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(2283) called with curMem=409469, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:06:20 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:54384 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:06:20 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 21:06:20 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 21:06:20 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 21:06:20 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 21:06:20 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 21:06:20 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 21:06:20 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 68 ms on localhost (1/4) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 68 ms on localhost (2/4) -2016-05-23 21:06:20 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 82 ms on localhost (3/4) -2016-05-23 21:06:20 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 92 ms on localhost (4/4) -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 21:06:20 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.093 s -2016-05-23 21:06:20 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:06:20 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:06:20 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 21:06:20 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=411752, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=414296, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 21:06:20 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:54384 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 21:06:20 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:20 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 21:06:20 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 21:06:20 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:20 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 19 ms on localhost (1/3) -2016-05-23 21:06:20 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 24 ms on localhost (2/3) -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:06:20 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 38 ms on localhost (3/3) -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 21:06:20 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.038 s -2016-05-23 21:06:20 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.167505 s -2016-05-23 21:06:20 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:06:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 167 bytes -2016-05-23 21:06:20 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:06:20 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=415864, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=420096, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 21:06:20 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:54384 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:06:20 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:06:20 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 21:06:20 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 21:06:20 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 21:06:20 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:06:20 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 21:06:20 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 74 ms on localhost (1/4) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 75 ms on localhost (2/4) -2016-05-23 21:06:20 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 21:06:20 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 129 ms on localhost (3/4) -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 130 ms on localhost (4/4) -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 21:06:20 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.131 s -2016-05-23 21:06:20 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:06:20 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:06:20 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 21:06:20 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=422402, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=427298, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:06:20 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:54384 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:06:20 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 21:06:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:20 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:20 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 21:06:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 20 ms on localhost (1/1) -2016-05-23 21:06:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 21:06:20 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.020 s -2016-05-23 21:06:20 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.185229 s -2016-05-23 21:06:20 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:06:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 167 bytes -2016-05-23 21:06:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 165 bytes -2016-05-23 21:06:20 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:06:20 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 21:06:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:20 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:06:20 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=429840, maxMem=766075207 -2016-05-23 21:06:20 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=435536, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:54384 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:06:21 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 21:06:21 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 21:06:21 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:21 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:21 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 32 ms on localhost (1/4) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 33 ms on localhost (2/4) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 37 ms on localhost (3/4) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 45 ms on localhost (4/4) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 21:06:21 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.045 s -2016-05-23 21:06:21 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.066502 s -2016-05-23 21:06:21 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:06:21 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=438445, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=444181, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:54384 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:06:21 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:21 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:06:21 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 39 ms on localhost (1/4) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 38 ms on localhost (2/4) -2016-05-23 21:06:21 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 21:06:21 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:21 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 63 ms on localhost (3/4) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 64 ms on localhost (4/4) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 21:06:21 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.068 s -2016-05-23 21:06:21 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.085741 s -2016-05-23 21:06:21 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:61 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Got job 20 (foreach at StatisticsCorrLearning.scala:61) with 4 output partitions -2016-05-23 21:06:21 INFO DAGScheduler:59 - Final stage: ResultStage 34(foreach at StatisticsCorrLearning.scala:61) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:58), which has no missing parents -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=447073, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=448297, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:54384 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:58) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 62, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 63, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 64, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:06:21 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 65) -2016-05-23 21:06:21 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 64) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 64). 915 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 65). 915 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 64) in 10 ms on localhost (1/4) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 65) in 10 ms on localhost (2/4) -2016-05-23 21:06:21 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 62) -2016-05-23 21:06:21 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 63) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 62). 915 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 63). 915 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 62) in 20 ms on localhost (3/4) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 63) in 19 ms on localhost (4/4) -2016-05-23 21:06:21 INFO DAGScheduler:59 - ResultStage 34 (foreach at StatisticsCorrLearning.scala:61) finished in 0.022 s -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-23 21:06:21 INFO DAGScheduler:59 - Job 20 finished: foreach at StatisticsCorrLearning.scala:61, took 0.034475 s -2016-05-23 21:06:21 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:63 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Got job 21 (foreach at StatisticsCorrLearning.scala:63) with 4 output partitions -2016-05-23 21:06:21 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at StatisticsCorrLearning.scala:63) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:59), which has no missing parents -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=449190, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=450414, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:54384 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:59) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 4 tasks -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 66, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:06:21 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 68) -2016-05-23 21:06:21 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 67) -2016-05-23 21:06:21 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 66) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 68). 915 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Running task 3.0 in stage 35.0 (TID 69) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 66). 915 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Finished task 3.0 in stage 35.0 (TID 69). 915 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 67). 915 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 68) in 11 ms on localhost (1/4) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 35.0 (TID 69) in 11 ms on localhost (2/4) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 66) in 15 ms on localhost (3/4) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 67) in 14 ms on localhost (4/4) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-23 21:06:21 INFO DAGScheduler:59 - ResultStage 35 (foreach at StatisticsCorrLearning.scala:63) finished in 0.016 s -2016-05-23 21:06:21 INFO DAGScheduler:59 - Job 21 finished: foreach at StatisticsCorrLearning.scala:63, took 0.029791 s -2016-05-23 21:06:21 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Got job 22 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:06:21 INFO DAGScheduler:59 - Final stage: ResultStage 36(first at RowMatrix.scala:65) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=451307, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(1523) called with curMem=453811, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1523.0 B, free 730.2 MB) -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:54384 (size: 1523.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:06:21 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 70) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 70). 935 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 70) in 6 ms on localhost (1/1) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-23 21:06:21 INFO DAGScheduler:59 - ResultStage 36 (first at RowMatrix.scala:65) finished in 0.006 s -2016-05-23 21:06:21 INFO DAGScheduler:59 - Job 22 finished: first at RowMatrix.scala:65, took 0.021314 s -2016-05-23 21:06:21 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Got job 23 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 21:06:21 INFO DAGScheduler:59 - Final stage: ResultStage 37(first at RowMatrix.scala:65) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=455334, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(1523) called with curMem=457838, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1523.0 B, free 730.1 MB) -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:54384 (size: 1523.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 37 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 3 tasks -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:06:21 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 72) -2016-05-23 21:06:21 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-23 21:06:21 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 73) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 1046 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 9 ms on localhost (1/3) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 73). 1046 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 72). 1046 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 73) in 9 ms on localhost (2/3) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 72) in 13 ms on localhost (3/3) -2016-05-23 21:06:21 INFO DAGScheduler:59 - ResultStage 37 (first at RowMatrix.scala:65) finished in 0.014 s -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-23 21:06:21 INFO DAGScheduler:59 - Job 23 finished: first at RowMatrix.scala:65, took 0.028740 s -2016-05-23 21:06:21 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Got job 24 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:06:21 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=459361, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(1933) called with curMem=462761, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1933.0 B, free 730.1 MB) -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:54384 (size: 1933.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 74, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 77, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:06:21 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 74) -2016-05-23 21:06:21 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 77) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 74). 1231 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 77). 1231 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 74) in 16 ms on localhost (1/4) -2016-05-23 21:06:21 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 75) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 77) in 14 ms on localhost (2/4) -2016-05-23 21:06:21 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 76) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 75). 1231 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 76). 1231 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 75) in 22 ms on localhost (3/4) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 76) in 22 ms on localhost (4/4) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-23 21:06:21 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at RowMatrix.scala:332) finished in 0.025 s -2016-05-23 21:06:21 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at RowMatrix.scala:332, took 0.039685 s -2016-05-23 21:06:21 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Got job 25 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:06:21 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=464694, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(1917) called with curMem=468134, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1917.0 B, free 730.1 MB) -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:54384 (size: 1917.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 39 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 4 tasks -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 79, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 80, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 39.0 (TID 81, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:06:21 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 80) -2016-05-23 21:06:21 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 78) -2016-05-23 21:06:21 INFO Executor:59 - Running task 3.0 in stage 39.0 (TID 81) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 80). 1109 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Finished task 3.0 in stage 39.0 (TID 81). 1109 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 80) in 6 ms on localhost (1/4) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 39.0 (TID 81) in 7 ms on localhost (2/4) -2016-05-23 21:06:21 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 79) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 78). 1109 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 79). 1109 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 78) in 15 ms on localhost (3/4) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 79) in 14 ms on localhost (4/4) -2016-05-23 21:06:21 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at RowMatrix.scala:124) finished in 0.017 s -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-23 21:06:21 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at RowMatrix.scala:124, took 0.032235 s -2016-05-23 21:06:21 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Got job 26 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:06:21 INFO DAGScheduler:59 - Final stage: ResultStage 40(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=470051, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(1852) called with curMem=473427, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 1852.0 B, free 730.1 MB) -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:54384 (size: 1852.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 40 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 4 tasks -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 82, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 83, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 84, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 40.0 (TID 85, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:06:21 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 82) -2016-05-23 21:06:21 INFO Executor:59 - Running task 3.0 in stage 40.0 (TID 85) -2016-05-23 21:06:21 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 84) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 82). 1104 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 83) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 84). 1249 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Finished task 3.0 in stage 40.0 (TID 85). 1254 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 83). 1254 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 82) in 15 ms on localhost (1/4) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 84) in 15 ms on localhost (2/4) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 83) in 17 ms on localhost (3/4) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 40.0 (TID 85) in 16 ms on localhost (4/4) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-23 21:06:21 INFO DAGScheduler:59 - ResultStage 40 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.019 s -2016-05-23 21:06:21 INFO DAGScheduler:59 - Job 26 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.031664 s -2016-05-23 21:06:21 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Registering RDD 51 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Got job 27 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:06:21 INFO DAGScheduler:59 - Final stage: ResultStage 42(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 41) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 41) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting ShuffleMapStage 41 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=475279, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(2280) called with curMem=479319, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:54384 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:06:21 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 41 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 4 tasks -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 86, localhost, PROCESS_LOCAL, 2221 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 87, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 88, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 41.0 (TID 89, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:06:21 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 87) -2016-05-23 21:06:21 INFO Executor:59 - Running task 3.0 in stage 41.0 (TID 89) -2016-05-23 21:06:21 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 86) -2016-05-23 21:06:21 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 88) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 86). 1161 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 86) in 16 ms on localhost (1/4) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 3.0 in stage 41.0 (TID 89). 1161 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 87). 1161 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 41.0 (TID 89) in 47 ms on localhost (2/4) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 2.0 in stage 41.0 (TID 88). 1161 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 87) in 49 ms on localhost (3/4) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 41.0 (TID 88) in 49 ms on localhost (4/4) -2016-05-23 21:06:21 INFO DAGScheduler:59 - ShuffleMapStage 41 (flatMap at SpearmanCorrelation.scala:49) finished in 0.051 s -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-23 21:06:21 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:06:21 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:06:21 INFO DAGScheduler:59 - waiting: Set(ResultStage 42) -2016-05-23 21:06:21 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Missing parents for ResultStage 42: List() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting ResultStage 42 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=481599, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(1563) called with curMem=484143, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 1563.0 B, free 730.1 MB) -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:54384 (size: 1563.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 42 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 3 tasks -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 90, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 91, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 42.0 (TID 92, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 90) -2016-05-23 21:06:21 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 91) -2016-05-23 21:06:21 INFO Executor:59 - Running task 2.0 in stage 42.0 (TID 92) -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:06:21 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 91). 1203 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Finished task 2.0 in stage 42.0 (TID 92). 1203 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 90). 1203 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 91) in 11 ms on localhost (1/3) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 42.0 (TID 92) in 12 ms on localhost (2/3) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 90) in 15 ms on localhost (3/3) -2016-05-23 21:06:21 INFO DAGScheduler:59 - ResultStage 42 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.016 s -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-23 21:06:21 INFO DAGScheduler:59 - Job 27 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.090964 s -2016-05-23 21:06:21 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:06:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 167 bytes -2016-05-23 21:06:21 INFO DAGScheduler:59 - Registering RDD 56 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Got job 28 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:06:21 INFO DAGScheduler:59 - Final stage: ResultStage 45(first at RowMatrix.scala:65) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 44) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 44) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting ShuffleMapStage 44 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=485706, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(2301) called with curMem=489938, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:54384 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:06:21 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 44 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 4 tasks -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 93, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 94, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 44.0 (TID 95, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 44.0 (TID 96, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:06:21 INFO Executor:59 - Running task 3.0 in stage 44.0 (TID 96) -2016-05-23 21:06:21 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 94) -2016-05-23 21:06:21 INFO Executor:59 - Running task 2.0 in stage 44.0 (TID 95) -2016-05-23 21:06:21 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 93) -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:21 INFO Executor:59 - Finished task 2.0 in stage 44.0 (TID 95). 1377 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 44.0 (TID 95) in 47 ms on localhost (1/4) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 94). 1377 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Finished task 3.0 in stage 44.0 (TID 96). 1377 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 94) in 60 ms on localhost (2/4) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 93). 1377 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 93) in 66 ms on localhost (3/4) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 44.0 (TID 96) in 66 ms on localhost (4/4) -2016-05-23 21:06:21 INFO DAGScheduler:59 - ShuffleMapStage 44 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.068 s -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-23 21:06:21 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:06:21 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:06:21 INFO DAGScheduler:59 - waiting: Set(ResultStage 45) -2016-05-23 21:06:21 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Missing parents for ResultStage 45: List() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=492239, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(2541) called with curMem=497135, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:54384 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:06:21 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 45 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 1 tasks -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 97, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 97) -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:21 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 97). 1185 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 97) in 9 ms on localhost (1/1) -2016-05-23 21:06:21 INFO DAGScheduler:59 - ResultStage 45 (first at RowMatrix.scala:65) finished in 0.009 s -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-23 21:06:21 INFO DAGScheduler:59 - Job 28 finished: first at RowMatrix.scala:65, took 0.100127 s -2016-05-23 21:06:21 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:06:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 167 bytes -2016-05-23 21:06:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 164 bytes -2016-05-23 21:06:21 INFO DAGScheduler:59 - Got job 29 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 21:06:21 INFO DAGScheduler:59 - Final stage: ResultStage 48(first at RowMatrix.scala:65) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 47) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which has no missing parents -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=499676, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(2541) called with curMem=504572, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:54384 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:06:21 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 48 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 3 tasks -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 98, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 99, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 48.0 (TID 100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 98) -2016-05-23 21:06:21 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 99) -2016-05-23 21:06:21 INFO Executor:59 - Running task 2.0 in stage 48.0 (TID 100) -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:06:21 INFO Executor:59 - Finished task 2.0 in stage 48.0 (TID 100). 1296 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 98). 1296 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 98) in 13 ms on localhost (1/3) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 48.0 (TID 100) in 13 ms on localhost (2/3) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 99). 1296 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 99) in 17 ms on localhost (3/3) -2016-05-23 21:06:21 INFO DAGScheduler:59 - ResultStage 48 (first at RowMatrix.scala:65) finished in 0.018 s -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-23 21:06:21 INFO DAGScheduler:59 - Job 29 finished: first at RowMatrix.scala:65, took 0.032555 s -2016-05-23 21:06:21 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Got job 30 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:06:21 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 50) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=507113, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(2910) called with curMem=512809, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:54384 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:06:21 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 51 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 4 tasks -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:54384 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 51.0 (TID 103, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 51.0 (TID 104, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-23 21:06:21 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-23 21:06:21 INFO Executor:59 - Running task 3.0 in stage 51.0 (TID 104) -2016-05-23 21:06:21 INFO Executor:59 - Running task 2.0 in stage 51.0 (TID 103) -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:54384 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:54384 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 1481 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Finished task 3.0 in stage 51.0 (TID 104). 1481 bytes result sent to driver -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-23 21:06:21 INFO Executor:59 - Finished task 2.0 in stage 51.0 (TID 103). 1481 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 16 ms on localhost (1/4) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 51.0 (TID 103) in 16 ms on localhost (2/4) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 51.0 (TID 104) in 17 ms on localhost (3/4) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 1481 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 20 ms on localhost (4/4) -2016-05-23 21:06:21 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at RowMatrix.scala:332) finished in 0.020 s -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:54384 in memory (size: 1563.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-23 21:06:21 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at RowMatrix.scala:332, took 0.051727 s -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:54384 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:54384 in memory (size: 1852.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Got job 31 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:06:21 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 53) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:54384 in memory (size: 1917.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=473300, maxMem=766075207 -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 21:06:21 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=477103, maxMem=766075207 -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:54384 in memory (size: 1933.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:54384 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:06:21 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 54 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 4 tasks -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 105, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:54384 in memory (size: 1523.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 106, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-23 21:06:21 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 105) -2016-05-23 21:06:21 INFO Executor:59 - Running task 3.0 in stage 54.0 (TID 108) -2016-05-23 21:06:21 INFO Executor:59 - Running task 2.0 in stage 54.0 (TID 107) -2016-05-23 21:06:21 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 106) -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:06:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:54384 in memory (size: 1523.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 3.0 in stage 54.0 (TID 108). 1359 bytes result sent to driver -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 54.0 (TID 108) in 17 ms on localhost (1/4) -2016-05-23 21:06:21 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 106). 1359 bytes result sent to driver -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-23 21:06:21 INFO Executor:59 - Finished task 2.0 in stage 54.0 (TID 107). 1359 bytes result sent to driver -2016-05-23 21:06:21 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 105). 1359 bytes result sent to driver -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:54384 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 54.0 (TID 107) in 26 ms on localhost (2/4) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 106) in 28 ms on localhost (3/4) -2016-05-23 21:06:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 105) in 31 ms on localhost (4/4) -2016-05-23 21:06:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-23 21:06:21 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at RowMatrix.scala:124) finished in 0.032 s -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:54384 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at RowMatrix.scala:124, took 0.047503 s -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:54384 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:54384 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:54384 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:54384 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 21:06:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:54384 in memory (size: 1568.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:54384 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:54384 in memory (size: 1861.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:54384 in memory (size: 1919.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:54384 in memory (size: 1935.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:54384 in memory (size: 1525.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:54384 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:54384 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:54384 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:54384 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:54384 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-23 21:06:21 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:54384 in memory (size: 2.2 KB, free: 730.6 MB) -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-23 21:06:21 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-23 21:06:21 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 21:06:21 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 21:06:22 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 21:06:22 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 21:06:22 INFO BlockManager:59 - BlockManager stopped -2016-05-23 21:06:22 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 21:06:22 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 21:06:22 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 21:06:22 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 21:06:22 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-669674e8-105a-4905-9258-921d98098ab5 -2016-05-23 21:06:22 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 21:08:37 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 21:08:38 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 21:08:38 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 21:08:38 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 21:08:38 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 21:08:39 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 21:08:39 INFO Remoting:74 - Starting remoting -2016-05-23 21:08:39 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:54421] -2016-05-23 21:08:39 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54421. -2016-05-23 21:08:39 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 21:08:39 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 21:08:39 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-f5e69d1c-7924-478c-bc83-2614214c8162 -2016-05-23 21:08:39 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 21:08:40 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-1b3e5db0-7080-424f-946b-82cbc1675c42\httpd-58860393-0c58-4cd7-baf5-9f91e1651355 -2016-05-23 21:08:40 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 21:08:40 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 21:08:40 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54422 -2016-05-23 21:08:40 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54422. -2016-05-23 21:08:40 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 21:08:40 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 21:08:40 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 21:08:40 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 21:08:40 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 21:08:40 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 21:08:40 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 21:08:40 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54441. -2016-05-23 21:08:40 INFO NettyBlockTransferService:59 - Server created on 54441 -2016-05-23 21:08:40 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 21:08:40 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54441 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54441) -2016-05-23 21:08:40 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 21:08:41 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 21:08:41 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 21:08:41 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 21:08:41 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 21:08:41 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54441 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 21:08:41 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:22 -2016-05-23 21:08:41 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 21:08:41 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 21:08:41 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 21:08:41 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 21:08:41 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54441 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 21:08:41 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:25 -2016-05-23 21:08:43 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 21:08:43 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 21:08:43 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 21:08:43 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 21:08:43 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 21:08:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:43 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:23), which has no missing parents -2016-05-23 21:08:43 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 21:08:43 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 21:08:43 INFO MemoryStore:59 - ensureFreeSpace(2047) called with curMem=346712, maxMem=766075207 -2016-05-23 21:08:43 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2047.0 B, free 730.3 MB) -2016-05-23 21:08:43 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54441 (size: 2047.0 B, free: 730.6 MB) -2016-05-23 21:08:43 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:23) -2016-05-23 21:08:43 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 21:08:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:08:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:08:43 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 21:08:43 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 21:08:43 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:08:43 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:08:43 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 21:08:43 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 21:08:43 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 21:08:43 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 21:08:43 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 21:08:43 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 21:08:43 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 21:08:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 95 ms on localhost (1/2) -2016-05-23 21:08:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 120 ms on localhost (2/2) -2016-05-23 21:08:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 21:08:43 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:29) finished in 0.140 s -2016-05-23 21:08:43 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:29, took 0.257398 s -2016-05-23 21:08:43 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 21:08:43 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:31 -2016-05-23 21:08:43 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:31) with 2 output partitions -2016-05-23 21:08:43 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:31) -2016-05-23 21:08:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:43 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:26), which has no missing parents -2016-05-23 21:08:43 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348759, maxMem=766075207 -2016-05-23 21:08:43 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 21:08:43 INFO MemoryStore:59 - ensureFreeSpace(2045) called with curMem=352271, maxMem=766075207 -2016-05-23 21:08:43 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2045.0 B, free 730.2 MB) -2016-05-23 21:08:43 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54441 (size: 2045.0 B, free: 730.6 MB) -2016-05-23 21:08:43 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:26) -2016-05-23 21:08:43 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 21:08:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:08:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:08:43 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 21:08:43 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 21:08:43 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:08:43 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:08:43 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 21:08:43 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 21:08:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 22 ms on localhost (1/2) -2016-05-23 21:08:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 21 ms on localhost (2/2) -2016-05-23 21:08:43 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:31) finished in 0.023 s -2016-05-23 21:08:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 21:08:43 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:31, took 0.040311 s -2016-05-23 21:08:43 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:08:43 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:08:43 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 21:08:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:43 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:08:43 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354316, maxMem=766075207 -2016-05-23 21:08:43 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 21:08:43 INFO MemoryStore:59 - ensureFreeSpace(2464) called with curMem=358876, maxMem=766075207 -2016-05-23 21:08:43 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:08:43 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54441 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 21:08:43 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:43 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 21:08:43 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 21:08:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:08:43 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 21:08:43 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:08:43 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:08:43 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 21:08:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 74 ms on localhost (1/1) -2016-05-23 21:08:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 21:08:43 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.075 s -2016-05-23 21:08:43 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.099860 s -2016-05-23 21:08:43 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:54441 in memory (size: 2047.0 B, free: 730.6 MB) -2016-05-23 21:08:43 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:54441 in memory (size: 2045.0 B, free: 730.6 MB) -2016-05-23 21:08:43 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-23 21:08:43 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-23 21:08:44 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:08:44 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:08:44 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:08:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:44 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:08:44 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=350224, maxMem=766075207 -2016-05-23 21:08:44 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 21:08:44 INFO MemoryStore:59 - ensureFreeSpace(2861) called with curMem=355672, maxMem=766075207 -2016-05-23 21:08:44 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:08:44 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:54441 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:08:44 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:08:44 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 21:08:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:08:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:08:44 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 21:08:44 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 21:08:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:08:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:08:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:08:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:08:44 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 21:08:44 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 21:08:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 44 ms on localhost (1/2) -2016-05-23 21:08:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 43 ms on localhost (2/2) -2016-05-23 21:08:44 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.046 s -2016-05-23 21:08:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 21:08:44 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 21:08:44 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 21:08:44 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.077277 s -2016-05-23 21:08:44 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:08:44 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:08:44 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:08:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:44 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:08:44 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=358533, maxMem=766075207 -2016-05-23 21:08:44 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 21:08:44 INFO MemoryStore:59 - ensureFreeSpace(2850) called with curMem=364029, maxMem=766075207 -2016-05-23 21:08:44 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:08:44 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:54441 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:08:44 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:08:44 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 21:08:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:08:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:08:44 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 21:08:44 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 21:08:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:08:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:08:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:08:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:08:44 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 21:08:44 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 21:08:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 32 ms on localhost (1/2) -2016-05-23 21:08:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 31 ms on localhost (2/2) -2016-05-23 21:08:44 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.034 s -2016-05-23 21:08:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 21:08:44 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.061793 s -2016-05-23 21:08:44 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:08:44 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 21:08:44 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:08:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:44 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:08:44 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=366879, maxMem=766075207 -2016-05-23 21:08:44 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 21:08:44 INFO MemoryStore:59 - ensureFreeSpace(2743) called with curMem=372223, maxMem=766075207 -2016-05-23 21:08:44 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 21:08:44 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:54441 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 21:08:44 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:08:44 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 21:08:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:08:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:08:44 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 21:08:44 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 21:08:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:08:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:08:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:08:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:08:44 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 21:08:44 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 21:08:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 37 ms on localhost (1/2) -2016-05-23 21:08:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 39 ms on localhost (2/2) -2016-05-23 21:08:44 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.039 s -2016-05-23 21:08:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 21:08:44 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.059389 s -2016-05-23 21:08:44 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:08:44 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:08:44 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 21:08:44 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:08:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 21:08:44 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 21:08:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:08:44 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=374966, maxMem=766075207 -2016-05-23 21:08:44 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 21:08:44 INFO MemoryStore:59 - ensureFreeSpace(3155) called with curMem=380894, maxMem=766075207 -2016-05-23 21:08:44 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 21:08:44 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:54441 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 21:08:44 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:08:44 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 21:08:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 21:08:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 21:08:44 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 21:08:44 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 21:08:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:08:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:08:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:08:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:08:44 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 21:08:44 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 21:08:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 68 ms on localhost (1/2) -2016-05-23 21:08:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 72 ms on localhost (2/2) -2016-05-23 21:08:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 21:08:44 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.074 s -2016-05-23 21:08:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:08:44 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:08:44 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 21:08:44 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:08:44 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=384049, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=386553, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 21:08:45 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:54441 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 21:08:45 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:45 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-23 21:08:45 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 59 ms on localhost (1/1) -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 21:08:45 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.060 s -2016-05-23 21:08:45 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.185432 s -2016-05-23 21:08:45 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:08:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 21:08:45 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:08:45 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=388098, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=392290, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:08:45 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:54441 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:08:45 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:45 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:54441 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 21:08:45 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:08:45 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:54441 in memory (size: 3.1 KB, free: 730.5 MB) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:08:45 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 21:08:45 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 21:08:45 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 21:08:45 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:54441 in memory (size: 2.7 KB, free: 730.5 MB) -2016-05-23 21:08:45 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:45 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:54441 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:08:45 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 21:08:45 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:54441 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:08:45 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 21:08:45 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:54441 in memory (size: 2.4 KB, free: 730.6 MB) -2016-05-23 21:08:45 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-23 21:08:45 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 43 ms on localhost (1/2) -2016-05-23 21:08:45 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 47 ms on localhost (2/2) -2016-05-23 21:08:45 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.051 s -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 21:08:45 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:08:45 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:08:45 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 21:08:45 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:08:45 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=349674, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=354530, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:08:45 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:54441 (size: 2.5 KB, free: 730.6 MB) -2016-05-23 21:08:45 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:45 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:45 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 23 ms on localhost (1/1) -2016-05-23 21:08:45 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.024 s -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 21:08:45 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.132897 s -2016-05-23 21:08:45 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:08:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 21:08:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 21:08:45 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:08:45 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=357061, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=362717, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:08:45 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:54441 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:08:45 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:45 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 21:08:45 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:45 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 23 ms on localhost (1/2) -2016-05-23 21:08:45 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 24 ms on localhost (2/2) -2016-05-23 21:08:45 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.025 s -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 21:08:45 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.041095 s -2016-05-23 21:08:45 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:08:45 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=365625, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=371321, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:08:45 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:54441 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:08:45 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:45 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 21:08:45 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:45 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 18 ms on localhost (1/2) -2016-05-23 21:08:45 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 28 ms on localhost (2/2) -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 21:08:45 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.030 s -2016-05-23 21:08:45 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.049507 s -2016-05-23 21:08:45 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:45 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:45) with 4 output partitions -2016-05-23 21:08:45 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:45) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:42), which has no missing parents -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=374210, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=375434, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:08:45 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:54441 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:08:45 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:42) -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:08:45 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 21:08:45 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 21:08:45 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 21:08:45 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 21:08:45 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 21:08:45 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 21:08:45 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 22 ms on localhost (1/4) -2016-05-23 21:08:45 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 13 ms on localhost (2/4) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 12 ms on localhost (3/4) -2016-05-23 21:08:45 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:45) finished in 0.024 s -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 12 ms on localhost (4/4) -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 21:08:45 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:45, took 0.039367 s -2016-05-23 21:08:45 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:47 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:47) with 4 output partitions -2016-05-23 21:08:45 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:47) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:43), which has no missing parents -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=376327, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=377551, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:08:45 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:54441 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:08:45 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:43) -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:08:45 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 21:08:45 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 21:08:45 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 21:08:45 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 21:08:45 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 21:08:45 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 7 ms on localhost (1/4) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 13 ms on localhost (2/4) -2016-05-23 21:08:45 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 21:08:45 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 14 ms on localhost (3/4) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 15 ms on localhost (4/4) -2016-05-23 21:08:45 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:47) finished in 0.016 s -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 21:08:45 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:47, took 0.028571 s -2016-05-23 21:08:45 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:08:45 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=378444, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=380948, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 21:08:45 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:54441 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:08:45 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:08:45 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 21:08:45 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 9 ms on localhost (1/1) -2016-05-23 21:08:45 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.010 s -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 21:08:45 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.022990 s -2016-05-23 21:08:45 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:08:45 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=382472, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=385872, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 21:08:45 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:54441 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 21:08:45 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:08:45 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 21:08:45 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 21:08:45 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 21:08:45 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 21:08:45 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 21:08:45 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 21:08:45 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 11 ms on localhost (1/4) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 11 ms on localhost (2/4) -2016-05-23 21:08:45 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 17 ms on localhost (3/4) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 17 ms on localhost (4/4) -2016-05-23 21:08:45 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.019 s -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 21:08:45 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.032252 s -2016-05-23 21:08:45 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:08:45 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=387806, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=391246, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 21:08:45 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:54441 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:08:45 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:08:45 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 21:08:45 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 21:08:45 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 21:08:45 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 21:08:45 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 21:08:45 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 15 ms on localhost (1/4) -2016-05-23 21:08:45 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 21:08:45 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 19 ms on localhost (2/4) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 19 ms on localhost (3/4) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 25 ms on localhost (4/4) -2016-05-23 21:08:45 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.025 s -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 21:08:45 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.039989 s -2016-05-23 21:08:45 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:08:45 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=393164, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(1860) called with curMem=396540, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1860.0 B, free 730.2 MB) -2016-05-23 21:08:45 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:54441 (size: 1860.0 B, free: 730.5 MB) -2016-05-23 21:08:45 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:08:45 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 21:08:45 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 21:08:45 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 21:08:45 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 21:08:45 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 21:08:45 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 15 ms on localhost (1/4) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 17 ms on localhost (2/4) -2016-05-23 21:08:45 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 21:08:45 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 28 ms on localhost (3/4) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 26 ms on localhost (4/4) -2016-05-23 21:08:45 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.031 s -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 21:08:45 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.047080 s -2016-05-23 21:08:45 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:08:45 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=398400, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=402440, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:08:45 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:54441 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:08:45 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 21:08:45 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 21:08:45 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 21:08:45 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 21:08:45 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 21:08:45 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 72 ms on localhost (1/4) -2016-05-23 21:08:45 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 21:08:45 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 82 ms on localhost (2/4) -2016-05-23 21:08:45 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 88 ms on localhost (3/4) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 91 ms on localhost (4/4) -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 21:08:45 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.091 s -2016-05-23 21:08:45 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:08:45 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:08:45 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 21:08:45 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:08:45 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=404722, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=407266, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 21:08:45 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:54441 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 21:08:45 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:45 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 21:08:45 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 21:08:45 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:45 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 18 ms on localhost (1/3) -2016-05-23 21:08:45 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 26 ms on localhost (2/3) -2016-05-23 21:08:45 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 21:08:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 31 ms on localhost (3/3) -2016-05-23 21:08:45 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.031 s -2016-05-23 21:08:45 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.160603 s -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 21:08:45 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:08:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 21:08:45 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:08:45 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=408834, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:08:45 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=413066, maxMem=766075207 -2016-05-23 21:08:45 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 21:08:45 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:54441 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:08:45 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:45 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:08:45 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:08:45 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:08:45 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 21:08:45 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 21:08:45 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 21:08:45 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-23 21:08:46 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 79 ms on localhost (1/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 82 ms on localhost (2/4) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 115 ms on localhost (3/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 119 ms on localhost (4/4) -2016-05-23 21:08:46 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.119 s -2016-05-23 21:08:46 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:08:46 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:08:46 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 21:08:46 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 21:08:46 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=415372, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=420268, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:54441 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:08:46 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:46 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 18 ms on localhost (1/1) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 21:08:46 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.019 s -2016-05-23 21:08:46 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.178675 s -2016-05-23 21:08:46 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:08:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 21:08:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 21:08:46 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:08:46 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=422810, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=428506, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:54441 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:08:46 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:54441 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:54441 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:08:46 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 21:08:46 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:46 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:46 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:46 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 33 ms on localhost (1/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 38 ms on localhost (2/4) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 54 ms on localhost (3/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 52 ms on localhost (4/4) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 21:08:46 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.056 s -2016-05-23 21:08:46 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.108915 s -2016-05-23 21:08:46 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-23 21:08:46 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:08:46 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:54441 in memory (size: 1568.0 B, free: 730.5 MB) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=413327, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=419063, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:54441 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:08:46 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-23 21:08:46 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:54441 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 21:08:46 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 21:08:46 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:46 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 32 ms on localhost (1/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 32 ms on localhost (2/4) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 41 ms on localhost (3/4) -2016-05-23 21:08:46 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 44 ms on localhost (4/4) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 21:08:46 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.047 s -2016-05-23 21:08:46 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.070943 s -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:54441 in memory (size: 1860.0 B, free: 730.5 MB) -2016-05-23 21:08:46 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:54441 in memory (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:08:46 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-23 21:08:46 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:59 -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:54441 in memory (size: 1934.0 B, free: 730.5 MB) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Got job 20 (foreach at StatisticsCorrLearning.scala:59) with 4 output partitions -2016-05-23 21:08:46 INFO DAGScheduler:59 - Final stage: ResultStage 34(foreach at StatisticsCorrLearning.scala:59) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:56), which has no missing parents -2016-05-23 21:08:46 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=399705, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:54441 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=396901, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:08:46 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:54441 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:08:46 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:56) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 62, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:54441 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 63, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 64, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:08:46 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:08:46 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 65) -2016-05-23 21:08:46 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 62) -2016-05-23 21:08:46 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 63) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 65). 915 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 62). 915 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 65) in 7 ms on localhost (1/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 62) in 11 ms on localhost (2/4) -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:54441 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:08:46 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 64) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 63). 915 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 63) in 15 ms on localhost (3/4) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 64). 915 bytes result sent to driver -2016-05-23 21:08:46 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 64) in 16 ms on localhost (4/4) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-23 21:08:46 INFO DAGScheduler:59 - ResultStage 34 (foreach at StatisticsCorrLearning.scala:59) finished in 0.019 s -2016-05-23 21:08:46 INFO DAGScheduler:59 - Job 20 finished: foreach at StatisticsCorrLearning.scala:59, took 0.036894 s -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:54441 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:08:46 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:54441 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:08:46 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:61 -2016-05-23 21:08:46 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Got job 21 (foreach at StatisticsCorrLearning.scala:61) with 4 output partitions -2016-05-23 21:08:46 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at StatisticsCorrLearning.scala:61) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:57), which has no missing parents -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=369024, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:54441 in memory (size: 2.5 KB, free: 730.6 MB) -2016-05-23 21:08:46 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=370248, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:54441 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:08:46 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:54441 in memory (size: 2.2 KB, free: 730.6 MB) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:57) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 4 tasks -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 66, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:08:46 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-23 21:08:46 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 3.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:08:46 INFO Executor:59 - Running task 3.0 in stage 35.0 (TID 69) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 3.0 in stage 35.0 (TID 69). 915 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 35.0 (TID 69) in 13 ms on localhost (1/4) -2016-05-23 21:08:46 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 68) -2016-05-23 21:08:46 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 67) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 68). 915 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 67). 915 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 66) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 68) in 22 ms on localhost (2/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 67) in 30 ms on localhost (3/4) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 66). 915 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 66) in 34 ms on localhost (4/4) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-23 21:08:46 INFO DAGScheduler:59 - ResultStage 35 (foreach at StatisticsCorrLearning.scala:61) finished in 0.036 s -2016-05-23 21:08:46 INFO DAGScheduler:59 - Job 21 finished: foreach at StatisticsCorrLearning.scala:61, took 0.052919 s -2016-05-23 21:08:46 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Got job 22 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:08:46 INFO DAGScheduler:59 - Final stage: ResultStage 36(first at RowMatrix.scala:65) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=364667, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(1522) called with curMem=367171, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1522.0 B, free 730.2 MB) -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:54441 (size: 1522.0 B, free: 730.6 MB) -2016-05-23 21:08:46 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:46 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 70) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 70). 1046 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 70) in 6 ms on localhost (1/1) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-23 21:08:46 INFO DAGScheduler:59 - ResultStage 36 (first at RowMatrix.scala:65) finished in 0.006 s -2016-05-23 21:08:46 INFO DAGScheduler:59 - Job 22 finished: first at RowMatrix.scala:65, took 0.020703 s -2016-05-23 21:08:46 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Got job 23 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:08:46 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=368693, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(1932) called with curMem=372093, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1932.0 B, free 730.2 MB) -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:54441 (size: 1932.0 B, free: 730.5 MB) -2016-05-23 21:08:46 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 4 tasks -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 3.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:08:46 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 72) -2016-05-23 21:08:46 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 73) -2016-05-23 21:08:46 INFO Executor:59 - Running task 3.0 in stage 37.0 (TID 74) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 72). 1231 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 3.0 in stage 37.0 (TID 74). 1231 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 1231 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 73). 1231 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 72) in 13 ms on localhost (1/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 37.0 (TID 74) in 11 ms on localhost (2/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 15 ms on localhost (3/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 73) in 15 ms on localhost (4/4) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-23 21:08:46 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at RowMatrix.scala:332) finished in 0.018 s -2016-05-23 21:08:46 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at RowMatrix.scala:332, took 0.031846 s -2016-05-23 21:08:46 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Got job 24 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:08:46 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=374025, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(1916) called with curMem=377465, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1916.0 B, free 730.2 MB) -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:54441 (size: 1916.0 B, free: 730.5 MB) -2016-05-23 21:08:46 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 77, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 78, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:08:46 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 78) -2016-05-23 21:08:46 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 77) -2016-05-23 21:08:46 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 78). 1109 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 78) in 7 ms on localhost (1/4) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 1109 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 1109 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 77). 1109 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 13 ms on localhost (2/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 14 ms on localhost (3/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 77) in 13 ms on localhost (4/4) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-23 21:08:46 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at RowMatrix.scala:124) finished in 0.016 s -2016-05-23 21:08:46 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at RowMatrix.scala:124, took 0.028274 s -2016-05-23 21:08:46 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Got job 25 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:08:46 INFO DAGScheduler:59 - Final stage: ResultStage 39(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=379381, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(1851) called with curMem=382757, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1851.0 B, free 730.2 MB) -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:54441 (size: 1851.0 B, free: 730.5 MB) -2016-05-23 21:08:46 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 4 tasks -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 79, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 80, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 81, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 3.0 in stage 39.0 (TID 82, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:08:46 INFO Executor:59 - Running task 3.0 in stage 39.0 (TID 82) -2016-05-23 21:08:46 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 81) -2016-05-23 21:08:46 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 79) -2016-05-23 21:08:46 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 80) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 79). 1254 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 80). 1254 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 81). 1249 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Finished task 3.0 in stage 39.0 (TID 82). 1294 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 79) in 13 ms on localhost (1/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 81) in 12 ms on localhost (2/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 80) in 13 ms on localhost (3/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 39.0 (TID 82) in 13 ms on localhost (4/4) -2016-05-23 21:08:46 INFO DAGScheduler:59 - ResultStage 39 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.016 s -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-23 21:08:46 INFO DAGScheduler:59 - Job 25 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.029940 s -2016-05-23 21:08:46 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Registering RDD 51 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Got job 26 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:08:46 INFO DAGScheduler:59 - Final stage: ResultStage 41(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 40) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 40) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=384608, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(2281) called with curMem=388648, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:54441 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:08:46 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 4 tasks -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 83, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 84, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 85, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 3.0 in stage 40.0 (TID 86, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:08:46 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 84) -2016-05-23 21:08:46 INFO Executor:59 - Running task 3.0 in stage 40.0 (TID 86) -2016-05-23 21:08:46 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 85) -2016-05-23 21:08:46 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 83) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 3.0 in stage 40.0 (TID 86). 1161 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 40.0 (TID 86) in 46 ms on localhost (1/4) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 83). 1161 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 84). 1161 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 83) in 55 ms on localhost (2/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 84) in 55 ms on localhost (3/4) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 85). 1161 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 85) in 57 ms on localhost (4/4) -2016-05-23 21:08:46 INFO DAGScheduler:59 - ShuffleMapStage 40 (flatMap at SpearmanCorrelation.scala:49) finished in 0.059 s -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-23 21:08:46 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:08:46 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:08:46 INFO DAGScheduler:59 - waiting: Set(ResultStage 41) -2016-05-23 21:08:46 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Missing parents for ResultStage 41: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=390929, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=393473, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.2 MB) -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:54441 (size: 1567.0 B, free: 730.5 MB) -2016-05-23 21:08:46 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 3 tasks -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 87, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 87) -2016-05-23 21:08:46 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 88) -2016-05-23 21:08:46 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 89) -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:46 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 87). 1203 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 87) in 12 ms on localhost (1/3) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 88). 1203 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 88) in 13 ms on localhost (2/3) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 2.0 in stage 41.0 (TID 89). 1203 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 41.0 (TID 89) in 14 ms on localhost (3/3) -2016-05-23 21:08:46 INFO DAGScheduler:59 - ResultStage 41 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.016 s -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-23 21:08:46 INFO DAGScheduler:59 - Job 26 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.095178 s -2016-05-23 21:08:46 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:08:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 163 bytes -2016-05-23 21:08:46 INFO DAGScheduler:59 - Registering RDD 56 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Got job 27 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:08:46 INFO DAGScheduler:59 - Final stage: ResultStage 44(first at RowMatrix.scala:65) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 43) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 43) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=395040, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(2305) called with curMem=399272, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:54441 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:08:46 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 4 tasks -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 90, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 91, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 43.0 (TID 92, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 3.0 in stage 43.0 (TID 93, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:08:46 INFO Executor:59 - Running task 3.0 in stage 43.0 (TID 93) -2016-05-23 21:08:46 INFO Executor:59 - Running task 2.0 in stage 43.0 (TID 92) -2016-05-23 21:08:46 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 90) -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:46 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 91) -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:46 INFO Executor:59 - Finished task 3.0 in stage 43.0 (TID 93). 1377 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Finished task 2.0 in stage 43.0 (TID 92). 1377 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 43.0 (TID 93) in 102 ms on localhost (1/4) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 91). 1377 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 43.0 (TID 92) in 107 ms on localhost (2/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 91) in 110 ms on localhost (3/4) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 90). 1377 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 90) in 113 ms on localhost (4/4) -2016-05-23 21:08:46 INFO DAGScheduler:59 - ShuffleMapStage 43 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.114 s -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-23 21:08:46 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:08:46 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:08:46 INFO DAGScheduler:59 - waiting: Set(ResultStage 44) -2016-05-23 21:08:46 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Missing parents for ResultStage 44: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=401577, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(2545) called with curMem=406473, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:54441 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:08:46 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 1 tasks -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 94, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 94) -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:46 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 94). 1296 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 94) in 10 ms on localhost (1/1) -2016-05-23 21:08:46 INFO DAGScheduler:59 - ResultStage 44 (first at RowMatrix.scala:65) finished in 0.011 s -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-23 21:08:46 INFO DAGScheduler:59 - Job 27 finished: first at RowMatrix.scala:65, took 0.147915 s -2016-05-23 21:08:46 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:08:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 163 bytes -2016-05-23 21:08:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 166 bytes -2016-05-23 21:08:46 INFO DAGScheduler:59 - Got job 28 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:08:46 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 46) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=409018, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(2914) called with curMem=414714, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:54441 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:08:46 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 4 tasks -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 95, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 96, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 47.0 (TID 97, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 3.0 in stage 47.0 (TID 98, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 95) -2016-05-23 21:08:46 INFO Executor:59 - Running task 2.0 in stage 47.0 (TID 97) -2016-05-23 21:08:46 INFO Executor:59 - Running task 3.0 in stage 47.0 (TID 98) -2016-05-23 21:08:46 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 96) -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:46 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 95). 1481 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Finished task 2.0 in stage 47.0 (TID 97). 1481 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 96). 1481 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Finished task 3.0 in stage 47.0 (TID 98). 1481 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 95) in 14 ms on localhost (1/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 47.0 (TID 97) in 14 ms on localhost (2/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 96) in 15 ms on localhost (3/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 47.0 (TID 98) in 15 ms on localhost (4/4) -2016-05-23 21:08:46 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at RowMatrix.scala:332) finished in 0.016 s -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-23 21:08:46 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at RowMatrix.scala:332, took 0.030925 s -2016-05-23 21:08:46 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Got job 29 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:08:46 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 49) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=417628, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=423364, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:08:46 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:54441 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:08:46 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 4 tasks -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 50.0 (TID 101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Starting task 3.0 in stage 50.0 (TID 102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:46 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-23 21:08:46 INFO Executor:59 - Running task 2.0 in stage 50.0 (TID 101) -2016-05-23 21:08:46 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-23 21:08:46 INFO Executor:59 - Running task 3.0 in stage 50.0 (TID 102) -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-23 21:08:46 INFO Executor:59 - Finished task 3.0 in stage 50.0 (TID 102). 1359 bytes result sent to driver -2016-05-23 21:08:46 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 1359 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 50.0 (TID 102) in 14 ms on localhost (1/4) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 2.0 in stage 50.0 (TID 101). 1359 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 50.0 (TID 101) in 19 ms on localhost (2/4) -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 20 ms on localhost (3/4) -2016-05-23 21:08:46 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 1359 bytes result sent to driver -2016-05-23 21:08:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 23 ms on localhost (4/4) -2016-05-23 21:08:46 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at RowMatrix.scala:124) finished in 0.023 s -2016-05-23 21:08:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-23 21:08:46 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at RowMatrix.scala:124, took 0.035520 s -2016-05-23 21:08:46 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:77 -2016-05-23 21:08:46 INFO DAGScheduler:59 - Got job 30 (foreach at StatisticsCorrLearning.scala:77) with 4 output partitions -2016-05-23 21:08:46 INFO DAGScheduler:59 - Final stage: ResultStage 51(foreach at StatisticsCorrLearning.scala:77) -2016-05-23 21:08:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:46 INFO DAGScheduler:59 - Submitting ResultStage 51 (ParallelCollectionRDD[61] at parallelize at StatisticsCorrLearning.scala:74), which has no missing parents -2016-05-23 21:08:46 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=426260, maxMem=766075207 -2016-05-23 21:08:46 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=427484, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:54441 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 51 (ParallelCollectionRDD[61] at parallelize at StatisticsCorrLearning.scala:74) -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 4 tasks -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 103, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 104, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 51.0 (TID 105, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 51.0 (TID 106, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:08:47 INFO Executor:59 - Running task 2.0 in stage 51.0 (TID 105) -2016-05-23 21:08:47 INFO Executor:59 - Running task 3.0 in stage 51.0 (TID 106) -2016-05-23 21:08:47 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 103) -2016-05-23 21:08:47 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 104) -2016-05-23 21:08:47 INFO Executor:59 - Finished task 2.0 in stage 51.0 (TID 105). 915 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 103). 915 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 3.0 in stage 51.0 (TID 106). 915 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 51.0 (TID 105) in 5 ms on localhost (1/4) -2016-05-23 21:08:47 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 104). 915 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 51.0 (TID 106) in 6 ms on localhost (2/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 103) in 9 ms on localhost (3/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 104) in 8 ms on localhost (4/4) -2016-05-23 21:08:47 INFO DAGScheduler:59 - ResultStage 51 (foreach at StatisticsCorrLearning.scala:77) finished in 0.009 s -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-23 21:08:47 INFO DAGScheduler:59 - Job 30 finished: foreach at StatisticsCorrLearning.scala:77, took 0.018650 s -2016-05-23 21:08:47 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:79 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Got job 31 (foreach at StatisticsCorrLearning.scala:79) with 4 output partitions -2016-05-23 21:08:47 INFO DAGScheduler:59 - Final stage: ResultStage 52(foreach at StatisticsCorrLearning.scala:79) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting ResultStage 52 (ParallelCollectionRDD[62] at parallelize at StatisticsCorrLearning.scala:75), which has no missing parents -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=428377, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=429601, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:54441 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 52 (ParallelCollectionRDD[62] at parallelize at StatisticsCorrLearning.scala:75) -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 4 tasks -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 107, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 108, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 52.0 (TID 109, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 52.0 (TID 110, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:08:47 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 108) -2016-05-23 21:08:47 INFO Executor:59 - Running task 2.0 in stage 52.0 (TID 109) -2016-05-23 21:08:47 INFO Executor:59 - Running task 3.0 in stage 52.0 (TID 110) -2016-05-23 21:08:47 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 107) -2016-05-23 21:08:47 INFO Executor:59 - Finished task 2.0 in stage 52.0 (TID 109). 915 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 107). 915 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 108). 915 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 3.0 in stage 52.0 (TID 110). 915 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 52.0 (TID 109) in 4 ms on localhost (1/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 107) in 6 ms on localhost (2/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 108) in 5 ms on localhost (3/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 52.0 (TID 110) in 6 ms on localhost (4/4) -2016-05-23 21:08:47 INFO DAGScheduler:59 - ResultStage 52 (foreach at StatisticsCorrLearning.scala:79) finished in 0.007 s -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-23 21:08:47 INFO DAGScheduler:59 - Job 31 finished: foreach at StatisticsCorrLearning.scala:79, took 0.014330 s -2016-05-23 21:08:47 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Got job 32 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:08:47 INFO DAGScheduler:59 - Final stage: ResultStage 53(first at RowMatrix.scala:65) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[64] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=430494, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=432998, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:54441 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 53 (MapPartitionsRDD[64] at map at Correlation.scala:45) -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 1 tasks -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 111, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:08:47 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 111) -2016-05-23 21:08:47 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 111). 935 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 111) in 3 ms on localhost (1/1) -2016-05-23 21:08:47 INFO DAGScheduler:59 - ResultStage 53 (first at RowMatrix.scala:65) finished in 0.003 s -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-23 21:08:47 INFO DAGScheduler:59 - Job 32 finished: first at RowMatrix.scala:65, took 0.011794 s -2016-05-23 21:08:47 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Got job 33 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 21:08:47 INFO DAGScheduler:59 - Final stage: ResultStage 54(first at RowMatrix.scala:65) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[64] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=434522, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=437026, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:54441 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 54 (MapPartitionsRDD[64] at map at Correlation.scala:45) -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 3 tasks -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 112, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 113, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 54.0 (TID 114, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:47 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 113) -2016-05-23 21:08:47 INFO Executor:59 - Running task 2.0 in stage 54.0 (TID 114) -2016-05-23 21:08:47 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 112) -2016-05-23 21:08:47 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 113). 1046 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 112). 1046 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 2.0 in stage 54.0 (TID 114). 1046 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 113) in 4 ms on localhost (1/3) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 112) in 6 ms on localhost (2/3) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 54.0 (TID 114) in 4 ms on localhost (3/3) -2016-05-23 21:08:47 INFO DAGScheduler:59 - ResultStage 54 (first at RowMatrix.scala:65) finished in 0.007 s -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-23 21:08:47 INFO DAGScheduler:59 - Job 33 finished: first at RowMatrix.scala:65, took 0.014604 s -2016-05-23 21:08:47 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Got job 34 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:08:47 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[65] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=438550, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=441950, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:54441 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 55 (MapPartitionsRDD[65] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 4 tasks -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 115, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 116, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 55.0 (TID 117, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 55.0 (TID 118, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:47 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 116) -2016-05-23 21:08:47 INFO Executor:59 - Running task 2.0 in stage 55.0 (TID 117) -2016-05-23 21:08:47 INFO Executor:59 - Running task 3.0 in stage 55.0 (TID 118) -2016-05-23 21:08:47 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 115) -2016-05-23 21:08:47 INFO Executor:59 - Finished task 2.0 in stage 55.0 (TID 117). 1231 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 116). 1231 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 3.0 in stage 55.0 (TID 118). 1231 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 115). 1231 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 115) in 10 ms on localhost (1/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 55.0 (TID 117) in 10 ms on localhost (2/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 116) in 11 ms on localhost (3/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 55.0 (TID 118) in 10 ms on localhost (4/4) -2016-05-23 21:08:47 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at RowMatrix.scala:332) finished in 0.011 s -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-23 21:08:47 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at RowMatrix.scala:332, took 0.021165 s -2016-05-23 21:08:47 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Got job 35 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:08:47 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[66] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=443884, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=447324, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:54441 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 56 (MapPartitionsRDD[66] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 4 tasks -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 119, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 120, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 56.0 (TID 121, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 56.0 (TID 122, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:47 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 119) -2016-05-23 21:08:47 INFO Executor:59 - Running task 2.0 in stage 56.0 (TID 121) -2016-05-23 21:08:47 INFO Executor:59 - Running task 3.0 in stage 56.0 (TID 122) -2016-05-23 21:08:47 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 120) -2016-05-23 21:08:47 INFO Executor:59 - Finished task 3.0 in stage 56.0 (TID 122). 1109 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 120). 1109 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 56.0 (TID 122) in 5 ms on localhost (1/4) -2016-05-23 21:08:47 INFO Executor:59 - Finished task 2.0 in stage 56.0 (TID 121). 1109 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 120) in 7 ms on localhost (2/4) -2016-05-23 21:08:47 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 119). 1109 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 56.0 (TID 121) in 8 ms on localhost (3/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 119) in 9 ms on localhost (4/4) -2016-05-23 21:08:47 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at RowMatrix.scala:124) finished in 0.010 s -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-23 21:08:47 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at RowMatrix.scala:124, took 0.019372 s -2016-05-23 21:08:47 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Got job 36 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:08:47 INFO DAGScheduler:59 - Final stage: ResultStage 57(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:08:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[72] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=449242, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(1858) called with curMem=452618, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 1858.0 B, free 730.2 MB) -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:54441 (size: 1858.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 57 (MapPartitionsRDD[72] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 4 tasks -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 123, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 124, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 57.0 (TID 125, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 57.0 (TID 126, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:08:47 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 123) -2016-05-23 21:08:47 INFO Executor:59 - Running task 2.0 in stage 57.0 (TID 125) -2016-05-23 21:08:47 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 124) -2016-05-23 21:08:47 INFO Executor:59 - Running task 3.0 in stage 57.0 (TID 126) -2016-05-23 21:08:47 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 124). 1254 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 2.0 in stage 57.0 (TID 125). 1249 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 123). 1104 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 3.0 in stage 57.0 (TID 126). 1254 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 124) in 9 ms on localhost (1/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 57.0 (TID 126) in 8 ms on localhost (2/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 57.0 (TID 125) in 9 ms on localhost (3/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 123) in 11 ms on localhost (4/4) -2016-05-23 21:08:47 INFO DAGScheduler:59 - ResultStage 57 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.011 s -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-23 21:08:47 INFO DAGScheduler:59 - Job 36 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.022086 s -2016-05-23 21:08:47 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Registering RDD 70 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Got job 37 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:08:47 INFO DAGScheduler:59 - Final stage: ResultStage 59(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 58) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 58) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 58 (MapPartitionsRDD[70] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=454476, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(2285) called with curMem=458516, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:54441 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:08:47 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 58 (MapPartitionsRDD[70] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 4 tasks -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 127, localhost, PROCESS_LOCAL, 2221 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 128, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 58.0 (TID 129, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 58.0 (TID 130, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:08:47 INFO Executor:59 - Running task 2.0 in stage 58.0 (TID 129) -2016-05-23 21:08:47 INFO Executor:59 - Running task 3.0 in stage 58.0 (TID 130) -2016-05-23 21:08:47 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 127) -2016-05-23 21:08:47 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 128) -2016-05-23 21:08:47 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 127). 1161 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 127) in 19 ms on localhost (1/4) -2016-05-23 21:08:47 INFO Executor:59 - Finished task 3.0 in stage 58.0 (TID 130). 1161 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 58.0 (TID 130) in 47 ms on localhost (2/4) -2016-05-23 21:08:47 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 128). 1161 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 2.0 in stage 58.0 (TID 129). 1161 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 128) in 56 ms on localhost (3/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 58.0 (TID 129) in 57 ms on localhost (4/4) -2016-05-23 21:08:47 INFO DAGScheduler:59 - ShuffleMapStage 58 (flatMap at SpearmanCorrelation.scala:49) finished in 0.058 s -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-05-23 21:08:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:08:47 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:08:47 INFO DAGScheduler:59 - waiting: Set(ResultStage 59) -2016-05-23 21:08:47 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:08:47 INFO DAGScheduler:59 - Missing parents for ResultStage 59: List() -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting ResultStage 59 (ShuffledRDD[73] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=460801, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=463345, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.1 MB) -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:54441 (size: 1567.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 59 (ShuffledRDD[73] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 3 tasks -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 131, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 132, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 59.0 (TID 133, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:47 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 132) -2016-05-23 21:08:47 INFO Executor:59 - Running task 2.0 in stage 59.0 (TID 133) -2016-05-23 21:08:47 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 131) -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned shuffle 4 -2016-05-23 21:08:47 INFO Executor:59 - Finished task 2.0 in stage 59.0 (TID 133). 1203 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 132). 1203 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 59.0 (TID 133) in 18 ms on localhost (1/3) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 132) in 20 ms on localhost (2/3) -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:54441 in memory (size: 1851.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 131). 1203 bytes result sent to driver -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 131) in 25 ms on localhost (3/3) -2016-05-23 21:08:47 INFO DAGScheduler:59 - ResultStage 59 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.025 s -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-05-23 21:08:47 INFO DAGScheduler:59 - Job 37 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.105578 s -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:54441 in memory (size: 1916.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:54441 in memory (size: 1932.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:54441 in memory (size: 1522.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:54441 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:54441 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:54441 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:54441 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:08:47 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:08:47 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 168 bytes -2016-05-23 21:08:47 INFO DAGScheduler:59 - Registering RDD 75 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Got job 38 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:08:47 INFO DAGScheduler:59 - Final stage: ResultStage 62(first at RowMatrix.scala:65) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 61) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 61) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 61 (MapPartitionsRDD[75] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:54441 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=415321, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:54441 in memory (size: 1858.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(2305) called with curMem=416177, maxMem=766075207 -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:54441 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:08:47 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:54441 in memory (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 61 (MapPartitionsRDD[75] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 4 tasks -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 134, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 135, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 61.0 (TID 136, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 61.0 (TID 137, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:08:47 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 135) -2016-05-23 21:08:47 INFO Executor:59 - Running task 3.0 in stage 61.0 (TID 137) -2016-05-23 21:08:47 INFO Executor:59 - Running task 2.0 in stage 61.0 (TID 136) -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:54441 in memory (size: 1934.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 134) -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:54441 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:54441 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:54441 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:54441 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:54441 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:54441 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:54441 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:54441 in memory (size: 2.3 KB, free: 730.6 MB) -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned shuffle 5 -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:54441 in memory (size: 1567.0 B, free: 730.6 MB) -2016-05-23 21:08:47 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:54441 in memory (size: 2.2 KB, free: 730.6 MB) -2016-05-23 21:08:47 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 134). 1377 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 2.0 in stage 61.0 (TID 136). 1377 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 134) in 78 ms on localhost (1/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 61.0 (TID 136) in 77 ms on localhost (2/4) -2016-05-23 21:08:47 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 135). 1377 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 3.0 in stage 61.0 (TID 137). 1377 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 135) in 83 ms on localhost (3/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 61.0 (TID 137) in 83 ms on localhost (4/4) -2016-05-23 21:08:47 INFO DAGScheduler:59 - ShuffleMapStage 61 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.086 s -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-05-23 21:08:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:08:47 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:08:47 INFO DAGScheduler:59 - waiting: Set(ResultStage 62) -2016-05-23 21:08:47 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:08:47 INFO DAGScheduler:59 - Missing parents for ResultStage 62: List() -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=353848, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=358744, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:54441 (size: 2.5 KB, free: 730.6 MB) -2016-05-23 21:08:47 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 62 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 1 tasks -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 138, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:47 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 138) -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:47 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 138). 1185 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 138) in 11 ms on localhost (1/1) -2016-05-23 21:08:47 INFO DAGScheduler:59 - ResultStage 62 (first at RowMatrix.scala:65) finished in 0.011 s -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-05-23 21:08:47 INFO DAGScheduler:59 - Job 38 finished: first at RowMatrix.scala:65, took 0.122346 s -2016-05-23 21:08:47 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:08:47 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 168 bytes -2016-05-23 21:08:47 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 165 bytes -2016-05-23 21:08:47 INFO DAGScheduler:59 - Got job 39 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 21:08:47 INFO DAGScheduler:59 - Final stage: ResultStage 65(first at RowMatrix.scala:65) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 64) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87), which has no missing parents -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=361288, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=366184, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:54441 (size: 2.5 KB, free: 730.6 MB) -2016-05-23 21:08:47 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 65 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 3 tasks -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 139, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 140, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 65.0 (TID 141, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:47 INFO Executor:59 - Running task 2.0 in stage 65.0 (TID 141) -2016-05-23 21:08:47 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 140) -2016-05-23 21:08:47 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 139) -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:47 INFO Executor:59 - Finished task 2.0 in stage 65.0 (TID 141). 1296 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 65.0 (TID 141) in 12 ms on localhost (1/3) -2016-05-23 21:08:47 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 139). 1296 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 140). 1296 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 139) in 16 ms on localhost (2/3) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 140) in 16 ms on localhost (3/3) -2016-05-23 21:08:47 INFO DAGScheduler:59 - ResultStage 65 (first at RowMatrix.scala:65) finished in 0.017 s -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-05-23 21:08:47 INFO DAGScheduler:59 - Job 39 finished: first at RowMatrix.scala:65, took 0.027648 s -2016-05-23 21:08:47 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Got job 40 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:08:47 INFO DAGScheduler:59 - Final stage: ResultStage 68(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 67) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting ResultStage 68 (MapPartitionsRDD[78] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=368728, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(2914) called with curMem=374424, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:54441 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:08:47 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 68 (MapPartitionsRDD[78] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Adding task set 68.0 with 4 tasks -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 68.0 (TID 142, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 68.0 (TID 143, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 68.0 (TID 144, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 68.0 (TID 145, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:47 INFO Executor:59 - Running task 3.0 in stage 68.0 (TID 145) -2016-05-23 21:08:47 INFO Executor:59 - Running task 2.0 in stage 68.0 (TID 144) -2016-05-23 21:08:47 INFO Executor:59 - Running task 0.0 in stage 68.0 (TID 142) -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:47 INFO Executor:59 - Running task 1.0 in stage 68.0 (TID 143) -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:47 INFO Executor:59 - Finished task 2.0 in stage 68.0 (TID 144). 1481 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 3.0 in stage 68.0 (TID 145). 1481 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 1.0 in stage 68.0 (TID 143). 1481 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 0.0 in stage 68.0 (TID 142). 1481 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 68.0 (TID 144) in 28 ms on localhost (1/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 68.0 (TID 145) in 28 ms on localhost (2/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 68.0 (TID 143) in 29 ms on localhost (3/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 68.0 (TID 142) in 31 ms on localhost (4/4) -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 68.0, whose tasks have all completed, from pool -2016-05-23 21:08:47 INFO DAGScheduler:59 - ResultStage 68 (treeAggregate at RowMatrix.scala:332) finished in 0.031 s -2016-05-23 21:08:47 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at RowMatrix.scala:332, took 0.042152 s -2016-05-23 21:08:47 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Got job 41 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:08:47 INFO DAGScheduler:59 - Final stage: ResultStage 71(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 70) -2016-05-23 21:08:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting ResultStage 71 (MapPartitionsRDD[79] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=377338, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:08:47 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=383074, maxMem=766075207 -2016-05-23 21:08:47 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:08:47 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:54441 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:08:47 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 71 (MapPartitionsRDD[79] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Adding task set 71.0 with 4 tasks -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 71.0 (TID 146, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 71.0 (TID 147, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 71.0 (TID 148, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 71.0 (TID 149, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:08:47 INFO Executor:59 - Running task 3.0 in stage 71.0 (TID 149) -2016-05-23 21:08:47 INFO Executor:59 - Running task 1.0 in stage 71.0 (TID 147) -2016-05-23 21:08:47 INFO Executor:59 - Running task 0.0 in stage 71.0 (TID 146) -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:47 INFO Executor:59 - Finished task 3.0 in stage 71.0 (TID 149). 1359 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Running task 2.0 in stage 71.0 (TID 148) -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:08:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:08:47 INFO Executor:59 - Finished task 1.0 in stage 71.0 (TID 147). 1359 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 0.0 in stage 71.0 (TID 146). 1359 bytes result sent to driver -2016-05-23 21:08:47 INFO Executor:59 - Finished task 2.0 in stage 71.0 (TID 148). 1359 bytes result sent to driver -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 71.0 (TID 149) in 21 ms on localhost (1/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 71.0 (TID 146) in 25 ms on localhost (2/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 71.0 (TID 147) in 25 ms on localhost (3/4) -2016-05-23 21:08:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 71.0 (TID 148) in 24 ms on localhost (4/4) -2016-05-23 21:08:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 71.0, whose tasks have all completed, from pool -2016-05-23 21:08:47 INFO DAGScheduler:59 - ResultStage 71 (treeAggregate at RowMatrix.scala:124) finished in 0.026 s -2016-05-23 21:08:47 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at RowMatrix.scala:124, took 0.039753 s -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 21:08:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 21:08:47 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 21:08:47 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 21:08:47 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 21:08:47 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 21:08:47 INFO BlockManager:59 - BlockManager stopped -2016-05-23 21:08:47 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 21:08:47 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 21:08:47 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 21:08:47 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 21:08:47 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 21:08:47 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-1b3e5db0-7080-424f-946b-82cbc1675c42 -2016-05-23 21:08:47 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 21:19:58 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 21:19:59 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 21:20:00 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 21:20:00 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 21:20:00 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 21:20:01 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 21:20:01 INFO Remoting:74 - Starting remoting -2016-05-23 21:20:01 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:55008] -2016-05-23 21:20:01 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55008. -2016-05-23 21:20:01 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 21:20:01 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 21:20:01 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-88e592c2-b512-4f0c-a747-38ac307269b0 -2016-05-23 21:20:01 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 21:20:02 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-a063dbf1-97f1-4f96-b27d-c9d7db857943\httpd-29e11ff3-fe6b-4b2e-a9f9-9205a55be0f6 -2016-05-23 21:20:02 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 21:20:02 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 21:20:02 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55009 -2016-05-23 21:20:02 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55009. -2016-05-23 21:20:02 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 21:20:02 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 21:20:02 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 21:20:02 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 21:20:02 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 21:20:02 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 21:20:02 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 21:20:03 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55028. -2016-05-23 21:20:03 INFO NettyBlockTransferService:59 - Server created on 55028 -2016-05-23 21:20:03 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 21:20:03 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55028 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55028) -2016-05-23 21:20:03 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 21:20:04 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 21:20:04 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 21:20:04 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 21:20:04 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 21:20:04 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55028 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 21:20:04 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:22 -2016-05-23 21:20:04 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 21:20:04 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 21:20:04 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 21:20:04 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 21:20:04 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55028 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 21:20:04 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:25 -2016-05-23 21:20:05 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 21:20:06 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 21:20:06 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 21:20:06 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 21:20:06 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 21:20:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:06 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:23), which has no missing parents -2016-05-23 21:20:06 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 21:20:06 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 21:20:06 INFO MemoryStore:59 - ensureFreeSpace(2046) called with curMem=346712, maxMem=766075207 -2016-05-23 21:20:06 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2046.0 B, free 730.3 MB) -2016-05-23 21:20:06 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55028 (size: 2046.0 B, free: 730.6 MB) -2016-05-23 21:20:06 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:23) -2016-05-23 21:20:06 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 21:20:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:20:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:20:06 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 21:20:06 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 21:20:06 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:20:06 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:20:06 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 21:20:06 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 21:20:06 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 21:20:06 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 21:20:06 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 21:20:06 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 21:20:06 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 21:20:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 214 ms on localhost (1/2) -2016-05-23 21:20:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 178 ms on localhost (2/2) -2016-05-23 21:20:06 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:29) finished in 0.244 s -2016-05-23 21:20:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 21:20:06 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:29, took 0.441646 s -2016-05-23 21:20:06 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 21:20:06 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:31 -2016-05-23 21:20:06 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:31) with 2 output partitions -2016-05-23 21:20:06 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:31) -2016-05-23 21:20:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:06 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:26), which has no missing parents -2016-05-23 21:20:06 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348758, maxMem=766075207 -2016-05-23 21:20:06 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 21:20:07 INFO MemoryStore:59 - ensureFreeSpace(2045) called with curMem=352270, maxMem=766075207 -2016-05-23 21:20:07 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2045.0 B, free 730.2 MB) -2016-05-23 21:20:07 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55028 (size: 2045.0 B, free: 730.6 MB) -2016-05-23 21:20:07 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:26) -2016-05-23 21:20:07 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 21:20:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:20:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:20:07 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 21:20:07 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:20:07 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 21:20:07 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 21:20:07 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:20:07 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 21:20:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 61 ms on localhost (1/2) -2016-05-23 21:20:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 69 ms on localhost (2/2) -2016-05-23 21:20:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 21:20:07 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:31) finished in 0.069 s -2016-05-23 21:20:07 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:31, took 0.518542 s -2016-05-23 21:20:07 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:20:07 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:20:07 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 21:20:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:07 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:20:07 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354315, maxMem=766075207 -2016-05-23 21:20:07 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 21:20:07 INFO MemoryStore:59 - ensureFreeSpace(2464) called with curMem=358875, maxMem=766075207 -2016-05-23 21:20:07 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:20:07 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55028 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 21:20:07 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:07 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 21:20:07 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 21:20:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:20:07 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 21:20:07 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:55028 in memory (size: 2045.0 B, free: 730.6 MB) -2016-05-23 21:20:07 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:20:07 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:20:07 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:55028 in memory (size: 2046.0 B, free: 730.6 MB) -2016-05-23 21:20:07 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 21:20:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 49 ms on localhost (1/1) -2016-05-23 21:20:07 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.050 s -2016-05-23 21:20:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 21:20:07 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.120611 s -2016-05-23 21:20:08 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:20:08 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:20:08 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:20:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:08 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:20:08 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=350224, maxMem=766075207 -2016-05-23 21:20:08 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 21:20:08 INFO MemoryStore:59 - ensureFreeSpace(2861) called with curMem=355672, maxMem=766075207 -2016-05-23 21:20:08 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:20:08 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55028 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:20:08 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:20:08 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 21:20:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:20:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:20:08 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 21:20:08 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 21:20:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:20:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:20:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:20:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:20:08 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 21:20:08 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 21:20:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 75 ms on localhost (1/2) -2016-05-23 21:20:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 76 ms on localhost (2/2) -2016-05-23 21:20:08 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.080 s -2016-05-23 21:20:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 21:20:08 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 21:20:08 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 21:20:08 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.121681 s -2016-05-23 21:20:08 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:20:08 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:20:08 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:20:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:08 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:20:08 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=358533, maxMem=766075207 -2016-05-23 21:20:08 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 21:20:08 INFO MemoryStore:59 - ensureFreeSpace(2850) called with curMem=364029, maxMem=766075207 -2016-05-23 21:20:08 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:20:08 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55028 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:20:08 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:20:08 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 21:20:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:20:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:20:08 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 21:20:08 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 21:20:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:20:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:20:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:20:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:20:08 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 21:20:08 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 21:20:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 39 ms on localhost (1/2) -2016-05-23 21:20:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 43 ms on localhost (2/2) -2016-05-23 21:20:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 21:20:08 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.043 s -2016-05-23 21:20:08 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.084478 s -2016-05-23 21:20:08 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:20:08 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 21:20:08 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:20:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:08 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:20:08 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=366879, maxMem=766075207 -2016-05-23 21:20:08 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 21:20:08 INFO MemoryStore:59 - ensureFreeSpace(2743) called with curMem=372223, maxMem=766075207 -2016-05-23 21:20:08 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 21:20:08 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55028 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 21:20:08 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:20:08 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 21:20:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:20:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:20:08 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 21:20:08 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 21:20:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:20:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:20:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:20:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:20:08 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 21:20:08 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 21:20:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 66 ms on localhost (1/2) -2016-05-23 21:20:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 73 ms on localhost (2/2) -2016-05-23 21:20:08 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.102 s -2016-05-23 21:20:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 21:20:08 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.132712 s -2016-05-23 21:20:09 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:20:09 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:20:09 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 21:20:09 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:20:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 21:20:09 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=374966, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(3155) called with curMem=380894, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 21:20:09 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55028 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 21:20:09 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:20:09 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 21:20:09 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 21:20:09 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 21:20:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:20:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:20:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:20:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:20:09 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 21:20:09 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 85 ms on localhost (1/2) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 90 ms on localhost (2/2) -2016-05-23 21:20:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 21:20:09 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.090 s -2016-05-23 21:20:09 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:20:09 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:20:09 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 21:20:09 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:20:09 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=384049, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=386553, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 21:20:09 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55028 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 21:20:09 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:20:09 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:09 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 21:20:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:20:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-23 21:20:09 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 70 ms on localhost (1/1) -2016-05-23 21:20:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 21:20:09 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.072 s -2016-05-23 21:20:09 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.233200 s -2016-05-23 21:20:09 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:20:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 21:20:09 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:20:09 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:20:09 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 21:20:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 21:20:09 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=388098, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=392290, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:20:09 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55028 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:20:09 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:20:09 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:20:09 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 21:20:09 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:55028 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 21:20:09 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 21:20:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:20:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:20:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:09 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 21:20:09 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:55028 in memory (size: 3.1 KB, free: 730.5 MB) -2016-05-23 21:20:09 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 21:20:09 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 21:20:09 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 53 ms on localhost (1/2) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 52 ms on localhost (2/2) -2016-05-23 21:20:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 21:20:09 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.057 s -2016-05-23 21:20:09 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:20:09 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:20:09 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 21:20:09 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:20:09 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:55028 in memory (size: 2.7 KB, free: 730.5 MB) -2016-05-23 21:20:09 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:20:09 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=370503, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 21:20:09 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:55028 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:20:09 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=369863, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:20:09 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55028 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:20:09 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 21:20:09 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55028 (size: 2.5 KB, free: 730.6 MB) -2016-05-23 21:20:09 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:20:09 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:09 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 21:20:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:20:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:09 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 31 ms on localhost (1/1) -2016-05-23 21:20:09 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.032 s -2016-05-23 21:20:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 21:20:09 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.165396 s -2016-05-23 21:20:09 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:20:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 21:20:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 21:20:09 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:20:09 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:20:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 21:20:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=364085, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=369741, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:20:09 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:55028 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:20:09 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:20:09 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:09 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 21:20:09 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 21:20:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:20:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:20:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:09 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 24 ms on localhost (1/2) -2016-05-23 21:20:09 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 34 ms on localhost (2/2) -2016-05-23 21:20:09 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.034 s -2016-05-23 21:20:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 21:20:09 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.060278 s -2016-05-23 21:20:09 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:20:09 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:20:09 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:20:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 21:20:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=372649, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=378345, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:20:09 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:55028 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:20:09 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:20:09 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:09 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 21:20:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:20:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:09 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 21:20:09 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 35 ms on localhost (1/2) -2016-05-23 21:20:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:20:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:09 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 86 ms on localhost (2/2) -2016-05-23 21:20:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 21:20:09 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.088 s -2016-05-23 21:20:09 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.119929 s -2016-05-23 21:20:09 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:45 -2016-05-23 21:20:09 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:45) with 4 output partitions -2016-05-23 21:20:09 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:45) -2016-05-23 21:20:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:42), which has no missing parents -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=381234, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=382458, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:20:09 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:55028 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:20:09 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:42) -2016-05-23 21:20:09 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:20:09 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 21:20:09 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 21:20:09 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 21:20:09 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 21:20:09 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 25 ms on localhost (1/4) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 12 ms on localhost (2/4) -2016-05-23 21:20:09 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 21:20:09 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 19 ms on localhost (3/4) -2016-05-23 21:20:09 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 21 ms on localhost (4/4) -2016-05-23 21:20:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 21:20:09 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:45) finished in 0.037 s -2016-05-23 21:20:09 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:45, took 0.059071 s -2016-05-23 21:20:09 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:47 -2016-05-23 21:20:09 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:47) with 4 output partitions -2016-05-23 21:20:09 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:47) -2016-05-23 21:20:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:43), which has no missing parents -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=383351, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=384575, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:20:09 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:55028 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:20:09 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:43) -2016-05-23 21:20:09 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:20:09 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 21:20:09 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 21:20:09 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 21:20:09 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 21:20:09 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 21:20:09 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 12 ms on localhost (1/4) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 15 ms on localhost (2/4) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 14 ms on localhost (3/4) -2016-05-23 21:20:09 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 21:20:09 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 20 ms on localhost (4/4) -2016-05-23 21:20:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 21:20:09 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:47) finished in 0.025 s -2016-05-23 21:20:09 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:47, took 0.039434 s -2016-05-23 21:20:09 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:20:09 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:20:09 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 21:20:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=385468, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=387972, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 21:20:09 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:55028 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:20:09 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 21:20:09 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:20:09 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 21:20:09 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 9 ms on localhost (1/1) -2016-05-23 21:20:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 21:20:09 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.010 s -2016-05-23 21:20:09 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.027351 s -2016-05-23 21:20:09 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:20:09 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:20:09 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:20:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=389496, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=392896, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 21:20:09 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:55028 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 21:20:09 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:20:09 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:20:09 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 21:20:09 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 21:20:09 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 21:20:09 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 21:20:09 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 21:20:09 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 21:20:09 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 23 ms on localhost (1/4) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 25 ms on localhost (2/4) -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 31 ms on localhost (3/4) -2016-05-23 21:20:09 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 21:20:09 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.040 s -2016-05-23 21:20:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 38 ms on localhost (4/4) -2016-05-23 21:20:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 21:20:09 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.058672 s -2016-05-23 21:20:09 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:20:09 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:20:09 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:20:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:09 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=394830, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:20:09 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=398270, maxMem=766075207 -2016-05-23 21:20:09 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 21:20:09 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:55028 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:20:09 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:10 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:20:10 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:20:10 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 21:20:10 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 21:20:10 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 21:20:10 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 21:20:10 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 21:20:10 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 17 ms on localhost (1/4) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 14 ms on localhost (2/4) -2016-05-23 21:20:10 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 22 ms on localhost (3/4) -2016-05-23 21:20:10 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 22 ms on localhost (4/4) -2016-05-23 21:20:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 21:20:10 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.027 s -2016-05-23 21:20:10 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.048087 s -2016-05-23 21:20:10 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:20:10 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:20:10 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:20:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:10 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:20:10 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=400188, maxMem=766075207 -2016-05-23 21:20:10 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:20:10 INFO MemoryStore:59 - ensureFreeSpace(1860) called with curMem=403564, maxMem=766075207 -2016-05-23 21:20:10 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1860.0 B, free 730.2 MB) -2016-05-23 21:20:10 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:55028 (size: 1860.0 B, free: 730.5 MB) -2016-05-23 21:20:10 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:10 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:20:10 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:20:10 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 21:20:10 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 21:20:10 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 33 ms on localhost (1/4) -2016-05-23 21:20:10 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 21:20:10 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 21:20:10 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 21:20:10 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 21:20:10 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 60 ms on localhost (2/4) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 64 ms on localhost (3/4) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 64 ms on localhost (4/4) -2016-05-23 21:20:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 21:20:10 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.067 s -2016-05-23 21:20:10 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.090060 s -2016-05-23 21:20:10 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:20:10 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:20:10 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:20:10 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:20:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 21:20:10 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 21:20:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:20:10 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=405424, maxMem=766075207 -2016-05-23 21:20:10 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 21:20:10 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=409464, maxMem=766075207 -2016-05-23 21:20:10 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:20:10 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:55028 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:20:10 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:10 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:20:10 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 21:20:10 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 21:20:10 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 21:20:10 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 21:20:10 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 21:20:10 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 66 ms on localhost (1/4) -2016-05-23 21:20:10 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 67 ms on localhost (2/4) -2016-05-23 21:20:10 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 73 ms on localhost (3/4) -2016-05-23 21:20:10 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 118 ms on localhost (4/4) -2016-05-23 21:20:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 21:20:10 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.121 s -2016-05-23 21:20:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:20:10 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:20:10 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 21:20:10 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:20:10 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 21:20:10 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:20:10 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=411746, maxMem=766075207 -2016-05-23 21:20:10 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:20:10 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=414290, maxMem=766075207 -2016-05-23 21:20:10 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 21:20:10 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:55028 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 21:20:10 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:10 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:20:10 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:10 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 21:20:10 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:10 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 22 ms on localhost (1/3) -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:10 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 42 ms on localhost (2/3) -2016-05-23 21:20:10 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:10 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 64 ms on localhost (3/3) -2016-05-23 21:20:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 21:20:10 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.067 s -2016-05-23 21:20:10 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.249266 s -2016-05-23 21:20:10 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:20:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 21:20:10 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:20:10 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:20:10 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 21:20:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 21:20:10 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 21:20:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:20:10 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=415858, maxMem=766075207 -2016-05-23 21:20:10 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:20:10 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=420090, maxMem=766075207 -2016-05-23 21:20:10 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 21:20:10 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:55028 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:20:10 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:10 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:20:10 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:20:10 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 21:20:10 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 21:20:10 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 21:20:10 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:10 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 21:20:10 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 21:20:10 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 154 ms on localhost (1/4) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 150 ms on localhost (2/4) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 155 ms on localhost (3/4) -2016-05-23 21:20:10 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 223 ms on localhost (4/4) -2016-05-23 21:20:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 21:20:10 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.227 s -2016-05-23 21:20:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:20:10 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:20:10 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 21:20:10 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:20:10 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 21:20:10 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:20:10 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=422396, maxMem=766075207 -2016-05-23 21:20:10 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:20:10 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=427292, maxMem=766075207 -2016-05-23 21:20:10 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:20:10 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:55028 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:20:10 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:10 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:20:10 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:10 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:10 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 20 ms on localhost (1/1) -2016-05-23 21:20:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 21:20:10 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.022 s -2016-05-23 21:20:10 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.307957 s -2016-05-23 21:20:10 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:20:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 21:20:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 21:20:10 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:20:10 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:20:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 21:20:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:10 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:20:10 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=429834, maxMem=766075207 -2016-05-23 21:20:10 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:20:10 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=435530, maxMem=766075207 -2016-05-23 21:20:10 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:20:10 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:55028 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:20:10 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:10 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:20:10 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:10 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 21:20:10 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:10 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:10 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 21:20:10 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:10 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 21:20:10 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:10 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 30 ms on localhost (1/4) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 95 ms on localhost (2/4) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 97 ms on localhost (3/4) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 96 ms on localhost (4/4) -2016-05-23 21:20:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 21:20:10 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.099 s -2016-05-23 21:20:10 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.118792 s -2016-05-23 21:20:10 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:20:10 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:20:10 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:20:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 21:20:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:10 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:20:10 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=438439, maxMem=766075207 -2016-05-23 21:20:10 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:20:10 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=444175, maxMem=766075207 -2016-05-23 21:20:10 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:20:10 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:55028 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:20:10 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:10 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:20:10 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:10 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 21:20:10 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:10 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 21:20:10 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:10 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 32 ms on localhost (1/4) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 39 ms on localhost (2/4) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 41 ms on localhost (3/4) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 46 ms on localhost (4/4) -2016-05-23 21:20:11 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.048 s -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 21:20:11 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.071612 s -2016-05-23 21:20:11 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:59 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Got job 20 (foreach at StatisticsCorrLearning.scala:59) with 4 output partitions -2016-05-23 21:20:11 INFO DAGScheduler:59 - Final stage: ResultStage 34(foreach at StatisticsCorrLearning.scala:59) -2016-05-23 21:20:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:56), which has no missing parents -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=447067, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=448291, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:20:11 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:55028 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:20:11 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:56) -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 62, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 63, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 64, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:20:11 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 63) -2016-05-23 21:20:11 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 65) -2016-05-23 21:20:11 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 64) -2016-05-23 21:20:11 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 62) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 64). 915 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 63). 915 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 65). 915 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 62). 915 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 62) in 15 ms on localhost (1/4) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 63) in 15 ms on localhost (2/4) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 65) in 14 ms on localhost (3/4) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 64) in 16 ms on localhost (4/4) -2016-05-23 21:20:11 INFO DAGScheduler:59 - ResultStage 34 (foreach at StatisticsCorrLearning.scala:59) finished in 0.018 s -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-23 21:20:11 INFO DAGScheduler:59 - Job 20 finished: foreach at StatisticsCorrLearning.scala:59, took 0.028649 s -2016-05-23 21:20:11 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:61 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Got job 21 (foreach at StatisticsCorrLearning.scala:61) with 4 output partitions -2016-05-23 21:20:11 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at StatisticsCorrLearning.scala:61) -2016-05-23 21:20:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:57), which has no missing parents -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=449184, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=450408, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:20:11 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:55028 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:20:11 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:57) -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 4 tasks -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 66, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 3.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:20:11 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 67) -2016-05-23 21:20:11 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 68) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 68). 915 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Running task 3.0 in stage 35.0 (TID 69) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 67). 915 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 66) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 3.0 in stage 35.0 (TID 69). 915 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 68) in 10 ms on localhost (1/4) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 66). 915 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 67) in 12 ms on localhost (2/4) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 66) in 14 ms on localhost (3/4) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 3.0 in stage 35.0 (TID 69) in 12 ms on localhost (4/4) -2016-05-23 21:20:11 INFO DAGScheduler:59 - ResultStage 35 (foreach at StatisticsCorrLearning.scala:61) finished in 0.016 s -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-23 21:20:11 INFO DAGScheduler:59 - Job 21 finished: foreach at StatisticsCorrLearning.scala:61, took 0.030038 s -2016-05-23 21:20:11 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Got job 22 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:20:11 INFO DAGScheduler:59 - Final stage: ResultStage 36(first at RowMatrix.scala:65) -2016-05-23 21:20:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=451301, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(1522) called with curMem=453805, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1522.0 B, free 730.2 MB) -2016-05-23 21:20:11 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:55028 (size: 1522.0 B, free: 730.5 MB) -2016-05-23 21:20:11 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:11 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 70) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 70). 1046 bytes result sent to driver -2016-05-23 21:20:11 INFO DAGScheduler:59 - ResultStage 36 (first at RowMatrix.scala:65) finished in 0.006 s -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 70) in 4 ms on localhost (1/1) -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-23 21:20:11 INFO DAGScheduler:59 - Job 22 finished: first at RowMatrix.scala:65, took 0.019072 s -2016-05-23 21:20:11 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Got job 23 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:20:11 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:20:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=455327, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(1932) called with curMem=458727, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1932.0 B, free 730.1 MB) -2016-05-23 21:20:11 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:55028 (size: 1932.0 B, free: 730.5 MB) -2016-05-23 21:20:11 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 4 tasks -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 3.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:20:11 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 72) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 72). 1231 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 73) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 73). 1231 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Running task 3.0 in stage 37.0 (TID 74) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 73) in 15 ms on localhost (1/4) -2016-05-23 21:20:11 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 3.0 in stage 37.0 (TID 74). 1231 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 1231 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 72) in 24 ms on localhost (2/4) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 3.0 in stage 37.0 (TID 74) in 23 ms on localhost (3/4) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 27 ms on localhost (4/4) -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-23 21:20:11 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at RowMatrix.scala:332) finished in 0.027 s -2016-05-23 21:20:11 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at RowMatrix.scala:332, took 0.041095 s -2016-05-23 21:20:11 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Got job 24 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:20:11 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:20:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=460659, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(1916) called with curMem=464099, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1916.0 B, free 730.1 MB) -2016-05-23 21:20:11 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:55028 (size: 1916.0 B, free: 730.5 MB) -2016-05-23 21:20:11 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 77, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 78, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:20:11 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 77) -2016-05-23 21:20:11 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 77). 1109 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 77) in 8 ms on localhost (1/4) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 1109 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 1109 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 78) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 78). 1109 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 37 ms on localhost (2/4) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 78) in 35 ms on localhost (3/4) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 47 ms on localhost (4/4) -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-23 21:20:11 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at RowMatrix.scala:124) finished in 0.051 s -2016-05-23 21:20:11 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at RowMatrix.scala:124, took 0.069055 s -2016-05-23 21:20:11 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Got job 25 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:20:11 INFO DAGScheduler:59 - Final stage: ResultStage 39(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:20:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=466015, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(1851) called with curMem=469391, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1851.0 B, free 730.1 MB) -2016-05-23 21:20:11 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:55028 (size: 1851.0 B, free: 730.5 MB) -2016-05-23 21:20:11 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 4 tasks -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 79, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 80, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 81, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 3.0 in stage 39.0 (TID 82, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:20:11 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 81) -2016-05-23 21:20:11 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 80) -2016-05-23 21:20:11 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 79) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 79). 1254 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 81). 1249 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 80). 1254 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Running task 3.0 in stage 39.0 (TID 82) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 3.0 in stage 39.0 (TID 82). 1294 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 79) in 37 ms on localhost (1/4) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 81) in 29 ms on localhost (2/4) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 80) in 31 ms on localhost (3/4) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 3.0 in stage 39.0 (TID 82) in 30 ms on localhost (4/4) -2016-05-23 21:20:11 INFO DAGScheduler:59 - ResultStage 39 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.040 s -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-23 21:20:11 INFO DAGScheduler:59 - Job 25 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.065989 s -2016-05-23 21:20:11 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Registering RDD 51 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:20:11 INFO DAGScheduler:59 - Got job 26 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:20:11 INFO DAGScheduler:59 - Final stage: ResultStage 41(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:20:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 40) -2016-05-23 21:20:11 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 40) -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=471242, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(2281) called with curMem=475282, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 21:20:11 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:55028 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:20:11 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 4 tasks -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 83, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 84, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 85, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 3.0 in stage 40.0 (TID 86, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:20:11 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 84) -2016-05-23 21:20:11 INFO Executor:59 - Running task 3.0 in stage 40.0 (TID 86) -2016-05-23 21:20:11 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 83) -2016-05-23 21:20:11 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 85) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 85). 1161 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 84). 1161 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Finished task 3.0 in stage 40.0 (TID 86). 1161 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 85) in 75 ms on localhost (1/4) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 3.0 in stage 40.0 (TID 86) in 76 ms on localhost (2/4) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 84) in 79 ms on localhost (3/4) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 83). 1161 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 83) in 117 ms on localhost (4/4) -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-23 21:20:11 INFO DAGScheduler:59 - ShuffleMapStage 40 (flatMap at SpearmanCorrelation.scala:49) finished in 0.118 s -2016-05-23 21:20:11 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:20:11 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:20:11 INFO DAGScheduler:59 - waiting: Set(ResultStage 41) -2016-05-23 21:20:11 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:20:11 INFO DAGScheduler:59 - Missing parents for ResultStage 41: List() -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=477563, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=480107, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.1 MB) -2016-05-23 21:20:11 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:55028 (size: 1567.0 B, free: 730.5 MB) -2016-05-23 21:20:11 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 3 tasks -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 87, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:11 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 88) -2016-05-23 21:20:11 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 89) -2016-05-23 21:20:11 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 87) -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:11 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 88). 1203 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Finished task 2.0 in stage 41.0 (TID 89). 1203 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 88) in 28 ms on localhost (1/3) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 41.0 (TID 89) in 28 ms on localhost (2/3) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 87). 1203 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 87) in 34 ms on localhost (3/3) -2016-05-23 21:20:11 INFO DAGScheduler:59 - ResultStage 41 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.035 s -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-23 21:20:11 INFO DAGScheduler:59 - Job 26 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.188394 s -2016-05-23 21:20:11 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:20:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 163 bytes -2016-05-23 21:20:11 INFO DAGScheduler:59 - Registering RDD 56 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:20:11 INFO DAGScheduler:59 - Got job 27 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:20:11 INFO DAGScheduler:59 - Final stage: ResultStage 44(first at RowMatrix.scala:65) -2016-05-23 21:20:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 43) -2016-05-23 21:20:11 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 43) -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=481674, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(2305) called with curMem=485906, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-23 21:20:11 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:55028 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:20:11 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 4 tasks -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 90, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 91, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 43.0 (TID 92, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 3.0 in stage 43.0 (TID 93, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:20:11 INFO Executor:59 - Running task 2.0 in stage 43.0 (TID 92) -2016-05-23 21:20:11 INFO Executor:59 - Running task 3.0 in stage 43.0 (TID 93) -2016-05-23 21:20:11 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 91) -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:11 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 90) -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:11 INFO Executor:59 - Finished task 3.0 in stage 43.0 (TID 93). 1377 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Finished task 2.0 in stage 43.0 (TID 92). 1377 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 3.0 in stage 43.0 (TID 93) in 106 ms on localhost (1/4) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 43.0 (TID 92) in 109 ms on localhost (2/4) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 91). 1377 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 91) in 141 ms on localhost (3/4) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 90). 1377 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 90) in 164 ms on localhost (4/4) -2016-05-23 21:20:11 INFO DAGScheduler:59 - ShuffleMapStage 43 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.165 s -2016-05-23 21:20:11 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:20:11 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:20:11 INFO DAGScheduler:59 - waiting: Set(ResultStage 44) -2016-05-23 21:20:11 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-23 21:20:11 INFO DAGScheduler:59 - Missing parents for ResultStage 44: List() -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=488211, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(2545) called with curMem=493107, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:20:11 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:55028 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:20:11 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 1 tasks -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 94, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:11 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 94) -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:11 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 94). 1296 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 94) in 13 ms on localhost (1/1) -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-23 21:20:11 INFO DAGScheduler:59 - ResultStage 44 (first at RowMatrix.scala:65) finished in 0.013 s -2016-05-23 21:20:11 INFO DAGScheduler:59 - Job 27 finished: first at RowMatrix.scala:65, took 0.213463 s -2016-05-23 21:20:11 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:20:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 163 bytes -2016-05-23 21:20:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 169 bytes -2016-05-23 21:20:11 INFO DAGScheduler:59 - Got job 28 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:20:11 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:20:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 46) -2016-05-23 21:20:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=495652, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(2914) called with curMem=501348, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 21:20:11 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:55028 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:20:11 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 4 tasks -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 95, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 96, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 47.0 (TID 97, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 3.0 in stage 47.0 (TID 98, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:11 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 96) -2016-05-23 21:20:11 INFO Executor:59 - Running task 2.0 in stage 47.0 (TID 97) -2016-05-23 21:20:11 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 95) -2016-05-23 21:20:11 INFO Executor:59 - Running task 3.0 in stage 47.0 (TID 98) -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:11 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 96). 1481 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Finished task 3.0 in stage 47.0 (TID 98). 1481 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 96) in 19 ms on localhost (1/4) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 3.0 in stage 47.0 (TID 98) in 20 ms on localhost (2/4) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 95). 1481 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 95) in 24 ms on localhost (3/4) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 2.0 in stage 47.0 (TID 97). 1481 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 47.0 (TID 97) in 24 ms on localhost (4/4) -2016-05-23 21:20:11 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at RowMatrix.scala:332) finished in 0.027 s -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-23 21:20:11 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at RowMatrix.scala:332, took 0.050339 s -2016-05-23 21:20:11 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Got job 29 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:20:11 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:20:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 49) -2016-05-23 21:20:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=504262, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=509998, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 21:20:11 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:55028 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:20:11 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 4 tasks -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 50.0 (TID 101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 3.0 in stage 50.0 (TID 102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:11 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-23 21:20:11 INFO Executor:59 - Running task 2.0 in stage 50.0 (TID 101) -2016-05-23 21:20:11 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-23 21:20:11 INFO Executor:59 - Running task 3.0 in stage 50.0 (TID 102) -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:11 INFO Executor:59 - Finished task 2.0 in stage 50.0 (TID 101). 1359 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Finished task 3.0 in stage 50.0 (TID 102). 1359 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 1359 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 50.0 (TID 101) in 15 ms on localhost (1/4) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 3.0 in stage 50.0 (TID 102) in 16 ms on localhost (2/4) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 19 ms on localhost (3/4) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 1359 bytes result sent to driver -2016-05-23 21:20:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 21 ms on localhost (4/4) -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-23 21:20:11 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at RowMatrix.scala:124) finished in 0.021 s -2016-05-23 21:20:11 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at RowMatrix.scala:124, took 0.034717 s -2016-05-23 21:20:11 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:76 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Got job 30 (foreach at StatisticsCorrLearning.scala:76) with 4 output partitions -2016-05-23 21:20:11 INFO DAGScheduler:59 - Final stage: ResultStage 51(foreach at StatisticsCorrLearning.scala:76) -2016-05-23 21:20:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting ResultStage 51 (ParallelCollectionRDD[61] at parallelize at StatisticsCorrLearning.scala:73), which has no missing parents -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=512894, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-05-23 21:20:11 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=514118, maxMem=766075207 -2016-05-23 21:20:11 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.1 MB) -2016-05-23 21:20:11 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:55028 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:20:11 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:11 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 51 (ParallelCollectionRDD[61] at parallelize at StatisticsCorrLearning.scala:73) -2016-05-23 21:20:11 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 4 tasks -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 103, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 21:20:11 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:55028 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 104, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 51.0 (TID 105, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:20:11 INFO TaskSetManager:59 - Starting task 3.0 in stage 51.0 (TID 106, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:20:11 INFO Executor:59 - Running task 3.0 in stage 51.0 (TID 106) -2016-05-23 21:20:11 INFO Executor:59 - Running task 2.0 in stage 51.0 (TID 105) -2016-05-23 21:20:11 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 103) -2016-05-23 21:20:11 INFO Executor:59 - Finished task 3.0 in stage 51.0 (TID 106). 915 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Finished task 2.0 in stage 51.0 (TID 105). 915 bytes result sent to driver -2016-05-23 21:20:11 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 104) -2016-05-23 21:20:11 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-23 21:20:11 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 103). 915 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 3.0 in stage 51.0 (TID 106) in 7 ms on localhost (1/4) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 104). 915 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 51.0 (TID 105) in 9 ms on localhost (2/4) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 104) in 12 ms on localhost (3/4) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 103) in 13 ms on localhost (4/4) -2016-05-23 21:20:12 INFO DAGScheduler:59 - ResultStage 51 (foreach at StatisticsCorrLearning.scala:76) finished in 0.014 s -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-23 21:20:12 INFO DAGScheduler:59 - Job 30 finished: foreach at StatisticsCorrLearning.scala:76, took 0.050770 s -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:55028 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:55028 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-23 21:20:12 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:78 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Got job 31 (foreach at StatisticsCorrLearning.scala:78) with 4 output partitions -2016-05-23 21:20:12 INFO DAGScheduler:59 - Final stage: ResultStage 52(foreach at StatisticsCorrLearning.scala:78) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting ResultStage 52 (ParallelCollectionRDD[62] at parallelize at StatisticsCorrLearning.scala:74), which has no missing parents -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:55028 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=483791, maxMem=766075207 -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=485015, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.1 MB) -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned shuffle 5 -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:55028 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 52 (ParallelCollectionRDD[62] at parallelize at StatisticsCorrLearning.scala:74) -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 4 tasks -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 107, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 108, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 52.0 (TID 109, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 3.0 in stage 52.0 (TID 110, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:55028 in memory (size: 1567.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO Executor:59 - Running task 3.0 in stage 52.0 (TID 110) -2016-05-23 21:20:12 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 107) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 3.0 in stage 52.0 (TID 110). 915 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 3.0 in stage 52.0 (TID 110) in 7 ms on localhost (1/4) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 107). 915 bytes result sent to driver -2016-05-23 21:20:12 INFO Executor:59 - Running task 2.0 in stage 52.0 (TID 109) -2016-05-23 21:20:12 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 108) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 108). 915 bytes result sent to driver -2016-05-23 21:20:12 INFO Executor:59 - Finished task 2.0 in stage 52.0 (TID 109). 915 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 107) in 19 ms on localhost (2/4) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 108) in 19 ms on localhost (3/4) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 52.0 (TID 109) in 19 ms on localhost (4/4) -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-23 21:20:12 INFO DAGScheduler:59 - ResultStage 52 (foreach at StatisticsCorrLearning.scala:78) finished in 0.026 s -2016-05-23 21:20:12 INFO DAGScheduler:59 - Job 31 finished: foreach at StatisticsCorrLearning.scala:78, took 0.036781 s -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:55028 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned shuffle 4 -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:55028 in memory (size: 1851.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-23 21:20:12 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Got job 32 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:20:12 INFO DAGScheduler:59 - Final stage: ResultStage 53(first at RowMatrix.scala:65) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[64] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=468333, maxMem=766075207 -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:55028 in memory (size: 1916.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=467397, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.1 MB) -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:55028 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:55028 in memory (size: 1932.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 53 (MapPartitionsRDD[64] at map at Correlation.scala:45) -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 1 tasks -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 111, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:20:12 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 111) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 111). 935 bytes result sent to driver -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:55028 in memory (size: 1522.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO DAGScheduler:59 - ResultStage 53 (first at RowMatrix.scala:65) finished in 0.006 s -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 111) in 6 ms on localhost (1/1) -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-23 21:20:12 INFO DAGScheduler:59 - Job 32 finished: first at RowMatrix.scala:65, took 0.019982 s -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:55028 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-23 21:20:12 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:55028 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Got job 33 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 21:20:12 INFO DAGScheduler:59 - Final stage: ResultStage 54(first at RowMatrix.scala:65) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[64] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:55028 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=446701, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=449205, maxMem=766075207 -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:55028 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 54 (MapPartitionsRDD[64] at map at Correlation.scala:45) -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 3 tasks -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:55028 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 112, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 113, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 54.0 (TID 114, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:12 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 112) -2016-05-23 21:20:12 INFO Executor:59 - Running task 2.0 in stage 54.0 (TID 114) -2016-05-23 21:20:12 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 113) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 2.0 in stage 54.0 (TID 114). 1046 bytes result sent to driver -2016-05-23 21:20:12 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 113). 1046 bytes result sent to driver -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:55028 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 112). 1046 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 113) in 6 ms on localhost (1/3) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 54.0 (TID 114) in 5 ms on localhost (2/3) -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 112) in 7 ms on localhost (3/3) -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-23 21:20:12 INFO DAGScheduler:59 - ResultStage 54 (first at RowMatrix.scala:65) finished in 0.008 s -2016-05-23 21:20:12 INFO DAGScheduler:59 - Job 33 finished: first at RowMatrix.scala:65, took 0.021291 s -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:55028 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:55028 in memory (size: 1568.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-23 21:20:12 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Got job 34 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:20:12 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[65] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=417714, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=421114, maxMem=766075207 -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:55028 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:55028 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 55 (MapPartitionsRDD[65] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 4 tasks -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 115, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 116, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 55.0 (TID 117, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 3.0 in stage 55.0 (TID 118, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:12 INFO Executor:59 - Running task 2.0 in stage 55.0 (TID 117) -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-05-23 21:20:12 INFO Executor:59 - Running task 3.0 in stage 55.0 (TID 118) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 3.0 in stage 55.0 (TID 118). 1231 bytes result sent to driver -2016-05-23 21:20:12 INFO Executor:59 - Finished task 2.0 in stage 55.0 (TID 117). 1231 bytes result sent to driver -2016-05-23 21:20:12 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 115) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 3.0 in stage 55.0 (TID 118) in 10 ms on localhost (1/4) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 115). 1231 bytes result sent to driver -2016-05-23 21:20:12 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 116) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 55.0 (TID 117) in 20 ms on localhost (2/4) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 115) in 24 ms on localhost (3/4) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 116). 1231 bytes result sent to driver -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:55028 in memory (size: 1860.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 116) in 27 ms on localhost (4/4) -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-23 21:20:12 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at RowMatrix.scala:332) finished in 0.030 s -2016-05-23 21:20:12 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at RowMatrix.scala:332, took 0.049270 s -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:55028 in memory (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Got job 35 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:20:12 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[66] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:55028 in memory (size: 1934.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=407120, maxMem=766075207 -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=409036, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:55028 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:55028 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-23 21:20:12 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 56 (MapPartitionsRDD[66] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 4 tasks -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 119, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:55028 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 120, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 56.0 (TID 121, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 3.0 in stage 56.0 (TID 122, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:12 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 120) -2016-05-23 21:20:12 INFO Executor:59 - Running task 3.0 in stage 56.0 (TID 122) -2016-05-23 21:20:12 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 119) -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:55028 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 120). 1109 bytes result sent to driver -2016-05-23 21:20:12 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 119). 1109 bytes result sent to driver -2016-05-23 21:20:12 INFO Executor:59 - Running task 2.0 in stage 56.0 (TID 121) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 3.0 in stage 56.0 (TID 122). 1109 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 120) in 8 ms on localhost (1/4) -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 119) in 11 ms on localhost (2/4) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 2.0 in stage 56.0 (TID 121). 1109 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 3.0 in stage 56.0 (TID 122) in 10 ms on localhost (3/4) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 56.0 (TID 121) in 12 ms on localhost (4/4) -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-23 21:20:12 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at RowMatrix.scala:124) finished in 0.015 s -2016-05-23 21:20:12 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at RowMatrix.scala:124, took 0.029179 s -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:55028 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:55028 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:55028 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:55028 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-23 21:20:12 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-23 21:20:12 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Got job 36 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:20:12 INFO DAGScheduler:59 - Final stage: ResultStage 57(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:20:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[72] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=373206, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(1858) called with curMem=376582, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 1858.0 B, free 730.2 MB) -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:55028 (size: 1858.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 57 (MapPartitionsRDD[72] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 4 tasks -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 123, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 124, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 57.0 (TID 125, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 3.0 in stage 57.0 (TID 126, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:20:12 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 123) -2016-05-23 21:20:12 INFO Executor:59 - Running task 2.0 in stage 57.0 (TID 125) -2016-05-23 21:20:12 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 124) -2016-05-23 21:20:12 INFO Executor:59 - Running task 3.0 in stage 57.0 (TID 126) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 123). 1104 bytes result sent to driver -2016-05-23 21:20:12 INFO Executor:59 - Finished task 2.0 in stage 57.0 (TID 125). 1249 bytes result sent to driver -2016-05-23 21:20:12 INFO Executor:59 - Finished task 3.0 in stage 57.0 (TID 126). 1254 bytes result sent to driver -2016-05-23 21:20:12 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 124). 1254 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 123) in 7 ms on localhost (1/4) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 57.0 (TID 125) in 6 ms on localhost (2/4) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 3.0 in stage 57.0 (TID 126) in 7 ms on localhost (3/4) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 124) in 8 ms on localhost (4/4) -2016-05-23 21:20:12 INFO DAGScheduler:59 - ResultStage 57 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.010 s -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-23 21:20:12 INFO DAGScheduler:59 - Job 36 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.018803 s -2016-05-23 21:20:12 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Registering RDD 70 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Got job 37 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:20:12 INFO DAGScheduler:59 - Final stage: ResultStage 59(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 58) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 58) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting ShuffleMapStage 58 (MapPartitionsRDD[70] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=378440, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(2285) called with curMem=382480, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:55028 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:20:12 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 58 (MapPartitionsRDD[70] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 4 tasks -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 127, localhost, PROCESS_LOCAL, 2221 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 128, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 58.0 (TID 129, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 3.0 in stage 58.0 (TID 130, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:20:12 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 128) -2016-05-23 21:20:12 INFO Executor:59 - Running task 3.0 in stage 58.0 (TID 130) -2016-05-23 21:20:12 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 127) -2016-05-23 21:20:12 INFO Executor:59 - Running task 2.0 in stage 58.0 (TID 129) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 127). 1161 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 127) in 23 ms on localhost (1/4) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 3.0 in stage 58.0 (TID 130). 1161 bytes result sent to driver -2016-05-23 21:20:12 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 128). 1161 bytes result sent to driver -2016-05-23 21:20:12 INFO Executor:59 - Finished task 2.0 in stage 58.0 (TID 129). 1161 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 3.0 in stage 58.0 (TID 130) in 48 ms on localhost (2/4) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 128) in 50 ms on localhost (3/4) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 58.0 (TID 129) in 51 ms on localhost (4/4) -2016-05-23 21:20:12 INFO DAGScheduler:59 - ShuffleMapStage 58 (flatMap at SpearmanCorrelation.scala:49) finished in 0.053 s -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-05-23 21:20:12 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:20:12 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:20:12 INFO DAGScheduler:59 - waiting: Set(ResultStage 59) -2016-05-23 21:20:12 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:20:12 INFO DAGScheduler:59 - Missing parents for ResultStage 59: List() -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting ResultStage 59 (ShuffledRDD[73] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=384765, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=387309, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.2 MB) -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:55028 (size: 1567.0 B, free: 730.5 MB) -2016-05-23 21:20:12 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 59 (ShuffledRDD[73] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 3 tasks -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 131, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 132, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 59.0 (TID 133, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:12 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 131) -2016-05-23 21:20:12 INFO Executor:59 - Running task 2.0 in stage 59.0 (TID 133) -2016-05-23 21:20:12 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 132) -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:12 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 131). 1203 bytes result sent to driver -2016-05-23 21:20:12 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 132). 1203 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 131) in 11 ms on localhost (1/3) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 2.0 in stage 59.0 (TID 133). 1203 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 132) in 10 ms on localhost (2/3) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 59.0 (TID 133) in 11 ms on localhost (3/3) -2016-05-23 21:20:12 INFO DAGScheduler:59 - ResultStage 59 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.012 s -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-05-23 21:20:12 INFO DAGScheduler:59 - Job 37 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.082994 s -2016-05-23 21:20:12 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:20:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 168 bytes -2016-05-23 21:20:12 INFO DAGScheduler:59 - Registering RDD 75 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Got job 38 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:20:12 INFO DAGScheduler:59 - Final stage: ResultStage 62(first at RowMatrix.scala:65) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 61) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 61) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting ShuffleMapStage 61 (MapPartitionsRDD[75] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=388876, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(2305) called with curMem=393108, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:55028 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:20:12 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 61 (MapPartitionsRDD[75] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 4 tasks -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 134, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 135, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 61.0 (TID 136, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 3.0 in stage 61.0 (TID 137, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:20:12 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 134) -2016-05-23 21:20:12 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 135) -2016-05-23 21:20:12 INFO Executor:59 - Running task 2.0 in stage 61.0 (TID 136) -2016-05-23 21:20:12 INFO Executor:59 - Running task 3.0 in stage 61.0 (TID 137) -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 3 ms -2016-05-23 21:20:12 INFO Executor:59 - Finished task 3.0 in stage 61.0 (TID 137). 1377 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 3.0 in stage 61.0 (TID 137) in 57 ms on localhost (1/4) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 2.0 in stage 61.0 (TID 136). 1377 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 61.0 (TID 136) in 63 ms on localhost (2/4) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 135). 1377 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 135) in 66 ms on localhost (3/4) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 134). 1377 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 134) in 69 ms on localhost (4/4) -2016-05-23 21:20:12 INFO DAGScheduler:59 - ShuffleMapStage 61 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.069 s -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-05-23 21:20:12 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:20:12 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:20:12 INFO DAGScheduler:59 - waiting: Set(ResultStage 62) -2016-05-23 21:20:12 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:20:12 INFO DAGScheduler:59 - Missing parents for ResultStage 62: List() -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=395413, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=400309, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:55028 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:20:12 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 62 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 1 tasks -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 138, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:12 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 138) -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:12 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 138). 1185 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 138) in 10 ms on localhost (1/1) -2016-05-23 21:20:12 INFO DAGScheduler:59 - ResultStage 62 (first at RowMatrix.scala:65) finished in 0.010 s -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-05-23 21:20:12 INFO DAGScheduler:59 - Job 38 finished: first at RowMatrix.scala:65, took 0.103545 s -2016-05-23 21:20:12 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:20:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 168 bytes -2016-05-23 21:20:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 165 bytes -2016-05-23 21:20:12 INFO DAGScheduler:59 - Got job 39 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 21:20:12 INFO DAGScheduler:59 - Final stage: ResultStage 65(first at RowMatrix.scala:65) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 64) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87), which has no missing parents -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=402853, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=407749, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:55028 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:20:12 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 65 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 3 tasks -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 139, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 140, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 65.0 (TID 141, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:12 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 139) -2016-05-23 21:20:12 INFO Executor:59 - Running task 2.0 in stage 65.0 (TID 141) -2016-05-23 21:20:12 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 140) -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:12 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 139). 1296 bytes result sent to driver -2016-05-23 21:20:12 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 140). 1296 bytes result sent to driver -2016-05-23 21:20:12 INFO Executor:59 - Finished task 2.0 in stage 65.0 (TID 141). 1296 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 139) in 18 ms on localhost (1/3) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 140) in 18 ms on localhost (2/3) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 65.0 (TID 141) in 19 ms on localhost (3/3) -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-05-23 21:20:12 INFO DAGScheduler:59 - ResultStage 65 (first at RowMatrix.scala:65) finished in 0.020 s -2016-05-23 21:20:12 INFO DAGScheduler:59 - Job 39 finished: first at RowMatrix.scala:65, took 0.038937 s -2016-05-23 21:20:12 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Got job 40 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:20:12 INFO DAGScheduler:59 - Final stage: ResultStage 68(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 67) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting ResultStage 68 (MapPartitionsRDD[78] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=410293, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(2914) called with curMem=415989, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:55028 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:20:12 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 68 (MapPartitionsRDD[78] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Adding task set 68.0 with 4 tasks -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 68.0 (TID 142, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 68.0 (TID 143, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 68.0 (TID 144, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 3.0 in stage 68.0 (TID 145, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:12 INFO Executor:59 - Running task 3.0 in stage 68.0 (TID 145) -2016-05-23 21:20:12 INFO Executor:59 - Running task 1.0 in stage 68.0 (TID 143) -2016-05-23 21:20:12 INFO Executor:59 - Running task 0.0 in stage 68.0 (TID 142) -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:12 INFO Executor:59 - Running task 2.0 in stage 68.0 (TID 144) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 3.0 in stage 68.0 (TID 145). 1481 bytes result sent to driver -2016-05-23 21:20:12 INFO Executor:59 - Finished task 1.0 in stage 68.0 (TID 143). 1481 bytes result sent to driver -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:12 INFO Executor:59 - Finished task 2.0 in stage 68.0 (TID 144). 1481 bytes result sent to driver -2016-05-23 21:20:12 INFO Executor:59 - Finished task 0.0 in stage 68.0 (TID 142). 1481 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 68.0 (TID 143) in 44 ms on localhost (1/4) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 68.0 (TID 144) in 44 ms on localhost (2/4) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 3.0 in stage 68.0 (TID 145) in 44 ms on localhost (3/4) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 68.0 (TID 142) in 48 ms on localhost (4/4) -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 68.0, whose tasks have all completed, from pool -2016-05-23 21:20:12 INFO DAGScheduler:59 - ResultStage 68 (treeAggregate at RowMatrix.scala:332) finished in 0.049 s -2016-05-23 21:20:12 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at RowMatrix.scala:332, took 0.061539 s -2016-05-23 21:20:12 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Got job 41 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:20:12 INFO DAGScheduler:59 - Final stage: ResultStage 71(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 70) -2016-05-23 21:20:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting ResultStage 71 (MapPartitionsRDD[79] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=418903, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:20:12 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=424639, maxMem=766075207 -2016-05-23 21:20:12 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:20:12 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:55028 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:20:12 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 71 (MapPartitionsRDD[79] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Adding task set 71.0 with 4 tasks -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 71.0 (TID 146, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 71.0 (TID 147, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 71.0 (TID 148, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Starting task 3.0 in stage 71.0 (TID 149, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:20:12 INFO Executor:59 - Running task 1.0 in stage 71.0 (TID 147) -2016-05-23 21:20:12 INFO Executor:59 - Running task 3.0 in stage 71.0 (TID 149) -2016-05-23 21:20:12 INFO Executor:59 - Running task 0.0 in stage 71.0 (TID 146) -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:12 INFO Executor:59 - Finished task 3.0 in stage 71.0 (TID 149). 1359 bytes result sent to driver -2016-05-23 21:20:12 INFO Executor:59 - Running task 2.0 in stage 71.0 (TID 148) -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 3.0 in stage 71.0 (TID 149) in 16 ms on localhost (1/4) -2016-05-23 21:20:12 INFO Executor:59 - Finished task 0.0 in stage 71.0 (TID 146). 1359 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 71.0 (TID 146) in 21 ms on localhost (2/4) -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:20:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:20:12 INFO Executor:59 - Finished task 2.0 in stage 71.0 (TID 148). 1359 bytes result sent to driver -2016-05-23 21:20:12 INFO Executor:59 - Finished task 1.0 in stage 71.0 (TID 147). 1359 bytes result sent to driver -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 71.0 (TID 147) in 34 ms on localhost (3/4) -2016-05-23 21:20:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 71.0 (TID 148) in 34 ms on localhost (4/4) -2016-05-23 21:20:12 INFO DAGScheduler:59 - ResultStage 71 (treeAggregate at RowMatrix.scala:124) finished in 0.037 s -2016-05-23 21:20:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 71.0, whose tasks have all completed, from pool -2016-05-23 21:20:12 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at RowMatrix.scala:124, took 0.053571 s -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 21:20:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 21:20:12 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 21:20:12 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 21:20:12 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 21:20:13 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 21:20:13 INFO BlockManager:59 - BlockManager stopped -2016-05-23 21:20:13 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 21:20:13 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 21:20:13 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 21:20:13 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 21:20:13 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-a063dbf1-97f1-4f96-b27d-c9d7db857943 -2016-05-23 21:20:13 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 21:20:13 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 21:24:10 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 21:24:11 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 21:24:11 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 21:24:11 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 21:24:11 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 21:24:12 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 21:24:12 INFO Remoting:74 - Starting remoting -2016-05-23 21:24:13 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:55073] -2016-05-23 21:24:13 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55073. -2016-05-23 21:24:13 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 21:24:13 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 21:24:13 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-d55b4040-187a-4ea0-816f-d05e9c2fa32e -2016-05-23 21:24:13 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 21:24:13 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-a046c892-3ef0-4588-b723-0d2db5431e12\httpd-af012ef5-6d1f-425c-b171-396ebfb19def -2016-05-23 21:24:13 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 21:24:13 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 21:24:14 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55074 -2016-05-23 21:24:14 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55074. -2016-05-23 21:24:14 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 21:24:14 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 21:24:14 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 21:24:14 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 21:24:14 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 21:24:14 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 21:24:14 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 21:24:15 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55093. -2016-05-23 21:24:15 INFO NettyBlockTransferService:59 - Server created on 55093 -2016-05-23 21:24:15 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 21:24:15 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55093 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55093) -2016-05-23 21:24:15 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 21:24:16 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 21:24:16 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 21:24:16 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 21:24:16 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 21:24:16 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55093 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 21:24:16 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:22 -2016-05-23 21:24:17 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 21:24:17 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 21:24:17 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 21:24:17 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 21:24:17 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55093 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 21:24:17 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:25 -2016-05-23 21:24:18 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 21:24:18 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 21:24:18 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 21:24:18 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 21:24:18 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 21:24:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:18 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:23), which has no missing parents -2016-05-23 21:24:18 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 21:24:18 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 21:24:18 INFO MemoryStore:59 - ensureFreeSpace(2047) called with curMem=346712, maxMem=766075207 -2016-05-23 21:24:18 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2047.0 B, free 730.3 MB) -2016-05-23 21:24:18 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55093 (size: 2047.0 B, free: 730.6 MB) -2016-05-23 21:24:18 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:23) -2016-05-23 21:24:18 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 21:24:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:24:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:24:19 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 21:24:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:24:19 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 21:24:19 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 21:24:19 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 21:24:19 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 21:24:19 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 21:24:19 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 21:24:19 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 21:24:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 599 ms on localhost (1/2) -2016-05-23 21:24:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:24:19 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 21:24:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 748 ms on localhost (2/2) -2016-05-23 21:24:19 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:29) finished in 0.851 s -2016-05-23 21:24:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 21:24:19 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:29, took 1.102897 s -2016-05-23 21:24:19 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 21:24:19 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:31 -2016-05-23 21:24:19 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:31) with 2 output partitions -2016-05-23 21:24:19 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:31) -2016-05-23 21:24:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:19 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:26), which has no missing parents -2016-05-23 21:24:19 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348759, maxMem=766075207 -2016-05-23 21:24:19 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 21:24:19 INFO MemoryStore:59 - ensureFreeSpace(2045) called with curMem=352271, maxMem=766075207 -2016-05-23 21:24:19 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2045.0 B, free 730.2 MB) -2016-05-23 21:24:19 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55093 (size: 2045.0 B, free: 730.6 MB) -2016-05-23 21:24:19 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:26) -2016-05-23 21:24:19 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 21:24:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:24:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:24:20 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 21:24:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:24:20 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 21:24:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:24:20 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 21:24:20 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 21:24:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 31 ms on localhost (1/2) -2016-05-23 21:24:20 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:31) finished in 0.038 s -2016-05-23 21:24:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 37 ms on localhost (2/2) -2016-05-23 21:24:20 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:31, took 0.067107 s -2016-05-23 21:24:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 21:24:20 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:24:20 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:24:20 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 21:24:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:20 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:24:20 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354316, maxMem=766075207 -2016-05-23 21:24:20 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 21:24:20 INFO MemoryStore:59 - ensureFreeSpace(2464) called with curMem=358876, maxMem=766075207 -2016-05-23 21:24:20 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:24:20 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55093 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 21:24:20 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 21:24:20 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 21:24:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:24:20 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 21:24:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:24:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:24:20 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 21:24:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 43 ms on localhost (1/1) -2016-05-23 21:24:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 21:24:20 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.046 s -2016-05-23 21:24:20 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.145832 s -2016-05-23 21:24:21 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:24:21 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:24:21 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:24:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:21 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:24:21 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=361340, maxMem=766075207 -2016-05-23 21:24:21 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 21:24:21 INFO MemoryStore:59 - ensureFreeSpace(2861) called with curMem=366788, maxMem=766075207 -2016-05-23 21:24:21 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:24:21 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55093 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:24:21 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:24:21 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 21:24:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:24:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:24:21 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 21:24:21 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 21:24:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:24:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:24:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:24:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:24:21 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 21:24:21 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 21:24:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 62 ms on localhost (1/2) -2016-05-23 21:24:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 66 ms on localhost (2/2) -2016-05-23 21:24:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 21:24:21 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.067 s -2016-05-23 21:24:21 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 21:24:21 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 21:24:21 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.105381 s -2016-05-23 21:24:21 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:24:21 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:24:21 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:24:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:21 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:24:21 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=369649, maxMem=766075207 -2016-05-23 21:24:21 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 21:24:21 INFO MemoryStore:59 - ensureFreeSpace(2850) called with curMem=375145, maxMem=766075207 -2016-05-23 21:24:21 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:24:21 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55093 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:24:21 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:24:21 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 21:24:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:24:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:24:21 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 21:24:21 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 21:24:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:24:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:24:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:24:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:24:21 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 21:24:21 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 21:24:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 49 ms on localhost (1/2) -2016-05-23 21:24:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 51 ms on localhost (2/2) -2016-05-23 21:24:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 21:24:21 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.055 s -2016-05-23 21:24:21 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.098604 s -2016-05-23 21:24:21 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:24:21 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 21:24:21 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:24:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:21 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:24:21 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=377995, maxMem=766075207 -2016-05-23 21:24:21 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 21:24:21 INFO MemoryStore:59 - ensureFreeSpace(2743) called with curMem=383339, maxMem=766075207 -2016-05-23 21:24:21 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 21:24:21 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55093 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 21:24:21 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:24:21 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 21:24:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:24:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:24:21 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 21:24:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:24:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:24:21 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 21:24:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 60 ms on localhost (1/2) -2016-05-23 21:24:21 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 21:24:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:24:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:24:21 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 21:24:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 98 ms on localhost (2/2) -2016-05-23 21:24:21 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.098 s -2016-05-23 21:24:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 21:24:21 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.132974 s -2016-05-23 21:24:22 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:24:22 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:24:22 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 21:24:22 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:24:22 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 21:24:22 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 21:24:22 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:24:22 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=386082, maxMem=766075207 -2016-05-23 21:24:22 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 21:24:22 INFO MemoryStore:59 - ensureFreeSpace(3155) called with curMem=392010, maxMem=766075207 -2016-05-23 21:24:22 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 21:24:22 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55093 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 21:24:22 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:24:22 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 21:24:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 21:24:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 21:24:22 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 21:24:22 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 21:24:22 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:24:22 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:24:22 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:24:22 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:24:22 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 21:24:22 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 21:24:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 97 ms on localhost (1/2) -2016-05-23 21:24:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 100 ms on localhost (2/2) -2016-05-23 21:24:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 21:24:22 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.102 s -2016-05-23 21:24:22 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:24:22 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:24:22 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 21:24:22 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:24:22 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 21:24:22 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:24:22 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=395165, maxMem=766075207 -2016-05-23 21:24:22 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:24:22 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=397669, maxMem=766075207 -2016-05-23 21:24:22 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 21:24:22 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55093 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 21:24:22 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:24:22 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 21:24:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:22 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 21:24:22 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:24:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-23 21:24:22 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 21:24:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 60 ms on localhost (1/1) -2016-05-23 21:24:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 21:24:22 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.061 s -2016-05-23 21:24:22 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.289267 s -2016-05-23 21:24:22 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:24:22 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 21:24:22 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:24:22 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:24:22 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 21:24:22 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 21:24:22 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 21:24:22 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:24:22 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=399214, maxMem=766075207 -2016-05-23 21:24:22 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:24:22 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=403406, maxMem=766075207 -2016-05-23 21:24:22 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:24:22 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55093 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:24:22 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:24:22 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 21:24:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:24:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:24:22 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 21:24:22 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 21:24:22 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:24:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:22 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:24:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 21:24:22 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 21:24:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 395 ms on localhost (1/2) -2016-05-23 21:24:22 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 21:24:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 403 ms on localhost (2/2) -2016-05-23 21:24:22 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.404 s -2016-05-23 21:24:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 21:24:22 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:24:22 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:24:22 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 21:24:22 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:24:22 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 21:24:22 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:24:22 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=405688, maxMem=766075207 -2016-05-23 21:24:22 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 21:24:22 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=410544, maxMem=766075207 -2016-05-23 21:24:22 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:24:23 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55093 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:24:23 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:24:23 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 21:24:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:23 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 21:24:23 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:24:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:24 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 21:24:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 611 ms on localhost (1/1) -2016-05-23 21:24:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 21:24:24 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.614 s -2016-05-23 21:24:24 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 1.833716 s -2016-05-23 21:24:24 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:55093 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 21:24:24 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:24:24 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 21:24:24 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 21:24:24 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 21:24:24 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:24:24 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:24:24 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 21:24:24 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:55093 in memory (size: 3.1 KB, free: 730.5 MB) -2016-05-23 21:24:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:24 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:24:24 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=399943, maxMem=766075207 -2016-05-23 21:24:24 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 21:24:24 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 21:24:24 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=400255, maxMem=766075207 -2016-05-23 21:24:24 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:24:24 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:55093 in memory (size: 2.7 KB, free: 730.5 MB) -2016-05-23 21:24:24 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:55093 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:24:24 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 21:24:24 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:24:24 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 21:24:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:24 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:55093 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:24:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:24 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 21:24:24 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 21:24:24 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 21:24:24 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:24:24 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55093 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:24:24 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:24:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:24 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 21:24:24 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 21:24:24 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 21:24:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 27 ms on localhost (1/2) -2016-05-23 21:24:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 30 ms on localhost (2/2) -2016-05-23 21:24:24 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.031 s -2016-05-23 21:24:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 21:24:24 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.050563 s -2016-05-23 21:24:24 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:24:24 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:24:24 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:24:24 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 21:24:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:24 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:24:24 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=383765, maxMem=766075207 -2016-05-23 21:24:24 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:24:24 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=389461, maxMem=766075207 -2016-05-23 21:24:24 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:24:24 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:55093 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:24:24 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:24:24 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 21:24:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:24 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 21:24:24 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 21:24:24 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:24:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:24 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:24:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:24 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 21:24:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 25 ms on localhost (1/2) -2016-05-23 21:24:24 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 21:24:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 32 ms on localhost (2/2) -2016-05-23 21:24:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 21:24:24 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.035 s -2016-05-23 21:24:24 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.056761 s -2016-05-23 21:24:25 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:45 -2016-05-23 21:24:25 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:45) with 4 output partitions -2016-05-23 21:24:25 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:45) -2016-05-23 21:24:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:25 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:42), which has no missing parents -2016-05-23 21:24:25 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=392350, maxMem=766075207 -2016-05-23 21:24:25 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:24:25 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=393574, maxMem=766075207 -2016-05-23 21:24:25 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:24:25 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:55093 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:24:25 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:42) -2016-05-23 21:24:25 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:24:25 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 21:24:25 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 21:24:25 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 21:24:25 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 21:24:25 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 21:24:25 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 21:24:25 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 21:24:25 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 21:24:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 143 ms on localhost (1/4) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 144 ms on localhost (2/4) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 226 ms on localhost (3/4) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 149 ms on localhost (4/4) -2016-05-23 21:24:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 21:24:25 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:45) finished in 0.230 s -2016-05-23 21:24:25 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:45, took 0.296486 s -2016-05-23 21:24:25 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:47 -2016-05-23 21:24:25 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:47) with 4 output partitions -2016-05-23 21:24:25 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:47) -2016-05-23 21:24:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:25 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:43), which has no missing parents -2016-05-23 21:24:25 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=394467, maxMem=766075207 -2016-05-23 21:24:25 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:24:25 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=395691, maxMem=766075207 -2016-05-23 21:24:25 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:24:25 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:55093 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:24:25 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:43) -2016-05-23 21:24:25 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:24:25 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 21:24:25 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 21:24:25 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 21:24:25 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 21:24:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 14 ms on localhost (1/4) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 11 ms on localhost (2/4) -2016-05-23 21:24:25 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 21:24:25 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 21:24:25 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 21:24:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 18 ms on localhost (3/4) -2016-05-23 21:24:25 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 21:24:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 25 ms on localhost (4/4) -2016-05-23 21:24:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 21:24:25 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:47) finished in 0.029 s -2016-05-23 21:24:25 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:47, took 0.045903 s -2016-05-23 21:24:25 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:24:25 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:24:25 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 21:24:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:25 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:24:25 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=396584, maxMem=766075207 -2016-05-23 21:24:25 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:24:25 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=399088, maxMem=766075207 -2016-05-23 21:24:25 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 21:24:25 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:55093 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:24:25 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 21:24:25 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:24:25 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 21:24:25 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 21:24:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 14 ms on localhost (1/1) -2016-05-23 21:24:25 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.014 s -2016-05-23 21:24:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 21:24:25 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.032931 s -2016-05-23 21:24:25 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:24:25 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:24:25 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:24:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:25 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:24:25 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=400612, maxMem=766075207 -2016-05-23 21:24:25 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:24:25 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=404012, maxMem=766075207 -2016-05-23 21:24:25 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 21:24:25 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:55093 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 21:24:25 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:24:25 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:24:25 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 21:24:25 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 21:24:25 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 21:24:25 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 21:24:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 10 ms on localhost (1/4) -2016-05-23 21:24:25 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 21:24:25 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 21:24:25 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 21:24:25 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 21:24:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 34 ms on localhost (2/4) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 37 ms on localhost (3/4) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 40 ms on localhost (4/4) -2016-05-23 21:24:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 21:24:25 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.046 s -2016-05-23 21:24:25 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.070803 s -2016-05-23 21:24:25 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:24:25 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:24:25 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:24:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:25 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:24:25 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=405946, maxMem=766075207 -2016-05-23 21:24:25 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:24:25 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=409386, maxMem=766075207 -2016-05-23 21:24:25 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 21:24:25 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:55093 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:24:25 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:24:25 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:24:25 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 21:24:25 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 21:24:25 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 21:24:25 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 21:24:25 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 21:24:25 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 21:24:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 17 ms on localhost (1/4) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 14 ms on localhost (2/4) -2016-05-23 21:24:25 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 21:24:25 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 21:24:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 31 ms on localhost (3/4) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 34 ms on localhost (4/4) -2016-05-23 21:24:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 21:24:25 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.037 s -2016-05-23 21:24:25 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.060952 s -2016-05-23 21:24:25 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:24:25 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:24:25 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:24:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:25 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:24:25 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=411304, maxMem=766075207 -2016-05-23 21:24:25 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:24:25 INFO MemoryStore:59 - ensureFreeSpace(1860) called with curMem=414680, maxMem=766075207 -2016-05-23 21:24:25 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1860.0 B, free 730.2 MB) -2016-05-23 21:24:25 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:55093 (size: 1860.0 B, free: 730.5 MB) -2016-05-23 21:24:25 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:24:25 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:24:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:24:25 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 21:24:25 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 21:24:25 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 21:24:25 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 21:24:26 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 59 ms on localhost (1/4) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 59 ms on localhost (2/4) -2016-05-23 21:24:26 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 80 ms on localhost (3/4) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 80 ms on localhost (4/4) -2016-05-23 21:24:26 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.158 s -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 21:24:26 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.192542 s -2016-05-23 21:24:26 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:24:26 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:24:26 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:24:26 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:24:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 21:24:26 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=416540, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=420580, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:24:26 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:55093 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:24:26 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 21:24:26 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 21:24:26 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 21:24:26 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 21:24:26 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 21:24:26 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 68 ms on localhost (1/4) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 73 ms on localhost (2/4) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 83 ms on localhost (3/4) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 105 ms on localhost (4/4) -2016-05-23 21:24:26 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.106 s -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 21:24:26 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:24:26 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:24:26 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 21:24:26 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:24:26 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=422862, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=425406, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 21:24:26 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:55093 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 21:24:26 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:26 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 21:24:26 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:26 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 21:24:26 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 28 ms on localhost (1/3) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 27 ms on localhost (2/3) -2016-05-23 21:24:26 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:26 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 53 ms on localhost (3/3) -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 21:24:26 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.054 s -2016-05-23 21:24:26 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.205848 s -2016-05-23 21:24:26 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:24:26 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 21:24:26 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:24:26 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:24:26 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 21:24:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 21:24:26 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=426974, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=431206, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 21:24:26 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:55093 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:24:26 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:24:26 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 21:24:26 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 21:24:26 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:26 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 68 ms on localhost (1/4) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 147 ms on localhost (2/4) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 170 ms on localhost (3/4) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 244 ms on localhost (4/4) -2016-05-23 21:24:26 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.246 s -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 21:24:26 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:24:26 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:24:26 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 21:24:26 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:24:26 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=433512, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=438408, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:24:26 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:55093 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:24:26 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:26 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:26 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 14 ms on localhost (1/1) -2016-05-23 21:24:26 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.014 s -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 21:24:26 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.361739 s -2016-05-23 21:24:26 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:24:26 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 21:24:26 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 21:24:26 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:24:26 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:24:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 21:24:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=440950, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=446646, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:24:26 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:55093 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:24:26 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:26 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 21:24:26 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 21:24:26 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 21:24:26 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:26 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 21:24:26 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 22 ms on localhost (1/4) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 23 ms on localhost (2/4) -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:26 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 21:24:26 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 42 ms on localhost (3/4) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 45 ms on localhost (4/4) -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 21:24:26 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.045 s -2016-05-23 21:24:26 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.057876 s -2016-05-23 21:24:26 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:24:26 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:24:26 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:24:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 21:24:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=449555, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=455291, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 21:24:26 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:55093 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:24:26 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:26 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 21:24:26 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 21:24:26 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:26 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:26 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 20 ms on localhost (1/4) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 24 ms on localhost (2/4) -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:26 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 32 ms on localhost (3/4) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 21:24:26 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.039 s -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 36 ms on localhost (4/4) -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 21:24:26 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.057528 s -2016-05-23 21:24:26 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:59 -2016-05-23 21:24:26 INFO DAGScheduler:59 - Got job 20 (foreach at StatisticsCorrLearning.scala:59) with 4 output partitions -2016-05-23 21:24:26 INFO DAGScheduler:59 - Final stage: ResultStage 34(foreach at StatisticsCorrLearning.scala:59) -2016-05-23 21:24:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:56), which has no missing parents -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=458183, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=459407, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.1 MB) -2016-05-23 21:24:26 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:55093 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:24:26 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:56) -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 62, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 63, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 64, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:24:26 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 62) -2016-05-23 21:24:26 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 63) -2016-05-23 21:24:26 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 64) -2016-05-23 21:24:26 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 65) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 62). 915 bytes result sent to driver -2016-05-23 21:24:26 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 63). 915 bytes result sent to driver -2016-05-23 21:24:26 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 65). 915 bytes result sent to driver -2016-05-23 21:24:26 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 64). 915 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 62) in 7 ms on localhost (1/4) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 65) in 6 ms on localhost (2/4) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 63) in 8 ms on localhost (3/4) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 64) in 9 ms on localhost (4/4) -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-23 21:24:26 INFO DAGScheduler:59 - ResultStage 34 (foreach at StatisticsCorrLearning.scala:59) finished in 0.011 s -2016-05-23 21:24:26 INFO DAGScheduler:59 - Job 20 finished: foreach at StatisticsCorrLearning.scala:59, took 0.021031 s -2016-05-23 21:24:26 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:61 -2016-05-23 21:24:26 INFO DAGScheduler:59 - Got job 21 (foreach at StatisticsCorrLearning.scala:61) with 4 output partitions -2016-05-23 21:24:26 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at StatisticsCorrLearning.scala:61) -2016-05-23 21:24:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:57), which has no missing parents -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=460300, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=461524, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.1 MB) -2016-05-23 21:24:26 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:55093 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:24:26 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:57) -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 4 tasks -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 66, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:24:26 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 66) -2016-05-23 21:24:26 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 67) -2016-05-23 21:24:26 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 68) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 67). 915 bytes result sent to driver -2016-05-23 21:24:26 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 66). 915 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 67) in 8 ms on localhost (1/4) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 66) in 11 ms on localhost (2/4) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 68). 915 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 68) in 10 ms on localhost (3/4) -2016-05-23 21:24:26 INFO Executor:59 - Running task 3.0 in stage 35.0 (TID 69) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 3.0 in stage 35.0 (TID 69). 915 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 35.0 (TID 69) in 15 ms on localhost (4/4) -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-23 21:24:26 INFO DAGScheduler:59 - ResultStage 35 (foreach at StatisticsCorrLearning.scala:61) finished in 0.019 s -2016-05-23 21:24:26 INFO DAGScheduler:59 - Job 21 finished: foreach at StatisticsCorrLearning.scala:61, took 0.038219 s -2016-05-23 21:24:26 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:24:26 INFO DAGScheduler:59 - Got job 22 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:24:26 INFO DAGScheduler:59 - Final stage: ResultStage 36(first at RowMatrix.scala:65) -2016-05-23 21:24:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=462417, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(1522) called with curMem=464921, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1522.0 B, free 730.1 MB) -2016-05-23 21:24:26 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:55093 (size: 1522.0 B, free: 730.5 MB) -2016-05-23 21:24:26 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:26 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 70) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 70). 1046 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 70) in 4 ms on localhost (1/1) -2016-05-23 21:24:26 INFO DAGScheduler:59 - ResultStage 36 (first at RowMatrix.scala:65) finished in 0.005 s -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-23 21:24:26 INFO DAGScheduler:59 - Job 22 finished: first at RowMatrix.scala:65, took 0.016003 s -2016-05-23 21:24:26 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:24:26 INFO DAGScheduler:59 - Got job 23 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:24:26 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:24:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=466443, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(1932) called with curMem=469843, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1932.0 B, free 730.1 MB) -2016-05-23 21:24:26 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:55093 (size: 1932.0 B, free: 730.5 MB) -2016-05-23 21:24:26 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 4 tasks -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:24:26 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 73) -2016-05-23 21:24:26 INFO Executor:59 - Running task 3.0 in stage 37.0 (TID 74) -2016-05-23 21:24:26 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 72) -2016-05-23 21:24:26 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 3.0 in stage 37.0 (TID 74). 1231 bytes result sent to driver -2016-05-23 21:24:26 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 73). 1231 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 37.0 (TID 74) in 7 ms on localhost (1/4) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 73) in 8 ms on localhost (2/4) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 72). 1231 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 72) in 10 ms on localhost (3/4) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 1231 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 15 ms on localhost (4/4) -2016-05-23 21:24:26 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at RowMatrix.scala:332) finished in 0.015 s -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-23 21:24:26 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at RowMatrix.scala:332, took 0.027754 s -2016-05-23 21:24:26 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:24:26 INFO DAGScheduler:59 - Got job 24 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:24:26 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:24:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=471775, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-23 21:24:26 INFO MemoryStore:59 - ensureFreeSpace(1916) called with curMem=475215, maxMem=766075207 -2016-05-23 21:24:26 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1916.0 B, free 730.1 MB) -2016-05-23 21:24:26 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:55093 (size: 1916.0 B, free: 730.5 MB) -2016-05-23 21:24:26 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 77, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 78, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:24:26 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-23 21:24:26 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-23 21:24:26 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 77) -2016-05-23 21:24:26 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 78) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 78). 1109 bytes result sent to driver -2016-05-23 21:24:26 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 77). 1109 bytes result sent to driver -2016-05-23 21:24:26 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 1109 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 78) in 8 ms on localhost (1/4) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 77) in 11 ms on localhost (2/4) -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 13 ms on localhost (3/4) -2016-05-23 21:24:26 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 1109 bytes result sent to driver -2016-05-23 21:24:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 18 ms on localhost (4/4) -2016-05-23 21:24:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-23 21:24:26 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at RowMatrix.scala:124) finished in 0.018 s -2016-05-23 21:24:26 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at RowMatrix.scala:124, took 0.032489 s -2016-05-23 21:24:27 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:24:27 INFO DAGScheduler:59 - Got job 25 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:24:27 INFO DAGScheduler:59 - Final stage: ResultStage 39(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:24:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:27 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:24:27 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=477131, maxMem=766075207 -2016-05-23 21:24:27 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 21:24:27 INFO MemoryStore:59 - ensureFreeSpace(1851) called with curMem=480507, maxMem=766075207 -2016-05-23 21:24:27 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1851.0 B, free 730.1 MB) -2016-05-23 21:24:27 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:55093 (size: 1851.0 B, free: 730.5 MB) -2016-05-23 21:24:27 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:24:27 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 4 tasks -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 79, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 80, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 81, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 39.0 (TID 82, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:24:27 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 81) -2016-05-23 21:24:27 INFO Executor:59 - Running task 3.0 in stage 39.0 (TID 82) -2016-05-23 21:24:27 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 80) -2016-05-23 21:24:27 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 81). 1249 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 81) in 8 ms on localhost (1/4) -2016-05-23 21:24:27 INFO Executor:59 - Finished task 3.0 in stage 39.0 (TID 82). 1294 bytes result sent to driver -2016-05-23 21:24:27 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 79) -2016-05-23 21:24:27 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 80). 1254 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 39.0 (TID 82) in 9 ms on localhost (2/4) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 80) in 13 ms on localhost (3/4) -2016-05-23 21:24:27 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 79). 1254 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 79) in 16 ms on localhost (4/4) -2016-05-23 21:24:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-23 21:24:27 INFO DAGScheduler:59 - ResultStage 39 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.016 s -2016-05-23 21:24:27 INFO DAGScheduler:59 - Job 25 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.032610 s -2016-05-23 21:24:27 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:24:27 INFO DAGScheduler:59 - Registering RDD 51 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:24:27 INFO DAGScheduler:59 - Got job 26 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:24:27 INFO DAGScheduler:59 - Final stage: ResultStage 41(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:24:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 40) -2016-05-23 21:24:27 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 40) -2016-05-23 21:24:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:24:27 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=482358, maxMem=766075207 -2016-05-23 21:24:27 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-05-23 21:24:27 INFO MemoryStore:59 - ensureFreeSpace(2281) called with curMem=486398, maxMem=766075207 -2016-05-23 21:24:27 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 21:24:27 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:55093 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:24:27 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:24:27 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 4 tasks -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 83, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 84, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 85, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 40.0 (TID 86, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:24:27 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 84) -2016-05-23 21:24:27 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 83) -2016-05-23 21:24:27 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 85) -2016-05-23 21:24:27 INFO Executor:59 - Running task 3.0 in stage 40.0 (TID 86) -2016-05-23 21:24:27 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 85). 1161 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 85) in 36 ms on localhost (1/4) -2016-05-23 21:24:27 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 83). 1161 bytes result sent to driver -2016-05-23 21:24:27 INFO Executor:59 - Finished task 3.0 in stage 40.0 (TID 86). 1161 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 83) in 206 ms on localhost (2/4) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 40.0 (TID 86) in 205 ms on localhost (3/4) -2016-05-23 21:24:27 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 84). 1161 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 84) in 445 ms on localhost (4/4) -2016-05-23 21:24:27 INFO DAGScheduler:59 - ShuffleMapStage 40 (flatMap at SpearmanCorrelation.scala:49) finished in 0.446 s -2016-05-23 21:24:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-23 21:24:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:24:27 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:24:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 41) -2016-05-23 21:24:27 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:24:27 INFO DAGScheduler:59 - Missing parents for ResultStage 41: List() -2016-05-23 21:24:27 INFO DAGScheduler:59 - Submitting ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:24:27 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=488679, maxMem=766075207 -2016-05-23 21:24:27 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:24:27 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=491223, maxMem=766075207 -2016-05-23 21:24:27 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.1 MB) -2016-05-23 21:24:27 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:55093 (size: 1567.0 B, free: 730.5 MB) -2016-05-23 21:24:27 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:27 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:24:27 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 3 tasks -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 87, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:27 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 87) -2016-05-23 21:24:27 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 89) -2016-05-23 21:24:27 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 88) -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:27 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 88). 1203 bytes result sent to driver -2016-05-23 21:24:27 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 87). 1203 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 88) in 16 ms on localhost (1/3) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 87) in 18 ms on localhost (2/3) -2016-05-23 21:24:27 INFO Executor:59 - Finished task 2.0 in stage 41.0 (TID 89). 1203 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 41.0 (TID 89) in 19 ms on localhost (3/3) -2016-05-23 21:24:27 INFO DAGScheduler:59 - ResultStage 41 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.021 s -2016-05-23 21:24:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-23 21:24:27 INFO DAGScheduler:59 - Job 26 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.496316 s -2016-05-23 21:24:27 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:24:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 163 bytes -2016-05-23 21:24:27 INFO DAGScheduler:59 - Registering RDD 56 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:24:27 INFO DAGScheduler:59 - Got job 27 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:24:27 INFO DAGScheduler:59 - Final stage: ResultStage 44(first at RowMatrix.scala:65) -2016-05-23 21:24:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 43) -2016-05-23 21:24:27 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 43) -2016-05-23 21:24:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:24:27 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=492790, maxMem=766075207 -2016-05-23 21:24:27 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 21:24:27 INFO MemoryStore:59 - ensureFreeSpace(2305) called with curMem=497022, maxMem=766075207 -2016-05-23 21:24:27 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-23 21:24:27 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:55093 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:24:27 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:24:27 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 4 tasks -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 90, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 91, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 43.0 (TID 92, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 43.0 (TID 93, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:24:27 INFO Executor:59 - Running task 2.0 in stage 43.0 (TID 92) -2016-05-23 21:24:27 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 91) -2016-05-23 21:24:27 INFO Executor:59 - Running task 3.0 in stage 43.0 (TID 93) -2016-05-23 21:24:27 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 90) -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:27 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 91). 1377 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 91) in 162 ms on localhost (1/4) -2016-05-23 21:24:27 INFO Executor:59 - Finished task 2.0 in stage 43.0 (TID 92). 1377 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 43.0 (TID 92) in 181 ms on localhost (2/4) -2016-05-23 21:24:27 INFO Executor:59 - Finished task 3.0 in stage 43.0 (TID 93). 1377 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 43.0 (TID 93) in 182 ms on localhost (3/4) -2016-05-23 21:24:27 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 90). 1377 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 90) in 190 ms on localhost (4/4) -2016-05-23 21:24:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-23 21:24:27 INFO DAGScheduler:59 - ShuffleMapStage 43 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.190 s -2016-05-23 21:24:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:24:27 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:24:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 44) -2016-05-23 21:24:27 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:24:27 INFO DAGScheduler:59 - Missing parents for ResultStage 44: List() -2016-05-23 21:24:27 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:24:27 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=499327, maxMem=766075207 -2016-05-23 21:24:27 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 21:24:27 INFO MemoryStore:59 - ensureFreeSpace(2545) called with curMem=504223, maxMem=766075207 -2016-05-23 21:24:27 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:24:27 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:55093 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:24:27 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:24:27 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 1 tasks -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 94, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:27 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 94) -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:27 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 94). 1296 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 94) in 9 ms on localhost (1/1) -2016-05-23 21:24:27 INFO DAGScheduler:59 - ResultStage 44 (first at RowMatrix.scala:65) finished in 0.010 s -2016-05-23 21:24:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-23 21:24:27 INFO DAGScheduler:59 - Job 27 finished: first at RowMatrix.scala:65, took 0.228897 s -2016-05-23 21:24:27 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:24:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 163 bytes -2016-05-23 21:24:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 166 bytes -2016-05-23 21:24:27 INFO DAGScheduler:59 - Got job 28 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:24:27 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:24:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 46) -2016-05-23 21:24:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:27 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:24:27 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=506768, maxMem=766075207 -2016-05-23 21:24:27 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 21:24:27 INFO MemoryStore:59 - ensureFreeSpace(2914) called with curMem=512464, maxMem=766075207 -2016-05-23 21:24:27 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 21:24:27 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:55093 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:24:27 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:24:27 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 4 tasks -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 95, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 96, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 47.0 (TID 97, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 47.0 (TID 98, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:27 INFO Executor:59 - Running task 2.0 in stage 47.0 (TID 97) -2016-05-23 21:24:27 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 95) -2016-05-23 21:24:27 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 96) -2016-05-23 21:24:27 INFO Executor:59 - Running task 3.0 in stage 47.0 (TID 98) -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:27 INFO Executor:59 - Finished task 3.0 in stage 47.0 (TID 98). 1481 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 47.0 (TID 98) in 11 ms on localhost (1/4) -2016-05-23 21:24:27 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 96). 1481 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 96) in 15 ms on localhost (2/4) -2016-05-23 21:24:27 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 95). 1481 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 95) in 18 ms on localhost (3/4) -2016-05-23 21:24:27 INFO Executor:59 - Finished task 2.0 in stage 47.0 (TID 97). 1481 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 47.0 (TID 97) in 19 ms on localhost (4/4) -2016-05-23 21:24:27 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at RowMatrix.scala:332) finished in 0.020 s -2016-05-23 21:24:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-23 21:24:27 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at RowMatrix.scala:332, took 0.034394 s -2016-05-23 21:24:27 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:24:27 INFO DAGScheduler:59 - Got job 29 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:24:27 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:24:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 49) -2016-05-23 21:24:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:27 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:24:27 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=515378, maxMem=766075207 -2016-05-23 21:24:27 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 21:24:27 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=521114, maxMem=766075207 -2016-05-23 21:24:27 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 21:24:27 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:55093 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:24:27 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:24:27 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 4 tasks -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 50.0 (TID 101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 50.0 (TID 102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:27 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-23 21:24:27 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-23 21:24:27 INFO Executor:59 - Running task 3.0 in stage 50.0 (TID 102) -2016-05-23 21:24:27 INFO Executor:59 - Running task 2.0 in stage 50.0 (TID 101) -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:27 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 1359 bytes result sent to driver -2016-05-23 21:24:27 INFO Executor:59 - Finished task 2.0 in stage 50.0 (TID 101). 1359 bytes result sent to driver -2016-05-23 21:24:27 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 1359 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 14 ms on localhost (1/4) -2016-05-23 21:24:27 INFO Executor:59 - Finished task 3.0 in stage 50.0 (TID 102). 1359 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 50.0 (TID 101) in 14 ms on localhost (2/4) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 16 ms on localhost (3/4) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 50.0 (TID 102) in 14 ms on localhost (4/4) -2016-05-23 21:24:27 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at RowMatrix.scala:124) finished in 0.016 s -2016-05-23 21:24:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-23 21:24:27 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at RowMatrix.scala:124, took 0.030781 s -2016-05-23 21:24:27 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:72 -2016-05-23 21:24:27 INFO DAGScheduler:59 - Got job 30 (foreach at StatisticsCorrLearning.scala:72) with 4 output partitions -2016-05-23 21:24:27 INFO DAGScheduler:59 - Final stage: ResultStage 51(foreach at StatisticsCorrLearning.scala:72) -2016-05-23 21:24:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:27 INFO DAGScheduler:59 - Submitting ResultStage 51 (ParallelCollectionRDD[61] at parallelize at StatisticsCorrLearning.scala:69), which has no missing parents -2016-05-23 21:24:27 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=524010, maxMem=766075207 -2016-05-23 21:24:27 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-05-23 21:24:27 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=525234, maxMem=766075207 -2016-05-23 21:24:27 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.1 MB) -2016-05-23 21:24:27 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:55093 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:24:27 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 51 (ParallelCollectionRDD[61] at parallelize at StatisticsCorrLearning.scala:69) -2016-05-23 21:24:27 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 4 tasks -2016-05-23 21:24:27 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 103, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 104, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 51.0 (TID 105, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 51.0 (TID 106, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:24:27 INFO Executor:59 - Running task 2.0 in stage 51.0 (TID 105) -2016-05-23 21:24:27 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 104) -2016-05-23 21:24:27 INFO Executor:59 - Finished task 2.0 in stage 51.0 (TID 105). 915 bytes result sent to driver -2016-05-23 21:24:27 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 104). 915 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 51.0 (TID 105) in 13 ms on localhost (1/4) -2016-05-23 21:24:27 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 103) -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 104) in 14 ms on localhost (2/4) -2016-05-23 21:24:27 INFO Executor:59 - Running task 3.0 in stage 51.0 (TID 106) -2016-05-23 21:24:27 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:55093 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:24:27 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 103). 915 bytes result sent to driver -2016-05-23 21:24:27 INFO Executor:59 - Finished task 3.0 in stage 51.0 (TID 106). 915 bytes result sent to driver -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 103) in 23 ms on localhost (3/4) -2016-05-23 21:24:27 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-23 21:24:27 INFO DAGScheduler:59 - ResultStage 51 (foreach at StatisticsCorrLearning.scala:72) finished in 0.027 s -2016-05-23 21:24:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 51.0 (TID 106) in 19 ms on localhost (4/4) -2016-05-23 21:24:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-23 21:24:27 INFO DAGScheduler:59 - Job 30 finished: foreach at StatisticsCorrLearning.scala:72, took 0.064054 s -2016-05-23 21:24:27 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:55093 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:24:27 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-23 21:24:27 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:55093 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:24:27 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-23 21:24:27 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:74 -2016-05-23 21:24:27 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:55093 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:24:27 INFO DAGScheduler:59 - Got job 31 (foreach at StatisticsCorrLearning.scala:74) with 4 output partitions -2016-05-23 21:24:27 INFO DAGScheduler:59 - Final stage: ResultStage 52(foreach at StatisticsCorrLearning.scala:74) -2016-05-23 21:24:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:27 INFO DAGScheduler:59 - Submitting ResultStage 52 (ParallelCollectionRDD[62] at parallelize at StatisticsCorrLearning.scala:70), which has no missing parents -2016-05-23 21:24:27 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-23 21:24:27 INFO ContextCleaner:59 - Cleaned shuffle 5 -2016-05-23 21:24:27 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=494907, maxMem=766075207 -2016-05-23 21:24:27 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-05-23 21:24:27 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=494564, maxMem=766075207 -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:55093 in memory (size: 1567.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.1 MB) -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:55093 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 52 (ParallelCollectionRDD[62] at parallelize at StatisticsCorrLearning.scala:70) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 4 tasks -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 107, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 108, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 52.0 (TID 109, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 52.0 (TID 110, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:24:28 INFO Executor:59 - Running task 3.0 in stage 52.0 (TID 110) -2016-05-23 21:24:28 INFO Executor:59 - Running task 2.0 in stage 52.0 (TID 109) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 3.0 in stage 52.0 (TID 110). 915 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 2.0 in stage 52.0 (TID 109). 915 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 107) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:55093 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 52.0 (TID 110) in 13 ms on localhost (1/4) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 107). 915 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 52.0 (TID 109) in 15 ms on localhost (2/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 107) in 17 ms on localhost (3/4) -2016-05-23 21:24:28 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 108) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 108). 915 bytes result sent to driver -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 108) in 24 ms on localhost (4/4) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-23 21:24:28 INFO DAGScheduler:59 - ResultStage 52 (foreach at StatisticsCorrLearning.scala:74) finished in 0.025 s -2016-05-23 21:24:28 INFO DAGScheduler:59 - Job 31 finished: foreach at StatisticsCorrLearning.scala:74, took 0.038400 s -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned shuffle 4 -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:55093 in memory (size: 1851.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:55093 in memory (size: 1916.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:55093 in memory (size: 1932.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-23 21:24:28 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Got job 32 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:24:28 INFO DAGScheduler:59 - Final stage: ResultStage 53(first at RowMatrix.scala:65) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[64] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:55093 in memory (size: 1522.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=466651, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=467038, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.1 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:55093 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:55093 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 53 (MapPartitionsRDD[64] at map at Correlation.scala:45) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 1 tasks -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 111, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:24:28 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 111) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:55093 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-23 21:24:28 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 111). 935 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 111) in 5 ms on localhost (1/1) -2016-05-23 21:24:28 INFO DAGScheduler:59 - ResultStage 53 (first at RowMatrix.scala:65) finished in 0.006 s -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-23 21:24:28 INFO DAGScheduler:59 - Job 32 finished: first at RowMatrix.scala:65, took 0.018505 s -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:55093 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:55093 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-23 21:24:28 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Got job 33 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:55093 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Final stage: ResultStage 54(first at RowMatrix.scala:65) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[64] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=439468, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:55093 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=441972, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:55093 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-05-23 21:24:28 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 54 (MapPartitionsRDD[64] at map at Correlation.scala:45) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 3 tasks -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 112, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 113, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 54.0 (TID 114, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO Executor:59 - Running task 2.0 in stage 54.0 (TID 114) -2016-05-23 21:24:28 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 113) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:55093 in memory (size: 1568.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 112) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 2.0 in stage 54.0 (TID 114). 1046 bytes result sent to driver -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-23 21:24:28 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 113). 1046 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 112). 1046 bytes result sent to driver -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:55093 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 54.0 (TID 114) in 9 ms on localhost (1/3) -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 113) in 12 ms on localhost (2/3) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 112) in 15 ms on localhost (3/3) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-23 21:24:28 INFO DAGScheduler:59 - ResultStage 54 (first at RowMatrix.scala:65) finished in 0.016 s -2016-05-23 21:24:28 INFO DAGScheduler:59 - Job 33 finished: first at RowMatrix.scala:65, took 0.031097 s -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:55093 in memory (size: 1860.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Got job 34 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:24:28 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[65] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:55093 in memory (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=418236, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=421636, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:55093 in memory (size: 1934.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:55093 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-23 21:24:28 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 55 (MapPartitionsRDD[65] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 4 tasks -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 115, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 116, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:55093 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 55.0 (TID 117, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 55.0 (TID 118, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-23 21:24:28 INFO Executor:59 - Running task 2.0 in stage 55.0 (TID 117) -2016-05-23 21:24:28 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 115) -2016-05-23 21:24:28 INFO Executor:59 - Running task 3.0 in stage 55.0 (TID 118) -2016-05-23 21:24:28 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 116) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 115). 1231 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 2.0 in stage 55.0 (TID 117). 1231 bytes result sent to driver -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:55093 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 3.0 in stage 55.0 (TID 118). 1231 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 116). 1231 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 55.0 (TID 117) in 12 ms on localhost (1/4) -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 55.0 (TID 118) in 12 ms on localhost (2/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 115) in 16 ms on localhost (3/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 116) in 16 ms on localhost (4/4) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-23 21:24:28 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at RowMatrix.scala:332) finished in 0.017 s -2016-05-23 21:24:28 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at RowMatrix.scala:332, took 0.034323 s -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:55093 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:55093 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-23 21:24:28 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Got job 35 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:24:28 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:55093 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[66] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=385438, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:55093 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=388878, maxMem=766075207 -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:55093 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 56 (MapPartitionsRDD[66] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 4 tasks -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:55093 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 119, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 120, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 56.0 (TID 121, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 56.0 (TID 122, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 120) -2016-05-23 21:24:28 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 119) -2016-05-23 21:24:28 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-23 21:24:28 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 119). 1109 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 120). 1109 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Running task 2.0 in stage 56.0 (TID 121) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 119) in 17 ms on localhost (1/4) -2016-05-23 21:24:28 INFO Executor:59 - Running task 3.0 in stage 56.0 (TID 122) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 120) in 17 ms on localhost (2/4) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 2.0 in stage 56.0 (TID 121). 1109 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 3.0 in stage 56.0 (TID 122). 1109 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 56.0 (TID 121) in 22 ms on localhost (3/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 56.0 (TID 122) in 23 ms on localhost (4/4) -2016-05-23 21:24:28 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at RowMatrix.scala:124) finished in 0.026 s -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-23 21:24:28 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at RowMatrix.scala:124, took 0.039977 s -2016-05-23 21:24:28 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Got job 36 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:24:28 INFO DAGScheduler:59 - Final stage: ResultStage 57(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[72] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=384322, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(1858) called with curMem=387698, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 1858.0 B, free 730.2 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:55093 (size: 1858.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 57 (MapPartitionsRDD[72] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 4 tasks -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 123, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 124, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 57.0 (TID 125, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 57.0 (TID 126, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 123) -2016-05-23 21:24:28 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 124) -2016-05-23 21:24:28 INFO Executor:59 - Running task 2.0 in stage 57.0 (TID 125) -2016-05-23 21:24:28 INFO Executor:59 - Running task 3.0 in stage 57.0 (TID 126) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 124). 1254 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 3.0 in stage 57.0 (TID 126). 1254 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 2.0 in stage 57.0 (TID 125). 1249 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 57.0 (TID 126) in 7 ms on localhost (1/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 57.0 (TID 125) in 9 ms on localhost (2/4) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 123). 1104 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 124) in 11 ms on localhost (3/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 123) in 14 ms on localhost (4/4) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-23 21:24:28 INFO DAGScheduler:59 - ResultStage 57 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.014 s -2016-05-23 21:24:28 INFO DAGScheduler:59 - Job 36 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.024979 s -2016-05-23 21:24:28 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Registering RDD 70 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Got job 37 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:24:28 INFO DAGScheduler:59 - Final stage: ResultStage 59(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 58) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 58) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 58 (MapPartitionsRDD[70] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=389556, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(2285) called with curMem=393596, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:55093 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:24:28 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 58 (MapPartitionsRDD[70] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 4 tasks -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 127, localhost, PROCESS_LOCAL, 2221 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 128, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 58.0 (TID 129, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 58.0 (TID 130, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:24:28 INFO Executor:59 - Running task 2.0 in stage 58.0 (TID 129) -2016-05-23 21:24:28 INFO Executor:59 - Running task 3.0 in stage 58.0 (TID 130) -2016-05-23 21:24:28 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 127) -2016-05-23 21:24:28 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 128) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 127). 1161 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 127) in 11 ms on localhost (1/4) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 3.0 in stage 58.0 (TID 130). 1161 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 2.0 in stage 58.0 (TID 129). 1161 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 58.0 (TID 129) in 40 ms on localhost (2/4) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 128). 1161 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 58.0 (TID 130) in 42 ms on localhost (3/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 128) in 43 ms on localhost (4/4) -2016-05-23 21:24:28 INFO DAGScheduler:59 - ShuffleMapStage 58 (flatMap at SpearmanCorrelation.scala:49) finished in 0.044 s -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-05-23 21:24:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:24:28 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:24:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 59) -2016-05-23 21:24:28 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Missing parents for ResultStage 59: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting ResultStage 59 (ShuffledRDD[73] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=395881, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=398425, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.2 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:55093 (size: 1567.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 59 (ShuffledRDD[73] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 3 tasks -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 131, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 132, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 59.0 (TID 133, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:28 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 131) -2016-05-23 21:24:28 INFO Executor:59 - Running task 2.0 in stage 59.0 (TID 133) -2016-05-23 21:24:28 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 132) -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:28 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 132). 1203 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 131). 1203 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 2.0 in stage 59.0 (TID 133). 1203 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 132) in 21 ms on localhost (1/3) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 131) in 23 ms on localhost (2/3) -2016-05-23 21:24:28 INFO DAGScheduler:59 - ResultStage 59 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.025 s -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 59.0 (TID 133) in 23 ms on localhost (3/3) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-05-23 21:24:28 INFO DAGScheduler:59 - Job 37 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.096348 s -2016-05-23 21:24:28 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:24:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 168 bytes -2016-05-23 21:24:28 INFO DAGScheduler:59 - Registering RDD 75 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Got job 38 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:24:28 INFO DAGScheduler:59 - Final stage: ResultStage 62(first at RowMatrix.scala:65) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 61) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 61) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 61 (MapPartitionsRDD[75] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=399992, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(2305) called with curMem=404224, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:55093 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:24:28 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 61 (MapPartitionsRDD[75] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 4 tasks -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 134, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 135, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 61.0 (TID 136, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 61.0 (TID 137, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:24:28 INFO Executor:59 - Running task 3.0 in stage 61.0 (TID 137) -2016-05-23 21:24:28 INFO Executor:59 - Running task 2.0 in stage 61.0 (TID 136) -2016-05-23 21:24:28 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 135) -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:28 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 134) -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:28 INFO Executor:59 - Finished task 3.0 in stage 61.0 (TID 137). 1377 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 134). 1377 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 2.0 in stage 61.0 (TID 136). 1377 bytes result sent to driver -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 61.0 (TID 137) in 142 ms on localhost (1/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 134) in 145 ms on localhost (2/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 61.0 (TID 136) in 145 ms on localhost (3/4) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 135). 1377 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 135) in 156 ms on localhost (4/4) -2016-05-23 21:24:28 INFO DAGScheduler:59 - ShuffleMapStage 61 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.160 s -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-05-23 21:24:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:24:28 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:24:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 62) -2016-05-23 21:24:28 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Missing parents for ResultStage 62: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=406529, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=411425, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:55093 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:24:28 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 62 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 1 tasks -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 138, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:28 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 138) -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:28 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 138). 1185 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 138) in 12 ms on localhost (1/1) -2016-05-23 21:24:28 INFO DAGScheduler:59 - ResultStage 62 (first at RowMatrix.scala:65) finished in 0.012 s -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-05-23 21:24:28 INFO DAGScheduler:59 - Job 38 finished: first at RowMatrix.scala:65, took 0.210203 s -2016-05-23 21:24:28 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:24:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 168 bytes -2016-05-23 21:24:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 165 bytes -2016-05-23 21:24:28 INFO DAGScheduler:59 - Got job 39 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 21:24:28 INFO DAGScheduler:59 - Final stage: ResultStage 65(first at RowMatrix.scala:65) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 64) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87), which has no missing parents -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=413969, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=418865, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:55093 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:24:28 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 65 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 3 tasks -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 139, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 140, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 65.0 (TID 141, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:28 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 139) -2016-05-23 21:24:28 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 140) -2016-05-23 21:24:28 INFO Executor:59 - Running task 2.0 in stage 65.0 (TID 141) -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:28 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 139). 1296 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 139) in 14 ms on localhost (1/3) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 140). 1296 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 140) in 15 ms on localhost (2/3) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 2.0 in stage 65.0 (TID 141). 1296 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 65.0 (TID 141) in 16 ms on localhost (3/3) -2016-05-23 21:24:28 INFO DAGScheduler:59 - ResultStage 65 (first at RowMatrix.scala:65) finished in 0.018 s -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-05-23 21:24:28 INFO DAGScheduler:59 - Job 39 finished: first at RowMatrix.scala:65, took 0.031060 s -2016-05-23 21:24:28 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Got job 40 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:24:28 INFO DAGScheduler:59 - Final stage: ResultStage 68(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 67) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting ResultStage 68 (MapPartitionsRDD[78] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=421409, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(2914) called with curMem=427105, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:55093 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:24:28 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 68 (MapPartitionsRDD[78] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Adding task set 68.0 with 4 tasks -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 68.0 (TID 142, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 68.0 (TID 143, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 68.0 (TID 144, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 68.0 (TID 145, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:28 INFO Executor:59 - Running task 2.0 in stage 68.0 (TID 144) -2016-05-23 21:24:28 INFO Executor:59 - Running task 3.0 in stage 68.0 (TID 145) -2016-05-23 21:24:28 INFO Executor:59 - Running task 0.0 in stage 68.0 (TID 142) -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:28 INFO Executor:59 - Finished task 2.0 in stage 68.0 (TID 144). 1481 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 0.0 in stage 68.0 (TID 142). 1481 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 68.0 (TID 144) in 13 ms on localhost (1/4) -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 10 ms -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 68.0 (TID 142) in 16 ms on localhost (2/4) -2016-05-23 21:24:28 INFO Executor:59 - Running task 1.0 in stage 68.0 (TID 143) -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:28 INFO Executor:59 - Finished task 3.0 in stage 68.0 (TID 145). 1481 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 68.0 (TID 145) in 23 ms on localhost (3/4) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 1.0 in stage 68.0 (TID 143). 1481 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 68.0 (TID 143) in 28 ms on localhost (4/4) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 68.0, whose tasks have all completed, from pool -2016-05-23 21:24:28 INFO DAGScheduler:59 - ResultStage 68 (treeAggregate at RowMatrix.scala:332) finished in 0.030 s -2016-05-23 21:24:28 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at RowMatrix.scala:332, took 0.041350 s -2016-05-23 21:24:28 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Got job 41 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:24:28 INFO DAGScheduler:59 - Final stage: ResultStage 71(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 70) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting ResultStage 71 (MapPartitionsRDD[79] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=430019, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=435755, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:55093 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:24:28 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 71 (MapPartitionsRDD[79] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Adding task set 71.0 with 4 tasks -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 71.0 (TID 146, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 71.0 (TID 147, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 71.0 (TID 148, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 71.0 (TID 149, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:28 INFO Executor:59 - Running task 0.0 in stage 71.0 (TID 146) -2016-05-23 21:24:28 INFO Executor:59 - Running task 3.0 in stage 71.0 (TID 149) -2016-05-23 21:24:28 INFO Executor:59 - Running task 1.0 in stage 71.0 (TID 147) -2016-05-23 21:24:28 INFO Executor:59 - Running task 2.0 in stage 71.0 (TID 148) -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:28 INFO Executor:59 - Finished task 0.0 in stage 71.0 (TID 146). 1359 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 71.0 (TID 146) in 14 ms on localhost (1/4) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 2.0 in stage 71.0 (TID 148). 1359 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 71.0 (TID 148) in 15 ms on localhost (2/4) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 3.0 in stage 71.0 (TID 149). 1359 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 1.0 in stage 71.0 (TID 147). 1359 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 71.0 (TID 149) in 18 ms on localhost (3/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 71.0 (TID 147) in 20 ms on localhost (4/4) -2016-05-23 21:24:28 INFO DAGScheduler:59 - ResultStage 71 (treeAggregate at RowMatrix.scala:124) finished in 0.021 s -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 71.0, whose tasks have all completed, from pool -2016-05-23 21:24:28 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at RowMatrix.scala:124, took 0.031357 s -2016-05-23 21:24:28 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:86 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Got job 42 (foreach at StatisticsCorrLearning.scala:86) with 4 output partitions -2016-05-23 21:24:28 INFO DAGScheduler:59 - Final stage: ResultStage 72(foreach at StatisticsCorrLearning.scala:86) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting ResultStage 72 (ParallelCollectionRDD[80] at parallelize at StatisticsCorrLearning.scala:83), which has no missing parents -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=438651, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=439875, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:55093 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 72 (ParallelCollectionRDD[80] at parallelize at StatisticsCorrLearning.scala:83) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Adding task set 72.0 with 4 tasks -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 72.0 (TID 150, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 72.0 (TID 151, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 72.0 (TID 152, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 72.0 (TID 153, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:24:28 INFO Executor:59 - Running task 0.0 in stage 72.0 (TID 150) -2016-05-23 21:24:28 INFO Executor:59 - Running task 2.0 in stage 72.0 (TID 152) -2016-05-23 21:24:28 INFO Executor:59 - Running task 1.0 in stage 72.0 (TID 151) -2016-05-23 21:24:28 INFO Executor:59 - Running task 3.0 in stage 72.0 (TID 153) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 0.0 in stage 72.0 (TID 150). 915 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 2.0 in stage 72.0 (TID 152). 915 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 3.0 in stage 72.0 (TID 153). 915 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 1.0 in stage 72.0 (TID 151). 915 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 72.0 (TID 152) in 5 ms on localhost (1/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 72.0 (TID 150) in 7 ms on localhost (2/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 72.0 (TID 151) in 6 ms on localhost (3/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 72.0 (TID 153) in 6 ms on localhost (4/4) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 72.0, whose tasks have all completed, from pool -2016-05-23 21:24:28 INFO DAGScheduler:59 - ResultStage 72 (foreach at StatisticsCorrLearning.scala:86) finished in 0.008 s -2016-05-23 21:24:28 INFO DAGScheduler:59 - Job 42 finished: foreach at StatisticsCorrLearning.scala:86, took 0.017760 s -2016-05-23 21:24:28 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:88 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Got job 43 (foreach at StatisticsCorrLearning.scala:88) with 4 output partitions -2016-05-23 21:24:28 INFO DAGScheduler:59 - Final stage: ResultStage 73(foreach at StatisticsCorrLearning.scala:88) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting ResultStage 73 (ParallelCollectionRDD[81] at parallelize at StatisticsCorrLearning.scala:84), which has no missing parents -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(1232) called with curMem=440768, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 1232.0 B, free 730.2 MB) -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(894) called with curMem=442000, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 894.0 B, free 730.2 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:55093 (size: 894.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 73 (ParallelCollectionRDD[81] at parallelize at StatisticsCorrLearning.scala:84) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Adding task set 73.0 with 4 tasks -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 73.0 (TID 154, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 73.0 (TID 155, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 73.0 (TID 156, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 73.0 (TID 157, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:24:28 INFO Executor:59 - Running task 1.0 in stage 73.0 (TID 155) -2016-05-23 21:24:28 INFO Executor:59 - Running task 0.0 in stage 73.0 (TID 154) -2016-05-23 21:24:28 INFO Executor:59 - Running task 3.0 in stage 73.0 (TID 157) -2016-05-23 21:24:28 INFO Executor:59 - Running task 2.0 in stage 73.0 (TID 156) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 0.0 in stage 73.0 (TID 154). 915 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 3.0 in stage 73.0 (TID 157). 915 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 73.0 (TID 154) in 6 ms on localhost (1/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 73.0 (TID 157) in 5 ms on localhost (2/4) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 1.0 in stage 73.0 (TID 155). 915 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 2.0 in stage 73.0 (TID 156). 915 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 73.0 (TID 156) in 8 ms on localhost (3/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 73.0 (TID 155) in 9 ms on localhost (4/4) -2016-05-23 21:24:28 INFO DAGScheduler:59 - ResultStage 73 (foreach at StatisticsCorrLearning.scala:88) finished in 0.010 s -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 73.0, whose tasks have all completed, from pool -2016-05-23 21:24:28 INFO DAGScheduler:59 - Job 43 finished: foreach at StatisticsCorrLearning.scala:88, took 0.018576 s -2016-05-23 21:24:28 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Got job 44 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:24:28 INFO DAGScheduler:59 - Final stage: ResultStage 74(first at RowMatrix.scala:65) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting ResultStage 74 (MapPartitionsRDD[83] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=442894, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=445398, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:55093 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 74 (MapPartitionsRDD[83] at map at Correlation.scala:45) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Adding task set 74.0 with 1 tasks -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 74.0 (TID 158, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO Executor:59 - Running task 0.0 in stage 74.0 (TID 158) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 0.0 in stage 74.0 (TID 158). 1046 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 74.0 (TID 158) in 3 ms on localhost (1/1) -2016-05-23 21:24:28 INFO DAGScheduler:59 - ResultStage 74 (first at RowMatrix.scala:65) finished in 0.003 s -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 74.0, whose tasks have all completed, from pool -2016-05-23 21:24:28 INFO DAGScheduler:59 - Job 44 finished: first at RowMatrix.scala:65, took 0.010014 s -2016-05-23 21:24:28 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Got job 45 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:24:28 INFO DAGScheduler:59 - Final stage: ResultStage 75(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting ResultStage 75 (MapPartitionsRDD[84] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=446922, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=450322, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.2 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:55093 (size: 1935.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 75 (MapPartitionsRDD[84] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Adding task set 75.0 with 4 tasks -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 75.0 (TID 159, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 75.0 (TID 160, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 75.0 (TID 161, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 75.0 (TID 162, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:24:28 INFO Executor:59 - Running task 2.0 in stage 75.0 (TID 161) -2016-05-23 21:24:28 INFO Executor:59 - Running task 0.0 in stage 75.0 (TID 159) -2016-05-23 21:24:28 INFO Executor:59 - Running task 3.0 in stage 75.0 (TID 162) -2016-05-23 21:24:28 INFO Executor:59 - Running task 1.0 in stage 75.0 (TID 160) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 2.0 in stage 75.0 (TID 161). 1231 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 1.0 in stage 75.0 (TID 160). 1231 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 3.0 in stage 75.0 (TID 162). 1231 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 0.0 in stage 75.0 (TID 159). 1231 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 75.0 (TID 161) in 4 ms on localhost (1/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 75.0 (TID 160) in 5 ms on localhost (2/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 75.0 (TID 162) in 5 ms on localhost (3/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 75.0 (TID 159) in 6 ms on localhost (4/4) -2016-05-23 21:24:28 INFO DAGScheduler:59 - ResultStage 75 (treeAggregate at RowMatrix.scala:332) finished in 0.006 s -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 75.0, whose tasks have all completed, from pool -2016-05-23 21:24:28 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at RowMatrix.scala:332, took 0.013913 s -2016-05-23 21:24:28 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Got job 46 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:24:28 INFO DAGScheduler:59 - Final stage: ResultStage 76(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting ResultStage 76 (MapPartitionsRDD[85] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=452257, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(1919) called with curMem=455697, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 1919.0 B, free 730.1 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:55093 (size: 1919.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 76 (MapPartitionsRDD[85] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 4 tasks -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 163, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 164, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 76.0 (TID 165, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 76.0 (TID 166, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:24:28 INFO Executor:59 - Running task 3.0 in stage 76.0 (TID 166) -2016-05-23 21:24:28 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 164) -2016-05-23 21:24:28 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 163) -2016-05-23 21:24:28 INFO Executor:59 - Running task 2.0 in stage 76.0 (TID 165) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 3.0 in stage 76.0 (TID 166). 1109 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 2.0 in stage 76.0 (TID 165). 1109 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 163). 1109 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 164). 1109 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 76.0 (TID 166) in 3 ms on localhost (1/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 76.0 (TID 165) in 4 ms on localhost (2/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 163) in 6 ms on localhost (3/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 164) in 6 ms on localhost (4/4) -2016-05-23 21:24:28 INFO DAGScheduler:59 - ResultStage 76 (treeAggregate at RowMatrix.scala:124) finished in 0.006 s -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-05-23 21:24:28 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at RowMatrix.scala:124, took 0.014476 s -2016-05-23 21:24:28 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Got job 47 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:24:28 INFO DAGScheduler:59 - Final stage: ResultStage 77(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[91] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=457616, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(1862) called with curMem=460992, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 1862.0 B, free 730.1 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:55093 (size: 1862.0 B, free: 730.5 MB) -2016-05-23 21:24:28 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 77 (MapPartitionsRDD[91] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 4 tasks -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 167, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 168, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 77.0 (TID 169, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 77.0 (TID 170, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:24:28 INFO Executor:59 - Running task 3.0 in stage 77.0 (TID 170) -2016-05-23 21:24:28 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 167) -2016-05-23 21:24:28 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 168) -2016-05-23 21:24:28 INFO Executor:59 - Running task 2.0 in stage 77.0 (TID 169) -2016-05-23 21:24:28 INFO Executor:59 - Finished task 3.0 in stage 77.0 (TID 170). 1294 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 2.0 in stage 77.0 (TID 169). 1249 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 167). 1254 bytes result sent to driver -2016-05-23 21:24:28 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 168). 1254 bytes result sent to driver -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 77.0 (TID 169) in 4 ms on localhost (1/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 168) in 6 ms on localhost (2/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 77.0 (TID 170) in 5 ms on localhost (3/4) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 167) in 7 ms on localhost (4/4) -2016-05-23 21:24:28 INFO DAGScheduler:59 - ResultStage 77 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.007 s -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-05-23 21:24:28 INFO DAGScheduler:59 - Job 47 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.015017 s -2016-05-23 21:24:28 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Registering RDD 89 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Got job 48 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:24:28 INFO DAGScheduler:59 - Final stage: ResultStage 79(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 78) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 78) -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 78 (MapPartitionsRDD[89] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=462854, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-05-23 21:24:28 INFO MemoryStore:59 - ensureFreeSpace(2289) called with curMem=466894, maxMem=766075207 -2016-05-23 21:24:28 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 21:24:28 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:55093 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:24:28 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 78 (MapPartitionsRDD[89] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:24:28 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 4 tasks -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 171, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 172, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 78.0 (TID 173, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:24:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 78.0 (TID 174, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:24:28 INFO Executor:59 - Running task 3.0 in stage 78.0 (TID 174) -2016-05-23 21:24:28 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 172) -2016-05-23 21:24:28 INFO Executor:59 - Running task 2.0 in stage 78.0 (TID 173) -2016-05-23 21:24:28 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 171) -2016-05-23 21:24:29 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 171). 1161 bytes result sent to driver -2016-05-23 21:24:29 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 172). 1161 bytes result sent to driver -2016-05-23 21:24:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 171) in 298 ms on localhost (1/4) -2016-05-23 21:24:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 172) in 297 ms on localhost (2/4) -2016-05-23 21:24:29 INFO Executor:59 - Finished task 2.0 in stage 78.0 (TID 173). 1161 bytes result sent to driver -2016-05-23 21:24:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 78.0 (TID 173) in 304 ms on localhost (3/4) -2016-05-23 21:24:29 INFO Executor:59 - Finished task 3.0 in stage 78.0 (TID 174). 1161 bytes result sent to driver -2016-05-23 21:24:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 78.0 (TID 174) in 315 ms on localhost (4/4) -2016-05-23 21:24:29 INFO DAGScheduler:59 - ShuffleMapStage 78 (flatMap at SpearmanCorrelation.scala:49) finished in 0.317 s -2016-05-23 21:24:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-05-23 21:24:29 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:24:29 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:24:29 INFO DAGScheduler:59 - waiting: Set(ResultStage 79) -2016-05-23 21:24:29 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:24:29 INFO DAGScheduler:59 - Missing parents for ResultStage 79: List() -2016-05-23 21:24:29 INFO DAGScheduler:59 - Submitting ResultStage 79 (ShuffledRDD[92] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:24:29 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=469183, maxMem=766075207 -2016-05-23 21:24:29 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:24:29 INFO MemoryStore:59 - ensureFreeSpace(1573) called with curMem=471727, maxMem=766075207 -2016-05-23 21:24:29 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 1573.0 B, free 730.1 MB) -2016-05-23 21:24:29 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:55093 (size: 1573.0 B, free: 730.5 MB) -2016-05-23 21:24:29 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:29 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 79 (ShuffledRDD[92] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:24:29 INFO TaskSchedulerImpl:59 - Adding task set 79.0 with 3 tasks -2016-05-23 21:24:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 79.0 (TID 175, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 79.0 (TID 176, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 79.0 (TID 177, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:29 INFO Executor:59 - Running task 2.0 in stage 79.0 (TID 177) -2016-05-23 21:24:29 INFO Executor:59 - Running task 0.0 in stage 79.0 (TID 175) -2016-05-23 21:24:29 INFO Executor:59 - Running task 1.0 in stage 79.0 (TID 176) -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:29 INFO Executor:59 - Finished task 0.0 in stage 79.0 (TID 175). 1203 bytes result sent to driver -2016-05-23 21:24:29 INFO Executor:59 - Finished task 2.0 in stage 79.0 (TID 177). 1203 bytes result sent to driver -2016-05-23 21:24:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 79.0 (TID 175) in 10 ms on localhost (1/3) -2016-05-23 21:24:29 INFO Executor:59 - Finished task 1.0 in stage 79.0 (TID 176). 1203 bytes result sent to driver -2016-05-23 21:24:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 79.0 (TID 177) in 9 ms on localhost (2/3) -2016-05-23 21:24:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 79.0 (TID 176) in 10 ms on localhost (3/3) -2016-05-23 21:24:29 INFO DAGScheduler:59 - ResultStage 79 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.011 s -2016-05-23 21:24:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 79.0, whose tasks have all completed, from pool -2016-05-23 21:24:29 INFO DAGScheduler:59 - Job 48 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.343614 s -2016-05-23 21:24:29 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:24:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 166 bytes -2016-05-23 21:24:29 INFO DAGScheduler:59 - Registering RDD 94 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:24:29 INFO DAGScheduler:59 - Got job 49 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:24:29 INFO DAGScheduler:59 - Final stage: ResultStage 82(first at RowMatrix.scala:65) -2016-05-23 21:24:29 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 81) -2016-05-23 21:24:29 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 81) -2016-05-23 21:24:29 INFO DAGScheduler:59 - Submitting ShuffleMapStage 81 (MapPartitionsRDD[94] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:24:29 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=473300, maxMem=766075207 -2016-05-23 21:24:29 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 21:24:29 INFO MemoryStore:59 - ensureFreeSpace(2313) called with curMem=477532, maxMem=766075207 -2016-05-23 21:24:29 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-23 21:24:29 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:55093 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:24:29 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 81 (MapPartitionsRDD[94] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:24:29 INFO TaskSchedulerImpl:59 - Adding task set 81.0 with 4 tasks -2016-05-23 21:24:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 81.0 (TID 178, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:24:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 81.0 (TID 179, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:24:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 81.0 (TID 180, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:24:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 81.0 (TID 181, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:24:29 INFO Executor:59 - Running task 2.0 in stage 81.0 (TID 180) -2016-05-23 21:24:29 INFO Executor:59 - Running task 1.0 in stage 81.0 (TID 179) -2016-05-23 21:24:29 INFO Executor:59 - Running task 0.0 in stage 81.0 (TID 178) -2016-05-23 21:24:29 INFO Executor:59 - Running task 3.0 in stage 81.0 (TID 181) -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:29 INFO Executor:59 - Finished task 1.0 in stage 81.0 (TID 179). 1377 bytes result sent to driver -2016-05-23 21:24:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 81.0 (TID 179) in 45 ms on localhost (1/4) -2016-05-23 21:24:29 INFO Executor:59 - Finished task 3.0 in stage 81.0 (TID 181). 1377 bytes result sent to driver -2016-05-23 21:24:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 81.0 (TID 181) in 91 ms on localhost (2/4) -2016-05-23 21:24:29 INFO Executor:59 - Finished task 2.0 in stage 81.0 (TID 180). 1377 bytes result sent to driver -2016-05-23 21:24:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 81.0 (TID 180) in 121 ms on localhost (3/4) -2016-05-23 21:24:29 INFO Executor:59 - Finished task 0.0 in stage 81.0 (TID 178). 1377 bytes result sent to driver -2016-05-23 21:24:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 81.0 (TID 178) in 134 ms on localhost (4/4) -2016-05-23 21:24:29 INFO DAGScheduler:59 - ShuffleMapStage 81 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.134 s -2016-05-23 21:24:29 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:24:29 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:24:29 INFO DAGScheduler:59 - waiting: Set(ResultStage 82) -2016-05-23 21:24:29 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:24:29 INFO DAGScheduler:59 - Missing parents for ResultStage 82: List() -2016-05-23 21:24:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 81.0, whose tasks have all completed, from pool -2016-05-23 21:24:29 INFO DAGScheduler:59 - Submitting ResultStage 82 (MapPartitionsRDD[96] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:24:29 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=479845, maxMem=766075207 -2016-05-23 21:24:29 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 21:24:29 INFO MemoryStore:59 - ensureFreeSpace(2562) called with curMem=484741, maxMem=766075207 -2016-05-23 21:24:29 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:24:29 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:55093 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:24:29 INFO SparkContext:59 - Created broadcast 61 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 82 (MapPartitionsRDD[96] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:24:29 INFO TaskSchedulerImpl:59 - Adding task set 82.0 with 1 tasks -2016-05-23 21:24:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 82.0 (TID 182, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:29 INFO Executor:59 - Running task 0.0 in stage 82.0 (TID 182) -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:29 INFO Executor:59 - Finished task 0.0 in stage 82.0 (TID 182). 1296 bytes result sent to driver -2016-05-23 21:24:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 82.0 (TID 182) in 8 ms on localhost (1/1) -2016-05-23 21:24:29 INFO DAGScheduler:59 - ResultStage 82 (first at RowMatrix.scala:65) finished in 0.008 s -2016-05-23 21:24:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 82.0, whose tasks have all completed, from pool -2016-05-23 21:24:29 INFO DAGScheduler:59 - Job 49 finished: first at RowMatrix.scala:65, took 0.159621 s -2016-05-23 21:24:29 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:24:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 166 bytes -2016-05-23 21:24:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 169 bytes -2016-05-23 21:24:29 INFO DAGScheduler:59 - Got job 50 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:24:29 INFO DAGScheduler:59 - Final stage: ResultStage 85(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:24:29 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 84) -2016-05-23 21:24:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:29 INFO DAGScheduler:59 - Submitting ResultStage 85 (MapPartitionsRDD[97] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:24:29 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=487303, maxMem=766075207 -2016-05-23 21:24:29 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 21:24:29 INFO MemoryStore:59 - ensureFreeSpace(2943) called with curMem=492999, maxMem=766075207 -2016-05-23 21:24:29 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.1 MB) -2016-05-23 21:24:29 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:55093 (size: 2.9 KB, free: 730.5 MB) -2016-05-23 21:24:29 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 85 (MapPartitionsRDD[97] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:24:29 INFO TaskSchedulerImpl:59 - Adding task set 85.0 with 4 tasks -2016-05-23 21:24:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 85.0 (TID 183, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 85.0 (TID 184, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 85.0 (TID 185, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 85.0 (TID 186, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:29 INFO Executor:59 - Running task 3.0 in stage 85.0 (TID 186) -2016-05-23 21:24:29 INFO Executor:59 - Running task 1.0 in stage 85.0 (TID 184) -2016-05-23 21:24:29 INFO Executor:59 - Running task 2.0 in stage 85.0 (TID 185) -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:29 INFO Executor:59 - Running task 0.0 in stage 85.0 (TID 183) -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:29 INFO Executor:59 - Finished task 1.0 in stage 85.0 (TID 184). 1481 bytes result sent to driver -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:29 INFO Executor:59 - Finished task 0.0 in stage 85.0 (TID 183). 1481 bytes result sent to driver -2016-05-23 21:24:29 INFO Executor:59 - Finished task 2.0 in stage 85.0 (TID 185). 1481 bytes result sent to driver -2016-05-23 21:24:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 85.0 (TID 184) in 25 ms on localhost (1/4) -2016-05-23 21:24:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 85.0 (TID 183) in 27 ms on localhost (2/4) -2016-05-23 21:24:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 85.0 (TID 185) in 26 ms on localhost (3/4) -2016-05-23 21:24:29 INFO Executor:59 - Finished task 3.0 in stage 85.0 (TID 186). 1481 bytes result sent to driver -2016-05-23 21:24:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 85.0 (TID 186) in 37 ms on localhost (4/4) -2016-05-23 21:24:29 INFO DAGScheduler:59 - ResultStage 85 (treeAggregate at RowMatrix.scala:332) finished in 0.039 s -2016-05-23 21:24:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 85.0, whose tasks have all completed, from pool -2016-05-23 21:24:29 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at RowMatrix.scala:332, took 0.047788 s -2016-05-23 21:24:29 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:24:29 INFO DAGScheduler:59 - Got job 51 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:24:29 INFO DAGScheduler:59 - Final stage: ResultStage 88(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:24:29 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 87) -2016-05-23 21:24:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:24:29 INFO DAGScheduler:59 - Submitting ResultStage 88 (MapPartitionsRDD[98] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:24:29 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=495942, maxMem=766075207 -2016-05-23 21:24:29 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 21:24:29 INFO MemoryStore:59 - ensureFreeSpace(2920) called with curMem=501678, maxMem=766075207 -2016-05-23 21:24:29 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.1 MB) -2016-05-23 21:24:29 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:55093 (size: 2.9 KB, free: 730.5 MB) -2016-05-23 21:24:29 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:24:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 88 (MapPartitionsRDD[98] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:24:29 INFO TaskSchedulerImpl:59 - Adding task set 88.0 with 4 tasks -2016-05-23 21:24:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 88.0 (TID 187, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 88.0 (TID 188, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 88.0 (TID 189, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 88.0 (TID 190, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:24:29 INFO Executor:59 - Running task 3.0 in stage 88.0 (TID 190) -2016-05-23 21:24:29 INFO Executor:59 - Running task 1.0 in stage 88.0 (TID 188) -2016-05-23 21:24:29 INFO Executor:59 - Running task 2.0 in stage 88.0 (TID 189) -2016-05-23 21:24:29 INFO Executor:59 - Running task 0.0 in stage 88.0 (TID 187) -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:24:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:24:29 INFO Executor:59 - Finished task 3.0 in stage 88.0 (TID 190). 1359 bytes result sent to driver -2016-05-23 21:24:29 INFO Executor:59 - Finished task 0.0 in stage 88.0 (TID 187). 1359 bytes result sent to driver -2016-05-23 21:24:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 88.0 (TID 190) in 11 ms on localhost (1/4) -2016-05-23 21:24:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 88.0 (TID 187) in 13 ms on localhost (2/4) -2016-05-23 21:24:29 INFO Executor:59 - Finished task 1.0 in stage 88.0 (TID 188). 1359 bytes result sent to driver -2016-05-23 21:24:29 INFO Executor:59 - Finished task 2.0 in stage 88.0 (TID 189). 1359 bytes result sent to driver -2016-05-23 21:24:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 88.0 (TID 188) in 14 ms on localhost (3/4) -2016-05-23 21:24:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 88.0 (TID 189) in 15 ms on localhost (4/4) -2016-05-23 21:24:29 INFO DAGScheduler:59 - ResultStage 88 (treeAggregate at RowMatrix.scala:124) finished in 0.016 s -2016-05-23 21:24:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 88.0, whose tasks have all completed, from pool -2016-05-23 21:24:29 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at RowMatrix.scala:124, took 0.027084 s -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 21:24:29 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 21:24:30 WARN QueuedThreadPool:145 - 4 threads could not be stopped -2016-05-23 21:24:30 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 21:24:30 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 21:24:31 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 21:24:33 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 21:24:33 INFO BlockManager:59 - BlockManager stopped -2016-05-23 21:24:33 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 21:24:33 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 21:24:33 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 21:24:33 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 21:24:33 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-a046c892-3ef0-4588-b723-0d2db5431e12 -2016-05-23 21:24:33 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 21:24:33 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 21:25:02 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 21:25:04 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 21:25:04 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 21:25:04 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 21:25:04 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 21:25:05 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 21:25:05 INFO Remoting:74 - Starting remoting -2016-05-23 21:25:06 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:55120] -2016-05-23 21:25:06 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55120. -2016-05-23 21:25:06 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 21:25:06 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 21:25:06 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-52728e15-531b-4818-989d-dc6190e552aa -2016-05-23 21:25:06 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 21:25:06 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-c6e8ecfb-a719-4081-8df2-f01d127f5d23\httpd-2968f041-93cf-4121-b165-ec3be72a0709 -2016-05-23 21:25:06 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 21:25:06 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 21:25:06 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55121 -2016-05-23 21:25:06 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55121. -2016-05-23 21:25:06 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 21:25:06 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 21:25:06 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 21:25:06 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 21:25:06 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 21:25:07 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 21:25:07 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 21:25:07 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55140. -2016-05-23 21:25:07 INFO NettyBlockTransferService:59 - Server created on 55140 -2016-05-23 21:25:07 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 21:25:07 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55140 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55140) -2016-05-23 21:25:07 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 21:25:08 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 21:25:08 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 21:25:08 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 21:25:08 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 21:25:08 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55140 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 21:25:08 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:22 -2016-05-23 21:25:08 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 21:25:08 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 21:25:08 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 21:25:08 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 21:25:08 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55140 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 21:25:08 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:25 -2016-05-23 21:25:10 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 21:25:10 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 21:25:10 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:29 -2016-05-23 21:25:10 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:29) with 2 output partitions -2016-05-23 21:25:10 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:29) -2016-05-23 21:25:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:10 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:23), which has no missing parents -2016-05-23 21:25:10 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 21:25:10 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 21:25:10 INFO MemoryStore:59 - ensureFreeSpace(2047) called with curMem=346712, maxMem=766075207 -2016-05-23 21:25:10 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2047.0 B, free 730.3 MB) -2016-05-23 21:25:10 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55140 (size: 2047.0 B, free: 730.6 MB) -2016-05-23 21:25:10 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:23) -2016-05-23 21:25:10 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 21:25:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:25:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:25:10 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 21:25:10 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 21:25:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:25:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:25:10 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 21:25:10 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 21:25:10 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 21:25:10 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 21:25:10 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 21:25:10 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 21:25:10 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 21:25:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 207 ms on localhost (1/2) -2016-05-23 21:25:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 269 ms on localhost (2/2) -2016-05-23 21:25:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 21:25:10 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:29) finished in 0.295 s -2016-05-23 21:25:10 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:29, took 0.617070 s -2016-05-23 21:25:11 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 21:25:11 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:31 -2016-05-23 21:25:11 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:31) with 2 output partitions -2016-05-23 21:25:11 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:31) -2016-05-23 21:25:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:11 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:26), which has no missing parents -2016-05-23 21:25:11 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348759, maxMem=766075207 -2016-05-23 21:25:11 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 21:25:11 INFO MemoryStore:59 - ensureFreeSpace(2045) called with curMem=352271, maxMem=766075207 -2016-05-23 21:25:11 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2045.0 B, free 730.2 MB) -2016-05-23 21:25:11 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55140 (size: 2045.0 B, free: 730.6 MB) -2016-05-23 21:25:11 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:26) -2016-05-23 21:25:11 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 21:25:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:25:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:25:11 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 21:25:11 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 21:25:11 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:25:11 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:25:11 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 21:25:11 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 21:25:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 24 ms on localhost (1/2) -2016-05-23 21:25:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 35 ms on localhost (2/2) -2016-05-23 21:25:11 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:31) finished in 0.036 s -2016-05-23 21:25:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 21:25:11 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:31, took 0.089341 s -2016-05-23 21:25:11 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:25:11 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:25:11 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 21:25:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:11 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:25:11 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354316, maxMem=766075207 -2016-05-23 21:25:11 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 21:25:11 INFO MemoryStore:59 - ensureFreeSpace(2465) called with curMem=358876, maxMem=766075207 -2016-05-23 21:25:11 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:25:11 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55140 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 21:25:11 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:11 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 21:25:11 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 21:25:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:25:11 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 21:25:11 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:25:11 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:25:11 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 21:25:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 49 ms on localhost (1/1) -2016-05-23 21:25:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 21:25:11 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.051 s -2016-05-23 21:25:11 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.103780 s -2016-05-23 21:25:12 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:25:12 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:25:12 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:25:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:12 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:25:12 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=361341, maxMem=766075207 -2016-05-23 21:25:12 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 21:25:12 INFO MemoryStore:59 - ensureFreeSpace(2862) called with curMem=366789, maxMem=766075207 -2016-05-23 21:25:12 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:25:12 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55140 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:25:12 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:25:12 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 21:25:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:25:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:25:12 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 21:25:12 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 21:25:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:25:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:25:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:25:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:25:12 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 21:25:12 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 21:25:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 59 ms on localhost (1/2) -2016-05-23 21:25:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 59 ms on localhost (2/2) -2016-05-23 21:25:12 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.061 s -2016-05-23 21:25:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 21:25:12 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 21:25:12 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 21:25:12 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.098068 s -2016-05-23 21:25:12 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:25:12 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:25:12 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:25:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:12 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:25:12 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=369651, maxMem=766075207 -2016-05-23 21:25:12 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 21:25:12 INFO MemoryStore:59 - ensureFreeSpace(2851) called with curMem=375147, maxMem=766075207 -2016-05-23 21:25:12 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:25:12 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55140 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:25:12 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:25:12 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 21:25:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:25:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:25:12 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 21:25:12 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 21:25:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:25:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:25:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:25:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:25:12 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 21:25:12 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 21:25:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 37 ms on localhost (1/2) -2016-05-23 21:25:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 35 ms on localhost (2/2) -2016-05-23 21:25:12 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.038 s -2016-05-23 21:25:12 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.059345 s -2016-05-23 21:25:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 21:25:12 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:25:12 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 21:25:12 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:25:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:12 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:25:12 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=377998, maxMem=766075207 -2016-05-23 21:25:12 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 21:25:12 INFO MemoryStore:59 - ensureFreeSpace(2744) called with curMem=383342, maxMem=766075207 -2016-05-23 21:25:12 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 21:25:12 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55140 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 21:25:12 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:25:12 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 21:25:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:25:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:25:12 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 21:25:12 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 21:25:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:25:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:25:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:25:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:25:12 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 21:25:12 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 21:25:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 44 ms on localhost (1/2) -2016-05-23 21:25:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 43 ms on localhost (2/2) -2016-05-23 21:25:12 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.046 s -2016-05-23 21:25:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 21:25:12 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.075068 s -2016-05-23 21:25:12 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:25:12 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:25:12 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 21:25:12 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:25:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 21:25:12 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 21:25:12 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:25:12 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=386086, maxMem=766075207 -2016-05-23 21:25:12 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 21:25:12 INFO MemoryStore:59 - ensureFreeSpace(3156) called with curMem=392014, maxMem=766075207 -2016-05-23 21:25:12 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 21:25:12 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55140 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 21:25:12 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:25:12 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 21:25:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 21:25:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 21:25:12 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 21:25:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:25:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:25:12 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 21:25:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:25:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:25:12 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 21:25:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 97 ms on localhost (1/2) -2016-05-23 21:25:12 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 21:25:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 125 ms on localhost (2/2) -2016-05-23 21:25:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 21:25:12 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.103 s -2016-05-23 21:25:12 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:25:12 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:25:12 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 21:25:12 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:25:12 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 21:25:12 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:25:12 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=395170, maxMem=766075207 -2016-05-23 21:25:12 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:25:12 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=397674, maxMem=766075207 -2016-05-23 21:25:12 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 21:25:12 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55140 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 21:25:12 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:12 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:25:12 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 21:25:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:12 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 21:25:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:25:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-23 21:25:13 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 21:25:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 89 ms on localhost (1/1) -2016-05-23 21:25:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 21:25:13 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.092 s -2016-05-23 21:25:13 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.315406 s -2016-05-23 21:25:13 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:25:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 21:25:13 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:25:13 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:25:13 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 21:25:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 21:25:13 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 21:25:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:25:13 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=399219, maxMem=766075207 -2016-05-23 21:25:13 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:25:13 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=403411, maxMem=766075207 -2016-05-23 21:25:13 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:25:13 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55140 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:25:13 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:55140 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 21:25:13 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:25:13 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 21:25:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:25:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:25:13 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 21:25:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:25:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:13 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 21:25:13 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 21:25:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:25:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 121 ms on localhost (1/2) -2016-05-23 21:25:13 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 21:25:13 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 21:25:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 290 ms on localhost (2/2) -2016-05-23 21:25:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 21:25:13 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.300 s -2016-05-23 21:25:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:25:13 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:25:13 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 21:25:13 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:25:13 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 21:25:13 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:25:13 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=393556, maxMem=766075207 -2016-05-23 21:25:13 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 21:25:13 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=398412, maxMem=766075207 -2016-05-23 21:25:13 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:25:13 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:55140 in memory (size: 2.7 KB, free: 730.5 MB) -2016-05-23 21:25:13 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55140 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:25:13 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 21:25:13 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:55140 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:25:13 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:13 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:25:13 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 21:25:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:13 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 21:25:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:25:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:13 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 21:25:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 46 ms on localhost (1/1) -2016-05-23 21:25:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 21:25:13 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.047 s -2016-05-23 21:25:13 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.609319 s -2016-05-23 21:25:13 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:25:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 21:25:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 21:25:13 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:25:13 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:25:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 21:25:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:13 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:25:13 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=392596, maxMem=766075207 -2016-05-23 21:25:13 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 21:25:13 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=398252, maxMem=766075207 -2016-05-23 21:25:13 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:25:13 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:55140 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:25:13 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:25:13 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 21:25:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:13 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 21:25:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:25:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:13 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 21:25:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 50 ms on localhost (1/2) -2016-05-23 21:25:13 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 21:25:13 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 21:25:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:25:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:13 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 21:25:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 85 ms on localhost (2/2) -2016-05-23 21:25:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 21:25:13 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.049 s -2016-05-23 21:25:13 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.122467 s -2016-05-23 21:25:13 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:25:13 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:25:13 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:25:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 21:25:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:13 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:25:13 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=398298, maxMem=766075207 -2016-05-23 21:25:13 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:25:13 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=403994, maxMem=766075207 -2016-05-23 21:25:13 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:25:13 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55140 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:25:13 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:55140 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:25:13 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:25:13 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 21:25:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:13 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 21:25:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:25:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:13 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 21:25:13 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 21:25:13 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 21:25:13 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:55140 in memory (size: 3.1 KB, free: 730.5 MB) -2016-05-23 21:25:13 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 21:25:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:25:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:13 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 21:25:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 71 ms on localhost (1/2) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 75 ms on localhost (2/2) -2016-05-23 21:25:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 21:25:14 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.079 s -2016-05-23 21:25:14 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.141764 s -2016-05-23 21:25:14 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:45 -2016-05-23 21:25:14 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:45) with 4 output partitions -2016-05-23 21:25:14 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:45) -2016-05-23 21:25:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:14 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:42), which has no missing parents -2016-05-23 21:25:14 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=392351, maxMem=766075207 -2016-05-23 21:25:14 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:25:14 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=393575, maxMem=766075207 -2016-05-23 21:25:14 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:25:14 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:55140 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:25:14 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:42) -2016-05-23 21:25:14 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:25:14 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 21:25:14 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 21:25:14 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 21:25:14 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 21:25:14 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 21:25:14 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 21:25:14 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 21:25:14 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 21:25:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 25 ms on localhost (1/4) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 41 ms on localhost (2/4) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 26 ms on localhost (3/4) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 31 ms on localhost (4/4) -2016-05-23 21:25:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 21:25:14 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:45) finished in 0.047 s -2016-05-23 21:25:14 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:45, took 0.495256 s -2016-05-23 21:25:14 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:47 -2016-05-23 21:25:14 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:47) with 4 output partitions -2016-05-23 21:25:14 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:47) -2016-05-23 21:25:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:14 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:43), which has no missing parents -2016-05-23 21:25:14 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=394468, maxMem=766075207 -2016-05-23 21:25:14 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:25:14 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=395692, maxMem=766075207 -2016-05-23 21:25:14 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:25:14 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:55140 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:25:14 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:43) -2016-05-23 21:25:14 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:25:14 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 21:25:14 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 21:25:14 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 21:25:14 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 21:25:14 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 21:25:14 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 21:25:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 18 ms on localhost (1/4) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 17 ms on localhost (2/4) -2016-05-23 21:25:14 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 19 ms on localhost (3/4) -2016-05-23 21:25:14 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 21:25:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 23 ms on localhost (4/4) -2016-05-23 21:25:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 21:25:14 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:47) finished in 0.028 s -2016-05-23 21:25:14 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:47, took 0.048634 s -2016-05-23 21:25:14 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:25:14 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:25:14 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 21:25:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:14 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:25:14 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=396585, maxMem=766075207 -2016-05-23 21:25:14 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:25:14 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=399089, maxMem=766075207 -2016-05-23 21:25:14 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 21:25:14 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:55140 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:25:14 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:14 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 21:25:14 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:25:14 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 21:25:14 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 21:25:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 12 ms on localhost (1/1) -2016-05-23 21:25:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 21:25:14 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.017 s -2016-05-23 21:25:14 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.036634 s -2016-05-23 21:25:14 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:25:14 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:25:14 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:25:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:14 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:25:14 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=400613, maxMem=766075207 -2016-05-23 21:25:14 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:25:14 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=404013, maxMem=766075207 -2016-05-23 21:25:14 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 21:25:14 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:55140 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 21:25:14 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:25:14 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:25:14 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 21:25:14 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 21:25:14 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 21:25:14 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 21:25:14 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 21:25:14 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 18 ms on localhost (1/4) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 17 ms on localhost (2/4) -2016-05-23 21:25:14 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 21:25:14 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 21:25:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 23 ms on localhost (3/4) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 28 ms on localhost (4/4) -2016-05-23 21:25:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 21:25:14 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.038 s -2016-05-23 21:25:14 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.065912 s -2016-05-23 21:25:14 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:25:14 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:25:14 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:25:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:14 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:25:14 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=405947, maxMem=766075207 -2016-05-23 21:25:14 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:25:14 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=409387, maxMem=766075207 -2016-05-23 21:25:14 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 21:25:14 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:55140 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:25:14 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:25:14 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:25:14 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 21:25:14 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 21:25:14 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 21:25:14 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 21:25:14 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 21:25:14 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 21:25:14 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 21:25:14 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 21:25:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 47 ms on localhost (1/4) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 47 ms on localhost (2/4) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 52 ms on localhost (3/4) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 48 ms on localhost (4/4) -2016-05-23 21:25:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 21:25:14 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.056 s -2016-05-23 21:25:14 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.081542 s -2016-05-23 21:25:14 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:25:14 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:25:14 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:25:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:14 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:25:14 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=411305, maxMem=766075207 -2016-05-23 21:25:14 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:25:14 INFO MemoryStore:59 - ensureFreeSpace(1860) called with curMem=414681, maxMem=766075207 -2016-05-23 21:25:14 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1860.0 B, free 730.2 MB) -2016-05-23 21:25:14 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:55140 (size: 1860.0 B, free: 730.5 MB) -2016-05-23 21:25:14 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:25:14 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:25:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:25:14 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 21:25:14 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 21:25:14 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 21:25:14 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 21:25:14 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 21:25:15 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 21:25:15 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 21:25:15 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 21:25:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 41 ms on localhost (1/4) -2016-05-23 21:25:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 41 ms on localhost (2/4) -2016-05-23 21:25:15 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 41 ms on localhost (3/4) -2016-05-23 21:25:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 47 ms on localhost (4/4) -2016-05-23 21:25:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 21:25:15 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.103 s -2016-05-23 21:25:15 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.128108 s -2016-05-23 21:25:15 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:25:15 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:25:15 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:25:15 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:25:15 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 21:25:15 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 21:25:15 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:25:15 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=416541, maxMem=766075207 -2016-05-23 21:25:15 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 21:25:15 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=420581, maxMem=766075207 -2016-05-23 21:25:15 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:25:15 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:55140 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:25:15 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:25:15 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 21:25:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:25:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 21:25:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:25:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 21:25:15 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 21:25:15 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 21:25:15 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 21:25:15 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 21:25:15 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 21:25:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 544 ms on localhost (1/4) -2016-05-23 21:25:15 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 21:25:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 558 ms on localhost (2/4) -2016-05-23 21:25:15 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 21:25:15 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 21:25:15 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 598 ms on localhost (3/4) -2016-05-23 21:25:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 605 ms on localhost (4/4) -2016-05-23 21:25:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 21:25:15 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.618 s -2016-05-23 21:25:15 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:25:15 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:25:15 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 21:25:15 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:25:15 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 21:25:15 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:25:15 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=422863, maxMem=766075207 -2016-05-23 21:25:15 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:25:15 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=425407, maxMem=766075207 -2016-05-23 21:25:15 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 21:25:15 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:55140 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 21:25:15 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:15 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:25:15 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 21:25:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:15 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 21:25:15 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 21:25:15 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 21:25:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:15 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 21:25:15 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 21:25:15 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 21:25:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 41 ms on localhost (1/3) -2016-05-23 21:25:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 44 ms on localhost (2/3) -2016-05-23 21:25:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 45 ms on localhost (3/3) -2016-05-23 21:25:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 21:25:15 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.050 s -2016-05-23 21:25:15 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.744761 s -2016-05-23 21:25:15 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:25:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 21:25:15 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:25:15 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:25:15 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 21:25:15 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 21:25:15 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 21:25:15 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:25:15 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=426975, maxMem=766075207 -2016-05-23 21:25:15 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:25:15 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=431207, maxMem=766075207 -2016-05-23 21:25:15 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 21:25:15 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:55140 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:25:15 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:25:15 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 21:25:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:25:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:25:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:25:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:25:15 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 21:25:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:15 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 21:25:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:15 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 21:25:15 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 21:25:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:16 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 149 ms on localhost (1/4) -2016-05-23 21:25:16 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 153 ms on localhost (2/4) -2016-05-23 21:25:16 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 21:25:16 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 161 ms on localhost (3/4) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 164 ms on localhost (4/4) -2016-05-23 21:25:16 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.167 s -2016-05-23 21:25:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:25:16 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:25:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 21:25:16 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 21:25:16 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:25:16 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 21:25:16 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:25:16 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=433513, maxMem=766075207 -2016-05-23 21:25:16 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:25:16 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=438409, maxMem=766075207 -2016-05-23 21:25:16 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:25:16 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:55140 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:25:16 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:16 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:25:16 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:16 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 21:25:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:16 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 20 ms on localhost (1/1) -2016-05-23 21:25:16 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.023 s -2016-05-23 21:25:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 21:25:16 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.235320 s -2016-05-23 21:25:16 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 21:25:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 21:25:16 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:25:16 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:25:16 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 21:25:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:16 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:25:16 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=440951, maxMem=766075207 -2016-05-23 21:25:16 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:25:16 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=446647, maxMem=766075207 -2016-05-23 21:25:16 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:25:16 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:55140 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:25:16 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:25:16 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:16 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 21:25:16 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 21:25:16 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 21:25:16 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 21:25:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:16 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 21:25:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 20 ms on localhost (1/4) -2016-05-23 21:25:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 21:25:16 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 29 ms on localhost (2/4) -2016-05-23 21:25:16 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 33 ms on localhost (3/4) -2016-05-23 21:25:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:16 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 61 ms on localhost (4/4) -2016-05-23 21:25:16 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.062 s -2016-05-23 21:25:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 21:25:16 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.077569 s -2016-05-23 21:25:16 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:25:16 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:25:16 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:25:16 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 21:25:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:16 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:25:16 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=449556, maxMem=766075207 -2016-05-23 21:25:16 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:25:16 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=455292, maxMem=766075207 -2016-05-23 21:25:16 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 21:25:16 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:55140 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:25:16 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:25:16 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:16 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 21:25:16 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 21:25:16 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 21:25:16 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 21:25:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:16 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 21:25:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 23 ms on localhost (1/4) -2016-05-23 21:25:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 21:25:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:16 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 38 ms on localhost (2/4) -2016-05-23 21:25:16 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 42 ms on localhost (3/4) -2016-05-23 21:25:16 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 46 ms on localhost (4/4) -2016-05-23 21:25:16 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.047 s -2016-05-23 21:25:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 21:25:16 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.064903 s -2016-05-23 21:25:16 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:59 -2016-05-23 21:25:16 INFO DAGScheduler:59 - Got job 20 (foreach at StatisticsCorrLearning.scala:59) with 4 output partitions -2016-05-23 21:25:16 INFO DAGScheduler:59 - Final stage: ResultStage 34(foreach at StatisticsCorrLearning.scala:59) -2016-05-23 21:25:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:16 INFO DAGScheduler:59 - Submitting ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:56), which has no missing parents -2016-05-23 21:25:16 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=458184, maxMem=766075207 -2016-05-23 21:25:16 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-05-23 21:25:16 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=459408, maxMem=766075207 -2016-05-23 21:25:16 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.1 MB) -2016-05-23 21:25:16 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:55140 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:25:16 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:56) -2016-05-23 21:25:16 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 62, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 63, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 64, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:25:16 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 64) -2016-05-23 21:25:16 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 64). 915 bytes result sent to driver -2016-05-23 21:25:16 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 65) -2016-05-23 21:25:16 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 65). 915 bytes result sent to driver -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 64) in 11 ms on localhost (1/4) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 65) in 12 ms on localhost (2/4) -2016-05-23 21:25:16 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 63) -2016-05-23 21:25:16 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 63). 915 bytes result sent to driver -2016-05-23 21:25:16 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 62) -2016-05-23 21:25:16 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 62). 915 bytes result sent to driver -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 63) in 27 ms on localhost (3/4) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 62) in 29 ms on localhost (4/4) -2016-05-23 21:25:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-23 21:25:16 INFO DAGScheduler:59 - ResultStage 34 (foreach at StatisticsCorrLearning.scala:59) finished in 0.037 s -2016-05-23 21:25:16 INFO DAGScheduler:59 - Job 20 finished: foreach at StatisticsCorrLearning.scala:59, took 0.117764 s -2016-05-23 21:25:16 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:61 -2016-05-23 21:25:16 INFO DAGScheduler:59 - Got job 21 (foreach at StatisticsCorrLearning.scala:61) with 4 output partitions -2016-05-23 21:25:16 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at StatisticsCorrLearning.scala:61) -2016-05-23 21:25:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:16 INFO DAGScheduler:59 - Submitting ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:57), which has no missing parents -2016-05-23 21:25:16 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=460301, maxMem=766075207 -2016-05-23 21:25:16 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-05-23 21:25:16 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=461525, maxMem=766075207 -2016-05-23 21:25:16 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.1 MB) -2016-05-23 21:25:16 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:55140 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:25:16 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:57) -2016-05-23 21:25:16 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 4 tasks -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 66, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:25:16 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 67) -2016-05-23 21:25:16 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 66) -2016-05-23 21:25:16 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 67). 915 bytes result sent to driver -2016-05-23 21:25:16 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 66). 915 bytes result sent to driver -2016-05-23 21:25:16 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 68) -2016-05-23 21:25:16 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 68). 915 bytes result sent to driver -2016-05-23 21:25:16 INFO Executor:59 - Running task 3.0 in stage 35.0 (TID 69) -2016-05-23 21:25:16 INFO Executor:59 - Finished task 3.0 in stage 35.0 (TID 69). 915 bytes result sent to driver -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 67) in 89 ms on localhost (1/4) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 66) in 92 ms on localhost (2/4) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 68) in 91 ms on localhost (3/4) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 35.0 (TID 69) in 91 ms on localhost (4/4) -2016-05-23 21:25:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-23 21:25:16 INFO DAGScheduler:59 - ResultStage 35 (foreach at StatisticsCorrLearning.scala:61) finished in 0.158 s -2016-05-23 21:25:16 INFO DAGScheduler:59 - Job 21 finished: foreach at StatisticsCorrLearning.scala:61, took 0.196422 s -2016-05-23 21:25:16 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:25:16 INFO DAGScheduler:59 - Got job 22 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:25:16 INFO DAGScheduler:59 - Final stage: ResultStage 36(first at RowMatrix.scala:65) -2016-05-23 21:25:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:16 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:25:16 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=462418, maxMem=766075207 -2016-05-23 21:25:16 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-05-23 21:25:16 INFO MemoryStore:59 - ensureFreeSpace(1522) called with curMem=464922, maxMem=766075207 -2016-05-23 21:25:16 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1522.0 B, free 730.1 MB) -2016-05-23 21:25:16 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:55140 (size: 1522.0 B, free: 730.5 MB) -2016-05-23 21:25:16 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:16 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 21:25:16 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:16 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 70) -2016-05-23 21:25:16 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 70). 1046 bytes result sent to driver -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 70) in 10 ms on localhost (1/1) -2016-05-23 21:25:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-23 21:25:16 INFO DAGScheduler:59 - ResultStage 36 (first at RowMatrix.scala:65) finished in 0.011 s -2016-05-23 21:25:16 INFO DAGScheduler:59 - Job 22 finished: first at RowMatrix.scala:65, took 0.041882 s -2016-05-23 21:25:16 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:25:16 INFO DAGScheduler:59 - Got job 23 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:25:16 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:25:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:16 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:25:16 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=466444, maxMem=766075207 -2016-05-23 21:25:16 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 21:25:16 INFO MemoryStore:59 - ensureFreeSpace(1932) called with curMem=469844, maxMem=766075207 -2016-05-23 21:25:16 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1932.0 B, free 730.1 MB) -2016-05-23 21:25:16 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:55140 (size: 1932.0 B, free: 730.5 MB) -2016-05-23 21:25:16 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:25:16 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 4 tasks -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:25:16 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-23 21:25:16 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 1231 bytes result sent to driver -2016-05-23 21:25:16 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 72) -2016-05-23 21:25:16 INFO Executor:59 - Running task 3.0 in stage 37.0 (TID 74) -2016-05-23 21:25:16 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 72). 1231 bytes result sent to driver -2016-05-23 21:25:16 INFO Executor:59 - Finished task 3.0 in stage 37.0 (TID 74). 1231 bytes result sent to driver -2016-05-23 21:25:16 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 73) -2016-05-23 21:25:16 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 73). 1231 bytes result sent to driver -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 22 ms on localhost (1/4) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 72) in 21 ms on localhost (2/4) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 73) in 21 ms on localhost (3/4) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 37.0 (TID 74) in 23 ms on localhost (4/4) -2016-05-23 21:25:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-23 21:25:16 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at RowMatrix.scala:332) finished in 0.032 s -2016-05-23 21:25:16 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at RowMatrix.scala:332, took 0.074765 s -2016-05-23 21:25:16 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:25:16 INFO DAGScheduler:59 - Got job 24 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:25:16 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:25:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:16 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:25:16 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=471776, maxMem=766075207 -2016-05-23 21:25:16 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-23 21:25:16 INFO MemoryStore:59 - ensureFreeSpace(1916) called with curMem=475216, maxMem=766075207 -2016-05-23 21:25:16 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1916.0 B, free 730.1 MB) -2016-05-23 21:25:16 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:55140 (size: 1916.0 B, free: 730.5 MB) -2016-05-23 21:25:16 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:25:16 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 77, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 78, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:25:16 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 77) -2016-05-23 21:25:16 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 77). 1109 bytes result sent to driver -2016-05-23 21:25:16 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 78) -2016-05-23 21:25:16 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 78). 1109 bytes result sent to driver -2016-05-23 21:25:16 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-23 21:25:16 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-23 21:25:16 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 1109 bytes result sent to driver -2016-05-23 21:25:16 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 1109 bytes result sent to driver -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 77) in 42 ms on localhost (1/4) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 78) in 42 ms on localhost (2/4) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 47 ms on localhost (3/4) -2016-05-23 21:25:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 46 ms on localhost (4/4) -2016-05-23 21:25:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-23 21:25:17 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at RowMatrix.scala:124) finished in 0.196 s -2016-05-23 21:25:17 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at RowMatrix.scala:124, took 0.235475 s -2016-05-23 21:25:17 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:25:17 INFO DAGScheduler:59 - Got job 25 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:25:17 INFO DAGScheduler:59 - Final stage: ResultStage 39(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:25:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:17 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:25:17 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=477132, maxMem=766075207 -2016-05-23 21:25:17 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 21:25:17 INFO MemoryStore:59 - ensureFreeSpace(1851) called with curMem=480508, maxMem=766075207 -2016-05-23 21:25:17 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1851.0 B, free 730.1 MB) -2016-05-23 21:25:17 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:55140 (size: 1851.0 B, free: 730.5 MB) -2016-05-23 21:25:17 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:25:17 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 4 tasks -2016-05-23 21:25:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 79, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 80, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 81, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 39.0 (TID 82, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:25:17 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 81) -2016-05-23 21:25:17 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 81). 1249 bytes result sent to driver -2016-05-23 21:25:17 INFO Executor:59 - Running task 3.0 in stage 39.0 (TID 82) -2016-05-23 21:25:17 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 79) -2016-05-23 21:25:17 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 80) -2016-05-23 21:25:17 INFO Executor:59 - Finished task 3.0 in stage 39.0 (TID 82). 1294 bytes result sent to driver -2016-05-23 21:25:17 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 79). 1254 bytes result sent to driver -2016-05-23 21:25:17 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 80). 1254 bytes result sent to driver -2016-05-23 21:25:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 81) in 18 ms on localhost (1/4) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 39.0 (TID 82) in 18 ms on localhost (2/4) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 79) in 22 ms on localhost (3/4) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 80) in 23 ms on localhost (4/4) -2016-05-23 21:25:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-23 21:25:17 INFO DAGScheduler:59 - ResultStage 39 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.043 s -2016-05-23 21:25:17 INFO DAGScheduler:59 - Job 25 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.092973 s -2016-05-23 21:25:17 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:25:17 INFO DAGScheduler:59 - Registering RDD 51 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:25:17 INFO DAGScheduler:59 - Got job 26 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:25:17 INFO DAGScheduler:59 - Final stage: ResultStage 41(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:25:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 40) -2016-05-23 21:25:17 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 40) -2016-05-23 21:25:17 INFO DAGScheduler:59 - Submitting ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:25:17 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=482359, maxMem=766075207 -2016-05-23 21:25:17 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-05-23 21:25:17 INFO MemoryStore:59 - ensureFreeSpace(2281) called with curMem=486399, maxMem=766075207 -2016-05-23 21:25:17 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 21:25:17 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:55140 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:25:17 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:25:17 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 4 tasks -2016-05-23 21:25:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 83, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 84, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 85, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 40.0 (TID 86, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:25:17 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 85) -2016-05-23 21:25:17 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 83) -2016-05-23 21:25:17 INFO Executor:59 - Running task 3.0 in stage 40.0 (TID 86) -2016-05-23 21:25:17 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 84) -2016-05-23 21:25:17 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 83). 1161 bytes result sent to driver -2016-05-23 21:25:17 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 85). 1161 bytes result sent to driver -2016-05-23 21:25:17 INFO Executor:59 - Finished task 3.0 in stage 40.0 (TID 86). 1161 bytes result sent to driver -2016-05-23 21:25:17 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 84). 1161 bytes result sent to driver -2016-05-23 21:25:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 83) in 100 ms on localhost (1/4) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 85) in 98 ms on localhost (2/4) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 40.0 (TID 86) in 98 ms on localhost (3/4) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 84) in 102 ms on localhost (4/4) -2016-05-23 21:25:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-23 21:25:17 INFO DAGScheduler:59 - ShuffleMapStage 40 (flatMap at SpearmanCorrelation.scala:49) finished in 0.131 s -2016-05-23 21:25:17 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:25:17 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:25:17 INFO DAGScheduler:59 - waiting: Set(ResultStage 41) -2016-05-23 21:25:17 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:25:17 INFO DAGScheduler:59 - Missing parents for ResultStage 41: List() -2016-05-23 21:25:17 INFO DAGScheduler:59 - Submitting ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:25:17 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=488680, maxMem=766075207 -2016-05-23 21:25:17 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:25:17 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=491224, maxMem=766075207 -2016-05-23 21:25:17 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.1 MB) -2016-05-23 21:25:17 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:55140 (size: 1567.0 B, free: 730.5 MB) -2016-05-23 21:25:17 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:17 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:25:17 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 3 tasks -2016-05-23 21:25:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 87, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:17 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 87) -2016-05-23 21:25:17 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 89) -2016-05-23 21:25:17 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 88) -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:17 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 88). 1203 bytes result sent to driver -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:17 INFO Executor:59 - Finished task 2.0 in stage 41.0 (TID 89). 1203 bytes result sent to driver -2016-05-23 21:25:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 88) in 33 ms on localhost (1/3) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 41.0 (TID 89) in 33 ms on localhost (2/3) -2016-05-23 21:25:17 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 87). 1203 bytes result sent to driver -2016-05-23 21:25:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 87) in 45 ms on localhost (3/3) -2016-05-23 21:25:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-23 21:25:17 INFO DAGScheduler:59 - ResultStage 41 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.047 s -2016-05-23 21:25:17 INFO DAGScheduler:59 - Job 26 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.250472 s -2016-05-23 21:25:17 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:25:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 163 bytes -2016-05-23 21:25:17 INFO DAGScheduler:59 - Registering RDD 56 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:25:17 INFO DAGScheduler:59 - Got job 27 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:25:17 INFO DAGScheduler:59 - Final stage: ResultStage 44(first at RowMatrix.scala:65) -2016-05-23 21:25:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 43) -2016-05-23 21:25:17 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 43) -2016-05-23 21:25:17 INFO DAGScheduler:59 - Submitting ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:25:17 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=492791, maxMem=766075207 -2016-05-23 21:25:17 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 21:25:17 INFO MemoryStore:59 - ensureFreeSpace(2305) called with curMem=497023, maxMem=766075207 -2016-05-23 21:25:17 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-23 21:25:17 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:55140 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:25:17 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:25:17 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 4 tasks -2016-05-23 21:25:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 90, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 91, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 43.0 (TID 92, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 43.0 (TID 93, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:25:17 INFO Executor:59 - Running task 3.0 in stage 43.0 (TID 93) -2016-05-23 21:25:17 INFO Executor:59 - Running task 2.0 in stage 43.0 (TID 92) -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:17 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 90) -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:17 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 91) -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 18 ms -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:17 INFO Executor:59 - Finished task 2.0 in stage 43.0 (TID 92). 1377 bytes result sent to driver -2016-05-23 21:25:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 43.0 (TID 92) in 66 ms on localhost (1/4) -2016-05-23 21:25:17 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 90). 1377 bytes result sent to driver -2016-05-23 21:25:17 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 91). 1377 bytes result sent to driver -2016-05-23 21:25:17 INFO Executor:59 - Finished task 3.0 in stage 43.0 (TID 93). 1377 bytes result sent to driver -2016-05-23 21:25:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 90) in 101 ms on localhost (2/4) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 91) in 101 ms on localhost (3/4) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 43.0 (TID 93) in 100 ms on localhost (4/4) -2016-05-23 21:25:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-23 21:25:17 INFO DAGScheduler:59 - ShuffleMapStage 43 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.125 s -2016-05-23 21:25:17 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:25:17 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:25:17 INFO DAGScheduler:59 - waiting: Set(ResultStage 44) -2016-05-23 21:25:17 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:25:17 INFO DAGScheduler:59 - Missing parents for ResultStage 44: List() -2016-05-23 21:25:17 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:25:17 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=499328, maxMem=766075207 -2016-05-23 21:25:17 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 21:25:17 INFO MemoryStore:59 - ensureFreeSpace(2545) called with curMem=504224, maxMem=766075207 -2016-05-23 21:25:17 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:25:17 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:55140 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:25:17 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:25:17 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 1 tasks -2016-05-23 21:25:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 94, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:17 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 94) -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:17 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 94). 1296 bytes result sent to driver -2016-05-23 21:25:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 94) in 21 ms on localhost (1/1) -2016-05-23 21:25:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-23 21:25:17 INFO DAGScheduler:59 - ResultStage 44 (first at RowMatrix.scala:65) finished in 0.022 s -2016-05-23 21:25:17 INFO DAGScheduler:59 - Job 27 finished: first at RowMatrix.scala:65, took 0.259705 s -2016-05-23 21:25:17 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:25:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 163 bytes -2016-05-23 21:25:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 166 bytes -2016-05-23 21:25:17 INFO DAGScheduler:59 - Got job 28 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:25:17 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:25:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 46) -2016-05-23 21:25:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:17 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:25:17 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=506769, maxMem=766075207 -2016-05-23 21:25:17 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 21:25:17 INFO MemoryStore:59 - ensureFreeSpace(2914) called with curMem=512465, maxMem=766075207 -2016-05-23 21:25:17 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 21:25:17 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:55140 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:25:17 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:25:17 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 4 tasks -2016-05-23 21:25:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 95, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 96, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 47.0 (TID 97, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 47.0 (TID 98, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:17 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 96) -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:17 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 96). 1481 bytes result sent to driver -2016-05-23 21:25:17 INFO Executor:59 - Running task 3.0 in stage 47.0 (TID 98) -2016-05-23 21:25:17 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 95) -2016-05-23 21:25:17 INFO Executor:59 - Running task 2.0 in stage 47.0 (TID 97) -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:17 INFO Executor:59 - Finished task 3.0 in stage 47.0 (TID 98). 1481 bytes result sent to driver -2016-05-23 21:25:17 INFO Executor:59 - Finished task 2.0 in stage 47.0 (TID 97). 1481 bytes result sent to driver -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:17 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 95). 1481 bytes result sent to driver -2016-05-23 21:25:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 96) in 48 ms on localhost (1/4) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 47.0 (TID 98) in 46 ms on localhost (2/4) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 95) in 50 ms on localhost (3/4) -2016-05-23 21:25:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 47.0 (TID 97) in 51 ms on localhost (4/4) -2016-05-23 21:25:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-23 21:25:17 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at RowMatrix.scala:332) finished in 0.067 s -2016-05-23 21:25:17 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at RowMatrix.scala:332, took 0.112262 s -2016-05-23 21:25:17 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:25:17 INFO DAGScheduler:59 - Got job 29 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:25:17 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:25:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 49) -2016-05-23 21:25:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:17 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:25:17 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=515379, maxMem=766075207 -2016-05-23 21:25:17 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 21:25:17 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=521115, maxMem=766075207 -2016-05-23 21:25:17 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:55140 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:25:18 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 4 tasks -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 50.0 (TID 101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 50.0 (TID 102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:18 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:18 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 1359 bytes result sent to driver -2016-05-23 21:25:18 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:18 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 1359 bytes result sent to driver -2016-05-23 21:25:18 INFO Executor:59 - Running task 2.0 in stage 50.0 (TID 101) -2016-05-23 21:25:18 INFO Executor:59 - Running task 3.0 in stage 50.0 (TID 102) -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 47 ms on localhost (1/4) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 46 ms on localhost (2/4) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 2.0 in stage 50.0 (TID 101). 1359 bytes result sent to driver -2016-05-23 21:25:18 INFO Executor:59 - Finished task 3.0 in stage 50.0 (TID 102). 1359 bytes result sent to driver -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 50.0 (TID 101) in 56 ms on localhost (3/4) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 50.0 (TID 102) in 60 ms on localhost (4/4) -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-23 21:25:18 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at RowMatrix.scala:124) finished in 0.069 s -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:55140 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-23 21:25:18 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at RowMatrix.scala:124, took 0.284182 s -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:55140 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:25:18 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:72 -2016-05-23 21:25:18 INFO DAGScheduler:59 - Got job 30 (foreach at StatisticsCorrLearning.scala:72) with 4 output partitions -2016-05-23 21:25:18 INFO DAGScheduler:59 - Final stage: ResultStage 51(foreach at StatisticsCorrLearning.scala:72) -2016-05-23 21:25:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting ResultStage 51 (ParallelCollectionRDD[61] at parallelize at StatisticsCorrLearning.scala:69), which has no missing parents -2016-05-23 21:25:18 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=507960, maxMem=766075207 -2016-05-23 21:25:18 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-05-23 21:25:18 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=509184, maxMem=766075207 -2016-05-23 21:25:18 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.1 MB) -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:55140 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 51 (ParallelCollectionRDD[61] at parallelize at StatisticsCorrLearning.scala:69) -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 4 tasks -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 103, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 104, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 51.0 (TID 105, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 51.0 (TID 106, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:25:18 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 103) -2016-05-23 21:25:18 INFO Executor:59 - Running task 2.0 in stage 51.0 (TID 105) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 103). 915 bytes result sent to driver -2016-05-23 21:25:18 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 104) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 2.0 in stage 51.0 (TID 105). 915 bytes result sent to driver -2016-05-23 21:25:18 INFO Executor:59 - Running task 3.0 in stage 51.0 (TID 106) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 104). 915 bytes result sent to driver -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 103) in 13 ms on localhost (1/4) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 51.0 (TID 105) in 12 ms on localhost (2/4) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 3.0 in stage 51.0 (TID 106). 915 bytes result sent to driver -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 51.0 (TID 106) in 14 ms on localhost (3/4) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 104) in 15 ms on localhost (4/4) -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-23 21:25:18 INFO DAGScheduler:59 - ResultStage 51 (foreach at StatisticsCorrLearning.scala:72) finished in 0.018 s -2016-05-23 21:25:18 INFO DAGScheduler:59 - Job 30 finished: foreach at StatisticsCorrLearning.scala:72, took 0.033217 s -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:55140 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:55140 in memory (size: 1567.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:55140 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:55140 in memory (size: 1851.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:74 -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-23 21:25:18 INFO DAGScheduler:59 - Got job 31 (foreach at StatisticsCorrLearning.scala:74) with 4 output partitions -2016-05-23 21:25:18 INFO DAGScheduler:59 - Final stage: ResultStage 52(foreach at StatisticsCorrLearning.scala:74) -2016-05-23 21:25:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting ResultStage 52 (ParallelCollectionRDD[62] at parallelize at StatisticsCorrLearning.scala:70), which has no missing parents -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:55140 in memory (size: 1916.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:55140 in memory (size: 1932.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:55140 in memory (size: 1522.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:55140 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:55140 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:55140 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-23 21:25:18 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=460305, maxMem=766075207 -2016-05-23 21:25:18 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-05-23 21:25:18 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=461529, maxMem=766075207 -2016-05-23 21:25:18 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.1 MB) -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:55140 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 52 (ParallelCollectionRDD[62] at parallelize at StatisticsCorrLearning.scala:70) -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 4 tasks -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:55140 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:55140 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:55140 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:55140 in memory (size: 1568.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:55140 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 107, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 108, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 52.0 (TID 109, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 52.0 (TID 110, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:25:18 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 108) -2016-05-23 21:25:18 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 107) -2016-05-23 21:25:18 INFO Executor:59 - Running task 2.0 in stage 52.0 (TID 109) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 108). 915 bytes result sent to driver -2016-05-23 21:25:18 INFO Executor:59 - Running task 3.0 in stage 52.0 (TID 110) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 2.0 in stage 52.0 (TID 109). 915 bytes result sent to driver -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-23 21:25:18 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 107). 915 bytes result sent to driver -2016-05-23 21:25:18 INFO Executor:59 - Finished task 3.0 in stage 52.0 (TID 110). 915 bytes result sent to driver -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 108) in 12 ms on localhost (1/4) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 52.0 (TID 109) in 11 ms on localhost (2/4) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 107) in 15 ms on localhost (3/4) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 52.0 (TID 110) in 13 ms on localhost (4/4) -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-23 21:25:18 INFO DAGScheduler:59 - ResultStage 52 (foreach at StatisticsCorrLearning.scala:74) finished in 0.023 s -2016-05-23 21:25:18 INFO DAGScheduler:59 - Job 31 finished: foreach at StatisticsCorrLearning.scala:74, took 0.195043 s -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:55140 in memory (size: 1860.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:25:18 INFO DAGScheduler:59 - Got job 32 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:25:18 INFO DAGScheduler:59 - Final stage: ResultStage 53(first at RowMatrix.scala:65) -2016-05-23 21:25:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[64] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:55140 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:25:18 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=417697, maxMem=766075207 -2016-05-23 21:25:18 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:25:18 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=420201, maxMem=766075207 -2016-05-23 21:25:18 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:55140 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 53 (MapPartitionsRDD[64] at map at Correlation.scala:45) -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 1 tasks -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 111, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-23 21:25:18 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 111) -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:55140 in memory (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 111). 935 bytes result sent to driver -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 111) in 19 ms on localhost (1/1) -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-23 21:25:18 INFO DAGScheduler:59 - ResultStage 53 (first at RowMatrix.scala:65) finished in 0.020 s -2016-05-23 21:25:18 INFO DAGScheduler:59 - Job 32 finished: first at RowMatrix.scala:65, took 0.037219 s -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:55140 in memory (size: 1934.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-23 21:25:18 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:25:18 INFO DAGScheduler:59 - Got job 33 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 21:25:18 INFO DAGScheduler:59 - Final stage: ResultStage 54(first at RowMatrix.scala:65) -2016-05-23 21:25:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[64] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:55140 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-23 21:25:18 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=407005, maxMem=766075207 -2016-05-23 21:25:18 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:25:18 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=408616, maxMem=766075207 -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:55140 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:55140 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 54 (MapPartitionsRDD[64] at map at Correlation.scala:45) -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 3 tasks -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 112, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:55140 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 113, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 54.0 (TID 114, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:18 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 112) -2016-05-23 21:25:18 INFO Executor:59 - Running task 2.0 in stage 54.0 (TID 114) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-23 21:25:18 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 113) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 112). 1046 bytes result sent to driver -2016-05-23 21:25:18 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 113). 1046 bytes result sent to driver -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:55140 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 2.0 in stage 54.0 (TID 114). 1046 bytes result sent to driver -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 54.0 (TID 114) in 6 ms on localhost (1/3) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 112) in 8 ms on localhost (2/3) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-23 21:25:18 INFO DAGScheduler:59 - ResultStage 54 (first at RowMatrix.scala:65) finished in 0.012 s -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 113) in 9 ms on localhost (3/3) -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-23 21:25:18 INFO DAGScheduler:59 - Job 33 finished: first at RowMatrix.scala:65, took 0.026483 s -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:55140 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:55140 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:25:18 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-23 21:25:18 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:25:18 INFO DAGScheduler:59 - Got job 34 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:25:18 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:25:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[65] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:25:18 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=382263, maxMem=766075207 -2016-05-23 21:25:18 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:25:18 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=385663, maxMem=766075207 -2016-05-23 21:25:18 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:55140 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 55 (MapPartitionsRDD[65] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 4 tasks -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 115, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 116, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 55.0 (TID 117, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 55.0 (TID 118, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:18 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 115) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 115). 1231 bytes result sent to driver -2016-05-23 21:25:18 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 116) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 116). 1231 bytes result sent to driver -2016-05-23 21:25:18 INFO Executor:59 - Running task 2.0 in stage 55.0 (TID 117) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 2.0 in stage 55.0 (TID 117). 1231 bytes result sent to driver -2016-05-23 21:25:18 INFO Executor:59 - Running task 3.0 in stage 55.0 (TID 118) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 3.0 in stage 55.0 (TID 118). 1231 bytes result sent to driver -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 115) in 26 ms on localhost (1/4) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 116) in 26 ms on localhost (2/4) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 55.0 (TID 117) in 26 ms on localhost (3/4) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 55.0 (TID 118) in 26 ms on localhost (4/4) -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-23 21:25:18 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at RowMatrix.scala:332) finished in 0.034 s -2016-05-23 21:25:18 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at RowMatrix.scala:332, took 0.078595 s -2016-05-23 21:25:18 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:25:18 INFO DAGScheduler:59 - Got job 35 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:25:18 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:25:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[66] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:25:18 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=387597, maxMem=766075207 -2016-05-23 21:25:18 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:25:18 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=391037, maxMem=766075207 -2016-05-23 21:25:18 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:55140 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 56 (MapPartitionsRDD[66] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 4 tasks -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 119, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 120, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 56.0 (TID 121, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 56.0 (TID 122, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:18 INFO Executor:59 - Running task 2.0 in stage 56.0 (TID 121) -2016-05-23 21:25:18 INFO Executor:59 - Running task 3.0 in stage 56.0 (TID 122) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 3.0 in stage 56.0 (TID 122). 1109 bytes result sent to driver -2016-05-23 21:25:18 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 120) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 120). 1109 bytes result sent to driver -2016-05-23 21:25:18 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 119) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 2.0 in stage 56.0 (TID 121). 1109 bytes result sent to driver -2016-05-23 21:25:18 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 119). 1109 bytes result sent to driver -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 56.0 (TID 122) in 13 ms on localhost (1/4) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 120) in 16 ms on localhost (2/4) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 56.0 (TID 121) in 16 ms on localhost (3/4) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 119) in 19 ms on localhost (4/4) -2016-05-23 21:25:18 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at RowMatrix.scala:124) finished in 0.020 s -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-23 21:25:18 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at RowMatrix.scala:124, took 0.035170 s -2016-05-23 21:25:18 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:25:18 INFO DAGScheduler:59 - Got job 36 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:25:18 INFO DAGScheduler:59 - Final stage: ResultStage 57(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:25:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[72] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:25:18 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=392955, maxMem=766075207 -2016-05-23 21:25:18 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:25:18 INFO MemoryStore:59 - ensureFreeSpace(1858) called with curMem=396331, maxMem=766075207 -2016-05-23 21:25:18 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 1858.0 B, free 730.2 MB) -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:55140 (size: 1858.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 57 (MapPartitionsRDD[72] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 4 tasks -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 123, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 124, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 57.0 (TID 125, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 57.0 (TID 126, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:18 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 123) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 123). 1104 bytes result sent to driver -2016-05-23 21:25:18 INFO Executor:59 - Running task 2.0 in stage 57.0 (TID 125) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 2.0 in stage 57.0 (TID 125). 1249 bytes result sent to driver -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 123) in 13 ms on localhost (1/4) -2016-05-23 21:25:18 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 124) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 57.0 (TID 125) in 12 ms on localhost (2/4) -2016-05-23 21:25:18 INFO Executor:59 - Running task 3.0 in stage 57.0 (TID 126) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 124). 1254 bytes result sent to driver -2016-05-23 21:25:18 INFO Executor:59 - Finished task 3.0 in stage 57.0 (TID 126). 1254 bytes result sent to driver -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 124) in 24 ms on localhost (3/4) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 57.0 (TID 126) in 23 ms on localhost (4/4) -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-23 21:25:18 INFO DAGScheduler:59 - ResultStage 57 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.028 s -2016-05-23 21:25:18 INFO DAGScheduler:59 - Job 36 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.048434 s -2016-05-23 21:25:18 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:25:18 INFO DAGScheduler:59 - Registering RDD 70 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:25:18 INFO DAGScheduler:59 - Got job 37 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:25:18 INFO DAGScheduler:59 - Final stage: ResultStage 59(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:25:18 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 58) -2016-05-23 21:25:18 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 58) -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting ShuffleMapStage 58 (MapPartitionsRDD[70] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:25:18 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=398189, maxMem=766075207 -2016-05-23 21:25:18 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 21:25:18 INFO MemoryStore:59 - ensureFreeSpace(2285) called with curMem=402229, maxMem=766075207 -2016-05-23 21:25:18 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:55140 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:25:18 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 58 (MapPartitionsRDD[70] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 4 tasks -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 127, localhost, PROCESS_LOCAL, 2221 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 128, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 58.0 (TID 129, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 58.0 (TID 130, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:25:18 INFO Executor:59 - Running task 2.0 in stage 58.0 (TID 129) -2016-05-23 21:25:18 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 127) -2016-05-23 21:25:18 INFO Executor:59 - Running task 3.0 in stage 58.0 (TID 130) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 127). 1161 bytes result sent to driver -2016-05-23 21:25:18 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 128) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 3.0 in stage 58.0 (TID 130). 1161 bytes result sent to driver -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 127) in 43 ms on localhost (1/4) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 58.0 (TID 130) in 62 ms on localhost (2/4) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 128). 1161 bytes result sent to driver -2016-05-23 21:25:18 INFO Executor:59 - Finished task 2.0 in stage 58.0 (TID 129). 1161 bytes result sent to driver -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 128) in 85 ms on localhost (3/4) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 58.0 (TID 129) in 85 ms on localhost (4/4) -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-05-23 21:25:18 INFO DAGScheduler:59 - ShuffleMapStage 58 (flatMap at SpearmanCorrelation.scala:49) finished in 0.090 s -2016-05-23 21:25:18 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:25:18 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:25:18 INFO DAGScheduler:59 - waiting: Set(ResultStage 59) -2016-05-23 21:25:18 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:25:18 INFO DAGScheduler:59 - Missing parents for ResultStage 59: List() -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting ResultStage 59 (ShuffledRDD[73] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:25:18 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=404514, maxMem=766075207 -2016-05-23 21:25:18 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:25:18 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=407058, maxMem=766075207 -2016-05-23 21:25:18 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.2 MB) -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:55140 (size: 1567.0 B, free: 730.5 MB) -2016-05-23 21:25:18 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 59 (ShuffledRDD[73] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 3 tasks -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 131, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 132, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 59.0 (TID 133, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:18 INFO Executor:59 - Running task 2.0 in stage 59.0 (TID 133) -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:18 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 132) -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:18 INFO Executor:59 - Finished task 2.0 in stage 59.0 (TID 133). 1203 bytes result sent to driver -2016-05-23 21:25:18 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 131) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 59.0 (TID 133) in 13 ms on localhost (1/3) -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:18 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 132). 1203 bytes result sent to driver -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 132) in 19 ms on localhost (2/3) -2016-05-23 21:25:18 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 131). 1203 bytes result sent to driver -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 131) in 26 ms on localhost (3/3) -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-05-23 21:25:18 INFO DAGScheduler:59 - ResultStage 59 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.026 s -2016-05-23 21:25:18 INFO DAGScheduler:59 - Job 37 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.145050 s -2016-05-23 21:25:18 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:25:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 168 bytes -2016-05-23 21:25:18 INFO DAGScheduler:59 - Registering RDD 75 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:25:18 INFO DAGScheduler:59 - Got job 38 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:25:18 INFO DAGScheduler:59 - Final stage: ResultStage 62(first at RowMatrix.scala:65) -2016-05-23 21:25:18 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 61) -2016-05-23 21:25:18 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 61) -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting ShuffleMapStage 61 (MapPartitionsRDD[75] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:25:18 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=408625, maxMem=766075207 -2016-05-23 21:25:18 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:25:18 INFO MemoryStore:59 - ensureFreeSpace(2305) called with curMem=412857, maxMem=766075207 -2016-05-23 21:25:18 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 21:25:18 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:55140 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:25:18 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 61 (MapPartitionsRDD[75] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:25:18 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 4 tasks -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 134, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 135, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 61.0 (TID 136, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 61.0 (TID 137, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:25:18 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 134) -2016-05-23 21:25:18 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 135) -2016-05-23 21:25:18 INFO Executor:59 - Running task 2.0 in stage 61.0 (TID 136) -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:18 INFO Executor:59 - Running task 3.0 in stage 61.0 (TID 137) -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:25:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:18 INFO Executor:59 - Finished task 2.0 in stage 61.0 (TID 136). 1377 bytes result sent to driver -2016-05-23 21:25:18 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 135). 1377 bytes result sent to driver -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 61.0 (TID 136) in 61 ms on localhost (1/4) -2016-05-23 21:25:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 135) in 62 ms on localhost (2/4) -2016-05-23 21:25:19 INFO Executor:59 - Finished task 3.0 in stage 61.0 (TID 137). 1377 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 134). 1377 bytes result sent to driver -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 61.0 (TID 137) in 82 ms on localhost (3/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 134) in 85 ms on localhost (4/4) -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-05-23 21:25:19 INFO DAGScheduler:59 - ShuffleMapStage 61 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.092 s -2016-05-23 21:25:19 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:25:19 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:25:19 INFO DAGScheduler:59 - waiting: Set(ResultStage 62) -2016-05-23 21:25:19 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:25:19 INFO DAGScheduler:59 - Missing parents for ResultStage 62: List() -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=415162, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=420058, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:25:19 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:55140 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:25:19 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 62 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 1 tasks -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 138, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:19 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 138) -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:19 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 138). 1185 bytes result sent to driver -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 138) in 14 ms on localhost (1/1) -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-05-23 21:25:19 INFO DAGScheduler:59 - ResultStage 62 (first at RowMatrix.scala:65) finished in 0.022 s -2016-05-23 21:25:19 INFO DAGScheduler:59 - Job 38 finished: first at RowMatrix.scala:65, took 0.155328 s -2016-05-23 21:25:19 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:25:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 168 bytes -2016-05-23 21:25:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 165 bytes -2016-05-23 21:25:19 INFO DAGScheduler:59 - Got job 39 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 21:25:19 INFO DAGScheduler:59 - Final stage: ResultStage 65(first at RowMatrix.scala:65) -2016-05-23 21:25:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 64) -2016-05-23 21:25:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87), which has no missing parents -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=422602, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=427498, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:25:19 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:55140 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:25:19 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 65 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 3 tasks -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 139, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 140, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 65.0 (TID 141, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:19 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 139) -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:19 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 139). 1296 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 140) -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:19 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 140). 1296 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Running task 2.0 in stage 65.0 (TID 141) -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:19 INFO Executor:59 - Finished task 2.0 in stage 65.0 (TID 141). 1296 bytes result sent to driver -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 139) in 56 ms on localhost (1/3) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 140) in 57 ms on localhost (2/3) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 65.0 (TID 141) in 57 ms on localhost (3/3) -2016-05-23 21:25:19 INFO DAGScheduler:59 - ResultStage 65 (first at RowMatrix.scala:65) finished in 0.059 s -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-05-23 21:25:19 INFO DAGScheduler:59 - Job 39 finished: first at RowMatrix.scala:65, took 0.079700 s -2016-05-23 21:25:19 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Got job 40 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:25:19 INFO DAGScheduler:59 - Final stage: ResultStage 68(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:25:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 67) -2016-05-23 21:25:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting ResultStage 68 (MapPartitionsRDD[78] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=430042, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(2914) called with curMem=435738, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:25:19 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:55140 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:25:19 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 68 (MapPartitionsRDD[78] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Adding task set 68.0 with 4 tasks -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 68.0 (TID 142, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 68.0 (TID 143, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 68.0 (TID 144, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 68.0 (TID 145, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:19 INFO Executor:59 - Running task 0.0 in stage 68.0 (TID 142) -2016-05-23 21:25:19 INFO Executor:59 - Running task 3.0 in stage 68.0 (TID 145) -2016-05-23 21:25:19 INFO Executor:59 - Running task 1.0 in stage 68.0 (TID 143) -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:19 INFO Executor:59 - Running task 2.0 in stage 68.0 (TID 144) -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:19 INFO Executor:59 - Finished task 2.0 in stage 68.0 (TID 144). 1481 bytes result sent to driver -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:19 INFO Executor:59 - Finished task 3.0 in stage 68.0 (TID 145). 1481 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Finished task 1.0 in stage 68.0 (TID 143). 1481 bytes result sent to driver -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:19 INFO Executor:59 - Finished task 0.0 in stage 68.0 (TID 142). 1481 bytes result sent to driver -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 68.0 (TID 144) in 47 ms on localhost (1/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 68.0 (TID 145) in 47 ms on localhost (2/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 68.0 (TID 142) in 51 ms on localhost (3/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 68.0 (TID 143) in 49 ms on localhost (4/4) -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 68.0, whose tasks have all completed, from pool -2016-05-23 21:25:19 INFO DAGScheduler:59 - ResultStage 68 (treeAggregate at RowMatrix.scala:332) finished in 0.053 s -2016-05-23 21:25:19 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at RowMatrix.scala:332, took 0.137674 s -2016-05-23 21:25:19 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Got job 41 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:25:19 INFO DAGScheduler:59 - Final stage: ResultStage 71(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:25:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 70) -2016-05-23 21:25:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting ResultStage 71 (MapPartitionsRDD[79] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=438652, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=444388, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:25:19 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:55140 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:25:19 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 71 (MapPartitionsRDD[79] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Adding task set 71.0 with 4 tasks -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 71.0 (TID 146, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 71.0 (TID 147, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 71.0 (TID 148, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 71.0 (TID 149, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:19 INFO Executor:59 - Running task 0.0 in stage 71.0 (TID 146) -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:19 INFO Executor:59 - Finished task 0.0 in stage 71.0 (TID 146). 1359 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Running task 1.0 in stage 71.0 (TID 147) -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:19 INFO Executor:59 - Finished task 1.0 in stage 71.0 (TID 147). 1359 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Running task 2.0 in stage 71.0 (TID 148) -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:19 INFO Executor:59 - Finished task 2.0 in stage 71.0 (TID 148). 1359 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Running task 3.0 in stage 71.0 (TID 149) -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:19 INFO Executor:59 - Finished task 3.0 in stage 71.0 (TID 149). 1359 bytes result sent to driver -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 71.0 (TID 146) in 135 ms on localhost (1/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 71.0 (TID 147) in 135 ms on localhost (2/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 71.0 (TID 148) in 137 ms on localhost (3/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 71.0 (TID 149) in 138 ms on localhost (4/4) -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 71.0, whose tasks have all completed, from pool -2016-05-23 21:25:19 INFO DAGScheduler:59 - ResultStage 71 (treeAggregate at RowMatrix.scala:124) finished in 0.164 s -2016-05-23 21:25:19 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at RowMatrix.scala:124, took 0.255354 s -2016-05-23 21:25:19 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:86 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Got job 42 (foreach at StatisticsCorrLearning.scala:86) with 4 output partitions -2016-05-23 21:25:19 INFO DAGScheduler:59 - Final stage: ResultStage 72(foreach at StatisticsCorrLearning.scala:86) -2016-05-23 21:25:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting ResultStage 72 (ParallelCollectionRDD[80] at parallelize at StatisticsCorrLearning.scala:83), which has no missing parents -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=447284, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=448508, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:25:19 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:55140 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:25:19 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 72 (ParallelCollectionRDD[80] at parallelize at StatisticsCorrLearning.scala:83) -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Adding task set 72.0 with 4 tasks -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 72.0 (TID 150, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 72.0 (TID 151, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 72.0 (TID 152, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 72.0 (TID 153, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:25:19 INFO Executor:59 - Running task 2.0 in stage 72.0 (TID 152) -2016-05-23 21:25:19 INFO Executor:59 - Running task 1.0 in stage 72.0 (TID 151) -2016-05-23 21:25:19 INFO Executor:59 - Running task 3.0 in stage 72.0 (TID 153) -2016-05-23 21:25:19 INFO Executor:59 - Finished task 1.0 in stage 72.0 (TID 151). 915 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Running task 0.0 in stage 72.0 (TID 150) -2016-05-23 21:25:19 INFO Executor:59 - Finished task 3.0 in stage 72.0 (TID 153). 915 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Finished task 2.0 in stage 72.0 (TID 152). 915 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Finished task 0.0 in stage 72.0 (TID 150). 915 bytes result sent to driver -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 72.0 (TID 151) in 9 ms on localhost (1/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 72.0 (TID 153) in 9 ms on localhost (2/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 72.0 (TID 152) in 10 ms on localhost (3/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 72.0 (TID 150) in 12 ms on localhost (4/4) -2016-05-23 21:25:19 INFO DAGScheduler:59 - ResultStage 72 (foreach at StatisticsCorrLearning.scala:86) finished in 0.013 s -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 72.0, whose tasks have all completed, from pool -2016-05-23 21:25:19 INFO DAGScheduler:59 - Job 42 finished: foreach at StatisticsCorrLearning.scala:86, took 0.033278 s -2016-05-23 21:25:19 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:88 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Got job 43 (foreach at StatisticsCorrLearning.scala:88) with 4 output partitions -2016-05-23 21:25:19 INFO DAGScheduler:59 - Final stage: ResultStage 73(foreach at StatisticsCorrLearning.scala:88) -2016-05-23 21:25:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting ResultStage 73 (ParallelCollectionRDD[81] at parallelize at StatisticsCorrLearning.scala:84), which has no missing parents -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(1232) called with curMem=449401, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 1232.0 B, free 730.2 MB) -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(894) called with curMem=450633, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 894.0 B, free 730.2 MB) -2016-05-23 21:25:19 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:55140 (size: 894.0 B, free: 730.5 MB) -2016-05-23 21:25:19 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 73 (ParallelCollectionRDD[81] at parallelize at StatisticsCorrLearning.scala:84) -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Adding task set 73.0 with 4 tasks -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 73.0 (TID 154, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 73.0 (TID 155, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 73.0 (TID 156, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 73.0 (TID 157, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:25:19 INFO Executor:59 - Running task 1.0 in stage 73.0 (TID 155) -2016-05-23 21:25:19 INFO Executor:59 - Running task 2.0 in stage 73.0 (TID 156) -2016-05-23 21:25:19 INFO Executor:59 - Finished task 1.0 in stage 73.0 (TID 155). 915 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Finished task 2.0 in stage 73.0 (TID 156). 915 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Running task 0.0 in stage 73.0 (TID 154) -2016-05-23 21:25:19 INFO Executor:59 - Running task 3.0 in stage 73.0 (TID 157) -2016-05-23 21:25:19 INFO Executor:59 - Finished task 0.0 in stage 73.0 (TID 154). 915 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Finished task 3.0 in stage 73.0 (TID 157). 915 bytes result sent to driver -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 73.0 (TID 156) in 8 ms on localhost (1/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 73.0 (TID 154) in 12 ms on localhost (2/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 73.0 (TID 155) in 11 ms on localhost (3/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 73.0 (TID 157) in 11 ms on localhost (4/4) -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 73.0, whose tasks have all completed, from pool -2016-05-23 21:25:19 INFO DAGScheduler:59 - ResultStage 73 (foreach at StatisticsCorrLearning.scala:88) finished in 0.014 s -2016-05-23 21:25:19 INFO DAGScheduler:59 - Job 43 finished: foreach at StatisticsCorrLearning.scala:88, took 0.030591 s -2016-05-23 21:25:19 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Got job 44 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:25:19 INFO DAGScheduler:59 - Final stage: ResultStage 74(first at RowMatrix.scala:65) -2016-05-23 21:25:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting ResultStage 74 (MapPartitionsRDD[83] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=451527, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=454031, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 21:25:19 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:55140 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:25:19 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 74 (MapPartitionsRDD[83] at map at Correlation.scala:45) -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Adding task set 74.0 with 1 tasks -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 74.0 (TID 158, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:19 INFO Executor:59 - Running task 0.0 in stage 74.0 (TID 158) -2016-05-23 21:25:19 INFO Executor:59 - Finished task 0.0 in stage 74.0 (TID 158). 1046 bytes result sent to driver -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 74.0 (TID 158) in 3 ms on localhost (1/1) -2016-05-23 21:25:19 INFO DAGScheduler:59 - ResultStage 74 (first at RowMatrix.scala:65) finished in 0.004 s -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 74.0, whose tasks have all completed, from pool -2016-05-23 21:25:19 INFO DAGScheduler:59 - Job 44 finished: first at RowMatrix.scala:65, took 0.011268 s -2016-05-23 21:25:19 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Got job 45 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:25:19 INFO DAGScheduler:59 - Final stage: ResultStage 75(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:25:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting ResultStage 75 (MapPartitionsRDD[84] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=455555, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=458955, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.1 MB) -2016-05-23 21:25:19 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:55140 (size: 1935.0 B, free: 730.5 MB) -2016-05-23 21:25:19 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 75 (MapPartitionsRDD[84] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Adding task set 75.0 with 4 tasks -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 75.0 (TID 159, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 75.0 (TID 160, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 75.0 (TID 161, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 75.0 (TID 162, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:25:19 INFO Executor:59 - Running task 0.0 in stage 75.0 (TID 159) -2016-05-23 21:25:19 INFO Executor:59 - Running task 1.0 in stage 75.0 (TID 160) -2016-05-23 21:25:19 INFO Executor:59 - Running task 3.0 in stage 75.0 (TID 162) -2016-05-23 21:25:19 INFO Executor:59 - Running task 2.0 in stage 75.0 (TID 161) -2016-05-23 21:25:19 INFO Executor:59 - Finished task 1.0 in stage 75.0 (TID 160). 1231 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Finished task 0.0 in stage 75.0 (TID 159). 1231 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Finished task 3.0 in stage 75.0 (TID 162). 1231 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Finished task 2.0 in stage 75.0 (TID 161). 1231 bytes result sent to driver -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 75.0 (TID 160) in 6 ms on localhost (1/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 75.0 (TID 162) in 6 ms on localhost (2/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 75.0 (TID 159) in 7 ms on localhost (3/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 75.0 (TID 161) in 7 ms on localhost (4/4) -2016-05-23 21:25:19 INFO DAGScheduler:59 - ResultStage 75 (treeAggregate at RowMatrix.scala:332) finished in 0.008 s -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 75.0, whose tasks have all completed, from pool -2016-05-23 21:25:19 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at RowMatrix.scala:332, took 0.017579 s -2016-05-23 21:25:19 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Got job 46 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:25:19 INFO DAGScheduler:59 - Final stage: ResultStage 76(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:25:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting ResultStage 76 (MapPartitionsRDD[85] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=460890, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(1919) called with curMem=464330, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 1919.0 B, free 730.1 MB) -2016-05-23 21:25:19 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:55140 (size: 1919.0 B, free: 730.5 MB) -2016-05-23 21:25:19 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 76 (MapPartitionsRDD[85] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 4 tasks -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 163, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 164, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 76.0 (TID 165, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 76.0 (TID 166, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:25:19 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 163) -2016-05-23 21:25:19 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 164) -2016-05-23 21:25:19 INFO Executor:59 - Running task 3.0 in stage 76.0 (TID 166) -2016-05-23 21:25:19 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 163). 1109 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Finished task 3.0 in stage 76.0 (TID 166). 1109 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Running task 2.0 in stage 76.0 (TID 165) -2016-05-23 21:25:19 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 164). 1109 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Finished task 2.0 in stage 76.0 (TID 165). 1109 bytes result sent to driver -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 163) in 11 ms on localhost (1/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 76.0 (TID 166) in 10 ms on localhost (2/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 164) in 11 ms on localhost (3/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 76.0 (TID 165) in 12 ms on localhost (4/4) -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-05-23 21:25:19 INFO DAGScheduler:59 - ResultStage 76 (treeAggregate at RowMatrix.scala:124) finished in 0.013 s -2016-05-23 21:25:19 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at RowMatrix.scala:124, took 0.023219 s -2016-05-23 21:25:19 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Got job 47 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:25:19 INFO DAGScheduler:59 - Final stage: ResultStage 77(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:25:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:25:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[91] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=466249, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(1862) called with curMem=469625, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 1862.0 B, free 730.1 MB) -2016-05-23 21:25:19 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:55140 (size: 1862.0 B, free: 730.5 MB) -2016-05-23 21:25:19 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 77 (MapPartitionsRDD[91] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 4 tasks -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 167, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 168, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 77.0 (TID 169, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 77.0 (TID 170, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:25:19 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 167) -2016-05-23 21:25:19 INFO Executor:59 - Running task 2.0 in stage 77.0 (TID 169) -2016-05-23 21:25:19 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 168) -2016-05-23 21:25:19 INFO Executor:59 - Running task 3.0 in stage 77.0 (TID 170) -2016-05-23 21:25:19 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 167). 1254 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Finished task 3.0 in stage 77.0 (TID 170). 1294 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Finished task 2.0 in stage 77.0 (TID 169). 1249 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 168). 1254 bytes result sent to driver -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 167) in 5 ms on localhost (1/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 77.0 (TID 169) in 5 ms on localhost (2/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 77.0 (TID 170) in 4 ms on localhost (3/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 168) in 6 ms on localhost (4/4) -2016-05-23 21:25:19 INFO DAGScheduler:59 - ResultStage 77 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.007 s -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-05-23 21:25:19 INFO DAGScheduler:59 - Job 47 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.021591 s -2016-05-23 21:25:19 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Registering RDD 89 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:25:19 INFO DAGScheduler:59 - Got job 48 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:25:19 INFO DAGScheduler:59 - Final stage: ResultStage 79(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:25:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 78) -2016-05-23 21:25:19 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 78) -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting ShuffleMapStage 78 (MapPartitionsRDD[89] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=471487, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(2289) called with curMem=475527, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 21:25:19 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:55140 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:25:19 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 78 (MapPartitionsRDD[89] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 4 tasks -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 171, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 172, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 78.0 (TID 173, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 78.0 (TID 174, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:25:19 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 171) -2016-05-23 21:25:19 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 172) -2016-05-23 21:25:19 INFO Executor:59 - Running task 2.0 in stage 78.0 (TID 173) -2016-05-23 21:25:19 INFO Executor:59 - Running task 3.0 in stage 78.0 (TID 174) -2016-05-23 21:25:19 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 171). 1161 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Finished task 2.0 in stage 78.0 (TID 173). 1161 bytes result sent to driver -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 171) in 72 ms on localhost (1/4) -2016-05-23 21:25:19 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 172). 1161 bytes result sent to driver -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 78.0 (TID 173) in 74 ms on localhost (2/4) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 172) in 76 ms on localhost (3/4) -2016-05-23 21:25:19 INFO Executor:59 - Finished task 3.0 in stage 78.0 (TID 174). 1161 bytes result sent to driver -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 78.0 (TID 174) in 82 ms on localhost (4/4) -2016-05-23 21:25:19 INFO DAGScheduler:59 - ShuffleMapStage 78 (flatMap at SpearmanCorrelation.scala:49) finished in 0.083 s -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-05-23 21:25:19 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:25:19 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:25:19 INFO DAGScheduler:59 - waiting: Set(ResultStage 79) -2016-05-23 21:25:19 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:25:19 INFO DAGScheduler:59 - Missing parents for ResultStage 79: List() -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting ResultStage 79 (ShuffledRDD[92] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=477816, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(1573) called with curMem=480360, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 1573.0 B, free 730.1 MB) -2016-05-23 21:25:19 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:55140 (size: 1573.0 B, free: 730.5 MB) -2016-05-23 21:25:19 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 79 (ShuffledRDD[92] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Adding task set 79.0 with 3 tasks -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 79.0 (TID 175, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 79.0 (TID 176, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 79.0 (TID 177, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:19 INFO Executor:59 - Running task 1.0 in stage 79.0 (TID 176) -2016-05-23 21:25:19 INFO Executor:59 - Running task 2.0 in stage 79.0 (TID 177) -2016-05-23 21:25:19 INFO Executor:59 - Running task 0.0 in stage 79.0 (TID 175) -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:19 INFO Executor:59 - Finished task 0.0 in stage 79.0 (TID 175). 1203 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Finished task 2.0 in stage 79.0 (TID 177). 1203 bytes result sent to driver -2016-05-23 21:25:19 INFO Executor:59 - Finished task 1.0 in stage 79.0 (TID 176). 1203 bytes result sent to driver -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 79.0 (TID 175) in 13 ms on localhost (1/3) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 79.0 (TID 176) in 13 ms on localhost (2/3) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 79.0 (TID 177) in 14 ms on localhost (3/3) -2016-05-23 21:25:19 INFO DAGScheduler:59 - ResultStage 79 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.015 s -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 79.0, whose tasks have all completed, from pool -2016-05-23 21:25:19 INFO DAGScheduler:59 - Job 48 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.117884 s -2016-05-23 21:25:19 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:25:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 166 bytes -2016-05-23 21:25:19 INFO DAGScheduler:59 - Registering RDD 94 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:25:19 INFO DAGScheduler:59 - Got job 49 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:25:19 INFO DAGScheduler:59 - Final stage: ResultStage 82(first at RowMatrix.scala:65) -2016-05-23 21:25:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 81) -2016-05-23 21:25:19 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 81) -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting ShuffleMapStage 81 (MapPartitionsRDD[94] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=481933, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 21:25:19 INFO MemoryStore:59 - ensureFreeSpace(2313) called with curMem=486165, maxMem=766075207 -2016-05-23 21:25:19 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-23 21:25:19 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:55140 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:25:19 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 81 (MapPartitionsRDD[94] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:25:19 INFO TaskSchedulerImpl:59 - Adding task set 81.0 with 4 tasks -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 81.0 (TID 178, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 81.0 (TID 179, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 81.0 (TID 180, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:25:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 81.0 (TID 181, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:25:19 INFO Executor:59 - Running task 0.0 in stage 81.0 (TID 178) -2016-05-23 21:25:19 INFO Executor:59 - Running task 3.0 in stage 81.0 (TID 181) -2016-05-23 21:25:19 INFO Executor:59 - Running task 1.0 in stage 81.0 (TID 179) -2016-05-23 21:25:19 INFO Executor:59 - Running task 2.0 in stage 81.0 (TID 180) -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:20 INFO Executor:59 - Finished task 1.0 in stage 81.0 (TID 179). 1377 bytes result sent to driver -2016-05-23 21:25:20 INFO Executor:59 - Finished task 3.0 in stage 81.0 (TID 181). 1377 bytes result sent to driver -2016-05-23 21:25:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 81.0 (TID 179) in 70 ms on localhost (1/4) -2016-05-23 21:25:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 81.0 (TID 181) in 70 ms on localhost (2/4) -2016-05-23 21:25:20 INFO Executor:59 - Finished task 0.0 in stage 81.0 (TID 178). 1377 bytes result sent to driver -2016-05-23 21:25:20 INFO Executor:59 - Finished task 2.0 in stage 81.0 (TID 180). 1377 bytes result sent to driver -2016-05-23 21:25:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 81.0 (TID 178) in 79 ms on localhost (3/4) -2016-05-23 21:25:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 81.0 (TID 180) in 79 ms on localhost (4/4) -2016-05-23 21:25:20 INFO DAGScheduler:59 - ShuffleMapStage 81 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.081 s -2016-05-23 21:25:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 81.0, whose tasks have all completed, from pool -2016-05-23 21:25:20 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:25:20 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:25:20 INFO DAGScheduler:59 - waiting: Set(ResultStage 82) -2016-05-23 21:25:20 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:25:20 INFO DAGScheduler:59 - Missing parents for ResultStage 82: List() -2016-05-23 21:25:20 INFO DAGScheduler:59 - Submitting ResultStage 82 (MapPartitionsRDD[96] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:25:20 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=488478, maxMem=766075207 -2016-05-23 21:25:20 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 21:25:20 INFO MemoryStore:59 - ensureFreeSpace(2562) called with curMem=493374, maxMem=766075207 -2016-05-23 21:25:20 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:25:20 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:55140 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:25:20 INFO SparkContext:59 - Created broadcast 61 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 82 (MapPartitionsRDD[96] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:25:20 INFO TaskSchedulerImpl:59 - Adding task set 82.0 with 1 tasks -2016-05-23 21:25:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 82.0 (TID 182, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:20 INFO Executor:59 - Running task 0.0 in stage 82.0 (TID 182) -2016-05-23 21:25:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:20 INFO Executor:59 - Finished task 0.0 in stage 82.0 (TID 182). 1296 bytes result sent to driver -2016-05-23 21:25:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 82.0 (TID 182) in 14 ms on localhost (1/1) -2016-05-23 21:25:20 INFO DAGScheduler:59 - ResultStage 82 (first at RowMatrix.scala:65) finished in 0.014 s -2016-05-23 21:25:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 82.0, whose tasks have all completed, from pool -2016-05-23 21:25:20 INFO DAGScheduler:59 - Job 49 finished: first at RowMatrix.scala:65, took 0.116727 s -2016-05-23 21:25:20 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:25:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 166 bytes -2016-05-23 21:25:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 169 bytes -2016-05-23 21:25:20 INFO DAGScheduler:59 - Got job 50 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:25:20 INFO DAGScheduler:59 - Final stage: ResultStage 85(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:25:20 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 84) -2016-05-23 21:25:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:20 INFO DAGScheduler:59 - Submitting ResultStage 85 (MapPartitionsRDD[97] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:25:20 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=495936, maxMem=766075207 -2016-05-23 21:25:20 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 21:25:20 INFO MemoryStore:59 - ensureFreeSpace(2943) called with curMem=501632, maxMem=766075207 -2016-05-23 21:25:20 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.1 MB) -2016-05-23 21:25:20 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:55140 (size: 2.9 KB, free: 730.5 MB) -2016-05-23 21:25:20 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 85 (MapPartitionsRDD[97] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:25:20 INFO TaskSchedulerImpl:59 - Adding task set 85.0 with 4 tasks -2016-05-23 21:25:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 85.0 (TID 183, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 85.0 (TID 184, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 85.0 (TID 185, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 85.0 (TID 186, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:20 INFO Executor:59 - Running task 0.0 in stage 85.0 (TID 183) -2016-05-23 21:25:20 INFO Executor:59 - Running task 2.0 in stage 85.0 (TID 185) -2016-05-23 21:25:20 INFO Executor:59 - Running task 1.0 in stage 85.0 (TID 184) -2016-05-23 21:25:20 INFO Executor:59 - Running task 3.0 in stage 85.0 (TID 186) -2016-05-23 21:25:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:25:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:20 INFO Executor:59 - Finished task 3.0 in stage 85.0 (TID 186). 1481 bytes result sent to driver -2016-05-23 21:25:20 INFO Executor:59 - Finished task 0.0 in stage 85.0 (TID 183). 1481 bytes result sent to driver -2016-05-23 21:25:20 INFO Executor:59 - Finished task 2.0 in stage 85.0 (TID 185). 1481 bytes result sent to driver -2016-05-23 21:25:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 85.0 (TID 183) in 18 ms on localhost (1/4) -2016-05-23 21:25:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 85.0 (TID 186) in 17 ms on localhost (2/4) -2016-05-23 21:25:20 INFO Executor:59 - Finished task 1.0 in stage 85.0 (TID 184). 1481 bytes result sent to driver -2016-05-23 21:25:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 85.0 (TID 185) in 18 ms on localhost (3/4) -2016-05-23 21:25:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 85.0 (TID 184) in 18 ms on localhost (4/4) -2016-05-23 21:25:20 INFO DAGScheduler:59 - ResultStage 85 (treeAggregate at RowMatrix.scala:332) finished in 0.020 s -2016-05-23 21:25:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 85.0, whose tasks have all completed, from pool -2016-05-23 21:25:20 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at RowMatrix.scala:332, took 0.031343 s -2016-05-23 21:25:20 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:25:20 INFO DAGScheduler:59 - Got job 51 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:25:20 INFO DAGScheduler:59 - Final stage: ResultStage 88(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:25:20 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 87) -2016-05-23 21:25:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:25:20 INFO DAGScheduler:59 - Submitting ResultStage 88 (MapPartitionsRDD[98] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:25:20 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=504575, maxMem=766075207 -2016-05-23 21:25:20 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 21:25:20 INFO MemoryStore:59 - ensureFreeSpace(2920) called with curMem=510311, maxMem=766075207 -2016-05-23 21:25:20 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.1 MB) -2016-05-23 21:25:20 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:55140 (size: 2.9 KB, free: 730.5 MB) -2016-05-23 21:25:20 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:25:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 88 (MapPartitionsRDD[98] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:25:20 INFO TaskSchedulerImpl:59 - Adding task set 88.0 with 4 tasks -2016-05-23 21:25:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 88.0 (TID 187, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 88.0 (TID 188, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 88.0 (TID 189, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 88.0 (TID 190, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:25:20 INFO Executor:59 - Running task 1.0 in stage 88.0 (TID 188) -2016-05-23 21:25:20 INFO Executor:59 - Running task 2.0 in stage 88.0 (TID 189) -2016-05-23 21:25:20 INFO Executor:59 - Running task 3.0 in stage 88.0 (TID 190) -2016-05-23 21:25:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:20 INFO Executor:59 - Finished task 2.0 in stage 88.0 (TID 189). 1359 bytes result sent to driver -2016-05-23 21:25:20 INFO Executor:59 - Finished task 3.0 in stage 88.0 (TID 190). 1359 bytes result sent to driver -2016-05-23 21:25:20 INFO Executor:59 - Running task 0.0 in stage 88.0 (TID 187) -2016-05-23 21:25:20 INFO Executor:59 - Finished task 1.0 in stage 88.0 (TID 188). 1359 bytes result sent to driver -2016-05-23 21:25:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 88.0 (TID 189) in 13 ms on localhost (1/4) -2016-05-23 21:25:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 88.0 (TID 188) in 13 ms on localhost (2/4) -2016-05-23 21:25:20 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:25:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:25:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 88.0 (TID 190) in 13 ms on localhost (3/4) -2016-05-23 21:25:20 INFO Executor:59 - Finished task 0.0 in stage 88.0 (TID 187). 1359 bytes result sent to driver -2016-05-23 21:25:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 88.0 (TID 187) in 23 ms on localhost (4/4) -2016-05-23 21:25:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 88.0, whose tasks have all completed, from pool -2016-05-23 21:25:20 INFO DAGScheduler:59 - ResultStage 88 (treeAggregate at RowMatrix.scala:124) finished in 0.024 s -2016-05-23 21:25:20 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at RowMatrix.scala:124, took 0.036263 s -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 21:25:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 21:25:20 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 21:25:20 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 21:25:20 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 21:25:20 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 21:25:20 INFO BlockManager:59 - BlockManager stopped -2016-05-23 21:25:20 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 21:25:20 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 21:25:20 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 21:25:20 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 21:25:20 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-c6e8ecfb-a719-4081-8df2-f01d127f5d23 -2016-05-23 21:25:20 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 21:25:20 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 21:52:34 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 21:52:36 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 21:52:38 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 21:52:38 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 21:52:38 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 21:52:44 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 21:52:45 INFO Remoting:74 - Starting remoting -2016-05-23 21:52:46 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53756] -2016-05-23 21:52:46 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53756. -2016-05-23 21:52:46 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 21:52:46 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 21:52:47 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-436ccf42-a6be-408b-b6a9-551c821ce95f -2016-05-23 21:52:48 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 21:52:50 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-e5fbf2db-2cc5-4c3d-87ba-b9f9fb509b61\httpd-7d675b75-15c8-445b-9cf9-12e87580b40b -2016-05-23 21:52:51 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 21:52:54 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 21:52:56 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53758 -2016-05-23 21:52:56 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53758. -2016-05-23 21:52:57 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 21:53:05 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 21:53:05 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 21:53:05 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 21:53:05 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 21:53:06 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 21:53:06 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 21:53:07 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53778. -2016-05-23 21:53:07 INFO NettyBlockTransferService:59 - Server created on 53778 -2016-05-23 21:53:07 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 21:53:07 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53778 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53778) -2016-05-23 21:53:07 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 21:53:23 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 21:53:23 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 21:53:23 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 21:53:23 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 21:53:23 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53778 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 21:53:23 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 21:53:24 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 21:53:24 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 21:53:24 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 21:53:24 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 21:53:24 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53778 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 21:53:24 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:26 -2016-05-23 21:53:26 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 21:53:26 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 21:53:27 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:30 -2016-05-23 21:53:27 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:30) with 2 output partitions -2016-05-23 21:53:27 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:30) -2016-05-23 21:53:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:53:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:53:27 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:24), which has no missing parents -2016-05-23 21:53:28 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 21:53:28 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 21:53:28 INFO MemoryStore:59 - ensureFreeSpace(2047) called with curMem=346712, maxMem=766075207 -2016-05-23 21:53:28 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2047.0 B, free 730.3 MB) -2016-05-23 21:53:28 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53778 (size: 2047.0 B, free: 730.6 MB) -2016-05-23 21:53:28 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:53:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:24) -2016-05-23 21:53:28 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 21:53:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:53:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:53:28 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 21:53:28 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 21:53:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:53:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:53:28 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 21:53:28 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 21:53:28 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 21:53:28 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 21:53:28 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 21:53:28 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 21:53:28 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 21:53:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 635 ms on localhost (1/2) -2016-05-23 21:53:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 763 ms on localhost (2/2) -2016-05-23 21:53:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 21:53:29 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:30) finished in 0.904 s -2016-05-23 21:53:29 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:30, took 2.500101 s -2016-05-23 21:53:29 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 21:53:29 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:32 -2016-05-23 21:53:29 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:32) with 2 output partitions -2016-05-23 21:53:29 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:32) -2016-05-23 21:53:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:53:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:53:29 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:27), which has no missing parents -2016-05-23 21:53:29 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348759, maxMem=766075207 -2016-05-23 21:53:29 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 21:53:29 INFO MemoryStore:59 - ensureFreeSpace(2046) called with curMem=352271, maxMem=766075207 -2016-05-23 21:53:29 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2046.0 B, free 730.2 MB) -2016-05-23 21:53:29 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53778 (size: 2046.0 B, free: 730.6 MB) -2016-05-23 21:53:29 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:53:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:27) -2016-05-23 21:53:29 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 21:53:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:53:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:53:29 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 21:53:29 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 21:53:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:53:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:53:29 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 21:53:29 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 21:53:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 27 ms on localhost (1/2) -2016-05-23 21:53:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 30 ms on localhost (2/2) -2016-05-23 21:53:29 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:32) finished in 0.030 s -2016-05-23 21:53:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 21:53:29 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:32, took 0.056568 s -2016-05-23 21:53:30 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:53:30 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:53:30 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 21:53:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:53:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:53:30 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:53:30 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354317, maxMem=766075207 -2016-05-23 21:53:30 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 21:53:30 INFO MemoryStore:59 - ensureFreeSpace(2465) called with curMem=358877, maxMem=766075207 -2016-05-23 21:53:30 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:53:30 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53778 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 21:53:30 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:53:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 21:53:30 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 21:53:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:53:30 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 21:53:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:53:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:53:30 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53778 in memory (size: 2046.0 B, free: 730.6 MB) -2016-05-23 21:53:30 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 21:53:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 336 ms on localhost (1/1) -2016-05-23 21:53:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 21:53:30 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.340 s -2016-05-23 21:53:30 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.411914 s -2016-05-23 21:53:31 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53778 in memory (size: 2047.0 B, free: 730.6 MB) -2016-05-23 21:54:03 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:54:03 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:54:03 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:03 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:03 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:54:03 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=350225, maxMem=766075207 -2016-05-23 21:54:03 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 21:54:03 INFO MemoryStore:59 - ensureFreeSpace(2862) called with curMem=355673, maxMem=766075207 -2016-05-23 21:54:03 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:54:03 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53778 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:54:03 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:03 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 21:54:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:54:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:54:03 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 21:54:03 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 21:54:03 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:54:03 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:54:03 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:54:03 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:54:03 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 21:54:03 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 21:54:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 84 ms on localhost (1/2) -2016-05-23 21:54:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 88 ms on localhost (2/2) -2016-05-23 21:54:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 21:54:03 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.089 s -2016-05-23 21:54:03 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 21:54:03 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 21:54:03 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.173622 s -2016-05-23 21:54:03 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:54:03 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:54:03 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:03 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:03 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:54:03 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=358535, maxMem=766075207 -2016-05-23 21:54:03 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 21:54:03 INFO MemoryStore:59 - ensureFreeSpace(2851) called with curMem=364031, maxMem=766075207 -2016-05-23 21:54:03 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:54:03 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53778 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:54:03 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:03 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 21:54:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:54:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:54:03 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 21:54:03 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 21:54:03 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:54:03 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:54:03 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:54:03 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:54:03 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 21:54:03 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 21:54:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 49 ms on localhost (1/2) -2016-05-23 21:54:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 49 ms on localhost (2/2) -2016-05-23 21:54:03 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.050 s -2016-05-23 21:54:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 21:54:03 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.083581 s -2016-05-23 21:54:09 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:54:09 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 21:54:09 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:54:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:09 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:54:09 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=366882, maxMem=766075207 -2016-05-23 21:54:09 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 21:54:09 INFO MemoryStore:59 - ensureFreeSpace(2744) called with curMem=372226, maxMem=766075207 -2016-05-23 21:54:09 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 21:54:09 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53778 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 21:54:09 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:54:09 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 21:54:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:54:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:54:09 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 21:54:09 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 21:54:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:54:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:54:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:54:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:54:09 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 21:54:09 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 21:54:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 464 ms on localhost (1/2) -2016-05-23 21:54:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 472 ms on localhost (2/2) -2016-05-23 21:54:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 21:54:09 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.475 s -2016-05-23 21:54:09 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.511701 s -2016-05-23 21:54:10 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:54:10 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:54:10 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 21:54:10 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:54:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 21:54:10 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 21:54:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:54:10 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=374970, maxMem=766075207 -2016-05-23 21:54:10 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 21:54:10 INFO MemoryStore:59 - ensureFreeSpace(3156) called with curMem=380898, maxMem=766075207 -2016-05-23 21:54:10 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 21:54:10 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53778 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 21:54:10 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:54:11 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 21:54:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 21:54:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 21:54:11 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 21:54:11 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 21:54:11 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:54:11 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:54:11 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:54:11 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:54:11 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 21:54:11 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 21:54:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 544 ms on localhost (1/2) -2016-05-23 21:54:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 538 ms on localhost (2/2) -2016-05-23 21:54:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 21:54:11 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.548 s -2016-05-23 21:54:11 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:54:11 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:54:11 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 21:54:11 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:54:11 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 21:54:11 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:54:11 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=384054, maxMem=766075207 -2016-05-23 21:54:11 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:54:11 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=386558, maxMem=766075207 -2016-05-23 21:54:11 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 21:54:11 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53778 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 21:54:11 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:11 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:54:11 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 21:54:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:11 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 21:54:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:54:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 18 ms -2016-05-23 21:54:11 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 21:54:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 182 ms on localhost (1/1) -2016-05-23 21:54:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 21:54:11 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.184 s -2016-05-23 21:54:11 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 1.583778 s -2016-05-23 21:54:11 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:54:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 21:54:11 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53778 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 21:54:12 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 21:54:12 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:54:12 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:54:12 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 21:54:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 21:54:12 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 21:54:12 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:54:12 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53778 in memory (size: 3.1 KB, free: 730.5 MB) -2016-05-23 21:54:12 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 21:54:12 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=374970, maxMem=766075207 -2016-05-23 21:54:12 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:54:12 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=371074, maxMem=766075207 -2016-05-23 21:54:12 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53778 in memory (size: 2.7 KB, free: 730.6 MB) -2016-05-23 21:54:12 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:54:12 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 21:54:12 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53778 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:54:12 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53778 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:54:12 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:54:12 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 21:54:12 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:54:12 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53778 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:54:12 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 21:54:12 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 21:54:12 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 21:54:12 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53778 in memory (size: 2.4 KB, free: 730.6 MB) -2016-05-23 21:54:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:54:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:54:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:12 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 21:54:12 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 166 ms on localhost (1/2) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 165 ms on localhost (2/2) -2016-05-23 21:54:12 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.168 s -2016-05-23 21:54:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 21:54:12 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:54:12 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:54:12 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 21:54:12 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:54:12 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 21:54:12 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:54:12 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=349674, maxMem=766075207 -2016-05-23 21:54:12 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 21:54:12 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=354530, maxMem=766075207 -2016-05-23 21:54:12 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:54:12 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53778 (size: 2.5 KB, free: 730.6 MB) -2016-05-23 21:54:12 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:12 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:54:12 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:12 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 21:54:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:54:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:12 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 66 ms on localhost (1/1) -2016-05-23 21:54:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 21:54:12 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.068 s -2016-05-23 21:54:12 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.393147 s -2016-05-23 21:54:12 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:54:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 21:54:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 21:54:12 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:54:12 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 21:54:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:12 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:54:12 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=357061, maxMem=766075207 -2016-05-23 21:54:12 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 21:54:12 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=362717, maxMem=766075207 -2016-05-23 21:54:12 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:54:12 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53778 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:54:12 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:12 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:12 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 21:54:12 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 21:54:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:54:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:12 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 21:54:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:54:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:12 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 31 ms on localhost (1/2) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 30 ms on localhost (2/2) -2016-05-23 21:54:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 21:54:12 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.032 s -2016-05-23 21:54:12 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.048608 s -2016-05-23 21:54:12 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:54:12 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:54:12 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 21:54:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:12 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:54:12 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=365625, maxMem=766075207 -2016-05-23 21:54:12 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:54:12 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=371321, maxMem=766075207 -2016-05-23 21:54:12 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:54:12 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53778 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:54:12 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:12 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:12 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 21:54:12 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 21:54:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:54:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:54:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:12 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 21:54:12 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 24 ms on localhost (1/2) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 31 ms on localhost (2/2) -2016-05-23 21:54:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 21:54:12 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.031 s -2016-05-23 21:54:12 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.114180 s -2016-05-23 21:54:12 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:46 -2016-05-23 21:54:12 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:46) with 4 output partitions -2016-05-23 21:54:12 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:46) -2016-05-23 21:54:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:12 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:43), which has no missing parents -2016-05-23 21:54:12 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=374210, maxMem=766075207 -2016-05-23 21:54:12 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:54:12 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=375434, maxMem=766075207 -2016-05-23 21:54:12 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:54:12 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53778 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:54:12 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:12 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:43) -2016-05-23 21:54:12 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:54:12 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 21:54:12 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 21:54:12 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 21:54:12 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 21:54:12 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 10 ms on localhost (1/4) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 13 ms on localhost (2/4) -2016-05-23 21:54:12 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 21:54:12 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 21:54:12 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 66 ms on localhost (3/4) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 22 ms on localhost (4/4) -2016-05-23 21:54:12 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:46) finished in 0.070 s -2016-05-23 21:54:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 21:54:12 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:46, took 0.089910 s -2016-05-23 21:54:12 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:48 -2016-05-23 21:54:12 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:48) with 4 output partitions -2016-05-23 21:54:12 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:48) -2016-05-23 21:54:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:12 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:44), which has no missing parents -2016-05-23 21:54:12 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=376327, maxMem=766075207 -2016-05-23 21:54:12 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:54:12 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=377551, maxMem=766075207 -2016-05-23 21:54:12 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:54:12 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53778 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:54:12 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:12 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:44) -2016-05-23 21:54:12 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:54:12 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 21:54:12 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 21:54:12 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 21:54:12 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 21:54:12 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 21:54:12 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 12 ms on localhost (1/4) -2016-05-23 21:54:12 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 9 ms on localhost (2/4) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 12 ms on localhost (3/4) -2016-05-23 21:54:12 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 17 ms on localhost (4/4) -2016-05-23 21:54:12 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:48) finished in 0.019 s -2016-05-23 21:54:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 21:54:12 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:48, took 0.031854 s -2016-05-23 21:54:12 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:54:12 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:54:12 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 21:54:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:12 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:54:12 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=378444, maxMem=766075207 -2016-05-23 21:54:12 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:54:12 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=380948, maxMem=766075207 -2016-05-23 21:54:12 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 21:54:12 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53778 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:54:12 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:12 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 21:54:12 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:54:12 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 21:54:12 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 8 ms on localhost (1/1) -2016-05-23 21:54:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 21:54:12 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.009 s -2016-05-23 21:54:12 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.025665 s -2016-05-23 21:54:12 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:54:12 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:54:12 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:12 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:54:12 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=382472, maxMem=766075207 -2016-05-23 21:54:12 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:54:12 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=385872, maxMem=766075207 -2016-05-23 21:54:12 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 21:54:12 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53778 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 21:54:12 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:12 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:12 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:54:12 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 21:54:12 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 21:54:12 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 21:54:12 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 21:54:12 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 21:54:12 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 21:54:12 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 19 ms on localhost (1/4) -2016-05-23 21:54:12 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 16 ms on localhost (2/4) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 20 ms on localhost (3/4) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 21 ms on localhost (4/4) -2016-05-23 21:54:12 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.024 s -2016-05-23 21:54:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 21:54:12 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.041004 s -2016-05-23 21:54:12 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:54:12 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:54:12 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:12 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:54:12 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=387806, maxMem=766075207 -2016-05-23 21:54:12 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:54:12 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=391246, maxMem=766075207 -2016-05-23 21:54:12 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 21:54:12 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53778 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:54:12 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:12 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:12 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:54:12 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 21:54:12 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 21:54:12 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 21:54:12 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 21:54:12 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 21:54:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 17 ms on localhost (1/4) -2016-05-23 21:54:13 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 21:54:13 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 21:54:13 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 76 ms on localhost (2/4) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 72 ms on localhost (3/4) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 79 ms on localhost (4/4) -2016-05-23 21:54:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 21:54:13 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.080 s -2016-05-23 21:54:13 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.101216 s -2016-05-23 21:54:13 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:54:13 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:54:13 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:54:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:13 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:54:13 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=393164, maxMem=766075207 -2016-05-23 21:54:13 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:54:13 INFO MemoryStore:59 - ensureFreeSpace(1860) called with curMem=396540, maxMem=766075207 -2016-05-23 21:54:13 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1860.0 B, free 730.2 MB) -2016-05-23 21:54:13 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53778 (size: 1860.0 B, free: 730.5 MB) -2016-05-23 21:54:13 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:13 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:54:13 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 21:54:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:54:13 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 21:54:13 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 21:54:13 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 21:54:13 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 21:54:13 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 21:54:13 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 21:54:13 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 21:54:13 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 21 ms on localhost (1/4) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 26 ms on localhost (2/4) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 27 ms on localhost (3/4) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 32 ms on localhost (4/4) -2016-05-23 21:54:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 21:54:13 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.032 s -2016-05-23 21:54:13 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.057635 s -2016-05-23 21:54:13 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:54:13 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:54:13 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:54:13 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:54:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 21:54:13 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 21:54:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:54:13 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=398400, maxMem=766075207 -2016-05-23 21:54:13 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 21:54:13 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=402440, maxMem=766075207 -2016-05-23 21:54:13 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:54:13 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:53778 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:54:13 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:13 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:54:13 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 21:54:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 21:54:13 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 21:54:13 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 21:54:13 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 21:54:13 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 21:54:13 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 252 ms on localhost (1/4) -2016-05-23 21:54:13 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 21:54:13 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 21:54:13 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 285 ms on localhost (2/4) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 289 ms on localhost (3/4) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 288 ms on localhost (4/4) -2016-05-23 21:54:13 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.293 s -2016-05-23 21:54:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 21:54:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:54:13 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:54:13 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 21:54:13 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:54:13 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 21:54:13 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:54:13 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=404722, maxMem=766075207 -2016-05-23 21:54:13 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:54:13 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=407266, maxMem=766075207 -2016-05-23 21:54:13 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 21:54:13 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:53778 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 21:54:13 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:13 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:54:13 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 21:54:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:13 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 21:54:13 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 21:54:13 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:13 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 21:54:13 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:13 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 37 ms on localhost (1/3) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 41 ms on localhost (2/3) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 41 ms on localhost (3/3) -2016-05-23 21:54:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 21:54:13 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.044 s -2016-05-23 21:54:13 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.390125 s -2016-05-23 21:54:13 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:54:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 21:54:13 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:54:13 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:54:13 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 21:54:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 21:54:13 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 21:54:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:54:13 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=408834, maxMem=766075207 -2016-05-23 21:54:13 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:54:13 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=413066, maxMem=766075207 -2016-05-23 21:54:13 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 21:54:13 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:53778 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:54:13 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:13 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:54:13 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 21:54:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:54:13 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 21:54:13 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 21:54:13 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 21:54:13 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:13 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 198 ms on localhost (1/4) -2016-05-23 21:54:13 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 237 ms on localhost (2/4) -2016-05-23 21:54:13 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 236 ms on localhost (3/4) -2016-05-23 21:54:13 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 260 ms on localhost (4/4) -2016-05-23 21:54:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 21:54:13 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.261 s -2016-05-23 21:54:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:54:13 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:54:13 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 21:54:13 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:54:13 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 21:54:13 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:54:13 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=415372, maxMem=766075207 -2016-05-23 21:54:13 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:54:13 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=420268, maxMem=766075207 -2016-05-23 21:54:13 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:54:13 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:53778 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:54:13 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:13 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:54:13 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 21:54:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:13 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:13 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 18 ms on localhost (1/1) -2016-05-23 21:54:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 21:54:13 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.019 s -2016-05-23 21:54:13 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.339561 s -2016-05-23 21:54:13 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:54:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 21:54:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 21:54:13 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:54:13 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 21:54:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:13 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:54:13 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=422810, maxMem=766075207 -2016-05-23 21:54:13 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:54:13 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=428506, maxMem=766075207 -2016-05-23 21:54:13 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:54:13 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:53778 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:54:13 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:13 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:13 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 21:54:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:13 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 21:54:13 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:13 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 21:54:13 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:13 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 21:54:13 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 39 ms on localhost (1/4) -2016-05-23 21:54:13 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 38 ms on localhost (2/4) -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:14 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 59 ms on localhost (3/4) -2016-05-23 21:54:14 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 60 ms on localhost (4/4) -2016-05-23 21:54:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 21:54:14 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.063 s -2016-05-23 21:54:14 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.085986 s -2016-05-23 21:54:14 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:54:14 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:54:14 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:14 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 21:54:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:14 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:54:14 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=431415, maxMem=766075207 -2016-05-23 21:54:14 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:54:14 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=437151, maxMem=766075207 -2016-05-23 21:54:14 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:54:14 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:53778 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:54:14 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:14 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:14 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 21:54:14 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 21:54:14 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 21:54:14 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 21:54:14 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:14 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:14 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:14 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 21:54:14 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 21:54:14 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 26 ms on localhost (1/4) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 30 ms on localhost (2/4) -2016-05-23 21:54:14 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 30 ms on localhost (3/4) -2016-05-23 21:54:14 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 46 ms on localhost (4/4) -2016-05-23 21:54:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 21:54:14 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.051 s -2016-05-23 21:54:14 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.069069 s -2016-05-23 21:54:14 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:60 -2016-05-23 21:54:14 INFO DAGScheduler:59 - Got job 20 (foreach at StatisticsCorrLearning.scala:60) with 4 output partitions -2016-05-23 21:54:14 INFO DAGScheduler:59 - Final stage: ResultStage 34(foreach at StatisticsCorrLearning.scala:60) -2016-05-23 21:54:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:14 INFO DAGScheduler:59 - Submitting ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:57), which has no missing parents -2016-05-23 21:54:14 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=440043, maxMem=766075207 -2016-05-23 21:54:14 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:54:14 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=441267, maxMem=766075207 -2016-05-23 21:54:14 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:54:14 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:53778 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:54:14 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:57) -2016-05-23 21:54:14 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 62, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 63, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 64, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:54:14 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 63) -2016-05-23 21:54:14 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 65) -2016-05-23 21:54:14 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 62) -2016-05-23 21:54:14 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 64) -2016-05-23 21:54:14 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 62). 915 bytes result sent to driver -2016-05-23 21:54:14 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 63). 915 bytes result sent to driver -2016-05-23 21:54:14 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 64). 915 bytes result sent to driver -2016-05-23 21:54:14 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 65). 915 bytes result sent to driver -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 63) in 10 ms on localhost (1/4) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 62) in 10 ms on localhost (2/4) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 65) in 10 ms on localhost (3/4) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 64) in 11 ms on localhost (4/4) -2016-05-23 21:54:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-23 21:54:14 INFO DAGScheduler:59 - ResultStage 34 (foreach at StatisticsCorrLearning.scala:60) finished in 0.013 s -2016-05-23 21:54:14 INFO DAGScheduler:59 - Job 20 finished: foreach at StatisticsCorrLearning.scala:60, took 0.024599 s -2016-05-23 21:54:14 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:62 -2016-05-23 21:54:14 INFO DAGScheduler:59 - Got job 21 (foreach at StatisticsCorrLearning.scala:62) with 4 output partitions -2016-05-23 21:54:14 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at StatisticsCorrLearning.scala:62) -2016-05-23 21:54:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:14 INFO DAGScheduler:59 - Submitting ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:58), which has no missing parents -2016-05-23 21:54:14 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=442160, maxMem=766075207 -2016-05-23 21:54:14 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:54:14 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=443384, maxMem=766075207 -2016-05-23 21:54:14 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:54:14 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:53778 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:54:14 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:58) -2016-05-23 21:54:14 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 4 tasks -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 66, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:54:14 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 68) -2016-05-23 21:54:14 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 67) -2016-05-23 21:54:14 INFO Executor:59 - Running task 3.0 in stage 35.0 (TID 69) -2016-05-23 21:54:14 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 68). 915 bytes result sent to driver -2016-05-23 21:54:14 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 67). 915 bytes result sent to driver -2016-05-23 21:54:14 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 66) -2016-05-23 21:54:14 INFO Executor:59 - Finished task 3.0 in stage 35.0 (TID 69). 915 bytes result sent to driver -2016-05-23 21:54:14 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 66). 915 bytes result sent to driver -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 68) in 10 ms on localhost (1/4) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 67) in 12 ms on localhost (2/4) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 35.0 (TID 69) in 10 ms on localhost (3/4) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 66) in 13 ms on localhost (4/4) -2016-05-23 21:54:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-23 21:54:14 INFO DAGScheduler:59 - ResultStage 35 (foreach at StatisticsCorrLearning.scala:62) finished in 0.014 s -2016-05-23 21:54:14 INFO DAGScheduler:59 - Job 21 finished: foreach at StatisticsCorrLearning.scala:62, took 0.025114 s -2016-05-23 21:54:14 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:54:14 INFO DAGScheduler:59 - Got job 22 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:54:14 INFO DAGScheduler:59 - Final stage: ResultStage 36(first at RowMatrix.scala:65) -2016-05-23 21:54:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:14 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:54:14 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=444277, maxMem=766075207 -2016-05-23 21:54:14 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:54:14 INFO MemoryStore:59 - ensureFreeSpace(1522) called with curMem=446781, maxMem=766075207 -2016-05-23 21:54:14 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1522.0 B, free 730.2 MB) -2016-05-23 21:54:14 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:53778 (size: 1522.0 B, free: 730.5 MB) -2016-05-23 21:54:14 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:14 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 21:54:14 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:14 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 70) -2016-05-23 21:54:14 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 70). 1046 bytes result sent to driver -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 70) in 4 ms on localhost (1/1) -2016-05-23 21:54:14 INFO DAGScheduler:59 - ResultStage 36 (first at RowMatrix.scala:65) finished in 0.005 s -2016-05-23 21:54:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-23 21:54:14 INFO DAGScheduler:59 - Job 22 finished: first at RowMatrix.scala:65, took 0.019088 s -2016-05-23 21:54:14 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:54:14 INFO DAGScheduler:59 - Got job 23 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:54:14 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:14 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:54:14 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=448303, maxMem=766075207 -2016-05-23 21:54:14 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:54:14 INFO MemoryStore:59 - ensureFreeSpace(1932) called with curMem=451703, maxMem=766075207 -2016-05-23 21:54:14 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1932.0 B, free 730.2 MB) -2016-05-23 21:54:14 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:53778 (size: 1932.0 B, free: 730.5 MB) -2016-05-23 21:54:14 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:14 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 4 tasks -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:54:14 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 72) -2016-05-23 21:54:14 INFO Executor:59 - Running task 3.0 in stage 37.0 (TID 74) -2016-05-23 21:54:14 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-23 21:54:14 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 73) -2016-05-23 21:54:14 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 72). 1231 bytes result sent to driver -2016-05-23 21:54:14 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 1231 bytes result sent to driver -2016-05-23 21:54:14 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 73). 1231 bytes result sent to driver -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 72) in 10 ms on localhost (1/4) -2016-05-23 21:54:14 INFO Executor:59 - Finished task 3.0 in stage 37.0 (TID 74). 1231 bytes result sent to driver -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 16 ms on localhost (2/4) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 37.0 (TID 74) in 13 ms on localhost (3/4) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 73) in 15 ms on localhost (4/4) -2016-05-23 21:54:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-23 21:54:14 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at RowMatrix.scala:332) finished in 0.017 s -2016-05-23 21:54:14 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at RowMatrix.scala:332, took 0.030879 s -2016-05-23 21:54:14 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:54:14 INFO DAGScheduler:59 - Got job 24 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:54:14 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:14 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:54:14 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=453635, maxMem=766075207 -2016-05-23 21:54:14 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:54:14 INFO MemoryStore:59 - ensureFreeSpace(1916) called with curMem=457075, maxMem=766075207 -2016-05-23 21:54:14 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1916.0 B, free 730.1 MB) -2016-05-23 21:54:14 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:53778 (size: 1916.0 B, free: 730.5 MB) -2016-05-23 21:54:14 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:14 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 77, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 78, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:54:14 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-23 21:54:14 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 77) -2016-05-23 21:54:14 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 78) -2016-05-23 21:54:14 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-23 21:54:14 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 77). 1109 bytes result sent to driver -2016-05-23 21:54:14 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 78). 1109 bytes result sent to driver -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 77) in 9 ms on localhost (1/4) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 78) in 9 ms on localhost (2/4) -2016-05-23 21:54:14 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 1109 bytes result sent to driver -2016-05-23 21:54:14 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 1109 bytes result sent to driver -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 15 ms on localhost (3/4) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 18 ms on localhost (4/4) -2016-05-23 21:54:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-23 21:54:14 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at RowMatrix.scala:124) finished in 0.028 s -2016-05-23 21:54:14 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at RowMatrix.scala:124, took 0.049742 s -2016-05-23 21:54:14 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:54:14 INFO DAGScheduler:59 - Got job 25 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:54:14 INFO DAGScheduler:59 - Final stage: ResultStage 39(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:54:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:14 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:54:14 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=458991, maxMem=766075207 -2016-05-23 21:54:14 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 21:54:14 INFO MemoryStore:59 - ensureFreeSpace(1851) called with curMem=462367, maxMem=766075207 -2016-05-23 21:54:14 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1851.0 B, free 730.1 MB) -2016-05-23 21:54:14 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:53778 (size: 1851.0 B, free: 730.5 MB) -2016-05-23 21:54:14 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:54:14 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 4 tasks -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 79, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 80, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 81, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 39.0 (TID 82, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:54:14 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 79) -2016-05-23 21:54:14 INFO Executor:59 - Running task 3.0 in stage 39.0 (TID 82) -2016-05-23 21:54:14 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 81) -2016-05-23 21:54:14 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 79). 1254 bytes result sent to driver -2016-05-23 21:54:14 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 81). 1249 bytes result sent to driver -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 79) in 9 ms on localhost (1/4) -2016-05-23 21:54:14 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 80) -2016-05-23 21:54:14 INFO Executor:59 - Finished task 3.0 in stage 39.0 (TID 82). 1294 bytes result sent to driver -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 81) in 9 ms on localhost (2/4) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 39.0 (TID 82) in 11 ms on localhost (3/4) -2016-05-23 21:54:14 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 80). 1254 bytes result sent to driver -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 80) in 15 ms on localhost (4/4) -2016-05-23 21:54:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-23 21:54:14 INFO DAGScheduler:59 - ResultStage 39 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.017 s -2016-05-23 21:54:14 INFO DAGScheduler:59 - Job 25 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.034038 s -2016-05-23 21:54:14 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:54:14 INFO DAGScheduler:59 - Registering RDD 51 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:54:14 INFO DAGScheduler:59 - Got job 26 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:54:14 INFO DAGScheduler:59 - Final stage: ResultStage 41(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:54:14 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 40) -2016-05-23 21:54:14 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 40) -2016-05-23 21:54:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:54:14 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=464218, maxMem=766075207 -2016-05-23 21:54:14 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-05-23 21:54:14 INFO MemoryStore:59 - ensureFreeSpace(2281) called with curMem=468258, maxMem=766075207 -2016-05-23 21:54:14 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 21:54:14 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:53778 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:54:14 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:54:14 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 4 tasks -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 83, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 84, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 85, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:54:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 40.0 (TID 86, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:54:14 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 83) -2016-05-23 21:54:14 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 85) -2016-05-23 21:54:14 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 84) -2016-05-23 21:54:14 INFO Executor:59 - Running task 3.0 in stage 40.0 (TID 86) -2016-05-23 21:54:14 INFO Executor:59 - Finished task 3.0 in stage 40.0 (TID 86). 1161 bytes result sent to driver -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 40.0 (TID 86) in 121 ms on localhost (1/4) -2016-05-23 21:54:14 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 83). 1161 bytes result sent to driver -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 83) in 171 ms on localhost (2/4) -2016-05-23 21:54:14 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 85). 1161 bytes result sent to driver -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 85) in 228 ms on localhost (3/4) -2016-05-23 21:54:14 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 84). 1161 bytes result sent to driver -2016-05-23 21:54:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 84) in 346 ms on localhost (4/4) -2016-05-23 21:54:14 INFO DAGScheduler:59 - ShuffleMapStage 40 (flatMap at SpearmanCorrelation.scala:49) finished in 0.348 s -2016-05-23 21:54:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-23 21:54:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:54:14 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:54:14 INFO DAGScheduler:59 - waiting: Set(ResultStage 41) -2016-05-23 21:54:14 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:54:14 INFO DAGScheduler:59 - Missing parents for ResultStage 41: List() -2016-05-23 21:54:14 INFO DAGScheduler:59 - Submitting ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:54:15 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=470539, maxMem=766075207 -2016-05-23 21:54:15 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:54:15 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=473083, maxMem=766075207 -2016-05-23 21:54:15 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.1 MB) -2016-05-23 21:54:15 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:53778 (size: 1567.0 B, free: 730.5 MB) -2016-05-23 21:54:15 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:15 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:54:15 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 3 tasks -2016-05-23 21:54:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 87, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:15 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 88) -2016-05-23 21:54:15 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 89) -2016-05-23 21:54:15 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 87) -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:15 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 88). 1203 bytes result sent to driver -2016-05-23 21:54:15 INFO Executor:59 - Finished task 2.0 in stage 41.0 (TID 89). 1203 bytes result sent to driver -2016-05-23 21:54:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 41.0 (TID 89) in 11 ms on localhost (1/3) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 88) in 11 ms on localhost (2/3) -2016-05-23 21:54:15 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 87). 1203 bytes result sent to driver -2016-05-23 21:54:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 87) in 13 ms on localhost (3/3) -2016-05-23 21:54:15 INFO DAGScheduler:59 - ResultStage 41 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.015 s -2016-05-23 21:54:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-23 21:54:15 INFO DAGScheduler:59 - Job 26 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.403206 s -2016-05-23 21:54:15 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:54:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 163 bytes -2016-05-23 21:54:15 INFO DAGScheduler:59 - Registering RDD 56 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:54:15 INFO DAGScheduler:59 - Got job 27 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:54:15 INFO DAGScheduler:59 - Final stage: ResultStage 44(first at RowMatrix.scala:65) -2016-05-23 21:54:15 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 43) -2016-05-23 21:54:15 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 43) -2016-05-23 21:54:15 INFO DAGScheduler:59 - Submitting ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:54:15 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=474650, maxMem=766075207 -2016-05-23 21:54:15 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 21:54:15 INFO MemoryStore:59 - ensureFreeSpace(2305) called with curMem=478882, maxMem=766075207 -2016-05-23 21:54:15 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-23 21:54:15 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:53778 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:54:15 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:54:15 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 4 tasks -2016-05-23 21:54:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 90, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 91, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 43.0 (TID 92, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 43.0 (TID 93, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:54:15 INFO Executor:59 - Running task 2.0 in stage 43.0 (TID 92) -2016-05-23 21:54:15 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 90) -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:15 INFO Executor:59 - Running task 3.0 in stage 43.0 (TID 93) -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:15 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 91) -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:15 INFO Executor:59 - Finished task 3.0 in stage 43.0 (TID 93). 1377 bytes result sent to driver -2016-05-23 21:54:15 INFO TaskSetManager:59 - Finished task 3.0 in stage 43.0 (TID 93) in 185 ms on localhost (1/4) -2016-05-23 21:54:15 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 90). 1377 bytes result sent to driver -2016-05-23 21:54:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 90) in 191 ms on localhost (2/4) -2016-05-23 21:54:15 INFO Executor:59 - Finished task 2.0 in stage 43.0 (TID 92). 1377 bytes result sent to driver -2016-05-23 21:54:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 43.0 (TID 92) in 197 ms on localhost (3/4) -2016-05-23 21:54:15 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 91). 1377 bytes result sent to driver -2016-05-23 21:54:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 91) in 202 ms on localhost (4/4) -2016-05-23 21:54:15 INFO DAGScheduler:59 - ShuffleMapStage 43 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.203 s -2016-05-23 21:54:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-23 21:54:15 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:54:15 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:54:15 INFO DAGScheduler:59 - waiting: Set(ResultStage 44) -2016-05-23 21:54:15 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:54:15 INFO DAGScheduler:59 - Missing parents for ResultStage 44: List() -2016-05-23 21:54:15 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:54:15 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=481187, maxMem=766075207 -2016-05-23 21:54:15 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 21:54:15 INFO MemoryStore:59 - ensureFreeSpace(2545) called with curMem=486083, maxMem=766075207 -2016-05-23 21:54:15 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:54:15 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:53778 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:54:15 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:15 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:54:15 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 1 tasks -2016-05-23 21:54:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 94, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:15 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 94) -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:15 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 94). 1296 bytes result sent to driver -2016-05-23 21:54:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 94) in 9 ms on localhost (1/1) -2016-05-23 21:54:15 INFO DAGScheduler:59 - ResultStage 44 (first at RowMatrix.scala:65) finished in 0.009 s -2016-05-23 21:54:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-23 21:54:15 INFO DAGScheduler:59 - Job 27 finished: first at RowMatrix.scala:65, took 0.236233 s -2016-05-23 21:54:15 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:54:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 163 bytes -2016-05-23 21:54:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 166 bytes -2016-05-23 21:54:15 INFO DAGScheduler:59 - Got job 28 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:54:15 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:15 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 46) -2016-05-23 21:54:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:15 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:54:15 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=488628, maxMem=766075207 -2016-05-23 21:54:15 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 21:54:15 INFO MemoryStore:59 - ensureFreeSpace(2914) called with curMem=494324, maxMem=766075207 -2016-05-23 21:54:15 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 21:54:15 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:53778 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:54:15 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:15 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 4 tasks -2016-05-23 21:54:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 95, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 96, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 47.0 (TID 97, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 47.0 (TID 98, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:15 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 95) -2016-05-23 21:54:15 INFO Executor:59 - Running task 2.0 in stage 47.0 (TID 97) -2016-05-23 21:54:15 INFO Executor:59 - Running task 3.0 in stage 47.0 (TID 98) -2016-05-23 21:54:15 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 96) -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:15 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 95). 1481 bytes result sent to driver -2016-05-23 21:54:15 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 96). 1481 bytes result sent to driver -2016-05-23 21:54:15 INFO Executor:59 - Finished task 3.0 in stage 47.0 (TID 98). 1481 bytes result sent to driver -2016-05-23 21:54:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 95) in 13 ms on localhost (1/4) -2016-05-23 21:54:15 INFO Executor:59 - Finished task 2.0 in stage 47.0 (TID 97). 1481 bytes result sent to driver -2016-05-23 21:54:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 96) in 13 ms on localhost (2/4) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Finished task 3.0 in stage 47.0 (TID 98) in 14 ms on localhost (3/4) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 47.0 (TID 97) in 14 ms on localhost (4/4) -2016-05-23 21:54:15 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at RowMatrix.scala:332) finished in 0.015 s -2016-05-23 21:54:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-23 21:54:15 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at RowMatrix.scala:332, took 0.028221 s -2016-05-23 21:54:15 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:54:15 INFO DAGScheduler:59 - Got job 29 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:54:15 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:15 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 49) -2016-05-23 21:54:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:15 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:54:15 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=497238, maxMem=766075207 -2016-05-23 21:54:15 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 21:54:15 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=502974, maxMem=766075207 -2016-05-23 21:54:15 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 21:54:15 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:53778 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:54:15 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:15 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 4 tasks -2016-05-23 21:54:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 50.0 (TID 101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 50.0 (TID 102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:15 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-23 21:54:15 INFO Executor:59 - Running task 3.0 in stage 50.0 (TID 102) -2016-05-23 21:54:15 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-23 21:54:15 INFO Executor:59 - Running task 2.0 in stage 50.0 (TID 101) -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 21:54:15 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 1359 bytes result sent to driver -2016-05-23 21:54:15 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 1359 bytes result sent to driver -2016-05-23 21:54:15 INFO Executor:59 - Finished task 2.0 in stage 50.0 (TID 101). 1359 bytes result sent to driver -2016-05-23 21:54:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 13 ms on localhost (1/4) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 15 ms on localhost (2/4) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 50.0 (TID 101) in 15 ms on localhost (3/4) -2016-05-23 21:54:15 INFO Executor:59 - Finished task 3.0 in stage 50.0 (TID 102). 1359 bytes result sent to driver -2016-05-23 21:54:15 INFO TaskSetManager:59 - Finished task 3.0 in stage 50.0 (TID 102) in 17 ms on localhost (4/4) -2016-05-23 21:54:15 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at RowMatrix.scala:124) finished in 0.019 s -2016-05-23 21:54:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-23 21:54:15 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at RowMatrix.scala:124, took 0.029461 s -2016-05-23 21:54:15 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:73 -2016-05-23 21:54:15 INFO DAGScheduler:59 - Got job 30 (foreach at StatisticsCorrLearning.scala:73) with 4 output partitions -2016-05-23 21:54:15 INFO DAGScheduler:59 - Final stage: ResultStage 51(foreach at StatisticsCorrLearning.scala:73) -2016-05-23 21:54:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:15 INFO DAGScheduler:59 - Submitting ResultStage 51 (ParallelCollectionRDD[61] at parallelize at StatisticsCorrLearning.scala:70), which has no missing parents -2016-05-23 21:54:15 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=505870, maxMem=766075207 -2016-05-23 21:54:15 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-05-23 21:54:15 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=507094, maxMem=766075207 -2016-05-23 21:54:15 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.1 MB) -2016-05-23 21:54:15 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:53778 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:54:15 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 51 (ParallelCollectionRDD[61] at parallelize at StatisticsCorrLearning.scala:70) -2016-05-23 21:54:15 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 4 tasks -2016-05-23 21:54:15 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:53778 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 103, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 104, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:54:15 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-23 21:54:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 51.0 (TID 105, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 51.0 (TID 106, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:54:15 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 104) -2016-05-23 21:54:15 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 103) -2016-05-23 21:54:15 INFO Executor:59 - Running task 3.0 in stage 51.0 (TID 106) -2016-05-23 21:54:15 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 104). 915 bytes result sent to driver -2016-05-23 21:54:15 INFO Executor:59 - Running task 2.0 in stage 51.0 (TID 105) -2016-05-23 21:54:15 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 103). 915 bytes result sent to driver -2016-05-23 21:54:15 INFO Executor:59 - Finished task 2.0 in stage 51.0 (TID 105). 915 bytes result sent to driver -2016-05-23 21:54:15 INFO Executor:59 - Finished task 3.0 in stage 51.0 (TID 106). 915 bytes result sent to driver -2016-05-23 21:54:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 104) in 12 ms on localhost (1/4) -2016-05-23 21:54:15 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:53778 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 103) in 13 ms on localhost (2/4) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Finished task 3.0 in stage 51.0 (TID 106) in 8 ms on localhost (3/4) -2016-05-23 21:54:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 51.0 (TID 105) in 8 ms on localhost (4/4) -2016-05-23 21:54:15 INFO DAGScheduler:59 - ResultStage 51 (foreach at StatisticsCorrLearning.scala:73) finished in 0.016 s -2016-05-23 21:54:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-23 21:54:15 INFO DAGScheduler:59 - Job 30 finished: foreach at StatisticsCorrLearning.scala:73, took 0.050380 s -2016-05-23 21:54:15 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-23 21:54:15 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:53778 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:54:15 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-23 21:54:15 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:53778 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:54:15 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-23 21:54:16 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:75 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Got job 31 (foreach at StatisticsCorrLearning.scala:75) with 4 output partitions -2016-05-23 21:54:16 INFO DAGScheduler:59 - Final stage: ResultStage 52(foreach at StatisticsCorrLearning.scala:75) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting ResultStage 52 (ParallelCollectionRDD[62] at parallelize at StatisticsCorrLearning.scala:71), which has no missing parents -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned shuffle 5 -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=476767, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:53778 in memory (size: 1567.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=476424, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.1 MB) -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:53778 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 52 (ParallelCollectionRDD[62] at parallelize at StatisticsCorrLearning.scala:71) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 4 tasks -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:53778 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 107, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 108, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 52.0 (TID 109, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 52.0 (TID 110, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:54:16 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 108) -2016-05-23 21:54:16 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 107) -2016-05-23 21:54:16 INFO Executor:59 - Running task 3.0 in stage 52.0 (TID 110) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 108). 915 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Finished task 3.0 in stage 52.0 (TID 110). 915 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 107). 915 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Running task 2.0 in stage 52.0 (TID 109) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 108) in 12 ms on localhost (1/4) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 52.0 (TID 110) in 12 ms on localhost (2/4) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 2.0 in stage 52.0 (TID 109). 915 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 107) in 17 ms on localhost (3/4) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 52.0 (TID 109) in 16 ms on localhost (4/4) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-23 21:54:16 INFO DAGScheduler:59 - ResultStage 52 (foreach at StatisticsCorrLearning.scala:75) finished in 0.020 s -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned shuffle 4 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Job 31 finished: foreach at StatisticsCorrLearning.scala:75, took 0.163832 s -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:53778 in memory (size: 1851.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:53778 in memory (size: 1916.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-23 21:54:16 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:53778 in memory (size: 1932.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Got job 32 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:54:16 INFO DAGScheduler:59 - Final stage: ResultStage 53(first at RowMatrix.scala:65) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[64] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=448511, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:53778 in memory (size: 1522.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=451015, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:53778 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 53 (MapPartitionsRDD[64] at map at Correlation.scala:45) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 1 tasks -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:53778 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 111, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-23 21:54:16 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 111) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 111). 935 bytes result sent to driver -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:53778 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO DAGScheduler:59 - ResultStage 53 (first at RowMatrix.scala:65) finished in 0.005 s -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 111) in 5 ms on localhost (1/1) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-23 21:54:16 INFO DAGScheduler:59 - Job 32 finished: first at RowMatrix.scala:65, took 0.016960 s -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:53778 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-23 21:54:16 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:53778 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Got job 33 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 21:54:16 INFO DAGScheduler:59 - Final stage: ResultStage 54(first at RowMatrix.scala:65) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[64] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=428530, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:53778 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=426138, maxMem=766075207 -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:53778 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:53778 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 54 (MapPartitionsRDD[64] at map at Correlation.scala:45) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 3 tasks -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 112, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 113, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 54.0 (TID 114, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:16 INFO Executor:59 - Running task 2.0 in stage 54.0 (TID 114) -2016-05-23 21:54:16 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 113) -2016-05-23 21:54:16 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 112) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 2.0 in stage 54.0 (TID 114). 1046 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 112). 1046 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 54.0 (TID 114) in 7 ms on localhost (1/3) -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:53778 in memory (size: 1568.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 112) in 11 ms on localhost (2/3) -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-23 21:54:16 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 113). 1046 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 113) in 16 ms on localhost (3/3) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-23 21:54:16 INFO DAGScheduler:59 - ResultStage 54 (first at RowMatrix.scala:65) finished in 0.017 s -2016-05-23 21:54:16 INFO DAGScheduler:59 - Job 33 finished: first at RowMatrix.scala:65, took 0.026989 s -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:53778 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:53778 in memory (size: 1860.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:53778 in memory (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-23 21:54:16 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Got job 34 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:54:16 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[65] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:53778 in memory (size: 1934.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=398162, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=398162, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:53778 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 55 (MapPartitionsRDD[65] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 4 tasks -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:53778 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 115, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 116, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 55.0 (TID 117, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 55.0 (TID 118, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:16 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 115) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 115). 1231 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Running task 2.0 in stage 55.0 (TID 117) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 115) in 11 ms on localhost (1/4) -2016-05-23 21:54:16 INFO Executor:59 - Running task 3.0 in stage 55.0 (TID 118) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 2.0 in stage 55.0 (TID 117). 1231 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Finished task 3.0 in stage 55.0 (TID 118). 1231 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 116) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 116). 1231 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 55.0 (TID 117) in 21 ms on localhost (2/4) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 55.0 (TID 118) in 21 ms on localhost (3/4) -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 116) in 25 ms on localhost (4/4) -2016-05-23 21:54:16 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at RowMatrix.scala:332) finished in 0.027 s -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:53778 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at RowMatrix.scala:332, took 0.046983 s -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:53778 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:53778 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-23 21:54:16 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Got job 35 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:54:16 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[66] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:53778 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=369829, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=370738, maxMem=766075207 -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:53778 in memory (size: 2.5 KB, free: 730.6 MB) -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:53778 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 56 (MapPartitionsRDD[66] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 4 tasks -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:53778 in memory (size: 2.2 KB, free: 730.6 MB) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 119, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 120, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 56.0 (TID 121, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 56.0 (TID 122, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:16 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 120) -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-23 21:54:16 INFO Executor:59 - Running task 2.0 in stage 56.0 (TID 121) -2016-05-23 21:54:16 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 119) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 120). 1109 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Finished task 2.0 in stage 56.0 (TID 121). 1109 bytes result sent to driver -2016-05-23 21:54:16 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-23 21:54:16 INFO Executor:59 - Running task 3.0 in stage 56.0 (TID 122) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 119). 1109 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 120) in 13 ms on localhost (1/4) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 56.0 (TID 121) in 13 ms on localhost (2/4) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 3.0 in stage 56.0 (TID 122). 1109 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 119) in 18 ms on localhost (3/4) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 56.0 (TID 122) in 17 ms on localhost (4/4) -2016-05-23 21:54:16 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at RowMatrix.scala:124) finished in 0.019 s -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-23 21:54:16 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at RowMatrix.scala:124, took 0.031487 s -2016-05-23 21:54:16 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Got job 36 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:54:16 INFO DAGScheduler:59 - Final stage: ResultStage 57(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[72] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=366182, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(1858) called with curMem=369558, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 1858.0 B, free 730.2 MB) -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:53778 (size: 1858.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 57 (MapPartitionsRDD[72] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 4 tasks -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 123, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 124, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 57.0 (TID 125, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 57.0 (TID 126, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:16 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 123) -2016-05-23 21:54:16 INFO Executor:59 - Running task 3.0 in stage 57.0 (TID 126) -2016-05-23 21:54:16 INFO Executor:59 - Running task 2.0 in stage 57.0 (TID 125) -2016-05-23 21:54:16 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 124) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 123). 1104 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Finished task 3.0 in stage 57.0 (TID 126). 1254 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Finished task 2.0 in stage 57.0 (TID 125). 1249 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 124). 1254 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 57.0 (TID 126) in 6 ms on localhost (1/4) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 123) in 8 ms on localhost (2/4) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 57.0 (TID 125) in 8 ms on localhost (3/4) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 124) in 10 ms on localhost (4/4) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-23 21:54:16 INFO DAGScheduler:59 - ResultStage 57 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.011 s -2016-05-23 21:54:16 INFO DAGScheduler:59 - Job 36 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.021466 s -2016-05-23 21:54:16 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Registering RDD 70 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Got job 37 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:54:16 INFO DAGScheduler:59 - Final stage: ResultStage 59(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 58) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 58) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 58 (MapPartitionsRDD[70] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=371416, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(2285) called with curMem=375456, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:53778 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:54:16 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 58 (MapPartitionsRDD[70] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 4 tasks -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 127, localhost, PROCESS_LOCAL, 2221 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 128, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 58.0 (TID 129, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 58.0 (TID 130, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:54:16 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 128) -2016-05-23 21:54:16 INFO Executor:59 - Running task 2.0 in stage 58.0 (TID 129) -2016-05-23 21:54:16 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 127) -2016-05-23 21:54:16 INFO Executor:59 - Running task 3.0 in stage 58.0 (TID 130) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 127). 1161 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 127) in 11 ms on localhost (1/4) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 2.0 in stage 58.0 (TID 129). 1161 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Finished task 3.0 in stage 58.0 (TID 130). 1161 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 128). 1161 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 58.0 (TID 129) in 42 ms on localhost (2/4) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 58.0 (TID 130) in 43 ms on localhost (3/4) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 128) in 45 ms on localhost (4/4) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-05-23 21:54:16 INFO DAGScheduler:59 - ShuffleMapStage 58 (flatMap at SpearmanCorrelation.scala:49) finished in 0.046 s -2016-05-23 21:54:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:54:16 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:54:16 INFO DAGScheduler:59 - waiting: Set(ResultStage 59) -2016-05-23 21:54:16 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:54:16 INFO DAGScheduler:59 - Missing parents for ResultStage 59: List() -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting ResultStage 59 (ShuffledRDD[73] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=377741, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=380285, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.2 MB) -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:53778 (size: 1567.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 59 (ShuffledRDD[73] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 3 tasks -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 131, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 132, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 59.0 (TID 133, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:16 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 132) -2016-05-23 21:54:16 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 131) -2016-05-23 21:54:16 INFO Executor:59 - Running task 2.0 in stage 59.0 (TID 133) -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:16 INFO Executor:59 - Finished task 2.0 in stage 59.0 (TID 133). 1203 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 131). 1203 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 132). 1203 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 59.0 (TID 133) in 13 ms on localhost (1/3) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 132) in 15 ms on localhost (2/3) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 131) in 16 ms on localhost (3/3) -2016-05-23 21:54:16 INFO DAGScheduler:59 - ResultStage 59 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.017 s -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-05-23 21:54:16 INFO DAGScheduler:59 - Job 37 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.079926 s -2016-05-23 21:54:16 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:54:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 168 bytes -2016-05-23 21:54:16 INFO DAGScheduler:59 - Registering RDD 75 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Got job 38 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:54:16 INFO DAGScheduler:59 - Final stage: ResultStage 62(first at RowMatrix.scala:65) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 61) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 61) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 61 (MapPartitionsRDD[75] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=381852, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(2305) called with curMem=386084, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:53778 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:54:16 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 61 (MapPartitionsRDD[75] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 4 tasks -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 134, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 135, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 61.0 (TID 136, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 61.0 (TID 137, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:54:16 INFO Executor:59 - Running task 2.0 in stage 61.0 (TID 136) -2016-05-23 21:54:16 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 135) -2016-05-23 21:54:16 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 134) -2016-05-23 21:54:16 INFO Executor:59 - Running task 3.0 in stage 61.0 (TID 137) -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:16 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 134). 1377 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 134) in 46 ms on localhost (1/4) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 2.0 in stage 61.0 (TID 136). 1377 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 61.0 (TID 136) in 57 ms on localhost (2/4) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 135). 1377 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 135) in 63 ms on localhost (3/4) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 3.0 in stage 61.0 (TID 137). 1377 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 61.0 (TID 137) in 66 ms on localhost (4/4) -2016-05-23 21:54:16 INFO DAGScheduler:59 - ShuffleMapStage 61 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.068 s -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-05-23 21:54:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:54:16 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:54:16 INFO DAGScheduler:59 - waiting: Set(ResultStage 62) -2016-05-23 21:54:16 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:54:16 INFO DAGScheduler:59 - Missing parents for ResultStage 62: List() -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=388389, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=393285, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:53778 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:54:16 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 62 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 1 tasks -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 138, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:16 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 138) -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:16 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 138). 1185 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 138) in 9 ms on localhost (1/1) -2016-05-23 21:54:16 INFO DAGScheduler:59 - ResultStage 62 (first at RowMatrix.scala:65) finished in 0.009 s -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-05-23 21:54:16 INFO DAGScheduler:59 - Job 38 finished: first at RowMatrix.scala:65, took 0.097009 s -2016-05-23 21:54:16 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:54:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 168 bytes -2016-05-23 21:54:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 165 bytes -2016-05-23 21:54:16 INFO DAGScheduler:59 - Got job 39 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 21:54:16 INFO DAGScheduler:59 - Final stage: ResultStage 65(first at RowMatrix.scala:65) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 64) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87), which has no missing parents -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=395829, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=400725, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:53778 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:54:16 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 65 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 3 tasks -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 139, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 140, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 65.0 (TID 141, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:16 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 139) -2016-05-23 21:54:16 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 140) -2016-05-23 21:54:16 INFO Executor:59 - Running task 2.0 in stage 65.0 (TID 141) -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:16 INFO Executor:59 - Finished task 2.0 in stage 65.0 (TID 141). 1296 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 140). 1296 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 140) in 13 ms on localhost (1/3) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 65.0 (TID 141) in 14 ms on localhost (2/3) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 139). 1296 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 139) in 18 ms on localhost (3/3) -2016-05-23 21:54:16 INFO DAGScheduler:59 - ResultStage 65 (first at RowMatrix.scala:65) finished in 0.018 s -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-05-23 21:54:16 INFO DAGScheduler:59 - Job 39 finished: first at RowMatrix.scala:65, took 0.031349 s -2016-05-23 21:54:16 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Got job 40 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:54:16 INFO DAGScheduler:59 - Final stage: ResultStage 68(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 67) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting ResultStage 68 (MapPartitionsRDD[78] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=403269, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(2914) called with curMem=408965, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:53778 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:54:16 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 68 (MapPartitionsRDD[78] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Adding task set 68.0 with 4 tasks -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 68.0 (TID 142, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 68.0 (TID 143, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 68.0 (TID 144, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 68.0 (TID 145, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:16 INFO Executor:59 - Running task 3.0 in stage 68.0 (TID 145) -2016-05-23 21:54:16 INFO Executor:59 - Running task 0.0 in stage 68.0 (TID 142) -2016-05-23 21:54:16 INFO Executor:59 - Running task 2.0 in stage 68.0 (TID 144) -2016-05-23 21:54:16 INFO Executor:59 - Running task 1.0 in stage 68.0 (TID 143) -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 3 ms -2016-05-23 21:54:16 INFO Executor:59 - Finished task 3.0 in stage 68.0 (TID 145). 1481 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Finished task 0.0 in stage 68.0 (TID 142). 1481 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Finished task 1.0 in stage 68.0 (TID 143). 1481 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Finished task 2.0 in stage 68.0 (TID 144). 1481 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 68.0 (TID 145) in 21 ms on localhost (1/4) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 68.0 (TID 142) in 24 ms on localhost (2/4) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 68.0 (TID 144) in 24 ms on localhost (3/4) -2016-05-23 21:54:16 INFO DAGScheduler:59 - ResultStage 68 (treeAggregate at RowMatrix.scala:332) finished in 0.027 s -2016-05-23 21:54:16 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at RowMatrix.scala:332, took 0.038030 s -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 68.0 (TID 143) in 25 ms on localhost (4/4) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 68.0, whose tasks have all completed, from pool -2016-05-23 21:54:16 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Got job 41 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:54:16 INFO DAGScheduler:59 - Final stage: ResultStage 71(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 70) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting ResultStage 71 (MapPartitionsRDD[79] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=411879, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=417615, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:53778 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:54:16 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 71 (MapPartitionsRDD[79] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Adding task set 71.0 with 4 tasks -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 71.0 (TID 146, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 71.0 (TID 147, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 71.0 (TID 148, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 71.0 (TID 149, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:16 INFO Executor:59 - Running task 0.0 in stage 71.0 (TID 146) -2016-05-23 21:54:16 INFO Executor:59 - Running task 2.0 in stage 71.0 (TID 148) -2016-05-23 21:54:16 INFO Executor:59 - Running task 1.0 in stage 71.0 (TID 147) -2016-05-23 21:54:16 INFO Executor:59 - Running task 3.0 in stage 71.0 (TID 149) -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:16 INFO Executor:59 - Finished task 1.0 in stage 71.0 (TID 147). 1359 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Finished task 0.0 in stage 71.0 (TID 146). 1359 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 71.0 (TID 147) in 17 ms on localhost (1/4) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 71.0 (TID 146) in 19 ms on localhost (2/4) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 3.0 in stage 71.0 (TID 149). 1359 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Finished task 2.0 in stage 71.0 (TID 148). 1359 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 71.0 (TID 149) in 23 ms on localhost (3/4) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 71.0 (TID 148) in 25 ms on localhost (4/4) -2016-05-23 21:54:16 INFO DAGScheduler:59 - ResultStage 71 (treeAggregate at RowMatrix.scala:124) finished in 0.026 s -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 71.0, whose tasks have all completed, from pool -2016-05-23 21:54:16 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at RowMatrix.scala:124, took 0.039773 s -2016-05-23 21:54:16 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:87 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Got job 42 (foreach at StatisticsCorrLearning.scala:87) with 4 output partitions -2016-05-23 21:54:16 INFO DAGScheduler:59 - Final stage: ResultStage 72(foreach at StatisticsCorrLearning.scala:87) -2016-05-23 21:54:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting ResultStage 72 (ParallelCollectionRDD[80] at parallelize at StatisticsCorrLearning.scala:84), which has no missing parents -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=420511, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:54:16 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=421735, maxMem=766075207 -2016-05-23 21:54:16 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:54:16 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:53778 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:54:16 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 72 (ParallelCollectionRDD[80] at parallelize at StatisticsCorrLearning.scala:84) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Adding task set 72.0 with 4 tasks -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 72.0 (TID 150, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 72.0 (TID 151, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 72.0 (TID 152, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 72.0 (TID 153, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:54:16 INFO Executor:59 - Running task 0.0 in stage 72.0 (TID 150) -2016-05-23 21:54:16 INFO Executor:59 - Running task 3.0 in stage 72.0 (TID 153) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 0.0 in stage 72.0 (TID 150). 915 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Running task 1.0 in stage 72.0 (TID 151) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 3.0 in stage 72.0 (TID 153). 915 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Finished task 1.0 in stage 72.0 (TID 151). 915 bytes result sent to driver -2016-05-23 21:54:16 INFO Executor:59 - Running task 2.0 in stage 72.0 (TID 152) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 72.0 (TID 150) in 6 ms on localhost (1/4) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 72.0 (TID 153) in 4 ms on localhost (2/4) -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 72.0 (TID 151) in 6 ms on localhost (3/4) -2016-05-23 21:54:16 INFO Executor:59 - Finished task 2.0 in stage 72.0 (TID 152). 915 bytes result sent to driver -2016-05-23 21:54:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 72.0 (TID 152) in 6 ms on localhost (4/4) -2016-05-23 21:54:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 72.0, whose tasks have all completed, from pool -2016-05-23 21:54:16 INFO DAGScheduler:59 - ResultStage 72 (foreach at StatisticsCorrLearning.scala:87) finished in 0.009 s -2016-05-23 21:54:16 INFO DAGScheduler:59 - Job 42 finished: foreach at StatisticsCorrLearning.scala:87, took 0.016930 s -2016-05-23 21:54:17 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:89 -2016-05-23 21:54:17 INFO DAGScheduler:59 - Got job 43 (foreach at StatisticsCorrLearning.scala:89) with 4 output partitions -2016-05-23 21:54:17 INFO DAGScheduler:59 - Final stage: ResultStage 73(foreach at StatisticsCorrLearning.scala:89) -2016-05-23 21:54:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting ResultStage 73 (ParallelCollectionRDD[81] at parallelize at StatisticsCorrLearning.scala:85), which has no missing parents -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(1232) called with curMem=422628, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 1232.0 B, free 730.2 MB) -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(894) called with curMem=423860, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 894.0 B, free 730.2 MB) -2016-05-23 21:54:17 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:53778 (size: 894.0 B, free: 730.5 MB) -2016-05-23 21:54:17 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 73 (ParallelCollectionRDD[81] at parallelize at StatisticsCorrLearning.scala:85) -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Adding task set 73.0 with 4 tasks -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 73.0 (TID 154, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 73.0 (TID 155, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 73.0 (TID 156, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 73.0 (TID 157, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:54:17 INFO Executor:59 - Running task 2.0 in stage 73.0 (TID 156) -2016-05-23 21:54:17 INFO Executor:59 - Finished task 2.0 in stage 73.0 (TID 156). 915 bytes result sent to driver -2016-05-23 21:54:17 INFO Executor:59 - Running task 3.0 in stage 73.0 (TID 157) -2016-05-23 21:54:17 INFO Executor:59 - Running task 1.0 in stage 73.0 (TID 155) -2016-05-23 21:54:17 INFO Executor:59 - Finished task 3.0 in stage 73.0 (TID 157). 915 bytes result sent to driver -2016-05-23 21:54:17 INFO Executor:59 - Finished task 1.0 in stage 73.0 (TID 155). 915 bytes result sent to driver -2016-05-23 21:54:17 INFO Executor:59 - Running task 0.0 in stage 73.0 (TID 154) -2016-05-23 21:54:17 INFO Executor:59 - Finished task 0.0 in stage 73.0 (TID 154). 915 bytes result sent to driver -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 73.0 (TID 156) in 7 ms on localhost (1/4) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 73.0 (TID 157) in 8 ms on localhost (2/4) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 73.0 (TID 155) in 9 ms on localhost (3/4) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 73.0 (TID 154) in 10 ms on localhost (4/4) -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 73.0, whose tasks have all completed, from pool -2016-05-23 21:54:17 INFO DAGScheduler:59 - ResultStage 73 (foreach at StatisticsCorrLearning.scala:89) finished in 0.017 s -2016-05-23 21:54:17 INFO DAGScheduler:59 - Job 43 finished: foreach at StatisticsCorrLearning.scala:89, took 0.026322 s -2016-05-23 21:54:17 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:54:17 INFO DAGScheduler:59 - Got job 44 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:54:17 INFO DAGScheduler:59 - Final stage: ResultStage 74(first at RowMatrix.scala:65) -2016-05-23 21:54:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting ResultStage 74 (MapPartitionsRDD[83] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=424754, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=427258, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 21:54:17 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:53778 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:54:17 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 74 (MapPartitionsRDD[83] at map at Correlation.scala:45) -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Adding task set 74.0 with 1 tasks -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 74.0 (TID 158, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:17 INFO Executor:59 - Running task 0.0 in stage 74.0 (TID 158) -2016-05-23 21:54:17 INFO Executor:59 - Finished task 0.0 in stage 74.0 (TID 158). 1046 bytes result sent to driver -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 74.0 (TID 158) in 2 ms on localhost (1/1) -2016-05-23 21:54:17 INFO DAGScheduler:59 - ResultStage 74 (first at RowMatrix.scala:65) finished in 0.003 s -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 74.0, whose tasks have all completed, from pool -2016-05-23 21:54:17 INFO DAGScheduler:59 - Job 44 finished: first at RowMatrix.scala:65, took 0.011608 s -2016-05-23 21:54:17 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:54:17 INFO DAGScheduler:59 - Got job 45 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:54:17 INFO DAGScheduler:59 - Final stage: ResultStage 75(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting ResultStage 75 (MapPartitionsRDD[84] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=428782, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=432182, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.2 MB) -2016-05-23 21:54:17 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:53778 (size: 1935.0 B, free: 730.5 MB) -2016-05-23 21:54:17 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 75 (MapPartitionsRDD[84] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Adding task set 75.0 with 4 tasks -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 75.0 (TID 159, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 75.0 (TID 160, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 75.0 (TID 161, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 75.0 (TID 162, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:54:17 INFO Executor:59 - Running task 2.0 in stage 75.0 (TID 161) -2016-05-23 21:54:17 INFO Executor:59 - Running task 1.0 in stage 75.0 (TID 160) -2016-05-23 21:54:17 INFO Executor:59 - Running task 3.0 in stage 75.0 (TID 162) -2016-05-23 21:54:17 INFO Executor:59 - Finished task 2.0 in stage 75.0 (TID 161). 1231 bytes result sent to driver -2016-05-23 21:54:17 INFO Executor:59 - Running task 0.0 in stage 75.0 (TID 159) -2016-05-23 21:54:17 INFO Executor:59 - Finished task 1.0 in stage 75.0 (TID 160). 1231 bytes result sent to driver -2016-05-23 21:54:17 INFO Executor:59 - Finished task 3.0 in stage 75.0 (TID 162). 1231 bytes result sent to driver -2016-05-23 21:54:17 INFO Executor:59 - Finished task 0.0 in stage 75.0 (TID 159). 1231 bytes result sent to driver -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 75.0 (TID 161) in 5 ms on localhost (1/4) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 75.0 (TID 160) in 6 ms on localhost (2/4) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 75.0 (TID 162) in 5 ms on localhost (3/4) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 75.0 (TID 159) in 7 ms on localhost (4/4) -2016-05-23 21:54:17 INFO DAGScheduler:59 - ResultStage 75 (treeAggregate at RowMatrix.scala:332) finished in 0.008 s -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 75.0, whose tasks have all completed, from pool -2016-05-23 21:54:17 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at RowMatrix.scala:332, took 0.016854 s -2016-05-23 21:54:17 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:54:17 INFO DAGScheduler:59 - Got job 46 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:54:17 INFO DAGScheduler:59 - Final stage: ResultStage 76(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting ResultStage 76 (MapPartitionsRDD[85] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=434117, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(1919) called with curMem=437557, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 1919.0 B, free 730.2 MB) -2016-05-23 21:54:17 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:53778 (size: 1919.0 B, free: 730.5 MB) -2016-05-23 21:54:17 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 76 (MapPartitionsRDD[85] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 4 tasks -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 163, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 164, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 76.0 (TID 165, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 76.0 (TID 166, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:54:17 INFO Executor:59 - Running task 3.0 in stage 76.0 (TID 166) -2016-05-23 21:54:17 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 164) -2016-05-23 21:54:17 INFO Executor:59 - Running task 2.0 in stage 76.0 (TID 165) -2016-05-23 21:54:17 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 163) -2016-05-23 21:54:17 INFO Executor:59 - Finished task 3.0 in stage 76.0 (TID 166). 1109 bytes result sent to driver -2016-05-23 21:54:17 INFO Executor:59 - Finished task 2.0 in stage 76.0 (TID 165). 1109 bytes result sent to driver -2016-05-23 21:54:17 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 164). 1109 bytes result sent to driver -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 76.0 (TID 166) in 4 ms on localhost (1/4) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 76.0 (TID 165) in 5 ms on localhost (2/4) -2016-05-23 21:54:17 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 163). 1109 bytes result sent to driver -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 164) in 6 ms on localhost (3/4) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 163) in 8 ms on localhost (4/4) -2016-05-23 21:54:17 INFO DAGScheduler:59 - ResultStage 76 (treeAggregate at RowMatrix.scala:124) finished in 0.008 s -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-05-23 21:54:17 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at RowMatrix.scala:124, took 0.016037 s -2016-05-23 21:54:17 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:54:17 INFO DAGScheduler:59 - Got job 47 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:54:17 INFO DAGScheduler:59 - Final stage: ResultStage 77(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:54:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[91] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=439476, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(1862) called with curMem=442852, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 1862.0 B, free 730.2 MB) -2016-05-23 21:54:17 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:53778 (size: 1862.0 B, free: 730.5 MB) -2016-05-23 21:54:17 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 77 (MapPartitionsRDD[91] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 4 tasks -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 167, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 168, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 77.0 (TID 169, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 77.0 (TID 170, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:54:17 INFO Executor:59 - Running task 3.0 in stage 77.0 (TID 170) -2016-05-23 21:54:17 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 168) -2016-05-23 21:54:17 INFO Executor:59 - Running task 2.0 in stage 77.0 (TID 169) -2016-05-23 21:54:17 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 167) -2016-05-23 21:54:17 INFO Executor:59 - Finished task 3.0 in stage 77.0 (TID 170). 1294 bytes result sent to driver -2016-05-23 21:54:17 INFO Executor:59 - Finished task 2.0 in stage 77.0 (TID 169). 1249 bytes result sent to driver -2016-05-23 21:54:17 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 167). 1254 bytes result sent to driver -2016-05-23 21:54:17 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 168). 1254 bytes result sent to driver -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 77.0 (TID 170) in 4 ms on localhost (1/4) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 77.0 (TID 169) in 4 ms on localhost (2/4) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 167) in 5 ms on localhost (3/4) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 168) in 6 ms on localhost (4/4) -2016-05-23 21:54:17 INFO DAGScheduler:59 - ResultStage 77 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.006 s -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-05-23 21:54:17 INFO DAGScheduler:59 - Job 47 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.014485 s -2016-05-23 21:54:17 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:54:17 INFO DAGScheduler:59 - Registering RDD 89 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:54:17 INFO DAGScheduler:59 - Got job 48 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:54:17 INFO DAGScheduler:59 - Final stage: ResultStage 79(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:54:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 78) -2016-05-23 21:54:17 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 78) -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting ShuffleMapStage 78 (MapPartitionsRDD[89] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=444714, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(2289) called with curMem=448754, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:54:17 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:53778 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:54:17 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 78 (MapPartitionsRDD[89] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 4 tasks -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 171, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 172, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 78.0 (TID 173, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 78.0 (TID 174, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:54:17 INFO Executor:59 - Running task 3.0 in stage 78.0 (TID 174) -2016-05-23 21:54:17 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 171) -2016-05-23 21:54:17 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 172) -2016-05-23 21:54:17 INFO Executor:59 - Running task 2.0 in stage 78.0 (TID 173) -2016-05-23 21:54:17 INFO Executor:59 - Finished task 2.0 in stage 78.0 (TID 173). 1161 bytes result sent to driver -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 78.0 (TID 173) in 40 ms on localhost (1/4) -2016-05-23 21:54:17 INFO Executor:59 - Finished task 3.0 in stage 78.0 (TID 174). 1161 bytes result sent to driver -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 78.0 (TID 174) in 49 ms on localhost (2/4) -2016-05-23 21:54:17 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 172). 1161 bytes result sent to driver -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 172) in 228 ms on localhost (3/4) -2016-05-23 21:54:17 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 171). 1161 bytes result sent to driver -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 171) in 243 ms on localhost (4/4) -2016-05-23 21:54:17 INFO DAGScheduler:59 - ShuffleMapStage 78 (flatMap at SpearmanCorrelation.scala:49) finished in 0.243 s -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-05-23 21:54:17 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:54:17 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:54:17 INFO DAGScheduler:59 - waiting: Set(ResultStage 79) -2016-05-23 21:54:17 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:54:17 INFO DAGScheduler:59 - Missing parents for ResultStage 79: List() -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting ResultStage 79 (ShuffledRDD[92] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=451043, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(1573) called with curMem=453587, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 1573.0 B, free 730.2 MB) -2016-05-23 21:54:17 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:53778 (size: 1573.0 B, free: 730.5 MB) -2016-05-23 21:54:17 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 79 (ShuffledRDD[92] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Adding task set 79.0 with 3 tasks -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 79.0 (TID 175, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 79.0 (TID 176, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 79.0 (TID 177, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:17 INFO Executor:59 - Running task 2.0 in stage 79.0 (TID 177) -2016-05-23 21:54:17 INFO Executor:59 - Running task 1.0 in stage 79.0 (TID 176) -2016-05-23 21:54:17 INFO Executor:59 - Running task 0.0 in stage 79.0 (TID 175) -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:17 INFO Executor:59 - Finished task 2.0 in stage 79.0 (TID 177). 1203 bytes result sent to driver -2016-05-23 21:54:17 INFO Executor:59 - Finished task 0.0 in stage 79.0 (TID 175). 1203 bytes result sent to driver -2016-05-23 21:54:17 INFO Executor:59 - Finished task 1.0 in stage 79.0 (TID 176). 1203 bytes result sent to driver -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 79.0 (TID 177) in 13 ms on localhost (1/3) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 79.0 (TID 175) in 14 ms on localhost (2/3) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 79.0 (TID 176) in 14 ms on localhost (3/3) -2016-05-23 21:54:17 INFO DAGScheduler:59 - ResultStage 79 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.015 s -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 79.0, whose tasks have all completed, from pool -2016-05-23 21:54:17 INFO DAGScheduler:59 - Job 48 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.279737 s -2016-05-23 21:54:17 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:54:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 166 bytes -2016-05-23 21:54:17 INFO DAGScheduler:59 - Registering RDD 94 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:54:17 INFO DAGScheduler:59 - Got job 49 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:54:17 INFO DAGScheduler:59 - Final stage: ResultStage 82(first at RowMatrix.scala:65) -2016-05-23 21:54:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 81) -2016-05-23 21:54:17 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 81) -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting ShuffleMapStage 81 (MapPartitionsRDD[94] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=455160, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(2313) called with curMem=459392, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-23 21:54:17 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:53778 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:54:17 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 81 (MapPartitionsRDD[94] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Adding task set 81.0 with 4 tasks -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 81.0 (TID 178, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 81.0 (TID 179, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 81.0 (TID 180, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 81.0 (TID 181, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:54:17 INFO Executor:59 - Running task 1.0 in stage 81.0 (TID 179) -2016-05-23 21:54:17 INFO Executor:59 - Running task 0.0 in stage 81.0 (TID 178) -2016-05-23 21:54:17 INFO Executor:59 - Running task 2.0 in stage 81.0 (TID 180) -2016-05-23 21:54:17 INFO Executor:59 - Running task 3.0 in stage 81.0 (TID 181) -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:17 INFO Executor:59 - Finished task 0.0 in stage 81.0 (TID 178). 1377 bytes result sent to driver -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 81.0 (TID 178) in 35 ms on localhost (1/4) -2016-05-23 21:54:17 INFO Executor:59 - Finished task 1.0 in stage 81.0 (TID 179). 1377 bytes result sent to driver -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 81.0 (TID 179) in 70 ms on localhost (2/4) -2016-05-23 21:54:17 INFO Executor:59 - Finished task 2.0 in stage 81.0 (TID 180). 1377 bytes result sent to driver -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 81.0 (TID 180) in 81 ms on localhost (3/4) -2016-05-23 21:54:17 INFO Executor:59 - Finished task 3.0 in stage 81.0 (TID 181). 1377 bytes result sent to driver -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 81.0 (TID 181) in 83 ms on localhost (4/4) -2016-05-23 21:54:17 INFO DAGScheduler:59 - ShuffleMapStage 81 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.085 s -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 81.0, whose tasks have all completed, from pool -2016-05-23 21:54:17 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:54:17 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:54:17 INFO DAGScheduler:59 - waiting: Set(ResultStage 82) -2016-05-23 21:54:17 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:54:17 INFO DAGScheduler:59 - Missing parents for ResultStage 82: List() -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting ResultStage 82 (MapPartitionsRDD[96] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=461705, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(2562) called with curMem=466601, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:54:17 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:53778 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:54:17 INFO SparkContext:59 - Created broadcast 61 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 82 (MapPartitionsRDD[96] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Adding task set 82.0 with 1 tasks -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 82.0 (TID 182, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:17 INFO Executor:59 - Running task 0.0 in stage 82.0 (TID 182) -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:17 INFO Executor:59 - Finished task 0.0 in stage 82.0 (TID 182). 1296 bytes result sent to driver -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 82.0 (TID 182) in 8 ms on localhost (1/1) -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 82.0, whose tasks have all completed, from pool -2016-05-23 21:54:17 INFO DAGScheduler:59 - ResultStage 82 (first at RowMatrix.scala:65) finished in 0.008 s -2016-05-23 21:54:17 INFO DAGScheduler:59 - Job 49 finished: first at RowMatrix.scala:65, took 0.111386 s -2016-05-23 21:54:17 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:54:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 166 bytes -2016-05-23 21:54:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 169 bytes -2016-05-23 21:54:17 INFO DAGScheduler:59 - Got job 50 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:54:17 INFO DAGScheduler:59 - Final stage: ResultStage 85(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 84) -2016-05-23 21:54:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting ResultStage 85 (MapPartitionsRDD[97] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=469163, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(2943) called with curMem=474859, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.1 MB) -2016-05-23 21:54:17 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:53778 (size: 2.9 KB, free: 730.5 MB) -2016-05-23 21:54:17 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 85 (MapPartitionsRDD[97] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Adding task set 85.0 with 4 tasks -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 85.0 (TID 183, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 85.0 (TID 184, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 85.0 (TID 185, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 85.0 (TID 186, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:17 INFO Executor:59 - Running task 1.0 in stage 85.0 (TID 184) -2016-05-23 21:54:17 INFO Executor:59 - Running task 3.0 in stage 85.0 (TID 186) -2016-05-23 21:54:17 INFO Executor:59 - Running task 2.0 in stage 85.0 (TID 185) -2016-05-23 21:54:17 INFO Executor:59 - Running task 0.0 in stage 85.0 (TID 183) -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-23 21:54:17 INFO Executor:59 - Finished task 1.0 in stage 85.0 (TID 184). 1481 bytes result sent to driver -2016-05-23 21:54:17 INFO Executor:59 - Finished task 3.0 in stage 85.0 (TID 186). 1481 bytes result sent to driver -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 85.0 (TID 186) in 10 ms on localhost (1/4) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 85.0 (TID 184) in 11 ms on localhost (2/4) -2016-05-23 21:54:17 INFO Executor:59 - Finished task 0.0 in stage 85.0 (TID 183). 1481 bytes result sent to driver -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 85.0 (TID 183) in 13 ms on localhost (3/4) -2016-05-23 21:54:17 INFO Executor:59 - Finished task 2.0 in stage 85.0 (TID 185). 1481 bytes result sent to driver -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 85.0 (TID 185) in 15 ms on localhost (4/4) -2016-05-23 21:54:17 INFO DAGScheduler:59 - ResultStage 85 (treeAggregate at RowMatrix.scala:332) finished in 0.016 s -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 85.0, whose tasks have all completed, from pool -2016-05-23 21:54:17 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at RowMatrix.scala:332, took 0.027420 s -2016-05-23 21:54:17 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:54:17 INFO DAGScheduler:59 - Got job 51 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:54:17 INFO DAGScheduler:59 - Final stage: ResultStage 88(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 87) -2016-05-23 21:54:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting ResultStage 88 (MapPartitionsRDD[98] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=477802, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(2920) called with curMem=483538, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.1 MB) -2016-05-23 21:54:17 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:53778 (size: 2.9 KB, free: 730.5 MB) -2016-05-23 21:54:17 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 88 (MapPartitionsRDD[98] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Adding task set 88.0 with 4 tasks -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 88.0 (TID 187, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 88.0 (TID 188, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 88.0 (TID 189, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 88.0 (TID 190, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:17 INFO Executor:59 - Running task 0.0 in stage 88.0 (TID 187) -2016-05-23 21:54:17 INFO Executor:59 - Running task 1.0 in stage 88.0 (TID 188) -2016-05-23 21:54:17 INFO Executor:59 - Running task 3.0 in stage 88.0 (TID 190) -2016-05-23 21:54:17 INFO Executor:59 - Running task 2.0 in stage 88.0 (TID 189) -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:54:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:17 INFO Executor:59 - Finished task 2.0 in stage 88.0 (TID 189). 1359 bytes result sent to driver -2016-05-23 21:54:17 INFO Executor:59 - Finished task 3.0 in stage 88.0 (TID 190). 1359 bytes result sent to driver -2016-05-23 21:54:17 INFO Executor:59 - Finished task 1.0 in stage 88.0 (TID 188). 1359 bytes result sent to driver -2016-05-23 21:54:17 INFO Executor:59 - Finished task 0.0 in stage 88.0 (TID 187). 1359 bytes result sent to driver -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 88.0 (TID 189) in 12 ms on localhost (1/4) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 88.0 (TID 188) in 12 ms on localhost (2/4) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 88.0 (TID 190) in 13 ms on localhost (3/4) -2016-05-23 21:54:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 88.0 (TID 187) in 14 ms on localhost (4/4) -2016-05-23 21:54:17 INFO DAGScheduler:59 - ResultStage 88 (treeAggregate at RowMatrix.scala:124) finished in 0.014 s -2016-05-23 21:54:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 88.0, whose tasks have all completed, from pool -2016-05-23 21:54:17 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at RowMatrix.scala:124, took 0.024161 s -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(88528) called with curMem=486458, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 86.5 KB, free 730.0 MB) -2016-05-23 21:54:17 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=574986, maxMem=766075207 -2016-05-23 21:54:17 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.0 MB) -2016-05-23 21:54:17 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:53778 (size: 19.3 KB, free: 730.5 MB) -2016-05-23 21:54:17 INFO SparkContext:59 - Created broadcast 64 from textFile at StatisticsCorrLearning.scala:98 -2016-05-23 21:54:18 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 21:54:18 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:54:18 INFO DAGScheduler:59 - Got job 52 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:54:18 INFO DAGScheduler:59 - Final stage: ResultStage 89(first at RowMatrix.scala:65) -2016-05-23 21:54:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:18 INFO DAGScheduler:59 - Submitting ResultStage 89 (MapPartitionsRDD[102] at map at StatisticsCorrLearning.scala:101), which has no missing parents -2016-05-23 21:54:18 INFO MemoryStore:59 - ensureFreeSpace(3552) called with curMem=594774, maxMem=766075207 -2016-05-23 21:54:18 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-23 21:54:18 INFO MemoryStore:59 - ensureFreeSpace(2003) called with curMem=598326, maxMem=766075207 -2016-05-23 21:54:18 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 2003.0 B, free 730.0 MB) -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:53778 (size: 2003.0 B, free: 730.5 MB) -2016-05-23 21:54:18 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:53778 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-23 21:54:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 89 (MapPartitionsRDD[102] at map at StatisticsCorrLearning.scala:101) -2016-05-23 21:54:18 INFO TaskSchedulerImpl:59 - Adding task set 89.0 with 1 tasks -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-05-23 21:54:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 89.0 (TID 191, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:54:18 INFO Executor:59 - Running task 0.0 in stage 89.0 (TID 191) -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:53778 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-23 21:54:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:53778 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:54:18 INFO Executor:59 - Finished task 0.0 in stage 89.0 (TID 191). 2199 bytes result sent to driver -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-05-23 21:54:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 89.0 (TID 191) in 12 ms on localhost (1/1) -2016-05-23 21:54:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 89.0, whose tasks have all completed, from pool -2016-05-23 21:54:18 INFO DAGScheduler:59 - ResultStage 89 (first at RowMatrix.scala:65) finished in 0.013 s -2016-05-23 21:54:18 INFO DAGScheduler:59 - Job 52 finished: first at RowMatrix.scala:65, took 0.040354 s -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:53778 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned shuffle 9 -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:53778 in memory (size: 1573.0 B, free: 730.5 MB) -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:53778 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned shuffle 8 -2016-05-23 21:54:18 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:54:18 INFO DAGScheduler:59 - Got job 53 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:54:18 INFO DAGScheduler:59 - Final stage: ResultStage 90(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:53778 in memory (size: 1862.0 B, free: 730.5 MB) -2016-05-23 21:54:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:18 INFO DAGScheduler:59 - Submitting ResultStage 90 (MapPartitionsRDD[103] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-05-23 21:54:18 INFO MemoryStore:59 - ensureFreeSpace(4472) called with curMem=553347, maxMem=766075207 -2016-05-23 21:54:18 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 4.4 KB, free 730.1 MB) -2016-05-23 21:54:18 INFO MemoryStore:59 - ensureFreeSpace(2372) called with curMem=552460, maxMem=766075207 -2016-05-23 21:54:18 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:53778 in memory (size: 1919.0 B, free: 730.5 MB) -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:53778 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-05-23 21:54:18 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 90 (MapPartitionsRDD[103] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:18 INFO TaskSchedulerImpl:59 - Adding task set 90.0 with 2 tasks -2016-05-23 21:54:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 90.0 (TID 192, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:54:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 90.0 (TID 193, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:53778 in memory (size: 1935.0 B, free: 730.5 MB) -2016-05-23 21:54:18 INFO Executor:59 - Running task 0.0 in stage 90.0 (TID 192) -2016-05-23 21:54:18 INFO Executor:59 - Running task 1.0 in stage 90.0 (TID 193) -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-05-23 21:54:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:53778 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:54:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-05-23 21:54:18 INFO Executor:59 - Finished task 1.0 in stage 90.0 (TID 193). 2384 bytes result sent to driver -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:53778 in memory (size: 894.0 B, free: 730.5 MB) -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-05-23 21:54:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 90.0 (TID 193) in 13 ms on localhost (1/2) -2016-05-23 21:54:18 INFO Executor:59 - Finished task 0.0 in stage 90.0 (TID 192). 2384 bytes result sent to driver -2016-05-23 21:54:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 90.0 (TID 192) in 16 ms on localhost (2/2) -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:53778 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:54:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 90.0, whose tasks have all completed, from pool -2016-05-23 21:54:18 INFO DAGScheduler:59 - ResultStage 90 (treeAggregate at RowMatrix.scala:332) finished in 0.017 s -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-05-23 21:54:18 INFO DAGScheduler:59 - Job 53 finished: treeAggregate at RowMatrix.scala:332, took 0.031242 s -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:53778 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:53778 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-05-23 21:54:18 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:54:18 INFO DAGScheduler:59 - Got job 54 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:53778 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:54:18 INFO DAGScheduler:59 - Final stage: ResultStage 91(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-05-23 21:54:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:18 INFO DAGScheduler:59 - Submitting ResultStage 91 (MapPartitionsRDD[104] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:54:18 INFO MemoryStore:59 - ensureFreeSpace(4584) called with curMem=514000, maxMem=766075207 -2016-05-23 21:54:18 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 4.5 KB, free 730.1 MB) -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:53778 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:54:18 INFO MemoryStore:59 - ensureFreeSpace(2364) called with curMem=513688, maxMem=766075207 -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-05-23 21:54:18 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:53778 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:54:18 INFO SparkContext:59 - Created broadcast 67 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 91 (MapPartitionsRDD[104] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:18 INFO TaskSchedulerImpl:59 - Adding task set 91.0 with 2 tasks -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:53778 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-05-23 21:54:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 91.0 (TID 194, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:54:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 91.0 (TID 195, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned shuffle 7 -2016-05-23 21:54:18 INFO Executor:59 - Running task 1.0 in stage 91.0 (TID 195) -2016-05-23 21:54:18 INFO Executor:59 - Running task 0.0 in stage 91.0 (TID 194) -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:53778 in memory (size: 1567.0 B, free: 730.5 MB) -2016-05-23 21:54:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:54:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-05-23 21:54:18 INFO Executor:59 - Finished task 0.0 in stage 91.0 (TID 194). 2337 bytes result sent to driver -2016-05-23 21:54:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 91.0 (TID 194) in 12 ms on localhost (1/2) -2016-05-23 21:54:18 INFO Executor:59 - Finished task 1.0 in stage 91.0 (TID 195). 2337 bytes result sent to driver -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:53778 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:54:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 91.0 (TID 195) in 18 ms on localhost (2/2) -2016-05-23 21:54:18 INFO DAGScheduler:59 - ResultStage 91 (treeAggregate at RowMatrix.scala:124) finished in 0.019 s -2016-05-23 21:54:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 91.0, whose tasks have all completed, from pool -2016-05-23 21:54:18 INFO DAGScheduler:59 - Job 54 finished: treeAggregate at RowMatrix.scala:124, took 0.030318 s -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned shuffle 6 -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:53778 in memory (size: 1858.0 B, free: 730.5 MB) -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:53778 in memory (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:53778 in memory (size: 1934.0 B, free: 730.5 MB) -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:53778 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:53778 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:53778 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:53778 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:54:18 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-23 21:54:18 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:54:18 INFO DAGScheduler:59 - Got job 55 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 21:54:18 INFO DAGScheduler:59 - Final stage: ResultStage 92(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:54:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:54:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:18 INFO DAGScheduler:59 - Submitting ResultStage 92 (MapPartitionsRDD[108] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:54:18 INFO MemoryStore:59 - ensureFreeSpace(4432) called with curMem=470863, maxMem=766075207 -2016-05-23 21:54:18 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 4.3 KB, free 730.1 MB) -2016-05-23 21:54:18 INFO MemoryStore:59 - ensureFreeSpace(2326) called with curMem=475295, maxMem=766075207 -2016-05-23 21:54:18 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:53778 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:54:18 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 92 (MapPartitionsRDD[108] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:54:18 INFO TaskSchedulerImpl:59 - Adding task set 92.0 with 2 tasks -2016-05-23 21:54:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 92.0 (TID 196, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:54:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 92.0 (TID 197, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:54:18 INFO Executor:59 - Running task 0.0 in stage 92.0 (TID 196) -2016-05-23 21:54:18 INFO Executor:59 - Running task 1.0 in stage 92.0 (TID 197) -2016-05-23 21:54:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:54:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:54:18 INFO Executor:59 - Finished task 0.0 in stage 92.0 (TID 196). 2443 bytes result sent to driver -2016-05-23 21:54:18 INFO Executor:59 - Finished task 1.0 in stage 92.0 (TID 197). 2443 bytes result sent to driver -2016-05-23 21:54:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 92.0 (TID 196) in 12 ms on localhost (1/2) -2016-05-23 21:54:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 92.0 (TID 197) in 13 ms on localhost (2/2) -2016-05-23 21:54:18 INFO DAGScheduler:59 - ResultStage 92 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.014 s -2016-05-23 21:54:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 92.0, whose tasks have all completed, from pool -2016-05-23 21:54:18 INFO DAGScheduler:59 - Job 55 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.023866 s -2016-05-23 21:54:18 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:54:18 INFO DAGScheduler:59 - Registering RDD 106 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:54:18 INFO DAGScheduler:59 - Got job 56 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 21:54:18 INFO DAGScheduler:59 - Final stage: ResultStage 94(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:54:18 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 93) -2016-05-23 21:54:18 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 93) -2016-05-23 21:54:18 INFO DAGScheduler:59 - Submitting ShuffleMapStage 93 (MapPartitionsRDD[106] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:54:18 INFO MemoryStore:59 - ensureFreeSpace(5024) called with curMem=477621, maxMem=766075207 -2016-05-23 21:54:18 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 4.9 KB, free 730.1 MB) -2016-05-23 21:54:18 INFO MemoryStore:59 - ensureFreeSpace(2736) called with curMem=482645, maxMem=766075207 -2016-05-23 21:54:18 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.1 MB) -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:53778 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 21:54:18 INFO SparkContext:59 - Created broadcast 69 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 93 (MapPartitionsRDD[106] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:54:18 INFO TaskSchedulerImpl:59 - Adding task set 93.0 with 2 tasks -2016-05-23 21:54:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 93.0 (TID 198, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-23 21:54:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 93.0 (TID 199, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-23 21:54:18 INFO Executor:59 - Running task 1.0 in stage 93.0 (TID 199) -2016-05-23 21:54:18 INFO Executor:59 - Running task 0.0 in stage 93.0 (TID 198) -2016-05-23 21:54:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:54:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:54:18 INFO Executor:59 - Finished task 1.0 in stage 93.0 (TID 199). 2254 bytes result sent to driver -2016-05-23 21:54:18 INFO Executor:59 - Finished task 0.0 in stage 93.0 (TID 198). 2254 bytes result sent to driver -2016-05-23 21:54:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 93.0 (TID 199) in 22 ms on localhost (1/2) -2016-05-23 21:54:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 93.0 (TID 198) in 24 ms on localhost (2/2) -2016-05-23 21:54:18 INFO DAGScheduler:59 - ShuffleMapStage 93 (flatMap at SpearmanCorrelation.scala:49) finished in 0.024 s -2016-05-23 21:54:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 93.0, whose tasks have all completed, from pool -2016-05-23 21:54:18 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:54:18 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:54:18 INFO DAGScheduler:59 - waiting: Set(ResultStage 94) -2016-05-23 21:54:18 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:54:18 INFO DAGScheduler:59 - Missing parents for ResultStage 94: List() -2016-05-23 21:54:18 INFO DAGScheduler:59 - Submitting ResultStage 94 (ShuffledRDD[109] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:54:18 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=485381, maxMem=766075207 -2016-05-23 21:54:18 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-05-23 21:54:18 INFO MemoryStore:59 - ensureFreeSpace(1551) called with curMem=487885, maxMem=766075207 -2016-05-23 21:54:18 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 1551.0 B, free 730.1 MB) -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:53778 (size: 1551.0 B, free: 730.5 MB) -2016-05-23 21:54:18 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 94 (ShuffledRDD[109] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:54:18 INFO TaskSchedulerImpl:59 - Adding task set 94.0 with 1 tasks -2016-05-23 21:54:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 94.0 (TID 200, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:18 INFO Executor:59 - Running task 0.0 in stage 94.0 (TID 200) -2016-05-23 21:54:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:54:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:18 INFO Executor:59 - Finished task 0.0 in stage 94.0 (TID 200). 1203 bytes result sent to driver -2016-05-23 21:54:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 94.0 (TID 200) in 8 ms on localhost (1/1) -2016-05-23 21:54:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 94.0, whose tasks have all completed, from pool -2016-05-23 21:54:18 INFO DAGScheduler:59 - ResultStage 94 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.011 s -2016-05-23 21:54:18 INFO DAGScheduler:59 - Job 56 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.054027 s -2016-05-23 21:54:18 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:54:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 152 bytes -2016-05-23 21:54:18 INFO DAGScheduler:59 - Registering RDD 111 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:54:18 INFO DAGScheduler:59 - Got job 57 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:54:18 INFO DAGScheduler:59 - Final stage: ResultStage 97(first at RowMatrix.scala:65) -2016-05-23 21:54:18 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 96) -2016-05-23 21:54:18 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 96) -2016-05-23 21:54:18 INFO DAGScheduler:59 - Submitting ShuffleMapStage 96 (MapPartitionsRDD[111] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:54:18 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=489436, maxMem=766075207 -2016-05-23 21:54:18 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 21:54:18 INFO MemoryStore:59 - ensureFreeSpace(2287) called with curMem=493628, maxMem=766075207 -2016-05-23 21:54:18 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:53778 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:54:18 INFO SparkContext:59 - Created broadcast 71 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 96 (MapPartitionsRDD[111] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:54:18 INFO TaskSchedulerImpl:59 - Adding task set 96.0 with 2 tasks -2016-05-23 21:54:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 96.0 (TID 201, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:54:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 96.0 (TID 202, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:54:18 INFO Executor:59 - Running task 0.0 in stage 96.0 (TID 201) -2016-05-23 21:54:18 INFO Executor:59 - Running task 1.0 in stage 96.0 (TID 202) -2016-05-23 21:54:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:54:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:54:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:18 INFO Executor:59 - Finished task 0.0 in stage 96.0 (TID 201). 1375 bytes result sent to driver -2016-05-23 21:54:18 INFO Executor:59 - Finished task 1.0 in stage 96.0 (TID 202). 1375 bytes result sent to driver -2016-05-23 21:54:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 96.0 (TID 201) in 24 ms on localhost (1/2) -2016-05-23 21:54:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 96.0 (TID 202) in 24 ms on localhost (2/2) -2016-05-23 21:54:18 INFO DAGScheduler:59 - ShuffleMapStage 96 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.024 s -2016-05-23 21:54:18 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:54:18 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:54:18 INFO DAGScheduler:59 - waiting: Set(ResultStage 97) -2016-05-23 21:54:18 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:54:18 INFO DAGScheduler:59 - Missing parents for ResultStage 97: List() -2016-05-23 21:54:18 INFO DAGScheduler:59 - Submitting ResultStage 97 (MapPartitionsRDD[113] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:54:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 96.0, whose tasks have all completed, from pool -2016-05-23 21:54:18 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=495915, maxMem=766075207 -2016-05-23 21:54:18 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 4.7 KB, free 730.1 MB) -2016-05-23 21:54:18 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=500771, maxMem=766075207 -2016-05-23 21:54:18 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:53778 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:54:18 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 97 (MapPartitionsRDD[113] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:54:18 INFO TaskSchedulerImpl:59 - Adding task set 97.0 with 1 tasks -2016-05-23 21:54:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 97.0 (TID 203, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:18 INFO Executor:59 - Running task 0.0 in stage 97.0 (TID 203) -2016-05-23 21:54:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:54:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:18 INFO Executor:59 - Finished task 0.0 in stage 97.0 (TID 203). 1320 bytes result sent to driver -2016-05-23 21:54:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 97.0 (TID 203) in 6 ms on localhost (1/1) -2016-05-23 21:54:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 97.0, whose tasks have all completed, from pool -2016-05-23 21:54:18 INFO DAGScheduler:59 - ResultStage 97 (first at RowMatrix.scala:65) finished in 0.008 s -2016-05-23 21:54:18 INFO DAGScheduler:59 - Job 57 finished: first at RowMatrix.scala:65, took 0.056036 s -2016-05-23 21:54:18 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:54:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 152 bytes -2016-05-23 21:54:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 152 bytes -2016-05-23 21:54:18 INFO DAGScheduler:59 - Got job 58 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:54:18 INFO DAGScheduler:59 - Final stage: ResultStage 100(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:18 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 99) -2016-05-23 21:54:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:18 INFO DAGScheduler:59 - Submitting ResultStage 100 (MapPartitionsRDD[114] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:54:18 INFO MemoryStore:59 - ensureFreeSpace(5680) called with curMem=503302, maxMem=766075207 -2016-05-23 21:54:18 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-23 21:54:18 INFO MemoryStore:59 - ensureFreeSpace(2901) called with curMem=508982, maxMem=766075207 -2016-05-23 21:54:18 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:53778 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:54:18 INFO SparkContext:59 - Created broadcast 73 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 100 (MapPartitionsRDD[114] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:54:18 INFO TaskSchedulerImpl:59 - Adding task set 100.0 with 2 tasks -2016-05-23 21:54:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 100.0 (TID 204, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 100.0 (TID 205, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:18 INFO Executor:59 - Running task 1.0 in stage 100.0 (TID 205) -2016-05-23 21:54:18 INFO Executor:59 - Running task 0.0 in stage 100.0 (TID 204) -2016-05-23 21:54:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:54:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:54:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:18 INFO Executor:59 - Finished task 1.0 in stage 100.0 (TID 205). 1505 bytes result sent to driver -2016-05-23 21:54:18 INFO Executor:59 - Finished task 0.0 in stage 100.0 (TID 204). 1505 bytes result sent to driver -2016-05-23 21:54:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 100.0 (TID 204) in 12 ms on localhost (1/2) -2016-05-23 21:54:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 100.0 (TID 205) in 13 ms on localhost (2/2) -2016-05-23 21:54:18 INFO DAGScheduler:59 - ResultStage 100 (treeAggregate at RowMatrix.scala:332) finished in 0.014 s -2016-05-23 21:54:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 100.0, whose tasks have all completed, from pool -2016-05-23 21:54:18 INFO DAGScheduler:59 - Job 58 finished: treeAggregate at RowMatrix.scala:332, took 0.029547 s -2016-05-23 21:54:18 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:54:18 INFO DAGScheduler:59 - Got job 59 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:54:18 INFO DAGScheduler:59 - Final stage: ResultStage 103(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:18 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 102) -2016-05-23 21:54:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:54:18 INFO DAGScheduler:59 - Submitting ResultStage 103 (MapPartitionsRDD[115] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:54:18 INFO MemoryStore:59 - ensureFreeSpace(5792) called with curMem=511883, maxMem=766075207 -2016-05-23 21:54:18 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-05-23 21:54:18 INFO MemoryStore:59 - ensureFreeSpace(2886) called with curMem=517675, maxMem=766075207 -2016-05-23 21:54:18 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 21:54:18 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:53778 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:54:18 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:54:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 103 (MapPartitionsRDD[115] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:54:18 INFO TaskSchedulerImpl:59 - Adding task set 103.0 with 2 tasks -2016-05-23 21:54:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 103.0 (TID 206, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 103.0 (TID 207, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:54:18 INFO Executor:59 - Running task 0.0 in stage 103.0 (TID 206) -2016-05-23 21:54:18 INFO Executor:59 - Running task 1.0 in stage 103.0 (TID 207) -2016-05-23 21:54:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:54:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:54:18 INFO Executor:59 - Finished task 0.0 in stage 103.0 (TID 206). 1458 bytes result sent to driver -2016-05-23 21:54:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:54:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:54:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 103.0 (TID 206) in 16 ms on localhost (1/2) -2016-05-23 21:54:18 INFO Executor:59 - Finished task 1.0 in stage 103.0 (TID 207). 1458 bytes result sent to driver -2016-05-23 21:54:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 103.0 (TID 207) in 18 ms on localhost (2/2) -2016-05-23 21:54:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 103.0, whose tasks have all completed, from pool -2016-05-23 21:54:18 INFO DAGScheduler:59 - ResultStage 103 (treeAggregate at RowMatrix.scala:124) finished in 0.020 s -2016-05-23 21:54:18 INFO DAGScheduler:59 - Job 59 finished: treeAggregate at RowMatrix.scala:124, took 0.030390 s -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 21:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 21:54:18 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 21:54:18 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 21:54:18 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 21:54:19 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 21:54:19 INFO BlockManager:59 - BlockManager stopped -2016-05-23 21:54:19 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 21:54:19 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 21:54:19 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 21:54:19 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 21:54:19 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-e5fbf2db-2cc5-4c3d-87ba-b9f9fb509b61 -2016-05-23 21:57:19 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 21:57:23 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 21:57:24 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 21:57:24 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 21:57:24 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 21:57:38 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 21:57:42 INFO Remoting:74 - Starting remoting -2016-05-23 21:57:50 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53846] -2016-05-23 21:57:50 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53846. -2016-05-23 21:57:51 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 21:57:52 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 21:57:53 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-ea2fc07a-8955-47a6-83ba-f32c2c137c35 -2016-05-23 21:57:55 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 21:57:56 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-7c9e68f3-b1a1-40e9-bf3c-209981a07ae1\httpd-b8bb5e08-3ad4-4727-9b8f-299d3269e556 -2016-05-23 21:57:57 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 21:57:59 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 21:58:01 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53850 -2016-05-23 21:58:01 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53850. -2016-05-23 21:58:03 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 21:58:14 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 21:58:14 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 21:58:14 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 21:58:14 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 21:58:19 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 21:58:20 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 21:58:30 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53869. -2016-05-23 21:58:30 INFO NettyBlockTransferService:59 - Server created on 53869 -2016-05-23 21:58:31 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 21:58:31 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53869 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53869) -2016-05-23 21:58:31 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 21:58:39 INFO Executor:59 - Told to re-register on heartbeat -2016-05-23 21:58:39 INFO BlockManager:59 - BlockManager re-registering with master -2016-05-23 21:58:39 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 21:58:39 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 21:58:39 INFO BlockManager:59 - Reporting 0 blocks to the master. -2016-05-23 21:58:47 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 21:58:47 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 21:58:47 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 21:58:47 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 21:58:47 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53869 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 21:58:47 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 21:58:47 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 21:58:47 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 21:58:47 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 21:58:47 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 21:58:47 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53869 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 21:58:47 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:26 -2016-05-23 21:58:49 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 21:58:49 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 21:58:49 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:30 -2016-05-23 21:58:50 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:30) with 2 output partitions -2016-05-23 21:58:50 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:30) -2016-05-23 21:58:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:58:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:58:50 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:24), which has no missing parents -2016-05-23 21:58:50 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 21:58:50 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 21:58:50 INFO MemoryStore:59 - ensureFreeSpace(2047) called with curMem=346712, maxMem=766075207 -2016-05-23 21:58:50 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2047.0 B, free 730.3 MB) -2016-05-23 21:58:50 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53869 (size: 2047.0 B, free: 730.6 MB) -2016-05-23 21:58:50 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:58:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:24) -2016-05-23 21:58:50 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 21:58:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:58:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:58:50 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 21:58:50 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 21:58:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:58:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:58:50 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 21:58:50 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 21:58:50 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 21:58:50 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 21:58:50 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 21:58:51 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 21:58:51 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 21:58:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 1053 ms on localhost (1/2) -2016-05-23 21:58:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 1196 ms on localhost (2/2) -2016-05-23 21:58:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 21:58:51 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:30) finished in 1.244 s -2016-05-23 21:58:51 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:30, took 2.078555 s -2016-05-23 21:58:52 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 21:58:52 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:32 -2016-05-23 21:58:52 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:32) with 2 output partitions -2016-05-23 21:58:52 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:32) -2016-05-23 21:58:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:58:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:58:52 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:27), which has no missing parents -2016-05-23 21:58:52 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348759, maxMem=766075207 -2016-05-23 21:58:52 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 21:58:52 INFO MemoryStore:59 - ensureFreeSpace(2045) called with curMem=352271, maxMem=766075207 -2016-05-23 21:58:52 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2045.0 B, free 730.2 MB) -2016-05-23 21:58:52 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53869 (size: 2045.0 B, free: 730.6 MB) -2016-05-23 21:58:52 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:58:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:27) -2016-05-23 21:58:52 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 21:58:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:58:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:58:52 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 21:58:52 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 21:58:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:58:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:58:52 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 21:58:52 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 21:58:52 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53869 in memory (size: 2047.0 B, free: 730.6 MB) -2016-05-23 21:58:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 75 ms on localhost (1/2) -2016-05-23 21:58:52 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-23 21:58:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 80 ms on localhost (2/2) -2016-05-23 21:58:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 21:58:52 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:32) finished in 0.084 s -2016-05-23 21:58:52 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:32, took 0.131957 s -2016-05-23 21:58:52 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:58:52 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:58:52 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 21:58:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:58:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:58:52 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:58:52 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=348757, maxMem=766075207 -2016-05-23 21:58:52 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 21:58:52 INFO MemoryStore:59 - ensureFreeSpace(2465) called with curMem=353317, maxMem=766075207 -2016-05-23 21:58:52 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:58:52 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53869 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 21:58:52 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:58:52 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 21:58:52 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 21:58:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:58:52 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 21:58:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:58:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:58:52 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 21:58:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 70 ms on localhost (1/1) -2016-05-23 21:58:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 21:58:52 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.071 s -2016-05-23 21:58:52 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.106997 s -2016-05-23 21:58:59 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-23 21:58:59 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53869 in memory (size: 2045.0 B, free: 730.6 MB) -2016-05-23 21:58:59 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53869 in memory (size: 2.4 KB, free: 730.6 MB) -2016-05-23 21:59:01 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:59:01 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:59:01 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:01 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:59:01 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=343200, maxMem=766075207 -2016-05-23 21:59:01 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.3 MB) -2016-05-23 21:59:01 INFO MemoryStore:59 - ensureFreeSpace(2862) called with curMem=348648, maxMem=766075207 -2016-05-23 21:59:01 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-05-23 21:59:01 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53869 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:59:01 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:01 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 21:59:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:59:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:59:01 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 21:59:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:59:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:59:01 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 21:59:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:59:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:59:01 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 21:59:01 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 21:59:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 111 ms on localhost (1/2) -2016-05-23 21:59:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 114 ms on localhost (2/2) -2016-05-23 21:59:01 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.121 s -2016-05-23 21:59:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 21:59:01 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 21:59:01 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 21:59:01 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.195072 s -2016-05-23 21:59:02 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:59:02 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:59:02 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:02 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:59:02 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=351510, maxMem=766075207 -2016-05-23 21:59:02 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 21:59:02 INFO MemoryStore:59 - ensureFreeSpace(2851) called with curMem=357006, maxMem=766075207 -2016-05-23 21:59:02 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:59:02 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53869 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:59:02 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:02 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 21:59:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:59:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:59:02 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 21:59:02 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 21:59:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:59:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:59:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:59:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:59:02 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 21:59:02 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 21:59:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 62 ms on localhost (1/2) -2016-05-23 21:59:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 64 ms on localhost (2/2) -2016-05-23 21:59:02 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.067 s -2016-05-23 21:59:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 21:59:02 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.091928 s -2016-05-23 21:59:05 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53869 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:59:05 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53869 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:59:05 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 21:59:05 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 21:59:06 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:59:06 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 21:59:06 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:59:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:06 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:59:06 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=343200, maxMem=766075207 -2016-05-23 21:59:06 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.3 MB) -2016-05-23 21:59:06 INFO MemoryStore:59 - ensureFreeSpace(2744) called with curMem=348544, maxMem=766075207 -2016-05-23 21:59:06 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.3 MB) -2016-05-23 21:59:06 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53869 (size: 2.7 KB, free: 730.6 MB) -2016-05-23 21:59:06 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:59:06 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 21:59:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:59:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 21:59:06 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 21:59:06 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 21:59:06 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:59:06 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:59:06 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:59:06 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:59:06 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 21:59:06 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 21:59:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 84 ms on localhost (1/2) -2016-05-23 21:59:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 88 ms on localhost (2/2) -2016-05-23 21:59:06 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.089 s -2016-05-23 21:59:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 21:59:06 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.118745 s -2016-05-23 21:59:06 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:59:06 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:59:06 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 21:59:06 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:59:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 21:59:06 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 21:59:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:59:06 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=351288, maxMem=766075207 -2016-05-23 21:59:06 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 21:59:06 INFO MemoryStore:59 - ensureFreeSpace(3156) called with curMem=357216, maxMem=766075207 -2016-05-23 21:59:06 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 21:59:06 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53869 (size: 3.1 KB, free: 730.6 MB) -2016-05-23 21:59:06 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:59:06 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 21:59:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 21:59:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 21:59:06 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 21:59:06 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 21:59:06 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:59:06 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:59:06 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 21:59:06 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 21:59:06 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 21:59:06 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 21:59:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 133 ms on localhost (1/2) -2016-05-23 21:59:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 136 ms on localhost (2/2) -2016-05-23 21:59:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 21:59:06 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.138 s -2016-05-23 21:59:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:59:06 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:59:06 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 21:59:06 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:59:06 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 21:59:06 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=360372, maxMem=766075207 -2016-05-23 21:59:07 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=362876, maxMem=766075207 -2016-05-23 21:59:07 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 21:59:07 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53869 (size: 1545.0 B, free: 730.6 MB) -2016-05-23 21:59:07 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:07 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:59:07 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 21:59:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:07 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 21:59:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:59:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 17 ms -2016-05-23 21:59:07 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 21:59:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 240 ms on localhost (1/1) -2016-05-23 21:59:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 21:59:07 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.242 s -2016-05-23 21:59:07 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.483056 s -2016-05-23 21:59:07 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:59:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 21:59:07 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:59:07 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:59:07 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 21:59:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 21:59:07 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 21:59:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=364421, maxMem=766075207 -2016-05-23 21:59:07 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=368613, maxMem=766075207 -2016-05-23 21:59:07 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:59:07 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53869 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:59:07 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:59:07 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 21:59:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:59:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:59:07 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 21:59:07 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 21:59:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:59:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:59:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 14 ms -2016-05-23 21:59:07 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 21:59:07 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 21:59:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 73 ms on localhost (1/2) -2016-05-23 21:59:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 80 ms on localhost (2/2) -2016-05-23 21:59:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 21:59:07 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.082 s -2016-05-23 21:59:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:59:07 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:59:07 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 21:59:07 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:59:07 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 21:59:07 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=370895, maxMem=766075207 -2016-05-23 21:59:07 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=375751, maxMem=766075207 -2016-05-23 21:59:07 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:59:07 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53869 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:59:07 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:07 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:59:07 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 21:59:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:07 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 21:59:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:59:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:07 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 21:59:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 48 ms on localhost (1/1) -2016-05-23 21:59:07 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.048 s -2016-05-23 21:59:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 21:59:07 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.192712 s -2016-05-23 21:59:07 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:59:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 21:59:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 21:59:07 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:59:07 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 21:59:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:07 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=378282, maxMem=766075207 -2016-05-23 21:59:07 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=383938, maxMem=766075207 -2016-05-23 21:59:07 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:59:07 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53869 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:59:07 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:07 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 21:59:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:07 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 21:59:07 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 21:59:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:59:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:59:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-23 21:59:07 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 21:59:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 30 ms on localhost (1/2) -2016-05-23 21:59:07 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 21:59:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 38 ms on localhost (2/2) -2016-05-23 21:59:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 21:59:07 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.041 s -2016-05-23 21:59:07 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.063563 s -2016-05-23 21:59:07 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:59:07 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:59:07 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 21:59:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:07 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=386846, maxMem=766075207 -2016-05-23 21:59:07 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:59:07 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-23 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=392542, maxMem=766075207 -2016-05-23 21:59:07 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:59:07 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53869 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:59:07 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:53869 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:59:07 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:07 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 21:59:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:07 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 21:59:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:59:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:07 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 21:59:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:59:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:07 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 21:59:07 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 21:59:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 57 ms on localhost (1/2) -2016-05-23 21:59:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 62 ms on localhost (2/2) -2016-05-23 21:59:07 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.062 s -2016-05-23 21:59:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 21:59:07 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-23 21:59:07 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.130646 s -2016-05-23 21:59:07 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:53869 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:59:07 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-23 21:59:07 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53869 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 21:59:07 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 21:59:07 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53869 in memory (size: 3.1 KB, free: 730.6 MB) -2016-05-23 21:59:07 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 21:59:07 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53869 in memory (size: 2.7 KB, free: 730.6 MB) -2016-05-23 21:59:07 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:46 -2016-05-23 21:59:07 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:46) with 4 output partitions -2016-05-23 21:59:07 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:46) -2016-05-23 21:59:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:07 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:43), which has no missing parents -2016-05-23 21:59:07 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=360349, maxMem=766075207 -2016-05-23 21:59:07 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=361573, maxMem=766075207 -2016-05-23 21:59:07 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:59:07 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53869 (size: 893.0 B, free: 730.6 MB) -2016-05-23 21:59:07 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:53869 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:59:07 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:07 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:43) -2016-05-23 21:59:07 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 21:59:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:59:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:59:07 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:59:07 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:59:07 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 21:59:07 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 21:59:07 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 21:59:07 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 21:59:07 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 21:59:07 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 21:59:07 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 21:59:07 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 12 ms on localhost (1/4) -2016-05-23 21:59:07 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 21:59:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 26 ms on localhost (2/4) -2016-05-23 21:59:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 18 ms on localhost (3/4) -2016-05-23 21:59:07 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 18 ms on localhost (4/4) -2016-05-23 21:59:07 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:46) finished in 0.029 s -2016-05-23 21:59:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 21:59:07 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:46, took 0.051296 s -2016-05-23 21:59:07 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:48 -2016-05-23 21:59:07 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:48) with 4 output partitions -2016-05-23 21:59:07 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:48) -2016-05-23 21:59:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:07 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:44), which has no missing parents -2016-05-23 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=353902, maxMem=766075207 -2016-05-23 21:59:07 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=355126, maxMem=766075207 -2016-05-23 21:59:07 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:59:07 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53869 (size: 893.0 B, free: 730.6 MB) -2016-05-23 21:59:07 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:07 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:44) -2016-05-23 21:59:07 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 21:59:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:59:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:59:07 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:59:07 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 21:59:07 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 21:59:07 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 21:59:07 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 21:59:07 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 21:59:07 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 21:59:07 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 21:59:07 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 21:59:07 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 21:59:07 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 8 ms on localhost (1/4) -2016-05-23 21:59:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 12 ms on localhost (2/4) -2016-05-23 21:59:07 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 12 ms on localhost (3/4) -2016-05-23 21:59:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 14 ms on localhost (4/4) -2016-05-23 21:59:07 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:48) finished in 0.016 s -2016-05-23 21:59:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 21:59:07 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:48, took 0.028133 s -2016-05-23 21:59:07 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:59:07 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:59:07 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 21:59:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:07 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=356019, maxMem=766075207 -2016-05-23 21:59:07 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=358523, maxMem=766075207 -2016-05-23 21:59:07 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 21:59:07 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53869 (size: 1524.0 B, free: 730.6 MB) -2016-05-23 21:59:07 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:07 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 21:59:07 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 21:59:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:59:07 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 21:59:07 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 21:59:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 9 ms on localhost (1/1) -2016-05-23 21:59:07 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.008 s -2016-05-23 21:59:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 21:59:07 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.027068 s -2016-05-23 21:59:07 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:59:08 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:59:08 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:08 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=360047, maxMem=766075207 -2016-05-23 21:59:08 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=363447, maxMem=766075207 -2016-05-23 21:59:08 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53869 (size: 1934.0 B, free: 730.6 MB) -2016-05-23 21:59:08 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:08 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:59:08 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 21:59:08 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 21:59:08 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 21:59:08 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 21:59:08 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 25 ms on localhost (1/4) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 23 ms on localhost (2/4) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 22 ms on localhost (3/4) -2016-05-23 21:59:08 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 35 ms on localhost (4/4) -2016-05-23 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 21:59:08 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.037 s -2016-05-23 21:59:08 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.056076 s -2016-05-23 21:59:08 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:59:08 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:59:08 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:08 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=365381, maxMem=766075207 -2016-05-23 21:59:08 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=368821, maxMem=766075207 -2016-05-23 21:59:08 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53869 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:59:08 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:08 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:59:08 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 21:59:08 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 21:59:08 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 21:59:08 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 18 ms on localhost (1/4) -2016-05-23 21:59:08 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 15 ms on localhost (2/4) -2016-05-23 21:59:08 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 19 ms on localhost (3/4) -2016-05-23 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 30 ms on localhost (4/4) -2016-05-23 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 21:59:08 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.031 s -2016-05-23 21:59:08 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.052863 s -2016-05-23 21:59:08 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:59:08 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:59:08 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:59:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:08 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=370739, maxMem=766075207 -2016-05-23 21:59:08 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(1860) called with curMem=374115, maxMem=766075207 -2016-05-23 21:59:08 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1860.0 B, free 730.2 MB) -2016-05-23 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53869 (size: 1860.0 B, free: 730.5 MB) -2016-05-23 21:59:08 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:08 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 21:59:08 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 21:59:08 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 21:59:08 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 21:59:08 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 21:59:08 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 21:59:08 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 26 ms on localhost (1/4) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 29 ms on localhost (2/4) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 28 ms on localhost (3/4) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 28 ms on localhost (4/4) -2016-05-23 21:59:08 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.034 s -2016-05-23 21:59:08 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.062850 s -2016-05-23 21:59:08 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:59:08 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:59:08 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:59:08 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:59:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 21:59:08 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 21:59:08 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=375975, maxMem=766075207 -2016-05-23 21:59:08 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=380015, maxMem=766075207 -2016-05-23 21:59:08 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:53869 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:59:08 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:08 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 21:59:08 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 21:59:08 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 21:59:08 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 21:59:08 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 21:59:08 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 21:59:08 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 82 ms on localhost (1/4) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 95 ms on localhost (2/4) -2016-05-23 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 121 ms on localhost (3/4) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 120 ms on localhost (4/4) -2016-05-23 21:59:08 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.122 s -2016-05-23 21:59:08 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:59:08 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:59:08 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 21:59:08 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 21:59:08 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 21:59:08 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=382297, maxMem=766075207 -2016-05-23 21:59:08 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=384841, maxMem=766075207 -2016-05-23 21:59:08 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:53869 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 21:59:08 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:53869 in memory (size: 1860.0 B, free: 730.5 MB) -2016-05-23 21:59:08 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 21:59:08 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:08 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 21:59:08 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:08 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 21:59:08 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:08 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 37 ms on localhost (1/3) -2016-05-23 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 21:59:08 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 21:59:08 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 55 ms on localhost (2/3) -2016-05-23 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:53869 in memory (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 71 ms on localhost (3/3) -2016-05-23 21:59:08 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.072 s -2016-05-23 21:59:08 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-23 21:59:08 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.298841 s -2016-05-23 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:53869 in memory (size: 1934.0 B, free: 730.5 MB) -2016-05-23 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 21:59:08 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-23 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:53869 in memory (size: 1524.0 B, free: 730.6 MB) -2016-05-23 21:59:08 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-23 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:53869 in memory (size: 893.0 B, free: 730.6 MB) -2016-05-23 21:59:08 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-23 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:53869 in memory (size: 893.0 B, free: 730.6 MB) -2016-05-23 21:59:08 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-23 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:53869 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 21:59:08 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:59:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 21:59:08 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:59:08 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:59:08 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 21:59:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 21:59:08 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 21:59:08 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=353634, maxMem=766075207 -2016-05-23 21:59:08 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=357866, maxMem=766075207 -2016-05-23 21:59:08 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:53869 (size: 2.3 KB, free: 730.6 MB) -2016-05-23 21:59:08 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:08 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:59:08 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 21:59:08 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 21:59:08 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:08 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:08 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 21:59:08 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:08 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 21:59:08 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 21:59:08 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 91 ms on localhost (1/4) -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 93 ms on localhost (2/4) -2016-05-23 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 234 ms on localhost (3/4) -2016-05-23 21:59:08 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 303 ms on localhost (4/4) -2016-05-23 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 21:59:08 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.310 s -2016-05-23 21:59:08 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:59:08 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:59:08 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 21:59:08 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:59:08 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 21:59:08 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=360172, maxMem=766075207 -2016-05-23 21:59:08 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=365068, maxMem=766075207 -2016-05-23 21:59:08 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:53869 (size: 2.5 KB, free: 730.6 MB) -2016-05-23 21:59:08 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:08 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:08 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 21:59:08 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 26 ms on localhost (1/1) -2016-05-23 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 21:59:08 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.028 s -2016-05-23 21:59:08 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.394965 s -2016-05-23 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:59:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 21:59:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 21:59:09 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=367610, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=373306, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:53869 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:59:09 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:09 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 21:59:09 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 21:59:09 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:09 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:09 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 45 ms on localhost (1/4) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 48 ms on localhost (2/4) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 48 ms on localhost (3/4) -2016-05-23 21:59:09 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 98 ms on localhost (4/4) -2016-05-23 21:59:09 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.099 s -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 21:59:09 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.118861 s -2016-05-23 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:59:09 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=376215, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=381951, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:53869 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:59:09 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:09 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 21:59:09 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 21:59:09 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:09 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:09 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 21:59:09 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 37 ms on localhost (1/4) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 39 ms on localhost (2/4) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 38 ms on localhost (3/4) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 39 ms on localhost (4/4) -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 21:59:09 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.044 s -2016-05-23 21:59:09 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.058980 s -2016-05-23 21:59:09 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:60 -2016-05-23 21:59:09 INFO DAGScheduler:59 - Got job 20 (foreach at StatisticsCorrLearning.scala:60) with 4 output partitions -2016-05-23 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 34(foreach at StatisticsCorrLearning.scala:60) -2016-05-23 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:57), which has no missing parents -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=384843, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=386067, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:53869 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:59:09 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:57) -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 62, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 63, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 64, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:59:09 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 64) -2016-05-23 21:59:09 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 65) -2016-05-23 21:59:09 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 63) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 64). 915 bytes result sent to driver -2016-05-23 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 63). 915 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 64) in 9 ms on localhost (1/4) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 65). 915 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 63) in 11 ms on localhost (2/4) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 65) in 9 ms on localhost (3/4) -2016-05-23 21:59:09 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 62) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 62). 915 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 62) in 53 ms on localhost (4/4) -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-23 21:59:09 INFO DAGScheduler:59 - ResultStage 34 (foreach at StatisticsCorrLearning.scala:60) finished in 0.080 s -2016-05-23 21:59:09 INFO DAGScheduler:59 - Job 20 finished: foreach at StatisticsCorrLearning.scala:60, took 0.097567 s -2016-05-23 21:59:09 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:62 -2016-05-23 21:59:09 INFO DAGScheduler:59 - Got job 21 (foreach at StatisticsCorrLearning.scala:62) with 4 output partitions -2016-05-23 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at StatisticsCorrLearning.scala:62) -2016-05-23 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:58), which has no missing parents -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=386960, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=388184, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:53869 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:59:09 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:58) -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 4 tasks -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 66, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:59:09 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 66) -2016-05-23 21:59:09 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 67) -2016-05-23 21:59:09 INFO Executor:59 - Running task 3.0 in stage 35.0 (TID 69) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 66). 915 bytes result sent to driver -2016-05-23 21:59:09 INFO Executor:59 - Finished task 3.0 in stage 35.0 (TID 69). 915 bytes result sent to driver -2016-05-23 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 67). 915 bytes result sent to driver -2016-05-23 21:59:09 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 68) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 68). 915 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 66) in 37 ms on localhost (1/4) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 67) in 56 ms on localhost (2/4) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 68) in 64 ms on localhost (3/4) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 35.0 (TID 69) in 66 ms on localhost (4/4) -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-23 21:59:09 INFO DAGScheduler:59 - ResultStage 35 (foreach at StatisticsCorrLearning.scala:62) finished in 0.072 s -2016-05-23 21:59:09 INFO DAGScheduler:59 - Job 21 finished: foreach at StatisticsCorrLearning.scala:62, took 0.091525 s -2016-05-23 21:59:09 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:59:09 INFO DAGScheduler:59 - Got job 22 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 36(first at RowMatrix.scala:65) -2016-05-23 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=389077, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(1522) called with curMem=391581, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1522.0 B, free 730.2 MB) -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:53869 (size: 1522.0 B, free: 730.5 MB) -2016-05-23 21:59:09 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:09 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 70) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 70). 1046 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 70) in 22 ms on localhost (1/1) -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-23 21:59:09 INFO DAGScheduler:59 - ResultStage 36 (first at RowMatrix.scala:65) finished in 0.023 s -2016-05-23 21:59:09 INFO DAGScheduler:59 - Job 22 finished: first at RowMatrix.scala:65, took 0.036758 s -2016-05-23 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:59:09 INFO DAGScheduler:59 - Got job 23 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=393103, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(1932) called with curMem=396503, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1932.0 B, free 730.2 MB) -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:53869 (size: 1932.0 B, free: 730.5 MB) -2016-05-23 21:59:09 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 4 tasks -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:59:09 INFO Executor:59 - Running task 3.0 in stage 37.0 (TID 74) -2016-05-23 21:59:09 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 72) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 3.0 in stage 37.0 (TID 74). 1231 bytes result sent to driver -2016-05-23 21:59:09 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 73) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 72). 1231 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 37.0 (TID 74) in 7 ms on localhost (1/4) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 72) in 10 ms on localhost (2/4) -2016-05-23 21:59:09 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 73). 1231 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 73) in 11 ms on localhost (3/4) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 1231 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 17 ms on localhost (4/4) -2016-05-23 21:59:09 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at RowMatrix.scala:332) finished in 0.017 s -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-23 21:59:09 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at RowMatrix.scala:332, took 0.030116 s -2016-05-23 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:59:09 INFO DAGScheduler:59 - Got job 24 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=398435, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(1916) called with curMem=401875, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1916.0 B, free 730.2 MB) -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:53869 (size: 1916.0 B, free: 730.5 MB) -2016-05-23 21:59:09 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 77, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 78, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:59:09 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 1109 bytes result sent to driver -2016-05-23 21:59:09 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 77) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 77). 1109 bytes result sent to driver -2016-05-23 21:59:09 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 1109 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 19 ms on localhost (1/4) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 77) in 19 ms on localhost (2/4) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 22 ms on localhost (3/4) -2016-05-23 21:59:09 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 78) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 78). 1109 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 78) in 23 ms on localhost (4/4) -2016-05-23 21:59:09 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at RowMatrix.scala:124) finished in 0.028 s -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-23 21:59:09 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at RowMatrix.scala:124, took 0.144899 s -2016-05-23 21:59:09 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:59:09 INFO DAGScheduler:59 - Got job 25 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 39(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=403791, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(1851) called with curMem=407167, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1851.0 B, free 730.2 MB) -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:53869 (size: 1851.0 B, free: 730.5 MB) -2016-05-23 21:59:09 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 4 tasks -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 79, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 80, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 81, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 39.0 (TID 82, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:59:09 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 81) -2016-05-23 21:59:09 INFO Executor:59 - Running task 3.0 in stage 39.0 (TID 82) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 81). 1249 bytes result sent to driver -2016-05-23 21:59:09 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 80) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 3.0 in stage 39.0 (TID 82). 1294 bytes result sent to driver -2016-05-23 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 80). 1254 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 81) in 9 ms on localhost (1/4) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 39.0 (TID 82) in 10 ms on localhost (2/4) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 80) in 12 ms on localhost (3/4) -2016-05-23 21:59:09 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 79) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 79). 1254 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 79) in 19 ms on localhost (4/4) -2016-05-23 21:59:09 INFO DAGScheduler:59 - ResultStage 39 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.020 s -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-23 21:59:09 INFO DAGScheduler:59 - Job 25 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.048614 s -2016-05-23 21:59:09 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:59:09 INFO DAGScheduler:59 - Registering RDD 51 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:59:09 INFO DAGScheduler:59 - Got job 26 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 41(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 40) -2016-05-23 21:59:09 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 40) -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=409018, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2281) called with curMem=413058, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:53869 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:59:09 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 4 tasks -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 83, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 84, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 85, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 40.0 (TID 86, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:59:09 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 84) -2016-05-23 21:59:09 INFO Executor:59 - Running task 3.0 in stage 40.0 (TID 86) -2016-05-23 21:59:09 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 85) -2016-05-23 21:59:09 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 83) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 84). 1161 bytes result sent to driver -2016-05-23 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 83). 1161 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 83) in 44 ms on localhost (1/4) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 84) in 44 ms on localhost (2/4) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 85). 1161 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 85) in 48 ms on localhost (3/4) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 3.0 in stage 40.0 (TID 86). 1161 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 40.0 (TID 86) in 50 ms on localhost (4/4) -2016-05-23 21:59:09 INFO DAGScheduler:59 - ShuffleMapStage 40 (flatMap at SpearmanCorrelation.scala:49) finished in 0.052 s -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-23 21:59:09 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:59:09 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:59:09 INFO DAGScheduler:59 - waiting: Set(ResultStage 41) -2016-05-23 21:59:09 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:59:09 INFO DAGScheduler:59 - Missing parents for ResultStage 41: List() -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=415339, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=417883, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.2 MB) -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:53869 (size: 1567.0 B, free: 730.5 MB) -2016-05-23 21:59:09 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 3 tasks -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 87, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:09 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 87) -2016-05-23 21:59:09 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 89) -2016-05-23 21:59:09 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 88) -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 88). 1203 bytes result sent to driver -2016-05-23 21:59:09 INFO Executor:59 - Finished task 2.0 in stage 41.0 (TID 89). 1203 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 88) in 11 ms on localhost (1/3) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 87). 1203 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 41.0 (TID 89) in 11 ms on localhost (2/3) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 87) in 13 ms on localhost (3/3) -2016-05-23 21:59:09 INFO DAGScheduler:59 - ResultStage 41 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.013 s -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-23 21:59:09 INFO DAGScheduler:59 - Job 26 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.088182 s -2016-05-23 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:53869 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:59:09 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-23 21:59:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 163 bytes -2016-05-23 21:59:09 INFO DAGScheduler:59 - Registering RDD 56 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:59:09 INFO DAGScheduler:59 - Got job 27 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 44(first at RowMatrix.scala:65) -2016-05-23 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 43) -2016-05-23 21:59:09 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 43) -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:53869 in memory (size: 1851.0 B, free: 730.5 MB) -2016-05-23 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=407902, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2305) called with curMem=410218, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:53869 in memory (size: 1916.0 B, free: 730.5 MB) -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:53869 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-23 21:59:09 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 4 tasks -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 90, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:53869 in memory (size: 1932.0 B, free: 730.5 MB) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 91, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 43.0 (TID 92, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 43.0 (TID 93, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:59:09 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 90) -2016-05-23 21:59:09 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 91) -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:09 INFO Executor:59 - Running task 3.0 in stage 43.0 (TID 93) -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:09 INFO Executor:59 - Running task 2.0 in stage 43.0 (TID 92) -2016-05-23 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:53869 in memory (size: 1522.0 B, free: 730.5 MB) -2016-05-23 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:53869 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-23 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 91). 1377 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 91) in 54 ms on localhost (1/4) -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:53869 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-23 21:59:09 INFO Executor:59 - Finished task 3.0 in stage 43.0 (TID 93). 1377 bytes result sent to driver -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:53869 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 43.0 (TID 93) in 61 ms on localhost (2/4) -2016-05-23 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:53869 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:59:09 INFO Executor:59 - Finished task 2.0 in stage 43.0 (TID 92). 1377 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 43.0 (TID 92) in 68 ms on localhost (3/4) -2016-05-23 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:53869 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-23 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 90). 1377 bytes result sent to driver -2016-05-23 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 90) in 76 ms on localhost (4/4) -2016-05-23 21:59:09 INFO DAGScheduler:59 - ShuffleMapStage 43 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.077 s -2016-05-23 21:59:09 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-23 21:59:09 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:59:09 INFO DAGScheduler:59 - waiting: Set(ResultStage 44) -2016-05-23 21:59:09 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:59:09 INFO DAGScheduler:59 - Missing parents for ResultStage 44: List() -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:53869 in memory (size: 2.3 KB, free: 730.6 MB) -2016-05-23 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=364282, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2545) called with curMem=369178, maxMem=766075207 -2016-05-23 21:59:09 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:59:09 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:53869 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:59:09 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:53869 in memory (size: 1568.0 B, free: 730.6 MB) -2016-05-23 21:59:09 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 1 tasks -2016-05-23 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 94, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:09 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 94) -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:53869 in memory (size: 1567.0 B, free: 730.6 MB) -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:10 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 94). 1296 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 94) in 14 ms on localhost (1/1) -2016-05-23 21:59:10 INFO DAGScheduler:59 - ResultStage 44 (first at RowMatrix.scala:65) finished in 0.015 s -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-23 21:59:10 INFO DAGScheduler:59 - Job 27 finished: first at RowMatrix.scala:65, took 0.119423 s -2016-05-23 21:59:10 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:59:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 163 bytes -2016-05-23 21:59:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 166 bytes -2016-05-23 21:59:10 INFO DAGScheduler:59 - Got job 28 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:59:10 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 46) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=363500, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(2914) called with curMem=369196, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:53869 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:59:10 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 4 tasks -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 95, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 96, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 47.0 (TID 97, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 47.0 (TID 98, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:10 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 95) -2016-05-23 21:59:10 INFO Executor:59 - Running task 2.0 in stage 47.0 (TID 97) -2016-05-23 21:59:10 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 96) -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:10 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 95). 1481 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 95) in 18 ms on localhost (1/4) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 96). 1481 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 96) in 19 ms on localhost (2/4) -2016-05-23 21:59:10 INFO Executor:59 - Running task 3.0 in stage 47.0 (TID 98) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 2.0 in stage 47.0 (TID 97). 1481 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 47.0 (TID 97) in 23 ms on localhost (3/4) -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:10 INFO Executor:59 - Finished task 3.0 in stage 47.0 (TID 98). 1481 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 47.0 (TID 98) in 34 ms on localhost (4/4) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-23 21:59:10 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at RowMatrix.scala:332) finished in 0.038 s -2016-05-23 21:59:10 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at RowMatrix.scala:332, took 0.054496 s -2016-05-23 21:59:10 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Got job 29 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:59:10 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 49) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=372110, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=377846, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:53869 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:59:10 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 4 tasks -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 50.0 (TID 101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 50.0 (TID 102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:10 INFO Executor:59 - Running task 3.0 in stage 50.0 (TID 102) -2016-05-23 21:59:10 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-23 21:59:10 INFO Executor:59 - Running task 2.0 in stage 50.0 (TID 101) -2016-05-23 21:59:10 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:10 INFO Executor:59 - Finished task 3.0 in stage 50.0 (TID 102). 1359 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 1359 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Finished task 2.0 in stage 50.0 (TID 101). 1359 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 50.0 (TID 102) in 14 ms on localhost (1/4) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 17 ms on localhost (2/4) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 50.0 (TID 101) in 16 ms on localhost (3/4) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 1359 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 20 ms on localhost (4/4) -2016-05-23 21:59:10 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at RowMatrix.scala:124) finished in 0.022 s -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-23 21:59:10 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at RowMatrix.scala:124, took 0.064042 s -2016-05-23 21:59:10 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:73 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Got job 30 (foreach at StatisticsCorrLearning.scala:73) with 4 output partitions -2016-05-23 21:59:10 INFO DAGScheduler:59 - Final stage: ResultStage 51(foreach at StatisticsCorrLearning.scala:73) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting ResultStage 51 (ParallelCollectionRDD[61] at parallelize at StatisticsCorrLearning.scala:70), which has no missing parents -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=380742, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=381966, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:53869 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:59:10 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 51 (ParallelCollectionRDD[61] at parallelize at StatisticsCorrLearning.scala:70) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 4 tasks -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 103, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 104, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 51.0 (TID 105, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 51.0 (TID 106, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:59:10 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 104) -2016-05-23 21:59:10 INFO Executor:59 - Running task 2.0 in stage 51.0 (TID 105) -2016-05-23 21:59:10 INFO Executor:59 - Running task 3.0 in stage 51.0 (TID 106) -2016-05-23 21:59:10 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 103) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 2.0 in stage 51.0 (TID 105). 915 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 104). 915 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Finished task 3.0 in stage 51.0 (TID 106). 915 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 103). 915 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 51.0 (TID 105) in 9 ms on localhost (1/4) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 51.0 (TID 106) in 9 ms on localhost (2/4) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 104) in 10 ms on localhost (3/4) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 103) in 12 ms on localhost (4/4) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-23 21:59:10 INFO DAGScheduler:59 - ResultStage 51 (foreach at StatisticsCorrLearning.scala:73) finished in 0.012 s -2016-05-23 21:59:10 INFO DAGScheduler:59 - Job 30 finished: foreach at StatisticsCorrLearning.scala:73, took 0.019696 s -2016-05-23 21:59:10 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:75 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Got job 31 (foreach at StatisticsCorrLearning.scala:75) with 4 output partitions -2016-05-23 21:59:10 INFO DAGScheduler:59 - Final stage: ResultStage 52(foreach at StatisticsCorrLearning.scala:75) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting ResultStage 52 (ParallelCollectionRDD[62] at parallelize at StatisticsCorrLearning.scala:71), which has no missing parents -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=382859, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=384083, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:53869 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:59:10 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 52 (ParallelCollectionRDD[62] at parallelize at StatisticsCorrLearning.scala:71) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 4 tasks -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 107, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 108, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 52.0 (TID 109, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 52.0 (TID 110, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:59:10 INFO Executor:59 - Running task 2.0 in stage 52.0 (TID 109) -2016-05-23 21:59:10 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 107) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 2.0 in stage 52.0 (TID 109). 915 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 108) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 107). 915 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 108). 915 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 52.0 (TID 109) in 9 ms on localhost (1/4) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 108) in 10 ms on localhost (2/4) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 107) in 12 ms on localhost (3/4) -2016-05-23 21:59:10 INFO Executor:59 - Running task 3.0 in stage 52.0 (TID 110) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 3.0 in stage 52.0 (TID 110). 915 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 52.0 (TID 110) in 15 ms on localhost (4/4) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-23 21:59:10 INFO DAGScheduler:59 - ResultStage 52 (foreach at StatisticsCorrLearning.scala:75) finished in 0.019 s -2016-05-23 21:59:10 INFO DAGScheduler:59 - Job 31 finished: foreach at StatisticsCorrLearning.scala:75, took 0.030615 s -2016-05-23 21:59:10 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Got job 32 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:59:10 INFO DAGScheduler:59 - Final stage: ResultStage 53(first at RowMatrix.scala:65) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[64] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=384976, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=387480, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:53869 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:59:10 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 53 (MapPartitionsRDD[64] at map at Correlation.scala:45) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 1 tasks -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 111, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:59:10 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 111) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 111). 935 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 111) in 10 ms on localhost (1/1) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-23 21:59:10 INFO DAGScheduler:59 - ResultStage 53 (first at RowMatrix.scala:65) finished in 0.015 s -2016-05-23 21:59:10 INFO DAGScheduler:59 - Job 32 finished: first at RowMatrix.scala:65, took 0.033069 s -2016-05-23 21:59:10 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Got job 33 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 21:59:10 INFO DAGScheduler:59 - Final stage: ResultStage 54(first at RowMatrix.scala:65) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[64] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=389004, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=391508, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:53869 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:59:10 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 54 (MapPartitionsRDD[64] at map at Correlation.scala:45) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 3 tasks -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 112, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 113, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 54.0 (TID 114, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:10 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 112) -2016-05-23 21:59:10 INFO Executor:59 - Running task 2.0 in stage 54.0 (TID 114) -2016-05-23 21:59:10 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 113) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 112). 1046 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 113). 1046 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Finished task 2.0 in stage 54.0 (TID 114). 1046 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 112) in 5 ms on localhost (1/3) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 113) in 5 ms on localhost (2/3) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 54.0 (TID 114) in 5 ms on localhost (3/3) -2016-05-23 21:59:10 INFO DAGScheduler:59 - ResultStage 54 (first at RowMatrix.scala:65) finished in 0.006 s -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-23 21:59:10 INFO DAGScheduler:59 - Job 33 finished: first at RowMatrix.scala:65, took 0.015863 s -2016-05-23 21:59:10 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Got job 34 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:59:10 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[65] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=393032, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=396432, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:53869 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 21:59:10 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 55 (MapPartitionsRDD[65] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 4 tasks -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 115, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 116, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 55.0 (TID 117, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 55.0 (TID 118, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:10 INFO Executor:59 - Running task 2.0 in stage 55.0 (TID 117) -2016-05-23 21:59:10 INFO Executor:59 - Running task 3.0 in stage 55.0 (TID 118) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 2.0 in stage 55.0 (TID 117). 1231 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Finished task 3.0 in stage 55.0 (TID 118). 1231 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 116) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 116). 1231 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 115) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 115). 1231 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 55.0 (TID 117) in 18 ms on localhost (1/4) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 115) in 21 ms on localhost (2/4) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 116) in 21 ms on localhost (3/4) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 55.0 (TID 118) in 21 ms on localhost (4/4) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-23 21:59:10 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at RowMatrix.scala:332) finished in 0.027 s -2016-05-23 21:59:10 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at RowMatrix.scala:332, took 0.048023 s -2016-05-23 21:59:10 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Got job 35 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:59:10 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[66] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=398366, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=401806, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:53869 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:59:10 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 56 (MapPartitionsRDD[66] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 4 tasks -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 119, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 120, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 56.0 (TID 121, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 56.0 (TID 122, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:10 INFO Executor:59 - Running task 2.0 in stage 56.0 (TID 121) -2016-05-23 21:59:10 INFO Executor:59 - Running task 3.0 in stage 56.0 (TID 122) -2016-05-23 21:59:10 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 120) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 120). 1109 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Finished task 3.0 in stage 56.0 (TID 122). 1109 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Finished task 2.0 in stage 56.0 (TID 121). 1109 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 119) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 119). 1109 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 120) in 18 ms on localhost (1/4) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 56.0 (TID 122) in 17 ms on localhost (2/4) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 56.0 (TID 121) in 19 ms on localhost (3/4) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 119) in 22 ms on localhost (4/4) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-23 21:59:10 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at RowMatrix.scala:124) finished in 0.025 s -2016-05-23 21:59:10 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at RowMatrix.scala:124, took 0.045628 s -2016-05-23 21:59:10 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Got job 36 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:59:10 INFO DAGScheduler:59 - Final stage: ResultStage 57(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[72] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=403724, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(1858) called with curMem=407100, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 1858.0 B, free 730.2 MB) -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:53869 (size: 1858.0 B, free: 730.5 MB) -2016-05-23 21:59:10 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 57 (MapPartitionsRDD[72] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 4 tasks -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 123, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 124, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 57.0 (TID 125, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 57.0 (TID 126, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:10 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 124) -2016-05-23 21:59:10 INFO Executor:59 - Running task 2.0 in stage 57.0 (TID 125) -2016-05-23 21:59:10 INFO Executor:59 - Running task 3.0 in stage 57.0 (TID 126) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 3.0 in stage 57.0 (TID 126). 1254 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 124). 1254 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Finished task 2.0 in stage 57.0 (TID 125). 1249 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 123) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 57.0 (TID 126) in 11 ms on localhost (1/4) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 123). 1104 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 57.0 (TID 125) in 18 ms on localhost (2/4) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 124) in 20 ms on localhost (3/4) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 123) in 25 ms on localhost (4/4) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-23 21:59:10 INFO DAGScheduler:59 - ResultStage 57 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.028 s -2016-05-23 21:59:10 INFO DAGScheduler:59 - Job 36 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.041414 s -2016-05-23 21:59:10 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Registering RDD 70 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Got job 37 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:59:10 INFO DAGScheduler:59 - Final stage: ResultStage 59(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 58) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 58) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 58 (MapPartitionsRDD[70] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=408958, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(2285) called with curMem=412998, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:53869 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:59:10 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 58 (MapPartitionsRDD[70] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 4 tasks -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 127, localhost, PROCESS_LOCAL, 2221 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 128, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 58.0 (TID 129, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 58.0 (TID 130, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:59:10 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 127) -2016-05-23 21:59:10 INFO Executor:59 - Running task 2.0 in stage 58.0 (TID 129) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 127). 1161 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 127) in 10 ms on localhost (1/4) -2016-05-23 21:59:10 INFO Executor:59 - Running task 3.0 in stage 58.0 (TID 130) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 3.0 in stage 58.0 (TID 130). 1161 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Finished task 2.0 in stage 58.0 (TID 129). 1161 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 58.0 (TID 130) in 53 ms on localhost (2/4) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 58.0 (TID 129) in 53 ms on localhost (3/4) -2016-05-23 21:59:10 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 128) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 128). 1161 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 128) in 125 ms on localhost (4/4) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-05-23 21:59:10 INFO DAGScheduler:59 - ShuffleMapStage 58 (flatMap at SpearmanCorrelation.scala:49) finished in 0.130 s -2016-05-23 21:59:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:59:10 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:59:10 INFO DAGScheduler:59 - waiting: Set(ResultStage 59) -2016-05-23 21:59:10 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:59:10 INFO DAGScheduler:59 - Missing parents for ResultStage 59: List() -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting ResultStage 59 (ShuffledRDD[73] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=415283, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=417827, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.2 MB) -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:53869 (size: 1567.0 B, free: 730.5 MB) -2016-05-23 21:59:10 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 59 (ShuffledRDD[73] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 3 tasks -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 131, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 132, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 59.0 (TID 133, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:10 INFO Executor:59 - Running task 2.0 in stage 59.0 (TID 133) -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:10 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 132) -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:10 INFO Executor:59 - Finished task 2.0 in stage 59.0 (TID 133). 1203 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 59.0 (TID 133) in 15 ms on localhost (1/3) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 132). 1203 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 132) in 19 ms on localhost (2/3) -2016-05-23 21:59:10 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 131) -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:10 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 131). 1203 bytes result sent to driver -2016-05-23 21:59:10 INFO DAGScheduler:59 - ResultStage 59 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.038 s -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 131) in 38 ms on localhost (3/3) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-05-23 21:59:10 INFO DAGScheduler:59 - Job 37 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.219327 s -2016-05-23 21:59:10 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:59:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 168 bytes -2016-05-23 21:59:10 INFO DAGScheduler:59 - Registering RDD 75 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Got job 38 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:59:10 INFO DAGScheduler:59 - Final stage: ResultStage 62(first at RowMatrix.scala:65) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 61) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 61) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 61 (MapPartitionsRDD[75] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=419394, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(2305) called with curMem=423626, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 21:59:10 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:53869 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:59:10 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 61 (MapPartitionsRDD[75] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 4 tasks -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 134, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 135, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 61.0 (TID 136, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 61.0 (TID 137, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:53869 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:59:10 INFO Executor:59 - Running task 3.0 in stage 61.0 (TID 137) -2016-05-23 21:59:10 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 134) -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:10 INFO Executor:59 - Running task 2.0 in stage 61.0 (TID 136) -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:10 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 135) -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:10 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:53869 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:59:10 INFO Executor:59 - Finished task 3.0 in stage 61.0 (TID 137). 1377 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 134). 1377 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Finished task 2.0 in stage 61.0 (TID 136). 1377 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 135). 1377 bytes result sent to driver -2016-05-23 21:59:10 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-23 21:59:10 INFO ContextCleaner:59 - Cleaned shuffle 5 -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 61.0 (TID 137) in 112 ms on localhost (1/4) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 134) in 115 ms on localhost (2/4) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 61.0 (TID 136) in 114 ms on localhost (3/4) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 135) in 116 ms on localhost (4/4) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-05-23 21:59:10 INFO DAGScheduler:59 - ShuffleMapStage 61 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.131 s -2016-05-23 21:59:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:59:10 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:59:10 INFO DAGScheduler:59 - waiting: Set(ResultStage 62) -2016-05-23 21:59:10 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:59:10 INFO DAGScheduler:59 - Missing parents for ResultStage 62: List() -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:53869 in memory (size: 1567.0 B, free: 730.5 MB) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:59:10 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=407842, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=412738, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:53869 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:59:10 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 62 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 1 tasks -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:53869 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:59:10 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 138, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:10 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 138) -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:53869 in memory (size: 1858.0 B, free: 730.5 MB) -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:10 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:10 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 138). 1185 bytes result sent to driver -2016-05-23 21:59:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 138) in 17 ms on localhost (1/1) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-05-23 21:59:10 INFO DAGScheduler:59 - ResultStage 62 (first at RowMatrix.scala:65) finished in 0.018 s -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:53869 in memory (size: 1918.0 B, free: 730.5 MB) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Job 38 finished: first at RowMatrix.scala:65, took 0.183239 s -2016-05-23 21:59:10 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:53869 in memory (size: 1934.0 B, free: 730.5 MB) -2016-05-23 21:59:10 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:53869 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:59:10 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:59:10 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-23 21:59:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 168 bytes -2016-05-23 21:59:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 165 bytes -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:53869 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Got job 39 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 21:59:10 INFO DAGScheduler:59 - Final stage: ResultStage 65(first at RowMatrix.scala:65) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 64) -2016-05-23 21:59:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87), which has no missing parents -2016-05-23 21:59:10 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=384082, maxMem=766075207 -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:53869 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:59:10 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-23 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=387754, maxMem=766075207 -2016-05-23 21:59:10 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:53869 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:53869 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:59:10 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-23 21:59:10 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:10 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 65 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:59:10 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 3 tasks -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 139, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 140, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 65.0 (TID 141, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:10 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 139) -2016-05-23 21:59:10 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:53869 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:10 INFO Executor:59 - Running task 2.0 in stage 65.0 (TID 141) -2016-05-23 21:59:10 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 140) -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:10 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 139). 1296 bytes result sent to driver -2016-05-23 21:59:10 INFO Executor:59 - Finished task 2.0 in stage 65.0 (TID 141). 1296 bytes result sent to driver -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:11 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 140). 1296 bytes result sent to driver -2016-05-23 21:59:11 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 139) in 64 ms on localhost (1/3) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 65.0 (TID 141) in 63 ms on localhost (2/3) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 140) in 65 ms on localhost (3/3) -2016-05-23 21:59:11 INFO DAGScheduler:59 - ResultStage 65 (first at RowMatrix.scala:65) finished in 0.066 s -2016-05-23 21:59:11 INFO DAGScheduler:59 - Job 39 finished: first at RowMatrix.scala:65, took 0.096494 s -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-05-23 21:59:11 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:59:11 INFO DAGScheduler:59 - Got job 40 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:59:11 INFO DAGScheduler:59 - Final stage: ResultStage 68(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 67) -2016-05-23 21:59:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting ResultStage 68 (MapPartitionsRDD[78] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=376635, maxMem=766075207 -2016-05-23 21:59:11 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:53869 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(2914) called with curMem=376635, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:59:11 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:53869 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:59:11 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 68 (MapPartitionsRDD[78] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Adding task set 68.0 with 4 tasks -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 68.0 (TID 142, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 68.0 (TID 143, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 68.0 (TID 144, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 3.0 in stage 68.0 (TID 145, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:11 INFO Executor:59 - Running task 2.0 in stage 68.0 (TID 144) -2016-05-23 21:59:11 INFO Executor:59 - Running task 1.0 in stage 68.0 (TID 143) -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:11 INFO Executor:59 - Running task 0.0 in stage 68.0 (TID 142) -2016-05-23 21:59:11 INFO Executor:59 - Finished task 1.0 in stage 68.0 (TID 143). 1481 bytes result sent to driver -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:11 INFO Executor:59 - Finished task 0.0 in stage 68.0 (TID 142). 1481 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Finished task 2.0 in stage 68.0 (TID 144). 1481 bytes result sent to driver -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 68.0 (TID 143) in 49 ms on localhost (1/4) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 68.0 (TID 142) in 51 ms on localhost (2/4) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 68.0 (TID 144) in 51 ms on localhost (3/4) -2016-05-23 21:59:11 INFO Executor:59 - Running task 3.0 in stage 68.0 (TID 145) -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:11 INFO Executor:59 - Finished task 3.0 in stage 68.0 (TID 145). 1481 bytes result sent to driver -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 3.0 in stage 68.0 (TID 145) in 64 ms on localhost (4/4) -2016-05-23 21:59:11 INFO DAGScheduler:59 - ResultStage 68 (treeAggregate at RowMatrix.scala:332) finished in 0.068 s -2016-05-23 21:59:11 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at RowMatrix.scala:332, took 0.159327 s -2016-05-23 21:59:11 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 68.0, whose tasks have all completed, from pool -2016-05-23 21:59:11 INFO DAGScheduler:59 - Got job 41 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:59:11 INFO DAGScheduler:59 - Final stage: ResultStage 71(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 70) -2016-05-23 21:59:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting ResultStage 71 (MapPartitionsRDD[79] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=379549, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=385285, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 21:59:11 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:53869 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:59:11 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 71 (MapPartitionsRDD[79] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Adding task set 71.0 with 4 tasks -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 71.0 (TID 146, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 71.0 (TID 147, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 71.0 (TID 148, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 3.0 in stage 71.0 (TID 149, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:11 INFO Executor:59 - Running task 3.0 in stage 71.0 (TID 149) -2016-05-23 21:59:11 INFO Executor:59 - Running task 1.0 in stage 71.0 (TID 147) -2016-05-23 21:59:11 INFO Executor:59 - Running task 2.0 in stage 71.0 (TID 148) -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:11 INFO Executor:59 - Finished task 3.0 in stage 71.0 (TID 149). 1359 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Finished task 2.0 in stage 71.0 (TID 148). 1359 bytes result sent to driver -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 3.0 in stage 71.0 (TID 149) in 13 ms on localhost (1/4) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 71.0 (TID 148) in 14 ms on localhost (2/4) -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:11 INFO Executor:59 - Finished task 1.0 in stage 71.0 (TID 147). 1359 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Running task 0.0 in stage 71.0 (TID 146) -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:11 INFO Executor:59 - Finished task 0.0 in stage 71.0 (TID 146). 1359 bytes result sent to driver -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 71.0 (TID 147) in 39 ms on localhost (3/4) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 71.0 (TID 146) in 41 ms on localhost (4/4) -2016-05-23 21:59:11 INFO DAGScheduler:59 - ResultStage 71 (treeAggregate at RowMatrix.scala:124) finished in 0.041 s -2016-05-23 21:59:11 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at RowMatrix.scala:124, took 0.054870 s -2016-05-23 21:59:11 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:87 -2016-05-23 21:59:11 INFO DAGScheduler:59 - Got job 42 (foreach at StatisticsCorrLearning.scala:87) with 4 output partitions -2016-05-23 21:59:11 INFO DAGScheduler:59 - Final stage: ResultStage 72(foreach at StatisticsCorrLearning.scala:87) -2016-05-23 21:59:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting ResultStage 72 (ParallelCollectionRDD[80] at parallelize at StatisticsCorrLearning.scala:84), which has no missing parents -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 71.0, whose tasks have all completed, from pool -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=388181, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=389405, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 21:59:11 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:53869 (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:59:11 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 72 (ParallelCollectionRDD[80] at parallelize at StatisticsCorrLearning.scala:84) -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Adding task set 72.0 with 4 tasks -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 72.0 (TID 150, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 72.0 (TID 151, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 72.0 (TID 152, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 3.0 in stage 72.0 (TID 153, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:59:11 INFO Executor:59 - Running task 0.0 in stage 72.0 (TID 150) -2016-05-23 21:59:11 INFO Executor:59 - Running task 1.0 in stage 72.0 (TID 151) -2016-05-23 21:59:11 INFO Executor:59 - Running task 2.0 in stage 72.0 (TID 152) -2016-05-23 21:59:11 INFO Executor:59 - Finished task 0.0 in stage 72.0 (TID 150). 915 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Finished task 2.0 in stage 72.0 (TID 152). 915 bytes result sent to driver -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 72.0 (TID 150) in 8 ms on localhost (1/4) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 72.0 (TID 152) in 7 ms on localhost (2/4) -2016-05-23 21:59:11 INFO Executor:59 - Running task 3.0 in stage 72.0 (TID 153) -2016-05-23 21:59:11 INFO Executor:59 - Finished task 1.0 in stage 72.0 (TID 151). 915 bytes result sent to driver -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 72.0 (TID 151) in 14 ms on localhost (3/4) -2016-05-23 21:59:11 INFO Executor:59 - Finished task 3.0 in stage 72.0 (TID 153). 915 bytes result sent to driver -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 3.0 in stage 72.0 (TID 153) in 15 ms on localhost (4/4) -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 72.0, whose tasks have all completed, from pool -2016-05-23 21:59:11 INFO DAGScheduler:59 - ResultStage 72 (foreach at StatisticsCorrLearning.scala:87) finished in 0.037 s -2016-05-23 21:59:11 INFO DAGScheduler:59 - Job 42 finished: foreach at StatisticsCorrLearning.scala:87, took 0.056291 s -2016-05-23 21:59:11 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:89 -2016-05-23 21:59:11 INFO DAGScheduler:59 - Got job 43 (foreach at StatisticsCorrLearning.scala:89) with 4 output partitions -2016-05-23 21:59:11 INFO DAGScheduler:59 - Final stage: ResultStage 73(foreach at StatisticsCorrLearning.scala:89) -2016-05-23 21:59:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting ResultStage 73 (ParallelCollectionRDD[81] at parallelize at StatisticsCorrLearning.scala:85), which has no missing parents -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(1232) called with curMem=390298, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 1232.0 B, free 730.2 MB) -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(894) called with curMem=391530, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 894.0 B, free 730.2 MB) -2016-05-23 21:59:11 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:53869 (size: 894.0 B, free: 730.5 MB) -2016-05-23 21:59:11 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 73 (ParallelCollectionRDD[81] at parallelize at StatisticsCorrLearning.scala:85) -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Adding task set 73.0 with 4 tasks -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 73.0 (TID 154, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 73.0 (TID 155, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 73.0 (TID 156, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 3.0 in stage 73.0 (TID 157, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 21:59:11 INFO Executor:59 - Running task 0.0 in stage 73.0 (TID 154) -2016-05-23 21:59:11 INFO Executor:59 - Running task 1.0 in stage 73.0 (TID 155) -2016-05-23 21:59:11 INFO Executor:59 - Running task 2.0 in stage 73.0 (TID 156) -2016-05-23 21:59:11 INFO Executor:59 - Finished task 0.0 in stage 73.0 (TID 154). 915 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Finished task 2.0 in stage 73.0 (TID 156). 915 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Finished task 1.0 in stage 73.0 (TID 155). 915 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Running task 3.0 in stage 73.0 (TID 157) -2016-05-23 21:59:11 INFO Executor:59 - Finished task 3.0 in stage 73.0 (TID 157). 915 bytes result sent to driver -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 73.0 (TID 154) in 17 ms on localhost (1/4) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 73.0 (TID 156) in 25 ms on localhost (2/4) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 73.0 (TID 155) in 26 ms on localhost (3/4) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 3.0 in stage 73.0 (TID 157) in 26 ms on localhost (4/4) -2016-05-23 21:59:11 INFO DAGScheduler:59 - ResultStage 73 (foreach at StatisticsCorrLearning.scala:89) finished in 0.028 s -2016-05-23 21:59:11 INFO DAGScheduler:59 - Job 43 finished: foreach at StatisticsCorrLearning.scala:89, took 0.040604 s -2016-05-23 21:59:11 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:59:11 INFO DAGScheduler:59 - Got job 44 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:59:11 INFO DAGScheduler:59 - Final stage: ResultStage 74(first at RowMatrix.scala:65) -2016-05-23 21:59:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting ResultStage 74 (MapPartitionsRDD[83] at map at Correlation.scala:45), which has no missing parents -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=392424, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=394928, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 73.0, whose tasks have all completed, from pool -2016-05-23 21:59:11 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:53869 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:59:11 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 74 (MapPartitionsRDD[83] at map at Correlation.scala:45) -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Adding task set 74.0 with 1 tasks -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 74.0 (TID 158, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:11 INFO Executor:59 - Running task 0.0 in stage 74.0 (TID 158) -2016-05-23 21:59:11 INFO Executor:59 - Finished task 0.0 in stage 74.0 (TID 158). 1046 bytes result sent to driver -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 74.0 (TID 158) in 6 ms on localhost (1/1) -2016-05-23 21:59:11 INFO DAGScheduler:59 - ResultStage 74 (first at RowMatrix.scala:65) finished in 0.007 s -2016-05-23 21:59:11 INFO DAGScheduler:59 - Job 44 finished: first at RowMatrix.scala:65, took 0.045082 s -2016-05-23 21:59:11 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:59:11 INFO DAGScheduler:59 - Got job 45 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:59:11 INFO DAGScheduler:59 - Final stage: ResultStage 75(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting ResultStage 75 (MapPartitionsRDD[84] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 74.0, whose tasks have all completed, from pool -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=396452, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=399852, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.2 MB) -2016-05-23 21:59:11 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:53869 (size: 1935.0 B, free: 730.5 MB) -2016-05-23 21:59:11 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 75 (MapPartitionsRDD[84] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Adding task set 75.0 with 4 tasks -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 75.0 (TID 159, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 75.0 (TID 160, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 75.0 (TID 161, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 3.0 in stage 75.0 (TID 162, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:59:11 INFO Executor:59 - Running task 3.0 in stage 75.0 (TID 162) -2016-05-23 21:59:11 INFO Executor:59 - Running task 1.0 in stage 75.0 (TID 160) -2016-05-23 21:59:11 INFO Executor:59 - Finished task 1.0 in stage 75.0 (TID 160). 1231 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Running task 2.0 in stage 75.0 (TID 161) -2016-05-23 21:59:11 INFO Executor:59 - Finished task 3.0 in stage 75.0 (TID 162). 1231 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Finished task 2.0 in stage 75.0 (TID 161). 1231 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Running task 0.0 in stage 75.0 (TID 159) -2016-05-23 21:59:11 INFO Executor:59 - Finished task 0.0 in stage 75.0 (TID 159). 1231 bytes result sent to driver -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 75.0 (TID 160) in 15 ms on localhost (1/4) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 3.0 in stage 75.0 (TID 162) in 14 ms on localhost (2/4) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 75.0 (TID 161) in 16 ms on localhost (3/4) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 75.0 (TID 159) in 20 ms on localhost (4/4) -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 75.0, whose tasks have all completed, from pool -2016-05-23 21:59:11 INFO DAGScheduler:59 - ResultStage 75 (treeAggregate at RowMatrix.scala:332) finished in 0.024 s -2016-05-23 21:59:11 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at RowMatrix.scala:332, took 0.038941 s -2016-05-23 21:59:11 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:59:11 INFO DAGScheduler:59 - Got job 46 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:59:11 INFO DAGScheduler:59 - Final stage: ResultStage 76(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting ResultStage 76 (MapPartitionsRDD[85] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=401787, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(1919) called with curMem=405227, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 1919.0 B, free 730.2 MB) -2016-05-23 21:59:11 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:53869 (size: 1919.0 B, free: 730.5 MB) -2016-05-23 21:59:11 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 76 (MapPartitionsRDD[85] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 4 tasks -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 163, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 164, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 76.0 (TID 165, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 3.0 in stage 76.0 (TID 166, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:59:11 INFO Executor:59 - Running task 2.0 in stage 76.0 (TID 165) -2016-05-23 21:59:11 INFO Executor:59 - Finished task 2.0 in stage 76.0 (TID 165). 1109 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Running task 3.0 in stage 76.0 (TID 166) -2016-05-23 21:59:11 INFO Executor:59 - Finished task 3.0 in stage 76.0 (TID 166). 1109 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 164) -2016-05-23 21:59:11 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 164). 1109 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 163) -2016-05-23 21:59:11 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 163). 1109 bytes result sent to driver -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 76.0 (TID 165) in 20 ms on localhost (1/4) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 3.0 in stage 76.0 (TID 166) in 21 ms on localhost (2/4) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 164) in 23 ms on localhost (3/4) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 163) in 24 ms on localhost (4/4) -2016-05-23 21:59:11 INFO DAGScheduler:59 - ResultStage 76 (treeAggregate at RowMatrix.scala:124) finished in 0.025 s -2016-05-23 21:59:11 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at RowMatrix.scala:124, took 0.042228 s -2016-05-23 21:59:11 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-05-23 21:59:11 INFO DAGScheduler:59 - Got job 47 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 21:59:11 INFO DAGScheduler:59 - Final stage: ResultStage 77(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:59:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[91] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=407146, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(1862) called with curMem=410522, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 1862.0 B, free 730.2 MB) -2016-05-23 21:59:11 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:53869 (size: 1862.0 B, free: 730.5 MB) -2016-05-23 21:59:11 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 77 (MapPartitionsRDD[91] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 4 tasks -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 167, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 168, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 77.0 (TID 169, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 3.0 in stage 77.0 (TID 170, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 21:59:11 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 167) -2016-05-23 21:59:11 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 168) -2016-05-23 21:59:11 INFO Executor:59 - Running task 3.0 in stage 77.0 (TID 170) -2016-05-23 21:59:11 INFO Executor:59 - Finished task 3.0 in stage 77.0 (TID 170). 1294 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 168). 1254 bytes result sent to driver -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 3.0 in stage 77.0 (TID 170) in 7 ms on localhost (1/4) -2016-05-23 21:59:11 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 167). 1254 bytes result sent to driver -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 168) in 9 ms on localhost (2/4) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 167) in 10 ms on localhost (3/4) -2016-05-23 21:59:11 INFO Executor:59 - Running task 2.0 in stage 77.0 (TID 169) -2016-05-23 21:59:11 INFO Executor:59 - Finished task 2.0 in stage 77.0 (TID 169). 1249 bytes result sent to driver -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 77.0 (TID 169) in 14 ms on localhost (4/4) -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-05-23 21:59:11 INFO DAGScheduler:59 - ResultStage 77 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.016 s -2016-05-23 21:59:11 INFO DAGScheduler:59 - Job 47 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.043626 s -2016-05-23 21:59:11 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:59:11 INFO DAGScheduler:59 - Registering RDD 89 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:59:11 INFO DAGScheduler:59 - Got job 48 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 21:59:11 INFO DAGScheduler:59 - Final stage: ResultStage 79(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:59:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 78) -2016-05-23 21:59:11 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 78) -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting ShuffleMapStage 78 (MapPartitionsRDD[89] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=412384, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(2289) called with curMem=416424, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 21:59:11 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:53869 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:59:11 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 78 (MapPartitionsRDD[89] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 4 tasks -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 171, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 172, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 78.0 (TID 173, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 3.0 in stage 78.0 (TID 174, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 21:59:11 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 172) -2016-05-23 21:59:11 INFO Executor:59 - Running task 2.0 in stage 78.0 (TID 173) -2016-05-23 21:59:11 INFO Executor:59 - Running task 3.0 in stage 78.0 (TID 174) -2016-05-23 21:59:11 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 171) -2016-05-23 21:59:11 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 172). 1161 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Finished task 3.0 in stage 78.0 (TID 174). 1161 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 171). 1161 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Finished task 2.0 in stage 78.0 (TID 173). 1161 bytes result sent to driver -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 172) in 94 ms on localhost (1/4) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 3.0 in stage 78.0 (TID 174) in 94 ms on localhost (2/4) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 171) in 96 ms on localhost (3/4) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 78.0 (TID 173) in 95 ms on localhost (4/4) -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-05-23 21:59:11 INFO DAGScheduler:59 - ShuffleMapStage 78 (flatMap at SpearmanCorrelation.scala:49) finished in 0.102 s -2016-05-23 21:59:11 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:59:11 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:59:11 INFO DAGScheduler:59 - waiting: Set(ResultStage 79) -2016-05-23 21:59:11 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:59:11 INFO DAGScheduler:59 - Missing parents for ResultStage 79: List() -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting ResultStage 79 (ShuffledRDD[92] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=418713, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(1573) called with curMem=421257, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 1573.0 B, free 730.2 MB) -2016-05-23 21:59:11 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:53869 (size: 1573.0 B, free: 730.5 MB) -2016-05-23 21:59:11 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 79 (ShuffledRDD[92] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Adding task set 79.0 with 3 tasks -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 79.0 (TID 175, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 79.0 (TID 176, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 79.0 (TID 177, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:11 INFO Executor:59 - Running task 0.0 in stage 79.0 (TID 175) -2016-05-23 21:59:11 INFO Executor:59 - Running task 1.0 in stage 79.0 (TID 176) -2016-05-23 21:59:11 INFO Executor:59 - Running task 2.0 in stage 79.0 (TID 177) -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:11 INFO Executor:59 - Finished task 1.0 in stage 79.0 (TID 176). 1203 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Finished task 0.0 in stage 79.0 (TID 175). 1203 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Finished task 2.0 in stage 79.0 (TID 177). 1203 bytes result sent to driver -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 79.0 (TID 176) in 27 ms on localhost (1/3) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 79.0 (TID 175) in 29 ms on localhost (2/3) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 79.0 (TID 177) in 27 ms on localhost (3/3) -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 79.0, whose tasks have all completed, from pool -2016-05-23 21:59:11 INFO DAGScheduler:59 - ResultStage 79 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.034 s -2016-05-23 21:59:11 INFO DAGScheduler:59 - Job 48 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.162441 s -2016-05-23 21:59:11 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:59:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 166 bytes -2016-05-23 21:59:11 INFO DAGScheduler:59 - Registering RDD 94 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:59:11 INFO DAGScheduler:59 - Got job 49 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:59:11 INFO DAGScheduler:59 - Final stage: ResultStage 82(first at RowMatrix.scala:65) -2016-05-23 21:59:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 81) -2016-05-23 21:59:11 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 81) -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting ShuffleMapStage 81 (MapPartitionsRDD[94] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=422830, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 21:59:11 INFO MemoryStore:59 - ensureFreeSpace(2313) called with curMem=427062, maxMem=766075207 -2016-05-23 21:59:11 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 21:59:11 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:53869 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:59:11 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:11 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 81 (MapPartitionsRDD[94] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:59:11 INFO TaskSchedulerImpl:59 - Adding task set 81.0 with 4 tasks -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 81.0 (TID 178, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 81.0 (TID 179, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 81.0 (TID 180, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:59:11 INFO TaskSetManager:59 - Starting task 3.0 in stage 81.0 (TID 181, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:59:11 INFO Executor:59 - Running task 1.0 in stage 81.0 (TID 179) -2016-05-23 21:59:11 INFO Executor:59 - Running task 3.0 in stage 81.0 (TID 181) -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 21:59:11 INFO Executor:59 - Running task 0.0 in stage 81.0 (TID 178) -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:11 INFO Executor:59 - Finished task 0.0 in stage 81.0 (TID 178). 1377 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Finished task 3.0 in stage 81.0 (TID 181). 1377 bytes result sent to driver -2016-05-23 21:59:11 INFO Executor:59 - Finished task 1.0 in stage 81.0 (TID 179). 1377 bytes result sent to driver -2016-05-23 21:59:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 81.0 (TID 178) in 86 ms on localhost (1/4) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 3.0 in stage 81.0 (TID 181) in 85 ms on localhost (2/4) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 81.0 (TID 179) in 87 ms on localhost (3/4) -2016-05-23 21:59:12 INFO Executor:59 - Running task 2.0 in stage 81.0 (TID 180) -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:12 INFO Executor:59 - Finished task 2.0 in stage 81.0 (TID 180). 1377 bytes result sent to driver -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 81.0 (TID 180) in 144 ms on localhost (4/4) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 81.0, whose tasks have all completed, from pool -2016-05-23 21:59:12 INFO DAGScheduler:59 - ShuffleMapStage 81 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.150 s -2016-05-23 21:59:12 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:59:12 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:59:12 INFO DAGScheduler:59 - waiting: Set(ResultStage 82) -2016-05-23 21:59:12 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:59:12 INFO DAGScheduler:59 - Missing parents for ResultStage 82: List() -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting ResultStage 82 (MapPartitionsRDD[96] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=429375, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(2562) called with curMem=434271, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:53869 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO SparkContext:59 - Created broadcast 61 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 82 (MapPartitionsRDD[96] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Adding task set 82.0 with 1 tasks -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 82.0 (TID 182, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:12 INFO Executor:59 - Running task 0.0 in stage 82.0 (TID 182) -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:12 INFO Executor:59 - Finished task 0.0 in stage 82.0 (TID 182). 1296 bytes result sent to driver -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 82.0 (TID 182) in 12 ms on localhost (1/1) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 82.0, whose tasks have all completed, from pool -2016-05-23 21:59:12 INFO DAGScheduler:59 - ResultStage 82 (first at RowMatrix.scala:65) finished in 0.015 s -2016-05-23 21:59:12 INFO DAGScheduler:59 - Job 49 finished: first at RowMatrix.scala:65, took 0.212866 s -2016-05-23 21:59:12 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:59:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 166 bytes -2016-05-23 21:59:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 169 bytes -2016-05-23 21:59:12 INFO DAGScheduler:59 - Got job 50 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 21:59:12 INFO DAGScheduler:59 - Final stage: ResultStage 85(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 84) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting ResultStage 85 (MapPartitionsRDD[97] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=436833, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(2943) called with curMem=442529, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.2 MB) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:53869 (size: 2.9 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 85 (MapPartitionsRDD[97] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Adding task set 85.0 with 4 tasks -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 85.0 (TID 183, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 85.0 (TID 184, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 85.0 (TID 185, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 3.0 in stage 85.0 (TID 186, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:12 INFO Executor:59 - Running task 2.0 in stage 85.0 (TID 185) -2016-05-23 21:59:12 INFO Executor:59 - Running task 1.0 in stage 85.0 (TID 184) -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:12 INFO Executor:59 - Running task 0.0 in stage 85.0 (TID 183) -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:12 INFO Executor:59 - Finished task 0.0 in stage 85.0 (TID 183). 1481 bytes result sent to driver -2016-05-23 21:59:12 INFO Executor:59 - Running task 3.0 in stage 85.0 (TID 186) -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:12 INFO Executor:59 - Finished task 2.0 in stage 85.0 (TID 185). 1481 bytes result sent to driver -2016-05-23 21:59:12 INFO Executor:59 - Finished task 3.0 in stage 85.0 (TID 186). 1481 bytes result sent to driver -2016-05-23 21:59:12 INFO Executor:59 - Finished task 1.0 in stage 85.0 (TID 184). 1481 bytes result sent to driver -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 85.0 (TID 183) in 43 ms on localhost (1/4) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 85.0 (TID 185) in 41 ms on localhost (2/4) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 3.0 in stage 85.0 (TID 186) in 41 ms on localhost (3/4) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 85.0 (TID 184) in 44 ms on localhost (4/4) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 85.0, whose tasks have all completed, from pool -2016-05-23 21:59:12 INFO DAGScheduler:59 - ResultStage 85 (treeAggregate at RowMatrix.scala:332) finished in 0.047 s -2016-05-23 21:59:12 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at RowMatrix.scala:332, took 0.073747 s -2016-05-23 21:59:12 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Got job 51 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 21:59:12 INFO DAGScheduler:59 - Final stage: ResultStage 88(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 87) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting ResultStage 88 (MapPartitionsRDD[98] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=445472, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(2920) called with curMem=451208, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.2 MB) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:53869 (size: 2.9 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 88 (MapPartitionsRDD[98] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Adding task set 88.0 with 4 tasks -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 88.0 (TID 187, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 88.0 (TID 188, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 88.0 (TID 189, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 3.0 in stage 88.0 (TID 190, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:12 INFO Executor:59 - Running task 0.0 in stage 88.0 (TID 187) -2016-05-23 21:59:12 INFO Executor:59 - Running task 2.0 in stage 88.0 (TID 189) -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 3 ms -2016-05-23 21:59:12 INFO Executor:59 - Running task 1.0 in stage 88.0 (TID 188) -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:12 INFO Executor:59 - Finished task 0.0 in stage 88.0 (TID 187). 1359 bytes result sent to driver -2016-05-23 21:59:12 INFO Executor:59 - Finished task 2.0 in stage 88.0 (TID 189). 1359 bytes result sent to driver -2016-05-23 21:59:12 INFO Executor:59 - Finished task 1.0 in stage 88.0 (TID 188). 1359 bytes result sent to driver -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 88.0 (TID 187) in 26 ms on localhost (1/4) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 88.0 (TID 189) in 26 ms on localhost (2/4) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 88.0 (TID 188) in 28 ms on localhost (3/4) -2016-05-23 21:59:12 INFO Executor:59 - Running task 3.0 in stage 88.0 (TID 190) -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:12 INFO Executor:59 - Finished task 3.0 in stage 88.0 (TID 190). 1359 bytes result sent to driver -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 3.0 in stage 88.0 (TID 190) in 41 ms on localhost (4/4) -2016-05-23 21:59:12 INFO DAGScheduler:59 - ResultStage 88 (treeAggregate at RowMatrix.scala:124) finished in 0.043 s -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 88.0, whose tasks have all completed, from pool -2016-05-23 21:59:12 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at RowMatrix.scala:124, took 0.063812 s -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(88528) called with curMem=454128, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 86.5 KB, free 730.1 MB) -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=542656, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.0 MB) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:53869 (size: 19.3 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO SparkContext:59 - Created broadcast 64 from textFile at StatisticsCorrLearning.scala:98 -2016-05-23 21:59:12 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 21:59:12 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:102 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Got job 52 (foreach at StatisticsCorrLearning.scala:102) with 2 output partitions -2016-05-23 21:59:12 INFO DAGScheduler:59 - Final stage: ResultStage 89(foreach at StatisticsCorrLearning.scala:102) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting ResultStage 89 (MapPartitionsRDD[102] at map at StatisticsCorrLearning.scala:101), which has no missing parents -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=562444, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(1982) called with curMem=565956, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 1982.0 B, free 730.0 MB) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:53869 (size: 1982.0 B, free: 730.5 MB) -2016-05-23 21:59:12 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 89 (MapPartitionsRDD[102] at map at StatisticsCorrLearning.scala:101) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Adding task set 89.0 with 2 tasks -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 89.0 (TID 191, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 89.0 (TID 192, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:59:12 INFO Executor:59 - Running task 1.0 in stage 89.0 (TID 192) -2016-05-23 21:59:12 INFO Executor:59 - Running task 0.0 in stage 89.0 (TID 191) -2016-05-23 21:59:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:59:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:59:12 INFO Executor:59 - Finished task 0.0 in stage 89.0 (TID 191). 2044 bytes result sent to driver -2016-05-23 21:59:12 INFO Executor:59 - Finished task 1.0 in stage 89.0 (TID 192). 2044 bytes result sent to driver -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 89.0 (TID 191) in 17 ms on localhost (1/2) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 89.0 (TID 192) in 18 ms on localhost (2/2) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 89.0, whose tasks have all completed, from pool -2016-05-23 21:59:12 INFO DAGScheduler:59 - ResultStage 89 (foreach at StatisticsCorrLearning.scala:102) finished in 0.020 s -2016-05-23 21:59:12 INFO DAGScheduler:59 - Job 52 finished: foreach at StatisticsCorrLearning.scala:102, took 0.032000 s -2016-05-23 21:59:12 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Got job 53 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:59:12 INFO DAGScheduler:59 - Final stage: ResultStage 90(first at RowMatrix.scala:65) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting ResultStage 90 (MapPartitionsRDD[102] at map at StatisticsCorrLearning.scala:101), which has no missing parents -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(3552) called with curMem=567938, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(2003) called with curMem=571490, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2003.0 B, free 730.0 MB) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:53869 (size: 2003.0 B, free: 730.5 MB) -2016-05-23 21:59:12 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 90 (MapPartitionsRDD[102] at map at StatisticsCorrLearning.scala:101) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Adding task set 90.0 with 1 tasks -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 90.0 (TID 193, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:59:12 INFO Executor:59 - Running task 0.0 in stage 90.0 (TID 193) -2016-05-23 21:59:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:59:12 INFO Executor:59 - Finished task 0.0 in stage 90.0 (TID 193). 2199 bytes result sent to driver -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 90.0 (TID 193) in 17 ms on localhost (1/1) -2016-05-23 21:59:12 INFO DAGScheduler:59 - ResultStage 90 (first at RowMatrix.scala:65) finished in 0.017 s -2016-05-23 21:59:12 INFO DAGScheduler:59 - Job 53 finished: first at RowMatrix.scala:65, took 0.030865 s -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 90.0, whose tasks have all completed, from pool -2016-05-23 21:59:12 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Got job 54 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:59:12 INFO DAGScheduler:59 - Final stage: ResultStage 91(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting ResultStage 91 (MapPartitionsRDD[103] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(4472) called with curMem=573493, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 4.4 KB, free 730.0 MB) -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(2372) called with curMem=577965, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.0 MB) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:53869 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO SparkContext:59 - Created broadcast 67 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 91 (MapPartitionsRDD[103] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Adding task set 91.0 with 2 tasks -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 91.0 (TID 194, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 91.0 (TID 195, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:59:12 INFO Executor:59 - Running task 0.0 in stage 91.0 (TID 194) -2016-05-23 21:59:12 INFO Executor:59 - Running task 1.0 in stage 91.0 (TID 195) -2016-05-23 21:59:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:59:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:59:12 INFO Executor:59 - Finished task 0.0 in stage 91.0 (TID 194). 2384 bytes result sent to driver -2016-05-23 21:59:12 INFO Executor:59 - Finished task 1.0 in stage 91.0 (TID 195). 2384 bytes result sent to driver -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 91.0 (TID 194) in 13 ms on localhost (1/2) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 91.0 (TID 195) in 13 ms on localhost (2/2) -2016-05-23 21:59:12 INFO DAGScheduler:59 - ResultStage 91 (treeAggregate at RowMatrix.scala:332) finished in 0.014 s -2016-05-23 21:59:12 INFO DAGScheduler:59 - Job 54 finished: treeAggregate at RowMatrix.scala:332, took 0.031176 s -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 91.0, whose tasks have all completed, from pool -2016-05-23 21:59:12 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Got job 55 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:59:12 INFO DAGScheduler:59 - Final stage: ResultStage 92(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting ResultStage 92 (MapPartitionsRDD[104] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(4584) called with curMem=580337, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 4.5 KB, free 730.0 MB) -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(2364) called with curMem=584921, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.0 MB) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:53869 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 92 (MapPartitionsRDD[104] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Adding task set 92.0 with 2 tasks -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 92.0 (TID 196, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 92.0 (TID 197, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:59:12 INFO Executor:59 - Running task 0.0 in stage 92.0 (TID 196) -2016-05-23 21:59:12 INFO Executor:59 - Running task 1.0 in stage 92.0 (TID 197) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:53869 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-05-23 21:59:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:59:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:53869 in memory (size: 2003.0 B, free: 730.5 MB) -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-05-23 21:59:12 INFO Executor:59 - Finished task 1.0 in stage 92.0 (TID 197). 2337 bytes result sent to driver -2016-05-23 21:59:12 INFO Executor:59 - Finished task 0.0 in stage 92.0 (TID 196). 2337 bytes result sent to driver -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 92.0 (TID 197) in 15 ms on localhost (1/2) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 92.0 (TID 196) in 21 ms on localhost (2/2) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 92.0, whose tasks have all completed, from pool -2016-05-23 21:59:12 INFO DAGScheduler:59 - ResultStage 92 (treeAggregate at RowMatrix.scala:124) finished in 0.022 s -2016-05-23 21:59:12 INFO DAGScheduler:59 - Job 55 finished: treeAggregate at RowMatrix.scala:124, took 0.035990 s -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:53869 in memory (size: 1982.0 B, free: 730.5 MB) -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:53869 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:53869 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:53869 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:53869 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned shuffle 9 -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:53869 in memory (size: 1573.0 B, free: 730.5 MB) -2016-05-23 21:59:12 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Got job 56 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 21:59:12 INFO DAGScheduler:59 - Final stage: ResultStage 93(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 21:59:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting ResultStage 93 (MapPartitionsRDD[108] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:53869 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(4432) called with curMem=527648, maxMem=766075207 -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 4.3 KB, free 730.1 MB) -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned shuffle 8 -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(2326) called with curMem=532080, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:53869 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:53869 in memory (size: 1862.0 B, free: 730.5 MB) -2016-05-23 21:59:12 INFO SparkContext:59 - Created broadcast 69 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 93 (MapPartitionsRDD[108] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Adding task set 93.0 with 2 tasks -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 93.0 (TID 198, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 93.0 (TID 199, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 21:59:12 INFO Executor:59 - Running task 0.0 in stage 93.0 (TID 198) -2016-05-23 21:59:12 INFO Executor:59 - Running task 1.0 in stage 93.0 (TID 199) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:53869 in memory (size: 1919.0 B, free: 730.5 MB) -2016-05-23 21:59:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:59:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-05-23 21:59:12 INFO Executor:59 - Finished task 1.0 in stage 93.0 (TID 199). 2443 bytes result sent to driver -2016-05-23 21:59:12 INFO Executor:59 - Finished task 0.0 in stage 93.0 (TID 198). 2443 bytes result sent to driver -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 93.0 (TID 199) in 13 ms on localhost (1/2) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:53869 in memory (size: 1935.0 B, free: 730.5 MB) -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 93.0 (TID 198) in 16 ms on localhost (2/2) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 93.0, whose tasks have all completed, from pool -2016-05-23 21:59:12 INFO DAGScheduler:59 - ResultStage 93 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.018 s -2016-05-23 21:59:12 INFO DAGScheduler:59 - Job 56 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.030088 s -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:53869 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:53869 in memory (size: 894.0 B, free: 730.5 MB) -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-05-23 21:59:12 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Registering RDD 106 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Got job 57 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 21:59:12 INFO DAGScheduler:59 - Final stage: ResultStage 95(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 94) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 94) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting ShuffleMapStage 94 (MapPartitionsRDD[106] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:53869 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(5024) called with curMem=510203, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 4.9 KB, free 730.1 MB) -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(2736) called with curMem=506595, maxMem=766075207 -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:53869 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.1 MB) -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:53869 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 94 (MapPartitionsRDD[106] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:53869 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Adding task set 94.0 with 2 tasks -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 94.0 (TID 200, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 94.0 (TID 201, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-23 21:59:12 INFO Executor:59 - Running task 0.0 in stage 94.0 (TID 200) -2016-05-23 21:59:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 21:59:12 INFO Executor:59 - Running task 1.0 in stage 94.0 (TID 201) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:53869 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 21:59:12 INFO Executor:59 - Finished task 0.0 in stage 94.0 (TID 200). 2254 bytes result sent to driver -2016-05-23 21:59:12 INFO Executor:59 - Finished task 1.0 in stage 94.0 (TID 201). 2254 bytes result sent to driver -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 94.0 (TID 200) in 29 ms on localhost (1/2) -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 94.0 (TID 201) in 30 ms on localhost (2/2) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 94.0, whose tasks have all completed, from pool -2016-05-23 21:59:12 INFO DAGScheduler:59 - ShuffleMapStage 94 (flatMap at SpearmanCorrelation.scala:49) finished in 0.033 s -2016-05-23 21:59:12 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:59:12 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:59:12 INFO DAGScheduler:59 - waiting: Set(ResultStage 95) -2016-05-23 21:59:12 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:59:12 INFO DAGScheduler:59 - Missing parents for ResultStage 95: List() -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting ResultStage 95 (ShuffledRDD[109] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:53869 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=485841, maxMem=766075207 -2016-05-23 21:59:12 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(1551) called with curMem=484113, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 1551.0 B, free 730.1 MB) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:53869 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:53869 (size: 1551.0 B, free: 730.5 MB) -2016-05-23 21:59:12 INFO SparkContext:59 - Created broadcast 71 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 95 (ShuffledRDD[109] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Adding task set 95.0 with 1 tasks -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 95.0 (TID 202, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:12 INFO Executor:59 - Running task 0.0 in stage 95.0 (TID 202) -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:12 INFO Executor:59 - Finished task 0.0 in stage 95.0 (TID 202). 1203 bytes result sent to driver -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 95.0 (TID 202) in 8 ms on localhost (1/1) -2016-05-23 21:59:12 INFO DAGScheduler:59 - ResultStage 95 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.008 s -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 95.0, whose tasks have all completed, from pool -2016-05-23 21:59:12 INFO DAGScheduler:59 - Job 57 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.065209 s -2016-05-23 21:59:12 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 21:59:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 152 bytes -2016-05-23 21:59:12 INFO DAGScheduler:59 - Registering RDD 111 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Got job 58 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 21:59:12 INFO DAGScheduler:59 - Final stage: ResultStage 98(first at RowMatrix.scala:65) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 97) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 97) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting ShuffleMapStage 97 (MapPartitionsRDD[111] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=483359, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(2287) called with curMem=487551, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:53869 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 97 (MapPartitionsRDD[111] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Adding task set 97.0 with 2 tasks -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 97.0 (TID 203, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 97.0 (TID 204, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 21:59:12 INFO Executor:59 - Running task 1.0 in stage 97.0 (TID 204) -2016-05-23 21:59:12 INFO Executor:59 - Running task 0.0 in stage 97.0 (TID 203) -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:12 INFO Executor:59 - Finished task 1.0 in stage 97.0 (TID 204). 1375 bytes result sent to driver -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 97.0 (TID 204) in 24 ms on localhost (1/2) -2016-05-23 21:59:12 INFO Executor:59 - Finished task 0.0 in stage 97.0 (TID 203). 1375 bytes result sent to driver -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 97.0 (TID 203) in 33 ms on localhost (2/2) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 97.0, whose tasks have all completed, from pool -2016-05-23 21:59:12 INFO DAGScheduler:59 - ShuffleMapStage 97 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.033 s -2016-05-23 21:59:12 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 21:59:12 INFO DAGScheduler:59 - running: Set() -2016-05-23 21:59:12 INFO DAGScheduler:59 - waiting: Set(ResultStage 98) -2016-05-23 21:59:12 INFO DAGScheduler:59 - failed: Set() -2016-05-23 21:59:12 INFO DAGScheduler:59 - Missing parents for ResultStage 98: List() -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting ResultStage 98 (MapPartitionsRDD[113] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=489838, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 4.7 KB, free 730.1 MB) -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=494694, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:53869 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO SparkContext:59 - Created broadcast 73 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 98 (MapPartitionsRDD[113] at map at SpearmanCorrelation.scala:87) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Adding task set 98.0 with 1 tasks -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 98.0 (TID 205, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:12 INFO Executor:59 - Running task 0.0 in stage 98.0 (TID 205) -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:12 INFO Executor:59 - Finished task 0.0 in stage 98.0 (TID 205). 1320 bytes result sent to driver -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 98.0 (TID 205) in 12 ms on localhost (1/1) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 98.0, whose tasks have all completed, from pool -2016-05-23 21:59:12 INFO DAGScheduler:59 - ResultStage 98 (first at RowMatrix.scala:65) finished in 0.013 s -2016-05-23 21:59:12 INFO DAGScheduler:59 - Job 58 finished: first at RowMatrix.scala:65, took 0.067473 s -2016-05-23 21:59:12 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 21:59:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 152 bytes -2016-05-23 21:59:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 152 bytes -2016-05-23 21:59:12 INFO DAGScheduler:59 - Got job 59 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 21:59:12 INFO DAGScheduler:59 - Final stage: ResultStage 101(treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 100) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting ResultStage 101 (MapPartitionsRDD[114] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(5680) called with curMem=497225, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(2901) called with curMem=502905, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:53869 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 101 (MapPartitionsRDD[114] at treeAggregate at RowMatrix.scala:332) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Adding task set 101.0 with 2 tasks -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 101.0 (TID 206, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 101.0 (TID 207, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:12 INFO Executor:59 - Running task 0.0 in stage 101.0 (TID 206) -2016-05-23 21:59:12 INFO Executor:59 - Running task 1.0 in stage 101.0 (TID 207) -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:12 INFO Executor:59 - Finished task 1.0 in stage 101.0 (TID 207). 1505 bytes result sent to driver -2016-05-23 21:59:12 INFO Executor:59 - Finished task 0.0 in stage 101.0 (TID 206). 1505 bytes result sent to driver -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 101.0 (TID 206) in 10 ms on localhost (1/2) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 101.0 (TID 207) in 10 ms on localhost (2/2) -2016-05-23 21:59:12 INFO DAGScheduler:59 - ResultStage 101 (treeAggregate at RowMatrix.scala:332) finished in 0.011 s -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 101.0, whose tasks have all completed, from pool -2016-05-23 21:59:12 INFO DAGScheduler:59 - Job 59 finished: treeAggregate at RowMatrix.scala:332, took 0.022397 s -2016-05-23 21:59:12 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Got job 60 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 21:59:12 INFO DAGScheduler:59 - Final stage: ResultStage 104(treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 103) -2016-05-23 21:59:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting ResultStage 104 (MapPartitionsRDD[115] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(5792) called with curMem=505806, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-05-23 21:59:12 INFO MemoryStore:59 - ensureFreeSpace(2886) called with curMem=511598, maxMem=766075207 -2016-05-23 21:59:12 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 21:59:12 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:53869 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 21:59:12 INFO SparkContext:59 - Created broadcast 75 from broadcast at DAGScheduler.scala:861 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 104 (MapPartitionsRDD[115] at treeAggregate at RowMatrix.scala:124) -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Adding task set 104.0 with 2 tasks -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 104.0 (TID 208, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 104.0 (TID 209, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 21:59:12 INFO Executor:59 - Running task 0.0 in stage 104.0 (TID 208) -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 21:59:12 INFO Executor:59 - Running task 1.0 in stage 104.0 (TID 209) -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 21:59:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 21:59:12 INFO Executor:59 - Finished task 0.0 in stage 104.0 (TID 208). 1458 bytes result sent to driver -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 104.0 (TID 208) in 10 ms on localhost (1/2) -2016-05-23 21:59:12 INFO Executor:59 - Finished task 1.0 in stage 104.0 (TID 209). 1458 bytes result sent to driver -2016-05-23 21:59:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 104.0 (TID 209) in 14 ms on localhost (2/2) -2016-05-23 21:59:12 INFO DAGScheduler:59 - ResultStage 104 (treeAggregate at RowMatrix.scala:124) finished in 0.015 s -2016-05-23 21:59:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 104.0, whose tasks have all completed, from pool -2016-05-23 21:59:12 INFO DAGScheduler:59 - Job 60 finished: treeAggregate at RowMatrix.scala:124, took 0.024062 s -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 21:59:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 21:59:12 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 21:59:12 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 21:59:13 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 21:59:13 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 21:59:13 INFO BlockManager:59 - BlockManager stopped -2016-05-23 21:59:13 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 21:59:13 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 21:59:13 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 21:59:13 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 21:59:13 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-7c9e68f3-b1a1-40e9-bf3c-209981a07ae1 -2016-05-23 22:02:22 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 22:02:22 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 22:02:23 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 22:02:23 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 22:02:23 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 22:02:24 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 22:02:25 INFO Remoting:74 - Starting remoting -2016-05-23 22:02:25 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53903] -2016-05-23 22:02:25 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53903. -2016-05-23 22:02:25 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 22:02:25 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 22:02:25 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-791e65cc-a41a-4106-9164-f8a6d552fcfd -2016-05-23 22:02:25 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 22:02:25 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-1bae0fac-a9bf-4fba-8ac2-21cd34cfe122\httpd-0e125e82-db9b-4997-9db5-4a4258eda575 -2016-05-23 22:02:25 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 22:02:26 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:02:26 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53904 -2016-05-23 22:02:26 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53904. -2016-05-23 22:02:26 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 22:02:26 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:02:26 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 22:02:26 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 22:02:26 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 22:02:26 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 22:02:26 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 22:02:26 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53923. -2016-05-23 22:02:26 INFO NettyBlockTransferService:59 - Server created on 53923 -2016-05-23 22:02:26 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 22:02:26 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53923 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53923) -2016-05-23 22:02:26 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 22:02:27 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 22:02:27 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 22:02:27 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 22:02:27 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 22:02:27 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53923 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 22:02:27 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 22:02:27 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 22:02:27 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 22:02:27 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 22:02:27 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 22:02:27 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53923 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 22:02:27 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:26 -2016-05-23 22:02:29 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 22:02:29 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 22:02:29 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:30 -2016-05-23 22:02:29 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:30) with 2 output partitions -2016-05-23 22:02:29 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:30) -2016-05-23 22:02:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:29 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:24), which has no missing parents -2016-05-23 22:02:29 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 22:02:29 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 22:02:29 INFO MemoryStore:59 - ensureFreeSpace(2047) called with curMem=346712, maxMem=766075207 -2016-05-23 22:02:29 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2047.0 B, free 730.3 MB) -2016-05-23 22:02:29 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53923 (size: 2047.0 B, free: 730.6 MB) -2016-05-23 22:02:29 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:24) -2016-05-23 22:02:29 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 22:02:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:02:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:02:29 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 22:02:29 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 22:02:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 22:02:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 22:02:29 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 22:02:29 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 22:02:29 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 22:02:29 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 22:02:29 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 22:02:29 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 22:02:29 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 22:02:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 180 ms on localhost (1/2) -2016-05-23 22:02:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 157 ms on localhost (2/2) -2016-05-23 22:02:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 22:02:29 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:30) finished in 0.222 s -2016-05-23 22:02:29 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:30, took 0.430082 s -2016-05-23 22:02:29 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 22:02:29 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:32 -2016-05-23 22:02:29 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:32) with 2 output partitions -2016-05-23 22:02:29 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:32) -2016-05-23 22:02:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:29 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:27), which has no missing parents -2016-05-23 22:02:29 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348759, maxMem=766075207 -2016-05-23 22:02:29 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 22:02:29 INFO MemoryStore:59 - ensureFreeSpace(2045) called with curMem=352271, maxMem=766075207 -2016-05-23 22:02:29 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2045.0 B, free 730.2 MB) -2016-05-23 22:02:29 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53923 (size: 2045.0 B, free: 730.6 MB) -2016-05-23 22:02:29 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:27) -2016-05-23 22:02:29 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 22:02:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:02:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:02:29 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 22:02:29 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 22:02:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 22:02:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 22:02:29 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 22:02:29 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 22:02:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 27 ms on localhost (1/2) -2016-05-23 22:02:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 43 ms on localhost (2/2) -2016-05-23 22:02:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 22:02:29 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:32) finished in 0.044 s -2016-05-23 22:02:29 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:32, took 0.077044 s -2016-05-23 22:02:30 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:02:30 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:02:30 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 22:02:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:30 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 22:02:30 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354316, maxMem=766075207 -2016-05-23 22:02:30 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 22:02:30 INFO MemoryStore:59 - ensureFreeSpace(2464) called with curMem=358876, maxMem=766075207 -2016-05-23 22:02:30 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 22:02:30 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53923 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 22:02:30 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 22:02:30 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 22:02:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 22:02:30 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 22:02:30 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53923 in memory (size: 2045.0 B, free: 730.6 MB) -2016-05-23 22:02:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 22:02:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 22:02:30 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 22:02:30 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53923 in memory (size: 2047.0 B, free: 730.6 MB) -2016-05-23 22:02:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 48 ms on localhost (1/1) -2016-05-23 22:02:30 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.045 s -2016-05-23 22:02:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 22:02:30 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.470214 s -2016-05-23 22:02:31 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:02:31 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 22:02:31 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:31 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:02:31 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=350224, maxMem=766075207 -2016-05-23 22:02:31 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 22:02:31 INFO MemoryStore:59 - ensureFreeSpace(2861) called with curMem=355672, maxMem=766075207 -2016-05-23 22:02:31 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 22:02:31 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53923 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 22:02:31 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:31 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 22:02:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 22:02:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 22:02:31 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 22:02:31 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 22:02:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 22:02:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 22:02:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 22:02:31 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 22:02:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 22:02:31 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 22:02:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 72 ms on localhost (1/2) -2016-05-23 22:02:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 74 ms on localhost (2/2) -2016-05-23 22:02:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 22:02:31 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.088 s -2016-05-23 22:02:31 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 22:02:31 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 22:02:31 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.131116 s -2016-05-23 22:02:31 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:02:31 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 22:02:31 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:02:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:31 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:02:31 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=358533, maxMem=766075207 -2016-05-23 22:02:31 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 22:02:31 INFO MemoryStore:59 - ensureFreeSpace(2850) called with curMem=364029, maxMem=766075207 -2016-05-23 22:02:31 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 22:02:31 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53923 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 22:02:31 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:02:31 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 22:02:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 22:02:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 22:02:31 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 22:02:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 22:02:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 22:02:31 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 22:02:31 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 22:02:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 22:02:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 22:02:31 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 22:02:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 95 ms on localhost (1/2) -2016-05-23 22:02:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 290 ms on localhost (2/2) -2016-05-23 22:02:31 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.302 s -2016-05-23 22:02:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 22:02:31 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.390209 s -2016-05-23 22:02:32 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 22:02:32 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 22:02:32 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:02:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:32 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:32 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 22:02:32 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=366879, maxMem=766075207 -2016-05-23 22:02:32 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 22:02:32 INFO MemoryStore:59 - ensureFreeSpace(2743) called with curMem=372223, maxMem=766075207 -2016-05-23 22:02:32 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 22:02:32 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53923 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 22:02:32 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:02:32 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 22:02:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 22:02:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 22:02:32 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 22:02:32 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 22:02:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 22:02:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 22:02:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 22:02:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 22:02:32 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 22:02:32 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 22:02:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 59 ms on localhost (1/2) -2016-05-23 22:02:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 74 ms on localhost (2/2) -2016-05-23 22:02:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 22:02:32 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.076 s -2016-05-23 22:02:32 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.105305 s -2016-05-23 22:02:32 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 22:02:32 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:02:32 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 22:02:32 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 22:02:32 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 22:02:32 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 22:02:32 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 22:02:32 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=374966, maxMem=766075207 -2016-05-23 22:02:32 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 22:02:32 INFO MemoryStore:59 - ensureFreeSpace(3155) called with curMem=380894, maxMem=766075207 -2016-05-23 22:02:32 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 22:02:32 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53923 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 22:02:32 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:02:32 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 22:02:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 22:02:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 22:02:32 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 22:02:32 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 22:02:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 22:02:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 22:02:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 22:02:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 22:02:32 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 22:02:32 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 22:02:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 111 ms on localhost (1/2) -2016-05-23 22:02:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 109 ms on localhost (2/2) -2016-05-23 22:02:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 22:02:32 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.113 s -2016-05-23 22:02:32 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:02:32 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:02:32 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 22:02:32 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:02:32 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 22:02:32 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 22:02:32 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=384049, maxMem=766075207 -2016-05-23 22:02:32 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 22:02:32 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=386553, maxMem=766075207 -2016-05-23 22:02:32 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 22:02:32 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53923 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 22:02:32 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:02:32 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 22:02:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:32 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 22:02:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:02:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-23 22:02:32 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 22:02:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 240 ms on localhost (1/1) -2016-05-23 22:02:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 22:02:32 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.246 s -2016-05-23 22:02:32 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.436389 s -2016-05-23 22:02:32 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:02:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 22:02:32 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:02:32 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:02:32 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 22:02:32 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 22:02:32 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 22:02:32 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 22:02:32 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=388098, maxMem=766075207 -2016-05-23 22:02:32 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 22:02:32 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=389786, maxMem=766075207 -2016-05-23 22:02:32 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 22:02:32 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53923 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 22:02:32 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53923 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 22:02:32 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:32 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 22:02:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:02:32 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 22:02:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:02:32 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53923 in memory (size: 3.1 KB, free: 730.5 MB) -2016-05-23 22:02:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:02:32 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 22:02:32 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 22:02:32 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 22:02:32 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53923 in memory (size: 2.7 KB, free: 730.5 MB) -2016-05-23 22:02:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:02:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:02:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:33 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 22:02:33 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53923 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 22:02:33 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 22:02:33 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53923 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 22:02:33 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 52 ms on localhost (1/2) -2016-05-23 22:02:33 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 22:02:33 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 61 ms on localhost (2/2) -2016-05-23 22:02:33 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.064 s -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 22:02:33 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:02:33 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:02:33 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 22:02:33 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:02:33 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=356698, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=361554, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 22:02:33 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53923 (size: 2.5 KB, free: 730.6 MB) -2016-05-23 22:02:33 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:33 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:33 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 34 ms on localhost (1/1) -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 22:02:33 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.034 s -2016-05-23 22:02:33 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.181109 s -2016-05-23 22:02:33 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:02:33 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 22:02:33 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 22:02:33 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 22:02:33 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:33 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 22:02:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=364085, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=369741, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 22:02:33 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53923 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:02:33 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:33 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 22:02:33 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:33 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 22:02:33 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 26 ms on localhost (1/2) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 29 ms on localhost (2/2) -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 22:02:33 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.029 s -2016-05-23 22:02:33 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.058317 s -2016-05-23 22:02:33 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:02:33 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 22:02:33 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:02:33 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 22:02:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=372649, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=378345, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 22:02:33 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53923 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:02:33 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:33 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 22:02:33 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:33 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 21 ms on localhost (1/2) -2016-05-23 22:02:33 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 22 ms on localhost (2/2) -2016-05-23 22:02:33 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.025 s -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 22:02:33 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.041978 s -2016-05-23 22:02:33 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:46 -2016-05-23 22:02:33 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:46) with 4 output partitions -2016-05-23 22:02:33 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:46) -2016-05-23 22:02:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:43), which has no missing parents -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=381234, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=382458, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 22:02:33 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53923 (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:02:33 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:43) -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 22:02:33 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 22:02:33 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 22:02:33 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 22:02:33 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 22:02:33 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 22:02:33 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 22:02:33 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 22:02:33 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 9 ms on localhost (1/4) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 17 ms on localhost (2/4) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 9 ms on localhost (3/4) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 10 ms on localhost (4/4) -2016-05-23 22:02:33 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:46) finished in 0.018 s -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 22:02:33 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:46, took 0.033701 s -2016-05-23 22:02:33 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:48 -2016-05-23 22:02:33 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:48) with 4 output partitions -2016-05-23 22:02:33 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:48) -2016-05-23 22:02:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:44), which has no missing parents -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=383351, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=384575, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 22:02:33 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53923 (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:02:33 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:44) -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 22:02:33 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 22:02:33 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 22:02:33 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 22:02:33 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 22:02:33 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 22:02:33 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 22:02:33 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 22:02:33 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 9 ms on localhost (1/4) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 11 ms on localhost (2/4) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 8 ms on localhost (3/4) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 10 ms on localhost (4/4) -2016-05-23 22:02:33 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:48) finished in 0.012 s -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 22:02:33 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:48, took 0.020271 s -2016-05-23 22:02:33 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:02:33 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:02:33 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 22:02:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=385468, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=387972, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 22:02:33 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53923 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 22:02:33 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:02:33 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 22:02:33 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 7 ms on localhost (1/1) -2016-05-23 22:02:33 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.007 s -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 22:02:33 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.020399 s -2016-05-23 22:02:33 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:02:33 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 22:02:33 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=389496, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=392896, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 22:02:33 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53923 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 22:02:33 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 22:02:33 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 22:02:33 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 22:02:33 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 22:02:33 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 22:02:33 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 22:02:33 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 13 ms on localhost (1/4) -2016-05-23 22:02:33 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 22:02:33 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 13 ms on localhost (2/4) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 12 ms on localhost (3/4) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 13 ms on localhost (4/4) -2016-05-23 22:02:33 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.016 s -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 22:02:33 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.027241 s -2016-05-23 22:02:33 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:02:33 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 22:02:33 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:02:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=394830, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=398270, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 22:02:33 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53923 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 22:02:33 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 22:02:33 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 22:02:33 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 22:02:33 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 22:02:33 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 22:02:33 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 22:02:33 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 12 ms on localhost (1/4) -2016-05-23 22:02:33 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 17 ms on localhost (2/4) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 19 ms on localhost (3/4) -2016-05-23 22:02:33 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 22:02:33 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.024 s -2016-05-23 22:02:33 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.036155 s -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 23 ms on localhost (4/4) -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 22:02:33 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 22:02:33 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 22:02:33 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:02:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=400188, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(1860) called with curMem=403564, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1860.0 B, free 730.2 MB) -2016-05-23 22:02:33 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53923 (size: 1860.0 B, free: 730.5 MB) -2016-05-23 22:02:33 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 22:02:33 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 22:02:33 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 22:02:33 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 22:02:33 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 22:02:33 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 22:02:33 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 22:02:33 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 15 ms on localhost (1/4) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 14 ms on localhost (2/4) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 19 ms on localhost (3/4) -2016-05-23 22:02:33 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 24 ms on localhost (4/4) -2016-05-23 22:02:33 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.025 s -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 22:02:33 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.041261 s -2016-05-23 22:02:33 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 22:02:33 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:02:33 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 22:02:33 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 22:02:33 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 22:02:33 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=405424, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=409464, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 22:02:33 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:53923 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 22:02:33 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 22:02:33 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 22:02:33 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 22:02:33 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 22:02:33 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 22:02:33 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 148 ms on localhost (1/4) -2016-05-23 22:02:33 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 148 ms on localhost (2/4) -2016-05-23 22:02:33 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 22:02:33 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 258 ms on localhost (3/4) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 259 ms on localhost (4/4) -2016-05-23 22:02:33 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.262 s -2016-05-23 22:02:33 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:02:33 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:02:33 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 22:02:33 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:02:33 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=411746, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=414290, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 22:02:33 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:53923 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 22:02:33 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:33 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 22:02:33 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 22:02:33 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:33 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 34 ms on localhost (1/3) -2016-05-23 22:02:33 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 34 ms on localhost (2/3) -2016-05-23 22:02:33 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 22:02:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 37 ms on localhost (3/3) -2016-05-23 22:02:33 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.039 s -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 22:02:33 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.337530 s -2016-05-23 22:02:33 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:02:33 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 22:02:33 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:02:33 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:02:33 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 22:02:33 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 22:02:33 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=415858, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 22:02:33 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=420090, maxMem=766075207 -2016-05-23 22:02:33 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 22:02:33 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:53923 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:02:33 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:02:33 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:02:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:02:33 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 22:02:33 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 22:02:33 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 22:02:33 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:33 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 22:02:33 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 22:02:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 106 ms on localhost (1/4) -2016-05-23 22:02:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 110 ms on localhost (2/4) -2016-05-23 22:02:34 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 22:02:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 132 ms on localhost (3/4) -2016-05-23 22:02:34 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 22:02:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 295 ms on localhost (4/4) -2016-05-23 22:02:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 22:02:34 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.297 s -2016-05-23 22:02:34 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:02:34 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:02:34 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 22:02:34 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:02:34 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 22:02:34 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 22:02:34 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=422396, maxMem=766075207 -2016-05-23 22:02:34 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 22:02:34 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=427292, maxMem=766075207 -2016-05-23 22:02:34 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 22:02:34 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:53923 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:02:34 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:34 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 22:02:34 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 22:02:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:34 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 22:02:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:34 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 22:02:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 61 ms on localhost (1/1) -2016-05-23 22:02:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 22:02:34 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.062 s -2016-05-23 22:02:34 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.396894 s -2016-05-23 22:02:34 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:02:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 22:02:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 22:02:34 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 22:02:34 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:34 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 22:02:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:34 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:02:34 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=429834, maxMem=766075207 -2016-05-23 22:02:34 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 22:02:34 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=435530, maxMem=766075207 -2016-05-23 22:02:34 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 22:02:34 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:53923 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:02:34 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:34 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 22:02:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:34 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 22:02:34 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 22:02:34 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 22:02:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:34 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 22:02:34 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 22:02:34 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 22:02:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 40 ms on localhost (1/4) -2016-05-23 22:02:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 39 ms on localhost (2/4) -2016-05-23 22:02:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 42 ms on localhost (3/4) -2016-05-23 22:02:34 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 22:02:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:34 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 22:02:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 356 ms on localhost (4/4) -2016-05-23 22:02:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 22:02:34 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.422 s -2016-05-23 22:02:34 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.456436 s -2016-05-23 22:02:34 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:02:34 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 22:02:34 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:02:34 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 22:02:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:34 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:02:34 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=438439, maxMem=766075207 -2016-05-23 22:02:34 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 22:02:34 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=444175, maxMem=766075207 -2016-05-23 22:02:34 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 22:02:34 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:53923 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:02:34 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:02:34 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 22:02:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:34 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 22:02:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:34 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 22:02:34 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 22:02:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:34 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 22:02:34 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 22:02:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 47 ms on localhost (1/4) -2016-05-23 22:02:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 50 ms on localhost (2/4) -2016-05-23 22:02:34 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 22:02:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 55 ms on localhost (3/4) -2016-05-23 22:02:34 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 22:02:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:34 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 22:02:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 80 ms on localhost (4/4) -2016-05-23 22:02:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 22:02:34 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.081 s -2016-05-23 22:02:34 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.109280 s -2016-05-23 22:02:34 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:60 -2016-05-23 22:02:34 INFO DAGScheduler:59 - Got job 20 (foreach at StatisticsCorrLearning.scala:60) with 4 output partitions -2016-05-23 22:02:34 INFO DAGScheduler:59 - Final stage: ResultStage 34(foreach at StatisticsCorrLearning.scala:60) -2016-05-23 22:02:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:34 INFO DAGScheduler:59 - Submitting ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:57), which has no missing parents -2016-05-23 22:02:34 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=447067, maxMem=766075207 -2016-05-23 22:02:34 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 22:02:34 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=448291, maxMem=766075207 -2016-05-23 22:02:34 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 22:02:34 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:53923 (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:02:34 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:57) -2016-05-23 22:02:34 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-05-23 22:02:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 62, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:02:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 63, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:02:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 64, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:02:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 22:02:34 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 63) -2016-05-23 22:02:34 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 62) -2016-05-23 22:02:34 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 64) -2016-05-23 22:02:34 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 63). 915 bytes result sent to driver -2016-05-23 22:02:34 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 62). 915 bytes result sent to driver -2016-05-23 22:02:34 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 64). 915 bytes result sent to driver -2016-05-23 22:02:34 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 65) -2016-05-23 22:02:34 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 65). 915 bytes result sent to driver -2016-05-23 22:02:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 63) in 17 ms on localhost (1/4) -2016-05-23 22:02:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 62) in 19 ms on localhost (2/4) -2016-05-23 22:02:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 64) in 17 ms on localhost (3/4) -2016-05-23 22:02:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 65) in 18 ms on localhost (4/4) -2016-05-23 22:02:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-23 22:02:34 INFO DAGScheduler:59 - ResultStage 34 (foreach at StatisticsCorrLearning.scala:60) finished in 0.024 s -2016-05-23 22:02:34 INFO DAGScheduler:59 - Job 20 finished: foreach at StatisticsCorrLearning.scala:60, took 0.039352 s -2016-05-23 22:02:34 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:62 -2016-05-23 22:02:34 INFO DAGScheduler:59 - Got job 21 (foreach at StatisticsCorrLearning.scala:62) with 4 output partitions -2016-05-23 22:02:34 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at StatisticsCorrLearning.scala:62) -2016-05-23 22:02:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:34 INFO DAGScheduler:59 - Submitting ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:58), which has no missing parents -2016-05-23 22:02:34 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=449184, maxMem=766075207 -2016-05-23 22:02:34 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 22:02:34 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=450408, maxMem=766075207 -2016-05-23 22:02:34 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 22:02:34 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:53923 (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:02:34 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:58) -2016-05-23 22:02:34 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 4 tasks -2016-05-23 22:02:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 66, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:02:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:02:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:02:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 22:02:34 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 67) -2016-05-23 22:02:34 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 68) -2016-05-23 22:02:34 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 67). 915 bytes result sent to driver -2016-05-23 22:02:34 INFO Executor:59 - Running task 3.0 in stage 35.0 (TID 69) -2016-05-23 22:02:34 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 68). 915 bytes result sent to driver -2016-05-23 22:02:34 INFO Executor:59 - Finished task 3.0 in stage 35.0 (TID 69). 915 bytes result sent to driver -2016-05-23 22:02:35 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 66) -2016-05-23 22:02:35 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 66). 915 bytes result sent to driver -2016-05-23 22:02:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 67) in 33 ms on localhost (1/4) -2016-05-23 22:02:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 68) in 33 ms on localhost (2/4) -2016-05-23 22:02:35 INFO TaskSetManager:59 - Finished task 3.0 in stage 35.0 (TID 69) in 34 ms on localhost (3/4) -2016-05-23 22:02:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 66) in 38 ms on localhost (4/4) -2016-05-23 22:02:35 INFO DAGScheduler:59 - ResultStage 35 (foreach at StatisticsCorrLearning.scala:62) finished in 0.039 s -2016-05-23 22:02:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-23 22:02:35 INFO DAGScheduler:59 - Job 21 finished: foreach at StatisticsCorrLearning.scala:62, took 0.055187 s -2016-05-23 22:02:35 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:02:35 INFO DAGScheduler:59 - Got job 22 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:02:35 INFO DAGScheduler:59 - Final stage: ResultStage 36(first at RowMatrix.scala:65) -2016-05-23 22:02:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:35 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 22:02:35 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=451301, maxMem=766075207 -2016-05-23 22:02:35 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 22:02:35 INFO MemoryStore:59 - ensureFreeSpace(1522) called with curMem=453805, maxMem=766075207 -2016-05-23 22:02:35 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1522.0 B, free 730.2 MB) -2016-05-23 22:02:35 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:53923 (size: 1522.0 B, free: 730.5 MB) -2016-05-23 22:02:35 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:35 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 22:02:35 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-23 22:02:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:35 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 70) -2016-05-23 22:02:35 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 70). 1046 bytes result sent to driver -2016-05-23 22:02:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 70) in 7 ms on localhost (1/1) -2016-05-23 22:02:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-23 22:02:35 INFO DAGScheduler:59 - ResultStage 36 (first at RowMatrix.scala:65) finished in 0.007 s -2016-05-23 22:02:35 INFO DAGScheduler:59 - Job 22 finished: first at RowMatrix.scala:65, took 0.024518 s -2016-05-23 22:02:35 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:02:35 INFO DAGScheduler:59 - Got job 23 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 22:02:35 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:35 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:02:35 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=455327, maxMem=766075207 -2016-05-23 22:02:35 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 22:02:35 INFO MemoryStore:59 - ensureFreeSpace(1932) called with curMem=458727, maxMem=766075207 -2016-05-23 22:02:35 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1932.0 B, free 730.1 MB) -2016-05-23 22:02:35 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:53923 (size: 1932.0 B, free: 730.5 MB) -2016-05-23 22:02:35 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:35 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:35 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 4 tasks -2016-05-23 22:02:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:35 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:35 INFO TaskSetManager:59 - Starting task 3.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:02:35 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-23 22:02:35 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 73) -2016-05-23 22:02:35 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 72) -2016-05-23 22:02:35 INFO Executor:59 - Running task 3.0 in stage 37.0 (TID 74) -2016-05-23 22:02:35 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 1231 bytes result sent to driver -2016-05-23 22:02:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 265 ms on localhost (1/4) -2016-05-23 22:02:35 INFO Executor:59 - Finished task 3.0 in stage 37.0 (TID 74). 1231 bytes result sent to driver -2016-05-23 22:02:35 INFO TaskSetManager:59 - Finished task 3.0 in stage 37.0 (TID 74) in 431 ms on localhost (2/4) -2016-05-23 22:02:35 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 72). 1231 bytes result sent to driver -2016-05-23 22:02:35 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 73). 1231 bytes result sent to driver -2016-05-23 22:02:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 72) in 820 ms on localhost (3/4) -2016-05-23 22:02:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 73) in 818 ms on localhost (4/4) -2016-05-23 22:02:35 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at RowMatrix.scala:332) finished in 0.824 s -2016-05-23 22:02:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-23 22:02:35 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at RowMatrix.scala:332, took 0.846082 s -2016-05-23 22:02:35 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:02:35 INFO DAGScheduler:59 - Got job 24 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 22:02:35 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:02:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:35 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:02:35 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=460659, maxMem=766075207 -2016-05-23 22:02:35 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-23 22:02:35 INFO MemoryStore:59 - ensureFreeSpace(1916) called with curMem=464099, maxMem=766075207 -2016-05-23 22:02:35 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1916.0 B, free 730.1 MB) -2016-05-23 22:02:35 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:53923 (size: 1916.0 B, free: 730.5 MB) -2016-05-23 22:02:35 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:35 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:02:35 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-23 22:02:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:35 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 77, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:35 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 78, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:02:35 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 78) -2016-05-23 22:02:35 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 78). 1109 bytes result sent to driver -2016-05-23 22:02:35 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-23 22:02:35 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 1109 bytes result sent to driver -2016-05-23 22:02:35 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 77) -2016-05-23 22:02:35 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 77). 1109 bytes result sent to driver -2016-05-23 22:02:35 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-23 22:02:35 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 1109 bytes result sent to driver -2016-05-23 22:02:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 28 ms on localhost (1/4) -2016-05-23 22:02:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 30 ms on localhost (2/4) -2016-05-23 22:02:35 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 78) in 27 ms on localhost (3/4) -2016-05-23 22:02:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 77) in 32 ms on localhost (4/4) -2016-05-23 22:02:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-23 22:02:35 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at RowMatrix.scala:124) finished in 0.043 s -2016-05-23 22:02:35 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at RowMatrix.scala:124, took 0.061773 s -2016-05-23 22:02:36 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 22:02:36 INFO DAGScheduler:59 - Got job 25 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 22:02:36 INFO DAGScheduler:59 - Final stage: ResultStage 39(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=466015, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(1851) called with curMem=469391, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1851.0 B, free 730.1 MB) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:53923 (size: 1851.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 4 tasks -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 79, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 80, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 81, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 3.0 in stage 39.0 (TID 82, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:02:36 INFO Executor:59 - Running task 3.0 in stage 39.0 (TID 82) -2016-05-23 22:02:36 INFO Executor:59 - Finished task 3.0 in stage 39.0 (TID 82). 1294 bytes result sent to driver -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 3.0 in stage 39.0 (TID 82) in 8 ms on localhost (1/4) -2016-05-23 22:02:36 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 80) -2016-05-23 22:02:36 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 81) -2016-05-23 22:02:36 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 80). 1254 bytes result sent to driver -2016-05-23 22:02:36 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 81). 1249 bytes result sent to driver -2016-05-23 22:02:36 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 79) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 80) in 25 ms on localhost (2/4) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 81) in 25 ms on localhost (3/4) -2016-05-23 22:02:36 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 79). 1254 bytes result sent to driver -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 79) in 30 ms on localhost (4/4) -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-23 22:02:36 INFO DAGScheduler:59 - ResultStage 39 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.031 s -2016-05-23 22:02:36 INFO DAGScheduler:59 - Job 25 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.054079 s -2016-05-23 22:02:36 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 22:02:36 INFO DAGScheduler:59 - Registering RDD 51 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Got job 26 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 22:02:36 INFO DAGScheduler:59 - Final stage: ResultStage 41(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 40) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 40) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=471242, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(2281) called with curMem=475282, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:53923 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 22:02:36 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 4 tasks -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 83, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 84, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 85, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 3.0 in stage 40.0 (TID 86, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 22:02:36 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 83) -2016-05-23 22:02:36 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 85) -2016-05-23 22:02:36 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 84) -2016-05-23 22:02:36 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 85). 1161 bytes result sent to driver -2016-05-23 22:02:36 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 83). 1161 bytes result sent to driver -2016-05-23 22:02:36 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 84). 1161 bytes result sent to driver -2016-05-23 22:02:36 INFO Executor:59 - Running task 3.0 in stage 40.0 (TID 86) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 85) in 131 ms on localhost (1/4) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 83) in 137 ms on localhost (2/4) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 84) in 137 ms on localhost (3/4) -2016-05-23 22:02:36 INFO Executor:59 - Finished task 3.0 in stage 40.0 (TID 86). 1161 bytes result sent to driver -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 3.0 in stage 40.0 (TID 86) in 171 ms on localhost (4/4) -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-23 22:02:36 INFO DAGScheduler:59 - ShuffleMapStage 40 (flatMap at SpearmanCorrelation.scala:49) finished in 0.176 s -2016-05-23 22:02:36 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:02:36 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:02:36 INFO DAGScheduler:59 - waiting: Set(ResultStage 41) -2016-05-23 22:02:36 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:02:36 INFO DAGScheduler:59 - Missing parents for ResultStage 41: List() -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=477563, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=480107, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.1 MB) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:53923 (size: 1567.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 3 tasks -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 87, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:36 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 87) -2016-05-23 22:02:36 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 88) -2016-05-23 22:02:36 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 89) -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:36 INFO Executor:59 - Finished task 2.0 in stage 41.0 (TID 89). 1203 bytes result sent to driver -2016-05-23 22:02:36 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 87). 1203 bytes result sent to driver -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 87) in 17 ms on localhost (1/3) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 2.0 in stage 41.0 (TID 89) in 15 ms on localhost (2/3) -2016-05-23 22:02:36 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 88). 1203 bytes result sent to driver -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 88) in 19 ms on localhost (3/3) -2016-05-23 22:02:36 INFO DAGScheduler:59 - ResultStage 41 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.020 s -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-23 22:02:36 INFO DAGScheduler:59 - Job 26 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.246244 s -2016-05-23 22:02:36 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:02:36 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 163 bytes -2016-05-23 22:02:36 INFO DAGScheduler:59 - Registering RDD 56 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Got job 27 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:02:36 INFO DAGScheduler:59 - Final stage: ResultStage 44(first at RowMatrix.scala:65) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 43) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 43) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=481674, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(2305) called with curMem=485906, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:53923 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:02:36 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 4 tasks -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 90, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 91, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 2.0 in stage 43.0 (TID 92, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 3.0 in stage 43.0 (TID 93, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:02:36 INFO Executor:59 - Running task 2.0 in stage 43.0 (TID 92) -2016-05-23 22:02:36 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 90) -2016-05-23 22:02:36 INFO Executor:59 - Running task 3.0 in stage 43.0 (TID 93) -2016-05-23 22:02:36 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 91) -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:36 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 91). 1377 bytes result sent to driver -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 91) in 51 ms on localhost (1/4) -2016-05-23 22:02:36 INFO Executor:59 - Finished task 3.0 in stage 43.0 (TID 93). 1377 bytes result sent to driver -2016-05-23 22:02:36 INFO Executor:59 - Finished task 2.0 in stage 43.0 (TID 92). 1377 bytes result sent to driver -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 3.0 in stage 43.0 (TID 93) in 183 ms on localhost (2/4) -2016-05-23 22:02:36 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 90). 1377 bytes result sent to driver -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 2.0 in stage 43.0 (TID 92) in 185 ms on localhost (3/4) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 90) in 188 ms on localhost (4/4) -2016-05-23 22:02:36 INFO DAGScheduler:59 - ShuffleMapStage 43 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.189 s -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-23 22:02:36 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:02:36 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:02:36 INFO DAGScheduler:59 - waiting: Set(ResultStage 44) -2016-05-23 22:02:36 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:02:36 INFO DAGScheduler:59 - Missing parents for ResultStage 44: List() -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=488211, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(2545) called with curMem=493107, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:53923 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:02:36 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 1 tasks -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 94, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:36 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 94) -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:36 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 94). 1296 bytes result sent to driver -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 94) in 9 ms on localhost (1/1) -2016-05-23 22:02:36 INFO DAGScheduler:59 - ResultStage 44 (first at RowMatrix.scala:65) finished in 0.010 s -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-23 22:02:36 INFO DAGScheduler:59 - Job 27 finished: first at RowMatrix.scala:65, took 0.228327 s -2016-05-23 22:02:36 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:02:36 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 163 bytes -2016-05-23 22:02:36 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 166 bytes -2016-05-23 22:02:36 INFO DAGScheduler:59 - Got job 28 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 22:02:36 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 46) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=495652, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(2914) called with curMem=501348, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:53923 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:02:36 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 4 tasks -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 95, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 96, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 2.0 in stage 47.0 (TID 97, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 3.0 in stage 47.0 (TID 98, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:36 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 95) -2016-05-23 22:02:36 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 96) -2016-05-23 22:02:36 INFO Executor:59 - Running task 2.0 in stage 47.0 (TID 97) -2016-05-23 22:02:36 INFO Executor:59 - Running task 3.0 in stage 47.0 (TID 98) -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:36 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 96). 1481 bytes result sent to driver -2016-05-23 22:02:36 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 95). 1481 bytes result sent to driver -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 96) in 15 ms on localhost (1/4) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 95) in 16 ms on localhost (2/4) -2016-05-23 22:02:36 INFO Executor:59 - Finished task 2.0 in stage 47.0 (TID 97). 1481 bytes result sent to driver -2016-05-23 22:02:36 INFO Executor:59 - Finished task 3.0 in stage 47.0 (TID 98). 1481 bytes result sent to driver -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 2.0 in stage 47.0 (TID 97) in 16 ms on localhost (3/4) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 3.0 in stage 47.0 (TID 98) in 17 ms on localhost (4/4) -2016-05-23 22:02:36 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at RowMatrix.scala:332) finished in 0.019 s -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-23 22:02:36 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at RowMatrix.scala:332, took 0.036556 s -2016-05-23 22:02:36 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:02:36 INFO DAGScheduler:59 - Got job 29 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 22:02:36 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 49) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=504262, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=509998, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:53923 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:02:36 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 4 tasks -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 2.0 in stage 50.0 (TID 101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 3.0 in stage 50.0 (TID 102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:36 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-23 22:02:36 INFO Executor:59 - Running task 2.0 in stage 50.0 (TID 101) -2016-05-23 22:02:36 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-23 22:02:36 INFO Executor:59 - Running task 3.0 in stage 50.0 (TID 102) -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:36 INFO Executor:59 - Finished task 2.0 in stage 50.0 (TID 101). 1359 bytes result sent to driver -2016-05-23 22:02:36 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 1359 bytes result sent to driver -2016-05-23 22:02:36 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 1359 bytes result sent to driver -2016-05-23 22:02:36 INFO Executor:59 - Finished task 3.0 in stage 50.0 (TID 102). 1359 bytes result sent to driver -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 31 ms on localhost (1/4) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 3.0 in stage 50.0 (TID 102) in 31 ms on localhost (2/4) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 32 ms on localhost (3/4) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 2.0 in stage 50.0 (TID 101) in 31 ms on localhost (4/4) -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-23 22:02:36 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at RowMatrix.scala:124) finished in 0.033 s -2016-05-23 22:02:36 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at RowMatrix.scala:124, took 0.047408 s -2016-05-23 22:02:36 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:73 -2016-05-23 22:02:36 INFO DAGScheduler:59 - Got job 30 (foreach at StatisticsCorrLearning.scala:73) with 4 output partitions -2016-05-23 22:02:36 INFO DAGScheduler:59 - Final stage: ResultStage 51(foreach at StatisticsCorrLearning.scala:73) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting ResultStage 51 (ParallelCollectionRDD[61] at parallelize at StatisticsCorrLearning.scala:70), which has no missing parents -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=512894, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=511222, maxMem=766075207 -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:53923 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.1 MB) -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:53923 (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 51 (ParallelCollectionRDD[61] at parallelize at StatisticsCorrLearning.scala:70) -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 4 tasks -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 103, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:53923 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 104, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 2.0 in stage 51.0 (TID 105, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 3.0 in stage 51.0 (TID 106, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:02:36 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 103) -2016-05-23 22:02:36 INFO Executor:59 - Running task 3.0 in stage 51.0 (TID 106) -2016-05-23 22:02:36 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 104) -2016-05-23 22:02:36 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 103). 915 bytes result sent to driver -2016-05-23 22:02:36 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 104). 915 bytes result sent to driver -2016-05-23 22:02:36 INFO Executor:59 - Finished task 3.0 in stage 51.0 (TID 106). 915 bytes result sent to driver -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 104) in 8 ms on localhost (1/4) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 103) in 9 ms on localhost (2/4) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:53923 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:02:36 INFO Executor:59 - Running task 2.0 in stage 51.0 (TID 105) -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 3.0 in stage 51.0 (TID 106) in 9 ms on localhost (3/4) -2016-05-23 22:02:36 INFO Executor:59 - Finished task 2.0 in stage 51.0 (TID 105). 915 bytes result sent to driver -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 2.0 in stage 51.0 (TID 105) in 13 ms on localhost (4/4) -2016-05-23 22:02:36 INFO DAGScheduler:59 - ResultStage 51 (foreach at StatisticsCorrLearning.scala:73) finished in 0.015 s -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-23 22:02:36 INFO DAGScheduler:59 - Job 30 finished: foreach at StatisticsCorrLearning.scala:73, took 0.055014 s -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:53923 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned shuffle 5 -2016-05-23 22:02:36 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:75 -2016-05-23 22:02:36 INFO DAGScheduler:59 - Got job 31 (foreach at StatisticsCorrLearning.scala:75) with 4 output partitions -2016-05-23 22:02:36 INFO DAGScheduler:59 - Final stage: ResultStage 52(foreach at StatisticsCorrLearning.scala:75) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:53923 in memory (size: 1567.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting ResultStage 52 (ParallelCollectionRDD[62] at parallelize at StatisticsCorrLearning.scala:71), which has no missing parents -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=479680, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=474583, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.1 MB) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:53923 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:53923 (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 52 (ParallelCollectionRDD[62] at parallelize at StatisticsCorrLearning.scala:71) -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 4 tasks -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 107, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 108, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 2.0 in stage 52.0 (TID 109, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 3.0 in stage 52.0 (TID 110, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:02:36 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 108) -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-23 22:02:36 INFO Executor:59 - Running task 2.0 in stage 52.0 (TID 109) -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned shuffle 4 -2016-05-23 22:02:36 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 107) -2016-05-23 22:02:36 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 108). 915 bytes result sent to driver -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 108) in 9 ms on localhost (1/4) -2016-05-23 22:02:36 INFO Executor:59 - Running task 3.0 in stage 52.0 (TID 110) -2016-05-23 22:02:36 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 107). 915 bytes result sent to driver -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:53923 in memory (size: 1851.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO Executor:59 - Finished task 2.0 in stage 52.0 (TID 109). 915 bytes result sent to driver -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 107) in 23 ms on localhost (2/4) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 2.0 in stage 52.0 (TID 109) in 22 ms on localhost (3/4) -2016-05-23 22:02:36 INFO Executor:59 - Finished task 3.0 in stage 52.0 (TID 110). 915 bytes result sent to driver -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 3.0 in stage 52.0 (TID 110) in 26 ms on localhost (4/4) -2016-05-23 22:02:36 INFO DAGScheduler:59 - ResultStage 52 (foreach at StatisticsCorrLearning.scala:75) finished in 0.026 s -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-23 22:02:36 INFO DAGScheduler:59 - Job 31 finished: foreach at StatisticsCorrLearning.scala:75, took 0.044015 s -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:53923 in memory (size: 1916.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:53923 in memory (size: 1932.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:53923 in memory (size: 1522.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-23 22:02:36 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:02:36 INFO DAGScheduler:59 - Got job 32 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:02:36 INFO DAGScheduler:59 - Final stage: ResultStage 53(first at RowMatrix.scala:65) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:53923 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[64] at map at Correlation.scala:45), which has no missing parents -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=451301, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:53923 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=453805, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:53923 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:53923 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 53 (MapPartitionsRDD[64] at map at Correlation.scala:45) -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 1 tasks -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 111, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 22:02:36 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 111) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:53923 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:02:36 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 111). 935 bytes result sent to driver -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-23 22:02:36 INFO DAGScheduler:59 - ResultStage 53 (first at RowMatrix.scala:65) finished in 0.004 s -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 111) in 4 ms on localhost (1/1) -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-23 22:02:36 INFO DAGScheduler:59 - Job 32 finished: first at RowMatrix.scala:65, took 0.016135 s -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:53923 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:53923 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:02:36 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-05-23 22:02:36 INFO DAGScheduler:59 - Got job 33 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 22:02:36 INFO DAGScheduler:59 - Final stage: ResultStage 54(first at RowMatrix.scala:65) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[64] at map at Correlation.scala:45), which has no missing parents -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:53923 in memory (size: 1568.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=422552, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=422512, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:53923 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:53923 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 54 (MapPartitionsRDD[64] at map at Correlation.scala:45) -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 3 tasks -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 112, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 113, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 2.0 in stage 54.0 (TID 114, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:36 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 112) -2016-05-23 22:02:36 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 113) -2016-05-23 22:02:36 INFO Executor:59 - Running task 2.0 in stage 54.0 (TID 114) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:53923 in memory (size: 1860.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 112). 1046 bytes result sent to driver -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-23 22:02:36 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 113). 1046 bytes result sent to driver -2016-05-23 22:02:36 INFO Executor:59 - Finished task 2.0 in stage 54.0 (TID 114). 1046 bytes result sent to driver -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 112) in 13 ms on localhost (1/3) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 113) in 13 ms on localhost (2/3) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:53923 in memory (size: 1918.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 2.0 in stage 54.0 (TID 114) in 14 ms on localhost (3/3) -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-23 22:02:36 INFO DAGScheduler:59 - ResultStage 54 (first at RowMatrix.scala:65) finished in 0.017 s -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-23 22:02:36 INFO DAGScheduler:59 - Job 33 finished: first at RowMatrix.scala:65, took 0.031692 s -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:53923 in memory (size: 1934.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:53923 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-23 22:02:36 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:02:36 INFO DAGScheduler:59 - Got job 34 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 22:02:36 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:53923 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[65] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=394748, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:53923 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=396924, maxMem=766075207 -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:53923 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:53923 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 55 (MapPartitionsRDD[65] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 4 tasks -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 115, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 116, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 2.0 in stage 55.0 (TID 117, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 3.0 in stage 55.0 (TID 118, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:36 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 116) -2016-05-23 22:02:36 INFO Executor:59 - Running task 2.0 in stage 55.0 (TID 117) -2016-05-23 22:02:36 INFO Executor:59 - Running task 3.0 in stage 55.0 (TID 118) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:53923 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:02:36 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 115) -2016-05-23 22:02:36 INFO Executor:59 - Finished task 2.0 in stage 55.0 (TID 117). 1231 bytes result sent to driver -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 2.0 in stage 55.0 (TID 117) in 5 ms on localhost (1/4) -2016-05-23 22:02:36 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 116). 1231 bytes result sent to driver -2016-05-23 22:02:36 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 115). 1231 bytes result sent to driver -2016-05-23 22:02:36 INFO Executor:59 - Finished task 3.0 in stage 55.0 (TID 118). 1231 bytes result sent to driver -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 116) in 10 ms on localhost (2/4) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 115) in 11 ms on localhost (3/4) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Finished task 3.0 in stage 55.0 (TID 118) in 10 ms on localhost (4/4) -2016-05-23 22:02:36 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at RowMatrix.scala:332) finished in 0.012 s -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:53923 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at RowMatrix.scala:332, took 0.027097 s -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:53923 in memory (size: 2.2 KB, free: 730.6 MB) -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-23 22:02:36 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-23 22:02:36 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:02:36 INFO DAGScheduler:59 - Got job 35 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 22:02:36 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:02:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[66] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=367848, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 22:02:36 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=371288, maxMem=766075207 -2016-05-23 22:02:36 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 22:02:36 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:53923 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 22:02:36 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:36 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 56 (MapPartitionsRDD[66] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:02:36 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 4 tasks -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 119, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 120, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 2.0 in stage 56.0 (TID 121, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:36 INFO TaskSetManager:59 - Starting task 3.0 in stage 56.0 (TID 122, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:37 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 119) -2016-05-23 22:02:37 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 120) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 119). 1109 bytes result sent to driver -2016-05-23 22:02:37 INFO Executor:59 - Running task 2.0 in stage 56.0 (TID 121) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 120). 1109 bytes result sent to driver -2016-05-23 22:02:37 INFO Executor:59 - Finished task 2.0 in stage 56.0 (TID 121). 1109 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 120) in 12 ms on localhost (1/4) -2016-05-23 22:02:37 INFO Executor:59 - Running task 3.0 in stage 56.0 (TID 122) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 3.0 in stage 56.0 (TID 122). 1109 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 119) in 14 ms on localhost (2/4) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 3.0 in stage 56.0 (TID 122) in 20 ms on localhost (3/4) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 2.0 in stage 56.0 (TID 121) in 21 ms on localhost (4/4) -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-23 22:02:37 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at RowMatrix.scala:124) finished in 0.023 s -2016-05-23 22:02:37 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at RowMatrix.scala:124, took 0.039081 s -2016-05-23 22:02:37 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Got job 36 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 22:02:37 INFO DAGScheduler:59 - Final stage: ResultStage 57(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[72] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=373206, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(1858) called with curMem=376582, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 1858.0 B, free 730.2 MB) -2016-05-23 22:02:37 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:53923 (size: 1858.0 B, free: 730.5 MB) -2016-05-23 22:02:37 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 57 (MapPartitionsRDD[72] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 4 tasks -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 123, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 124, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 2.0 in stage 57.0 (TID 125, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 3.0 in stage 57.0 (TID 126, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:37 INFO Executor:59 - Running task 2.0 in stage 57.0 (TID 125) -2016-05-23 22:02:37 INFO Executor:59 - Running task 3.0 in stage 57.0 (TID 126) -2016-05-23 22:02:37 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 123) -2016-05-23 22:02:37 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 124) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 2.0 in stage 57.0 (TID 125). 1249 bytes result sent to driver -2016-05-23 22:02:37 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 124). 1254 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 2.0 in stage 57.0 (TID 125) in 6 ms on localhost (1/4) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 3.0 in stage 57.0 (TID 126). 1254 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 124) in 8 ms on localhost (2/4) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 123). 1104 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 3.0 in stage 57.0 (TID 126) in 8 ms on localhost (3/4) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 123) in 11 ms on localhost (4/4) -2016-05-23 22:02:37 INFO DAGScheduler:59 - ResultStage 57 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.011 s -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-23 22:02:37 INFO DAGScheduler:59 - Job 36 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.022967 s -2016-05-23 22:02:37 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Registering RDD 70 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Got job 37 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 22:02:37 INFO DAGScheduler:59 - Final stage: ResultStage 59(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 58) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 58) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 58 (MapPartitionsRDD[70] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=378440, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(2285) called with curMem=382480, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 22:02:37 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:53923 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 22:02:37 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 58 (MapPartitionsRDD[70] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 4 tasks -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 127, localhost, PROCESS_LOCAL, 2221 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 128, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 2.0 in stage 58.0 (TID 129, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 3.0 in stage 58.0 (TID 130, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 22:02:37 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 128) -2016-05-23 22:02:37 INFO Executor:59 - Running task 3.0 in stage 58.0 (TID 130) -2016-05-23 22:02:37 INFO Executor:59 - Running task 2.0 in stage 58.0 (TID 129) -2016-05-23 22:02:37 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 127) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 127). 1161 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 127) in 34 ms on localhost (1/4) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 3.0 in stage 58.0 (TID 130). 1161 bytes result sent to driver -2016-05-23 22:02:37 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 128). 1161 bytes result sent to driver -2016-05-23 22:02:37 INFO Executor:59 - Finished task 2.0 in stage 58.0 (TID 129). 1161 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 3.0 in stage 58.0 (TID 130) in 81 ms on localhost (2/4) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 128) in 84 ms on localhost (3/4) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 2.0 in stage 58.0 (TID 129) in 83 ms on localhost (4/4) -2016-05-23 22:02:37 INFO DAGScheduler:59 - ShuffleMapStage 58 (flatMap at SpearmanCorrelation.scala:49) finished in 0.085 s -2016-05-23 22:02:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:02:37 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:02:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 59) -2016-05-23 22:02:37 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-05-23 22:02:37 INFO DAGScheduler:59 - Missing parents for ResultStage 59: List() -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting ResultStage 59 (ShuffledRDD[73] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=384765, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=387309, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.2 MB) -2016-05-23 22:02:37 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:53923 (size: 1567.0 B, free: 730.5 MB) -2016-05-23 22:02:37 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 59 (ShuffledRDD[73] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 3 tasks -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 131, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 132, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 2.0 in stage 59.0 (TID 133, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:37 INFO Executor:59 - Running task 2.0 in stage 59.0 (TID 133) -2016-05-23 22:02:37 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 132) -2016-05-23 22:02:37 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 131) -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:37 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 132). 1203 bytes result sent to driver -2016-05-23 22:02:37 INFO Executor:59 - Finished task 2.0 in stage 59.0 (TID 133). 1203 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 132) in 11 ms on localhost (1/3) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 2.0 in stage 59.0 (TID 133) in 12 ms on localhost (2/3) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 131). 1203 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 131) in 15 ms on localhost (3/3) -2016-05-23 22:02:37 INFO DAGScheduler:59 - ResultStage 59 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.015 s -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-05-23 22:02:37 INFO DAGScheduler:59 - Job 37 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.129162 s -2016-05-23 22:02:37 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:02:37 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 168 bytes -2016-05-23 22:02:37 INFO DAGScheduler:59 - Registering RDD 75 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Got job 38 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:02:37 INFO DAGScheduler:59 - Final stage: ResultStage 62(first at RowMatrix.scala:65) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 61) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 61) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 61 (MapPartitionsRDD[75] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=388876, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(2305) called with curMem=393108, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 22:02:37 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:53923 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:02:37 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 61 (MapPartitionsRDD[75] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 4 tasks -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 134, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 135, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 2.0 in stage 61.0 (TID 136, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 3.0 in stage 61.0 (TID 137, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:02:37 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 135) -2016-05-23 22:02:37 INFO Executor:59 - Running task 2.0 in stage 61.0 (TID 136) -2016-05-23 22:02:37 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 134) -2016-05-23 22:02:37 INFO Executor:59 - Running task 3.0 in stage 61.0 (TID 137) -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:37 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 135). 1377 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 135) in 45 ms on localhost (1/4) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 3.0 in stage 61.0 (TID 137). 1377 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 3.0 in stage 61.0 (TID 137) in 48 ms on localhost (2/4) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 134). 1377 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 134) in 56 ms on localhost (3/4) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 2.0 in stage 61.0 (TID 136). 1377 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 2.0 in stage 61.0 (TID 136) in 66 ms on localhost (4/4) -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-05-23 22:02:37 INFO DAGScheduler:59 - ShuffleMapStage 61 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.069 s -2016-05-23 22:02:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:02:37 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:02:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 62) -2016-05-23 22:02:37 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:02:37 INFO DAGScheduler:59 - Missing parents for ResultStage 62: List() -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=395413, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=400309, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 22:02:37 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:53923 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:02:37 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 62 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87) -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 1 tasks -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 138, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:37 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 138) -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:37 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 138). 1185 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 138) in 9 ms on localhost (1/1) -2016-05-23 22:02:37 INFO DAGScheduler:59 - ResultStage 62 (first at RowMatrix.scala:65) finished in 0.009 s -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-05-23 22:02:37 INFO DAGScheduler:59 - Job 38 finished: first at RowMatrix.scala:65, took 0.098907 s -2016-05-23 22:02:37 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:02:37 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 168 bytes -2016-05-23 22:02:37 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 165 bytes -2016-05-23 22:02:37 INFO DAGScheduler:59 - Got job 39 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 22:02:37 INFO DAGScheduler:59 - Final stage: ResultStage 65(first at RowMatrix.scala:65) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 64) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87), which has no missing parents -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=402853, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=407749, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 22:02:37 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:53923 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:02:37 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 65 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87) -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 3 tasks -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 139, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 140, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 2.0 in stage 65.0 (TID 141, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:37 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 140) -2016-05-23 22:02:37 INFO Executor:59 - Running task 2.0 in stage 65.0 (TID 141) -2016-05-23 22:02:37 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 139) -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 22:02:37 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 139). 1296 bytes result sent to driver -2016-05-23 22:02:37 INFO Executor:59 - Finished task 2.0 in stage 65.0 (TID 141). 1296 bytes result sent to driver -2016-05-23 22:02:37 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 140). 1296 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 139) in 15 ms on localhost (1/3) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 2.0 in stage 65.0 (TID 141) in 15 ms on localhost (2/3) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 140) in 17 ms on localhost (3/3) -2016-05-23 22:02:37 INFO DAGScheduler:59 - ResultStage 65 (first at RowMatrix.scala:65) finished in 0.017 s -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-05-23 22:02:37 INFO DAGScheduler:59 - Job 39 finished: first at RowMatrix.scala:65, took 0.030376 s -2016-05-23 22:02:37 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Got job 40 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 22:02:37 INFO DAGScheduler:59 - Final stage: ResultStage 68(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 67) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting ResultStage 68 (MapPartitionsRDD[78] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=410293, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(2914) called with curMem=415989, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 22:02:37 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:53923 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:02:37 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 68 (MapPartitionsRDD[78] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Adding task set 68.0 with 4 tasks -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 68.0 (TID 142, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 68.0 (TID 143, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 2.0 in stage 68.0 (TID 144, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 3.0 in stage 68.0 (TID 145, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:37 INFO Executor:59 - Running task 0.0 in stage 68.0 (TID 142) -2016-05-23 22:02:37 INFO Executor:59 - Running task 1.0 in stage 68.0 (TID 143) -2016-05-23 22:02:37 INFO Executor:59 - Running task 3.0 in stage 68.0 (TID 145) -2016-05-23 22:02:37 INFO Executor:59 - Running task 2.0 in stage 68.0 (TID 144) -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 8 ms -2016-05-23 22:02:37 INFO Executor:59 - Finished task 0.0 in stage 68.0 (TID 142). 1481 bytes result sent to driver -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 10 ms -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 68.0 (TID 142) in 21 ms on localhost (1/4) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 2.0 in stage 68.0 (TID 144). 1481 bytes result sent to driver -2016-05-23 22:02:37 INFO Executor:59 - Finished task 3.0 in stage 68.0 (TID 145). 1481 bytes result sent to driver -2016-05-23 22:02:37 INFO Executor:59 - Finished task 1.0 in stage 68.0 (TID 143). 1481 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 2.0 in stage 68.0 (TID 144) in 21 ms on localhost (2/4) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 3.0 in stage 68.0 (TID 145) in 22 ms on localhost (3/4) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 68.0 (TID 143) in 23 ms on localhost (4/4) -2016-05-23 22:02:37 INFO DAGScheduler:59 - ResultStage 68 (treeAggregate at RowMatrix.scala:332) finished in 0.023 s -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 68.0, whose tasks have all completed, from pool -2016-05-23 22:02:37 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at RowMatrix.scala:332, took 0.032804 s -2016-05-23 22:02:37 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Got job 41 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 22:02:37 INFO DAGScheduler:59 - Final stage: ResultStage 71(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 70) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting ResultStage 71 (MapPartitionsRDD[79] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=418903, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=424639, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 22:02:37 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:53923 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:02:37 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 71 (MapPartitionsRDD[79] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Adding task set 71.0 with 4 tasks -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 71.0 (TID 146, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 71.0 (TID 147, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 2.0 in stage 71.0 (TID 148, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 3.0 in stage 71.0 (TID 149, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:37 INFO Executor:59 - Running task 1.0 in stage 71.0 (TID 147) -2016-05-23 22:02:37 INFO Executor:59 - Running task 3.0 in stage 71.0 (TID 149) -2016-05-23 22:02:37 INFO Executor:59 - Running task 0.0 in stage 71.0 (TID 146) -2016-05-23 22:02:37 INFO Executor:59 - Running task 2.0 in stage 71.0 (TID 148) -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:37 INFO Executor:59 - Finished task 1.0 in stage 71.0 (TID 147). 1359 bytes result sent to driver -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:37 INFO Executor:59 - Finished task 3.0 in stage 71.0 (TID 149). 1359 bytes result sent to driver -2016-05-23 22:02:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-23 22:02:37 INFO Executor:59 - Finished task 0.0 in stage 71.0 (TID 146). 1359 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 71.0 (TID 147) in 13 ms on localhost (1/4) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 3.0 in stage 71.0 (TID 149) in 13 ms on localhost (2/4) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 71.0 (TID 146) in 15 ms on localhost (3/4) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 2.0 in stage 71.0 (TID 148). 1359 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 2.0 in stage 71.0 (TID 148) in 18 ms on localhost (4/4) -2016-05-23 22:02:37 INFO DAGScheduler:59 - ResultStage 71 (treeAggregate at RowMatrix.scala:124) finished in 0.019 s -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 71.0, whose tasks have all completed, from pool -2016-05-23 22:02:37 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at RowMatrix.scala:124, took 0.028964 s -2016-05-23 22:02:37 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:87 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Got job 42 (foreach at StatisticsCorrLearning.scala:87) with 4 output partitions -2016-05-23 22:02:37 INFO DAGScheduler:59 - Final stage: ResultStage 72(foreach at StatisticsCorrLearning.scala:87) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting ResultStage 72 (ParallelCollectionRDD[80] at parallelize at StatisticsCorrLearning.scala:84), which has no missing parents -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=427535, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=428759, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 22:02:37 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:53923 (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:02:37 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 72 (ParallelCollectionRDD[80] at parallelize at StatisticsCorrLearning.scala:84) -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Adding task set 72.0 with 4 tasks -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 72.0 (TID 150, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 72.0 (TID 151, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 2.0 in stage 72.0 (TID 152, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 3.0 in stage 72.0 (TID 153, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 22:02:37 INFO Executor:59 - Running task 0.0 in stage 72.0 (TID 150) -2016-05-23 22:02:37 INFO Executor:59 - Running task 3.0 in stage 72.0 (TID 153) -2016-05-23 22:02:37 INFO Executor:59 - Running task 1.0 in stage 72.0 (TID 151) -2016-05-23 22:02:37 INFO Executor:59 - Running task 2.0 in stage 72.0 (TID 152) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 1.0 in stage 72.0 (TID 151). 915 bytes result sent to driver -2016-05-23 22:02:37 INFO Executor:59 - Finished task 3.0 in stage 72.0 (TID 153). 915 bytes result sent to driver -2016-05-23 22:02:37 INFO Executor:59 - Finished task 2.0 in stage 72.0 (TID 152). 915 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 72.0 (TID 151) in 5 ms on localhost (1/4) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 0.0 in stage 72.0 (TID 150). 915 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 3.0 in stage 72.0 (TID 153) in 4 ms on localhost (2/4) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 2.0 in stage 72.0 (TID 152) in 6 ms on localhost (3/4) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 72.0 (TID 150) in 6 ms on localhost (4/4) -2016-05-23 22:02:37 INFO DAGScheduler:59 - ResultStage 72 (foreach at StatisticsCorrLearning.scala:87) finished in 0.007 s -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 72.0, whose tasks have all completed, from pool -2016-05-23 22:02:37 INFO DAGScheduler:59 - Job 42 finished: foreach at StatisticsCorrLearning.scala:87, took 0.013927 s -2016-05-23 22:02:37 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:89 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Got job 43 (foreach at StatisticsCorrLearning.scala:89) with 4 output partitions -2016-05-23 22:02:37 INFO DAGScheduler:59 - Final stage: ResultStage 73(foreach at StatisticsCorrLearning.scala:89) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting ResultStage 73 (ParallelCollectionRDD[81] at parallelize at StatisticsCorrLearning.scala:85), which has no missing parents -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(1232) called with curMem=429652, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 1232.0 B, free 730.2 MB) -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(894) called with curMem=430884, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 894.0 B, free 730.2 MB) -2016-05-23 22:02:37 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:53923 (size: 894.0 B, free: 730.5 MB) -2016-05-23 22:02:37 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 73 (ParallelCollectionRDD[81] at parallelize at StatisticsCorrLearning.scala:85) -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Adding task set 73.0 with 4 tasks -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 73.0 (TID 154, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 73.0 (TID 155, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 2.0 in stage 73.0 (TID 156, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 3.0 in stage 73.0 (TID 157, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 22:02:37 INFO Executor:59 - Running task 0.0 in stage 73.0 (TID 154) -2016-05-23 22:02:37 INFO Executor:59 - Running task 1.0 in stage 73.0 (TID 155) -2016-05-23 22:02:37 INFO Executor:59 - Running task 3.0 in stage 73.0 (TID 157) -2016-05-23 22:02:37 INFO Executor:59 - Running task 2.0 in stage 73.0 (TID 156) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 0.0 in stage 73.0 (TID 154). 915 bytes result sent to driver -2016-05-23 22:02:37 INFO Executor:59 - Finished task 3.0 in stage 73.0 (TID 157). 915 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 73.0 (TID 154) in 5 ms on localhost (1/4) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 1.0 in stage 73.0 (TID 155). 915 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 3.0 in stage 73.0 (TID 157) in 5 ms on localhost (2/4) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 2.0 in stage 73.0 (TID 156). 915 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 73.0 (TID 155) in 5 ms on localhost (3/4) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 2.0 in stage 73.0 (TID 156) in 6 ms on localhost (4/4) -2016-05-23 22:02:37 INFO DAGScheduler:59 - ResultStage 73 (foreach at StatisticsCorrLearning.scala:89) finished in 0.007 s -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 73.0, whose tasks have all completed, from pool -2016-05-23 22:02:37 INFO DAGScheduler:59 - Job 43 finished: foreach at StatisticsCorrLearning.scala:89, took 0.014783 s -2016-05-23 22:02:37 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Got job 44 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:02:37 INFO DAGScheduler:59 - Final stage: ResultStage 74(first at RowMatrix.scala:65) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting ResultStage 74 (MapPartitionsRDD[83] at map at Correlation.scala:45), which has no missing parents -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=431778, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=434282, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 22:02:37 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:53923 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 22:02:37 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 74 (MapPartitionsRDD[83] at map at Correlation.scala:45) -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Adding task set 74.0 with 1 tasks -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 74.0 (TID 158, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:37 INFO Executor:59 - Running task 0.0 in stage 74.0 (TID 158) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 0.0 in stage 74.0 (TID 158). 1046 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 74.0 (TID 158) in 4 ms on localhost (1/1) -2016-05-23 22:02:37 INFO DAGScheduler:59 - ResultStage 74 (first at RowMatrix.scala:65) finished in 0.004 s -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 74.0, whose tasks have all completed, from pool -2016-05-23 22:02:37 INFO DAGScheduler:59 - Job 44 finished: first at RowMatrix.scala:65, took 0.013676 s -2016-05-23 22:02:37 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Got job 45 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 22:02:37 INFO DAGScheduler:59 - Final stage: ResultStage 75(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting ResultStage 75 (MapPartitionsRDD[84] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=435806, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=439206, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.2 MB) -2016-05-23 22:02:37 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:53923 (size: 1935.0 B, free: 730.5 MB) -2016-05-23 22:02:37 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 75 (MapPartitionsRDD[84] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Adding task set 75.0 with 4 tasks -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 75.0 (TID 159, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 75.0 (TID 160, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 2.0 in stage 75.0 (TID 161, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 3.0 in stage 75.0 (TID 162, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:02:37 INFO Executor:59 - Running task 0.0 in stage 75.0 (TID 159) -2016-05-23 22:02:37 INFO Executor:59 - Running task 3.0 in stage 75.0 (TID 162) -2016-05-23 22:02:37 INFO Executor:59 - Running task 2.0 in stage 75.0 (TID 161) -2016-05-23 22:02:37 INFO Executor:59 - Running task 1.0 in stage 75.0 (TID 160) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 0.0 in stage 75.0 (TID 159). 1231 bytes result sent to driver -2016-05-23 22:02:37 INFO Executor:59 - Finished task 2.0 in stage 75.0 (TID 161). 1231 bytes result sent to driver -2016-05-23 22:02:37 INFO Executor:59 - Finished task 3.0 in stage 75.0 (TID 162). 1231 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 75.0 (TID 159) in 6 ms on localhost (1/4) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 1.0 in stage 75.0 (TID 160). 1231 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 2.0 in stage 75.0 (TID 161) in 6 ms on localhost (2/4) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 75.0 (TID 160) in 8 ms on localhost (3/4) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 3.0 in stage 75.0 (TID 162) in 9 ms on localhost (4/4) -2016-05-23 22:02:37 INFO DAGScheduler:59 - ResultStage 75 (treeAggregate at RowMatrix.scala:332) finished in 0.010 s -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 75.0, whose tasks have all completed, from pool -2016-05-23 22:02:37 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at RowMatrix.scala:332, took 0.021111 s -2016-05-23 22:02:37 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Got job 46 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 22:02:37 INFO DAGScheduler:59 - Final stage: ResultStage 76(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting ResultStage 76 (MapPartitionsRDD[85] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=441141, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(1919) called with curMem=444581, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 1919.0 B, free 730.2 MB) -2016-05-23 22:02:37 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:53923 (size: 1919.0 B, free: 730.5 MB) -2016-05-23 22:02:37 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 76 (MapPartitionsRDD[85] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 4 tasks -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 163, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 164, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 2.0 in stage 76.0 (TID 165, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 3.0 in stage 76.0 (TID 166, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:02:37 INFO Executor:59 - Running task 3.0 in stage 76.0 (TID 166) -2016-05-23 22:02:37 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 164) -2016-05-23 22:02:37 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 163) -2016-05-23 22:02:37 INFO Executor:59 - Running task 2.0 in stage 76.0 (TID 165) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 3.0 in stage 76.0 (TID 166). 1109 bytes result sent to driver -2016-05-23 22:02:37 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 163). 1109 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 3.0 in stage 76.0 (TID 166) in 4 ms on localhost (1/4) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 164). 1109 bytes result sent to driver -2016-05-23 22:02:37 INFO Executor:59 - Finished task 2.0 in stage 76.0 (TID 165). 1109 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 163) in 8 ms on localhost (2/4) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 164) in 9 ms on localhost (3/4) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 2.0 in stage 76.0 (TID 165) in 10 ms on localhost (4/4) -2016-05-23 22:02:37 INFO DAGScheduler:59 - ResultStage 76 (treeAggregate at RowMatrix.scala:124) finished in 0.011 s -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-05-23 22:02:37 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at RowMatrix.scala:124, took 0.021282 s -2016-05-23 22:02:37 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Got job 47 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 22:02:37 INFO DAGScheduler:59 - Final stage: ResultStage 77(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[91] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=446500, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(1862) called with curMem=449876, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 1862.0 B, free 730.2 MB) -2016-05-23 22:02:37 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:53923 (size: 1862.0 B, free: 730.5 MB) -2016-05-23 22:02:37 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 77 (MapPartitionsRDD[91] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 4 tasks -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 167, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 168, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 2.0 in stage 77.0 (TID 169, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 3.0 in stage 77.0 (TID 170, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:02:37 INFO Executor:59 - Running task 3.0 in stage 77.0 (TID 170) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 3.0 in stage 77.0 (TID 170). 1294 bytes result sent to driver -2016-05-23 22:02:37 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 168) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 168). 1254 bytes result sent to driver -2016-05-23 22:02:37 INFO Executor:59 - Running task 2.0 in stage 77.0 (TID 169) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 2.0 in stage 77.0 (TID 169). 1249 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 3.0 in stage 77.0 (TID 170) in 16 ms on localhost (1/4) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 168) in 16 ms on localhost (2/4) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 2.0 in stage 77.0 (TID 169) in 16 ms on localhost (3/4) -2016-05-23 22:02:37 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 167) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 167). 1254 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 167) in 16 ms on localhost (4/4) -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-05-23 22:02:37 INFO DAGScheduler:59 - ResultStage 77 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.016 s -2016-05-23 22:02:37 INFO DAGScheduler:59 - Job 47 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.039193 s -2016-05-23 22:02:37 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Registering RDD 89 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Got job 48 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 22:02:37 INFO DAGScheduler:59 - Final stage: ResultStage 79(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 78) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 78) -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 78 (MapPartitionsRDD[89] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=451738, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 22:02:37 INFO MemoryStore:59 - ensureFreeSpace(2289) called with curMem=455778, maxMem=766075207 -2016-05-23 22:02:37 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 22:02:37 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:53923 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 22:02:37 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:37 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 78 (MapPartitionsRDD[89] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:02:37 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 4 tasks -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 171, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 172, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 2.0 in stage 78.0 (TID 173, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Starting task 3.0 in stage 78.0 (TID 174, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 22:02:37 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 171) -2016-05-23 22:02:37 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 172) -2016-05-23 22:02:37 INFO Executor:59 - Running task 2.0 in stage 78.0 (TID 173) -2016-05-23 22:02:37 INFO Executor:59 - Running task 3.0 in stage 78.0 (TID 174) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 171). 1161 bytes result sent to driver -2016-05-23 22:02:37 INFO Executor:59 - Finished task 2.0 in stage 78.0 (TID 173). 1161 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 171) in 121 ms on localhost (1/4) -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 2.0 in stage 78.0 (TID 173) in 125 ms on localhost (2/4) -2016-05-23 22:02:37 INFO Executor:59 - Finished task 3.0 in stage 78.0 (TID 174). 1161 bytes result sent to driver -2016-05-23 22:02:37 INFO TaskSetManager:59 - Finished task 3.0 in stage 78.0 (TID 174) in 130 ms on localhost (3/4) -2016-05-23 22:02:38 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 172). 1161 bytes result sent to driver -2016-05-23 22:02:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 172) in 243 ms on localhost (4/4) -2016-05-23 22:02:38 INFO DAGScheduler:59 - ShuffleMapStage 78 (flatMap at SpearmanCorrelation.scala:49) finished in 0.244 s -2016-05-23 22:02:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-05-23 22:02:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:02:38 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:02:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 79) -2016-05-23 22:02:38 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:02:38 INFO DAGScheduler:59 - Missing parents for ResultStage 79: List() -2016-05-23 22:02:38 INFO DAGScheduler:59 - Submitting ResultStage 79 (ShuffledRDD[92] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 22:02:38 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=458067, maxMem=766075207 -2016-05-23 22:02:38 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 22:02:38 INFO MemoryStore:59 - ensureFreeSpace(1573) called with curMem=460611, maxMem=766075207 -2016-05-23 22:02:38 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 1573.0 B, free 730.1 MB) -2016-05-23 22:02:38 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:53923 (size: 1573.0 B, free: 730.5 MB) -2016-05-23 22:02:38 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:38 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 79 (ShuffledRDD[92] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:02:38 INFO TaskSchedulerImpl:59 - Adding task set 79.0 with 3 tasks -2016-05-23 22:02:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 79.0 (TID 175, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 79.0 (TID 176, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:38 INFO TaskSetManager:59 - Starting task 2.0 in stage 79.0 (TID 177, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:38 INFO Executor:59 - Running task 2.0 in stage 79.0 (TID 177) -2016-05-23 22:02:38 INFO Executor:59 - Running task 1.0 in stage 79.0 (TID 176) -2016-05-23 22:02:38 INFO Executor:59 - Running task 0.0 in stage 79.0 (TID 175) -2016-05-23 22:02:38 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:38 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:38 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:38 INFO Executor:59 - Finished task 2.0 in stage 79.0 (TID 177). 1203 bytes result sent to driver -2016-05-23 22:02:38 INFO TaskSetManager:59 - Finished task 2.0 in stage 79.0 (TID 177) in 21 ms on localhost (1/3) -2016-05-23 22:02:38 INFO Executor:59 - Finished task 0.0 in stage 79.0 (TID 175). 1203 bytes result sent to driver -2016-05-23 22:02:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 79.0 (TID 175) in 26 ms on localhost (2/3) -2016-05-23 22:02:38 INFO Executor:59 - Finished task 1.0 in stage 79.0 (TID 176). 1203 bytes result sent to driver -2016-05-23 22:02:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 79.0 (TID 176) in 26 ms on localhost (3/3) -2016-05-23 22:02:38 INFO DAGScheduler:59 - ResultStage 79 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.028 s -2016-05-23 22:02:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 79.0, whose tasks have all completed, from pool -2016-05-23 22:02:38 INFO DAGScheduler:59 - Job 48 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.474541 s -2016-05-23 22:02:38 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:02:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 166 bytes -2016-05-23 22:02:38 INFO DAGScheduler:59 - Registering RDD 94 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:02:38 INFO DAGScheduler:59 - Got job 49 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:02:38 INFO DAGScheduler:59 - Final stage: ResultStage 82(first at RowMatrix.scala:65) -2016-05-23 22:02:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 81) -2016-05-23 22:02:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 81) -2016-05-23 22:02:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 81 (MapPartitionsRDD[94] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 22:02:38 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=462184, maxMem=766075207 -2016-05-23 22:02:38 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 22:02:38 INFO MemoryStore:59 - ensureFreeSpace(2313) called with curMem=466416, maxMem=766075207 -2016-05-23 22:02:38 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-23 22:02:38 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:53923 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:02:38 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:38 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 81 (MapPartitionsRDD[94] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:02:38 INFO TaskSchedulerImpl:59 - Adding task set 81.0 with 4 tasks -2016-05-23 22:02:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 81.0 (TID 178, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:02:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 81.0 (TID 179, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:02:38 INFO TaskSetManager:59 - Starting task 2.0 in stage 81.0 (TID 180, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:02:38 INFO TaskSetManager:59 - Starting task 3.0 in stage 81.0 (TID 181, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:02:38 INFO Executor:59 - Running task 1.0 in stage 81.0 (TID 179) -2016-05-23 22:02:38 INFO Executor:59 - Running task 3.0 in stage 81.0 (TID 181) -2016-05-23 22:02:38 INFO Executor:59 - Running task 0.0 in stage 81.0 (TID 178) -2016-05-23 22:02:38 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:38 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:38 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 15 ms -2016-05-23 22:02:38 INFO Executor:59 - Running task 2.0 in stage 81.0 (TID 180) -2016-05-23 22:02:38 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:39 INFO Executor:59 - Finished task 3.0 in stage 81.0 (TID 181). 1377 bytes result sent to driver -2016-05-23 22:02:39 INFO TaskSetManager:59 - Finished task 3.0 in stage 81.0 (TID 181) in 185 ms on localhost (1/4) -2016-05-23 22:02:39 INFO Executor:59 - Finished task 0.0 in stage 81.0 (TID 178). 1377 bytes result sent to driver -2016-05-23 22:02:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 81.0 (TID 178) in 185 ms on localhost (2/4) -2016-05-23 22:02:39 INFO Executor:59 - Finished task 2.0 in stage 81.0 (TID 180). 1377 bytes result sent to driver -2016-05-23 22:02:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 81.0 (TID 180) in 350 ms on localhost (3/4) -2016-05-23 22:02:39 INFO Executor:59 - Finished task 1.0 in stage 81.0 (TID 179). 1377 bytes result sent to driver -2016-05-23 22:02:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 81.0 (TID 179) in 431 ms on localhost (4/4) -2016-05-23 22:02:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 81.0, whose tasks have all completed, from pool -2016-05-23 22:02:39 INFO DAGScheduler:59 - ShuffleMapStage 81 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.431 s -2016-05-23 22:02:39 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:02:39 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:02:39 INFO DAGScheduler:59 - waiting: Set(ResultStage 82) -2016-05-23 22:02:39 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:02:39 INFO DAGScheduler:59 - Missing parents for ResultStage 82: List() -2016-05-23 22:02:39 INFO DAGScheduler:59 - Submitting ResultStage 82 (MapPartitionsRDD[96] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 22:02:39 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=468729, maxMem=766075207 -2016-05-23 22:02:39 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-05-23 22:02:39 INFO MemoryStore:59 - ensureFreeSpace(2562) called with curMem=473625, maxMem=766075207 -2016-05-23 22:02:39 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:53923 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:02:39 INFO SparkContext:59 - Created broadcast 61 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:39 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 82 (MapPartitionsRDD[96] at map at SpearmanCorrelation.scala:87) -2016-05-23 22:02:39 INFO TaskSchedulerImpl:59 - Adding task set 82.0 with 1 tasks -2016-05-23 22:02:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 82.0 (TID 182, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:39 INFO Executor:59 - Running task 0.0 in stage 82.0 (TID 182) -2016-05-23 22:02:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:39 INFO Executor:59 - Finished task 0.0 in stage 82.0 (TID 182). 1296 bytes result sent to driver -2016-05-23 22:02:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 82.0 (TID 182) in 24 ms on localhost (1/1) -2016-05-23 22:02:39 INFO DAGScheduler:59 - ResultStage 82 (first at RowMatrix.scala:65) finished in 0.024 s -2016-05-23 22:02:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 82.0, whose tasks have all completed, from pool -2016-05-23 22:02:39 INFO DAGScheduler:59 - Job 49 finished: first at RowMatrix.scala:65, took 1.244061 s -2016-05-23 22:02:39 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:02:39 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 166 bytes -2016-05-23 22:02:39 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 169 bytes -2016-05-23 22:02:39 INFO DAGScheduler:59 - Got job 50 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 22:02:39 INFO DAGScheduler:59 - Final stage: ResultStage 85(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 84) -2016-05-23 22:02:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:39 INFO DAGScheduler:59 - Submitting ResultStage 85 (MapPartitionsRDD[97] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:02:39 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=476187, maxMem=766075207 -2016-05-23 22:02:39 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 22:02:39 INFO MemoryStore:59 - ensureFreeSpace(2943) called with curMem=481883, maxMem=766075207 -2016-05-23 22:02:39 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.1 MB) -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:53923 (size: 2.9 KB, free: 730.5 MB) -2016-05-23 22:02:39 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:39 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 85 (MapPartitionsRDD[97] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:39 INFO TaskSchedulerImpl:59 - Adding task set 85.0 with 4 tasks -2016-05-23 22:02:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 85.0 (TID 183, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 85.0 (TID 184, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 85.0 (TID 185, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:39 INFO TaskSetManager:59 - Starting task 3.0 in stage 85.0 (TID 186, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:39 INFO Executor:59 - Running task 0.0 in stage 85.0 (TID 183) -2016-05-23 22:02:39 INFO Executor:59 - Running task 1.0 in stage 85.0 (TID 184) -2016-05-23 22:02:39 INFO Executor:59 - Running task 3.0 in stage 85.0 (TID 186) -2016-05-23 22:02:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:39 INFO Executor:59 - Running task 2.0 in stage 85.0 (TID 185) -2016-05-23 22:02:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:39 INFO Executor:59 - Finished task 0.0 in stage 85.0 (TID 183). 1481 bytes result sent to driver -2016-05-23 22:02:39 INFO Executor:59 - Finished task 2.0 in stage 85.0 (TID 185). 1481 bytes result sent to driver -2016-05-23 22:02:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 16 ms -2016-05-23 22:02:39 INFO Executor:59 - Finished task 1.0 in stage 85.0 (TID 184). 1481 bytes result sent to driver -2016-05-23 22:02:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 85.0 (TID 183) in 16 ms on localhost (1/4) -2016-05-23 22:02:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 85.0 (TID 185) in 16 ms on localhost (2/4) -2016-05-23 22:02:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 85.0 (TID 184) in 16 ms on localhost (3/4) -2016-05-23 22:02:39 INFO Executor:59 - Finished task 3.0 in stage 85.0 (TID 186). 1481 bytes result sent to driver -2016-05-23 22:02:39 INFO TaskSetManager:59 - Finished task 3.0 in stage 85.0 (TID 186) in 16 ms on localhost (4/4) -2016-05-23 22:02:39 INFO DAGScheduler:59 - ResultStage 85 (treeAggregate at RowMatrix.scala:332) finished in 0.016 s -2016-05-23 22:02:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 85.0, whose tasks have all completed, from pool -2016-05-23 22:02:39 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at RowMatrix.scala:332, took 0.027183 s -2016-05-23 22:02:39 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:02:39 INFO DAGScheduler:59 - Got job 51 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 22:02:39 INFO DAGScheduler:59 - Final stage: ResultStage 88(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:02:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 87) -2016-05-23 22:02:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:39 INFO DAGScheduler:59 - Submitting ResultStage 88 (MapPartitionsRDD[98] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:02:39 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=484826, maxMem=766075207 -2016-05-23 22:02:39 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-23 22:02:39 INFO MemoryStore:59 - ensureFreeSpace(2920) called with curMem=490562, maxMem=766075207 -2016-05-23 22:02:39 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.1 MB) -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:53923 (size: 2.9 KB, free: 730.5 MB) -2016-05-23 22:02:39 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:39 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 88 (MapPartitionsRDD[98] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:02:39 INFO TaskSchedulerImpl:59 - Adding task set 88.0 with 4 tasks -2016-05-23 22:02:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 88.0 (TID 187, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 88.0 (TID 188, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 88.0 (TID 189, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:39 INFO TaskSetManager:59 - Starting task 3.0 in stage 88.0 (TID 190, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:02:39 INFO Executor:59 - Running task 3.0 in stage 88.0 (TID 190) -2016-05-23 22:02:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:39 INFO Executor:59 - Finished task 3.0 in stage 88.0 (TID 190). 1359 bytes result sent to driver -2016-05-23 22:02:39 INFO Executor:59 - Running task 1.0 in stage 88.0 (TID 188) -2016-05-23 22:02:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:39 INFO Executor:59 - Running task 2.0 in stage 88.0 (TID 189) -2016-05-23 22:02:39 INFO Executor:59 - Running task 0.0 in stage 88.0 (TID 187) -2016-05-23 22:02:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:02:39 INFO TaskSetManager:59 - Finished task 3.0 in stage 88.0 (TID 190) in 16 ms on localhost (1/4) -2016-05-23 22:02:39 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:02:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:02:39 INFO Executor:59 - Finished task 2.0 in stage 88.0 (TID 189). 1359 bytes result sent to driver -2016-05-23 22:02:39 INFO Executor:59 - Finished task 0.0 in stage 88.0 (TID 187). 1359 bytes result sent to driver -2016-05-23 22:02:39 INFO Executor:59 - Finished task 1.0 in stage 88.0 (TID 188). 1359 bytes result sent to driver -2016-05-23 22:02:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 88.0 (TID 189) in 56 ms on localhost (2/4) -2016-05-23 22:02:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 88.0 (TID 188) in 58 ms on localhost (3/4) -2016-05-23 22:02:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 88.0 (TID 187) in 58 ms on localhost (4/4) -2016-05-23 22:02:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 88.0, whose tasks have all completed, from pool -2016-05-23 22:02:39 INFO DAGScheduler:59 - ResultStage 88 (treeAggregate at RowMatrix.scala:124) finished in 0.059 s -2016-05-23 22:02:39 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at RowMatrix.scala:124, took 0.077017 s -2016-05-23 22:02:39 INFO MemoryStore:59 - ensureFreeSpace(88528) called with curMem=493482, maxMem=766075207 -2016-05-23 22:02:39 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 86.5 KB, free 730.0 MB) -2016-05-23 22:02:39 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=582010, maxMem=766075207 -2016-05-23 22:02:39 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.0 MB) -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:53923 (size: 19.3 KB, free: 730.5 MB) -2016-05-23 22:02:39 INFO SparkContext:59 - Created broadcast 64 from textFile at StatisticsCorrLearning.scala:98 -2016-05-23 22:02:39 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 22:02:39 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:102 -2016-05-23 22:02:39 INFO DAGScheduler:59 - Got job 52 (foreach at StatisticsCorrLearning.scala:102) with 3 output partitions -2016-05-23 22:02:39 INFO DAGScheduler:59 - Final stage: ResultStage 89(foreach at StatisticsCorrLearning.scala:102) -2016-05-23 22:02:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:39 INFO DAGScheduler:59 - Submitting ResultStage 89 (MapPartitionsRDD[102] at map at StatisticsCorrLearning.scala:101), which has no missing parents -2016-05-23 22:02:39 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=601798, maxMem=766075207 -2016-05-23 22:02:39 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-23 22:02:39 INFO MemoryStore:59 - ensureFreeSpace(1982) called with curMem=605310, maxMem=766075207 -2016-05-23 22:02:39 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 1982.0 B, free 730.0 MB) -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:53923 (size: 1982.0 B, free: 730.5 MB) -2016-05-23 22:02:39 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:39 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 89 (MapPartitionsRDD[102] at map at StatisticsCorrLearning.scala:101) -2016-05-23 22:02:39 INFO TaskSchedulerImpl:59 - Adding task set 89.0 with 3 tasks -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:53923 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-23 22:02:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 89.0 (TID 191, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:02:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 89.0 (TID 192, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:02:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 89.0 (TID 193, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:02:39 INFO Executor:59 - Running task 1.0 in stage 89.0 (TID 192) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-05-23 22:02:39 INFO Executor:59 - Running task 0.0 in stage 89.0 (TID 191) -2016-05-23 22:02:39 INFO Executor:59 - Running task 2.0 in stage 89.0 (TID 193) -2016-05-23 22:02:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningz.txt:4+4 -2016-05-23 22:02:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningz.txt:8+1 -2016-05-23 22:02:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningz.txt:0+4 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:53923 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-23 22:02:39 INFO Executor:59 - Finished task 1.0 in stage 89.0 (TID 192). 2044 bytes result sent to driver -2016-05-23 22:02:39 INFO Executor:59 - Finished task 2.0 in stage 89.0 (TID 193). 2044 bytes result sent to driver -2016-05-23 22:02:39 INFO Executor:59 - Finished task 0.0 in stage 89.0 (TID 191). 2044 bytes result sent to driver -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-05-23 22:02:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 89.0 (TID 193) in 12 ms on localhost (1/3) -2016-05-23 22:02:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 89.0 (TID 191) in 13 ms on localhost (2/3) -2016-05-23 22:02:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 89.0 (TID 192) in 13 ms on localhost (3/3) -2016-05-23 22:02:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 89.0, whose tasks have all completed, from pool -2016-05-23 22:02:39 INFO DAGScheduler:59 - ResultStage 89 (foreach at StatisticsCorrLearning.scala:102) finished in 0.014 s -2016-05-23 22:02:39 INFO DAGScheduler:59 - Job 52 finished: foreach at StatisticsCorrLearning.scala:102, took 0.039391 s -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:53923 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:53923 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-05-23 22:02:39 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:02:39 INFO DAGScheduler:59 - Got job 53 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:02:39 INFO DAGScheduler:59 - Final stage: ResultStage 90(first at RowMatrix.scala:65) -2016-05-23 22:02:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:39 INFO DAGScheduler:59 - Submitting ResultStage 90 (MapPartitionsRDD[102] at map at StatisticsCorrLearning.scala:101), which has no missing parents -2016-05-23 22:02:39 INFO MemoryStore:59 - ensureFreeSpace(3552) called with curMem=575994, maxMem=766075207 -2016-05-23 22:02:39 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-23 22:02:39 INFO MemoryStore:59 - ensureFreeSpace(2003) called with curMem=579546, maxMem=766075207 -2016-05-23 22:02:39 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2003.0 B, free 730.0 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned shuffle 9 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:53923 (size: 2003.0 B, free: 730.5 MB) -2016-05-23 22:02:39 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:39 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 90 (MapPartitionsRDD[102] at map at StatisticsCorrLearning.scala:101) -2016-05-23 22:02:39 INFO TaskSchedulerImpl:59 - Adding task set 90.0 with 1 tasks -2016-05-23 22:02:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 90.0 (TID 194, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:02:39 INFO Executor:59 - Running task 0.0 in stage 90.0 (TID 194) -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:53923 in memory (size: 1573.0 B, free: 730.5 MB) -2016-05-23 22:02:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningz.txt:0+4 -2016-05-23 22:02:39 INFO Executor:59 - Finished task 0.0 in stage 90.0 (TID 194). 2199 bytes result sent to driver -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:53923 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-05-23 22:02:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 90.0 (TID 194) in 19 ms on localhost (1/1) -2016-05-23 22:02:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 90.0, whose tasks have all completed, from pool -2016-05-23 22:02:39 INFO DAGScheduler:59 - ResultStage 90 (first at RowMatrix.scala:65) finished in 0.020 s -2016-05-23 22:02:39 INFO DAGScheduler:59 - Job 53 finished: first at RowMatrix.scala:65, took 0.033405 s -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned shuffle 8 -2016-05-23 22:02:39 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:02:39 INFO DAGScheduler:59 - Got job 54 (treeAggregate at RowMatrix.scala:332) with 3 output partitions -2016-05-23 22:02:39 INFO DAGScheduler:59 - Final stage: ResultStage 91(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:02:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:02:39 INFO DAGScheduler:59 - Submitting ResultStage 91 (MapPartitionsRDD[103] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:02:39 INFO MemoryStore:59 - ensureFreeSpace(4472) called with curMem=571103, maxMem=766075207 -2016-05-23 22:02:39 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 4.4 KB, free 730.0 MB) -2016-05-23 22:02:39 INFO MemoryStore:59 - ensureFreeSpace(2372) called with curMem=570337, maxMem=766075207 -2016-05-23 22:02:39 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.0 MB) -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:53923 in memory (size: 1862.0 B, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:53923 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:02:39 INFO SparkContext:59 - Created broadcast 67 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:02:39 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 91 (MapPartitionsRDD[103] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:02:39 INFO TaskSchedulerImpl:59 - Adding task set 91.0 with 3 tasks -2016-05-23 22:02:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 91.0 (TID 195, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:02:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 91.0 (TID 196, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:02:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 91.0 (TID 197, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:53923 in memory (size: 1919.0 B, free: 730.5 MB) -2016-05-23 22:02:39 INFO Executor:59 - Running task 1.0 in stage 91.0 (TID 196) -2016-05-23 22:02:39 INFO Executor:59 - Running task 2.0 in stage 91.0 (TID 197) -2016-05-23 22:02:39 INFO Executor:59 - Running task 0.0 in stage 91.0 (TID 195) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-05-23 22:02:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningz.txt:4+4 -2016-05-23 22:02:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningz.txt:8+1 -2016-05-23 22:02:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningz.txt:0+4 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:53923 in memory (size: 1935.0 B, free: 730.5 MB) -2016-05-23 22:02:39 INFO Executor:59 - Finished task 1.0 in stage 91.0 (TID 196). 2384 bytes result sent to driver -2016-05-23 22:02:39 INFO Executor:59 - Finished task 0.0 in stage 91.0 (TID 195). 2384 bytes result sent to driver -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-05-23 22:02:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 91.0 (TID 196) in 13 ms on localhost (1/3) -2016-05-23 22:02:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 91.0 (TID 195) in 14 ms on localhost (2/3) -2016-05-23 22:02:39 INFO Executor:59 - Finished task 2.0 in stage 91.0 (TID 197). 2384 bytes result sent to driver -2016-05-23 22:02:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 91.0 (TID 197) in 15 ms on localhost (3/3) -2016-05-23 22:02:39 INFO DAGScheduler:59 - ResultStage 91 (treeAggregate at RowMatrix.scala:332) finished in 0.017 s -2016-05-23 22:02:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 91.0, whose tasks have all completed, from pool -2016-05-23 22:02:39 INFO DAGScheduler:59 - Job 54 finished: treeAggregate at RowMatrix.scala:332, took 0.029299 s -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:53923 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:53923 in memory (size: 894.0 B, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:53923 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:53923 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:53923 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:53923 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:53923 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:53923 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned shuffle 7 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:53923 in memory (size: 1567.0 B, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:53923 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned shuffle 6 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:53923 in memory (size: 1858.0 B, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:53923 in memory (size: 1918.0 B, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:53923 in memory (size: 1934.0 B, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-23 22:02:39 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:53923 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:53923 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:53923 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-23 22:02:39 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:53923 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:02:39 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 22:02:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 22:02:40 WARN QueuedThreadPool:145 - 4 threads could not be stopped -2016-05-23 22:02:41 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 22:02:41 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 22:02:42 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 22:02:44 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 22:02:44 INFO BlockManager:59 - BlockManager stopped -2016-05-23 22:02:44 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 22:02:44 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 22:02:44 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 22:02:44 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 22:02:44 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-1bae0fac-a9bf-4fba-8ac2-21cd34cfe122 -2016-05-23 22:02:44 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 22:02:44 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 22:07:22 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 22:07:23 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 22:07:24 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 22:07:24 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 22:07:24 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 22:07:25 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 22:07:25 INFO Remoting:74 - Starting remoting -2016-05-23 22:07:26 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53963. -2016-05-23 22:07:26 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53963] -2016-05-23 22:07:26 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 22:07:26 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 22:07:26 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-edd5b454-08c8-46bb-9545-ea42d9d8107e -2016-05-23 22:07:26 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 22:07:26 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-c731b2d8-9b67-422c-88e0-b03b82fe281b\httpd-6bccd01c-7fba-423f-a63f-73cbf45db577 -2016-05-23 22:07:26 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 22:07:26 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:07:26 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53964 -2016-05-23 22:07:26 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53964. -2016-05-23 22:07:26 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 22:07:27 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:07:27 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 22:07:27 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 22:07:27 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 22:07:27 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 22:07:27 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 22:07:27 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53983. -2016-05-23 22:07:27 INFO NettyBlockTransferService:59 - Server created on 53983 -2016-05-23 22:07:27 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 22:07:27 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53983 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53983) -2016-05-23 22:07:27 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 22:07:28 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 22:07:28 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 22:07:28 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 22:07:28 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 22:07:28 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53983 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 22:07:28 INFO SparkContext:59 - Created broadcast 0 from textFile at StatisticsCorrLearning.scala:23 -2016-05-23 22:07:28 INFO MemoryStore:59 - ensureFreeSpace(157344) called with curMem=171580, maxMem=766075207 -2016-05-23 22:07:28 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 153.7 KB, free 730.3 MB) -2016-05-23 22:07:28 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=328924, maxMem=766075207 -2016-05-23 22:07:28 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.3 MB) -2016-05-23 22:07:28 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53983 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 22:07:28 INFO SparkContext:59 - Created broadcast 1 from textFile at StatisticsCorrLearning.scala:26 -2016-05-23 22:07:30 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 22:07:30 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 22:07:30 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:30 -2016-05-23 22:07:30 INFO DAGScheduler:59 - Got job 0 (foreach at StatisticsCorrLearning.scala:30) with 2 output partitions -2016-05-23 22:07:30 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StatisticsCorrLearning.scala:30) -2016-05-23 22:07:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:30 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:24), which has no missing parents -2016-05-23 22:07:30 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=343200, maxMem=766075207 -2016-05-23 22:07:30 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 22:07:30 INFO MemoryStore:59 - ensureFreeSpace(2047) called with curMem=346712, maxMem=766075207 -2016-05-23 22:07:30 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2047.0 B, free 730.3 MB) -2016-05-23 22:07:30 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53983 (size: 2047.0 B, free: 730.6 MB) -2016-05-23 22:07:30 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at flatMap at StatisticsCorrLearning.scala:24) -2016-05-23 22:07:30 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 22:07:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:07:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:07:30 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 22:07:30 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 22:07:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 22:07:30 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 22:07:30 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 22:07:30 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 22:07:30 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 22:07:30 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 22:07:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 22:07:30 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 22:07:30 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 22:07:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 188 ms on localhost (1/2) -2016-05-23 22:07:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 159 ms on localhost (2/2) -2016-05-23 22:07:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 22:07:30 INFO DAGScheduler:59 - ResultStage 0 (foreach at StatisticsCorrLearning.scala:30) finished in 0.235 s -2016-05-23 22:07:30 INFO DAGScheduler:59 - Job 0 finished: foreach at StatisticsCorrLearning.scala:30, took 0.515553 s -2016-05-23 22:07:30 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 22:07:30 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:32 -2016-05-23 22:07:30 INFO DAGScheduler:59 - Got job 1 (foreach at StatisticsCorrLearning.scala:32) with 2 output partitions -2016-05-23 22:07:30 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StatisticsCorrLearning.scala:32) -2016-05-23 22:07:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:30 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:27), which has no missing parents -2016-05-23 22:07:30 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=348759, maxMem=766075207 -2016-05-23 22:07:30 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-05-23 22:07:31 INFO MemoryStore:59 - ensureFreeSpace(2045) called with curMem=352271, maxMem=766075207 -2016-05-23 22:07:31 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2045.0 B, free 730.2 MB) -2016-05-23 22:07:31 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53983 (size: 2045.0 B, free: 730.6 MB) -2016-05-23 22:07:31 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at flatMap at StatisticsCorrLearning.scala:27) -2016-05-23 22:07:31 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 22:07:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:07:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:07:31 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 22:07:31 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 22:07:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 22:07:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 22:07:31 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 22:07:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 24 ms on localhost (1/2) -2016-05-23 22:07:31 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 22:07:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 33 ms on localhost (2/2) -2016-05-23 22:07:31 INFO DAGScheduler:59 - ResultStage 1 (foreach at StatisticsCorrLearning.scala:32) finished in 0.033 s -2016-05-23 22:07:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 22:07:31 INFO DAGScheduler:59 - Job 1 finished: foreach at StatisticsCorrLearning.scala:32, took 0.077530 s -2016-05-23 22:07:31 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:07:31 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:07:31 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-05-23 22:07:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:31 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45), which has no missing parents -2016-05-23 22:07:31 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=354316, maxMem=766075207 -2016-05-23 22:07:31 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-23 22:07:31 INFO MemoryStore:59 - ensureFreeSpace(2464) called with curMem=358876, maxMem=766075207 -2016-05-23 22:07:31 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 22:07:31 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53983 (size: 2.4 KB, free: 730.6 MB) -2016-05-23 22:07:31 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at Correlation.scala:45) -2016-05-23 22:07:31 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-05-23 22:07:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 22:07:31 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 22:07:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 22:07:31 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53983 in memory (size: 2045.0 B, free: 730.6 MB) -2016-05-23 22:07:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 22:07:31 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2175 bytes result sent to driver -2016-05-23 22:07:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 39 ms on localhost (1/1) -2016-05-23 22:07:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 22:07:31 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-23 22:07:31 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.036 s -2016-05-23 22:07:31 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.231373 s -2016-05-23 22:07:31 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53983 in memory (size: 2047.0 B, free: 730.6 MB) -2016-05-23 22:07:31 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-23 22:07:31 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:07:31 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 22:07:31 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:31 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:07:31 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=350224, maxMem=766075207 -2016-05-23 22:07:31 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.3 KB, free 730.2 MB) -2016-05-23 22:07:31 INFO MemoryStore:59 - ensureFreeSpace(2861) called with curMem=355672, maxMem=766075207 -2016-05-23 22:07:31 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 22:07:31 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53983 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 22:07:31 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:31 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 22:07:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 22:07:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 22:07:31 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-23 22:07:31 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-23 22:07:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 22:07:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 22:07:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 22:07:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 22:07:32 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2360 bytes result sent to driver -2016-05-23 22:07:32 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2360 bytes result sent to driver -2016-05-23 22:07:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 50 ms on localhost (1/2) -2016-05-23 22:07:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 56 ms on localhost (2/2) -2016-05-23 22:07:32 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.056 s -2016-05-23 22:07:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 22:07:32 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-23 22:07:32 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-23 22:07:32 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.088166 s -2016-05-23 22:07:32 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:07:32 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 22:07:32 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:32 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:32 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:07:32 INFO MemoryStore:59 - ensureFreeSpace(5496) called with curMem=358533, maxMem=766075207 -2016-05-23 22:07:32 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.2 MB) -2016-05-23 22:07:32 INFO MemoryStore:59 - ensureFreeSpace(2850) called with curMem=364029, maxMem=766075207 -2016-05-23 22:07:32 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 22:07:32 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53983 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 22:07:32 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:32 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 22:07:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 22:07:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 22:07:32 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-23 22:07:32 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-23 22:07:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 22:07:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 22:07:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 22:07:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 22:07:32 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2238 bytes result sent to driver -2016-05-23 22:07:32 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2238 bytes result sent to driver -2016-05-23 22:07:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 42 ms on localhost (1/2) -2016-05-23 22:07:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 43 ms on localhost (2/2) -2016-05-23 22:07:32 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.046 s -2016-05-23 22:07:32 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.095727 s -2016-05-23 22:07:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 22:07:32 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 22:07:32 INFO DAGScheduler:59 - Got job 5 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 22:07:32 INFO DAGScheduler:59 - Final stage: ResultStage 5(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:07:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:32 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:32 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 22:07:32 INFO MemoryStore:59 - ensureFreeSpace(5344) called with curMem=366879, maxMem=766075207 -2016-05-23 22:07:32 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-23 22:07:32 INFO MemoryStore:59 - ensureFreeSpace(2743) called with curMem=372223, maxMem=766075207 -2016-05-23 22:07:32 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-05-23 22:07:32 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53983 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 22:07:32 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:07:32 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-23 22:07:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 22:07:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2568 bytes) -2016-05-23 22:07:32 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-23 22:07:32 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-23 22:07:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 22:07:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 22:07:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 22:07:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 22:07:32 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2543 bytes result sent to driver -2016-05-23 22:07:32 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2543 bytes result sent to driver -2016-05-23 22:07:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 49 ms on localhost (1/2) -2016-05-23 22:07:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 53 ms on localhost (2/2) -2016-05-23 22:07:32 INFO DAGScheduler:59 - ResultStage 5 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.053 s -2016-05-23 22:07:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 22:07:32 INFO DAGScheduler:59 - Job 5 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.088233 s -2016-05-23 22:07:32 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 22:07:32 INFO DAGScheduler:59 - Registering RDD 13 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:07:32 INFO DAGScheduler:59 - Got job 6 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 22:07:32 INFO DAGScheduler:59 - Final stage: ResultStage 7(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 22:07:32 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-23 22:07:32 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-23 22:07:32 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 22:07:32 INFO MemoryStore:59 - ensureFreeSpace(5928) called with curMem=374966, maxMem=766075207 -2016-05-23 22:07:32 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-05-23 22:07:32 INFO MemoryStore:59 - ensureFreeSpace(3155) called with curMem=380894, maxMem=766075207 -2016-05-23 22:07:32 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-23 22:07:32 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53983 (size: 3.1 KB, free: 730.5 MB) -2016-05-23 22:07:32 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:07:32 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-23 22:07:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 22:07:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2557 bytes) -2016-05-23 22:07:32 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-23 22:07:32 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-23 22:07:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 22:07:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 22:07:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:10+11 -2016-05-23 22:07:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningy.txt:0+10 -2016-05-23 22:07:32 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-23 22:07:32 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-23 22:07:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 96 ms on localhost (1/2) -2016-05-23 22:07:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 101 ms on localhost (2/2) -2016-05-23 22:07:32 INFO DAGScheduler:59 - ShuffleMapStage 6 (flatMap at SpearmanCorrelation.scala:49) finished in 0.102 s -2016-05-23 22:07:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-23 22:07:32 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:07:32 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:07:32 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-23 22:07:32 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:07:32 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-23 22:07:32 INFO DAGScheduler:59 - Submitting ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 22:07:32 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=384049, maxMem=766075207 -2016-05-23 22:07:32 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 22:07:32 INFO MemoryStore:59 - ensureFreeSpace(1545) called with curMem=386553, maxMem=766075207 -2016-05-23 22:07:32 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1545.0 B, free 730.2 MB) -2016-05-23 22:07:32 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53983 (size: 1545.0 B, free: 730.5 MB) -2016-05-23 22:07:32 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (ShuffledRDD[16] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:07:32 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-23 22:07:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:32 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-23 22:07:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:07:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-23 22:07:32 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 1203 bytes result sent to driver -2016-05-23 22:07:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 59 ms on localhost (1/1) -2016-05-23 22:07:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-23 22:07:32 INFO DAGScheduler:59 - ResultStage 7 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.060 s -2016-05-23 22:07:32 INFO DAGScheduler:59 - Job 6 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.222310 s -2016-05-23 22:07:33 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53983 in memory (size: 1545.0 B, free: 730.5 MB) -2016-05-23 22:07:33 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:07:33 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-23 22:07:33 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53983 in memory (size: 3.1 KB, free: 730.5 MB) -2016-05-23 22:07:33 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-23 22:07:33 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 22:07:33 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53983 in memory (size: 2.7 KB, free: 730.6 MB) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:07:33 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Got job 7 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:07:33 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at RowMatrix.scala:65) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 22:07:33 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53983 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 22:07:33 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=358533, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 22:07:33 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53983 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=359864, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 22:07:33 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 22:07:33 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53983 (size: 2.2 KB, free: 730.6 MB) -2016-05-23 22:07:33 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:33 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53983 in memory (size: 2.4 KB, free: 730.6 MB) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[18] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-23 22:07:33 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 14, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:07:33 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 14) -2016-05-23 22:07:33 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 15) -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:33 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 14). 1375 bytes result sent to driver -2016-05-23 22:07:33 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 15). 1375 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 14) in 38 ms on localhost (1/2) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 15) in 38 ms on localhost (2/2) -2016-05-23 22:07:33 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.040 s -2016-05-23 22:07:33 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-23 22:07:33 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:07:33 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-23 22:07:33 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:07:33 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=349674, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=354530, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 22:07:33 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53983 (size: 2.5 KB, free: 730.6 MB) -2016-05-23 22:07:33 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at map at SpearmanCorrelation.scala:87) -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:33 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:33 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 1296 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 22 ms on localhost (1/1) -2016-05-23 22:07:33 INFO DAGScheduler:59 - ResultStage 10 (first at RowMatrix.scala:65) finished in 0.024 s -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-23 22:07:33 INFO DAGScheduler:59 - Job 7 finished: first at RowMatrix.scala:65, took 0.102228 s -2016-05-23 22:07:33 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:07:33 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 152 bytes -2016-05-23 22:07:33 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-23 22:07:33 INFO DAGScheduler:59 - Got job 8 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 22:07:33 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=357061, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(2908) called with curMem=362717, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 22:07:33 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53983 (size: 2.8 KB, free: 730.6 MB) -2016-05-23 22:07:33 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[21] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:33 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 17) -2016-05-23 22:07:33 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 18) -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:33 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 17). 1481 bytes result sent to driver -2016-05-23 22:07:33 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 18). 1481 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 17) in 18 ms on localhost (1/2) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 18) in 18 ms on localhost (2/2) -2016-05-23 22:07:33 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at RowMatrix.scala:332) finished in 0.019 s -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-23 22:07:33 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at RowMatrix.scala:332, took 0.034608 s -2016-05-23 22:07:33 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Got job 9 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 22:07:33 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=365625, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(2889) called with curMem=371321, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 22:07:33 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53983 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:07:33 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[22] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:33 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 19) -2016-05-23 22:07:33 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 20) -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:33 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 20). 1359 bytes result sent to driver -2016-05-23 22:07:33 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 19). 1359 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 20) in 20 ms on localhost (1/2) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 19) in 23 ms on localhost (2/2) -2016-05-23 22:07:33 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at RowMatrix.scala:124) finished in 0.023 s -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-23 22:07:33 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at RowMatrix.scala:124, took 0.040283 s -2016-05-23 22:07:33 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:46 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Got job 10 (foreach at StatisticsCorrLearning.scala:46) with 4 output partitions -2016-05-23 22:07:33 INFO DAGScheduler:59 - Final stage: ResultStage 17(foreach at StatisticsCorrLearning.scala:46) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:43), which has no missing parents -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=374210, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=375434, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 22:07:33 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53983 (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:07:33 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (ParallelCollectionRDD[23] at parallelize at StatisticsCorrLearning.scala:43) -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 21, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 22, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 23, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 24, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 22:07:33 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 21) -2016-05-23 22:07:33 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 22) -2016-05-23 22:07:33 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 24) -2016-05-23 22:07:33 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 23) -2016-05-23 22:07:33 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 24). 915 bytes result sent to driver -2016-05-23 22:07:33 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 21). 915 bytes result sent to driver -2016-05-23 22:07:33 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 22). 915 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 24) in 9 ms on localhost (1/4) -2016-05-23 22:07:33 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 23). 915 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 21) in 20 ms on localhost (2/4) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 22) in 14 ms on localhost (3/4) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 23) in 13 ms on localhost (4/4) -2016-05-23 22:07:33 INFO DAGScheduler:59 - ResultStage 17 (foreach at StatisticsCorrLearning.scala:46) finished in 0.022 s -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-23 22:07:33 INFO DAGScheduler:59 - Job 10 finished: foreach at StatisticsCorrLearning.scala:46, took 0.036646 s -2016-05-23 22:07:33 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:48 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Got job 11 (foreach at StatisticsCorrLearning.scala:48) with 4 output partitions -2016-05-23 22:07:33 INFO DAGScheduler:59 - Final stage: ResultStage 18(foreach at StatisticsCorrLearning.scala:48) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:44), which has no missing parents -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=376327, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=377551, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 22:07:33 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53983 (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:07:33 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (ParallelCollectionRDD[24] at parallelize at StatisticsCorrLearning.scala:44) -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 25, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 26, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 27, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 28, localhost, PROCESS_LOCAL, 2052 bytes) -2016-05-23 22:07:33 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 25) -2016-05-23 22:07:33 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 28) -2016-05-23 22:07:33 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 27) -2016-05-23 22:07:33 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 26) -2016-05-23 22:07:33 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 28). 915 bytes result sent to driver -2016-05-23 22:07:33 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 27). 915 bytes result sent to driver -2016-05-23 22:07:33 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 26). 915 bytes result sent to driver -2016-05-23 22:07:33 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 25). 915 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 28) in 9 ms on localhost (1/4) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 27) in 11 ms on localhost (2/4) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 25) in 14 ms on localhost (3/4) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 26) in 14 ms on localhost (4/4) -2016-05-23 22:07:33 INFO DAGScheduler:59 - ResultStage 18 (foreach at StatisticsCorrLearning.scala:48) finished in 0.015 s -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-23 22:07:33 INFO DAGScheduler:59 - Job 11 finished: foreach at StatisticsCorrLearning.scala:48, took 0.026039 s -2016-05-23 22:07:33 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Got job 12 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:07:33 INFO DAGScheduler:59 - Final stage: ResultStage 19(first at RowMatrix.scala:65) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45), which has no missing parents -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=378444, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=380948, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 22:07:33 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53983 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 22:07:33 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[26] at map at Correlation.scala:45) -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 29, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:07:33 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 29) -2016-05-23 22:07:33 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 29). 1046 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 29) in 7 ms on localhost (1/1) -2016-05-23 22:07:33 INFO DAGScheduler:59 - ResultStage 19 (first at RowMatrix.scala:65) finished in 0.007 s -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-23 22:07:33 INFO DAGScheduler:59 - Job 12 finished: first at RowMatrix.scala:65, took 0.020661 s -2016-05-23 22:07:33 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Got job 13 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 22:07:33 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=382472, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=385872, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 22:07:33 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53983 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 22:07:33 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[27] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 30, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 31, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 32, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 33, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 22:07:33 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 32) -2016-05-23 22:07:33 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 31) -2016-05-23 22:07:33 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 33) -2016-05-23 22:07:33 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 30) -2016-05-23 22:07:33 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 33). 1231 bytes result sent to driver -2016-05-23 22:07:33 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 32). 1231 bytes result sent to driver -2016-05-23 22:07:33 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 30). 1231 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 33) in 11 ms on localhost (1/4) -2016-05-23 22:07:33 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 31). 1231 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 32) in 13 ms on localhost (2/4) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 30) in 17 ms on localhost (3/4) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 31) in 17 ms on localhost (4/4) -2016-05-23 22:07:33 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at RowMatrix.scala:332) finished in 0.019 s -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-23 22:07:33 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at RowMatrix.scala:332, took 0.030491 s -2016-05-23 22:07:33 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Got job 14 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 22:07:33 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=387806, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=391246, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 22:07:33 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53983 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 22:07:33 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[28] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 34, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 35, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 36, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 37, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 22:07:33 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 36) -2016-05-23 22:07:33 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 35) -2016-05-23 22:07:33 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 37) -2016-05-23 22:07:33 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 34) -2016-05-23 22:07:33 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 36). 1109 bytes result sent to driver -2016-05-23 22:07:33 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 37). 1109 bytes result sent to driver -2016-05-23 22:07:33 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 34). 1109 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 36) in 12 ms on localhost (1/4) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 37) in 12 ms on localhost (2/4) -2016-05-23 22:07:33 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 35). 1109 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 34) in 18 ms on localhost (3/4) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 35) in 18 ms on localhost (4/4) -2016-05-23 22:07:33 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at RowMatrix.scala:124) finished in 0.019 s -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-23 22:07:33 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at RowMatrix.scala:124, took 0.032980 s -2016-05-23 22:07:33 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Got job 15 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 22:07:33 INFO DAGScheduler:59 - Final stage: ResultStage 22(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=393164, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(1860) called with curMem=396540, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1860.0 B, free 730.2 MB) -2016-05-23 22:07:33 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53983 (size: 1860.0 B, free: 730.5 MB) -2016-05-23 22:07:33 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[34] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 38, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 39, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 40, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 22:07:33 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 39) -2016-05-23 22:07:33 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 41) -2016-05-23 22:07:33 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 40) -2016-05-23 22:07:33 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 39). 1334 bytes result sent to driver -2016-05-23 22:07:33 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 38) -2016-05-23 22:07:33 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 40). 1294 bytes result sent to driver -2016-05-23 22:07:33 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 41). 1334 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 39) in 17 ms on localhost (1/4) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 40) in 17 ms on localhost (2/4) -2016-05-23 22:07:33 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 38). 1294 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 41) in 18 ms on localhost (3/4) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 38) in 23 ms on localhost (4/4) -2016-05-23 22:07:33 INFO DAGScheduler:59 - ResultStage 22 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.024 s -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-23 22:07:33 INFO DAGScheduler:59 - Job 15 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.040350 s -2016-05-23 22:07:33 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Registering RDD 32 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Got job 16 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 22:07:33 INFO DAGScheduler:59 - Final stage: ResultStage 24(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=398400, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(2282) called with curMem=402440, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 22:07:33 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:53983 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 22:07:33 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[32] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 42, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2269 bytes) -2016-05-23 22:07:33 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 43) -2016-05-23 22:07:33 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 45) -2016-05-23 22:07:33 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 44) -2016-05-23 22:07:33 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 42) -2016-05-23 22:07:33 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 45). 1161 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 45) in 55 ms on localhost (1/4) -2016-05-23 22:07:33 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 44) in 62 ms on localhost (2/4) -2016-05-23 22:07:33 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 43). 1161 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 43) in 77 ms on localhost (3/4) -2016-05-23 22:07:33 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 42). 1161 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 42) in 88 ms on localhost (4/4) -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-23 22:07:33 INFO DAGScheduler:59 - ShuffleMapStage 23 (flatMap at SpearmanCorrelation.scala:49) finished in 0.089 s -2016-05-23 22:07:33 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:07:33 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:07:33 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-23 22:07:33 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:07:33 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=404722, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=407266, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.2 MB) -2016-05-23 22:07:33 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:53983 (size: 1568.0 B, free: 730.5 MB) -2016-05-23 22:07:33 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 24 (ShuffledRDD[35] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 3 tasks -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:33 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 46) -2016-05-23 22:07:33 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 47) -2016-05-23 22:07:33 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 48) -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:33 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 47). 1203 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 47) in 22 ms on localhost (1/3) -2016-05-23 22:07:33 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 48). 1203 bytes result sent to driver -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 48) in 26 ms on localhost (2/3) -2016-05-23 22:07:33 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 46). 1203 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 46) in 41 ms on localhost (3/3) -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-23 22:07:33 INFO DAGScheduler:59 - ResultStage 24 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.043 s -2016-05-23 22:07:33 INFO DAGScheduler:59 - Job 16 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.168398 s -2016-05-23 22:07:33 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:07:33 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 22:07:33 INFO DAGScheduler:59 - Registering RDD 37 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Got job 17 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:07:33 INFO DAGScheduler:59 - Final stage: ResultStage 27(first at RowMatrix.scala:65) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=408834, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 22:07:33 INFO MemoryStore:59 - ensureFreeSpace(2306) called with curMem=413066, maxMem=766075207 -2016-05-23 22:07:33 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 22:07:33 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:53983 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:07:33 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[37] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:07:33 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:07:33 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 52) -2016-05-23 22:07:33 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-23 22:07:33 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 51) -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:33 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:33 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 52). 1377 bytes result sent to driver -2016-05-23 22:07:33 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1377 bytes result sent to driver -2016-05-23 22:07:33 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 51). 1377 bytes result sent to driver -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 52) in 140 ms on localhost (1/4) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 143 ms on localhost (2/4) -2016-05-23 22:07:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 51) in 144 ms on localhost (3/4) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1377 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 198 ms on localhost (4/4) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-23 22:07:34 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.201 s -2016-05-23 22:07:34 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:07:34 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:07:34 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-23 22:07:34 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=415372, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=420268, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:53983 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:07:34 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 27 (MapPartitionsRDD[39] at map at SpearmanCorrelation.scala:87) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 1 tasks -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:34 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 1296 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 14 ms on localhost (1/1) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-23 22:07:34 INFO DAGScheduler:59 - ResultStage 27 (first at RowMatrix.scala:65) finished in 0.014 s -2016-05-23 22:07:34 INFO DAGScheduler:59 - Job 17 finished: first at RowMatrix.scala:65, took 0.273160 s -2016-05-23 22:07:34 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:07:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 168 bytes -2016-05-23 22:07:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 166 bytes -2016-05-23 22:07:34 INFO DAGScheduler:59 - Got job 18 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 22:07:34 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=422810, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(2909) called with curMem=428506, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:53983 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:07:34 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:53983 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[40] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 55) -2016-05-23 22:07:34 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 56) -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:34 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 55). 1481 bytes result sent to driver -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:34 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 56). 1481 bytes result sent to driver -2016-05-23 22:07:34 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 54) -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:34 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 54). 1481 bytes result sent to driver -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:53983 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:07:34 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 57) -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:34 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 57). 1481 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 56) in 83 ms on localhost (1/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 55) in 85 ms on localhost (2/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 54) in 87 ms on localhost (3/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 57) in 84 ms on localhost (4/4) -2016-05-23 22:07:34 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at RowMatrix.scala:332) finished in 0.089 s -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-23 22:07:34 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at RowMatrix.scala:332, took 0.136039 s -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:53983 in memory (size: 1568.0 B, free: 730.5 MB) -2016-05-23 22:07:34 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Got job 19 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 22:07:34 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:34 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=413327, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:53983 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=412741, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:53983 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:53983 in memory (size: 1860.0 B, free: 730.5 MB) -2016-05-23 22:07:34 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[41] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 59) -2016-05-23 22:07:34 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 61) -2016-05-23 22:07:34 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 60) -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:34 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 61). 1359 bytes result sent to driver -2016-05-23 22:07:34 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 59). 1359 bytes result sent to driver -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:34 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 60). 1359 bytes result sent to driver -2016-05-23 22:07:34 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:53983 in memory (size: 1918.0 B, free: 730.5 MB) -2016-05-23 22:07:34 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:34 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 1359 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 61) in 60 ms on localhost (1/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 59) in 62 ms on localhost (2/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 60) in 62 ms on localhost (3/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 66 ms on localhost (4/4) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-23 22:07:34 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:53983 in memory (size: 1934.0 B, free: 730.5 MB) -2016-05-23 22:07:34 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at RowMatrix.scala:124) finished in 0.072 s -2016-05-23 22:07:34 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at RowMatrix.scala:124, took 0.098953 s -2016-05-23 22:07:34 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:53983 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 22:07:34 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:60 -2016-05-23 22:07:34 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Got job 20 (foreach at StatisticsCorrLearning.scala:60) with 4 output partitions -2016-05-23 22:07:34 INFO DAGScheduler:59 - Final stage: ResultStage 34(foreach at StatisticsCorrLearning.scala:60) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:57), which has no missing parents -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:53983 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=394784, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=396008, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 22:07:34 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:53983 (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:07:34 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (ParallelCollectionRDD[42] at parallelize at StatisticsCorrLearning.scala:57) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 62, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 63, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:53983 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 64, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 22:07:34 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 63) -2016-05-23 22:07:34 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 65) -2016-05-23 22:07:34 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 62) -2016-05-23 22:07:34 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 64) -2016-05-23 22:07:34 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-23 22:07:34 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 63). 915 bytes result sent to driver -2016-05-23 22:07:34 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 62). 915 bytes result sent to driver -2016-05-23 22:07:34 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 65). 915 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 62) in 9 ms on localhost (1/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 63) in 9 ms on localhost (2/4) -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:53983 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 64). 915 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 65) in 9 ms on localhost (3/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 64) in 12 ms on localhost (4/4) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-23 22:07:34 INFO DAGScheduler:59 - ResultStage 34 (foreach at StatisticsCorrLearning.scala:60) finished in 0.015 s -2016-05-23 22:07:34 INFO DAGScheduler:59 - Job 20 finished: foreach at StatisticsCorrLearning.scala:60, took 0.029167 s -2016-05-23 22:07:34 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:62 -2016-05-23 22:07:34 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Got job 21 (foreach at StatisticsCorrLearning.scala:62) with 4 output partitions -2016-05-23 22:07:34 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at StatisticsCorrLearning.scala:62) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:58), which has no missing parents -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=384975, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=386199, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:53983 (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:53983 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:07:34 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:53983 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:07:34 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 35 (ParallelCollectionRDD[43] at parallelize at StatisticsCorrLearning.scala:58) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 4 tasks -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 66, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 22:07:34 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 66) -2016-05-23 22:07:34 INFO Executor:59 - Running task 3.0 in stage 35.0 (TID 69) -2016-05-23 22:07:34 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 67) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 66). 915 bytes result sent to driver -2016-05-23 22:07:34 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 67). 915 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 66) in 11 ms on localhost (1/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 67) in 11 ms on localhost (2/4) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 3.0 in stage 35.0 (TID 69). 915 bytes result sent to driver -2016-05-23 22:07:34 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-23 22:07:34 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 68) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 68). 915 bytes result sent to driver -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:53983 in memory (size: 2.2 KB, free: 730.6 MB) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 35.0 (TID 69) in 20 ms on localhost (3/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 68) in 22 ms on localhost (4/4) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-23 22:07:34 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-23 22:07:34 INFO DAGScheduler:59 - ResultStage 35 (foreach at StatisticsCorrLearning.scala:62) finished in 0.027 s -2016-05-23 22:07:34 INFO DAGScheduler:59 - Job 21 finished: foreach at StatisticsCorrLearning.scala:62, took 0.064033 s -2016-05-23 22:07:34 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:07:34 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Got job 22 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:07:34 INFO DAGScheduler:59 - Final stage: ResultStage 36(first at RowMatrix.scala:65) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45), which has no missing parents -2016-05-23 22:07:34 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=364667, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(1522) called with curMem=367171, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1522.0 B, free 730.2 MB) -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:53983 (size: 1522.0 B, free: 730.6 MB) -2016-05-23 22:07:34 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[45] at map at Correlation.scala:45) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:34 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 70) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 70). 1046 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 70) in 7 ms on localhost (1/1) -2016-05-23 22:07:34 INFO DAGScheduler:59 - ResultStage 36 (first at RowMatrix.scala:65) finished in 0.008 s -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-23 22:07:34 INFO DAGScheduler:59 - Job 22 finished: first at RowMatrix.scala:65, took 0.027200 s -2016-05-23 22:07:34 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Got job 23 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 22:07:34 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=368693, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(1932) called with curMem=372093, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1932.0 B, free 730.2 MB) -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:53983 (size: 1932.0 B, free: 730.5 MB) -2016-05-23 22:07:34 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 37 (MapPartitionsRDD[46] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 4 tasks -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:07:34 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 72) -2016-05-23 22:07:34 INFO Executor:59 - Running task 3.0 in stage 37.0 (TID 74) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 72). 1231 bytes result sent to driver -2016-05-23 22:07:34 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 73) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 3.0 in stage 37.0 (TID 74). 1231 bytes result sent to driver -2016-05-23 22:07:34 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 73). 1231 bytes result sent to driver -2016-05-23 22:07:34 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 1231 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 72) in 14 ms on localhost (1/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 16 ms on localhost (2/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 37.0 (TID 74) in 14 ms on localhost (3/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 73) in 15 ms on localhost (4/4) -2016-05-23 22:07:34 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at RowMatrix.scala:332) finished in 0.018 s -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-23 22:07:34 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at RowMatrix.scala:332, took 0.029824 s -2016-05-23 22:07:34 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Got job 24 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 22:07:34 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=374025, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(1916) called with curMem=377465, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1916.0 B, free 730.2 MB) -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:53983 (size: 1916.0 B, free: 730.5 MB) -2016-05-23 22:07:34 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[47] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 77, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 78, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:07:34 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 77) -2016-05-23 22:07:34 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 78) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 78). 1109 bytes result sent to driver -2016-05-23 22:07:34 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 77). 1109 bytes result sent to driver -2016-05-23 22:07:34 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 1109 bytes result sent to driver -2016-05-23 22:07:34 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 1109 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 78) in 16 ms on localhost (1/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 77) in 19 ms on localhost (2/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 20 ms on localhost (3/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 24 ms on localhost (4/4) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-23 22:07:34 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at RowMatrix.scala:124) finished in 0.025 s -2016-05-23 22:07:34 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at RowMatrix.scala:124, took 0.035600 s -2016-05-23 22:07:34 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Got job 25 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 22:07:34 INFO DAGScheduler:59 - Final stage: ResultStage 39(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=379381, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(1851) called with curMem=382757, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1851.0 B, free 730.2 MB) -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:53983 (size: 1851.0 B, free: 730.5 MB) -2016-05-23 22:07:34 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 39 (MapPartitionsRDD[53] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 4 tasks -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 79, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 80, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 81, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 39.0 (TID 82, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:07:34 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 81) -2016-05-23 22:07:34 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 80) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 81). 1249 bytes result sent to driver -2016-05-23 22:07:34 INFO Executor:59 - Running task 3.0 in stage 39.0 (TID 82) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 80). 1254 bytes result sent to driver -2016-05-23 22:07:34 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 79) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 3.0 in stage 39.0 (TID 82). 1294 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 80) in 13 ms on localhost (1/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 81) in 13 ms on localhost (2/4) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 79). 1254 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 39.0 (TID 82) in 13 ms on localhost (3/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 79) in 17 ms on localhost (4/4) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-23 22:07:34 INFO DAGScheduler:59 - ResultStage 39 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.017 s -2016-05-23 22:07:34 INFO DAGScheduler:59 - Job 25 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.032380 s -2016-05-23 22:07:34 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Registering RDD 51 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Got job 26 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 22:07:34 INFO DAGScheduler:59 - Final stage: ResultStage 41(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 40) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 40) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=384608, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(2281) called with curMem=388648, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:53983 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 22:07:34 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 40 (MapPartitionsRDD[51] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 4 tasks -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 83, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 84, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 85, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 40.0 (TID 86, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 22:07:34 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 83) -2016-05-23 22:07:34 INFO Executor:59 - Running task 3.0 in stage 40.0 (TID 86) -2016-05-23 22:07:34 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 84) -2016-05-23 22:07:34 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 85) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 83). 1161 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 83) in 57 ms on localhost (1/4) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 84). 1161 bytes result sent to driver -2016-05-23 22:07:34 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 85). 1161 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 85) in 61 ms on localhost (2/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 84) in 63 ms on localhost (3/4) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 3.0 in stage 40.0 (TID 86). 1161 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 40.0 (TID 86) in 80 ms on localhost (4/4) -2016-05-23 22:07:34 INFO DAGScheduler:59 - ShuffleMapStage 40 (flatMap at SpearmanCorrelation.scala:49) finished in 0.083 s -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-23 22:07:34 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:07:34 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:07:34 INFO DAGScheduler:59 - waiting: Set(ResultStage 41) -2016-05-23 22:07:34 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Missing parents for ResultStage 41: List() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=390929, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=393473, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.2 MB) -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:53983 (size: 1567.0 B, free: 730.5 MB) -2016-05-23 22:07:34 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 41 (ShuffledRDD[54] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 3 tasks -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 87, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 87) -2016-05-23 22:07:34 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 88) -2016-05-23 22:07:34 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 89) -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:34 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 87). 1203 bytes result sent to driver -2016-05-23 22:07:34 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 88). 1203 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 87) in 13 ms on localhost (1/3) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 88) in 14 ms on localhost (2/3) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 2.0 in stage 41.0 (TID 89). 1203 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 41.0 (TID 89) in 15 ms on localhost (3/3) -2016-05-23 22:07:34 INFO DAGScheduler:59 - ResultStage 41 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.017 s -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-23 22:07:34 INFO DAGScheduler:59 - Job 26 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.122635 s -2016-05-23 22:07:34 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:07:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 163 bytes -2016-05-23 22:07:34 INFO DAGScheduler:59 - Registering RDD 56 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Got job 27 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:07:34 INFO DAGScheduler:59 - Final stage: ResultStage 44(first at RowMatrix.scala:65) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 43) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 43) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=395040, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(2305) called with curMem=399272, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:53983 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:07:34 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 43 (MapPartitionsRDD[56] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 4 tasks -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 90, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 91, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 43.0 (TID 92, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 43.0 (TID 93, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:07:34 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 90) -2016-05-23 22:07:34 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 91) -2016-05-23 22:07:34 INFO Executor:59 - Running task 2.0 in stage 43.0 (TID 92) -2016-05-23 22:07:34 INFO Executor:59 - Running task 3.0 in stage 43.0 (TID 93) -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:34 INFO Executor:59 - Finished task 2.0 in stage 43.0 (TID 92). 1377 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 43.0 (TID 92) in 44 ms on localhost (1/4) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 91). 1377 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 91) in 51 ms on localhost (2/4) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 3.0 in stage 43.0 (TID 93). 1377 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 43.0 (TID 93) in 62 ms on localhost (3/4) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 90). 1377 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 90) in 67 ms on localhost (4/4) -2016-05-23 22:07:34 INFO DAGScheduler:59 - ShuffleMapStage 43 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.068 s -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-23 22:07:34 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:07:34 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:07:34 INFO DAGScheduler:59 - waiting: Set(ResultStage 44) -2016-05-23 22:07:34 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Missing parents for ResultStage 44: List() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=401577, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(2545) called with curMem=406473, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:53983 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:07:34 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 44 (MapPartitionsRDD[58] at map at SpearmanCorrelation.scala:87) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 1 tasks -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 94, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 94) -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:34 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 94). 1296 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 94) in 11 ms on localhost (1/1) -2016-05-23 22:07:34 INFO DAGScheduler:59 - ResultStage 44 (first at RowMatrix.scala:65) finished in 0.012 s -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-23 22:07:34 INFO DAGScheduler:59 - Job 27 finished: first at RowMatrix.scala:65, took 0.105081 s -2016-05-23 22:07:34 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:07:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 163 bytes -2016-05-23 22:07:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 166 bytes -2016-05-23 22:07:34 INFO DAGScheduler:59 - Got job 28 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 22:07:34 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 46) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=409018, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(2914) called with curMem=414714, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:53983 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:07:34 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 47 (MapPartitionsRDD[59] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 4 tasks -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 95, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 96, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 47.0 (TID 97, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 47.0 (TID 98, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 95) -2016-05-23 22:07:34 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 96) -2016-05-23 22:07:34 INFO Executor:59 - Running task 2.0 in stage 47.0 (TID 97) -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:34 INFO Executor:59 - Running task 3.0 in stage 47.0 (TID 98) -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:34 INFO Executor:59 - Finished task 2.0 in stage 47.0 (TID 97). 1481 bytes result sent to driver -2016-05-23 22:07:34 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 96). 1481 bytes result sent to driver -2016-05-23 22:07:34 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 95). 1481 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 95) in 17 ms on localhost (1/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 47.0 (TID 97) in 16 ms on localhost (2/4) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 96) in 17 ms on localhost (3/4) -2016-05-23 22:07:34 INFO Executor:59 - Finished task 3.0 in stage 47.0 (TID 98). 1481 bytes result sent to driver -2016-05-23 22:07:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 47.0 (TID 98) in 19 ms on localhost (4/4) -2016-05-23 22:07:34 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at RowMatrix.scala:332) finished in 0.021 s -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-23 22:07:34 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at RowMatrix.scala:332, took 0.035385 s -2016-05-23 22:07:34 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Got job 29 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 22:07:34 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 49) -2016-05-23 22:07:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=417628, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=423364, maxMem=766075207 -2016-05-23 22:07:34 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 22:07:34 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:53983 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:07:34 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 50 (MapPartitionsRDD[60] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:34 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 4 tasks -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 50.0 (TID 101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 50.0 (TID 102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:34 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-23 22:07:34 INFO Executor:59 - Running task 2.0 in stage 50.0 (TID 101) -2016-05-23 22:07:34 INFO Executor:59 - Running task 3.0 in stage 50.0 (TID 102) -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:34 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:34 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 1359 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 1359 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 3.0 in stage 50.0 (TID 102). 1359 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 22 ms on localhost (1/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 24 ms on localhost (2/4) -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 3.0 in stage 50.0 (TID 102) in 21 ms on localhost (3/4) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 2.0 in stage 50.0 (TID 101). 1359 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 50.0 (TID 101) in 30 ms on localhost (4/4) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-23 22:07:35 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at RowMatrix.scala:124) finished in 0.032 s -2016-05-23 22:07:35 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at RowMatrix.scala:124, took 0.044029 s -2016-05-23 22:07:35 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:73 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Got job 30 (foreach at StatisticsCorrLearning.scala:73) with 4 output partitions -2016-05-23 22:07:35 INFO DAGScheduler:59 - Final stage: ResultStage 51(foreach at StatisticsCorrLearning.scala:73) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting ResultStage 51 (ParallelCollectionRDD[61] at parallelize at StatisticsCorrLearning.scala:70), which has no missing parents -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=426260, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=427484, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:53983 (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 51 (ParallelCollectionRDD[61] at parallelize at StatisticsCorrLearning.scala:70) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 4 tasks -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 103, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 104, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 2.0 in stage 51.0 (TID 105, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 3.0 in stage 51.0 (TID 106, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:07:35 INFO Executor:59 - Running task 3.0 in stage 51.0 (TID 106) -2016-05-23 22:07:35 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 103) -2016-05-23 22:07:35 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 104) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 3.0 in stage 51.0 (TID 106). 915 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 104). 915 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 103). 915 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Running task 2.0 in stage 51.0 (TID 105) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 2.0 in stage 51.0 (TID 105). 915 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 3.0 in stage 51.0 (TID 106) in 17 ms on localhost (1/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 104) in 22 ms on localhost (2/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 51.0 (TID 105) in 23 ms on localhost (3/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 103) in 25 ms on localhost (4/4) -2016-05-23 22:07:35 INFO DAGScheduler:59 - ResultStage 51 (foreach at StatisticsCorrLearning.scala:73) finished in 0.025 s -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-23 22:07:35 INFO DAGScheduler:59 - Job 30 finished: foreach at StatisticsCorrLearning.scala:73, took 0.036231 s -2016-05-23 22:07:35 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:75 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Got job 31 (foreach at StatisticsCorrLearning.scala:75) with 4 output partitions -2016-05-23 22:07:35 INFO DAGScheduler:59 - Final stage: ResultStage 52(foreach at StatisticsCorrLearning.scala:75) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting ResultStage 52 (ParallelCollectionRDD[62] at parallelize at StatisticsCorrLearning.scala:71), which has no missing parents -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=428377, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=429601, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:53983 (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 52 (ParallelCollectionRDD[62] at parallelize at StatisticsCorrLearning.scala:71) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 4 tasks -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 107, localhost, PROCESS_LOCAL, 2028 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 108, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 2.0 in stage 52.0 (TID 109, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 3.0 in stage 52.0 (TID 110, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:07:35 INFO Executor:59 - Running task 3.0 in stage 52.0 (TID 110) -2016-05-23 22:07:35 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 108) -2016-05-23 22:07:35 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 107) -2016-05-23 22:07:35 INFO Executor:59 - Running task 2.0 in stage 52.0 (TID 109) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 3.0 in stage 52.0 (TID 110). 915 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 107). 915 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 2.0 in stage 52.0 (TID 109). 915 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 3.0 in stage 52.0 (TID 110) in 4 ms on localhost (1/4) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 108). 915 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 107) in 8 ms on localhost (2/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 52.0 (TID 109) in 8 ms on localhost (3/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 108) in 11 ms on localhost (4/4) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-23 22:07:35 INFO DAGScheduler:59 - ResultStage 52 (foreach at StatisticsCorrLearning.scala:75) finished in 0.012 s -2016-05-23 22:07:35 INFO DAGScheduler:59 - Job 31 finished: foreach at StatisticsCorrLearning.scala:75, took 0.021619 s -2016-05-23 22:07:35 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Got job 32 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:07:35 INFO DAGScheduler:59 - Final stage: ResultStage 53(first at RowMatrix.scala:65) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[64] at map at Correlation.scala:45), which has no missing parents -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=430494, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=432998, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:53983 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 53 (MapPartitionsRDD[64] at map at Correlation.scala:45) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 1 tasks -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 111, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 22:07:35 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 111) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 111). 935 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 111) in 5 ms on localhost (1/1) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-23 22:07:35 INFO DAGScheduler:59 - ResultStage 53 (first at RowMatrix.scala:65) finished in 0.006 s -2016-05-23 22:07:35 INFO DAGScheduler:59 - Job 32 finished: first at RowMatrix.scala:65, took 0.017009 s -2016-05-23 22:07:35 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Got job 33 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 22:07:35 INFO DAGScheduler:59 - Final stage: ResultStage 54(first at RowMatrix.scala:65) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[64] at map at Correlation.scala:45), which has no missing parents -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=434522, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=437026, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:53983 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 54 (MapPartitionsRDD[64] at map at Correlation.scala:45) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 3 tasks -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 112, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 113, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 2.0 in stage 54.0 (TID 114, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:35 INFO Executor:59 - Running task 2.0 in stage 54.0 (TID 114) -2016-05-23 22:07:35 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 112) -2016-05-23 22:07:35 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 113) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 112). 1046 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 113). 1046 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 2.0 in stage 54.0 (TID 114). 1046 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 112) in 6 ms on localhost (1/3) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 54.0 (TID 114) in 5 ms on localhost (2/3) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 113) in 6 ms on localhost (3/3) -2016-05-23 22:07:35 INFO DAGScheduler:59 - ResultStage 54 (first at RowMatrix.scala:65) finished in 0.007 s -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-23 22:07:35 INFO DAGScheduler:59 - Job 33 finished: first at RowMatrix.scala:65, took 0.017976 s -2016-05-23 22:07:35 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Got job 34 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 22:07:35 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[65] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=438550, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(1934) called with curMem=441950, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1934.0 B, free 730.2 MB) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:53983 (size: 1934.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 55 (MapPartitionsRDD[65] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 4 tasks -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 115, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 116, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 2.0 in stage 55.0 (TID 117, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 3.0 in stage 55.0 (TID 118, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:35 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 115) -2016-05-23 22:07:35 INFO Executor:59 - Running task 2.0 in stage 55.0 (TID 117) -2016-05-23 22:07:35 INFO Executor:59 - Running task 3.0 in stage 55.0 (TID 118) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 115). 1231 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 3.0 in stage 55.0 (TID 118). 1231 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 116) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 116). 1231 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 2.0 in stage 55.0 (TID 117). 1231 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 115) in 19 ms on localhost (1/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 3.0 in stage 55.0 (TID 118) in 18 ms on localhost (2/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 55.0 (TID 117) in 19 ms on localhost (3/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 116) in 20 ms on localhost (4/4) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-23 22:07:35 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at RowMatrix.scala:332) finished in 0.023 s -2016-05-23 22:07:35 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at RowMatrix.scala:332, took 0.036187 s -2016-05-23 22:07:35 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Got job 35 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 22:07:35 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[66] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=443884, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(1918) called with curMem=447324, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 1918.0 B, free 730.2 MB) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:53983 (size: 1918.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 56 (MapPartitionsRDD[66] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 4 tasks -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 119, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 120, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 2.0 in stage 56.0 (TID 121, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 3.0 in stage 56.0 (TID 122, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:35 INFO Executor:59 - Running task 2.0 in stage 56.0 (TID 121) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 2.0 in stage 56.0 (TID 121). 1109 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Running task 3.0 in stage 56.0 (TID 122) -2016-05-23 22:07:35 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 119) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 3.0 in stage 56.0 (TID 122). 1109 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 119). 1109 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 120) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 56.0 (TID 121) in 16 ms on localhost (1/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 3.0 in stage 56.0 (TID 122) in 12 ms on localhost (2/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 119) in 20 ms on localhost (3/4) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 120). 1109 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 120) in 28 ms on localhost (4/4) -2016-05-23 22:07:35 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at RowMatrix.scala:124) finished in 0.033 s -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-23 22:07:35 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at RowMatrix.scala:124, took 0.050341 s -2016-05-23 22:07:35 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Got job 36 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 22:07:35 INFO DAGScheduler:59 - Final stage: ResultStage 57(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[72] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=449242, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(1858) called with curMem=452618, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 1858.0 B, free 730.2 MB) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:53983 (size: 1858.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 57 (MapPartitionsRDD[72] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 4 tasks -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 123, localhost, PROCESS_LOCAL, 2232 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 124, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 2.0 in stage 57.0 (TID 125, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 3.0 in stage 57.0 (TID 126, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:35 INFO Executor:59 - Running task 2.0 in stage 57.0 (TID 125) -2016-05-23 22:07:35 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 124) -2016-05-23 22:07:35 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 123) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 2.0 in stage 57.0 (TID 125). 1249 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Running task 3.0 in stage 57.0 (TID 126) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 123). 1104 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 124). 1254 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 3.0 in stage 57.0 (TID 126). 1254 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 57.0 (TID 125) in 9 ms on localhost (1/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 123) in 13 ms on localhost (2/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 124) in 13 ms on localhost (3/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 3.0 in stage 57.0 (TID 126) in 18 ms on localhost (4/4) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-23 22:07:35 INFO DAGScheduler:59 - ResultStage 57 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.022 s -2016-05-23 22:07:35 INFO DAGScheduler:59 - Job 36 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.037136 s -2016-05-23 22:07:35 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Registering RDD 70 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Got job 37 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 22:07:35 INFO DAGScheduler:59 - Final stage: ResultStage 59(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 58) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 58) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting ShuffleMapStage 58 (MapPartitionsRDD[70] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=454476, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(2285) called with curMem=458516, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:53983 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 22:07:35 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 58 (MapPartitionsRDD[70] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 4 tasks -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 127, localhost, PROCESS_LOCAL, 2221 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 128, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 2.0 in stage 58.0 (TID 129, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 3.0 in stage 58.0 (TID 130, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 22:07:35 INFO Executor:59 - Running task 3.0 in stage 58.0 (TID 130) -2016-05-23 22:07:35 INFO Executor:59 - Running task 2.0 in stage 58.0 (TID 129) -2016-05-23 22:07:35 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 128) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 128). 1161 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 127) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 128) in 46 ms on localhost (1/4) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 2.0 in stage 58.0 (TID 129). 1161 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 127). 1161 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 58.0 (TID 129) in 56 ms on localhost (2/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 127) in 59 ms on localhost (3/4) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 3.0 in stage 58.0 (TID 130). 1161 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 3.0 in stage 58.0 (TID 130) in 60 ms on localhost (4/4) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-05-23 22:07:35 INFO DAGScheduler:59 - ShuffleMapStage 58 (flatMap at SpearmanCorrelation.scala:49) finished in 0.064 s -2016-05-23 22:07:35 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:07:35 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:07:35 INFO DAGScheduler:59 - waiting: Set(ResultStage 59) -2016-05-23 22:07:35 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Missing parents for ResultStage 59: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting ResultStage 59 (ShuffledRDD[73] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=460801, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=463345, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.1 MB) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:53983 (size: 1567.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 59 (ShuffledRDD[73] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 3 tasks -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 131, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 132, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 2.0 in stage 59.0 (TID 133, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:35 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 131) -2016-05-23 22:07:35 INFO Executor:59 - Running task 2.0 in stage 59.0 (TID 133) -2016-05-23 22:07:35 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 132) -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:35 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 132). 1203 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 2.0 in stage 59.0 (TID 133). 1203 bytes result sent to driver -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 132) in 14 ms on localhost (1/3) -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 59.0 (TID 133) in 15 ms on localhost (2/3) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 131). 1203 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 131) in 22 ms on localhost (3/3) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-05-23 22:07:35 INFO DAGScheduler:59 - ResultStage 59 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.022 s -2016-05-23 22:07:35 INFO DAGScheduler:59 - Job 37 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.113625 s -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:53983 in memory (size: 1567.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:53983 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:53983 in memory (size: 1858.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-23 22:07:35 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:53983 in memory (size: 1918.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-23 22:07:35 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 168 bytes -2016-05-23 22:07:35 INFO DAGScheduler:59 - Registering RDD 75 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Got job 38 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:53983 in memory (size: 1934.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Final stage: ResultStage 62(first at RowMatrix.scala:65) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 61) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 61) -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting ShuffleMapStage 61 (MapPartitionsRDD[75] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=437026, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:53983 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(2305) called with curMem=438754, maxMem=766075207 -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:53983 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:07:35 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 61 (MapPartitionsRDD[75] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:53983 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 4 tasks -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 134, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 135, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 2.0 in stage 61.0 (TID 136, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 3.0 in stage 61.0 (TID 137, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:07:35 INFO Executor:59 - Running task 2.0 in stage 61.0 (TID 136) -2016-05-23 22:07:35 INFO Executor:59 - Running task 3.0 in stage 61.0 (TID 137) -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:53983 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 135) -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:35 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 134) -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 4 blocks -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:53983 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:53983 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 2.0 in stage 61.0 (TID 136). 1377 bytes result sent to driver -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 61.0 (TID 136) in 53 ms on localhost (1/4) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:53983 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 134). 1377 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 135). 1377 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 134) in 65 ms on localhost (2/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 135) in 65 ms on localhost (3/4) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 3.0 in stage 61.0 (TID 137). 1377 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 3.0 in stage 61.0 (TID 137) in 65 ms on localhost (4/4) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-05-23 22:07:35 INFO DAGScheduler:59 - ShuffleMapStage 61 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.068 s -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-23 22:07:35 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:07:35 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:07:35 INFO DAGScheduler:59 - waiting: Set(ResultStage 62) -2016-05-23 22:07:35 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Missing parents for ResultStage 62: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=408114, maxMem=766075207 -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:53983 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=413010, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:53983 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:07:35 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 62 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:53983 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 1 tasks -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 138, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:35 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 138) -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned shuffle 5 -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:53983 in memory (size: 1567.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 138). 1185 bytes result sent to driver -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 138) in 13 ms on localhost (1/1) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-05-23 22:07:35 INFO DAGScheduler:59 - ResultStage 62 (first at RowMatrix.scala:65) finished in 0.014 s -2016-05-23 22:07:35 INFO DAGScheduler:59 - Job 38 finished: first at RowMatrix.scala:65, took 0.111652 s -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:53983 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned shuffle 4 -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:53983 in memory (size: 1851.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-23 22:07:35 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 168 bytes -2016-05-23 22:07:35 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 165 bytes -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:53983 in memory (size: 1916.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Got job 39 (first at RowMatrix.scala:65) with 3 output partitions -2016-05-23 22:07:35 INFO DAGScheduler:59 - Final stage: ResultStage 65(first at RowMatrix.scala:65) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 64) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87), which has no missing parents -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=391442, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=396338, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:53983 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:07:35 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 65 (MapPartitionsRDD[77] at map at SpearmanCorrelation.scala:87) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 3 tasks -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 139, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:53983 in memory (size: 1932.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 140, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 2.0 in stage 65.0 (TID 141, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:35 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 139) -2016-05-23 22:07:35 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 140) -2016-05-23 22:07:35 INFO Executor:59 - Running task 2.0 in stage 65.0 (TID 141) -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 14 ms -2016-05-23 22:07:35 INFO Executor:59 - Finished task 2.0 in stage 65.0 (TID 141). 1296 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 139). 1296 bytes result sent to driver -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:53983 in memory (size: 1522.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 65.0 (TID 141) in 29 ms on localhost (1/3) -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 13 ms -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 139) in 32 ms on localhost (2/3) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 140). 1296 bytes result sent to driver -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:53983 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 140) in 44 ms on localhost (3/3) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-05-23 22:07:35 INFO DAGScheduler:59 - ResultStage 65 (first at RowMatrix.scala:65) finished in 0.046 s -2016-05-23 22:07:35 INFO DAGScheduler:59 - Job 39 finished: first at RowMatrix.scala:65, took 0.064836 s -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:53983 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-23 22:07:35 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:53983 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:07:35 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Got job 40 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 22:07:35 INFO DAGScheduler:59 - Final stage: ResultStage 68(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 67) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting ResultStage 68 (MapPartitionsRDD[78] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:53983 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=370313, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(2914) called with curMem=376009, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:53983 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:07:35 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 68 (MapPartitionsRDD[78] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Adding task set 68.0 with 4 tasks -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 68.0 (TID 142, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 68.0 (TID 143, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 2.0 in stage 68.0 (TID 144, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 3.0 in stage 68.0 (TID 145, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:35 INFO Executor:59 - Running task 2.0 in stage 68.0 (TID 144) -2016-05-23 22:07:35 INFO Executor:59 - Running task 1.0 in stage 68.0 (TID 143) -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:35 INFO Executor:59 - Running task 3.0 in stage 68.0 (TID 145) -2016-05-23 22:07:35 INFO Executor:59 - Running task 0.0 in stage 68.0 (TID 142) -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:35 INFO Executor:59 - Finished task 3.0 in stage 68.0 (TID 145). 1481 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 0.0 in stage 68.0 (TID 142). 1481 bytes result sent to driver -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:35 INFO Executor:59 - Finished task 2.0 in stage 68.0 (TID 144). 1481 bytes result sent to driver -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 3.0 in stage 68.0 (TID 145) in 22 ms on localhost (1/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 68.0 (TID 142) in 26 ms on localhost (2/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 68.0 (TID 144) in 25 ms on localhost (3/4) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 1.0 in stage 68.0 (TID 143). 1481 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 68.0 (TID 143) in 32 ms on localhost (4/4) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 68.0, whose tasks have all completed, from pool -2016-05-23 22:07:35 INFO DAGScheduler:59 - ResultStage 68 (treeAggregate at RowMatrix.scala:332) finished in 0.034 s -2016-05-23 22:07:35 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at RowMatrix.scala:332, took 0.051519 s -2016-05-23 22:07:35 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Got job 41 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 22:07:35 INFO DAGScheduler:59 - Final stage: ResultStage 71(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 70) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting ResultStage 71 (MapPartitionsRDD[79] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=373227, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=378963, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:53983 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:07:35 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 71 (MapPartitionsRDD[79] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Adding task set 71.0 with 4 tasks -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 71.0 (TID 146, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 71.0 (TID 147, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 2.0 in stage 71.0 (TID 148, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 3.0 in stage 71.0 (TID 149, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:35 INFO Executor:59 - Running task 0.0 in stage 71.0 (TID 146) -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:35 INFO Executor:59 - Finished task 0.0 in stage 71.0 (TID 146). 1359 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Running task 2.0 in stage 71.0 (TID 148) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 71.0 (TID 146) in 13 ms on localhost (1/4) -2016-05-23 22:07:35 INFO Executor:59 - Running task 1.0 in stage 71.0 (TID 147) -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:35 INFO Executor:59 - Finished task 2.0 in stage 71.0 (TID 148). 1359 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Running task 3.0 in stage 71.0 (TID 149) -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:35 INFO Executor:59 - Finished task 1.0 in stage 71.0 (TID 147). 1359 bytes result sent to driver -2016-05-23 22:07:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 71.0 (TID 148) in 23 ms on localhost (2/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 71.0 (TID 147) in 25 ms on localhost (3/4) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 3.0 in stage 71.0 (TID 149). 1359 bytes result sent to driver -2016-05-23 22:07:35 INFO DAGScheduler:59 - ResultStage 71 (treeAggregate at RowMatrix.scala:124) finished in 0.048 s -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 3.0 in stage 71.0 (TID 149) in 46 ms on localhost (4/4) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 71.0, whose tasks have all completed, from pool -2016-05-23 22:07:35 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at RowMatrix.scala:124, took 0.085221 s -2016-05-23 22:07:35 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:87 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Got job 42 (foreach at StatisticsCorrLearning.scala:87) with 4 output partitions -2016-05-23 22:07:35 INFO DAGScheduler:59 - Final stage: ResultStage 72(foreach at StatisticsCorrLearning.scala:87) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting ResultStage 72 (ParallelCollectionRDD[80] at parallelize at StatisticsCorrLearning.scala:84), which has no missing parents -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=381859, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(893) called with curMem=383083, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 893.0 B, free 730.2 MB) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:53983 (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 72 (ParallelCollectionRDD[80] at parallelize at StatisticsCorrLearning.scala:84) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Adding task set 72.0 with 4 tasks -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 72.0 (TID 150, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 72.0 (TID 151, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 2.0 in stage 72.0 (TID 152, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 3.0 in stage 72.0 (TID 153, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 22:07:35 INFO Executor:59 - Running task 1.0 in stage 72.0 (TID 151) -2016-05-23 22:07:35 INFO Executor:59 - Running task 3.0 in stage 72.0 (TID 153) -2016-05-23 22:07:35 INFO Executor:59 - Running task 2.0 in stage 72.0 (TID 152) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 1.0 in stage 72.0 (TID 151). 915 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 2.0 in stage 72.0 (TID 152). 915 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Running task 0.0 in stage 72.0 (TID 150) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 72.0 (TID 151) in 5 ms on localhost (1/4) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 0.0 in stage 72.0 (TID 150). 915 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 72.0 (TID 152) in 5 ms on localhost (2/4) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 3.0 in stage 72.0 (TID 153). 915 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 72.0 (TID 150) in 7 ms on localhost (3/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 3.0 in stage 72.0 (TID 153) in 8 ms on localhost (4/4) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 72.0, whose tasks have all completed, from pool -2016-05-23 22:07:35 INFO DAGScheduler:59 - ResultStage 72 (foreach at StatisticsCorrLearning.scala:87) finished in 0.010 s -2016-05-23 22:07:35 INFO DAGScheduler:59 - Job 42 finished: foreach at StatisticsCorrLearning.scala:87, took 0.019782 s -2016-05-23 22:07:35 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:89 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Got job 43 (foreach at StatisticsCorrLearning.scala:89) with 4 output partitions -2016-05-23 22:07:35 INFO DAGScheduler:59 - Final stage: ResultStage 73(foreach at StatisticsCorrLearning.scala:89) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting ResultStage 73 (ParallelCollectionRDD[81] at parallelize at StatisticsCorrLearning.scala:85), which has no missing parents -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(1232) called with curMem=383976, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 1232.0 B, free 730.2 MB) -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(894) called with curMem=385208, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 894.0 B, free 730.2 MB) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:53983 (size: 894.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 73 (ParallelCollectionRDD[81] at parallelize at StatisticsCorrLearning.scala:85) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Adding task set 73.0 with 4 tasks -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 73.0 (TID 154, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 73.0 (TID 155, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 2.0 in stage 73.0 (TID 156, localhost, PROCESS_LOCAL, 2036 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 3.0 in stage 73.0 (TID 157, localhost, PROCESS_LOCAL, 2044 bytes) -2016-05-23 22:07:35 INFO Executor:59 - Running task 1.0 in stage 73.0 (TID 155) -2016-05-23 22:07:35 INFO Executor:59 - Running task 2.0 in stage 73.0 (TID 156) -2016-05-23 22:07:35 INFO Executor:59 - Running task 0.0 in stage 73.0 (TID 154) -2016-05-23 22:07:35 INFO Executor:59 - Running task 3.0 in stage 73.0 (TID 157) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 2.0 in stage 73.0 (TID 156). 915 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 0.0 in stage 73.0 (TID 154). 915 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 1.0 in stage 73.0 (TID 155). 915 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 73.0 (TID 156) in 4 ms on localhost (1/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 73.0 (TID 154) in 6 ms on localhost (2/4) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 3.0 in stage 73.0 (TID 157). 915 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 73.0 (TID 155) in 8 ms on localhost (3/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 3.0 in stage 73.0 (TID 157) in 7 ms on localhost (4/4) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 73.0, whose tasks have all completed, from pool -2016-05-23 22:07:35 INFO DAGScheduler:59 - ResultStage 73 (foreach at StatisticsCorrLearning.scala:89) finished in 0.010 s -2016-05-23 22:07:35 INFO DAGScheduler:59 - Job 43 finished: foreach at StatisticsCorrLearning.scala:89, took 0.019964 s -2016-05-23 22:07:35 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Got job 44 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:07:35 INFO DAGScheduler:59 - Final stage: ResultStage 74(first at RowMatrix.scala:65) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting ResultStage 74 (MapPartitionsRDD[83] at map at Correlation.scala:45), which has no missing parents -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=386102, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(1524) called with curMem=388606, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 1524.0 B, free 730.2 MB) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:53983 (size: 1524.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 74 (MapPartitionsRDD[83] at map at Correlation.scala:45) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Adding task set 74.0 with 1 tasks -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 74.0 (TID 158, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:35 INFO Executor:59 - Running task 0.0 in stage 74.0 (TID 158) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 0.0 in stage 74.0 (TID 158). 1046 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 74.0 (TID 158) in 4 ms on localhost (1/1) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 74.0, whose tasks have all completed, from pool -2016-05-23 22:07:35 INFO DAGScheduler:59 - ResultStage 74 (first at RowMatrix.scala:65) finished in 0.004 s -2016-05-23 22:07:35 INFO DAGScheduler:59 - Job 44 finished: first at RowMatrix.scala:65, took 0.014518 s -2016-05-23 22:07:35 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Got job 45 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 22:07:35 INFO DAGScheduler:59 - Final stage: ResultStage 75(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting ResultStage 75 (MapPartitionsRDD[84] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=390130, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=393530, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.2 MB) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:53983 (size: 1935.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 75 (MapPartitionsRDD[84] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Adding task set 75.0 with 4 tasks -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 75.0 (TID 159, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 75.0 (TID 160, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 2.0 in stage 75.0 (TID 161, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Starting task 3.0 in stage 75.0 (TID 162, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:07:35 INFO Executor:59 - Running task 2.0 in stage 75.0 (TID 161) -2016-05-23 22:07:35 INFO Executor:59 - Running task 3.0 in stage 75.0 (TID 162) -2016-05-23 22:07:35 INFO Executor:59 - Running task 0.0 in stage 75.0 (TID 159) -2016-05-23 22:07:35 INFO Executor:59 - Running task 1.0 in stage 75.0 (TID 160) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 2.0 in stage 75.0 (TID 161). 1231 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 3.0 in stage 75.0 (TID 162). 1231 bytes result sent to driver -2016-05-23 22:07:35 INFO Executor:59 - Finished task 0.0 in stage 75.0 (TID 159). 1231 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 75.0 (TID 161) in 4 ms on localhost (1/4) -2016-05-23 22:07:35 INFO Executor:59 - Finished task 1.0 in stage 75.0 (TID 160). 1231 bytes result sent to driver -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 75.0 (TID 159) in 7 ms on localhost (2/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 3.0 in stage 75.0 (TID 162) in 6 ms on localhost (3/4) -2016-05-23 22:07:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 75.0 (TID 160) in 8 ms on localhost (4/4) -2016-05-23 22:07:35 INFO DAGScheduler:59 - ResultStage 75 (treeAggregate at RowMatrix.scala:332) finished in 0.009 s -2016-05-23 22:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 75.0, whose tasks have all completed, from pool -2016-05-23 22:07:35 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at RowMatrix.scala:332, took 0.017639 s -2016-05-23 22:07:35 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:07:35 INFO DAGScheduler:59 - Got job 46 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 22:07:35 INFO DAGScheduler:59 - Final stage: ResultStage 76(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:35 INFO DAGScheduler:59 - Submitting ResultStage 76 (MapPartitionsRDD[85] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=395465, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-23 22:07:35 INFO MemoryStore:59 - ensureFreeSpace(1919) called with curMem=398905, maxMem=766075207 -2016-05-23 22:07:35 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 1919.0 B, free 730.2 MB) -2016-05-23 22:07:35 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:53983 (size: 1919.0 B, free: 730.5 MB) -2016-05-23 22:07:35 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 76 (MapPartitionsRDD[85] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 4 tasks -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 163, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 164, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 2.0 in stage 76.0 (TID 165, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 3.0 in stage 76.0 (TID 166, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:07:36 INFO Executor:59 - Running task 3.0 in stage 76.0 (TID 166) -2016-05-23 22:07:36 INFO Executor:59 - Running task 2.0 in stage 76.0 (TID 165) -2016-05-23 22:07:36 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 164) -2016-05-23 22:07:36 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 163) -2016-05-23 22:07:36 INFO Executor:59 - Finished task 3.0 in stage 76.0 (TID 166). 1109 bytes result sent to driver -2016-05-23 22:07:36 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 163). 1109 bytes result sent to driver -2016-05-23 22:07:36 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 164). 1109 bytes result sent to driver -2016-05-23 22:07:36 INFO Executor:59 - Finished task 2.0 in stage 76.0 (TID 165). 1109 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 163) in 10 ms on localhost (1/4) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 2.0 in stage 76.0 (TID 165) in 9 ms on localhost (2/4) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 164) in 11 ms on localhost (3/4) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 3.0 in stage 76.0 (TID 166) in 10 ms on localhost (4/4) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-05-23 22:07:36 INFO DAGScheduler:59 - ResultStage 76 (treeAggregate at RowMatrix.scala:124) finished in 0.013 s -2016-05-23 22:07:36 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at RowMatrix.scala:124, took 0.031794 s -2016-05-23 22:07:36 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Got job 47 (sortByKey at SpearmanCorrelation.scala:55) with 4 output partitions -2016-05-23 22:07:36 INFO DAGScheduler:59 - Final stage: ResultStage 77(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[91] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=400824, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(1862) called with curMem=404200, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 1862.0 B, free 730.2 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:53983 (size: 1862.0 B, free: 730.5 MB) -2016-05-23 22:07:36 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 77 (MapPartitionsRDD[91] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 4 tasks -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 167, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 168, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 2.0 in stage 77.0 (TID 169, localhost, PROCESS_LOCAL, 2248 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 3.0 in stage 77.0 (TID 170, localhost, PROCESS_LOCAL, 2264 bytes) -2016-05-23 22:07:36 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 167) -2016-05-23 22:07:36 INFO Executor:59 - Running task 2.0 in stage 77.0 (TID 169) -2016-05-23 22:07:36 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 168) -2016-05-23 22:07:36 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 167). 1254 bytes result sent to driver -2016-05-23 22:07:36 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 168). 1254 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 167) in 6 ms on localhost (1/4) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 168) in 6 ms on localhost (2/4) -2016-05-23 22:07:36 INFO Executor:59 - Running task 3.0 in stage 77.0 (TID 170) -2016-05-23 22:07:36 INFO Executor:59 - Finished task 2.0 in stage 77.0 (TID 169). 1249 bytes result sent to driver -2016-05-23 22:07:36 INFO Executor:59 - Finished task 3.0 in stage 77.0 (TID 170). 1294 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 2.0 in stage 77.0 (TID 169) in 9 ms on localhost (3/4) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 3.0 in stage 77.0 (TID 170) in 9 ms on localhost (4/4) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-05-23 22:07:36 INFO DAGScheduler:59 - ResultStage 77 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.011 s -2016-05-23 22:07:36 INFO DAGScheduler:59 - Job 47 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.021961 s -2016-05-23 22:07:36 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Registering RDD 89 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Got job 48 (zipWithIndex at SpearmanCorrelation.scala:57) with 3 output partitions -2016-05-23 22:07:36 INFO DAGScheduler:59 - Final stage: ResultStage 79(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 78) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 78) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting ShuffleMapStage 78 (MapPartitionsRDD[89] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=406062, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(2289) called with curMem=410102, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:53983 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 78 (MapPartitionsRDD[89] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 4 tasks -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 171, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 172, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 2.0 in stage 78.0 (TID 173, localhost, PROCESS_LOCAL, 2237 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 3.0 in stage 78.0 (TID 174, localhost, PROCESS_LOCAL, 2253 bytes) -2016-05-23 22:07:36 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 172) -2016-05-23 22:07:36 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 171) -2016-05-23 22:07:36 INFO Executor:59 - Running task 2.0 in stage 78.0 (TID 173) -2016-05-23 22:07:36 INFO Executor:59 - Running task 3.0 in stage 78.0 (TID 174) -2016-05-23 22:07:36 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 172). 1161 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 172) in 38 ms on localhost (1/4) -2016-05-23 22:07:36 INFO Executor:59 - Finished task 3.0 in stage 78.0 (TID 174). 1161 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 3.0 in stage 78.0 (TID 174) in 62 ms on localhost (2/4) -2016-05-23 22:07:36 INFO Executor:59 - Finished task 2.0 in stage 78.0 (TID 173). 1161 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 2.0 in stage 78.0 (TID 173) in 72 ms on localhost (3/4) -2016-05-23 22:07:36 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 171). 1161 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 171) in 75 ms on localhost (4/4) -2016-05-23 22:07:36 INFO DAGScheduler:59 - ShuffleMapStage 78 (flatMap at SpearmanCorrelation.scala:49) finished in 0.075 s -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-05-23 22:07:36 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:07:36 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:07:36 INFO DAGScheduler:59 - waiting: Set(ResultStage 79) -2016-05-23 22:07:36 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Missing parents for ResultStage 79: List() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting ResultStage 79 (ShuffledRDD[92] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=412391, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(1573) called with curMem=414935, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 1573.0 B, free 730.2 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:53983 (size: 1573.0 B, free: 730.5 MB) -2016-05-23 22:07:36 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 79 (ShuffledRDD[92] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Adding task set 79.0 with 3 tasks -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 79.0 (TID 175, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 79.0 (TID 176, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 2.0 in stage 79.0 (TID 177, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:36 INFO Executor:59 - Running task 1.0 in stage 79.0 (TID 176) -2016-05-23 22:07:36 INFO Executor:59 - Running task 2.0 in stage 79.0 (TID 177) -2016-05-23 22:07:36 INFO Executor:59 - Running task 0.0 in stage 79.0 (TID 175) -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:36 INFO Executor:59 - Finished task 0.0 in stage 79.0 (TID 175). 1203 bytes result sent to driver -2016-05-23 22:07:36 INFO Executor:59 - Finished task 1.0 in stage 79.0 (TID 176). 1203 bytes result sent to driver -2016-05-23 22:07:36 INFO Executor:59 - Finished task 2.0 in stage 79.0 (TID 177). 1203 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 79.0 (TID 175) in 13 ms on localhost (1/3) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 2.0 in stage 79.0 (TID 177) in 13 ms on localhost (2/3) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 79.0 (TID 176) in 14 ms on localhost (3/3) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 79.0, whose tasks have all completed, from pool -2016-05-23 22:07:36 INFO DAGScheduler:59 - ResultStage 79 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.016 s -2016-05-23 22:07:36 INFO DAGScheduler:59 - Job 48 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.116337 s -2016-05-23 22:07:36 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:07:36 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 166 bytes -2016-05-23 22:07:36 INFO DAGScheduler:59 - Registering RDD 94 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Got job 49 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:07:36 INFO DAGScheduler:59 - Final stage: ResultStage 82(first at RowMatrix.scala:65) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 81) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 81) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting ShuffleMapStage 81 (MapPartitionsRDD[94] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(4232) called with curMem=416508, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(2313) called with curMem=420740, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:53983 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 81 (MapPartitionsRDD[94] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Adding task set 81.0 with 4 tasks -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 81.0 (TID 178, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 81.0 (TID 179, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 2.0 in stage 81.0 (TID 180, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 3.0 in stage 81.0 (TID 181, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:07:36 INFO Executor:59 - Running task 2.0 in stage 81.0 (TID 180) -2016-05-23 22:07:36 INFO Executor:59 - Running task 0.0 in stage 81.0 (TID 178) -2016-05-23 22:07:36 INFO Executor:59 - Running task 3.0 in stage 81.0 (TID 181) -2016-05-23 22:07:36 INFO Executor:59 - Running task 1.0 in stage 81.0 (TID 179) -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:36 INFO Executor:59 - Finished task 3.0 in stage 81.0 (TID 181). 1377 bytes result sent to driver -2016-05-23 22:07:36 INFO Executor:59 - Finished task 0.0 in stage 81.0 (TID 178). 1377 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 81.0 (TID 178) in 68 ms on localhost (1/4) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 3.0 in stage 81.0 (TID 181) in 67 ms on localhost (2/4) -2016-05-23 22:07:36 INFO Executor:59 - Finished task 1.0 in stage 81.0 (TID 179). 1377 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 81.0 (TID 179) in 84 ms on localhost (3/4) -2016-05-23 22:07:36 INFO Executor:59 - Finished task 2.0 in stage 81.0 (TID 180). 1377 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 2.0 in stage 81.0 (TID 180) in 87 ms on localhost (4/4) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 81.0, whose tasks have all completed, from pool -2016-05-23 22:07:36 INFO DAGScheduler:59 - ShuffleMapStage 81 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.088 s -2016-05-23 22:07:36 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:07:36 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:07:36 INFO DAGScheduler:59 - waiting: Set(ResultStage 82) -2016-05-23 22:07:36 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Missing parents for ResultStage 82: List() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting ResultStage 82 (MapPartitionsRDD[96] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=423053, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(2562) called with curMem=427949, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:53983 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO SparkContext:59 - Created broadcast 61 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 82 (MapPartitionsRDD[96] at map at SpearmanCorrelation.scala:87) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Adding task set 82.0 with 1 tasks -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 82.0 (TID 182, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:36 INFO Executor:59 - Running task 0.0 in stage 82.0 (TID 182) -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:36 INFO Executor:59 - Finished task 0.0 in stage 82.0 (TID 182). 1296 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 82.0 (TID 182) in 10 ms on localhost (1/1) -2016-05-23 22:07:36 INFO DAGScheduler:59 - ResultStage 82 (first at RowMatrix.scala:65) finished in 0.010 s -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 82.0, whose tasks have all completed, from pool -2016-05-23 22:07:36 INFO DAGScheduler:59 - Job 49 finished: first at RowMatrix.scala:65, took 0.117979 s -2016-05-23 22:07:36 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:07:36 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 166 bytes -2016-05-23 22:07:36 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 169 bytes -2016-05-23 22:07:36 INFO DAGScheduler:59 - Got job 50 (treeAggregate at RowMatrix.scala:332) with 4 output partitions -2016-05-23 22:07:36 INFO DAGScheduler:59 - Final stage: ResultStage 85(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 84) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting ResultStage 85 (MapPartitionsRDD[97] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(5696) called with curMem=430511, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(2943) called with curMem=436207, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.2 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:53983 (size: 2.9 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 85 (MapPartitionsRDD[97] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Adding task set 85.0 with 4 tasks -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 85.0 (TID 183, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 85.0 (TID 184, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 2.0 in stage 85.0 (TID 185, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 3.0 in stage 85.0 (TID 186, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:36 INFO Executor:59 - Running task 2.0 in stage 85.0 (TID 185) -2016-05-23 22:07:36 INFO Executor:59 - Running task 3.0 in stage 85.0 (TID 186) -2016-05-23 22:07:36 INFO Executor:59 - Running task 0.0 in stage 85.0 (TID 183) -2016-05-23 22:07:36 INFO Executor:59 - Running task 1.0 in stage 85.0 (TID 184) -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:36 INFO Executor:59 - Finished task 2.0 in stage 85.0 (TID 185). 1481 bytes result sent to driver -2016-05-23 22:07:36 INFO Executor:59 - Finished task 0.0 in stage 85.0 (TID 183). 1481 bytes result sent to driver -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:36 INFO Executor:59 - Finished task 1.0 in stage 85.0 (TID 184). 1481 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 85.0 (TID 184) in 14 ms on localhost (1/4) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 85.0 (TID 183) in 16 ms on localhost (2/4) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 2.0 in stage 85.0 (TID 185) in 15 ms on localhost (3/4) -2016-05-23 22:07:36 INFO Executor:59 - Finished task 3.0 in stage 85.0 (TID 186). 1481 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 3.0 in stage 85.0 (TID 186) in 19 ms on localhost (4/4) -2016-05-23 22:07:36 INFO DAGScheduler:59 - ResultStage 85 (treeAggregate at RowMatrix.scala:332) finished in 0.020 s -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 85.0, whose tasks have all completed, from pool -2016-05-23 22:07:36 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at RowMatrix.scala:332, took 0.033692 s -2016-05-23 22:07:36 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Got job 51 (treeAggregate at RowMatrix.scala:124) with 4 output partitions -2016-05-23 22:07:36 INFO DAGScheduler:59 - Final stage: ResultStage 88(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 87) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting ResultStage 88 (MapPartitionsRDD[98] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(5736) called with curMem=439150, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(2920) called with curMem=444886, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.2 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:53983 (size: 2.9 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 88 (MapPartitionsRDD[98] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Adding task set 88.0 with 4 tasks -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 88.0 (TID 187, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 88.0 (TID 188, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 2.0 in stage 88.0 (TID 189, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 3.0 in stage 88.0 (TID 190, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:36 INFO Executor:59 - Running task 1.0 in stage 88.0 (TID 188) -2016-05-23 22:07:36 INFO Executor:59 - Running task 2.0 in stage 88.0 (TID 189) -2016-05-23 22:07:36 INFO Executor:59 - Running task 3.0 in stage 88.0 (TID 190) -2016-05-23 22:07:36 INFO Executor:59 - Running task 0.0 in stage 88.0 (TID 187) -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:36 INFO Executor:59 - Finished task 2.0 in stage 88.0 (TID 189). 1359 bytes result sent to driver -2016-05-23 22:07:36 INFO Executor:59 - Finished task 0.0 in stage 88.0 (TID 187). 1359 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 2.0 in stage 88.0 (TID 189) in 10 ms on localhost (1/4) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 88.0 (TID 187) in 12 ms on localhost (2/4) -2016-05-23 22:07:36 INFO Executor:59 - Finished task 1.0 in stage 88.0 (TID 188). 1359 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 88.0 (TID 188) in 16 ms on localhost (3/4) -2016-05-23 22:07:36 INFO Executor:59 - Finished task 3.0 in stage 88.0 (TID 190). 1359 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 3.0 in stage 88.0 (TID 190) in 17 ms on localhost (4/4) -2016-05-23 22:07:36 INFO DAGScheduler:59 - ResultStage 88 (treeAggregate at RowMatrix.scala:124) finished in 0.018 s -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 88.0, whose tasks have all completed, from pool -2016-05-23 22:07:36 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at RowMatrix.scala:124, took 0.028276 s -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(88528) called with curMem=447806, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 86.5 KB, free 730.1 MB) -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=536334, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:53983 (size: 19.3 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO SparkContext:59 - Created broadcast 64 from textFile at StatisticsCorrLearning.scala:98 -2016-05-23 22:07:36 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 22:07:36 INFO SparkContext:59 - Starting job: foreach at StatisticsCorrLearning.scala:103 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Got job 52 (foreach at StatisticsCorrLearning.scala:103) with 2 output partitions -2016-05-23 22:07:36 INFO DAGScheduler:59 - Final stage: ResultStage 89(foreach at StatisticsCorrLearning.scala:103) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting ResultStage 89 (MapPartitionsRDD[102] at map at StatisticsCorrLearning.scala:101), which has no missing parents -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=556122, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(1982) called with curMem=559634, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 1982.0 B, free 730.1 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:53983 (size: 1982.0 B, free: 730.5 MB) -2016-05-23 22:07:36 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 89 (MapPartitionsRDD[102] at map at StatisticsCorrLearning.scala:101) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Adding task set 89.0 with 2 tasks -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 89.0 (TID 191, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 89.0 (TID 192, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:07:36 INFO Executor:59 - Running task 0.0 in stage 89.0 (TID 191) -2016-05-23 22:07:36 INFO Executor:59 - Running task 1.0 in stage 89.0 (TID 192) -2016-05-23 22:07:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 22:07:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 22:07:36 INFO Executor:59 - Finished task 1.0 in stage 89.0 (TID 192). 2044 bytes result sent to driver -2016-05-23 22:07:36 INFO Executor:59 - Finished task 0.0 in stage 89.0 (TID 191). 2044 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 89.0 (TID 192) in 9 ms on localhost (1/2) -2016-05-23 22:07:36 INFO DAGScheduler:59 - ResultStage 89 (foreach at StatisticsCorrLearning.scala:103) finished in 0.010 s -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 89.0 (TID 191) in 10 ms on localhost (2/2) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 89.0, whose tasks have all completed, from pool -2016-05-23 22:07:36 INFO DAGScheduler:59 - Job 52 finished: foreach at StatisticsCorrLearning.scala:103, took 0.019978 s -2016-05-23 22:07:36 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Got job 53 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:07:36 INFO DAGScheduler:59 - Final stage: ResultStage 90(first at RowMatrix.scala:65) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting ResultStage 90 (MapPartitionsRDD[102] at map at StatisticsCorrLearning.scala:101), which has no missing parents -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(3552) called with curMem=561616, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(2003) called with curMem=565168, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2003.0 B, free 730.0 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:53983 (size: 2003.0 B, free: 730.5 MB) -2016-05-23 22:07:36 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 90 (MapPartitionsRDD[102] at map at StatisticsCorrLearning.scala:101) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Adding task set 90.0 with 1 tasks -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 90.0 (TID 193, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:07:36 INFO Executor:59 - Running task 0.0 in stage 90.0 (TID 193) -2016-05-23 22:07:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 22:07:36 INFO Executor:59 - Finished task 0.0 in stage 90.0 (TID 193). 2199 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 90.0 (TID 193) in 8 ms on localhost (1/1) -2016-05-23 22:07:36 INFO DAGScheduler:59 - ResultStage 90 (first at RowMatrix.scala:65) finished in 0.009 s -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 90.0, whose tasks have all completed, from pool -2016-05-23 22:07:36 INFO DAGScheduler:59 - Job 53 finished: first at RowMatrix.scala:65, took 0.015942 s -2016-05-23 22:07:36 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Got job 54 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 22:07:36 INFO DAGScheduler:59 - Final stage: ResultStage 91(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting ResultStage 91 (MapPartitionsRDD[103] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(4472) called with curMem=567171, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 4.4 KB, free 730.0 MB) -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(2372) called with curMem=571643, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.0 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:53983 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO SparkContext:59 - Created broadcast 67 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 91 (MapPartitionsRDD[103] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Adding task set 91.0 with 2 tasks -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 91.0 (TID 194, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 91.0 (TID 195, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:07:36 INFO Executor:59 - Running task 1.0 in stage 91.0 (TID 195) -2016-05-23 22:07:36 INFO Executor:59 - Running task 0.0 in stage 91.0 (TID 194) -2016-05-23 22:07:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 22:07:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 22:07:36 INFO Executor:59 - Finished task 1.0 in stage 91.0 (TID 195). 2384 bytes result sent to driver -2016-05-23 22:07:36 INFO Executor:59 - Finished task 0.0 in stage 91.0 (TID 194). 2384 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 91.0 (TID 195) in 9 ms on localhost (1/2) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 91.0 (TID 194) in 10 ms on localhost (2/2) -2016-05-23 22:07:36 INFO DAGScheduler:59 - ResultStage 91 (treeAggregate at RowMatrix.scala:332) finished in 0.010 s -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 91.0, whose tasks have all completed, from pool -2016-05-23 22:07:36 INFO DAGScheduler:59 - Job 54 finished: treeAggregate at RowMatrix.scala:332, took 0.018739 s -2016-05-23 22:07:36 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Got job 55 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 22:07:36 INFO DAGScheduler:59 - Final stage: ResultStage 92(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting ResultStage 92 (MapPartitionsRDD[104] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(4584) called with curMem=574015, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 4.5 KB, free 730.0 MB) -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(2364) called with curMem=578599, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.0 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:53983 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 92 (MapPartitionsRDD[104] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Adding task set 92.0 with 2 tasks -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 92.0 (TID 196, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 92.0 (TID 197, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:07:36 INFO Executor:59 - Running task 0.0 in stage 92.0 (TID 196) -2016-05-23 22:07:36 INFO Executor:59 - Running task 1.0 in stage 92.0 (TID 197) -2016-05-23 22:07:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 22:07:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 22:07:36 INFO Executor:59 - Finished task 1.0 in stage 92.0 (TID 197). 2337 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 92.0 (TID 197) in 8 ms on localhost (1/2) -2016-05-23 22:07:36 INFO Executor:59 - Finished task 0.0 in stage 92.0 (TID 196). 2337 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 92.0 (TID 196) in 9 ms on localhost (2/2) -2016-05-23 22:07:36 INFO DAGScheduler:59 - ResultStage 92 (treeAggregate at RowMatrix.scala:124) finished in 0.009 s -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 92.0, whose tasks have all completed, from pool -2016-05-23 22:07:36 INFO DAGScheduler:59 - Job 55 finished: treeAggregate at RowMatrix.scala:124, took 0.018156 s -2016-05-23 22:07:36 INFO SparkContext:59 - Starting job: sortByKey at SpearmanCorrelation.scala:55 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Got job 56 (sortByKey at SpearmanCorrelation.scala:55) with 2 output partitions -2016-05-23 22:07:36 INFO DAGScheduler:59 - Final stage: ResultStage 93(sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting ResultStage 93 (MapPartitionsRDD[108] at sortByKey at SpearmanCorrelation.scala:55), which has no missing parents -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(4432) called with curMem=580963, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 4.3 KB, free 730.0 MB) -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(2326) called with curMem=585395, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.0 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:53983 (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO SparkContext:59 - Created broadcast 69 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 93 (MapPartitionsRDD[108] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Adding task set 93.0 with 2 tasks -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 93.0 (TID 198, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 93.0 (TID 199, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-23 22:07:36 INFO Executor:59 - Running task 1.0 in stage 93.0 (TID 199) -2016-05-23 22:07:36 INFO Executor:59 - Running task 0.0 in stage 93.0 (TID 198) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:53983 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-05-23 22:07:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 22:07:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:53983 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO Executor:59 - Finished task 0.0 in stage 93.0 (TID 198). 2443 bytes result sent to driver -2016-05-23 22:07:36 INFO Executor:59 - Finished task 1.0 in stage 93.0 (TID 199). 2443 bytes result sent to driver -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 93.0 (TID 198) in 22 ms on localhost (1/2) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 93.0 (TID 199) in 23 ms on localhost (2/2) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 93.0, whose tasks have all completed, from pool -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:53983 in memory (size: 2003.0 B, free: 730.5 MB) -2016-05-23 22:07:36 INFO DAGScheduler:59 - ResultStage 93 (sortByKey at SpearmanCorrelation.scala:55) finished in 0.024 s -2016-05-23 22:07:36 INFO DAGScheduler:59 - Job 56 finished: sortByKey at SpearmanCorrelation.scala:55, took 0.057290 s -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:53983 in memory (size: 1982.0 B, free: 730.5 MB) -2016-05-23 22:07:36 INFO SparkContext:59 - Starting job: zipWithIndex at SpearmanCorrelation.scala:57 -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Registering RDD 106 (flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Got job 57 (zipWithIndex at SpearmanCorrelation.scala:57) with 1 output partitions -2016-05-23 22:07:36 INFO DAGScheduler:59 - Final stage: ResultStage 95(zipWithIndex at SpearmanCorrelation.scala:57) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 94) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 94) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting ShuffleMapStage 94 (MapPartitionsRDD[106] at flatMap at SpearmanCorrelation.scala:49), which has no missing parents -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:53983 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(5024) called with curMem=554224, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 4.9 KB, free 730.1 MB) -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(2736) called with curMem=559248, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.1 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:53983 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:53983 (size: 2.7 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-05-23 22:07:36 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 94 (MapPartitionsRDD[106] at flatMap at SpearmanCorrelation.scala:49) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Adding task set 94.0 with 2 tasks -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 94.0 (TID 200, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 94.0 (TID 201, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-23 22:07:36 INFO Executor:59 - Running task 0.0 in stage 94.0 (TID 200) -2016-05-23 22:07:36 INFO Executor:59 - Running task 1.0 in stage 94.0 (TID 201) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:53983 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-05-23 22:07:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:10+11 -2016-05-23 22:07:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StatisticsCorrLearningx.txt:0+10 -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:53983 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned shuffle 9 -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:53983 in memory (size: 1573.0 B, free: 730.5 MB) -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:53983 in memory (size: 2.2 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned shuffle 8 -2016-05-23 22:07:36 INFO Executor:59 - Finished task 0.0 in stage 94.0 (TID 200). 2254 bytes result sent to driver -2016-05-23 22:07:36 INFO Executor:59 - Finished task 1.0 in stage 94.0 (TID 201). 2254 bytes result sent to driver -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:53983 in memory (size: 1862.0 B, free: 730.5 MB) -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 94.0 (TID 201) in 34 ms on localhost (1/2) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 94.0 (TID 200) in 36 ms on localhost (2/2) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 94.0, whose tasks have all completed, from pool -2016-05-23 22:07:36 INFO DAGScheduler:59 - ShuffleMapStage 94 (flatMap at SpearmanCorrelation.scala:49) finished in 0.036 s -2016-05-23 22:07:36 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:07:36 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:07:36 INFO DAGScheduler:59 - waiting: Set(ResultStage 95) -2016-05-23 22:07:36 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Missing parents for ResultStage 95: List() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting ResultStage 95 (ShuffledRDD[109] at sortByKey at SpearmanCorrelation.scala:55), which is now runnable -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:53983 in memory (size: 1919.0 B, free: 730.5 MB) -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=518299, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(1551) called with curMem=518868, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 1551.0 B, free 730.1 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:53983 in memory (size: 1935.0 B, free: 730.5 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:53983 (size: 1551.0 B, free: 730.5 MB) -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-05-23 22:07:36 INFO SparkContext:59 - Created broadcast 71 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 95 (ShuffledRDD[109] at sortByKey at SpearmanCorrelation.scala:55) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Adding task set 95.0 with 1 tasks -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 95.0 (TID 202, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:36 INFO Executor:59 - Running task 0.0 in stage 95.0 (TID 202) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:53983 in memory (size: 1524.0 B, free: 730.5 MB) -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:53983 in memory (size: 894.0 B, free: 730.5 MB) -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-05-23 22:07:36 INFO Executor:59 - Finished task 0.0 in stage 95.0 (TID 202). 1203 bytes result sent to driver -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:53983 in memory (size: 893.0 B, free: 730.5 MB) -2016-05-23 22:07:36 INFO DAGScheduler:59 - ResultStage 95 (zipWithIndex at SpearmanCorrelation.scala:57) finished in 0.010 s -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 95.0 (TID 202) in 9 ms on localhost (1/1) -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Job 57 finished: zipWithIndex at SpearmanCorrelation.scala:57, took 0.070392 s -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 95.0, whose tasks have all completed, from pool -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:53983 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:53983 in memory (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:53983 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:53983 in memory (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:53983 in memory (size: 2.3 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-05-23 22:07:36 INFO ContextCleaner:59 - Cleaned shuffle 7 -2016-05-23 22:07:36 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-05-23 22:07:36 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 152 bytes -2016-05-23 22:07:36 INFO DAGScheduler:59 - Registering RDD 111 (mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Got job 58 (first at RowMatrix.scala:65) with 1 output partitions -2016-05-23 22:07:36 INFO DAGScheduler:59 - Final stage: ResultStage 98(first at RowMatrix.scala:65) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 97) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 97) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting ShuffleMapStage 97 (MapPartitionsRDD[111] at mapPartitions at SpearmanCorrelation.scala:57), which has no missing parents -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=470089, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(2287) called with curMem=474281, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:53983 (size: 2.2 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 97 (MapPartitionsRDD[111] at mapPartitions at SpearmanCorrelation.scala:57) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Adding task set 97.0 with 2 tasks -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 97.0 (TID 203, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 97.0 (TID 204, localhost, PROCESS_LOCAL, 2000 bytes) -2016-05-23 22:07:36 INFO Executor:59 - Running task 1.0 in stage 97.0 (TID 204) -2016-05-23 22:07:36 INFO Executor:59 - Running task 0.0 in stage 97.0 (TID 203) -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:36 INFO Executor:59 - Finished task 1.0 in stage 97.0 (TID 204). 1375 bytes result sent to driver -2016-05-23 22:07:36 INFO Executor:59 - Finished task 0.0 in stage 97.0 (TID 203). 1375 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 97.0 (TID 204) in 23 ms on localhost (1/2) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 97.0 (TID 203) in 25 ms on localhost (2/2) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 97.0, whose tasks have all completed, from pool -2016-05-23 22:07:36 INFO DAGScheduler:59 - ShuffleMapStage 97 (mapPartitions at SpearmanCorrelation.scala:57) finished in 0.027 s -2016-05-23 22:07:36 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-23 22:07:36 INFO DAGScheduler:59 - running: Set() -2016-05-23 22:07:36 INFO DAGScheduler:59 - waiting: Set(ResultStage 98) -2016-05-23 22:07:36 INFO DAGScheduler:59 - failed: Set() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Missing parents for ResultStage 98: List() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting ResultStage 98 (MapPartitionsRDD[113] at map at SpearmanCorrelation.scala:87), which is now runnable -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(4856) called with curMem=476568, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 4.7 KB, free 730.1 MB) -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(2531) called with curMem=481424, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:53983 (size: 2.5 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO SparkContext:59 - Created broadcast 73 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 98 (MapPartitionsRDD[113] at map at SpearmanCorrelation.scala:87) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Adding task set 98.0 with 1 tasks -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 98.0 (TID 205, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:36 INFO Executor:59 - Running task 0.0 in stage 98.0 (TID 205) -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:36 INFO Executor:59 - Finished task 0.0 in stage 98.0 (TID 205). 1320 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 98.0 (TID 205) in 8 ms on localhost (1/1) -2016-05-23 22:07:36 INFO DAGScheduler:59 - ResultStage 98 (first at RowMatrix.scala:65) finished in 0.008 s -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 98.0, whose tasks have all completed, from pool -2016-05-23 22:07:36 INFO DAGScheduler:59 - Job 58 finished: first at RowMatrix.scala:65, took 0.056000 s -2016-05-23 22:07:36 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-05-23 22:07:36 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 152 bytes -2016-05-23 22:07:36 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 152 bytes -2016-05-23 22:07:36 INFO DAGScheduler:59 - Got job 59 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-05-23 22:07:36 INFO DAGScheduler:59 - Final stage: ResultStage 101(treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 100) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting ResultStage 101 (MapPartitionsRDD[114] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(5680) called with curMem=483955, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(2901) called with curMem=489635, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:53983 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 101 (MapPartitionsRDD[114] at treeAggregate at RowMatrix.scala:332) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Adding task set 101.0 with 2 tasks -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 101.0 (TID 206, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 101.0 (TID 207, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:36 INFO Executor:59 - Running task 1.0 in stage 101.0 (TID 207) -2016-05-23 22:07:36 INFO Executor:59 - Running task 0.0 in stage 101.0 (TID 206) -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:36 INFO Executor:59 - Finished task 1.0 in stage 101.0 (TID 207). 1505 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 101.0 (TID 207) in 9 ms on localhost (1/2) -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-23 22:07:36 INFO Executor:59 - Finished task 0.0 in stage 101.0 (TID 206). 1505 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 101.0 (TID 206) in 17 ms on localhost (2/2) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 101.0, whose tasks have all completed, from pool -2016-05-23 22:07:36 INFO DAGScheduler:59 - ResultStage 101 (treeAggregate at RowMatrix.scala:332) finished in 0.018 s -2016-05-23 22:07:36 INFO DAGScheduler:59 - Job 59 finished: treeAggregate at RowMatrix.scala:332, took 0.029105 s -2016-05-23 22:07:36 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Got job 60 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-05-23 22:07:36 INFO DAGScheduler:59 - Final stage: ResultStage 104(treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 103) -2016-05-23 22:07:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting ResultStage 104 (MapPartitionsRDD[115] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(5792) called with curMem=492536, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-05-23 22:07:36 INFO MemoryStore:59 - ensureFreeSpace(2886) called with curMem=498328, maxMem=766075207 -2016-05-23 22:07:36 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-23 22:07:36 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:53983 (size: 2.8 KB, free: 730.5 MB) -2016-05-23 22:07:36 INFO SparkContext:59 - Created broadcast 75 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:07:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 104 (MapPartitionsRDD[115] at treeAggregate at RowMatrix.scala:124) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Adding task set 104.0 with 2 tasks -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 104.0 (TID 208, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 104.0 (TID 209, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-23 22:07:36 INFO Executor:59 - Running task 0.0 in stage 104.0 (TID 208) -2016-05-23 22:07:36 INFO Executor:59 - Running task 1.0 in stage 104.0 (TID 209) -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-23 22:07:36 INFO Executor:59 - Finished task 1.0 in stage 104.0 (TID 209). 1458 bytes result sent to driver -2016-05-23 22:07:36 INFO Executor:59 - Finished task 0.0 in stage 104.0 (TID 208). 1458 bytes result sent to driver -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 104.0 (TID 209) in 11 ms on localhost (1/2) -2016-05-23 22:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 104.0 (TID 208) in 11 ms on localhost (2/2) -2016-05-23 22:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 104.0, whose tasks have all completed, from pool -2016-05-23 22:07:36 INFO DAGScheduler:59 - ResultStage 104 (treeAggregate at RowMatrix.scala:124) finished in 0.012 s -2016-05-23 22:07:36 INFO DAGScheduler:59 - Job 60 finished: treeAggregate at RowMatrix.scala:124, took 0.023093 s -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 22:07:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 22:07:37 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 22:07:37 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 22:07:37 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 22:07:37 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 22:07:37 INFO BlockManager:59 - BlockManager stopped -2016-05-23 22:07:37 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 22:07:37 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 22:07:37 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 22:07:37 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 22:07:37 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-c731b2d8-9b67-422c-88e0-b03b82fe281b -2016-05-23 22:07:37 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 22:07:37 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 22:13:14 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 22:13:15 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 22:13:15 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 22:13:15 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 22:13:15 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 22:13:20 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 22:13:20 INFO Remoting:74 - Starting remoting -2016-05-23 22:13:22 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53047] -2016-05-23 22:13:22 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53047. -2016-05-23 22:13:22 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 22:13:22 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 22:13:23 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-1b169d8d-a33a-47bd-b817-a6d73ad77908 -2016-05-23 22:13:23 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 22:13:23 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-e4afdfcd-49de-4ece-aad7-9f8acef47e74\httpd-6c6778e2-ece5-4bcc-bf64-b02446942841 -2016-05-23 22:13:23 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 22:13:23 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:13:23 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53048 -2016-05-23 22:13:23 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53048. -2016-05-23 22:13:24 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 22:13:24 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:13:24 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 22:13:24 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 22:13:24 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 22:13:25 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 22:13:25 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 22:13:26 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53067. -2016-05-23 22:13:26 INFO NettyBlockTransferService:59 - Server created on 53067 -2016-05-23 22:13:26 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 22:13:26 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53067 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53067) -2016-05-23 22:13:26 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 22:13:28 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 22:13:28 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 22:13:29 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 22:13:29 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 22:13:29 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53067 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 22:13:29 INFO SparkContext:59 - Created broadcast 0 from textFile at StratifiedSamplingLearning.scala:18 -2016-05-23 22:13:30 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 22:13:31 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 22:13:31 INFO SparkContext:59 - Starting job: foreach at StratifiedSamplingLearning.scala:27 -2016-05-23 22:13:31 INFO DAGScheduler:59 - Got job 0 (foreach at StratifiedSamplingLearning.scala:27) with 2 output partitions -2016-05-23 22:13:31 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StratifiedSamplingLearning.scala:27) -2016-05-23 22:13:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:13:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:13:31 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at sampleByKey at StratifiedSamplingLearning.scala:26), which has no missing parents -2016-05-23 22:13:31 INFO MemoryStore:59 - ensureFreeSpace(3880) called with curMem=171580, maxMem=766075207 -2016-05-23 22:13:31 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-23 22:13:31 INFO MemoryStore:59 - ensureFreeSpace(2220) called with curMem=175460, maxMem=766075207 -2016-05-23 22:13:31 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-05-23 22:13:31 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53067 (size: 2.2 KB, free: 730.6 MB) -2016-05-23 22:13:31 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:13:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at sampleByKey at StratifiedSamplingLearning.scala:26) -2016-05-23 22:13:31 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 22:13:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:13:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:13:31 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 22:13:31 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 22:13:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:0+10 -2016-05-23 22:13:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:10+11 -2016-05-23 22:13:31 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 22:13:31 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 22:13:31 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 22:13:31 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 22:13:31 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 22:13:31 ERROR Executor:96 - Exception in task 1.0 in stage 0.0 (TID 1) -java.util.NoSuchElementException: key not found: bbb - at scala.collection.MapLike$class.default(MapLike.scala:228) - at scala.collection.AbstractMap.default(Map.scala:58) - at scala.collection.MapLike$class.apply(MapLike.scala:141) - at scala.collection.AbstractMap.apply(Map.scala:58) - at org.apache.spark.util.random.StratifiedSamplingUtils$$anonfun$getBernoulliSamplingFunction$1$$anonfun$apply$5.apply(StratifiedSamplingUtils.scala:214) - at org.apache.spark.util.random.StratifiedSamplingUtils$$anonfun$getBernoulliSamplingFunction$1$$anonfun$apply$5.apply(StratifiedSamplingUtils.scala:214) - at scala.collection.Iterator$$anon$14.hasNext(Iterator.scala:390) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-05-23 22:13:31 ERROR Executor:96 - Exception in task 0.0 in stage 0.0 (TID 0) -java.util.NoSuchElementException: key not found: bb - at scala.collection.MapLike$class.default(MapLike.scala:228) - at scala.collection.AbstractMap.default(Map.scala:58) - at scala.collection.MapLike$class.apply(MapLike.scala:141) - at scala.collection.AbstractMap.apply(Map.scala:58) - at org.apache.spark.util.random.StratifiedSamplingUtils$$anonfun$getBernoulliSamplingFunction$1$$anonfun$apply$5.apply(StratifiedSamplingUtils.scala:214) - at org.apache.spark.util.random.StratifiedSamplingUtils$$anonfun$getBernoulliSamplingFunction$1$$anonfun$apply$5.apply(StratifiedSamplingUtils.scala:214) - at scala.collection.Iterator$$anon$14.hasNext(Iterator.scala:390) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-05-23 22:13:31 WARN TaskSetManager:71 - Lost task 0.0 in stage 0.0 (TID 0, localhost): java.util.NoSuchElementException: key not found: bb - at scala.collection.MapLike$class.default(MapLike.scala:228) - at scala.collection.AbstractMap.default(Map.scala:58) - at scala.collection.MapLike$class.apply(MapLike.scala:141) - at scala.collection.AbstractMap.apply(Map.scala:58) - at org.apache.spark.util.random.StratifiedSamplingUtils$$anonfun$getBernoulliSamplingFunction$1$$anonfun$apply$5.apply(StratifiedSamplingUtils.scala:214) - at org.apache.spark.util.random.StratifiedSamplingUtils$$anonfun$getBernoulliSamplingFunction$1$$anonfun$apply$5.apply(StratifiedSamplingUtils.scala:214) - at scala.collection.Iterator$$anon$14.hasNext(Iterator.scala:390) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) - -2016-05-23 22:13:31 ERROR TaskSetManager:75 - Task 0 in stage 0.0 failed 1 times; aborting job -2016-05-23 22:13:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 22:13:31 WARN TaskSetManager:71 - Lost task 1.0 in stage 0.0 (TID 1, localhost): java.util.NoSuchElementException: key not found: bbb - at scala.collection.MapLike$class.default(MapLike.scala:228) - at scala.collection.AbstractMap.default(Map.scala:58) - at scala.collection.MapLike$class.apply(MapLike.scala:141) - at scala.collection.AbstractMap.apply(Map.scala:58) - at org.apache.spark.util.random.StratifiedSamplingUtils$$anonfun$getBernoulliSamplingFunction$1$$anonfun$apply$5.apply(StratifiedSamplingUtils.scala:214) - at org.apache.spark.util.random.StratifiedSamplingUtils$$anonfun$getBernoulliSamplingFunction$1$$anonfun$apply$5.apply(StratifiedSamplingUtils.scala:214) - at scala.collection.Iterator$$anon$14.hasNext(Iterator.scala:390) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) - -2016-05-23 22:13:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 22:13:31 INFO TaskSchedulerImpl:59 - Cancelling stage 0 -2016-05-23 22:13:31 INFO DAGScheduler:59 - ResultStage 0 (foreach at StratifiedSamplingLearning.scala:27) failed in 0.488 s -2016-05-23 22:13:31 INFO DAGScheduler:59 - Job 0 failed: foreach at StratifiedSamplingLearning.scala:27, took 0.758134 s -2016-05-23 22:13:32 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 22:13:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 22:13:32 WARN QueuedThreadPool:145 - 5 threads could not be stopped -2016-05-23 22:13:32 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 22:13:32 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 22:13:32 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 22:13:32 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 22:13:32 INFO BlockManager:59 - BlockManager stopped -2016-05-23 22:13:32 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 22:13:33 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 22:13:33 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 22:13:33 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 22:13:33 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-e4afdfcd-49de-4ece-aad7-9f8acef47e74 -2016-05-23 22:13:33 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 22:13:33 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 22:15:28 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 22:15:29 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 22:15:29 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 22:15:29 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 22:15:29 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 22:15:30 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 22:15:31 INFO Remoting:74 - Starting remoting -2016-05-23 22:15:33 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53123] -2016-05-23 22:15:33 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53123. -2016-05-23 22:15:33 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 22:15:34 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 22:15:34 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-c4ebeb9a-d99f-432c-966b-5a6a15b9f3af -2016-05-23 22:15:34 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 22:15:35 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-4104107d-1cc1-46c3-86f1-4791bd5c497c\httpd-9a37af54-c363-459f-8a12-c59a89e05066 -2016-05-23 22:15:35 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 22:15:35 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:15:35 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53126 -2016-05-23 22:15:35 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53126. -2016-05-23 22:15:36 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 22:15:37 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:15:37 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 22:15:37 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 22:15:37 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 22:15:39 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 22:15:39 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 22:15:49 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53174. -2016-05-23 22:15:49 INFO NettyBlockTransferService:59 - Server created on 53174 -2016-05-23 22:15:49 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 22:15:49 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53174 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53174) -2016-05-23 22:15:49 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 22:15:52 INFO Executor:59 - Told to re-register on heartbeat -2016-05-23 22:15:52 INFO BlockManager:59 - BlockManager re-registering with master -2016-05-23 22:15:52 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 22:15:52 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 22:15:52 INFO BlockManager:59 - Reporting 0 blocks to the master. -2016-05-23 22:16:16 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 22:16:17 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 22:16:20 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 22:16:20 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 22:16:20 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53174 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 22:16:20 INFO SparkContext:59 - Created broadcast 0 from textFile at StratifiedSamplingLearning.scala:18 -2016-05-23 22:16:30 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 22:16:35 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 22:16:36 INFO SparkContext:59 - Starting job: foreach at StratifiedSamplingLearning.scala:27 -2016-05-23 22:16:38 INFO DAGScheduler:59 - Got job 0 (foreach at StratifiedSamplingLearning.scala:27) with 2 output partitions -2016-05-23 22:16:38 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StratifiedSamplingLearning.scala:27) -2016-05-23 22:16:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:16:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:16:38 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at sampleByKey at StratifiedSamplingLearning.scala:26), which has no missing parents -2016-05-23 22:16:40 INFO MemoryStore:59 - ensureFreeSpace(3880) called with curMem=171580, maxMem=766075207 -2016-05-23 22:16:40 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-23 22:16:40 INFO MemoryStore:59 - ensureFreeSpace(2220) called with curMem=175460, maxMem=766075207 -2016-05-23 22:16:40 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-05-23 22:16:40 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53174 (size: 2.2 KB, free: 730.6 MB) -2016-05-23 22:16:40 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:16:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at sampleByKey at StratifiedSamplingLearning.scala:26) -2016-05-23 22:16:40 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 22:16:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:16:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:16:44 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 22:16:44 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 22:16:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:0+10 -2016-05-23 22:16:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:10+11 -2016-05-23 22:16:45 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 22:16:45 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 22:16:45 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 22:16:45 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 22:16:45 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 22:16:45 ERROR Executor:96 - Exception in task 1.0 in stage 0.0 (TID 1) -java.util.NoSuchElementException: key not found: bbb - at scala.collection.MapLike$class.default(MapLike.scala:228) - at scala.collection.AbstractMap.default(Map.scala:58) - at scala.collection.MapLike$class.apply(MapLike.scala:141) - at scala.collection.AbstractMap.apply(Map.scala:58) - at org.apache.spark.util.random.StratifiedSamplingUtils$$anonfun$getBernoulliSamplingFunction$1$$anonfun$apply$5.apply(StratifiedSamplingUtils.scala:214) - at org.apache.spark.util.random.StratifiedSamplingUtils$$anonfun$getBernoulliSamplingFunction$1$$anonfun$apply$5.apply(StratifiedSamplingUtils.scala:214) - at scala.collection.Iterator$$anon$14.hasNext(Iterator.scala:390) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-05-23 22:16:45 ERROR Executor:96 - Exception in task 0.0 in stage 0.0 (TID 0) -java.util.NoSuchElementException: key not found: bb - at scala.collection.MapLike$class.default(MapLike.scala:228) - at scala.collection.AbstractMap.default(Map.scala:58) - at scala.collection.MapLike$class.apply(MapLike.scala:141) - at scala.collection.AbstractMap.apply(Map.scala:58) - at org.apache.spark.util.random.StratifiedSamplingUtils$$anonfun$getBernoulliSamplingFunction$1$$anonfun$apply$5.apply(StratifiedSamplingUtils.scala:214) - at org.apache.spark.util.random.StratifiedSamplingUtils$$anonfun$getBernoulliSamplingFunction$1$$anonfun$apply$5.apply(StratifiedSamplingUtils.scala:214) - at scala.collection.Iterator$$anon$14.hasNext(Iterator.scala:390) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-05-23 22:16:46 WARN TaskSetManager:71 - Lost task 0.0 in stage 0.0 (TID 0, localhost): java.util.NoSuchElementException: key not found: bb - at scala.collection.MapLike$class.default(MapLike.scala:228) - at scala.collection.AbstractMap.default(Map.scala:58) - at scala.collection.MapLike$class.apply(MapLike.scala:141) - at scala.collection.AbstractMap.apply(Map.scala:58) - at org.apache.spark.util.random.StratifiedSamplingUtils$$anonfun$getBernoulliSamplingFunction$1$$anonfun$apply$5.apply(StratifiedSamplingUtils.scala:214) - at org.apache.spark.util.random.StratifiedSamplingUtils$$anonfun$getBernoulliSamplingFunction$1$$anonfun$apply$5.apply(StratifiedSamplingUtils.scala:214) - at scala.collection.Iterator$$anon$14.hasNext(Iterator.scala:390) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) - -2016-05-23 22:16:46 ERROR TaskSetManager:75 - Task 0 in stage 0.0 failed 1 times; aborting job -2016-05-23 22:16:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 22:16:46 WARN TaskSetManager:71 - Lost task 1.0 in stage 0.0 (TID 1, localhost): java.util.NoSuchElementException: key not found: bbb - at scala.collection.MapLike$class.default(MapLike.scala:228) - at scala.collection.AbstractMap.default(Map.scala:58) - at scala.collection.MapLike$class.apply(MapLike.scala:141) - at scala.collection.AbstractMap.apply(Map.scala:58) - at org.apache.spark.util.random.StratifiedSamplingUtils$$anonfun$getBernoulliSamplingFunction$1$$anonfun$apply$5.apply(StratifiedSamplingUtils.scala:214) - at org.apache.spark.util.random.StratifiedSamplingUtils$$anonfun$getBernoulliSamplingFunction$1$$anonfun$apply$5.apply(StratifiedSamplingUtils.scala:214) - at scala.collection.Iterator$$anon$14.hasNext(Iterator.scala:390) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) - -2016-05-23 22:16:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 22:16:46 INFO TaskSchedulerImpl:59 - Cancelling stage 0 -2016-05-23 22:16:46 INFO DAGScheduler:59 - ResultStage 0 (foreach at StratifiedSamplingLearning.scala:27) failed in 5.348 s -2016-05-23 22:16:46 INFO DAGScheduler:59 - Job 0 failed: foreach at StratifiedSamplingLearning.scala:27, took 10.179482 s -2016-05-23 22:16:46 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 22:16:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 22:16:46 WARN QueuedThreadPool:145 - 6 threads could not be stopped -2016-05-23 22:16:46 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 22:16:47 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 22:16:47 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 22:16:48 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 22:16:48 INFO BlockManager:59 - BlockManager stopped -2016-05-23 22:16:48 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 22:16:48 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 22:16:48 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 22:16:48 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 22:16:48 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-4104107d-1cc1-46c3-86f1-4791bd5c497c -2016-05-23 22:24:40 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 22:24:41 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 22:24:42 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 22:24:42 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 22:24:42 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 22:24:51 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 22:24:51 INFO Remoting:74 - Starting remoting -2016-05-23 22:24:52 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53505] -2016-05-23 22:24:52 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53505. -2016-05-23 22:24:52 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 22:24:52 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 22:24:52 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-c863d3c5-f6bb-4ad6-9d90-cd131bfa3eaf -2016-05-23 22:24:52 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 22:24:52 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-3deafb79-72b1-4dbe-ad87-7605ed47b2c6\httpd-e9312841-6f0b-4f37-b33b-7cd6701df84a -2016-05-23 22:24:52 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 22:24:53 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:24:53 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53506 -2016-05-23 22:24:53 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53506. -2016-05-23 22:24:53 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 22:24:53 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:24:53 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 22:24:53 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 22:24:53 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 22:24:54 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 22:24:54 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 22:24:54 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53525. -2016-05-23 22:24:54 INFO NettyBlockTransferService:59 - Server created on 53525 -2016-05-23 22:24:54 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 22:24:54 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53525 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53525) -2016-05-23 22:24:54 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 22:24:55 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:33 -2016-05-23 22:24:55 INFO DAGScheduler:59 - Got job 0 (collect at StratifiedSamplingLearning.scala:33) with 4 output partitions -2016-05-23 22:24:55 INFO DAGScheduler:59 - Final stage: ResultStage 0(collect at StratifiedSamplingLearning.scala:33) -2016-05-23 22:24:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:24:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:24:55 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[1] at sampleByKey at StratifiedSamplingLearning.scala:33), which has no missing parents -2016-05-23 22:24:56 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=0, maxMem=766075207 -2016-05-23 22:24:56 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 2.3 KB, free 730.6 MB) -2016-05-23 22:24:56 INFO MemoryStore:59 - ensureFreeSpace(1465) called with curMem=2336, maxMem=766075207 -2016-05-23 22:24:56 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1465.0 B, free 730.6 MB) -2016-05-23 22:24:56 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53525 (size: 1465.0 B, free: 730.6 MB) -2016-05-23 22:24:56 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:24:56 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at sampleByKey at StratifiedSamplingLearning.scala:33) -2016-05-23 22:24:56 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 4 tasks -2016-05-23 22:24:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2188 bytes) -2016-05-23 22:24:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2212 bytes) -2016-05-23 22:24:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2210 bytes) -2016-05-23 22:24:56 INFO TaskSetManager:59 - Starting task 3.0 in stage 0.0 (TID 3, localhost, PROCESS_LOCAL, 2216 bytes) -2016-05-23 22:24:56 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-05-23 22:24:56 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 22:24:56 INFO Executor:59 - Running task 3.0 in stage 0.0 (TID 3) -2016-05-23 22:24:56 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 22:24:56 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1061 bytes result sent to driver -2016-05-23 22:24:56 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 1060 bytes result sent to driver -2016-05-23 22:24:56 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1059 bytes result sent to driver -2016-05-23 22:24:56 INFO Executor:59 - Finished task 3.0 in stage 0.0 (TID 3). 1087 bytes result sent to driver -2016-05-23 22:24:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 91 ms on localhost (1/4) -2016-05-23 22:24:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 92 ms on localhost (2/4) -2016-05-23 22:24:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 121 ms on localhost (3/4) -2016-05-23 22:24:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 0.0 (TID 3) in 94 ms on localhost (4/4) -2016-05-23 22:24:56 INFO DAGScheduler:59 - ResultStage 0 (collect at StratifiedSamplingLearning.scala:33) finished in 0.135 s -2016-05-23 22:24:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 22:24:56 INFO DAGScheduler:59 - Job 0 finished: collect at StratifiedSamplingLearning.scala:33, took 0.360304 s -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 22:24:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 22:24:56 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 22:24:56 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 22:24:56 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 22:24:56 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 22:24:56 INFO BlockManager:59 - BlockManager stopped -2016-05-23 22:24:56 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 22:24:56 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 22:24:56 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 22:24:56 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 22:24:56 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-3deafb79-72b1-4dbe-ad87-7605ed47b2c6 -2016-05-23 22:25:06 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 22:25:06 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 22:25:07 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 22:25:07 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 22:25:07 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 22:25:07 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 22:25:07 INFO Remoting:74 - Starting remoting -2016-05-23 22:25:08 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53542] -2016-05-23 22:25:08 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53542. -2016-05-23 22:25:08 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 22:25:08 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 22:25:08 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-fa953a51-0ef6-4bdb-8808-ebfc4d5ff3ab -2016-05-23 22:25:08 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 22:25:08 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-ae8b991a-5078-4c6d-b6c6-083144e957d0\httpd-b00f3f3e-5e95-4da9-8d24-0c701d3cbc68 -2016-05-23 22:25:08 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 22:25:08 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:25:08 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53543 -2016-05-23 22:25:08 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53543. -2016-05-23 22:25:08 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 22:25:08 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:25:08 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 22:25:08 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 22:25:08 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 22:25:08 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 22:25:08 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 22:25:09 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53562. -2016-05-23 22:25:09 INFO NettyBlockTransferService:59 - Server created on 53562 -2016-05-23 22:25:09 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 22:25:09 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53562 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53562) -2016-05-23 22:25:09 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 22:25:09 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:34 -2016-05-23 22:25:09 INFO DAGScheduler:59 - Got job 0 (collect at StratifiedSamplingLearning.scala:34) with 4 output partitions -2016-05-23 22:25:09 INFO DAGScheduler:59 - Final stage: ResultStage 0(collect at StratifiedSamplingLearning.scala:34) -2016-05-23 22:25:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:25:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:25:09 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[1] at sampleByKey at StratifiedSamplingLearning.scala:34), which has no missing parents -2016-05-23 22:25:09 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=0, maxMem=766075207 -2016-05-23 22:25:09 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 2.3 KB, free 730.6 MB) -2016-05-23 22:25:09 INFO MemoryStore:59 - ensureFreeSpace(1465) called with curMem=2336, maxMem=766075207 -2016-05-23 22:25:09 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1465.0 B, free 730.6 MB) -2016-05-23 22:25:09 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53562 (size: 1465.0 B, free: 730.6 MB) -2016-05-23 22:25:09 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:25:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at sampleByKey at StratifiedSamplingLearning.scala:34) -2016-05-23 22:25:09 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 4 tasks -2016-05-23 22:25:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2188 bytes) -2016-05-23 22:25:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2212 bytes) -2016-05-23 22:25:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2210 bytes) -2016-05-23 22:25:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 0.0 (TID 3, localhost, PROCESS_LOCAL, 2216 bytes) -2016-05-23 22:25:09 INFO Executor:59 - Running task 3.0 in stage 0.0 (TID 3) -2016-05-23 22:25:09 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-05-23 22:25:09 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 22:25:09 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 22:25:09 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1059 bytes result sent to driver -2016-05-23 22:25:09 INFO Executor:59 - Finished task 3.0 in stage 0.0 (TID 3). 1087 bytes result sent to driver -2016-05-23 22:25:09 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1061 bytes result sent to driver -2016-05-23 22:25:09 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 1060 bytes result sent to driver -2016-05-23 22:25:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 84 ms on localhost (1/4) -2016-05-23 22:25:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 61 ms on localhost (2/4) -2016-05-23 22:25:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 0.0 (TID 3) in 60 ms on localhost (3/4) -2016-05-23 22:25:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 64 ms on localhost (4/4) -2016-05-23 22:25:09 INFO DAGScheduler:59 - ResultStage 0 (collect at StratifiedSamplingLearning.scala:34) finished in 0.098 s -2016-05-23 22:25:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 22:25:09 INFO DAGScheduler:59 - Job 0 finished: collect at StratifiedSamplingLearning.scala:34, took 0.283817 s -2016-05-23 22:25:10 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:41 -2016-05-23 22:25:10 INFO DAGScheduler:59 - Got job 1 (collect at StratifiedSamplingLearning.scala:41) with 3 output partitions -2016-05-23 22:25:10 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at StratifiedSamplingLearning.scala:41) -2016-05-23 22:25:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:25:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:25:10 INFO DAGScheduler:59 - Submitting ResultStage 1 (ParallelCollectionRDD[2] at parallelize at StratifiedSamplingLearning.scala:38), which has no missing parents -2016-05-23 22:25:10 INFO MemoryStore:59 - ensureFreeSpace(1272) called with curMem=3801, maxMem=766075207 -2016-05-23 22:25:10 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1272.0 B, free 730.6 MB) -2016-05-23 22:25:10 INFO MemoryStore:59 - ensureFreeSpace(843) called with curMem=5073, maxMem=766075207 -2016-05-23 22:25:10 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 843.0 B, free 730.6 MB) -2016-05-23 22:25:10 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53562 (size: 843.0 B, free: 730.6 MB) -2016-05-23 22:25:10 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:25:10 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 1 (ParallelCollectionRDD[2] at parallelize at StratifiedSamplingLearning.scala:38) -2016-05-23 22:25:10 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 3 tasks -2016-05-23 22:25:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2085 bytes) -2016-05-23 22:25:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 5, localhost, PROCESS_LOCAL, 2085 bytes) -2016-05-23 22:25:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 6, localhost, PROCESS_LOCAL, 2142 bytes) -2016-05-23 22:25:10 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 4) -2016-05-23 22:25:10 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 5) -2016-05-23 22:25:10 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 6) -2016-05-23 22:25:10 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 5). 926 bytes result sent to driver -2016-05-23 22:25:10 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 6). 926 bytes result sent to driver -2016-05-23 22:25:10 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 4). 922 bytes result sent to driver -2016-05-23 22:25:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 5) in 15 ms on localhost (1/3) -2016-05-23 22:25:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 6) in 16 ms on localhost (2/3) -2016-05-23 22:25:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 4) in 23 ms on localhost (3/3) -2016-05-23 22:25:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 22:25:10 INFO DAGScheduler:59 - ResultStage 1 (collect at StratifiedSamplingLearning.scala:41) finished in 0.024 s -2016-05-23 22:25:10 INFO DAGScheduler:59 - Job 1 finished: collect at StratifiedSamplingLearning.scala:41, took 0.037086 s -2016-05-23 22:25:10 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:42 -2016-05-23 22:25:10 INFO DAGScheduler:59 - Got job 2 (collect at StratifiedSamplingLearning.scala:42) with 3 output partitions -2016-05-23 22:25:10 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at StratifiedSamplingLearning.scala:42) -2016-05-23 22:25:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:25:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:25:10 INFO DAGScheduler:59 - Submitting ResultStage 2 (PartitionwiseSampledRDD[3] at sample at StratifiedSamplingLearning.scala:39), which has no missing parents -2016-05-23 22:25:10 INFO MemoryStore:59 - ensureFreeSpace(18856) called with curMem=5916, maxMem=766075207 -2016-05-23 22:25:10 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 18.4 KB, free 730.6 MB) -2016-05-23 22:25:10 INFO MemoryStore:59 - ensureFreeSpace(7707) called with curMem=24772, maxMem=766075207 -2016-05-23 22:25:10 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.6 MB) -2016-05-23 22:25:10 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53562 (size: 7.5 KB, free: 730.6 MB) -2016-05-23 22:25:10 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:25:10 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 2 (PartitionwiseSampledRDD[3] at sample at StratifiedSamplingLearning.scala:39) -2016-05-23 22:25:10 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-05-23 22:25:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:25:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 8, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:25:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 9, localhost, PROCESS_LOCAL, 2251 bytes) -2016-05-23 22:25:10 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 7) -2016-05-23 22:25:10 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 9) -2016-05-23 22:25:10 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 8) -2016-05-23 22:25:10 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 8). 902 bytes result sent to driver -2016-05-23 22:25:10 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 7). 914 bytes result sent to driver -2016-05-23 22:25:10 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 9). 910 bytes result sent to driver -2016-05-23 22:25:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 8) in 16 ms on localhost (1/3) -2016-05-23 22:25:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 9) in 16 ms on localhost (2/3) -2016-05-23 22:25:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 7) in 19 ms on localhost (3/3) -2016-05-23 22:25:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 22:25:10 INFO DAGScheduler:59 - ResultStage 2 (collect at StratifiedSamplingLearning.scala:42) finished in 0.019 s -2016-05-23 22:25:10 INFO DAGScheduler:59 - Job 2 finished: collect at StratifiedSamplingLearning.scala:42, took 0.037949 s -2016-05-23 22:25:10 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:43 -2016-05-23 22:25:10 INFO DAGScheduler:59 - Got job 3 (collect at StratifiedSamplingLearning.scala:43) with 3 output partitions -2016-05-23 22:25:10 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at StratifiedSamplingLearning.scala:43) -2016-05-23 22:25:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:25:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:25:10 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[4] at sample at StratifiedSamplingLearning.scala:40), which has no missing parents -2016-05-23 22:25:10 INFO MemoryStore:59 - ensureFreeSpace(2192) called with curMem=32479, maxMem=766075207 -2016-05-23 22:25:10 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.1 KB, free 730.6 MB) -2016-05-23 22:25:10 INFO MemoryStore:59 - ensureFreeSpace(1350) called with curMem=34671, maxMem=766075207 -2016-05-23 22:25:10 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1350.0 B, free 730.6 MB) -2016-05-23 22:25:10 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53562 (size: 1350.0 B, free: 730.6 MB) -2016-05-23 22:25:10 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:25:10 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[4] at sample at StratifiedSamplingLearning.scala:40) -2016-05-23 22:25:10 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 3 tasks -2016-05-23 22:25:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:25:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 11, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:25:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 12, localhost, PROCESS_LOCAL, 2251 bytes) -2016-05-23 22:25:10 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 11) -2016-05-23 22:25:10 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 12) -2016-05-23 22:25:10 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 10) -2016-05-23 22:25:10 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 12). 926 bytes result sent to driver -2016-05-23 22:25:10 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 11). 914 bytes result sent to driver -2016-05-23 22:25:10 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 10). 914 bytes result sent to driver -2016-05-23 22:25:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 12) in 10 ms on localhost (1/3) -2016-05-23 22:25:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 11) in 12 ms on localhost (2/3) -2016-05-23 22:25:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 10) in 14 ms on localhost (3/3) -2016-05-23 22:25:10 INFO DAGScheduler:59 - ResultStage 3 (collect at StratifiedSamplingLearning.scala:43) finished in 0.014 s -2016-05-23 22:25:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 22:25:10 INFO DAGScheduler:59 - Job 3 finished: collect at StratifiedSamplingLearning.scala:43, took 0.026611 s -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 22:25:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 22:25:10 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 22:25:10 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 22:25:10 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 22:25:10 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 22:25:10 INFO BlockManager:59 - BlockManager stopped -2016-05-23 22:25:10 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 22:25:10 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 22:25:10 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 22:25:10 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 22:25:10 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-ae8b991a-5078-4c6d-b6c6-083144e957d0 -2016-05-23 22:28:05 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 22:28:06 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 22:28:06 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 22:28:06 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 22:28:06 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 22:28:07 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 22:28:07 INFO Remoting:74 - Starting remoting -2016-05-23 22:28:07 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53586] -2016-05-23 22:28:07 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53586. -2016-05-23 22:28:07 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 22:28:07 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 22:28:07 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-f3628931-de60-4a93-9d73-d27409e9f0c9 -2016-05-23 22:28:07 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 22:28:07 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-599e0a88-2638-410a-8082-973434390f85\httpd-560ff14e-de04-4608-84a7-fa788483d69d -2016-05-23 22:28:07 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 22:28:07 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:28:07 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53587 -2016-05-23 22:28:07 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53587. -2016-05-23 22:28:07 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 22:28:07 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:28:07 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 22:28:07 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 22:28:07 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 22:28:08 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 22:28:08 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 22:28:08 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53606. -2016-05-23 22:28:08 INFO NettyBlockTransferService:59 - Server created on 53606 -2016-05-23 22:28:08 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 22:28:08 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53606 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53606) -2016-05-23 22:28:08 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 22:28:09 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 22:28:09 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 22:28:09 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 22:28:09 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 22:28:09 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53606 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 22:28:09 INFO SparkContext:59 - Created broadcast 0 from textFile at StratifiedSamplingLearning.scala:19 -2016-05-23 22:28:10 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 22:28:10 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 22:28:10 INFO SparkContext:59 - Starting job: foreach at StratifiedSamplingLearning.scala:26 -2016-05-23 22:28:10 INFO DAGScheduler:59 - Got job 0 (foreach at StratifiedSamplingLearning.scala:26) with 2 output partitions -2016-05-23 22:28:10 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StratifiedSamplingLearning.scala:26) -2016-05-23 22:28:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:28:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:28:10 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at StratifiedSamplingLearning.scala:20), which has no missing parents -2016-05-23 22:28:10 INFO MemoryStore:59 - ensureFreeSpace(3304) called with curMem=171580, maxMem=766075207 -2016-05-23 22:28:10 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-05-23 22:28:10 INFO MemoryStore:59 - ensureFreeSpace(1915) called with curMem=174884, maxMem=766075207 -2016-05-23 22:28:10 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1915.0 B, free 730.4 MB) -2016-05-23 22:28:10 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53606 (size: 1915.0 B, free: 730.6 MB) -2016-05-23 22:28:10 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:28:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at StratifiedSamplingLearning.scala:20) -2016-05-23 22:28:10 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 22:28:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:28:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:28:10 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 22:28:10 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 22:28:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:10+11 -2016-05-23 22:28:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:0+10 -2016-05-23 22:28:10 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 22:28:10 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 22:28:10 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 22:28:10 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 22:28:10 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 22:28:10 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 22:28:10 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 22:28:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 90 ms on localhost (1/2) -2016-05-23 22:28:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 116 ms on localhost (2/2) -2016-05-23 22:28:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 22:28:10 INFO DAGScheduler:59 - ResultStage 0 (foreach at StratifiedSamplingLearning.scala:26) finished in 0.133 s -2016-05-23 22:28:10 INFO DAGScheduler:59 - Job 0 finished: foreach at StratifiedSamplingLearning.scala:26, took 0.293902 s -2016-05-23 22:28:10 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:35 -2016-05-23 22:28:10 INFO DAGScheduler:59 - Got job 1 (collect at StratifiedSamplingLearning.scala:35) with 4 output partitions -2016-05-23 22:28:10 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at StratifiedSamplingLearning.scala:35) -2016-05-23 22:28:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:28:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:28:10 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at sampleByKey at StratifiedSamplingLearning.scala:35), which has no missing parents -2016-05-23 22:28:10 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=176799, maxMem=766075207 -2016-05-23 22:28:10 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 22:28:10 INFO MemoryStore:59 - ensureFreeSpace(1460) called with curMem=179135, maxMem=766075207 -2016-05-23 22:28:10 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1460.0 B, free 730.4 MB) -2016-05-23 22:28:10 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53606 (size: 1460.0 B, free: 730.6 MB) -2016-05-23 22:28:10 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:28:10 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at sampleByKey at StratifiedSamplingLearning.scala:35) -2016-05-23 22:28:10 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 4 tasks -2016-05-23 22:28:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2188 bytes) -2016-05-23 22:28:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2212 bytes) -2016-05-23 22:28:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2210 bytes) -2016-05-23 22:28:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 1.0 (TID 5, localhost, PROCESS_LOCAL, 2216 bytes) -2016-05-23 22:28:10 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 22:28:10 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 22:28:10 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 4) -2016-05-23 22:28:10 INFO Executor:59 - Running task 3.0 in stage 1.0 (TID 5) -2016-05-23 22:28:10 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 4). 1060 bytes result sent to driver -2016-05-23 22:28:10 INFO Executor:59 - Finished task 3.0 in stage 1.0 (TID 5). 1087 bytes result sent to driver -2016-05-23 22:28:10 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1061 bytes result sent to driver -2016-05-23 22:28:10 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1059 bytes result sent to driver -2016-05-23 22:28:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 4) in 17 ms on localhost (1/4) -2016-05-23 22:28:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 1.0 (TID 5) in 17 ms on localhost (2/4) -2016-05-23 22:28:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 20 ms on localhost (3/4) -2016-05-23 22:28:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 29 ms on localhost (4/4) -2016-05-23 22:28:10 INFO DAGScheduler:59 - ResultStage 1 (collect at StratifiedSamplingLearning.scala:35) finished in 0.029 s -2016-05-23 22:28:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 22:28:10 INFO DAGScheduler:59 - Job 1 finished: collect at StratifiedSamplingLearning.scala:35, took 0.049831 s -2016-05-23 22:28:11 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:42 -2016-05-23 22:28:11 INFO DAGScheduler:59 - Got job 2 (collect at StratifiedSamplingLearning.scala:42) with 3 output partitions -2016-05-23 22:28:11 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at StratifiedSamplingLearning.scala:42) -2016-05-23 22:28:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:28:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:28:11 INFO DAGScheduler:59 - Submitting ResultStage 2 (ParallelCollectionRDD[5] at parallelize at StratifiedSamplingLearning.scala:39), which has no missing parents -2016-05-23 22:28:11 INFO MemoryStore:59 - ensureFreeSpace(1272) called with curMem=180595, maxMem=766075207 -2016-05-23 22:28:11 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 1272.0 B, free 730.4 MB) -2016-05-23 22:28:11 INFO MemoryStore:59 - ensureFreeSpace(842) called with curMem=181867, maxMem=766075207 -2016-05-23 22:28:11 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 842.0 B, free 730.4 MB) -2016-05-23 22:28:11 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53606 (size: 842.0 B, free: 730.6 MB) -2016-05-23 22:28:11 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:28:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 2 (ParallelCollectionRDD[5] at parallelize at StratifiedSamplingLearning.scala:39) -2016-05-23 22:28:11 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-05-23 22:28:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2085 bytes) -2016-05-23 22:28:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2085 bytes) -2016-05-23 22:28:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 8, localhost, PROCESS_LOCAL, 2142 bytes) -2016-05-23 22:28:11 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 6) -2016-05-23 22:28:11 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 8) -2016-05-23 22:28:11 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 7) -2016-05-23 22:28:11 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 7). 926 bytes result sent to driver -2016-05-23 22:28:11 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 6). 922 bytes result sent to driver -2016-05-23 22:28:11 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 8). 926 bytes result sent to driver -2016-05-23 22:28:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 7) in 17 ms on localhost (1/3) -2016-05-23 22:28:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 6) in 24 ms on localhost (2/3) -2016-05-23 22:28:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 8) in 18 ms on localhost (3/3) -2016-05-23 22:28:11 INFO DAGScheduler:59 - ResultStage 2 (collect at StratifiedSamplingLearning.scala:42) finished in 0.026 s -2016-05-23 22:28:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 22:28:11 INFO DAGScheduler:59 - Job 2 finished: collect at StratifiedSamplingLearning.scala:42, took 0.042755 s -2016-05-23 22:28:11 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:43 -2016-05-23 22:28:11 INFO DAGScheduler:59 - Got job 3 (collect at StratifiedSamplingLearning.scala:43) with 3 output partitions -2016-05-23 22:28:11 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at StratifiedSamplingLearning.scala:43) -2016-05-23 22:28:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:28:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:28:11 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[6] at sample at StratifiedSamplingLearning.scala:40), which has no missing parents -2016-05-23 22:28:11 INFO MemoryStore:59 - ensureFreeSpace(18856) called with curMem=182709, maxMem=766075207 -2016-05-23 22:28:11 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 18.4 KB, free 730.4 MB) -2016-05-23 22:28:11 INFO MemoryStore:59 - ensureFreeSpace(7723) called with curMem=201565, maxMem=766075207 -2016-05-23 22:28:11 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.4 MB) -2016-05-23 22:28:11 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53606 (size: 7.5 KB, free: 730.6 MB) -2016-05-23 22:28:11 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:28:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[6] at sample at StratifiedSamplingLearning.scala:40) -2016-05-23 22:28:11 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 3 tasks -2016-05-23 22:28:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:28:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:28:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 11, localhost, PROCESS_LOCAL, 2251 bytes) -2016-05-23 22:28:11 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 9) -2016-05-23 22:28:11 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 11) -2016-05-23 22:28:11 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 10) -2016-05-23 22:28:11 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 9). 914 bytes result sent to driver -2016-05-23 22:28:11 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 10). 902 bytes result sent to driver -2016-05-23 22:28:11 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 11). 910 bytes result sent to driver -2016-05-23 22:28:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 9) in 41 ms on localhost (1/3) -2016-05-23 22:28:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 10) in 39 ms on localhost (2/3) -2016-05-23 22:28:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 11) in 38 ms on localhost (3/3) -2016-05-23 22:28:11 INFO DAGScheduler:59 - ResultStage 3 (collect at StratifiedSamplingLearning.scala:43) finished in 0.042 s -2016-05-23 22:28:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 22:28:11 INFO DAGScheduler:59 - Job 3 finished: collect at StratifiedSamplingLearning.scala:43, took 0.091853 s -2016-05-23 22:28:11 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53606 in memory (size: 842.0 B, free: 730.6 MB) -2016-05-23 22:28:11 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:44 -2016-05-23 22:28:11 INFO DAGScheduler:59 - Got job 4 (collect at StratifiedSamplingLearning.scala:44) with 3 output partitions -2016-05-23 22:28:11 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at StratifiedSamplingLearning.scala:44) -2016-05-23 22:28:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:28:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:28:11 INFO DAGScheduler:59 - Submitting ResultStage 4 (PartitionwiseSampledRDD[7] at sample at StratifiedSamplingLearning.scala:41), which has no missing parents -2016-05-23 22:28:11 INFO MemoryStore:59 - ensureFreeSpace(2192) called with curMem=207174, maxMem=766075207 -2016-05-23 22:28:11 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-23 22:28:11 INFO MemoryStore:59 - ensureFreeSpace(1355) called with curMem=205570, maxMem=766075207 -2016-05-23 22:28:11 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53606 in memory (size: 1460.0 B, free: 730.6 MB) -2016-05-23 22:28:11 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 1355.0 B, free 730.4 MB) -2016-05-23 22:28:11 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53606 (size: 1355.0 B, free: 730.6 MB) -2016-05-23 22:28:11 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53606 in memory (size: 1915.0 B, free: 730.6 MB) -2016-05-23 22:28:11 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:28:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 4 (PartitionwiseSampledRDD[7] at sample at StratifiedSamplingLearning.scala:41) -2016-05-23 22:28:11 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 3 tasks -2016-05-23 22:28:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 12, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:28:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:28:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 14, localhost, PROCESS_LOCAL, 2251 bytes) -2016-05-23 22:28:11 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 12) -2016-05-23 22:28:11 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 14) -2016-05-23 22:28:11 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 13) -2016-05-23 22:28:11 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 14). 926 bytes result sent to driver -2016-05-23 22:28:11 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 13). 914 bytes result sent to driver -2016-05-23 22:28:11 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 12). 914 bytes result sent to driver -2016-05-23 22:28:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 14) in 9 ms on localhost (1/3) -2016-05-23 22:28:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 13) in 10 ms on localhost (2/3) -2016-05-23 22:28:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 12) in 12 ms on localhost (3/3) -2016-05-23 22:28:11 INFO DAGScheduler:59 - ResultStage 4 (collect at StratifiedSamplingLearning.scala:44) finished in 0.012 s -2016-05-23 22:28:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 22:28:11 INFO DAGScheduler:59 - Job 4 finished: collect at StratifiedSamplingLearning.scala:44, took 0.029143 s -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 22:28:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 22:28:11 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 22:28:11 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 22:28:11 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 22:28:11 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 22:28:11 INFO BlockManager:59 - BlockManager stopped -2016-05-23 22:28:11 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 22:28:11 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 22:28:11 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 22:28:11 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 22:28:11 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-599e0a88-2638-410a-8082-973434390f85 -2016-05-23 22:29:04 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 22:29:05 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 22:29:05 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 22:29:05 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 22:29:05 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 22:29:06 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 22:29:06 INFO Remoting:74 - Starting remoting -2016-05-23 22:29:07 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53628] -2016-05-23 22:29:07 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53628. -2016-05-23 22:29:07 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 22:29:07 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 22:29:07 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-49c40195-c75d-4f8e-8efc-a5e077f5a727 -2016-05-23 22:29:07 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 22:29:07 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-bde4c5f2-c3e4-4f37-a54f-eb2b25bf9c11\httpd-90af0650-eada-4dba-9b5d-14188d8d988d -2016-05-23 22:29:07 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 22:29:07 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:29:07 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53629 -2016-05-23 22:29:07 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53629. -2016-05-23 22:29:07 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 22:29:07 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:29:07 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 22:29:07 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 22:29:07 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 22:29:07 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 22:29:07 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 22:29:08 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53648. -2016-05-23 22:29:08 INFO NettyBlockTransferService:59 - Server created on 53648 -2016-05-23 22:29:08 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 22:29:08 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53648 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53648) -2016-05-23 22:29:08 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 22:29:09 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 22:29:09 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 22:29:09 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 22:29:09 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 22:29:09 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53648 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 22:29:09 INFO SparkContext:59 - Created broadcast 0 from textFile at StratifiedSamplingLearning.scala:19 -2016-05-23 22:29:10 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 22:29:10 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 22:29:10 INFO SparkContext:59 - Starting job: foreach at StratifiedSamplingLearning.scala:26 -2016-05-23 22:29:10 INFO DAGScheduler:59 - Got job 0 (foreach at StratifiedSamplingLearning.scala:26) with 2 output partitions -2016-05-23 22:29:10 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StratifiedSamplingLearning.scala:26) -2016-05-23 22:29:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:29:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:29:10 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at StratifiedSamplingLearning.scala:25), which has no missing parents -2016-05-23 22:29:10 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=171580, maxMem=766075207 -2016-05-23 22:29:10 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 22:29:10 INFO MemoryStore:59 - ensureFreeSpace(1939) called with curMem=175036, maxMem=766075207 -2016-05-23 22:29:10 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1939.0 B, free 730.4 MB) -2016-05-23 22:29:10 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53648 (size: 1939.0 B, free: 730.6 MB) -2016-05-23 22:29:10 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:29:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at StratifiedSamplingLearning.scala:25) -2016-05-23 22:29:10 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 22:29:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:29:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:29:10 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 22:29:10 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 22:29:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:21+22 -2016-05-23 22:29:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:0+21 -2016-05-23 22:29:10 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 22:29:10 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 22:29:10 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 22:29:10 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 22:29:10 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 22:29:11 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 22:29:11 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 22:29:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 111 ms on localhost (1/2) -2016-05-23 22:29:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 88 ms on localhost (2/2) -2016-05-23 22:29:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 22:29:11 INFO DAGScheduler:59 - ResultStage 0 (foreach at StratifiedSamplingLearning.scala:26) finished in 0.132 s -2016-05-23 22:29:11 INFO DAGScheduler:59 - Job 0 finished: foreach at StratifiedSamplingLearning.scala:26, took 0.233760 s -2016-05-23 22:29:11 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:35 -2016-05-23 22:29:11 INFO DAGScheduler:59 - Got job 1 (collect at StratifiedSamplingLearning.scala:35) with 4 output partitions -2016-05-23 22:29:11 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at StratifiedSamplingLearning.scala:35) -2016-05-23 22:29:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:29:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:29:11 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at sampleByKey at StratifiedSamplingLearning.scala:35), which has no missing parents -2016-05-23 22:29:11 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=176975, maxMem=766075207 -2016-05-23 22:29:11 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 22:29:11 INFO MemoryStore:59 - ensureFreeSpace(1461) called with curMem=179311, maxMem=766075207 -2016-05-23 22:29:11 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1461.0 B, free 730.4 MB) -2016-05-23 22:29:11 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53648 (size: 1461.0 B, free: 730.6 MB) -2016-05-23 22:29:11 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:29:11 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at sampleByKey at StratifiedSamplingLearning.scala:35) -2016-05-23 22:29:11 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 4 tasks -2016-05-23 22:29:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2188 bytes) -2016-05-23 22:29:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2212 bytes) -2016-05-23 22:29:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2210 bytes) -2016-05-23 22:29:11 INFO TaskSetManager:59 - Starting task 3.0 in stage 1.0 (TID 5, localhost, PROCESS_LOCAL, 2216 bytes) -2016-05-23 22:29:11 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 22:29:11 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 22:29:11 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 4) -2016-05-23 22:29:11 INFO Executor:59 - Running task 3.0 in stage 1.0 (TID 5) -2016-05-23 22:29:11 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1061 bytes result sent to driver -2016-05-23 22:29:11 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 4). 1060 bytes result sent to driver -2016-05-23 22:29:11 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1059 bytes result sent to driver -2016-05-23 22:29:11 INFO Executor:59 - Finished task 3.0 in stage 1.0 (TID 5). 1087 bytes result sent to driver -2016-05-23 22:29:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 26 ms on localhost (1/4) -2016-05-23 22:29:11 INFO TaskSetManager:59 - Finished task 3.0 in stage 1.0 (TID 5) in 17 ms on localhost (2/4) -2016-05-23 22:29:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 4) in 21 ms on localhost (3/4) -2016-05-23 22:29:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 24 ms on localhost (4/4) -2016-05-23 22:29:11 INFO DAGScheduler:59 - ResultStage 1 (collect at StratifiedSamplingLearning.scala:35) finished in 0.032 s -2016-05-23 22:29:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 22:29:11 INFO DAGScheduler:59 - Job 1 finished: collect at StratifiedSamplingLearning.scala:35, took 0.048303 s -2016-05-23 22:29:11 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:42 -2016-05-23 22:29:11 INFO DAGScheduler:59 - Got job 2 (collect at StratifiedSamplingLearning.scala:42) with 3 output partitions -2016-05-23 22:29:11 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at StratifiedSamplingLearning.scala:42) -2016-05-23 22:29:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:29:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:29:11 INFO DAGScheduler:59 - Submitting ResultStage 2 (ParallelCollectionRDD[6] at parallelize at StratifiedSamplingLearning.scala:39), which has no missing parents -2016-05-23 22:29:11 INFO MemoryStore:59 - ensureFreeSpace(1272) called with curMem=180772, maxMem=766075207 -2016-05-23 22:29:11 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 1272.0 B, free 730.4 MB) -2016-05-23 22:29:11 INFO MemoryStore:59 - ensureFreeSpace(843) called with curMem=182044, maxMem=766075207 -2016-05-23 22:29:11 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 843.0 B, free 730.4 MB) -2016-05-23 22:29:11 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53648 (size: 843.0 B, free: 730.6 MB) -2016-05-23 22:29:11 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:29:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 2 (ParallelCollectionRDD[6] at parallelize at StratifiedSamplingLearning.scala:39) -2016-05-23 22:29:11 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-05-23 22:29:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2085 bytes) -2016-05-23 22:29:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2085 bytes) -2016-05-23 22:29:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 8, localhost, PROCESS_LOCAL, 2142 bytes) -2016-05-23 22:29:11 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 7) -2016-05-23 22:29:11 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 8) -2016-05-23 22:29:11 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 6) -2016-05-23 22:29:11 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 6). 922 bytes result sent to driver -2016-05-23 22:29:11 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 8). 926 bytes result sent to driver -2016-05-23 22:29:11 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 7). 926 bytes result sent to driver -2016-05-23 22:29:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 6) in 18 ms on localhost (1/3) -2016-05-23 22:29:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 8) in 13 ms on localhost (2/3) -2016-05-23 22:29:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 7) in 15 ms on localhost (3/3) -2016-05-23 22:29:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 22:29:11 INFO DAGScheduler:59 - ResultStage 2 (collect at StratifiedSamplingLearning.scala:42) finished in 0.020 s -2016-05-23 22:29:11 INFO DAGScheduler:59 - Job 2 finished: collect at StratifiedSamplingLearning.scala:42, took 0.034783 s -2016-05-23 22:29:11 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:43 -2016-05-23 22:29:11 INFO DAGScheduler:59 - Got job 3 (collect at StratifiedSamplingLearning.scala:43) with 3 output partitions -2016-05-23 22:29:11 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at StratifiedSamplingLearning.scala:43) -2016-05-23 22:29:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:29:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:29:11 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[7] at sample at StratifiedSamplingLearning.scala:40), which has no missing parents -2016-05-23 22:29:11 INFO MemoryStore:59 - ensureFreeSpace(18856) called with curMem=182887, maxMem=766075207 -2016-05-23 22:29:11 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 18.4 KB, free 730.4 MB) -2016-05-23 22:29:11 INFO MemoryStore:59 - ensureFreeSpace(7709) called with curMem=201743, maxMem=766075207 -2016-05-23 22:29:11 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.4 MB) -2016-05-23 22:29:11 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53648 (size: 7.5 KB, free: 730.6 MB) -2016-05-23 22:29:11 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:29:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[7] at sample at StratifiedSamplingLearning.scala:40) -2016-05-23 22:29:11 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 3 tasks -2016-05-23 22:29:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:29:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:29:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 11, localhost, PROCESS_LOCAL, 2251 bytes) -2016-05-23 22:29:11 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 9) -2016-05-23 22:29:11 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 11) -2016-05-23 22:29:11 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 9). 914 bytes result sent to driver -2016-05-23 22:29:11 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 11). 910 bytes result sent to driver -2016-05-23 22:29:11 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 10) -2016-05-23 22:29:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 11) in 28 ms on localhost (1/3) -2016-05-23 22:29:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 9) in 60 ms on localhost (2/3) -2016-05-23 22:29:11 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 10). 902 bytes result sent to driver -2016-05-23 22:29:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 10) in 37 ms on localhost (3/3) -2016-05-23 22:29:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 22:29:11 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53648 in memory (size: 1461.0 B, free: 730.6 MB) -2016-05-23 22:29:11 INFO DAGScheduler:59 - ResultStage 3 (collect at StratifiedSamplingLearning.scala:43) finished in 0.068 s -2016-05-23 22:29:11 INFO DAGScheduler:59 - Job 3 finished: collect at StratifiedSamplingLearning.scala:43, took 0.088869 s -2016-05-23 22:29:11 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53648 in memory (size: 843.0 B, free: 730.6 MB) -2016-05-23 22:29:11 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53648 in memory (size: 1939.0 B, free: 730.6 MB) -2016-05-23 22:29:11 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:44 -2016-05-23 22:29:11 INFO DAGScheduler:59 - Got job 4 (collect at StratifiedSamplingLearning.scala:44) with 3 output partitions -2016-05-23 22:29:11 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at StratifiedSamplingLearning.scala:44) -2016-05-23 22:29:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:29:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:29:11 INFO DAGScheduler:59 - Submitting ResultStage 4 (PartitionwiseSampledRDD[8] at sample at StratifiedSamplingLearning.scala:41), which has no missing parents -2016-05-23 22:29:11 INFO MemoryStore:59 - ensureFreeSpace(2192) called with curMem=198145, maxMem=766075207 -2016-05-23 22:29:11 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-23 22:29:11 INFO MemoryStore:59 - ensureFreeSpace(1351) called with curMem=200337, maxMem=766075207 -2016-05-23 22:29:11 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 1351.0 B, free 730.4 MB) -2016-05-23 22:29:11 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53648 (size: 1351.0 B, free: 730.6 MB) -2016-05-23 22:29:11 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:29:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 4 (PartitionwiseSampledRDD[8] at sample at StratifiedSamplingLearning.scala:41) -2016-05-23 22:29:11 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 3 tasks -2016-05-23 22:29:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 12, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:29:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:29:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 14, localhost, PROCESS_LOCAL, 2251 bytes) -2016-05-23 22:29:11 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 12) -2016-05-23 22:29:11 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 14) -2016-05-23 22:29:11 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 13) -2016-05-23 22:29:11 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 13). 914 bytes result sent to driver -2016-05-23 22:29:11 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 12). 914 bytes result sent to driver -2016-05-23 22:29:11 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 14). 926 bytes result sent to driver -2016-05-23 22:29:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 13) in 11 ms on localhost (1/3) -2016-05-23 22:29:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 12) in 14 ms on localhost (2/3) -2016-05-23 22:29:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 14) in 12 ms on localhost (3/3) -2016-05-23 22:29:11 INFO DAGScheduler:59 - ResultStage 4 (collect at StratifiedSamplingLearning.scala:44) finished in 0.016 s -2016-05-23 22:29:11 INFO DAGScheduler:59 - Job 4 finished: collect at StratifiedSamplingLearning.scala:44, took 0.030671 s -2016-05-23 22:29:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 22:29:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 22:29:11 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 22:29:11 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 22:29:11 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 22:29:11 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 22:29:11 INFO BlockManager:59 - BlockManager stopped -2016-05-23 22:29:11 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 22:29:11 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 22:29:11 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 22:29:11 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 22:29:11 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-bde4c5f2-c3e4-4f37-a54f-eb2b25bf9c11 -2016-05-23 22:29:11 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 22:30:18 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 22:30:19 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 22:30:19 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 22:30:19 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 22:30:19 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 22:30:20 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 22:30:20 INFO Remoting:74 - Starting remoting -2016-05-23 22:30:20 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53669] -2016-05-23 22:30:20 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53669. -2016-05-23 22:30:20 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 22:30:20 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 22:30:21 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-404043d0-5d22-47a5-8e09-7155727741b4 -2016-05-23 22:30:21 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 22:30:21 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-1a66ca26-0eb1-4a8b-aa17-af3b51ac8b4a\httpd-fade5506-6f63-48f0-a06a-a95f26380096 -2016-05-23 22:30:21 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 22:30:21 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:30:21 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53670 -2016-05-23 22:30:21 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53670. -2016-05-23 22:30:21 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 22:30:21 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:30:21 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 22:30:21 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 22:30:21 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 22:30:21 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 22:30:21 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 22:30:21 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53689. -2016-05-23 22:30:21 INFO NettyBlockTransferService:59 - Server created on 53689 -2016-05-23 22:30:21 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 22:30:21 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53689 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53689) -2016-05-23 22:30:21 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 22:30:22 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 22:30:22 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 22:30:22 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 22:30:22 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 22:30:22 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53689 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 22:30:22 INFO SparkContext:59 - Created broadcast 0 from textFile at StratifiedSamplingLearning.scala:19 -2016-05-23 22:30:23 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 22:30:24 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 22:30:24 INFO SparkContext:59 - Starting job: foreach at StratifiedSamplingLearning.scala:26 -2016-05-23 22:30:24 INFO DAGScheduler:59 - Got job 0 (foreach at StratifiedSamplingLearning.scala:26) with 2 output partitions -2016-05-23 22:30:24 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StratifiedSamplingLearning.scala:26) -2016-05-23 22:30:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:30:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:30:24 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at StratifiedSamplingLearning.scala:25), which has no missing parents -2016-05-23 22:30:24 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=171580, maxMem=766075207 -2016-05-23 22:30:24 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 22:30:24 INFO MemoryStore:59 - ensureFreeSpace(1939) called with curMem=175036, maxMem=766075207 -2016-05-23 22:30:24 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1939.0 B, free 730.4 MB) -2016-05-23 22:30:24 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53689 (size: 1939.0 B, free: 730.6 MB) -2016-05-23 22:30:24 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:30:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at StratifiedSamplingLearning.scala:25) -2016-05-23 22:30:24 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 22:30:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:30:24 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 22:30:24 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 22:30:24 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:21+22 -2016-05-23 22:30:24 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:0+21 -2016-05-23 22:30:24 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 22:30:24 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 22:30:24 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 22:30:24 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 22:30:24 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 22:30:24 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 22:30:24 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 22:30:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 99 ms on localhost (1/2) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 138 ms on localhost (2/2) -2016-05-23 22:30:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 22:30:24 INFO DAGScheduler:59 - ResultStage 0 (foreach at StratifiedSamplingLearning.scala:26) finished in 0.159 s -2016-05-23 22:30:24 INFO DAGScheduler:59 - Job 0 finished: foreach at StratifiedSamplingLearning.scala:26, took 0.330175 s -2016-05-23 22:30:24 INFO SparkContext:59 - Starting job: foreach at StratifiedSamplingLearning.scala:29 -2016-05-23 22:30:24 INFO DAGScheduler:59 - Got job 1 (foreach at StratifiedSamplingLearning.scala:29) with 2 output partitions -2016-05-23 22:30:24 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StratifiedSamplingLearning.scala:29) -2016-05-23 22:30:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:30:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:30:24 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at sampleByKey at StratifiedSamplingLearning.scala:28), which has no missing parents -2016-05-23 22:30:24 INFO MemoryStore:59 - ensureFreeSpace(4128) called with curMem=176975, maxMem=766075207 -2016-05-23 22:30:24 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-05-23 22:30:24 INFO MemoryStore:59 - ensureFreeSpace(2294) called with curMem=181103, maxMem=766075207 -2016-05-23 22:30:24 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-05-23 22:30:24 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53689 (size: 2.2 KB, free: 730.6 MB) -2016-05-23 22:30:24 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:30:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at sampleByKey at StratifiedSamplingLearning.scala:28) -2016-05-23 22:30:24 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 22:30:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:30:24 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 22:30:24 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 22:30:24 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:0+21 -2016-05-23 22:30:24 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:21+22 -2016-05-23 22:30:24 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 22:30:24 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 22:30:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 27 ms on localhost (1/2) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 29 ms on localhost (2/2) -2016-05-23 22:30:24 INFO DAGScheduler:59 - ResultStage 1 (foreach at StratifiedSamplingLearning.scala:29) finished in 0.029 s -2016-05-23 22:30:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 22:30:24 INFO DAGScheduler:59 - Job 1 finished: foreach at StratifiedSamplingLearning.scala:29, took 0.051555 s -2016-05-23 22:30:24 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:35 -2016-05-23 22:30:24 INFO DAGScheduler:59 - Got job 2 (collect at StratifiedSamplingLearning.scala:35) with 4 output partitions -2016-05-23 22:30:24 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at StratifiedSamplingLearning.scala:35) -2016-05-23 22:30:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:30:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:30:24 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[6] at sampleByKey at StratifiedSamplingLearning.scala:35), which has no missing parents -2016-05-23 22:30:24 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=183397, maxMem=766075207 -2016-05-23 22:30:24 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 22:30:24 INFO MemoryStore:59 - ensureFreeSpace(1460) called with curMem=185733, maxMem=766075207 -2016-05-23 22:30:24 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1460.0 B, free 730.4 MB) -2016-05-23 22:30:24 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53689 (size: 1460.0 B, free: 730.6 MB) -2016-05-23 22:30:24 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:30:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 2 (MapPartitionsRDD[6] at sampleByKey at StratifiedSamplingLearning.scala:35) -2016-05-23 22:30:24 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 4 tasks -2016-05-23 22:30:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2188 bytes) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2212 bytes) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2210 bytes) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2216 bytes) -2016-05-23 22:30:24 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-23 22:30:24 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 22:30:24 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-05-23 22:30:24 INFO Executor:59 - Running task 3.0 in stage 2.0 (TID 7) -2016-05-23 22:30:24 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1061 bytes result sent to driver -2016-05-23 22:30:24 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 6). 1060 bytes result sent to driver -2016-05-23 22:30:24 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1059 bytes result sent to driver -2016-05-23 22:30:24 INFO Executor:59 - Finished task 3.0 in stage 2.0 (TID 7). 1087 bytes result sent to driver -2016-05-23 22:30:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 39 ms on localhost (1/4) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 6) in 29 ms on localhost (2/4) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 2.0 (TID 7) in 30 ms on localhost (3/4) -2016-05-23 22:30:24 INFO DAGScheduler:59 - ResultStage 2 (collect at StratifiedSamplingLearning.scala:35) finished in 0.044 s -2016-05-23 22:30:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 35 ms on localhost (4/4) -2016-05-23 22:30:24 INFO DAGScheduler:59 - Job 2 finished: collect at StratifiedSamplingLearning.scala:35, took 0.057715 s -2016-05-23 22:30:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 22:30:24 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:42 -2016-05-23 22:30:24 INFO DAGScheduler:59 - Got job 3 (collect at StratifiedSamplingLearning.scala:42) with 3 output partitions -2016-05-23 22:30:24 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at StratifiedSamplingLearning.scala:42) -2016-05-23 22:30:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:30:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:30:24 INFO DAGScheduler:59 - Submitting ResultStage 3 (ParallelCollectionRDD[7] at parallelize at StratifiedSamplingLearning.scala:39), which has no missing parents -2016-05-23 22:30:24 INFO MemoryStore:59 - ensureFreeSpace(1272) called with curMem=187193, maxMem=766075207 -2016-05-23 22:30:24 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 1272.0 B, free 730.4 MB) -2016-05-23 22:30:24 INFO MemoryStore:59 - ensureFreeSpace(843) called with curMem=188465, maxMem=766075207 -2016-05-23 22:30:24 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 843.0 B, free 730.4 MB) -2016-05-23 22:30:24 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53689 (size: 843.0 B, free: 730.6 MB) -2016-05-23 22:30:24 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:30:24 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 3 (ParallelCollectionRDD[7] at parallelize at StratifiedSamplingLearning.scala:39) -2016-05-23 22:30:24 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 3 tasks -2016-05-23 22:30:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 2085 bytes) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2085 bytes) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2142 bytes) -2016-05-23 22:30:24 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 9) -2016-05-23 22:30:24 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 10) -2016-05-23 22:30:24 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 8) -2016-05-23 22:30:24 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 8). 922 bytes result sent to driver -2016-05-23 22:30:24 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 9). 926 bytes result sent to driver -2016-05-23 22:30:24 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 10). 926 bytes result sent to driver -2016-05-23 22:30:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 8) in 14 ms on localhost (1/3) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 9) in 9 ms on localhost (2/3) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 10) in 9 ms on localhost (3/3) -2016-05-23 22:30:24 INFO DAGScheduler:59 - ResultStage 3 (collect at StratifiedSamplingLearning.scala:42) finished in 0.015 s -2016-05-23 22:30:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 22:30:24 INFO DAGScheduler:59 - Job 3 finished: collect at StratifiedSamplingLearning.scala:42, took 0.028578 s -2016-05-23 22:30:24 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:43 -2016-05-23 22:30:24 INFO DAGScheduler:59 - Got job 4 (collect at StratifiedSamplingLearning.scala:43) with 3 output partitions -2016-05-23 22:30:24 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at StratifiedSamplingLearning.scala:43) -2016-05-23 22:30:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:30:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:30:24 INFO DAGScheduler:59 - Submitting ResultStage 4 (PartitionwiseSampledRDD[8] at sample at StratifiedSamplingLearning.scala:40), which has no missing parents -2016-05-23 22:30:24 INFO MemoryStore:59 - ensureFreeSpace(18856) called with curMem=189308, maxMem=766075207 -2016-05-23 22:30:24 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 18.4 KB, free 730.4 MB) -2016-05-23 22:30:24 INFO MemoryStore:59 - ensureFreeSpace(7702) called with curMem=208164, maxMem=766075207 -2016-05-23 22:30:24 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.4 MB) -2016-05-23 22:30:24 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53689 (size: 7.5 KB, free: 730.6 MB) -2016-05-23 22:30:24 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:30:24 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 4 (PartitionwiseSampledRDD[8] at sample at StratifiedSamplingLearning.scala:40) -2016-05-23 22:30:24 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 3 tasks -2016-05-23 22:30:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 11, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 12, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 2251 bytes) -2016-05-23 22:30:24 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 12) -2016-05-23 22:30:24 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 13) -2016-05-23 22:30:24 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 11) -2016-05-23 22:30:24 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 11). 914 bytes result sent to driver -2016-05-23 22:30:24 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 12). 902 bytes result sent to driver -2016-05-23 22:30:24 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 13). 910 bytes result sent to driver -2016-05-23 22:30:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 11) in 17 ms on localhost (1/3) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 12) in 17 ms on localhost (2/3) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 13) in 16 ms on localhost (3/3) -2016-05-23 22:30:24 INFO DAGScheduler:59 - ResultStage 4 (collect at StratifiedSamplingLearning.scala:43) finished in 0.019 s -2016-05-23 22:30:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 22:30:24 INFO DAGScheduler:59 - Job 4 finished: collect at StratifiedSamplingLearning.scala:43, took 0.034169 s -2016-05-23 22:30:24 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:44 -2016-05-23 22:30:24 INFO DAGScheduler:59 - Got job 5 (collect at StratifiedSamplingLearning.scala:44) with 3 output partitions -2016-05-23 22:30:24 INFO DAGScheduler:59 - Final stage: ResultStage 5(collect at StratifiedSamplingLearning.scala:44) -2016-05-23 22:30:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:30:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:30:24 INFO DAGScheduler:59 - Submitting ResultStage 5 (PartitionwiseSampledRDD[9] at sample at StratifiedSamplingLearning.scala:41), which has no missing parents -2016-05-23 22:30:24 INFO MemoryStore:59 - ensureFreeSpace(2192) called with curMem=215866, maxMem=766075207 -2016-05-23 22:30:24 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-23 22:30:24 INFO MemoryStore:59 - ensureFreeSpace(1351) called with curMem=218058, maxMem=766075207 -2016-05-23 22:30:24 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1351.0 B, free 730.4 MB) -2016-05-23 22:30:24 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53689 (size: 1351.0 B, free: 730.6 MB) -2016-05-23 22:30:24 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:30:24 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 5 (PartitionwiseSampledRDD[9] at sample at StratifiedSamplingLearning.scala:41) -2016-05-23 22:30:24 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 3 tasks -2016-05-23 22:30:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 14, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 15, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 16, localhost, PROCESS_LOCAL, 2251 bytes) -2016-05-23 22:30:24 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 14) -2016-05-23 22:30:24 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 16) -2016-05-23 22:30:24 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 15) -2016-05-23 22:30:24 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 14). 914 bytes result sent to driver -2016-05-23 22:30:24 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 16). 926 bytes result sent to driver -2016-05-23 22:30:24 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 15). 914 bytes result sent to driver -2016-05-23 22:30:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 16) in 10 ms on localhost (1/3) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 14) in 13 ms on localhost (2/3) -2016-05-23 22:30:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 15) in 13 ms on localhost (3/3) -2016-05-23 22:30:24 INFO DAGScheduler:59 - ResultStage 5 (collect at StratifiedSamplingLearning.scala:44) finished in 0.015 s -2016-05-23 22:30:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 22:30:24 INFO DAGScheduler:59 - Job 5 finished: collect at StratifiedSamplingLearning.scala:44, took 0.028822 s -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 22:30:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 22:30:24 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 22:30:24 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 22:30:24 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 22:30:25 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 22:30:25 INFO BlockManager:59 - BlockManager stopped -2016-05-23 22:30:25 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 22:30:25 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 22:30:25 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 22:30:25 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 22:30:25 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-1a66ca26-0eb1-4a8b-aa17-af3b51ac8b4a -2016-05-23 22:30:25 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 22:33:12 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 22:33:12 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 22:33:13 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 22:33:13 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 22:33:13 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 22:33:13 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 22:33:13 INFO Remoting:74 - Starting remoting -2016-05-23 22:33:14 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53712] -2016-05-23 22:33:14 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53712. -2016-05-23 22:33:14 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 22:33:14 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 22:33:14 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-55f34be3-bb05-488d-8ccd-4ef3533bf3bd -2016-05-23 22:33:14 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 22:33:14 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-8754471e-7069-4a4a-90f5-60a66e178cc0\httpd-535af536-092f-46ba-a14f-b405eb3a3fd1 -2016-05-23 22:33:14 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 22:33:14 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:33:14 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53713 -2016-05-23 22:33:14 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53713. -2016-05-23 22:33:14 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 22:33:14 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:33:14 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 22:33:14 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 22:33:14 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 22:33:14 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 22:33:14 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 22:33:15 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53732. -2016-05-23 22:33:15 INFO NettyBlockTransferService:59 - Server created on 53732 -2016-05-23 22:33:15 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 22:33:15 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53732 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53732) -2016-05-23 22:33:15 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 22:33:15 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 22:33:15 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 22:33:15 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 22:33:15 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 22:33:15 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53732 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 22:33:15 INFO SparkContext:59 - Created broadcast 0 from textFile at StratifiedSamplingLearning.scala:19 -2016-05-23 22:33:17 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 22:33:17 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 22:33:17 INFO SparkContext:59 - Starting job: foreach at StratifiedSamplingLearning.scala:26 -2016-05-23 22:33:17 INFO DAGScheduler:59 - Got job 0 (foreach at StratifiedSamplingLearning.scala:26) with 2 output partitions -2016-05-23 22:33:17 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StratifiedSamplingLearning.scala:26) -2016-05-23 22:33:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:33:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:33:17 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at StratifiedSamplingLearning.scala:25), which has no missing parents -2016-05-23 22:33:17 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=171580, maxMem=766075207 -2016-05-23 22:33:17 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 22:33:17 INFO MemoryStore:59 - ensureFreeSpace(1939) called with curMem=175036, maxMem=766075207 -2016-05-23 22:33:17 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1939.0 B, free 730.4 MB) -2016-05-23 22:33:17 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53732 (size: 1939.0 B, free: 730.6 MB) -2016-05-23 22:33:17 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:33:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at StratifiedSamplingLearning.scala:25) -2016-05-23 22:33:17 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 22:33:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:33:17 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 22:33:17 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 22:33:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:21+22 -2016-05-23 22:33:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:0+21 -2016-05-23 22:33:17 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 22:33:17 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 22:33:17 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 22:33:17 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 22:33:17 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 22:33:17 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 22:33:17 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 22:33:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 84 ms on localhost (1/2) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 107 ms on localhost (2/2) -2016-05-23 22:33:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 22:33:17 INFO DAGScheduler:59 - ResultStage 0 (foreach at StratifiedSamplingLearning.scala:26) finished in 0.122 s -2016-05-23 22:33:17 INFO DAGScheduler:59 - Job 0 finished: foreach at StratifiedSamplingLearning.scala:26, took 0.251991 s -2016-05-23 22:33:17 INFO SparkContext:59 - Starting job: foreach at StratifiedSamplingLearning.scala:29 -2016-05-23 22:33:17 INFO DAGScheduler:59 - Got job 1 (foreach at StratifiedSamplingLearning.scala:29) with 2 output partitions -2016-05-23 22:33:17 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StratifiedSamplingLearning.scala:29) -2016-05-23 22:33:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:33:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:33:17 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at sampleByKey at StratifiedSamplingLearning.scala:28), which has no missing parents -2016-05-23 22:33:17 INFO MemoryStore:59 - ensureFreeSpace(4128) called with curMem=176975, maxMem=766075207 -2016-05-23 22:33:17 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-05-23 22:33:17 INFO MemoryStore:59 - ensureFreeSpace(2294) called with curMem=181103, maxMem=766075207 -2016-05-23 22:33:17 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-05-23 22:33:17 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53732 (size: 2.2 KB, free: 730.6 MB) -2016-05-23 22:33:17 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:33:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at sampleByKey at StratifiedSamplingLearning.scala:28) -2016-05-23 22:33:17 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 22:33:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:33:17 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 22:33:17 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 22:33:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:21+22 -2016-05-23 22:33:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:0+21 -2016-05-23 22:33:17 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 22:33:17 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 22:33:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 22 ms on localhost (1/2) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 25 ms on localhost (2/2) -2016-05-23 22:33:17 INFO DAGScheduler:59 - ResultStage 1 (foreach at StratifiedSamplingLearning.scala:29) finished in 0.026 s -2016-05-23 22:33:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 22:33:17 INFO DAGScheduler:59 - Job 1 finished: foreach at StratifiedSamplingLearning.scala:29, took 0.048697 s -2016-05-23 22:33:17 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:35 -2016-05-23 22:33:17 INFO DAGScheduler:59 - Got job 2 (collect at StratifiedSamplingLearning.scala:35) with 4 output partitions -2016-05-23 22:33:17 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at StratifiedSamplingLearning.scala:35) -2016-05-23 22:33:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:33:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:33:17 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[6] at sampleByKey at StratifiedSamplingLearning.scala:35), which has no missing parents -2016-05-23 22:33:17 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=183397, maxMem=766075207 -2016-05-23 22:33:17 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 22:33:17 INFO MemoryStore:59 - ensureFreeSpace(1462) called with curMem=185733, maxMem=766075207 -2016-05-23 22:33:17 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1462.0 B, free 730.4 MB) -2016-05-23 22:33:17 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53732 (size: 1462.0 B, free: 730.6 MB) -2016-05-23 22:33:17 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:33:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 2 (MapPartitionsRDD[6] at sampleByKey at StratifiedSamplingLearning.scala:35) -2016-05-23 22:33:17 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 4 tasks -2016-05-23 22:33:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2188 bytes) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2212 bytes) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2210 bytes) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2216 bytes) -2016-05-23 22:33:17 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 22:33:17 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-23 22:33:17 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-05-23 22:33:17 INFO Executor:59 - Running task 3.0 in stage 2.0 (TID 7) -2016-05-23 22:33:17 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 6). 1060 bytes result sent to driver -2016-05-23 22:33:17 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 911 bytes result sent to driver -2016-05-23 22:33:17 INFO Executor:59 - Finished task 3.0 in stage 2.0 (TID 7). 1062 bytes result sent to driver -2016-05-23 22:33:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 6) in 16 ms on localhost (1/4) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 26 ms on localhost (2/4) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 2.0 (TID 7) in 19 ms on localhost (3/4) -2016-05-23 22:33:17 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1061 bytes result sent to driver -2016-05-23 22:33:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 26 ms on localhost (4/4) -2016-05-23 22:33:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 22:33:17 INFO DAGScheduler:59 - ResultStage 2 (collect at StratifiedSamplingLearning.scala:35) finished in 0.033 s -2016-05-23 22:33:17 INFO DAGScheduler:59 - Job 2 finished: collect at StratifiedSamplingLearning.scala:35, took 0.047664 s -2016-05-23 22:33:17 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:43 -2016-05-23 22:33:17 INFO DAGScheduler:59 - Got job 3 (collect at StratifiedSamplingLearning.scala:43) with 3 output partitions -2016-05-23 22:33:17 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at StratifiedSamplingLearning.scala:43) -2016-05-23 22:33:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:33:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:33:17 INFO DAGScheduler:59 - Submitting ResultStage 3 (ParallelCollectionRDD[7] at parallelize at StratifiedSamplingLearning.scala:40), which has no missing parents -2016-05-23 22:33:17 INFO MemoryStore:59 - ensureFreeSpace(1272) called with curMem=187195, maxMem=766075207 -2016-05-23 22:33:17 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 1272.0 B, free 730.4 MB) -2016-05-23 22:33:17 INFO MemoryStore:59 - ensureFreeSpace(843) called with curMem=188467, maxMem=766075207 -2016-05-23 22:33:17 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 843.0 B, free 730.4 MB) -2016-05-23 22:33:17 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53732 (size: 843.0 B, free: 730.6 MB) -2016-05-23 22:33:17 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:33:17 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 3 (ParallelCollectionRDD[7] at parallelize at StratifiedSamplingLearning.scala:40) -2016-05-23 22:33:17 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 3 tasks -2016-05-23 22:33:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 2085 bytes) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2085 bytes) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2142 bytes) -2016-05-23 22:33:17 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 8) -2016-05-23 22:33:17 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 10) -2016-05-23 22:33:17 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 9) -2016-05-23 22:33:17 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 10). 926 bytes result sent to driver -2016-05-23 22:33:17 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 9). 926 bytes result sent to driver -2016-05-23 22:33:17 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 8). 922 bytes result sent to driver -2016-05-23 22:33:17 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53732 in memory (size: 1462.0 B, free: 730.6 MB) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 10) in 10 ms on localhost (1/3) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 9) in 12 ms on localhost (2/3) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 8) in 21 ms on localhost (3/3) -2016-05-23 22:33:17 INFO DAGScheduler:59 - ResultStage 3 (collect at StratifiedSamplingLearning.scala:43) finished in 0.028 s -2016-05-23 22:33:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 22:33:17 INFO DAGScheduler:59 - Job 3 finished: collect at StratifiedSamplingLearning.scala:43, took 0.067936 s -2016-05-23 22:33:17 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53732 in memory (size: 2.2 KB, free: 730.6 MB) -2016-05-23 22:33:17 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53732 in memory (size: 1939.0 B, free: 730.6 MB) -2016-05-23 22:33:17 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:44 -2016-05-23 22:33:17 INFO DAGScheduler:59 - Got job 4 (collect at StratifiedSamplingLearning.scala:44) with 3 output partitions -2016-05-23 22:33:17 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at StratifiedSamplingLearning.scala:44) -2016-05-23 22:33:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:33:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:33:17 INFO DAGScheduler:59 - Submitting ResultStage 4 (PartitionwiseSampledRDD[8] at sample at StratifiedSamplingLearning.scala:41), which has no missing parents -2016-05-23 22:33:17 INFO MemoryStore:59 - ensureFreeSpace(18856) called with curMem=173695, maxMem=766075207 -2016-05-23 22:33:17 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 18.4 KB, free 730.4 MB) -2016-05-23 22:33:17 INFO MemoryStore:59 - ensureFreeSpace(7699) called with curMem=192551, maxMem=766075207 -2016-05-23 22:33:17 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.4 MB) -2016-05-23 22:33:17 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53732 (size: 7.5 KB, free: 730.6 MB) -2016-05-23 22:33:17 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:33:17 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 4 (PartitionwiseSampledRDD[8] at sample at StratifiedSamplingLearning.scala:41) -2016-05-23 22:33:17 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 3 tasks -2016-05-23 22:33:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 11, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 12, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 2251 bytes) -2016-05-23 22:33:17 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 12) -2016-05-23 22:33:17 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 13) -2016-05-23 22:33:17 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 11) -2016-05-23 22:33:17 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 13). 910 bytes result sent to driver -2016-05-23 22:33:17 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 11). 914 bytes result sent to driver -2016-05-23 22:33:17 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 12). 902 bytes result sent to driver -2016-05-23 22:33:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 13) in 19 ms on localhost (1/3) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 12) in 21 ms on localhost (2/3) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 11) in 23 ms on localhost (3/3) -2016-05-23 22:33:17 INFO DAGScheduler:59 - ResultStage 4 (collect at StratifiedSamplingLearning.scala:44) finished in 0.023 s -2016-05-23 22:33:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 22:33:17 INFO DAGScheduler:59 - Job 4 finished: collect at StratifiedSamplingLearning.scala:44, took 0.039518 s -2016-05-23 22:33:17 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:45 -2016-05-23 22:33:17 INFO DAGScheduler:59 - Got job 5 (collect at StratifiedSamplingLearning.scala:45) with 3 output partitions -2016-05-23 22:33:17 INFO DAGScheduler:59 - Final stage: ResultStage 5(collect at StratifiedSamplingLearning.scala:45) -2016-05-23 22:33:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:33:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:33:17 INFO DAGScheduler:59 - Submitting ResultStage 5 (PartitionwiseSampledRDD[9] at sample at StratifiedSamplingLearning.scala:42), which has no missing parents -2016-05-23 22:33:17 INFO MemoryStore:59 - ensureFreeSpace(2192) called with curMem=200250, maxMem=766075207 -2016-05-23 22:33:17 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-23 22:33:17 INFO MemoryStore:59 - ensureFreeSpace(1351) called with curMem=202442, maxMem=766075207 -2016-05-23 22:33:17 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1351.0 B, free 730.4 MB) -2016-05-23 22:33:17 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53732 (size: 1351.0 B, free: 730.6 MB) -2016-05-23 22:33:17 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:33:17 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 5 (PartitionwiseSampledRDD[9] at sample at StratifiedSamplingLearning.scala:42) -2016-05-23 22:33:17 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 3 tasks -2016-05-23 22:33:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 14, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 15, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 16, localhost, PROCESS_LOCAL, 2251 bytes) -2016-05-23 22:33:17 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 15) -2016-05-23 22:33:17 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 14) -2016-05-23 22:33:17 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 16) -2016-05-23 22:33:17 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 15). 914 bytes result sent to driver -2016-05-23 22:33:17 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 14). 914 bytes result sent to driver -2016-05-23 22:33:17 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 16). 926 bytes result sent to driver -2016-05-23 22:33:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 15) in 10 ms on localhost (1/3) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 14) in 11 ms on localhost (2/3) -2016-05-23 22:33:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 16) in 11 ms on localhost (3/3) -2016-05-23 22:33:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 22:33:17 INFO DAGScheduler:59 - ResultStage 5 (collect at StratifiedSamplingLearning.scala:45) finished in 0.015 s -2016-05-23 22:33:17 INFO DAGScheduler:59 - Job 5 finished: collect at StratifiedSamplingLearning.scala:45, took 0.027709 s -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 22:33:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 22:33:17 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 22:33:17 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 22:33:18 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 22:33:18 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 22:33:18 INFO BlockManager:59 - BlockManager stopped -2016-05-23 22:33:18 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 22:33:18 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 22:33:18 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 22:33:18 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 22:33:18 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-8754471e-7069-4a4a-90f5-60a66e178cc0 -2016-05-23 22:33:18 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 22:33:18 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 22:33:42 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 22:33:43 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 22:33:43 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 22:33:43 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 22:33:43 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 22:33:44 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 22:33:44 INFO Remoting:74 - Starting remoting -2016-05-23 22:33:44 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53752] -2016-05-23 22:33:44 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53752. -2016-05-23 22:33:44 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 22:33:45 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 22:33:45 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-f4fc034d-0061-4299-9047-5f1921f9dae7 -2016-05-23 22:33:45 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 22:33:45 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-a786e34b-4e1b-4a36-beed-7ac5ec250252\httpd-b1a14eed-61c5-4c52-a5d5-ff1bf04bc888 -2016-05-23 22:33:45 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 22:33:45 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:33:45 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53753 -2016-05-23 22:33:45 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53753. -2016-05-23 22:33:45 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 22:33:45 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:33:45 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 22:33:45 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 22:33:45 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 22:33:45 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 22:33:45 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 22:33:45 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53772. -2016-05-23 22:33:45 INFO NettyBlockTransferService:59 - Server created on 53772 -2016-05-23 22:33:45 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 22:33:45 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53772 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53772) -2016-05-23 22:33:45 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 22:33:46 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 22:33:46 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 22:33:46 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 22:33:46 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 22:33:46 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53772 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 22:33:46 INFO SparkContext:59 - Created broadcast 0 from textFile at StratifiedSamplingLearning.scala:19 -2016-05-23 22:33:47 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 22:33:48 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 22:33:48 INFO SparkContext:59 - Starting job: foreach at StratifiedSamplingLearning.scala:26 -2016-05-23 22:33:48 INFO DAGScheduler:59 - Got job 0 (foreach at StratifiedSamplingLearning.scala:26) with 2 output partitions -2016-05-23 22:33:48 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StratifiedSamplingLearning.scala:26) -2016-05-23 22:33:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:33:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:33:48 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at StratifiedSamplingLearning.scala:25), which has no missing parents -2016-05-23 22:33:48 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=171580, maxMem=766075207 -2016-05-23 22:33:48 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 22:33:48 INFO MemoryStore:59 - ensureFreeSpace(1939) called with curMem=175036, maxMem=766075207 -2016-05-23 22:33:48 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1939.0 B, free 730.4 MB) -2016-05-23 22:33:48 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53772 (size: 1939.0 B, free: 730.6 MB) -2016-05-23 22:33:48 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:33:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at StratifiedSamplingLearning.scala:25) -2016-05-23 22:33:48 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 22:33:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:33:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:33:48 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 22:33:48 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 22:33:48 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:21+22 -2016-05-23 22:33:48 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:0+21 -2016-05-23 22:33:48 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 22:33:48 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 22:33:48 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 22:33:48 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 22:33:48 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 22:33:48 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 22:33:48 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 22:33:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 88 ms on localhost (1/2) -2016-05-23 22:33:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 114 ms on localhost (2/2) -2016-05-23 22:33:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 22:33:48 INFO DAGScheduler:59 - ResultStage 0 (foreach at StratifiedSamplingLearning.scala:26) finished in 0.132 s -2016-05-23 22:33:48 INFO DAGScheduler:59 - Job 0 finished: foreach at StratifiedSamplingLearning.scala:26, took 0.223992 s -2016-05-23 22:33:48 INFO SparkContext:59 - Starting job: foreach at StratifiedSamplingLearning.scala:29 -2016-05-23 22:33:48 INFO DAGScheduler:59 - Got job 1 (foreach at StratifiedSamplingLearning.scala:29) with 2 output partitions -2016-05-23 22:33:48 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StratifiedSamplingLearning.scala:29) -2016-05-23 22:33:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:33:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:33:48 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at sampleByKey at StratifiedSamplingLearning.scala:28), which has no missing parents -2016-05-23 22:33:48 INFO MemoryStore:59 - ensureFreeSpace(4128) called with curMem=176975, maxMem=766075207 -2016-05-23 22:33:48 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-05-23 22:33:48 INFO MemoryStore:59 - ensureFreeSpace(2294) called with curMem=181103, maxMem=766075207 -2016-05-23 22:33:48 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-05-23 22:33:48 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53772 (size: 2.2 KB, free: 730.6 MB) -2016-05-23 22:33:48 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:33:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at sampleByKey at StratifiedSamplingLearning.scala:28) -2016-05-23 22:33:48 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 22:33:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:33:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:33:48 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 22:33:48 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:21+22 -2016-05-23 22:33:48 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 22:33:48 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 22:33:48 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:0+21 -2016-05-23 22:33:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 45 ms on localhost (1/2) -2016-05-23 22:33:48 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 22:33:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 56 ms on localhost (2/2) -2016-05-23 22:33:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 22:33:48 INFO DAGScheduler:59 - ResultStage 1 (foreach at StratifiedSamplingLearning.scala:29) finished in 0.056 s -2016-05-23 22:33:48 INFO DAGScheduler:59 - Job 1 finished: foreach at StratifiedSamplingLearning.scala:29, took 0.096499 s -2016-05-23 22:33:48 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:35 -2016-05-23 22:33:48 INFO DAGScheduler:59 - Got job 2 (collect at StratifiedSamplingLearning.scala:35) with 4 output partitions -2016-05-23 22:33:48 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at StratifiedSamplingLearning.scala:35) -2016-05-23 22:33:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:33:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:33:48 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[6] at sampleByKey at StratifiedSamplingLearning.scala:35), which has no missing parents -2016-05-23 22:33:48 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=183397, maxMem=766075207 -2016-05-23 22:33:48 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 22:33:48 INFO MemoryStore:59 - ensureFreeSpace(1462) called with curMem=185733, maxMem=766075207 -2016-05-23 22:33:48 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1462.0 B, free 730.4 MB) -2016-05-23 22:33:48 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53772 (size: 1462.0 B, free: 730.6 MB) -2016-05-23 22:33:48 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:33:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 2 (MapPartitionsRDD[6] at sampleByKey at StratifiedSamplingLearning.scala:35) -2016-05-23 22:33:48 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 4 tasks -2016-05-23 22:33:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2188 bytes) -2016-05-23 22:33:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2212 bytes) -2016-05-23 22:33:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2210 bytes) -2016-05-23 22:33:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2216 bytes) -2016-05-23 22:33:48 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-23 22:33:48 INFO Executor:59 - Running task 3.0 in stage 2.0 (TID 7) -2016-05-23 22:33:48 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-05-23 22:33:48 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 22:33:48 INFO Executor:59 - Finished task 3.0 in stage 2.0 (TID 7). 1062 bytes result sent to driver -2016-05-23 22:33:48 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 6). 1060 bytes result sent to driver -2016-05-23 22:33:48 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 911 bytes result sent to driver -2016-05-23 22:33:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 6) in 20 ms on localhost (1/4) -2016-05-23 22:33:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 2.0 (TID 7) in 20 ms on localhost (2/4) -2016-05-23 22:33:48 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1061 bytes result sent to driver -2016-05-23 22:33:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 35 ms on localhost (3/4) -2016-05-23 22:33:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 27 ms on localhost (4/4) -2016-05-23 22:33:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 22:33:48 INFO DAGScheduler:59 - ResultStage 2 (collect at StratifiedSamplingLearning.scala:35) finished in 0.039 s -2016-05-23 22:33:48 INFO DAGScheduler:59 - Job 2 finished: collect at StratifiedSamplingLearning.scala:35, took 0.059579 s -2016-05-23 22:33:48 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:43 -2016-05-23 22:33:48 INFO DAGScheduler:59 - Got job 3 (collect at StratifiedSamplingLearning.scala:43) with 3 output partitions -2016-05-23 22:33:48 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at StratifiedSamplingLearning.scala:43) -2016-05-23 22:33:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:33:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:33:48 INFO DAGScheduler:59 - Submitting ResultStage 3 (ParallelCollectionRDD[7] at parallelize at StratifiedSamplingLearning.scala:40), which has no missing parents -2016-05-23 22:33:48 INFO MemoryStore:59 - ensureFreeSpace(1272) called with curMem=187195, maxMem=766075207 -2016-05-23 22:33:48 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 1272.0 B, free 730.4 MB) -2016-05-23 22:33:48 INFO MemoryStore:59 - ensureFreeSpace(843) called with curMem=188467, maxMem=766075207 -2016-05-23 22:33:48 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 843.0 B, free 730.4 MB) -2016-05-23 22:33:48 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53772 (size: 843.0 B, free: 730.6 MB) -2016-05-23 22:33:48 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:33:48 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 3 (ParallelCollectionRDD[7] at parallelize at StratifiedSamplingLearning.scala:40) -2016-05-23 22:33:48 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 3 tasks -2016-05-23 22:33:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 2085 bytes) -2016-05-23 22:33:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2085 bytes) -2016-05-23 22:33:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2142 bytes) -2016-05-23 22:33:48 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 9) -2016-05-23 22:33:48 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 10) -2016-05-23 22:33:48 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 8) -2016-05-23 22:33:48 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 10). 926 bytes result sent to driver -2016-05-23 22:33:48 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 8). 922 bytes result sent to driver -2016-05-23 22:33:48 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 9). 926 bytes result sent to driver -2016-05-23 22:33:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 10) in 10 ms on localhost (1/3) -2016-05-23 22:33:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 9) in 12 ms on localhost (2/3) -2016-05-23 22:33:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 8) in 20 ms on localhost (3/3) -2016-05-23 22:33:48 INFO DAGScheduler:59 - ResultStage 3 (collect at StratifiedSamplingLearning.scala:43) finished in 0.020 s -2016-05-23 22:33:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 22:33:48 INFO DAGScheduler:59 - Job 3 finished: collect at StratifiedSamplingLearning.scala:43, took 0.039053 s -2016-05-23 22:33:48 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:44 -2016-05-23 22:33:48 INFO DAGScheduler:59 - Got job 4 (collect at StratifiedSamplingLearning.scala:44) with 3 output partitions -2016-05-23 22:33:48 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at StratifiedSamplingLearning.scala:44) -2016-05-23 22:33:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:33:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:33:48 INFO DAGScheduler:59 - Submitting ResultStage 4 (PartitionwiseSampledRDD[8] at sample at StratifiedSamplingLearning.scala:41), which has no missing parents -2016-05-23 22:33:48 INFO MemoryStore:59 - ensureFreeSpace(18856) called with curMem=189310, maxMem=766075207 -2016-05-23 22:33:48 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 18.4 KB, free 730.4 MB) -2016-05-23 22:33:48 INFO MemoryStore:59 - ensureFreeSpace(7700) called with curMem=208166, maxMem=766075207 -2016-05-23 22:33:48 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.4 MB) -2016-05-23 22:33:48 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53772 (size: 7.5 KB, free: 730.6 MB) -2016-05-23 22:33:48 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:33:48 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 4 (PartitionwiseSampledRDD[8] at sample at StratifiedSamplingLearning.scala:41) -2016-05-23 22:33:48 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 3 tasks -2016-05-23 22:33:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 11, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:33:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 12, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:33:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 2251 bytes) -2016-05-23 22:33:48 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 11) -2016-05-23 22:33:48 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 13) -2016-05-23 22:33:48 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 12) -2016-05-23 22:33:48 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-23 22:33:48 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 13). 910 bytes result sent to driver -2016-05-23 22:33:48 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 11). 914 bytes result sent to driver -2016-05-23 22:33:48 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 12). 902 bytes result sent to driver -2016-05-23 22:33:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 13) in 17 ms on localhost (1/3) -2016-05-23 22:33:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 11) in 21 ms on localhost (2/3) -2016-05-23 22:33:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 12) in 22 ms on localhost (3/3) -2016-05-23 22:33:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 22:33:48 INFO DAGScheduler:59 - ResultStage 4 (collect at StratifiedSamplingLearning.scala:44) finished in 0.025 s -2016-05-23 22:33:48 INFO DAGScheduler:59 - Job 4 finished: collect at StratifiedSamplingLearning.scala:44, took 0.055299 s -2016-05-23 22:33:48 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53772 in memory (size: 843.0 B, free: 730.6 MB) -2016-05-23 22:33:48 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-23 22:33:48 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:45 -2016-05-23 22:33:48 INFO DAGScheduler:59 - Got job 5 (collect at StratifiedSamplingLearning.scala:45) with 3 output partitions -2016-05-23 22:33:48 INFO DAGScheduler:59 - Final stage: ResultStage 5(collect at StratifiedSamplingLearning.scala:45) -2016-05-23 22:33:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:33:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:33:48 INFO DAGScheduler:59 - Submitting ResultStage 5 (PartitionwiseSampledRDD[9] at sample at StratifiedSamplingLearning.scala:42), which has no missing parents -2016-05-23 22:33:48 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53772 in memory (size: 1462.0 B, free: 730.6 MB) -2016-05-23 22:33:48 INFO MemoryStore:59 - ensureFreeSpace(2192) called with curMem=209953, maxMem=766075207 -2016-05-23 22:33:48 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-23 22:33:48 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-23 22:33:48 INFO MemoryStore:59 - ensureFreeSpace(1351) called with curMem=212145, maxMem=766075207 -2016-05-23 22:33:48 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1351.0 B, free 730.4 MB) -2016-05-23 22:33:48 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53772 in memory (size: 2.2 KB, free: 730.6 MB) -2016-05-23 22:33:48 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53772 (size: 1351.0 B, free: 730.6 MB) -2016-05-23 22:33:48 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:33:48 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 5 (PartitionwiseSampledRDD[9] at sample at StratifiedSamplingLearning.scala:42) -2016-05-23 22:33:48 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 3 tasks -2016-05-23 22:33:48 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53772 in memory (size: 1939.0 B, free: 730.6 MB) -2016-05-23 22:33:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 14, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:33:48 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-23 22:33:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 15, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:33:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 16, localhost, PROCESS_LOCAL, 2251 bytes) -2016-05-23 22:33:48 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 14) -2016-05-23 22:33:48 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 16) -2016-05-23 22:33:48 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 15) -2016-05-23 22:33:48 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 16). 926 bytes result sent to driver -2016-05-23 22:33:48 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 14). 914 bytes result sent to driver -2016-05-23 22:33:48 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 15). 914 bytes result sent to driver -2016-05-23 22:33:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 16) in 9 ms on localhost (1/3) -2016-05-23 22:33:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 15) in 12 ms on localhost (2/3) -2016-05-23 22:33:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 14) in 13 ms on localhost (3/3) -2016-05-23 22:33:48 INFO DAGScheduler:59 - ResultStage 5 (collect at StratifiedSamplingLearning.scala:45) finished in 0.013 s -2016-05-23 22:33:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 22:33:48 INFO DAGScheduler:59 - Job 5 finished: collect at StratifiedSamplingLearning.scala:45, took 0.034799 s -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 22:33:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 22:33:48 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 22:33:48 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 22:33:49 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 22:33:49 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 22:33:49 INFO BlockManager:59 - BlockManager stopped -2016-05-23 22:33:49 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 22:33:49 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 22:33:49 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 22:33:49 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 22:33:49 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-a786e34b-4e1b-4a36-beed-7ac5ec250252 -2016-05-23 22:33:49 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 22:33:49 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-23 22:34:01 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 22:34:03 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 22:34:03 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 22:34:03 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 22:34:03 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 22:34:04 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 22:34:04 INFO Remoting:74 - Starting remoting -2016-05-23 22:34:04 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53798] -2016-05-23 22:34:04 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53798. -2016-05-23 22:34:04 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 22:34:04 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 22:34:04 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-e61bf07a-7700-40c9-a3d4-1d1877a055b2 -2016-05-23 22:34:04 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 22:34:04 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-adc9f7b8-1467-48ef-b228-804ca0d4c7ad\httpd-a4eb7a9b-e72b-4003-975b-7b2dfd9f287b -2016-05-23 22:34:04 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 22:34:04 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:34:04 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53799 -2016-05-23 22:34:04 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53799. -2016-05-23 22:34:05 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 22:34:05 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:34:05 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 22:34:05 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 22:34:05 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 22:34:05 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 22:34:05 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 22:34:05 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53818. -2016-05-23 22:34:05 INFO NettyBlockTransferService:59 - Server created on 53818 -2016-05-23 22:34:05 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 22:34:05 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53818 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53818) -2016-05-23 22:34:05 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 22:34:06 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 22:34:06 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 22:34:06 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 22:34:06 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 22:34:06 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53818 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 22:34:06 INFO SparkContext:59 - Created broadcast 0 from textFile at StratifiedSamplingLearning.scala:19 -2016-05-23 22:34:08 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 22:34:08 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 22:34:08 INFO SparkContext:59 - Starting job: foreach at StratifiedSamplingLearning.scala:26 -2016-05-23 22:34:08 INFO DAGScheduler:59 - Got job 0 (foreach at StratifiedSamplingLearning.scala:26) with 2 output partitions -2016-05-23 22:34:08 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StratifiedSamplingLearning.scala:26) -2016-05-23 22:34:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:34:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:34:08 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at StratifiedSamplingLearning.scala:25), which has no missing parents -2016-05-23 22:34:08 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=171580, maxMem=766075207 -2016-05-23 22:34:08 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 22:34:08 INFO MemoryStore:59 - ensureFreeSpace(1939) called with curMem=175036, maxMem=766075207 -2016-05-23 22:34:08 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1939.0 B, free 730.4 MB) -2016-05-23 22:34:08 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53818 (size: 1939.0 B, free: 730.6 MB) -2016-05-23 22:34:08 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:34:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at StratifiedSamplingLearning.scala:25) -2016-05-23 22:34:08 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 22:34:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:34:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:34:08 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 22:34:08 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 22:34:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:21+22 -2016-05-23 22:34:08 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 22:34:08 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 22:34:08 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 22:34:08 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 22:34:08 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 22:34:08 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 22:34:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 152 ms on localhost (1/2) -2016-05-23 22:34:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:0+21 -2016-05-23 22:34:08 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 22:34:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 336 ms on localhost (2/2) -2016-05-23 22:34:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 22:34:08 INFO DAGScheduler:59 - ResultStage 0 (foreach at StratifiedSamplingLearning.scala:26) finished in 0.368 s -2016-05-23 22:34:08 INFO DAGScheduler:59 - Job 0 finished: foreach at StratifiedSamplingLearning.scala:26, took 0.627319 s -2016-05-23 22:34:09 INFO SparkContext:59 - Starting job: foreach at StratifiedSamplingLearning.scala:29 -2016-05-23 22:34:09 INFO DAGScheduler:59 - Got job 1 (foreach at StratifiedSamplingLearning.scala:29) with 2 output partitions -2016-05-23 22:34:09 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StratifiedSamplingLearning.scala:29) -2016-05-23 22:34:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:34:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:34:09 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at sampleByKey at StratifiedSamplingLearning.scala:28), which has no missing parents -2016-05-23 22:34:09 INFO MemoryStore:59 - ensureFreeSpace(4128) called with curMem=176975, maxMem=766075207 -2016-05-23 22:34:09 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-05-23 22:34:09 INFO MemoryStore:59 - ensureFreeSpace(2296) called with curMem=181103, maxMem=766075207 -2016-05-23 22:34:09 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-05-23 22:34:09 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53818 (size: 2.2 KB, free: 730.6 MB) -2016-05-23 22:34:09 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:34:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at sampleByKey at StratifiedSamplingLearning.scala:28) -2016-05-23 22:34:09 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 22:34:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:34:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:34:09 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 22:34:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:21+22 -2016-05-23 22:34:09 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 22:34:09 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 22:34:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:0+21 -2016-05-23 22:34:09 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 22:34:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 44 ms on localhost (1/2) -2016-05-23 22:34:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 89 ms on localhost (2/2) -2016-05-23 22:34:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 22:34:09 INFO DAGScheduler:59 - ResultStage 1 (foreach at StratifiedSamplingLearning.scala:29) finished in 0.094 s -2016-05-23 22:34:09 INFO DAGScheduler:59 - Job 1 finished: foreach at StratifiedSamplingLearning.scala:29, took 0.168572 s -2016-05-23 22:34:09 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:35 -2016-05-23 22:34:09 INFO DAGScheduler:59 - Got job 2 (collect at StratifiedSamplingLearning.scala:35) with 4 output partitions -2016-05-23 22:34:09 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at StratifiedSamplingLearning.scala:35) -2016-05-23 22:34:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:34:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:34:09 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[6] at sampleByKey at StratifiedSamplingLearning.scala:35), which has no missing parents -2016-05-23 22:34:09 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=183399, maxMem=766075207 -2016-05-23 22:34:09 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 22:34:09 INFO MemoryStore:59 - ensureFreeSpace(1462) called with curMem=185735, maxMem=766075207 -2016-05-23 22:34:09 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1462.0 B, free 730.4 MB) -2016-05-23 22:34:09 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53818 (size: 1462.0 B, free: 730.6 MB) -2016-05-23 22:34:09 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:34:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 2 (MapPartitionsRDD[6] at sampleByKey at StratifiedSamplingLearning.scala:35) -2016-05-23 22:34:09 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 4 tasks -2016-05-23 22:34:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2188 bytes) -2016-05-23 22:34:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2212 bytes) -2016-05-23 22:34:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2210 bytes) -2016-05-23 22:34:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2216 bytes) -2016-05-23 22:34:09 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-23 22:34:09 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 22:34:09 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-05-23 22:34:09 INFO Executor:59 - Running task 3.0 in stage 2.0 (TID 7) -2016-05-23 22:34:09 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1061 bytes result sent to driver -2016-05-23 22:34:09 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1059 bytes result sent to driver -2016-05-23 22:34:09 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 6). 1060 bytes result sent to driver -2016-05-23 22:34:09 INFO Executor:59 - Finished task 3.0 in stage 2.0 (TID 7). 1087 bytes result sent to driver -2016-05-23 22:34:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 19 ms on localhost (1/4) -2016-05-23 22:34:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 29 ms on localhost (2/4) -2016-05-23 22:34:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 6) in 21 ms on localhost (3/4) -2016-05-23 22:34:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 2.0 (TID 7) in 21 ms on localhost (4/4) -2016-05-23 22:34:09 INFO DAGScheduler:59 - ResultStage 2 (collect at StratifiedSamplingLearning.scala:35) finished in 0.032 s -2016-05-23 22:34:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 22:34:09 INFO DAGScheduler:59 - Job 2 finished: collect at StratifiedSamplingLearning.scala:35, took 0.047685 s -2016-05-23 22:34:09 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:43 -2016-05-23 22:34:09 INFO DAGScheduler:59 - Got job 3 (collect at StratifiedSamplingLearning.scala:43) with 3 output partitions -2016-05-23 22:34:09 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at StratifiedSamplingLearning.scala:43) -2016-05-23 22:34:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:34:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:34:09 INFO DAGScheduler:59 - Submitting ResultStage 3 (ParallelCollectionRDD[7] at parallelize at StratifiedSamplingLearning.scala:40), which has no missing parents -2016-05-23 22:34:09 INFO MemoryStore:59 - ensureFreeSpace(1272) called with curMem=187197, maxMem=766075207 -2016-05-23 22:34:09 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 1272.0 B, free 730.4 MB) -2016-05-23 22:34:09 INFO MemoryStore:59 - ensureFreeSpace(843) called with curMem=188469, maxMem=766075207 -2016-05-23 22:34:09 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 843.0 B, free 730.4 MB) -2016-05-23 22:34:09 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53818 (size: 843.0 B, free: 730.6 MB) -2016-05-23 22:34:09 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:34:09 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 3 (ParallelCollectionRDD[7] at parallelize at StratifiedSamplingLearning.scala:40) -2016-05-23 22:34:09 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 3 tasks -2016-05-23 22:34:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 2085 bytes) -2016-05-23 22:34:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2085 bytes) -2016-05-23 22:34:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2142 bytes) -2016-05-23 22:34:09 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 10) -2016-05-23 22:34:09 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 8) -2016-05-23 22:34:09 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 9) -2016-05-23 22:34:09 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 8). 922 bytes result sent to driver -2016-05-23 22:34:09 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 10). 926 bytes result sent to driver -2016-05-23 22:34:09 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 9). 926 bytes result sent to driver -2016-05-23 22:34:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 8) in 18 ms on localhost (1/3) -2016-05-23 22:34:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 10) in 13 ms on localhost (2/3) -2016-05-23 22:34:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 9) in 16 ms on localhost (3/3) -2016-05-23 22:34:09 INFO DAGScheduler:59 - ResultStage 3 (collect at StratifiedSamplingLearning.scala:43) finished in 0.023 s -2016-05-23 22:34:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 22:34:09 INFO DAGScheduler:59 - Job 3 finished: collect at StratifiedSamplingLearning.scala:43, took 0.064682 s -2016-05-23 22:34:09 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53818 in memory (size: 1462.0 B, free: 730.6 MB) -2016-05-23 22:34:09 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53818 in memory (size: 2.2 KB, free: 730.6 MB) -2016-05-23 22:34:09 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53818 in memory (size: 1939.0 B, free: 730.6 MB) -2016-05-23 22:34:09 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:44 -2016-05-23 22:34:09 INFO DAGScheduler:59 - Got job 4 (collect at StratifiedSamplingLearning.scala:44) with 3 output partitions -2016-05-23 22:34:09 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at StratifiedSamplingLearning.scala:44) -2016-05-23 22:34:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:34:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:34:09 INFO DAGScheduler:59 - Submitting ResultStage 4 (PartitionwiseSampledRDD[8] at sample at StratifiedSamplingLearning.scala:41), which has no missing parents -2016-05-23 22:34:09 INFO MemoryStore:59 - ensureFreeSpace(18856) called with curMem=173695, maxMem=766075207 -2016-05-23 22:34:09 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 18.4 KB, free 730.4 MB) -2016-05-23 22:34:09 INFO MemoryStore:59 - ensureFreeSpace(7702) called with curMem=192551, maxMem=766075207 -2016-05-23 22:34:09 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.4 MB) -2016-05-23 22:34:09 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53818 (size: 7.5 KB, free: 730.6 MB) -2016-05-23 22:34:09 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:34:09 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 4 (PartitionwiseSampledRDD[8] at sample at StratifiedSamplingLearning.scala:41) -2016-05-23 22:34:09 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 3 tasks -2016-05-23 22:34:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 11, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:34:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 12, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:34:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 2251 bytes) -2016-05-23 22:34:09 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 11) -2016-05-23 22:34:09 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 13) -2016-05-23 22:34:09 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 12) -2016-05-23 22:34:09 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 12). 902 bytes result sent to driver -2016-05-23 22:34:09 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 11). 914 bytes result sent to driver -2016-05-23 22:34:09 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 13). 910 bytes result sent to driver -2016-05-23 22:34:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 12) in 21 ms on localhost (1/3) -2016-05-23 22:34:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 11) in 24 ms on localhost (2/3) -2016-05-23 22:34:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 13) in 21 ms on localhost (3/3) -2016-05-23 22:34:09 INFO DAGScheduler:59 - ResultStage 4 (collect at StratifiedSamplingLearning.scala:44) finished in 0.025 s -2016-05-23 22:34:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 22:34:09 INFO DAGScheduler:59 - Job 4 finished: collect at StratifiedSamplingLearning.scala:44, took 0.041895 s -2016-05-23 22:34:09 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:45 -2016-05-23 22:34:09 INFO DAGScheduler:59 - Got job 5 (collect at StratifiedSamplingLearning.scala:45) with 3 output partitions -2016-05-23 22:34:09 INFO DAGScheduler:59 - Final stage: ResultStage 5(collect at StratifiedSamplingLearning.scala:45) -2016-05-23 22:34:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:34:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:34:09 INFO DAGScheduler:59 - Submitting ResultStage 5 (PartitionwiseSampledRDD[9] at sample at StratifiedSamplingLearning.scala:42), which has no missing parents -2016-05-23 22:34:09 INFO MemoryStore:59 - ensureFreeSpace(2192) called with curMem=200253, maxMem=766075207 -2016-05-23 22:34:09 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-23 22:34:09 INFO MemoryStore:59 - ensureFreeSpace(1353) called with curMem=202445, maxMem=766075207 -2016-05-23 22:34:09 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1353.0 B, free 730.4 MB) -2016-05-23 22:34:09 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53818 (size: 1353.0 B, free: 730.6 MB) -2016-05-23 22:34:09 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:34:09 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 5 (PartitionwiseSampledRDD[9] at sample at StratifiedSamplingLearning.scala:42) -2016-05-23 22:34:09 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 3 tasks -2016-05-23 22:34:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 14, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:34:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 15, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:34:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 16, localhost, PROCESS_LOCAL, 2251 bytes) -2016-05-23 22:34:09 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 16) -2016-05-23 22:34:09 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 15) -2016-05-23 22:34:09 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 14) -2016-05-23 22:34:09 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 15). 914 bytes result sent to driver -2016-05-23 22:34:09 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 16). 926 bytes result sent to driver -2016-05-23 22:34:09 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 14). 914 bytes result sent to driver -2016-05-23 22:34:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 16) in 13 ms on localhost (1/3) -2016-05-23 22:34:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 14) in 16 ms on localhost (2/3) -2016-05-23 22:34:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 15) in 16 ms on localhost (3/3) -2016-05-23 22:34:09 INFO DAGScheduler:59 - ResultStage 5 (collect at StratifiedSamplingLearning.scala:45) finished in 0.017 s -2016-05-23 22:34:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 22:34:09 INFO DAGScheduler:59 - Job 5 finished: collect at StratifiedSamplingLearning.scala:45, took 0.032499 s -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 22:34:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 22:34:09 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 22:34:09 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 22:34:09 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 22:34:09 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 22:34:09 INFO BlockManager:59 - BlockManager stopped -2016-05-23 22:34:09 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 22:34:09 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 22:34:09 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 22:34:09 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 22:34:09 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-adc9f7b8-1467-48ef-b228-804ca0d4c7ad -2016-05-23 22:34:09 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 22:34:47 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 22:34:48 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 22:34:48 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 22:34:48 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 22:34:48 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 22:34:49 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 22:34:49 INFO Remoting:74 - Starting remoting -2016-05-23 22:34:49 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53839] -2016-05-23 22:34:49 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53839. -2016-05-23 22:34:49 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 22:34:49 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 22:34:49 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-188ead8a-23d3-4b28-8412-0455c61c2996 -2016-05-23 22:34:49 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 22:34:49 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-dbf9ba56-9320-4181-a832-83d5af109dcc\httpd-f55dfd0d-9870-4697-b43f-762f538a2e02 -2016-05-23 22:34:49 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 22:34:49 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:34:49 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53840 -2016-05-23 22:34:49 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53840. -2016-05-23 22:34:49 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 22:34:50 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:34:50 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 22:34:50 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 22:34:50 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 22:34:50 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 22:34:50 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 22:34:50 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53859. -2016-05-23 22:34:50 INFO NettyBlockTransferService:59 - Server created on 53859 -2016-05-23 22:34:50 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 22:34:50 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53859 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53859) -2016-05-23 22:34:50 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 22:34:51 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 22:34:51 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 22:34:51 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 22:34:51 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 22:34:51 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53859 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 22:34:51 INFO SparkContext:59 - Created broadcast 0 from textFile at StratifiedSamplingLearning.scala:19 -2016-05-23 22:34:52 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 22:34:52 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 22:34:52 INFO SparkContext:59 - Starting job: foreach at StratifiedSamplingLearning.scala:26 -2016-05-23 22:34:52 INFO DAGScheduler:59 - Got job 0 (foreach at StratifiedSamplingLearning.scala:26) with 2 output partitions -2016-05-23 22:34:52 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StratifiedSamplingLearning.scala:26) -2016-05-23 22:34:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:34:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:34:52 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at StratifiedSamplingLearning.scala:25), which has no missing parents -2016-05-23 22:34:52 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=171580, maxMem=766075207 -2016-05-23 22:34:52 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 22:34:52 INFO MemoryStore:59 - ensureFreeSpace(1939) called with curMem=175036, maxMem=766075207 -2016-05-23 22:34:52 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1939.0 B, free 730.4 MB) -2016-05-23 22:34:52 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53859 (size: 1939.0 B, free: 730.6 MB) -2016-05-23 22:34:52 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:34:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at StratifiedSamplingLearning.scala:25) -2016-05-23 22:34:52 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 22:34:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:34:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:34:52 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 22:34:52 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 22:34:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:0+21 -2016-05-23 22:34:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:21+22 -2016-05-23 22:34:52 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 22:34:52 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 22:34:52 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 22:34:52 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 22:34:52 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 22:34:52 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 22:34:52 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 22:34:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 94 ms on localhost (1/2) -2016-05-23 22:34:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 122 ms on localhost (2/2) -2016-05-23 22:34:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 22:34:52 INFO DAGScheduler:59 - ResultStage 0 (foreach at StratifiedSamplingLearning.scala:26) finished in 0.138 s -2016-05-23 22:34:52 INFO DAGScheduler:59 - Job 0 finished: foreach at StratifiedSamplingLearning.scala:26, took 0.286315 s -2016-05-23 22:34:52 INFO SparkContext:59 - Starting job: foreach at StratifiedSamplingLearning.scala:30 -2016-05-23 22:34:52 INFO DAGScheduler:59 - Got job 1 (foreach at StratifiedSamplingLearning.scala:30) with 2 output partitions -2016-05-23 22:34:52 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StratifiedSamplingLearning.scala:30) -2016-05-23 22:34:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:34:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:34:52 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at sampleByKey at StratifiedSamplingLearning.scala:29), which has no missing parents -2016-05-23 22:34:52 INFO MemoryStore:59 - ensureFreeSpace(4128) called with curMem=176975, maxMem=766075207 -2016-05-23 22:34:52 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-05-23 22:34:52 INFO MemoryStore:59 - ensureFreeSpace(2296) called with curMem=181103, maxMem=766075207 -2016-05-23 22:34:52 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-05-23 22:34:52 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53859 (size: 2.2 KB, free: 730.6 MB) -2016-05-23 22:34:52 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:34:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at sampleByKey at StratifiedSamplingLearning.scala:29) -2016-05-23 22:34:52 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 22:34:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:34:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:34:52 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 22:34:52 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 22:34:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:21+22 -2016-05-23 22:34:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:0+21 -2016-05-23 22:34:52 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 22:34:52 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 22:34:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 23 ms on localhost (1/2) -2016-05-23 22:34:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 26 ms on localhost (2/2) -2016-05-23 22:34:53 INFO DAGScheduler:59 - ResultStage 1 (foreach at StratifiedSamplingLearning.scala:30) finished in 0.026 s -2016-05-23 22:34:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 22:34:53 INFO DAGScheduler:59 - Job 1 finished: foreach at StratifiedSamplingLearning.scala:30, took 0.047083 s -2016-05-23 22:34:53 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:36 -2016-05-23 22:34:53 INFO DAGScheduler:59 - Got job 2 (collect at StratifiedSamplingLearning.scala:36) with 4 output partitions -2016-05-23 22:34:53 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at StratifiedSamplingLearning.scala:36) -2016-05-23 22:34:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:34:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:34:53 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[6] at sampleByKey at StratifiedSamplingLearning.scala:36), which has no missing parents -2016-05-23 22:34:53 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=183399, maxMem=766075207 -2016-05-23 22:34:53 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 22:34:53 INFO MemoryStore:59 - ensureFreeSpace(1462) called with curMem=185735, maxMem=766075207 -2016-05-23 22:34:53 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1462.0 B, free 730.4 MB) -2016-05-23 22:34:53 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53859 (size: 1462.0 B, free: 730.6 MB) -2016-05-23 22:34:53 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:34:53 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 2 (MapPartitionsRDD[6] at sampleByKey at StratifiedSamplingLearning.scala:36) -2016-05-23 22:34:53 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 4 tasks -2016-05-23 22:34:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2188 bytes) -2016-05-23 22:34:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2212 bytes) -2016-05-23 22:34:53 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2210 bytes) -2016-05-23 22:34:53 INFO TaskSetManager:59 - Starting task 3.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2216 bytes) -2016-05-23 22:34:53 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 22:34:53 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-23 22:34:53 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-05-23 22:34:53 INFO Executor:59 - Running task 3.0 in stage 2.0 (TID 7) -2016-05-23 22:34:53 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1059 bytes result sent to driver -2016-05-23 22:34:53 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1061 bytes result sent to driver -2016-05-23 22:34:53 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 6). 1060 bytes result sent to driver -2016-05-23 22:34:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 32 ms on localhost (1/4) -2016-05-23 22:34:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 22 ms on localhost (2/4) -2016-05-23 22:34:53 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 6) in 21 ms on localhost (3/4) -2016-05-23 22:34:53 INFO Executor:59 - Finished task 3.0 in stage 2.0 (TID 7). 1087 bytes result sent to driver -2016-05-23 22:34:53 INFO TaskSetManager:59 - Finished task 3.0 in stage 2.0 (TID 7) in 29 ms on localhost (4/4) -2016-05-23 22:34:53 INFO DAGScheduler:59 - ResultStage 2 (collect at StratifiedSamplingLearning.scala:36) finished in 0.044 s -2016-05-23 22:34:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 22:34:53 INFO DAGScheduler:59 - Job 2 finished: collect at StratifiedSamplingLearning.scala:36, took 0.067695 s -2016-05-23 22:34:53 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:44 -2016-05-23 22:34:53 INFO DAGScheduler:59 - Got job 3 (collect at StratifiedSamplingLearning.scala:44) with 3 output partitions -2016-05-23 22:34:53 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at StratifiedSamplingLearning.scala:44) -2016-05-23 22:34:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:34:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:34:53 INFO DAGScheduler:59 - Submitting ResultStage 3 (ParallelCollectionRDD[7] at parallelize at StratifiedSamplingLearning.scala:41), which has no missing parents -2016-05-23 22:34:53 INFO MemoryStore:59 - ensureFreeSpace(1272) called with curMem=187197, maxMem=766075207 -2016-05-23 22:34:53 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 1272.0 B, free 730.4 MB) -2016-05-23 22:34:53 INFO MemoryStore:59 - ensureFreeSpace(843) called with curMem=188469, maxMem=766075207 -2016-05-23 22:34:53 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 843.0 B, free 730.4 MB) -2016-05-23 22:34:53 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53859 (size: 843.0 B, free: 730.6 MB) -2016-05-23 22:34:53 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:34:53 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 3 (ParallelCollectionRDD[7] at parallelize at StratifiedSamplingLearning.scala:41) -2016-05-23 22:34:53 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 3 tasks -2016-05-23 22:34:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 2085 bytes) -2016-05-23 22:34:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2085 bytes) -2016-05-23 22:34:53 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2142 bytes) -2016-05-23 22:34:53 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 8) -2016-05-23 22:34:53 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 10) -2016-05-23 22:34:53 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 9) -2016-05-23 22:34:53 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 8). 922 bytes result sent to driver -2016-05-23 22:34:53 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 9). 926 bytes result sent to driver -2016-05-23 22:34:53 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 10). 926 bytes result sent to driver -2016-05-23 22:34:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 8) in 41 ms on localhost (1/3) -2016-05-23 22:34:53 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 10) in 37 ms on localhost (2/3) -2016-05-23 22:34:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 9) in 39 ms on localhost (3/3) -2016-05-23 22:34:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 22:34:53 INFO DAGScheduler:59 - ResultStage 3 (collect at StratifiedSamplingLearning.scala:44) finished in 0.045 s -2016-05-23 22:34:53 INFO DAGScheduler:59 - Job 3 finished: collect at StratifiedSamplingLearning.scala:44, took 0.061729 s -2016-05-23 22:34:53 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53859 in memory (size: 1462.0 B, free: 730.6 MB) -2016-05-23 22:34:53 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:45 -2016-05-23 22:34:53 INFO DAGScheduler:59 - Got job 4 (collect at StratifiedSamplingLearning.scala:45) with 3 output partitions -2016-05-23 22:34:53 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at StratifiedSamplingLearning.scala:45) -2016-05-23 22:34:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:34:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:34:53 INFO DAGScheduler:59 - Submitting ResultStage 4 (PartitionwiseSampledRDD[8] at sample at StratifiedSamplingLearning.scala:42), which has no missing parents -2016-05-23 22:34:53 INFO MemoryStore:59 - ensureFreeSpace(18856) called with curMem=185514, maxMem=766075207 -2016-05-23 22:34:53 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 18.4 KB, free 730.4 MB) -2016-05-23 22:34:53 INFO MemoryStore:59 - ensureFreeSpace(7701) called with curMem=197946, maxMem=766075207 -2016-05-23 22:34:53 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.4 MB) -2016-05-23 22:34:53 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53859 in memory (size: 2.2 KB, free: 730.6 MB) -2016-05-23 22:34:53 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53859 (size: 7.5 KB, free: 730.6 MB) -2016-05-23 22:34:53 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:34:53 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53859 in memory (size: 1939.0 B, free: 730.6 MB) -2016-05-23 22:34:53 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 4 (PartitionwiseSampledRDD[8] at sample at StratifiedSamplingLearning.scala:42) -2016-05-23 22:34:53 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 3 tasks -2016-05-23 22:34:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 11, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:34:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 12, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:34:53 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 2251 bytes) -2016-05-23 22:34:53 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 11) -2016-05-23 22:34:53 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 13) -2016-05-23 22:34:53 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 12) -2016-05-23 22:34:53 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 13). 910 bytes result sent to driver -2016-05-23 22:34:53 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 11). 914 bytes result sent to driver -2016-05-23 22:34:53 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 12). 902 bytes result sent to driver -2016-05-23 22:34:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 11) in 26 ms on localhost (1/3) -2016-05-23 22:34:53 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 13) in 23 ms on localhost (2/3) -2016-05-23 22:34:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 12) in 24 ms on localhost (3/3) -2016-05-23 22:34:53 INFO DAGScheduler:59 - ResultStage 4 (collect at StratifiedSamplingLearning.scala:45) finished in 0.028 s -2016-05-23 22:34:53 INFO DAGScheduler:59 - Job 4 finished: collect at StratifiedSamplingLearning.scala:45, took 0.049622 s -2016-05-23 22:34:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 22:34:53 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:46 -2016-05-23 22:34:53 INFO DAGScheduler:59 - Got job 5 (collect at StratifiedSamplingLearning.scala:46) with 3 output partitions -2016-05-23 22:34:53 INFO DAGScheduler:59 - Final stage: ResultStage 5(collect at StratifiedSamplingLearning.scala:46) -2016-05-23 22:34:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:34:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:34:53 INFO DAGScheduler:59 - Submitting ResultStage 5 (PartitionwiseSampledRDD[9] at sample at StratifiedSamplingLearning.scala:43), which has no missing parents -2016-05-23 22:34:53 INFO MemoryStore:59 - ensureFreeSpace(2192) called with curMem=200252, maxMem=766075207 -2016-05-23 22:34:53 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-23 22:34:53 INFO MemoryStore:59 - ensureFreeSpace(1352) called with curMem=202444, maxMem=766075207 -2016-05-23 22:34:53 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1352.0 B, free 730.4 MB) -2016-05-23 22:34:53 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53859 (size: 1352.0 B, free: 730.6 MB) -2016-05-23 22:34:53 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:34:53 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 5 (PartitionwiseSampledRDD[9] at sample at StratifiedSamplingLearning.scala:43) -2016-05-23 22:34:53 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 3 tasks -2016-05-23 22:34:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 14, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:34:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 15, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:34:53 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 16, localhost, PROCESS_LOCAL, 2251 bytes) -2016-05-23 22:34:53 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 14) -2016-05-23 22:34:53 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 16) -2016-05-23 22:34:53 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 15) -2016-05-23 22:34:53 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 16). 926 bytes result sent to driver -2016-05-23 22:34:53 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 14). 914 bytes result sent to driver -2016-05-23 22:34:53 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 15). 914 bytes result sent to driver -2016-05-23 22:34:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 14) in 23 ms on localhost (1/3) -2016-05-23 22:34:53 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 16) in 20 ms on localhost (2/3) -2016-05-23 22:34:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 15) in 23 ms on localhost (3/3) -2016-05-23 22:34:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 22:34:53 INFO DAGScheduler:59 - ResultStage 5 (collect at StratifiedSamplingLearning.scala:46) finished in 0.026 s -2016-05-23 22:34:53 INFO DAGScheduler:59 - Job 5 finished: collect at StratifiedSamplingLearning.scala:46, took 0.041429 s -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 22:34:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 22:34:53 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 22:34:53 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 22:34:53 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 22:34:53 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 22:34:53 INFO BlockManager:59 - BlockManager stopped -2016-05-23 22:34:53 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 22:34:53 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 22:34:53 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 22:34:53 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 22:34:53 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-dbf9ba56-9320-4181-a832-83d5af109dcc -2016-05-23 22:34:53 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 22:37:29 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 22:37:29 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 22:37:30 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 22:37:30 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 22:37:30 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 22:37:30 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 22:37:30 INFO Remoting:74 - Starting remoting -2016-05-23 22:37:31 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53883] -2016-05-23 22:37:31 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53883. -2016-05-23 22:37:31 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 22:37:31 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 22:37:31 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-fbe60033-77f7-4fe3-a104-412f89df091f -2016-05-23 22:37:31 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 22:37:31 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-21ab0075-8fe4-4f2b-a700-c1523328c085\httpd-47e79a44-5352-4140-a74b-604d32780a48 -2016-05-23 22:37:31 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 22:37:31 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:37:31 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53884 -2016-05-23 22:37:31 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53884. -2016-05-23 22:37:31 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 22:37:31 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:37:31 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 22:37:31 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 22:37:31 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 22:37:31 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 22:37:31 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 22:37:32 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53903. -2016-05-23 22:37:32 INFO NettyBlockTransferService:59 - Server created on 53903 -2016-05-23 22:37:32 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 22:37:32 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53903 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53903) -2016-05-23 22:37:32 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 22:37:32 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-23 22:37:32 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-23 22:37:32 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-23 22:37:32 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-23 22:37:32 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53903 (size: 13.9 KB, free: 730.6 MB) -2016-05-23 22:37:32 INFO SparkContext:59 - Created broadcast 0 from textFile at StratifiedSamplingLearning.scala:19 -2016-05-23 22:37:34 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-23 22:37:34 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-23 22:37:34 INFO SparkContext:59 - Starting job: foreach at StratifiedSamplingLearning.scala:26 -2016-05-23 22:37:34 INFO DAGScheduler:59 - Got job 0 (foreach at StratifiedSamplingLearning.scala:26) with 2 output partitions -2016-05-23 22:37:34 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at StratifiedSamplingLearning.scala:26) -2016-05-23 22:37:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:37:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:37:34 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at StratifiedSamplingLearning.scala:25), which has no missing parents -2016-05-23 22:37:34 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=171580, maxMem=766075207 -2016-05-23 22:37:34 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-23 22:37:34 INFO MemoryStore:59 - ensureFreeSpace(1939) called with curMem=175036, maxMem=766075207 -2016-05-23 22:37:34 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1939.0 B, free 730.4 MB) -2016-05-23 22:37:34 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53903 (size: 1939.0 B, free: 730.6 MB) -2016-05-23 22:37:34 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:37:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at StratifiedSamplingLearning.scala:25) -2016-05-23 22:37:34 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 22:37:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:37:34 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 22:37:34 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 22:37:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:0+21 -2016-05-23 22:37:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:21+22 -2016-05-23 22:37:34 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-23 22:37:34 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-23 22:37:34 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-23 22:37:34 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-23 22:37:34 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-23 22:37:34 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-23 22:37:34 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-23 22:37:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 109 ms on localhost (1/2) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 93 ms on localhost (2/2) -2016-05-23 22:37:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 22:37:34 INFO DAGScheduler:59 - ResultStage 0 (foreach at StratifiedSamplingLearning.scala:26) finished in 0.128 s -2016-05-23 22:37:34 INFO DAGScheduler:59 - Job 0 finished: foreach at StratifiedSamplingLearning.scala:26, took 0.313532 s -2016-05-23 22:37:34 INFO SparkContext:59 - Starting job: foreach at StratifiedSamplingLearning.scala:30 -2016-05-23 22:37:34 INFO DAGScheduler:59 - Got job 1 (foreach at StratifiedSamplingLearning.scala:30) with 2 output partitions -2016-05-23 22:37:34 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at StratifiedSamplingLearning.scala:30) -2016-05-23 22:37:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:37:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:37:34 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at sampleByKey at StratifiedSamplingLearning.scala:29), which has no missing parents -2016-05-23 22:37:34 INFO MemoryStore:59 - ensureFreeSpace(4128) called with curMem=176975, maxMem=766075207 -2016-05-23 22:37:34 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-05-23 22:37:34 INFO MemoryStore:59 - ensureFreeSpace(2296) called with curMem=181103, maxMem=766075207 -2016-05-23 22:37:34 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-05-23 22:37:34 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53903 (size: 2.2 KB, free: 730.6 MB) -2016-05-23 22:37:34 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:37:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at sampleByKey at StratifiedSamplingLearning.scala:29) -2016-05-23 22:37:34 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 22:37:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-23 22:37:34 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 22:37:34 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 22:37:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:21+22 -2016-05-23 22:37:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/StratifiedSampling.txt:0+21 -2016-05-23 22:37:34 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-05-23 22:37:34 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-05-23 22:37:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 30 ms on localhost (1/2) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 29 ms on localhost (2/2) -2016-05-23 22:37:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 22:37:34 INFO DAGScheduler:59 - ResultStage 1 (foreach at StratifiedSamplingLearning.scala:30) finished in 0.032 s -2016-05-23 22:37:34 INFO DAGScheduler:59 - Job 1 finished: foreach at StratifiedSamplingLearning.scala:30, took 0.056504 s -2016-05-23 22:37:34 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:36 -2016-05-23 22:37:34 INFO DAGScheduler:59 - Got job 2 (collect at StratifiedSamplingLearning.scala:36) with 4 output partitions -2016-05-23 22:37:34 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at StratifiedSamplingLearning.scala:36) -2016-05-23 22:37:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:37:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:37:34 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[6] at sampleByKey at StratifiedSamplingLearning.scala:36), which has no missing parents -2016-05-23 22:37:34 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=183399, maxMem=766075207 -2016-05-23 22:37:34 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-23 22:37:34 INFO MemoryStore:59 - ensureFreeSpace(1462) called with curMem=185735, maxMem=766075207 -2016-05-23 22:37:34 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1462.0 B, free 730.4 MB) -2016-05-23 22:37:34 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53903 (size: 1462.0 B, free: 730.6 MB) -2016-05-23 22:37:34 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:37:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 2 (MapPartitionsRDD[6] at sampleByKey at StratifiedSamplingLearning.scala:36) -2016-05-23 22:37:34 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 4 tasks -2016-05-23 22:37:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2188 bytes) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2212 bytes) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2210 bytes) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2216 bytes) -2016-05-23 22:37:34 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-23 22:37:34 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 22:37:34 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-05-23 22:37:34 INFO Executor:59 - Running task 3.0 in stage 2.0 (TID 7) -2016-05-23 22:37:34 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 6). 1060 bytes result sent to driver -2016-05-23 22:37:34 INFO Executor:59 - Finished task 3.0 in stage 2.0 (TID 7). 1087 bytes result sent to driver -2016-05-23 22:37:34 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1061 bytes result sent to driver -2016-05-23 22:37:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 6) in 15 ms on localhost (1/4) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 2.0 (TID 7) in 14 ms on localhost (2/4) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 17 ms on localhost (3/4) -2016-05-23 22:37:34 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1059 bytes result sent to driver -2016-05-23 22:37:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 30 ms on localhost (4/4) -2016-05-23 22:37:34 INFO DAGScheduler:59 - ResultStage 2 (collect at StratifiedSamplingLearning.scala:36) finished in 0.030 s -2016-05-23 22:37:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 22:37:34 INFO DAGScheduler:59 - Job 2 finished: collect at StratifiedSamplingLearning.scala:36, took 0.043638 s -2016-05-23 22:37:34 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:44 -2016-05-23 22:37:34 INFO DAGScheduler:59 - Got job 3 (collect at StratifiedSamplingLearning.scala:44) with 3 output partitions -2016-05-23 22:37:34 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at StratifiedSamplingLearning.scala:44) -2016-05-23 22:37:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:37:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:37:34 INFO DAGScheduler:59 - Submitting ResultStage 3 (ParallelCollectionRDD[7] at parallelize at StratifiedSamplingLearning.scala:41), which has no missing parents -2016-05-23 22:37:34 INFO MemoryStore:59 - ensureFreeSpace(1272) called with curMem=187197, maxMem=766075207 -2016-05-23 22:37:34 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 1272.0 B, free 730.4 MB) -2016-05-23 22:37:34 INFO MemoryStore:59 - ensureFreeSpace(843) called with curMem=188469, maxMem=766075207 -2016-05-23 22:37:34 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 843.0 B, free 730.4 MB) -2016-05-23 22:37:34 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53903 (size: 843.0 B, free: 730.6 MB) -2016-05-23 22:37:34 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:37:34 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 3 (ParallelCollectionRDD[7] at parallelize at StratifiedSamplingLearning.scala:41) -2016-05-23 22:37:34 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 3 tasks -2016-05-23 22:37:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 2085 bytes) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2085 bytes) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2142 bytes) -2016-05-23 22:37:34 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 9) -2016-05-23 22:37:34 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 8) -2016-05-23 22:37:34 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 9). 926 bytes result sent to driver -2016-05-23 22:37:34 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 10) -2016-05-23 22:37:34 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 8). 922 bytes result sent to driver -2016-05-23 22:37:34 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53903 in memory (size: 2.2 KB, free: 730.6 MB) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 9) in 12 ms on localhost (1/3) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 8) in 17 ms on localhost (2/3) -2016-05-23 22:37:34 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 10). 926 bytes result sent to driver -2016-05-23 22:37:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 10) in 14 ms on localhost (3/3) -2016-05-23 22:37:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 22:37:34 INFO DAGScheduler:59 - ResultStage 3 (collect at StratifiedSamplingLearning.scala:44) finished in 0.022 s -2016-05-23 22:37:34 INFO DAGScheduler:59 - Job 3 finished: collect at StratifiedSamplingLearning.scala:44, took 0.061037 s -2016-05-23 22:37:34 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53903 in memory (size: 1462.0 B, free: 730.6 MB) -2016-05-23 22:37:34 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53903 in memory (size: 1939.0 B, free: 730.6 MB) -2016-05-23 22:37:34 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:45 -2016-05-23 22:37:34 INFO DAGScheduler:59 - Got job 4 (collect at StratifiedSamplingLearning.scala:45) with 3 output partitions -2016-05-23 22:37:34 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at StratifiedSamplingLearning.scala:45) -2016-05-23 22:37:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:37:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:37:34 INFO DAGScheduler:59 - Submitting ResultStage 4 (PartitionwiseSampledRDD[8] at sample at StratifiedSamplingLearning.scala:42), which has no missing parents -2016-05-23 22:37:34 INFO MemoryStore:59 - ensureFreeSpace(18856) called with curMem=173695, maxMem=766075207 -2016-05-23 22:37:34 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 18.4 KB, free 730.4 MB) -2016-05-23 22:37:34 INFO MemoryStore:59 - ensureFreeSpace(7699) called with curMem=192551, maxMem=766075207 -2016-05-23 22:37:34 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.4 MB) -2016-05-23 22:37:34 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53903 (size: 7.5 KB, free: 730.6 MB) -2016-05-23 22:37:34 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:37:34 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 4 (PartitionwiseSampledRDD[8] at sample at StratifiedSamplingLearning.scala:42) -2016-05-23 22:37:34 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 3 tasks -2016-05-23 22:37:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 11, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 12, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 2251 bytes) -2016-05-23 22:37:34 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 11) -2016-05-23 22:37:34 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 12) -2016-05-23 22:37:34 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 13) -2016-05-23 22:37:34 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 12). 902 bytes result sent to driver -2016-05-23 22:37:34 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 13). 910 bytes result sent to driver -2016-05-23 22:37:34 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 11). 914 bytes result sent to driver -2016-05-23 22:37:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 12) in 18 ms on localhost (1/3) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 13) in 17 ms on localhost (2/3) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 11) in 20 ms on localhost (3/3) -2016-05-23 22:37:34 INFO DAGScheduler:59 - ResultStage 4 (collect at StratifiedSamplingLearning.scala:45) finished in 0.021 s -2016-05-23 22:37:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 22:37:34 INFO DAGScheduler:59 - Job 4 finished: collect at StratifiedSamplingLearning.scala:45, took 0.035089 s -2016-05-23 22:37:34 INFO SparkContext:59 - Starting job: collect at StratifiedSamplingLearning.scala:46 -2016-05-23 22:37:34 INFO DAGScheduler:59 - Got job 5 (collect at StratifiedSamplingLearning.scala:46) with 3 output partitions -2016-05-23 22:37:34 INFO DAGScheduler:59 - Final stage: ResultStage 5(collect at StratifiedSamplingLearning.scala:46) -2016-05-23 22:37:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:37:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:37:34 INFO DAGScheduler:59 - Submitting ResultStage 5 (PartitionwiseSampledRDD[9] at sample at StratifiedSamplingLearning.scala:43), which has no missing parents -2016-05-23 22:37:34 INFO MemoryStore:59 - ensureFreeSpace(2192) called with curMem=200250, maxMem=766075207 -2016-05-23 22:37:34 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-23 22:37:34 INFO MemoryStore:59 - ensureFreeSpace(1351) called with curMem=202442, maxMem=766075207 -2016-05-23 22:37:34 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1351.0 B, free 730.4 MB) -2016-05-23 22:37:34 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53903 (size: 1351.0 B, free: 730.6 MB) -2016-05-23 22:37:34 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:37:34 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 5 (PartitionwiseSampledRDD[9] at sample at StratifiedSamplingLearning.scala:43) -2016-05-23 22:37:34 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 3 tasks -2016-05-23 22:37:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 14, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 15, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 16, localhost, PROCESS_LOCAL, 2251 bytes) -2016-05-23 22:37:34 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 14) -2016-05-23 22:37:34 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 16) -2016-05-23 22:37:34 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 15) -2016-05-23 22:37:34 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 14). 914 bytes result sent to driver -2016-05-23 22:37:34 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 16). 926 bytes result sent to driver -2016-05-23 22:37:34 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 15). 914 bytes result sent to driver -2016-05-23 22:37:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 14) in 11 ms on localhost (1/3) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 15) in 12 ms on localhost (2/3) -2016-05-23 22:37:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 16) in 12 ms on localhost (3/3) -2016-05-23 22:37:34 INFO DAGScheduler:59 - ResultStage 5 (collect at StratifiedSamplingLearning.scala:46) finished in 0.015 s -2016-05-23 22:37:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-23 22:37:34 INFO DAGScheduler:59 - Job 5 finished: collect at StratifiedSamplingLearning.scala:46, took 0.027102 s -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 22:37:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 22:37:34 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 22:37:34 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 22:37:35 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 22:37:35 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 22:37:35 INFO BlockManager:59 - BlockManager stopped -2016-05-23 22:37:35 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 22:37:35 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 22:37:35 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 22:37:35 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 22:37:35 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-21ab0075-8fe4-4f2b-a700-c1523328c085 -2016-05-23 22:37:35 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 22:49:32 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 22:49:33 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 22:49:33 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 22:49:33 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 22:49:33 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 22:49:34 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 22:49:34 INFO Remoting:74 - Starting remoting -2016-05-23 22:49:34 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:54109] -2016-05-23 22:49:34 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54109. -2016-05-23 22:49:34 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 22:49:34 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 22:49:34 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-7522b10f-dd3e-4e46-ab49-bd4783bbc554 -2016-05-23 22:49:34 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 22:49:34 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-8893e165-d67c-4ff0-b8f0-660e662e9a32\httpd-b05d122e-040b-4e24-a233-7c4c8462816f -2016-05-23 22:49:34 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 22:49:34 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:49:34 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54110 -2016-05-23 22:49:34 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54110. -2016-05-23 22:49:35 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 22:49:35 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:49:35 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 22:49:35 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 22:49:35 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 22:49:35 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 22:49:35 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 22:49:36 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54129. -2016-05-23 22:49:36 INFO NettyBlockTransferService:59 - Server created on 54129 -2016-05-23 22:49:36 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 22:49:36 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54129 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54129) -2016-05-23 22:49:36 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 22:49:36 INFO SparkContext:59 - Starting job: foreach at RandomRDDLearning.scala:21 -2016-05-23 22:49:36 INFO DAGScheduler:59 - Got job 0 (foreach at RandomRDDLearning.scala:21) with 2 output partitions -2016-05-23 22:49:36 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at RandomRDDLearning.scala:21) -2016-05-23 22:49:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:49:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:49:36 INFO DAGScheduler:59 - Submitting ResultStage 0 (RandomRDD[0] at RDD at RandomRDD.scala:38), which has no missing parents -2016-05-23 22:49:36 INFO MemoryStore:59 - ensureFreeSpace(1096) called with curMem=0, maxMem=766075207 -2016-05-23 22:49:36 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1096.0 B, free 730.6 MB) -2016-05-23 22:49:36 INFO MemoryStore:59 - ensureFreeSpace(827) called with curMem=1096, maxMem=766075207 -2016-05-23 22:49:36 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 827.0 B, free 730.6 MB) -2016-05-23 22:49:36 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54129 (size: 827.0 B, free: 730.6 MB) -2016-05-23 22:49:36 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:49:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (RandomRDD[0] at RDD at RandomRDD.scala:38) -2016-05-23 22:49:37 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 22:49:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2287 bytes) -2016-05-23 22:49:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2287 bytes) -2016-05-23 22:49:37 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 22:49:37 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 22:49:37 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 915 bytes result sent to driver -2016-05-23 22:49:37 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 915 bytes result sent to driver -2016-05-23 22:49:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 51 ms on localhost (1/2) -2016-05-23 22:49:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 74 ms on localhost (2/2) -2016-05-23 22:49:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 22:49:37 INFO DAGScheduler:59 - ResultStage 0 (foreach at RandomRDDLearning.scala:21) finished in 0.085 s -2016-05-23 22:49:37 INFO DAGScheduler:59 - Job 0 finished: foreach at RandomRDDLearning.scala:21, took 0.294338 s -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 22:49:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 22:49:37 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 22:49:37 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 22:49:37 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 22:49:37 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 22:49:37 INFO BlockManager:59 - BlockManager stopped -2016-05-23 22:49:37 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 22:49:37 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 22:49:37 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 22:49:37 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 22:49:37 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-8893e165-d67c-4ff0-b8f0-660e662e9a32 -2016-05-23 22:49:37 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 22:52:57 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 22:52:58 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 22:52:58 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 22:52:58 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 22:52:58 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 22:52:59 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 22:52:59 INFO Remoting:74 - Starting remoting -2016-05-23 22:52:59 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:54204] -2016-05-23 22:52:59 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54204. -2016-05-23 22:52:59 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 22:52:59 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 22:52:59 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-242d4a35-76c9-4594-b633-51e94eac83d6 -2016-05-23 22:52:59 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 22:52:59 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-ed36b233-73a5-4dca-bbc6-8879ca44e64c\httpd-83f09ad9-58bb-4ac2-9d18-e2cbdbcd1926 -2016-05-23 22:52:59 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 22:52:59 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:52:59 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54205 -2016-05-23 22:52:59 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54205. -2016-05-23 22:52:59 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 22:52:59 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:52:59 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 22:52:59 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 22:52:59 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 22:53:00 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 22:53:00 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 22:53:00 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54224. -2016-05-23 22:53:00 INFO NettyBlockTransferService:59 - Server created on 54224 -2016-05-23 22:53:00 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 22:53:00 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54224 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54224) -2016-05-23 22:53:00 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 22:53:00 INFO SparkContext:59 - Starting job: foreach at RandomRDDLearning.scala:21 -2016-05-23 22:53:00 INFO DAGScheduler:59 - Got job 0 (foreach at RandomRDDLearning.scala:21) with 2 output partitions -2016-05-23 22:53:00 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at RandomRDDLearning.scala:21) -2016-05-23 22:53:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:53:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:53:00 INFO DAGScheduler:59 - Submitting ResultStage 0 (RandomRDD[0] at RDD at RandomRDD.scala:38), which has no missing parents -2016-05-23 22:53:01 INFO MemoryStore:59 - ensureFreeSpace(1096) called with curMem=0, maxMem=766075207 -2016-05-23 22:53:01 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1096.0 B, free 730.6 MB) -2016-05-23 22:53:01 INFO MemoryStore:59 - ensureFreeSpace(827) called with curMem=1096, maxMem=766075207 -2016-05-23 22:53:01 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 827.0 B, free 730.6 MB) -2016-05-23 22:53:01 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54224 (size: 827.0 B, free: 730.6 MB) -2016-05-23 22:53:01 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:53:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (RandomRDD[0] at RDD at RandomRDD.scala:38) -2016-05-23 22:53:01 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 22:53:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2287 bytes) -2016-05-23 22:53:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2287 bytes) -2016-05-23 22:53:01 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 22:53:01 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 22:53:01 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 915 bytes result sent to driver -2016-05-23 22:53:01 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 915 bytes result sent to driver -2016-05-23 22:53:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 71 ms on localhost (1/2) -2016-05-23 22:53:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 53 ms on localhost (2/2) -2016-05-23 22:53:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 22:53:01 INFO DAGScheduler:59 - ResultStage 0 (foreach at RandomRDDLearning.scala:21) finished in 0.085 s -2016-05-23 22:53:01 INFO DAGScheduler:59 - Job 0 finished: foreach at RandomRDDLearning.scala:21, took 0.330831 s -2016-05-23 22:53:01 INFO SparkContext:59 - Starting job: foreach at RandomRDDLearning.scala:23 -2016-05-23 22:53:01 INFO DAGScheduler:59 - Got job 1 (foreach at RandomRDDLearning.scala:23) with 2 output partitions -2016-05-23 22:53:01 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at RandomRDDLearning.scala:23) -2016-05-23 22:53:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:53:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:53:01 INFO DAGScheduler:59 - Submitting ResultStage 1 (RandomRDD[1] at RDD at RandomRDD.scala:38), which has no missing parents -2016-05-23 22:53:01 INFO MemoryStore:59 - ensureFreeSpace(1096) called with curMem=1923, maxMem=766075207 -2016-05-23 22:53:01 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1096.0 B, free 730.6 MB) -2016-05-23 22:53:01 INFO MemoryStore:59 - ensureFreeSpace(827) called with curMem=3019, maxMem=766075207 -2016-05-23 22:53:01 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 827.0 B, free 730.6 MB) -2016-05-23 22:53:01 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54224 (size: 827.0 B, free: 730.6 MB) -2016-05-23 22:53:01 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:53:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (RandomRDD[1] at RDD at RandomRDD.scala:38) -2016-05-23 22:53:01 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 22:53:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 22:53:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 22:53:01 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 22:53:01 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 22:53:01 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 915 bytes result sent to driver -2016-05-23 22:53:01 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 915 bytes result sent to driver -2016-05-23 22:53:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 16 ms on localhost (1/2) -2016-05-23 22:53:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 18 ms on localhost (2/2) -2016-05-23 22:53:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 22:53:01 INFO DAGScheduler:59 - ResultStage 1 (foreach at RandomRDDLearning.scala:23) finished in 0.019 s -2016-05-23 22:53:01 INFO DAGScheduler:59 - Job 1 finished: foreach at RandomRDDLearning.scala:23, took 0.031972 s -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 22:53:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 22:53:01 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 22:53:01 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 22:53:01 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 22:53:01 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 22:53:01 INFO BlockManager:59 - BlockManager stopped -2016-05-23 22:53:01 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 22:53:01 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 22:53:01 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 22:53:01 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 22:53:01 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-ed36b233-73a5-4dca-bbc6-8879ca44e64c -2016-05-23 22:54:28 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-23 22:54:29 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-23 22:54:29 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-23 22:54:29 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-23 22:54:29 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-23 22:54:30 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-23 22:54:30 INFO Remoting:74 - Starting remoting -2016-05-23 22:54:30 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:54241] -2016-05-23 22:54:30 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54241. -2016-05-23 22:54:30 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-23 22:54:30 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-23 22:54:30 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-33d700c3-38d8-4182-90a9-53d5b58b0246 -2016-05-23 22:54:30 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-23 22:54:30 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-d9d8df54-35c6-4686-9a1a-4b5312fb67e5\httpd-27a2db80-0999-4d40-9521-eed77fafc8c8 -2016-05-23 22:54:30 INFO HttpServer:59 - Starting HTTP Server -2016-05-23 22:54:30 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:54:30 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54242 -2016-05-23 22:54:30 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54242. -2016-05-23 22:54:30 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-23 22:54:30 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-23 22:54:30 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-23 22:54:30 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-23 22:54:30 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-23 22:54:30 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-23 22:54:30 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-23 22:54:31 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54261. -2016-05-23 22:54:31 INFO NettyBlockTransferService:59 - Server created on 54261 -2016-05-23 22:54:31 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-23 22:54:31 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54261 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54261) -2016-05-23 22:54:31 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-23 22:54:31 INFO SparkContext:59 - Starting job: foreach at RandomRDDLearning.scala:21 -2016-05-23 22:54:31 INFO DAGScheduler:59 - Got job 0 (foreach at RandomRDDLearning.scala:21) with 2 output partitions -2016-05-23 22:54:31 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at RandomRDDLearning.scala:21) -2016-05-23 22:54:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:54:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:54:31 INFO DAGScheduler:59 - Submitting ResultStage 0 (RandomRDD[0] at RDD at RandomRDD.scala:38), which has no missing parents -2016-05-23 22:54:31 INFO MemoryStore:59 - ensureFreeSpace(1096) called with curMem=0, maxMem=766075207 -2016-05-23 22:54:31 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1096.0 B, free 730.6 MB) -2016-05-23 22:54:31 INFO MemoryStore:59 - ensureFreeSpace(827) called with curMem=1096, maxMem=766075207 -2016-05-23 22:54:31 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 827.0 B, free 730.6 MB) -2016-05-23 22:54:31 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54261 (size: 827.0 B, free: 730.6 MB) -2016-05-23 22:54:31 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:54:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (RandomRDD[0] at RDD at RandomRDD.scala:38) -2016-05-23 22:54:31 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-23 22:54:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2287 bytes) -2016-05-23 22:54:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2287 bytes) -2016-05-23 22:54:31 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-23 22:54:31 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-23 22:54:32 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 915 bytes result sent to driver -2016-05-23 22:54:32 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 915 bytes result sent to driver -2016-05-23 22:54:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 71 ms on localhost (1/2) -2016-05-23 22:54:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 95 ms on localhost (2/2) -2016-05-23 22:54:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-23 22:54:32 INFO DAGScheduler:59 - ResultStage 0 (foreach at RandomRDDLearning.scala:21) finished in 0.108 s -2016-05-23 22:54:32 INFO DAGScheduler:59 - Job 0 finished: foreach at RandomRDDLearning.scala:21, took 0.313634 s -2016-05-23 22:54:32 INFO SparkContext:59 - Starting job: foreach at RandomRDDLearning.scala:23 -2016-05-23 22:54:32 INFO DAGScheduler:59 - Got job 1 (foreach at RandomRDDLearning.scala:23) with 2 output partitions -2016-05-23 22:54:32 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at RandomRDDLearning.scala:23) -2016-05-23 22:54:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:54:32 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:54:32 INFO DAGScheduler:59 - Submitting ResultStage 1 (RandomRDD[1] at RDD at RandomRDD.scala:38), which has no missing parents -2016-05-23 22:54:32 INFO MemoryStore:59 - ensureFreeSpace(1096) called with curMem=1923, maxMem=766075207 -2016-05-23 22:54:32 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1096.0 B, free 730.6 MB) -2016-05-23 22:54:32 INFO MemoryStore:59 - ensureFreeSpace(827) called with curMem=3019, maxMem=766075207 -2016-05-23 22:54:32 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 827.0 B, free 730.6 MB) -2016-05-23 22:54:32 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54261 (size: 827.0 B, free: 730.6 MB) -2016-05-23 22:54:32 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:54:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (RandomRDD[1] at RDD at RandomRDD.scala:38) -2016-05-23 22:54:32 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-23 22:54:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 22:54:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-23 22:54:32 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-23 22:54:32 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-23 22:54:32 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 915 bytes result sent to driver -2016-05-23 22:54:32 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 915 bytes result sent to driver -2016-05-23 22:54:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 11 ms on localhost (1/2) -2016-05-23 22:54:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 13 ms on localhost (2/2) -2016-05-23 22:54:32 INFO DAGScheduler:59 - ResultStage 1 (foreach at RandomRDDLearning.scala:23) finished in 0.014 s -2016-05-23 22:54:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-23 22:54:32 INFO DAGScheduler:59 - Job 1 finished: foreach at RandomRDDLearning.scala:23, took 0.025057 s -2016-05-23 22:54:32 INFO SparkContext:59 - Starting job: foreach at RandomRDDLearning.scala:25 -2016-05-23 22:54:32 INFO DAGScheduler:59 - Got job 2 (foreach at RandomRDDLearning.scala:25) with 2 output partitions -2016-05-23 22:54:32 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at RandomRDDLearning.scala:25) -2016-05-23 22:54:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:54:32 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:54:32 INFO DAGScheduler:59 - Submitting ResultStage 2 (RandomRDD[2] at RDD at RandomRDD.scala:38), which has no missing parents -2016-05-23 22:54:32 INFO MemoryStore:59 - ensureFreeSpace(1096) called with curMem=3846, maxMem=766075207 -2016-05-23 22:54:32 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 1096.0 B, free 730.6 MB) -2016-05-23 22:54:32 INFO MemoryStore:59 - ensureFreeSpace(827) called with curMem=4942, maxMem=766075207 -2016-05-23 22:54:32 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 827.0 B, free 730.6 MB) -2016-05-23 22:54:32 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54261 (size: 827.0 B, free: 730.6 MB) -2016-05-23 22:54:32 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:54:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (RandomRDD[2] at RDD at RandomRDD.scala:38) -2016-05-23 22:54:32 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-23 22:54:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 18703 bytes) -2016-05-23 22:54:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 18703 bytes) -2016-05-23 22:54:32 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-23 22:54:32 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-23 22:54:32 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 915 bytes result sent to driver -2016-05-23 22:54:32 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 915 bytes result sent to driver -2016-05-23 22:54:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 11 ms on localhost (1/2) -2016-05-23 22:54:32 INFO DAGScheduler:59 - ResultStage 2 (foreach at RandomRDDLearning.scala:25) finished in 0.019 s -2016-05-23 22:54:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 18 ms on localhost (2/2) -2016-05-23 22:54:32 INFO DAGScheduler:59 - Job 2 finished: foreach at RandomRDDLearning.scala:25, took 0.043043 s -2016-05-23 22:54:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-23 22:54:32 INFO SparkContext:59 - Starting job: foreach at RandomRDDLearning.scala:27 -2016-05-23 22:54:32 INFO DAGScheduler:59 - Got job 3 (foreach at RandomRDDLearning.scala:27) with 2 output partitions -2016-05-23 22:54:32 INFO DAGScheduler:59 - Final stage: ResultStage 3(foreach at RandomRDDLearning.scala:27) -2016-05-23 22:54:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:54:32 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:54:32 INFO DAGScheduler:59 - Submitting ResultStage 3 (RandomRDD[3] at RDD at RandomRDD.scala:38), which has no missing parents -2016-05-23 22:54:32 INFO MemoryStore:59 - ensureFreeSpace(1096) called with curMem=5769, maxMem=766075207 -2016-05-23 22:54:32 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 1096.0 B, free 730.6 MB) -2016-05-23 22:54:32 INFO MemoryStore:59 - ensureFreeSpace(827) called with curMem=6865, maxMem=766075207 -2016-05-23 22:54:32 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 827.0 B, free 730.6 MB) -2016-05-23 22:54:32 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54261 (size: 827.0 B, free: 730.6 MB) -2016-05-23 22:54:32 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:54:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (RandomRDD[3] at RDD at RandomRDD.scala:38) -2016-05-23 22:54:32 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-23 22:54:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 18078 bytes) -2016-05-23 22:54:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 18078 bytes) -2016-05-23 22:54:32 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-23 22:54:32 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-23 22:54:32 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 915 bytes result sent to driver -2016-05-23 22:54:32 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 915 bytes result sent to driver -2016-05-23 22:54:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 32 ms on localhost (1/2) -2016-05-23 22:54:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 24 ms on localhost (2/2) -2016-05-23 22:54:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-23 22:54:32 INFO DAGScheduler:59 - ResultStage 3 (foreach at RandomRDDLearning.scala:27) finished in 0.035 s -2016-05-23 22:54:32 INFO DAGScheduler:59 - Job 3 finished: foreach at RandomRDDLearning.scala:27, took 0.169966 s -2016-05-23 22:54:32 INFO SparkContext:59 - Starting job: foreach at RandomRDDLearning.scala:29 -2016-05-23 22:54:32 INFO DAGScheduler:59 - Got job 4 (foreach at RandomRDDLearning.scala:29) with 2 output partitions -2016-05-23 22:54:32 INFO DAGScheduler:59 - Final stage: ResultStage 4(foreach at RandomRDDLearning.scala:29) -2016-05-23 22:54:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-23 22:54:32 INFO DAGScheduler:59 - Missing parents: List() -2016-05-23 22:54:32 INFO DAGScheduler:59 - Submitting ResultStage 4 (RandomRDD[4] at RDD at RandomRDD.scala:38), which has no missing parents -2016-05-23 22:54:32 INFO MemoryStore:59 - ensureFreeSpace(1096) called with curMem=7692, maxMem=766075207 -2016-05-23 22:54:32 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 1096.0 B, free 730.6 MB) -2016-05-23 22:54:32 INFO MemoryStore:59 - ensureFreeSpace(827) called with curMem=8788, maxMem=766075207 -2016-05-23 22:54:32 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 827.0 B, free 730.6 MB) -2016-05-23 22:54:32 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54261 (size: 827.0 B, free: 730.6 MB) -2016-05-23 22:54:32 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-23 22:54:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (RandomRDD[4] at RDD at RandomRDD.scala:38) -2016-05-23 22:54:32 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-23 22:54:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 18250 bytes) -2016-05-23 22:54:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 18250 bytes) -2016-05-23 22:54:32 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-23 22:54:32 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-23 22:54:32 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 915 bytes result sent to driver -2016-05-23 22:54:32 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 915 bytes result sent to driver -2016-05-23 22:54:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 15 ms on localhost (1/2) -2016-05-23 22:54:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 11 ms on localhost (2/2) -2016-05-23 22:54:32 INFO DAGScheduler:59 - ResultStage 4 (foreach at RandomRDDLearning.scala:29) finished in 0.016 s -2016-05-23 22:54:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-23 22:54:32 INFO DAGScheduler:59 - Job 4 finished: foreach at RandomRDDLearning.scala:29, took 0.032609 s -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-23 22:54:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-23 22:54:32 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-23 22:54:32 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-23 22:54:32 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-23 22:54:32 INFO MemoryStore:59 - MemoryStore cleared -2016-05-23 22:54:32 INFO BlockManager:59 - BlockManager stopped -2016-05-23 22:54:32 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-23 22:54:32 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-23 22:54:32 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-23 22:54:32 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-23 22:54:32 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-d9d8df54-35c6-4686-9a1a-4b5312fb67e5 -2016-05-23 22:54:32 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-23 22:54:32 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 16:00:04 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 16:00:17 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 16:00:19 INFO Remoting:74 - Starting remoting -2016-05-24 16:00:22 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:49262] -2016-05-24 16:00:26 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 16:00:26 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:49263 -2016-05-24 16:00:27 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 16:00:27 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 16:00:46 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 16:00:46 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 16:00:49 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 16:00:49 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 16:00:49 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 16:00:49 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 16:00:49 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 16:00:54 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 16:00:54 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 16:01:03 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605241601_0079_m_000000_79' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/LinearRegression/20160524160101539/result/_temporary/0/task_201605241601_0079_m_000000 -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 16:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 16:01:04 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 16:01:04 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 16:07:19 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 16:07:20 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 16:07:20 INFO Remoting:74 - Starting remoting -2016-05-24 16:07:20 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:49311] -2016-05-24 16:07:21 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 16:07:21 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:49312 -2016-05-24 16:07:21 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 16:07:21 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 16:07:24 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 16:07:24 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 16:07:24 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 16:07:24 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 16:07:24 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 16:07:24 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 16:07:24 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 16:07:25 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 16:07:25 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 16:07:38 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605241607_0079_m_000000_79' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/LinearRegression/20160524160729870/result/_temporary/0/task_201605241607_0079_m_000000 -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 16:07:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 16:07:39 WARN QueuedThreadPool:145 - 6 threads could not be stopped -2016-05-24 16:07:41 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 16:07:41 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 17:10:01 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 17:10:06 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 17:10:07 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 17:10:07 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 17:10:07 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 17:10:12 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 17:10:12 INFO Remoting:74 - Starting remoting -2016-05-24 17:10:13 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:50848] -2016-05-24 17:10:13 INFO Utils:59 - Successfully started service 'sparkDriver' on port 50848. -2016-05-24 17:10:13 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 17:10:13 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 17:10:13 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-cfad4305-1cd3-4331-baf2-4b0b67f4f82d -2016-05-24 17:10:13 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 17:10:14 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-9ee011be-041c-401b-b79f-15bd1e12ef00\httpd-e243afac-f1ad-40e4-8d14-98d4d0311f55 -2016-05-24 17:10:14 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 17:10:14 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 17:10:14 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:50849 -2016-05-24 17:10:14 INFO Utils:59 - Successfully started service 'HTTP file server' on port 50849. -2016-05-24 17:10:14 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 17:10:15 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 17:10:15 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 17:10:15 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 17:10:15 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 17:10:15 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 17:10:15 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 17:10:16 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 50868. -2016-05-24 17:10:16 INFO NettyBlockTransferService:59 - Server created on 50868 -2016-05-24 17:10:16 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 17:10:16 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:50868 with 730.6 MB RAM, BlockManagerId(driver, localhost, 50868) -2016-05-24 17:10:16 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 17:10:19 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 17:10:19 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 17:10:19 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 17:10:19 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 17:10:19 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:50868 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 17:10:19 INFO SparkContext:59 - Created broadcast 0 from textFile at LinearRegression2Learning.scala:21 -2016-05-24 17:10:21 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 17:10:21 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 17:10:22 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 17:10:22 INFO DAGScheduler:59 - Got job 0 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 17:10:22 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 17:10:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:22 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 17:10:22 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=171580, maxMem=766075207 -2016-05-24 17:10:22 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-24 17:10:22 INFO MemoryStore:59 - ensureFreeSpace(2182) called with curMem=175316, maxMem=766075207 -2016-05-24 17:10:22 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 17:10:22 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:50868 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 17:10:22 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 17:10:22 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-24 17:10:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 17:10:22 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 17:10:22 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 17:10:22 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/ridge-data/lpsa2.data:0+19 -2016-05-24 17:10:22 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 17:10:22 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 17:10:22 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 17:10:22 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 17:10:22 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 17:10:23 INFO MemoryStore:59 - ensureFreeSpace(320) called with curMem=177498, maxMem=766075207 -2016-05-24 17:10:23 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 320.0 B, free 730.4 MB) -2016-05-24 17:10:23 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:50868 (size: 320.0 B, free: 730.6 MB) -2016-05-24 17:10:23 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2611 bytes result sent to driver -2016-05-24 17:10:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 449 ms on localhost (1/1) -2016-05-24 17:10:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 17:10:23 INFO DAGScheduler:59 - ResultStage 0 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.492 s -2016-05-24 17:10:23 INFO DAGScheduler:59 - Job 0 finished: first at GeneralizedLinearAlgorithm.scala:206, took 1.125480 s -2016-05-24 17:10:23 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 17:10:23 INFO DAGScheduler:59 - Got job 1 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 17:10:23 INFO DAGScheduler:59 - Final stage: ResultStage 1(count at GradientDescent.scala:195) -2016-05-24 17:10:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:23 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 17:10:23 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=177818, maxMem=766075207 -2016-05-24 17:10:23 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 17:10:23 INFO MemoryStore:59 - ensureFreeSpace(2020) called with curMem=181258, maxMem=766075207 -2016-05-24 17:10:23 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2020.0 B, free 730.4 MB) -2016-05-24 17:10:23 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:50868 (size: 2020.0 B, free: 730.6 MB) -2016-05-24 17:10:23 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 17:10:23 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 17:10:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 17:10:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 17:10:23 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-24 17:10:23 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-24 17:10:23 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 17:10:23 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/ridge-data/lpsa2.data:19+19 -2016-05-24 17:10:23 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:23 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-05-24 17:10:23 INFO MemoryStore:59 - ensureFreeSpace(168) called with curMem=183278, maxMem=766075207 -2016-05-24 17:10:23 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 168.0 B, free 730.4 MB) -2016-05-24 17:10:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 61 ms on localhost (1/2) -2016-05-24 17:10:23 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:50868 (size: 168.0 B, free: 730.6 MB) -2016-05-24 17:10:23 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2662 bytes result sent to driver -2016-05-24 17:10:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 71 ms on localhost (2/2) -2016-05-24 17:10:23 INFO DAGScheduler:59 - ResultStage 1 (count at GradientDescent.scala:195) finished in 0.078 s -2016-05-24 17:10:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 17:10:23 INFO DAGScheduler:59 - Job 1 finished: count at GradientDescent.scala:195, took 0.157090 s -2016-05-24 17:10:24 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 17:10:24 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:50868 in memory (size: 2020.0 B, free: 730.6 MB) -2016-05-24 17:10:24 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:50868 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 17:10:24 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 17:10:26 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 17:10:26 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 17:10:26 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=172068, maxMem=766075207 -2016-05-24 17:10:26 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:26 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=172140, maxMem=766075207 -2016-05-24 17:10:26 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:26 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:26 INFO SparkContext:59 - Created broadcast 3 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:26 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:26 INFO DAGScheduler:59 - Got job 2 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:26 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:26 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:26 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=172278, maxMem=766075207 -2016-05-24 17:10:26 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:26 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=177798, maxMem=766075207 -2016-05-24 17:10:26 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:26 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:26 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:26 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 17:10:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:26 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 17:10:26 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 17:10:26 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:26 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:26 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2423 bytes result sent to driver -2016-05-24 17:10:26 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2423 bytes result sent to driver -2016-05-24 17:10:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 40 ms on localhost (1/2) -2016-05-24 17:10:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 39 ms on localhost (2/2) -2016-05-24 17:10:26 INFO DAGScheduler:59 - ResultStage 2 (treeAggregate at GradientDescent.scala:225) finished in 0.041 s -2016-05-24 17:10:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 17:10:26 INFO DAGScheduler:59 - Job 2 finished: treeAggregate at GradientDescent.scala:225, took 0.083671 s -2016-05-24 17:10:26 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=180769, maxMem=766075207 -2016-05-24 17:10:26 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:26 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=180841, maxMem=766075207 -2016-05-24 17:10:26 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:26 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:26 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:26 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:26 INFO DAGScheduler:59 - Got job 3 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:26 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:26 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:26 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=180979, maxMem=766075207 -2016-05-24 17:10:26 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:26 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=186499, maxMem=766075207 -2016-05-24 17:10:26 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:26 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:26 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:26 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 17:10:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:26 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 17:10:26 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 17:10:26 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:26 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2423 bytes result sent to driver -2016-05-24 17:10:26 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:26 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2423 bytes result sent to driver -2016-05-24 17:10:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 26 ms on localhost (1/2) -2016-05-24 17:10:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 30 ms on localhost (2/2) -2016-05-24 17:10:26 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at GradientDescent.scala:225) finished in 0.030 s -2016-05-24 17:10:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 17:10:26 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at GradientDescent.scala:225, took 0.054038 s -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=189471, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:27 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=189543, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:27 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:27 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 17:10:27 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:27 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:27 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:27 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[10] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=172488, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=178008, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:27 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[10] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:27 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 17:10:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:27 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 17:10:27 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 17:10:27 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:27 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2423 bytes result sent to driver -2016-05-24 17:10:27 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:27 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2423 bytes result sent to driver -2016-05-24 17:10:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 30 ms on localhost (1/2) -2016-05-24 17:10:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 32 ms on localhost (2/2) -2016-05-24 17:10:27 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.035 s -2016-05-24 17:10:27 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.084760 s -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=180980, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=181052, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:27 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:27 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:27 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:27 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:27 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[12] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=181190, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=186710, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:27 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[12] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:27 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 17:10:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:27 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 17:10:27 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 17:10:27 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 17:10:27 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:27 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:27 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2423 bytes result sent to driver -2016-05-24 17:10:27 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2423 bytes result sent to driver -2016-05-24 17:10:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 25 ms on localhost (1/2) -2016-05-24 17:10:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 27 ms on localhost (2/2) -2016-05-24 17:10:27 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.031 s -2016-05-24 17:10:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 17:10:27 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.080937 s -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=180977, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=181049, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:27 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:27 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:27 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:27 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:27 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[14] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=181187, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=186707, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:27 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[14] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:27 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 17:10:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:27 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 17:10:27 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 17:10:27 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:27 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:27 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2423 bytes result sent to driver -2016-05-24 17:10:27 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2423 bytes result sent to driver -2016-05-24 17:10:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 21 ms on localhost (1/2) -2016-05-24 17:10:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 24 ms on localhost (2/2) -2016-05-24 17:10:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 17:10:27 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.027 s -2016-05-24 17:10:27 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.051629 s -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=189678, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=189750, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:27 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:27 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:27 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:27 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:27 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:27 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[16] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=181187, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=186707, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:27 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[16] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:27 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 17:10:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:27 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 17:10:27 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 17:10:27 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:27 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2423 bytes result sent to driver -2016-05-24 17:10:27 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 18 ms on localhost (1/2) -2016-05-24 17:10:27 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2423 bytes result sent to driver -2016-05-24 17:10:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 32 ms on localhost (2/2) -2016-05-24 17:10:27 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.033 s -2016-05-24 17:10:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 17:10:27 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.059611 s -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=181188, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=181260, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:27 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:27 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:27 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:27 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:27 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[18] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=181398, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=186918, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:27 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[18] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:27 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 17:10:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:27 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 17:10:27 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 17:10:27 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:27 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:27 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2423 bytes result sent to driver -2016-05-24 17:10:27 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2423 bytes result sent to driver -2016-05-24 17:10:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 24 ms on localhost (1/2) -2016-05-24 17:10:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 22 ms on localhost (2/2) -2016-05-24 17:10:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 17:10:27 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.025 s -2016-05-24 17:10:27 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.049360 s -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=189889, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=189961, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:27 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:27 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:27 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:27 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:27 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[20] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=190099, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=195619, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:27 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[20] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:27 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 17:10:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:27 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 17:10:27 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 17:10:27 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:27 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2423 bytes result sent to driver -2016-05-24 17:10:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 22 ms on localhost (1/2) -2016-05-24 17:10:27 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:27 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2423 bytes result sent to driver -2016-05-24 17:10:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 40 ms on localhost (2/2) -2016-05-24 17:10:27 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.042 s -2016-05-24 17:10:27 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.067536 s -2016-05-24 17:10:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=198590, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=198662, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:27 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:27 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 17:10:27 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:27 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:27 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:27 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[22] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=181609, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=184158, maxMem=766075207 -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:27 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[22] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:27 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 17:10:27 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 17:10:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:27 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 17:10:27 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 17:10:27 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 17:10:27 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:27 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:27 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2423 bytes result sent to driver -2016-05-24 17:10:27 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2423 bytes result sent to driver -2016-05-24 17:10:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 22 ms on localhost (1/2) -2016-05-24 17:10:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 22 ms on localhost (2/2) -2016-05-24 17:10:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 17:10:27 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.025 s -2016-05-24 17:10:27 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.069896 s -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=181400, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=181472, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:27 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:27 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:27 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:27 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:27 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[24] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=181610, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=187130, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:27 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[24] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:27 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 17:10:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:27 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 17:10:27 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 17:10:27 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:27 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:27 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2423 bytes result sent to driver -2016-05-24 17:10:27 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2423 bytes result sent to driver -2016-05-24 17:10:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 21 ms on localhost (1/2) -2016-05-24 17:10:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 21 ms on localhost (2/2) -2016-05-24 17:10:27 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 17:10:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 17:10:27 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.047265 s -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=190100, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=190172, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:27 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:27 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:27 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:27 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:27 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[26] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=190310, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:27 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=195830, maxMem=766075207 -2016-05-24 17:10:27 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:27 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:27 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[26] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:27 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 17:10:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:27 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 17:10:27 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:28 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 17 ms on localhost (1/2) -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 20 ms on localhost (2/2) -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 17:10:28 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 17:10:28 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.049351 s -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=198802, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=198874, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:28 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[28] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:28 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=181820, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(2973) called with curMem=187202, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[28] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 17:10:28 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:28 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 23 ms on localhost (1/2) -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 29 ms on localhost (2/2) -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 17:10:28 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.032 s -2016-05-24 17:10:28 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.055862 s -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=181611, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=181683, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:28 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[30] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=181611, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(2976) called with curMem=187131, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[30] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:28 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 17 ms on localhost (1/2) -2016-05-24 17:10:28 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:28 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 34 ms on localhost (2/2) -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 17:10:28 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.037 s -2016-05-24 17:10:28 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.056808 s -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=190107, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=190179, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:28 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:28 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[32] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=190317, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=195837, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[32] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 17:10:28 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:28 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 19 ms on localhost (1/2) -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 23 ms on localhost (2/2) -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 17:10:28 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 17:10:28 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.044861 s -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=198808, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=198880, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:28 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:28 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[34] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=199018, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=204538, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[34] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 17:10:28 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:28 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 18 ms on localhost (1/2) -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 23 ms on localhost (2/2) -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 17:10:28 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.025 s -2016-05-24 17:10:28 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.044701 s -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=207510, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=207582, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:28 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:28 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[36] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=207720, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=213240, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[36] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:28 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 21 ms on localhost (1/2) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:28 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 43 ms on localhost (2/2) -2016-05-24 17:10:28 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.045 s -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 17:10:28 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.082925 s -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=198595, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=198667, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:28 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:28 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[38] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=181610, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=187130, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[38] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:28 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 19 ms on localhost (1/2) -2016-05-24 17:10:28 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:28 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 43 ms on localhost (2/2) -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 17:10:28 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.046 s -2016-05-24 17:10:28 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.101882 s -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=190101, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=190173, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:28 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:28 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[40] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=190311, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=195831, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[40] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:28 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:28 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 29 ms on localhost (1/2) -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 35 ms on localhost (2/2) -2016-05-24 17:10:28 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.036 s -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 17:10:28 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.065743 s -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=198803, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=198875, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:28 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:28 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[42] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=199013, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=204533, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[42] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 17:10:28 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:28 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 26 ms on localhost (1/2) -2016-05-24 17:10:28 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 32 ms on localhost (2/2) -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 17:10:28 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.033 s -2016-05-24 17:10:28 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.047829 s -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=207504, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=207576, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:28 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:28 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[44] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=207714, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=213234, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[44] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:28 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 16 ms on localhost (1/2) -2016-05-24 17:10:28 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:28 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 36 ms on localhost (2/2) -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 17:10:28 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.038 s -2016-05-24 17:10:28 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.056087 s -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=216206, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=216278, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:28 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[46] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=207924, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=204952, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[46] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:28 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 15 ms on localhost (1/2) -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:28 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 2423 bytes result sent to driver -2016-05-24 17:10:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 27 ms on localhost (2/2) -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 17:10:28 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.029 s -2016-05-24 17:10:28 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.048525 s -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=207714, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=207786, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:28 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:28 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[48] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=204953, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=204953, maxMem=766075207 -2016-05-24 17:10:28 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:28 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[48] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:28 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:28 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:28 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:28 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:28 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:28 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 12 ms on localhost (1/2) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 15 ms on localhost (2/2) -2016-05-24 17:10:29 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 17:10:29 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.042012 s -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=199084, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=199156, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 17:10:29 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:29 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[50] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=190521, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=196041, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[50] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:29 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 20 ms on localhost (1/2) -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:29 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 25 ms on localhost (2/2) -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 17:10:29 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.027 s -2016-05-24 17:10:29 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.043631 s -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=199012, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=199084, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:29 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:29 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[52] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=199222, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=204742, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[52] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 17:10:29 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:29 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:29 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 25 ms on localhost (1/2) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 25 ms on localhost (2/2) -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 17:10:29 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.001 s -2016-05-24 17:10:29 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.230533 s -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=207711, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=207783, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:29 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:29 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[54] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=207921, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=213441, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[54] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 17:10:29 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:29 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:29 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 21 ms on localhost (1/2) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 23 ms on localhost (2/2) -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 17:10:29 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 17:10:29 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.039212 s -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=216410, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=216482, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:29 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:29 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[56] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=216620, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=222140, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[56] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 17:10:29 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:29 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 10 ms on localhost (1/2) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 12 ms on localhost (2/2) -2016-05-24 17:10:29 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 17:10:29 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.026358 s -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=225112, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=225184, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:29 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:29 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[58] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=225322, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=230842, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[58] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:29 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 19 ms on localhost (1/2) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 17:10:29 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:29 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 43 ms on localhost (2/2) -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 17:10:29 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.045 s -2016-05-24 17:10:29 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.070902 s -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=204742, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=199294, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:29 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:29 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[60] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=190731, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=196251, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[60] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:29 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 15 ms on localhost (1/2) -2016-05-24 17:10:29 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:29 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 36 ms on localhost (2/2) -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 17:10:29 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.039 s -2016-05-24 17:10:29 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.057376 s -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=190311, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=190383, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:29 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:29 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[62] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=190521, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=196041, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[62] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 17:10:29 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:29 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 14 ms on localhost (1/2) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 14 ms on localhost (2/2) -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 17:10:29 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 17:10:29 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.032699 s -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=199012, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=199084, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:29 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:29 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[64] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=199222, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=204742, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[64] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 17:10:29 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:29 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 10 ms on localhost (1/2) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 10 ms on localhost (2/2) -2016-05-24 17:10:29 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 17:10:29 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.024853 s -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=207713, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=207785, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:29 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:29 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[66] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=207923, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=213443, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[66] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 17:10:29 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:29 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 7 ms on localhost (1/2) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 8 ms on localhost (2/2) -2016-05-24 17:10:29 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 17:10:29 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.023030 s -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=216412, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=216484, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:29 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:29 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[68] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=216622, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=222142, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[68] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 17:10:29 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:29 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 10 ms on localhost (1/2) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 10 ms on localhost (2/2) -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 17:10:29 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 17:10:29 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.025291 s -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=225114, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=225186, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:29 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:29 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[70] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=225324, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=230844, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[70] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:29 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 17:10:29 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:29 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:29 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 2423 bytes result sent to driver -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 8 ms on localhost (1/2) -2016-05-24 17:10:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 10 ms on localhost (2/2) -2016-05-24 17:10:29 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:10:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 17:10:29 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.025822 s -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=233816, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233888, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 17:10:29 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:29 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:29 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:29 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[72] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=208133, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:29 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 17:10:29 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=213515, maxMem=766075207 -2016-05-24 17:10:29 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:29 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[72] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:30 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:30 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 12 ms on localhost (1/2) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 15 ms on localhost (2/2) -2016-05-24 17:10:30 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 17:10:30 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.115694 s -2016-05-24 17:10:30 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=207924, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=207996, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 17:10:30 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[74] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=199223, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=196251, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:30 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[74] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 17:10:30 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:30 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:30 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 18 ms on localhost (1/2) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 17:10:30 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.038529 s -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 21 ms on localhost (2/2) -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=190521, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=190593, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:30 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[76] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=190731, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=196251, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[76] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 17:10:30 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:30 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:30 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 7 ms on localhost (1/2) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 9 ms on localhost (2/2) -2016-05-24 17:10:30 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 17:10:30 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.021571 s -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=199222, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=199294, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:30 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[78] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=199432, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=204952, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[78] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 17:10:30 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:30 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 12 ms on localhost (1/2) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 13 ms on localhost (2/2) -2016-05-24 17:10:30 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 17:10:30 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.025259 s -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=207923, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=207995, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:30 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[80] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=208133, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=213653, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[80] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 17:10:30 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:30 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 10 ms on localhost (1/2) -2016-05-24 17:10:30 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 10 ms on localhost (2/2) -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 17:10:30 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.026217 s -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=216624, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=216696, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:30 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[82] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=216834, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=222354, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[82] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 17:10:30 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:30 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 7 ms on localhost (1/2) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 9 ms on localhost (2/2) -2016-05-24 17:10:30 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 17:10:30 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.065313 s -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=225325, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=225397, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:30 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[84] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=225535, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(2974) called with curMem=231055, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[84] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 17:10:30 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:30 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 11 ms on localhost (1/2) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 14 ms on localhost (2/2) -2016-05-24 17:10:30 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 17:10:30 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.031665 s -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=234029, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=234101, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:30 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[86] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=234239, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(2973) called with curMem=239759, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[86] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 17:10:30 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:30 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 8 ms on localhost (1/2) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 9 ms on localhost (2/2) -2016-05-24 17:10:30 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 17:10:30 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.023391 s -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=242732, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=242804, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:30 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[88] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=242942, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=248462, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[88] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 17:10:30 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:30 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 8 ms on localhost (1/2) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 9 ms on localhost (2/2) -2016-05-24 17:10:30 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 17:10:30 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.023658 s -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=251434, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=251506, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:30 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:10:30 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[90] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=242942, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=248462, maxMem=766075207 -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[90] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:10:30 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 17:10:30 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:30 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 10 ms on localhost (1/2) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 13 ms on localhost (2/2) -2016-05-24 17:10:30 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.028287 s -2016-05-24 17:10:30 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=234101, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=234173, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 17:10:30 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[92] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=216837, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=219386, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[92] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 17:10:30 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:30 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 9 ms on localhost (1/2) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 13 ms on localhost (2/2) -2016-05-24 17:10:30 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 17:10:30 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.042021 s -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=204954, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=199506, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 91 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:30 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[94] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=190731, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=196251, maxMem=766075207 -2016-05-24 17:10:30 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[94] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 17:10:30 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:30 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 9 ms on localhost (1/2) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 13 ms on localhost (2/2) -2016-05-24 17:10:30 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 17:10:30 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.028935 s -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=199223, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=199295, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 93 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:30 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Got job 47 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[96] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=199433, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=204953, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 47 (MapPartitionsRDD[96] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 2 tasks -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 93, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 94, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 93) -2016-05-24 17:10:30 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 94) -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:30 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 93). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 94). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 93) in 10 ms on localhost (1/2) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 94) in 10 ms on localhost (2/2) -2016-05-24 17:10:30 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 17:10:30 INFO DAGScheduler:59 - Job 47 finished: treeAggregate at GradientDescent.scala:225, took 0.026548 s -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=207924, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=207996, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 95 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:30 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Got job 48 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 48(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[98] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=208134, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=213654, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 48 (MapPartitionsRDD[98] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 2 tasks -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 95, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 96, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 95) -2016-05-24 17:10:30 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 96) -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:30 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 95). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 96). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 95) in 8 ms on localhost (1/2) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 96) in 7 ms on localhost (2/2) -2016-05-24 17:10:30 INFO DAGScheduler:59 - ResultStage 48 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-24 17:10:30 INFO DAGScheduler:59 - Job 48 finished: treeAggregate at GradientDescent.scala:225, took 0.021524 s -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=216624, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=216696, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 97 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:30 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Got job 49 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 49(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[100] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=216834, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=222354, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 49 (MapPartitionsRDD[100] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 2 tasks -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 97, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 98, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 97) -2016-05-24 17:10:30 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 98) -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:30 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 98). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 97). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 98) in 7 ms on localhost (1/2) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 97) in 8 ms on localhost (2/2) -2016-05-24 17:10:30 INFO DAGScheduler:59 - ResultStage 49 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-05-24 17:10:30 INFO DAGScheduler:59 - Job 49 finished: treeAggregate at GradientDescent.scala:225, took 0.020393 s -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=225325, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=225397, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 99 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:30 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Got job 50 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[102] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=225535, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(2974) called with curMem=231055, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 50 (MapPartitionsRDD[102] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 2 tasks -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-24 17:10:30 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:30 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 8 ms on localhost (1/2) -2016-05-24 17:10:30 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 2423 bytes result sent to driver -2016-05-24 17:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 11 ms on localhost (2/2) -2016-05-24 17:10:30 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-24 17:10:30 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at GradientDescent.scala:225, took 0.052518 s -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=234029, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=234101, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 101 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:30 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Got job 51 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[104] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=234239, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=239759, maxMem=766075207 -2016-05-24 17:10:30 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:30 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:10:30 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 51 (MapPartitionsRDD[104] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:30 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:31 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-24 17:10:31 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-24 17:10:31 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:31 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:31 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 2423 bytes result sent to driver -2016-05-24 17:10:31 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 2423 bytes result sent to driver -2016-05-24 17:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 8 ms on localhost (1/2) -2016-05-24 17:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 8 ms on localhost (2/2) -2016-05-24 17:10:31 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 17:10:31 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at GradientDescent.scala:225, took 0.076630 s -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=242730, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=242802, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:10:31 INFO SparkContext:59 - Created broadcast 103 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:31 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:31 INFO DAGScheduler:59 - Got job 52 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 52(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:31 INFO DAGScheduler:59 - Submitting ResultStage 52 (MapPartitionsRDD[106] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=242940, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=248460, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:10:31 INFO SparkContext:59 - Created broadcast 104 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 52 (MapPartitionsRDD[106] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:31 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-05-24 17:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-24 17:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 103, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 104, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:31 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 103) -2016-05-24 17:10:31 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 104) -2016-05-24 17:10:31 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:31 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:31 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 103). 2423 bytes result sent to driver -2016-05-24 17:10:31 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 104). 2423 bytes result sent to driver -2016-05-24 17:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 103) in 8 ms on localhost (1/2) -2016-05-24 17:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 104) in 8 ms on localhost (2/2) -2016-05-24 17:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-24 17:10:31 INFO DAGScheduler:59 - ResultStage 52 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 17:10:31 INFO DAGScheduler:59 - Job 52 finished: treeAggregate at GradientDescent.scala:225, took 0.022468 s -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=251431, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=251503, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:10:31 INFO SparkContext:59 - Created broadcast 105 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:31 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:31 INFO DAGScheduler:59 - Got job 53 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 53(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:31 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[108] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=251641, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=257161, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:10:31 INFO SparkContext:59 - Created broadcast 106 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[108] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:31 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-05-24 17:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 105, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 106, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:31 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 105) -2016-05-24 17:10:31 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 106) -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Removed broadcast_104_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:10:31 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:31 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-05-24 17:10:31 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 106). 2423 bytes result sent to driver -2016-05-24 17:10:31 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:31 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 105). 2423 bytes result sent to driver -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Removed broadcast_103_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 105) in 20 ms on localhost (1/2) -2016-05-24 17:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 106) in 20 ms on localhost (2/2) -2016-05-24 17:10:31 INFO DAGScheduler:59 - ResultStage 53 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 17:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-24 17:10:31 INFO DAGScheduler:59 - Job 53 finished: treeAggregate at GradientDescent.scala:225, took 0.034225 s -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=251432, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=248533, maxMem=766075207 -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Removed broadcast_102_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:10:31 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-05-24 17:10:31 INFO SparkContext:59 - Created broadcast 107 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Removed broadcast_101_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Removed broadcast_100_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:31 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-05-24 17:10:31 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Removed broadcast_99_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:31 INFO DAGScheduler:59 - Got job 54 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:31 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[110] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Removed broadcast_98_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=225746, maxMem=766075207 -2016-05-24 17:10:31 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=231056, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Removed broadcast_97_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:31 INFO SparkContext:59 - Created broadcast 108 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 54 (MapPartitionsRDD[110] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:31 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 2 tasks -2016-05-24 17:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Removed broadcast_96_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:31 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 108) -2016-05-24 17:10:31 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 107) -2016-05-24 17:10:31 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-05-24 17:10:31 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:31 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:31 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 108). 2423 bytes result sent to driver -2016-05-24 17:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 108) in 7 ms on localhost (1/2) -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Removed broadcast_95_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:31 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 107). 2423 bytes result sent to driver -2016-05-24 17:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 107) in 12 ms on localhost (2/2) -2016-05-24 17:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-24 17:10:31 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Removed broadcast_94_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:31 INFO DAGScheduler:59 - Job 54 finished: treeAggregate at GradientDescent.scala:225, took 0.025388 s -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=216836, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=216908, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:31 INFO SparkContext:59 - Created broadcast 109 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:31 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Removed broadcast_93_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Removed broadcast_92_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:31 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:31 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-05-24 17:10:31 INFO DAGScheduler:59 - Got job 55 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:31 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[112] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Removed broadcast_91_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=208134, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=205163, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Removed broadcast_90_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:31 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:31 INFO SparkContext:59 - Created broadcast 110 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Removed broadcast_89_piece0 on localhost:50868 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 55 (MapPartitionsRDD[112] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:31 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 2 tasks -2016-05-24 17:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 109, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 110, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:31 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 110) -2016-05-24 17:10:31 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 109) -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Removed broadcast_88_piece0 on localhost:50868 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:31 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:31 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-05-24 17:10:31 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:31 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 110). 2423 bytes result sent to driver -2016-05-24 17:10:31 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 109). 2423 bytes result sent to driver -2016-05-24 17:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 110) in 14 ms on localhost (1/2) -2016-05-24 17:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 109) in 16 ms on localhost (2/2) -2016-05-24 17:10:31 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 17:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-24 17:10:31 INFO DAGScheduler:59 - Job 55 finished: treeAggregate at GradientDescent.scala:225, took 0.150262 s -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=199431, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=199503, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:31 INFO SparkContext:59 - Created broadcast 111 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:31 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:31 INFO DAGScheduler:59 - Got job 56 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:31 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[114] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=199641, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=205161, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:31 INFO SparkContext:59 - Created broadcast 112 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 56 (MapPartitionsRDD[114] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:31 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 2 tasks -2016-05-24 17:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 111, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 112, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:31 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 112) -2016-05-24 17:10:31 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:31 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 112). 2423 bytes result sent to driver -2016-05-24 17:10:31 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 111) -2016-05-24 17:10:31 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:31 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 111). 2423 bytes result sent to driver -2016-05-24 17:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 112) in 8 ms on localhost (1/2) -2016-05-24 17:10:31 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 17:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 111) in 10 ms on localhost (2/2) -2016-05-24 17:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-24 17:10:31 INFO DAGScheduler:59 - Job 56 finished: treeAggregate at GradientDescent.scala:225, took 0.040763 s -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=208132, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=208204, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:50868 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:10:31 INFO SparkContext:59 - Created broadcast 113 from broadcast at GradientDescent.scala:221 -2016-05-24 17:10:31 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:10:31 INFO DAGScheduler:59 - Got job 57 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 57(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:10:31 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[116] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=208342, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:10:31 INFO MemoryStore:59 - ensureFreeSpace(2968) called with curMem=213862, maxMem=766075207 -2016-05-24 17:10:31 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:10:31 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:50868 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:10:31 INFO SparkContext:59 - Created broadcast 114 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 57 (MapPartitionsRDD[116] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:10:31 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 2 tasks -2016-05-24 17:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 113, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 114, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:10:31 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 113) -2016-05-24 17:10:31 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 114) -2016-05-24 17:10:31 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:10:31 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 113). 2423 bytes result sent to driver -2016-05-24 17:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 113) in 10 ms on localhost (1/2) -2016-05-24 17:10:31 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:10:31 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 114). 2423 bytes result sent to driver -2016-05-24 17:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 114) in 15 ms on localhost (2/2) -2016-05-24 17:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-24 17:10:31 INFO DAGScheduler:59 - ResultStage 57 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 17:10:31 INFO DAGScheduler:59 - Job 57 finished: treeAggregate at GradientDescent.scala:225, took 0.030380 s -2016-05-24 17:10:31 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 0.6105671927144823, 0.6097961496848042, 0.6090406315074238, 0.6083000889873139, 0.6075740019440139, 0.6068618771610624, 0.6061632465163228, 0.6054776652741959, 0.6048047105229598, 0.604143979742525 -2016-05-24 17:10:32 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 17:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 17:10:33 WARN QueuedThreadPool:145 - 5 threads could not be stopped -2016-05-24 17:10:33 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 17:10:33 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 17:10:34 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 17:10:34 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 17:10:34 INFO BlockManager:59 - BlockManager stopped -2016-05-24 17:10:34 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 17:10:34 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 17:10:34 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 17:10:35 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 17:10:35 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-9ee011be-041c-401b-b79f-15bd1e12ef00 -2016-05-24 17:11:57 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 17:11:58 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 17:11:58 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 17:11:58 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 17:11:58 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 17:11:59 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 17:11:59 INFO Remoting:74 - Starting remoting -2016-05-24 17:12:00 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:50889] -2016-05-24 17:12:00 INFO Utils:59 - Successfully started service 'sparkDriver' on port 50889. -2016-05-24 17:12:00 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 17:12:00 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 17:12:00 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-9d9643b7-81ea-4fdf-8b7a-f8cb2b73f4fc -2016-05-24 17:12:00 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 17:12:00 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-f2b73c83-7ab9-46cb-84fe-1a0b0250977f\httpd-e3749691-648d-47c1-b011-f4268ee11abe -2016-05-24 17:12:00 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 17:12:00 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 17:12:00 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:50890 -2016-05-24 17:12:00 INFO Utils:59 - Successfully started service 'HTTP file server' on port 50890. -2016-05-24 17:12:00 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 17:12:00 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 17:12:00 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 17:12:00 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 17:12:00 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 17:12:01 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 17:12:01 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 17:12:01 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 50909. -2016-05-24 17:12:01 INFO NettyBlockTransferService:59 - Server created on 50909 -2016-05-24 17:12:01 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 17:12:01 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:50909 with 730.6 MB RAM, BlockManagerId(driver, localhost, 50909) -2016-05-24 17:12:01 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 17:12:02 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 17:12:02 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 17:12:02 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 17:12:02 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 17:12:02 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:50909 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 17:12:02 INFO SparkContext:59 - Created broadcast 0 from textFile at LinearRegression2Learning.scala:21 -2016-05-24 17:12:04 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 17:12:04 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 17:12:04 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 17:12:04 INFO DAGScheduler:59 - Got job 0 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 17:12:04 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 17:12:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:04 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 17:12:04 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=171580, maxMem=766075207 -2016-05-24 17:12:04 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-24 17:12:04 INFO MemoryStore:59 - ensureFreeSpace(2182) called with curMem=175316, maxMem=766075207 -2016-05-24 17:12:04 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 17:12:04 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:50909 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 17:12:04 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:04 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 17:12:04 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-24 17:12:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 17:12:04 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 17:12:04 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 17:12:04 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/ridge-data/lpsa2.data:0+19 -2016-05-24 17:12:04 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 17:12:04 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 17:12:04 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 17:12:04 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 17:12:04 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 17:12:04 INFO MemoryStore:59 - ensureFreeSpace(320) called with curMem=177498, maxMem=766075207 -2016-05-24 17:12:04 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 320.0 B, free 730.4 MB) -2016-05-24 17:12:04 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:50909 (size: 320.0 B, free: 730.6 MB) -2016-05-24 17:12:04 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2611 bytes result sent to driver -2016-05-24 17:12:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 162 ms on localhost (1/1) -2016-05-24 17:12:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 17:12:04 INFO DAGScheduler:59 - ResultStage 0 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.196 s -2016-05-24 17:12:04 INFO DAGScheduler:59 - Job 0 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.330132 s -2016-05-24 17:12:04 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 17:12:04 INFO DAGScheduler:59 - Got job 1 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 17:12:04 INFO DAGScheduler:59 - Final stage: ResultStage 1(count at GradientDescent.scala:195) -2016-05-24 17:12:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:04 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 17:12:04 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=177818, maxMem=766075207 -2016-05-24 17:12:04 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 17:12:04 INFO MemoryStore:59 - ensureFreeSpace(2020) called with curMem=181258, maxMem=766075207 -2016-05-24 17:12:04 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2020.0 B, free 730.4 MB) -2016-05-24 17:12:04 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:50909 (size: 2020.0 B, free: 730.6 MB) -2016-05-24 17:12:04 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 17:12:04 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 17:12:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 17:12:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 17:12:04 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-24 17:12:04 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-24 17:12:04 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:04 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-05-24 17:12:04 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 17:12:04 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/ridge-data/lpsa2.data:19+19 -2016-05-24 17:12:04 INFO MemoryStore:59 - ensureFreeSpace(168) called with curMem=183278, maxMem=766075207 -2016-05-24 17:12:04 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 168.0 B, free 730.4 MB) -2016-05-24 17:12:04 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:50909 (size: 168.0 B, free: 730.6 MB) -2016-05-24 17:12:04 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2662 bytes result sent to driver -2016-05-24 17:12:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 77 ms on localhost (1/2) -2016-05-24 17:12:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 90 ms on localhost (2/2) -2016-05-24 17:12:04 INFO DAGScheduler:59 - ResultStage 1 (count at GradientDescent.scala:195) finished in 0.090 s -2016-05-24 17:12:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 17:12:04 INFO DAGScheduler:59 - Job 1 finished: count at GradientDescent.scala:195, took 0.131201 s -2016-05-24 17:12:05 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:50909 in memory (size: 2020.0 B, free: 730.6 MB) -2016-05-24 17:12:05 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:50909 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 17:12:05 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 17:12:05 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=172068, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=172140, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:05 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:05 INFO SparkContext:59 - Created broadcast 3 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:05 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:05 INFO DAGScheduler:59 - Got job 2 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:05 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:05 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=172278, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=177798, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:05 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:05 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:05 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 17:12:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:05 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 17:12:05 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 17:12:05 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:05 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:05 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2423 bytes result sent to driver -2016-05-24 17:12:05 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2423 bytes result sent to driver -2016-05-24 17:12:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 30 ms on localhost (1/2) -2016-05-24 17:12:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 34 ms on localhost (2/2) -2016-05-24 17:12:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 17:12:05 INFO DAGScheduler:59 - ResultStage 2 (treeAggregate at GradientDescent.scala:225) finished in 0.036 s -2016-05-24 17:12:05 INFO DAGScheduler:59 - Job 2 finished: treeAggregate at GradientDescent.scala:225, took 0.060111 s -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=180769, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=180841, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:05 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:05 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:05 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:05 INFO DAGScheduler:59 - Got job 3 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:05 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:05 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=180979, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=186499, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:05 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:05 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:05 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 17:12:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:05 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 17:12:05 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 17:12:05 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:05 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2423 bytes result sent to driver -2016-05-24 17:12:05 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 30 ms on localhost (1/2) -2016-05-24 17:12:05 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2423 bytes result sent to driver -2016-05-24 17:12:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 36 ms on localhost (2/2) -2016-05-24 17:12:05 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at GradientDescent.scala:225) finished in 0.038 s -2016-05-24 17:12:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 17:12:05 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at GradientDescent.scala:225, took 0.061120 s -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=189470, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=189542, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:05 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:05 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:05 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:05 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:05 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:05 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[10] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=189680, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=195200, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:05 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:05 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[10] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:05 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 17:12:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:05 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 17:12:05 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 17:12:05 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:05 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:05 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2423 bytes result sent to driver -2016-05-24 17:12:05 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2423 bytes result sent to driver -2016-05-24 17:12:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 38 ms on localhost (1/2) -2016-05-24 17:12:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 37 ms on localhost (2/2) -2016-05-24 17:12:05 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.040 s -2016-05-24 17:12:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 17:12:05 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.071364 s -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=198172, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=198244, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:05 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:05 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:05 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:05 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:05 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:05 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[12] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=198382, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=203902, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:05 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:05 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[12] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:05 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 17:12:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:05 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 17:12:05 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:05 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 17:12:05 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2423 bytes result sent to driver -2016-05-24 17:12:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 27 ms on localhost (1/2) -2016-05-24 17:12:05 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:05 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2423 bytes result sent to driver -2016-05-24 17:12:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 46 ms on localhost (2/2) -2016-05-24 17:12:05 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.047 s -2016-05-24 17:12:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 17:12:05 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.073781 s -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=206872, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=206944, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:05 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:05 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:05 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:05 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:05 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:05 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[14] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=207082, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=212602, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:05 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:05 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[14] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:05 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 17:12:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:05 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 17:12:05 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 17:12:05 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:05 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2423 bytes result sent to driver -2016-05-24 17:12:05 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:05 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2423 bytes result sent to driver -2016-05-24 17:12:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 44 ms on localhost (1/2) -2016-05-24 17:12:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 47 ms on localhost (2/2) -2016-05-24 17:12:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 17:12:05 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.047 s -2016-05-24 17:12:05 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.069220 s -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=215573, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=215645, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:05 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:05 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:05 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:05 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:05 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:05 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[16] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=215783, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:05 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=221303, maxMem=766075207 -2016-05-24 17:12:05 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:05 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:05 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[16] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:05 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 17:12:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:05 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 17:12:05 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:06 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 22 ms on localhost (1/2) -2016-05-24 17:12:06 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.033 s -2016-05-24 17:12:06 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.057107 s -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 31 ms on localhost (2/2) -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224272, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224344, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:06 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[18] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224482, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=230002, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[18] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 17:12:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:06 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 17:12:06 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:06 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:06 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 23 ms on localhost (1/2) -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 30 ms on localhost (2/2) -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 17:12:06 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.031 s -2016-05-24 17:12:06 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.053908 s -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=232973, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233045, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:06 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[20] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224692, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:06 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=230074, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[20] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 17:12:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:06 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 17:12:06 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 17:12:06 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:06 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 24 ms on localhost (1/2) -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:06 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:06 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 34 ms on localhost (2/2) -2016-05-24 17:12:06 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.035 s -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 17:12:06 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.065518 s -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=215573, maxMem=766075207 -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=215645, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:06 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 17:12:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:06 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[22] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=207293, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(2973) called with curMem=212813, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[22] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 17:12:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:06 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 17:12:06 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:06 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 20 ms on localhost (1/2) -2016-05-24 17:12:06 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 30 ms on localhost (2/2) -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 17:12:06 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.033 s -2016-05-24 17:12:06 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.060248 s -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=207084, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=207156, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:06 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:06 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:06 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:06 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[24] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=189682, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=195202, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[24] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 17:12:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:06 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 17:12:06 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:06 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 18 ms on localhost (1/2) -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 26 ms on localhost (2/2) -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 17:12:06 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.029 s -2016-05-24 17:12:06 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.072553 s -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=198172, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=198244, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:06 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[26] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=198382, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=203902, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[26] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 17:12:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:06 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 17:12:06 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:06 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 23 ms on localhost (1/2) -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 28 ms on localhost (2/2) -2016-05-24 17:12:06 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.030 s -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 17:12:06 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.054768 s -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=206873, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=206945, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:06 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[28] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=207083, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(2973) called with curMem=212603, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[28] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 17:12:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:06 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 17:12:06 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:06 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 22 ms on localhost (1/2) -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 28 ms on localhost (2/2) -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 17:12:06 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.029 s -2016-05-24 17:12:06 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.048410 s -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=215576, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=215648, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:06 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[30] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=215786, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(2977) called with curMem=221306, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[30] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 17:12:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:06 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:06 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:06 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 37 ms on localhost (1/2) -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 39 ms on localhost (2/2) -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 17:12:06 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.043 s -2016-05-24 17:12:06 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.067382 s -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224283, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224355, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:06 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[32] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224493, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=230013, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[32] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 17:12:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:06 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 17:12:06 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:06 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 15 ms on localhost (1/2) -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 18 ms on localhost (2/2) -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 17:12:06 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 17:12:06 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.040193 s -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=232984, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233056, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:06 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[34] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233194, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=238714, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[34] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 17:12:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:06 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 17:12:06 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:06 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 16 ms on localhost (1/2) -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 16 ms on localhost (2/2) -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 17:12:06 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 17:12:06 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.037356 s -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241685, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241757, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:06 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[36] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=241895, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=247415, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[36] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 17:12:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:06 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:06 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 17:12:06 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 19 ms on localhost (1/2) -2016-05-24 17:12:06 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:06 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2423 bytes result sent to driver -2016-05-24 17:12:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 28 ms on localhost (2/2) -2016-05-24 17:12:06 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 17:12:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 17:12:06 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.045764 s -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250386, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250458, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:06 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:06 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:06 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:06 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:06 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[38] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:06 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250596, maxMem=766075207 -2016-05-24 17:12:06 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=256116, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[38] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:07 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 15 ms on localhost (1/2) -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:07 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 29 ms on localhost (2/2) -2016-05-24 17:12:07 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.030 s -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 17:12:07 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.057646 s -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=259087, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=259159, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:07 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[40] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=259297, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=264817, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[40] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 17:12:07 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:07 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 12 ms on localhost (1/2) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 15 ms on localhost (2/2) -2016-05-24 17:12:07 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 17:12:07 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.043706 s -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=267788, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=267860, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:07 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[42] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=267998, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=273518, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[42] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 17:12:07 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:07 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 19 ms on localhost (1/2) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 21 ms on localhost (2/2) -2016-05-24 17:12:07 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 17:12:07 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.040558 s -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276489, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276561, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:07 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[44] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=276699, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=282219, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[44] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 17:12:07 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:07 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:07 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 20 ms on localhost (1/2) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 20 ms on localhost (2/2) -2016-05-24 17:12:07 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 17:12:07 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.042713 s -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285190, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285262, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:07 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[46] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=285400, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=290920, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[46] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:07 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 15 ms on localhost (1/2) -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:07 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 25 ms on localhost (2/2) -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 17:12:07 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 17:12:07 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.058328 s -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=293891, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=293963, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:07 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[48] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=294101, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=299621, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[48] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:07 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 19 ms on localhost (1/2) -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:07 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 26 ms on localhost (2/2) -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 17:12:07 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.041 s -2016-05-24 17:12:07 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.063533 s -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=302592, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=302664, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:07 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[50] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=302802, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=308322, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[50] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 17:12:07 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:07 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 13 ms on localhost (1/2) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 12 ms on localhost (2/2) -2016-05-24 17:12:07 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 17:12:07 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.034862 s -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=311293, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=311365, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:07 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[52] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=311503, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=317023, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[52] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 17:12:07 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:07 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:07 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 19 ms on localhost (1/2) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 19 ms on localhost (2/2) -2016-05-24 17:12:07 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 17:12:07 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.044567 s -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=319993, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=320065, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:07 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[54] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=320203, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=325723, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[54] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 17:12:07 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:07 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 17 ms on localhost (1/2) -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:07 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 22 ms on localhost (2/2) -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 17:12:07 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 17:12:07 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.041212 s -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=328692, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=328764, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:07 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:07 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[56] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=311503, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=317023, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[56] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 17:12:07 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 17:12:07 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 13 ms on localhost (1/2) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 13 ms on localhost (2/2) -2016-05-24 17:12:07 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 17:12:07 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.029753 s -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=302593, maxMem=766075207 -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=299694, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 17:12:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:07 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[58] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=285401, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=290921, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[58] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 17:12:07 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 17:12:07 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:07 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 12 ms on localhost (1/2) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 16 ms on localhost (2/2) -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 17:12:07 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 17:12:07 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.037803 s -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=282221, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=282293, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:07 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:07 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[60] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=276701, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=282221, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[60] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 17:12:07 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:07 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:07 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 19 ms on localhost (1/2) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 19 ms on localhost (2/2) -2016-05-24 17:12:07 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 17:12:07 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.053865 s -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276491, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=273592, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:07 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[62] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=268000, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=273520, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[62] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:07 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 17:12:07 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:07 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:07 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 2423 bytes result sent to driver -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 9 ms on localhost (1/2) -2016-05-24 17:12:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 12 ms on localhost (2/2) -2016-05-24 17:12:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 17:12:07 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 17:12:07 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.031884 s -2016-05-24 17:12:07 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=268001, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=267863, maxMem=766075207 -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:07 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:07 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:07 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:07 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[64] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250803, maxMem=766075207 -2016-05-24 17:12:07 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:07 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 17:12:07 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=256323, maxMem=766075207 -2016-05-24 17:12:07 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[64] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 10 ms on localhost (1/2) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 11 ms on localhost (2/2) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.028188 s -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241890, maxMem=766075207 -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241824, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[66] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224487, maxMem=766075207 -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=230007, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[66] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 8 ms on localhost (1/2) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 10 ms on localhost (2/2) -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.024850 s -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224278, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224350, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[68] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224488, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=230008, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[68] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 9 ms on localhost (1/2) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 9 ms on localhost (2/2) -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.024667 s -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=232980, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233052, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[70] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233190, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=238710, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[70] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 6 ms on localhost (1/2) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 12 ms on localhost (2/2) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.026391 s -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241682, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241754, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[72] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=241892, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=247412, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[72] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 8 ms on localhost (1/2) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 10 ms on localhost (2/2) -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.028217 s -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250383, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250455, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[74] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250593, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=256113, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[74] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 9 ms on localhost (1/2) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 10 ms on localhost (2/2) -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.023075 s -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=259085, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=259157, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[76] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=259295, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=264815, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[76] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 7 ms on localhost (1/2) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 9 ms on localhost (2/2) -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.021654 s -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=267787, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=267859, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[78] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=267997, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=273517, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[78] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 8 ms on localhost (1/2) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 9 ms on localhost (2/2) -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.019986 s -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276488, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276560, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[80] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=276698, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=282218, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[80] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 8 ms on localhost (1/2) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 10 ms on localhost (2/2) -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.024109 s -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285189, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285261, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[82] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=285399, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=290919, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[82] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 7 ms on localhost (1/2) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 9 ms on localhost (2/2) -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.025677 s -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=293891, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=293963, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[84] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=294101, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2974) called with curMem=299621, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[84] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 8 ms on localhost (1/2) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 8 ms on localhost (2/2) -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.022834 s -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=302595, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=302667, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[86] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=302805, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2974) called with curMem=308325, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[86] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 11 ms on localhost (1/2) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 11 ms on localhost (2/2) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.027952 s -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=311299, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=311371, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[88] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=311509, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=317029, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[88] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 9 ms on localhost (1/2) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 8 ms on localhost (2/2) -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.025024 s -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=320001, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=320073, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[90] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=320211, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=325731, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[90] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 11 ms on localhost (1/2) -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 13 ms on localhost (2/2) -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.040008 s -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=311506, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=311578, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[92] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=294310, maxMem=766075207 -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=299692, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[92] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 7 ms on localhost (1/2) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 11 ms on localhost (2/2) -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.025692 s -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285397, maxMem=766075207 -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285331, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 91 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[94] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=268203, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=273723, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[94] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 9 ms on localhost (1/2) -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 16 ms on localhost (2/2) -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.034588 s -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276694, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276628, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 93 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 47 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[96] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=259291, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=261839, maxMem=766075207 -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 47 (MapPartitionsRDD[96] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 93, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 94, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 93) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 94) -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 93). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 94). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 94) in 9 ms on localhost (1/2) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 93) in 10 ms on localhost (2/2) -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 47 finished: treeAggregate at GradientDescent.scala:225, took 0.025647 s -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250450, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250450, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 95 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 48 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 48(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[98] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=241887, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=238915, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 48 (MapPartitionsRDD[98] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 95, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 96, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 96) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 95) -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 96). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 95). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 95) in 9 ms on localhost (1/2) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 96) in 9 ms on localhost (2/2) -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 48 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 48 finished: treeAggregate at GradientDescent.scala:225, took 0.025450 s -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=233184, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233046, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 97 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:50909 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:08 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 49 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 49(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:50909 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[100] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=215780, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=221300, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 49 (MapPartitionsRDD[100] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 97, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 98, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 98) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 97) -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 97). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 98). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 97) in 7 ms on localhost (1/2) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 98) in 7 ms on localhost (2/2) -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 49 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 49 finished: treeAggregate at GradientDescent.scala:225, took 0.023298 s -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224271, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224343, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 99 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 50 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[102] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224481, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2973) called with curMem=230001, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 50 (MapPartitionsRDD[102] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 9 ms on localhost (1/2) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 9 ms on localhost (2/2) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at GradientDescent.scala:225, took 0.022918 s -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=232974, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233046, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 101 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 51 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[104] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233184, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=238704, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 51 (MapPartitionsRDD[104] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:08 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 2423 bytes result sent to driver -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 8 ms on localhost (1/2) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 10 ms on localhost (2/2) -2016-05-24 17:12:08 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-24 17:12:08 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at GradientDescent.scala:225, took 0.024396 s -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241674, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241746, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 103 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Got job 52 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:08 INFO DAGScheduler:59 - Final stage: ResultStage 52(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting ResultStage 52 (MapPartitionsRDD[106] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=241884, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:12:08 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=247404, maxMem=766075207 -2016-05-24 17:12:08 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:08 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:08 INFO SparkContext:59 - Created broadcast 104 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 52 (MapPartitionsRDD[106] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:08 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 103, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 104, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:08 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 103) -2016-05-24 17:12:08 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 104) -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:09 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 104). 2423 bytes result sent to driver -2016-05-24 17:12:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 104) in 7 ms on localhost (1/2) -2016-05-24 17:12:09 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 103). 2423 bytes result sent to driver -2016-05-24 17:12:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 103) in 11 ms on localhost (2/2) -2016-05-24 17:12:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-24 17:12:09 INFO DAGScheduler:59 - ResultStage 52 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 17:12:09 INFO DAGScheduler:59 - Job 52 finished: treeAggregate at GradientDescent.scala:225, took 0.025062 s -2016-05-24 17:12:09 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250376, maxMem=766075207 -2016-05-24 17:12:09 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:09 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250448, maxMem=766075207 -2016-05-24 17:12:09 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:09 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:09 INFO SparkContext:59 - Created broadcast 105 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:09 INFO DAGScheduler:59 - Got job 53 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:09 INFO DAGScheduler:59 - Final stage: ResultStage 53(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:09 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[108] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:09 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250586, maxMem=766075207 -2016-05-24 17:12:09 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:09 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=256106, maxMem=766075207 -2016-05-24 17:12:09 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:09 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:09 INFO SparkContext:59 - Created broadcast 106 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[108] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:09 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-05-24 17:12:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 105, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 106, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:09 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 105) -2016-05-24 17:12:09 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 106) -2016-05-24 17:12:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:09 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 105). 2423 bytes result sent to driver -2016-05-24 17:12:09 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 106). 2423 bytes result sent to driver -2016-05-24 17:12:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 105) in 11 ms on localhost (1/2) -2016-05-24 17:12:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 106) in 10 ms on localhost (2/2) -2016-05-24 17:12:09 INFO DAGScheduler:59 - ResultStage 53 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 17:12:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-24 17:12:09 INFO DAGScheduler:59 - Job 53 finished: treeAggregate at GradientDescent.scala:225, took 0.023647 s -2016-05-24 17:12:09 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=259078, maxMem=766075207 -2016-05-24 17:12:09 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:09 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=259150, maxMem=766075207 -2016-05-24 17:12:09 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:09 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:09 INFO SparkContext:59 - Created broadcast 107 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:09 INFO DAGScheduler:59 - Got job 54 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:09 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:09 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[110] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:09 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=259288, maxMem=766075207 -2016-05-24 17:12:09 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:09 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=264808, maxMem=766075207 -2016-05-24 17:12:09 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:09 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:09 INFO SparkContext:59 - Created broadcast 108 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 54 (MapPartitionsRDD[110] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:09 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 2 tasks -2016-05-24 17:12:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:09 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 108) -2016-05-24 17:12:09 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 107) -2016-05-24 17:12:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:09 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 107). 2423 bytes result sent to driver -2016-05-24 17:12:09 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 108). 2423 bytes result sent to driver -2016-05-24 17:12:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 107) in 9 ms on localhost (1/2) -2016-05-24 17:12:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 108) in 9 ms on localhost (2/2) -2016-05-24 17:12:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-24 17:12:09 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 17:12:09 INFO DAGScheduler:59 - Job 54 finished: treeAggregate at GradientDescent.scala:225, took 0.023916 s -2016-05-24 17:12:09 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=267780, maxMem=766075207 -2016-05-24 17:12:09 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:09 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=267852, maxMem=766075207 -2016-05-24 17:12:09 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:09 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:09 INFO SparkContext:59 - Created broadcast 109 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:09 INFO DAGScheduler:59 - Got job 55 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:09 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:09 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[112] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:09 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=267990, maxMem=766075207 -2016-05-24 17:12:09 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:09 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=273510, maxMem=766075207 -2016-05-24 17:12:09 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:09 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:09 INFO SparkContext:59 - Created broadcast 110 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 55 (MapPartitionsRDD[112] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:09 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 2 tasks -2016-05-24 17:12:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 109, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 110, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:09 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 109) -2016-05-24 17:12:09 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 110) -2016-05-24 17:12:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:09 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 109). 2423 bytes result sent to driver -2016-05-24 17:12:09 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 110). 2423 bytes result sent to driver -2016-05-24 17:12:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 109) in 8 ms on localhost (1/2) -2016-05-24 17:12:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 110) in 7 ms on localhost (2/2) -2016-05-24 17:12:09 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 17:12:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-24 17:12:09 INFO DAGScheduler:59 - Job 55 finished: treeAggregate at GradientDescent.scala:225, took 0.040222 s -2016-05-24 17:12:09 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276481, maxMem=766075207 -2016-05-24 17:12:09 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:09 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276553, maxMem=766075207 -2016-05-24 17:12:09 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:09 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:09 INFO SparkContext:59 - Created broadcast 111 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:09 INFO DAGScheduler:59 - Got job 56 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:09 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:09 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[114] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:09 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=276691, maxMem=766075207 -2016-05-24 17:12:09 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:09 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=282211, maxMem=766075207 -2016-05-24 17:12:09 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:09 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:09 INFO SparkContext:59 - Created broadcast 112 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 56 (MapPartitionsRDD[114] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:09 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 2 tasks -2016-05-24 17:12:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 111, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 112, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:09 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 111) -2016-05-24 17:12:09 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 112) -2016-05-24 17:12:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:09 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 111). 2423 bytes result sent to driver -2016-05-24 17:12:09 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 112). 2423 bytes result sent to driver -2016-05-24 17:12:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 111) in 6 ms on localhost (1/2) -2016-05-24 17:12:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 112) in 6 ms on localhost (2/2) -2016-05-24 17:12:09 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 17:12:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-24 17:12:09 INFO DAGScheduler:59 - Job 56 finished: treeAggregate at GradientDescent.scala:225, took 0.022594 s -2016-05-24 17:12:09 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285182, maxMem=766075207 -2016-05-24 17:12:09 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:12:09 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285254, maxMem=766075207 -2016-05-24 17:12:09 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:12:09 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:50909 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:12:09 INFO SparkContext:59 - Created broadcast 113 from broadcast at GradientDescent.scala:221 -2016-05-24 17:12:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:12:09 INFO DAGScheduler:59 - Got job 57 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:12:09 INFO DAGScheduler:59 - Final stage: ResultStage 57(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:12:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:12:09 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[116] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:12:09 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=285392, maxMem=766075207 -2016-05-24 17:12:09 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:12:09 INFO MemoryStore:59 - ensureFreeSpace(2968) called with curMem=290912, maxMem=766075207 -2016-05-24 17:12:09 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:12:09 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:50909 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:12:09 INFO SparkContext:59 - Created broadcast 114 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:12:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 57 (MapPartitionsRDD[116] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:12:09 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 2 tasks -2016-05-24 17:12:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 113, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 114, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:12:09 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 113) -2016-05-24 17:12:09 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 114) -2016-05-24 17:12:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:12:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:12:09 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 114). 2423 bytes result sent to driver -2016-05-24 17:12:09 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 113). 2423 bytes result sent to driver -2016-05-24 17:12:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 114) in 6 ms on localhost (1/2) -2016-05-24 17:12:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 113) in 8 ms on localhost (2/2) -2016-05-24 17:12:09 INFO DAGScheduler:59 - ResultStage 57 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 17:12:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-24 17:12:09 INFO DAGScheduler:59 - Job 57 finished: treeAggregate at GradientDescent.scala:225, took 0.022372 s -2016-05-24 17:12:09 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 0.6105671927144823, 0.6097961496848042, 0.6090406315074238, 0.6083000889873139, 0.6075740019440139, 0.6068618771610624, 0.6061632465163228, 0.6054776652741959, 0.6048047105229598, 0.604143979742525 -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 17:12:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 17:12:09 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 17:12:09 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 17:12:09 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 17:12:09 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 17:12:09 INFO BlockManager:59 - BlockManager stopped -2016-05-24 17:12:09 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 17:12:09 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 17:12:09 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 17:12:09 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 17:12:09 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-f2b73c83-7ab9-46cb-84fe-1a0b0250977f -2016-05-24 17:12:09 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 17:12:09 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 17:18:24 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 17:18:25 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 17:18:25 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 17:18:25 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 17:18:25 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 17:18:26 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 17:18:26 INFO Remoting:74 - Starting remoting -2016-05-24 17:18:27 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:51091] -2016-05-24 17:18:27 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51091. -2016-05-24 17:18:27 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 17:18:27 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 17:18:27 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-e550d51e-edba-4e21-b283-728a3cdf2d7a -2016-05-24 17:18:27 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 17:18:27 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-d0d4ca54-7c1f-4e15-a82f-cb0ddafa374c\httpd-26d92453-c3ac-4c47-9e59-1bbfae2e4f7f -2016-05-24 17:18:27 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 17:18:27 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 17:18:27 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51092 -2016-05-24 17:18:27 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51092. -2016-05-24 17:18:27 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 17:18:27 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 17:18:27 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 17:18:27 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 17:18:27 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 17:18:28 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 17:18:28 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 17:18:28 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51111. -2016-05-24 17:18:28 INFO NettyBlockTransferService:59 - Server created on 51111 -2016-05-24 17:18:28 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 17:18:28 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51111 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51111) -2016-05-24 17:18:28 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 17:18:29 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 17:18:29 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 17:18:30 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 17:18:30 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 17:18:30 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51111 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 17:18:30 INFO SparkContext:59 - Created broadcast 0 from textFile at LinearRegression2Learning.scala:21 -2016-05-24 17:18:31 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 17:18:31 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 17:18:31 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 17:18:31 INFO DAGScheduler:59 - Got job 0 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 17:18:31 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 17:18:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:31 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 17:18:31 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=171580, maxMem=766075207 -2016-05-24 17:18:31 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-24 17:18:31 INFO MemoryStore:59 - ensureFreeSpace(2182) called with curMem=175316, maxMem=766075207 -2016-05-24 17:18:31 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 17:18:31 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51111 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 17:18:31 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 17:18:31 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-24 17:18:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 17:18:31 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 17:18:31 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 17:18:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/ridge-data/lpsa2.data:0+19 -2016-05-24 17:18:32 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 17:18:32 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 17:18:32 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 17:18:32 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 17:18:32 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 17:18:32 INFO MemoryStore:59 - ensureFreeSpace(320) called with curMem=177498, maxMem=766075207 -2016-05-24 17:18:32 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 320.0 B, free 730.4 MB) -2016-05-24 17:18:32 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:51111 (size: 320.0 B, free: 730.6 MB) -2016-05-24 17:18:32 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2611 bytes result sent to driver -2016-05-24 17:18:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 188 ms on localhost (1/1) -2016-05-24 17:18:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 17:18:32 INFO DAGScheduler:59 - ResultStage 0 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.218 s -2016-05-24 17:18:32 INFO DAGScheduler:59 - Job 0 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.354139 s -2016-05-24 17:18:32 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 17:18:32 INFO DAGScheduler:59 - Got job 1 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 17:18:32 INFO DAGScheduler:59 - Final stage: ResultStage 1(count at GradientDescent.scala:195) -2016-05-24 17:18:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:32 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:32 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 17:18:32 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=177818, maxMem=766075207 -2016-05-24 17:18:32 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 17:18:32 INFO MemoryStore:59 - ensureFreeSpace(2020) called with curMem=181258, maxMem=766075207 -2016-05-24 17:18:32 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2020.0 B, free 730.4 MB) -2016-05-24 17:18:32 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51111 (size: 2020.0 B, free: 730.6 MB) -2016-05-24 17:18:32 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 17:18:32 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 17:18:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 17:18:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 17:18:32 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-24 17:18:32 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-24 17:18:32 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:32 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 17:18:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/ridge-data/lpsa2.data:19+19 -2016-05-24 17:18:32 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-05-24 17:18:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 28 ms on localhost (1/2) -2016-05-24 17:18:32 INFO MemoryStore:59 - ensureFreeSpace(168) called with curMem=183278, maxMem=766075207 -2016-05-24 17:18:32 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 168.0 B, free 730.4 MB) -2016-05-24 17:18:32 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:51111 (size: 168.0 B, free: 730.6 MB) -2016-05-24 17:18:32 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2662 bytes result sent to driver -2016-05-24 17:18:32 INFO DAGScheduler:59 - ResultStage 1 (count at GradientDescent.scala:195) finished in 0.039 s -2016-05-24 17:18:32 INFO DAGScheduler:59 - Job 1 finished: count at GradientDescent.scala:195, took 0.116620 s -2016-05-24 17:18:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 35 ms on localhost (2/2) -2016-05-24 17:18:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 17:18:32 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:51111 in memory (size: 2020.0 B, free: 730.6 MB) -2016-05-24 17:18:32 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:51111 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 17:18:32 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 17:18:32 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 17:18:32 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=172068, maxMem=766075207 -2016-05-24 17:18:32 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:32 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=172140, maxMem=766075207 -2016-05-24 17:18:32 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:32 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:32 INFO SparkContext:59 - Created broadcast 3 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:32 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:32 INFO DAGScheduler:59 - Got job 2 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:32 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:32 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:32 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:32 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=172278, maxMem=766075207 -2016-05-24 17:18:32 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:32 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=177798, maxMem=766075207 -2016-05-24 17:18:32 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:32 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:32 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:32 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 17:18:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:32 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 17:18:32 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 17:18:32 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:32 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:32 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2423 bytes result sent to driver -2016-05-24 17:18:32 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2423 bytes result sent to driver -2016-05-24 17:18:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 28 ms on localhost (1/2) -2016-05-24 17:18:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 31 ms on localhost (2/2) -2016-05-24 17:18:32 INFO DAGScheduler:59 - ResultStage 2 (treeAggregate at GradientDescent.scala:225) finished in 0.031 s -2016-05-24 17:18:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 17:18:32 INFO DAGScheduler:59 - Job 2 finished: treeAggregate at GradientDescent.scala:225, took 0.053492 s -2016-05-24 17:18:32 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=180770, maxMem=766075207 -2016-05-24 17:18:32 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:32 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=180842, maxMem=766075207 -2016-05-24 17:18:32 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:32 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:32 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:32 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:32 INFO DAGScheduler:59 - Got job 3 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:32 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:32 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:32 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:32 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=180980, maxMem=766075207 -2016-05-24 17:18:32 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:32 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=186500, maxMem=766075207 -2016-05-24 17:18:32 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:32 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:32 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:32 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 17:18:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:32 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 17:18:32 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 17:18:32 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:32 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2423 bytes result sent to driver -2016-05-24 17:18:32 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:33 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2423 bytes result sent to driver -2016-05-24 17:18:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 27 ms on localhost (1/2) -2016-05-24 17:18:33 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at GradientDescent.scala:225) finished in 0.030 s -2016-05-24 17:18:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 28 ms on localhost (2/2) -2016-05-24 17:18:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 17:18:33 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at GradientDescent.scala:225, took 0.050357 s -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=189472, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=189544, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:33 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:33 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:33 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:33 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:33 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[10] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=189682, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=195202, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:33 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[10] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:33 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 17:18:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:33 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 17:18:33 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 17:18:33 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:33 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:33 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2423 bytes result sent to driver -2016-05-24 17:18:33 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2423 bytes result sent to driver -2016-05-24 17:18:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 33 ms on localhost (1/2) -2016-05-24 17:18:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 35 ms on localhost (2/2) -2016-05-24 17:18:33 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.036 s -2016-05-24 17:18:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 17:18:33 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.066516 s -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=198174, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=198246, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:33 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:33 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:33 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:33 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:33 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[12] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=198384, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=203904, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:33 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[12] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:33 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 17:18:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:33 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 17:18:33 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 17:18:33 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:33 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:33 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2423 bytes result sent to driver -2016-05-24 17:18:33 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2423 bytes result sent to driver -2016-05-24 17:18:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 28 ms on localhost (1/2) -2016-05-24 17:18:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 28 ms on localhost (2/2) -2016-05-24 17:18:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 17:18:33 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.032 s -2016-05-24 17:18:33 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.060474 s -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=206873, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=206945, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:33 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:33 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:33 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:33 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:33 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[14] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=207083, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=212603, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:33 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[14] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:33 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 17:18:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:33 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 17:18:33 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 17:18:33 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:33 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:33 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2423 bytes result sent to driver -2016-05-24 17:18:33 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2423 bytes result sent to driver -2016-05-24 17:18:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 23 ms on localhost (1/2) -2016-05-24 17:18:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 29 ms on localhost (2/2) -2016-05-24 17:18:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 17:18:33 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.030 s -2016-05-24 17:18:33 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.055316 s -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=215574, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=215646, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:33 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:33 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:33 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:33 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:33 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[16] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=215784, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=221304, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:33 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[16] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:33 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 17:18:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:33 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 17:18:33 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 17:18:33 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:33 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:33 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2423 bytes result sent to driver -2016-05-24 17:18:33 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2423 bytes result sent to driver -2016-05-24 17:18:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 20 ms on localhost (1/2) -2016-05-24 17:18:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 21 ms on localhost (2/2) -2016-05-24 17:18:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 17:18:33 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.025 s -2016-05-24 17:18:33 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.050551 s -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224273, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224345, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:33 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:33 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:33 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:33 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:33 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[18] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224483, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=230003, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:33 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[18] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:33 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 17:18:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:33 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 17:18:33 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 17:18:33 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:33 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2423 bytes result sent to driver -2016-05-24 17:18:33 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:33 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2423 bytes result sent to driver -2016-05-24 17:18:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 29 ms on localhost (1/2) -2016-05-24 17:18:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 33 ms on localhost (2/2) -2016-05-24 17:18:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 17:18:33 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.035 s -2016-05-24 17:18:33 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.056226 s -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=232974, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233046, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:33 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:33 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:33 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 17:18:33 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:33 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:33 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:33 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[20] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=207293, maxMem=766075207 -2016-05-24 17:18:33 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=212603, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:33 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[20] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:33 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:33 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 17:18:33 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 17:18:33 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:33 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:33 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:33 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2423 bytes result sent to driver -2016-05-24 17:18:33 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2423 bytes result sent to driver -2016-05-24 17:18:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 24 ms on localhost (1/2) -2016-05-24 17:18:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 24 ms on localhost (2/2) -2016-05-24 17:18:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 17:18:33 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.030 s -2016-05-24 17:18:33 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.058742 s -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=198382, maxMem=766075207 -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:33 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=198244, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:33 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:33 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:33 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:33 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:33 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:33 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[22] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=181188, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=186708, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:33 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[22] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:33 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 17:18:33 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 17:18:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:33 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 17:18:33 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:33 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:33 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:33 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2423 bytes result sent to driver -2016-05-24 17:18:33 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2423 bytes result sent to driver -2016-05-24 17:18:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 23 ms on localhost (1/2) -2016-05-24 17:18:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 22 ms on localhost (2/2) -2016-05-24 17:18:33 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 17:18:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 17:18:33 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.055133 s -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=189470, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=189542, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:33 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:33 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:33 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:33 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:33 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[24] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=189680, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=195200, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:33 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[24] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:33 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 17:18:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:33 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 17:18:33 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 17:18:33 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:33 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:33 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2423 bytes result sent to driver -2016-05-24 17:18:33 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2423 bytes result sent to driver -2016-05-24 17:18:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 19 ms on localhost (1/2) -2016-05-24 17:18:33 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 17:18:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 25 ms on localhost (2/2) -2016-05-24 17:18:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 17:18:33 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.051115 s -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=198171, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=198243, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:33 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:33 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:33 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:33 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:33 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[26] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=198381, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=203901, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:33 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[26] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:33 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 17:18:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:33 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 17:18:33 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 17:18:33 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:33 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:33 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2423 bytes result sent to driver -2016-05-24 17:18:33 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2423 bytes result sent to driver -2016-05-24 17:18:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 20 ms on localhost (1/2) -2016-05-24 17:18:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 19 ms on localhost (2/2) -2016-05-24 17:18:33 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 17:18:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 17:18:33 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.048804 s -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=206872, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:33 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=206944, maxMem=766075207 -2016-05-24 17:18:33 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:33 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:33 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:33 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:33 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:33 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:33 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[28] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=207082, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=212602, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[28] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 17:18:34 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:34 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 20 ms on localhost (1/2) -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:34 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 32 ms on localhost (2/2) -2016-05-24 17:18:34 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.032 s -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 17:18:34 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.054518 s -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=215574, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=215646, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:34 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:34 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[30] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=215784, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(2976) called with curMem=221304, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[30] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:34 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:34 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 37 ms on localhost (1/2) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 36 ms on localhost (2/2) -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 17:18:34 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.042 s -2016-05-24 17:18:34 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.083203 s -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224280, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224352, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:34 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:34 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[32] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224490, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=230010, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[32] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 17:18:34 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:34 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 20 ms on localhost (1/2) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 22 ms on localhost (2/2) -2016-05-24 17:18:34 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 17:18:34 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.045776 s -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=232982, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233054, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:34 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:34 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[34] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233192, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=238712, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[34] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 17:18:34 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:34 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 22 ms on localhost (1/2) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 24 ms on localhost (2/2) -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 17:18:34 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.027 s -2016-05-24 17:18:34 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.046079 s -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241678, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241750, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:34 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:34 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[36] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=241888, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=247408, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[36] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 17:18:34 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:34 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 16 ms on localhost (1/2) -2016-05-24 17:18:34 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 22 ms on localhost (2/2) -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 17:18:34 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 17:18:34 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.041094 s -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250379, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250451, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:34 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:34 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[38] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250589, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=256109, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[38] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:34 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 18 ms on localhost (1/2) -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:34 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 40 ms on localhost (2/2) -2016-05-24 17:18:34 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.041 s -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 17:18:34 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.066473 s -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=259080, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=259152, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:34 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:34 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[40] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=259290, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=264810, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[40] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 17:18:34 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:34 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 17 ms on localhost (1/2) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 16 ms on localhost (2/2) -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 17:18:34 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 17:18:34 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.042825 s -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=267781, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=267853, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:34 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:34 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[42] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=267991, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=273511, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[42] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 17:18:34 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:34 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:34 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 22 ms on localhost (1/2) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 22 ms on localhost (2/2) -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 17:18:34 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.025 s -2016-05-24 17:18:34 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.042722 s -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276482, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276554, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:34 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:34 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[44] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=276692, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=282212, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[44] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 17:18:34 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:34 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 15 ms on localhost (1/2) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 18 ms on localhost (2/2) -2016-05-24 17:18:34 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 17:18:34 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.041472 s -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285184, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285256, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:34 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:34 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[46] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=285394, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=290914, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[46] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 17:18:34 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:34 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:34 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 24 ms on localhost (1/2) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 29 ms on localhost (2/2) -2016-05-24 17:18:34 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.029 s -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 17:18:34 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.055203 s -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=293886, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=293958, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:34 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:34 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[48] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=294096, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=299616, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[48] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:34 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:34 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 29 ms on localhost (1/2) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 34 ms on localhost (2/2) -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 17:18:34 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.036 s -2016-05-24 17:18:34 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.075851 s -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=302588, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=302660, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:34 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:34 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[50] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=302798, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:34 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=308318, maxMem=766075207 -2016-05-24 17:18:34 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:34 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:34 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[50] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:34 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:34 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 18 ms on localhost (1/2) -2016-05-24 17:18:34 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:34 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 2423 bytes result sent to driver -2016-05-24 17:18:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 30 ms on localhost (2/2) -2016-05-24 17:18:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 17:18:35 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.031 s -2016-05-24 17:18:35 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.056915 s -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=311287, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=311359, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:35 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[52] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=311497, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=317017, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[52] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 17:18:35 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:35 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 20 ms on localhost (1/2) -2016-05-24 17:18:35 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 21 ms on localhost (2/2) -2016-05-24 17:18:35 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 17:18:35 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.045780 s -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=319987, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=320059, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:35 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[54] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=320197, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=325717, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[54] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 17:18:35 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:35 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 34 ms on localhost (1/2) -2016-05-24 17:18:35 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 38 ms on localhost (2/2) -2016-05-24 17:18:35 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.038 s -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 17:18:35 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.053285 s -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=320196, maxMem=766075207 -2016-05-24 17:18:35 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=320196, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:35 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:35 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[56] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=302795, maxMem=766075207 -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=308315, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[56] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 17:18:35 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 17:18:35 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:35 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 11 ms on localhost (1/2) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 16 ms on localhost (2/2) -2016-05-24 17:18:35 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 17:18:35 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.033026 s -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=293882, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=290983, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 17:18:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:35 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[58] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=268199, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=273509, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[58] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:35 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 14 ms on localhost (1/2) -2016-05-24 17:18:35 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:35 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 27 ms on localhost (2/2) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.033 s -2016-05-24 17:18:35 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.051248 s -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=267989, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=268061, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:35 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:35 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[60] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=267989, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=265023, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[60] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 17:18:35 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:35 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 14 ms on localhost (1/2) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 17 ms on localhost (2/2) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 17:18:35 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 17:18:35 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.034158 s -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=267784, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=259364, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 17:18:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:35 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[62] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250586, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=247614, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[62] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 17:18:35 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:35 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 15 ms on localhost (1/2) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 16 ms on localhost (2/2) -2016-05-24 17:18:35 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.036750 s -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250375, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=247476, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 17:18:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:35 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[64] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=238913, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:35 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=238775, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[64] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 17:18:35 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 17:18:35 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:35 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 12 ms on localhost (1/2) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 15 ms on localhost (2/2) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 17:18:35 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 17:18:35 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.030036 s -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224345, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224345, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:35 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[66] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224483, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=230003, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[66] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 17:18:35 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:35 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 12 ms on localhost (1/2) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 12 ms on localhost (2/2) -2016-05-24 17:18:35 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 17:18:35 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.026633 s -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=232974, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233046, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:35 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[68] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233184, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=238704, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[68] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 17:18:35 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:35 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 9 ms on localhost (1/2) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 9 ms on localhost (2/2) -2016-05-24 17:18:35 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 17:18:35 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.033826 s -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241676, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241748, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:35 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[70] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=241886, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=247406, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[70] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 17:18:35 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:35 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 7 ms on localhost (1/2) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 8 ms on localhost (2/2) -2016-05-24 17:18:35 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 17:18:35 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.021207 s -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250377, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250449, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:35 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[72] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250587, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=256107, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[72] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 17:18:35 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:35 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 9 ms on localhost (1/2) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 9 ms on localhost (2/2) -2016-05-24 17:18:35 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 17:18:35 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.032032 s -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=259078, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=259150, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:35 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[74] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=259288, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=264808, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[74] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 17:18:35 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:35 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 8 ms on localhost (1/2) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 10 ms on localhost (2/2) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 17:18:35 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:18:35 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.025057 s -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=267779, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=267851, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:35 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[76] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=267989, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=273509, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[76] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 17:18:35 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:35 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 8 ms on localhost (1/2) -2016-05-24 17:18:35 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 9 ms on localhost (2/2) -2016-05-24 17:18:35 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 17:18:35 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.022628 s -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276480, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276552, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:35 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[78] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=276690, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=282210, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[78] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 17:18:35 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:35 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 10 ms on localhost (1/2) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 11 ms on localhost (2/2) -2016-05-24 17:18:35 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 17:18:35 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.024480 s -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285182, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285254, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:35 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[80] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=285392, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=290912, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[80] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 17:18:35 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:35 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 6 ms on localhost (1/2) -2016-05-24 17:18:35 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 9 ms on localhost (2/2) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 17:18:35 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:18:35 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.023248 s -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=293884, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=293956, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:35 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[82] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=294094, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=299614, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[82] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 17:18:35 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:35 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 9 ms on localhost (1/2) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 9 ms on localhost (2/2) -2016-05-24 17:18:35 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 17:18:35 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.025893 s -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=302585, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=302657, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:35 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[84] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=302795, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(2973) called with curMem=308315, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[84] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 17:18:35 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:35 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 7 ms on localhost (1/2) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 9 ms on localhost (2/2) -2016-05-24 17:18:35 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 17:18:35 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.021716 s -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=311288, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=311360, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:35 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[86] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=311498, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO MemoryStore:59 - ensureFreeSpace(2973) called with curMem=317018, maxMem=766075207 -2016-05-24 17:18:35 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:35 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:35 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[86] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:35 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:35 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 11 ms on localhost (1/2) -2016-05-24 17:18:35 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 17:18:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:35 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 2423 bytes result sent to driver -2016-05-24 17:18:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 24 ms on localhost (2/2) -2016-05-24 17:18:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 17:18:35 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 17:18:35 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.041356 s -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=319991, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=320063, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:36 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:36 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[88] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=320201, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=325721, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[88] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 17:18:36 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:36 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 11 ms on localhost (1/2) -2016-05-24 17:18:36 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 15 ms on localhost (2/2) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 17:18:36 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 17:18:36 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.029715 s -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=328693, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=328765, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:36 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:36 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[90] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=328903, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=334423, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[90] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-24 17:18:36 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 17:18:36 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 8 ms on localhost (1/2) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 11 ms on localhost (2/2) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 17:18:36 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 17:18:36 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.039233 s -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=320200, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=320062, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 17:18:36 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:36 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[92] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=303006, maxMem=766075207 -2016-05-24 17:18:36 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=308388, maxMem=766075207 -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[92] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 17:18:36 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 17:18:36 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 8 ms on localhost (1/2) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 10 ms on localhost (2/2) -2016-05-24 17:18:36 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.025857 s -2016-05-24 17:18:36 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=294093, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=294027, maxMem=766075207 -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 91 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:36 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[94] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=276691, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=279240, maxMem=766075207 -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:36 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[94] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:36 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 11 ms on localhost (1/2) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 11 ms on localhost (2/2) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 17:18:36 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 17:18:36 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.029997 s -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=267918, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=267852, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 93 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 17:18:36 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO DAGScheduler:59 - Got job 47 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:36 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[96] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250797, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=256317, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 47 (MapPartitionsRDD[96] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 2 tasks -2016-05-24 17:18:36 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 93, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 94, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 94) -2016-05-24 17:18:36 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 93) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:36 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 94). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:36 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 93). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 93) in 14 ms on localhost (1/2) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 94) in 19 ms on localhost (2/2) -2016-05-24 17:18:36 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-24 17:18:36 INFO DAGScheduler:59 - Job 47 finished: treeAggregate at GradientDescent.scala:225, took 0.036349 s -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=256106, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250658, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 95 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO DAGScheduler:59 - Got job 48 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:36 INFO DAGScheduler:59 - Final stage: ResultStage 48(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[98] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=242095, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=247615, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 48 (MapPartitionsRDD[98] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 2 tasks -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 95, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 96, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 95) -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:36 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 17:18:36 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 96) -2016-05-24 17:18:36 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 95). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 95) in 16 ms on localhost (1/2) -2016-05-24 17:18:36 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 96). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 96) in 19 ms on localhost (2/2) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-24 17:18:36 INFO DAGScheduler:59 - ResultStage 48 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 17:18:36 INFO DAGScheduler:59 - Job 48 finished: treeAggregate at GradientDescent.scala:225, took 0.040890 s -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=244855, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241956, maxMem=766075207 -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:36 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 97 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:36 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Got job 49 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:36 INFO DAGScheduler:59 - Final stage: ResultStage 49(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[100] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233255, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=238703, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 49 (MapPartitionsRDD[100] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 2 tasks -2016-05-24 17:18:36 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 97, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 98, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 97) -2016-05-24 17:18:36 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 98) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:36 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 97). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 98). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 97) in 9 ms on localhost (1/2) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 98) in 9 ms on localhost (2/2) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-05-24 17:18:36 INFO DAGScheduler:59 - ResultStage 49 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:18:36 INFO DAGScheduler:59 - Job 49 finished: treeAggregate at GradientDescent.scala:225, took 0.024531 s -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:51111 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224484, maxMem=766075207 -2016-05-24 17:18:36 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224484, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:51111 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 99 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:36 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Got job 50 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:36 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[102] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224484, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(2974) called with curMem=230004, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 50 (MapPartitionsRDD[102] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 2 tasks -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-24 17:18:36 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:36 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 6 ms on localhost (1/2) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 8 ms on localhost (2/2) -2016-05-24 17:18:36 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-24 17:18:36 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at GradientDescent.scala:225, took 0.022068 s -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=232978, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233050, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 101 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:36 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Got job 51 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:36 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[104] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233188, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(2973) called with curMem=238708, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 51 (MapPartitionsRDD[104] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-24 17:18:36 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:36 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 8 ms on localhost (1/2) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 7 ms on localhost (2/2) -2016-05-24 17:18:36 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-24 17:18:36 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at GradientDescent.scala:225, took 0.023470 s -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241681, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241753, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 103 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:36 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Got job 52 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:36 INFO DAGScheduler:59 - Final stage: ResultStage 52(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting ResultStage 52 (MapPartitionsRDD[106] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=241891, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=247411, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 104 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 52 (MapPartitionsRDD[106] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 103, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 104, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 103) -2016-05-24 17:18:36 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 104) -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:36 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 104). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 103). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 104) in 8 ms on localhost (1/2) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 103) in 9 ms on localhost (2/2) -2016-05-24 17:18:36 INFO DAGScheduler:59 - ResultStage 52 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-24 17:18:36 INFO DAGScheduler:59 - Job 52 finished: treeAggregate at GradientDescent.scala:225, took 0.019631 s -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250383, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250455, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 105 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:36 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Got job 53 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:36 INFO DAGScheduler:59 - Final stage: ResultStage 53(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[108] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250593, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=256113, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 106 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[108] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 105, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 106, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 105) -2016-05-24 17:18:36 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 106) -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:36 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 105). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:36 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 106). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 105) in 19 ms on localhost (1/2) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 106) in 20 ms on localhost (2/2) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-24 17:18:36 INFO DAGScheduler:59 - ResultStage 53 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 17:18:36 INFO DAGScheduler:59 - Job 53 finished: treeAggregate at GradientDescent.scala:225, took 0.034427 s -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=259084, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=259156, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 107 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:36 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Got job 54 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:36 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[110] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=259294, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=264814, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 108 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 54 (MapPartitionsRDD[110] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 2 tasks -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 107) -2016-05-24 17:18:36 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 108) -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:36 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 108). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 107). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 108) in 7 ms on localhost (1/2) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 107) in 9 ms on localhost (2/2) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-24 17:18:36 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:18:36 INFO DAGScheduler:59 - Job 54 finished: treeAggregate at GradientDescent.scala:225, took 0.022092 s -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=267785, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=267857, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 109 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:36 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Got job 55 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:36 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[112] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=267995, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=273515, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 110 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 55 (MapPartitionsRDD[112] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 2 tasks -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 109, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 110, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 110) -2016-05-24 17:18:36 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 109) -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:36 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 110). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 109). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 110) in 7 ms on localhost (1/2) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 109) in 8 ms on localhost (2/2) -2016-05-24 17:18:36 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-24 17:18:36 INFO DAGScheduler:59 - Job 55 finished: treeAggregate at GradientDescent.scala:225, took 0.024291 s -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276485, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276557, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 111 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:36 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Got job 56 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:36 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[114] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=276695, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=282215, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 112 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 56 (MapPartitionsRDD[114] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 2 tasks -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 111, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 112, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:36 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 111) -2016-05-24 17:18:36 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 112) -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:36 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:36 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 112). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 112) in 209 ms on localhost (1/2) -2016-05-24 17:18:36 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 111). 2423 bytes result sent to driver -2016-05-24 17:18:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 111) in 213 ms on localhost (2/2) -2016-05-24 17:18:36 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at GradientDescent.scala:225) finished in 0.213 s -2016-05-24 17:18:36 INFO DAGScheduler:59 - Job 56 finished: treeAggregate at GradientDescent.scala:225, took 0.316791 s -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285187, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285259, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:18:36 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:51111 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:18:36 INFO SparkContext:59 - Created broadcast 113 from broadcast at GradientDescent.scala:221 -2016-05-24 17:18:36 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:18:36 INFO DAGScheduler:59 - Got job 57 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:18:36 INFO DAGScheduler:59 - Final stage: ResultStage 57(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:18:36 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[116] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:18:36 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=285397, maxMem=766075207 -2016-05-24 17:18:36 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:18:37 INFO MemoryStore:59 - ensureFreeSpace(2968) called with curMem=290917, maxMem=766075207 -2016-05-24 17:18:37 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:18:37 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:51111 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:18:37 INFO SparkContext:59 - Created broadcast 114 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:18:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 57 (MapPartitionsRDD[116] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:18:37 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 2 tasks -2016-05-24 17:18:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 113, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 114, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:18:37 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 113) -2016-05-24 17:18:37 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 114) -2016-05-24 17:18:37 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:18:37 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:18:37 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 113). 2423 bytes result sent to driver -2016-05-24 17:18:37 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 114). 2423 bytes result sent to driver -2016-05-24 17:18:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 113) in 6 ms on localhost (1/2) -2016-05-24 17:18:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 114) in 6 ms on localhost (2/2) -2016-05-24 17:18:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-24 17:18:37 INFO DAGScheduler:59 - ResultStage 57 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 17:18:37 INFO DAGScheduler:59 - Job 57 finished: treeAggregate at GradientDescent.scala:225, took 0.067760 s -2016-05-24 17:18:37 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 0.6105671927144823, 0.6097961496848042, 0.6090406315074238, 0.6083000889873139, 0.6075740019440139, 0.6068618771610624, 0.6061632465163228, 0.6054776652741959, 0.6048047105229598, 0.604143979742525 -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 17:18:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 17:18:37 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 17:18:37 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 17:18:37 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 17:18:37 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 17:18:37 INFO BlockManager:59 - BlockManager stopped -2016-05-24 17:18:37 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 17:18:37 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 17:18:37 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 17:18:37 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 17:18:37 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-d0d4ca54-7c1f-4e15-a82f-cb0ddafa374c -2016-05-24 17:18:37 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 17:18:37 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 17:20:15 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 17:20:16 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 17:20:16 INFO Remoting:74 - Starting remoting -2016-05-24 17:20:17 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:51198] -2016-05-24 17:20:17 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 17:20:17 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51200 -2016-05-24 17:20:17 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 17:20:17 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 17:20:20 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 17:20:20 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 17:20:21 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 17:20:21 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 17:20:21 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 17:20:21 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 17:20:21 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 17:20:21 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 17:20:21 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 17:20:32 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605241720_0079_m_000000_79' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/LinearRegression/20160524172026114/result/_temporary/0/task_201605241720_0079_m_000000 -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 17:20:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 17:20:47 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 17:20:48 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 17:20:48 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 17:20:48 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 17:20:48 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 17:20:49 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 17:20:49 INFO Remoting:74 - Starting remoting -2016-05-24 17:20:50 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:51245] -2016-05-24 17:20:50 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51245. -2016-05-24 17:20:50 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 17:20:50 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 17:20:50 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-d749a122-3684-4eaf-9643-5547203aeb68 -2016-05-24 17:20:50 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 17:20:50 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-ffd773bd-e351-4ba5-b6a4-8d869f2d4760\httpd-ed6bb097-67ac-4947-a1e8-c35178f161c0 -2016-05-24 17:20:50 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 17:20:50 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 17:20:50 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51246 -2016-05-24 17:20:50 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51246. -2016-05-24 17:20:50 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 17:20:50 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 17:20:50 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 17:20:50 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 17:20:50 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 17:20:50 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 17:20:50 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 17:20:51 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51267. -2016-05-24 17:20:51 INFO NettyBlockTransferService:59 - Server created on 51267 -2016-05-24 17:20:51 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 17:20:51 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51267 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51267) -2016-05-24 17:20:51 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 17:20:52 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 17:20:52 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 17:20:52 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 17:20:52 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 17:20:52 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51267 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 17:20:52 INFO SparkContext:59 - Created broadcast 0 from textFile at LinearRegression2Learning.scala:21 -2016-05-24 17:20:53 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 17:20:53 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 17:20:53 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 17:20:53 INFO DAGScheduler:59 - Got job 0 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 17:20:53 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 17:20:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:53 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 17:20:54 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=171580, maxMem=766075207 -2016-05-24 17:20:54 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-24 17:20:54 INFO MemoryStore:59 - ensureFreeSpace(2182) called with curMem=175316, maxMem=766075207 -2016-05-24 17:20:54 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 17:20:54 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51267 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 17:20:54 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:54 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 17:20:54 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-24 17:20:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 17:20:54 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 17:20:54 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 17:20:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/ridge-data/lpsa2.data:0+19 -2016-05-24 17:20:54 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 17:20:54 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 17:20:54 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 17:20:54 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 17:20:54 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 17:20:54 INFO MemoryStore:59 - ensureFreeSpace(320) called with curMem=177498, maxMem=766075207 -2016-05-24 17:20:54 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 320.0 B, free 730.4 MB) -2016-05-24 17:20:54 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:51267 (size: 320.0 B, free: 730.6 MB) -2016-05-24 17:20:54 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2611 bytes result sent to driver -2016-05-24 17:20:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 180 ms on localhost (1/1) -2016-05-24 17:20:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 17:20:54 INFO DAGScheduler:59 - ResultStage 0 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.208 s -2016-05-24 17:20:54 INFO DAGScheduler:59 - Job 0 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.331556 s -2016-05-24 17:20:54 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 17:20:54 INFO DAGScheduler:59 - Got job 1 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 17:20:54 INFO DAGScheduler:59 - Final stage: ResultStage 1(count at GradientDescent.scala:195) -2016-05-24 17:20:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:54 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 17:20:54 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=177818, maxMem=766075207 -2016-05-24 17:20:54 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 17:20:54 INFO MemoryStore:59 - ensureFreeSpace(2020) called with curMem=181258, maxMem=766075207 -2016-05-24 17:20:54 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2020.0 B, free 730.4 MB) -2016-05-24 17:20:54 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51267 (size: 2020.0 B, free: 730.6 MB) -2016-05-24 17:20:54 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 17:20:54 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 17:20:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 17:20:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 17:20:54 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-24 17:20:54 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-24 17:20:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:54 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-05-24 17:20:54 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 17:20:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/ridge-data/lpsa2.data:19+19 -2016-05-24 17:20:54 INFO MemoryStore:59 - ensureFreeSpace(168) called with curMem=183278, maxMem=766075207 -2016-05-24 17:20:54 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 168.0 B, free 730.4 MB) -2016-05-24 17:20:54 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:51267 (size: 168.0 B, free: 730.6 MB) -2016-05-24 17:20:54 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2662 bytes result sent to driver -2016-05-24 17:20:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 85 ms on localhost (1/2) -2016-05-24 17:20:54 INFO DAGScheduler:59 - ResultStage 1 (count at GradientDescent.scala:195) finished in 0.093 s -2016-05-24 17:20:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 85 ms on localhost (2/2) -2016-05-24 17:20:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 17:20:54 INFO DAGScheduler:59 - Job 1 finished: count at GradientDescent.scala:195, took 0.138019 s -2016-05-24 17:20:54 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:51267 in memory (size: 2020.0 B, free: 730.6 MB) -2016-05-24 17:20:54 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:51267 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 17:20:55 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 17:20:55 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=172068, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=172140, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:55 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:55 INFO SparkContext:59 - Created broadcast 3 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:55 INFO DAGScheduler:59 - Got job 2 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:55 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:55 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=172278, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=177798, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:20:55 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:55 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:55 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 17:20:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:55 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 17:20:55 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 17:20:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:55 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2423 bytes result sent to driver -2016-05-24 17:20:55 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2423 bytes result sent to driver -2016-05-24 17:20:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 29 ms on localhost (1/2) -2016-05-24 17:20:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 27 ms on localhost (2/2) -2016-05-24 17:20:55 INFO DAGScheduler:59 - ResultStage 2 (treeAggregate at GradientDescent.scala:225) finished in 0.030 s -2016-05-24 17:20:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 17:20:55 INFO DAGScheduler:59 - Job 2 finished: treeAggregate at GradientDescent.scala:225, took 0.051687 s -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=180770, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=180842, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:55 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:55 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:55 INFO DAGScheduler:59 - Got job 3 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:55 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:55 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=180980, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=186500, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:20:55 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:55 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:55 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 17:20:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:55 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 17:20:55 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 17:20:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:55 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2423 bytes result sent to driver -2016-05-24 17:20:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:55 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2423 bytes result sent to driver -2016-05-24 17:20:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 35 ms on localhost (1/2) -2016-05-24 17:20:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 41 ms on localhost (2/2) -2016-05-24 17:20:55 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at GradientDescent.scala:225) finished in 0.041 s -2016-05-24 17:20:55 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at GradientDescent.scala:225, took 0.059942 s -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=189471, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=189543, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:55 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:55 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 17:20:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:55 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:55 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:55 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[10] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=189681, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=195201, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:20:55 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:55 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[10] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:55 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 17:20:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:55 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 17:20:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:55 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 17:20:55 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2423 bytes result sent to driver -2016-05-24 17:20:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:55 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2423 bytes result sent to driver -2016-05-24 17:20:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 38 ms on localhost (1/2) -2016-05-24 17:20:55 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.048 s -2016-05-24 17:20:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 41 ms on localhost (2/2) -2016-05-24 17:20:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 17:20:55 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.083318 s -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=198172, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=198244, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:55 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:55 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:55 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:55 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:55 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[12] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=198382, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=203902, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:20:55 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:55 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[12] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:55 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 17:20:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:55 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 17:20:55 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 17:20:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:55 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2423 bytes result sent to driver -2016-05-24 17:20:55 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2423 bytes result sent to driver -2016-05-24 17:20:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 23 ms on localhost (1/2) -2016-05-24 17:20:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 30 ms on localhost (2/2) -2016-05-24 17:20:55 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.031 s -2016-05-24 17:20:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 17:20:55 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.056707 s -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=206871, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=206943, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:55 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:55 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:55 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:55 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:55 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[14] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=207081, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=212601, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:20:55 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:55 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[14] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:55 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 17:20:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:55 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 17:20:55 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 17:20:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:55 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2423 bytes result sent to driver -2016-05-24 17:20:55 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2423 bytes result sent to driver -2016-05-24 17:20:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 35 ms on localhost (1/2) -2016-05-24 17:20:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 40 ms on localhost (2/2) -2016-05-24 17:20:55 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.040 s -2016-05-24 17:20:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 17:20:55 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.071962 s -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=215573, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=215645, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:55 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:55 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:55 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:55 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:55 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[16] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=215783, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(2977) called with curMem=221303, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:20:55 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:55 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[16] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:55 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 17:20:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:55 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 17:20:55 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 17:20:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:55 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2423 bytes result sent to driver -2016-05-24 17:20:55 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2423 bytes result sent to driver -2016-05-24 17:20:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 19 ms on localhost (1/2) -2016-05-24 17:20:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 19 ms on localhost (2/2) -2016-05-24 17:20:55 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 17:20:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 17:20:55 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.048153 s -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224280, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224352, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:55 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:55 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:55 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:55 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:55 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[18] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224490, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=230010, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:20:55 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:55 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[18] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:55 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 17:20:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:55 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 17:20:55 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 17:20:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:55 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2423 bytes result sent to driver -2016-05-24 17:20:55 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2423 bytes result sent to driver -2016-05-24 17:20:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 22 ms on localhost (1/2) -2016-05-24 17:20:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 22 ms on localhost (2/2) -2016-05-24 17:20:55 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 17:20:55 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.052713 s -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=232981, maxMem=766075207 -2016-05-24 17:20:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233053, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:55 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:55 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:55 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 17:20:55 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:55 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:55 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[20] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233191, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:55 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=238711, maxMem=766075207 -2016-05-24 17:20:55 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:20:55 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:55 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[20] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:55 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 17:20:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:55 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 17:20:55 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 17:20:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:55 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2423 bytes result sent to driver -2016-05-24 17:20:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:55 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2423 bytes result sent to driver -2016-05-24 17:20:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 22 ms on localhost (1/2) -2016-05-24 17:20:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 26 ms on localhost (2/2) -2016-05-24 17:20:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 17:20:55 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.027 s -2016-05-24 17:20:55 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.051265 s -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241682, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241754, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:56 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 17:20:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:56 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:56 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[22] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:56 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224694, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=230076, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[22] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:56 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:56 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 17:20:56 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 20 ms on localhost (1/2) -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:56 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:56 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 27 ms on localhost (2/2) -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 17:20:56 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.030 s -2016-05-24 17:20:56 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.061560 s -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224274, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=215857, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:56 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:56 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:56 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[24] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=204113, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=209633, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:20:56 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[24] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:56 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:56 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 20 ms on localhost (1/2) -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:56 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 33 ms on localhost (2/2) -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 17:20:56 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.034 s -2016-05-24 17:20:56 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.063406 s -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=198171, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=198243, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:56 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[26] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=198381, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=203901, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[26] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 17:20:56 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:56 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:56 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 23 ms on localhost (1/2) -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 29 ms on localhost (2/2) -2016-05-24 17:20:56 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.030 s -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 17:20:56 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.057235 s -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=206872, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=206944, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:56 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[28] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=207082, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=212602, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[28] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 17:20:56 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:56 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 21 ms on localhost (1/2) -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 21 ms on localhost (2/2) -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 17:20:56 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 17:20:56 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.047469 s -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=215574, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=215646, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:56 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[30] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=215784, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(2977) called with curMem=221304, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[30] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:56 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 20 ms on localhost (1/2) -2016-05-24 17:20:56 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:56 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 38 ms on localhost (2/2) -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 17:20:56 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.041 s -2016-05-24 17:20:56 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.062854 s -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224281, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224353, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:56 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[32] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224491, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=230011, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[32] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 17:20:56 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:56 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 16 ms on localhost (1/2) -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 19 ms on localhost (2/2) -2016-05-24 17:20:56 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 17:20:56 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.043680 s -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=232983, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233055, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:56 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[34] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233193, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=238713, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[34] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 17:20:56 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:56 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 19 ms on localhost (1/2) -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 18 ms on localhost (2/2) -2016-05-24 17:20:56 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 17:20:56 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.041359 s -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241685, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241757, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:56 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[36] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=241895, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=247415, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[36] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:56 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 21 ms on localhost (1/2) -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:56 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 26 ms on localhost (2/2) -2016-05-24 17:20:56 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.029 s -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 17:20:56 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.048244 s -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250387, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250459, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:56 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[38] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250597, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(2974) called with curMem=256117, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[38] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 17:20:56 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:56 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:56 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 23 ms on localhost (1/2) -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 28 ms on localhost (2/2) -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 17:20:56 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.029 s -2016-05-24 17:20:56 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.055317 s -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=259091, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=259163, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:56 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[40] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=259301, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=264821, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[40] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 17:20:56 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:56 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:56 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 28 ms on localhost (1/2) -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 32 ms on localhost (2/2) -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 17:20:56 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.034 s -2016-05-24 17:20:56 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.060602 s -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=267793, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=267865, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:56 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[42] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=268003, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=273523, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[42] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:56 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 17:20:56 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:56 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 2423 bytes result sent to driver -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 20 ms on localhost (1/2) -2016-05-24 17:20:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 20 ms on localhost (2/2) -2016-05-24 17:20:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 17:20:56 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 17:20:56 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.048366 s -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276494, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276566, maxMem=766075207 -2016-05-24 17:20:56 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:56 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:56 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:57 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[44] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=276704, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=282224, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[44] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 17:20:57 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:57 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 16 ms on localhost (1/2) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 17 ms on localhost (2/2) -2016-05-24 17:20:57 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 17:20:57 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.035003 s -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285196, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285268, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:57 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[46] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=285406, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=290926, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[46] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 17:20:57 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:57 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:57 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 27 ms on localhost (1/2) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 32 ms on localhost (2/2) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 17:20:57 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.036 s -2016-05-24 17:20:57 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.059982 s -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=293897, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=293969, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:57 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[48] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=294107, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=299627, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[48] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:57 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 30 ms on localhost (1/2) -2016-05-24 17:20:57 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 35 ms on localhost (2/2) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 17:20:57 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.039 s -2016-05-24 17:20:57 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.064301 s -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=302599, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=302671, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:57 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[50] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=302809, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=308329, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[50] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:57 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 22 ms on localhost (1/2) -2016-05-24 17:20:57 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:57 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 38 ms on localhost (2/2) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 17:20:57 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.040 s -2016-05-24 17:20:57 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.069643 s -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=311299, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=311371, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:57 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[52] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=311509, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=317029, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[52] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 17:20:57 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:57 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:57 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 15 ms on localhost (1/2) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 19 ms on localhost (2/2) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 17:20:57 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 17:20:57 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.048569 s -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=319998, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=320070, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:57 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[54] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=320208, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=325728, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[54] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 17:20:57 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:57 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 13 ms on localhost (1/2) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 18 ms on localhost (2/2) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 17:20:57 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 17:20:57 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.037625 s -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=328697, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=328769, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:57 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:57 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[56] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=319998, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=322549, maxMem=766075207 -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[56] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 17:20:57 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:57 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 11 ms on localhost (1/2) -2016-05-24 17:20:57 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 13 ms on localhost (2/2) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 17:20:57 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.029790 s -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=311090, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=308190, maxMem=766075207 -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:57 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[58] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=285405, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=290925, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[58] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 17:20:57 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:57 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 12 ms on localhost (1/2) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 15 ms on localhost (2/2) -2016-05-24 17:20:57 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 17:20:57 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.037721 s -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285196, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285268, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:57 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[60] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=285406, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=290926, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[60] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 17:20:57 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:57 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 17 ms on localhost (1/2) -2016-05-24 17:20:57 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.025 s -2016-05-24 17:20:57 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.064111 s -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 24 ms on localhost (2/2) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=284986, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=282084, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:57 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:57 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:57 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[62] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=267862, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=273382, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[62] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 17:20:57 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:57 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 12 ms on localhost (1/2) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 13 ms on localhost (2/2) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 17:20:57 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 17:20:57 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.035160 s -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=270762, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=267862, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:57 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[64] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=264818, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=264680, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[64] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 17:20:57 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:57 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:57 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 12 ms on localhost (1/2) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 16 ms on localhost (2/2) -2016-05-24 17:20:57 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 17:20:57 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.036630 s -2016-05-24 17:20:57 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=259082, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=258944, maxMem=766075207 -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:57 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[66] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=241751, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=247199, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[66] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 17:20:57 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:57 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:57 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 10 ms on localhost (1/2) -2016-05-24 17:20:57 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 12 ms on localhost (2/2) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 17:20:57 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:57 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.032567 s -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241471, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233051, maxMem=766075207 -2016-05-24 17:20:57 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:57 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:57 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[68] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224488, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=230008, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[68] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:57 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 17:20:57 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:57 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 2423 bytes result sent to driver -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 9 ms on localhost (1/2) -2016-05-24 17:20:57 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 17:20:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 10 ms on localhost (2/2) -2016-05-24 17:20:57 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.026253 s -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=232978, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233050, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:57 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[70] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233188, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:57 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=238708, maxMem=766075207 -2016-05-24 17:20:57 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:20:57 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:57 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[70] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:57 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 17:20:58 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 9 ms on localhost (1/2) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 11 ms on localhost (2/2) -2016-05-24 17:20:58 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 17:20:58 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.030216 s -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241680, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241752, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:58 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[72] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=241890, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=247410, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[72] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 17:20:58 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 12 ms on localhost (1/2) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 15 ms on localhost (2/2) -2016-05-24 17:20:58 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 17:20:58 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.037890 s -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250381, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250453, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:58 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[74] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250591, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=256111, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[74] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 17:20:58 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 8 ms on localhost (1/2) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 12 ms on localhost (2/2) -2016-05-24 17:20:58 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 17:20:58 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.026435 s -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=259083, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=259155, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:58 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[76] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=259293, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=264813, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[76] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 17:20:58 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 10 ms on localhost (1/2) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 11 ms on localhost (2/2) -2016-05-24 17:20:58 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 17:20:58 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.038970 s -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=267785, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=267857, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:58 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[78] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=267995, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=273515, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[78] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 17:20:58 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 8 ms on localhost (1/2) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 11 ms on localhost (2/2) -2016-05-24 17:20:58 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 17:20:58 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.023900 s -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276486, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276558, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:58 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[80] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=276696, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=282216, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[80] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 17:20:58 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 10 ms on localhost (1/2) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 12 ms on localhost (2/2) -2016-05-24 17:20:58 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 17:20:58 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.029774 s -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285188, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285260, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:58 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[82] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=285398, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=290918, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[82] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 17:20:58 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 8 ms on localhost (1/2) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 10 ms on localhost (2/2) -2016-05-24 17:20:58 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 17:20:58 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.023919 s -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=293889, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=293961, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:58 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[84] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=294099, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(2975) called with curMem=299619, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[84] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 17:20:58 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 12 ms on localhost (1/2) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 16 ms on localhost (2/2) -2016-05-24 17:20:58 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 17:20:58 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.031935 s -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=302594, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=302666, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:58 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[86] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=302804, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(2973) called with curMem=308324, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[86] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 17:20:58 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 9 ms on localhost (1/2) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 9 ms on localhost (2/2) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 17:20:58 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 17:20:58 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.029032 s -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=311297, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=311369, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:58 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[88] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=311507, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=317027, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[88] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 17:20:58 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 7 ms on localhost (1/2) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 8 ms on localhost (2/2) -2016-05-24 17:20:58 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 17:20:58 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.021376 s -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=319998, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=320070, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:58 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[90] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=320208, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=325728, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[90] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 17:20:58 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-24 17:20:58 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 13 ms on localhost (1/2) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 11 ms on localhost (2/2) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 17:20:58 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 17:20:58 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.044789 s -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=311506, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=311578, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:58 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[92] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=311716, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=317236, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[92] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 17:20:58 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 6 ms on localhost (1/2) -2016-05-24 17:20:58 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 12 ms on localhost (2/2) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 17:20:58 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 17:20:58 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.029415 s -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=311291, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=311363, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 91 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:58 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:58 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[94] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=294098, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=299618, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[94] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 17:20:58 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 13 ms on localhost (1/2) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 15 ms on localhost (2/2) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 17:20:58 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 17:20:58 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.033281 s -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=302590, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=302662, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 93 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:58 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Got job 47 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:58 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[96] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=285397, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=290917, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 47 (MapPartitionsRDD[96] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 2 tasks -2016-05-24 17:20:58 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 93, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 94, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 93) -2016-05-24 17:20:58 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 94) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 94). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 93). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 93) in 10 ms on localhost (1/2) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 94) in 9 ms on localhost (2/2) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-24 17:20:58 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 17:20:58 INFO DAGScheduler:59 - Job 47 finished: treeAggregate at GradientDescent.scala:225, took 0.027036 s -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=282216, maxMem=766075207 -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276768, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 95 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Got job 48 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:58 INFO DAGScheduler:59 - Final stage: ResultStage 48(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[98] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:58 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=253774, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=256322, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 48 (MapPartitionsRDD[98] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 2 tasks -2016-05-24 17:20:58 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 95, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 96, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 96) -2016-05-24 17:20:58 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 95) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 96). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 96) in 9 ms on localhost (1/2) -2016-05-24 17:20:58 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 95). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 95) in 13 ms on localhost (2/2) -2016-05-24 17:20:58 INFO DAGScheduler:59 - ResultStage 48 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-24 17:20:58 INFO DAGScheduler:59 - Job 48 finished: treeAggregate at GradientDescent.scala:225, took 0.035282 s -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=256112, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=256184, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 97 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:58 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Got job 49 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:58 INFO DAGScheduler:59 - Final stage: ResultStage 49(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[100] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250592, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=256112, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 49 (MapPartitionsRDD[100] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 2 tasks -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 97, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 98, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 98) -2016-05-24 17:20:58 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 97) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 97). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 97) in 18 ms on localhost (1/2) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 98). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 98) in 22 ms on localhost (2/2) -2016-05-24 17:20:58 INFO DAGScheduler:59 - ResultStage 49 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 17:20:58 INFO DAGScheduler:59 - Job 49 finished: treeAggregate at GradientDescent.scala:225, took 0.044085 s -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250590, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250662, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 99 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Got job 50 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:58 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[102] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250590, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(2974) called with curMem=256110, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 50 (MapPartitionsRDD[102] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 2 tasks -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-24 17:20:58 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-24 17:20:58 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 16 ms on localhost (1/2) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 27 ms on localhost (2/2) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-24 17:20:58 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 17:20:58 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at GradientDescent.scala:225, took 0.048468 s -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241890, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241962, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 101 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:51267 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:20:58 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 17:20:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:51267 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Got job 51 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:58 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[104] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233189, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=238709, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 51 (MapPartitionsRDD[104] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-24 17:20:58 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 7 ms on localhost (1/2) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 7 ms on localhost (2/2) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-24 17:20:58 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 17:20:58 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at GradientDescent.scala:225, took 0.022765 s -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241679, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241751, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 103 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Got job 52 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:58 INFO DAGScheduler:59 - Final stage: ResultStage 52(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting ResultStage 52 (MapPartitionsRDD[106] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=241889, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=247409, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 104 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 52 (MapPartitionsRDD[106] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 103, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 104, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:58 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 103) -2016-05-24 17:20:58 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 104) -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:58 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 103). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 104). 2423 bytes result sent to driver -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 103) in 9 ms on localhost (1/2) -2016-05-24 17:20:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 104) in 8 ms on localhost (2/2) -2016-05-24 17:20:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-24 17:20:58 INFO DAGScheduler:59 - ResultStage 52 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 17:20:58 INFO DAGScheduler:59 - Job 52 finished: treeAggregate at GradientDescent.scala:225, took 0.022452 s -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250380, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250452, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:58 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:58 INFO SparkContext:59 - Created broadcast 105 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:58 INFO DAGScheduler:59 - Got job 53 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:58 INFO DAGScheduler:59 - Final stage: ResultStage 53(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:58 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[108] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250590, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:58 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=256110, maxMem=766075207 -2016-05-24 17:20:58 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:59 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:59 INFO SparkContext:59 - Created broadcast 106 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[108] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:59 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-05-24 17:20:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 105, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 106, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:59 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 105) -2016-05-24 17:20:59 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 106) -2016-05-24 17:20:59 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:59 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:59 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 105). 2423 bytes result sent to driver -2016-05-24 17:20:59 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 106). 2423 bytes result sent to driver -2016-05-24 17:20:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 105) in 7 ms on localhost (1/2) -2016-05-24 17:20:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 106) in 7 ms on localhost (2/2) -2016-05-24 17:20:59 INFO DAGScheduler:59 - ResultStage 53 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 17:20:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-24 17:20:59 INFO DAGScheduler:59 - Job 53 finished: treeAggregate at GradientDescent.scala:225, took 0.017510 s -2016-05-24 17:20:59 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=259082, maxMem=766075207 -2016-05-24 17:20:59 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:59 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=259154, maxMem=766075207 -2016-05-24 17:20:59 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:59 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:59 INFO SparkContext:59 - Created broadcast 107 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:59 INFO DAGScheduler:59 - Got job 54 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:59 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:59 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[110] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:59 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=259292, maxMem=766075207 -2016-05-24 17:20:59 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:59 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=264812, maxMem=766075207 -2016-05-24 17:20:59 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:59 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:59 INFO SparkContext:59 - Created broadcast 108 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 54 (MapPartitionsRDD[110] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:59 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 2 tasks -2016-05-24 17:20:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:59 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 107) -2016-05-24 17:20:59 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 108) -2016-05-24 17:20:59 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:59 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:59 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 107). 2423 bytes result sent to driver -2016-05-24 17:20:59 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 108). 2423 bytes result sent to driver -2016-05-24 17:20:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 108) in 10 ms on localhost (1/2) -2016-05-24 17:20:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 107) in 12 ms on localhost (2/2) -2016-05-24 17:20:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-24 17:20:59 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 17:20:59 INFO DAGScheduler:59 - Job 54 finished: treeAggregate at GradientDescent.scala:225, took 0.022969 s -2016-05-24 17:20:59 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=267784, maxMem=766075207 -2016-05-24 17:20:59 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:59 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=267856, maxMem=766075207 -2016-05-24 17:20:59 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:59 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:59 INFO SparkContext:59 - Created broadcast 109 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:59 INFO DAGScheduler:59 - Got job 55 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:59 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:59 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[112] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:59 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=267994, maxMem=766075207 -2016-05-24 17:20:59 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:59 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=273514, maxMem=766075207 -2016-05-24 17:20:59 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:59 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:59 INFO SparkContext:59 - Created broadcast 110 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 55 (MapPartitionsRDD[112] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:59 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 2 tasks -2016-05-24 17:20:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 109, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 110, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:59 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 110) -2016-05-24 17:20:59 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 109) -2016-05-24 17:20:59 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:59 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:59 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 109). 2423 bytes result sent to driver -2016-05-24 17:20:59 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 110). 2423 bytes result sent to driver -2016-05-24 17:20:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 109) in 7 ms on localhost (1/2) -2016-05-24 17:20:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 110) in 6 ms on localhost (2/2) -2016-05-24 17:20:59 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 17:20:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-24 17:20:59 INFO DAGScheduler:59 - Job 55 finished: treeAggregate at GradientDescent.scala:225, took 0.026506 s -2016-05-24 17:20:59 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276484, maxMem=766075207 -2016-05-24 17:20:59 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:59 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276556, maxMem=766075207 -2016-05-24 17:20:59 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:59 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:59 INFO SparkContext:59 - Created broadcast 111 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:59 INFO DAGScheduler:59 - Got job 56 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:59 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:59 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[114] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:59 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=276694, maxMem=766075207 -2016-05-24 17:20:59 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:59 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=282214, maxMem=766075207 -2016-05-24 17:20:59 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:59 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:59 INFO SparkContext:59 - Created broadcast 112 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 56 (MapPartitionsRDD[114] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:59 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 2 tasks -2016-05-24 17:20:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 111, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 112, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:59 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 111) -2016-05-24 17:20:59 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 112) -2016-05-24 17:20:59 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:59 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:59 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 111). 2423 bytes result sent to driver -2016-05-24 17:20:59 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 112). 2423 bytes result sent to driver -2016-05-24 17:20:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 111) in 10 ms on localhost (1/2) -2016-05-24 17:20:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 112) in 10 ms on localhost (2/2) -2016-05-24 17:20:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-24 17:20:59 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 17:20:59 INFO DAGScheduler:59 - Job 56 finished: treeAggregate at GradientDescent.scala:225, took 0.027111 s -2016-05-24 17:20:59 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285186, maxMem=766075207 -2016-05-24 17:20:59 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:20:59 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285258, maxMem=766075207 -2016-05-24 17:20:59 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:20:59 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:51267 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:20:59 INFO SparkContext:59 - Created broadcast 113 from broadcast at GradientDescent.scala:221 -2016-05-24 17:20:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:20:59 INFO DAGScheduler:59 - Got job 57 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:20:59 INFO DAGScheduler:59 - Final stage: ResultStage 57(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:20:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:20:59 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[116] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:20:59 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=285396, maxMem=766075207 -2016-05-24 17:20:59 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:20:59 INFO MemoryStore:59 - ensureFreeSpace(2973) called with curMem=290916, maxMem=766075207 -2016-05-24 17:20:59 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:20:59 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:51267 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:20:59 INFO SparkContext:59 - Created broadcast 114 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:20:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 57 (MapPartitionsRDD[116] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:20:59 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 2 tasks -2016-05-24 17:20:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 113, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 114, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:20:59 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 113) -2016-05-24 17:20:59 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 114) -2016-05-24 17:20:59 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:20:59 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:20:59 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 114). 2423 bytes result sent to driver -2016-05-24 17:20:59 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 113). 2423 bytes result sent to driver -2016-05-24 17:20:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 114) in 6 ms on localhost (1/2) -2016-05-24 17:20:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 113) in 7 ms on localhost (2/2) -2016-05-24 17:20:59 INFO DAGScheduler:59 - ResultStage 57 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 17:20:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-24 17:20:59 INFO DAGScheduler:59 - Job 57 finished: treeAggregate at GradientDescent.scala:225, took 0.022433 s -2016-05-24 17:20:59 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 0.6105671927144823, 0.6097961496848042, 0.6090406315074238, 0.6083000889873139, 0.6075740019440139, 0.6068618771610624, 0.6061632465163228, 0.6054776652741959, 0.6048047105229598, 0.604143979742525 -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 17:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 17:20:59 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 17:20:59 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 17:20:59 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 17:20:59 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 17:20:59 INFO BlockManager:59 - BlockManager stopped -2016-05-24 17:20:59 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 17:20:59 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 17:20:59 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 17:20:59 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 17:20:59 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-ffd773bd-e351-4ba5-b6a4-8d869f2d4760 -2016-05-24 17:20:59 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 17:20:59 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 17:21:35 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 17:21:36 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 17:21:36 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 17:21:36 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 17:21:36 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 17:21:37 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 17:21:37 INFO Remoting:74 - Starting remoting -2016-05-24 17:21:38 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:51294] -2016-05-24 17:21:38 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51294. -2016-05-24 17:21:38 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 17:21:38 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 17:21:38 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-f7a9b6b7-94b9-4492-ab1f-ac4254a3aad2 -2016-05-24 17:21:38 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 17:21:39 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-60ce8b78-3b5b-4dec-adca-7204cca2958d\httpd-4e8b5f24-c303-4519-adf9-8fb0e218d739 -2016-05-24 17:21:39 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 17:21:39 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 17:21:39 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51295 -2016-05-24 17:21:39 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51295. -2016-05-24 17:21:39 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 17:21:39 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 17:21:39 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 17:21:39 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 17:21:39 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 17:21:39 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 17:21:39 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 17:21:40 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51315. -2016-05-24 17:21:40 INFO NettyBlockTransferService:59 - Server created on 51315 -2016-05-24 17:21:40 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 17:21:40 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51315 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51315) -2016-05-24 17:21:40 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 17:21:41 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 17:21:41 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 17:21:42 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 17:21:42 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 17:21:42 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51315 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 17:21:42 INFO SparkContext:59 - Created broadcast 0 from textFile at LinearRegression2Learning.scala:21 -2016-05-24 17:21:43 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 17:21:43 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 17:21:43 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 17:21:43 INFO DAGScheduler:59 - Got job 0 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 17:21:43 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 17:21:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:43 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 17:21:43 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=171580, maxMem=766075207 -2016-05-24 17:21:43 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-24 17:21:43 INFO MemoryStore:59 - ensureFreeSpace(2182) called with curMem=175316, maxMem=766075207 -2016-05-24 17:21:43 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 17:21:43 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51315 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 17:21:43 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:43 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 17:21:43 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-24 17:21:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 17:21:44 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 17:21:44 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 17:21:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/ridge-data/lpsa2.data:0+19 -2016-05-24 17:21:44 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 17:21:44 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 17:21:44 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 17:21:44 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 17:21:44 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 17:21:44 INFO MemoryStore:59 - ensureFreeSpace(320) called with curMem=177498, maxMem=766075207 -2016-05-24 17:21:44 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 320.0 B, free 730.4 MB) -2016-05-24 17:21:44 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:51315 (size: 320.0 B, free: 730.6 MB) -2016-05-24 17:21:44 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2611 bytes result sent to driver -2016-05-24 17:21:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 146 ms on localhost (1/1) -2016-05-24 17:21:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 17:21:44 INFO DAGScheduler:59 - ResultStage 0 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.169 s -2016-05-24 17:21:44 INFO DAGScheduler:59 - Job 0 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.273924 s -2016-05-24 17:21:44 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 17:21:44 INFO DAGScheduler:59 - Got job 1 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 17:21:44 INFO DAGScheduler:59 - Final stage: ResultStage 1(count at GradientDescent.scala:195) -2016-05-24 17:21:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:44 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 17:21:44 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=177818, maxMem=766075207 -2016-05-24 17:21:44 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 17:21:44 INFO MemoryStore:59 - ensureFreeSpace(2020) called with curMem=181258, maxMem=766075207 -2016-05-24 17:21:44 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2020.0 B, free 730.4 MB) -2016-05-24 17:21:44 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51315 (size: 2020.0 B, free: 730.6 MB) -2016-05-24 17:21:44 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 17:21:44 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 17:21:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 17:21:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 17:21:44 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-24 17:21:44 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-24 17:21:44 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:44 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 17:21:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/ridge-data/lpsa2.data:19+19 -2016-05-24 17:21:44 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-05-24 17:21:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 26 ms on localhost (1/2) -2016-05-24 17:21:44 INFO MemoryStore:59 - ensureFreeSpace(168) called with curMem=183278, maxMem=766075207 -2016-05-24 17:21:44 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 168.0 B, free 730.4 MB) -2016-05-24 17:21:44 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:51315 (size: 168.0 B, free: 730.6 MB) -2016-05-24 17:21:44 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2662 bytes result sent to driver -2016-05-24 17:21:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 32 ms on localhost (2/2) -2016-05-24 17:21:44 INFO DAGScheduler:59 - ResultStage 1 (count at GradientDescent.scala:195) finished in 0.037 s -2016-05-24 17:21:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 17:21:44 INFO DAGScheduler:59 - Job 1 finished: count at GradientDescent.scala:195, took 0.074777 s -2016-05-24 17:21:44 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 17:21:44 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 17:21:44 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=183446, maxMem=766075207 -2016-05-24 17:21:44 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:44 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=183518, maxMem=766075207 -2016-05-24 17:21:44 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:44 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:44 INFO SparkContext:59 - Created broadcast 3 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Got job 2 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:45 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=183656, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=189176, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 17:21:45 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:45 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 23 ms on localhost (1/2) -2016-05-24 17:21:45 INFO DAGScheduler:59 - ResultStage 2 (treeAggregate at GradientDescent.scala:225) finished in 0.025 s -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 25 ms on localhost (2/2) -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 17:21:45 INFO DAGScheduler:59 - Job 2 finished: treeAggregate at GradientDescent.scala:225, took 0.046145 s -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=192147, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=192219, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Got job 3 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:45 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=192357, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=197877, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 17:21:45 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:45 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:45 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 27 ms on localhost (1/2) -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 25 ms on localhost (2/2) -2016-05-24 17:21:45 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 17:21:45 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at GradientDescent.scala:225, took 0.073151 s -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:51315 in memory (size: 2020.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:51315 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=180770, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=180842, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:45 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[10] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=180980, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=186500, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[10] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 17:21:45 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:45 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 23 ms on localhost (1/2) -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 22 ms on localhost (2/2) -2016-05-24 17:21:45 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 17:21:45 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.048444 s -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=189471, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=189543, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:45 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[12] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=189681, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=195201, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[12] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 17:21:45 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:45 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 18 ms on localhost (1/2) -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 22 ms on localhost (2/2) -2016-05-24 17:21:45 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 17:21:45 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.045774 s -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=198170, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=198242, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:45 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[14] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=198380, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=203900, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[14] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 17:21:45 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:45 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 21 ms on localhost (1/2) -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 24 ms on localhost (2/2) -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 17:21:45 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 17:21:45 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.044738 s -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=206871, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=206943, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:45 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[16] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=207081, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=212601, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[16] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 17:21:45 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:45 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 19 ms on localhost (1/2) -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 18 ms on localhost (2/2) -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 17:21:45 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 17:21:45 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.039729 s -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=215564, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=215636, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:45 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[18] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=215774, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=221294, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[18] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 17:21:45 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:45 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 21 ms on localhost (1/2) -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 25 ms on localhost (2/2) -2016-05-24 17:21:45 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 17:21:45 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.047578 s -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224265, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224337, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:45 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[20] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224475, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=229995, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[20] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 17:21:45 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:45 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 19 ms on localhost (1/2) -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 22 ms on localhost (2/2) -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 17:21:45 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 17:21:45 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.040966 s -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=232966, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233038, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:45 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[22] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233176, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=238696, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[22] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:45 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 17:21:45 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 21 ms on localhost (1/2) -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:45 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 36 ms on localhost (2/2) -2016-05-24 17:21:45 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.036 s -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 17:21:45 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.060597 s -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241668, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241740, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:45 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[24] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233386, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=238696, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:45 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[24] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 17:21:45 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:45 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 21 ms on localhost (1/2) -2016-05-24 17:21:45 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 26 ms on localhost (2/2) -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 17:21:45 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.029 s -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.069494 s -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224264, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=221373, maxMem=766075207 -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:45 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[26] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=198591, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=204039, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[26] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 17:21:45 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:45 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:45 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 17:21:45 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 20 ms on localhost (1/2) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.027 s -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 26 ms on localhost (2/2) -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 17:21:45 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.054034 s -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=189470, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=189542, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:45 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[28] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=189680, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(2973) called with curMem=195200, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[28] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:45 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 17:21:45 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:45 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 2423 bytes result sent to driver -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 18 ms on localhost (1/2) -2016-05-24 17:21:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 22 ms on localhost (2/2) -2016-05-24 17:21:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 17:21:45 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 17:21:45 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.039010 s -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=198173, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=198245, maxMem=766075207 -2016-05-24 17:21:45 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:45 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:45 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:46 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[30] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=198383, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(2976) called with curMem=203903, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[30] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 17:21:46 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:46 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:46 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 23 ms on localhost (1/2) -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 29 ms on localhost (2/2) -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 17:21:46 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.033 s -2016-05-24 17:21:46 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.069160 s -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=206879, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=206951, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:46 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[32] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=207089, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=212609, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[32] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 17:21:46 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:46 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:46 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 27 ms on localhost (1/2) -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 34 ms on localhost (2/2) -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 17:21:46 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.037 s -2016-05-24 17:21:46 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.058670 s -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=215581, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=215653, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:46 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[34] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=215791, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=221311, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[34] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 17:21:46 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:46 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 16 ms on localhost (1/2) -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 18 ms on localhost (2/2) -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 17:21:46 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 17:21:46 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.045570 s -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224283, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224355, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:46 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[36] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224493, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=230013, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[36] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 17:21:46 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:46 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 22 ms on localhost (1/2) -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 22 ms on localhost (2/2) -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 17:21:46 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 17:21:46 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.045109 s -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=232984, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233056, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:46 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[38] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233194, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=238714, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[38] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 17:21:46 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:46 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 17 ms on localhost (1/2) -2016-05-24 17:21:46 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 25 ms on localhost (2/2) -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 17:21:46 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 17:21:46 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.061336 s -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241686, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241758, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:46 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[40] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=241896, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=247416, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[40] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 17:21:46 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:46 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 13 ms on localhost (1/2) -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 12 ms on localhost (2/2) -2016-05-24 17:21:46 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 17:21:46 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.141257 s -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250388, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250460, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:46 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[42] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250598, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=256118, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[42] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 17:21:46 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:46 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 26 ms on localhost (1/2) -2016-05-24 17:21:46 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 40 ms on localhost (2/2) -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 17:21:46 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.042 s -2016-05-24 17:21:46 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.066439 s -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=259090, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=259162, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:46 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[44] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=259300, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=264820, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[44] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 17:21:46 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:46 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 19 ms on localhost (1/2) -2016-05-24 17:21:46 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 28 ms on localhost (2/2) -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 17:21:46 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.031 s -2016-05-24 17:21:46 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.051082 s -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=267792, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=267864, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:46 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[46] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=268002, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=273522, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[46] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 17:21:46 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:46 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 12 ms on localhost (1/2) -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 16 ms on localhost (2/2) -2016-05-24 17:21:46 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 17:21:46 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.047509 s -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276494, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276566, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:46 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[48] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=276704, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=282224, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[48] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 17:21:46 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:46 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 12 ms on localhost (1/2) -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 14 ms on localhost (2/2) -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 17:21:46 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 17:21:46 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.032241 s -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285196, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285268, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:46 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[50] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=285406, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=290926, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:46 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:46 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[50] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:46 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 17:21:46 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:46 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 2423 bytes result sent to driver -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 13 ms on localhost (1/2) -2016-05-24 17:21:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 13 ms on localhost (2/2) -2016-05-24 17:21:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 17:21:46 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 17:21:46 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.035026 s -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=293897, maxMem=766075207 -2016-05-24 17:21:46 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=293969, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:47 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[52] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=294107, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=299627, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[52] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 17:21:47 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:47 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 13 ms on localhost (1/2) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 16 ms on localhost (2/2) -2016-05-24 17:21:47 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 17:21:47 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.040940 s -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=302596, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=302668, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:47 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[54] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=302806, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=308326, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[54] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 17:21:47 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:47 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 13 ms on localhost (1/2) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 16 ms on localhost (2/2) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 17:21:47 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 17:21:47 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.035236 s -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=311296, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=311368, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:47 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[56] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=311506, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=317026, maxMem=766075207 -2016-05-24 17:21:47 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[56] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 17:21:47 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 17:21:47 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:47 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 12 ms on localhost (1/2) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 15 ms on localhost (2/2) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 17:21:47 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 17:21:47 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.053428 s -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=308328, maxMem=766075207 -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=302880, maxMem=766075207 -2016-05-24 17:21:47 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:47 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[58] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=285615, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=291135, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[58] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:47 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 17:21:47 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:47 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 19 ms on localhost (1/2) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 20 ms on localhost (2/2) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 17:21:47 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 17:21:47 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.053154 s -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285195, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285267, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:47 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:47 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[60] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=276775, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=282223, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[60] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 17:21:47 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:47 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 14 ms on localhost (1/2) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 16 ms on localhost (2/2) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.033 s -2016-05-24 17:21:47 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.068683 s -2016-05-24 17:21:47 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=268000, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=267934, maxMem=766075207 -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:47 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:47 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[62] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=259298, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=264818, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[62] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:47 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 12 ms on localhost (1/2) -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:47 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 26 ms on localhost (2/2) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 17:21:47 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.032 s -2016-05-24 17:21:47 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.056291 s -2016-05-24 17:21:47 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=259297, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=259369, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:47 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[64] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250595, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=256115, maxMem=766075207 -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[64] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 17:21:47 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:47 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 18 ms on localhost (1/2) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 20 ms on localhost (2/2) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 17:21:47 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.046652 s -2016-05-24 17:21:47 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241888, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241750, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:47 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 17:21:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:47 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:47 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[66] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=230213, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=235733, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:47 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[66] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 17:21:47 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:47 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 9 ms on localhost (1/2) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 11 ms on localhost (2/2) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 17:21:47 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 17:21:47 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.030161 s -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224275, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224347, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:47 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[68] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224485, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=230005, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[68] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 17:21:47 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:47 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 8 ms on localhost (1/2) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 8 ms on localhost (2/2) -2016-05-24 17:21:47 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 17:21:47 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.023568 s -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=232977, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233049, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:47 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[70] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233187, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=238707, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[70] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 17:21:47 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:47 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 6 ms on localhost (1/2) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 8 ms on localhost (2/2) -2016-05-24 17:21:47 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 17:21:47 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.018104 s -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241678, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241750, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:47 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[72] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=241888, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=247408, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[72] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 17:21:47 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:47 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 7 ms on localhost (1/2) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 7 ms on localhost (2/2) -2016-05-24 17:21:47 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 17:21:47 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.061338 s -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250380, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250452, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:47 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[74] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250590, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=256110, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[74] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:47 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 12 ms on localhost (1/2) -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:47 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 18 ms on localhost (2/2) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 17:21:47 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 17:21:47 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.031451 s -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=259081, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=259153, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:47 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[76] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=259291, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=264811, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[76] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 17:21:47 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:47 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 9 ms on localhost (1/2) -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:47 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 11 ms on localhost (2/2) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 17:21:47 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 17:21:47 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.035565 s -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=267783, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=267855, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:47 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[78] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=267993, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=273513, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[78] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 17:21:47 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:47 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:47 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 14 ms on localhost (1/2) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 16 ms on localhost (2/2) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 17:21:47 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 17:21:47 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.031368 s -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276484, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276556, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:47 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[80] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=276694, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=282214, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[80] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 17:21:47 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:47 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 8 ms on localhost (1/2) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 10 ms on localhost (2/2) -2016-05-24 17:21:47 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 17:21:47 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.027612 s -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285185, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285257, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:47 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[82] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=285395, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=290915, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:47 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:47 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[82] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:47 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 17:21:47 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:47 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 2423 bytes result sent to driver -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 8 ms on localhost (1/2) -2016-05-24 17:21:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 8 ms on localhost (2/2) -2016-05-24 17:21:47 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:21:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 17:21:47 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.026393 s -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=293887, maxMem=766075207 -2016-05-24 17:21:47 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=293959, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:48 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[84] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=294097, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(2974) called with curMem=299617, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[84] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 17:21:48 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:48 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 9 ms on localhost (1/2) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 9 ms on localhost (2/2) -2016-05-24 17:21:48 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 17:21:48 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.025383 s -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=302591, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=302663, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:48 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[86] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=302801, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(2973) called with curMem=308321, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[86] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 17:21:48 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:48 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 9 ms on localhost (1/2) -2016-05-24 17:21:48 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 15 ms on localhost (2/2) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 17:21:48 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 17:21:48 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.035013 s -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=311294, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=311366, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:48 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[88] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=302801, maxMem=766075207 -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=308321, maxMem=766075207 -2016-05-24 17:21:48 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[88] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 17:21:48 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:48 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 9 ms on localhost (1/2) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 9 ms on localhost (2/2) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 17:21:48 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 17:21:48 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.034178 s -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=302379, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=293959, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 17:21:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:48 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[90] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=276695, maxMem=766075207 -2016-05-24 17:21:48 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=282077, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[90] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 17:21:48 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:48 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 12 ms on localhost (1/2) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 12 ms on localhost (2/2) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 17:21:48 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 17:21:48 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.030710 s -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276274, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276346, maxMem=766075207 -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:48 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:48 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[92] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=259081, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=264601, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[92] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 17:21:48 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 17:21:48 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:48 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 9 ms on localhost (1/2) -2016-05-24 17:21:48 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 17:21:48 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 10 ms on localhost (2/2) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 17:21:48 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.030107 s -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250379, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250451, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 91 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:48 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[94] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=241887, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=247407, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[94] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 17:21:48 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 17:21:48 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:48 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 12 ms on localhost (1/2) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 14 ms on localhost (2/2) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 17:21:48 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 17:21:48 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.046536 s -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=247197, maxMem=766075207 -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:48 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241749, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 93 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:48 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Got job 47 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:48 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[96] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=232977, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=238497, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 47 (MapPartitionsRDD[96] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 2 tasks -2016-05-24 17:21:48 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 93, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 94, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 94) -2016-05-24 17:21:48 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 93) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:48 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 93). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 94). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 93) in 13 ms on localhost (1/2) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 94) in 12 ms on localhost (2/2) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-24 17:21:48 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 17:21:48 INFO DAGScheduler:59 - Job 47 finished: treeAggregate at GradientDescent.scala:225, took 0.028889 s -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224276, maxMem=766075207 -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:48 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224348, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 95 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:51315 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:51315 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:48 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 17:21:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Got job 48 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:48 INFO DAGScheduler:59 - Final stage: ResultStage 48(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[98] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=215785, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=221305, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 48 (MapPartitionsRDD[98] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 2 tasks -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 95, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 96, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 96) -2016-05-24 17:21:48 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 95) -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:48 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 96). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 95). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 96) in 7 ms on localhost (1/2) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 95) in 8 ms on localhost (2/2) -2016-05-24 17:21:48 INFO DAGScheduler:59 - ResultStage 48 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-24 17:21:48 INFO DAGScheduler:59 - Job 48 finished: treeAggregate at GradientDescent.scala:225, took 0.022049 s -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224276, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224348, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 97 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Got job 49 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:48 INFO DAGScheduler:59 - Final stage: ResultStage 49(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[100] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224486, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=230006, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 49 (MapPartitionsRDD[100] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 2 tasks -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 97, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 98, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 97) -2016-05-24 17:21:48 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 98) -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:48 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 97). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 98). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 97) in 9 ms on localhost (1/2) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 98) in 9 ms on localhost (2/2) -2016-05-24 17:21:48 INFO DAGScheduler:59 - ResultStage 49 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-05-24 17:21:48 INFO DAGScheduler:59 - Job 49 finished: treeAggregate at GradientDescent.scala:225, took 0.021671 s -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=232976, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233048, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.6 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 99 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Got job 50 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:48 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[102] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233186, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(2974) called with curMem=238706, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 50 (MapPartitionsRDD[102] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 2 tasks -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-24 17:21:48 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:48 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 9 ms on localhost (1/2) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 12 ms on localhost (2/2) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-24 17:21:48 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 17:21:48 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at GradientDescent.scala:225, took 0.028984 s -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241680, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241752, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 101 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Got job 51 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:48 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[104] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=241890, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=247410, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 51 (MapPartitionsRDD[104] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-24 17:21:48 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:48 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 7 ms on localhost (1/2) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 7 ms on localhost (2/2) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-24 17:21:48 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:21:48 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at GradientDescent.scala:225, took 0.025746 s -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250380, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250452, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 103 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Got job 52 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:48 INFO DAGScheduler:59 - Final stage: ResultStage 52(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting ResultStage 52 (MapPartitionsRDD[106] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250590, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=256110, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 104 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 52 (MapPartitionsRDD[106] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 103, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 104, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 103) -2016-05-24 17:21:48 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 104) -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:48 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 104). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 103). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 104) in 6 ms on localhost (1/2) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 103) in 8 ms on localhost (2/2) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-24 17:21:48 INFO DAGScheduler:59 - ResultStage 52 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 17:21:48 INFO DAGScheduler:59 - Job 52 finished: treeAggregate at GradientDescent.scala:225, took 0.021405 s -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=259081, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=259153, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 105 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Got job 53 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:48 INFO DAGScheduler:59 - Final stage: ResultStage 53(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[108] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=259291, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=264811, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 106 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[108] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 105, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 106, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 105) -2016-05-24 17:21:48 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 106) -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:48 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 105). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:48 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 106). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 106) in 14 ms on localhost (1/2) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 105) in 15 ms on localhost (2/2) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-24 17:21:48 INFO DAGScheduler:59 - ResultStage 53 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 17:21:48 INFO DAGScheduler:59 - Job 53 finished: treeAggregate at GradientDescent.scala:225, took 0.030566 s -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=267782, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=267854, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 107 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Got job 54 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:48 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[110] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=267992, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=273512, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 108 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 54 (MapPartitionsRDD[110] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 2 tasks -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 107) -2016-05-24 17:21:48 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 108) -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:48 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 107). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 108). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 108) in 15 ms on localhost (1/2) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 107) in 17 ms on localhost (2/2) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-24 17:21:48 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 17:21:48 INFO DAGScheduler:59 - Job 54 finished: treeAggregate at GradientDescent.scala:225, took 0.030571 s -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276483, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276555, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 109 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Got job 55 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:48 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[112] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=276693, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=282213, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 110 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 55 (MapPartitionsRDD[112] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 2 tasks -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 109, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 110, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 110) -2016-05-24 17:21:48 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 109) -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:48 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 110). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 109). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 110) in 6 ms on localhost (1/2) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 109) in 8 ms on localhost (2/2) -2016-05-24 17:21:48 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-24 17:21:48 INFO DAGScheduler:59 - Job 55 finished: treeAggregate at GradientDescent.scala:225, took 0.028902 s -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285184, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285256, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 111 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Got job 56 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:48 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[114] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=285394, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=290914, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 112 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 56 (MapPartitionsRDD[114] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 2 tasks -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 111, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 112, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:48 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 111) -2016-05-24 17:21:48 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 112) -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:48 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 111). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 112). 2423 bytes result sent to driver -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 111) in 6 ms on localhost (1/2) -2016-05-24 17:21:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 112) in 6 ms on localhost (2/2) -2016-05-24 17:21:48 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 17:21:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-24 17:21:48 INFO DAGScheduler:59 - Job 56 finished: treeAggregate at GradientDescent.scala:225, took 0.019231 s -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=293886, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=293958, maxMem=766075207 -2016-05-24 17:21:48 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 17:21:48 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:51315 (size: 138.0 B, free: 730.5 MB) -2016-05-24 17:21:48 INFO SparkContext:59 - Created broadcast 113 from broadcast at GradientDescent.scala:221 -2016-05-24 17:21:49 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 17:21:49 INFO DAGScheduler:59 - Got job 57 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 17:21:49 INFO DAGScheduler:59 - Final stage: ResultStage 57(treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 17:21:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 17:21:49 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[116] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 17:21:49 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=294096, maxMem=766075207 -2016-05-24 17:21:49 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 17:21:49 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=299616, maxMem=766075207 -2016-05-24 17:21:49 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 17:21:49 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:51315 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 17:21:49 INFO SparkContext:59 - Created broadcast 114 from broadcast at DAGScheduler.scala:861 -2016-05-24 17:21:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 57 (MapPartitionsRDD[116] at treeAggregate at GradientDescent.scala:225) -2016-05-24 17:21:49 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 2 tasks -2016-05-24 17:21:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 113, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 114, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 17:21:49 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 113) -2016-05-24 17:21:49 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 114) -2016-05-24 17:21:49 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 17:21:49 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 17:21:49 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 114). 2423 bytes result sent to driver -2016-05-24 17:21:49 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 113). 2423 bytes result sent to driver -2016-05-24 17:21:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 114) in 6 ms on localhost (1/2) -2016-05-24 17:21:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 113) in 7 ms on localhost (2/2) -2016-05-24 17:21:49 INFO DAGScheduler:59 - ResultStage 57 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 17:21:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-24 17:21:49 INFO DAGScheduler:59 - Job 57 finished: treeAggregate at GradientDescent.scala:225, took 0.022367 s -2016-05-24 17:21:49 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 0.6105671927144823, 0.6097961496848042, 0.6090406315074238, 0.6083000889873139, 0.6075740019440139, 0.6068618771610624, 0.6061632465163228, 0.6054776652741959, 0.6048047105229598, 0.604143979742525 -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 17:21:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 17:21:49 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 17:21:49 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 17:21:49 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 17:21:49 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 17:21:49 INFO BlockManager:59 - BlockManager stopped -2016-05-24 17:21:49 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 17:21:49 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 17:21:49 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 17:21:49 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 17:21:49 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-60ce8b78-3b5b-4dec-adca-7204cca2958d -2016-05-24 17:21:49 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 17:21:49 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 17:25:42 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 17:25:44 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 17:25:44 INFO Remoting:74 - Starting remoting -2016-05-24 17:25:44 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:51517] -2016-05-24 17:25:44 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 17:25:44 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51518 -2016-05-24 17:25:45 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 17:25:45 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 17:25:48 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 17:25:48 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 17:25:48 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 17:25:48 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 17:25:48 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 17:25:48 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 17:25:48 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 17:25:49 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 17:25:49 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 17:25:57 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605241725_0079_m_000000_79' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/LinearRegression/20160524172553815/result/_temporary/0/task_201605241725_0079_m_000000 -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 17:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 17:25:58 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 17:25:58 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 20:22:31 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 20:22:32 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 20:22:32 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 20:22:32 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 20:22:32 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 20:22:35 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 20:22:35 INFO Remoting:74 - Starting remoting -2016-05-24 20:22:35 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53324] -2016-05-24 20:22:35 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53324. -2016-05-24 20:22:35 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 20:22:35 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 20:22:35 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-ffab34d3-f462-45d9-80d0-ca99edeebea6 -2016-05-24 20:22:35 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 20:22:35 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-caca4742-26fb-41a2-be6c-af6c64a6dbd1\httpd-9922612d-5e7b-453a-bce6-5d19e862af4b -2016-05-24 20:22:35 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 20:22:35 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 20:22:35 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53325 -2016-05-24 20:22:35 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53325. -2016-05-24 20:22:36 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 20:22:36 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 20:22:36 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 20:22:36 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 20:22:36 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 20:22:36 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 20:22:36 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 20:22:36 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53344. -2016-05-24 20:22:36 INFO NettyBlockTransferService:59 - Server created on 53344 -2016-05-24 20:22:36 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 20:22:36 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53344 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53344) -2016-05-24 20:22:36 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 20:22:38 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 20:22:38 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 20:22:38 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 20:22:38 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 20:22:38 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53344 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 20:22:38 INFO SparkContext:59 - Created broadcast 0 from textFile at LinearRegression3Learning.scala:21 -2016-05-24 20:22:40 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 20:22:40 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 20:22:40 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 20:22:40 INFO DAGScheduler:59 - Got job 0 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 20:22:40 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 20:22:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:22:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:22:40 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(3744) called with curMem=171580, maxMem=766075207 -2016-05-24 20:22:40 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-24 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(2174) called with curMem=175324, maxMem=766075207 -2016-05-24 20:22:40 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 20:22:40 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53344 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 20:22:40 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:22:40 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 20:22:40 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-24 20:22:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:22:40 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 20:22:40 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 20:22:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/linearRegression3.txt:0+52 -2016-05-24 20:22:40 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 20:22:40 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 20:22:40 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 20:22:40 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 20:22:40 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(400) called with curMem=177498, maxMem=766075207 -2016-05-24 20:22:40 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 400.0 B, free 730.4 MB) -2016-05-24 20:22:40 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:53344 (size: 400.0 B, free: 730.6 MB) -2016-05-24 20:22:40 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2611 bytes result sent to driver -2016-05-24 20:22:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 143 ms on localhost (1/1) -2016-05-24 20:22:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 20:22:40 INFO DAGScheduler:59 - ResultStage 0 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.167 s -2016-05-24 20:22:40 INFO DAGScheduler:59 - Job 0 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.253017 s -2016-05-24 20:22:40 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 20:22:40 INFO DAGScheduler:59 - Got job 1 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 20:22:40 INFO DAGScheduler:59 - Final stage: ResultStage 1(count at GradientDescent.scala:195) -2016-05-24 20:22:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:22:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:22:40 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=177898, maxMem=766075207 -2016-05-24 20:22:40 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(2012) called with curMem=181354, maxMem=766075207 -2016-05-24 20:22:40 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2012.0 B, free 730.4 MB) -2016-05-24 20:22:40 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53344 (size: 2012.0 B, free: 730.6 MB) -2016-05-24 20:22:40 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:22:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 20:22:40 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 20:22:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:22:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:22:40 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-24 20:22:40 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-24 20:22:40 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:22:40 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-05-24 20:22:40 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 20:22:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/linearRegression3.txt:52+53 -2016-05-24 20:22:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 21 ms on localhost (1/2) -2016-05-24 20:22:40 INFO MemoryStore:59 - ensureFreeSpace(400) called with curMem=183366, maxMem=766075207 -2016-05-24 20:22:40 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 400.0 B, free 730.4 MB) -2016-05-24 20:22:40 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:53344 (size: 400.0 B, free: 730.6 MB) -2016-05-24 20:22:40 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2662 bytes result sent to driver -2016-05-24 20:22:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 32 ms on localhost (2/2) -2016-05-24 20:22:40 INFO DAGScheduler:59 - ResultStage 1 (count at GradientDescent.scala:195) finished in 0.035 s -2016-05-24 20:22:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 20:22:40 INFO DAGScheduler:59 - Job 1 finished: count at GradientDescent.scala:195, took 0.068029 s -2016-05-24 20:22:40 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53344 in memory (size: 2012.0 B, free: 730.6 MB) -2016-05-24 20:22:40 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53344 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 20:22:41 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 20:22:41 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=172380, maxMem=766075207 -2016-05-24 20:22:41 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=172452, maxMem=766075207 -2016-05-24 20:22:41 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:22:41 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53344 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:22:41 INFO SparkContext:59 - Created broadcast 3 from broadcast at GradientDescent.scala:221 -2016-05-24 20:22:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:22:41 INFO DAGScheduler:59 - Got job 2 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:22:41 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:22:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:22:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:22:41 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=172590, maxMem=766075207 -2016-05-24 20:22:41 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=178118, maxMem=766075207 -2016-05-24 20:22:41 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:22:41 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53344 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:22:41 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:22:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:22:41 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 20:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:22:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:22:41 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 20:22:41 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 20:22:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:22:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:22:41 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2423 bytes result sent to driver -2016-05-24 20:22:41 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2423 bytes result sent to driver -2016-05-24 20:22:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 21 ms on localhost (1/2) -2016-05-24 20:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 23 ms on localhost (2/2) -2016-05-24 20:22:41 INFO DAGScheduler:59 - ResultStage 2 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 20:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 20:22:41 INFO DAGScheduler:59 - Job 2 finished: treeAggregate at GradientDescent.scala:225, took 0.041123 s -2016-05-24 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=181084, maxMem=766075207 -2016-05-24 20:22:41 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=181156, maxMem=766075207 -2016-05-24 20:22:41 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:22:41 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53344 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:22:41 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 20:22:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:22:41 INFO DAGScheduler:59 - Got job 3 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:22:41 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:22:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:22:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:22:41 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=181294, maxMem=766075207 -2016-05-24 20:22:41 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:22:41 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=186822, maxMem=766075207 -2016-05-24 20:22:41 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:22:41 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53344 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:22:41 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:22:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:22:41 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 20:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:22:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:22:41 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 20:22:41 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 20:22:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:22:41 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2423 bytes result sent to driver -2016-05-24 20:22:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:22:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 15 ms on localhost (1/2) -2016-05-24 20:22:41 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2423 bytes result sent to driver -2016-05-24 20:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 22 ms on localhost (2/2) -2016-05-24 20:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 20:22:41 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 20:22:41 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at GradientDescent.scala:225, took 0.037488 s -2016-05-24 20:22:41 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 3372.5, 1.8866811720935E13 -2016-05-24 20:22:41 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 20:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 20:22:41 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 20:22:41 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 20:22:41 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 20:22:41 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 20:22:41 INFO BlockManager:59 - BlockManager stopped -2016-05-24 20:22:41 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 20:22:41 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 20:22:41 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 20:22:41 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 20:22:41 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-caca4742-26fb-41a2-be6c-af6c64a6dbd1 -2016-05-24 20:22:41 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 20:22:41 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 20:23:49 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 20:23:49 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 20:23:50 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 20:23:50 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 20:23:50 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 20:23:50 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 20:23:50 INFO Remoting:74 - Starting remoting -2016-05-24 20:23:51 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53366] -2016-05-24 20:23:51 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53366. -2016-05-24 20:23:51 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 20:23:51 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 20:23:51 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-1a4ecef6-09ed-4e99-b10e-983c3c8bbed9 -2016-05-24 20:23:51 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 20:23:51 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-6b4546e4-dc5d-49f2-a93e-cfbc64fd9875\httpd-46fbcbf2-950b-4d21-b828-65d03f16d34c -2016-05-24 20:23:51 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 20:23:51 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 20:23:51 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53367 -2016-05-24 20:23:51 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53367. -2016-05-24 20:23:51 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 20:23:51 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 20:23:51 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 20:23:51 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 20:23:51 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 20:23:51 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 20:23:51 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 20:23:52 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53386. -2016-05-24 20:23:52 INFO NettyBlockTransferService:59 - Server created on 53386 -2016-05-24 20:23:52 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 20:23:52 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53386 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53386) -2016-05-24 20:23:52 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 20:23:52 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 20:23:52 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 20:23:52 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 20:23:52 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 20:23:52 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53386 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 20:23:52 INFO SparkContext:59 - Created broadcast 0 from textFile at LinearRegression3Learning.scala:21 -2016-05-24 20:23:54 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 20:23:54 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 20:23:54 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 20:23:54 INFO DAGScheduler:59 - Got job 0 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 20:23:54 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 20:23:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:23:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:23:54 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 20:23:54 INFO MemoryStore:59 - ensureFreeSpace(3744) called with curMem=171580, maxMem=766075207 -2016-05-24 20:23:54 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-24 20:23:54 INFO MemoryStore:59 - ensureFreeSpace(2174) called with curMem=175324, maxMem=766075207 -2016-05-24 20:23:54 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 20:23:54 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53386 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 20:23:54 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:23:54 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 20:23:54 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-24 20:23:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:23:54 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 20:23:54 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 20:23:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/linearRegression3.txt:0+52 -2016-05-24 20:23:54 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 20:23:54 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 20:23:54 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 20:23:54 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 20:23:54 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 20:23:54 INFO MemoryStore:59 - ensureFreeSpace(400) called with curMem=177498, maxMem=766075207 -2016-05-24 20:23:54 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 400.0 B, free 730.4 MB) -2016-05-24 20:23:54 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:53386 (size: 400.0 B, free: 730.6 MB) -2016-05-24 20:23:54 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2611 bytes result sent to driver -2016-05-24 20:23:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 114 ms on localhost (1/1) -2016-05-24 20:23:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 20:23:54 INFO DAGScheduler:59 - ResultStage 0 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.133 s -2016-05-24 20:23:54 INFO DAGScheduler:59 - Job 0 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.232431 s -2016-05-24 20:23:54 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 20:23:54 INFO DAGScheduler:59 - Got job 1 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 20:23:54 INFO DAGScheduler:59 - Final stage: ResultStage 1(count at GradientDescent.scala:195) -2016-05-24 20:23:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:23:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:23:54 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 20:23:54 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=177898, maxMem=766075207 -2016-05-24 20:23:54 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 20:23:54 INFO MemoryStore:59 - ensureFreeSpace(2012) called with curMem=181354, maxMem=766075207 -2016-05-24 20:23:54 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2012.0 B, free 730.4 MB) -2016-05-24 20:23:54 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53386 (size: 2012.0 B, free: 730.6 MB) -2016-05-24 20:23:54 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:23:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 20:23:54 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 20:23:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:23:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:23:54 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-24 20:23:54 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-24 20:23:54 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 20:23:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/linearRegression3.txt:52+53 -2016-05-24 20:23:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:23:54 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-05-24 20:23:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 31 ms on localhost (1/2) -2016-05-24 20:23:54 INFO MemoryStore:59 - ensureFreeSpace(400) called with curMem=183366, maxMem=766075207 -2016-05-24 20:23:54 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 400.0 B, free 730.4 MB) -2016-05-24 20:23:54 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:53386 (size: 400.0 B, free: 730.6 MB) -2016-05-24 20:23:54 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2662 bytes result sent to driver -2016-05-24 20:23:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 39 ms on localhost (2/2) -2016-05-24 20:23:54 INFO DAGScheduler:59 - ResultStage 1 (count at GradientDescent.scala:195) finished in 0.045 s -2016-05-24 20:23:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 20:23:54 INFO DAGScheduler:59 - Job 1 finished: count at GradientDescent.scala:195, took 0.079651 s -2016-05-24 20:23:55 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53386 in memory (size: 2012.0 B, free: 730.6 MB) -2016-05-24 20:23:55 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53386 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 20:23:55 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 20:23:55 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 20:23:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=172380, maxMem=766075207 -2016-05-24 20:23:55 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:23:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=172452, maxMem=766075207 -2016-05-24 20:23:55 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:23:55 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53386 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:23:55 INFO SparkContext:59 - Created broadcast 3 from broadcast at GradientDescent.scala:221 -2016-05-24 20:23:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:23:55 INFO DAGScheduler:59 - Got job 2 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:23:55 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:23:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:23:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:23:55 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:23:55 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=172590, maxMem=766075207 -2016-05-24 20:23:55 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:23:55 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=178118, maxMem=766075207 -2016-05-24 20:23:55 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:23:55 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53386 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:23:55 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:23:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:23:55 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 20:23:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:23:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:23:55 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 20:23:55 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 20:23:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:23:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:23:55 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2423 bytes result sent to driver -2016-05-24 20:23:55 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2423 bytes result sent to driver -2016-05-24 20:23:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 32 ms on localhost (1/2) -2016-05-24 20:23:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 32 ms on localhost (2/2) -2016-05-24 20:23:55 INFO DAGScheduler:59 - ResultStage 2 (treeAggregate at GradientDescent.scala:225) finished in 0.035 s -2016-05-24 20:23:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 20:23:55 INFO DAGScheduler:59 - Job 2 finished: treeAggregate at GradientDescent.scala:225, took 0.055385 s -2016-05-24 20:23:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=181084, maxMem=766075207 -2016-05-24 20:23:55 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:23:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=181156, maxMem=766075207 -2016-05-24 20:23:55 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:23:55 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53386 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:23:55 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 20:23:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:23:55 INFO DAGScheduler:59 - Got job 3 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:23:55 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:23:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:23:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:23:55 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:23:55 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=181294, maxMem=766075207 -2016-05-24 20:23:55 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:23:55 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=186822, maxMem=766075207 -2016-05-24 20:23:55 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:23:55 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53386 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:23:55 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:23:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:23:55 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 20:23:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:23:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:23:55 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 20:23:55 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 20:23:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:23:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:23:55 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2423 bytes result sent to driver -2016-05-24 20:23:55 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2423 bytes result sent to driver -2016-05-24 20:23:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 24 ms on localhost (1/2) -2016-05-24 20:23:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 27 ms on localhost (2/2) -2016-05-24 20:23:55 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at GradientDescent.scala:225) finished in 0.027 s -2016-05-24 20:23:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 20:23:55 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at GradientDescent.scala:225, took 0.046861 s -2016-05-24 20:23:55 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 3372.5, 1.8866811720935E13 -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 20:23:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 20:23:55 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 20:23:55 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 20:23:55 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 20:23:55 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 20:23:55 INFO BlockManager:59 - BlockManager stopped -2016-05-24 20:23:55 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 20:23:55 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 20:23:55 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 20:23:55 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 20:23:55 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-6b4546e4-dc5d-49f2-a93e-cfbc64fd9875 -2016-05-24 20:23:55 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 20:23:55 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 20:24:13 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 20:24:14 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 20:24:14 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 20:24:14 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 20:24:14 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 20:24:15 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 20:24:15 INFO Remoting:74 - Starting remoting -2016-05-24 20:24:15 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53408] -2016-05-24 20:24:15 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53408. -2016-05-24 20:24:15 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 20:24:15 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 20:24:15 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-8f13dd3f-386c-4feb-96e3-ee5450b60ca9 -2016-05-24 20:24:15 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 20:24:15 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-5b284d05-f3ec-4883-9271-527d46f98cf5\httpd-4abd4b26-b705-497b-974e-4a876dbaba63 -2016-05-24 20:24:15 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 20:24:16 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 20:24:16 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53409 -2016-05-24 20:24:16 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53409. -2016-05-24 20:24:16 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 20:24:16 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 20:24:16 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 20:24:16 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 20:24:16 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 20:24:16 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 20:24:16 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 20:24:16 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53428. -2016-05-24 20:24:16 INFO NettyBlockTransferService:59 - Server created on 53428 -2016-05-24 20:24:16 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 20:24:16 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53428 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53428) -2016-05-24 20:24:16 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 20:24:17 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 20:24:17 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 20:24:17 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 20:24:17 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 20:24:17 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53428 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 20:24:17 INFO SparkContext:59 - Created broadcast 0 from textFile at LinearRegression3Learning.scala:21 -2016-05-24 20:24:18 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 20:24:18 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 20:24:19 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 20:24:19 INFO DAGScheduler:59 - Got job 0 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 20:24:19 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 20:24:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:19 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 20:24:19 INFO MemoryStore:59 - ensureFreeSpace(3744) called with curMem=171580, maxMem=766075207 -2016-05-24 20:24:19 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-24 20:24:19 INFO MemoryStore:59 - ensureFreeSpace(2174) called with curMem=175324, maxMem=766075207 -2016-05-24 20:24:19 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 20:24:19 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53428 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 20:24:19 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 20:24:19 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-24 20:24:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:24:19 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 20:24:19 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 20:24:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/linearRegression3.txt:0+52 -2016-05-24 20:24:19 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 20:24:19 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 20:24:19 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 20:24:19 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 20:24:19 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 20:24:19 INFO MemoryStore:59 - ensureFreeSpace(400) called with curMem=177498, maxMem=766075207 -2016-05-24 20:24:19 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 400.0 B, free 730.4 MB) -2016-05-24 20:24:19 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:53428 (size: 400.0 B, free: 730.6 MB) -2016-05-24 20:24:19 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2611 bytes result sent to driver -2016-05-24 20:24:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 113 ms on localhost (1/1) -2016-05-24 20:24:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 20:24:19 INFO DAGScheduler:59 - ResultStage 0 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.135 s -2016-05-24 20:24:19 INFO DAGScheduler:59 - Job 0 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.228389 s -2016-05-24 20:24:19 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 20:24:19 INFO DAGScheduler:59 - Got job 1 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 20:24:19 INFO DAGScheduler:59 - Final stage: ResultStage 1(count at GradientDescent.scala:195) -2016-05-24 20:24:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:19 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 20:24:19 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=177898, maxMem=766075207 -2016-05-24 20:24:19 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 20:24:19 INFO MemoryStore:59 - ensureFreeSpace(2012) called with curMem=181354, maxMem=766075207 -2016-05-24 20:24:19 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2012.0 B, free 730.4 MB) -2016-05-24 20:24:19 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53428 (size: 2012.0 B, free: 730.6 MB) -2016-05-24 20:24:19 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 20:24:19 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 20:24:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:24:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:24:19 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-24 20:24:19 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-24 20:24:19 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 20:24:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/linearRegression3.txt:52+53 -2016-05-24 20:24:19 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:19 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-05-24 20:24:19 INFO MemoryStore:59 - ensureFreeSpace(400) called with curMem=183366, maxMem=766075207 -2016-05-24 20:24:19 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 400.0 B, free 730.4 MB) -2016-05-24 20:24:19 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:53428 (size: 400.0 B, free: 730.6 MB) -2016-05-24 20:24:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 31 ms on localhost (1/2) -2016-05-24 20:24:19 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2662 bytes result sent to driver -2016-05-24 20:24:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 36 ms on localhost (2/2) -2016-05-24 20:24:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 20:24:19 INFO DAGScheduler:59 - ResultStage 1 (count at GradientDescent.scala:195) finished in 0.040 s -2016-05-24 20:24:19 INFO DAGScheduler:59 - Job 1 finished: count at GradientDescent.scala:195, took 0.073062 s -2016-05-24 20:24:19 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 20:24:19 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 20:24:19 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=183766, maxMem=766075207 -2016-05-24 20:24:19 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:19 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=183838, maxMem=766075207 -2016-05-24 20:24:19 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:19 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53428 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:19 INFO SparkContext:59 - Created broadcast 3 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:19 INFO DAGScheduler:59 - Got job 2 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:19 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:19 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:19 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=183976, maxMem=766075207 -2016-05-24 20:24:19 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:19 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=189504, maxMem=766075207 -2016-05-24 20:24:19 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:19 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53428 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:19 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:19 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 20:24:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:19 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 20:24:19 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 20:24:19 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:19 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:19 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2423 bytes result sent to driver -2016-05-24 20:24:19 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2423 bytes result sent to driver -2016-05-24 20:24:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 22 ms on localhost (1/2) -2016-05-24 20:24:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 21 ms on localhost (2/2) -2016-05-24 20:24:19 INFO DAGScheduler:59 - ResultStage 2 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 20:24:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 20:24:19 INFO DAGScheduler:59 - Job 2 finished: treeAggregate at GradientDescent.scala:225, took 0.042966 s -2016-05-24 20:24:19 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=192474, maxMem=766075207 -2016-05-24 20:24:19 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:19 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=192546, maxMem=766075207 -2016-05-24 20:24:19 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:19 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53428 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:19 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:19 INFO DAGScheduler:59 - Got job 3 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:19 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:19 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:19 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=192684, maxMem=766075207 -2016-05-24 20:24:19 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:19 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=198212, maxMem=766075207 -2016-05-24 20:24:19 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:19 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53428 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:19 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:19 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 20:24:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:19 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 20:24:19 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 20:24:19 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:19 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:19 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2423 bytes result sent to driver -2016-05-24 20:24:19 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2423 bytes result sent to driver -2016-05-24 20:24:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 17 ms on localhost (1/2) -2016-05-24 20:24:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 20 ms on localhost (2/2) -2016-05-24 20:24:19 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 20:24:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 20:24:19 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at GradientDescent.scala:225, took 0.036715 s -2016-05-24 20:24:19 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 3372.5, 1.8866811720935E13 -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 20:24:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 20:24:19 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 20:24:20 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 20:24:20 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 20:24:20 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 20:24:20 INFO BlockManager:59 - BlockManager stopped -2016-05-24 20:24:20 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 20:24:20 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 20:24:20 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 20:24:20 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 20:24:20 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-5b284d05-f3ec-4883-9271-527d46f98cf5 -2016-05-24 20:24:20 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 20:24:20 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 20:24:40 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 20:24:40 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 20:24:41 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 20:24:41 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 20:24:41 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 20:24:41 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 20:24:42 INFO Remoting:74 - Starting remoting -2016-05-24 20:24:42 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53449] -2016-05-24 20:24:42 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53449. -2016-05-24 20:24:42 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 20:24:42 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 20:24:42 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-5919cf60-ba33-4317-b11a-7b1cd0ae425a -2016-05-24 20:24:42 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 20:24:42 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-07910bee-3543-4e7b-a4c9-04146473789c\httpd-a0be83ba-0e7f-4bca-a8bf-762e3d02726f -2016-05-24 20:24:42 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 20:24:42 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 20:24:42 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53450 -2016-05-24 20:24:42 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53450. -2016-05-24 20:24:42 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 20:24:42 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 20:24:42 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 20:24:42 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 20:24:42 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 20:24:42 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 20:24:42 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 20:24:43 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53469. -2016-05-24 20:24:43 INFO NettyBlockTransferService:59 - Server created on 53469 -2016-05-24 20:24:43 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 20:24:43 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53469 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53469) -2016-05-24 20:24:43 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 20:24:43 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 20:24:43 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 20:24:43 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 20:24:43 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 20:24:43 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53469 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 20:24:43 INFO SparkContext:59 - Created broadcast 0 from textFile at LinearRegression3Learning.scala:21 -2016-05-24 20:24:45 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 20:24:45 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 20:24:45 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 20:24:45 INFO DAGScheduler:59 - Got job 0 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 20:24:45 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 20:24:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:45 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 20:24:45 INFO MemoryStore:59 - ensureFreeSpace(3744) called with curMem=171580, maxMem=766075207 -2016-05-24 20:24:45 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-24 20:24:45 INFO MemoryStore:59 - ensureFreeSpace(2174) called with curMem=175324, maxMem=766075207 -2016-05-24 20:24:45 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 20:24:45 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53469 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 20:24:45 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:45 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 20:24:45 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-24 20:24:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:24:45 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 20:24:45 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 20:24:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/linearRegression3.txt:0+52 -2016-05-24 20:24:45 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 20:24:45 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 20:24:45 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 20:24:45 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 20:24:45 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 20:24:45 INFO MemoryStore:59 - ensureFreeSpace(400) called with curMem=177498, maxMem=766075207 -2016-05-24 20:24:45 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 400.0 B, free 730.4 MB) -2016-05-24 20:24:45 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:53469 (size: 400.0 B, free: 730.6 MB) -2016-05-24 20:24:45 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2611 bytes result sent to driver -2016-05-24 20:24:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 117 ms on localhost (1/1) -2016-05-24 20:24:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 20:24:45 INFO DAGScheduler:59 - ResultStage 0 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.139 s -2016-05-24 20:24:45 INFO DAGScheduler:59 - Job 0 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.237389 s -2016-05-24 20:24:45 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 20:24:45 INFO DAGScheduler:59 - Got job 1 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 20:24:45 INFO DAGScheduler:59 - Final stage: ResultStage 1(count at GradientDescent.scala:195) -2016-05-24 20:24:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:45 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 20:24:45 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=177898, maxMem=766075207 -2016-05-24 20:24:45 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 20:24:45 INFO MemoryStore:59 - ensureFreeSpace(2012) called with curMem=181354, maxMem=766075207 -2016-05-24 20:24:45 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2012.0 B, free 730.4 MB) -2016-05-24 20:24:45 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53469 (size: 2012.0 B, free: 730.6 MB) -2016-05-24 20:24:45 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 20:24:45 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 20:24:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:24:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:24:45 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-24 20:24:45 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-24 20:24:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:45 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 20:24:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/linearRegression3.txt:52+53 -2016-05-24 20:24:45 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-05-24 20:24:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 27 ms on localhost (1/2) -2016-05-24 20:24:45 INFO MemoryStore:59 - ensureFreeSpace(400) called with curMem=183366, maxMem=766075207 -2016-05-24 20:24:45 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 400.0 B, free 730.4 MB) -2016-05-24 20:24:45 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:53469 (size: 400.0 B, free: 730.6 MB) -2016-05-24 20:24:45 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2662 bytes result sent to driver -2016-05-24 20:24:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 34 ms on localhost (2/2) -2016-05-24 20:24:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 20:24:45 INFO DAGScheduler:59 - ResultStage 1 (count at GradientDescent.scala:195) finished in 0.039 s -2016-05-24 20:24:45 INFO DAGScheduler:59 - Job 1 finished: count at GradientDescent.scala:195, took 0.072165 s -2016-05-24 20:24:45 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53469 in memory (size: 2012.0 B, free: 730.6 MB) -2016-05-24 20:24:45 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53469 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 20:24:46 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 20:24:46 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=172380, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=172452, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 3 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Got job 2 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:46 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=172590, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=178118, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 20:24:46 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:46 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 29 ms on localhost (1/2) -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 26 ms on localhost (2/2) -2016-05-24 20:24:46 INFO DAGScheduler:59 - ResultStage 2 (treeAggregate at GradientDescent.scala:225) finished in 0.030 s -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 20:24:46 INFO DAGScheduler:59 - Job 2 finished: treeAggregate at GradientDescent.scala:225, took 0.054083 s -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=181083, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=181155, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Got job 3 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:46 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=181293, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=186821, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 20:24:46 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:46 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:46 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 15 ms on localhost (1/2) -2016-05-24 20:24:46 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 19 ms on localhost (2/2) -2016-05-24 20:24:46 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at GradientDescent.scala:225, took 0.035345 s -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=189786, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=189858, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:46 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[10] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=189996, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=195524, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[10] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 20:24:46 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:46 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 18 ms on localhost (1/2) -2016-05-24 20:24:46 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 24 ms on localhost (2/2) -2016-05-24 20:24:46 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.025 s -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 20:24:46 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.046486 s -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=198489, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=198561, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:46 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[12] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=198699, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=204227, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[12] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 20:24:46 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:46 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 15 ms on localhost (1/2) -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 18 ms on localhost (2/2) -2016-05-24 20:24:46 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 20:24:46 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.040209 s -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=207191, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=207263, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:46 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[14] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=207401, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=212929, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[14] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 20:24:46 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:46 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 17 ms on localhost (1/2) -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 17 ms on localhost (2/2) -2016-05-24 20:24:46 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 20:24:46 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.036292 s -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=215894, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=215966, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:46 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[16] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=216104, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=221632, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[16] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 20:24:46 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:46 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 12 ms on localhost (1/2) -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 15 ms on localhost (2/2) -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 20:24:46 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 20:24:46 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.035495 s -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224594, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224666, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:46 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[18] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=224804, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=230332, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[18] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 20:24:46 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:46 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 17 ms on localhost (1/2) -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 20 ms on localhost (2/2) -2016-05-24 20:24:46 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 20:24:46 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.037442 s -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=233298, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233370, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:46 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[20] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=230542, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=230542, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[20] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 20:24:46 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 20:24:46 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:46 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:46 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 19 ms on localhost (1/2) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:46 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 21 ms on localhost (2/2) -2016-05-24 20:24:46 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.073669 s -2016-05-24 20:24:46 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=215894, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=215966, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:46 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:46 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:46 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:46 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[22] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=190068, maxMem=766075207 -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=189996, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:46 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[22] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 20:24:46 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:46 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 16 ms on localhost (1/2) -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 15 ms on localhost (2/2) -2016-05-24 20:24:46 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 20:24:46 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.038430 s -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=189786, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=189858, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:46 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[24] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=189996, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=195524, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[24] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 20:24:46 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:46 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:46 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 11 ms on localhost (1/2) -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 15 ms on localhost (2/2) -2016-05-24 20:24:46 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 20:24:46 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.029878 s -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=198488, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=198560, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:46 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[26] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=198698, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=204226, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[26] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:46 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 20:24:46 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:46 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2423 bytes result sent to driver -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 13 ms on localhost (1/2) -2016-05-24 20:24:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 16 ms on localhost (2/2) -2016-05-24 20:24:46 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 20:24:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 20:24:46 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.037251 s -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=207192, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=207264, maxMem=766075207 -2016-05-24 20:24:46 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:46 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:46 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:46 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:46 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:46 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[28] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=207402, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=212930, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[28] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 20:24:47 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:47 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 11 ms on localhost (1/2) -2016-05-24 20:24:47 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 16 ms on localhost (2/2) -2016-05-24 20:24:47 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 20:24:47 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.030890 s -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=215896, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=215968, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:47 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[30] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=216106, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=221634, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[30] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 20:24:47 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:47 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 16 ms on localhost (1/2) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 18 ms on localhost (2/2) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 20:24:47 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 20:24:47 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.038301 s -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224605, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224677, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:47 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[32] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=224815, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=230343, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[32] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 20:24:47 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:47 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 14 ms on localhost (1/2) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 17 ms on localhost (2/2) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 20:24:47 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 20:24:47 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.038958 s -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=233309, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233381, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:47 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[34] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=233519, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=239047, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[34] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 20:24:47 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:47 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 13 ms on localhost (1/2) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 14 ms on localhost (2/2) -2016-05-24 20:24:47 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 20:24:47 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.029292 s -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=242013, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=242085, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:47 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[36] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=242223, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=247751, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[36] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 20:24:47 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:47 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 14 ms on localhost (1/2) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 15 ms on localhost (2/2) -2016-05-24 20:24:47 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 20:24:47 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.027906 s -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250716, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250788, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:47 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[38] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=250926, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=256454, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[38] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 20:24:47 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:47 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 17 ms on localhost (1/2) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 18 ms on localhost (2/2) -2016-05-24 20:24:47 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 20:24:47 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.034274 s -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=259420, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=259492, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:47 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[40] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=259630, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=265158, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[40] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 20:24:47 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:47 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:47 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 16 ms on localhost (1/2) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 16 ms on localhost (2/2) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 20:24:47 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 20:24:47 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.035628 s -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=268123, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=268195, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:47 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[42] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=268333, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=273861, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[42] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 20:24:47 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:47 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 16 ms on localhost (1/2) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 15 ms on localhost (2/2) -2016-05-24 20:24:47 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 20:24:47 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.034704 s -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276827, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276899, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:47 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[44] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=277037, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=282565, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[44] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 20:24:47 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:47 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 13 ms on localhost (1/2) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 13 ms on localhost (2/2) -2016-05-24 20:24:47 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 20:24:47 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.031475 s -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285530, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285602, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:47 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[46] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=285740, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=291268, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[46] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 20:24:47 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:47 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 13 ms on localhost (1/2) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 13 ms on localhost (2/2) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 20:24:47 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 20:24:47 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.033911 s -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=294234, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=294306, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:47 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[48] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=294444, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=299972, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[48] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 20:24:47 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:47 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 13 ms on localhost (1/2) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 15 ms on localhost (2/2) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 20:24:47 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 20:24:47 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.035165 s -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=302937, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=303009, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:47 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[50] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=303147, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=308675, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[50] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 20:24:47 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:47 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 13 ms on localhost (1/2) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 15 ms on localhost (2/2) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 20:24:47 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 20:24:47 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.035455 s -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=311639, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=311711, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:47 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[52] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=311849, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=317377, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[52] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 20:24:47 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:47 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 8 ms on localhost (1/2) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 9 ms on localhost (2/2) -2016-05-24 20:24:47 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 20:24:47 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.027440 s -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=320341, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=320413, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:47 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[54] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=320551, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=326079, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[54] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 20:24:47 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:47 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 14 ms on localhost (1/2) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 15 ms on localhost (2/2) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 20:24:47 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 20:24:47 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.028513 s -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=329042, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=329114, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:47 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:47 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[56] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=308885, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=308885, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[56] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 20:24:47 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:47 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 8 ms on localhost (1/2) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 10 ms on localhost (2/2) -2016-05-24 20:24:47 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 20:24:47 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.021797 s -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=294443, maxMem=766075207 -2016-05-24 20:24:47 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=294305, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:47 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:47 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[58] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=277108, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=282564, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[58] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 20:24:47 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 20:24:47 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:47 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 11 ms on localhost (1/2) -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:47 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 18 ms on localhost (2/2) -2016-05-24 20:24:47 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 20:24:47 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.030717 s -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=268333, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=268405, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 20:24:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:47 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[60] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=259630, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=256664, maxMem=766075207 -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:47 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[60] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 20:24:47 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:47 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:47 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 20:24:47 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 12 ms on localhost (1/2) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 12 ms on localhost (2/2) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 20:24:47 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 20:24:47 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.028299 s -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250787, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250787, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:47 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:47 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 20:24:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:47 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[62] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=233512, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=236074, maxMem=766075207 -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:47 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[62] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:47 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:47 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:47 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 2423 bytes result sent to driver -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 10 ms on localhost (1/2) -2016-05-24 20:24:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 12 ms on localhost (2/2) -2016-05-24 20:24:47 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 20:24:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 20:24:47 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.027973 s -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=230338, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:47 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224882, maxMem=766075207 -2016-05-24 20:24:47 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:47 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:47 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:47 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 20:24:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:47 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:47 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:47 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:47 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[64] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=213142, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:48 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=213142, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[64] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 6 ms on localhost (1/2) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 6 ms on localhost (2/2) -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.018881 s -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=215897, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=215969, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:48 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[66] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=216107, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=221635, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[66] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 5 ms on localhost (1/2) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 7 ms on localhost (2/2) -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.017776 s -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224600, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224672, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:48 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[68] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=224810, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=230338, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[68] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 6 ms on localhost (1/2) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 6 ms on localhost (2/2) -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.016431 s -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=233303, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233375, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:48 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[70] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=233513, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=239041, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[70] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 5 ms on localhost (1/2) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 5 ms on localhost (2/2) -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.014284 s -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=242007, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=242079, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:48 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[72] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=242217, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=247745, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[72] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 10 ms on localhost (1/2) -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 14 ms on localhost (2/2) -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.091435 s -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250710, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250782, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:48 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[74] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=250920, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=256448, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[74] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 5 ms on localhost (1/2) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 6 ms on localhost (2/2) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.015120 s -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=259414, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=259486, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:48 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[76] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=259624, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=265152, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[76] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 5 ms on localhost (1/2) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 5 ms on localhost (2/2) -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.014484 s -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=268117, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=268189, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:48 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[78] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=268327, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=273855, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[78] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 6 ms on localhost (1/2) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 5 ms on localhost (2/2) -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.014845 s -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276821, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276893, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:48 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[80] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=277031, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=282559, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[80] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 5 ms on localhost (1/2) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 6 ms on localhost (2/2) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.016406 s -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285524, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285596, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:48 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[82] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=285734, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=291262, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[82] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 6 ms on localhost (1/2) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 5 ms on localhost (2/2) -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.016080 s -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=294227, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=294299, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:48 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[84] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=294437, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2967) called with curMem=299965, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[84] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 5 ms on localhost (1/2) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 6 ms on localhost (2/2) -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.015739 s -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=302932, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=303004, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:48 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[86] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=303142, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2967) called with curMem=308670, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[86] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 9 ms on localhost (1/2) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 9 ms on localhost (2/2) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.021094 s -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=311637, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=311709, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:48 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[88] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=311847, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=317375, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[88] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 6 ms on localhost (1/2) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 7 ms on localhost (2/2) -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.018134 s -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=320341, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=320413, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:48 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[90] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=320551, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=326079, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[90] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 7 ms on localhost (1/2) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 15 ms on localhost (2/2) -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.033568 s -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=303143, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=303215, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:48 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[92] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=285946, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=291264, maxMem=766075207 -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[92] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 20:24:48 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 6 ms on localhost (1/2) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 7 ms on localhost (2/2) -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.018934 s -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276822, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276894, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 91 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 20:24:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:48 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[94] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=259625, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=256660, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[94] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 9 ms on localhost (1/2) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 12 ms on localhost (2/2) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.024430 s -2016-05-24 20:24:48 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250922, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250856, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 93 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Got job 47 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:48 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[96] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=233515, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=230549, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:48 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 47 (MapPartitionsRDD[96] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 2 tasks -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 93, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 94, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 93) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 94) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 93). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 94). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 93) in 10 ms on localhost (1/2) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 94) in 10 ms on localhost (2/2) -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 47 finished: treeAggregate at GradientDescent.scala:225, took 0.022028 s -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=221635, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=216179, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:48 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 95 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:53469 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:53469 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Got job 48 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:48 INFO DAGScheduler:59 - Final stage: ResultStage 48(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[98] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=207402, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=212930, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 48 (MapPartitionsRDD[98] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 2 tasks -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 95, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 96, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 95) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 96) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 96). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 95). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 96) in 4 ms on localhost (1/2) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 95) in 5 ms on localhost (2/2) -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 48 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 48 finished: treeAggregate at GradientDescent.scala:225, took 0.014006 s -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=215896, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=215968, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 97 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Got job 49 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:48 INFO DAGScheduler:59 - Final stage: ResultStage 49(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[100] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=216106, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=221634, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 49 (MapPartitionsRDD[100] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 2 tasks -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 97, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 98, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 98) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 97) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 97). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 98). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 97) in 10 ms on localhost (1/2) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 98) in 10 ms on localhost (2/2) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 49 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 49 finished: treeAggregate at GradientDescent.scala:225, took 0.024750 s -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224597, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224669, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 99 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Got job 50 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:48 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[102] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=224807, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2968) called with curMem=230335, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 50 (MapPartitionsRDD[102] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 2 tasks -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 5 ms on localhost (1/2) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 6 ms on localhost (2/2) -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at GradientDescent.scala:225, took 0.017591 s -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=233303, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233375, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:53469 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 101 from broadcast at GradientDescent.scala:221 -2016-05-24 20:24:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Got job 51 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:24:48 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[104] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=233513, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:24:48 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=239041, maxMem=766075207 -2016-05-24 20:24:48 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:24:48 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:53469 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:24:48 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 51 (MapPartitionsRDD[104] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:24:48 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-24 20:24:48 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:24:48 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:24:48 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 2423 bytes result sent to driver -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 5 ms on localhost (1/2) -2016-05-24 20:24:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 5 ms on localhost (2/2) -2016-05-24 20:24:48 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 20:24:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-24 20:24:48 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at GradientDescent.scala:225, took 0.014983 s -2016-05-24 20:24:48 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 20:24:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 20:24:48 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 20:24:48 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 20:24:48 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 20:24:49 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 20:24:49 INFO BlockManager:59 - BlockManager stopped -2016-05-24 20:24:49 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 20:24:49 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 20:24:49 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 20:24:49 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 20:24:49 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-07910bee-3543-4e7b-a4c9-04146473789c -2016-05-24 20:24:49 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 20:25:50 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 20:25:50 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 20:25:51 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 20:25:51 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 20:25:51 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 20:25:52 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 20:25:52 INFO Remoting:74 - Starting remoting -2016-05-24 20:25:52 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53490] -2016-05-24 20:25:52 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53490. -2016-05-24 20:25:52 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 20:25:52 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 20:25:52 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-38b4d34a-ac5a-4d1d-b8b2-44bbcddfb562 -2016-05-24 20:25:52 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 20:25:52 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-a3bd3f8e-46e8-418a-b30e-9670e4afdc79\httpd-35a9081c-e1a8-40b0-81e8-bb3dda9301c1 -2016-05-24 20:25:52 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 20:25:52 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 20:25:52 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53491 -2016-05-24 20:25:52 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53491. -2016-05-24 20:25:52 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 20:25:52 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 20:25:52 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 20:25:52 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 20:25:52 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 20:25:52 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 20:25:52 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 20:25:53 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53510. -2016-05-24 20:25:53 INFO NettyBlockTransferService:59 - Server created on 53510 -2016-05-24 20:25:53 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 20:25:53 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53510 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53510) -2016-05-24 20:25:53 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 20:25:53 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 20:25:53 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 20:25:54 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 20:25:54 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 20:25:54 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53510 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 20:25:54 INFO SparkContext:59 - Created broadcast 0 from textFile at LinearRegression3Learning.scala:21 -2016-05-24 20:25:55 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 20:25:55 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 20:25:55 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 20:25:55 INFO DAGScheduler:59 - Got job 0 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 20:25:55 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 20:25:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:55 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 20:25:55 INFO MemoryStore:59 - ensureFreeSpace(3744) called with curMem=171580, maxMem=766075207 -2016-05-24 20:25:55 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-24 20:25:55 INFO MemoryStore:59 - ensureFreeSpace(2177) called with curMem=175324, maxMem=766075207 -2016-05-24 20:25:55 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 20:25:55 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53510 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 20:25:55 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:55 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 20:25:55 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-24 20:25:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:25:55 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 20:25:55 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 20:25:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/linearRegression3.txt:0+52 -2016-05-24 20:25:55 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 20:25:55 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 20:25:55 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 20:25:55 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 20:25:55 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 20:25:55 INFO MemoryStore:59 - ensureFreeSpace(400) called with curMem=177501, maxMem=766075207 -2016-05-24 20:25:55 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 400.0 B, free 730.4 MB) -2016-05-24 20:25:55 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:53510 (size: 400.0 B, free: 730.6 MB) -2016-05-24 20:25:55 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2611 bytes result sent to driver -2016-05-24 20:25:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 132 ms on localhost (1/1) -2016-05-24 20:25:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 20:25:55 INFO DAGScheduler:59 - ResultStage 0 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.152 s -2016-05-24 20:25:55 INFO DAGScheduler:59 - Job 0 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.238490 s -2016-05-24 20:25:55 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 20:25:55 INFO DAGScheduler:59 - Got job 1 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 20:25:55 INFO DAGScheduler:59 - Final stage: ResultStage 1(count at GradientDescent.scala:195) -2016-05-24 20:25:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:55 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 20:25:55 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=177901, maxMem=766075207 -2016-05-24 20:25:55 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 20:25:55 INFO MemoryStore:59 - ensureFreeSpace(2015) called with curMem=181357, maxMem=766075207 -2016-05-24 20:25:55 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2015.0 B, free 730.4 MB) -2016-05-24 20:25:55 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53510 (size: 2015.0 B, free: 730.6 MB) -2016-05-24 20:25:55 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 20:25:55 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 20:25:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:25:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:25:55 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-24 20:25:55 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-24 20:25:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:55 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-05-24 20:25:55 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 20:25:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/linearRegression3.txt:52+53 -2016-05-24 20:25:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 20 ms on localhost (1/2) -2016-05-24 20:25:55 INFO MemoryStore:59 - ensureFreeSpace(400) called with curMem=183372, maxMem=766075207 -2016-05-24 20:25:55 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 400.0 B, free 730.4 MB) -2016-05-24 20:25:55 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:53510 (size: 400.0 B, free: 730.6 MB) -2016-05-24 20:25:55 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2662 bytes result sent to driver -2016-05-24 20:25:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 30 ms on localhost (2/2) -2016-05-24 20:25:55 INFO DAGScheduler:59 - ResultStage 1 (count at GradientDescent.scala:195) finished in 0.034 s -2016-05-24 20:25:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 20:25:55 INFO DAGScheduler:59 - Job 1 finished: count at GradientDescent.scala:195, took 0.066078 s -2016-05-24 20:25:56 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 20:25:56 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=183772, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=183844, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 3 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Got job 2 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:56 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=183982, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=189510, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 20:25:56 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 20:25:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:56 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2423 bytes result sent to driver -2016-05-24 20:25:56 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2423 bytes result sent to driver -2016-05-24 20:25:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 19 ms on localhost (1/2) -2016-05-24 20:25:56 INFO DAGScheduler:59 - ResultStage 2 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 20:25:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 22 ms on localhost (2/2) -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 20:25:56 INFO DAGScheduler:59 - Job 2 finished: treeAggregate at GradientDescent.scala:225, took 0.038947 s -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=192475, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=192547, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Got job 3 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:56 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=192685, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=198213, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 20:25:56 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 20:25:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:56 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2423 bytes result sent to driver -2016-05-24 20:25:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:56 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2423 bytes result sent to driver -2016-05-24 20:25:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 17 ms on localhost (1/2) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 18 ms on localhost (2/2) -2016-05-24 20:25:56 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 20:25:56 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at GradientDescent.scala:225, took 0.035655 s -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=201178, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=201250, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:56 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[10] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=201388, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=206916, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[10] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 20:25:56 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 20:25:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:56 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2423 bytes result sent to driver -2016-05-24 20:25:56 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2423 bytes result sent to driver -2016-05-24 20:25:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 20 ms on localhost (1/2) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 19 ms on localhost (2/2) -2016-05-24 20:25:56 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 20:25:56 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.049539 s -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=209881, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=209953, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:56 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[12] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=210091, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=215619, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[12] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 20:25:56 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 20:25:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:56 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2423 bytes result sent to driver -2016-05-24 20:25:56 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2423 bytes result sent to driver -2016-05-24 20:25:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 17 ms on localhost (1/2) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 22 ms on localhost (2/2) -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 20:25:56 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 20:25:56 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.045387 s -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=218583, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=218655, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:56 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[14] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=218793, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=224321, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[14] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 20:25:56 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 20:25:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:56 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2423 bytes result sent to driver -2016-05-24 20:25:56 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2423 bytes result sent to driver -2016-05-24 20:25:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 16 ms on localhost (1/2) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 18 ms on localhost (2/2) -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 20:25:56 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 20:25:56 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.037579 s -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=227287, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=227359, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:56 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[16] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=227497, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=233025, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[16] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 20:25:56 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 20:25:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:56 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2423 bytes result sent to driver -2016-05-24 20:25:56 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2423 bytes result sent to driver -2016-05-24 20:25:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 17 ms on localhost (1/2) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 17 ms on localhost (2/2) -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 20:25:56 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 20:25:56 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.040070 s -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=235989, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=236061, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:56 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[18] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=236199, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=241727, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[18] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 20:25:56 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 20:25:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:56 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2423 bytes result sent to driver -2016-05-24 20:25:56 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2423 bytes result sent to driver -2016-05-24 20:25:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 15 ms on localhost (1/2) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 18 ms on localhost (2/2) -2016-05-24 20:25:56 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 20:25:56 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.034590 s -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=244693, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=244765, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:56 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[20] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=244903, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=250431, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[20] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 20:25:56 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 20:25:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:56 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2423 bytes result sent to driver -2016-05-24 20:25:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:56 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2423 bytes result sent to driver -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 25 ms on localhost (1/2) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 23 ms on localhost (2/2) -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 20:25:56 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.027 s -2016-05-24 20:25:56 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.081499 s -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=253187, maxMem=766075207 -2016-05-24 20:25:56 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=253049, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:56 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:56 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:56 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[22] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:56 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=227149, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=232605, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[22] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 20:25:56 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:25:56 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 20:25:56 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 20:25:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:56 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2423 bytes result sent to driver -2016-05-24 20:25:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:56 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2423 bytes result sent to driver -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 18 ms on localhost (1/2) -2016-05-24 20:25:56 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 20:25:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 21 ms on localhost (2/2) -2016-05-24 20:25:56 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 20:25:56 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.044409 s -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=218164, maxMem=766075207 -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=212708, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:56 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:56 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[24] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=201389, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=206917, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[24] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 20:25:56 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 20:25:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:56 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2423 bytes result sent to driver -2016-05-24 20:25:56 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2423 bytes result sent to driver -2016-05-24 20:25:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 14 ms on localhost (1/2) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 14 ms on localhost (2/2) -2016-05-24 20:25:56 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 20:25:56 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.032052 s -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=209882, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=209954, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:56 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[26] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=210092, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:25:56 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=215620, maxMem=766075207 -2016-05-24 20:25:56 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:25:56 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:56 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[26] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:56 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:56 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 20:25:56 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 13 ms on localhost (1/2) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 16 ms on localhost (2/2) -2016-05-24 20:25:57 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 20:25:57 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.034001 s -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=218585, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=218657, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:57 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[28] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=218795, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=224323, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[28] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 20:25:57 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 12 ms on localhost (1/2) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 15 ms on localhost (2/2) -2016-05-24 20:25:57 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 20:25:57 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.029816 s -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=227289, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=227361, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:57 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[30] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=227499, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=233027, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[30] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 20:25:57 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 17 ms on localhost (1/2) -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 23 ms on localhost (2/2) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 20:25:57 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.027 s -2016-05-24 20:25:57 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.044693 s -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=235998, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=236070, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:57 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[32] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=236208, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=241736, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[32] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 20:25:57 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 13 ms on localhost (1/2) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 16 ms on localhost (2/2) -2016-05-24 20:25:57 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 20:25:57 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.032896 s -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=244702, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=244774, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:57 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[34] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=244912, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=250440, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[34] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 20:25:57 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 13 ms on localhost (1/2) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 15 ms on localhost (2/2) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 20:25:57 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 20:25:57 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.027627 s -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=253406, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=253478, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:57 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[36] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=253616, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=259144, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[36] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 20:25:57 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 12 ms on localhost (1/2) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 15 ms on localhost (2/2) -2016-05-24 20:25:57 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 20:25:57 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.029514 s -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=262109, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=262181, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:57 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[38] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=262319, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=267847, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[38] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 20:25:57 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 14 ms on localhost (1/2) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 18 ms on localhost (2/2) -2016-05-24 20:25:57 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 20:25:57 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.033106 s -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=270810, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=270882, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:57 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[40] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=271020, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=276548, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[40] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 20:25:57 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 13 ms on localhost (1/2) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 13 ms on localhost (2/2) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 20:25:57 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 20:25:57 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.029267 s -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=279513, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=279585, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:57 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[42] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=279723, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=285251, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[42] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 20:25:57 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 14 ms on localhost (1/2) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 13 ms on localhost (2/2) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 20:25:57 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 20:25:57 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.029987 s -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=288217, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=288289, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:57 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[44] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=288427, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=293955, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[44] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 20:25:57 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 12 ms on localhost (1/2) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 12 ms on localhost (2/2) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 20:25:57 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 20:25:57 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.027919 s -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=296920, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=296992, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:57 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[46] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=297130, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=302658, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[46] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 20:25:57 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 12 ms on localhost (1/2) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 16 ms on localhost (2/2) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 20:25:57 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 20:25:57 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.031900 s -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=305623, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=305695, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:57 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[48] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=305833, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=311361, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[48] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 20:25:57 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 12 ms on localhost (1/2) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 13 ms on localhost (2/2) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 20:25:57 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 20:25:57 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.034471 s -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=314326, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=314398, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:57 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[50] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=314536, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=320064, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[50] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 20:25:57 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 11 ms on localhost (1/2) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 12 ms on localhost (2/2) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 20:25:57 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 20:25:57 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.028410 s -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=323029, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=323101, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:57 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[52] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=323239, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=328767, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[52] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 20:25:57 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 12 ms on localhost (1/2) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 14 ms on localhost (2/2) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 20:25:57 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 20:25:57 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.037463 s -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=331730, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=331802, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:57 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[54] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=331940, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=337468, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[54] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 20:25:57 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 7 ms on localhost (1/2) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 9 ms on localhost (2/2) -2016-05-24 20:25:57 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 20:25:57 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.024213 s -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=340432, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=340504, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 20:25:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:57 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[56] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=326201, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=328764, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[56] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 20:25:57 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 9 ms on localhost (1/2) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 9 ms on localhost (2/2) -2016-05-24 20:25:57 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 20:25:57 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.026702 s -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=314323, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=314395, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:57 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:57 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[58] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=296919, maxMem=766075207 -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=302447, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[58] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 20:25:57 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 20:25:57 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 11 ms on localhost (1/2) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 10 ms on localhost (2/2) -2016-05-24 20:25:57 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 20:25:57 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.026732 s -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=293744, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=288288, maxMem=766075207 -2016-05-24 20:25:57 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:57 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[60] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=276547, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=276547, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[60] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:57 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 13 ms on localhost (1/2) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 16 ms on localhost (2/2) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 20:25:57 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 20:25:57 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.033500 s -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=267636, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=262180, maxMem=766075207 -2016-05-24 20:25:57 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:57 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:57 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[62] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=253405, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=250439, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[62] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 20:25:57 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:57 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:57 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 2423 bytes result sent to driver -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 12 ms on localhost (1/2) -2016-05-24 20:25:57 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 20:25:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 12 ms on localhost (2/2) -2016-05-24 20:25:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 20:25:57 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.029838 s -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250229, maxMem=766075207 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:57 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=244773, maxMem=766075207 -2016-05-24 20:25:57 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 20:25:57 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:57 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:57 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[64] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:58 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=227498, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=232888, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[64] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 20:25:58 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 7 ms on localhost (1/2) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 9 ms on localhost (2/2) -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.022183 s -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=227289, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=227361, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[66] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=227499, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=233027, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[66] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 5 ms on localhost (1/2) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 6 ms on localhost (2/2) -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.015537 s -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=235993, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=236065, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[68] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=236203, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=241731, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[68] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 6 ms on localhost (1/2) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 7 ms on localhost (2/2) -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.016180 s -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=244696, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=244768, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[70] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=244906, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=250434, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[70] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 5 ms on localhost (1/2) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 6 ms on localhost (2/2) -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.015549 s -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=253400, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=253472, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[72] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=253610, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=259138, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[72] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 5 ms on localhost (1/2) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 6 ms on localhost (2/2) -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.018598 s -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=262104, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=262176, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[74] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=262314, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=267842, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[74] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 6 ms on localhost (1/2) -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 6 ms on localhost (2/2) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.015993 s -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=270807, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=270879, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[76] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=271017, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=276545, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[76] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 6 ms on localhost (1/2) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 7 ms on localhost (2/2) -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.016123 s -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=279509, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=279581, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[78] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=279719, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=285247, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[78] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 6 ms on localhost (1/2) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 7 ms on localhost (2/2) -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.016750 s -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=288213, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=288285, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[80] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=288423, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=293951, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[80] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 5 ms on localhost (1/2) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 5 ms on localhost (2/2) -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.015548 s -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=296915, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=296987, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[82] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=297125, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=302653, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[82] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 5 ms on localhost (1/2) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 6 ms on localhost (2/2) -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.018083 s -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=305618, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=305690, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[84] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=305828, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2968) called with curMem=311356, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[84] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 5 ms on localhost (1/2) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 6 ms on localhost (2/2) -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.014714 s -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=314324, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=314396, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[86] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=314534, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2968) called with curMem=320062, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[86] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 9 ms on localhost (1/2) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 14 ms on localhost (2/2) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.024998 s -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=323030, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=323102, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[88] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=323240, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=328768, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[88] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 6 ms on localhost (1/2) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 6 ms on localhost (2/2) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.020003 s -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=331734, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=331806, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[90] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=331944, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=337472, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[90] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 7 ms on localhost (1/2) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 7 ms on localhost (2/2) -2016-05-24 20:25:58 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.026824 s -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=323032, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=323104, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[92] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=305625, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=308187, maxMem=766075207 -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[92] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 20:25:58 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 7 ms on localhost (1/2) -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 9 ms on localhost (2/2) -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.021115 s -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=296782, maxMem=766075207 -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=296782, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 91 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 20:25:58 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[94] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=279301, maxMem=766075207 -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=279301, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[94] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 13 ms on localhost (1/2) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 15 ms on localhost (2/2) -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.026804 s -2016-05-24 20:25:58 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=270528, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=270462, maxMem=766075207 -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 93 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 47 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[96] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=253194, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=258722, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 47 (MapPartitionsRDD[96] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 2 tasks -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 93, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 94, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 94) -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 93) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 93). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 94). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 93) in 9 ms on localhost (1/2) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 94) in 10 ms on localhost (2/2) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 47 finished: treeAggregate at GradientDescent.scala:225, took 0.024375 s -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=244280, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=238824, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 95 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:53510 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:53510 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 48 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 48(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[98] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=218796, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=224324, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 48 (MapPartitionsRDD[98] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 2 tasks -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 95, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 96, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 95) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 96) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 96). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 95). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 96) in 5 ms on localhost (1/2) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 95) in 5 ms on localhost (2/2) -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 48 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 48 finished: treeAggregate at GradientDescent.scala:225, took 0.013609 s -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=227289, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=227361, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 97 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 49 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 49(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[100] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=227499, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=233027, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 49 (MapPartitionsRDD[100] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 2 tasks -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 97, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 98, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 98) -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 97) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 97). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 98). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 97) in 6 ms on localhost (1/2) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 98) in 5 ms on localhost (2/2) -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 49 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 49 finished: treeAggregate at GradientDescent.scala:225, took 0.016371 s -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=235992, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=236064, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 99 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 50 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[102] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=236202, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2967) called with curMem=241730, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 50 (MapPartitionsRDD[102] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 2 tasks -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 6 ms on localhost (1/2) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 7 ms on localhost (2/2) -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at GradientDescent.scala:225, took 0.016545 s -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=244697, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=244769, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:53510 (size: 138.0 B, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 101 from broadcast at GradientDescent.scala:221 -2016-05-24 20:25:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Got job 51 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:25:58 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[104] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=244907, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=250435, maxMem=766075207 -2016-05-24 20:25:58 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 20:25:58 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:53510 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 20:25:58 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 51 (MapPartitionsRDD[104] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:25:58 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-24 20:25:58 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:25:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:25:58 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 2423 bytes result sent to driver -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 6 ms on localhost (1/2) -2016-05-24 20:25:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 6 ms on localhost (2/2) -2016-05-24 20:25:58 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 20:25:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-24 20:25:58 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at GradientDescent.scala:225, took 0.016901 s -2016-05-24 20:25:58 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity, Infinity -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 20:25:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 20:25:58 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 20:25:58 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 20:25:58 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 20:25:59 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 20:25:59 INFO BlockManager:59 - BlockManager stopped -2016-05-24 20:25:59 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 20:25:59 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 20:25:59 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 20:25:59 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 20:25:59 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-a3bd3f8e-46e8-418a-b30e-9670e4afdc79 -2016-05-24 20:25:59 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 20:29:04 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 20:29:04 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 20:29:05 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 20:29:05 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 20:29:05 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 20:29:05 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 20:29:06 INFO Remoting:74 - Starting remoting -2016-05-24 20:29:06 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53553] -2016-05-24 20:29:06 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53553. -2016-05-24 20:29:06 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 20:29:06 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 20:29:06 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-41284263-0831-411e-8342-45b2c6e06f57 -2016-05-24 20:29:06 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 20:29:06 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-81b8cde9-b13e-4f32-87e6-96daee21b20a\httpd-5ff1e53f-855e-4fa2-a9f5-a94ed1932a88 -2016-05-24 20:29:06 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 20:29:06 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 20:29:06 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53554 -2016-05-24 20:29:06 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53554. -2016-05-24 20:29:06 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 20:29:06 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 20:29:06 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 20:29:06 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 20:29:06 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 20:29:06 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 20:29:06 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 20:29:07 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53573. -2016-05-24 20:29:07 INFO NettyBlockTransferService:59 - Server created on 53573 -2016-05-24 20:29:07 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 20:29:07 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53573 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53573) -2016-05-24 20:29:07 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 20:29:07 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 20:29:07 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 20:29:07 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 20:29:07 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 20:29:07 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53573 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 20:29:07 INFO SparkContext:59 - Created broadcast 0 from textFile at LinearRegression3Learning.scala:21 -2016-05-24 20:29:09 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 20:29:09 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 20:29:09 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 20:29:09 INFO DAGScheduler:59 - Got job 0 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 20:29:09 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 20:29:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:29:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:29:09 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 20:29:09 INFO MemoryStore:59 - ensureFreeSpace(3744) called with curMem=171580, maxMem=766075207 -2016-05-24 20:29:09 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-24 20:29:09 INFO MemoryStore:59 - ensureFreeSpace(2174) called with curMem=175324, maxMem=766075207 -2016-05-24 20:29:09 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 20:29:09 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53573 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 20:29:09 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:29:09 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 20:29:09 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-24 20:29:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:29:09 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 20:29:09 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 20:29:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/linearRegression3.txt:0+52 -2016-05-24 20:29:09 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 20:29:09 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 20:29:09 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 20:29:09 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 20:29:09 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 20:29:09 INFO MemoryStore:59 - ensureFreeSpace(400) called with curMem=177498, maxMem=766075207 -2016-05-24 20:29:09 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 400.0 B, free 730.4 MB) -2016-05-24 20:29:09 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:53573 (size: 400.0 B, free: 730.6 MB) -2016-05-24 20:29:09 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2611 bytes result sent to driver -2016-05-24 20:29:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 114 ms on localhost (1/1) -2016-05-24 20:29:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 20:29:09 INFO DAGScheduler:59 - ResultStage 0 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.133 s -2016-05-24 20:29:09 INFO DAGScheduler:59 - Job 0 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.229850 s -2016-05-24 20:29:09 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 20:29:09 INFO DAGScheduler:59 - Got job 1 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 20:29:09 INFO DAGScheduler:59 - Final stage: ResultStage 1(count at GradientDescent.scala:195) -2016-05-24 20:29:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:29:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:29:09 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 20:29:09 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=177898, maxMem=766075207 -2016-05-24 20:29:09 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 20:29:09 INFO MemoryStore:59 - ensureFreeSpace(2012) called with curMem=181354, maxMem=766075207 -2016-05-24 20:29:09 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2012.0 B, free 730.4 MB) -2016-05-24 20:29:09 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53573 (size: 2012.0 B, free: 730.6 MB) -2016-05-24 20:29:09 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:29:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 20:29:09 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 20:29:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:29:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:29:09 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-24 20:29:09 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-24 20:29:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:29:09 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-05-24 20:29:09 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 20:29:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/linearRegression3.txt:52+53 -2016-05-24 20:29:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 25 ms on localhost (1/2) -2016-05-24 20:29:09 INFO MemoryStore:59 - ensureFreeSpace(400) called with curMem=183366, maxMem=766075207 -2016-05-24 20:29:09 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 400.0 B, free 730.4 MB) -2016-05-24 20:29:09 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:53573 (size: 400.0 B, free: 730.6 MB) -2016-05-24 20:29:09 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2662 bytes result sent to driver -2016-05-24 20:29:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 33 ms on localhost (2/2) -2016-05-24 20:29:09 INFO DAGScheduler:59 - ResultStage 1 (count at GradientDescent.scala:195) finished in 0.038 s -2016-05-24 20:29:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 20:29:09 INFO DAGScheduler:59 - Job 1 finished: count at GradientDescent.scala:195, took 0.070452 s -2016-05-24 20:29:10 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 20:29:10 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 20:29:10 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=183766, maxMem=766075207 -2016-05-24 20:29:10 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:29:10 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=183838, maxMem=766075207 -2016-05-24 20:29:10 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:29:10 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53573 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:29:10 INFO SparkContext:59 - Created broadcast 3 from broadcast at GradientDescent.scala:221 -2016-05-24 20:29:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:29:10 INFO DAGScheduler:59 - Got job 2 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:29:10 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:29:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:29:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:29:10 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:29:10 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=183976, maxMem=766075207 -2016-05-24 20:29:10 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:29:10 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=189504, maxMem=766075207 -2016-05-24 20:29:10 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:29:10 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53573 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:29:10 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:29:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:29:10 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 20:29:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:29:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:29:10 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 20:29:10 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 20:29:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:29:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:29:10 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2423 bytes result sent to driver -2016-05-24 20:29:10 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2423 bytes result sent to driver -2016-05-24 20:29:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 22 ms on localhost (1/2) -2016-05-24 20:29:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 24 ms on localhost (2/2) -2016-05-24 20:29:10 INFO DAGScheduler:59 - ResultStage 2 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 20:29:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 20:29:10 INFO DAGScheduler:59 - Job 2 finished: treeAggregate at GradientDescent.scala:225, took 0.040753 s -2016-05-24 20:29:10 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=192470, maxMem=766075207 -2016-05-24 20:29:10 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:29:10 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=192542, maxMem=766075207 -2016-05-24 20:29:10 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:29:10 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53573 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:29:10 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 20:29:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:29:10 INFO DAGScheduler:59 - Got job 3 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:29:10 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:29:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:29:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:29:10 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:29:10 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=192680, maxMem=766075207 -2016-05-24 20:29:10 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:29:10 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=198208, maxMem=766075207 -2016-05-24 20:29:10 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:29:10 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53573 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:29:10 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:29:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:29:10 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 20:29:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:29:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:29:10 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 20:29:10 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 20:29:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:29:10 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2423 bytes result sent to driver -2016-05-24 20:29:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:29:10 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2423 bytes result sent to driver -2016-05-24 20:29:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 17 ms on localhost (1/2) -2016-05-24 20:29:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 21 ms on localhost (2/2) -2016-05-24 20:29:10 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 20:29:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 20:29:10 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at GradientDescent.scala:225, took 0.036310 s -2016-05-24 20:29:10 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 3372.5, 1.8866811720935E13 -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 20:29:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 20:29:10 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 20:29:10 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 20:29:10 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 20:29:10 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 20:29:10 INFO BlockManager:59 - BlockManager stopped -2016-05-24 20:29:10 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 20:29:10 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 20:29:10 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 20:29:10 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 20:29:10 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-81b8cde9-b13e-4f32-87e6-96daee21b20a -2016-05-24 20:29:10 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 20:29:10 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 20:31:15 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 20:31:15 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 20:31:16 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 20:31:16 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 20:31:16 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 20:31:16 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 20:31:16 INFO Remoting:74 - Starting remoting -2016-05-24 20:31:17 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:53608] -2016-05-24 20:31:17 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53608. -2016-05-24 20:31:17 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 20:31:17 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 20:31:17 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-b965e5a6-6ae9-456f-aab7-2f31f879f83b -2016-05-24 20:31:17 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 20:31:17 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-95de653f-dba5-424f-b92a-68b6154b7515\httpd-b6a17c02-c5a2-4c93-b7f2-e5c4d51d6580 -2016-05-24 20:31:17 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 20:31:17 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 20:31:17 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53609 -2016-05-24 20:31:17 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53609. -2016-05-24 20:31:17 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 20:31:17 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 20:31:17 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 20:31:17 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 20:31:17 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 20:31:17 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 20:31:17 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 20:31:18 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53628. -2016-05-24 20:31:18 INFO NettyBlockTransferService:59 - Server created on 53628 -2016-05-24 20:31:18 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 20:31:18 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53628 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53628) -2016-05-24 20:31:18 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 20:31:18 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 20:31:18 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 20:31:18 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 20:31:18 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 20:31:18 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53628 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 20:31:18 INFO SparkContext:59 - Created broadcast 0 from textFile at LinearRegression3Learning.scala:21 -2016-05-24 20:31:20 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 20:31:20 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 20:31:20 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 20:31:20 INFO DAGScheduler:59 - Got job 0 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 20:31:20 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 20:31:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:31:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:31:20 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 20:31:20 INFO MemoryStore:59 - ensureFreeSpace(3744) called with curMem=171580, maxMem=766075207 -2016-05-24 20:31:20 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-24 20:31:20 INFO MemoryStore:59 - ensureFreeSpace(2174) called with curMem=175324, maxMem=766075207 -2016-05-24 20:31:20 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 20:31:20 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53628 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 20:31:20 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:31:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 20:31:20 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-24 20:31:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:31:20 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 20:31:20 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 20:31:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/linearRegression3.txt:0+52 -2016-05-24 20:31:20 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 20:31:20 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 20:31:20 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 20:31:20 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 20:31:20 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 20:31:20 INFO MemoryStore:59 - ensureFreeSpace(400) called with curMem=177498, maxMem=766075207 -2016-05-24 20:31:20 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 400.0 B, free 730.4 MB) -2016-05-24 20:31:20 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:53628 (size: 400.0 B, free: 730.6 MB) -2016-05-24 20:31:20 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2611 bytes result sent to driver -2016-05-24 20:31:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 121 ms on localhost (1/1) -2016-05-24 20:31:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 20:31:20 INFO DAGScheduler:59 - ResultStage 0 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.143 s -2016-05-24 20:31:20 INFO DAGScheduler:59 - Job 0 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.232438 s -2016-05-24 20:31:20 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 20:31:20 INFO DAGScheduler:59 - Got job 1 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 20:31:20 INFO DAGScheduler:59 - Final stage: ResultStage 1(count at GradientDescent.scala:195) -2016-05-24 20:31:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:31:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:31:20 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 20:31:20 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=177898, maxMem=766075207 -2016-05-24 20:31:20 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 20:31:20 INFO MemoryStore:59 - ensureFreeSpace(2012) called with curMem=181354, maxMem=766075207 -2016-05-24 20:31:20 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2012.0 B, free 730.4 MB) -2016-05-24 20:31:20 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53628 (size: 2012.0 B, free: 730.6 MB) -2016-05-24 20:31:20 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:31:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 20:31:20 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 20:31:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:31:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:31:20 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-24 20:31:20 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-24 20:31:20 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:31:20 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-05-24 20:31:20 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 20:31:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/linearRegression3.txt:52+53 -2016-05-24 20:31:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 23 ms on localhost (1/2) -2016-05-24 20:31:20 INFO MemoryStore:59 - ensureFreeSpace(400) called with curMem=183366, maxMem=766075207 -2016-05-24 20:31:20 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 400.0 B, free 730.4 MB) -2016-05-24 20:31:20 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:53628 (size: 400.0 B, free: 730.6 MB) -2016-05-24 20:31:20 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2662 bytes result sent to driver -2016-05-24 20:31:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 33 ms on localhost (2/2) -2016-05-24 20:31:20 INFO DAGScheduler:59 - ResultStage 1 (count at GradientDescent.scala:195) finished in 0.037 s -2016-05-24 20:31:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 20:31:20 INFO DAGScheduler:59 - Job 1 finished: count at GradientDescent.scala:195, took 0.069306 s -2016-05-24 20:31:21 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53628 in memory (size: 2012.0 B, free: 730.6 MB) -2016-05-24 20:31:21 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53628 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 20:31:21 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 20:31:21 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 20:31:21 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=172380, maxMem=766075207 -2016-05-24 20:31:21 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:31:21 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=172452, maxMem=766075207 -2016-05-24 20:31:21 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:31:21 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53628 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:31:21 INFO SparkContext:59 - Created broadcast 3 from broadcast at GradientDescent.scala:221 -2016-05-24 20:31:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:31:21 INFO DAGScheduler:59 - Got job 2 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:31:21 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:31:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:31:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:31:21 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:31:21 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=172590, maxMem=766075207 -2016-05-24 20:31:21 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:31:21 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=178118, maxMem=766075207 -2016-05-24 20:31:21 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:31:21 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53628 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:31:21 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:31:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:31:21 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 20:31:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:31:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:31:21 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 20:31:21 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 20:31:21 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:31:21 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:31:21 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2423 bytes result sent to driver -2016-05-24 20:31:21 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2423 bytes result sent to driver -2016-05-24 20:31:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 18 ms on localhost (1/2) -2016-05-24 20:31:21 INFO DAGScheduler:59 - ResultStage 2 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 20:31:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 21 ms on localhost (2/2) -2016-05-24 20:31:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 20:31:21 INFO DAGScheduler:59 - Job 2 finished: treeAggregate at GradientDescent.scala:225, took 0.038960 s -2016-05-24 20:31:21 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=181084, maxMem=766075207 -2016-05-24 20:31:21 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:31:21 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=181156, maxMem=766075207 -2016-05-24 20:31:21 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:31:21 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53628 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:31:21 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 20:31:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:31:21 INFO DAGScheduler:59 - Got job 3 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:31:21 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:31:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:31:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:31:21 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:31:21 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=181294, maxMem=766075207 -2016-05-24 20:31:21 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:31:21 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=186822, maxMem=766075207 -2016-05-24 20:31:21 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:31:21 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53628 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:31:21 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:31:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:31:21 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 20:31:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:31:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:31:21 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 20:31:21 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 20:31:21 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:31:21 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2423 bytes result sent to driver -2016-05-24 20:31:21 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:31:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 17 ms on localhost (1/2) -2016-05-24 20:31:21 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2423 bytes result sent to driver -2016-05-24 20:31:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 20 ms on localhost (2/2) -2016-05-24 20:31:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 20:31:21 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 20:31:21 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at GradientDescent.scala:225, took 0.035719 s -2016-05-24 20:31:21 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 3372.5, 1.8866811720935E13 -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 20:31:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 20:31:21 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 20:31:21 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 20:31:21 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 20:31:21 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 20:31:21 INFO BlockManager:59 - BlockManager stopped -2016-05-24 20:31:21 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 20:31:21 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 20:31:21 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 20:31:21 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 20:31:21 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-95de653f-dba5-424f-b92a-68b6154b7515 -2016-05-24 20:31:21 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 20:43:25 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 20:43:26 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 20:43:26 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 20:43:26 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 20:43:26 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 20:43:27 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 20:43:27 INFO Remoting:74 - Starting remoting -2016-05-24 20:43:27 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:54018] -2016-05-24 20:43:27 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54018. -2016-05-24 20:43:27 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 20:43:27 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 20:43:27 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-ee6f7ea3-dc80-4172-8dbe-7c9f230c217f -2016-05-24 20:43:27 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 20:43:27 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-fcee7d38-3ffc-4b0f-940b-cd0895515751\httpd-8cd55945-05cc-4b37-9651-fa19cecf236f -2016-05-24 20:43:27 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 20:43:27 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 20:43:27 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54019 -2016-05-24 20:43:27 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54019. -2016-05-24 20:43:27 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 20:43:27 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 20:43:27 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 20:43:27 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 20:43:27 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 20:43:28 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 20:43:28 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 20:43:28 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54038. -2016-05-24 20:43:28 INFO NettyBlockTransferService:59 - Server created on 54038 -2016-05-24 20:43:28 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 20:43:28 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54038 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54038) -2016-05-24 20:43:28 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 20:43:29 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 20:43:29 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 20:43:29 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 20:43:29 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 20:43:29 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54038 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 20:43:29 INFO SparkContext:59 - Created broadcast 0 from textFile at LinearRegression4Learning.scala:21 -2016-05-24 20:43:30 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 20:43:30 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 20:43:30 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 20:43:30 INFO DAGScheduler:59 - Got job 0 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 20:43:30 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 20:43:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:43:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:43:30 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 20:43:30 INFO MemoryStore:59 - ensureFreeSpace(3744) called with curMem=171580, maxMem=766075207 -2016-05-24 20:43:30 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-24 20:43:30 INFO MemoryStore:59 - ensureFreeSpace(2177) called with curMem=175324, maxMem=766075207 -2016-05-24 20:43:30 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 20:43:30 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54038 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 20:43:30 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:43:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 20:43:30 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-24 20:43:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:43:30 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 20:43:30 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 20:43:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/linearRegression3.txt:0+52 -2016-05-24 20:43:30 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 20:43:30 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 20:43:30 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 20:43:30 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 20:43:30 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 20:43:30 INFO MemoryStore:59 - ensureFreeSpace(400) called with curMem=177501, maxMem=766075207 -2016-05-24 20:43:30 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 400.0 B, free 730.4 MB) -2016-05-24 20:43:30 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:54038 (size: 400.0 B, free: 730.6 MB) -2016-05-24 20:43:30 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2611 bytes result sent to driver -2016-05-24 20:43:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 147 ms on localhost (1/1) -2016-05-24 20:43:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 20:43:30 INFO DAGScheduler:59 - ResultStage 0 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.179 s -2016-05-24 20:43:30 INFO DAGScheduler:59 - Job 0 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.281530 s -2016-05-24 20:43:30 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 20:43:30 INFO DAGScheduler:59 - Got job 1 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 20:43:30 INFO DAGScheduler:59 - Final stage: ResultStage 1(count at GradientDescent.scala:195) -2016-05-24 20:43:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:43:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:43:30 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 20:43:30 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=177901, maxMem=766075207 -2016-05-24 20:43:30 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 20:43:30 INFO MemoryStore:59 - ensureFreeSpace(2013) called with curMem=181357, maxMem=766075207 -2016-05-24 20:43:30 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2013.0 B, free 730.4 MB) -2016-05-24 20:43:30 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54038 (size: 2013.0 B, free: 730.6 MB) -2016-05-24 20:43:30 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:43:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 20:43:30 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 20:43:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:43:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:43:30 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-24 20:43:30 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-24 20:43:30 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 20:43:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/linearRegression3.txt:52+53 -2016-05-24 20:43:31 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:43:31 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-05-24 20:43:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 27 ms on localhost (1/2) -2016-05-24 20:43:31 INFO MemoryStore:59 - ensureFreeSpace(400) called with curMem=183370, maxMem=766075207 -2016-05-24 20:43:31 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 400.0 B, free 730.4 MB) -2016-05-24 20:43:31 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:54038 (size: 400.0 B, free: 730.6 MB) -2016-05-24 20:43:31 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2662 bytes result sent to driver -2016-05-24 20:43:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 31 ms on localhost (2/2) -2016-05-24 20:43:31 INFO DAGScheduler:59 - ResultStage 1 (count at GradientDescent.scala:195) finished in 0.036 s -2016-05-24 20:43:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 20:43:31 INFO DAGScheduler:59 - Job 1 finished: count at GradientDescent.scala:195, took 0.072393 s -2016-05-24 20:43:31 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:54038 in memory (size: 2013.0 B, free: 730.6 MB) -2016-05-24 20:43:31 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:54038 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 20:43:31 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 20:43:31 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 20:43:31 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=172380, maxMem=766075207 -2016-05-24 20:43:31 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:43:31 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=172452, maxMem=766075207 -2016-05-24 20:43:31 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:43:31 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54038 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:43:31 INFO SparkContext:59 - Created broadcast 3 from broadcast at GradientDescent.scala:221 -2016-05-24 20:43:31 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:43:31 INFO DAGScheduler:59 - Got job 2 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:43:31 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:43:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:43:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:43:31 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:43:31 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=172590, maxMem=766075207 -2016-05-24 20:43:31 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:43:31 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=178118, maxMem=766075207 -2016-05-24 20:43:31 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:43:31 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54038 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:43:31 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:43:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:43:31 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 20:43:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:43:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:43:31 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 20:43:31 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 20:43:31 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:43:31 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:43:31 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2423 bytes result sent to driver -2016-05-24 20:43:31 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2423 bytes result sent to driver -2016-05-24 20:43:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 26 ms on localhost (1/2) -2016-05-24 20:43:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 29 ms on localhost (2/2) -2016-05-24 20:43:31 INFO DAGScheduler:59 - ResultStage 2 (treeAggregate at GradientDescent.scala:225) finished in 0.030 s -2016-05-24 20:43:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 20:43:31 INFO DAGScheduler:59 - Job 2 finished: treeAggregate at GradientDescent.scala:225, took 0.050225 s -2016-05-24 20:43:31 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=181084, maxMem=766075207 -2016-05-24 20:43:31 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 20:43:31 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=181156, maxMem=766075207 -2016-05-24 20:43:31 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 20:43:31 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:54038 (size: 138.0 B, free: 730.6 MB) -2016-05-24 20:43:31 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 20:43:31 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 20:43:31 INFO DAGScheduler:59 - Got job 3 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 20:43:31 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at GradientDescent.scala:225) -2016-05-24 20:43:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:43:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:43:31 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 20:43:31 INFO MemoryStore:59 - ensureFreeSpace(5528) called with curMem=181294, maxMem=766075207 -2016-05-24 20:43:31 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 20:43:31 INFO MemoryStore:59 - ensureFreeSpace(2967) called with curMem=186822, maxMem=766075207 -2016-05-24 20:43:31 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 20:43:31 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:54038 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 20:43:31 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:43:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225) -2016-05-24 20:43:31 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 20:43:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:43:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 20:43:31 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 20:43:31 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 20:43:31 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:43:31 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:43:31 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2423 bytes result sent to driver -2016-05-24 20:43:31 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2423 bytes result sent to driver -2016-05-24 20:43:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 23 ms on localhost (1/2) -2016-05-24 20:43:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 26 ms on localhost (2/2) -2016-05-24 20:43:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 20:43:31 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at GradientDescent.scala:225) finished in 0.027 s -2016-05-24 20:43:31 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at GradientDescent.scala:225, took 0.048877 s -2016-05-24 20:43:31 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 3372.5, 1.8866811720935E13 -2016-05-24 20:43:31 INFO SparkContext:59 - Starting job: mean at LinearRegression4Learning.scala:34 -2016-05-24 20:43:31 INFO DAGScheduler:59 - Got job 4 (mean at LinearRegression4Learning.scala:34) with 2 output partitions -2016-05-24 20:43:31 INFO DAGScheduler:59 - Final stage: ResultStage 4(mean at LinearRegression4Learning.scala:34) -2016-05-24 20:43:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 20:43:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 20:43:31 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[11] at mean at LinearRegression4Learning.scala:34), which has no missing parents -2016-05-24 20:43:31 INFO MemoryStore:59 - ensureFreeSpace(4576) called with curMem=189789, maxMem=766075207 -2016-05-24 20:43:31 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 4.5 KB, free 730.4 MB) -2016-05-24 20:43:31 INFO MemoryStore:59 - ensureFreeSpace(2484) called with curMem=194365, maxMem=766075207 -2016-05-24 20:43:31 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-05-24 20:43:31 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:54038 (size: 2.4 KB, free: 730.6 MB) -2016-05-24 20:43:31 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-24 20:43:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[11] at mean at LinearRegression4Learning.scala:34) -2016-05-24 20:43:31 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 20:43:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:43:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2195 bytes) -2016-05-24 20:43:31 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 20:43:31 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 20:43:31 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 20:43:31 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 20:43:31 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2209 bytes result sent to driver -2016-05-24 20:43:31 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2209 bytes result sent to driver -2016-05-24 20:43:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 36 ms on localhost (1/2) -2016-05-24 20:43:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 36 ms on localhost (2/2) -2016-05-24 20:43:31 INFO DAGScheduler:59 - ResultStage 4 (mean at LinearRegression4Learning.scala:34) finished in 0.037 s -2016-05-24 20:43:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 20:43:31 INFO DAGScheduler:59 - Job 4 finished: mean at LinearRegression4Learning.scala:34, took 0.069274 s -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 20:43:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 20:43:31 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 20:43:31 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 20:43:31 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 20:43:31 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 20:43:31 INFO BlockManager:59 - BlockManager stopped -2016-05-24 20:43:31 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 20:43:31 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 20:43:31 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 20:43:31 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 20:43:31 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-fcee7d38-3ffc-4b0f-940b-cd0895515751 -2016-05-24 21:19:46 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 21:19:47 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 21:19:48 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 21:19:48 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 21:19:48 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 21:19:49 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 21:19:49 INFO Remoting:74 - Starting remoting -2016-05-24 21:19:49 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:54830] -2016-05-24 21:19:49 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54830. -2016-05-24 21:19:49 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 21:19:49 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 21:19:49 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-3470cd2e-600e-45c4-94dc-fe6dd3666f68 -2016-05-24 21:19:49 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 21:19:49 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-a94376b5-d659-4eb8-baa4-2e64c4177e83\httpd-5f174dcd-2dcb-4a82-9f0e-e96cecb60480 -2016-05-24 21:19:49 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 21:19:49 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:19:49 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54831 -2016-05-24 21:19:49 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54831. -2016-05-24 21:19:49 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 21:19:49 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:19:49 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 21:19:49 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 21:19:49 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 21:19:50 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 21:19:50 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 21:19:50 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54850. -2016-05-24 21:19:50 INFO NettyBlockTransferService:59 - Server created on 54850 -2016-05-24 21:19:50 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 21:19:50 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54850 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54850) -2016-05-24 21:19:50 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 21:19:51 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 21:19:51 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 21:19:51 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 21:19:51 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 21:19:51 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54850 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 21:19:51 INFO SparkContext:59 - Created broadcast 0 from textFile at LinearRegression2Learning.scala:21 -2016-05-24 21:19:52 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 21:19:52 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 21:19:52 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 21:19:52 INFO DAGScheduler:59 - Got job 0 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 21:19:52 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:19:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:52 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 21:19:52 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=171580, maxMem=766075207 -2016-05-24 21:19:52 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-24 21:19:52 INFO MemoryStore:59 - ensureFreeSpace(2182) called with curMem=175316, maxMem=766075207 -2016-05-24 21:19:52 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 21:19:52 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54850 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:19:52 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:52 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:19:52 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-24 21:19:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 21:19:52 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 21:19:52 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 21:19:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/ridge-data/lpsa2.data:0+19 -2016-05-24 21:19:52 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 21:19:52 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 21:19:52 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 21:19:52 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 21:19:52 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 21:19:52 INFO MemoryStore:59 - ensureFreeSpace(320) called with curMem=177498, maxMem=766075207 -2016-05-24 21:19:52 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 320.0 B, free 730.4 MB) -2016-05-24 21:19:52 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:54850 (size: 320.0 B, free: 730.6 MB) -2016-05-24 21:19:53 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2611 bytes result sent to driver -2016-05-24 21:19:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 133 ms on localhost (1/1) -2016-05-24 21:19:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 21:19:53 INFO DAGScheduler:59 - ResultStage 0 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.155 s -2016-05-24 21:19:53 INFO DAGScheduler:59 - Job 0 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.258547 s -2016-05-24 21:19:53 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 21:19:53 INFO DAGScheduler:59 - Got job 1 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 21:19:53 INFO DAGScheduler:59 - Final stage: ResultStage 1(count at GradientDescent.scala:195) -2016-05-24 21:19:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:53 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=177818, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(2020) called with curMem=181258, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2020.0 B, free 730.4 MB) -2016-05-24 21:19:53 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54850 (size: 2020.0 B, free: 730.6 MB) -2016-05-24 21:19:53 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 21:19:53 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 21:19:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 21:19:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 21:19:53 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-24 21:19:53 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-24 21:19:53 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:53 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 21:19:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/ridge-data/lpsa2.data:19+19 -2016-05-24 21:19:53 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-05-24 21:19:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 23 ms on localhost (1/2) -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(168) called with curMem=183278, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 168.0 B, free 730.4 MB) -2016-05-24 21:19:53 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:54850 (size: 168.0 B, free: 730.6 MB) -2016-05-24 21:19:53 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2662 bytes result sent to driver -2016-05-24 21:19:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 28 ms on localhost (2/2) -2016-05-24 21:19:53 INFO DAGScheduler:59 - ResultStage 1 (count at GradientDescent.scala:195) finished in 0.034 s -2016-05-24 21:19:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 21:19:53 INFO DAGScheduler:59 - Job 1 finished: count at GradientDescent.scala:195, took 0.061789 s -2016-05-24 21:19:53 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:54850 in memory (size: 2020.0 B, free: 730.6 MB) -2016-05-24 21:19:53 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:54850 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:19:53 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 21:19:53 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=172068, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=172140, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:53 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:53 INFO SparkContext:59 - Created broadcast 3 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:53 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:53 INFO DAGScheduler:59 - Got job 2 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:53 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:53 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=172278, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=177798, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:53 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:53 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[6] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:53 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 21:19:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:53 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 21:19:53 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 21:19:53 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:53 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:53 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2423 bytes result sent to driver -2016-05-24 21:19:53 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2423 bytes result sent to driver -2016-05-24 21:19:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 21 ms on localhost (1/2) -2016-05-24 21:19:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 23 ms on localhost (2/2) -2016-05-24 21:19:53 INFO DAGScheduler:59 - ResultStage 2 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 21:19:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 21:19:53 INFO DAGScheduler:59 - Job 2 finished: treeAggregate at GradientDescent.scala:225, took 0.039073 s -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=180769, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=180841, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:53 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:53 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:53 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:53 INFO DAGScheduler:59 - Got job 3 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:53 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:53 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=180979, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=186499, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:53 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:53 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:53 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 21:19:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:53 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 21:19:53 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 21:19:53 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:53 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2423 bytes result sent to driver -2016-05-24 21:19:53 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 17 ms on localhost (1/2) -2016-05-24 21:19:53 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2423 bytes result sent to driver -2016-05-24 21:19:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 22 ms on localhost (2/2) -2016-05-24 21:19:53 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 21:19:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 21:19:53 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at GradientDescent.scala:225, took 0.038113 s -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=189470, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=189542, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:53 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:53 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:53 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:53 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:53 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:53 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[10] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=189680, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=195200, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:53 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:53 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[10] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:53 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 21:19:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:53 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 21:19:53 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 21:19:53 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:53 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:53 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2423 bytes result sent to driver -2016-05-24 21:19:53 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2423 bytes result sent to driver -2016-05-24 21:19:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 23 ms on localhost (1/2) -2016-05-24 21:19:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 22 ms on localhost (2/2) -2016-05-24 21:19:53 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.025 s -2016-05-24 21:19:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 21:19:53 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.047210 s -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=198171, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=198243, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:53 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:53 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:53 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:53 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:53 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:53 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[12] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=198381, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=203901, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:53 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:53 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[12] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:53 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 21:19:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:53 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 21:19:53 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 21:19:53 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:53 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:53 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2423 bytes result sent to driver -2016-05-24 21:19:53 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2423 bytes result sent to driver -2016-05-24 21:19:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 15 ms on localhost (1/2) -2016-05-24 21:19:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 16 ms on localhost (2/2) -2016-05-24 21:19:53 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:19:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 21:19:53 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.036992 s -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=206871, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=206943, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:53 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:53 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:53 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:53 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:53 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:53 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[14] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=207081, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=212601, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:53 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:53 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[14] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:53 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 21:19:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:53 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 21:19:53 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 21:19:53 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:53 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:53 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2423 bytes result sent to driver -2016-05-24 21:19:53 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2423 bytes result sent to driver -2016-05-24 21:19:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 20 ms on localhost (1/2) -2016-05-24 21:19:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 21 ms on localhost (2/2) -2016-05-24 21:19:53 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 21:19:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 21:19:53 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.040193 s -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=215573, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=215645, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:53 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:53 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:53 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:53 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:53 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:53 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[16] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=215783, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=221303, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:53 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:53 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[16] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:53 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 21:19:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:53 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 21:19:53 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 21:19:53 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:53 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:53 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2423 bytes result sent to driver -2016-05-24 21:19:53 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2423 bytes result sent to driver -2016-05-24 21:19:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 12 ms on localhost (1/2) -2016-05-24 21:19:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 14 ms on localhost (2/2) -2016-05-24 21:19:53 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 21:19:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 21:19:53 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.034574 s -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224273, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224345, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:53 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:53 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:53 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:53 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:53 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:53 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[18] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224483, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:53 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=230003, maxMem=766075207 -2016-05-24 21:19:53 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:53 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:53 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[18] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:53 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 21:19:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:53 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 21:19:53 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 21:19:53 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:53 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:53 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 19 ms on localhost (1/2) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 18 ms on localhost (2/2) -2016-05-24 21:19:54 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 21:19:54 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.035093 s -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=232974, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233046, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:54 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[20] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233184, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=238704, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[20] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:54 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 21:19:54 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:54 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 17 ms on localhost (1/2) -2016-05-24 21:19:54 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 22 ms on localhost (2/2) -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 21:19:54 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.025 s -2016-05-24 21:19:54 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.057777 s -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224555, maxMem=766075207 -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=221585, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:54 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:54 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:54 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:54 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[22] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=198591, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=195620, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:54 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[22] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 21:19:54 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:54 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:54 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 28 ms on localhost (1/2) -2016-05-24 21:19:54 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 30 ms on localhost (2/2) -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 21:19:54 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.033 s -2016-05-24 21:19:54 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.059074 s -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=189542, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=189542, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:54 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[24] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=189680, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=195200, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[24] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 21:19:54 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:54 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 13 ms on localhost (1/2) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 14 ms on localhost (2/2) -2016-05-24 21:19:54 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 21:19:54 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.031732 s -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=198170, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=198242, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:54 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[26] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=198380, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=203900, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[26] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 21:19:54 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:54 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 16 ms on localhost (1/2) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 15 ms on localhost (2/2) -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 21:19:54 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:19:54 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.035157 s -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=206872, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=206944, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:54 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[28] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=207082, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(2973) called with curMem=212602, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[28] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 21:19:54 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:54 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 17 ms on localhost (1/2) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 15 ms on localhost (2/2) -2016-05-24 21:19:54 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 21:19:54 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.034101 s -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=215575, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=215647, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:54 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[30] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=215785, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(2976) called with curMem=221305, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[30] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:54 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 66 ms on localhost (1/2) -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:54 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 76 ms on localhost (2/2) -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 21:19:54 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.098 s -2016-05-24 21:19:54 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.174158 s -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224281, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224353, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:54 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[32] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224491, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=230011, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[32] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 21:19:54 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:54 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 14 ms on localhost (1/2) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 17 ms on localhost (2/2) -2016-05-24 21:19:54 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 21:19:54 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.036589 s -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=232983, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233055, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:54 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[34] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233193, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=238713, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[34] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 21:19:54 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:54 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 13 ms on localhost (1/2) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 14 ms on localhost (2/2) -2016-05-24 21:19:54 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 21:19:54 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.027566 s -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241685, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241757, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:54 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[36] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=241895, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=247415, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[36] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 21:19:54 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:54 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 10 ms on localhost (1/2) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 12 ms on localhost (2/2) -2016-05-24 21:19:54 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 21:19:54 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.028033 s -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250387, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250459, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:54 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[38] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250597, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=256117, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[38] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 21:19:54 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:54 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 14 ms on localhost (1/2) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 16 ms on localhost (2/2) -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 21:19:54 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:19:54 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.029352 s -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=259086, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=259158, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:54 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[40] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=259296, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=264816, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[40] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 21:19:54 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:54 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 14 ms on localhost (1/2) -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:54 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 19 ms on localhost (2/2) -2016-05-24 21:19:54 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 21:19:54 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.037628 s -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=267788, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=267860, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:54 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[42] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=267998, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=273518, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[42] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 21:19:54 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:54 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 14 ms on localhost (1/2) -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:54 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 22 ms on localhost (2/2) -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 21:19:54 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 21:19:54 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.044723 s -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276490, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276562, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:54 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[44] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=276700, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=282220, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[44] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:54 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:54 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 16 ms on localhost (1/2) -2016-05-24 21:19:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:54 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 2423 bytes result sent to driver -2016-05-24 21:19:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 24 ms on localhost (2/2) -2016-05-24 21:19:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 21:19:54 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 21:19:54 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.047022 s -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285191, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:54 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285263, maxMem=766075207 -2016-05-24 21:19:54 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:54 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:54 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[46] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=285401, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=290921, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[46] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 13 ms on localhost (1/2) -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 17 ms on localhost (2/2) -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.046055 s -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=293892, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=293964, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[48] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=294102, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=299622, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[48] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 13 ms on localhost (1/2) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 16 ms on localhost (2/2) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.055355 s -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=302594, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=302666, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[50] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=302804, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=308324, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[50] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 12 ms on localhost (1/2) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 11 ms on localhost (2/2) -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.034830 s -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=311294, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=311366, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[52] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=311504, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=317024, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[52] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 11 ms on localhost (1/2) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 14 ms on localhost (2/2) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.034086 s -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=319994, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=320066, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[54] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=320204, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=325724, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[54] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 8 ms on localhost (1/2) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 9 ms on localhost (2/2) -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.023123 s -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=328694, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=328766, maxMem=766075207 -2016-05-24 21:19:55 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[56] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=303014, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=308462, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[56] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 10 ms on localhost (1/2) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 9 ms on localhost (2/2) -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.022632 s -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=299621, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=294173, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[58] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=276770, maxMem=766075207 -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=282218, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[58] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 9 ms on localhost (1/2) -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 13 ms on localhost (2/2) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.028424 s -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276698, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276770, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[60] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=265027, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=265027, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[60] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 11 ms on localhost (1/2) -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 14 ms on localhost (2/2) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.030396 s -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=259086, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=256186, maxMem=766075207 -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[62] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233395, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=238915, maxMem=766075207 -2016-05-24 21:19:55 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[62] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 12 ms on localhost (1/2) -2016-05-24 21:19:55 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 16 ms on localhost (2/2) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.032794 s -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=232975, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224557, maxMem=766075207 -2016-05-24 21:19:55 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:55 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[64] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=207155, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=212603, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[64] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 6 ms on localhost (1/2) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 6 ms on localhost (2/2) -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.015702 s -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=215575, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=215647, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[66] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=215785, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=221305, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[66] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 6 ms on localhost (1/2) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 7 ms on localhost (2/2) -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.015948 s -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224276, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224348, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[68] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224486, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=230006, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[68] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 8 ms on localhost (1/2) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 9 ms on localhost (2/2) -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.019594 s -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=232977, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233049, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[70] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233187, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=238707, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[70] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 6 ms on localhost (1/2) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 6 ms on localhost (2/2) -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.016648 s -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241679, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241751, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[72] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=241889, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=247409, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[72] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 7 ms on localhost (1/2) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 11 ms on localhost (2/2) -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.024720 s -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250380, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250452, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[74] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250590, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=256110, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[74] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 6 ms on localhost (1/2) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 8 ms on localhost (2/2) -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.018091 s -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=259082, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=259154, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[76] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=259292, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=264812, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[76] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 5 ms on localhost (1/2) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 5 ms on localhost (2/2) -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.014805 s -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=267784, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=267856, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[78] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=267994, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=273514, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[78] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 5 ms on localhost (1/2) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 5 ms on localhost (2/2) -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.014788 s -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276486, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276558, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[80] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=276696, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=282216, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[80] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 6 ms on localhost (1/2) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 6 ms on localhost (2/2) -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.015404 s -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285187, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285259, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[82] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=285397, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=290917, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[82] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 6 ms on localhost (1/2) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 7 ms on localhost (2/2) -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.020659 s -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=293888, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=293960, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[84] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=294098, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2974) called with curMem=299618, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[84] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 5 ms on localhost (1/2) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 5 ms on localhost (2/2) -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.016189 s -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=302592, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=302664, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[86] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=302802, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2974) called with curMem=308322, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[86] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 8 ms on localhost (1/2) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 10 ms on localhost (2/2) -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.022861 s -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=311296, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=311368, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[88] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=311506, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=317026, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[88] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:55 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 21:19:55 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:55 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 2423 bytes result sent to driver -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 7 ms on localhost (1/2) -2016-05-24 21:19:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 8 ms on localhost (2/2) -2016-05-24 21:19:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 21:19:55 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 21:19:55 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.024670 s -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=319998, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=320070, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:55 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:55 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:55 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:55 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:55 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[90] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:55 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=320208, maxMem=766075207 -2016-05-24 21:19:55 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=325728, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[90] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 21:19:56 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:56 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-24 21:19:56 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:56 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 8 ms on localhost (1/2) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 10 ms on localhost (2/2) -2016-05-24 21:19:56 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 21:19:56 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.033979 s -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=319788, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=311366, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 21:19:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:56 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[92] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=294099, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=299481, maxMem=766075207 -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[92] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 21:19:56 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 21:19:56 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:56 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 7 ms on localhost (1/2) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 8 ms on localhost (2/2) -2016-05-24 21:19:56 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 21:19:56 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.020103 s -2016-05-24 21:19:56 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285049, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285049, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 91 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 21:19:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:56 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[94] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=264812, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:56 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=264812, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[94] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 21:19:56 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:56 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 7 ms on localhost (1/2) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 8 ms on localhost (2/2) -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 21:19:56 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 21:19:56 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.019960 s -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=255901, maxMem=766075207 -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:56 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250453, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 93 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Got job 47 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:56 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[96] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=238708, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:56 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=238708, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 47 (MapPartitionsRDD[96] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 2 tasks -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 93, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 94, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 94) -2016-05-24 21:19:56 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 93) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:56 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:56 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 94). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 93). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 94) in 6 ms on localhost (1/2) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 93) in 10 ms on localhost (2/2) -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-24 21:19:56 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 21:19:56 INFO DAGScheduler:59 - Job 47 finished: treeAggregate at GradientDescent.scala:225, took 0.023100 s -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224276, maxMem=766075207 -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:56 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224348, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 95 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:56 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:54850 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:56 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 21:19:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Got job 48 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:56 INFO DAGScheduler:59 - Final stage: ResultStage 48(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:54850 in memory (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[98] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=207084, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=212604, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 48 (MapPartitionsRDD[98] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 2 tasks -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 95, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 96, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 96) -2016-05-24 21:19:56 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 95) -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:56 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 96). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 95). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 95) in 5 ms on localhost (1/2) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 96) in 5 ms on localhost (2/2) -2016-05-24 21:19:56 INFO DAGScheduler:59 - ResultStage 48 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-24 21:19:56 INFO DAGScheduler:59 - Job 48 finished: treeAggregate at GradientDescent.scala:225, took 0.015843 s -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=215575, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=215647, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 97 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Got job 49 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:56 INFO DAGScheduler:59 - Final stage: ResultStage 49(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[100] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=215785, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=221305, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 49 (MapPartitionsRDD[100] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 2 tasks -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 97, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 98, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 98) -2016-05-24 21:19:56 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 97) -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:56 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 97). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 98). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 97) in 5 ms on localhost (1/2) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 98) in 5 ms on localhost (2/2) -2016-05-24 21:19:56 INFO DAGScheduler:59 - ResultStage 49 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-05-24 21:19:56 INFO DAGScheduler:59 - Job 49 finished: treeAggregate at GradientDescent.scala:225, took 0.014543 s -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=224275, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=224347, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 99 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Got job 50 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:56 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[102] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=224485, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(2974) called with curMem=230005, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 50 (MapPartitionsRDD[102] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 2 tasks -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-24 21:19:56 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:56 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:56 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 15 ms on localhost (1/2) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 15 ms on localhost (2/2) -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-24 21:19:56 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 21:19:56 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at GradientDescent.scala:225, took 0.026308 s -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=232979, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=233051, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.6 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 101 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Got job 51 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:56 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[104] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=233189, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=238709, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 51 (MapPartitionsRDD[104] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-24 21:19:56 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:56 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 6 ms on localhost (1/2) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 5 ms on localhost (2/2) -2016-05-24 21:19:56 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-24 21:19:56 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at GradientDescent.scala:225, took 0.016719 s -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=241680, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=241752, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 103 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Got job 52 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:56 INFO DAGScheduler:59 - Final stage: ResultStage 52(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting ResultStage 52 (MapPartitionsRDD[106] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=241890, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=247410, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 104 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 52 (MapPartitionsRDD[106] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 103, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 104, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 103) -2016-05-24 21:19:56 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 104) -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:56 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 103). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 104). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 103) in 6 ms on localhost (1/2) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 104) in 6 ms on localhost (2/2) -2016-05-24 21:19:56 INFO DAGScheduler:59 - ResultStage 52 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-24 21:19:56 INFO DAGScheduler:59 - Job 52 finished: treeAggregate at GradientDescent.scala:225, took 0.014456 s -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=250381, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=250453, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 105 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Got job 53 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:56 INFO DAGScheduler:59 - Final stage: ResultStage 53(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[108] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=250591, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=256111, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 106 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[108] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 105, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 106, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 105) -2016-05-24 21:19:56 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 106) -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:56 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 106). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 106) in 8 ms on localhost (1/2) -2016-05-24 21:19:56 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 105). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 105) in 12 ms on localhost (2/2) -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-24 21:19:56 INFO DAGScheduler:59 - ResultStage 53 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:19:56 INFO DAGScheduler:59 - Job 53 finished: treeAggregate at GradientDescent.scala:225, took 0.020750 s -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=259083, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=259155, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 107 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Got job 54 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:56 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[110] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=259293, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=264813, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 108 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 54 (MapPartitionsRDD[110] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 2 tasks -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 107) -2016-05-24 21:19:56 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 108) -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:56 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 107). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:56 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 108). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 107) in 11 ms on localhost (1/2) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 108) in 11 ms on localhost (2/2) -2016-05-24 21:19:56 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 21:19:56 INFO DAGScheduler:59 - Job 54 finished: treeAggregate at GradientDescent.scala:225, took 0.020252 s -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=267784, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=267856, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 109 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Got job 55 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:56 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[112] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=267994, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=273514, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 110 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 55 (MapPartitionsRDD[112] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 2 tasks -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 109, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 110, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 110) -2016-05-24 21:19:56 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 109) -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:56 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 109). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 110). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 109) in 5 ms on localhost (1/2) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 110) in 4 ms on localhost (2/2) -2016-05-24 21:19:56 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-24 21:19:56 INFO DAGScheduler:59 - Job 55 finished: treeAggregate at GradientDescent.scala:225, took 0.017302 s -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=276484, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=276556, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 111 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Got job 56 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:56 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[114] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=276694, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=282214, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 112 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 56 (MapPartitionsRDD[114] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 2 tasks -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 111, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 112, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 112) -2016-05-24 21:19:56 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 111) -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:56 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 111). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 112). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 111) in 5 ms on localhost (1/2) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 112) in 6 ms on localhost (2/2) -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-24 21:19:56 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:19:56 INFO DAGScheduler:59 - Job 56 finished: treeAggregate at GradientDescent.scala:225, took 0.015669 s -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=285185, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 72.0 B, free 730.3 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=285257, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.3 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:54850 (size: 138.0 B, free: 730.5 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 113 from broadcast at GradientDescent.scala:221 -2016-05-24 21:19:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Got job 57 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:19:56 INFO DAGScheduler:59 - Final stage: ResultStage 57(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[116] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(5520) called with curMem=285395, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:19:56 INFO MemoryStore:59 - ensureFreeSpace(2968) called with curMem=290915, maxMem=766075207 -2016-05-24 21:19:56 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:19:56 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:54850 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:19:56 INFO SparkContext:59 - Created broadcast 114 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 57 (MapPartitionsRDD[116] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 2 tasks -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 113, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 114, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-24 21:19:56 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 113) -2016-05-24 21:19:56 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 114) -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:19:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:19:56 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 113). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 114). 2423 bytes result sent to driver -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 113) in 5 ms on localhost (1/2) -2016-05-24 21:19:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 114) in 4 ms on localhost (2/2) -2016-05-24 21:19:56 INFO DAGScheduler:59 - ResultStage 57 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 21:19:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-24 21:19:56 INFO DAGScheduler:59 - Job 57 finished: treeAggregate at GradientDescent.scala:225, took 0.014119 s -2016-05-24 21:19:56 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 0.6105671927144823, 0.6097961496848042, 0.6090406315074238, 0.6083000889873139, 0.6075740019440139, 0.6068618771610624, 0.6061632465163228, 0.6054776652741959, 0.6048047105229598, 0.604143979742525 -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 21:19:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 21:19:56 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 21:19:56 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 21:19:56 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 21:19:56 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 21:19:56 INFO BlockManager:59 - BlockManager stopped -2016-05-24 21:19:56 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 21:19:56 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 21:19:56 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 21:19:56 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 21:19:56 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-a94376b5-d659-4eb8-baa4-2e64c4177e83 -2016-05-24 21:23:42 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 21:23:42 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 21:23:43 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 21:23:43 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 21:23:43 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 21:23:44 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 21:23:44 INFO Remoting:74 - Starting remoting -2016-05-24 21:23:44 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:54875] -2016-05-24 21:23:44 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54875. -2016-05-24 21:23:44 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 21:23:44 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 21:23:44 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-adad01de-615f-4159-8272-db64d1d92d0a -2016-05-24 21:23:44 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 21:23:44 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-93ae946f-5c4a-4867-b0dc-77fba7d577a9\httpd-d63a283a-1504-4ce8-8449-5bcf179431f8 -2016-05-24 21:23:44 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 21:23:44 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:23:44 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54876 -2016-05-24 21:23:44 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54876. -2016-05-24 21:23:44 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 21:23:44 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:23:44 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 21:23:44 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 21:23:44 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 21:23:45 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 21:23:45 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 21:23:45 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54895. -2016-05-24 21:23:45 INFO NettyBlockTransferService:59 - Server created on 54895 -2016-05-24 21:23:45 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 21:23:45 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54895 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54895) -2016-05-24 21:23:45 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 21:23:46 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 21:23:46 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 21:23:46 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 21:23:46 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 21:23:46 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54895 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 21:23:46 INFO SparkContext:59 - Created broadcast 0 from textFile at LogisticRegressionLearning.scala:22 -2016-05-24 21:23:47 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 21:23:47 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 21:23:47 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 21:23:47 INFO DAGScheduler:59 - Got job 0 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 21:23:47 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:23:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:23:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:23:47 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 21:23:47 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=171580, maxMem=766075207 -2016-05-24 21:23:47 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-24 21:23:47 INFO MemoryStore:59 - ensureFreeSpace(2186) called with curMem=175316, maxMem=766075207 -2016-05-24 21:23:47 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 21:23:47 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54895 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:23:47 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:23:47 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:23:47 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-24 21:23:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 21:23:47 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 21:23:47 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 21:23:47 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/ridge-data/lpsa2.data:0+19 -2016-05-24 21:23:47 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 21:23:47 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 21:23:47 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 21:23:47 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 21:23:47 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 21:23:47 INFO MemoryStore:59 - ensureFreeSpace(320) called with curMem=177502, maxMem=766075207 -2016-05-24 21:23:47 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 320.0 B, free 730.4 MB) -2016-05-24 21:23:47 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:54895 (size: 320.0 B, free: 730.6 MB) -2016-05-24 21:23:47 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2611 bytes result sent to driver -2016-05-24 21:23:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 112 ms on localhost (1/1) -2016-05-24 21:23:47 INFO DAGScheduler:59 - ResultStage 0 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.130 s -2016-05-24 21:23:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 21:23:47 INFO DAGScheduler:59 - Job 0 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.274228 s -2016-05-24 21:23:47 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 21:23:47 INFO DAGScheduler:59 - Got job 1 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 21:23:47 INFO DAGScheduler:59 - Final stage: ResultStage 1(count at DataValidators.scala:40) -2016-05-24 21:23:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:23:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:23:47 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 21:23:47 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=177822, maxMem=766075207 -2016-05-24 21:23:47 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:23:47 INFO MemoryStore:59 - ensureFreeSpace(2044) called with curMem=181334, maxMem=766075207 -2016-05-24 21:23:47 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2044.0 B, free 730.4 MB) -2016-05-24 21:23:47 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54895 (size: 2044.0 B, free: 730.6 MB) -2016-05-24 21:23:47 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:23:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at filter at DataValidators.scala:40) -2016-05-24 21:23:47 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 21:23:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 21:23:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 21:23:47 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-24 21:23:47 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-24 21:23:47 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:23:47 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 21:23:47 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/ridge-data/lpsa2.data:19+19 -2016-05-24 21:23:47 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-05-24 21:23:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 21 ms on localhost (1/2) -2016-05-24 21:23:48 INFO MemoryStore:59 - ensureFreeSpace(168) called with curMem=183378, maxMem=766075207 -2016-05-24 21:23:48 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 168.0 B, free 730.4 MB) -2016-05-24 21:23:48 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:54895 (size: 168.0 B, free: 730.6 MB) -2016-05-24 21:23:48 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2662 bytes result sent to driver -2016-05-24 21:23:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 30 ms on localhost (2/2) -2016-05-24 21:23:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 21:23:48 INFO DAGScheduler:59 - ResultStage 1 (count at DataValidators.scala:40) finished in 0.034 s -2016-05-24 21:23:48 INFO DAGScheduler:59 - Job 1 finished: count at DataValidators.scala:40, took 0.064966 s -2016-05-24 21:23:48 ERROR DataValidators:75 - Classification labels should be 0 or 1. Found 6 invalid labels -2016-05-24 21:23:48 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 21:23:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 21:23:48 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 21:23:48 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 21:23:48 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 21:23:48 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 21:23:48 INFO BlockManager:59 - BlockManager stopped -2016-05-24 21:23:48 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 21:23:48 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 21:23:48 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 21:23:48 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 21:23:48 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-93ae946f-5c4a-4867-b0dc-77fba7d577a9 -2016-05-24 21:23:48 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 21:23:48 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 21:24:40 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 21:24:41 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 21:24:41 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 21:24:41 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 21:24:41 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 21:24:42 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 21:24:42 INFO Remoting:74 - Starting remoting -2016-05-24 21:24:42 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:54916] -2016-05-24 21:24:42 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54916. -2016-05-24 21:24:42 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 21:24:42 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 21:24:42 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-e3261464-01d7-49dc-9b4f-de941b4e765a -2016-05-24 21:24:42 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 21:24:43 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-8d674a08-b521-473b-aec0-37018ec9883b\httpd-78b31e5c-881e-4b46-9903-6c2fa7a7e58c -2016-05-24 21:24:43 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 21:24:43 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:24:43 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54917 -2016-05-24 21:24:43 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54917. -2016-05-24 21:24:43 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 21:24:43 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:24:43 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 21:24:43 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 21:24:43 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 21:24:43 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 21:24:43 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 21:24:43 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54936. -2016-05-24 21:24:43 INFO NettyBlockTransferService:59 - Server created on 54936 -2016-05-24 21:24:43 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 21:24:43 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54936 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54936) -2016-05-24 21:24:43 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 21:24:44 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 21:24:44 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 21:24:44 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 21:24:44 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 21:24:44 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54936 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 21:24:44 INFO SparkContext:59 - Created broadcast 0 from textFile at LogisticRegressionLearning.scala:22 -2016-05-24 21:24:45 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 21:24:45 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 21:24:45 INFO SparkContext:59 - Starting job: foreach at LogisticRegressionLearning.scala:27 -2016-05-24 21:24:45 INFO DAGScheduler:59 - Got job 0 (foreach at LogisticRegressionLearning.scala:27) with 2 output partitions -2016-05-24 21:24:45 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at LogisticRegressionLearning.scala:27) -2016-05-24 21:24:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:24:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:24:45 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at LogisticRegressionLearning.scala:23), which has no missing parents -2016-05-24 21:24:46 INFO MemoryStore:59 - ensureFreeSpace(3424) called with curMem=171580, maxMem=766075207 -2016-05-24 21:24:46 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-05-24 21:24:46 INFO MemoryStore:59 - ensureFreeSpace(1997) called with curMem=175004, maxMem=766075207 -2016-05-24 21:24:46 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1997.0 B, free 730.4 MB) -2016-05-24 21:24:46 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54936 (size: 1997.0 B, free: 730.6 MB) -2016-05-24 21:24:46 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:24:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at LogisticRegressionLearning.scala:23) -2016-05-24 21:24:46 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 21:24:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 21:24:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 21:24:46 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 21:24:46 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 21:24:46 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 21:24:46 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 21:24:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/ridge-data/lpsa2.data:0+19 -2016-05-24 21:24:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/ridge-data/lpsa2.data:19+19 -2016-05-24 21:24:46 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 21:24:46 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 21:24:46 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 21:24:46 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 21:24:46 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 21:24:46 INFO MemoryStore:59 - ensureFreeSpace(168) called with curMem=177001, maxMem=766075207 -2016-05-24 21:24:46 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 168.0 B, free 730.4 MB) -2016-05-24 21:24:46 INFO MemoryStore:59 - ensureFreeSpace(320) called with curMem=177169, maxMem=766075207 -2016-05-24 21:24:46 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 320.0 B, free 730.4 MB) -2016-05-24 21:24:46 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:54936 (size: 168.0 B, free: 730.6 MB) -2016-05-24 21:24:46 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:54936 (size: 320.0 B, free: 730.6 MB) -2016-05-24 21:24:46 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2624 bytes result sent to driver -2016-05-24 21:24:46 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2624 bytes result sent to driver -2016-05-24 21:24:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 157 ms on localhost (1/2) -2016-05-24 21:24:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 185 ms on localhost (2/2) -2016-05-24 21:24:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 21:24:46 INFO DAGScheduler:59 - ResultStage 0 (foreach at LogisticRegressionLearning.scala:27) finished in 0.208 s -2016-05-24 21:24:46 INFO DAGScheduler:59 - Job 0 finished: foreach at LogisticRegressionLearning.scala:27, took 0.330024 s -2016-05-24 21:24:46 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 21:24:46 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 21:24:46 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:24:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:24:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:24:46 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 21:24:46 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=177489, maxMem=766075207 -2016-05-24 21:24:46 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-24 21:24:46 INFO MemoryStore:59 - ensureFreeSpace(2186) called with curMem=181225, maxMem=766075207 -2016-05-24 21:24:46 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 21:24:46 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54936 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:24:46 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:24:46 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:24:46 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 21:24:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 21:24:46 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 21:24:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:24:46 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-24 21:24:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 20 ms on localhost (1/1) -2016-05-24 21:24:46 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.022 s -2016-05-24 21:24:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 21:24:46 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.050507 s -2016-05-24 21:24:46 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 21:24:46 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 21:24:46 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 21:24:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:24:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:24:46 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[4] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 21:24:46 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=183411, maxMem=766075207 -2016-05-24 21:24:46 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:24:46 INFO MemoryStore:59 - ensureFreeSpace(2044) called with curMem=186923, maxMem=766075207 -2016-05-24 21:24:46 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2044.0 B, free 730.4 MB) -2016-05-24 21:24:46 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54936 (size: 2044.0 B, free: 730.6 MB) -2016-05-24 21:24:46 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:24:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[4] at filter at DataValidators.scala:40) -2016-05-24 21:24:46 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 21:24:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 21:24:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-24 21:24:46 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 21:24:46 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 21:24:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:24:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:24:46 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 21:24:46 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 21:24:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 17 ms on localhost (1/2) -2016-05-24 21:24:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 18 ms on localhost (2/2) -2016-05-24 21:24:46 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.019 s -2016-05-24 21:24:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 21:24:46 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.035384 s -2016-05-24 21:24:46 ERROR DataValidators:75 - Classification labels should be 0 or 1. Found 6 invalid labels -2016-05-24 21:24:46 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 21:24:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 21:24:46 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 21:24:46 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 21:24:46 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 21:24:46 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 21:24:46 INFO BlockManager:59 - BlockManager stopped -2016-05-24 21:24:46 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 21:24:46 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 21:24:46 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 21:24:46 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 21:24:46 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-8d674a08-b521-473b-aec0-37018ec9883b -2016-05-24 21:26:31 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 21:26:32 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 21:26:32 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 21:26:32 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 21:26:32 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 21:26:33 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 21:26:33 INFO Remoting:74 - Starting remoting -2016-05-24 21:26:33 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:54957] -2016-05-24 21:26:33 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54957. -2016-05-24 21:26:33 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 21:26:33 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 21:26:33 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-63e940f2-0e40-4700-8944-d90ebb2334f0 -2016-05-24 21:26:33 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 21:26:33 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-751f8360-30b7-46a2-8b76-9923776fe52d\httpd-11dbb511-a819-4731-a55c-2e7eb0d38cdc -2016-05-24 21:26:33 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 21:26:33 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:26:33 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54958 -2016-05-24 21:26:33 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54958. -2016-05-24 21:26:33 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 21:26:34 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:26:34 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 21:26:34 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 21:26:34 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 21:26:34 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 21:26:34 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 21:26:34 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54977. -2016-05-24 21:26:34 INFO NettyBlockTransferService:59 - Server created on 54977 -2016-05-24 21:26:34 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 21:26:34 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54977 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54977) -2016-05-24 21:26:34 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 21:26:35 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 21:26:35 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 21:26:35 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 21:26:35 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 21:26:35 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54977 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 21:26:35 INFO SparkContext:59 - Created broadcast 0 from textFile at LogisticRegressionLearning.scala:22 -2016-05-24 21:26:36 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 21:26:36 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 21:26:36 INFO SparkContext:59 - Starting job: foreach at LogisticRegressionLearning.scala:28 -2016-05-24 21:26:37 INFO DAGScheduler:59 - Got job 0 (foreach at LogisticRegressionLearning.scala:28) with 2 output partitions -2016-05-24 21:26:37 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at LogisticRegressionLearning.scala:28) -2016-05-24 21:26:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:26:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:26:37 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at LogisticRegressionLearning.scala:23), which has no missing parents -2016-05-24 21:26:37 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=171580, maxMem=766075207 -2016-05-24 21:26:37 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:26:37 INFO MemoryStore:59 - ensureFreeSpace(1993) called with curMem=175020, maxMem=766075207 -2016-05-24 21:26:37 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1993.0 B, free 730.4 MB) -2016-05-24 21:26:37 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54977 (size: 1993.0 B, free: 730.6 MB) -2016-05-24 21:26:37 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:26:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at LogisticRegressionLearning.scala:23) -2016-05-24 21:26:37 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 21:26:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:26:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:26:37 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 21:26:37 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 21:26:37 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 21:26:37 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 21:26:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:12+13 -2016-05-24 21:26:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:0+12 -2016-05-24 21:26:37 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 21:26:37 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 21:26:37 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 21:26:37 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 21:26:37 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 21:26:37 INFO MemoryStore:59 - ensureFreeSpace(152) called with curMem=177013, maxMem=766075207 -2016-05-24 21:26:37 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 152.0 B, free 730.4 MB) -2016-05-24 21:26:37 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=177165, maxMem=766075207 -2016-05-24 21:26:37 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-05-24 21:26:37 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:54977 (size: 152.0 B, free: 730.6 MB) -2016-05-24 21:26:37 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:54977 (size: 288.0 B, free: 730.6 MB) -2016-05-24 21:26:37 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2624 bytes result sent to driver -2016-05-24 21:26:37 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2624 bytes result sent to driver -2016-05-24 21:26:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 175 ms on localhost (1/2) -2016-05-24 21:26:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 150 ms on localhost (2/2) -2016-05-24 21:26:37 INFO DAGScheduler:59 - ResultStage 0 (foreach at LogisticRegressionLearning.scala:28) finished in 0.198 s -2016-05-24 21:26:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 21:26:37 INFO DAGScheduler:59 - Job 0 finished: foreach at LogisticRegressionLearning.scala:28, took 0.331789 s -2016-05-24 21:26:37 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 21:26:37 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 21:26:37 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:26:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:26:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:26:37 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 21:26:37 INFO MemoryStore:59 - ensureFreeSpace(3752) called with curMem=177453, maxMem=766075207 -2016-05-24 21:26:37 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-24 21:26:37 INFO MemoryStore:59 - ensureFreeSpace(2181) called with curMem=181205, maxMem=766075207 -2016-05-24 21:26:37 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 21:26:37 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54977 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:26:37 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:26:37 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:26:37 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 21:26:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:26:37 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 21:26:37 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:26:37 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-24 21:26:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 19 ms on localhost (1/1) -2016-05-24 21:26:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 21:26:37 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.019 s -2016-05-24 21:26:37 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.054990 s -2016-05-24 21:26:37 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 21:26:37 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 21:26:37 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 21:26:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:26:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:26:37 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[4] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 21:26:37 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=183386, maxMem=766075207 -2016-05-24 21:26:37 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:26:37 INFO MemoryStore:59 - ensureFreeSpace(2039) called with curMem=186914, maxMem=766075207 -2016-05-24 21:26:37 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2039.0 B, free 730.4 MB) -2016-05-24 21:26:37 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54977 (size: 2039.0 B, free: 730.6 MB) -2016-05-24 21:26:37 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:26:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[4] at filter at DataValidators.scala:40) -2016-05-24 21:26:37 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 21:26:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:26:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:26:37 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 21:26:37 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 21:26:37 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:26:37 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 21:26:37 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:26:37 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 21:26:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 13 ms on localhost (1/2) -2016-05-24 21:26:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 15 ms on localhost (2/2) -2016-05-24 21:26:37 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.016 s -2016-05-24 21:26:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 21:26:37 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.031006 s -2016-05-24 21:26:37 ERROR DataValidators:75 - Classification labels should be 0 or 1. Found 6 invalid labels -2016-05-24 21:26:37 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 21:26:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 21:26:37 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 21:26:37 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 21:26:37 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 21:26:37 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 21:26:37 INFO BlockManager:59 - BlockManager stopped -2016-05-24 21:26:37 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 21:26:37 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 21:26:37 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 21:26:37 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 21:26:37 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-751f8360-30b7-46a2-8b76-9923776fe52d -2016-05-24 21:26:37 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 21:27:16 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 21:27:17 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 21:27:17 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 21:27:17 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 21:27:17 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 21:27:18 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 21:27:18 INFO Remoting:74 - Starting remoting -2016-05-24 21:27:18 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:54998] -2016-05-24 21:27:18 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54998. -2016-05-24 21:27:18 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 21:27:18 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 21:27:18 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-49baf709-9930-4ee2-8bb8-b93cffcd67c4 -2016-05-24 21:27:18 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 21:27:18 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-4d9c00a6-8b41-4438-b7c7-c17709f54db2\httpd-277d10fb-2e60-40ab-b543-8c7ccbd3100d -2016-05-24 21:27:18 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 21:27:18 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:27:18 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54999 -2016-05-24 21:27:18 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54999. -2016-05-24 21:27:18 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 21:27:18 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:27:18 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 21:27:18 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 21:27:18 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 21:27:18 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 21:27:18 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 21:27:19 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55018. -2016-05-24 21:27:19 INFO NettyBlockTransferService:59 - Server created on 55018 -2016-05-24 21:27:19 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 21:27:19 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55018 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55018) -2016-05-24 21:27:19 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 21:27:19 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 21:27:19 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 21:27:19 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 21:27:19 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 21:27:19 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55018 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 21:27:19 INFO SparkContext:59 - Created broadcast 0 from textFile at LogisticRegressionLearning.scala:22 -2016-05-24 21:27:21 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 21:27:21 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 21:27:21 INFO SparkContext:59 - Starting job: foreach at LogisticRegressionLearning.scala:28 -2016-05-24 21:27:21 INFO DAGScheduler:59 - Got job 0 (foreach at LogisticRegressionLearning.scala:28) with 2 output partitions -2016-05-24 21:27:21 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at LogisticRegressionLearning.scala:28) -2016-05-24 21:27:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:27:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:27:21 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at LogisticRegressionLearning.scala:23), which has no missing parents -2016-05-24 21:27:21 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=171580, maxMem=766075207 -2016-05-24 21:27:21 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:27:21 INFO MemoryStore:59 - ensureFreeSpace(1993) called with curMem=175020, maxMem=766075207 -2016-05-24 21:27:21 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1993.0 B, free 730.4 MB) -2016-05-24 21:27:21 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55018 (size: 1993.0 B, free: 730.6 MB) -2016-05-24 21:27:21 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:27:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at LogisticRegressionLearning.scala:23) -2016-05-24 21:27:21 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 21:27:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:27:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:27:21 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 21:27:21 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 21:27:21 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 21:27:21 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 21:27:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:12+13 -2016-05-24 21:27:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:0+12 -2016-05-24 21:27:21 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 21:27:21 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 21:27:21 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 21:27:21 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 21:27:21 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 21:27:21 INFO MemoryStore:59 - ensureFreeSpace(152) called with curMem=177013, maxMem=766075207 -2016-05-24 21:27:21 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 152.0 B, free 730.4 MB) -2016-05-24 21:27:21 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=177165, maxMem=766075207 -2016-05-24 21:27:21 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-05-24 21:27:21 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:55018 (size: 152.0 B, free: 730.6 MB) -2016-05-24 21:27:21 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:55018 (size: 288.0 B, free: 730.6 MB) -2016-05-24 21:27:21 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2624 bytes result sent to driver -2016-05-24 21:27:21 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2624 bytes result sent to driver -2016-05-24 21:27:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 110 ms on localhost (1/2) -2016-05-24 21:27:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 136 ms on localhost (2/2) -2016-05-24 21:27:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 21:27:21 INFO DAGScheduler:59 - ResultStage 0 (foreach at LogisticRegressionLearning.scala:28) finished in 0.152 s -2016-05-24 21:27:21 INFO DAGScheduler:59 - Job 0 finished: foreach at LogisticRegressionLearning.scala:28, took 0.311547 s -2016-05-24 21:27:21 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 21:27:21 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 21:27:21 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:27:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:27:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:27:21 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 21:27:21 INFO MemoryStore:59 - ensureFreeSpace(3752) called with curMem=177453, maxMem=766075207 -2016-05-24 21:27:21 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-24 21:27:21 INFO MemoryStore:59 - ensureFreeSpace(2181) called with curMem=181205, maxMem=766075207 -2016-05-24 21:27:21 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 21:27:21 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55018 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:27:21 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:27:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:27:21 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 21:27:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:27:21 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 21:27:21 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:27:21 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-24 21:27:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 16 ms on localhost (1/1) -2016-05-24 21:27:21 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.017 s -2016-05-24 21:27:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 21:27:21 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.045542 s -2016-05-24 21:27:21 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 21:27:21 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 21:27:21 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 21:27:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:27:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:27:21 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[4] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 21:27:21 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=183386, maxMem=766075207 -2016-05-24 21:27:21 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:27:21 INFO MemoryStore:59 - ensureFreeSpace(2039) called with curMem=186914, maxMem=766075207 -2016-05-24 21:27:21 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2039.0 B, free 730.4 MB) -2016-05-24 21:27:21 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55018 (size: 2039.0 B, free: 730.6 MB) -2016-05-24 21:27:21 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:27:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[4] at filter at DataValidators.scala:40) -2016-05-24 21:27:21 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 21:27:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:27:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:27:21 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 21:27:21 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 21:27:21 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:27:21 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:27:21 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 21:27:21 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 21:27:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 14 ms on localhost (1/2) -2016-05-24 21:27:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 14 ms on localhost (2/2) -2016-05-24 21:27:21 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.015 s -2016-05-24 21:27:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 21:27:21 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.029658 s -2016-05-24 21:27:21 ERROR DataValidators:75 - Classification labels should be 0 or 1. Found 6 invalid labels -2016-05-24 21:27:21 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 21:27:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 21:27:21 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 21:27:21 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 21:27:22 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 21:27:22 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 21:27:22 INFO BlockManager:59 - BlockManager stopped -2016-05-24 21:27:22 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 21:27:22 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 21:27:22 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 21:27:22 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 21:27:22 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-4d9c00a6-8b41-4438-b7c7-c17709f54db2 -2016-05-24 21:30:38 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 21:30:39 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 21:30:39 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 21:30:39 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 21:30:39 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 21:30:40 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 21:30:40 INFO Remoting:74 - Starting remoting -2016-05-24 21:30:40 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:55040] -2016-05-24 21:30:40 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55040. -2016-05-24 21:30:40 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 21:30:40 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 21:30:40 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-07377210-4103-4ad0-9682-42219d4496c6 -2016-05-24 21:30:40 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 21:30:40 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-8aa63e71-9026-4b31-8f50-6b867ceb0fa6\httpd-5902fe93-c014-4cf6-852d-59c5e7ad543b -2016-05-24 21:30:40 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 21:30:40 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:30:40 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55041 -2016-05-24 21:30:40 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55041. -2016-05-24 21:30:40 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 21:30:41 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:30:41 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 21:30:41 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 21:30:41 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 21:30:41 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 21:30:41 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 21:30:41 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55060. -2016-05-24 21:30:41 INFO NettyBlockTransferService:59 - Server created on 55060 -2016-05-24 21:30:41 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 21:30:41 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55060 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55060) -2016-05-24 21:30:41 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 21:30:42 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 21:30:42 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 21:30:42 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 21:30:42 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 21:30:42 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55060 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 21:30:42 INFO SparkContext:59 - Created broadcast 0 from textFile at LogisticRegressionLearning.scala:22 -2016-05-24 21:30:43 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 21:30:43 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 21:30:43 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 21:30:43 INFO DAGScheduler:59 - Got job 0 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 21:30:43 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:30:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:43 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 21:30:43 INFO MemoryStore:59 - ensureFreeSpace(3752) called with curMem=171580, maxMem=766075207 -2016-05-24 21:30:43 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-24 21:30:43 INFO MemoryStore:59 - ensureFreeSpace(2181) called with curMem=175332, maxMem=766075207 -2016-05-24 21:30:43 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 21:30:43 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55060 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:30:43 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:43 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:30:43 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-24 21:30:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:30:43 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 21:30:43 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 21:30:43 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:0+20 -2016-05-24 21:30:43 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 21:30:43 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 21:30:43 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 21:30:43 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 21:30:43 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(424) called with curMem=177513, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 424.0 B, free 730.4 MB) -2016-05-24 21:30:44 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:55060 (size: 424.0 B, free: 730.6 MB) -2016-05-24 21:30:44 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2611 bytes result sent to driver -2016-05-24 21:30:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 112 ms on localhost (1/1) -2016-05-24 21:30:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 21:30:44 INFO DAGScheduler:59 - ResultStage 0 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.134 s -2016-05-24 21:30:44 INFO DAGScheduler:59 - Job 0 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.242314 s -2016-05-24 21:30:44 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 21:30:44 INFO DAGScheduler:59 - Got job 1 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 21:30:44 INFO DAGScheduler:59 - Final stage: ResultStage 1(count at DataValidators.scala:40) -2016-05-24 21:30:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:44 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=177937, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(2039) called with curMem=181465, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2039.0 B, free 730.4 MB) -2016-05-24 21:30:44 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55060 (size: 2039.0 B, free: 730.6 MB) -2016-05-24 21:30:44 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at filter at DataValidators.scala:40) -2016-05-24 21:30:44 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 21:30:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:30:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:30:44 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-24 21:30:44 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-24 21:30:44 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:44 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-05-24 21:30:44 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 21:30:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:20+21 -2016-05-24 21:30:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 22 ms on localhost (1/2) -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=183504, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-05-24 21:30:44 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:55060 (size: 288.0 B, free: 730.6 MB) -2016-05-24 21:30:44 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2662 bytes result sent to driver -2016-05-24 21:30:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 36 ms on localhost (2/2) -2016-05-24 21:30:44 INFO DAGScheduler:59 - ResultStage 1 (count at DataValidators.scala:40) finished in 0.039 s -2016-05-24 21:30:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 21:30:44 INFO DAGScheduler:59 - Job 1 finished: count at DataValidators.scala:40, took 0.100455 s -2016-05-24 21:30:44 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 21:30:44 INFO DAGScheduler:59 - Got job 2 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 21:30:44 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at GradientDescent.scala:195) -2016-05-24 21:30:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:44 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[5] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=183792, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(2018) called with curMem=187248, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2018.0 B, free 730.4 MB) -2016-05-24 21:30:44 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55060 (size: 2018.0 B, free: 730.6 MB) -2016-05-24 21:30:44 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[5] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 21:30:44 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 21:30:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:30:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:30:44 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 21:30:44 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 21:30:44 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:44 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:44 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 21:30:44 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 21:30:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 14 ms on localhost (1/2) -2016-05-24 21:30:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 17 ms on localhost (2/2) -2016-05-24 21:30:44 INFO DAGScheduler:59 - ResultStage 2 (count at GradientDescent.scala:195) finished in 0.017 s -2016-05-24 21:30:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 21:30:44 INFO DAGScheduler:59 - Job 2 finished: count at GradientDescent.scala:195, took 0.035771 s -2016-05-24 21:30:44 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 21:30:44 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:55060 in memory (size: 2018.0 B, free: 730.6 MB) -2016-05-24 21:30:44 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 21:30:44 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:55060 in memory (size: 2039.0 B, free: 730.6 MB) -2016-05-24 21:30:44 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:55060 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:30:44 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 21:30:44 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 21:30:44 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=172292, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=172356, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:30:44 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:44 INFO SparkContext:59 - Created broadcast 4 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:44 INFO DAGScheduler:59 - Got job 3 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:44 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:44 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[7] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=172486, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=178022, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:30:44 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:44 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[7] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:44 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 21:30:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:44 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 21:30:44 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 21:30:44 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:44 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:44 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2415 bytes result sent to driver -2016-05-24 21:30:44 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2415 bytes result sent to driver -2016-05-24 21:30:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 23 ms on localhost (1/2) -2016-05-24 21:30:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 21 ms on localhost (2/2) -2016-05-24 21:30:44 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 21:30:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 21:30:44 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at GradientDescent.scala:225, took 0.039114 s -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=180986, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=181050, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:30:44 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:44 INFO SparkContext:59 - Created broadcast 6 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:44 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:44 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:44 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=181180, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=186716, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:30:44 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:44 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:44 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 21:30:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:44 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 21:30:44 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 21:30:44 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:44 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:44 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2415 bytes result sent to driver -2016-05-24 21:30:44 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2415 bytes result sent to driver -2016-05-24 21:30:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 15 ms on localhost (1/2) -2016-05-24 21:30:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 15 ms on localhost (2/2) -2016-05-24 21:30:44 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 21:30:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 21:30:44 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.031433 s -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=189680, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=189744, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:30:44 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:44 INFO SparkContext:59 - Created broadcast 8 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:44 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:44 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:44 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[11] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=189874, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=195410, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:30:44 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:44 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[11] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:44 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 21:30:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:44 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 21:30:44 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:44 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2415 bytes result sent to driver -2016-05-24 21:30:44 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 21:30:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 21 ms on localhost (1/2) -2016-05-24 21:30:44 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:44 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2415 bytes result sent to driver -2016-05-24 21:30:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 31 ms on localhost (2/2) -2016-05-24 21:30:44 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.030 s -2016-05-24 21:30:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 21:30:44 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.048100 s -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=198375, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=198439, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:30:44 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:44 INFO SparkContext:59 - Created broadcast 10 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:44 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:44 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:44 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=198569, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=204105, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:30:44 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:44 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:44 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 21:30:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:44 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 21:30:44 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 21:30:44 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:44 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:44 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2415 bytes result sent to driver -2016-05-24 21:30:44 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2415 bytes result sent to driver -2016-05-24 21:30:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 18 ms on localhost (1/2) -2016-05-24 21:30:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 21 ms on localhost (2/2) -2016-05-24 21:30:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 21:30:44 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 21:30:44 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.043450 s -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=207068, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=207132, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:30:44 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:44 INFO SparkContext:59 - Created broadcast 12 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:44 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:44 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:44 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=207262, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=212798, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:30:44 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:44 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:44 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 21:30:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:44 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 21:30:44 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 21:30:44 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:44 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:44 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2415 bytes result sent to driver -2016-05-24 21:30:44 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2415 bytes result sent to driver -2016-05-24 21:30:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 15 ms on localhost (1/2) -2016-05-24 21:30:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 16 ms on localhost (2/2) -2016-05-24 21:30:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 21:30:44 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:30:44 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.037435 s -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=215762, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=215826, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:30:44 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:44 INFO SparkContext:59 - Created broadcast 14 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:44 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:44 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:44 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=215956, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:30:44 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=221492, maxMem=766075207 -2016-05-24 21:30:44 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:30:44 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:44 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:44 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 21:30:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:44 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 21:30:44 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:45 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 17 ms on localhost (1/2) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 15 ms on localhost (2/2) -2016-05-24 21:30:45 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 21:30:45 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.031587 s -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224457, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224521, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 16 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=224651, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=230187, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 21:30:45 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:45 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 40 ms on localhost (1/2) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 42 ms on localhost (2/2) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 21:30:45 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.043 s -2016-05-24 21:30:45 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.057812 s -2016-05-24 21:30:45 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224521, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224521, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 18 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:45 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:45 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 21:30:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:45 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:45 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=198569, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=204105, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 21:30:45 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 21:30:45 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:45 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 16 ms on localhost (1/2) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 21 ms on localhost (2/2) -2016-05-24 21:30:45 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 21:30:45 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.047924 s -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=195411, maxMem=766075207 -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:30:45 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=189939, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 20 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=189875, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=195411, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 21:30:45 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:45 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 14 ms on localhost (1/2) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 13 ms on localhost (2/2) -2016-05-24 21:30:45 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 21:30:45 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.031436 s -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=198376, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=198440, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 22 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=198570, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=204106, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 21:30:45 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:45 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 17 ms on localhost (1/2) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 21 ms on localhost (2/2) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 21:30:45 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 21:30:45 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.043618 s -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=207070, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=207134, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 24 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=207264, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=212800, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 21:30:45 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:45 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 15 ms on localhost (1/2) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 14 ms on localhost (2/2) -2016-05-24 21:30:45 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 21:30:45 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.037031 s -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=215764, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=215828, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 26 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=215958, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=221494, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 21:30:45 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:45 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 20 ms on localhost (1/2) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 21 ms on localhost (2/2) -2016-05-24 21:30:45 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 21:30:45 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.043038 s -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224459, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224523, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 28 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=224653, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=230189, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 21:30:45 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:45 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 18 ms on localhost (1/2) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 22 ms on localhost (2/2) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 21:30:45 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 21:30:45 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.042640 s -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=233153, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=233217, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 30 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=233347, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=238883, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 21:30:45 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:45 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 11 ms on localhost (1/2) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 14 ms on localhost (2/2) -2016-05-24 21:30:45 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 21:30:45 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.029075 s -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=241847, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=241911, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 32 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=242041, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=247577, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 21:30:45 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:45 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 15 ms on localhost (1/2) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 15 ms on localhost (2/2) -2016-05-24 21:30:45 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 21:30:45 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.029368 s -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=250542, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=250606, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 34 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=250736, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=256272, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 21:30:45 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:45 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 14 ms on localhost (1/2) -2016-05-24 21:30:45 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 18 ms on localhost (2/2) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 21:30:45 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 21:30:45 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.034327 s -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=259236, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=259300, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 36 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=259430, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=264966, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 21:30:45 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:45 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 13 ms on localhost (1/2) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 13 ms on localhost (2/2) -2016-05-24 21:30:45 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 21:30:45 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.034035 s -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=267930, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=267994, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 38 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=268124, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(2961) called with curMem=273660, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 21:30:45 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:45 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 15 ms on localhost (1/2) -2016-05-24 21:30:45 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 17 ms on localhost (2/2) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 21:30:45 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 21:30:45 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.031629 s -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=276621, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=276685, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 40 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=276815, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=282351, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 21:30:45 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:45 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 12 ms on localhost (1/2) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 14 ms on localhost (2/2) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 21:30:45 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 21:30:45 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.032680 s -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=285315, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=285379, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 42 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=285509, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=291045, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 21:30:45 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:45 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 13 ms on localhost (1/2) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 16 ms on localhost (2/2) -2016-05-24 21:30:45 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 21:30:45 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.032353 s -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=294008, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=294072, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 44 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=294202, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(2961) called with curMem=299738, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 21:30:45 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:45 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 12 ms on localhost (1/2) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 12 ms on localhost (2/2) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 21:30:45 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:30:45 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.031445 s -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=302699, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=302763, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 46 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=302893, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(2961) called with curMem=308429, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:45 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 21:30:45 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:45 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:45 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 2415 bytes result sent to driver -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 11 ms on localhost (1/2) -2016-05-24 21:30:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 12 ms on localhost (2/2) -2016-05-24 21:30:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 21:30:45 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:30:45 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.030609 s -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=311390, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:45 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=311454, maxMem=766075207 -2016-05-24 21:30:45 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:45 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:45 INFO SparkContext:59 - Created broadcast 48 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:45 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:45 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=311584, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2961) called with curMem=317120, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 13 ms on localhost (1/2) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 14 ms on localhost (2/2) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.033141 s -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=320081, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=320145, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 50 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=320275, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=325811, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 9 ms on localhost (1/2) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 10 ms on localhost (2/2) -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.028515 s -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=328773, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=328837, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 52 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=328967, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=334503, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 10 ms on localhost (1/2) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 12 ms on localhost (2/2) -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.047527 s -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=328774, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=320341, maxMem=766075207 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 54 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=303089, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=308431, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 11 ms on localhost (1/2) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 11 ms on localhost (2/2) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.028577 s -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=294202, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=294266, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 56 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 21:30:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=285575, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=291047, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 20 ms on localhost (1/2) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 25 ms on localhost (2/2) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.043337 s -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=285511, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=285575, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 58 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=276881, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=282417, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 10 ms on localhost (1/2) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 11 ms on localhost (2/2) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.035429 s -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=282353, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=276881, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 60 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=268123, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=270695, maxMem=766075207 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 61 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 10 ms on localhost (1/2) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 12 ms on localhost (2/2) -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.027099 s -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=259234, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=256334, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 62 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 21:30:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=233539, maxMem=766075207 -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=238881, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 8 ms on localhost (1/2) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 8 ms on localhost (2/2) -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.018763 s -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224650, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224584, maxMem=766075207 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 64 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=224650, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=230186, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 9 ms on localhost (1/2) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 9 ms on localhost (2/2) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.020956 s -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=233150, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=233214, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 66 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=233344, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=238880, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 67 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 6 ms on localhost (1/2) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 5 ms on localhost (2/2) -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.015025 s -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=241845, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=241909, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 68 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=242039, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=247575, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 69 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 5 ms on localhost (1/2) -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 5 ms on localhost (2/2) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.014385 s -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=250539, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=250603, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 70 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=250733, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=256269, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 71 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 5 ms on localhost (1/2) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 7 ms on localhost (2/2) -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.015181 s -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=259233, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=259297, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 72 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=259427, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=264963, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 73 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 6 ms on localhost (1/2) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 6 ms on localhost (2/2) -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.016330 s -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=267925, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=267989, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 74 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=268119, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=273655, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 75 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 8 ms on localhost (1/2) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 8 ms on localhost (2/2) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.022007 s -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=276619, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=276683, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 76 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=276813, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=282349, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 77 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 6 ms on localhost (1/2) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 5 ms on localhost (2/2) -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.015042 s -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=285312, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=285376, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 78 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=285506, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=291042, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 79 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 6 ms on localhost (1/2) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 6 ms on localhost (2/2) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.017060 s -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=294006, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=294070, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 80 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=294200, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=299736, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 81 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 5 ms on localhost (1/2) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 6 ms on localhost (2/2) -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.016192 s -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=302700, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=302764, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 82 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=302894, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=308430, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 83 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 9 ms on localhost (1/2) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 9 ms on localhost (2/2) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.019138 s -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=311395, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=311459, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 84 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=311589, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=317125, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 85 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 9 ms on localhost (1/2) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 11 ms on localhost (2/2) -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.023765 s -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=320090, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=320154, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 86 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=320284, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=325820, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 87 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 5 ms on localhost (1/2) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 6 ms on localhost (2/2) -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.015860 s -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=328785, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=328849, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:55060 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 88 from broadcast at GradientDescent.scala:221 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_87_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:30:46 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=311589, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=308624, maxMem=766075207 -2016-05-24 21:30:46 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:55060 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO SparkContext:59 - Created broadcast 89 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:30:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:30:46 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 21:30:46 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 21:30:46 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 2415 bytes result sent to driver -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 8 ms on localhost (1/2) -2016-05-24 21:30:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 9 ms on localhost (2/2) -2016-05-24 21:30:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 21:30:46 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.020471 s -2016-05-24 21:30:46 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 0.6307644134901649, 0.6301608260893138, 0.62961849864768, 0.6293831586494314, 0.6292340893598364, 0.629169306843859, 0.6291372385030662, 0.6291242484799721, 0.629118922682065, 0.6291169842584431 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 21:30:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:46 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:55060 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:30:46 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 21:30:46 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:55060 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:30:46 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 21:30:47 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 21:30:47 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 21:30:47 INFO BlockManager:59 - BlockManager stopped -2016-05-24 21:30:47 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 21:30:47 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 21:30:47 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 21:30:47 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 21:30:47 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-8aa63e71-9026-4b31-8f50-6b867ceb0fa6 -2016-05-24 21:30:47 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 21:31:02 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 21:31:03 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 21:31:03 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 21:31:03 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 21:31:03 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 21:31:04 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 21:31:04 INFO Remoting:74 - Starting remoting -2016-05-24 21:31:04 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:55081] -2016-05-24 21:31:04 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55081. -2016-05-24 21:31:04 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 21:31:04 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 21:31:04 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-4d406d3c-33e9-470d-a6de-3e464d5a33bc -2016-05-24 21:31:04 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 21:31:04 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-30653218-a506-4258-b1f6-85b36822c17f\httpd-80f113c8-c61d-4369-9e66-ffe8dbe9d713 -2016-05-24 21:31:04 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 21:31:04 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:31:04 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55082 -2016-05-24 21:31:04 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55082. -2016-05-24 21:31:04 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 21:31:04 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:31:04 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 21:31:04 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 21:31:04 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 21:31:05 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 21:31:05 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 21:31:05 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55101. -2016-05-24 21:31:05 INFO NettyBlockTransferService:59 - Server created on 55101 -2016-05-24 21:31:05 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 21:31:05 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55101 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55101) -2016-05-24 21:31:05 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 21:31:05 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 21:31:05 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 21:31:05 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 21:31:05 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 21:31:05 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55101 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 21:31:05 INFO SparkContext:59 - Created broadcast 0 from textFile at LogisticRegressionLearning.scala:22 -2016-05-24 21:31:07 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 21:31:07 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 21:31:07 INFO SparkContext:59 - Starting job: foreach at LogisticRegressionLearning.scala:27 -2016-05-24 21:31:07 INFO DAGScheduler:59 - Got job 0 (foreach at LogisticRegressionLearning.scala:27) with 2 output partitions -2016-05-24 21:31:07 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at LogisticRegressionLearning.scala:27) -2016-05-24 21:31:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:07 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at LogisticRegressionLearning.scala:23), which has no missing parents -2016-05-24 21:31:07 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=171580, maxMem=766075207 -2016-05-24 21:31:07 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:31:07 INFO MemoryStore:59 - ensureFreeSpace(1994) called with curMem=175020, maxMem=766075207 -2016-05-24 21:31:07 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1994.0 B, free 730.4 MB) -2016-05-24 21:31:07 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55101 (size: 1994.0 B, free: 730.6 MB) -2016-05-24 21:31:07 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at LogisticRegressionLearning.scala:23) -2016-05-24 21:31:07 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 21:31:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:07 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 21:31:07 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 21:31:07 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 21:31:07 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 21:31:07 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:20+21 -2016-05-24 21:31:07 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:0+20 -2016-05-24 21:31:07 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 21:31:07 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 21:31:07 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 21:31:07 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 21:31:07 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 21:31:07 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=177014, maxMem=766075207 -2016-05-24 21:31:07 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-05-24 21:31:07 INFO MemoryStore:59 - ensureFreeSpace(424) called with curMem=177302, maxMem=766075207 -2016-05-24 21:31:07 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 424.0 B, free 730.4 MB) -2016-05-24 21:31:07 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:55101 (size: 288.0 B, free: 730.6 MB) -2016-05-24 21:31:07 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:55101 (size: 424.0 B, free: 730.6 MB) -2016-05-24 21:31:07 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2624 bytes result sent to driver -2016-05-24 21:31:07 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2624 bytes result sent to driver -2016-05-24 21:31:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 131 ms on localhost (1/2) -2016-05-24 21:31:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 114 ms on localhost (2/2) -2016-05-24 21:31:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 21:31:07 INFO DAGScheduler:59 - ResultStage 0 (foreach at LogisticRegressionLearning.scala:27) finished in 0.148 s -2016-05-24 21:31:07 INFO DAGScheduler:59 - Job 0 finished: foreach at LogisticRegressionLearning.scala:27, took 0.273567 s -2016-05-24 21:31:07 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 21:31:07 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 21:31:07 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:31:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:07 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 21:31:07 INFO MemoryStore:59 - ensureFreeSpace(3752) called with curMem=177726, maxMem=766075207 -2016-05-24 21:31:07 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-24 21:31:07 INFO MemoryStore:59 - ensureFreeSpace(2182) called with curMem=181478, maxMem=766075207 -2016-05-24 21:31:07 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 21:31:07 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55101 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:31:07 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:07 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:31:07 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 21:31:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:07 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 21:31:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:07 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-24 21:31:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 18 ms on localhost (1/1) -2016-05-24 21:31:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 21:31:07 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.020 s -2016-05-24 21:31:07 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.050979 s -2016-05-24 21:31:07 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 21:31:07 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 21:31:07 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 21:31:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:07 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[4] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 21:31:07 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=183660, maxMem=766075207 -2016-05-24 21:31:07 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:31:07 INFO MemoryStore:59 - ensureFreeSpace(2040) called with curMem=187188, maxMem=766075207 -2016-05-24 21:31:07 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2040.0 B, free 730.4 MB) -2016-05-24 21:31:07 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55101 (size: 2040.0 B, free: 730.6 MB) -2016-05-24 21:31:07 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[4] at filter at DataValidators.scala:40) -2016-05-24 21:31:07 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 21:31:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:07 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 21:31:07 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 21:31:07 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:07 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 21:31:07 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 21:31:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 21 ms on localhost (1/2) -2016-05-24 21:31:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 23 ms on localhost (2/2) -2016-05-24 21:31:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 21:31:07 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.024 s -2016-05-24 21:31:07 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.050087 s -2016-05-24 21:31:07 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 21:31:07 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 21:31:07 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 21:31:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:07 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[5] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 21:31:07 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=189228, maxMem=766075207 -2016-05-24 21:31:07 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:31:07 INFO MemoryStore:59 - ensureFreeSpace(2019) called with curMem=192684, maxMem=766075207 -2016-05-24 21:31:07 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2019.0 B, free 730.4 MB) -2016-05-24 21:31:07 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55101 (size: 2019.0 B, free: 730.6 MB) -2016-05-24 21:31:07 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[5] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 21:31:07 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 21:31:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:07 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 21:31:07 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 21:31:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:07 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:07 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 21:31:07 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 21:31:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 15 ms on localhost (1/2) -2016-05-24 21:31:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 14 ms on localhost (2/2) -2016-05-24 21:31:07 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.017 s -2016-05-24 21:31:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 21:31:07 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.034628 s -2016-05-24 21:31:07 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:55101 in memory (size: 2019.0 B, free: 730.6 MB) -2016-05-24 21:31:07 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:55101 in memory (size: 2040.0 B, free: 730.6 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:55101 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:55101 in memory (size: 1994.0 B, free: 730.6 MB) -2016-05-24 21:31:08 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 21:31:08 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=172292, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=172356, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:08 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:08 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:08 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[7] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=172486, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=178022, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[7] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:08 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 21:31:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:08 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 21:31:08 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 21:31:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:08 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2415 bytes result sent to driver -2016-05-24 21:31:08 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2415 bytes result sent to driver -2016-05-24 21:31:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 20 ms on localhost (1/2) -2016-05-24 21:31:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 20 ms on localhost (2/2) -2016-05-24 21:31:08 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 21:31:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 21:31:08 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.037481 s -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=180986, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=181050, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:08 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:08 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:08 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[9] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=181180, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=186716, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[9] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:08 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 21:31:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:08 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 21:31:08 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 21:31:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:08 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2415 bytes result sent to driver -2016-05-24 21:31:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:08 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2415 bytes result sent to driver -2016-05-24 21:31:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 18 ms on localhost (1/2) -2016-05-24 21:31:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 24 ms on localhost (2/2) -2016-05-24 21:31:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 21:31:08 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 21:31:08 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.060556 s -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=189681, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=189745, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:08 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:08 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:08 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[11] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=189875, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=195411, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[11] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:08 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 21:31:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:08 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 21:31:08 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 21:31:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:08 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2415 bytes result sent to driver -2016-05-24 21:31:08 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2415 bytes result sent to driver -2016-05-24 21:31:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 17 ms on localhost (1/2) -2016-05-24 21:31:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 20 ms on localhost (2/2) -2016-05-24 21:31:08 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 21:31:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 21:31:08 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.041562 s -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=198377, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=198441, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:08 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:08 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:08 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=198571, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=204107, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:08 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 21:31:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:08 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 21:31:08 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 21:31:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:08 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2415 bytes result sent to driver -2016-05-24 21:31:08 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2415 bytes result sent to driver -2016-05-24 21:31:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 16 ms on localhost (1/2) -2016-05-24 21:31:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 18 ms on localhost (2/2) -2016-05-24 21:31:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 21:31:08 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 21:31:08 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.037701 s -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=207071, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=207135, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:08 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:08 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:08 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=207265, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=212801, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:08 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 21:31:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:08 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 21:31:08 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 21:31:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:08 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2415 bytes result sent to driver -2016-05-24 21:31:08 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2415 bytes result sent to driver -2016-05-24 21:31:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 19 ms on localhost (1/2) -2016-05-24 21:31:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 24 ms on localhost (2/2) -2016-05-24 21:31:08 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 21:31:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 21:31:08 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.065854 s -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=215765, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=215829, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:08 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:08 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:08 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=215959, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=221495, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:08 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 21:31:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:08 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 21:31:08 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 21:31:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:08 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2415 bytes result sent to driver -2016-05-24 21:31:08 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2415 bytes result sent to driver -2016-05-24 21:31:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 13 ms on localhost (1/2) -2016-05-24 21:31:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 14 ms on localhost (2/2) -2016-05-24 21:31:08 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 21:31:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 21:31:08 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.030534 s -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224460, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224524, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:08 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:08 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:08 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:08 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:08 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=207459, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=212865, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:08 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 21:31:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:08 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 21:31:08 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 21:31:08 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:08 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2415 bytes result sent to driver -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 16 ms on localhost (1/2) -2016-05-24 21:31:08 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 21:31:08 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2415 bytes result sent to driver -2016-05-24 21:31:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 22 ms on localhost (2/2) -2016-05-24 21:31:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 21:31:08 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:08 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.043209 s -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=189876, maxMem=766075207 -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:08 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=189746, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:08 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:08 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:08 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:08 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=181182, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=186718, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:08 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 21:31:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:08 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 21:31:08 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 21:31:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:08 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2415 bytes result sent to driver -2016-05-24 21:31:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:08 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2415 bytes result sent to driver -2016-05-24 21:31:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 12 ms on localhost (1/2) -2016-05-24 21:31:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 13 ms on localhost (2/2) -2016-05-24 21:31:08 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:31:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 21:31:08 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.032825 s -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=189683, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=189747, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:08 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:08 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:08 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=189877, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=195413, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:08 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 21:31:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:08 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 21:31:08 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 21:31:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:08 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2415 bytes result sent to driver -2016-05-24 21:31:08 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2415 bytes result sent to driver -2016-05-24 21:31:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 13 ms on localhost (1/2) -2016-05-24 21:31:08 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:31:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 17 ms on localhost (2/2) -2016-05-24 21:31:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 21:31:08 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.035499 s -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=198379, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=198443, maxMem=766075207 -2016-05-24 21:31:08 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:08 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:08 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:08 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:08 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:08 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=198573, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=204109, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 21:31:09 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:09 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 15 ms on localhost (1/2) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 14 ms on localhost (2/2) -2016-05-24 21:31:09 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 21:31:09 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.035281 s -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=207075, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=207139, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:09 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=207269, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=212805, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 21:31:09 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:09 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 17 ms on localhost (1/2) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 20 ms on localhost (2/2) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 21:31:09 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 21:31:09 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.036928 s -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=215770, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=215834, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:09 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=215964, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=221500, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 21:31:09 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:09 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 16 ms on localhost (1/2) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 19 ms on localhost (2/2) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 21:31:09 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 21:31:09 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.040047 s -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224471, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224535, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:09 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=224665, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=230201, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 21:31:09 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:09 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 12 ms on localhost (1/2) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 13 ms on localhost (2/2) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 21:31:09 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:31:09 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.028803 s -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=233166, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=233230, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:09 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=233360, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=238896, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 21:31:09 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:09 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 15 ms on localhost (1/2) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 16 ms on localhost (2/2) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 21:31:09 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 21:31:09 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.032264 s -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=241862, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=241926, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:09 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=242056, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=247592, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 21:31:09 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:09 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 12 ms on localhost (1/2) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 15 ms on localhost (2/2) -2016-05-24 21:31:09 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 21:31:09 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.029732 s -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=250558, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=250622, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:09 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=250752, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=256288, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 21:31:09 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:09 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 12 ms on localhost (1/2) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 16 ms on localhost (2/2) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 21:31:09 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:31:09 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.031566 s -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=259253, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=259317, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:09 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=259447, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=264983, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 21:31:09 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:09 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 14 ms on localhost (1/2) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 16 ms on localhost (2/2) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 21:31:09 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 21:31:09 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.034410 s -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=267949, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=268013, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:09 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=268143, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=273679, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 21:31:09 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:09 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 14 ms on localhost (1/2) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 15 ms on localhost (2/2) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 21:31:09 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:31:09 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.036838 s -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=276641, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=276705, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:09 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=276835, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=282371, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 21:31:09 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:09 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 12 ms on localhost (1/2) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 15 ms on localhost (2/2) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 21:31:09 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 21:31:09 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.032520 s -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=285337, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=285401, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:09 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=285531, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=291067, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 21:31:09 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:09 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 13 ms on localhost (1/2) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 13 ms on localhost (2/2) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 21:31:09 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:31:09 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.028992 s -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=294031, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=294095, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:09 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=294225, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=299761, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 21:31:09 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:09 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 12 ms on localhost (1/2) -2016-05-24 21:31:09 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 13 ms on localhost (2/2) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 21:31:09 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 21:31:09 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.031698 s -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=302723, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=302787, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:09 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=302917, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=308453, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 21:31:09 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:09 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 12 ms on localhost (1/2) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 12 ms on localhost (2/2) -2016-05-24 21:31:09 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 21:31:09 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.029909 s -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=311417, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=311481, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:09 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=311611, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=317147, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 21:31:09 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:09 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 10 ms on localhost (1/2) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 12 ms on localhost (2/2) -2016-05-24 21:31:09 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 21:31:09 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.028567 s -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=320113, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=320177, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:09 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=320307, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=325843, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:09 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 21:31:09 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:09 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 12 ms on localhost (1/2) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 13 ms on localhost (2/2) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 21:31:09 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 21:31:09 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.043757 s -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=311604, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=308704, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:09 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:09 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=285718, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=291254, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 21:31:09 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:09 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 8 ms on localhost (1/2) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 11 ms on localhost (2/2) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 21:31:09 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 21:31:09 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.023816 s -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=282560, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=282624, maxMem=766075207 -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:09 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=259827, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=265169, maxMem=766075207 -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 21:31:09 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:09 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 10 ms on localhost (1/2) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 13 ms on localhost (2/2) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 21:31:09 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.027916 s -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=259439, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=251001, maxMem=766075207 -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:09 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 21:31:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:09 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=242242, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=247778, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 21:31:09 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:09 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:09 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 2415 bytes result sent to driver -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 10 ms on localhost (1/2) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 12 ms on localhost (2/2) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 21:31:09 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 21:31:09 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.027350 s -2016-05-24 21:31:09 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=242048, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=241982, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:09 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:09 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=224850, maxMem=766075207 -2016-05-24 21:31:09 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:09 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=230256, maxMem=766075207 -2016-05-24 21:31:09 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:09 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:09 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:09 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:09 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 21:31:09 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:10 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 21:31:10 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 10 ms on localhost (1/2) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 12 ms on localhost (2/2) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 21:31:10 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:31:10 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.027679 s -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224463, maxMem=766075207 -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=221561, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:10 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:10 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=215961, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=221497, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 21:31:10 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:10 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 9 ms on localhost (1/2) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 9 ms on localhost (2/2) -2016-05-24 21:31:10 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 21:31:10 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.021995 s -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224462, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224526, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:10 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=224656, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=230192, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 21:31:10 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:10 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 6 ms on localhost (1/2) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 6 ms on localhost (2/2) -2016-05-24 21:31:10 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 21:31:10 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.015341 s -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=233158, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=233222, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:10 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=233352, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=238888, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 21:31:10 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:10 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 5 ms on localhost (1/2) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 7 ms on localhost (2/2) -2016-05-24 21:31:10 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 21:31:10 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.016856 s -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=241853, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=241917, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:10 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=242047, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=247583, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 21:31:10 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:10 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 6 ms on localhost (1/2) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 8 ms on localhost (2/2) -2016-05-24 21:31:10 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 21:31:10 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.018582 s -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=250549, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=250613, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:10 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=250743, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=256279, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 21:31:10 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:10 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 9 ms on localhost (1/2) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 9 ms on localhost (2/2) -2016-05-24 21:31:10 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 21:31:10 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.025713 s -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=259245, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=259309, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:10 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=259439, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=264975, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 21:31:10 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:10 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 10 ms on localhost (1/2) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 12 ms on localhost (2/2) -2016-05-24 21:31:10 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 21:31:10 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.030104 s -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=267941, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=268005, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:10 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=268135, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=273671, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 21:31:10 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:10 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 8 ms on localhost (1/2) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 8 ms on localhost (2/2) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 21:31:10 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 21:31:10 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.023262 s -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=276637, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=276701, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:10 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=276831, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=282367, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 21:31:10 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:10 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 8 ms on localhost (1/2) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 8 ms on localhost (2/2) -2016-05-24 21:31:10 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 21:31:10 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.022891 s -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=285333, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=285397, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:10 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=285527, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=291063, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 21:31:10 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:10 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 9 ms on localhost (1/2) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 8 ms on localhost (2/2) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 21:31:10 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 21:31:10 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.020974 s -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=294027, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=294091, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:10 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=294221, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=299757, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 21:31:10 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:10 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 6 ms on localhost (1/2) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 8 ms on localhost (2/2) -2016-05-24 21:31:10 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 21:31:10 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.023613 s -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=302722, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=302786, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:10 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=302916, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=308452, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 21:31:10 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:10 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 5 ms on localhost (1/2) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 7 ms on localhost (2/2) -2016-05-24 21:31:10 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 21:31:10 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.018053 s -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=311417, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=311481, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:10 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=311611, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=317147, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 21:31:10 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:10 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 10 ms on localhost (1/2) -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:10 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 12 ms on localhost (2/2) -2016-05-24 21:31:10 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 21:31:10 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.025800 s -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=320112, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=320176, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:10 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=320306, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=325842, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 21:31:10 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:10 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 49 ms on localhost (1/2) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 49 ms on localhost (2/2) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 21:31:10 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.052 s -2016-05-24 21:31:10 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.062237 s -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=328808, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=328872, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:10 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=329002, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=331572, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 21:31:10 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:10 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 8 ms on localhost (1/2) -2016-05-24 21:31:10 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 9 ms on localhost (2/2) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 21:31:10 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.036005 s -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=320113, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=311676, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:55101 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:10 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:10 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:10 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=294353, maxMem=766075207 -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=299759, maxMem=766075207 -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:55101 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 21:31:10 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:10 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 21:31:10 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:10 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:10 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 2415 bytes result sent to driver -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 16 ms on localhost (1/2) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 19 ms on localhost (2/2) -2016-05-24 21:31:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 21:31:10 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 21:31:10 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.034619 s -2016-05-24 21:31:10 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 0.6307644134901649, 0.6301608260893138, 0.62961849864768, 0.6293831586494314, 0.6292340893598364, 0.629169306843859, 0.6291372385030662, 0.6291242484799721, 0.629118922682065, 0.6291169842584431 -2016-05-24 21:31:10 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 21:31:10 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 21:31:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:10 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:10 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:10 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:10 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 21:31:10 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:55101 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:10 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:55101 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:10 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 21:31:10 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 21:31:10 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 21:31:10 INFO BlockManager:59 - BlockManager stopped -2016-05-24 21:31:11 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 21:31:11 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 21:31:11 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 21:31:11 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 21:31:11 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-30653218-a506-4258-b1f6-85b36822c17f -2016-05-24 21:31:11 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 21:31:11 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 21:31:32 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 21:31:33 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 21:31:33 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 21:31:33 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 21:31:33 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 21:31:34 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 21:31:34 INFO Remoting:74 - Starting remoting -2016-05-24 21:31:34 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:55122] -2016-05-24 21:31:34 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55122. -2016-05-24 21:31:34 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 21:31:34 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 21:31:34 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-630fcb3a-c4eb-472c-baa4-800788bc7a37 -2016-05-24 21:31:34 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 21:31:34 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-21c60e4c-2616-4003-8473-7ab60acd1b33\httpd-6466ca2d-15e0-4a73-a9bb-b1d382ab86eb -2016-05-24 21:31:34 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 21:31:34 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:31:34 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55123 -2016-05-24 21:31:34 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55123. -2016-05-24 21:31:34 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 21:31:34 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:31:34 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 21:31:34 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 21:31:34 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 21:31:35 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 21:31:35 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 21:31:35 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55142. -2016-05-24 21:31:35 INFO NettyBlockTransferService:59 - Server created on 55142 -2016-05-24 21:31:35 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 21:31:35 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55142 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55142) -2016-05-24 21:31:35 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 21:31:36 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 21:31:36 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 21:31:36 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 21:31:36 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 21:31:36 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55142 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 21:31:36 INFO SparkContext:59 - Created broadcast 0 from textFile at LogisticRegressionLearning.scala:22 -2016-05-24 21:31:37 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 21:31:37 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 21:31:37 INFO SparkContext:59 - Starting job: foreach at LogisticRegressionLearning.scala:27 -2016-05-24 21:31:37 INFO DAGScheduler:59 - Got job 0 (foreach at LogisticRegressionLearning.scala:27) with 2 output partitions -2016-05-24 21:31:37 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at LogisticRegressionLearning.scala:27) -2016-05-24 21:31:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:37 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at LogisticRegressionLearning.scala:23), which has no missing parents -2016-05-24 21:31:38 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=171580, maxMem=766075207 -2016-05-24 21:31:38 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:31:38 INFO MemoryStore:59 - ensureFreeSpace(1993) called with curMem=175020, maxMem=766075207 -2016-05-24 21:31:38 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1993.0 B, free 730.4 MB) -2016-05-24 21:31:38 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55142 (size: 1993.0 B, free: 730.6 MB) -2016-05-24 21:31:38 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at LogisticRegressionLearning.scala:23) -2016-05-24 21:31:38 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 21:31:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:38 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 21:31:38 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 21:31:38 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 21:31:38 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 21:31:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:0+20 -2016-05-24 21:31:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:20+21 -2016-05-24 21:31:38 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 21:31:38 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 21:31:38 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 21:31:38 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 21:31:38 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 21:31:38 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=177013, maxMem=766075207 -2016-05-24 21:31:38 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-05-24 21:31:38 INFO MemoryStore:59 - ensureFreeSpace(424) called with curMem=177301, maxMem=766075207 -2016-05-24 21:31:38 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 424.0 B, free 730.4 MB) -2016-05-24 21:31:38 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:55142 (size: 288.0 B, free: 730.6 MB) -2016-05-24 21:31:38 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:55142 (size: 424.0 B, free: 730.6 MB) -2016-05-24 21:31:38 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2624 bytes result sent to driver -2016-05-24 21:31:38 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2624 bytes result sent to driver -2016-05-24 21:31:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 162 ms on localhost (1/2) -2016-05-24 21:31:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 141 ms on localhost (2/2) -2016-05-24 21:31:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 21:31:38 INFO DAGScheduler:59 - ResultStage 0 (foreach at LogisticRegressionLearning.scala:27) finished in 0.190 s -2016-05-24 21:31:38 INFO DAGScheduler:59 - Job 0 finished: foreach at LogisticRegressionLearning.scala:27, took 0.353265 s -2016-05-24 21:31:38 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 21:31:38 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 21:31:38 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:31:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:38 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 21:31:38 INFO MemoryStore:59 - ensureFreeSpace(3752) called with curMem=177725, maxMem=766075207 -2016-05-24 21:31:38 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-24 21:31:38 INFO MemoryStore:59 - ensureFreeSpace(2181) called with curMem=181477, maxMem=766075207 -2016-05-24 21:31:38 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 21:31:38 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55142 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:31:38 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:38 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:31:38 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 21:31:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:38 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 21:31:38 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:38 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-24 21:31:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 25 ms on localhost (1/1) -2016-05-24 21:31:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 21:31:38 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.029 s -2016-05-24 21:31:38 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.318991 s -2016-05-24 21:31:38 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 21:31:38 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 21:31:38 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 21:31:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:38 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[4] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 21:31:38 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=183658, maxMem=766075207 -2016-05-24 21:31:38 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:31:38 INFO MemoryStore:59 - ensureFreeSpace(2039) called with curMem=187186, maxMem=766075207 -2016-05-24 21:31:38 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2039.0 B, free 730.4 MB) -2016-05-24 21:31:38 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55142 (size: 2039.0 B, free: 730.6 MB) -2016-05-24 21:31:38 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[4] at filter at DataValidators.scala:40) -2016-05-24 21:31:38 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 21:31:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:38 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 21:31:38 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 21:31:38 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:38 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:38 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 21:31:38 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 21:31:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 15 ms on localhost (1/2) -2016-05-24 21:31:38 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.017 s -2016-05-24 21:31:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 15 ms on localhost (2/2) -2016-05-24 21:31:38 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.034646 s -2016-05-24 21:31:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 21:31:38 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 21:31:38 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 21:31:38 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 21:31:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:38 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[5] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 21:31:38 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=189225, maxMem=766075207 -2016-05-24 21:31:38 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:31:38 INFO MemoryStore:59 - ensureFreeSpace(2018) called with curMem=192681, maxMem=766075207 -2016-05-24 21:31:38 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2018.0 B, free 730.4 MB) -2016-05-24 21:31:38 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55142 (size: 2018.0 B, free: 730.6 MB) -2016-05-24 21:31:38 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[5] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 21:31:38 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 21:31:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:38 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 21:31:38 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 21:31:38 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:38 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:38 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 21:31:38 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 21:31:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 11 ms on localhost (1/2) -2016-05-24 21:31:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 12 ms on localhost (2/2) -2016-05-24 21:31:38 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.013 s -2016-05-24 21:31:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 21:31:38 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.029181 s -2016-05-24 21:31:38 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 21:31:38 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:55142 in memory (size: 2039.0 B, free: 730.6 MB) -2016-05-24 21:31:38 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 21:31:38 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:55142 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:31:38 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 21:31:38 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:55142 in memory (size: 1993.0 B, free: 730.6 MB) -2016-05-24 21:31:38 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 21:31:38 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:55142 in memory (size: 2018.0 B, free: 730.6 MB) -2016-05-24 21:31:39 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 21:31:39 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=172292, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=172356, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:39 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[7] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=172486, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=178022, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[7] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 21:31:39 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:39 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 20 ms on localhost (1/2) -2016-05-24 21:31:39 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 18 ms on localhost (2/2) -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 21:31:39 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.036980 s -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=180986, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=181050, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:39 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[9] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=181180, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=186716, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[9] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 21:31:39 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:39 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:39 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 18 ms on localhost (1/2) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 24 ms on localhost (2/2) -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 21:31:39 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.027 s -2016-05-24 21:31:39 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.044567 s -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=189680, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=189744, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:39 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[11] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=189874, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=195410, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[11] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 21:31:39 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:39 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 19 ms on localhost (1/2) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 17 ms on localhost (2/2) -2016-05-24 21:31:39 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 21:31:39 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.043142 s -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=198374, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=198438, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:39 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=198568, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=204104, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 21:31:39 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:39 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 18 ms on localhost (1/2) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 17 ms on localhost (2/2) -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 21:31:39 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 21:31:39 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.039347 s -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=207067, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=207131, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:39 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=207261, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=212797, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 21:31:39 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:39 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 17 ms on localhost (1/2) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 21 ms on localhost (2/2) -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 21:31:39 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 21:31:39 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.037274 s -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=215759, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=215823, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:39 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=215953, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=221489, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 21:31:39 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:39 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 19 ms on localhost (1/2) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 25 ms on localhost (2/2) -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 21:31:39 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.029 s -2016-05-24 21:31:39 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.044971 s -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224453, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224517, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:39 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:39 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:39 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=207261, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=204298, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:39 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 21:31:39 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:39 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:39 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 16 ms on localhost (1/2) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 17 ms on localhost (2/2) -2016-05-24 21:31:39 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 21:31:39 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.034958 s -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=189874, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:39 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=189938, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:39 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:39 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=181180, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=186716, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 21:31:39 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:39 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 14 ms on localhost (1/2) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 16 ms on localhost (2/2) -2016-05-24 21:31:39 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 21:31:39 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.040467 s -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=189681, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=189745, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:39 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=189875, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=195411, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 21:31:39 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:39 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 16 ms on localhost (1/2) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 18 ms on localhost (2/2) -2016-05-24 21:31:39 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 21:31:39 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.037399 s -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=198376, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=198440, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:39 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=198570, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=204106, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 21:31:39 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:39 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 14 ms on localhost (1/2) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 17 ms on localhost (2/2) -2016-05-24 21:31:39 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 21:31:39 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.038654 s -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=207070, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=207134, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:39 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=207264, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=212800, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 21:31:39 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:39 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 15 ms on localhost (1/2) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 18 ms on localhost (2/2) -2016-05-24 21:31:39 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 21:31:39 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.035909 s -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=215765, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=215829, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:39 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=215959, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO MemoryStore:59 - ensureFreeSpace(2968) called with curMem=221495, maxMem=766075207 -2016-05-24 21:31:39 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:39 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:39 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:39 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:39 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 21:31:39 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:39 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 2415 bytes result sent to driver -2016-05-24 21:31:39 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 22 ms on localhost (1/2) -2016-05-24 21:31:40 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 26 ms on localhost (2/2) -2016-05-24 21:31:40 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 21:31:40 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.049075 s -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224463, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224527, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:40 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:40 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=224657, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=230193, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 21:31:40 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:40 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 13 ms on localhost (1/2) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 13 ms on localhost (2/2) -2016-05-24 21:31:40 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:31:40 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.030174 s -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=233157, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=233221, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:40 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:40 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=233351, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=238887, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 21:31:40 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:40 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:40 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 26 ms on localhost (1/2) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 27 ms on localhost (2/2) -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 21:31:40 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.030 s -2016-05-24 21:31:40 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.042755 s -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=241852, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=241916, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:40 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:40 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=242046, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=247582, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 21:31:40 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:40 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 11 ms on localhost (1/2) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 12 ms on localhost (2/2) -2016-05-24 21:31:40 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 21:31:40 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.025855 s -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=250547, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=250611, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:40 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:40 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=250741, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=256277, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:40 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:40 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 43 ms on localhost (1/2) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 42 ms on localhost (2/2) -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 21:31:40 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.046 s -2016-05-24 21:31:40 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.098770 s -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=259242, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=259306, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:40 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:40 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=259436, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=264972, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:40 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:40 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 24 ms on localhost (1/2) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 28 ms on localhost (2/2) -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 21:31:40 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.030 s -2016-05-24 21:31:40 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.051408 s -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=267936, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=268000, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:40 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:40 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=268130, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=273666, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:40 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:40 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 22 ms on localhost (1/2) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 23 ms on localhost (2/2) -2016-05-24 21:31:40 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.025 s -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 21:31:40 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.045731 s -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=276626, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=276690, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:40 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:40 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=276820, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=282356, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 21:31:40 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:40 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 13 ms on localhost (1/2) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 16 ms on localhost (2/2) -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 21:31:40 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 21:31:40 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.045415 s -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=285319, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=285383, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:40 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:40 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=285513, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=291049, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 21:31:40 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:40 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 22 ms on localhost (1/2) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 26 ms on localhost (2/2) -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 21:31:40 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 21:31:40 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.042454 s -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=294012, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=294076, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:40 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:40 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=294206, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=299742, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:40 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:40 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 27 ms on localhost (1/2) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 30 ms on localhost (2/2) -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 21:31:40 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.032 s -2016-05-24 21:31:40 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.082795 s -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=302705, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=302769, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:40 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:40 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=302899, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=308435, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 21:31:40 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:40 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:40 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 15 ms on localhost (1/2) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 26 ms on localhost (2/2) -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 21:31:40 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.027 s -2016-05-24 21:31:40 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.051141 s -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=311397, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=311461, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:40 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:40 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=311591, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=317127, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 21:31:40 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:40 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 9 ms on localhost (1/2) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 10 ms on localhost (2/2) -2016-05-24 21:31:40 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 21:31:40 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.025214 s -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=320089, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=320153, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:40 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:40 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=320283, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=325819, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:40 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 21:31:40 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:40 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 66 ms on localhost (1/2) -2016-05-24 21:31:40 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 2415 bytes result sent to driver -2016-05-24 21:31:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 10 ms on localhost (2/2) -2016-05-24 21:31:40 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.069 s -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:40 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.105116 s -2016-05-24 21:31:40 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=311592, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=311462, maxMem=766075207 -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:40 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:40 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:40 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=294206, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=294206, maxMem=766075207 -2016-05-24 21:31:40 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:40 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:40 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:40 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:40 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 21:31:40 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:40 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 14 ms on localhost (1/2) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 16 ms on localhost (2/2) -2016-05-24 21:31:41 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 21:31:41 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.028496 s -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=285386, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=282422, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 21:31:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:41 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=259626, maxMem=766075207 -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=265032, maxMem=766075207 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 21:31:41 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 15 ms on localhost (1/2) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 17 ms on localhost (2/2) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 21:31:41 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.040337 s -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=250737, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=250801, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:41 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=242233, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=247575, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 21:31:41 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 11 ms on localhost (1/2) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 13 ms on localhost (2/2) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 21:31:41 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:31:41 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.028748 s -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=241845, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=238945, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 21:31:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=216149, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=221685, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 21:31:41 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 11 ms on localhost (1/2) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 14 ms on localhost (2/2) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 21:31:41 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 21:31:41 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.030817 s -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=221491, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=216019, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:41 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=215955, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=221491, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 21:31:41 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 7 ms on localhost (1/2) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 9 ms on localhost (2/2) -2016-05-24 21:31:41 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 21:31:41 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.021098 s -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224456, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224520, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:41 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=224650, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=230186, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 21:31:41 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 7 ms on localhost (1/2) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 7 ms on localhost (2/2) -2016-05-24 21:31:41 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 21:31:41 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.016153 s -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=233151, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=233215, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:41 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=233345, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=238881, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 21:31:41 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 6 ms on localhost (1/2) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 6 ms on localhost (2/2) -2016-05-24 21:31:41 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 21:31:41 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.015386 s -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=241846, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=241910, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:41 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=242040, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=247576, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 21:31:41 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 6 ms on localhost (1/2) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 6 ms on localhost (2/2) -2016-05-24 21:31:41 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 21:31:41 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.017454 s -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=250540, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=250604, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:41 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=250734, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=256270, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 21:31:41 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 5 ms on localhost (1/2) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 7 ms on localhost (2/2) -2016-05-24 21:31:41 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 21:31:41 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.019304 s -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=259233, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=259297, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:41 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=259427, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=264963, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 21:31:41 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 6 ms on localhost (1/2) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 7 ms on localhost (2/2) -2016-05-24 21:31:41 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 21:31:41 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.019523 s -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=267927, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=267991, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:41 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=268121, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=273657, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 21:31:41 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 8 ms on localhost (1/2) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 11 ms on localhost (2/2) -2016-05-24 21:31:41 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 21:31:41 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.031237 s -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=276619, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=276683, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:41 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=276813, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=282349, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 7 ms on localhost (1/2) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 15 ms on localhost (2/2) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 21:31:41 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:31:41 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.040202 s -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=285313, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=285377, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:41 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=285507, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=291043, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 21:31:41 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 9 ms on localhost (1/2) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 30 ms on localhost (2/2) -2016-05-24 21:31:41 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.032 s -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 21:31:41 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.046678 s -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=294008, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=294072, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:41 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=294202, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=299738, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 38 ms on localhost (1/2) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 38 ms on localhost (2/2) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 21:31:41 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.048 s -2016-05-24 21:31:41 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.066060 s -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=302703, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=302767, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:41 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=302897, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=308433, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 21:31:41 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 7 ms on localhost (1/2) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 7 ms on localhost (2/2) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 21:31:41 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 21:31:41 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.023893 s -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=311397, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=311461, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:41 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=311591, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=317127, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 21:31:41 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 15 ms on localhost (1/2) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 14 ms on localhost (2/2) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 21:31:41 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 21:31:41 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.048101 s -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=320091, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=320155, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:41 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=320285, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=325821, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 21:31:41 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 6 ms on localhost (1/2) -2016-05-24 21:31:41 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 8 ms on localhost (2/2) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 21:31:41 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.018651 s -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=328786, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=328850, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:41 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=328980, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=334516, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 21:31:41 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 6 ms on localhost (1/2) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 7 ms on localhost (2/2) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 21:31:41 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 21:31:41 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.028506 s -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=320092, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=311656, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:55142 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:41 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:41 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=294396, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=299738, maxMem=766075207 -2016-05-24 21:31:41 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:55142 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:41 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 9 ms on localhost (1/2) -2016-05-24 21:31:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:41 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 2415 bytes result sent to driver -2016-05-24 21:31:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 12 ms on localhost (2/2) -2016-05-24 21:31:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 21:31:41 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.025665 s -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 21:31:41 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 0.6307644134901649, 0.6301608260893138, 0.62961849864768, 0.6293831586494314, 0.6292340893598364, 0.629169306843859, 0.6291372385030662, 0.6291242484799721, 0.629118922682065, 0.6291169842584431 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 21:31:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:55142 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:41 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 21:31:41 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 21:31:41 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:55142 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:41 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 21:31:42 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 21:31:42 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 21:31:42 INFO BlockManager:59 - BlockManager stopped -2016-05-24 21:31:42 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 21:31:42 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 21:31:42 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 21:31:42 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 21:31:42 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 21:31:42 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 21:31:42 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-21c60e4c-2616-4003-8473-7ab60acd1b33 -2016-05-24 21:31:42 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-05-24 21:31:52 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 21:31:53 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 21:31:53 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 21:31:53 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 21:31:53 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 21:31:54 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 21:31:54 INFO Remoting:74 - Starting remoting -2016-05-24 21:31:54 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:55163] -2016-05-24 21:31:54 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55163. -2016-05-24 21:31:54 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 21:31:54 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 21:31:54 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-6f82dcfb-d47a-4c72-85cc-997868f2d64d -2016-05-24 21:31:54 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 21:31:55 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-756b1a07-b65f-4668-89a7-5c6cdc676486\httpd-cfb4fe4f-f205-4448-a85d-dbe4b327ce22 -2016-05-24 21:31:55 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 21:31:55 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:31:55 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55164 -2016-05-24 21:31:55 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55164. -2016-05-24 21:31:55 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 21:31:55 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:31:55 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 21:31:55 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 21:31:55 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 21:31:55 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 21:31:55 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 21:31:55 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55183. -2016-05-24 21:31:55 INFO NettyBlockTransferService:59 - Server created on 55183 -2016-05-24 21:31:55 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 21:31:55 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55183 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55183) -2016-05-24 21:31:55 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 21:31:56 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 21:31:56 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 21:31:56 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 21:31:56 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 21:31:56 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55183 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 21:31:56 INFO SparkContext:59 - Created broadcast 0 from textFile at LogisticRegressionLearning.scala:22 -2016-05-24 21:31:57 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 21:31:57 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 21:31:57 INFO SparkContext:59 - Starting job: foreach at LogisticRegressionLearning.scala:27 -2016-05-24 21:31:58 INFO DAGScheduler:59 - Got job 0 (foreach at LogisticRegressionLearning.scala:27) with 2 output partitions -2016-05-24 21:31:58 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at LogisticRegressionLearning.scala:27) -2016-05-24 21:31:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:58 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at LogisticRegressionLearning.scala:23), which has no missing parents -2016-05-24 21:31:58 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=171580, maxMem=766075207 -2016-05-24 21:31:58 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:31:58 INFO MemoryStore:59 - ensureFreeSpace(1993) called with curMem=175020, maxMem=766075207 -2016-05-24 21:31:58 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1993.0 B, free 730.4 MB) -2016-05-24 21:31:58 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55183 (size: 1993.0 B, free: 730.6 MB) -2016-05-24 21:31:58 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at LogisticRegressionLearning.scala:23) -2016-05-24 21:31:58 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 21:31:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:58 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 21:31:58 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 21:31:58 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 21:31:58 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 21:31:58 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:0+20 -2016-05-24 21:31:58 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:20+21 -2016-05-24 21:31:58 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 21:31:58 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 21:31:58 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 21:31:58 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 21:31:58 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 21:31:58 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=177013, maxMem=766075207 -2016-05-24 21:31:58 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-05-24 21:31:58 INFO MemoryStore:59 - ensureFreeSpace(424) called with curMem=177301, maxMem=766075207 -2016-05-24 21:31:58 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 424.0 B, free 730.4 MB) -2016-05-24 21:31:58 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:55183 (size: 288.0 B, free: 730.6 MB) -2016-05-24 21:31:58 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:55183 (size: 424.0 B, free: 730.6 MB) -2016-05-24 21:31:58 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2624 bytes result sent to driver -2016-05-24 21:31:58 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2624 bytes result sent to driver -2016-05-24 21:31:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 123 ms on localhost (1/2) -2016-05-24 21:31:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 150 ms on localhost (2/2) -2016-05-24 21:31:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 21:31:58 INFO DAGScheduler:59 - ResultStage 0 (foreach at LogisticRegressionLearning.scala:27) finished in 0.169 s -2016-05-24 21:31:58 INFO DAGScheduler:59 - Job 0 finished: foreach at LogisticRegressionLearning.scala:27, took 0.314015 s -2016-05-24 21:31:58 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 21:31:58 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 21:31:58 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:31:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:58 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 21:31:58 INFO MemoryStore:59 - ensureFreeSpace(3752) called with curMem=177725, maxMem=766075207 -2016-05-24 21:31:58 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-24 21:31:58 INFO MemoryStore:59 - ensureFreeSpace(2181) called with curMem=181477, maxMem=766075207 -2016-05-24 21:31:58 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 21:31:58 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55183 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:31:58 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:58 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:31:58 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 21:31:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:58 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 21:31:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:58 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-24 21:31:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 23 ms on localhost (1/1) -2016-05-24 21:31:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 21:31:58 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.026 s -2016-05-24 21:31:58 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.063748 s -2016-05-24 21:31:58 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 21:31:58 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 21:31:58 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 21:31:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:58 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[4] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 21:31:58 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=183658, maxMem=766075207 -2016-05-24 21:31:58 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:31:58 INFO MemoryStore:59 - ensureFreeSpace(2039) called with curMem=187186, maxMem=766075207 -2016-05-24 21:31:58 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2039.0 B, free 730.4 MB) -2016-05-24 21:31:58 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55183 (size: 2039.0 B, free: 730.6 MB) -2016-05-24 21:31:58 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[4] at filter at DataValidators.scala:40) -2016-05-24 21:31:58 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 21:31:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:58 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 21:31:58 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 21:31:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:58 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 21:31:58 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 21:31:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 17 ms on localhost (1/2) -2016-05-24 21:31:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 20 ms on localhost (2/2) -2016-05-24 21:31:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 21:31:58 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.022 s -2016-05-24 21:31:58 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.050485 s -2016-05-24 21:31:58 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 21:31:58 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 21:31:58 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 21:31:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:58 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[5] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 21:31:58 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=189225, maxMem=766075207 -2016-05-24 21:31:58 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:31:58 INFO MemoryStore:59 - ensureFreeSpace(2018) called with curMem=192681, maxMem=766075207 -2016-05-24 21:31:58 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2018.0 B, free 730.4 MB) -2016-05-24 21:31:58 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55183 (size: 2018.0 B, free: 730.6 MB) -2016-05-24 21:31:58 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[5] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 21:31:58 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 21:31:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:31:58 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 21:31:58 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 21:31:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:58 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 21:31:58 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 21:31:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 13 ms on localhost (1/2) -2016-05-24 21:31:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 15 ms on localhost (2/2) -2016-05-24 21:31:58 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.015 s -2016-05-24 21:31:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 21:31:58 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.032876 s -2016-05-24 21:31:58 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 21:31:58 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 21:31:58 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=194699, maxMem=766075207 -2016-05-24 21:31:58 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:58 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=194763, maxMem=766075207 -2016-05-24 21:31:58 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:58 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:58 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:58 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:58 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:58 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[7] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:58 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=194893, maxMem=766075207 -2016-05-24 21:31:58 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:58 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=200429, maxMem=766075207 -2016-05-24 21:31:58 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:58 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:58 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[7] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:58 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 21:31:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:58 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 21:31:58 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 21:31:58 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:55183 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:31:58 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:58 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:55183 in memory (size: 2018.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 24 ms on localhost (1/2) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:55183 in memory (size: 2039.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 23 ms on localhost (2/2) -2016-05-24 21:31:59 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 21:31:59 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.068549 s -2016-05-24 21:31:59 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=186419, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:55183 in memory (size: 1993.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=181050, maxMem=766075207 -2016-05-24 21:31:59 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:59 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[9] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=181180, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=186716, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[9] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 21:31:59 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:59 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 23 ms on localhost (1/2) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 25 ms on localhost (2/2) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 21:31:59 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.027 s -2016-05-24 21:31:59 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.043714 s -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=189678, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=189742, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:59 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[11] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=189872, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=195408, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[11] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 21:31:59 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:59 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 17 ms on localhost (1/2) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 19 ms on localhost (2/2) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 21:31:59 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 21:31:59 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.042729 s -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=198373, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=198437, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:59 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=198567, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=204103, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 21:31:59 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:59 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 18 ms on localhost (1/2) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 18 ms on localhost (2/2) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 21:31:59 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 21:31:59 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.038654 s -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=207066, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=207130, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:59 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=207260, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=212796, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 21:31:59 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:59 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 16 ms on localhost (1/2) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 16 ms on localhost (2/2) -2016-05-24 21:31:59 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 21:31:59 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.038340 s -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=215758, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=215822, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:59 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=215952, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=221488, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 21:31:59 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:59 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 12 ms on localhost (1/2) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 14 ms on localhost (2/2) -2016-05-24 21:31:59 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 21:31:59 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.032849 s -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224452, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224516, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:59 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=224646, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=230182, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 21:31:59 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:59 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 15 ms on localhost (1/2) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 15 ms on localhost (2/2) -2016-05-24 21:31:59 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 21:31:59 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.030540 s -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=233146, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=233210, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:59 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=233340, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=238876, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 21:31:59 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:59 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:59 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 11 ms on localhost (1/2) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 14 ms on localhost (2/2) -2016-05-24 21:31:59 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 21:31:59 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.031324 s -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=241840, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=241904, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:59 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=242034, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=244606, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:59 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 21:31:59 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:59 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 21:31:59 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 16 ms on localhost (1/2) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 14 ms on localhost (2/2) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 21:31:59 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:31:59 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.054716 s -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=230184, maxMem=766075207 -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224712, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:59 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:59 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:59 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=207263, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=207263, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 21:31:59 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:59 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:59 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 17 ms on localhost (1/2) -2016-05-24 21:31:59 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 21:31:59 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 18 ms on localhost (2/2) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 21:31:59 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.041444 s -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=198377, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=195477, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:59 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:59 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=189877, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=195413, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 21:31:59 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:59 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 17 ms on localhost (1/2) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 18 ms on localhost (2/2) -2016-05-24 21:31:59 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 21:31:59 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.038150 s -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=198378, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=198442, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:59 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=198572, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=204108, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 21:31:59 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:59 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 18 ms on localhost (1/2) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 16 ms on localhost (2/2) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 21:31:59 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 21:31:59 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.039463 s -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=207078, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=207142, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:59 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=207272, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=212808, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 21:31:59 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:59 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 13 ms on localhost (1/2) -2016-05-24 21:31:59 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 19 ms on localhost (2/2) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 21:31:59 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 21:31:59 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.037923 s -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=215772, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=215836, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:59 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=215966, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=221502, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 21:31:59 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:59 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 15 ms on localhost (1/2) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 17 ms on localhost (2/2) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 21:31:59 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 21:31:59 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.033405 s -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224467, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224531, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:59 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=224661, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=230197, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 21:31:59 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:59 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 14 ms on localhost (1/2) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 16 ms on localhost (2/2) -2016-05-24 21:31:59 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 21:31:59 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.032212 s -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=233161, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=233225, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:59 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=233355, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=238891, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:31:59 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 21:31:59 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:31:59 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:31:59 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 2415 bytes result sent to driver -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 15 ms on localhost (1/2) -2016-05-24 21:31:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 13 ms on localhost (2/2) -2016-05-24 21:31:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 21:31:59 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 21:31:59 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.028990 s -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=241855, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=241919, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:31:59 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:31:59 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 21:31:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:31:59 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:31:59 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:31:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:31:59 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=242049, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:31:59 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=247585, maxMem=766075207 -2016-05-24 21:31:59 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 13 ms on localhost (1/2) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 14 ms on localhost (2/2) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.028365 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=250550, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=250614, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=250744, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=256280, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 12 ms on localhost (1/2) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 14 ms on localhost (2/2) -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.029878 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=259240, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=259304, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=259434, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=264970, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 12 ms on localhost (1/2) -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 17 ms on localhost (2/2) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.033246 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=267934, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=267998, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=268128, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=273664, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 13 ms on localhost (1/2) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 13 ms on localhost (2/2) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.027374 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=276627, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=276691, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=276821, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=282357, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 12 ms on localhost (1/2) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 14 ms on localhost (2/2) -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.031584 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=285320, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=285384, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=285514, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=291050, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 13 ms on localhost (1/2) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 12 ms on localhost (2/2) -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.029100 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=294013, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=294077, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=294207, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=299743, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 13 ms on localhost (1/2) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 13 ms on localhost (2/2) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.030346 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=302705, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=302769, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=302899, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=308435, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 11 ms on localhost (1/2) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 10 ms on localhost (2/2) -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.028045 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=311397, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=311461, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=311591, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=317127, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 10 ms on localhost (1/2) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 9 ms on localhost (2/2) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.032436 s -2016-05-24 21:32:00 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=302901, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=302771, maxMem=766075207 -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=277016, maxMem=766075207 -2016-05-24 21:32:00 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=282488, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 11 ms on localhost (1/2) -2016-05-24 21:32:00 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 10 ms on localhost (2/2) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.024890 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=276692, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=276692, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=259501, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=264973, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 10 ms on localhost (1/2) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 12 ms on localhost (2/2) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.032521 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=250744, maxMem=766075207 -2016-05-24 21:32:00 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=250678, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=233419, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=238955, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 9 ms on localhost (1/2) -2016-05-24 21:32:00 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 11 ms on localhost (2/2) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.026058 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=233156, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=233220, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:00 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=216154, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=221560, maxMem=766075207 -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 21:32:00 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 8 ms on localhost (1/2) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 10 ms on localhost (2/2) -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.026788 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=215765, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=215829, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=215959, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=221495, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 8 ms on localhost (1/2) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 9 ms on localhost (2/2) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.018799 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224460, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224524, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=224654, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=230190, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 5 ms on localhost (1/2) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 6 ms on localhost (2/2) -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.015043 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=233154, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=233218, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=233348, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=238884, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 5 ms on localhost (1/2) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 5 ms on localhost (2/2) -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.014425 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=241849, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=241913, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=242043, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=247579, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 7 ms on localhost (1/2) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 6 ms on localhost (2/2) -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.018231 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=250544, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=250608, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=250738, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=256274, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 5 ms on localhost (1/2) -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 7 ms on localhost (2/2) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.018568 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=259239, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=259303, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=259433, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=264969, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 7 ms on localhost (1/2) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 7 ms on localhost (2/2) -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.018158 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=267932, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=267996, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=268126, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2967) called with curMem=273662, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 7 ms on localhost (1/2) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 8 ms on localhost (2/2) -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.019694 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=276629, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=276693, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=276823, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=282359, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 7 ms on localhost (1/2) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 7 ms on localhost (2/2) -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.018735 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=285322, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=285386, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=285516, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=291052, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 5 ms on localhost (1/2) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 7 ms on localhost (2/2) -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.019560 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=294017, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=294081, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=294211, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2966) called with curMem=299747, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 7 ms on localhost (1/2) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 9 ms on localhost (2/2) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.017967 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=302713, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=302777, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=302907, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=308443, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:00 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 21:32:00 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:00 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:00 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 2415 bytes result sent to driver -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 8 ms on localhost (1/2) -2016-05-24 21:32:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 8 ms on localhost (2/2) -2016-05-24 21:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 21:32:00 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 21:32:00 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.021397 s -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=311407, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=311471, maxMem=766075207 -2016-05-24 21:32:00 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:00 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:00 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:01 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:01 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:01 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:01 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:01 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=311601, maxMem=766075207 -2016-05-24 21:32:01 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:01 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=317137, maxMem=766075207 -2016-05-24 21:32:01 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:01 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:01 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 21:32:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:01 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 21:32:01 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:01 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-24 21:32:01 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:01 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:01 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 2415 bytes result sent to driver -2016-05-24 21:32:01 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 2415 bytes result sent to driver -2016-05-24 21:32:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 6 ms on localhost (1/2) -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 6 ms on localhost (2/2) -2016-05-24 21:32:01 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:32:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 21:32:01 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.022106 s -2016-05-24 21:32:01 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=294405, maxMem=766075207 -2016-05-24 21:32:01 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:01 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 21:32:01 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=294469, maxMem=766075207 -2016-05-24 21:32:01 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:01 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:01 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:01 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:01 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:01 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:01 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 21:32:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:01 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:01 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=277017, maxMem=766075207 -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:01 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:01 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=279586, maxMem=766075207 -2016-05-24 21:32:01 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:01 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 21:32:01 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:01 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 21:32:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:01 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 21:32:01 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 21:32:01 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:01 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:01 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 2415 bytes result sent to driver -2016-05-24 21:32:01 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 2415 bytes result sent to driver -2016-05-24 21:32:01 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 21:32:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 6 ms on localhost (1/2) -2016-05-24 21:32:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 7 ms on localhost (2/2) -2016-05-24 21:32:01 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:32:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 21:32:01 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.017803 s -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:01 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=268128, maxMem=766075207 -2016-05-24 21:32:01 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:01 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=259691, maxMem=766075207 -2016-05-24 21:32:01 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:01 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:55183 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:01 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:01 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 21:32:01 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:01 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:01 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:01 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 21:32:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:01 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:01 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=250932, maxMem=766075207 -2016-05-24 21:32:01 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:01 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=256404, maxMem=766075207 -2016-05-24 21:32:01 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:55183 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:01 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:01 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 21:32:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:01 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 21:32:01 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 21:32:01 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 21:32:01 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:01 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:01 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 2415 bytes result sent to driver -2016-05-24 21:32:01 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 2415 bytes result sent to driver -2016-05-24 21:32:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 7 ms on localhost (1/2) -2016-05-24 21:32:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 8 ms on localhost (2/2) -2016-05-24 21:32:01 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 21:32:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 21:32:01 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.024935 s -2016-05-24 21:32:01 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 0.6307644134901649, 0.6301608260893138, 0.62961849864768, 0.6293831586494314, 0.6292340893598364, 0.629169306843859, 0.6291372385030662, 0.6291242484799721, 0.629118922682065, 0.6291169842584431 -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:01 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:01 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 21:32:01 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 21:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:01 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:01 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:55183 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:01 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 21:32:01 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:55183 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:01 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 21:32:01 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 21:32:01 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 21:32:01 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 21:32:01 INFO BlockManager:59 - BlockManager stopped -2016-05-24 21:32:01 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 21:32:01 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 21:32:01 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 21:32:01 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 21:32:01 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-756b1a07-b65f-4668-89a7-5c6cdc676486 -2016-05-24 21:32:01 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 21:32:47 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 21:32:48 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 21:32:49 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 21:32:49 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 21:32:49 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 21:32:49 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 21:32:49 INFO Remoting:74 - Starting remoting -2016-05-24 21:32:50 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:55207] -2016-05-24 21:32:50 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55207. -2016-05-24 21:32:50 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 21:32:50 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 21:32:50 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-53199f61-9c4e-425d-8037-2ecf198c4ae1 -2016-05-24 21:32:50 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 21:32:50 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-2b8ee3c2-396e-4cc2-8277-7a468d9d4530\httpd-b707222d-6f63-4792-a723-302c8f8a5afe -2016-05-24 21:32:50 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 21:32:50 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:32:50 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55208 -2016-05-24 21:32:50 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55208. -2016-05-24 21:32:50 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 21:32:50 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:32:50 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 21:32:50 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 21:32:50 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 21:32:50 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 21:32:50 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 21:32:51 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55227. -2016-05-24 21:32:51 INFO NettyBlockTransferService:59 - Server created on 55227 -2016-05-24 21:32:51 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 21:32:51 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55227 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55227) -2016-05-24 21:32:51 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 21:32:51 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 21:32:51 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 21:32:51 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 21:32:51 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 21:32:51 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55227 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 21:32:51 INFO SparkContext:59 - Created broadcast 0 from textFile at LogisticRegressionLearning.scala:22 -2016-05-24 21:32:53 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 21:32:53 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 21:32:53 INFO SparkContext:59 - Starting job: foreach at LogisticRegressionLearning.scala:27 -2016-05-24 21:32:53 INFO DAGScheduler:59 - Got job 0 (foreach at LogisticRegressionLearning.scala:27) with 2 output partitions -2016-05-24 21:32:53 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at LogisticRegressionLearning.scala:27) -2016-05-24 21:32:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:53 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at LogisticRegressionLearning.scala:23), which has no missing parents -2016-05-24 21:32:53 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=171580, maxMem=766075207 -2016-05-24 21:32:53 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:32:53 INFO MemoryStore:59 - ensureFreeSpace(1993) called with curMem=175020, maxMem=766075207 -2016-05-24 21:32:53 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1993.0 B, free 730.4 MB) -2016-05-24 21:32:53 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55227 (size: 1993.0 B, free: 730.6 MB) -2016-05-24 21:32:53 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at LogisticRegressionLearning.scala:23) -2016-05-24 21:32:53 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 21:32:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:32:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:32:53 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 21:32:53 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 21:32:53 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 21:32:53 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 21:32:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:20+21 -2016-05-24 21:32:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:0+20 -2016-05-24 21:32:53 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 21:32:53 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 21:32:53 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 21:32:53 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 21:32:53 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 21:32:53 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=177013, maxMem=766075207 -2016-05-24 21:32:53 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-05-24 21:32:53 INFO MemoryStore:59 - ensureFreeSpace(424) called with curMem=177301, maxMem=766075207 -2016-05-24 21:32:53 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:55227 (size: 288.0 B, free: 730.6 MB) -2016-05-24 21:32:53 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 424.0 B, free 730.4 MB) -2016-05-24 21:32:53 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:55227 (size: 424.0 B, free: 730.6 MB) -2016-05-24 21:32:53 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2624 bytes result sent to driver -2016-05-24 21:32:53 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2624 bytes result sent to driver -2016-05-24 21:32:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 131 ms on localhost (1/2) -2016-05-24 21:32:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 158 ms on localhost (2/2) -2016-05-24 21:32:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 21:32:53 INFO DAGScheduler:59 - ResultStage 0 (foreach at LogisticRegressionLearning.scala:27) finished in 0.173 s -2016-05-24 21:32:53 INFO DAGScheduler:59 - Job 0 finished: foreach at LogisticRegressionLearning.scala:27, took 0.306189 s -2016-05-24 21:32:53 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 21:32:53 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 21:32:53 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:32:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:53 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 21:32:53 INFO MemoryStore:59 - ensureFreeSpace(3752) called with curMem=177725, maxMem=766075207 -2016-05-24 21:32:53 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-24 21:32:53 INFO MemoryStore:59 - ensureFreeSpace(2181) called with curMem=181477, maxMem=766075207 -2016-05-24 21:32:53 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 21:32:53 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55227 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:32:53 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:53 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:32:53 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 21:32:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:32:53 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 21:32:53 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:53 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-24 21:32:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 17 ms on localhost (1/1) -2016-05-24 21:32:53 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.017 s -2016-05-24 21:32:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 21:32:53 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.046165 s -2016-05-24 21:32:53 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 21:32:53 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 21:32:53 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 21:32:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:53 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[4] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 21:32:53 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=183658, maxMem=766075207 -2016-05-24 21:32:53 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:32:53 INFO MemoryStore:59 - ensureFreeSpace(2039) called with curMem=187186, maxMem=766075207 -2016-05-24 21:32:53 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2039.0 B, free 730.4 MB) -2016-05-24 21:32:53 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55227 (size: 2039.0 B, free: 730.6 MB) -2016-05-24 21:32:53 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[4] at filter at DataValidators.scala:40) -2016-05-24 21:32:53 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 21:32:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:32:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:32:53 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 21:32:53 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 21:32:53 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:53 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:53 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 21:32:53 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 21:32:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 12 ms on localhost (1/2) -2016-05-24 21:32:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 12 ms on localhost (2/2) -2016-05-24 21:32:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 21:32:53 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.014 s -2016-05-24 21:32:53 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.028263 s -2016-05-24 21:32:53 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 21:32:53 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 21:32:53 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 21:32:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:53 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[5] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 21:32:53 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=189225, maxMem=766075207 -2016-05-24 21:32:53 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:32:53 INFO MemoryStore:59 - ensureFreeSpace(2018) called with curMem=192681, maxMem=766075207 -2016-05-24 21:32:53 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2018.0 B, free 730.4 MB) -2016-05-24 21:32:53 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55227 (size: 2018.0 B, free: 730.6 MB) -2016-05-24 21:32:53 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[5] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 21:32:53 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 21:32:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:32:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:32:53 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 21:32:53 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 21:32:53 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:53 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:53 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 21:32:53 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 21:32:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 11 ms on localhost (1/2) -2016-05-24 21:32:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 13 ms on localhost (2/2) -2016-05-24 21:32:53 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.013 s -2016-05-24 21:32:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 21:32:53 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.029962 s -2016-05-24 21:32:53 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:55227 in memory (size: 2018.0 B, free: 730.6 MB) -2016-05-24 21:32:53 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:55227 in memory (size: 2039.0 B, free: 730.6 MB) -2016-05-24 21:32:53 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:55227 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:32:53 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:55227 in memory (size: 1993.0 B, free: 730.6 MB) -2016-05-24 21:32:54 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 21:32:54 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=172292, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=172356, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:54 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[7] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=172486, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=178022, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[7] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 21:32:54 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:54 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 19 ms on localhost (1/2) -2016-05-24 21:32:54 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 18 ms on localhost (2/2) -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 21:32:54 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.035421 s -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=180985, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=181049, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:54 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[9] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=181179, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=186715, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[9] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 21:32:54 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:54 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 19 ms on localhost (1/2) -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 22 ms on localhost (2/2) -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 21:32:54 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 21:32:54 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.047804 s -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=189679, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=189743, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:54 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[11] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=189873, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=195409, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[11] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 21:32:54 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:54 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 16 ms on localhost (1/2) -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 19 ms on localhost (2/2) -2016-05-24 21:32:54 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 21:32:54 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.042263 s -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=198373, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=198437, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:54 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=198567, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=204103, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 21:32:54 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:54 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 16 ms on localhost (1/2) -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 16 ms on localhost (2/2) -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 21:32:54 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 21:32:54 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.035938 s -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=207066, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=207130, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:54 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=207260, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=212796, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 21:32:54 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:54 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 17 ms on localhost (1/2) -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 20 ms on localhost (2/2) -2016-05-24 21:32:54 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 21:32:54 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.037459 s -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=215758, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=215822, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:54 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=215952, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=221488, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 21:32:54 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:54 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 13 ms on localhost (1/2) -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 15 ms on localhost (2/2) -2016-05-24 21:32:54 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 21:32:54 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.033375 s -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224452, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224516, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:54 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:54 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:54 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=201724, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:54 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=204297, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:54 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 21:32:54 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 21:32:54 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:54 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 19 ms on localhost (1/2) -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 18 ms on localhost (2/2) -2016-05-24 21:32:54 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 21:32:54 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.037342 s -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=189874, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:54 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=189938, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:54 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:54 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=181181, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=186717, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 21:32:54 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:54 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 11 ms on localhost (1/2) -2016-05-24 21:32:54 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 15 ms on localhost (2/2) -2016-05-24 21:32:54 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 21:32:54 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.031387 s -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=189682, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=189746, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:54 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=189876, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=195412, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 21:32:54 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:54 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 10 ms on localhost (1/2) -2016-05-24 21:32:54 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 14 ms on localhost (2/2) -2016-05-24 21:32:54 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 21:32:54 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.036581 s -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=198377, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=198441, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:54 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=198571, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=204107, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 21:32:54 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:54 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 16 ms on localhost (1/2) -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 15 ms on localhost (2/2) -2016-05-24 21:32:54 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 21:32:54 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.036835 s -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=207071, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=207135, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:54 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=207265, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=212801, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:54 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 21:32:54 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:54 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:54 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2415 bytes result sent to driver -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 15 ms on localhost (1/2) -2016-05-24 21:32:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 16 ms on localhost (2/2) -2016-05-24 21:32:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 21:32:54 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 21:32:54 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.033547 s -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=215766, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=215830, maxMem=766075207 -2016-05-24 21:32:54 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:54 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:54 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:55 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=215960, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=221496, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 21:32:55 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:55 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 16 ms on localhost (1/2) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 21 ms on localhost (2/2) -2016-05-24 21:32:55 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 21:32:55 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.040477 s -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224465, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224529, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:55 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=224659, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=230195, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 21:32:55 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:55 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 12 ms on localhost (1/2) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 14 ms on localhost (2/2) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 21:32:55 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 21:32:55 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.030984 s -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=233160, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=233224, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:55 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=233354, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=238890, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 21:32:55 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:55 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 14 ms on localhost (1/2) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 17 ms on localhost (2/2) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 21:32:55 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:32:55 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.032197 s -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=241855, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=241919, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:55 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=242049, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=247585, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 21:32:55 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:55 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 12 ms on localhost (1/2) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 13 ms on localhost (2/2) -2016-05-24 21:32:55 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 21:32:55 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.027275 s -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=250550, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=250614, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:55 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=250744, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=256280, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 21:32:55 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:55 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 16 ms on localhost (1/2) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 19 ms on localhost (2/2) -2016-05-24 21:32:55 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 21:32:55 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.036773 s -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=259244, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=259308, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:55 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=259438, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=264974, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 21:32:55 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:55 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 15 ms on localhost (1/2) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 19 ms on localhost (2/2) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 21:32:55 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 21:32:55 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.038719 s -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=267938, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=268002, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:55 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=268132, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(2961) called with curMem=273668, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:55 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:55 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 27 ms on localhost (1/2) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 32 ms on localhost (2/2) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 21:32:55 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.033 s -2016-05-24 21:32:55 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.074343 s -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=276629, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=276693, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:55 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=276823, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=282359, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 21:32:55 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:55 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 12 ms on localhost (1/2) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 11 ms on localhost (2/2) -2016-05-24 21:32:55 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 21:32:55 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.031348 s -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=285322, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=285386, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:55 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=285516, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=291052, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 21:32:55 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:55 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 11 ms on localhost (1/2) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 13 ms on localhost (2/2) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 21:32:55 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:32:55 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.031971 s -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=294016, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=294080, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:55 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=294210, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=299746, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 21:32:55 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:55 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 11 ms on localhost (1/2) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 11 ms on localhost (2/2) -2016-05-24 21:32:55 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 21:32:55 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.026304 s -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=302710, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=302774, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:55 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=302904, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=308440, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 21:32:55 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:55 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 11 ms on localhost (1/2) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 11 ms on localhost (2/2) -2016-05-24 21:32:55 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 21:32:55 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.027537 s -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=311402, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=311466, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:55 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=311596, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(2961) called with curMem=317132, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 21:32:55 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:55 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 7 ms on localhost (1/2) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 7 ms on localhost (2/2) -2016-05-24 21:32:55 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 21:32:55 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.025058 s -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=320093, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=320157, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:55 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=320287, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=325823, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 21:32:55 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:55 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 23 ms on localhost (1/2) -2016-05-24 21:32:55 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 27 ms on localhost (2/2) -2016-05-24 21:32:55 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.027 s -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 21:32:55 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.041275 s -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=320094, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=320158, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 21:32:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=294402, maxMem=766075207 -2016-05-24 21:32:55 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=299744, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 21:32:55 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 21:32:55 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:55 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 9 ms on localhost (1/2) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 11 ms on localhost (2/2) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 21:32:55 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 21:32:55 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.024127 s -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=291053, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=285581, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:55 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:55 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=268129, maxMem=766075207 -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=270700, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 21:32:55 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:55 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:55 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 12 ms on localhost (1/2) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 12 ms on localhost (2/2) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 21:32:55 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:32:55 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.026986 s -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=259433, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=259303, maxMem=766075207 -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:55 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:55 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=242039, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=247575, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 21:32:55 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 21:32:55 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:55 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 11 ms on localhost (1/2) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 11 ms on localhost (2/2) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 21:32:55 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.028125 s -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=241845, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=238945, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:55 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:55 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 21:32:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:55 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=219114, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=221685, maxMem=766075207 -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:55 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 21:32:55 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:55 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 10 ms on localhost (1/2) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 12 ms on localhost (2/2) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:55 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 21:32:55 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.028440 s -2016-05-24 21:32:55 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=215954, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=215888, maxMem=766075207 -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:55 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=215954, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=221490, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 21:32:55 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:55 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:55 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 2415 bytes result sent to driver -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 7 ms on localhost (1/2) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 9 ms on localhost (2/2) -2016-05-24 21:32:55 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 21:32:55 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.020500 s -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224455, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224519, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:55 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=224649, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:55 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=230185, maxMem=766075207 -2016-05-24 21:32:55 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:55 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:55 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:55 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:55 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 21:32:55 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:56 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 6 ms on localhost (1/2) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 6 ms on localhost (2/2) -2016-05-24 21:32:56 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 21:32:56 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.014940 s -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=233150, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=233214, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:56 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=233344, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=238880, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 21:32:56 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:56 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 5 ms on localhost (1/2) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 6 ms on localhost (2/2) -2016-05-24 21:32:56 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 21:32:56 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.014610 s -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=241844, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=241908, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:56 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=242038, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=247574, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 21:32:56 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:56 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 6 ms on localhost (1/2) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 5 ms on localhost (2/2) -2016-05-24 21:32:56 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 21:32:56 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.014718 s -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=250539, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=250603, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:56 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=250733, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=256269, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 21:32:56 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:56 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 6 ms on localhost (1/2) -2016-05-24 21:32:56 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 6 ms on localhost (2/2) -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 21:32:56 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.018788 s -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=259234, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=259298, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:56 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=259428, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=264964, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 21:32:56 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:56 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 11 ms on localhost (1/2) -2016-05-24 21:32:56 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 13 ms on localhost (2/2) -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 21:32:56 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.022289 s -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=267927, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=267991, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:56 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=268121, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=273657, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 21:32:56 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:56 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 5 ms on localhost (1/2) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 6 ms on localhost (2/2) -2016-05-24 21:32:56 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 21:32:56 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.015098 s -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=276620, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=276684, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:56 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=276814, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=282350, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 21:32:56 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:56 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 5 ms on localhost (1/2) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 7 ms on localhost (2/2) -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 21:32:56 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:32:56 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.015662 s -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=285315, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=285379, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:56 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=285509, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=291045, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 21:32:56 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:56 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 6 ms on localhost (1/2) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 6 ms on localhost (2/2) -2016-05-24 21:32:56 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 21:32:56 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.015905 s -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=294008, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=294072, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:56 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=294202, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=299738, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 21:32:56 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:56 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 5 ms on localhost (1/2) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 6 ms on localhost (2/2) -2016-05-24 21:32:56 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 21:32:56 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.015845 s -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=302703, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=302767, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:56 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=302897, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=308433, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 21:32:56 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:56 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 5 ms on localhost (1/2) -2016-05-24 21:32:56 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 6 ms on localhost (2/2) -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 21:32:56 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 21:32:56 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.016990 s -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=311398, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=311462, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:56 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=311592, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=317128, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 21:32:56 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:56 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 6 ms on localhost (1/2) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 7 ms on localhost (2/2) -2016-05-24 21:32:56 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 21:32:56 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.022023 s -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=320092, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=320156, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:56 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=320286, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=325822, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 21:32:56 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:56 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 6 ms on localhost (1/2) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 7 ms on localhost (2/2) -2016-05-24 21:32:56 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 21:32:56 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.016372 s -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=328787, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=328851, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:56 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=328981, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=334517, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 21:32:56 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 21:32:56 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:56 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:56 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 8 ms on localhost (1/2) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 8 ms on localhost (2/2) -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 21:32:56 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 21:32:56 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.030278 s -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=320092, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=320156, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:55227 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 21:32:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:32:56 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=294397, maxMem=766075207 -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 21:32:56 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=299933, maxMem=766075207 -2016-05-24 21:32:56 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:55227 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:32:56 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 21:32:56 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:32:56 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:32:56 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 2415 bytes result sent to driver -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 8 ms on localhost (1/2) -2016-05-24 21:32:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 10 ms on localhost (2/2) -2016-05-24 21:32:56 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 21:32:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 21:32:56 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.021782 s -2016-05-24 21:32:56 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 0.6307644134901649, 0.6301608260893138, 0.62961849864768, 0.6293831586494314, 0.6292340893598364, 0.629169306843859, 0.6291372385030662, 0.6291242484799721, 0.629118922682065, 0.6291169842584431 -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 21:32:56 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 21:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:32:56 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:56 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:56 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:56 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:56 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:56 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:55227 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:32:56 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 21:32:56 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:55227 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:32:56 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 21:32:56 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 21:32:56 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 21:32:56 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 21:32:56 INFO BlockManager:59 - BlockManager stopped -2016-05-24 21:32:56 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 21:32:56 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 21:32:56 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 21:32:56 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 21:32:56 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-2b8ee3c2-396e-4cc2-8277-7a468d9d4530 -2016-05-24 21:32:56 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 21:32:56 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 21:35:42 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 21:35:44 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 21:35:44 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 21:35:44 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 21:35:44 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 21:35:45 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 21:35:45 INFO Remoting:74 - Starting remoting -2016-05-24 21:35:45 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:55252] -2016-05-24 21:35:45 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55252. -2016-05-24 21:35:45 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 21:35:45 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 21:35:45 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-ce64f775-5769-4ed4-9ece-a9647482dfcf -2016-05-24 21:35:46 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 21:35:46 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-287f9157-c32b-4d73-affd-ec4c79ded551\httpd-8341568f-1e9c-4060-84db-153a59b2c9b2 -2016-05-24 21:35:46 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 21:35:46 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:35:46 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55255 -2016-05-24 21:35:46 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55255. -2016-05-24 21:35:46 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 21:35:46 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:35:46 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 21:35:46 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 21:35:46 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 21:35:46 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 21:35:46 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 21:35:47 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55274. -2016-05-24 21:35:47 INFO NettyBlockTransferService:59 - Server created on 55274 -2016-05-24 21:35:47 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 21:35:47 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55274 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55274) -2016-05-24 21:35:47 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 21:35:48 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 21:35:48 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 21:35:48 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 21:35:48 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 21:35:48 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55274 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 21:35:48 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 21:35:50 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 21:35:50 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 21:35:50 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 21:35:50 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 21:35:50 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 21:35:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:35:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:35:50 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 21:35:50 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 21:35:50 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 21:35:50 INFO MemoryStore:59 - ensureFreeSpace(2147) called with curMem=175476, maxMem=766075207 -2016-05-24 21:35:50 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 21:35:50 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55274 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:35:50 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:35:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 21:35:50 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 21:35:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:35:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:35:50 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 21:35:50 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 21:35:50 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 21:35:50 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 21:35:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 21:35:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 21:35:50 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 21:35:50 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 21:35:50 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 21:35:50 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 21:35:50 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 21:35:50 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177623, maxMem=766075207 -2016-05-24 21:35:50 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 21:35:51 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262703, maxMem=766075207 -2016-05-24 21:35:51 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 21:35:51 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:55274 (size: 83.1 KB, free: 730.5 MB) -2016-05-24 21:35:51 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:55274 (size: 83.6 KB, free: 730.4 MB) -2016-05-24 21:35:51 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 21:35:51 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 21:35:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 571 ms on localhost (1/2) -2016-05-24 21:35:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 600 ms on localhost (2/2) -2016-05-24 21:35:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 21:35:51 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.637 s -2016-05-24 21:35:51 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.831963 s -2016-05-24 21:35:51 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 21:35:51 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 21:35:51 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:35:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:35:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:35:51 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 21:35:51 INFO MemoryStore:59 - ensureFreeSpace(4256) called with curMem=348351, maxMem=766075207 -2016-05-24 21:35:51 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-24 21:35:51 INFO MemoryStore:59 - ensureFreeSpace(2343) called with curMem=352607, maxMem=766075207 -2016-05-24 21:35:51 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-24 21:35:51 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55274 (size: 2.3 KB, free: 730.4 MB) -2016-05-24 21:35:51 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:35:51 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:35:51 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 21:35:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:35:51 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 21:35:51 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:35:51 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-24 21:35:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 22 ms on localhost (1/1) -2016-05-24 21:35:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 21:35:51 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.024 s -2016-05-24 21:35:51 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.056763 s -2016-05-24 21:35:51 WARN LogisticRegressionWithSGD:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-05-24 21:35:51 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 21:35:51 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 21:35:51 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 21:35:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:35:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:35:51 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[8] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 21:35:51 INFO MemoryStore:59 - ensureFreeSpace(3952) called with curMem=354950, maxMem=766075207 -2016-05-24 21:35:51 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-24 21:35:51 INFO MemoryStore:59 - ensureFreeSpace(2176) called with curMem=358902, maxMem=766075207 -2016-05-24 21:35:51 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.2 MB) -2016-05-24 21:35:51 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55274 (size: 2.1 KB, free: 730.4 MB) -2016-05-24 21:35:51 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:35:51 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[8] at filter at DataValidators.scala:40) -2016-05-24 21:35:51 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 21:35:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:35:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:35:51 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 21:35:51 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 21:35:51 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:35:51 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:35:51 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 21:35:51 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 21:35:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 18 ms on localhost (1/2) -2016-05-24 21:35:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 17 ms on localhost (2/2) -2016-05-24 21:35:51 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.019 s -2016-05-24 21:35:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 21:35:51 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.036306 s -2016-05-24 21:35:51 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 21:35:51 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 21:35:51 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 21:35:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:35:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:35:51 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 21:35:51 INFO MemoryStore:59 - ensureFreeSpace(3960) called with curMem=361078, maxMem=766075207 -2016-05-24 21:35:51 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-24 21:35:51 INFO MemoryStore:59 - ensureFreeSpace(2193) called with curMem=365038, maxMem=766075207 -2016-05-24 21:35:51 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.2 MB) -2016-05-24 21:35:51 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55274 (size: 2.1 KB, free: 730.4 MB) -2016-05-24 21:35:51 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:35:51 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 21:35:51 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 21:35:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:35:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:35:51 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 21:35:51 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 21:35:51 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:35:51 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 21:35:51 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:35:51 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 21:35:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 18 ms on localhost (1/2) -2016-05-24 21:35:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 21 ms on localhost (2/2) -2016-05-24 21:35:51 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.022 s -2016-05-24 21:35:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 21:35:51 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.041064 s -2016-05-24 21:35:51 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 21:35:51 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 21:35:51 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=367231, maxMem=766075207 -2016-05-24 21:35:51 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:35:51 INFO MemoryStore:59 - ensureFreeSpace(388) called with curMem=372823, maxMem=766075207 -2016-05-24 21:35:51 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 388.0 B, free 730.2 MB) -2016-05-24 21:35:51 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55274 (size: 388.0 B, free: 730.4 MB) -2016-05-24 21:35:51 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 21:35:51 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:35:51 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:35:51 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:35:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:35:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:35:51 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[11] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:35:51 INFO MemoryStore:59 - ensureFreeSpace(11520) called with curMem=373211, maxMem=766075207 -2016-05-24 21:35:51 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 11.3 KB, free 730.2 MB) -2016-05-24 21:35:51 INFO MemoryStore:59 - ensureFreeSpace(3383) called with curMem=384731, maxMem=766075207 -2016-05-24 21:35:51 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-24 21:35:51 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55274 (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:35:51 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:35:51 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[11] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:35:51 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 21:35:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:35:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:35:51 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 21:35:51 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 21:35:51 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:35:51 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:35:51 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 7968 bytes result sent to driver -2016-05-24 21:35:52 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 7968 bytes result sent to driver -2016-05-24 21:35:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 29 ms on localhost (1/2) -2016-05-24 21:35:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 29 ms on localhost (2/2) -2016-05-24 21:35:52 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.032 s -2016-05-24 21:35:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 21:35:52 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.056465 s -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=388114, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(2527) called with curMem=393706, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55274 (size: 2.5 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 21:35:52 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:35:52 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:35:52 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:35:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:35:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:35:52 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(11520) called with curMem=396233, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 11.3 KB, free 730.2 MB) -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(3381) called with curMem=407753, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55274 (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:35:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:35:52 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 21:35:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:35:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:35:52 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 21:35:52 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 21:35:52 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:35:52 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:35:52 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 7968 bytes result sent to driver -2016-05-24 21:35:52 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 7968 bytes result sent to driver -2016-05-24 21:35:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 24 ms on localhost (1/2) -2016-05-24 21:35:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 27 ms on localhost (2/2) -2016-05-24 21:35:52 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.029 s -2016-05-24 21:35:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 21:35:52 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.053506 s -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=411134, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(3026) called with curMem=416726, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55274 (size: 3.0 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 21:35:52 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:35:52 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:35:52 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:35:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:35:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:35:52 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(11520) called with curMem=419752, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 11.3 KB, free 730.2 MB) -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(3382) called with curMem=431272, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55274 (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:35:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:35:52 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 21:35:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:35:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:35:52 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 21:35:52 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 21:35:52 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:35:52 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 7968 bytes result sent to driver -2016-05-24 21:35:52 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:35:52 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 7968 bytes result sent to driver -2016-05-24 21:35:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 72 ms on localhost (1/2) -2016-05-24 21:35:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 73 ms on localhost (2/2) -2016-05-24 21:35:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 21:35:52 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.077 s -2016-05-24 21:35:52 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.101815 s -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=434654, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(3029) called with curMem=440246, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55274 (size: 3.0 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 21:35:52 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:35:52 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:35:52 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:35:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:35:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:35:52 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(11520) called with curMem=443275, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 11.3 KB, free 730.2 MB) -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(3384) called with curMem=454795, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:55274 (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:35:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:35:52 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 21:35:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:35:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:35:52 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 21:35:52 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 21:35:52 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:35:52 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 7968 bytes result sent to driver -2016-05-24 21:35:52 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:35:52 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 7968 bytes result sent to driver -2016-05-24 21:35:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 26 ms on localhost (1/2) -2016-05-24 21:35:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 30 ms on localhost (2/2) -2016-05-24 21:35:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 21:35:52 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.033 s -2016-05-24 21:35:52 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.058497 s -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=458179, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(3013) called with curMem=463771, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.1 MB) -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:55274 (size: 2.9 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 21:35:52 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:35:52 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:35:52 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:35:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:35:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:35:52 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(11520) called with curMem=466784, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 11.3 KB, free 730.1 MB) -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(3381) called with curMem=478304, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:55274 (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:55274 in memory (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:35:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:35:52 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 21:35:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:35:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:35:52 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 21:35:52 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 21:35:52 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:55274 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:35:52 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 7968 bytes result sent to driver -2016-05-24 21:35:52 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:35:52 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 7968 bytes result sent to driver -2016-05-24 21:35:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 34 ms on localhost (1/2) -2016-05-24 21:35:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 36 ms on localhost (2/2) -2016-05-24 21:35:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 21:35:52 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.043 s -2016-05-24 21:35:52 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.106884 s -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=458665, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(3032) called with curMem=464257, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:55274 (size: 3.0 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:55274 in memory (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 21:35:52 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:35:52 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:35:52 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:35:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55274 in memory (size: 388.0 B, free: 730.4 MB) -2016-05-24 21:35:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:35:52 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(11520) called with curMem=446406, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 11.3 KB, free 730.1 MB) -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(3383) called with curMem=457926, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:55274 (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:35:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:35:52 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 21:35:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:35:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:35:52 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 21:35:52 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 21:35:52 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:35:52 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 7968 bytes result sent to driver -2016-05-24 21:35:52 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:35:52 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 7968 bytes result sent to driver -2016-05-24 21:35:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 33 ms on localhost (1/2) -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:55274 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 33 ms on localhost (2/2) -2016-05-24 21:35:52 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.036 s -2016-05-24 21:35:52 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.062293 s -2016-05-24 21:35:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 21:35:52 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=455156, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(3019) called with curMem=460748, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.1 MB) -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:55274 (size: 2.9 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:55274 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:55274 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 21:35:52 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:55274 in memory (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:35:52 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:35:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:35:52 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 21:35:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:35:52 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:55274 in memory (size: 3.0 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(11520) called with curMem=427515, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.3 KB, free 730.2 MB) -2016-05-24 21:35:52 INFO MemoryStore:59 - ensureFreeSpace(3384) called with curMem=439035, maxMem=766075207 -2016-05-24 21:35:52 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:55274 (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:35:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:35:52 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 21:35:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:35:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:35:52 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:55274 in memory (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 21:35:52 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:35:52 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:35:52 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 7968 bytes result sent to driver -2016-05-24 21:35:52 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 21:35:52 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 7968 bytes result sent to driver -2016-05-24 21:35:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 20 ms on localhost (1/2) -2016-05-24 21:35:52 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:55274 in memory (size: 3.0 KB, free: 730.4 MB) -2016-05-24 21:35:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 23 ms on localhost (2/2) -2016-05-24 21:35:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 21:35:52 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.025 s -2016-05-24 21:35:52 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.053904 s -2016-05-24 21:35:52 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 0.6931471805599446, 1389.54945, 1000.6912077715741, 683.1897825265019, 408.2254825265008, 162.28993605224332, 0.0 -2016-05-24 21:35:52 WARN LogisticRegressionWithSGD:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 21:35:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 21:35:52 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 21:35:52 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 21:35:52 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 21:35:53 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 21:35:53 INFO BlockManager:59 - BlockManager stopped -2016-05-24 21:35:53 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 21:35:53 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 21:35:53 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 21:35:53 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 21:35:53 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-287f9157-c32b-4d73-affd-ec4c79ded551 -2016-05-24 21:35:53 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 21:35:53 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 21:40:32 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 21:40:33 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 21:40:33 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 21:40:33 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 21:40:33 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 21:40:34 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 21:40:34 INFO Remoting:74 - Starting remoting -2016-05-24 21:40:34 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:55517] -2016-05-24 21:40:34 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55517. -2016-05-24 21:40:34 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 21:40:34 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 21:40:34 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-0d532f51-ee30-49de-9baf-32230c7b3fde -2016-05-24 21:40:34 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 21:40:34 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-cb5bd136-8a2f-4dd4-8cea-834c0ac31304\httpd-949eea25-c8e4-45e4-86af-735c735ce3f1 -2016-05-24 21:40:34 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 21:40:35 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:40:35 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55518 -2016-05-24 21:40:35 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55518. -2016-05-24 21:40:35 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 21:40:35 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:40:35 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 21:40:35 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 21:40:35 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 21:40:35 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 21:40:35 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 21:40:35 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55537. -2016-05-24 21:40:35 INFO NettyBlockTransferService:59 - Server created on 55537 -2016-05-24 21:40:35 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 21:40:35 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55537 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55537) -2016-05-24 21:40:35 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 21:40:36 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 21:40:36 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 21:40:36 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 21:40:36 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 21:40:36 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55537 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 21:40:36 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 21:40:37 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 21:40:38 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 21:40:38 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 21:40:38 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 21:40:38 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 21:40:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:40:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:40:38 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 21:40:38 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 21:40:38 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 21:40:38 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=175476, maxMem=766075207 -2016-05-24 21:40:38 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 21:40:38 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55537 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:40:38 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:40:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 21:40:38 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 21:40:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:40:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:40:38 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 21:40:38 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 21:40:38 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 21:40:38 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 21:40:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 21:40:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 21:40:38 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 21:40:38 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 21:40:38 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 21:40:38 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 21:40:38 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 21:40:38 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177621, maxMem=766075207 -2016-05-24 21:40:38 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 21:40:38 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:55537 (size: 83.1 KB, free: 730.5 MB) -2016-05-24 21:40:38 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262701, maxMem=766075207 -2016-05-24 21:40:38 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 21:40:38 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:55537 (size: 83.6 KB, free: 730.4 MB) -2016-05-24 21:40:38 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 21:40:38 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 21:40:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 389 ms on localhost (1/2) -2016-05-24 21:40:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 373 ms on localhost (2/2) -2016-05-24 21:40:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 21:40:38 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.408 s -2016-05-24 21:40:38 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.566997 s -2016-05-24 21:40:38 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 21:40:38 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 21:40:38 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:40:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:40:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:40:38 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 21:40:38 INFO MemoryStore:59 - ensureFreeSpace(4256) called with curMem=348349, maxMem=766075207 -2016-05-24 21:40:38 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-24 21:40:38 INFO MemoryStore:59 - ensureFreeSpace(2343) called with curMem=352605, maxMem=766075207 -2016-05-24 21:40:38 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-24 21:40:38 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55537 (size: 2.3 KB, free: 730.4 MB) -2016-05-24 21:40:38 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:40:38 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:40:38 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 21:40:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:40:38 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 21:40:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:40:38 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-24 21:40:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 17 ms on localhost (1/1) -2016-05-24 21:40:38 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.017 s -2016-05-24 21:40:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 21:40:38 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.035052 s -2016-05-24 21:40:38 WARN LogisticRegressionWithSGD:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-05-24 21:40:38 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 21:40:38 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 21:40:38 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 21:40:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:40:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:40:38 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[8] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 21:40:38 INFO MemoryStore:59 - ensureFreeSpace(3952) called with curMem=354948, maxMem=766075207 -2016-05-24 21:40:38 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-24 21:40:38 INFO MemoryStore:59 - ensureFreeSpace(2176) called with curMem=358900, maxMem=766075207 -2016-05-24 21:40:38 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.2 MB) -2016-05-24 21:40:38 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55537 (size: 2.1 KB, free: 730.4 MB) -2016-05-24 21:40:38 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:40:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[8] at filter at DataValidators.scala:40) -2016-05-24 21:40:38 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 21:40:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:40:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:40:38 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 21:40:38 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 21:40:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:40:38 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:40:38 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 21:40:38 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 21:40:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 15 ms on localhost (1/2) -2016-05-24 21:40:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 13 ms on localhost (2/2) -2016-05-24 21:40:38 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.016 s -2016-05-24 21:40:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 21:40:38 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.027990 s -2016-05-24 21:40:38 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 21:40:38 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 21:40:38 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 21:40:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:40:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:40:38 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 21:40:38 INFO MemoryStore:59 - ensureFreeSpace(3960) called with curMem=361076, maxMem=766075207 -2016-05-24 21:40:38 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-24 21:40:38 INFO MemoryStore:59 - ensureFreeSpace(2193) called with curMem=365036, maxMem=766075207 -2016-05-24 21:40:38 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.2 MB) -2016-05-24 21:40:38 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55537 (size: 2.1 KB, free: 730.4 MB) -2016-05-24 21:40:38 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:40:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 21:40:38 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 21:40:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:40:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:40:38 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 21:40:38 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 21:40:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:40:38 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:40:38 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 21:40:38 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 21:40:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 13 ms on localhost (1/2) -2016-05-24 21:40:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 12 ms on localhost (2/2) -2016-05-24 21:40:38 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.014 s -2016-05-24 21:40:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 21:40:38 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.028006 s -2016-05-24 21:40:39 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 21:40:39 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=367229, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(388) called with curMem=372821, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 388.0 B, free 730.2 MB) -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55537 (size: 388.0 B, free: 730.4 MB) -2016-05-24 21:40:39 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 21:40:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:40:39 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:40:39 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:40:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:40:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:40:39 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[11] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(11520) called with curMem=373209, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 11.3 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(3383) called with curMem=384729, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55537 (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:40:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[11] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:40:39 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 21:40:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:40:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:40:39 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 21:40:39 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 21:40:39 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:40:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:40:39 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 7968 bytes result sent to driver -2016-05-24 21:40:39 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 7968 bytes result sent to driver -2016-05-24 21:40:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 21 ms on localhost (1/2) -2016-05-24 21:40:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 26 ms on localhost (2/2) -2016-05-24 21:40:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 21:40:39 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 21:40:39 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.049425 s -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=388112, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(2527) called with curMem=393704, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55537 (size: 2.5 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 21:40:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:40:39 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:40:39 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:40:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:40:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:40:39 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(11520) called with curMem=396231, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 11.3 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(3381) called with curMem=407751, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55537 (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:40:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:40:39 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 21:40:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:40:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:40:39 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 21:40:39 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 21:40:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:40:39 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:40:39 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 7968 bytes result sent to driver -2016-05-24 21:40:39 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 7968 bytes result sent to driver -2016-05-24 21:40:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 19 ms on localhost (1/2) -2016-05-24 21:40:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 20 ms on localhost (2/2) -2016-05-24 21:40:39 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 21:40:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 21:40:39 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.040229 s -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=411132, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(3026) called with curMem=416724, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55537 (size: 3.0 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 21:40:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:40:39 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:40:39 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:40:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:40:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:40:39 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(11520) called with curMem=419750, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 11.3 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(3383) called with curMem=431270, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55537 (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:40:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:40:39 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 21:40:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:40:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:40:39 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 21:40:39 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 21:40:39 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:40:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:40:39 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 7968 bytes result sent to driver -2016-05-24 21:40:39 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 7968 bytes result sent to driver -2016-05-24 21:40:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 19 ms on localhost (1/2) -2016-05-24 21:40:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 22 ms on localhost (2/2) -2016-05-24 21:40:39 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 21:40:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 21:40:39 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.042592 s -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=434653, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(3029) called with curMem=440245, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55537 (size: 3.0 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 21:40:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:40:39 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:40:39 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:40:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:40:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:40:39 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(11520) called with curMem=443274, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 11.3 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(3383) called with curMem=454794, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:55537 (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:40:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:40:39 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 21:40:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:40:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:40:39 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 21:40:39 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 21:40:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:40:39 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:40:39 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 7968 bytes result sent to driver -2016-05-24 21:40:39 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 7968 bytes result sent to driver -2016-05-24 21:40:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 16 ms on localhost (1/2) -2016-05-24 21:40:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 16 ms on localhost (2/2) -2016-05-24 21:40:39 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:40:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 21:40:39 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.037737 s -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=458177, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(3013) called with curMem=463769, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.1 MB) -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:55537 (size: 2.9 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:55537 in memory (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:40:39 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:40:39 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:40:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:40:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:40:39 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:40:39 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:55537 in memory (size: 3.0 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(11520) called with curMem=443258, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 11.3 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(3382) called with curMem=439875, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:55537 in memory (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:55537 (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:40:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:40:39 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 21:40:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:55537 in memory (size: 3.0 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:40:39 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 21:40:39 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:55537 in memory (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 21:40:39 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:40:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:40:39 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 7968 bytes result sent to driver -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:55537 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 7968 bytes result sent to driver -2016-05-24 21:40:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 21 ms on localhost (1/2) -2016-05-24 21:40:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 25 ms on localhost (2/2) -2016-05-24 21:40:39 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 21:40:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 21:40:39 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.047083 s -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:55537 in memory (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55537 in memory (size: 388.0 B, free: 730.4 MB) -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:55537 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=384583, maxMem=766075207 -2016-05-24 21:40:39 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(3032) called with curMem=387999, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:55537 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:55537 (size: 3.0 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:55537 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:55537 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 21:40:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:40:39 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:40:39 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:40:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:40:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:40:39 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(11520) called with curMem=374439, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 11.3 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(3383) called with curMem=385959, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:55537 (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:40:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:40:39 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 21:40:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:40:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:40:39 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 21:40:39 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 21:40:39 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:40:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:40:39 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 7968 bytes result sent to driver -2016-05-24 21:40:39 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 7968 bytes result sent to driver -2016-05-24 21:40:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 15 ms on localhost (1/2) -2016-05-24 21:40:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 18 ms on localhost (2/2) -2016-05-24 21:40:39 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 21:40:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 21:40:39 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.036184 s -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=389342, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(3019) called with curMem=394934, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:55537 (size: 2.9 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 21:40:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:40:39 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:40:39 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:40:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:40:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:40:39 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(11520) called with curMem=397953, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.3 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(3384) called with curMem=409473, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:55537 (size: 3.3 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:40:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:40:39 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 21:40:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:40:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:40:39 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 21:40:39 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 21:40:39 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:40:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:40:39 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 7968 bytes result sent to driver -2016-05-24 21:40:39 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 7968 bytes result sent to driver -2016-05-24 21:40:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 15 ms on localhost (1/2) -2016-05-24 21:40:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 17 ms on localhost (2/2) -2016-05-24 21:40:39 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:40:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 21:40:39 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.034075 s -2016-05-24 21:40:39 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 0.6931471805599446, 1389.54945, 1000.6912077715741, 683.1897825265019, 408.2254825265008, 162.28993605224332, 0.0 -2016-05-24 21:40:39 WARN LogisticRegressionWithSGD:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-05-24 21:40:39 INFO SparkContext:59 - Starting job: take at LogisticRegression2Learning.scala:29 -2016-05-24 21:40:39 INFO DAGScheduler:59 - Got job 11 (take at LogisticRegression2Learning.scala:29) with 1 output partitions -2016-05-24 21:40:39 INFO DAGScheduler:59 - Final stage: ResultStage 11(take at LogisticRegression2Learning.scala:29) -2016-05-24 21:40:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:40:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:40:39 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(3976) called with curMem=412857, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=416833, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:55537 (size: 2.1 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:40:39 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 11 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-24 21:40:39 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 1 tasks -2016-05-24 21:40:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:40:39 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 21:40:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:40:39 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 18702 bytes result sent to driver -2016-05-24 21:40:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 11 ms on localhost (1/1) -2016-05-24 21:40:39 INFO DAGScheduler:59 - ResultStage 11 (take at LogisticRegression2Learning.scala:29) finished in 0.011 s -2016-05-24 21:40:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 21:40:39 INFO DAGScheduler:59 - Job 11 finished: take at LogisticRegression2Learning.scala:29, took 0.025541 s -2016-05-24 21:40:39 INFO SparkContext:59 - Starting job: take at LogisticRegression2Learning.scala:30 -2016-05-24 21:40:39 INFO DAGScheduler:59 - Got job 12 (take at LogisticRegression2Learning.scala:30) with 1 output partitions -2016-05-24 21:40:39 INFO DAGScheduler:59 - Final stage: ResultStage 12(take at LogisticRegression2Learning.scala:30) -2016-05-24 21:40:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:40:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:40:39 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(3976) called with curMem=419027, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.9 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO MemoryStore:59 - ensureFreeSpace(2190) called with curMem=423003, maxMem=766075207 -2016-05-24 21:40:39 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.2 MB) -2016-05-24 21:40:39 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:55537 (size: 2.1 KB, free: 730.4 MB) -2016-05-24 21:40:39 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:40:39 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-24 21:40:39 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-24 21:40:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 22, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:40:39 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 22) -2016-05-24 21:40:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:40:39 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 22). 4475 bytes result sent to driver -2016-05-24 21:40:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 22) in 12 ms on localhost (1/1) -2016-05-24 21:40:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 21:40:39 INFO DAGScheduler:59 - ResultStage 12 (take at LogisticRegression2Learning.scala:30) finished in 0.012 s -2016-05-24 21:40:39 INFO DAGScheduler:59 - Job 12 finished: take at LogisticRegression2Learning.scala:30, took 0.031450 s -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 21:40:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 21:40:39 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 21:40:39 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 21:40:40 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 21:40:40 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 21:40:40 INFO BlockManager:59 - BlockManager stopped -2016-05-24 21:40:40 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 21:40:40 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 21:40:40 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 21:40:40 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 21:40:40 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-cb5bd136-8a2f-4dd4-8cea-834c0ac31304 -2016-05-24 21:40:40 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 21:41:22 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 21:41:23 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 21:41:23 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 21:41:23 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 21:41:23 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 21:41:24 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 21:41:24 INFO Remoting:74 - Starting remoting -2016-05-24 21:41:24 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:55559] -2016-05-24 21:41:24 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55559. -2016-05-24 21:41:24 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 21:41:24 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 21:41:24 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-e994915b-7f81-4ba3-84f6-63a66c50e327 -2016-05-24 21:41:24 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 21:41:24 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-e927798e-3313-407d-b0b1-5a1e4f7e1b08\httpd-235b8e67-4142-4e3c-bf8b-3301fb7f0f1d -2016-05-24 21:41:24 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 21:41:24 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:41:24 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55560 -2016-05-24 21:41:24 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55560. -2016-05-24 21:41:24 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 21:41:24 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:41:24 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 21:41:24 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 21:41:24 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 21:41:25 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 21:41:25 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 21:41:25 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55579. -2016-05-24 21:41:25 INFO NettyBlockTransferService:59 - Server created on 55579 -2016-05-24 21:41:25 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 21:41:25 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55579 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55579) -2016-05-24 21:41:25 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 21:41:26 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 21:41:26 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 21:41:26 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 21:41:26 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 21:41:26 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55579 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 21:41:26 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 21:41:27 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 21:41:27 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 21:41:27 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 21:41:27 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 21:41:27 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 21:41:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:41:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:41:27 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 21:41:27 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 21:41:27 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 21:41:27 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=175476, maxMem=766075207 -2016-05-24 21:41:27 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 21:41:27 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55579 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:41:27 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:41:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 21:41:27 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 21:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:41:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:41:28 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 21:41:28 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 21:41:28 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 21:41:28 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 21:41:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 21:41:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 21:41:28 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 21:41:28 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 21:41:28 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 21:41:28 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 21:41:28 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 21:41:28 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177621, maxMem=766075207 -2016-05-24 21:41:28 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 21:41:28 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:55579 (size: 83.1 KB, free: 730.5 MB) -2016-05-24 21:41:28 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262701, maxMem=766075207 -2016-05-24 21:41:28 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 21:41:28 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:55579 (size: 83.6 KB, free: 730.4 MB) -2016-05-24 21:41:28 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 21:41:28 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 21:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 380 ms on localhost (1/2) -2016-05-24 21:41:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 392 ms on localhost (2/2) -2016-05-24 21:41:28 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.438 s -2016-05-24 21:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 21:41:28 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.614299 s -2016-05-24 21:41:28 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 21:41:28 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 21:41:28 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:41:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:41:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:41:28 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 21:41:28 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=348349, maxMem=766075207 -2016-05-24 21:41:28 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-24 21:41:28 INFO MemoryStore:59 - ensureFreeSpace(2481) called with curMem=352909, maxMem=766075207 -2016-05-24 21:41:28 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-24 21:41:28 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55579 (size: 2.4 KB, free: 730.4 MB) -2016-05-24 21:41:28 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:41:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:41:28 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 21:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:41:28 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 21:41:28 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:41:28 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-24 21:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 27 ms on localhost (1/1) -2016-05-24 21:41:28 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.027 s -2016-05-24 21:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 21:41:28 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.049652 s -2016-05-24 21:41:28 WARN LogisticRegressionWithSGD:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-05-24 21:41:28 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 21:41:28 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 21:41:28 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 21:41:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:41:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:41:28 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 21:41:28 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=355390, maxMem=766075207 -2016-05-24 21:41:28 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-24 21:41:28 INFO MemoryStore:59 - ensureFreeSpace(2294) called with curMem=359638, maxMem=766075207 -2016-05-24 21:41:28 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 21:41:28 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55579 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:41:28 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:41:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40) -2016-05-24 21:41:28 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 21:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:41:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:41:28 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 21:41:28 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 21:41:28 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:41:28 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:41:28 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 21:41:28 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 21:41:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 13 ms on localhost (1/2) -2016-05-24 21:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 15 ms on localhost (2/2) -2016-05-24 21:41:28 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.016 s -2016-05-24 21:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 21:41:28 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.029337 s -2016-05-24 21:41:28 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 21:41:28 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 21:41:28 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 21:41:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:41:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:41:28 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 21:41:28 INFO MemoryStore:59 - ensureFreeSpace(4264) called with curMem=361932, maxMem=766075207 -2016-05-24 21:41:28 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-24 21:41:28 INFO MemoryStore:59 - ensureFreeSpace(2318) called with curMem=366196, maxMem=766075207 -2016-05-24 21:41:28 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-24 21:41:28 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55579 (size: 2.3 KB, free: 730.4 MB) -2016-05-24 21:41:28 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:41:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 21:41:28 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 21:41:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:41:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:41:28 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 21:41:28 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 21:41:28 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:41:28 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 21:41:28 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:41:28 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 21:41:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 15 ms on localhost (1/2) -2016-05-24 21:41:28 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.019 s -2016-05-24 21:41:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 17 ms on localhost (2/2) -2016-05-24 21:41:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 21:41:28 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.039182 s -2016-05-24 21:41:29 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 21:41:29 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=368514, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(388) called with curMem=374106, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 388.0 B, free 730.2 MB) -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55579 (size: 388.0 B, free: 730.4 MB) -2016-05-24 21:41:29 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 21:41:29 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:41:29 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:41:29 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:41:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:41:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:41:29 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=374494, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(3493) called with curMem=386294, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55579 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:41:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:41:29 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 21:41:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:41:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:41:29 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 21:41:29 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 21:41:29 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:41:29 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:41:29 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 7968 bytes result sent to driver -2016-05-24 21:41:29 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 7968 bytes result sent to driver -2016-05-24 21:41:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 23 ms on localhost (1/2) -2016-05-24 21:41:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 27 ms on localhost (2/2) -2016-05-24 21:41:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 21:41:29 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.027 s -2016-05-24 21:41:29 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.050437 s -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=389787, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(2249) called with curMem=395379, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55579 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 21:41:29 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:41:29 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:41:29 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:41:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:41:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:41:29 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=397628, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=409428, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55579 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:41:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:41:29 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 21:41:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:41:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:41:29 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 21:41:29 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 21:41:29 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:41:29 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:41:29 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 7968 bytes result sent to driver -2016-05-24 21:41:29 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 7968 bytes result sent to driver -2016-05-24 21:41:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 20 ms on localhost (1/2) -2016-05-24 21:41:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 22 ms on localhost (2/2) -2016-05-24 21:41:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 21:41:29 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 21:41:29 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.049926 s -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=412922, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(3115) called with curMem=418514, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55579 (size: 3.0 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 21:41:29 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:41:29 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:41:29 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:41:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:41:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:41:29 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=421629, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=433429, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55579 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:41:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:41:29 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 21:41:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:41:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:41:29 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 21:41:29 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 21:41:29 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:41:29 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:41:29 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 7968 bytes result sent to driver -2016-05-24 21:41:29 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 7968 bytes result sent to driver -2016-05-24 21:41:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 18 ms on localhost (1/2) -2016-05-24 21:41:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 20 ms on localhost (2/2) -2016-05-24 21:41:29 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 21:41:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 21:41:29 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.041757 s -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=436923, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(3151) called with curMem=442515, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55579 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 21:41:29 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:41:29 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:41:29 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:41:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:41:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:41:29 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=445666, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=457466, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:55579 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:41:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:41:29 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 21:41:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:41:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:41:29 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 21:41:29 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 21:41:29 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:41:29 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:41:29 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 7968 bytes result sent to driver -2016-05-24 21:41:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 12 ms on localhost (1/2) -2016-05-24 21:41:29 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 7968 bytes result sent to driver -2016-05-24 21:41:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 17 ms on localhost (2/2) -2016-05-24 21:41:29 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:41:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 21:41:29 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.038643 s -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=460960, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(3152) called with curMem=466552, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:55579 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 21:41:29 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:41:29 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:41:29 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:41:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:41:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:41:29 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:55579 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=454410, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(3492) called with curMem=466210, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 21:41:29 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:55579 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:41:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:41:29 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:55579 in memory (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:41:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:41:29 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:55579 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 21:41:29 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55579 in memory (size: 388.0 B, free: 730.4 MB) -2016-05-24 21:41:29 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:41:29 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:41:29 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 7968 bytes result sent to driver -2016-05-24 21:41:29 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 7968 bytes result sent to driver -2016-05-24 21:41:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 22 ms on localhost (1/2) -2016-05-24 21:41:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 21 ms on localhost (2/2) -2016-05-24 21:41:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 21:41:29 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 21:41:29 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.046166 s -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:55579 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:55579 in memory (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=427464, maxMem=766075207 -2016-05-24 21:41:29 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=426015, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:55579 in memory (size: 2.4 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:55579 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:55579 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:55579 in memory (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:55579 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:55579 in memory (size: 3.0 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:41:29 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:41:29 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:41:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:41:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:41:29 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=381141, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=392941, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:55579 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:41:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:41:29 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 21:41:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:41:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:41:29 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 21:41:29 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 21:41:29 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:41:29 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:41:29 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 7968 bytes result sent to driver -2016-05-24 21:41:29 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 7968 bytes result sent to driver -2016-05-24 21:41:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 13 ms on localhost (1/2) -2016-05-24 21:41:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 16 ms on localhost (2/2) -2016-05-24 21:41:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 21:41:29 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:41:29 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.033817 s -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=396435, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=402027, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:55579 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 21:41:29 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:41:29 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:41:29 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:41:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:41:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:41:29 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=405191, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=416991, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:55579 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:41:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:41:29 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 21:41:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:41:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:41:29 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 21:41:29 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 21:41:29 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:41:29 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:41:29 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 7968 bytes result sent to driver -2016-05-24 21:41:29 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 7968 bytes result sent to driver -2016-05-24 21:41:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 11 ms on localhost (1/2) -2016-05-24 21:41:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 13 ms on localhost (2/2) -2016-05-24 21:41:29 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:41:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 21:41:29 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.026580 s -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=420485, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=426077, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:55579 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 21:41:29 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:41:29 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:41:29 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:41:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:41:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:41:29 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=429241, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(3495) called with curMem=441041, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:55579 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:41:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:41:29 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 21:41:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:41:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:41:29 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 21:41:29 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 21:41:29 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:41:29 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:41:29 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 7968 bytes result sent to driver -2016-05-24 21:41:29 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 7968 bytes result sent to driver -2016-05-24 21:41:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 10 ms on localhost (1/2) -2016-05-24 21:41:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 12 ms on localhost (2/2) -2016-05-24 21:41:29 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 21:41:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 21:41:29 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.035972 s -2016-05-24 21:41:29 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 0.6931471805599448, 10057.41015625, 3992.030200294059, 2280.4067582415346, 1609.1071976946598, 1008.6786174352385, 460.5648214704288, 0.0 -2016-05-24 21:41:29 WARN LogisticRegressionWithSGD:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-05-24 21:41:29 INFO SparkContext:59 - Starting job: collectAsMap at MulticlassMetrics.scala:51 -2016-05-24 21:41:29 INFO DAGScheduler:59 - Registering RDD 29 (map at MulticlassMetrics.scala:48) -2016-05-24 21:41:29 INFO DAGScheduler:59 - Got job 12 (collectAsMap at MulticlassMetrics.scala:51) with 2 output partitions -2016-05-24 21:41:29 INFO DAGScheduler:59 - Final stage: ResultStage 13(collectAsMap at MulticlassMetrics.scala:51) -2016-05-24 21:41:29 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-24 21:41:29 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-05-24 21:41:29 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (MapPartitionsRDD[29] at map at MulticlassMetrics.scala:48), which has no missing parents -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(11408) called with curMem=444536, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 11.1 KB, free 730.2 MB) -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(6171) called with curMem=455944, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 6.0 KB, free 730.1 MB) -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:55579 (size: 6.0 KB, free: 730.4 MB) -2016-05-24 21:41:29 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:41:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (MapPartitionsRDD[29] at map at MulticlassMetrics.scala:48) -2016-05-24 21:41:29 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 21:41:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 21:41:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 21:41:29 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 21:41:29 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 21:41:29 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:41:29 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:41:29 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2254 bytes result sent to driver -2016-05-24 21:41:29 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2254 bytes result sent to driver -2016-05-24 21:41:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 49 ms on localhost (1/2) -2016-05-24 21:41:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 49 ms on localhost (2/2) -2016-05-24 21:41:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 21:41:29 INFO DAGScheduler:59 - ShuffleMapStage 12 (map at MulticlassMetrics.scala:48) finished in 0.052 s -2016-05-24 21:41:29 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 21:41:29 INFO DAGScheduler:59 - running: Set() -2016-05-24 21:41:29 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-05-24 21:41:29 INFO DAGScheduler:59 - failed: Set() -2016-05-24 21:41:29 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-05-24 21:41:29 INFO DAGScheduler:59 - Submitting ResultStage 13 (ShuffledRDD[30] at reduceByKey at MulticlassMetrics.scala:50), which is now runnable -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=462115, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 21:41:29 INFO MemoryStore:59 - ensureFreeSpace(1412) called with curMem=464451, maxMem=766075207 -2016-05-24 21:41:29 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 1412.0 B, free 730.1 MB) -2016-05-24 21:41:29 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:55579 (size: 1412.0 B, free: 730.4 MB) -2016-05-24 21:41:29 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:41:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (ShuffledRDD[30] at reduceByKey at MulticlassMetrics.scala:50) -2016-05-24 21:41:29 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 21:41:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 21:41:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 21:41:29 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 21:41:29 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 21:41:29 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-24 21:41:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 21:41:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-24 21:41:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-24 21:41:29 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 1161 bytes result sent to driver -2016-05-24 21:41:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 55 ms on localhost (1/2) -2016-05-24 21:41:29 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 1389 bytes result sent to driver -2016-05-24 21:41:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 58 ms on localhost (2/2) -2016-05-24 21:41:29 INFO DAGScheduler:59 - ResultStage 13 (collectAsMap at MulticlassMetrics.scala:51) finished in 0.058 s -2016-05-24 21:41:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 21:41:29 INFO DAGScheduler:59 - Job 12 finished: collectAsMap at MulticlassMetrics.scala:51, took 0.146054 s -2016-05-24 21:41:30 INFO SparkContext:59 - Starting job: countByValue at MulticlassMetrics.scala:45 -2016-05-24 21:41:30 INFO DAGScheduler:59 - Registering RDD 33 (countByValue at MulticlassMetrics.scala:45) -2016-05-24 21:41:30 INFO DAGScheduler:59 - Got job 13 (countByValue at MulticlassMetrics.scala:45) with 2 output partitions -2016-05-24 21:41:30 INFO DAGScheduler:59 - Final stage: ResultStage 15(countByValue at MulticlassMetrics.scala:45) -2016-05-24 21:41:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 14) -2016-05-24 21:41:30 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 14) -2016-05-24 21:41:30 INFO DAGScheduler:59 - Submitting ShuffleMapStage 14 (MapPartitionsRDD[33] at countByValue at MulticlassMetrics.scala:45), which has no missing parents -2016-05-24 21:41:30 INFO MemoryStore:59 - ensureFreeSpace(11936) called with curMem=465863, maxMem=766075207 -2016-05-24 21:41:30 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 11.7 KB, free 730.1 MB) -2016-05-24 21:41:30 INFO MemoryStore:59 - ensureFreeSpace(6356) called with curMem=477799, maxMem=766075207 -2016-05-24 21:41:30 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 6.2 KB, free 730.1 MB) -2016-05-24 21:41:30 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:55579 (size: 6.2 KB, free: 730.4 MB) -2016-05-24 21:41:30 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:41:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 14 (MapPartitionsRDD[33] at countByValue at MulticlassMetrics.scala:45) -2016-05-24 21:41:30 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 21:41:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 21:41:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 21:41:30 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 21:41:30 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 21:41:30 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:41:30 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:41:30 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2254 bytes result sent to driver -2016-05-24 21:41:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 19 ms on localhost (1/2) -2016-05-24 21:41:30 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2254 bytes result sent to driver -2016-05-24 21:41:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 22 ms on localhost (2/2) -2016-05-24 21:41:30 INFO DAGScheduler:59 - ShuffleMapStage 14 (countByValue at MulticlassMetrics.scala:45) finished in 0.023 s -2016-05-24 21:41:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 21:41:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 21:41:30 INFO DAGScheduler:59 - running: Set() -2016-05-24 21:41:30 INFO DAGScheduler:59 - waiting: Set(ResultStage 15) -2016-05-24 21:41:30 INFO DAGScheduler:59 - failed: Set() -2016-05-24 21:41:30 INFO DAGScheduler:59 - Missing parents for ResultStage 15: List() -2016-05-24 21:41:30 INFO DAGScheduler:59 - Submitting ResultStage 15 (ShuffledRDD[34] at countByValue at MulticlassMetrics.scala:45), which is now runnable -2016-05-24 21:41:30 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=484155, maxMem=766075207 -2016-05-24 21:41:30 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 21:41:30 INFO MemoryStore:59 - ensureFreeSpace(1371) called with curMem=486491, maxMem=766075207 -2016-05-24 21:41:30 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 1371.0 B, free 730.1 MB) -2016-05-24 21:41:30 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:55579 (size: 1371.0 B, free: 730.4 MB) -2016-05-24 21:41:30 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:41:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (ShuffledRDD[34] at countByValue at MulticlassMetrics.scala:45) -2016-05-24 21:41:30 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 21:41:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 21:41:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 21:41:30 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 21:41:30 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 21:41:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 21:41:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 21:41:30 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-24 21:41:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 21:41:30 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 1161 bytes result sent to driver -2016-05-24 21:41:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 8 ms on localhost (1/2) -2016-05-24 21:41:30 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 1394 bytes result sent to driver -2016-05-24 21:41:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 12 ms on localhost (2/2) -2016-05-24 21:41:30 INFO DAGScheduler:59 - ResultStage 15 (countByValue at MulticlassMetrics.scala:45) finished in 0.013 s -2016-05-24 21:41:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 21:41:30 INFO DAGScheduler:59 - Job 13 finished: countByValue at MulticlassMetrics.scala:45, took 0.064861 s -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 21:41:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 21:41:30 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 21:41:30 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 21:41:30 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 21:41:30 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 21:41:30 INFO BlockManager:59 - BlockManager stopped -2016-05-24 21:41:30 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 21:41:30 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 21:41:30 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 21:41:30 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 21:41:30 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-e927798e-3313-407d-b0b1-5a1e4f7e1b08 -2016-05-24 21:41:30 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 21:41:30 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 21:42:32 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 21:42:33 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 21:42:33 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 21:42:33 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 21:42:33 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 21:42:34 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 21:42:34 INFO Remoting:74 - Starting remoting -2016-05-24 21:42:34 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:55602] -2016-05-24 21:42:34 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55602. -2016-05-24 21:42:34 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 21:42:34 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 21:42:34 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-127c36b5-1f36-4876-a49f-05f3e2f111d1 -2016-05-24 21:42:34 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 21:42:35 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-4537d917-0ffc-4a4b-ac56-f28c84f6681c\httpd-638b7ea8-6236-49b5-8e78-37487f1f45e3 -2016-05-24 21:42:35 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 21:42:35 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:42:35 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55603 -2016-05-24 21:42:35 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55603. -2016-05-24 21:42:35 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 21:42:35 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:42:35 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 21:42:35 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 21:42:35 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 21:42:35 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 21:42:35 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 21:42:35 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55622. -2016-05-24 21:42:35 INFO NettyBlockTransferService:59 - Server created on 55622 -2016-05-24 21:42:35 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 21:42:35 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55622 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55622) -2016-05-24 21:42:35 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 21:42:36 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 21:42:36 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 21:42:36 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 21:42:36 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 21:42:36 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55622 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 21:42:36 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 21:42:37 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 21:42:38 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 21:42:38 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 21:42:38 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 21:42:38 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 21:42:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:42:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:42:38 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 21:42:38 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 21:42:38 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 21:42:38 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=175476, maxMem=766075207 -2016-05-24 21:42:38 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 21:42:38 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55622 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:42:38 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:42:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 21:42:38 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 21:42:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:42:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:42:38 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 21:42:38 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 21:42:38 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 21:42:38 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 21:42:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 21:42:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 21:42:38 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 21:42:38 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 21:42:38 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 21:42:38 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 21:42:38 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 21:42:38 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177621, maxMem=766075207 -2016-05-24 21:42:38 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 21:42:38 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262701, maxMem=766075207 -2016-05-24 21:42:38 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 21:42:38 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:55622 (size: 83.1 KB, free: 730.5 MB) -2016-05-24 21:42:38 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:55622 (size: 83.6 KB, free: 730.4 MB) -2016-05-24 21:42:38 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 21:42:38 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 21:42:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 424 ms on localhost (1/2) -2016-05-24 21:42:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 454 ms on localhost (2/2) -2016-05-24 21:42:38 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.472 s -2016-05-24 21:42:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 21:42:38 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.636067 s -2016-05-24 21:42:38 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 21:42:38 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 21:42:38 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:42:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:42:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:42:38 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 21:42:38 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=348349, maxMem=766075207 -2016-05-24 21:42:38 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-24 21:42:38 INFO MemoryStore:59 - ensureFreeSpace(2481) called with curMem=352909, maxMem=766075207 -2016-05-24 21:42:38 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-24 21:42:38 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55622 (size: 2.4 KB, free: 730.4 MB) -2016-05-24 21:42:38 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:42:38 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:42:38 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 21:42:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:42:38 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 21:42:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:42:38 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-24 21:42:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 23 ms on localhost (1/1) -2016-05-24 21:42:38 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.023 s -2016-05-24 21:42:38 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.040534 s -2016-05-24 21:42:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 21:42:38 WARN LogisticRegressionWithSGD:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-05-24 21:42:38 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 21:42:38 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 21:42:38 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 21:42:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:42:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:42:38 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 21:42:38 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=355390, maxMem=766075207 -2016-05-24 21:42:38 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-24 21:42:38 INFO MemoryStore:59 - ensureFreeSpace(2294) called with curMem=359638, maxMem=766075207 -2016-05-24 21:42:38 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 21:42:38 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55622 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:42:38 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:42:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40) -2016-05-24 21:42:38 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 21:42:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:42:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:42:38 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 21:42:38 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 21:42:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:42:38 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:42:38 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 21:42:38 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 21:42:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 13 ms on localhost (1/2) -2016-05-24 21:42:38 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.014 s -2016-05-24 21:42:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 13 ms on localhost (2/2) -2016-05-24 21:42:38 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.026497 s -2016-05-24 21:42:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 21:42:38 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 21:42:38 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 21:42:38 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 21:42:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:42:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:42:38 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 21:42:38 INFO MemoryStore:59 - ensureFreeSpace(4264) called with curMem=361932, maxMem=766075207 -2016-05-24 21:42:38 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-24 21:42:38 INFO MemoryStore:59 - ensureFreeSpace(2318) called with curMem=366196, maxMem=766075207 -2016-05-24 21:42:38 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-24 21:42:38 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55622 (size: 2.3 KB, free: 730.4 MB) -2016-05-24 21:42:38 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:42:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 21:42:38 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 21:42:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:42:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:42:38 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 21:42:38 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 21:42:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:42:38 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 21:42:38 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:42:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 17 ms on localhost (1/2) -2016-05-24 21:42:38 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 21:42:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 19 ms on localhost (2/2) -2016-05-24 21:42:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 21:42:38 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.021 s -2016-05-24 21:42:38 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.036459 s -2016-05-24 21:42:39 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 21:42:39 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=368514, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(388) called with curMem=374106, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 388.0 B, free 730.2 MB) -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55622 (size: 388.0 B, free: 730.4 MB) -2016-05-24 21:42:39 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 21:42:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:42:39 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:42:39 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:42:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:42:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:42:39 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=374494, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(3493) called with curMem=386294, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55622 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:42:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:42:39 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 21:42:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:42:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:42:39 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 21:42:39 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 21:42:39 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:42:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:42:39 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 7968 bytes result sent to driver -2016-05-24 21:42:39 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 7968 bytes result sent to driver -2016-05-24 21:42:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 22 ms on localhost (1/2) -2016-05-24 21:42:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 26 ms on localhost (2/2) -2016-05-24 21:42:39 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 21:42:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 21:42:39 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.046259 s -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=389787, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(2249) called with curMem=395379, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55622 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 21:42:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:42:39 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:42:39 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:42:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:42:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:42:39 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=397628, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=409428, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55622 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:42:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:42:39 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 21:42:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:42:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:42:39 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 21:42:39 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 21:42:39 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:42:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:42:39 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 7968 bytes result sent to driver -2016-05-24 21:42:39 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 7968 bytes result sent to driver -2016-05-24 21:42:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 20 ms on localhost (1/2) -2016-05-24 21:42:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 25 ms on localhost (2/2) -2016-05-24 21:42:39 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 21:42:39 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.049524 s -2016-05-24 21:42:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=412922, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(3115) called with curMem=418514, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55622 (size: 3.0 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 21:42:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:42:39 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:42:39 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:42:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:42:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:42:39 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=421629, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=433429, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55622 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:42:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:42:39 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 21:42:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:42:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:42:39 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 21:42:39 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 21:42:39 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:42:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:42:39 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 7968 bytes result sent to driver -2016-05-24 21:42:39 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 7968 bytes result sent to driver -2016-05-24 21:42:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 18 ms on localhost (1/2) -2016-05-24 21:42:39 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 21:42:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 21 ms on localhost (2/2) -2016-05-24 21:42:39 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.043208 s -2016-05-24 21:42:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=436923, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(3151) called with curMem=442515, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55622 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 21:42:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:42:39 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:42:39 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:42:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:42:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:42:39 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=445666, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(3493) called with curMem=457466, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:55622 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:42:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:42:39 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 21:42:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:42:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:42:39 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 21:42:39 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 21:42:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:42:39 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:42:39 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 7968 bytes result sent to driver -2016-05-24 21:42:39 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 7968 bytes result sent to driver -2016-05-24 21:42:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 53 ms on localhost (1/2) -2016-05-24 21:42:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 56 ms on localhost (2/2) -2016-05-24 21:42:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 21:42:39 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.058 s -2016-05-24 21:42:39 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.076419 s -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=460959, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(3152) called with curMem=466551, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:55622 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:55622 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:55622 in memory (size: 3.0 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:42:39 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:55622 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:42:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:42:39 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 21:42:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:42:39 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:55622 in memory (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=422567, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(3492) called with curMem=430874, maxMem=766075207 -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:55622 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:55622 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:42:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:42:39 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55622 in memory (size: 388.0 B, free: 730.4 MB) -2016-05-24 21:42:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:42:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:42:39 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 21:42:39 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:55622 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:55622 in memory (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:42:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:42:39 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 7968 bytes result sent to driver -2016-05-24 21:42:39 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 7968 bytes result sent to driver -2016-05-24 21:42:39 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 21:42:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 21 ms on localhost (1/2) -2016-05-24 21:42:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 20 ms on localhost (2/2) -2016-05-24 21:42:39 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:55622 in memory (size: 2.4 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.042611 s -2016-05-24 21:42:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 21:42:39 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=396421, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=402013, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:55622 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 21:42:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:42:39 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:42:39 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:42:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:42:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:42:39 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=405177, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=416977, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:55622 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:42:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:42:39 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 21:42:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:42:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:42:39 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 21:42:39 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 21:42:39 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:42:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:42:39 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 7968 bytes result sent to driver -2016-05-24 21:42:39 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 7968 bytes result sent to driver -2016-05-24 21:42:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 15 ms on localhost (1/2) -2016-05-24 21:42:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 15 ms on localhost (2/2) -2016-05-24 21:42:39 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 21:42:39 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.032498 s -2016-05-24 21:42:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=420471, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=426063, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:55622 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 21:42:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:42:39 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:42:39 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:42:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:42:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:42:39 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=429227, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(3495) called with curMem=441027, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:55622 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:42:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:42:39 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 21:42:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:42:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:42:39 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 21:42:39 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 21:42:39 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:42:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:42:39 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 7968 bytes result sent to driver -2016-05-24 21:42:39 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 7968 bytes result sent to driver -2016-05-24 21:42:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 11 ms on localhost (1/2) -2016-05-24 21:42:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 13 ms on localhost (2/2) -2016-05-24 21:42:39 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:42:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 21:42:39 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.029511 s -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=444522, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=450114, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:55622 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 21:42:39 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:42:39 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:42:39 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:42:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:42:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:42:39 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=453278, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(3495) called with curMem=465078, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:55622 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:42:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:42:39 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 21:42:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:42:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:42:39 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 21:42:39 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 21:42:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:42:39 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:42:39 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 7968 bytes result sent to driver -2016-05-24 21:42:39 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 7968 bytes result sent to driver -2016-05-24 21:42:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 11 ms on localhost (1/2) -2016-05-24 21:42:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 11 ms on localhost (2/2) -2016-05-24 21:42:39 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 21:42:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 21:42:39 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.026962 s -2016-05-24 21:42:39 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 0.6931471805599448, 10057.41015625, 3992.030200294059, 2280.4067582415346, 1609.1071976946598, 1008.6786174352385, 460.5648214704288, 0.0 -2016-05-24 21:42:39 WARN LogisticRegressionWithSGD:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-05-24 21:42:39 INFO SparkContext:59 - Starting job: collectAsMap at MulticlassMetrics.scala:51 -2016-05-24 21:42:39 INFO DAGScheduler:59 - Registering RDD 29 (map at MulticlassMetrics.scala:48) -2016-05-24 21:42:39 INFO DAGScheduler:59 - Got job 12 (collectAsMap at MulticlassMetrics.scala:51) with 2 output partitions -2016-05-24 21:42:39 INFO DAGScheduler:59 - Final stage: ResultStage 13(collectAsMap at MulticlassMetrics.scala:51) -2016-05-24 21:42:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-24 21:42:39 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-05-24 21:42:39 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (MapPartitionsRDD[29] at map at MulticlassMetrics.scala:48), which has no missing parents -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(11408) called with curMem=468573, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 11.1 KB, free 730.1 MB) -2016-05-24 21:42:39 INFO MemoryStore:59 - ensureFreeSpace(6171) called with curMem=479981, maxMem=766075207 -2016-05-24 21:42:39 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 6.0 KB, free 730.1 MB) -2016-05-24 21:42:39 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:55622 (size: 6.0 KB, free: 730.4 MB) -2016-05-24 21:42:39 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:42:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (MapPartitionsRDD[29] at map at MulticlassMetrics.scala:48) -2016-05-24 21:42:39 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 21:42:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 21:42:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 21:42:40 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 21:42:40 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 21:42:40 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:42:40 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:42:40 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2254 bytes result sent to driver -2016-05-24 21:42:40 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2254 bytes result sent to driver -2016-05-24 21:42:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 52 ms on localhost (1/2) -2016-05-24 21:42:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 54 ms on localhost (2/2) -2016-05-24 21:42:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 21:42:40 INFO DAGScheduler:59 - ShuffleMapStage 12 (map at MulticlassMetrics.scala:48) finished in 0.055 s -2016-05-24 21:42:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 21:42:40 INFO DAGScheduler:59 - running: Set() -2016-05-24 21:42:40 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-05-24 21:42:40 INFO DAGScheduler:59 - failed: Set() -2016-05-24 21:42:40 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-05-24 21:42:40 INFO DAGScheduler:59 - Submitting ResultStage 13 (ShuffledRDD[30] at reduceByKey at MulticlassMetrics.scala:50), which is now runnable -2016-05-24 21:42:40 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=486152, maxMem=766075207 -2016-05-24 21:42:40 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 21:42:40 INFO MemoryStore:59 - ensureFreeSpace(1412) called with curMem=488488, maxMem=766075207 -2016-05-24 21:42:40 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 1412.0 B, free 730.1 MB) -2016-05-24 21:42:40 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:55622 (size: 1412.0 B, free: 730.4 MB) -2016-05-24 21:42:40 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:42:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (ShuffledRDD[30] at reduceByKey at MulticlassMetrics.scala:50) -2016-05-24 21:42:40 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 21:42:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 21:42:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 21:42:40 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 21:42:40 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 21:42:40 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-24 21:42:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 21:42:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-24 21:42:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-24 21:42:40 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 1161 bytes result sent to driver -2016-05-24 21:42:40 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 1389 bytes result sent to driver -2016-05-24 21:42:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 27 ms on localhost (1/2) -2016-05-24 21:42:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 29 ms on localhost (2/2) -2016-05-24 21:42:40 INFO DAGScheduler:59 - ResultStage 13 (collectAsMap at MulticlassMetrics.scala:51) finished in 0.029 s -2016-05-24 21:42:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 21:42:40 INFO DAGScheduler:59 - Job 12 finished: collectAsMap at MulticlassMetrics.scala:51, took 0.119005 s -2016-05-24 21:42:40 INFO SparkContext:59 - Starting job: countByValue at MulticlassMetrics.scala:45 -2016-05-24 21:42:40 INFO DAGScheduler:59 - Registering RDD 33 (countByValue at MulticlassMetrics.scala:45) -2016-05-24 21:42:40 INFO DAGScheduler:59 - Got job 13 (countByValue at MulticlassMetrics.scala:45) with 2 output partitions -2016-05-24 21:42:40 INFO DAGScheduler:59 - Final stage: ResultStage 15(countByValue at MulticlassMetrics.scala:45) -2016-05-24 21:42:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 14) -2016-05-24 21:42:40 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 14) -2016-05-24 21:42:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 14 (MapPartitionsRDD[33] at countByValue at MulticlassMetrics.scala:45), which has no missing parents -2016-05-24 21:42:40 INFO MemoryStore:59 - ensureFreeSpace(11936) called with curMem=489900, maxMem=766075207 -2016-05-24 21:42:40 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 11.7 KB, free 730.1 MB) -2016-05-24 21:42:40 INFO MemoryStore:59 - ensureFreeSpace(6356) called with curMem=501836, maxMem=766075207 -2016-05-24 21:42:40 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 6.2 KB, free 730.1 MB) -2016-05-24 21:42:40 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:55622 (size: 6.2 KB, free: 730.4 MB) -2016-05-24 21:42:40 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:42:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 14 (MapPartitionsRDD[33] at countByValue at MulticlassMetrics.scala:45) -2016-05-24 21:42:40 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 21:42:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 21:42:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 21:42:40 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 21:42:40 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 21:42:40 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:42:40 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:42:40 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2254 bytes result sent to driver -2016-05-24 21:42:40 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2254 bytes result sent to driver -2016-05-24 21:42:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 23 ms on localhost (1/2) -2016-05-24 21:42:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 23 ms on localhost (2/2) -2016-05-24 21:42:40 INFO DAGScheduler:59 - ShuffleMapStage 14 (countByValue at MulticlassMetrics.scala:45) finished in 0.024 s -2016-05-24 21:42:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 21:42:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 21:42:40 INFO DAGScheduler:59 - running: Set() -2016-05-24 21:42:40 INFO DAGScheduler:59 - waiting: Set(ResultStage 15) -2016-05-24 21:42:40 INFO DAGScheduler:59 - failed: Set() -2016-05-24 21:42:40 INFO DAGScheduler:59 - Missing parents for ResultStage 15: List() -2016-05-24 21:42:40 INFO DAGScheduler:59 - Submitting ResultStage 15 (ShuffledRDD[34] at countByValue at MulticlassMetrics.scala:45), which is now runnable -2016-05-24 21:42:40 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=508192, maxMem=766075207 -2016-05-24 21:42:40 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 21:42:40 INFO MemoryStore:59 - ensureFreeSpace(1371) called with curMem=510528, maxMem=766075207 -2016-05-24 21:42:40 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 1371.0 B, free 730.1 MB) -2016-05-24 21:42:40 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:55622 (size: 1371.0 B, free: 730.4 MB) -2016-05-24 21:42:40 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:42:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (ShuffledRDD[34] at countByValue at MulticlassMetrics.scala:45) -2016-05-24 21:42:40 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 21:42:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 21:42:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 21:42:40 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 21:42:40 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 21:42:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 21:42:40 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-24 21:42:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 21:42:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 21:42:40 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 1161 bytes result sent to driver -2016-05-24 21:42:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 9 ms on localhost (1/2) -2016-05-24 21:42:40 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 1394 bytes result sent to driver -2016-05-24 21:42:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 14 ms on localhost (2/2) -2016-05-24 21:42:40 INFO DAGScheduler:59 - ResultStage 15 (countByValue at MulticlassMetrics.scala:45) finished in 0.015 s -2016-05-24 21:42:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 21:42:40 INFO DAGScheduler:59 - Job 13 finished: countByValue at MulticlassMetrics.scala:45, took 0.065552 s -2016-05-24 21:42:40 INFO SparkContext:59 - Starting job: count at LogisticRegression3Learning.scala:39 -2016-05-24 21:42:40 INFO DAGScheduler:59 - Got job 14 (count at LogisticRegression3Learning.scala:39) with 2 output partitions -2016-05-24 21:42:40 INFO DAGScheduler:59 - Final stage: ResultStage 16(count at LogisticRegression3Learning.scala:39) -2016-05-24 21:42:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:42:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:42:40 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[7] at randomSplit at LogisticRegression3Learning.scala:25), which has no missing parents -2016-05-24 21:42:40 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=511899, maxMem=766075207 -2016-05-24 21:42:40 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 4.0 KB, free 730.1 MB) -2016-05-24 21:42:40 INFO MemoryStore:59 - ensureFreeSpace(2257) called with curMem=516011, maxMem=766075207 -2016-05-24 21:42:40 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-24 21:42:40 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:55622 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:42:40 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:42:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[7] at randomSplit at LogisticRegression3Learning.scala:25) -2016-05-24 21:42:40 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 21:42:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:42:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:42:40 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 21:42:40 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 21:42:40 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:42:40 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:42:40 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2082 bytes result sent to driver -2016-05-24 21:42:40 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2082 bytes result sent to driver -2016-05-24 21:42:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 9 ms on localhost (1/2) -2016-05-24 21:42:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 11 ms on localhost (2/2) -2016-05-24 21:42:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 21:42:40 INFO DAGScheduler:59 - ResultStage 16 (count at LogisticRegression3Learning.scala:39) finished in 0.011 s -2016-05-24 21:42:40 INFO DAGScheduler:59 - Job 14 finished: count at LogisticRegression3Learning.scala:39, took 0.058316 s -2016-05-24 21:42:40 INFO SparkContext:59 - Starting job: count at LogisticRegression3Learning.scala:40 -2016-05-24 21:42:40 INFO DAGScheduler:59 - Got job 15 (count at LogisticRegression3Learning.scala:40) with 2 output partitions -2016-05-24 21:42:40 INFO DAGScheduler:59 - Final stage: ResultStage 17(count at LogisticRegression3Learning.scala:40) -2016-05-24 21:42:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:42:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:42:40 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[8] at randomSplit at LogisticRegression3Learning.scala:25), which has no missing parents -2016-05-24 21:42:40 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=518268, maxMem=766075207 -2016-05-24 21:42:40 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 4.0 KB, free 730.1 MB) -2016-05-24 21:42:40 INFO MemoryStore:59 - ensureFreeSpace(2261) called with curMem=522380, maxMem=766075207 -2016-05-24 21:42:40 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-24 21:42:40 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:55622 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:42:40 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:42:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[8] at randomSplit at LogisticRegression3Learning.scala:25) -2016-05-24 21:42:40 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 21:42:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:42:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:42:40 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 21:42:40 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 21:42:40 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:42:40 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:42:40 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2082 bytes result sent to driver -2016-05-24 21:42:40 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2082 bytes result sent to driver -2016-05-24 21:42:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 11 ms on localhost (1/2) -2016-05-24 21:42:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 11 ms on localhost (2/2) -2016-05-24 21:42:40 INFO DAGScheduler:59 - ResultStage 17 (count at LogisticRegression3Learning.scala:40) finished in 0.012 s -2016-05-24 21:42:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 21:42:40 INFO DAGScheduler:59 - Job 15 finished: count at LogisticRegression3Learning.scala:40, took 0.022999 s -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 21:42:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 21:42:40 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 21:42:40 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 21:42:40 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 21:42:40 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 21:42:40 INFO BlockManager:59 - BlockManager stopped -2016-05-24 21:42:40 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 21:42:40 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 21:42:40 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 21:42:40 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 21:42:40 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-4537d917-0ffc-4a4b-ac56-f28c84f6681c -2016-05-24 21:43:48 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 21:43:48 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 21:43:49 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 21:43:49 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 21:43:49 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 21:43:50 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 21:43:50 INFO Remoting:74 - Starting remoting -2016-05-24 21:43:50 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:55643] -2016-05-24 21:43:50 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55643. -2016-05-24 21:43:50 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 21:43:50 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 21:43:50 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-ed35426a-1ad0-4f64-a540-05e935f7f1dd -2016-05-24 21:43:50 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 21:43:50 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-df012fd0-3a69-4a79-a1c6-9f1b07d1a8a9\httpd-d2ae2aab-9b63-4418-9eda-d39ec8fc1252 -2016-05-24 21:43:50 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 21:43:50 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:43:50 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55644 -2016-05-24 21:43:50 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55644. -2016-05-24 21:43:50 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 21:43:50 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:43:50 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 21:43:50 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 21:43:50 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 21:43:50 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 21:43:50 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 21:43:51 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55663. -2016-05-24 21:43:51 INFO NettyBlockTransferService:59 - Server created on 55663 -2016-05-24 21:43:51 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 21:43:51 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55663 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55663) -2016-05-24 21:43:51 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 21:43:51 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 21:43:51 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 21:43:51 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 21:43:51 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 21:43:51 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55663 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 21:43:51 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 21:43:53 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 21:43:53 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 21:43:53 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 21:43:53 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 21:43:53 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 21:43:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:43:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:43:53 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 21:43:53 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 21:43:53 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 21:43:53 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=175476, maxMem=766075207 -2016-05-24 21:43:53 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 21:43:53 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55663 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:43:53 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:43:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 21:43:53 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 21:43:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:43:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:43:53 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 21:43:53 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 21:43:53 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 21:43:53 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 21:43:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 21:43:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 21:43:53 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 21:43:53 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 21:43:53 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 21:43:53 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 21:43:53 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 21:43:53 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177621, maxMem=766075207 -2016-05-24 21:43:53 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 21:43:53 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:55663 (size: 83.1 KB, free: 730.5 MB) -2016-05-24 21:43:53 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262701, maxMem=766075207 -2016-05-24 21:43:53 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 21:43:53 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:55663 (size: 83.6 KB, free: 730.4 MB) -2016-05-24 21:43:53 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 21:43:53 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 21:43:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 367 ms on localhost (1/2) -2016-05-24 21:43:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 388 ms on localhost (2/2) -2016-05-24 21:43:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 21:43:54 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.403 s -2016-05-24 21:43:54 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.552650 s -2016-05-24 21:43:54 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:55663 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-24 21:43:54 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 21:43:54 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 21:43:54 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 21:43:54 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:43:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:43:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:43:54 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=342308, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.3 MB) -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(2481) called with curMem=346868, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.3 MB) -2016-05-24 21:43:54 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55663 (size: 2.4 KB, free: 730.4 MB) -2016-05-24 21:43:54 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:43:54 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:43:54 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 21:43:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:43:54 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 21:43:54 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:43:54 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-24 21:43:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 21 ms on localhost (1/1) -2016-05-24 21:43:54 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.021 s -2016-05-24 21:43:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 21:43:54 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.039044 s -2016-05-24 21:43:54 WARN LogisticRegressionWithSGD:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-05-24 21:43:54 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 21:43:54 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 21:43:54 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 21:43:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:43:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:43:54 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=349349, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(2294) called with curMem=353597, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 21:43:54 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55663 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:43:54 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:43:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40) -2016-05-24 21:43:54 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 21:43:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:43:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:43:54 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 21:43:54 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 21:43:54 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:43:54 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:43:54 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 21:43:54 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 21:43:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 12 ms on localhost (1/2) -2016-05-24 21:43:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 14 ms on localhost (2/2) -2016-05-24 21:43:54 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.014 s -2016-05-24 21:43:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 21:43:54 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.027266 s -2016-05-24 21:43:54 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 21:43:54 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 21:43:54 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 21:43:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:43:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:43:54 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(4264) called with curMem=355891, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(2318) called with curMem=360155, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-24 21:43:54 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55663 (size: 2.3 KB, free: 730.4 MB) -2016-05-24 21:43:54 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:43:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 21:43:54 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 21:43:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:43:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:43:54 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 21:43:54 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 21:43:54 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:43:54 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:43:54 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 21:43:54 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 21:43:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 20 ms on localhost (1/2) -2016-05-24 21:43:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 20 ms on localhost (2/2) -2016-05-24 21:43:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 21:43:54 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.022 s -2016-05-24 21:43:54 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.040661 s -2016-05-24 21:43:54 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 21:43:54 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=362473, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(388) called with curMem=368065, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 388.0 B, free 730.2 MB) -2016-05-24 21:43:54 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55663 (size: 388.0 B, free: 730.4 MB) -2016-05-24 21:43:54 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 21:43:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:43:54 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:43:54 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:43:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:43:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:43:54 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=368453, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(3492) called with curMem=380253, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:43:54 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55663 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:43:54 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:43:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:43:54 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 21:43:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:43:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:43:54 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 21:43:54 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 21:43:54 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:43:54 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:43:54 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 7968 bytes result sent to driver -2016-05-24 21:43:54 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 7968 bytes result sent to driver -2016-05-24 21:43:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 25 ms on localhost (1/2) -2016-05-24 21:43:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 25 ms on localhost (2/2) -2016-05-24 21:43:54 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.027 s -2016-05-24 21:43:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 21:43:54 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.048123 s -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=383745, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(2249) called with curMem=389337, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 21:43:54 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55663 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:43:54 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 21:43:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:43:54 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:43:54 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:43:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:43:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:43:54 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=391586, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(3493) called with curMem=403386, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:43:54 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55663 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:43:54 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:43:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:43:54 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 21:43:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:43:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:43:54 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 21:43:54 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 21:43:54 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:43:54 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:43:54 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 7968 bytes result sent to driver -2016-05-24 21:43:54 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 7968 bytes result sent to driver -2016-05-24 21:43:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 21 ms on localhost (1/2) -2016-05-24 21:43:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 21 ms on localhost (2/2) -2016-05-24 21:43:54 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 21:43:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 21:43:54 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.047460 s -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=406879, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(3115) called with curMem=412471, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-05-24 21:43:54 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55663 (size: 3.0 KB, free: 730.4 MB) -2016-05-24 21:43:54 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 21:43:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:43:54 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:43:54 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:43:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:43:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:43:54 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=415586, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=427386, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:43:54 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55663 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:43:54 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:43:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:43:54 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 21:43:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:43:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:43:54 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 21:43:54 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 21:43:54 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:43:54 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:43:54 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 7968 bytes result sent to driver -2016-05-24 21:43:54 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 7968 bytes result sent to driver -2016-05-24 21:43:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 20 ms on localhost (1/2) -2016-05-24 21:43:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 21 ms on localhost (2/2) -2016-05-24 21:43:54 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 21:43:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 21:43:54 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.041159 s -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=430880, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(3151) called with curMem=436472, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:43:54 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55663 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:43:54 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 21:43:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:43:54 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:43:54 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:43:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:43:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:43:54 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:43:54 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=439623, maxMem=766075207 -2016-05-24 21:43:54 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(3493) called with curMem=451423, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:55663 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:43:55 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 21:43:55 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 21:43:55 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:43:55 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:43:55 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 7968 bytes result sent to driver -2016-05-24 21:43:55 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 7968 bytes result sent to driver -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 13 ms on localhost (1/2) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 17 ms on localhost (2/2) -2016-05-24 21:43:55 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 21:43:55 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.039245 s -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=454916, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(3152) called with curMem=460508, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:55663 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:55663 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:55663 in memory (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:55663 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 21:43:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:55663 in memory (size: 3.0 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:43:55 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:43:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:43:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:55663 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=400330, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:43:55 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(3493) called with curMem=412130, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:55663 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:55663 in memory (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:43:55 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 21:43:55 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:55663 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55663 in memory (size: 388.0 B, free: 730.4 MB) -2016-05-24 21:43:55 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:43:55 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:43:55 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 7968 bytes result sent to driver -2016-05-24 21:43:55 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 7968 bytes result sent to driver -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:55663 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 22 ms on localhost (1/2) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 20 ms on localhost (2/2) -2016-05-24 21:43:55 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 21:43:55 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.043481 s -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 21:43:55 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:55663 in memory (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:55663 in memory (size: 2.4 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=366345, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:43:55 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=371937, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:55663 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 21:43:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:43:55 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:43:55 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:43:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:43:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=375101, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(3495) called with curMem=386901, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:55663 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:43:55 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 21:43:55 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 21:43:55 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:43:55 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:43:55 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 7968 bytes result sent to driver -2016-05-24 21:43:55 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 7968 bytes result sent to driver -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 14 ms on localhost (1/2) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 14 ms on localhost (2/2) -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 21:43:55 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 21:43:55 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.031645 s -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=390396, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=395988, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:55663 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 21:43:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:43:55 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:43:55 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:43:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:43:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=399152, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=410952, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:55663 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:43:55 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 21:43:55 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 21:43:55 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:43:55 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:43:55 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 7968 bytes result sent to driver -2016-05-24 21:43:55 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 7968 bytes result sent to driver -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 13 ms on localhost (1/2) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 14 ms on localhost (2/2) -2016-05-24 21:43:55 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 21:43:55 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.027086 s -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=414446, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=420038, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:55663 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 21:43:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:43:55 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:43:55 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:43:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:43:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=423202, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=435002, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:55663 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:43:55 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 21:43:55 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 21:43:55 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:43:55 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:43:55 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 7968 bytes result sent to driver -2016-05-24 21:43:55 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 7968 bytes result sent to driver -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 11 ms on localhost (1/2) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 13 ms on localhost (2/2) -2016-05-24 21:43:55 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 21:43:55 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.027943 s -2016-05-24 21:43:55 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 0.6931471805599448, 10057.41015625, 3992.030200294059, 2280.4067582415346, 1609.1071976946598, 1008.6786174352385, 460.5648214704288, 0.0 -2016-05-24 21:43:55 WARN LogisticRegressionWithSGD:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-05-24 21:43:55 INFO SparkContext:59 - Starting job: collectAsMap at MulticlassMetrics.scala:51 -2016-05-24 21:43:55 INFO DAGScheduler:59 - Registering RDD 29 (map at MulticlassMetrics.scala:48) -2016-05-24 21:43:55 INFO DAGScheduler:59 - Got job 12 (collectAsMap at MulticlassMetrics.scala:51) with 2 output partitions -2016-05-24 21:43:55 INFO DAGScheduler:59 - Final stage: ResultStage 13(collectAsMap at MulticlassMetrics.scala:51) -2016-05-24 21:43:55 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-24 21:43:55 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (MapPartitionsRDD[29] at map at MulticlassMetrics.scala:48), which has no missing parents -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(11408) called with curMem=438496, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 11.1 KB, free 730.2 MB) -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(6171) called with curMem=449904, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 6.0 KB, free 730.2 MB) -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:55663 (size: 6.0 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (MapPartitionsRDD[29] at map at MulticlassMetrics.scala:48) -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 21:43:55 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 21:43:55 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 21:43:55 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:43:55 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:43:55 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2254 bytes result sent to driver -2016-05-24 21:43:55 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2254 bytes result sent to driver -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 62 ms on localhost (1/2) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 65 ms on localhost (2/2) -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 21:43:55 INFO DAGScheduler:59 - ShuffleMapStage 12 (map at MulticlassMetrics.scala:48) finished in 0.066 s -2016-05-24 21:43:55 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 21:43:55 INFO DAGScheduler:59 - running: Set() -2016-05-24 21:43:55 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-05-24 21:43:55 INFO DAGScheduler:59 - failed: Set() -2016-05-24 21:43:55 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting ResultStage 13 (ShuffledRDD[30] at reduceByKey at MulticlassMetrics.scala:50), which is now runnable -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=456075, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(1412) called with curMem=458411, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 1412.0 B, free 730.1 MB) -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:55663 (size: 1412.0 B, free: 730.4 MB) -2016-05-24 21:43:55 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (ShuffledRDD[30] at reduceByKey at MulticlassMetrics.scala:50) -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 21:43:55 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 21:43:55 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 21:43:55 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-24 21:43:55 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 21:43:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-24 21:43:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-24 21:43:55 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 1161 bytes result sent to driver -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 31 ms on localhost (1/2) -2016-05-24 21:43:55 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 1389 bytes result sent to driver -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 40 ms on localhost (2/2) -2016-05-24 21:43:55 INFO DAGScheduler:59 - ResultStage 13 (collectAsMap at MulticlassMetrics.scala:51) finished in 0.040 s -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 21:43:55 INFO DAGScheduler:59 - Job 12 finished: collectAsMap at MulticlassMetrics.scala:51, took 0.145919 s -2016-05-24 21:43:55 INFO SparkContext:59 - Starting job: countByValue at MulticlassMetrics.scala:45 -2016-05-24 21:43:55 INFO DAGScheduler:59 - Registering RDD 33 (countByValue at MulticlassMetrics.scala:45) -2016-05-24 21:43:55 INFO DAGScheduler:59 - Got job 13 (countByValue at MulticlassMetrics.scala:45) with 2 output partitions -2016-05-24 21:43:55 INFO DAGScheduler:59 - Final stage: ResultStage 15(countByValue at MulticlassMetrics.scala:45) -2016-05-24 21:43:55 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 14) -2016-05-24 21:43:55 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 14) -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting ShuffleMapStage 14 (MapPartitionsRDD[33] at countByValue at MulticlassMetrics.scala:45), which has no missing parents -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(11936) called with curMem=459823, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 11.7 KB, free 730.1 MB) -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(6356) called with curMem=471759, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 6.2 KB, free 730.1 MB) -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:55663 (size: 6.2 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 14 (MapPartitionsRDD[33] at countByValue at MulticlassMetrics.scala:45) -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 21:43:55 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 21:43:55 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 21:43:55 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:43:55 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:43:55 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2254 bytes result sent to driver -2016-05-24 21:43:55 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2254 bytes result sent to driver -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 24 ms on localhost (1/2) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 27 ms on localhost (2/2) -2016-05-24 21:43:55 INFO DAGScheduler:59 - ShuffleMapStage 14 (countByValue at MulticlassMetrics.scala:45) finished in 0.028 s -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 21:43:55 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 21:43:55 INFO DAGScheduler:59 - running: Set() -2016-05-24 21:43:55 INFO DAGScheduler:59 - waiting: Set(ResultStage 15) -2016-05-24 21:43:55 INFO DAGScheduler:59 - failed: Set() -2016-05-24 21:43:55 INFO DAGScheduler:59 - Missing parents for ResultStage 15: List() -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting ResultStage 15 (ShuffledRDD[34] at countByValue at MulticlassMetrics.scala:45), which is now runnable -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=478115, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(1371) called with curMem=480451, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 1371.0 B, free 730.1 MB) -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:55663 (size: 1371.0 B, free: 730.4 MB) -2016-05-24 21:43:55 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (ShuffledRDD[34] at countByValue at MulticlassMetrics.scala:45) -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 21:43:55 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 21:43:55 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 21:43:55 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-24 21:43:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 21:43:55 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 1161 bytes result sent to driver -2016-05-24 21:43:55 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 21:43:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 7 ms on localhost (1/2) -2016-05-24 21:43:55 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 1394 bytes result sent to driver -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 14 ms on localhost (2/2) -2016-05-24 21:43:55 INFO DAGScheduler:59 - ResultStage 15 (countByValue at MulticlassMetrics.scala:45) finished in 0.014 s -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 21:43:55 INFO DAGScheduler:59 - Job 13 finished: countByValue at MulticlassMetrics.scala:45, took 0.067822 s -2016-05-24 21:43:55 INFO SparkContext:59 - Starting job: count at LogisticRegression3Learning.scala:39 -2016-05-24 21:43:55 INFO DAGScheduler:59 - Got job 14 (count at LogisticRegression3Learning.scala:39) with 2 output partitions -2016-05-24 21:43:55 INFO DAGScheduler:59 - Final stage: ResultStage 16(count at LogisticRegression3Learning.scala:39) -2016-05-24 21:43:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:43:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[7] at randomSplit at LogisticRegression3Learning.scala:25), which has no missing parents -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=481822, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 4.0 KB, free 730.1 MB) -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(2257) called with curMem=485934, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:55663 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[7] at randomSplit at LogisticRegression3Learning.scala:25) -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:43:55 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 21:43:55 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 21:43:55 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:43:55 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:43:55 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2082 bytes result sent to driver -2016-05-24 21:43:55 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2082 bytes result sent to driver -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 13 ms on localhost (1/2) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 15 ms on localhost (2/2) -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 21:43:55 INFO DAGScheduler:59 - ResultStage 16 (count at LogisticRegression3Learning.scala:39) finished in 0.018 s -2016-05-24 21:43:55 INFO DAGScheduler:59 - Job 14 finished: count at LogisticRegression3Learning.scala:39, took 0.032648 s -2016-05-24 21:43:55 INFO SparkContext:59 - Starting job: count at LogisticRegression3Learning.scala:40 -2016-05-24 21:43:55 INFO DAGScheduler:59 - Got job 15 (count at LogisticRegression3Learning.scala:40) with 2 output partitions -2016-05-24 21:43:55 INFO DAGScheduler:59 - Final stage: ResultStage 17(count at LogisticRegression3Learning.scala:40) -2016-05-24 21:43:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:43:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[8] at randomSplit at LogisticRegression3Learning.scala:25), which has no missing parents -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=488191, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 4.0 KB, free 730.1 MB) -2016-05-24 21:43:55 INFO MemoryStore:59 - ensureFreeSpace(2261) called with curMem=492303, maxMem=766075207 -2016-05-24 21:43:55 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-24 21:43:55 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:55663 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:43:55 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:43:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[8] at randomSplit at LogisticRegression3Learning.scala:25) -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:43:55 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 21:43:55 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 21:43:55 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:43:55 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:43:55 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2082 bytes result sent to driver -2016-05-24 21:43:55 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2082 bytes result sent to driver -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 9 ms on localhost (1/2) -2016-05-24 21:43:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 10 ms on localhost (2/2) -2016-05-24 21:43:55 INFO DAGScheduler:59 - ResultStage 17 (count at LogisticRegression3Learning.scala:40) finished in 0.011 s -2016-05-24 21:43:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 21:43:55 INFO DAGScheduler:59 - Job 15 finished: count at LogisticRegression3Learning.scala:40, took 0.021952 s -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 21:43:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 21:43:55 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 21:43:55 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 21:43:55 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 21:43:55 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 21:43:55 INFO BlockManager:59 - BlockManager stopped -2016-05-24 21:43:55 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 21:43:56 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 21:43:56 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 21:43:56 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 21:43:56 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 21:43:56 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 21:43:56 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-df012fd0-3a69-4a79-a1c6-9f1b07d1a8a9 -2016-05-24 21:45:09 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 21:45:10 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 21:45:10 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 21:45:10 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 21:45:10 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 21:45:11 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 21:45:11 INFO Remoting:74 - Starting remoting -2016-05-24 21:45:11 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:55684] -2016-05-24 21:45:11 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55684. -2016-05-24 21:45:11 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 21:45:11 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 21:45:11 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-ff109839-426e-40c1-b36f-beb83cafca0e -2016-05-24 21:45:11 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 21:45:11 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-65ced127-6458-42e8-9979-4a50d92414f1\httpd-32b804ef-8f68-43c9-8484-57948670b08f -2016-05-24 21:45:11 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 21:45:11 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:45:12 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55685 -2016-05-24 21:45:12 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55685. -2016-05-24 21:45:12 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 21:45:12 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:45:12 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 21:45:12 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 21:45:12 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 21:45:12 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 21:45:12 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 21:45:12 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55704. -2016-05-24 21:45:12 INFO NettyBlockTransferService:59 - Server created on 55704 -2016-05-24 21:45:12 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 21:45:12 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55704 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55704) -2016-05-24 21:45:12 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 21:45:13 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 21:45:13 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 21:45:13 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 21:45:13 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 21:45:13 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55704 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 21:45:13 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 21:45:14 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 21:45:14 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 21:45:14 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 21:45:14 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 21:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 21:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:45:14 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 21:45:14 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 21:45:14 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 21:45:15 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=175476, maxMem=766075207 -2016-05-24 21:45:15 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 21:45:15 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55704 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:45:15 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:45:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 21:45:15 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 21:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:45:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:45:15 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 21:45:15 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 21:45:15 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 21:45:15 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 21:45:15 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 21:45:15 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 21:45:15 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 21:45:15 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 21:45:15 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 21:45:15 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 21:45:15 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 21:45:15 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177621, maxMem=766075207 -2016-05-24 21:45:15 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 21:45:15 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:55704 (size: 83.1 KB, free: 730.5 MB) -2016-05-24 21:45:15 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262701, maxMem=766075207 -2016-05-24 21:45:15 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 21:45:15 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:55704 (size: 83.6 KB, free: 730.4 MB) -2016-05-24 21:45:15 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 21:45:15 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 21:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 395 ms on localhost (1/2) -2016-05-24 21:45:15 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.417 s -2016-05-24 21:45:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 380 ms on localhost (2/2) -2016-05-24 21:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 21:45:15 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.563845 s -2016-05-24 21:45:15 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 21:45:15 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 21:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:45:15 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 21:45:15 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=348349, maxMem=766075207 -2016-05-24 21:45:15 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-24 21:45:15 INFO MemoryStore:59 - ensureFreeSpace(2481) called with curMem=352909, maxMem=766075207 -2016-05-24 21:45:15 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-24 21:45:15 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55704 (size: 2.4 KB, free: 730.4 MB) -2016-05-24 21:45:15 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:45:15 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:45:15 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 21:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:45:15 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 21:45:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:45:15 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-24 21:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 27 ms on localhost (1/1) -2016-05-24 21:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 21:45:15 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.027 s -2016-05-24 21:45:15 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.049016 s -2016-05-24 21:45:15 WARN LogisticRegressionWithSGD:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-05-24 21:45:15 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 21:45:15 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 21:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 21:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:45:15 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 21:45:15 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=355390, maxMem=766075207 -2016-05-24 21:45:15 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-24 21:45:15 INFO MemoryStore:59 - ensureFreeSpace(2294) called with curMem=359638, maxMem=766075207 -2016-05-24 21:45:15 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 21:45:15 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55704 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:45:15 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:45:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40) -2016-05-24 21:45:15 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 21:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:45:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:45:15 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 21:45:15 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 21:45:15 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:45:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:45:15 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 21:45:15 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 21:45:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 16 ms on localhost (1/2) -2016-05-24 21:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 18 ms on localhost (2/2) -2016-05-24 21:45:15 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.019 s -2016-05-24 21:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 21:45:15 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.035262 s -2016-05-24 21:45:15 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 21:45:15 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 21:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 21:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:45:15 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 21:45:15 INFO MemoryStore:59 - ensureFreeSpace(4264) called with curMem=361932, maxMem=766075207 -2016-05-24 21:45:15 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-24 21:45:15 INFO MemoryStore:59 - ensureFreeSpace(2318) called with curMem=366196, maxMem=766075207 -2016-05-24 21:45:15 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-24 21:45:15 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55704 (size: 2.3 KB, free: 730.4 MB) -2016-05-24 21:45:15 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:45:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 21:45:15 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 21:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:45:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:45:15 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 21:45:15 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 21:45:15 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:45:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:45:15 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 21:45:15 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 21:45:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 16 ms on localhost (1/2) -2016-05-24 21:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 17 ms on localhost (2/2) -2016-05-24 21:45:15 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.017 s -2016-05-24 21:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 21:45:15 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.037712 s -2016-05-24 21:45:16 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 21:45:16 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=368514, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(388) called with curMem=374106, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 388.0 B, free 730.2 MB) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55704 (size: 388.0 B, free: 730.4 MB) -2016-05-24 21:45:16 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 21:45:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:45:16 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:45:16 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=374494, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(3493) called with curMem=386294, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55704 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:45:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:45:16 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 21:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:45:16 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 21:45:16 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 21:45:16 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:45:16 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:45:16 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 7968 bytes result sent to driver -2016-05-24 21:45:16 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 7968 bytes result sent to driver -2016-05-24 21:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 25 ms on localhost (1/2) -2016-05-24 21:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 24 ms on localhost (2/2) -2016-05-24 21:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 21:45:16 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 21:45:16 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.055003 s -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=389787, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(2249) called with curMem=395379, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55704 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 21:45:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:45:16 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:45:16 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=397628, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=409428, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55704 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:45:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:45:16 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 21:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:45:16 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 21:45:16 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 21:45:16 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:45:16 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:45:16 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 7968 bytes result sent to driver -2016-05-24 21:45:16 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 7968 bytes result sent to driver -2016-05-24 21:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 21 ms on localhost (1/2) -2016-05-24 21:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 21 ms on localhost (2/2) -2016-05-24 21:45:16 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 21:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 21:45:16 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.045554 s -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=412922, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(3115) called with curMem=418514, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55704 (size: 3.0 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 21:45:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:45:16 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:45:16 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=421629, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(3495) called with curMem=433429, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55704 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:45:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:45:16 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 21:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:45:16 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 21:45:16 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 21:45:16 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:45:16 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 7968 bytes result sent to driver -2016-05-24 21:45:16 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:45:16 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 7968 bytes result sent to driver -2016-05-24 21:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 24 ms on localhost (1/2) -2016-05-24 21:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 27 ms on localhost (2/2) -2016-05-24 21:45:16 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 21:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 21:45:16 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.051304 s -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=436924, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(3151) called with curMem=442516, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55704 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 21:45:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:45:16 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:45:16 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=445667, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=457467, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:55704 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:45:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:45:16 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 21:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:45:16 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 21:45:16 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:45:16 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 7968 bytes result sent to driver -2016-05-24 21:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 21 ms on localhost (1/2) -2016-05-24 21:45:16 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:55704 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 21:45:16 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:55704 in memory (size: 3.0 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 7968 bytes result sent to driver -2016-05-24 21:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 51 ms on localhost (2/2) -2016-05-24 21:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 21:45:16 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.054 s -2016-05-24 21:45:16 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.127177 s -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:55704 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:55704 in memory (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:55704 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=398531, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55704 in memory (size: 388.0 B, free: 730.4 MB) -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(3152) called with curMem=403735, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:55704 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:55704 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:55704 in memory (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:55704 in memory (size: 2.4 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 21:45:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:45:16 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:55704 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:45:16 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=375089, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(3493) called with curMem=386889, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:55704 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:45:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:45:16 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 21:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:45:16 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 21:45:16 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 21:45:16 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:45:16 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:45:16 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 7968 bytes result sent to driver -2016-05-24 21:45:16 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 7968 bytes result sent to driver -2016-05-24 21:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 18 ms on localhost (1/2) -2016-05-24 21:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 19 ms on localhost (2/2) -2016-05-24 21:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 21:45:16 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 21:45:16 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.039543 s -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=390382, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=395974, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:55704 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 21:45:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:45:16 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:45:16 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=399138, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=410938, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:55704 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:45:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:45:16 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 21:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:45:16 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 21:45:16 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 21:45:16 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:45:16 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:45:16 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 7968 bytes result sent to driver -2016-05-24 21:45:16 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 7968 bytes result sent to driver -2016-05-24 21:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 12 ms on localhost (1/2) -2016-05-24 21:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 12 ms on localhost (2/2) -2016-05-24 21:45:16 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 21:45:16 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.026139 s -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=414432, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=420024, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:55704 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 21:45:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:45:16 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:45:16 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=423188, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=434988, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:55704 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:45:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:45:16 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 21:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:45:16 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 21:45:16 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 21:45:16 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:45:16 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:45:16 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 7968 bytes result sent to driver -2016-05-24 21:45:16 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 7968 bytes result sent to driver -2016-05-24 21:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 12 ms on localhost (1/2) -2016-05-24 21:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 12 ms on localhost (2/2) -2016-05-24 21:45:16 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 21:45:16 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.026837 s -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=438482, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=444074, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:55704 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 21:45:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:45:16 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:45:16 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=447238, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=459038, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:55704 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:45:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:45:16 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 21:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:45:16 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 21:45:16 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 21:45:16 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:45:16 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:45:16 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 7968 bytes result sent to driver -2016-05-24 21:45:16 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 7968 bytes result sent to driver -2016-05-24 21:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 11 ms on localhost (1/2) -2016-05-24 21:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 11 ms on localhost (2/2) -2016-05-24 21:45:16 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 21:45:16 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.028547 s -2016-05-24 21:45:16 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 0.6931471805599448, 10057.41015625, 3992.030200294059, 2280.4067582415346, 1609.1071976946598, 1008.6786174352385, 460.5648214704288, 0.0 -2016-05-24 21:45:16 WARN LogisticRegressionWithSGD:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-05-24 21:45:16 INFO SparkContext:59 - Starting job: collectAsMap at MulticlassMetrics.scala:51 -2016-05-24 21:45:16 INFO DAGScheduler:59 - Registering RDD 29 (map at MulticlassMetrics.scala:48) -2016-05-24 21:45:16 INFO DAGScheduler:59 - Got job 12 (collectAsMap at MulticlassMetrics.scala:51) with 2 output partitions -2016-05-24 21:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 13(collectAsMap at MulticlassMetrics.scala:51) -2016-05-24 21:45:16 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-24 21:45:16 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-05-24 21:45:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (MapPartitionsRDD[29] at map at MulticlassMetrics.scala:48), which has no missing parents -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(11408) called with curMem=462532, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 11.1 KB, free 730.1 MB) -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(6171) called with curMem=473940, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 6.0 KB, free 730.1 MB) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:55704 (size: 6.0 KB, free: 730.4 MB) -2016-05-24 21:45:16 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:45:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (MapPartitionsRDD[29] at map at MulticlassMetrics.scala:48) -2016-05-24 21:45:16 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 21:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 21:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 21:45:16 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 21:45:16 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 21:45:16 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:45:16 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:45:16 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2254 bytes result sent to driver -2016-05-24 21:45:16 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2254 bytes result sent to driver -2016-05-24 21:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 54 ms on localhost (1/2) -2016-05-24 21:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 52 ms on localhost (2/2) -2016-05-24 21:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 21:45:16 INFO DAGScheduler:59 - ShuffleMapStage 12 (map at MulticlassMetrics.scala:48) finished in 0.055 s -2016-05-24 21:45:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 21:45:16 INFO DAGScheduler:59 - running: Set() -2016-05-24 21:45:16 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-05-24 21:45:16 INFO DAGScheduler:59 - failed: Set() -2016-05-24 21:45:16 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-05-24 21:45:16 INFO DAGScheduler:59 - Submitting ResultStage 13 (ShuffledRDD[30] at reduceByKey at MulticlassMetrics.scala:50), which is now runnable -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=480111, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 21:45:16 INFO MemoryStore:59 - ensureFreeSpace(1412) called with curMem=482447, maxMem=766075207 -2016-05-24 21:45:16 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 1412.0 B, free 730.1 MB) -2016-05-24 21:45:16 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:55704 (size: 1412.0 B, free: 730.4 MB) -2016-05-24 21:45:16 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:45:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (ShuffledRDD[30] at reduceByKey at MulticlassMetrics.scala:50) -2016-05-24 21:45:16 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 21:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 21:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 21:45:16 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 21:45:16 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 21:45:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 21:45:16 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-24 21:45:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-24 21:45:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-24 21:45:16 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 1161 bytes result sent to driver -2016-05-24 21:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 27 ms on localhost (1/2) -2016-05-24 21:45:16 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 1389 bytes result sent to driver -2016-05-24 21:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 30 ms on localhost (2/2) -2016-05-24 21:45:16 INFO DAGScheduler:59 - ResultStage 13 (collectAsMap at MulticlassMetrics.scala:51) finished in 0.031 s -2016-05-24 21:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 21:45:16 INFO DAGScheduler:59 - Job 12 finished: collectAsMap at MulticlassMetrics.scala:51, took 0.121542 s -2016-05-24 21:45:17 INFO SparkContext:59 - Starting job: countByValue at MulticlassMetrics.scala:45 -2016-05-24 21:45:17 INFO DAGScheduler:59 - Registering RDD 33 (countByValue at MulticlassMetrics.scala:45) -2016-05-24 21:45:17 INFO DAGScheduler:59 - Got job 13 (countByValue at MulticlassMetrics.scala:45) with 2 output partitions -2016-05-24 21:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 15(countByValue at MulticlassMetrics.scala:45) -2016-05-24 21:45:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 14) -2016-05-24 21:45:17 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 14) -2016-05-24 21:45:17 INFO DAGScheduler:59 - Submitting ShuffleMapStage 14 (MapPartitionsRDD[33] at countByValue at MulticlassMetrics.scala:45), which has no missing parents -2016-05-24 21:45:17 INFO MemoryStore:59 - ensureFreeSpace(11936) called with curMem=483859, maxMem=766075207 -2016-05-24 21:45:17 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 11.7 KB, free 730.1 MB) -2016-05-24 21:45:17 INFO MemoryStore:59 - ensureFreeSpace(6355) called with curMem=495795, maxMem=766075207 -2016-05-24 21:45:17 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 6.2 KB, free 730.1 MB) -2016-05-24 21:45:17 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:55704 (size: 6.2 KB, free: 730.4 MB) -2016-05-24 21:45:17 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:45:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 14 (MapPartitionsRDD[33] at countByValue at MulticlassMetrics.scala:45) -2016-05-24 21:45:17 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 21:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 21:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 21:45:17 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 21:45:17 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 21:45:17 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:45:17 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:45:17 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2254 bytes result sent to driver -2016-05-24 21:45:17 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2254 bytes result sent to driver -2016-05-24 21:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 23 ms on localhost (1/2) -2016-05-24 21:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 24 ms on localhost (2/2) -2016-05-24 21:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 21:45:17 INFO DAGScheduler:59 - ShuffleMapStage 14 (countByValue at MulticlassMetrics.scala:45) finished in 0.025 s -2016-05-24 21:45:17 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 21:45:17 INFO DAGScheduler:59 - running: Set() -2016-05-24 21:45:17 INFO DAGScheduler:59 - waiting: Set(ResultStage 15) -2016-05-24 21:45:17 INFO DAGScheduler:59 - failed: Set() -2016-05-24 21:45:17 INFO DAGScheduler:59 - Missing parents for ResultStage 15: List() -2016-05-24 21:45:17 INFO DAGScheduler:59 - Submitting ResultStage 15 (ShuffledRDD[34] at countByValue at MulticlassMetrics.scala:45), which is now runnable -2016-05-24 21:45:17 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=502150, maxMem=766075207 -2016-05-24 21:45:17 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 21:45:17 INFO MemoryStore:59 - ensureFreeSpace(1371) called with curMem=504486, maxMem=766075207 -2016-05-24 21:45:17 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 1371.0 B, free 730.1 MB) -2016-05-24 21:45:17 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:55704 (size: 1371.0 B, free: 730.4 MB) -2016-05-24 21:45:17 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:45:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (ShuffledRDD[34] at countByValue at MulticlassMetrics.scala:45) -2016-05-24 21:45:17 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 21:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 21:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 21:45:17 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 21:45:17 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 21:45:17 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-24 21:45:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 21:45:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 21:45:17 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 1161 bytes result sent to driver -2016-05-24 21:45:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 21:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 6 ms on localhost (1/2) -2016-05-24 21:45:17 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 1394 bytes result sent to driver -2016-05-24 21:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 13 ms on localhost (2/2) -2016-05-24 21:45:17 INFO DAGScheduler:59 - ResultStage 15 (countByValue at MulticlassMetrics.scala:45) finished in 0.013 s -2016-05-24 21:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 21:45:17 INFO DAGScheduler:59 - Job 13 finished: countByValue at MulticlassMetrics.scala:45, took 0.064082 s -2016-05-24 21:45:17 INFO SparkContext:59 - Starting job: count at LogisticRegression3Learning.scala:39 -2016-05-24 21:45:17 INFO DAGScheduler:59 - Got job 14 (count at LogisticRegression3Learning.scala:39) with 2 output partitions -2016-05-24 21:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 16(count at LogisticRegression3Learning.scala:39) -2016-05-24 21:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:45:17 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-24 21:45:17 INFO MemoryStore:59 - ensureFreeSpace(3808) called with curMem=505857, maxMem=766075207 -2016-05-24 21:45:17 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 3.7 KB, free 730.1 MB) -2016-05-24 21:45:17 INFO MemoryStore:59 - ensureFreeSpace(2133) called with curMem=509665, maxMem=766075207 -2016-05-24 21:45:17 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.1 MB) -2016-05-24 21:45:17 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:55704 (size: 2.1 KB, free: 730.4 MB) -2016-05-24 21:45:17 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:45:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-24 21:45:17 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 21:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:45:17 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 21:45:17 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 21:45:17 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:45:17 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2082 bytes result sent to driver -2016-05-24 21:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 12 ms on localhost (1/2) -2016-05-24 21:45:17 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:45:17 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2082 bytes result sent to driver -2016-05-24 21:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 18 ms on localhost (2/2) -2016-05-24 21:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 21:45:17 INFO DAGScheduler:59 - ResultStage 16 (count at LogisticRegression3Learning.scala:39) finished in 0.019 s -2016-05-24 21:45:17 INFO DAGScheduler:59 - Job 14 finished: count at LogisticRegression3Learning.scala:39, took 0.033276 s -2016-05-24 21:45:17 INFO SparkContext:59 - Starting job: count at LogisticRegression3Learning.scala:40 -2016-05-24 21:45:17 INFO DAGScheduler:59 - Got job 15 (count at LogisticRegression3Learning.scala:40) with 2 output partitions -2016-05-24 21:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 17(count at LogisticRegression3Learning.scala:40) -2016-05-24 21:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:45:17 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[7] at randomSplit at LogisticRegression3Learning.scala:25), which has no missing parents -2016-05-24 21:45:17 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=511798, maxMem=766075207 -2016-05-24 21:45:17 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 4.0 KB, free 730.1 MB) -2016-05-24 21:45:17 INFO MemoryStore:59 - ensureFreeSpace(2257) called with curMem=515910, maxMem=766075207 -2016-05-24 21:45:17 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-24 21:45:17 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:55704 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:45:17 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:45:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[7] at randomSplit at LogisticRegression3Learning.scala:25) -2016-05-24 21:45:17 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 21:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:45:17 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 21:45:17 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 21:45:17 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:45:17 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:45:17 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2082 bytes result sent to driver -2016-05-24 21:45:17 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2082 bytes result sent to driver -2016-05-24 21:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 9 ms on localhost (1/2) -2016-05-24 21:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 11 ms on localhost (2/2) -2016-05-24 21:45:17 INFO DAGScheduler:59 - ResultStage 17 (count at LogisticRegression3Learning.scala:40) finished in 0.011 s -2016-05-24 21:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 21:45:17 INFO DAGScheduler:59 - Job 15 finished: count at LogisticRegression3Learning.scala:40, took 0.024457 s -2016-05-24 21:45:17 INFO SparkContext:59 - Starting job: count at LogisticRegression3Learning.scala:41 -2016-05-24 21:45:17 INFO DAGScheduler:59 - Got job 16 (count at LogisticRegression3Learning.scala:41) with 2 output partitions -2016-05-24 21:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 18(count at LogisticRegression3Learning.scala:41) -2016-05-24 21:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:45:17 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[8] at randomSplit at LogisticRegression3Learning.scala:25), which has no missing parents -2016-05-24 21:45:17 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=518167, maxMem=766075207 -2016-05-24 21:45:17 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 4.0 KB, free 730.1 MB) -2016-05-24 21:45:17 INFO MemoryStore:59 - ensureFreeSpace(2261) called with curMem=522279, maxMem=766075207 -2016-05-24 21:45:17 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-24 21:45:17 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:55704 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:45:17 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:45:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[8] at randomSplit at LogisticRegression3Learning.scala:25) -2016-05-24 21:45:17 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 21:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:45:17 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 21:45:17 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 21:45:17 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:45:17 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:45:17 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2082 bytes result sent to driver -2016-05-24 21:45:17 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2082 bytes result sent to driver -2016-05-24 21:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 12 ms on localhost (1/2) -2016-05-24 21:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 12 ms on localhost (2/2) -2016-05-24 21:45:17 INFO DAGScheduler:59 - ResultStage 18 (count at LogisticRegression3Learning.scala:41) finished in 0.014 s -2016-05-24 21:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 21:45:17 INFO DAGScheduler:59 - Job 16 finished: count at LogisticRegression3Learning.scala:41, took 0.027803 s -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 21:45:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 21:45:17 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 21:45:17 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 21:45:17 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 21:45:17 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 21:45:17 INFO BlockManager:59 - BlockManager stopped -2016-05-24 21:45:17 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 21:45:17 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 21:45:17 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 21:45:17 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 21:45:17 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-65ced127-6458-42e8-9979-4a50d92414f1 -2016-05-24 21:45:17 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 21:46:54 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 21:46:55 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 21:46:55 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 21:46:55 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 21:46:55 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 21:46:56 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 21:46:56 INFO Remoting:74 - Starting remoting -2016-05-24 21:46:56 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:55728] -2016-05-24 21:46:56 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55728. -2016-05-24 21:46:56 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 21:46:56 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 21:46:56 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-e5e23d27-28fe-4ee1-ac88-a00693495509 -2016-05-24 21:46:56 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 21:46:56 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-cbb3f851-981b-48d2-a896-12a2990a6eea\httpd-b23642e2-dbef-404c-815e-59d956a00108 -2016-05-24 21:46:56 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 21:46:57 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:46:57 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55729 -2016-05-24 21:46:57 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55729. -2016-05-24 21:46:57 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 21:46:57 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:46:57 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 21:46:57 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 21:46:57 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 21:46:57 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 21:46:57 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 21:46:57 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55748. -2016-05-24 21:46:57 INFO NettyBlockTransferService:59 - Server created on 55748 -2016-05-24 21:46:57 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 21:46:57 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55748 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55748) -2016-05-24 21:46:57 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 21:46:58 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 21:46:58 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 21:46:58 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 21:46:58 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 21:46:58 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55748 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 21:46:58 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 21:46:59 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 21:47:00 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 21:47:00 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 21:47:00 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 21:47:00 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 21:47:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:47:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:47:00 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 21:47:00 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 21:47:00 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 21:47:00 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=175476, maxMem=766075207 -2016-05-24 21:47:00 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 21:47:00 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55748 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:47:00 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:47:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 21:47:00 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 21:47:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:47:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:47:00 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 21:47:00 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 21:47:00 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 21:47:00 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 21:47:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 21:47:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 21:47:00 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 21:47:00 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 21:47:00 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 21:47:00 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 21:47:00 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 21:47:00 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177621, maxMem=766075207 -2016-05-24 21:47:00 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 21:47:00 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262701, maxMem=766075207 -2016-05-24 21:47:00 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 21:47:00 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:55748 (size: 83.1 KB, free: 730.5 MB) -2016-05-24 21:47:00 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:55748 (size: 83.6 KB, free: 730.4 MB) -2016-05-24 21:47:00 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 21:47:00 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 21:47:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 389 ms on localhost (1/2) -2016-05-24 21:47:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 373 ms on localhost (2/2) -2016-05-24 21:47:00 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.412 s -2016-05-24 21:47:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 21:47:00 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.572047 s -2016-05-24 21:47:00 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 21:47:00 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 21:47:00 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:47:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:47:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:47:00 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 21:47:00 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=348349, maxMem=766075207 -2016-05-24 21:47:00 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-24 21:47:00 INFO MemoryStore:59 - ensureFreeSpace(2481) called with curMem=352909, maxMem=766075207 -2016-05-24 21:47:00 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-24 21:47:00 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55748 (size: 2.4 KB, free: 730.4 MB) -2016-05-24 21:47:00 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:47:00 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:47:00 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 21:47:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:47:00 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 21:47:00 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:47:00 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-24 21:47:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 21 ms on localhost (1/1) -2016-05-24 21:47:00 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.022 s -2016-05-24 21:47:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 21:47:00 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.041415 s -2016-05-24 21:47:00 WARN LogisticRegressionWithSGD:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-05-24 21:47:00 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 21:47:00 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 21:47:00 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 21:47:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:47:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:47:00 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 21:47:00 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=355390, maxMem=766075207 -2016-05-24 21:47:00 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-24 21:47:00 INFO MemoryStore:59 - ensureFreeSpace(2294) called with curMem=359638, maxMem=766075207 -2016-05-24 21:47:00 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 21:47:00 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55748 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:47:00 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:47:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40) -2016-05-24 21:47:00 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 21:47:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:47:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:47:00 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 21:47:00 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 21:47:00 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:47:00 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:47:00 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 21:47:00 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 21:47:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 12 ms on localhost (1/2) -2016-05-24 21:47:00 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.013 s -2016-05-24 21:47:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 12 ms on localhost (2/2) -2016-05-24 21:47:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 21:47:00 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.027362 s -2016-05-24 21:47:00 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 21:47:00 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 21:47:00 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 21:47:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:47:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:47:00 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 21:47:00 INFO MemoryStore:59 - ensureFreeSpace(4264) called with curMem=361932, maxMem=766075207 -2016-05-24 21:47:00 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-24 21:47:00 INFO MemoryStore:59 - ensureFreeSpace(2318) called with curMem=366196, maxMem=766075207 -2016-05-24 21:47:00 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-24 21:47:00 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55748 (size: 2.3 KB, free: 730.4 MB) -2016-05-24 21:47:00 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:47:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 21:47:00 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 21:47:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:47:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:47:00 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 21:47:00 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 21:47:00 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:47:00 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 21:47:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 14 ms on localhost (1/2) -2016-05-24 21:47:00 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:47:00 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 21:47:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 19 ms on localhost (2/2) -2016-05-24 21:47:00 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.021 s -2016-05-24 21:47:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 21:47:00 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.040931 s -2016-05-24 21:47:01 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 21:47:01 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=368514, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(388) called with curMem=374106, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 388.0 B, free 730.2 MB) -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55748 (size: 388.0 B, free: 730.4 MB) -2016-05-24 21:47:01 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 21:47:01 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:47:01 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:47:01 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:47:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:47:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:47:01 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=374494, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(3492) called with curMem=386294, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55748 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:47:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:47:01 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 21:47:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:47:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:47:01 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 21:47:01 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 21:47:01 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:47:01 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:47:01 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 7968 bytes result sent to driver -2016-05-24 21:47:01 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 7968 bytes result sent to driver -2016-05-24 21:47:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 24 ms on localhost (1/2) -2016-05-24 21:47:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 29 ms on localhost (2/2) -2016-05-24 21:47:01 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.029 s -2016-05-24 21:47:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 21:47:01 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.057813 s -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=389786, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(2249) called with curMem=395378, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55748 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 21:47:01 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:47:01 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:47:01 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:47:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:47:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:47:01 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=397627, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=409427, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55748 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:47:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:47:01 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 21:47:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:47:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:47:01 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 21:47:01 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 21:47:01 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:47:01 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:47:01 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 7968 bytes result sent to driver -2016-05-24 21:47:01 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 7968 bytes result sent to driver -2016-05-24 21:47:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 25 ms on localhost (1/2) -2016-05-24 21:47:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 27 ms on localhost (2/2) -2016-05-24 21:47:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 21:47:01 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.032 s -2016-05-24 21:47:01 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.059737 s -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=412921, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(3115) called with curMem=418513, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55748 (size: 3.0 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 21:47:01 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:47:01 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:47:01 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:47:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:47:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:47:01 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=421628, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=433428, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55748 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:47:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:47:01 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 21:47:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:47:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:47:01 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 21:47:01 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 21:47:01 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:47:01 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:47:01 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 7968 bytes result sent to driver -2016-05-24 21:47:01 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 7968 bytes result sent to driver -2016-05-24 21:47:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 22 ms on localhost (1/2) -2016-05-24 21:47:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 23 ms on localhost (2/2) -2016-05-24 21:47:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 21:47:01 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.027 s -2016-05-24 21:47:01 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.053155 s -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=436922, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(3151) called with curMem=442514, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55748 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 21:47:01 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:47:01 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:47:01 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:47:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:47:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:47:01 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=445665, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=457465, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:55748 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:47:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:47:01 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 21:47:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:47:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:47:01 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 21:47:01 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 21:47:01 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:47:01 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:47:01 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 7968 bytes result sent to driver -2016-05-24 21:47:01 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 7968 bytes result sent to driver -2016-05-24 21:47:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 16 ms on localhost (1/2) -2016-05-24 21:47:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 15 ms on localhost (2/2) -2016-05-24 21:47:01 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:47:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 21:47:01 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.039566 s -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=460959, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(3152) called with curMem=466551, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:55748 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 21:47:01 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:47:01 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:47:01 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:47:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:47:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:47:01 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:55748 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=454409, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(3492) called with curMem=466209, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:55748 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 21:47:01 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:47:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:47:01 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 21:47:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:55748 in memory (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:47:01 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 21:47:01 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:55748 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 21:47:01 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:55748 in memory (size: 3.0 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 7968 bytes result sent to driver -2016-05-24 21:47:01 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:47:01 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 7968 bytes result sent to driver -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:55748 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 20 ms on localhost (1/2) -2016-05-24 21:47:01 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 21:47:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 19 ms on localhost (2/2) -2016-05-24 21:47:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 21:47:01 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 21:47:01 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.044236 s -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:55748 in memory (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:55748 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55748 in memory (size: 388.0 B, free: 730.4 MB) -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=398142, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=393878, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:55748 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:55748 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:55748 in memory (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:55748 in memory (size: 2.4 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 21:47:01 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:47:01 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:47:01 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:47:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:47:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:47:01 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=381141, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(3495) called with curMem=392941, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:55748 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:47:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:47:01 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 21:47:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:47:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:47:01 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 21:47:01 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 21:47:01 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:47:01 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:47:01 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 7968 bytes result sent to driver -2016-05-24 21:47:01 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 7968 bytes result sent to driver -2016-05-24 21:47:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 14 ms on localhost (1/2) -2016-05-24 21:47:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 16 ms on localhost (2/2) -2016-05-24 21:47:01 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 21:47:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 21:47:01 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.031021 s -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=396436, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=402028, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:55748 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 21:47:01 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:47:01 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:47:01 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:47:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:47:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:47:01 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=405192, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(3495) called with curMem=416992, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:55748 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:47:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:47:01 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 21:47:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:47:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:47:01 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 21:47:01 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 21:47:01 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:47:01 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:47:01 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 7968 bytes result sent to driver -2016-05-24 21:47:01 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 7968 bytes result sent to driver -2016-05-24 21:47:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 13 ms on localhost (1/2) -2016-05-24 21:47:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 12 ms on localhost (2/2) -2016-05-24 21:47:01 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:47:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 21:47:01 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.026176 s -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=420487, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=426079, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:55748 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 21:47:01 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:47:01 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:47:01 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:47:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:47:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:47:01 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=429243, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=441043, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 21:47:01 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:55748 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 21:47:01 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:47:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:47:01 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 21:47:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:47:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 21:47:01 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 21:47:01 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 21:47:01 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:47:01 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:47:01 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 7968 bytes result sent to driver -2016-05-24 21:47:01 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 7968 bytes result sent to driver -2016-05-24 21:47:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 11 ms on localhost (1/2) -2016-05-24 21:47:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 10 ms on localhost (2/2) -2016-05-24 21:47:01 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 21:47:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 21:47:01 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.028176 s -2016-05-24 21:47:01 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 0.6931471805599448, 10057.41015625, 3992.030200294059, 2280.4067582415346, 1609.1071976946598, 1008.6786174352385, 460.5648214704288, 0.0 -2016-05-24 21:47:01 WARN LogisticRegressionWithSGD:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-05-24 21:47:01 INFO SparkContext:59 - Starting job: collectAsMap at MulticlassMetrics.scala:51 -2016-05-24 21:47:01 INFO DAGScheduler:59 - Registering RDD 29 (map at MulticlassMetrics.scala:48) -2016-05-24 21:47:01 INFO DAGScheduler:59 - Got job 12 (collectAsMap at MulticlassMetrics.scala:51) with 2 output partitions -2016-05-24 21:47:01 INFO DAGScheduler:59 - Final stage: ResultStage 13(collectAsMap at MulticlassMetrics.scala:51) -2016-05-24 21:47:01 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-24 21:47:01 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-05-24 21:47:01 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (MapPartitionsRDD[29] at map at MulticlassMetrics.scala:48), which has no missing parents -2016-05-24 21:47:01 INFO MemoryStore:59 - ensureFreeSpace(11408) called with curMem=444537, maxMem=766075207 -2016-05-24 21:47:01 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 11.1 KB, free 730.2 MB) -2016-05-24 21:47:02 INFO MemoryStore:59 - ensureFreeSpace(6171) called with curMem=455945, maxMem=766075207 -2016-05-24 21:47:02 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 6.0 KB, free 730.1 MB) -2016-05-24 21:47:02 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:55748 (size: 6.0 KB, free: 730.4 MB) -2016-05-24 21:47:02 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:47:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (MapPartitionsRDD[29] at map at MulticlassMetrics.scala:48) -2016-05-24 21:47:02 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 21:47:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 21:47:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 21:47:02 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 21:47:02 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 21:47:02 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:47:02 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:47:02 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2254 bytes result sent to driver -2016-05-24 21:47:02 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2254 bytes result sent to driver -2016-05-24 21:47:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 53 ms on localhost (1/2) -2016-05-24 21:47:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 52 ms on localhost (2/2) -2016-05-24 21:47:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 21:47:02 INFO DAGScheduler:59 - ShuffleMapStage 12 (map at MulticlassMetrics.scala:48) finished in 0.055 s -2016-05-24 21:47:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 21:47:02 INFO DAGScheduler:59 - running: Set() -2016-05-24 21:47:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-05-24 21:47:02 INFO DAGScheduler:59 - failed: Set() -2016-05-24 21:47:02 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-05-24 21:47:02 INFO DAGScheduler:59 - Submitting ResultStage 13 (ShuffledRDD[30] at reduceByKey at MulticlassMetrics.scala:50), which is now runnable -2016-05-24 21:47:02 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=462116, maxMem=766075207 -2016-05-24 21:47:02 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 21:47:02 INFO MemoryStore:59 - ensureFreeSpace(1412) called with curMem=464452, maxMem=766075207 -2016-05-24 21:47:02 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 1412.0 B, free 730.1 MB) -2016-05-24 21:47:02 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:55748 (size: 1412.0 B, free: 730.4 MB) -2016-05-24 21:47:02 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:47:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (ShuffledRDD[30] at reduceByKey at MulticlassMetrics.scala:50) -2016-05-24 21:47:02 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 21:47:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 21:47:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 21:47:02 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 21:47:02 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 21:47:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 21:47:02 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-24 21:47:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-24 21:47:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-24 21:47:02 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 1161 bytes result sent to driver -2016-05-24 21:47:02 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 1389 bytes result sent to driver -2016-05-24 21:47:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 28 ms on localhost (1/2) -2016-05-24 21:47:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 31 ms on localhost (2/2) -2016-05-24 21:47:02 INFO DAGScheduler:59 - ResultStage 13 (collectAsMap at MulticlassMetrics.scala:51) finished in 0.032 s -2016-05-24 21:47:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 21:47:02 INFO DAGScheduler:59 - Job 12 finished: collectAsMap at MulticlassMetrics.scala:51, took 0.122053 s -2016-05-24 21:47:02 INFO SparkContext:59 - Starting job: countByValue at MulticlassMetrics.scala:45 -2016-05-24 21:47:02 INFO DAGScheduler:59 - Registering RDD 33 (countByValue at MulticlassMetrics.scala:45) -2016-05-24 21:47:02 INFO DAGScheduler:59 - Got job 13 (countByValue at MulticlassMetrics.scala:45) with 2 output partitions -2016-05-24 21:47:02 INFO DAGScheduler:59 - Final stage: ResultStage 15(countByValue at MulticlassMetrics.scala:45) -2016-05-24 21:47:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 14) -2016-05-24 21:47:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 14) -2016-05-24 21:47:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 14 (MapPartitionsRDD[33] at countByValue at MulticlassMetrics.scala:45), which has no missing parents -2016-05-24 21:47:02 INFO MemoryStore:59 - ensureFreeSpace(11936) called with curMem=465864, maxMem=766075207 -2016-05-24 21:47:02 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 11.7 KB, free 730.1 MB) -2016-05-24 21:47:02 INFO MemoryStore:59 - ensureFreeSpace(6356) called with curMem=477800, maxMem=766075207 -2016-05-24 21:47:02 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 6.2 KB, free 730.1 MB) -2016-05-24 21:47:02 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:55748 (size: 6.2 KB, free: 730.4 MB) -2016-05-24 21:47:02 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:47:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 14 (MapPartitionsRDD[33] at countByValue at MulticlassMetrics.scala:45) -2016-05-24 21:47:02 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 21:47:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 21:47:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 21:47:02 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 21:47:02 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 21:47:02 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:47:02 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:47:02 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2254 bytes result sent to driver -2016-05-24 21:47:02 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2254 bytes result sent to driver -2016-05-24 21:47:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 24 ms on localhost (1/2) -2016-05-24 21:47:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 24 ms on localhost (2/2) -2016-05-24 21:47:02 INFO DAGScheduler:59 - ShuffleMapStage 14 (countByValue at MulticlassMetrics.scala:45) finished in 0.026 s -2016-05-24 21:47:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 21:47:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 21:47:02 INFO DAGScheduler:59 - running: Set() -2016-05-24 21:47:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 15) -2016-05-24 21:47:02 INFO DAGScheduler:59 - failed: Set() -2016-05-24 21:47:02 INFO DAGScheduler:59 - Missing parents for ResultStage 15: List() -2016-05-24 21:47:02 INFO DAGScheduler:59 - Submitting ResultStage 15 (ShuffledRDD[34] at countByValue at MulticlassMetrics.scala:45), which is now runnable -2016-05-24 21:47:02 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=484156, maxMem=766075207 -2016-05-24 21:47:02 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 21:47:02 INFO MemoryStore:59 - ensureFreeSpace(1371) called with curMem=486492, maxMem=766075207 -2016-05-24 21:47:02 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 1371.0 B, free 730.1 MB) -2016-05-24 21:47:02 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:55748 (size: 1371.0 B, free: 730.4 MB) -2016-05-24 21:47:02 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:47:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (ShuffledRDD[34] at countByValue at MulticlassMetrics.scala:45) -2016-05-24 21:47:02 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 21:47:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 21:47:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 21:47:02 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 21:47:02 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 21:47:02 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-24 21:47:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 21:47:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 21:47:02 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 1161 bytes result sent to driver -2016-05-24 21:47:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 21:47:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 8 ms on localhost (1/2) -2016-05-24 21:47:02 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 1394 bytes result sent to driver -2016-05-24 21:47:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 13 ms on localhost (2/2) -2016-05-24 21:47:02 INFO DAGScheduler:59 - ResultStage 15 (countByValue at MulticlassMetrics.scala:45) finished in 0.013 s -2016-05-24 21:47:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 21:47:02 INFO DAGScheduler:59 - Job 13 finished: countByValue at MulticlassMetrics.scala:45, took 0.067285 s -2016-05-24 21:47:02 INFO SparkContext:59 - Starting job: count at LogisticRegression3Learning.scala:39 -2016-05-24 21:47:02 INFO DAGScheduler:59 - Got job 14 (count at LogisticRegression3Learning.scala:39) with 2 output partitions -2016-05-24 21:47:02 INFO DAGScheduler:59 - Final stage: ResultStage 16(count at LogisticRegression3Learning.scala:39) -2016-05-24 21:47:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:47:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:47:02 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-24 21:47:02 INFO MemoryStore:59 - ensureFreeSpace(3808) called with curMem=487863, maxMem=766075207 -2016-05-24 21:47:02 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 3.7 KB, free 730.1 MB) -2016-05-24 21:47:02 INFO MemoryStore:59 - ensureFreeSpace(2133) called with curMem=491671, maxMem=766075207 -2016-05-24 21:47:02 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.1 MB) -2016-05-24 21:47:02 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:55748 (size: 2.1 KB, free: 730.4 MB) -2016-05-24 21:47:02 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:47:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-24 21:47:02 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 21:47:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:47:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:47:02 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 21:47:02 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 21:47:02 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:47:02 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:47:02 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2082 bytes result sent to driver -2016-05-24 21:47:02 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2082 bytes result sent to driver -2016-05-24 21:47:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 10 ms on localhost (1/2) -2016-05-24 21:47:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 11 ms on localhost (2/2) -2016-05-24 21:47:02 INFO DAGScheduler:59 - ResultStage 16 (count at LogisticRegression3Learning.scala:39) finished in 0.011 s -2016-05-24 21:47:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 21:47:02 INFO DAGScheduler:59 - Job 14 finished: count at LogisticRegression3Learning.scala:39, took 0.026047 s -2016-05-24 21:47:02 INFO SparkContext:59 - Starting job: count at LogisticRegression3Learning.scala:40 -2016-05-24 21:47:02 INFO DAGScheduler:59 - Got job 15 (count at LogisticRegression3Learning.scala:40) with 2 output partitions -2016-05-24 21:47:02 INFO DAGScheduler:59 - Final stage: ResultStage 17(count at LogisticRegression3Learning.scala:40) -2016-05-24 21:47:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:47:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:47:02 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[7] at randomSplit at LogisticRegression3Learning.scala:25), which has no missing parents -2016-05-24 21:47:02 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=493804, maxMem=766075207 -2016-05-24 21:47:02 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 4.0 KB, free 730.1 MB) -2016-05-24 21:47:02 INFO MemoryStore:59 - ensureFreeSpace(2257) called with curMem=497916, maxMem=766075207 -2016-05-24 21:47:02 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-24 21:47:02 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:55748 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:47:02 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:47:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[7] at randomSplit at LogisticRegression3Learning.scala:25) -2016-05-24 21:47:02 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 21:47:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:47:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:47:02 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 21:47:02 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 21:47:02 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:47:02 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:47:02 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2082 bytes result sent to driver -2016-05-24 21:47:02 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2082 bytes result sent to driver -2016-05-24 21:47:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 13 ms on localhost (1/2) -2016-05-24 21:47:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 14 ms on localhost (2/2) -2016-05-24 21:47:02 INFO DAGScheduler:59 - ResultStage 17 (count at LogisticRegression3Learning.scala:40) finished in 0.015 s -2016-05-24 21:47:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 21:47:02 INFO DAGScheduler:59 - Job 15 finished: count at LogisticRegression3Learning.scala:40, took 0.027659 s -2016-05-24 21:47:02 INFO SparkContext:59 - Starting job: count at LogisticRegression3Learning.scala:41 -2016-05-24 21:47:02 INFO DAGScheduler:59 - Got job 16 (count at LogisticRegression3Learning.scala:41) with 2 output partitions -2016-05-24 21:47:02 INFO DAGScheduler:59 - Final stage: ResultStage 18(count at LogisticRegression3Learning.scala:41) -2016-05-24 21:47:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:47:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:47:02 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[8] at randomSplit at LogisticRegression3Learning.scala:25), which has no missing parents -2016-05-24 21:47:02 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=500173, maxMem=766075207 -2016-05-24 21:47:02 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 4.0 KB, free 730.1 MB) -2016-05-24 21:47:02 INFO MemoryStore:59 - ensureFreeSpace(2261) called with curMem=504285, maxMem=766075207 -2016-05-24 21:47:02 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-24 21:47:02 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:55748 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 21:47:02 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:47:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[8] at randomSplit at LogisticRegression3Learning.scala:25) -2016-05-24 21:47:02 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 21:47:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:47:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:47:02 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 21:47:02 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 21:47:02 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:47:02 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 21:47:02 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2082 bytes result sent to driver -2016-05-24 21:47:02 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2082 bytes result sent to driver -2016-05-24 21:47:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 11 ms on localhost (1/2) -2016-05-24 21:47:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 10 ms on localhost (2/2) -2016-05-24 21:47:02 INFO DAGScheduler:59 - ResultStage 18 (count at LogisticRegression3Learning.scala:41) finished in 0.011 s -2016-05-24 21:47:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 21:47:02 INFO DAGScheduler:59 - Job 16 finished: count at LogisticRegression3Learning.scala:41, took 0.063787 s -2016-05-24 21:47:02 INFO SparkContext:59 - Starting job: take at LogisticRegression3Learning.scala:43 -2016-05-24 21:47:02 INFO DAGScheduler:59 - Got job 17 (take at LogisticRegression3Learning.scala:43) with 1 output partitions -2016-05-24 21:47:02 INFO DAGScheduler:59 - Final stage: ResultStage 19(take at LogisticRegression3Learning.scala:43) -2016-05-24 21:47:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:47:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:47:02 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[28] at map at LogisticRegression3Learning.scala:31), which has no missing parents -2016-05-24 21:47:02 INFO MemoryStore:59 - ensureFreeSpace(10640) called with curMem=506546, maxMem=766075207 -2016-05-24 21:47:02 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 10.4 KB, free 730.1 MB) -2016-05-24 21:47:02 INFO MemoryStore:59 - ensureFreeSpace(5781) called with curMem=517186, maxMem=766075207 -2016-05-24 21:47:02 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-24 21:47:02 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:55748 (size: 5.6 KB, free: 730.4 MB) -2016-05-24 21:47:02 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:47:02 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[28] at map at LogisticRegression3Learning.scala:31) -2016-05-24 21:47:02 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-24 21:47:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 21:47:02 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 21:47:02 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 21:47:02 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 2401 bytes result sent to driver -2016-05-24 21:47:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 9 ms on localhost (1/1) -2016-05-24 21:47:02 INFO DAGScheduler:59 - ResultStage 19 (take at LogisticRegression3Learning.scala:43) finished in 0.009 s -2016-05-24 21:47:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 21:47:02 INFO DAGScheduler:59 - Job 17 finished: take at LogisticRegression3Learning.scala:43, took 0.021379 s -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 21:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 21:47:02 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 21:47:02 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 21:47:02 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 21:47:02 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 21:47:02 INFO BlockManager:59 - BlockManager stopped -2016-05-24 21:47:02 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 21:47:02 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 21:47:02 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 21:47:02 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 21:47:02 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-cbb3f851-981b-48d2-a896-12a2990a6eea -2016-05-24 21:47:02 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 21:47:02 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 21:59:01 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 21:59:02 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 21:59:02 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 21:59:02 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 21:59:02 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 21:59:03 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 21:59:03 INFO Remoting:74 - Starting remoting -2016-05-24 21:59:03 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:56168] -2016-05-24 21:59:03 INFO Utils:59 - Successfully started service 'sparkDriver' on port 56168. -2016-05-24 21:59:03 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 21:59:03 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 21:59:03 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-3ba81243-b5e6-4274-a30d-d8ff6fe68408 -2016-05-24 21:59:03 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 21:59:04 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-c4e31c6e-3f90-4657-8648-626bb1f62193\httpd-2e73856e-2d0c-47c4-8879-9d31b3e894d5 -2016-05-24 21:59:04 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 21:59:04 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:59:04 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:56169 -2016-05-24 21:59:04 INFO Utils:59 - Successfully started service 'HTTP file server' on port 56169. -2016-05-24 21:59:04 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 21:59:04 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 21:59:04 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 21:59:04 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 21:59:04 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 21:59:04 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 21:59:04 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 21:59:04 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 56188. -2016-05-24 21:59:04 INFO NettyBlockTransferService:59 - Server created on 56188 -2016-05-24 21:59:04 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 21:59:04 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:56188 with 730.6 MB RAM, BlockManagerId(driver, localhost, 56188) -2016-05-24 21:59:04 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 21:59:05 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 21:59:05 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 21:59:05 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 21:59:05 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 21:59:05 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:56188 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 21:59:05 INFO SparkContext:59 - Created broadcast 0 from textFile at LogisticRegressionLearning.scala:23 -2016-05-24 21:59:06 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 21:59:06 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 21:59:07 INFO SparkContext:59 - Starting job: foreach at LogisticRegressionLearning.scala:28 -2016-05-24 21:59:07 INFO DAGScheduler:59 - Got job 0 (foreach at LogisticRegressionLearning.scala:28) with 2 output partitions -2016-05-24 21:59:07 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at LogisticRegressionLearning.scala:28) -2016-05-24 21:59:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:07 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at LogisticRegressionLearning.scala:24), which has no missing parents -2016-05-24 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=171580, maxMem=766075207 -2016-05-24 21:59:07 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(1993) called with curMem=175020, maxMem=766075207 -2016-05-24 21:59:07 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1993.0 B, free 730.4 MB) -2016-05-24 21:59:07 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:56188 (size: 1993.0 B, free: 730.6 MB) -2016-05-24 21:59:07 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at LogisticRegressionLearning.scala:24) -2016-05-24 21:59:07 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 21:59:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:59:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:59:07 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 21:59:07 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 21:59:07 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 21:59:07 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 21:59:07 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:38+39 -2016-05-24 21:59:07 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:0+38 -2016-05-24 21:59:07 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 21:59:07 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 21:59:07 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 21:59:07 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 21:59:07 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=177013, maxMem=766075207 -2016-05-24 21:59:07 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-05-24 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(424) called with curMem=177301, maxMem=766075207 -2016-05-24 21:59:07 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 424.0 B, free 730.4 MB) -2016-05-24 21:59:07 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:56188 (size: 288.0 B, free: 730.6 MB) -2016-05-24 21:59:07 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:56188 (size: 424.0 B, free: 730.6 MB) -2016-05-24 21:59:07 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2624 bytes result sent to driver -2016-05-24 21:59:07 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2624 bytes result sent to driver -2016-05-24 21:59:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 112 ms on localhost (1/2) -2016-05-24 21:59:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 135 ms on localhost (2/2) -2016-05-24 21:59:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 21:59:07 INFO DAGScheduler:59 - ResultStage 0 (foreach at LogisticRegressionLearning.scala:28) finished in 0.150 s -2016-05-24 21:59:07 INFO DAGScheduler:59 - Job 0 finished: foreach at LogisticRegressionLearning.scala:28, took 0.317767 s -2016-05-24 21:59:07 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 21:59:07 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 21:59:07 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:59:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:07 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(3752) called with curMem=177725, maxMem=766075207 -2016-05-24 21:59:07 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-24 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(2181) called with curMem=181477, maxMem=766075207 -2016-05-24 21:59:07 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 21:59:07 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:56188 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:59:07 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:07 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 21:59:07 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 21:59:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:59:07 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 21:59:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:07 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-24 21:59:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 15 ms on localhost (1/1) -2016-05-24 21:59:07 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.015 s -2016-05-24 21:59:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 21:59:07 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.044030 s -2016-05-24 21:59:07 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 21:59:07 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 21:59:07 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 21:59:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:07 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[4] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=183658, maxMem=766075207 -2016-05-24 21:59:07 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(2039) called with curMem=187186, maxMem=766075207 -2016-05-24 21:59:07 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2039.0 B, free 730.4 MB) -2016-05-24 21:59:07 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:56188 (size: 2039.0 B, free: 730.6 MB) -2016-05-24 21:59:07 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[4] at filter at DataValidators.scala:40) -2016-05-24 21:59:07 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 21:59:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:59:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:59:07 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 21:59:07 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 21:59:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:07 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:07 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 21:59:07 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 21:59:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 14 ms on localhost (1/2) -2016-05-24 21:59:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 16 ms on localhost (2/2) -2016-05-24 21:59:07 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.016 s -2016-05-24 21:59:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 21:59:07 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.031768 s -2016-05-24 21:59:07 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 21:59:07 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 21:59:07 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 21:59:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:07 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[5] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=189225, maxMem=766075207 -2016-05-24 21:59:07 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(2018) called with curMem=192681, maxMem=766075207 -2016-05-24 21:59:07 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2018.0 B, free 730.4 MB) -2016-05-24 21:59:07 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:56188 (size: 2018.0 B, free: 730.6 MB) -2016-05-24 21:59:07 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[5] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 21:59:07 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 21:59:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:59:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 21:59:07 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 21:59:07 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 21:59:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:07 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 21:59:07 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:07 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 21:59:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 12 ms on localhost (1/2) -2016-05-24 21:59:07 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.017 s -2016-05-24 21:59:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 14 ms on localhost (2/2) -2016-05-24 21:59:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 21:59:07 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.030825 s -2016-05-24 21:59:07 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 21:59:07 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=194699, maxMem=766075207 -2016-05-24 21:59:07 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=194763, maxMem=766075207 -2016-05-24 21:59:07 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:07 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:07 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:07 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:07 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:07 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[7] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=194893, maxMem=766075207 -2016-05-24 21:59:07 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=200429, maxMem=766075207 -2016-05-24 21:59:07 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:07 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:07 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[7] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:07 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 21:59:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:07 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 21:59:07 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 21:59:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:07 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:07 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2415 bytes result sent to driver -2016-05-24 21:59:07 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2415 bytes result sent to driver -2016-05-24 21:59:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 18 ms on localhost (1/2) -2016-05-24 21:59:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 21 ms on localhost (2/2) -2016-05-24 21:59:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 21:59:07 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 21:59:07 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.039429 s -2016-05-24 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=203393, maxMem=766075207 -2016-05-24 21:59:07 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=203457, maxMem=766075207 -2016-05-24 21:59:07 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:07 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:07 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:07 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:07 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:07 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[9] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=203587, maxMem=766075207 -2016-05-24 21:59:07 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:07 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=209123, maxMem=766075207 -2016-05-24 21:59:07 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:07 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:07 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[9] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:07 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 21:59:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:07 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 21:59:07 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 21:59:07 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:07 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2415 bytes result sent to driver -2016-05-24 21:59:07 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 20 ms on localhost (1/2) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 21 ms on localhost (2/2) -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 21:59:08 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 21:59:08 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.043955 s -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=212088, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=212152, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:08 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[11] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=203781, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=209317, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[11] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 21:59:08 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 21:59:08 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 29 ms on localhost (1/2) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 29 ms on localhost (2/2) -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 21:59:08 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.032 s -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.065178 s -2016-05-24 21:59:08 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=203587, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=203521, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:56188 in memory (size: 2018.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:56188 in memory (size: 2039.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 21:59:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:56188 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:08 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:56188 in memory (size: 1993.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:08 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=181180, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=186716, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 21:59:08 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 15 ms on localhost (1/2) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 18 ms on localhost (2/2) -2016-05-24 21:59:08 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 21:59:08 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.043149 s -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=189678, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=189742, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:08 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=189872, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=195408, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 21:59:08 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 16 ms on localhost (1/2) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 19 ms on localhost (2/2) -2016-05-24 21:59:08 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 21:59:08 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.038679 s -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=198370, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=198434, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:08 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=198564, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=204100, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 21:59:08 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 16 ms on localhost (1/2) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 14 ms on localhost (2/2) -2016-05-24 21:59:08 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 21:59:08 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.032693 s -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=207064, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=207128, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:08 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=207258, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=212794, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 21:59:08 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 14 ms on localhost (1/2) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 15 ms on localhost (2/2) -2016-05-24 21:59:08 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 21:59:08 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.031118 s -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=215757, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=215821, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:08 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=215951, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=221487, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 21:59:08 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 12 ms on localhost (1/2) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 13 ms on localhost (2/2) -2016-05-24 21:59:08 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 21:59:08 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.031507 s -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224452, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224516, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:08 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=224646, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=230182, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 21:59:08 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 15 ms on localhost (1/2) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 13 ms on localhost (2/2) -2016-05-24 21:59:08 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 21:59:08 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.031703 s -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=233147, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=233211, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:08 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=233341, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=238877, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 21:59:08 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 13 ms on localhost (1/2) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 16 ms on localhost (2/2) -2016-05-24 21:59:08 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 21:59:08 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.036685 s -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=241841, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=241905, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:08 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=242035, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=247571, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 21:59:08 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 21:59:08 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 19 ms on localhost (1/2) -2016-05-24 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 21 ms on localhost (2/2) -2016-05-24 21:59:08 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 21:59:08 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.077952 s -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224839, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224773, maxMem=766075207 -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 21:59:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:08 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=212988, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=218524, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 21:59:08 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 21 ms on localhost (1/2) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 27 ms on localhost (2/2) -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 21:59:08 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 21:59:08 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.050338 s -2016-05-24 21:59:08 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=207266, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=207200, maxMem=766075207 -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:08 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:08 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=189880, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=195416, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 21:59:08 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 14 ms on localhost (1/2) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 14 ms on localhost (2/2) -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 21:59:08 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 21:59:08 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.036468 s -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=198381, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=198445, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:08 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=198575, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=204111, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 21:59:08 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 15 ms on localhost (1/2) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 15 ms on localhost (2/2) -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 21:59:08 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:59:08 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.033104 s -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=207076, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=207140, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:08 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=207270, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=212806, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 21:59:08 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 13 ms on localhost (1/2) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 12 ms on localhost (2/2) -2016-05-24 21:59:08 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 21:59:08 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.025980 s -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=215770, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=215834, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:08 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=215964, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=221500, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:08 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 21:59:08 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:08 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 14 ms on localhost (1/2) -2016-05-24 21:59:08 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 2415 bytes result sent to driver -2016-05-24 21:59:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 16 ms on localhost (2/2) -2016-05-24 21:59:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 21:59:08 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 21:59:08 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.034211 s -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224465, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224529, maxMem=766075207 -2016-05-24 21:59:08 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:08 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:08 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=224659, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=230195, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 13 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 16 ms on localhost (2/2) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.031003 s -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=233159, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=233223, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=233353, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=238889, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 14 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 13 ms on localhost (2/2) -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.029660 s -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=241849, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=241913, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=242043, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=247579, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 15 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 16 ms on localhost (2/2) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.035917 s -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=250544, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=250608, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=250738, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=256274, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 12 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 15 ms on localhost (2/2) -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.030417 s -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=259237, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=259301, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=259431, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=264967, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 13 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 13 ms on localhost (2/2) -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.030412 s -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=267931, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=267995, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=268125, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=273661, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 12 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 12 ms on localhost (2/2) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.029265 s -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=276623, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=276687, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=276817, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2961) called with curMem=282353, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 11 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 12 ms on localhost (2/2) -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.032838 s -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=285314, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=285378, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=285508, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2961) called with curMem=291044, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 9 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 11 ms on localhost (2/2) -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.026336 s -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=294005, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=294069, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=294199, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=299735, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 9 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 9 ms on localhost (2/2) -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.021683 s -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=302699, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=302763, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=302893, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=308429, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 12 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 12 ms on localhost (2/2) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.026484 s -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=311394, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=311458, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=311588, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=317124, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 11 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 11 ms on localhost (2/2) -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.027110 s -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=320088, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=320152, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=320282, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=325818, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 7 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 8 ms on localhost (2/2) -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.024449 s -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=328787, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=328851, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=311588, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=317124, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 6 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 8 ms on localhost (2/2) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.020842 s -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=302764, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=302764, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=282550, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=282550, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 8 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 9 ms on localhost (2/2) -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.020958 s -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=276626, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=268191, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=250806, maxMem=766075207 -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=256278, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 7 ms on localhost (1/2) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 9 ms on localhost (2/2) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.020275 s -2016-05-24 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=241854, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=241918, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=216158, maxMem=766075207 -2016-05-24 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=221564, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 6 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 7 ms on localhost (2/2) -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.018004 s -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:56188 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=207265, maxMem=766075207 -2016-05-24 21:59:09 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=207199, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:56188 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=207265, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=212801, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 6 ms on localhost (1/2) -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 7 ms on localhost (2/2) -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.019248 s -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=215763, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=215827, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=215957, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=221493, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 6 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 5 ms on localhost (2/2) -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.015341 s -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224457, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224521, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=224651, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=230187, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 6 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 6 ms on localhost (2/2) -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.015016 s -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=233150, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=233214, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.6 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=233344, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=238880, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 7 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 6 ms on localhost (2/2) -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.015316 s -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=241845, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=241909, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=242039, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=247575, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 6 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 8 ms on localhost (2/2) -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.018165 s -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=250538, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=250602, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=250732, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2967) called with curMem=256268, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 5 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 6 ms on localhost (2/2) -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.016728 s -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=259235, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=259299, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=259429, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2969) called with curMem=264965, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 21:59:09 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:09 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 2415 bytes result sent to driver -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 8 ms on localhost (1/2) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 10 ms on localhost (2/2) -2016-05-24 21:59:09 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 21:59:09 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.020983 s -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=267934, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=267998, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:09 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=268128, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO MemoryStore:59 - ensureFreeSpace(2964) called with curMem=273664, maxMem=766075207 -2016-05-24 21:59:09 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:09 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:09 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:09 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:10 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 21:59:10 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 21:59:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:10 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 2415 bytes result sent to driver -2016-05-24 21:59:10 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 2415 bytes result sent to driver -2016-05-24 21:59:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 6 ms on localhost (1/2) -2016-05-24 21:59:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 5 ms on localhost (2/2) -2016-05-24 21:59:10 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:59:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 21:59:10 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.014517 s -2016-05-24 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=276628, maxMem=766075207 -2016-05-24 21:59:10 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=276692, maxMem=766075207 -2016-05-24 21:59:10 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:59:10 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:10 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:10 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:10 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:10 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=276822, maxMem=766075207 -2016-05-24 21:59:10 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=282358, maxMem=766075207 -2016-05-24 21:59:10 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:10 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:10 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:10 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 21:59:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:10 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 21:59:10 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 21:59:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:10 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 2415 bytes result sent to driver -2016-05-24 21:59:10 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 2415 bytes result sent to driver -2016-05-24 21:59:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 5 ms on localhost (1/2) -2016-05-24 21:59:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 5 ms on localhost (2/2) -2016-05-24 21:59:10 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 21:59:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 21:59:10 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.014858 s -2016-05-24 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=285323, maxMem=766075207 -2016-05-24 21:59:10 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=285387, maxMem=766075207 -2016-05-24 21:59:10 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:59:10 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:10 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:10 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:10 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:10 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=285517, maxMem=766075207 -2016-05-24 21:59:10 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=291053, maxMem=766075207 -2016-05-24 21:59:10 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:10 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:10 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:10 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 21:59:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:10 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 21:59:10 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 21:59:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:10 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 2415 bytes result sent to driver -2016-05-24 21:59:10 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 2415 bytes result sent to driver -2016-05-24 21:59:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 5 ms on localhost (1/2) -2016-05-24 21:59:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 5 ms on localhost (2/2) -2016-05-24 21:59:10 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 21:59:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 21:59:10 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.013776 s -2016-05-24 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=294018, maxMem=766075207 -2016-05-24 21:59:10 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 64.0 B, free 730.3 MB) -2016-05-24 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=294082, maxMem=766075207 -2016-05-24 21:59:10 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.3 MB) -2016-05-24 21:59:10 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:56188 (size: 130.0 B, free: 730.5 MB) -2016-05-24 21:59:10 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 21:59:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 21:59:10 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 21:59:10 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 21:59:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 21:59:10 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=294212, maxMem=766075207 -2016-05-24 21:59:10 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 5.4 KB, free 730.3 MB) -2016-05-24 21:59:10 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=299748, maxMem=766075207 -2016-05-24 21:59:10 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-24 21:59:10 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:56188 (size: 2.9 KB, free: 730.5 MB) -2016-05-24 21:59:10 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 21:59:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225) -2016-05-24 21:59:10 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 21:59:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 21:59:10 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 21:59:10 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 21:59:10 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 21:59:10 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 2415 bytes result sent to driver -2016-05-24 21:59:10 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 21:59:10 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 2415 bytes result sent to driver -2016-05-24 21:59:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 9 ms on localhost (1/2) -2016-05-24 21:59:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 10 ms on localhost (2/2) -2016-05-24 21:59:10 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 21:59:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 21:59:10 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.023853 s -2016-05-24 21:59:10 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 0.6307644134901649, 0.6301608260893138, 0.62961849864768, 0.6293831586494314, 0.6292340893598364, 0.629169306843859, 0.6291372385030662, 0.6291242484799721, 0.629118922682065, 0.6291169842584431 -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 21:59:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 21:59:10 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 21:59:10 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 21:59:10 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 21:59:10 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 21:59:10 INFO BlockManager:59 - BlockManager stopped -2016-05-24 21:59:10 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 21:59:10 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 21:59:10 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 21:59:10 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 21:59:10 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-c4e31c6e-3f90-4657-8648-626bb1f62193 -2016-05-24 21:59:10 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 21:59:10 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 22:01:20 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 22:01:21 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 22:01:21 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 22:01:21 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 22:01:21 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 22:01:22 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 22:01:22 INFO Remoting:74 - Starting remoting -2016-05-24 22:01:22 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:56327] -2016-05-24 22:01:22 INFO Utils:59 - Successfully started service 'sparkDriver' on port 56327. -2016-05-24 22:01:22 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 22:01:22 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 22:01:22 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-b3c83fb7-dfba-4e51-9d38-a2387079c2ac -2016-05-24 22:01:22 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 22:01:23 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-92789420-1d2f-4a29-9b4a-64375c7df260\httpd-89ca64f2-f12e-4947-840f-ebd95eabd3cd -2016-05-24 22:01:23 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 22:01:23 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:01:23 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:56328 -2016-05-24 22:01:23 INFO Utils:59 - Successfully started service 'HTTP file server' on port 56328. -2016-05-24 22:01:23 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 22:01:23 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:01:23 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 22:01:23 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 22:01:23 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 22:01:23 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 22:01:23 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 22:01:23 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 56348. -2016-05-24 22:01:23 INFO NettyBlockTransferService:59 - Server created on 56348 -2016-05-24 22:01:23 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 22:01:23 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:56348 with 730.6 MB RAM, BlockManagerId(driver, localhost, 56348) -2016-05-24 22:01:23 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 22:01:24 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 22:01:24 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 22:01:24 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 22:01:24 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 22:01:24 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:56348 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 22:01:24 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 22:01:25 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 22:01:25 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:01:26 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 22:01:26 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 22:01:26 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 22:01:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:01:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:01:26 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 22:01:26 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 22:01:26 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 22:01:26 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=175476, maxMem=766075207 -2016-05-24 22:01:26 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:01:26 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:56348 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:01:26 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:01:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 22:01:26 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 22:01:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:01:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:01:26 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 22:01:26 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 22:01:26 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 22:01:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 22:01:26 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 22:01:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 22:01:26 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 22:01:26 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 22:01:26 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 22:01:26 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 22:01:26 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 22:01:26 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177621, maxMem=766075207 -2016-05-24 22:01:26 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 22:01:26 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:56348 (size: 83.1 KB, free: 730.5 MB) -2016-05-24 22:01:26 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262701, maxMem=766075207 -2016-05-24 22:01:26 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 22:01:26 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:56348 (size: 83.6 KB, free: 730.4 MB) -2016-05-24 22:01:26 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 22:01:26 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 22:01:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 458 ms on localhost (1/2) -2016-05-24 22:01:26 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.517 s -2016-05-24 22:01:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 501 ms on localhost (2/2) -2016-05-24 22:01:26 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.722588 s -2016-05-24 22:01:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 22:01:26 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 22:01:26 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 22:01:26 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:01:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:01:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:01:26 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 22:01:26 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=348349, maxMem=766075207 -2016-05-24 22:01:26 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-24 22:01:26 INFO MemoryStore:59 - ensureFreeSpace(2481) called with curMem=352909, maxMem=766075207 -2016-05-24 22:01:26 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-24 22:01:26 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:56348 (size: 2.4 KB, free: 730.4 MB) -2016-05-24 22:01:26 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:01:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:01:26 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 22:01:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:01:26 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 22:01:26 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:01:26 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-24 22:01:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 31 ms on localhost (1/1) -2016-05-24 22:01:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 22:01:26 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.031 s -2016-05-24 22:01:26 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.057969 s -2016-05-24 22:01:26 WARN LogisticRegressionWithSGD:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-05-24 22:01:26 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 22:01:26 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 22:01:26 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 22:01:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:01:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:01:26 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 22:01:26 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=355390, maxMem=766075207 -2016-05-24 22:01:26 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-24 22:01:26 INFO MemoryStore:59 - ensureFreeSpace(2294) called with curMem=359638, maxMem=766075207 -2016-05-24 22:01:26 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 22:01:26 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:56348 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 22:01:26 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:01:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40) -2016-05-24 22:01:26 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 22:01:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:01:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:01:26 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 22:01:26 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 22:01:26 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:01:26 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:01:26 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 22:01:26 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 22:01:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 17 ms on localhost (1/2) -2016-05-24 22:01:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 19 ms on localhost (2/2) -2016-05-24 22:01:26 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.019 s -2016-05-24 22:01:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 22:01:26 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.038492 s -2016-05-24 22:01:26 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 22:01:26 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 22:01:26 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 22:01:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:01:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:01:26 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 22:01:26 INFO MemoryStore:59 - ensureFreeSpace(4264) called with curMem=361932, maxMem=766075207 -2016-05-24 22:01:26 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-24 22:01:26 INFO MemoryStore:59 - ensureFreeSpace(2318) called with curMem=366196, maxMem=766075207 -2016-05-24 22:01:26 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-24 22:01:26 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:56348 (size: 2.3 KB, free: 730.4 MB) -2016-05-24 22:01:26 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:01:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 22:01:26 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 22:01:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:01:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:01:26 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 22:01:26 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 22:01:26 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:01:26 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:01:26 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 22:01:26 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 22:01:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 23 ms on localhost (1/2) -2016-05-24 22:01:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 22 ms on localhost (2/2) -2016-05-24 22:01:27 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.024 s -2016-05-24 22:01:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 22:01:27 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.044843 s -2016-05-24 22:01:27 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 22:01:27 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=368514, maxMem=766075207 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(388) called with curMem=374106, maxMem=766075207 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 388.0 B, free 730.2 MB) -2016-05-24 22:01:27 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:56348 (size: 388.0 B, free: 730.4 MB) -2016-05-24 22:01:27 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 22:01:27 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:01:27 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:01:27 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:01:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:01:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:01:27 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=374494, maxMem=766075207 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(3493) called with curMem=386294, maxMem=766075207 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 22:01:27 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:56348 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 22:01:27 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:01:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:01:27 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 22:01:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:01:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:01:27 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 22:01:27 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 22:01:27 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:01:27 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:01:27 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 7968 bytes result sent to driver -2016-05-24 22:01:27 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 7968 bytes result sent to driver -2016-05-24 22:01:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 22 ms on localhost (1/2) -2016-05-24 22:01:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 22 ms on localhost (2/2) -2016-05-24 22:01:27 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 22:01:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 22:01:27 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.046837 s -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=389787, maxMem=766075207 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(2249) called with curMem=395379, maxMem=766075207 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 22:01:27 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:56348 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 22:01:27 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 22:01:27 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:01:27 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:01:27 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:01:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:01:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:01:27 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=397628, maxMem=766075207 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(3493) called with curMem=409428, maxMem=766075207 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 22:01:27 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:56348 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 22:01:27 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:01:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:01:27 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 22:01:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:01:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:01:27 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 22:01:27 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 22:01:27 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:01:27 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:01:27 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 7968 bytes result sent to driver -2016-05-24 22:01:27 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 7968 bytes result sent to driver -2016-05-24 22:01:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 19 ms on localhost (1/2) -2016-05-24 22:01:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 24 ms on localhost (2/2) -2016-05-24 22:01:27 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 22:01:27 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.049433 s -2016-05-24 22:01:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=412921, maxMem=766075207 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(3115) called with curMem=418513, maxMem=766075207 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-05-24 22:01:27 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:56348 (size: 3.0 KB, free: 730.4 MB) -2016-05-24 22:01:27 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 22:01:27 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:01:27 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:01:27 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:01:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:01:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:01:27 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=421628, maxMem=766075207 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=433428, maxMem=766075207 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 22:01:27 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:56348 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 22:01:27 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:01:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:01:27 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 22:01:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:01:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:01:27 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 22:01:27 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 22:01:27 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:01:27 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:01:27 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 7968 bytes result sent to driver -2016-05-24 22:01:27 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 7968 bytes result sent to driver -2016-05-24 22:01:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 17 ms on localhost (1/2) -2016-05-24 22:01:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 20 ms on localhost (2/2) -2016-05-24 22:01:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 22:01:27 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 22:01:27 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.042555 s -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=436922, maxMem=766075207 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(3151) called with curMem=442514, maxMem=766075207 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 22:01:27 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:56348 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 22:01:27 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 22:01:27 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:01:27 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:01:27 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:01:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:01:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:01:27 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=445665, maxMem=766075207 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=457465, maxMem=766075207 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:01:27 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:56348 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 22:01:27 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:01:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:01:27 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 22:01:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:01:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:01:27 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 22:01:27 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 22:01:27 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:01:27 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:01:27 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 7968 bytes result sent to driver -2016-05-24 22:01:27 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 7968 bytes result sent to driver -2016-05-24 22:01:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 15 ms on localhost (1/2) -2016-05-24 22:01:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 15 ms on localhost (2/2) -2016-05-24 22:01:27 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:01:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 22:01:27 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.035833 s -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=460959, maxMem=766075207 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(3152) called with curMem=466551, maxMem=766075207 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 22:01:27 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:56348 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 22:01:27 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 22:01:27 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:01:27 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:01:27 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:01:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:01:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:01:27 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=469703, maxMem=766075207 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(3493) called with curMem=469703, maxMem=766075207 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:01:27 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:56348 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 22:01:27 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:01:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:01:27 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 22:01:27 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:56348 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 22:01:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:01:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:01:27 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 22:01:27 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 22:01:27 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 22:01:27 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:56348 in memory (size: 3.1 KB, free: 730.4 MB) -2016-05-24 22:01:27 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:01:27 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 7968 bytes result sent to driver -2016-05-24 22:01:27 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:56348 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 22:01:27 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:01:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 20 ms on localhost (1/2) -2016-05-24 22:01:27 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 22:01:27 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 7968 bytes result sent to driver -2016-05-24 22:01:27 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:56348 in memory (size: 3.0 KB, free: 730.4 MB) -2016-05-24 22:01:27 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 22:01:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 28 ms on localhost (2/2) -2016-05-24 22:01:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 22:01:27 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.058238 s -2016-05-24 22:01:27 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:56348 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 22:01:27 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=421665, maxMem=766075207 -2016-05-24 22:01:27 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 22:01:27 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=427257, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 22:01:28 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:56348 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 22:01:28 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 22:01:28 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:56348 in memory (size: 2.2 KB, free: 730.4 MB) -2016-05-24 22:01:28 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:56348 in memory (size: 3.4 KB, free: 730.4 MB) -2016-05-24 22:01:28 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 22:01:28 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:56348 in memory (size: 388.0 B, free: 730.4 MB) -2016-05-24 22:01:28 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:56348 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-24 22:01:28 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 22:01:28 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:01:28 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:01:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:01:28 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:56348 in memory (size: 2.2 KB, free: 730.4 MB) -2016-05-24 22:01:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:01:28 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:01:28 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:56348 in memory (size: 2.4 KB, free: 730.4 MB) -2016-05-24 22:01:28 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=381142, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(3495) called with curMem=392942, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 22:01:28 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:56348 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 22:01:28 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:01:28 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 22:01:28 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 22:01:28 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:01:28 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:01:28 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 7968 bytes result sent to driver -2016-05-24 22:01:28 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 7968 bytes result sent to driver -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 14 ms on localhost (1/2) -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 17 ms on localhost (2/2) -2016-05-24 22:01:28 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 22:01:28 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.035935 s -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=396437, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=402029, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 22:01:28 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:56348 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 22:01:28 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 22:01:28 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:01:28 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:01:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:01:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=405193, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(3494) called with curMem=416993, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 22:01:28 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:56348 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 22:01:28 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:01:28 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 22:01:28 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 22:01:28 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:01:28 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:01:28 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 7968 bytes result sent to driver -2016-05-24 22:01:28 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 7968 bytes result sent to driver -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 12 ms on localhost (1/2) -2016-05-24 22:01:28 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 12 ms on localhost (2/2) -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 22:01:28 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.030952 s -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=420487, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.5 KB, free 730.2 MB) -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=426079, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-24 22:01:28 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:56348 (size: 3.1 KB, free: 730.4 MB) -2016-05-24 22:01:28 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 22:01:28 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:01:28 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:01:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:01:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(11800) called with curMem=429243, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 11.5 KB, free 730.2 MB) -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(3495) called with curMem=441043, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-05-24 22:01:28 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:56348 (size: 3.4 KB, free: 730.4 MB) -2016-05-24 22:01:28 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:01:28 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 22:01:28 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 22:01:28 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:01:28 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:01:28 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 7968 bytes result sent to driver -2016-05-24 22:01:28 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 7968 bytes result sent to driver -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 12 ms on localhost (1/2) -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 12 ms on localhost (2/2) -2016-05-24 22:01:28 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 22:01:28 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.029429 s -2016-05-24 22:01:28 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 0.6931471805599448, 10057.41015625, 3992.030200294059, 2280.4067582415346, 1609.1071976946598, 1008.6786174352385, 460.5648214704288, 0.0 -2016-05-24 22:01:28 WARN LogisticRegressionWithSGD:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-05-24 22:01:28 INFO SparkContext:59 - Starting job: collectAsMap at MulticlassMetrics.scala:51 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Registering RDD 29 (map at MulticlassMetrics.scala:48) -2016-05-24 22:01:28 INFO DAGScheduler:59 - Got job 12 (collectAsMap at MulticlassMetrics.scala:51) with 2 output partitions -2016-05-24 22:01:28 INFO DAGScheduler:59 - Final stage: ResultStage 13(collectAsMap at MulticlassMetrics.scala:51) -2016-05-24 22:01:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-24 22:01:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (MapPartitionsRDD[29] at map at MulticlassMetrics.scala:48), which has no missing parents -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(11408) called with curMem=444538, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 11.1 KB, free 730.2 MB) -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(6171) called with curMem=455946, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 6.0 KB, free 730.1 MB) -2016-05-24 22:01:28 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:56348 (size: 6.0 KB, free: 730.4 MB) -2016-05-24 22:01:28 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (MapPartitionsRDD[29] at map at MulticlassMetrics.scala:48) -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:01:28 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 22:01:28 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 22:01:28 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:01:28 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:01:28 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2254 bytes result sent to driver -2016-05-24 22:01:28 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2254 bytes result sent to driver -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 50 ms on localhost (1/2) -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 48 ms on localhost (2/2) -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 22:01:28 INFO DAGScheduler:59 - ShuffleMapStage 12 (map at MulticlassMetrics.scala:48) finished in 0.051 s -2016-05-24 22:01:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:01:28 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:01:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-05-24 22:01:28 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:01:28 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting ResultStage 13 (ShuffledRDD[30] at reduceByKey at MulticlassMetrics.scala:50), which is now runnable -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=462117, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(1412) called with curMem=464453, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 1412.0 B, free 730.1 MB) -2016-05-24 22:01:28 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:56348 (size: 1412.0 B, free: 730.4 MB) -2016-05-24 22:01:28 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (ShuffledRDD[30] at reduceByKey at MulticlassMetrics.scala:50) -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:01:28 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 22:01:28 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 22:01:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:01:28 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-24 22:01:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-24 22:01:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-24 22:01:28 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 1161 bytes result sent to driver -2016-05-24 22:01:28 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 1389 bytes result sent to driver -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 29 ms on localhost (1/2) -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 33 ms on localhost (2/2) -2016-05-24 22:01:28 INFO DAGScheduler:59 - ResultStage 13 (collectAsMap at MulticlassMetrics.scala:51) finished in 0.034 s -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 22:01:28 INFO DAGScheduler:59 - Job 12 finished: collectAsMap at MulticlassMetrics.scala:51, took 0.119995 s -2016-05-24 22:01:28 INFO SparkContext:59 - Starting job: countByValue at MulticlassMetrics.scala:45 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Registering RDD 33 (countByValue at MulticlassMetrics.scala:45) -2016-05-24 22:01:28 INFO DAGScheduler:59 - Got job 13 (countByValue at MulticlassMetrics.scala:45) with 2 output partitions -2016-05-24 22:01:28 INFO DAGScheduler:59 - Final stage: ResultStage 15(countByValue at MulticlassMetrics.scala:45) -2016-05-24 22:01:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 14) -2016-05-24 22:01:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 14) -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 14 (MapPartitionsRDD[33] at countByValue at MulticlassMetrics.scala:45), which has no missing parents -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(11936) called with curMem=465865, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 11.7 KB, free 730.1 MB) -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(6356) called with curMem=477801, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 6.2 KB, free 730.1 MB) -2016-05-24 22:01:28 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:56348 (size: 6.2 KB, free: 730.4 MB) -2016-05-24 22:01:28 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 14 (MapPartitionsRDD[33] at countByValue at MulticlassMetrics.scala:45) -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:01:28 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 22:01:28 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 22:01:28 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:01:28 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:01:28 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2254 bytes result sent to driver -2016-05-24 22:01:28 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2254 bytes result sent to driver -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 19 ms on localhost (1/2) -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 21 ms on localhost (2/2) -2016-05-24 22:01:28 INFO DAGScheduler:59 - ShuffleMapStage 14 (countByValue at MulticlassMetrics.scala:45) finished in 0.021 s -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 22:01:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:01:28 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:01:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 15) -2016-05-24 22:01:28 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:01:28 INFO DAGScheduler:59 - Missing parents for ResultStage 15: List() -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting ResultStage 15 (ShuffledRDD[34] at countByValue at MulticlassMetrics.scala:45), which is now runnable -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=484157, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(1371) called with curMem=486493, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 1371.0 B, free 730.1 MB) -2016-05-24 22:01:28 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:56348 (size: 1371.0 B, free: 730.4 MB) -2016-05-24 22:01:28 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (ShuffledRDD[34] at countByValue at MulticlassMetrics.scala:45) -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:01:28 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 22:01:28 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 22:01:28 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-24 22:01:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:01:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:01:28 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 1161 bytes result sent to driver -2016-05-24 22:01:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 7 ms on localhost (1/2) -2016-05-24 22:01:28 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 1394 bytes result sent to driver -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 14 ms on localhost (2/2) -2016-05-24 22:01:28 INFO DAGScheduler:59 - ResultStage 15 (countByValue at MulticlassMetrics.scala:45) finished in 0.014 s -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 22:01:28 INFO DAGScheduler:59 - Job 13 finished: countByValue at MulticlassMetrics.scala:45, took 0.060822 s -2016-05-24 22:01:28 INFO SparkContext:59 - Starting job: count at LogisticRegression3Learning.scala:39 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Got job 14 (count at LogisticRegression3Learning.scala:39) with 2 output partitions -2016-05-24 22:01:28 INFO DAGScheduler:59 - Final stage: ResultStage 16(count at LogisticRegression3Learning.scala:39) -2016-05-24 22:01:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:01:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(3808) called with curMem=487864, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 3.7 KB, free 730.1 MB) -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(2133) called with curMem=491672, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.1 MB) -2016-05-24 22:01:28 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:56348 (size: 2.1 KB, free: 730.4 MB) -2016-05-24 22:01:28 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:01:28 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 22:01:28 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 22:01:28 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:01:28 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:01:28 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2082 bytes result sent to driver -2016-05-24 22:01:28 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2082 bytes result sent to driver -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 12 ms on localhost (1/2) -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 13 ms on localhost (2/2) -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 22:01:28 INFO DAGScheduler:59 - ResultStage 16 (count at LogisticRegression3Learning.scala:39) finished in 0.015 s -2016-05-24 22:01:28 INFO DAGScheduler:59 - Job 14 finished: count at LogisticRegression3Learning.scala:39, took 0.027386 s -2016-05-24 22:01:28 INFO SparkContext:59 - Starting job: count at LogisticRegression3Learning.scala:40 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Got job 15 (count at LogisticRegression3Learning.scala:40) with 2 output partitions -2016-05-24 22:01:28 INFO DAGScheduler:59 - Final stage: ResultStage 17(count at LogisticRegression3Learning.scala:40) -2016-05-24 22:01:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:01:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[7] at randomSplit at LogisticRegression3Learning.scala:25), which has no missing parents -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=493805, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 4.0 KB, free 730.1 MB) -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(2257) called with curMem=497917, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-24 22:01:28 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:56348 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 22:01:28 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[7] at randomSplit at LogisticRegression3Learning.scala:25) -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:01:28 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 22:01:28 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 22:01:28 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:01:28 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:01:28 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2082 bytes result sent to driver -2016-05-24 22:01:28 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2082 bytes result sent to driver -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 11 ms on localhost (1/2) -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 11 ms on localhost (2/2) -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 22:01:28 INFO DAGScheduler:59 - ResultStage 17 (count at LogisticRegression3Learning.scala:40) finished in 0.012 s -2016-05-24 22:01:28 INFO DAGScheduler:59 - Job 15 finished: count at LogisticRegression3Learning.scala:40, took 0.023337 s -2016-05-24 22:01:28 INFO SparkContext:59 - Starting job: count at LogisticRegression3Learning.scala:41 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Got job 16 (count at LogisticRegression3Learning.scala:41) with 2 output partitions -2016-05-24 22:01:28 INFO DAGScheduler:59 - Final stage: ResultStage 18(count at LogisticRegression3Learning.scala:41) -2016-05-24 22:01:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:01:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[8] at randomSplit at LogisticRegression3Learning.scala:25), which has no missing parents -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=500174, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 4.0 KB, free 730.1 MB) -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(2261) called with curMem=504286, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-24 22:01:28 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:56348 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 22:01:28 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[8] at randomSplit at LogisticRegression3Learning.scala:25) -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:01:28 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 22:01:28 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:01:28 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2082 bytes result sent to driver -2016-05-24 22:01:28 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 14 ms on localhost (1/2) -2016-05-24 22:01:28 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:01:28 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2082 bytes result sent to driver -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 23 ms on localhost (2/2) -2016-05-24 22:01:28 INFO DAGScheduler:59 - ResultStage 18 (count at LogisticRegression3Learning.scala:41) finished in 0.023 s -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 22:01:28 INFO DAGScheduler:59 - Job 16 finished: count at LogisticRegression3Learning.scala:41, took 0.041338 s -2016-05-24 22:01:28 INFO SparkContext:59 - Starting job: take at LogisticRegression3Learning.scala:43 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Got job 17 (take at LogisticRegression3Learning.scala:43) with 1 output partitions -2016-05-24 22:01:28 INFO DAGScheduler:59 - Final stage: ResultStage 19(take at LogisticRegression3Learning.scala:43) -2016-05-24 22:01:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:01:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[28] at map at LogisticRegression3Learning.scala:31), which has no missing parents -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(10640) called with curMem=506547, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 10.4 KB, free 730.1 MB) -2016-05-24 22:01:28 INFO MemoryStore:59 - ensureFreeSpace(5781) called with curMem=517187, maxMem=766075207 -2016-05-24 22:01:28 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-24 22:01:28 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:56348 (size: 5.6 KB, free: 730.4 MB) -2016-05-24 22:01:28 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[28] at map at LogisticRegression3Learning.scala:31) -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-24 22:01:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:01:28 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 22:01:28 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:01:28 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 2401 bytes result sent to driver -2016-05-24 22:01:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 13 ms on localhost (1/1) -2016-05-24 22:01:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 22:01:28 INFO DAGScheduler:59 - ResultStage 19 (take at LogisticRegression3Learning.scala:43) finished in 0.014 s -2016-05-24 22:01:28 INFO DAGScheduler:59 - Job 17 finished: take at LogisticRegression3Learning.scala:43, took 0.030980 s -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 22:01:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 22:01:28 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 22:01:28 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 22:01:28 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 22:01:28 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 22:01:28 INFO BlockManager:59 - BlockManager stopped -2016-05-24 22:01:28 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 22:01:28 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 22:01:28 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 22:01:28 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 22:01:28 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-92789420-1d2f-4a29-9b4a-64375c7df260 -2016-05-24 22:07:21 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 22:07:22 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 22:07:22 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 22:07:22 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 22:07:22 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 22:07:23 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 22:07:23 INFO Remoting:74 - Starting remoting -2016-05-24 22:07:23 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:56472] -2016-05-24 22:07:23 INFO Utils:59 - Successfully started service 'sparkDriver' on port 56472. -2016-05-24 22:07:23 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 22:07:23 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 22:07:23 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-c1d9a950-5f6a-4022-addb-541f0bc16171 -2016-05-24 22:07:23 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 22:07:23 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-497f539c-c728-4ec4-ad63-67cf31b3ee7a\httpd-8d3a7526-3e8c-4061-8de1-7ec2da1dc8d9 -2016-05-24 22:07:23 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 22:07:23 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:07:23 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:56473 -2016-05-24 22:07:23 INFO Utils:59 - Successfully started service 'HTTP file server' on port 56473. -2016-05-24 22:07:23 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 22:07:23 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:07:23 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 22:07:23 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 22:07:23 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 22:07:24 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 22:07:24 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 22:07:24 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 56492. -2016-05-24 22:07:24 INFO NettyBlockTransferService:59 - Server created on 56492 -2016-05-24 22:07:24 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 22:07:24 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:56492 with 730.6 MB RAM, BlockManagerId(driver, localhost, 56492) -2016-05-24 22:07:24 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 22:07:25 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 22:07:25 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 22:07:25 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 22:07:25 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 22:07:25 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:56492 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 22:07:25 INFO SparkContext:59 - Created broadcast 0 from textFile at SVMLearning.scala:22 -2016-05-24 22:07:26 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 22:07:26 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:07:26 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 22:07:26 INFO DAGScheduler:59 - Got job 0 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 22:07:26 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:07:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:07:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:07:26 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 22:07:26 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=171580, maxMem=766075207 -2016-05-24 22:07:26 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-24 22:07:26 INFO MemoryStore:59 - ensureFreeSpace(2181) called with curMem=175316, maxMem=766075207 -2016-05-24 22:07:26 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:07:26 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:56492 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:07:26 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:07:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:07:26 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-24 22:07:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 22:07:26 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 22:07:26 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 22:07:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:0+38 -2016-05-24 22:07:26 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 22:07:26 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 22:07:26 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 22:07:26 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 22:07:26 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 22:07:26 INFO MemoryStore:59 - ensureFreeSpace(472) called with curMem=177497, maxMem=766075207 -2016-05-24 22:07:26 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 472.0 B, free 730.4 MB) -2016-05-24 22:07:26 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:56492 (size: 472.0 B, free: 730.6 MB) -2016-05-24 22:07:26 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2611 bytes result sent to driver -2016-05-24 22:07:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 115 ms on localhost (1/1) -2016-05-24 22:07:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 22:07:26 INFO DAGScheduler:59 - ResultStage 0 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.136 s -2016-05-24 22:07:26 INFO DAGScheduler:59 - Job 0 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.216014 s -2016-05-24 22:07:26 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 22:07:26 INFO DAGScheduler:59 - Got job 1 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 22:07:26 INFO DAGScheduler:59 - Final stage: ResultStage 1(count at DataValidators.scala:40) -2016-05-24 22:07:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:07:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:07:26 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 22:07:26 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=177969, maxMem=766075207 -2016-05-24 22:07:26 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 22:07:26 INFO MemoryStore:59 - ensureFreeSpace(2039) called with curMem=181481, maxMem=766075207 -2016-05-24 22:07:26 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2039.0 B, free 730.4 MB) -2016-05-24 22:07:26 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:56492 (size: 2039.0 B, free: 730.6 MB) -2016-05-24 22:07:26 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:07:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at filter at DataValidators.scala:40) -2016-05-24 22:07:26 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 22:07:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 22:07:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 22:07:26 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-24 22:07:26 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-24 22:07:27 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 22:07:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:38+39 -2016-05-24 22:07:27 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:07:27 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-05-24 22:07:27 INFO MemoryStore:59 - ensureFreeSpace(320) called with curMem=183520, maxMem=766075207 -2016-05-24 22:07:27 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 320.0 B, free 730.4 MB) -2016-05-24 22:07:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 31 ms on localhost (1/2) -2016-05-24 22:07:27 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:56492 (size: 320.0 B, free: 730.6 MB) -2016-05-24 22:07:27 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2662 bytes result sent to driver -2016-05-24 22:07:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 36 ms on localhost (2/2) -2016-05-24 22:07:27 INFO DAGScheduler:59 - ResultStage 1 (count at DataValidators.scala:40) finished in 0.040 s -2016-05-24 22:07:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 22:07:27 INFO DAGScheduler:59 - Job 1 finished: count at DataValidators.scala:40, took 0.079753 s -2016-05-24 22:07:27 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 22:07:27 INFO DAGScheduler:59 - Got job 2 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 22:07:27 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at GradientDescent.scala:195) -2016-05-24 22:07:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:07:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:07:27 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[5] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 22:07:27 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=183840, maxMem=766075207 -2016-05-24 22:07:27 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 22:07:27 INFO MemoryStore:59 - ensureFreeSpace(2018) called with curMem=187280, maxMem=766075207 -2016-05-24 22:07:27 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2018.0 B, free 730.4 MB) -2016-05-24 22:07:27 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:56492 (size: 2018.0 B, free: 730.6 MB) -2016-05-24 22:07:27 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:07:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[5] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 22:07:27 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 22:07:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 22:07:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 22:07:27 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 22:07:27 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 22:07:27 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:07:27 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:07:27 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 22:07:27 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 22:07:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 19 ms on localhost (1/2) -2016-05-24 22:07:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 23 ms on localhost (2/2) -2016-05-24 22:07:27 INFO DAGScheduler:59 - ResultStage 2 (count at GradientDescent.scala:195) finished in 0.023 s -2016-05-24 22:07:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 22:07:27 INFO DAGScheduler:59 - Job 2 finished: count at GradientDescent.scala:195, took 0.048332 s -2016-05-24 22:07:27 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:56492 in memory (size: 2018.0 B, free: 730.6 MB) -2016-05-24 22:07:27 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:56492 in memory (size: 2039.0 B, free: 730.6 MB) -2016-05-24 22:07:27 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:56492 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:07:27 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 22:07:27 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 22:07:27 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=172372, maxMem=766075207 -2016-05-24 22:07:27 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 22:07:27 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=172444, maxMem=766075207 -2016-05-24 22:07:27 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 22:07:27 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:56492 (size: 138.0 B, free: 730.6 MB) -2016-05-24 22:07:27 INFO SparkContext:59 - Created broadcast 4 from broadcast at GradientDescent.scala:221 -2016-05-24 22:07:27 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:07:27 INFO DAGScheduler:59 - Got job 3 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:07:27 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:07:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:07:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:07:27 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[7] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:07:27 INFO MemoryStore:59 - ensureFreeSpace(5512) called with curMem=172582, maxMem=766075207 -2016-05-24 22:07:27 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:07:27 INFO MemoryStore:59 - ensureFreeSpace(2961) called with curMem=178094, maxMem=766075207 -2016-05-24 22:07:27 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:07:27 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:56492 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:07:27 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:07:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[7] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:07:27 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 22:07:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:07:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:07:27 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 22:07:27 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 22:07:27 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:07:27 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:07:27 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2423 bytes result sent to driver -2016-05-24 22:07:27 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2423 bytes result sent to driver -2016-05-24 22:07:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 23 ms on localhost (1/2) -2016-05-24 22:07:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 28 ms on localhost (2/2) -2016-05-24 22:07:27 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 22:07:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 22:07:27 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at GradientDescent.scala:225, took 0.043632 s -2016-05-24 22:07:27 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=181055, maxMem=766075207 -2016-05-24 22:07:27 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-05-24 22:07:27 INFO MemoryStore:59 - ensureFreeSpace(138) called with curMem=181127, maxMem=766075207 -2016-05-24 22:07:27 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 138.0 B, free 730.4 MB) -2016-05-24 22:07:27 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:56492 (size: 138.0 B, free: 730.6 MB) -2016-05-24 22:07:27 INFO SparkContext:59 - Created broadcast 6 from broadcast at GradientDescent.scala:221 -2016-05-24 22:07:27 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:07:27 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:07:27 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:07:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:07:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:07:27 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:07:27 INFO MemoryStore:59 - ensureFreeSpace(5512) called with curMem=181265, maxMem=766075207 -2016-05-24 22:07:27 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:07:27 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=186777, maxMem=766075207 -2016-05-24 22:07:27 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:07:27 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:56492 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:07:27 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:07:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:07:27 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 22:07:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:07:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:07:27 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 22:07:27 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 22:07:27 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:07:27 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:07:27 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2423 bytes result sent to driver -2016-05-24 22:07:27 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2423 bytes result sent to driver -2016-05-24 22:07:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 14 ms on localhost (1/2) -2016-05-24 22:07:27 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:07:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 17 ms on localhost (2/2) -2016-05-24 22:07:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 22:07:27 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.031885 s -2016-05-24 22:07:27 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 1.0, 1.0 -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 22:07:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 22:07:27 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 22:07:27 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 22:07:27 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 22:07:27 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 22:07:27 INFO BlockManager:59 - BlockManager stopped -2016-05-24 22:07:27 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 22:07:27 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 22:07:27 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 22:07:27 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 22:07:27 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-497f539c-c728-4ec4-ad63-67cf31b3ee7a -2016-05-24 22:07:27 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 22:08:35 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 22:08:36 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 22:08:36 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 22:08:36 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 22:08:36 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 22:08:37 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 22:08:37 INFO Remoting:74 - Starting remoting -2016-05-24 22:08:37 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:56517] -2016-05-24 22:08:37 INFO Utils:59 - Successfully started service 'sparkDriver' on port 56517. -2016-05-24 22:08:37 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 22:08:37 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 22:08:37 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-9bff86b1-d1b6-4cf0-9bda-c2a76eb90545 -2016-05-24 22:08:37 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 22:08:37 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-3f4996fa-e754-43cb-a5d4-ea88a319e0d9\httpd-86743b7b-6a78-454d-93ea-abde343e77a3 -2016-05-24 22:08:37 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 22:08:37 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:08:37 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:56518 -2016-05-24 22:08:37 INFO Utils:59 - Successfully started service 'HTTP file server' on port 56518. -2016-05-24 22:08:37 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 22:08:37 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:08:37 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 22:08:37 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 22:08:37 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 22:08:37 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 22:08:37 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 22:08:38 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 56537. -2016-05-24 22:08:38 INFO NettyBlockTransferService:59 - Server created on 56537 -2016-05-24 22:08:38 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 22:08:38 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:56537 with 730.6 MB RAM, BlockManagerId(driver, localhost, 56537) -2016-05-24 22:08:38 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 22:08:39 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 22:08:39 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 22:08:39 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 22:08:39 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 22:08:39 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:56537 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 22:08:39 INFO SparkContext:59 - Created broadcast 0 from textFile at SVMLearning.scala:22 -2016-05-24 22:08:40 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 22:08:40 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:08:40 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 22:08:40 INFO DAGScheduler:59 - Got job 0 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 22:08:40 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:08:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:08:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:08:40 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 22:08:40 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=171580, maxMem=766075207 -2016-05-24 22:08:40 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-24 22:08:40 INFO MemoryStore:59 - ensureFreeSpace(2181) called with curMem=175316, maxMem=766075207 -2016-05-24 22:08:40 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:08:40 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:56537 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:08:40 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:08:40 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:08:40 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-24 22:08:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 22:08:40 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 22:08:40 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 22:08:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:0+38 -2016-05-24 22:08:40 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 22:08:40 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 22:08:40 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 22:08:40 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 22:08:40 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 22:08:40 INFO MemoryStore:59 - ensureFreeSpace(424) called with curMem=177497, maxMem=766075207 -2016-05-24 22:08:40 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 424.0 B, free 730.4 MB) -2016-05-24 22:08:40 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:56537 (size: 424.0 B, free: 730.6 MB) -2016-05-24 22:08:40 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2611 bytes result sent to driver -2016-05-24 22:08:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 116 ms on localhost (1/1) -2016-05-24 22:08:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 22:08:40 INFO DAGScheduler:59 - ResultStage 0 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.139 s -2016-05-24 22:08:40 INFO DAGScheduler:59 - Job 0 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.244408 s -2016-05-24 22:08:40 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 22:08:40 INFO DAGScheduler:59 - Got job 1 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 22:08:40 INFO DAGScheduler:59 - Final stage: ResultStage 1(count at DataValidators.scala:40) -2016-05-24 22:08:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:08:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:08:40 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 22:08:40 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=177921, maxMem=766075207 -2016-05-24 22:08:40 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 22:08:40 INFO MemoryStore:59 - ensureFreeSpace(2039) called with curMem=181433, maxMem=766075207 -2016-05-24 22:08:40 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2039.0 B, free 730.4 MB) -2016-05-24 22:08:40 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:56537 (size: 2039.0 B, free: 730.6 MB) -2016-05-24 22:08:41 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:08:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at filter at DataValidators.scala:40) -2016-05-24 22:08:41 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 22:08:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 22:08:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 22:08:41 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-24 22:08:41 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-24 22:08:41 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 22:08:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:38+39 -2016-05-24 22:08:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:08:41 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-05-24 22:08:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 25 ms on localhost (1/2) -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=183472, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-05-24 22:08:41 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:56537 (size: 288.0 B, free: 730.6 MB) -2016-05-24 22:08:41 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2662 bytes result sent to driver -2016-05-24 22:08:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 30 ms on localhost (2/2) -2016-05-24 22:08:41 INFO DAGScheduler:59 - ResultStage 1 (count at DataValidators.scala:40) finished in 0.035 s -2016-05-24 22:08:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 22:08:41 INFO DAGScheduler:59 - Job 1 finished: count at DataValidators.scala:40, took 0.098281 s -2016-05-24 22:08:41 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 22:08:41 INFO DAGScheduler:59 - Got job 2 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 22:08:41 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at GradientDescent.scala:195) -2016-05-24 22:08:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:08:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:08:41 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[5] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=183760, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(2018) called with curMem=187200, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2018.0 B, free 730.4 MB) -2016-05-24 22:08:41 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:56537 (size: 2018.0 B, free: 730.6 MB) -2016-05-24 22:08:41 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:08:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[5] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 22:08:41 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 22:08:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 22:08:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 22:08:41 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 22:08:41 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 22:08:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:08:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:08:41 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 22:08:41 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 22:08:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 14 ms on localhost (1/2) -2016-05-24 22:08:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 17 ms on localhost (2/2) -2016-05-24 22:08:41 INFO DAGScheduler:59 - ResultStage 2 (count at GradientDescent.scala:195) finished in 0.017 s -2016-05-24 22:08:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 22:08:41 INFO DAGScheduler:59 - Job 2 finished: count at GradientDescent.scala:195, took 0.033560 s -2016-05-24 22:08:41 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:56537 in memory (size: 2018.0 B, free: 730.6 MB) -2016-05-24 22:08:41 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:56537 in memory (size: 2039.0 B, free: 730.6 MB) -2016-05-24 22:08:41 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:56537 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:08:41 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 22:08:41 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=172292, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=172356, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 22:08:41 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:56537 (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:08:41 INFO SparkContext:59 - Created broadcast 4 from broadcast at GradientDescent.scala:221 -2016-05-24 22:08:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:08:41 INFO DAGScheduler:59 - Got job 3 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:08:41 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:08:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:08:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:08:41 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[7] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(5504) called with curMem=172486, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(2959) called with curMem=177990, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:08:41 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:56537 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:08:41 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:08:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[7] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:08:41 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 22:08:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:08:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:08:41 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 22:08:41 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 22:08:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:08:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:08:41 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2415 bytes result sent to driver -2016-05-24 22:08:41 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2415 bytes result sent to driver -2016-05-24 22:08:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 25 ms on localhost (1/2) -2016-05-24 22:08:41 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 22:08:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 23 ms on localhost (2/2) -2016-05-24 22:08:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 22:08:41 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at GradientDescent.scala:225, took 0.042698 s -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=180949, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=181013, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 22:08:41 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:56537 (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:08:41 INFO SparkContext:59 - Created broadcast 6 from broadcast at GradientDescent.scala:221 -2016-05-24 22:08:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:08:41 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:08:41 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:08:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:08:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:08:41 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(5504) called with curMem=181143, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=186647, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:08:41 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:56537 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:08:41 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:08:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:08:41 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 22:08:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:08:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:08:41 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 22:08:41 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 22:08:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:08:41 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2415 bytes result sent to driver -2016-05-24 22:08:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:08:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 18 ms on localhost (1/2) -2016-05-24 22:08:41 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2415 bytes result sent to driver -2016-05-24 22:08:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 24 ms on localhost (2/2) -2016-05-24 22:08:41 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 22:08:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 22:08:41 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.042365 s -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=189607, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=189671, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 22:08:41 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:56537 (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:08:41 INFO SparkContext:59 - Created broadcast 8 from broadcast at GradientDescent.scala:221 -2016-05-24 22:08:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:08:41 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:08:41 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:08:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:08:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:08:41 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[11] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(5504) called with curMem=189801, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=195305, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:08:41 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:56537 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:08:41 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:08:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[11] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:08:41 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 22:08:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:08:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:08:41 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 22:08:41 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 22:08:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:08:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:08:41 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2415 bytes result sent to driver -2016-05-24 22:08:41 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2415 bytes result sent to driver -2016-05-24 22:08:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 16 ms on localhost (1/2) -2016-05-24 22:08:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 17 ms on localhost (2/2) -2016-05-24 22:08:41 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:08:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 22:08:41 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.038220 s -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=198265, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=198329, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 22:08:41 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:56537 (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:08:41 INFO SparkContext:59 - Created broadcast 10 from broadcast at GradientDescent.scala:221 -2016-05-24 22:08:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:08:41 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:08:41 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:08:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:08:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:08:41 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(5504) called with curMem=198459, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(2958) called with curMem=203963, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:08:41 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:56537 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:08:41 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:08:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:08:41 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 22:08:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:08:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:08:41 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 22:08:41 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 22:08:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:08:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:08:41 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2415 bytes result sent to driver -2016-05-24 22:08:41 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2415 bytes result sent to driver -2016-05-24 22:08:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 19 ms on localhost (1/2) -2016-05-24 22:08:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 20 ms on localhost (2/2) -2016-05-24 22:08:41 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 22:08:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 22:08:41 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.040250 s -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=206921, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=206985, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 22:08:41 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:56537 (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:08:41 INFO SparkContext:59 - Created broadcast 12 from broadcast at GradientDescent.scala:221 -2016-05-24 22:08:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:08:41 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:08:41 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:08:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:08:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:08:41 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(5504) called with curMem=207115, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(2959) called with curMem=212619, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:08:41 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:56537 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:08:41 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:08:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:08:41 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 22:08:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:08:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:08:41 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 22:08:41 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 22:08:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:08:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:08:41 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2415 bytes result sent to driver -2016-05-24 22:08:41 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2415 bytes result sent to driver -2016-05-24 22:08:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 15 ms on localhost (1/2) -2016-05-24 22:08:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 17 ms on localhost (2/2) -2016-05-24 22:08:41 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:08:41 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.037517 s -2016-05-24 22:08:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=215578, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=215642, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 22:08:41 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:56537 (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:08:41 INFO SparkContext:59 - Created broadcast 14 from broadcast at GradientDescent.scala:221 -2016-05-24 22:08:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:08:41 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:08:41 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:08:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:08:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:08:41 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(5504) called with curMem=215772, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=221276, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:08:41 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:56537 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:08:41 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:08:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:08:41 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 22:08:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:08:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:08:41 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 22:08:41 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 22:08:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:08:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:08:41 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2415 bytes result sent to driver -2016-05-24 22:08:41 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2415 bytes result sent to driver -2016-05-24 22:08:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 18 ms on localhost (1/2) -2016-05-24 22:08:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 20 ms on localhost (2/2) -2016-05-24 22:08:41 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:08:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 22:08:41 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.036704 s -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224236, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224300, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 22:08:41 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:56537 (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:08:41 INFO SparkContext:59 - Created broadcast 16 from broadcast at GradientDescent.scala:221 -2016-05-24 22:08:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:08:41 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:08:41 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:08:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:08:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:08:41 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(5504) called with curMem=224430, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:08:41 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=229934, maxMem=766075207 -2016-05-24 22:08:41 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:08:41 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:56537 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:08:41 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:08:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:08:41 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 22:08:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:08:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:08:41 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 22:08:41 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 22:08:41 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:08:41 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:56537 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:08:41 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2415 bytes result sent to driver -2016-05-24 22:08:41 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:08:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 35 ms on localhost (1/2) -2016-05-24 22:08:41 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2415 bytes result sent to driver -2016-05-24 22:08:42 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 22:08:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 41 ms on localhost (2/2) -2016-05-24 22:08:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 22:08:42 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.041 s -2016-05-24 22:08:42 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.057534 s -2016-05-24 22:08:42 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:56537 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:08:42 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224300, maxMem=766075207 -2016-05-24 22:08:42 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 22:08:42 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=221341, maxMem=766075207 -2016-05-24 22:08:42 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 22:08:42 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:56537 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:08:42 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 22:08:42 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:56537 (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:08:42 INFO SparkContext:59 - Created broadcast 18 from broadcast at GradientDescent.scala:221 -2016-05-24 22:08:42 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:56537 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:08:42 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:56537 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:08:42 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 22:08:42 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:56537 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:08:42 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:56537 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:08:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:08:42 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 22:08:42 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:08:42 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:08:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:08:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:08:42 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:08:42 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:56537 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:08:42 INFO MemoryStore:59 - ensureFreeSpace(5504) called with curMem=189995, maxMem=766075207 -2016-05-24 22:08:42 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:08:42 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:56537 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:08:42 INFO MemoryStore:59 - ensureFreeSpace(2959) called with curMem=195499, maxMem=766075207 -2016-05-24 22:08:42 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 22:08:42 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:08:42 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:56537 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:08:42 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:08:42 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:56537 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:08:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:08:42 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 22:08:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:08:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:08:42 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:56537 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:08:42 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 22:08:42 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 22:08:42 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 22:08:42 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:56537 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:08:42 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:08:42 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:08:42 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2415 bytes result sent to driver -2016-05-24 22:08:42 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2415 bytes result sent to driver -2016-05-24 22:08:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 15 ms on localhost (1/2) -2016-05-24 22:08:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 18 ms on localhost (2/2) -2016-05-24 22:08:42 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 22:08:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 22:08:42 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.038228 s -2016-05-24 22:08:42 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=189607, maxMem=766075207 -2016-05-24 22:08:42 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 22:08:42 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=189671, maxMem=766075207 -2016-05-24 22:08:42 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 22:08:42 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:56537 (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:08:42 INFO SparkContext:59 - Created broadcast 20 from broadcast at GradientDescent.scala:221 -2016-05-24 22:08:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:08:42 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:08:42 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:08:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:08:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:08:42 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:08:42 INFO MemoryStore:59 - ensureFreeSpace(5504) called with curMem=189801, maxMem=766075207 -2016-05-24 22:08:42 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:08:42 INFO MemoryStore:59 - ensureFreeSpace(2958) called with curMem=195305, maxMem=766075207 -2016-05-24 22:08:42 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:08:42 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:56537 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:08:42 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:08:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:08:42 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 22:08:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:08:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:08:42 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 22:08:42 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 22:08:42 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:08:42 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:08:42 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2415 bytes result sent to driver -2016-05-24 22:08:42 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2415 bytes result sent to driver -2016-05-24 22:08:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 11 ms on localhost (1/2) -2016-05-24 22:08:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 13 ms on localhost (2/2) -2016-05-24 22:08:42 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:08:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 22:08:42 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.033163 s -2016-05-24 22:08:42 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=198263, maxMem=766075207 -2016-05-24 22:08:42 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 22:08:42 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=198327, maxMem=766075207 -2016-05-24 22:08:42 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 22:08:42 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:56537 (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:08:42 INFO SparkContext:59 - Created broadcast 22 from broadcast at GradientDescent.scala:221 -2016-05-24 22:08:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:08:42 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:08:42 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:08:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:08:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:08:42 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:08:42 INFO MemoryStore:59 - ensureFreeSpace(5504) called with curMem=198457, maxMem=766075207 -2016-05-24 22:08:42 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:08:42 INFO MemoryStore:59 - ensureFreeSpace(2959) called with curMem=203961, maxMem=766075207 -2016-05-24 22:08:42 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:08:42 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:56537 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:08:42 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:08:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:08:42 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 22:08:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:08:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:08:42 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 22:08:42 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 22:08:42 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:08:42 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:08:42 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2415 bytes result sent to driver -2016-05-24 22:08:42 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2415 bytes result sent to driver -2016-05-24 22:08:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 14 ms on localhost (1/2) -2016-05-24 22:08:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 14 ms on localhost (2/2) -2016-05-24 22:08:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 22:08:42 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:08:42 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.033760 s -2016-05-24 22:08:42 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 1.0, 5.53125, 2.6419218178969137, 1.231939268520448, 2.822111994463019, 1.0165344536191285, 3.0190880547164793, 2.7935481462482343, 1.3656454187550682, 1.6489867875360198 -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 22:08:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 22:08:42 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 22:08:42 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 22:08:42 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 22:08:42 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 22:08:42 INFO BlockManager:59 - BlockManager stopped -2016-05-24 22:08:42 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 22:08:42 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 22:08:42 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 22:08:42 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 22:08:42 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-3f4996fa-e754-43cb-a5d4-ea88a319e0d9 -2016-05-24 22:08:42 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 22:09:28 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 22:09:29 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 22:09:29 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 22:09:29 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 22:09:29 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 22:09:30 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 22:09:30 INFO Remoting:74 - Starting remoting -2016-05-24 22:09:31 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:56561] -2016-05-24 22:09:31 INFO Utils:59 - Successfully started service 'sparkDriver' on port 56561. -2016-05-24 22:09:31 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 22:09:31 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 22:09:31 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-c1fda440-31cf-4095-90f6-592f73d09176 -2016-05-24 22:09:31 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 22:09:31 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-b3800e61-b958-44d3-9009-32be1ca62b96\httpd-177cc053-f35c-400d-9ce0-0e309e3d00d4 -2016-05-24 22:09:31 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 22:09:31 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:09:31 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:56562 -2016-05-24 22:09:31 INFO Utils:59 - Successfully started service 'HTTP file server' on port 56562. -2016-05-24 22:09:31 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 22:09:31 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:09:31 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 22:09:31 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 22:09:31 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 22:09:31 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 22:09:31 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 22:09:32 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 56581. -2016-05-24 22:09:32 INFO NettyBlockTransferService:59 - Server created on 56581 -2016-05-24 22:09:32 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 22:09:32 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:56581 with 730.6 MB RAM, BlockManagerId(driver, localhost, 56581) -2016-05-24 22:09:32 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 22:09:32 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 22:09:32 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 22:09:32 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 22:09:32 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 22:09:32 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:56581 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 22:09:32 INFO SparkContext:59 - Created broadcast 0 from textFile at SVMLearning.scala:22 -2016-05-24 22:09:34 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 22:09:34 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:09:34 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 22:09:34 INFO DAGScheduler:59 - Got job 0 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 22:09:34 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:09:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:09:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:09:34 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 22:09:34 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=171580, maxMem=766075207 -2016-05-24 22:09:34 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-24 22:09:34 INFO MemoryStore:59 - ensureFreeSpace(2181) called with curMem=175316, maxMem=766075207 -2016-05-24 22:09:34 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:09:34 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:56581 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:09:34 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:09:34 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:09:34 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-05-24 22:09:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 22:09:34 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 22:09:34 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-05-24 22:09:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:0+38 -2016-05-24 22:09:34 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 22:09:34 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 22:09:34 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 22:09:34 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 22:09:34 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 22:09:34 INFO MemoryStore:59 - ensureFreeSpace(424) called with curMem=177497, maxMem=766075207 -2016-05-24 22:09:34 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 424.0 B, free 730.4 MB) -2016-05-24 22:09:34 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:56581 (size: 424.0 B, free: 730.6 MB) -2016-05-24 22:09:34 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2611 bytes result sent to driver -2016-05-24 22:09:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 118 ms on localhost (1/1) -2016-05-24 22:09:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 22:09:34 INFO DAGScheduler:59 - ResultStage 0 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.139 s -2016-05-24 22:09:34 INFO DAGScheduler:59 - Job 0 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.247190 s -2016-05-24 22:09:34 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 22:09:34 INFO DAGScheduler:59 - Got job 1 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 22:09:34 INFO DAGScheduler:59 - Final stage: ResultStage 1(count at DataValidators.scala:40) -2016-05-24 22:09:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:09:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:09:34 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 22:09:34 INFO MemoryStore:59 - ensureFreeSpace(3512) called with curMem=177921, maxMem=766075207 -2016-05-24 22:09:34 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 22:09:34 INFO MemoryStore:59 - ensureFreeSpace(2039) called with curMem=181433, maxMem=766075207 -2016-05-24 22:09:34 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2039.0 B, free 730.4 MB) -2016-05-24 22:09:34 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:56581 (size: 2039.0 B, free: 730.6 MB) -2016-05-24 22:09:34 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:09:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at filter at DataValidators.scala:40) -2016-05-24 22:09:34 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 22:09:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 22:09:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 22:09:34 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-05-24 22:09:34 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-05-24 22:09:34 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:09:34 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-05-24 22:09:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/logisticRegression1.data:38+39 -2016-05-24 22:09:34 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-05-24 22:09:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 32 ms on localhost (1/2) -2016-05-24 22:09:34 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=183472, maxMem=766075207 -2016-05-24 22:09:34 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-05-24 22:09:34 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:56581 (size: 288.0 B, free: 730.6 MB) -2016-05-24 22:09:34 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2662 bytes result sent to driver -2016-05-24 22:09:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 45 ms on localhost (2/2) -2016-05-24 22:09:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 22:09:34 INFO DAGScheduler:59 - ResultStage 1 (count at DataValidators.scala:40) finished in 0.052 s -2016-05-24 22:09:34 INFO DAGScheduler:59 - Job 1 finished: count at DataValidators.scala:40, took 0.086185 s -2016-05-24 22:09:34 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 22:09:34 INFO DAGScheduler:59 - Got job 2 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 22:09:34 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at GradientDescent.scala:195) -2016-05-24 22:09:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:09:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:09:34 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[5] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 22:09:34 INFO MemoryStore:59 - ensureFreeSpace(3440) called with curMem=183760, maxMem=766075207 -2016-05-24 22:09:34 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-24 22:09:34 INFO MemoryStore:59 - ensureFreeSpace(2018) called with curMem=187200, maxMem=766075207 -2016-05-24 22:09:34 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2018.0 B, free 730.4 MB) -2016-05-24 22:09:34 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:56581 (size: 2018.0 B, free: 730.6 MB) -2016-05-24 22:09:34 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:09:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[5] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 22:09:34 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 22:09:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 22:09:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2198 bytes) -2016-05-24 22:09:34 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 22:09:34 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 22:09:34 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:09:34 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:09:34 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 22:09:34 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 22:09:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 17 ms on localhost (1/2) -2016-05-24 22:09:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 17 ms on localhost (2/2) -2016-05-24 22:09:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 22:09:34 INFO DAGScheduler:59 - ResultStage 2 (count at GradientDescent.scala:195) finished in 0.021 s -2016-05-24 22:09:34 INFO DAGScheduler:59 - Job 2 finished: count at GradientDescent.scala:195, took 0.038742 s -2016-05-24 22:09:34 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:56581 in memory (size: 2018.0 B, free: 730.6 MB) -2016-05-24 22:09:34 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 22:09:34 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:56581 in memory (size: 2039.0 B, free: 730.6 MB) -2016-05-24 22:09:34 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:56581 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:09:35 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 22:09:35 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 22:09:35 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=172292, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=172356, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:56581 (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:09:35 INFO SparkContext:59 - Created broadcast 4 from broadcast at GradientDescent.scala:221 -2016-05-24 22:09:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:09:35 INFO DAGScheduler:59 - Got job 3 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:09:35 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:09:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:09:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:09:35 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[7] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(5504) called with curMem=172486, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(2959) called with curMem=177990, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:56581 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:09:35 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:09:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[7] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:09:35 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 22:09:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:09:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:09:35 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 22:09:35 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 22:09:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:09:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:09:35 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2415 bytes result sent to driver -2016-05-24 22:09:35 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2415 bytes result sent to driver -2016-05-24 22:09:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 20 ms on localhost (1/2) -2016-05-24 22:09:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 25 ms on localhost (2/2) -2016-05-24 22:09:35 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at GradientDescent.scala:225) finished in 0.025 s -2016-05-24 22:09:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 22:09:35 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at GradientDescent.scala:225, took 0.042429 s -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=180949, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=181013, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:56581 (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:09:35 INFO SparkContext:59 - Created broadcast 6 from broadcast at GradientDescent.scala:221 -2016-05-24 22:09:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:09:35 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:09:35 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:09:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:09:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:09:35 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(5504) called with curMem=181143, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=186647, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:56581 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:09:35 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:09:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:09:35 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 22:09:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:09:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:09:35 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 22:09:35 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 22:09:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:09:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:09:35 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2415 bytes result sent to driver -2016-05-24 22:09:35 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2415 bytes result sent to driver -2016-05-24 22:09:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 18 ms on localhost (1/2) -2016-05-24 22:09:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 18 ms on localhost (2/2) -2016-05-24 22:09:35 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:09:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 22:09:35 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.035288 s -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=189607, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=189671, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:56581 (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:09:35 INFO SparkContext:59 - Created broadcast 8 from broadcast at GradientDescent.scala:221 -2016-05-24 22:09:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:09:35 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:09:35 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:09:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:09:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:09:35 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[11] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(5504) called with curMem=189801, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=195305, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:56581 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:09:35 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:09:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[11] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:09:35 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 22:09:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:09:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:09:35 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 22:09:35 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 22:09:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:09:35 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2415 bytes result sent to driver -2016-05-24 22:09:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:09:35 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2415 bytes result sent to driver -2016-05-24 22:09:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 18 ms on localhost (1/2) -2016-05-24 22:09:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 19 ms on localhost (2/2) -2016-05-24 22:09:35 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:09:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 22:09:35 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.042499 s -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=198265, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=198329, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:56581 (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:09:35 INFO SparkContext:59 - Created broadcast 10 from broadcast at GradientDescent.scala:221 -2016-05-24 22:09:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:09:35 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:09:35 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:09:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:09:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:09:35 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(5504) called with curMem=198459, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(2958) called with curMem=203963, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:56581 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:09:35 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:09:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:09:35 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 22:09:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:09:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:09:35 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 22:09:35 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 22:09:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:09:35 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2415 bytes result sent to driver -2016-05-24 22:09:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:09:35 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2415 bytes result sent to driver -2016-05-24 22:09:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 21 ms on localhost (1/2) -2016-05-24 22:09:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 27 ms on localhost (2/2) -2016-05-24 22:09:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 22:09:35 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.029 s -2016-05-24 22:09:35 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.051372 s -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=206921, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=206985, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:56581 (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:09:35 INFO SparkContext:59 - Created broadcast 12 from broadcast at GradientDescent.scala:221 -2016-05-24 22:09:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:09:35 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:09:35 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:09:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:09:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:09:35 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(5504) called with curMem=207115, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=212619, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:56581 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:09:35 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:09:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:09:35 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 22:09:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:09:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:09:35 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 22:09:35 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 22:09:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:09:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:09:35 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2415 bytes result sent to driver -2016-05-24 22:09:35 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2415 bytes result sent to driver -2016-05-24 22:09:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 17 ms on localhost (1/2) -2016-05-24 22:09:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 18 ms on localhost (2/2) -2016-05-24 22:09:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 22:09:35 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:09:35 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.050161 s -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=215579, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=215643, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:56581 (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:09:35 INFO SparkContext:59 - Created broadcast 14 from broadcast at GradientDescent.scala:221 -2016-05-24 22:09:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:09:35 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:09:35 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:09:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:09:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:09:35 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(5504) called with curMem=215773, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(2959) called with curMem=221277, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:56581 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:09:35 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:09:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:09:35 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 22:09:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:09:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:09:35 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 22:09:35 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 22:09:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:09:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:09:35 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2415 bytes result sent to driver -2016-05-24 22:09:35 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2415 bytes result sent to driver -2016-05-24 22:09:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 19 ms on localhost (1/2) -2016-05-24 22:09:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 17 ms on localhost (2/2) -2016-05-24 22:09:35 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:09:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 22:09:35 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.039890 s -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=224236, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=224300, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:56581 (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:09:35 INFO SparkContext:59 - Created broadcast 16 from broadcast at GradientDescent.scala:221 -2016-05-24 22:09:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:09:35 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:09:35 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:09:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:09:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:09:35 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(5504) called with curMem=224430, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=229934, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:56581 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:09:35 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:09:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:09:35 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 22:09:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:09:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:09:35 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 22:09:35 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 22:09:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:09:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:09:35 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2415 bytes result sent to driver -2016-05-24 22:09:35 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2415 bytes result sent to driver -2016-05-24 22:09:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 16 ms on localhost (1/2) -2016-05-24 22:09:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 18 ms on localhost (2/2) -2016-05-24 22:09:35 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 22:09:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 22:09:35 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.035023 s -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=232894, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=232958, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:56581 (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:09:35 INFO SparkContext:59 - Created broadcast 18 from broadcast at GradientDescent.scala:221 -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:56581 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:09:35 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:56581 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:56581 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:09:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:09:35 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 22:09:35 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:09:35 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:09:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:09:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:09:35 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:56581 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(5504) called with curMem=215773, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=218317, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:56581 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:56581 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:09:35 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 22:09:35 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:09:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:09:35 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 22:09:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:56581 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:09:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:09:35 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 22:09:35 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:56581 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:09:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:09:35 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2415 bytes result sent to driver -2016-05-24 22:09:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:09:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 19 ms on localhost (1/2) -2016-05-24 22:09:35 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 22:09:35 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2415 bytes result sent to driver -2016-05-24 22:09:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 22 ms on localhost (2/2) -2016-05-24 22:09:35 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 22:09:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:56581 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:09:35 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.045581 s -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=198459, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:56581 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:09:35 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=198329, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:56581 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:56581 (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:09:35 INFO SparkContext:59 - Created broadcast 20 from broadcast at GradientDescent.scala:221 -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:56581 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:09:35 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:56581 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:56581 in memory (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:09:35 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 22:09:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:56581 in memory (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:09:35 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:09:35 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:09:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:09:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:09:35 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(5504) called with curMem=181144, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(2959) called with curMem=186648, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:56581 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:09:35 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:09:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:09:35 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 22:09:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:09:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:09:35 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 22:09:35 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 22:09:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:09:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:09:35 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2415 bytes result sent to driver -2016-05-24 22:09:35 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2415 bytes result sent to driver -2016-05-24 22:09:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 13 ms on localhost (1/2) -2016-05-24 22:09:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 16 ms on localhost (2/2) -2016-05-24 22:09:35 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:09:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 22:09:35 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.035406 s -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=189607, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 64.0 B, free 730.4 MB) -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(130) called with curMem=189671, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 130.0 B, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:56581 (size: 130.0 B, free: 730.6 MB) -2016-05-24 22:09:35 INFO SparkContext:59 - Created broadcast 22 from broadcast at GradientDescent.scala:221 -2016-05-24 22:09:35 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:09:35 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:09:35 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:09:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:09:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:09:35 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(5504) called with curMem=189801, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-24 22:09:35 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=195305, maxMem=766075207 -2016-05-24 22:09:35 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:09:35 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:56581 (size: 2.9 KB, free: 730.6 MB) -2016-05-24 22:09:35 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:09:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:09:35 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 22:09:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:09:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-24 22:09:35 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 22:09:35 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 22:09:35 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-05-24 22:09:35 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-05-24 22:09:35 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2415 bytes result sent to driver -2016-05-24 22:09:35 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2415 bytes result sent to driver -2016-05-24 22:09:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 16 ms on localhost (1/2) -2016-05-24 22:09:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 16 ms on localhost (2/2) -2016-05-24 22:09:35 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 22:09:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 22:09:35 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.034133 s -2016-05-24 22:09:35 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 1.0, 5.53125, 2.6419218178969137, 1.231939268520448, 2.822111994463019, 1.0165344536191285, 3.0190880547164793, 2.7935481462482343, 1.3656454187550682, 1.6489867875360198 -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 22:09:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 22:09:36 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 22:09:36 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 22:09:36 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 22:09:36 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 22:09:36 INFO BlockManager:59 - BlockManager stopped -2016-05-24 22:09:36 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 22:09:36 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 22:09:36 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 22:09:36 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 22:09:36 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-b3800e61-b958-44d3-9009-32be1ca62b96 -2016-05-24 22:09:36 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 22:09:36 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 22:14:08 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 22:14:09 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 22:14:09 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 22:14:09 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 22:14:09 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 22:14:10 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 22:14:10 INFO Remoting:74 - Starting remoting -2016-05-24 22:14:10 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:56644] -2016-05-24 22:14:10 INFO Utils:59 - Successfully started service 'sparkDriver' on port 56644. -2016-05-24 22:14:10 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 22:14:10 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 22:14:10 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-ddbce537-07e9-4579-be0e-4659f01e3ad7 -2016-05-24 22:14:10 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 22:14:10 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-604c0433-83d9-468b-a3f2-3b173042361c\httpd-e07e3499-2469-4691-b153-446850a58867 -2016-05-24 22:14:10 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 22:14:10 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:14:10 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:56645 -2016-05-24 22:14:10 INFO Utils:59 - Successfully started service 'HTTP file server' on port 56645. -2016-05-24 22:14:10 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 22:14:11 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:14:11 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 22:14:11 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 22:14:11 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 22:14:11 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 22:14:11 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 22:14:11 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 56664. -2016-05-24 22:14:11 INFO NettyBlockTransferService:59 - Server created on 56664 -2016-05-24 22:14:11 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 22:14:11 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:56664 with 730.6 MB RAM, BlockManagerId(driver, localhost, 56664) -2016-05-24 22:14:11 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 22:14:12 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 22:14:12 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 22:14:12 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 22:14:12 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 22:14:12 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:56664 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 22:14:12 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 22:14:13 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 22:14:13 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:14:13 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 22:14:13 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 22:14:13 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 22:14:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:13 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 22:14:14 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 22:14:14 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 22:14:14 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=175476, maxMem=766075207 -2016-05-24 22:14:14 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:14:14 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:56664 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:14:14 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 22:14:14 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 22:14:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:14:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:14:14 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 22:14:14 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 22:14:14 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 22:14:14 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 22:14:14 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 22:14:14 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 22:14:14 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 22:14:14 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 22:14:14 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 22:14:14 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 22:14:14 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 22:14:14 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177621, maxMem=766075207 -2016-05-24 22:14:14 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 22:14:14 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:56664 (size: 83.6 KB, free: 730.5 MB) -2016-05-24 22:14:14 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 22:14:14 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263269, maxMem=766075207 -2016-05-24 22:14:14 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 22:14:14 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:56664 (size: 83.1 KB, free: 730.4 MB) -2016-05-24 22:14:14 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 22:14:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 623 ms on localhost (1/2) -2016-05-24 22:14:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 589 ms on localhost (2/2) -2016-05-24 22:14:14 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.654 s -2016-05-24 22:14:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 22:14:14 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.865931 s -2016-05-24 22:14:14 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 22:14:14 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 22:14:14 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:14:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:14 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 22:14:14 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=348349, maxMem=766075207 -2016-05-24 22:14:14 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-24 22:14:14 INFO MemoryStore:59 - ensureFreeSpace(2483) called with curMem=352909, maxMem=766075207 -2016-05-24 22:14:14 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-24 22:14:14 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:56664 (size: 2.4 KB, free: 730.4 MB) -2016-05-24 22:14:14 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:14 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:14:14 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 22:14:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:14:14 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 22:14:14 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-05-24 22:14:14 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:14:14 INFO MemoryStore:59 - ensureFreeSpace(54392) called with curMem=355392, maxMem=766075207 -2016-05-24 22:14:14 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 53.1 KB, free 730.2 MB) -2016-05-24 22:14:14 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:56664 (size: 53.1 KB, free: 730.4 MB) -2016-05-24 22:14:14 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2611 bytes result sent to driver -2016-05-24 22:14:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 33 ms on localhost (1/1) -2016-05-24 22:14:14 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.033 s -2016-05-24 22:14:14 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.051152 s -2016-05-24 22:14:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 22:14:14 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 22:14:14 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 22:14:14 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 22:14:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:14 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 22:14:14 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=409784, maxMem=766075207 -2016-05-24 22:14:14 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-24 22:14:14 INFO MemoryStore:59 - ensureFreeSpace(2300) called with curMem=414032, maxMem=766075207 -2016-05-24 22:14:14 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 22:14:14 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:56664 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 22:14:14 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40) -2016-05-24 22:14:14 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 22:14:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:14:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:14:14 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 22:14:14 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 22:14:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:14 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-05-24 22:14:14 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:14:14 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 22:14:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 13 ms on localhost (1/2) -2016-05-24 22:14:14 INFO MemoryStore:59 - ensureFreeSpace(62184) called with curMem=416332, maxMem=766075207 -2016-05-24 22:14:14 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 60.7 KB, free 730.1 MB) -2016-05-24 22:14:14 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:56664 (size: 60.7 KB, free: 730.3 MB) -2016-05-24 22:14:14 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2662 bytes result sent to driver -2016-05-24 22:14:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 24 ms on localhost (2/2) -2016-05-24 22:14:14 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.025 s -2016-05-24 22:14:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 22:14:14 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.039408 s -2016-05-24 22:14:14 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 22:14:14 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 22:14:14 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 22:14:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:14 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 22:14:14 INFO MemoryStore:59 - ensureFreeSpace(4264) called with curMem=478516, maxMem=766075207 -2016-05-24 22:14:14 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.2 KB, free 730.1 MB) -2016-05-24 22:14:14 INFO MemoryStore:59 - ensureFreeSpace(2322) called with curMem=482780, maxMem=766075207 -2016-05-24 22:14:14 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 22:14:14 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:56664 (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:14:14 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 22:14:14 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 22:14:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:14:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:14:14 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 22:14:14 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 22:14:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:14 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 22:14:14 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 22:14:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 19 ms on localhost (1/2) -2016-05-24 22:14:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 19 ms on localhost (2/2) -2016-05-24 22:14:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 22:14:14 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.020 s -2016-05-24 22:14:14 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.035692 s -2016-05-24 22:14:15 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 22:14:15 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=485102, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(388) called with curMem=490694, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 388.0 B, free 730.1 MB) -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:56664 (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:14:15 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:15 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:15 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:15 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=491082, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=502866, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:15 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:15 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 22:14:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:15 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 22:14:15 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 22:14:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:15 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 7968 bytes result sent to driver -2016-05-24 22:14:15 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 7968 bytes result sent to driver -2016-05-24 22:14:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 25 ms on localhost (1/2) -2016-05-24 22:14:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 28 ms on localhost (2/2) -2016-05-24 22:14:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 22:14:15 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.029 s -2016-05-24 22:14:15 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.052281 s -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=506348, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=511940, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:56664 (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:14:15 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:15 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:15 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:15 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=514178, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=525962, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:15 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:15 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 22:14:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:15 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 22:14:15 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 22:14:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:15 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 7968 bytes result sent to driver -2016-05-24 22:14:15 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 7968 bytes result sent to driver -2016-05-24 22:14:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 23 ms on localhost (1/2) -2016-05-24 22:14:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 22 ms on localhost (2/2) -2016-05-24 22:14:15 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 22:14:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 22:14:15 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.057757 s -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=529444, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=535036, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:15 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:15 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:15 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:15 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=538645, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=550429, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:15 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:15 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 22:14:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:15 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 22:14:15 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 22:14:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:15 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 7968 bytes result sent to driver -2016-05-24 22:14:15 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 7968 bytes result sent to driver -2016-05-24 22:14:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 18 ms on localhost (1/2) -2016-05-24 22:14:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 21 ms on localhost (2/2) -2016-05-24 22:14:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 22:14:15 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:14:15 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.043622 s -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=553913, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(3603) called with curMem=559505, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:15 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:15 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:15 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:15 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=563108, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=574892, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:15 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:15 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 22:14:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:15 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 22:14:15 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 22:14:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:15 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 7968 bytes result sent to driver -2016-05-24 22:14:15 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 7968 bytes result sent to driver -2016-05-24 22:14:15 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 22:14:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 60 ms on localhost (1/2) -2016-05-24 22:14:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 65 ms on localhost (2/2) -2016-05-24 22:14:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 22:14:15 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.065 s -2016-05-24 22:14:15 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.087833 s -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=578375, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=583967, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:15 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:56664 in memory (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:15 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:56664 in memory (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:56664 in memory (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:14:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:15 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 22:14:15 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:15 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:56664 in memory (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:14:15 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:15 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=538320, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:56664 in memory (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:14:15 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=534836, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:15 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:15 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:15 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:15 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 22:14:15 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:15 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 7968 bytes result sent to driver -2016-05-24 22:14:15 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 7968 bytes result sent to driver -2016-05-24 22:14:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 19 ms on localhost (1/2) -2016-05-24 22:14:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 17 ms on localhost (2/2) -2016-05-24 22:14:15 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:14:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 22:14:15 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.039228 s -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=513851, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=519443, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:15 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:15 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:15 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:15 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=523049, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=534833, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:15 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:15 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 22:14:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:15 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 22:14:15 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 22:14:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:15 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 7968 bytes result sent to driver -2016-05-24 22:14:15 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 7968 bytes result sent to driver -2016-05-24 22:14:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 12 ms on localhost (1/2) -2016-05-24 22:14:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 13 ms on localhost (2/2) -2016-05-24 22:14:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 22:14:15 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:14:15 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.031403 s -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=538317, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:14:15 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=543909, maxMem=766075207 -2016-05-24 22:14:15 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:14:15 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:15 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:16 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:16 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:16 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=547515, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=559299, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:14:16 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:16 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:16 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 22:14:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:16 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 22:14:16 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 22:14:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:16 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 7968 bytes result sent to driver -2016-05-24 22:14:16 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 7968 bytes result sent to driver -2016-05-24 22:14:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 14 ms on localhost (1/2) -2016-05-24 22:14:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 17 ms on localhost (2/2) -2016-05-24 22:14:16 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 22:14:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 22:14:16 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.033577 s -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=562783, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(3604) called with curMem=568375, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:14:16 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:16 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:16 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:16 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:16 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=571979, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=583763, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:14:16 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:16 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:16 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 22:14:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:16 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 22:14:16 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 22:14:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:16 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 7968 bytes result sent to driver -2016-05-24 22:14:16 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 7968 bytes result sent to driver -2016-05-24 22:14:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 10 ms on localhost (1/2) -2016-05-24 22:14:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 11 ms on localhost (2/2) -2016-05-24 22:14:16 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:14:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 22:14:16 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.028409 s -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=587246, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=592838, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:14:16 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:16 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:16 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:16 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:16 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=596444, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=608228, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:14:16 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:16 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:16 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 22:14:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:16 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 22:14:16 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 22:14:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:16 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 7968 bytes result sent to driver -2016-05-24 22:14:16 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 7968 bytes result sent to driver -2016-05-24 22:14:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 12 ms on localhost (1/2) -2016-05-24 22:14:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 11 ms on localhost (2/2) -2016-05-24 22:14:16 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:14:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 22:14:16 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.024908 s -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=611712, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=617304, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:14:16 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:16 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:16 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:16 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:16 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=620910, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=632694, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:14:16 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:16 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:16 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 22:14:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:16 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 22:14:16 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 22:14:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:16 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 7968 bytes result sent to driver -2016-05-24 22:14:16 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 7968 bytes result sent to driver -2016-05-24 22:14:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 12 ms on localhost (1/2) -2016-05-24 22:14:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 12 ms on localhost (2/2) -2016-05-24 22:14:16 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:14:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 22:14:16 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.027457 s -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=636178, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(3610) called with curMem=641770, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:14:16 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:16 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:16 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:16 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:16 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=645380, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=657164, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:14:16 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:16 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:16 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 22:14:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:16 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 22:14:16 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 22:14:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:16 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 7968 bytes result sent to driver -2016-05-24 22:14:16 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 7968 bytes result sent to driver -2016-05-24 22:14:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 12 ms on localhost (1/2) -2016-05-24 22:14:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 13 ms on localhost (2/2) -2016-05-24 22:14:16 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:14:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 22:14:16 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.025188 s -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=660646, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=666238, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:16 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:16 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:16 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:16 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:16 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=669844, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=681628, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:14:16 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:16 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:16 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 22:14:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:16 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 22:14:16 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 22:14:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:16 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 7968 bytes result sent to driver -2016-05-24 22:14:16 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 7968 bytes result sent to driver -2016-05-24 22:14:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 31 ms on localhost (1/2) -2016-05-24 22:14:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 30 ms on localhost (2/2) -2016-05-24 22:14:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 22:14:16 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.033 s -2016-05-24 22:14:16 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.050335 s -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=685111, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(3610) called with curMem=690703, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:16 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:16 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:16 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:16 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:16 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=694313, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=706097, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:14:16 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:16 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:16 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 22:14:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:16 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 22:14:16 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 22:14:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:16 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 7968 bytes result sent to driver -2016-05-24 22:14:16 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 7968 bytes result sent to driver -2016-05-24 22:14:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 12 ms on localhost (1/2) -2016-05-24 22:14:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 14 ms on localhost (2/2) -2016-05-24 22:14:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 22:14:16 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:14:16 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.028731 s -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=709579, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:14:16 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=715171, maxMem=766075207 -2016-05-24 22:14:16 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:16 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:16 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:17 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:17 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:17 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=718777, maxMem=766075207 -2016-05-24 22:14:17 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:14:17 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=730561, maxMem=766075207 -2016-05-24 22:14:17 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:14:17 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:17 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:17 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 22:14:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:17 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 22:14:17 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 22:14:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:17 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 7968 bytes result sent to driver -2016-05-24 22:14:17 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 7968 bytes result sent to driver -2016-05-24 22:14:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 10 ms on localhost (1/2) -2016-05-24 22:14:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 11 ms on localhost (2/2) -2016-05-24 22:14:17 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:14:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 22:14:17 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.023559 s -2016-05-24 22:14:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=734044, maxMem=766075207 -2016-05-24 22:14:17 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:14:17 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=739636, maxMem=766075207 -2016-05-24 22:14:17 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:17 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:17 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:17 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:17 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:17 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=743242, maxMem=766075207 -2016-05-24 22:14:17 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:14:17 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=755026, maxMem=766075207 -2016-05-24 22:14:17 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:14:17 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:17 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:17 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 22:14:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:17 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 22:14:17 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 22:14:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:17 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 7968 bytes result sent to driver -2016-05-24 22:14:17 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 7968 bytes result sent to driver -2016-05-24 22:14:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 13 ms on localhost (1/2) -2016-05-24 22:14:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 15 ms on localhost (2/2) -2016-05-24 22:14:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 22:14:17 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:14:17 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.033555 s -2016-05-24 22:14:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=758508, maxMem=766075207 -2016-05-24 22:14:17 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:14:17 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=764100, maxMem=766075207 -2016-05-24 22:14:17 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:17 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:17 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:17 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:17 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:17 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=767713, maxMem=766075207 -2016-05-24 22:14:17 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:17 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=779497, maxMem=766075207 -2016-05-24 22:14:17 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:17 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:17 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:17 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 22:14:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:17 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 22:14:17 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 22:14:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:17 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 7968 bytes result sent to driver -2016-05-24 22:14:17 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 7968 bytes result sent to driver -2016-05-24 22:14:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 15 ms on localhost (1/2) -2016-05-24 22:14:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 15 ms on localhost (2/2) -2016-05-24 22:14:17 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:14:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 22:14:17 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.037144 s -2016-05-24 22:14:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=782980, maxMem=766075207 -2016-05-24 22:14:17 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:14:17 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=788572, maxMem=766075207 -2016-05-24 22:14:17 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:17 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:17 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:17 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:17 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:17 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=792178, maxMem=766075207 -2016-05-24 22:14:17 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:17 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=803962, maxMem=766075207 -2016-05-24 22:14:17 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:17 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:17 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:17 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 22:14:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:17 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 22:14:17 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 22:14:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 10 ms on localhost (1/2) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 10 ms on localhost (2/2) -2016-05-24 22:14:18 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 22:14:18 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.030965 s -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=807446, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=813038, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:18 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=816650, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=828434, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 22:14:18 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 9 ms on localhost (1/2) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 9 ms on localhost (2/2) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 22:14:18 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:14:18 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.021544 s -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=831918, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3621) called with curMem=837510, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:18 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=841131, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=852915, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 22:14:18 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 12 ms on localhost (1/2) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 12 ms on localhost (2/2) -2016-05-24 22:14:18 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 22:14:18 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.025563 s -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=856398, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=861990, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:18 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=865602, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=877386, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 22:14:18 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 10 ms on localhost (1/2) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 12 ms on localhost (2/2) -2016-05-24 22:14:18 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 22:14:18 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.026711 s -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=880869, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=886461, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:18 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=890073, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=901857, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 22:14:18 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 12 ms on localhost (1/2) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 14 ms on localhost (2/2) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 22:14:18 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:14:18 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.044778 s -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=880877, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=882985, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:18 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:18 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=846743, maxMem=766075207 -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=852935, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 22:14:18 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 18 ms on localhost (1/2) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 18 ms on localhost (2/2) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 22:14:18 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.036305 s -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=807491, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=809600, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:18 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 22:14:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:18 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=773338, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=779530, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 22:14:18 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 22:14:18 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 9 ms on localhost (1/2) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 12 ms on localhost (2/2) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 22:14:18 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:14:18 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.033979 s -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=755054, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3634) called with curMem=748862, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:18 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:18 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=699954, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=706146, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 22:14:18 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 11 ms on localhost (1/2) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 12 ms on localhost (2/2) -2016-05-24 22:14:18 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 22:14:18 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.026978 s -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=685162, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=690754, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:18 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=654631, maxMem=766075207 -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=666415, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 22:14:18 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 22:14:18 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 11 ms on localhost (1/2) -2016-05-24 22:14:18 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 18 ms on localhost (2/2) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 22:14:18 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:14:18 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.038090 s -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=648915, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=651025, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:18 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 22:14:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:18 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=620970, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=629270, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:14:18 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:18 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 22:14:18 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 11 ms on localhost (1/2) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 13 ms on localhost (2/2) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 22:14:18 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:14:18 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.029013 s -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=611772, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=617364, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:18 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=620976, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=632760, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 22:14:18 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 10 ms on localhost (1/2) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 9 ms on localhost (2/2) -2016-05-24 22:14:18 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 22:14:18 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.021616 s -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=636243, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=641835, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:18 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=645441, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=657225, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 22:14:18 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 7 ms on localhost (1/2) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 8 ms on localhost (2/2) -2016-05-24 22:14:18 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 22:14:18 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.018846 s -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=660709, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3623) called with curMem=666301, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:18 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=669924, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=681708, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 22:14:18 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 7 ms on localhost (1/2) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 7 ms on localhost (2/2) -2016-05-24 22:14:18 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 22:14:18 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.016769 s -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=685191, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=690783, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:18 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=694389, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=706173, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 22:14:18 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 5 ms on localhost (1/2) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 7 ms on localhost (2/2) -2016-05-24 22:14:18 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 22:14:18 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.016682 s -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=709656, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3626) called with curMem=715248, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:18 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=718874, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=730658, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 22:14:18 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 6 ms on localhost (1/2) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 8 ms on localhost (2/2) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 22:14:18 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:14:18 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.018203 s -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=734142, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=739734, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:18 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=743346, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=755130, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 22:14:18 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 6 ms on localhost (1/2) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 5 ms on localhost (2/2) -2016-05-24 22:14:18 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 22:14:18 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.014825 s -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=758614, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=764206, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:18 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=767818, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=779602, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 22:14:18 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 14 ms on localhost (1/2) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 18 ms on localhost (2/2) -2016-05-24 22:14:18 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 22:14:18 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.220036 s -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=783086, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=788678, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:18 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=792290, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=804074, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 22:14:18 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 5 ms on localhost (1/2) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 6 ms on localhost (2/2) -2016-05-24 22:14:18 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 22:14:18 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.014739 s -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=807558, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=813150, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:18 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=816762, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=828546, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 22:14:18 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 6 ms on localhost (1/2) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 5 ms on localhost (2/2) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 22:14:18 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:14:18 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.015453 s -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=832029, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=837621, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:18 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=841233, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=853017, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:18 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:18 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 7 ms on localhost (1/2) -2016-05-24 22:14:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:18 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 7968 bytes result sent to driver -2016-05-24 22:14:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 8 ms on localhost (2/2) -2016-05-24 22:14:18 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:14:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 22:14:18 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.018593 s -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=856500, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=862092, maxMem=766075207 -2016-05-24 22:14:18 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:18 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=865704, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=877488, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 6 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 7 ms on localhost (2/2) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.016174 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=880972, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=886564, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=890178, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=901962, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 8 ms on localhost (1/2) -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 8 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.030413 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=877360, maxMem=766075207 -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=862094, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=816747, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=819325, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 22:14:19 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 8 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 8 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.018685 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=794853, maxMem=766075207 -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=788661, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=739718, maxMem=766075207 -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=745910, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 22:14:19 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 7 ms on localhost (1/2) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 8 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.021417 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=706040, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=706040, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[93] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=660708, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=669008, maxMem=766075207 -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[93] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 7 ms on localhost (1/2) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 9 ms on localhost (2/2) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.019738 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=623542, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:19 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=617350, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:19 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[95] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=572014, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=583798, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[95] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 7 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 9 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.017872 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=587281, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=592873, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[97] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=596486, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=608270, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[97] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 11 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 13 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.069915 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=611752, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3618) called with curMem=617344, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 91 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 47 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[99] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=620962, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=632746, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 47 (MapPartitionsRDD[99] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 93, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 94, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 94) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 93) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 93). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 94). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 93) in 7 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 94) in 6 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 47 finished: treeAggregate at GradientDescent.scala:225, took 0.017343 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=636229, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=641821, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 93 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 48 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 48(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[101] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=645435, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=657219, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 48 (MapPartitionsRDD[101] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 95, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 96, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 95) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 96) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 95). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 96). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 96) in 7 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 95) in 7 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 48 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 48 finished: treeAggregate at GradientDescent.scala:225, took 0.018745 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=660702, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3618) called with curMem=666294, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 95 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 49 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 49(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[103] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=669912, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=681696, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 49 (MapPartitionsRDD[103] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 97, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 98, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 97) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 98) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 98). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 97). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 98) in 8 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 97) in 8 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 49 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 49 finished: treeAggregate at GradientDescent.scala:225, took 0.018273 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=685179, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3622) called with curMem=690771, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 97 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 50 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[105] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=694393, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=706177, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 50 (MapPartitionsRDD[105] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 7 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 9 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at GradientDescent.scala:225, took 0.021858 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=709658, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=715250, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 99 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 51 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[107] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=718863, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=730647, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 51 (MapPartitionsRDD[107] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 7 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 9 ms on localhost (2/2) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at GradientDescent.scala:225, took 0.022675 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=734129, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3628) called with curMem=739721, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 101 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 52 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 52(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 52 (MapPartitionsRDD[109] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=743349, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=755133, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 52 (MapPartitionsRDD[109] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 103, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 104, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 104) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 103) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 104). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 103). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 104) in 6 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 103) in 7 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 52 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 52 finished: treeAggregate at GradientDescent.scala:225, took 0.017195 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=758616, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=764208, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 103 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 53 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 53(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[111] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=767840, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=779624, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 104 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[111] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 105, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 106, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 105) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 106) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 105). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 106). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 105) in 5 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 106) in 4 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 53 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 53 finished: treeAggregate at GradientDescent.scala:225, took 0.013663 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=783106, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=788698, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 105 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 54 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[113] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=792311, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=804095, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 106 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 54 (MapPartitionsRDD[113] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 107) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 108) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 107). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 108). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 107) in 4 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 108) in 4 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 54 finished: treeAggregate at GradientDescent.scala:225, took 0.012571 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=807578, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=813170, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 107 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 55 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[115] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=816802, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=828586, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 108 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 55 (MapPartitionsRDD[115] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 109, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 110, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 110) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 109) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 110). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 110) in 4 ms on localhost (1/2) -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 109). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 109) in 6 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 55 finished: treeAggregate at GradientDescent.scala:225, took 0.014610 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=832068, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=837660, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 109 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 56 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[117] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=841273, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=853057, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 110 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 56 (MapPartitionsRDD[117] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 111, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 112, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 112) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 111) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 111). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 112). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 112) in 4 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 111) in 5 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 56 finished: treeAggregate at GradientDescent.scala:225, took 0.016135 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=856539, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=862131, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 111 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 57 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 57(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[119] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=865744, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=877528, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 112 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 57 (MapPartitionsRDD[119] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 113, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 114, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 113) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 114) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 114). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 113). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 114) in 3 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 113) in 5 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 57 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 57 finished: treeAggregate at GradientDescent.scala:225, took 0.013655 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=881009, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=886601, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 113 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 58 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 58(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 58 (MapPartitionsRDD[121] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=890213, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=901997, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 114 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 58 (MapPartitionsRDD[121] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 115, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 116, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 116) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 115) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 116). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 115). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 116) in 4 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 115) in 5 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 58 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 58 finished: treeAggregate at GradientDescent.scala:225, took 0.014502 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=905480, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_115 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3620) called with curMem=911072, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_115_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_115_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 115 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 59 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 59(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 59 (MapPartitionsRDD[123] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=914692, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_116 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3479) called with curMem=926476, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_116_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_116_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 116 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 59 (MapPartitionsRDD[123] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 117, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 118, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 117) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 118) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 118). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 117). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 118) in 5 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 117) in 6 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 59 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 59 finished: treeAggregate at GradientDescent.scala:225, took 0.015486 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=929955, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_117 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=935547, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_117_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_117_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 117 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 60 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 60(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 60 (MapPartitionsRDD[125] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=939159, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_118 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=950943, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_118_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_118_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 118 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 60 (MapPartitionsRDD[125] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 60.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 60.0 (TID 119, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 60.0 (TID 120, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 60.0 (TID 120) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 60.0 (TID 119) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 60.0 (TID 119). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 60.0 (TID 120). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 60.0 (TID 119) in 5 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 60.0 (TID 120) in 4 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 60 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 60.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 60 finished: treeAggregate at GradientDescent.scala:225, took 0.013698 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=954426, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_119 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3616) called with curMem=960018, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_119_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_119_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 119 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 61 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 61(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 61 (MapPartitionsRDD[127] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=963634, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_120 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=975418, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_120_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_120_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 120 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 61 (MapPartitionsRDD[127] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 121, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 122, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 122) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 121) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 121). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 122). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 121) in 4 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 122) in 5 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 61 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 61 finished: treeAggregate at GradientDescent.scala:225, took 0.016199 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=978902, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_121 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=984494, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_121_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_121_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 121 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 62 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 62(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[129] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=988106, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_122 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=999890, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_122_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_122_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 122 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 62 (MapPartitionsRDD[129] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 123, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 62.0 (TID 124, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 123) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 62.0 (TID 124) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 123). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 62.0 (TID 124). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 123) in 5 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 62.0 (TID 124) in 5 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 62 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 62 finished: treeAggregate at GradientDescent.scala:225, took 0.015292 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1003372, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_123 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3630) called with curMem=1008964, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_123_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_123_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 123 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 63 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 63(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 63 (MapPartitionsRDD[131] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1012594, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_124 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=1024378, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_124_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_124_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 124 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 63 (MapPartitionsRDD[131] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 63.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 63.0 (TID 125, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 63.0 (TID 126, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 63.0 (TID 125) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 63.0 (TID 126) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 63.0 (TID 125). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 63.0 (TID 126). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 63.0 (TID 125) in 6 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 63.0 (TID 126) in 5 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 63 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 63.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 63 finished: treeAggregate at GradientDescent.scala:225, took 0.015141 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1027859, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_125 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1033451, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_125_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_125_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 125 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 64 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 64(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 64 (MapPartitionsRDD[133] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1037064, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_126 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=1048848, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_126_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_126_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 126 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 64 (MapPartitionsRDD[133] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 64.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 64.0 (TID 127, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 64.0 (TID 128, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 64.0 (TID 127) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 64.0 (TID 128) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 64.0 (TID 128). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 64.0 (TID 127). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 64.0 (TID 128) in 4 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 64.0 (TID 127) in 5 ms on localhost (2/2) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 64.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 64 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 64 finished: treeAggregate at GradientDescent.scala:225, took 0.014974 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1052330, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_127 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1057922, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_127_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_127_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 127 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 65 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 65(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[135] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1061534, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_128 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1073318, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_128_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_128_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 128 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 65 (MapPartitionsRDD[135] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 129, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 130, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 129) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 130) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 129). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 130). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 129) in 4 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 130) in 5 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 65 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 65 finished: treeAggregate at GradientDescent.scala:225, took 0.015623 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1076802, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_129 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1082394, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_129_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_129_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 129 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 66 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 66(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 66 (MapPartitionsRDD[137] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1086007, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_130 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=1097791, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_130_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_130_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 130 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 66 (MapPartitionsRDD[137] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 66.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 66.0 (TID 131, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 66.0 (TID 132, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 66.0 (TID 132) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 66.0 (TID 131) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 66.0 (TID 132). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 66.0 (TID 131). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 66.0 (TID 132) in 4 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 66.0 (TID 131) in 5 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 66 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 66.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 66 finished: treeAggregate at GradientDescent.scala:225, took 0.016118 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1101278, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_131 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3633) called with curMem=1106870, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_131_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_131_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 131 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 67 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 67(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 67 (MapPartitionsRDD[139] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1110503, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_132 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1122287, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_132_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_132_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 132 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 67 (MapPartitionsRDD[139] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 67.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 67.0 (TID 133, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 67.0 (TID 134, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 67.0 (TID 134) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 67.0 (TID 133) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 67.0 (TID 133). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 67.0 (TID 134). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 67.0 (TID 133) in 5 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 67.0 (TID 134) in 4 ms on localhost (2/2) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 67.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 67 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 67 finished: treeAggregate at GradientDescent.scala:225, took 0.015371 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1125770, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_133 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1131362, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_133_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_133_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 133 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 68 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 68(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 68 (MapPartitionsRDD[141] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1134975, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_134 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1146759, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_134_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_134_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 134 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 68 (MapPartitionsRDD[141] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 68.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 68.0 (TID 135, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 68.0 (TID 136, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 68.0 (TID 136) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 68.0 (TID 135) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 68.0 (TID 136). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 68.0 (TID 135). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 68.0 (TID 136) in 5 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 68.0 (TID 135) in 5 ms on localhost (2/2) -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 68 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 68.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 68 finished: treeAggregate at GradientDescent.scala:225, took 0.016120 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1150242, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_135 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3608) called with curMem=1155834, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_135_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_135_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 135 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Got job 69 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:19 INFO DAGScheduler:59 - Final stage: ResultStage 69(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting ResultStage 69 (MapPartitionsRDD[143] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1159442, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_136 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1171226, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_136_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_136_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 136 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 69 (MapPartitionsRDD[143] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Adding task set 69.0 with 2 tasks -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 69.0 (TID 137, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 69.0 (TID 138, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:19 INFO Executor:59 - Running task 0.0 in stage 69.0 (TID 137) -2016-05-24 22:14:19 INFO Executor:59 - Running task 1.0 in stage 69.0 (TID 138) -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:19 INFO Executor:59 - Finished task 0.0 in stage 69.0 (TID 137). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO Executor:59 - Finished task 1.0 in stage 69.0 (TID 138). 7968 bytes result sent to driver -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 69.0 (TID 137) in 5 ms on localhost (1/2) -2016-05-24 22:14:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 69.0 (TID 138) in 4 ms on localhost (2/2) -2016-05-24 22:14:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 69.0, whose tasks have all completed, from pool -2016-05-24 22:14:19 INFO DAGScheduler:59 - ResultStage 69 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:14:19 INFO DAGScheduler:59 - Job 69 finished: treeAggregate at GradientDescent.scala:225, took 0.017253 s -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1174709, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_137 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:14:19 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1180301, maxMem=766075207 -2016-05-24 22:14:19 INFO MemoryStore:59 - Block broadcast_137_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:14:19 INFO BlockManagerInfo:59 - Added broadcast_137_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:19 INFO SparkContext:59 - Created broadcast 137 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 70 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 70(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 70 (MapPartitionsRDD[145] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1183907, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_138 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1195691, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_138_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_138_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 138 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 70 (MapPartitionsRDD[145] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 70.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 70.0 (TID 139, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 70.0 (TID 140, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 70.0 (TID 139) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 70.0 (TID 140) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 70.0 (TID 139). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 70.0 (TID 140). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 70.0 (TID 139) in 4 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 70.0 (TID 140) in 5 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 70 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 70.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 70 finished: treeAggregate at GradientDescent.scala:225, took 0.014591 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1199174, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_139 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3619) called with curMem=1204766, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_139_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_139_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 139 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 71 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 71(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 71 (MapPartitionsRDD[147] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1208385, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_140 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1220169, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_140_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_140_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 140 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 71 (MapPartitionsRDD[147] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 71.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 71.0 (TID 141, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 71.0 (TID 142, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 71.0 (TID 141) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 71.0 (TID 142) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 71.0 (TID 141). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 71.0 (TID 141) in 5 ms on localhost (1/2) -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 71.0 (TID 142). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 71.0 (TID 142) in 5 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 71 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 71.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 71 finished: treeAggregate at GradientDescent.scala:225, took 0.015364 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1223653, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_141 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3604) called with curMem=1229245, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_141_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_141_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 141 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 72 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 72(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 72 (MapPartitionsRDD[149] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1232849, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_142 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1244633, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_142_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_142_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 142 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 72 (MapPartitionsRDD[149] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 72.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 72.0 (TID 143, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 72.0 (TID 144, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 72.0 (TID 143) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 72.0 (TID 144) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 72.0 (TID 144). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 72.0 (TID 143). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 72.0 (TID 144) in 6 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 72.0 (TID 143) in 7 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 72 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 72.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 72 finished: treeAggregate at GradientDescent.scala:225, took 0.017436 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1248116, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_143 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=1253708, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_143_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_143_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 143 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 73 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 73(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 73 (MapPartitionsRDD[151] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1257317, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_144 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1269101, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_144_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_144_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 144 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 73 (MapPartitionsRDD[151] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 73.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 73.0 (TID 145, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 73.0 (TID 146, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 73.0 (TID 145) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 73.0 (TID 146) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 73.0 (TID 145). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 73.0 (TID 146). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 73.0 (TID 145) in 6 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 73.0 (TID 146) in 6 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 73 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 73.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 73 finished: treeAggregate at GradientDescent.scala:225, took 0.019260 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1272584, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_145 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1278176, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_145_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_145_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 145 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 74 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 74(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 74 (MapPartitionsRDD[153] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1281782, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_146 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1293566, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_146_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.3 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_146_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 146 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 74 (MapPartitionsRDD[153] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 74.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 74.0 (TID 147, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 74.0 (TID 148, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 74.0 (TID 148) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 74.0 (TID 147) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 74.0 (TID 147). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 74.0 (TID 148). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 74.0 (TID 147) in 5 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 74.0 (TID 148) in 5 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 74 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 74.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 74 finished: treeAggregate at GradientDescent.scala:225, took 0.016578 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1297049, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_147 stored as values in memory (estimated size 5.5 KB, free 729.3 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3624) called with curMem=1302641, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_147_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.3 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_147_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 147 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 75 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 75(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 75 (MapPartitionsRDD[155] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1306265, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_148 stored as values in memory (estimated size 11.5 KB, free 729.3 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=1318049, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_148_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.3 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_148_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 148 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 75 (MapPartitionsRDD[155] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 75.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 75.0 (TID 149, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 75.0 (TID 150, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 75.0 (TID 150) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 75.0 (TID 149) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 75.0 (TID 149). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 75.0 (TID 150). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 75.0 (TID 149) in 5 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 75.0 (TID 150) in 5 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 75 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 75.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 75 finished: treeAggregate at GradientDescent.scala:225, took 0.015301 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1321531, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_149 stored as values in memory (estimated size 5.5 KB, free 729.3 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3626) called with curMem=1327123, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_149_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.3 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_149_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 149 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_148_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 76 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 76 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 76(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_147_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 76 (MapPartitionsRDD[157] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1287517, maxMem=766075207 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_146_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_150 stored as values in memory (estimated size 11.5 KB, free 729.3 MB) -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 75 -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1287517, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_150_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_150_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 150 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 76 (MapPartitionsRDD[157] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_145_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 151, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 152, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 151) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 152) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_144_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 152). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 151). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 152) in 6 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 151) in 8 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 76 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 76 finished: treeAggregate at GradientDescent.scala:225, took 0.016804 s -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 74 -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1266535, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_151 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_143_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3611) called with curMem=1268518, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_151_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_151_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 151 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_142_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 73 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_141_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_140_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 77 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 77(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 72 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[159] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1223187, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_152 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_139_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1229379, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_152_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_152_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 152 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_138_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 77 (MapPartitionsRDD[159] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 2 tasks -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 71 -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 153, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 154, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 154) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 153) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_137_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 154). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 153). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 154) in 4 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 153) in 5 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 77 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 77 finished: treeAggregate at GradientDescent.scala:225, took 0.014588 s -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_136_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1193131, maxMem=766075207 -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 70 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_153 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1189523, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_153_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_135_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_153_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 153 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_134_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 69 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_133_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_132_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 78 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 78(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 78 (MapPartitionsRDD[161] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1149763, maxMem=766075207 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_131_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_154 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1140684, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_154_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_130_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_154_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 154 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 78 (MapPartitionsRDD[161] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 155, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 156, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_129_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 155) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 156) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 155). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 156). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_128_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 155) in 5 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 156) in 5 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 78 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 78 finished: treeAggregate at GradientDescent.scala:225, took 0.015908 s -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1114102, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_155 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_127_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1116082, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_155_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_155_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_126_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 155 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_125_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_124_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 79 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 79(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_123_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 79 (MapPartitionsRDD[163] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1070736, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_156 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1067254, maxMem=766075207 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_122_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_156_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_156_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 156 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 79 (MapPartitionsRDD[163] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 79.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 79.0 (TID 157, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 79.0 (TID 158, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_121_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 79.0 (TID 158) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 79.0 (TID 157) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 79.0 (TID 158). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 79.0 (TID 157). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 79.0 (TID 158) in 7 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 79.0 (TID 157) in 9 ms on localhost (2/2) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 79.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 79 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_120_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 79 finished: treeAggregate at GradientDescent.scala:225, took 0.023264 s -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1046266, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_157 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1048242, maxMem=766075207 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_119_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_157_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_157_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 157 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_118_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 80 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 80(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_117_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 80 (MapPartitionsRDD[165] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1021791, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_158 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1030096, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_158_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_116_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_158_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 158 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 80 (MapPartitionsRDD[165] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 80.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 80.0 (TID 159, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 80.0 (TID 160, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 80.0 (TID 160) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 80.0 (TID 159) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_115_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 80.0 (TID 160). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 80.0 (TID 159). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 80.0 (TID 160) in 7 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 80.0 (TID 159) in 8 ms on localhost (2/2) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 80.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 80 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 80 finished: treeAggregate at GradientDescent.scala:225, took 0.020517 s -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_114_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=997316, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_159 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1002908, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_159_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_159_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 159 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_113_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_112_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_111_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 81 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 81(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 81 (MapPartitionsRDD[167] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=969358, maxMem=766075207 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_110_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_160 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=969358, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_160_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_160_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 160 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 81 (MapPartitionsRDD[167] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 81.0 with 2 tasks -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_109_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 81.0 (TID 161, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 81.0 (TID 162, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 81.0 (TID 161) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 81.0 (TID 162) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 81.0 (TID 161). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 81.0 (TID 161) in 7 ms on localhost (1/2) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_108_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 81.0 (TID 162). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 81.0 (TID 162) in 9 ms on localhost (2/2) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 81.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 81 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 81 finished: treeAggregate at GradientDescent.scala:225, took 0.021441 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=942779, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_161 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=944739, maxMem=766075207 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_107_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_161_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_161_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 161 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_106_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_105_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 82 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 82(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 82 (MapPartitionsRDD[169] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_104_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=908613, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_162 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=920397, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_162_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_162_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 162 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 82 (MapPartitionsRDD[169] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 82.0 with 2 tasks -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_103_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 82.0 (TID 163, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 82.0 (TID 164, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 82.0 (TID 164) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_102_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 82.0 (TID 163) -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 82.0 (TID 164). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 82.0 (TID 164) in 6 ms on localhost (1/2) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 82.0 (TID 163). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 82.0 (TID 163) in 10 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 82 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 82.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 82 finished: treeAggregate at GradientDescent.scala:225, took 0.022015 s -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_101_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=890169, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_163 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=880495, maxMem=766075207 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_100_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_163_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_163_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 163 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_99_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_98_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 83 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 83(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_97_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 83 (MapPartitionsRDD[171] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=850425, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_164 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=858726, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_164_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_96_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_164_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 164 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 83 (MapPartitionsRDD[171] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 83.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 83.0 (TID 165, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 83.0 (TID 166, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 83.0 (TID 166) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 83.0 (TID 165) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_95_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 83.0 (TID 165). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 83.0 (TID 166). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 83.0 (TID 165) in 7 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 83.0 (TID 166) in 8 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 83 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 83.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 83 finished: treeAggregate at GradientDescent.scala:225, took 0.020256 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=837733, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_165 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_94_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=831541, maxMem=766075207 -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_165_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_165_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 165 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_93_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_92_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_91_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 84 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 84(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 84 (MapPartitionsRDD[173] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=786204, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_166 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_90_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=797988, maxMem=766075207 -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_166_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_166_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 166 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 84 (MapPartitionsRDD[173] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_89_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 84.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 84.0 (TID 167, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 84.0 (TID 168, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 84.0 (TID 167) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_88_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 84.0 (TID 168) -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 84.0 (TID 167). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 84.0 (TID 168). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 84.0 (TID 167) in 9 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 84.0 (TID 168) in 10 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 84 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 84.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 84 finished: treeAggregate at GradientDescent.scala:225, took 0.022987 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=773387, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_167 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_87_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3620) called with curMem=773387, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_167_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_167_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 167 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 85 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 85(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 85 (MapPartitionsRDD[175] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=728066, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_168 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=739850, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_168_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_168_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 168 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 85 (MapPartitionsRDD[175] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 85.0 with 2 tasks -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 85.0 (TID 169, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 85.0 (TID 170, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 85.0 (TID 169) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 85.0 (TID 170) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 85.0 (TID 169). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 85.0 (TID 170). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 85.0 (TID 169) in 7 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 85.0 (TID 170) in 7 ms on localhost (2/2) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 85.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 85 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 85 finished: treeAggregate at GradientDescent.scala:225, took 0.019461 s -2016-05-24 22:14:20 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=718862, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_169 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3621) called with curMem=715248, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_169_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_169_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 169 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 86 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 86(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 86 (MapPartitionsRDD[177] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=718869, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_170 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=730653, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_170_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_170_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 170 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 86 (MapPartitionsRDD[177] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 86.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 86.0 (TID 171, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 86.0 (TID 172, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 86.0 (TID 171) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 86.0 (TID 172) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 86.0 (TID 172). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 86.0 (TID 171). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 86.0 (TID 172) in 6 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 86.0 (TID 171) in 8 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 86 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 86.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 86 finished: treeAggregate at GradientDescent.scala:225, took 0.018110 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=734137, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_171 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=739729, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_171_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_171_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 171 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 87 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 87(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 87 (MapPartitionsRDD[179] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=743341, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_172 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=755125, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_172_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_172_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 172 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 87 (MapPartitionsRDD[179] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 87.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 87.0 (TID 173, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 87.0 (TID 174, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 87.0 (TID 173) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 87.0 (TID 174) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 87.0 (TID 174). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 87.0 (TID 173). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 87.0 (TID 174) in 6 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 87.0 (TID 173) in 7 ms on localhost (2/2) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 87.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 87 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 87 finished: treeAggregate at GradientDescent.scala:225, took 0.019152 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=758608, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_173 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=764200, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_173_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_173_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 173 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 88 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 88(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 88 (MapPartitionsRDD[181] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=767813, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_174 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3486) called with curMem=779597, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_174_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_174_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 174 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 88 (MapPartitionsRDD[181] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 88.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 88.0 (TID 175, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 88.0 (TID 176, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 88.0 (TID 175) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 88.0 (TID 176) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 88.0 (TID 175). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 88.0 (TID 176). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 88.0 (TID 175) in 7 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 88.0 (TID 176) in 6 ms on localhost (2/2) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 88.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 88 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 88 finished: treeAggregate at GradientDescent.scala:225, took 0.016761 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=783083, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_175 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3615) called with curMem=788675, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_175_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_175_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 175 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 89 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 89(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 89 (MapPartitionsRDD[183] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=792290, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_176 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=804074, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_176_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_176_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 176 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 89 (MapPartitionsRDD[183] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 89.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 89.0 (TID 177, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 89.0 (TID 178, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 89.0 (TID 177) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 89.0 (TID 178) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 89.0 (TID 178). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 89.0 (TID 177). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 89.0 (TID 177) in 7 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 89.0 (TID 178) in 7 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 89 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 89.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 89 finished: treeAggregate at GradientDescent.scala:225, took 0.017464 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=807557, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_177 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=813149, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_177_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_177_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 177 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 90 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 90(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 90 (MapPartitionsRDD[185] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=816761, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_178 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=828545, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_178_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_178_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 178 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 90 (MapPartitionsRDD[185] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 90.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 90.0 (TID 179, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 90.0 (TID 180, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 90.0 (TID 179) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 90.0 (TID 180) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 90.0 (TID 179). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 90.0 (TID 180). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 90.0 (TID 179) in 8 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 90.0 (TID 180) in 7 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 90 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 90.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 90 finished: treeAggregate at GradientDescent.scala:225, took 0.019065 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=832029, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_179 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=837621, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_179_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_179_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 179 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 91 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 91(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 91 (MapPartitionsRDD[187] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=841234, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_180 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=853018, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_180_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_180_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 180 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 91 (MapPartitionsRDD[187] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 91.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 91.0 (TID 181, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 91.0 (TID 182, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 91.0 (TID 182) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 91.0 (TID 181) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 91.0 (TID 182). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 91.0 (TID 182) in 4 ms on localhost (1/2) -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 91.0 (TID 181). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 91.0 (TID 181) in 6 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 91 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 91.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 91 finished: treeAggregate at GradientDescent.scala:225, took 0.015682 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=856501, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_181 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3608) called with curMem=862093, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_181_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_181_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 181 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 92 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 92(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 92 (MapPartitionsRDD[189] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=865701, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_182 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=877485, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_182_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_182_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 182 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 92 (MapPartitionsRDD[189] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 92.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 92.0 (TID 183, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 92.0 (TID 184, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 92.0 (TID 183) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 92.0 (TID 184) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 92.0 (TID 184). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 92.0 (TID 183). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 92.0 (TID 184) in 4 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 92.0 (TID 183) in 4 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 92 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 92.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 92 finished: treeAggregate at GradientDescent.scala:225, took 0.011823 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=880968, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_183 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=886560, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_183_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_183_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 183 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 93 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 93(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 93 (MapPartitionsRDD[191] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=890174, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_184 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=901958, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_184_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_184_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 184 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 93 (MapPartitionsRDD[191] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 93.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 93.0 (TID 185, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 93.0 (TID 186, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 93.0 (TID 185) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 93.0 (TID 185). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 93.0 (TID 186) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 93.0 (TID 185) in 7 ms on localhost (1/2) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 93.0 (TID 186). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 93.0 (TID 186) in 10 ms on localhost (2/2) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 93.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 93 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 93 finished: treeAggregate at GradientDescent.scala:225, took 0.020988 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=905441, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_185 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=911033, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_185_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_185_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 185 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 94 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 94(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 94 (MapPartitionsRDD[193] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=914645, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_186 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=926429, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_186_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_186_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 186 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 94 (MapPartitionsRDD[193] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 94.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 94.0 (TID 187, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 94.0 (TID 188, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 94.0 (TID 187) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 94.0 (TID 188) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 94.0 (TID 187). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 94.0 (TID 188). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 94.0 (TID 187) in 11 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 94.0 (TID 188) in 10 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 94 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 94.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 94 finished: treeAggregate at GradientDescent.scala:225, took 0.023883 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=929912, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_187 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=935504, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_187_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_187_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 187 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 95 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 95(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 95 (MapPartitionsRDD[195] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=939117, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_188 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=950901, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_188_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_188_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 188 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 95 (MapPartitionsRDD[195] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 95.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 95.0 (TID 189, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 95.0 (TID 190, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 95.0 (TID 189) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 95.0 (TID 190) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 95.0 (TID 190). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 95.0 (TID 189). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 95.0 (TID 190) in 4 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 95.0 (TID 189) in 5 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 95 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 95.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 95 finished: treeAggregate at GradientDescent.scala:225, took 0.011479 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=954385, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_189 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3634) called with curMem=959977, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_189_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_189_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 189 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 96 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 96(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 96 (MapPartitionsRDD[197] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=963611, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_190 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=975395, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_190_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_190_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 190 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 96 (MapPartitionsRDD[197] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 96.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 96.0 (TID 191, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 96.0 (TID 192, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 96.0 (TID 191) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 96.0 (TID 192) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 96.0 (TID 191). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 96.0 (TID 192). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 96.0 (TID 191) in 6 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 96.0 (TID 192) in 6 ms on localhost (2/2) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 96.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 96 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 96 finished: treeAggregate at GradientDescent.scala:225, took 0.014344 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=978878, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_191 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3619) called with curMem=984470, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_191_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_191_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 191 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 97 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 97(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 97 (MapPartitionsRDD[199] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=988089, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_192 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=999873, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_192_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_192_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 192 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 97 (MapPartitionsRDD[199] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 97.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 97.0 (TID 193, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 97.0 (TID 194, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 97.0 (TID 193) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 97.0 (TID 194) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 97.0 (TID 193). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 97.0 (TID 194). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 97.0 (TID 193) in 6 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 97.0 (TID 194) in 6 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 97 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 97.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 97 finished: treeAggregate at GradientDescent.scala:225, took 0.013245 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1003356, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_193 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1008948, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_193_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_193_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 193 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 98 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 98(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 98 (MapPartitionsRDD[201] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1012560, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_194 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1024344, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_194_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_194_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 194 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 98 (MapPartitionsRDD[201] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 98.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 98.0 (TID 195, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 98.0 (TID 196, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 98.0 (TID 195) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 98.0 (TID 196) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 98.0 (TID 195). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 98.0 (TID 196). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 98.0 (TID 195) in 5 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 98.0 (TID 196) in 5 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 98 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 98.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 98 finished: treeAggregate at GradientDescent.scala:225, took 0.012362 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1027828, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_195 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=1033420, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_195_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_195_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 195 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 99 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 99(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 99 (MapPartitionsRDD[203] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1037029, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_196 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1048813, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_196_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_196_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 196 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 99 (MapPartitionsRDD[203] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 99.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 99.0 (TID 197, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 99.0 (TID 198, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 99.0 (TID 197) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 99.0 (TID 198) -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:20 INFO Executor:59 - Finished task 1.0 in stage 99.0 (TID 198). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO Executor:59 - Finished task 0.0 in stage 99.0 (TID 197). 7968 bytes result sent to driver -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 99.0 (TID 198) in 4 ms on localhost (1/2) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 99.0 (TID 197) in 4 ms on localhost (2/2) -2016-05-24 22:14:20 INFO DAGScheduler:59 - ResultStage 99 (treeAggregate at GradientDescent.scala:225) finished in 0.004 s -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 99.0, whose tasks have all completed, from pool -2016-05-24 22:14:20 INFO DAGScheduler:59 - Job 99 finished: treeAggregate at GradientDescent.scala:225, took 0.011719 s -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1052296, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_197 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3616) called with curMem=1057888, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_197_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_197_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 197 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Got job 100 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:20 INFO DAGScheduler:59 - Final stage: ResultStage 100(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting ResultStage 100 (MapPartitionsRDD[205] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1061504, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_198 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO MemoryStore:59 - ensureFreeSpace(3485) called with curMem=1073288, maxMem=766075207 -2016-05-24 22:14:20 INFO MemoryStore:59 - Block broadcast_198_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:14:20 INFO BlockManagerInfo:59 - Added broadcast_198_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:20 INFO SparkContext:59 - Created broadcast 198 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 100 (MapPartitionsRDD[205] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:20 INFO TaskSchedulerImpl:59 - Adding task set 100.0 with 2 tasks -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 100.0 (TID 199, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 100.0 (TID 200, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:20 INFO Executor:59 - Running task 0.0 in stage 100.0 (TID 199) -2016-05-24 22:14:20 INFO Executor:59 - Running task 1.0 in stage 100.0 (TID 200) -2016-05-24 22:14:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:21 INFO Executor:59 - Finished task 1.0 in stage 100.0 (TID 200). 7968 bytes result sent to driver -2016-05-24 22:14:21 INFO Executor:59 - Finished task 0.0 in stage 100.0 (TID 199). 7968 bytes result sent to driver -2016-05-24 22:14:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 100.0 (TID 200) in 4 ms on localhost (1/2) -2016-05-24 22:14:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 100.0 (TID 199) in 5 ms on localhost (2/2) -2016-05-24 22:14:21 INFO DAGScheduler:59 - ResultStage 100 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:14:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 100.0, whose tasks have all completed, from pool -2016-05-24 22:14:21 INFO DAGScheduler:59 - Job 100 finished: treeAggregate at GradientDescent.scala:225, took 0.011537 s -2016-05-24 22:14:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1076773, maxMem=766075207 -2016-05-24 22:14:21 INFO MemoryStore:59 - Block broadcast_199 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:14:21 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1082365, maxMem=766075207 -2016-05-24 22:14:21 INFO MemoryStore:59 - Block broadcast_199_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:14:21 INFO BlockManagerInfo:59 - Added broadcast_199_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:21 INFO SparkContext:59 - Created broadcast 199 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:21 INFO DAGScheduler:59 - Got job 101 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:21 INFO DAGScheduler:59 - Final stage: ResultStage 101(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:21 INFO DAGScheduler:59 - Submitting ResultStage 101 (MapPartitionsRDD[207] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1085971, maxMem=766075207 -2016-05-24 22:14:21 INFO MemoryStore:59 - Block broadcast_200 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:14:21 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1097755, maxMem=766075207 -2016-05-24 22:14:21 INFO MemoryStore:59 - Block broadcast_200_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:14:21 INFO BlockManagerInfo:59 - Added broadcast_200_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:21 INFO SparkContext:59 - Created broadcast 200 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 101 (MapPartitionsRDD[207] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:21 INFO TaskSchedulerImpl:59 - Adding task set 101.0 with 2 tasks -2016-05-24 22:14:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 101.0 (TID 201, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 101.0 (TID 202, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:21 INFO Executor:59 - Running task 0.0 in stage 101.0 (TID 201) -2016-05-24 22:14:21 INFO Executor:59 - Running task 1.0 in stage 101.0 (TID 202) -2016-05-24 22:14:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:21 INFO Executor:59 - Finished task 1.0 in stage 101.0 (TID 202). 7968 bytes result sent to driver -2016-05-24 22:14:21 INFO Executor:59 - Finished task 0.0 in stage 101.0 (TID 201). 7968 bytes result sent to driver -2016-05-24 22:14:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 101.0 (TID 202) in 4 ms on localhost (1/2) -2016-05-24 22:14:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 101.0 (TID 201) in 6 ms on localhost (2/2) -2016-05-24 22:14:21 INFO DAGScheduler:59 - ResultStage 101 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 101.0, whose tasks have all completed, from pool -2016-05-24 22:14:21 INFO DAGScheduler:59 - Job 101 finished: treeAggregate at GradientDescent.scala:225, took 0.013884 s -2016-05-24 22:14:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1101239, maxMem=766075207 -2016-05-24 22:14:21 INFO MemoryStore:59 - Block broadcast_201 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:14:21 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1106831, maxMem=766075207 -2016-05-24 22:14:21 INFO MemoryStore:59 - Block broadcast_201_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:14:21 INFO BlockManagerInfo:59 - Added broadcast_201_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:21 INFO SparkContext:59 - Created broadcast 201 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:21 INFO DAGScheduler:59 - Got job 102 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:21 INFO DAGScheduler:59 - Final stage: ResultStage 102(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:21 INFO DAGScheduler:59 - Submitting ResultStage 102 (MapPartitionsRDD[209] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1110444, maxMem=766075207 -2016-05-24 22:14:21 INFO MemoryStore:59 - Block broadcast_202 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:14:21 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1122228, maxMem=766075207 -2016-05-24 22:14:21 INFO MemoryStore:59 - Block broadcast_202_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:14:21 INFO BlockManagerInfo:59 - Added broadcast_202_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:21 INFO SparkContext:59 - Created broadcast 202 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 102 (MapPartitionsRDD[209] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:21 INFO TaskSchedulerImpl:59 - Adding task set 102.0 with 2 tasks -2016-05-24 22:14:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 102.0 (TID 203, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 102.0 (TID 204, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:21 INFO Executor:59 - Running task 0.0 in stage 102.0 (TID 203) -2016-05-24 22:14:21 INFO Executor:59 - Running task 1.0 in stage 102.0 (TID 204) -2016-05-24 22:14:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:21 INFO Executor:59 - Finished task 1.0 in stage 102.0 (TID 204). 7968 bytes result sent to driver -2016-05-24 22:14:21 INFO Executor:59 - Finished task 0.0 in stage 102.0 (TID 203). 7968 bytes result sent to driver -2016-05-24 22:14:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 102.0 (TID 204) in 3 ms on localhost (1/2) -2016-05-24 22:14:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 102.0 (TID 203) in 5 ms on localhost (2/2) -2016-05-24 22:14:21 INFO DAGScheduler:59 - ResultStage 102 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:14:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 102.0, whose tasks have all completed, from pool -2016-05-24 22:14:21 INFO DAGScheduler:59 - Job 102 finished: treeAggregate at GradientDescent.scala:225, took 0.011803 s -2016-05-24 22:14:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1125712, maxMem=766075207 -2016-05-24 22:14:21 INFO MemoryStore:59 - Block broadcast_203 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:14:21 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1131304, maxMem=766075207 -2016-05-24 22:14:21 INFO MemoryStore:59 - Block broadcast_203_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:14:21 INFO BlockManagerInfo:59 - Added broadcast_203_piece0 in memory on localhost:56664 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:21 INFO SparkContext:59 - Created broadcast 203 from broadcast at GradientDescent.scala:221 -2016-05-24 22:14:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:14:21 INFO DAGScheduler:59 - Got job 103 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:14:21 INFO DAGScheduler:59 - Final stage: ResultStage 103(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:21 INFO DAGScheduler:59 - Submitting ResultStage 103 (MapPartitionsRDD[211] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:14:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1134916, maxMem=766075207 -2016-05-24 22:14:21 INFO MemoryStore:59 - Block broadcast_204 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:14:21 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=1146700, maxMem=766075207 -2016-05-24 22:14:21 INFO MemoryStore:59 - Block broadcast_204_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:14:21 INFO BlockManagerInfo:59 - Added broadcast_204_piece0 in memory on localhost:56664 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:21 INFO SparkContext:59 - Created broadcast 204 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 103 (MapPartitionsRDD[211] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:14:21 INFO TaskSchedulerImpl:59 - Adding task set 103.0 with 2 tasks -2016-05-24 22:14:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 103.0 (TID 205, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 103.0 (TID 206, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:14:21 INFO Executor:59 - Running task 1.0 in stage 103.0 (TID 206) -2016-05-24 22:14:21 INFO Executor:59 - Running task 0.0 in stage 103.0 (TID 205) -2016-05-24 22:14:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:14:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:14:21 INFO Executor:59 - Finished task 1.0 in stage 103.0 (TID 206). 7968 bytes result sent to driver -2016-05-24 22:14:21 INFO Executor:59 - Finished task 0.0 in stage 103.0 (TID 205). 7968 bytes result sent to driver -2016-05-24 22:14:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 103.0 (TID 206) in 4 ms on localhost (1/2) -2016-05-24 22:14:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 103.0 (TID 205) in 6 ms on localhost (2/2) -2016-05-24 22:14:21 INFO DAGScheduler:59 - ResultStage 103 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:14:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 103.0, whose tasks have all completed, from pool -2016-05-24 22:14:21 INFO DAGScheduler:59 - Job 103 finished: treeAggregate at GradientDescent.scala:225, took 0.012797 s -2016-05-24 22:14:21 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 3363.579936927662, 3356.5316534793847, 3349.536449573466, 3342.59343622967, 3335.701748404667, 3328.860544103912, 3322.069003535364, 3315.3263283026713, 3308.6317406356134, 3301.984482655661 -2016-05-24 22:14:21 INFO SparkContext:59 - Starting job: sortByKey at BinaryClassificationMetrics.scala:156 -2016-05-24 22:14:21 INFO DAGScheduler:59 - Registering RDD 212 (map at SVMFromSparkLearning.scala:42) -2016-05-24 22:14:21 INFO DAGScheduler:59 - Got job 104 (sortByKey at BinaryClassificationMetrics.scala:156) with 2 output partitions -2016-05-24 22:14:21 INFO DAGScheduler:59 - Final stage: ResultStage 105(sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:14:21 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 104) -2016-05-24 22:14:21 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 104) -2016-05-24 22:14:21 INFO DAGScheduler:59 - Submitting ShuffleMapStage 104 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:42), which has no missing parents -2016-05-24 22:14:21 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=1150180, maxMem=766075207 -2016-05-24 22:14:21 INFO MemoryStore:59 - Block broadcast_205 stored as values in memory (estimated size 10.8 KB, free 729.5 MB) -2016-05-24 22:14:21 INFO MemoryStore:59 - ensureFreeSpace(6433) called with curMem=1161188, maxMem=766075207 -2016-05-24 22:14:21 INFO MemoryStore:59 - Block broadcast_205_piece0 stored as bytes in memory (estimated size 6.3 KB, free 729.5 MB) -2016-05-24 22:14:21 INFO BlockManagerInfo:59 - Added broadcast_205_piece0 in memory on localhost:56664 (size: 6.3 KB, free: 730.1 MB) -2016-05-24 22:14:21 INFO SparkContext:59 - Created broadcast 205 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 104 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:42) -2016-05-24 22:14:21 INFO TaskSchedulerImpl:59 - Adding task set 104.0 with 2 tasks -2016-05-24 22:14:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 104.0 (TID 207, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:14:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 104.0 (TID 208, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:14:21 INFO Executor:59 - Running task 0.0 in stage 104.0 (TID 207) -2016-05-24 22:14:21 INFO Executor:59 - Running task 1.0 in stage 104.0 (TID 208) -2016-05-24 22:14:21 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:14:21 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:14:21 INFO Executor:59 - Finished task 0.0 in stage 104.0 (TID 207). 2254 bytes result sent to driver -2016-05-24 22:14:21 INFO Executor:59 - Finished task 1.0 in stage 104.0 (TID 208). 2254 bytes result sent to driver -2016-05-24 22:14:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 104.0 (TID 208) in 829 ms on localhost (1/2) -2016-05-24 22:14:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 104.0 (TID 207) in 830 ms on localhost (2/2) -2016-05-24 22:14:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 104.0, whose tasks have all completed, from pool -2016-05-24 22:14:21 INFO DAGScheduler:59 - ShuffleMapStage 104 (map at SVMFromSparkLearning.scala:42) finished in 0.832 s -2016-05-24 22:14:21 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:14:21 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:14:21 INFO DAGScheduler:59 - waiting: Set(ResultStage 105) -2016-05-24 22:14:21 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:14:21 INFO DAGScheduler:59 - Missing parents for ResultStage 105: List() -2016-05-24 22:14:21 INFO DAGScheduler:59 - Submitting ResultStage 105 (MapPartitionsRDD[215] at sortByKey at BinaryClassificationMetrics.scala:156), which is now runnable -2016-05-24 22:14:21 INFO MemoryStore:59 - ensureFreeSpace(3344) called with curMem=1167621, maxMem=766075207 -2016-05-24 22:14:21 INFO MemoryStore:59 - Block broadcast_206 stored as values in memory (estimated size 3.3 KB, free 729.5 MB) -2016-05-24 22:14:21 INFO MemoryStore:59 - ensureFreeSpace(1900) called with curMem=1170965, maxMem=766075207 -2016-05-24 22:14:21 INFO MemoryStore:59 - Block broadcast_206_piece0 stored as bytes in memory (estimated size 1900.0 B, free 729.5 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Added broadcast_206_piece0 in memory on localhost:56664 (size: 1900.0 B, free: 730.1 MB) -2016-05-24 22:14:22 INFO SparkContext:59 - Created broadcast 206 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 105 (MapPartitionsRDD[215] at sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:14:22 INFO TaskSchedulerImpl:59 - Adding task set 105.0 with 2 tasks -2016-05-24 22:14:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 105.0 (TID 209, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:14:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 105.0 (TID 210, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:14:22 INFO Executor:59 - Running task 1.0 in stage 105.0 (TID 210) -2016-05-24 22:14:22 INFO Executor:59 - Running task 0.0 in stage 105.0 (TID 209) -2016-05-24 22:14:22 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:14:22 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:14:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-24 22:14:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-24 22:14:22 INFO Executor:59 - Finished task 0.0 in stage 105.0 (TID 209). 1469 bytes result sent to driver -2016-05-24 22:14:22 INFO Executor:59 - Finished task 1.0 in stage 105.0 (TID 210). 1517 bytes result sent to driver -2016-05-24 22:14:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 105.0 (TID 209) in 37 ms on localhost (1/2) -2016-05-24 22:14:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 105.0 (TID 210) in 37 ms on localhost (2/2) -2016-05-24 22:14:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 105.0, whose tasks have all completed, from pool -2016-05-24 22:14:22 INFO DAGScheduler:59 - ResultStage 105 (sortByKey at BinaryClassificationMetrics.scala:156) finished in 0.038 s -2016-05-24 22:14:22 INFO DAGScheduler:59 - Job 104 finished: sortByKey at BinaryClassificationMetrics.scala:156, took 0.895900 s -2016-05-24 22:14:22 INFO SparkContext:59 - Starting job: collect at BinaryClassificationMetrics.scala:193 -2016-05-24 22:14:22 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 154 bytes -2016-05-24 22:14:22 INFO DAGScheduler:59 - Registering RDD 213 (combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:14:22 INFO DAGScheduler:59 - Got job 105 (collect at BinaryClassificationMetrics.scala:193) with 2 output partitions -2016-05-24 22:14:22 INFO DAGScheduler:59 - Final stage: ResultStage 108(collect at BinaryClassificationMetrics.scala:193) -2016-05-24 22:14:22 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 107) -2016-05-24 22:14:22 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 107) -2016-05-24 22:14:22 INFO DAGScheduler:59 - Submitting ShuffleMapStage 107 (ShuffledRDD[213] at combineByKey at BinaryClassificationMetrics.scala:152), which has no missing parents -2016-05-24 22:14:22 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=1172865, maxMem=766075207 -2016-05-24 22:14:22 INFO MemoryStore:59 - Block broadcast_207 stored as values in memory (estimated size 2.8 KB, free 729.5 MB) -2016-05-24 22:14:22 INFO MemoryStore:59 - ensureFreeSpace(1658) called with curMem=1175761, maxMem=766075207 -2016-05-24 22:14:22 INFO MemoryStore:59 - Block broadcast_207_piece0 stored as bytes in memory (estimated size 1658.0 B, free 729.5 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Added broadcast_207_piece0 in memory on localhost:56664 (size: 1658.0 B, free: 730.1 MB) -2016-05-24 22:14:22 INFO SparkContext:59 - Created broadcast 207 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 107 (ShuffledRDD[213] at combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:14:22 INFO TaskSchedulerImpl:59 - Adding task set 107.0 with 2 tasks -2016-05-24 22:14:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 107.0 (TID 211, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:14:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 107.0 (TID 212, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:14:22 INFO Executor:59 - Running task 0.0 in stage 107.0 (TID 211) -2016-05-24 22:14:22 INFO Executor:59 - Running task 1.0 in stage 107.0 (TID 212) -2016-05-24 22:14:22 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:14:22 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:14:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:14:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:14:22 INFO Executor:59 - Finished task 1.0 in stage 107.0 (TID 212). 1375 bytes result sent to driver -2016-05-24 22:14:22 INFO Executor:59 - Finished task 0.0 in stage 107.0 (TID 211). 1375 bytes result sent to driver -2016-05-24 22:14:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 107.0 (TID 212) in 102 ms on localhost (1/2) -2016-05-24 22:14:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 107.0 (TID 211) in 104 ms on localhost (2/2) -2016-05-24 22:14:22 INFO DAGScheduler:59 - ShuffleMapStage 107 (combineByKey at BinaryClassificationMetrics.scala:152) finished in 0.105 s -2016-05-24 22:14:22 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:14:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 107.0, whose tasks have all completed, from pool -2016-05-24 22:14:22 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:14:22 INFO DAGScheduler:59 - waiting: Set(ResultStage 108) -2016-05-24 22:14:22 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:14:22 INFO DAGScheduler:59 - Missing parents for ResultStage 108: List() -2016-05-24 22:14:22 INFO DAGScheduler:59 - Submitting ResultStage 108 (MapPartitionsRDD[218] at mapPartitions at BinaryClassificationMetrics.scala:189), which is now runnable -2016-05-24 22:14:22 INFO MemoryStore:59 - ensureFreeSpace(3472) called with curMem=1177419, maxMem=766075207 -2016-05-24 22:14:22 INFO MemoryStore:59 - Block broadcast_208 stored as values in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:14:22 INFO MemoryStore:59 - ensureFreeSpace(1981) called with curMem=1180891, maxMem=766075207 -2016-05-24 22:14:22 INFO MemoryStore:59 - Block broadcast_208_piece0 stored as bytes in memory (estimated size 1981.0 B, free 729.5 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Added broadcast_208_piece0 in memory on localhost:56664 (size: 1981.0 B, free: 730.1 MB) -2016-05-24 22:14:22 INFO SparkContext:59 - Created broadcast 208 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 108 (MapPartitionsRDD[218] at mapPartitions at BinaryClassificationMetrics.scala:189) -2016-05-24 22:14:22 INFO TaskSchedulerImpl:59 - Adding task set 108.0 with 2 tasks -2016-05-24 22:14:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 108.0 (TID 213, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:14:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 108.0 (TID 214, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:14:22 INFO Executor:59 - Running task 1.0 in stage 108.0 (TID 214) -2016-05-24 22:14:22 INFO Executor:59 - Running task 0.0 in stage 108.0 (TID 213) -2016-05-24 22:14:22 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:14:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:14:22 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:14:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:14:22 INFO Executor:59 - Finished task 1.0 in stage 108.0 (TID 214). 1330 bytes result sent to driver -2016-05-24 22:14:22 INFO Executor:59 - Finished task 0.0 in stage 108.0 (TID 213). 1330 bytes result sent to driver -2016-05-24 22:14:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 108.0 (TID 214) in 20 ms on localhost (1/2) -2016-05-24 22:14:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 108.0 (TID 213) in 21 ms on localhost (2/2) -2016-05-24 22:14:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 108.0, whose tasks have all completed, from pool -2016-05-24 22:14:22 INFO DAGScheduler:59 - ResultStage 108 (collect at BinaryClassificationMetrics.scala:193) finished in 0.021 s -2016-05-24 22:14:22 INFO DAGScheduler:59 - Job 105 finished: collect at BinaryClassificationMetrics.scala:193, took 0.158185 s -2016-05-24 22:14:22 INFO BinaryClassificationMetrics:59 - Total counts: {numPos: 25, numNeg: 11} -2016-05-24 22:14:22 INFO SparkContext:59 - Starting job: runJob at SlidingRDD.scala:75 -2016-05-24 22:14:22 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 154 bytes -2016-05-24 22:14:22 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-24 22:14:22 INFO DAGScheduler:59 - Got job 106 (runJob at SlidingRDD.scala:75) with 3 output partitions -2016-05-24 22:14:22 INFO DAGScheduler:59 - Final stage: ResultStage 111(runJob at SlidingRDD.scala:75) -2016-05-24 22:14:22 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 110) -2016-05-24 22:14:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:22 INFO DAGScheduler:59 - Submitting ResultStage 111 (UnionRDD[224] at UnionRDD at BinaryClassificationMetrics.scala:92), which has no missing parents -2016-05-24 22:14:22 INFO MemoryStore:59 - ensureFreeSpace(5184) called with curMem=1182872, maxMem=766075207 -2016-05-24 22:14:22 INFO MemoryStore:59 - Block broadcast_209 stored as values in memory (estimated size 5.1 KB, free 729.5 MB) -2016-05-24 22:14:22 INFO MemoryStore:59 - ensureFreeSpace(2788) called with curMem=1188056, maxMem=766075207 -2016-05-24 22:14:22 INFO MemoryStore:59 - Block broadcast_209_piece0 stored as bytes in memory (estimated size 2.7 KB, free 729.5 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Added broadcast_209_piece0 in memory on localhost:56664 (size: 2.7 KB, free: 730.1 MB) -2016-05-24 22:14:22 INFO SparkContext:59 - Created broadcast 209 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:22 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 111 (UnionRDD[224] at UnionRDD at BinaryClassificationMetrics.scala:92) -2016-05-24 22:14:22 INFO TaskSchedulerImpl:59 - Adding task set 111.0 with 3 tasks -2016-05-24 22:14:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 111.0 (TID 215, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:14:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 111.0 (TID 216, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:14:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 111.0 (TID 217, localhost, PROCESS_LOCAL, 2318 bytes) -2016-05-24 22:14:22 INFO Executor:59 - Running task 0.0 in stage 111.0 (TID 215) -2016-05-24 22:14:22 INFO Executor:59 - Running task 1.0 in stage 111.0 (TID 216) -2016-05-24 22:14:22 INFO Executor:59 - Running task 2.0 in stage 111.0 (TID 217) -2016-05-24 22:14:22 INFO Executor:59 - Finished task 2.0 in stage 111.0 (TID 217). 1053 bytes result sent to driver -2016-05-24 22:14:22 INFO CacheManager:59 - Partition rdd_219_0 not found, computing it -2016-05-24 22:14:22 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:14:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_204_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:22 INFO MemoryStore:59 - ensureFreeSpace(1448) called with curMem=1175580, maxMem=766075207 -2016-05-24 22:14:22 INFO MemoryStore:59 - Block rdd_219_0 stored as values in memory (estimated size 1448.0 B, free 729.5 MB) -2016-05-24 22:14:22 INFO CacheManager:59 - Partition rdd_219_1 not found, computing it -2016-05-24 22:14:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 111.0 (TID 217) in 53 ms on localhost (1/3) -2016-05-24 22:14:22 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:14:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Added rdd_219_0 in memory on localhost:56664 (size: 1448.0 B, free: 730.1 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 104 -2016-05-24 22:14:22 INFO MemoryStore:59 - ensureFreeSpace(1616) called with curMem=1173416, maxMem=766075207 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_203_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:22 INFO MemoryStore:59 - Block rdd_219_1 stored as values in memory (estimated size 1616.0 B, free 729.5 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Added rdd_219_1 in memory on localhost:56664 (size: 1616.0 B, free: 730.1 MB) -2016-05-24 22:14:22 INFO Executor:59 - Finished task 0.0 in stage 111.0 (TID 215). 1883 bytes result sent to driver -2016-05-24 22:14:22 INFO Executor:59 - Finished task 1.0 in stage 111.0 (TID 216). 1883 bytes result sent to driver -2016-05-24 22:14:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 111.0 (TID 216) in 63 ms on localhost (2/3) -2016-05-24 22:14:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 111.0 (TID 215) in 65 ms on localhost (3/3) -2016-05-24 22:14:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 111.0, whose tasks have all completed, from pool -2016-05-24 22:14:22 INFO DAGScheduler:59 - ResultStage 111 (runJob at SlidingRDD.scala:75) finished in 0.066 s -2016-05-24 22:14:22 INFO DAGScheduler:59 - Job 106 finished: runJob at SlidingRDD.scala:75, took 0.097317 s -2016-05-24 22:14:22 INFO SparkContext:59 - Starting job: aggregate at AreaUnderCurve.scala:45 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_202_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 103 -2016-05-24 22:14:22 INFO DAGScheduler:59 - Got job 107 (aggregate at AreaUnderCurve.scala:45) with 4 output partitions -2016-05-24 22:14:22 INFO DAGScheduler:59 - Final stage: ResultStage 114(aggregate at AreaUnderCurve.scala:45) -2016-05-24 22:14:22 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 113) -2016-05-24 22:14:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:22 INFO DAGScheduler:59 - Submitting ResultStage 114 (SlidingRDD[225] at RDD at SlidingRDD.scala:47), which has no missing parents -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_201_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:22 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=1144967, maxMem=766075207 -2016-05-24 22:14:22 INFO MemoryStore:59 - Block broadcast_210 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:14:22 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=1135355, maxMem=766075207 -2016-05-24 22:14:22 INFO MemoryStore:59 - Block broadcast_210_piece0 stored as bytes in memory (estimated size 2.9 KB, free 729.5 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_200_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 102 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Added broadcast_210_piece0 in memory on localhost:56664 (size: 2.9 KB, free: 730.1 MB) -2016-05-24 22:14:22 INFO SparkContext:59 - Created broadcast 210 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 114 (SlidingRDD[225] at RDD at SlidingRDD.scala:47) -2016-05-24 22:14:22 INFO TaskSchedulerImpl:59 - Adding task set 114.0 with 4 tasks -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_199_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 114.0 (TID 218, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:14:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 114.0 (TID 219, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_198_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 114.0 (TID 220, localhost, PROCESS_LOCAL, 2635 bytes) -2016-05-24 22:14:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 114.0 (TID 221, localhost, PROCESS_LOCAL, 2475 bytes) -2016-05-24 22:14:22 INFO Executor:59 - Running task 2.0 in stage 114.0 (TID 219) -2016-05-24 22:14:22 INFO Executor:59 - Running task 1.0 in stage 114.0 (TID 218) -2016-05-24 22:14:22 INFO Executor:59 - Running task 0.0 in stage 114.0 (TID 220) -2016-05-24 22:14:22 INFO BlockManager:59 - Found block rdd_219_1 locally -2016-05-24 22:14:22 INFO Executor:59 - Finished task 0.0 in stage 114.0 (TID 220). 955 bytes result sent to driver -2016-05-24 22:14:22 INFO BlockManager:59 - Found block rdd_219_0 locally -2016-05-24 22:14:22 INFO Executor:59 - Finished task 2.0 in stage 114.0 (TID 219). 2084 bytes result sent to driver -2016-05-24 22:14:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 114.0 (TID 220) in 11 ms on localhost (1/4) -2016-05-24 22:14:22 INFO Executor:59 - Running task 3.0 in stage 114.0 (TID 221) -2016-05-24 22:14:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 114.0 (TID 219) in 13 ms on localhost (2/4) -2016-05-24 22:14:22 INFO Executor:59 - Finished task 1.0 in stage 114.0 (TID 218). 2084 bytes result sent to driver -2016-05-24 22:14:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 114.0 (TID 218) in 18 ms on localhost (3/4) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 101 -2016-05-24 22:14:22 INFO Executor:59 - Finished task 3.0 in stage 114.0 (TID 221). 955 bytes result sent to driver -2016-05-24 22:14:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 114.0 (TID 221) in 14 ms on localhost (4/4) -2016-05-24 22:14:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 114.0, whose tasks have all completed, from pool -2016-05-24 22:14:22 INFO DAGScheduler:59 - ResultStage 114 (aggregate at AreaUnderCurve.scala:45) finished in 0.021 s -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_197_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:22 INFO DAGScheduler:59 - Job 107 finished: aggregate at AreaUnderCurve.scala:45, took 0.033103 s -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_196_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 100 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_195_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_194_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 99 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_193_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_192_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 98 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_191_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_190_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 97 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_189_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_188_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 96 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_187_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_186_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 95 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_185_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_184_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 94 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_183_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_182_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 93 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_181_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_180_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 92 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_179_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_178_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 91 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_177_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_176_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 90 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_175_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_174_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 89 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_173_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_172_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 88 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_171_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_170_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 87 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_169_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_168_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 86 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_167_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_166_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 85 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_165_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_164_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 84 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_163_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_162_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 83 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_161_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_160_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 82 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_159_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_158_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 81 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_157_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_156_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 80 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_155_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_154_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 79 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_153_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_152_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 78 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_151_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_150_piece0 on localhost:56664 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 77 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_149_piece0 on localhost:56664 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_208_piece0 on localhost:56664 in memory (size: 1981.0 B, free: 730.3 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 108 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_207_piece0 on localhost:56664 in memory (size: 1658.0 B, free: 730.3 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 107 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_206_piece0 on localhost:56664 in memory (size: 1900.0 B, free: 730.3 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 106 -2016-05-24 22:14:22 INFO BlockManagerInfo:59 - Removed broadcast_205_piece0 on localhost:56664 in memory (size: 6.3 KB, free: 730.3 MB) -2016-05-24 22:14:22 INFO ContextCleaner:59 - Cleaned accumulator 105 -2016-05-24 22:14:23 INFO SparkContext:59 - Starting job: saveAsTextFile at GLMClassificationModel.scala:61 -2016-05-24 22:14:23 INFO DAGScheduler:59 - Got job 108 (saveAsTextFile at GLMClassificationModel.scala:61) with 1 output partitions -2016-05-24 22:14:23 INFO DAGScheduler:59 - Final stage: ResultStage 115(saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:14:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:23 INFO DAGScheduler:59 - Submitting ResultStage 115 (MapPartitionsRDD[227] at saveAsTextFile at GLMClassificationModel.scala:61), which has no missing parents -2016-05-24 22:14:23 INFO MemoryStore:59 - ensureFreeSpace(126856) called with curMem=484588, maxMem=766075207 -2016-05-24 22:14:23 INFO MemoryStore:59 - Block broadcast_211 stored as values in memory (estimated size 123.9 KB, free 730.0 MB) -2016-05-24 22:14:23 INFO MemoryStore:59 - ensureFreeSpace(42460) called with curMem=611444, maxMem=766075207 -2016-05-24 22:14:23 INFO MemoryStore:59 - Block broadcast_211_piece0 stored as bytes in memory (estimated size 41.5 KB, free 730.0 MB) -2016-05-24 22:14:23 INFO BlockManagerInfo:59 - Added broadcast_211_piece0 in memory on localhost:56664 (size: 41.5 KB, free: 730.2 MB) -2016-05-24 22:14:23 INFO SparkContext:59 - Created broadcast 211 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 115 (MapPartitionsRDD[227] at saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:14:23 INFO TaskSchedulerImpl:59 - Adding task set 115.0 with 1 tasks -2016-05-24 22:14:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 115.0 (TID 222, localhost, PROCESS_LOCAL, 2197 bytes) -2016-05-24 22:14:23 INFO Executor:59 - Running task 0.0 in stage 115.0 (TID 222) -2016-05-24 22:14:23 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242214_0115_m_000000_222' to file:/D:/all/idea/SparkLearning/myModelPath/metadata/_temporary/0/task_201605242214_0115_m_000000 -2016-05-24 22:14:23 INFO SparkHadoopMapRedUtil:59 - attempt_201605242214_0115_m_000000_222: Committed -2016-05-24 22:14:23 INFO Executor:59 - Finished task 0.0 in stage 115.0 (TID 222). 1864 bytes result sent to driver -2016-05-24 22:14:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 115.0 (TID 222) in 273 ms on localhost (1/1) -2016-05-24 22:14:23 INFO DAGScheduler:59 - ResultStage 115 (saveAsTextFile at GLMClassificationModel.scala:61) finished in 0.273 s -2016-05-24 22:14:23 INFO DAGScheduler:59 - Job 108 finished: saveAsTextFile at GLMClassificationModel.scala:61, took 0.341010 s -2016-05-24 22:14:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 115.0, whose tasks have all completed, from pool -2016-05-24 22:14:26 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:14:26 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:14:26 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:65 -2016-05-24 22:14:26 INFO DAGScheduler:59 - Got job 109 (parquet at GLMClassificationModel.scala:65) with 1 output partitions -2016-05-24 22:14:26 INFO DAGScheduler:59 - Final stage: ResultStage 116(parquet at GLMClassificationModel.scala:65) -2016-05-24 22:14:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:26 INFO DAGScheduler:59 - Submitting ResultStage 116 (MapPartitionsRDD[229] at rddToDataFrameHolder at GLMClassificationModel.scala:65), which has no missing parents -2016-05-24 22:14:26 INFO MemoryStore:59 - ensureFreeSpace(66976) called with curMem=653904, maxMem=766075207 -2016-05-24 22:14:26 INFO MemoryStore:59 - Block broadcast_212 stored as values in memory (estimated size 65.4 KB, free 729.9 MB) -2016-05-24 22:14:26 INFO MemoryStore:59 - ensureFreeSpace(23370) called with curMem=720880, maxMem=766075207 -2016-05-24 22:14:26 INFO MemoryStore:59 - Block broadcast_212_piece0 stored as bytes in memory (estimated size 22.8 KB, free 729.9 MB) -2016-05-24 22:14:26 INFO BlockManagerInfo:59 - Added broadcast_212_piece0 in memory on localhost:56664 (size: 22.8 KB, free: 730.2 MB) -2016-05-24 22:14:26 INFO SparkContext:59 - Created broadcast 212 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 116 (MapPartitionsRDD[229] at rddToDataFrameHolder at GLMClassificationModel.scala:65) -2016-05-24 22:14:26 INFO TaskSchedulerImpl:59 - Adding task set 116.0 with 1 tasks -2016-05-24 22:14:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 116.0 (TID 223, localhost, PROCESS_LOCAL, 7978 bytes) -2016-05-24 22:14:26 INFO Executor:59 - Running task 0.0 in stage 116.0 (TID 223) -2016-05-24 22:14:26 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:14:26 INFO CodecConfig:151 - Compression: GZIP -2016-05-24 22:14:26 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-24 22:14:26 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-24 22:14:26 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-24 22:14:26 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-24 22:14:26 INFO ParquetOutputFormat:151 - Validation is off -2016-05-24 22:14:26 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-24 22:14:26 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-24 22:14:27 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 6,156 -2016-05-24 22:14:27 INFO ColumnChunkPageWriteStore:151 - written 61B for [weights, type] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-24 22:14:27 INFO ColumnChunkPageWriteStore:151 - written 48B for [weights, size] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-24 22:14:27 INFO ColumnChunkPageWriteStore:151 - written 51B for [weights, indices, array] INT32: 1 values, 13B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-24 22:14:27 INFO ColumnChunkPageWriteStore:151 - written 637B for [weights, values, array] DOUBLE: 692 values, 668B raw, 593B comp, 1 pages, encodings: [RLE, PLAIN_DICTIONARY], dic { 413 entries, 3,304B raw, 413B comp} -2016-05-24 22:14:27 INFO ColumnChunkPageWriteStore:151 - written 69B for [intercept] DOUBLE: 1 values, 14B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-24 22:14:27 INFO ColumnChunkPageWriteStore:151 - written 47B for [threshold] DOUBLE: 1 values, 6B raw, 26B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-24 22:14:27 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242214_0116_m_000000_0' to file:/D:/all/idea/SparkLearning/myModelPath/data/_temporary/0/task_201605242214_0116_m_000000 -2016-05-24 22:14:27 INFO SparkHadoopMapRedUtil:59 - attempt_201605242214_0116_m_000000_0: Committed -2016-05-24 22:14:27 INFO Executor:59 - Finished task 0.0 in stage 116.0 (TID 223). 915 bytes result sent to driver -2016-05-24 22:14:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 116.0 (TID 223) in 1059 ms on localhost (1/1) -2016-05-24 22:14:27 INFO DAGScheduler:59 - ResultStage 116 (parquet at GLMClassificationModel.scala:65) finished in 1.059 s -2016-05-24 22:14:27 INFO DAGScheduler:59 - Job 109 finished: parquet at GLMClassificationModel.scala:65, took 1.095817 s -2016-05-24 22:14:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 116.0, whose tasks have all completed, from pool -2016-05-24 22:14:27 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:14:27 INFO DefaultWriterContainer:59 - Job job_201605242214_0000 committed. -2016-05-24 22:14:27 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/myModelPath/data on driver -2016-05-24 22:14:27 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/myModelPath/data on driver -2016-05-24 22:14:27 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=744250, maxMem=766075207 -2016-05-24 22:14:27 INFO MemoryStore:59 - Block broadcast_213 stored as values in memory (estimated size 225.3 KB, free 729.7 MB) -2016-05-24 22:14:28 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=974954, maxMem=766075207 -2016-05-24 22:14:28 INFO MemoryStore:59 - Block broadcast_213_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.6 MB) -2016-05-24 22:14:28 INFO BlockManagerInfo:59 - Added broadcast_213_piece0 in memory on localhost:56664 (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:14:28 INFO SparkContext:59 - Created broadcast 213 from textFile at modelSaveLoad.scala:129 -2016-05-24 22:14:28 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:14:28 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-24 22:14:28 INFO DAGScheduler:59 - Got job 110 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-24 22:14:28 INFO DAGScheduler:59 - Final stage: ResultStage 117(first at modelSaveLoad.scala:129) -2016-05-24 22:14:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:28 INFO DAGScheduler:59 - Submitting ResultStage 117 (MapPartitionsRDD[232] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-24 22:14:28 INFO MemoryStore:59 - ensureFreeSpace(3152) called with curMem=994742, maxMem=766075207 -2016-05-24 22:14:28 INFO MemoryStore:59 - Block broadcast_214 stored as values in memory (estimated size 3.1 KB, free 729.6 MB) -2016-05-24 22:14:28 INFO MemoryStore:59 - ensureFreeSpace(1829) called with curMem=997894, maxMem=766075207 -2016-05-24 22:14:28 INFO MemoryStore:59 - Block broadcast_214_piece0 stored as bytes in memory (estimated size 1829.0 B, free 729.6 MB) -2016-05-24 22:14:28 INFO BlockManagerInfo:59 - Added broadcast_214_piece0 in memory on localhost:56664 (size: 1829.0 B, free: 730.2 MB) -2016-05-24 22:14:28 INFO SparkContext:59 - Created broadcast 214 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 117 (MapPartitionsRDD[232] at textFile at modelSaveLoad.scala:129) -2016-05-24 22:14:28 INFO TaskSchedulerImpl:59 - Adding task set 117.0 with 1 tasks -2016-05-24 22:14:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 117.0 (TID 224, localhost, PROCESS_LOCAL, 2172 bytes) -2016-05-24 22:14:28 INFO Executor:59 - Running task 0.0 in stage 117.0 (TID 224) -2016-05-24 22:14:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/myModelPath/metadata/part-00000:0+54 -2016-05-24 22:14:28 INFO Executor:59 - Finished task 0.0 in stage 117.0 (TID 224). 2154 bytes result sent to driver -2016-05-24 22:14:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 117.0 (TID 224) in 11 ms on localhost (1/1) -2016-05-24 22:14:28 INFO DAGScheduler:59 - ResultStage 117 (first at modelSaveLoad.scala:129) finished in 0.011 s -2016-05-24 22:14:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 117.0, whose tasks have all completed, from pool -2016-05-24 22:14:28 INFO DAGScheduler:59 - Job 110 finished: first at modelSaveLoad.scala:129, took 0.022051 s -2016-05-24 22:14:28 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/myModelPath/data on driver -2016-05-24 22:14:28 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:78 -2016-05-24 22:14:28 INFO DAGScheduler:59 - Got job 111 (parquet at GLMClassificationModel.scala:78) with 4 output partitions -2016-05-24 22:14:28 INFO DAGScheduler:59 - Final stage: ResultStage 118(parquet at GLMClassificationModel.scala:78) -2016-05-24 22:14:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:28 INFO DAGScheduler:59 - Submitting ResultStage 118 (MapPartitionsRDD[234] at parquet at GLMClassificationModel.scala:78), which has no missing parents -2016-05-24 22:14:28 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=999723, maxMem=766075207 -2016-05-24 22:14:28 INFO MemoryStore:59 - Block broadcast_215 stored as values in memory (estimated size 61.5 KB, free 729.6 MB) -2016-05-24 22:14:28 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=1062723, maxMem=766075207 -2016-05-24 22:14:28 INFO MemoryStore:59 - Block broadcast_215_piece0 stored as bytes in memory (estimated size 20.6 KB, free 729.6 MB) -2016-05-24 22:14:28 INFO BlockManagerInfo:59 - Added broadcast_215_piece0 in memory on localhost:56664 (size: 20.6 KB, free: 730.2 MB) -2016-05-24 22:14:28 INFO SparkContext:59 - Created broadcast 215 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 118 (MapPartitionsRDD[234] at parquet at GLMClassificationModel.scala:78) -2016-05-24 22:14:28 INFO TaskSchedulerImpl:59 - Adding task set 118.0 with 4 tasks -2016-05-24 22:14:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 118.0 (TID 225, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:14:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 118.0 (TID 226, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:14:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 118.0 (TID 227, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:14:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 118.0 (TID 228, localhost, PROCESS_LOCAL, 2214 bytes) -2016-05-24 22:14:28 INFO Executor:59 - Running task 1.0 in stage 118.0 (TID 226) -2016-05-24 22:14:28 INFO Executor:59 - Running task 3.0 in stage 118.0 (TID 228) -2016-05-24 22:14:28 INFO Executor:59 - Running task 2.0 in stage 118.0 (TID 227) -2016-05-24 22:14:28 INFO Executor:59 - Running task 0.0 in stage 118.0 (TID 225) -2016-05-24 22:14:28 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:14:28 INFO Executor:59 - Finished task 2.0 in stage 118.0 (TID 227). 936 bytes result sent to driver -2016-05-24 22:14:28 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:14:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 118.0 (TID 227) in 75 ms on localhost (1/4) -2016-05-24 22:14:28 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:14:28 INFO Executor:59 - Finished task 1.0 in stage 118.0 (TID 226). 936 bytes result sent to driver -2016-05-24 22:14:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 118.0 (TID 226) in 80 ms on localhost (2/4) -2016-05-24 22:14:28 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:14:28 INFO Executor:59 - Finished task 0.0 in stage 118.0 (TID 225). 936 bytes result sent to driver -2016-05-24 22:14:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 118.0 (TID 225) in 95 ms on localhost (3/4) -2016-05-24 22:14:28 INFO Executor:59 - Finished task 3.0 in stage 118.0 (TID 228). 1802 bytes result sent to driver -2016-05-24 22:14:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 118.0 (TID 228) in 123 ms on localhost (4/4) -2016-05-24 22:14:28 INFO DAGScheduler:59 - ResultStage 118 (parquet at GLMClassificationModel.scala:78) finished in 0.125 s -2016-05-24 22:14:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 118.0, whose tasks have all completed, from pool -2016-05-24 22:14:28 INFO DAGScheduler:59 - Job 111 finished: parquet at GLMClassificationModel.scala:78, took 0.153086 s -2016-05-24 22:14:28 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=1083796, maxMem=766075207 -2016-05-24 22:14:28 INFO MemoryStore:59 - Block broadcast_216 stored as values in memory (estimated size 86.4 KB, free 729.5 MB) -2016-05-24 22:14:28 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1172308, maxMem=766075207 -2016-05-24 22:14:28 INFO MemoryStore:59 - Block broadcast_216_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.4 MB) -2016-05-24 22:14:28 INFO BlockManagerInfo:59 - Added broadcast_216_piece0 in memory on localhost:56664 (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:14:28 INFO SparkContext:59 - Created broadcast 216 from take at GLMClassificationModel.scala:79 -2016-05-24 22:14:29 INFO BlockManagerInfo:59 - Removed broadcast_209_piece0 on localhost:56664 in memory (size: 2.7 KB, free: 730.2 MB) -2016-05-24 22:14:29 INFO BlockManagerInfo:59 - Removed broadcast_215_piece0 on localhost:56664 in memory (size: 20.6 KB, free: 730.2 MB) -2016-05-24 22:14:29 INFO ContextCleaner:59 - Cleaned accumulator 114 -2016-05-24 22:14:29 INFO BlockManagerInfo:59 - Removed broadcast_214_piece0 on localhost:56664 in memory (size: 1829.0 B, free: 730.2 MB) -2016-05-24 22:14:29 INFO ContextCleaner:59 - Cleaned accumulator 113 -2016-05-24 22:14:29 INFO BlockManagerInfo:59 - Removed broadcast_213_piece0 on localhost:56664 in memory (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:14:29 INFO BlockManagerInfo:59 - Removed broadcast_212_piece0 on localhost:56664 in memory (size: 22.8 KB, free: 730.2 MB) -2016-05-24 22:14:29 INFO ContextCleaner:59 - Cleaned accumulator 112 -2016-05-24 22:14:29 INFO BlockManagerInfo:59 - Removed broadcast_211_piece0 on localhost:56664 in memory (size: 41.5 KB, free: 730.3 MB) -2016-05-24 22:14:29 INFO ContextCleaner:59 - Cleaned accumulator 111 -2016-05-24 22:14:29 INFO BlockManagerInfo:59 - Removed broadcast_210_piece0 on localhost:56664 in memory (size: 2.9 KB, free: 730.3 MB) -2016-05-24 22:14:29 INFO ContextCleaner:59 - Cleaned accumulator 110 -2016-05-24 22:14:29 INFO ContextCleaner:59 - Cleaned accumulator 109 -2016-05-24 22:14:29 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:56664 in memory (size: 2.1 KB, free: 730.3 MB) -2016-05-24 22:14:29 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 22:14:30 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-24 22:14:30 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/myModelPath/data/part-r-00000-8753de36-6942-4f31-a27a-ba436d7c572f.gz.parquet -2016-05-24 22:14:30 INFO SparkContext:59 - Starting job: take at GLMClassificationModel.scala:79 -2016-05-24 22:14:30 INFO DAGScheduler:59 - Got job 112 (take at GLMClassificationModel.scala:79) with 1 output partitions -2016-05-24 22:14:30 INFO DAGScheduler:59 - Final stage: ResultStage 119(take at GLMClassificationModel.scala:79) -2016-05-24 22:14:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:14:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:14:30 INFO DAGScheduler:59 - Submitting ResultStage 119 (MapPartitionsRDD[236] at take at GLMClassificationModel.scala:79), which has no missing parents -2016-05-24 22:14:30 INFO MemoryStore:59 - ensureFreeSpace(4168) called with curMem=570248, maxMem=766075207 -2016-05-24 22:14:30 INFO MemoryStore:59 - Block broadcast_217 stored as values in memory (estimated size 4.1 KB, free 730.0 MB) -2016-05-24 22:14:30 INFO MemoryStore:59 - ensureFreeSpace(2438) called with curMem=574416, maxMem=766075207 -2016-05-24 22:14:30 INFO MemoryStore:59 - Block broadcast_217_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.0 MB) -2016-05-24 22:14:30 INFO BlockManagerInfo:59 - Added broadcast_217_piece0 in memory on localhost:56664 (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:14:30 INFO SparkContext:59 - Created broadcast 217 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:14:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 119 (MapPartitionsRDD[236] at take at GLMClassificationModel.scala:79) -2016-05-24 22:14:30 INFO TaskSchedulerImpl:59 - Adding task set 119.0 with 1 tasks -2016-05-24 22:14:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 119.0 (TID 229, localhost, PROCESS_LOCAL, 2254 bytes) -2016-05-24 22:14:30 INFO Executor:59 - Running task 0.0 in stage 119.0 (TID 229) -2016-05-24 22:14:30 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/myModelPath/data/part-r-00000-8753de36-6942-4f31-a27a-ba436d7c572f.gz.parquet start: 0 end: 5410 length: 5410 hosts: []} -2016-05-24 22:14:30 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-24 22:14:30 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group weights { - required int32 type (INT_8); - optional int32 size; - optional group indices (LIST) { - repeated int32 array; - } - optional group values (LIST) { - repeated double array; - } - } - optional double intercept; - optional double threshold; -} - -Catalyst form: -StructType(StructField(weights,org.apache.spark.mllib.linalg.VectorUDT@f71b0bce,true), StructField(intercept,DoubleType,true), StructField(threshold,DoubleType,true)) - -2016-05-24 22:14:31 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-24 22:14:31 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-24 22:14:31 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-24 22:14:31 INFO InternalParquetRecordReader:151 - block read in memory in 36 ms. row count = 1 -2016-05-24 22:17:53 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 22:17:54 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 22:17:54 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 22:17:54 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 22:17:54 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 22:17:56 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 22:17:56 INFO Remoting:74 - Starting remoting -2016-05-24 22:17:56 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:56697] -2016-05-24 22:17:56 INFO Utils:59 - Successfully started service 'sparkDriver' on port 56697. -2016-05-24 22:17:56 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 22:17:56 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 22:17:56 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-a9150fbd-d618-461b-b2b5-f7285b3d626e -2016-05-24 22:17:56 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 22:17:56 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-7736e226-cd3c-4af7-8c00-b09628b9f89c\httpd-f1397615-c321-4eca-bed7-edb140dd8cec -2016-05-24 22:17:56 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 22:17:56 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:17:56 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:56698 -2016-05-24 22:17:56 INFO Utils:59 - Successfully started service 'HTTP file server' on port 56698. -2016-05-24 22:17:57 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 22:17:57 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:17:57 WARN AbstractLifeCycle:204 - FAILED SelectChannelConnector@0.0.0.0:4040: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.mllib.learning.regression.SVMFromSparkLearning$.main(SVMFromSparkLearning.scala:26) - at org.apache.spark.mllib.learning.regression.SVMFromSparkLearning.main(SVMFromSparkLearning.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-05-24 22:17:57 WARN AbstractLifeCycle:204 - FAILED org.spark-project.jetty.server.Server@c9a8f1d: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.mllib.learning.regression.SVMFromSparkLearning$.main(SVMFromSparkLearning.scala:26) - at org.apache.spark.mllib.learning.regression.SVMFromSparkLearning.main(SVMFromSparkLearning.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 22:17:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 22:17:57 WARN Utils:71 - Service 'SparkUI' could not bind on port 4040. Attempting port 4041. -2016-05-24 22:17:57 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:17:57 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4041 -2016-05-24 22:17:57 INFO Utils:59 - Successfully started service 'SparkUI' on port 4041. -2016-05-24 22:17:57 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4041 -2016-05-24 22:17:57 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 22:17:57 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 22:17:58 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 56717. -2016-05-24 22:17:58 INFO NettyBlockTransferService:59 - Server created on 56717 -2016-05-24 22:17:58 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 22:17:58 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:56717 with 730.6 MB RAM, BlockManagerId(driver, localhost, 56717) -2016-05-24 22:17:58 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 22:17:59 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 22:17:59 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 22:17:59 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 22:17:59 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 22:17:59 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:56717 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 22:17:59 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 22:18:01 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 22:18:01 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:18:01 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 22:18:01 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 22:18:01 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 22:18:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:01 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 22:18:01 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 22:18:01 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 22:18:01 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=175476, maxMem=766075207 -2016-05-24 22:18:01 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:18:01 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:56717 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:18:01 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 22:18:01 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 22:18:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:18:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:18:01 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 22:18:01 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 22:18:01 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 22:18:01 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 22:18:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 22:18:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 22:18:01 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 22:18:01 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 22:18:01 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 22:18:01 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 22:18:01 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 22:18:02 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177621, maxMem=766075207 -2016-05-24 22:18:02 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 22:18:02 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:56717 (size: 83.6 KB, free: 730.5 MB) -2016-05-24 22:18:02 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263269, maxMem=766075207 -2016-05-24 22:18:02 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 22:18:02 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:56717 (size: 83.1 KB, free: 730.4 MB) -2016-05-24 22:18:02 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 22:18:02 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 22:18:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 588 ms on localhost (1/2) -2016-05-24 22:18:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 621 ms on localhost (2/2) -2016-05-24 22:18:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 22:18:02 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.646 s -2016-05-24 22:18:02 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.902696 s -2016-05-24 22:18:02 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 22:18:02 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 22:18:02 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:18:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:02 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 22:18:02 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=348349, maxMem=766075207 -2016-05-24 22:18:02 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-24 22:18:02 INFO MemoryStore:59 - ensureFreeSpace(2483) called with curMem=352909, maxMem=766075207 -2016-05-24 22:18:02 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-24 22:18:02 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:56717 (size: 2.4 KB, free: 730.4 MB) -2016-05-24 22:18:02 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:02 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:18:02 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 22:18:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:18:02 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 22:18:02 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-05-24 22:18:02 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:18:02 INFO MemoryStore:59 - ensureFreeSpace(54392) called with curMem=355392, maxMem=766075207 -2016-05-24 22:18:02 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 53.1 KB, free 730.2 MB) -2016-05-24 22:18:02 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:56717 (size: 53.1 KB, free: 730.4 MB) -2016-05-24 22:18:02 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2611 bytes result sent to driver -2016-05-24 22:18:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 45 ms on localhost (1/1) -2016-05-24 22:18:02 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.045 s -2016-05-24 22:18:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 22:18:02 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.067890 s -2016-05-24 22:18:02 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 22:18:02 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 22:18:02 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 22:18:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:02 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 22:18:02 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=409784, maxMem=766075207 -2016-05-24 22:18:02 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-24 22:18:02 INFO MemoryStore:59 - ensureFreeSpace(2300) called with curMem=414032, maxMem=766075207 -2016-05-24 22:18:02 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 22:18:02 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:56717 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 22:18:02 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40) -2016-05-24 22:18:02 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 22:18:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:18:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:18:02 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 22:18:02 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 22:18:02 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:02 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 22:18:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 18 ms on localhost (1/2) -2016-05-24 22:18:02 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-05-24 22:18:02 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:18:02 INFO MemoryStore:59 - ensureFreeSpace(62184) called with curMem=416332, maxMem=766075207 -2016-05-24 22:18:02 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 60.7 KB, free 730.1 MB) -2016-05-24 22:18:02 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:56717 (size: 60.7 KB, free: 730.3 MB) -2016-05-24 22:18:02 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2662 bytes result sent to driver -2016-05-24 22:18:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 39 ms on localhost (2/2) -2016-05-24 22:18:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 22:18:02 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.071 s -2016-05-24 22:18:02 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.115895 s -2016-05-24 22:18:02 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 22:18:02 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 22:18:02 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 22:18:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:02 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 22:18:02 INFO MemoryStore:59 - ensureFreeSpace(4264) called with curMem=478516, maxMem=766075207 -2016-05-24 22:18:02 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.2 KB, free 730.1 MB) -2016-05-24 22:18:02 INFO MemoryStore:59 - ensureFreeSpace(2322) called with curMem=482780, maxMem=766075207 -2016-05-24 22:18:02 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 22:18:02 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:56717 (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:18:02 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 22:18:02 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 22:18:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:18:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:18:02 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 22:18:02 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:02 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 22:18:02 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 22:18:02 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:02 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 22:18:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 32 ms on localhost (1/2) -2016-05-24 22:18:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 34 ms on localhost (2/2) -2016-05-24 22:18:02 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.036 s -2016-05-24 22:18:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 22:18:02 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.056328 s -2016-05-24 22:18:03 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 22:18:03 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 22:18:03 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=485102, maxMem=766075207 -2016-05-24 22:18:03 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:18:03 INFO MemoryStore:59 - ensureFreeSpace(388) called with curMem=490694, maxMem=766075207 -2016-05-24 22:18:03 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 388.0 B, free 730.1 MB) -2016-05-24 22:18:03 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:56717 (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:18:03 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:03 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:03 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:03 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:03 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:03 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:03 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=491082, maxMem=766075207 -2016-05-24 22:18:03 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:18:03 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=502866, maxMem=766075207 -2016-05-24 22:18:03 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:18:03 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:03 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:03 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 22:18:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:03 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 22:18:03 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 22:18:03 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:03 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 7968 bytes result sent to driver -2016-05-24 22:18:03 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 27 ms on localhost (1/2) -2016-05-24 22:18:03 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 7968 bytes result sent to driver -2016-05-24 22:18:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 36 ms on localhost (2/2) -2016-05-24 22:18:03 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.037 s -2016-05-24 22:18:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 22:18:03 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.124548 s -2016-05-24 22:18:03 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=506347, maxMem=766075207 -2016-05-24 22:18:03 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:18:03 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=511939, maxMem=766075207 -2016-05-24 22:18:03 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-24 22:18:03 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:56717 (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:18:03 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:03 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:03 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:03 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:03 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:03 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:03 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=514177, maxMem=766075207 -2016-05-24 22:18:03 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:18:03 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=525961, maxMem=766075207 -2016-05-24 22:18:03 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:18:03 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:03 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:03 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 22:18:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:03 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 22:18:03 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 22:18:03 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:03 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:03 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 7968 bytes result sent to driver -2016-05-24 22:18:03 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 7968 bytes result sent to driver -2016-05-24 22:18:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 18 ms on localhost (1/2) -2016-05-24 22:18:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 20 ms on localhost (2/2) -2016-05-24 22:18:03 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:18:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 22:18:03 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.059444 s -2016-05-24 22:18:03 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=529444, maxMem=766075207 -2016-05-24 22:18:03 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:18:03 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=535036, maxMem=766075207 -2016-05-24 22:18:03 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:18:03 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:03 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:03 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:03 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:03 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:03 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:03 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:03 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=538645, maxMem=766075207 -2016-05-24 22:18:03 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:18:03 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=550429, maxMem=766075207 -2016-05-24 22:18:03 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:18:03 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:03 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:03 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 22:18:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:03 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 22:18:03 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 22:18:03 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:03 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:03 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 7968 bytes result sent to driver -2016-05-24 22:18:03 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 7968 bytes result sent to driver -2016-05-24 22:18:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 18 ms on localhost (1/2) -2016-05-24 22:18:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 23 ms on localhost (2/2) -2016-05-24 22:18:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 22:18:03 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.025 s -2016-05-24 22:18:03 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.044735 s -2016-05-24 22:18:03 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=553912, maxMem=766075207 -2016-05-24 22:18:03 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:18:03 INFO MemoryStore:59 - ensureFreeSpace(3603) called with curMem=559504, maxMem=766075207 -2016-05-24 22:18:03 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:03 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:03 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:03 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:03 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:03 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:03 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:03 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:03 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=563107, maxMem=766075207 -2016-05-24 22:18:03 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:03 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=574891, maxMem=766075207 -2016-05-24 22:18:03 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:03 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:03 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:03 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 22:18:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 22:18:04 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:04 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:04 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 109 ms on localhost (1/2) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 109 ms on localhost (2/2) -2016-05-24 22:18:04 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.111 s -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 22:18:04 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.137691 s -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:56717 in memory (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=570544, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=572655, maxMem=766075207 -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:04 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:56717 in memory (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:56717 in memory (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:56717 in memory (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 22:18:04 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:56717 in memory (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:04 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:56717 in memory (size: 2.1 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=517012, maxMem=766075207 -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:18:04 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=519595, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 22:18:04 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 22:18:04 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:04 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:04 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 28 ms on localhost (1/2) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 26 ms on localhost (2/2) -2016-05-24 22:18:04 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.029 s -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 22:18:04 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.085942 s -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=507810, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=513402, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:04 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:04 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=517008, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=528792, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 22:18:04 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:04 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 15 ms on localhost (1/2) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 18 ms on localhost (2/2) -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 22:18:04 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 22:18:04 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.033458 s -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=532276, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=537868, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:04 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:04 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=541474, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=553258, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 22:18:04 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:04 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 11 ms on localhost (1/2) -2016-05-24 22:18:04 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 16 ms on localhost (2/2) -2016-05-24 22:18:04 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 22:18:04 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.033370 s -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=556741, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3604) called with curMem=562333, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:04 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:04 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=565937, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=577721, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 22:18:04 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:04 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 13 ms on localhost (1/2) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 15 ms on localhost (2/2) -2016-05-24 22:18:04 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 22:18:04 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.032788 s -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=581204, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=586796, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:04 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:04 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=590402, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=602186, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 22:18:04 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:04 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 16 ms on localhost (1/2) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 18 ms on localhost (2/2) -2016-05-24 22:18:04 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 22:18:04 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.052277 s -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=605669, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=611261, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:04 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:04 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=614867, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=626651, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:04 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 22:18:04 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 18 ms on localhost (1/2) -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:04 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 24 ms on localhost (2/2) -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 22:18:04 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 22:18:04 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.044554 s -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=630135, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3610) called with curMem=635727, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:04 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:04 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=639337, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=651121, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 22:18:04 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:04 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 10 ms on localhost (1/2) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 11 ms on localhost (2/2) -2016-05-24 22:18:04 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 22:18:04 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.022208 s -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=654604, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=660196, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:04 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:04 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=663802, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=675586, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 22:18:04 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:04 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 16 ms on localhost (1/2) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 15 ms on localhost (2/2) -2016-05-24 22:18:04 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 22:18:04 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.029660 s -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=679070, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3610) called with curMem=684662, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:04 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:04 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=688272, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=700056, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 22:18:04 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:04 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 13 ms on localhost (1/2) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 17 ms on localhost (2/2) -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 22:18:04 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:18:04 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.080039 s -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=703539, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=709131, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:04 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:04 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=712737, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=724521, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 22:18:04 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:04 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 15 ms on localhost (1/2) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 17 ms on localhost (2/2) -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 22:18:04 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:18:04 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.037236 s -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=728004, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=733596, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:04 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:04 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=737202, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=748986, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:04 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 22:18:04 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:04 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:04 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 7968 bytes result sent to driver -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 16 ms on localhost (1/2) -2016-05-24 22:18:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 19 ms on localhost (2/2) -2016-05-24 22:18:04 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:18:04 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.078470 s -2016-05-24 22:18:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=752468, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:04 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=758060, maxMem=766075207 -2016-05-24 22:18:04 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:04 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:04 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:05 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:05 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=761673, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=773457, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 22:18:05 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:05 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 14 ms on localhost (1/2) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 16 ms on localhost (2/2) -2016-05-24 22:18:05 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 22:18:05 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.038836 s -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=776940, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=782532, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:05 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:05 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=786138, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=797922, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 22:18:05 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:05 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 15 ms on localhost (1/2) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 15 ms on localhost (2/2) -2016-05-24 22:18:05 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 22:18:05 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.059489 s -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=801405, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=806997, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:05 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:05 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=810609, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=822393, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 22:18:05 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:05 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 15 ms on localhost (1/2) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 16 ms on localhost (2/2) -2016-05-24 22:18:05 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 22:18:05 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.042263 s -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=825876, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3621) called with curMem=831468, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:05 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:05 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=835089, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=846873, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 22:18:05 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:05 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 27 ms on localhost (1/2) -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:05 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 31 ms on localhost (2/2) -2016-05-24 22:18:05 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.050 s -2016-05-24 22:18:05 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.100930 s -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=850356, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=855948, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:05 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:05 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=859560, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=871344, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 22:18:05 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:05 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:05 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 15 ms on localhost (1/2) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 19 ms on localhost (2/2) -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 22:18:05 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 22:18:05 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.053560 s -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=874827, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=880419, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:05 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:05 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=884031, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=895815, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 22:18:05 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:05 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:05 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 12 ms on localhost (1/2) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 14 ms on localhost (2/2) -2016-05-24 22:18:05 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 22:18:05 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.083212 s -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=899299, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=904891, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:05 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=893238, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=905022, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:18:05 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 22:18:05 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:05 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 10 ms on localhost (1/2) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 11 ms on localhost (2/2) -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 22:18:05 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:18:05 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.029183 s -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=884033, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=889625, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:05 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:05 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=859555, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=856072, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 22:18:05 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:05 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 22:18:05 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 8 ms on localhost (1/2) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 10 ms on localhost (2/2) -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 22:18:05 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.041244 s -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=822400, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3634) called with curMem=816208, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:05 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:05 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=767302, maxMem=766075207 -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=773494, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 22:18:05 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 22:18:05 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:05 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 8 ms on localhost (1/2) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 8 ms on localhost (2/2) -2016-05-24 22:18:05 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 22:18:05 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.019046 s -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=728039, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=730147, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:05 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=673045, maxMem=766075207 -2016-05-24 22:18:05 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=675633, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 22:18:05 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 22:18:05 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:05 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 10 ms on localhost (1/2) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 10 ms on localhost (2/2) -2016-05-24 22:18:05 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 22:18:05 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.025741 s -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=654649, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=656757, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:05 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:05 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:05 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=614924, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=626708, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 22:18:05 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 22:18:05 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:05 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 20 ms on localhost (1/2) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 21 ms on localhost (2/2) -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 22:18:05 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 22:18:05 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.038579 s -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=605730, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=611322, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:05 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:05 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=614934, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=626718, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 22:18:05 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:05 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 8 ms on localhost (1/2) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 8 ms on localhost (2/2) -2016-05-24 22:18:05 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 22:18:05 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.020359 s -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=630201, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=635793, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:05 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:05 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=639399, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=651183, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 22:18:05 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:05 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 10 ms on localhost (1/2) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 10 ms on localhost (2/2) -2016-05-24 22:18:05 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 22:18:05 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.024260 s -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=654667, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3623) called with curMem=660259, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:05 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:05 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=663882, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:05 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=675666, maxMem=766075207 -2016-05-24 22:18:05 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:05 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:05 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:05 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:05 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 22:18:05 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:05 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:05 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 7968 bytes result sent to driver -2016-05-24 22:18:05 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 8 ms on localhost (1/2) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 10 ms on localhost (2/2) -2016-05-24 22:18:06 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 22:18:06 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.022268 s -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=679149, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=684741, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:06 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=688347, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=700131, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 22:18:06 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:06 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 19 ms on localhost (1/2) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 20 ms on localhost (2/2) -2016-05-24 22:18:06 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:18:06 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.034063 s -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=703615, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3626) called with curMem=709207, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:06 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=712833, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=724617, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 22:18:06 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:06 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 10 ms on localhost (1/2) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 11 ms on localhost (2/2) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 22:18:06 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:18:06 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.062313 s -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=728101, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=733693, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:06 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=737305, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=749089, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 22:18:06 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:06 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:06 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 12 ms on localhost (1/2) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 21 ms on localhost (2/2) -2016-05-24 22:18:06 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 22:18:06 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.041874 s -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=752573, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=758165, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:06 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=761777, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=773561, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:06 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 22:18:06 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 14 ms on localhost (1/2) -2016-05-24 22:18:06 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 18 ms on localhost (2/2) -2016-05-24 22:18:06 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 22:18:06 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.098816 s -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=777044, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=782636, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:06 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=786248, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=798032, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 22:18:06 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:06 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 7 ms on localhost (1/2) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 7 ms on localhost (2/2) -2016-05-24 22:18:06 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 22:18:06 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.019876 s -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=801516, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=807108, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:06 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=810720, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=822504, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 22:18:06 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:06 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 9 ms on localhost (1/2) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 10 ms on localhost (2/2) -2016-05-24 22:18:06 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 22:18:06 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.028070 s -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=825986, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=831578, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:06 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=835190, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=846974, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 22:18:06 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:06 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 7 ms on localhost (1/2) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 9 ms on localhost (2/2) -2016-05-24 22:18:06 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 22:18:06 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.017836 s -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=850458, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=856050, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:06 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=859662, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=871446, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 22:18:06 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:06 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 6 ms on localhost (1/2) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 7 ms on localhost (2/2) -2016-05-24 22:18:06 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 22:18:06 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.017078 s -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=874930, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=880522, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:06 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=884136, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=895920, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:18:06 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:06 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 7 ms on localhost (1/2) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 9 ms on localhost (2/2) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 22:18:06 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:18:06 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.071549 s -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=886698, maxMem=766075207 -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:06 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=874914, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:06 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=825968, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=837752, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 22:18:06 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:06 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 10 ms on localhost (1/2) -2016-05-24 22:18:06 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 13 ms on localhost (2/2) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 22:18:06 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:18:06 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.030182 s -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=801495, maxMem=766075207 -2016-05-24 22:18:06 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=807087, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:06 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:06 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=777025, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=788809, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 22:18:06 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:06 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 12 ms on localhost (1/2) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 12 ms on localhost (2/2) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 22:18:06 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:18:06 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.035314 s -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=767818, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=769927, maxMem=766075207 -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:06 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:06 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[93] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=737283, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=749067, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[93] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 22:18:06 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:06 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 8 ms on localhost (1/2) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 10 ms on localhost (2/2) -2016-05-24 22:18:06 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 22:18:06 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.024580 s -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=703593, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=709185, maxMem=766075207 -2016-05-24 22:18:06 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:06 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[95] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=679117, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=690901, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[95] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 22:18:06 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:06 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 22:18:06 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 9 ms on localhost (1/2) -2016-05-24 22:18:06 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 12 ms on localhost (2/2) -2016-05-24 22:18:06 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 22:18:06 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.031309 s -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=654652, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=660244, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:06 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[97] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=630180, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=641964, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[97] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 22:18:06 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:06 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:06 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 45 ms on localhost (1/2) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 46 ms on localhost (2/2) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 22:18:06 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.047 s -2016-05-24 22:18:06 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.066072 s -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=630181, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3618) called with curMem=632161, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 91 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Got job 47 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:06 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[99] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=614923, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=626707, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 47 (MapPartitionsRDD[99] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 2 tasks -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 93, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 94, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 93) -2016-05-24 22:18:06 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 94) -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:06 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 93). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 94). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 93) in 7 ms on localhost (1/2) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 94) in 7 ms on localhost (2/2) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-24 22:18:06 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:18:06 INFO DAGScheduler:59 - Job 47 finished: treeAggregate at GradientDescent.scala:225, took 0.019208 s -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=630190, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=635782, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 93 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:06 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Got job 48 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:06 INFO DAGScheduler:59 - Final stage: ResultStage 48(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[101] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=639396, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=651180, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 48 (MapPartitionsRDD[101] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 2 tasks -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 95, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 96, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:06 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 96) -2016-05-24 22:18:06 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 95) -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:06 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:06 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 96). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 95). 7968 bytes result sent to driver -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 96) in 7 ms on localhost (1/2) -2016-05-24 22:18:06 INFO DAGScheduler:59 - ResultStage 48 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:18:06 INFO DAGScheduler:59 - Job 48 finished: treeAggregate at GradientDescent.scala:225, took 0.018991 s -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=654663, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:06 INFO MemoryStore:59 - ensureFreeSpace(3618) called with curMem=660255, maxMem=766075207 -2016-05-24 22:18:06 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 95) in 7 ms on localhost (2/2) -2016-05-24 22:18:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-24 22:18:06 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:06 INFO SparkContext:59 - Created broadcast 95 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Got job 49 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:07 INFO DAGScheduler:59 - Final stage: ResultStage 49(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[103] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=663873, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=675657, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 49 (MapPartitionsRDD[103] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 2 tasks -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 97, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 98, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 98) -2016-05-24 22:18:07 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 97) -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:07 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 98). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 97). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 97) in 15 ms on localhost (1/2) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 98) in 16 ms on localhost (2/2) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-05-24 22:18:07 INFO DAGScheduler:59 - ResultStage 49 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:18:07 INFO DAGScheduler:59 - Job 49 finished: treeAggregate at GradientDescent.scala:225, took 0.063451 s -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=679140, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3622) called with curMem=684732, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 97 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Got job 50 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:07 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[105] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=688354, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=700138, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 50 (MapPartitionsRDD[105] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 2 tasks -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-24 22:18:07 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:07 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 7 ms on localhost (1/2) -2016-05-24 22:18:07 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 9 ms on localhost (2/2) -2016-05-24 22:18:07 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-24 22:18:07 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at GradientDescent.scala:225, took 0.020511 s -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=703619, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=709211, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 99 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Got job 51 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:07 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[107] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=712824, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=724608, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 51 (MapPartitionsRDD[107] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-24 22:18:07 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:07 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 36 ms on localhost (1/2) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 36 ms on localhost (2/2) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-24 22:18:07 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at GradientDescent.scala:225) finished in 0.038 s -2016-05-24 22:18:07 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at GradientDescent.scala:225, took 0.052098 s -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=728090, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3628) called with curMem=733682, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 101 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Got job 52 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:07 INFO DAGScheduler:59 - Final stage: ResultStage 52(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting ResultStage 52 (MapPartitionsRDD[109] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=737310, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=749094, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 52 (MapPartitionsRDD[109] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 103, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 104, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 103) -2016-05-24 22:18:07 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 104) -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:07 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 104). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 103). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 104) in 4 ms on localhost (1/2) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 103) in 5 ms on localhost (2/2) -2016-05-24 22:18:07 INFO DAGScheduler:59 - ResultStage 52 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-24 22:18:07 INFO DAGScheduler:59 - Job 52 finished: treeAggregate at GradientDescent.scala:225, took 0.014253 s -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=752576, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=758168, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 103 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Got job 53 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:07 INFO DAGScheduler:59 - Final stage: ResultStage 53(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[111] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=761800, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=773584, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 104 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[111] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 105, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 106, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 106) -2016-05-24 22:18:07 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 105) -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:07 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 105). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 106). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 105) in 5 ms on localhost (1/2) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 106) in 5 ms on localhost (2/2) -2016-05-24 22:18:07 INFO DAGScheduler:59 - ResultStage 53 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-24 22:18:07 INFO DAGScheduler:59 - Job 53 finished: treeAggregate at GradientDescent.scala:225, took 0.014934 s -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=777065, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=782657, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 105 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Got job 54 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:07 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[113] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=786270, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=798054, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 106 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 54 (MapPartitionsRDD[113] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 2 tasks -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 107) -2016-05-24 22:18:07 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 108) -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:07 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 107). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 108). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 107) in 6 ms on localhost (1/2) -2016-05-24 22:18:07 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 108) in 7 ms on localhost (2/2) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-24 22:18:07 INFO DAGScheduler:59 - Job 54 finished: treeAggregate at GradientDescent.scala:225, took 0.019083 s -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=801538, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=807130, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 107 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Got job 55 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:07 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[115] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=810762, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=822546, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 108 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 55 (MapPartitionsRDD[115] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 2 tasks -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 109, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 110, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 110) -2016-05-24 22:18:07 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 109) -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:07 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 110). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 109). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 110) in 6 ms on localhost (1/2) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 109) in 7 ms on localhost (2/2) -2016-05-24 22:18:07 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-24 22:18:07 INFO DAGScheduler:59 - Job 55 finished: treeAggregate at GradientDescent.scala:225, took 0.048658 s -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=826028, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=831620, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 109 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Got job 56 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:07 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[117] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=835233, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=847017, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 110 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 56 (MapPartitionsRDD[117] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 2 tasks -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 111, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 112, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 111) -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:07 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 112) -2016-05-24 22:18:07 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 111). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:07 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 112). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 111) in 21 ms on localhost (1/2) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 112) in 21 ms on localhost (2/2) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-24 22:18:07 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 22:18:07 INFO DAGScheduler:59 - Job 56 finished: treeAggregate at GradientDescent.scala:225, took 0.077905 s -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=850500, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=856092, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 111 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Got job 57 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:07 INFO DAGScheduler:59 - Final stage: ResultStage 57(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[119] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=859705, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3479) called with curMem=871489, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 112 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 57 (MapPartitionsRDD[119] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 2 tasks -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 113, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 114, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 114) -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:07 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 114). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 114) in 7 ms on localhost (1/2) -2016-05-24 22:18:07 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 113) -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:07 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 113). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 113) in 18 ms on localhost (2/2) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-24 22:18:07 INFO DAGScheduler:59 - ResultStage 57 (treeAggregate at GradientDescent.scala:225) finished in 0.047 s -2016-05-24 22:18:07 INFO DAGScheduler:59 - Job 57 finished: treeAggregate at GradientDescent.scala:225, took 0.180286 s -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=874968, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=880560, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 113 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Got job 58 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:07 INFO DAGScheduler:59 - Final stage: ResultStage 58(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting ResultStage 58 (MapPartitionsRDD[121] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=884172, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=895956, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 114 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 58 (MapPartitionsRDD[121] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 2 tasks -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 115, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 116, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 115) -2016-05-24 22:18:07 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 116) -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:07 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 115). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 116). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 115) in 5 ms on localhost (1/2) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 116) in 6 ms on localhost (2/2) -2016-05-24 22:18:07 INFO DAGScheduler:59 - ResultStage 58 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-05-24 22:18:07 INFO DAGScheduler:59 - Job 58 finished: treeAggregate at GradientDescent.scala:225, took 0.051356 s -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=899438, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_115 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3620) called with curMem=905030, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_115_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_115_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 115 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Got job 59 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:07 INFO DAGScheduler:59 - Final stage: ResultStage 59(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting ResultStage 59 (MapPartitionsRDD[123] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=908650, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_116 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3478) called with curMem=920434, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_116_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_116_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 116 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 59 (MapPartitionsRDD[123] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 2 tasks -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 117, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 118, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 117) -2016-05-24 22:18:07 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 118) -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:07 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 117). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 118). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 117) in 6 ms on localhost (1/2) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 118) in 7 ms on localhost (2/2) -2016-05-24 22:18:07 INFO DAGScheduler:59 - ResultStage 59 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-05-24 22:18:07 INFO DAGScheduler:59 - Job 59 finished: treeAggregate at GradientDescent.scala:225, took 0.018813 s -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=923912, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_117 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=929504, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_117_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_117_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 117 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Got job 60 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:07 INFO DAGScheduler:59 - Final stage: ResultStage 60(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting ResultStage 60 (MapPartitionsRDD[125] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=933116, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_118 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=944900, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_118_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_118_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 118 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 60 (MapPartitionsRDD[125] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Adding task set 60.0 with 2 tasks -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 60.0 (TID 119, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 60.0 (TID 120, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO Executor:59 - Running task 1.0 in stage 60.0 (TID 120) -2016-05-24 22:18:07 INFO Executor:59 - Running task 0.0 in stage 60.0 (TID 119) -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:07 INFO Executor:59 - Finished task 0.0 in stage 60.0 (TID 119). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO Executor:59 - Finished task 1.0 in stage 60.0 (TID 120). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 60.0 (TID 119) in 14 ms on localhost (1/2) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 60.0 (TID 120) in 13 ms on localhost (2/2) -2016-05-24 22:18:07 INFO DAGScheduler:59 - ResultStage 60 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 60.0, whose tasks have all completed, from pool -2016-05-24 22:18:07 INFO DAGScheduler:59 - Job 60 finished: treeAggregate at GradientDescent.scala:225, took 0.024768 s -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=948384, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_119 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3616) called with curMem=953976, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_119_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_119_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 119 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Got job 61 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:07 INFO DAGScheduler:59 - Final stage: ResultStage 61(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting ResultStage 61 (MapPartitionsRDD[127] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=957592, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_120 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=969376, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_120_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_120_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 120 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 61 (MapPartitionsRDD[127] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 2 tasks -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 121, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 122, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 121) -2016-05-24 22:18:07 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 122) -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:07 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 121). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 121) in 6 ms on localhost (1/2) -2016-05-24 22:18:07 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 122). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 122) in 6 ms on localhost (2/2) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-05-24 22:18:07 INFO DAGScheduler:59 - ResultStage 61 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:18:07 INFO DAGScheduler:59 - Job 61 finished: treeAggregate at GradientDescent.scala:225, took 0.021246 s -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=972858, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_121 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=978450, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_121_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_121_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 121 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Got job 62 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:07 INFO DAGScheduler:59 - Final stage: ResultStage 62(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[129] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=982062, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_122 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=993846, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_122_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_122_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 122 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 62 (MapPartitionsRDD[129] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 2 tasks -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 123, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 62.0 (TID 124, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO Executor:59 - Running task 1.0 in stage 62.0 (TID 124) -2016-05-24 22:18:07 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 123) -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:07 INFO Executor:59 - Finished task 1.0 in stage 62.0 (TID 124). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 123). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 62.0 (TID 124) in 6 ms on localhost (1/2) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 123) in 6 ms on localhost (2/2) -2016-05-24 22:18:07 INFO DAGScheduler:59 - ResultStage 62 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-05-24 22:18:07 INFO DAGScheduler:59 - Job 62 finished: treeAggregate at GradientDescent.scala:225, took 0.018408 s -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=997329, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_123 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3630) called with curMem=1002921, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_123_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_123_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 123 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:07 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Got job 63 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:07 INFO DAGScheduler:59 - Final stage: ResultStage 63(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting ResultStage 63 (MapPartitionsRDD[131] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1006551, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_124 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=1018335, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_124_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:18:07 INFO BlockManagerInfo:59 - Added broadcast_124_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:07 INFO SparkContext:59 - Created broadcast 124 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 63 (MapPartitionsRDD[131] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Adding task set 63.0 with 2 tasks -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 63.0 (TID 125, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 63.0 (TID 126, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:07 INFO Executor:59 - Running task 1.0 in stage 63.0 (TID 126) -2016-05-24 22:18:07 INFO Executor:59 - Running task 0.0 in stage 63.0 (TID 125) -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:07 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:07 INFO Executor:59 - Finished task 0.0 in stage 63.0 (TID 125). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO Executor:59 - Finished task 1.0 in stage 63.0 (TID 126). 7968 bytes result sent to driver -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 63.0 (TID 125) in 5 ms on localhost (1/2) -2016-05-24 22:18:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 63.0 (TID 126) in 5 ms on localhost (2/2) -2016-05-24 22:18:07 INFO DAGScheduler:59 - ResultStage 63 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:18:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 63.0, whose tasks have all completed, from pool -2016-05-24 22:18:07 INFO DAGScheduler:59 - Job 63 finished: treeAggregate at GradientDescent.scala:225, took 0.019942 s -2016-05-24 22:18:07 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1021816, maxMem=766075207 -2016-05-24 22:18:07 INFO MemoryStore:59 - Block broadcast_125 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1027408, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_125_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_125_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 125 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Got job 64 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:08 INFO DAGScheduler:59 - Final stage: ResultStage 64(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting ResultStage 64 (MapPartitionsRDD[133] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1031021, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_126 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1042805, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_126_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_126_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 126 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 64 (MapPartitionsRDD[133] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Adding task set 64.0 with 2 tasks -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 64.0 (TID 127, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 64.0 (TID 128, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO Executor:59 - Running task 0.0 in stage 64.0 (TID 127) -2016-05-24 22:18:08 INFO Executor:59 - Running task 1.0 in stage 64.0 (TID 128) -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:08 INFO Executor:59 - Finished task 0.0 in stage 64.0 (TID 127). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO Executor:59 - Finished task 1.0 in stage 64.0 (TID 128). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 64.0 (TID 127) in 5 ms on localhost (1/2) -2016-05-24 22:18:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 64.0 (TID 128) in 5 ms on localhost (2/2) -2016-05-24 22:18:08 INFO DAGScheduler:59 - ResultStage 64 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 64.0, whose tasks have all completed, from pool -2016-05-24 22:18:08 INFO DAGScheduler:59 - Job 64 finished: treeAggregate at GradientDescent.scala:225, took 0.013709 s -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1046288, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_127 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1051880, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_127_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_127_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 127 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Got job 65 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:08 INFO DAGScheduler:59 - Final stage: ResultStage 65(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[135] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1055492, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_128 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1067276, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_128_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_128_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 128 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 65 (MapPartitionsRDD[135] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 2 tasks -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 129, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 130, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 129) -2016-05-24 22:18:08 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 130) -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:08 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 130). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 129). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 130) in 5 ms on localhost (1/2) -2016-05-24 22:18:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 129) in 6 ms on localhost (2/2) -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-05-24 22:18:08 INFO DAGScheduler:59 - ResultStage 65 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:18:08 INFO DAGScheduler:59 - Job 65 finished: treeAggregate at GradientDescent.scala:225, took 0.015470 s -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1070760, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_129 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1076352, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_129_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_129_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 129 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Got job 66 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:08 INFO DAGScheduler:59 - Final stage: ResultStage 66(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting ResultStage 66 (MapPartitionsRDD[137] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1079965, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_130 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=1091749, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_130_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_130_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 130 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 66 (MapPartitionsRDD[137] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Adding task set 66.0 with 2 tasks -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 66.0 (TID 131, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 66.0 (TID 132, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO Executor:59 - Running task 0.0 in stage 66.0 (TID 131) -2016-05-24 22:18:08 INFO Executor:59 - Running task 1.0 in stage 66.0 (TID 132) -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:08 INFO Executor:59 - Finished task 0.0 in stage 66.0 (TID 131). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 66.0 (TID 131) in 6 ms on localhost (1/2) -2016-05-24 22:18:08 INFO Executor:59 - Finished task 1.0 in stage 66.0 (TID 132). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 66.0 (TID 132) in 6 ms on localhost (2/2) -2016-05-24 22:18:08 INFO DAGScheduler:59 - ResultStage 66 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 66.0, whose tasks have all completed, from pool -2016-05-24 22:18:08 INFO DAGScheduler:59 - Job 66 finished: treeAggregate at GradientDescent.scala:225, took 0.016681 s -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1095231, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_131 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3633) called with curMem=1100823, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_131_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_131_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 131 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Got job 67 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:08 INFO DAGScheduler:59 - Final stage: ResultStage 67(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting ResultStage 67 (MapPartitionsRDD[139] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1104456, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_132 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1116240, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_132_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_132_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 132 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 67 (MapPartitionsRDD[139] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Adding task set 67.0 with 2 tasks -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 67.0 (TID 133, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 67.0 (TID 134, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO Executor:59 - Running task 0.0 in stage 67.0 (TID 133) -2016-05-24 22:18:08 INFO Executor:59 - Running task 1.0 in stage 67.0 (TID 134) -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:08 INFO Executor:59 - Finished task 1.0 in stage 67.0 (TID 134). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO Executor:59 - Finished task 0.0 in stage 67.0 (TID 133). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 67.0 (TID 134) in 8 ms on localhost (1/2) -2016-05-24 22:18:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 67.0 (TID 133) in 10 ms on localhost (2/2) -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 67.0, whose tasks have all completed, from pool -2016-05-24 22:18:08 INFO DAGScheduler:59 - ResultStage 67 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:18:08 INFO DAGScheduler:59 - Job 67 finished: treeAggregate at GradientDescent.scala:225, took 0.076809 s -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1119723, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_133 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1125315, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_133_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_133_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 133 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Got job 68 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:08 INFO DAGScheduler:59 - Final stage: ResultStage 68(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting ResultStage 68 (MapPartitionsRDD[141] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1128928, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_134 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1140712, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_134_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_134_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 134 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 68 (MapPartitionsRDD[141] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Adding task set 68.0 with 2 tasks -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 68.0 (TID 135, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 68.0 (TID 136, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO Executor:59 - Running task 0.0 in stage 68.0 (TID 135) -2016-05-24 22:18:08 INFO Executor:59 - Running task 1.0 in stage 68.0 (TID 136) -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:08 INFO Executor:59 - Finished task 1.0 in stage 68.0 (TID 136). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO Executor:59 - Finished task 0.0 in stage 68.0 (TID 135). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 68.0 (TID 136) in 7 ms on localhost (1/2) -2016-05-24 22:18:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 68.0 (TID 135) in 8 ms on localhost (2/2) -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 68.0, whose tasks have all completed, from pool -2016-05-24 22:18:08 INFO DAGScheduler:59 - ResultStage 68 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:18:08 INFO DAGScheduler:59 - Job 68 finished: treeAggregate at GradientDescent.scala:225, took 0.023359 s -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1144195, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_135 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3608) called with curMem=1149787, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_135_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_135_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 135 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Got job 69 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:08 INFO DAGScheduler:59 - Final stage: ResultStage 69(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting ResultStage 69 (MapPartitionsRDD[143] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1153395, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_136 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3485) called with curMem=1165179, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_136_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_136_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 136 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 69 (MapPartitionsRDD[143] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Adding task set 69.0 with 2 tasks -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 69.0 (TID 137, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 69.0 (TID 138, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO Executor:59 - Running task 0.0 in stage 69.0 (TID 137) -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:08 INFO Executor:59 - Finished task 0.0 in stage 69.0 (TID 137). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO Executor:59 - Running task 1.0 in stage 69.0 (TID 138) -2016-05-24 22:18:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 69.0 (TID 137) in 14 ms on localhost (1/2) -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:08 INFO Executor:59 - Finished task 1.0 in stage 69.0 (TID 138). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 69.0 (TID 138) in 20 ms on localhost (2/2) -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 69.0, whose tasks have all completed, from pool -2016-05-24 22:18:08 INFO DAGScheduler:59 - ResultStage 69 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 22:18:08 INFO DAGScheduler:59 - Job 69 finished: treeAggregate at GradientDescent.scala:225, took 0.229424 s -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1168664, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_137 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1174256, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_137_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_137_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 137 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Got job 70 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:08 INFO DAGScheduler:59 - Final stage: ResultStage 70(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting ResultStage 70 (MapPartitionsRDD[145] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1177862, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_138 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=1189646, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_138_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_138_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 138 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 70 (MapPartitionsRDD[145] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Adding task set 70.0 with 2 tasks -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 70.0 (TID 139, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 70.0 (TID 140, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO Executor:59 - Running task 0.0 in stage 70.0 (TID 139) -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:08 INFO Executor:59 - Finished task 0.0 in stage 70.0 (TID 139). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 70.0 (TID 139) in 6 ms on localhost (1/2) -2016-05-24 22:18:08 INFO Executor:59 - Running task 1.0 in stage 70.0 (TID 140) -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:08 INFO Executor:59 - Finished task 1.0 in stage 70.0 (TID 140). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 70.0 (TID 140) in 23 ms on localhost (2/2) -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 70.0, whose tasks have all completed, from pool -2016-05-24 22:18:08 INFO DAGScheduler:59 - ResultStage 70 (treeAggregate at GradientDescent.scala:225) finished in 0.095 s -2016-05-24 22:18:08 INFO DAGScheduler:59 - Job 70 finished: treeAggregate at GradientDescent.scala:225, took 0.203017 s -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1193128, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_139 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3619) called with curMem=1198720, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_139_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_139_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 139 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Got job 71 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:08 INFO DAGScheduler:59 - Final stage: ResultStage 71(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting ResultStage 71 (MapPartitionsRDD[147] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1202339, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_140 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1214123, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_140_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_140_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 140 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 71 (MapPartitionsRDD[147] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Adding task set 71.0 with 2 tasks -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 71.0 (TID 141, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 71.0 (TID 142, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO Executor:59 - Running task 0.0 in stage 71.0 (TID 141) -2016-05-24 22:18:08 INFO Executor:59 - Running task 1.0 in stage 71.0 (TID 142) -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:08 INFO Executor:59 - Finished task 0.0 in stage 71.0 (TID 141). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO Executor:59 - Finished task 1.0 in stage 71.0 (TID 142). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 71.0 (TID 141) in 8 ms on localhost (1/2) -2016-05-24 22:18:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 71.0 (TID 142) in 8 ms on localhost (2/2) -2016-05-24 22:18:08 INFO DAGScheduler:59 - ResultStage 71 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 71.0, whose tasks have all completed, from pool -2016-05-24 22:18:08 INFO DAGScheduler:59 - Job 71 finished: treeAggregate at GradientDescent.scala:225, took 0.023035 s -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1217606, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_141 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3604) called with curMem=1223198, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_141_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_141_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 141 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Got job 72 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:08 INFO DAGScheduler:59 - Final stage: ResultStage 72(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting ResultStage 72 (MapPartitionsRDD[149] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1226802, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_142 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1238586, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_142_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_142_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 142 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 72 (MapPartitionsRDD[149] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Adding task set 72.0 with 2 tasks -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 72.0 (TID 143, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 72.0 (TID 144, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO Executor:59 - Running task 0.0 in stage 72.0 (TID 143) -2016-05-24 22:18:08 INFO Executor:59 - Running task 1.0 in stage 72.0 (TID 144) -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:08 INFO Executor:59 - Finished task 0.0 in stage 72.0 (TID 143). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 72.0 (TID 143) in 9 ms on localhost (1/2) -2016-05-24 22:18:08 INFO Executor:59 - Finished task 1.0 in stage 72.0 (TID 144). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 72.0 (TID 144) in 10 ms on localhost (2/2) -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 72.0, whose tasks have all completed, from pool -2016-05-24 22:18:08 INFO DAGScheduler:59 - ResultStage 72 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:18:08 INFO DAGScheduler:59 - Job 72 finished: treeAggregate at GradientDescent.scala:225, took 0.027967 s -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1242069, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_143 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=1247661, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_143_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_143_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 143 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Got job 73 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:08 INFO DAGScheduler:59 - Final stage: ResultStage 73(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting ResultStage 73 (MapPartitionsRDD[151] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1251270, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_144 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=1263054, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_144_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_144_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 144 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 73 (MapPartitionsRDD[151] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Adding task set 73.0 with 2 tasks -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 73.0 (TID 145, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 73.0 (TID 146, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO Executor:59 - Running task 1.0 in stage 73.0 (TID 146) -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:08 INFO Executor:59 - Running task 0.0 in stage 73.0 (TID 145) -2016-05-24 22:18:08 INFO Executor:59 - Finished task 1.0 in stage 73.0 (TID 146). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 73.0 (TID 146) in 7 ms on localhost (1/2) -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:08 INFO Executor:59 - Finished task 0.0 in stage 73.0 (TID 145). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 73.0 (TID 145) in 10 ms on localhost (2/2) -2016-05-24 22:18:08 INFO DAGScheduler:59 - ResultStage 73 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 73.0, whose tasks have all completed, from pool -2016-05-24 22:18:08 INFO DAGScheduler:59 - Job 73 finished: treeAggregate at GradientDescent.scala:225, took 0.063644 s -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1266536, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_145 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1272128, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_145_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_145_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 145 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:08 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Got job 74 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:08 INFO DAGScheduler:59 - Final stage: ResultStage 74(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting ResultStage 74 (MapPartitionsRDD[153] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1275734, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_146 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:18:08 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=1287518, maxMem=766075207 -2016-05-24 22:18:08 INFO MemoryStore:59 - Block broadcast_146_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:18:08 INFO BlockManagerInfo:59 - Added broadcast_146_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:08 INFO SparkContext:59 - Created broadcast 146 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 74 (MapPartitionsRDD[153] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:08 INFO TaskSchedulerImpl:59 - Adding task set 74.0 with 2 tasks -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 74.0 (TID 147, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 74.0 (TID 148, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:08 INFO Executor:59 - Running task 1.0 in stage 74.0 (TID 148) -2016-05-24 22:18:08 INFO Executor:59 - Running task 0.0 in stage 74.0 (TID 147) -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:08 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:08 INFO Executor:59 - Finished task 0.0 in stage 74.0 (TID 147). 7968 bytes result sent to driver -2016-05-24 22:18:08 INFO Executor:59 - Finished task 1.0 in stage 74.0 (TID 148). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 74.0 (TID 147) in 35 ms on localhost (1/2) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 74.0 (TID 148) in 35 ms on localhost (2/2) -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 74.0, whose tasks have all completed, from pool -2016-05-24 22:18:09 INFO DAGScheduler:59 - ResultStage 74 (treeAggregate at GradientDescent.scala:225) finished in 0.036 s -2016-05-24 22:18:09 INFO DAGScheduler:59 - Job 74 finished: treeAggregate at GradientDescent.scala:225, took 0.085264 s -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1291000, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_147 stored as values in memory (estimated size 5.5 KB, free 729.3 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3624) called with curMem=1296592, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_147_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.3 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_147_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 147 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Got job 75 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:09 INFO DAGScheduler:59 - Final stage: ResultStage 75(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting ResultStage 75 (MapPartitionsRDD[155] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1300216, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_148 stored as values in memory (estimated size 11.5 KB, free 729.3 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=1312000, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_148_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.3 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_148_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 148 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 75 (MapPartitionsRDD[155] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Adding task set 75.0 with 2 tasks -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 75.0 (TID 149, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 75.0 (TID 150, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO Executor:59 - Running task 1.0 in stage 75.0 (TID 150) -2016-05-24 22:18:09 INFO Executor:59 - Running task 0.0 in stage 75.0 (TID 149) -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:09 INFO Executor:59 - Finished task 1.0 in stage 75.0 (TID 150). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO Executor:59 - Finished task 0.0 in stage 75.0 (TID 149). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 75.0 (TID 150) in 6 ms on localhost (1/2) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 75.0 (TID 149) in 7 ms on localhost (2/2) -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 75.0, whose tasks have all completed, from pool -2016-05-24 22:18:09 INFO DAGScheduler:59 - ResultStage 75 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:18:09 INFO DAGScheduler:59 - Job 75 finished: treeAggregate at GradientDescent.scala:225, took 0.020870 s -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1315482, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_149 stored as values in memory (estimated size 5.5 KB, free 729.3 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3626) called with curMem=1321074, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_149_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.3 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_149_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 149 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_148_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 76 -2016-05-24 22:18:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_147_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO DAGScheduler:59 - Got job 76 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:09 INFO DAGScheduler:59 - Final stage: ResultStage 76(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting ResultStage 76 (MapPartitionsRDD[157] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1300218, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_150 stored as values in memory (estimated size 11.5 KB, free 729.3 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1308520, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_150_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.3 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_150_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 150 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_146_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 76 (MapPartitionsRDD[157] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 2 tasks -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 75 -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 151, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 152, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 151) -2016-05-24 22:18:09 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 152) -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:09 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 151). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_145_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 152). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 151) in 7 ms on localhost (1/2) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 152) in 8 ms on localhost (2/2) -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-05-24 22:18:09 INFO DAGScheduler:59 - ResultStage 76 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:18:09 INFO DAGScheduler:59 - Job 76 finished: treeAggregate at GradientDescent.scala:225, took 0.019255 s -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1275755, maxMem=766075207 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_144_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_151 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 74 -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3611) called with curMem=1277738, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_151_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_143_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_151_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 151 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_142_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 73 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Got job 77 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:09 INFO DAGScheduler:59 - Final stage: ResultStage 77(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_141_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[159] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1251294, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_152 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1263078, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_152_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_152_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 152 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 77 (MapPartitionsRDD[159] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 2 tasks -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 153, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 154, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 154) -2016-05-24 22:18:09 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 153) -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:09 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 154). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:09 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 153). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 154) in 12 ms on localhost (1/2) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 153) in 14 ms on localhost (2/2) -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-05-24 22:18:09 INFO DAGScheduler:59 - ResultStage 77 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:18:09 INFO DAGScheduler:59 - Job 77 finished: treeAggregate at GradientDescent.scala:225, took 0.066626 s -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_140_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1263078, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_153 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1256886, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_153_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_153_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 72 -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 153 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_139_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_138_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Got job 78 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 71 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Final stage: ResultStage 78(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting ResultStage 78 (MapPartitionsRDD[161] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1232415, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_154 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1244199, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_154_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_137_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_154_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_136_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 154 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 78 (MapPartitionsRDD[161] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 2 tasks -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 155, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 156, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 156) -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:09 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 156). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 155) -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:09 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 155). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 70 -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 156) in 42 ms on localhost (1/2) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 155) in 43 ms on localhost (2/2) -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-05-24 22:18:09 INFO DAGScheduler:59 - ResultStage 78 (treeAggregate at GradientDescent.scala:225) finished in 0.048 s -2016-05-24 22:18:09 INFO DAGScheduler:59 - Job 78 finished: treeAggregate at GradientDescent.scala:225, took 0.073431 s -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1217621, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_155 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1223213, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_155_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_135_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_155_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 155 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Got job 79 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:09 INFO DAGScheduler:59 - Final stage: ResultStage 79(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting ResultStage 79 (MapPartitionsRDD[163] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1211558, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_156 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_134_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1223342, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_156_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_156_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 69 -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 156 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 79 (MapPartitionsRDD[163] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Adding task set 79.0 with 2 tasks -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 79.0 (TID 157, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 79.0 (TID 158, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_133_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO Executor:59 - Running task 0.0 in stage 79.0 (TID 157) -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:09 INFO Executor:59 - Finished task 0.0 in stage 79.0 (TID 157). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO Executor:59 - Running task 1.0 in stage 79.0 (TID 158) -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:09 INFO Executor:59 - Finished task 1.0 in stage 79.0 (TID 158). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 79.0 (TID 157) in 37 ms on localhost (1/2) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 79.0 (TID 158) in 37 ms on localhost (2/2) -2016-05-24 22:18:09 INFO DAGScheduler:59 - ResultStage 79 (treeAggregate at GradientDescent.scala:225) finished in 0.038 s -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 79.0, whose tasks have all completed, from pool -2016-05-24 22:18:09 INFO DAGScheduler:59 - Job 79 finished: treeAggregate at GradientDescent.scala:225, took 0.357304 s -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1217620, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_157 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1223212, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_157_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_157_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 157 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Got job 80 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:09 INFO DAGScheduler:59 - Final stage: ResultStage 80(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting ResultStage 80 (MapPartitionsRDD[165] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1226824, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_158 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1238608, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_158_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_158_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 158 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 80 (MapPartitionsRDD[165] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Adding task set 80.0 with 2 tasks -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_132_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 80.0 (TID 159, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 80.0 (TID 160, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-05-24 22:18:09 INFO Executor:59 - Running task 0.0 in stage 80.0 (TID 159) -2016-05-24 22:18:09 INFO Executor:59 - Running task 1.0 in stage 80.0 (TID 160) -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_131_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO Executor:59 - Finished task 0.0 in stage 80.0 (TID 159). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO Executor:59 - Finished task 1.0 in stage 80.0 (TID 160). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 80.0 (TID 159) in 6 ms on localhost (1/2) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 80.0 (TID 160) in 6 ms on localhost (2/2) -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 80.0, whose tasks have all completed, from pool -2016-05-24 22:18:09 INFO DAGScheduler:59 - ResultStage 80 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:18:09 INFO DAGScheduler:59 - Job 80 finished: treeAggregate at GradientDescent.scala:225, took 0.021669 s -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1205816, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_159 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1207926, maxMem=766075207 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_130_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_159_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_159_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 159 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_129_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_128_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-05-24 22:18:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Got job 81 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:09 INFO DAGScheduler:59 - Final stage: ResultStage 81(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting ResultStage 81 (MapPartitionsRDD[167] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_127_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1177855, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_160 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1186156, maxMem=766075207 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_126_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_160_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_160_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 160 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 81 (MapPartitionsRDD[167] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Adding task set 81.0 with 2 tasks -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 81.0 (TID 161, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 81.0 (TID 162, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_125_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO Executor:59 - Running task 0.0 in stage 81.0 (TID 161) -2016-05-24 22:18:09 INFO Executor:59 - Running task 1.0 in stage 81.0 (TID 162) -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_124_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO Executor:59 - Finished task 0.0 in stage 81.0 (TID 161). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO Executor:59 - Finished task 1.0 in stage 81.0 (TID 162). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 81.0 (TID 161) in 7 ms on localhost (1/2) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 81.0 (TID 162) in 6 ms on localhost (2/2) -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 81.0, whose tasks have all completed, from pool -2016-05-24 22:18:09 INFO DAGScheduler:59 - ResultStage 81 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:18:09 INFO DAGScheduler:59 - Job 81 finished: treeAggregate at GradientDescent.scala:225, took 0.017419 s -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_123_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1149756, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_161 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1149756, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_161_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_161_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 161 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_122_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_121_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_120_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-05-24 22:18:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Got job 82 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:09 INFO DAGScheduler:59 - Final stage: ResultStage 82(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_119_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting ResultStage 82 (MapPartitionsRDD[169] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1100939, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_162 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_118_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1100939, maxMem=766075207 -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_162_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_162_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 162 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 82 (MapPartitionsRDD[169] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_117_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Adding task set 82.0 with 2 tasks -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 82.0 (TID 163, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 82.0 (TID 164, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO Executor:59 - Running task 1.0 in stage 82.0 (TID 164) -2016-05-24 22:18:09 INFO Executor:59 - Running task 0.0 in stage 82.0 (TID 163) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_116_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:09 INFO Executor:59 - Finished task 0.0 in stage 82.0 (TID 163). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-05-24 22:18:09 INFO Executor:59 - Finished task 1.0 in stage 82.0 (TID 164). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 82.0 (TID 164) in 7 ms on localhost (1/2) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 82.0 (TID 163) in 8 ms on localhost (2/2) -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 82.0, whose tasks have all completed, from pool -2016-05-24 22:18:09 INFO DAGScheduler:59 - ResultStage 82 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:18:09 INFO DAGScheduler:59 - Job 82 finished: treeAggregate at GradientDescent.scala:225, took 0.019482 s -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_115_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1070744, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_163 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=1061070, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_163_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_114_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_163_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 163 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_113_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Got job 83 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:09 INFO DAGScheduler:59 - Final stage: ResultStage 83(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_112_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting ResultStage 83 (MapPartitionsRDD[171] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1034625, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_164 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_111_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1042796, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_164_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_164_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 164 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 83 (MapPartitionsRDD[171] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Adding task set 83.0 with 2 tasks -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_110_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 83.0 (TID 165, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 83.0 (TID 166, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO Executor:59 - Running task 1.0 in stage 83.0 (TID 166) -2016-05-24 22:18:09 INFO Executor:59 - Running task 0.0 in stage 83.0 (TID 165) -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:09 INFO Executor:59 - Finished task 1.0 in stage 83.0 (TID 166). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO Executor:59 - Finished task 0.0 in stage 83.0 (TID 165). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 83.0 (TID 166) in 7 ms on localhost (1/2) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 83.0 (TID 165) in 8 ms on localhost (2/2) -2016-05-24 22:18:09 INFO DAGScheduler:59 - ResultStage 83 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 83.0, whose tasks have all completed, from pool -2016-05-24 22:18:09 INFO DAGScheduler:59 - Job 83 finished: treeAggregate at GradientDescent.scala:225, took 0.053985 s -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1031012, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_165 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_109_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1012133, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_165_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_108_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_165_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 165 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_107_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_106_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-05-24 22:18:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_105_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:09 INFO DAGScheduler:59 - Got job 84 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:09 INFO DAGScheduler:59 - Final stage: ResultStage 84(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting ResultStage 84 (MapPartitionsRDD[173] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_104_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=978567, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_166 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=978567, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_166_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_166_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 166 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 84 (MapPartitionsRDD[173] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Adding task set 84.0 with 2 tasks -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_103_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 84.0 (TID 167, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 84.0 (TID 168, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO Executor:59 - Running task 0.0 in stage 84.0 (TID 167) -2016-05-24 22:18:09 INFO Executor:59 - Running task 1.0 in stage 84.0 (TID 168) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_102_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-05-24 22:18:09 INFO Executor:59 - Finished task 1.0 in stage 84.0 (TID 168). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO Executor:59 - Finished task 0.0 in stage 84.0 (TID 167). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 84.0 (TID 168) in 5 ms on localhost (1/2) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 84.0 (TID 167) in 7 ms on localhost (2/2) -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 84.0, whose tasks have all completed, from pool -2016-05-24 22:18:09 INFO DAGScheduler:59 - ResultStage 84 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:18:09 INFO DAGScheduler:59 - Job 84 finished: treeAggregate at GradientDescent.scala:225, took 0.018198 s -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_101_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=948341, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_167 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3620) called with curMem=953933, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_167_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_167_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 167 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_100_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_99_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Got job 85 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:09 INFO DAGScheduler:59 - Final stage: ResultStage 85(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_98_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting ResultStage 85 (MapPartitionsRDD[175] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=917817, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_168 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=929601, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_168_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_168_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 168 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 85 (MapPartitionsRDD[175] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Adding task set 85.0 with 2 tasks -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 85.0 (TID 169, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 85.0 (TID 170, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:09 INFO Executor:59 - Running task 1.0 in stage 85.0 (TID 170) -2016-05-24 22:18:09 INFO Executor:59 - Running task 0.0 in stage 85.0 (TID 169) -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:09 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-05-24 22:18:09 INFO Executor:59 - Finished task 0.0 in stage 85.0 (TID 169). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO Executor:59 - Finished task 1.0 in stage 85.0 (TID 170). 7968 bytes result sent to driver -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 85.0 (TID 169) in 7 ms on localhost (1/2) -2016-05-24 22:18:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 85.0 (TID 170) in 7 ms on localhost (2/2) -2016-05-24 22:18:09 INFO DAGScheduler:59 - ResultStage 85 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:18:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 85.0, whose tasks have all completed, from pool -2016-05-24 22:18:09 INFO DAGScheduler:59 - Job 85 finished: treeAggregate at GradientDescent.scala:225, took 0.024158 s -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_97_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=923870, maxMem=766075207 -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_169 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - ensureFreeSpace(3621) called with curMem=925979, maxMem=766075207 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_96_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:09 INFO MemoryStore:59 - Block broadcast_169_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Added broadcast_169_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Created broadcast 169 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_95_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_94_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:09 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_93_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:09 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:09 INFO DAGScheduler:59 - Got job 86 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:09 INFO DAGScheduler:59 - Final stage: ResultStage 86(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:09 INFO BlockManagerInfo:59 - Removed broadcast_92_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:09 INFO DAGScheduler:59 - Submitting ResultStage 86 (MapPartitionsRDD[177] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:10 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=868866, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_170 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=877032, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_170_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Removed broadcast_91_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_170_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 170 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 86 (MapPartitionsRDD[177] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Adding task set 86.0 with 2 tasks -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 86.0 (TID 171, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 86.0 (TID 172, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO Executor:59 - Running task 0.0 in stage 86.0 (TID 171) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Removed broadcast_90_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO Executor:59 - Running task 1.0 in stage 86.0 (TID 172) -2016-05-24 22:18:10 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 0.0 in stage 86.0 (TID 171). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO Executor:59 - Finished task 1.0 in stage 86.0 (TID 172). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Removed broadcast_89_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 86.0 (TID 171) in 7 ms on localhost (1/2) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 86.0 (TID 172) in 6 ms on localhost (2/2) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 86.0, whose tasks have all completed, from pool -2016-05-24 22:18:10 INFO DAGScheduler:59 - ResultStage 86 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:18:10 INFO DAGScheduler:59 - Job 86 finished: treeAggregate at GradientDescent.scala:225, took 0.072538 s -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=846965, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_171 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Removed broadcast_88_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=840773, maxMem=766075207 -2016-05-24 22:18:10 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_171_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_171_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 171 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Removed broadcast_87_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Got job 87 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:10 INFO DAGScheduler:59 - Final stage: ResultStage 87(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting ResultStage 87 (MapPartitionsRDD[179] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:10 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=795444, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_172 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=798024, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_172_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_172_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 172 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 87 (MapPartitionsRDD[179] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Adding task set 87.0 with 2 tasks -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 87.0 (TID 173, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 87.0 (TID 174, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO Executor:59 - Running task 0.0 in stage 87.0 (TID 173) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO Executor:59 - Running task 1.0 in stage 87.0 (TID 174) -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:10 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 22:18:10 INFO Executor:59 - Finished task 0.0 in stage 87.0 (TID 173). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO Executor:59 - Finished task 1.0 in stage 87.0 (TID 174). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 87.0 (TID 173) in 7 ms on localhost (1/2) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 87.0 (TID 174) in 6 ms on localhost (2/2) -2016-05-24 22:18:10 INFO DAGScheduler:59 - ResultStage 87 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 87.0, whose tasks have all completed, from pool -2016-05-24 22:18:10 INFO DAGScheduler:59 - Job 87 finished: treeAggregate at GradientDescent.scala:225, took 0.017874 s -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=777036, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_173 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=767360, maxMem=766075207 -2016-05-24 22:18:10 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_173_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_173_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 173 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Got job 88 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:10 INFO DAGScheduler:59 - Final stage: ResultStage 88(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting ResultStage 88 (MapPartitionsRDD[181] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=761767, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_174 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3485) called with curMem=773551, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_174_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_174_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 174 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 88 (MapPartitionsRDD[181] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Adding task set 88.0 with 2 tasks -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 88.0 (TID 175, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 88.0 (TID 176, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO Executor:59 - Running task 0.0 in stage 88.0 (TID 175) -2016-05-24 22:18:10 INFO Executor:59 - Running task 1.0 in stage 88.0 (TID 176) -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 1.0 in stage 88.0 (TID 176). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO Executor:59 - Finished task 0.0 in stage 88.0 (TID 175). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 88.0 (TID 176) in 5 ms on localhost (1/2) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 88.0 (TID 175) in 6 ms on localhost (2/2) -2016-05-24 22:18:10 INFO DAGScheduler:59 - ResultStage 88 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 88.0, whose tasks have all completed, from pool -2016-05-24 22:18:10 INFO DAGScheduler:59 - Job 88 finished: treeAggregate at GradientDescent.scala:225, took 0.017134 s -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=777036, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_175 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3615) called with curMem=782628, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_175_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_175_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 175 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Got job 89 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:10 INFO DAGScheduler:59 - Final stage: ResultStage 89(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting ResultStage 89 (MapPartitionsRDD[183] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=786243, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_176 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=798027, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_176_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_176_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 176 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 89 (MapPartitionsRDD[183] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Adding task set 89.0 with 2 tasks -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 89.0 (TID 177, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 89.0 (TID 178, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO Executor:59 - Running task 1.0 in stage 89.0 (TID 178) -2016-05-24 22:18:10 INFO Executor:59 - Running task 0.0 in stage 89.0 (TID 177) -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 1.0 in stage 89.0 (TID 178). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO Executor:59 - Finished task 0.0 in stage 89.0 (TID 177). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 89.0 (TID 178) in 4 ms on localhost (1/2) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 89.0 (TID 177) in 5 ms on localhost (2/2) -2016-05-24 22:18:10 INFO DAGScheduler:59 - ResultStage 89 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 89.0, whose tasks have all completed, from pool -2016-05-24 22:18:10 INFO DAGScheduler:59 - Job 89 finished: treeAggregate at GradientDescent.scala:225, took 0.012915 s -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=801510, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_177 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=807102, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_177_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_177_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 177 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Got job 90 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:10 INFO DAGScheduler:59 - Final stage: ResultStage 90(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting ResultStage 90 (MapPartitionsRDD[185] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=810714, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_178 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=822498, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_178_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_178_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 178 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 90 (MapPartitionsRDD[185] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Adding task set 90.0 with 2 tasks -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 90.0 (TID 179, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 90.0 (TID 180, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO Executor:59 - Running task 1.0 in stage 90.0 (TID 180) -2016-05-24 22:18:10 INFO Executor:59 - Running task 0.0 in stage 90.0 (TID 179) -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 1.0 in stage 90.0 (TID 180). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO Executor:59 - Finished task 0.0 in stage 90.0 (TID 179). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 90.0 (TID 180) in 4 ms on localhost (1/2) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 90.0 (TID 179) in 4 ms on localhost (2/2) -2016-05-24 22:18:10 INFO DAGScheduler:59 - ResultStage 90 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 90.0, whose tasks have all completed, from pool -2016-05-24 22:18:10 INFO DAGScheduler:59 - Job 90 finished: treeAggregate at GradientDescent.scala:225, took 0.011744 s -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=825982, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_179 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=831574, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_179_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_179_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 179 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Got job 91 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:10 INFO DAGScheduler:59 - Final stage: ResultStage 91(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting ResultStage 91 (MapPartitionsRDD[187] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=835187, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_180 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=846971, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_180_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_180_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 180 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 91 (MapPartitionsRDD[187] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Adding task set 91.0 with 2 tasks -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 91.0 (TID 181, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 91.0 (TID 182, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO Executor:59 - Running task 1.0 in stage 91.0 (TID 182) -2016-05-24 22:18:10 INFO Executor:59 - Running task 0.0 in stage 91.0 (TID 181) -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 0.0 in stage 91.0 (TID 181). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO Executor:59 - Finished task 1.0 in stage 91.0 (TID 182). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 91.0 (TID 182) in 5 ms on localhost (1/2) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 91.0 (TID 181) in 6 ms on localhost (2/2) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 91.0, whose tasks have all completed, from pool -2016-05-24 22:18:10 INFO DAGScheduler:59 - ResultStage 91 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:18:10 INFO DAGScheduler:59 - Job 91 finished: treeAggregate at GradientDescent.scala:225, took 0.013770 s -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=850454, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_181 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3608) called with curMem=856046, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_181_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_181_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 181 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Got job 92 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:10 INFO DAGScheduler:59 - Final stage: ResultStage 92(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting ResultStage 92 (MapPartitionsRDD[189] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=859654, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_182 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=871438, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_182_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_182_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 182 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 92 (MapPartitionsRDD[189] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Adding task set 92.0 with 2 tasks -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 92.0 (TID 183, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 92.0 (TID 184, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO Executor:59 - Running task 0.0 in stage 92.0 (TID 183) -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 0.0 in stage 92.0 (TID 183). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO Executor:59 - Running task 1.0 in stage 92.0 (TID 184) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 92.0 (TID 183) in 9 ms on localhost (1/2) -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 1.0 in stage 92.0 (TID 184). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 92.0 (TID 184) in 12 ms on localhost (2/2) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 92.0, whose tasks have all completed, from pool -2016-05-24 22:18:10 INFO DAGScheduler:59 - ResultStage 92 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:18:10 INFO DAGScheduler:59 - Job 92 finished: treeAggregate at GradientDescent.scala:225, took 0.022840 s -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=874922, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_183 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=880514, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_183_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_183_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 183 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Got job 93 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:10 INFO DAGScheduler:59 - Final stage: ResultStage 93(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting ResultStage 93 (MapPartitionsRDD[191] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=884128, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_184 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=895912, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_184_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_184_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 184 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 93 (MapPartitionsRDD[191] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Adding task set 93.0 with 2 tasks -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 93.0 (TID 185, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 93.0 (TID 186, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO Executor:59 - Running task 0.0 in stage 93.0 (TID 185) -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 0.0 in stage 93.0 (TID 185). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO Executor:59 - Running task 1.0 in stage 93.0 (TID 186) -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 1.0 in stage 93.0 (TID 186). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 93.0 (TID 185) in 11 ms on localhost (1/2) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 93.0 (TID 186) in 11 ms on localhost (2/2) -2016-05-24 22:18:10 INFO DAGScheduler:59 - ResultStage 93 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 93.0, whose tasks have all completed, from pool -2016-05-24 22:18:10 INFO DAGScheduler:59 - Job 93 finished: treeAggregate at GradientDescent.scala:225, took 0.051517 s -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=899396, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_185 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=904988, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_185_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_185_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 185 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Got job 94 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:10 INFO DAGScheduler:59 - Final stage: ResultStage 94(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting ResultStage 94 (MapPartitionsRDD[193] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=908600, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_186 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=920384, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_186_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_186_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 186 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 94 (MapPartitionsRDD[193] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Adding task set 94.0 with 2 tasks -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 94.0 (TID 187, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 94.0 (TID 188, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO Executor:59 - Running task 1.0 in stage 94.0 (TID 188) -2016-05-24 22:18:10 INFO Executor:59 - Running task 0.0 in stage 94.0 (TID 187) -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 0.0 in stage 94.0 (TID 187). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO Executor:59 - Finished task 1.0 in stage 94.0 (TID 188). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 94.0 (TID 187) in 6 ms on localhost (1/2) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 94.0 (TID 188) in 5 ms on localhost (2/2) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 94.0, whose tasks have all completed, from pool -2016-05-24 22:18:10 INFO DAGScheduler:59 - ResultStage 94 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:18:10 INFO DAGScheduler:59 - Job 94 finished: treeAggregate at GradientDescent.scala:225, took 0.018362 s -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=923866, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_187 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=929458, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_187_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_187_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 187 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Got job 95 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:10 INFO DAGScheduler:59 - Final stage: ResultStage 95(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting ResultStage 95 (MapPartitionsRDD[195] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=933071, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_188 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=944855, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_188_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_188_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 188 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 95 (MapPartitionsRDD[195] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Adding task set 95.0 with 2 tasks -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 95.0 (TID 189, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 95.0 (TID 190, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO Executor:59 - Running task 0.0 in stage 95.0 (TID 189) -2016-05-24 22:18:10 INFO Executor:59 - Running task 1.0 in stage 95.0 (TID 190) -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 0.0 in stage 95.0 (TID 189). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO Executor:59 - Finished task 1.0 in stage 95.0 (TID 190). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 95.0 (TID 190) in 7 ms on localhost (1/2) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 95.0 (TID 189) in 9 ms on localhost (2/2) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 95.0, whose tasks have all completed, from pool -2016-05-24 22:18:10 INFO DAGScheduler:59 - ResultStage 95 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:18:10 INFO DAGScheduler:59 - Job 95 finished: treeAggregate at GradientDescent.scala:225, took 0.021275 s -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=948339, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_189 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3634) called with curMem=953931, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_189_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_189_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 189 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Got job 96 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:10 INFO DAGScheduler:59 - Final stage: ResultStage 96(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting ResultStage 96 (MapPartitionsRDD[197] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=957565, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_190 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=969349, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_190_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_190_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 190 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 96 (MapPartitionsRDD[197] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Adding task set 96.0 with 2 tasks -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 96.0 (TID 191, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 96.0 (TID 192, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO Executor:59 - Running task 0.0 in stage 96.0 (TID 191) -2016-05-24 22:18:10 INFO Executor:59 - Running task 1.0 in stage 96.0 (TID 192) -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 0.0 in stage 96.0 (TID 191). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 1.0 in stage 96.0 (TID 192). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 96.0 (TID 192) in 42 ms on localhost (1/2) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 96.0 (TID 191) in 44 ms on localhost (2/2) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 96.0, whose tasks have all completed, from pool -2016-05-24 22:18:10 INFO DAGScheduler:59 - ResultStage 96 (treeAggregate at GradientDescent.scala:225) finished in 0.044 s -2016-05-24 22:18:10 INFO DAGScheduler:59 - Job 96 finished: treeAggregate at GradientDescent.scala:225, took 0.057479 s -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=972832, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_191 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3619) called with curMem=978424, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_191_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_191_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 191 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Got job 97 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:10 INFO DAGScheduler:59 - Final stage: ResultStage 97(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting ResultStage 97 (MapPartitionsRDD[199] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=982043, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_192 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=993827, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_192_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_192_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 192 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 97 (MapPartitionsRDD[199] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Adding task set 97.0 with 2 tasks -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 97.0 (TID 193, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 97.0 (TID 194, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO Executor:59 - Running task 0.0 in stage 97.0 (TID 193) -2016-05-24 22:18:10 INFO Executor:59 - Running task 1.0 in stage 97.0 (TID 194) -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 0.0 in stage 97.0 (TID 193). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO Executor:59 - Finished task 1.0 in stage 97.0 (TID 194). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 97.0 (TID 193) in 8 ms on localhost (1/2) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 97.0 (TID 194) in 7 ms on localhost (2/2) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 97.0, whose tasks have all completed, from pool -2016-05-24 22:18:10 INFO DAGScheduler:59 - ResultStage 97 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:18:10 INFO DAGScheduler:59 - Job 97 finished: treeAggregate at GradientDescent.scala:225, took 0.017498 s -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=997310, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_193 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1002902, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_193_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_193_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 193 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Got job 98 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:10 INFO DAGScheduler:59 - Final stage: ResultStage 98(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting ResultStage 98 (MapPartitionsRDD[201] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1006514, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_194 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1018298, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_194_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_194_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 194 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 98 (MapPartitionsRDD[201] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Adding task set 98.0 with 2 tasks -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 98.0 (TID 195, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 98.0 (TID 196, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO Executor:59 - Running task 0.0 in stage 98.0 (TID 195) -2016-05-24 22:18:10 INFO Executor:59 - Running task 1.0 in stage 98.0 (TID 196) -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 0.0 in stage 98.0 (TID 195). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO Executor:59 - Finished task 1.0 in stage 98.0 (TID 196). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 98.0 (TID 195) in 7 ms on localhost (1/2) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 98.0 (TID 196) in 8 ms on localhost (2/2) -2016-05-24 22:18:10 INFO DAGScheduler:59 - ResultStage 98 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 98.0, whose tasks have all completed, from pool -2016-05-24 22:18:10 INFO DAGScheduler:59 - Job 98 finished: treeAggregate at GradientDescent.scala:225, took 0.017592 s -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1021781, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_195 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=1027373, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_195_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_195_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 195 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Got job 99 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:10 INFO DAGScheduler:59 - Final stage: ResultStage 99(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting ResultStage 99 (MapPartitionsRDD[203] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1030982, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_196 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1042766, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_196_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_196_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 196 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 99 (MapPartitionsRDD[203] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Adding task set 99.0 with 2 tasks -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 99.0 (TID 197, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 99.0 (TID 198, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO Executor:59 - Running task 0.0 in stage 99.0 (TID 197) -2016-05-24 22:18:10 INFO Executor:59 - Running task 1.0 in stage 99.0 (TID 198) -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 0.0 in stage 99.0 (TID 197). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 1.0 in stage 99.0 (TID 198). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 99.0 (TID 197) in 36 ms on localhost (1/2) -2016-05-24 22:18:10 INFO DAGScheduler:59 - ResultStage 99 (treeAggregate at GradientDescent.scala:225) finished in 0.053 s -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 99.0 (TID 198) in 51 ms on localhost (2/2) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 99.0, whose tasks have all completed, from pool -2016-05-24 22:18:10 INFO DAGScheduler:59 - Job 99 finished: treeAggregate at GradientDescent.scala:225, took 0.068021 s -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1046249, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_197 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3616) called with curMem=1051841, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_197_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_197_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 197 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Got job 100 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:10 INFO DAGScheduler:59 - Final stage: ResultStage 100(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting ResultStage 100 (MapPartitionsRDD[205] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1055457, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_198 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3485) called with curMem=1067241, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_198_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_198_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 198 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 100 (MapPartitionsRDD[205] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Adding task set 100.0 with 2 tasks -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 100.0 (TID 199, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 100.0 (TID 200, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO Executor:59 - Running task 0.0 in stage 100.0 (TID 199) -2016-05-24 22:18:10 INFO Executor:59 - Running task 1.0 in stage 100.0 (TID 200) -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 0.0 in stage 100.0 (TID 199). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO Executor:59 - Finished task 1.0 in stage 100.0 (TID 200). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 100.0 (TID 199) in 8 ms on localhost (1/2) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 100.0 (TID 200) in 8 ms on localhost (2/2) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 100.0, whose tasks have all completed, from pool -2016-05-24 22:18:10 INFO DAGScheduler:59 - ResultStage 100 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:18:10 INFO DAGScheduler:59 - Job 100 finished: treeAggregate at GradientDescent.scala:225, took 0.023066 s -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1070726, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_199 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1076318, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_199_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_199_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 199 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Got job 101 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:10 INFO DAGScheduler:59 - Final stage: ResultStage 101(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting ResultStage 101 (MapPartitionsRDD[207] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1079924, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_200 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1091708, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_200_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_200_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 200 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 101 (MapPartitionsRDD[207] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Adding task set 101.0 with 2 tasks -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 101.0 (TID 201, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 101.0 (TID 202, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO Executor:59 - Running task 1.0 in stage 101.0 (TID 202) -2016-05-24 22:18:10 INFO Executor:59 - Running task 0.0 in stage 101.0 (TID 201) -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 0.0 in stage 101.0 (TID 201). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO Executor:59 - Finished task 1.0 in stage 101.0 (TID 202). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 101.0 (TID 201) in 8 ms on localhost (1/2) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 101.0 (TID 202) in 7 ms on localhost (2/2) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 101.0, whose tasks have all completed, from pool -2016-05-24 22:18:10 INFO DAGScheduler:59 - ResultStage 101 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:18:10 INFO DAGScheduler:59 - Job 101 finished: treeAggregate at GradientDescent.scala:225, took 0.019039 s -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1095191, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_201 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1100783, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_201_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_201_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 201 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Got job 102 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:10 INFO DAGScheduler:59 - Final stage: ResultStage 102(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting ResultStage 102 (MapPartitionsRDD[209] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1104396, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_202 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1116180, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_202_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_202_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 202 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 102 (MapPartitionsRDD[209] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Adding task set 102.0 with 2 tasks -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 102.0 (TID 203, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 102.0 (TID 204, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO Executor:59 - Running task 0.0 in stage 102.0 (TID 203) -2016-05-24 22:18:10 INFO Executor:59 - Running task 1.0 in stage 102.0 (TID 204) -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 1.0 in stage 102.0 (TID 204). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO Executor:59 - Finished task 0.0 in stage 102.0 (TID 203). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 102.0 (TID 204) in 7 ms on localhost (1/2) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 102.0 (TID 203) in 9 ms on localhost (2/2) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 102.0, whose tasks have all completed, from pool -2016-05-24 22:18:10 INFO DAGScheduler:59 - ResultStage 102 (treeAggregate at GradientDescent.scala:225) finished in 0.039 s -2016-05-24 22:18:10 INFO DAGScheduler:59 - Job 102 finished: treeAggregate at GradientDescent.scala:225, took 0.070208 s -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1119664, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_203 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1125256, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_203_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_203_piece0 in memory on localhost:56717 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 203 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:10 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Got job 103 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:10 INFO DAGScheduler:59 - Final stage: ResultStage 103(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting ResultStage 103 (MapPartitionsRDD[211] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1128868, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_204 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:18:10 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1140652, maxMem=766075207 -2016-05-24 22:18:10 INFO MemoryStore:59 - Block broadcast_204_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:18:10 INFO BlockManagerInfo:59 - Added broadcast_204_piece0 in memory on localhost:56717 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:10 INFO SparkContext:59 - Created broadcast 204 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 103 (MapPartitionsRDD[211] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Adding task set 103.0 with 2 tasks -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 103.0 (TID 205, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 103.0 (TID 206, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:10 INFO Executor:59 - Running task 1.0 in stage 103.0 (TID 206) -2016-05-24 22:18:10 INFO Executor:59 - Running task 0.0 in stage 103.0 (TID 205) -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:10 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:10 INFO Executor:59 - Finished task 0.0 in stage 103.0 (TID 205). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO Executor:59 - Finished task 1.0 in stage 103.0 (TID 206). 7968 bytes result sent to driver -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 103.0 (TID 205) in 7 ms on localhost (1/2) -2016-05-24 22:18:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 103.0 (TID 206) in 7 ms on localhost (2/2) -2016-05-24 22:18:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 103.0, whose tasks have all completed, from pool -2016-05-24 22:18:10 INFO DAGScheduler:59 - ResultStage 103 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:18:10 INFO DAGScheduler:59 - Job 103 finished: treeAggregate at GradientDescent.scala:225, took 0.019223 s -2016-05-24 22:18:10 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 3363.579936927662, 3356.5316534793847, 3349.536449573466, 3342.59343622967, 3335.701748404667, 3328.860544103912, 3322.069003535364, 3315.3263283026713, 3308.6317406356134, 3301.984482655661 -2016-05-24 22:18:10 INFO SparkContext:59 - Starting job: sortByKey at BinaryClassificationMetrics.scala:156 -2016-05-24 22:18:10 INFO DAGScheduler:59 - Registering RDD 212 (map at SVMFromSparkLearning.scala:44) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Got job 104 (sortByKey at BinaryClassificationMetrics.scala:156) with 2 output partitions -2016-05-24 22:18:10 INFO DAGScheduler:59 - Final stage: ResultStage 105(sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 104) -2016-05-24 22:18:10 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 104) -2016-05-24 22:18:11 INFO DAGScheduler:59 - Submitting ShuffleMapStage 104 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44), which has no missing parents -2016-05-24 22:18:11 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=1144135, maxMem=766075207 -2016-05-24 22:18:11 INFO MemoryStore:59 - Block broadcast_205 stored as values in memory (estimated size 10.8 KB, free 729.5 MB) -2016-05-24 22:18:11 INFO MemoryStore:59 - ensureFreeSpace(6433) called with curMem=1155143, maxMem=766075207 -2016-05-24 22:18:11 INFO MemoryStore:59 - Block broadcast_205_piece0 stored as bytes in memory (estimated size 6.3 KB, free 729.5 MB) -2016-05-24 22:18:11 INFO BlockManagerInfo:59 - Added broadcast_205_piece0 in memory on localhost:56717 (size: 6.3 KB, free: 730.1 MB) -2016-05-24 22:18:11 INFO SparkContext:59 - Created broadcast 205 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 104 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44) -2016-05-24 22:18:11 INFO TaskSchedulerImpl:59 - Adding task set 104.0 with 2 tasks -2016-05-24 22:18:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 104.0 (TID 207, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:18:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 104.0 (TID 208, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:18:11 INFO Executor:59 - Running task 0.0 in stage 104.0 (TID 207) -2016-05-24 22:18:11 INFO Executor:59 - Running task 1.0 in stage 104.0 (TID 208) -2016-05-24 22:18:11 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:18:11 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:18:11 INFO Executor:59 - Finished task 0.0 in stage 104.0 (TID 207). 2254 bytes result sent to driver -2016-05-24 22:18:11 INFO Executor:59 - Finished task 1.0 in stage 104.0 (TID 208). 2254 bytes result sent to driver -2016-05-24 22:18:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 104.0 (TID 208) in 715 ms on localhost (1/2) -2016-05-24 22:18:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 104.0 (TID 207) in 718 ms on localhost (2/2) -2016-05-24 22:18:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 104.0, whose tasks have all completed, from pool -2016-05-24 22:18:11 INFO DAGScheduler:59 - ShuffleMapStage 104 (map at SVMFromSparkLearning.scala:44) finished in 0.721 s -2016-05-24 22:18:11 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:18:11 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:18:11 INFO DAGScheduler:59 - waiting: Set(ResultStage 105) -2016-05-24 22:18:11 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:18:11 INFO DAGScheduler:59 - Missing parents for ResultStage 105: List() -2016-05-24 22:18:11 INFO DAGScheduler:59 - Submitting ResultStage 105 (MapPartitionsRDD[215] at sortByKey at BinaryClassificationMetrics.scala:156), which is now runnable -2016-05-24 22:18:11 INFO MemoryStore:59 - ensureFreeSpace(3344) called with curMem=1161576, maxMem=766075207 -2016-05-24 22:18:11 INFO MemoryStore:59 - Block broadcast_206 stored as values in memory (estimated size 3.3 KB, free 729.5 MB) -2016-05-24 22:18:11 INFO MemoryStore:59 - ensureFreeSpace(1900) called with curMem=1164920, maxMem=766075207 -2016-05-24 22:18:11 INFO MemoryStore:59 - Block broadcast_206_piece0 stored as bytes in memory (estimated size 1900.0 B, free 729.5 MB) -2016-05-24 22:18:11 INFO BlockManagerInfo:59 - Added broadcast_206_piece0 in memory on localhost:56717 (size: 1900.0 B, free: 730.1 MB) -2016-05-24 22:18:11 INFO SparkContext:59 - Created broadcast 206 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 105 (MapPartitionsRDD[215] at sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:18:11 INFO TaskSchedulerImpl:59 - Adding task set 105.0 with 2 tasks -2016-05-24 22:18:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 105.0 (TID 209, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:18:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 105.0 (TID 210, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:18:11 INFO Executor:59 - Running task 0.0 in stage 105.0 (TID 209) -2016-05-24 22:18:11 INFO Executor:59 - Running task 1.0 in stage 105.0 (TID 210) -2016-05-24 22:18:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:18:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:18:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-24 22:18:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-24 22:18:11 INFO Executor:59 - Finished task 1.0 in stage 105.0 (TID 210). 1517 bytes result sent to driver -2016-05-24 22:18:11 INFO Executor:59 - Finished task 0.0 in stage 105.0 (TID 209). 1469 bytes result sent to driver -2016-05-24 22:18:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 105.0 (TID 210) in 49 ms on localhost (1/2) -2016-05-24 22:18:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 105.0 (TID 209) in 51 ms on localhost (2/2) -2016-05-24 22:18:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 105.0, whose tasks have all completed, from pool -2016-05-24 22:18:11 INFO DAGScheduler:59 - ResultStage 105 (sortByKey at BinaryClassificationMetrics.scala:156) finished in 0.052 s -2016-05-24 22:18:11 INFO DAGScheduler:59 - Job 104 finished: sortByKey at BinaryClassificationMetrics.scala:156, took 0.861185 s -2016-05-24 22:18:11 INFO SparkContext:59 - Starting job: collect at BinaryClassificationMetrics.scala:193 -2016-05-24 22:18:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 154 bytes -2016-05-24 22:18:11 INFO DAGScheduler:59 - Registering RDD 213 (combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:18:11 INFO DAGScheduler:59 - Got job 105 (collect at BinaryClassificationMetrics.scala:193) with 2 output partitions -2016-05-24 22:18:11 INFO DAGScheduler:59 - Final stage: ResultStage 108(collect at BinaryClassificationMetrics.scala:193) -2016-05-24 22:18:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 107) -2016-05-24 22:18:11 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 107) -2016-05-24 22:18:11 INFO DAGScheduler:59 - Submitting ShuffleMapStage 107 (ShuffledRDD[213] at combineByKey at BinaryClassificationMetrics.scala:152), which has no missing parents -2016-05-24 22:18:11 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=1166820, maxMem=766075207 -2016-05-24 22:18:11 INFO MemoryStore:59 - Block broadcast_207 stored as values in memory (estimated size 2.8 KB, free 729.5 MB) -2016-05-24 22:18:11 INFO MemoryStore:59 - ensureFreeSpace(1658) called with curMem=1169716, maxMem=766075207 -2016-05-24 22:18:11 INFO MemoryStore:59 - Block broadcast_207_piece0 stored as bytes in memory (estimated size 1658.0 B, free 729.5 MB) -2016-05-24 22:18:11 INFO BlockManagerInfo:59 - Added broadcast_207_piece0 in memory on localhost:56717 (size: 1658.0 B, free: 730.1 MB) -2016-05-24 22:18:11 INFO SparkContext:59 - Created broadcast 207 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 107 (ShuffledRDD[213] at combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:18:11 INFO TaskSchedulerImpl:59 - Adding task set 107.0 with 2 tasks -2016-05-24 22:18:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 107.0 (TID 211, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:18:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 107.0 (TID 212, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:18:11 INFO Executor:59 - Running task 0.0 in stage 107.0 (TID 211) -2016-05-24 22:18:11 INFO Executor:59 - Running task 1.0 in stage 107.0 (TID 212) -2016-05-24 22:18:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:18:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:18:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:18:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:18:11 INFO Executor:59 - Finished task 1.0 in stage 107.0 (TID 212). 1375 bytes result sent to driver -2016-05-24 22:18:11 INFO Executor:59 - Finished task 0.0 in stage 107.0 (TID 211). 1375 bytes result sent to driver -2016-05-24 22:18:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 107.0 (TID 212) in 31 ms on localhost (1/2) -2016-05-24 22:18:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 107.0 (TID 211) in 33 ms on localhost (2/2) -2016-05-24 22:18:11 INFO DAGScheduler:59 - ShuffleMapStage 107 (combineByKey at BinaryClassificationMetrics.scala:152) finished in 0.033 s -2016-05-24 22:18:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 107.0, whose tasks have all completed, from pool -2016-05-24 22:18:11 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:18:11 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:18:11 INFO DAGScheduler:59 - waiting: Set(ResultStage 108) -2016-05-24 22:18:11 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:18:11 INFO DAGScheduler:59 - Missing parents for ResultStage 108: List() -2016-05-24 22:18:11 INFO DAGScheduler:59 - Submitting ResultStage 108 (MapPartitionsRDD[218] at mapPartitions at BinaryClassificationMetrics.scala:189), which is now runnable -2016-05-24 22:18:11 INFO MemoryStore:59 - ensureFreeSpace(3472) called with curMem=1171374, maxMem=766075207 -2016-05-24 22:18:11 INFO MemoryStore:59 - Block broadcast_208 stored as values in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:18:11 INFO MemoryStore:59 - ensureFreeSpace(1981) called with curMem=1174846, maxMem=766075207 -2016-05-24 22:18:11 INFO MemoryStore:59 - Block broadcast_208_piece0 stored as bytes in memory (estimated size 1981.0 B, free 729.5 MB) -2016-05-24 22:18:11 INFO BlockManagerInfo:59 - Added broadcast_208_piece0 in memory on localhost:56717 (size: 1981.0 B, free: 730.1 MB) -2016-05-24 22:18:11 INFO SparkContext:59 - Created broadcast 208 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 108 (MapPartitionsRDD[218] at mapPartitions at BinaryClassificationMetrics.scala:189) -2016-05-24 22:18:11 INFO TaskSchedulerImpl:59 - Adding task set 108.0 with 2 tasks -2016-05-24 22:18:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 108.0 (TID 213, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:18:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 108.0 (TID 214, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:18:11 INFO Executor:59 - Running task 1.0 in stage 108.0 (TID 214) -2016-05-24 22:18:11 INFO Executor:59 - Running task 0.0 in stage 108.0 (TID 213) -2016-05-24 22:18:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:18:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:18:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:18:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:18:11 INFO Executor:59 - Finished task 0.0 in stage 108.0 (TID 213). 1330 bytes result sent to driver -2016-05-24 22:18:11 INFO Executor:59 - Finished task 1.0 in stage 108.0 (TID 214). 1330 bytes result sent to driver -2016-05-24 22:18:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 108.0 (TID 213) in 17 ms on localhost (1/2) -2016-05-24 22:18:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 108.0 (TID 214) in 16 ms on localhost (2/2) -2016-05-24 22:18:11 INFO DAGScheduler:59 - ResultStage 108 (collect at BinaryClassificationMetrics.scala:193) finished in 0.017 s -2016-05-24 22:18:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 108.0, whose tasks have all completed, from pool -2016-05-24 22:18:11 INFO DAGScheduler:59 - Job 105 finished: collect at BinaryClassificationMetrics.scala:193, took 0.111616 s -2016-05-24 22:18:11 INFO BinaryClassificationMetrics:59 - Total counts: {numPos: 25, numNeg: 11} -2016-05-24 22:18:12 INFO SparkContext:59 - Starting job: runJob at SlidingRDD.scala:75 -2016-05-24 22:18:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 154 bytes -2016-05-24 22:18:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-24 22:18:12 INFO DAGScheduler:59 - Got job 106 (runJob at SlidingRDD.scala:75) with 3 output partitions -2016-05-24 22:18:12 INFO DAGScheduler:59 - Final stage: ResultStage 111(runJob at SlidingRDD.scala:75) -2016-05-24 22:18:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 110) -2016-05-24 22:18:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:12 INFO DAGScheduler:59 - Submitting ResultStage 111 (UnionRDD[224] at UnionRDD at BinaryClassificationMetrics.scala:92), which has no missing parents -2016-05-24 22:18:12 INFO MemoryStore:59 - ensureFreeSpace(5184) called with curMem=1176827, maxMem=766075207 -2016-05-24 22:18:12 INFO MemoryStore:59 - Block broadcast_209 stored as values in memory (estimated size 5.1 KB, free 729.5 MB) -2016-05-24 22:18:12 INFO MemoryStore:59 - ensureFreeSpace(2788) called with curMem=1182011, maxMem=766075207 -2016-05-24 22:18:12 INFO MemoryStore:59 - Block broadcast_209_piece0 stored as bytes in memory (estimated size 2.7 KB, free 729.5 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_208_piece0 on localhost:56717 in memory (size: 1981.0 B, free: 730.1 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Added broadcast_209_piece0 in memory on localhost:56717 (size: 2.7 KB, free: 730.1 MB) -2016-05-24 22:18:12 INFO SparkContext:59 - Created broadcast 209 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:12 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 111 (UnionRDD[224] at UnionRDD at BinaryClassificationMetrics.scala:92) -2016-05-24 22:18:12 INFO TaskSchedulerImpl:59 - Adding task set 111.0 with 3 tasks -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 108 -2016-05-24 22:18:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 111.0 (TID 215, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:18:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 111.0 (TID 216, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_207_piece0 on localhost:56717 in memory (size: 1658.0 B, free: 730.1 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 107 -2016-05-24 22:18:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 111.0 (TID 217, localhost, PROCESS_LOCAL, 2318 bytes) -2016-05-24 22:18:12 INFO Executor:59 - Running task 0.0 in stage 111.0 (TID 215) -2016-05-24 22:18:12 INFO Executor:59 - Running task 1.0 in stage 111.0 (TID 216) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_206_piece0 on localhost:56717 in memory (size: 1900.0 B, free: 730.1 MB) -2016-05-24 22:18:12 INFO CacheManager:59 - Partition rdd_219_0 not found, computing it -2016-05-24 22:18:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:18:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:18:12 INFO Executor:59 - Running task 2.0 in stage 111.0 (TID 217) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 106 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_205_piece0 on localhost:56717 in memory (size: 6.3 KB, free: 730.1 MB) -2016-05-24 22:18:12 INFO CacheManager:59 - Partition rdd_219_1 not found, computing it -2016-05-24 22:18:12 INFO MemoryStore:59 - ensureFreeSpace(1448) called with curMem=1152107, maxMem=766075207 -2016-05-24 22:18:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:18:12 INFO MemoryStore:59 - Block rdd_219_0 stored as values in memory (estimated size 1448.0 B, free 729.5 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 105 -2016-05-24 22:18:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:18:12 INFO Executor:59 - Finished task 2.0 in stage 111.0 (TID 217). 1053 bytes result sent to driver -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Added rdd_219_0 in memory on localhost:56717 (size: 1448.0 B, free: 730.1 MB) -2016-05-24 22:18:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 111.0 (TID 217) in 77 ms on localhost (1/3) -2016-05-24 22:18:12 INFO MemoryStore:59 - ensureFreeSpace(1616) called with curMem=1150072, maxMem=766075207 -2016-05-24 22:18:12 INFO MemoryStore:59 - Block rdd_219_1 stored as values in memory (estimated size 1616.0 B, free 729.5 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_204_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Added rdd_219_1 in memory on localhost:56717 (size: 1616.0 B, free: 730.1 MB) -2016-05-24 22:18:12 INFO Executor:59 - Finished task 0.0 in stage 111.0 (TID 215). 1883 bytes result sent to driver -2016-05-24 22:18:12 INFO Executor:59 - Finished task 1.0 in stage 111.0 (TID 216). 1883 bytes result sent to driver -2016-05-24 22:18:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 111.0 (TID 215) in 87 ms on localhost (2/3) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 104 -2016-05-24 22:18:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 111.0 (TID 216) in 86 ms on localhost (3/3) -2016-05-24 22:18:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 111.0, whose tasks have all completed, from pool -2016-05-24 22:18:12 INFO DAGScheduler:59 - ResultStage 111 (runJob at SlidingRDD.scala:75) finished in 0.090 s -2016-05-24 22:18:12 INFO DAGScheduler:59 - Job 106 finished: runJob at SlidingRDD.scala:75, took 0.165891 s -2016-05-24 22:18:12 INFO SparkContext:59 - Starting job: aggregate at AreaUnderCurve.scala:45 -2016-05-24 22:18:12 INFO DAGScheduler:59 - Got job 107 (aggregate at AreaUnderCurve.scala:45) with 4 output partitions -2016-05-24 22:18:12 INFO DAGScheduler:59 - Final stage: ResultStage 114(aggregate at AreaUnderCurve.scala:45) -2016-05-24 22:18:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 113) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_203_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:12 INFO DAGScheduler:59 - Submitting ResultStage 114 (SlidingRDD[225] at RDD at SlidingRDD.scala:47), which has no missing parents -2016-05-24 22:18:12 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=1130700, maxMem=766075207 -2016-05-24 22:18:12 INFO MemoryStore:59 - Block broadcast_210 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:18:12 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=1132872, maxMem=766075207 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_202_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:12 INFO MemoryStore:59 - Block broadcast_210_piece0 stored as bytes in memory (estimated size 2.9 KB, free 729.5 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Added broadcast_210_piece0 in memory on localhost:56717 (size: 2.9 KB, free: 730.1 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 103 -2016-05-24 22:18:12 INFO SparkContext:59 - Created broadcast 210 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:12 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 114 (SlidingRDD[225] at RDD at SlidingRDD.scala:47) -2016-05-24 22:18:12 INFO TaskSchedulerImpl:59 - Adding task set 114.0 with 4 tasks -2016-05-24 22:18:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 114.0 (TID 218, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:18:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 114.0 (TID 219, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:18:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 114.0 (TID 220, localhost, PROCESS_LOCAL, 2635 bytes) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_201_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:12 INFO TaskSetManager:59 - Starting task 3.0 in stage 114.0 (TID 221, localhost, PROCESS_LOCAL, 2475 bytes) -2016-05-24 22:18:12 INFO Executor:59 - Running task 1.0 in stage 114.0 (TID 218) -2016-05-24 22:18:12 INFO Executor:59 - Running task 2.0 in stage 114.0 (TID 219) -2016-05-24 22:18:12 INFO BlockManager:59 - Found block rdd_219_0 locally -2016-05-24 22:18:12 INFO BlockManager:59 - Found block rdd_219_1 locally -2016-05-24 22:18:12 INFO Executor:59 - Running task 0.0 in stage 114.0 (TID 220) -2016-05-24 22:18:12 INFO Executor:59 - Finished task 1.0 in stage 114.0 (TID 218). 2084 bytes result sent to driver -2016-05-24 22:18:12 INFO Executor:59 - Finished task 2.0 in stage 114.0 (TID 219). 2084 bytes result sent to driver -2016-05-24 22:18:12 INFO Executor:59 - Finished task 0.0 in stage 114.0 (TID 220). 955 bytes result sent to driver -2016-05-24 22:18:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 114.0 (TID 218) in 21 ms on localhost (1/4) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_200_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:12 INFO Executor:59 - Running task 3.0 in stage 114.0 (TID 221) -2016-05-24 22:18:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 114.0 (TID 219) in 18 ms on localhost (2/4) -2016-05-24 22:18:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 114.0 (TID 220) in 17 ms on localhost (3/4) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 102 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_199_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:12 INFO Executor:59 - Finished task 3.0 in stage 114.0 (TID 221). 955 bytes result sent to driver -2016-05-24 22:18:12 INFO TaskSetManager:59 - Finished task 3.0 in stage 114.0 (TID 221) in 19 ms on localhost (4/4) -2016-05-24 22:18:12 INFO DAGScheduler:59 - ResultStage 114 (aggregate at AreaUnderCurve.scala:45) finished in 0.028 s -2016-05-24 22:18:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 114.0, whose tasks have all completed, from pool -2016-05-24 22:18:12 INFO DAGScheduler:59 - Job 107 finished: aggregate at AreaUnderCurve.scala:45, took 0.088984 s -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_198_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 101 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_197_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:12 INFO SparkContext:59 - Starting job: take at SVMFromSparkLearning.scala:54 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_196_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:12 INFO DAGScheduler:59 - Got job 108 (take at SVMFromSparkLearning.scala:54) with 1 output partitions -2016-05-24 22:18:12 INFO DAGScheduler:59 - Final stage: ResultStage 115(take at SVMFromSparkLearning.scala:54) -2016-05-24 22:18:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 100 -2016-05-24 22:18:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:12 INFO DAGScheduler:59 - Submitting ResultStage 115 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44), which has no missing parents -2016-05-24 22:18:12 INFO MemoryStore:59 - ensureFreeSpace(10312) called with curMem=1041444, maxMem=766075207 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_195_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:12 INFO MemoryStore:59 - Block broadcast_211 stored as values in memory (estimated size 10.1 KB, free 729.6 MB) -2016-05-24 22:18:12 INFO MemoryStore:59 - ensureFreeSpace(6082) called with curMem=1048273, maxMem=766075207 -2016-05-24 22:18:12 INFO MemoryStore:59 - Block broadcast_211_piece0 stored as bytes in memory (estimated size 5.9 KB, free 729.6 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_194_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Added broadcast_211_piece0 in memory on localhost:56717 (size: 5.9 KB, free: 730.1 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 99 -2016-05-24 22:18:12 INFO SparkContext:59 - Created broadcast 211 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:12 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 115 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44) -2016-05-24 22:18:12 INFO TaskSchedulerImpl:59 - Adding task set 115.0 with 1 tasks -2016-05-24 22:18:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 115.0 (TID 222, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:18:12 INFO Executor:59 - Running task 0.0 in stage 115.0 (TID 222) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_193_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:12 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_192_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:12 INFO Executor:59 - Finished task 0.0 in stage 115.0 (TID 222). 2401 bytes result sent to driver -2016-05-24 22:18:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 115.0 (TID 222) in 9 ms on localhost (1/1) -2016-05-24 22:18:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 115.0, whose tasks have all completed, from pool -2016-05-24 22:18:12 INFO DAGScheduler:59 - ResultStage 115 (take at SVMFromSparkLearning.scala:54) finished in 0.010 s -2016-05-24 22:18:12 INFO DAGScheduler:59 - Job 108 finished: take at SVMFromSparkLearning.scala:54, took 0.020838 s -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 98 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_191_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_190_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 97 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_189_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_188_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 96 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_187_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_186_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 95 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_185_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_184_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 94 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_183_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_182_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 93 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_181_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_180_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 92 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_179_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_178_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 91 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_177_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_176_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 90 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_175_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_174_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 89 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_173_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_172_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 88 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_171_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_170_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 87 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_169_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_168_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 86 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_167_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_166_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 85 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_165_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_164_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 84 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_163_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_162_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 83 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_161_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_160_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 82 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_159_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_158_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 81 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_157_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_156_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 80 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_155_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_154_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 79 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_153_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_152_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 78 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_151_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_150_piece0 on localhost:56717 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:12 INFO ContextCleaner:59 - Cleaned accumulator 77 -2016-05-24 22:18:12 INFO BlockManagerInfo:59 - Removed broadcast_149_piece0 on localhost:56717 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:12 INFO SparkContext:59 - Starting job: saveAsTextFile at GLMClassificationModel.scala:61 -2016-05-24 22:18:12 INFO DAGScheduler:59 - Got job 109 (saveAsTextFile at GLMClassificationModel.scala:61) with 1 output partitions -2016-05-24 22:18:12 INFO DAGScheduler:59 - Final stage: ResultStage 116(saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:18:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:12 INFO DAGScheduler:59 - Submitting ResultStage 116 (MapPartitionsRDD[227] at saveAsTextFile at GLMClassificationModel.scala:61), which has no missing parents -2016-05-24 22:18:13 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=494941, maxMem=766075207 -2016-05-24 22:18:13 INFO MemoryStore:59 - Block broadcast_212 stored as values in memory (estimated size 124.0 KB, free 730.0 MB) -2016-05-24 22:18:13 INFO MemoryStore:59 - ensureFreeSpace(42589) called with curMem=621933, maxMem=766075207 -2016-05-24 22:18:13 INFO MemoryStore:59 - Block broadcast_212_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.0 MB) -2016-05-24 22:18:13 INFO BlockManagerInfo:59 - Added broadcast_212_piece0 in memory on localhost:56717 (size: 41.6 KB, free: 730.2 MB) -2016-05-24 22:18:13 INFO SparkContext:59 - Created broadcast 212 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:13 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 116 (MapPartitionsRDD[227] at saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:18:13 INFO TaskSchedulerImpl:59 - Adding task set 116.0 with 1 tasks -2016-05-24 22:18:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 116.0 (TID 223, localhost, PROCESS_LOCAL, 2197 bytes) -2016-05-24 22:18:13 INFO Executor:59 - Running task 0.0 in stage 116.0 (TID 223) -2016-05-24 22:18:13 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242218_0116_m_000000_223' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524221812327/result/metadata/_temporary/0/task_201605242218_0116_m_000000 -2016-05-24 22:18:13 INFO SparkHadoopMapRedUtil:59 - attempt_201605242218_0116_m_000000_223: Committed -2016-05-24 22:18:13 INFO Executor:59 - Finished task 0.0 in stage 116.0 (TID 223). 1864 bytes result sent to driver -2016-05-24 22:18:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 116.0 (TID 223) in 835 ms on localhost (1/1) -2016-05-24 22:18:13 INFO DAGScheduler:59 - ResultStage 116 (saveAsTextFile at GLMClassificationModel.scala:61) finished in 0.835 s -2016-05-24 22:18:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 116.0, whose tasks have all completed, from pool -2016-05-24 22:18:13 INFO DAGScheduler:59 - Job 109 finished: saveAsTextFile at GLMClassificationModel.scala:61, took 0.942451 s -2016-05-24 22:18:15 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:18:15 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:18:15 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:65 -2016-05-24 22:18:15 INFO DAGScheduler:59 - Got job 110 (parquet at GLMClassificationModel.scala:65) with 1 output partitions -2016-05-24 22:18:15 INFO DAGScheduler:59 - Final stage: ResultStage 117(parquet at GLMClassificationModel.scala:65) -2016-05-24 22:18:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:15 INFO DAGScheduler:59 - Submitting ResultStage 117 (MapPartitionsRDD[229] at rddToDataFrameHolder at GLMClassificationModel.scala:65), which has no missing parents -2016-05-24 22:18:16 INFO MemoryStore:59 - ensureFreeSpace(67104) called with curMem=664522, maxMem=766075207 -2016-05-24 22:18:16 INFO MemoryStore:59 - Block broadcast_213 stored as values in memory (estimated size 65.5 KB, free 729.9 MB) -2016-05-24 22:18:16 INFO MemoryStore:59 - ensureFreeSpace(23441) called with curMem=731626, maxMem=766075207 -2016-05-24 22:18:16 INFO MemoryStore:59 - Block broadcast_213_piece0 stored as bytes in memory (estimated size 22.9 KB, free 729.9 MB) -2016-05-24 22:18:16 INFO BlockManagerInfo:59 - Added broadcast_213_piece0 in memory on localhost:56717 (size: 22.9 KB, free: 730.2 MB) -2016-05-24 22:18:16 INFO SparkContext:59 - Created broadcast 213 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:16 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 117 (MapPartitionsRDD[229] at rddToDataFrameHolder at GLMClassificationModel.scala:65) -2016-05-24 22:18:16 INFO TaskSchedulerImpl:59 - Adding task set 117.0 with 1 tasks -2016-05-24 22:18:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 117.0 (TID 224, localhost, PROCESS_LOCAL, 7978 bytes) -2016-05-24 22:18:16 INFO Executor:59 - Running task 0.0 in stage 117.0 (TID 224) -2016-05-24 22:18:16 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:18:16 INFO CodecConfig:151 - Compression: GZIP -2016-05-24 22:18:16 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-24 22:18:16 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-24 22:18:16 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-24 22:18:16 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-24 22:18:16 INFO ParquetOutputFormat:151 - Validation is off -2016-05-24 22:18:16 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-24 22:18:16 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-24 22:18:16 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 6,156 -2016-05-24 22:18:16 INFO ColumnChunkPageWriteStore:151 - written 61B for [weights, type] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [RLE, PLAIN, BIT_PACKED] -2016-05-24 22:18:16 INFO ColumnChunkPageWriteStore:151 - written 48B for [weights, size] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [RLE, PLAIN, BIT_PACKED] -2016-05-24 22:18:16 INFO ColumnChunkPageWriteStore:151 - written 51B for [weights, indices, array] INT32: 1 values, 13B raw, 30B comp, 1 pages, encodings: [RLE, PLAIN] -2016-05-24 22:18:16 INFO ColumnChunkPageWriteStore:151 - written 637B for [weights, values, array] DOUBLE: 692 values, 668B raw, 593B comp, 1 pages, encodings: [RLE, PLAIN_DICTIONARY], dic { 413 entries, 3,304B raw, 413B comp} -2016-05-24 22:18:16 INFO ColumnChunkPageWriteStore:151 - written 69B for [intercept] DOUBLE: 1 values, 14B raw, 28B comp, 1 pages, encodings: [RLE, PLAIN, BIT_PACKED] -2016-05-24 22:18:16 INFO ColumnChunkPageWriteStore:151 - written 47B for [threshold] DOUBLE: 1 values, 6B raw, 26B comp, 1 pages, encodings: [RLE, PLAIN, BIT_PACKED] -2016-05-24 22:18:16 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242218_0117_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524221812327/result/data/_temporary/0/task_201605242218_0117_m_000000 -2016-05-24 22:18:16 INFO SparkHadoopMapRedUtil:59 - attempt_201605242218_0117_m_000000_0: Committed -2016-05-24 22:18:16 INFO Executor:59 - Finished task 0.0 in stage 117.0 (TID 224). 915 bytes result sent to driver -2016-05-24 22:18:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 117.0 (TID 224) in 708 ms on localhost (1/1) -2016-05-24 22:18:16 INFO DAGScheduler:59 - ResultStage 117 (parquet at GLMClassificationModel.scala:65) finished in 0.709 s -2016-05-24 22:18:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 117.0, whose tasks have all completed, from pool -2016-05-24 22:18:16 INFO DAGScheduler:59 - Job 110 finished: parquet at GLMClassificationModel.scala:65, took 0.753378 s -2016-05-24 22:18:16 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:18:17 INFO DefaultWriterContainer:59 - Job job_201605242218_0000 committed. -2016-05-24 22:18:17 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524221812327/result/data on driver -2016-05-24 22:18:17 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524221812327/result/data on driver -2016-05-24 22:18:17 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=755067, maxMem=766075207 -2016-05-24 22:18:17 INFO MemoryStore:59 - Block broadcast_214 stored as values in memory (estimated size 225.3 KB, free 729.6 MB) -2016-05-24 22:18:17 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=985771, maxMem=766075207 -2016-05-24 22:18:17 INFO MemoryStore:59 - Block broadcast_214_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.6 MB) -2016-05-24 22:18:17 INFO BlockManagerInfo:59 - Added broadcast_214_piece0 in memory on localhost:56717 (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:18:17 INFO SparkContext:59 - Created broadcast 214 from textFile at modelSaveLoad.scala:129 -2016-05-24 22:18:17 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:18:17 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-24 22:18:17 INFO DAGScheduler:59 - Got job 111 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-24 22:18:17 INFO DAGScheduler:59 - Final stage: ResultStage 118(first at modelSaveLoad.scala:129) -2016-05-24 22:18:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:17 INFO DAGScheduler:59 - Submitting ResultStage 118 (MapPartitionsRDD[232] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-24 22:18:17 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=1005559, maxMem=766075207 -2016-05-24 22:18:17 INFO MemoryStore:59 - Block broadcast_215 stored as values in memory (estimated size 3.1 KB, free 729.6 MB) -2016-05-24 22:18:17 INFO MemoryStore:59 - ensureFreeSpace(1898) called with curMem=1008775, maxMem=766075207 -2016-05-24 22:18:17 INFO MemoryStore:59 - Block broadcast_215_piece0 stored as bytes in memory (estimated size 1898.0 B, free 729.6 MB) -2016-05-24 22:18:17 INFO BlockManagerInfo:59 - Added broadcast_215_piece0 in memory on localhost:56717 (size: 1898.0 B, free: 730.2 MB) -2016-05-24 22:18:17 INFO SparkContext:59 - Created broadcast 215 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 118 (MapPartitionsRDD[232] at textFile at modelSaveLoad.scala:129) -2016-05-24 22:18:17 INFO TaskSchedulerImpl:59 - Adding task set 118.0 with 1 tasks -2016-05-24 22:18:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 118.0 (TID 225, localhost, PROCESS_LOCAL, 2238 bytes) -2016-05-24 22:18:17 INFO Executor:59 - Running task 0.0 in stage 118.0 (TID 225) -2016-05-24 22:18:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524221812327/result/metadata/part-00000:0+54 -2016-05-24 22:18:17 INFO Executor:59 - Finished task 0.0 in stage 118.0 (TID 225). 2154 bytes result sent to driver -2016-05-24 22:18:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 118.0 (TID 225) in 12 ms on localhost (1/1) -2016-05-24 22:18:17 INFO DAGScheduler:59 - ResultStage 118 (first at modelSaveLoad.scala:129) finished in 0.012 s -2016-05-24 22:18:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 118.0, whose tasks have all completed, from pool -2016-05-24 22:18:17 INFO DAGScheduler:59 - Job 111 finished: first at modelSaveLoad.scala:129, took 0.049013 s -2016-05-24 22:18:17 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524221812327/result/data on driver -2016-05-24 22:18:17 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:78 -2016-05-24 22:18:17 INFO DAGScheduler:59 - Got job 112 (parquet at GLMClassificationModel.scala:78) with 4 output partitions -2016-05-24 22:18:17 INFO DAGScheduler:59 - Final stage: ResultStage 119(parquet at GLMClassificationModel.scala:78) -2016-05-24 22:18:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:17 INFO DAGScheduler:59 - Submitting ResultStage 119 (MapPartitionsRDD[234] at parquet at GLMClassificationModel.scala:78), which has no missing parents -2016-05-24 22:18:17 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=1010673, maxMem=766075207 -2016-05-24 22:18:17 INFO MemoryStore:59 - Block broadcast_216 stored as values in memory (estimated size 61.5 KB, free 729.6 MB) -2016-05-24 22:18:17 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=1073673, maxMem=766075207 -2016-05-24 22:18:17 INFO MemoryStore:59 - Block broadcast_216_piece0 stored as bytes in memory (estimated size 20.6 KB, free 729.5 MB) -2016-05-24 22:18:17 INFO BlockManagerInfo:59 - Added broadcast_216_piece0 in memory on localhost:56717 (size: 20.6 KB, free: 730.2 MB) -2016-05-24 22:18:17 INFO SparkContext:59 - Created broadcast 216 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 119 (MapPartitionsRDD[234] at parquet at GLMClassificationModel.scala:78) -2016-05-24 22:18:17 INFO TaskSchedulerImpl:59 - Adding task set 119.0 with 4 tasks -2016-05-24 22:18:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 119.0 (TID 226, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:18:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 119.0 (TID 227, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:18:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 119.0 (TID 228, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:18:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 119.0 (TID 229, localhost, PROCESS_LOCAL, 2279 bytes) -2016-05-24 22:18:17 INFO Executor:59 - Running task 1.0 in stage 119.0 (TID 227) -2016-05-24 22:18:17 INFO Executor:59 - Running task 3.0 in stage 119.0 (TID 229) -2016-05-24 22:18:18 INFO Executor:59 - Running task 2.0 in stage 119.0 (TID 228) -2016-05-24 22:18:18 INFO Executor:59 - Running task 0.0 in stage 119.0 (TID 226) -2016-05-24 22:18:18 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:18:18 INFO Executor:59 - Finished task 1.0 in stage 119.0 (TID 227). 936 bytes result sent to driver -2016-05-24 22:18:18 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:18:18 INFO Executor:59 - Finished task 2.0 in stage 119.0 (TID 228). 936 bytes result sent to driver -2016-05-24 22:18:18 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:18:18 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:18:18 INFO Executor:59 - Finished task 0.0 in stage 119.0 (TID 226). 936 bytes result sent to driver -2016-05-24 22:18:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 119.0 (TID 227) in 112 ms on localhost (1/4) -2016-05-24 22:18:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 119.0 (TID 228) in 111 ms on localhost (2/4) -2016-05-24 22:18:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 119.0 (TID 226) in 114 ms on localhost (3/4) -2016-05-24 22:18:18 INFO Executor:59 - Finished task 3.0 in stage 119.0 (TID 229). 1802 bytes result sent to driver -2016-05-24 22:18:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 119.0 (TID 229) in 147 ms on localhost (4/4) -2016-05-24 22:18:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 119.0, whose tasks have all completed, from pool -2016-05-24 22:18:18 INFO DAGScheduler:59 - ResultStage 119 (parquet at GLMClassificationModel.scala:78) finished in 0.172 s -2016-05-24 22:18:18 INFO DAGScheduler:59 - Job 112 finished: parquet at GLMClassificationModel.scala:78, took 0.232301 s -2016-05-24 22:18:18 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=1094746, maxMem=766075207 -2016-05-24 22:18:18 INFO MemoryStore:59 - Block broadcast_217 stored as values in memory (estimated size 86.4 KB, free 729.5 MB) -2016-05-24 22:18:18 INFO BlockManagerInfo:59 - Removed broadcast_216_piece0 on localhost:56717 in memory (size: 20.6 KB, free: 730.2 MB) -2016-05-24 22:18:18 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1099185, maxMem=766075207 -2016-05-24 22:18:18 INFO MemoryStore:59 - Block broadcast_217_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.5 MB) -2016-05-24 22:18:18 INFO ContextCleaner:59 - Cleaned accumulator 115 -2016-05-24 22:18:18 INFO BlockManagerInfo:59 - Added broadcast_217_piece0 in memory on localhost:56717 (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:18:18 INFO SparkContext:59 - Created broadcast 217 from take at GLMClassificationModel.scala:79 -2016-05-24 22:18:18 INFO BlockManagerInfo:59 - Removed broadcast_215_piece0 on localhost:56717 in memory (size: 1898.0 B, free: 730.2 MB) -2016-05-24 22:18:18 INFO ContextCleaner:59 - Cleaned accumulator 114 -2016-05-24 22:18:18 INFO BlockManagerInfo:59 - Removed broadcast_214_piece0 on localhost:56717 in memory (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:18:19 INFO BlockManagerInfo:59 - Removed broadcast_213_piece0 on localhost:56717 in memory (size: 22.9 KB, free: 730.2 MB) -2016-05-24 22:18:19 INFO ContextCleaner:59 - Cleaned accumulator 113 -2016-05-24 22:18:19 INFO BlockManagerInfo:59 - Removed broadcast_212_piece0 on localhost:56717 in memory (size: 41.6 KB, free: 730.3 MB) -2016-05-24 22:18:19 INFO ContextCleaner:59 - Cleaned accumulator 112 -2016-05-24 22:18:19 INFO BlockManagerInfo:59 - Removed broadcast_211_piece0 on localhost:56717 in memory (size: 5.9 KB, free: 730.3 MB) -2016-05-24 22:18:19 INFO ContextCleaner:59 - Cleaned accumulator 111 -2016-05-24 22:18:19 INFO BlockManagerInfo:59 - Removed broadcast_210_piece0 on localhost:56717 in memory (size: 2.9 KB, free: 730.3 MB) -2016-05-24 22:18:19 INFO ContextCleaner:59 - Cleaned accumulator 110 -2016-05-24 22:18:19 INFO BlockManagerInfo:59 - Removed broadcast_209_piece0 on localhost:56717 in memory (size: 2.7 KB, free: 730.3 MB) -2016-05-24 22:18:19 INFO ContextCleaner:59 - Cleaned accumulator 109 -2016-05-24 22:18:19 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 22:18:19 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-24 22:18:19 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524221812327/result/data/part-r-00000-1c6d32c7-ff57-4483-8223-87756ce4734a.gz.parquet -2016-05-24 22:18:19 INFO SparkContext:59 - Starting job: take at GLMClassificationModel.scala:79 -2016-05-24 22:18:19 INFO DAGScheduler:59 - Got job 113 (take at GLMClassificationModel.scala:79) with 1 output partitions -2016-05-24 22:18:19 INFO DAGScheduler:59 - Final stage: ResultStage 120(take at GLMClassificationModel.scala:79) -2016-05-24 22:18:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:19 INFO DAGScheduler:59 - Submitting ResultStage 120 (MapPartitionsRDD[236] at take at GLMClassificationModel.scala:79), which has no missing parents -2016-05-24 22:18:19 INFO MemoryStore:59 - ensureFreeSpace(4168) called with curMem=570248, maxMem=766075207 -2016-05-24 22:18:19 INFO MemoryStore:59 - Block broadcast_218 stored as values in memory (estimated size 4.1 KB, free 730.0 MB) -2016-05-24 22:18:19 INFO MemoryStore:59 - ensureFreeSpace(2437) called with curMem=574416, maxMem=766075207 -2016-05-24 22:18:19 INFO MemoryStore:59 - Block broadcast_218_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.0 MB) -2016-05-24 22:18:19 INFO BlockManagerInfo:59 - Added broadcast_218_piece0 in memory on localhost:56717 (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:18:19 INFO SparkContext:59 - Created broadcast 218 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 120 (MapPartitionsRDD[236] at take at GLMClassificationModel.scala:79) -2016-05-24 22:18:19 INFO TaskSchedulerImpl:59 - Adding task set 120.0 with 1 tasks -2016-05-24 22:18:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 120.0 (TID 230, localhost, PROCESS_LOCAL, 2301 bytes) -2016-05-24 22:18:19 INFO Executor:59 - Running task 0.0 in stage 120.0 (TID 230) -2016-05-24 22:18:19 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524221812327/result/data/part-r-00000-1c6d32c7-ff57-4483-8223-87756ce4734a.gz.parquet start: 0 end: 5410 length: 5410 hosts: []} -2016-05-24 22:18:19 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-24 22:18:19 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group weights { - required int32 type (INT_8); - optional int32 size; - optional group indices (LIST) { - repeated int32 array; - } - optional group values (LIST) { - repeated double array; - } - } - optional double intercept; - optional double threshold; -} - -Catalyst form: -StructType(StructField(weights,org.apache.spark.mllib.linalg.VectorUDT@f71b0bce,true), StructField(intercept,DoubleType,true), StructField(threshold,DoubleType,true)) - -2016-05-24 22:18:20 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-24 22:18:20 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-24 22:18:20 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-24 22:18:32 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 22:18:34 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 22:18:34 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 22:18:34 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 22:18:35 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 22:18:36 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 22:18:36 INFO Remoting:74 - Starting remoting -2016-05-24 22:18:37 INFO Utils:59 - Successfully started service 'sparkDriver' on port 56739. -2016-05-24 22:18:37 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:56739] -2016-05-24 22:18:37 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 22:18:37 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 22:18:37 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-499bb4c6-dd8d-4328-bdf2-be6eefb3820c -2016-05-24 22:18:37 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 22:18:37 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-36ddcf14-eeac-463d-bfec-73c0383dfb96\httpd-f7b8c941-6b22-4d06-bd2e-be1e60267f27 -2016-05-24 22:18:37 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 22:18:38 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:18:38 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:56740 -2016-05-24 22:18:38 INFO Utils:59 - Successfully started service 'HTTP file server' on port 56740. -2016-05-24 22:18:38 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 22:18:38 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:18:38 WARN AbstractLifeCycle:204 - FAILED SelectChannelConnector@0.0.0.0:4040: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.mllib.learning.regression.SVMFromSparkLearning$.main(SVMFromSparkLearning.scala:26) - at org.apache.spark.mllib.learning.regression.SVMFromSparkLearning.main(SVMFromSparkLearning.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-05-24 22:18:38 WARN AbstractLifeCycle:204 - FAILED org.spark-project.jetty.server.Server@29822ba9: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.mllib.learning.regression.SVMFromSparkLearning$.main(SVMFromSparkLearning.scala:26) - at org.apache.spark.mllib.learning.regression.SVMFromSparkLearning.main(SVMFromSparkLearning.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 22:18:38 WARN Utils:71 - Service 'SparkUI' could not bind on port 4040. Attempting port 4041. -2016-05-24 22:18:38 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:18:38 WARN AbstractLifeCycle:204 - FAILED SelectChannelConnector@0.0.0.0:4041: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.mllib.learning.regression.SVMFromSparkLearning$.main(SVMFromSparkLearning.scala:26) - at org.apache.spark.mllib.learning.regression.SVMFromSparkLearning.main(SVMFromSparkLearning.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-05-24 22:18:38 WARN AbstractLifeCycle:204 - FAILED org.spark-project.jetty.server.Server@c6ee2d8: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.mllib.learning.regression.SVMFromSparkLearning$.main(SVMFromSparkLearning.scala:26) - at org.apache.spark.mllib.learning.regression.SVMFromSparkLearning.main(SVMFromSparkLearning.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 22:18:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 22:18:38 WARN Utils:71 - Service 'SparkUI' could not bind on port 4041. Attempting port 4042. -2016-05-24 22:18:38 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:18:38 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4042 -2016-05-24 22:18:38 INFO Utils:59 - Successfully started service 'SparkUI' on port 4042. -2016-05-24 22:18:38 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4042 -2016-05-24 22:18:39 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 22:18:39 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 22:18:41 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 56759. -2016-05-24 22:18:41 INFO NettyBlockTransferService:59 - Server created on 56759 -2016-05-24 22:18:41 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 22:18:41 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:56759 with 730.6 MB RAM, BlockManagerId(driver, localhost, 56759) -2016-05-24 22:18:41 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 22:18:43 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 22:18:43 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 22:18:43 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 22:18:43 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 22:18:43 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:56759 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 22:18:43 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 22:18:45 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 22:18:46 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:18:46 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 22:18:46 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 22:18:46 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 22:18:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:46 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 22:18:46 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 22:18:46 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 22:18:46 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=175476, maxMem=766075207 -2016-05-24 22:18:46 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:18:46 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:56759 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:18:46 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 22:18:46 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 22:18:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:18:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:18:46 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 22:18:46 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 22:18:46 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 22:18:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 22:18:46 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 22:18:46 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 22:18:46 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 22:18:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 22:18:46 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 22:18:46 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 22:18:46 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 22:18:47 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177621, maxMem=766075207 -2016-05-24 22:18:47 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 22:18:47 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262701, maxMem=766075207 -2016-05-24 22:18:47 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:56759 (size: 83.1 KB, free: 730.5 MB) -2016-05-24 22:18:47 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 22:18:47 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:56759 (size: 83.6 KB, free: 730.4 MB) -2016-05-24 22:18:47 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 22:18:47 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 22:18:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 948 ms on localhost (1/2) -2016-05-24 22:18:47 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 1.047 s -2016-05-24 22:18:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 1005 ms on localhost (2/2) -2016-05-24 22:18:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 22:18:47 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 1.291937 s -2016-05-24 22:18:47 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 22:18:47 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 22:18:47 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:18:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:47 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 22:18:47 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=348349, maxMem=766075207 -2016-05-24 22:18:47 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-24 22:18:47 INFO MemoryStore:59 - ensureFreeSpace(2483) called with curMem=352909, maxMem=766075207 -2016-05-24 22:18:47 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-24 22:18:47 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:56759 (size: 2.4 KB, free: 730.4 MB) -2016-05-24 22:18:47 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:47 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:18:47 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 22:18:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:18:47 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 22:18:47 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-05-24 22:18:47 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:18:47 INFO MemoryStore:59 - ensureFreeSpace(54392) called with curMem=355392, maxMem=766075207 -2016-05-24 22:18:47 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 53.1 KB, free 730.2 MB) -2016-05-24 22:18:47 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:56759 (size: 53.1 KB, free: 730.4 MB) -2016-05-24 22:18:47 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2611 bytes result sent to driver -2016-05-24 22:18:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 57 ms on localhost (1/1) -2016-05-24 22:18:47 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.084 s -2016-05-24 22:18:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 22:18:47 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.184414 s -2016-05-24 22:18:47 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 22:18:47 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 22:18:47 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 22:18:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:47 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 22:18:47 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=409784, maxMem=766075207 -2016-05-24 22:18:47 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-24 22:18:47 INFO MemoryStore:59 - ensureFreeSpace(2300) called with curMem=414032, maxMem=766075207 -2016-05-24 22:18:47 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 22:18:47 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:56759 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 22:18:47 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40) -2016-05-24 22:18:47 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 22:18:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:18:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:18:47 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 22:18:47 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 22:18:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:47 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-05-24 22:18:47 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 22:18:47 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:18:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 43 ms on localhost (1/2) -2016-05-24 22:18:47 INFO MemoryStore:59 - ensureFreeSpace(62184) called with curMem=416332, maxMem=766075207 -2016-05-24 22:18:47 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 60.7 KB, free 730.1 MB) -2016-05-24 22:18:47 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:56759 (size: 60.7 KB, free: 730.3 MB) -2016-05-24 22:18:47 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2662 bytes result sent to driver -2016-05-24 22:18:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 60 ms on localhost (2/2) -2016-05-24 22:18:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 22:18:47 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.068 s -2016-05-24 22:18:47 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.118977 s -2016-05-24 22:18:47 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 22:18:47 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 22:18:47 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 22:18:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:47 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 22:18:47 INFO MemoryStore:59 - ensureFreeSpace(4264) called with curMem=478516, maxMem=766075207 -2016-05-24 22:18:47 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.2 KB, free 730.1 MB) -2016-05-24 22:18:47 INFO MemoryStore:59 - ensureFreeSpace(2322) called with curMem=482780, maxMem=766075207 -2016-05-24 22:18:47 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 22:18:47 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:56759 (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:18:47 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 22:18:47 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 22:18:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:18:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:18:47 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 22:18:47 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 22:18:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:47 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 22:18:47 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 22:18:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 26 ms on localhost (1/2) -2016-05-24 22:18:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 28 ms on localhost (2/2) -2016-05-24 22:18:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 22:18:48 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.032 s -2016-05-24 22:18:48 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.129741 s -2016-05-24 22:18:48 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 22:18:48 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 22:18:48 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=485102, maxMem=766075207 -2016-05-24 22:18:48 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:18:48 INFO MemoryStore:59 - ensureFreeSpace(388) called with curMem=490694, maxMem=766075207 -2016-05-24 22:18:48 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 388.0 B, free 730.1 MB) -2016-05-24 22:18:48 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:56759 (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:18:48 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:48 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:48 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:48 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:48 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=491082, maxMem=766075207 -2016-05-24 22:18:48 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:18:48 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=502866, maxMem=766075207 -2016-05-24 22:18:48 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:18:48 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:49 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:49 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 22:18:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:49 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 22:18:49 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:49 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 7968 bytes result sent to driver -2016-05-24 22:18:49 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 22:18:49 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:49 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 7968 bytes result sent to driver -2016-05-24 22:18:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 56 ms on localhost (1/2) -2016-05-24 22:18:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 124 ms on localhost (2/2) -2016-05-24 22:18:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 22:18:49 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.127 s -2016-05-24 22:18:49 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.165690 s -2016-05-24 22:18:49 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=506348, maxMem=766075207 -2016-05-24 22:18:49 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:18:49 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=511940, maxMem=766075207 -2016-05-24 22:18:49 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-24 22:18:49 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:56759 (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:18:49 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:49 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:49 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:49 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:49 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:49 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=514178, maxMem=766075207 -2016-05-24 22:18:49 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:18:49 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=525962, maxMem=766075207 -2016-05-24 22:18:49 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:18:49 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:49 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:49 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 22:18:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:49 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 22:18:49 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 22:18:49 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:49 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 7968 bytes result sent to driver -2016-05-24 22:18:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 35 ms on localhost (1/2) -2016-05-24 22:18:49 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:49 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 7968 bytes result sent to driver -2016-05-24 22:18:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 52 ms on localhost (2/2) -2016-05-24 22:18:49 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.053 s -2016-05-24 22:18:49 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.092946 s -2016-05-24 22:18:49 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=529444, maxMem=766075207 -2016-05-24 22:18:49 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:18:49 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=535036, maxMem=766075207 -2016-05-24 22:18:49 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:18:49 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:49 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:49 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:49 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:49 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:49 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 22:18:49 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=538645, maxMem=766075207 -2016-05-24 22:18:49 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:18:49 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=550429, maxMem=766075207 -2016-05-24 22:18:49 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:18:49 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:49 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:49 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 22:18:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:49 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 22:18:49 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 22:18:49 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:49 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:49 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 7968 bytes result sent to driver -2016-05-24 22:18:49 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 7968 bytes result sent to driver -2016-05-24 22:18:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 21 ms on localhost (1/2) -2016-05-24 22:18:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 23 ms on localhost (2/2) -2016-05-24 22:18:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 22:18:49 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.027 s -2016-05-24 22:18:49 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.069589 s -2016-05-24 22:18:49 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=553912, maxMem=766075207 -2016-05-24 22:18:49 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:18:49 INFO MemoryStore:59 - ensureFreeSpace(3603) called with curMem=559504, maxMem=766075207 -2016-05-24 22:18:49 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:49 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:49 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:49 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:49 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:49 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:49 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:49 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=563107, maxMem=766075207 -2016-05-24 22:18:49 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:49 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=574891, maxMem=766075207 -2016-05-24 22:18:49 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:49 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:49 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:49 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 22:18:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:49 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 22:18:49 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:49 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 22:18:49 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 7968 bytes result sent to driver -2016-05-24 22:18:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 21 ms on localhost (1/2) -2016-05-24 22:18:49 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:49 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:49 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 7968 bytes result sent to driver -2016-05-24 22:18:49 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 22:18:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 33 ms on localhost (2/2) -2016-05-24 22:18:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 22:18:49 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.049 s -2016-05-24 22:18:49 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.117328 s -2016-05-24 22:18:49 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:49 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:49 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 22:18:49 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:56759 in memory (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:18:49 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=527327, maxMem=766075207 -2016-05-24 22:18:49 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:18:49 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:49 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 22:18:49 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=521135, maxMem=766075207 -2016-05-24 22:18:49 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:18:49 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:49 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:56759 in memory (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:18:49 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:49 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:49 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:49 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:49 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:49 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:56759 in memory (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:18:49 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 22:18:49 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=512175, maxMem=766075207 -2016-05-24 22:18:49 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:18:49 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=523959, maxMem=766075207 -2016-05-24 22:18:49 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:18:49 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:49 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:49 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 22:18:49 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:56759 in memory (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:18:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:49 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 22:18:49 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:49 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 7968 bytes result sent to driver -2016-05-24 22:18:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 26 ms on localhost (1/2) -2016-05-24 22:18:49 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 22:18:49 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 22:18:49 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:49 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 7968 bytes result sent to driver -2016-05-24 22:18:49 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:56759 in memory (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:18:49 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.077 s -2016-05-24 22:18:49 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.124100 s -2016-05-24 22:18:49 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 22:18:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 76 ms on localhost (2/2) -2016-05-24 22:18:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 22:18:50 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:56759 in memory (size: 2.1 KB, free: 730.3 MB) -2016-05-24 22:18:50 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=507809, maxMem=766075207 -2016-05-24 22:18:50 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:18:50 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=513401, maxMem=766075207 -2016-05-24 22:18:50 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:18:50 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:50 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:50 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:50 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:50 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:50 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:50 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=517007, maxMem=766075207 -2016-05-24 22:18:50 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:18:50 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=528791, maxMem=766075207 -2016-05-24 22:18:50 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:18:50 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:50 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:50 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 22:18:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:50 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 22:18:50 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 22:18:50 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:50 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:50 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 7968 bytes result sent to driver -2016-05-24 22:18:50 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 7968 bytes result sent to driver -2016-05-24 22:18:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 20 ms on localhost (1/2) -2016-05-24 22:18:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 22 ms on localhost (2/2) -2016-05-24 22:18:50 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 22:18:50 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.046108 s -2016-05-24 22:18:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 22:18:50 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=532274, maxMem=766075207 -2016-05-24 22:18:50 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:18:50 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=537866, maxMem=766075207 -2016-05-24 22:18:50 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:18:50 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:50 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:50 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:50 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:50 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:50 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:50 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=541472, maxMem=766075207 -2016-05-24 22:18:50 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:18:50 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=553256, maxMem=766075207 -2016-05-24 22:18:50 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:18:50 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:50 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:50 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 22:18:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:50 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 22:18:50 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:50 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 7968 bytes result sent to driver -2016-05-24 22:18:50 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 22:18:50 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:50 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 7968 bytes result sent to driver -2016-05-24 22:18:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 82 ms on localhost (1/2) -2016-05-24 22:18:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 84 ms on localhost (2/2) -2016-05-24 22:18:50 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.085 s -2016-05-24 22:18:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 22:18:50 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.185512 s -2016-05-24 22:18:50 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=556740, maxMem=766075207 -2016-05-24 22:18:50 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:50 INFO MemoryStore:59 - ensureFreeSpace(3604) called with curMem=562332, maxMem=766075207 -2016-05-24 22:18:50 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:50 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:50 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:50 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:50 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:50 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:50 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:50 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=565936, maxMem=766075207 -2016-05-24 22:18:50 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:50 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=577720, maxMem=766075207 -2016-05-24 22:18:50 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:50 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:50 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:50 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 22:18:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:50 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 22:18:50 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 22:18:50 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:50 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:50 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 7968 bytes result sent to driver -2016-05-24 22:18:50 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 7968 bytes result sent to driver -2016-05-24 22:18:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 18 ms on localhost (1/2) -2016-05-24 22:18:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 17 ms on localhost (2/2) -2016-05-24 22:18:50 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:18:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 22:18:50 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.081881 s -2016-05-24 22:18:50 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=581204, maxMem=766075207 -2016-05-24 22:18:50 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:50 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=586796, maxMem=766075207 -2016-05-24 22:18:50 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:50 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:50 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:50 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:50 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:50 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:50 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:50 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=590402, maxMem=766075207 -2016-05-24 22:18:50 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:50 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=602186, maxMem=766075207 -2016-05-24 22:18:50 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:50 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:50 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:50 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 22:18:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:50 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 22:18:50 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 22:18:50 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:50 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 7968 bytes result sent to driver -2016-05-24 22:18:50 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:50 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 7968 bytes result sent to driver -2016-05-24 22:18:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 20 ms on localhost (1/2) -2016-05-24 22:18:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 26 ms on localhost (2/2) -2016-05-24 22:18:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 22:18:50 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.030 s -2016-05-24 22:18:50 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.064280 s -2016-05-24 22:18:50 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=605670, maxMem=766075207 -2016-05-24 22:18:50 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:50 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=611262, maxMem=766075207 -2016-05-24 22:18:50 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:50 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:50 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:50 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:50 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:50 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:50 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:50 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=614868, maxMem=766075207 -2016-05-24 22:18:50 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:50 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=626652, maxMem=766075207 -2016-05-24 22:18:50 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:50 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:50 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:50 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 22:18:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:51 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 22:18:51 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 22:18:51 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:51 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:51 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 7968 bytes result sent to driver -2016-05-24 22:18:51 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 7968 bytes result sent to driver -2016-05-24 22:18:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 17 ms on localhost (1/2) -2016-05-24 22:18:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 20 ms on localhost (2/2) -2016-05-24 22:18:51 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.050 s -2016-05-24 22:18:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 22:18:51 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.084151 s -2016-05-24 22:18:51 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=630135, maxMem=766075207 -2016-05-24 22:18:51 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:51 INFO MemoryStore:59 - ensureFreeSpace(3610) called with curMem=635727, maxMem=766075207 -2016-05-24 22:18:51 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:51 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:51 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:51 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:51 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:51 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:51 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:51 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=639337, maxMem=766075207 -2016-05-24 22:18:51 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:51 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=651121, maxMem=766075207 -2016-05-24 22:18:51 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:51 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:51 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:51 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:51 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 22:18:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:51 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 22:18:51 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 22:18:51 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:51 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:51 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 7968 bytes result sent to driver -2016-05-24 22:18:51 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 7968 bytes result sent to driver -2016-05-24 22:18:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 25 ms on localhost (1/2) -2016-05-24 22:18:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 29 ms on localhost (2/2) -2016-05-24 22:18:51 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.050 s -2016-05-24 22:18:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 22:18:51 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.119045 s -2016-05-24 22:18:51 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=654605, maxMem=766075207 -2016-05-24 22:18:51 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:51 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=660197, maxMem=766075207 -2016-05-24 22:18:51 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:51 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:51 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:51 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:51 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:51 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:51 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:51 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=663803, maxMem=766075207 -2016-05-24 22:18:51 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:51 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=675587, maxMem=766075207 -2016-05-24 22:18:51 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:51 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:51 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:51 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:51 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 22:18:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:51 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 22:18:51 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 22:18:51 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:51 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:51 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 7968 bytes result sent to driver -2016-05-24 22:18:51 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 7968 bytes result sent to driver -2016-05-24 22:18:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 59 ms on localhost (1/2) -2016-05-24 22:18:51 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.064 s -2016-05-24 22:18:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 63 ms on localhost (2/2) -2016-05-24 22:18:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 22:18:51 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.183056 s -2016-05-24 22:18:51 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=679071, maxMem=766075207 -2016-05-24 22:18:51 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:51 INFO MemoryStore:59 - ensureFreeSpace(3610) called with curMem=684663, maxMem=766075207 -2016-05-24 22:18:51 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:51 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:51 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:51 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:51 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:51 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:51 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:51 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=688273, maxMem=766075207 -2016-05-24 22:18:51 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:51 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=700057, maxMem=766075207 -2016-05-24 22:18:51 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:51 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:51 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:51 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:51 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 22:18:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:51 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 22:18:51 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 22:18:51 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:51 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 7968 bytes result sent to driver -2016-05-24 22:18:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 81 ms on localhost (1/2) -2016-05-24 22:18:51 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:51 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 7968 bytes result sent to driver -2016-05-24 22:18:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 91 ms on localhost (2/2) -2016-05-24 22:18:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 22:18:51 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.096 s -2016-05-24 22:18:51 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.152595 s -2016-05-24 22:18:51 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=703540, maxMem=766075207 -2016-05-24 22:18:51 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:51 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=709132, maxMem=766075207 -2016-05-24 22:18:51 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:51 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:51 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:51 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:51 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:51 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:51 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:51 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=712738, maxMem=766075207 -2016-05-24 22:18:51 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:51 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=724522, maxMem=766075207 -2016-05-24 22:18:51 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:51 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:51 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:51 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:51 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 22:18:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:51 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 22:18:51 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 22:18:51 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:51 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:51 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 7968 bytes result sent to driver -2016-05-24 22:18:51 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 7968 bytes result sent to driver -2016-05-24 22:18:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 19 ms on localhost (1/2) -2016-05-24 22:18:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 27 ms on localhost (2/2) -2016-05-24 22:18:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 22:18:51 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 22:18:51 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.060171 s -2016-05-24 22:18:51 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=728005, maxMem=766075207 -2016-05-24 22:18:51 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:51 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=733597, maxMem=766075207 -2016-05-24 22:18:51 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:51 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:51 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:51 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:51 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:51 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:51 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:51 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=737203, maxMem=766075207 -2016-05-24 22:18:51 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:51 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=748987, maxMem=766075207 -2016-05-24 22:18:51 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:51 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:52 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:52 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 22:18:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:52 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 22:18:52 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:52 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 7968 bytes result sent to driver -2016-05-24 22:18:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 17 ms on localhost (1/2) -2016-05-24 22:18:52 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 22:18:52 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:52 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 7968 bytes result sent to driver -2016-05-24 22:18:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 31 ms on localhost (2/2) -2016-05-24 22:18:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 22:18:52 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.034 s -2016-05-24 22:18:52 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.290415 s -2016-05-24 22:18:52 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=752469, maxMem=766075207 -2016-05-24 22:18:52 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:52 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=758061, maxMem=766075207 -2016-05-24 22:18:52 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:52 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:52 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:52 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:52 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:52 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:52 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:52 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=761674, maxMem=766075207 -2016-05-24 22:18:52 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:52 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=773458, maxMem=766075207 -2016-05-24 22:18:52 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:52 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:52 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:52 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 22:18:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:52 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 22:18:52 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 22:18:52 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:52 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:52 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 7968 bytes result sent to driver -2016-05-24 22:18:52 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 7968 bytes result sent to driver -2016-05-24 22:18:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 30 ms on localhost (1/2) -2016-05-24 22:18:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 39 ms on localhost (2/2) -2016-05-24 22:18:52 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.091 s -2016-05-24 22:18:52 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.128778 s -2016-05-24 22:18:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 22:18:52 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=776941, maxMem=766075207 -2016-05-24 22:18:52 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:52 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=782533, maxMem=766075207 -2016-05-24 22:18:52 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:52 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:52 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:52 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:52 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:52 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:52 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:52 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=786139, maxMem=766075207 -2016-05-24 22:18:52 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:52 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=797923, maxMem=766075207 -2016-05-24 22:18:52 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:52 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:52 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:52 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 22:18:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:52 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 22:18:52 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:52 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 22:18:52 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 7968 bytes result sent to driver -2016-05-24 22:18:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 58 ms on localhost (1/2) -2016-05-24 22:18:52 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:52 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 7968 bytes result sent to driver -2016-05-24 22:18:52 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.084 s -2016-05-24 22:18:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 82 ms on localhost (2/2) -2016-05-24 22:18:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 22:18:52 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.180328 s -2016-05-24 22:18:52 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=801407, maxMem=766075207 -2016-05-24 22:18:52 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:52 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=806999, maxMem=766075207 -2016-05-24 22:18:52 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:52 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:52 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:52 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:52 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:52 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:52 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:52 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=810611, maxMem=766075207 -2016-05-24 22:18:52 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:52 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=822395, maxMem=766075207 -2016-05-24 22:18:52 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:52 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:52 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:52 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 22:18:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:52 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 22:18:52 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:52 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 7968 bytes result sent to driver -2016-05-24 22:18:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 28 ms on localhost (1/2) -2016-05-24 22:18:52 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 22:18:52 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:52 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 7968 bytes result sent to driver -2016-05-24 22:18:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 73 ms on localhost (2/2) -2016-05-24 22:18:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 22:18:52 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.106 s -2016-05-24 22:18:52 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.133645 s -2016-05-24 22:18:52 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=825878, maxMem=766075207 -2016-05-24 22:18:52 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:52 INFO MemoryStore:59 - ensureFreeSpace(3621) called with curMem=831470, maxMem=766075207 -2016-05-24 22:18:52 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:52 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:52 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:52 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:52 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:52 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:52 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:52 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=835091, maxMem=766075207 -2016-05-24 22:18:52 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:52 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=846875, maxMem=766075207 -2016-05-24 22:18:52 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:52 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:52 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:52 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 22:18:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:52 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 22:18:52 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:52 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 7968 bytes result sent to driver -2016-05-24 22:18:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 13 ms on localhost (1/2) -2016-05-24 22:18:52 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 22:18:52 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:52 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 7968 bytes result sent to driver -2016-05-24 22:18:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 57 ms on localhost (2/2) -2016-05-24 22:18:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 22:18:52 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.066 s -2016-05-24 22:18:52 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.111720 s -2016-05-24 22:18:52 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=850358, maxMem=766075207 -2016-05-24 22:18:52 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:52 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=855950, maxMem=766075207 -2016-05-24 22:18:52 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:52 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:52 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:52 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:53 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:53 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:53 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=859562, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=871346, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:53 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 22:18:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:53 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 22:18:53 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:53 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 7968 bytes result sent to driver -2016-05-24 22:18:53 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 22:18:53 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:53 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 7968 bytes result sent to driver -2016-05-24 22:18:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 38 ms on localhost (1/2) -2016-05-24 22:18:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 40 ms on localhost (2/2) -2016-05-24 22:18:53 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.047 s -2016-05-24 22:18:53 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.136292 s -2016-05-24 22:18:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=874829, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=880421, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:53 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:53 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:53 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:53 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=880550, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=880550, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:53 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 22:18:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:53 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 22:18:53 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 22:18:53 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:53 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:53 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 7968 bytes result sent to driver -2016-05-24 22:18:53 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 22:18:53 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 7968 bytes result sent to driver -2016-05-24 22:18:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 10 ms on localhost (1/2) -2016-05-24 22:18:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 12 ms on localhost (2/2) -2016-05-24 22:18:53 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:18:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 22:18:53 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.103002 s -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=874830, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=880422, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:53 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:53 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:53 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=819817, maxMem=766075207 -2016-05-24 22:18:53 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=827995, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:53 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 22:18:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:53 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 22:18:53 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 22:18:53 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:53 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:53 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 7968 bytes result sent to driver -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 7968 bytes result sent to driver -2016-05-24 22:18:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 69 ms on localhost (1/2) -2016-05-24 22:18:53 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 22:18:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 77 ms on localhost (2/2) -2016-05-24 22:18:53 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.079 s -2016-05-24 22:18:53 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.095223 s -2016-05-24 22:18:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=801414, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=803524, maxMem=766075207 -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:53 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 22:18:53 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:53 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:53 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:53 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=761691, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=769992, maxMem=766075207 -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:53 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:53 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 22:18:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:53 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 22:18:53 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 22:18:53 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 7968 bytes result sent to driver -2016-05-24 22:18:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 14 ms on localhost (1/2) -2016-05-24 22:18:53 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:53 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 7968 bytes result sent to driver -2016-05-24 22:18:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 19 ms on localhost (2/2) -2016-05-24 22:18:53 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 22:18:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 22:18:53 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 22:18:53 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.119450 s -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=728021, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(3634) called with curMem=733613, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:53 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:53 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:53 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=684828, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=696612, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:53 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:53 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 22:18:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:53 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 22:18:53 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 22:18:53 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:53 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:53 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 7968 bytes result sent to driver -2016-05-24 22:18:53 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 7968 bytes result sent to driver -2016-05-24 22:18:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 9 ms on localhost (1/2) -2016-05-24 22:18:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 10 ms on localhost (2/2) -2016-05-24 22:18:53 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:18:53 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.052127 s -2016-05-24 22:18:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=688309, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=693901, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:53 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:53 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:53 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=693907, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=705691, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:53 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 22:18:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:53 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:53 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 7968 bytes result sent to driver -2016-05-24 22:18:53 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 22:18:53 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 22:18:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 34 ms on localhost (1/2) -2016-05-24 22:18:53 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:53 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 7968 bytes result sent to driver -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 50 ms on localhost (2/2) -2016-05-24 22:18:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 22:18:53 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.051 s -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.140828 s -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=651167, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:53 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=644975, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:53 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:53 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:53 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:53 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:53 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=624124, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=632302, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:53 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:53 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 22:18:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:53 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 22:18:53 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:53 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 7968 bytes result sent to driver -2016-05-24 22:18:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 28 ms on localhost (1/2) -2016-05-24 22:18:53 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 22:18:53 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:53 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 7968 bytes result sent to driver -2016-05-24 22:18:53 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 22:18:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 38 ms on localhost (2/2) -2016-05-24 22:18:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 22:18:53 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.041 s -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:53 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.089225 s -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=605732, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=611324, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:53 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:53 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:53 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:53 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:53 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=614936, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:53 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=626720, maxMem=766075207 -2016-05-24 22:18:53 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:53 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:53 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:53 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 22:18:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:53 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 22:18:53 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:53 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 7968 bytes result sent to driver -2016-05-24 22:18:53 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 22:18:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 19 ms on localhost (1/2) -2016-05-24 22:18:53 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:53 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 7968 bytes result sent to driver -2016-05-24 22:18:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 28 ms on localhost (2/2) -2016-05-24 22:18:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 22:18:53 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.030 s -2016-05-24 22:18:54 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.067738 s -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=630202, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=635794, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:54 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:54 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:54 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:54 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:54 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=639400, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=651184, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:54 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:54 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:54 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 22:18:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:54 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 22:18:54 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 22:18:54 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:54 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:54 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 7968 bytes result sent to driver -2016-05-24 22:18:54 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 7968 bytes result sent to driver -2016-05-24 22:18:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 12 ms on localhost (1/2) -2016-05-24 22:18:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 15 ms on localhost (2/2) -2016-05-24 22:18:54 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:18:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 22:18:54 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.126502 s -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=654668, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(3623) called with curMem=660260, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:54 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:54 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:54 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:54 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:54 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=663883, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=675667, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:54 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:54 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:54 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 22:18:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:54 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 22:18:54 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 22:18:54 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:54 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:54 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 7968 bytes result sent to driver -2016-05-24 22:18:54 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 7968 bytes result sent to driver -2016-05-24 22:18:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 9 ms on localhost (1/2) -2016-05-24 22:18:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 11 ms on localhost (2/2) -2016-05-24 22:18:54 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:18:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 22:18:54 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.027708 s -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=679149, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=684741, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:54 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:54 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:54 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:54 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:54 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=688347, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=700131, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:54 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:54 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:54 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 22:18:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:54 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 22:18:54 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 22:18:54 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:54 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:54 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 7968 bytes result sent to driver -2016-05-24 22:18:54 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 7968 bytes result sent to driver -2016-05-24 22:18:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 9 ms on localhost (1/2) -2016-05-24 22:18:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 9 ms on localhost (2/2) -2016-05-24 22:18:54 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:18:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 22:18:54 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.022522 s -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=703615, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(3626) called with curMem=709207, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:54 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:54 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:54 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:54 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:54 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=712833, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=724617, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:54 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:54 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:54 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 22:18:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:54 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 22:18:54 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:54 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 7968 bytes result sent to driver -2016-05-24 22:18:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 11 ms on localhost (1/2) -2016-05-24 22:18:54 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 22:18:54 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:54 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 7968 bytes result sent to driver -2016-05-24 22:18:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 30 ms on localhost (2/2) -2016-05-24 22:18:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 22:18:54 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.060 s -2016-05-24 22:18:54 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.098673 s -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=728101, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=733693, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:54 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:54 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:54 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:54 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:54 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=737305, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=749089, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:54 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:54 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:54 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 22:18:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:54 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 22:18:54 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 22:18:54 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:54 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:54 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 7968 bytes result sent to driver -2016-05-24 22:18:54 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 7968 bytes result sent to driver -2016-05-24 22:18:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 8 ms on localhost (1/2) -2016-05-24 22:18:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 10 ms on localhost (2/2) -2016-05-24 22:18:54 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:18:54 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.024979 s -2016-05-24 22:18:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=752573, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=758165, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:54 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:54 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:54 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:54 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:54 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:54 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=761777, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:54 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=773561, maxMem=766075207 -2016-05-24 22:18:54 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:55 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:55 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:55 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 22:18:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:55 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 22:18:55 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 22:18:55 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:55 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:55 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 7968 bytes result sent to driver -2016-05-24 22:18:55 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 7968 bytes result sent to driver -2016-05-24 22:18:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 34 ms on localhost (1/2) -2016-05-24 22:18:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 36 ms on localhost (2/2) -2016-05-24 22:18:55 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.037 s -2016-05-24 22:18:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 22:18:55 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.755696 s -2016-05-24 22:18:55 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=777044, maxMem=766075207 -2016-05-24 22:18:55 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:55 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=782636, maxMem=766075207 -2016-05-24 22:18:55 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:55 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:55 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:55 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:55 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:55 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:55 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=786248, maxMem=766075207 -2016-05-24 22:18:55 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:55 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=798032, maxMem=766075207 -2016-05-24 22:18:55 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:55 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:55 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:55 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 22:18:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:55 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 22:18:55 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 22:18:55 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:55 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:55 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 7968 bytes result sent to driver -2016-05-24 22:18:55 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 7968 bytes result sent to driver -2016-05-24 22:18:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 10 ms on localhost (1/2) -2016-05-24 22:18:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 10 ms on localhost (2/2) -2016-05-24 22:18:55 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:18:55 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.028349 s -2016-05-24 22:18:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 22:18:55 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=801515, maxMem=766075207 -2016-05-24 22:18:55 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:55 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=807107, maxMem=766075207 -2016-05-24 22:18:55 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:55 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:55 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:55 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:55 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:55 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:55 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=810719, maxMem=766075207 -2016-05-24 22:18:55 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:55 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=822503, maxMem=766075207 -2016-05-24 22:18:55 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:55 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:55 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:55 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 22:18:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:55 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 22:18:55 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 22:18:55 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:55 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:55 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 7968 bytes result sent to driver -2016-05-24 22:18:55 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 7968 bytes result sent to driver -2016-05-24 22:18:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 9 ms on localhost (1/2) -2016-05-24 22:18:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 9 ms on localhost (2/2) -2016-05-24 22:18:55 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:18:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 22:18:55 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.025437 s -2016-05-24 22:18:55 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=825987, maxMem=766075207 -2016-05-24 22:18:55 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:55 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=831579, maxMem=766075207 -2016-05-24 22:18:55 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:55 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:55 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:55 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:56 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=835191, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=846975, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:56 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:56 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 24 ms on localhost (1/2) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 42 ms on localhost (2/2) -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 22:18:56 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.056 s -2016-05-24 22:18:56 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.676127 s -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=850458, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=856050, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:56 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=859662, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=871446, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 22:18:56 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:56 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 13 ms on localhost (1/2) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 13 ms on localhost (2/2) -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 22:18:56 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:18:56 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.027007 s -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=874929, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=880521, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:56 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=884135, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=895919, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 22:18:56 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 22:18:56 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:56 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 7 ms on localhost (1/2) -2016-05-24 22:18:56 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 9 ms on localhost (2/2) -2016-05-24 22:18:56 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 22:18:56 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.040856 s -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=856053, maxMem=766075207 -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=852569, maxMem=766075207 -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:56 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 22:18:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:56 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=810722, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=822506, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 22:18:56 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 22:18:56 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:56 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 12 ms on localhost (1/2) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 13 ms on localhost (2/2) -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 22:18:56 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:18:56 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.033003 s -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=786251, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:56 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=791843, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 22:18:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:56 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=767357, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=779141, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 22:18:56 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:56 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:56 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 18 ms on localhost (1/2) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 20 ms on localhost (2/2) -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 22:18:56 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:18:56 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.063898 s -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=749082, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=733675, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:56 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[93] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=718413, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=730197, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[93] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 22:18:56 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:56 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:56 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 16 ms on localhost (1/2) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 15 ms on localhost (2/2) -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 22:18:56 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.025 s -2016-05-24 22:18:56 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.050513 s -2016-05-24 22:18:56 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=712823, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=718415, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:56 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[95] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:56 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=673086, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=681258, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[95] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:56 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 22:18:56 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 13 ms on localhost (1/2) -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:56 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 18 ms on localhost (2/2) -2016-05-24 22:18:56 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 22:18:56 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.035022 s -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=651177, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=656769, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:56 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:56 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[97] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=635909, maxMem=766075207 -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=647693, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[97] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 22:18:56 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:56 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:56 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 26 ms on localhost (1/2) -2016-05-24 22:18:56 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.032 s -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 32 ms on localhost (2/2) -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:56 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.055260 s -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=626701, maxMem=766075207 -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3618) called with curMem=620509, maxMem=766075207 -2016-05-24 22:18:56 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 91 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Got job 47 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:56 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[99] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=614923, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=626707, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 47 (MapPartitionsRDD[99] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 2 tasks -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 93, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 94, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 94) -2016-05-24 22:18:56 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 93) -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:56 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 94). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 93). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 94) in 8 ms on localhost (1/2) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 93) in 10 ms on localhost (2/2) -2016-05-24 22:18:56 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-24 22:18:56 INFO DAGScheduler:59 - Job 47 finished: treeAggregate at GradientDescent.scala:225, took 0.024837 s -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=630190, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=635782, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 93 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Got job 48 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:56 INFO DAGScheduler:59 - Final stage: ResultStage 48(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[101] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=639396, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=651180, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 48 (MapPartitionsRDD[101] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 2 tasks -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 95, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 96, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 95) -2016-05-24 22:18:56 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 96) -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:56 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 96). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:56 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 95). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 96) in 9 ms on localhost (1/2) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 95) in 12 ms on localhost (2/2) -2016-05-24 22:18:56 INFO DAGScheduler:59 - ResultStage 48 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-24 22:18:56 INFO DAGScheduler:59 - Job 48 finished: treeAggregate at GradientDescent.scala:225, took 0.026118 s -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=654663, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3618) called with curMem=660255, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 95 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Got job 49 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:56 INFO DAGScheduler:59 - Final stage: ResultStage 49(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[103] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=663873, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=675657, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 49 (MapPartitionsRDD[103] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 2 tasks -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 97, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 98, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 97) -2016-05-24 22:18:56 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 98) -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:56 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 98). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 98) in 8 ms on localhost (1/2) -2016-05-24 22:18:56 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 97). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 97) in 12 ms on localhost (2/2) -2016-05-24 22:18:56 INFO DAGScheduler:59 - ResultStage 49 (treeAggregate at GradientDescent.scala:225) finished in 0.055 s -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-05-24 22:18:56 INFO DAGScheduler:59 - Job 49 finished: treeAggregate at GradientDescent.scala:225, took 0.071395 s -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=679140, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3622) called with curMem=684732, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 97 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Got job 50 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:56 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[105] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=688354, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=700138, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 50 (MapPartitionsRDD[105] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 2 tasks -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:56 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:56 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 11 ms on localhost (1/2) -2016-05-24 22:18:56 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-24 22:18:56 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:56 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 7968 bytes result sent to driver -2016-05-24 22:18:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 22 ms on localhost (2/2) -2016-05-24 22:18:56 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 22:18:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-24 22:18:56 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at GradientDescent.scala:225, took 0.043555 s -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=703620, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=709212, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:56 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:56 INFO SparkContext:59 - Created broadcast 99 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:56 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:56 INFO DAGScheduler:59 - Got job 51 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:56 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:56 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[107] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=712825, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:56 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=724609, maxMem=766075207 -2016-05-24 22:18:56 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:57 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:57 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 51 (MapPartitionsRDD[107] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:57 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-05-24 22:18:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:57 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-24 22:18:57 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-24 22:18:57 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:57 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 7968 bytes result sent to driver -2016-05-24 22:18:57 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:57 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 7968 bytes result sent to driver -2016-05-24 22:18:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 27 ms on localhost (1/2) -2016-05-24 22:18:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 25 ms on localhost (2/2) -2016-05-24 22:18:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-24 22:18:57 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at GradientDescent.scala:225) finished in 0.032 s -2016-05-24 22:18:57 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at GradientDescent.scala:225, took 0.312273 s -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=728092, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(3628) called with curMem=733684, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:57 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:57 INFO SparkContext:59 - Created broadcast 101 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:57 INFO DAGScheduler:59 - Got job 52 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:57 INFO DAGScheduler:59 - Final stage: ResultStage 52(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:57 INFO DAGScheduler:59 - Submitting ResultStage 52 (MapPartitionsRDD[109] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=737312, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=749096, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:18:57 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:57 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 52 (MapPartitionsRDD[109] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:57 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-05-24 22:18:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 103, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 104, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:57 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 103) -2016-05-24 22:18:57 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 104) -2016-05-24 22:18:57 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:57 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:57 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 103). 7968 bytes result sent to driver -2016-05-24 22:18:57 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 104). 7968 bytes result sent to driver -2016-05-24 22:18:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 103) in 7 ms on localhost (1/2) -2016-05-24 22:18:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 104) in 8 ms on localhost (2/2) -2016-05-24 22:18:57 INFO DAGScheduler:59 - ResultStage 52 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:18:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-24 22:18:57 INFO DAGScheduler:59 - Job 52 finished: treeAggregate at GradientDescent.scala:225, took 0.048349 s -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=752579, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=758171, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:18:57 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:57 INFO SparkContext:59 - Created broadcast 103 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:57 INFO DAGScheduler:59 - Got job 53 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:57 INFO DAGScheduler:59 - Final stage: ResultStage 53(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:57 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[111] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=761803, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=773587, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:57 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:57 INFO SparkContext:59 - Created broadcast 104 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[111] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:57 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-05-24 22:18:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 105, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 106, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:57 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 105) -2016-05-24 22:18:57 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 106) -2016-05-24 22:18:57 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:57 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 105). 7968 bytes result sent to driver -2016-05-24 22:18:57 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:57 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 106). 7968 bytes result sent to driver -2016-05-24 22:18:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 105) in 9 ms on localhost (1/2) -2016-05-24 22:18:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 106) in 11 ms on localhost (2/2) -2016-05-24 22:18:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-24 22:18:57 INFO DAGScheduler:59 - ResultStage 53 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:18:57 INFO DAGScheduler:59 - Job 53 finished: treeAggregate at GradientDescent.scala:225, took 0.025296 s -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=777068, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=782660, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:57 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:57 INFO SparkContext:59 - Created broadcast 105 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:57 INFO DAGScheduler:59 - Got job 54 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:57 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:57 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[113] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=786273, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=798057, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:57 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:57 INFO SparkContext:59 - Created broadcast 106 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 54 (MapPartitionsRDD[113] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:57 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 2 tasks -2016-05-24 22:18:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:57 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 107) -2016-05-24 22:18:57 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 108) -2016-05-24 22:18:57 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:57 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:57 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 107). 7968 bytes result sent to driver -2016-05-24 22:18:57 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 108). 7968 bytes result sent to driver -2016-05-24 22:18:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 107) in 10 ms on localhost (1/2) -2016-05-24 22:18:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 108) in 10 ms on localhost (2/2) -2016-05-24 22:18:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-24 22:18:57 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 22:18:57 INFO DAGScheduler:59 - Job 54 finished: treeAggregate at GradientDescent.scala:225, took 0.077730 s -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=801541, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=807133, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:57 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:57 INFO SparkContext:59 - Created broadcast 107 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:57 INFO DAGScheduler:59 - Got job 55 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:57 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:57 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[115] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=810765, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=822549, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:57 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:57 INFO SparkContext:59 - Created broadcast 108 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 55 (MapPartitionsRDD[115] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:57 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 2 tasks -2016-05-24 22:18:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 109, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 110, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:57 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 109) -2016-05-24 22:18:57 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 110) -2016-05-24 22:18:57 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:57 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:57 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 109). 7968 bytes result sent to driver -2016-05-24 22:18:57 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 110). 7968 bytes result sent to driver -2016-05-24 22:18:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 109) in 8 ms on localhost (1/2) -2016-05-24 22:18:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 110) in 8 ms on localhost (2/2) -2016-05-24 22:18:57 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:18:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-24 22:18:57 INFO DAGScheduler:59 - Job 55 finished: treeAggregate at GradientDescent.scala:225, took 0.020597 s -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=826030, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=831622, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:57 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:57 INFO SparkContext:59 - Created broadcast 109 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:57 INFO DAGScheduler:59 - Got job 56 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:57 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:57 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[117] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=835235, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=847019, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:57 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:57 INFO SparkContext:59 - Created broadcast 110 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 56 (MapPartitionsRDD[117] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:57 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 2 tasks -2016-05-24 22:18:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 111, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 112, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:57 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 111) -2016-05-24 22:18:57 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 112) -2016-05-24 22:18:57 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:57 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:57 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 111). 7968 bytes result sent to driver -2016-05-24 22:18:57 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 112). 7968 bytes result sent to driver -2016-05-24 22:18:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 111) in 10 ms on localhost (1/2) -2016-05-24 22:18:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 112) in 8 ms on localhost (2/2) -2016-05-24 22:18:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-24 22:18:57 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:18:57 INFO DAGScheduler:59 - Job 56 finished: treeAggregate at GradientDescent.scala:225, took 0.239554 s -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=850501, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=856093, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:57 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:57 INFO SparkContext:59 - Created broadcast 111 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:57 INFO DAGScheduler:59 - Got job 57 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:57 INFO DAGScheduler:59 - Final stage: ResultStage 57(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:57 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[119] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=859706, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=871490, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:57 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:57 INFO SparkContext:59 - Created broadcast 112 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 57 (MapPartitionsRDD[119] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:57 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 2 tasks -2016-05-24 22:18:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 113, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 114, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:57 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 113) -2016-05-24 22:18:57 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 114) -2016-05-24 22:18:57 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:57 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:57 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 113). 7968 bytes result sent to driver -2016-05-24 22:18:57 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 114). 7968 bytes result sent to driver -2016-05-24 22:18:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 113) in 9 ms on localhost (1/2) -2016-05-24 22:18:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 114) in 8 ms on localhost (2/2) -2016-05-24 22:18:57 INFO DAGScheduler:59 - ResultStage 57 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:18:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-24 22:18:57 INFO DAGScheduler:59 - Job 57 finished: treeAggregate at GradientDescent.scala:225, took 0.025993 s -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=874971, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=880563, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:57 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:57 INFO SparkContext:59 - Created broadcast 113 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:57 INFO DAGScheduler:59 - Got job 58 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:57 INFO DAGScheduler:59 - Final stage: ResultStage 58(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:57 INFO DAGScheduler:59 - Submitting ResultStage 58 (MapPartitionsRDD[121] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=884175, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=895959, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:18:57 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:57 INFO SparkContext:59 - Created broadcast 114 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 58 (MapPartitionsRDD[121] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:57 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 2 tasks -2016-05-24 22:18:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 115, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 116, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:57 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 115) -2016-05-24 22:18:57 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 116) -2016-05-24 22:18:57 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:57 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:57 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 115). 7968 bytes result sent to driver -2016-05-24 22:18:57 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 116). 7968 bytes result sent to driver -2016-05-24 22:18:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 115) in 8 ms on localhost (1/2) -2016-05-24 22:18:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 116) in 8 ms on localhost (2/2) -2016-05-24 22:18:57 INFO DAGScheduler:59 - ResultStage 58 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:18:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-05-24 22:18:57 INFO DAGScheduler:59 - Job 58 finished: treeAggregate at GradientDescent.scala:225, took 0.022728 s -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=899441, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_115 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(3620) called with curMem=905033, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_115_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:57 INFO BlockManagerInfo:59 - Added broadcast_115_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:57 INFO SparkContext:59 - Created broadcast 115 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:57 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:57 INFO DAGScheduler:59 - Got job 59 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:57 INFO DAGScheduler:59 - Final stage: ResultStage 59(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:57 INFO DAGScheduler:59 - Submitting ResultStage 59 (MapPartitionsRDD[123] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:57 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=908653, maxMem=766075207 -2016-05-24 22:18:57 INFO MemoryStore:59 - Block broadcast_116 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3478) called with curMem=920437, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_116_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_116_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 116 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 59 (MapPartitionsRDD[123] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 2 tasks -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 117, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 118, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 117) -2016-05-24 22:18:58 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 118) -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:58 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 117). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 118). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 117) in 6 ms on localhost (1/2) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 118) in 5 ms on localhost (2/2) -2016-05-24 22:18:58 INFO DAGScheduler:59 - ResultStage 59 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-05-24 22:18:58 INFO DAGScheduler:59 - Job 59 finished: treeAggregate at GradientDescent.scala:225, took 0.086910 s -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=923915, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_117 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=929507, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_117_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_117_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 117 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Got job 60 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:58 INFO DAGScheduler:59 - Final stage: ResultStage 60(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting ResultStage 60 (MapPartitionsRDD[125] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=933119, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_118 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=944903, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_118_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_118_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 118 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 60 (MapPartitionsRDD[125] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Adding task set 60.0 with 2 tasks -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 60.0 (TID 119, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 60.0 (TID 120, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO Executor:59 - Running task 0.0 in stage 60.0 (TID 119) -2016-05-24 22:18:58 INFO Executor:59 - Running task 1.0 in stage 60.0 (TID 120) -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:58 INFO Executor:59 - Finished task 1.0 in stage 60.0 (TID 120). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:58 INFO Executor:59 - Finished task 0.0 in stage 60.0 (TID 119). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 60.0 (TID 120) in 8 ms on localhost (1/2) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 60.0 (TID 119) in 9 ms on localhost (2/2) -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 60.0, whose tasks have all completed, from pool -2016-05-24 22:18:58 INFO DAGScheduler:59 - ResultStage 60 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:18:58 INFO DAGScheduler:59 - Job 60 finished: treeAggregate at GradientDescent.scala:225, took 0.022079 s -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=948386, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_119 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3616) called with curMem=953978, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_119_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_119_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 119 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Got job 61 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:58 INFO DAGScheduler:59 - Final stage: ResultStage 61(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting ResultStage 61 (MapPartitionsRDD[127] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=957594, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_120 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=969378, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_120_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_120_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 120 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 61 (MapPartitionsRDD[127] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 2 tasks -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 121, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 122, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 121) -2016-05-24 22:18:58 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 122) -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:58 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 121). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 121) in 15 ms on localhost (1/2) -2016-05-24 22:18:58 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 122). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 122) in 17 ms on localhost (2/2) -2016-05-24 22:18:58 INFO DAGScheduler:59 - ResultStage 61 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-05-24 22:18:58 INFO DAGScheduler:59 - Job 61 finished: treeAggregate at GradientDescent.scala:225, took 0.067311 s -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=972862, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_121 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=978454, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_121_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_121_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 121 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Got job 62 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:58 INFO DAGScheduler:59 - Final stage: ResultStage 62(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[129] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=982066, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_122 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=993850, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_122_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_122_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 122 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 62 (MapPartitionsRDD[129] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 2 tasks -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 123, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 62.0 (TID 124, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 123) -2016-05-24 22:18:58 INFO Executor:59 - Running task 1.0 in stage 62.0 (TID 124) -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:58 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 123). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO Executor:59 - Finished task 1.0 in stage 62.0 (TID 124). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 123) in 9 ms on localhost (1/2) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 62.0 (TID 124) in 9 ms on localhost (2/2) -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-05-24 22:18:58 INFO DAGScheduler:59 - ResultStage 62 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:18:58 INFO DAGScheduler:59 - Job 62 finished: treeAggregate at GradientDescent.scala:225, took 0.026478 s -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=997333, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_123 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3630) called with curMem=1002925, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_123_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_123_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 123 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Got job 63 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:58 INFO DAGScheduler:59 - Final stage: ResultStage 63(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting ResultStage 63 (MapPartitionsRDD[131] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1006555, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_124 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=1018339, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_124_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_124_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 124 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 63 (MapPartitionsRDD[131] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Adding task set 63.0 with 2 tasks -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 63.0 (TID 125, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 63.0 (TID 126, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO Executor:59 - Running task 0.0 in stage 63.0 (TID 125) -2016-05-24 22:18:58 INFO Executor:59 - Running task 1.0 in stage 63.0 (TID 126) -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:58 INFO Executor:59 - Finished task 1.0 in stage 63.0 (TID 126). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO Executor:59 - Finished task 0.0 in stage 63.0 (TID 125). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 63.0 (TID 125) in 8 ms on localhost (1/2) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 63.0 (TID 126) in 9 ms on localhost (2/2) -2016-05-24 22:18:58 INFO DAGScheduler:59 - ResultStage 63 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:18:58 INFO DAGScheduler:59 - Job 63 finished: treeAggregate at GradientDescent.scala:225, took 0.057748 s -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 63.0, whose tasks have all completed, from pool -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1021820, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_125 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1027412, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_125_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_125_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 125 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Got job 64 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:58 INFO DAGScheduler:59 - Final stage: ResultStage 64(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting ResultStage 64 (MapPartitionsRDD[133] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1031025, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_126 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1042809, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_126_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_126_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 126 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 64 (MapPartitionsRDD[133] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Adding task set 64.0 with 2 tasks -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 64.0 (TID 127, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 64.0 (TID 128, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO Executor:59 - Running task 1.0 in stage 64.0 (TID 128) -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:58 INFO Executor:59 - Finished task 1.0 in stage 64.0 (TID 128). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 64.0 (TID 128) in 13 ms on localhost (1/2) -2016-05-24 22:18:58 INFO Executor:59 - Running task 0.0 in stage 64.0 (TID 127) -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:58 INFO Executor:59 - Finished task 0.0 in stage 64.0 (TID 127). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 64.0 (TID 127) in 23 ms on localhost (2/2) -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 64.0, whose tasks have all completed, from pool -2016-05-24 22:18:58 INFO DAGScheduler:59 - ResultStage 64 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 22:18:58 INFO DAGScheduler:59 - Job 64 finished: treeAggregate at GradientDescent.scala:225, took 0.039174 s -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1046292, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_127 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1051884, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_127_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_127_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 127 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Got job 65 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:58 INFO DAGScheduler:59 - Final stage: ResultStage 65(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[135] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1055496, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_128 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1067280, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_128_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_128_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 128 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 65 (MapPartitionsRDD[135] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 2 tasks -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 129, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 130, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 129) -2016-05-24 22:18:58 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 130) -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:58 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 129). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 130). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 129) in 9 ms on localhost (1/2) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 130) in 9 ms on localhost (2/2) -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-05-24 22:18:58 INFO DAGScheduler:59 - ResultStage 65 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:18:58 INFO DAGScheduler:59 - Job 65 finished: treeAggregate at GradientDescent.scala:225, took 0.025016 s -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1070763, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_129 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1076355, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_129_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_129_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 129 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Got job 66 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:58 INFO DAGScheduler:59 - Final stage: ResultStage 66(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting ResultStage 66 (MapPartitionsRDD[137] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1079968, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_130 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1091752, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_130_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_130_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 130 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 66 (MapPartitionsRDD[137] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Adding task set 66.0 with 2 tasks -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 66.0 (TID 131, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 66.0 (TID 132, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO Executor:59 - Running task 1.0 in stage 66.0 (TID 132) -2016-05-24 22:18:58 INFO Executor:59 - Running task 0.0 in stage 66.0 (TID 131) -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:58 INFO Executor:59 - Finished task 0.0 in stage 66.0 (TID 131). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO Executor:59 - Finished task 1.0 in stage 66.0 (TID 132). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 66.0 (TID 132) in 15 ms on localhost (1/2) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 66.0 (TID 131) in 16 ms on localhost (2/2) -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 66.0, whose tasks have all completed, from pool -2016-05-24 22:18:58 INFO DAGScheduler:59 - ResultStage 66 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 22:18:58 INFO DAGScheduler:59 - Job 66 finished: treeAggregate at GradientDescent.scala:225, took 0.067144 s -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1095236, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_131 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3633) called with curMem=1100828, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_131_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_131_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 131 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Got job 67 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:58 INFO DAGScheduler:59 - Final stage: ResultStage 67(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting ResultStage 67 (MapPartitionsRDD[139] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1104461, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_132 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1116245, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_132_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_132_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 132 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 67 (MapPartitionsRDD[139] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Adding task set 67.0 with 2 tasks -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 67.0 (TID 133, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 67.0 (TID 134, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO Executor:59 - Running task 0.0 in stage 67.0 (TID 133) -2016-05-24 22:18:58 INFO Executor:59 - Running task 1.0 in stage 67.0 (TID 134) -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:58 INFO Executor:59 - Finished task 0.0 in stage 67.0 (TID 133). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO Executor:59 - Finished task 1.0 in stage 67.0 (TID 134). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 67.0 (TID 133) in 8 ms on localhost (1/2) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 67.0 (TID 134) in 8 ms on localhost (2/2) -2016-05-24 22:18:58 INFO DAGScheduler:59 - ResultStage 67 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 67.0, whose tasks have all completed, from pool -2016-05-24 22:18:58 INFO DAGScheduler:59 - Job 67 finished: treeAggregate at GradientDescent.scala:225, took 0.023403 s -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1119728, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_133 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1125320, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_133_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_133_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 133 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Got job 68 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:58 INFO DAGScheduler:59 - Final stage: ResultStage 68(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting ResultStage 68 (MapPartitionsRDD[141] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1128933, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_134 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1140717, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_134_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_134_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 134 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 68 (MapPartitionsRDD[141] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Adding task set 68.0 with 2 tasks -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 68.0 (TID 135, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 68.0 (TID 136, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO Executor:59 - Running task 1.0 in stage 68.0 (TID 136) -2016-05-24 22:18:58 INFO Executor:59 - Running task 0.0 in stage 68.0 (TID 135) -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:58 INFO Executor:59 - Finished task 1.0 in stage 68.0 (TID 136). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO Executor:59 - Finished task 0.0 in stage 68.0 (TID 135). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 68.0 (TID 136) in 6 ms on localhost (1/2) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 68.0 (TID 135) in 9 ms on localhost (2/2) -2016-05-24 22:18:58 INFO DAGScheduler:59 - ResultStage 68 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 68.0, whose tasks have all completed, from pool -2016-05-24 22:18:58 INFO DAGScheduler:59 - Job 68 finished: treeAggregate at GradientDescent.scala:225, took 0.022507 s -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1144201, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_135 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3608) called with curMem=1149793, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_135_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_135_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 135 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Got job 69 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:58 INFO DAGScheduler:59 - Final stage: ResultStage 69(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting ResultStage 69 (MapPartitionsRDD[143] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1153401, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_136 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1165185, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_136_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_136_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 136 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 69 (MapPartitionsRDD[143] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Adding task set 69.0 with 2 tasks -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 69.0 (TID 137, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 69.0 (TID 138, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO Executor:59 - Running task 0.0 in stage 69.0 (TID 137) -2016-05-24 22:18:58 INFO Executor:59 - Running task 1.0 in stage 69.0 (TID 138) -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:58 INFO Executor:59 - Finished task 1.0 in stage 69.0 (TID 138). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO Executor:59 - Finished task 0.0 in stage 69.0 (TID 137). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 69.0 (TID 138) in 5 ms on localhost (1/2) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 69.0 (TID 137) in 6 ms on localhost (2/2) -2016-05-24 22:18:58 INFO DAGScheduler:59 - ResultStage 69 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 69.0, whose tasks have all completed, from pool -2016-05-24 22:18:58 INFO DAGScheduler:59 - Job 69 finished: treeAggregate at GradientDescent.scala:225, took 0.019183 s -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1168669, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_137 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1174261, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_137_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_137_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 137 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Got job 70 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:58 INFO DAGScheduler:59 - Final stage: ResultStage 70(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting ResultStage 70 (MapPartitionsRDD[145] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1177867, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_138 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=1189651, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_138_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_138_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 138 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 70 (MapPartitionsRDD[145] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Adding task set 70.0 with 2 tasks -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 70.0 (TID 139, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 70.0 (TID 140, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO Executor:59 - Running task 0.0 in stage 70.0 (TID 139) -2016-05-24 22:18:58 INFO Executor:59 - Running task 1.0 in stage 70.0 (TID 140) -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:58 INFO Executor:59 - Finished task 0.0 in stage 70.0 (TID 139). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO Executor:59 - Finished task 1.0 in stage 70.0 (TID 140). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 70.0 (TID 139) in 7 ms on localhost (1/2) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 70.0 (TID 140) in 6 ms on localhost (2/2) -2016-05-24 22:18:58 INFO DAGScheduler:59 - ResultStage 70 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 70.0, whose tasks have all completed, from pool -2016-05-24 22:18:58 INFO DAGScheduler:59 - Job 70 finished: treeAggregate at GradientDescent.scala:225, took 0.018162 s -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1193132, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_139 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3619) called with curMem=1198724, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_139_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_139_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 139 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Got job 71 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:58 INFO DAGScheduler:59 - Final stage: ResultStage 71(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting ResultStage 71 (MapPartitionsRDD[147] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1202343, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_140 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1214127, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_140_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_140_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 140 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 71 (MapPartitionsRDD[147] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Adding task set 71.0 with 2 tasks -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 71.0 (TID 141, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 71.0 (TID 142, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:58 INFO Executor:59 - Running task 0.0 in stage 71.0 (TID 141) -2016-05-24 22:18:58 INFO Executor:59 - Running task 1.0 in stage 71.0 (TID 142) -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:58 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:58 INFO Executor:59 - Finished task 1.0 in stage 71.0 (TID 142). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO Executor:59 - Finished task 0.0 in stage 71.0 (TID 141). 7968 bytes result sent to driver -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 71.0 (TID 142) in 6 ms on localhost (1/2) -2016-05-24 22:18:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 71.0 (TID 141) in 7 ms on localhost (2/2) -2016-05-24 22:18:58 INFO DAGScheduler:59 - ResultStage 71 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:18:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 71.0, whose tasks have all completed, from pool -2016-05-24 22:18:58 INFO DAGScheduler:59 - Job 71 finished: treeAggregate at GradientDescent.scala:225, took 0.019588 s -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1217611, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_141 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:18:58 INFO MemoryStore:59 - ensureFreeSpace(3604) called with curMem=1223203, maxMem=766075207 -2016-05-24 22:18:58 INFO MemoryStore:59 - Block broadcast_141_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:18:58 INFO BlockManagerInfo:59 - Added broadcast_141_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:58 INFO SparkContext:59 - Created broadcast 141 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:58 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:58 INFO DAGScheduler:59 - Got job 72 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:58 INFO DAGScheduler:59 - Final stage: ResultStage 72(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting ResultStage 72 (MapPartitionsRDD[149] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1226807, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_142 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1238591, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_142_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_142_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 142 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 72 (MapPartitionsRDD[149] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Adding task set 72.0 with 2 tasks -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 72.0 (TID 143, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 72.0 (TID 144, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO Executor:59 - Running task 1.0 in stage 72.0 (TID 144) -2016-05-24 22:18:59 INFO Executor:59 - Running task 0.0 in stage 72.0 (TID 143) -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:59 INFO Executor:59 - Finished task 1.0 in stage 72.0 (TID 144). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO Executor:59 - Finished task 0.0 in stage 72.0 (TID 143). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 72.0 (TID 144) in 8 ms on localhost (1/2) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 72.0 (TID 143) in 10 ms on localhost (2/2) -2016-05-24 22:18:59 INFO DAGScheduler:59 - ResultStage 72 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 72.0, whose tasks have all completed, from pool -2016-05-24 22:18:59 INFO DAGScheduler:59 - Job 72 finished: treeAggregate at GradientDescent.scala:225, took 0.027190 s -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1242075, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_143 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=1247667, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_143_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_143_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 143 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Got job 73 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:59 INFO DAGScheduler:59 - Final stage: ResultStage 73(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting ResultStage 73 (MapPartitionsRDD[151] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1251276, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_144 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1263060, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_144_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_144_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 144 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 73 (MapPartitionsRDD[151] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Adding task set 73.0 with 2 tasks -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 73.0 (TID 145, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 73.0 (TID 146, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO Executor:59 - Running task 0.0 in stage 73.0 (TID 145) -2016-05-24 22:18:59 INFO Executor:59 - Running task 1.0 in stage 73.0 (TID 146) -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:59 INFO Executor:59 - Finished task 1.0 in stage 73.0 (TID 146). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO Executor:59 - Finished task 0.0 in stage 73.0 (TID 145). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 73.0 (TID 145) in 8 ms on localhost (1/2) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 73.0 (TID 146) in 8 ms on localhost (2/2) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 73.0, whose tasks have all completed, from pool -2016-05-24 22:18:59 INFO DAGScheduler:59 - ResultStage 73 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:18:59 INFO DAGScheduler:59 - Job 73 finished: treeAggregate at GradientDescent.scala:225, took 0.024471 s -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1266544, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_145 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1272136, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_145_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_145_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 145 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Got job 74 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:59 INFO DAGScheduler:59 - Final stage: ResultStage 74(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting ResultStage 74 (MapPartitionsRDD[153] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1275742, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_146 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1287526, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_146_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_146_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 146 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 74 (MapPartitionsRDD[153] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Adding task set 74.0 with 2 tasks -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 74.0 (TID 147, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 74.0 (TID 148, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO Executor:59 - Running task 1.0 in stage 74.0 (TID 148) -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:59 INFO Executor:59 - Finished task 1.0 in stage 74.0 (TID 148). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO Executor:59 - Running task 0.0 in stage 74.0 (TID 147) -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:59 INFO Executor:59 - Finished task 0.0 in stage 74.0 (TID 147). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 74.0 (TID 148) in 59 ms on localhost (1/2) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 74.0 (TID 147) in 61 ms on localhost (2/2) -2016-05-24 22:18:59 INFO DAGScheduler:59 - ResultStage 74 (treeAggregate at GradientDescent.scala:225) finished in 0.061 s -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 74.0, whose tasks have all completed, from pool -2016-05-24 22:18:59 INFO DAGScheduler:59 - Job 74 finished: treeAggregate at GradientDescent.scala:225, took 0.082125 s -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1291010, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_147 stored as values in memory (estimated size 5.5 KB, free 729.3 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3624) called with curMem=1296602, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_147_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.3 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_147_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 147 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Got job 75 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:59 INFO DAGScheduler:59 - Final stage: ResultStage 75(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting ResultStage 75 (MapPartitionsRDD[155] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1300226, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_148 stored as values in memory (estimated size 11.5 KB, free 729.3 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1312010, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_148_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.3 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_148_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 148 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 75 (MapPartitionsRDD[155] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Adding task set 75.0 with 2 tasks -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 75.0 (TID 149, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 75.0 (TID 150, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO Executor:59 - Running task 0.0 in stage 75.0 (TID 149) -2016-05-24 22:18:59 INFO Executor:59 - Running task 1.0 in stage 75.0 (TID 150) -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:59 INFO Executor:59 - Finished task 1.0 in stage 75.0 (TID 150). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 75.0 (TID 150) in 5 ms on localhost (1/2) -2016-05-24 22:18:59 INFO Executor:59 - Finished task 0.0 in stage 75.0 (TID 149). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 75.0 (TID 149) in 8 ms on localhost (2/2) -2016-05-24 22:18:59 INFO DAGScheduler:59 - ResultStage 75 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 75.0, whose tasks have all completed, from pool -2016-05-24 22:18:59 INFO DAGScheduler:59 - Job 75 finished: treeAggregate at GradientDescent.scala:225, took 0.020212 s -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1315493, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_149 stored as values in memory (estimated size 5.5 KB, free 729.3 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3626) called with curMem=1321085, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_149_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.3 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_149_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 149 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_148_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 76 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_147_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Got job 76 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:59 INFO DAGScheduler:59 - Final stage: ResultStage 76(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_146_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting ResultStage 76 (MapPartitionsRDD[157] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 75 -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1284960, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_150 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1287546, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_150_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_145_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_150_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 150 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 76 (MapPartitionsRDD[157] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 2 tasks -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 151, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 152, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 152) -2016-05-24 22:18:59 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 151) -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_144_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:59 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 151). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 152). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 151) in 8 ms on localhost (1/2) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 152) in 8 ms on localhost (2/2) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-05-24 22:18:59 INFO DAGScheduler:59 - ResultStage 76 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:18:59 INFO DAGScheduler:59 - Job 76 finished: treeAggregate at GradientDescent.scala:225, took 0.061421 s -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1275762, maxMem=766075207 -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 74 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_151 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3611) called with curMem=1277745, maxMem=766075207 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_143_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_151_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_151_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 151 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_142_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 73 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_141_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Got job 77 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:59 INFO DAGScheduler:59 - Final stage: ResultStage 77(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_140_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 72 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[159] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1232413, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_152 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_139_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1238605, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_152_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_152_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 152 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 77 (MapPartitionsRDD[159] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 2 tasks -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_138_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 153, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 71 -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 154, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 153) -2016-05-24 22:18:59 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 154) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_137_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:59 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 154). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 153). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 154) in 7 ms on localhost (1/2) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 153) in 9 ms on localhost (2/2) -2016-05-24 22:18:59 INFO DAGScheduler:59 - ResultStage 77 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-05-24 22:18:59 INFO DAGScheduler:59 - Job 77 finished: treeAggregate at GradientDescent.scala:225, took 0.023281 s -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_136_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 70 -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1202357, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_153 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1198749, maxMem=766075207 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_135_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_153_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_153_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_134_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 69 -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 153 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_133_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_132_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_131_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_130_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Got job 78 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:59 INFO DAGScheduler:59 - Final stage: ResultStage 78(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting ResultStage 78 (MapPartitionsRDD[161] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1134515, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_154 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_129_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1140707, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_154_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_154_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 154 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 78 (MapPartitionsRDD[161] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 2 tasks -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_128_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 155, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 156, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-05-24 22:18:59 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 155) -2016-05-24 22:18:59 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 156) -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_127_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 156). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 155). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 156) in 8 ms on localhost (1/2) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 155) in 8 ms on localhost (2/2) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-05-24 22:18:59 INFO DAGScheduler:59 - ResultStage 78 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:18:59 INFO DAGScheduler:59 - Job 78 finished: treeAggregate at GradientDescent.scala:225, took 0.029003 s -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1116236, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_155 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_126_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1110044, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_155_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_155_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 155 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_125_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_124_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_123_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Got job 79 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:59 INFO DAGScheduler:59 - Final stage: ResultStage 79(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_122_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting ResultStage 79 (MapPartitionsRDD[163] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1064697, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_156 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1076481, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_156_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_156_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 156 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 79 (MapPartitionsRDD[163] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Adding task set 79.0 with 2 tasks -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 79.0 (TID 157, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 79.0 (TID 158, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO Executor:59 - Running task 1.0 in stage 79.0 (TID 158) -2016-05-24 22:18:59 INFO Executor:59 - Running task 0.0 in stage 79.0 (TID 157) -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:59 INFO Executor:59 - Finished task 1.0 in stage 79.0 (TID 158). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_121_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:59 INFO Executor:59 - Finished task 0.0 in stage 79.0 (TID 157). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 79.0 (TID 158) in 25 ms on localhost (1/2) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 79.0 (TID 157) in 27 ms on localhost (2/2) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 79.0, whose tasks have all completed, from pool -2016-05-24 22:18:59 INFO DAGScheduler:59 - ResultStage 79 (treeAggregate at GradientDescent.scala:225) finished in 0.029 s -2016-05-24 22:18:59 INFO DAGScheduler:59 - Job 79 finished: treeAggregate at GradientDescent.scala:225, took 0.047509 s -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1058977, maxMem=766075207 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_120_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_157 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1061085, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_157_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_157_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 157 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Got job 80 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:59 INFO DAGScheduler:59 - Final stage: ResultStage 80(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting ResultStage 80 (MapPartitionsRDD[165] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1064697, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_158 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1076481, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_158_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_158_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 158 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 80 (MapPartitionsRDD[165] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Adding task set 80.0 with 2 tasks -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 80.0 (TID 159, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 80.0 (TID 160, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO Executor:59 - Running task 0.0 in stage 80.0 (TID 159) -2016-05-24 22:18:59 INFO Executor:59 - Running task 1.0 in stage 80.0 (TID 160) -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:59 INFO Executor:59 - Finished task 1.0 in stage 80.0 (TID 160). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO Executor:59 - Finished task 0.0 in stage 80.0 (TID 159). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 80.0 (TID 160) in 7 ms on localhost (1/2) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 80.0 (TID 159) in 9 ms on localhost (2/2) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 80.0, whose tasks have all completed, from pool -2016-05-24 22:18:59 INFO DAGScheduler:59 - ResultStage 80 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:18:59 INFO DAGScheduler:59 - Job 80 finished: treeAggregate at GradientDescent.scala:225, took 0.023748 s -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1079965, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_159 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1085557, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_159_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_159_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 159 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Got job 81 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:59 INFO DAGScheduler:59 - Final stage: ResultStage 81(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting ResultStage 81 (MapPartitionsRDD[167] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1089163, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_160 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1100947, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_160_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_160_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 160 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 81 (MapPartitionsRDD[167] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Adding task set 81.0 with 2 tasks -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 81.0 (TID 161, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 81.0 (TID 162, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO Executor:59 - Running task 0.0 in stage 81.0 (TID 161) -2016-05-24 22:18:59 INFO Executor:59 - Running task 1.0 in stage 81.0 (TID 162) -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:59 INFO Executor:59 - Finished task 0.0 in stage 81.0 (TID 161). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO Executor:59 - Finished task 1.0 in stage 81.0 (TID 162). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 81.0 (TID 162) in 39 ms on localhost (1/2) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-05-24 22:18:59 INFO DAGScheduler:59 - ResultStage 81 (treeAggregate at GradientDescent.scala:225) finished in 0.042 s -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 81.0 (TID 161) in 42 ms on localhost (2/2) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 81.0, whose tasks have all completed, from pool -2016-05-24 22:18:59 INFO DAGScheduler:59 - Job 81 finished: treeAggregate at GradientDescent.scala:225, took 0.056510 s -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1104430, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_161 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1110022, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_161_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_161_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 161 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_119_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Got job 82 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:59 INFO DAGScheduler:59 - Final stage: ResultStage 82(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting ResultStage 82 (MapPartitionsRDD[169] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1104426, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_162 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1116210, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_162_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_162_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 162 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 82 (MapPartitionsRDD[169] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Adding task set 82.0 with 2 tasks -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 82.0 (TID 163, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 82.0 (TID 164, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO Executor:59 - Running task 1.0 in stage 82.0 (TID 164) -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:59 INFO Executor:59 - Finished task 1.0 in stage 82.0 (TID 164). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 82.0 (TID 164) in 11 ms on localhost (1/2) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_118_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_117_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_116_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_115_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_114_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_113_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_112_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_111_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_110_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-05-24 22:18:59 INFO Executor:59 - Running task 0.0 in stage 82.0 (TID 163) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_109_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:59 INFO Executor:59 - Finished task 0.0 in stage 82.0 (TID 163). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 82.0 (TID 163) in 102 ms on localhost (2/2) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 82.0, whose tasks have all completed, from pool -2016-05-24 22:18:59 INFO DAGScheduler:59 - ResultStage 82 (treeAggregate at GradientDescent.scala:225) finished in 0.104 s -2016-05-24 22:18:59 INFO DAGScheduler:59 - Job 82 finished: treeAggregate at GradientDescent.scala:225, took 0.131986 s -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=985553, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_163 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_108_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=987664, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_163_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_163_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 163 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_107_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_106_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_105_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_104_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Got job 83 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:59 INFO DAGScheduler:59 - Final stage: ResultStage 83(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting ResultStage 83 (MapPartitionsRDD[171] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=933092, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_164 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_103_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=944876, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_164_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_102_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_164_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 164 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 83 (MapPartitionsRDD[171] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Adding task set 83.0 with 2 tasks -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 83.0 (TID 165, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 83.0 (TID 166, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO Executor:59 - Running task 0.0 in stage 83.0 (TID 165) -2016-05-24 22:18:59 INFO Executor:59 - Running task 1.0 in stage 83.0 (TID 166) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_101_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:59 INFO Executor:59 - Finished task 1.0 in stage 83.0 (TID 166). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO Executor:59 - Finished task 0.0 in stage 83.0 (TID 165). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 83.0 (TID 166) in 6 ms on localhost (1/2) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_100_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 83.0 (TID 165) in 7 ms on localhost (2/2) -2016-05-24 22:18:59 INFO DAGScheduler:59 - ResultStage 83 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 83.0, whose tasks have all completed, from pool -2016-05-24 22:18:59 INFO DAGScheduler:59 - Job 83 finished: treeAggregate at GradientDescent.scala:225, took 0.020208 s -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=904992, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_165 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_99_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=904992, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_165_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_165_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 165 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_98_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_97_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_96_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-05-24 22:18:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Got job 84 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:59 INFO DAGScheduler:59 - Final stage: ResultStage 84(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_95_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting ResultStage 84 (MapPartitionsRDD[173] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=856164, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_166 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_94_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=856164, maxMem=766075207 -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_166_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_166_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 166 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_93_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 84 (MapPartitionsRDD[173] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Adding task set 84.0 with 2 tasks -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 84.0 (TID 167, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 84.0 (TID 168, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO Executor:59 - Running task 1.0 in stage 84.0 (TID 168) -2016-05-24 22:18:59 INFO Executor:59 - Running task 0.0 in stage 84.0 (TID 167) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_92_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:18:59 INFO Executor:59 - Finished task 0.0 in stage 84.0 (TID 167). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_91_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO Executor:59 - Finished task 1.0 in stage 84.0 (TID 168). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 84.0 (TID 167) in 8 ms on localhost (1/2) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 84.0 (TID 168) in 8 ms on localhost (2/2) -2016-05-24 22:18:59 INFO DAGScheduler:59 - ResultStage 84 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 84.0, whose tasks have all completed, from pool -2016-05-24 22:18:59 INFO DAGScheduler:59 - Job 84 finished: treeAggregate at GradientDescent.scala:225, took 0.021795 s -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=810698, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_167 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_90_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3620) called with curMem=816290, maxMem=766075207 -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_167_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_167_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 167 from broadcast at GradientDescent.scala:221 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_89_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_88_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_87_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Got job 85 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Final stage: ResultStage 85(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:18:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting ResultStage 85 (MapPartitionsRDD[175] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=765374, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_168 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:18:59 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=773546, maxMem=766075207 -2016-05-24 22:18:59 INFO MemoryStore:59 - Block broadcast_168_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Added broadcast_168_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO SparkContext:59 - Created broadcast 168 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:18:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 85 (MapPartitionsRDD[175] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:18:59 INFO TaskSchedulerImpl:59 - Adding task set 85.0 with 2 tasks -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 85.0 (TID 169, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 85.0 (TID 170, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:18:59 INFO Executor:59 - Running task 0.0 in stage 85.0 (TID 169) -2016-05-24 22:18:59 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:18:59 INFO Executor:59 - Finished task 0.0 in stage 85.0 (TID 169). 7968 bytes result sent to driver -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:18:59 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO Executor:59 - Running task 1.0 in stage 85.0 (TID 170) -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 1.0 in stage 85.0 (TID 170). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 85.0 (TID 169) in 53 ms on localhost (1/2) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 85.0 (TID 170) in 52 ms on localhost (2/2) -2016-05-24 22:19:00 INFO DAGScheduler:59 - ResultStage 85 (treeAggregate at GradientDescent.scala:225) finished in 0.052 s -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 85.0, whose tasks have all completed, from pool -2016-05-24 22:19:00 INFO DAGScheduler:59 - Job 85 finished: treeAggregate at GradientDescent.scala:225, took 0.076402 s -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=703613, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_169 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3621) called with curMem=709205, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_169_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_169_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 169 from broadcast at GradientDescent.scala:221 -2016-05-24 22:19:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Got job 86 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 86(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting ResultStage 86 (MapPartitionsRDD[177] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=712826, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_170 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=724610, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_170_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_170_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 170 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 86 (MapPartitionsRDD[177] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Adding task set 86.0 with 2 tasks -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 86.0 (TID 171, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 86.0 (TID 172, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO Executor:59 - Running task 0.0 in stage 86.0 (TID 171) -2016-05-24 22:19:00 INFO Executor:59 - Running task 1.0 in stage 86.0 (TID 172) -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 1.0 in stage 86.0 (TID 172). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 86.0 (TID 172) in 12 ms on localhost (1/2) -2016-05-24 22:19:00 INFO Executor:59 - Finished task 0.0 in stage 86.0 (TID 171). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 86.0 (TID 171) in 15 ms on localhost (2/2) -2016-05-24 22:19:00 INFO DAGScheduler:59 - ResultStage 86 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 86.0, whose tasks have all completed, from pool -2016-05-24 22:19:00 INFO DAGScheduler:59 - Job 86 finished: treeAggregate at GradientDescent.scala:225, took 0.026056 s -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=728093, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_171 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=733685, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_171_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_171_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 171 from broadcast at GradientDescent.scala:221 -2016-05-24 22:19:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Got job 87 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 87(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting ResultStage 87 (MapPartitionsRDD[179] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=737297, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_172 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=749081, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_172_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_172_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 172 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 87 (MapPartitionsRDD[179] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Adding task set 87.0 with 2 tasks -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 87.0 (TID 173, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 87.0 (TID 174, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO Executor:59 - Running task 1.0 in stage 87.0 (TID 174) -2016-05-24 22:19:00 INFO Executor:59 - Running task 0.0 in stage 87.0 (TID 173) -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 0.0 in stage 87.0 (TID 173). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO Executor:59 - Finished task 1.0 in stage 87.0 (TID 174). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 87.0 (TID 173) in 6 ms on localhost (1/2) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 87.0 (TID 174) in 6 ms on localhost (2/2) -2016-05-24 22:19:00 INFO DAGScheduler:59 - ResultStage 87 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 87.0, whose tasks have all completed, from pool -2016-05-24 22:19:00 INFO DAGScheduler:59 - Job 87 finished: treeAggregate at GradientDescent.scala:225, took 0.018869 s -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=752564, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_173 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=758156, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_173_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_173_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 173 from broadcast at GradientDescent.scala:221 -2016-05-24 22:19:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Got job 88 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 88(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting ResultStage 88 (MapPartitionsRDD[181] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=761769, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_174 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=773553, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_174_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_174_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 174 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 88 (MapPartitionsRDD[181] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Adding task set 88.0 with 2 tasks -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 88.0 (TID 175, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 88.0 (TID 176, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO Executor:59 - Running task 0.0 in stage 88.0 (TID 175) -2016-05-24 22:19:00 INFO Executor:59 - Running task 1.0 in stage 88.0 (TID 176) -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 0.0 in stage 88.0 (TID 175). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO Executor:59 - Finished task 1.0 in stage 88.0 (TID 176). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 88.0 (TID 175) in 8 ms on localhost (1/2) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 88.0 (TID 176) in 7 ms on localhost (2/2) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 88.0, whose tasks have all completed, from pool -2016-05-24 22:19:00 INFO DAGScheduler:59 - ResultStage 88 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:19:00 INFO DAGScheduler:59 - Job 88 finished: treeAggregate at GradientDescent.scala:225, took 0.052091 s -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=777036, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_175 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3615) called with curMem=782628, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_175_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_175_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 175 from broadcast at GradientDescent.scala:221 -2016-05-24 22:19:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Got job 89 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 89(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting ResultStage 89 (MapPartitionsRDD[183] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=786243, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_176 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=798027, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_176_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_176_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 176 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 89 (MapPartitionsRDD[183] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Adding task set 89.0 with 2 tasks -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 89.0 (TID 177, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 89.0 (TID 178, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO Executor:59 - Running task 0.0 in stage 89.0 (TID 177) -2016-05-24 22:19:00 INFO Executor:59 - Running task 1.0 in stage 89.0 (TID 178) -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 0.0 in stage 89.0 (TID 177). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 89.0 (TID 177) in 6 ms on localhost (1/2) -2016-05-24 22:19:00 INFO Executor:59 - Finished task 1.0 in stage 89.0 (TID 178). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 89.0 (TID 178) in 6 ms on localhost (2/2) -2016-05-24 22:19:00 INFO DAGScheduler:59 - ResultStage 89 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 89.0, whose tasks have all completed, from pool -2016-05-24 22:19:00 INFO DAGScheduler:59 - Job 89 finished: treeAggregate at GradientDescent.scala:225, took 0.017824 s -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=801510, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_177 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=807102, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_177_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_177_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 177 from broadcast at GradientDescent.scala:221 -2016-05-24 22:19:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Got job 90 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 90(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting ResultStage 90 (MapPartitionsRDD[185] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=810714, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_178 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=822498, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_178_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_178_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 178 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 90 (MapPartitionsRDD[185] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Adding task set 90.0 with 2 tasks -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 90.0 (TID 179, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 90.0 (TID 180, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO Executor:59 - Running task 1.0 in stage 90.0 (TID 180) -2016-05-24 22:19:00 INFO Executor:59 - Running task 0.0 in stage 90.0 (TID 179) -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 1.0 in stage 90.0 (TID 180). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO Executor:59 - Finished task 0.0 in stage 90.0 (TID 179). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 90.0 (TID 180) in 5 ms on localhost (1/2) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 90.0 (TID 179) in 7 ms on localhost (2/2) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 90.0, whose tasks have all completed, from pool -2016-05-24 22:19:00 INFO DAGScheduler:59 - ResultStage 90 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:19:00 INFO DAGScheduler:59 - Job 90 finished: treeAggregate at GradientDescent.scala:225, took 0.018371 s -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=825981, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_179 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=831573, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_179_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_179_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 179 from broadcast at GradientDescent.scala:221 -2016-05-24 22:19:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Got job 91 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 91(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting ResultStage 91 (MapPartitionsRDD[187] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=835186, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_180 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=846970, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_180_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_180_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 180 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 91 (MapPartitionsRDD[187] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Adding task set 91.0 with 2 tasks -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 91.0 (TID 181, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 91.0 (TID 182, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO Executor:59 - Running task 1.0 in stage 91.0 (TID 182) -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 1.0 in stage 91.0 (TID 182). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 91.0 (TID 182) in 8 ms on localhost (1/2) -2016-05-24 22:19:00 INFO Executor:59 - Running task 0.0 in stage 91.0 (TID 181) -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 0.0 in stage 91.0 (TID 181). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 91.0 (TID 181) in 24 ms on localhost (2/2) -2016-05-24 22:19:00 INFO DAGScheduler:59 - ResultStage 91 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 91.0, whose tasks have all completed, from pool -2016-05-24 22:19:00 INFO DAGScheduler:59 - Job 91 finished: treeAggregate at GradientDescent.scala:225, took 0.037582 s -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=850454, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_181 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3608) called with curMem=856046, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_181_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_181_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 181 from broadcast at GradientDescent.scala:221 -2016-05-24 22:19:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Got job 92 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 92(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting ResultStage 92 (MapPartitionsRDD[189] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=859654, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_182 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=871438, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_182_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_182_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 182 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 92 (MapPartitionsRDD[189] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Adding task set 92.0 with 2 tasks -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 92.0 (TID 183, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 92.0 (TID 184, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO Executor:59 - Running task 0.0 in stage 92.0 (TID 183) -2016-05-24 22:19:00 INFO Executor:59 - Running task 1.0 in stage 92.0 (TID 184) -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 0.0 in stage 92.0 (TID 183). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO Executor:59 - Finished task 1.0 in stage 92.0 (TID 184). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 92.0 (TID 183) in 9 ms on localhost (1/2) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 92.0 (TID 184) in 9 ms on localhost (2/2) -2016-05-24 22:19:00 INFO DAGScheduler:59 - ResultStage 92 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 92.0, whose tasks have all completed, from pool -2016-05-24 22:19:00 INFO DAGScheduler:59 - Job 92 finished: treeAggregate at GradientDescent.scala:225, took 0.063772 s -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=874921, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_183 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=880513, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_183_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_183_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 183 from broadcast at GradientDescent.scala:221 -2016-05-24 22:19:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Got job 93 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 93(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting ResultStage 93 (MapPartitionsRDD[191] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=884127, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_184 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=895911, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_184_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_184_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 184 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 93 (MapPartitionsRDD[191] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Adding task set 93.0 with 2 tasks -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 93.0 (TID 185, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 93.0 (TID 186, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO Executor:59 - Running task 1.0 in stage 93.0 (TID 186) -2016-05-24 22:19:00 INFO Executor:59 - Running task 0.0 in stage 93.0 (TID 185) -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 1.0 in stage 93.0 (TID 186). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 93.0 (TID 186) in 9 ms on localhost (1/2) -2016-05-24 22:19:00 INFO Executor:59 - Finished task 0.0 in stage 93.0 (TID 185). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 93.0 (TID 185) in 12 ms on localhost (2/2) -2016-05-24 22:19:00 INFO DAGScheduler:59 - ResultStage 93 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 93.0, whose tasks have all completed, from pool -2016-05-24 22:19:00 INFO DAGScheduler:59 - Job 93 finished: treeAggregate at GradientDescent.scala:225, took 0.023967 s -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=899395, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_185 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=904987, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_185_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_185_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 185 from broadcast at GradientDescent.scala:221 -2016-05-24 22:19:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Got job 94 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 94(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting ResultStage 94 (MapPartitionsRDD[193] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=908599, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_186 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=920383, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_186_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_186_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 186 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 94 (MapPartitionsRDD[193] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Adding task set 94.0 with 2 tasks -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 94.0 (TID 187, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 94.0 (TID 188, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO Executor:59 - Running task 1.0 in stage 94.0 (TID 188) -2016-05-24 22:19:00 INFO Executor:59 - Running task 0.0 in stage 94.0 (TID 187) -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 1.0 in stage 94.0 (TID 188). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 0.0 in stage 94.0 (TID 187). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 94.0 (TID 188) in 17 ms on localhost (1/2) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 94.0 (TID 187) in 19 ms on localhost (2/2) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 94.0, whose tasks have all completed, from pool -2016-05-24 22:19:00 INFO DAGScheduler:59 - ResultStage 94 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:19:00 INFO DAGScheduler:59 - Job 94 finished: treeAggregate at GradientDescent.scala:225, took 0.079720 s -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=923867, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_187 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=929459, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_187_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_187_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 187 from broadcast at GradientDescent.scala:221 -2016-05-24 22:19:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Got job 95 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 95(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting ResultStage 95 (MapPartitionsRDD[195] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=933072, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_188 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=944856, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_188_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_188_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 188 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 95 (MapPartitionsRDD[195] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Adding task set 95.0 with 2 tasks -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 95.0 (TID 189, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 95.0 (TID 190, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO Executor:59 - Running task 1.0 in stage 95.0 (TID 190) -2016-05-24 22:19:00 INFO Executor:59 - Running task 0.0 in stage 95.0 (TID 189) -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 1.0 in stage 95.0 (TID 190). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO Executor:59 - Finished task 0.0 in stage 95.0 (TID 189). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 95.0 (TID 190) in 6 ms on localhost (1/2) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 95.0 (TID 189) in 7 ms on localhost (2/2) -2016-05-24 22:19:00 INFO DAGScheduler:59 - ResultStage 95 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 95.0, whose tasks have all completed, from pool -2016-05-24 22:19:00 INFO DAGScheduler:59 - Job 95 finished: treeAggregate at GradientDescent.scala:225, took 0.018192 s -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=948340, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_189 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3634) called with curMem=953932, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_189_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_189_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 189 from broadcast at GradientDescent.scala:221 -2016-05-24 22:19:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Got job 96 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 96(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting ResultStage 96 (MapPartitionsRDD[197] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=957566, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_190 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=969350, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_190_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_190_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 190 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 96 (MapPartitionsRDD[197] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Adding task set 96.0 with 2 tasks -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 96.0 (TID 191, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 96.0 (TID 192, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO Executor:59 - Running task 1.0 in stage 96.0 (TID 192) -2016-05-24 22:19:00 INFO Executor:59 - Running task 0.0 in stage 96.0 (TID 191) -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 1.0 in stage 96.0 (TID 192). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO Executor:59 - Finished task 0.0 in stage 96.0 (TID 191). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 96.0 (TID 192) in 5 ms on localhost (1/2) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 96.0 (TID 191) in 6 ms on localhost (2/2) -2016-05-24 22:19:00 INFO DAGScheduler:59 - ResultStage 96 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 96.0, whose tasks have all completed, from pool -2016-05-24 22:19:00 INFO DAGScheduler:59 - Job 96 finished: treeAggregate at GradientDescent.scala:225, took 0.016866 s -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=972833, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_191 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3619) called with curMem=978425, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_191_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_191_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 191 from broadcast at GradientDescent.scala:221 -2016-05-24 22:19:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Got job 97 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 97(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting ResultStage 97 (MapPartitionsRDD[199] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=982044, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_192 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=993828, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_192_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_192_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 192 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 97 (MapPartitionsRDD[199] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Adding task set 97.0 with 2 tasks -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 97.0 (TID 193, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 97.0 (TID 194, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO Executor:59 - Running task 0.0 in stage 97.0 (TID 193) -2016-05-24 22:19:00 INFO Executor:59 - Running task 1.0 in stage 97.0 (TID 194) -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 0.0 in stage 97.0 (TID 193). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO Executor:59 - Finished task 1.0 in stage 97.0 (TID 194). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 97.0 (TID 193) in 7 ms on localhost (1/2) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 97.0 (TID 194) in 6 ms on localhost (2/2) -2016-05-24 22:19:00 INFO DAGScheduler:59 - ResultStage 97 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 97.0, whose tasks have all completed, from pool -2016-05-24 22:19:00 INFO DAGScheduler:59 - Job 97 finished: treeAggregate at GradientDescent.scala:225, took 0.018771 s -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=997312, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_193 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1002904, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_193_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_193_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 193 from broadcast at GradientDescent.scala:221 -2016-05-24 22:19:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Got job 98 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 98(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting ResultStage 98 (MapPartitionsRDD[201] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1006516, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_194 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1018300, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_194_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_194_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 194 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 98 (MapPartitionsRDD[201] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Adding task set 98.0 with 2 tasks -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 98.0 (TID 195, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 98.0 (TID 196, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO Executor:59 - Running task 0.0 in stage 98.0 (TID 195) -2016-05-24 22:19:00 INFO Executor:59 - Running task 1.0 in stage 98.0 (TID 196) -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 1.0 in stage 98.0 (TID 196). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO Executor:59 - Finished task 0.0 in stage 98.0 (TID 195). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 98.0 (TID 196) in 5 ms on localhost (1/2) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 98.0 (TID 195) in 5 ms on localhost (2/2) -2016-05-24 22:19:00 INFO DAGScheduler:59 - ResultStage 98 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 98.0, whose tasks have all completed, from pool -2016-05-24 22:19:00 INFO DAGScheduler:59 - Job 98 finished: treeAggregate at GradientDescent.scala:225, took 0.014718 s -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1021783, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_195 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=1027375, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_195_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_195_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 195 from broadcast at GradientDescent.scala:221 -2016-05-24 22:19:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Got job 99 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 99(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting ResultStage 99 (MapPartitionsRDD[203] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1030984, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_196 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1042768, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_196_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_196_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 196 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 99 (MapPartitionsRDD[203] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Adding task set 99.0 with 2 tasks -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 99.0 (TID 197, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 99.0 (TID 198, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO Executor:59 - Running task 0.0 in stage 99.0 (TID 197) -2016-05-24 22:19:00 INFO Executor:59 - Running task 1.0 in stage 99.0 (TID 198) -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 1.0 in stage 99.0 (TID 198). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO Executor:59 - Finished task 0.0 in stage 99.0 (TID 197). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 99.0 (TID 198) in 4 ms on localhost (1/2) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 99.0 (TID 197) in 6 ms on localhost (2/2) -2016-05-24 22:19:00 INFO DAGScheduler:59 - ResultStage 99 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 99.0, whose tasks have all completed, from pool -2016-05-24 22:19:00 INFO DAGScheduler:59 - Job 99 finished: treeAggregate at GradientDescent.scala:225, took 0.014316 s -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1046252, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_197 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3616) called with curMem=1051844, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_197_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_197_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 197 from broadcast at GradientDescent.scala:221 -2016-05-24 22:19:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Got job 100 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 100(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting ResultStage 100 (MapPartitionsRDD[205] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1055460, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_198 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3485) called with curMem=1067244, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_198_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_198_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 198 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 100 (MapPartitionsRDD[205] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Adding task set 100.0 with 2 tasks -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 100.0 (TID 199, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 100.0 (TID 200, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO Executor:59 - Running task 0.0 in stage 100.0 (TID 199) -2016-05-24 22:19:00 INFO Executor:59 - Running task 1.0 in stage 100.0 (TID 200) -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 0.0 in stage 100.0 (TID 199). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO Executor:59 - Finished task 1.0 in stage 100.0 (TID 200). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 100.0 (TID 199) in 6 ms on localhost (1/2) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 100.0 (TID 200) in 7 ms on localhost (2/2) -2016-05-24 22:19:00 INFO DAGScheduler:59 - ResultStage 100 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 100.0, whose tasks have all completed, from pool -2016-05-24 22:19:00 INFO DAGScheduler:59 - Job 100 finished: treeAggregate at GradientDescent.scala:225, took 0.015558 s -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1070729, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_199 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1076321, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_199_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_199_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 199 from broadcast at GradientDescent.scala:221 -2016-05-24 22:19:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Got job 101 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 101(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting ResultStage 101 (MapPartitionsRDD[207] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1079927, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_200 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1091711, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_200_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_200_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 200 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 101 (MapPartitionsRDD[207] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Adding task set 101.0 with 2 tasks -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 101.0 (TID 201, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 101.0 (TID 202, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO Executor:59 - Running task 1.0 in stage 101.0 (TID 202) -2016-05-24 22:19:00 INFO Executor:59 - Running task 0.0 in stage 101.0 (TID 201) -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 1.0 in stage 101.0 (TID 202). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO Executor:59 - Finished task 0.0 in stage 101.0 (TID 201). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 101.0 (TID 202) in 46 ms on localhost (1/2) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 101.0 (TID 201) in 48 ms on localhost (2/2) -2016-05-24 22:19:00 INFO DAGScheduler:59 - ResultStage 101 (treeAggregate at GradientDescent.scala:225) finished in 0.048 s -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 101.0, whose tasks have all completed, from pool -2016-05-24 22:19:00 INFO DAGScheduler:59 - Job 101 finished: treeAggregate at GradientDescent.scala:225, took 0.062352 s -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1095194, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_201 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1100786, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_201_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_201_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 201 from broadcast at GradientDescent.scala:221 -2016-05-24 22:19:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Got job 102 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 102(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting ResultStage 102 (MapPartitionsRDD[209] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1104399, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_202 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1116183, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_202_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_202_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 202 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 102 (MapPartitionsRDD[209] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Adding task set 102.0 with 2 tasks -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 102.0 (TID 203, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 102.0 (TID 204, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO Executor:59 - Running task 0.0 in stage 102.0 (TID 203) -2016-05-24 22:19:00 INFO Executor:59 - Running task 1.0 in stage 102.0 (TID 204) -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 0.0 in stage 102.0 (TID 203). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 102.0 (TID 203) in 8 ms on localhost (1/2) -2016-05-24 22:19:00 INFO Executor:59 - Finished task 1.0 in stage 102.0 (TID 204). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 102.0 (TID 204) in 10 ms on localhost (2/2) -2016-05-24 22:19:00 INFO DAGScheduler:59 - ResultStage 102 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 102.0, whose tasks have all completed, from pool -2016-05-24 22:19:00 INFO DAGScheduler:59 - Job 102 finished: treeAggregate at GradientDescent.scala:225, took 0.020847 s -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1119667, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_203 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1125259, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_203_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_203_piece0 in memory on localhost:56759 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 203 from broadcast at GradientDescent.scala:221 -2016-05-24 22:19:00 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Got job 103 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 103(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting ResultStage 103 (MapPartitionsRDD[211] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1128871, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_204 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:19:00 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1140655, maxMem=766075207 -2016-05-24 22:19:00 INFO MemoryStore:59 - Block broadcast_204_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:19:00 INFO BlockManagerInfo:59 - Added broadcast_204_piece0 in memory on localhost:56759 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:19:00 INFO SparkContext:59 - Created broadcast 204 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 103 (MapPartitionsRDD[211] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Adding task set 103.0 with 2 tasks -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 103.0 (TID 205, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 103.0 (TID 206, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:19:00 INFO Executor:59 - Running task 1.0 in stage 103.0 (TID 206) -2016-05-24 22:19:00 INFO Executor:59 - Running task 0.0 in stage 103.0 (TID 205) -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:19:00 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:19:00 INFO Executor:59 - Finished task 0.0 in stage 103.0 (TID 205). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO Executor:59 - Finished task 1.0 in stage 103.0 (TID 206). 7968 bytes result sent to driver -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 103.0 (TID 205) in 6 ms on localhost (1/2) -2016-05-24 22:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 103.0 (TID 206) in 7 ms on localhost (2/2) -2016-05-24 22:19:00 INFO DAGScheduler:59 - ResultStage 103 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 103.0, whose tasks have all completed, from pool -2016-05-24 22:19:00 INFO DAGScheduler:59 - Job 103 finished: treeAggregate at GradientDescent.scala:225, took 0.016689 s -2016-05-24 22:19:00 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 3363.579936927662, 3356.5316534793847, 3349.536449573466, 3342.59343622967, 3335.701748404667, 3328.860544103912, 3322.069003535364, 3315.3263283026713, 3308.6317406356134, 3301.984482655661 -2016-05-24 22:19:00 INFO SparkContext:59 - Starting job: sortByKey at BinaryClassificationMetrics.scala:156 -2016-05-24 22:19:00 INFO DAGScheduler:59 - Registering RDD 212 (map at SVMFromSparkLearning.scala:44) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Got job 104 (sortByKey at BinaryClassificationMetrics.scala:156) with 2 output partitions -2016-05-24 22:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 105(sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 104) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 104) -2016-05-24 22:19:00 INFO DAGScheduler:59 - Submitting ShuffleMapStage 104 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44), which has no missing parents -2016-05-24 22:19:01 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=1144139, maxMem=766075207 -2016-05-24 22:19:01 INFO MemoryStore:59 - Block broadcast_205 stored as values in memory (estimated size 10.8 KB, free 729.5 MB) -2016-05-24 22:19:01 INFO MemoryStore:59 - ensureFreeSpace(6433) called with curMem=1155147, maxMem=766075207 -2016-05-24 22:19:01 INFO MemoryStore:59 - Block broadcast_205_piece0 stored as bytes in memory (estimated size 6.3 KB, free 729.5 MB) -2016-05-24 22:19:01 INFO BlockManagerInfo:59 - Added broadcast_205_piece0 in memory on localhost:56759 (size: 6.3 KB, free: 730.1 MB) -2016-05-24 22:19:01 INFO SparkContext:59 - Created broadcast 205 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 104 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44) -2016-05-24 22:19:01 INFO TaskSchedulerImpl:59 - Adding task set 104.0 with 2 tasks -2016-05-24 22:19:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 104.0 (TID 207, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:19:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 104.0 (TID 208, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:19:01 INFO Executor:59 - Running task 0.0 in stage 104.0 (TID 207) -2016-05-24 22:19:01 INFO Executor:59 - Running task 1.0 in stage 104.0 (TID 208) -2016-05-24 22:19:01 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:19:01 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:19:02 INFO Executor:59 - Finished task 0.0 in stage 104.0 (TID 207). 2254 bytes result sent to driver -2016-05-24 22:19:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 104.0 (TID 207) in 1010 ms on localhost (1/2) -2016-05-24 22:19:02 INFO Executor:59 - Finished task 1.0 in stage 104.0 (TID 208). 2254 bytes result sent to driver -2016-05-24 22:19:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 104.0 (TID 208) in 1071 ms on localhost (2/2) -2016-05-24 22:19:02 INFO DAGScheduler:59 - ShuffleMapStage 104 (map at SVMFromSparkLearning.scala:44) finished in 1.074 s -2016-05-24 22:19:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 104.0, whose tasks have all completed, from pool -2016-05-24 22:19:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:19:02 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:19:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 105) -2016-05-24 22:19:02 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:19:02 INFO DAGScheduler:59 - Missing parents for ResultStage 105: List() -2016-05-24 22:19:02 INFO DAGScheduler:59 - Submitting ResultStage 105 (MapPartitionsRDD[215] at sortByKey at BinaryClassificationMetrics.scala:156), which is now runnable -2016-05-24 22:19:02 INFO MemoryStore:59 - ensureFreeSpace(3344) called with curMem=1161580, maxMem=766075207 -2016-05-24 22:19:02 INFO MemoryStore:59 - Block broadcast_206 stored as values in memory (estimated size 3.3 KB, free 729.5 MB) -2016-05-24 22:19:02 INFO MemoryStore:59 - ensureFreeSpace(1900) called with curMem=1164924, maxMem=766075207 -2016-05-24 22:19:02 INFO MemoryStore:59 - Block broadcast_206_piece0 stored as bytes in memory (estimated size 1900.0 B, free 729.5 MB) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Added broadcast_206_piece0 in memory on localhost:56759 (size: 1900.0 B, free: 730.1 MB) -2016-05-24 22:19:02 INFO SparkContext:59 - Created broadcast 206 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 105 (MapPartitionsRDD[215] at sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:19:02 INFO TaskSchedulerImpl:59 - Adding task set 105.0 with 2 tasks -2016-05-24 22:19:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 105.0 (TID 209, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:19:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 105.0 (TID 210, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:19:02 INFO Executor:59 - Running task 0.0 in stage 105.0 (TID 209) -2016-05-24 22:19:02 INFO Executor:59 - Running task 1.0 in stage 105.0 (TID 210) -2016-05-24 22:19:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:19:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:19:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 8 ms -2016-05-24 22:19:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 8 ms -2016-05-24 22:19:02 INFO Executor:59 - Finished task 0.0 in stage 105.0 (TID 209). 1469 bytes result sent to driver -2016-05-24 22:19:02 INFO Executor:59 - Finished task 1.0 in stage 105.0 (TID 210). 1517 bytes result sent to driver -2016-05-24 22:19:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 105.0 (TID 210) in 72 ms on localhost (1/2) -2016-05-24 22:19:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 105.0 (TID 209) in 74 ms on localhost (2/2) -2016-05-24 22:19:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 105.0, whose tasks have all completed, from pool -2016-05-24 22:19:02 INFO DAGScheduler:59 - ResultStage 105 (sortByKey at BinaryClassificationMetrics.scala:156) finished in 0.076 s -2016-05-24 22:19:02 INFO DAGScheduler:59 - Job 104 finished: sortByKey at BinaryClassificationMetrics.scala:156, took 1.195968 s -2016-05-24 22:19:02 INFO SparkContext:59 - Starting job: collect at BinaryClassificationMetrics.scala:193 -2016-05-24 22:19:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 154 bytes -2016-05-24 22:19:02 INFO DAGScheduler:59 - Registering RDD 213 (combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:19:02 INFO DAGScheduler:59 - Got job 105 (collect at BinaryClassificationMetrics.scala:193) with 2 output partitions -2016-05-24 22:19:02 INFO DAGScheduler:59 - Final stage: ResultStage 108(collect at BinaryClassificationMetrics.scala:193) -2016-05-24 22:19:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 107) -2016-05-24 22:19:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 107) -2016-05-24 22:19:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 107 (ShuffledRDD[213] at combineByKey at BinaryClassificationMetrics.scala:152), which has no missing parents -2016-05-24 22:19:02 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=1166824, maxMem=766075207 -2016-05-24 22:19:02 INFO MemoryStore:59 - Block broadcast_207 stored as values in memory (estimated size 2.8 KB, free 729.5 MB) -2016-05-24 22:19:02 INFO MemoryStore:59 - ensureFreeSpace(1658) called with curMem=1169720, maxMem=766075207 -2016-05-24 22:19:02 INFO MemoryStore:59 - Block broadcast_207_piece0 stored as bytes in memory (estimated size 1658.0 B, free 729.5 MB) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Added broadcast_207_piece0 in memory on localhost:56759 (size: 1658.0 B, free: 730.1 MB) -2016-05-24 22:19:02 INFO SparkContext:59 - Created broadcast 207 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 107 (ShuffledRDD[213] at combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:19:02 INFO TaskSchedulerImpl:59 - Adding task set 107.0 with 2 tasks -2016-05-24 22:19:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 107.0 (TID 211, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:19:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 107.0 (TID 212, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:19:02 INFO Executor:59 - Running task 0.0 in stage 107.0 (TID 211) -2016-05-24 22:19:02 INFO Executor:59 - Running task 1.0 in stage 107.0 (TID 212) -2016-05-24 22:19:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:19:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:19:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:19:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:19:02 INFO Executor:59 - Finished task 1.0 in stage 107.0 (TID 212). 1375 bytes result sent to driver -2016-05-24 22:19:02 INFO Executor:59 - Finished task 0.0 in stage 107.0 (TID 211). 1375 bytes result sent to driver -2016-05-24 22:19:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 107.0 (TID 212) in 157 ms on localhost (1/2) -2016-05-24 22:19:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 107.0 (TID 211) in 175 ms on localhost (2/2) -2016-05-24 22:19:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 107.0, whose tasks have all completed, from pool -2016-05-24 22:19:02 INFO DAGScheduler:59 - ShuffleMapStage 107 (combineByKey at BinaryClassificationMetrics.scala:152) finished in 0.176 s -2016-05-24 22:19:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:19:02 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:19:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 108) -2016-05-24 22:19:02 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:19:02 INFO DAGScheduler:59 - Missing parents for ResultStage 108: List() -2016-05-24 22:19:02 INFO DAGScheduler:59 - Submitting ResultStage 108 (MapPartitionsRDD[218] at mapPartitions at BinaryClassificationMetrics.scala:189), which is now runnable -2016-05-24 22:19:02 INFO MemoryStore:59 - ensureFreeSpace(3472) called with curMem=1171378, maxMem=766075207 -2016-05-24 22:19:02 INFO MemoryStore:59 - Block broadcast_208 stored as values in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:19:02 INFO MemoryStore:59 - ensureFreeSpace(1981) called with curMem=1174850, maxMem=766075207 -2016-05-24 22:19:02 INFO MemoryStore:59 - Block broadcast_208_piece0 stored as bytes in memory (estimated size 1981.0 B, free 729.5 MB) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Added broadcast_208_piece0 in memory on localhost:56759 (size: 1981.0 B, free: 730.1 MB) -2016-05-24 22:19:02 INFO SparkContext:59 - Created broadcast 208 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 108 (MapPartitionsRDD[218] at mapPartitions at BinaryClassificationMetrics.scala:189) -2016-05-24 22:19:02 INFO TaskSchedulerImpl:59 - Adding task set 108.0 with 2 tasks -2016-05-24 22:19:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 108.0 (TID 213, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:19:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 108.0 (TID 214, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:19:02 INFO Executor:59 - Running task 1.0 in stage 108.0 (TID 214) -2016-05-24 22:19:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:19:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:19:02 INFO Executor:59 - Running task 0.0 in stage 108.0 (TID 213) -2016-05-24 22:19:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:19:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:19:02 INFO Executor:59 - Finished task 1.0 in stage 108.0 (TID 214). 1330 bytes result sent to driver -2016-05-24 22:19:02 INFO Executor:59 - Finished task 0.0 in stage 108.0 (TID 213). 1330 bytes result sent to driver -2016-05-24 22:19:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 108.0 (TID 214) in 35 ms on localhost (1/2) -2016-05-24 22:19:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 108.0 (TID 213) in 38 ms on localhost (2/2) -2016-05-24 22:19:02 INFO DAGScheduler:59 - ResultStage 108 (collect at BinaryClassificationMetrics.scala:193) finished in 0.038 s -2016-05-24 22:19:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 108.0, whose tasks have all completed, from pool -2016-05-24 22:19:02 INFO DAGScheduler:59 - Job 105 finished: collect at BinaryClassificationMetrics.scala:193, took 0.242118 s -2016-05-24 22:19:02 INFO BinaryClassificationMetrics:59 - Total counts: {numPos: 25, numNeg: 11} -2016-05-24 22:19:02 INFO SparkContext:59 - Starting job: runJob at SlidingRDD.scala:75 -2016-05-24 22:19:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 154 bytes -2016-05-24 22:19:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-24 22:19:02 INFO DAGScheduler:59 - Got job 106 (runJob at SlidingRDD.scala:75) with 3 output partitions -2016-05-24 22:19:02 INFO DAGScheduler:59 - Final stage: ResultStage 111(runJob at SlidingRDD.scala:75) -2016-05-24 22:19:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 110) -2016-05-24 22:19:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:02 INFO DAGScheduler:59 - Submitting ResultStage 111 (UnionRDD[224] at UnionRDD at BinaryClassificationMetrics.scala:92), which has no missing parents -2016-05-24 22:19:02 INFO MemoryStore:59 - ensureFreeSpace(5184) called with curMem=1176831, maxMem=766075207 -2016-05-24 22:19:02 INFO MemoryStore:59 - Block broadcast_209 stored as values in memory (estimated size 5.1 KB, free 729.5 MB) -2016-05-24 22:19:02 INFO MemoryStore:59 - ensureFreeSpace(2788) called with curMem=1182015, maxMem=766075207 -2016-05-24 22:19:02 INFO MemoryStore:59 - Block broadcast_209_piece0 stored as bytes in memory (estimated size 2.7 KB, free 729.5 MB) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Added broadcast_209_piece0 in memory on localhost:56759 (size: 2.7 KB, free: 730.1 MB) -2016-05-24 22:19:02 INFO SparkContext:59 - Created broadcast 209 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:02 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 111 (UnionRDD[224] at UnionRDD at BinaryClassificationMetrics.scala:92) -2016-05-24 22:19:02 INFO TaskSchedulerImpl:59 - Adding task set 111.0 with 3 tasks -2016-05-24 22:19:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 111.0 (TID 215, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:19:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 111.0 (TID 216, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:19:02 INFO TaskSetManager:59 - Starting task 2.0 in stage 111.0 (TID 217, localhost, PROCESS_LOCAL, 2318 bytes) -2016-05-24 22:19:02 INFO Executor:59 - Running task 1.0 in stage 111.0 (TID 216) -2016-05-24 22:19:02 INFO Executor:59 - Running task 0.0 in stage 111.0 (TID 215) -2016-05-24 22:19:02 INFO CacheManager:59 - Partition rdd_219_1 not found, computing it -2016-05-24 22:19:02 INFO CacheManager:59 - Partition rdd_219_0 not found, computing it -2016-05-24 22:19:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:19:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:19:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:19:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:19:02 INFO MemoryStore:59 - ensureFreeSpace(1616) called with curMem=1184803, maxMem=766075207 -2016-05-24 22:19:02 INFO MemoryStore:59 - Block rdd_219_1 stored as values in memory (estimated size 1616.0 B, free 729.5 MB) -2016-05-24 22:19:02 INFO MemoryStore:59 - ensureFreeSpace(1448) called with curMem=1186419, maxMem=766075207 -2016-05-24 22:19:02 INFO MemoryStore:59 - Block rdd_219_0 stored as values in memory (estimated size 1448.0 B, free 729.5 MB) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Added rdd_219_1 in memory on localhost:56759 (size: 1616.0 B, free: 730.1 MB) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Added rdd_219_0 in memory on localhost:56759 (size: 1448.0 B, free: 730.1 MB) -2016-05-24 22:19:02 INFO Executor:59 - Finished task 1.0 in stage 111.0 (TID 216). 1883 bytes result sent to driver -2016-05-24 22:19:02 INFO Executor:59 - Finished task 0.0 in stage 111.0 (TID 215). 1883 bytes result sent to driver -2016-05-24 22:19:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 111.0 (TID 216) in 28 ms on localhost (1/3) -2016-05-24 22:19:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 111.0 (TID 215) in 32 ms on localhost (2/3) -2016-05-24 22:19:02 INFO Executor:59 - Running task 2.0 in stage 111.0 (TID 217) -2016-05-24 22:19:02 INFO Executor:59 - Finished task 2.0 in stage 111.0 (TID 217). 1053 bytes result sent to driver -2016-05-24 22:19:02 INFO TaskSetManager:59 - Finished task 2.0 in stage 111.0 (TID 217) in 55 ms on localhost (3/3) -2016-05-24 22:19:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 111.0, whose tasks have all completed, from pool -2016-05-24 22:19:02 INFO DAGScheduler:59 - ResultStage 111 (runJob at SlidingRDD.scala:75) finished in 0.061 s -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_152_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:19:02 INFO DAGScheduler:59 - Job 106 finished: runJob at SlidingRDD.scala:75, took 0.152939 s -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_208_piece0 on localhost:56759 in memory (size: 1981.0 B, free: 730.1 MB) -2016-05-24 22:19:02 INFO ContextCleaner:59 - Cleaned accumulator 108 -2016-05-24 22:19:02 INFO SparkContext:59 - Starting job: aggregate at AreaUnderCurve.scala:45 -2016-05-24 22:19:02 INFO DAGScheduler:59 - Got job 107 (aggregate at AreaUnderCurve.scala:45) with 4 output partitions -2016-05-24 22:19:02 INFO DAGScheduler:59 - Final stage: ResultStage 114(aggregate at AreaUnderCurve.scala:45) -2016-05-24 22:19:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 113) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_207_piece0 on localhost:56759 in memory (size: 1658.0 B, free: 730.1 MB) -2016-05-24 22:19:02 INFO ContextCleaner:59 - Cleaned accumulator 107 -2016-05-24 22:19:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:02 INFO DAGScheduler:59 - Submitting ResultStage 114 (SlidingRDD[225] at RDD at SlidingRDD.scala:47), which has no missing parents -2016-05-24 22:19:02 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=1162593, maxMem=766075207 -2016-05-24 22:19:02 INFO MemoryStore:59 - Block broadcast_210 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:19:02 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=1168249, maxMem=766075207 -2016-05-24 22:19:02 INFO MemoryStore:59 - Block broadcast_210_piece0 stored as bytes in memory (estimated size 2.9 KB, free 729.5 MB) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_206_piece0 on localhost:56759 in memory (size: 1900.0 B, free: 730.1 MB) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Added broadcast_210_piece0 in memory on localhost:56759 (size: 2.9 KB, free: 730.1 MB) -2016-05-24 22:19:02 INFO ContextCleaner:59 - Cleaned accumulator 106 -2016-05-24 22:19:02 INFO SparkContext:59 - Created broadcast 210 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:02 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 114 (SlidingRDD[225] at RDD at SlidingRDD.scala:47) -2016-05-24 22:19:02 INFO TaskSchedulerImpl:59 - Adding task set 114.0 with 4 tasks -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_205_piece0 on localhost:56759 in memory (size: 6.3 KB, free: 730.1 MB) -2016-05-24 22:19:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 114.0 (TID 218, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:19:02 INFO TaskSetManager:59 - Starting task 2.0 in stage 114.0 (TID 219, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:19:02 INFO ContextCleaner:59 - Cleaned accumulator 105 -2016-05-24 22:19:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 114.0 (TID 220, localhost, PROCESS_LOCAL, 2635 bytes) -2016-05-24 22:19:02 INFO TaskSetManager:59 - Starting task 3.0 in stage 114.0 (TID 221, localhost, PROCESS_LOCAL, 2475 bytes) -2016-05-24 22:19:02 INFO Executor:59 - Running task 0.0 in stage 114.0 (TID 220) -2016-05-24 22:19:02 INFO Executor:59 - Running task 1.0 in stage 114.0 (TID 218) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_204_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:19:02 INFO Executor:59 - Running task 2.0 in stage 114.0 (TID 219) -2016-05-24 22:19:02 INFO BlockManager:59 - Found block rdd_219_0 locally -2016-05-24 22:19:02 INFO Executor:59 - Finished task 0.0 in stage 114.0 (TID 220). 955 bytes result sent to driver -2016-05-24 22:19:02 INFO Executor:59 - Running task 3.0 in stage 114.0 (TID 221) -2016-05-24 22:19:02 INFO ContextCleaner:59 - Cleaned accumulator 104 -2016-05-24 22:19:02 INFO Executor:59 - Finished task 3.0 in stage 114.0 (TID 221). 955 bytes result sent to driver -2016-05-24 22:19:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 114.0 (TID 220) in 13 ms on localhost (1/4) -2016-05-24 22:19:02 INFO TaskSetManager:59 - Finished task 3.0 in stage 114.0 (TID 221) in 13 ms on localhost (2/4) -2016-05-24 22:19:02 INFO Executor:59 - Finished task 1.0 in stage 114.0 (TID 218). 2084 bytes result sent to driver -2016-05-24 22:19:02 INFO BlockManager:59 - Found block rdd_219_1 locally -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_203_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:19:02 INFO Executor:59 - Finished task 2.0 in stage 114.0 (TID 219). 2084 bytes result sent to driver -2016-05-24 22:19:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 114.0 (TID 218) in 33 ms on localhost (3/4) -2016-05-24 22:19:02 INFO TaskSetManager:59 - Finished task 2.0 in stage 114.0 (TID 219) in 29 ms on localhost (4/4) -2016-05-24 22:19:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 114.0, whose tasks have all completed, from pool -2016-05-24 22:19:02 INFO DAGScheduler:59 - ResultStage 114 (aggregate at AreaUnderCurve.scala:45) finished in 0.034 s -2016-05-24 22:19:02 INFO DAGScheduler:59 - Job 107 finished: aggregate at AreaUnderCurve.scala:45, took 0.062386 s -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_202_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:19:02 INFO ContextCleaner:59 - Cleaned accumulator 103 -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_201_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_200_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:19:02 INFO ContextCleaner:59 - Cleaned accumulator 102 -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_199_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_198_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:19:02 INFO ContextCleaner:59 - Cleaned accumulator 101 -2016-05-24 22:19:02 INFO SparkContext:59 - Starting job: take at SVMFromSparkLearning.scala:55 -2016-05-24 22:19:02 INFO DAGScheduler:59 - Got job 108 (take at SVMFromSparkLearning.scala:55) with 1 output partitions -2016-05-24 22:19:02 INFO DAGScheduler:59 - Final stage: ResultStage 115(take at SVMFromSparkLearning.scala:55) -2016-05-24 22:19:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_197_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:19:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:02 INFO DAGScheduler:59 - Submitting ResultStage 115 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44), which has no missing parents -2016-05-24 22:19:02 INFO MemoryStore:59 - ensureFreeSpace(10312) called with curMem=1050648, maxMem=766075207 -2016-05-24 22:19:02 INFO MemoryStore:59 - Block broadcast_211 stored as values in memory (estimated size 10.1 KB, free 729.6 MB) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_196_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:19:02 INFO MemoryStore:59 - ensureFreeSpace(6082) called with curMem=1057476, maxMem=766075207 -2016-05-24 22:19:02 INFO MemoryStore:59 - Block broadcast_211_piece0 stored as bytes in memory (estimated size 5.9 KB, free 729.6 MB) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Added broadcast_211_piece0 in memory on localhost:56759 (size: 5.9 KB, free: 730.1 MB) -2016-05-24 22:19:02 INFO SparkContext:59 - Created broadcast 211 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:02 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 115 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44) -2016-05-24 22:19:02 INFO TaskSchedulerImpl:59 - Adding task set 115.0 with 1 tasks -2016-05-24 22:19:02 INFO ContextCleaner:59 - Cleaned accumulator 100 -2016-05-24 22:19:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 115.0 (TID 222, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:19:02 INFO Executor:59 - Running task 0.0 in stage 115.0 (TID 222) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_195_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:19:02 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:19:02 INFO Executor:59 - Finished task 0.0 in stage 115.0 (TID 222). 2401 bytes result sent to driver -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_194_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:19:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 115.0 (TID 222) in 8 ms on localhost (1/1) -2016-05-24 22:19:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 115.0, whose tasks have all completed, from pool -2016-05-24 22:19:02 INFO DAGScheduler:59 - ResultStage 115 (take at SVMFromSparkLearning.scala:55) finished in 0.031 s -2016-05-24 22:19:02 INFO DAGScheduler:59 - Job 108 finished: take at SVMFromSparkLearning.scala:55, took 0.049995 s -2016-05-24 22:19:02 INFO ContextCleaner:59 - Cleaned accumulator 99 -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_193_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_192_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:19:02 INFO ContextCleaner:59 - Cleaned accumulator 98 -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_191_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_190_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:19:02 INFO ContextCleaner:59 - Cleaned accumulator 97 -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_189_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_188_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:02 INFO ContextCleaner:59 - Cleaned accumulator 96 -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_187_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_186_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:02 INFO ContextCleaner:59 - Cleaned accumulator 95 -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_185_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_184_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:02 INFO ContextCleaner:59 - Cleaned accumulator 94 -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_183_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_182_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:02 INFO ContextCleaner:59 - Cleaned accumulator 93 -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_181_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:02 INFO BlockManagerInfo:59 - Removed broadcast_180_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO ContextCleaner:59 - Cleaned accumulator 92 -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_179_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_178_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO ContextCleaner:59 - Cleaned accumulator 91 -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_177_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_176_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO ContextCleaner:59 - Cleaned accumulator 90 -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_175_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_174_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO ContextCleaner:59 - Cleaned accumulator 89 -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_173_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_172_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO ContextCleaner:59 - Cleaned accumulator 88 -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_171_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_170_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO ContextCleaner:59 - Cleaned accumulator 87 -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_169_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_168_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO ContextCleaner:59 - Cleaned accumulator 86 -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_167_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_166_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO ContextCleaner:59 - Cleaned accumulator 85 -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_165_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_164_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO ContextCleaner:59 - Cleaned accumulator 84 -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_163_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_162_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO ContextCleaner:59 - Cleaned accumulator 83 -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_161_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_160_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:19:03 INFO ContextCleaner:59 - Cleaned accumulator 82 -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_159_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_158_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:19:03 INFO ContextCleaner:59 - Cleaned accumulator 81 -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_157_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_156_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:19:03 INFO ContextCleaner:59 - Cleaned accumulator 80 -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_155_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_154_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:19:03 INFO ContextCleaner:59 - Cleaned accumulator 79 -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_153_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:19:03 INFO ContextCleaner:59 - Cleaned accumulator 78 -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_151_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_150_piece0 on localhost:56759 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:19:03 INFO ContextCleaner:59 - Cleaned accumulator 77 -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Removed broadcast_149_piece0 on localhost:56759 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:19:03 INFO SparkContext:59 - Starting job: saveAsTextFile at GLMClassificationModel.scala:61 -2016-05-24 22:19:03 INFO DAGScheduler:59 - Got job 109 (saveAsTextFile at GLMClassificationModel.scala:61) with 1 output partitions -2016-05-24 22:19:03 INFO DAGScheduler:59 - Final stage: ResultStage 116(saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:19:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:03 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:03 INFO DAGScheduler:59 - Submitting ResultStage 116 (MapPartitionsRDD[227] at saveAsTextFile at GLMClassificationModel.scala:61), which has no missing parents -2016-05-24 22:19:03 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=494941, maxMem=766075207 -2016-05-24 22:19:03 INFO MemoryStore:59 - Block broadcast_212 stored as values in memory (estimated size 124.0 KB, free 730.0 MB) -2016-05-24 22:19:03 INFO MemoryStore:59 - ensureFreeSpace(42581) called with curMem=621933, maxMem=766075207 -2016-05-24 22:19:03 INFO MemoryStore:59 - Block broadcast_212_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.0 MB) -2016-05-24 22:19:03 INFO BlockManagerInfo:59 - Added broadcast_212_piece0 in memory on localhost:56759 (size: 41.6 KB, free: 730.2 MB) -2016-05-24 22:19:03 INFO SparkContext:59 - Created broadcast 212 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:03 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 116 (MapPartitionsRDD[227] at saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:19:03 INFO TaskSchedulerImpl:59 - Adding task set 116.0 with 1 tasks -2016-05-24 22:19:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 116.0 (TID 223, localhost, PROCESS_LOCAL, 2197 bytes) -2016-05-24 22:19:03 INFO Executor:59 - Running task 0.0 in stage 116.0 (TID 223) -2016-05-24 22:19:04 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242219_0116_m_000000_223' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524221902829/result/metadata/_temporary/0/task_201605242219_0116_m_000000 -2016-05-24 22:19:04 INFO SparkHadoopMapRedUtil:59 - attempt_201605242219_0116_m_000000_223: Committed -2016-05-24 22:19:04 INFO Executor:59 - Finished task 0.0 in stage 116.0 (TID 223). 1864 bytes result sent to driver -2016-05-24 22:19:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 116.0 (TID 223) in 584 ms on localhost (1/1) -2016-05-24 22:19:04 INFO DAGScheduler:59 - ResultStage 116 (saveAsTextFile at GLMClassificationModel.scala:61) finished in 0.585 s -2016-05-24 22:19:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 116.0, whose tasks have all completed, from pool -2016-05-24 22:19:04 INFO DAGScheduler:59 - Job 109 finished: saveAsTextFile at GLMClassificationModel.scala:61, took 0.703125 s -2016-05-24 22:19:07 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:19:07 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:19:07 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:65 -2016-05-24 22:19:07 INFO DAGScheduler:59 - Got job 110 (parquet at GLMClassificationModel.scala:65) with 1 output partitions -2016-05-24 22:19:07 INFO DAGScheduler:59 - Final stage: ResultStage 117(parquet at GLMClassificationModel.scala:65) -2016-05-24 22:19:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:07 INFO DAGScheduler:59 - Submitting ResultStage 117 (MapPartitionsRDD[229] at rddToDataFrameHolder at GLMClassificationModel.scala:65), which has no missing parents -2016-05-24 22:19:07 INFO MemoryStore:59 - ensureFreeSpace(67104) called with curMem=664514, maxMem=766075207 -2016-05-24 22:19:07 INFO MemoryStore:59 - Block broadcast_213 stored as values in memory (estimated size 65.5 KB, free 729.9 MB) -2016-05-24 22:19:07 INFO MemoryStore:59 - ensureFreeSpace(23443) called with curMem=731618, maxMem=766075207 -2016-05-24 22:19:07 INFO MemoryStore:59 - Block broadcast_213_piece0 stored as bytes in memory (estimated size 22.9 KB, free 729.9 MB) -2016-05-24 22:19:07 INFO BlockManagerInfo:59 - Added broadcast_213_piece0 in memory on localhost:56759 (size: 22.9 KB, free: 730.2 MB) -2016-05-24 22:19:07 INFO SparkContext:59 - Created broadcast 213 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:07 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 117 (MapPartitionsRDD[229] at rddToDataFrameHolder at GLMClassificationModel.scala:65) -2016-05-24 22:19:07 INFO TaskSchedulerImpl:59 - Adding task set 117.0 with 1 tasks -2016-05-24 22:19:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 117.0 (TID 224, localhost, PROCESS_LOCAL, 7978 bytes) -2016-05-24 22:19:07 INFO Executor:59 - Running task 0.0 in stage 117.0 (TID 224) -2016-05-24 22:19:07 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:19:07 INFO CodecConfig:151 - Compression: GZIP -2016-05-24 22:19:07 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-24 22:19:07 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-24 22:19:07 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-24 22:19:07 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-24 22:19:07 INFO ParquetOutputFormat:151 - Validation is off -2016-05-24 22:19:07 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-24 22:19:07 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-24 22:19:08 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 6,156 -2016-05-24 22:19:08 INFO ColumnChunkPageWriteStore:151 - written 61B for [weights, type] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN] -2016-05-24 22:19:08 INFO ColumnChunkPageWriteStore:151 - written 48B for [weights, size] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN] -2016-05-24 22:19:08 INFO ColumnChunkPageWriteStore:151 - written 51B for [weights, indices, array] INT32: 1 values, 13B raw, 30B comp, 1 pages, encodings: [RLE, PLAIN] -2016-05-24 22:19:08 INFO ColumnChunkPageWriteStore:151 - written 637B for [weights, values, array] DOUBLE: 692 values, 668B raw, 593B comp, 1 pages, encodings: [RLE, PLAIN_DICTIONARY], dic { 413 entries, 3,304B raw, 413B comp} -2016-05-24 22:19:08 INFO ColumnChunkPageWriteStore:151 - written 69B for [intercept] DOUBLE: 1 values, 14B raw, 28B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN] -2016-05-24 22:19:08 INFO ColumnChunkPageWriteStore:151 - written 47B for [threshold] DOUBLE: 1 values, 6B raw, 26B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN] -2016-05-24 22:19:09 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242219_0117_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524221902829/result/data/_temporary/0/task_201605242219_0117_m_000000 -2016-05-24 22:19:09 INFO SparkHadoopMapRedUtil:59 - attempt_201605242219_0117_m_000000_0: Committed -2016-05-24 22:19:09 INFO Executor:59 - Finished task 0.0 in stage 117.0 (TID 224). 915 bytes result sent to driver -2016-05-24 22:19:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 117.0 (TID 224) in 1520 ms on localhost (1/1) -2016-05-24 22:19:09 INFO DAGScheduler:59 - ResultStage 117 (parquet at GLMClassificationModel.scala:65) finished in 1.521 s -2016-05-24 22:19:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 117.0, whose tasks have all completed, from pool -2016-05-24 22:19:09 INFO DAGScheduler:59 - Job 110 finished: parquet at GLMClassificationModel.scala:65, took 1.577358 s -2016-05-24 22:19:09 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:19:09 INFO DefaultWriterContainer:59 - Job job_201605242219_0000 committed. -2016-05-24 22:19:09 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524221902829/result/data on driver -2016-05-24 22:19:09 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524221902829/result/data on driver -2016-05-24 22:19:09 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=755061, maxMem=766075207 -2016-05-24 22:19:09 INFO MemoryStore:59 - Block broadcast_214 stored as values in memory (estimated size 225.3 KB, free 729.6 MB) -2016-05-24 22:19:09 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=985765, maxMem=766075207 -2016-05-24 22:19:09 INFO MemoryStore:59 - Block broadcast_214_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.6 MB) -2016-05-24 22:19:10 INFO BlockManagerInfo:59 - Added broadcast_214_piece0 in memory on localhost:56759 (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:19:10 INFO SparkContext:59 - Created broadcast 214 from textFile at modelSaveLoad.scala:129 -2016-05-24 22:19:10 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:19:10 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-24 22:19:10 INFO DAGScheduler:59 - Got job 111 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-24 22:19:10 INFO DAGScheduler:59 - Final stage: ResultStage 118(first at modelSaveLoad.scala:129) -2016-05-24 22:19:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:10 INFO DAGScheduler:59 - Submitting ResultStage 118 (MapPartitionsRDD[232] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-24 22:19:10 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=1005553, maxMem=766075207 -2016-05-24 22:19:10 INFO MemoryStore:59 - Block broadcast_215 stored as values in memory (estimated size 3.1 KB, free 729.6 MB) -2016-05-24 22:19:10 INFO MemoryStore:59 - ensureFreeSpace(1898) called with curMem=1008769, maxMem=766075207 -2016-05-24 22:19:10 INFO MemoryStore:59 - Block broadcast_215_piece0 stored as bytes in memory (estimated size 1898.0 B, free 729.6 MB) -2016-05-24 22:19:10 INFO BlockManagerInfo:59 - Added broadcast_215_piece0 in memory on localhost:56759 (size: 1898.0 B, free: 730.2 MB) -2016-05-24 22:19:10 INFO SparkContext:59 - Created broadcast 215 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:10 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 118 (MapPartitionsRDD[232] at textFile at modelSaveLoad.scala:129) -2016-05-24 22:19:10 INFO TaskSchedulerImpl:59 - Adding task set 118.0 with 1 tasks -2016-05-24 22:19:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 118.0 (TID 225, localhost, PROCESS_LOCAL, 2238 bytes) -2016-05-24 22:19:10 INFO Executor:59 - Running task 0.0 in stage 118.0 (TID 225) -2016-05-24 22:19:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524221902829/result/metadata/part-00000:0+54 -2016-05-24 22:19:10 INFO Executor:59 - Finished task 0.0 in stage 118.0 (TID 225). 2154 bytes result sent to driver -2016-05-24 22:19:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 118.0 (TID 225) in 9 ms on localhost (1/1) -2016-05-24 22:19:10 INFO DAGScheduler:59 - ResultStage 118 (first at modelSaveLoad.scala:129) finished in 0.009 s -2016-05-24 22:19:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 118.0, whose tasks have all completed, from pool -2016-05-24 22:19:10 INFO DAGScheduler:59 - Job 111 finished: first at modelSaveLoad.scala:129, took 0.016416 s -2016-05-24 22:19:10 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524221902829/result/data on driver -2016-05-24 22:19:10 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:78 -2016-05-24 22:19:10 INFO DAGScheduler:59 - Got job 112 (parquet at GLMClassificationModel.scala:78) with 4 output partitions -2016-05-24 22:19:10 INFO DAGScheduler:59 - Final stage: ResultStage 119(parquet at GLMClassificationModel.scala:78) -2016-05-24 22:19:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:10 INFO DAGScheduler:59 - Submitting ResultStage 119 (MapPartitionsRDD[234] at parquet at GLMClassificationModel.scala:78), which has no missing parents -2016-05-24 22:19:10 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=1010667, maxMem=766075207 -2016-05-24 22:19:10 INFO MemoryStore:59 - Block broadcast_216 stored as values in memory (estimated size 61.5 KB, free 729.6 MB) -2016-05-24 22:19:10 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=1073667, maxMem=766075207 -2016-05-24 22:19:10 INFO MemoryStore:59 - Block broadcast_216_piece0 stored as bytes in memory (estimated size 20.6 KB, free 729.5 MB) -2016-05-24 22:19:10 INFO BlockManagerInfo:59 - Added broadcast_216_piece0 in memory on localhost:56759 (size: 20.6 KB, free: 730.2 MB) -2016-05-24 22:19:10 INFO SparkContext:59 - Created broadcast 216 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:10 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 119 (MapPartitionsRDD[234] at parquet at GLMClassificationModel.scala:78) -2016-05-24 22:19:10 INFO TaskSchedulerImpl:59 - Adding task set 119.0 with 4 tasks -2016-05-24 22:19:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 119.0 (TID 226, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:19:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 119.0 (TID 227, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:19:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 119.0 (TID 228, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:19:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 119.0 (TID 229, localhost, PROCESS_LOCAL, 2279 bytes) -2016-05-24 22:19:10 INFO Executor:59 - Running task 2.0 in stage 119.0 (TID 228) -2016-05-24 22:19:10 INFO Executor:59 - Running task 3.0 in stage 119.0 (TID 229) -2016-05-24 22:19:10 INFO Executor:59 - Running task 0.0 in stage 119.0 (TID 226) -2016-05-24 22:19:10 INFO Executor:59 - Running task 1.0 in stage 119.0 (TID 227) -2016-05-24 22:19:10 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:19:10 INFO Executor:59 - Finished task 2.0 in stage 119.0 (TID 228). 936 bytes result sent to driver -2016-05-24 22:19:10 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:19:10 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:19:10 INFO Executor:59 - Finished task 0.0 in stage 119.0 (TID 226). 936 bytes result sent to driver -2016-05-24 22:19:10 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:19:10 INFO Executor:59 - Finished task 1.0 in stage 119.0 (TID 227). 936 bytes result sent to driver -2016-05-24 22:19:10 INFO Executor:59 - Finished task 3.0 in stage 119.0 (TID 229). 1802 bytes result sent to driver -2016-05-24 22:19:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 119.0 (TID 228) in 163 ms on localhost (1/4) -2016-05-24 22:19:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 119.0 (TID 226) in 164 ms on localhost (2/4) -2016-05-24 22:19:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 119.0 (TID 227) in 164 ms on localhost (3/4) -2016-05-24 22:19:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 119.0 (TID 229) in 164 ms on localhost (4/4) -2016-05-24 22:19:10 INFO DAGScheduler:59 - ResultStage 119 (parquet at GLMClassificationModel.scala:78) finished in 0.167 s -2016-05-24 22:19:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 119.0, whose tasks have all completed, from pool -2016-05-24 22:19:10 INFO DAGScheduler:59 - Job 112 finished: parquet at GLMClassificationModel.scala:78, took 0.221192 s -2016-05-24 22:19:11 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=1094740, maxMem=766075207 -2016-05-24 22:19:11 INFO MemoryStore:59 - Block broadcast_217 stored as values in memory (estimated size 86.4 KB, free 729.5 MB) -2016-05-24 22:19:11 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1183252, maxMem=766075207 -2016-05-24 22:19:11 INFO MemoryStore:59 - Block broadcast_217_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.4 MB) -2016-05-24 22:19:11 INFO BlockManagerInfo:59 - Added broadcast_217_piece0 in memory on localhost:56759 (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:19:11 INFO SparkContext:59 - Created broadcast 217 from take at GLMClassificationModel.scala:79 -2016-05-24 22:19:12 INFO BlockManagerInfo:59 - Removed broadcast_216_piece0 on localhost:56759 in memory (size: 20.6 KB, free: 730.2 MB) -2016-05-24 22:19:12 INFO ContextCleaner:59 - Cleaned accumulator 115 -2016-05-24 22:19:12 INFO BlockManagerInfo:59 - Removed broadcast_215_piece0 on localhost:56759 in memory (size: 1898.0 B, free: 730.2 MB) -2016-05-24 22:19:12 INFO ContextCleaner:59 - Cleaned accumulator 114 -2016-05-24 22:19:12 INFO BlockManagerInfo:59 - Removed broadcast_214_piece0 on localhost:56759 in memory (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:19:12 INFO BlockManagerInfo:59 - Removed broadcast_213_piece0 on localhost:56759 in memory (size: 22.9 KB, free: 730.2 MB) -2016-05-24 22:19:12 INFO ContextCleaner:59 - Cleaned accumulator 113 -2016-05-24 22:19:12 INFO BlockManagerInfo:59 - Removed broadcast_212_piece0 on localhost:56759 in memory (size: 41.6 KB, free: 730.3 MB) -2016-05-24 22:19:12 INFO ContextCleaner:59 - Cleaned accumulator 112 -2016-05-24 22:19:12 INFO BlockManagerInfo:59 - Removed broadcast_211_piece0 on localhost:56759 in memory (size: 5.9 KB, free: 730.3 MB) -2016-05-24 22:19:12 INFO ContextCleaner:59 - Cleaned accumulator 111 -2016-05-24 22:19:12 INFO BlockManagerInfo:59 - Removed broadcast_210_piece0 on localhost:56759 in memory (size: 2.9 KB, free: 730.3 MB) -2016-05-24 22:19:12 INFO ContextCleaner:59 - Cleaned accumulator 110 -2016-05-24 22:19:12 INFO BlockManagerInfo:59 - Removed broadcast_209_piece0 on localhost:56759 in memory (size: 2.7 KB, free: 730.3 MB) -2016-05-24 22:19:12 INFO ContextCleaner:59 - Cleaned accumulator 109 -2016-05-24 22:19:12 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 22:19:12 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-24 22:19:12 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524221902829/result/data/part-r-00000-9541f357-7d06-4d26-ab33-3c33e573f91c.gz.parquet -2016-05-24 22:19:12 INFO SparkContext:59 - Starting job: take at GLMClassificationModel.scala:79 -2016-05-24 22:19:12 INFO DAGScheduler:59 - Got job 113 (take at GLMClassificationModel.scala:79) with 1 output partitions -2016-05-24 22:19:12 INFO DAGScheduler:59 - Final stage: ResultStage 120(take at GLMClassificationModel.scala:79) -2016-05-24 22:19:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:19:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:19:12 INFO DAGScheduler:59 - Submitting ResultStage 120 (MapPartitionsRDD[236] at take at GLMClassificationModel.scala:79), which has no missing parents -2016-05-24 22:19:12 INFO MemoryStore:59 - ensureFreeSpace(4168) called with curMem=570248, maxMem=766075207 -2016-05-24 22:19:12 INFO MemoryStore:59 - Block broadcast_218 stored as values in memory (estimated size 4.1 KB, free 730.0 MB) -2016-05-24 22:19:12 INFO MemoryStore:59 - ensureFreeSpace(2438) called with curMem=574416, maxMem=766075207 -2016-05-24 22:19:12 INFO MemoryStore:59 - Block broadcast_218_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.0 MB) -2016-05-24 22:19:12 INFO BlockManagerInfo:59 - Added broadcast_218_piece0 in memory on localhost:56759 (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:19:12 INFO SparkContext:59 - Created broadcast 218 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:19:12 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 120 (MapPartitionsRDD[236] at take at GLMClassificationModel.scala:79) -2016-05-24 22:19:12 INFO TaskSchedulerImpl:59 - Adding task set 120.0 with 1 tasks -2016-05-24 22:19:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 120.0 (TID 230, localhost, PROCESS_LOCAL, 2302 bytes) -2016-05-24 22:19:12 INFO Executor:59 - Running task 0.0 in stage 120.0 (TID 230) -2016-05-24 22:19:12 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524221902829/result/data/part-r-00000-9541f357-7d06-4d26-ab33-3c33e573f91c.gz.parquet start: 0 end: 5410 length: 5410 hosts: []} -2016-05-24 22:19:12 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-24 22:19:12 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group weights { - required int32 type (INT_8); - optional int32 size; - optional group indices (LIST) { - repeated int32 array; - } - optional group values (LIST) { - repeated double array; - } - } - optional double intercept; - optional double threshold; -} - -Catalyst form: -StructType(StructField(weights,org.apache.spark.mllib.linalg.VectorUDT@f71b0bce,true), StructField(intercept,DoubleType,true), StructField(threshold,DoubleType,true)) - -2016-05-24 22:19:12 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-24 22:19:12 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-24 22:19:13 ERROR Executor:96 - Exception in task 0.0 in stage 120.0 (TID 230) -java.lang.OutOfMemoryError: PermGen space - at java.lang.ClassLoader.defineClass1(Native Method) - at java.lang.ClassLoader.defineClass(Unknown Source) - at java.security.SecureClassLoader.defineClass(Unknown Source) - at java.net.URLClassLoader.defineClass(Unknown Source) - at java.net.URLClassLoader.access$100(Unknown Source) - at java.net.URLClassLoader$1.run(Unknown Source) - at java.net.URLClassLoader$1.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at java.net.URLClassLoader.findClass(Unknown Source) - at java.lang.ClassLoader.loadClass(Unknown Source) - at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source) - at java.lang.ClassLoader.loadClass(Unknown Source) - at org.apache.hadoop.io.compress.GzipCodec.getDecompressorType(GzipCodec.java:166) - at org.apache.hadoop.io.compress.CodecPool.getDecompressor(CodecPool.java:176) - at org.apache.parquet.hadoop.CodecFactory$BytesDecompressor.(CodecFactory.java:50) - at org.apache.parquet.hadoop.CodecFactory.getDecompressor(CodecFactory.java:179) - at org.apache.parquet.hadoop.ParquetFileReader$Chunk.readAllPages(ParquetFileReader.java:610) - at org.apache.parquet.hadoop.ParquetFileReader.readNextRowGroup(ParquetFileReader.java:496) - at org.apache.parquet.hadoop.InternalParquetRecordReader.checkRead(InternalParquetRecordReader.java:127) - at org.apache.parquet.hadoop.InternalParquetRecordReader.nextKeyValue(InternalParquetRecordReader.java:208) - at org.apache.parquet.hadoop.ParquetRecordReader.nextKeyValue(ParquetRecordReader.java:201) - at org.apache.spark.rdd.SqlNewHadoopRDD$$anon$1.hasNext(SqlNewHadoopRDD.scala:168) - at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) - at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:308) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48) - at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:103) - at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:47) - at scala.collection.TraversableOnce$class.to(TraversableOnce.scala:273) - at scala.collection.AbstractIterator.to(Iterator.scala:1157) - at scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:265) -2016-05-24 22:20:01 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 22:20:02 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 22:20:03 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 22:20:03 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 22:20:03 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 22:20:04 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 22:20:04 INFO Remoting:74 - Starting remoting -2016-05-24 22:20:05 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:56812] -2016-05-24 22:20:05 INFO Utils:59 - Successfully started service 'sparkDriver' on port 56812. -2016-05-24 22:20:05 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 22:20:05 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 22:20:05 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-c26bcbce-7d89-4859-a9c1-da00494eb181 -2016-05-24 22:20:05 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 22:20:05 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-959a451b-c77d-4164-93b8-76f36d8201b7\httpd-2836b5ea-dbd4-40a6-bbca-47061e80f414 -2016-05-24 22:20:05 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 22:20:05 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:20:05 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:56813 -2016-05-24 22:20:05 INFO Utils:59 - Successfully started service 'HTTP file server' on port 56813. -2016-05-24 22:20:05 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 22:20:06 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:20:06 WARN AbstractLifeCycle:204 - FAILED SelectChannelConnector@0.0.0.0:4040: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.mllib.learning.regression.SVMFromSparkLearning$.main(SVMFromSparkLearning.scala:26) - at org.apache.spark.mllib.learning.regression.SVMFromSparkLearning.main(SVMFromSparkLearning.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-05-24 22:20:06 WARN AbstractLifeCycle:204 - FAILED org.spark-project.jetty.server.Server@10705573: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.mllib.learning.regression.SVMFromSparkLearning$.main(SVMFromSparkLearning.scala:26) - at org.apache.spark.mllib.learning.regression.SVMFromSparkLearning.main(SVMFromSparkLearning.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 22:20:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 22:20:06 WARN Utils:71 - Service 'SparkUI' could not bind on port 4040. Attempting port 4041. -2016-05-24 22:20:06 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:20:06 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4041 -2016-05-24 22:20:06 INFO Utils:59 - Successfully started service 'SparkUI' on port 4041. -2016-05-24 22:20:06 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4041 -2016-05-24 22:20:06 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 22:20:06 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 22:20:07 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 56832. -2016-05-24 22:20:07 INFO NettyBlockTransferService:59 - Server created on 56832 -2016-05-24 22:20:07 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 22:20:07 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:56832 with 730.6 MB RAM, BlockManagerId(driver, localhost, 56832) -2016-05-24 22:20:07 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 22:20:08 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 22:20:08 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 22:20:08 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 22:20:08 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 22:20:08 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:56832 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 22:20:08 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 22:20:10 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 22:20:10 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:20:10 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 22:20:10 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 22:20:10 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 22:20:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:10 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 22:20:10 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 22:20:10 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 22:20:10 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=175476, maxMem=766075207 -2016-05-24 22:20:10 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:20:10 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:56832 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:20:10 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 22:20:10 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 22:20:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:20:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:20:10 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 22:20:10 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 22:20:10 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 22:20:10 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 22:20:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 22:20:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 22:20:10 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 22:20:10 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 22:20:10 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 22:20:10 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 22:20:10 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 22:20:11 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177621, maxMem=766075207 -2016-05-24 22:20:11 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 22:20:11 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:56832 (size: 83.6 KB, free: 730.5 MB) -2016-05-24 22:20:11 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 22:20:11 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263269, maxMem=766075207 -2016-05-24 22:20:11 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 22:20:11 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:56832 (size: 83.1 KB, free: 730.4 MB) -2016-05-24 22:20:11 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 22:20:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 703 ms on localhost (1/2) -2016-05-24 22:20:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 766 ms on localhost (2/2) -2016-05-24 22:20:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 22:20:11 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.797 s -2016-05-24 22:20:11 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 1.047840 s -2016-05-24 22:20:11 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 22:20:11 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 22:20:11 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:20:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:11 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 22:20:11 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=348349, maxMem=766075207 -2016-05-24 22:20:11 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-24 22:20:11 INFO MemoryStore:59 - ensureFreeSpace(2483) called with curMem=352909, maxMem=766075207 -2016-05-24 22:20:11 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-24 22:20:11 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:56832 (size: 2.4 KB, free: 730.4 MB) -2016-05-24 22:20:11 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:11 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:20:11 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 22:20:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:20:11 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 22:20:11 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-05-24 22:20:11 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:20:11 INFO MemoryStore:59 - ensureFreeSpace(54392) called with curMem=355392, maxMem=766075207 -2016-05-24 22:20:11 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 53.1 KB, free 730.2 MB) -2016-05-24 22:20:11 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:56832 (size: 53.1 KB, free: 730.4 MB) -2016-05-24 22:20:11 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2611 bytes result sent to driver -2016-05-24 22:20:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 139 ms on localhost (1/1) -2016-05-24 22:20:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 22:20:11 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.141 s -2016-05-24 22:20:11 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.200656 s -2016-05-24 22:20:11 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 22:20:11 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 22:20:11 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 22:20:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:11 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 22:20:11 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=409784, maxMem=766075207 -2016-05-24 22:20:11 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-24 22:20:11 INFO MemoryStore:59 - ensureFreeSpace(2300) called with curMem=414032, maxMem=766075207 -2016-05-24 22:20:11 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 22:20:11 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:56832 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 22:20:11 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40) -2016-05-24 22:20:11 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 22:20:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:20:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:20:11 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 22:20:11 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 22:20:11 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:11 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 22:20:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 21 ms on localhost (1/2) -2016-05-24 22:20:11 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-05-24 22:20:11 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:20:11 INFO MemoryStore:59 - ensureFreeSpace(62184) called with curMem=416332, maxMem=766075207 -2016-05-24 22:20:11 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 60.7 KB, free 730.1 MB) -2016-05-24 22:20:11 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:56832 (size: 60.7 KB, free: 730.3 MB) -2016-05-24 22:20:11 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2662 bytes result sent to driver -2016-05-24 22:20:11 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.054 s -2016-05-24 22:20:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 53 ms on localhost (2/2) -2016-05-24 22:20:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 22:20:11 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.076617 s -2016-05-24 22:20:11 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 22:20:11 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 22:20:11 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 22:20:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:11 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 22:20:11 INFO MemoryStore:59 - ensureFreeSpace(4264) called with curMem=478516, maxMem=766075207 -2016-05-24 22:20:11 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.2 KB, free 730.1 MB) -2016-05-24 22:20:11 INFO MemoryStore:59 - ensureFreeSpace(2322) called with curMem=482780, maxMem=766075207 -2016-05-24 22:20:11 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 22:20:11 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:56832 (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:20:11 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 22:20:11 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 22:20:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:20:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:20:11 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 22:20:11 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 22:20:11 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:11 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 22:20:11 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:11 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 22:20:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 22 ms on localhost (1/2) -2016-05-24 22:20:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 28 ms on localhost (2/2) -2016-05-24 22:20:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 22:20:11 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.028 s -2016-05-24 22:20:11 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.125506 s -2016-05-24 22:20:12 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 22:20:12 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 22:20:12 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=485102, maxMem=766075207 -2016-05-24 22:20:12 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:20:12 INFO MemoryStore:59 - ensureFreeSpace(388) called with curMem=490694, maxMem=766075207 -2016-05-24 22:20:12 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 388.0 B, free 730.1 MB) -2016-05-24 22:20:12 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:56832 (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:20:12 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:13 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:13 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:13 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:13 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=491082, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=502866, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:13 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 22:20:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:13 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 22:20:13 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 22:20:13 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:13 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:13 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 7968 bytes result sent to driver -2016-05-24 22:20:13 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 7968 bytes result sent to driver -2016-05-24 22:20:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 33 ms on localhost (1/2) -2016-05-24 22:20:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 37 ms on localhost (2/2) -2016-05-24 22:20:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 22:20:13 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.038 s -2016-05-24 22:20:13 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.068048 s -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=506347, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=511939, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:56832 (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:13 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:13 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:13 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:13 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=514177, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=525961, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:13 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 22:20:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:13 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 22:20:13 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 22:20:13 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:13 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:13 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 7968 bytes result sent to driver -2016-05-24 22:20:13 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 7968 bytes result sent to driver -2016-05-24 22:20:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 25 ms on localhost (1/2) -2016-05-24 22:20:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 25 ms on localhost (2/2) -2016-05-24 22:20:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 22:20:13 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 22:20:13 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.114317 s -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=529443, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=535035, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:13 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:13 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:13 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:13 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=538644, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=550428, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:13 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 22:20:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:13 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 22:20:13 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 22:20:13 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:13 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:13 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 7968 bytes result sent to driver -2016-05-24 22:20:13 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 7968 bytes result sent to driver -2016-05-24 22:20:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 45 ms on localhost (1/2) -2016-05-24 22:20:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 48 ms on localhost (2/2) -2016-05-24 22:20:13 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.050 s -2016-05-24 22:20:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 22:20:13 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.110501 s -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=553911, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(3603) called with curMem=559503, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:13 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:13 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:13 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:13 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=563106, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=574890, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:13 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 22:20:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:13 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 22:20:13 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 22:20:13 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:13 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:13 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 7968 bytes result sent to driver -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 7968 bytes result sent to driver -2016-05-24 22:20:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 21 ms on localhost (1/2) -2016-05-24 22:20:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 28 ms on localhost (2/2) -2016-05-24 22:20:13 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.029 s -2016-05-24 22:20:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 22:20:13 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.097204 s -2016-05-24 22:20:13 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:56832 in memory (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=546077, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:56832 in memory (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=545689, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:56832 in memory (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:56832 in memory (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:56832 in memory (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:13 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:13 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:13 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=498585, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=510369, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:13 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 22:20:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:13 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 22:20:13 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 22:20:13 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:13 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:13 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 7968 bytes result sent to driver -2016-05-24 22:20:13 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 7968 bytes result sent to driver -2016-05-24 22:20:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 24 ms on localhost (1/2) -2016-05-24 22:20:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 23 ms on localhost (2/2) -2016-05-24 22:20:13 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.025 s -2016-05-24 22:20:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 22:20:13 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.076075 s -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=513851, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=519443, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:13 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:13 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:13 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:13 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=523049, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=534833, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:13 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 22:20:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:13 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 22:20:13 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 22:20:13 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:13 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 7968 bytes result sent to driver -2016-05-24 22:20:13 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 22 ms on localhost (1/2) -2016-05-24 22:20:13 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 7968 bytes result sent to driver -2016-05-24 22:20:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 26 ms on localhost (2/2) -2016-05-24 22:20:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 22:20:13 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.030 s -2016-05-24 22:20:13 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.053015 s -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=538316, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=543908, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:13 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:13 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:13 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:13 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=547514, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=559298, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:13 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 22:20:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:13 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 22:20:13 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 22:20:13 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:13 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:13 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 7968 bytes result sent to driver -2016-05-24 22:20:13 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 7968 bytes result sent to driver -2016-05-24 22:20:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 19 ms on localhost (1/2) -2016-05-24 22:20:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 19 ms on localhost (2/2) -2016-05-24 22:20:13 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:20:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 22:20:13 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.060177 s -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=562780, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(3604) called with curMem=568372, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:13 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:13 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:13 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:13 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=571976, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=583760, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:13 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 22:20:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:13 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 22:20:13 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 22:20:13 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:13 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:13 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 7968 bytes result sent to driver -2016-05-24 22:20:13 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 7968 bytes result sent to driver -2016-05-24 22:20:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 18 ms on localhost (1/2) -2016-05-24 22:20:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 17 ms on localhost (2/2) -2016-05-24 22:20:13 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:20:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 22:20:13 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.044095 s -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=587244, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:13 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=592836, maxMem=766075207 -2016-05-24 22:20:13 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:13 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:13 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:14 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=596442, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=608226, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 22:20:14 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:14 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 19 ms on localhost (1/2) -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 18 ms on localhost (2/2) -2016-05-24 22:20:14 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 22:20:14 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.079825 s -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=611709, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=617301, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:14 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=620907, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=632691, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 22:20:14 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:14 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:14 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 22 ms on localhost (1/2) -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 25 ms on localhost (2/2) -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 22:20:14 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 22:20:14 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.053118 s -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=636174, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3610) called with curMem=641766, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:14 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=645376, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=657160, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:14 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 22:20:14 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 18 ms on localhost (1/2) -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:14 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 29 ms on localhost (2/2) -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 22:20:14 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.029 s -2016-05-24 22:20:14 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.049192 s -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=660644, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=666236, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:14 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=669842, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=681626, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 22:20:14 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:14 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 15 ms on localhost (1/2) -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 21 ms on localhost (2/2) -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 22:20:14 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 22:20:14 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.038550 s -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=685110, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3610) called with curMem=690702, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:14 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=694312, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=706096, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 22:20:14 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:14 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 11 ms on localhost (1/2) -2016-05-24 22:20:14 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 13 ms on localhost (2/2) -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 22:20:14 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.035197 s -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=709579, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=715171, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:14 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=718777, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=730561, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 22:20:14 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:14 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 14 ms on localhost (1/2) -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 15 ms on localhost (2/2) -2016-05-24 22:20:14 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 22:20:14 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.031330 s -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=734045, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=739637, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:14 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=743243, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=755027, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 22:20:14 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:14 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 16 ms on localhost (1/2) -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 18 ms on localhost (2/2) -2016-05-24 22:20:14 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 22:20:14 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.071732 s -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=758509, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=764101, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:14 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=767714, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=779498, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 22:20:14 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:14 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 45 ms on localhost (1/2) -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 46 ms on localhost (2/2) -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 22:20:14 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.093 s -2016-05-24 22:20:14 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.115426 s -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=782982, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=788574, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:14 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=792180, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=803964, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 22:20:14 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:14 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 15 ms on localhost (1/2) -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:14 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 22 ms on localhost (2/2) -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 22:20:14 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 22:20:14 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.043212 s -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=807448, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=813040, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:14 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=816652, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=828436, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 22:20:14 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:14 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 44 ms on localhost (1/2) -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 48 ms on localhost (2/2) -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 22:20:14 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.057 s -2016-05-24 22:20:14 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.074763 s -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=831919, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3621) called with curMem=837511, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:14 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=841132, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=852916, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:14 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:14 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:14 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 22:20:14 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:14 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 13 ms on localhost (1/2) -2016-05-24 22:20:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:14 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 7968 bytes result sent to driver -2016-05-24 22:20:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 25 ms on localhost (2/2) -2016-05-24 22:20:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 22:20:14 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 22:20:14 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.045605 s -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=856400, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:14 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=861992, maxMem=766075207 -2016-05-24 22:20:14 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:15 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=865604, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=877388, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:15 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:15 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 24 ms on localhost (1/2) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 25 ms on localhost (2/2) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 22:20:15 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.042 s -2016-05-24 22:20:15 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.064976 s -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=880871, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=886463, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:15 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=890075, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=901859, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 22:20:15 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:15 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 14 ms on localhost (1/2) -2016-05-24 22:20:15 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 21 ms on localhost (2/2) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 22:20:15 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 22:20:15 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.073 s -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.185189 s -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=792693, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=794681, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:15 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:15 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=749364, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=757665, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:15 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:15 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 35 ms on localhost (1/2) -2016-05-24 22:20:15 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 37 ms on localhost (2/2) -2016-05-24 22:20:15 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.038 s -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 22:20:15 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.059307 s -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=666741, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=672333, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 22:20:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:15 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=627010, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=638794, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 22:20:15 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 22:20:15 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:15 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:15 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 11 ms on localhost (1/2) -2016-05-24 22:20:15 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 12 ms on localhost (2/2) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 22:20:15 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:20:15 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.035729 s -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=593340, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3634) called with curMem=598932, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:15 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:15 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=562832, maxMem=766075207 -2016-05-24 22:20:15 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=571133, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:15 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 22:20:15 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:15 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 15 ms on localhost (1/2) -2016-05-24 22:20:15 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 19 ms on localhost (2/2) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 22:20:15 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 22:20:15 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.047830 s -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=562829, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=568421, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:15 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=572033, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=583817, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 22:20:15 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:15 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 6 ms on localhost (1/2) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 7 ms on localhost (2/2) -2016-05-24 22:20:15 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 22:20:15 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.023716 s -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=587299, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=592891, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:15 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=596504, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=608288, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 22:20:15 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:15 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:15 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 20 ms on localhost (1/2) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 22 ms on localhost (2/2) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 22:20:15 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 22:20:15 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.035845 s -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=611771, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=617363, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:15 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=620975, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=632759, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 22:20:15 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:15 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 7 ms on localhost (1/2) -2016-05-24 22:20:15 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 9 ms on localhost (2/2) -2016-05-24 22:20:15 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 22:20:15 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.020452 s -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=636242, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=641834, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:15 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=645440, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=657224, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 22:20:15 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:15 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 8 ms on localhost (1/2) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 11 ms on localhost (2/2) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 22:20:15 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:20:15 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.031669 s -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=660707, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3623) called with curMem=666299, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:15 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=669922, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=681706, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 22:20:15 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:15 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 7 ms on localhost (1/2) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 7 ms on localhost (2/2) -2016-05-24 22:20:15 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 22:20:15 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.021881 s -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=685189, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=690781, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:15 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=694387, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=706171, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 22:20:15 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:15 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 36 ms on localhost (1/2) -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:15 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 43 ms on localhost (2/2) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 22:20:15 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.046 s -2016-05-24 22:20:15 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.064982 s -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=709655, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3626) called with curMem=715247, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:15 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=718873, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=730657, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 22:20:15 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:15 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 7968 bytes result sent to driver -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 11 ms on localhost (1/2) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 11 ms on localhost (2/2) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 22:20:15 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:20:15 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.028542 s -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=734140, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=739732, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:15 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=743344, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:15 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=755128, maxMem=766075207 -2016-05-24 22:20:15 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:15 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:15 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:15 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:15 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 22:20:15 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 22:20:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 13 ms on localhost (1/2) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 14 ms on localhost (2/2) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 22:20:16 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.025 s -2016-05-24 22:20:16 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.044252 s -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=758612, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=764204, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:16 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=767816, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=779600, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 18 ms on localhost (1/2) -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 26 ms on localhost (2/2) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 22:20:16 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.027 s -2016-05-24 22:20:16 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.071317 s -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=783084, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=788676, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:16 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=792288, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=804072, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 22:20:16 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 9 ms on localhost (1/2) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 12 ms on localhost (2/2) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 22:20:16 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:20:16 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.027000 s -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=807556, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=813148, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:16 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=816760, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=828544, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 22:20:16 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 18 ms on localhost (1/2) -2016-05-24 22:20:16 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 21 ms on localhost (2/2) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 22:20:16 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.025 s -2016-05-24 22:20:16 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.068168 s -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=832027, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=837619, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:16 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=841231, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=853015, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 22:20:16 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 15 ms on localhost (1/2) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 15 ms on localhost (2/2) -2016-05-24 22:20:16 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 22:20:16 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.033979 s -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=856499, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=862091, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:16 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=865703, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=877487, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 22:20:16 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 9 ms on localhost (1/2) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 8 ms on localhost (2/2) -2016-05-24 22:20:16 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 22:20:16 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.024126 s -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=880970, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=886562, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:16 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=890176, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=901960, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 22:20:16 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 22:20:16 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 10 ms on localhost (1/2) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 9 ms on localhost (2/2) -2016-05-24 22:20:16 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 22:20:16 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.047287 s -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=856501, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=858609, maxMem=766075207 -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:16 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=801494, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=809666, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 22:20:16 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 8 ms on localhost (1/2) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 10 ms on localhost (2/2) -2016-05-24 22:20:16 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 22:20:16 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.023293 s -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=767817, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=764205, maxMem=766075207 -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 22:20:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=715383, maxMem=766075207 -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:16 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=715383, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 22:20:16 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 16 ms on localhost (1/2) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 18 ms on localhost (2/2) -2016-05-24 22:20:16 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 22:20:16 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.036630 s -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=685184, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=675509, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:16 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[93] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=654650, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=666434, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[93] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 22:20:16 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 8 ms on localhost (1/2) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 11 ms on localhost (2/2) -2016-05-24 22:20:16 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 22:20:16 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.026988 s -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=669917, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=675509, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:16 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 22:20:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:16 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[95] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=675508, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=687292, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:16 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:16 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[95] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 22:20:16 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 12 ms on localhost (1/2) -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 18 ms on localhost (2/2) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 22:20:16 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:20:16 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.113061 s -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=587281, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=592873, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:16 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[97] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=596486, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=608270, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[97] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 22:20:16 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 9 ms on localhost (1/2) -2016-05-24 22:20:16 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 13 ms on localhost (2/2) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 22:20:16 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.029401 s -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=611752, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3618) called with curMem=617344, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 91 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Got job 47 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:16 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[99] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=620962, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=632746, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 47 (MapPartitionsRDD[99] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 2 tasks -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 93, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 94, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 94) -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 94). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 94) in 9 ms on localhost (1/2) -2016-05-24 22:20:16 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 93) -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 93). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 93) in 55 ms on localhost (2/2) -2016-05-24 22:20:16 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at GradientDescent.scala:225) finished in 0.056 s -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-24 22:20:16 INFO DAGScheduler:59 - Job 47 finished: treeAggregate at GradientDescent.scala:225, took 0.070403 s -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=636230, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=641822, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 93 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Got job 48 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:16 INFO DAGScheduler:59 - Final stage: ResultStage 48(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[101] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=645436, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=657220, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 48 (MapPartitionsRDD[101] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 2 tasks -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 95, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 96, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 96) -2016-05-24 22:20:16 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 95) -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 95). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 96). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 95) in 7 ms on localhost (1/2) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 96) in 8 ms on localhost (2/2) -2016-05-24 22:20:16 INFO DAGScheduler:59 - ResultStage 48 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-24 22:20:16 INFO DAGScheduler:59 - Job 48 finished: treeAggregate at GradientDescent.scala:225, took 0.020903 s -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=660703, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3618) called with curMem=666295, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 95 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Got job 49 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:16 INFO DAGScheduler:59 - Final stage: ResultStage 49(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[103] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=669913, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=681697, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 49 (MapPartitionsRDD[103] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 2 tasks -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 97, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 98, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:16 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 98) -2016-05-24 22:20:16 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 97) -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:16 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 98). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 97). 7968 bytes result sent to driver -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 98) in 7 ms on localhost (1/2) -2016-05-24 22:20:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 97) in 7 ms on localhost (2/2) -2016-05-24 22:20:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-05-24 22:20:16 INFO DAGScheduler:59 - ResultStage 49 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:16 INFO DAGScheduler:59 - Job 49 finished: treeAggregate at GradientDescent.scala:225, took 0.020209 s -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=685180, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(3622) called with curMem=690772, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:16 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:16 INFO SparkContext:59 - Created broadcast 97 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:16 INFO DAGScheduler:59 - Got job 50 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:16 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:16 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[105] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=694394, maxMem=766075207 -2016-05-24 22:20:16 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=706178, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 50 (MapPartitionsRDD[105] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 2 tasks -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-24 22:20:17 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 13 ms on localhost (1/2) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 15 ms on localhost (2/2) -2016-05-24 22:20:17 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-24 22:20:17 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at GradientDescent.scala:225, took 0.089204 s -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=709659, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=715251, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 99 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Got job 51 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:17 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[107] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=718864, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=730648, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 51 (MapPartitionsRDD[107] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-24 22:20:17 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 9 ms on localhost (1/2) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 11 ms on localhost (2/2) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-24 22:20:17 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:20:17 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at GradientDescent.scala:225, took 0.028761 s -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=734131, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3628) called with curMem=739723, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 101 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Got job 52 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:17 INFO DAGScheduler:59 - Final stage: ResultStage 52(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting ResultStage 52 (MapPartitionsRDD[109] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=743351, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=755135, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 52 (MapPartitionsRDD[109] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 103, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 104, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 103) -2016-05-24 22:20:17 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 104) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 103). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 104). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 103) in 13 ms on localhost (1/2) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 104) in 14 ms on localhost (2/2) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-24 22:20:17 INFO DAGScheduler:59 - ResultStage 52 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:20:17 INFO DAGScheduler:59 - Job 52 finished: treeAggregate at GradientDescent.scala:225, took 0.032918 s -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=758617, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=764209, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 103 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Got job 53 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:17 INFO DAGScheduler:59 - Final stage: ResultStage 53(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[111] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=767841, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=779625, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 104 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[111] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 105, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 106, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 106) -2016-05-24 22:20:17 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 105) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 106). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 105). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 106) in 8 ms on localhost (1/2) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 105) in 10 ms on localhost (2/2) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-24 22:20:17 INFO DAGScheduler:59 - ResultStage 53 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:20:17 INFO DAGScheduler:59 - Job 53 finished: treeAggregate at GradientDescent.scala:225, took 0.030234 s -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=783107, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=788699, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 105 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Got job 54 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:17 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[113] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=792312, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=804096, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 106 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 54 (MapPartitionsRDD[113] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 2 tasks -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 108) -2016-05-24 22:20:17 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 107) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 108). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 108) in 32 ms on localhost (1/2) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 107). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 107) in 37 ms on localhost (2/2) -2016-05-24 22:20:17 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at GradientDescent.scala:225) finished in 0.037 s -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-24 22:20:17 INFO DAGScheduler:59 - Job 54 finished: treeAggregate at GradientDescent.scala:225, took 0.050236 s -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=807579, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=813171, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 107 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Got job 55 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:17 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[115] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=816803, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=828587, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 108 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 55 (MapPartitionsRDD[115] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 2 tasks -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 109, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 110, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 109) -2016-05-24 22:20:17 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 110) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 109). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 110). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 109) in 7 ms on localhost (1/2) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 110) in 7 ms on localhost (2/2) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-24 22:20:17 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:17 INFO DAGScheduler:59 - Job 55 finished: treeAggregate at GradientDescent.scala:225, took 0.020029 s -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=832069, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=837661, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 109 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Got job 56 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:17 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[117] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=841274, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=853058, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 110 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 56 (MapPartitionsRDD[117] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 2 tasks -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 111, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 112, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 112) -2016-05-24 22:20:17 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 111) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 111). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 112). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 111) in 6 ms on localhost (1/2) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 112) in 6 ms on localhost (2/2) -2016-05-24 22:20:17 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-24 22:20:17 INFO DAGScheduler:59 - Job 56 finished: treeAggregate at GradientDescent.scala:225, took 0.023637 s -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=856540, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=862132, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 111 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Got job 57 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:17 INFO DAGScheduler:59 - Final stage: ResultStage 57(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[119] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=865745, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=877529, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 112 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 57 (MapPartitionsRDD[119] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 2 tasks -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 113, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 114, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 114) -2016-05-24 22:20:17 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 113) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 114). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 113). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 114) in 5 ms on localhost (1/2) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 113) in 6 ms on localhost (2/2) -2016-05-24 22:20:17 INFO DAGScheduler:59 - ResultStage 57 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-24 22:20:17 INFO DAGScheduler:59 - Job 57 finished: treeAggregate at GradientDescent.scala:225, took 0.017069 s -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=881009, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=886601, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 113 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Got job 58 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:17 INFO DAGScheduler:59 - Final stage: ResultStage 58(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting ResultStage 58 (MapPartitionsRDD[121] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=890213, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=901997, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 114 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 58 (MapPartitionsRDD[121] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 2 tasks -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 115, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 116, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 115) -2016-05-24 22:20:17 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 116) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 116). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 116) in 22 ms on localhost (1/2) -2016-05-24 22:20:17 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 115). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 115) in 24 ms on localhost (2/2) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-05-24 22:20:17 INFO DAGScheduler:59 - ResultStage 58 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 22:20:17 INFO DAGScheduler:59 - Job 58 finished: treeAggregate at GradientDescent.scala:225, took 0.036965 s -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=905480, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_115 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3620) called with curMem=911072, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_115_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_115_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 115 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Got job 59 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:17 INFO DAGScheduler:59 - Final stage: ResultStage 59(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting ResultStage 59 (MapPartitionsRDD[123] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=914692, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_116 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3476) called with curMem=926476, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_116_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_116_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 116 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 59 (MapPartitionsRDD[123] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 2 tasks -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 117, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 118, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 118) -2016-05-24 22:20:17 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 117) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 118). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 117). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 118) in 7 ms on localhost (1/2) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 117) in 9 ms on localhost (2/2) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-05-24 22:20:17 INFO DAGScheduler:59 - ResultStage 59 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:20:17 INFO DAGScheduler:59 - Job 59 finished: treeAggregate at GradientDescent.scala:225, took 0.055066 s -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=929952, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_117 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=935544, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_117_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_117_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 117 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Got job 60 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:17 INFO DAGScheduler:59 - Final stage: ResultStage 60(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting ResultStage 60 (MapPartitionsRDD[125] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=939156, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_118 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=950940, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_118_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_118_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 118 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 60 (MapPartitionsRDD[125] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Adding task set 60.0 with 2 tasks -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 60.0 (TID 119, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 60.0 (TID 120, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO Executor:59 - Running task 1.0 in stage 60.0 (TID 120) -2016-05-24 22:20:17 INFO Executor:59 - Running task 0.0 in stage 60.0 (TID 119) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 1.0 in stage 60.0 (TID 120). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO Executor:59 - Finished task 0.0 in stage 60.0 (TID 119). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 60.0 (TID 120) in 7 ms on localhost (1/2) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 60.0 (TID 119) in 8 ms on localhost (2/2) -2016-05-24 22:20:17 INFO DAGScheduler:59 - ResultStage 60 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 60.0, whose tasks have all completed, from pool -2016-05-24 22:20:17 INFO DAGScheduler:59 - Job 60 finished: treeAggregate at GradientDescent.scala:225, took 0.020107 s -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=954424, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_119 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3616) called with curMem=960016, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_119_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_119_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 119 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Got job 61 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:17 INFO DAGScheduler:59 - Final stage: ResultStage 61(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting ResultStage 61 (MapPartitionsRDD[127] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=963632, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_120 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=975416, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_120_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_120_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 120 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 61 (MapPartitionsRDD[127] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 2 tasks -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 121, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 122, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 122) -2016-05-24 22:20:17 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 121) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 122). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 121). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 122) in 6 ms on localhost (1/2) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 121) in 7 ms on localhost (2/2) -2016-05-24 22:20:17 INFO DAGScheduler:59 - ResultStage 61 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-05-24 22:20:17 INFO DAGScheduler:59 - Job 61 finished: treeAggregate at GradientDescent.scala:225, took 0.022124 s -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=978899, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_121 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=984491, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_121_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_121_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 121 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Got job 62 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:17 INFO DAGScheduler:59 - Final stage: ResultStage 62(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[129] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=988103, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_122 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=999887, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_122_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_122_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 122 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 62 (MapPartitionsRDD[129] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 2 tasks -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 123, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 62.0 (TID 124, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO Executor:59 - Running task 1.0 in stage 62.0 (TID 124) -2016-05-24 22:20:17 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 123) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 1.0 in stage 62.0 (TID 124). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 123). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 62.0 (TID 124) in 7 ms on localhost (1/2) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 123) in 7 ms on localhost (2/2) -2016-05-24 22:20:17 INFO DAGScheduler:59 - ResultStage 62 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-05-24 22:20:17 INFO DAGScheduler:59 - Job 62 finished: treeAggregate at GradientDescent.scala:225, took 0.020798 s -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1003369, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_123 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3630) called with curMem=1008961, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_123_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_123_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 123 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Got job 63 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:17 INFO DAGScheduler:59 - Final stage: ResultStage 63(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting ResultStage 63 (MapPartitionsRDD[131] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1012591, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_124 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=1024375, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_124_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_124_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 124 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 63 (MapPartitionsRDD[131] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Adding task set 63.0 with 2 tasks -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 63.0 (TID 125, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 63.0 (TID 126, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO Executor:59 - Running task 0.0 in stage 63.0 (TID 125) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 0.0 in stage 63.0 (TID 125). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 63.0 (TID 125) in 8 ms on localhost (1/2) -2016-05-24 22:20:17 INFO Executor:59 - Running task 1.0 in stage 63.0 (TID 126) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 1.0 in stage 63.0 (TID 126). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 63.0 (TID 126) in 51 ms on localhost (2/2) -2016-05-24 22:20:17 INFO DAGScheduler:59 - ResultStage 63 (treeAggregate at GradientDescent.scala:225) finished in 0.053 s -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 63.0, whose tasks have all completed, from pool -2016-05-24 22:20:17 INFO DAGScheduler:59 - Job 63 finished: treeAggregate at GradientDescent.scala:225, took 0.068358 s -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1027856, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_125 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1033448, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_125_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_125_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 125 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Got job 64 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:17 INFO DAGScheduler:59 - Final stage: ResultStage 64(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting ResultStage 64 (MapPartitionsRDD[133] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1037061, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_126 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1048845, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_126_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_126_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 126 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 64 (MapPartitionsRDD[133] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Adding task set 64.0 with 2 tasks -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 64.0 (TID 127, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 64.0 (TID 128, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO Executor:59 - Running task 0.0 in stage 64.0 (TID 127) -2016-05-24 22:20:17 INFO Executor:59 - Running task 1.0 in stage 64.0 (TID 128) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 1.0 in stage 64.0 (TID 128). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO Executor:59 - Finished task 0.0 in stage 64.0 (TID 127). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 64.0 (TID 128) in 8 ms on localhost (1/2) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 64.0 (TID 127) in 9 ms on localhost (2/2) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 64.0, whose tasks have all completed, from pool -2016-05-24 22:20:17 INFO DAGScheduler:59 - ResultStage 64 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:20:17 INFO DAGScheduler:59 - Job 64 finished: treeAggregate at GradientDescent.scala:225, took 0.022622 s -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1052328, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_127 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1057920, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_127_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_127_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 127 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Got job 65 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:17 INFO DAGScheduler:59 - Final stage: ResultStage 65(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[135] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1061532, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_128 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1073316, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_128_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_128_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 128 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 65 (MapPartitionsRDD[135] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 2 tasks -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 129, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 130, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 129) -2016-05-24 22:20:17 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 130) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 130). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 129). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 130) in 8 ms on localhost (1/2) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 129) in 10 ms on localhost (2/2) -2016-05-24 22:20:17 INFO DAGScheduler:59 - ResultStage 65 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-05-24 22:20:17 INFO DAGScheduler:59 - Job 65 finished: treeAggregate at GradientDescent.scala:225, took 0.022690 s -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1076799, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_129 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1082391, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_129_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_129_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 129 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Got job 66 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:17 INFO DAGScheduler:59 - Final stage: ResultStage 66(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting ResultStage 66 (MapPartitionsRDD[137] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1086004, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_130 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1097788, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_130_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_130_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 130 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 66 (MapPartitionsRDD[137] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Adding task set 66.0 with 2 tasks -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 66.0 (TID 131, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 66.0 (TID 132, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO Executor:59 - Running task 1.0 in stage 66.0 (TID 132) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 1.0 in stage 66.0 (TID 132). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO Executor:59 - Running task 0.0 in stage 66.0 (TID 131) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 0.0 in stage 66.0 (TID 131). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 66.0 (TID 132) in 14 ms on localhost (1/2) -2016-05-24 22:20:17 INFO DAGScheduler:59 - ResultStage 66 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:20:17 INFO DAGScheduler:59 - Job 66 finished: treeAggregate at GradientDescent.scala:225, took 0.030762 s -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 66.0 (TID 131) in 16 ms on localhost (2/2) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 66.0, whose tasks have all completed, from pool -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1101271, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_131 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3633) called with curMem=1106863, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_131_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_131_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 131 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Got job 67 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:17 INFO DAGScheduler:59 - Final stage: ResultStage 67(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting ResultStage 67 (MapPartitionsRDD[139] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1110496, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_132 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1122280, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_132_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_132_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 132 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 67 (MapPartitionsRDD[139] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Adding task set 67.0 with 2 tasks -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 67.0 (TID 133, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 67.0 (TID 134, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:17 INFO Executor:59 - Running task 0.0 in stage 67.0 (TID 133) -2016-05-24 22:20:17 INFO Executor:59 - Running task 1.0 in stage 67.0 (TID 134) -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:17 INFO Executor:59 - Finished task 1.0 in stage 67.0 (TID 134). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO Executor:59 - Finished task 0.0 in stage 67.0 (TID 133). 7968 bytes result sent to driver -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 67.0 (TID 134) in 7 ms on localhost (1/2) -2016-05-24 22:20:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 67.0 (TID 133) in 9 ms on localhost (2/2) -2016-05-24 22:20:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 67.0, whose tasks have all completed, from pool -2016-05-24 22:20:17 INFO DAGScheduler:59 - ResultStage 67 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:20:17 INFO DAGScheduler:59 - Job 67 finished: treeAggregate at GradientDescent.scala:225, took 0.023473 s -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1125764, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_133 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:20:17 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1131356, maxMem=766075207 -2016-05-24 22:20:17 INFO MemoryStore:59 - Block broadcast_133_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:20:17 INFO BlockManagerInfo:59 - Added broadcast_133_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:17 INFO SparkContext:59 - Created broadcast 133 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:17 INFO DAGScheduler:59 - Got job 68 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:17 INFO DAGScheduler:59 - Final stage: ResultStage 68(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:17 INFO DAGScheduler:59 - Submitting ResultStage 68 (MapPartitionsRDD[141] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1134969, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_134 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1146753, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_134_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_134_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 134 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 68 (MapPartitionsRDD[141] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Adding task set 68.0 with 2 tasks -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 68.0 (TID 135, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 68.0 (TID 136, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO Executor:59 - Running task 0.0 in stage 68.0 (TID 135) -2016-05-24 22:20:18 INFO Executor:59 - Running task 1.0 in stage 68.0 (TID 136) -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:18 INFO Executor:59 - Finished task 0.0 in stage 68.0 (TID 135). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO Executor:59 - Finished task 1.0 in stage 68.0 (TID 136). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 68.0 (TID 135) in 7 ms on localhost (1/2) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 68.0 (TID 136) in 7 ms on localhost (2/2) -2016-05-24 22:20:18 INFO DAGScheduler:59 - ResultStage 68 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 68.0, whose tasks have all completed, from pool -2016-05-24 22:20:18 INFO DAGScheduler:59 - Job 68 finished: treeAggregate at GradientDescent.scala:225, took 0.023268 s -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1150237, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_135 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3608) called with curMem=1155829, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_135_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_135_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 135 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Got job 69 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:18 INFO DAGScheduler:59 - Final stage: ResultStage 69(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting ResultStage 69 (MapPartitionsRDD[143] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1159437, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_136 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=1171221, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_136_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_136_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 136 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 69 (MapPartitionsRDD[143] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Adding task set 69.0 with 2 tasks -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 69.0 (TID 137, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 69.0 (TID 138, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO Executor:59 - Running task 1.0 in stage 69.0 (TID 138) -2016-05-24 22:20:18 INFO Executor:59 - Running task 0.0 in stage 69.0 (TID 137) -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:18 INFO Executor:59 - Finished task 0.0 in stage 69.0 (TID 137). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO Executor:59 - Finished task 1.0 in stage 69.0 (TID 138). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 69.0 (TID 137) in 8 ms on localhost (1/2) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 69.0 (TID 138) in 8 ms on localhost (2/2) -2016-05-24 22:20:18 INFO DAGScheduler:59 - ResultStage 69 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 69.0, whose tasks have all completed, from pool -2016-05-24 22:20:18 INFO DAGScheduler:59 - Job 69 finished: treeAggregate at GradientDescent.scala:225, took 0.020988 s -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1174703, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_137 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1180295, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_137_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_137_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 137 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Got job 70 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:18 INFO DAGScheduler:59 - Final stage: ResultStage 70(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting ResultStage 70 (MapPartitionsRDD[145] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1183901, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_138 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=1195685, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_138_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_138_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 138 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 70 (MapPartitionsRDD[145] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Adding task set 70.0 with 2 tasks -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 70.0 (TID 139, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 70.0 (TID 140, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO Executor:59 - Running task 0.0 in stage 70.0 (TID 139) -2016-05-24 22:20:18 INFO Executor:59 - Running task 1.0 in stage 70.0 (TID 140) -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:18 INFO Executor:59 - Finished task 0.0 in stage 70.0 (TID 139). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO Executor:59 - Finished task 1.0 in stage 70.0 (TID 140). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 70.0 (TID 139) in 8 ms on localhost (1/2) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 70.0 (TID 140) in 8 ms on localhost (2/2) -2016-05-24 22:20:18 INFO DAGScheduler:59 - ResultStage 70 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 70.0, whose tasks have all completed, from pool -2016-05-24 22:20:18 INFO DAGScheduler:59 - Job 70 finished: treeAggregate at GradientDescent.scala:225, took 0.019988 s -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1199167, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_139 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3619) called with curMem=1204759, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_139_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_139_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 139 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Got job 71 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:18 INFO DAGScheduler:59 - Final stage: ResultStage 71(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting ResultStage 71 (MapPartitionsRDD[147] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1208378, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_140 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1220162, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_140_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_140_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 140 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 71 (MapPartitionsRDD[147] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Adding task set 71.0 with 2 tasks -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 71.0 (TID 141, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 71.0 (TID 142, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO Executor:59 - Running task 1.0 in stage 71.0 (TID 142) -2016-05-24 22:20:18 INFO Executor:59 - Running task 0.0 in stage 71.0 (TID 141) -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:18 INFO Executor:59 - Finished task 1.0 in stage 71.0 (TID 142). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO Executor:59 - Finished task 0.0 in stage 71.0 (TID 141). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 71.0 (TID 142) in 6 ms on localhost (1/2) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 71.0 (TID 141) in 7 ms on localhost (2/2) -2016-05-24 22:20:18 INFO DAGScheduler:59 - ResultStage 71 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 71.0, whose tasks have all completed, from pool -2016-05-24 22:20:18 INFO DAGScheduler:59 - Job 71 finished: treeAggregate at GradientDescent.scala:225, took 0.018798 s -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1223646, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_141 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3604) called with curMem=1229238, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_141_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_141_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 141 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Got job 72 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:18 INFO DAGScheduler:59 - Final stage: ResultStage 72(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting ResultStage 72 (MapPartitionsRDD[149] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1232842, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_142 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1244626, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_142_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_142_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 142 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 72 (MapPartitionsRDD[149] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Adding task set 72.0 with 2 tasks -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 72.0 (TID 143, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 72.0 (TID 144, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO Executor:59 - Running task 1.0 in stage 72.0 (TID 144) -2016-05-24 22:20:18 INFO Executor:59 - Running task 0.0 in stage 72.0 (TID 143) -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:18 INFO Executor:59 - Finished task 1.0 in stage 72.0 (TID 144). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 72.0 (TID 144) in 7 ms on localhost (1/2) -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:18 INFO Executor:59 - Finished task 0.0 in stage 72.0 (TID 143). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 72.0 (TID 143) in 12 ms on localhost (2/2) -2016-05-24 22:20:18 INFO DAGScheduler:59 - ResultStage 72 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 72.0, whose tasks have all completed, from pool -2016-05-24 22:20:18 INFO DAGScheduler:59 - Job 72 finished: treeAggregate at GradientDescent.scala:225, took 0.027466 s -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1248109, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_143 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=1253701, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_143_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_143_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 143 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Got job 73 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:18 INFO DAGScheduler:59 - Final stage: ResultStage 73(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting ResultStage 73 (MapPartitionsRDD[151] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1257310, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_144 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1269094, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_144_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_144_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 144 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 73 (MapPartitionsRDD[151] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Adding task set 73.0 with 2 tasks -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 73.0 (TID 145, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 73.0 (TID 146, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO Executor:59 - Running task 0.0 in stage 73.0 (TID 145) -2016-05-24 22:20:18 INFO Executor:59 - Running task 1.0 in stage 73.0 (TID 146) -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:18 INFO Executor:59 - Finished task 1.0 in stage 73.0 (TID 146). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO Executor:59 - Finished task 0.0 in stage 73.0 (TID 145). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 73.0 (TID 146) in 5 ms on localhost (1/2) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 73.0 (TID 145) in 5 ms on localhost (2/2) -2016-05-24 22:20:18 INFO DAGScheduler:59 - ResultStage 73 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 73.0, whose tasks have all completed, from pool -2016-05-24 22:20:18 INFO DAGScheduler:59 - Job 73 finished: treeAggregate at GradientDescent.scala:225, took 0.019145 s -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1272577, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_145 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1278169, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_145_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_145_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 145 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Got job 74 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:18 INFO DAGScheduler:59 - Final stage: ResultStage 74(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting ResultStage 74 (MapPartitionsRDD[153] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1281775, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_146 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1293559, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_146_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.3 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_146_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 146 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 74 (MapPartitionsRDD[153] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Adding task set 74.0 with 2 tasks -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 74.0 (TID 147, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 74.0 (TID 148, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO Executor:59 - Running task 0.0 in stage 74.0 (TID 147) -2016-05-24 22:20:18 INFO Executor:59 - Running task 1.0 in stage 74.0 (TID 148) -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:18 INFO Executor:59 - Finished task 0.0 in stage 74.0 (TID 147). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO Executor:59 - Finished task 1.0 in stage 74.0 (TID 148). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 74.0 (TID 147) in 6 ms on localhost (1/2) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 74.0 (TID 148) in 5 ms on localhost (2/2) -2016-05-24 22:20:18 INFO DAGScheduler:59 - ResultStage 74 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 74.0, whose tasks have all completed, from pool -2016-05-24 22:20:18 INFO DAGScheduler:59 - Job 74 finished: treeAggregate at GradientDescent.scala:225, took 0.015722 s -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1297042, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_147 stored as values in memory (estimated size 5.5 KB, free 729.3 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3624) called with curMem=1302634, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_147_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.3 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_147_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 147 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Got job 75 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:18 INFO DAGScheduler:59 - Final stage: ResultStage 75(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting ResultStage 75 (MapPartitionsRDD[155] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1306258, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_148 stored as values in memory (estimated size 11.5 KB, free 729.3 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=1318042, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_148_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.3 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_148_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 148 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 75 (MapPartitionsRDD[155] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Adding task set 75.0 with 2 tasks -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 75.0 (TID 149, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 75.0 (TID 150, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO Executor:59 - Running task 0.0 in stage 75.0 (TID 149) -2016-05-24 22:20:18 INFO Executor:59 - Running task 1.0 in stage 75.0 (TID 150) -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:18 INFO Executor:59 - Finished task 0.0 in stage 75.0 (TID 149). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO Executor:59 - Finished task 1.0 in stage 75.0 (TID 150). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 75.0 (TID 149) in 7 ms on localhost (1/2) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 75.0 (TID 150) in 7 ms on localhost (2/2) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 75.0, whose tasks have all completed, from pool -2016-05-24 22:20:18 INFO DAGScheduler:59 - ResultStage 75 (treeAggregate at GradientDescent.scala:225) finished in 0.041 s -2016-05-24 22:20:18 INFO DAGScheduler:59 - Job 75 finished: treeAggregate at GradientDescent.scala:225, took 0.053048 s -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1321524, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_149 stored as values in memory (estimated size 5.5 KB, free 729.3 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3626) called with curMem=1327116, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_149_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.3 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_149_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 149 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Got job 76 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:18 INFO DAGScheduler:59 - Final stage: ResultStage 76(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting ResultStage 76 (MapPartitionsRDD[157] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1330742, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_150 stored as values in memory (estimated size 11.5 KB, free 729.3 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1342526, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_150_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.3 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_150_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 150 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_148_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 76 (MapPartitionsRDD[157] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 2 tasks -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 151, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO ContextCleaner:59 - Cleaned accumulator 76 -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 152, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 152) -2016-05-24 22:20:18 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 151) -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_147_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 151). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 151) in 9 ms on localhost (1/2) -2016-05-24 22:20:18 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 152). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 152) in 10 ms on localhost (2/2) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-05-24 22:20:18 INFO DAGScheduler:59 - ResultStage 76 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:20:18 INFO DAGScheduler:59 - Job 76 finished: treeAggregate at GradientDescent.scala:225, took 0.038071 s -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1318045, maxMem=766075207 -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_146_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_151 stored as values in memory (estimated size 5.5 KB, free 729.3 MB) -2016-05-24 22:20:18 INFO ContextCleaner:59 - Cleaned accumulator 75 -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3611) called with curMem=1302655, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_151_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.3 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_145_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_151_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 151 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_144_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO ContextCleaner:59 - Cleaned accumulator 74 -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_143_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Got job 77 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:18 INFO DAGScheduler:59 - Final stage: ResultStage 77(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_142_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[159] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:18 INFO ContextCleaner:59 - Cleaned accumulator 73 -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1266531, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_152 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1269119, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_152_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_141_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_152_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 152 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 77 (MapPartitionsRDD[159] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 2 tasks -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 153, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 154, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_140_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 153) -2016-05-24 22:20:18 INFO ContextCleaner:59 - Cleaned accumulator 72 -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:18 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 153). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 154) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_139_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 153) in 11 ms on localhost (1/2) -2016-05-24 22:20:18 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 154). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 154) in 13 ms on localhost (2/2) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-05-24 22:20:18 INFO DAGScheduler:59 - ResultStage 77 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:20:18 INFO DAGScheduler:59 - Job 77 finished: treeAggregate at GradientDescent.scala:225, took 0.030743 s -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1248124, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_153 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1253716, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_153_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_153_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 153 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Got job 78 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:18 INFO DAGScheduler:59 - Final stage: ResultStage 78(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting ResultStage 78 (MapPartitionsRDD[161] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1253846, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_154 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_138_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1265630, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_154_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_154_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO ContextCleaner:59 - Cleaned accumulator 71 -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 154 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 78 (MapPartitionsRDD[161] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 2 tasks -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 155, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 156, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 155) -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:18 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 155). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 155) in 11 ms on localhost (1/2) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_137_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 156) -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:18 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 156). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_136_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 156) in 155 ms on localhost (2/2) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-05-24 22:20:18 INFO DAGScheduler:59 - ResultStage 78 (treeAggregate at GradientDescent.scala:225) finished in 0.158 s -2016-05-24 22:20:18 INFO DAGScheduler:59 - Job 78 finished: treeAggregate at GradientDescent.scala:225, took 0.174820 s -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1232865, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_155 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1238457, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_155_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_155_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 155 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Got job 79 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:18 INFO DAGScheduler:59 - Final stage: ResultStage 79(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting ResultStage 79 (MapPartitionsRDD[163] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1242069, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_156 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1253853, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_156_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_156_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 156 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 79 (MapPartitionsRDD[163] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Adding task set 79.0 with 2 tasks -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 79.0 (TID 157, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 79.0 (TID 158, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO Executor:59 - Running task 1.0 in stage 79.0 (TID 158) -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:18 INFO Executor:59 - Finished task 1.0 in stage 79.0 (TID 158). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO Executor:59 - Running task 0.0 in stage 79.0 (TID 157) -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:18 INFO Executor:59 - Finished task 0.0 in stage 79.0 (TID 157). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 79.0 (TID 158) in 38 ms on localhost (1/2) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 79.0 (TID 157) in 40 ms on localhost (2/2) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 79.0, whose tasks have all completed, from pool -2016-05-24 22:20:18 INFO ContextCleaner:59 - Cleaned accumulator 70 -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_135_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_134_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO ContextCleaner:59 - Cleaned accumulator 69 -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_133_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_132_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-05-24 22:20:18 INFO DAGScheduler:59 - ResultStage 79 (treeAggregate at GradientDescent.scala:225) finished in 0.093 s -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_131_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO DAGScheduler:59 - Job 79 finished: treeAggregate at GradientDescent.scala:225, took 0.107287 s -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1204763, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_157 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1204763, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_157_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_157_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 157 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_130_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Got job 80 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:18 INFO DAGScheduler:59 - Final stage: ResultStage 80(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:18 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting ResultStage 80 (MapPartitionsRDD[165] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1193108, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_158 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1204892, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_158_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_158_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 158 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 80 (MapPartitionsRDD[165] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_129_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Adding task set 80.0 with 2 tasks -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 80.0 (TID 159, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 80.0 (TID 160, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO Executor:59 - Running task 1.0 in stage 80.0 (TID 160) -2016-05-24 22:20:18 INFO Executor:59 - Running task 0.0 in stage 80.0 (TID 159) -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:18 INFO Executor:59 - Finished task 1.0 in stage 80.0 (TID 160). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_128_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO Executor:59 - Finished task 0.0 in stage 80.0 (TID 159). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 80.0 (TID 159) in 7 ms on localhost (1/2) -2016-05-24 22:20:18 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 80.0 (TID 160) in 9 ms on localhost (2/2) -2016-05-24 22:20:18 INFO DAGScheduler:59 - ResultStage 80 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:20:18 INFO DAGScheduler:59 - Job 80 finished: treeAggregate at GradientDescent.scala:225, took 0.050512 s -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1174700, maxMem=766075207 -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 80.0, whose tasks have all completed, from pool -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_159 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1180292, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_159_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_127_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_159_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 159 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Got job 81 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:18 INFO DAGScheduler:59 - Final stage: ResultStage 81(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_126_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting ResultStage 81 (MapPartitionsRDD[167] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:18 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1168631, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_160 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1171210, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_160_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_125_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_160_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 160 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 81 (MapPartitionsRDD[167] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Adding task set 81.0 with 2 tasks -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 81.0 (TID 161, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 81.0 (TID 162, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_124_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO Executor:59 - Running task 1.0 in stage 81.0 (TID 162) -2016-05-24 22:20:18 INFO Executor:59 - Running task 0.0 in stage 81.0 (TID 161) -2016-05-24 22:20:18 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:18 INFO Executor:59 - Finished task 0.0 in stage 81.0 (TID 161). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO Executor:59 - Finished task 1.0 in stage 81.0 (TID 162). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_123_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 81.0 (TID 161) in 8 ms on localhost (1/2) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 81.0 (TID 162) in 8 ms on localhost (2/2) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 81.0, whose tasks have all completed, from pool -2016-05-24 22:20:18 INFO DAGScheduler:59 - ResultStage 81 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:20:18 INFO DAGScheduler:59 - Job 81 finished: treeAggregate at GradientDescent.scala:225, took 0.021929 s -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1134940, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_161 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_122_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1140532, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_161_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_161_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 161 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_121_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_120_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-05-24 22:20:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_119_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO DAGScheduler:59 - Got job 82 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:18 INFO DAGScheduler:59 - Final stage: ResultStage 82(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting ResultStage 82 (MapPartitionsRDD[169] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_118_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1095197, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_162 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:20:18 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=1106981, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_162_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Added broadcast_162_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO SparkContext:59 - Created broadcast 162 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 82 (MapPartitionsRDD[169] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_117_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Adding task set 82.0 with 2 tasks -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 82.0 (TID 163, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 82.0 (TID 164, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:18 INFO Executor:59 - Running task 1.0 in stage 82.0 (TID 164) -2016-05-24 22:20:18 INFO Executor:59 - Running task 0.0 in stage 82.0 (TID 163) -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_116_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:18 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-05-24 22:20:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:18 INFO Executor:59 - Finished task 0.0 in stage 82.0 (TID 163). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO Executor:59 - Finished task 1.0 in stage 82.0 (TID 164). 7968 bytes result sent to driver -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 82.0 (TID 163) in 8 ms on localhost (1/2) -2016-05-24 22:20:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 82.0 (TID 164) in 7 ms on localhost (2/2) -2016-05-24 22:20:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 82.0, whose tasks have all completed, from pool -2016-05-24 22:20:18 INFO DAGScheduler:59 - ResultStage 82 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:20:18 INFO BlockManagerInfo:59 - Removed broadcast_115_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:18 INFO DAGScheduler:59 - Job 82 finished: treeAggregate at GradientDescent.scala:225, took 0.025125 s -2016-05-24 22:20:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1065001, maxMem=766075207 -2016-05-24 22:20:18 INFO MemoryStore:59 - Block broadcast_163 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_114_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=1067110, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_163_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_163_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 163 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_113_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_112_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_111_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_110_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Got job 83 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:19 INFO DAGScheduler:59 - Final stage: ResultStage 83(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:19 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting ResultStage 83 (MapPartitionsRDD[171] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_109_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1012580, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_164 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1012580, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_164_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_108_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_164_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 164 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 83 (MapPartitionsRDD[171] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Adding task set 83.0 with 2 tasks -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 83.0 (TID 165, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 83.0 (TID 166, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_107_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO Executor:59 - Running task 1.0 in stage 83.0 (TID 166) -2016-05-24 22:20:19 INFO Executor:59 - Running task 0.0 in stage 83.0 (TID 165) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_106_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO Executor:59 - Finished task 1.0 in stage 83.0 (TID 166). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 83.0 (TID 166) in 14 ms on localhost (1/2) -2016-05-24 22:20:19 INFO Executor:59 - Finished task 0.0 in stage 83.0 (TID 165). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 83.0 (TID 165) in 19 ms on localhost (2/2) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 83.0, whose tasks have all completed, from pool -2016-05-24 22:20:19 INFO DAGScheduler:59 - ResultStage 83 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_105_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Job 83 finished: treeAggregate at GradientDescent.scala:225, took 0.036630 s -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=978885, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_165 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=984477, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_165_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_165_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_104_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 165 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_103_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_102_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_101_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Got job 84 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:19 INFO DAGScheduler:59 - Final stage: ResultStage 84(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting ResultStage 84 (MapPartitionsRDD[173] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_100_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=923846, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_166 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=932017, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_166_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_99_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_166_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 166 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 84 (MapPartitionsRDD[173] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Adding task set 84.0 with 2 tasks -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 84.0 (TID 167, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 84.0 (TID 168, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO Executor:59 - Running task 1.0 in stage 84.0 (TID 168) -2016-05-24 22:20:19 INFO Executor:59 - Running task 0.0 in stage 84.0 (TID 167) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_98_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 1.0 in stage 84.0 (TID 168). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO Executor:59 - Finished task 0.0 in stage 84.0 (TID 167). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 84.0 (TID 167) in 9 ms on localhost (1/2) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 84.0 (TID 168) in 10 ms on localhost (2/2) -2016-05-24 22:20:19 INFO DAGScheduler:59 - ResultStage 84 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 84.0, whose tasks have all completed, from pool -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_97_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Job 84 finished: treeAggregate at GradientDescent.scala:225, took 0.024266 s -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=901947, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_167 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3620) called with curMem=907539, maxMem=766075207 -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_96_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_167_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_167_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 167 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_95_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_94_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-05-24 22:20:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Got job 85 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:19 INFO DAGScheduler:59 - Final stage: ResultStage 85(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_93_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting ResultStage 85 (MapPartitionsRDD[175] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=865692, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_168 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3485) called with curMem=862208, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_168_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_92_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_168_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 168 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 85 (MapPartitionsRDD[175] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Adding task set 85.0 with 2 tasks -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 85.0 (TID 169, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 85.0 (TID 170, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_91_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO Executor:59 - Running task 0.0 in stage 85.0 (TID 169) -2016-05-24 22:20:19 INFO Executor:59 - Running task 1.0 in stage 85.0 (TID 170) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 0.0 in stage 85.0 (TID 169). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO Executor:59 - Finished task 1.0 in stage 85.0 (TID 170). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 85.0 (TID 169) in 9 ms on localhost (1/2) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 85.0 (TID 170) in 10 ms on localhost (2/2) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_90_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO DAGScheduler:59 - ResultStage 85 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 85.0, whose tasks have all completed, from pool -2016-05-24 22:20:19 INFO DAGScheduler:59 - Job 85 finished: treeAggregate at GradientDescent.scala:225, took 0.025552 s -2016-05-24 22:20:19 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=841217, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_169 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_89_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3621) called with curMem=843196, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_169_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_169_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 169 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_88_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_87_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Got job 86 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:19 INFO DAGScheduler:59 - Final stage: ResultStage 86(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting ResultStage 86 (MapPartitionsRDD[177] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:19 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=801486, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_170 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=813270, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_170_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_170_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 170 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 86 (MapPartitionsRDD[177] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Adding task set 86.0 with 2 tasks -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 86.0 (TID 171, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 86.0 (TID 172, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO Executor:59 - Running task 0.0 in stage 86.0 (TID 171) -2016-05-24 22:20:19 INFO Executor:59 - Running task 1.0 in stage 86.0 (TID 172) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 1.0 in stage 86.0 (TID 172). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO Executor:59 - Finished task 0.0 in stage 86.0 (TID 171). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 86.0 (TID 172) in 8 ms on localhost (1/2) -2016-05-24 22:20:19 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 86.0 (TID 171) in 10 ms on localhost (2/2) -2016-05-24 22:20:19 INFO DAGScheduler:59 - ResultStage 86 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:20:19 INFO DAGScheduler:59 - Job 86 finished: treeAggregate at GradientDescent.scala:225, took 0.023670 s -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 86.0, whose tasks have all completed, from pool -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=783081, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_171 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=773406, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_171_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_171_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 171 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Got job 87 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:19 INFO DAGScheduler:59 - Final stage: ResultStage 87(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting ResultStage 87 (MapPartitionsRDD[179] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:19 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=752547, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_172 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=760717, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_172_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_172_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 172 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 87 (MapPartitionsRDD[179] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Adding task set 87.0 with 2 tasks -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 87.0 (TID 173, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 87.0 (TID 174, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO Executor:59 - Running task 0.0 in stage 87.0 (TID 173) -2016-05-24 22:20:19 INFO Executor:59 - Running task 1.0 in stage 87.0 (TID 174) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 1.0 in stage 87.0 (TID 174). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO Executor:59 - Finished task 0.0 in stage 87.0 (TID 173). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 87.0 (TID 174) in 4 ms on localhost (1/2) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 87.0 (TID 173) in 5 ms on localhost (2/2) -2016-05-24 22:20:19 INFO DAGScheduler:59 - ResultStage 87 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 87.0, whose tasks have all completed, from pool -2016-05-24 22:20:19 INFO DAGScheduler:59 - Job 87 finished: treeAggregate at GradientDescent.scala:225, took 0.017253 s -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=758608, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_173 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=764200, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_173_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_173_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 173 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Got job 88 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:19 INFO DAGScheduler:59 - Final stage: ResultStage 88(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting ResultStage 88 (MapPartitionsRDD[181] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=767813, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_174 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3486) called with curMem=779597, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_174_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_174_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 174 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 88 (MapPartitionsRDD[181] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Adding task set 88.0 with 2 tasks -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 88.0 (TID 175, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 88.0 (TID 176, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO Executor:59 - Running task 1.0 in stage 88.0 (TID 176) -2016-05-24 22:20:19 INFO Executor:59 - Running task 0.0 in stage 88.0 (TID 175) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 0.0 in stage 88.0 (TID 175). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO Executor:59 - Finished task 1.0 in stage 88.0 (TID 176). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 88.0 (TID 175) in 4 ms on localhost (1/2) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 88.0 (TID 176) in 5 ms on localhost (2/2) -2016-05-24 22:20:19 INFO DAGScheduler:59 - ResultStage 88 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 88.0, whose tasks have all completed, from pool -2016-05-24 22:20:19 INFO DAGScheduler:59 - Job 88 finished: treeAggregate at GradientDescent.scala:225, took 0.013664 s -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=783083, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_175 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3615) called with curMem=788675, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_175_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_175_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 175 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Got job 89 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:19 INFO DAGScheduler:59 - Final stage: ResultStage 89(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting ResultStage 89 (MapPartitionsRDD[183] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=792290, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_176 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=804074, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_176_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_176_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 176 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 89 (MapPartitionsRDD[183] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Adding task set 89.0 with 2 tasks -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 89.0 (TID 177, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 89.0 (TID 178, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO Executor:59 - Running task 0.0 in stage 89.0 (TID 177) -2016-05-24 22:20:19 INFO Executor:59 - Running task 1.0 in stage 89.0 (TID 178) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 1.0 in stage 89.0 (TID 178). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO Executor:59 - Finished task 0.0 in stage 89.0 (TID 177). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 89.0 (TID 178) in 4 ms on localhost (1/2) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 89.0 (TID 177) in 5 ms on localhost (2/2) -2016-05-24 22:20:19 INFO DAGScheduler:59 - ResultStage 89 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:20:19 INFO DAGScheduler:59 - Job 89 finished: treeAggregate at GradientDescent.scala:225, took 0.013139 s -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 89.0, whose tasks have all completed, from pool -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=807557, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_177 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=813149, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_177_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_177_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 177 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Got job 90 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:19 INFO DAGScheduler:59 - Final stage: ResultStage 90(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting ResultStage 90 (MapPartitionsRDD[185] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=816761, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_178 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=828545, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_178_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_178_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 178 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 90 (MapPartitionsRDD[185] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Adding task set 90.0 with 2 tasks -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 90.0 (TID 179, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 90.0 (TID 180, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO Executor:59 - Running task 0.0 in stage 90.0 (TID 179) -2016-05-24 22:20:19 INFO Executor:59 - Running task 1.0 in stage 90.0 (TID 180) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 0.0 in stage 90.0 (TID 179). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO Executor:59 - Finished task 1.0 in stage 90.0 (TID 180). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 90.0 (TID 179) in 5 ms on localhost (1/2) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 90.0 (TID 180) in 6 ms on localhost (2/2) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 90.0, whose tasks have all completed, from pool -2016-05-24 22:20:19 INFO DAGScheduler:59 - ResultStage 90 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:20:19 INFO DAGScheduler:59 - Job 90 finished: treeAggregate at GradientDescent.scala:225, took 0.014455 s -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=832029, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_179 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=837621, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_179_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_179_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 179 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Got job 91 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:19 INFO DAGScheduler:59 - Final stage: ResultStage 91(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting ResultStage 91 (MapPartitionsRDD[187] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=841234, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_180 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=853018, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_180_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_180_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 180 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 91 (MapPartitionsRDD[187] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Adding task set 91.0 with 2 tasks -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 91.0 (TID 181, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 91.0 (TID 182, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO Executor:59 - Running task 0.0 in stage 91.0 (TID 181) -2016-05-24 22:20:19 INFO Executor:59 - Running task 1.0 in stage 91.0 (TID 182) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 1.0 in stage 91.0 (TID 182). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 91.0 (TID 182) in 6 ms on localhost (1/2) -2016-05-24 22:20:19 INFO Executor:59 - Finished task 0.0 in stage 91.0 (TID 181). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 91.0 (TID 181) in 9 ms on localhost (2/2) -2016-05-24 22:20:19 INFO DAGScheduler:59 - ResultStage 91 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:20:19 INFO DAGScheduler:59 - Job 91 finished: treeAggregate at GradientDescent.scala:225, took 0.080080 s -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 91.0, whose tasks have all completed, from pool -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=856501, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_181 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3608) called with curMem=862093, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_181_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_181_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 181 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Got job 92 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:19 INFO DAGScheduler:59 - Final stage: ResultStage 92(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting ResultStage 92 (MapPartitionsRDD[189] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=865701, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_182 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=877485, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_182_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_182_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 182 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 92 (MapPartitionsRDD[189] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Adding task set 92.0 with 2 tasks -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 92.0 (TID 183, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 92.0 (TID 184, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO Executor:59 - Running task 1.0 in stage 92.0 (TID 184) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 1.0 in stage 92.0 (TID 184). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO Executor:59 - Running task 0.0 in stage 92.0 (TID 183) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 0.0 in stage 92.0 (TID 183). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 92.0 (TID 184) in 13 ms on localhost (1/2) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 92.0 (TID 183) in 18 ms on localhost (2/2) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 92.0, whose tasks have all completed, from pool -2016-05-24 22:20:19 INFO DAGScheduler:59 - ResultStage 92 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:20:19 INFO DAGScheduler:59 - Job 92 finished: treeAggregate at GradientDescent.scala:225, took 0.029106 s -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=880967, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_183 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=886559, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_183_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_183_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 183 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Got job 93 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:19 INFO DAGScheduler:59 - Final stage: ResultStage 93(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting ResultStage 93 (MapPartitionsRDD[191] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=890173, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_184 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=901957, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_184_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_184_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 184 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 93 (MapPartitionsRDD[191] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Adding task set 93.0 with 2 tasks -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 93.0 (TID 185, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 93.0 (TID 186, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO Executor:59 - Running task 0.0 in stage 93.0 (TID 185) -2016-05-24 22:20:19 INFO Executor:59 - Running task 1.0 in stage 93.0 (TID 186) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 0.0 in stage 93.0 (TID 185). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 1.0 in stage 93.0 (TID 186). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 93.0 (TID 185) in 42 ms on localhost (1/2) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 93.0 (TID 186) in 42 ms on localhost (2/2) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 93.0, whose tasks have all completed, from pool -2016-05-24 22:20:19 INFO DAGScheduler:59 - ResultStage 93 (treeAggregate at GradientDescent.scala:225) finished in 0.043 s -2016-05-24 22:20:19 INFO DAGScheduler:59 - Job 93 finished: treeAggregate at GradientDescent.scala:225, took 0.061893 s -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=905440, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_185 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=911032, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_185_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_185_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 185 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Got job 94 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:19 INFO DAGScheduler:59 - Final stage: ResultStage 94(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting ResultStage 94 (MapPartitionsRDD[193] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=914644, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_186 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=926428, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_186_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_186_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 186 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 94 (MapPartitionsRDD[193] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Adding task set 94.0 with 2 tasks -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 94.0 (TID 187, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 94.0 (TID 188, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO Executor:59 - Running task 1.0 in stage 94.0 (TID 188) -2016-05-24 22:20:19 INFO Executor:59 - Running task 0.0 in stage 94.0 (TID 187) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 1.0 in stage 94.0 (TID 188). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO Executor:59 - Finished task 0.0 in stage 94.0 (TID 187). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 94.0 (TID 188) in 9 ms on localhost (1/2) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 94.0 (TID 187) in 10 ms on localhost (2/2) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 94.0, whose tasks have all completed, from pool -2016-05-24 22:20:19 INFO DAGScheduler:59 - ResultStage 94 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:20:19 INFO DAGScheduler:59 - Job 94 finished: treeAggregate at GradientDescent.scala:225, took 0.039127 s -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=929911, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_187 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=935503, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_187_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_187_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 187 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Got job 95 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:19 INFO DAGScheduler:59 - Final stage: ResultStage 95(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting ResultStage 95 (MapPartitionsRDD[195] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=939116, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_188 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=950900, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_188_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_188_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 188 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 95 (MapPartitionsRDD[195] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Adding task set 95.0 with 2 tasks -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 95.0 (TID 189, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 95.0 (TID 190, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO Executor:59 - Running task 1.0 in stage 95.0 (TID 190) -2016-05-24 22:20:19 INFO Executor:59 - Running task 0.0 in stage 95.0 (TID 189) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 0.0 in stage 95.0 (TID 189). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO Executor:59 - Finished task 1.0 in stage 95.0 (TID 190). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 95.0 (TID 189) in 7 ms on localhost (1/2) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 95.0 (TID 190) in 7 ms on localhost (2/2) -2016-05-24 22:20:19 INFO DAGScheduler:59 - ResultStage 95 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 95.0, whose tasks have all completed, from pool -2016-05-24 22:20:19 INFO DAGScheduler:59 - Job 95 finished: treeAggregate at GradientDescent.scala:225, took 0.020585 s -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=954383, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_189 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3634) called with curMem=959975, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_189_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_189_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 189 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Got job 96 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:19 INFO DAGScheduler:59 - Final stage: ResultStage 96(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting ResultStage 96 (MapPartitionsRDD[197] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=963609, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_190 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=975393, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_190_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_190_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 190 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 96 (MapPartitionsRDD[197] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Adding task set 96.0 with 2 tasks -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 96.0 (TID 191, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 96.0 (TID 192, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO Executor:59 - Running task 1.0 in stage 96.0 (TID 192) -2016-05-24 22:20:19 INFO Executor:59 - Running task 0.0 in stage 96.0 (TID 191) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 0.0 in stage 96.0 (TID 191). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO Executor:59 - Finished task 1.0 in stage 96.0 (TID 192). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 96.0 (TID 192) in 7 ms on localhost (1/2) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 96.0 (TID 191) in 8 ms on localhost (2/2) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 96.0, whose tasks have all completed, from pool -2016-05-24 22:20:19 INFO DAGScheduler:59 - ResultStage 96 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:20:19 INFO DAGScheduler:59 - Job 96 finished: treeAggregate at GradientDescent.scala:225, took 0.020347 s -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=978875, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_191 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3619) called with curMem=984467, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_191_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_191_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 191 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Got job 97 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:19 INFO DAGScheduler:59 - Final stage: ResultStage 97(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting ResultStage 97 (MapPartitionsRDD[199] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=988086, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_192 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=999870, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_192_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_192_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 192 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 97 (MapPartitionsRDD[199] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Adding task set 97.0 with 2 tasks -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 97.0 (TID 193, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 97.0 (TID 194, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO Executor:59 - Running task 1.0 in stage 97.0 (TID 194) -2016-05-24 22:20:19 INFO Executor:59 - Running task 0.0 in stage 97.0 (TID 193) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 0.0 in stage 97.0 (TID 193). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO Executor:59 - Finished task 1.0 in stage 97.0 (TID 194). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 97.0 (TID 193) in 7 ms on localhost (1/2) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 97.0 (TID 194) in 6 ms on localhost (2/2) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 97.0, whose tasks have all completed, from pool -2016-05-24 22:20:19 INFO DAGScheduler:59 - ResultStage 97 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:19 INFO DAGScheduler:59 - Job 97 finished: treeAggregate at GradientDescent.scala:225, took 0.050745 s -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1003353, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_193 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1008945, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_193_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_193_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 193 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Got job 98 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:19 INFO DAGScheduler:59 - Final stage: ResultStage 98(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting ResultStage 98 (MapPartitionsRDD[201] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1012557, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_194 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1024341, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_194_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_194_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 194 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 98 (MapPartitionsRDD[201] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Adding task set 98.0 with 2 tasks -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 98.0 (TID 195, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 98.0 (TID 196, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO Executor:59 - Running task 0.0 in stage 98.0 (TID 195) -2016-05-24 22:20:19 INFO Executor:59 - Running task 1.0 in stage 98.0 (TID 196) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 1.0 in stage 98.0 (TID 196). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO Executor:59 - Finished task 0.0 in stage 98.0 (TID 195). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 98.0 (TID 196) in 6 ms on localhost (1/2) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 98.0 (TID 195) in 8 ms on localhost (2/2) -2016-05-24 22:20:19 INFO DAGScheduler:59 - ResultStage 98 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 98.0, whose tasks have all completed, from pool -2016-05-24 22:20:19 INFO DAGScheduler:59 - Job 98 finished: treeAggregate at GradientDescent.scala:225, took 0.017298 s -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1027824, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_195 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=1033416, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_195_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_195_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 195 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Got job 99 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:19 INFO DAGScheduler:59 - Final stage: ResultStage 99(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting ResultStage 99 (MapPartitionsRDD[203] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1037025, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_196 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1048809, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_196_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_196_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 196 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 99 (MapPartitionsRDD[203] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Adding task set 99.0 with 2 tasks -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 99.0 (TID 197, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 99.0 (TID 198, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO Executor:59 - Running task 0.0 in stage 99.0 (TID 197) -2016-05-24 22:20:19 INFO Executor:59 - Running task 1.0 in stage 99.0 (TID 198) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 1.0 in stage 99.0 (TID 198). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO Executor:59 - Finished task 0.0 in stage 99.0 (TID 197). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 99.0 (TID 198) in 6 ms on localhost (1/2) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 99.0 (TID 197) in 7 ms on localhost (2/2) -2016-05-24 22:20:19 INFO DAGScheduler:59 - ResultStage 99 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 99.0, whose tasks have all completed, from pool -2016-05-24 22:20:19 INFO DAGScheduler:59 - Job 99 finished: treeAggregate at GradientDescent.scala:225, took 0.016409 s -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1052293, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_197 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3616) called with curMem=1057885, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_197_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_197_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 197 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Got job 100 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:19 INFO DAGScheduler:59 - Final stage: ResultStage 100(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting ResultStage 100 (MapPartitionsRDD[205] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1061501, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_198 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1073285, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_198_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_198_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 198 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 100 (MapPartitionsRDD[205] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Adding task set 100.0 with 2 tasks -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 100.0 (TID 199, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 100.0 (TID 200, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO Executor:59 - Running task 0.0 in stage 100.0 (TID 199) -2016-05-24 22:20:19 INFO Executor:59 - Running task 1.0 in stage 100.0 (TID 200) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 0.0 in stage 100.0 (TID 199). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 100.0 (TID 199) in 9 ms on localhost (1/2) -2016-05-24 22:20:19 INFO Executor:59 - Finished task 1.0 in stage 100.0 (TID 200). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 100.0 (TID 200) in 45 ms on localhost (2/2) -2016-05-24 22:20:19 INFO DAGScheduler:59 - ResultStage 100 (treeAggregate at GradientDescent.scala:225) finished in 0.044 s -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 100.0, whose tasks have all completed, from pool -2016-05-24 22:20:19 INFO DAGScheduler:59 - Job 100 finished: treeAggregate at GradientDescent.scala:225, took 0.058600 s -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1076769, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_199 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1082361, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_199_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_199_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 199 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Got job 101 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:19 INFO DAGScheduler:59 - Final stage: ResultStage 101(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting ResultStage 101 (MapPartitionsRDD[207] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1085967, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_200 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1097751, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_200_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_200_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 200 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 101 (MapPartitionsRDD[207] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Adding task set 101.0 with 2 tasks -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 101.0 (TID 201, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 101.0 (TID 202, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO Executor:59 - Running task 0.0 in stage 101.0 (TID 201) -2016-05-24 22:20:19 INFO Executor:59 - Running task 1.0 in stage 101.0 (TID 202) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 1.0 in stage 101.0 (TID 202). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO Executor:59 - Finished task 0.0 in stage 101.0 (TID 201). 7968 bytes result sent to driver -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 101.0 (TID 202) in 7 ms on localhost (1/2) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 101.0 (TID 201) in 7 ms on localhost (2/2) -2016-05-24 22:20:19 INFO DAGScheduler:59 - ResultStage 101 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 101.0, whose tasks have all completed, from pool -2016-05-24 22:20:19 INFO DAGScheduler:59 - Job 101 finished: treeAggregate at GradientDescent.scala:225, took 0.017750 s -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1101235, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_201 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1106827, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_201_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_201_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 201 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Got job 102 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:19 INFO DAGScheduler:59 - Final stage: ResultStage 102(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting ResultStage 102 (MapPartitionsRDD[209] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1110440, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_202 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:20:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1122224, maxMem=766075207 -2016-05-24 22:20:19 INFO MemoryStore:59 - Block broadcast_202_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:20:19 INFO BlockManagerInfo:59 - Added broadcast_202_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:19 INFO SparkContext:59 - Created broadcast 202 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 102 (MapPartitionsRDD[209] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:19 INFO TaskSchedulerImpl:59 - Adding task set 102.0 with 2 tasks -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 102.0 (TID 203, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 102.0 (TID 204, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:19 INFO Executor:59 - Running task 0.0 in stage 102.0 (TID 203) -2016-05-24 22:20:19 INFO Executor:59 - Running task 1.0 in stage 102.0 (TID 204) -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:19 INFO Executor:59 - Finished task 0.0 in stage 102.0 (TID 203). 7968 bytes result sent to driver -2016-05-24 22:20:20 INFO Executor:59 - Finished task 1.0 in stage 102.0 (TID 204). 7968 bytes result sent to driver -2016-05-24 22:20:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 102.0 (TID 203) in 7 ms on localhost (1/2) -2016-05-24 22:20:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 102.0 (TID 204) in 8 ms on localhost (2/2) -2016-05-24 22:20:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 102.0, whose tasks have all completed, from pool -2016-05-24 22:20:20 INFO DAGScheduler:59 - ResultStage 102 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:20 INFO DAGScheduler:59 - Job 102 finished: treeAggregate at GradientDescent.scala:225, took 0.021045 s -2016-05-24 22:20:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1125707, maxMem=766075207 -2016-05-24 22:20:20 INFO MemoryStore:59 - Block broadcast_203 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:20:20 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1131299, maxMem=766075207 -2016-05-24 22:20:20 INFO MemoryStore:59 - Block broadcast_203_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:20:20 INFO BlockManagerInfo:59 - Added broadcast_203_piece0 in memory on localhost:56832 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:20 INFO SparkContext:59 - Created broadcast 203 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:20 INFO DAGScheduler:59 - Got job 103 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:20 INFO DAGScheduler:59 - Final stage: ResultStage 103(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:20 INFO DAGScheduler:59 - Submitting ResultStage 103 (MapPartitionsRDD[211] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1134911, maxMem=766075207 -2016-05-24 22:20:20 INFO MemoryStore:59 - Block broadcast_204 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:20:20 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1146695, maxMem=766075207 -2016-05-24 22:20:20 INFO MemoryStore:59 - Block broadcast_204_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:20:20 INFO BlockManagerInfo:59 - Added broadcast_204_piece0 in memory on localhost:56832 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:20 INFO SparkContext:59 - Created broadcast 204 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 103 (MapPartitionsRDD[211] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:20 INFO TaskSchedulerImpl:59 - Adding task set 103.0 with 2 tasks -2016-05-24 22:20:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 103.0 (TID 205, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 103.0 (TID 206, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:20 INFO Executor:59 - Running task 1.0 in stage 103.0 (TID 206) -2016-05-24 22:20:20 INFO Executor:59 - Running task 0.0 in stage 103.0 (TID 205) -2016-05-24 22:20:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:20 INFO Executor:59 - Finished task 1.0 in stage 103.0 (TID 206). 7968 bytes result sent to driver -2016-05-24 22:20:20 INFO Executor:59 - Finished task 0.0 in stage 103.0 (TID 205). 7968 bytes result sent to driver -2016-05-24 22:20:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 103.0 (TID 206) in 6 ms on localhost (1/2) -2016-05-24 22:20:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 103.0 (TID 205) in 8 ms on localhost (2/2) -2016-05-24 22:20:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 103.0, whose tasks have all completed, from pool -2016-05-24 22:20:20 INFO DAGScheduler:59 - ResultStage 103 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:20 INFO DAGScheduler:59 - Job 103 finished: treeAggregate at GradientDescent.scala:225, took 0.022020 s -2016-05-24 22:20:20 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 3363.579936927662, 3356.5316534793847, 3349.536449573466, 3342.59343622967, 3335.701748404667, 3328.860544103912, 3322.069003535364, 3315.3263283026713, 3308.6317406356134, 3301.984482655661 -2016-05-24 22:20:20 INFO SparkContext:59 - Starting job: sortByKey at BinaryClassificationMetrics.scala:156 -2016-05-24 22:20:20 INFO DAGScheduler:59 - Registering RDD 212 (map at SVMFromSparkLearning.scala:44) -2016-05-24 22:20:20 INFO DAGScheduler:59 - Got job 104 (sortByKey at BinaryClassificationMetrics.scala:156) with 2 output partitions -2016-05-24 22:20:20 INFO DAGScheduler:59 - Final stage: ResultStage 105(sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:20:20 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 104) -2016-05-24 22:20:20 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 104) -2016-05-24 22:20:20 INFO DAGScheduler:59 - Submitting ShuffleMapStage 104 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44), which has no missing parents -2016-05-24 22:20:20 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=1150179, maxMem=766075207 -2016-05-24 22:20:20 INFO MemoryStore:59 - Block broadcast_205 stored as values in memory (estimated size 10.8 KB, free 729.5 MB) -2016-05-24 22:20:20 INFO MemoryStore:59 - ensureFreeSpace(6433) called with curMem=1161187, maxMem=766075207 -2016-05-24 22:20:20 INFO MemoryStore:59 - Block broadcast_205_piece0 stored as bytes in memory (estimated size 6.3 KB, free 729.5 MB) -2016-05-24 22:20:20 INFO BlockManagerInfo:59 - Added broadcast_205_piece0 in memory on localhost:56832 (size: 6.3 KB, free: 730.1 MB) -2016-05-24 22:20:20 INFO SparkContext:59 - Created broadcast 205 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 104 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44) -2016-05-24 22:20:20 INFO TaskSchedulerImpl:59 - Adding task set 104.0 with 2 tasks -2016-05-24 22:20:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 104.0 (TID 207, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:20:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 104.0 (TID 208, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:20:20 INFO Executor:59 - Running task 1.0 in stage 104.0 (TID 208) -2016-05-24 22:20:20 INFO Executor:59 - Running task 0.0 in stage 104.0 (TID 207) -2016-05-24 22:20:20 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:20:20 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:20:20 INFO Executor:59 - Finished task 1.0 in stage 104.0 (TID 208). 2254 bytes result sent to driver -2016-05-24 22:20:20 INFO Executor:59 - Finished task 0.0 in stage 104.0 (TID 207). 2254 bytes result sent to driver -2016-05-24 22:20:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 104.0 (TID 208) in 819 ms on localhost (1/2) -2016-05-24 22:20:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 104.0 (TID 207) in 821 ms on localhost (2/2) -2016-05-24 22:20:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 104.0, whose tasks have all completed, from pool -2016-05-24 22:20:20 INFO DAGScheduler:59 - ShuffleMapStage 104 (map at SVMFromSparkLearning.scala:44) finished in 0.823 s -2016-05-24 22:20:20 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:20:20 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:20:20 INFO DAGScheduler:59 - waiting: Set(ResultStage 105) -2016-05-24 22:20:20 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:20:20 INFO DAGScheduler:59 - Missing parents for ResultStage 105: List() -2016-05-24 22:20:20 INFO DAGScheduler:59 - Submitting ResultStage 105 (MapPartitionsRDD[215] at sortByKey at BinaryClassificationMetrics.scala:156), which is now runnable -2016-05-24 22:20:20 INFO MemoryStore:59 - ensureFreeSpace(3344) called with curMem=1167620, maxMem=766075207 -2016-05-24 22:20:20 INFO MemoryStore:59 - Block broadcast_206 stored as values in memory (estimated size 3.3 KB, free 729.5 MB) -2016-05-24 22:20:20 INFO MemoryStore:59 - ensureFreeSpace(1900) called with curMem=1170964, maxMem=766075207 -2016-05-24 22:20:20 INFO MemoryStore:59 - Block broadcast_206_piece0 stored as bytes in memory (estimated size 1900.0 B, free 729.5 MB) -2016-05-24 22:20:20 INFO BlockManagerInfo:59 - Added broadcast_206_piece0 in memory on localhost:56832 (size: 1900.0 B, free: 730.1 MB) -2016-05-24 22:20:20 INFO SparkContext:59 - Created broadcast 206 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 105 (MapPartitionsRDD[215] at sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:20:20 INFO TaskSchedulerImpl:59 - Adding task set 105.0 with 2 tasks -2016-05-24 22:20:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 105.0 (TID 209, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:20:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 105.0 (TID 210, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:20:20 INFO Executor:59 - Running task 0.0 in stage 105.0 (TID 209) -2016-05-24 22:20:20 INFO Executor:59 - Running task 1.0 in stage 105.0 (TID 210) -2016-05-24 22:20:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:20:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-24 22:20:21 INFO Executor:59 - Finished task 0.0 in stage 105.0 (TID 209). 1469 bytes result sent to driver -2016-05-24 22:20:21 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:20:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 48 ms -2016-05-24 22:20:21 INFO Executor:59 - Finished task 1.0 in stage 105.0 (TID 210). 1517 bytes result sent to driver -2016-05-24 22:20:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 105.0 (TID 209) in 107 ms on localhost (1/2) -2016-05-24 22:20:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 105.0 (TID 210) in 106 ms on localhost (2/2) -2016-05-24 22:20:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 105.0, whose tasks have all completed, from pool -2016-05-24 22:20:21 INFO DAGScheduler:59 - ResultStage 105 (sortByKey at BinaryClassificationMetrics.scala:156) finished in 0.108 s -2016-05-24 22:20:21 INFO DAGScheduler:59 - Job 104 finished: sortByKey at BinaryClassificationMetrics.scala:156, took 0.966493 s -2016-05-24 22:20:21 INFO SparkContext:59 - Starting job: collect at BinaryClassificationMetrics.scala:193 -2016-05-24 22:20:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 154 bytes -2016-05-24 22:20:21 INFO DAGScheduler:59 - Registering RDD 213 (combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:20:21 INFO DAGScheduler:59 - Got job 105 (collect at BinaryClassificationMetrics.scala:193) with 2 output partitions -2016-05-24 22:20:21 INFO DAGScheduler:59 - Final stage: ResultStage 108(collect at BinaryClassificationMetrics.scala:193) -2016-05-24 22:20:21 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 107) -2016-05-24 22:20:21 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 107) -2016-05-24 22:20:21 INFO DAGScheduler:59 - Submitting ShuffleMapStage 107 (ShuffledRDD[213] at combineByKey at BinaryClassificationMetrics.scala:152), which has no missing parents -2016-05-24 22:20:21 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=1172864, maxMem=766075207 -2016-05-24 22:20:21 INFO MemoryStore:59 - Block broadcast_207 stored as values in memory (estimated size 2.8 KB, free 729.5 MB) -2016-05-24 22:20:21 INFO MemoryStore:59 - ensureFreeSpace(1658) called with curMem=1175760, maxMem=766075207 -2016-05-24 22:20:21 INFO MemoryStore:59 - Block broadcast_207_piece0 stored as bytes in memory (estimated size 1658.0 B, free 729.5 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Added broadcast_207_piece0 in memory on localhost:56832 (size: 1658.0 B, free: 730.1 MB) -2016-05-24 22:20:21 INFO SparkContext:59 - Created broadcast 207 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 107 (ShuffledRDD[213] at combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:20:21 INFO TaskSchedulerImpl:59 - Adding task set 107.0 with 2 tasks -2016-05-24 22:20:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 107.0 (TID 211, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:20:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 107.0 (TID 212, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:20:21 INFO Executor:59 - Running task 0.0 in stage 107.0 (TID 211) -2016-05-24 22:20:21 INFO Executor:59 - Running task 1.0 in stage 107.0 (TID 212) -2016-05-24 22:20:21 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:20:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:20:21 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:20:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:20:21 INFO Executor:59 - Finished task 0.0 in stage 107.0 (TID 211). 1375 bytes result sent to driver -2016-05-24 22:20:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 107.0 (TID 211) in 41 ms on localhost (1/2) -2016-05-24 22:20:21 INFO Executor:59 - Finished task 1.0 in stage 107.0 (TID 212). 1375 bytes result sent to driver -2016-05-24 22:20:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 107.0 (TID 212) in 55 ms on localhost (2/2) -2016-05-24 22:20:21 INFO DAGScheduler:59 - ShuffleMapStage 107 (combineByKey at BinaryClassificationMetrics.scala:152) finished in 0.057 s -2016-05-24 22:20:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 107.0, whose tasks have all completed, from pool -2016-05-24 22:20:21 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:20:21 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:20:21 INFO DAGScheduler:59 - waiting: Set(ResultStage 108) -2016-05-24 22:20:21 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:20:21 INFO DAGScheduler:59 - Missing parents for ResultStage 108: List() -2016-05-24 22:20:21 INFO DAGScheduler:59 - Submitting ResultStage 108 (MapPartitionsRDD[218] at mapPartitions at BinaryClassificationMetrics.scala:189), which is now runnable -2016-05-24 22:20:21 INFO MemoryStore:59 - ensureFreeSpace(3472) called with curMem=1177418, maxMem=766075207 -2016-05-24 22:20:21 INFO MemoryStore:59 - Block broadcast_208 stored as values in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:20:21 INFO MemoryStore:59 - ensureFreeSpace(1981) called with curMem=1180890, maxMem=766075207 -2016-05-24 22:20:21 INFO MemoryStore:59 - Block broadcast_208_piece0 stored as bytes in memory (estimated size 1981.0 B, free 729.5 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Added broadcast_208_piece0 in memory on localhost:56832 (size: 1981.0 B, free: 730.1 MB) -2016-05-24 22:20:21 INFO SparkContext:59 - Created broadcast 208 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 108 (MapPartitionsRDD[218] at mapPartitions at BinaryClassificationMetrics.scala:189) -2016-05-24 22:20:21 INFO TaskSchedulerImpl:59 - Adding task set 108.0 with 2 tasks -2016-05-24 22:20:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 108.0 (TID 213, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:20:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 108.0 (TID 214, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:20:21 INFO Executor:59 - Running task 1.0 in stage 108.0 (TID 214) -2016-05-24 22:20:21 INFO Executor:59 - Running task 0.0 in stage 108.0 (TID 213) -2016-05-24 22:20:21 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:20:21 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:20:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:20:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:20:21 INFO Executor:59 - Finished task 0.0 in stage 108.0 (TID 213). 1330 bytes result sent to driver -2016-05-24 22:20:21 INFO Executor:59 - Finished task 1.0 in stage 108.0 (TID 214). 1330 bytes result sent to driver -2016-05-24 22:20:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 108.0 (TID 213) in 22 ms on localhost (1/2) -2016-05-24 22:20:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 108.0 (TID 214) in 22 ms on localhost (2/2) -2016-05-24 22:20:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 108.0, whose tasks have all completed, from pool -2016-05-24 22:20:21 INFO DAGScheduler:59 - ResultStage 108 (collect at BinaryClassificationMetrics.scala:193) finished in 0.050 s -2016-05-24 22:20:21 INFO DAGScheduler:59 - Job 105 finished: collect at BinaryClassificationMetrics.scala:193, took 0.136071 s -2016-05-24 22:20:21 INFO BinaryClassificationMetrics:59 - Total counts: {numPos: 25, numNeg: 11} -2016-05-24 22:20:21 INFO SparkContext:59 - Starting job: runJob at SlidingRDD.scala:75 -2016-05-24 22:20:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 154 bytes -2016-05-24 22:20:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-24 22:20:21 INFO DAGScheduler:59 - Got job 106 (runJob at SlidingRDD.scala:75) with 3 output partitions -2016-05-24 22:20:21 INFO DAGScheduler:59 - Final stage: ResultStage 111(runJob at SlidingRDD.scala:75) -2016-05-24 22:20:21 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 110) -2016-05-24 22:20:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:21 INFO DAGScheduler:59 - Submitting ResultStage 111 (UnionRDD[224] at UnionRDD at BinaryClassificationMetrics.scala:92), which has no missing parents -2016-05-24 22:20:21 INFO MemoryStore:59 - ensureFreeSpace(5184) called with curMem=1182871, maxMem=766075207 -2016-05-24 22:20:21 INFO MemoryStore:59 - Block broadcast_209 stored as values in memory (estimated size 5.1 KB, free 729.5 MB) -2016-05-24 22:20:21 INFO MemoryStore:59 - ensureFreeSpace(2788) called with curMem=1188055, maxMem=766075207 -2016-05-24 22:20:21 INFO MemoryStore:59 - Block broadcast_209_piece0 stored as bytes in memory (estimated size 2.7 KB, free 729.5 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Added broadcast_209_piece0 in memory on localhost:56832 (size: 2.7 KB, free: 730.1 MB) -2016-05-24 22:20:21 INFO SparkContext:59 - Created broadcast 209 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:21 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 111 (UnionRDD[224] at UnionRDD at BinaryClassificationMetrics.scala:92) -2016-05-24 22:20:21 INFO TaskSchedulerImpl:59 - Adding task set 111.0 with 3 tasks -2016-05-24 22:20:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 111.0 (TID 215, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:20:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 111.0 (TID 216, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:20:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 111.0 (TID 217, localhost, PROCESS_LOCAL, 2318 bytes) -2016-05-24 22:20:21 INFO Executor:59 - Running task 1.0 in stage 111.0 (TID 216) -2016-05-24 22:20:21 INFO Executor:59 - Running task 0.0 in stage 111.0 (TID 215) -2016-05-24 22:20:21 INFO Executor:59 - Running task 2.0 in stage 111.0 (TID 217) -2016-05-24 22:20:21 INFO Executor:59 - Finished task 2.0 in stage 111.0 (TID 217). 1053 bytes result sent to driver -2016-05-24 22:20:21 INFO CacheManager:59 - Partition rdd_219_1 not found, computing it -2016-05-24 22:20:21 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:20:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:20:21 INFO CacheManager:59 - Partition rdd_219_0 not found, computing it -2016-05-24 22:20:21 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:20:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:20:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 111.0 (TID 217) in 32 ms on localhost (1/3) -2016-05-24 22:20:21 INFO MemoryStore:59 - ensureFreeSpace(1616) called with curMem=1190843, maxMem=766075207 -2016-05-24 22:20:21 INFO MemoryStore:59 - Block rdd_219_1 stored as values in memory (estimated size 1616.0 B, free 729.4 MB) -2016-05-24 22:20:21 INFO MemoryStore:59 - ensureFreeSpace(1448) called with curMem=1192459, maxMem=766075207 -2016-05-24 22:20:21 INFO MemoryStore:59 - Block rdd_219_0 stored as values in memory (estimated size 1448.0 B, free 729.4 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Added rdd_219_1 in memory on localhost:56832 (size: 1616.0 B, free: 730.1 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Added rdd_219_0 in memory on localhost:56832 (size: 1448.0 B, free: 730.1 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_208_piece0 on localhost:56832 in memory (size: 1981.0 B, free: 730.1 MB) -2016-05-24 22:20:21 INFO Executor:59 - Finished task 0.0 in stage 111.0 (TID 215). 1883 bytes result sent to driver -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 108 -2016-05-24 22:20:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 111.0 (TID 215) in 58 ms on localhost (2/3) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_207_piece0 on localhost:56832 in memory (size: 1658.0 B, free: 730.1 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 107 -2016-05-24 22:20:21 INFO Executor:59 - Finished task 1.0 in stage 111.0 (TID 216). 1883 bytes result sent to driver -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_206_piece0 on localhost:56832 in memory (size: 1900.0 B, free: 730.1 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 106 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_205_piece0 on localhost:56832 in memory (size: 6.3 KB, free: 730.1 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 105 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_204_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 104 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_203_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 111.0 (TID 216) in 76 ms on localhost (3/3) -2016-05-24 22:20:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 111.0, whose tasks have all completed, from pool -2016-05-24 22:20:21 INFO DAGScheduler:59 - ResultStage 111 (runJob at SlidingRDD.scala:75) finished in 0.080 s -2016-05-24 22:20:21 INFO DAGScheduler:59 - Job 106 finished: runJob at SlidingRDD.scala:75, took 0.103607 s -2016-05-24 22:20:21 INFO SparkContext:59 - Starting job: aggregate at AreaUnderCurve.scala:45 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_202_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:21 INFO DAGScheduler:59 - Got job 107 (aggregate at AreaUnderCurve.scala:45) with 4 output partitions -2016-05-24 22:20:21 INFO DAGScheduler:59 - Final stage: ResultStage 114(aggregate at AreaUnderCurve.scala:45) -2016-05-24 22:20:21 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 113) -2016-05-24 22:20:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:21 INFO DAGScheduler:59 - Submitting ResultStage 114 (SlidingRDD[225] at RDD at SlidingRDD.scala:47), which has no missing parents -2016-05-24 22:20:21 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=1133260, maxMem=766075207 -2016-05-24 22:20:21 INFO MemoryStore:59 - Block broadcast_210 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:20:21 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=1138916, maxMem=766075207 -2016-05-24 22:20:21 INFO MemoryStore:59 - Block broadcast_210_piece0 stored as bytes in memory (estimated size 2.9 KB, free 729.5 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 103 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Added broadcast_210_piece0 in memory on localhost:56832 (size: 2.9 KB, free: 730.1 MB) -2016-05-24 22:20:21 INFO SparkContext:59 - Created broadcast 210 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 114 (SlidingRDD[225] at RDD at SlidingRDD.scala:47) -2016-05-24 22:20:21 INFO TaskSchedulerImpl:59 - Adding task set 114.0 with 4 tasks -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_201_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 114.0 (TID 218, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:20:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 114.0 (TID 219, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:20:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 114.0 (TID 220, localhost, PROCESS_LOCAL, 2635 bytes) -2016-05-24 22:20:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 114.0 (TID 221, localhost, PROCESS_LOCAL, 2475 bytes) -2016-05-24 22:20:21 INFO Executor:59 - Running task 1.0 in stage 114.0 (TID 218) -2016-05-24 22:20:21 INFO Executor:59 - Running task 2.0 in stage 114.0 (TID 219) -2016-05-24 22:20:21 INFO Executor:59 - Running task 0.0 in stage 114.0 (TID 220) -2016-05-24 22:20:21 INFO BlockManager:59 - Found block rdd_219_0 locally -2016-05-24 22:20:21 INFO BlockManager:59 - Found block rdd_219_1 locally -2016-05-24 22:20:21 INFO Executor:59 - Running task 3.0 in stage 114.0 (TID 221) -2016-05-24 22:20:21 INFO Executor:59 - Finished task 1.0 in stage 114.0 (TID 218). 2084 bytes result sent to driver -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_200_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:21 INFO Executor:59 - Finished task 3.0 in stage 114.0 (TID 221). 955 bytes result sent to driver -2016-05-24 22:20:21 INFO Executor:59 - Finished task 2.0 in stage 114.0 (TID 219). 2084 bytes result sent to driver -2016-05-24 22:20:21 INFO Executor:59 - Finished task 0.0 in stage 114.0 (TID 220). 955 bytes result sent to driver -2016-05-24 22:20:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 114.0 (TID 221) in 14 ms on localhost (1/4) -2016-05-24 22:20:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 114.0 (TID 218) in 21 ms on localhost (2/4) -2016-05-24 22:20:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 114.0 (TID 220) in 15 ms on localhost (3/4) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 102 -2016-05-24 22:20:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 114.0 (TID 219) in 19 ms on localhost (4/4) -2016-05-24 22:20:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 114.0, whose tasks have all completed, from pool -2016-05-24 22:20:21 INFO DAGScheduler:59 - ResultStage 114 (aggregate at AreaUnderCurve.scala:45) finished in 0.022 s -2016-05-24 22:20:21 INFO DAGScheduler:59 - Job 107 finished: aggregate at AreaUnderCurve.scala:45, took 0.072543 s -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_199_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_198_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 101 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_197_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:21 INFO SparkContext:59 - Starting job: take at SVMFromSparkLearning.scala:55 -2016-05-24 22:20:21 INFO DAGScheduler:59 - Got job 108 (take at SVMFromSparkLearning.scala:55) with 1 output partitions -2016-05-24 22:20:21 INFO DAGScheduler:59 - Final stage: ResultStage 115(take at SVMFromSparkLearning.scala:55) -2016-05-24 22:20:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:21 INFO DAGScheduler:59 - Submitting ResultStage 115 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44), which has no missing parents -2016-05-24 22:20:21 INFO MemoryStore:59 - ensureFreeSpace(10312) called with curMem=1071956, maxMem=766075207 -2016-05-24 22:20:21 INFO MemoryStore:59 - Block broadcast_211 stored as values in memory (estimated size 10.1 KB, free 729.6 MB) -2016-05-24 22:20:21 INFO MemoryStore:59 - ensureFreeSpace(6082) called with curMem=1082268, maxMem=766075207 -2016-05-24 22:20:21 INFO MemoryStore:59 - Block broadcast_211_piece0 stored as bytes in memory (estimated size 5.9 KB, free 729.5 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Added broadcast_211_piece0 in memory on localhost:56832 (size: 5.9 KB, free: 730.1 MB) -2016-05-24 22:20:21 INFO SparkContext:59 - Created broadcast 211 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_196_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 115 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44) -2016-05-24 22:20:21 INFO TaskSchedulerImpl:59 - Adding task set 115.0 with 1 tasks -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 100 -2016-05-24 22:20:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 115.0 (TID 222, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:20:21 INFO Executor:59 - Running task 0.0 in stage 115.0 (TID 222) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_195_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:21 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:20:21 INFO Executor:59 - Finished task 0.0 in stage 115.0 (TID 222). 2401 bytes result sent to driver -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_194_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 99 -2016-05-24 22:20:21 INFO DAGScheduler:59 - ResultStage 115 (take at SVMFromSparkLearning.scala:55) finished in 0.009 s -2016-05-24 22:20:21 INFO DAGScheduler:59 - Job 108 finished: take at SVMFromSparkLearning.scala:55, took 0.022216 s -2016-05-24 22:20:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 115.0 (TID 222) in 9 ms on localhost (1/1) -2016-05-24 22:20:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 115.0, whose tasks have all completed, from pool -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_193_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_192_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 98 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_191_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_190_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 97 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_189_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_188_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 96 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_187_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_186_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 95 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_185_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_184_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 94 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_183_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_182_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 93 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_181_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_180_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 92 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_179_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_178_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 91 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_177_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_176_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 90 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_175_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_174_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 89 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_173_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_172_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 88 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_171_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_170_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 87 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_169_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_168_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 86 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_167_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_166_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 85 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_165_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_164_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 84 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_163_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_162_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 83 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_161_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_160_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 82 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_159_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_158_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 81 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_157_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_156_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 80 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_155_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_154_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 79 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_153_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_152_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 78 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_151_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_150_piece0 on localhost:56832 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:21 INFO ContextCleaner:59 - Cleaned accumulator 77 -2016-05-24 22:20:21 INFO BlockManagerInfo:59 - Removed broadcast_149_piece0 on localhost:56832 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:22 INFO SparkContext:59 - Starting job: saveAsTextFile at GLMClassificationModel.scala:61 -2016-05-24 22:20:22 INFO DAGScheduler:59 - Got job 109 (saveAsTextFile at GLMClassificationModel.scala:61) with 1 output partitions -2016-05-24 22:20:22 INFO DAGScheduler:59 - Final stage: ResultStage 116(saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:20:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:22 INFO DAGScheduler:59 - Submitting ResultStage 116 (MapPartitionsRDD[227] at saveAsTextFile at GLMClassificationModel.scala:61), which has no missing parents -2016-05-24 22:20:22 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=500982, maxMem=766075207 -2016-05-24 22:20:22 INFO MemoryStore:59 - Block broadcast_212 stored as values in memory (estimated size 124.0 KB, free 730.0 MB) -2016-05-24 22:20:22 INFO MemoryStore:59 - ensureFreeSpace(42575) called with curMem=627974, maxMem=766075207 -2016-05-24 22:20:22 INFO MemoryStore:59 - Block broadcast_212_piece0 stored as bytes in memory (estimated size 41.6 KB, free 729.9 MB) -2016-05-24 22:20:22 INFO BlockManagerInfo:59 - Added broadcast_212_piece0 in memory on localhost:56832 (size: 41.6 KB, free: 730.2 MB) -2016-05-24 22:20:22 INFO SparkContext:59 - Created broadcast 212 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 116 (MapPartitionsRDD[227] at saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:20:22 INFO TaskSchedulerImpl:59 - Adding task set 116.0 with 1 tasks -2016-05-24 22:20:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 116.0 (TID 223, localhost, PROCESS_LOCAL, 2197 bytes) -2016-05-24 22:20:22 INFO Executor:59 - Running task 0.0 in stage 116.0 (TID 223) -2016-05-24 22:20:22 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242220_0116_m_000000_223' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222021522/result/metadata/_temporary/0/task_201605242220_0116_m_000000 -2016-05-24 22:20:22 INFO SparkHadoopMapRedUtil:59 - attempt_201605242220_0116_m_000000_223: Committed -2016-05-24 22:20:22 INFO Executor:59 - Finished task 0.0 in stage 116.0 (TID 223). 1864 bytes result sent to driver -2016-05-24 22:20:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 116.0 (TID 223) in 361 ms on localhost (1/1) -2016-05-24 22:20:22 INFO DAGScheduler:59 - ResultStage 116 (saveAsTextFile at GLMClassificationModel.scala:61) finished in 0.360 s -2016-05-24 22:20:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 116.0, whose tasks have all completed, from pool -2016-05-24 22:20:22 INFO DAGScheduler:59 - Job 109 finished: saveAsTextFile at GLMClassificationModel.scala:61, took 0.475670 s -2016-05-24 22:20:25 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:20:25 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:20:25 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:65 -2016-05-24 22:20:25 INFO DAGScheduler:59 - Got job 110 (parquet at GLMClassificationModel.scala:65) with 1 output partitions -2016-05-24 22:20:25 INFO DAGScheduler:59 - Final stage: ResultStage 117(parquet at GLMClassificationModel.scala:65) -2016-05-24 22:20:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:25 INFO DAGScheduler:59 - Submitting ResultStage 117 (MapPartitionsRDD[229] at rddToDataFrameHolder at GLMClassificationModel.scala:65), which has no missing parents -2016-05-24 22:20:25 INFO MemoryStore:59 - ensureFreeSpace(67104) called with curMem=670549, maxMem=766075207 -2016-05-24 22:20:25 INFO MemoryStore:59 - Block broadcast_213 stored as values in memory (estimated size 65.5 KB, free 729.9 MB) -2016-05-24 22:20:25 INFO MemoryStore:59 - ensureFreeSpace(23438) called with curMem=737653, maxMem=766075207 -2016-05-24 22:20:25 INFO MemoryStore:59 - Block broadcast_213_piece0 stored as bytes in memory (estimated size 22.9 KB, free 729.9 MB) -2016-05-24 22:20:25 INFO BlockManagerInfo:59 - Added broadcast_213_piece0 in memory on localhost:56832 (size: 22.9 KB, free: 730.2 MB) -2016-05-24 22:20:25 INFO SparkContext:59 - Created broadcast 213 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 117 (MapPartitionsRDD[229] at rddToDataFrameHolder at GLMClassificationModel.scala:65) -2016-05-24 22:20:25 INFO TaskSchedulerImpl:59 - Adding task set 117.0 with 1 tasks -2016-05-24 22:20:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 117.0 (TID 224, localhost, PROCESS_LOCAL, 7978 bytes) -2016-05-24 22:20:25 INFO Executor:59 - Running task 0.0 in stage 117.0 (TID 224) -2016-05-24 22:20:25 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:20:25 INFO CodecConfig:151 - Compression: GZIP -2016-05-24 22:20:25 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-24 22:20:25 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-24 22:20:25 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-24 22:20:25 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-24 22:20:25 INFO ParquetOutputFormat:151 - Validation is off -2016-05-24 22:20:25 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-24 22:20:25 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-24 22:20:25 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 6,156 -2016-05-24 22:20:25 INFO ColumnChunkPageWriteStore:151 - written 61B for [weights, type] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [RLE, PLAIN, BIT_PACKED] -2016-05-24 22:20:25 INFO ColumnChunkPageWriteStore:151 - written 48B for [weights, size] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [RLE, PLAIN, BIT_PACKED] -2016-05-24 22:20:25 INFO ColumnChunkPageWriteStore:151 - written 51B for [weights, indices, array] INT32: 1 values, 13B raw, 30B comp, 1 pages, encodings: [RLE, PLAIN] -2016-05-24 22:20:25 INFO ColumnChunkPageWriteStore:151 - written 637B for [weights, values, array] DOUBLE: 692 values, 668B raw, 593B comp, 1 pages, encodings: [RLE, PLAIN_DICTIONARY], dic { 413 entries, 3,304B raw, 413B comp} -2016-05-24 22:20:25 INFO ColumnChunkPageWriteStore:151 - written 69B for [intercept] DOUBLE: 1 values, 14B raw, 28B comp, 1 pages, encodings: [RLE, PLAIN, BIT_PACKED] -2016-05-24 22:20:25 INFO ColumnChunkPageWriteStore:151 - written 47B for [threshold] DOUBLE: 1 values, 6B raw, 26B comp, 1 pages, encodings: [RLE, PLAIN, BIT_PACKED] -2016-05-24 22:20:26 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242220_0117_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222021522/result/data/_temporary/0/task_201605242220_0117_m_000000 -2016-05-24 22:20:26 INFO SparkHadoopMapRedUtil:59 - attempt_201605242220_0117_m_000000_0: Committed -2016-05-24 22:20:26 INFO Executor:59 - Finished task 0.0 in stage 117.0 (TID 224). 915 bytes result sent to driver -2016-05-24 22:20:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 117.0 (TID 224) in 815 ms on localhost (1/1) -2016-05-24 22:20:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 117.0, whose tasks have all completed, from pool -2016-05-24 22:20:26 INFO DAGScheduler:59 - ResultStage 117 (parquet at GLMClassificationModel.scala:65) finished in 0.817 s -2016-05-24 22:20:26 INFO DAGScheduler:59 - Job 110 finished: parquet at GLMClassificationModel.scala:65, took 0.867408 s -2016-05-24 22:20:26 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:20:27 INFO DefaultWriterContainer:59 - Job job_201605242220_0000 committed. -2016-05-24 22:20:27 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222021522/result/data on driver -2016-05-24 22:20:27 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222021522/result/data on driver -2016-05-24 22:20:27 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=761091, maxMem=766075207 -2016-05-24 22:20:27 INFO MemoryStore:59 - Block broadcast_214 stored as values in memory (estimated size 225.3 KB, free 729.6 MB) -2016-05-24 22:20:27 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=991795, maxMem=766075207 -2016-05-24 22:20:27 INFO MemoryStore:59 - Block broadcast_214_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.6 MB) -2016-05-24 22:20:27 INFO BlockManagerInfo:59 - Added broadcast_214_piece0 in memory on localhost:56832 (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:20:27 INFO SparkContext:59 - Created broadcast 214 from textFile at modelSaveLoad.scala:129 -2016-05-24 22:20:27 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:20:27 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-24 22:20:27 INFO DAGScheduler:59 - Got job 111 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-24 22:20:27 INFO DAGScheduler:59 - Final stage: ResultStage 118(first at modelSaveLoad.scala:129) -2016-05-24 22:20:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:27 INFO DAGScheduler:59 - Submitting ResultStage 118 (MapPartitionsRDD[232] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-24 22:20:27 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=1011583, maxMem=766075207 -2016-05-24 22:20:27 INFO MemoryStore:59 - Block broadcast_215 stored as values in memory (estimated size 3.1 KB, free 729.6 MB) -2016-05-24 22:20:27 INFO MemoryStore:59 - ensureFreeSpace(1898) called with curMem=1014799, maxMem=766075207 -2016-05-24 22:20:27 INFO MemoryStore:59 - Block broadcast_215_piece0 stored as bytes in memory (estimated size 1898.0 B, free 729.6 MB) -2016-05-24 22:20:27 INFO BlockManagerInfo:59 - Added broadcast_215_piece0 in memory on localhost:56832 (size: 1898.0 B, free: 730.2 MB) -2016-05-24 22:20:27 INFO SparkContext:59 - Created broadcast 215 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 118 (MapPartitionsRDD[232] at textFile at modelSaveLoad.scala:129) -2016-05-24 22:20:27 INFO TaskSchedulerImpl:59 - Adding task set 118.0 with 1 tasks -2016-05-24 22:20:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 118.0 (TID 225, localhost, PROCESS_LOCAL, 2238 bytes) -2016-05-24 22:20:27 INFO Executor:59 - Running task 0.0 in stage 118.0 (TID 225) -2016-05-24 22:20:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222021522/result/metadata/part-00000:0+54 -2016-05-24 22:20:27 INFO Executor:59 - Finished task 0.0 in stage 118.0 (TID 225). 2154 bytes result sent to driver -2016-05-24 22:20:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 118.0 (TID 225) in 23 ms on localhost (1/1) -2016-05-24 22:20:27 INFO DAGScheduler:59 - ResultStage 118 (first at modelSaveLoad.scala:129) finished in 0.023 s -2016-05-24 22:20:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 118.0, whose tasks have all completed, from pool -2016-05-24 22:20:27 INFO DAGScheduler:59 - Job 111 finished: first at modelSaveLoad.scala:129, took 0.034891 s -2016-05-24 22:20:27 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222021522/result/data on driver -2016-05-24 22:20:27 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:78 -2016-05-24 22:20:27 INFO DAGScheduler:59 - Got job 112 (parquet at GLMClassificationModel.scala:78) with 4 output partitions -2016-05-24 22:20:27 INFO DAGScheduler:59 - Final stage: ResultStage 119(parquet at GLMClassificationModel.scala:78) -2016-05-24 22:20:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:27 INFO DAGScheduler:59 - Submitting ResultStage 119 (MapPartitionsRDD[234] at parquet at GLMClassificationModel.scala:78), which has no missing parents -2016-05-24 22:20:27 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=1016697, maxMem=766075207 -2016-05-24 22:20:27 INFO MemoryStore:59 - Block broadcast_216 stored as values in memory (estimated size 61.5 KB, free 729.6 MB) -2016-05-24 22:20:27 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=1079697, maxMem=766075207 -2016-05-24 22:20:27 INFO MemoryStore:59 - Block broadcast_216_piece0 stored as bytes in memory (estimated size 20.6 KB, free 729.5 MB) -2016-05-24 22:20:27 INFO BlockManagerInfo:59 - Added broadcast_216_piece0 in memory on localhost:56832 (size: 20.6 KB, free: 730.2 MB) -2016-05-24 22:20:27 INFO SparkContext:59 - Created broadcast 216 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 119 (MapPartitionsRDD[234] at parquet at GLMClassificationModel.scala:78) -2016-05-24 22:20:27 INFO TaskSchedulerImpl:59 - Adding task set 119.0 with 4 tasks -2016-05-24 22:20:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 119.0 (TID 226, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:20:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 119.0 (TID 227, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:20:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 119.0 (TID 228, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:20:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 119.0 (TID 229, localhost, PROCESS_LOCAL, 2279 bytes) -2016-05-24 22:20:27 INFO Executor:59 - Running task 1.0 in stage 119.0 (TID 227) -2016-05-24 22:20:27 INFO Executor:59 - Running task 2.0 in stage 119.0 (TID 228) -2016-05-24 22:20:27 INFO Executor:59 - Running task 3.0 in stage 119.0 (TID 229) -2016-05-24 22:20:27 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:20:27 INFO Executor:59 - Finished task 2.0 in stage 119.0 (TID 228). 936 bytes result sent to driver -2016-05-24 22:20:27 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:20:27 INFO Executor:59 - Finished task 1.0 in stage 119.0 (TID 227). 936 bytes result sent to driver -2016-05-24 22:20:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 119.0 (TID 228) in 69 ms on localhost (1/4) -2016-05-24 22:20:27 INFO Executor:59 - Running task 0.0 in stage 119.0 (TID 226) -2016-05-24 22:20:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 119.0 (TID 227) in 115 ms on localhost (2/4) -2016-05-24 22:20:28 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:20:28 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:20:28 INFO Executor:59 - Finished task 0.0 in stage 119.0 (TID 226). 936 bytes result sent to driver -2016-05-24 22:20:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 119.0 (TID 226) in 219 ms on localhost (3/4) -2016-05-24 22:20:28 INFO Executor:59 - Finished task 3.0 in stage 119.0 (TID 229). 1802 bytes result sent to driver -2016-05-24 22:20:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 119.0 (TID 229) in 241 ms on localhost (4/4) -2016-05-24 22:20:28 INFO DAGScheduler:59 - ResultStage 119 (parquet at GLMClassificationModel.scala:78) finished in 0.240 s -2016-05-24 22:20:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 119.0, whose tasks have all completed, from pool -2016-05-24 22:20:28 INFO DAGScheduler:59 - Job 112 finished: parquet at GLMClassificationModel.scala:78, took 0.288720 s -2016-05-24 22:20:28 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=1100770, maxMem=766075207 -2016-05-24 22:20:28 INFO MemoryStore:59 - Block broadcast_217 stored as values in memory (estimated size 86.4 KB, free 729.5 MB) -2016-05-24 22:20:28 INFO BlockManagerInfo:59 - Removed broadcast_216_piece0 on localhost:56832 in memory (size: 20.6 KB, free: 730.2 MB) -2016-05-24 22:20:28 INFO ContextCleaner:59 - Cleaned accumulator 115 -2016-05-24 22:20:28 INFO BlockManagerInfo:59 - Removed broadcast_215_piece0 on localhost:56832 in memory (size: 1898.0 B, free: 730.2 MB) -2016-05-24 22:20:28 INFO ContextCleaner:59 - Cleaned accumulator 114 -2016-05-24 22:20:28 INFO BlockManagerInfo:59 - Removed broadcast_214_piece0 on localhost:56832 in memory (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:20:28 INFO BlockManagerInfo:59 - Removed broadcast_213_piece0 on localhost:56832 in memory (size: 22.9 KB, free: 730.2 MB) -2016-05-24 22:20:28 INFO ContextCleaner:59 - Cleaned accumulator 113 -2016-05-24 22:20:29 INFO BlockManagerInfo:59 - Removed broadcast_212_piece0 on localhost:56832 in memory (size: 41.6 KB, free: 730.3 MB) -2016-05-24 22:20:29 INFO ContextCleaner:59 - Cleaned accumulator 112 -2016-05-24 22:20:29 INFO BlockManagerInfo:59 - Removed broadcast_211_piece0 on localhost:56832 in memory (size: 5.9 KB, free: 730.3 MB) -2016-05-24 22:20:29 INFO ContextCleaner:59 - Cleaned accumulator 111 -2016-05-24 22:20:29 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=570129, maxMem=766075207 -2016-05-24 22:20:29 INFO MemoryStore:59 - Block broadcast_217_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.0 MB) -2016-05-24 22:20:29 INFO BlockManagerInfo:59 - Removed broadcast_210_piece0 on localhost:56832 in memory (size: 2.9 KB, free: 730.3 MB) -2016-05-24 22:20:29 INFO BlockManagerInfo:59 - Added broadcast_217_piece0 in memory on localhost:56832 (size: 19.3 KB, free: 730.3 MB) -2016-05-24 22:20:29 INFO ContextCleaner:59 - Cleaned accumulator 110 -2016-05-24 22:20:29 INFO SparkContext:59 - Created broadcast 217 from take at GLMClassificationModel.scala:79 -2016-05-24 22:20:29 INFO BlockManagerInfo:59 - Removed broadcast_209_piece0 on localhost:56832 in memory (size: 2.7 KB, free: 730.3 MB) -2016-05-24 22:20:29 INFO ContextCleaner:59 - Cleaned accumulator 109 -2016-05-24 22:20:29 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:56832 in memory (size: 2.1 KB, free: 730.3 MB) -2016-05-24 22:20:29 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 22:20:29 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-24 22:20:29 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222021522/result/data/part-r-00000-b7b1320a-2597-4e2a-9ea0-5fa670ee53a5.gz.parquet -2016-05-24 22:20:29 INFO SparkContext:59 - Starting job: take at GLMClassificationModel.scala:79 -2016-05-24 22:20:29 INFO DAGScheduler:59 - Got job 113 (take at GLMClassificationModel.scala:79) with 1 output partitions -2016-05-24 22:20:29 INFO DAGScheduler:59 - Final stage: ResultStage 120(take at GLMClassificationModel.scala:79) -2016-05-24 22:20:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:29 INFO DAGScheduler:59 - Submitting ResultStage 120 (MapPartitionsRDD[236] at take at GLMClassificationModel.scala:79), which has no missing parents -2016-05-24 22:20:29 INFO MemoryStore:59 - ensureFreeSpace(4168) called with curMem=570248, maxMem=766075207 -2016-05-24 22:20:29 INFO MemoryStore:59 - Block broadcast_218 stored as values in memory (estimated size 4.1 KB, free 730.0 MB) -2016-05-24 22:20:29 INFO MemoryStore:59 - ensureFreeSpace(2438) called with curMem=574416, maxMem=766075207 -2016-05-24 22:20:29 INFO MemoryStore:59 - Block broadcast_218_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.0 MB) -2016-05-24 22:20:29 INFO BlockManagerInfo:59 - Added broadcast_218_piece0 in memory on localhost:56832 (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:20:29 INFO SparkContext:59 - Created broadcast 218 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 120 (MapPartitionsRDD[236] at take at GLMClassificationModel.scala:79) -2016-05-24 22:20:29 INFO TaskSchedulerImpl:59 - Adding task set 120.0 with 1 tasks -2016-05-24 22:20:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 120.0 (TID 230, localhost, PROCESS_LOCAL, 2299 bytes) -2016-05-24 22:20:29 INFO Executor:59 - Running task 0.0 in stage 120.0 (TID 230) -2016-05-24 22:20:29 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222021522/result/data/part-r-00000-b7b1320a-2597-4e2a-9ea0-5fa670ee53a5.gz.parquet start: 0 end: 5410 length: 5410 hosts: []} -2016-05-24 22:20:29 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-24 22:20:29 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group weights { - required int32 type (INT_8); - optional int32 size; - optional group indices (LIST) { - repeated int32 array; - } - optional group values (LIST) { - repeated double array; - } - } - optional double intercept; - optional double threshold; -} - -Catalyst form: -StructType(StructField(weights,org.apache.spark.mllib.linalg.VectorUDT@f71b0bce,true), StructField(intercept,DoubleType,true), StructField(threshold,DoubleType,true)) - -2016-05-24 22:20:30 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-24 22:20:30 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-24 22:20:35 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 22:20:36 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 22:20:36 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 22:20:36 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 22:20:36 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 22:20:37 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 22:20:37 INFO Remoting:74 - Starting remoting -2016-05-24 22:20:37 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:56852] -2016-05-24 22:20:37 INFO Utils:59 - Successfully started service 'sparkDriver' on port 56852. -2016-05-24 22:20:37 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 22:20:37 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 22:20:37 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-2a803143-b4be-4a62-967b-a1849d0f09b9 -2016-05-24 22:20:37 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 22:20:37 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-a01be572-b5de-449c-b810-7ff83f16ab55\httpd-fa3e486b-4e77-49e7-ae61-3f193f753ed7 -2016-05-24 22:20:37 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 22:20:38 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:20:38 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:56853 -2016-05-24 22:20:38 INFO Utils:59 - Successfully started service 'HTTP file server' on port 56853. -2016-05-24 22:20:38 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 22:20:38 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:20:38 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 22:20:38 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 22:20:38 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 22:20:38 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 22:20:38 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 22:20:38 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 56872. -2016-05-24 22:20:38 INFO NettyBlockTransferService:59 - Server created on 56872 -2016-05-24 22:20:38 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 22:20:38 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:56872 with 730.6 MB RAM, BlockManagerId(driver, localhost, 56872) -2016-05-24 22:20:38 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 22:20:39 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 22:20:39 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 22:20:39 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 22:20:39 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 22:20:39 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:56872 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 22:20:39 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 22:20:40 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 22:20:40 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:20:40 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 22:20:40 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 22:20:40 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 22:20:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:40 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 22:20:41 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 22:20:41 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 22:20:41 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=175476, maxMem=766075207 -2016-05-24 22:20:41 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:20:41 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:56872 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:20:41 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 22:20:41 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 22:20:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:20:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:20:41 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 22:20:41 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 22:20:41 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 22:20:41 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 22:20:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 22:20:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 22:20:41 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 22:20:41 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 22:20:41 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 22:20:41 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 22:20:41 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 22:20:41 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177621, maxMem=766075207 -2016-05-24 22:20:41 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 22:20:41 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263269, maxMem=766075207 -2016-05-24 22:20:41 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 22:20:41 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:56872 (size: 83.6 KB, free: 730.5 MB) -2016-05-24 22:20:41 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:56872 (size: 83.1 KB, free: 730.4 MB) -2016-05-24 22:20:41 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 22:20:41 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 22:20:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 378 ms on localhost (1/2) -2016-05-24 22:20:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 354 ms on localhost (2/2) -2016-05-24 22:20:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 22:20:41 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.403 s -2016-05-24 22:20:41 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.570620 s -2016-05-24 22:20:41 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:56872 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-24 22:20:41 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 22:20:41 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 22:20:41 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 22:20:41 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:20:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:41 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 22:20:41 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=342308, maxMem=766075207 -2016-05-24 22:20:41 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.3 MB) -2016-05-24 22:20:41 INFO MemoryStore:59 - ensureFreeSpace(2483) called with curMem=346868, maxMem=766075207 -2016-05-24 22:20:41 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.3 MB) -2016-05-24 22:20:41 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:56872 (size: 2.4 KB, free: 730.4 MB) -2016-05-24 22:20:41 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:41 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:20:41 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 22:20:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:20:41 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 22:20:41 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-05-24 22:20:41 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:20:41 INFO MemoryStore:59 - ensureFreeSpace(54392) called with curMem=349351, maxMem=766075207 -2016-05-24 22:20:41 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 53.1 KB, free 730.2 MB) -2016-05-24 22:20:41 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:56872 (size: 53.1 KB, free: 730.4 MB) -2016-05-24 22:20:41 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2611 bytes result sent to driver -2016-05-24 22:20:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 45 ms on localhost (1/1) -2016-05-24 22:20:41 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.047 s -2016-05-24 22:20:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 22:20:41 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.126839 s -2016-05-24 22:20:41 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 22:20:41 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 22:20:41 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 22:20:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:41 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 22:20:41 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=403743, maxMem=766075207 -2016-05-24 22:20:41 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-24 22:20:41 INFO MemoryStore:59 - ensureFreeSpace(2300) called with curMem=407991, maxMem=766075207 -2016-05-24 22:20:41 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 22:20:41 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:56872 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 22:20:41 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40) -2016-05-24 22:20:41 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 22:20:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:20:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:20:41 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 22:20:41 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 22:20:41 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:41 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-05-24 22:20:41 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 22:20:41 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:20:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 14 ms on localhost (1/2) -2016-05-24 22:20:41 INFO MemoryStore:59 - ensureFreeSpace(62184) called with curMem=410291, maxMem=766075207 -2016-05-24 22:20:41 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 60.7 KB, free 730.1 MB) -2016-05-24 22:20:41 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:56872 (size: 60.7 KB, free: 730.3 MB) -2016-05-24 22:20:41 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2662 bytes result sent to driver -2016-05-24 22:20:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 24 ms on localhost (2/2) -2016-05-24 22:20:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 22:20:41 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.025 s -2016-05-24 22:20:41 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.052143 s -2016-05-24 22:20:41 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 22:20:41 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 22:20:41 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 22:20:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:41 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 22:20:41 INFO MemoryStore:59 - ensureFreeSpace(4264) called with curMem=472475, maxMem=766075207 -2016-05-24 22:20:41 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.2 KB, free 730.1 MB) -2016-05-24 22:20:41 INFO MemoryStore:59 - ensureFreeSpace(2322) called with curMem=476739, maxMem=766075207 -2016-05-24 22:20:41 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 22:20:41 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:56872 (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:20:41 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 22:20:41 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 22:20:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:20:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:20:41 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 22:20:41 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 22:20:41 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:41 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 22:20:41 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:41 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 22:20:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 26 ms on localhost (1/2) -2016-05-24 22:20:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 29 ms on localhost (2/2) -2016-05-24 22:20:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 22:20:41 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.030 s -2016-05-24 22:20:41 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.095427 s -2016-05-24 22:20:42 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 22:20:42 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=479061, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(388) called with curMem=484653, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 388.0 B, free 730.1 MB) -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:56872 (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:20:42 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:42 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:42 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:42 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=485041, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=496825, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:42 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 22:20:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:42 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 22:20:42 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 22:20:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:42 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 7968 bytes result sent to driver -2016-05-24 22:20:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 19 ms on localhost (1/2) -2016-05-24 22:20:42 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 7968 bytes result sent to driver -2016-05-24 22:20:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 21 ms on localhost (2/2) -2016-05-24 22:20:42 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.025 s -2016-05-24 22:20:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 22:20:42 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.079031 s -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=500306, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=505898, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:56872 (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:42 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:42 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:42 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=508136, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=519920, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:42 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 22:20:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:42 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 22:20:42 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 22:20:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:42 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 7968 bytes result sent to driver -2016-05-24 22:20:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 21 ms on localhost (1/2) -2016-05-24 22:20:42 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 7968 bytes result sent to driver -2016-05-24 22:20:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 23 ms on localhost (2/2) -2016-05-24 22:20:42 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.025 s -2016-05-24 22:20:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 22:20:42 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.071662 s -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=523403, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=528995, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:42 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:42 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:42 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=532604, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=544388, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:42 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 22:20:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:42 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 22:20:42 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 22:20:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:42 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 7968 bytes result sent to driver -2016-05-24 22:20:42 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 7968 bytes result sent to driver -2016-05-24 22:20:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 19 ms on localhost (1/2) -2016-05-24 22:20:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 22 ms on localhost (2/2) -2016-05-24 22:20:42 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 22:20:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 22:20:42 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.062551 s -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=547872, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(3603) called with curMem=553464, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:42 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:42 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:42 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=557067, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=568851, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:42 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 22:20:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:42 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 22:20:42 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 22:20:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:42 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 7968 bytes result sent to driver -2016-05-24 22:20:42 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 7968 bytes result sent to driver -2016-05-24 22:20:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 28 ms on localhost (1/2) -2016-05-24 22:20:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 33 ms on localhost (2/2) -2016-05-24 22:20:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.035 s -2016-05-24 22:20:42 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 22:20:42 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.086498 s -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=527006, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:56872 in memory (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=530360, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:56872 in memory (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:56872 in memory (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:56872 in memory (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 22:20:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:56872 in memory (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 22:20:42 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:42 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:42 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=492544, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=504328, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:42 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 22:20:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:42 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 22:20:42 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 22:20:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:42 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 7968 bytes result sent to driver -2016-05-24 22:20:42 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 7968 bytes result sent to driver -2016-05-24 22:20:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 16 ms on localhost (1/2) -2016-05-24 22:20:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 17 ms on localhost (2/2) -2016-05-24 22:20:42 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:20:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 22:20:42 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.033391 s -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=507809, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=513401, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:42 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:42 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:42 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=517007, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=528791, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:42 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 22:20:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:42 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 22:20:42 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 22:20:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:42 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 7968 bytes result sent to driver -2016-05-24 22:20:42 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 7968 bytes result sent to driver -2016-05-24 22:20:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 14 ms on localhost (1/2) -2016-05-24 22:20:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 14 ms on localhost (2/2) -2016-05-24 22:20:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 22:20:42 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:20:42 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.031545 s -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=532274, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=537866, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:42 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:42 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:42 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=541472, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=553256, maxMem=766075207 -2016-05-24 22:20:42 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:20:42 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:42 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:42 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 22:20:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:42 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 22:20:42 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 22:20:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:43 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 13 ms on localhost (1/2) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 12 ms on localhost (2/2) -2016-05-24 22:20:43 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 22:20:43 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.025965 s -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=556740, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3604) called with curMem=562332, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:43 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=565936, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=577720, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 22:20:43 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:43 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:43 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 16 ms on localhost (1/2) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 17 ms on localhost (2/2) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 22:20:43 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:20:43 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.033688 s -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=581204, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=586796, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:43 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=590402, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=602186, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 22:20:43 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:43 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 13 ms on localhost (1/2) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 12 ms on localhost (2/2) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 22:20:43 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:20:43 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.033771 s -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=605670, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=611262, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:43 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=614868, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=626652, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 22:20:43 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:43 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 11 ms on localhost (1/2) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 12 ms on localhost (2/2) -2016-05-24 22:20:43 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 22:20:43 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.033008 s -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=630136, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3610) called with curMem=635728, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:43 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=639338, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=651122, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 22:20:43 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:43 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 13 ms on localhost (1/2) -2016-05-24 22:20:43 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 14 ms on localhost (2/2) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 22:20:43 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:20:43 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.028038 s -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=654605, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=660197, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:43 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=663803, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=675587, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 22:20:43 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:43 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 16 ms on localhost (1/2) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 16 ms on localhost (2/2) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 22:20:43 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 22:20:43 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.036044 s -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=679069, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3610) called with curMem=684661, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 22:20:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:43 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=644930, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=651122, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 22:20:43 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 22:20:43 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 12 ms on localhost (1/2) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 13 ms on localhost (2/2) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 22:20:43 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:20:43 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.029549 s -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=590405, maxMem=766075207 -2016-05-24 22:20:43 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=586801, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 22:20:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:43 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=522605, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=528797, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 22:20:43 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:43 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 11 ms on localhost (1/2) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 11 ms on localhost (2/2) -2016-05-24 22:20:43 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 22:20:43 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.024042 s -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=507819, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=513411, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:43 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=517017, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=528801, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 22:20:43 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:43 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 11 ms on localhost (1/2) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 14 ms on localhost (2/2) -2016-05-24 22:20:43 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 22:20:43 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.029891 s -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=532284, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=537876, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:43 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=541489, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=553273, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:43 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 15 ms on localhost (1/2) -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:43 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 29 ms on localhost (2/2) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 22:20:43 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.030 s -2016-05-24 22:20:43 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.050696 s -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=556757, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=562349, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:43 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=565955, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=577739, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 22:20:43 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:43 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 14 ms on localhost (1/2) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 16 ms on localhost (2/2) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 22:20:43 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:20:43 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.031007 s -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=581222, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=586814, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:43 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=590426, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=602210, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 22:20:43 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:43 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 14 ms on localhost (1/2) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 16 ms on localhost (2/2) -2016-05-24 22:20:43 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 22:20:43 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.033207 s -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=605693, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3621) called with curMem=611285, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:43 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=614906, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=626690, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 22:20:43 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:43 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 13 ms on localhost (1/2) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 13 ms on localhost (2/2) -2016-05-24 22:20:43 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 22:20:43 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.030533 s -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=630174, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=635766, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:43 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=639378, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=651162, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 22:20:43 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:43 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 13 ms on localhost (1/2) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 13 ms on localhost (2/2) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 22:20:43 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:20:43 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.028641 s -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=654645, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=660237, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:43 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=663849, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=675633, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 22:20:43 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:43 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 10 ms on localhost (1/2) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 12 ms on localhost (2/2) -2016-05-24 22:20:43 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 22:20:43 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.029698 s -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=679117, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=684709, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:43 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=688323, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=700107, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 22:20:43 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:43 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 12 ms on localhost (1/2) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 14 ms on localhost (2/2) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 22:20:43 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:20:43 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.029451 s -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=703590, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=709182, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:43 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=712796, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=724580, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 22:20:43 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:43 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 18 ms on localhost (1/2) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 16 ms on localhost (2/2) -2016-05-24 22:20:43 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 22:20:43 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.090466 s -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=728060, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3634) called with curMem=733652, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:43 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=737286, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:43 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=749070, maxMem=766075207 -2016-05-24 22:20:43 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:43 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:43 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:43 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:43 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:43 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 22:20:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:43 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 7968 bytes result sent to driver -2016-05-24 22:20:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 16 ms on localhost (1/2) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 18 ms on localhost (2/2) -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 22:20:44 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 22:20:44 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.036632 s -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=752551, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=758143, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:44 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=761755, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=773539, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 22:20:44 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:44 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 7 ms on localhost (1/2) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 7 ms on localhost (2/2) -2016-05-24 22:20:44 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 22:20:44 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.023869 s -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=777021, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=782613, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:44 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=786226, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=798010, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 22:20:44 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:44 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 11 ms on localhost (1/2) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 12 ms on localhost (2/2) -2016-05-24 22:20:44 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.023364 s -2016-05-24 22:20:44 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=782615, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=782615, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:44 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=737266, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=749050, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 22:20:44 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 22:20:44 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:44 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 10 ms on localhost (1/2) -2016-05-24 22:20:44 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 11 ms on localhost (2/2) -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 22:20:44 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:20:44 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.028859 s -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=703588, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=709180, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 22:20:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:44 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=660351, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=660351, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 22:20:44 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:44 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 8 ms on localhost (1/2) -2016-05-24 22:20:44 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 11 ms on localhost (2/2) -2016-05-24 22:20:44 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 22:20:44 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.022354 s -2016-05-24 22:20:44 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=614897, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3623) called with curMem=611284, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:44 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:44 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 22:20:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:44 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=562493, maxMem=766075207 -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=562493, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 22:20:44 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:44 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 10 ms on localhost (1/2) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 10 ms on localhost (2/2) -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 22:20:44 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:20:44 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.023729 s -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=565977, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=571569, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:44 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=575175, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=586959, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 22:20:44 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:44 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 5 ms on localhost (1/2) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 6 ms on localhost (2/2) -2016-05-24 22:20:44 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 22:20:44 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.017603 s -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=590443, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3626) called with curMem=596035, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:44 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=599661, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=611445, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 22:20:44 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:44 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 6 ms on localhost (1/2) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 6 ms on localhost (2/2) -2016-05-24 22:20:44 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 22:20:44 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.014638 s -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=614929, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=620521, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:44 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=624133, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=635917, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 22:20:44 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:44 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 5 ms on localhost (1/2) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 6 ms on localhost (2/2) -2016-05-24 22:20:44 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 22:20:44 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.014574 s -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=639400, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=644992, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:44 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=648604, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=660388, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:44 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 22:20:44 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 15 ms on localhost (1/2) -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:44 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 38 ms on localhost (2/2) -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 22:20:44 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.043 s -2016-05-24 22:20:44 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.357942 s -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=663872, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=669464, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:44 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=673076, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=684860, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 22:20:44 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:44 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 10 ms on localhost (1/2) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 10 ms on localhost (2/2) -2016-05-24 22:20:44 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 22:20:44 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.015451 s -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=688344, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=693936, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:44 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=697548, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=709332, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 22:20:44 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:44 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 10 ms on localhost (1/2) -2016-05-24 22:20:44 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 10 ms on localhost (2/2) -2016-05-24 22:20:44 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 22:20:44 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.019176 s -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=712815, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=718407, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:44 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=722019, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=733803, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 22:20:44 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:44 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 12 ms on localhost (1/2) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 12 ms on localhost (2/2) -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 22:20:44 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:20:44 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.021234 s -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=737286, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=742878, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:44 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=746490, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=758274, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:44 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 22:20:44 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:44 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 7968 bytes result sent to driver -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 0 ms on localhost (1/2) -2016-05-24 22:20:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 0 ms on localhost (2/2) -2016-05-24 22:20:44 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.000 s -2016-05-24 22:20:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 22:20:44 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.014437 s -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=761758, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=767350, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:44 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:44 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:44 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:44 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:44 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=770964, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:44 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=782748, maxMem=766075207 -2016-05-24 22:20:44 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 5 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 8 ms on localhost (2/2) -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.203957 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=786232, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=791824, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=795436, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3486) called with curMem=807220, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 11 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 14 ms on localhost (2/2) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.031037 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=810706, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=816298, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=819910, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=831694, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 8 ms on localhost (1/2) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 9 ms on localhost (2/2) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.025028 s -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=795434, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=797412, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[93] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=746489, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=754790, maxMem=766075207 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[93] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 14 ms on localhost (1/2) -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 13 ms on localhost (2/2) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.027147 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=718405, maxMem=766075207 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=718405, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[95] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=673074, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=681374, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[95] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 9 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 9 ms on localhost (2/2) -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.020830 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=644982, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=629714, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[97] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=575176, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=586960, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[97] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 9 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 9 ms on localhost (2/2) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.027642 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=590443, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3618) called with curMem=596035, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 91 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 47 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[99] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=599653, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=611437, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 47 (MapPartitionsRDD[99] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 93, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 94, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 93) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 94) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 93). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 94). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 94) in 7 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 93) in 8 ms on localhost (2/2) -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 47 finished: treeAggregate at GradientDescent.scala:225, took 0.017479 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=614921, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=620513, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 93 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 48 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 48(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[101] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=624127, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=635911, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 48 (MapPartitionsRDD[101] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 95, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 96, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 96) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 95) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 95). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 96). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 95) in 6 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 96) in 5 ms on localhost (2/2) -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 48 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 48 finished: treeAggregate at GradientDescent.scala:225, took 0.015078 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=639394, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3618) called with curMem=644986, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 95 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 49 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 49(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[103] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=648604, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=660388, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 49 (MapPartitionsRDD[103] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 97, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 98, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 97) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 98) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 98). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 97). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 98) in 6 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 97) in 7 ms on localhost (2/2) -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 49 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 49 finished: treeAggregate at GradientDescent.scala:225, took 0.017527 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=663871, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3622) called with curMem=669463, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 97 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 50 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[105] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=673085, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=684869, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 50 (MapPartitionsRDD[105] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 6 ms on localhost (1/2) -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 10 ms on localhost (2/2) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at GradientDescent.scala:225, took 0.019715 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=688351, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=693943, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 99 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 51 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[107] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=697556, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=709340, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 51 (MapPartitionsRDD[107] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 8 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 8 ms on localhost (2/2) -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at GradientDescent.scala:225, took 0.020661 s -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=712821, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3628) called with curMem=718413, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 101 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 52 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 52(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 52 (MapPartitionsRDD[109] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=722041, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=733825, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 52 (MapPartitionsRDD[109] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 103, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 104, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 104) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 103) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 104). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 103). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 104) in 6 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 103) in 7 ms on localhost (2/2) -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 52 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 52 finished: treeAggregate at GradientDescent.scala:225, took 0.017082 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=737308, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=742900, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 103 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 53 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 53(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[111] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=746532, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=758316, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 104 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[111] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 105, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 106, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 105) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 106) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 105). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 106). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 105) in 6 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 106) in 6 ms on localhost (2/2) -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 53 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 53 finished: treeAggregate at GradientDescent.scala:225, took 0.016479 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=761797, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=767389, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 105 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 54 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[113] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=771002, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=782786, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 106 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 54 (MapPartitionsRDD[113] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 107) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 108) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 107). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 108). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 107) in 5 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 108) in 6 ms on localhost (2/2) -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 54 finished: treeAggregate at GradientDescent.scala:225, took 0.015765 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=786269, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=791861, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 107 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 55 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[115] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=795493, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=807277, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 108 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 55 (MapPartitionsRDD[115] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 109, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 110, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 110) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 109) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 109). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 110). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 110) in 7 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 109) in 7 ms on localhost (2/2) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 55 finished: treeAggregate at GradientDescent.scala:225, took 0.015954 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=810759, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=816351, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 109 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 56 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[117] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=819964, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=831748, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 110 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 56 (MapPartitionsRDD[117] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 111, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 112, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 112) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 111) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 112). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 111). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 112) in 4 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 111) in 6 ms on localhost (2/2) -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 56 finished: treeAggregate at GradientDescent.scala:225, took 0.016393 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=835230, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=840822, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 111 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 57 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 57(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[119] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=844435, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=856219, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 112 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 57 (MapPartitionsRDD[119] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 113, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 114, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 114) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 113) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 114). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 113). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 114) in 5 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 113) in 6 ms on localhost (2/2) -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 57 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 57 finished: treeAggregate at GradientDescent.scala:225, took 0.015685 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=859700, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=865292, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 113 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 58 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 58(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 58 (MapPartitionsRDD[121] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=868904, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=880688, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 114 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 58 (MapPartitionsRDD[121] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 115, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 116, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 116) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 115) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 115). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 116). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 115) in 5 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 116) in 6 ms on localhost (2/2) -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 58 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 58 finished: treeAggregate at GradientDescent.scala:225, took 0.020005 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=884168, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_115 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3620) called with curMem=889760, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_115_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_115_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 115 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 59 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 59(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 59 (MapPartitionsRDD[123] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=893380, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_116 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3478) called with curMem=905164, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_116_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_116_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 116 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 59 (MapPartitionsRDD[123] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 117, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 118, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 117) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 118) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 118). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 117). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 118) in 5 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 117) in 7 ms on localhost (2/2) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 59 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 59 finished: treeAggregate at GradientDescent.scala:225, took 0.018612 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=908642, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_117 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=914234, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_117_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_117_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 117 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 60 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 60(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 60 (MapPartitionsRDD[125] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=917846, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_118 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=929630, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_118_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_118_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 118 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 60 (MapPartitionsRDD[125] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 60.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 60.0 (TID 119, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 60.0 (TID 120, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 60.0 (TID 120) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 60.0 (TID 119) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 60.0 (TID 120). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 60.0 (TID 119). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 60.0 (TID 120) in 6 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 60.0 (TID 119) in 7 ms on localhost (2/2) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 60.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 60 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 60 finished: treeAggregate at GradientDescent.scala:225, took 0.020495 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=933113, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_119 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3616) called with curMem=938705, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_119_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_119_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 119 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 61 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 61(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 61 (MapPartitionsRDD[127] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=942321, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_120 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=954105, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_120_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_120_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 120 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 61 (MapPartitionsRDD[127] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 121, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 122, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 121) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 122) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 122). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 121). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 122) in 6 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 121) in 7 ms on localhost (2/2) -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 61 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 61 finished: treeAggregate at GradientDescent.scala:225, took 0.038847 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=957588, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_121 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=963180, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_121_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_121_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 121 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 62 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 62(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[129] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=966792, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_122 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=978576, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_122_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_122_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 122 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 62 (MapPartitionsRDD[129] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 123, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 62.0 (TID 124, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 62.0 (TID 124) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 123) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 62.0 (TID 124). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 123). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 62.0 (TID 124) in 5 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 123) in 6 ms on localhost (2/2) -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 62 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 62 finished: treeAggregate at GradientDescent.scala:225, took 0.017673 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=982059, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_123 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3630) called with curMem=987651, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_123_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_123_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 123 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 63 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 63(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 63 (MapPartitionsRDD[131] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=991281, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_124 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=1003065, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_124_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_124_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 124 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 63 (MapPartitionsRDD[131] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 63.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 63.0 (TID 125, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 63.0 (TID 126, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 63.0 (TID 126) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 63.0 (TID 125) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 63.0 (TID 126). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 63.0 (TID 125). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 63.0 (TID 126) in 5 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 63.0 (TID 125) in 5 ms on localhost (2/2) -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 63 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 63.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 63 finished: treeAggregate at GradientDescent.scala:225, took 0.015736 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1006545, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_125 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1012137, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_125_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_125_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 125 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 64 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 64(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 64 (MapPartitionsRDD[133] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1015750, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_126 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1027534, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_126_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_126_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 126 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 64 (MapPartitionsRDD[133] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 64.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 64.0 (TID 127, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 64.0 (TID 128, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 64.0 (TID 128) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 64.0 (TID 127) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 64.0 (TID 128). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 64.0 (TID 127). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 64.0 (TID 128) in 6 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 64.0 (TID 127) in 6 ms on localhost (2/2) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 64.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 64 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 64 finished: treeAggregate at GradientDescent.scala:225, took 0.016102 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1031017, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_127 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1036609, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_127_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_127_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 127 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_126_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 65 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 65(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[135] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_125_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1015749, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_128 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1012269, maxMem=766075207 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_124_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_128_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_128_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 128 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 65 (MapPartitionsRDD[135] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 2 tasks -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_123_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 129, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 130, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 130) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 129) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_122_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 129). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 129) in 7 ms on localhost (1/2) -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 130). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_121_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 65 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 130) in 8 ms on localhost (2/2) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 65 finished: treeAggregate at GradientDescent.scala:225, took 0.019102 s -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=966792, maxMem=766075207 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_120_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_129 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=972384, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_129_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_119_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_129_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 129 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_118_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_117_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_116_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 66 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 66(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 66 (MapPartitionsRDD[137] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_115_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=917844, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_130 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=914364, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_130_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_114_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_130_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 130 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 66 (MapPartitionsRDD[137] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Adding task set 66.0 with 2 tasks -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 66.0 (TID 131, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_113_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 66.0 (TID 132, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:45 INFO Executor:59 - Running task 1.0 in stage 66.0 (TID 132) -2016-05-24 22:20:45 INFO Executor:59 - Running task 0.0 in stage 66.0 (TID 131) -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_112_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-05-24 22:20:45 INFO Executor:59 - Finished task 1.0 in stage 66.0 (TID 132). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO Executor:59 - Finished task 0.0 in stage 66.0 (TID 131). 7968 bytes result sent to driver -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 66.0 (TID 132) in 5 ms on localhost (1/2) -2016-05-24 22:20:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 66.0 (TID 131) in 7 ms on localhost (2/2) -2016-05-24 22:20:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 66.0, whose tasks have all completed, from pool -2016-05-24 22:20:45 INFO DAGScheduler:59 - ResultStage 66 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:45 INFO DAGScheduler:59 - Job 66 finished: treeAggregate at GradientDescent.scala:225, took 0.016630 s -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_111_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=884173, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_131 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3633) called with curMem=886283, maxMem=766075207 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_110_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_131_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Added broadcast_131_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Created broadcast 131 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_109_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_108_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_107_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Got job 67 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:45 INFO DAGScheduler:59 - Final stage: ResultStage 67(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_106_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-05-24 22:20:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:45 INFO DAGScheduler:59 - Submitting ResultStage 67 (MapPartitionsRDD[139] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=825557, maxMem=766075207 -2016-05-24 22:20:45 INFO BlockManagerInfo:59 - Removed broadcast_105_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_132 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:45 INFO MemoryStore:59 - ensureFreeSpace(3486) called with curMem=831749, maxMem=766075207 -2016-05-24 22:20:45 INFO MemoryStore:59 - Block broadcast_132_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_132_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 132 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_104_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 67 (MapPartitionsRDD[139] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 67.0 with 2 tasks -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 67.0 (TID 133, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 67.0 (TID 134, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 67.0 (TID 133) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 67.0 (TID 134) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_103_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 67.0 (TID 134). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 67.0 (TID 133). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 67.0 (TID 134) in 6 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 67.0 (TID 133) in 7 ms on localhost (2/2) -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 67 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 67.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 67 finished: treeAggregate at GradientDescent.scala:225, took 0.016458 s -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_102_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=795479, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_133 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=791851, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_133_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_101_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_133_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 133 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_100_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_99_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_98_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 68 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 68(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 68 (MapPartitionsRDD[141] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_97_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=746514, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_134 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=754815, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_134_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_96_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_134_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 134 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 68 (MapPartitionsRDD[141] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 68.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 68.0 (TID 135, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 68.0 (TID 136, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 68.0 (TID 136) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 68.0 (TID 135) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_95_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 68.0 (TID 135). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_94_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 68.0 (TID 136). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 68.0 (TID 135) in 6 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 68.0 (TID 136) in 5 ms on localhost (2/2) -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 68 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 68.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 68 finished: treeAggregate at GradientDescent.scala:225, took 0.015567 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=718424, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_135 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_93_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3608) called with curMem=718424, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_135_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_135_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 135 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_92_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_91_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_90_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_89_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 69 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 69(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 69 (MapPartitionsRDD[143] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_88_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=657814, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_136 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=665986, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_136_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_87_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_136_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 136 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 69 (MapPartitionsRDD[143] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 69.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 69.0 (TID 137, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 69.0 (TID 138, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 69.0 (TID 138) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 69.0 (TID 137) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 69.0 (TID 138). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 69.0 (TID 137). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 69.0 (TID 137) in 7 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 69.0 (TID 138) in 8 ms on localhost (2/2) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 69 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 69.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 69 finished: treeAggregate at GradientDescent.scala:225, took 0.032969 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=624141, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_137 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=629733, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_137_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_137_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 137 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 70 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 70(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 70 (MapPartitionsRDD[145] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=633339, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_138 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=645123, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_138_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_138_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 138 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 70 (MapPartitionsRDD[145] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 70.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 70.0 (TID 139, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 70.0 (TID 140, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 70.0 (TID 139) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 70.0 (TID 140) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 70.0 (TID 140). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 70.0 (TID 139). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 70.0 (TID 140) in 7 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 70.0 (TID 139) in 8 ms on localhost (2/2) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 70.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 70 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 70 finished: treeAggregate at GradientDescent.scala:225, took 0.021312 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=648605, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_139 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3619) called with curMem=654197, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_139_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_139_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 139 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 71 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 71(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 71 (MapPartitionsRDD[147] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=657816, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_140 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=669600, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_140_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_140_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 140 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 71 (MapPartitionsRDD[147] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 71.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 71.0 (TID 141, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 71.0 (TID 142, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 71.0 (TID 141) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 71.0 (TID 142) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 71.0 (TID 141). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 71.0 (TID 142). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 71.0 (TID 141) in 15 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 71.0 (TID 142) in 15 ms on localhost (2/2) -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 71 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 71.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 71 finished: treeAggregate at GradientDescent.scala:225, took 0.025778 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=673084, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_141 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3604) called with curMem=678676, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_141_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_141_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 141 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 72 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 72(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 72 (MapPartitionsRDD[149] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=682280, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_142 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=694064, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_142_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_142_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 142 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 72 (MapPartitionsRDD[149] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 72.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 72.0 (TID 143, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 72.0 (TID 144, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 72.0 (TID 143) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 72.0 (TID 144) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 72.0 (TID 143). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 72.0 (TID 144). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 72.0 (TID 143) in 7 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 72.0 (TID 144) in 9 ms on localhost (2/2) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 72.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 72 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 72 finished: treeAggregate at GradientDescent.scala:225, took 0.020181 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=697547, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_143 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=703139, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_143_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_143_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 143 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 73 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 73(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 73 (MapPartitionsRDD[151] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=706748, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_144 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=718532, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_144_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_144_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 144 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 73 (MapPartitionsRDD[151] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 73.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 73.0 (TID 145, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 73.0 (TID 146, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 73.0 (TID 146) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 73.0 (TID 145) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 73.0 (TID 146). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 73.0 (TID 145). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 73.0 (TID 146) in 5 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 73.0 (TID 145) in 6 ms on localhost (2/2) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 73.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 73 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 73 finished: treeAggregate at GradientDescent.scala:225, took 0.018399 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=722016, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_145 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=727608, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_145_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_145_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 145 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 74 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 74(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 74 (MapPartitionsRDD[153] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=731214, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_146 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=742998, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_146_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_146_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 146 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 74 (MapPartitionsRDD[153] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 74.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 74.0 (TID 147, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 74.0 (TID 148, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 74.0 (TID 147) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 74.0 (TID 148) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 74.0 (TID 148). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 74.0 (TID 147). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 74.0 (TID 148) in 4 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 74.0 (TID 147) in 5 ms on localhost (2/2) -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 74 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 74.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 74 finished: treeAggregate at GradientDescent.scala:225, took 0.012084 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=746482, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_147 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3624) called with curMem=752074, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_147_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_147_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 147 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 75 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 75(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 75 (MapPartitionsRDD[155] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=755698, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_148 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=767482, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_148_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_148_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 148 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 75 (MapPartitionsRDD[155] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 75.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 75.0 (TID 149, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 75.0 (TID 150, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 75.0 (TID 150) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 75.0 (TID 149) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 75.0 (TID 150). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 75.0 (TID 149). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 75.0 (TID 150) in 5 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 75.0 (TID 149) in 6 ms on localhost (2/2) -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 75 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 75.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 75 finished: treeAggregate at GradientDescent.scala:225, took 0.014601 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=770964, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_149 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3626) called with curMem=776556, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_149_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_149_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 149 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 76 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 76(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 76 (MapPartitionsRDD[157] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=780182, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_150 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3478) called with curMem=791966, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_150_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_150_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 150 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 76 (MapPartitionsRDD[157] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 151, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 152, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 151) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 152) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 151). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 152). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 152) in 4 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 151) in 5 ms on localhost (2/2) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 76 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 76 finished: treeAggregate at GradientDescent.scala:225, took 0.013941 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=795444, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_151 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3611) called with curMem=801036, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_151_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_151_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 151 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 77 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 77(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[159] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=804647, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_152 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=816431, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_152_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_152_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 152 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 77 (MapPartitionsRDD[159] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 153, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 154, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 153) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 154) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 153). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 154). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 153) in 4 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 154) in 4 ms on localhost (2/2) -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 77 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 77 finished: treeAggregate at GradientDescent.scala:225, took 0.012788 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=819915, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_153 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=825507, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_153_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_153_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 153 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 78 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 78(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 78 (MapPartitionsRDD[161] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=829119, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_154 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=840903, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_154_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_154_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 154 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 78 (MapPartitionsRDD[161] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 155, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 156, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 156) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 156). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 156) in 5 ms on localhost (1/2) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 155) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 155). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 155) in 10 ms on localhost (2/2) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 78 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 78 finished: treeAggregate at GradientDescent.scala:225, took 0.018726 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=844386, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_155 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=849978, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_155_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_155_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 155 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 79 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 79(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 79 (MapPartitionsRDD[163] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=853590, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_156 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=865374, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_156_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_156_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 156 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 79 (MapPartitionsRDD[163] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 79.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 79.0 (TID 157, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 79.0 (TID 158, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 79.0 (TID 158) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 79.0 (TID 157) -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 79.0 (TID 158). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 79.0 (TID 158) in 6 ms on localhost (1/2) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 79.0 (TID 157). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 79.0 (TID 157) in 11 ms on localhost (2/2) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 79.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 79 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 79 finished: treeAggregate at GradientDescent.scala:225, took 0.024511 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=868858, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_157 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=874450, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_157_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_157_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 157 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 80 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 80(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 80 (MapPartitionsRDD[165] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=878062, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_158 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=889846, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_158_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_158_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 158 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 80 (MapPartitionsRDD[165] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 80.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 80.0 (TID 159, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 80.0 (TID 160, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 80.0 (TID 160) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 80.0 (TID 159) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 80.0 (TID 160). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 80.0 (TID 159). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 80.0 (TID 160) in 9 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 80.0 (TID 159) in 12 ms on localhost (2/2) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 80.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 80 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 80 finished: treeAggregate at GradientDescent.scala:225, took 0.021917 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=893329, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_159 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=898921, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_159_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_159_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 159 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 81 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 81(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 81 (MapPartitionsRDD[167] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=902527, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_160 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=914311, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_160_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_160_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 160 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 81 (MapPartitionsRDD[167] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 81.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 81.0 (TID 161, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 81.0 (TID 162, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 81.0 (TID 161) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 81.0 (TID 162) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 81.0 (TID 162). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 81.0 (TID 161). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 81.0 (TID 162) in 6 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 81.0 (TID 161) in 7 ms on localhost (2/2) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 81.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 81 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 81 finished: treeAggregate at GradientDescent.scala:225, took 0.017748 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=917795, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_161 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=923387, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_161_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_161_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 161 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 82 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 82(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 82 (MapPartitionsRDD[169] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=926999, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_162 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=938783, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_162_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_162_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 162 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 82 (MapPartitionsRDD[169] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 82.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 82.0 (TID 163, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 82.0 (TID 164, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 82.0 (TID 164) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 82.0 (TID 163) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 82.0 (TID 164). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 82.0 (TID 163). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 82.0 (TID 164) in 7 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 82.0 (TID 163) in 8 ms on localhost (2/2) -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 82 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 82.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 82 finished: treeAggregate at GradientDescent.scala:225, took 0.018773 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=942266, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_163 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=947858, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_163_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_163_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 163 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 83 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 83(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 83 (MapPartitionsRDD[171] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=951472, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_164 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=963256, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_164_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_164_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 164 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 83 (MapPartitionsRDD[171] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 83.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 83.0 (TID 165, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 83.0 (TID 166, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 83.0 (TID 165) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 83.0 (TID 166) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 83.0 (TID 166). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 83.0 (TID 165). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 83.0 (TID 166) in 5 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 83.0 (TID 165) in 6 ms on localhost (2/2) -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 83 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 83.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 83 finished: treeAggregate at GradientDescent.scala:225, took 0.016784 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=966740, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_165 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=972332, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_165_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_165_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 165 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 84 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 84(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 84 (MapPartitionsRDD[173] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=975944, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_166 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=987728, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_166_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_166_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 166 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 84 (MapPartitionsRDD[173] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 84.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 84.0 (TID 167, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 84.0 (TID 168, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 84.0 (TID 167) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 84.0 (TID 168) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 84.0 (TID 168). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 84.0 (TID 167). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 84.0 (TID 168) in 5 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 84.0 (TID 167) in 7 ms on localhost (2/2) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 84.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 84 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 84 finished: treeAggregate at GradientDescent.scala:225, took 0.018956 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=991212, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_167 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3620) called with curMem=996804, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_167_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_167_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 167 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 85 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 85(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 85 (MapPartitionsRDD[175] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1000424, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_168 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1012208, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_168_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_168_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 168 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 85 (MapPartitionsRDD[175] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 85.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 85.0 (TID 169, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 85.0 (TID 170, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 85.0 (TID 169) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 85.0 (TID 170) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 85.0 (TID 170). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 85.0 (TID 169). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 85.0 (TID 170) in 7 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 85.0 (TID 169) in 8 ms on localhost (2/2) -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 85 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 85.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 85 finished: treeAggregate at GradientDescent.scala:225, took 0.019900 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1015692, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_169 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3621) called with curMem=1021284, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_169_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_169_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 169 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_168_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 86 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_167_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_166_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 85 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 86 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 86(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 86 (MapPartitionsRDD[177] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_165_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=975953, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_170 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=972469, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_170_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_164_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_170_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 170 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 86 (MapPartitionsRDD[177] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 86.0 with 2 tasks -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 84 -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 86.0 (TID 171, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 86.0 (TID 172, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 86.0 (TID 171) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_163_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 86.0 (TID 172) -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 86.0 (TID 171). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_162_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 86.0 (TID 172). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 86.0 (TID 171) in 12 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 86.0 (TID 172) in 12 ms on localhost (2/2) -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 86 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 86.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 83 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 86 finished: treeAggregate at GradientDescent.scala:225, took 0.024966 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=947868, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_171 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_161_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=947868, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_171_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_171_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 171 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_160_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 82 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_159_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_158_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 81 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 87 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 87(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 87 (MapPartitionsRDD[179] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_157_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=902543, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_172 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=910843, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_172_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_156_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_172_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 172 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 80 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 87 (MapPartitionsRDD[179] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 87.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 87.0 (TID 173, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 87.0 (TID 174, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 87.0 (TID 174) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 87.0 (TID 174). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_155_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 87.0 (TID 173) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 87.0 (TID 174) in 12 ms on localhost (1/2) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 87.0 (TID 173). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_154_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 87.0 (TID 173) in 20 ms on localhost (2/2) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 87.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 87 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 79 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 87 finished: treeAggregate at GradientDescent.scala:225, took 0.035277 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=878071, maxMem=766075207 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_153_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_173 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=874459, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_173_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_173_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_152_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 78 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_151_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 173 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_150_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 77 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_149_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_148_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 88 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 88(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 88 (MapPartitionsRDD[181] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=813855, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_174 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3485) called with curMem=825639, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_174_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_174_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 174 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 88 (MapPartitionsRDD[181] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 88.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 88.0 (TID 175, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 88.0 (TID 176, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 88.0 (TID 176) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 88.0 (TID 175) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 76 -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 88.0 (TID 176). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 88.0 (TID 175). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_147_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 88.0 (TID 176) in 11 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 88.0 (TID 175) in 13 ms on localhost (2/2) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 88.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 88 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 88 finished: treeAggregate at GradientDescent.scala:225, took 0.029892 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=819908, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_175 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3615) called with curMem=825500, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_175_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_175_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 175 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_146_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 75 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_145_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_144_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 74 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 89 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 89(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 89 (MapPartitionsRDD[183] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_143_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=780180, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_176 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=788481, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_176_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_142_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_176_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 73 -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 176 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 89 (MapPartitionsRDD[183] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 89.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 89.0 (TID 177, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 89.0 (TID 178, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 89.0 (TID 177) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 89.0 (TID 178) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_141_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 89.0 (TID 178). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 89.0 (TID 177). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_140_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 89.0 (TID 178) in 6 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 89.0 (TID 177) in 7 ms on localhost (2/2) -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 89 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 72 -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 89.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 89 finished: treeAggregate at GradientDescent.scala:225, took 0.015771 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=752094, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_177 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_139_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=752094, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_177_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_177_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 177 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_138_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 71 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_137_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_136_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 70 -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 90 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 90(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 90 (MapPartitionsRDD[185] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_135_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=694990, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_178 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=703290, maxMem=766075207 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_134_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_178_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 69 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_178_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 178 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 90 (MapPartitionsRDD[185] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 90.0 with 2 tasks -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_133_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 90.0 (TID 179, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 90.0 (TID 180, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 90.0 (TID 180) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 90.0 (TID 179) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_132_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 90.0 (TID 179). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 90.0 (TID 180). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 90.0 (TID 179) in 7 ms on localhost (1/2) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_131_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 90 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 90.0 (TID 180) in 7 ms on localhost (2/2) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 90.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 90 finished: treeAggregate at GradientDescent.scala:225, took 0.017001 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=673073, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_179 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=678665, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_179_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_179_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 179 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_130_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_129_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Removed broadcast_128_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 91 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 91(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 91 (MapPartitionsRDD[187] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=642539, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_180 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=654323, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_180_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_180_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 180 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 91 (MapPartitionsRDD[187] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 91.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 91.0 (TID 181, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 91.0 (TID 182, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 91.0 (TID 182) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 91.0 (TID 181) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 91.0 (TID 181). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 91.0 (TID 182). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 91.0 (TID 181) in 6 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 91.0 (TID 182) in 6 ms on localhost (2/2) -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 91 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 91.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 91 finished: treeAggregate at GradientDescent.scala:225, took 0.018075 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=657806, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_181 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3608) called with curMem=663398, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_181_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_181_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 181 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 92 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 92(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 92 (MapPartitionsRDD[189] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=667006, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_182 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=678790, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_182_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_182_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 182 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 92 (MapPartitionsRDD[189] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 92.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 92.0 (TID 183, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 92.0 (TID 184, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 92.0 (TID 184) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 92.0 (TID 183) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 92.0 (TID 183). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 92.0 (TID 184). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 92.0 (TID 183) in 9 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 92.0 (TID 184) in 9 ms on localhost (2/2) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 92.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 92 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 92 finished: treeAggregate at GradientDescent.scala:225, took 0.019024 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=682274, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_183 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=687866, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_183_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_183_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 183 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Got job 93 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 93(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting ResultStage 93 (MapPartitionsRDD[191] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=691480, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_184 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=703264, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_184_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_184_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 184 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 93 (MapPartitionsRDD[191] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Adding task set 93.0 with 2 tasks -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 93.0 (TID 185, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 93.0 (TID 186, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:46 INFO Executor:59 - Running task 1.0 in stage 93.0 (TID 186) -2016-05-24 22:20:46 INFO Executor:59 - Running task 0.0 in stage 93.0 (TID 185) -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:46 INFO Executor:59 - Finished task 1.0 in stage 93.0 (TID 186). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO Executor:59 - Finished task 0.0 in stage 93.0 (TID 185). 7968 bytes result sent to driver -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 93.0 (TID 186) in 5 ms on localhost (1/2) -2016-05-24 22:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 93.0 (TID 185) in 6 ms on localhost (2/2) -2016-05-24 22:20:46 INFO DAGScheduler:59 - ResultStage 93 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 93.0, whose tasks have all completed, from pool -2016-05-24 22:20:46 INFO DAGScheduler:59 - Job 93 finished: treeAggregate at GradientDescent.scala:225, took 0.018041 s -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=706747, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_185 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=712339, maxMem=766075207 -2016-05-24 22:20:46 INFO MemoryStore:59 - Block broadcast_185_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:46 INFO BlockManagerInfo:59 - Added broadcast_185_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:46 INFO SparkContext:59 - Created broadcast 185 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Got job 94 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:47 INFO DAGScheduler:59 - Final stage: ResultStage 94(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting ResultStage 94 (MapPartitionsRDD[193] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=715951, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_186 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=727735, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_186_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_186_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 186 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 94 (MapPartitionsRDD[193] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Adding task set 94.0 with 2 tasks -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 94.0 (TID 187, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 94.0 (TID 188, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:47 INFO Executor:59 - Running task 0.0 in stage 94.0 (TID 187) -2016-05-24 22:20:47 INFO Executor:59 - Running task 1.0 in stage 94.0 (TID 188) -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:47 INFO Executor:59 - Finished task 0.0 in stage 94.0 (TID 187). 7968 bytes result sent to driver -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 94.0 (TID 187) in 7 ms on localhost (1/2) -2016-05-24 22:20:47 INFO Executor:59 - Finished task 1.0 in stage 94.0 (TID 188). 7968 bytes result sent to driver -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 94.0 (TID 188) in 8 ms on localhost (2/2) -2016-05-24 22:20:47 INFO DAGScheduler:59 - ResultStage 94 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 94.0, whose tasks have all completed, from pool -2016-05-24 22:20:47 INFO DAGScheduler:59 - Job 94 finished: treeAggregate at GradientDescent.scala:225, took 0.021771 s -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=731219, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_187 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=736811, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_187_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_187_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 187 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Got job 95 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:47 INFO DAGScheduler:59 - Final stage: ResultStage 95(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting ResultStage 95 (MapPartitionsRDD[195] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=740424, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_188 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=752208, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_188_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_188_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 188 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 95 (MapPartitionsRDD[195] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Adding task set 95.0 with 2 tasks -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 95.0 (TID 189, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 95.0 (TID 190, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:47 INFO Executor:59 - Running task 1.0 in stage 95.0 (TID 190) -2016-05-24 22:20:47 INFO Executor:59 - Running task 0.0 in stage 95.0 (TID 189) -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:47 INFO Executor:59 - Finished task 0.0 in stage 95.0 (TID 189). 7968 bytes result sent to driver -2016-05-24 22:20:47 INFO Executor:59 - Finished task 1.0 in stage 95.0 (TID 190). 7968 bytes result sent to driver -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 95.0 (TID 189) in 7 ms on localhost (1/2) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 95.0 (TID 190) in 8 ms on localhost (2/2) -2016-05-24 22:20:47 INFO DAGScheduler:59 - ResultStage 95 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 95.0, whose tasks have all completed, from pool -2016-05-24 22:20:47 INFO DAGScheduler:59 - Job 95 finished: treeAggregate at GradientDescent.scala:225, took 0.022369 s -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=755691, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_189 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(3634) called with curMem=761283, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_189_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_189_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 189 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Got job 96 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:47 INFO DAGScheduler:59 - Final stage: ResultStage 96(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting ResultStage 96 (MapPartitionsRDD[197] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=764917, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_190 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=776701, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_190_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_190_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 190 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 96 (MapPartitionsRDD[197] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Adding task set 96.0 with 2 tasks -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 96.0 (TID 191, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 96.0 (TID 192, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:47 INFO Executor:59 - Running task 1.0 in stage 96.0 (TID 192) -2016-05-24 22:20:47 INFO Executor:59 - Running task 0.0 in stage 96.0 (TID 191) -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:47 INFO Executor:59 - Finished task 1.0 in stage 96.0 (TID 192). 7968 bytes result sent to driver -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:47 INFO Executor:59 - Finished task 0.0 in stage 96.0 (TID 191). 7968 bytes result sent to driver -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 96.0 (TID 191) in 13 ms on localhost (1/2) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 96.0 (TID 192) in 13 ms on localhost (2/2) -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 96.0, whose tasks have all completed, from pool -2016-05-24 22:20:47 INFO DAGScheduler:59 - ResultStage 96 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:20:47 INFO DAGScheduler:59 - Job 96 finished: treeAggregate at GradientDescent.scala:225, took 0.024760 s -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=780185, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_191 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(3619) called with curMem=785777, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_191_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_191_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 191 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Got job 97 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:47 INFO DAGScheduler:59 - Final stage: ResultStage 97(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting ResultStage 97 (MapPartitionsRDD[199] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=789396, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_192 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=801180, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_192_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_192_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 192 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 97 (MapPartitionsRDD[199] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Adding task set 97.0 with 2 tasks -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 97.0 (TID 193, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 97.0 (TID 194, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:47 INFO Executor:59 - Running task 0.0 in stage 97.0 (TID 193) -2016-05-24 22:20:47 INFO Executor:59 - Running task 1.0 in stage 97.0 (TID 194) -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:47 INFO Executor:59 - Finished task 1.0 in stage 97.0 (TID 194). 7968 bytes result sent to driver -2016-05-24 22:20:47 INFO Executor:59 - Finished task 0.0 in stage 97.0 (TID 193). 7968 bytes result sent to driver -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 97.0 (TID 194) in 6 ms on localhost (1/2) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 97.0 (TID 193) in 7 ms on localhost (2/2) -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 97.0, whose tasks have all completed, from pool -2016-05-24 22:20:47 INFO DAGScheduler:59 - ResultStage 97 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:20:47 INFO DAGScheduler:59 - Job 97 finished: treeAggregate at GradientDescent.scala:225, took 0.023333 s -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=804663, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_193 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=810255, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_193_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_193_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 193 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Got job 98 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:47 INFO DAGScheduler:59 - Final stage: ResultStage 98(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting ResultStage 98 (MapPartitionsRDD[201] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=813867, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_194 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=825651, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_194_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_194_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 194 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 98 (MapPartitionsRDD[201] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Adding task set 98.0 with 2 tasks -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 98.0 (TID 195, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 98.0 (TID 196, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:47 INFO Executor:59 - Running task 1.0 in stage 98.0 (TID 196) -2016-05-24 22:20:47 INFO Executor:59 - Running task 0.0 in stage 98.0 (TID 195) -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:47 INFO Executor:59 - Finished task 1.0 in stage 98.0 (TID 196). 7968 bytes result sent to driver -2016-05-24 22:20:47 INFO Executor:59 - Finished task 0.0 in stage 98.0 (TID 195). 7968 bytes result sent to driver -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 98.0 (TID 196) in 5 ms on localhost (1/2) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 98.0 (TID 195) in 6 ms on localhost (2/2) -2016-05-24 22:20:47 INFO DAGScheduler:59 - ResultStage 98 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 98.0, whose tasks have all completed, from pool -2016-05-24 22:20:47 INFO DAGScheduler:59 - Job 98 finished: treeAggregate at GradientDescent.scala:225, took 0.013707 s -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=829134, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_195 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=834726, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_195_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_195_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 195 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Got job 99 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:47 INFO DAGScheduler:59 - Final stage: ResultStage 99(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting ResultStage 99 (MapPartitionsRDD[203] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=838335, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_196 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=850119, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_196_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_196_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 196 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 99 (MapPartitionsRDD[203] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Adding task set 99.0 with 2 tasks -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 99.0 (TID 197, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 99.0 (TID 198, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:47 INFO Executor:59 - Running task 1.0 in stage 99.0 (TID 198) -2016-05-24 22:20:47 INFO Executor:59 - Running task 0.0 in stage 99.0 (TID 197) -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:47 INFO Executor:59 - Finished task 0.0 in stage 99.0 (TID 197). 7968 bytes result sent to driver -2016-05-24 22:20:47 INFO Executor:59 - Finished task 1.0 in stage 99.0 (TID 198). 7968 bytes result sent to driver -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 99.0 (TID 197) in 6 ms on localhost (1/2) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 99.0 (TID 198) in 7 ms on localhost (2/2) -2016-05-24 22:20:47 INFO DAGScheduler:59 - ResultStage 99 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 99.0, whose tasks have all completed, from pool -2016-05-24 22:20:47 INFO DAGScheduler:59 - Job 99 finished: treeAggregate at GradientDescent.scala:225, took 0.018731 s -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=853603, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_197 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(3616) called with curMem=859195, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_197_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_197_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 197 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Got job 100 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:47 INFO DAGScheduler:59 - Final stage: ResultStage 100(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting ResultStage 100 (MapPartitionsRDD[205] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=862811, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_198 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=874595, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_198_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_198_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 198 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 100 (MapPartitionsRDD[205] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Adding task set 100.0 with 2 tasks -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 100.0 (TID 199, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 100.0 (TID 200, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:47 INFO Executor:59 - Running task 0.0 in stage 100.0 (TID 199) -2016-05-24 22:20:47 INFO Executor:59 - Running task 1.0 in stage 100.0 (TID 200) -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:47 INFO Executor:59 - Finished task 0.0 in stage 100.0 (TID 199). 7968 bytes result sent to driver -2016-05-24 22:20:47 INFO Executor:59 - Finished task 1.0 in stage 100.0 (TID 200). 7968 bytes result sent to driver -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 100.0 (TID 199) in 5 ms on localhost (1/2) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 100.0 (TID 200) in 5 ms on localhost (2/2) -2016-05-24 22:20:47 INFO DAGScheduler:59 - ResultStage 100 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 100.0, whose tasks have all completed, from pool -2016-05-24 22:20:47 INFO DAGScheduler:59 - Job 100 finished: treeAggregate at GradientDescent.scala:225, took 0.013383 s -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=878078, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_199 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=883670, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_199_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_199_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 199 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Got job 101 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:47 INFO DAGScheduler:59 - Final stage: ResultStage 101(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting ResultStage 101 (MapPartitionsRDD[207] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=887276, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_200 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=899060, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_200_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_200_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 200 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 101 (MapPartitionsRDD[207] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Adding task set 101.0 with 2 tasks -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 101.0 (TID 201, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 101.0 (TID 202, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:47 INFO Executor:59 - Running task 0.0 in stage 101.0 (TID 201) -2016-05-24 22:20:47 INFO Executor:59 - Running task 1.0 in stage 101.0 (TID 202) -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:47 INFO Executor:59 - Finished task 0.0 in stage 101.0 (TID 201). 7968 bytes result sent to driver -2016-05-24 22:20:47 INFO Executor:59 - Finished task 1.0 in stage 101.0 (TID 202). 7968 bytes result sent to driver -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 101.0 (TID 201) in 6 ms on localhost (1/2) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 101.0 (TID 202) in 6 ms on localhost (2/2) -2016-05-24 22:20:47 INFO DAGScheduler:59 - ResultStage 101 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 101.0, whose tasks have all completed, from pool -2016-05-24 22:20:47 INFO DAGScheduler:59 - Job 101 finished: treeAggregate at GradientDescent.scala:225, took 0.015229 s -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=902542, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_201 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=908134, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_201_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_201_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 201 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Got job 102 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:47 INFO DAGScheduler:59 - Final stage: ResultStage 102(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting ResultStage 102 (MapPartitionsRDD[209] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=911747, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_202 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=923531, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_202_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_202_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 202 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 102 (MapPartitionsRDD[209] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Adding task set 102.0 with 2 tasks -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 102.0 (TID 203, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 102.0 (TID 204, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:47 INFO Executor:59 - Running task 0.0 in stage 102.0 (TID 203) -2016-05-24 22:20:47 INFO Executor:59 - Running task 1.0 in stage 102.0 (TID 204) -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:47 INFO Executor:59 - Finished task 0.0 in stage 102.0 (TID 203). 7968 bytes result sent to driver -2016-05-24 22:20:47 INFO Executor:59 - Finished task 1.0 in stage 102.0 (TID 204). 7968 bytes result sent to driver -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 102.0 (TID 203) in 4 ms on localhost (1/2) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 102.0 (TID 204) in 5 ms on localhost (2/2) -2016-05-24 22:20:47 INFO DAGScheduler:59 - ResultStage 102 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 102.0, whose tasks have all completed, from pool -2016-05-24 22:20:47 INFO DAGScheduler:59 - Job 102 finished: treeAggregate at GradientDescent.scala:225, took 0.014351 s -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=927014, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_203 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=932606, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_203_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_203_piece0 in memory on localhost:56872 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 203 from broadcast at GradientDescent.scala:221 -2016-05-24 22:20:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Got job 103 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:20:47 INFO DAGScheduler:59 - Final stage: ResultStage 103(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting ResultStage 103 (MapPartitionsRDD[211] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=936218, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_204 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=948002, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_204_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_204_piece0 in memory on localhost:56872 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 204 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 103 (MapPartitionsRDD[211] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Adding task set 103.0 with 2 tasks -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 103.0 (TID 205, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 103.0 (TID 206, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:20:47 INFO Executor:59 - Running task 0.0 in stage 103.0 (TID 205) -2016-05-24 22:20:47 INFO Executor:59 - Running task 1.0 in stage 103.0 (TID 206) -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:20:47 INFO Executor:59 - Finished task 0.0 in stage 103.0 (TID 205). 7968 bytes result sent to driver -2016-05-24 22:20:47 INFO Executor:59 - Finished task 1.0 in stage 103.0 (TID 206). 7968 bytes result sent to driver -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 103.0 (TID 205) in 4 ms on localhost (1/2) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 103.0 (TID 206) in 4 ms on localhost (2/2) -2016-05-24 22:20:47 INFO DAGScheduler:59 - ResultStage 103 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 103.0, whose tasks have all completed, from pool -2016-05-24 22:20:47 INFO DAGScheduler:59 - Job 103 finished: treeAggregate at GradientDescent.scala:225, took 0.012253 s -2016-05-24 22:20:47 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 3363.579936927662, 3356.5316534793847, 3349.536449573466, 3342.59343622967, 3335.701748404667, 3328.860544103912, 3322.069003535364, 3315.3263283026713, 3308.6317406356134, 3301.984482655661 -2016-05-24 22:20:47 INFO SparkContext:59 - Starting job: sortByKey at BinaryClassificationMetrics.scala:156 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Registering RDD 212 (map at SVMFromSparkLearning.scala:44) -2016-05-24 22:20:47 INFO DAGScheduler:59 - Got job 104 (sortByKey at BinaryClassificationMetrics.scala:156) with 2 output partitions -2016-05-24 22:20:47 INFO DAGScheduler:59 - Final stage: ResultStage 105(sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:20:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 104) -2016-05-24 22:20:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 104) -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 104 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44), which has no missing parents -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=951485, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_205 stored as values in memory (estimated size 10.8 KB, free 729.7 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(6433) called with curMem=962493, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_205_piece0 stored as bytes in memory (estimated size 6.3 KB, free 729.7 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_205_piece0 in memory on localhost:56872 (size: 6.3 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 205 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 104 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44) -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Adding task set 104.0 with 2 tasks -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 104.0 (TID 207, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 104.0 (TID 208, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:20:47 INFO Executor:59 - Running task 0.0 in stage 104.0 (TID 207) -2016-05-24 22:20:47 INFO Executor:59 - Running task 1.0 in stage 104.0 (TID 208) -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:20:47 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_204_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO ContextCleaner:59 - Cleaned accumulator 104 -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_203_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_202_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO ContextCleaner:59 - Cleaned accumulator 103 -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_201_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_200_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO ContextCleaner:59 - Cleaned accumulator 102 -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_199_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_198_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO ContextCleaner:59 - Cleaned accumulator 101 -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_197_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_196_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO ContextCleaner:59 - Cleaned accumulator 100 -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_195_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_194_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO ContextCleaner:59 - Cleaned accumulator 99 -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_193_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_192_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO ContextCleaner:59 - Cleaned accumulator 98 -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_191_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_190_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO ContextCleaner:59 - Cleaned accumulator 97 -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_189_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_188_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO ContextCleaner:59 - Cleaned accumulator 96 -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_187_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_186_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO ContextCleaner:59 - Cleaned accumulator 95 -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_185_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_184_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO ContextCleaner:59 - Cleaned accumulator 94 -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_183_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_182_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO ContextCleaner:59 - Cleaned accumulator 93 -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_181_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_180_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO ContextCleaner:59 - Cleaned accumulator 92 -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_179_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_178_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO ContextCleaner:59 - Cleaned accumulator 91 -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_177_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_176_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:47 INFO ContextCleaner:59 - Cleaned accumulator 90 -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_175_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_174_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:47 INFO ContextCleaner:59 - Cleaned accumulator 89 -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_173_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_172_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:47 INFO ContextCleaner:59 - Cleaned accumulator 88 -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_171_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_170_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:47 INFO ContextCleaner:59 - Cleaned accumulator 87 -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Removed broadcast_169_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:47 INFO Executor:59 - Finished task 0.0 in stage 104.0 (TID 207). 2254 bytes result sent to driver -2016-05-24 22:20:47 INFO Executor:59 - Finished task 1.0 in stage 104.0 (TID 208). 2254 bytes result sent to driver -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 104.0 (TID 208) in 510 ms on localhost (1/2) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 104.0 (TID 207) in 512 ms on localhost (2/2) -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 104.0, whose tasks have all completed, from pool -2016-05-24 22:20:47 INFO DAGScheduler:59 - ShuffleMapStage 104 (map at SVMFromSparkLearning.scala:44) finished in 0.513 s -2016-05-24 22:20:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:20:47 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:20:47 INFO DAGScheduler:59 - waiting: Set(ResultStage 105) -2016-05-24 22:20:47 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Missing parents for ResultStage 105: List() -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting ResultStage 105 (MapPartitionsRDD[215] at sortByKey at BinaryClassificationMetrics.scala:156), which is now runnable -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(3344) called with curMem=528407, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_206 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(1900) called with curMem=531751, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_206_piece0 stored as bytes in memory (estimated size 1900.0 B, free 730.1 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_206_piece0 in memory on localhost:56872 (size: 1900.0 B, free: 730.3 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 206 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 105 (MapPartitionsRDD[215] at sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Adding task set 105.0 with 2 tasks -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 105.0 (TID 209, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 105.0 (TID 210, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:20:47 INFO Executor:59 - Running task 0.0 in stage 105.0 (TID 209) -2016-05-24 22:20:47 INFO Executor:59 - Running task 1.0 in stage 105.0 (TID 210) -2016-05-24 22:20:47 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:20:47 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:20:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-24 22:20:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-24 22:20:47 INFO Executor:59 - Finished task 0.0 in stage 105.0 (TID 209). 1469 bytes result sent to driver -2016-05-24 22:20:47 INFO Executor:59 - Finished task 1.0 in stage 105.0 (TID 210). 1517 bytes result sent to driver -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 105.0 (TID 209) in 39 ms on localhost (1/2) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 105.0 (TID 210) in 39 ms on localhost (2/2) -2016-05-24 22:20:47 INFO DAGScheduler:59 - ResultStage 105 (sortByKey at BinaryClassificationMetrics.scala:156) finished in 0.041 s -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 105.0, whose tasks have all completed, from pool -2016-05-24 22:20:47 INFO DAGScheduler:59 - Job 104 finished: sortByKey at BinaryClassificationMetrics.scala:156, took 0.580885 s -2016-05-24 22:20:47 INFO SparkContext:59 - Starting job: collect at BinaryClassificationMetrics.scala:193 -2016-05-24 22:20:47 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 154 bytes -2016-05-24 22:20:47 INFO DAGScheduler:59 - Registering RDD 213 (combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:20:47 INFO DAGScheduler:59 - Got job 105 (collect at BinaryClassificationMetrics.scala:193) with 2 output partitions -2016-05-24 22:20:47 INFO DAGScheduler:59 - Final stage: ResultStage 108(collect at BinaryClassificationMetrics.scala:193) -2016-05-24 22:20:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 107) -2016-05-24 22:20:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 107) -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 107 (ShuffledRDD[213] at combineByKey at BinaryClassificationMetrics.scala:152), which has no missing parents -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=533651, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_207 stored as values in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-24 22:20:47 INFO MemoryStore:59 - ensureFreeSpace(1658) called with curMem=536547, maxMem=766075207 -2016-05-24 22:20:47 INFO MemoryStore:59 - Block broadcast_207_piece0 stored as bytes in memory (estimated size 1658.0 B, free 730.1 MB) -2016-05-24 22:20:47 INFO BlockManagerInfo:59 - Added broadcast_207_piece0 in memory on localhost:56872 (size: 1658.0 B, free: 730.3 MB) -2016-05-24 22:20:47 INFO SparkContext:59 - Created broadcast 207 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 107 (ShuffledRDD[213] at combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:20:47 INFO TaskSchedulerImpl:59 - Adding task set 107.0 with 2 tasks -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 107.0 (TID 211, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 107.0 (TID 212, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:20:47 INFO Executor:59 - Running task 1.0 in stage 107.0 (TID 212) -2016-05-24 22:20:47 INFO Executor:59 - Running task 0.0 in stage 107.0 (TID 211) -2016-05-24 22:20:47 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:20:47 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:20:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:20:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:20:48 INFO Executor:59 - Finished task 1.0 in stage 107.0 (TID 212). 1375 bytes result sent to driver -2016-05-24 22:20:48 INFO Executor:59 - Finished task 0.0 in stage 107.0 (TID 211). 1375 bytes result sent to driver -2016-05-24 22:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 107.0 (TID 212) in 56 ms on localhost (1/2) -2016-05-24 22:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 107.0 (TID 211) in 57 ms on localhost (2/2) -2016-05-24 22:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 107.0, whose tasks have all completed, from pool -2016-05-24 22:20:48 INFO DAGScheduler:59 - ShuffleMapStage 107 (combineByKey at BinaryClassificationMetrics.scala:152) finished in 0.057 s -2016-05-24 22:20:48 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:20:48 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:20:48 INFO DAGScheduler:59 - waiting: Set(ResultStage 108) -2016-05-24 22:20:48 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:20:48 INFO DAGScheduler:59 - Missing parents for ResultStage 108: List() -2016-05-24 22:20:48 INFO DAGScheduler:59 - Submitting ResultStage 108 (MapPartitionsRDD[218] at mapPartitions at BinaryClassificationMetrics.scala:189), which is now runnable -2016-05-24 22:20:48 INFO MemoryStore:59 - ensureFreeSpace(3472) called with curMem=538205, maxMem=766075207 -2016-05-24 22:20:48 INFO MemoryStore:59 - Block broadcast_208 stored as values in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:20:48 INFO MemoryStore:59 - ensureFreeSpace(1981) called with curMem=541677, maxMem=766075207 -2016-05-24 22:20:48 INFO MemoryStore:59 - Block broadcast_208_piece0 stored as bytes in memory (estimated size 1981.0 B, free 730.1 MB) -2016-05-24 22:20:48 INFO BlockManagerInfo:59 - Added broadcast_208_piece0 in memory on localhost:56872 (size: 1981.0 B, free: 730.3 MB) -2016-05-24 22:20:48 INFO SparkContext:59 - Created broadcast 208 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 108 (MapPartitionsRDD[218] at mapPartitions at BinaryClassificationMetrics.scala:189) -2016-05-24 22:20:48 INFO TaskSchedulerImpl:59 - Adding task set 108.0 with 2 tasks -2016-05-24 22:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 108.0 (TID 213, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 108.0 (TID 214, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:20:48 INFO Executor:59 - Running task 1.0 in stage 108.0 (TID 214) -2016-05-24 22:20:48 INFO Executor:59 - Running task 0.0 in stage 108.0 (TID 213) -2016-05-24 22:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:20:48 INFO Executor:59 - Finished task 1.0 in stage 108.0 (TID 214). 1330 bytes result sent to driver -2016-05-24 22:20:48 INFO Executor:59 - Finished task 0.0 in stage 108.0 (TID 213). 1330 bytes result sent to driver -2016-05-24 22:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 108.0 (TID 214) in 13 ms on localhost (1/2) -2016-05-24 22:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 108.0 (TID 213) in 14 ms on localhost (2/2) -2016-05-24 22:20:48 INFO DAGScheduler:59 - ResultStage 108 (collect at BinaryClassificationMetrics.scala:193) finished in 0.015 s -2016-05-24 22:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 108.0, whose tasks have all completed, from pool -2016-05-24 22:20:48 INFO DAGScheduler:59 - Job 105 finished: collect at BinaryClassificationMetrics.scala:193, took 0.092503 s -2016-05-24 22:20:48 INFO BinaryClassificationMetrics:59 - Total counts: {numPos: 25, numNeg: 11} -2016-05-24 22:20:48 INFO SparkContext:59 - Starting job: runJob at SlidingRDD.scala:75 -2016-05-24 22:20:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 154 bytes -2016-05-24 22:20:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-24 22:20:48 INFO DAGScheduler:59 - Got job 106 (runJob at SlidingRDD.scala:75) with 3 output partitions -2016-05-24 22:20:48 INFO DAGScheduler:59 - Final stage: ResultStage 111(runJob at SlidingRDD.scala:75) -2016-05-24 22:20:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 110) -2016-05-24 22:20:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:48 INFO DAGScheduler:59 - Submitting ResultStage 111 (UnionRDD[224] at UnionRDD at BinaryClassificationMetrics.scala:92), which has no missing parents -2016-05-24 22:20:48 INFO MemoryStore:59 - ensureFreeSpace(5184) called with curMem=543658, maxMem=766075207 -2016-05-24 22:20:48 INFO MemoryStore:59 - Block broadcast_209 stored as values in memory (estimated size 5.1 KB, free 730.1 MB) -2016-05-24 22:20:48 INFO MemoryStore:59 - ensureFreeSpace(2788) called with curMem=548842, maxMem=766075207 -2016-05-24 22:20:48 INFO MemoryStore:59 - Block broadcast_209_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.1 MB) -2016-05-24 22:20:48 INFO BlockManagerInfo:59 - Added broadcast_209_piece0 in memory on localhost:56872 (size: 2.7 KB, free: 730.3 MB) -2016-05-24 22:20:48 INFO SparkContext:59 - Created broadcast 209 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:48 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 111 (UnionRDD[224] at UnionRDD at BinaryClassificationMetrics.scala:92) -2016-05-24 22:20:48 INFO TaskSchedulerImpl:59 - Adding task set 111.0 with 3 tasks -2016-05-24 22:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 111.0 (TID 215, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 111.0 (TID 216, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:20:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 111.0 (TID 217, localhost, PROCESS_LOCAL, 2318 bytes) -2016-05-24 22:20:48 INFO Executor:59 - Running task 1.0 in stage 111.0 (TID 216) -2016-05-24 22:20:48 INFO Executor:59 - Running task 0.0 in stage 111.0 (TID 215) -2016-05-24 22:20:48 INFO CacheManager:59 - Partition rdd_219_0 not found, computing it -2016-05-24 22:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:20:48 INFO MemoryStore:59 - ensureFreeSpace(1448) called with curMem=551630, maxMem=766075207 -2016-05-24 22:20:48 INFO MemoryStore:59 - Block rdd_219_0 stored as values in memory (estimated size 1448.0 B, free 730.1 MB) -2016-05-24 22:20:48 INFO BlockManagerInfo:59 - Added rdd_219_0 in memory on localhost:56872 (size: 1448.0 B, free: 730.3 MB) -2016-05-24 22:20:48 INFO Executor:59 - Finished task 0.0 in stage 111.0 (TID 215). 1883 bytes result sent to driver -2016-05-24 22:20:48 INFO CacheManager:59 - Partition rdd_219_1 not found, computing it -2016-05-24 22:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:20:48 INFO MemoryStore:59 - ensureFreeSpace(1616) called with curMem=553078, maxMem=766075207 -2016-05-24 22:20:48 INFO MemoryStore:59 - Block rdd_219_1 stored as values in memory (estimated size 1616.0 B, free 730.1 MB) -2016-05-24 22:20:48 INFO BlockManagerInfo:59 - Added rdd_219_1 in memory on localhost:56872 (size: 1616.0 B, free: 730.3 MB) -2016-05-24 22:20:48 INFO Executor:59 - Running task 2.0 in stage 111.0 (TID 217) -2016-05-24 22:20:48 INFO Executor:59 - Finished task 1.0 in stage 111.0 (TID 216). 1883 bytes result sent to driver -2016-05-24 22:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 111.0 (TID 215) in 33 ms on localhost (1/3) -2016-05-24 22:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 111.0 (TID 216) in 31 ms on localhost (2/3) -2016-05-24 22:20:48 INFO Executor:59 - Finished task 2.0 in stage 111.0 (TID 217). 1053 bytes result sent to driver -2016-05-24 22:20:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 111.0 (TID 217) in 34 ms on localhost (3/3) -2016-05-24 22:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 111.0, whose tasks have all completed, from pool -2016-05-24 22:20:48 INFO DAGScheduler:59 - ResultStage 111 (runJob at SlidingRDD.scala:75) finished in 0.037 s -2016-05-24 22:20:48 INFO DAGScheduler:59 - Job 106 finished: runJob at SlidingRDD.scala:75, took 0.056878 s -2016-05-24 22:20:48 INFO SparkContext:59 - Starting job: aggregate at AreaUnderCurve.scala:45 -2016-05-24 22:20:48 INFO DAGScheduler:59 - Got job 107 (aggregate at AreaUnderCurve.scala:45) with 4 output partitions -2016-05-24 22:20:48 INFO DAGScheduler:59 - Final stage: ResultStage 114(aggregate at AreaUnderCurve.scala:45) -2016-05-24 22:20:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 113) -2016-05-24 22:20:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:48 INFO DAGScheduler:59 - Submitting ResultStage 114 (SlidingRDD[225] at RDD at SlidingRDD.scala:47), which has no missing parents -2016-05-24 22:20:48 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=554694, maxMem=766075207 -2016-05-24 22:20:48 INFO MemoryStore:59 - Block broadcast_210 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:20:48 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=560350, maxMem=766075207 -2016-05-24 22:20:48 INFO MemoryStore:59 - Block broadcast_210_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.0 MB) -2016-05-24 22:20:48 INFO BlockManagerInfo:59 - Added broadcast_210_piece0 in memory on localhost:56872 (size: 2.9 KB, free: 730.3 MB) -2016-05-24 22:20:48 INFO SparkContext:59 - Created broadcast 210 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 114 (SlidingRDD[225] at RDD at SlidingRDD.scala:47) -2016-05-24 22:20:48 INFO TaskSchedulerImpl:59 - Adding task set 114.0 with 4 tasks -2016-05-24 22:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 114.0 (TID 218, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:20:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 114.0 (TID 219, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 114.0 (TID 220, localhost, PROCESS_LOCAL, 2635 bytes) -2016-05-24 22:20:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 114.0 (TID 221, localhost, PROCESS_LOCAL, 2475 bytes) -2016-05-24 22:20:48 INFO Executor:59 - Running task 2.0 in stage 114.0 (TID 219) -2016-05-24 22:20:48 INFO Executor:59 - Running task 1.0 in stage 114.0 (TID 218) -2016-05-24 22:20:48 INFO BlockManager:59 - Found block rdd_219_0 locally -2016-05-24 22:20:48 INFO BlockManager:59 - Found block rdd_219_1 locally -2016-05-24 22:20:48 INFO Executor:59 - Running task 0.0 in stage 114.0 (TID 220) -2016-05-24 22:20:48 INFO Executor:59 - Finished task 1.0 in stage 114.0 (TID 218). 2084 bytes result sent to driver -2016-05-24 22:20:48 INFO Executor:59 - Running task 3.0 in stage 114.0 (TID 221) -2016-05-24 22:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 114.0 (TID 218) in 11 ms on localhost (1/4) -2016-05-24 22:20:48 INFO Executor:59 - Finished task 2.0 in stage 114.0 (TID 219). 2084 bytes result sent to driver -2016-05-24 22:20:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 114.0 (TID 219) in 11 ms on localhost (2/4) -2016-05-24 22:20:48 INFO Executor:59 - Finished task 3.0 in stage 114.0 (TID 221). 955 bytes result sent to driver -2016-05-24 22:20:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 114.0 (TID 221) in 12 ms on localhost (3/4) -2016-05-24 22:20:48 INFO Executor:59 - Finished task 0.0 in stage 114.0 (TID 220). 955 bytes result sent to driver -2016-05-24 22:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 114.0 (TID 220) in 14 ms on localhost (4/4) -2016-05-24 22:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 114.0, whose tasks have all completed, from pool -2016-05-24 22:20:48 INFO DAGScheduler:59 - ResultStage 114 (aggregate at AreaUnderCurve.scala:45) finished in 0.018 s -2016-05-24 22:20:48 INFO DAGScheduler:59 - Job 107 finished: aggregate at AreaUnderCurve.scala:45, took 0.029631 s -2016-05-24 22:20:48 INFO SparkContext:59 - Starting job: take at SVMFromSparkLearning.scala:55 -2016-05-24 22:20:48 INFO DAGScheduler:59 - Got job 108 (take at SVMFromSparkLearning.scala:55) with 1 output partitions -2016-05-24 22:20:48 INFO DAGScheduler:59 - Final stage: ResultStage 115(take at SVMFromSparkLearning.scala:55) -2016-05-24 22:20:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:48 INFO DAGScheduler:59 - Submitting ResultStage 115 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44), which has no missing parents -2016-05-24 22:20:48 INFO MemoryStore:59 - ensureFreeSpace(10312) called with curMem=563321, maxMem=766075207 -2016-05-24 22:20:48 INFO MemoryStore:59 - Block broadcast_211 stored as values in memory (estimated size 10.1 KB, free 730.0 MB) -2016-05-24 22:20:48 INFO MemoryStore:59 - ensureFreeSpace(6082) called with curMem=573633, maxMem=766075207 -2016-05-24 22:20:48 INFO MemoryStore:59 - Block broadcast_211_piece0 stored as bytes in memory (estimated size 5.9 KB, free 730.0 MB) -2016-05-24 22:20:48 INFO BlockManagerInfo:59 - Added broadcast_211_piece0 in memory on localhost:56872 (size: 5.9 KB, free: 730.3 MB) -2016-05-24 22:20:48 INFO SparkContext:59 - Created broadcast 211 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:48 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 115 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44) -2016-05-24 22:20:48 INFO TaskSchedulerImpl:59 - Adding task set 115.0 with 1 tasks -2016-05-24 22:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 115.0 (TID 222, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:20:48 INFO Executor:59 - Running task 0.0 in stage 115.0 (TID 222) -2016-05-24 22:20:48 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:20:48 INFO Executor:59 - Finished task 0.0 in stage 115.0 (TID 222). 2401 bytes result sent to driver -2016-05-24 22:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 115.0 (TID 222) in 4 ms on localhost (1/1) -2016-05-24 22:20:48 INFO DAGScheduler:59 - ResultStage 115 (take at SVMFromSparkLearning.scala:55) finished in 0.004 s -2016-05-24 22:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 115.0, whose tasks have all completed, from pool -2016-05-24 22:20:48 INFO DAGScheduler:59 - Job 108 finished: take at SVMFromSparkLearning.scala:55, took 0.011458 s -2016-05-24 22:20:48 INFO SparkContext:59 - Starting job: saveAsTextFile at GLMClassificationModel.scala:61 -2016-05-24 22:20:48 INFO DAGScheduler:59 - Got job 109 (saveAsTextFile at GLMClassificationModel.scala:61) with 1 output partitions -2016-05-24 22:20:48 INFO DAGScheduler:59 - Final stage: ResultStage 116(saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:20:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:48 INFO DAGScheduler:59 - Submitting ResultStage 116 (MapPartitionsRDD[227] at saveAsTextFile at GLMClassificationModel.scala:61), which has no missing parents -2016-05-24 22:20:48 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=579715, maxMem=766075207 -2016-05-24 22:20:48 INFO MemoryStore:59 - Block broadcast_212 stored as values in memory (estimated size 124.0 KB, free 729.9 MB) -2016-05-24 22:20:48 INFO MemoryStore:59 - ensureFreeSpace(42575) called with curMem=706707, maxMem=766075207 -2016-05-24 22:20:48 INFO MemoryStore:59 - Block broadcast_212_piece0 stored as bytes in memory (estimated size 41.6 KB, free 729.9 MB) -2016-05-24 22:20:48 INFO BlockManagerInfo:59 - Added broadcast_212_piece0 in memory on localhost:56872 (size: 41.6 KB, free: 730.2 MB) -2016-05-24 22:20:48 INFO SparkContext:59 - Created broadcast 212 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:48 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 116 (MapPartitionsRDD[227] at saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:20:48 INFO TaskSchedulerImpl:59 - Adding task set 116.0 with 1 tasks -2016-05-24 22:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 116.0 (TID 223, localhost, PROCESS_LOCAL, 2197 bytes) -2016-05-24 22:20:48 INFO Executor:59 - Running task 0.0 in stage 116.0 (TID 223) -2016-05-24 22:20:48 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242220_0116_m_000000_223' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222048221/result/metadata/_temporary/0/task_201605242220_0116_m_000000 -2016-05-24 22:20:48 INFO SparkHadoopMapRedUtil:59 - attempt_201605242220_0116_m_000000_223: Committed -2016-05-24 22:20:48 INFO Executor:59 - Finished task 0.0 in stage 116.0 (TID 223). 1864 bytes result sent to driver -2016-05-24 22:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 116.0 (TID 223) in 135 ms on localhost (1/1) -2016-05-24 22:20:48 INFO DAGScheduler:59 - ResultStage 116 (saveAsTextFile at GLMClassificationModel.scala:61) finished in 0.135 s -2016-05-24 22:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 116.0, whose tasks have all completed, from pool -2016-05-24 22:20:48 INFO DAGScheduler:59 - Job 109 finished: saveAsTextFile at GLMClassificationModel.scala:61, took 0.195354 s -2016-05-24 22:20:50 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:20:50 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:20:50 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:65 -2016-05-24 22:20:50 INFO DAGScheduler:59 - Got job 110 (parquet at GLMClassificationModel.scala:65) with 1 output partitions -2016-05-24 22:20:50 INFO DAGScheduler:59 - Final stage: ResultStage 117(parquet at GLMClassificationModel.scala:65) -2016-05-24 22:20:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:50 INFO DAGScheduler:59 - Submitting ResultStage 117 (MapPartitionsRDD[229] at rddToDataFrameHolder at GLMClassificationModel.scala:65), which has no missing parents -2016-05-24 22:20:50 INFO MemoryStore:59 - ensureFreeSpace(67104) called with curMem=749282, maxMem=766075207 -2016-05-24 22:20:50 INFO MemoryStore:59 - Block broadcast_213 stored as values in memory (estimated size 65.5 KB, free 729.8 MB) -2016-05-24 22:20:50 INFO MemoryStore:59 - ensureFreeSpace(23437) called with curMem=816386, maxMem=766075207 -2016-05-24 22:20:50 INFO MemoryStore:59 - Block broadcast_213_piece0 stored as bytes in memory (estimated size 22.9 KB, free 729.8 MB) -2016-05-24 22:20:50 INFO BlockManagerInfo:59 - Added broadcast_213_piece0 in memory on localhost:56872 (size: 22.9 KB, free: 730.2 MB) -2016-05-24 22:20:50 INFO SparkContext:59 - Created broadcast 213 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 117 (MapPartitionsRDD[229] at rddToDataFrameHolder at GLMClassificationModel.scala:65) -2016-05-24 22:20:50 INFO TaskSchedulerImpl:59 - Adding task set 117.0 with 1 tasks -2016-05-24 22:20:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 117.0 (TID 224, localhost, PROCESS_LOCAL, 7978 bytes) -2016-05-24 22:20:50 INFO Executor:59 - Running task 0.0 in stage 117.0 (TID 224) -2016-05-24 22:20:50 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:20:50 INFO CodecConfig:151 - Compression: GZIP -2016-05-24 22:20:50 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-24 22:20:50 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-24 22:20:50 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-24 22:20:50 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-24 22:20:50 INFO ParquetOutputFormat:151 - Validation is off -2016-05-24 22:20:50 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-24 22:20:50 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-24 22:20:50 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 6,156 -2016-05-24 22:20:50 INFO ColumnChunkPageWriteStore:151 - written 61B for [weights, type] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-24 22:20:50 INFO ColumnChunkPageWriteStore:151 - written 48B for [weights, size] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-24 22:20:50 INFO ColumnChunkPageWriteStore:151 - written 51B for [weights, indices, array] INT32: 1 values, 13B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-24 22:20:50 INFO ColumnChunkPageWriteStore:151 - written 637B for [weights, values, array] DOUBLE: 692 values, 668B raw, 593B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 413 entries, 3,304B raw, 413B comp} -2016-05-24 22:20:50 INFO ColumnChunkPageWriteStore:151 - written 69B for [intercept] DOUBLE: 1 values, 14B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-24 22:20:50 INFO ColumnChunkPageWriteStore:151 - written 47B for [threshold] DOUBLE: 1 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-24 22:20:50 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242220_0117_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222048221/result/data/_temporary/0/task_201605242220_0117_m_000000 -2016-05-24 22:20:50 INFO SparkHadoopMapRedUtil:59 - attempt_201605242220_0117_m_000000_0: Committed -2016-05-24 22:20:50 INFO Executor:59 - Finished task 0.0 in stage 117.0 (TID 224). 915 bytes result sent to driver -2016-05-24 22:20:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 117.0 (TID 224) in 457 ms on localhost (1/1) -2016-05-24 22:20:50 INFO DAGScheduler:59 - ResultStage 117 (parquet at GLMClassificationModel.scala:65) finished in 0.458 s -2016-05-24 22:20:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 117.0, whose tasks have all completed, from pool -2016-05-24 22:20:50 INFO DAGScheduler:59 - Job 110 finished: parquet at GLMClassificationModel.scala:65, took 0.490373 s -2016-05-24 22:20:50 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:20:50 INFO DefaultWriterContainer:59 - Job job_201605242220_0000 committed. -2016-05-24 22:20:50 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222048221/result/data on driver -2016-05-24 22:20:50 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222048221/result/data on driver -2016-05-24 22:20:50 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=839823, maxMem=766075207 -2016-05-24 22:20:50 INFO MemoryStore:59 - Block broadcast_214 stored as values in memory (estimated size 225.3 KB, free 729.6 MB) -2016-05-24 22:20:50 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1070527, maxMem=766075207 -2016-05-24 22:20:50 INFO MemoryStore:59 - Block broadcast_214_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.5 MB) -2016-05-24 22:20:50 INFO BlockManagerInfo:59 - Added broadcast_214_piece0 in memory on localhost:56872 (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:20:50 INFO SparkContext:59 - Created broadcast 214 from textFile at modelSaveLoad.scala:129 -2016-05-24 22:20:51 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:20:51 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-24 22:20:51 INFO DAGScheduler:59 - Got job 111 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-24 22:20:51 INFO DAGScheduler:59 - Final stage: ResultStage 118(first at modelSaveLoad.scala:129) -2016-05-24 22:20:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:51 INFO DAGScheduler:59 - Submitting ResultStage 118 (MapPartitionsRDD[232] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-24 22:20:51 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=1090315, maxMem=766075207 -2016-05-24 22:20:51 INFO MemoryStore:59 - Block broadcast_215 stored as values in memory (estimated size 3.1 KB, free 729.5 MB) -2016-05-24 22:20:51 INFO MemoryStore:59 - ensureFreeSpace(1900) called with curMem=1093531, maxMem=766075207 -2016-05-24 22:20:51 INFO MemoryStore:59 - Block broadcast_215_piece0 stored as bytes in memory (estimated size 1900.0 B, free 729.5 MB) -2016-05-24 22:20:51 INFO BlockManagerInfo:59 - Added broadcast_215_piece0 in memory on localhost:56872 (size: 1900.0 B, free: 730.2 MB) -2016-05-24 22:20:51 INFO SparkContext:59 - Created broadcast 215 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:51 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 118 (MapPartitionsRDD[232] at textFile at modelSaveLoad.scala:129) -2016-05-24 22:20:51 INFO TaskSchedulerImpl:59 - Adding task set 118.0 with 1 tasks -2016-05-24 22:20:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 118.0 (TID 225, localhost, PROCESS_LOCAL, 2238 bytes) -2016-05-24 22:20:51 INFO Executor:59 - Running task 0.0 in stage 118.0 (TID 225) -2016-05-24 22:20:51 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222048221/result/metadata/part-00000:0+54 -2016-05-24 22:20:51 INFO Executor:59 - Finished task 0.0 in stage 118.0 (TID 225). 2154 bytes result sent to driver -2016-05-24 22:20:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 118.0 (TID 225) in 9 ms on localhost (1/1) -2016-05-24 22:20:51 INFO DAGScheduler:59 - ResultStage 118 (first at modelSaveLoad.scala:129) finished in 0.009 s -2016-05-24 22:20:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 118.0, whose tasks have all completed, from pool -2016-05-24 22:20:51 INFO DAGScheduler:59 - Job 111 finished: first at modelSaveLoad.scala:129, took 0.015444 s -2016-05-24 22:20:51 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222048221/result/data on driver -2016-05-24 22:20:51 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:78 -2016-05-24 22:20:51 INFO DAGScheduler:59 - Got job 112 (parquet at GLMClassificationModel.scala:78) with 4 output partitions -2016-05-24 22:20:51 INFO DAGScheduler:59 - Final stage: ResultStage 119(parquet at GLMClassificationModel.scala:78) -2016-05-24 22:20:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:51 INFO DAGScheduler:59 - Submitting ResultStage 119 (MapPartitionsRDD[234] at parquet at GLMClassificationModel.scala:78), which has no missing parents -2016-05-24 22:20:51 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=1095431, maxMem=766075207 -2016-05-24 22:20:51 INFO MemoryStore:59 - Block broadcast_216 stored as values in memory (estimated size 61.5 KB, free 729.5 MB) -2016-05-24 22:20:51 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=1158431, maxMem=766075207 -2016-05-24 22:20:51 INFO MemoryStore:59 - Block broadcast_216_piece0 stored as bytes in memory (estimated size 20.6 KB, free 729.5 MB) -2016-05-24 22:20:51 INFO BlockManagerInfo:59 - Added broadcast_216_piece0 in memory on localhost:56872 (size: 20.6 KB, free: 730.2 MB) -2016-05-24 22:20:51 INFO SparkContext:59 - Created broadcast 216 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 119 (MapPartitionsRDD[234] at parquet at GLMClassificationModel.scala:78) -2016-05-24 22:20:51 INFO TaskSchedulerImpl:59 - Adding task set 119.0 with 4 tasks -2016-05-24 22:20:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 119.0 (TID 226, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:20:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 119.0 (TID 227, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:20:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 119.0 (TID 228, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:20:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 119.0 (TID 229, localhost, PROCESS_LOCAL, 2279 bytes) -2016-05-24 22:20:51 INFO Executor:59 - Running task 0.0 in stage 119.0 (TID 226) -2016-05-24 22:20:51 INFO Executor:59 - Running task 2.0 in stage 119.0 (TID 228) -2016-05-24 22:20:51 INFO Executor:59 - Running task 3.0 in stage 119.0 (TID 229) -2016-05-24 22:20:51 INFO Executor:59 - Running task 1.0 in stage 119.0 (TID 227) -2016-05-24 22:20:51 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:20:51 INFO Executor:59 - Finished task 1.0 in stage 119.0 (TID 227). 936 bytes result sent to driver -2016-05-24 22:20:51 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:20:51 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:20:51 INFO Executor:59 - Finished task 0.0 in stage 119.0 (TID 226). 936 bytes result sent to driver -2016-05-24 22:20:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 119.0 (TID 227) in 128 ms on localhost (1/4) -2016-05-24 22:20:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 119.0 (TID 226) in 130 ms on localhost (2/4) -2016-05-24 22:20:51 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:20:51 INFO Executor:59 - Finished task 2.0 in stage 119.0 (TID 228). 936 bytes result sent to driver -2016-05-24 22:20:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 119.0 (TID 228) in 138 ms on localhost (3/4) -2016-05-24 22:20:51 INFO BlockManagerInfo:59 - Removed broadcast_215_piece0 on localhost:56872 in memory (size: 1900.0 B, free: 730.2 MB) -2016-05-24 22:20:51 INFO ContextCleaner:59 - Cleaned accumulator 114 -2016-05-24 22:20:51 INFO BlockManagerInfo:59 - Removed broadcast_214_piece0 on localhost:56872 in memory (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:20:51 INFO BlockManagerInfo:59 - Removed broadcast_213_piece0 on localhost:56872 in memory (size: 22.9 KB, free: 730.2 MB) -2016-05-24 22:20:51 INFO ContextCleaner:59 - Cleaned accumulator 113 -2016-05-24 22:20:51 INFO BlockManagerInfo:59 - Removed broadcast_212_piece0 on localhost:56872 in memory (size: 41.6 KB, free: 730.2 MB) -2016-05-24 22:20:51 INFO Executor:59 - Finished task 3.0 in stage 119.0 (TID 229). 1802 bytes result sent to driver -2016-05-24 22:20:51 INFO ContextCleaner:59 - Cleaned accumulator 112 -2016-05-24 22:20:51 INFO TaskSetManager:59 - Finished task 3.0 in stage 119.0 (TID 229) in 160 ms on localhost (4/4) -2016-05-24 22:20:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 119.0, whose tasks have all completed, from pool -2016-05-24 22:20:51 INFO DAGScheduler:59 - ResultStage 119 (parquet at GLMClassificationModel.scala:78) finished in 0.162 s -2016-05-24 22:20:51 INFO DAGScheduler:59 - Job 112 finished: parquet at GLMClassificationModel.scala:78, took 0.187061 s -2016-05-24 22:20:51 INFO BlockManagerInfo:59 - Removed broadcast_211_piece0 on localhost:56872 in memory (size: 5.9 KB, free: 730.2 MB) -2016-05-24 22:20:51 INFO ContextCleaner:59 - Cleaned accumulator 111 -2016-05-24 22:20:51 INFO BlockManagerInfo:59 - Removed broadcast_210_piece0 on localhost:56872 in memory (size: 2.9 KB, free: 730.2 MB) -2016-05-24 22:20:51 INFO ContextCleaner:59 - Cleaned accumulator 110 -2016-05-24 22:20:51 INFO BlockManagerInfo:59 - Removed broadcast_209_piece0 on localhost:56872 in memory (size: 2.7 KB, free: 730.2 MB) -2016-05-24 22:20:51 INFO ContextCleaner:59 - Cleaned accumulator 109 -2016-05-24 22:20:51 INFO BlockManagerInfo:59 - Removed broadcast_208_piece0 on localhost:56872 in memory (size: 1981.0 B, free: 730.2 MB) -2016-05-24 22:20:51 INFO ContextCleaner:59 - Cleaned accumulator 108 -2016-05-24 22:20:51 INFO BlockManagerInfo:59 - Removed broadcast_207_piece0 on localhost:56872 in memory (size: 1658.0 B, free: 730.3 MB) -2016-05-24 22:20:51 INFO ContextCleaner:59 - Cleaned accumulator 107 -2016-05-24 22:20:51 INFO BlockManagerInfo:59 - Removed broadcast_206_piece0 on localhost:56872 in memory (size: 1900.0 B, free: 730.3 MB) -2016-05-24 22:20:51 INFO ContextCleaner:59 - Cleaned accumulator 106 -2016-05-24 22:20:51 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=615544, maxMem=766075207 -2016-05-24 22:20:51 INFO MemoryStore:59 - Block broadcast_217 stored as values in memory (estimated size 86.4 KB, free 729.9 MB) -2016-05-24 22:20:51 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=704056, maxMem=766075207 -2016-05-24 22:20:51 INFO MemoryStore:59 - Block broadcast_217_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-24 22:20:51 INFO BlockManagerInfo:59 - Added broadcast_217_piece0 in memory on localhost:56872 (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:20:51 INFO SparkContext:59 - Created broadcast 217 from take at GLMClassificationModel.scala:79 -2016-05-24 22:20:51 INFO BlockManagerInfo:59 - Removed broadcast_216_piece0 on localhost:56872 in memory (size: 20.6 KB, free: 730.3 MB) -2016-05-24 22:20:51 INFO ContextCleaner:59 - Cleaned accumulator 115 -2016-05-24 22:20:51 INFO BlockManagerInfo:59 - Removed broadcast_205_piece0 on localhost:56872 in memory (size: 6.3 KB, free: 730.3 MB) -2016-05-24 22:20:51 INFO ContextCleaner:59 - Cleaned accumulator 105 -2016-05-24 22:20:52 INFO BlockManagerInfo:59 - Removed broadcast_127_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:52 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-24 22:20:52 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:52 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:56872 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:20:52 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 22:20:52 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:52 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:56872 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:20:52 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-24 22:20:52 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222048221/result/data/part-r-00000-56919489-94df-4521-8ffe-b78cdf2b166d.gz.parquet -2016-05-24 22:20:52 INFO SparkContext:59 - Starting job: take at GLMClassificationModel.scala:79 -2016-05-24 22:20:52 INFO DAGScheduler:59 - Got job 113 (take at GLMClassificationModel.scala:79) with 1 output partitions -2016-05-24 22:20:52 INFO DAGScheduler:59 - Final stage: ResultStage 120(take at GLMClassificationModel.scala:79) -2016-05-24 22:20:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:20:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:20:52 INFO DAGScheduler:59 - Submitting ResultStage 120 (MapPartitionsRDD[236] at take at GLMClassificationModel.scala:79), which has no missing parents -2016-05-24 22:20:52 INFO MemoryStore:59 - ensureFreeSpace(4168) called with curMem=570248, maxMem=766075207 -2016-05-24 22:20:52 INFO MemoryStore:59 - Block broadcast_218 stored as values in memory (estimated size 4.1 KB, free 730.0 MB) -2016-05-24 22:20:52 INFO MemoryStore:59 - ensureFreeSpace(2438) called with curMem=574416, maxMem=766075207 -2016-05-24 22:20:52 INFO MemoryStore:59 - Block broadcast_218_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.0 MB) -2016-05-24 22:20:52 INFO BlockManagerInfo:59 - Added broadcast_218_piece0 in memory on localhost:56872 (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:20:52 INFO SparkContext:59 - Created broadcast 218 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:20:52 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 120 (MapPartitionsRDD[236] at take at GLMClassificationModel.scala:79) -2016-05-24 22:20:52 INFO TaskSchedulerImpl:59 - Adding task set 120.0 with 1 tasks -2016-05-24 22:20:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 120.0 (TID 230, localhost, PROCESS_LOCAL, 2301 bytes) -2016-05-24 22:20:52 INFO Executor:59 - Running task 0.0 in stage 120.0 (TID 230) -2016-05-24 22:20:52 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222048221/result/data/part-r-00000-56919489-94df-4521-8ffe-b78cdf2b166d.gz.parquet start: 0 end: 5410 length: 5410 hosts: []} -2016-05-24 22:20:52 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-24 22:20:52 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group weights { - required int32 type (INT_8); - optional int32 size; - optional group indices (LIST) { - repeated int32 array; - } - optional group values (LIST) { - repeated double array; - } - } - optional double intercept; - optional double threshold; -} - -Catalyst form: -StructType(StructField(weights,org.apache.spark.mllib.linalg.VectorUDT@f71b0bce,true), StructField(intercept,DoubleType,true), StructField(threshold,DoubleType,true)) - -2016-05-24 22:20:52 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-24 22:20:52 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-24 22:20:52 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-24 22:20:52 INFO InternalParquetRecordReader:151 - block read in memory in 8 ms. row count = 1 -2016-05-24 22:22:32 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 22:22:33 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 22:22:34 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 22:22:34 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 22:22:34 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 22:22:35 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 22:22:35 INFO Remoting:74 - Starting remoting -2016-05-24 22:22:35 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:56898] -2016-05-24 22:22:35 INFO Utils:59 - Successfully started service 'sparkDriver' on port 56898. -2016-05-24 22:22:35 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 22:22:35 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 22:22:35 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-d9a34556-9d74-44ff-bd38-57dbdf63eb17 -2016-05-24 22:22:35 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 22:22:36 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-0c249c60-19bc-4c94-82a7-4704860b1a37\httpd-2bf4b108-0ef5-4f34-849b-2528484613c7 -2016-05-24 22:22:36 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 22:22:36 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:22:36 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:56899 -2016-05-24 22:22:36 INFO Utils:59 - Successfully started service 'HTTP file server' on port 56899. -2016-05-24 22:22:36 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 22:22:36 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:22:36 WARN AbstractLifeCycle:204 - FAILED SelectChannelConnector@0.0.0.0:4040: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.mllib.learning.regression.SVMFromSparkLearning$.main(SVMFromSparkLearning.scala:26) - at org.apache.spark.mllib.learning.regression.SVMFromSparkLearning.main(SVMFromSparkLearning.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-05-24 22:22:36 WARN AbstractLifeCycle:204 - FAILED org.spark-project.jetty.server.Server@3d73c8c9: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.mllib.learning.regression.SVMFromSparkLearning$.main(SVMFromSparkLearning.scala:26) - at org.apache.spark.mllib.learning.regression.SVMFromSparkLearning.main(SVMFromSparkLearning.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 22:22:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 22:22:36 WARN Utils:71 - Service 'SparkUI' could not bind on port 4040. Attempting port 4041. -2016-05-24 22:22:36 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:22:36 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4041 -2016-05-24 22:22:36 INFO Utils:59 - Successfully started service 'SparkUI' on port 4041. -2016-05-24 22:22:36 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4041 -2016-05-24 22:22:37 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 22:22:37 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 22:22:37 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 56918. -2016-05-24 22:22:37 INFO NettyBlockTransferService:59 - Server created on 56918 -2016-05-24 22:22:37 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 22:22:37 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:56918 with 730.6 MB RAM, BlockManagerId(driver, localhost, 56918) -2016-05-24 22:22:37 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 22:22:38 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 22:22:38 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 22:22:38 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 22:22:38 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 22:22:38 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:56918 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 22:22:38 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 22:22:40 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 22:22:40 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:22:40 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 22:22:40 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 22:22:40 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 22:22:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:40 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 22:22:40 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 22:22:40 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 22:22:40 INFO MemoryStore:59 - ensureFreeSpace(2151) called with curMem=175476, maxMem=766075207 -2016-05-24 22:22:40 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:22:40 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:56918 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:22:40 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 22:22:40 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 22:22:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:22:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:22:40 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 22:22:40 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 22:22:40 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 22:22:40 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 22:22:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 22:22:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 22:22:40 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 22:22:40 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 22:22:40 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 22:22:40 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 22:22:40 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 22:22:41 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177627, maxMem=766075207 -2016-05-24 22:22:41 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 22:22:41 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262707, maxMem=766075207 -2016-05-24 22:22:41 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 22:22:41 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:56918 (size: 83.1 KB, free: 730.5 MB) -2016-05-24 22:22:41 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:56918 (size: 83.6 KB, free: 730.4 MB) -2016-05-24 22:22:41 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 22:22:41 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 22:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 442 ms on localhost (1/2) -2016-05-24 22:22:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 425 ms on localhost (2/2) -2016-05-24 22:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 22:22:41 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.464 s -2016-05-24 22:22:41 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.612193 s -2016-05-24 22:22:41 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 22:22:41 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 22:22:41 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:22:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:41 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 22:22:41 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=348355, maxMem=766075207 -2016-05-24 22:22:41 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-24 22:22:41 INFO MemoryStore:59 - ensureFreeSpace(2490) called with curMem=352915, maxMem=766075207 -2016-05-24 22:22:41 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-24 22:22:41 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:56918 (size: 2.4 KB, free: 730.4 MB) -2016-05-24 22:22:41 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:41 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:22:41 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 22:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:22:41 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 22:22:41 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-05-24 22:22:41 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:22:41 INFO MemoryStore:59 - ensureFreeSpace(54392) called with curMem=355405, maxMem=766075207 -2016-05-24 22:22:41 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 53.1 KB, free 730.2 MB) -2016-05-24 22:22:41 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:56918 (size: 53.1 KB, free: 730.4 MB) -2016-05-24 22:22:41 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2611 bytes result sent to driver -2016-05-24 22:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 42 ms on localhost (1/1) -2016-05-24 22:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 22:22:41 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.043 s -2016-05-24 22:22:41 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.067231 s -2016-05-24 22:22:41 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 22:22:41 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 22:22:41 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 22:22:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:41 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 22:22:41 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=409797, maxMem=766075207 -2016-05-24 22:22:41 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-24 22:22:41 INFO MemoryStore:59 - ensureFreeSpace(2307) called with curMem=414045, maxMem=766075207 -2016-05-24 22:22:41 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-24 22:22:41 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:56918 (size: 2.3 KB, free: 730.4 MB) -2016-05-24 22:22:41 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40) -2016-05-24 22:22:41 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 22:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:22:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:22:41 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 22:22:41 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 22:22:41 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:41 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-05-24 22:22:41 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:22:41 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 22:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 17 ms on localhost (1/2) -2016-05-24 22:22:41 INFO MemoryStore:59 - ensureFreeSpace(62184) called with curMem=416352, maxMem=766075207 -2016-05-24 22:22:41 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 60.7 KB, free 730.1 MB) -2016-05-24 22:22:41 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:56918 (size: 60.7 KB, free: 730.3 MB) -2016-05-24 22:22:41 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2662 bytes result sent to driver -2016-05-24 22:22:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 29 ms on localhost (2/2) -2016-05-24 22:22:41 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.031 s -2016-05-24 22:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 22:22:41 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.058240 s -2016-05-24 22:22:41 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 22:22:41 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 22:22:41 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 22:22:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:41 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 22:22:41 INFO MemoryStore:59 - ensureFreeSpace(4264) called with curMem=478536, maxMem=766075207 -2016-05-24 22:22:41 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.2 KB, free 730.1 MB) -2016-05-24 22:22:41 INFO MemoryStore:59 - ensureFreeSpace(2329) called with curMem=482800, maxMem=766075207 -2016-05-24 22:22:41 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 22:22:41 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:56918 (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:22:41 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 22:22:41 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 22:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:22:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:22:41 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 22:22:41 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 22:22:41 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:41 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 22:22:41 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:41 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 22:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 17 ms on localhost (1/2) -2016-05-24 22:22:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 18 ms on localhost (2/2) -2016-05-24 22:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 22:22:41 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.022 s -2016-05-24 22:22:41 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.041976 s -2016-05-24 22:22:42 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 22:22:42 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=485129, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(388) called with curMem=490721, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 388.0 B, free 730.1 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:56918 (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:42 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=491109, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=502893, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 22:22:42 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:42 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 28 ms on localhost (1/2) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 27 ms on localhost (2/2) -2016-05-24 22:22:42 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.030 s -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 22:22:42 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.053119 s -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=506381, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=511973, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:56918 (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:42 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=514211, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3489) called with curMem=525995, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 22:22:42 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:42 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 19 ms on localhost (1/2) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 22 ms on localhost (2/2) -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 22:22:42 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 22:22:42 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.045393 s -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=529484, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=535076, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:42 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=538685, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=550469, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 22:22:42 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:42 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 21 ms on localhost (1/2) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 25 ms on localhost (2/2) -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 22:22:42 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 22:22:42 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.045172 s -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=553959, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3603) called with curMem=559551, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:42 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=563154, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3489) called with curMem=574938, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 22:22:42 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:42 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:42 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 23 ms on localhost (1/2) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 23 ms on localhost (2/2) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 22:22:42 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 22:22:42 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.085799 s -2016-05-24 22:22:42 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=553952, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=544271, maxMem=766075207 -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:56918 in memory (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:56918 in memory (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:56918 in memory (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:42 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:42 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:56918 in memory (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:42 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:56918 in memory (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=498597, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3489) called with curMem=508230, maxMem=766075207 -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:56918 in memory (size: 2.1 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 22:22:42 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:42 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 20 ms on localhost (1/2) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 23 ms on localhost (2/2) -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 22:22:42 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 22:22:42 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.044081 s -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=507823, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=513415, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:42 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=517021, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3489) called with curMem=528805, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 22:22:42 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:42 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 14 ms on localhost (1/2) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 14 ms on localhost (2/2) -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 22:22:42 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:22:42 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.030831 s -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=532294, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=537886, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:42 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=541492, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=553276, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 22:22:42 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:42 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 11 ms on localhost (1/2) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 11 ms on localhost (2/2) -2016-05-24 22:22:42 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 22:22:42 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.025420 s -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=556766, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3604) called with curMem=562358, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:42 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=565962, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=577746, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 22:22:42 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:42 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 10 ms on localhost (1/2) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 11 ms on localhost (2/2) -2016-05-24 22:22:42 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 22:22:42 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.029476 s -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=581237, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=586829, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:42 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=590435, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=602219, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 22:22:42 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:42 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 9 ms on localhost (1/2) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 12 ms on localhost (2/2) -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 22:22:42 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:22:42 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.025676 s -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=605709, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=611301, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:42 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=614907, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=626691, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 22:22:42 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:42 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 16 ms on localhost (1/2) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 15 ms on localhost (2/2) -2016-05-24 22:22:42 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 22:22:42 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.034104 s -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=630181, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3610) called with curMem=635773, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:42 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=639383, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=651167, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 22:22:42 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:42 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 12 ms on localhost (1/2) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 12 ms on localhost (2/2) -2016-05-24 22:22:42 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 22:22:42 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.025209 s -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=654657, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=660249, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:42 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=663855, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=675639, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:42 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 22:22:42 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:42 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 7968 bytes result sent to driver -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 13 ms on localhost (1/2) -2016-05-24 22:22:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 14 ms on localhost (2/2) -2016-05-24 22:22:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 22:22:42 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:22:42 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.032781 s -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=679130, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:22:42 INFO MemoryStore:59 - ensureFreeSpace(3610) called with curMem=684722, maxMem=766075207 -2016-05-24 22:22:42 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:22:42 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:42 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=688332, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=700116, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 11 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 13 ms on localhost (2/2) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.028278 s -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=703606, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=709198, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=712804, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=724588, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 9 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 10 ms on localhost (2/2) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.021990 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=728079, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=733671, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=737277, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=749061, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 12 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 14 ms on localhost (2/2) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.030322 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=752551, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=758143, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=761756, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=773540, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 15 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 14 ms on localhost (2/2) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.035913 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=777031, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=782623, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=786229, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=798013, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 14 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 14 ms on localhost (2/2) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.030961 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=801504, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=807096, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=810708, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=822492, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 11 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 13 ms on localhost (2/2) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.031272 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=825982, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3621) called with curMem=831574, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=835195, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=846979, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 11 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 13 ms on localhost (2/2) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.027257 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=850469, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=856061, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=859673, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3489) called with curMem=871457, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 11 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 13 ms on localhost (2/2) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.030412 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=874946, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=880538, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=884150, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=895934, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 22:22:43 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 15 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 14 ms on localhost (2/2) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.037 s -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.070089 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=850460, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=852562, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=819913, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=828091, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 22:22:43 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 10 ms on localhost (1/2) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 12 ms on localhost (2/2) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.029125 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=782629, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=782629, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=737294, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=733803, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 10 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 9 ms on localhost (2/2) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.026990 s -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=700126, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3634) called with curMem=690328, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=639427, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=647721, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:22:43 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:43 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 8 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 9 ms on localhost (2/2) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.020031 s -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=590482, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:22:43 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=592468, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:43 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=566020, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3489) called with curMem=577804, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 9 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 9 ms on localhost (2/2) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.020632 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=581293, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=586885, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=590498, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=602282, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 11 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 11 ms on localhost (2/2) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.022869 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=605772, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=611364, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=614976, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=626760, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 6 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 6 ms on localhost (2/2) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.017088 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=630250, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=635842, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=639448, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=651232, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 6 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 8 ms on localhost (2/2) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.016609 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=654722, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3623) called with curMem=660314, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=663937, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=675721, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 6 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 8 ms on localhost (2/2) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.017311 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=679212, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=684804, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=688410, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=700194, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 6 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 6 ms on localhost (2/2) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.016589 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=703684, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3626) called with curMem=709276, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=712902, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=724686, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 6 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 6 ms on localhost (2/2) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.016345 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=728177, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=733769, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=737381, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3489) called with curMem=749165, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 5 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 6 ms on localhost (2/2) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.015586 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=752654, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=758246, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=761858, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=773642, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 6 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 8 ms on localhost (2/2) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.019483 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=777133, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=782725, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=786337, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=798121, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 5 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 6 ms on localhost (2/2) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.015216 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=801612, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=807204, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=810816, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=822600, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 7 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 7 ms on localhost (2/2) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.016643 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=826090, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=831682, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:43 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=835294, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=847078, maxMem=766075207 -2016-05-24 22:22:43 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:43 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:43 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:43 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 22:22:43 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:43 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 7968 bytes result sent to driver -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 8 ms on localhost (1/2) -2016-05-24 22:22:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 7 ms on localhost (2/2) -2016-05-24 22:22:43 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:22:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 22:22:43 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.017758 s -2016-05-24 22:22:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=850568, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=856160, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:44 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=859772, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=871556, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 22:22:44 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:44 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 6 ms on localhost (1/2) -2016-05-24 22:22:44 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 6 ms on localhost (2/2) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.015387 s -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=875046, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=880638, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:44 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=884252, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=896036, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:22:44 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 22:22:44 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:44 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 6 ms on localhost (1/2) -2016-05-24 22:22:44 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 8 ms on localhost (2/2) -2016-05-24 22:22:44 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 22:22:44 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.026749 s -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=856162, maxMem=766075207 -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=852672, maxMem=766075207 -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:44 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 22:22:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:44 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=807326, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:44 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=807326, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 22:22:44 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:44 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 9 ms on localhost (1/2) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 9 ms on localhost (2/2) -2016-05-24 22:22:44 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 22:22:44 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.023828 s -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=761860, maxMem=766075207 -2016-05-24 22:22:44 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=763826, maxMem=766075207 -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:44 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:44 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=718476, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=724668, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 22:22:44 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 22:22:44 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:44 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 8 ms on localhost (1/2) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 10 ms on localhost (2/2) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 22:22:44 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.024060 s -2016-05-24 22:22:44 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=682818, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=684798, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 22:22:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:44 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[93] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=624183, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:22:44 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=630375, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[93] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 22:22:44 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 22:22:44 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:44 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 7 ms on localhost (1/2) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 8 ms on localhost (2/2) -2016-05-24 22:22:44 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 22:22:44 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.022745 s -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=590480, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:22:44 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=596072, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:44 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[95] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=566000, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=577784, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[95] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 22:22:44 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:44 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 7 ms on localhost (1/2) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 8 ms on localhost (2/2) -2016-05-24 22:22:44 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 22:22:44 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.018330 s -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=581275, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=586867, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:44 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[97] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=590480, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3489) called with curMem=602264, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[97] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 22:22:44 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:44 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 7 ms on localhost (1/2) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 8 ms on localhost (2/2) -2016-05-24 22:22:44 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 22:22:44 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.025197 s -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=605753, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3618) called with curMem=611345, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 91 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Got job 47 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:44 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[99] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=614963, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=626747, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 47 (MapPartitionsRDD[99] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 2 tasks -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 93, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 94, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 94) -2016-05-24 22:22:44 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 93) -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:44 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 94). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 93). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 94) in 5 ms on localhost (1/2) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 93) in 6 ms on localhost (2/2) -2016-05-24 22:22:44 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-24 22:22:44 INFO DAGScheduler:59 - Job 47 finished: treeAggregate at GradientDescent.scala:225, took 0.014373 s -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=630237, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=635829, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 93 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Got job 48 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:44 INFO DAGScheduler:59 - Final stage: ResultStage 48(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[101] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=639443, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3489) called with curMem=651227, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 48 (MapPartitionsRDD[101] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 2 tasks -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 95, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 96, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 95) -2016-05-24 22:22:44 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 96) -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:44 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 95). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 96). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 95) in 6 ms on localhost (1/2) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 96) in 5 ms on localhost (2/2) -2016-05-24 22:22:44 INFO DAGScheduler:59 - ResultStage 48 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-24 22:22:44 INFO DAGScheduler:59 - Job 48 finished: treeAggregate at GradientDescent.scala:225, took 0.014978 s -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=654716, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3618) called with curMem=660308, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 95 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Got job 49 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:44 INFO DAGScheduler:59 - Final stage: ResultStage 49(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[103] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=663926, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=675710, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 49 (MapPartitionsRDD[103] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 2 tasks -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 97, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 98, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 98) -2016-05-24 22:22:44 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 97) -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:44 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 97). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 98). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 97) in 7 ms on localhost (1/2) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 98) in 7 ms on localhost (2/2) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-05-24 22:22:44 INFO DAGScheduler:59 - ResultStage 49 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:22:44 INFO DAGScheduler:59 - Job 49 finished: treeAggregate at GradientDescent.scala:225, took 0.017756 s -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=679201, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3622) called with curMem=684793, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 97 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Got job 50 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:44 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[105] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=688415, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=700199, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 50 (MapPartitionsRDD[105] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 2 tasks -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-24 22:22:44 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:44 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 7 ms on localhost (1/2) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 7 ms on localhost (2/2) -2016-05-24 22:22:44 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-24 22:22:44 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at GradientDescent.scala:225, took 0.016364 s -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=703687, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=709279, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 99 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Got job 51 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:44 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[107] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=712892, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3489) called with curMem=724676, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 51 (MapPartitionsRDD[107] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-24 22:22:44 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:44 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 7 ms on localhost (1/2) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 7 ms on localhost (2/2) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-24 22:22:44 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:22:44 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at GradientDescent.scala:225, took 0.018883 s -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=728165, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3628) called with curMem=733757, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 101 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Got job 52 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:44 INFO DAGScheduler:59 - Final stage: ResultStage 52(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting ResultStage 52 (MapPartitionsRDD[109] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=737385, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3489) called with curMem=749169, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 52 (MapPartitionsRDD[109] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 103, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 104, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 103) -2016-05-24 22:22:44 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 104) -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:44 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 103). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 104). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 103) in 6 ms on localhost (1/2) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 104) in 7 ms on localhost (2/2) -2016-05-24 22:22:44 INFO DAGScheduler:59 - ResultStage 52 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:22:44 INFO DAGScheduler:59 - Job 52 finished: treeAggregate at GradientDescent.scala:225, took 0.017265 s -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=752658, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=758250, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 103 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Got job 53 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:44 INFO DAGScheduler:59 - Final stage: ResultStage 53(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[111] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=761882, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=773666, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 104 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[111] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 105, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 106, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 106) -2016-05-24 22:22:44 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 105) -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:44 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 106). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 105). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 105) in 19 ms on localhost (1/2) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 106) in 18 ms on localhost (2/2) -2016-05-24 22:22:44 INFO DAGScheduler:59 - ResultStage 53 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-24 22:22:44 INFO DAGScheduler:59 - Job 53 finished: treeAggregate at GradientDescent.scala:225, took 0.038705 s -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=777154, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=782746, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 105 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Got job 54 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:44 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[113] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=786359, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=798143, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 106 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 54 (MapPartitionsRDD[113] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 2 tasks -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 108) -2016-05-24 22:22:44 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 107) -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:44 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 107). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 108). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 107) in 7 ms on localhost (1/2) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 108) in 6 ms on localhost (2/2) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-24 22:22:44 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:22:44 INFO DAGScheduler:59 - Job 54 finished: treeAggregate at GradientDescent.scala:225, took 0.018881 s -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=801633, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=807225, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 107 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Got job 55 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:44 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[115] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=810857, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3489) called with curMem=822641, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 108 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 55 (MapPartitionsRDD[115] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 2 tasks -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 109, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 110, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:44 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 109) -2016-05-24 22:22:44 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 110) -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:44 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 110). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 109). 7968 bytes result sent to driver -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 110) in 5 ms on localhost (1/2) -2016-05-24 22:22:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 109) in 7 ms on localhost (2/2) -2016-05-24 22:22:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-24 22:22:44 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:22:44 INFO DAGScheduler:59 - Job 55 finished: treeAggregate at GradientDescent.scala:225, took 0.019647 s -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=826130, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=831722, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:44 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:44 INFO SparkContext:59 - Created broadcast 109 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:44 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:44 INFO DAGScheduler:59 - Got job 56 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:44 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:44 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[117] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:44 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=835335, maxMem=766075207 -2016-05-24 22:22:44 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3489) called with curMem=847119, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 110 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 56 (MapPartitionsRDD[117] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 2 tasks -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 111, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 112, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 111) -2016-05-24 22:22:45 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 112) -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:45 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 112). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 111). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 112) in 6 ms on localhost (1/2) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 111) in 7 ms on localhost (2/2) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-24 22:22:45 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:22:45 INFO DAGScheduler:59 - Job 56 finished: treeAggregate at GradientDescent.scala:225, took 0.573245 s -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=850608, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=856200, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 111 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Got job 57 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:45 INFO DAGScheduler:59 - Final stage: ResultStage 57(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[119] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=859813, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3489) called with curMem=871597, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 112 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 57 (MapPartitionsRDD[119] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 2 tasks -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 113, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 114, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 113) -2016-05-24 22:22:45 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 114) -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:45 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 114). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 113). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 114) in 7 ms on localhost (1/2) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 113) in 9 ms on localhost (2/2) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-24 22:22:45 INFO DAGScheduler:59 - ResultStage 57 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:22:45 INFO DAGScheduler:59 - Job 57 finished: treeAggregate at GradientDescent.scala:225, took 0.023054 s -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=875086, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=880678, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 113 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Got job 58 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:45 INFO DAGScheduler:59 - Final stage: ResultStage 58(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting ResultStage 58 (MapPartitionsRDD[121] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=884290, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=896074, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 114 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 58 (MapPartitionsRDD[121] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 2 tasks -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 115, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 116, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 116) -2016-05-24 22:22:45 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 115) -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:45 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 115). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 116). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 115) in 6 ms on localhost (1/2) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 116) in 6 ms on localhost (2/2) -2016-05-24 22:22:45 INFO DAGScheduler:59 - ResultStage 58 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-05-24 22:22:45 INFO DAGScheduler:59 - Job 58 finished: treeAggregate at GradientDescent.scala:225, took 0.017897 s -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=899564, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_115 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3620) called with curMem=905156, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_115_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_115_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 115 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Got job 59 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:45 INFO DAGScheduler:59 - Final stage: ResultStage 59(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting ResultStage 59 (MapPartitionsRDD[123] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=908776, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_116 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=920560, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_116_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_116_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 116 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 59 (MapPartitionsRDD[123] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 2 tasks -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 117, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 118, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 117) -2016-05-24 22:22:45 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 118) -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:45 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 117). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 118). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 118) in 8 ms on localhost (1/2) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 117) in 9 ms on localhost (2/2) -2016-05-24 22:22:45 INFO DAGScheduler:59 - ResultStage 59 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-05-24 22:22:45 INFO DAGScheduler:59 - Job 59 finished: treeAggregate at GradientDescent.scala:225, took 0.051117 s -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=924047, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_117 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=929639, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_117_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_117_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 117 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Got job 60 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:45 INFO DAGScheduler:59 - Final stage: ResultStage 60(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting ResultStage 60 (MapPartitionsRDD[125] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=933251, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_118 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3489) called with curMem=945035, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_118_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_118_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 118 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 60 (MapPartitionsRDD[125] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Adding task set 60.0 with 2 tasks -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 60.0 (TID 119, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 60.0 (TID 120, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO Executor:59 - Running task 1.0 in stage 60.0 (TID 120) -2016-05-24 22:22:45 INFO Executor:59 - Running task 0.0 in stage 60.0 (TID 119) -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:45 INFO Executor:59 - Finished task 0.0 in stage 60.0 (TID 119). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO Executor:59 - Finished task 1.0 in stage 60.0 (TID 120). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 60.0 (TID 119) in 6 ms on localhost (1/2) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 60.0 (TID 120) in 7 ms on localhost (2/2) -2016-05-24 22:22:45 INFO DAGScheduler:59 - ResultStage 60 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 60.0, whose tasks have all completed, from pool -2016-05-24 22:22:45 INFO DAGScheduler:59 - Job 60 finished: treeAggregate at GradientDescent.scala:225, took 0.018281 s -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=948524, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_119 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3616) called with curMem=954116, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_119_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_119_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 119 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Got job 61 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:45 INFO DAGScheduler:59 - Final stage: ResultStage 61(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting ResultStage 61 (MapPartitionsRDD[127] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=957732, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_120 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=969516, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_120_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_120_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 120 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 61 (MapPartitionsRDD[127] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 2 tasks -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 121, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 122, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 122) -2016-05-24 22:22:45 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 121) -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:45 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 122). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 121). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 122) in 4 ms on localhost (1/2) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 121) in 6 ms on localhost (2/2) -2016-05-24 22:22:45 INFO DAGScheduler:59 - ResultStage 61 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-05-24 22:22:45 INFO DAGScheduler:59 - Job 61 finished: treeAggregate at GradientDescent.scala:225, took 0.018369 s -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=973007, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_121 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=978599, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_121_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_121_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 121 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Got job 62 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:45 INFO DAGScheduler:59 - Final stage: ResultStage 62(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[129] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=982211, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_122 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=993995, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_122_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_122_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 122 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 62 (MapPartitionsRDD[129] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 2 tasks -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 123, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 62.0 (TID 124, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 123) -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:45 INFO Executor:59 - Running task 1.0 in stage 62.0 (TID 124) -2016-05-24 22:22:45 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 123). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 123) in 8 ms on localhost (1/2) -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:45 INFO Executor:59 - Finished task 1.0 in stage 62.0 (TID 124). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 62.0 (TID 124) in 43 ms on localhost (2/2) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-05-24 22:22:45 INFO DAGScheduler:59 - ResultStage 62 (treeAggregate at GradientDescent.scala:225) finished in 0.045 s -2016-05-24 22:22:45 INFO DAGScheduler:59 - Job 62 finished: treeAggregate at GradientDescent.scala:225, took 0.057374 s -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=997485, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_123 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3630) called with curMem=1003077, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_123_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_123_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 123 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Got job 63 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:45 INFO DAGScheduler:59 - Final stage: ResultStage 63(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting ResultStage 63 (MapPartitionsRDD[131] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1006707, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_124 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=1018491, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_124_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_124_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 124 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 63 (MapPartitionsRDD[131] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Adding task set 63.0 with 2 tasks -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 63.0 (TID 125, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 63.0 (TID 126, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO Executor:59 - Running task 1.0 in stage 63.0 (TID 126) -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:45 INFO Executor:59 - Running task 0.0 in stage 63.0 (TID 125) -2016-05-24 22:22:45 INFO Executor:59 - Finished task 1.0 in stage 63.0 (TID 126). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 63.0 (TID 126) in 7 ms on localhost (1/2) -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:45 INFO Executor:59 - Finished task 0.0 in stage 63.0 (TID 125). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 63.0 (TID 125) in 10 ms on localhost (2/2) -2016-05-24 22:22:45 INFO DAGScheduler:59 - ResultStage 63 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 63.0, whose tasks have all completed, from pool -2016-05-24 22:22:45 INFO DAGScheduler:59 - Job 63 finished: treeAggregate at GradientDescent.scala:225, took 0.021264 s -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1021979, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_125 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1027571, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_125_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_125_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 125 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Got job 64 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:45 INFO DAGScheduler:59 - Final stage: ResultStage 64(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting ResultStage 64 (MapPartitionsRDD[133] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1031184, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_126 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=1042968, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_126_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_126_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 126 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 64 (MapPartitionsRDD[133] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Adding task set 64.0 with 2 tasks -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 64.0 (TID 127, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 64.0 (TID 128, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO Executor:59 - Running task 0.0 in stage 64.0 (TID 127) -2016-05-24 22:22:45 INFO Executor:59 - Running task 1.0 in stage 64.0 (TID 128) -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:45 INFO Executor:59 - Finished task 0.0 in stage 64.0 (TID 127). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO Executor:59 - Finished task 1.0 in stage 64.0 (TID 128). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 64.0 (TID 127) in 7 ms on localhost (1/2) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 64.0 (TID 128) in 6 ms on localhost (2/2) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 64.0, whose tasks have all completed, from pool -2016-05-24 22:22:45 INFO DAGScheduler:59 - ResultStage 64 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:22:45 INFO DAGScheduler:59 - Job 64 finished: treeAggregate at GradientDescent.scala:225, took 0.019976 s -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1046458, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_127 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1052050, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_127_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_127_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 127 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Got job 65 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:45 INFO DAGScheduler:59 - Final stage: ResultStage 65(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[135] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1055662, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_128 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3489) called with curMem=1067446, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_128_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_128_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 128 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 65 (MapPartitionsRDD[135] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 2 tasks -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 129, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 130, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 129) -2016-05-24 22:22:45 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 130) -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:45 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 129). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 130). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 129) in 7 ms on localhost (1/2) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 130) in 7 ms on localhost (2/2) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-05-24 22:22:45 INFO DAGScheduler:59 - ResultStage 65 (treeAggregate at GradientDescent.scala:225) finished in 0.039 s -2016-05-24 22:22:45 INFO DAGScheduler:59 - Job 65 finished: treeAggregate at GradientDescent.scala:225, took 0.053222 s -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1070935, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_129 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1076527, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_129_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_129_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 129 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Got job 66 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:45 INFO DAGScheduler:59 - Final stage: ResultStage 66(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting ResultStage 66 (MapPartitionsRDD[137] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1080140, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_130 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=1091924, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_130_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_130_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 130 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 66 (MapPartitionsRDD[137] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Adding task set 66.0 with 2 tasks -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 66.0 (TID 131, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 66.0 (TID 132, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO Executor:59 - Running task 1.0 in stage 66.0 (TID 132) -2016-05-24 22:22:45 INFO Executor:59 - Running task 0.0 in stage 66.0 (TID 131) -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:45 INFO Executor:59 - Finished task 0.0 in stage 66.0 (TID 131). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO Executor:59 - Finished task 1.0 in stage 66.0 (TID 132). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 66.0 (TID 131) in 5 ms on localhost (1/2) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 66.0 (TID 132) in 6 ms on localhost (2/2) -2016-05-24 22:22:45 INFO DAGScheduler:59 - ResultStage 66 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 66.0, whose tasks have all completed, from pool -2016-05-24 22:22:45 INFO DAGScheduler:59 - Job 66 finished: treeAggregate at GradientDescent.scala:225, took 0.018082 s -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1095414, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_131 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3633) called with curMem=1101006, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_131_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_131_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 131 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Got job 67 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:45 INFO DAGScheduler:59 - Final stage: ResultStage 67(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting ResultStage 67 (MapPartitionsRDD[139] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1104639, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_132 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=1116423, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_132_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_132_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 132 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 67 (MapPartitionsRDD[139] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Adding task set 67.0 with 2 tasks -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 67.0 (TID 133, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 67.0 (TID 134, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO Executor:59 - Running task 1.0 in stage 67.0 (TID 134) -2016-05-24 22:22:45 INFO Executor:59 - Running task 0.0 in stage 67.0 (TID 133) -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:45 INFO Executor:59 - Finished task 0.0 in stage 67.0 (TID 133). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO Executor:59 - Finished task 1.0 in stage 67.0 (TID 134). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 67.0 (TID 133) in 7 ms on localhost (1/2) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 67.0 (TID 134) in 6 ms on localhost (2/2) -2016-05-24 22:22:45 INFO DAGScheduler:59 - ResultStage 67 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 67.0, whose tasks have all completed, from pool -2016-05-24 22:22:45 INFO DAGScheduler:59 - Job 67 finished: treeAggregate at GradientDescent.scala:225, took 0.019402 s -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1119913, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_133 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1125505, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_133_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_133_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 133 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Got job 68 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:45 INFO DAGScheduler:59 - Final stage: ResultStage 68(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting ResultStage 68 (MapPartitionsRDD[141] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1129118, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_134 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=1140902, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_134_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_134_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 134 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 68 (MapPartitionsRDD[141] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Adding task set 68.0 with 2 tasks -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 68.0 (TID 135, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 68.0 (TID 136, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO Executor:59 - Running task 1.0 in stage 68.0 (TID 136) -2016-05-24 22:22:45 INFO Executor:59 - Running task 0.0 in stage 68.0 (TID 135) -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:45 INFO Executor:59 - Finished task 1.0 in stage 68.0 (TID 136). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 68.0 (TID 136) in 8 ms on localhost (1/2) -2016-05-24 22:22:45 INFO Executor:59 - Finished task 0.0 in stage 68.0 (TID 135). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 68.0 (TID 135) in 9 ms on localhost (2/2) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 68.0, whose tasks have all completed, from pool -2016-05-24 22:22:45 INFO DAGScheduler:59 - ResultStage 68 (treeAggregate at GradientDescent.scala:225) finished in 0.042 s -2016-05-24 22:22:45 INFO DAGScheduler:59 - Job 68 finished: treeAggregate at GradientDescent.scala:225, took 0.054561 s -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1144393, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_135 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3608) called with curMem=1149985, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_135_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_135_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 135 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Got job 69 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:45 INFO DAGScheduler:59 - Final stage: ResultStage 69(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting ResultStage 69 (MapPartitionsRDD[143] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1153593, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_136 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=1165377, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_136_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_136_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 136 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 69 (MapPartitionsRDD[143] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Adding task set 69.0 with 2 tasks -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 69.0 (TID 137, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 69.0 (TID 138, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO Executor:59 - Running task 0.0 in stage 69.0 (TID 137) -2016-05-24 22:22:45 INFO Executor:59 - Running task 1.0 in stage 69.0 (TID 138) -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:45 INFO Executor:59 - Finished task 0.0 in stage 69.0 (TID 137). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO Executor:59 - Finished task 1.0 in stage 69.0 (TID 138). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 69.0 (TID 137) in 7 ms on localhost (1/2) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 69.0 (TID 138) in 7 ms on localhost (2/2) -2016-05-24 22:22:45 INFO DAGScheduler:59 - ResultStage 69 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 69.0, whose tasks have all completed, from pool -2016-05-24 22:22:45 INFO DAGScheduler:59 - Job 69 finished: treeAggregate at GradientDescent.scala:225, took 0.022429 s -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1168867, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_137 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1174459, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_137_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_137_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 137 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Got job 70 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:45 INFO DAGScheduler:59 - Final stage: ResultStage 70(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting ResultStage 70 (MapPartitionsRDD[145] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1178065, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_138 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=1189849, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_138_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_138_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 138 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 70 (MapPartitionsRDD[145] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Adding task set 70.0 with 2 tasks -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 70.0 (TID 139, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 70.0 (TID 140, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO Executor:59 - Running task 0.0 in stage 70.0 (TID 139) -2016-05-24 22:22:45 INFO Executor:59 - Running task 1.0 in stage 70.0 (TID 140) -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:45 INFO Executor:59 - Finished task 1.0 in stage 70.0 (TID 140). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO Executor:59 - Finished task 0.0 in stage 70.0 (TID 139). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 70.0 (TID 140) in 5 ms on localhost (1/2) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 70.0 (TID 139) in 7 ms on localhost (2/2) -2016-05-24 22:22:45 INFO DAGScheduler:59 - ResultStage 70 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 70.0, whose tasks have all completed, from pool -2016-05-24 22:22:45 INFO DAGScheduler:59 - Job 70 finished: treeAggregate at GradientDescent.scala:225, took 0.017456 s -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1193339, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_139 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3619) called with curMem=1198931, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_139_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_139_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 139 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Got job 71 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:45 INFO DAGScheduler:59 - Final stage: ResultStage 71(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting ResultStage 71 (MapPartitionsRDD[147] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1202550, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_140 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=1214334, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_140_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_140_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 140 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 71 (MapPartitionsRDD[147] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Adding task set 71.0 with 2 tasks -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 71.0 (TID 141, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 71.0 (TID 142, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO Executor:59 - Running task 0.0 in stage 71.0 (TID 141) -2016-05-24 22:22:45 INFO Executor:59 - Running task 1.0 in stage 71.0 (TID 142) -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:45 INFO Executor:59 - Finished task 0.0 in stage 71.0 (TID 141). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:45 INFO Executor:59 - Finished task 1.0 in stage 71.0 (TID 142). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 71.0 (TID 141) in 39 ms on localhost (1/2) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 71.0 (TID 142) in 39 ms on localhost (2/2) -2016-05-24 22:22:45 INFO DAGScheduler:59 - ResultStage 71 (treeAggregate at GradientDescent.scala:225) finished in 0.041 s -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 71.0, whose tasks have all completed, from pool -2016-05-24 22:22:45 INFO DAGScheduler:59 - Job 71 finished: treeAggregate at GradientDescent.scala:225, took 0.058370 s -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1217825, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_141 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3604) called with curMem=1223417, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_141_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_141_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 141 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Got job 72 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:45 INFO DAGScheduler:59 - Final stage: ResultStage 72(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting ResultStage 72 (MapPartitionsRDD[149] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1227021, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_142 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=1238805, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_142_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_142_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 142 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 72 (MapPartitionsRDD[149] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Adding task set 72.0 with 2 tasks -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 72.0 (TID 143, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 72.0 (TID 144, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO Executor:59 - Running task 0.0 in stage 72.0 (TID 143) -2016-05-24 22:22:45 INFO Executor:59 - Running task 1.0 in stage 72.0 (TID 144) -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:45 INFO Executor:59 - Finished task 0.0 in stage 72.0 (TID 143). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 72.0 (TID 143) in 8 ms on localhost (1/2) -2016-05-24 22:22:45 INFO Executor:59 - Finished task 1.0 in stage 72.0 (TID 144). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 72.0 (TID 144) in 9 ms on localhost (2/2) -2016-05-24 22:22:45 INFO DAGScheduler:59 - ResultStage 72 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 72.0, whose tasks have all completed, from pool -2016-05-24 22:22:45 INFO DAGScheduler:59 - Job 72 finished: treeAggregate at GradientDescent.scala:225, took 0.022598 s -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1242296, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_143 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=1247888, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_143_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_143_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 143 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:45 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Got job 73 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:45 INFO DAGScheduler:59 - Final stage: ResultStage 73(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting ResultStage 73 (MapPartitionsRDD[151] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1251497, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_144 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=1263281, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_144_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_144_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 144 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 73 (MapPartitionsRDD[151] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Adding task set 73.0 with 2 tasks -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 73.0 (TID 145, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 73.0 (TID 146, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:45 INFO Executor:59 - Running task 0.0 in stage 73.0 (TID 145) -2016-05-24 22:22:45 INFO Executor:59 - Running task 1.0 in stage 73.0 (TID 146) -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:45 INFO Executor:59 - Finished task 0.0 in stage 73.0 (TID 145). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO Executor:59 - Finished task 1.0 in stage 73.0 (TID 146). 7968 bytes result sent to driver -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 73.0 (TID 145) in 6 ms on localhost (1/2) -2016-05-24 22:22:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 73.0 (TID 146) in 5 ms on localhost (2/2) -2016-05-24 22:22:45 INFO DAGScheduler:59 - ResultStage 73 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:22:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 73.0, whose tasks have all completed, from pool -2016-05-24 22:22:45 INFO DAGScheduler:59 - Job 73 finished: treeAggregate at GradientDescent.scala:225, took 0.017836 s -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1266771, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_145 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:22:45 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1272363, maxMem=766075207 -2016-05-24 22:22:45 INFO MemoryStore:59 - Block broadcast_145_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:22:45 INFO BlockManagerInfo:59 - Added broadcast_145_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:45 INFO SparkContext:59 - Created broadcast 145 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Got job 74 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:46 INFO DAGScheduler:59 - Final stage: ResultStage 74(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting ResultStage 74 (MapPartitionsRDD[153] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1275969, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_146 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=1287753, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_146_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_146_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 146 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 74 (MapPartitionsRDD[153] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Adding task set 74.0 with 2 tasks -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 74.0 (TID 147, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 74.0 (TID 148, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO Executor:59 - Running task 1.0 in stage 74.0 (TID 148) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 1.0 in stage 74.0 (TID 148). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO Executor:59 - Running task 0.0 in stage 74.0 (TID 147) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 0.0 in stage 74.0 (TID 147). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 74.0 (TID 148) in 34 ms on localhost (1/2) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 74.0 (TID 147) in 35 ms on localhost (2/2) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 74.0, whose tasks have all completed, from pool -2016-05-24 22:22:46 INFO DAGScheduler:59 - ResultStage 74 (treeAggregate at GradientDescent.scala:225) finished in 0.056 s -2016-05-24 22:22:46 INFO DAGScheduler:59 - Job 74 finished: treeAggregate at GradientDescent.scala:225, took 0.071694 s -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1291243, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_147 stored as values in memory (estimated size 5.5 KB, free 729.3 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3624) called with curMem=1296835, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_147_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.3 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_147_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 147 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Got job 75 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:46 INFO DAGScheduler:59 - Final stage: ResultStage 75(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting ResultStage 75 (MapPartitionsRDD[155] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1300459, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_148 stored as values in memory (estimated size 11.5 KB, free 729.3 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=1312243, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_148_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.3 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_148_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 148 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 75 (MapPartitionsRDD[155] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Adding task set 75.0 with 2 tasks -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 75.0 (TID 149, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 75.0 (TID 150, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO Executor:59 - Running task 1.0 in stage 75.0 (TID 150) -2016-05-24 22:22:46 INFO Executor:59 - Running task 0.0 in stage 75.0 (TID 149) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 0.0 in stage 75.0 (TID 149). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 75.0 (TID 149) in 98 ms on localhost (1/2) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 1.0 in stage 75.0 (TID 150). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 75.0 (TID 150) in 103 ms on localhost (2/2) -2016-05-24 22:22:46 INFO DAGScheduler:59 - ResultStage 75 (treeAggregate at GradientDescent.scala:225) finished in 0.104 s -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 75.0, whose tasks have all completed, from pool -2016-05-24 22:22:46 INFO DAGScheduler:59 - Job 75 finished: treeAggregate at GradientDescent.scala:225, took 0.114495 s -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1315733, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_149 stored as values in memory (estimated size 5.5 KB, free 729.3 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3626) called with curMem=1321325, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_149_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.3 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_149_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 149 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_125_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_124_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Got job 76 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:46 INFO DAGScheduler:59 - Final stage: ResultStage 76(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting ResultStage 76 (MapPartitionsRDD[157] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_123_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1291252, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_150 stored as values in memory (estimated size 11.5 KB, free 729.3 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=1287762, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_150_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_122_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_150_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 150 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 76 (MapPartitionsRDD[157] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 2 tasks -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 151, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 152, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 151) -2016-05-24 22:22:46 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 152) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_121_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 151). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_120_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 151) in 7 ms on localhost (1/2) -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-05-24 22:22:46 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 152). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 152) in 10 ms on localhost (2/2) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-05-24 22:22:46 INFO DAGScheduler:59 - ResultStage 76 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:22:46 INFO DAGScheduler:59 - Job 76 finished: treeAggregate at GradientDescent.scala:225, took 0.022869 s -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_119_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1263157, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_151 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3611) called with curMem=1259668, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_151_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_118_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_151_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 151 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_117_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_116_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-05-24 22:22:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Got job 77 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:46 INFO DAGScheduler:59 - Final stage: ResultStage 77(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_115_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[159] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1202534, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_152 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_114_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=1214318, maxMem=766075207 -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_152_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_152_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 152 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 77 (MapPartitionsRDD[159] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 2 tasks -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_113_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 153, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 154, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 154) -2016-05-24 22:22:46 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 153) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 154). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 153). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 154) in 36 ms on localhost (1/2) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 153) in 37 ms on localhost (2/2) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_112_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO DAGScheduler:59 - ResultStage 77 (treeAggregate at GradientDescent.scala:225) finished in 0.037 s -2016-05-24 22:22:46 INFO DAGScheduler:59 - Job 77 finished: treeAggregate at GradientDescent.scala:225, took 0.056218 s -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1193331, maxMem=766075207 -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_153 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1198923, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_153_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_153_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 153 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_111_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Got job 78 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:46 INFO DAGScheduler:59 - Final stage: ResultStage 78(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting ResultStage 78 (MapPartitionsRDD[161] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1189841, maxMem=766075207 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_110_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_154 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=1189841, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_154_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_154_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 154 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 78 (MapPartitionsRDD[161] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 2 tasks -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 155, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_109_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 156, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 155) -2016-05-24 22:22:46 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 156) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_108_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 155). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 156). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 155) in 6 ms on localhost (1/2) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 156) in 7 ms on localhost (2/2) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-05-24 22:22:46 INFO DAGScheduler:59 - ResultStage 78 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Job 78 finished: treeAggregate at GradientDescent.scala:225, took 0.019964 s -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1163261, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_155 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_107_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1165221, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_155_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_155_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 155 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_106_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_105_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_104_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-05-24 22:22:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_103_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Got job 79 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:46 INFO DAGScheduler:59 - Final stage: ResultStage 79(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting ResultStage 79 (MapPartitionsRDD[163] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_102_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1104585, maxMem=766075207 -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_156 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=1107149, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_156_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_101_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_156_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 156 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 79 (MapPartitionsRDD[163] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Adding task set 79.0 with 2 tasks -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 79.0 (TID 157, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 79.0 (TID 158, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO Executor:59 - Running task 0.0 in stage 79.0 (TID 157) -2016-05-24 22:22:46 INFO Executor:59 - Running task 1.0 in stage 79.0 (TID 158) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_100_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 0.0 in stage 79.0 (TID 157). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO Executor:59 - Finished task 1.0 in stage 79.0 (TID 158). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_99_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 79.0 (TID 157) in 6 ms on localhost (1/2) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 79.0 (TID 158) in 7 ms on localhost (2/2) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 79.0, whose tasks have all completed, from pool -2016-05-24 22:22:46 INFO DAGScheduler:59 - ResultStage 79 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:22:46 INFO DAGScheduler:59 - Job 79 finished: treeAggregate at GradientDescent.scala:225, took 0.018125 s -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1082674, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_157 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_98_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1076482, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_157_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_157_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 157 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_97_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_96_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_95_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Got job 80 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:46 INFO DAGScheduler:59 - Final stage: ResultStage 80(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_94_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting ResultStage 80 (MapPartitionsRDD[165] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1027508, maxMem=766075207 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_93_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_158 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=1033700, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_158_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_158_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 158 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_92_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 80 (MapPartitionsRDD[165] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Adding task set 80.0 with 2 tasks -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 80.0 (TID 159, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 80.0 (TID 160, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO Executor:59 - Running task 0.0 in stage 80.0 (TID 159) -2016-05-24 22:22:46 INFO Executor:59 - Running task 1.0 in stage 80.0 (TID 160) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_91_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 1.0 in stage 80.0 (TID 160). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO Executor:59 - Finished task 0.0 in stage 80.0 (TID 159). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 80.0 (TID 159) in 7 ms on localhost (1/2) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_90_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO DAGScheduler:59 - ResultStage 80 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:22:46 INFO DAGScheduler:59 - Job 80 finished: treeAggregate at GradientDescent.scala:225, took 0.019795 s -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 80.0 (TID 160) in 8 ms on localhost (2/2) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 80.0, whose tasks have all completed, from pool -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=997434, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_159 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1003026, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_159_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_159_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 159 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_89_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Got job 81 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:46 INFO DAGScheduler:59 - Final stage: ResultStage 81(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_88_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting ResultStage 81 (MapPartitionsRDD[167] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=993936, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_160 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=1005720, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_160_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_160_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 160 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 81 (MapPartitionsRDD[167] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Adding task set 81.0 with 2 tasks -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 81.0 (TID 161, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 81.0 (TID 162, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO Executor:59 - Running task 0.0 in stage 81.0 (TID 161) -2016-05-24 22:22:46 INFO Executor:59 - Running task 1.0 in stage 81.0 (TID 162) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 0.0 in stage 81.0 (TID 161). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 81.0 (TID 161) in 7 ms on localhost (1/2) -2016-05-24 22:22:46 INFO Executor:59 - Finished task 1.0 in stage 81.0 (TID 162). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 81.0 (TID 162) in 13 ms on localhost (2/2) -2016-05-24 22:22:46 INFO DAGScheduler:59 - ResultStage 81 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 81.0, whose tasks have all completed, from pool -2016-05-24 22:22:46 INFO DAGScheduler:59 - Job 81 finished: treeAggregate at GradientDescent.scala:225, took 0.052559 s -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=997426, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_161 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1003018, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_161_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_161_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 161 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_87_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-05-24 22:22:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Got job 82 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:46 INFO DAGScheduler:59 - Final stage: ResultStage 82(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting ResultStage 82 (MapPartitionsRDD[169] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=957676, maxMem=766075207 -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_162 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=969460, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_162_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_162_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 162 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 82 (MapPartitionsRDD[169] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Adding task set 82.0 with 2 tasks -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 82.0 (TID 163, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 82.0 (TID 164, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO Executor:59 - Running task 0.0 in stage 82.0 (TID 163) -2016-05-24 22:22:46 INFO Executor:59 - Running task 1.0 in stage 82.0 (TID 164) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 0.0 in stage 82.0 (TID 163). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 1.0 in stage 82.0 (TID 164). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 82.0 (TID 164) in 14 ms on localhost (1/2) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 82.0 (TID 163) in 15 ms on localhost (2/2) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 82.0, whose tasks have all completed, from pool -2016-05-24 22:22:46 INFO DAGScheduler:59 - ResultStage 82 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Job 82 finished: treeAggregate at GradientDescent.scala:225, took 0.030151 s -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=942881, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_163 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=944861, maxMem=766075207 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_163_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_163_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 163 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_148_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 76 -2016-05-24 22:22:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Got job 83 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:46 INFO DAGScheduler:59 - Final stage: ResultStage 83(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting ResultStage 83 (MapPartitionsRDD[171] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_147_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=899505, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_164 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_146_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=896015, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_164_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 75 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_164_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 164 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 83 (MapPartitionsRDD[171] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Adding task set 83.0 with 2 tasks -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_145_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 83.0 (TID 165, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 83.0 (TID 166, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO Executor:59 - Running task 1.0 in stage 83.0 (TID 166) -2016-05-24 22:22:46 INFO Executor:59 - Running task 0.0 in stage 83.0 (TID 165) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_144_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 74 -2016-05-24 22:22:46 INFO Executor:59 - Finished task 1.0 in stage 83.0 (TID 166). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO Executor:59 - Finished task 0.0 in stage 83.0 (TID 165). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 83.0 (TID 166) in 6 ms on localhost (1/2) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 83.0 (TID 165) in 7 ms on localhost (2/2) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 83.0, whose tasks have all completed, from pool -2016-05-24 22:22:46 INFO DAGScheduler:59 - ResultStage 83 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:22:46 INFO DAGScheduler:59 - Job 83 finished: treeAggregate at GradientDescent.scala:225, took 0.018165 s -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_143_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=865832, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_165 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=867933, maxMem=766075207 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_142_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_165_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_165_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 73 -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 165 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_141_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_140_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Got job 84 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:46 INFO DAGScheduler:59 - Final stage: ResultStage 84(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 72 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting ResultStage 84 (MapPartitionsRDD[173] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=831671, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_166 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_139_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=837863, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_166_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_166_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_138_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 166 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 84 (MapPartitionsRDD[173] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Adding task set 84.0 with 2 tasks -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 71 -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 84.0 (TID 167, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 84.0 (TID 168, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO Executor:59 - Running task 0.0 in stage 84.0 (TID 167) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 0.0 in stage 84.0 (TID 167). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO Executor:59 - Running task 1.0 in stage 84.0 (TID 168) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_137_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 1.0 in stage 84.0 (TID 168). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 84.0 (TID 167) in 21 ms on localhost (1/2) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 84.0 (TID 168) in 20 ms on localhost (2/2) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 84.0, whose tasks have all completed, from pool -2016-05-24 22:22:46 INFO DAGScheduler:59 - ResultStage 84 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 22:22:46 INFO DAGScheduler:59 - Job 84 finished: treeAggregate at GradientDescent.scala:225, took 0.068944 s -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=801608, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_167 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_136_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3620) called with curMem=807200, maxMem=766075207 -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 70 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_167_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_167_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 167 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_135_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_134_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 69 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_133_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_132_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Got job 85 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Final stage: ResultStage 85(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting ResultStage 85 (MapPartitionsRDD[175] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_131_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=758233, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_168 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=764425, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_168_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_168_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 168 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_130_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 85 (MapPartitionsRDD[175] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Adding task set 85.0 with 2 tasks -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 85.0 (TID 169, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 85.0 (TID 170, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO Executor:59 - Running task 1.0 in stage 85.0 (TID 170) -2016-05-24 22:22:46 INFO Executor:59 - Running task 0.0 in stage 85.0 (TID 169) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_129_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 1.0 in stage 85.0 (TID 170). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO Executor:59 - Finished task 0.0 in stage 85.0 (TID 169). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 85.0 (TID 170) in 7 ms on localhost (1/2) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 85.0 (TID 169) in 8 ms on localhost (2/2) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 85.0, whose tasks have all completed, from pool -2016-05-24 22:22:46 INFO DAGScheduler:59 - ResultStage 85 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:22:46 INFO DAGScheduler:59 - Job 85 finished: treeAggregate at GradientDescent.scala:225, took 0.021005 s -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_128_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=728163, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_169 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3621) called with curMem=728163, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_169_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_127_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_169_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 169 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Removed broadcast_126_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-05-24 22:22:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Got job 86 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:46 INFO DAGScheduler:59 - Final stage: ResultStage 86(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting ResultStage 86 (MapPartitionsRDD[177] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=712898, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_170 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=724682, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_170_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_170_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 170 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 86 (MapPartitionsRDD[177] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Adding task set 86.0 with 2 tasks -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 86.0 (TID 171, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 86.0 (TID 172, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO Executor:59 - Running task 0.0 in stage 86.0 (TID 171) -2016-05-24 22:22:46 INFO Executor:59 - Running task 1.0 in stage 86.0 (TID 172) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 1.0 in stage 86.0 (TID 172). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO Executor:59 - Finished task 0.0 in stage 86.0 (TID 171). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 86.0 (TID 172) in 5 ms on localhost (1/2) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 86.0 (TID 171) in 5 ms on localhost (2/2) -2016-05-24 22:22:46 INFO DAGScheduler:59 - ResultStage 86 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 86.0, whose tasks have all completed, from pool -2016-05-24 22:22:46 INFO DAGScheduler:59 - Job 86 finished: treeAggregate at GradientDescent.scala:225, took 0.015749 s -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=728172, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_171 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=733764, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_171_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_171_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 171 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Got job 87 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:46 INFO DAGScheduler:59 - Final stage: ResultStage 87(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting ResultStage 87 (MapPartitionsRDD[179] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=737376, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_172 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=749160, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_172_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_172_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 172 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 87 (MapPartitionsRDD[179] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Adding task set 87.0 with 2 tasks -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 87.0 (TID 173, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 87.0 (TID 174, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO Executor:59 - Running task 0.0 in stage 87.0 (TID 173) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 0.0 in stage 87.0 (TID 173). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO Executor:59 - Running task 1.0 in stage 87.0 (TID 174) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 87.0 (TID 173) in 9 ms on localhost (1/2) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 1.0 in stage 87.0 (TID 174). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 87.0 (TID 174) in 23 ms on localhost (2/2) -2016-05-24 22:22:46 INFO DAGScheduler:59 - ResultStage 87 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 87.0, whose tasks have all completed, from pool -2016-05-24 22:22:46 INFO DAGScheduler:59 - Job 87 finished: treeAggregate at GradientDescent.scala:225, took 0.057325 s -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=752651, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_173 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=758243, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_173_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_173_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 173 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Got job 88 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:46 INFO DAGScheduler:59 - Final stage: ResultStage 88(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting ResultStage 88 (MapPartitionsRDD[181] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=761856, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_174 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=773640, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_174_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_174_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 174 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 88 (MapPartitionsRDD[181] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Adding task set 88.0 with 2 tasks -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 88.0 (TID 175, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 88.0 (TID 176, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO Executor:59 - Running task 0.0 in stage 88.0 (TID 175) -2016-05-24 22:22:46 INFO Executor:59 - Running task 1.0 in stage 88.0 (TID 176) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 1.0 in stage 88.0 (TID 176). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO Executor:59 - Finished task 0.0 in stage 88.0 (TID 175). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 88.0 (TID 176) in 5 ms on localhost (1/2) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 88.0 (TID 175) in 7 ms on localhost (2/2) -2016-05-24 22:22:46 INFO DAGScheduler:59 - ResultStage 88 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 88.0, whose tasks have all completed, from pool -2016-05-24 22:22:46 INFO DAGScheduler:59 - Job 88 finished: treeAggregate at GradientDescent.scala:225, took 0.016380 s -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=777131, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_175 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3615) called with curMem=782723, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_175_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_175_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 175 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Got job 89 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:46 INFO DAGScheduler:59 - Final stage: ResultStage 89(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting ResultStage 89 (MapPartitionsRDD[183] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=786338, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_176 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=798122, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_176_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_176_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 176 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 89 (MapPartitionsRDD[183] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Adding task set 89.0 with 2 tasks -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 89.0 (TID 177, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 89.0 (TID 178, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO Executor:59 - Running task 0.0 in stage 89.0 (TID 177) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 0.0 in stage 89.0 (TID 177). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO Executor:59 - Running task 1.0 in stage 89.0 (TID 178) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 1.0 in stage 89.0 (TID 178). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 89.0 (TID 177) in 13 ms on localhost (1/2) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 89.0 (TID 178) in 13 ms on localhost (2/2) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 89.0, whose tasks have all completed, from pool -2016-05-24 22:22:46 INFO DAGScheduler:59 - ResultStage 89 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:22:46 INFO DAGScheduler:59 - Job 89 finished: treeAggregate at GradientDescent.scala:225, took 0.030917 s -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=801613, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_177 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=807205, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_177_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_177_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 177 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Got job 90 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:46 INFO DAGScheduler:59 - Final stage: ResultStage 90(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting ResultStage 90 (MapPartitionsRDD[185] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=810817, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_178 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=822601, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_178_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:46 INFO BlockManagerInfo:59 - Added broadcast_178_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:46 INFO SparkContext:59 - Created broadcast 178 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 90 (MapPartitionsRDD[185] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Adding task set 90.0 with 2 tasks -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 90.0 (TID 179, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 90.0 (TID 180, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:46 INFO Executor:59 - Running task 1.0 in stage 90.0 (TID 180) -2016-05-24 22:22:46 INFO Executor:59 - Running task 0.0 in stage 90.0 (TID 179) -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:46 INFO Executor:59 - Finished task 1.0 in stage 90.0 (TID 180). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO Executor:59 - Finished task 0.0 in stage 90.0 (TID 179). 7968 bytes result sent to driver -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 90.0 (TID 180) in 6 ms on localhost (1/2) -2016-05-24 22:22:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 90.0 (TID 179) in 7 ms on localhost (2/2) -2016-05-24 22:22:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 90.0, whose tasks have all completed, from pool -2016-05-24 22:22:46 INFO DAGScheduler:59 - ResultStage 90 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:22:46 INFO DAGScheduler:59 - Job 90 finished: treeAggregate at GradientDescent.scala:225, took 0.046071 s -2016-05-24 22:22:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=826091, maxMem=766075207 -2016-05-24 22:22:46 INFO MemoryStore:59 - Block broadcast_179 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=831683, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_179_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_179_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 179 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Got job 91 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:47 INFO DAGScheduler:59 - Final stage: ResultStage 91(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting ResultStage 91 (MapPartitionsRDD[187] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=835296, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_180 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=847080, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_180_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_180_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 180 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 91 (MapPartitionsRDD[187] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Adding task set 91.0 with 2 tasks -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 91.0 (TID 181, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 91.0 (TID 182, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO Executor:59 - Running task 0.0 in stage 91.0 (TID 181) -2016-05-24 22:22:47 INFO Executor:59 - Running task 1.0 in stage 91.0 (TID 182) -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:47 INFO Executor:59 - Finished task 0.0 in stage 91.0 (TID 181). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO Executor:59 - Finished task 1.0 in stage 91.0 (TID 182). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 91.0 (TID 181) in 6 ms on localhost (1/2) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 91.0 (TID 182) in 7 ms on localhost (2/2) -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 91.0, whose tasks have all completed, from pool -2016-05-24 22:22:47 INFO DAGScheduler:59 - ResultStage 91 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:22:47 INFO DAGScheduler:59 - Job 91 finished: treeAggregate at GradientDescent.scala:225, took 0.017131 s -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=850570, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_181 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3608) called with curMem=856162, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_181_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_181_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 181 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Got job 92 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:47 INFO DAGScheduler:59 - Final stage: ResultStage 92(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting ResultStage 92 (MapPartitionsRDD[189] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=859770, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_182 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=871554, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_182_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_182_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 182 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 92 (MapPartitionsRDD[189] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Adding task set 92.0 with 2 tasks -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 92.0 (TID 183, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 92.0 (TID 184, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO Executor:59 - Running task 0.0 in stage 92.0 (TID 183) -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:47 INFO Executor:59 - Finished task 0.0 in stage 92.0 (TID 183). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO Executor:59 - Running task 1.0 in stage 92.0 (TID 184) -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:47 INFO Executor:59 - Finished task 1.0 in stage 92.0 (TID 184). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 92.0 (TID 183) in 13 ms on localhost (1/2) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 92.0 (TID 184) in 13 ms on localhost (2/2) -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 92.0, whose tasks have all completed, from pool -2016-05-24 22:22:47 INFO DAGScheduler:59 - ResultStage 92 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:22:47 INFO DAGScheduler:59 - Job 92 finished: treeAggregate at GradientDescent.scala:225, took 0.026079 s -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=875045, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_183 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=880637, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_183_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_183_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 183 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Got job 93 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:47 INFO DAGScheduler:59 - Final stage: ResultStage 93(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting ResultStage 93 (MapPartitionsRDD[191] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=884251, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_184 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=896035, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_184_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_184_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 184 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 93 (MapPartitionsRDD[191] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Adding task set 93.0 with 2 tasks -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 93.0 (TID 185, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 93.0 (TID 186, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO Executor:59 - Running task 1.0 in stage 93.0 (TID 186) -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:47 INFO Executor:59 - Finished task 1.0 in stage 93.0 (TID 186). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO Executor:59 - Running task 0.0 in stage 93.0 (TID 185) -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:47 INFO Executor:59 - Finished task 0.0 in stage 93.0 (TID 185). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 93.0 (TID 186) in 43 ms on localhost (1/2) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 93.0 (TID 185) in 44 ms on localhost (2/2) -2016-05-24 22:22:47 INFO DAGScheduler:59 - ResultStage 93 (treeAggregate at GradientDescent.scala:225) finished in 0.044 s -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 93.0, whose tasks have all completed, from pool -2016-05-24 22:22:47 INFO DAGScheduler:59 - Job 93 finished: treeAggregate at GradientDescent.scala:225, took 0.061029 s -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=899525, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_185 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=905117, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_185_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_185_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 185 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Got job 94 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:47 INFO DAGScheduler:59 - Final stage: ResultStage 94(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting ResultStage 94 (MapPartitionsRDD[193] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=908729, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_186 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=920513, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_186_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_186_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 186 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 94 (MapPartitionsRDD[193] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Adding task set 94.0 with 2 tasks -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 94.0 (TID 187, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 94.0 (TID 188, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO Executor:59 - Running task 1.0 in stage 94.0 (TID 188) -2016-05-24 22:22:47 INFO Executor:59 - Running task 0.0 in stage 94.0 (TID 187) -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:47 INFO Executor:59 - Finished task 1.0 in stage 94.0 (TID 188). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO Executor:59 - Finished task 0.0 in stage 94.0 (TID 187). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 94.0 (TID 188) in 6 ms on localhost (1/2) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 94.0 (TID 187) in 9 ms on localhost (2/2) -2016-05-24 22:22:47 INFO DAGScheduler:59 - ResultStage 94 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 94.0, whose tasks have all completed, from pool -2016-05-24 22:22:47 INFO DAGScheduler:59 - Job 94 finished: treeAggregate at GradientDescent.scala:225, took 0.028326 s -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=924003, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_187 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=929595, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_187_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_187_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 187 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Got job 95 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:47 INFO DAGScheduler:59 - Final stage: ResultStage 95(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting ResultStage 95 (MapPartitionsRDD[195] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=933208, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_188 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=944992, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_188_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_188_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 188 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 95 (MapPartitionsRDD[195] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Adding task set 95.0 with 2 tasks -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 95.0 (TID 189, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 95.0 (TID 190, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO Executor:59 - Running task 1.0 in stage 95.0 (TID 190) -2016-05-24 22:22:47 INFO Executor:59 - Running task 0.0 in stage 95.0 (TID 189) -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:47 INFO Executor:59 - Finished task 1.0 in stage 95.0 (TID 190). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO Executor:59 - Finished task 0.0 in stage 95.0 (TID 189). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 95.0 (TID 190) in 4 ms on localhost (1/2) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 95.0 (TID 189) in 6 ms on localhost (2/2) -2016-05-24 22:22:47 INFO DAGScheduler:59 - ResultStage 95 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 95.0, whose tasks have all completed, from pool -2016-05-24 22:22:47 INFO DAGScheduler:59 - Job 95 finished: treeAggregate at GradientDescent.scala:225, took 0.015462 s -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=948483, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_189 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3634) called with curMem=954075, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_189_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_189_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 189 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Got job 96 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:47 INFO DAGScheduler:59 - Final stage: ResultStage 96(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting ResultStage 96 (MapPartitionsRDD[197] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=957709, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_190 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=969493, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_190_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_190_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 190 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 96 (MapPartitionsRDD[197] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Adding task set 96.0 with 2 tasks -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 96.0 (TID 191, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 96.0 (TID 192, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO Executor:59 - Running task 0.0 in stage 96.0 (TID 191) -2016-05-24 22:22:47 INFO Executor:59 - Running task 1.0 in stage 96.0 (TID 192) -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:47 INFO Executor:59 - Finished task 0.0 in stage 96.0 (TID 191). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO Executor:59 - Finished task 1.0 in stage 96.0 (TID 192). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 96.0 (TID 191) in 6 ms on localhost (1/2) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 96.0 (TID 192) in 6 ms on localhost (2/2) -2016-05-24 22:22:47 INFO DAGScheduler:59 - ResultStage 96 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 96.0, whose tasks have all completed, from pool -2016-05-24 22:22:47 INFO DAGScheduler:59 - Job 96 finished: treeAggregate at GradientDescent.scala:225, took 0.015392 s -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=972984, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_191 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3619) called with curMem=978576, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_191_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_191_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 191 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Got job 97 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:47 INFO DAGScheduler:59 - Final stage: ResultStage 97(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting ResultStage 97 (MapPartitionsRDD[199] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=982195, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_192 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=993979, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_192_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_192_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 192 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 97 (MapPartitionsRDD[199] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Adding task set 97.0 with 2 tasks -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 97.0 (TID 193, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 97.0 (TID 194, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO Executor:59 - Running task 1.0 in stage 97.0 (TID 194) -2016-05-24 22:22:47 INFO Executor:59 - Running task 0.0 in stage 97.0 (TID 193) -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:47 INFO Executor:59 - Finished task 1.0 in stage 97.0 (TID 194). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO Executor:59 - Finished task 0.0 in stage 97.0 (TID 193). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 97.0 (TID 194) in 6 ms on localhost (1/2) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 97.0 (TID 193) in 7 ms on localhost (2/2) -2016-05-24 22:22:47 INFO DAGScheduler:59 - ResultStage 97 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 97.0, whose tasks have all completed, from pool -2016-05-24 22:22:47 INFO DAGScheduler:59 - Job 97 finished: treeAggregate at GradientDescent.scala:225, took 0.037904 s -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=997469, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_193 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1003061, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_193_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_193_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 193 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Got job 98 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:47 INFO DAGScheduler:59 - Final stage: ResultStage 98(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting ResultStage 98 (MapPartitionsRDD[201] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1006673, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_194 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=1018457, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_194_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_194_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 194 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 98 (MapPartitionsRDD[201] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Adding task set 98.0 with 2 tasks -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 98.0 (TID 195, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 98.0 (TID 196, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO Executor:59 - Running task 1.0 in stage 98.0 (TID 196) -2016-05-24 22:22:47 INFO Executor:59 - Running task 0.0 in stage 98.0 (TID 195) -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:47 INFO Executor:59 - Finished task 1.0 in stage 98.0 (TID 196). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO Executor:59 - Finished task 0.0 in stage 98.0 (TID 195). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 98.0 (TID 196) in 5 ms on localhost (1/2) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 98.0 (TID 195) in 6 ms on localhost (2/2) -2016-05-24 22:22:47 INFO DAGScheduler:59 - ResultStage 98 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 98.0, whose tasks have all completed, from pool -2016-05-24 22:22:47 INFO DAGScheduler:59 - Job 98 finished: treeAggregate at GradientDescent.scala:225, took 0.016552 s -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1021947, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_195 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=1027539, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_195_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_195_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 195 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Got job 99 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:47 INFO DAGScheduler:59 - Final stage: ResultStage 99(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting ResultStage 99 (MapPartitionsRDD[203] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1031148, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_196 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=1042932, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_196_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_196_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 196 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 99 (MapPartitionsRDD[203] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Adding task set 99.0 with 2 tasks -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 99.0 (TID 197, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 99.0 (TID 198, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO Executor:59 - Running task 0.0 in stage 99.0 (TID 197) -2016-05-24 22:22:47 INFO Executor:59 - Running task 1.0 in stage 99.0 (TID 198) -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:47 INFO Executor:59 - Finished task 1.0 in stage 99.0 (TID 198). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO Executor:59 - Finished task 0.0 in stage 99.0 (TID 197). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 99.0 (TID 198) in 5 ms on localhost (1/2) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 99.0 (TID 197) in 5 ms on localhost (2/2) -2016-05-24 22:22:47 INFO DAGScheduler:59 - ResultStage 99 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 99.0, whose tasks have all completed, from pool -2016-05-24 22:22:47 INFO DAGScheduler:59 - Job 99 finished: treeAggregate at GradientDescent.scala:225, took 0.014216 s -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1046422, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_197 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3616) called with curMem=1052014, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_197_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_197_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 197 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Got job 100 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:47 INFO DAGScheduler:59 - Final stage: ResultStage 100(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting ResultStage 100 (MapPartitionsRDD[205] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1055630, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_198 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3492) called with curMem=1067414, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_198_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_198_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 198 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 100 (MapPartitionsRDD[205] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Adding task set 100.0 with 2 tasks -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 100.0 (TID 199, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 100.0 (TID 200, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO Executor:59 - Running task 1.0 in stage 100.0 (TID 200) -2016-05-24 22:22:47 INFO Executor:59 - Running task 0.0 in stage 100.0 (TID 199) -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:47 INFO Executor:59 - Finished task 0.0 in stage 100.0 (TID 199). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO Executor:59 - Finished task 1.0 in stage 100.0 (TID 200). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 100.0 (TID 199) in 6 ms on localhost (1/2) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 100.0 (TID 200) in 5 ms on localhost (2/2) -2016-05-24 22:22:47 INFO DAGScheduler:59 - ResultStage 100 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 100.0, whose tasks have all completed, from pool -2016-05-24 22:22:47 INFO DAGScheduler:59 - Job 100 finished: treeAggregate at GradientDescent.scala:225, took 0.014546 s -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1070906, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_199 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1076498, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_199_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_199_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 199 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Got job 101 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:47 INFO DAGScheduler:59 - Final stage: ResultStage 101(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting ResultStage 101 (MapPartitionsRDD[207] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1080104, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_200 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=1091888, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_200_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_200_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 200 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 101 (MapPartitionsRDD[207] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Adding task set 101.0 with 2 tasks -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 101.0 (TID 201, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 101.0 (TID 202, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO Executor:59 - Running task 1.0 in stage 101.0 (TID 202) -2016-05-24 22:22:47 INFO Executor:59 - Running task 0.0 in stage 101.0 (TID 201) -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:47 INFO Executor:59 - Finished task 1.0 in stage 101.0 (TID 202). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO Executor:59 - Finished task 0.0 in stage 101.0 (TID 201). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 101.0 (TID 202) in 6 ms on localhost (1/2) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 101.0 (TID 201) in 7 ms on localhost (2/2) -2016-05-24 22:22:47 INFO DAGScheduler:59 - ResultStage 101 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 101.0, whose tasks have all completed, from pool -2016-05-24 22:22:47 INFO DAGScheduler:59 - Job 101 finished: treeAggregate at GradientDescent.scala:225, took 0.047985 s -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1095379, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_201 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1100971, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_201_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_201_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 201 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Got job 102 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:47 INFO DAGScheduler:59 - Final stage: ResultStage 102(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting ResultStage 102 (MapPartitionsRDD[209] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1104584, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_202 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=1116368, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_202_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_202_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 202 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 102 (MapPartitionsRDD[209] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Adding task set 102.0 with 2 tasks -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 102.0 (TID 203, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 102.0 (TID 204, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO Executor:59 - Running task 0.0 in stage 102.0 (TID 203) -2016-05-24 22:22:47 INFO Executor:59 - Running task 1.0 in stage 102.0 (TID 204) -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:47 INFO Executor:59 - Finished task 0.0 in stage 102.0 (TID 203). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO Executor:59 - Finished task 1.0 in stage 102.0 (TID 204). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 102.0 (TID 203) in 5 ms on localhost (1/2) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 102.0 (TID 204) in 5 ms on localhost (2/2) -2016-05-24 22:22:47 INFO DAGScheduler:59 - ResultStage 102 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 102.0, whose tasks have all completed, from pool -2016-05-24 22:22:47 INFO DAGScheduler:59 - Job 102 finished: treeAggregate at GradientDescent.scala:225, took 0.018488 s -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1119858, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_203 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1125450, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_203_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_203_piece0 in memory on localhost:56918 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 203 from broadcast at GradientDescent.scala:221 -2016-05-24 22:22:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Got job 103 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:22:47 INFO DAGScheduler:59 - Final stage: ResultStage 103(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting ResultStage 103 (MapPartitionsRDD[211] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1129062, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_204 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=1140846, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_204_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_204_piece0 in memory on localhost:56918 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 204 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 103 (MapPartitionsRDD[211] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Adding task set 103.0 with 2 tasks -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 103.0 (TID 205, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 103.0 (TID 206, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:22:47 INFO Executor:59 - Running task 0.0 in stage 103.0 (TID 205) -2016-05-24 22:22:47 INFO Executor:59 - Running task 1.0 in stage 103.0 (TID 206) -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:22:47 INFO Executor:59 - Finished task 1.0 in stage 103.0 (TID 206). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO Executor:59 - Finished task 0.0 in stage 103.0 (TID 205). 7968 bytes result sent to driver -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 103.0 (TID 206) in 6 ms on localhost (1/2) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 103.0 (TID 205) in 6 ms on localhost (2/2) -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 103.0, whose tasks have all completed, from pool -2016-05-24 22:22:47 INFO DAGScheduler:59 - ResultStage 103 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:22:47 INFO DAGScheduler:59 - Job 103 finished: treeAggregate at GradientDescent.scala:225, took 0.017246 s -2016-05-24 22:22:47 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 3363.579936927662, 3356.5316534793847, 3349.536449573466, 3342.59343622967, 3335.701748404667, 3328.860544103912, 3322.069003535364, 3315.3263283026713, 3308.6317406356134, 3301.984482655661 -2016-05-24 22:22:47 INFO SparkContext:59 - Starting job: sortByKey at BinaryClassificationMetrics.scala:156 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Registering RDD 212 (map at SVMFromSparkLearning.scala:44) -2016-05-24 22:22:47 INFO DAGScheduler:59 - Got job 104 (sortByKey at BinaryClassificationMetrics.scala:156) with 2 output partitions -2016-05-24 22:22:47 INFO DAGScheduler:59 - Final stage: ResultStage 105(sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:22:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 104) -2016-05-24 22:22:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 104) -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 104 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44), which has no missing parents -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=1144336, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_205 stored as values in memory (estimated size 10.8 KB, free 729.5 MB) -2016-05-24 22:22:47 INFO MemoryStore:59 - ensureFreeSpace(6440) called with curMem=1155344, maxMem=766075207 -2016-05-24 22:22:47 INFO MemoryStore:59 - Block broadcast_205_piece0 stored as bytes in memory (estimated size 6.3 KB, free 729.5 MB) -2016-05-24 22:22:47 INFO BlockManagerInfo:59 - Added broadcast_205_piece0 in memory on localhost:56918 (size: 6.3 KB, free: 730.1 MB) -2016-05-24 22:22:47 INFO SparkContext:59 - Created broadcast 205 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 104 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44) -2016-05-24 22:22:47 INFO TaskSchedulerImpl:59 - Adding task set 104.0 with 2 tasks -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 104.0 (TID 207, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:22:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 104.0 (TID 208, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:22:47 INFO Executor:59 - Running task 0.0 in stage 104.0 (TID 207) -2016-05-24 22:22:47 INFO Executor:59 - Running task 1.0 in stage 104.0 (TID 208) -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:22:47 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:22:48 INFO Executor:59 - Finished task 1.0 in stage 104.0 (TID 208). 2254 bytes result sent to driver -2016-05-24 22:22:48 INFO Executor:59 - Finished task 0.0 in stage 104.0 (TID 207). 2254 bytes result sent to driver -2016-05-24 22:22:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 104.0 (TID 208) in 866 ms on localhost (1/2) -2016-05-24 22:22:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 104.0 (TID 207) in 868 ms on localhost (2/2) -2016-05-24 22:22:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 104.0, whose tasks have all completed, from pool -2016-05-24 22:22:48 INFO DAGScheduler:59 - ShuffleMapStage 104 (map at SVMFromSparkLearning.scala:44) finished in 0.905 s -2016-05-24 22:22:48 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:22:48 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:22:48 INFO DAGScheduler:59 - waiting: Set(ResultStage 105) -2016-05-24 22:22:48 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:22:48 INFO DAGScheduler:59 - Missing parents for ResultStage 105: List() -2016-05-24 22:22:48 INFO DAGScheduler:59 - Submitting ResultStage 105 (MapPartitionsRDD[215] at sortByKey at BinaryClassificationMetrics.scala:156), which is now runnable -2016-05-24 22:22:48 INFO MemoryStore:59 - ensureFreeSpace(3344) called with curMem=1161784, maxMem=766075207 -2016-05-24 22:22:48 INFO MemoryStore:59 - Block broadcast_206 stored as values in memory (estimated size 3.3 KB, free 729.5 MB) -2016-05-24 22:22:48 INFO MemoryStore:59 - ensureFreeSpace(1900) called with curMem=1165128, maxMem=766075207 -2016-05-24 22:22:48 INFO MemoryStore:59 - Block broadcast_206_piece0 stored as bytes in memory (estimated size 1900.0 B, free 729.5 MB) -2016-05-24 22:22:48 INFO BlockManagerInfo:59 - Added broadcast_206_piece0 in memory on localhost:56918 (size: 1900.0 B, free: 730.1 MB) -2016-05-24 22:22:48 INFO SparkContext:59 - Created broadcast 206 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 105 (MapPartitionsRDD[215] at sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:22:48 INFO TaskSchedulerImpl:59 - Adding task set 105.0 with 2 tasks -2016-05-24 22:22:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 105.0 (TID 209, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:22:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 105.0 (TID 210, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:22:48 INFO Executor:59 - Running task 1.0 in stage 105.0 (TID 210) -2016-05-24 22:22:48 INFO Executor:59 - Running task 0.0 in stage 105.0 (TID 209) -2016-05-24 22:22:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:22:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:22:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-24 22:22:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-24 22:22:48 INFO Executor:59 - Finished task 1.0 in stage 105.0 (TID 210). 1517 bytes result sent to driver -2016-05-24 22:22:48 INFO Executor:59 - Finished task 0.0 in stage 105.0 (TID 209). 1469 bytes result sent to driver -2016-05-24 22:22:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 105.0 (TID 210) in 50 ms on localhost (1/2) -2016-05-24 22:22:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 105.0 (TID 209) in 51 ms on localhost (2/2) -2016-05-24 22:22:48 INFO DAGScheduler:59 - ResultStage 105 (sortByKey at BinaryClassificationMetrics.scala:156) finished in 0.053 s -2016-05-24 22:22:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 105.0, whose tasks have all completed, from pool -2016-05-24 22:22:48 INFO DAGScheduler:59 - Job 104 finished: sortByKey at BinaryClassificationMetrics.scala:156, took 0.996072 s -2016-05-24 22:22:48 INFO SparkContext:59 - Starting job: collect at BinaryClassificationMetrics.scala:193 -2016-05-24 22:22:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 154 bytes -2016-05-24 22:22:48 INFO DAGScheduler:59 - Registering RDD 213 (combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:22:48 INFO DAGScheduler:59 - Got job 105 (collect at BinaryClassificationMetrics.scala:193) with 2 output partitions -2016-05-24 22:22:48 INFO DAGScheduler:59 - Final stage: ResultStage 108(collect at BinaryClassificationMetrics.scala:193) -2016-05-24 22:22:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 107) -2016-05-24 22:22:48 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 107) -2016-05-24 22:22:48 INFO DAGScheduler:59 - Submitting ShuffleMapStage 107 (ShuffledRDD[213] at combineByKey at BinaryClassificationMetrics.scala:152), which has no missing parents -2016-05-24 22:22:48 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=1167028, maxMem=766075207 -2016-05-24 22:22:48 INFO MemoryStore:59 - Block broadcast_207 stored as values in memory (estimated size 2.8 KB, free 729.5 MB) -2016-05-24 22:22:48 INFO MemoryStore:59 - ensureFreeSpace(1658) called with curMem=1169924, maxMem=766075207 -2016-05-24 22:22:48 INFO MemoryStore:59 - Block broadcast_207_piece0 stored as bytes in memory (estimated size 1658.0 B, free 729.5 MB) -2016-05-24 22:22:48 INFO BlockManagerInfo:59 - Added broadcast_207_piece0 in memory on localhost:56918 (size: 1658.0 B, free: 730.1 MB) -2016-05-24 22:22:48 INFO SparkContext:59 - Created broadcast 207 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 107 (ShuffledRDD[213] at combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:22:48 INFO TaskSchedulerImpl:59 - Adding task set 107.0 with 2 tasks -2016-05-24 22:22:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 107.0 (TID 211, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:22:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 107.0 (TID 212, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:22:48 INFO Executor:59 - Running task 1.0 in stage 107.0 (TID 212) -2016-05-24 22:22:48 INFO Executor:59 - Running task 0.0 in stage 107.0 (TID 211) -2016-05-24 22:22:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:22:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:22:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:22:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:22:48 INFO Executor:59 - Finished task 0.0 in stage 107.0 (TID 211). 1375 bytes result sent to driver -2016-05-24 22:22:48 INFO Executor:59 - Finished task 1.0 in stage 107.0 (TID 212). 1375 bytes result sent to driver -2016-05-24 22:22:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 107.0 (TID 211) in 75 ms on localhost (1/2) -2016-05-24 22:22:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 107.0 (TID 212) in 74 ms on localhost (2/2) -2016-05-24 22:22:48 INFO DAGScheduler:59 - ShuffleMapStage 107 (combineByKey at BinaryClassificationMetrics.scala:152) finished in 0.077 s -2016-05-24 22:22:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 107.0, whose tasks have all completed, from pool -2016-05-24 22:22:48 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:22:48 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:22:48 INFO DAGScheduler:59 - waiting: Set(ResultStage 108) -2016-05-24 22:22:48 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:22:48 INFO DAGScheduler:59 - Missing parents for ResultStage 108: List() -2016-05-24 22:22:48 INFO DAGScheduler:59 - Submitting ResultStage 108 (MapPartitionsRDD[218] at mapPartitions at BinaryClassificationMetrics.scala:189), which is now runnable -2016-05-24 22:22:48 INFO MemoryStore:59 - ensureFreeSpace(3472) called with curMem=1171582, maxMem=766075207 -2016-05-24 22:22:48 INFO MemoryStore:59 - Block broadcast_208 stored as values in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:22:48 INFO MemoryStore:59 - ensureFreeSpace(1981) called with curMem=1175054, maxMem=766075207 -2016-05-24 22:22:48 INFO MemoryStore:59 - Block broadcast_208_piece0 stored as bytes in memory (estimated size 1981.0 B, free 729.5 MB) -2016-05-24 22:22:48 INFO BlockManagerInfo:59 - Added broadcast_208_piece0 in memory on localhost:56918 (size: 1981.0 B, free: 730.1 MB) -2016-05-24 22:22:48 INFO SparkContext:59 - Created broadcast 208 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 108 (MapPartitionsRDD[218] at mapPartitions at BinaryClassificationMetrics.scala:189) -2016-05-24 22:22:48 INFO TaskSchedulerImpl:59 - Adding task set 108.0 with 2 tasks -2016-05-24 22:22:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 108.0 (TID 213, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:22:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 108.0 (TID 214, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:22:48 INFO Executor:59 - Running task 0.0 in stage 108.0 (TID 213) -2016-05-24 22:22:48 INFO Executor:59 - Running task 1.0 in stage 108.0 (TID 214) -2016-05-24 22:22:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:22:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:22:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:22:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:22:48 INFO Executor:59 - Finished task 0.0 in stage 108.0 (TID 213). 1330 bytes result sent to driver -2016-05-24 22:22:48 INFO Executor:59 - Finished task 1.0 in stage 108.0 (TID 214). 1330 bytes result sent to driver -2016-05-24 22:22:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 108.0 (TID 213) in 22 ms on localhost (1/2) -2016-05-24 22:22:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 108.0 (TID 214) in 21 ms on localhost (2/2) -2016-05-24 22:22:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 108.0, whose tasks have all completed, from pool -2016-05-24 22:22:48 INFO DAGScheduler:59 - ResultStage 108 (collect at BinaryClassificationMetrics.scala:193) finished in 0.022 s -2016-05-24 22:22:48 INFO DAGScheduler:59 - Job 105 finished: collect at BinaryClassificationMetrics.scala:193, took 0.125339 s -2016-05-24 22:22:48 INFO BinaryClassificationMetrics:59 - Total counts: {numPos: 25, numNeg: 11} -2016-05-24 22:22:48 INFO SparkContext:59 - Starting job: runJob at SlidingRDD.scala:75 -2016-05-24 22:22:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 154 bytes -2016-05-24 22:22:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-24 22:22:48 INFO DAGScheduler:59 - Got job 106 (runJob at SlidingRDD.scala:75) with 3 output partitions -2016-05-24 22:22:48 INFO DAGScheduler:59 - Final stage: ResultStage 111(runJob at SlidingRDD.scala:75) -2016-05-24 22:22:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 110) -2016-05-24 22:22:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:48 INFO DAGScheduler:59 - Submitting ResultStage 111 (UnionRDD[224] at UnionRDD at BinaryClassificationMetrics.scala:92), which has no missing parents -2016-05-24 22:22:48 INFO MemoryStore:59 - ensureFreeSpace(5184) called with curMem=1177035, maxMem=766075207 -2016-05-24 22:22:48 INFO MemoryStore:59 - Block broadcast_209 stored as values in memory (estimated size 5.1 KB, free 729.5 MB) -2016-05-24 22:22:48 INFO MemoryStore:59 - ensureFreeSpace(2787) called with curMem=1182219, maxMem=766075207 -2016-05-24 22:22:48 INFO MemoryStore:59 - Block broadcast_209_piece0 stored as bytes in memory (estimated size 2.7 KB, free 729.5 MB) -2016-05-24 22:22:48 INFO BlockManagerInfo:59 - Added broadcast_209_piece0 in memory on localhost:56918 (size: 2.7 KB, free: 730.1 MB) -2016-05-24 22:22:48 INFO SparkContext:59 - Created broadcast 209 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:48 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 111 (UnionRDD[224] at UnionRDD at BinaryClassificationMetrics.scala:92) -2016-05-24 22:22:48 INFO TaskSchedulerImpl:59 - Adding task set 111.0 with 3 tasks -2016-05-24 22:22:48 INFO BlockManagerInfo:59 - Removed broadcast_208_piece0 on localhost:56918 in memory (size: 1981.0 B, free: 730.1 MB) -2016-05-24 22:22:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 111.0 (TID 215, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:22:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 111.0 (TID 216, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:22:48 INFO ContextCleaner:59 - Cleaned accumulator 108 -2016-05-24 22:22:48 INFO BlockManagerInfo:59 - Removed broadcast_207_piece0 on localhost:56918 in memory (size: 1658.0 B, free: 730.1 MB) -2016-05-24 22:22:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 111.0 (TID 217, localhost, PROCESS_LOCAL, 2318 bytes) -2016-05-24 22:22:48 INFO Executor:59 - Running task 0.0 in stage 111.0 (TID 215) -2016-05-24 22:22:48 INFO ContextCleaner:59 - Cleaned accumulator 107 -2016-05-24 22:22:48 INFO Executor:59 - Running task 1.0 in stage 111.0 (TID 216) -2016-05-24 22:22:48 INFO BlockManagerInfo:59 - Removed broadcast_206_piece0 on localhost:56918 in memory (size: 1900.0 B, free: 730.1 MB) -2016-05-24 22:22:48 INFO Executor:59 - Running task 2.0 in stage 111.0 (TID 217) -2016-05-24 22:22:48 INFO CacheManager:59 - Partition rdd_219_0 not found, computing it -2016-05-24 22:22:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:22:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:22:48 INFO Executor:59 - Finished task 2.0 in stage 111.0 (TID 217). 1053 bytes result sent to driver -2016-05-24 22:22:48 INFO CacheManager:59 - Partition rdd_219_1 not found, computing it -2016-05-24 22:22:48 INFO ContextCleaner:59 - Cleaned accumulator 106 -2016-05-24 22:22:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:22:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:22:48 INFO MemoryStore:59 - ensureFreeSpace(1448) called with curMem=1158747, maxMem=766075207 -2016-05-24 22:22:48 INFO MemoryStore:59 - Block rdd_219_0 stored as values in memory (estimated size 1448.0 B, free 729.5 MB) -2016-05-24 22:22:48 INFO BlockManagerInfo:59 - Added rdd_219_0 in memory on localhost:56918 (size: 1448.0 B, free: 730.1 MB) -2016-05-24 22:22:48 INFO BlockManagerInfo:59 - Removed broadcast_205_piece0 on localhost:56918 in memory (size: 6.3 KB, free: 730.1 MB) -2016-05-24 22:22:48 INFO Executor:59 - Finished task 0.0 in stage 111.0 (TID 215). 1883 bytes result sent to driver -2016-05-24 22:22:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 111.0 (TID 217) in 20 ms on localhost (1/3) -2016-05-24 22:22:48 INFO ContextCleaner:59 - Cleaned accumulator 105 -2016-05-24 22:22:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 111.0 (TID 215) in 29 ms on localhost (2/3) -2016-05-24 22:22:48 INFO MemoryStore:59 - ensureFreeSpace(1616) called with curMem=1150265, maxMem=766075207 -2016-05-24 22:22:48 INFO MemoryStore:59 - Block rdd_219_1 stored as values in memory (estimated size 1616.0 B, free 729.5 MB) -2016-05-24 22:22:48 INFO BlockManagerInfo:59 - Removed broadcast_204_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:48 INFO BlockManagerInfo:59 - Added rdd_219_1 in memory on localhost:56918 (size: 1616.0 B, free: 730.1 MB) -2016-05-24 22:22:48 INFO ContextCleaner:59 - Cleaned accumulator 104 -2016-05-24 22:22:48 INFO Executor:59 - Finished task 1.0 in stage 111.0 (TID 216). 1883 bytes result sent to driver -2016-05-24 22:22:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 111.0 (TID 216) in 32 ms on localhost (3/3) -2016-05-24 22:22:48 INFO DAGScheduler:59 - ResultStage 111 (runJob at SlidingRDD.scala:75) finished in 0.035 s -2016-05-24 22:22:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 111.0, whose tasks have all completed, from pool -2016-05-24 22:22:48 INFO DAGScheduler:59 - Job 106 finished: runJob at SlidingRDD.scala:75, took 0.127507 s -2016-05-24 22:22:48 INFO BlockManagerInfo:59 - Removed broadcast_203_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:48 INFO BlockManagerInfo:59 - Removed broadcast_202_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:48 INFO ContextCleaner:59 - Cleaned accumulator 103 -2016-05-24 22:22:48 INFO SparkContext:59 - Starting job: aggregate at AreaUnderCurve.scala:45 -2016-05-24 22:22:48 INFO DAGScheduler:59 - Got job 107 (aggregate at AreaUnderCurve.scala:45) with 4 output partitions -2016-05-24 22:22:48 INFO DAGScheduler:59 - Final stage: ResultStage 114(aggregate at AreaUnderCurve.scala:45) -2016-05-24 22:22:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 113) -2016-05-24 22:22:48 INFO BlockManagerInfo:59 - Removed broadcast_201_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:48 INFO DAGScheduler:59 - Submitting ResultStage 114 (SlidingRDD[225] at RDD at SlidingRDD.scala:47), which has no missing parents -2016-05-24 22:22:48 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=1094630, maxMem=766075207 -2016-05-24 22:22:48 INFO MemoryStore:59 - Block broadcast_210 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:22:48 INFO MemoryStore:59 - ensureFreeSpace(2970) called with curMem=1096795, maxMem=766075207 -2016-05-24 22:22:48 INFO BlockManagerInfo:59 - Removed broadcast_200_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:48 INFO MemoryStore:59 - Block broadcast_210_piece0 stored as bytes in memory (estimated size 2.9 KB, free 729.5 MB) -2016-05-24 22:22:48 INFO ContextCleaner:59 - Cleaned accumulator 102 -2016-05-24 22:22:48 INFO BlockManagerInfo:59 - Added broadcast_210_piece0 in memory on localhost:56918 (size: 2.9 KB, free: 730.1 MB) -2016-05-24 22:22:48 INFO SparkContext:59 - Created broadcast 210 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 114 (SlidingRDD[225] at RDD at SlidingRDD.scala:47) -2016-05-24 22:22:48 INFO TaskSchedulerImpl:59 - Adding task set 114.0 with 4 tasks -2016-05-24 22:22:48 INFO BlockManagerInfo:59 - Removed broadcast_199_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:48 INFO BlockManagerInfo:59 - Removed broadcast_198_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 114.0 (TID 218, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:22:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 114.0 (TID 219, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:22:48 INFO ContextCleaner:59 - Cleaned accumulator 101 -2016-05-24 22:22:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 114.0 (TID 220, localhost, PROCESS_LOCAL, 2635 bytes) -2016-05-24 22:22:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 114.0 (TID 221, localhost, PROCESS_LOCAL, 2475 bytes) -2016-05-24 22:22:48 INFO Executor:59 - Running task 1.0 in stage 114.0 (TID 218) -2016-05-24 22:22:48 INFO Executor:59 - Running task 3.0 in stage 114.0 (TID 221) -2016-05-24 22:22:48 INFO BlockManagerInfo:59 - Removed broadcast_197_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:48 INFO Executor:59 - Running task 2.0 in stage 114.0 (TID 219) -2016-05-24 22:22:48 INFO BlockManager:59 - Found block rdd_219_0 locally -2016-05-24 22:22:48 INFO BlockManager:59 - Found block rdd_219_1 locally -2016-05-24 22:22:48 INFO Executor:59 - Finished task 3.0 in stage 114.0 (TID 221). 955 bytes result sent to driver -2016-05-24 22:22:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 114.0 (TID 221) in 13 ms on localhost (1/4) -2016-05-24 22:22:48 INFO Executor:59 - Finished task 1.0 in stage 114.0 (TID 218). 2084 bytes result sent to driver -2016-05-24 22:22:48 INFO Executor:59 - Finished task 2.0 in stage 114.0 (TID 219). 2084 bytes result sent to driver -2016-05-24 22:22:48 INFO Executor:59 - Running task 0.0 in stage 114.0 (TID 220) -2016-05-24 22:22:49 INFO Executor:59 - Finished task 0.0 in stage 114.0 (TID 220). 955 bytes result sent to driver -2016-05-24 22:22:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 114.0 (TID 220) in 25 ms on localhost (2/4) -2016-05-24 22:22:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 114.0 (TID 219) in 27 ms on localhost (3/4) -2016-05-24 22:22:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 114.0 (TID 218) in 32 ms on localhost (4/4) -2016-05-24 22:22:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 114.0, whose tasks have all completed, from pool -2016-05-24 22:22:49 INFO DAGScheduler:59 - ResultStage 114 (aggregate at AreaUnderCurve.scala:45) finished in 0.033 s -2016-05-24 22:22:49 INFO DAGScheduler:59 - Job 107 finished: aggregate at AreaUnderCurve.scala:45, took 0.048519 s -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_196_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:49 INFO SparkContext:59 - Starting job: take at SVMFromSparkLearning.scala:56 -2016-05-24 22:22:49 INFO DAGScheduler:59 - Got job 108 (take at SVMFromSparkLearning.scala:56) with 1 output partitions -2016-05-24 22:22:49 INFO DAGScheduler:59 - Final stage: ResultStage 115(take at SVMFromSparkLearning.scala:56) -2016-05-24 22:22:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 100 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_195_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:49 INFO DAGScheduler:59 - Submitting ResultStage 115 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44), which has no missing parents -2016-05-24 22:22:49 INFO MemoryStore:59 - ensureFreeSpace(10312) called with curMem=1041608, maxMem=766075207 -2016-05-24 22:22:49 INFO MemoryStore:59 - Block broadcast_211 stored as values in memory (estimated size 10.1 KB, free 729.6 MB) -2016-05-24 22:22:49 INFO MemoryStore:59 - ensureFreeSpace(6089) called with curMem=1051920, maxMem=766075207 -2016-05-24 22:22:49 INFO MemoryStore:59 - Block broadcast_211_piece0 stored as bytes in memory (estimated size 5.9 KB, free 729.6 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Added broadcast_211_piece0 in memory on localhost:56918 (size: 5.9 KB, free: 730.1 MB) -2016-05-24 22:22:49 INFO SparkContext:59 - Created broadcast 211 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:49 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 115 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44) -2016-05-24 22:22:49 INFO TaskSchedulerImpl:59 - Adding task set 115.0 with 1 tasks -2016-05-24 22:22:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 115.0 (TID 222, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:22:49 INFO Executor:59 - Running task 0.0 in stage 115.0 (TID 222) -2016-05-24 22:22:49 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_194_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:49 INFO Executor:59 - Finished task 0.0 in stage 115.0 (TID 222). 2401 bytes result sent to driver -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 99 -2016-05-24 22:22:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 115.0 (TID 222) in 27 ms on localhost (1/1) -2016-05-24 22:22:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 115.0, whose tasks have all completed, from pool -2016-05-24 22:22:49 INFO DAGScheduler:59 - ResultStage 115 (take at SVMFromSparkLearning.scala:56) finished in 0.027 s -2016-05-24 22:22:49 INFO DAGScheduler:59 - Job 108 finished: take at SVMFromSparkLearning.scala:56, took 0.075364 s -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_193_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_192_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 98 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_191_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_190_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 97 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_189_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_188_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 96 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_187_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_186_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 95 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_185_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_184_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 94 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_183_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_182_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 93 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_181_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_180_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 92 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_179_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_178_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 91 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_177_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_176_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 90 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_175_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_174_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 89 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_173_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_172_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 88 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_171_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_170_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 87 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_169_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_168_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 86 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_167_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_166_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 85 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_165_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_164_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 84 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_163_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_162_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 83 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_161_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_160_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 82 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_159_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_158_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 81 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_157_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_156_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 80 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_155_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_154_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 79 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_153_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_152_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 78 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_151_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_150_piece0 on localhost:56918 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:22:49 INFO ContextCleaner:59 - Cleaned accumulator 77 -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Removed broadcast_149_piece0 on localhost:56918 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:22:49 INFO SparkContext:59 - Starting job: saveAsTextFile at GLMClassificationModel.scala:61 -2016-05-24 22:22:49 INFO DAGScheduler:59 - Got job 109 (saveAsTextFile at GLMClassificationModel.scala:61) with 1 output partitions -2016-05-24 22:22:49 INFO DAGScheduler:59 - Final stage: ResultStage 116(saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:22:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:49 INFO DAGScheduler:59 - Submitting ResultStage 116 (MapPartitionsRDD[227] at saveAsTextFile at GLMClassificationModel.scala:61), which has no missing parents -2016-05-24 22:22:49 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=494946, maxMem=766075207 -2016-05-24 22:22:49 INFO MemoryStore:59 - Block broadcast_212 stored as values in memory (estimated size 124.0 KB, free 730.0 MB) -2016-05-24 22:22:49 INFO MemoryStore:59 - ensureFreeSpace(42575) called with curMem=621938, maxMem=766075207 -2016-05-24 22:22:49 INFO MemoryStore:59 - Block broadcast_212_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.0 MB) -2016-05-24 22:22:49 INFO BlockManagerInfo:59 - Added broadcast_212_piece0 in memory on localhost:56918 (size: 41.6 KB, free: 730.2 MB) -2016-05-24 22:22:49 INFO SparkContext:59 - Created broadcast 212 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:49 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 116 (MapPartitionsRDD[227] at saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:22:49 INFO TaskSchedulerImpl:59 - Adding task set 116.0 with 1 tasks -2016-05-24 22:22:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 116.0 (TID 223, localhost, PROCESS_LOCAL, 2197 bytes) -2016-05-24 22:22:49 INFO Executor:59 - Running task 0.0 in stage 116.0 (TID 223) -2016-05-24 22:22:50 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242222_0116_m_000000_223' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222249102/result/metadata/_temporary/0/task_201605242222_0116_m_000000 -2016-05-24 22:22:50 INFO SparkHadoopMapRedUtil:59 - attempt_201605242222_0116_m_000000_223: Committed -2016-05-24 22:22:50 INFO Executor:59 - Finished task 0.0 in stage 116.0 (TID 223). 1864 bytes result sent to driver -2016-05-24 22:22:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 116.0 (TID 223) in 265 ms on localhost (1/1) -2016-05-24 22:22:50 INFO DAGScheduler:59 - ResultStage 116 (saveAsTextFile at GLMClassificationModel.scala:61) finished in 0.265 s -2016-05-24 22:22:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 116.0, whose tasks have all completed, from pool -2016-05-24 22:22:50 INFO DAGScheduler:59 - Job 109 finished: saveAsTextFile at GLMClassificationModel.scala:61, took 0.362472 s -2016-05-24 22:22:53 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:22:53 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:22:53 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:65 -2016-05-24 22:22:53 INFO DAGScheduler:59 - Got job 110 (parquet at GLMClassificationModel.scala:65) with 1 output partitions -2016-05-24 22:22:53 INFO DAGScheduler:59 - Final stage: ResultStage 117(parquet at GLMClassificationModel.scala:65) -2016-05-24 22:22:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:53 INFO DAGScheduler:59 - Submitting ResultStage 117 (MapPartitionsRDD[229] at rddToDataFrameHolder at GLMClassificationModel.scala:65), which has no missing parents -2016-05-24 22:22:53 INFO MemoryStore:59 - ensureFreeSpace(67104) called with curMem=664513, maxMem=766075207 -2016-05-24 22:22:53 INFO MemoryStore:59 - Block broadcast_213 stored as values in memory (estimated size 65.5 KB, free 729.9 MB) -2016-05-24 22:22:53 INFO MemoryStore:59 - ensureFreeSpace(23437) called with curMem=731617, maxMem=766075207 -2016-05-24 22:22:53 INFO MemoryStore:59 - Block broadcast_213_piece0 stored as bytes in memory (estimated size 22.9 KB, free 729.9 MB) -2016-05-24 22:22:53 INFO BlockManagerInfo:59 - Added broadcast_213_piece0 in memory on localhost:56918 (size: 22.9 KB, free: 730.2 MB) -2016-05-24 22:22:53 INFO SparkContext:59 - Created broadcast 213 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:53 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 117 (MapPartitionsRDD[229] at rddToDataFrameHolder at GLMClassificationModel.scala:65) -2016-05-24 22:22:53 INFO TaskSchedulerImpl:59 - Adding task set 117.0 with 1 tasks -2016-05-24 22:22:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 117.0 (TID 224, localhost, PROCESS_LOCAL, 7978 bytes) -2016-05-24 22:22:53 INFO Executor:59 - Running task 0.0 in stage 117.0 (TID 224) -2016-05-24 22:22:53 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:22:53 INFO CodecConfig:151 - Compression: GZIP -2016-05-24 22:22:53 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-24 22:22:53 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-24 22:22:53 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-24 22:22:53 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-24 22:22:53 INFO ParquetOutputFormat:151 - Validation is off -2016-05-24 22:22:53 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-24 22:22:53 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-24 22:22:53 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 6,156 -2016-05-24 22:22:53 INFO ColumnChunkPageWriteStore:151 - written 61B for [weights, type] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-24 22:22:53 INFO ColumnChunkPageWriteStore:151 - written 48B for [weights, size] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-24 22:22:53 INFO ColumnChunkPageWriteStore:151 - written 51B for [weights, indices, array] INT32: 1 values, 13B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-24 22:22:53 INFO ColumnChunkPageWriteStore:151 - written 637B for [weights, values, array] DOUBLE: 692 values, 668B raw, 593B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 413 entries, 3,304B raw, 413B comp} -2016-05-24 22:22:53 INFO ColumnChunkPageWriteStore:151 - written 69B for [intercept] DOUBLE: 1 values, 14B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-24 22:22:53 INFO ColumnChunkPageWriteStore:151 - written 47B for [threshold] DOUBLE: 1 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-24 22:22:54 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242222_0117_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222249102/result/data/_temporary/0/task_201605242222_0117_m_000000 -2016-05-24 22:22:54 INFO SparkHadoopMapRedUtil:59 - attempt_201605242222_0117_m_000000_0: Committed -2016-05-24 22:22:54 INFO Executor:59 - Finished task 0.0 in stage 117.0 (TID 224). 915 bytes result sent to driver -2016-05-24 22:22:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 117.0 (TID 224) in 1074 ms on localhost (1/1) -2016-05-24 22:22:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 117.0, whose tasks have all completed, from pool -2016-05-24 22:22:54 INFO DAGScheduler:59 - ResultStage 117 (parquet at GLMClassificationModel.scala:65) finished in 1.078 s -2016-05-24 22:22:54 INFO DAGScheduler:59 - Job 110 finished: parquet at GLMClassificationModel.scala:65, took 1.127042 s -2016-05-24 22:22:54 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:22:54 INFO DefaultWriterContainer:59 - Job job_201605242222_0000 committed. -2016-05-24 22:22:54 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222249102/result/data on driver -2016-05-24 22:22:54 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222249102/result/data on driver -2016-05-24 22:22:54 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=755054, maxMem=766075207 -2016-05-24 22:22:55 INFO MemoryStore:59 - Block broadcast_214 stored as values in memory (estimated size 225.3 KB, free 729.6 MB) -2016-05-24 22:22:55 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=985758, maxMem=766075207 -2016-05-24 22:22:55 INFO MemoryStore:59 - Block broadcast_214_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.6 MB) -2016-05-24 22:22:55 INFO BlockManagerInfo:59 - Added broadcast_214_piece0 in memory on localhost:56918 (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:22:55 INFO SparkContext:59 - Created broadcast 214 from textFile at modelSaveLoad.scala:129 -2016-05-24 22:22:55 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:22:55 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-24 22:22:55 INFO DAGScheduler:59 - Got job 111 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-24 22:22:55 INFO DAGScheduler:59 - Final stage: ResultStage 118(first at modelSaveLoad.scala:129) -2016-05-24 22:22:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:55 INFO DAGScheduler:59 - Submitting ResultStage 118 (MapPartitionsRDD[232] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-24 22:22:55 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=1005546, maxMem=766075207 -2016-05-24 22:22:55 INFO MemoryStore:59 - Block broadcast_215 stored as values in memory (estimated size 3.1 KB, free 729.6 MB) -2016-05-24 22:22:55 INFO MemoryStore:59 - ensureFreeSpace(1898) called with curMem=1008762, maxMem=766075207 -2016-05-24 22:22:55 INFO MemoryStore:59 - Block broadcast_215_piece0 stored as bytes in memory (estimated size 1898.0 B, free 729.6 MB) -2016-05-24 22:22:55 INFO BlockManagerInfo:59 - Added broadcast_215_piece0 in memory on localhost:56918 (size: 1898.0 B, free: 730.2 MB) -2016-05-24 22:22:55 INFO SparkContext:59 - Created broadcast 215 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:55 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 118 (MapPartitionsRDD[232] at textFile at modelSaveLoad.scala:129) -2016-05-24 22:22:55 INFO TaskSchedulerImpl:59 - Adding task set 118.0 with 1 tasks -2016-05-24 22:22:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 118.0 (TID 225, localhost, PROCESS_LOCAL, 2238 bytes) -2016-05-24 22:22:55 INFO Executor:59 - Running task 0.0 in stage 118.0 (TID 225) -2016-05-24 22:22:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222249102/result/metadata/part-00000:0+54 -2016-05-24 22:22:55 INFO Executor:59 - Finished task 0.0 in stage 118.0 (TID 225). 2154 bytes result sent to driver -2016-05-24 22:22:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 118.0 (TID 225) in 14 ms on localhost (1/1) -2016-05-24 22:22:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 118.0, whose tasks have all completed, from pool -2016-05-24 22:22:55 INFO DAGScheduler:59 - ResultStage 118 (first at modelSaveLoad.scala:129) finished in 0.016 s -2016-05-24 22:22:55 INFO DAGScheduler:59 - Job 111 finished: first at modelSaveLoad.scala:129, took 0.025024 s -2016-05-24 22:22:55 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222249102/result/data on driver -2016-05-24 22:22:55 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:78 -2016-05-24 22:22:55 INFO DAGScheduler:59 - Got job 112 (parquet at GLMClassificationModel.scala:78) with 4 output partitions -2016-05-24 22:22:55 INFO DAGScheduler:59 - Final stage: ResultStage 119(parquet at GLMClassificationModel.scala:78) -2016-05-24 22:22:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:55 INFO DAGScheduler:59 - Submitting ResultStage 119 (MapPartitionsRDD[234] at parquet at GLMClassificationModel.scala:78), which has no missing parents -2016-05-24 22:22:55 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=1010660, maxMem=766075207 -2016-05-24 22:22:55 INFO MemoryStore:59 - Block broadcast_216 stored as values in memory (estimated size 61.5 KB, free 729.6 MB) -2016-05-24 22:22:55 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=1073660, maxMem=766075207 -2016-05-24 22:22:55 INFO MemoryStore:59 - Block broadcast_216_piece0 stored as bytes in memory (estimated size 20.6 KB, free 729.5 MB) -2016-05-24 22:22:55 INFO BlockManagerInfo:59 - Added broadcast_216_piece0 in memory on localhost:56918 (size: 20.6 KB, free: 730.2 MB) -2016-05-24 22:22:55 INFO SparkContext:59 - Created broadcast 216 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 119 (MapPartitionsRDD[234] at parquet at GLMClassificationModel.scala:78) -2016-05-24 22:22:55 INFO TaskSchedulerImpl:59 - Adding task set 119.0 with 4 tasks -2016-05-24 22:22:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 119.0 (TID 226, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:22:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 119.0 (TID 227, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:22:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 119.0 (TID 228, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:22:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 119.0 (TID 229, localhost, PROCESS_LOCAL, 2279 bytes) -2016-05-24 22:22:55 INFO Executor:59 - Running task 0.0 in stage 119.0 (TID 226) -2016-05-24 22:22:55 INFO Executor:59 - Running task 1.0 in stage 119.0 (TID 227) -2016-05-24 22:22:55 INFO Executor:59 - Running task 3.0 in stage 119.0 (TID 229) -2016-05-24 22:22:55 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:22:55 INFO Executor:59 - Finished task 1.0 in stage 119.0 (TID 227). 936 bytes result sent to driver -2016-05-24 22:22:55 INFO Executor:59 - Running task 2.0 in stage 119.0 (TID 228) -2016-05-24 22:22:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 119.0 (TID 227) in 118 ms on localhost (1/4) -2016-05-24 22:22:55 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:22:55 INFO Executor:59 - Finished task 3.0 in stage 119.0 (TID 229). 1802 bytes result sent to driver -2016-05-24 22:22:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 119.0 (TID 229) in 157 ms on localhost (2/4) -2016-05-24 22:22:55 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:22:55 INFO Executor:59 - Finished task 0.0 in stage 119.0 (TID 226). 936 bytes result sent to driver -2016-05-24 22:22:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 119.0 (TID 226) in 169 ms on localhost (3/4) -2016-05-24 22:22:55 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:22:55 INFO Executor:59 - Finished task 2.0 in stage 119.0 (TID 228). 936 bytes result sent to driver -2016-05-24 22:22:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 119.0 (TID 228) in 192 ms on localhost (4/4) -2016-05-24 22:22:55 INFO DAGScheduler:59 - ResultStage 119 (parquet at GLMClassificationModel.scala:78) finished in 0.193 s -2016-05-24 22:22:55 INFO DAGScheduler:59 - Job 112 finished: parquet at GLMClassificationModel.scala:78, took 0.230427 s -2016-05-24 22:22:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 119.0, whose tasks have all completed, from pool -2016-05-24 22:22:56 INFO ContextCleaner:59 - Cleaned accumulator 109 -2016-05-24 22:22:56 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 22:22:56 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=1010660, maxMem=766075207 -2016-05-24 22:22:56 INFO MemoryStore:59 - Block broadcast_217 stored as values in memory (estimated size 86.4 KB, free 729.5 MB) -2016-05-24 22:22:56 INFO BlockManagerInfo:59 - Removed broadcast_216_piece0 on localhost:56918 in memory (size: 20.6 KB, free: 730.2 MB) -2016-05-24 22:22:56 INFO ContextCleaner:59 - Cleaned accumulator 115 -2016-05-24 22:22:56 INFO BlockManagerInfo:59 - Removed broadcast_215_piece0 on localhost:56918 in memory (size: 1898.0 B, free: 730.2 MB) -2016-05-24 22:22:56 INFO ContextCleaner:59 - Cleaned accumulator 114 -2016-05-24 22:22:56 INFO BlockManagerInfo:59 - Removed broadcast_214_piece0 on localhost:56918 in memory (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:22:56 INFO BlockManagerInfo:59 - Removed broadcast_213_piece0 on localhost:56918 in memory (size: 22.9 KB, free: 730.2 MB) -2016-05-24 22:22:56 INFO ContextCleaner:59 - Cleaned accumulator 113 -2016-05-24 22:22:56 INFO BlockManagerInfo:59 - Removed broadcast_212_piece0 on localhost:56918 in memory (size: 41.6 KB, free: 730.3 MB) -2016-05-24 22:22:56 INFO ContextCleaner:59 - Cleaned accumulator 112 -2016-05-24 22:22:56 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=577369, maxMem=766075207 -2016-05-24 22:22:56 INFO MemoryStore:59 - Block broadcast_217_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.0 MB) -2016-05-24 22:22:56 INFO BlockManagerInfo:59 - Removed broadcast_211_piece0 on localhost:56918 in memory (size: 5.9 KB, free: 730.3 MB) -2016-05-24 22:22:56 INFO BlockManagerInfo:59 - Added broadcast_217_piece0 in memory on localhost:56918 (size: 19.3 KB, free: 730.3 MB) -2016-05-24 22:22:56 INFO ContextCleaner:59 - Cleaned accumulator 111 -2016-05-24 22:22:56 INFO SparkContext:59 - Created broadcast 217 from take at GLMClassificationModel.scala:79 -2016-05-24 22:22:56 INFO BlockManagerInfo:59 - Removed broadcast_210_piece0 on localhost:56918 in memory (size: 2.9 KB, free: 730.3 MB) -2016-05-24 22:22:56 INFO ContextCleaner:59 - Cleaned accumulator 110 -2016-05-24 22:22:56 INFO BlockManagerInfo:59 - Removed broadcast_209_piece0 on localhost:56918 in memory (size: 2.7 KB, free: 730.3 MB) -2016-05-24 22:22:57 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-24 22:22:57 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222249102/result/data/part-r-00000-a42428b1-6e4a-4386-b947-38fb9a34adb8.gz.parquet -2016-05-24 22:22:57 INFO SparkContext:59 - Starting job: take at GLMClassificationModel.scala:79 -2016-05-24 22:22:57 INFO DAGScheduler:59 - Got job 113 (take at GLMClassificationModel.scala:79) with 1 output partitions -2016-05-24 22:22:57 INFO DAGScheduler:59 - Final stage: ResultStage 120(take at GLMClassificationModel.scala:79) -2016-05-24 22:22:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:22:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:22:57 INFO DAGScheduler:59 - Submitting ResultStage 120 (MapPartitionsRDD[236] at take at GLMClassificationModel.scala:79), which has no missing parents -2016-05-24 22:22:57 INFO MemoryStore:59 - ensureFreeSpace(4168) called with curMem=570248, maxMem=766075207 -2016-05-24 22:22:57 INFO MemoryStore:59 - Block broadcast_218 stored as values in memory (estimated size 4.1 KB, free 730.0 MB) -2016-05-24 22:22:57 INFO MemoryStore:59 - ensureFreeSpace(2438) called with curMem=574416, maxMem=766075207 -2016-05-24 22:22:57 INFO MemoryStore:59 - Block broadcast_218_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.0 MB) -2016-05-24 22:22:57 INFO BlockManagerInfo:59 - Added broadcast_218_piece0 in memory on localhost:56918 (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:22:57 INFO SparkContext:59 - Created broadcast 218 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:22:57 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 120 (MapPartitionsRDD[236] at take at GLMClassificationModel.scala:79) -2016-05-24 22:22:57 INFO TaskSchedulerImpl:59 - Adding task set 120.0 with 1 tasks -2016-05-24 22:22:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 120.0 (TID 230, localhost, PROCESS_LOCAL, 2299 bytes) -2016-05-24 22:22:57 INFO Executor:59 - Running task 0.0 in stage 120.0 (TID 230) -2016-05-24 22:22:57 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222249102/result/data/part-r-00000-a42428b1-6e4a-4386-b947-38fb9a34adb8.gz.parquet start: 0 end: 5410 length: 5410 hosts: []} -2016-05-24 22:22:57 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-24 22:22:57 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group weights { - required int32 type (INT_8); - optional int32 size; - optional group indices (LIST) { - repeated int32 array; - } - optional group values (LIST) { - repeated double array; - } - } - optional double intercept; - optional double threshold; -} - -Catalyst form: -StructType(StructField(weights,org.apache.spark.mllib.linalg.VectorUDT@f71b0bce,true), StructField(intercept,DoubleType,true), StructField(threshold,DoubleType,true)) - -2016-05-24 22:22:57 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-24 22:22:57 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-24 22:22:58 ERROR Executor:96 - Exception in task 0.0 in stage 120.0 (TID 230) -java.lang.OutOfMemoryError: PermGen space - at java.lang.ClassLoader.defineClass1(Native Method) - at java.lang.ClassLoader.defineClass(Unknown Source) - at java.security.SecureClassLoader.defineClass(Unknown Source) - at java.net.URLClassLoader.defineClass(Unknown Source) - at java.net.URLClassLoader.access$100(Unknown Source) - at java.net.URLClassLoader$1.run(Unknown Source) - at java.net.URLClassLoader$1.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at java.net.URLClassLoader.findClass(Unknown Source) - at java.lang.ClassLoader.loadClass(Unknown Source) - at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source) - at java.lang.ClassLoader.loadClass(Unknown Source) - at org.apache.hadoop.io.compress.GzipCodec.getDecompressorType(GzipCodec.java:166) - at org.apache.hadoop.io.compress.CodecPool.getDecompressor(CodecPool.java:176) - at org.apache.parquet.hadoop.CodecFactory$BytesDecompressor.(CodecFactory.java:50) - at org.apache.parquet.hadoop.CodecFactory.getDecompressor(CodecFactory.java:179) - at org.apache.parquet.hadoop.ParquetFileReader$Chunk.readAllPages(ParquetFileReader.java:610) - at org.apache.parquet.hadoop.ParquetFileReader.readNextRowGroup(ParquetFileReader.java:496) - at org.apache.parquet.hadoop.InternalParquetRecordReader.checkRead(InternalParquetRecordReader.java:127) - at org.apache.parquet.hadoop.InternalParquetRecordReader.nextKeyValue(InternalParquetRecordReader.java:208) - at org.apache.parquet.hadoop.ParquetRecordReader.nextKeyValue(ParquetRecordReader.java:201) - at org.apache.spark.rdd.SqlNewHadoopRDD$$anon$1.hasNext(SqlNewHadoopRDD.scala:168) - at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) - at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:308) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48) - at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:103) - at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:47) - at scala.collection.TraversableOnce$class.to(TraversableOnce.scala:273) - at scala.collection.AbstractIterator.to(Iterator.scala:1157) - at scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:265) -2016-05-24 22:23:12 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 22:23:12 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 22:23:13 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 22:23:13 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 22:23:13 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 22:23:14 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 22:23:14 INFO Remoting:74 - Starting remoting -2016-05-24 22:23:14 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:56939] -2016-05-24 22:23:14 INFO Utils:59 - Successfully started service 'sparkDriver' on port 56939. -2016-05-24 22:23:14 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 22:23:14 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 22:23:14 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-201f6c4c-d5c1-4aca-839f-962e8b4b3060 -2016-05-24 22:23:14 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 22:23:14 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-fd5a021b-3106-46fe-b969-439d92550474\httpd-ea267ad8-b88f-4851-8c2d-e66a59415fb0 -2016-05-24 22:23:14 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 22:23:14 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:23:14 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:56940 -2016-05-24 22:23:14 INFO Utils:59 - Successfully started service 'HTTP file server' on port 56940. -2016-05-24 22:23:14 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 22:23:14 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:23:14 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 22:23:14 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 22:23:14 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 22:23:14 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 22:23:14 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 22:23:15 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 56959. -2016-05-24 22:23:15 INFO NettyBlockTransferService:59 - Server created on 56959 -2016-05-24 22:23:15 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 22:23:15 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:56959 with 730.6 MB RAM, BlockManagerId(driver, localhost, 56959) -2016-05-24 22:23:15 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 22:23:15 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 22:23:15 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 22:23:15 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 22:23:15 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 22:23:15 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:56959 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 22:23:15 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 22:23:17 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 22:23:17 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:23:17 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 22:23:17 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 22:23:17 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 22:23:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:17 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 22:23:17 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 22:23:17 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 22:23:17 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=175476, maxMem=766075207 -2016-05-24 22:23:17 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:23:17 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:56959 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:23:17 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 22:23:17 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 22:23:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:23:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:23:17 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 22:23:17 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 22:23:17 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 22:23:17 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 22:23:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 22:23:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 22:23:17 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 22:23:17 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 22:23:17 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 22:23:17 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 22:23:17 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 22:23:17 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177621, maxMem=766075207 -2016-05-24 22:23:17 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 22:23:17 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:56959 (size: 83.1 KB, free: 730.5 MB) -2016-05-24 22:23:17 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 22:23:17 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262701, maxMem=766075207 -2016-05-24 22:23:17 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 22:23:17 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:56959 (size: 83.6 KB, free: 730.4 MB) -2016-05-24 22:23:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 385 ms on localhost (1/2) -2016-05-24 22:23:17 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 22:23:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 422 ms on localhost (2/2) -2016-05-24 22:23:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 22:23:17 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.443 s -2016-05-24 22:23:17 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.626966 s -2016-05-24 22:23:18 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 22:23:18 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 22:23:18 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:23:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:18 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 22:23:18 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=348349, maxMem=766075207 -2016-05-24 22:23:18 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-24 22:23:18 INFO MemoryStore:59 - ensureFreeSpace(2483) called with curMem=352909, maxMem=766075207 -2016-05-24 22:23:18 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-24 22:23:18 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:56959 (size: 2.4 KB, free: 730.4 MB) -2016-05-24 22:23:18 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:23:18 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 22:23:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:23:18 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 22:23:18 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-05-24 22:23:18 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:23:18 INFO MemoryStore:59 - ensureFreeSpace(54392) called with curMem=355392, maxMem=766075207 -2016-05-24 22:23:18 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 53.1 KB, free 730.2 MB) -2016-05-24 22:23:18 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:56959 (size: 53.1 KB, free: 730.4 MB) -2016-05-24 22:23:18 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2611 bytes result sent to driver -2016-05-24 22:23:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 48 ms on localhost (1/1) -2016-05-24 22:23:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 22:23:18 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.049 s -2016-05-24 22:23:18 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.084213 s -2016-05-24 22:23:18 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 22:23:18 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 22:23:18 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 22:23:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:18 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 22:23:18 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=409784, maxMem=766075207 -2016-05-24 22:23:18 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-24 22:23:18 INFO MemoryStore:59 - ensureFreeSpace(2300) called with curMem=414032, maxMem=766075207 -2016-05-24 22:23:18 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 22:23:18 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:56959 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 22:23:18 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40) -2016-05-24 22:23:18 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 22:23:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:23:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:23:18 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 22:23:18 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 22:23:18 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-05-24 22:23:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:18 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:23:18 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 22:23:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 14 ms on localhost (1/2) -2016-05-24 22:23:18 INFO MemoryStore:59 - ensureFreeSpace(62184) called with curMem=416332, maxMem=766075207 -2016-05-24 22:23:18 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 60.7 KB, free 730.1 MB) -2016-05-24 22:23:18 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:56959 (size: 60.7 KB, free: 730.3 MB) -2016-05-24 22:23:18 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2662 bytes result sent to driver -2016-05-24 22:23:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 22 ms on localhost (2/2) -2016-05-24 22:23:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 22:23:18 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.025 s -2016-05-24 22:23:18 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.065786 s -2016-05-24 22:23:18 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 22:23:18 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 22:23:18 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 22:23:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:18 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 22:23:18 INFO MemoryStore:59 - ensureFreeSpace(4264) called with curMem=478516, maxMem=766075207 -2016-05-24 22:23:18 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.2 KB, free 730.1 MB) -2016-05-24 22:23:18 INFO MemoryStore:59 - ensureFreeSpace(2322) called with curMem=482780, maxMem=766075207 -2016-05-24 22:23:18 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 22:23:18 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:56959 (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:23:18 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 22:23:18 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 22:23:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:23:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:23:18 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 22:23:18 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:18 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 22:23:18 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 22:23:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 20 ms on localhost (1/2) -2016-05-24 22:23:18 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:18 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 22:23:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 26 ms on localhost (2/2) -2016-05-24 22:23:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 22:23:18 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.035 s -2016-05-24 22:23:18 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.054165 s -2016-05-24 22:23:18 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 22:23:18 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 22:23:18 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=485102, maxMem=766075207 -2016-05-24 22:23:18 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:23:18 INFO MemoryStore:59 - ensureFreeSpace(388) called with curMem=484108, maxMem=766075207 -2016-05-24 22:23:18 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:56959 in memory (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:23:18 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 388.0 B, free 730.1 MB) -2016-05-24 22:23:18 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 22:23:18 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:56959 (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:23:18 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:56959 in memory (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:23:18 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:18 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 22:23:18 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:56959 in memory (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:23:18 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 22:23:18 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:56959 in memory (size: 2.1 KB, free: 730.3 MB) -2016-05-24 22:23:18 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 22:23:18 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:18 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:18 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:18 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:18 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=464864, maxMem=766075207 -2016-05-24 22:23:18 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:23:18 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=476648, maxMem=766075207 -2016-05-24 22:23:18 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:23:18 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:18 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:18 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 22:23:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:18 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 22:23:18 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 22:23:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:19 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 7968 bytes result sent to driver -2016-05-24 22:23:19 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 7968 bytes result sent to driver -2016-05-24 22:23:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 25 ms on localhost (1/2) -2016-05-24 22:23:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 23 ms on localhost (2/2) -2016-05-24 22:23:19 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.027 s -2016-05-24 22:23:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 22:23:19 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.044609 s -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=480130, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=485722, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:56959 (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:19 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:19 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:19 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=487960, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(3479) called with curMem=499744, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:19 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 22:23:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:19 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 22:23:19 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 22:23:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:19 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 7968 bytes result sent to driver -2016-05-24 22:23:19 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 7968 bytes result sent to driver -2016-05-24 22:23:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 18 ms on localhost (1/2) -2016-05-24 22:23:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 19 ms on localhost (2/2) -2016-05-24 22:23:19 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:23:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 22:23:19 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.039599 s -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=503223, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=508815, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:19 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:19 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:19 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=512424, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=524208, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:19 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 22:23:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:19 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 22:23:19 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 22:23:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:19 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 7968 bytes result sent to driver -2016-05-24 22:23:19 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 7968 bytes result sent to driver -2016-05-24 22:23:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 18 ms on localhost (1/2) -2016-05-24 22:23:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 22 ms on localhost (2/2) -2016-05-24 22:23:19 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 22:23:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 22:23:19 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.040213 s -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=527691, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(3603) called with curMem=533283, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:19 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:19 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:19 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=536886, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=548670, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:19 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 22:23:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:19 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 22:23:19 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 22:23:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:19 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 7968 bytes result sent to driver -2016-05-24 22:23:19 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 7968 bytes result sent to driver -2016-05-24 22:23:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 16 ms on localhost (1/2) -2016-05-24 22:23:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 15 ms on localhost (2/2) -2016-05-24 22:23:19 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 22:23:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 22:23:19 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.033464 s -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=552153, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=557745, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:19 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:19 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:19 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=561351, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=573135, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:19 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 22:23:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:19 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 22:23:19 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 22:23:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:19 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 7968 bytes result sent to driver -2016-05-24 22:23:19 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 7968 bytes result sent to driver -2016-05-24 22:23:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 16 ms on localhost (1/2) -2016-05-24 22:23:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 17 ms on localhost (2/2) -2016-05-24 22:23:19 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:23:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 22:23:19 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.031801 s -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=576616, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=582208, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:19 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:19 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:19 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=585814, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=597598, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:19 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 22:23:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:19 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 22:23:19 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 22:23:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:19 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 7968 bytes result sent to driver -2016-05-24 22:23:19 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 7968 bytes result sent to driver -2016-05-24 22:23:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 12 ms on localhost (1/2) -2016-05-24 22:23:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 12 ms on localhost (2/2) -2016-05-24 22:23:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 22:23:19 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:23:19 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.025964 s -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=601081, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=606673, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:19 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:19 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:19 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=610279, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=622063, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:19 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 22:23:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:19 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 22:23:19 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 22:23:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:19 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 7968 bytes result sent to driver -2016-05-24 22:23:19 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 7968 bytes result sent to driver -2016-05-24 22:23:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 13 ms on localhost (1/2) -2016-05-24 22:23:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 12 ms on localhost (2/2) -2016-05-24 22:23:19 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:23:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 22:23:19 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.030976 s -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=625546, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(3604) called with curMem=631138, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:19 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:19 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:19 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=634742, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=646526, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:19 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:19 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 22:23:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:19 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 22:23:19 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 22:23:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:19 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 7968 bytes result sent to driver -2016-05-24 22:23:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:19 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 7968 bytes result sent to driver -2016-05-24 22:23:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 11 ms on localhost (1/2) -2016-05-24 22:23:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 14 ms on localhost (2/2) -2016-05-24 22:23:19 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:23:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 22:23:19 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.028878 s -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=650009, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=655601, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:19 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:19 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 22:23:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:19 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:19 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=595012, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=603190, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:19 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 22:23:19 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 22:23:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:19 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 22:23:19 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 22:23:19 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 7968 bytes result sent to driver -2016-05-24 22:23:19 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 7968 bytes result sent to driver -2016-05-24 22:23:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 15 ms on localhost (1/2) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 15 ms on localhost (2/2) -2016-05-24 22:23:19 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:23:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 22:23:19 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.031576 s -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=527689, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=533281, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:56959 in memory (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:19 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:19 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:19 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=498528, maxMem=766075207 -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:23:19 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=509924, maxMem=766075207 -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:56959 in memory (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:23:19 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:23:19 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:19 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:19 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 22:23:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:19 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 22:23:19 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 22:23:19 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:19 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 30 ms on localhost (1/2) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 29 ms on localhost (2/2) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 22:23:20 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.035 s -2016-05-24 22:23:20 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.392373 s -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=507816, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3610) called with curMem=513408, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:20 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=517018, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=528802, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 22:23:20 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 15 ms on localhost (1/2) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 17 ms on localhost (2/2) -2016-05-24 22:23:20 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 22:23:20 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.036304 s -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=532286, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=537878, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:20 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=541484, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=553268, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 22:23:20 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 15 ms on localhost (1/2) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 27 ms on localhost (2/2) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 22:23:20 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 22:23:20 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.047199 s -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=556751, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3610) called with curMem=562343, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:20 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=565953, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=577737, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 22:23:20 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 15 ms on localhost (1/2) -2016-05-24 22:23:20 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 19 ms on localhost (2/2) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 22:23:20 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 22:23:20 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.040848 s -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=581220, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=586812, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:20 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=590418, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=602202, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 22:23:20 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 13 ms on localhost (1/2) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 13 ms on localhost (2/2) -2016-05-24 22:23:20 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 22:23:20 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.025633 s -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=605685, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=611277, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:20 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=614883, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=626667, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 22:23:20 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 18 ms on localhost (1/2) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 19 ms on localhost (2/2) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 22:23:20 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:23:20 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.039346 s -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=630150, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=635742, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:20 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=639355, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=651139, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:20 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 22:23:20 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 14 ms on localhost (1/2) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 21 ms on localhost (2/2) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 22:23:20 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:23:20 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.035680 s -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=654623, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=660215, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:20 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=663821, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=675605, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 22:23:20 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 15 ms on localhost (1/2) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 17 ms on localhost (2/2) -2016-05-24 22:23:20 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 22:23:20 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.031567 s -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=679089, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=684681, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:20 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=688293, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=700077, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 22:23:20 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 14 ms on localhost (1/2) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 14 ms on localhost (2/2) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 22:23:20 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:23:20 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.030686 s -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=703560, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3621) called with curMem=709152, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:20 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=712773, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=724557, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 22:23:20 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 12 ms on localhost (1/2) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 12 ms on localhost (2/2) -2016-05-24 22:23:20 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 22:23:20 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.027010 s -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=728040, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=733632, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:20 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=737244, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=749028, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 22:23:20 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 10 ms on localhost (1/2) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 12 ms on localhost (2/2) -2016-05-24 22:23:20 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 22:23:20 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.024836 s -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=752511, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=758103, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:20 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=761715, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=773499, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 22:23:20 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 10 ms on localhost (1/2) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 12 ms on localhost (2/2) -2016-05-24 22:23:20 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 22:23:20 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.026363 s -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=776982, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=782574, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:20 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=786188, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=797972, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 22:23:20 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 10 ms on localhost (1/2) -2016-05-24 22:23:20 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 13 ms on localhost (2/2) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 22:23:20 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:23:20 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.029879 s -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=801454, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=807046, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:20 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=810660, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=822444, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 22:23:20 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 12 ms on localhost (1/2) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 13 ms on localhost (2/2) -2016-05-24 22:23:20 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 22:23:20 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.031787 s -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=825924, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3634) called with curMem=831516, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:20 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=835150, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=846934, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 22:23:20 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 9 ms on localhost (1/2) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 12 ms on localhost (2/2) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 22:23:20 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:23:20 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.027710 s -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=850415, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=856007, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:20 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=859619, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=871403, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 22:23:20 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 14 ms on localhost (1/2) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 14 ms on localhost (2/2) -2016-05-24 22:23:20 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 22:23:20 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.066097 s -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=835129, maxMem=766075207 -2016-05-24 22:23:20 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=831515, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:20 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:20 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=782702, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=782702, maxMem=766075207 -2016-05-24 22:23:20 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 22:23:20 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:20 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 22:23:20 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 8 ms on localhost (1/2) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 10 ms on localhost (2/2) -2016-05-24 22:23:20 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 22:23:20 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.021574 s -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=749019, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:20 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=742827, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:20 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:20 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=697501, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=700080, maxMem=766075207 -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 22:23:20 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 22:23:20 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 7968 bytes result sent to driver -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 7 ms on localhost (1/2) -2016-05-24 22:23:20 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 22:23:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 8 ms on localhost (2/2) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 22:23:20 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:23:20 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.021693 s -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=654634, maxMem=766075207 -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=660226, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:20 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 22:23:20 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:20 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=599630, maxMem=766075207 -2016-05-24 22:23:20 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=607804, maxMem=766075207 -2016-05-24 22:23:20 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:20 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:20 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:20 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 22:23:20 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 22:23:20 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:20 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:20 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:20 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 8 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 9 ms on localhost (2/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.020001 s -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=565962, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3623) called with curMem=567948, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=565979, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=577763, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 5 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 7 ms on localhost (2/2) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.016564 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=581246, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=586838, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=590444, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=602228, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 6 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 7 ms on localhost (2/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.018326 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=605711, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3626) called with curMem=611303, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=614929, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=626713, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 8 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 8 ms on localhost (2/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.019946 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=630196, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=635788, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=639400, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=651184, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 5 ms on localhost (1/2) -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 8 ms on localhost (2/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.019030 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=654667, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=660259, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=663871, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=675655, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 6 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 7 ms on localhost (2/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.039333 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=679138, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=684730, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=688342, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=700126, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 7 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 8 ms on localhost (2/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.021768 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=703610, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=709202, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=712814, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=724598, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 8 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 10 ms on localhost (2/2) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.020514 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=728081, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=733673, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=737285, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=749069, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 8 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 8 ms on localhost (2/2) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.018630 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=752553, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=758145, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=761757, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=773541, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 5 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 6 ms on localhost (2/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.015494 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=777024, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=782616, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=786230, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=798014, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 6 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 5 ms on localhost (2/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.016917 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=801497, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=807089, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=810701, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=822485, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 8 ms on localhost (1/2) -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 10 ms on localhost (2/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.019657 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=825969, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=831561, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=835173, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=846957, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 10 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 9 ms on localhost (2/2) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.021189 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=850439, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=856031, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[93] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=859643, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=871427, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[93] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 7 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 7 ms on localhost (2/2) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.016960 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=874911, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=880503, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[95] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=880631, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:23:21 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=880631, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[95] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 9 ms on localhost (1/2) -2016-05-24 22:23:21 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 9 ms on localhost (2/2) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.026383 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=850440, maxMem=766075207 -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=840764, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[97] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=795433, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=798013, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[97] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 22:23:21 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 8 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 8 ms on localhost (2/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.037938 s -2016-05-24 22:23:21 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=756164, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3618) called with curMem=758144, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 91 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 47 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[99] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=697552, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=700132, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 47 (MapPartitionsRDD[99] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 93, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 94, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 93) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 94) -2016-05-24 22:23:21 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 94). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 93). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 94) in 6 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 93) in 7 ms on localhost (2/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 47 finished: treeAggregate at GradientDescent.scala:225, took 0.016694 s -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=663863, maxMem=766075207 -2016-05-24 22:23:21 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=665849, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 93 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 48 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 48(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[101] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=614923, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=611439, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 48 (MapPartitionsRDD[101] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 95, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 96, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 96) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 95) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 96). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 96) in 7 ms on localhost (1/2) -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 95). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 95) in 10 ms on localhost (2/2) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 48 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 48 finished: treeAggregate at GradientDescent.scala:225, took 0.023903 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=565979, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3618) called with curMem=571571, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 95 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 49 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 49(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[103] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=565985, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=577769, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 49 (MapPartitionsRDD[103] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 97, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 98, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 98) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 97) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 98). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 97). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 98) in 5 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 97) in 7 ms on localhost (2/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 49 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 49 finished: treeAggregate at GradientDescent.scala:225, took 0.015183 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=581253, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3622) called with curMem=586845, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 97 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 50 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[105] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=590467, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=602251, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 50 (MapPartitionsRDD[105] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 9 ms on localhost (1/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 9 ms on localhost (2/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at GradientDescent.scala:225, took 0.019305 s -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=605733, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=611325, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 99 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 51 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[107] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=614938, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=626722, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 51 (MapPartitionsRDD[107] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 5 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 6 ms on localhost (2/2) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at GradientDescent.scala:225, took 0.017123 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=630204, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3628) called with curMem=635796, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 101 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 52 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 52(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 52 (MapPartitionsRDD[109] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=639424, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=651208, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 52 (MapPartitionsRDD[109] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 103, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 104, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 103) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 104) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 103). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 104). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 103) in 7 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 104) in 6 ms on localhost (2/2) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 52 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 52 finished: treeAggregate at GradientDescent.scala:225, took 0.017605 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=654690, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=660282, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 103 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 53 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 53(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[111] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=663914, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=675698, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 104 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[111] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 105, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 106, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 105) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 106) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 105). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 106). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 105) in 5 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 106) in 5 ms on localhost (2/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 53 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 53 finished: treeAggregate at GradientDescent.scala:225, took 0.014046 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=679179, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=684771, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 105 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 54 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[113] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=688384, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=700168, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 106 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 54 (MapPartitionsRDD[113] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 107) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 108) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 108). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 107). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 108) in 4 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 107) in 5 ms on localhost (2/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 54 finished: treeAggregate at GradientDescent.scala:225, took 0.012627 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=703652, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=709244, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 107 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 55 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[115] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=712876, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=724660, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 108 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 55 (MapPartitionsRDD[115] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 109, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 110, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 110) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 109) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 110). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 109). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 110) in 5 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 109) in 5 ms on localhost (2/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 55 finished: treeAggregate at GradientDescent.scala:225, took 0.016027 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=728142, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=733734, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 109 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 56 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[117] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=737347, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=749131, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 110 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 56 (MapPartitionsRDD[117] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 111, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 112, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 111) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 112) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 112). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 111). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 112) in 5 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 111) in 6 ms on localhost (2/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 56 finished: treeAggregate at GradientDescent.scala:225, took 0.031053 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=752614, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=758206, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 111 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 57 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 57(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[119] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=761819, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=773603, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 112 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 57 (MapPartitionsRDD[119] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 113, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 114, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 113) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 114) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 113). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 114). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 113) in 4 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 114) in 5 ms on localhost (2/2) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 57 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 57 finished: treeAggregate at GradientDescent.scala:225, took 0.013611 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=777083, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=782675, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 113 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 58 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 58(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 58 (MapPartitionsRDD[121] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=786287, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=798071, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 114 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 58 (MapPartitionsRDD[121] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 115, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 116, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 115) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 116) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 115). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 115) in 4 ms on localhost (1/2) -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 116). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 116) in 5 ms on localhost (2/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 58 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 58 finished: treeAggregate at GradientDescent.scala:225, took 0.024409 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=801553, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_115 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3620) called with curMem=807145, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_115_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_115_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 115 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 59 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 59(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 59 (MapPartitionsRDD[123] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=810765, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_116 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3479) called with curMem=822549, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_116_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_116_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 116 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 59 (MapPartitionsRDD[123] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 117, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 118, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 118) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 117) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 117). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 118). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 117) in 4 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 118) in 5 ms on localhost (2/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 59 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 59 finished: treeAggregate at GradientDescent.scala:225, took 0.014147 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=826028, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_117 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=831620, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_117_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_117_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 117 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 60 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 60(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 60 (MapPartitionsRDD[125] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=835232, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_118 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=847016, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_118_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_118_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 118 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 60 (MapPartitionsRDD[125] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 60.0 with 2 tasks -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 60.0 (TID 119, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 60.0 (TID 120, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:21 INFO Executor:59 - Running task 1.0 in stage 60.0 (TID 120) -2016-05-24 22:23:21 INFO Executor:59 - Running task 0.0 in stage 60.0 (TID 119) -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:21 INFO Executor:59 - Finished task 1.0 in stage 60.0 (TID 120). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO Executor:59 - Finished task 0.0 in stage 60.0 (TID 119). 7968 bytes result sent to driver -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 60.0 (TID 120) in 4 ms on localhost (1/2) -2016-05-24 22:23:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 60.0 (TID 119) in 5 ms on localhost (2/2) -2016-05-24 22:23:21 INFO DAGScheduler:59 - ResultStage 60 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 60.0, whose tasks have all completed, from pool -2016-05-24 22:23:21 INFO DAGScheduler:59 - Job 60 finished: treeAggregate at GradientDescent.scala:225, took 0.013851 s -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=850496, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_119 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3616) called with curMem=856088, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_119_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_119_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 119 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:21 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Got job 61 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:21 INFO DAGScheduler:59 - Final stage: ResultStage 61(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting ResultStage 61 (MapPartitionsRDD[127] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=859704, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_120 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=871488, maxMem=766075207 -2016-05-24 22:23:21 INFO MemoryStore:59 - Block broadcast_120_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:21 INFO BlockManagerInfo:59 - Added broadcast_120_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:21 INFO SparkContext:59 - Created broadcast 120 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 61 (MapPartitionsRDD[127] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:21 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 121, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 122, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 122) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 121) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 121). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 122). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 121) in 4 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 122) in 5 ms on localhost (2/2) -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 61 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 61 finished: treeAggregate at GradientDescent.scala:225, took 0.013627 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=874971, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_121 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=880563, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_121_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_121_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 121 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 62 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 62(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[129] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=884175, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_122 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=895959, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_122_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_122_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 122 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 62 (MapPartitionsRDD[129] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 123, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 62.0 (TID 124, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 123) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 62.0 (TID 124) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 62.0 (TID 124). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 123). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 62.0 (TID 124) in 4 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 123) in 5 ms on localhost (2/2) -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 62 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 62 finished: treeAggregate at GradientDescent.scala:225, took 0.012996 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=899441, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_123 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3630) called with curMem=905033, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_123_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_123_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 123 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 63 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 63(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 63 (MapPartitionsRDD[131] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=908663, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_124 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=920447, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_124_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_124_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 124 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 63 (MapPartitionsRDD[131] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 63.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 63.0 (TID 125, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 63.0 (TID 126, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 63.0 (TID 126) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 63.0 (TID 125) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 63.0 (TID 126). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 63.0 (TID 125). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 63.0 (TID 126) in 4 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 63.0 (TID 125) in 5 ms on localhost (2/2) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 63.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 63 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 63 finished: treeAggregate at GradientDescent.scala:225, took 0.013957 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=923929, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_125 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=929521, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_125_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_125_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 125 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 64 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 64(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 64 (MapPartitionsRDD[133] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=933134, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_126 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=944918, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_126_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_126_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 126 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 64 (MapPartitionsRDD[133] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 64.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 64.0 (TID 127, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 64.0 (TID 128, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 64.0 (TID 128) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 64.0 (TID 127) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 64.0 (TID 128). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 64.0 (TID 127). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 64.0 (TID 128) in 5 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 64.0 (TID 127) in 5 ms on localhost (2/2) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 64.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 64 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 64 finished: treeAggregate at GradientDescent.scala:225, took 0.014111 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=948399, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_127 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=953991, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_127_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_127_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 127 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 65 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 65(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[135] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=957603, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_128 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=969387, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_128_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_128_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 128 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 65 (MapPartitionsRDD[135] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 129, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 130, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 130) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 129) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 130). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 129). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 129) in 7 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 130) in 6 ms on localhost (2/2) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 65 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 65 finished: treeAggregate at GradientDescent.scala:225, took 0.017424 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=972870, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_129 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=978462, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_129_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_129_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 129 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 66 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 66(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 66 (MapPartitionsRDD[137] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=982075, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_130 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=993859, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_130_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_130_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 130 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 66 (MapPartitionsRDD[137] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 66.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 66.0 (TID 131, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 66.0 (TID 132, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 66.0 (TID 131) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 66.0 (TID 132) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 66.0 (TID 132). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 66.0 (TID 131). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 66.0 (TID 132) in 7 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 66.0 (TID 131) in 8 ms on localhost (2/2) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 66.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 66 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 66 finished: treeAggregate at GradientDescent.scala:225, took 0.018827 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=997343, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_131 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3633) called with curMem=1002935, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_131_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_131_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 131 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 67 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 67(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 67 (MapPartitionsRDD[139] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1006568, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_132 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1018352, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_132_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_132_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 132 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 67 (MapPartitionsRDD[139] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 67.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 67.0 (TID 133, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 67.0 (TID 134, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 67.0 (TID 133) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 67.0 (TID 134) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 67.0 (TID 133). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 67.0 (TID 134). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 67.0 (TID 133) in 5 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 67.0 (TID 134) in 4 ms on localhost (2/2) -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 67 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 67.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 67 finished: treeAggregate at GradientDescent.scala:225, took 0.014313 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1021835, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_133 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1027427, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_133_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_133_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 133 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 68 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 68(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 68 (MapPartitionsRDD[141] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1031040, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_134 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1042824, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_134_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_134_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 134 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 68 (MapPartitionsRDD[141] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 68.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 68.0 (TID 135, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 68.0 (TID 136, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 68.0 (TID 135) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 68.0 (TID 136) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 68.0 (TID 135). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 68.0 (TID 136). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 68.0 (TID 135) in 4 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 68.0 (TID 136) in 4 ms on localhost (2/2) -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 68 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 68.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 68 finished: treeAggregate at GradientDescent.scala:225, took 0.013799 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1046308, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_135 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3608) called with curMem=1051900, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_135_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_135_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 135 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 69 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 69(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 69 (MapPartitionsRDD[143] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1055508, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_136 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1067292, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_136_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_136_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 136 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 69 (MapPartitionsRDD[143] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 69.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 69.0 (TID 137, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 69.0 (TID 138, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 69.0 (TID 138) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 69.0 (TID 137) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 69.0 (TID 137). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 69.0 (TID 138). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 69.0 (TID 137) in 5 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 69.0 (TID 138) in 5 ms on localhost (2/2) -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 69 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 69.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 69 finished: treeAggregate at GradientDescent.scala:225, took 0.014697 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1070775, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_137 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1076367, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_137_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_137_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 137 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 70 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 70(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 70 (MapPartitionsRDD[145] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1079973, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_138 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1091757, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_138_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_138_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 138 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 70 (MapPartitionsRDD[145] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 70.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 70.0 (TID 139, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 70.0 (TID 140, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 70.0 (TID 139) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 70.0 (TID 140) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 70.0 (TID 140). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 70.0 (TID 139). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 70.0 (TID 140) in 5 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 70.0 (TID 139) in 5 ms on localhost (2/2) -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 70 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 70.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 70 finished: treeAggregate at GradientDescent.scala:225, took 0.014005 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1095240, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_139 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3619) called with curMem=1100832, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_139_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_139_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 139 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 71 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 71(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 71 (MapPartitionsRDD[147] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1104451, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_140 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1116235, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_140_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_140_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 140 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 71 (MapPartitionsRDD[147] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 71.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 71.0 (TID 141, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 71.0 (TID 142, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 71.0 (TID 141) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 71.0 (TID 142) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 71.0 (TID 142). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 71.0 (TID 141). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 71.0 (TID 142) in 4 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 71.0 (TID 141) in 5 ms on localhost (2/2) -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 71 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 71.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 71 finished: treeAggregate at GradientDescent.scala:225, took 0.013733 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1119718, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_141 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3604) called with curMem=1125310, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_141_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_141_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 141 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 72 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 72(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 72 (MapPartitionsRDD[149] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1128914, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_142 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1140698, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_142_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_142_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 142 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 72 (MapPartitionsRDD[149] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 72.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 72.0 (TID 143, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 72.0 (TID 144, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 72.0 (TID 143) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 72.0 (TID 144) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 72.0 (TID 143). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 72.0 (TID 144). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 72.0 (TID 143) in 6 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 72.0 (TID 144) in 7 ms on localhost (2/2) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 72.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 72 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 72 finished: treeAggregate at GradientDescent.scala:225, took 0.020874 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1140698, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_143 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_140_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 72 -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=1134506, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_143_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_143_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 143 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_139_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_138_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 71 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 73 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 73(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_137_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 73 (MapPartitionsRDD[151] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1089172, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_144 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_136_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1100956, maxMem=766075207 -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 70 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_144_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_144_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 144 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 73 (MapPartitionsRDD[151] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 73.0 with 2 tasks -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_135_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 73.0 (TID 145, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 73.0 (TID 146, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 73.0 (TID 146) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 73.0 (TID 145) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_134_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 69 -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 73.0 (TID 146). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 73.0 (TID 145). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 73.0 (TID 146) in 5 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 73.0 (TID 145) in 6 ms on localhost (2/2) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_133_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 73 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 73.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 73 finished: treeAggregate at GradientDescent.scala:225, took 0.019625 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1070767, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_145 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1061092, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_145_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_132_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_145_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 145 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_131_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_130_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_129_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 74 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 74(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 74 (MapPartitionsRDD[153] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_128_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1015733, maxMem=766075207 -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_146 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1018313, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_146_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_127_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_146_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 146 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 74 (MapPartitionsRDD[153] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 74.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 74.0 (TID 147, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_126_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 74.0 (TID 148, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 74.0 (TID 147) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 74.0 (TID 148) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 74.0 (TID 148). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 74.0 (TID 147). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_125_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 74.0 (TID 148) in 5 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 74.0 (TID 147) in 6 ms on localhost (2/2) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 74.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 74 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 74 finished: treeAggregate at GradientDescent.scala:225, took 0.015001 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=982060, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_147 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_124_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3624) called with curMem=987652, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_147_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_147_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 147 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_123_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_122_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_121_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 75 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 75(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_120_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 75 (MapPartitionsRDD[155] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=938701, maxMem=766075207 -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_119_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_148 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=944893, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_148_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_148_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 148 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_118_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 75 (MapPartitionsRDD[155] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 75.0 with 2 tasks -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 75.0 (TID 149, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 75.0 (TID 150, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 75.0 (TID 149) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 75.0 (TID 150) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_117_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 75.0 (TID 149). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 75.0 (TID 150). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 75.0 (TID 149) in 5 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 75.0 (TID 150) in 5 ms on localhost (2/2) -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 75 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 75.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 75 finished: treeAggregate at GradientDescent.scala:225, took 0.015743 s -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_116_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=908645, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_149 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3626) called with curMem=905025, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_149_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_115_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_149_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 149 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_114_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_113_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 76 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 76(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_112_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 76 (MapPartitionsRDD[157] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=859712, maxMem=766075207 -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_111_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_150 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=868013, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_150_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_110_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_150_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 150 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 76 (MapPartitionsRDD[157] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 151, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 152, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 151) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 152) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_109_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_108_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 152). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 151). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 152) in 5 ms on localhost (1/2) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 151) in 6 ms on localhost (2/2) -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 76 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 76 finished: treeAggregate at GradientDescent.scala:225, took 0.014593 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=826018, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_151 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_107_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3611) called with curMem=816342, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_151_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_106_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_151_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 151 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_105_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_104_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_103_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 77 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 77(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[159] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_102_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=770993, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_152 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=777185, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_152_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_101_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_152_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 152 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 77 (MapPartitionsRDD[159] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 153, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 154, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_100_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 154) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 153) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_99_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 154). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 153). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 154) in 6 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 153) in 7 ms on localhost (2/2) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 77 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 77 finished: treeAggregate at GradientDescent.scala:225, took 0.017920 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=749087, maxMem=766075207 -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_98_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_153 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=742895, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_153_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_153_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 153 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_97_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_96_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_95_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 78 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 78(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 78 (MapPartitionsRDD[161] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_94_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=697549, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_154 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=705719, maxMem=766075207 -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_93_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_154_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_154_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 154 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 78 (MapPartitionsRDD[161] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 155, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 156, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_92_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 155) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 156) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_91_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 155). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 155) in 5 ms on localhost (1/2) -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 156). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 156) in 6 ms on localhost (2/2) -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 78 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 78 finished: treeAggregate at GradientDescent.scala:225, took 0.016966 s -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_90_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=663867, maxMem=766075207 -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_155 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=665846, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_155_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_89_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_155_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 155 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Removed broadcast_88_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 79 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 79(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 79 (MapPartitionsRDD[163] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=648599, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_156 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=660383, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_156_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_156_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 156 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 79 (MapPartitionsRDD[163] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 79.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 79.0 (TID 157, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 79.0 (TID 158, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 79.0 (TID 157) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 79.0 (TID 158) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 79.0 (TID 157). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 79.0 (TID 158). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 79.0 (TID 157) in 7 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 79.0 (TID 158) in 7 ms on localhost (2/2) -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 79 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 79.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 79 finished: treeAggregate at GradientDescent.scala:225, took 0.018407 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=663866, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_157 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=669458, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_157_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_157_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 157 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 80 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 80(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 80 (MapPartitionsRDD[165] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=673070, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_158 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=684854, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_158_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_158_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 158 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 80 (MapPartitionsRDD[165] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 80.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 80.0 (TID 159, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 80.0 (TID 160, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 80.0 (TID 160) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 80.0 (TID 159) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 80.0 (TID 159). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 80.0 (TID 160). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 80.0 (TID 159) in 6 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 80.0 (TID 160) in 6 ms on localhost (2/2) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 80.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 80 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 80 finished: treeAggregate at GradientDescent.scala:225, took 0.018207 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=688338, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_159 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=693930, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_159_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_159_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 159 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 81 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 81(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 81 (MapPartitionsRDD[167] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=697536, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_160 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=709320, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_160_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_160_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 160 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 81 (MapPartitionsRDD[167] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 81.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 81.0 (TID 161, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 81.0 (TID 162, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 81.0 (TID 161) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 81.0 (TID 162) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 81.0 (TID 161). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 81.0 (TID 162). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 81.0 (TID 161) in 7 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 81.0 (TID 162) in 6 ms on localhost (2/2) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 81.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 81 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 81 finished: treeAggregate at GradientDescent.scala:225, took 0.018863 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=712803, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_161 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=718395, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_161_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_161_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 161 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 82 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 82(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 82 (MapPartitionsRDD[169] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=722007, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_162 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=733791, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_162_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_162_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 162 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 82 (MapPartitionsRDD[169] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 82.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 82.0 (TID 163, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 82.0 (TID 164, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 82.0 (TID 163) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 82.0 (TID 164) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 82.0 (TID 164). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 82.0 (TID 163). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 82.0 (TID 164) in 7 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 82.0 (TID 163) in 9 ms on localhost (2/2) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 82.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 82 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 82 finished: treeAggregate at GradientDescent.scala:225, took 0.019517 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=737274, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_163 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=742866, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_163_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_163_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 163 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 83 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 83(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 83 (MapPartitionsRDD[171] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=746480, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_164 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=758264, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_164_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_164_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 164 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 83 (MapPartitionsRDD[171] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 83.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 83.0 (TID 165, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 83.0 (TID 166, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 83.0 (TID 165) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 83.0 (TID 166) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 83.0 (TID 165). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 83.0 (TID 166). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 83.0 (TID 166) in 7 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 83.0 (TID 165) in 7 ms on localhost (2/2) -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 83 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 83.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 83 finished: treeAggregate at GradientDescent.scala:225, took 0.016219 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=761748, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_165 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=767340, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_165_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_165_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 165 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 84 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 84(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 84 (MapPartitionsRDD[173] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=770952, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_166 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=782736, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_166_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_166_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 166 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 84 (MapPartitionsRDD[173] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 84.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 84.0 (TID 167, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 84.0 (TID 168, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 84.0 (TID 167) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 84.0 (TID 168) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 84.0 (TID 167). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 84.0 (TID 168). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 84.0 (TID 167) in 7 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 84.0 (TID 168) in 7 ms on localhost (2/2) -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 84 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 84.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 84 finished: treeAggregate at GradientDescent.scala:225, took 0.018922 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=786220, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_167 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3620) called with curMem=791812, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_167_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_167_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 167 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 85 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 85(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 85 (MapPartitionsRDD[175] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=795432, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_168 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=807216, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_168_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_168_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 168 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 85 (MapPartitionsRDD[175] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 85.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 85.0 (TID 169, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 85.0 (TID 170, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 85.0 (TID 169) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 85.0 (TID 170) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 85.0 (TID 170). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 85.0 (TID 169). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 85.0 (TID 170) in 6 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 85.0 (TID 169) in 6 ms on localhost (2/2) -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 85 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 85.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 85 finished: treeAggregate at GradientDescent.scala:225, took 0.016551 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=810698, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_169 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3621) called with curMem=816290, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_169_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_169_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 169 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 86 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 86(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 86 (MapPartitionsRDD[177] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=819911, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_170 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=831695, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_170_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_170_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 170 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 86 (MapPartitionsRDD[177] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 86.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 86.0 (TID 171, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 86.0 (TID 172, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 86.0 (TID 171) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 86.0 (TID 172) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 86.0 (TID 172). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 86.0 (TID 171). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 86.0 (TID 172) in 5 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 86.0 (TID 171) in 7 ms on localhost (2/2) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 86.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 86 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 86 finished: treeAggregate at GradientDescent.scala:225, took 0.015941 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=835178, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_171 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=840770, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_171_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_171_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 171 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 87 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 87(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 87 (MapPartitionsRDD[179] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=844382, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_172 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=856166, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_172_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_172_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 172 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 87 (MapPartitionsRDD[179] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 87.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 87.0 (TID 173, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 87.0 (TID 174, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 87.0 (TID 173) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 87.0 (TID 174) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 87.0 (TID 174). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 87.0 (TID 173). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 87.0 (TID 174) in 6 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 87.0 (TID 173) in 6 ms on localhost (2/2) -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 87 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 87.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 87 finished: treeAggregate at GradientDescent.scala:225, took 0.015561 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=859649, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_173 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=865241, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_173_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_173_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 173 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 88 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 88(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 88 (MapPartitionsRDD[181] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=868854, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_174 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3486) called with curMem=880638, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_174_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_174_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 174 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 88 (MapPartitionsRDD[181] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 88.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 88.0 (TID 175, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 88.0 (TID 176, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 88.0 (TID 175) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 88.0 (TID 176) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 88.0 (TID 176). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 88.0 (TID 175). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 88.0 (TID 176) in 7 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 88.0 (TID 175) in 8 ms on localhost (2/2) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 88.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 88 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 88 finished: treeAggregate at GradientDescent.scala:225, took 0.019182 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=884124, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_175 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3615) called with curMem=889716, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_175_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_175_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 175 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 89 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 89(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 89 (MapPartitionsRDD[183] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=893331, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_176 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=905115, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_176_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_176_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 176 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 89 (MapPartitionsRDD[183] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 89.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 89.0 (TID 177, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 89.0 (TID 178, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 89.0 (TID 177) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 89.0 (TID 178) -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 89.0 (TID 177). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 89.0 (TID 177) in 7 ms on localhost (1/2) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 89.0 (TID 178). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 89.0 (TID 178) in 10 ms on localhost (2/2) -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 89 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 89.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 89 finished: treeAggregate at GradientDescent.scala:225, took 0.022178 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=908598, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_177 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=914190, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_177_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_177_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 177 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 90 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 90(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 90 (MapPartitionsRDD[185] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=917802, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_178 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=929586, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_178_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_178_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 178 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 90 (MapPartitionsRDD[185] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 90.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 90.0 (TID 179, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 90.0 (TID 180, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 90.0 (TID 179) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 90.0 (TID 180) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 90.0 (TID 179). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 90.0 (TID 180). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 90.0 (TID 179) in 6 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 90.0 (TID 180) in 7 ms on localhost (2/2) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 90.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 90 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 90 finished: treeAggregate at GradientDescent.scala:225, took 0.015780 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=933070, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_179 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=938662, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_179_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_179_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 179 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 91 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 91(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 91 (MapPartitionsRDD[187] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=942275, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_180 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=954059, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_180_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_180_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 180 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 91 (MapPartitionsRDD[187] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 91.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 91.0 (TID 181, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 91.0 (TID 182, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 91.0 (TID 181) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 91.0 (TID 182) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 91.0 (TID 181). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 91.0 (TID 182). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 91.0 (TID 181) in 5 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 91.0 (TID 182) in 5 ms on localhost (2/2) -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 91 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 91.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 91 finished: treeAggregate at GradientDescent.scala:225, took 0.016235 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=957543, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_181 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3608) called with curMem=963135, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_181_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_181_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 181 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 92 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 92(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting ResultStage 92 (MapPartitionsRDD[189] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=966743, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_182 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=978527, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_182_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_182_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 182 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 92 (MapPartitionsRDD[189] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Adding task set 92.0 with 2 tasks -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 92.0 (TID 183, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 92.0 (TID 184, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:22 INFO Executor:59 - Running task 0.0 in stage 92.0 (TID 183) -2016-05-24 22:23:22 INFO Executor:59 - Running task 1.0 in stage 92.0 (TID 184) -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:22 INFO Executor:59 - Finished task 0.0 in stage 92.0 (TID 183). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO Executor:59 - Finished task 1.0 in stage 92.0 (TID 184). 7968 bytes result sent to driver -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 92.0 (TID 183) in 4 ms on localhost (1/2) -2016-05-24 22:23:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 92.0 (TID 184) in 5 ms on localhost (2/2) -2016-05-24 22:23:22 INFO DAGScheduler:59 - ResultStage 92 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:23:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 92.0, whose tasks have all completed, from pool -2016-05-24 22:23:22 INFO DAGScheduler:59 - Job 92 finished: treeAggregate at GradientDescent.scala:225, took 0.011913 s -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=982011, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_183 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=987603, maxMem=766075207 -2016-05-24 22:23:22 INFO MemoryStore:59 - Block broadcast_183_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:23:22 INFO BlockManagerInfo:59 - Added broadcast_183_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:22 INFO SparkContext:59 - Created broadcast 183 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:22 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:22 INFO DAGScheduler:59 - Got job 93 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:22 INFO DAGScheduler:59 - Final stage: ResultStage 93(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting ResultStage 93 (MapPartitionsRDD[191] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=991217, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_184 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1003001, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_184_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_184_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 184 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 93 (MapPartitionsRDD[191] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Adding task set 93.0 with 2 tasks -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 93.0 (TID 185, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 93.0 (TID 186, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO Executor:59 - Running task 1.0 in stage 93.0 (TID 186) -2016-05-24 22:23:23 INFO Executor:59 - Running task 0.0 in stage 93.0 (TID 185) -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:23 INFO Executor:59 - Finished task 1.0 in stage 93.0 (TID 186). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO Executor:59 - Finished task 0.0 in stage 93.0 (TID 185). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 93.0 (TID 186) in 7 ms on localhost (1/2) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 93.0 (TID 185) in 9 ms on localhost (2/2) -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 93.0, whose tasks have all completed, from pool -2016-05-24 22:23:23 INFO DAGScheduler:59 - ResultStage 93 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:23:23 INFO DAGScheduler:59 - Job 93 finished: treeAggregate at GradientDescent.scala:225, took 0.016748 s -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1006484, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_185 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1012076, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_185_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_185_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 185 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:23 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Got job 94 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:23 INFO DAGScheduler:59 - Final stage: ResultStage 94(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting ResultStage 94 (MapPartitionsRDD[193] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1015688, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_186 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1027472, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_186_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_186_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 186 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 94 (MapPartitionsRDD[193] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Adding task set 94.0 with 2 tasks -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 94.0 (TID 187, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 94.0 (TID 188, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO Executor:59 - Running task 1.0 in stage 94.0 (TID 188) -2016-05-24 22:23:23 INFO Executor:59 - Running task 0.0 in stage 94.0 (TID 187) -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:23 INFO Executor:59 - Finished task 1.0 in stage 94.0 (TID 188). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 94.0 (TID 188) in 5 ms on localhost (1/2) -2016-05-24 22:23:23 INFO Executor:59 - Finished task 0.0 in stage 94.0 (TID 187). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 94.0 (TID 187) in 7 ms on localhost (2/2) -2016-05-24 22:23:23 INFO DAGScheduler:59 - ResultStage 94 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 94.0, whose tasks have all completed, from pool -2016-05-24 22:23:23 INFO DAGScheduler:59 - Job 94 finished: treeAggregate at GradientDescent.scala:225, took 0.016068 s -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1030956, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_187 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1036548, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_187_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_187_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 187 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:23 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Got job 95 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:23 INFO DAGScheduler:59 - Final stage: ResultStage 95(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting ResultStage 95 (MapPartitionsRDD[195] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1040161, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_188 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=1051945, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_188_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_188_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 188 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 95 (MapPartitionsRDD[195] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Adding task set 95.0 with 2 tasks -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 95.0 (TID 189, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 95.0 (TID 190, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO Executor:59 - Running task 0.0 in stage 95.0 (TID 189) -2016-05-24 22:23:23 INFO Executor:59 - Running task 1.0 in stage 95.0 (TID 190) -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:23 INFO Executor:59 - Finished task 0.0 in stage 95.0 (TID 189). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 95.0 (TID 189) in 6 ms on localhost (1/2) -2016-05-24 22:23:23 INFO Executor:59 - Finished task 1.0 in stage 95.0 (TID 190). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 95.0 (TID 190) in 7 ms on localhost (2/2) -2016-05-24 22:23:23 INFO DAGScheduler:59 - ResultStage 95 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 95.0, whose tasks have all completed, from pool -2016-05-24 22:23:23 INFO DAGScheduler:59 - Job 95 finished: treeAggregate at GradientDescent.scala:225, took 0.016295 s -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1055427, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_189 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3634) called with curMem=1061019, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_189_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_189_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 189 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:23 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Got job 96 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:23 INFO DAGScheduler:59 - Final stage: ResultStage 96(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting ResultStage 96 (MapPartitionsRDD[197] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1064653, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_190 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=1076437, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_190_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_190_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 190 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 96 (MapPartitionsRDD[197] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Adding task set 96.0 with 2 tasks -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 96.0 (TID 191, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 96.0 (TID 192, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO Executor:59 - Running task 1.0 in stage 96.0 (TID 192) -2016-05-24 22:23:23 INFO Executor:59 - Running task 0.0 in stage 96.0 (TID 191) -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:23 INFO Executor:59 - Finished task 0.0 in stage 96.0 (TID 191). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO Executor:59 - Finished task 1.0 in stage 96.0 (TID 192). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 96.0 (TID 191) in 5 ms on localhost (1/2) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 96.0 (TID 192) in 5 ms on localhost (2/2) -2016-05-24 22:23:23 INFO DAGScheduler:59 - ResultStage 96 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 96.0, whose tasks have all completed, from pool -2016-05-24 22:23:23 INFO DAGScheduler:59 - Job 96 finished: treeAggregate at GradientDescent.scala:225, took 0.012381 s -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1079920, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_191 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3619) called with curMem=1085512, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_191_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_191_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 191 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:23 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Got job 97 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:23 INFO DAGScheduler:59 - Final stage: ResultStage 97(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting ResultStage 97 (MapPartitionsRDD[199] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1089131, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_192 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1100915, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_192_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_192_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 192 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 97 (MapPartitionsRDD[199] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Adding task set 97.0 with 2 tasks -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 97.0 (TID 193, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 97.0 (TID 194, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO Executor:59 - Running task 0.0 in stage 97.0 (TID 193) -2016-05-24 22:23:23 INFO Executor:59 - Running task 1.0 in stage 97.0 (TID 194) -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:23 INFO Executor:59 - Finished task 0.0 in stage 97.0 (TID 193). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO Executor:59 - Finished task 1.0 in stage 97.0 (TID 194). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 97.0 (TID 193) in 6 ms on localhost (1/2) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 97.0 (TID 194) in 5 ms on localhost (2/2) -2016-05-24 22:23:23 INFO DAGScheduler:59 - ResultStage 97 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 97.0, whose tasks have all completed, from pool -2016-05-24 22:23:23 INFO DAGScheduler:59 - Job 97 finished: treeAggregate at GradientDescent.scala:225, took 0.015132 s -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1104399, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_193 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1109991, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_193_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_193_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 193 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:23 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Got job 98 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:23 INFO DAGScheduler:59 - Final stage: ResultStage 98(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting ResultStage 98 (MapPartitionsRDD[201] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1113603, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_194 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1125387, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_194_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_194_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 194 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 98 (MapPartitionsRDD[201] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Adding task set 98.0 with 2 tasks -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 98.0 (TID 195, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 98.0 (TID 196, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO Executor:59 - Running task 0.0 in stage 98.0 (TID 195) -2016-05-24 22:23:23 INFO Executor:59 - Running task 1.0 in stage 98.0 (TID 196) -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:23 INFO Executor:59 - Finished task 1.0 in stage 98.0 (TID 196). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO Executor:59 - Finished task 0.0 in stage 98.0 (TID 195). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 98.0 (TID 196) in 3 ms on localhost (1/2) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 98.0 (TID 195) in 5 ms on localhost (2/2) -2016-05-24 22:23:23 INFO DAGScheduler:59 - ResultStage 98 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 98.0, whose tasks have all completed, from pool -2016-05-24 22:23:23 INFO DAGScheduler:59 - Job 98 finished: treeAggregate at GradientDescent.scala:225, took 0.011809 s -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1128871, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_195 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=1134463, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_195_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_195_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 195 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:23 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Got job 99 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:23 INFO DAGScheduler:59 - Final stage: ResultStage 99(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting ResultStage 99 (MapPartitionsRDD[203] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1138072, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_196 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1149856, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_196_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_196_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 196 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_172_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 99 (MapPartitionsRDD[203] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Adding task set 99.0 with 2 tasks -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 99.0 (TID 197, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 99.0 (TID 198, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO Executor:59 - Running task 0.0 in stage 99.0 (TID 197) -2016-05-24 22:23:23 INFO Executor:59 - Running task 1.0 in stage 99.0 (TID 198) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_194_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 99 -2016-05-24 22:23:23 INFO Executor:59 - Finished task 0.0 in stage 99.0 (TID 197). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO Executor:59 - Finished task 1.0 in stage 99.0 (TID 198). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 99.0 (TID 197) in 6 ms on localhost (1/2) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 99.0 (TID 198) in 5 ms on localhost (2/2) -2016-05-24 22:23:23 INFO DAGScheduler:59 - ResultStage 99 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 99.0, whose tasks have all completed, from pool -2016-05-24 22:23:23 INFO DAGScheduler:59 - Job 99 finished: treeAggregate at GradientDescent.scala:225, took 0.019691 s -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_193_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1113601, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_197 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3616) called with curMem=1115709, maxMem=766075207 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_192_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_197_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 98 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_197_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 197 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_191_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_190_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 97 -2016-05-24 22:23:23 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Got job 100 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:23 INFO DAGScheduler:59 - Final stage: ResultStage 100(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_189_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting ResultStage 100 (MapPartitionsRDD[205] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1070355, maxMem=766075207 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_188_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_198 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 96 -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1070355, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_198_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_198_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 198 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 100 (MapPartitionsRDD[205] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Adding task set 100.0 with 2 tasks -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_187_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 100.0 (TID 199, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 100.0 (TID 200, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO Executor:59 - Running task 1.0 in stage 100.0 (TID 200) -2016-05-24 22:23:23 INFO Executor:59 - Running task 0.0 in stage 100.0 (TID 199) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_186_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 95 -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:23 INFO Executor:59 - Finished task 1.0 in stage 100.0 (TID 200). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO Executor:59 - Finished task 0.0 in stage 100.0 (TID 199). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_185_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 100.0 (TID 200) in 6 ms on localhost (1/2) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 100.0 (TID 199) in 6 ms on localhost (2/2) -2016-05-24 22:23:23 INFO DAGScheduler:59 - ResultStage 100 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 100.0, whose tasks have all completed, from pool -2016-05-24 22:23:23 INFO DAGScheduler:59 - Job 100 finished: treeAggregate at GradientDescent.scala:225, took 0.014500 s -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1036679, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_199 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_184_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1030487, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_199_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 94 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_199_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 199 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_183_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_182_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 93 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_181_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Got job 101 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:23 INFO DAGScheduler:59 - Final stage: ResultStage 101(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting ResultStage 101 (MapPartitionsRDD[207] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_180_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 92 -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=979559, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_200 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=987730, maxMem=766075207 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_179_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_200_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_200_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 200 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 101 (MapPartitionsRDD[207] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Adding task set 101.0 with 2 tasks -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_178_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 101.0 (TID 201, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 101.0 (TID 202, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 91 -2016-05-24 22:23:23 INFO Executor:59 - Running task 0.0 in stage 101.0 (TID 201) -2016-05-24 22:23:23 INFO Executor:59 - Running task 1.0 in stage 101.0 (TID 202) -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_143_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO Executor:59 - Finished task 1.0 in stage 101.0 (TID 202). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 101.0 (TID 202) in 6 ms on localhost (1/2) -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 74 -2016-05-24 22:23:23 INFO Executor:59 - Finished task 0.0 in stage 101.0 (TID 201). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 101.0 (TID 201) in 9 ms on localhost (2/2) -2016-05-24 22:23:23 INFO DAGScheduler:59 - ResultStage 101 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 101.0, whose tasks have all completed, from pool -2016-05-24 22:23:23 INFO DAGScheduler:59 - Job 101 finished: treeAggregate at GradientDescent.scala:225, took 0.017125 s -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_144_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=951476, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_201 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=947870, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_201_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_145_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 75 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_201_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 201 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_146_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_147_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 76 -2016-05-24 22:23:23 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Got job 102 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:23 INFO DAGScheduler:59 - Final stage: ResultStage 102(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_148_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting ResultStage 102 (MapPartitionsRDD[209] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=908107, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_202 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_149_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=914299, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_202_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 77 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_202_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 202 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 102 (MapPartitionsRDD[209] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Adding task set 102.0 with 2 tasks -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 102.0 (TID 203, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_150_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 102.0 (TID 204, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO Executor:59 - Running task 1.0 in stage 102.0 (TID 204) -2016-05-24 22:23:23 INFO Executor:59 - Running task 0.0 in stage 102.0 (TID 203) -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_151_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO Executor:59 - Finished task 1.0 in stage 102.0 (TID 204). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO Executor:59 - Finished task 0.0 in stage 102.0 (TID 203). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 78 -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 102.0 (TID 204) in 6 ms on localhost (1/2) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 102.0 (TID 203) in 7 ms on localhost (2/2) -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 102.0, whose tasks have all completed, from pool -2016-05-24 22:23:23 INFO DAGScheduler:59 - ResultStage 102 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:23:23 INFO DAGScheduler:59 - Job 102 finished: treeAggregate at GradientDescent.scala:225, took 0.017219 s -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_152_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=878045, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_203 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=880025, maxMem=766075207 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_153_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_203_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 79 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_203_piece0 in memory on localhost:56959 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 203 from broadcast at GradientDescent.scala:221 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_154_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_155_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_177_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_176_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Got job 103 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:23:23 INFO DAGScheduler:59 - Final stage: ResultStage 103(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 90 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting ResultStage 103 (MapPartitionsRDD[211] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_175_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=825488, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_204 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=819896, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_204_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_204_piece0 in memory on localhost:56959 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 204 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_174_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 103 (MapPartitionsRDD[211] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Adding task set 103.0 with 2 tasks -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 89 -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 103.0 (TID 205, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 103.0 (TID 206, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:23:23 INFO Executor:59 - Running task 0.0 in stage 103.0 (TID 205) -2016-05-24 22:23:23 INFO Executor:59 - Running task 1.0 in stage 103.0 (TID 206) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_173_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 88 -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:23:23 INFO Executor:59 - Finished task 0.0 in stage 103.0 (TID 205). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO Executor:59 - Finished task 1.0 in stage 103.0 (TID 206). 7968 bytes result sent to driver -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_171_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 103.0 (TID 205) in 5 ms on localhost (1/2) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 103.0 (TID 206) in 5 ms on localhost (2/2) -2016-05-24 22:23:23 INFO DAGScheduler:59 - ResultStage 103 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 103.0, whose tasks have all completed, from pool -2016-05-24 22:23:23 INFO DAGScheduler:59 - Job 103 finished: treeAggregate at GradientDescent.scala:225, took 0.013519 s -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_170_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 3363.579936927662, 3356.5316534793847, 3349.536449573466, 3342.59343622967, 3335.701748404667, 3328.860544103912, 3322.069003535364, 3315.3263283026713, 3308.6317406356134, 3301.984482655661 -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 87 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_169_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_168_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 86 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_167_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_166_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 85 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_165_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_164_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 84 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_163_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_162_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 83 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_161_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_160_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 82 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_159_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_158_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 81 -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_157_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Removed broadcast_156_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:23 INFO ContextCleaner:59 - Cleaned accumulator 80 -2016-05-24 22:23:23 INFO SparkContext:59 - Starting job: sortByKey at BinaryClassificationMetrics.scala:156 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Registering RDD 212 (map at SVMFromSparkLearning.scala:44) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Got job 104 (sortByKey at BinaryClassificationMetrics.scala:156) with 2 output partitions -2016-05-24 22:23:23 INFO DAGScheduler:59 - Final stage: ResultStage 105(sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 104) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 104) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting ShuffleMapStage 104 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44), which has no missing parents -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=614906, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_205 stored as values in memory (estimated size 10.8 KB, free 730.0 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(6433) called with curMem=625914, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_205_piece0 stored as bytes in memory (estimated size 6.3 KB, free 730.0 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_205_piece0 in memory on localhost:56959 (size: 6.3 KB, free: 730.2 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 205 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 104 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44) -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Adding task set 104.0 with 2 tasks -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 104.0 (TID 207, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 104.0 (TID 208, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:23:23 INFO Executor:59 - Running task 1.0 in stage 104.0 (TID 208) -2016-05-24 22:23:23 INFO Executor:59 - Running task 0.0 in stage 104.0 (TID 207) -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:23:23 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:23:23 INFO Executor:59 - Finished task 0.0 in stage 104.0 (TID 207). 2254 bytes result sent to driver -2016-05-24 22:23:23 INFO Executor:59 - Finished task 1.0 in stage 104.0 (TID 208). 2254 bytes result sent to driver -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 104.0 (TID 208) in 453 ms on localhost (1/2) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 104.0 (TID 207) in 456 ms on localhost (2/2) -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 104.0, whose tasks have all completed, from pool -2016-05-24 22:23:23 INFO DAGScheduler:59 - ShuffleMapStage 104 (map at SVMFromSparkLearning.scala:44) finished in 0.457 s -2016-05-24 22:23:23 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:23:23 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:23:23 INFO DAGScheduler:59 - waiting: Set(ResultStage 105) -2016-05-24 22:23:23 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Missing parents for ResultStage 105: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting ResultStage 105 (MapPartitionsRDD[215] at sortByKey at BinaryClassificationMetrics.scala:156), which is now runnable -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3344) called with curMem=632347, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_206 stored as values in memory (estimated size 3.3 KB, free 730.0 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(1900) called with curMem=635691, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_206_piece0 stored as bytes in memory (estimated size 1900.0 B, free 730.0 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_206_piece0 in memory on localhost:56959 (size: 1900.0 B, free: 730.2 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 206 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 105 (MapPartitionsRDD[215] at sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Adding task set 105.0 with 2 tasks -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 105.0 (TID 209, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 105.0 (TID 210, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:23:23 INFO Executor:59 - Running task 0.0 in stage 105.0 (TID 209) -2016-05-24 22:23:23 INFO Executor:59 - Running task 1.0 in stage 105.0 (TID 210) -2016-05-24 22:23:23 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:23:23 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:23:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-24 22:23:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-24 22:23:23 INFO Executor:59 - Finished task 0.0 in stage 105.0 (TID 209). 1469 bytes result sent to driver -2016-05-24 22:23:23 INFO Executor:59 - Finished task 1.0 in stage 105.0 (TID 210). 1517 bytes result sent to driver -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 105.0 (TID 209) in 38 ms on localhost (1/2) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 105.0 (TID 210) in 37 ms on localhost (2/2) -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 105.0, whose tasks have all completed, from pool -2016-05-24 22:23:23 INFO DAGScheduler:59 - ResultStage 105 (sortByKey at BinaryClassificationMetrics.scala:156) finished in 0.039 s -2016-05-24 22:23:23 INFO DAGScheduler:59 - Job 104 finished: sortByKey at BinaryClassificationMetrics.scala:156, took 0.521365 s -2016-05-24 22:23:23 INFO SparkContext:59 - Starting job: collect at BinaryClassificationMetrics.scala:193 -2016-05-24 22:23:23 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 154 bytes -2016-05-24 22:23:23 INFO DAGScheduler:59 - Registering RDD 213 (combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Got job 105 (collect at BinaryClassificationMetrics.scala:193) with 2 output partitions -2016-05-24 22:23:23 INFO DAGScheduler:59 - Final stage: ResultStage 108(collect at BinaryClassificationMetrics.scala:193) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 107) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 107) -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting ShuffleMapStage 107 (ShuffledRDD[213] at combineByKey at BinaryClassificationMetrics.scala:152), which has no missing parents -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=637591, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_207 stored as values in memory (estimated size 2.8 KB, free 730.0 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(1658) called with curMem=640487, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_207_piece0 stored as bytes in memory (estimated size 1658.0 B, free 730.0 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_207_piece0 in memory on localhost:56959 (size: 1658.0 B, free: 730.2 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 207 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 107 (ShuffledRDD[213] at combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Adding task set 107.0 with 2 tasks -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 107.0 (TID 211, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 107.0 (TID 212, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:23:23 INFO Executor:59 - Running task 0.0 in stage 107.0 (TID 211) -2016-05-24 22:23:23 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:23:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:23:23 INFO Executor:59 - Running task 1.0 in stage 107.0 (TID 212) -2016-05-24 22:23:23 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:23:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:23:23 INFO Executor:59 - Finished task 0.0 in stage 107.0 (TID 211). 1375 bytes result sent to driver -2016-05-24 22:23:23 INFO Executor:59 - Finished task 1.0 in stage 107.0 (TID 212). 1375 bytes result sent to driver -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 107.0 (TID 211) in 34 ms on localhost (1/2) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 107.0 (TID 212) in 33 ms on localhost (2/2) -2016-05-24 22:23:23 INFO DAGScheduler:59 - ShuffleMapStage 107 (combineByKey at BinaryClassificationMetrics.scala:152) finished in 0.034 s -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 107.0, whose tasks have all completed, from pool -2016-05-24 22:23:23 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:23:23 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:23:23 INFO DAGScheduler:59 - waiting: Set(ResultStage 108) -2016-05-24 22:23:23 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Missing parents for ResultStage 108: List() -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting ResultStage 108 (MapPartitionsRDD[218] at mapPartitions at BinaryClassificationMetrics.scala:189), which is now runnable -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(3472) called with curMem=642145, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_208 stored as values in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:23:23 INFO MemoryStore:59 - ensureFreeSpace(1981) called with curMem=645617, maxMem=766075207 -2016-05-24 22:23:23 INFO MemoryStore:59 - Block broadcast_208_piece0 stored as bytes in memory (estimated size 1981.0 B, free 730.0 MB) -2016-05-24 22:23:23 INFO BlockManagerInfo:59 - Added broadcast_208_piece0 in memory on localhost:56959 (size: 1981.0 B, free: 730.2 MB) -2016-05-24 22:23:23 INFO SparkContext:59 - Created broadcast 208 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 108 (MapPartitionsRDD[218] at mapPartitions at BinaryClassificationMetrics.scala:189) -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Adding task set 108.0 with 2 tasks -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 108.0 (TID 213, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 108.0 (TID 214, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:23:23 INFO Executor:59 - Running task 1.0 in stage 108.0 (TID 214) -2016-05-24 22:23:23 INFO Executor:59 - Running task 0.0 in stage 108.0 (TID 213) -2016-05-24 22:23:23 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:23:23 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:23:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:23:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:23:23 INFO Executor:59 - Finished task 0.0 in stage 108.0 (TID 213). 1330 bytes result sent to driver -2016-05-24 22:23:23 INFO Executor:59 - Finished task 1.0 in stage 108.0 (TID 214). 1330 bytes result sent to driver -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 108.0 (TID 213) in 14 ms on localhost (1/2) -2016-05-24 22:23:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 108.0 (TID 214) in 15 ms on localhost (2/2) -2016-05-24 22:23:23 INFO DAGScheduler:59 - ResultStage 108 (collect at BinaryClassificationMetrics.scala:193) finished in 0.015 s -2016-05-24 22:23:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 108.0, whose tasks have all completed, from pool -2016-05-24 22:23:23 INFO DAGScheduler:59 - Job 105 finished: collect at BinaryClassificationMetrics.scala:193, took 0.072251 s -2016-05-24 22:23:23 INFO BinaryClassificationMetrics:59 - Total counts: {numPos: 25, numNeg: 11} -2016-05-24 22:23:24 INFO SparkContext:59 - Starting job: runJob at SlidingRDD.scala:75 -2016-05-24 22:23:24 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 154 bytes -2016-05-24 22:23:24 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-24 22:23:24 INFO DAGScheduler:59 - Got job 106 (runJob at SlidingRDD.scala:75) with 3 output partitions -2016-05-24 22:23:24 INFO DAGScheduler:59 - Final stage: ResultStage 111(runJob at SlidingRDD.scala:75) -2016-05-24 22:23:24 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 110) -2016-05-24 22:23:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:24 INFO DAGScheduler:59 - Submitting ResultStage 111 (UnionRDD[224] at UnionRDD at BinaryClassificationMetrics.scala:92), which has no missing parents -2016-05-24 22:23:24 INFO MemoryStore:59 - ensureFreeSpace(5184) called with curMem=647598, maxMem=766075207 -2016-05-24 22:23:24 INFO MemoryStore:59 - Block broadcast_209 stored as values in memory (estimated size 5.1 KB, free 730.0 MB) -2016-05-24 22:23:24 INFO MemoryStore:59 - ensureFreeSpace(2788) called with curMem=652782, maxMem=766075207 -2016-05-24 22:23:24 INFO MemoryStore:59 - Block broadcast_209_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.0 MB) -2016-05-24 22:23:24 INFO BlockManagerInfo:59 - Added broadcast_209_piece0 in memory on localhost:56959 (size: 2.7 KB, free: 730.2 MB) -2016-05-24 22:23:24 INFO SparkContext:59 - Created broadcast 209 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:24 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 111 (UnionRDD[224] at UnionRDD at BinaryClassificationMetrics.scala:92) -2016-05-24 22:23:24 INFO TaskSchedulerImpl:59 - Adding task set 111.0 with 3 tasks -2016-05-24 22:23:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 111.0 (TID 215, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:23:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 111.0 (TID 216, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:23:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 111.0 (TID 217, localhost, PROCESS_LOCAL, 2318 bytes) -2016-05-24 22:23:24 INFO Executor:59 - Running task 0.0 in stage 111.0 (TID 215) -2016-05-24 22:23:24 INFO Executor:59 - Running task 1.0 in stage 111.0 (TID 216) -2016-05-24 22:23:24 INFO Executor:59 - Running task 2.0 in stage 111.0 (TID 217) -2016-05-24 22:23:24 INFO CacheManager:59 - Partition rdd_219_0 not found, computing it -2016-05-24 22:23:24 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:23:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:23:24 INFO Executor:59 - Finished task 2.0 in stage 111.0 (TID 217). 1053 bytes result sent to driver -2016-05-24 22:23:24 INFO CacheManager:59 - Partition rdd_219_1 not found, computing it -2016-05-24 22:23:24 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:23:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:23:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 111.0 (TID 217) in 11 ms on localhost (1/3) -2016-05-24 22:23:24 INFO MemoryStore:59 - ensureFreeSpace(1448) called with curMem=655570, maxMem=766075207 -2016-05-24 22:23:24 INFO MemoryStore:59 - Block rdd_219_0 stored as values in memory (estimated size 1448.0 B, free 730.0 MB) -2016-05-24 22:23:24 INFO BlockManagerInfo:59 - Added rdd_219_0 in memory on localhost:56959 (size: 1448.0 B, free: 730.2 MB) -2016-05-24 22:23:24 INFO MemoryStore:59 - ensureFreeSpace(1616) called with curMem=657018, maxMem=766075207 -2016-05-24 22:23:24 INFO MemoryStore:59 - Block rdd_219_1 stored as values in memory (estimated size 1616.0 B, free 730.0 MB) -2016-05-24 22:23:24 INFO Executor:59 - Finished task 0.0 in stage 111.0 (TID 215). 1883 bytes result sent to driver -2016-05-24 22:23:24 INFO BlockManagerInfo:59 - Added rdd_219_1 in memory on localhost:56959 (size: 1616.0 B, free: 730.2 MB) -2016-05-24 22:23:24 INFO Executor:59 - Finished task 1.0 in stage 111.0 (TID 216). 1883 bytes result sent to driver -2016-05-24 22:23:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 111.0 (TID 215) in 20 ms on localhost (2/3) -2016-05-24 22:23:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 111.0 (TID 216) in 19 ms on localhost (3/3) -2016-05-24 22:23:24 INFO DAGScheduler:59 - ResultStage 111 (runJob at SlidingRDD.scala:75) finished in 0.022 s -2016-05-24 22:23:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 111.0, whose tasks have all completed, from pool -2016-05-24 22:23:24 INFO DAGScheduler:59 - Job 106 finished: runJob at SlidingRDD.scala:75, took 0.039009 s -2016-05-24 22:23:24 INFO SparkContext:59 - Starting job: aggregate at AreaUnderCurve.scala:45 -2016-05-24 22:23:24 INFO DAGScheduler:59 - Got job 107 (aggregate at AreaUnderCurve.scala:45) with 4 output partitions -2016-05-24 22:23:24 INFO DAGScheduler:59 - Final stage: ResultStage 114(aggregate at AreaUnderCurve.scala:45) -2016-05-24 22:23:24 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 113) -2016-05-24 22:23:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:24 INFO DAGScheduler:59 - Submitting ResultStage 114 (SlidingRDD[225] at RDD at SlidingRDD.scala:47), which has no missing parents -2016-05-24 22:23:24 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=658634, maxMem=766075207 -2016-05-24 22:23:24 INFO MemoryStore:59 - Block broadcast_210 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:23:24 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=664290, maxMem=766075207 -2016-05-24 22:23:24 INFO MemoryStore:59 - Block broadcast_210_piece0 stored as bytes in memory (estimated size 2.9 KB, free 729.9 MB) -2016-05-24 22:23:24 INFO BlockManagerInfo:59 - Added broadcast_210_piece0 in memory on localhost:56959 (size: 2.9 KB, free: 730.2 MB) -2016-05-24 22:23:24 INFO SparkContext:59 - Created broadcast 210 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 114 (SlidingRDD[225] at RDD at SlidingRDD.scala:47) -2016-05-24 22:23:24 INFO TaskSchedulerImpl:59 - Adding task set 114.0 with 4 tasks -2016-05-24 22:23:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 114.0 (TID 218, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:23:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 114.0 (TID 219, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:23:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 114.0 (TID 220, localhost, PROCESS_LOCAL, 2635 bytes) -2016-05-24 22:23:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 114.0 (TID 221, localhost, PROCESS_LOCAL, 2475 bytes) -2016-05-24 22:23:24 INFO Executor:59 - Running task 2.0 in stage 114.0 (TID 219) -2016-05-24 22:23:24 INFO Executor:59 - Running task 0.0 in stage 114.0 (TID 220) -2016-05-24 22:23:24 INFO Executor:59 - Running task 3.0 in stage 114.0 (TID 221) -2016-05-24 22:23:24 INFO Executor:59 - Running task 1.0 in stage 114.0 (TID 218) -2016-05-24 22:23:24 INFO BlockManager:59 - Found block rdd_219_1 locally -2016-05-24 22:23:24 INFO Executor:59 - Finished task 0.0 in stage 114.0 (TID 220). 955 bytes result sent to driver -2016-05-24 22:23:24 INFO Executor:59 - Finished task 2.0 in stage 114.0 (TID 219). 2084 bytes result sent to driver -2016-05-24 22:23:24 INFO BlockManager:59 - Found block rdd_219_0 locally -2016-05-24 22:23:24 INFO Executor:59 - Finished task 3.0 in stage 114.0 (TID 221). 955 bytes result sent to driver -2016-05-24 22:23:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 114.0 (TID 220) in 12 ms on localhost (1/4) -2016-05-24 22:23:24 INFO Executor:59 - Finished task 1.0 in stage 114.0 (TID 218). 2084 bytes result sent to driver -2016-05-24 22:23:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 114.0 (TID 219) in 13 ms on localhost (2/4) -2016-05-24 22:23:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 114.0 (TID 221) in 12 ms on localhost (3/4) -2016-05-24 22:23:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 114.0 (TID 218) in 17 ms on localhost (4/4) -2016-05-24 22:23:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 114.0, whose tasks have all completed, from pool -2016-05-24 22:23:24 INFO DAGScheduler:59 - ResultStage 114 (aggregate at AreaUnderCurve.scala:45) finished in 0.017 s -2016-05-24 22:23:24 INFO DAGScheduler:59 - Job 107 finished: aggregate at AreaUnderCurve.scala:45, took 0.029413 s -2016-05-24 22:23:24 INFO SparkContext:59 - Starting job: take at SVMFromSparkLearning.scala:56 -2016-05-24 22:23:24 INFO DAGScheduler:59 - Got job 108 (take at SVMFromSparkLearning.scala:56) with 1 output partitions -2016-05-24 22:23:24 INFO DAGScheduler:59 - Final stage: ResultStage 115(take at SVMFromSparkLearning.scala:56) -2016-05-24 22:23:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:24 INFO DAGScheduler:59 - Submitting ResultStage 115 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44), which has no missing parents -2016-05-24 22:23:24 INFO MemoryStore:59 - ensureFreeSpace(10312) called with curMem=667261, maxMem=766075207 -2016-05-24 22:23:24 INFO MemoryStore:59 - Block broadcast_211 stored as values in memory (estimated size 10.1 KB, free 729.9 MB) -2016-05-24 22:23:24 INFO MemoryStore:59 - ensureFreeSpace(6082) called with curMem=677573, maxMem=766075207 -2016-05-24 22:23:24 INFO MemoryStore:59 - Block broadcast_211_piece0 stored as bytes in memory (estimated size 5.9 KB, free 729.9 MB) -2016-05-24 22:23:24 INFO BlockManagerInfo:59 - Added broadcast_211_piece0 in memory on localhost:56959 (size: 5.9 KB, free: 730.2 MB) -2016-05-24 22:23:24 INFO SparkContext:59 - Created broadcast 211 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 115 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44) -2016-05-24 22:23:24 INFO TaskSchedulerImpl:59 - Adding task set 115.0 with 1 tasks -2016-05-24 22:23:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 115.0 (TID 222, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:23:24 INFO Executor:59 - Running task 0.0 in stage 115.0 (TID 222) -2016-05-24 22:23:24 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:23:24 INFO Executor:59 - Finished task 0.0 in stage 115.0 (TID 222). 2401 bytes result sent to driver -2016-05-24 22:23:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 115.0 (TID 222) in 6 ms on localhost (1/1) -2016-05-24 22:23:24 INFO DAGScheduler:59 - ResultStage 115 (take at SVMFromSparkLearning.scala:56) finished in 0.006 s -2016-05-24 22:23:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 115.0, whose tasks have all completed, from pool -2016-05-24 22:23:24 INFO DAGScheduler:59 - Job 108 finished: take at SVMFromSparkLearning.scala:56, took 0.014647 s -2016-05-24 22:23:24 INFO SparkContext:59 - Starting job: saveAsTextFile at GLMClassificationModel.scala:61 -2016-05-24 22:23:24 INFO DAGScheduler:59 - Got job 109 (saveAsTextFile at GLMClassificationModel.scala:61) with 1 output partitions -2016-05-24 22:23:24 INFO DAGScheduler:59 - Final stage: ResultStage 116(saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:23:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:24 INFO DAGScheduler:59 - Submitting ResultStage 116 (MapPartitionsRDD[227] at saveAsTextFile at GLMClassificationModel.scala:61), which has no missing parents -2016-05-24 22:23:24 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=683655, maxMem=766075207 -2016-05-24 22:23:24 INFO MemoryStore:59 - Block broadcast_212 stored as values in memory (estimated size 124.0 KB, free 729.8 MB) -2016-05-24 22:23:24 INFO MemoryStore:59 - ensureFreeSpace(42575) called with curMem=810647, maxMem=766075207 -2016-05-24 22:23:24 INFO MemoryStore:59 - Block broadcast_212_piece0 stored as bytes in memory (estimated size 41.6 KB, free 729.8 MB) -2016-05-24 22:23:24 INFO BlockManagerInfo:59 - Added broadcast_212_piece0 in memory on localhost:56959 (size: 41.6 KB, free: 730.2 MB) -2016-05-24 22:23:24 INFO SparkContext:59 - Created broadcast 212 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 116 (MapPartitionsRDD[227] at saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:23:24 INFO TaskSchedulerImpl:59 - Adding task set 116.0 with 1 tasks -2016-05-24 22:23:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 116.0 (TID 223, localhost, PROCESS_LOCAL, 2197 bytes) -2016-05-24 22:23:24 INFO Executor:59 - Running task 0.0 in stage 116.0 (TID 223) -2016-05-24 22:23:24 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242223_0116_m_000000_223' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222324107/result/metadata/_temporary/0/task_201605242223_0116_m_000000 -2016-05-24 22:23:24 INFO SparkHadoopMapRedUtil:59 - attempt_201605242223_0116_m_000000_223: Committed -2016-05-24 22:23:24 INFO Executor:59 - Finished task 0.0 in stage 116.0 (TID 223). 1864 bytes result sent to driver -2016-05-24 22:23:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 116.0 (TID 223) in 132 ms on localhost (1/1) -2016-05-24 22:23:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 116.0, whose tasks have all completed, from pool -2016-05-24 22:23:24 INFO DAGScheduler:59 - ResultStage 116 (saveAsTextFile at GLMClassificationModel.scala:61) finished in 0.133 s -2016-05-24 22:23:24 INFO DAGScheduler:59 - Job 109 finished: saveAsTextFile at GLMClassificationModel.scala:61, took 0.200607 s -2016-05-24 22:23:26 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:23:26 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:23:26 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:65 -2016-05-24 22:23:26 INFO DAGScheduler:59 - Got job 110 (parquet at GLMClassificationModel.scala:65) with 1 output partitions -2016-05-24 22:23:26 INFO DAGScheduler:59 - Final stage: ResultStage 117(parquet at GLMClassificationModel.scala:65) -2016-05-24 22:23:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:26 INFO DAGScheduler:59 - Submitting ResultStage 117 (MapPartitionsRDD[229] at rddToDataFrameHolder at GLMClassificationModel.scala:65), which has no missing parents -2016-05-24 22:23:26 INFO MemoryStore:59 - ensureFreeSpace(67104) called with curMem=853222, maxMem=766075207 -2016-05-24 22:23:26 INFO MemoryStore:59 - Block broadcast_213 stored as values in memory (estimated size 65.5 KB, free 729.7 MB) -2016-05-24 22:23:26 INFO MemoryStore:59 - ensureFreeSpace(23438) called with curMem=920326, maxMem=766075207 -2016-05-24 22:23:26 INFO MemoryStore:59 - Block broadcast_213_piece0 stored as bytes in memory (estimated size 22.9 KB, free 729.7 MB) -2016-05-24 22:23:26 INFO BlockManagerInfo:59 - Added broadcast_213_piece0 in memory on localhost:56959 (size: 22.9 KB, free: 730.2 MB) -2016-05-24 22:23:26 INFO SparkContext:59 - Created broadcast 213 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 117 (MapPartitionsRDD[229] at rddToDataFrameHolder at GLMClassificationModel.scala:65) -2016-05-24 22:23:26 INFO TaskSchedulerImpl:59 - Adding task set 117.0 with 1 tasks -2016-05-24 22:23:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 117.0 (TID 224, localhost, PROCESS_LOCAL, 7978 bytes) -2016-05-24 22:23:26 INFO Executor:59 - Running task 0.0 in stage 117.0 (TID 224) -2016-05-24 22:23:26 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:23:26 INFO CodecConfig:151 - Compression: GZIP -2016-05-24 22:23:26 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-24 22:23:26 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-24 22:23:26 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-24 22:23:26 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-24 22:23:26 INFO ParquetOutputFormat:151 - Validation is off -2016-05-24 22:23:26 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-24 22:23:26 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-24 22:23:26 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 6,156 -2016-05-24 22:23:26 INFO ColumnChunkPageWriteStore:151 - written 61B for [weights, type] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [PLAIN, BIT_PACKED, RLE] -2016-05-24 22:23:26 INFO ColumnChunkPageWriteStore:151 - written 48B for [weights, size] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [PLAIN, BIT_PACKED, RLE] -2016-05-24 22:23:26 INFO ColumnChunkPageWriteStore:151 - written 51B for [weights, indices, array] INT32: 1 values, 13B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-24 22:23:26 INFO ColumnChunkPageWriteStore:151 - written 637B for [weights, values, array] DOUBLE: 692 values, 668B raw, 593B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 413 entries, 3,304B raw, 413B comp} -2016-05-24 22:23:26 INFO ColumnChunkPageWriteStore:151 - written 69B for [intercept] DOUBLE: 1 values, 14B raw, 28B comp, 1 pages, encodings: [PLAIN, BIT_PACKED, RLE] -2016-05-24 22:23:26 INFO ColumnChunkPageWriteStore:151 - written 47B for [threshold] DOUBLE: 1 values, 6B raw, 26B comp, 1 pages, encodings: [PLAIN, BIT_PACKED, RLE] -2016-05-24 22:23:26 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242223_0117_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222324107/result/data/_temporary/0/task_201605242223_0117_m_000000 -2016-05-24 22:23:26 INFO SparkHadoopMapRedUtil:59 - attempt_201605242223_0117_m_000000_0: Committed -2016-05-24 22:23:26 INFO Executor:59 - Finished task 0.0 in stage 117.0 (TID 224). 915 bytes result sent to driver -2016-05-24 22:23:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 117.0 (TID 224) in 452 ms on localhost (1/1) -2016-05-24 22:23:26 INFO DAGScheduler:59 - ResultStage 117 (parquet at GLMClassificationModel.scala:65) finished in 0.452 s -2016-05-24 22:23:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 117.0, whose tasks have all completed, from pool -2016-05-24 22:23:26 INFO DAGScheduler:59 - Job 110 finished: parquet at GLMClassificationModel.scala:65, took 0.489010 s -2016-05-24 22:23:26 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:23:26 INFO DefaultWriterContainer:59 - Job job_201605242223_0000 committed. -2016-05-24 22:23:26 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222324107/result/data on driver -2016-05-24 22:23:26 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222324107/result/data on driver -2016-05-24 22:23:26 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=943764, maxMem=766075207 -2016-05-24 22:23:26 INFO MemoryStore:59 - Block broadcast_214 stored as values in memory (estimated size 225.3 KB, free 729.5 MB) -2016-05-24 22:23:26 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1174468, maxMem=766075207 -2016-05-24 22:23:26 INFO MemoryStore:59 - Block broadcast_214_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.4 MB) -2016-05-24 22:23:26 INFO BlockManagerInfo:59 - Added broadcast_214_piece0 in memory on localhost:56959 (size: 19.3 KB, free: 730.1 MB) -2016-05-24 22:23:26 INFO SparkContext:59 - Created broadcast 214 from textFile at modelSaveLoad.scala:129 -2016-05-24 22:23:27 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:23:27 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-24 22:23:27 INFO DAGScheduler:59 - Got job 111 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-24 22:23:27 INFO DAGScheduler:59 - Final stage: ResultStage 118(first at modelSaveLoad.scala:129) -2016-05-24 22:23:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:27 INFO DAGScheduler:59 - Submitting ResultStage 118 (MapPartitionsRDD[232] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-24 22:23:27 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=1194256, maxMem=766075207 -2016-05-24 22:23:27 INFO MemoryStore:59 - Block broadcast_215 stored as values in memory (estimated size 3.1 KB, free 729.4 MB) -2016-05-24 22:23:27 INFO MemoryStore:59 - ensureFreeSpace(1898) called with curMem=1197472, maxMem=766075207 -2016-05-24 22:23:27 INFO MemoryStore:59 - Block broadcast_215_piece0 stored as bytes in memory (estimated size 1898.0 B, free 729.4 MB) -2016-05-24 22:23:27 INFO BlockManagerInfo:59 - Added broadcast_215_piece0 in memory on localhost:56959 (size: 1898.0 B, free: 730.1 MB) -2016-05-24 22:23:27 INFO SparkContext:59 - Created broadcast 215 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 118 (MapPartitionsRDD[232] at textFile at modelSaveLoad.scala:129) -2016-05-24 22:23:27 INFO TaskSchedulerImpl:59 - Adding task set 118.0 with 1 tasks -2016-05-24 22:23:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 118.0 (TID 225, localhost, PROCESS_LOCAL, 2238 bytes) -2016-05-24 22:23:27 INFO Executor:59 - Running task 0.0 in stage 118.0 (TID 225) -2016-05-24 22:23:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222324107/result/metadata/part-00000:0+54 -2016-05-24 22:23:27 INFO Executor:59 - Finished task 0.0 in stage 118.0 (TID 225). 2154 bytes result sent to driver -2016-05-24 22:23:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 118.0 (TID 225) in 11 ms on localhost (1/1) -2016-05-24 22:23:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 118.0, whose tasks have all completed, from pool -2016-05-24 22:23:27 INFO DAGScheduler:59 - ResultStage 118 (first at modelSaveLoad.scala:129) finished in 0.012 s -2016-05-24 22:23:27 INFO DAGScheduler:59 - Job 111 finished: first at modelSaveLoad.scala:129, took 0.019292 s -2016-05-24 22:23:27 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222324107/result/data on driver -2016-05-24 22:23:27 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:78 -2016-05-24 22:23:27 INFO DAGScheduler:59 - Got job 112 (parquet at GLMClassificationModel.scala:78) with 4 output partitions -2016-05-24 22:23:27 INFO DAGScheduler:59 - Final stage: ResultStage 119(parquet at GLMClassificationModel.scala:78) -2016-05-24 22:23:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:27 INFO DAGScheduler:59 - Submitting ResultStage 119 (MapPartitionsRDD[234] at parquet at GLMClassificationModel.scala:78), which has no missing parents -2016-05-24 22:23:27 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=1199370, maxMem=766075207 -2016-05-24 22:23:27 INFO MemoryStore:59 - Block broadcast_216 stored as values in memory (estimated size 61.5 KB, free 729.4 MB) -2016-05-24 22:23:27 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=1262370, maxMem=766075207 -2016-05-24 22:23:27 INFO MemoryStore:59 - Block broadcast_216_piece0 stored as bytes in memory (estimated size 20.6 KB, free 729.4 MB) -2016-05-24 22:23:27 INFO BlockManagerInfo:59 - Added broadcast_216_piece0 in memory on localhost:56959 (size: 20.6 KB, free: 730.1 MB) -2016-05-24 22:23:27 INFO SparkContext:59 - Created broadcast 216 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 119 (MapPartitionsRDD[234] at parquet at GLMClassificationModel.scala:78) -2016-05-24 22:23:27 INFO TaskSchedulerImpl:59 - Adding task set 119.0 with 4 tasks -2016-05-24 22:23:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 119.0 (TID 226, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:23:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 119.0 (TID 227, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:23:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 119.0 (TID 228, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:23:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 119.0 (TID 229, localhost, PROCESS_LOCAL, 2279 bytes) -2016-05-24 22:23:27 INFO Executor:59 - Running task 2.0 in stage 119.0 (TID 228) -2016-05-24 22:23:27 INFO Executor:59 - Running task 1.0 in stage 119.0 (TID 227) -2016-05-24 22:23:27 INFO Executor:59 - Running task 3.0 in stage 119.0 (TID 229) -2016-05-24 22:23:27 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:23:27 INFO Executor:59 - Finished task 1.0 in stage 119.0 (TID 227). 936 bytes result sent to driver -2016-05-24 22:23:27 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:23:27 INFO Executor:59 - Finished task 2.0 in stage 119.0 (TID 228). 936 bytes result sent to driver -2016-05-24 22:23:27 INFO Executor:59 - Running task 0.0 in stage 119.0 (TID 226) -2016-05-24 22:23:27 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:23:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 119.0 (TID 227) in 72 ms on localhost (1/4) -2016-05-24 22:23:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 119.0 (TID 228) in 71 ms on localhost (2/4) -2016-05-24 22:23:27 INFO Executor:59 - Finished task 3.0 in stage 119.0 (TID 229). 1802 bytes result sent to driver -2016-05-24 22:23:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 119.0 (TID 229) in 98 ms on localhost (3/4) -2016-05-24 22:23:27 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:23:27 INFO Executor:59 - Finished task 0.0 in stage 119.0 (TID 226). 936 bytes result sent to driver -2016-05-24 22:23:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 119.0 (TID 226) in 108 ms on localhost (4/4) -2016-05-24 22:23:27 INFO DAGScheduler:59 - ResultStage 119 (parquet at GLMClassificationModel.scala:78) finished in 0.108 s -2016-05-24 22:23:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 119.0, whose tasks have all completed, from pool -2016-05-24 22:23:27 INFO DAGScheduler:59 - Job 112 finished: parquet at GLMClassificationModel.scala:78, took 0.152806 s -2016-05-24 22:23:27 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=1283443, maxMem=766075207 -2016-05-24 22:23:27 INFO MemoryStore:59 - Block broadcast_217 stored as values in memory (estimated size 86.4 KB, free 729.3 MB) -2016-05-24 22:23:27 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1371955, maxMem=766075207 -2016-05-24 22:23:27 INFO MemoryStore:59 - Block broadcast_217_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.3 MB) -2016-05-24 22:23:27 INFO BlockManagerInfo:59 - Added broadcast_217_piece0 in memory on localhost:56959 (size: 19.3 KB, free: 730.1 MB) -2016-05-24 22:23:27 INFO SparkContext:59 - Created broadcast 217 from take at GLMClassificationModel.scala:79 -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_216_piece0 on localhost:56959 in memory (size: 20.6 KB, free: 730.1 MB) -2016-05-24 22:23:28 INFO ContextCleaner:59 - Cleaned accumulator 115 -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_215_piece0 on localhost:56959 in memory (size: 1898.0 B, free: 730.1 MB) -2016-05-24 22:23:28 INFO ContextCleaner:59 - Cleaned accumulator 114 -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_214_piece0 on localhost:56959 in memory (size: 19.3 KB, free: 730.1 MB) -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_213_piece0 on localhost:56959 in memory (size: 22.9 KB, free: 730.2 MB) -2016-05-24 22:23:28 INFO ContextCleaner:59 - Cleaned accumulator 113 -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_212_piece0 on localhost:56959 in memory (size: 41.6 KB, free: 730.2 MB) -2016-05-24 22:23:28 INFO ContextCleaner:59 - Cleaned accumulator 112 -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_211_piece0 on localhost:56959 in memory (size: 5.9 KB, free: 730.2 MB) -2016-05-24 22:23:28 INFO ContextCleaner:59 - Cleaned accumulator 111 -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_210_piece0 on localhost:56959 in memory (size: 2.9 KB, free: 730.2 MB) -2016-05-24 22:23:28 INFO ContextCleaner:59 - Cleaned accumulator 110 -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_209_piece0 on localhost:56959 in memory (size: 2.7 KB, free: 730.2 MB) -2016-05-24 22:23:28 INFO ContextCleaner:59 - Cleaned accumulator 109 -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_208_piece0 on localhost:56959 in memory (size: 1981.0 B, free: 730.2 MB) -2016-05-24 22:23:28 INFO ContextCleaner:59 - Cleaned accumulator 108 -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_207_piece0 on localhost:56959 in memory (size: 1658.0 B, free: 730.2 MB) -2016-05-24 22:23:28 INFO ContextCleaner:59 - Cleaned accumulator 107 -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_206_piece0 on localhost:56959 in memory (size: 1900.0 B, free: 730.2 MB) -2016-05-24 22:23:28 INFO ContextCleaner:59 - Cleaned accumulator 106 -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_205_piece0 on localhost:56959 in memory (size: 6.3 KB, free: 730.2 MB) -2016-05-24 22:23:28 INFO ContextCleaner:59 - Cleaned accumulator 105 -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_204_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:28 INFO ContextCleaner:59 - Cleaned accumulator 104 -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_203_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_202_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:28 INFO ContextCleaner:59 - Cleaned accumulator 103 -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_201_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_200_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:23:28 INFO ContextCleaner:59 - Cleaned accumulator 102 -2016-05-24 22:23:28 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_199_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:28 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222324107/result/data/part-r-00000-ced5f951-2d85-431c-b72f-19bc855bac49.gz.parquet -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_198_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:28 INFO ContextCleaner:59 - Cleaned accumulator 101 -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_197_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:28 INFO SparkContext:59 - Starting job: take at GLMClassificationModel.scala:79 -2016-05-24 22:23:28 INFO DAGScheduler:59 - Got job 113 (take at GLMClassificationModel.scala:79) with 1 output partitions -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_196_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:28 INFO DAGScheduler:59 - Final stage: ResultStage 120(take at GLMClassificationModel.scala:79) -2016-05-24 22:23:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:23:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:23:28 INFO DAGScheduler:59 - Submitting ResultStage 120 (MapPartitionsRDD[236] at take at GLMClassificationModel.scala:79), which has no missing parents -2016-05-24 22:23:28 INFO ContextCleaner:59 - Cleaned accumulator 100 -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_195_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_142_piece0 on localhost:56959 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:23:28 INFO ContextCleaner:59 - Cleaned accumulator 73 -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_141_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Removed broadcast_87_piece0 on localhost:56959 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:23:28 INFO MemoryStore:59 - ensureFreeSpace(4168) called with curMem=570248, maxMem=766075207 -2016-05-24 22:23:28 INFO MemoryStore:59 - Block broadcast_218 stored as values in memory (estimated size 4.1 KB, free 730.0 MB) -2016-05-24 22:23:28 INFO MemoryStore:59 - ensureFreeSpace(2438) called with curMem=574416, maxMem=766075207 -2016-05-24 22:23:28 INFO MemoryStore:59 - Block broadcast_218_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.0 MB) -2016-05-24 22:23:28 INFO BlockManagerInfo:59 - Added broadcast_218_piece0 in memory on localhost:56959 (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:23:28 INFO SparkContext:59 - Created broadcast 218 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:23:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 120 (MapPartitionsRDD[236] at take at GLMClassificationModel.scala:79) -2016-05-24 22:23:28 INFO TaskSchedulerImpl:59 - Adding task set 120.0 with 1 tasks -2016-05-24 22:23:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 120.0 (TID 230, localhost, PROCESS_LOCAL, 2302 bytes) -2016-05-24 22:23:28 INFO Executor:59 - Running task 0.0 in stage 120.0 (TID 230) -2016-05-24 22:23:28 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222324107/result/data/part-r-00000-ced5f951-2d85-431c-b72f-19bc855bac49.gz.parquet start: 0 end: 5410 length: 5410 hosts: []} -2016-05-24 22:23:28 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-24 22:23:28 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group weights { - required int32 type (INT_8); - optional int32 size; - optional group indices (LIST) { - repeated int32 array; - } - optional group values (LIST) { - repeated double array; - } - } - optional double intercept; - optional double threshold; -} - -Catalyst form: -StructType(StructField(weights,org.apache.spark.mllib.linalg.VectorUDT@f71b0bce,true), StructField(intercept,DoubleType,true), StructField(threshold,DoubleType,true)) - -2016-05-24 22:23:29 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-24 22:23:29 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-24 22:23:29 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-24 22:23:29 INFO InternalParquetRecordReader:151 - block read in memory in 9 ms. row count = 1 -2016-05-24 22:24:06 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 22:24:07 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 22:24:07 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 22:24:07 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 22:24:07 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 22:24:08 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 22:24:08 INFO Remoting:74 - Starting remoting -2016-05-24 22:24:08 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:56980] -2016-05-24 22:24:08 INFO Utils:59 - Successfully started service 'sparkDriver' on port 56980. -2016-05-24 22:24:08 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 22:24:08 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 22:24:08 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-d8035bf0-3c7e-44d7-a6a4-2250a3af4690 -2016-05-24 22:24:08 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 22:24:08 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-c1803529-1664-499f-b67d-4ef72f7bfc76\httpd-84267416-048d-46c2-9f9f-a95d0b457d09 -2016-05-24 22:24:08 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 22:24:08 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:24:08 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:56981 -2016-05-24 22:24:08 INFO Utils:59 - Successfully started service 'HTTP file server' on port 56981. -2016-05-24 22:24:08 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 22:24:09 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:24:09 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 22:24:09 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 22:24:09 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 22:24:09 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 22:24:09 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 22:24:09 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 57000. -2016-05-24 22:24:09 INFO NettyBlockTransferService:59 - Server created on 57000 -2016-05-24 22:24:09 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 22:24:09 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:57000 with 730.6 MB RAM, BlockManagerId(driver, localhost, 57000) -2016-05-24 22:24:09 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 22:24:10 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 22:24:10 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 22:24:10 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 22:24:10 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 22:24:10 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:57000 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 22:24:10 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 22:24:11 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 22:24:11 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:24:12 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 22:24:12 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 22:24:12 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 22:24:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:12 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 22:24:12 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 22:24:12 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 22:24:12 INFO MemoryStore:59 - ensureFreeSpace(2149) called with curMem=175476, maxMem=766075207 -2016-05-24 22:24:12 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:24:12 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:57000 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:24:12 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 22:24:12 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 22:24:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:24:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:24:12 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 22:24:12 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 22:24:12 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 22:24:12 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 22:24:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 22:24:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 22:24:12 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 22:24:12 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 22:24:12 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 22:24:12 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 22:24:12 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 22:24:12 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177625, maxMem=766075207 -2016-05-24 22:24:12 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 22:24:12 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:57000 (size: 83.1 KB, free: 730.5 MB) -2016-05-24 22:24:12 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262705, maxMem=766075207 -2016-05-24 22:24:12 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 22:24:12 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:57000 (size: 83.6 KB, free: 730.4 MB) -2016-05-24 22:24:12 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 22:24:12 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 22:24:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 382 ms on localhost (1/2) -2016-05-24 22:24:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 407 ms on localhost (2/2) -2016-05-24 22:24:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 22:24:12 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.428 s -2016-05-24 22:24:12 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.581096 s -2016-05-24 22:24:12 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 22:24:12 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 22:24:12 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:24:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:12 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 22:24:12 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=348353, maxMem=766075207 -2016-05-24 22:24:12 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-24 22:24:12 INFO MemoryStore:59 - ensureFreeSpace(2487) called with curMem=352913, maxMem=766075207 -2016-05-24 22:24:12 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-24 22:24:12 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:57000 (size: 2.4 KB, free: 730.4 MB) -2016-05-24 22:24:12 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:12 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:24:12 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 22:24:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:24:12 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 22:24:12 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-05-24 22:24:12 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:24:12 INFO MemoryStore:59 - ensureFreeSpace(54392) called with curMem=355400, maxMem=766075207 -2016-05-24 22:24:12 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 53.1 KB, free 730.2 MB) -2016-05-24 22:24:12 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:57000 (size: 53.1 KB, free: 730.4 MB) -2016-05-24 22:24:12 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2611 bytes result sent to driver -2016-05-24 22:24:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 41 ms on localhost (1/1) -2016-05-24 22:24:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 22:24:12 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.042 s -2016-05-24 22:24:12 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.075814 s -2016-05-24 22:24:12 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 22:24:12 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 22:24:12 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 22:24:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:12 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 22:24:12 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=409792, maxMem=766075207 -2016-05-24 22:24:12 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-24 22:24:12 INFO MemoryStore:59 - ensureFreeSpace(2304) called with curMem=414040, maxMem=766075207 -2016-05-24 22:24:12 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-24 22:24:12 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:57000 (size: 2.3 KB, free: 730.4 MB) -2016-05-24 22:24:12 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40) -2016-05-24 22:24:12 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 22:24:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:24:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:24:12 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 22:24:12 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 22:24:12 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-05-24 22:24:12 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:24:12 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:12 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 22:24:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 17 ms on localhost (1/2) -2016-05-24 22:24:12 INFO MemoryStore:59 - ensureFreeSpace(62184) called with curMem=416344, maxMem=766075207 -2016-05-24 22:24:12 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 60.7 KB, free 730.1 MB) -2016-05-24 22:24:12 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:57000 (size: 60.7 KB, free: 730.3 MB) -2016-05-24 22:24:12 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2662 bytes result sent to driver -2016-05-24 22:24:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 24 ms on localhost (2/2) -2016-05-24 22:24:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 22:24:12 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.027 s -2016-05-24 22:24:12 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.042284 s -2016-05-24 22:24:12 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 22:24:12 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 22:24:12 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 22:24:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:12 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 22:24:12 INFO MemoryStore:59 - ensureFreeSpace(4264) called with curMem=478528, maxMem=766075207 -2016-05-24 22:24:12 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.2 KB, free 730.1 MB) -2016-05-24 22:24:12 INFO MemoryStore:59 - ensureFreeSpace(2326) called with curMem=482792, maxMem=766075207 -2016-05-24 22:24:12 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 22:24:12 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:57000 (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:24:12 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 22:24:12 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 22:24:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:24:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:24:12 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 22:24:12 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 22:24:12 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:12 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:12 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 22:24:12 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 22:24:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 17 ms on localhost (1/2) -2016-05-24 22:24:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 17 ms on localhost (2/2) -2016-05-24 22:24:12 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.018 s -2016-05-24 22:24:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 22:24:12 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.037144 s -2016-05-24 22:24:13 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 22:24:13 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=485118, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(388) called with curMem=490710, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 388.0 B, free 730.1 MB) -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:57000 (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:24:13 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:13 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:13 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:13 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:13 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=491098, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(3486) called with curMem=502882, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:13 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 22:24:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:13 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 22:24:13 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 22:24:13 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:13 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:13 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 7968 bytes result sent to driver -2016-05-24 22:24:13 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 7968 bytes result sent to driver -2016-05-24 22:24:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 26 ms on localhost (1/2) -2016-05-24 22:24:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 24 ms on localhost (2/2) -2016-05-24 22:24:13 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.027 s -2016-05-24 22:24:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 22:24:13 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.046920 s -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=506368, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=511960, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:57000 (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:13 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:13 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:13 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:13 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=514198, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(3486) called with curMem=525982, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:13 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 22:24:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:13 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 22:24:13 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 22:24:13 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:13 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:13 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 7968 bytes result sent to driver -2016-05-24 22:24:13 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 7968 bytes result sent to driver -2016-05-24 22:24:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 19 ms on localhost (1/2) -2016-05-24 22:24:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 19 ms on localhost (2/2) -2016-05-24 22:24:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 22:24:13 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 22:24:13 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.045456 s -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=529468, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=535060, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:13 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:13 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:13 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:13 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=538669, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=550453, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:13 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 22:24:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:13 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 22:24:13 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 22:24:13 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:13 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:13 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 7968 bytes result sent to driver -2016-05-24 22:24:13 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 7968 bytes result sent to driver -2016-05-24 22:24:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 18 ms on localhost (1/2) -2016-05-24 22:24:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 20 ms on localhost (2/2) -2016-05-24 22:24:13 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:24:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 22:24:13 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.038397 s -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=553940, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(3603) called with curMem=559532, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:13 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:13 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:13 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:13 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=563135, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(3486) called with curMem=574919, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:13 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 22:24:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:13 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 22:24:13 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 22:24:13 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:13 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:13 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 7968 bytes result sent to driver -2016-05-24 22:24:13 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 7968 bytes result sent to driver -2016-05-24 22:24:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 40 ms on localhost (1/2) -2016-05-24 22:24:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 44 ms on localhost (2/2) -2016-05-24 22:24:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 22:24:13 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.046 s -2016-05-24 22:24:13 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 22:24:13 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.065834 s -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=578405, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=583997, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:57000 in memory (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:57000 in memory (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:24:13 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:13 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:13 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:57000 in memory (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:13 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 22:24:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:13 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:57000 in memory (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=549629, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(3485) called with curMem=550118, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:57000 in memory (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:13 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:13 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 22:24:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:13 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 22:24:13 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:13 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:13 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 7968 bytes result sent to driver -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 7968 bytes result sent to driver -2016-05-24 22:24:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 21 ms on localhost (1/2) -2016-05-24 22:24:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 19 ms on localhost (2/2) -2016-05-24 22:24:13 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 22:24:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 22:24:13 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.040385 s -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=513861, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=519453, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:13 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:13 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:13 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:13 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=523059, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=534843, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:13 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 22:24:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:13 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 22:24:13 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 22:24:13 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:13 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:13 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 7968 bytes result sent to driver -2016-05-24 22:24:13 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 7968 bytes result sent to driver -2016-05-24 22:24:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 15 ms on localhost (1/2) -2016-05-24 22:24:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 15 ms on localhost (2/2) -2016-05-24 22:24:13 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:24:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 22:24:13 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.033895 s -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=538331, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=543923, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:13 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:13 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:13 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:13 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=547529, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=559313, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:13 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 22:24:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:13 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 22:24:13 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 22:24:13 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:13 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:13 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 7968 bytes result sent to driver -2016-05-24 22:24:13 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 7968 bytes result sent to driver -2016-05-24 22:24:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 19 ms on localhost (1/2) -2016-05-24 22:24:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 18 ms on localhost (2/2) -2016-05-24 22:24:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 22:24:13 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:24:13 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.038370 s -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=562800, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(3604) called with curMem=568392, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:13 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:13 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:13 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:13 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=571996, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:24:13 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=583780, maxMem=766075207 -2016-05-24 22:24:13 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:24:13 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:13 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:13 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 22:24:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:13 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 22:24:13 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 22:24:13 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:13 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:13 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 7968 bytes result sent to driver -2016-05-24 22:24:13 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 7968 bytes result sent to driver -2016-05-24 22:24:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 11 ms on localhost (1/2) -2016-05-24 22:24:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 11 ms on localhost (2/2) -2016-05-24 22:24:13 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:24:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 22:24:13 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.027074 s -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=587267, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=592859, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:14 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=596465, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=608249, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 22:24:14 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:14 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:14 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 11 ms on localhost (1/2) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 14 ms on localhost (2/2) -2016-05-24 22:24:14 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 22:24:14 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.026080 s -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=611736, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=617328, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:14 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=620934, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=632718, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 22:24:14 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:14 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 13 ms on localhost (1/2) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 13 ms on localhost (2/2) -2016-05-24 22:24:14 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 22:24:14 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.027607 s -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=636205, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3610) called with curMem=641797, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:14 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=645407, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=657191, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 22:24:14 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:14 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:14 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 14 ms on localhost (1/2) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 16 ms on localhost (2/2) -2016-05-24 22:24:14 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 22:24:14 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.029548 s -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=660679, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=666271, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:14 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=669877, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=681661, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 22:24:14 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:14 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 16 ms on localhost (1/2) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 16 ms on localhost (2/2) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 22:24:14 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:24:14 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.030713 s -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=685148, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3610) called with curMem=690740, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:14 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=694350, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=706134, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 22:24:14 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:14 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 13 ms on localhost (1/2) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 16 ms on localhost (2/2) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 22:24:14 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:24:14 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.033249 s -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=709621, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=715213, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:14 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=718819, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=730603, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 22:24:14 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:14 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 16 ms on localhost (1/2) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 17 ms on localhost (2/2) -2016-05-24 22:24:14 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 22:24:14 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.044299 s -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=734090, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=739682, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:14 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=743288, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3486) called with curMem=755072, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 22:24:14 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:14 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 13 ms on localhost (1/2) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 13 ms on localhost (2/2) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 22:24:14 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:24:14 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.031409 s -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=758558, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=764150, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:14 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=767763, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=779547, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 22:24:14 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:14 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 15 ms on localhost (1/2) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 14 ms on localhost (2/2) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 22:24:14 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:24:14 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.034405 s -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=783034, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=788626, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:14 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=792232, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=804016, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 22:24:14 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:14 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 16 ms on localhost (1/2) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 16 ms on localhost (2/2) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 22:24:14 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 22:24:14 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.032998 s -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=807504, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=813096, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:14 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=816708, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=828492, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 22:24:14 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:14 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 12 ms on localhost (1/2) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 14 ms on localhost (2/2) -2016-05-24 22:24:14 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 22:24:14 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.031839 s -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=831979, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3621) called with curMem=837571, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:14 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=841192, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=852976, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 22:24:14 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:14 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 13 ms on localhost (1/2) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 13 ms on localhost (2/2) -2016-05-24 22:24:14 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 22:24:14 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.028294 s -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=856464, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=862056, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 43 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Got job 23 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:14 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=865668, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=877452, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[51] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 22:24:14 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:14 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 11 ms on localhost (1/2) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 12 ms on localhost (2/2) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 22:24:14 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:24:14 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at GradientDescent.scala:225, took 0.028961 s -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=880939, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=886531, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 45 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Got job 24 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:14 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=890143, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=901927, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[53] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 22:24:14 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:14 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 22:24:14 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 11 ms on localhost (1/2) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 13 ms on localhost (2/2) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 22:24:14 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:24:14 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at GradientDescent.scala:225, took 0.063536 s -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=880939, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=886531, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 47 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Got job 25 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:14 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=874873, maxMem=766075207 -2016-05-24 22:24:14 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3486) called with curMem=883036, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[55] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 22:24:14 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:14 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 12 ms on localhost (1/2) -2016-05-24 22:24:14 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 12 ms on localhost (2/2) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 22:24:14 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:24:14 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at GradientDescent.scala:225, took 0.030689 s -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=862047, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=862047, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 49 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Got job 26 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:14 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:14 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=825920, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3485) called with curMem=828499, maxMem=766075207 -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[57] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 22:24:14 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:14 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 10 ms on localhost (1/2) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 12 ms on localhost (2/2) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:24:14 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at GradientDescent.scala:225, took 0.033582 s -2016-05-24 22:24:14 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=783047, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3634) called with curMem=788639, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 51 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:14 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 22:24:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Got job 27 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:14 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=728059, maxMem=766075207 -2016-05-24 22:24:14 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=736233, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[59] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 22:24:14 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 22:24:14 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:14 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 11 ms on localhost (1/2) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 11 ms on localhost (2/2) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 22:24:14 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:24:14 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at GradientDescent.scala:225, took 0.026263 s -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=694385, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=696371, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 53 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Got job 28 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:14 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:14 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=654653, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(3485) called with curMem=662831, maxMem=766075207 -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[61] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:14 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 22:24:14 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:14 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:14 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 7968 bytes result sent to driver -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 10 ms on localhost (1/2) -2016-05-24 22:24:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 10 ms on localhost (2/2) -2016-05-24 22:24:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 22:24:14 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 22:24:14 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:24:14 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at GradientDescent.scala:225, took 0.026852 s -2016-05-24 22:24:14 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=641846, maxMem=766075207 -2016-05-24 22:24:14 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=641846, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 55 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:15 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 29 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 29(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=596527, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=608311, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[63] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 8 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 9 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 29 (treeAggregate at GradientDescent.scala:225) finished in 0.010 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 29 finished: treeAggregate at GradientDescent.scala:225, took 0.021643 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=611799, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=617391, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 57 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 30 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 30(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=621003, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=632787, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[65] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 6 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 7 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 30 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 30 finished: treeAggregate at GradientDescent.scala:225, took 0.015503 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=636275, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=641867, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 59 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 31 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 31(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=645473, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=657257, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[67] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 7 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 8 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 31 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 31 finished: treeAggregate at GradientDescent.scala:225, took 0.017332 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=660744, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3623) called with curMem=666336, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 61 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 32 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 32(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=669959, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=681743, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[69] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 7 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 8 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 32 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 32 finished: treeAggregate at GradientDescent.scala:225, took 0.017956 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=685231, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=690823, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 63 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 33 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 33(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=694429, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3486) called with curMem=706213, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[71] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 6 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 7 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 33 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 33 finished: treeAggregate at GradientDescent.scala:225, took 0.016571 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=709699, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3626) called with curMem=715291, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 65 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 34 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 34(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=718917, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=730701, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[73] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 8 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 10 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 34 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 34 finished: treeAggregate at GradientDescent.scala:225, took 0.022763 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=734189, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=739781, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 67 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 35 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 35(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=743393, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=755177, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[75] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 5 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 6 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 35 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 35 finished: treeAggregate at GradientDescent.scala:225, took 0.014448 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=758664, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=764256, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 69 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 36 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 36(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=767868, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=779652, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[77] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 7 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 8 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 36 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 36 finished: treeAggregate at GradientDescent.scala:225, took 0.020551 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=783139, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=788731, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 71 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 37 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 37(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=792343, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=804127, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[79] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 6 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 6 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 37 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 37 finished: treeAggregate at GradientDescent.scala:225, took 0.015502 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=807615, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=813207, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 73 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 38 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 38(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=816819, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=828603, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[81] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 6 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 7 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 38 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 38 finished: treeAggregate at GradientDescent.scala:225, took 0.017894 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=832091, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=837683, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 75 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 39 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 39(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=841295, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=853079, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[83] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 7 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 7 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 39 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 39 finished: treeAggregate at GradientDescent.scala:225, took 0.017333 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=856567, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=862159, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 77 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 40 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 40(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=865771, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=877555, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[85] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 8 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 8 ms on localhost (2/2) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 40 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 40 finished: treeAggregate at GradientDescent.scala:225, took 0.019890 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=881043, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=886635, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 79 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 41 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 41(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=890249, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=902033, maxMem=766075207 -2016-05-24 22:24:15 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[87] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 10 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 10 ms on localhost (2/2) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 41 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 41 finished: treeAggregate at GradientDescent.scala:225, took 0.036518 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=856564, maxMem=766075207 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=852952, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 81 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 42 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 42(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=807614, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=804130, maxMem=766075207 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[89] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 8 ms on localhost (1/2) -2016-05-24 22:24:15 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 42 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 42 finished: treeAggregate at GradientDescent.scala:225, took 0.020245 s -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 9 ms on localhost (2/2) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=773916, maxMem=766075207 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=776022, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 83 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 43 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 43(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=709693, maxMem=766075207 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3485) called with curMem=717989, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[91] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-05-24 22:24:15 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 8 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 8 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 43 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 43 finished: treeAggregate at GradientDescent.scala:225, took 0.021178 s -2016-05-24 22:24:15 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=679622, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=681602, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 85 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 44 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 44(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[93] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=636263, maxMem=766075207 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=632776, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:15 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[93] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 8 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 9 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 44 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 44 finished: treeAggregate at GradientDescent.scala:225, took 0.019215 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=602569, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=592891, maxMem=766075207 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 87 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 45 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 45(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[95] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=572033, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=583817, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[95] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 6 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 7 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 45 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 45 finished: treeAggregate at GradientDescent.scala:225, took 0.016088 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=587304, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=592896, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 89 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 46 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 46(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[97] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=596509, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=608293, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[97] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 6 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 7 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 46 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 46 finished: treeAggregate at GradientDescent.scala:225, took 0.018244 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=611780, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3618) called with curMem=617372, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 91 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 47 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 47(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[99] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=620990, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=632774, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 47 (MapPartitionsRDD[99] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 93, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 94, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 93) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 94) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 94). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 93). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 94) in 5 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 93) in 6 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 47 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 47 finished: treeAggregate at GradientDescent.scala:225, took 0.014176 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=636262, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=641854, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 93 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 48 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 48(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[101] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=645468, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3486) called with curMem=657252, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 48 (MapPartitionsRDD[101] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 95, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 96, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 95) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 96) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 96). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 95). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 96) in 5 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 95) in 6 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 48 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 48 finished: treeAggregate at GradientDescent.scala:225, took 0.015794 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=660738, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3618) called with curMem=666330, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 95 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 49 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 49(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[103] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=669948, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=681732, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 49 (MapPartitionsRDD[103] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 97, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 98, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 98) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 97) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 97). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 98). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 97) in 6 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 98) in 6 ms on localhost (2/2) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 49 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 49 finished: treeAggregate at GradientDescent.scala:225, took 0.016051 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=685219, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3622) called with curMem=690811, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 97 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 50 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 50(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[105] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=694433, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3485) called with curMem=706217, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 50 (MapPartitionsRDD[105] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 7 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 8 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 50 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 50 finished: treeAggregate at GradientDescent.scala:225, took 0.019498 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=709702, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=715294, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 99 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 51 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 51(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[107] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=718907, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3486) called with curMem=730691, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 51 (MapPartitionsRDD[107] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 6 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 7 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 51 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 51 finished: treeAggregate at GradientDescent.scala:225, took 0.018193 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=734177, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3628) called with curMem=739769, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 101 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 52 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 52(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 52 (MapPartitionsRDD[109] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=743397, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3486) called with curMem=755181, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 52 (MapPartitionsRDD[109] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 103, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 104, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 103) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 104) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 103). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 104). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 103) in 6 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 104) in 6 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 52 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 52 finished: treeAggregate at GradientDescent.scala:225, took 0.018889 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=758667, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=764259, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 103 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 53 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 53(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[111] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=767891, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3485) called with curMem=779675, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 104 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[111] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 105, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 106, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 105) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 106) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 106). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 105). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 106) in 5 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 105) in 5 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 53 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 53 finished: treeAggregate at GradientDescent.scala:225, took 0.014259 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=783160, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=788752, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 105 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 54 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 54(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[113] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=792365, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=804149, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 106 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 54 (MapPartitionsRDD[113] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 107) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 108) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 108). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 107). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 107) in 4 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 108) in 5 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 54 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 54 finished: treeAggregate at GradientDescent.scala:225, took 0.013895 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=807637, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=813229, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 107 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 55 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 55(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[115] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=816861, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3485) called with curMem=828645, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 108 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 55 (MapPartitionsRDD[115] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 109, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 110, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 109) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 110) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 109). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 110). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 110) in 5 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 109) in 6 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 55 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 55 finished: treeAggregate at GradientDescent.scala:225, took 0.014516 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=832130, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=837722, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 109 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 56 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 56(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[117] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=841335, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=853119, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 110 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 56 (MapPartitionsRDD[117] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 111, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 112, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 111) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 112) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 111). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 112). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 111) in 7 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 112) in 7 ms on localhost (2/2) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 56 (treeAggregate at GradientDescent.scala:225) finished in 0.008 s -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 56 finished: treeAggregate at GradientDescent.scala:225, took 0.023102 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=856606, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=862198, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 111 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 57 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 57(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[119] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=865811, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3485) called with curMem=877595, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 112 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 57 (MapPartitionsRDD[119] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 113, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 114, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 113) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 114) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 114). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 113). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 114) in 4 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 113) in 5 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 57 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 57 finished: treeAggregate at GradientDescent.scala:225, took 0.014997 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=881080, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=886672, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 113 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 58 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 58(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 58 (MapPartitionsRDD[121] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=890284, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=902068, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 114 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 58 (MapPartitionsRDD[121] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 115, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 116, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 116) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 115) -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 116). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 116) in 6 ms on localhost (1/2) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 115). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 115) in 9 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 58 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 58 finished: treeAggregate at GradientDescent.scala:225, took 0.019368 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=905555, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_115 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3620) called with curMem=911147, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_115_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_115_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 115 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:15 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Got job 59 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:15 INFO DAGScheduler:59 - Final stage: ResultStage 59(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting ResultStage 59 (MapPartitionsRDD[123] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=914767, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_116 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=926551, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_116_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:24:15 INFO BlockManagerInfo:59 - Added broadcast_116_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:15 INFO SparkContext:59 - Created broadcast 116 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 59 (MapPartitionsRDD[123] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 2 tasks -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 117, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 118, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:15 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 117) -2016-05-24 22:24:15 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 118) -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:15 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:15 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 117). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 118). 7968 bytes result sent to driver -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 117) in 5 ms on localhost (1/2) -2016-05-24 22:24:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 118) in 5 ms on localhost (2/2) -2016-05-24 22:24:15 INFO DAGScheduler:59 - ResultStage 59 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:24:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-05-24 22:24:15 INFO DAGScheduler:59 - Job 59 finished: treeAggregate at GradientDescent.scala:225, took 0.015221 s -2016-05-24 22:24:15 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=930034, maxMem=766075207 -2016-05-24 22:24:15 INFO MemoryStore:59 - Block broadcast_117 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=935626, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_117_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_117_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 117 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 60 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 60(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 60 (MapPartitionsRDD[125] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=939238, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_118 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=951022, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_118_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_118_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 118 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 60 (MapPartitionsRDD[125] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 60.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 60.0 (TID 119, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 60.0 (TID 120, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 60.0 (TID 119) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 60.0 (TID 120) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 60.0 (TID 119). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 60.0 (TID 120). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 60.0 (TID 119) in 5 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 60.0 (TID 120) in 5 ms on localhost (2/2) -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 60 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 60.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 60 finished: treeAggregate at GradientDescent.scala:225, took 0.014136 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=954510, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_119 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3616) called with curMem=960102, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_119_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_119_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 119 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 61 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 61(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 61 (MapPartitionsRDD[127] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=963718, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_120 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=975502, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_120_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_120_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 120 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 61 (MapPartitionsRDD[127] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 121, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 122, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 122) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 121) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 121). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 122). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 121) in 4 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 122) in 5 ms on localhost (2/2) -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 61 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 61 finished: treeAggregate at GradientDescent.scala:225, took 0.015135 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=978990, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_121 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=984582, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_121_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_121_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 121 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 62 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 62(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[129] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=988194, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_122 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=999978, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_122_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_122_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 122 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 62 (MapPartitionsRDD[129] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 123, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 62.0 (TID 124, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 123) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 62.0 (TID 124) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 62.0 (TID 124). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 123). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 62.0 (TID 124) in 5 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 123) in 5 ms on localhost (2/2) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 62 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 62 finished: treeAggregate at GradientDescent.scala:225, took 0.015158 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1003465, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_123 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3630) called with curMem=1009057, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_123_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_123_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 123 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 63 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 63(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 63 (MapPartitionsRDD[131] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1012687, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_124 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3485) called with curMem=1024471, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_124_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_124_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 124 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 63 (MapPartitionsRDD[131] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 63.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 63.0 (TID 125, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 63.0 (TID 126, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 63.0 (TID 126) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 63.0 (TID 125) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 63.0 (TID 125). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 63.0 (TID 126). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 63.0 (TID 125) in 6 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 63.0 (TID 126) in 6 ms on localhost (2/2) -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 63 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 63 finished: treeAggregate at GradientDescent.scala:225, took 0.016047 s -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 63.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1027956, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_125 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1033548, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_125_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_125_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 125 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 64 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 64(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 64 (MapPartitionsRDD[133] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1037161, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_126 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3485) called with curMem=1048945, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_126_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_126_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 126 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 64 (MapPartitionsRDD[133] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 64.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 64.0 (TID 127, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 64.0 (TID 128, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 64.0 (TID 127) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 64.0 (TID 128) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 64.0 (TID 127). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 64.0 (TID 128). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 64.0 (TID 127) in 4 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 64.0 (TID 128) in 5 ms on localhost (2/2) -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 64 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 64.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 64 finished: treeAggregate at GradientDescent.scala:225, took 0.013835 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1052430, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_127 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1058022, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_127_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_127_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 127 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 65 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 65(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[135] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1061634, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_128 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=1073418, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_128_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_128_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 128 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 65 (MapPartitionsRDD[135] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 129, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 130, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 130) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 129) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 130). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 129). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 130) in 5 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 129) in 6 ms on localhost (2/2) -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 65 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 65 finished: treeAggregate at GradientDescent.scala:225, took 0.015309 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1076905, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_129 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1082497, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_129_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_129_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 129 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 66 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 66(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 66 (MapPartitionsRDD[137] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1086110, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_130 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=1097894, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_130_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_130_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 130 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 66 (MapPartitionsRDD[137] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 66.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 66.0 (TID 131, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 66.0 (TID 132, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 66.0 (TID 131) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 66.0 (TID 132) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 66.0 (TID 132). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 66.0 (TID 131). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 66.0 (TID 132) in 4 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 66.0 (TID 131) in 5 ms on localhost (2/2) -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 66 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 66.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 66 finished: treeAggregate at GradientDescent.scala:225, took 0.014580 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1101382, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_131 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3633) called with curMem=1106974, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_131_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_131_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 131 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 67 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 67(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 67 (MapPartitionsRDD[139] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1110607, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_132 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=1122391, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_132_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_132_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 132 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 67 (MapPartitionsRDD[139] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 67.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 67.0 (TID 133, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 67.0 (TID 134, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 67.0 (TID 133) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 67.0 (TID 134) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 67.0 (TID 133). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 67.0 (TID 134). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 67.0 (TID 133) in 5 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 67.0 (TID 134) in 5 ms on localhost (2/2) -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 67 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 67.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 67 finished: treeAggregate at GradientDescent.scala:225, took 0.014283 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1125878, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_133 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1131470, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_133_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_133_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 133 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 68 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 68(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 68 (MapPartitionsRDD[141] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1135083, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_134 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=1146867, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_134_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_134_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 134 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 68 (MapPartitionsRDD[141] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 68.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 68.0 (TID 135, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 68.0 (TID 136, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 68.0 (TID 136) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 68.0 (TID 135) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 68.0 (TID 136). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 68.0 (TID 135). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 68.0 (TID 136) in 5 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 68.0 (TID 135) in 6 ms on localhost (2/2) -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 68 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 68.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 68 finished: treeAggregate at GradientDescent.scala:225, took 0.014883 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1150355, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_135 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3608) called with curMem=1155947, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_135_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_135_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 135 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 69 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 69(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 69 (MapPartitionsRDD[143] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1159555, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_136 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=1171339, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_136_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_136_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 136 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 69 (MapPartitionsRDD[143] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 69.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 69.0 (TID 137, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 69.0 (TID 138, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 69.0 (TID 137) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 69.0 (TID 138) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 69.0 (TID 138). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 69.0 (TID 137). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 69.0 (TID 138) in 4 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 69.0 (TID 137) in 5 ms on localhost (2/2) -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 69 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 69.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 69 finished: treeAggregate at GradientDescent.scala:225, took 0.015873 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1174826, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_137 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1180418, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_137_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_137_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 137 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 70 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 70(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 70 (MapPartitionsRDD[145] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1184024, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_138 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3486) called with curMem=1195808, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_138_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_138_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 138 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 70 (MapPartitionsRDD[145] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 70.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 70.0 (TID 139, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 70.0 (TID 140, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 70.0 (TID 140) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 70.0 (TID 139) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 70.0 (TID 139). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 70.0 (TID 140). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 70.0 (TID 139) in 5 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 70.0 (TID 140) in 5 ms on localhost (2/2) -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 70 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 70.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 70 finished: treeAggregate at GradientDescent.scala:225, took 0.016167 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1199294, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_139 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3619) called with curMem=1204886, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_139_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_139_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 139 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 71 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 71(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 71 (MapPartitionsRDD[147] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1208505, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_140 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=1220289, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_140_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_140_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 140 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 71 (MapPartitionsRDD[147] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 71.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 71.0 (TID 141, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 71.0 (TID 142, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 71.0 (TID 141) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 71.0 (TID 142) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 71.0 (TID 142). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 71.0 (TID 141). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 71.0 (TID 142) in 4 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 71.0 (TID 141) in 5 ms on localhost (2/2) -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 71 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 71.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 71 finished: treeAggregate at GradientDescent.scala:225, took 0.013784 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1223776, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_141 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3604) called with curMem=1229368, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_141_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_141_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 141 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 72 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 72(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 72 (MapPartitionsRDD[149] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1232972, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_142 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=1244756, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_142_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_142_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 142 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 72 (MapPartitionsRDD[149] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 72.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 72.0 (TID 143, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 72.0 (TID 144, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 72.0 (TID 143) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 72.0 (TID 143). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 72.0 (TID 144) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 72.0 (TID 143) in 6 ms on localhost (1/2) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 72.0 (TID 144). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 72.0 (TID 144) in 8 ms on localhost (2/2) -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 72 (treeAggregate at GradientDescent.scala:225) finished in 0.009 s -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 72.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 72 finished: treeAggregate at GradientDescent.scala:225, took 0.017221 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1248244, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_143 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=1253836, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_143_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_143_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 143 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 73 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 73(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 73 (MapPartitionsRDD[151] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1257445, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_144 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=1269229, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_144_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_144_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 144 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 73 (MapPartitionsRDD[151] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 73.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 73.0 (TID 145, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 73.0 (TID 146, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 73.0 (TID 145) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 73.0 (TID 146) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 73.0 (TID 145). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 73.0 (TID 146). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 73.0 (TID 146) in 5 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 73.0 (TID 145) in 6 ms on localhost (2/2) -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 73 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 73.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 73 finished: treeAggregate at GradientDescent.scala:225, took 0.016229 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1272717, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_145 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1278309, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_145_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_145_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 145 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 74 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 74(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 74 (MapPartitionsRDD[153] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1281915, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_146 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=1293699, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_146_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.3 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_146_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 146 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 74 (MapPartitionsRDD[153] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 74.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 74.0 (TID 147, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 74.0 (TID 148, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 74.0 (TID 147) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 74.0 (TID 148) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 74.0 (TID 147). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 74.0 (TID 148). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 74.0 (TID 147) in 5 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 74.0 (TID 148) in 5 ms on localhost (2/2) -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 74 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 74.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 74 finished: treeAggregate at GradientDescent.scala:225, took 0.014955 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1297186, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_147 stored as values in memory (estimated size 5.5 KB, free 729.3 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3624) called with curMem=1302778, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_147_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.3 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_147_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 147 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 75 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 75(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 75 (MapPartitionsRDD[155] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1306402, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_148 stored as values in memory (estimated size 11.5 KB, free 729.3 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3486) called with curMem=1318186, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_148_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.3 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_148_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 148 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 75 (MapPartitionsRDD[155] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 75.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 75.0 (TID 149, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 75.0 (TID 150, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 75.0 (TID 149) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 75.0 (TID 150) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 75.0 (TID 149). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 75.0 (TID 150). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 75.0 (TID 149) in 5 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 75.0 (TID 150) in 4 ms on localhost (2/2) -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 75 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 75.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 75 finished: treeAggregate at GradientDescent.scala:225, took 0.013183 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1321672, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_149 stored as values in memory (estimated size 5.5 KB, free 729.3 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3626) called with curMem=1327264, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_149_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.3 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_149_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 149 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_88_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_148_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 76 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 76 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 76(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_147_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 76 (MapPartitionsRDD[157] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_146_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1275862, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_150 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 75 -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=1282054, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_150_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_150_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 150 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_145_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 76 (MapPartitionsRDD[157] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 151, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 152, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 151) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 152) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_144_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 74 -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 152). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 152) in 7 ms on localhost (1/2) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_143_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 151). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 151) in 11 ms on localhost (2/2) -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 76 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 76 finished: treeAggregate at GradientDescent.scala:225, took 0.020405 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1253975, maxMem=766075207 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_142_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_151 stored as values in memory (estimated size 5.5 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 73 -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3611) called with curMem=1238587, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_151_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_141_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_151_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 151 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_140_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 72 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_139_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 77 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 77(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_138_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 71 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[159] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1198840, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_152 stored as values in memory (estimated size 11.5 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_137_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=1205032, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_152_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_152_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 152 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 77 (MapPartitionsRDD[159] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 2 tasks -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_136_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 70 -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 153, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 154, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 154) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 153) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_135_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 153). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 154). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 153) in 6 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 154) in 7 ms on localhost (2/2) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 77 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 77 finished: treeAggregate at GradientDescent.scala:225, took 0.016765 s -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_134_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1168776, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_153 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 69 -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1174368, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_153_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_153_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 153 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_133_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_132_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_131_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 78 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 78(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 78 (MapPartitionsRDD[161] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1144279, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_154 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=1140791, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_154_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_130_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_154_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 154 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 78 (MapPartitionsRDD[161] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 155, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 156, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_129_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 156) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 156). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 156) in 12 ms on localhost (1/2) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 155) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 155). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_128_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 155) in 22 ms on localhost (2/2) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 78 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 78 finished: treeAggregate at GradientDescent.scala:225, took 0.034301 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1110598, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_155 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_127_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1116190, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_155_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_155_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 155 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_126_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_125_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_124_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 79 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 79(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 79 (MapPartitionsRDD[163] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1076429, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_156 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_123_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=1082621, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_156_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_156_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 156 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 79 (MapPartitionsRDD[163] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 79.0 with 2 tasks -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_122_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 79.0 (TID 157, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 79.0 (TID 158, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 79.0 (TID 157) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 79.0 (TID 158) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 79.0 (TID 158). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 79.0 (TID 157). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 79.0 (TID 158) in 15 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 79.0 (TID 157) in 17 ms on localhost (2/2) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 79.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_121_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 79 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 79 finished: treeAggregate at GradientDescent.scala:225, took 0.056855 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1061633, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_157 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1067225, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_157_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_157_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 157 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_120_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 80 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 80(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 80 (MapPartitionsRDD[165] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1051949, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_158 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_119_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=1063733, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_158_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_158_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 158 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 80 (MapPartitionsRDD[165] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 80.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 80.0 (TID 159, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_118_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 80.0 (TID 160, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 80.0 (TID 159) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 80.0 (TID 159). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 80.0 (TID 159) in 8 ms on localhost (1/2) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 80.0 (TID 160) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 80.0 (TID 160). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 80.0 (TID 160) in 44 ms on localhost (2/2) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 80.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 80 (treeAggregate at GradientDescent.scala:225) finished in 0.046 s -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 80 finished: treeAggregate at GradientDescent.scala:225, took 0.059293 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1046356, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_159 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1051948, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_159_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_159_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 159 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 81 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 81(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_117_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 81 (MapPartitionsRDD[167] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1051942, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_160 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3486) called with curMem=1063726, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_160_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_160_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 160 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 81 (MapPartitionsRDD[167] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 81.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 81.0 (TID 161, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 81.0 (TID 162, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 81.0 (TID 162) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 81.0 (TID 162). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 81.0 (TID 161) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 81.0 (TID 161). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 81.0 (TID 162) in 27 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 81.0 (TID 161) in 29 ms on localhost (2/2) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 81.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 81 (treeAggregate at GradientDescent.scala:225) finished in 0.037 s -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 81 finished: treeAggregate at GradientDescent.scala:225, took 0.049063 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1061620, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_161 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1067212, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_161_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_161_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 161 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_116_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 82 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 82(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_115_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 82 (MapPartitionsRDD[169] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1046345, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_162 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=1058129, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_162_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_162_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 162 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 82 (MapPartitionsRDD[169] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 82.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 82.0 (TID 163, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_114_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 82.0 (TID 164, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 82.0 (TID 163) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_113_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 82.0 (TID 164) -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 82.0 (TID 163). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 82.0 (TID 164). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 82.0 (TID 163) in 9 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 82.0 (TID 164) in 15 ms on localhost (2/2) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 82.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 82 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 82 finished: treeAggregate at GradientDescent.scala:225, took 0.033918 s -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1033656, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_163 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_112_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=1027464, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_163_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_163_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 163 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_111_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_110_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_109_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 83 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 83(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_108_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 83 (MapPartitionsRDD[171] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=972904, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_164 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_107_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=972904, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_164_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_164_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 164 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_106_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 83 (MapPartitionsRDD[171] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 83.0 with 2 tasks -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 83.0 (TID 165, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 83.0 (TID 166, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 83.0 (TID 165) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 83.0 (TID 166) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_105_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 83.0 (TID 166). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 83.0 (TID 165). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_104_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 83.0 (TID 166) in 5 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 83.0 (TID 165) in 6 ms on localhost (2/2) -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 83 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 83.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 83 finished: treeAggregate at GradientDescent.scala:225, took 0.015672 s -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=942837, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_165 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_103_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=944797, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_165_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_165_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 165 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_102_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_101_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_100_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 84 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 84(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_99_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 84 (MapPartitionsRDD[173] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=895959, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_166 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_98_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3486) called with curMem=895959, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_166_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_166_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 166 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 84 (MapPartitionsRDD[173] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_97_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 84.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 84.0 (TID 167, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 84.0 (TID 168, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 84.0 (TID 167) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 84.0 (TID 168) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_96_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 84.0 (TID 168). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 84.0 (TID 167). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_95_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 84.0 (TID 168) in 6 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 84.0 (TID 167) in 6 ms on localhost (2/2) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 84.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 84 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 84 finished: treeAggregate at GradientDescent.scala:225, took 0.017638 s -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_94_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=862264, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_167 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3620) called with curMem=852458, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_167_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_93_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_167_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 167 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_92_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_91_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 85 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 85(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_90_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 85 (MapPartitionsRDD[175] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=807120, maxMem=766075207 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_89_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_168 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=813312, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_168_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_168_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 168 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 85 (MapPartitionsRDD[175] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_87_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 85.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 85.0 (TID 169, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 85.0 (TID 170, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 85.0 (TID 170) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 85.0 (TID 169) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 85.0 (TID 169). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 85.0 (TID 170). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 85.0 (TID 169) in 6 ms on localhost (1/2) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 85.0 (TID 170) in 7 ms on localhost (2/2) -2016-05-24 22:24:16 INFO DAGScheduler:59 - ResultStage 85 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 85.0, whose tasks have all completed, from pool -2016-05-24 22:24:16 INFO DAGScheduler:59 - Job 85 finished: treeAggregate at GradientDescent.scala:225, took 0.017087 s -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=783120, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_169 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3621) called with curMem=773443, maxMem=766075207 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_169_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_169_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 169 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Got job 86 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:16 INFO DAGScheduler:59 - Final stage: ResultStage 86(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting ResultStage 86 (MapPartitionsRDD[177] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:16 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=724500, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_170 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=730692, maxMem=766075207 -2016-05-24 22:24:16 INFO MemoryStore:59 - Block broadcast_170_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:24:16 INFO BlockManagerInfo:59 - Added broadcast_170_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:16 INFO SparkContext:59 - Created broadcast 170 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 86 (MapPartitionsRDD[177] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:16 INFO TaskSchedulerImpl:59 - Adding task set 86.0 with 2 tasks -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 86.0 (TID 171, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 86.0 (TID 172, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:16 INFO Executor:59 - Running task 0.0 in stage 86.0 (TID 171) -2016-05-24 22:24:16 INFO Executor:59 - Running task 1.0 in stage 86.0 (TID 172) -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:16 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:16 INFO Executor:59 - Finished task 1.0 in stage 86.0 (TID 172). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO Executor:59 - Finished task 0.0 in stage 86.0 (TID 171). 7968 bytes result sent to driver -2016-05-24 22:24:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 86.0 (TID 172) in 4 ms on localhost (1/2) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 86.0 (TID 171) in 4 ms on localhost (2/2) -2016-05-24 22:24:17 INFO DAGScheduler:59 - ResultStage 86 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 86.0, whose tasks have all completed, from pool -2016-05-24 22:24:17 INFO DAGScheduler:59 - Job 86 finished: treeAggregate at GradientDescent.scala:225, took 0.012664 s -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=734179, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_171 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=739771, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_171_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_171_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 171 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Got job 87 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:17 INFO DAGScheduler:59 - Final stage: ResultStage 87(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting ResultStage 87 (MapPartitionsRDD[179] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=743383, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_172 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=755167, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_172_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_172_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 172 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 87 (MapPartitionsRDD[179] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Adding task set 87.0 with 2 tasks -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 87.0 (TID 173, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 87.0 (TID 174, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO Executor:59 - Running task 1.0 in stage 87.0 (TID 174) -2016-05-24 22:24:17 INFO Executor:59 - Running task 0.0 in stage 87.0 (TID 173) -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:17 INFO Executor:59 - Finished task 0.0 in stage 87.0 (TID 173). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO Executor:59 - Finished task 1.0 in stage 87.0 (TID 174). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 87.0 (TID 173) in 4 ms on localhost (1/2) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 87.0 (TID 174) in 4 ms on localhost (2/2) -2016-05-24 22:24:17 INFO DAGScheduler:59 - ResultStage 87 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 87.0, whose tasks have all completed, from pool -2016-05-24 22:24:17 INFO DAGScheduler:59 - Job 87 finished: treeAggregate at GradientDescent.scala:225, took 0.013493 s -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=758654, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_173 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=764246, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_173_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_173_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 173 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Got job 88 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:17 INFO DAGScheduler:59 - Final stage: ResultStage 88(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting ResultStage 88 (MapPartitionsRDD[181] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=767859, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_174 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3490) called with curMem=779643, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_174_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_174_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 174 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 88 (MapPartitionsRDD[181] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Adding task set 88.0 with 2 tasks -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 88.0 (TID 175, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 88.0 (TID 176, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO Executor:59 - Running task 0.0 in stage 88.0 (TID 175) -2016-05-24 22:24:17 INFO Executor:59 - Running task 1.0 in stage 88.0 (TID 176) -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:17 INFO Executor:59 - Finished task 1.0 in stage 88.0 (TID 176). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO Executor:59 - Finished task 0.0 in stage 88.0 (TID 175). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 88.0 (TID 176) in 4 ms on localhost (1/2) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 88.0 (TID 175) in 5 ms on localhost (2/2) -2016-05-24 22:24:17 INFO DAGScheduler:59 - ResultStage 88 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 88.0, whose tasks have all completed, from pool -2016-05-24 22:24:17 INFO DAGScheduler:59 - Job 88 finished: treeAggregate at GradientDescent.scala:225, took 0.013938 s -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=783133, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_175 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3615) called with curMem=788725, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_175_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_175_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 175 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Got job 89 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:17 INFO DAGScheduler:59 - Final stage: ResultStage 89(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting ResultStage 89 (MapPartitionsRDD[183] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=792340, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_176 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=804124, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_176_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_176_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 176 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 89 (MapPartitionsRDD[183] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Adding task set 89.0 with 2 tasks -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 89.0 (TID 177, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 89.0 (TID 178, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO Executor:59 - Running task 1.0 in stage 89.0 (TID 178) -2016-05-24 22:24:17 INFO Executor:59 - Running task 0.0 in stage 89.0 (TID 177) -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:17 INFO Executor:59 - Finished task 1.0 in stage 89.0 (TID 178). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO Executor:59 - Finished task 0.0 in stage 89.0 (TID 177). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 89.0 (TID 178) in 3 ms on localhost (1/2) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 89.0 (TID 177) in 5 ms on localhost (2/2) -2016-05-24 22:24:17 INFO DAGScheduler:59 - ResultStage 89 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 89.0, whose tasks have all completed, from pool -2016-05-24 22:24:17 INFO DAGScheduler:59 - Job 89 finished: treeAggregate at GradientDescent.scala:225, took 0.011754 s -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=807612, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_177 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=813204, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_177_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_177_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 177 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Got job 90 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:17 INFO DAGScheduler:59 - Final stage: ResultStage 90(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting ResultStage 90 (MapPartitionsRDD[185] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=816816, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_178 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=828600, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_178_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_178_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 178 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 90 (MapPartitionsRDD[185] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Adding task set 90.0 with 2 tasks -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 90.0 (TID 179, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 90.0 (TID 180, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO Executor:59 - Running task 1.0 in stage 90.0 (TID 180) -2016-05-24 22:24:17 INFO Executor:59 - Running task 0.0 in stage 90.0 (TID 179) -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:17 INFO Executor:59 - Finished task 1.0 in stage 90.0 (TID 180). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO Executor:59 - Finished task 0.0 in stage 90.0 (TID 179). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 90.0 (TID 180) in 4 ms on localhost (1/2) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 90.0 (TID 179) in 6 ms on localhost (2/2) -2016-05-24 22:24:17 INFO DAGScheduler:59 - ResultStage 90 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 90.0, whose tasks have all completed, from pool -2016-05-24 22:24:17 INFO DAGScheduler:59 - Job 90 finished: treeAggregate at GradientDescent.scala:225, took 0.013147 s -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=832088, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_179 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=837680, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_179_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_179_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 179 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Got job 91 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:17 INFO DAGScheduler:59 - Final stage: ResultStage 91(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting ResultStage 91 (MapPartitionsRDD[187] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=841293, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_180 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=853077, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_180_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_180_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 180 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 91 (MapPartitionsRDD[187] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Adding task set 91.0 with 2 tasks -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 91.0 (TID 181, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 91.0 (TID 182, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO Executor:59 - Running task 1.0 in stage 91.0 (TID 182) -2016-05-24 22:24:17 INFO Executor:59 - Running task 0.0 in stage 91.0 (TID 181) -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:17 INFO Executor:59 - Finished task 1.0 in stage 91.0 (TID 182). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO Executor:59 - Finished task 0.0 in stage 91.0 (TID 181). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 91.0 (TID 182) in 4 ms on localhost (1/2) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 91.0 (TID 181) in 5 ms on localhost (2/2) -2016-05-24 22:24:17 INFO DAGScheduler:59 - ResultStage 91 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 91.0, whose tasks have all completed, from pool -2016-05-24 22:24:17 INFO DAGScheduler:59 - Job 91 finished: treeAggregate at GradientDescent.scala:225, took 0.014594 s -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=856564, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_181 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3608) called with curMem=862156, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_181_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_181_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 181 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Got job 92 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:17 INFO DAGScheduler:59 - Final stage: ResultStage 92(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting ResultStage 92 (MapPartitionsRDD[189] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=865764, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_182 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=877548, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_182_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_182_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 182 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 92 (MapPartitionsRDD[189] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Adding task set 92.0 with 2 tasks -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 92.0 (TID 183, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 92.0 (TID 184, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO Executor:59 - Running task 0.0 in stage 92.0 (TID 183) -2016-05-24 22:24:17 INFO Executor:59 - Running task 1.0 in stage 92.0 (TID 184) -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:17 INFO Executor:59 - Finished task 0.0 in stage 92.0 (TID 183). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO Executor:59 - Finished task 1.0 in stage 92.0 (TID 184). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 92.0 (TID 183) in 4 ms on localhost (1/2) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 92.0 (TID 184) in 4 ms on localhost (2/2) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 92.0, whose tasks have all completed, from pool -2016-05-24 22:24:17 INFO DAGScheduler:59 - ResultStage 92 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:24:17 INFO DAGScheduler:59 - Job 92 finished: treeAggregate at GradientDescent.scala:225, took 0.012200 s -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=881035, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_183 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3614) called with curMem=886627, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_183_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_183_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 183 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Got job 93 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:17 INFO DAGScheduler:59 - Final stage: ResultStage 93(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting ResultStage 93 (MapPartitionsRDD[191] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=890241, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_184 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=902025, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_184_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_184_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 184 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 93 (MapPartitionsRDD[191] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Adding task set 93.0 with 2 tasks -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 93.0 (TID 185, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 93.0 (TID 186, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO Executor:59 - Running task 0.0 in stage 93.0 (TID 185) -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:17 INFO Executor:59 - Finished task 0.0 in stage 93.0 (TID 185). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO Executor:59 - Running task 1.0 in stage 93.0 (TID 186) -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 93.0 (TID 185) in 7 ms on localhost (1/2) -2016-05-24 22:24:17 INFO Executor:59 - Finished task 1.0 in stage 93.0 (TID 186). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 93.0 (TID 186) in 12 ms on localhost (2/2) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 93.0, whose tasks have all completed, from pool -2016-05-24 22:24:17 INFO DAGScheduler:59 - ResultStage 93 (treeAggregate at GradientDescent.scala:225) finished in 0.013 s -2016-05-24 22:24:17 INFO DAGScheduler:59 - Job 93 finished: treeAggregate at GradientDescent.scala:225, took 0.021894 s -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=905513, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_185 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=911105, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_185_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_185_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 185 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Got job 94 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:17 INFO DAGScheduler:59 - Final stage: ResultStage 94(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting ResultStage 94 (MapPartitionsRDD[193] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=914717, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_186 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=926501, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_186_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_186_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 186 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 94 (MapPartitionsRDD[193] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Adding task set 94.0 with 2 tasks -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 94.0 (TID 187, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 94.0 (TID 188, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO Executor:59 - Running task 0.0 in stage 94.0 (TID 187) -2016-05-24 22:24:17 INFO Executor:59 - Running task 1.0 in stage 94.0 (TID 188) -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:17 INFO Executor:59 - Finished task 1.0 in stage 94.0 (TID 188). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO Executor:59 - Finished task 0.0 in stage 94.0 (TID 187). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 94.0 (TID 188) in 6 ms on localhost (1/2) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 94.0 (TID 187) in 7 ms on localhost (2/2) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 94.0, whose tasks have all completed, from pool -2016-05-24 22:24:17 INFO DAGScheduler:59 - ResultStage 94 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:24:17 INFO DAGScheduler:59 - Job 94 finished: treeAggregate at GradientDescent.scala:225, took 0.018157 s -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=929988, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_187 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=935580, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_187_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_187_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 187 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Got job 95 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:17 INFO DAGScheduler:59 - Final stage: ResultStage 95(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting ResultStage 95 (MapPartitionsRDD[195] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=939193, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_188 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=950977, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_188_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_188_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 188 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 95 (MapPartitionsRDD[195] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Adding task set 95.0 with 2 tasks -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 95.0 (TID 189, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 95.0 (TID 190, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO Executor:59 - Running task 1.0 in stage 95.0 (TID 190) -2016-05-24 22:24:17 INFO Executor:59 - Running task 0.0 in stage 95.0 (TID 189) -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:17 INFO Executor:59 - Finished task 1.0 in stage 95.0 (TID 190). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO Executor:59 - Finished task 0.0 in stage 95.0 (TID 189). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 95.0 (TID 190) in 4 ms on localhost (1/2) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 95.0 (TID 189) in 5 ms on localhost (2/2) -2016-05-24 22:24:17 INFO DAGScheduler:59 - ResultStage 95 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 95.0, whose tasks have all completed, from pool -2016-05-24 22:24:17 INFO DAGScheduler:59 - Job 95 finished: treeAggregate at GradientDescent.scala:225, took 0.012194 s -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=954464, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_189 stored as values in memory (estimated size 5.5 KB, free 729.7 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3634) called with curMem=960056, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_189_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.7 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_189_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 189 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Got job 96 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:17 INFO DAGScheduler:59 - Final stage: ResultStage 96(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting ResultStage 96 (MapPartitionsRDD[197] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=963690, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_190 stored as values in memory (estimated size 11.5 KB, free 729.7 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=975474, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_190_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.7 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_190_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 190 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 96 (MapPartitionsRDD[197] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Adding task set 96.0 with 2 tasks -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 96.0 (TID 191, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 96.0 (TID 192, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO Executor:59 - Running task 0.0 in stage 96.0 (TID 191) -2016-05-24 22:24:17 INFO Executor:59 - Running task 1.0 in stage 96.0 (TID 192) -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:17 INFO Executor:59 - Finished task 0.0 in stage 96.0 (TID 191). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO Executor:59 - Finished task 1.0 in stage 96.0 (TID 192). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 96.0 (TID 191) in 5 ms on localhost (1/2) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 96.0 (TID 192) in 5 ms on localhost (2/2) -2016-05-24 22:24:17 INFO DAGScheduler:59 - ResultStage 96 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 96.0, whose tasks have all completed, from pool -2016-05-24 22:24:17 INFO DAGScheduler:59 - Job 96 finished: treeAggregate at GradientDescent.scala:225, took 0.013228 s -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=978961, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_191 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3619) called with curMem=984553, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_191_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_191_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 191 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Got job 97 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:17 INFO DAGScheduler:59 - Final stage: ResultStage 97(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting ResultStage 97 (MapPartitionsRDD[199] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=988172, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_192 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=999956, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_192_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_192_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 192 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 97 (MapPartitionsRDD[199] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Adding task set 97.0 with 2 tasks -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 97.0 (TID 193, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 97.0 (TID 194, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO Executor:59 - Running task 1.0 in stage 97.0 (TID 194) -2016-05-24 22:24:17 INFO Executor:59 - Running task 0.0 in stage 97.0 (TID 193) -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:17 INFO Executor:59 - Finished task 0.0 in stage 97.0 (TID 193). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO Executor:59 - Finished task 1.0 in stage 97.0 (TID 194). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 97.0 (TID 193) in 5 ms on localhost (1/2) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 97.0 (TID 194) in 4 ms on localhost (2/2) -2016-05-24 22:24:17 INFO DAGScheduler:59 - ResultStage 97 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 97.0, whose tasks have all completed, from pool -2016-05-24 22:24:17 INFO DAGScheduler:59 - Job 97 finished: treeAggregate at GradientDescent.scala:225, took 0.013601 s -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1003444, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_193 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1009036, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_193_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_193_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 193 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Got job 98 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:17 INFO DAGScheduler:59 - Final stage: ResultStage 98(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting ResultStage 98 (MapPartitionsRDD[201] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1012648, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_194 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=1024432, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_194_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_194_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 194 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 98 (MapPartitionsRDD[201] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Adding task set 98.0 with 2 tasks -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 98.0 (TID 195, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 98.0 (TID 196, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO Executor:59 - Running task 0.0 in stage 98.0 (TID 195) -2016-05-24 22:24:17 INFO Executor:59 - Running task 1.0 in stage 98.0 (TID 196) -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:17 INFO Executor:59 - Finished task 1.0 in stage 98.0 (TID 196). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO Executor:59 - Finished task 0.0 in stage 98.0 (TID 195). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 98.0 (TID 196) in 4 ms on localhost (1/2) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 98.0 (TID 195) in 6 ms on localhost (2/2) -2016-05-24 22:24:17 INFO DAGScheduler:59 - ResultStage 98 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 98.0, whose tasks have all completed, from pool -2016-05-24 22:24:17 INFO DAGScheduler:59 - Job 98 finished: treeAggregate at GradientDescent.scala:225, took 0.013333 s -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1027920, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_195 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3609) called with curMem=1033512, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_195_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_195_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 195 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Got job 99 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:17 INFO DAGScheduler:59 - Final stage: ResultStage 99(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting ResultStage 99 (MapPartitionsRDD[203] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1037121, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_196 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=1048905, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_196_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_196_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 196 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 99 (MapPartitionsRDD[203] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Adding task set 99.0 with 2 tasks -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 99.0 (TID 197, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 99.0 (TID 198, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO Executor:59 - Running task 0.0 in stage 99.0 (TID 197) -2016-05-24 22:24:17 INFO Executor:59 - Running task 1.0 in stage 99.0 (TID 198) -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:17 INFO Executor:59 - Finished task 0.0 in stage 99.0 (TID 197). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO Executor:59 - Finished task 1.0 in stage 99.0 (TID 198). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 99.0 (TID 197) in 4 ms on localhost (1/2) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 99.0 (TID 198) in 5 ms on localhost (2/2) -2016-05-24 22:24:17 INFO DAGScheduler:59 - ResultStage 99 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 99.0, whose tasks have all completed, from pool -2016-05-24 22:24:17 INFO DAGScheduler:59 - Job 99 finished: treeAggregate at GradientDescent.scala:225, took 0.012250 s -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1052392, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_197 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3616) called with curMem=1057984, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_197_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_197_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 197 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Got job 100 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:17 INFO DAGScheduler:59 - Final stage: ResultStage 100(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting ResultStage 100 (MapPartitionsRDD[205] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1061600, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_198 stored as values in memory (estimated size 11.5 KB, free 729.6 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=1073384, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_198_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.6 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_198_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 198 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 100 (MapPartitionsRDD[205] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Adding task set 100.0 with 2 tasks -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 100.0 (TID 199, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 100.0 (TID 200, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO Executor:59 - Running task 0.0 in stage 100.0 (TID 199) -2016-05-24 22:24:17 INFO Executor:59 - Running task 1.0 in stage 100.0 (TID 200) -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:17 INFO Executor:59 - Finished task 1.0 in stage 100.0 (TID 200). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO Executor:59 - Finished task 0.0 in stage 100.0 (TID 199). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 100.0 (TID 200) in 5 ms on localhost (1/2) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 100.0 (TID 199) in 5 ms on localhost (2/2) -2016-05-24 22:24:17 INFO DAGScheduler:59 - ResultStage 100 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 100.0, whose tasks have all completed, from pool -2016-05-24 22:24:17 INFO DAGScheduler:59 - Job 100 finished: treeAggregate at GradientDescent.scala:225, took 0.012675 s -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1076872, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_199 stored as values in memory (estimated size 5.5 KB, free 729.6 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3606) called with curMem=1082464, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_199_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.6 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_199_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 199 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Got job 101 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:17 INFO DAGScheduler:59 - Final stage: ResultStage 101(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting ResultStage 101 (MapPartitionsRDD[207] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1086070, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_200 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=1097854, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_200_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_200_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 200 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 101 (MapPartitionsRDD[207] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Adding task set 101.0 with 2 tasks -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 101.0 (TID 201, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 101.0 (TID 202, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO Executor:59 - Running task 0.0 in stage 101.0 (TID 201) -2016-05-24 22:24:17 INFO Executor:59 - Running task 1.0 in stage 101.0 (TID 202) -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:17 INFO Executor:59 - Finished task 0.0 in stage 101.0 (TID 201). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO Executor:59 - Finished task 1.0 in stage 101.0 (TID 202). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 101.0 (TID 201) in 5 ms on localhost (1/2) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 101.0 (TID 202) in 5 ms on localhost (2/2) -2016-05-24 22:24:17 INFO DAGScheduler:59 - ResultStage 101 (treeAggregate at GradientDescent.scala:225) finished in 0.005 s -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 101.0, whose tasks have all completed, from pool -2016-05-24 22:24:17 INFO DAGScheduler:59 - Job 101 finished: treeAggregate at GradientDescent.scala:225, took 0.013486 s -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1101341, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_201 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3613) called with curMem=1106933, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_201_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_201_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 201 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Got job 102 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:17 INFO DAGScheduler:59 - Final stage: ResultStage 102(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting ResultStage 102 (MapPartitionsRDD[209] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1110546, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_202 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=1122330, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_202_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_202_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 202 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 102 (MapPartitionsRDD[209] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Adding task set 102.0 with 2 tasks -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 102.0 (TID 203, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 102.0 (TID 204, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO Executor:59 - Running task 0.0 in stage 102.0 (TID 203) -2016-05-24 22:24:17 INFO Executor:59 - Running task 1.0 in stage 102.0 (TID 204) -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:17 INFO Executor:59 - Finished task 1.0 in stage 102.0 (TID 204). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO Executor:59 - Finished task 0.0 in stage 102.0 (TID 203). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 102.0 (TID 204) in 4 ms on localhost (1/2) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 102.0 (TID 203) in 5 ms on localhost (2/2) -2016-05-24 22:24:17 INFO DAGScheduler:59 - ResultStage 102 (treeAggregate at GradientDescent.scala:225) finished in 0.006 s -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 102.0, whose tasks have all completed, from pool -2016-05-24 22:24:17 INFO DAGScheduler:59 - Job 102 finished: treeAggregate at GradientDescent.scala:225, took 0.013950 s -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=1125818, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_203 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3612) called with curMem=1131410, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_203_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.5 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_203_piece0 in memory on localhost:57000 (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 203 from broadcast at GradientDescent.scala:221 -2016-05-24 22:24:17 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Got job 103 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:24:17 INFO DAGScheduler:59 - Final stage: ResultStage 103(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting ResultStage 103 (MapPartitionsRDD[211] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=1135022, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_204 stored as values in memory (estimated size 11.5 KB, free 729.5 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=1146806, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_204_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_204_piece0 in memory on localhost:57000 (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 204 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 103 (MapPartitionsRDD[211] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Adding task set 103.0 with 2 tasks -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 103.0 (TID 205, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 103.0 (TID 206, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:24:17 INFO Executor:59 - Running task 0.0 in stage 103.0 (TID 205) -2016-05-24 22:24:17 INFO Executor:59 - Running task 1.0 in stage 103.0 (TID 206) -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:24:17 INFO Executor:59 - Finished task 0.0 in stage 103.0 (TID 205). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO Executor:59 - Finished task 1.0 in stage 103.0 (TID 206). 7968 bytes result sent to driver -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 103.0 (TID 205) in 6 ms on localhost (1/2) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 103.0 (TID 206) in 6 ms on localhost (2/2) -2016-05-24 22:24:17 INFO DAGScheduler:59 - ResultStage 103 (treeAggregate at GradientDescent.scala:225) finished in 0.007 s -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 103.0, whose tasks have all completed, from pool -2016-05-24 22:24:17 INFO DAGScheduler:59 - Job 103 finished: treeAggregate at GradientDescent.scala:225, took 0.014970 s -2016-05-24 22:24:17 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 3363.579936927662, 3356.5316534793847, 3349.536449573466, 3342.59343622967, 3335.701748404667, 3328.860544103912, 3322.069003535364, 3315.3263283026713, 3308.6317406356134, 3301.984482655661 -2016-05-24 22:24:17 INFO SparkContext:59 - Starting job: sortByKey at BinaryClassificationMetrics.scala:156 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Registering RDD 212 (map at SVMFromSparkLearning.scala:44) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Got job 104 (sortByKey at BinaryClassificationMetrics.scala:156) with 2 output partitions -2016-05-24 22:24:17 INFO DAGScheduler:59 - Final stage: ResultStage 105(sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 104) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 104) -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting ShuffleMapStage 104 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44), which has no missing parents -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=1150293, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_205 stored as values in memory (estimated size 10.8 KB, free 729.5 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(6437) called with curMem=1161301, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_205_piece0 stored as bytes in memory (estimated size 6.3 KB, free 729.5 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_205_piece0 in memory on localhost:57000 (size: 6.3 KB, free: 730.1 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 205 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 104 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Adding task set 104.0 with 2 tasks -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 104.0 (TID 207, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 104.0 (TID 208, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:24:17 INFO Executor:59 - Running task 0.0 in stage 104.0 (TID 207) -2016-05-24 22:24:17 INFO Executor:59 - Running task 1.0 in stage 104.0 (TID 208) -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:24:17 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:24:17 INFO Executor:59 - Finished task 1.0 in stage 104.0 (TID 208). 2254 bytes result sent to driver -2016-05-24 22:24:17 INFO Executor:59 - Finished task 0.0 in stage 104.0 (TID 207). 2254 bytes result sent to driver -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 104.0 (TID 208) in 468 ms on localhost (1/2) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 104.0 (TID 207) in 470 ms on localhost (2/2) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 104.0, whose tasks have all completed, from pool -2016-05-24 22:24:17 INFO DAGScheduler:59 - ShuffleMapStage 104 (map at SVMFromSparkLearning.scala:44) finished in 0.471 s -2016-05-24 22:24:17 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:24:17 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:24:17 INFO DAGScheduler:59 - waiting: Set(ResultStage 105) -2016-05-24 22:24:17 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Missing parents for ResultStage 105: List() -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting ResultStage 105 (MapPartitionsRDD[215] at sortByKey at BinaryClassificationMetrics.scala:156), which is now runnable -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(3344) called with curMem=1167738, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_206 stored as values in memory (estimated size 3.3 KB, free 729.5 MB) -2016-05-24 22:24:17 INFO MemoryStore:59 - ensureFreeSpace(1900) called with curMem=1171082, maxMem=766075207 -2016-05-24 22:24:17 INFO MemoryStore:59 - Block broadcast_206_piece0 stored as bytes in memory (estimated size 1900.0 B, free 729.5 MB) -2016-05-24 22:24:17 INFO BlockManagerInfo:59 - Added broadcast_206_piece0 in memory on localhost:57000 (size: 1900.0 B, free: 730.1 MB) -2016-05-24 22:24:17 INFO SparkContext:59 - Created broadcast 206 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 105 (MapPartitionsRDD[215] at sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:24:17 INFO TaskSchedulerImpl:59 - Adding task set 105.0 with 2 tasks -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 105.0 (TID 209, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:24:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 105.0 (TID 210, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:24:17 INFO Executor:59 - Running task 0.0 in stage 105.0 (TID 209) -2016-05-24 22:24:17 INFO Executor:59 - Running task 1.0 in stage 105.0 (TID 210) -2016-05-24 22:24:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:24:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:24:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-24 22:24:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-24 22:24:18 INFO Executor:59 - Finished task 0.0 in stage 105.0 (TID 209). 1469 bytes result sent to driver -2016-05-24 22:24:18 INFO Executor:59 - Finished task 1.0 in stage 105.0 (TID 210). 1517 bytes result sent to driver -2016-05-24 22:24:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 105.0 (TID 209) in 42 ms on localhost (1/2) -2016-05-24 22:24:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 105.0 (TID 210) in 40 ms on localhost (2/2) -2016-05-24 22:24:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 105.0, whose tasks have all completed, from pool -2016-05-24 22:24:18 INFO DAGScheduler:59 - ResultStage 105 (sortByKey at BinaryClassificationMetrics.scala:156) finished in 0.043 s -2016-05-24 22:24:18 INFO DAGScheduler:59 - Job 104 finished: sortByKey at BinaryClassificationMetrics.scala:156, took 0.541331 s -2016-05-24 22:24:18 INFO SparkContext:59 - Starting job: collect at BinaryClassificationMetrics.scala:193 -2016-05-24 22:24:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 154 bytes -2016-05-24 22:24:18 INFO DAGScheduler:59 - Registering RDD 213 (combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:24:18 INFO DAGScheduler:59 - Got job 105 (collect at BinaryClassificationMetrics.scala:193) with 2 output partitions -2016-05-24 22:24:18 INFO DAGScheduler:59 - Final stage: ResultStage 108(collect at BinaryClassificationMetrics.scala:193) -2016-05-24 22:24:18 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 107) -2016-05-24 22:24:18 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 107) -2016-05-24 22:24:18 INFO DAGScheduler:59 - Submitting ShuffleMapStage 107 (ShuffledRDD[213] at combineByKey at BinaryClassificationMetrics.scala:152), which has no missing parents -2016-05-24 22:24:18 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=1172982, maxMem=766075207 -2016-05-24 22:24:18 INFO MemoryStore:59 - Block broadcast_207 stored as values in memory (estimated size 2.8 KB, free 729.5 MB) -2016-05-24 22:24:18 INFO MemoryStore:59 - ensureFreeSpace(1658) called with curMem=1175878, maxMem=766075207 -2016-05-24 22:24:18 INFO MemoryStore:59 - Block broadcast_207_piece0 stored as bytes in memory (estimated size 1658.0 B, free 729.5 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Added broadcast_207_piece0 in memory on localhost:57000 (size: 1658.0 B, free: 730.1 MB) -2016-05-24 22:24:18 INFO SparkContext:59 - Created broadcast 207 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 107 (ShuffledRDD[213] at combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:24:18 INFO TaskSchedulerImpl:59 - Adding task set 107.0 with 2 tasks -2016-05-24 22:24:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 107.0 (TID 211, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:24:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 107.0 (TID 212, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:24:18 INFO Executor:59 - Running task 0.0 in stage 107.0 (TID 211) -2016-05-24 22:24:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:24:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:24:18 INFO Executor:59 - Running task 1.0 in stage 107.0 (TID 212) -2016-05-24 22:24:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:24:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:24:18 INFO Executor:59 - Finished task 0.0 in stage 107.0 (TID 211). 1375 bytes result sent to driver -2016-05-24 22:24:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 107.0 (TID 211) in 38 ms on localhost (1/2) -2016-05-24 22:24:18 INFO Executor:59 - Finished task 1.0 in stage 107.0 (TID 212). 1375 bytes result sent to driver -2016-05-24 22:24:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 107.0 (TID 212) in 40 ms on localhost (2/2) -2016-05-24 22:24:18 INFO DAGScheduler:59 - ShuffleMapStage 107 (combineByKey at BinaryClassificationMetrics.scala:152) finished in 0.042 s -2016-05-24 22:24:18 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:24:18 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:24:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 107.0, whose tasks have all completed, from pool -2016-05-24 22:24:18 INFO DAGScheduler:59 - waiting: Set(ResultStage 108) -2016-05-24 22:24:18 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:24:18 INFO DAGScheduler:59 - Missing parents for ResultStage 108: List() -2016-05-24 22:24:18 INFO DAGScheduler:59 - Submitting ResultStage 108 (MapPartitionsRDD[218] at mapPartitions at BinaryClassificationMetrics.scala:189), which is now runnable -2016-05-24 22:24:18 INFO MemoryStore:59 - ensureFreeSpace(3472) called with curMem=1177536, maxMem=766075207 -2016-05-24 22:24:18 INFO MemoryStore:59 - Block broadcast_208 stored as values in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-24 22:24:18 INFO MemoryStore:59 - ensureFreeSpace(1981) called with curMem=1181008, maxMem=766075207 -2016-05-24 22:24:18 INFO MemoryStore:59 - Block broadcast_208_piece0 stored as bytes in memory (estimated size 1981.0 B, free 729.5 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Added broadcast_208_piece0 in memory on localhost:57000 (size: 1981.0 B, free: 730.1 MB) -2016-05-24 22:24:18 INFO SparkContext:59 - Created broadcast 208 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 108 (MapPartitionsRDD[218] at mapPartitions at BinaryClassificationMetrics.scala:189) -2016-05-24 22:24:18 INFO TaskSchedulerImpl:59 - Adding task set 108.0 with 2 tasks -2016-05-24 22:24:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 108.0 (TID 213, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:24:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 108.0 (TID 214, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:24:18 INFO Executor:59 - Running task 1.0 in stage 108.0 (TID 214) -2016-05-24 22:24:18 INFO Executor:59 - Running task 0.0 in stage 108.0 (TID 213) -2016-05-24 22:24:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:24:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:24:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:24:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:24:18 INFO Executor:59 - Finished task 0.0 in stage 108.0 (TID 213). 1330 bytes result sent to driver -2016-05-24 22:24:18 INFO Executor:59 - Finished task 1.0 in stage 108.0 (TID 214). 1330 bytes result sent to driver -2016-05-24 22:24:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 108.0 (TID 213) in 18 ms on localhost (1/2) -2016-05-24 22:24:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 108.0 (TID 214) in 17 ms on localhost (2/2) -2016-05-24 22:24:18 INFO DAGScheduler:59 - ResultStage 108 (collect at BinaryClassificationMetrics.scala:193) finished in 0.018 s -2016-05-24 22:24:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 108.0, whose tasks have all completed, from pool -2016-05-24 22:24:18 INFO DAGScheduler:59 - Job 105 finished: collect at BinaryClassificationMetrics.scala:193, took 0.090542 s -2016-05-24 22:24:18 INFO BinaryClassificationMetrics:59 - Total counts: {numPos: 25, numNeg: 11} -2016-05-24 22:24:18 INFO SparkContext:59 - Starting job: runJob at SlidingRDD.scala:75 -2016-05-24 22:24:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 154 bytes -2016-05-24 22:24:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-24 22:24:18 INFO DAGScheduler:59 - Got job 106 (runJob at SlidingRDD.scala:75) with 3 output partitions -2016-05-24 22:24:18 INFO DAGScheduler:59 - Final stage: ResultStage 111(runJob at SlidingRDD.scala:75) -2016-05-24 22:24:18 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 110) -2016-05-24 22:24:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:18 INFO DAGScheduler:59 - Submitting ResultStage 111 (UnionRDD[224] at UnionRDD at BinaryClassificationMetrics.scala:92), which has no missing parents -2016-05-24 22:24:18 INFO MemoryStore:59 - ensureFreeSpace(5184) called with curMem=1182989, maxMem=766075207 -2016-05-24 22:24:18 INFO MemoryStore:59 - Block broadcast_209 stored as values in memory (estimated size 5.1 KB, free 729.5 MB) -2016-05-24 22:24:18 INFO MemoryStore:59 - ensureFreeSpace(2788) called with curMem=1188173, maxMem=766075207 -2016-05-24 22:24:18 INFO MemoryStore:59 - Block broadcast_209_piece0 stored as bytes in memory (estimated size 2.7 KB, free 729.5 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Added broadcast_209_piece0 in memory on localhost:57000 (size: 2.7 KB, free: 730.1 MB) -2016-05-24 22:24:18 INFO SparkContext:59 - Created broadcast 209 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:18 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 111 (UnionRDD[224] at UnionRDD at BinaryClassificationMetrics.scala:92) -2016-05-24 22:24:18 INFO TaskSchedulerImpl:59 - Adding task set 111.0 with 3 tasks -2016-05-24 22:24:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 111.0 (TID 215, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:24:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 111.0 (TID 216, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:24:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 111.0 (TID 217, localhost, PROCESS_LOCAL, 2318 bytes) -2016-05-24 22:24:18 INFO Executor:59 - Running task 1.0 in stage 111.0 (TID 216) -2016-05-24 22:24:18 INFO Executor:59 - Running task 0.0 in stage 111.0 (TID 215) -2016-05-24 22:24:18 INFO CacheManager:59 - Partition rdd_219_0 not found, computing it -2016-05-24 22:24:18 INFO CacheManager:59 - Partition rdd_219_1 not found, computing it -2016-05-24 22:24:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:24:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:24:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:24:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:24:18 INFO MemoryStore:59 - ensureFreeSpace(1448) called with curMem=1190961, maxMem=766075207 -2016-05-24 22:24:18 INFO MemoryStore:59 - Block rdd_219_0 stored as values in memory (estimated size 1448.0 B, free 729.4 MB) -2016-05-24 22:24:18 INFO MemoryStore:59 - ensureFreeSpace(1616) called with curMem=1192409, maxMem=766075207 -2016-05-24 22:24:18 INFO Executor:59 - Running task 2.0 in stage 111.0 (TID 217) -2016-05-24 22:24:18 INFO MemoryStore:59 - Block rdd_219_1 stored as values in memory (estimated size 1616.0 B, free 729.4 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Added rdd_219_0 in memory on localhost:57000 (size: 1448.0 B, free: 730.1 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Added rdd_219_1 in memory on localhost:57000 (size: 1616.0 B, free: 730.1 MB) -2016-05-24 22:24:18 INFO Executor:59 - Finished task 0.0 in stage 111.0 (TID 215). 1883 bytes result sent to driver -2016-05-24 22:24:18 INFO Executor:59 - Finished task 1.0 in stage 111.0 (TID 216). 1883 bytes result sent to driver -2016-05-24 22:24:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 111.0 (TID 215) in 57 ms on localhost (1/3) -2016-05-24 22:24:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 111.0 (TID 216) in 31 ms on localhost (2/3) -2016-05-24 22:24:18 INFO Executor:59 - Finished task 2.0 in stage 111.0 (TID 217). 1053 bytes result sent to driver -2016-05-24 22:24:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 111.0 (TID 217) in 33 ms on localhost (3/3) -2016-05-24 22:24:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 111.0, whose tasks have all completed, from pool -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_159_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:18 INFO DAGScheduler:59 - ResultStage 111 (runJob at SlidingRDD.scala:75) finished in 0.062 s -2016-05-24 22:24:18 INFO DAGScheduler:59 - Job 106 finished: runJob at SlidingRDD.scala:75, took 0.084343 s -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_158_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 81 -2016-05-24 22:24:18 INFO SparkContext:59 - Starting job: aggregate at AreaUnderCurve.scala:45 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_157_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:18 INFO DAGScheduler:59 - Got job 107 (aggregate at AreaUnderCurve.scala:45) with 4 output partitions -2016-05-24 22:24:18 INFO DAGScheduler:59 - Final stage: ResultStage 114(aggregate at AreaUnderCurve.scala:45) -2016-05-24 22:24:18 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 113) -2016-05-24 22:24:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:18 INFO DAGScheduler:59 - Submitting ResultStage 114 (SlidingRDD[225] at RDD at SlidingRDD.scala:47), which has no missing parents -2016-05-24 22:24:18 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=1156865, maxMem=766075207 -2016-05-24 22:24:18 INFO MemoryStore:59 - Block broadcast_210 stored as values in memory (estimated size 5.5 KB, free 729.5 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_156_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:18 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=1162521, maxMem=766075207 -2016-05-24 22:24:18 INFO MemoryStore:59 - Block broadcast_210_piece0 stored as bytes in memory (estimated size 2.9 KB, free 729.5 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Added broadcast_210_piece0 in memory on localhost:57000 (size: 2.9 KB, free: 730.1 MB) -2016-05-24 22:24:18 INFO SparkContext:59 - Created broadcast 210 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 80 -2016-05-24 22:24:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 114 (SlidingRDD[225] at RDD at SlidingRDD.scala:47) -2016-05-24 22:24:18 INFO TaskSchedulerImpl:59 - Adding task set 114.0 with 4 tasks -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_155_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 114.0 (TID 218, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:24:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 114.0 (TID 219, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:24:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 114.0 (TID 220, localhost, PROCESS_LOCAL, 2635 bytes) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_154_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 114.0 (TID 221, localhost, PROCESS_LOCAL, 2475 bytes) -2016-05-24 22:24:18 INFO Executor:59 - Running task 0.0 in stage 114.0 (TID 220) -2016-05-24 22:24:18 INFO Executor:59 - Running task 1.0 in stage 114.0 (TID 218) -2016-05-24 22:24:18 INFO Executor:59 - Running task 2.0 in stage 114.0 (TID 219) -2016-05-24 22:24:18 INFO BlockManager:59 - Found block rdd_219_1 locally -2016-05-24 22:24:18 INFO BlockManager:59 - Found block rdd_219_0 locally -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 79 -2016-05-24 22:24:18 INFO Executor:59 - Finished task 0.0 in stage 114.0 (TID 220). 955 bytes result sent to driver -2016-05-24 22:24:18 INFO Executor:59 - Finished task 2.0 in stage 114.0 (TID 219). 2084 bytes result sent to driver -2016-05-24 22:24:18 INFO Executor:59 - Finished task 1.0 in stage 114.0 (TID 218). 2084 bytes result sent to driver -2016-05-24 22:24:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 114.0 (TID 220) in 15 ms on localhost (1/4) -2016-05-24 22:24:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 114.0 (TID 219) in 16 ms on localhost (2/4) -2016-05-24 22:24:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 114.0 (TID 218) in 21 ms on localhost (3/4) -2016-05-24 22:24:18 INFO Executor:59 - Running task 3.0 in stage 114.0 (TID 221) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_153_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:18 INFO Executor:59 - Finished task 3.0 in stage 114.0 (TID 221). 955 bytes result sent to driver -2016-05-24 22:24:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 114.0 (TID 221) in 21 ms on localhost (4/4) -2016-05-24 22:24:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 114.0, whose tasks have all completed, from pool -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_152_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:18 INFO DAGScheduler:59 - ResultStage 114 (aggregate at AreaUnderCurve.scala:45) finished in 0.028 s -2016-05-24 22:24:18 INFO DAGScheduler:59 - Job 107 finished: aggregate at AreaUnderCurve.scala:45, took 0.041602 s -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 78 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_151_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_150_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 77 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_149_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_208_piece0 on localhost:57000 in memory (size: 1981.0 B, free: 730.1 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 108 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_207_piece0 on localhost:57000 in memory (size: 1658.0 B, free: 730.1 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 107 -2016-05-24 22:24:18 INFO SparkContext:59 - Starting job: take at SVMFromSparkLearning.scala:56 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_206_piece0 on localhost:57000 in memory (size: 1900.0 B, free: 730.1 MB) -2016-05-24 22:24:18 INFO DAGScheduler:59 - Got job 108 (take at SVMFromSparkLearning.scala:56) with 1 output partitions -2016-05-24 22:24:18 INFO DAGScheduler:59 - Final stage: ResultStage 115(take at SVMFromSparkLearning.scala:56) -2016-05-24 22:24:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 106 -2016-05-24 22:24:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:18 INFO DAGScheduler:59 - Submitting ResultStage 115 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44), which has no missing parents -2016-05-24 22:24:18 INFO MemoryStore:59 - ensureFreeSpace(10312) called with curMem=1038369, maxMem=766075207 -2016-05-24 22:24:18 INFO MemoryStore:59 - Block broadcast_211 stored as values in memory (estimated size 10.1 KB, free 729.6 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_205_piece0 on localhost:57000 in memory (size: 6.3 KB, free: 730.1 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 105 -2016-05-24 22:24:18 INFO MemoryStore:59 - ensureFreeSpace(6086) called with curMem=1048681, maxMem=766075207 -2016-05-24 22:24:18 INFO MemoryStore:59 - Block broadcast_211_piece0 stored as bytes in memory (estimated size 5.9 KB, free 729.6 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Added broadcast_211_piece0 in memory on localhost:57000 (size: 5.9 KB, free: 730.1 MB) -2016-05-24 22:24:18 INFO SparkContext:59 - Created broadcast 211 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_204_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 115 (MapPartitionsRDD[212] at map at SVMFromSparkLearning.scala:44) -2016-05-24 22:24:18 INFO TaskSchedulerImpl:59 - Adding task set 115.0 with 1 tasks -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 104 -2016-05-24 22:24:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 115.0 (TID 222, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:24:18 INFO Executor:59 - Running task 0.0 in stage 115.0 (TID 222) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_203_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:18 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:24:18 INFO Executor:59 - Finished task 0.0 in stage 115.0 (TID 222). 2401 bytes result sent to driver -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_202_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 103 -2016-05-24 22:24:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 115.0 (TID 222) in 11 ms on localhost (1/1) -2016-05-24 22:24:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 115.0, whose tasks have all completed, from pool -2016-05-24 22:24:18 INFO DAGScheduler:59 - ResultStage 115 (take at SVMFromSparkLearning.scala:56) finished in 0.013 s -2016-05-24 22:24:18 INFO DAGScheduler:59 - Job 108 finished: take at SVMFromSparkLearning.scala:56, took 0.022546 s -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_201_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_200_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.1 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 102 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_199_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_198_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 101 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_197_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_196_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 100 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_195_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_194_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 99 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_193_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_192_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 98 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_191_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_190_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 97 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_189_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_188_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 96 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_187_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_186_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 95 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_185_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_184_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 94 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_183_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_182_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 93 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_181_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_180_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 92 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_179_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_178_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 91 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_177_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_176_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 90 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_175_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_174_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 89 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_173_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_172_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 88 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_171_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_170_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 87 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_169_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_168_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 86 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_167_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_166_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 85 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_165_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_164_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 84 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_163_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_162_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 83 -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_161_piece0 on localhost:57000 in memory (size: 3.5 KB, free: 730.3 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Removed broadcast_160_piece0 on localhost:57000 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:24:18 INFO ContextCleaner:59 - Cleaned accumulator 82 -2016-05-24 22:24:18 INFO SparkContext:59 - Starting job: saveAsTextFile at GLMClassificationModel.scala:61 -2016-05-24 22:24:18 INFO DAGScheduler:59 - Got job 109 (saveAsTextFile at GLMClassificationModel.scala:61) with 1 output partitions -2016-05-24 22:24:18 INFO DAGScheduler:59 - Final stage: ResultStage 116(saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:24:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:18 INFO DAGScheduler:59 - Submitting ResultStage 116 (MapPartitionsRDD[227] at saveAsTextFile at GLMClassificationModel.scala:61), which has no missing parents -2016-05-24 22:24:18 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=500990, maxMem=766075207 -2016-05-24 22:24:18 INFO MemoryStore:59 - Block broadcast_212 stored as values in memory (estimated size 124.0 KB, free 730.0 MB) -2016-05-24 22:24:18 INFO MemoryStore:59 - ensureFreeSpace(42575) called with curMem=627982, maxMem=766075207 -2016-05-24 22:24:18 INFO MemoryStore:59 - Block broadcast_212_piece0 stored as bytes in memory (estimated size 41.6 KB, free 729.9 MB) -2016-05-24 22:24:18 INFO BlockManagerInfo:59 - Added broadcast_212_piece0 in memory on localhost:57000 (size: 41.6 KB, free: 730.2 MB) -2016-05-24 22:24:18 INFO SparkContext:59 - Created broadcast 212 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 116 (MapPartitionsRDD[227] at saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:24:18 INFO TaskSchedulerImpl:59 - Adding task set 116.0 with 1 tasks -2016-05-24 22:24:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 116.0 (TID 223, localhost, PROCESS_LOCAL, 2197 bytes) -2016-05-24 22:24:18 INFO Executor:59 - Running task 0.0 in stage 116.0 (TID 223) -2016-05-24 22:24:18 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242224_0116_m_000000_223' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222418363/result/metadata/_temporary/0/task_201605242224_0116_m_000000 -2016-05-24 22:24:18 INFO SparkHadoopMapRedUtil:59 - attempt_201605242224_0116_m_000000_223: Committed -2016-05-24 22:24:18 INFO Executor:59 - Finished task 0.0 in stage 116.0 (TID 223). 1864 bytes result sent to driver -2016-05-24 22:24:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 116.0 (TID 223) in 155 ms on localhost (1/1) -2016-05-24 22:24:18 INFO DAGScheduler:59 - ResultStage 116 (saveAsTextFile at GLMClassificationModel.scala:61) finished in 0.155 s -2016-05-24 22:24:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 116.0, whose tasks have all completed, from pool -2016-05-24 22:24:18 INFO DAGScheduler:59 - Job 109 finished: saveAsTextFile at GLMClassificationModel.scala:61, took 0.219669 s -2016-05-24 22:24:20 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:24:20 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:24:20 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:65 -2016-05-24 22:24:20 INFO DAGScheduler:59 - Got job 110 (parquet at GLMClassificationModel.scala:65) with 1 output partitions -2016-05-24 22:24:20 INFO DAGScheduler:59 - Final stage: ResultStage 117(parquet at GLMClassificationModel.scala:65) -2016-05-24 22:24:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:20 INFO DAGScheduler:59 - Submitting ResultStage 117 (MapPartitionsRDD[229] at rddToDataFrameHolder at GLMClassificationModel.scala:65), which has no missing parents -2016-05-24 22:24:20 INFO MemoryStore:59 - ensureFreeSpace(67104) called with curMem=670557, maxMem=766075207 -2016-05-24 22:24:20 INFO MemoryStore:59 - Block broadcast_213 stored as values in memory (estimated size 65.5 KB, free 729.9 MB) -2016-05-24 22:24:20 INFO MemoryStore:59 - ensureFreeSpace(23437) called with curMem=737661, maxMem=766075207 -2016-05-24 22:24:20 INFO MemoryStore:59 - Block broadcast_213_piece0 stored as bytes in memory (estimated size 22.9 KB, free 729.9 MB) -2016-05-24 22:24:20 INFO BlockManagerInfo:59 - Added broadcast_213_piece0 in memory on localhost:57000 (size: 22.9 KB, free: 730.2 MB) -2016-05-24 22:24:20 INFO SparkContext:59 - Created broadcast 213 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 117 (MapPartitionsRDD[229] at rddToDataFrameHolder at GLMClassificationModel.scala:65) -2016-05-24 22:24:20 INFO TaskSchedulerImpl:59 - Adding task set 117.0 with 1 tasks -2016-05-24 22:24:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 117.0 (TID 224, localhost, PROCESS_LOCAL, 7978 bytes) -2016-05-24 22:24:20 INFO Executor:59 - Running task 0.0 in stage 117.0 (TID 224) -2016-05-24 22:24:20 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:24:20 INFO CodecConfig:151 - Compression: GZIP -2016-05-24 22:24:20 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-24 22:24:20 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-24 22:24:20 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-24 22:24:20 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-24 22:24:20 INFO ParquetOutputFormat:151 - Validation is off -2016-05-24 22:24:20 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-24 22:24:20 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-24 22:24:20 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 6,156 -2016-05-24 22:24:20 INFO ColumnChunkPageWriteStore:151 - written 61B for [weights, type] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-24 22:24:20 INFO ColumnChunkPageWriteStore:151 - written 48B for [weights, size] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-24 22:24:20 INFO ColumnChunkPageWriteStore:151 - written 51B for [weights, indices, array] INT32: 1 values, 13B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-24 22:24:20 INFO ColumnChunkPageWriteStore:151 - written 637B for [weights, values, array] DOUBLE: 692 values, 668B raw, 593B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 413 entries, 3,304B raw, 413B comp} -2016-05-24 22:24:20 INFO ColumnChunkPageWriteStore:151 - written 69B for [intercept] DOUBLE: 1 values, 14B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-24 22:24:20 INFO ColumnChunkPageWriteStore:151 - written 47B for [threshold] DOUBLE: 1 values, 6B raw, 26B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-24 22:24:20 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242224_0117_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222418363/result/data/_temporary/0/task_201605242224_0117_m_000000 -2016-05-24 22:24:20 INFO SparkHadoopMapRedUtil:59 - attempt_201605242224_0117_m_000000_0: Committed -2016-05-24 22:24:20 INFO Executor:59 - Finished task 0.0 in stage 117.0 (TID 224). 915 bytes result sent to driver -2016-05-24 22:24:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 117.0 (TID 224) in 471 ms on localhost (1/1) -2016-05-24 22:24:20 INFO DAGScheduler:59 - ResultStage 117 (parquet at GLMClassificationModel.scala:65) finished in 0.471 s -2016-05-24 22:24:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 117.0, whose tasks have all completed, from pool -2016-05-24 22:24:20 INFO DAGScheduler:59 - Job 110 finished: parquet at GLMClassificationModel.scala:65, took 0.507632 s -2016-05-24 22:24:21 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:24:21 INFO DefaultWriterContainer:59 - Job job_201605242224_0000 committed. -2016-05-24 22:24:21 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222418363/result/data on driver -2016-05-24 22:24:21 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222418363/result/data on driver -2016-05-24 22:24:21 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=761098, maxMem=766075207 -2016-05-24 22:24:21 INFO MemoryStore:59 - Block broadcast_214 stored as values in memory (estimated size 225.3 KB, free 729.6 MB) -2016-05-24 22:24:21 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=991802, maxMem=766075207 -2016-05-24 22:24:21 INFO MemoryStore:59 - Block broadcast_214_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.6 MB) -2016-05-24 22:24:21 INFO BlockManagerInfo:59 - Added broadcast_214_piece0 in memory on localhost:57000 (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:24:21 INFO SparkContext:59 - Created broadcast 214 from textFile at modelSaveLoad.scala:129 -2016-05-24 22:24:21 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:24:21 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-24 22:24:21 INFO DAGScheduler:59 - Got job 111 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-24 22:24:21 INFO DAGScheduler:59 - Final stage: ResultStage 118(first at modelSaveLoad.scala:129) -2016-05-24 22:24:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:21 INFO DAGScheduler:59 - Submitting ResultStage 118 (MapPartitionsRDD[232] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-24 22:24:21 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=1011590, maxMem=766075207 -2016-05-24 22:24:21 INFO MemoryStore:59 - Block broadcast_215 stored as values in memory (estimated size 3.1 KB, free 729.6 MB) -2016-05-24 22:24:21 INFO MemoryStore:59 - ensureFreeSpace(1898) called with curMem=1014806, maxMem=766075207 -2016-05-24 22:24:21 INFO MemoryStore:59 - Block broadcast_215_piece0 stored as bytes in memory (estimated size 1898.0 B, free 729.6 MB) -2016-05-24 22:24:21 INFO BlockManagerInfo:59 - Added broadcast_215_piece0 in memory on localhost:57000 (size: 1898.0 B, free: 730.2 MB) -2016-05-24 22:24:21 INFO SparkContext:59 - Created broadcast 215 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 118 (MapPartitionsRDD[232] at textFile at modelSaveLoad.scala:129) -2016-05-24 22:24:21 INFO TaskSchedulerImpl:59 - Adding task set 118.0 with 1 tasks -2016-05-24 22:24:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 118.0 (TID 225, localhost, PROCESS_LOCAL, 2238 bytes) -2016-05-24 22:24:21 INFO Executor:59 - Running task 0.0 in stage 118.0 (TID 225) -2016-05-24 22:24:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222418363/result/metadata/part-00000:0+54 -2016-05-24 22:24:21 INFO Executor:59 - Finished task 0.0 in stage 118.0 (TID 225). 2154 bytes result sent to driver -2016-05-24 22:24:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 118.0 (TID 225) in 12 ms on localhost (1/1) -2016-05-24 22:24:21 INFO DAGScheduler:59 - ResultStage 118 (first at modelSaveLoad.scala:129) finished in 0.013 s -2016-05-24 22:24:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 118.0, whose tasks have all completed, from pool -2016-05-24 22:24:21 INFO DAGScheduler:59 - Job 111 finished: first at modelSaveLoad.scala:129, took 0.023131 s -2016-05-24 22:24:21 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222418363/result/data on driver -2016-05-24 22:24:21 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:78 -2016-05-24 22:24:21 INFO DAGScheduler:59 - Got job 112 (parquet at GLMClassificationModel.scala:78) with 4 output partitions -2016-05-24 22:24:21 INFO DAGScheduler:59 - Final stage: ResultStage 119(parquet at GLMClassificationModel.scala:78) -2016-05-24 22:24:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:21 INFO DAGScheduler:59 - Submitting ResultStage 119 (MapPartitionsRDD[234] at parquet at GLMClassificationModel.scala:78), which has no missing parents -2016-05-24 22:24:21 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=1016704, maxMem=766075207 -2016-05-24 22:24:21 INFO MemoryStore:59 - Block broadcast_216 stored as values in memory (estimated size 61.5 KB, free 729.6 MB) -2016-05-24 22:24:21 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=1079704, maxMem=766075207 -2016-05-24 22:24:21 INFO MemoryStore:59 - Block broadcast_216_piece0 stored as bytes in memory (estimated size 20.6 KB, free 729.5 MB) -2016-05-24 22:24:21 INFO BlockManagerInfo:59 - Added broadcast_216_piece0 in memory on localhost:57000 (size: 20.6 KB, free: 730.2 MB) -2016-05-24 22:24:21 INFO SparkContext:59 - Created broadcast 216 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 119 (MapPartitionsRDD[234] at parquet at GLMClassificationModel.scala:78) -2016-05-24 22:24:21 INFO TaskSchedulerImpl:59 - Adding task set 119.0 with 4 tasks -2016-05-24 22:24:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 119.0 (TID 226, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:24:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 119.0 (TID 227, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:24:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 119.0 (TID 228, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:24:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 119.0 (TID 229, localhost, PROCESS_LOCAL, 2279 bytes) -2016-05-24 22:24:21 INFO Executor:59 - Running task 3.0 in stage 119.0 (TID 229) -2016-05-24 22:24:21 INFO Executor:59 - Running task 0.0 in stage 119.0 (TID 226) -2016-05-24 22:24:21 INFO Executor:59 - Running task 2.0 in stage 119.0 (TID 228) -2016-05-24 22:24:21 INFO Executor:59 - Running task 1.0 in stage 119.0 (TID 227) -2016-05-24 22:24:21 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:24:21 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:24:21 INFO Executor:59 - Finished task 0.0 in stage 119.0 (TID 226). 936 bytes result sent to driver -2016-05-24 22:24:21 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:24:21 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:24:21 INFO Executor:59 - Finished task 2.0 in stage 119.0 (TID 228). 936 bytes result sent to driver -2016-05-24 22:24:21 INFO Executor:59 - Finished task 1.0 in stage 119.0 (TID 227). 936 bytes result sent to driver -2016-05-24 22:24:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 119.0 (TID 226) in 93 ms on localhost (1/4) -2016-05-24 22:24:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 119.0 (TID 228) in 91 ms on localhost (2/4) -2016-05-24 22:24:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 119.0 (TID 227) in 93 ms on localhost (3/4) -2016-05-24 22:24:21 INFO Executor:59 - Finished task 3.0 in stage 119.0 (TID 229). 1802 bytes result sent to driver -2016-05-24 22:24:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 119.0 (TID 229) in 114 ms on localhost (4/4) -2016-05-24 22:24:21 INFO DAGScheduler:59 - ResultStage 119 (parquet at GLMClassificationModel.scala:78) finished in 0.116 s -2016-05-24 22:24:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 119.0, whose tasks have all completed, from pool -2016-05-24 22:24:21 INFO DAGScheduler:59 - Job 112 finished: parquet at GLMClassificationModel.scala:78, took 0.168733 s -2016-05-24 22:24:21 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=1100777, maxMem=766075207 -2016-05-24 22:24:21 INFO MemoryStore:59 - Block broadcast_217 stored as values in memory (estimated size 86.4 KB, free 729.5 MB) -2016-05-24 22:24:21 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1189289, maxMem=766075207 -2016-05-24 22:24:21 INFO MemoryStore:59 - Block broadcast_217_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.4 MB) -2016-05-24 22:24:21 INFO BlockManagerInfo:59 - Added broadcast_217_piece0 in memory on localhost:57000 (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:24:21 INFO SparkContext:59 - Created broadcast 217 from take at GLMClassificationModel.scala:79 -2016-05-24 22:24:22 INFO BlockManagerInfo:59 - Removed broadcast_216_piece0 on localhost:57000 in memory (size: 20.6 KB, free: 730.2 MB) -2016-05-24 22:24:22 INFO ContextCleaner:59 - Cleaned accumulator 115 -2016-05-24 22:24:22 INFO BlockManagerInfo:59 - Removed broadcast_215_piece0 on localhost:57000 in memory (size: 1898.0 B, free: 730.2 MB) -2016-05-24 22:24:22 INFO ContextCleaner:59 - Cleaned accumulator 114 -2016-05-24 22:24:22 INFO BlockManagerInfo:59 - Removed broadcast_214_piece0 on localhost:57000 in memory (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:24:22 INFO BlockManagerInfo:59 - Removed broadcast_213_piece0 on localhost:57000 in memory (size: 22.9 KB, free: 730.2 MB) -2016-05-24 22:24:22 INFO ContextCleaner:59 - Cleaned accumulator 113 -2016-05-24 22:24:22 INFO BlockManagerInfo:59 - Removed broadcast_212_piece0 on localhost:57000 in memory (size: 41.6 KB, free: 730.3 MB) -2016-05-24 22:24:22 INFO ContextCleaner:59 - Cleaned accumulator 112 -2016-05-24 22:24:22 INFO BlockManagerInfo:59 - Removed broadcast_211_piece0 on localhost:57000 in memory (size: 5.9 KB, free: 730.3 MB) -2016-05-24 22:24:22 INFO ContextCleaner:59 - Cleaned accumulator 111 -2016-05-24 22:24:22 INFO BlockManagerInfo:59 - Removed broadcast_210_piece0 on localhost:57000 in memory (size: 2.9 KB, free: 730.3 MB) -2016-05-24 22:24:22 INFO ContextCleaner:59 - Cleaned accumulator 110 -2016-05-24 22:24:22 INFO BlockManagerInfo:59 - Removed broadcast_209_piece0 on localhost:57000 in memory (size: 2.7 KB, free: 730.3 MB) -2016-05-24 22:24:22 INFO ContextCleaner:59 - Cleaned accumulator 109 -2016-05-24 22:24:22 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:57000 in memory (size: 2.1 KB, free: 730.3 MB) -2016-05-24 22:24:22 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 22:24:23 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-24 22:24:23 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222418363/result/data/part-r-00000-8b3d532e-0933-432a-96d9-5686ca803e5b.gz.parquet -2016-05-24 22:24:23 INFO SparkContext:59 - Starting job: take at GLMClassificationModel.scala:79 -2016-05-24 22:24:23 INFO DAGScheduler:59 - Got job 113 (take at GLMClassificationModel.scala:79) with 1 output partitions -2016-05-24 22:24:23 INFO DAGScheduler:59 - Final stage: ResultStage 120(take at GLMClassificationModel.scala:79) -2016-05-24 22:24:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:24:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:24:23 INFO DAGScheduler:59 - Submitting ResultStage 120 (MapPartitionsRDD[236] at take at GLMClassificationModel.scala:79), which has no missing parents -2016-05-24 22:24:23 INFO MemoryStore:59 - ensureFreeSpace(4168) called with curMem=570248, maxMem=766075207 -2016-05-24 22:24:23 INFO MemoryStore:59 - Block broadcast_218 stored as values in memory (estimated size 4.1 KB, free 730.0 MB) -2016-05-24 22:24:23 INFO MemoryStore:59 - ensureFreeSpace(2438) called with curMem=574416, maxMem=766075207 -2016-05-24 22:24:23 INFO MemoryStore:59 - Block broadcast_218_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.0 MB) -2016-05-24 22:24:23 INFO BlockManagerInfo:59 - Added broadcast_218_piece0 in memory on localhost:57000 (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:24:23 INFO SparkContext:59 - Created broadcast 218 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:24:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 120 (MapPartitionsRDD[236] at take at GLMClassificationModel.scala:79) -2016-05-24 22:24:23 INFO TaskSchedulerImpl:59 - Adding task set 120.0 with 1 tasks -2016-05-24 22:24:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 120.0 (TID 230, localhost, PROCESS_LOCAL, 2301 bytes) -2016-05-24 22:24:23 INFO Executor:59 - Running task 0.0 in stage 120.0 (TID 230) -2016-05-24 22:24:23 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222418363/result/data/part-r-00000-8b3d532e-0933-432a-96d9-5686ca803e5b.gz.parquet start: 0 end: 5410 length: 5410 hosts: []} -2016-05-24 22:24:23 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-24 22:24:23 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group weights { - required int32 type (INT_8); - optional int32 size; - optional group indices (LIST) { - repeated int32 array; - } - optional group values (LIST) { - repeated double array; - } - } - optional double intercept; - optional double threshold; -} - -Catalyst form: -StructType(StructField(weights,org.apache.spark.mllib.linalg.VectorUDT@f71b0bce,true), StructField(intercept,DoubleType,true), StructField(threshold,DoubleType,true)) - -2016-05-24 22:24:23 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-24 22:24:23 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-24 22:24:23 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-24 22:24:23 INFO InternalParquetRecordReader:151 - block read in memory in 323 ms. row count = 1 -2016-05-24 22:28:36 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 22:28:37 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 22:28:38 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 22:28:38 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 22:28:38 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 22:28:39 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 22:28:39 INFO Remoting:74 - Starting remoting -2016-05-24 22:28:39 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:57200] -2016-05-24 22:28:39 INFO Utils:59 - Successfully started service 'sparkDriver' on port 57200. -2016-05-24 22:28:39 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 22:28:39 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 22:28:39 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-a36dc5b1-7c47-4607-aa73-2e9af142f396 -2016-05-24 22:28:39 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 22:28:40 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-59959578-c340-4800-bf60-7eabbed05f00\httpd-567aa31d-1bfe-49f8-975f-5d4513c4d592 -2016-05-24 22:28:40 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 22:28:40 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:28:40 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:57201 -2016-05-24 22:28:40 INFO Utils:59 - Successfully started service 'HTTP file server' on port 57201. -2016-05-24 22:28:40 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 22:28:40 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:28:40 WARN AbstractLifeCycle:204 - FAILED SelectChannelConnector@0.0.0.0:4040: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.mllib.learning.regression.SVMFromSpark2Learning$.main(SVMFromSpark2Learning.scala:25) - at org.apache.spark.mllib.learning.regression.SVMFromSpark2Learning.main(SVMFromSpark2Learning.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-05-24 22:28:40 WARN AbstractLifeCycle:204 - FAILED org.spark-project.jetty.server.Server@22038fa5: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.mllib.learning.regression.SVMFromSpark2Learning$.main(SVMFromSpark2Learning.scala:25) - at org.apache.spark.mllib.learning.regression.SVMFromSpark2Learning.main(SVMFromSpark2Learning.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 22:28:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 22:28:40 WARN Utils:71 - Service 'SparkUI' could not bind on port 4040. Attempting port 4041. -2016-05-24 22:28:40 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:28:40 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4041 -2016-05-24 22:28:40 INFO Utils:59 - Successfully started service 'SparkUI' on port 4041. -2016-05-24 22:28:40 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4041 -2016-05-24 22:28:40 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 22:28:40 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 22:28:41 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 57220. -2016-05-24 22:28:41 INFO NettyBlockTransferService:59 - Server created on 57220 -2016-05-24 22:28:41 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 22:28:41 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:57220 with 730.6 MB RAM, BlockManagerId(driver, localhost, 57220) -2016-05-24 22:28:41 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 22:28:42 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 22:28:42 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 22:28:42 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 22:28:42 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 22:28:42 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:57220 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 22:28:42 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 22:28:44 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 22:28:44 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:28:44 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 22:28:44 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 22:28:44 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 22:28:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:44 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 22:28:44 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 22:28:44 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 22:28:44 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=175476, maxMem=766075207 -2016-05-24 22:28:44 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:28:44 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:57220 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:28:44 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 22:28:44 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 22:28:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:28:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:28:44 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 22:28:44 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 22:28:44 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 22:28:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 22:28:44 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 22:28:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 22:28:44 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 22:28:44 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 22:28:44 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 22:28:44 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 22:28:44 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 22:28:45 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177621, maxMem=766075207 -2016-05-24 22:28:45 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 22:28:45 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262701, maxMem=766075207 -2016-05-24 22:28:45 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 22:28:45 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:57220 (size: 83.1 KB, free: 730.5 MB) -2016-05-24 22:28:45 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:57220 (size: 83.6 KB, free: 730.4 MB) -2016-05-24 22:28:45 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 22:28:45 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 22:28:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 452 ms on localhost (1/2) -2016-05-24 22:28:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 485 ms on localhost (2/2) -2016-05-24 22:28:45 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.510 s -2016-05-24 22:28:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 22:28:45 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.731598 s -2016-05-24 22:28:45 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 22:28:45 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 22:28:45 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:28:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:45 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 22:28:45 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=348349, maxMem=766075207 -2016-05-24 22:28:45 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-24 22:28:45 INFO MemoryStore:59 - ensureFreeSpace(2483) called with curMem=352909, maxMem=766075207 -2016-05-24 22:28:45 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-24 22:28:45 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:57220 (size: 2.4 KB, free: 730.4 MB) -2016-05-24 22:28:45 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:45 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:28:45 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 22:28:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:28:45 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 22:28:45 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-05-24 22:28:45 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:28:45 INFO MemoryStore:59 - ensureFreeSpace(54392) called with curMem=355392, maxMem=766075207 -2016-05-24 22:28:45 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 53.1 KB, free 730.2 MB) -2016-05-24 22:28:45 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:57220 (size: 53.1 KB, free: 730.4 MB) -2016-05-24 22:28:45 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2611 bytes result sent to driver -2016-05-24 22:28:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 45 ms on localhost (1/1) -2016-05-24 22:28:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 22:28:45 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.047 s -2016-05-24 22:28:45 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.073586 s -2016-05-24 22:28:45 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 22:28:45 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 22:28:45 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 22:28:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:45 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 22:28:45 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=409784, maxMem=766075207 -2016-05-24 22:28:45 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-24 22:28:45 INFO MemoryStore:59 - ensureFreeSpace(2300) called with curMem=414032, maxMem=766075207 -2016-05-24 22:28:45 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 22:28:45 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:57220 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 22:28:45 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40) -2016-05-24 22:28:45 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 22:28:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:28:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:28:45 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 22:28:45 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 22:28:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:45 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-05-24 22:28:45 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:28:45 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 22:28:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 24 ms on localhost (1/2) -2016-05-24 22:28:45 INFO MemoryStore:59 - ensureFreeSpace(62184) called with curMem=416332, maxMem=766075207 -2016-05-24 22:28:45 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 60.7 KB, free 730.1 MB) -2016-05-24 22:28:45 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:57220 (size: 60.7 KB, free: 730.3 MB) -2016-05-24 22:28:45 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2662 bytes result sent to driver -2016-05-24 22:28:45 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.041 s -2016-05-24 22:28:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 39 ms on localhost (2/2) -2016-05-24 22:28:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 22:28:45 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.097198 s -2016-05-24 22:28:45 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 22:28:45 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 22:28:45 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 22:28:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:45 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 22:28:45 INFO MemoryStore:59 - ensureFreeSpace(4264) called with curMem=478516, maxMem=766075207 -2016-05-24 22:28:45 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.2 KB, free 730.1 MB) -2016-05-24 22:28:45 INFO MemoryStore:59 - ensureFreeSpace(2322) called with curMem=482780, maxMem=766075207 -2016-05-24 22:28:45 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 22:28:45 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:57220 (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:28:45 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 22:28:45 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 22:28:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:28:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:28:45 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 22:28:45 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 22:28:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:28:45 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 22:28:45 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 22:28:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 24 ms on localhost (1/2) -2016-05-24 22:28:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 24 ms on localhost (2/2) -2016-05-24 22:28:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 22:28:45 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.027 s -2016-05-24 22:28:45 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.055286 s -2016-05-24 22:28:46 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 22:28:46 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=485102, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(388) called with curMem=490694, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 388.0 B, free 730.1 MB) -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:57220 (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:28:46 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 22:28:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:28:46 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:28:46 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:46 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=491082, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=502866, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:57220 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:28:46 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:46 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 22:28:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:46 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 22:28:46 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 22:28:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:28:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:46 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 7968 bytes result sent to driver -2016-05-24 22:28:46 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 7968 bytes result sent to driver -2016-05-24 22:28:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 23 ms on localhost (1/2) -2016-05-24 22:28:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 26 ms on localhost (2/2) -2016-05-24 22:28:46 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 22:28:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 22:28:46 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.057664 s -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=506347, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(2345) called with curMem=511939, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:57220 (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:28:46 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 22:28:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:28:46 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:28:46 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:46 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=514284, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=526068, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:57220 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:28:46 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:46 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 22:28:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:46 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 22:28:46 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 22:28:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:46 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 7968 bytes result sent to driver -2016-05-24 22:28:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 21 ms on localhost (1/2) -2016-05-24 22:28:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:28:46 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 7968 bytes result sent to driver -2016-05-24 22:28:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 61 ms on localhost (2/2) -2016-05-24 22:28:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 22:28:46 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.089 s -2016-05-24 22:28:46 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.108578 s -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=529550, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(3190) called with curMem=535142, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:57220 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:28:46 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 22:28:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:28:46 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:28:46 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:46 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=538332, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=550116, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:57220 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:28:46 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:46 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 22:28:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:46 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 22:28:46 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 22:28:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:46 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 7968 bytes result sent to driver -2016-05-24 22:28:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 30 ms on localhost (1/2) -2016-05-24 22:28:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:28:46 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 7968 bytes result sent to driver -2016-05-24 22:28:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 102 ms on localhost (2/2) -2016-05-24 22:28:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 22:28:46 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.106 s -2016-05-24 22:28:46 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.128912 s -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=553600, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(3199) called with curMem=559192, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.0 MB) -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:57220 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:28:46 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 22:28:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:28:46 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:28:46 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:46 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=562391, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=574175, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:57220 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:28:46 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:46 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 22:28:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:46 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 22:28:46 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 22:28:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:28:46 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 7968 bytes result sent to driver -2016-05-24 22:28:46 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 7968 bytes result sent to driver -2016-05-24 22:28:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 18 ms on localhost (1/2) -2016-05-24 22:28:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 16 ms on localhost (2/2) -2016-05-24 22:28:46 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:28:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 22:28:46 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.061286 s -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:57220 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:28:46 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=556798, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(3214) called with curMem=559200, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.0 MB) -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:57220 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:57220 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:28:46 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:57220 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:28:46 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:57220 in memory (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:28:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:28:46 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:28:46 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:57220 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:28:46 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=523946, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:28:46 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=535730, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:57220 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:28:46 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:57220 in memory (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:28:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:46 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 22:28:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:57220 in memory (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:28:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:46 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 22:28:46 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 22:28:46 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:57220 in memory (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:28:46 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 22:28:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:46 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 7968 bytes result sent to driver -2016-05-24 22:28:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:57220 in memory (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:28:46 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 7968 bytes result sent to driver -2016-05-24 22:28:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 24 ms on localhost (1/2) -2016-05-24 22:28:46 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 22:28:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 24 ms on localhost (2/2) -2016-05-24 22:28:46 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.026 s -2016-05-24 22:28:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 22:28:46 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.046644 s -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:57220 in memory (size: 2.1 KB, free: 730.3 MB) -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=507013, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(3209) called with curMem=512605, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:57220 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:28:46 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 22:28:46 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:28:46 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:28:46 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:46 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=515814, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:28:46 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=527598, maxMem=766075207 -2016-05-24 22:28:46 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:28:46 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:57220 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:28:46 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:46 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 22:28:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:46 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:47 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 22:28:47 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 17 ms on localhost (1/2) -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:28:47 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 24 ms on localhost (2/2) -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 22:28:47 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 22:28:47 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.042616 s -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=531081, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3208) called with curMem=536673, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:57220 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 22:28:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:28:47 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=539881, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=551665, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:57220 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 22:28:47 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:47 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 19 ms on localhost (1/2) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 23 ms on localhost (2/2) -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 22:28:47 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 22:28:47 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.045045 s -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=555148, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3233) called with curMem=560740, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:57220 (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 22:28:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:28:47 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=563973, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=575757, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:57220 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 22:28:47 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:28:47 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 18 ms on localhost (1/2) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 18 ms on localhost (2/2) -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 22:28:47 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:28:47 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.047053 s -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=579241, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3237) called with curMem=584833, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:57220 (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 22:28:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:28:47 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=588070, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=599854, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:57220 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 22:28:47 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:47 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 16 ms on localhost (1/2) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 19 ms on localhost (2/2) -2016-05-24 22:28:47 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 22:28:47 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.037033 s -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=603338, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3220) called with curMem=608930, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.0 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:57220 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 22:28:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:28:47 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=612150, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=623934, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:57220 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 22:28:47 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:28:47 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 26 ms on localhost (1/2) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 23 ms on localhost (2/2) -2016-05-24 22:28:47 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 22:28:47 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.043511 s -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=627418, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3222) called with curMem=633010, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.0 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:57220 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 22:28:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:28:47 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=636232, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=648016, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:57220 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 22:28:47 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:47 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 11 ms on localhost (1/2) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 14 ms on localhost (2/2) -2016-05-24 22:28:47 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 22:28:47 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.037345 s -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=651500, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3223) called with curMem=657092, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.0 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:57220 (size: 3.1 KB, free: 730.2 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 22:28:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:28:47 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=660315, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=672099, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:57220 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 22:28:47 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:28:47 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:47 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 49 ms on localhost (1/2) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 51 ms on localhost (2/2) -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 22:28:47 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.051 s -2016-05-24 22:28:47 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.068917 s -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=675583, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3239) called with curMem=681175, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.2 KB, free 729.9 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:57220 (size: 3.2 KB, free: 730.2 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 22:28:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:28:47 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=684414, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=696198, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:57220 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 22:28:47 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:47 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:28:47 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 16 ms on localhost (1/2) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 17 ms on localhost (2/2) -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 22:28:47 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:28:47 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.067763 s -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=699681, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3233) called with curMem=705273, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.2 KB, free 729.9 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:57220 (size: 3.2 KB, free: 730.2 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 22:28:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:28:47 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=708506, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=720290, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:57220 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 22:28:47 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:28:47 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 15 ms on localhost (1/2) -2016-05-24 22:28:47 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 21 ms on localhost (2/2) -2016-05-24 22:28:47 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 22:28:47 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.066942 s -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=723773, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3226) called with curMem=729365, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 3.2 KB, free 729.9 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:57220 (size: 3.2 KB, free: 730.2 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 22:28:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:28:47 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=732591, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=744375, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:57220 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 22:28:47 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:47 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 14 ms on localhost (1/2) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 18 ms on localhost (2/2) -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 22:28:47 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 22:28:47 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.035069 s -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=747858, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3220) called with curMem=753450, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 3.1 KB, free 729.9 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:57220 (size: 3.1 KB, free: 730.2 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 22:28:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:28:47 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=756670, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=768454, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:57220 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:47 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:28:47 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 28 ms on localhost (1/2) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 29 ms on localhost (2/2) -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 22:28:47 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.032 s -2016-05-24 22:28:47 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.057079 s -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=771937, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3223) called with curMem=777529, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 3.1 KB, free 729.8 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:57220 (size: 3.1 KB, free: 730.2 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 22:28:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:28:47 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=780752, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=792536, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:57220 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:47 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 22:28:47 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:28:47 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 7968 bytes result sent to driver -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 17 ms on localhost (1/2) -2016-05-24 22:28:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 19 ms on localhost (2/2) -2016-05-24 22:28:47 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:28:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 22:28:47 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.038007 s -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=796019, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:28:47 INFO MemoryStore:59 - ensureFreeSpace(3229) called with curMem=801611, maxMem=766075207 -2016-05-24 22:28:47 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 3.2 KB, free 729.8 MB) -2016-05-24 22:28:47 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:57220 (size: 3.2 KB, free: 730.2 MB) -2016-05-24 22:28:47 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 22:28:47 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:28:47 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:28:47 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:48 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:28:48 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=804840, maxMem=766075207 -2016-05-24 22:28:48 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:28:48 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=816624, maxMem=766075207 -2016-05-24 22:28:48 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:57220 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:48 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 22:28:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:48 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 22:28:48 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 22:28:48 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:48 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:28:48 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 7968 bytes result sent to driver -2016-05-24 22:28:48 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 7968 bytes result sent to driver -2016-05-24 22:28:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 13 ms on localhost (1/2) -2016-05-24 22:28:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 14 ms on localhost (2/2) -2016-05-24 22:28:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 22:28:48 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:28:48 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.116026 s -2016-05-24 22:28:48 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=820107, maxMem=766075207 -2016-05-24 22:28:48 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 5.5 KB, free 729.8 MB) -2016-05-24 22:28:48 INFO MemoryStore:59 - ensureFreeSpace(3234) called with curMem=825699, maxMem=766075207 -2016-05-24 22:28:48 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.2 KB, free 729.8 MB) -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:57220 (size: 3.2 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 22:28:48 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:28:48 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:28:48 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:48 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:28:48 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=828933, maxMem=766075207 -2016-05-24 22:28:48 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 11.5 KB, free 729.8 MB) -2016-05-24 22:28:48 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=840717, maxMem=766075207 -2016-05-24 22:28:48 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.8 MB) -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:57220 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:28:48 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 22:28:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:28:48 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 22:28:48 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:28:48 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 7968 bytes result sent to driver -2016-05-24 22:28:48 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 22:28:48 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:28:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 17 ms on localhost (1/2) -2016-05-24 22:28:48 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 7968 bytes result sent to driver -2016-05-24 22:28:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 31 ms on localhost (2/2) -2016-05-24 22:28:48 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.031 s -2016-05-24 22:28:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 22:28:48 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.047708 s -2016-05-24 22:28:48 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 4756.985924113548, 4319.927896434357, 3903.4385808079614, 3505.841537598476, 3124.097453008626, 2756.2417647061375, 2401.155210966652, 2057.4601181932067, 1724.026901602298, 1463.7810900203021 -2016-05-24 22:28:48 INFO SparkContext:59 - Starting job: sortByKey at BinaryClassificationMetrics.scala:156 -2016-05-24 22:28:48 INFO DAGScheduler:59 - Registering RDD 50 (map at SVMFromSpark2Learning.scala:54) -2016-05-24 22:28:48 INFO DAGScheduler:59 - Got job 23 (sortByKey at BinaryClassificationMetrics.scala:156) with 2 output partitions -2016-05-24 22:28:48 INFO DAGScheduler:59 - Final stage: ResultStage 24(sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:28:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-24 22:28:48 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-24 22:28:48 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[50] at map at SVMFromSpark2Learning.scala:54), which has no missing parents -2016-05-24 22:28:48 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=844200, maxMem=766075207 -2016-05-24 22:28:48 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 10.8 KB, free 729.8 MB) -2016-05-24 22:28:48 INFO MemoryStore:59 - ensureFreeSpace(6056) called with curMem=855208, maxMem=766075207 -2016-05-24 22:28:48 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 5.9 KB, free 729.8 MB) -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:57220 (size: 5.9 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[50] at map at SVMFromSpark2Learning.scala:54) -2016-05-24 22:28:48 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 22:28:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:28:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:28:48 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 22:28:48 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 22:28:48 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:28:48 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:57220 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:57220 in memory (size: 3.2 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:57220 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:57220 in memory (size: 3.2 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:57220 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:57220 in memory (size: 3.1 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:57220 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:57220 in memory (size: 3.1 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:57220 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:57220 in memory (size: 3.2 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:57220 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:57220 in memory (size: 3.2 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:57220 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:57220 in memory (size: 3.2 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:57220 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:57220 in memory (size: 3.1 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:57220 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:57220 in memory (size: 3.1 KB, free: 730.2 MB) -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:57220 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:28:48 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:57220 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:57220 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:28:48 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:57220 in memory (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:57220 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:28:48 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:57220 in memory (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:57220 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:28:48 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:57220 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:57220 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:28:48 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:57220 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:57220 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:28:48 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:57220 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:57220 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:28:48 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 22:28:48 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:57220 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:28:49 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2254 bytes result sent to driver -2016-05-24 22:28:49 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2254 bytes result sent to driver -2016-05-24 22:28:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 900 ms on localhost (1/2) -2016-05-24 22:28:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 901 ms on localhost (2/2) -2016-05-24 22:28:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 22:28:49 INFO DAGScheduler:59 - ShuffleMapStage 23 (map at SVMFromSpark2Learning.scala:54) finished in 0.904 s -2016-05-24 22:28:49 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:28:49 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:28:49 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-24 22:28:49 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:28:49 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-24 22:28:49 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[53] at sortByKey at BinaryClassificationMetrics.scala:156), which is now runnable -2016-05-24 22:28:49 INFO MemoryStore:59 - ensureFreeSpace(3344) called with curMem=475948, maxMem=766075207 -2016-05-24 22:28:49 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-05-24 22:28:49 INFO MemoryStore:59 - ensureFreeSpace(1898) called with curMem=479292, maxMem=766075207 -2016-05-24 22:28:49 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 1898.0 B, free 730.1 MB) -2016-05-24 22:28:49 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:57220 (size: 1898.0 B, free: 730.3 MB) -2016-05-24 22:28:49 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[53] at sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:28:49 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 22:28:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:28:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:28:49 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 22:28:49 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 22:28:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:28:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:28:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-24 22:28:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-24 22:28:49 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 1501 bytes result sent to driver -2016-05-24 22:28:49 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 1485 bytes result sent to driver -2016-05-24 22:28:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 54 ms on localhost (1/2) -2016-05-24 22:28:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 54 ms on localhost (2/2) -2016-05-24 22:28:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 22:28:49 INFO DAGScheduler:59 - ResultStage 24 (sortByKey at BinaryClassificationMetrics.scala:156) finished in 0.114 s -2016-05-24 22:28:49 INFO DAGScheduler:59 - Job 23 finished: sortByKey at BinaryClassificationMetrics.scala:156, took 1.061974 s -2016-05-24 22:28:49 INFO SparkContext:59 - Starting job: collect at BinaryClassificationMetrics.scala:193 -2016-05-24 22:28:49 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-05-24 22:28:49 INFO DAGScheduler:59 - Registering RDD 51 (combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:28:49 INFO DAGScheduler:59 - Got job 24 (collect at BinaryClassificationMetrics.scala:193) with 2 output partitions -2016-05-24 22:28:49 INFO DAGScheduler:59 - Final stage: ResultStage 27(collect at BinaryClassificationMetrics.scala:193) -2016-05-24 22:28:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-24 22:28:49 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-24 22:28:49 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (ShuffledRDD[51] at combineByKey at BinaryClassificationMetrics.scala:152), which has no missing parents -2016-05-24 22:28:49 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=481190, maxMem=766075207 -2016-05-24 22:28:49 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 2.8 KB, free 730.1 MB) -2016-05-24 22:28:49 INFO MemoryStore:59 - ensureFreeSpace(1658) called with curMem=484086, maxMem=766075207 -2016-05-24 22:28:49 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 1658.0 B, free 730.1 MB) -2016-05-24 22:28:49 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:57220 (size: 1658.0 B, free: 730.3 MB) -2016-05-24 22:28:49 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 26 (ShuffledRDD[51] at combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:28:49 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 22:28:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:28:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:28:49 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-24 22:28:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:28:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:28:49 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-24 22:28:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:28:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:28:49 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1375 bytes result sent to driver -2016-05-24 22:28:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 71 ms on localhost (1/2) -2016-05-24 22:28:49 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1375 bytes result sent to driver -2016-05-24 22:28:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 81 ms on localhost (2/2) -2016-05-24 22:28:49 INFO DAGScheduler:59 - ShuffleMapStage 26 (combineByKey at BinaryClassificationMetrics.scala:152) finished in 0.083 s -2016-05-24 22:28:49 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:28:49 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:28:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 22:28:49 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-24 22:28:49 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:28:49 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-24 22:28:49 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[56] at mapPartitions at BinaryClassificationMetrics.scala:189), which is now runnable -2016-05-24 22:28:49 INFO MemoryStore:59 - ensureFreeSpace(3472) called with curMem=485744, maxMem=766075207 -2016-05-24 22:28:49 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:28:49 INFO MemoryStore:59 - ensureFreeSpace(1983) called with curMem=489216, maxMem=766075207 -2016-05-24 22:28:49 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 1983.0 B, free 730.1 MB) -2016-05-24 22:28:49 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:57220 (size: 1983.0 B, free: 730.3 MB) -2016-05-24 22:28:49 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[56] at mapPartitions at BinaryClassificationMetrics.scala:189) -2016-05-24 22:28:49 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 22:28:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 51, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:28:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 52, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:28:49 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 52) -2016-05-24 22:28:49 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 51) -2016-05-24 22:28:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:28:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:28:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:28:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:28:49 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 51). 1330 bytes result sent to driver -2016-05-24 22:28:49 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 52). 1330 bytes result sent to driver -2016-05-24 22:28:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 51) in 21 ms on localhost (1/2) -2016-05-24 22:28:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 52) in 21 ms on localhost (2/2) -2016-05-24 22:28:49 INFO DAGScheduler:59 - ResultStage 27 (collect at BinaryClassificationMetrics.scala:193) finished in 0.023 s -2016-05-24 22:28:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 22:28:49 INFO DAGScheduler:59 - Job 24 finished: collect at BinaryClassificationMetrics.scala:193, took 0.139988 s -2016-05-24 22:28:49 INFO BinaryClassificationMetrics:59 - Total counts: {numPos: 25, numNeg: 11} -2016-05-24 22:28:49 INFO SparkContext:59 - Starting job: runJob at SlidingRDD.scala:75 -2016-05-24 22:28:49 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-05-24 22:28:49 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-24 22:28:49 INFO DAGScheduler:59 - Got job 25 (runJob at SlidingRDD.scala:75) with 3 output partitions -2016-05-24 22:28:49 INFO DAGScheduler:59 - Final stage: ResultStage 30(runJob at SlidingRDD.scala:75) -2016-05-24 22:28:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-24 22:28:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:49 INFO DAGScheduler:59 - Submitting ResultStage 30 (UnionRDD[62] at UnionRDD at BinaryClassificationMetrics.scala:92), which has no missing parents -2016-05-24 22:28:49 INFO MemoryStore:59 - ensureFreeSpace(5184) called with curMem=491199, maxMem=766075207 -2016-05-24 22:28:49 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 5.1 KB, free 730.1 MB) -2016-05-24 22:28:49 INFO MemoryStore:59 - ensureFreeSpace(2788) called with curMem=496383, maxMem=766075207 -2016-05-24 22:28:49 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.1 MB) -2016-05-24 22:28:49 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:57220 (size: 2.7 KB, free: 730.3 MB) -2016-05-24 22:28:49 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:49 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 30 (UnionRDD[62] at UnionRDD at BinaryClassificationMetrics.scala:92) -2016-05-24 22:28:49 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 3 tasks -2016-05-24 22:28:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 53, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:28:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:28:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 2318 bytes) -2016-05-24 22:28:49 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 54) -2016-05-24 22:28:49 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 53) -2016-05-24 22:28:49 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 55) -2016-05-24 22:28:49 INFO CacheManager:59 - Partition rdd_57_1 not found, computing it -2016-05-24 22:28:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:28:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:28:49 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 55). 1053 bytes result sent to driver -2016-05-24 22:28:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 55) in 14 ms on localhost (1/3) -2016-05-24 22:28:49 INFO CacheManager:59 - Partition rdd_57_0 not found, computing it -2016-05-24 22:28:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:28:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:28:49 INFO MemoryStore:59 - ensureFreeSpace(1616) called with curMem=499171, maxMem=766075207 -2016-05-24 22:28:49 INFO MemoryStore:59 - Block rdd_57_1 stored as values in memory (estimated size 1616.0 B, free 730.1 MB) -2016-05-24 22:28:49 INFO BlockManagerInfo:59 - Added rdd_57_1 in memory on localhost:57220 (size: 1616.0 B, free: 730.3 MB) -2016-05-24 22:28:49 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 54). 1883 bytes result sent to driver -2016-05-24 22:28:49 INFO MemoryStore:59 - ensureFreeSpace(1448) called with curMem=500787, maxMem=766075207 -2016-05-24 22:28:49 INFO MemoryStore:59 - Block rdd_57_0 stored as values in memory (estimated size 1448.0 B, free 730.1 MB) -2016-05-24 22:28:49 INFO BlockManagerInfo:59 - Added rdd_57_0 in memory on localhost:57220 (size: 1448.0 B, free: 730.3 MB) -2016-05-24 22:28:49 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 53). 1883 bytes result sent to driver -2016-05-24 22:28:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 54) in 33 ms on localhost (2/3) -2016-05-24 22:28:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 53) in 37 ms on localhost (3/3) -2016-05-24 22:28:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 22:28:49 INFO DAGScheduler:59 - ResultStage 30 (runJob at SlidingRDD.scala:75) finished in 0.038 s -2016-05-24 22:28:49 INFO DAGScheduler:59 - Job 25 finished: runJob at SlidingRDD.scala:75, took 0.066010 s -2016-05-24 22:28:49 INFO SparkContext:59 - Starting job: aggregate at AreaUnderCurve.scala:45 -2016-05-24 22:28:49 INFO DAGScheduler:59 - Got job 26 (aggregate at AreaUnderCurve.scala:45) with 4 output partitions -2016-05-24 22:28:49 INFO DAGScheduler:59 - Final stage: ResultStage 33(aggregate at AreaUnderCurve.scala:45) -2016-05-24 22:28:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-24 22:28:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:49 INFO DAGScheduler:59 - Submitting ResultStage 33 (SlidingRDD[63] at RDD at SlidingRDD.scala:47), which has no missing parents -2016-05-24 22:28:49 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=502235, maxMem=766075207 -2016-05-24 22:28:49 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:28:49 INFO MemoryStore:59 - ensureFreeSpace(2972) called with curMem=507891, maxMem=766075207 -2016-05-24 22:28:49 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.1 MB) -2016-05-24 22:28:49 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:57220 (size: 2.9 KB, free: 730.3 MB) -2016-05-24 22:28:49 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:49 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (SlidingRDD[63] at RDD at SlidingRDD.scala:47) -2016-05-24 22:28:49 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-24 22:28:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 56, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:28:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 57, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:28:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 2635 bytes) -2016-05-24 22:28:49 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 2475 bytes) -2016-05-24 22:28:49 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 56) -2016-05-24 22:28:49 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 59) -2016-05-24 22:28:49 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 57) -2016-05-24 22:28:49 INFO BlockManager:59 - Found block rdd_57_0 locally -2016-05-24 22:28:49 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 59). 955 bytes result sent to driver -2016-05-24 22:28:49 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-24 22:28:49 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 56). 2084 bytes result sent to driver -2016-05-24 22:28:49 INFO BlockManager:59 - Found block rdd_57_1 locally -2016-05-24 22:28:49 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 59) in 15 ms on localhost (1/4) -2016-05-24 22:28:49 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 955 bytes result sent to driver -2016-05-24 22:28:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 56) in 28 ms on localhost (2/4) -2016-05-24 22:28:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 23 ms on localhost (3/4) -2016-05-24 22:28:49 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 57). 2084 bytes result sent to driver -2016-05-24 22:28:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 57) in 27 ms on localhost (4/4) -2016-05-24 22:28:49 INFO DAGScheduler:59 - ResultStage 33 (aggregate at AreaUnderCurve.scala:45) finished in 0.031 s -2016-05-24 22:28:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 22:28:49 INFO DAGScheduler:59 - Job 26 finished: aggregate at AreaUnderCurve.scala:45, took 0.046844 s -2016-05-24 22:28:49 INFO SparkContext:59 - Starting job: take at SVMFromSpark2Learning.scala:66 -2016-05-24 22:28:49 INFO DAGScheduler:59 - Got job 27 (take at SVMFromSpark2Learning.scala:66) with 1 output partitions -2016-05-24 22:28:49 INFO DAGScheduler:59 - Final stage: ResultStage 34(take at SVMFromSpark2Learning.scala:66) -2016-05-24 22:28:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:49 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[50] at map at SVMFromSpark2Learning.scala:54), which has no missing parents -2016-05-24 22:28:49 INFO MemoryStore:59 - ensureFreeSpace(10320) called with curMem=510863, maxMem=766075207 -2016-05-24 22:28:49 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 10.1 KB, free 730.1 MB) -2016-05-24 22:28:49 INFO MemoryStore:59 - ensureFreeSpace(5705) called with curMem=521183, maxMem=766075207 -2016-05-24 22:28:49 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-24 22:28:49 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:57220 (size: 5.6 KB, free: 730.3 MB) -2016-05-24 22:28:49 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:49 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 34 (MapPartitionsRDD[50] at map at SVMFromSpark2Learning.scala:54) -2016-05-24 22:28:49 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 1 tasks -2016-05-24 22:28:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 60, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:28:49 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 60) -2016-05-24 22:28:49 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:28:49 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 60). 2401 bytes result sent to driver -2016-05-24 22:28:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 60) in 9 ms on localhost (1/1) -2016-05-24 22:28:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 22:28:49 INFO DAGScheduler:59 - ResultStage 34 (take at SVMFromSpark2Learning.scala:66) finished in 0.009 s -2016-05-24 22:28:49 INFO DAGScheduler:59 - Job 27 finished: take at SVMFromSpark2Learning.scala:66, took 0.048284 s -2016-05-24 22:28:50 INFO SparkContext:59 - Starting job: saveAsTextFile at GLMClassificationModel.scala:61 -2016-05-24 22:28:50 INFO DAGScheduler:59 - Got job 28 (saveAsTextFile at GLMClassificationModel.scala:61) with 1 output partitions -2016-05-24 22:28:50 INFO DAGScheduler:59 - Final stage: ResultStage 35(saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:28:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:50 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[65] at saveAsTextFile at GLMClassificationModel.scala:61), which has no missing parents -2016-05-24 22:28:50 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=526888, maxMem=766075207 -2016-05-24 22:28:50 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 124.0 KB, free 730.0 MB) -2016-05-24 22:28:50 INFO MemoryStore:59 - ensureFreeSpace(42574) called with curMem=653880, maxMem=766075207 -2016-05-24 22:28:50 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 41.6 KB, free 729.9 MB) -2016-05-24 22:28:50 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:57220 (size: 41.6 KB, free: 730.2 MB) -2016-05-24 22:28:50 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 35 (MapPartitionsRDD[65] at saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:28:50 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 1 tasks -2016-05-24 22:28:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 61, localhost, PROCESS_LOCAL, 2197 bytes) -2016-05-24 22:28:50 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 61) -2016-05-24 22:28:50 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242228_0035_m_000000_61' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222849752/result/metadata/_temporary/0/task_201605242228_0035_m_000000 -2016-05-24 22:28:50 INFO SparkHadoopMapRedUtil:59 - attempt_201605242228_0035_m_000000_61: Committed -2016-05-24 22:28:50 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 61). 1864 bytes result sent to driver -2016-05-24 22:28:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 61) in 200 ms on localhost (1/1) -2016-05-24 22:28:50 INFO DAGScheduler:59 - ResultStage 35 (saveAsTextFile at GLMClassificationModel.scala:61) finished in 0.201 s -2016-05-24 22:28:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 22:28:50 INFO DAGScheduler:59 - Job 28 finished: saveAsTextFile at GLMClassificationModel.scala:61, took 0.293939 s -2016-05-24 22:28:52 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:28:52 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:28:52 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:65 -2016-05-24 22:28:52 INFO DAGScheduler:59 - Got job 29 (parquet at GLMClassificationModel.scala:65) with 1 output partitions -2016-05-24 22:28:52 INFO DAGScheduler:59 - Final stage: ResultStage 36(parquet at GLMClassificationModel.scala:65) -2016-05-24 22:28:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:52 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[67] at rddToDataFrameHolder at GLMClassificationModel.scala:65), which has no missing parents -2016-05-24 22:28:52 INFO MemoryStore:59 - ensureFreeSpace(67104) called with curMem=696454, maxMem=766075207 -2016-05-24 22:28:52 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 65.5 KB, free 729.9 MB) -2016-05-24 22:28:52 INFO MemoryStore:59 - ensureFreeSpace(23439) called with curMem=763558, maxMem=766075207 -2016-05-24 22:28:52 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 22.9 KB, free 729.8 MB) -2016-05-24 22:28:52 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:57220 (size: 22.9 KB, free: 730.2 MB) -2016-05-24 22:28:52 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:52 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[67] at rddToDataFrameHolder at GLMClassificationModel.scala:65) -2016-05-24 22:28:52 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-24 22:28:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 62, localhost, PROCESS_LOCAL, 7978 bytes) -2016-05-24 22:28:52 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 62) -2016-05-24 22:28:52 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:28:52 INFO CodecConfig:151 - Compression: GZIP -2016-05-24 22:28:52 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-24 22:28:52 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-24 22:28:52 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-24 22:28:52 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-24 22:28:52 INFO ParquetOutputFormat:151 - Validation is off -2016-05-24 22:28:52 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-24 22:28:53 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:57220 in memory (size: 41.6 KB, free: 730.3 MB) -2016-05-24 22:28:53 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 22:28:53 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:57220 in memory (size: 5.6 KB, free: 730.3 MB) -2016-05-24 22:28:53 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 22:28:53 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:57220 in memory (size: 2.9 KB, free: 730.3 MB) -2016-05-24 22:28:53 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 22:28:53 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:57220 in memory (size: 2.7 KB, free: 730.3 MB) -2016-05-24 22:28:53 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 22:28:53 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:57220 in memory (size: 1983.0 B, free: 730.3 MB) -2016-05-24 22:28:53 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 22:28:53 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:57220 in memory (size: 1658.0 B, free: 730.3 MB) -2016-05-24 22:28:53 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 22:28:53 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:57220 in memory (size: 1898.0 B, free: 730.3 MB) -2016-05-24 22:28:53 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 22:28:53 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:57220 in memory (size: 5.9 KB, free: 730.3 MB) -2016-05-24 22:28:53 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-24 22:28:53 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 22:28:53 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 6,084 -2016-05-24 22:28:53 INFO ColumnChunkPageWriteStore:151 - written 61B for [weights, type] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-24 22:28:53 INFO ColumnChunkPageWriteStore:151 - written 48B for [weights, size] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-24 22:28:53 INFO ColumnChunkPageWriteStore:151 - written 51B for [weights, indices, array] INT32: 1 values, 13B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-24 22:28:53 INFO ColumnChunkPageWriteStore:151 - written 635B for [weights, values, array] DOUBLE: 692 values, 668B raw, 591B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 404 entries, 3,232B raw, 404B comp} -2016-05-24 22:28:53 INFO ColumnChunkPageWriteStore:151 - written 69B for [intercept] DOUBLE: 1 values, 14B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-24 22:28:53 INFO ColumnChunkPageWriteStore:151 - written 47B for [threshold] DOUBLE: 1 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-24 22:28:53 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242228_0036_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222849752/result/data/_temporary/0/task_201605242228_0036_m_000000 -2016-05-24 22:28:53 INFO SparkHadoopMapRedUtil:59 - attempt_201605242228_0036_m_000000_0: Committed -2016-05-24 22:28:53 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 62). 915 bytes result sent to driver -2016-05-24 22:28:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 62) in 707 ms on localhost (1/1) -2016-05-24 22:28:53 INFO DAGScheduler:59 - ResultStage 36 (parquet at GLMClassificationModel.scala:65) finished in 0.708 s -2016-05-24 22:28:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 22:28:53 INFO DAGScheduler:59 - Job 29 finished: parquet at GLMClassificationModel.scala:65, took 0.782002 s -2016-05-24 22:28:53 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:28:53 INFO DefaultWriterContainer:59 - Job job_201605242228_0000 committed. -2016-05-24 22:28:53 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222849752/result/data on driver -2016-05-24 22:28:53 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222849752/result/data on driver -2016-05-24 22:28:54 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=552491, maxMem=766075207 -2016-05-24 22:28:54 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 225.3 KB, free 729.8 MB) -2016-05-24 22:28:54 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=783195, maxMem=766075207 -2016-05-24 22:28:54 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.8 MB) -2016-05-24 22:28:54 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:57220 (size: 19.3 KB, free: 730.3 MB) -2016-05-24 22:28:54 INFO SparkContext:59 - Created broadcast 52 from textFile at modelSaveLoad.scala:129 -2016-05-24 22:28:54 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:28:54 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-24 22:28:54 INFO DAGScheduler:59 - Got job 30 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-24 22:28:54 INFO DAGScheduler:59 - Final stage: ResultStage 37(first at modelSaveLoad.scala:129) -2016-05-24 22:28:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:54 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[70] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-24 22:28:54 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=802983, maxMem=766075207 -2016-05-24 22:28:54 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 3.1 KB, free 729.8 MB) -2016-05-24 22:28:54 INFO MemoryStore:59 - ensureFreeSpace(1894) called with curMem=806199, maxMem=766075207 -2016-05-24 22:28:54 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 1894.0 B, free 729.8 MB) -2016-05-24 22:28:54 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:57220 (size: 1894.0 B, free: 730.3 MB) -2016-05-24 22:28:54 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:54 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 37 (MapPartitionsRDD[70] at textFile at modelSaveLoad.scala:129) -2016-05-24 22:28:54 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 1 tasks -2016-05-24 22:28:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 63, localhost, PROCESS_LOCAL, 2238 bytes) -2016-05-24 22:28:54 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 63) -2016-05-24 22:28:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222849752/result/metadata/part-00000:0+54 -2016-05-24 22:28:54 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 63). 2154 bytes result sent to driver -2016-05-24 22:28:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 63) in 13 ms on localhost (1/1) -2016-05-24 22:28:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 22:28:54 INFO DAGScheduler:59 - ResultStage 37 (first at modelSaveLoad.scala:129) finished in 0.021 s -2016-05-24 22:28:54 INFO DAGScheduler:59 - Job 30 finished: first at modelSaveLoad.scala:129, took 0.031617 s -2016-05-24 22:28:54 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222849752/result/data on driver -2016-05-24 22:28:54 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:78 -2016-05-24 22:28:54 INFO DAGScheduler:59 - Got job 31 (parquet at GLMClassificationModel.scala:78) with 4 output partitions -2016-05-24 22:28:54 INFO DAGScheduler:59 - Final stage: ResultStage 38(parquet at GLMClassificationModel.scala:78) -2016-05-24 22:28:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:54 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[72] at parquet at GLMClassificationModel.scala:78), which has no missing parents -2016-05-24 22:28:54 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=808093, maxMem=766075207 -2016-05-24 22:28:54 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 61.5 KB, free 729.8 MB) -2016-05-24 22:28:54 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=871093, maxMem=766075207 -2016-05-24 22:28:54 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 20.6 KB, free 729.7 MB) -2016-05-24 22:28:54 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:57220 (size: 20.6 KB, free: 730.2 MB) -2016-05-24 22:28:54 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:54 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[72] at parquet at GLMClassificationModel.scala:78) -2016-05-24 22:28:54 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-24 22:28:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 64, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:28:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 65, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:28:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 66, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:28:54 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 67, localhost, PROCESS_LOCAL, 2279 bytes) -2016-05-24 22:28:54 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 67) -2016-05-24 22:28:54 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 66) -2016-05-24 22:28:54 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 64) -2016-05-24 22:28:54 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:28:54 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 66). 936 bytes result sent to driver -2016-05-24 22:28:54 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 65) -2016-05-24 22:28:54 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:28:54 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 64). 936 bytes result sent to driver -2016-05-24 22:28:54 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:28:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 66) in 130 ms on localhost (1/4) -2016-05-24 22:28:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 64) in 133 ms on localhost (2/4) -2016-05-24 22:28:54 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:28:54 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 65). 936 bytes result sent to driver -2016-05-24 22:28:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 65) in 152 ms on localhost (3/4) -2016-05-24 22:28:54 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 67). 1802 bytes result sent to driver -2016-05-24 22:28:54 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 67) in 157 ms on localhost (4/4) -2016-05-24 22:28:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 22:28:54 INFO DAGScheduler:59 - ResultStage 38 (parquet at GLMClassificationModel.scala:78) finished in 0.160 s -2016-05-24 22:28:54 INFO DAGScheduler:59 - Job 31 finished: parquet at GLMClassificationModel.scala:78, took 0.197649 s -2016-05-24 22:28:55 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=892166, maxMem=766075207 -2016-05-24 22:28:55 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 86.4 KB, free 729.7 MB) -2016-05-24 22:28:55 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=980678, maxMem=766075207 -2016-05-24 22:28:55 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.6 MB) -2016-05-24 22:28:55 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:57220 (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:28:55 INFO SparkContext:59 - Created broadcast 55 from take at GLMClassificationModel.scala:79 -2016-05-24 22:28:55 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 22:28:55 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:57220 in memory (size: 20.6 KB, free: 730.2 MB) -2016-05-24 22:28:55 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 22:28:55 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-24 22:28:55 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222849752/result/data/part-r-00000-f893e64a-8a41-4adc-9187-b5852378272c.gz.parquet -2016-05-24 22:28:55 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:57220 in memory (size: 1894.0 B, free: 730.2 MB) -2016-05-24 22:28:56 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 22:28:56 INFO SparkContext:59 - Starting job: take at GLMClassificationModel.scala:79 -2016-05-24 22:28:56 INFO DAGScheduler:59 - Got job 32 (take at GLMClassificationModel.scala:79) with 1 output partitions -2016-05-24 22:28:56 INFO DAGScheduler:59 - Final stage: ResultStage 39(take at GLMClassificationModel.scala:79) -2016-05-24 22:28:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:28:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:28:56 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[74] at take at GLMClassificationModel.scala:79), which has no missing parents -2016-05-24 22:28:56 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:57220 in memory (size: 19.3 KB, free: 730.3 MB) -2016-05-24 22:28:56 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:57220 in memory (size: 22.9 KB, free: 730.3 MB) -2016-05-24 22:28:56 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 22:28:56 INFO MemoryStore:59 - ensureFreeSpace(4168) called with curMem=570248, maxMem=766075207 -2016-05-24 22:28:56 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 4.1 KB, free 730.0 MB) -2016-05-24 22:28:56 INFO MemoryStore:59 - ensureFreeSpace(2438) called with curMem=574416, maxMem=766075207 -2016-05-24 22:28:56 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.0 MB) -2016-05-24 22:28:56 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:57220 (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:28:56 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:28:56 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 39 (MapPartitionsRDD[74] at take at GLMClassificationModel.scala:79) -2016-05-24 22:28:56 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 1 tasks -2016-05-24 22:28:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 68, localhost, PROCESS_LOCAL, 2301 bytes) -2016-05-24 22:28:56 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 68) -2016-05-24 22:28:56 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222849752/result/data/part-r-00000-f893e64a-8a41-4adc-9187-b5852378272c.gz.parquet start: 0 end: 4863 length: 4863 hosts: []} -2016-05-24 22:28:56 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-24 22:28:56 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group weights { - required int32 type (INT_8); - optional int32 size; - optional group indices (LIST) { - repeated int32 array; - } - optional group values (LIST) { - repeated double array; - } - } - optional double intercept; - optional double threshold; -} - -Catalyst form: -StructType(StructField(weights,org.apache.spark.mllib.linalg.VectorUDT@f71b0bce,true), StructField(intercept,DoubleType,true), StructField(threshold,DoubleType,true)) - -2016-05-24 22:28:56 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-24 22:28:56 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-24 22:28:56 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-24 22:28:56 INFO InternalParquetRecordReader:151 - block read in memory in 13 ms. row count = 1 -2016-05-24 22:28:58 ERROR Executor:96 - Exception in task 0.0 in stage 39.0 (TID 68) -java.lang.OutOfMemoryError: PermGen space - at java.lang.ClassLoader.defineClass1(Native Method) - at java.lang.ClassLoader.defineClass(Unknown Source) - at java.security.SecureClassLoader.defineClass(Unknown Source) - at java.net.URLClassLoader.defineClass(Unknown Source) - at java.net.URLClassLoader.access$100(Unknown Source) - at java.net.URLClassLoader$1.run(Unknown Source) - at java.net.URLClassLoader$1.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at java.net.URLClassLoader.findClass(Unknown Source) - at java.lang.ClassLoader.loadClass(Unknown Source) - at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source) - at java.lang.ClassLoader.loadClass(Unknown Source) - at java.lang.Class.getDeclaredMethods0(Native Method) - at java.lang.Class.privateGetDeclaredMethods(Unknown Source) - at java.lang.Class.getDeclaredMethod(Unknown Source) - at java.io.ObjectStreamClass.getPrivateMethod(Unknown Source) - at java.io.ObjectStreamClass.access$1700(Unknown Source) - at java.io.ObjectStreamClass$2.run(Unknown Source) - at java.io.ObjectStreamClass$2.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at java.io.ObjectStreamClass.(Unknown Source) - at java.io.ObjectStreamClass.lookup(Unknown Source) - at java.io.ObjectOutputStream.writeObject0(Unknown Source) - at java.io.ObjectOutputStream.writeArray(Unknown Source) - at java.io.ObjectOutputStream.writeObject0(Unknown Source) - at java.io.ObjectOutputStream.writeObject(Unknown Source) - at org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:44) - at org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:84) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:240) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-05-24 22:29:32 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 22:29:33 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 22:29:33 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 22:29:33 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 22:29:33 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 22:29:34 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 22:29:34 INFO Remoting:74 - Starting remoting -2016-05-24 22:29:35 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:57240] -2016-05-24 22:29:35 INFO Utils:59 - Successfully started service 'sparkDriver' on port 57240. -2016-05-24 22:29:35 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 22:29:35 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 22:29:35 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-0909b956-6a30-4111-8e65-e97e39fe83ed -2016-05-24 22:29:35 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 22:29:35 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-25afd4ca-33b5-4a1a-8774-36204672b632\httpd-bc714b3a-9d72-4189-bc1d-97db41b6ec39 -2016-05-24 22:29:35 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 22:29:35 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:29:35 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:57241 -2016-05-24 22:29:35 INFO Utils:59 - Successfully started service 'HTTP file server' on port 57241. -2016-05-24 22:29:35 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 22:29:35 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:29:35 WARN AbstractLifeCycle:204 - FAILED SelectChannelConnector@0.0.0.0:4040: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.mllib.learning.regression.SVMFromSpark2Learning$.main(SVMFromSpark2Learning.scala:25) - at org.apache.spark.mllib.learning.regression.SVMFromSpark2Learning.main(SVMFromSpark2Learning.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-05-24 22:29:35 WARN AbstractLifeCycle:204 - FAILED org.spark-project.jetty.server.Server@5a26b058: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.mllib.learning.regression.SVMFromSpark2Learning$.main(SVMFromSpark2Learning.scala:25) - at org.apache.spark.mllib.learning.regression.SVMFromSpark2Learning.main(SVMFromSpark2Learning.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 22:29:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 22:29:35 WARN Utils:71 - Service 'SparkUI' could not bind on port 4040. Attempting port 4041. -2016-05-24 22:29:35 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:29:35 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4041 -2016-05-24 22:29:35 INFO Utils:59 - Successfully started service 'SparkUI' on port 4041. -2016-05-24 22:29:35 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4041 -2016-05-24 22:29:36 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 22:29:36 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 22:29:36 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 57260. -2016-05-24 22:29:36 INFO NettyBlockTransferService:59 - Server created on 57260 -2016-05-24 22:29:36 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 22:29:36 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:57260 with 730.6 MB RAM, BlockManagerId(driver, localhost, 57260) -2016-05-24 22:29:36 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 22:29:37 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 22:29:37 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 22:29:37 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 22:29:37 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 22:29:37 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:57260 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 22:29:37 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 22:29:39 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 22:29:39 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:29:39 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 22:29:39 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 22:29:39 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 22:29:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:39 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 22:29:39 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 22:29:39 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 22:29:39 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=175476, maxMem=766075207 -2016-05-24 22:29:39 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:29:39 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:57260 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:29:39 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 22:29:39 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 22:29:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:29:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:29:39 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 22:29:39 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 22:29:39 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 22:29:39 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 22:29:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 22:29:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 22:29:39 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 22:29:39 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 22:29:39 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 22:29:39 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 22:29:39 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 22:29:40 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177621, maxMem=766075207 -2016-05-24 22:29:40 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 22:29:40 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:57260 (size: 83.6 KB, free: 730.5 MB) -2016-05-24 22:29:40 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263269, maxMem=766075207 -2016-05-24 22:29:40 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 22:29:40 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:57260 (size: 83.1 KB, free: 730.4 MB) -2016-05-24 22:29:40 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 22:29:40 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 22:29:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 512 ms on localhost (1/2) -2016-05-24 22:29:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 656 ms on localhost (2/2) -2016-05-24 22:29:40 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.658 s -2016-05-24 22:29:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 22:29:40 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.943073 s -2016-05-24 22:29:40 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 22:29:40 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 22:29:40 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:29:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:40 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 22:29:40 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=348349, maxMem=766075207 -2016-05-24 22:29:40 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-24 22:29:40 INFO MemoryStore:59 - ensureFreeSpace(2483) called with curMem=352909, maxMem=766075207 -2016-05-24 22:29:40 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-24 22:29:40 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:57260 (size: 2.4 KB, free: 730.4 MB) -2016-05-24 22:29:40 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:40 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:29:40 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 22:29:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:29:40 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 22:29:40 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-05-24 22:29:40 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:29:40 INFO MemoryStore:59 - ensureFreeSpace(54392) called with curMem=355392, maxMem=766075207 -2016-05-24 22:29:40 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 53.1 KB, free 730.2 MB) -2016-05-24 22:29:40 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:57260 (size: 53.1 KB, free: 730.4 MB) -2016-05-24 22:29:40 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2611 bytes result sent to driver -2016-05-24 22:29:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 40 ms on localhost (1/1) -2016-05-24 22:29:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 22:29:40 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.037 s -2016-05-24 22:29:40 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.112290 s -2016-05-24 22:29:40 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 22:29:40 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 22:29:40 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 22:29:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:40 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 22:29:40 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=409784, maxMem=766075207 -2016-05-24 22:29:40 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-24 22:29:40 INFO MemoryStore:59 - ensureFreeSpace(2300) called with curMem=414032, maxMem=766075207 -2016-05-24 22:29:40 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 22:29:40 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:57260 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 22:29:40 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40) -2016-05-24 22:29:40 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 22:29:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:29:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:29:40 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 22:29:40 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 22:29:40 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:40 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 22:29:40 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-05-24 22:29:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 59 ms on localhost (1/2) -2016-05-24 22:29:40 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:29:40 INFO MemoryStore:59 - ensureFreeSpace(62184) called with curMem=416332, maxMem=766075207 -2016-05-24 22:29:40 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 60.7 KB, free 730.1 MB) -2016-05-24 22:29:40 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:57260 (size: 60.7 KB, free: 730.3 MB) -2016-05-24 22:29:40 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2662 bytes result sent to driver -2016-05-24 22:29:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 85 ms on localhost (2/2) -2016-05-24 22:29:40 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.087 s -2016-05-24 22:29:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 22:29:40 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.105028 s -2016-05-24 22:29:40 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 22:29:40 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 22:29:40 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 22:29:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:40 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 22:29:40 INFO MemoryStore:59 - ensureFreeSpace(4264) called with curMem=478516, maxMem=766075207 -2016-05-24 22:29:40 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.2 KB, free 730.1 MB) -2016-05-24 22:29:40 INFO MemoryStore:59 - ensureFreeSpace(2322) called with curMem=482780, maxMem=766075207 -2016-05-24 22:29:40 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 22:29:40 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:57260 (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:29:40 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 22:29:40 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 22:29:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:29:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:29:40 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 22:29:40 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 22:29:40 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:29:40 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 22:29:40 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:40 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 22:29:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 24 ms on localhost (1/2) -2016-05-24 22:29:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 25 ms on localhost (2/2) -2016-05-24 22:29:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 22:29:40 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.027 s -2016-05-24 22:29:40 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.047481 s -2016-05-24 22:29:41 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 22:29:41 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=485102, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(388) called with curMem=490694, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 388.0 B, free 730.1 MB) -2016-05-24 22:29:41 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:57260 (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:29:41 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 22:29:41 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:57260 in memory (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:29:41 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 22:29:41 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:57260 in memory (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:29:41 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 22:29:41 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:57260 in memory (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:29:41 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 22:29:41 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:57260 in memory (size: 2.1 KB, free: 730.3 MB) -2016-05-24 22:29:41 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 22:29:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:29:41 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:29:41 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:41 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=464864, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=476648, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:29:41 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:57260 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:41 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:41 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 22:29:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:41 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 22:29:41 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:29:41 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 7968 bytes result sent to driver -2016-05-24 22:29:41 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 22:29:41 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:41 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 7968 bytes result sent to driver -2016-05-24 22:29:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 72 ms on localhost (1/2) -2016-05-24 22:29:41 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.077 s -2016-05-24 22:29:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 76 ms on localhost (2/2) -2016-05-24 22:29:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 22:29:41 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.118863 s -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=480129, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(2345) called with curMem=485721, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 22:29:41 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:57260 (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:29:41 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 22:29:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:29:41 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:29:41 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:41 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=488066, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=499850, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:29:41 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:57260 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:41 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:41 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 22:29:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:41 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 22:29:41 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 22:29:41 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:41 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:29:41 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 7968 bytes result sent to driver -2016-05-24 22:29:41 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 7968 bytes result sent to driver -2016-05-24 22:29:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 24 ms on localhost (1/2) -2016-05-24 22:29:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 25 ms on localhost (2/2) -2016-05-24 22:29:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 22:29:41 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.034 s -2016-05-24 22:29:41 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.060876 s -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=503333, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(3190) called with curMem=508925, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 22:29:41 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:57260 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:29:41 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 22:29:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:29:41 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:29:41 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:41 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=512115, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=523899, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:29:41 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:57260 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:41 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:41 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 22:29:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:41 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 22:29:41 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:29:41 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 7968 bytes result sent to driver -2016-05-24 22:29:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 39 ms on localhost (1/2) -2016-05-24 22:29:41 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 22:29:41 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:41 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 7968 bytes result sent to driver -2016-05-24 22:29:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 78 ms on localhost (2/2) -2016-05-24 22:29:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 22:29:41 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.078 s -2016-05-24 22:29:41 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.100355 s -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=527382, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(3199) called with curMem=532974, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 22:29:41 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:57260 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:29:41 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 22:29:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:29:41 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:29:41 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:41 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=536173, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=547957, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:29:41 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:57260 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:41 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:41 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 22:29:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:41 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 22:29:41 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 22:29:41 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:41 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:29:41 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 7968 bytes result sent to driver -2016-05-24 22:29:41 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 7968 bytes result sent to driver -2016-05-24 22:29:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 16 ms on localhost (1/2) -2016-05-24 22:29:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 17 ms on localhost (2/2) -2016-05-24 22:29:41 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:29:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 22:29:41 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.039013 s -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=551439, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(3214) called with curMem=557031, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 22:29:41 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:57260 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:29:41 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 22:29:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:29:41 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:29:41 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:41 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=560245, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=572029, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:29:41 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:57260 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:41 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:41 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 22:29:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:41 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 22:29:41 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 22:29:41 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:29:41 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:41 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 7968 bytes result sent to driver -2016-05-24 22:29:41 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 7968 bytes result sent to driver -2016-05-24 22:29:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 14 ms on localhost (1/2) -2016-05-24 22:29:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 17 ms on localhost (2/2) -2016-05-24 22:29:41 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:29:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 22:29:41 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.030418 s -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=575510, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(3209) called with curMem=581102, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.0 MB) -2016-05-24 22:29:41 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:57260 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:29:41 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 22:29:41 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:29:41 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:29:41 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:41 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=584311, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=596095, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:29:41 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:57260 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:41 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:41 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 22:29:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:41 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 22:29:41 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 22:29:41 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:29:41 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:41 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 7968 bytes result sent to driver -2016-05-24 22:29:41 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 7968 bytes result sent to driver -2016-05-24 22:29:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 11 ms on localhost (1/2) -2016-05-24 22:29:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 13 ms on localhost (2/2) -2016-05-24 22:29:41 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.014 s -2016-05-24 22:29:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 22:29:41 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.027682 s -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=599578, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:29:41 INFO MemoryStore:59 - ensureFreeSpace(3208) called with curMem=605170, maxMem=766075207 -2016-05-24 22:29:41 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.0 MB) -2016-05-24 22:29:41 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:57260 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:29:41 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 22:29:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:29:42 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=608378, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=620162, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:57260 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:42 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:29:42 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 29 ms on localhost (1/2) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 30 ms on localhost (2/2) -2016-05-24 22:29:42 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.031 s -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 22:29:42 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.055009 s -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=623646, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3233) called with curMem=629238, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:57260 (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 22:29:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:29:42 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=632471, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=644255, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:57260 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:57260 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 22:29:42 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:57260 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 22:29:42 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:57260 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:29:42 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:57260 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 15 ms on localhost (1/2) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 16 ms on localhost (2/2) -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 22:29:42 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:29:42 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.062041 s -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:57260 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:57260 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=575530, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3237) called with curMem=581122, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:57260 (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:57260 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:29:42 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:29:42 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:57260 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=560302, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=572086, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:57260 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:57260 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 22:29:42 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:57260 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 22:29:42 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:57260 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:29:42 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 22:29:42 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:57260 in memory (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 19 ms on localhost (1/2) -2016-05-24 22:29:42 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 24 ms on localhost (2/2) -2016-05-24 22:29:42 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 22:29:42 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.045624 s -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:57260 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=513052, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3220) called with curMem=518644, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:57260 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 22:29:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:29:42 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=521864, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=533648, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:29:42 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:57260 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:57260 in memory (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:42 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 15 ms on localhost (1/2) -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:29:42 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 59 ms on localhost (2/2) -2016-05-24 22:29:42 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.062 s -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 22:29:42 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.094153 s -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=531152, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3222) called with curMem=536744, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:57260 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 22:29:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:29:42 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=539966, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=551750, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:57260 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 22:29:42 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:29:42 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 14 ms on localhost (1/2) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 16 ms on localhost (2/2) -2016-05-24 22:29:42 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 22:29:42 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.030582 s -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=555234, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3223) called with curMem=560826, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:57260 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 22:29:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:29:42 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=564049, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=575833, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:57260 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 22:29:42 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:29:42 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 19 ms on localhost (1/2) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 22 ms on localhost (2/2) -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 22:29:42 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.040 s -2016-05-24 22:29:42 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.057450 s -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=579316, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3239) called with curMem=584908, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:57260 (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 22:29:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:29:42 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=588147, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=599931, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:57260 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 22:29:42 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:42 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 14 ms on localhost (1/2) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 14 ms on localhost (2/2) -2016-05-24 22:29:42 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 22:29:42 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.051923 s -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=603415, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3233) called with curMem=609007, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:57260 (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 22:29:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:29:42 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=612240, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=624024, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:57260 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 22:29:42 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:42 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 12 ms on localhost (1/2) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 14 ms on localhost (2/2) -2016-05-24 22:29:42 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 22:29:42 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.026542 s -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=627507, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3226) called with curMem=633099, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:57260 (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 22:29:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:29:42 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=636325, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=648109, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:57260 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 22:29:42 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:42 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 11 ms on localhost (1/2) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 14 ms on localhost (2/2) -2016-05-24 22:29:42 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 22:29:42 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.042410 s -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=651591, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3220) called with curMem=657183, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.0 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:57260 (size: 3.1 KB, free: 730.2 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 22:29:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:29:42 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=660403, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=672187, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:57260 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 22:29:42 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:42 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 13 ms on localhost (1/2) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 16 ms on localhost (2/2) -2016-05-24 22:29:42 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 22:29:42 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.033040 s -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=675671, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3223) called with curMem=681263, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 3.1 KB, free 729.9 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:57260 (size: 3.1 KB, free: 730.2 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 22:29:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:29:42 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=684486, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=696270, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:57260 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 22:29:42 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:42 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 22 ms on localhost (1/2) -2016-05-24 22:29:42 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 7968 bytes result sent to driver -2016-05-24 22:29:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 25 ms on localhost (2/2) -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 22:29:42 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.028 s -2016-05-24 22:29:42 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.143064 s -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=699753, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3229) called with curMem=705345, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 3.2 KB, free 729.9 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:57260 (size: 3.2 KB, free: 730.2 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 22:29:42 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:29:42 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=708574, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:29:42 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=720358, maxMem=766075207 -2016-05-24 22:29:42 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:29:42 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:57260 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:29:42 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:42 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:42 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 22:29:42 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:29:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:43 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 7968 bytes result sent to driver -2016-05-24 22:29:43 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 7968 bytes result sent to driver -2016-05-24 22:29:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 21 ms on localhost (1/2) -2016-05-24 22:29:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 23 ms on localhost (2/2) -2016-05-24 22:29:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 22:29:43 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.024 s -2016-05-24 22:29:43 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.037815 s -2016-05-24 22:29:43 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=723841, maxMem=766075207 -2016-05-24 22:29:43 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:29:43 INFO MemoryStore:59 - ensureFreeSpace(3234) called with curMem=729433, maxMem=766075207 -2016-05-24 22:29:43 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.2 KB, free 729.9 MB) -2016-05-24 22:29:43 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:57260 (size: 3.2 KB, free: 730.2 MB) -2016-05-24 22:29:43 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 22:29:43 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:29:43 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:29:43 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:43 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:29:43 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=732667, maxMem=766075207 -2016-05-24 22:29:43 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:29:43 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=744451, maxMem=766075207 -2016-05-24 22:29:43 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:29:43 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:57260 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:29:43 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:29:43 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 22:29:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:29:43 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 22:29:43 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 22:29:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:29:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:29:43 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 7968 bytes result sent to driver -2016-05-24 22:29:43 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 7968 bytes result sent to driver -2016-05-24 22:29:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 16 ms on localhost (1/2) -2016-05-24 22:29:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 16 ms on localhost (2/2) -2016-05-24 22:29:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 22:29:43 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.019 s -2016-05-24 22:29:43 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.066956 s -2016-05-24 22:29:43 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 4756.985924113548, 4319.927896434357, 3903.4385808079614, 3505.841537598476, 3124.097453008626, 2756.2417647061375, 2401.155210966652, 2057.4601181932067, 1724.026901602298, 1463.7810900203021 -2016-05-24 22:29:43 INFO SparkContext:59 - Starting job: sortByKey at BinaryClassificationMetrics.scala:156 -2016-05-24 22:29:43 INFO DAGScheduler:59 - Registering RDD 50 (map at SVMFromSpark2Learning.scala:54) -2016-05-24 22:29:43 INFO DAGScheduler:59 - Got job 23 (sortByKey at BinaryClassificationMetrics.scala:156) with 2 output partitions -2016-05-24 22:29:43 INFO DAGScheduler:59 - Final stage: ResultStage 24(sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:29:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-24 22:29:43 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-24 22:29:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[50] at map at SVMFromSpark2Learning.scala:54), which has no missing parents -2016-05-24 22:29:43 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=747934, maxMem=766075207 -2016-05-24 22:29:43 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 10.8 KB, free 729.9 MB) -2016-05-24 22:29:43 INFO MemoryStore:59 - ensureFreeSpace(6056) called with curMem=758942, maxMem=766075207 -2016-05-24 22:29:43 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 5.9 KB, free 729.9 MB) -2016-05-24 22:29:43 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:57260 (size: 5.9 KB, free: 730.2 MB) -2016-05-24 22:29:43 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[50] at map at SVMFromSpark2Learning.scala:54) -2016-05-24 22:29:43 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 22:29:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:29:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:29:43 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 22:29:43 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:29:43 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 22:29:43 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:29:43 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2254 bytes result sent to driver -2016-05-24 22:29:43 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2254 bytes result sent to driver -2016-05-24 22:29:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 750 ms on localhost (1/2) -2016-05-24 22:29:43 INFO DAGScheduler:59 - ShuffleMapStage 23 (map at SVMFromSpark2Learning.scala:54) finished in 0.768 s -2016-05-24 22:29:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:29:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 767 ms on localhost (2/2) -2016-05-24 22:29:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 22:29:43 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:29:43 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-24 22:29:43 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:29:43 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-24 22:29:43 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[53] at sortByKey at BinaryClassificationMetrics.scala:156), which is now runnable -2016-05-24 22:29:44 INFO MemoryStore:59 - ensureFreeSpace(3344) called with curMem=764998, maxMem=766075207 -2016-05-24 22:29:44 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 3.3 KB, free 729.9 MB) -2016-05-24 22:29:44 INFO MemoryStore:59 - ensureFreeSpace(1898) called with curMem=768342, maxMem=766075207 -2016-05-24 22:29:44 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 1898.0 B, free 729.9 MB) -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:57260 (size: 1898.0 B, free: 730.2 MB) -2016-05-24 22:29:44 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[53] at sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:29:44 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 22:29:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:29:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:29:44 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 22:29:44 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 22:29:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:29:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:29:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 8 ms -2016-05-24 22:29:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 8 ms -2016-05-24 22:29:44 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 1485 bytes result sent to driver -2016-05-24 22:29:44 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 1501 bytes result sent to driver -2016-05-24 22:29:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 57 ms on localhost (1/2) -2016-05-24 22:29:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 60 ms on localhost (2/2) -2016-05-24 22:29:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 22:29:44 INFO DAGScheduler:59 - ResultStage 24 (sortByKey at BinaryClassificationMetrics.scala:156) finished in 0.074 s -2016-05-24 22:29:44 INFO DAGScheduler:59 - Job 23 finished: sortByKey at BinaryClassificationMetrics.scala:156, took 0.899467 s -2016-05-24 22:29:44 INFO SparkContext:59 - Starting job: collect at BinaryClassificationMetrics.scala:193 -2016-05-24 22:29:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-05-24 22:29:44 INFO DAGScheduler:59 - Registering RDD 51 (combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:29:44 INFO DAGScheduler:59 - Got job 24 (collect at BinaryClassificationMetrics.scala:193) with 2 output partitions -2016-05-24 22:29:44 INFO DAGScheduler:59 - Final stage: ResultStage 27(collect at BinaryClassificationMetrics.scala:193) -2016-05-24 22:29:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-24 22:29:44 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-24 22:29:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (ShuffledRDD[51] at combineByKey at BinaryClassificationMetrics.scala:152), which has no missing parents -2016-05-24 22:29:44 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=770240, maxMem=766075207 -2016-05-24 22:29:44 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 2.8 KB, free 729.8 MB) -2016-05-24 22:29:44 INFO MemoryStore:59 - ensureFreeSpace(1658) called with curMem=773136, maxMem=766075207 -2016-05-24 22:29:44 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 1658.0 B, free 729.8 MB) -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:57260 (size: 1658.0 B, free: 730.2 MB) -2016-05-24 22:29:44 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 26 (ShuffledRDD[51] at combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:29:44 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 22:29:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:29:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:29:44 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-24 22:29:44 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-24 22:29:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:29:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:29:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:29:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:29:44 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1375 bytes result sent to driver -2016-05-24 22:29:44 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1375 bytes result sent to driver -2016-05-24 22:29:44 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 22:29:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 72 ms on localhost (1/2) -2016-05-24 22:29:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 75 ms on localhost (2/2) -2016-05-24 22:29:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 22:29:44 INFO DAGScheduler:59 - ShuffleMapStage 26 (combineByKey at BinaryClassificationMetrics.scala:152) finished in 0.077 s -2016-05-24 22:29:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:29:44 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:29:44 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-24 22:29:44 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:29:44 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:57260 in memory (size: 1898.0 B, free: 730.2 MB) -2016-05-24 22:29:44 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[56] at mapPartitions at BinaryClassificationMetrics.scala:189), which is now runnable -2016-05-24 22:29:44 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 22:29:44 INFO MemoryStore:59 - ensureFreeSpace(3472) called with curMem=758544, maxMem=766075207 -2016-05-24 22:29:44 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:57260 in memory (size: 5.9 KB, free: 730.2 MB) -2016-05-24 22:29:44 INFO MemoryStore:59 - ensureFreeSpace(1982) called with curMem=755960, maxMem=766075207 -2016-05-24 22:29:44 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 22:29:44 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 1982.0 B, free 729.9 MB) -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:57260 (size: 1982.0 B, free: 730.2 MB) -2016-05-24 22:29:44 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[56] at mapPartitions at BinaryClassificationMetrics.scala:189) -2016-05-24 22:29:44 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:57260 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:29:44 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 22:29:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 51, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:29:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 52, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:29:44 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 52) -2016-05-24 22:29:44 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 51) -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:57260 in memory (size: 3.2 KB, free: 730.2 MB) -2016-05-24 22:29:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:29:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:29:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:29:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:57260 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:29:44 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:57260 in memory (size: 3.2 KB, free: 730.2 MB) -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:57260 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:29:44 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 51). 1330 bytes result sent to driver -2016-05-24 22:29:44 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 52). 1330 bytes result sent to driver -2016-05-24 22:29:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 51) in 18 ms on localhost (1/2) -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:57260 in memory (size: 3.1 KB, free: 730.2 MB) -2016-05-24 22:29:44 INFO DAGScheduler:59 - ResultStage 27 (collect at BinaryClassificationMetrics.scala:193) finished in 0.018 s -2016-05-24 22:29:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 52) in 17 ms on localhost (2/2) -2016-05-24 22:29:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 22:29:44 INFO DAGScheduler:59 - Job 24 finished: collect at BinaryClassificationMetrics.scala:193, took 0.171022 s -2016-05-24 22:29:44 INFO BinaryClassificationMetrics:59 - Total counts: {numPos: 25, numNeg: 11} -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:57260 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:29:44 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:57260 in memory (size: 3.1 KB, free: 730.2 MB) -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:57260 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:29:44 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:57260 in memory (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:57260 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:44 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:57260 in memory (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:57260 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:44 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:57260 in memory (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:57260 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:44 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:57260 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:57260 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:44 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:57260 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:29:44 INFO SparkContext:59 - Starting job: runJob at SlidingRDD.scala:75 -2016-05-24 22:29:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:57260 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-24 22:29:44 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 22:29:44 INFO DAGScheduler:59 - Got job 25 (runJob at SlidingRDD.scala:75) with 3 output partitions -2016-05-24 22:29:44 INFO DAGScheduler:59 - Final stage: ResultStage 30(runJob at SlidingRDD.scala:75) -2016-05-24 22:29:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-24 22:29:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:44 INFO DAGScheduler:59 - Submitting ResultStage 30 (UnionRDD[62] at UnionRDD at BinaryClassificationMetrics.scala:92), which has no missing parents -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:57260 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:57260 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:44 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:57260 in memory (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:57260 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:29:44 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 22:29:44 INFO MemoryStore:59 - ensureFreeSpace(5184) called with curMem=477717, maxMem=766075207 -2016-05-24 22:29:44 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 5.1 KB, free 730.1 MB) -2016-05-24 22:29:44 INFO MemoryStore:59 - ensureFreeSpace(2788) called with curMem=474076, maxMem=766075207 -2016-05-24 22:29:44 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.1 MB) -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:57260 in memory (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:57260 (size: 2.7 KB, free: 730.3 MB) -2016-05-24 22:29:44 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:44 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 30 (UnionRDD[62] at UnionRDD at BinaryClassificationMetrics.scala:92) -2016-05-24 22:29:44 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 3 tasks -2016-05-24 22:29:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 53, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:29:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:29:44 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 2318 bytes) -2016-05-24 22:29:44 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 54) -2016-05-24 22:29:44 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 53) -2016-05-24 22:29:44 INFO CacheManager:59 - Partition rdd_57_0 not found, computing it -2016-05-24 22:29:44 INFO CacheManager:59 - Partition rdd_57_1 not found, computing it -2016-05-24 22:29:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:29:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:29:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:29:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:29:44 INFO MemoryStore:59 - ensureFreeSpace(1448) called with curMem=476864, maxMem=766075207 -2016-05-24 22:29:44 INFO MemoryStore:59 - Block rdd_57_0 stored as values in memory (estimated size 1448.0 B, free 730.1 MB) -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Added rdd_57_0 in memory on localhost:57260 (size: 1448.0 B, free: 730.3 MB) -2016-05-24 22:29:44 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 53). 1883 bytes result sent to driver -2016-05-24 22:29:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 53) in 56 ms on localhost (1/3) -2016-05-24 22:29:44 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 55) -2016-05-24 22:29:44 INFO MemoryStore:59 - ensureFreeSpace(1616) called with curMem=478312, maxMem=766075207 -2016-05-24 22:29:44 INFO MemoryStore:59 - Block rdd_57_1 stored as values in memory (estimated size 1616.0 B, free 730.1 MB) -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Added rdd_57_1 in memory on localhost:57260 (size: 1616.0 B, free: 730.3 MB) -2016-05-24 22:29:44 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 54). 1883 bytes result sent to driver -2016-05-24 22:29:44 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 55). 1053 bytes result sent to driver -2016-05-24 22:29:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 54) in 62 ms on localhost (2/3) -2016-05-24 22:29:44 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 55) in 62 ms on localhost (3/3) -2016-05-24 22:29:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 22:29:44 INFO DAGScheduler:59 - ResultStage 30 (runJob at SlidingRDD.scala:75) finished in 0.067 s -2016-05-24 22:29:44 INFO DAGScheduler:59 - Job 25 finished: runJob at SlidingRDD.scala:75, took 0.092870 s -2016-05-24 22:29:44 INFO SparkContext:59 - Starting job: aggregate at AreaUnderCurve.scala:45 -2016-05-24 22:29:44 INFO DAGScheduler:59 - Got job 26 (aggregate at AreaUnderCurve.scala:45) with 4 output partitions -2016-05-24 22:29:44 INFO DAGScheduler:59 - Final stage: ResultStage 33(aggregate at AreaUnderCurve.scala:45) -2016-05-24 22:29:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-24 22:29:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:44 INFO DAGScheduler:59 - Submitting ResultStage 33 (SlidingRDD[63] at RDD at SlidingRDD.scala:47), which has no missing parents -2016-05-24 22:29:44 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=479928, maxMem=766075207 -2016-05-24 22:29:44 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:29:44 INFO MemoryStore:59 - ensureFreeSpace(2973) called with curMem=485584, maxMem=766075207 -2016-05-24 22:29:44 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.1 MB) -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:57260 (size: 2.9 KB, free: 730.3 MB) -2016-05-24 22:29:44 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:44 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (SlidingRDD[63] at RDD at SlidingRDD.scala:47) -2016-05-24 22:29:44 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-24 22:29:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 56, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:29:44 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 57, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:29:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 2635 bytes) -2016-05-24 22:29:44 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 2475 bytes) -2016-05-24 22:29:44 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 57) -2016-05-24 22:29:44 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-24 22:29:44 INFO BlockManager:59 - Found block rdd_57_1 locally -2016-05-24 22:29:44 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 955 bytes result sent to driver -2016-05-24 22:29:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 11 ms on localhost (1/4) -2016-05-24 22:29:44 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 57). 2084 bytes result sent to driver -2016-05-24 22:29:44 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 56) -2016-05-24 22:29:44 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 59) -2016-05-24 22:29:44 INFO BlockManager:59 - Found block rdd_57_0 locally -2016-05-24 22:29:44 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 56). 2084 bytes result sent to driver -2016-05-24 22:29:44 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 59). 955 bytes result sent to driver -2016-05-24 22:29:44 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 57) in 25 ms on localhost (2/4) -2016-05-24 22:29:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 56) in 29 ms on localhost (3/4) -2016-05-24 22:29:44 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 59) in 24 ms on localhost (4/4) -2016-05-24 22:29:44 INFO DAGScheduler:59 - ResultStage 33 (aggregate at AreaUnderCurve.scala:45) finished in 0.031 s -2016-05-24 22:29:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 22:29:44 INFO DAGScheduler:59 - Job 26 finished: aggregate at AreaUnderCurve.scala:45, took 0.047122 s -2016-05-24 22:29:44 INFO SparkContext:59 - Starting job: take at SVMFromSpark2Learning.scala:66 -2016-05-24 22:29:44 INFO DAGScheduler:59 - Got job 27 (take at SVMFromSpark2Learning.scala:66) with 1 output partitions -2016-05-24 22:29:44 INFO DAGScheduler:59 - Final stage: ResultStage 34(take at SVMFromSpark2Learning.scala:66) -2016-05-24 22:29:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:44 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[50] at map at SVMFromSpark2Learning.scala:54), which has no missing parents -2016-05-24 22:29:44 INFO MemoryStore:59 - ensureFreeSpace(10320) called with curMem=488557, maxMem=766075207 -2016-05-24 22:29:44 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 10.1 KB, free 730.1 MB) -2016-05-24 22:29:44 INFO MemoryStore:59 - ensureFreeSpace(5705) called with curMem=498877, maxMem=766075207 -2016-05-24 22:29:44 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-24 22:29:44 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:57260 (size: 5.6 KB, free: 730.3 MB) -2016-05-24 22:29:44 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:44 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 34 (MapPartitionsRDD[50] at map at SVMFromSpark2Learning.scala:54) -2016-05-24 22:29:44 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 1 tasks -2016-05-24 22:29:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 60, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:29:44 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 60) -2016-05-24 22:29:44 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:29:44 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 60). 2401 bytes result sent to driver -2016-05-24 22:29:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 60) in 39 ms on localhost (1/1) -2016-05-24 22:29:44 INFO DAGScheduler:59 - ResultStage 34 (take at SVMFromSpark2Learning.scala:66) finished in 0.040 s -2016-05-24 22:29:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 22:29:44 INFO DAGScheduler:59 - Job 27 finished: take at SVMFromSpark2Learning.scala:66, took 0.058752 s -2016-05-24 22:29:45 INFO SparkContext:59 - Starting job: saveAsTextFile at GLMClassificationModel.scala:61 -2016-05-24 22:29:45 INFO DAGScheduler:59 - Got job 28 (saveAsTextFile at GLMClassificationModel.scala:61) with 1 output partitions -2016-05-24 22:29:45 INFO DAGScheduler:59 - Final stage: ResultStage 35(saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:29:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:45 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[65] at saveAsTextFile at GLMClassificationModel.scala:61), which has no missing parents -2016-05-24 22:29:45 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=504582, maxMem=766075207 -2016-05-24 22:29:45 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 124.0 KB, free 730.0 MB) -2016-05-24 22:29:45 INFO MemoryStore:59 - ensureFreeSpace(42574) called with curMem=631574, maxMem=766075207 -2016-05-24 22:29:45 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 41.6 KB, free 729.9 MB) -2016-05-24 22:29:45 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:57260 (size: 41.6 KB, free: 730.2 MB) -2016-05-24 22:29:45 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:45 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 35 (MapPartitionsRDD[65] at saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:29:45 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 1 tasks -2016-05-24 22:29:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 61, localhost, PROCESS_LOCAL, 2197 bytes) -2016-05-24 22:29:45 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 61) -2016-05-24 22:29:45 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242229_0035_m_000000_61' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222944563/result/metadata/_temporary/0/task_201605242229_0035_m_000000 -2016-05-24 22:29:45 INFO SparkHadoopMapRedUtil:59 - attempt_201605242229_0035_m_000000_61: Committed -2016-05-24 22:29:45 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 61). 1864 bytes result sent to driver -2016-05-24 22:29:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 61) in 212 ms on localhost (1/1) -2016-05-24 22:29:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 22:29:45 INFO DAGScheduler:59 - ResultStage 35 (saveAsTextFile at GLMClassificationModel.scala:61) finished in 0.212 s -2016-05-24 22:29:45 INFO DAGScheduler:59 - Job 28 finished: saveAsTextFile at GLMClassificationModel.scala:61, took 0.314874 s -2016-05-24 22:29:47 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:29:47 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:29:47 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:65 -2016-05-24 22:29:47 INFO DAGScheduler:59 - Got job 29 (parquet at GLMClassificationModel.scala:65) with 1 output partitions -2016-05-24 22:29:47 INFO DAGScheduler:59 - Final stage: ResultStage 36(parquet at GLMClassificationModel.scala:65) -2016-05-24 22:29:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:47 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[67] at rddToDataFrameHolder at GLMClassificationModel.scala:65), which has no missing parents -2016-05-24 22:29:47 INFO MemoryStore:59 - ensureFreeSpace(67104) called with curMem=674148, maxMem=766075207 -2016-05-24 22:29:47 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 65.5 KB, free 729.9 MB) -2016-05-24 22:29:47 INFO MemoryStore:59 - ensureFreeSpace(23437) called with curMem=741252, maxMem=766075207 -2016-05-24 22:29:47 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 22.9 KB, free 729.9 MB) -2016-05-24 22:29:47 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:57260 (size: 22.9 KB, free: 730.2 MB) -2016-05-24 22:29:47 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:47 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[67] at rddToDataFrameHolder at GLMClassificationModel.scala:65) -2016-05-24 22:29:47 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-24 22:29:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 62, localhost, PROCESS_LOCAL, 7978 bytes) -2016-05-24 22:29:47 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 62) -2016-05-24 22:29:47 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:29:47 INFO CodecConfig:151 - Compression: GZIP -2016-05-24 22:29:47 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-24 22:29:47 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-24 22:29:47 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-24 22:29:47 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-24 22:29:47 INFO ParquetOutputFormat:151 - Validation is off -2016-05-24 22:29:47 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-24 22:29:47 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-24 22:29:47 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 6,084 -2016-05-24 22:29:47 INFO ColumnChunkPageWriteStore:151 - written 61B for [weights, type] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-24 22:29:47 INFO ColumnChunkPageWriteStore:151 - written 48B for [weights, size] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-24 22:29:47 INFO ColumnChunkPageWriteStore:151 - written 51B for [weights, indices, array] INT32: 1 values, 13B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-24 22:29:47 INFO ColumnChunkPageWriteStore:151 - written 635B for [weights, values, array] DOUBLE: 692 values, 668B raw, 591B comp, 1 pages, encodings: [RLE, PLAIN_DICTIONARY], dic { 404 entries, 3,232B raw, 404B comp} -2016-05-24 22:29:47 INFO ColumnChunkPageWriteStore:151 - written 69B for [intercept] DOUBLE: 1 values, 14B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-24 22:29:47 INFO ColumnChunkPageWriteStore:151 - written 47B for [threshold] DOUBLE: 1 values, 6B raw, 26B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-24 22:29:48 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242229_0036_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222944563/result/data/_temporary/0/task_201605242229_0036_m_000000 -2016-05-24 22:29:48 INFO SparkHadoopMapRedUtil:59 - attempt_201605242229_0036_m_000000_0: Committed -2016-05-24 22:29:48 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 62). 915 bytes result sent to driver -2016-05-24 22:29:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 62) in 644 ms on localhost (1/1) -2016-05-24 22:29:48 INFO DAGScheduler:59 - ResultStage 36 (parquet at GLMClassificationModel.scala:65) finished in 0.651 s -2016-05-24 22:29:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 22:29:48 INFO DAGScheduler:59 - Job 29 finished: parquet at GLMClassificationModel.scala:65, took 0.690768 s -2016-05-24 22:29:48 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:29:48 INFO DefaultWriterContainer:59 - Job job_201605242229_0000 committed. -2016-05-24 22:29:48 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222944563/result/data on driver -2016-05-24 22:29:48 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222944563/result/data on driver -2016-05-24 22:29:48 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=764689, maxMem=766075207 -2016-05-24 22:29:48 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 225.3 KB, free 729.6 MB) -2016-05-24 22:29:48 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=995393, maxMem=766075207 -2016-05-24 22:29:48 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.6 MB) -2016-05-24 22:29:48 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:57260 (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:29:48 INFO SparkContext:59 - Created broadcast 52 from textFile at modelSaveLoad.scala:129 -2016-05-24 22:29:49 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:29:49 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-24 22:29:49 INFO DAGScheduler:59 - Got job 30 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-24 22:29:49 INFO DAGScheduler:59 - Final stage: ResultStage 37(first at modelSaveLoad.scala:129) -2016-05-24 22:29:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:49 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[70] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-24 22:29:49 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=1015181, maxMem=766075207 -2016-05-24 22:29:49 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 3.1 KB, free 729.6 MB) -2016-05-24 22:29:49 INFO MemoryStore:59 - ensureFreeSpace(1894) called with curMem=1018397, maxMem=766075207 -2016-05-24 22:29:49 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 1894.0 B, free 729.6 MB) -2016-05-24 22:29:49 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:57260 (size: 1894.0 B, free: 730.2 MB) -2016-05-24 22:29:49 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:49 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 37 (MapPartitionsRDD[70] at textFile at modelSaveLoad.scala:129) -2016-05-24 22:29:49 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 1 tasks -2016-05-24 22:29:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 63, localhost, PROCESS_LOCAL, 2238 bytes) -2016-05-24 22:29:49 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 63) -2016-05-24 22:29:49 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:57260 in memory (size: 22.9 KB, free: 730.2 MB) -2016-05-24 22:29:49 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 22:29:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222944563/result/metadata/part-00000:0+54 -2016-05-24 22:29:49 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:57260 in memory (size: 41.6 KB, free: 730.3 MB) -2016-05-24 22:29:49 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 22:29:49 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 63). 2154 bytes result sent to driver -2016-05-24 22:29:49 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:57260 in memory (size: 5.6 KB, free: 730.3 MB) -2016-05-24 22:29:49 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 22:29:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 63) in 13 ms on localhost (1/1) -2016-05-24 22:29:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 22:29:49 INFO DAGScheduler:59 - ResultStage 37 (first at modelSaveLoad.scala:129) finished in 0.014 s -2016-05-24 22:29:49 INFO DAGScheduler:59 - Job 30 finished: first at modelSaveLoad.scala:129, took 0.116437 s -2016-05-24 22:29:49 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:57260 in memory (size: 2.9 KB, free: 730.3 MB) -2016-05-24 22:29:49 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 22:29:49 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:57260 in memory (size: 2.7 KB, free: 730.3 MB) -2016-05-24 22:29:49 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 22:29:49 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:57260 in memory (size: 1982.0 B, free: 730.3 MB) -2016-05-24 22:29:49 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 22:29:49 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:57260 in memory (size: 1658.0 B, free: 730.3 MB) -2016-05-24 22:29:49 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 22:29:49 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222944563/result/data on driver -2016-05-24 22:29:49 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:78 -2016-05-24 22:29:49 INFO DAGScheduler:59 - Got job 31 (parquet at GLMClassificationModel.scala:78) with 4 output partitions -2016-05-24 22:29:49 INFO DAGScheduler:59 - Final stage: ResultStage 38(parquet at GLMClassificationModel.scala:78) -2016-05-24 22:29:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:49 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[72] at parquet at GLMClassificationModel.scala:78), which has no missing parents -2016-05-24 22:29:49 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=717550, maxMem=766075207 -2016-05-24 22:29:49 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 61.5 KB, free 729.8 MB) -2016-05-24 22:29:49 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=780550, maxMem=766075207 -2016-05-24 22:29:49 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 20.6 KB, free 729.8 MB) -2016-05-24 22:29:49 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:57260 (size: 20.6 KB, free: 730.3 MB) -2016-05-24 22:29:49 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:49 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[72] at parquet at GLMClassificationModel.scala:78) -2016-05-24 22:29:49 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-24 22:29:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 64, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:29:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 65, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:29:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 66, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:29:49 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 67, localhost, PROCESS_LOCAL, 2279 bytes) -2016-05-24 22:29:49 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 64) -2016-05-24 22:29:49 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 67) -2016-05-24 22:29:49 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 66) -2016-05-24 22:29:49 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 65) -2016-05-24 22:29:49 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:29:49 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:29:49 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 64). 936 bytes result sent to driver -2016-05-24 22:29:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 64) in 94 ms on localhost (1/4) -2016-05-24 22:29:49 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 67). 1802 bytes result sent to driver -2016-05-24 22:29:49 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 67) in 102 ms on localhost (2/4) -2016-05-24 22:29:49 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:29:49 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 66). 936 bytes result sent to driver -2016-05-24 22:29:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 66) in 109 ms on localhost (3/4) -2016-05-24 22:29:49 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:29:49 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 65). 936 bytes result sent to driver -2016-05-24 22:29:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 65) in 120 ms on localhost (4/4) -2016-05-24 22:29:49 INFO DAGScheduler:59 - ResultStage 38 (parquet at GLMClassificationModel.scala:78) finished in 0.121 s -2016-05-24 22:29:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 22:29:49 INFO DAGScheduler:59 - Job 31 finished: parquet at GLMClassificationModel.scala:78, took 0.154971 s -2016-05-24 22:29:49 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=801623, maxMem=766075207 -2016-05-24 22:29:49 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 86.4 KB, free 729.7 MB) -2016-05-24 22:29:49 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=890135, maxMem=766075207 -2016-05-24 22:29:49 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.7 MB) -2016-05-24 22:29:49 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:57260 (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:29:49 INFO SparkContext:59 - Created broadcast 55 from take at GLMClassificationModel.scala:79 -2016-05-24 22:29:50 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:57260 in memory (size: 1894.0 B, free: 730.2 MB) -2016-05-24 22:29:50 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-24 22:29:50 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222944563/result/data/part-r-00000-c891fab1-6fff-4c6f-aea5-ec88078bcad5.gz.parquet -2016-05-24 22:29:51 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:57260 in memory (size: 20.6 KB, free: 730.3 MB) -2016-05-24 22:29:51 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 22:29:51 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 22:29:51 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:57260 in memory (size: 19.3 KB, free: 730.3 MB) -2016-05-24 22:29:51 INFO SparkContext:59 - Starting job: take at GLMClassificationModel.scala:79 -2016-05-24 22:29:51 INFO DAGScheduler:59 - Got job 32 (take at GLMClassificationModel.scala:79) with 1 output partitions -2016-05-24 22:29:51 INFO DAGScheduler:59 - Final stage: ResultStage 39(take at GLMClassificationModel.scala:79) -2016-05-24 22:29:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:29:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:29:51 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[74] at take at GLMClassificationModel.scala:79), which has no missing parents -2016-05-24 22:29:51 INFO MemoryStore:59 - ensureFreeSpace(4168) called with curMem=570248, maxMem=766075207 -2016-05-24 22:29:51 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 4.1 KB, free 730.0 MB) -2016-05-24 22:29:51 INFO MemoryStore:59 - ensureFreeSpace(2438) called with curMem=574416, maxMem=766075207 -2016-05-24 22:29:51 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.0 MB) -2016-05-24 22:29:51 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:57260 (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:29:51 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:29:51 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 39 (MapPartitionsRDD[74] at take at GLMClassificationModel.scala:79) -2016-05-24 22:29:51 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 1 tasks -2016-05-24 22:29:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 68, localhost, PROCESS_LOCAL, 2301 bytes) -2016-05-24 22:29:51 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 68) -2016-05-24 22:29:51 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524222944563/result/data/part-r-00000-c891fab1-6fff-4c6f-aea5-ec88078bcad5.gz.parquet start: 0 end: 4863 length: 4863 hosts: []} -2016-05-24 22:29:51 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-24 22:29:51 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group weights { - required int32 type (INT_8); - optional int32 size; - optional group indices (LIST) { - repeated int32 array; - } - optional group values (LIST) { - repeated double array; - } - } - optional double intercept; - optional double threshold; -} - -Catalyst form: -StructType(StructField(weights,org.apache.spark.mllib.linalg.VectorUDT@f71b0bce,true), StructField(intercept,DoubleType,true), StructField(threshold,DoubleType,true)) - -2016-05-24 22:29:51 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-24 22:29:51 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-24 22:29:51 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-24 22:29:51 INFO InternalParquetRecordReader:151 - block read in memory in 27 ms. row count = 1 -2016-05-24 22:29:53 ERROR Executor:96 - Exception in task 0.0 in stage 39.0 (TID 68) -java.lang.OutOfMemoryError: PermGen space - at java.lang.ClassLoader.defineClass(Unknown Source) - at java.security.SecureClassLoader.defineClass(Unknown Source) - at java.net.URLClassLoader.defineClass(Unknown Source) - at java.net.URLClassLoader.access$100(Unknown Source) - at java.net.URLClassLoader$1.run(Unknown Source) - at java.net.URLClassLoader$1.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at java.net.URLClassLoader.findClass(Unknown Source) - at java.lang.ClassLoader.loadClass(Unknown Source) - at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source) - at java.lang.ClassLoader.loadClass(Unknown Source) - at java.lang.Class.getDeclaredMethods0(Native Method) - at java.lang.Class.privateGetDeclaredMethods(Unknown Source) - at java.lang.Class.getDeclaredMethod(Unknown Source) - at java.io.ObjectStreamClass.getPrivateMethod(Unknown Source) - at java.io.ObjectStreamClass.access$1700(Unknown Source) - at java.io.ObjectStreamClass$2.run(Unknown Source) - at java.io.ObjectStreamClass$2.run(Unknown Source) - at java.security.AccessController.doPrivileged(Native Method) - at java.io.ObjectStreamClass.(Unknown Source) - at java.io.ObjectStreamClass.lookup(Unknown Source) - at java.io.ObjectOutputStream.writeObject0(Unknown Source) - at java.io.ObjectOutputStream.writeArray(Unknown Source) - at java.io.ObjectOutputStream.writeObject0(Unknown Source) - at java.io.ObjectOutputStream.writeObject(Unknown Source) - at org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:44) - at org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:84) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:240) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-05-24 22:33:17 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 22:33:18 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 22:33:18 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 22:33:18 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 22:33:18 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 22:33:19 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 22:33:19 INFO Remoting:74 - Starting remoting -2016-05-24 22:33:20 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:57356] -2016-05-24 22:33:20 INFO Utils:59 - Successfully started service 'sparkDriver' on port 57356. -2016-05-24 22:33:20 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 22:33:20 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 22:33:20 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-d47597ee-f5ad-4754-9c54-dc36ca664100 -2016-05-24 22:33:20 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 22:33:20 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-013ce97e-05ef-4c7e-ac0e-a3647423771a\httpd-1e64ba80-1e7a-42c3-b9a0-5fd6018eb355 -2016-05-24 22:33:20 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 22:33:20 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:33:20 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:57357 -2016-05-24 22:33:20 INFO Utils:59 - Successfully started service 'HTTP file server' on port 57357. -2016-05-24 22:33:20 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 22:33:20 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:33:20 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 22:33:20 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 22:33:20 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 22:33:20 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 22:33:20 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 22:33:21 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 57376. -2016-05-24 22:33:21 INFO NettyBlockTransferService:59 - Server created on 57376 -2016-05-24 22:33:21 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 22:33:21 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:57376 with 730.6 MB RAM, BlockManagerId(driver, localhost, 57376) -2016-05-24 22:33:21 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 22:33:21 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 22:33:21 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 22:33:21 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 22:33:21 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 22:33:21 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:57376 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 22:33:21 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 22:33:23 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 22:33:23 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:33:23 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 22:33:23 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 22:33:23 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 22:33:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:23 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 22:33:23 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 22:33:23 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 22:33:23 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=175476, maxMem=766075207 -2016-05-24 22:33:23 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:33:23 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:57376 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:33:23 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 22:33:23 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 22:33:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:33:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:33:23 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 22:33:23 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 22:33:23 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 22:33:23 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 22:33:23 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 22:33:23 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 22:33:23 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 22:33:23 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 22:33:23 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 22:33:23 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 22:33:23 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 22:33:24 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177621, maxMem=766075207 -2016-05-24 22:33:24 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 22:33:24 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:57376 (size: 83.1 KB, free: 730.5 MB) -2016-05-24 22:33:24 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262701, maxMem=766075207 -2016-05-24 22:33:24 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 22:33:24 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:57376 (size: 83.6 KB, free: 730.4 MB) -2016-05-24 22:33:24 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 22:33:24 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 22:33:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 388 ms on localhost (1/2) -2016-05-24 22:33:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 417 ms on localhost (2/2) -2016-05-24 22:33:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 22:33:24 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.441 s -2016-05-24 22:33:24 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.636377 s -2016-05-24 22:33:24 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 22:33:24 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 22:33:24 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:33:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:24 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 22:33:24 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=348349, maxMem=766075207 -2016-05-24 22:33:24 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-24 22:33:24 INFO MemoryStore:59 - ensureFreeSpace(2483) called with curMem=352909, maxMem=766075207 -2016-05-24 22:33:24 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-24 22:33:24 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:57376 (size: 2.4 KB, free: 730.4 MB) -2016-05-24 22:33:24 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:33:24 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 22:33:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:33:24 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 22:33:24 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-05-24 22:33:24 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:33:24 INFO MemoryStore:59 - ensureFreeSpace(54392) called with curMem=355392, maxMem=766075207 -2016-05-24 22:33:24 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 53.1 KB, free 730.2 MB) -2016-05-24 22:33:24 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:57376 (size: 53.1 KB, free: 730.4 MB) -2016-05-24 22:33:24 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2611 bytes result sent to driver -2016-05-24 22:33:24 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.055 s -2016-05-24 22:33:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 55 ms on localhost (1/1) -2016-05-24 22:33:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 22:33:24 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.087572 s -2016-05-24 22:33:24 INFO SparkContext:59 - Starting job: count at DataValidators.scala:40 -2016-05-24 22:33:24 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:40) with 2 output partitions -2016-05-24 22:33:24 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:40) -2016-05-24 22:33:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:24 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40), which has no missing parents -2016-05-24 22:33:24 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=409784, maxMem=766075207 -2016-05-24 22:33:24 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-05-24 22:33:24 INFO MemoryStore:59 - ensureFreeSpace(2300) called with curMem=414032, maxMem=766075207 -2016-05-24 22:33:24 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-05-24 22:33:24 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:57376 (size: 2.2 KB, free: 730.4 MB) -2016-05-24 22:33:24 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:40) -2016-05-24 22:33:24 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 22:33:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:33:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:33:24 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 22:33:24 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 22:33:24 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-05-24 22:33:24 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:33:24 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:24 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 22:33:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 19 ms on localhost (1/2) -2016-05-24 22:33:24 INFO MemoryStore:59 - ensureFreeSpace(62184) called with curMem=416332, maxMem=766075207 -2016-05-24 22:33:24 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 60.7 KB, free 730.1 MB) -2016-05-24 22:33:24 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:57376 (size: 60.7 KB, free: 730.3 MB) -2016-05-24 22:33:24 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2662 bytes result sent to driver -2016-05-24 22:33:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 31 ms on localhost (2/2) -2016-05-24 22:33:24 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:40) finished in 0.033 s -2016-05-24 22:33:24 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:40, took 0.052144 s -2016-05-24 22:33:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 22:33:24 INFO SparkContext:59 - Starting job: count at GradientDescent.scala:195 -2016-05-24 22:33:24 INFO DAGScheduler:59 - Got job 3 (count at GradientDescent.scala:195) with 2 output partitions -2016-05-24 22:33:24 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at GradientDescent.scala:195) -2016-05-24 22:33:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:24 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292), which has no missing parents -2016-05-24 22:33:24 INFO MemoryStore:59 - ensureFreeSpace(4264) called with curMem=478516, maxMem=766075207 -2016-05-24 22:33:24 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.2 KB, free 730.1 MB) -2016-05-24 22:33:24 INFO MemoryStore:59 - ensureFreeSpace(2322) called with curMem=482780, maxMem=766075207 -2016-05-24 22:33:24 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 22:33:24 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:57376 (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:33:24 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[11] at map at GeneralizedLinearAlgorithm.scala:292) -2016-05-24 22:33:24 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 22:33:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:33:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:33:24 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 22:33:24 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 22:33:24 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:24 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 22:33:24 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:33:24 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 22:33:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 30 ms on localhost (1/2) -2016-05-24 22:33:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 29 ms on localhost (2/2) -2016-05-24 22:33:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 22:33:24 INFO DAGScheduler:59 - ResultStage 3 (count at GradientDescent.scala:195) finished in 0.032 s -2016-05-24 22:33:24 INFO DAGScheduler:59 - Job 3 finished: count at GradientDescent.scala:195, took 0.054758 s -2016-05-24 22:33:24 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 22:33:24 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 22:33:24 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=485102, maxMem=766075207 -2016-05-24 22:33:24 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:33:24 INFO MemoryStore:59 - ensureFreeSpace(388) called with curMem=490694, maxMem=766075207 -2016-05-24 22:33:24 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 388.0 B, free 730.1 MB) -2016-05-24 22:33:24 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:57376 (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:33:24 INFO SparkContext:59 - Created broadcast 5 from broadcast at GradientDescent.scala:221 -2016-05-24 22:33:24 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:57376 in memory (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:57376 in memory (size: 2.2 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:57376 in memory (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:57376 in memory (size: 2.1 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 22:33:25 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Got job 4 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:33:25 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:25 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=464864, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=476648, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:57376 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:25 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 22:33:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:25 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 22:33:25 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 22:33:25 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:33:25 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:25 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 7968 bytes result sent to driver -2016-05-24 22:33:25 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 7968 bytes result sent to driver -2016-05-24 22:33:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 22 ms on localhost (1/2) -2016-05-24 22:33:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 21 ms on localhost (2/2) -2016-05-24 22:33:25 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 22:33:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 22:33:25 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at GradientDescent.scala:225, took 0.046994 s -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=480129, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(2345) called with curMem=485721, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:57376 (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Created broadcast 7 from broadcast at GradientDescent.scala:221 -2016-05-24 22:33:25 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Got job 5 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:33:25 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:25 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=488066, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=499850, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:57376 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:25 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 22:33:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:25 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 22:33:25 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 22:33:25 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:33:25 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:25 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 7968 bytes result sent to driver -2016-05-24 22:33:25 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 7968 bytes result sent to driver -2016-05-24 22:33:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 15 ms on localhost (1/2) -2016-05-24 22:33:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 18 ms on localhost (2/2) -2016-05-24 22:33:25 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 22:33:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 22:33:25 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at GradientDescent.scala:225, took 0.039216 s -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=503332, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(3190) called with curMem=508924, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:57376 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Created broadcast 9 from broadcast at GradientDescent.scala:221 -2016-05-24 22:33:25 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Got job 6 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:33:25 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:25 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=512114, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=523898, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:57376 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[17] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:25 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 22:33:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:25 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 22:33:25 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 22:33:25 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:33:25 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:25 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 7968 bytes result sent to driver -2016-05-24 22:33:25 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 7968 bytes result sent to driver -2016-05-24 22:33:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 19 ms on localhost (1/2) -2016-05-24 22:33:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 23 ms on localhost (2/2) -2016-05-24 22:33:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 22:33:25 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at GradientDescent.scala:225) finished in 0.023 s -2016-05-24 22:33:25 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at GradientDescent.scala:225, took 0.040590 s -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=527381, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(3199) called with curMem=532973, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:57376 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Created broadcast 11 from broadcast at GradientDescent.scala:221 -2016-05-24 22:33:25 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Got job 7 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:33:25 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:25 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=536172, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=547956, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:57376 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:25 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 22:33:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:25 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 22:33:25 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 22:33:25 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:33:25 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:25 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 7968 bytes result sent to driver -2016-05-24 22:33:25 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 7968 bytes result sent to driver -2016-05-24 22:33:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 15 ms on localhost (1/2) -2016-05-24 22:33:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 17 ms on localhost (2/2) -2016-05-24 22:33:25 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 22:33:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 22:33:25 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at GradientDescent.scala:225, took 0.034156 s -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=551439, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(3214) called with curMem=557031, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:57376 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Created broadcast 13 from broadcast at GradientDescent.scala:221 -2016-05-24 22:33:25 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Got job 8 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:33:25 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:25 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=560245, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(3481) called with curMem=572029, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:57376 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:25 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 22:33:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:25 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 22:33:25 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 22:33:25 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:25 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:33:25 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 7968 bytes result sent to driver -2016-05-24 22:33:25 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 7968 bytes result sent to driver -2016-05-24 22:33:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 17 ms on localhost (1/2) -2016-05-24 22:33:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 16 ms on localhost (2/2) -2016-05-24 22:33:25 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at GradientDescent.scala:225) finished in 0.018 s -2016-05-24 22:33:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 22:33:25 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at GradientDescent.scala:225, took 0.035135 s -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=575510, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(3209) called with curMem=581102, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.0 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:57376 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Created broadcast 15 from broadcast at GradientDescent.scala:221 -2016-05-24 22:33:25 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Got job 9 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:33:25 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:25 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=584311, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=596095, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:57376 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[23] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:25 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 22:33:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:25 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 22:33:25 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 22:33:25 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:25 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:33:25 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 7968 bytes result sent to driver -2016-05-24 22:33:25 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 7968 bytes result sent to driver -2016-05-24 22:33:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 15 ms on localhost (1/2) -2016-05-24 22:33:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 13 ms on localhost (2/2) -2016-05-24 22:33:25 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:33:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 22:33:25 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at GradientDescent.scala:225, took 0.031624 s -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=599578, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(3208) called with curMem=605170, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.0 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:57376 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Created broadcast 17 from broadcast at GradientDescent.scala:221 -2016-05-24 22:33:25 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Got job 10 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:33:25 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:25 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=608378, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=620162, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:57376 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[25] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:25 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 22:33:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:25 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 22:33:25 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 22:33:25 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:33:25 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:25 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 7968 bytes result sent to driver -2016-05-24 22:33:25 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 7968 bytes result sent to driver -2016-05-24 22:33:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 13 ms on localhost (1/2) -2016-05-24 22:33:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 15 ms on localhost (2/2) -2016-05-24 22:33:25 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:33:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 22:33:25 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at GradientDescent.scala:225, took 0.028065 s -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=623646, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(3233) called with curMem=629238, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:57376 (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Created broadcast 19 from broadcast at GradientDescent.scala:221 -2016-05-24 22:33:25 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Got job 11 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:33:25 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:25 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=632471, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=644255, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:57376 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[27] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:25 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 22:33:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:25 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 22:33:25 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 22:33:25 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:25 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 7968 bytes result sent to driver -2016-05-24 22:33:25 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:33:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 35 ms on localhost (1/2) -2016-05-24 22:33:25 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 7968 bytes result sent to driver -2016-05-24 22:33:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 35 ms on localhost (2/2) -2016-05-24 22:33:25 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at GradientDescent.scala:225) finished in 0.040 s -2016-05-24 22:33:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 22:33:25 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at GradientDescent.scala:225, took 0.120945 s -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=647739, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(3237) called with curMem=653331, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:57376 (size: 3.2 KB, free: 730.2 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Created broadcast 21 from broadcast at GradientDescent.scala:221 -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:57376 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:57376 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:57376 in memory (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Got job 12 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:33:25 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:57376 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:25 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:57376 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=593142, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=601443, maxMem=766075207 -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:57376 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:33:25 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:57376 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[29] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:25 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:57376 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:25 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 22:33:25 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:57376 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:57376 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:33:25 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:25 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 7968 bytes result sent to driver -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:57376 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 7968 bytes result sent to driver -2016-05-24 22:33:25 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 22:33:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 14 ms on localhost (1/2) -2016-05-24 22:33:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 16 ms on localhost (2/2) -2016-05-24 22:33:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 22:33:25 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at GradientDescent.scala:225) finished in 0.017 s -2016-05-24 22:33:25 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at GradientDescent.scala:225, took 0.032238 s -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:57376 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:57376 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=520946, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(3220) called with curMem=517756, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:57376 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:57376 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Created broadcast 23 from broadcast at GradientDescent.scala:221 -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:57376 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:57376 in memory (size: 2.3 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:57376 in memory (size: 388.0 B, free: 730.3 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Got job 13 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:33:25 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:25 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=491793, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:33:25 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=503577, maxMem=766075207 -2016-05-24 22:33:25 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:33:25 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:57376 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:25 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[31] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:25 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 22:33:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:25 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 22:33:25 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 22:33:25 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:25 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 7968 bytes result sent to driver -2016-05-24 22:33:25 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:33:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 23 ms on localhost (1/2) -2016-05-24 22:33:25 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 7968 bytes result sent to driver -2016-05-24 22:33:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 25 ms on localhost (2/2) -2016-05-24 22:33:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 22:33:25 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at GradientDescent.scala:225) finished in 0.029 s -2016-05-24 22:33:25 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at GradientDescent.scala:225, took 0.237398 s -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=507059, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(3222) called with curMem=512651, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 22:33:26 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:57376 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:33:26 INFO SparkContext:59 - Created broadcast 25 from broadcast at GradientDescent.scala:221 -2016-05-24 22:33:26 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:33:26 INFO DAGScheduler:59 - Got job 14 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:33:26 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:26 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=515873, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=527657, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:33:26 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:57376 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:26 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[33] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:26 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 22:33:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:26 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 22:33:26 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:33:26 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 7968 bytes result sent to driver -2016-05-24 22:33:26 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 22:33:26 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:26 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 7968 bytes result sent to driver -2016-05-24 22:33:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 16 ms on localhost (1/2) -2016-05-24 22:33:26 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at GradientDescent.scala:225) finished in 0.020 s -2016-05-24 22:33:26 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at GradientDescent.scala:225, took 0.037557 s -2016-05-24 22:33:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 19 ms on localhost (2/2) -2016-05-24 22:33:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=531141, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(3223) called with curMem=536733, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 22:33:26 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:57376 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:33:26 INFO SparkContext:59 - Created broadcast 27 from broadcast at GradientDescent.scala:221 -2016-05-24 22:33:26 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:33:26 INFO DAGScheduler:59 - Got job 15 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:33:26 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:26 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=539956, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 11.5 KB, free 730.1 MB) -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=551740, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.1 MB) -2016-05-24 22:33:26 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:57376 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:26 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[35] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:26 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 22:33:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:26 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 22:33:26 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 22:33:26 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:33:26 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 7968 bytes result sent to driver -2016-05-24 22:33:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 15 ms on localhost (1/2) -2016-05-24 22:33:26 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:26 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 7968 bytes result sent to driver -2016-05-24 22:33:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 21 ms on localhost (2/2) -2016-05-24 22:33:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 22:33:26 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at GradientDescent.scala:225) finished in 0.022 s -2016-05-24 22:33:26 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at GradientDescent.scala:225, took 0.035012 s -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=555223, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.5 KB, free 730.1 MB) -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(3239) called with curMem=560815, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-05-24 22:33:26 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:57376 (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:33:26 INFO SparkContext:59 - Created broadcast 29 from broadcast at GradientDescent.scala:221 -2016-05-24 22:33:26 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:33:26 INFO DAGScheduler:59 - Got job 16 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:33:26 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:26 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=564054, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=575838, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:33:26 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:57376 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:26 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[37] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:26 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 22:33:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:26 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 22:33:26 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 22:33:26 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:26 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:33:26 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 7968 bytes result sent to driver -2016-05-24 22:33:26 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 7968 bytes result sent to driver -2016-05-24 22:33:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 11 ms on localhost (1/2) -2016-05-24 22:33:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 11 ms on localhost (2/2) -2016-05-24 22:33:26 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at GradientDescent.scala:225) finished in 0.012 s -2016-05-24 22:33:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 22:33:26 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at GradientDescent.scala:225, took 0.029858 s -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=579321, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(3233) called with curMem=584913, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-05-24 22:33:26 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:57376 (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:33:26 INFO SparkContext:59 - Created broadcast 31 from broadcast at GradientDescent.scala:221 -2016-05-24 22:33:26 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:33:26 INFO DAGScheduler:59 - Got job 17 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:33:26 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:26 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=588146, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=599930, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:33:26 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:57376 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:26 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[39] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:26 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 22:33:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:26 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 22:33:26 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 22:33:26 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:26 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:33:26 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 7968 bytes result sent to driver -2016-05-24 22:33:26 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 7968 bytes result sent to driver -2016-05-24 22:33:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 10 ms on localhost (1/2) -2016-05-24 22:33:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 10 ms on localhost (2/2) -2016-05-24 22:33:26 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at GradientDescent.scala:225) finished in 0.011 s -2016-05-24 22:33:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 22:33:26 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at GradientDescent.scala:225, took 0.023550 s -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=603413, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(3226) called with curMem=609005, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-05-24 22:33:26 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:57376 (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:33:26 INFO SparkContext:59 - Created broadcast 33 from broadcast at GradientDescent.scala:221 -2016-05-24 22:33:26 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:33:26 INFO DAGScheduler:59 - Got job 18 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:33:26 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:26 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=612231, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=624015, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:33:26 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:57376 (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:26 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[41] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:26 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 22:33:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:26 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 22:33:26 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 22:33:26 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:26 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:33:26 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 7968 bytes result sent to driver -2016-05-24 22:33:26 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 7968 bytes result sent to driver -2016-05-24 22:33:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 13 ms on localhost (1/2) -2016-05-24 22:33:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 13 ms on localhost (2/2) -2016-05-24 22:33:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 22:33:26 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at GradientDescent.scala:225) finished in 0.015 s -2016-05-24 22:33:26 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at GradientDescent.scala:225, took 0.030798 s -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=627497, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(3220) called with curMem=633089, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.0 MB) -2016-05-24 22:33:26 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:57376 (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:33:26 INFO SparkContext:59 - Created broadcast 35 from broadcast at GradientDescent.scala:221 -2016-05-24 22:33:26 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:33:26 INFO DAGScheduler:59 - Got job 19 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:33:26 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:26 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=636309, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 11.5 KB, free 730.0 MB) -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(3483) called with curMem=648093, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.0 MB) -2016-05-24 22:33:26 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:57376 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:33:26 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[43] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:26 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 22:33:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:26 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 22:33:26 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 22:33:26 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:33:26 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:26 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 7968 bytes result sent to driver -2016-05-24 22:33:26 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 7968 bytes result sent to driver -2016-05-24 22:33:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 19 ms on localhost (1/2) -2016-05-24 22:33:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 19 ms on localhost (2/2) -2016-05-24 22:33:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 22:33:26 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:33:26 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at GradientDescent.scala:225, took 0.039428 s -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=651576, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(3223) called with curMem=657168, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.0 MB) -2016-05-24 22:33:26 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:57376 (size: 3.1 KB, free: 730.2 MB) -2016-05-24 22:33:26 INFO SparkContext:59 - Created broadcast 37 from broadcast at GradientDescent.scala:221 -2016-05-24 22:33:26 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:33:26 INFO DAGScheduler:59 - Got job 20 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:33:26 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:26 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=660391, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=672175, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:33:26 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:57376 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:33:26 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[45] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:26 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 22:33:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:26 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 22:33:26 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 22:33:26 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:26 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:33:26 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 7968 bytes result sent to driver -2016-05-24 22:33:26 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 7968 bytes result sent to driver -2016-05-24 22:33:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 17 ms on localhost (1/2) -2016-05-24 22:33:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 20 ms on localhost (2/2) -2016-05-24 22:33:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 22:33:26 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at GradientDescent.scala:225) finished in 0.021 s -2016-05-24 22:33:26 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at GradientDescent.scala:225, took 0.038685 s -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=675659, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(3229) called with curMem=681251, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 3.2 KB, free 729.9 MB) -2016-05-24 22:33:26 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:57376 (size: 3.2 KB, free: 730.2 MB) -2016-05-24 22:33:26 INFO SparkContext:59 - Created broadcast 39 from broadcast at GradientDescent.scala:221 -2016-05-24 22:33:26 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:33:26 INFO DAGScheduler:59 - Got job 21 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:33:26 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:26 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=684480, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=696264, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:33:26 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:57376 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:33:26 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[47] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:26 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 22:33:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:26 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 22:33:26 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 22:33:26 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:26 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:33:26 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 7968 bytes result sent to driver -2016-05-24 22:33:26 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 7968 bytes result sent to driver -2016-05-24 22:33:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 13 ms on localhost (1/2) -2016-05-24 22:33:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 14 ms on localhost (2/2) -2016-05-24 22:33:26 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:33:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 22:33:26 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at GradientDescent.scala:225, took 0.029553 s -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(5592) called with curMem=699748, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 5.5 KB, free 729.9 MB) -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(3234) called with curMem=705340, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.2 KB, free 729.9 MB) -2016-05-24 22:33:26 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:57376 (size: 3.2 KB, free: 730.2 MB) -2016-05-24 22:33:26 INFO SparkContext:59 - Created broadcast 41 from broadcast at GradientDescent.scala:221 -2016-05-24 22:33:26 INFO SparkContext:59 - Starting job: treeAggregate at GradientDescent.scala:225 -2016-05-24 22:33:26 INFO DAGScheduler:59 - Got job 22 (treeAggregate at GradientDescent.scala:225) with 2 output partitions -2016-05-24 22:33:26 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:26 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225), which has no missing parents -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(11784) called with curMem=708574, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 11.5 KB, free 729.9 MB) -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(3482) called with curMem=720358, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:33:26 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:57376 (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:33:26 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[49] at treeAggregate at GradientDescent.scala:225) -2016-05-24 22:33:26 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 22:33:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:33:26 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 22:33:26 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 22:33:26 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:33:26 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:33:26 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 7968 bytes result sent to driver -2016-05-24 22:33:26 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 7968 bytes result sent to driver -2016-05-24 22:33:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 14 ms on localhost (1/2) -2016-05-24 22:33:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 14 ms on localhost (2/2) -2016-05-24 22:33:26 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at GradientDescent.scala:225) finished in 0.016 s -2016-05-24 22:33:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 22:33:26 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at GradientDescent.scala:225, took 0.033583 s -2016-05-24 22:33:26 INFO GradientDescent:59 - GradientDescent.runMiniBatchSGD finished. Last 10 stochastic losses 4756.985924113548, 4319.927896434357, 3903.4385808079614, 3505.841537598476, 3124.097453008626, 2756.2417647061375, 2401.155210966652, 2057.4601181932067, 1724.026901602298, 1463.7810900203021 -2016-05-24 22:33:26 INFO SparkContext:59 - Starting job: sortByKey at BinaryClassificationMetrics.scala:156 -2016-05-24 22:33:26 INFO DAGScheduler:59 - Registering RDD 50 (map at SVMFromSpark2Learning.scala:54) -2016-05-24 22:33:26 INFO DAGScheduler:59 - Got job 23 (sortByKey at BinaryClassificationMetrics.scala:156) with 2 output partitions -2016-05-24 22:33:26 INFO DAGScheduler:59 - Final stage: ResultStage 24(sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:33:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 23) -2016-05-24 22:33:26 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 23) -2016-05-24 22:33:26 INFO DAGScheduler:59 - Submitting ShuffleMapStage 23 (MapPartitionsRDD[50] at map at SVMFromSpark2Learning.scala:54), which has no missing parents -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=723840, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 10.8 KB, free 729.9 MB) -2016-05-24 22:33:26 INFO MemoryStore:59 - ensureFreeSpace(6056) called with curMem=734848, maxMem=766075207 -2016-05-24 22:33:26 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 5.9 KB, free 729.9 MB) -2016-05-24 22:33:26 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:57376 (size: 5.9 KB, free: 730.2 MB) -2016-05-24 22:33:26 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 23 (MapPartitionsRDD[50] at map at SVMFromSpark2Learning.scala:54) -2016-05-24 22:33:26 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 22:33:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:33:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:33:26 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 22:33:26 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 22:33:26 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:33:26 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:33:27 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2254 bytes result sent to driver -2016-05-24 22:33:27 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2254 bytes result sent to driver -2016-05-24 22:33:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 493 ms on localhost (1/2) -2016-05-24 22:33:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 495 ms on localhost (2/2) -2016-05-24 22:33:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 22:33:27 INFO DAGScheduler:59 - ShuffleMapStage 23 (map at SVMFromSpark2Learning.scala:54) finished in 0.496 s -2016-05-24 22:33:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:33:27 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:33:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 24) -2016-05-24 22:33:27 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:33:27 INFO DAGScheduler:59 - Missing parents for ResultStage 24: List() -2016-05-24 22:33:27 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[53] at sortByKey at BinaryClassificationMetrics.scala:156), which is now runnable -2016-05-24 22:33:27 INFO MemoryStore:59 - ensureFreeSpace(3344) called with curMem=740904, maxMem=766075207 -2016-05-24 22:33:27 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 3.3 KB, free 729.9 MB) -2016-05-24 22:33:27 INFO MemoryStore:59 - ensureFreeSpace(1898) called with curMem=744248, maxMem=766075207 -2016-05-24 22:33:27 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 1898.0 B, free 729.9 MB) -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:57376 (size: 1898.0 B, free: 730.2 MB) -2016-05-24 22:33:27 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[53] at sortByKey at BinaryClassificationMetrics.scala:156) -2016-05-24 22:33:27 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 22:33:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:33:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:33:27 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 22:33:27 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 22:33:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:33:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:33:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-24 22:33:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-24 22:33:27 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 1485 bytes result sent to driver -2016-05-24 22:33:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 48 ms on localhost (1/2) -2016-05-24 22:33:27 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 1501 bytes result sent to driver -2016-05-24 22:33:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 52 ms on localhost (2/2) -2016-05-24 22:33:27 INFO DAGScheduler:59 - ResultStage 24 (sortByKey at BinaryClassificationMetrics.scala:156) finished in 0.053 s -2016-05-24 22:33:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 22:33:27 INFO DAGScheduler:59 - Job 23 finished: sortByKey at BinaryClassificationMetrics.scala:156, took 0.587256 s -2016-05-24 22:33:27 INFO SparkContext:59 - Starting job: collect at BinaryClassificationMetrics.scala:193 -2016-05-24 22:33:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-05-24 22:33:27 INFO DAGScheduler:59 - Registering RDD 51 (combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:33:27 INFO DAGScheduler:59 - Got job 24 (collect at BinaryClassificationMetrics.scala:193) with 2 output partitions -2016-05-24 22:33:27 INFO DAGScheduler:59 - Final stage: ResultStage 27(collect at BinaryClassificationMetrics.scala:193) -2016-05-24 22:33:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-24 22:33:27 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-24 22:33:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (ShuffledRDD[51] at combineByKey at BinaryClassificationMetrics.scala:152), which has no missing parents -2016-05-24 22:33:27 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=746146, maxMem=766075207 -2016-05-24 22:33:27 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 2.8 KB, free 729.9 MB) -2016-05-24 22:33:27 INFO MemoryStore:59 - ensureFreeSpace(1658) called with curMem=749042, maxMem=766075207 -2016-05-24 22:33:27 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 1658.0 B, free 729.9 MB) -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:57376 (size: 1658.0 B, free: 730.2 MB) -2016-05-24 22:33:27 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 26 (ShuffledRDD[51] at combineByKey at BinaryClassificationMetrics.scala:152) -2016-05-24 22:33:27 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 22:33:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:33:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 1890 bytes) -2016-05-24 22:33:27 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-24 22:33:27 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-24 22:33:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:33:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:33:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:33:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:33:27 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 1375 bytes result sent to driver -2016-05-24 22:33:27 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 1375 bytes result sent to driver -2016-05-24 22:33:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 33 ms on localhost (1/2) -2016-05-24 22:33:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 34 ms on localhost (2/2) -2016-05-24 22:33:27 INFO DAGScheduler:59 - ShuffleMapStage 26 (combineByKey at BinaryClassificationMetrics.scala:152) finished in 0.035 s -2016-05-24 22:33:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 22:33:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:33:27 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:33:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-24 22:33:27 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:33:27 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-24 22:33:27 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[56] at mapPartitions at BinaryClassificationMetrics.scala:189), which is now runnable -2016-05-24 22:33:27 INFO MemoryStore:59 - ensureFreeSpace(3472) called with curMem=750700, maxMem=766075207 -2016-05-24 22:33:27 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 3.4 KB, free 729.9 MB) -2016-05-24 22:33:27 INFO MemoryStore:59 - ensureFreeSpace(1981) called with curMem=754172, maxMem=766075207 -2016-05-24 22:33:27 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 1981.0 B, free 729.9 MB) -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:57376 (size: 1981.0 B, free: 730.2 MB) -2016-05-24 22:33:27 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[56] at mapPartitions at BinaryClassificationMetrics.scala:189) -2016-05-24 22:33:27 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 22:33:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 51, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:33:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 52, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:33:27 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 51) -2016-05-24 22:33:27 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 52) -2016-05-24 22:33:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:33:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:33:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:33:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:33:27 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 51). 1330 bytes result sent to driver -2016-05-24 22:33:27 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 52). 1330 bytes result sent to driver -2016-05-24 22:33:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 52) in 16 ms on localhost (1/2) -2016-05-24 22:33:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 51) in 18 ms on localhost (2/2) -2016-05-24 22:33:27 INFO DAGScheduler:59 - ResultStage 27 (collect at BinaryClassificationMetrics.scala:193) finished in 0.018 s -2016-05-24 22:33:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 22:33:27 INFO DAGScheduler:59 - Job 24 finished: collect at BinaryClassificationMetrics.scala:193, took 0.080429 s -2016-05-24 22:33:27 INFO BinaryClassificationMetrics:59 - Total counts: {numPos: 25, numNeg: 11} -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:57376 in memory (size: 1981.0 B, free: 730.2 MB) -2016-05-24 22:33:27 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:57376 in memory (size: 1658.0 B, free: 730.2 MB) -2016-05-24 22:33:27 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 22:33:27 INFO SparkContext:59 - Starting job: runJob at SlidingRDD.scala:75 -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:57376 in memory (size: 1898.0 B, free: 730.2 MB) -2016-05-24 22:33:27 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 22:33:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-05-24 22:33:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:57376 in memory (size: 5.9 KB, free: 730.2 MB) -2016-05-24 22:33:27 INFO DAGScheduler:59 - Got job 25 (runJob at SlidingRDD.scala:75) with 3 output partitions -2016-05-24 22:33:27 INFO DAGScheduler:59 - Final stage: ResultStage 30(runJob at SlidingRDD.scala:75) -2016-05-24 22:33:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-24 22:33:27 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 22:33:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:27 INFO DAGScheduler:59 - Submitting ResultStage 30 (UnionRDD[62] at UnionRDD at BinaryClassificationMetrics.scala:92), which has no missing parents -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:57376 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:33:27 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:57376 in memory (size: 3.2 KB, free: 730.2 MB) -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:57376 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:33:27 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 22:33:27 INFO MemoryStore:59 - ensureFreeSpace(5184) called with curMem=681251, maxMem=766075207 -2016-05-24 22:33:27 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 5.1 KB, free 729.9 MB) -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:57376 in memory (size: 3.2 KB, free: 730.2 MB) -2016-05-24 22:33:27 INFO MemoryStore:59 - ensureFreeSpace(2787) called with curMem=680843, maxMem=766075207 -2016-05-24 22:33:27 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.7 KB, free 729.9 MB) -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:57376 (size: 2.7 KB, free: 730.2 MB) -2016-05-24 22:33:27 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:57376 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:33:27 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 30 (UnionRDD[62] at UnionRDD at BinaryClassificationMetrics.scala:92) -2016-05-24 22:33:27 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 3 tasks -2016-05-24 22:33:27 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 22:33:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 53, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:57376 in memory (size: 3.1 KB, free: 730.2 MB) -2016-05-24 22:33:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 54, localhost, PROCESS_LOCAL, 2010 bytes) -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:57376 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-24 22:33:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 55, localhost, PROCESS_LOCAL, 2318 bytes) -2016-05-24 22:33:27 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 22:33:27 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 53) -2016-05-24 22:33:27 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 54) -2016-05-24 22:33:27 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 55) -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:57376 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:33:27 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 55). 1053 bytes result sent to driver -2016-05-24 22:33:27 INFO CacheManager:59 - Partition rdd_57_0 not found, computing it -2016-05-24 22:33:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:33:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:33:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 55) in 12 ms on localhost (1/3) -2016-05-24 22:33:27 INFO CacheManager:59 - Partition rdd_57_1 not found, computing it -2016-05-24 22:33:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:33:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:57376 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:27 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 22:33:27 INFO MemoryStore:59 - ensureFreeSpace(1448) called with curMem=620202, maxMem=766075207 -2016-05-24 22:33:27 INFO MemoryStore:59 - Block rdd_57_0 stored as values in memory (estimated size 1448.0 B, free 730.0 MB) -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Added rdd_57_0 in memory on localhost:57376 (size: 1448.0 B, free: 730.3 MB) -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:57376 in memory (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:33:27 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 53). 1883 bytes result sent to driver -2016-05-24 22:33:27 INFO MemoryStore:59 - ensureFreeSpace(1616) called with curMem=612832, maxMem=766075207 -2016-05-24 22:33:27 INFO MemoryStore:59 - Block rdd_57_1 stored as values in memory (estimated size 1616.0 B, free 730.0 MB) -2016-05-24 22:33:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 53) in 28 ms on localhost (2/3) -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Added rdd_57_1 in memory on localhost:57376 (size: 1616.0 B, free: 730.3 MB) -2016-05-24 22:33:27 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 54). 1883 bytes result sent to driver -2016-05-24 22:33:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 54) in 29 ms on localhost (3/3) -2016-05-24 22:33:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 22:33:27 INFO DAGScheduler:59 - ResultStage 30 (runJob at SlidingRDD.scala:75) finished in 0.031 s -2016-05-24 22:33:27 INFO DAGScheduler:59 - Job 25 finished: runJob at SlidingRDD.scala:75, took 0.057014 s -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:57376 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:27 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:57376 in memory (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:33:27 INFO SparkContext:59 - Starting job: aggregate at AreaUnderCurve.scala:45 -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:57376 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:27 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 22:33:27 INFO DAGScheduler:59 - Got job 26 (aggregate at AreaUnderCurve.scala:45) with 4 output partitions -2016-05-24 22:33:27 INFO DAGScheduler:59 - Final stage: ResultStage 33(aggregate at AreaUnderCurve.scala:45) -2016-05-24 22:33:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-05-24 22:33:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:27 INFO DAGScheduler:59 - Submitting ResultStage 33 (SlidingRDD[63] at RDD at SlidingRDD.scala:47), which has no missing parents -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:57376 in memory (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:33:27 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=566258, maxMem=766075207 -2016-05-24 22:33:27 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 5.5 KB, free 730.0 MB) -2016-05-24 22:33:27 INFO MemoryStore:59 - ensureFreeSpace(2971) called with curMem=556647, maxMem=766075207 -2016-05-24 22:33:27 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.1 MB) -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:57376 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:27 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:57376 (size: 2.9 KB, free: 730.3 MB) -2016-05-24 22:33:27 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (SlidingRDD[63] at RDD at SlidingRDD.scala:47) -2016-05-24 22:33:27 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:57376 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:33:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 56, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:33:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 57, localhost, PROCESS_LOCAL, 2429 bytes) -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:57376 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 58, localhost, PROCESS_LOCAL, 2635 bytes) -2016-05-24 22:33:27 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 22:33:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 59, localhost, PROCESS_LOCAL, 2475 bytes) -2016-05-24 22:33:27 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 57) -2016-05-24 22:33:27 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 58) -2016-05-24 22:33:27 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 56) -2016-05-24 22:33:27 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 59) -2016-05-24 22:33:27 INFO BlockManager:59 - Found block rdd_57_1 locally -2016-05-24 22:33:27 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 59). 955 bytes result sent to driver -2016-05-24 22:33:27 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 57). 2084 bytes result sent to driver -2016-05-24 22:33:27 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 58). 955 bytes result sent to driver -2016-05-24 22:33:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 59) in 12 ms on localhost (1/4) -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:57376 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:33:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 57) in 15 ms on localhost (2/4) -2016-05-24 22:33:27 INFO BlockManager:59 - Found block rdd_57_0 locally -2016-05-24 22:33:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 58) in 15 ms on localhost (3/4) -2016-05-24 22:33:27 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 56). 2084 bytes result sent to driver -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:57376 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 56) in 23 ms on localhost (4/4) -2016-05-24 22:33:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 22:33:27 INFO DAGScheduler:59 - ResultStage 33 (aggregate at AreaUnderCurve.scala:45) finished in 0.025 s -2016-05-24 22:33:27 INFO DAGScheduler:59 - Job 26 finished: aggregate at AreaUnderCurve.scala:45, took 0.039412 s -2016-05-24 22:33:27 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:57376 in memory (size: 3.1 KB, free: 730.3 MB) -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:57376 in memory (size: 3.4 KB, free: 730.3 MB) -2016-05-24 22:33:27 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:57376 in memory (size: 3.2 KB, free: 730.3 MB) -2016-05-24 22:33:27 INFO SparkContext:59 - Starting job: take at SVMFromSpark2Learning.scala:66 -2016-05-24 22:33:27 INFO DAGScheduler:59 - Got job 27 (take at SVMFromSpark2Learning.scala:66) with 1 output partitions -2016-05-24 22:33:27 INFO DAGScheduler:59 - Final stage: ResultStage 34(take at SVMFromSpark2Learning.scala:66) -2016-05-24 22:33:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:27 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[50] at map at SVMFromSpark2Learning.scala:54), which has no missing parents -2016-05-24 22:33:27 INFO MemoryStore:59 - ensureFreeSpace(10320) called with curMem=478546, maxMem=766075207 -2016-05-24 22:33:27 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 10.1 KB, free 730.1 MB) -2016-05-24 22:33:27 INFO MemoryStore:59 - ensureFreeSpace(5705) called with curMem=488866, maxMem=766075207 -2016-05-24 22:33:27 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 5.6 KB, free 730.1 MB) -2016-05-24 22:33:27 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:57376 (size: 5.6 KB, free: 730.3 MB) -2016-05-24 22:33:27 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 34 (MapPartitionsRDD[50] at map at SVMFromSpark2Learning.scala:54) -2016-05-24 22:33:27 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 1 tasks -2016-05-24 22:33:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 60, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:33:27 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 60) -2016-05-24 22:33:27 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:33:27 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 60). 2401 bytes result sent to driver -2016-05-24 22:33:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 60) in 6 ms on localhost (1/1) -2016-05-24 22:33:27 INFO DAGScheduler:59 - ResultStage 34 (take at SVMFromSpark2Learning.scala:66) finished in 0.006 s -2016-05-24 22:33:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 22:33:27 INFO DAGScheduler:59 - Job 27 finished: take at SVMFromSpark2Learning.scala:66, took 0.015183 s -2016-05-24 22:33:27 INFO SparkContext:59 - Starting job: saveAsTextFile at GLMClassificationModel.scala:61 -2016-05-24 22:33:27 INFO DAGScheduler:59 - Got job 28 (saveAsTextFile at GLMClassificationModel.scala:61) with 1 output partitions -2016-05-24 22:33:27 INFO DAGScheduler:59 - Final stage: ResultStage 35(saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:33:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:27 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[65] at saveAsTextFile at GLMClassificationModel.scala:61), which has no missing parents -2016-05-24 22:33:27 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=494571, maxMem=766075207 -2016-05-24 22:33:27 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 124.0 KB, free 730.0 MB) -2016-05-24 22:33:28 INFO MemoryStore:59 - ensureFreeSpace(42574) called with curMem=621563, maxMem=766075207 -2016-05-24 22:33:28 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.0 MB) -2016-05-24 22:33:28 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:57376 (size: 41.6 KB, free: 730.2 MB) -2016-05-24 22:33:28 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 35 (MapPartitionsRDD[65] at saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:33:28 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 1 tasks -2016-05-24 22:33:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 61, localhost, PROCESS_LOCAL, 2197 bytes) -2016-05-24 22:33:28 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 61) -2016-05-24 22:33:28 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242233_0035_m_000000_61' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524223327463/result/metadata/_temporary/0/task_201605242233_0035_m_000000 -2016-05-24 22:33:28 INFO SparkHadoopMapRedUtil:59 - attempt_201605242233_0035_m_000000_61: Committed -2016-05-24 22:33:28 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 61). 1864 bytes result sent to driver -2016-05-24 22:33:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 61) in 170 ms on localhost (1/1) -2016-05-24 22:33:28 INFO DAGScheduler:59 - ResultStage 35 (saveAsTextFile at GLMClassificationModel.scala:61) finished in 0.170 s -2016-05-24 22:33:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 22:33:28 INFO DAGScheduler:59 - Job 28 finished: saveAsTextFile at GLMClassificationModel.scala:61, took 0.421622 s -2016-05-24 22:33:29 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:33:29 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:33:29 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:65 -2016-05-24 22:33:29 INFO DAGScheduler:59 - Got job 29 (parquet at GLMClassificationModel.scala:65) with 1 output partitions -2016-05-24 22:33:29 INFO DAGScheduler:59 - Final stage: ResultStage 36(parquet at GLMClassificationModel.scala:65) -2016-05-24 22:33:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:29 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[67] at rddToDataFrameHolder at GLMClassificationModel.scala:65), which has no missing parents -2016-05-24 22:33:29 INFO MemoryStore:59 - ensureFreeSpace(67104) called with curMem=664137, maxMem=766075207 -2016-05-24 22:33:29 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 65.5 KB, free 729.9 MB) -2016-05-24 22:33:29 INFO MemoryStore:59 - ensureFreeSpace(23443) called with curMem=731241, maxMem=766075207 -2016-05-24 22:33:29 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 22.9 KB, free 729.9 MB) -2016-05-24 22:33:29 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:57376 (size: 22.9 KB, free: 730.2 MB) -2016-05-24 22:33:29 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 36 (MapPartitionsRDD[67] at rddToDataFrameHolder at GLMClassificationModel.scala:65) -2016-05-24 22:33:29 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 1 tasks -2016-05-24 22:33:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 62, localhost, PROCESS_LOCAL, 7978 bytes) -2016-05-24 22:33:29 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 62) -2016-05-24 22:33:29 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:33:29 INFO CodecConfig:151 - Compression: GZIP -2016-05-24 22:33:29 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-24 22:33:29 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-24 22:33:29 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-24 22:33:29 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-24 22:33:29 INFO ParquetOutputFormat:151 - Validation is off -2016-05-24 22:33:29 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-24 22:33:29 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-24 22:33:30 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 6,084 -2016-05-24 22:33:30 INFO ColumnChunkPageWriteStore:151 - written 61B for [weights, type] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN] -2016-05-24 22:33:30 INFO ColumnChunkPageWriteStore:151 - written 48B for [weights, size] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN] -2016-05-24 22:33:30 INFO ColumnChunkPageWriteStore:151 - written 51B for [weights, indices, array] INT32: 1 values, 13B raw, 30B comp, 1 pages, encodings: [RLE, PLAIN] -2016-05-24 22:33:30 INFO ColumnChunkPageWriteStore:151 - written 635B for [weights, values, array] DOUBLE: 692 values, 668B raw, 591B comp, 1 pages, encodings: [RLE, PLAIN_DICTIONARY], dic { 404 entries, 3,232B raw, 404B comp} -2016-05-24 22:33:30 INFO ColumnChunkPageWriteStore:151 - written 69B for [intercept] DOUBLE: 1 values, 14B raw, 28B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN] -2016-05-24 22:33:30 INFO ColumnChunkPageWriteStore:151 - written 47B for [threshold] DOUBLE: 1 values, 6B raw, 26B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN] -2016-05-24 22:33:30 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242233_0036_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524223327463/result/data/_temporary/0/task_201605242233_0036_m_000000 -2016-05-24 22:33:30 INFO SparkHadoopMapRedUtil:59 - attempt_201605242233_0036_m_000000_0: Committed -2016-05-24 22:33:30 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 62). 915 bytes result sent to driver -2016-05-24 22:33:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 62) in 456 ms on localhost (1/1) -2016-05-24 22:33:30 INFO DAGScheduler:59 - ResultStage 36 (parquet at GLMClassificationModel.scala:65) finished in 0.457 s -2016-05-24 22:33:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 22:33:30 INFO DAGScheduler:59 - Job 29 finished: parquet at GLMClassificationModel.scala:65, took 0.493699 s -2016-05-24 22:33:30 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:33:30 INFO DefaultWriterContainer:59 - Job job_201605242233_0000 committed. -2016-05-24 22:33:30 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524223327463/result/data on driver -2016-05-24 22:33:30 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524223327463/result/data on driver -2016-05-24 22:33:30 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=754684, maxMem=766075207 -2016-05-24 22:33:30 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 225.3 KB, free 729.6 MB) -2016-05-24 22:33:30 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=985388, maxMem=766075207 -2016-05-24 22:33:30 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.6 MB) -2016-05-24 22:33:30 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:57376 (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:33:30 INFO SparkContext:59 - Created broadcast 52 from textFile at modelSaveLoad.scala:129 -2016-05-24 22:33:30 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:33:30 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-24 22:33:30 INFO DAGScheduler:59 - Got job 30 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-24 22:33:30 INFO DAGScheduler:59 - Final stage: ResultStage 37(first at modelSaveLoad.scala:129) -2016-05-24 22:33:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:30 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[70] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-24 22:33:30 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=1005176, maxMem=766075207 -2016-05-24 22:33:30 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 3.1 KB, free 729.6 MB) -2016-05-24 22:33:30 INFO MemoryStore:59 - ensureFreeSpace(1894) called with curMem=1008392, maxMem=766075207 -2016-05-24 22:33:30 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 1894.0 B, free 729.6 MB) -2016-05-24 22:33:30 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:57376 (size: 1894.0 B, free: 730.2 MB) -2016-05-24 22:33:30 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 37 (MapPartitionsRDD[70] at textFile at modelSaveLoad.scala:129) -2016-05-24 22:33:30 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 1 tasks -2016-05-24 22:33:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 63, localhost, PROCESS_LOCAL, 2238 bytes) -2016-05-24 22:33:30 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 63) -2016-05-24 22:33:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524223327463/result/metadata/part-00000:0+54 -2016-05-24 22:33:30 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 63). 2154 bytes result sent to driver -2016-05-24 22:33:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 63) in 14 ms on localhost (1/1) -2016-05-24 22:33:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 22:33:30 INFO DAGScheduler:59 - ResultStage 37 (first at modelSaveLoad.scala:129) finished in 0.014 s -2016-05-24 22:33:30 INFO DAGScheduler:59 - Job 30 finished: first at modelSaveLoad.scala:129, took 0.030758 s -2016-05-24 22:33:30 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524223327463/result/data on driver -2016-05-24 22:33:31 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:57376 in memory (size: 1894.0 B, free: 730.2 MB) -2016-05-24 22:33:31 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 22:33:31 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:57376 in memory (size: 19.3 KB, free: 730.2 MB) -2016-05-24 22:33:31 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:57376 in memory (size: 22.9 KB, free: 730.2 MB) -2016-05-24 22:33:31 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 22:33:31 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:57376 in memory (size: 41.6 KB, free: 730.3 MB) -2016-05-24 22:33:31 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 22:33:31 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:57376 in memory (size: 5.6 KB, free: 730.3 MB) -2016-05-24 22:33:31 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 22:33:31 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:57376 in memory (size: 2.9 KB, free: 730.3 MB) -2016-05-24 22:33:31 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 22:33:31 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:57376 in memory (size: 2.7 KB, free: 730.3 MB) -2016-05-24 22:33:31 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 22:33:31 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:78 -2016-05-24 22:33:31 INFO DAGScheduler:59 - Got job 31 (parquet at GLMClassificationModel.scala:78) with 4 output partitions -2016-05-24 22:33:31 INFO DAGScheduler:59 - Final stage: ResultStage 38(parquet at GLMClassificationModel.scala:78) -2016-05-24 22:33:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:31 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[72] at parquet at GLMClassificationModel.scala:78), which has no missing parents -2016-05-24 22:33:31 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=461948, maxMem=766075207 -2016-05-24 22:33:31 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 61.5 KB, free 730.1 MB) -2016-05-24 22:33:31 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=524948, maxMem=766075207 -2016-05-24 22:33:31 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.1 MB) -2016-05-24 22:33:31 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:57376 (size: 20.6 KB, free: 730.3 MB) -2016-05-24 22:33:31 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[72] at parquet at GLMClassificationModel.scala:78) -2016-05-24 22:33:31 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-05-24 22:33:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 64, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:33:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 65, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:33:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 66, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:33:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 67, localhost, PROCESS_LOCAL, 2279 bytes) -2016-05-24 22:33:31 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 64) -2016-05-24 22:33:31 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 66) -2016-05-24 22:33:31 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 65) -2016-05-24 22:33:31 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 67) -2016-05-24 22:33:31 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:33:31 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:33:31 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 66). 936 bytes result sent to driver -2016-05-24 22:33:31 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 64). 936 bytes result sent to driver -2016-05-24 22:33:31 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:33:31 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:33:31 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 65). 936 bytes result sent to driver -2016-05-24 22:33:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 66) in 94 ms on localhost (1/4) -2016-05-24 22:33:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 64) in 97 ms on localhost (2/4) -2016-05-24 22:33:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 65) in 97 ms on localhost (3/4) -2016-05-24 22:33:31 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 67). 1802 bytes result sent to driver -2016-05-24 22:33:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 67) in 113 ms on localhost (4/4) -2016-05-24 22:33:31 INFO DAGScheduler:59 - ResultStage 38 (parquet at GLMClassificationModel.scala:78) finished in 0.115 s -2016-05-24 22:33:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-05-24 22:33:31 INFO DAGScheduler:59 - Job 31 finished: parquet at GLMClassificationModel.scala:78, took 0.169232 s -2016-05-24 22:33:31 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=546021, maxMem=766075207 -2016-05-24 22:33:31 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 86.4 KB, free 730.0 MB) -2016-05-24 22:33:31 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=634533, maxMem=766075207 -2016-05-24 22:33:31 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.0 MB) -2016-05-24 22:33:31 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:57376 (size: 19.3 KB, free: 730.3 MB) -2016-05-24 22:33:31 INFO SparkContext:59 - Created broadcast 55 from take at GLMClassificationModel.scala:79 -2016-05-24 22:33:31 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-24 22:33:31 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524223327463/result/data/part-r-00000-3cc25a32-9a61-4a09-8a20-8d51b1445579.gz.parquet -2016-05-24 22:33:31 INFO SparkContext:59 - Starting job: take at GLMClassificationModel.scala:79 -2016-05-24 22:33:31 INFO DAGScheduler:59 - Got job 32 (take at GLMClassificationModel.scala:79) with 1 output partitions -2016-05-24 22:33:31 INFO DAGScheduler:59 - Final stage: ResultStage 39(take at GLMClassificationModel.scala:79) -2016-05-24 22:33:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:33:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:33:31 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[74] at take at GLMClassificationModel.scala:79), which has no missing parents -2016-05-24 22:33:31 INFO MemoryStore:59 - ensureFreeSpace(4168) called with curMem=654321, maxMem=766075207 -2016-05-24 22:33:31 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 4.1 KB, free 730.0 MB) -2016-05-24 22:33:31 INFO MemoryStore:59 - ensureFreeSpace(2438) called with curMem=658489, maxMem=766075207 -2016-05-24 22:33:31 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.0 MB) -2016-05-24 22:33:31 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:57376 (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:33:31 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:33:31 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:57376 in memory (size: 20.6 KB, free: 730.3 MB) -2016-05-24 22:33:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 39 (MapPartitionsRDD[74] at take at GLMClassificationModel.scala:79) -2016-05-24 22:33:31 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 1 tasks -2016-05-24 22:33:31 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 22:33:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 68, localhost, PROCESS_LOCAL, 2300 bytes) -2016-05-24 22:33:31 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 68) -2016-05-24 22:33:31 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/sample_libsvm_data20160524223327463/result/data/part-r-00000-3cc25a32-9a61-4a09-8a20-8d51b1445579.gz.parquet start: 0 end: 4863 length: 4863 hosts: []} -2016-05-24 22:33:32 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-24 22:33:32 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group weights { - required int32 type (INT_8); - optional int32 size; - optional group indices (LIST) { - repeated int32 array; - } - optional group values (LIST) { - repeated double array; - } - } - optional double intercept; - optional double threshold; -} - -Catalyst form: -StructType(StructField(weights,org.apache.spark.mllib.linalg.VectorUDT@f71b0bce,true), StructField(intercept,DoubleType,true), StructField(threshold,DoubleType,true)) - -2016-05-24 22:33:32 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-24 22:33:32 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-24 22:33:32 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-24 22:33:32 INFO InternalParquetRecordReader:151 - block read in memory in 10 ms. row count = 1 -2016-05-24 22:33:32 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 68). 12288 bytes result sent to driver -2016-05-24 22:33:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 68) in 534 ms on localhost (1/1) -2016-05-24 22:33:32 INFO DAGScheduler:59 - ResultStage 39 (take at GLMClassificationModel.scala:79) finished in 0.534 s -2016-05-24 22:33:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-05-24 22:33:32 INFO DAGScheduler:59 - Job 32 finished: take at GLMClassificationModel.scala:79, took 0.881133 s -2016-05-24 22:33:32 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:57376 in memory (size: 2.4 KB, free: 730.3 MB) -2016-05-24 22:33:32 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-05-24 22:33:32 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:57376 in memory (size: 19.3 KB, free: 730.3 MB) -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 22:33:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 22:33:32 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 22:33:32 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 22:36:00 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 22:36:00 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 22:36:01 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 22:36:01 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 22:36:01 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 22:36:01 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 22:36:02 INFO Remoting:74 - Starting remoting -2016-05-24 22:36:02 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:57486] -2016-05-24 22:36:02 INFO Utils:59 - Successfully started service 'sparkDriver' on port 57486. -2016-05-24 22:36:02 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 22:36:02 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 22:36:02 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-00d40251-37e3-428d-a7f6-fe04a9bf10df -2016-05-24 22:36:02 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 22:36:02 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-feed7b92-32d4-4789-86e7-6ab38906df52\httpd-2b0ae218-4ab3-490b-aeef-8cc25fc26a5e -2016-05-24 22:36:02 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 22:36:02 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:36:02 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:57491 -2016-05-24 22:36:02 INFO Utils:59 - Successfully started service 'HTTP file server' on port 57491. -2016-05-24 22:36:02 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 22:36:02 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:36:02 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 22:36:02 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 22:36:02 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 22:36:02 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 22:36:02 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 22:36:03 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 57514. -2016-05-24 22:36:03 INFO NettyBlockTransferService:59 - Server created on 57514 -2016-05-24 22:36:03 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 22:36:03 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:57514 with 730.6 MB RAM, BlockManagerId(driver, localhost, 57514) -2016-05-24 22:36:03 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 22:36:04 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 22:36:04 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 22:36:04 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 22:36:04 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 22:36:04 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:57514 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 22:36:04 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 22:36:05 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 22:36:05 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:36:05 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 22:36:05 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 22:36:05 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 22:36:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:05 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:05 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 22:36:05 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 22:36:05 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 22:36:05 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=175476, maxMem=766075207 -2016-05-24 22:36:05 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:36:05 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:57514 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:36:05 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 22:36:05 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 22:36:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:05 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 22:36:05 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 22:36:05 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 22:36:05 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 22:36:05 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 22:36:05 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 22:36:05 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 22:36:05 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 22:36:05 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 22:36:05 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 22:36:05 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 22:36:06 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177621, maxMem=766075207 -2016-05-24 22:36:06 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 22:36:06 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:57514 (size: 83.1 KB, free: 730.5 MB) -2016-05-24 22:36:06 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262701, maxMem=766075207 -2016-05-24 22:36:06 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 22:36:06 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:57514 (size: 83.6 KB, free: 730.4 MB) -2016-05-24 22:36:06 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 22:36:06 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 22:36:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 483 ms on localhost (1/2) -2016-05-24 22:36:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 467 ms on localhost (2/2) -2016-05-24 22:36:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 22:36:06 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.504 s -2016-05-24 22:36:06 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.681560 s -2016-05-24 22:36:06 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 22:36:06 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 22:36:06 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:36:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:06 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 22:36:06 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=348349, maxMem=766075207 -2016-05-24 22:36:06 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-24 22:36:06 INFO MemoryStore:59 - ensureFreeSpace(2483) called with curMem=352909, maxMem=766075207 -2016-05-24 22:36:06 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-24 22:36:06 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:57514 (size: 2.4 KB, free: 730.4 MB) -2016-05-24 22:36:06 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:06 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:36:06 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 22:36:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:06 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 22:36:06 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-05-24 22:36:06 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:36:06 INFO MemoryStore:59 - ensureFreeSpace(54392) called with curMem=355392, maxMem=766075207 -2016-05-24 22:36:06 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 53.1 KB, free 730.2 MB) -2016-05-24 22:36:06 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:57514 (size: 53.1 KB, free: 730.4 MB) -2016-05-24 22:36:06 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2611 bytes result sent to driver -2016-05-24 22:36:06 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.041 s -2016-05-24 22:36:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 40 ms on localhost (1/1) -2016-05-24 22:36:06 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.087801 s -2016-05-24 22:36:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 22:36:06 INFO SparkContext:59 - Starting job: count at DataValidators.scala:56 -2016-05-24 22:36:06 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:56) with 2 output partitions -2016-05-24 22:36:06 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:56) -2016-05-24 22:36:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:06 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:55), which has no missing parents -2016-05-24 22:36:06 INFO MemoryStore:59 - ensureFreeSpace(4456) called with curMem=409784, maxMem=766075207 -2016-05-24 22:36:06 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.4 KB, free 730.2 MB) -2016-05-24 22:36:06 INFO MemoryStore:59 - ensureFreeSpace(2351) called with curMem=414240, maxMem=766075207 -2016-05-24 22:36:06 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-24 22:36:06 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:57514 (size: 2.3 KB, free: 730.4 MB) -2016-05-24 22:36:06 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:55) -2016-05-24 22:36:06 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 22:36:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:06 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 22:36:06 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 22:36:06 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-05-24 22:36:06 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:36:06 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:36:06 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 22:36:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 31 ms on localhost (1/2) -2016-05-24 22:36:06 INFO MemoryStore:59 - ensureFreeSpace(62184) called with curMem=416591, maxMem=766075207 -2016-05-24 22:36:06 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 60.7 KB, free 730.1 MB) -2016-05-24 22:36:06 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:57514 (size: 60.7 KB, free: 730.3 MB) -2016-05-24 22:36:06 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2662 bytes result sent to driver -2016-05-24 22:36:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 40 ms on localhost (2/2) -2016-05-24 22:36:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 22:36:06 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:56) finished in 0.044 s -2016-05-24 22:36:06 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:56, took 0.084798 s -2016-05-24 22:36:06 INFO SparkContext:59 - Starting job: treeAggregate at StandardScaler.scala:55 -2016-05-24 22:36:06 INFO DAGScheduler:59 - Got job 3 (treeAggregate at StandardScaler.scala:55) with 2 output partitions -2016-05-24 22:36:06 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at StandardScaler.scala:55) -2016-05-24 22:36:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:06 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:06 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[12] at treeAggregate at StandardScaler.scala:55), which has no missing parents -2016-05-24 22:36:07 INFO MemoryStore:59 - ensureFreeSpace(5392) called with curMem=478775, maxMem=766075207 -2016-05-24 22:36:07 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.3 KB, free 730.1 MB) -2016-05-24 22:36:07 INFO MemoryStore:59 - ensureFreeSpace(2767) called with curMem=484167, maxMem=766075207 -2016-05-24 22:36:07 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.1 MB) -2016-05-24 22:36:07 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:57514 (size: 2.7 KB, free: 730.3 MB) -2016-05-24 22:36:07 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[12] at treeAggregate at StandardScaler.scala:55) -2016-05-24 22:36:07 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 22:36:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:07 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 22:36:07 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 22:36:07 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:36:07 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:36:07 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 41369 bytes result sent to driver -2016-05-24 22:36:07 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 41369 bytes result sent to driver -2016-05-24 22:36:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 157 ms on localhost (1/2) -2016-05-24 22:36:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 162 ms on localhost (2/2) -2016-05-24 22:36:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 22:36:07 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at StandardScaler.scala:55) finished in 0.164 s -2016-05-24 22:36:07 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at StandardScaler.scala:55, took 0.215626 s -2016-05-24 22:36:07 INFO SparkContext:59 - Starting job: count at LBFGS.scala:170 -2016-05-24 22:36:07 INFO DAGScheduler:59 - Got job 4 (count at LBFGS.scala:170) with 2 output partitions -2016-05-24 22:36:07 INFO DAGScheduler:59 - Final stage: ResultStage 4(count at LBFGS.scala:170) -2016-05-24 22:36:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:07 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at map at GeneralizedLinearAlgorithm.scala:290), which has no missing parents -2016-05-24 22:36:07 INFO MemoryStore:59 - ensureFreeSpace(15648) called with curMem=486934, maxMem=766075207 -2016-05-24 22:36:07 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 15.3 KB, free 730.1 MB) -2016-05-24 22:36:07 INFO MemoryStore:59 - ensureFreeSpace(8226) called with curMem=502582, maxMem=766075207 -2016-05-24 22:36:07 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 8.0 KB, free 730.1 MB) -2016-05-24 22:36:07 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:57514 (size: 8.0 KB, free: 730.3 MB) -2016-05-24 22:36:07 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at map at GeneralizedLinearAlgorithm.scala:290) -2016-05-24 22:36:07 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 22:36:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:07 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 22:36:07 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 22:36:07 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-24 22:36:07 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-24 22:36:07 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:36:07 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:36:07 INFO MemoryStore:59 - ensureFreeSpace(55112) called with curMem=510808, maxMem=766075207 -2016-05-24 22:36:07 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 53.8 KB, free 730.0 MB) -2016-05-24 22:36:07 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=565920, maxMem=766075207 -2016-05-24 22:36:07 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-24 22:36:07 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:57514 (size: 53.8 KB, free: 730.2 MB) -2016-05-24 22:36:07 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:57514 (size: 61.5 KB, free: 730.2 MB) -2016-05-24 22:36:07 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2662 bytes result sent to driver -2016-05-24 22:36:07 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2662 bytes result sent to driver -2016-05-24 22:36:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 64 ms on localhost (1/2) -2016-05-24 22:36:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 63 ms on localhost (2/2) -2016-05-24 22:36:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 22:36:07 INFO DAGScheduler:59 - ResultStage 4 (count at LBFGS.scala:170) finished in 0.065 s -2016-05-24 22:36:07 INFO DAGScheduler:59 - Job 4 finished: count at LBFGS.scala:170, took 0.103114 s -2016-05-24 22:36:10 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=628920, maxMem=766075207 -2016-05-24 22:36:10 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 48.7 KB, free 729.9 MB) -2016-05-24 22:36:10 INFO MemoryStore:59 - ensureFreeSpace(2476) called with curMem=678800, maxMem=766075207 -2016-05-24 22:36:10 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.4 KB, free 729.9 MB) -2016-05-24 22:36:10 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:57514 (size: 2.4 KB, free: 730.2 MB) -2016-05-24 22:36:10 INFO SparkContext:59 - Created broadcast 6 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:10 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:10 INFO DAGScheduler:59 - Got job 5 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:10 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:10 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[14] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:10 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=681276, maxMem=766075207 -2016-05-24 22:36:10 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 65.2 KB, free 729.9 MB) -2016-05-24 22:36:11 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=748004, maxMem=766075207 -2016-05-24 22:36:11 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.9 MB) -2016-05-24 22:36:12 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.2 MB) -2016-05-24 22:36:12 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[14] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:12 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 22:36:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:12 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 22:36:12 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:12 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 22:36:12 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 52374 bytes result sent to driver -2016-05-24 22:36:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 47 ms on localhost (1/2) -2016-05-24 22:36:12 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:12 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 52374 bytes result sent to driver -2016-05-24 22:36:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 56 ms on localhost (2/2) -2016-05-24 22:36:12 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at LBFGS.scala:218) finished in 0.061 s -2016-05-24 22:36:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 22:36:12 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 22:36:12 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 22:36:12 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at LBFGS.scala:218, took 1.361340 s -2016-05-24 22:36:13 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=759227, maxMem=766075207 -2016-05-24 22:36:13 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 48.7 KB, free 729.8 MB) -2016-05-24 22:36:13 INFO MemoryStore:59 - ensureFreeSpace(11374) called with curMem=809107, maxMem=766075207 -2016-05-24 22:36:13 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 11.1 KB, free 729.8 MB) -2016-05-24 22:36:13 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:57514 (size: 11.1 KB, free: 730.1 MB) -2016-05-24 22:36:13 INFO SparkContext:59 - Created broadcast 8 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:13 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:13 INFO DAGScheduler:59 - Got job 6 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:13 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:13 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[15] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:13 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=820481, maxMem=766075207 -2016-05-24 22:36:13 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 65.2 KB, free 729.7 MB) -2016-05-24 22:36:13 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=887209, maxMem=766075207 -2016-05-24 22:36:13 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.7 MB) -2016-05-24 22:36:13 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:13 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[15] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:13 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 22:36:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:13 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 22:36:13 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 22:36:13 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:13 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:13 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 52374 bytes result sent to driver -2016-05-24 22:36:13 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 52374 bytes result sent to driver -2016-05-24 22:36:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 42 ms on localhost (1/2) -2016-05-24 22:36:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 41 ms on localhost (2/2) -2016-05-24 22:36:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 22:36:13 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at LBFGS.scala:218) finished in 0.045 s -2016-05-24 22:36:13 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at LBFGS.scala:218, took 0.161368 s -2016-05-24 22:36:13 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=898432, maxMem=766075207 -2016-05-24 22:36:13 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 48.7 KB, free 729.7 MB) -2016-05-24 22:36:13 INFO MemoryStore:59 - ensureFreeSpace(11373) called with curMem=948312, maxMem=766075207 -2016-05-24 22:36:13 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 11.1 KB, free 729.7 MB) -2016-05-24 22:36:13 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:57514 (size: 11.1 KB, free: 730.1 MB) -2016-05-24 22:36:13 INFO SparkContext:59 - Created broadcast 10 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:13 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:13 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:57514 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:13 INFO DAGScheduler:59 - Got job 7 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:13 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:13 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[16] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:13 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=881734, maxMem=766075207 -2016-05-24 22:36:13 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 65.2 KB, free 729.7 MB) -2016-05-24 22:36:13 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 22:36:13 INFO MemoryStore:59 - ensureFreeSpace(11220) called with curMem=887208, maxMem=766075207 -2016-05-24 22:36:13 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.7 MB) -2016-05-24 22:36:13 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:57514 in memory (size: 11.1 KB, free: 730.1 MB) -2016-05-24 22:36:13 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:13 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[16] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:13 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 22:36:13 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:57514 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:13 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 22:36:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:13 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:57514 in memory (size: 2.4 KB, free: 730.1 MB) -2016-05-24 22:36:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:13 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 22:36:13 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 22:36:13 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:57514 in memory (size: 8.0 KB, free: 730.2 MB) -2016-05-24 22:36:13 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 22:36:13 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:13 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:13 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:57514 in memory (size: 2.7 KB, free: 730.2 MB) -2016-05-24 22:36:13 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 52374 bytes result sent to driver -2016-05-24 22:36:13 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 22:36:13 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 52374 bytes result sent to driver -2016-05-24 22:36:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 22 ms on localhost (1/2) -2016-05-24 22:36:13 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:57514 in memory (size: 2.3 KB, free: 730.2 MB) -2016-05-24 22:36:13 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 22:36:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 26 ms on localhost (2/2) -2016-05-24 22:36:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 22:36:13 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at LBFGS.scala:218) finished in 0.026 s -2016-05-24 22:36:13 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at LBFGS.scala:218, took 0.067283 s -2016-05-24 22:36:13 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:57514 in memory (size: 2.4 KB, free: 730.2 MB) -2016-05-24 22:36:13 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 22:36:13 INFO StrongWolfeLineSearch:11 - Line search t: 0.05601643361637265 fval: 1.2089737414014867 rhs: 2.3023346338192554 cdd: 5.371949174352343 -2016-05-24 22:36:13 INFO LBFGS:11 - Step Size: 0.05602 -2016-05-24 22:36:13 INFO LBFGS:11 - Val and Grad Norm: 1.20897 (rel: 0.475) 7.36229 -2016-05-24 22:36:13 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=722238, maxMem=766075207 -2016-05-24 22:36:13 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 48.7 KB, free 729.8 MB) -2016-05-24 22:36:13 INFO MemoryStore:59 - ensureFreeSpace(11987) called with curMem=772118, maxMem=766075207 -2016-05-24 22:36:13 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.8 MB) -2016-05-24 22:36:13 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.2 MB) -2016-05-24 22:36:13 INFO SparkContext:59 - Created broadcast 12 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:13 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:13 INFO DAGScheduler:59 - Got job 8 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:13 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:13 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[17] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:13 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=784105, maxMem=766075207 -2016-05-24 22:36:13 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 65.2 KB, free 729.8 MB) -2016-05-24 22:36:13 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=850833, maxMem=766075207 -2016-05-24 22:36:13 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.8 MB) -2016-05-24 22:36:13 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:13 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[17] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:13 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 22:36:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:13 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 22:36:13 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 22:36:13 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:13 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:13 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 52374 bytes result sent to driver -2016-05-24 22:36:13 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 52374 bytes result sent to driver -2016-05-24 22:36:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 19 ms on localhost (1/2) -2016-05-24 22:36:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 24 ms on localhost (2/2) -2016-05-24 22:36:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 22:36:13 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at LBFGS.scala:218) finished in 0.025 s -2016-05-24 22:36:13 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at LBFGS.scala:218, took 0.063487 s -2016-05-24 22:36:13 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:13 INFO LBFGS:11 - Val and Grad Norm: 0.454344 (rel: 0.624) 2.33981 -2016-05-24 22:36:13 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=862056, maxMem=766075207 -2016-05-24 22:36:13 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 48.7 KB, free 729.7 MB) -2016-05-24 22:36:13 INFO MemoryStore:59 - ensureFreeSpace(11989) called with curMem=911936, maxMem=766075207 -2016-05-24 22:36:13 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.7 MB) -2016-05-24 22:36:13 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:13 INFO SparkContext:59 - Created broadcast 14 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:13 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:13 INFO DAGScheduler:59 - Got job 9 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:13 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:13 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[18] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:13 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=923925, maxMem=766075207 -2016-05-24 22:36:13 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 65.2 KB, free 729.6 MB) -2016-05-24 22:36:13 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=990653, maxMem=766075207 -2016-05-24 22:36:13 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.6 MB) -2016-05-24 22:36:13 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:13 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[18] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:13 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 22:36:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:13 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 22:36:13 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 22:36:13 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:13 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 52374 bytes result sent to driver -2016-05-24 22:36:13 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 15 ms on localhost (1/2) -2016-05-24 22:36:14 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 18 ms on localhost (2/2) -2016-05-24 22:36:14 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at LBFGS.scala:218) finished in 0.019 s -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 22:36:14 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at LBFGS.scala:218, took 0.031282 s -2016-05-24 22:36:14 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:14 INFO LBFGS:11 - Val and Grad Norm: 0.208220 (rel: 0.542) 1.03863 -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1001876, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 48.7 KB, free 729.6 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11983) called with curMem=1051756, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.6 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 16 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:14 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Got job 10 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:14 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[19] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1063739, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 65.2 KB, free 729.5 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=1130467, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.5 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[19] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 22:36:14 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:14 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 16 ms on localhost (1/2) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 18 ms on localhost (2/2) -2016-05-24 22:36:14 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at LBFGS.scala:218) finished in 0.018 s -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 22:36:14 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at LBFGS.scala:218, took 0.031523 s -2016-05-24 22:36:14 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:14 INFO LBFGS:11 - Val and Grad Norm: 0.100482 (rel: 0.517) 0.486243 -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1141690, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 48.7 KB, free 729.4 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11986) called with curMem=1191570, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.4 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 18 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:14 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Got job 11 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:14 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[20] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1203556, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 65.2 KB, free 729.4 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=1270284, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.4 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[20] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 22:36:14 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:14 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 17 ms on localhost (1/2) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 19 ms on localhost (2/2) -2016-05-24 22:36:14 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at LBFGS.scala:218) finished in 0.019 s -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 22:36:14 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at LBFGS.scala:218, took 0.038774 s -2016-05-24 22:36:14 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:14 INFO LBFGS:11 - Val and Grad Norm: 0.0511397 (rel: 0.491) 0.228894 -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1281507, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 48.7 KB, free 729.3 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11987) called with curMem=1331387, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.3 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 20 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:14 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Got job 12 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:14 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[21] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1343374, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 65.2 KB, free 729.2 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=1410102, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.2 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[21] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 22:36:14 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:14 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 17 ms on localhost (1/2) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 19 ms on localhost (2/2) -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 22:36:14 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at LBFGS.scala:218) finished in 0.020 s -2016-05-24 22:36:14 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at LBFGS.scala:218, took 0.037265 s -2016-05-24 22:36:14 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:14 INFO LBFGS:11 - Val and Grad Norm: 0.0265910 (rel: 0.480) 0.113874 -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1421325, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 48.7 KB, free 729.2 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11996) called with curMem=1471205, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.2 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.0 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 22 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:14 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Got job 13 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:14 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[22] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1483201, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 65.2 KB, free 729.1 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11224) called with curMem=1549929, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.1 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.0 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[22] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 22:36:14 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:14 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 15 ms on localhost (1/2) -2016-05-24 22:36:14 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 19 ms on localhost (2/2) -2016-05-24 22:36:14 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at LBFGS.scala:218) finished in 0.021 s -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 22:36:14 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at LBFGS.scala:218, took 0.038906 s -2016-05-24 22:36:14 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:14 INFO LBFGS:11 - Val and Grad Norm: 0.0134428 (rel: 0.494) 0.0568713 -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1561153, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 48.7 KB, free 729.0 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11987) called with curMem=1611033, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.0 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.0 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 24 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:14 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Got job 14 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:14 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[23] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:57514 in memory (size: 11.0 KB, free: 730.0 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1611796, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 65.2 KB, free 729.0 MB) -2016-05-24 22:36:14 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:57514 in memory (size: 11.7 KB, free: 730.0 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=1599800, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.1 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.0 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[23] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:57514 in memory (size: 11.0 KB, free: 730.0 MB) -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 22:36:14 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 22:36:14 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:57514 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:57514 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:57514 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:57514 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 17 ms on localhost (1/2) -2016-05-24 22:36:14 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 19 ms on localhost (2/2) -2016-05-24 22:36:14 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at LBFGS.scala:218) finished in 0.020 s -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 22:36:14 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at LBFGS.scala:218, took 0.050503 s -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:57514 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:14 INFO LBFGS:11 - Val and Grad Norm: 0.00669083 (rel: 0.502) 0.0282528 -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:57514 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:57514 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=990651, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 48.7 KB, free 729.6 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:57514 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(12017) called with curMem=973803, maxMem=766075207 -2016-05-24 22:36:14 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.6 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 26 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:57514 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:57514 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:57514 in memory (size: 11.1 KB, free: 730.2 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Got job 15 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:14 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[24] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=784752, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 65.2 KB, free 729.8 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=851480, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.8 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[24] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 22:36:14 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:14 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 14 ms on localhost (1/2) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 13 ms on localhost (2/2) -2016-05-24 22:36:14 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at LBFGS.scala:218) finished in 0.015 s -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 22:36:14 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at LBFGS.scala:218, took 0.026650 s -2016-05-24 22:36:14 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:14 INFO LBFGS:11 - Val and Grad Norm: 0.00333297 (rel: 0.502) 0.0140530 -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=862703, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 48.7 KB, free 729.7 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11983) called with curMem=912583, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.7 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 28 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:14 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Got job 16 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:14 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[25] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=924566, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 65.2 KB, free 729.6 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=991294, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.6 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[25] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 22:36:14 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:14 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 17 ms on localhost (1/2) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 17 ms on localhost (2/2) -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 22:36:14 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at LBFGS.scala:218) finished in 0.019 s -2016-05-24 22:36:14 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at LBFGS.scala:218, took 0.031994 s -2016-05-24 22:36:14 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:14 INFO LBFGS:11 - Val and Grad Norm: 0.00166698 (rel: 0.500) 0.00701518 -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1002517, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 48.7 KB, free 729.6 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11989) called with curMem=1052397, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.6 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 30 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:14 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Got job 17 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:14 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[26] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1064386, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 65.2 KB, free 729.5 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=1131114, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.5 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[26] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 22:36:14 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:14 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 16 ms on localhost (1/2) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 15 ms on localhost (2/2) -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 22:36:14 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at LBFGS.scala:218) finished in 0.016 s -2016-05-24 22:36:14 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at LBFGS.scala:218, took 0.029507 s -2016-05-24 22:36:14 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:14 INFO LBFGS:11 - Val and Grad Norm: 0.000835695 (rel: 0.499) 0.00350992 -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1142337, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 48.7 KB, free 729.4 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11987) called with curMem=1192217, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.4 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 32 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:14 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Got job 18 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:14 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[27] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1204204, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 65.2 KB, free 729.4 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=1270932, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.4 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[27] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 22:36:14 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:14 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 12 ms on localhost (1/2) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 12 ms on localhost (2/2) -2016-05-24 22:36:14 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at LBFGS.scala:218) finished in 0.013 s -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 22:36:14 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at LBFGS.scala:218, took 0.024764 s -2016-05-24 22:36:14 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:14 INFO LBFGS:11 - Val and Grad Norm: 0.000419197 (rel: 0.498) 0.00175794 -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1282155, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 48.7 KB, free 729.3 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11987) called with curMem=1332035, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.3 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 34 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:14 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Got job 19 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:14 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[28] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1344022, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 65.2 KB, free 729.2 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=1410750, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.2 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[28] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:14 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 31 ms on localhost (1/2) -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:14 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 50 ms on localhost (2/2) -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 22:36:14 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at LBFGS.scala:218) finished in 0.055 s -2016-05-24 22:36:14 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at LBFGS.scala:218, took 0.069041 s -2016-05-24 22:36:14 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:14 INFO LBFGS:11 - Val and Grad Norm: 0.000210335 (rel: 0.498) 0.000881261 -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1421973, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 48.7 KB, free 729.2 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11995) called with curMem=1471853, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.2 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.0 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:57514 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 36 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:14 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:57514 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:57514 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 22:36:14 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Got job 20 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:57514 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[29] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:57514 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1126261, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 65.2 KB, free 729.4 MB) -2016-05-24 22:36:14 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=1181000, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.4 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:57514 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[29] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:57514 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 22:36:14 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:57514 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:14 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 28 ms on localhost (1/2) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:57514 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 30 ms on localhost (2/2) -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 22:36:14 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at LBFGS.scala:218) finished in 0.032 s -2016-05-24 22:36:14 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at LBFGS.scala:218, took 0.052927 s -2016-05-24 22:36:14 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 22:36:14 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:14 INFO LBFGS:11 - Val and Grad Norm: 0.000105370 (rel: 0.499) 0.000444642 -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:57514 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=862681, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 48.7 KB, free 729.7 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:57514 in memory (size: 11.0 KB, free: 730.2 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11990) called with curMem=901338, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.7 MB) -2016-05-24 22:36:14 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 38 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:57514 in memory (size: 11.7 KB, free: 730.2 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Got job 21 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:14 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[30] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=784733, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 65.2 KB, free 729.8 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=851461, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.8 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[30] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 22:36:14 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:14 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 17 ms on localhost (1/2) -2016-05-24 22:36:14 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at LBFGS.scala:218) finished in 0.019 s -2016-05-24 22:36:14 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at LBFGS.scala:218, took 0.034845 s -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 17 ms on localhost (2/2) -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 22:36:14 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:14 INFO LBFGS:11 - Val and Grad Norm: 5.19643e-05 (rel: 0.507) 0.000229782 -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=862684, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 48.7 KB, free 729.7 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11987) called with curMem=912564, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.7 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 40 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:14 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Got job 22 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:14 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[31] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=924551, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 65.2 KB, free 729.6 MB) -2016-05-24 22:36:14 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=991279, maxMem=766075207 -2016-05-24 22:36:14 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.6 MB) -2016-05-24 22:36:14 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:14 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[31] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:14 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 22:36:14 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:14 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:14 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 52374 bytes result sent to driver -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 15 ms on localhost (1/2) -2016-05-24 22:36:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 15 ms on localhost (2/2) -2016-05-24 22:36:14 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at LBFGS.scala:218) finished in 0.016 s -2016-05-24 22:36:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 22:36:14 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at LBFGS.scala:218, took 0.031608 s -2016-05-24 22:36:14 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:14 INFO LBFGS:11 - Val and Grad Norm: 2.33196e-05 (rel: 0.551) 0.000104590 -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1002502, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 48.7 KB, free 729.6 MB) -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(11988) called with curMem=1052382, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.6 MB) -2016-05-24 22:36:15 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:15 INFO SparkContext:59 - Created broadcast 42 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:15 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:15 INFO DAGScheduler:59 - Got job 23 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:15 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:15 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[32] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1064370, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 65.2 KB, free 729.5 MB) -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=1131098, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.5 MB) -2016-05-24 22:36:15 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:15 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[32] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:15 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 22:36:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:15 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 22:36:15 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 22:36:15 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:15 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:15 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 52374 bytes result sent to driver -2016-05-24 22:36:15 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 52374 bytes result sent to driver -2016-05-24 22:36:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 10 ms on localhost (1/2) -2016-05-24 22:36:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 10 ms on localhost (2/2) -2016-05-24 22:36:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 22:36:15 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at LBFGS.scala:218) finished in 0.011 s -2016-05-24 22:36:15 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at LBFGS.scala:218, took 0.022617 s -2016-05-24 22:36:15 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:15 INFO LBFGS:11 - Val and Grad Norm: 1.19501e-05 (rel: 0.488) 5.00526e-05 -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1142321, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 48.7 KB, free 729.4 MB) -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(11987) called with curMem=1192201, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.4 MB) -2016-05-24 22:36:15 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:15 INFO SparkContext:59 - Created broadcast 44 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:15 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:15 INFO DAGScheduler:59 - Got job 24 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:15 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:15 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[33] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1204188, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 65.2 KB, free 729.4 MB) -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=1270916, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.4 MB) -2016-05-24 22:36:15 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:15 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[33] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:15 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 22:36:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:15 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 22:36:15 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 22:36:15 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:15 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:15 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 52374 bytes result sent to driver -2016-05-24 22:36:15 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 52374 bytes result sent to driver -2016-05-24 22:36:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 12 ms on localhost (1/2) -2016-05-24 22:36:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 14 ms on localhost (2/2) -2016-05-24 22:36:15 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at LBFGS.scala:218) finished in 0.014 s -2016-05-24 22:36:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 22:36:15 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at LBFGS.scala:218, took 0.027278 s -2016-05-24 22:36:15 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:15 INFO LBFGS:11 - Val and Grad Norm: 5.96391e-06 (rel: 0.501) 2.49293e-05 -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1282139, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 48.7 KB, free 729.3 MB) -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(11987) called with curMem=1332019, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.3 MB) -2016-05-24 22:36:15 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:36:15 INFO SparkContext:59 - Created broadcast 46 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:15 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:15 INFO DAGScheduler:59 - Got job 25 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:15 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:15 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[34] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1344006, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 65.2 KB, free 729.2 MB) -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=1410734, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.2 MB) -2016-05-24 22:36:15 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:36:15 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[34] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:15 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 22:36:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:15 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 22:36:15 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 22:36:15 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:15 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:15 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 52374 bytes result sent to driver -2016-05-24 22:36:15 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 52374 bytes result sent to driver -2016-05-24 22:36:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 11 ms on localhost (1/2) -2016-05-24 22:36:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 12 ms on localhost (2/2) -2016-05-24 22:36:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 22:36:15 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at LBFGS.scala:218) finished in 0.012 s -2016-05-24 22:36:15 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at LBFGS.scala:218, took 0.029145 s -2016-05-24 22:36:15 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:15 INFO LBFGS:11 - Val and Grad Norm: 2.99165e-06 (rel: 0.498) 1.24838e-05 -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1421957, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 48.7 KB, free 729.2 MB) -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(11987) called with curMem=1471837, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.2 MB) -2016-05-24 22:36:15 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.0 MB) -2016-05-24 22:36:15 INFO SparkContext:59 - Created broadcast 48 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:15 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:15 INFO DAGScheduler:59 - Got job 26 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:15 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:15 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[35] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1483824, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 65.2 KB, free 729.1 MB) -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=1550552, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.1 MB) -2016-05-24 22:36:15 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.0 MB) -2016-05-24 22:36:15 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[35] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:15 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 22:36:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:15 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 22:36:15 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 22:36:15 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:15 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:15 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 52374 bytes result sent to driver -2016-05-24 22:36:15 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 52374 bytes result sent to driver -2016-05-24 22:36:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 11 ms on localhost (1/2) -2016-05-24 22:36:15 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at LBFGS.scala:218) finished in 0.013 s -2016-05-24 22:36:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 12 ms on localhost (2/2) -2016-05-24 22:36:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 22:36:15 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at LBFGS.scala:218, took 0.029588 s -2016-05-24 22:36:15 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:15 INFO LBFGS:11 - Val and Grad Norm: 1.49719e-06 (rel: 0.500) 6.24077e-06 -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1561775, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 48.7 KB, free 729.0 MB) -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(11990) called with curMem=1611655, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.0 MB) -2016-05-24 22:36:15 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.0 MB) -2016-05-24 22:36:15 INFO SparkContext:59 - Created broadcast 50 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:15 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:15 INFO DAGScheduler:59 - Got job 27 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:15 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:15 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[36] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1623645, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 65.2 KB, free 729.0 MB) -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=1690373, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.0 MB) -2016-05-24 22:36:15 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.0 MB) -2016-05-24 22:36:15 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[36] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:15 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 22:36:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:15 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 22:36:15 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 22:36:15 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:15 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:15 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 52374 bytes result sent to driver -2016-05-24 22:36:15 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 52374 bytes result sent to driver -2016-05-24 22:36:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 11 ms on localhost (1/2) -2016-05-24 22:36:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 11 ms on localhost (2/2) -2016-05-24 22:36:15 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at LBFGS.scala:218) finished in 0.013 s -2016-05-24 22:36:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 22:36:15 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at LBFGS.scala:218, took 0.025769 s -2016-05-24 22:36:15 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:15 INFO LBFGS:11 - Val and Grad Norm: 7.49739e-07 (rel: 0.325) 3.12222e-06 -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1701595, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 48.7 KB, free 728.9 MB) -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(11989) called with curMem=1751475, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 11.7 KB, free 728.9 MB) -2016-05-24 22:36:15 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:57514 (size: 11.7 KB, free: 730.0 MB) -2016-05-24 22:36:15 INFO SparkContext:59 - Created broadcast 52 from broadcast at LBFGS.scala:215 -2016-05-24 22:36:15 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:36:15 INFO DAGScheduler:59 - Got job 28 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:36:15 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:36:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:36:15 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[37] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1763464, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 65.2 KB, free 728.8 MB) -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=1830192, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 11.0 KB, free 728.8 MB) -2016-05-24 22:36:15 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:57514 (size: 11.0 KB, free: 730.0 MB) -2016-05-24 22:36:15 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[37] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:36:15 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 22:36:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:36:15 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 22:36:15 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 22:36:15 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:36:15 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:36:15 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 52374 bytes result sent to driver -2016-05-24 22:36:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 7 ms on localhost (1/2) -2016-05-24 22:36:15 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 52374 bytes result sent to driver -2016-05-24 22:36:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 9 ms on localhost (2/2) -2016-05-24 22:36:15 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at LBFGS.scala:218) finished in 0.009 s -2016-05-24 22:36:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 22:36:15 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at LBFGS.scala:218, took 0.019814 s -2016-05-24 22:36:15 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:36:15 INFO LBFGS:11 - Val and Grad Norm: 3.75261e-07 (rel: 0.163) 1.56142e-06 -2016-05-24 22:36:15 INFO LBFGS:59 - LBFGS.runLBFGS finished. Last 10 losses 2.1033505359918756E-4, 1.0537047761752091E-4, 5.196431900265709E-5, 2.331955661624809E-5, 1.1950134074796737E-5, 5.963910399387688E-6, 2.9916512107964574E-6, 1.4971881987870313E-6, 7.497391833372197E-7, 3.7526137909957747E-7 -2016-05-24 22:36:15 INFO SparkContext:59 - Starting job: collectAsMap at MulticlassMetrics.scala:51 -2016-05-24 22:36:15 INFO DAGScheduler:59 - Registering RDD 39 (map at MulticlassMetrics.scala:48) -2016-05-24 22:36:15 INFO DAGScheduler:59 - Got job 29 (collectAsMap at MulticlassMetrics.scala:51) with 2 output partitions -2016-05-24 22:36:15 INFO DAGScheduler:59 - Final stage: ResultStage 30(collectAsMap at MulticlassMetrics.scala:51) -2016-05-24 22:36:15 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-24 22:36:15 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 29) -2016-05-24 22:36:15 INFO DAGScheduler:59 - Submitting ShuffleMapStage 29 (MapPartitionsRDD[39] at map at MulticlassMetrics.scala:48), which has no missing parents -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(55696) called with curMem=1841415, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 54.4 KB, free 728.8 MB) -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(15356) called with curMem=1897111, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 15.0 KB, free 728.8 MB) -2016-05-24 22:36:15 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:57514 (size: 15.0 KB, free: 730.0 MB) -2016-05-24 22:36:15 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 29 (MapPartitionsRDD[39] at map at MulticlassMetrics.scala:48) -2016-05-24 22:36:15 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 22:36:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:36:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:36:15 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 22:36:15 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 22:36:15 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:36:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:36:15 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 2254 bytes result sent to driver -2016-05-24 22:36:15 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 2254 bytes result sent to driver -2016-05-24 22:36:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 46 ms on localhost (1/2) -2016-05-24 22:36:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 49 ms on localhost (2/2) -2016-05-24 22:36:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 22:36:15 INFO DAGScheduler:59 - ShuffleMapStage 29 (map at MulticlassMetrics.scala:48) finished in 0.050 s -2016-05-24 22:36:15 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:36:15 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:36:15 INFO DAGScheduler:59 - waiting: Set(ResultStage 30) -2016-05-24 22:36:15 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:36:15 INFO DAGScheduler:59 - Missing parents for ResultStage 30: List() -2016-05-24 22:36:15 INFO DAGScheduler:59 - Submitting ResultStage 30 (ShuffledRDD[40] at reduceByKey at MulticlassMetrics.scala:50), which is now runnable -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=1912467, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 2.3 KB, free 728.8 MB) -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(1412) called with curMem=1914803, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 1412.0 B, free 728.8 MB) -2016-05-24 22:36:15 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:57514 (size: 1412.0 B, free: 730.0 MB) -2016-05-24 22:36:15 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (ShuffledRDD[40] at reduceByKey at MulticlassMetrics.scala:50) -2016-05-24 22:36:15 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 22:36:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:36:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:36:15 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 22:36:15 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 22:36:15 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:36:15 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-24 22:36:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-24 22:36:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-24 22:36:15 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 1161 bytes result sent to driver -2016-05-24 22:36:15 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 1389 bytes result sent to driver -2016-05-24 22:36:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 26 ms on localhost (1/2) -2016-05-24 22:36:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 28 ms on localhost (2/2) -2016-05-24 22:36:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 22:36:15 INFO DAGScheduler:59 - ResultStage 30 (collectAsMap at MulticlassMetrics.scala:51) finished in 0.029 s -2016-05-24 22:36:15 INFO DAGScheduler:59 - Job 29 finished: collectAsMap at MulticlassMetrics.scala:51, took 0.109288 s -2016-05-24 22:36:15 INFO SparkContext:59 - Starting job: countByValue at MulticlassMetrics.scala:45 -2016-05-24 22:36:15 INFO DAGScheduler:59 - Registering RDD 43 (countByValue at MulticlassMetrics.scala:45) -2016-05-24 22:36:15 INFO DAGScheduler:59 - Got job 30 (countByValue at MulticlassMetrics.scala:45) with 2 output partitions -2016-05-24 22:36:15 INFO DAGScheduler:59 - Final stage: ResultStage 32(countByValue at MulticlassMetrics.scala:45) -2016-05-24 22:36:15 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 31) -2016-05-24 22:36:15 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 31) -2016-05-24 22:36:15 INFO DAGScheduler:59 - Submitting ShuffleMapStage 31 (MapPartitionsRDD[43] at countByValue at MulticlassMetrics.scala:45), which has no missing parents -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(56224) called with curMem=1916215, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 54.9 KB, free 728.7 MB) -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(15543) called with curMem=1972439, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 15.2 KB, free 728.7 MB) -2016-05-24 22:36:15 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:57514 (size: 15.2 KB, free: 730.0 MB) -2016-05-24 22:36:15 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 31 (MapPartitionsRDD[43] at countByValue at MulticlassMetrics.scala:45) -2016-05-24 22:36:15 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 22:36:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:36:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:36:15 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 22:36:15 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 22:36:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:36:15 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:36:15 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 2254 bytes result sent to driver -2016-05-24 22:36:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 25 ms on localhost (1/2) -2016-05-24 22:36:15 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 2254 bytes result sent to driver -2016-05-24 22:36:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 29 ms on localhost (2/2) -2016-05-24 22:36:15 INFO DAGScheduler:59 - ShuffleMapStage 31 (countByValue at MulticlassMetrics.scala:45) finished in 0.031 s -2016-05-24 22:36:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 22:36:15 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:36:15 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:36:15 INFO DAGScheduler:59 - waiting: Set(ResultStage 32) -2016-05-24 22:36:15 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:36:15 INFO DAGScheduler:59 - Missing parents for ResultStage 32: List() -2016-05-24 22:36:15 INFO DAGScheduler:59 - Submitting ResultStage 32 (ShuffledRDD[44] at countByValue at MulticlassMetrics.scala:45), which is now runnable -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=1987982, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 2.3 KB, free 728.7 MB) -2016-05-24 22:36:15 INFO MemoryStore:59 - ensureFreeSpace(1371) called with curMem=1990318, maxMem=766075207 -2016-05-24 22:36:15 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 1371.0 B, free 728.7 MB) -2016-05-24 22:36:15 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:57514 (size: 1371.0 B, free: 730.0 MB) -2016-05-24 22:36:15 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:36:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (ShuffledRDD[44] at countByValue at MulticlassMetrics.scala:45) -2016-05-24 22:36:15 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 22:36:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:36:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:36:15 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 22:36:15 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 22:36:15 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:36:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:36:15 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-24 22:36:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:36:15 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 1161 bytes result sent to driver -2016-05-24 22:36:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 5 ms on localhost (1/2) -2016-05-24 22:36:15 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 1394 bytes result sent to driver -2016-05-24 22:36:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 7 ms on localhost (2/2) -2016-05-24 22:36:15 INFO DAGScheduler:59 - ResultStage 32 (countByValue at MulticlassMetrics.scala:45) finished in 0.007 s -2016-05-24 22:36:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 22:36:15 INFO DAGScheduler:59 - Job 30 finished: countByValue at MulticlassMetrics.scala:45, took 0.058175 s -2016-05-24 22:36:15 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 22:36:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 22:36:16 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 22:36:16 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 22:36:16 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 22:36:16 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 22:36:16 INFO BlockManager:59 - BlockManager stopped -2016-05-24 22:36:16 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 22:36:16 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 22:36:16 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 22:36:16 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 22:36:16 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-feed7b92-32d4-4789-86e7-6ab38906df52 -2016-05-24 22:37:14 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 22:37:15 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 22:37:15 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 22:37:15 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 22:37:15 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 22:37:16 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 22:37:16 INFO Remoting:74 - Starting remoting -2016-05-24 22:37:16 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:57565] -2016-05-24 22:37:16 INFO Utils:59 - Successfully started service 'sparkDriver' on port 57565. -2016-05-24 22:37:16 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 22:37:16 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 22:37:17 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-841feda7-e0fb-4db1-971d-1e7a55c51be7 -2016-05-24 22:37:17 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 22:37:17 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-8c3de5cc-bc83-40c6-93a6-67ed945c30a9\httpd-21fbc620-f9da-4ef6-b88a-eae4e06a89f9 -2016-05-24 22:37:17 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 22:37:17 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:37:17 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:57566 -2016-05-24 22:37:17 INFO Utils:59 - Successfully started service 'HTTP file server' on port 57566. -2016-05-24 22:37:17 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 22:37:17 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:37:17 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 22:37:17 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 22:37:17 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 22:37:17 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 22:37:17 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 22:37:18 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 57585. -2016-05-24 22:37:18 INFO NettyBlockTransferService:59 - Server created on 57585 -2016-05-24 22:37:18 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 22:37:18 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:57585 with 730.6 MB RAM, BlockManagerId(driver, localhost, 57585) -2016-05-24 22:37:18 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 22:37:18 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 22:37:18 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 22:37:18 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 22:37:18 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 22:37:18 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:57585 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 22:37:18 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-24 22:37:20 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 22:37:20 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:37:20 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-24 22:37:20 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-24 22:37:20 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-24 22:37:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:20 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-24 22:37:20 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-05-24 22:37:20 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-24 22:37:20 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=175476, maxMem=766075207 -2016-05-24 22:37:20 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:37:20 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:57585 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:37:20 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-24 22:37:20 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 22:37:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:20 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 22:37:20 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 22:37:20 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-24 22:37:20 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-24 22:37:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:52368+52368 -2016-05-24 22:37:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/regression/sample_libsvm_data.txt:0+52368 -2016-05-24 22:37:20 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 22:37:20 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 22:37:20 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 22:37:20 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 22:37:20 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 22:37:20 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177621, maxMem=766075207 -2016-05-24 22:37:20 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-24 22:37:20 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:57585 (size: 83.1 KB, free: 730.5 MB) -2016-05-24 22:37:20 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262701, maxMem=766075207 -2016-05-24 22:37:20 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-24 22:37:20 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:57585 (size: 83.6 KB, free: 730.4 MB) -2016-05-24 22:37:20 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-24 22:37:20 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-24 22:37:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 448 ms on localhost (1/2) -2016-05-24 22:37:21 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.493 s -2016-05-24 22:37:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 471 ms on localhost (2/2) -2016-05-24 22:37:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 22:37:21 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.684452 s -2016-05-24 22:37:21 INFO SparkContext:59 - Starting job: first at GeneralizedLinearAlgorithm.scala:206 -2016-05-24 22:37:21 INFO DAGScheduler:59 - Got job 1 (first at GeneralizedLinearAlgorithm.scala:206) with 1 output partitions -2016-05-24 22:37:21 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:37:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:21 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206), which has no missing parents -2016-05-24 22:37:21 INFO MemoryStore:59 - ensureFreeSpace(4560) called with curMem=348349, maxMem=766075207 -2016-05-24 22:37:21 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.2 MB) -2016-05-24 22:37:21 INFO MemoryStore:59 - ensureFreeSpace(2483) called with curMem=352909, maxMem=766075207 -2016-05-24 22:37:21 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-05-24 22:37:21 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:57585 (size: 2.4 KB, free: 730.4 MB) -2016-05-24 22:37:21 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at GeneralizedLinearAlgorithm.scala:206) -2016-05-24 22:37:21 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-24 22:37:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:21 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 22:37:21 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-05-24 22:37:21 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:37:21 INFO MemoryStore:59 - ensureFreeSpace(54392) called with curMem=355392, maxMem=766075207 -2016-05-24 22:37:21 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 53.1 KB, free 730.2 MB) -2016-05-24 22:37:21 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:57585 (size: 53.1 KB, free: 730.4 MB) -2016-05-24 22:37:21 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2611 bytes result sent to driver -2016-05-24 22:37:21 INFO DAGScheduler:59 - ResultStage 1 (first at GeneralizedLinearAlgorithm.scala:206) finished in 0.037 s -2016-05-24 22:37:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 36 ms on localhost (1/1) -2016-05-24 22:37:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 22:37:21 INFO DAGScheduler:59 - Job 1 finished: first at GeneralizedLinearAlgorithm.scala:206, took 0.055941 s -2016-05-24 22:37:21 INFO SparkContext:59 - Starting job: count at DataValidators.scala:56 -2016-05-24 22:37:21 INFO DAGScheduler:59 - Got job 2 (count at DataValidators.scala:56) with 2 output partitions -2016-05-24 22:37:21 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DataValidators.scala:56) -2016-05-24 22:37:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:21 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:55), which has no missing parents -2016-05-24 22:37:21 INFO MemoryStore:59 - ensureFreeSpace(4456) called with curMem=409784, maxMem=766075207 -2016-05-24 22:37:21 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.4 KB, free 730.2 MB) -2016-05-24 22:37:21 INFO MemoryStore:59 - ensureFreeSpace(2351) called with curMem=414240, maxMem=766075207 -2016-05-24 22:37:21 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-24 22:37:21 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:57585 (size: 2.3 KB, free: 730.4 MB) -2016-05-24 22:37:21 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at filter at DataValidators.scala:55) -2016-05-24 22:37:21 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 22:37:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:21 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-24 22:37:21 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-24 22:37:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:37:21 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-24 22:37:21 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-05-24 22:37:21 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:37:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 17 ms on localhost (1/2) -2016-05-24 22:37:21 INFO MemoryStore:59 - ensureFreeSpace(62184) called with curMem=416591, maxMem=766075207 -2016-05-24 22:37:21 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 60.7 KB, free 730.1 MB) -2016-05-24 22:37:21 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:57585 (size: 60.7 KB, free: 730.3 MB) -2016-05-24 22:37:21 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2662 bytes result sent to driver -2016-05-24 22:37:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 28 ms on localhost (2/2) -2016-05-24 22:37:21 INFO DAGScheduler:59 - ResultStage 2 (count at DataValidators.scala:56) finished in 0.031 s -2016-05-24 22:37:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 22:37:21 INFO DAGScheduler:59 - Job 2 finished: count at DataValidators.scala:56, took 0.226746 s -2016-05-24 22:37:21 INFO SparkContext:59 - Starting job: treeAggregate at StandardScaler.scala:55 -2016-05-24 22:37:21 INFO DAGScheduler:59 - Got job 3 (treeAggregate at StandardScaler.scala:55) with 2 output partitions -2016-05-24 22:37:21 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at StandardScaler.scala:55) -2016-05-24 22:37:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:21 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[12] at treeAggregate at StandardScaler.scala:55), which has no missing parents -2016-05-24 22:37:21 INFO MemoryStore:59 - ensureFreeSpace(5392) called with curMem=478775, maxMem=766075207 -2016-05-24 22:37:21 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.3 KB, free 730.1 MB) -2016-05-24 22:37:21 INFO MemoryStore:59 - ensureFreeSpace(2767) called with curMem=484167, maxMem=766075207 -2016-05-24 22:37:21 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.1 MB) -2016-05-24 22:37:21 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:57585 (size: 2.7 KB, free: 730.3 MB) -2016-05-24 22:37:21 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[12] at treeAggregate at StandardScaler.scala:55) -2016-05-24 22:37:21 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 22:37:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:21 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-24 22:37:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:37:21 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-24 22:37:21 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 41369 bytes result sent to driver -2016-05-24 22:37:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:37:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 32 ms on localhost (1/2) -2016-05-24 22:37:21 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 41369 bytes result sent to driver -2016-05-24 22:37:21 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at StandardScaler.scala:55) finished in 0.042 s -2016-05-24 22:37:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 39 ms on localhost (2/2) -2016-05-24 22:37:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 22:37:21 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at StandardScaler.scala:55, took 0.065255 s -2016-05-24 22:37:21 INFO SparkContext:59 - Starting job: count at LBFGS.scala:170 -2016-05-24 22:37:21 INFO DAGScheduler:59 - Got job 4 (count at LBFGS.scala:170) with 2 output partitions -2016-05-24 22:37:21 INFO DAGScheduler:59 - Final stage: ResultStage 4(count at LBFGS.scala:170) -2016-05-24 22:37:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:21 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at map at GeneralizedLinearAlgorithm.scala:290), which has no missing parents -2016-05-24 22:37:21 INFO MemoryStore:59 - ensureFreeSpace(15648) called with curMem=486934, maxMem=766075207 -2016-05-24 22:37:21 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 15.3 KB, free 730.1 MB) -2016-05-24 22:37:21 INFO MemoryStore:59 - ensureFreeSpace(8225) called with curMem=502582, maxMem=766075207 -2016-05-24 22:37:21 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 8.0 KB, free 730.1 MB) -2016-05-24 22:37:21 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:57585 (size: 8.0 KB, free: 730.3 MB) -2016-05-24 22:37:21 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at map at GeneralizedLinearAlgorithm.scala:290) -2016-05-24 22:37:21 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 22:37:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:21 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-24 22:37:21 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-24 22:37:21 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-24 22:37:21 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-24 22:37:21 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-24 22:37:21 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-24 22:37:21 INFO MemoryStore:59 - ensureFreeSpace(55112) called with curMem=510807, maxMem=766075207 -2016-05-24 22:37:21 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 53.8 KB, free 730.0 MB) -2016-05-24 22:37:21 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=565919, maxMem=766075207 -2016-05-24 22:37:21 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-24 22:37:21 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:57585 (size: 53.8 KB, free: 730.2 MB) -2016-05-24 22:37:21 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:57585 (size: 61.5 KB, free: 730.2 MB) -2016-05-24 22:37:21 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2662 bytes result sent to driver -2016-05-24 22:37:21 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2662 bytes result sent to driver -2016-05-24 22:37:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 34 ms on localhost (1/2) -2016-05-24 22:37:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 33 ms on localhost (2/2) -2016-05-24 22:37:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 22:37:21 INFO DAGScheduler:59 - ResultStage 4 (count at LBFGS.scala:170) finished in 0.036 s -2016-05-24 22:37:21 INFO DAGScheduler:59 - Job 4 finished: count at LBFGS.scala:170, took 0.054428 s -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=628919, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 48.7 KB, free 729.9 MB) -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(2476) called with curMem=678799, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.4 KB, free 729.9 MB) -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:57585 (size: 2.4 KB, free: 730.2 MB) -2016-05-24 22:37:22 INFO SparkContext:59 - Created broadcast 6 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:22 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:22 INFO DAGScheduler:59 - Got job 5 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:22 INFO DAGScheduler:59 - Final stage: ResultStage 5(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:22 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[14] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=681275, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 65.2 KB, free 729.9 MB) -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(11221) called with curMem=748003, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.9 MB) -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.2 MB) -2016-05-24 22:37:22 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[14] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:22 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 22:37:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:22 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 22:37:22 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-24 22:37:22 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:22 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:22 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 52374 bytes result sent to driver -2016-05-24 22:37:22 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 52374 bytes result sent to driver -2016-05-24 22:37:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 35 ms on localhost (1/2) -2016-05-24 22:37:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 38 ms on localhost (2/2) -2016-05-24 22:37:22 INFO DAGScheduler:59 - ResultStage 5 (treeAggregate at LBFGS.scala:218) finished in 0.039 s -2016-05-24 22:37:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 22:37:22 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 22:37:22 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 22:37:22 INFO DAGScheduler:59 - Job 5 finished: treeAggregate at LBFGS.scala:218, took 0.065966 s -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=759224, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 48.7 KB, free 729.8 MB) -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(11374) called with curMem=809104, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 11.1 KB, free 729.8 MB) -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:57585 (size: 11.1 KB, free: 730.1 MB) -2016-05-24 22:37:22 INFO SparkContext:59 - Created broadcast 8 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:22 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:22 INFO DAGScheduler:59 - Got job 6 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:22 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:22 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[15] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=820478, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 65.2 KB, free 729.7 MB) -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=887206, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.7 MB) -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:22 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[15] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:22 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-24 22:37:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:22 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-24 22:37:22 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-24 22:37:22 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:22 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:22 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 52374 bytes result sent to driver -2016-05-24 22:37:22 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 52374 bytes result sent to driver -2016-05-24 22:37:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 21 ms on localhost (1/2) -2016-05-24 22:37:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 27 ms on localhost (2/2) -2016-05-24 22:37:22 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at LBFGS.scala:218) finished in 0.028 s -2016-05-24 22:37:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 22:37:22 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at LBFGS.scala:218, took 0.052890 s -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=898428, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 48.7 KB, free 729.7 MB) -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(11373) called with curMem=948308, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 11.1 KB, free 729.7 MB) -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:57585 (size: 11.1 KB, free: 730.1 MB) -2016-05-24 22:37:22 INFO SparkContext:59 - Created broadcast 10 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:22 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:22 INFO DAGScheduler:59 - Got job 7 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:22 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:22 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[16] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=959681, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 65.2 KB, free 729.6 MB) -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(11219) called with curMem=948459, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.7 MB) -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:22 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[16] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:22 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-24 22:37:22 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 22:37:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:22 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 22:37:22 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:57585 in memory (size: 11.1 KB, free: 730.1 MB) -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:22 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:22 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:22 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:57585 in memory (size: 2.4 KB, free: 730.1 MB) -2016-05-24 22:37:22 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 52374 bytes result sent to driver -2016-05-24 22:37:22 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 52374 bytes result sent to driver -2016-05-24 22:37:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 22 ms on localhost (1/2) -2016-05-24 22:37:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 21 ms on localhost (2/2) -2016-05-24 22:37:22 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at LBFGS.scala:218) finished in 0.023 s -2016-05-24 22:37:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:57585 in memory (size: 8.0 KB, free: 730.2 MB) -2016-05-24 22:37:22 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at LBFGS.scala:218, took 0.042241 s -2016-05-24 22:37:22 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 22:37:22 INFO StrongWolfeLineSearch:11 - Line search t: 0.05601643361637265 fval: 1.2089737414014867 rhs: 2.3023346338192554 cdd: 5.371949174352343 -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:57585 in memory (size: 2.7 KB, free: 730.2 MB) -2016-05-24 22:37:22 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 22:37:22 INFO LBFGS:11 - Step Size: 0.05602 -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:57585 in memory (size: 2.3 KB, free: 730.2 MB) -2016-05-24 22:37:22 INFO LBFGS:11 - Val and Grad Norm: 1.20897 (rel: 0.475) 7.36229 -2016-05-24 22:37:22 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:57585 in memory (size: 2.4 KB, free: 730.2 MB) -2016-05-24 22:37:22 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:57585 in memory (size: 2.1 KB, free: 730.2 MB) -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=716196, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 48.7 KB, free 729.9 MB) -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(11987) called with curMem=766076, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.8 MB) -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.2 MB) -2016-05-24 22:37:22 INFO SparkContext:59 - Created broadcast 12 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:22 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:22 INFO DAGScheduler:59 - Got job 8 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:22 INFO DAGScheduler:59 - Final stage: ResultStage 8(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:22 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[17] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=778063, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 65.2 KB, free 729.8 MB) -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=844791, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.8 MB) -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:22 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[17] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:22 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-24 22:37:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:22 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 22:37:22 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-24 22:37:22 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:22 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:22 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 52374 bytes result sent to driver -2016-05-24 22:37:22 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 52374 bytes result sent to driver -2016-05-24 22:37:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 20 ms on localhost (1/2) -2016-05-24 22:37:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 24 ms on localhost (2/2) -2016-05-24 22:37:22 INFO DAGScheduler:59 - ResultStage 8 (treeAggregate at LBFGS.scala:218) finished in 0.024 s -2016-05-24 22:37:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 22:37:22 INFO DAGScheduler:59 - Job 8 finished: treeAggregate at LBFGS.scala:218, took 0.043379 s -2016-05-24 22:37:22 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:22 INFO LBFGS:11 - Val and Grad Norm: 0.454344 (rel: 0.624) 2.33981 -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=856013, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 48.7 KB, free 729.7 MB) -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(11989) called with curMem=905893, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.7 MB) -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:22 INFO SparkContext:59 - Created broadcast 14 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:22 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:22 INFO DAGScheduler:59 - Got job 9 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:22 INFO DAGScheduler:59 - Final stage: ResultStage 9(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:22 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[18] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=917882, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 65.2 KB, free 729.6 MB) -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=984610, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.6 MB) -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:22 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[18] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:22 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-24 22:37:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:22 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-24 22:37:22 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-24 22:37:22 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:22 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:22 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 52374 bytes result sent to driver -2016-05-24 22:37:22 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 52374 bytes result sent to driver -2016-05-24 22:37:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 14 ms on localhost (1/2) -2016-05-24 22:37:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 14 ms on localhost (2/2) -2016-05-24 22:37:22 INFO DAGScheduler:59 - ResultStage 9 (treeAggregate at LBFGS.scala:218) finished in 0.015 s -2016-05-24 22:37:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 22:37:22 INFO DAGScheduler:59 - Job 9 finished: treeAggregate at LBFGS.scala:218, took 0.026256 s -2016-05-24 22:37:22 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:22 INFO LBFGS:11 - Val and Grad Norm: 0.208220 (rel: 0.542) 1.03863 -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=995832, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 48.7 KB, free 729.6 MB) -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(11983) called with curMem=1045712, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.6 MB) -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:22 INFO SparkContext:59 - Created broadcast 16 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:22 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:22 INFO DAGScheduler:59 - Got job 10 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:22 INFO DAGScheduler:59 - Final stage: ResultStage 10(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:22 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[19] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1057695, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 65.2 KB, free 729.5 MB) -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=1124423, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.5 MB) -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:22 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[19] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:22 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-24 22:37:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:22 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-24 22:37:22 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 22:37:22 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:22 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:22 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 52374 bytes result sent to driver -2016-05-24 22:37:22 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 52374 bytes result sent to driver -2016-05-24 22:37:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 13 ms on localhost (1/2) -2016-05-24 22:37:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 14 ms on localhost (2/2) -2016-05-24 22:37:22 INFO DAGScheduler:59 - ResultStage 10 (treeAggregate at LBFGS.scala:218) finished in 0.015 s -2016-05-24 22:37:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 22:37:22 INFO DAGScheduler:59 - Job 10 finished: treeAggregate at LBFGS.scala:218, took 0.027806 s -2016-05-24 22:37:22 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:22 INFO LBFGS:11 - Val and Grad Norm: 0.100482 (rel: 0.517) 0.486243 -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1135645, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 48.7 KB, free 729.5 MB) -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(11983) called with curMem=1185525, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.4 MB) -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:22 INFO SparkContext:59 - Created broadcast 18 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:22 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:22 INFO DAGScheduler:59 - Got job 11 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:22 INFO DAGScheduler:59 - Final stage: ResultStage 11(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:22 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[20] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1197508, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 65.2 KB, free 729.4 MB) -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=1264236, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.4 MB) -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:22 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[20] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:22 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-24 22:37:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:22 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-24 22:37:22 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-24 22:37:22 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:22 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:22 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 52374 bytes result sent to driver -2016-05-24 22:37:22 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 52374 bytes result sent to driver -2016-05-24 22:37:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 13 ms on localhost (1/2) -2016-05-24 22:37:22 INFO DAGScheduler:59 - ResultStage 11 (treeAggregate at LBFGS.scala:218) finished in 0.014 s -2016-05-24 22:37:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 13 ms on localhost (2/2) -2016-05-24 22:37:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 22:37:22 INFO DAGScheduler:59 - Job 11 finished: treeAggregate at LBFGS.scala:218, took 0.031808 s -2016-05-24 22:37:22 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:22 INFO LBFGS:11 - Val and Grad Norm: 0.0511397 (rel: 0.491) 0.228894 -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1275458, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 48.7 KB, free 729.3 MB) -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(11987) called with curMem=1325338, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.3 MB) -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:22 INFO SparkContext:59 - Created broadcast 20 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:22 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:22 INFO DAGScheduler:59 - Got job 12 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:22 INFO DAGScheduler:59 - Final stage: ResultStage 12(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:22 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[21] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1337325, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 65.2 KB, free 729.2 MB) -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=1404053, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.2 MB) -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:22 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[21] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:22 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-24 22:37:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:22 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-24 22:37:22 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-24 22:37:22 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:22 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:22 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 52374 bytes result sent to driver -2016-05-24 22:37:22 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 52374 bytes result sent to driver -2016-05-24 22:37:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 15 ms on localhost (1/2) -2016-05-24 22:37:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 18 ms on localhost (2/2) -2016-05-24 22:37:22 INFO DAGScheduler:59 - ResultStage 12 (treeAggregate at LBFGS.scala:218) finished in 0.019 s -2016-05-24 22:37:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 22:37:22 INFO DAGScheduler:59 - Job 12 finished: treeAggregate at LBFGS.scala:218, took 0.035418 s -2016-05-24 22:37:22 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:22 INFO LBFGS:11 - Val and Grad Norm: 0.0265910 (rel: 0.480) 0.113874 -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1415275, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 48.7 KB, free 729.2 MB) -2016-05-24 22:37:22 INFO MemoryStore:59 - ensureFreeSpace(11990) called with curMem=1465155, maxMem=766075207 -2016-05-24 22:37:22 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.2 MB) -2016-05-24 22:37:22 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.0 MB) -2016-05-24 22:37:22 INFO SparkContext:59 - Created broadcast 22 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:23 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Got job 13 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:23 INFO DAGScheduler:59 - Final stage: ResultStage 13(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[22] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1477145, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 65.2 KB, free 729.1 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11223) called with curMem=1543873, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.1 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.0 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[22] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-24 22:37:23 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:23 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 19 ms on localhost (1/2) -2016-05-24 22:37:23 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 25 ms on localhost (2/2) -2016-05-24 22:37:23 INFO DAGScheduler:59 - ResultStage 13 (treeAggregate at LBFGS.scala:218) finished in 0.028 s -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-24 22:37:23 INFO DAGScheduler:59 - Job 13 finished: treeAggregate at LBFGS.scala:218, took 0.047158 s -2016-05-24 22:37:23 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:23 INFO LBFGS:11 - Val and Grad Norm: 0.0134428 (rel: 0.494) 0.0568713 -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1555096, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 48.7 KB, free 729.1 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11988) called with curMem=1604976, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.0 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.0 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 24 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:23 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Got job 14 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:23 INFO DAGScheduler:59 - Final stage: ResultStage 14(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[23] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1616964, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 65.2 KB, free 729.0 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.0 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=1672470, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.0 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.0 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.0 MB) -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[23] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-24 22:37:23 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-24 22:37:23 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 730.0 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 18 ms on localhost (1/2) -2016-05-24 22:37:23 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-24 22:37:23 INFO DAGScheduler:59 - ResultStage 14 (treeAggregate at LBFGS.scala:218) finished in 0.019 s -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 17 ms on localhost (2/2) -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-24 22:37:23 INFO DAGScheduler:59 - Job 14 finished: treeAggregate at LBFGS.scala:218, took 0.046892 s -2016-05-24 22:37:23 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO LBFGS:11 - Val and Grad Norm: 0.00669083 (rel: 0.502) 0.0282528 -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1057700, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 48.7 KB, free 729.5 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(12002) called with curMem=1095591, maxMem=766075207 -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.5 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 26 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 22:37:23 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:57585 in memory (size: 11.1 KB, free: 730.2 MB) -2016-05-24 22:37:23 INFO DAGScheduler:59 - Got job 15 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:23 INFO DAGScheduler:59 - Final stage: ResultStage 15(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[24] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=778696, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 65.2 KB, free 729.8 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=845424, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.8 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[24] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-24 22:37:23 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:23 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 16 ms on localhost (1/2) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 17 ms on localhost (2/2) -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-24 22:37:23 INFO DAGScheduler:59 - ResultStage 15 (treeAggregate at LBFGS.scala:218) finished in 0.019 s -2016-05-24 22:37:23 INFO DAGScheduler:59 - Job 15 finished: treeAggregate at LBFGS.scala:218, took 0.033773 s -2016-05-24 22:37:23 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:23 INFO LBFGS:11 - Val and Grad Norm: 0.00333297 (rel: 0.502) 0.0140530 -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=856646, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 48.7 KB, free 729.7 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11986) called with curMem=906526, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.7 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 28 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:23 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Got job 16 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:23 INFO DAGScheduler:59 - Final stage: ResultStage 16(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[25] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=918512, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 65.2 KB, free 729.6 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=985240, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.6 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[25] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-24 22:37:23 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:23 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 18 ms on localhost (1/2) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 17 ms on localhost (2/2) -2016-05-24 22:37:23 INFO DAGScheduler:59 - ResultStage 16 (treeAggregate at LBFGS.scala:218) finished in 0.019 s -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-24 22:37:23 INFO DAGScheduler:59 - Job 16 finished: treeAggregate at LBFGS.scala:218, took 0.032697 s -2016-05-24 22:37:23 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:23 INFO LBFGS:11 - Val and Grad Norm: 0.00166698 (rel: 0.500) 0.00701518 -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=996462, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 48.7 KB, free 729.6 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11987) called with curMem=1046342, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.6 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 30 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:23 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Got job 17 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:23 INFO DAGScheduler:59 - Final stage: ResultStage 17(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[26] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1058329, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 65.2 KB, free 729.5 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=1125057, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.5 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[26] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-24 22:37:23 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:23 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 30 ms on localhost (1/2) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 32 ms on localhost (2/2) -2016-05-24 22:37:23 INFO DAGScheduler:59 - ResultStage 17 (treeAggregate at LBFGS.scala:218) finished in 0.032 s -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-24 22:37:23 INFO DAGScheduler:59 - Job 17 finished: treeAggregate at LBFGS.scala:218, took 0.044791 s -2016-05-24 22:37:23 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:23 INFO LBFGS:11 - Val and Grad Norm: 0.000835695 (rel: 0.499) 0.00350992 -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1136279, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 48.7 KB, free 729.5 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11987) called with curMem=1186159, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.4 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 32 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:23 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Got job 18 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:23 INFO DAGScheduler:59 - Final stage: ResultStage 18(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[27] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1198146, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 65.2 KB, free 729.4 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=1264874, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.4 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[27] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-24 22:37:23 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:23 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 16 ms on localhost (1/2) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 19 ms on localhost (2/2) -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-24 22:37:23 INFO DAGScheduler:59 - ResultStage 18 (treeAggregate at LBFGS.scala:218) finished in 0.019 s -2016-05-24 22:37:23 INFO DAGScheduler:59 - Job 18 finished: treeAggregate at LBFGS.scala:218, took 0.032220 s -2016-05-24 22:37:23 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:23 INFO LBFGS:11 - Val and Grad Norm: 0.000419197 (rel: 0.498) 0.00175794 -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1276096, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 48.7 KB, free 729.3 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11987) called with curMem=1325976, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.3 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 34 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:23 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Got job 19 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:23 INFO DAGScheduler:59 - Final stage: ResultStage 19(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[28] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1337963, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 65.2 KB, free 729.2 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=1404691, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.2 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[28] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-05-24 22:37:23 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:23 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 15 ms on localhost (1/2) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 15 ms on localhost (2/2) -2016-05-24 22:37:23 INFO DAGScheduler:59 - ResultStage 19 (treeAggregate at LBFGS.scala:218) finished in 0.017 s -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-24 22:37:23 INFO DAGScheduler:59 - Job 19 finished: treeAggregate at LBFGS.scala:218, took 0.030098 s -2016-05-24 22:37:23 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:23 INFO LBFGS:11 - Val and Grad Norm: 0.000210335 (rel: 0.498) 0.000881261 -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1337963, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 48.7 KB, free 729.3 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11995) called with curMem=1375856, maxMem=766075207 -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.3 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 36 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Got job 20 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:23 INFO DAGScheduler:59 - Final stage: ResultStage 20(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[29] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1108217, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 65.2 KB, free 729.5 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=1125065, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.5 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[29] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-24 22:37:23 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-05-24 22:37:23 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 19 ms on localhost (1/2) -2016-05-24 22:37:23 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-24 22:37:23 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 28 ms on localhost (2/2) -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-24 22:37:23 INFO DAGScheduler:59 - ResultStage 20 (treeAggregate at LBFGS.scala:218) finished in 0.031 s -2016-05-24 22:37:23 INFO DAGScheduler:59 - Job 20 finished: treeAggregate at LBFGS.scala:218, took 0.049459 s -2016-05-24 22:37:23 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:23 INFO LBFGS:11 - Val and Grad Norm: 0.000105370 (rel: 0.499) 0.000444642 -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.2 MB) -2016-05-24 22:37:23 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 730.2 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=716821, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 48.7 KB, free 729.9 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11989) called with curMem=766701, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.8 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.2 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 38 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:23 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Got job 21 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:23 INFO DAGScheduler:59 - Final stage: ResultStage 21(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[30] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=778690, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 65.2 KB, free 729.8 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=845418, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.8 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[30] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:23 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 23 ms on localhost (1/2) -2016-05-24 22:37:23 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 29 ms on localhost (2/2) -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-24 22:37:23 INFO DAGScheduler:59 - ResultStage 21 (treeAggregate at LBFGS.scala:218) finished in 0.031 s -2016-05-24 22:37:23 INFO DAGScheduler:59 - Job 21 finished: treeAggregate at LBFGS.scala:218, took 0.046449 s -2016-05-24 22:37:23 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:23 INFO LBFGS:11 - Val and Grad Norm: 5.19643e-05 (rel: 0.507) 0.000229782 -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=856640, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 48.7 KB, free 729.7 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11988) called with curMem=906520, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.7 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 40 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:23 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Got job 22 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:23 INFO DAGScheduler:59 - Final stage: ResultStage 22(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[31] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=918508, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 65.2 KB, free 729.6 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=985236, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.6 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[31] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-05-24 22:37:23 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:23 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 15 ms on localhost (1/2) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 16 ms on localhost (2/2) -2016-05-24 22:37:23 INFO DAGScheduler:59 - ResultStage 22 (treeAggregate at LBFGS.scala:218) finished in 0.017 s -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-24 22:37:23 INFO DAGScheduler:59 - Job 22 finished: treeAggregate at LBFGS.scala:218, took 0.033435 s -2016-05-24 22:37:23 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:23 INFO LBFGS:11 - Val and Grad Norm: 2.33196e-05 (rel: 0.551) 0.000104590 -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=996458, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 48.7 KB, free 729.6 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11995) called with curMem=1046338, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.6 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 42 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:23 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Got job 23 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:23 INFO DAGScheduler:59 - Final stage: ResultStage 23(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[32] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1058333, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 65.2 KB, free 729.5 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=1125061, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.5 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[32] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-05-24 22:37:23 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:23 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 13 ms on localhost (1/2) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 17 ms on localhost (2/2) -2016-05-24 22:37:23 INFO DAGScheduler:59 - ResultStage 23 (treeAggregate at LBFGS.scala:218) finished in 0.017 s -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-24 22:37:23 INFO DAGScheduler:59 - Job 23 finished: treeAggregate at LBFGS.scala:218, took 0.032848 s -2016-05-24 22:37:23 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:23 INFO LBFGS:11 - Val and Grad Norm: 1.19501e-05 (rel: 0.488) 5.00526e-05 -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1136283, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 48.7 KB, free 729.5 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11988) called with curMem=1186163, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.4 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 44 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:23 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Got job 24 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:23 INFO DAGScheduler:59 - Final stage: ResultStage 24(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[33] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1198151, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 65.2 KB, free 729.4 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=1264879, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.4 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[33] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-05-24 22:37:23 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:23 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 12 ms on localhost (1/2) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 12 ms on localhost (2/2) -2016-05-24 22:37:23 INFO DAGScheduler:59 - ResultStage 24 (treeAggregate at LBFGS.scala:218) finished in 0.014 s -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-24 22:37:23 INFO DAGScheduler:59 - Job 24 finished: treeAggregate at LBFGS.scala:218, took 0.028793 s -2016-05-24 22:37:23 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:23 INFO LBFGS:11 - Val and Grad Norm: 5.96391e-06 (rel: 0.501) 2.49293e-05 -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1276101, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 48.7 KB, free 729.3 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11988) called with curMem=1325981, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.3 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 46 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:23 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Got job 25 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:23 INFO DAGScheduler:59 - Final stage: ResultStage 25(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[34] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1337969, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 65.2 KB, free 729.2 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=1404697, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.2 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[34] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-05-24 22:37:23 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:23 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 12 ms on localhost (1/2) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 13 ms on localhost (2/2) -2016-05-24 22:37:23 INFO DAGScheduler:59 - ResultStage 25 (treeAggregate at LBFGS.scala:218) finished in 0.014 s -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-24 22:37:23 INFO DAGScheduler:59 - Job 25 finished: treeAggregate at LBFGS.scala:218, took 0.027812 s -2016-05-24 22:37:23 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:23 INFO LBFGS:11 - Val and Grad Norm: 2.99165e-06 (rel: 0.498) 1.24838e-05 -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1415919, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 48.7 KB, free 729.2 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11987) called with curMem=1465799, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.2 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.0 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 48 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:23 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Got job 26 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:23 INFO DAGScheduler:59 - Final stage: ResultStage 26(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[35] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1477786, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 65.2 KB, free 729.1 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=1544514, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.1 MB) -2016-05-24 22:37:23 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.0 MB) -2016-05-24 22:37:23 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[35] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:23 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:23 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 12 ms on localhost (1/2) -2016-05-24 22:37:23 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-05-24 22:37:23 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:23 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 52374 bytes result sent to driver -2016-05-24 22:37:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 23 ms on localhost (2/2) -2016-05-24 22:37:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-24 22:37:23 INFO DAGScheduler:59 - ResultStage 26 (treeAggregate at LBFGS.scala:218) finished in 0.025 s -2016-05-24 22:37:23 INFO DAGScheduler:59 - Job 26 finished: treeAggregate at LBFGS.scala:218, took 0.044642 s -2016-05-24 22:37:23 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:23 INFO LBFGS:11 - Val and Grad Norm: 1.49719e-06 (rel: 0.500) 6.24077e-06 -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1555736, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 48.7 KB, free 729.1 MB) -2016-05-24 22:37:23 INFO MemoryStore:59 - ensureFreeSpace(11989) called with curMem=1605616, maxMem=766075207 -2016-05-24 22:37:23 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 11.7 KB, free 729.0 MB) -2016-05-24 22:37:24 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.0 MB) -2016-05-24 22:37:24 INFO SparkContext:59 - Created broadcast 50 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:24 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:24 INFO DAGScheduler:59 - Got job 27 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:24 INFO DAGScheduler:59 - Final stage: ResultStage 27(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:24 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[36] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:24 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1617605, maxMem=766075207 -2016-05-24 22:37:24 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 65.2 KB, free 729.0 MB) -2016-05-24 22:37:24 INFO MemoryStore:59 - ensureFreeSpace(11221) called with curMem=1684333, maxMem=766075207 -2016-05-24 22:37:24 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 11.0 KB, free 729.0 MB) -2016-05-24 22:37:24 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.0 MB) -2016-05-24 22:37:24 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[36] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:24 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-24 22:37:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:24 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-05-24 22:37:24 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-05-24 22:37:24 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:24 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:24 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 52374 bytes result sent to driver -2016-05-24 22:37:24 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 52374 bytes result sent to driver -2016-05-24 22:37:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 10 ms on localhost (1/2) -2016-05-24 22:37:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 12 ms on localhost (2/2) -2016-05-24 22:37:24 INFO DAGScheduler:59 - ResultStage 27 (treeAggregate at LBFGS.scala:218) finished in 0.013 s -2016-05-24 22:37:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-24 22:37:24 INFO DAGScheduler:59 - Job 27 finished: treeAggregate at LBFGS.scala:218, took 0.028054 s -2016-05-24 22:37:24 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:24 INFO LBFGS:11 - Val and Grad Norm: 7.49739e-07 (rel: 0.325) 3.12222e-06 -2016-05-24 22:37:24 INFO MemoryStore:59 - ensureFreeSpace(49880) called with curMem=1695554, maxMem=766075207 -2016-05-24 22:37:24 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 48.7 KB, free 728.9 MB) -2016-05-24 22:37:24 INFO MemoryStore:59 - ensureFreeSpace(11990) called with curMem=1745434, maxMem=766075207 -2016-05-24 22:37:24 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 11.7 KB, free 728.9 MB) -2016-05-24 22:37:24 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:57585 (size: 11.7 KB, free: 730.0 MB) -2016-05-24 22:37:24 INFO SparkContext:59 - Created broadcast 52 from broadcast at LBFGS.scala:215 -2016-05-24 22:37:24 INFO SparkContext:59 - Starting job: treeAggregate at LBFGS.scala:218 -2016-05-24 22:37:24 INFO DAGScheduler:59 - Got job 28 (treeAggregate at LBFGS.scala:218) with 2 output partitions -2016-05-24 22:37:24 INFO DAGScheduler:59 - Final stage: ResultStage 28(treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:24 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[37] at treeAggregate at LBFGS.scala:218), which has no missing parents -2016-05-24 22:37:24 INFO MemoryStore:59 - ensureFreeSpace(66728) called with curMem=1757424, maxMem=766075207 -2016-05-24 22:37:24 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 65.2 KB, free 728.8 MB) -2016-05-24 22:37:24 INFO MemoryStore:59 - ensureFreeSpace(11222) called with curMem=1824152, maxMem=766075207 -2016-05-24 22:37:24 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 11.0 KB, free 728.8 MB) -2016-05-24 22:37:24 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:57585 (size: 11.0 KB, free: 730.0 MB) -2016-05-24 22:37:24 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[37] at treeAggregate at LBFGS.scala:218) -2016-05-24 22:37:24 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-24 22:37:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-24 22:37:24 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-05-24 22:37:24 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-05-24 22:37:24 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-24 22:37:24 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-24 22:37:24 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 52374 bytes result sent to driver -2016-05-24 22:37:24 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 52374 bytes result sent to driver -2016-05-24 22:37:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 8 ms on localhost (1/2) -2016-05-24 22:37:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 8 ms on localhost (2/2) -2016-05-24 22:37:24 INFO DAGScheduler:59 - ResultStage 28 (treeAggregate at LBFGS.scala:218) finished in 0.009 s -2016-05-24 22:37:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-24 22:37:24 INFO DAGScheduler:59 - Job 28 finished: treeAggregate at LBFGS.scala:218, took 0.019983 s -2016-05-24 22:37:24 INFO LBFGS:11 - Step Size: 1.000 -2016-05-24 22:37:24 INFO LBFGS:11 - Val and Grad Norm: 3.75261e-07 (rel: 0.163) 1.56142e-06 -2016-05-24 22:37:24 INFO LBFGS:59 - LBFGS.runLBFGS finished. Last 10 losses 2.1033505359918713E-4, 1.0537047761752114E-4, 5.196431900265667E-5, 2.3319556616247852E-5, 1.1950134074796801E-5, 5.9639103993876725E-6, 2.9916512107964705E-6, 1.4971881987870306E-6, 7.497391833372216E-7, 3.752613790995776E-7 -2016-05-24 22:37:24 INFO SparkContext:59 - Starting job: collectAsMap at MulticlassMetrics.scala:51 -2016-05-24 22:37:24 INFO DAGScheduler:59 - Registering RDD 39 (map at MulticlassMetrics.scala:48) -2016-05-24 22:37:24 INFO DAGScheduler:59 - Got job 29 (collectAsMap at MulticlassMetrics.scala:51) with 2 output partitions -2016-05-24 22:37:24 INFO DAGScheduler:59 - Final stage: ResultStage 30(collectAsMap at MulticlassMetrics.scala:51) -2016-05-24 22:37:24 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 29) -2016-05-24 22:37:24 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 29) -2016-05-24 22:37:24 INFO DAGScheduler:59 - Submitting ShuffleMapStage 29 (MapPartitionsRDD[39] at map at MulticlassMetrics.scala:48), which has no missing parents -2016-05-24 22:37:24 INFO MemoryStore:59 - ensureFreeSpace(55696) called with curMem=1835374, maxMem=766075207 -2016-05-24 22:37:24 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 54.4 KB, free 728.8 MB) -2016-05-24 22:37:24 INFO MemoryStore:59 - ensureFreeSpace(15342) called with curMem=1891070, maxMem=766075207 -2016-05-24 22:37:24 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 15.0 KB, free 728.8 MB) -2016-05-24 22:37:24 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:57585 (size: 15.0 KB, free: 730.0 MB) -2016-05-24 22:37:24 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 29 (MapPartitionsRDD[39] at map at MulticlassMetrics.scala:48) -2016-05-24 22:37:24 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-24 22:37:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:37:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:37:24 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-05-24 22:37:24 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-05-24 22:37:24 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:37:24 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:37:24 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 2254 bytes result sent to driver -2016-05-24 22:37:24 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 2254 bytes result sent to driver -2016-05-24 22:37:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 55 ms on localhost (1/2) -2016-05-24 22:37:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 57 ms on localhost (2/2) -2016-05-24 22:37:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-24 22:37:24 INFO DAGScheduler:59 - ShuffleMapStage 29 (map at MulticlassMetrics.scala:48) finished in 0.059 s -2016-05-24 22:37:24 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:37:24 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:37:24 INFO DAGScheduler:59 - waiting: Set(ResultStage 30) -2016-05-24 22:37:24 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:37:24 INFO DAGScheduler:59 - Missing parents for ResultStage 30: List() -2016-05-24 22:37:24 INFO DAGScheduler:59 - Submitting ResultStage 30 (ShuffledRDD[40] at reduceByKey at MulticlassMetrics.scala:50), which is now runnable -2016-05-24 22:37:24 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=1906412, maxMem=766075207 -2016-05-24 22:37:24 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 2.3 KB, free 728.8 MB) -2016-05-24 22:37:24 INFO MemoryStore:59 - ensureFreeSpace(1412) called with curMem=1908748, maxMem=766075207 -2016-05-24 22:37:24 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 1412.0 B, free 728.8 MB) -2016-05-24 22:37:24 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:57585 (size: 1412.0 B, free: 730.0 MB) -2016-05-24 22:37:24 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (ShuffledRDD[40] at reduceByKey at MulticlassMetrics.scala:50) -2016-05-24 22:37:24 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-24 22:37:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:37:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:37:24 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-05-24 22:37:24 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-05-24 22:37:24 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-24 22:37:24 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:37:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-24 22:37:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-24 22:37:24 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 1161 bytes result sent to driver -2016-05-24 22:37:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 25 ms on localhost (1/2) -2016-05-24 22:37:24 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 1389 bytes result sent to driver -2016-05-24 22:37:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 28 ms on localhost (2/2) -2016-05-24 22:37:24 INFO DAGScheduler:59 - ResultStage 30 (collectAsMap at MulticlassMetrics.scala:51) finished in 0.028 s -2016-05-24 22:37:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-24 22:37:24 INFO DAGScheduler:59 - Job 29 finished: collectAsMap at MulticlassMetrics.scala:51, took 0.124565 s -2016-05-24 22:37:24 INFO SparkContext:59 - Starting job: countByValue at MulticlassMetrics.scala:45 -2016-05-24 22:37:24 INFO DAGScheduler:59 - Registering RDD 43 (countByValue at MulticlassMetrics.scala:45) -2016-05-24 22:37:24 INFO DAGScheduler:59 - Got job 30 (countByValue at MulticlassMetrics.scala:45) with 2 output partitions -2016-05-24 22:37:24 INFO DAGScheduler:59 - Final stage: ResultStage 32(countByValue at MulticlassMetrics.scala:45) -2016-05-24 22:37:24 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 31) -2016-05-24 22:37:24 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 31) -2016-05-24 22:37:24 INFO DAGScheduler:59 - Submitting ShuffleMapStage 31 (MapPartitionsRDD[43] at countByValue at MulticlassMetrics.scala:45), which has no missing parents -2016-05-24 22:37:24 INFO MemoryStore:59 - ensureFreeSpace(56224) called with curMem=1910160, maxMem=766075207 -2016-05-24 22:37:24 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 54.9 KB, free 728.7 MB) -2016-05-24 22:37:24 INFO MemoryStore:59 - ensureFreeSpace(15529) called with curMem=1966384, maxMem=766075207 -2016-05-24 22:37:24 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 15.2 KB, free 728.7 MB) -2016-05-24 22:37:24 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:57585 (size: 15.2 KB, free: 730.0 MB) -2016-05-24 22:37:24 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 31 (MapPartitionsRDD[43] at countByValue at MulticlassMetrics.scala:45) -2016-05-24 22:37:24 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-24 22:37:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:37:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2185 bytes) -2016-05-24 22:37:24 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-05-24 22:37:24 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-05-24 22:37:24 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-24 22:37:24 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-24 22:37:24 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 2254 bytes result sent to driver -2016-05-24 22:37:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 15 ms on localhost (1/2) -2016-05-24 22:37:24 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 2254 bytes result sent to driver -2016-05-24 22:37:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 18 ms on localhost (2/2) -2016-05-24 22:37:24 INFO DAGScheduler:59 - ShuffleMapStage 31 (countByValue at MulticlassMetrics.scala:45) finished in 0.019 s -2016-05-24 22:37:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-24 22:37:24 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:37:24 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:37:24 INFO DAGScheduler:59 - waiting: Set(ResultStage 32) -2016-05-24 22:37:24 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:37:24 INFO DAGScheduler:59 - Missing parents for ResultStage 32: List() -2016-05-24 22:37:24 INFO DAGScheduler:59 - Submitting ResultStage 32 (ShuffledRDD[44] at countByValue at MulticlassMetrics.scala:45), which is now runnable -2016-05-24 22:37:24 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=1981913, maxMem=766075207 -2016-05-24 22:37:24 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 2.3 KB, free 728.7 MB) -2016-05-24 22:37:24 INFO MemoryStore:59 - ensureFreeSpace(1371) called with curMem=1984249, maxMem=766075207 -2016-05-24 22:37:24 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 1371.0 B, free 728.7 MB) -2016-05-24 22:37:24 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:57585 (size: 1371.0 B, free: 730.0 MB) -2016-05-24 22:37:24 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (ShuffledRDD[44] at countByValue at MulticlassMetrics.scala:45) -2016-05-24 22:37:24 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-24 22:37:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:37:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:37:24 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-05-24 22:37:24 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-05-24 22:37:24 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:37:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-24 22:37:24 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-24 22:37:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-24 22:37:24 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 1161 bytes result sent to driver -2016-05-24 22:37:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 5 ms on localhost (1/2) -2016-05-24 22:37:24 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 1394 bytes result sent to driver -2016-05-24 22:37:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 7 ms on localhost (2/2) -2016-05-24 22:37:24 INFO DAGScheduler:59 - ResultStage 32 (countByValue at MulticlassMetrics.scala:45) finished in 0.007 s -2016-05-24 22:37:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-24 22:37:24 INFO DAGScheduler:59 - Job 30 finished: countByValue at MulticlassMetrics.scala:45, took 0.048433 s -2016-05-24 22:37:24 INFO SparkContext:59 - Starting job: saveAsTextFile at GLMClassificationModel.scala:61 -2016-05-24 22:37:24 INFO DAGScheduler:59 - Got job 31 (saveAsTextFile at GLMClassificationModel.scala:61) with 1 output partitions -2016-05-24 22:37:24 INFO DAGScheduler:59 - Final stage: ResultStage 33(saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:37:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:24 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[46] at saveAsTextFile at GLMClassificationModel.scala:61), which has no missing parents -2016-05-24 22:37:24 INFO MemoryStore:59 - ensureFreeSpace(127008) called with curMem=1985620, maxMem=766075207 -2016-05-24 22:37:24 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 124.0 KB, free 728.6 MB) -2016-05-24 22:37:24 INFO MemoryStore:59 - ensureFreeSpace(42653) called with curMem=2112628, maxMem=766075207 -2016-05-24 22:37:24 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 41.7 KB, free 728.5 MB) -2016-05-24 22:37:24 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:57585 (size: 41.7 KB, free: 729.9 MB) -2016-05-24 22:37:24 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 33 (MapPartitionsRDD[46] at saveAsTextFile at GLMClassificationModel.scala:61) -2016-05-24 22:37:24 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 1 tasks -2016-05-24 22:37:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-24 22:37:24 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-05-24 22:37:24 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242237_0033_m_000000_65' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/LogisticRegressionWithLDFGS20160524223724357/result/metadata/_temporary/0/task_201605242237_0033_m_000000 -2016-05-24 22:37:24 INFO SparkHadoopMapRedUtil:59 - attempt_201605242237_0033_m_000000_65: Committed -2016-05-24 22:37:24 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 1864 bytes result sent to driver -2016-05-24 22:37:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 186 ms on localhost (1/1) -2016-05-24 22:37:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-24 22:37:24 INFO DAGScheduler:59 - ResultStage 33 (saveAsTextFile at GLMClassificationModel.scala:61) finished in 0.187 s -2016-05-24 22:37:24 INFO DAGScheduler:59 - Job 31 finished: saveAsTextFile at GLMClassificationModel.scala:61, took 0.254608 s -2016-05-24 22:37:25 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 729.9 MB) -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 729.9 MB) -2016-05-24 22:37:25 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 729.9 MB) -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:57585 in memory (size: 41.7 KB, free: 730.0 MB) -2016-05-24 22:37:25 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:57585 in memory (size: 1371.0 B, free: 730.0 MB) -2016-05-24 22:37:25 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:57585 in memory (size: 15.2 KB, free: 730.0 MB) -2016-05-24 22:37:25 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-24 22:37:25 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:57585 in memory (size: 1412.0 B, free: 730.0 MB) -2016-05-24 22:37:25 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:57585 in memory (size: 15.0 KB, free: 730.0 MB) -2016-05-24 22:37:25 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-24 22:37:25 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.0 MB) -2016-05-24 22:37:25 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 730.0 MB) -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:25 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:25 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:25 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:25 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 730.1 MB) -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.1 MB) -2016-05-24 22:37:25 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 730.2 MB) -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.2 MB) -2016-05-24 22:37:25 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:57585 in memory (size: 11.7 KB, free: 730.2 MB) -2016-05-24 22:37:25 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:57585 in memory (size: 11.0 KB, free: 730.2 MB) -2016-05-24 22:37:26 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:37:26 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:37:26 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:65 -2016-05-24 22:37:26 INFO DAGScheduler:59 - Got job 32 (parquet at GLMClassificationModel.scala:65) with 1 output partitions -2016-05-24 22:37:26 INFO DAGScheduler:59 - Final stage: ResultStage 34(parquet at GLMClassificationModel.scala:65) -2016-05-24 22:37:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:26 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[48] at rddToDataFrameHolder at GLMClassificationModel.scala:65), which has no missing parents -2016-05-24 22:37:26 INFO MemoryStore:59 - ensureFreeSpace(67128) called with curMem=576996, maxMem=766075207 -2016-05-24 22:37:26 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 65.6 KB, free 730.0 MB) -2016-05-24 22:37:26 INFO MemoryStore:59 - ensureFreeSpace(23497) called with curMem=644124, maxMem=766075207 -2016-05-24 22:37:26 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 22.9 KB, free 729.9 MB) -2016-05-24 22:37:26 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:57585 (size: 22.9 KB, free: 730.2 MB) -2016-05-24 22:37:26 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 34 (MapPartitionsRDD[48] at rddToDataFrameHolder at GLMClassificationModel.scala:65) -2016-05-24 22:37:26 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 1 tasks -2016-05-24 22:37:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 66, localhost, PROCESS_LOCAL, 52320 bytes) -2016-05-24 22:37:26 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 66) -2016-05-24 22:37:26 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:37:26 INFO CodecConfig:151 - Compression: GZIP -2016-05-24 22:37:26 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-24 22:37:26 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-24 22:37:26 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-24 22:37:26 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-24 22:37:26 INFO ParquetOutputFormat:151 - Validation is off -2016-05-24 22:37:26 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-24 22:37:26 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-24 22:37:26 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 31,608 -2016-05-24 22:37:26 INFO ColumnChunkPageWriteStore:151 - written 61B for [weights, type] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [PLAIN, BIT_PACKED, RLE] -2016-05-24 22:37:26 INFO ColumnChunkPageWriteStore:151 - written 48B for [weights, size] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [PLAIN, BIT_PACKED, RLE] -2016-05-24 22:37:26 INFO ColumnChunkPageWriteStore:151 - written 51B for [weights, indices, array] INT32: 1 values, 13B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-24 22:37:26 INFO ColumnChunkPageWriteStore:151 - written 1,354B for [weights, values, array] DOUBLE: 6,228 values, 6,506B raw, 1,310B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 825 entries, 6,600B raw, 825B comp} -2016-05-24 22:37:26 INFO ColumnChunkPageWriteStore:151 - written 69B for [intercept] DOUBLE: 1 values, 14B raw, 28B comp, 1 pages, encodings: [PLAIN, BIT_PACKED, RLE] -2016-05-24 22:37:26 INFO ColumnChunkPageWriteStore:151 - written 71B for [threshold] DOUBLE: 1 values, 14B raw, 30B comp, 1 pages, encodings: [PLAIN, BIT_PACKED, RLE] -2016-05-24 22:37:27 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242237_0034_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/LogisticRegressionWithLDFGS20160524223724357/result/data/_temporary/0/task_201605242237_0034_m_000000 -2016-05-24 22:37:27 INFO SparkHadoopMapRedUtil:59 - attempt_201605242237_0034_m_000000_0: Committed -2016-05-24 22:37:27 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 66). 915 bytes result sent to driver -2016-05-24 22:37:27 INFO DAGScheduler:59 - ResultStage 34 (parquet at GLMClassificationModel.scala:65) finished in 0.540 s -2016-05-24 22:37:27 INFO DAGScheduler:59 - Job 32 finished: parquet at GLMClassificationModel.scala:65, took 0.575601 s -2016-05-24 22:37:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 66) in 540 ms on localhost (1/1) -2016-05-24 22:37:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-24 22:37:27 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:37:27 INFO DefaultWriterContainer:59 - Job job_201605242237_0000 committed. -2016-05-24 22:37:27 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/LogisticRegressionWithLDFGS20160524223724357/result/data on driver -2016-05-24 22:37:27 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/LogisticRegressionWithLDFGS20160524223724357/result/data on driver -2016-05-24 22:37:27 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=667621, maxMem=766075207 -2016-05-24 22:37:27 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 225.3 KB, free 729.7 MB) -2016-05-24 22:37:27 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=898325, maxMem=766075207 -2016-05-24 22:37:27 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.7 MB) -2016-05-24 22:37:27 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:57585 (size: 19.3 KB, free: 730.1 MB) -2016-05-24 22:37:27 INFO SparkContext:59 - Created broadcast 60 from textFile at modelSaveLoad.scala:129 -2016-05-24 22:37:27 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:37:27 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-24 22:37:27 INFO DAGScheduler:59 - Got job 33 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-24 22:37:27 INFO DAGScheduler:59 - Final stage: ResultStage 35(first at modelSaveLoad.scala:129) -2016-05-24 22:37:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:27 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[51] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-24 22:37:27 INFO MemoryStore:59 - ensureFreeSpace(3224) called with curMem=918113, maxMem=766075207 -2016-05-24 22:37:27 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 3.1 KB, free 729.7 MB) -2016-05-24 22:37:27 INFO MemoryStore:59 - ensureFreeSpace(1899) called with curMem=921337, maxMem=766075207 -2016-05-24 22:37:27 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 1899.0 B, free 729.7 MB) -2016-05-24 22:37:27 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:57585 (size: 1899.0 B, free: 730.1 MB) -2016-05-24 22:37:27 INFO SparkContext:59 - Created broadcast 61 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 35 (MapPartitionsRDD[51] at textFile at modelSaveLoad.scala:129) -2016-05-24 22:37:27 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 1 tasks -2016-05-24 22:37:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2247 bytes) -2016-05-24 22:37:27 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 67) -2016-05-24 22:37:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/LogisticRegressionWithLDFGS20160524223724357/result/metadata/part-00000:0+62 -2016-05-24 22:37:27 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 67). 2170 bytes result sent to driver -2016-05-24 22:37:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 67) in 11 ms on localhost (1/1) -2016-05-24 22:37:27 INFO DAGScheduler:59 - ResultStage 35 (first at modelSaveLoad.scala:129) finished in 0.011 s -2016-05-24 22:37:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-24 22:37:27 INFO DAGScheduler:59 - Job 33 finished: first at modelSaveLoad.scala:129, took 0.022116 s -2016-05-24 22:37:27 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/LogisticRegressionWithLDFGS20160524223724357/result/data on driver -2016-05-24 22:37:27 INFO SparkContext:59 - Starting job: parquet at GLMClassificationModel.scala:78 -2016-05-24 22:37:27 INFO DAGScheduler:59 - Got job 34 (parquet at GLMClassificationModel.scala:78) with 4 output partitions -2016-05-24 22:37:27 INFO DAGScheduler:59 - Final stage: ResultStage 36(parquet at GLMClassificationModel.scala:78) -2016-05-24 22:37:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:27 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[53] at parquet at GLMClassificationModel.scala:78), which has no missing parents -2016-05-24 22:37:27 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=923236, maxMem=766075207 -2016-05-24 22:37:27 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 61.5 KB, free 729.6 MB) -2016-05-24 22:37:27 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=986236, maxMem=766075207 -2016-05-24 22:37:27 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 20.6 KB, free 729.6 MB) -2016-05-24 22:37:27 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:57585 (size: 20.6 KB, free: 730.1 MB) -2016-05-24 22:37:27 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 36 (MapPartitionsRDD[53] at parquet at GLMClassificationModel.scala:78) -2016-05-24 22:37:27 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 4 tasks -2016-05-24 22:37:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 68, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:37:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 69, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:37:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:37:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2288 bytes) -2016-05-24 22:37:27 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 68) -2016-05-24 22:37:27 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 69) -2016-05-24 22:37:27 INFO Executor:59 - Running task 2.0 in stage 36.0 (TID 70) -2016-05-24 22:37:27 INFO Executor:59 - Running task 3.0 in stage 36.0 (TID 71) -2016-05-24 22:37:27 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:37:27 INFO Executor:59 - Finished task 2.0 in stage 36.0 (TID 70). 936 bytes result sent to driver -2016-05-24 22:37:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 36.0 (TID 70) in 65 ms on localhost (1/4) -2016-05-24 22:37:27 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:37:27 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 69). 936 bytes result sent to driver -2016-05-24 22:37:27 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:37:27 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 68). 936 bytes result sent to driver -2016-05-24 22:37:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 69) in 75 ms on localhost (2/4) -2016-05-24 22:37:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 68) in 76 ms on localhost (3/4) -2016-05-24 22:37:27 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:37:27 INFO Executor:59 - Finished task 3.0 in stage 36.0 (TID 71). 1802 bytes result sent to driver -2016-05-24 22:37:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 36.0 (TID 71) in 95 ms on localhost (4/4) -2016-05-24 22:37:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-24 22:37:27 INFO DAGScheduler:59 - ResultStage 36 (parquet at GLMClassificationModel.scala:78) finished in 0.098 s -2016-05-24 22:37:27 INFO DAGScheduler:59 - Job 34 finished: parquet at GLMClassificationModel.scala:78, took 0.124962 s -2016-05-24 22:37:27 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=1007309, maxMem=766075207 -2016-05-24 22:37:27 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 86.4 KB, free 729.5 MB) -2016-05-24 22:37:27 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1095821, maxMem=766075207 -2016-05-24 22:37:27 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.5 MB) -2016-05-24 22:37:27 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:57585 (size: 19.3 KB, free: 730.1 MB) -2016-05-24 22:37:27 INFO SparkContext:59 - Created broadcast 63 from take at GLMClassificationModel.scala:79 -2016-05-24 22:37:27 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-24 22:37:27 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/LogisticRegressionWithLDFGS20160524223724357/result/data/part-r-00000-3f5e4cc8-ffa7-45a9-ba64-f255adce628b.gz.parquet -2016-05-24 22:37:27 INFO SparkContext:59 - Starting job: take at GLMClassificationModel.scala:79 -2016-05-24 22:37:27 INFO DAGScheduler:59 - Got job 35 (take at GLMClassificationModel.scala:79) with 1 output partitions -2016-05-24 22:37:27 INFO DAGScheduler:59 - Final stage: ResultStage 37(take at GLMClassificationModel.scala:79) -2016-05-24 22:37:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:37:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:37:27 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[55] at take at GLMClassificationModel.scala:79), which has no missing parents -2016-05-24 22:37:27 INFO MemoryStore:59 - ensureFreeSpace(4168) called with curMem=1115609, maxMem=766075207 -2016-05-24 22:37:27 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 4.1 KB, free 729.5 MB) -2016-05-24 22:37:27 INFO MemoryStore:59 - ensureFreeSpace(2436) called with curMem=1119777, maxMem=766075207 -2016-05-24 22:37:27 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 2.4 KB, free 729.5 MB) -2016-05-24 22:37:27 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:57585 (size: 2.4 KB, free: 730.1 MB) -2016-05-24 22:37:27 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:37:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 37 (MapPartitionsRDD[55] at take at GLMClassificationModel.scala:79) -2016-05-24 22:37:27 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 1 tasks -2016-05-24 22:37:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 72, localhost, PROCESS_LOCAL, 2306 bytes) -2016-05-24 22:37:27 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 72) -2016-05-24 22:37:27 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/LogisticRegressionWithLDFGS20160524223724357/result/data/part-r-00000-3f5e4cc8-ffa7-45a9-ba64-f255adce628b.gz.parquet start: 0 end: 9320 length: 9320 hosts: []} -2016-05-24 22:37:27 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-24 22:37:28 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group weights { - required int32 type (INT_8); - optional int32 size; - optional group indices (LIST) { - repeated int32 array; - } - optional group values (LIST) { - repeated double array; - } - } - optional double intercept; - optional double threshold; -} - -Catalyst form: -StructType(StructField(weights,org.apache.spark.mllib.linalg.VectorUDT@f71b0bce,true), StructField(intercept,DoubleType,true), StructField(threshold,DoubleType,true)) - -2016-05-24 22:37:28 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-24 22:37:28 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-24 22:37:28 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-24 22:37:28 INFO InternalParquetRecordReader:151 - block read in memory in 9 ms. row count = 1 -2016-05-24 22:37:28 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 72). 90183 bytes result sent to driver -2016-05-24 22:37:28 INFO DAGScheduler:59 - ResultStage 37 (take at GLMClassificationModel.scala:79) finished in 0.176 s -2016-05-24 22:37:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 72) in 176 ms on localhost (1/1) -2016-05-24 22:37:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-24 22:37:28 INFO DAGScheduler:59 - Job 35 finished: take at GLMClassificationModel.scala:79, took 0.193330 s -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 22:37:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 22:37:28 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 22:37:28 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 22:37:28 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 22:37:28 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 22:37:28 INFO BlockManager:59 - BlockManager stopped -2016-05-24 22:37:28 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 22:37:28 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 22:37:28 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 22:37:28 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 22:37:28 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-8c3de5cc-bc83-40c6-93a6-67ed945c30a9 -2016-05-24 22:54:07 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 22:54:08 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 22:54:08 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 22:54:08 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 22:54:08 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 22:54:09 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 22:54:09 INFO Remoting:74 - Starting remoting -2016-05-24 22:54:09 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:49463] -2016-05-24 22:54:09 INFO Utils:59 - Successfully started service 'sparkDriver' on port 49463. -2016-05-24 22:54:09 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 22:54:09 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 22:54:09 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-3e748b6c-9b28-4fc2-9c2b-aa5ba968a5f2 -2016-05-24 22:54:09 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 22:54:09 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-03058b4e-16e6-4abb-aab7-e04353ae3d95\httpd-3d89ae23-3b0f-4bdf-a80e-a3231b66e3b3 -2016-05-24 22:54:09 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 22:54:09 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:54:09 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:49464 -2016-05-24 22:54:09 INFO Utils:59 - Successfully started service 'HTTP file server' on port 49464. -2016-05-24 22:54:09 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 22:54:10 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:54:10 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 22:54:10 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 22:54:10 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 22:54:10 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 22:54:10 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 22:54:10 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 49483. -2016-05-24 22:54:10 INFO NettyBlockTransferService:59 - Server created on 49483 -2016-05-24 22:54:10 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 22:54:10 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:49483 with 730.6 MB RAM, BlockManagerId(driver, localhost, 49483) -2016-05-24 22:54:10 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 22:54:11 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 22:54:11 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 22:54:11 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 22:54:11 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 22:54:11 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:49483 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 22:54:11 INFO SparkContext:59 - Created broadcast 0 from textFile at NaiveBayesLearning.scala:26 -2016-05-24 22:54:13 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 22:54:13 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:54:13 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-24 22:54:13 INFO DAGScheduler:59 - Registering RDD 5 (map at NaiveBayes.scala:383) -2016-05-24 22:54:13 INFO DAGScheduler:59 - Got job 0 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-24 22:54:13 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at NaiveBayes.scala:401) -2016-05-24 22:54:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-05-24 22:54:13 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-05-24 22:54:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-24 22:54:13 INFO MemoryStore:59 - ensureFreeSpace(5104) called with curMem=171580, maxMem=766075207 -2016-05-24 22:54:13 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-24 22:54:13 INFO MemoryStore:59 - ensureFreeSpace(2692) called with curMem=176684, maxMem=766075207 -2016-05-24 22:54:13 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-24 22:54:13 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:49483 (size: 2.6 KB, free: 730.6 MB) -2016-05-24 22:54:13 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:54:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383) -2016-05-24 22:54:13 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 22:54:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-24 22:54:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-24 22:54:13 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 22:54:13 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 22:54:13 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-24 22:54:13 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-24 22:54:13 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 22:54:13 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 22:54:13 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 22:54:13 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 22:54:13 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 22:54:13 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 22:54:13 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 22:54:13 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver -2016-05-24 22:54:13 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver -2016-05-24 22:54:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 219 ms on localhost (1/2) -2016-05-24 22:54:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 242 ms on localhost (2/2) -2016-05-24 22:54:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 22:54:13 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at NaiveBayes.scala:383) finished in 0.257 s -2016-05-24 22:54:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:54:13 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:54:13 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-05-24 22:54:13 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:54:13 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-05-24 22:54:13 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-24 22:54:13 INFO MemoryStore:59 - ensureFreeSpace(2936) called with curMem=179376, maxMem=766075207 -2016-05-24 22:54:13 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:54:13 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=182312, maxMem=766075207 -2016-05-24 22:54:13 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.4 MB) -2016-05-24 22:54:13 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:49483 (size: 1567.0 B, free: 730.6 MB) -2016-05-24 22:54:13 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:54:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383) -2016-05-24 22:54:13 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 22:54:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:54:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:54:13 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-24 22:54:13 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 22:54:13 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-24 22:54:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:54:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-24 22:54:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-24 22:54:13 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-05-24 22:54:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 33 ms on localhost (1/2) -2016-05-24 22:54:13 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1645 bytes result sent to driver -2016-05-24 22:54:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 41 ms on localhost (2/2) -2016-05-24 22:54:13 INFO DAGScheduler:59 - ResultStage 1 (collect at NaiveBayes.scala:401) finished in 0.041 s -2016-05-24 22:54:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 22:54:13 INFO DAGScheduler:59 - Job 0 finished: collect at NaiveBayes.scala:401, took 0.425924 s -2016-05-24 22:54:13 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:39 -2016-05-24 22:54:13 INFO DAGScheduler:59 - Got job 1 (count at NaiveBayesLearning.scala:39) with 2 output partitions -2016-05-24 22:54:13 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at NaiveBayesLearning.scala:39) -2016-05-24 22:54:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:54:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:54:13 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-24 22:54:13 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=183879, maxMem=766075207 -2016-05-24 22:54:13 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-24 22:54:13 INFO MemoryStore:59 - ensureFreeSpace(2671) called with curMem=188775, maxMem=766075207 -2016-05-24 22:54:13 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-24 22:54:13 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:49483 (size: 2.6 KB, free: 730.6 MB) -2016-05-24 22:54:13 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:54:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:39) -2016-05-24 22:54:13 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 22:54:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 22:54:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 22:54:13 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-24 22:54:13 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-24 22:54:13 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-24 22:54:13 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-24 22:54:13 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 22:54:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 46 ms on localhost (1/2) -2016-05-24 22:54:13 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 22:54:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 37 ms on localhost (2/2) -2016-05-24 22:54:13 INFO DAGScheduler:59 - ResultStage 2 (count at NaiveBayesLearning.scala:39) finished in 0.060 s -2016-05-24 22:54:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 22:54:13 INFO DAGScheduler:59 - Job 1 finished: count at NaiveBayesLearning.scala:39, took 0.074443 s -2016-05-24 22:54:13 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:39 -2016-05-24 22:54:13 INFO DAGScheduler:59 - Got job 2 (count at NaiveBayesLearning.scala:39) with 2 output partitions -2016-05-24 22:54:13 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at NaiveBayesLearning.scala:39) -2016-05-24 22:54:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:54:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:54:13 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-24 22:54:13 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=191446, maxMem=766075207 -2016-05-24 22:54:13 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-24 22:54:13 INFO MemoryStore:59 - ensureFreeSpace(2117) called with curMem=195046, maxMem=766075207 -2016-05-24 22:54:13 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:54:13 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:49483 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:54:13 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:54:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-24 22:54:13 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 22:54:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 22:54:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 22:54:13 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-24 22:54:13 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-24 22:54:13 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-24 22:54:13 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-24 22:54:13 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 22:54:13 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 2082 bytes result sent to driver -2016-05-24 22:54:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 18 ms on localhost (1/2) -2016-05-24 22:54:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 18 ms on localhost (2/2) -2016-05-24 22:54:13 INFO DAGScheduler:59 - ResultStage 3 (count at NaiveBayesLearning.scala:39) finished in 0.019 s -2016-05-24 22:54:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 22:54:13 INFO DAGScheduler:59 - Job 2 finished: count at NaiveBayesLearning.scala:39, took 0.031254 s -2016-05-24 22:54:14 INFO SparkContext:59 - Starting job: saveAsTextFile at NaiveBayes.scala:202 -2016-05-24 22:54:14 INFO DAGScheduler:59 - Got job 3 (saveAsTextFile at NaiveBayes.scala:202) with 1 output partitions -2016-05-24 22:54:14 INFO DAGScheduler:59 - Final stage: ResultStage 4(saveAsTextFile at NaiveBayes.scala:202) -2016-05-24 22:54:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:54:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:54:14 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202), which has no missing parents -2016-05-24 22:54:14 INFO MemoryStore:59 - ensureFreeSpace(126984) called with curMem=197163, maxMem=766075207 -2016-05-24 22:54:14 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 124.0 KB, free 730.3 MB) -2016-05-24 22:54:14 INFO MemoryStore:59 - ensureFreeSpace(42576) called with curMem=324147, maxMem=766075207 -2016-05-24 22:54:14 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.2 MB) -2016-05-24 22:54:14 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:49483 (size: 41.6 KB, free: 730.5 MB) -2016-05-24 22:54:14 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:54:14 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202) -2016-05-24 22:54:14 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-05-24 22:54:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-24 22:54:14 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-24 22:54:14 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242254_0004_m_000000_8' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/NaiveBayesModel20160524225413800/result/metadata/_temporary/0/task_201605242254_0004_m_000000 -2016-05-24 22:54:14 INFO SparkHadoopMapRedUtil:59 - attempt_201605242254_0004_m_000000_8: Committed -2016-05-24 22:54:14 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 1864 bytes result sent to driver -2016-05-24 22:54:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 279 ms on localhost (1/1) -2016-05-24 22:54:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 22:54:14 INFO DAGScheduler:59 - ResultStage 4 (saveAsTextFile at NaiveBayes.scala:202) finished in 0.279 s -2016-05-24 22:54:14 INFO DAGScheduler:59 - Job 3 finished: saveAsTextFile at NaiveBayes.scala:202, took 0.549215 s -2016-05-24 22:54:15 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:49483 in memory (size: 41.6 KB, free: 730.6 MB) -2016-05-24 22:54:15 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 22:54:15 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:49483 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:54:15 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 22:54:15 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:49483 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-24 22:54:15 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 22:54:15 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:49483 in memory (size: 1567.0 B, free: 730.6 MB) -2016-05-24 22:54:15 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 22:54:15 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:49483 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-24 22:54:15 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 22:54:15 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-24 22:54:16 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:54:16 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:54:16 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:206 -2016-05-24 22:54:16 INFO DAGScheduler:59 - Got job 4 (parquet at NaiveBayes.scala:206) with 1 output partitions -2016-05-24 22:54:16 INFO DAGScheduler:59 - Final stage: ResultStage 5(parquet at NaiveBayes.scala:206) -2016-05-24 22:54:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:54:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:54:16 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205), which has no missing parents -2016-05-24 22:54:16 INFO MemoryStore:59 - ensureFreeSpace(66928) called with curMem=171580, maxMem=766075207 -2016-05-24 22:54:16 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 65.4 KB, free 730.4 MB) -2016-05-24 22:54:16 INFO MemoryStore:59 - ensureFreeSpace(23365) called with curMem=238508, maxMem=766075207 -2016-05-24 22:54:16 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 22.8 KB, free 730.3 MB) -2016-05-24 22:54:16 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:49483 (size: 22.8 KB, free: 730.6 MB) -2016-05-24 22:54:16 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:54:16 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 5 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205) -2016-05-24 22:54:16 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 1 tasks -2016-05-24 22:54:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2506 bytes) -2016-05-24 22:54:16 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 22:54:16 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:54:16 INFO CodecConfig:151 - Compression: GZIP -2016-05-24 22:54:16 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-24 22:54:16 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-24 22:54:16 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-24 22:54:16 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-24 22:54:16 INFO ParquetOutputFormat:151 - Validation is off -2016-05-24 22:54:16 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-24 22:54:17 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-24 22:54:17 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 231 -2016-05-24 22:54:17 INFO ColumnChunkPageWriteStore:151 - written 77B for [labels, bag, array] DOUBLE: 3 values, 37B raw, 36B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-24 22:54:17 INFO ColumnChunkPageWriteStore:151 - written 74B for [pi, bag, array] DOUBLE: 3 values, 16B raw, 33B comp, 1 pages, encodings: [RLE, PLAIN_DICTIONARY], dic { 2 entries, 16B raw, 2B comp} -2016-05-24 22:54:17 INFO ColumnChunkPageWriteStore:151 - written 83B for [theta, bag, array, bag, array] DOUBLE: 9 values, 23B raw, 42B comp, 1 pages, encodings: [RLE, PLAIN_DICTIONARY], dic { 6 entries, 48B raw, 6B comp} -2016-05-24 22:54:17 INFO ColumnChunkPageWriteStore:151 - written 86B for [modelType] BINARY: 1 values, 21B raw, 39B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-24 22:54:17 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242254_0005_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/NaiveBayesModel20160524225413800/result/data/_temporary/0/task_201605242254_0005_m_000000 -2016-05-24 22:54:17 INFO SparkHadoopMapRedUtil:59 - attempt_201605242254_0005_m_000000_0: Committed -2016-05-24 22:54:17 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 915 bytes result sent to driver -2016-05-24 22:54:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 574 ms on localhost (1/1) -2016-05-24 22:54:17 INFO DAGScheduler:59 - ResultStage 5 (parquet at NaiveBayes.scala:206) finished in 0.574 s -2016-05-24 22:54:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 22:54:17 INFO DAGScheduler:59 - Job 4 finished: parquet at NaiveBayes.scala:206, took 0.621154 s -2016-05-24 22:54:17 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:54:17 INFO DefaultWriterContainer:59 - Job job_201605242254_0000 committed. -2016-05-24 22:54:17 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/NaiveBayesModel20160524225413800/result/data on driver -2016-05-24 22:54:17 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/NaiveBayesModel20160524225413800/result/data on driver -2016-05-24 22:54:17 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=261873, maxMem=766075207 -2016-05-24 22:54:17 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-24 22:54:17 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=492577, maxMem=766075207 -2016-05-24 22:54:17 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-24 22:54:17 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:49483 (size: 19.3 KB, free: 730.5 MB) -2016-05-24 22:54:17 INFO SparkContext:59 - Created broadcast 7 from textFile at modelSaveLoad.scala:129 -2016-05-24 22:54:17 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:54:17 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-24 22:54:17 INFO DAGScheduler:59 - Got job 5 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-24 22:54:17 INFO DAGScheduler:59 - Final stage: ResultStage 6(first at modelSaveLoad.scala:129) -2016-05-24 22:54:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:54:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:54:17 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-24 22:54:17 INFO MemoryStore:59 - ensureFreeSpace(3208) called with curMem=512365, maxMem=766075207 -2016-05-24 22:54:17 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 22:54:17 INFO MemoryStore:59 - ensureFreeSpace(1895) called with curMem=515573, maxMem=766075207 -2016-05-24 22:54:17 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 1895.0 B, free 730.1 MB) -2016-05-24 22:54:17 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:49483 (size: 1895.0 B, free: 730.5 MB) -2016-05-24 22:54:17 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:54:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 6 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129) -2016-05-24 22:54:17 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 1 tasks -2016-05-24 22:54:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 10, localhost, PROCESS_LOCAL, 2234 bytes) -2016-05-24 22:54:17 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 10) -2016-05-24 22:54:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/NaiveBayesModel20160524225413800/result/metadata/part-00000:0+56 -2016-05-24 22:54:17 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 10). 2159 bytes result sent to driver -2016-05-24 22:54:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 10) in 15 ms on localhost (1/1) -2016-05-24 22:54:17 INFO DAGScheduler:59 - ResultStage 6 (first at modelSaveLoad.scala:129) finished in 0.015 s -2016-05-24 22:54:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 22:54:17 INFO DAGScheduler:59 - Job 5 finished: first at modelSaveLoad.scala:129, took 0.031492 s -2016-05-24 22:54:18 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/NaiveBayesModel20160524225413800/result/data on driver -2016-05-24 22:54:18 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:213 -2016-05-24 22:54:18 INFO DAGScheduler:59 - Got job 6 (parquet at NaiveBayes.scala:213) with 4 output partitions -2016-05-24 22:54:18 INFO DAGScheduler:59 - Final stage: ResultStage 7(parquet at NaiveBayes.scala:213) -2016-05-24 22:54:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:54:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:54:18 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213), which has no missing parents -2016-05-24 22:54:18 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=517468, maxMem=766075207 -2016-05-24 22:54:18 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-24 22:54:18 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=580468, maxMem=766075207 -2016-05-24 22:54:18 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.0 MB) -2016-05-24 22:54:18 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:49483 (size: 20.6 KB, free: 730.5 MB) -2016-05-24 22:54:18 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:54:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 7 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213) -2016-05-24 22:54:18 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 4 tasks -2016-05-24 22:54:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 11, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:54:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 12, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:54:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:54:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2276 bytes) -2016-05-24 22:54:18 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 11) -2016-05-24 22:54:18 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 12) -2016-05-24 22:54:18 INFO Executor:59 - Running task 3.0 in stage 7.0 (TID 14) -2016-05-24 22:54:18 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:54:18 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 12). 936 bytes result sent to driver -2016-05-24 22:54:18 INFO Executor:59 - Running task 2.0 in stage 7.0 (TID 13) -2016-05-24 22:54:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 12) in 51 ms on localhost (1/4) -2016-05-24 22:54:18 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:54:18 INFO Executor:59 - Finished task 3.0 in stage 7.0 (TID 14). 1876 bytes result sent to driver -2016-05-24 22:54:18 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:54:18 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:54:18 INFO Executor:59 - Finished task 2.0 in stage 7.0 (TID 13). 936 bytes result sent to driver -2016-05-24 22:54:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 7.0 (TID 14) in 85 ms on localhost (2/4) -2016-05-24 22:54:18 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 11). 936 bytes result sent to driver -2016-05-24 22:54:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 7.0 (TID 13) in 88 ms on localhost (3/4) -2016-05-24 22:54:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 11) in 92 ms on localhost (4/4) -2016-05-24 22:54:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 22:54:18 INFO DAGScheduler:59 - ResultStage 7 (parquet at NaiveBayes.scala:213) finished in 0.093 s -2016-05-24 22:54:18 INFO DAGScheduler:59 - Job 6 finished: parquet at NaiveBayes.scala:213, took 0.125878 s -2016-05-24 22:54:18 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=601541, maxMem=766075207 -2016-05-24 22:54:18 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 86.4 KB, free 729.9 MB) -2016-05-24 22:54:18 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=690053, maxMem=766075207 -2016-05-24 22:54:18 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-24 22:54:18 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:49483 (size: 19.3 KB, free: 730.5 MB) -2016-05-24 22:54:18 INFO SparkContext:59 - Created broadcast 10 from take at NaiveBayes.scala:216 -2016-05-24 22:54:18 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-24 22:54:18 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/NaiveBayesModel20160524225413800/result/data/part-r-00000-1496331f-3e7b-47d1-a32f-f752124ecbc6.gz.parquet -2016-05-24 22:54:18 INFO SparkContext:59 - Starting job: take at NaiveBayes.scala:216 -2016-05-24 22:54:18 INFO DAGScheduler:59 - Got job 7 (take at NaiveBayes.scala:216) with 1 output partitions -2016-05-24 22:54:18 INFO DAGScheduler:59 - Final stage: ResultStage 8(take at NaiveBayes.scala:216) -2016-05-24 22:54:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:54:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:54:18 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216), which has no missing parents -2016-05-24 22:54:18 INFO MemoryStore:59 - ensureFreeSpace(4280) called with curMem=709841, maxMem=766075207 -2016-05-24 22:54:18 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.2 KB, free 729.9 MB) -2016-05-24 22:54:18 INFO MemoryStore:59 - ensureFreeSpace(2466) called with curMem=714121, maxMem=766075207 -2016-05-24 22:54:18 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.4 KB, free 729.9 MB) -2016-05-24 22:54:18 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:49483 (size: 2.4 KB, free: 730.5 MB) -2016-05-24 22:54:18 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:54:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216) -2016-05-24 22:54:18 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-05-24 22:54:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2302 bytes) -2016-05-24 22:54:18 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 22:54:18 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/regression/NaiveBayesModel20160524225413800/result/data/part-r-00000-1496331f-3e7b-47d1-a32f-f752124ecbc6.gz.parquet start: 0 end: 1477 length: 1477 hosts: []} -2016-05-24 22:54:18 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-24 22:54:18 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group labels (LIST) { - repeated group bag { - optional double array; - } - } - optional group pi (LIST) { - repeated group bag { - optional double array; - } - } - optional group theta (LIST) { - repeated group bag { - optional group array (LIST) { - repeated group bag { - optional double array; - } - } - } - } - optional binary modelType (UTF8); -} - -Catalyst form: -StructType(StructField(labels,ArrayType(DoubleType,true),true), StructField(pi,ArrayType(DoubleType,true),true), StructField(theta,ArrayType(ArrayType(DoubleType,true),true),true), StructField(modelType,StringType,true)) - -2016-05-24 22:54:18 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-24 22:54:18 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-24 22:54:18 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-24 22:54:18 INFO InternalParquetRecordReader:151 - block read in memory in 8 ms. row count = 1 -2016-05-24 22:54:18 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2841 bytes result sent to driver -2016-05-24 22:54:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 92 ms on localhost (1/1) -2016-05-24 22:54:18 INFO DAGScheduler:59 - ResultStage 8 (take at NaiveBayes.scala:216) finished in 0.094 s -2016-05-24 22:54:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 22:54:18 INFO DAGScheduler:59 - Job 7 finished: take at NaiveBayes.scala:216, took 0.109822 s -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 22:54:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 22:54:18 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 22:54:18 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 22:54:18 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 22:54:18 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 22:54:18 INFO BlockManager:59 - BlockManager stopped -2016-05-24 22:54:18 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 22:54:18 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 22:54:18 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 22:54:18 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 22:54:18 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-03058b4e-16e6-4abb-aab7-e04353ae3d95 -2016-05-24 22:54:18 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 22:54:18 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-24 22:55:15 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 22:55:16 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 22:55:16 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 22:55:16 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 22:55:16 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 22:55:17 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 22:55:17 INFO Remoting:74 - Starting remoting -2016-05-24 22:55:17 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:49504] -2016-05-24 22:55:17 INFO Utils:59 - Successfully started service 'sparkDriver' on port 49504. -2016-05-24 22:55:17 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 22:55:17 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 22:55:17 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-48258356-31c7-40f2-beac-64f9ebe99276 -2016-05-24 22:55:17 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 22:55:17 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-e69f6567-1807-43cc-a9de-3cde23135eb1\httpd-0f39badf-4f09-476f-a532-64d836e7e463 -2016-05-24 22:55:17 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 22:55:17 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:55:17 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:49505 -2016-05-24 22:55:17 INFO Utils:59 - Successfully started service 'HTTP file server' on port 49505. -2016-05-24 22:55:17 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 22:55:17 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:55:17 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 22:55:17 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 22:55:17 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 22:55:18 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 22:55:18 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 22:55:18 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 49524. -2016-05-24 22:55:18 INFO NettyBlockTransferService:59 - Server created on 49524 -2016-05-24 22:55:18 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 22:55:18 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:49524 with 730.6 MB RAM, BlockManagerId(driver, localhost, 49524) -2016-05-24 22:55:18 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 22:55:18 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 22:55:18 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 22:55:19 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 22:55:19 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 22:55:19 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:49524 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 22:55:19 INFO SparkContext:59 - Created broadcast 0 from textFile at NaiveBayesLearning.scala:26 -2016-05-24 22:55:20 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 22:55:20 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:55:20 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-24 22:55:20 INFO DAGScheduler:59 - Registering RDD 5 (map at NaiveBayes.scala:383) -2016-05-24 22:55:20 INFO DAGScheduler:59 - Got job 0 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-24 22:55:20 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at NaiveBayes.scala:401) -2016-05-24 22:55:20 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-05-24 22:55:20 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-05-24 22:55:20 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-24 22:55:20 INFO MemoryStore:59 - ensureFreeSpace(5104) called with curMem=171580, maxMem=766075207 -2016-05-24 22:55:20 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-24 22:55:20 INFO MemoryStore:59 - ensureFreeSpace(2692) called with curMem=176684, maxMem=766075207 -2016-05-24 22:55:20 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-24 22:55:20 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:49524 (size: 2.6 KB, free: 730.6 MB) -2016-05-24 22:55:20 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:55:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383) -2016-05-24 22:55:20 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 22:55:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-24 22:55:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-24 22:55:20 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 22:55:20 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 22:55:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-24 22:55:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-24 22:55:20 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 22:55:20 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 22:55:20 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 22:55:20 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 22:55:20 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 22:55:20 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 22:55:20 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 22:55:20 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver -2016-05-24 22:55:20 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver -2016-05-24 22:55:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 204 ms on localhost (1/2) -2016-05-24 22:55:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 183 ms on localhost (2/2) -2016-05-24 22:55:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 22:55:20 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at NaiveBayes.scala:383) finished in 0.223 s -2016-05-24 22:55:20 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:55:20 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:55:20 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-05-24 22:55:20 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:55:20 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-05-24 22:55:20 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-24 22:55:20 INFO MemoryStore:59 - ensureFreeSpace(2936) called with curMem=179376, maxMem=766075207 -2016-05-24 22:55:20 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:55:20 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=182312, maxMem=766075207 -2016-05-24 22:55:20 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.4 MB) -2016-05-24 22:55:20 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:49524 (size: 1567.0 B, free: 730.6 MB) -2016-05-24 22:55:20 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:55:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383) -2016-05-24 22:55:20 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 22:55:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:55:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:55:20 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 22:55:20 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-24 22:55:21 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:55:21 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-24 22:55:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-24 22:55:21 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-24 22:55:21 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-05-24 22:55:21 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1645 bytes result sent to driver -2016-05-24 22:55:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 39 ms on localhost (1/2) -2016-05-24 22:55:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 43 ms on localhost (2/2) -2016-05-24 22:55:21 INFO DAGScheduler:59 - ResultStage 1 (collect at NaiveBayes.scala:401) finished in 0.044 s -2016-05-24 22:55:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 22:55:21 INFO DAGScheduler:59 - Job 0 finished: collect at NaiveBayes.scala:401, took 0.416072 s -2016-05-24 22:55:21 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:39 -2016-05-24 22:55:21 INFO DAGScheduler:59 - Got job 1 (count at NaiveBayesLearning.scala:39) with 2 output partitions -2016-05-24 22:55:21 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at NaiveBayesLearning.scala:39) -2016-05-24 22:55:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:55:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:55:21 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-24 22:55:21 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=183879, maxMem=766075207 -2016-05-24 22:55:21 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-24 22:55:21 INFO MemoryStore:59 - ensureFreeSpace(2671) called with curMem=188775, maxMem=766075207 -2016-05-24 22:55:21 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-24 22:55:21 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:49524 (size: 2.6 KB, free: 730.6 MB) -2016-05-24 22:55:21 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:55:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:39) -2016-05-24 22:55:21 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 22:55:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 22:55:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 22:55:21 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-24 22:55:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-24 22:55:21 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-24 22:55:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-24 22:55:21 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 22:55:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 34 ms on localhost (1/2) -2016-05-24 22:55:21 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:49524 in memory (size: 1567.0 B, free: 730.6 MB) -2016-05-24 22:55:21 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 22:55:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 42 ms on localhost (2/2) -2016-05-24 22:55:21 INFO DAGScheduler:59 - ResultStage 2 (count at NaiveBayesLearning.scala:39) finished in 0.041 s -2016-05-24 22:55:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 22:55:21 INFO DAGScheduler:59 - Job 1 finished: count at NaiveBayesLearning.scala:39, took 0.083389 s -2016-05-24 22:55:21 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:49524 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-24 22:55:21 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:39 -2016-05-24 22:55:21 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-24 22:55:21 INFO DAGScheduler:59 - Got job 2 (count at NaiveBayesLearning.scala:39) with 2 output partitions -2016-05-24 22:55:21 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at NaiveBayesLearning.scala:39) -2016-05-24 22:55:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:55:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:55:21 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-24 22:55:21 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=179147, maxMem=766075207 -2016-05-24 22:55:21 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-24 22:55:21 INFO MemoryStore:59 - ensureFreeSpace(2117) called with curMem=182747, maxMem=766075207 -2016-05-24 22:55:21 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:55:21 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:49524 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:55:21 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:55:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-24 22:55:21 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 22:55:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 22:55:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 22:55:21 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-24 22:55:21 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-24 22:55:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-24 22:55:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-24 22:55:21 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 2082 bytes result sent to driver -2016-05-24 22:55:21 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 22:55:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 16 ms on localhost (1/2) -2016-05-24 22:55:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 18 ms on localhost (2/2) -2016-05-24 22:55:21 INFO DAGScheduler:59 - ResultStage 3 (count at NaiveBayesLearning.scala:39) finished in 0.019 s -2016-05-24 22:55:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 22:55:21 INFO DAGScheduler:59 - Job 2 finished: count at NaiveBayesLearning.scala:39, took 0.032097 s -2016-05-24 22:55:21 INFO SparkContext:59 - Starting job: saveAsTextFile at NaiveBayes.scala:202 -2016-05-24 22:55:21 INFO DAGScheduler:59 - Got job 3 (saveAsTextFile at NaiveBayes.scala:202) with 1 output partitions -2016-05-24 22:55:21 INFO DAGScheduler:59 - Final stage: ResultStage 4(saveAsTextFile at NaiveBayes.scala:202) -2016-05-24 22:55:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:55:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:55:21 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202), which has no missing parents -2016-05-24 22:55:21 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=184864, maxMem=766075207 -2016-05-24 22:55:21 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 124.0 KB, free 730.3 MB) -2016-05-24 22:55:21 INFO MemoryStore:59 - ensureFreeSpace(42591) called with curMem=311856, maxMem=766075207 -2016-05-24 22:55:21 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.2 MB) -2016-05-24 22:55:21 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:49524 (size: 41.6 KB, free: 730.5 MB) -2016-05-24 22:55:21 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:55:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202) -2016-05-24 22:55:21 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-05-24 22:55:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-24 22:55:21 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-24 22:55:21 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242255_0004_m_000000_8' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524225521208/result/metadata/_temporary/0/task_201605242255_0004_m_000000 -2016-05-24 22:55:21 INFO SparkHadoopMapRedUtil:59 - attempt_201605242255_0004_m_000000_8: Committed -2016-05-24 22:55:21 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 1864 bytes result sent to driver -2016-05-24 22:55:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 249 ms on localhost (1/1) -2016-05-24 22:55:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 22:55:21 INFO DAGScheduler:59 - ResultStage 4 (saveAsTextFile at NaiveBayes.scala:202) finished in 0.251 s -2016-05-24 22:55:21 INFO DAGScheduler:59 - Job 3 finished: saveAsTextFile at NaiveBayes.scala:202, took 0.341859 s -2016-05-24 22:55:23 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:49524 in memory (size: 41.6 KB, free: 730.6 MB) -2016-05-24 22:55:23 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 22:55:23 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:49524 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:55:23 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 22:55:23 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:49524 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-24 22:55:23 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:55:23 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:55:23 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:206 -2016-05-24 22:55:23 INFO DAGScheduler:59 - Got job 4 (parquet at NaiveBayes.scala:206) with 1 output partitions -2016-05-24 22:55:23 INFO DAGScheduler:59 - Final stage: ResultStage 5(parquet at NaiveBayes.scala:206) -2016-05-24 22:55:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:55:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:55:23 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205), which has no missing parents -2016-05-24 22:55:23 INFO MemoryStore:59 - ensureFreeSpace(66936) called with curMem=171580, maxMem=766075207 -2016-05-24 22:55:23 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 65.4 KB, free 730.4 MB) -2016-05-24 22:55:23 INFO MemoryStore:59 - ensureFreeSpace(23367) called with curMem=238516, maxMem=766075207 -2016-05-24 22:55:23 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 22.8 KB, free 730.3 MB) -2016-05-24 22:55:23 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:49524 (size: 22.8 KB, free: 730.6 MB) -2016-05-24 22:55:23 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:55:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 5 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205) -2016-05-24 22:55:23 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 1 tasks -2016-05-24 22:55:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2506 bytes) -2016-05-24 22:55:23 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-24 22:55:23 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:55:23 INFO CodecConfig:151 - Compression: GZIP -2016-05-24 22:55:23 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-24 22:55:23 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-24 22:55:23 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-24 22:55:23 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-24 22:55:23 INFO ParquetOutputFormat:151 - Validation is off -2016-05-24 22:55:23 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-24 22:55:24 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-24 22:55:24 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 231 -2016-05-24 22:55:24 INFO ColumnChunkPageWriteStore:151 - written 77B for [labels, bag, array] DOUBLE: 3 values, 37B raw, 36B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-24 22:55:24 INFO ColumnChunkPageWriteStore:151 - written 74B for [pi, bag, array] DOUBLE: 3 values, 16B raw, 33B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 2 entries, 16B raw, 2B comp} -2016-05-24 22:55:24 INFO ColumnChunkPageWriteStore:151 - written 83B for [theta, bag, array, bag, array] DOUBLE: 9 values, 23B raw, 42B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 6 entries, 48B raw, 6B comp} -2016-05-24 22:55:24 INFO ColumnChunkPageWriteStore:151 - written 86B for [modelType] BINARY: 1 values, 21B raw, 39B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-24 22:55:24 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242255_0005_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524225521208/result/data/_temporary/0/task_201605242255_0005_m_000000 -2016-05-24 22:55:24 INFO SparkHadoopMapRedUtil:59 - attempt_201605242255_0005_m_000000_0: Committed -2016-05-24 22:55:24 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 915 bytes result sent to driver -2016-05-24 22:55:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 827 ms on localhost (1/1) -2016-05-24 22:55:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 22:55:24 INFO DAGScheduler:59 - ResultStage 5 (parquet at NaiveBayes.scala:206) finished in 0.831 s -2016-05-24 22:55:24 INFO DAGScheduler:59 - Job 4 finished: parquet at NaiveBayes.scala:206, took 0.875876 s -2016-05-24 22:55:24 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:55:24 INFO DefaultWriterContainer:59 - Job job_201605242255_0000 committed. -2016-05-24 22:55:24 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524225521208/result/data on driver -2016-05-24 22:55:24 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524225521208/result/data on driver -2016-05-24 22:55:24 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=261883, maxMem=766075207 -2016-05-24 22:55:24 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-24 22:55:24 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=492587, maxMem=766075207 -2016-05-24 22:55:24 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-24 22:55:24 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:49524 (size: 19.3 KB, free: 730.5 MB) -2016-05-24 22:55:24 INFO SparkContext:59 - Created broadcast 7 from textFile at modelSaveLoad.scala:129 -2016-05-24 22:55:25 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:55:25 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-24 22:55:25 INFO DAGScheduler:59 - Got job 5 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-24 22:55:25 INFO DAGScheduler:59 - Final stage: ResultStage 6(first at modelSaveLoad.scala:129) -2016-05-24 22:55:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:55:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:55:25 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-24 22:55:25 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=512375, maxMem=766075207 -2016-05-24 22:55:25 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 22:55:25 INFO MemoryStore:59 - ensureFreeSpace(1891) called with curMem=515591, maxMem=766075207 -2016-05-24 22:55:25 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 1891.0 B, free 730.1 MB) -2016-05-24 22:55:25 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:49524 (size: 1891.0 B, free: 730.5 MB) -2016-05-24 22:55:25 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:55:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 6 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129) -2016-05-24 22:55:25 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 1 tasks -2016-05-24 22:55:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 10, localhost, PROCESS_LOCAL, 2239 bytes) -2016-05-24 22:55:25 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 10) -2016-05-24 22:55:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524225521208/result/metadata/part-00000:0+56 -2016-05-24 22:55:25 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 10). 2159 bytes result sent to driver -2016-05-24 22:55:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 10) in 16 ms on localhost (1/1) -2016-05-24 22:55:25 INFO DAGScheduler:59 - ResultStage 6 (first at modelSaveLoad.scala:129) finished in 0.016 s -2016-05-24 22:55:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 22:55:25 INFO DAGScheduler:59 - Job 5 finished: first at modelSaveLoad.scala:129, took 0.033215 s -2016-05-24 22:55:25 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524225521208/result/data on driver -2016-05-24 22:55:25 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:213 -2016-05-24 22:55:25 INFO DAGScheduler:59 - Got job 6 (parquet at NaiveBayes.scala:213) with 4 output partitions -2016-05-24 22:55:25 INFO DAGScheduler:59 - Final stage: ResultStage 7(parquet at NaiveBayes.scala:213) -2016-05-24 22:55:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:55:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:55:25 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213), which has no missing parents -2016-05-24 22:55:25 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=517482, maxMem=766075207 -2016-05-24 22:55:25 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-24 22:55:25 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=580482, maxMem=766075207 -2016-05-24 22:55:25 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.0 MB) -2016-05-24 22:55:25 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:49524 (size: 20.6 KB, free: 730.5 MB) -2016-05-24 22:55:25 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:55:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 7 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213) -2016-05-24 22:55:25 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 4 tasks -2016-05-24 22:55:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 11, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:55:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 12, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:55:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:55:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-24 22:55:25 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 12) -2016-05-24 22:55:25 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 11) -2016-05-24 22:55:25 INFO Executor:59 - Running task 2.0 in stage 7.0 (TID 13) -2016-05-24 22:55:25 INFO Executor:59 - Running task 3.0 in stage 7.0 (TID 14) -2016-05-24 22:55:25 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:55:25 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:55:25 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 11). 936 bytes result sent to driver -2016-05-24 22:55:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 11) in 62 ms on localhost (1/4) -2016-05-24 22:55:25 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:55:25 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 12). 936 bytes result sent to driver -2016-05-24 22:55:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 12) in 66 ms on localhost (2/4) -2016-05-24 22:55:25 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:55:25 INFO Executor:59 - Finished task 2.0 in stage 7.0 (TID 13). 936 bytes result sent to driver -2016-05-24 22:55:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 7.0 (TID 13) in 69 ms on localhost (3/4) -2016-05-24 22:55:25 INFO Executor:59 - Finished task 3.0 in stage 7.0 (TID 14). 1876 bytes result sent to driver -2016-05-24 22:55:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 7.0 (TID 14) in 78 ms on localhost (4/4) -2016-05-24 22:55:25 INFO DAGScheduler:59 - ResultStage 7 (parquet at NaiveBayes.scala:213) finished in 0.081 s -2016-05-24 22:55:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 22:55:25 INFO DAGScheduler:59 - Job 6 finished: parquet at NaiveBayes.scala:213, took 0.107038 s -2016-05-24 22:55:25 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=601555, maxMem=766075207 -2016-05-24 22:55:25 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 86.4 KB, free 729.9 MB) -2016-05-24 22:55:25 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=690067, maxMem=766075207 -2016-05-24 22:55:25 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-24 22:55:25 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:49524 (size: 19.3 KB, free: 730.5 MB) -2016-05-24 22:55:25 INFO SparkContext:59 - Created broadcast 10 from take at NaiveBayes.scala:216 -2016-05-24 22:55:25 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-24 22:55:25 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524225521208/result/data/part-r-00000-d46e03c9-7816-4fdc-bdec-a99d0d195d23.gz.parquet -2016-05-24 22:55:25 INFO SparkContext:59 - Starting job: take at NaiveBayes.scala:216 -2016-05-24 22:55:25 INFO DAGScheduler:59 - Got job 7 (take at NaiveBayes.scala:216) with 1 output partitions -2016-05-24 22:55:25 INFO DAGScheduler:59 - Final stage: ResultStage 8(take at NaiveBayes.scala:216) -2016-05-24 22:55:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:55:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:55:25 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216), which has no missing parents -2016-05-24 22:55:25 INFO MemoryStore:59 - ensureFreeSpace(4280) called with curMem=709855, maxMem=766075207 -2016-05-24 22:55:25 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.2 KB, free 729.9 MB) -2016-05-24 22:55:25 INFO MemoryStore:59 - ensureFreeSpace(2466) called with curMem=714135, maxMem=766075207 -2016-05-24 22:55:25 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.4 KB, free 729.9 MB) -2016-05-24 22:55:25 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:49524 (size: 2.4 KB, free: 730.5 MB) -2016-05-24 22:55:25 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:55:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216) -2016-05-24 22:55:25 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-05-24 22:55:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:55:25 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-24 22:55:25 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524225521208/result/data/part-r-00000-d46e03c9-7816-4fdc-bdec-a99d0d195d23.gz.parquet start: 0 end: 1477 length: 1477 hosts: []} -2016-05-24 22:55:25 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-24 22:55:25 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group labels (LIST) { - repeated group bag { - optional double array; - } - } - optional group pi (LIST) { - repeated group bag { - optional double array; - } - } - optional group theta (LIST) { - repeated group bag { - optional group array (LIST) { - repeated group bag { - optional double array; - } - } - } - } - optional binary modelType (UTF8); -} - -Catalyst form: -StructType(StructField(labels,ArrayType(DoubleType,true),true), StructField(pi,ArrayType(DoubleType,true),true), StructField(theta,ArrayType(ArrayType(DoubleType,true),true),true), StructField(modelType,StringType,true)) - -2016-05-24 22:55:25 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-24 22:55:25 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-24 22:55:25 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-24 22:55:25 INFO InternalParquetRecordReader:151 - block read in memory in 9 ms. row count = 1 -2016-05-24 22:55:25 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2841 bytes result sent to driver -2016-05-24 22:55:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 94 ms on localhost (1/1) -2016-05-24 22:55:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 22:55:25 INFO DAGScheduler:59 - ResultStage 8 (take at NaiveBayes.scala:216) finished in 0.095 s -2016-05-24 22:55:25 INFO DAGScheduler:59 - Job 7 finished: take at NaiveBayes.scala:216, took 0.124143 s -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 22:55:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 22:55:25 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 22:55:25 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 22:55:25 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 22:55:25 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 22:55:25 INFO BlockManager:59 - BlockManager stopped -2016-05-24 22:55:25 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 22:55:25 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 22:55:25 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 22:55:25 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 22:55:25 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-e69f6567-1807-43cc-a9de-3cde23135eb1 -2016-05-24 22:55:25 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-24 22:58:36 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 22:58:37 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 22:58:37 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 22:58:37 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 22:58:37 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 22:58:38 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 22:58:38 INFO Remoting:74 - Starting remoting -2016-05-24 22:58:38 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:49547] -2016-05-24 22:58:38 INFO Utils:59 - Successfully started service 'sparkDriver' on port 49547. -2016-05-24 22:58:38 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 22:58:38 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 22:58:38 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-e18ba592-af93-4a84-ac6e-10d7ff16f947 -2016-05-24 22:58:38 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 22:58:38 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-2b2418f9-cf17-4d29-bfd0-cd0d35c41ec3\httpd-9a2b3878-a50a-470b-81f6-d32a6104278a -2016-05-24 22:58:38 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 22:58:39 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:58:39 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:49548 -2016-05-24 22:58:39 INFO Utils:59 - Successfully started service 'HTTP file server' on port 49548. -2016-05-24 22:58:39 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 22:58:39 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 22:58:39 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 22:58:39 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 22:58:39 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 22:58:39 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 22:58:39 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 22:58:39 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 49567. -2016-05-24 22:58:39 INFO NettyBlockTransferService:59 - Server created on 49567 -2016-05-24 22:58:39 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 22:58:39 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:49567 with 730.6 MB RAM, BlockManagerId(driver, localhost, 49567) -2016-05-24 22:58:39 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 22:58:40 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 22:58:40 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 22:58:40 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 22:58:40 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 22:58:40 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:49567 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 22:58:40 INFO SparkContext:59 - Created broadcast 0 from textFile at NaiveBayesLearning.scala:26 -2016-05-24 22:58:41 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 22:58:41 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:58:41 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-24 22:58:41 INFO DAGScheduler:59 - Registering RDD 5 (map at NaiveBayes.scala:383) -2016-05-24 22:58:41 INFO DAGScheduler:59 - Got job 0 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-24 22:58:41 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at NaiveBayes.scala:401) -2016-05-24 22:58:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-05-24 22:58:42 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-05-24 22:58:42 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-24 22:58:42 INFO MemoryStore:59 - ensureFreeSpace(5104) called with curMem=171580, maxMem=766075207 -2016-05-24 22:58:42 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-24 22:58:42 INFO MemoryStore:59 - ensureFreeSpace(2692) called with curMem=176684, maxMem=766075207 -2016-05-24 22:58:42 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-24 22:58:42 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:49567 (size: 2.6 KB, free: 730.6 MB) -2016-05-24 22:58:42 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:58:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383) -2016-05-24 22:58:42 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 22:58:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-24 22:58:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-24 22:58:42 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 22:58:42 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 22:58:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-24 22:58:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-24 22:58:42 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 22:58:42 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 22:58:42 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 22:58:42 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 22:58:42 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 22:58:42 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 22:58:42 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 22:58:42 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver -2016-05-24 22:58:42 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver -2016-05-24 22:58:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 150 ms on localhost (1/2) -2016-05-24 22:58:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 172 ms on localhost (2/2) -2016-05-24 22:58:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 22:58:42 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at NaiveBayes.scala:383) finished in 0.187 s -2016-05-24 22:58:42 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 22:58:42 INFO DAGScheduler:59 - running: Set() -2016-05-24 22:58:42 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-05-24 22:58:42 INFO DAGScheduler:59 - failed: Set() -2016-05-24 22:58:42 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-05-24 22:58:42 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-24 22:58:42 INFO MemoryStore:59 - ensureFreeSpace(2936) called with curMem=179376, maxMem=766075207 -2016-05-24 22:58:42 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 22:58:42 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=182312, maxMem=766075207 -2016-05-24 22:58:42 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.4 MB) -2016-05-24 22:58:42 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:49567 (size: 1567.0 B, free: 730.6 MB) -2016-05-24 22:58:42 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:58:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383) -2016-05-24 22:58:42 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 22:58:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:58:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 22:58:42 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-24 22:58:42 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 22:58:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 22:58:42 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-24 22:58:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-24 22:58:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-24 22:58:42 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-05-24 22:58:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 31 ms on localhost (1/2) -2016-05-24 22:58:42 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1645 bytes result sent to driver -2016-05-24 22:58:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 38 ms on localhost (2/2) -2016-05-24 22:58:42 INFO DAGScheduler:59 - ResultStage 1 (collect at NaiveBayes.scala:401) finished in 0.039 s -2016-05-24 22:58:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 22:58:42 INFO DAGScheduler:59 - Job 0 finished: collect at NaiveBayes.scala:401, took 0.361721 s -2016-05-24 22:58:42 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:39 -2016-05-24 22:58:42 INFO DAGScheduler:59 - Got job 1 (count at NaiveBayesLearning.scala:39) with 2 output partitions -2016-05-24 22:58:42 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at NaiveBayesLearning.scala:39) -2016-05-24 22:58:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:58:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:58:42 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-24 22:58:42 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=183879, maxMem=766075207 -2016-05-24 22:58:42 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-24 22:58:42 INFO MemoryStore:59 - ensureFreeSpace(2671) called with curMem=188775, maxMem=766075207 -2016-05-24 22:58:42 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-24 22:58:42 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:49567 (size: 2.6 KB, free: 730.6 MB) -2016-05-24 22:58:42 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:58:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:39) -2016-05-24 22:58:42 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 22:58:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 22:58:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 22:58:42 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-24 22:58:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-24 22:58:42 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-24 22:58:42 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:49567 in memory (size: 1567.0 B, free: 730.6 MB) -2016-05-24 22:58:42 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 22:58:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-24 22:58:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 31 ms on localhost (1/2) -2016-05-24 22:58:42 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:49567 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-24 22:58:42 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-24 22:58:42 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 22:58:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 42 ms on localhost (2/2) -2016-05-24 22:58:42 INFO DAGScheduler:59 - ResultStage 2 (count at NaiveBayesLearning.scala:39) finished in 0.044 s -2016-05-24 22:58:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 22:58:42 INFO DAGScheduler:59 - Job 1 finished: count at NaiveBayesLearning.scala:39, took 0.099396 s -2016-05-24 22:58:42 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:39 -2016-05-24 22:58:42 INFO DAGScheduler:59 - Got job 2 (count at NaiveBayesLearning.scala:39) with 2 output partitions -2016-05-24 22:58:42 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at NaiveBayesLearning.scala:39) -2016-05-24 22:58:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:58:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:58:42 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-24 22:58:42 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=179147, maxMem=766075207 -2016-05-24 22:58:42 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-24 22:58:42 INFO MemoryStore:59 - ensureFreeSpace(2117) called with curMem=182747, maxMem=766075207 -2016-05-24 22:58:42 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:58:42 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:49567 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:58:42 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:58:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-24 22:58:42 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 22:58:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 22:58:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 22:58:42 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-24 22:58:42 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-24 22:58:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-24 22:58:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-24 22:58:42 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 22:58:42 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 2082 bytes result sent to driver -2016-05-24 22:58:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 19 ms on localhost (1/2) -2016-05-24 22:58:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 17 ms on localhost (2/2) -2016-05-24 22:58:42 INFO DAGScheduler:59 - ResultStage 3 (count at NaiveBayesLearning.scala:39) finished in 0.019 s -2016-05-24 22:58:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 22:58:42 INFO DAGScheduler:59 - Job 2 finished: count at NaiveBayesLearning.scala:39, took 0.032233 s -2016-05-24 22:58:42 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:43 -2016-05-24 22:58:42 INFO DAGScheduler:59 - Got job 3 (count at NaiveBayesLearning.scala:43) with 2 output partitions -2016-05-24 22:58:42 INFO DAGScheduler:59 - Final stage: ResultStage 4(count at NaiveBayesLearning.scala:43) -2016-05-24 22:58:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:58:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:58:42 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-24 22:58:42 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=184864, maxMem=766075207 -2016-05-24 22:58:42 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-24 22:58:42 INFO MemoryStore:59 - ensureFreeSpace(2113) called with curMem=188464, maxMem=766075207 -2016-05-24 22:58:42 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:58:42 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:49567 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:58:42 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:58:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-24 22:58:42 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 22:58:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 22:58:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 22:58:42 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-24 22:58:42 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-24 22:58:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-24 22:58:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-24 22:58:42 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2082 bytes result sent to driver -2016-05-24 22:58:42 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2082 bytes result sent to driver -2016-05-24 22:58:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 16 ms on localhost (1/2) -2016-05-24 22:58:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 16 ms on localhost (2/2) -2016-05-24 22:58:42 INFO DAGScheduler:59 - ResultStage 4 (count at NaiveBayesLearning.scala:43) finished in 0.017 s -2016-05-24 22:58:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 22:58:42 INFO DAGScheduler:59 - Job 3 finished: count at NaiveBayesLearning.scala:43, took 0.032987 s -2016-05-24 22:58:42 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:44 -2016-05-24 22:58:42 INFO DAGScheduler:59 - Got job 4 (count at NaiveBayesLearning.scala:44) with 2 output partitions -2016-05-24 22:58:42 INFO DAGScheduler:59 - Final stage: ResultStage 5(count at NaiveBayesLearning.scala:44) -2016-05-24 22:58:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:58:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:58:42 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-24 22:58:42 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=190577, maxMem=766075207 -2016-05-24 22:58:42 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-24 22:58:42 INFO MemoryStore:59 - ensureFreeSpace(2117) called with curMem=194177, maxMem=766075207 -2016-05-24 22:58:42 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 22:58:42 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:49567 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:58:42 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:58:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-24 22:58:42 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 22:58:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 22:58:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 22:58:42 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-05-24 22:58:42 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-05-24 22:58:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-24 22:58:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-24 22:58:42 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 2082 bytes result sent to driver -2016-05-24 22:58:42 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 2082 bytes result sent to driver -2016-05-24 22:58:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 19 ms on localhost (1/2) -2016-05-24 22:58:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 21 ms on localhost (2/2) -2016-05-24 22:58:42 INFO DAGScheduler:59 - ResultStage 5 (count at NaiveBayesLearning.scala:44) finished in 0.022 s -2016-05-24 22:58:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 22:58:42 INFO DAGScheduler:59 - Job 4 finished: count at NaiveBayesLearning.scala:44, took 0.035274 s -2016-05-24 22:58:42 INFO SparkContext:59 - Starting job: saveAsTextFile at NaiveBayes.scala:202 -2016-05-24 22:58:42 INFO DAGScheduler:59 - Got job 5 (saveAsTextFile at NaiveBayes.scala:202) with 1 output partitions -2016-05-24 22:58:42 INFO DAGScheduler:59 - Final stage: ResultStage 6(saveAsTextFile at NaiveBayes.scala:202) -2016-05-24 22:58:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:58:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:58:42 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202), which has no missing parents -2016-05-24 22:58:43 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=196294, maxMem=766075207 -2016-05-24 22:58:43 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 124.0 KB, free 730.3 MB) -2016-05-24 22:58:43 INFO MemoryStore:59 - ensureFreeSpace(42591) called with curMem=323286, maxMem=766075207 -2016-05-24 22:58:43 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.2 MB) -2016-05-24 22:58:43 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:49567 (size: 41.6 KB, free: 730.5 MB) -2016-05-24 22:58:43 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:58:43 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 6 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202) -2016-05-24 22:58:43 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 1 tasks -2016-05-24 22:58:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-24 22:58:43 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-05-24 22:58:43 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242258_0006_m_000000_12' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524225842577/result/metadata/_temporary/0/task_201605242258_0006_m_000000 -2016-05-24 22:58:43 INFO SparkHadoopMapRedUtil:59 - attempt_201605242258_0006_m_000000_12: Committed -2016-05-24 22:58:43 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 1864 bytes result sent to driver -2016-05-24 22:58:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 213 ms on localhost (1/1) -2016-05-24 22:58:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 22:58:43 INFO DAGScheduler:59 - ResultStage 6 (saveAsTextFile at NaiveBayes.scala:202) finished in 0.215 s -2016-05-24 22:58:43 INFO DAGScheduler:59 - Job 5 finished: saveAsTextFile at NaiveBayes.scala:202, took 0.301170 s -2016-05-24 22:58:44 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:49567 in memory (size: 41.6 KB, free: 730.6 MB) -2016-05-24 22:58:44 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 22:58:44 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:49567 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:58:44 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 22:58:44 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:49567 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:58:44 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 22:58:44 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:49567 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 22:58:44 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 22:58:44 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:58:44 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:58:44 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:206 -2016-05-24 22:58:44 INFO DAGScheduler:59 - Got job 6 (parquet at NaiveBayes.scala:206) with 1 output partitions -2016-05-24 22:58:44 INFO DAGScheduler:59 - Final stage: ResultStage 7(parquet at NaiveBayes.scala:206) -2016-05-24 22:58:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:58:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:58:44 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205), which has no missing parents -2016-05-24 22:58:44 INFO MemoryStore:59 - ensureFreeSpace(66936) called with curMem=179147, maxMem=766075207 -2016-05-24 22:58:44 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 65.4 KB, free 730.4 MB) -2016-05-24 22:58:44 INFO MemoryStore:59 - ensureFreeSpace(23367) called with curMem=246083, maxMem=766075207 -2016-05-24 22:58:44 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 22.8 KB, free 730.3 MB) -2016-05-24 22:58:44 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:49567 (size: 22.8 KB, free: 730.5 MB) -2016-05-24 22:58:44 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:58:44 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205) -2016-05-24 22:58:44 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-24 22:58:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2506 bytes) -2016-05-24 22:58:44 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 22:58:45 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 22:58:45 INFO CodecConfig:151 - Compression: GZIP -2016-05-24 22:58:45 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-24 22:58:45 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-24 22:58:45 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-24 22:58:45 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-24 22:58:45 INFO ParquetOutputFormat:151 - Validation is off -2016-05-24 22:58:45 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-24 22:58:45 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-24 22:58:45 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 231 -2016-05-24 22:58:45 INFO ColumnChunkPageWriteStore:151 - written 77B for [labels, bag, array] DOUBLE: 3 values, 37B raw, 36B comp, 1 pages, encodings: [RLE, PLAIN] -2016-05-24 22:58:45 INFO ColumnChunkPageWriteStore:151 - written 74B for [pi, bag, array] DOUBLE: 3 values, 16B raw, 33B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 2 entries, 16B raw, 2B comp} -2016-05-24 22:58:45 INFO ColumnChunkPageWriteStore:151 - written 83B for [theta, bag, array, bag, array] DOUBLE: 9 values, 23B raw, 42B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 6 entries, 48B raw, 6B comp} -2016-05-24 22:58:45 INFO ColumnChunkPageWriteStore:151 - written 86B for [modelType] BINARY: 1 values, 21B raw, 39B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-24 22:58:45 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242258_0007_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524225842577/result/data/_temporary/0/task_201605242258_0007_m_000000 -2016-05-24 22:58:45 INFO SparkHadoopMapRedUtil:59 - attempt_201605242258_0007_m_000000_0: Committed -2016-05-24 22:58:45 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 915 bytes result sent to driver -2016-05-24 22:58:45 INFO DAGScheduler:59 - ResultStage 7 (parquet at NaiveBayes.scala:206) finished in 0.454 s -2016-05-24 22:58:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 454 ms on localhost (1/1) -2016-05-24 22:58:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 22:58:45 INFO DAGScheduler:59 - Job 6 finished: parquet at NaiveBayes.scala:206, took 0.492842 s -2016-05-24 22:58:45 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:58:45 INFO DefaultWriterContainer:59 - Job job_201605242258_0000 committed. -2016-05-24 22:58:45 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524225842577/result/data on driver -2016-05-24 22:58:45 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524225842577/result/data on driver -2016-05-24 22:58:45 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=269450, maxMem=766075207 -2016-05-24 22:58:45 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-24 22:58:45 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=500154, maxMem=766075207 -2016-05-24 22:58:45 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-24 22:58:45 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:49567 (size: 19.3 KB, free: 730.5 MB) -2016-05-24 22:58:45 INFO SparkContext:59 - Created broadcast 9 from textFile at modelSaveLoad.scala:129 -2016-05-24 22:58:45 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 22:58:45 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-24 22:58:45 INFO DAGScheduler:59 - Got job 7 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-24 22:58:45 INFO DAGScheduler:59 - Final stage: ResultStage 8(first at modelSaveLoad.scala:129) -2016-05-24 22:58:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:58:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:58:45 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-24 22:58:45 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=519942, maxMem=766075207 -2016-05-24 22:58:45 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 22:58:45 INFO MemoryStore:59 - ensureFreeSpace(1893) called with curMem=523158, maxMem=766075207 -2016-05-24 22:58:45 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 1893.0 B, free 730.1 MB) -2016-05-24 22:58:45 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:49567 (size: 1893.0 B, free: 730.5 MB) -2016-05-24 22:58:45 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:58:45 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129) -2016-05-24 22:58:45 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-05-24 22:58:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 14, localhost, PROCESS_LOCAL, 2239 bytes) -2016-05-24 22:58:45 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 14) -2016-05-24 22:58:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524225842577/result/metadata/part-00000:0+56 -2016-05-24 22:58:45 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 14). 2159 bytes result sent to driver -2016-05-24 22:58:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 14) in 13 ms on localhost (1/1) -2016-05-24 22:58:45 INFO DAGScheduler:59 - ResultStage 8 (first at modelSaveLoad.scala:129) finished in 0.014 s -2016-05-24 22:58:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 22:58:45 INFO DAGScheduler:59 - Job 7 finished: first at modelSaveLoad.scala:129, took 0.027429 s -2016-05-24 22:58:45 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524225842577/result/data on driver -2016-05-24 22:58:46 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:213 -2016-05-24 22:58:46 INFO DAGScheduler:59 - Got job 8 (parquet at NaiveBayes.scala:213) with 4 output partitions -2016-05-24 22:58:46 INFO DAGScheduler:59 - Final stage: ResultStage 9(parquet at NaiveBayes.scala:213) -2016-05-24 22:58:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:58:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:58:46 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213), which has no missing parents -2016-05-24 22:58:46 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=525051, maxMem=766075207 -2016-05-24 22:58:46 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-24 22:58:46 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=588051, maxMem=766075207 -2016-05-24 22:58:46 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.0 MB) -2016-05-24 22:58:46 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:49567 (size: 20.6 KB, free: 730.5 MB) -2016-05-24 22:58:46 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:58:46 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 9 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213) -2016-05-24 22:58:46 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 4 tasks -2016-05-24 22:58:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:58:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 16, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:58:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 22:58:46 INFO TaskSetManager:59 - Starting task 3.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-24 22:58:46 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 16) -2016-05-24 22:58:46 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 15) -2016-05-24 22:58:46 INFO Executor:59 - Running task 2.0 in stage 9.0 (TID 17) -2016-05-24 22:58:46 INFO Executor:59 - Running task 3.0 in stage 9.0 (TID 18) -2016-05-24 22:58:46 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:58:46 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 15). 936 bytes result sent to driver -2016-05-24 22:58:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 15) in 63 ms on localhost (1/4) -2016-05-24 22:58:46 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:58:46 INFO Executor:59 - Finished task 2.0 in stage 9.0 (TID 17). 936 bytes result sent to driver -2016-05-24 22:58:46 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:58:46 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 16). 936 bytes result sent to driver -2016-05-24 22:58:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 9.0 (TID 17) in 75 ms on localhost (2/4) -2016-05-24 22:58:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 16) in 78 ms on localhost (3/4) -2016-05-24 22:58:46 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 22:58:46 INFO Executor:59 - Finished task 3.0 in stage 9.0 (TID 18). 1876 bytes result sent to driver -2016-05-24 22:58:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 9.0 (TID 18) in 110 ms on localhost (4/4) -2016-05-24 22:58:46 INFO DAGScheduler:59 - ResultStage 9 (parquet at NaiveBayes.scala:213) finished in 0.114 s -2016-05-24 22:58:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 22:58:46 INFO DAGScheduler:59 - Job 8 finished: parquet at NaiveBayes.scala:213, took 0.146733 s -2016-05-24 22:58:46 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=609124, maxMem=766075207 -2016-05-24 22:58:46 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 86.4 KB, free 729.9 MB) -2016-05-24 22:58:46 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=697636, maxMem=766075207 -2016-05-24 22:58:46 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-24 22:58:46 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:49567 (size: 19.3 KB, free: 730.5 MB) -2016-05-24 22:58:46 INFO SparkContext:59 - Created broadcast 12 from take at NaiveBayes.scala:216 -2016-05-24 22:58:46 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-24 22:58:46 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524225842577/result/data/part-r-00000-4e2ebbbf-1581-4eac-93cc-498b531db03c.gz.parquet -2016-05-24 22:58:46 INFO SparkContext:59 - Starting job: take at NaiveBayes.scala:216 -2016-05-24 22:58:46 INFO DAGScheduler:59 - Got job 9 (take at NaiveBayes.scala:216) with 1 output partitions -2016-05-24 22:58:46 INFO DAGScheduler:59 - Final stage: ResultStage 10(take at NaiveBayes.scala:216) -2016-05-24 22:58:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 22:58:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 22:58:46 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216), which has no missing parents -2016-05-24 22:58:46 INFO MemoryStore:59 - ensureFreeSpace(4280) called with curMem=717424, maxMem=766075207 -2016-05-24 22:58:46 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 4.2 KB, free 729.9 MB) -2016-05-24 22:58:46 INFO MemoryStore:59 - ensureFreeSpace(2466) called with curMem=721704, maxMem=766075207 -2016-05-24 22:58:46 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.4 KB, free 729.9 MB) -2016-05-24 22:58:46 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:49567 (size: 2.4 KB, free: 730.5 MB) -2016-05-24 22:58:46 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-24 22:58:46 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216) -2016-05-24 22:58:46 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-24 22:58:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-24 22:58:46 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-24 22:58:46 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524225842577/result/data/part-r-00000-4e2ebbbf-1581-4eac-93cc-498b531db03c.gz.parquet start: 0 end: 1477 length: 1477 hosts: []} -2016-05-24 22:58:46 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-24 22:58:46 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group labels (LIST) { - repeated group bag { - optional double array; - } - } - optional group pi (LIST) { - repeated group bag { - optional double array; - } - } - optional group theta (LIST) { - repeated group bag { - optional group array (LIST) { - repeated group bag { - optional double array; - } - } - } - } - optional binary modelType (UTF8); -} - -Catalyst form: -StructType(StructField(labels,ArrayType(DoubleType,true),true), StructField(pi,ArrayType(DoubleType,true),true), StructField(theta,ArrayType(ArrayType(DoubleType,true),true),true), StructField(modelType,StringType,true)) - -2016-05-24 22:58:46 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-24 22:58:46 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-24 22:58:46 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-24 22:58:46 INFO InternalParquetRecordReader:151 - block read in memory in 13 ms. row count = 1 -2016-05-24 22:58:46 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2841 bytes result sent to driver -2016-05-24 22:58:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 145 ms on localhost (1/1) -2016-05-24 22:58:46 INFO DAGScheduler:59 - ResultStage 10 (take at NaiveBayes.scala:216) finished in 0.146 s -2016-05-24 22:58:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 22:58:46 INFO DAGScheduler:59 - Job 9 finished: take at NaiveBayes.scala:216, took 0.168458 s -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 22:58:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 22:58:46 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 22:58:46 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 22:58:46 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 22:58:46 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 22:58:46 INFO BlockManager:59 - BlockManager stopped -2016-05-24 22:58:46 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 22:58:46 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 22:58:46 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 22:58:46 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 22:58:46 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-2b2418f9-cf17-4d29-bfd0-cd0d35c41ec3 -2016-05-24 23:00:35 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-24 23:00:35 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-24 23:00:36 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-24 23:00:36 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-24 23:00:36 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-24 23:00:36 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-24 23:00:37 INFO Remoting:74 - Starting remoting -2016-05-24 23:00:37 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.184:49687] -2016-05-24 23:00:37 INFO Utils:59 - Successfully started service 'sparkDriver' on port 49687. -2016-05-24 23:00:37 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-24 23:00:37 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-24 23:00:37 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-5159f91d-aad8-46b9-afa6-38c81bd9edf8 -2016-05-24 23:00:37 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-24 23:00:37 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-b3bc0f29-20cd-4255-9375-212b2354359b\httpd-53995ae1-9cda-4c91-92ec-fcdfe34aea64 -2016-05-24 23:00:37 INFO HttpServer:59 - Starting HTTP Server -2016-05-24 23:00:37 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 23:00:37 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:49688 -2016-05-24 23:00:37 INFO Utils:59 - Successfully started service 'HTTP file server' on port 49688. -2016-05-24 23:00:37 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-24 23:00:37 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-24 23:00:37 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-24 23:00:37 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-24 23:00:37 INFO SparkUI:59 - Started SparkUI at http://211.86.159.184:4040 -2016-05-24 23:00:37 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-24 23:00:37 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-24 23:00:38 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 49707. -2016-05-24 23:00:38 INFO NettyBlockTransferService:59 - Server created on 49707 -2016-05-24 23:00:38 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-24 23:00:38 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:49707 with 730.6 MB RAM, BlockManagerId(driver, localhost, 49707) -2016-05-24 23:00:38 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-24 23:00:38 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-24 23:00:38 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-24 23:00:38 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-24 23:00:38 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-24 23:00:38 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:49707 (size: 13.9 KB, free: 730.6 MB) -2016-05-24 23:00:38 INFO SparkContext:59 - Created broadcast 0 from textFile at NaiveBayesLearning.scala:26 -2016-05-24 23:00:40 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%20, but we couldn't find any external IP address! -2016-05-24 23:00:40 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 23:00:40 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-24 23:00:40 INFO DAGScheduler:59 - Registering RDD 5 (map at NaiveBayes.scala:383) -2016-05-24 23:00:40 INFO DAGScheduler:59 - Got job 0 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-24 23:00:40 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at NaiveBayes.scala:401) -2016-05-24 23:00:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-05-24 23:00:40 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-05-24 23:00:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-24 23:00:40 INFO MemoryStore:59 - ensureFreeSpace(5104) called with curMem=171580, maxMem=766075207 -2016-05-24 23:00:40 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-24 23:00:40 INFO MemoryStore:59 - ensureFreeSpace(2692) called with curMem=176684, maxMem=766075207 -2016-05-24 23:00:40 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-24 23:00:40 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:49707 (size: 2.6 KB, free: 730.6 MB) -2016-05-24 23:00:40 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-24 23:00:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383) -2016-05-24 23:00:40 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-24 23:00:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-24 23:00:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-24 23:00:40 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-24 23:00:40 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-24 23:00:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-24 23:00:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-24 23:00:40 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-24 23:00:40 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-24 23:00:40 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-24 23:00:40 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-24 23:00:40 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-24 23:00:40 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-24 23:00:40 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-24 23:00:40 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver -2016-05-24 23:00:40 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver -2016-05-24 23:00:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 256 ms on localhost (1/2) -2016-05-24 23:00:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 240 ms on localhost (2/2) -2016-05-24 23:00:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-24 23:00:40 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at NaiveBayes.scala:383) finished in 0.276 s -2016-05-24 23:00:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-24 23:00:40 INFO DAGScheduler:59 - running: Set() -2016-05-24 23:00:40 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-05-24 23:00:40 INFO DAGScheduler:59 - failed: Set() -2016-05-24 23:00:40 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-05-24 23:00:40 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-24 23:00:40 INFO MemoryStore:59 - ensureFreeSpace(2936) called with curMem=179376, maxMem=766075207 -2016-05-24 23:00:40 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-24 23:00:40 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=182312, maxMem=766075207 -2016-05-24 23:00:40 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.4 MB) -2016-05-24 23:00:40 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:49707 (size: 1567.0 B, free: 730.6 MB) -2016-05-24 23:00:40 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-24 23:00:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383) -2016-05-24 23:00:40 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-24 23:00:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 23:00:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-24 23:00:40 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-24 23:00:40 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-24 23:00:40 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-24 23:00:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-24 23:00:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-24 23:00:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-24 23:00:40 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-05-24 23:00:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 36 ms on localhost (1/2) -2016-05-24 23:00:40 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1645 bytes result sent to driver -2016-05-24 23:00:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 44 ms on localhost (2/2) -2016-05-24 23:00:40 INFO DAGScheduler:59 - ResultStage 1 (collect at NaiveBayes.scala:401) finished in 0.046 s -2016-05-24 23:00:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-24 23:00:40 INFO DAGScheduler:59 - Job 0 finished: collect at NaiveBayes.scala:401, took 0.438960 s -2016-05-24 23:00:40 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:39 -2016-05-24 23:00:40 INFO DAGScheduler:59 - Got job 1 (count at NaiveBayesLearning.scala:39) with 2 output partitions -2016-05-24 23:00:40 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at NaiveBayesLearning.scala:39) -2016-05-24 23:00:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 23:00:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 23:00:40 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-24 23:00:40 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=183879, maxMem=766075207 -2016-05-24 23:00:40 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-24 23:00:40 INFO MemoryStore:59 - ensureFreeSpace(2671) called with curMem=188775, maxMem=766075207 -2016-05-24 23:00:40 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-24 23:00:40 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:49707 (size: 2.6 KB, free: 730.6 MB) -2016-05-24 23:00:40 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-24 23:00:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:39) -2016-05-24 23:00:40 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-24 23:00:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 23:00:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 23:00:40 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-24 23:00:40 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-24 23:00:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-24 23:00:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-24 23:00:41 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-24 23:00:41 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2082 bytes result sent to driver -2016-05-24 23:00:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 20 ms on localhost (1/2) -2016-05-24 23:00:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 20 ms on localhost (2/2) -2016-05-24 23:00:41 INFO DAGScheduler:59 - ResultStage 2 (count at NaiveBayesLearning.scala:39) finished in 0.023 s -2016-05-24 23:00:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-24 23:00:41 INFO DAGScheduler:59 - Job 1 finished: count at NaiveBayesLearning.scala:39, took 0.039579 s -2016-05-24 23:00:41 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:39 -2016-05-24 23:00:41 INFO DAGScheduler:59 - Got job 2 (count at NaiveBayesLearning.scala:39) with 2 output partitions -2016-05-24 23:00:41 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at NaiveBayesLearning.scala:39) -2016-05-24 23:00:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 23:00:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 23:00:41 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-24 23:00:41 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=191446, maxMem=766075207 -2016-05-24 23:00:41 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-24 23:00:41 INFO MemoryStore:59 - ensureFreeSpace(2117) called with curMem=195046, maxMem=766075207 -2016-05-24 23:00:41 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 23:00:41 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:49707 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 23:00:41 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-24 23:00:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-24 23:00:41 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-24 23:00:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 23:00:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 23:00:41 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-24 23:00:41 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-24 23:00:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-24 23:00:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-24 23:00:41 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-24 23:00:41 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 2082 bytes result sent to driver -2016-05-24 23:00:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 16 ms on localhost (1/2) -2016-05-24 23:00:41 INFO DAGScheduler:59 - ResultStage 3 (count at NaiveBayesLearning.scala:39) finished in 0.018 s -2016-05-24 23:00:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 17 ms on localhost (2/2) -2016-05-24 23:00:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-24 23:00:41 INFO DAGScheduler:59 - Job 2 finished: count at NaiveBayesLearning.scala:39, took 0.030224 s -2016-05-24 23:00:41 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:43 -2016-05-24 23:00:41 INFO DAGScheduler:59 - Got job 3 (count at NaiveBayesLearning.scala:43) with 2 output partitions -2016-05-24 23:00:41 INFO DAGScheduler:59 - Final stage: ResultStage 4(count at NaiveBayesLearning.scala:43) -2016-05-24 23:00:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 23:00:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 23:00:41 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-24 23:00:41 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=197163, maxMem=766075207 -2016-05-24 23:00:41 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-24 23:00:41 INFO MemoryStore:59 - ensureFreeSpace(2113) called with curMem=200763, maxMem=766075207 -2016-05-24 23:00:41 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 23:00:41 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:49707 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 23:00:41 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-24 23:00:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-24 23:00:41 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-24 23:00:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 23:00:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 23:00:41 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-24 23:00:41 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-24 23:00:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-24 23:00:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-24 23:00:41 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2082 bytes result sent to driver -2016-05-24 23:00:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 22 ms on localhost (1/2) -2016-05-24 23:00:41 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2082 bytes result sent to driver -2016-05-24 23:00:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 25 ms on localhost (2/2) -2016-05-24 23:00:41 INFO DAGScheduler:59 - ResultStage 4 (count at NaiveBayesLearning.scala:43) finished in 0.027 s -2016-05-24 23:00:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-24 23:00:41 INFO DAGScheduler:59 - Job 3 finished: count at NaiveBayesLearning.scala:43, took 0.041248 s -2016-05-24 23:00:41 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:44 -2016-05-24 23:00:41 INFO DAGScheduler:59 - Got job 4 (count at NaiveBayesLearning.scala:44) with 2 output partitions -2016-05-24 23:00:41 INFO DAGScheduler:59 - Final stage: ResultStage 5(count at NaiveBayesLearning.scala:44) -2016-05-24 23:00:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 23:00:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 23:00:41 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-24 23:00:41 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=202876, maxMem=766075207 -2016-05-24 23:00:41 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-24 23:00:41 INFO MemoryStore:59 - ensureFreeSpace(2117) called with curMem=206476, maxMem=766075207 -2016-05-24 23:00:41 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-24 23:00:41 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:49707 (size: 2.1 KB, free: 730.6 MB) -2016-05-24 23:00:41 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-24 23:00:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-24 23:00:41 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-24 23:00:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 23:00:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 23:00:41 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-05-24 23:00:41 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-05-24 23:00:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-24 23:00:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-24 23:00:41 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 2082 bytes result sent to driver -2016-05-24 23:00:41 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 2082 bytes result sent to driver -2016-05-24 23:00:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 25 ms on localhost (1/2) -2016-05-24 23:00:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 24 ms on localhost (2/2) -2016-05-24 23:00:41 INFO DAGScheduler:59 - ResultStage 5 (count at NaiveBayesLearning.scala:44) finished in 0.025 s -2016-05-24 23:00:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-24 23:00:41 INFO DAGScheduler:59 - Job 4 finished: count at NaiveBayesLearning.scala:44, took 0.039222 s -2016-05-24 23:00:41 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:47 -2016-05-24 23:00:41 INFO DAGScheduler:59 - Got job 5 (take at NaiveBayesLearning.scala:47) with 1 output partitions -2016-05-24 23:00:41 INFO DAGScheduler:59 - Final stage: ResultStage 6(take at NaiveBayesLearning.scala:47) -2016-05-24 23:00:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 23:00:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 23:00:41 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:38), which has no missing parents -2016-05-24 23:00:41 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=208593, maxMem=766075207 -2016-05-24 23:00:41 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-24 23:00:41 INFO MemoryStore:59 - ensureFreeSpace(2669) called with curMem=213417, maxMem=766075207 -2016-05-24 23:00:41 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-24 23:00:41 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:49707 (size: 2.6 KB, free: 730.6 MB) -2016-05-24 23:00:41 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-24 23:00:41 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 6 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:38) -2016-05-24 23:00:41 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 1 tasks -2016-05-24 23:00:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 23:00:41 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-05-24 23:00:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-24 23:00:41 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 2040 bytes result sent to driver -2016-05-24 23:00:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 17 ms on localhost (1/1) -2016-05-24 23:00:41 INFO DAGScheduler:59 - ResultStage 6 (take at NaiveBayesLearning.scala:47) finished in 0.017 s -2016-05-24 23:00:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-24 23:00:41 INFO DAGScheduler:59 - Job 5 finished: take at NaiveBayesLearning.scala:47, took 0.033882 s -2016-05-24 23:00:41 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:47 -2016-05-24 23:00:41 INFO DAGScheduler:59 - Got job 6 (take at NaiveBayesLearning.scala:47) with 1 output partitions -2016-05-24 23:00:41 INFO DAGScheduler:59 - Final stage: ResultStage 7(take at NaiveBayesLearning.scala:47) -2016-05-24 23:00:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 23:00:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 23:00:41 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:38), which has no missing parents -2016-05-24 23:00:41 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=216086, maxMem=766075207 -2016-05-24 23:00:41 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-24 23:00:41 INFO MemoryStore:59 - ensureFreeSpace(2669) called with curMem=220910, maxMem=766075207 -2016-05-24 23:00:41 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-24 23:00:41 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:49707 (size: 2.6 KB, free: 730.6 MB) -2016-05-24 23:00:41 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-24 23:00:41 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:38) -2016-05-24 23:00:41 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-24 23:00:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-24 23:00:41 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-24 23:00:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-24 23:00:41 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2206 bytes result sent to driver -2016-05-24 23:00:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 20 ms on localhost (1/1) -2016-05-24 23:00:41 INFO DAGScheduler:59 - ResultStage 7 (take at NaiveBayesLearning.scala:47) finished in 0.021 s -2016-05-24 23:00:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-24 23:00:41 INFO DAGScheduler:59 - Job 6 finished: take at NaiveBayesLearning.scala:47, took 0.040058 s -2016-05-24 23:00:41 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:49707 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-24 23:00:41 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-24 23:00:41 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:49707 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-24 23:00:41 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-24 23:00:41 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:49707 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 23:00:41 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-24 23:00:41 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:49707 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 23:00:41 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-24 23:00:41 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:49707 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-24 23:00:41 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-24 23:00:41 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:49707 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-24 23:00:41 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-24 23:00:41 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:49707 in memory (size: 1567.0 B, free: 730.6 MB) -2016-05-24 23:00:41 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-24 23:00:41 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:49707 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-24 23:00:41 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-24 23:00:41 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-24 23:00:41 INFO SparkContext:59 - Starting job: saveAsTextFile at NaiveBayes.scala:202 -2016-05-24 23:00:41 INFO DAGScheduler:59 - Got job 7 (saveAsTextFile at NaiveBayes.scala:202) with 1 output partitions -2016-05-24 23:00:41 INFO DAGScheduler:59 - Final stage: ResultStage 8(saveAsTextFile at NaiveBayes.scala:202) -2016-05-24 23:00:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 23:00:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 23:00:41 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202), which has no missing parents -2016-05-24 23:00:41 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=171580, maxMem=766075207 -2016-05-24 23:00:41 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 124.0 KB, free 730.3 MB) -2016-05-24 23:00:41 INFO MemoryStore:59 - ensureFreeSpace(42580) called with curMem=298572, maxMem=766075207 -2016-05-24 23:00:41 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.3 MB) -2016-05-24 23:00:41 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:49707 (size: 41.6 KB, free: 730.5 MB) -2016-05-24 23:00:41 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-24 23:00:41 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202) -2016-05-24 23:00:41 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-05-24 23:00:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 14, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-24 23:00:41 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 14) -2016-05-24 23:00:42 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242300_0008_m_000000_14' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524230041237/result/metadata/_temporary/0/task_201605242300_0008_m_000000 -2016-05-24 23:00:42 INFO SparkHadoopMapRedUtil:59 - attempt_201605242300_0008_m_000000_14: Committed -2016-05-24 23:00:42 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 14). 1864 bytes result sent to driver -2016-05-24 23:00:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 14) in 207 ms on localhost (1/1) -2016-05-24 23:00:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-24 23:00:42 INFO DAGScheduler:59 - ResultStage 8 (saveAsTextFile at NaiveBayes.scala:202) finished in 0.210 s -2016-05-24 23:00:42 INFO DAGScheduler:59 - Job 7 finished: saveAsTextFile at NaiveBayes.scala:202, took 0.306729 s -2016-05-24 23:00:43 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 23:00:43 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 23:00:43 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:206 -2016-05-24 23:00:43 INFO DAGScheduler:59 - Got job 8 (parquet at NaiveBayes.scala:206) with 1 output partitions -2016-05-24 23:00:43 INFO DAGScheduler:59 - Final stage: ResultStage 9(parquet at NaiveBayes.scala:206) -2016-05-24 23:00:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 23:00:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 23:00:43 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205), which has no missing parents -2016-05-24 23:00:43 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:49707 in memory (size: 41.6 KB, free: 730.6 MB) -2016-05-24 23:00:43 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-24 23:00:43 INFO MemoryStore:59 - ensureFreeSpace(66936) called with curMem=171580, maxMem=766075207 -2016-05-24 23:00:43 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 65.4 KB, free 730.4 MB) -2016-05-24 23:00:43 INFO MemoryStore:59 - ensureFreeSpace(23364) called with curMem=238516, maxMem=766075207 -2016-05-24 23:00:43 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 22.8 KB, free 730.3 MB) -2016-05-24 23:00:43 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:49707 (size: 22.8 KB, free: 730.6 MB) -2016-05-24 23:00:43 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-24 23:00:43 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 9 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205) -2016-05-24 23:00:43 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 1 tasks -2016-05-24 23:00:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2506 bytes) -2016-05-24 23:00:43 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 15) -2016-05-24 23:00:43 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-24 23:00:43 INFO CodecConfig:151 - Compression: GZIP -2016-05-24 23:00:43 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-24 23:00:43 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-24 23:00:43 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-24 23:00:43 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-24 23:00:43 INFO ParquetOutputFormat:151 - Validation is off -2016-05-24 23:00:43 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-24 23:00:43 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-24 23:00:43 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 231 -2016-05-24 23:00:43 INFO ColumnChunkPageWriteStore:151 - written 77B for [labels, bag, array] DOUBLE: 3 values, 37B raw, 36B comp, 1 pages, encodings: [RLE, PLAIN] -2016-05-24 23:00:43 INFO ColumnChunkPageWriteStore:151 - written 74B for [pi, bag, array] DOUBLE: 3 values, 16B raw, 33B comp, 1 pages, encodings: [RLE, PLAIN_DICTIONARY], dic { 2 entries, 16B raw, 2B comp} -2016-05-24 23:00:43 INFO ColumnChunkPageWriteStore:151 - written 83B for [theta, bag, array, bag, array] DOUBLE: 9 values, 23B raw, 42B comp, 1 pages, encodings: [RLE, PLAIN_DICTIONARY], dic { 6 entries, 48B raw, 6B comp} -2016-05-24 23:00:43 INFO ColumnChunkPageWriteStore:151 - written 86B for [modelType] BINARY: 1 values, 21B raw, 39B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-24 23:00:44 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605242300_0009_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524230041237/result/data/_temporary/0/task_201605242300_0009_m_000000 -2016-05-24 23:00:44 INFO SparkHadoopMapRedUtil:59 - attempt_201605242300_0009_m_000000_0: Committed -2016-05-24 23:00:44 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 15). 915 bytes result sent to driver -2016-05-24 23:00:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 15) in 452 ms on localhost (1/1) -2016-05-24 23:00:44 INFO DAGScheduler:59 - ResultStage 9 (parquet at NaiveBayes.scala:206) finished in 0.454 s -2016-05-24 23:00:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-24 23:00:44 INFO DAGScheduler:59 - Job 8 finished: parquet at NaiveBayes.scala:206, took 0.524006 s -2016-05-24 23:00:44 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 23:00:44 INFO DefaultWriterContainer:59 - Job job_201605242300_0000 committed. -2016-05-24 23:00:44 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524230041237/result/data on driver -2016-05-24 23:00:44 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524230041237/result/data on driver -2016-05-24 23:00:44 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=261880, maxMem=766075207 -2016-05-24 23:00:44 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-24 23:00:44 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=492584, maxMem=766075207 -2016-05-24 23:00:44 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-24 23:00:44 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:49707 (size: 19.3 KB, free: 730.5 MB) -2016-05-24 23:00:44 INFO SparkContext:59 - Created broadcast 11 from textFile at modelSaveLoad.scala:129 -2016-05-24 23:00:44 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-24 23:00:44 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-24 23:00:44 INFO DAGScheduler:59 - Got job 9 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-24 23:00:44 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at modelSaveLoad.scala:129) -2016-05-24 23:00:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 23:00:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 23:00:44 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-24 23:00:44 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=512372, maxMem=766075207 -2016-05-24 23:00:44 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-24 23:00:44 INFO MemoryStore:59 - ensureFreeSpace(1892) called with curMem=515588, maxMem=766075207 -2016-05-24 23:00:44 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 1892.0 B, free 730.1 MB) -2016-05-24 23:00:44 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:49707 (size: 1892.0 B, free: 730.5 MB) -2016-05-24 23:00:44 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-24 23:00:44 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129) -2016-05-24 23:00:44 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-24 23:00:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 2239 bytes) -2016-05-24 23:00:44 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-24 23:00:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524230041237/result/metadata/part-00000:0+56 -2016-05-24 23:00:44 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 2159 bytes result sent to driver -2016-05-24 23:00:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 14 ms on localhost (1/1) -2016-05-24 23:00:44 INFO DAGScheduler:59 - ResultStage 10 (first at modelSaveLoad.scala:129) finished in 0.014 s -2016-05-24 23:00:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-24 23:00:44 INFO DAGScheduler:59 - Job 9 finished: first at modelSaveLoad.scala:129, took 0.029393 s -2016-05-24 23:00:44 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524230041237/result/data on driver -2016-05-24 23:00:44 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:213 -2016-05-24 23:00:44 INFO DAGScheduler:59 - Got job 10 (parquet at NaiveBayes.scala:213) with 4 output partitions -2016-05-24 23:00:44 INFO DAGScheduler:59 - Final stage: ResultStage 11(parquet at NaiveBayes.scala:213) -2016-05-24 23:00:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 23:00:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 23:00:44 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213), which has no missing parents -2016-05-24 23:00:44 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=517480, maxMem=766075207 -2016-05-24 23:00:44 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-24 23:00:44 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=580480, maxMem=766075207 -2016-05-24 23:00:44 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.0 MB) -2016-05-24 23:00:44 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:49707 (size: 20.6 KB, free: 730.5 MB) -2016-05-24 23:00:44 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-24 23:00:44 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 11 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213) -2016-05-24 23:00:44 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 4 tasks -2016-05-24 23:00:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 17, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 23:00:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 18, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 23:00:44 INFO TaskSetManager:59 - Starting task 2.0 in stage 11.0 (TID 19, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-24 23:00:44 INFO TaskSetManager:59 - Starting task 3.0 in stage 11.0 (TID 20, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-24 23:00:44 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 17) -2016-05-24 23:00:44 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 18) -2016-05-24 23:00:44 INFO Executor:59 - Running task 3.0 in stage 11.0 (TID 20) -2016-05-24 23:00:44 INFO Executor:59 - Running task 2.0 in stage 11.0 (TID 19) -2016-05-24 23:00:44 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 23:00:44 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 18). 936 bytes result sent to driver -2016-05-24 23:00:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 18) in 55 ms on localhost (1/4) -2016-05-24 23:00:44 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 23:00:44 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 23:00:44 INFO Executor:59 - Finished task 2.0 in stage 11.0 (TID 19). 936 bytes result sent to driver -2016-05-24 23:00:44 INFO TaskSetManager:59 - Finished task 2.0 in stage 11.0 (TID 19) in 72 ms on localhost (2/4) -2016-05-24 23:00:44 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-24 23:00:44 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 17). 936 bytes result sent to driver -2016-05-24 23:00:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 17) in 78 ms on localhost (3/4) -2016-05-24 23:00:44 INFO Executor:59 - Finished task 3.0 in stage 11.0 (TID 20). 1876 bytes result sent to driver -2016-05-24 23:00:44 INFO TaskSetManager:59 - Finished task 3.0 in stage 11.0 (TID 20) in 76 ms on localhost (4/4) -2016-05-24 23:00:44 INFO DAGScheduler:59 - ResultStage 11 (parquet at NaiveBayes.scala:213) finished in 0.080 s -2016-05-24 23:00:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-24 23:00:44 INFO DAGScheduler:59 - Job 10 finished: parquet at NaiveBayes.scala:213, took 0.112485 s -2016-05-24 23:00:44 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=601553, maxMem=766075207 -2016-05-24 23:00:44 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 86.4 KB, free 729.9 MB) -2016-05-24 23:00:44 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=690065, maxMem=766075207 -2016-05-24 23:00:44 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-24 23:00:44 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:49707 (size: 19.3 KB, free: 730.5 MB) -2016-05-24 23:00:44 INFO SparkContext:59 - Created broadcast 14 from take at NaiveBayes.scala:216 -2016-05-24 23:00:45 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-24 23:00:45 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524230041237/result/data/part-r-00000-904e3bec-46b3-4cb9-860f-f6cbc838b48f.gz.parquet -2016-05-24 23:00:45 INFO SparkContext:59 - Starting job: take at NaiveBayes.scala:216 -2016-05-24 23:00:45 INFO DAGScheduler:59 - Got job 11 (take at NaiveBayes.scala:216) with 1 output partitions -2016-05-24 23:00:45 INFO DAGScheduler:59 - Final stage: ResultStage 12(take at NaiveBayes.scala:216) -2016-05-24 23:00:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-24 23:00:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-24 23:00:45 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216), which has no missing parents -2016-05-24 23:00:45 INFO MemoryStore:59 - ensureFreeSpace(4280) called with curMem=709853, maxMem=766075207 -2016-05-24 23:00:45 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 4.2 KB, free 729.9 MB) -2016-05-24 23:00:45 INFO MemoryStore:59 - ensureFreeSpace(2466) called with curMem=714133, maxMem=766075207 -2016-05-24 23:00:45 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.4 KB, free 729.9 MB) -2016-05-24 23:00:45 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:49707 (size: 2.4 KB, free: 730.5 MB) -2016-05-24 23:00:45 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-24 23:00:45 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216) -2016-05-24 23:00:45 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-24 23:00:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 21, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-24 23:00:45 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 21) -2016-05-24 23:00:45 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160524230041237/result/data/part-r-00000-904e3bec-46b3-4cb9-860f-f6cbc838b48f.gz.parquet start: 0 end: 1477 length: 1477 hosts: []} -2016-05-24 23:00:45 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-24 23:00:45 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group labels (LIST) { - repeated group bag { - optional double array; - } - } - optional group pi (LIST) { - repeated group bag { - optional double array; - } - } - optional group theta (LIST) { - repeated group bag { - optional group array (LIST) { - repeated group bag { - optional double array; - } - } - } - } - optional binary modelType (UTF8); -} - -Catalyst form: -StructType(StructField(labels,ArrayType(DoubleType,true),true), StructField(pi,ArrayType(DoubleType,true),true), StructField(theta,ArrayType(ArrayType(DoubleType,true),true),true), StructField(modelType,StringType,true)) - -2016-05-24 23:00:45 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-24 23:00:45 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-24 23:00:45 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-24 23:00:45 INFO InternalParquetRecordReader:151 - block read in memory in 11 ms. row count = 1 -2016-05-24 23:00:45 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 21). 2841 bytes result sent to driver -2016-05-24 23:00:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 21) in 108 ms on localhost (1/1) -2016-05-24 23:00:45 INFO DAGScheduler:59 - ResultStage 12 (take at NaiveBayes.scala:216) finished in 0.108 s -2016-05-24 23:00:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-24 23:00:45 INFO DAGScheduler:59 - Job 11 finished: take at NaiveBayes.scala:216, took 0.126259 s -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-24 23:00:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-24 23:00:45 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.184:4040 -2016-05-24 23:00:45 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-24 23:00:45 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-24 23:00:45 INFO MemoryStore:59 - MemoryStore cleared -2016-05-24 23:00:45 INFO BlockManager:59 - BlockManager stopped -2016-05-24 23:00:45 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-24 23:00:45 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-24 23:00:45 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-24 23:00:45 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-24 23:00:45 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-b3bc0f29-20cd-4255-9375-212b2354359b -2016-05-25 09:59:53 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 09:59:59 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 10:00:01 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 10:00:01 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 10:00:01 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 10:00:20 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 10:00:21 INFO Remoting:74 - Starting remoting -2016-05-25 10:00:24 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:51252] -2016-05-25 10:00:25 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51252. -2016-05-25 10:00:25 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 10:00:26 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 10:00:27 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-b888f580-a256-4bf3-9d8e-8680ca05e1b6 -2016-05-25 10:00:27 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 10:00:27 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-1492960a-310e-476b-8d82-df9d7d3100a9\httpd-4ff48f94-34c9-43a5-aa53-6e19e550ae1b -2016-05-25 10:00:28 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 10:00:29 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:00:29 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51262 -2016-05-25 10:00:29 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51262. -2016-05-25 10:00:29 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 10:00:31 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:00:32 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 10:00:32 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 10:00:32 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 10:00:36 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 10:00:36 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 10:00:42 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51292. -2016-05-25 10:00:42 INFO NettyBlockTransferService:59 - Server created on 51292 -2016-05-25 10:00:42 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 10:00:42 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51292 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51292) -2016-05-25 10:00:42 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 10:00:50 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 10:00:50 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 10:00:50 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 10:00:50 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 10:00:50 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51292 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 10:00:50 INFO SparkContext:59 - Created broadcast 0 from textFile at NaiveBayesLearning.scala:26 -2016-05-25 10:00:55 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 10:00:57 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:00:57 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:00:57 INFO DAGScheduler:59 - Registering RDD 5 (map at NaiveBayes.scala:383) -2016-05-25 10:00:57 INFO DAGScheduler:59 - Got job 0 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:00:57 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at NaiveBayes.scala:401) -2016-05-25 10:00:57 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-05-25 10:00:57 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-05-25 10:00:57 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:00:58 INFO MemoryStore:59 - ensureFreeSpace(5104) called with curMem=171580, maxMem=766075207 -2016-05-25 10:00:58 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 10:00:58 INFO MemoryStore:59 - ensureFreeSpace(2692) called with curMem=176684, maxMem=766075207 -2016-05-25 10:00:58 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:00:58 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51292 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:00:58 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:00:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383) -2016-05-25 10:00:58 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 10:00:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 10:00:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 10:00:58 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 10:00:58 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 10:00:58 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:00:58 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:00:58 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 10:00:58 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 10:00:58 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 10:00:58 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 10:00:58 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 10:00:59 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 10:00:59 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 10:00:59 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver -2016-05-25 10:00:59 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver -2016-05-25 10:00:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 1219 ms on localhost (1/2) -2016-05-25 10:00:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 1324 ms on localhost (2/2) -2016-05-25 10:00:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 10:00:59 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at NaiveBayes.scala:383) finished in 1.424 s -2016-05-25 10:00:59 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 10:00:59 INFO DAGScheduler:59 - running: Set() -2016-05-25 10:00:59 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-05-25 10:00:59 INFO DAGScheduler:59 - failed: Set() -2016-05-25 10:00:59 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-05-25 10:00:59 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 10:00:59 INFO MemoryStore:59 - ensureFreeSpace(2936) called with curMem=179376, maxMem=766075207 -2016-05-25 10:00:59 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-25 10:00:59 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=182312, maxMem=766075207 -2016-05-25 10:00:59 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.4 MB) -2016-05-25 10:00:59 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51292 (size: 1567.0 B, free: 730.6 MB) -2016-05-25 10:00:59 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:00:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383) -2016-05-25 10:00:59 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-25 10:00:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:00:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:00:59 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 10:00:59 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-25 10:00:59 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 10:00:59 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 10:00:59 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 18 ms -2016-05-25 10:00:59 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 18 ms -2016-05-25 10:00:59 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-05-25 10:00:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 67 ms on localhost (1/2) -2016-05-25 10:00:59 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1645 bytes result sent to driver -2016-05-25 10:00:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 77 ms on localhost (2/2) -2016-05-25 10:00:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 10:00:59 INFO DAGScheduler:59 - ResultStage 1 (collect at NaiveBayes.scala:401) finished in 0.078 s -2016-05-25 10:00:59 INFO DAGScheduler:59 - Job 0 finished: collect at NaiveBayes.scala:401, took 2.119002 s -2016-05-25 10:01:00 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:39 -2016-05-25 10:01:00 INFO DAGScheduler:59 - Got job 1 (count at NaiveBayesLearning.scala:39) with 2 output partitions -2016-05-25 10:01:00 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at NaiveBayesLearning.scala:39) -2016-05-25 10:01:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:01:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:01:00 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-25 10:01:00 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=183879, maxMem=766075207 -2016-05-25 10:01:00 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-25 10:01:00 INFO MemoryStore:59 - ensureFreeSpace(2671) called with curMem=188775, maxMem=766075207 -2016-05-25 10:01:00 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:01:00 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:51292 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:01:00 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:01:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:39) -2016-05-25 10:01:00 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 10:01:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:01:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:01:00 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-25 10:01:00 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-25 10:01:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:01:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:01:00 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 10:01:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 38 ms on localhost (1/2) -2016-05-25 10:01:00 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2082 bytes result sent to driver -2016-05-25 10:01:00 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:51292 in memory (size: 1567.0 B, free: 730.6 MB) -2016-05-25 10:01:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 72 ms on localhost (2/2) -2016-05-25 10:01:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 10:01:00 INFO DAGScheduler:59 - ResultStage 2 (count at NaiveBayesLearning.scala:39) finished in 0.073 s -2016-05-25 10:01:00 INFO DAGScheduler:59 - Job 1 finished: count at NaiveBayesLearning.scala:39, took 0.092854 s -2016-05-25 10:01:00 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 10:01:00 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:39 -2016-05-25 10:01:00 INFO DAGScheduler:59 - Got job 2 (count at NaiveBayesLearning.scala:39) with 2 output partitions -2016-05-25 10:01:00 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at NaiveBayesLearning.scala:39) -2016-05-25 10:01:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:01:00 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:51292 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:01:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:01:00 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-25 10:01:00 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 10:01:00 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=179147, maxMem=766075207 -2016-05-25 10:01:00 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:01:00 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 10:01:00 INFO MemoryStore:59 - ensureFreeSpace(2117) called with curMem=182747, maxMem=766075207 -2016-05-25 10:01:00 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:01:00 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:51292 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:01:00 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:01:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-25 10:01:00 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 10:01:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:01:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:01:00 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-25 10:01:00 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-25 10:01:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:01:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:01:00 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-25 10:01:00 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 2082 bytes result sent to driver -2016-05-25 10:01:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 21 ms on localhost (1/2) -2016-05-25 10:01:00 INFO DAGScheduler:59 - ResultStage 3 (count at NaiveBayesLearning.scala:39) finished in 0.021 s -2016-05-25 10:01:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 20 ms on localhost (2/2) -2016-05-25 10:01:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 10:01:00 INFO DAGScheduler:59 - Job 2 finished: count at NaiveBayesLearning.scala:39, took 0.036897 s -2016-05-25 10:01:00 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:43 -2016-05-25 10:01:00 INFO DAGScheduler:59 - Got job 3 (count at NaiveBayesLearning.scala:43) with 2 output partitions -2016-05-25 10:01:00 INFO DAGScheduler:59 - Final stage: ResultStage 4(count at NaiveBayesLearning.scala:43) -2016-05-25 10:01:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:01:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:01:00 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-25 10:01:00 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=184864, maxMem=766075207 -2016-05-25 10:01:00 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:01:00 INFO MemoryStore:59 - ensureFreeSpace(2113) called with curMem=188464, maxMem=766075207 -2016-05-25 10:01:00 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:01:00 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:51292 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:01:00 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:01:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-25 10:01:00 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 10:01:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:01:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:01:00 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-25 10:01:00 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-25 10:01:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:01:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:01:00 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2082 bytes result sent to driver -2016-05-25 10:01:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 16 ms on localhost (1/2) -2016-05-25 10:01:00 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2082 bytes result sent to driver -2016-05-25 10:01:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 20 ms on localhost (2/2) -2016-05-25 10:01:00 INFO DAGScheduler:59 - ResultStage 4 (count at NaiveBayesLearning.scala:43) finished in 0.021 s -2016-05-25 10:01:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 10:01:00 INFO DAGScheduler:59 - Job 3 finished: count at NaiveBayesLearning.scala:43, took 0.034189 s -2016-05-25 10:01:00 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:44 -2016-05-25 10:01:00 INFO DAGScheduler:59 - Got job 4 (count at NaiveBayesLearning.scala:44) with 2 output partitions -2016-05-25 10:01:00 INFO DAGScheduler:59 - Final stage: ResultStage 5(count at NaiveBayesLearning.scala:44) -2016-05-25 10:01:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:01:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:01:00 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-25 10:01:00 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=190577, maxMem=766075207 -2016-05-25 10:01:00 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:01:00 INFO MemoryStore:59 - ensureFreeSpace(2117) called with curMem=194177, maxMem=766075207 -2016-05-25 10:01:00 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:01:00 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:51292 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:01:00 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:01:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-25 10:01:00 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 10:01:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:01:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:01:00 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-05-25 10:01:00 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-05-25 10:01:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:01:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:01:00 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 2082 bytes result sent to driver -2016-05-25 10:01:00 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 2082 bytes result sent to driver -2016-05-25 10:01:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 21 ms on localhost (1/2) -2016-05-25 10:01:00 INFO DAGScheduler:59 - ResultStage 5 (count at NaiveBayesLearning.scala:44) finished in 0.024 s -2016-05-25 10:01:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 24 ms on localhost (2/2) -2016-05-25 10:01:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 10:01:00 INFO DAGScheduler:59 - Job 4 finished: count at NaiveBayesLearning.scala:44, took 0.037590 s -2016-05-25 10:01:00 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:47 -2016-05-25 10:01:00 INFO DAGScheduler:59 - Got job 5 (take at NaiveBayesLearning.scala:47) with 1 output partitions -2016-05-25 10:01:00 INFO DAGScheduler:59 - Final stage: ResultStage 6(take at NaiveBayesLearning.scala:47) -2016-05-25 10:01:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:01:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:01:00 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:38), which has no missing parents -2016-05-25 10:01:00 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=196294, maxMem=766075207 -2016-05-25 10:01:00 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 10:01:00 INFO MemoryStore:59 - ensureFreeSpace(2669) called with curMem=201118, maxMem=766075207 -2016-05-25 10:01:00 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:01:00 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:51292 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:01:00 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:01:00 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 6 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:38) -2016-05-25 10:01:00 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 1 tasks -2016-05-25 10:01:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:01:00 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-05-25 10:01:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:01:00 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 2040 bytes result sent to driver -2016-05-25 10:01:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 20 ms on localhost (1/1) -2016-05-25 10:01:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 10:01:00 INFO DAGScheduler:59 - ResultStage 6 (take at NaiveBayesLearning.scala:47) finished in 0.021 s -2016-05-25 10:01:00 INFO DAGScheduler:59 - Job 5 finished: take at NaiveBayesLearning.scala:47, took 0.037653 s -2016-05-25 10:01:00 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:47 -2016-05-25 10:01:00 INFO DAGScheduler:59 - Got job 6 (take at NaiveBayesLearning.scala:47) with 1 output partitions -2016-05-25 10:01:00 INFO DAGScheduler:59 - Final stage: ResultStage 7(take at NaiveBayesLearning.scala:47) -2016-05-25 10:01:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:01:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:01:00 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:38), which has no missing parents -2016-05-25 10:01:00 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=203787, maxMem=766075207 -2016-05-25 10:01:00 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 10:01:00 INFO MemoryStore:59 - ensureFreeSpace(2669) called with curMem=208611, maxMem=766075207 -2016-05-25 10:01:00 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:01:00 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:51292 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:01:00 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:01:00 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:38) -2016-05-25 10:01:00 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-25 10:01:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:01:00 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 10:01:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:01:00 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2206 bytes result sent to driver -2016-05-25 10:01:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 22 ms on localhost (1/1) -2016-05-25 10:01:00 INFO DAGScheduler:59 - ResultStage 7 (take at NaiveBayesLearning.scala:47) finished in 0.024 s -2016-05-25 10:01:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 10:01:00 INFO DAGScheduler:59 - Job 6 finished: take at NaiveBayesLearning.scala:47, took 0.044712 s -2016-05-25 10:01:01 INFO SparkContext:59 - Starting job: saveAsTextFile at NaiveBayes.scala:202 -2016-05-25 10:01:01 INFO DAGScheduler:59 - Got job 7 (saveAsTextFile at NaiveBayes.scala:202) with 1 output partitions -2016-05-25 10:01:01 INFO DAGScheduler:59 - Final stage: ResultStage 8(saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 10:01:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:01:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:01:01 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202), which has no missing parents -2016-05-25 10:01:01 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=211280, maxMem=766075207 -2016-05-25 10:01:01 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 124.0 KB, free 730.3 MB) -2016-05-25 10:01:01 INFO MemoryStore:59 - ensureFreeSpace(42584) called with curMem=338272, maxMem=766075207 -2016-05-25 10:01:01 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.2 MB) -2016-05-25 10:01:01 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:51292 (size: 41.6 KB, free: 730.5 MB) -2016-05-25 10:01:01 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:01:01 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 10:01:01 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-05-25 10:01:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 14, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-25 10:01:01 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 14) -2016-05-25 10:01:02 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251001_0008_m_000000_14' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525100100516/result/metadata/_temporary/0/task_201605251001_0008_m_000000 -2016-05-25 10:01:02 INFO SparkHadoopMapRedUtil:59 - attempt_201605251001_0008_m_000000_14: Committed -2016-05-25 10:01:02 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 14). 1864 bytes result sent to driver -2016-05-25 10:01:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 14) in 495 ms on localhost (1/1) -2016-05-25 10:01:02 INFO DAGScheduler:59 - ResultStage 8 (saveAsTextFile at NaiveBayes.scala:202) finished in 0.495 s -2016-05-25 10:01:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 10:01:02 INFO DAGScheduler:59 - Job 7 finished: saveAsTextFile at NaiveBayes.scala:202, took 0.583401 s -2016-05-25 10:01:03 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:51292 in memory (size: 41.6 KB, free: 730.6 MB) -2016-05-25 10:01:03 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 10:01:03 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:51292 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:01:03 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 10:01:03 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:51292 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:01:03 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 10:01:03 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:51292 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:01:03 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 10:01:03 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:51292 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:01:03 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 10:01:03 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:51292 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:01:03 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 10:01:03 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:51292 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:01:03 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 10:01:06 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:01:06 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:01:07 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:206 -2016-05-25 10:01:07 INFO DAGScheduler:59 - Got job 8 (parquet at NaiveBayes.scala:206) with 1 output partitions -2016-05-25 10:01:07 INFO DAGScheduler:59 - Final stage: ResultStage 9(parquet at NaiveBayes.scala:206) -2016-05-25 10:01:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:01:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:01:07 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205), which has no missing parents -2016-05-25 10:01:07 INFO MemoryStore:59 - ensureFreeSpace(66936) called with curMem=171580, maxMem=766075207 -2016-05-25 10:01:07 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 65.4 KB, free 730.4 MB) -2016-05-25 10:01:07 INFO MemoryStore:59 - ensureFreeSpace(23362) called with curMem=238516, maxMem=766075207 -2016-05-25 10:01:07 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 22.8 KB, free 730.3 MB) -2016-05-25 10:01:07 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:51292 (size: 22.8 KB, free: 730.6 MB) -2016-05-25 10:01:07 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:01:07 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 9 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205) -2016-05-25 10:01:07 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 1 tasks -2016-05-25 10:01:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2506 bytes) -2016-05-25 10:01:07 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 15) -2016-05-25 10:01:07 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:01:07 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 10:01:07 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 10:01:07 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 10:01:07 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 10:01:07 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 10:01:07 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 10:01:07 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 10:01:07 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 10:01:07 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 231 -2016-05-25 10:01:08 INFO ColumnChunkPageWriteStore:151 - written 77B for [labels, bag, array] DOUBLE: 3 values, 37B raw, 36B comp, 1 pages, encodings: [RLE, PLAIN] -2016-05-25 10:01:08 INFO ColumnChunkPageWriteStore:151 - written 74B for [pi, bag, array] DOUBLE: 3 values, 16B raw, 33B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 2 entries, 16B raw, 2B comp} -2016-05-25 10:01:08 INFO ColumnChunkPageWriteStore:151 - written 83B for [theta, bag, array, bag, array] DOUBLE: 9 values, 23B raw, 42B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 6 entries, 48B raw, 6B comp} -2016-05-25 10:01:08 INFO ColumnChunkPageWriteStore:151 - written 86B for [modelType] BINARY: 1 values, 21B raw, 39B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 10:01:08 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251001_0009_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525100100516/result/data/_temporary/0/task_201605251001_0009_m_000000 -2016-05-25 10:01:08 INFO SparkHadoopMapRedUtil:59 - attempt_201605251001_0009_m_000000_0: Committed -2016-05-25 10:01:08 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 15). 915 bytes result sent to driver -2016-05-25 10:01:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 15) in 1612 ms on localhost (1/1) -2016-05-25 10:01:08 INFO DAGScheduler:59 - ResultStage 9 (parquet at NaiveBayes.scala:206) finished in 1.612 s -2016-05-25 10:01:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 10:01:08 INFO DAGScheduler:59 - Job 8 finished: parquet at NaiveBayes.scala:206, took 1.653876 s -2016-05-25 10:01:08 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:01:08 INFO DefaultWriterContainer:59 - Job job_201605251001_0000 committed. -2016-05-25 10:01:08 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525100100516/result/data on driver -2016-05-25 10:01:08 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525100100516/result/data on driver -2016-05-25 10:01:09 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=261878, maxMem=766075207 -2016-05-25 10:01:09 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-25 10:01:09 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=492582, maxMem=766075207 -2016-05-25 10:01:09 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-25 10:01:09 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:51292 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 10:01:09 INFO SparkContext:59 - Created broadcast 11 from textFile at modelSaveLoad.scala:129 -2016-05-25 10:01:09 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:51292 in memory (size: 22.8 KB, free: 730.6 MB) -2016-05-25 10:01:09 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 10:01:09 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:01:09 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 10:01:09 INFO DAGScheduler:59 - Got job 9 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 10:01:09 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at modelSaveLoad.scala:129) -2016-05-25 10:01:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:01:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:01:09 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 10:01:09 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=422072, maxMem=766075207 -2016-05-25 10:01:09 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 3.1 KB, free 730.2 MB) -2016-05-25 10:01:09 INFO MemoryStore:59 - ensureFreeSpace(1891) called with curMem=425288, maxMem=766075207 -2016-05-25 10:01:09 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 1891.0 B, free 730.2 MB) -2016-05-25 10:01:09 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:51292 (size: 1891.0 B, free: 730.6 MB) -2016-05-25 10:01:09 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:01:09 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129) -2016-05-25 10:01:09 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-25 10:01:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 16, localhost, PROCESS_LOCAL, 2239 bytes) -2016-05-25 10:01:09 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 16) -2016-05-25 10:01:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525100100516/result/metadata/part-00000:0+56 -2016-05-25 10:01:09 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 16). 2159 bytes result sent to driver -2016-05-25 10:01:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 16) in 19 ms on localhost (1/1) -2016-05-25 10:01:09 INFO DAGScheduler:59 - ResultStage 10 (first at modelSaveLoad.scala:129) finished in 0.019 s -2016-05-25 10:01:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 10:01:09 INFO DAGScheduler:59 - Job 9 finished: first at modelSaveLoad.scala:129, took 0.040325 s -2016-05-25 10:01:09 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525100100516/result/data on driver -2016-05-25 10:01:09 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:213 -2016-05-25 10:01:09 INFO DAGScheduler:59 - Got job 10 (parquet at NaiveBayes.scala:213) with 4 output partitions -2016-05-25 10:01:09 INFO DAGScheduler:59 - Final stage: ResultStage 11(parquet at NaiveBayes.scala:213) -2016-05-25 10:01:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:01:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:01:09 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213), which has no missing parents -2016-05-25 10:01:09 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=427179, maxMem=766075207 -2016-05-25 10:01:09 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 61.5 KB, free 730.1 MB) -2016-05-25 10:01:09 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=490179, maxMem=766075207 -2016-05-25 10:01:09 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.1 MB) -2016-05-25 10:01:09 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:51292 (size: 20.6 KB, free: 730.5 MB) -2016-05-25 10:01:09 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:01:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 11 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213) -2016-05-25 10:01:09 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 4 tasks -2016-05-25 10:01:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 17, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:01:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 18, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:01:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 11.0 (TID 19, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:01:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 11.0 (TID 20, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-25 10:01:09 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 17) -2016-05-25 10:01:09 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 18) -2016-05-25 10:01:09 INFO Executor:59 - Running task 2.0 in stage 11.0 (TID 19) -2016-05-25 10:01:09 INFO Executor:59 - Running task 3.0 in stage 11.0 (TID 20) -2016-05-25 10:01:09 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:01:09 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:01:09 INFO Executor:59 - Finished task 2.0 in stage 11.0 (TID 19). 936 bytes result sent to driver -2016-05-25 10:01:09 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:01:09 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:01:09 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 17). 936 bytes result sent to driver -2016-05-25 10:01:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 11.0 (TID 19) in 82 ms on localhost (1/4) -2016-05-25 10:01:09 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 18). 936 bytes result sent to driver -2016-05-25 10:01:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 17) in 88 ms on localhost (2/4) -2016-05-25 10:01:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 18) in 88 ms on localhost (3/4) -2016-05-25 10:01:09 INFO Executor:59 - Finished task 3.0 in stage 11.0 (TID 20). 1876 bytes result sent to driver -2016-05-25 10:01:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 11.0 (TID 20) in 139 ms on localhost (4/4) -2016-05-25 10:01:09 INFO DAGScheduler:59 - ResultStage 11 (parquet at NaiveBayes.scala:213) finished in 0.142 s -2016-05-25 10:01:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 10:01:09 INFO DAGScheduler:59 - Job 10 finished: parquet at NaiveBayes.scala:213, took 0.185584 s -2016-05-25 10:01:10 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=511252, maxMem=766075207 -2016-05-25 10:01:10 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 86.4 KB, free 730.0 MB) -2016-05-25 10:01:10 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=599764, maxMem=766075207 -2016-05-25 10:01:10 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.0 MB) -2016-05-25 10:01:10 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:51292 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 10:01:10 INFO SparkContext:59 - Created broadcast 14 from take at NaiveBayes.scala:216 -2016-05-25 10:01:10 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 10:01:10 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525100100516/result/data/part-r-00000-0928d565-87e5-4c2c-80f5-f883e9c64deb.gz.parquet -2016-05-25 10:01:10 INFO SparkContext:59 - Starting job: take at NaiveBayes.scala:216 -2016-05-25 10:01:10 INFO DAGScheduler:59 - Got job 11 (take at NaiveBayes.scala:216) with 1 output partitions -2016-05-25 10:01:10 INFO DAGScheduler:59 - Final stage: ResultStage 12(take at NaiveBayes.scala:216) -2016-05-25 10:01:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:01:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:01:10 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216), which has no missing parents -2016-05-25 10:01:10 INFO MemoryStore:59 - ensureFreeSpace(4280) called with curMem=619552, maxMem=766075207 -2016-05-25 10:01:10 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 4.2 KB, free 730.0 MB) -2016-05-25 10:01:10 INFO MemoryStore:59 - ensureFreeSpace(2466) called with curMem=623832, maxMem=766075207 -2016-05-25 10:01:10 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.0 MB) -2016-05-25 10:01:10 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:51292 (size: 2.4 KB, free: 730.5 MB) -2016-05-25 10:01:10 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:01:10 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216) -2016-05-25 10:01:10 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-25 10:01:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 21, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-25 10:01:10 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 21) -2016-05-25 10:01:10 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525100100516/result/data/part-r-00000-0928d565-87e5-4c2c-80f5-f883e9c64deb.gz.parquet start: 0 end: 1477 length: 1477 hosts: []} -2016-05-25 10:01:10 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 10:01:10 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group labels (LIST) { - repeated group bag { - optional double array; - } - } - optional group pi (LIST) { - repeated group bag { - optional double array; - } - } - optional group theta (LIST) { - repeated group bag { - optional group array (LIST) { - repeated group bag { - optional double array; - } - } - } - } - optional binary modelType (UTF8); -} - -Catalyst form: -StructType(StructField(labels,ArrayType(DoubleType,true),true), StructField(pi,ArrayType(DoubleType,true),true), StructField(theta,ArrayType(ArrayType(DoubleType,true),true),true), StructField(modelType,StringType,true)) - -2016-05-25 10:01:10 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 10:01:10 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 10:01:10 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 10:01:10 INFO InternalParquetRecordReader:151 - block read in memory in 36 ms. row count = 1 -2016-05-25 10:01:10 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 21). 2841 bytes result sent to driver -2016-05-25 10:01:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 21) in 173 ms on localhost (1/1) -2016-05-25 10:01:10 INFO DAGScheduler:59 - ResultStage 12 (take at NaiveBayes.scala:216) finished in 0.174 s -2016-05-25 10:01:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 10:01:10 INFO DAGScheduler:59 - Job 11 finished: take at NaiveBayes.scala:216, took 0.188806 s -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 10:01:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 10:01:11 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 10:01:11 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 10:01:11 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 10:01:11 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 10:01:11 INFO BlockManager:59 - BlockManager stopped -2016-05-25 10:01:11 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 10:01:11 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 10:01:11 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 10:01:11 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 10:01:11 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-1492960a-310e-476b-8d82-df9d7d3100a9 -2016-05-25 10:15:03 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 10:15:05 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 10:15:05 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 10:15:05 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 10:15:05 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 10:15:07 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 10:15:07 INFO Remoting:74 - Starting remoting -2016-05-25 10:15:07 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:51806] -2016-05-25 10:15:07 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51806. -2016-05-25 10:15:07 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 10:15:07 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 10:15:07 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-2d9a3f40-f721-4846-a2ec-ad85a5cb4cd2 -2016-05-25 10:15:08 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 10:15:08 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-cae0a584-238f-4f5b-b076-21ec253aff5c\httpd-6afddeb4-797f-4c4e-9827-d08a02b487e5 -2016-05-25 10:15:08 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 10:15:08 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:15:08 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51807 -2016-05-25 10:15:08 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51807. -2016-05-25 10:15:08 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 10:15:09 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:15:09 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 10:15:09 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 10:15:09 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 10:15:09 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 10:15:10 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 10:15:10 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51826. -2016-05-25 10:15:10 INFO NettyBlockTransferService:59 - Server created on 51826 -2016-05-25 10:15:10 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 10:15:10 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51826 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51826) -2016-05-25 10:15:10 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 10:15:13 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 10:15:13 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 10:15:13 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 10:15:13 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 10:15:13 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51826 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 10:15:13 INFO SparkContext:59 - Created broadcast 0 from textFile at NaiveBayesLearning.scala:26 -2016-05-25 10:15:15 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 10:15:16 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:15:20 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:15:20 INFO DAGScheduler:59 - Registering RDD 5 (map at NaiveBayes.scala:383) -2016-05-25 10:15:20 INFO DAGScheduler:59 - Got job 0 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:15:20 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at NaiveBayes.scala:401) -2016-05-25 10:15:20 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-05-25 10:15:20 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-05-25 10:15:20 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:15:20 INFO MemoryStore:59 - ensureFreeSpace(5104) called with curMem=171580, maxMem=766075207 -2016-05-25 10:15:20 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 10:15:20 INFO MemoryStore:59 - ensureFreeSpace(2692) called with curMem=176684, maxMem=766075207 -2016-05-25 10:15:20 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:15:20 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51826 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:15:20 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:15:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383) -2016-05-25 10:15:20 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 10:15:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 10:15:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 10:15:20 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 10:15:20 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 10:15:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:15:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:15:20 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 10:15:20 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 10:15:20 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 10:15:20 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 10:15:20 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 10:15:20 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 10:15:20 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 10:15:20 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver -2016-05-25 10:15:20 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver -2016-05-25 10:15:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 435 ms on localhost (1/2) -2016-05-25 10:15:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 418 ms on localhost (2/2) -2016-05-25 10:15:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 10:15:20 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at NaiveBayes.scala:383) finished in 0.458 s -2016-05-25 10:15:20 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 10:15:20 INFO DAGScheduler:59 - running: Set() -2016-05-25 10:15:20 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-05-25 10:15:20 INFO DAGScheduler:59 - failed: Set() -2016-05-25 10:15:20 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-05-25 10:15:20 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 10:15:20 INFO MemoryStore:59 - ensureFreeSpace(2936) called with curMem=179376, maxMem=766075207 -2016-05-25 10:15:20 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-25 10:15:20 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=182312, maxMem=766075207 -2016-05-25 10:15:20 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.4 MB) -2016-05-25 10:15:20 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51826 (size: 1567.0 B, free: 730.6 MB) -2016-05-25 10:15:20 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:15:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383) -2016-05-25 10:15:20 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-25 10:15:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:15:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:15:20 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-25 10:15:20 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 10:15:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 10:15:20 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 10:15:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 10:15:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-25 10:15:20 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1645 bytes result sent to driver -2016-05-25 10:15:20 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-05-25 10:15:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 51 ms on localhost (1/2) -2016-05-25 10:15:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 61 ms on localhost (2/2) -2016-05-25 10:15:20 INFO DAGScheduler:59 - ResultStage 1 (collect at NaiveBayes.scala:401) finished in 0.063 s -2016-05-25 10:15:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 10:15:20 INFO DAGScheduler:59 - Job 0 finished: collect at NaiveBayes.scala:401, took 0.970401 s -2016-05-25 10:15:21 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:39 -2016-05-25 10:15:21 INFO DAGScheduler:59 - Got job 1 (count at NaiveBayesLearning.scala:39) with 2 output partitions -2016-05-25 10:15:21 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at NaiveBayesLearning.scala:39) -2016-05-25 10:15:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:15:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:15:21 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-25 10:15:21 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=183879, maxMem=766075207 -2016-05-25 10:15:21 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-25 10:15:21 INFO MemoryStore:59 - ensureFreeSpace(2671) called with curMem=188775, maxMem=766075207 -2016-05-25 10:15:21 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:15:21 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:51826 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:15:21 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:15:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:39) -2016-05-25 10:15:21 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 10:15:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:15:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:15:21 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-25 10:15:21 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-25 10:15:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:15:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:15:21 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 10:15:21 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2082 bytes result sent to driver -2016-05-25 10:15:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 34 ms on localhost (1/2) -2016-05-25 10:15:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 37 ms on localhost (2/2) -2016-05-25 10:15:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 10:15:21 INFO DAGScheduler:59 - ResultStage 2 (count at NaiveBayesLearning.scala:39) finished in 0.037 s -2016-05-25 10:15:21 INFO DAGScheduler:59 - Job 1 finished: count at NaiveBayesLearning.scala:39, took 0.055293 s -2016-05-25 10:15:21 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:39 -2016-05-25 10:15:21 INFO DAGScheduler:59 - Got job 2 (count at NaiveBayesLearning.scala:39) with 2 output partitions -2016-05-25 10:15:21 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at NaiveBayesLearning.scala:39) -2016-05-25 10:15:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:15:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:15:21 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-25 10:15:21 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=191446, maxMem=766075207 -2016-05-25 10:15:21 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:15:21 INFO MemoryStore:59 - ensureFreeSpace(2117) called with curMem=195046, maxMem=766075207 -2016-05-25 10:15:21 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:15:21 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:51826 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:15:21 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:15:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-25 10:15:21 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 10:15:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:15:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:15:21 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-25 10:15:21 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-25 10:15:23 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:15:23 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-25 10:15:23 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:15:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 1965 ms on localhost (1/2) -2016-05-25 10:15:23 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 2082 bytes result sent to driver -2016-05-25 10:15:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 1978 ms on localhost (2/2) -2016-05-25 10:15:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 10:15:23 INFO DAGScheduler:59 - ResultStage 3 (count at NaiveBayesLearning.scala:39) finished in 1.979 s -2016-05-25 10:15:23 INFO DAGScheduler:59 - Job 2 finished: count at NaiveBayesLearning.scala:39, took 2.013264 s -2016-05-25 10:15:24 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:43 -2016-05-25 10:15:24 INFO DAGScheduler:59 - Got job 3 (count at NaiveBayesLearning.scala:43) with 2 output partitions -2016-05-25 10:15:24 INFO DAGScheduler:59 - Final stage: ResultStage 4(count at NaiveBayesLearning.scala:43) -2016-05-25 10:15:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:15:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:15:24 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-25 10:15:24 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=197163, maxMem=766075207 -2016-05-25 10:15:24 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:15:24 INFO MemoryStore:59 - ensureFreeSpace(2113) called with curMem=200763, maxMem=766075207 -2016-05-25 10:15:24 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:15:24 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:51826 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:15:24 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:51826 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:15:24 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:15:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-25 10:15:24 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 10:15:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:15:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:15:24 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-25 10:15:24 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:15:24 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-25 10:15:24 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:15:24 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2082 bytes result sent to driver -2016-05-25 10:15:24 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2082 bytes result sent to driver -2016-05-25 10:15:24 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 10:15:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 38 ms on localhost (1/2) -2016-05-25 10:15:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 46 ms on localhost (2/2) -2016-05-25 10:15:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 10:15:24 INFO DAGScheduler:59 - ResultStage 4 (count at NaiveBayesLearning.scala:43) finished in 0.054 s -2016-05-25 10:15:24 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:51826 in memory (size: 1567.0 B, free: 730.6 MB) -2016-05-25 10:15:24 INFO DAGScheduler:59 - Job 3 finished: count at NaiveBayesLearning.scala:43, took 0.089078 s -2016-05-25 10:15:24 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 10:15:24 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:51826 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:15:24 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 10:15:24 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 10:15:25 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:44 -2016-05-25 10:15:25 INFO DAGScheduler:59 - Got job 4 (foreach at NaiveBayesLearning.scala:44) with 2 output partitions -2016-05-25 10:15:25 INFO DAGScheduler:59 - Final stage: ResultStage 5(foreach at NaiveBayesLearning.scala:44) -2016-05-25 10:15:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:15:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:15:25 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-25 10:15:25 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=183010, maxMem=766075207 -2016-05-25 10:15:25 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-25 10:15:25 INFO MemoryStore:59 - ensureFreeSpace(2143) called with curMem=186746, maxMem=766075207 -2016-05-25 10:15:25 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:15:25 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:51826 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:15:25 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:15:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-25 10:15:25 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 10:15:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:15:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:15:25 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-05-25 10:15:25 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-05-25 10:15:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:15:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:15:25 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 2044 bytes result sent to driver -2016-05-25 10:15:25 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 2044 bytes result sent to driver -2016-05-25 10:15:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 40 ms on localhost (1/2) -2016-05-25 10:15:25 INFO DAGScheduler:59 - ResultStage 5 (foreach at NaiveBayesLearning.scala:44) finished in 0.045 s -2016-05-25 10:15:25 INFO DAGScheduler:59 - Job 4 finished: foreach at NaiveBayesLearning.scala:44, took 0.062236 s -2016-05-25 10:15:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 44 ms on localhost (2/2) -2016-05-25 10:15:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 10:15:25 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:45 -2016-05-25 10:15:25 INFO DAGScheduler:59 - Got job 5 (count at NaiveBayesLearning.scala:45) with 2 output partitions -2016-05-25 10:15:25 INFO DAGScheduler:59 - Final stage: ResultStage 6(count at NaiveBayesLearning.scala:45) -2016-05-25 10:15:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:15:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:15:25 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-25 10:15:25 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=188889, maxMem=766075207 -2016-05-25 10:15:25 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:15:25 INFO MemoryStore:59 - ensureFreeSpace(2117) called with curMem=192489, maxMem=766075207 -2016-05-25 10:15:25 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:15:25 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:51826 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:15:25 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:15:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-25 10:15:25 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 10:15:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:15:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:15:25 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 13) -2016-05-25 10:15:25 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-05-25 10:15:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:15:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:15:25 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 13). 2082 bytes result sent to driver -2016-05-25 10:15:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 13) in 23 ms on localhost (1/2) -2016-05-25 10:15:25 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 2082 bytes result sent to driver -2016-05-25 10:15:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 32 ms on localhost (2/2) -2016-05-25 10:15:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 10:15:25 INFO DAGScheduler:59 - ResultStage 6 (count at NaiveBayesLearning.scala:45) finished in 0.048 s -2016-05-25 10:15:25 INFO DAGScheduler:59 - Job 5 finished: count at NaiveBayesLearning.scala:45, took 0.071077 s -2016-05-25 10:15:25 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:46 -2016-05-25 10:15:25 INFO DAGScheduler:59 - Got job 6 (foreach at NaiveBayesLearning.scala:46) with 2 output partitions -2016-05-25 10:15:25 INFO DAGScheduler:59 - Final stage: ResultStage 7(foreach at NaiveBayesLearning.scala:46) -2016-05-25 10:15:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:15:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:15:25 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-25 10:15:25 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=194606, maxMem=766075207 -2016-05-25 10:15:25 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-25 10:15:25 INFO MemoryStore:59 - ensureFreeSpace(2147) called with curMem=198342, maxMem=766075207 -2016-05-25 10:15:25 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:15:25 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:51826 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:15:25 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:15:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-25 10:15:25 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 10:15:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:15:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 15, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:15:25 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 15) -2016-05-25 10:15:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:15:25 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 14) -2016-05-25 10:15:25 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 15). 2044 bytes result sent to driver -2016-05-25 10:15:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 15) in 20 ms on localhost (1/2) -2016-05-25 10:15:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:15:25 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 14). 2044 bytes result sent to driver -2016-05-25 10:15:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 14) in 46 ms on localhost (2/2) -2016-05-25 10:15:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 10:15:25 INFO DAGScheduler:59 - ResultStage 7 (foreach at NaiveBayesLearning.scala:46) finished in 0.053 s -2016-05-25 10:15:25 INFO DAGScheduler:59 - Job 6 finished: foreach at NaiveBayesLearning.scala:46, took 0.227373 s -2016-05-25 10:15:25 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:49 -2016-05-25 10:15:25 INFO DAGScheduler:59 - Got job 7 (take at NaiveBayesLearning.scala:49) with 1 output partitions -2016-05-25 10:15:25 INFO DAGScheduler:59 - Final stage: ResultStage 8(take at NaiveBayesLearning.scala:49) -2016-05-25 10:15:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:15:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:15:25 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:38), which has no missing parents -2016-05-25 10:15:25 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=200489, maxMem=766075207 -2016-05-25 10:15:25 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 10:15:25 INFO MemoryStore:59 - ensureFreeSpace(2669) called with curMem=205313, maxMem=766075207 -2016-05-25 10:15:25 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:15:25 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:51826 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:15:25 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:15:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:38) -2016-05-25 10:15:25 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-05-25 10:15:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:15:25 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 16) -2016-05-25 10:15:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:15:25 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 16). 2040 bytes result sent to driver -2016-05-25 10:15:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 16) in 32 ms on localhost (1/1) -2016-05-25 10:15:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 10:15:25 INFO DAGScheduler:59 - ResultStage 8 (take at NaiveBayesLearning.scala:49) finished in 0.037 s -2016-05-25 10:15:25 INFO DAGScheduler:59 - Job 7 finished: take at NaiveBayesLearning.scala:49, took 0.117875 s -2016-05-25 10:15:25 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:49 -2016-05-25 10:15:25 INFO DAGScheduler:59 - Got job 8 (take at NaiveBayesLearning.scala:49) with 1 output partitions -2016-05-25 10:15:25 INFO DAGScheduler:59 - Final stage: ResultStage 9(take at NaiveBayesLearning.scala:49) -2016-05-25 10:15:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:15:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:15:25 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:38), which has no missing parents -2016-05-25 10:15:25 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=207982, maxMem=766075207 -2016-05-25 10:15:25 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 10:15:25 INFO MemoryStore:59 - ensureFreeSpace(2669) called with curMem=212806, maxMem=766075207 -2016-05-25 10:15:25 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:15:25 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:51826 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:15:25 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:15:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 9 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:38) -2016-05-25 10:15:25 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 1 tasks -2016-05-25 10:15:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:15:25 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 10:15:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:15:25 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2206 bytes result sent to driver -2016-05-25 10:15:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 42 ms on localhost (1/1) -2016-05-25 10:15:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 10:15:25 INFO DAGScheduler:59 - ResultStage 9 (take at NaiveBayesLearning.scala:49) finished in 0.050 s -2016-05-25 10:15:25 INFO DAGScheduler:59 - Job 8 finished: take at NaiveBayesLearning.scala:49, took 0.153270 s -2016-05-25 10:15:26 INFO SparkContext:59 - Starting job: saveAsTextFile at NaiveBayes.scala:202 -2016-05-25 10:15:26 INFO DAGScheduler:59 - Got job 9 (saveAsTextFile at NaiveBayes.scala:202) with 1 output partitions -2016-05-25 10:15:26 INFO DAGScheduler:59 - Final stage: ResultStage 10(saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 10:15:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:15:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:15:26 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202), which has no missing parents -2016-05-25 10:15:26 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=215475, maxMem=766075207 -2016-05-25 10:15:26 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 124.0 KB, free 730.3 MB) -2016-05-25 10:15:26 INFO MemoryStore:59 - ensureFreeSpace(42583) called with curMem=342467, maxMem=766075207 -2016-05-25 10:15:26 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.2 MB) -2016-05-25 10:15:26 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:51826 (size: 41.6 KB, free: 730.5 MB) -2016-05-25 10:15:26 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:15:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 10:15:26 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-25 10:15:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 18, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-25 10:15:26 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 18) -2016-05-25 10:15:27 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251015_0010_m_000000_18' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525101525822/result/metadata/_temporary/0/task_201605251015_0010_m_000000 -2016-05-25 10:15:27 INFO SparkHadoopMapRedUtil:59 - attempt_201605251015_0010_m_000000_18: Committed -2016-05-25 10:15:27 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 18). 1864 bytes result sent to driver -2016-05-25 10:15:27 INFO DAGScheduler:59 - ResultStage 10 (saveAsTextFile at NaiveBayes.scala:202) finished in 0.907 s -2016-05-25 10:15:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 18) in 896 ms on localhost (1/1) -2016-05-25 10:15:27 INFO DAGScheduler:59 - Job 9 finished: saveAsTextFile at NaiveBayes.scala:202, took 1.028118 s -2016-05-25 10:15:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 10:15:28 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:51826 in memory (size: 41.6 KB, free: 730.6 MB) -2016-05-25 10:15:28 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 10:15:28 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:51826 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:15:28 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 10:15:28 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:51826 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:15:28 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 10:15:28 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:51826 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:15:28 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 10:15:28 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:51826 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:15:28 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 10:15:28 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:51826 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:15:28 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 10:15:28 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:51826 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:15:28 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 10:15:28 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:51826 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:15:28 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 10:15:30 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:15:30 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:15:30 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:206 -2016-05-25 10:15:30 INFO DAGScheduler:59 - Got job 10 (parquet at NaiveBayes.scala:206) with 1 output partitions -2016-05-25 10:15:30 INFO DAGScheduler:59 - Final stage: ResultStage 11(parquet at NaiveBayes.scala:206) -2016-05-25 10:15:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:15:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:15:30 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205), which has no missing parents -2016-05-25 10:15:30 INFO MemoryStore:59 - ensureFreeSpace(66936) called with curMem=171580, maxMem=766075207 -2016-05-25 10:15:30 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 65.4 KB, free 730.4 MB) -2016-05-25 10:15:30 INFO MemoryStore:59 - ensureFreeSpace(23362) called with curMem=238516, maxMem=766075207 -2016-05-25 10:15:30 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 22.8 KB, free 730.3 MB) -2016-05-25 10:15:30 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:51826 (size: 22.8 KB, free: 730.6 MB) -2016-05-25 10:15:30 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:15:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 11 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205) -2016-05-25 10:15:30 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 1 tasks -2016-05-25 10:15:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 19, localhost, PROCESS_LOCAL, 2506 bytes) -2016-05-25 10:15:30 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 19) -2016-05-25 10:15:30 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:15:30 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 10:15:30 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 10:15:30 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 10:15:30 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 10:15:30 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 10:15:30 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 10:15:30 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 10:15:30 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 10:15:31 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 231 -2016-05-25 10:15:31 INFO ColumnChunkPageWriteStore:151 - written 77B for [labels, bag, array] DOUBLE: 3 values, 37B raw, 36B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 10:15:31 INFO ColumnChunkPageWriteStore:151 - written 74B for [pi, bag, array] DOUBLE: 3 values, 16B raw, 33B comp, 1 pages, encodings: [RLE, PLAIN_DICTIONARY], dic { 2 entries, 16B raw, 2B comp} -2016-05-25 10:15:31 INFO ColumnChunkPageWriteStore:151 - written 83B for [theta, bag, array, bag, array] DOUBLE: 9 values, 23B raw, 42B comp, 1 pages, encodings: [RLE, PLAIN_DICTIONARY], dic { 6 entries, 48B raw, 6B comp} -2016-05-25 10:15:31 INFO ColumnChunkPageWriteStore:151 - written 86B for [modelType] BINARY: 1 values, 21B raw, 39B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 10:15:31 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251015_0011_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525101525822/result/data/_temporary/0/task_201605251015_0011_m_000000 -2016-05-25 10:15:31 INFO SparkHadoopMapRedUtil:59 - attempt_201605251015_0011_m_000000_0: Committed -2016-05-25 10:15:31 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 19). 915 bytes result sent to driver -2016-05-25 10:15:31 INFO DAGScheduler:59 - ResultStage 11 (parquet at NaiveBayes.scala:206) finished in 1.670 s -2016-05-25 10:15:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 19) in 1670 ms on localhost (1/1) -2016-05-25 10:15:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 10:15:31 INFO DAGScheduler:59 - Job 10 finished: parquet at NaiveBayes.scala:206, took 1.729642 s -2016-05-25 10:15:35 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:15:37 INFO DefaultWriterContainer:59 - Job job_201605251015_0000 committed. -2016-05-25 10:15:37 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525101525822/result/data on driver -2016-05-25 10:15:37 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525101525822/result/data on driver -2016-05-25 10:15:38 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=261878, maxMem=766075207 -2016-05-25 10:15:38 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-25 10:15:38 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=492582, maxMem=766075207 -2016-05-25 10:15:38 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-25 10:15:38 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:51826 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 10:15:38 INFO SparkContext:59 - Created broadcast 13 from textFile at modelSaveLoad.scala:129 -2016-05-25 10:15:40 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:15:40 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 10:15:40 INFO DAGScheduler:59 - Got job 11 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 10:15:40 INFO DAGScheduler:59 - Final stage: ResultStage 12(first at modelSaveLoad.scala:129) -2016-05-25 10:15:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:15:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:15:40 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 10:15:40 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=512370, maxMem=766075207 -2016-05-25 10:15:40 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-25 10:15:40 INFO MemoryStore:59 - ensureFreeSpace(1892) called with curMem=515586, maxMem=766075207 -2016-05-25 10:15:40 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 1892.0 B, free 730.1 MB) -2016-05-25 10:15:40 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:51826 (size: 1892.0 B, free: 730.5 MB) -2016-05-25 10:15:40 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:15:40 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129) -2016-05-25 10:15:40 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-25 10:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 20, localhost, PROCESS_LOCAL, 2239 bytes) -2016-05-25 10:15:40 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 20) -2016-05-25 10:15:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525101525822/result/metadata/part-00000:0+56 -2016-05-25 10:15:40 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 20). 2159 bytes result sent to driver -2016-05-25 10:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 20) in 17 ms on localhost (1/1) -2016-05-25 10:15:40 INFO DAGScheduler:59 - ResultStage 12 (first at modelSaveLoad.scala:129) finished in 0.019 s -2016-05-25 10:15:40 INFO DAGScheduler:59 - Job 11 finished: first at modelSaveLoad.scala:129, took 0.036003 s -2016-05-25 10:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 10:15:41 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525101525822/result/data on driver -2016-05-25 10:15:41 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:213 -2016-05-25 10:15:41 INFO DAGScheduler:59 - Got job 12 (parquet at NaiveBayes.scala:213) with 4 output partitions -2016-05-25 10:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 13(parquet at NaiveBayes.scala:213) -2016-05-25 10:15:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:15:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:15:41 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213), which has no missing parents -2016-05-25 10:15:41 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=517478, maxMem=766075207 -2016-05-25 10:15:41 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-25 10:15:41 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=580478, maxMem=766075207 -2016-05-25 10:15:41 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.0 MB) -2016-05-25 10:15:41 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:51826 (size: 20.6 KB, free: 730.5 MB) -2016-05-25 10:15:41 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:15:41 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 13 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213) -2016-05-25 10:15:41 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 4 tasks -2016-05-25 10:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 21, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 22, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:15:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 13.0 (TID 23, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:15:41 INFO TaskSetManager:59 - Starting task 3.0 in stage 13.0 (TID 24, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-25 10:15:41 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 21) -2016-05-25 10:15:41 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 22) -2016-05-25 10:15:41 INFO Executor:59 - Running task 2.0 in stage 13.0 (TID 23) -2016-05-25 10:15:41 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:15:41 INFO Executor:59 - Finished task 2.0 in stage 13.0 (TID 23). 936 bytes result sent to driver -2016-05-25 10:15:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 13.0 (TID 23) in 60 ms on localhost (1/4) -2016-05-25 10:15:41 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:15:41 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 22). 936 bytes result sent to driver -2016-05-25 10:15:41 INFO Executor:59 - Running task 3.0 in stage 13.0 (TID 24) -2016-05-25 10:15:41 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:15:41 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:15:41 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 21). 936 bytes result sent to driver -2016-05-25 10:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 22) in 173 ms on localhost (2/4) -2016-05-25 10:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 21) in 176 ms on localhost (3/4) -2016-05-25 10:15:43 INFO Executor:59 - Finished task 3.0 in stage 13.0 (TID 24). 1876 bytes result sent to driver -2016-05-25 10:15:43 INFO TaskSetManager:59 - Finished task 3.0 in stage 13.0 (TID 24) in 1372 ms on localhost (4/4) -2016-05-25 10:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 10:15:43 INFO DAGScheduler:59 - ResultStage 13 (parquet at NaiveBayes.scala:213) finished in 1.379 s -2016-05-25 10:15:43 INFO DAGScheduler:59 - Job 12 finished: parquet at NaiveBayes.scala:213, took 1.436403 s -2016-05-25 10:15:45 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=601551, maxMem=766075207 -2016-05-25 10:15:45 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 86.4 KB, free 729.9 MB) -2016-05-25 10:15:45 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=690063, maxMem=766075207 -2016-05-25 10:15:45 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-25 10:15:45 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:51826 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 10:15:45 INFO SparkContext:59 - Created broadcast 16 from take at NaiveBayes.scala:216 -2016-05-25 10:15:46 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 10:15:47 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525101525822/result/data/part-r-00000-044ee1fc-5709-4f8e-a895-a87261b30027.gz.parquet -2016-05-25 10:15:47 INFO SparkContext:59 - Starting job: take at NaiveBayes.scala:216 -2016-05-25 10:15:47 INFO DAGScheduler:59 - Got job 13 (take at NaiveBayes.scala:216) with 1 output partitions -2016-05-25 10:15:47 INFO DAGScheduler:59 - Final stage: ResultStage 14(take at NaiveBayes.scala:216) -2016-05-25 10:15:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:15:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:15:47 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216), which has no missing parents -2016-05-25 10:15:47 INFO MemoryStore:59 - ensureFreeSpace(4280) called with curMem=709851, maxMem=766075207 -2016-05-25 10:15:47 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 4.2 KB, free 729.9 MB) -2016-05-25 10:15:47 INFO MemoryStore:59 - ensureFreeSpace(2466) called with curMem=714131, maxMem=766075207 -2016-05-25 10:15:47 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.4 KB, free 729.9 MB) -2016-05-25 10:15:47 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:51826 (size: 2.4 KB, free: 730.5 MB) -2016-05-25 10:15:47 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:15:47 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 14 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216) -2016-05-25 10:15:47 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 1 tasks -2016-05-25 10:15:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 25, localhost, PROCESS_LOCAL, 2303 bytes) -2016-05-25 10:15:47 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 25) -2016-05-25 10:15:47 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525101525822/result/data/part-r-00000-044ee1fc-5709-4f8e-a895-a87261b30027.gz.parquet start: 0 end: 1477 length: 1477 hosts: []} -2016-05-25 10:15:47 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 10:15:47 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group labels (LIST) { - repeated group bag { - optional double array; - } - } - optional group pi (LIST) { - repeated group bag { - optional double array; - } - } - optional group theta (LIST) { - repeated group bag { - optional group array (LIST) { - repeated group bag { - optional double array; - } - } - } - } - optional binary modelType (UTF8); -} - -Catalyst form: -StructType(StructField(labels,ArrayType(DoubleType,true),true), StructField(pi,ArrayType(DoubleType,true),true), StructField(theta,ArrayType(ArrayType(DoubleType,true),true),true), StructField(modelType,StringType,true)) - -2016-05-25 10:15:48 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 10:15:48 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 10:15:49 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 10:15:49 INFO InternalParquetRecordReader:151 - block read in memory in 1294 ms. row count = 1 -2016-05-25 10:15:49 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 25). 2841 bytes result sent to driver -2016-05-25 10:15:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 25) in 2730 ms on localhost (1/1) -2016-05-25 10:15:49 INFO DAGScheduler:59 - ResultStage 14 (take at NaiveBayes.scala:216) finished in 2.730 s -2016-05-25 10:15:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 10:15:49 INFO DAGScheduler:59 - Job 13 finished: take at NaiveBayes.scala:216, took 2.756470 s -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 10:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 10:15:50 WARN QueuedThreadPool:145 - 8 threads could not be stopped -2016-05-25 10:15:50 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 10:15:50 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 10:15:51 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 10:15:51 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 10:15:51 INFO BlockManager:59 - BlockManager stopped -2016-05-25 10:15:51 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 10:15:51 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 10:15:52 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 10:15:52 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-25 10:15:52 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 10:15:52 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 10:15:52 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-cae0a584-238f-4f5b-b076-21ec253aff5c -2016-05-25 10:29:02 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 10:29:03 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 10:29:03 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 10:29:03 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 10:29:03 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 10:29:04 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 10:29:04 INFO Remoting:74 - Starting remoting -2016-05-25 10:29:04 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:52465] -2016-05-25 10:29:04 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52465. -2016-05-25 10:29:04 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 10:29:04 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 10:29:04 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-b743ad83-01b5-4886-af7b-954831a40f8f -2016-05-25 10:29:04 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 10:29:04 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-651868d8-4032-4e99-a85a-33b0da0e6b39\httpd-2f713edb-2370-4111-8637-d2b4d2d3e078 -2016-05-25 10:29:04 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 10:29:04 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:29:04 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52466 -2016-05-25 10:29:04 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52466. -2016-05-25 10:29:05 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 10:29:05 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:29:05 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 10:29:05 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 10:29:05 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 10:29:05 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 10:29:05 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 10:29:05 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52485. -2016-05-25 10:29:05 INFO NettyBlockTransferService:59 - Server created on 52485 -2016-05-25 10:29:05 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 10:29:05 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52485 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52485) -2016-05-25 10:29:05 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 10:29:06 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 10:29:06 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 10:29:06 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 10:29:06 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 10:29:06 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52485 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 10:29:06 INFO SparkContext:59 - Created broadcast 0 from textFile at NaiveBayesLearning.scala:26 -2016-05-25 10:29:07 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 10:29:08 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:29:08 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:29:08 INFO DAGScheduler:59 - Registering RDD 5 (map at NaiveBayes.scala:383) -2016-05-25 10:29:08 INFO DAGScheduler:59 - Got job 0 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:29:08 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at NaiveBayes.scala:401) -2016-05-25 10:29:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-05-25 10:29:08 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-05-25 10:29:08 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:29:08 INFO MemoryStore:59 - ensureFreeSpace(5104) called with curMem=171580, maxMem=766075207 -2016-05-25 10:29:08 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 10:29:08 INFO MemoryStore:59 - ensureFreeSpace(2697) called with curMem=176684, maxMem=766075207 -2016-05-25 10:29:08 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:29:08 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52485 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:29:08 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383) -2016-05-25 10:29:08 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 10:29:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 10:29:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 10:29:08 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 10:29:08 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 10:29:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:29:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:29:08 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 10:29:08 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 10:29:08 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 10:29:08 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 10:29:08 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 10:29:08 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 10:29:08 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 10:29:08 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver -2016-05-25 10:29:08 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver -2016-05-25 10:29:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 178 ms on localhost (1/2) -2016-05-25 10:29:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 159 ms on localhost (2/2) -2016-05-25 10:29:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 10:29:08 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at NaiveBayes.scala:383) finished in 0.195 s -2016-05-25 10:29:08 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 10:29:08 INFO DAGScheduler:59 - running: Set() -2016-05-25 10:29:08 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-05-25 10:29:08 INFO DAGScheduler:59 - failed: Set() -2016-05-25 10:29:08 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-05-25 10:29:08 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 10:29:08 INFO MemoryStore:59 - ensureFreeSpace(2936) called with curMem=179381, maxMem=766075207 -2016-05-25 10:29:08 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-25 10:29:08 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=182317, maxMem=766075207 -2016-05-25 10:29:08 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.4 MB) -2016-05-25 10:29:08 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52485 (size: 1567.0 B, free: 730.6 MB) -2016-05-25 10:29:08 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383) -2016-05-25 10:29:08 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-25 10:29:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:29:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:29:08 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-25 10:29:08 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 10:29:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 10:29:08 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 10:29:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 10:29:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 10:29:08 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-05-25 10:29:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 32 ms on localhost (1/2) -2016-05-25 10:29:08 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1645 bytes result sent to driver -2016-05-25 10:29:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 39 ms on localhost (2/2) -2016-05-25 10:29:08 INFO DAGScheduler:59 - ResultStage 1 (collect at NaiveBayes.scala:401) finished in 0.039 s -2016-05-25 10:29:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 10:29:08 INFO DAGScheduler:59 - Job 0 finished: collect at NaiveBayes.scala:401, took 0.354649 s -2016-05-25 10:29:08 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:39 -2016-05-25 10:29:08 INFO DAGScheduler:59 - Got job 1 (count at NaiveBayesLearning.scala:39) with 2 output partitions -2016-05-25 10:29:08 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at NaiveBayesLearning.scala:39) -2016-05-25 10:29:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:08 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-25 10:29:08 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=183884, maxMem=766075207 -2016-05-25 10:29:08 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-25 10:29:08 INFO MemoryStore:59 - ensureFreeSpace(2670) called with curMem=188780, maxMem=766075207 -2016-05-25 10:29:08 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:29:08 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52485 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:29:08 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:39) -2016-05-25 10:29:08 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 10:29:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:08 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-25 10:29:08 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-25 10:29:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:29:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:29:08 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52485 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:29:08 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2082 bytes result sent to driver -2016-05-25 10:29:08 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 10:29:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 24 ms on localhost (1/2) -2016-05-25 10:29:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 29 ms on localhost (2/2) -2016-05-25 10:29:08 INFO DAGScheduler:59 - ResultStage 2 (count at NaiveBayesLearning.scala:39) finished in 0.029 s -2016-05-25 10:29:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 10:29:08 INFO DAGScheduler:59 - Job 1 finished: count at NaiveBayesLearning.scala:39, took 0.070942 s -2016-05-25 10:29:08 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52485 in memory (size: 1567.0 B, free: 730.6 MB) -2016-05-25 10:29:08 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:39 -2016-05-25 10:29:08 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 10:29:08 INFO DAGScheduler:59 - Got job 2 (count at NaiveBayesLearning.scala:39) with 2 output partitions -2016-05-25 10:29:08 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at NaiveBayesLearning.scala:39) -2016-05-25 10:29:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:08 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-25 10:29:08 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=179146, maxMem=766075207 -2016-05-25 10:29:08 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:29:08 INFO MemoryStore:59 - ensureFreeSpace(2116) called with curMem=182746, maxMem=766075207 -2016-05-25 10:29:08 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:29:08 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52485 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:29:08 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-25 10:29:08 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 10:29:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:08 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-25 10:29:08 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-25 10:29:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:29:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:29:08 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-25 10:29:08 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 2082 bytes result sent to driver -2016-05-25 10:29:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 19 ms on localhost (1/2) -2016-05-25 10:29:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 18 ms on localhost (2/2) -2016-05-25 10:29:08 INFO DAGScheduler:59 - ResultStage 3 (count at NaiveBayesLearning.scala:39) finished in 0.020 s -2016-05-25 10:29:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 10:29:08 INFO DAGScheduler:59 - Job 2 finished: count at NaiveBayesLearning.scala:39, took 0.033562 s -2016-05-25 10:29:08 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:43 -2016-05-25 10:29:08 INFO DAGScheduler:59 - Got job 3 (count at NaiveBayesLearning.scala:43) with 2 output partitions -2016-05-25 10:29:08 INFO DAGScheduler:59 - Final stage: ResultStage 4(count at NaiveBayesLearning.scala:43) -2016-05-25 10:29:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:08 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-25 10:29:08 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=184862, maxMem=766075207 -2016-05-25 10:29:08 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:29:08 INFO MemoryStore:59 - ensureFreeSpace(2112) called with curMem=188462, maxMem=766075207 -2016-05-25 10:29:08 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:29:08 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52485 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:29:08 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-25 10:29:08 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 10:29:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:08 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-25 10:29:08 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-25 10:29:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:29:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:29:08 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2082 bytes result sent to driver -2016-05-25 10:29:08 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2082 bytes result sent to driver -2016-05-25 10:29:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 16 ms on localhost (1/2) -2016-05-25 10:29:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 19 ms on localhost (2/2) -2016-05-25 10:29:08 INFO DAGScheduler:59 - ResultStage 4 (count at NaiveBayesLearning.scala:43) finished in 0.019 s -2016-05-25 10:29:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 10:29:08 INFO DAGScheduler:59 - Job 3 finished: count at NaiveBayesLearning.scala:43, took 0.031909 s -2016-05-25 10:29:08 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:44 -2016-05-25 10:29:08 INFO DAGScheduler:59 - Got job 4 (foreach at NaiveBayesLearning.scala:44) with 2 output partitions -2016-05-25 10:29:08 INFO DAGScheduler:59 - Final stage: ResultStage 5(foreach at NaiveBayesLearning.scala:44) -2016-05-25 10:29:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:08 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-25 10:29:08 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=190574, maxMem=766075207 -2016-05-25 10:29:08 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-25 10:29:08 INFO MemoryStore:59 - ensureFreeSpace(2142) called with curMem=194310, maxMem=766075207 -2016-05-25 10:29:08 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:29:08 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52485 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:29:08 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-25 10:29:08 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 10:29:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:08 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-05-25 10:29:08 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-05-25 10:29:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:29:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:29:08 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 2044 bytes result sent to driver -2016-05-25 10:29:08 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 2044 bytes result sent to driver -2016-05-25 10:29:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 19 ms on localhost (1/2) -2016-05-25 10:29:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 21 ms on localhost (2/2) -2016-05-25 10:29:08 INFO DAGScheduler:59 - ResultStage 5 (foreach at NaiveBayesLearning.scala:44) finished in 0.021 s -2016-05-25 10:29:08 INFO DAGScheduler:59 - Job 4 finished: foreach at NaiveBayesLearning.scala:44, took 0.035086 s -2016-05-25 10:29:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 10:29:08 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:45 -2016-05-25 10:29:08 INFO DAGScheduler:59 - Got job 5 (count at NaiveBayesLearning.scala:45) with 2 output partitions -2016-05-25 10:29:08 INFO DAGScheduler:59 - Final stage: ResultStage 6(count at NaiveBayesLearning.scala:45) -2016-05-25 10:29:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:08 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-25 10:29:08 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=196452, maxMem=766075207 -2016-05-25 10:29:08 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:29:08 INFO MemoryStore:59 - ensureFreeSpace(2116) called with curMem=200052, maxMem=766075207 -2016-05-25 10:29:08 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:29:08 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52485 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:29:08 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-25 10:29:08 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 10:29:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:08 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 13) -2016-05-25 10:29:08 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-05-25 10:29:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:29:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:29:08 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 13). 2082 bytes result sent to driver -2016-05-25 10:29:08 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 2082 bytes result sent to driver -2016-05-25 10:29:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 13) in 16 ms on localhost (1/2) -2016-05-25 10:29:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 18 ms on localhost (2/2) -2016-05-25 10:29:08 INFO DAGScheduler:59 - ResultStage 6 (count at NaiveBayesLearning.scala:45) finished in 0.019 s -2016-05-25 10:29:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 10:29:08 INFO DAGScheduler:59 - Job 5 finished: count at NaiveBayesLearning.scala:45, took 0.032345 s -2016-05-25 10:29:08 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:46 -2016-05-25 10:29:08 INFO DAGScheduler:59 - Got job 6 (foreach at NaiveBayesLearning.scala:46) with 2 output partitions -2016-05-25 10:29:08 INFO DAGScheduler:59 - Final stage: ResultStage 7(foreach at NaiveBayesLearning.scala:46) -2016-05-25 10:29:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:08 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32), which has no missing parents -2016-05-25 10:29:08 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=202168, maxMem=766075207 -2016-05-25 10:29:08 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-25 10:29:08 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=205904, maxMem=766075207 -2016-05-25 10:29:08 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:29:08 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52485 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:29:08 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:32) -2016-05-25 10:29:08 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 10:29:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 15, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:08 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 15) -2016-05-25 10:29:08 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 14) -2016-05-25 10:29:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:29:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:29:08 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 14). 2044 bytes result sent to driver -2016-05-25 10:29:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 14) in 18 ms on localhost (1/2) -2016-05-25 10:29:08 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 15). 2044 bytes result sent to driver -2016-05-25 10:29:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 15) in 20 ms on localhost (2/2) -2016-05-25 10:29:08 INFO DAGScheduler:59 - ResultStage 7 (foreach at NaiveBayesLearning.scala:46) finished in 0.022 s -2016-05-25 10:29:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 10:29:08 INFO DAGScheduler:59 - Job 6 finished: foreach at NaiveBayesLearning.scala:46, took 0.040588 s -2016-05-25 10:29:08 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:49 -2016-05-25 10:29:08 INFO DAGScheduler:59 - Got job 7 (take at NaiveBayesLearning.scala:49) with 1 output partitions -2016-05-25 10:29:08 INFO DAGScheduler:59 - Final stage: ResultStage 8(take at NaiveBayesLearning.scala:49) -2016-05-25 10:29:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:08 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:08 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:38), which has no missing parents -2016-05-25 10:29:08 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=208050, maxMem=766075207 -2016-05-25 10:29:08 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 10:29:09 INFO MemoryStore:59 - ensureFreeSpace(2669) called with curMem=212874, maxMem=766075207 -2016-05-25 10:29:09 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:29:09 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52485 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:29:09 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:09 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:38) -2016-05-25 10:29:09 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-05-25 10:29:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:09 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 16) -2016-05-25 10:29:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:29:09 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 16). 2040 bytes result sent to driver -2016-05-25 10:29:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 16) in 19 ms on localhost (1/1) -2016-05-25 10:29:09 INFO DAGScheduler:59 - ResultStage 8 (take at NaiveBayesLearning.scala:49) finished in 0.019 s -2016-05-25 10:29:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 10:29:09 INFO DAGScheduler:59 - Job 7 finished: take at NaiveBayesLearning.scala:49, took 0.037102 s -2016-05-25 10:29:09 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:49 -2016-05-25 10:29:09 INFO DAGScheduler:59 - Got job 8 (take at NaiveBayesLearning.scala:49) with 1 output partitions -2016-05-25 10:29:09 INFO DAGScheduler:59 - Final stage: ResultStage 9(take at NaiveBayesLearning.scala:49) -2016-05-25 10:29:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:09 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:38), which has no missing parents -2016-05-25 10:29:09 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=215543, maxMem=766075207 -2016-05-25 10:29:09 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 10:29:09 INFO MemoryStore:59 - ensureFreeSpace(2669) called with curMem=220367, maxMem=766075207 -2016-05-25 10:29:09 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:29:09 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:52485 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:29:09 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:09 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 9 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:38) -2016-05-25 10:29:09 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 1 tasks -2016-05-25 10:29:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:09 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 10:29:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:29:09 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2206 bytes result sent to driver -2016-05-25 10:29:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 19 ms on localhost (1/1) -2016-05-25 10:29:09 INFO DAGScheduler:59 - ResultStage 9 (take at NaiveBayesLearning.scala:49) finished in 0.019 s -2016-05-25 10:29:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 10:29:09 INFO DAGScheduler:59 - Job 8 finished: take at NaiveBayesLearning.scala:49, took 0.035353 s -2016-05-25 10:29:09 INFO SparkContext:59 - Starting job: saveAsTextFile at NaiveBayes.scala:202 -2016-05-25 10:29:09 INFO DAGScheduler:59 - Got job 9 (saveAsTextFile at NaiveBayes.scala:202) with 1 output partitions -2016-05-25 10:29:09 INFO DAGScheduler:59 - Final stage: ResultStage 10(saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 10:29:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:09 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202), which has no missing parents -2016-05-25 10:29:09 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=223036, maxMem=766075207 -2016-05-25 10:29:09 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 124.0 KB, free 730.3 MB) -2016-05-25 10:29:09 INFO MemoryStore:59 - ensureFreeSpace(42584) called with curMem=350028, maxMem=766075207 -2016-05-25 10:29:09 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.2 MB) -2016-05-25 10:29:09 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:52485 (size: 41.6 KB, free: 730.5 MB) -2016-05-25 10:29:09 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:09 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 10:29:09 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-25 10:29:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 18, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-25 10:29:09 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 18) -2016-05-25 10:29:10 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251029_0010_m_000000_18' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525102909076/result/metadata/_temporary/0/task_201605251029_0010_m_000000 -2016-05-25 10:29:10 INFO SparkHadoopMapRedUtil:59 - attempt_201605251029_0010_m_000000_18: Committed -2016-05-25 10:29:10 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 18). 1864 bytes result sent to driver -2016-05-25 10:29:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 18) in 444 ms on localhost (1/1) -2016-05-25 10:29:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 10:29:10 INFO DAGScheduler:59 - ResultStage 10 (saveAsTextFile at NaiveBayes.scala:202) finished in 0.444 s -2016-05-25 10:29:10 INFO DAGScheduler:59 - Job 9 finished: saveAsTextFile at NaiveBayes.scala:202, took 0.532028 s -2016-05-25 10:29:10 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:52485 in memory (size: 41.6 KB, free: 730.6 MB) -2016-05-25 10:29:10 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 10:29:10 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:52485 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:29:10 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 10:29:10 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:52485 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:29:10 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 10:29:10 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:52485 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:29:10 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 10:29:10 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:52485 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:29:10 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 10:29:10 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:52485 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:29:10 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 10:29:10 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52485 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:29:10 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 10:29:10 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52485 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:29:10 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 10:29:11 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:29:11 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:29:11 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:206 -2016-05-25 10:29:11 INFO DAGScheduler:59 - Got job 10 (parquet at NaiveBayes.scala:206) with 1 output partitions -2016-05-25 10:29:11 INFO DAGScheduler:59 - Final stage: ResultStage 11(parquet at NaiveBayes.scala:206) -2016-05-25 10:29:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:11 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205), which has no missing parents -2016-05-25 10:29:11 INFO MemoryStore:59 - ensureFreeSpace(66936) called with curMem=179146, maxMem=766075207 -2016-05-25 10:29:11 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 65.4 KB, free 730.4 MB) -2016-05-25 10:29:11 INFO MemoryStore:59 - ensureFreeSpace(23364) called with curMem=246082, maxMem=766075207 -2016-05-25 10:29:11 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 22.8 KB, free 730.3 MB) -2016-05-25 10:29:11 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:52485 (size: 22.8 KB, free: 730.5 MB) -2016-05-25 10:29:11 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:11 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 11 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205) -2016-05-25 10:29:11 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 1 tasks -2016-05-25 10:29:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 19, localhost, PROCESS_LOCAL, 2506 bytes) -2016-05-25 10:29:11 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 19) -2016-05-25 10:29:11 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:29:11 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 10:29:11 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 10:29:11 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 10:29:11 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 10:29:11 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 10:29:11 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 10:29:11 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 10:29:11 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 10:29:11 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 231 -2016-05-25 10:29:11 INFO ColumnChunkPageWriteStore:151 - written 77B for [labels, bag, array] DOUBLE: 3 values, 37B raw, 36B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 10:29:11 INFO ColumnChunkPageWriteStore:151 - written 74B for [pi, bag, array] DOUBLE: 3 values, 16B raw, 33B comp, 1 pages, encodings: [RLE, PLAIN_DICTIONARY], dic { 2 entries, 16B raw, 2B comp} -2016-05-25 10:29:11 INFO ColumnChunkPageWriteStore:151 - written 83B for [theta, bag, array, bag, array] DOUBLE: 9 values, 23B raw, 42B comp, 1 pages, encodings: [RLE, PLAIN_DICTIONARY], dic { 6 entries, 48B raw, 6B comp} -2016-05-25 10:29:11 INFO ColumnChunkPageWriteStore:151 - written 86B for [modelType] BINARY: 1 values, 21B raw, 39B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 10:29:12 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251029_0011_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525102909076/result/data/_temporary/0/task_201605251029_0011_m_000000 -2016-05-25 10:29:12 INFO SparkHadoopMapRedUtil:59 - attempt_201605251029_0011_m_000000_0: Committed -2016-05-25 10:29:12 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 19). 915 bytes result sent to driver -2016-05-25 10:29:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 19) in 460 ms on localhost (1/1) -2016-05-25 10:29:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 10:29:12 INFO DAGScheduler:59 - ResultStage 11 (parquet at NaiveBayes.scala:206) finished in 0.460 s -2016-05-25 10:29:12 INFO DAGScheduler:59 - Job 10 finished: parquet at NaiveBayes.scala:206, took 0.500389 s -2016-05-25 10:29:12 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:29:12 INFO DefaultWriterContainer:59 - Job job_201605251029_0000 committed. -2016-05-25 10:29:12 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525102909076/result/data on driver -2016-05-25 10:29:12 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525102909076/result/data on driver -2016-05-25 10:29:12 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=269446, maxMem=766075207 -2016-05-25 10:29:12 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-25 10:29:12 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=500150, maxMem=766075207 -2016-05-25 10:29:12 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-25 10:29:12 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:52485 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 10:29:12 INFO SparkContext:59 - Created broadcast 13 from textFile at modelSaveLoad.scala:129 -2016-05-25 10:29:12 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:29:12 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 10:29:12 INFO DAGScheduler:59 - Got job 11 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 10:29:12 INFO DAGScheduler:59 - Final stage: ResultStage 12(first at modelSaveLoad.scala:129) -2016-05-25 10:29:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:12 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 10:29:12 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=519938, maxMem=766075207 -2016-05-25 10:29:12 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-25 10:29:12 INFO MemoryStore:59 - ensureFreeSpace(1891) called with curMem=523154, maxMem=766075207 -2016-05-25 10:29:12 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 1891.0 B, free 730.1 MB) -2016-05-25 10:29:12 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:52485 (size: 1891.0 B, free: 730.5 MB) -2016-05-25 10:29:12 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:12 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129) -2016-05-25 10:29:12 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-25 10:29:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 20, localhost, PROCESS_LOCAL, 2239 bytes) -2016-05-25 10:29:12 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 20) -2016-05-25 10:29:12 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525102909076/result/metadata/part-00000:0+56 -2016-05-25 10:29:12 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 20). 2159 bytes result sent to driver -2016-05-25 10:29:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 20) in 13 ms on localhost (1/1) -2016-05-25 10:29:12 INFO DAGScheduler:59 - ResultStage 12 (first at modelSaveLoad.scala:129) finished in 0.014 s -2016-05-25 10:29:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 10:29:12 INFO DAGScheduler:59 - Job 11 finished: first at modelSaveLoad.scala:129, took 0.027736 s -2016-05-25 10:29:12 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525102909076/result/data on driver -2016-05-25 10:29:12 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:213 -2016-05-25 10:29:12 INFO DAGScheduler:59 - Got job 12 (parquet at NaiveBayes.scala:213) with 4 output partitions -2016-05-25 10:29:12 INFO DAGScheduler:59 - Final stage: ResultStage 13(parquet at NaiveBayes.scala:213) -2016-05-25 10:29:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:12 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213), which has no missing parents -2016-05-25 10:29:12 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=525045, maxMem=766075207 -2016-05-25 10:29:12 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-25 10:29:12 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=588045, maxMem=766075207 -2016-05-25 10:29:12 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.0 MB) -2016-05-25 10:29:12 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:52485 (size: 20.6 KB, free: 730.5 MB) -2016-05-25 10:29:12 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:12 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 13 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213) -2016-05-25 10:29:12 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 4 tasks -2016-05-25 10:29:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 21, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:29:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 22, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:29:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 13.0 (TID 23, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:29:12 INFO TaskSetManager:59 - Starting task 3.0 in stage 13.0 (TID 24, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-25 10:29:12 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 21) -2016-05-25 10:29:12 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 22) -2016-05-25 10:29:12 INFO Executor:59 - Running task 2.0 in stage 13.0 (TID 23) -2016-05-25 10:29:12 INFO Executor:59 - Running task 3.0 in stage 13.0 (TID 24) -2016-05-25 10:29:12 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:29:12 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:29:12 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 22). 936 bytes result sent to driver -2016-05-25 10:29:12 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:29:12 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 21). 936 bytes result sent to driver -2016-05-25 10:29:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 22) in 61 ms on localhost (1/4) -2016-05-25 10:29:12 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:29:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 21) in 65 ms on localhost (2/4) -2016-05-25 10:29:12 INFO Executor:59 - Finished task 2.0 in stage 13.0 (TID 23). 936 bytes result sent to driver -2016-05-25 10:29:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 13.0 (TID 23) in 66 ms on localhost (3/4) -2016-05-25 10:29:12 INFO Executor:59 - Finished task 3.0 in stage 13.0 (TID 24). 1876 bytes result sent to driver -2016-05-25 10:29:12 INFO TaskSetManager:59 - Finished task 3.0 in stage 13.0 (TID 24) in 82 ms on localhost (4/4) -2016-05-25 10:29:12 INFO DAGScheduler:59 - ResultStage 13 (parquet at NaiveBayes.scala:213) finished in 0.085 s -2016-05-25 10:29:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 10:29:12 INFO DAGScheduler:59 - Job 12 finished: parquet at NaiveBayes.scala:213, took 0.111646 s -2016-05-25 10:29:12 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=609118, maxMem=766075207 -2016-05-25 10:29:12 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 86.4 KB, free 729.9 MB) -2016-05-25 10:29:12 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=697630, maxMem=766075207 -2016-05-25 10:29:12 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-25 10:29:12 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:52485 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 10:29:12 INFO SparkContext:59 - Created broadcast 16 from take at NaiveBayes.scala:216 -2016-05-25 10:29:13 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 10:29:13 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525102909076/result/data/part-r-00000-c48aec0b-b0d2-4f2c-9fad-b774d09b7ea9.gz.parquet -2016-05-25 10:29:13 INFO SparkContext:59 - Starting job: take at NaiveBayes.scala:216 -2016-05-25 10:29:13 INFO DAGScheduler:59 - Got job 13 (take at NaiveBayes.scala:216) with 1 output partitions -2016-05-25 10:29:13 INFO DAGScheduler:59 - Final stage: ResultStage 14(take at NaiveBayes.scala:216) -2016-05-25 10:29:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:13 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216), which has no missing parents -2016-05-25 10:29:13 INFO MemoryStore:59 - ensureFreeSpace(4280) called with curMem=717418, maxMem=766075207 -2016-05-25 10:29:13 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 4.2 KB, free 729.9 MB) -2016-05-25 10:29:13 INFO MemoryStore:59 - ensureFreeSpace(2466) called with curMem=721698, maxMem=766075207 -2016-05-25 10:29:13 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.4 KB, free 729.9 MB) -2016-05-25 10:29:13 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:52485 (size: 2.4 KB, free: 730.5 MB) -2016-05-25 10:29:13 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:13 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 14 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216) -2016-05-25 10:29:13 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 1 tasks -2016-05-25 10:29:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 25, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-25 10:29:13 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 25) -2016-05-25 10:29:13 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525102909076/result/data/part-r-00000-c48aec0b-b0d2-4f2c-9fad-b774d09b7ea9.gz.parquet start: 0 end: 1477 length: 1477 hosts: []} -2016-05-25 10:29:13 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 10:29:13 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group labels (LIST) { - repeated group bag { - optional double array; - } - } - optional group pi (LIST) { - repeated group bag { - optional double array; - } - } - optional group theta (LIST) { - repeated group bag { - optional group array (LIST) { - repeated group bag { - optional double array; - } - } - } - } - optional binary modelType (UTF8); -} - -Catalyst form: -StructType(StructField(labels,ArrayType(DoubleType,true),true), StructField(pi,ArrayType(DoubleType,true),true), StructField(theta,ArrayType(ArrayType(DoubleType,true),true),true), StructField(modelType,StringType,true)) - -2016-05-25 10:29:13 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 10:29:13 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 10:29:13 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 10:29:13 INFO InternalParquetRecordReader:151 - block read in memory in 12 ms. row count = 1 -2016-05-25 10:29:13 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 25). 2841 bytes result sent to driver -2016-05-25 10:29:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 25) in 107 ms on localhost (1/1) -2016-05-25 10:29:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 10:29:13 INFO DAGScheduler:59 - ResultStage 14 (take at NaiveBayes.scala:216) finished in 0.108 s -2016-05-25 10:29:13 INFO DAGScheduler:59 - Job 13 finished: take at NaiveBayes.scala:216, took 0.125427 s -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 10:29:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 10:29:13 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 10:29:13 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 10:29:13 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 10:29:13 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 10:29:13 INFO BlockManager:59 - BlockManager stopped -2016-05-25 10:29:13 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 10:29:13 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 10:29:13 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 10:29:13 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 10:29:13 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 10:29:13 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-651868d8-4032-4e99-a85a-33b0da0e6b39 -2016-05-25 10:29:13 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-25 10:29:46 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 10:29:47 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 10:29:47 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 10:29:47 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 10:29:47 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 10:29:48 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 10:29:48 INFO Remoting:74 - Starting remoting -2016-05-25 10:29:48 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:52506] -2016-05-25 10:29:48 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52506. -2016-05-25 10:29:48 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 10:29:48 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 10:29:48 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-52e02906-a103-4b6f-851b-accb520640f7 -2016-05-25 10:29:48 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 10:29:48 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-3f75591b-eeba-4605-9860-6f17ad184f8e\httpd-5cde8e59-b211-46cc-8aa5-674a868ce4f4 -2016-05-25 10:29:48 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 10:29:48 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:29:48 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52507 -2016-05-25 10:29:48 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52507. -2016-05-25 10:29:48 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 10:29:48 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:29:48 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 10:29:48 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 10:29:48 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 10:29:49 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 10:29:49 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 10:29:49 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52526. -2016-05-25 10:29:49 INFO NettyBlockTransferService:59 - Server created on 52526 -2016-05-25 10:29:49 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 10:29:49 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52526 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52526) -2016-05-25 10:29:49 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 10:29:50 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 10:29:50 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 10:29:50 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 10:29:50 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 10:29:50 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52526 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 10:29:50 INFO SparkContext:59 - Created broadcast 0 from textFile at NaiveBayesLearning.scala:26 -2016-05-25 10:29:51 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 10:29:51 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:29:51 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:32 -2016-05-25 10:29:51 INFO DAGScheduler:59 - Got job 0 (foreach at NaiveBayesLearning.scala:32) with 2 output partitions -2016-05-25 10:29:51 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at NaiveBayesLearning.scala:32) -2016-05-25 10:29:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:51 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at NaiveBayesLearning.scala:27), which has no missing parents -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(3432) called with curMem=171580, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(2015) called with curMem=175012, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2015.0 B, free 730.4 MB) -2016-05-25 10:29:52 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52526 (size: 2015.0 B, free: 730.6 MB) -2016-05-25 10:29:52 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at NaiveBayesLearning.scala:27) -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 10:29:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:52 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 10:29:52 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 10:29:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:29:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:29:52 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 10:29:52 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 10:29:52 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 10:29:52 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 10:29:52 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 10:29:52 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-25 10:29:52 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-25 10:29:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 130 ms on localhost (1/2) -2016-05-25 10:29:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 109 ms on localhost (2/2) -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 10:29:52 INFO DAGScheduler:59 - ResultStage 0 (foreach at NaiveBayesLearning.scala:32) finished in 0.155 s -2016-05-25 10:29:52 INFO DAGScheduler:59 - Job 0 finished: foreach at NaiveBayesLearning.scala:32, took 0.311225 s -2016-05-25 10:29:52 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:29:52 INFO DAGScheduler:59 - Registering RDD 5 (map at NaiveBayes.scala:383) -2016-05-25 10:29:52 INFO DAGScheduler:59 - Got job 1 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:29:52 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at NaiveBayes.scala:401) -2016-05-25 10:29:52 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1) -2016-05-25 10:29:52 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1) -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(5104) called with curMem=177027, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(2697) called with curMem=182131, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:29:52 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52526 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:29:52 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383) -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-25 10:29:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 10:29:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 10:29:52 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 10:29:52 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-25 10:29:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:29:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:29:52 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 10:29:52 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 10:29:52 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2254 bytes result sent to driver -2016-05-25 10:29:52 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2254 bytes result sent to driver -2016-05-25 10:29:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 114 ms on localhost (1/2) -2016-05-25 10:29:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 117 ms on localhost (2/2) -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 10:29:52 INFO DAGScheduler:59 - ShuffleMapStage 1 (map at NaiveBayes.scala:383) finished in 0.118 s -2016-05-25 10:29:52 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 10:29:52 INFO DAGScheduler:59 - running: Set() -2016-05-25 10:29:52 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-05-25 10:29:52 INFO DAGScheduler:59 - failed: Set() -2016-05-25 10:29:52 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List() -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting ResultStage 2 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(2936) called with curMem=184828, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=187764, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.4 MB) -2016-05-25 10:29:52 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52526 (size: 1567.0 B, free: 730.6 MB) -2016-05-25 10:29:52 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383) -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 10:29:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:29:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:29:52 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-25 10:29:52 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-25 10:29:52 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52526 in memory (size: 2015.0 B, free: 730.6 MB) -2016-05-25 10:29:52 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 10:29:52 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 10:29:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 10:29:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 10:29:52 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1161 bytes result sent to driver -2016-05-25 10:29:52 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1645 bytes result sent to driver -2016-05-25 10:29:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 47 ms on localhost (1/2) -2016-05-25 10:29:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 55 ms on localhost (2/2) -2016-05-25 10:29:52 INFO DAGScheduler:59 - ResultStage 2 (collect at NaiveBayes.scala:401) finished in 0.056 s -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 10:29:52 INFO DAGScheduler:59 - Job 1 finished: collect at NaiveBayes.scala:401, took 0.245233 s -2016-05-25 10:29:52 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:40 -2016-05-25 10:29:52 INFO DAGScheduler:59 - Got job 2 (count at NaiveBayesLearning.scala:40) with 2 output partitions -2016-05-25 10:29:52 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at NaiveBayesLearning.scala:40) -2016-05-25 10:29:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:40), which has no missing parents -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=183884, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(2670) called with curMem=188780, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:29:52 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52526 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:29:52 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:40) -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 10:29:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:52 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-25 10:29:52 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-25 10:29:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:29:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:29:52 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 2082 bytes result sent to driver -2016-05-25 10:29:52 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-25 10:29:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 22 ms on localhost (1/2) -2016-05-25 10:29:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 25 ms on localhost (2/2) -2016-05-25 10:29:52 INFO DAGScheduler:59 - ResultStage 3 (count at NaiveBayesLearning.scala:40) finished in 0.025 s -2016-05-25 10:29:52 INFO DAGScheduler:59 - Job 2 finished: count at NaiveBayesLearning.scala:40, took 0.045071 s -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 10:29:52 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:40 -2016-05-25 10:29:52 INFO DAGScheduler:59 - Got job 3 (count at NaiveBayesLearning.scala:40) with 2 output partitions -2016-05-25 10:29:52 INFO DAGScheduler:59 - Final stage: ResultStage 4(count at NaiveBayesLearning.scala:40) -2016-05-25 10:29:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=191450, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(2118) called with curMem=195050, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:29:52 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52526 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:29:52 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 10:29:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:52 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-25 10:29:52 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-25 10:29:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:29:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:29:52 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2082 bytes result sent to driver -2016-05-25 10:29:52 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2082 bytes result sent to driver -2016-05-25 10:29:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 16 ms on localhost (1/2) -2016-05-25 10:29:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 20 ms on localhost (2/2) -2016-05-25 10:29:52 INFO DAGScheduler:59 - ResultStage 4 (count at NaiveBayesLearning.scala:40) finished in 0.021 s -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 10:29:52 INFO DAGScheduler:59 - Job 3 finished: count at NaiveBayesLearning.scala:40, took 0.034189 s -2016-05-25 10:29:52 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:44 -2016-05-25 10:29:52 INFO DAGScheduler:59 - Got job 4 (count at NaiveBayesLearning.scala:44) with 2 output partitions -2016-05-25 10:29:52 INFO DAGScheduler:59 - Final stage: ResultStage 5(count at NaiveBayesLearning.scala:44) -2016-05-25 10:29:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=197168, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(2114) called with curMem=200768, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:29:52 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52526 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:29:52 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 10:29:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:52 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-05-25 10:29:52 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-05-25 10:29:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:29:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:29:52 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 2082 bytes result sent to driver -2016-05-25 10:29:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 19 ms on localhost (1/2) -2016-05-25 10:29:52 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 2082 bytes result sent to driver -2016-05-25 10:29:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 24 ms on localhost (2/2) -2016-05-25 10:29:52 INFO DAGScheduler:59 - ResultStage 5 (count at NaiveBayesLearning.scala:44) finished in 0.026 s -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 10:29:52 INFO DAGScheduler:59 - Job 4 finished: count at NaiveBayesLearning.scala:44, took 0.039021 s -2016-05-25 10:29:52 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:45 -2016-05-25 10:29:52 INFO DAGScheduler:59 - Got job 5 (foreach at NaiveBayesLearning.scala:45) with 2 output partitions -2016-05-25 10:29:52 INFO DAGScheduler:59 - Final stage: ResultStage 6(foreach at NaiveBayesLearning.scala:45) -2016-05-25 10:29:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=202882, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(2144) called with curMem=206618, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:29:52 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52526 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:29:52 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 10:29:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:52 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-05-25 10:29:52 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 13) -2016-05-25 10:29:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:29:52 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 2044 bytes result sent to driver -2016-05-25 10:29:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:29:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 18 ms on localhost (1/2) -2016-05-25 10:29:52 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 13). 2044 bytes result sent to driver -2016-05-25 10:29:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 13) in 24 ms on localhost (2/2) -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 10:29:52 INFO DAGScheduler:59 - ResultStage 6 (foreach at NaiveBayesLearning.scala:45) finished in 0.025 s -2016-05-25 10:29:52 INFO DAGScheduler:59 - Job 5 finished: foreach at NaiveBayesLearning.scala:45, took 0.039968 s -2016-05-25 10:29:52 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:46 -2016-05-25 10:29:52 INFO DAGScheduler:59 - Got job 6 (count at NaiveBayesLearning.scala:46) with 2 output partitions -2016-05-25 10:29:52 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at NaiveBayesLearning.scala:46) -2016-05-25 10:29:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=208762, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(2118) called with curMem=212362, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:29:52 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52526 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:29:52 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 10:29:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 15, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:52 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 15) -2016-05-25 10:29:52 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 14) -2016-05-25 10:29:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:29:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:29:52 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 10:29:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 15) in 14 ms on localhost (1/2) -2016-05-25 10:29:52 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 14). 2082 bytes result sent to driver -2016-05-25 10:29:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 14) in 23 ms on localhost (2/2) -2016-05-25 10:29:52 INFO DAGScheduler:59 - ResultStage 7 (count at NaiveBayesLearning.scala:46) finished in 0.024 s -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 10:29:52 INFO DAGScheduler:59 - Job 6 finished: count at NaiveBayesLearning.scala:46, took 0.042235 s -2016-05-25 10:29:52 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:47 -2016-05-25 10:29:52 INFO DAGScheduler:59 - Got job 7 (foreach at NaiveBayesLearning.scala:47) with 2 output partitions -2016-05-25 10:29:52 INFO DAGScheduler:59 - Final stage: ResultStage 8(foreach at NaiveBayesLearning.scala:47) -2016-05-25 10:29:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=214480, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(2148) called with curMem=218216, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:29:52 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52526 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:29:52 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 10:29:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 17, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:52 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 16) -2016-05-25 10:29:52 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 17) -2016-05-25 10:29:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:29:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:29:52 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 16). 2044 bytes result sent to driver -2016-05-25 10:29:52 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 17). 2044 bytes result sent to driver -2016-05-25 10:29:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 16) in 19 ms on localhost (1/2) -2016-05-25 10:29:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 17) in 19 ms on localhost (2/2) -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 10:29:52 INFO DAGScheduler:59 - ResultStage 8 (foreach at NaiveBayesLearning.scala:47) finished in 0.020 s -2016-05-25 10:29:52 INFO DAGScheduler:59 - Job 7 finished: foreach at NaiveBayesLearning.scala:47, took 0.040457 s -2016-05-25 10:29:52 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:50 -2016-05-25 10:29:52 INFO DAGScheduler:59 - Got job 8 (take at NaiveBayesLearning.scala:50) with 1 output partitions -2016-05-25 10:29:52 INFO DAGScheduler:59 - Final stage: ResultStage 9(take at NaiveBayesLearning.scala:50) -2016-05-25 10:29:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=220364, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(2669) called with curMem=225188, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:29:52 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:52526 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:29:52 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 9 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39) -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 1 tasks -2016-05-25 10:29:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:52 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 18) -2016-05-25 10:29:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:29:52 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 18). 2040 bytes result sent to driver -2016-05-25 10:29:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 18) in 18 ms on localhost (1/1) -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 10:29:52 INFO DAGScheduler:59 - ResultStage 9 (take at NaiveBayesLearning.scala:50) finished in 0.019 s -2016-05-25 10:29:52 INFO DAGScheduler:59 - Job 8 finished: take at NaiveBayesLearning.scala:50, took 0.043772 s -2016-05-25 10:29:52 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:50 -2016-05-25 10:29:52 INFO DAGScheduler:59 - Got job 9 (take at NaiveBayesLearning.scala:50) with 1 output partitions -2016-05-25 10:29:52 INFO DAGScheduler:59 - Final stage: ResultStage 10(take at NaiveBayesLearning.scala:50) -2016-05-25 10:29:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=227857, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 10:29:52 INFO MemoryStore:59 - ensureFreeSpace(2669) called with curMem=232681, maxMem=766075207 -2016-05-25 10:29:52 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:29:52 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:52526 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:29:52 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:52 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39) -2016-05-25 10:29:52 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-25 10:29:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:29:52 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 10:29:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:29:53 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2206 bytes result sent to driver -2016-05-25 10:29:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 17 ms on localhost (1/1) -2016-05-25 10:29:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 10:29:53 INFO DAGScheduler:59 - ResultStage 10 (take at NaiveBayesLearning.scala:50) finished in 0.018 s -2016-05-25 10:29:53 INFO DAGScheduler:59 - Job 9 finished: take at NaiveBayesLearning.scala:50, took 0.033486 s -2016-05-25 10:29:53 INFO SparkContext:59 - Starting job: saveAsTextFile at NaiveBayes.scala:202 -2016-05-25 10:29:53 INFO DAGScheduler:59 - Got job 10 (saveAsTextFile at NaiveBayes.scala:202) with 1 output partitions -2016-05-25 10:29:53 INFO DAGScheduler:59 - Final stage: ResultStage 11(saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 10:29:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:53 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202), which has no missing parents -2016-05-25 10:29:53 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=235350, maxMem=766075207 -2016-05-25 10:29:53 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 124.0 KB, free 730.2 MB) -2016-05-25 10:29:53 INFO MemoryStore:59 - ensureFreeSpace(42584) called with curMem=362342, maxMem=766075207 -2016-05-25 10:29:53 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.2 MB) -2016-05-25 10:29:53 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:52526 (size: 41.6 KB, free: 730.5 MB) -2016-05-25 10:29:53 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:53 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 11 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 10:29:53 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 1 tasks -2016-05-25 10:29:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 20, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-25 10:29:53 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 20) -2016-05-25 10:29:53 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251029_0011_m_000000_20' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525102953008/result/metadata/_temporary/0/task_201605251029_0011_m_000000 -2016-05-25 10:29:53 INFO SparkHadoopMapRedUtil:59 - attempt_201605251029_0011_m_000000_20: Committed -2016-05-25 10:29:53 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 20). 1864 bytes result sent to driver -2016-05-25 10:29:53 INFO DAGScheduler:59 - ResultStage 11 (saveAsTextFile at NaiveBayes.scala:202) finished in 0.255 s -2016-05-25 10:29:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 20) in 253 ms on localhost (1/1) -2016-05-25 10:29:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 10:29:53 INFO DAGScheduler:59 - Job 10 finished: saveAsTextFile at NaiveBayes.scala:202, took 0.365517 s -2016-05-25 10:29:54 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:52526 in memory (size: 41.6 KB, free: 730.6 MB) -2016-05-25 10:29:54 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-25 10:29:54 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:52526 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:29:54 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 10:29:54 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:52526 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:29:54 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 10:29:54 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:52526 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:29:54 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 10:29:54 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:52526 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:29:54 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 10:29:54 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:52526 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:29:54 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 10:29:54 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:52526 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:29:54 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 10:29:54 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52526 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:29:54 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 10:29:54 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52526 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:29:54 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 10:29:54 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52526 in memory (size: 1567.0 B, free: 730.6 MB) -2016-05-25 10:29:54 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52526 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:29:54 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 10:29:55 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:29:55 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:29:55 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:206 -2016-05-25 10:29:55 INFO DAGScheduler:59 - Got job 11 (parquet at NaiveBayes.scala:206) with 1 output partitions -2016-05-25 10:29:55 INFO DAGScheduler:59 - Final stage: ResultStage 12(parquet at NaiveBayes.scala:206) -2016-05-25 10:29:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:55 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205), which has no missing parents -2016-05-25 10:29:55 INFO MemoryStore:59 - ensureFreeSpace(66936) called with curMem=171580, maxMem=766075207 -2016-05-25 10:29:55 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 65.4 KB, free 730.4 MB) -2016-05-25 10:29:55 INFO MemoryStore:59 - ensureFreeSpace(23362) called with curMem=238516, maxMem=766075207 -2016-05-25 10:29:55 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 22.8 KB, free 730.3 MB) -2016-05-25 10:29:55 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:52526 (size: 22.8 KB, free: 730.6 MB) -2016-05-25 10:29:55 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:55 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205) -2016-05-25 10:29:55 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-25 10:29:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 21, localhost, PROCESS_LOCAL, 2506 bytes) -2016-05-25 10:29:55 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 21) -2016-05-25 10:29:55 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:29:55 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 10:29:55 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 10:29:55 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 10:29:55 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 10:29:55 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 10:29:55 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 10:29:55 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 10:29:55 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 10:29:55 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 231 -2016-05-25 10:29:55 INFO ColumnChunkPageWriteStore:151 - written 77B for [labels, bag, array] DOUBLE: 3 values, 37B raw, 36B comp, 1 pages, encodings: [RLE, PLAIN] -2016-05-25 10:29:55 INFO ColumnChunkPageWriteStore:151 - written 74B for [pi, bag, array] DOUBLE: 3 values, 16B raw, 33B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 2 entries, 16B raw, 2B comp} -2016-05-25 10:29:55 INFO ColumnChunkPageWriteStore:151 - written 83B for [theta, bag, array, bag, array] DOUBLE: 9 values, 23B raw, 42B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 6 entries, 48B raw, 6B comp} -2016-05-25 10:29:55 INFO ColumnChunkPageWriteStore:151 - written 86B for [modelType] BINARY: 1 values, 21B raw, 39B comp, 1 pages, encodings: [RLE, PLAIN, BIT_PACKED] -2016-05-25 10:29:55 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251029_0012_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525102953008/result/data/_temporary/0/task_201605251029_0012_m_000000 -2016-05-25 10:29:55 INFO SparkHadoopMapRedUtil:59 - attempt_201605251029_0012_m_000000_0: Committed -2016-05-25 10:29:55 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 21). 915 bytes result sent to driver -2016-05-25 10:29:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 21) in 456 ms on localhost (1/1) -2016-05-25 10:29:55 INFO DAGScheduler:59 - ResultStage 12 (parquet at NaiveBayes.scala:206) finished in 0.456 s -2016-05-25 10:29:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 10:29:55 INFO DAGScheduler:59 - Job 11 finished: parquet at NaiveBayes.scala:206, took 0.498192 s -2016-05-25 10:29:56 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:29:56 INFO DefaultWriterContainer:59 - Job job_201605251029_0000 committed. -2016-05-25 10:29:56 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525102953008/result/data on driver -2016-05-25 10:29:56 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525102953008/result/data on driver -2016-05-25 10:29:56 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=261878, maxMem=766075207 -2016-05-25 10:29:56 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-25 10:29:56 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=492582, maxMem=766075207 -2016-05-25 10:29:56 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-25 10:29:56 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:52526 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 10:29:56 INFO SparkContext:59 - Created broadcast 14 from textFile at modelSaveLoad.scala:129 -2016-05-25 10:29:56 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:29:56 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 10:29:56 INFO DAGScheduler:59 - Got job 12 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 10:29:56 INFO DAGScheduler:59 - Final stage: ResultStage 13(first at modelSaveLoad.scala:129) -2016-05-25 10:29:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:56 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 10:29:56 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=512370, maxMem=766075207 -2016-05-25 10:29:56 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-25 10:29:56 INFO MemoryStore:59 - ensureFreeSpace(1889) called with curMem=515586, maxMem=766075207 -2016-05-25 10:29:56 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1889.0 B, free 730.1 MB) -2016-05-25 10:29:56 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:52526 (size: 1889.0 B, free: 730.5 MB) -2016-05-25 10:29:56 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:56 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 13 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129) -2016-05-25 10:29:56 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 1 tasks -2016-05-25 10:29:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 22, localhost, PROCESS_LOCAL, 2239 bytes) -2016-05-25 10:29:56 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 22) -2016-05-25 10:29:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525102953008/result/metadata/part-00000:0+56 -2016-05-25 10:29:56 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 22). 2159 bytes result sent to driver -2016-05-25 10:29:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 22) in 33 ms on localhost (1/1) -2016-05-25 10:29:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 10:29:56 INFO DAGScheduler:59 - ResultStage 13 (first at modelSaveLoad.scala:129) finished in 0.038 s -2016-05-25 10:29:56 INFO DAGScheduler:59 - Job 12 finished: first at modelSaveLoad.scala:129, took 0.259083 s -2016-05-25 10:29:56 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525102953008/result/data on driver -2016-05-25 10:29:56 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:213 -2016-05-25 10:29:56 INFO DAGScheduler:59 - Got job 13 (parquet at NaiveBayes.scala:213) with 4 output partitions -2016-05-25 10:29:56 INFO DAGScheduler:59 - Final stage: ResultStage 14(parquet at NaiveBayes.scala:213) -2016-05-25 10:29:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:56 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213), which has no missing parents -2016-05-25 10:29:56 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=517475, maxMem=766075207 -2016-05-25 10:29:56 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-25 10:29:56 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=580475, maxMem=766075207 -2016-05-25 10:29:56 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.0 MB) -2016-05-25 10:29:56 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:52526 (size: 20.6 KB, free: 730.5 MB) -2016-05-25 10:29:56 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:56 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 14 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213) -2016-05-25 10:29:56 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 4 tasks -2016-05-25 10:29:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 23, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:29:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 24, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:29:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 25, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:29:56 INFO TaskSetManager:59 - Starting task 3.0 in stage 14.0 (TID 26, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-25 10:29:56 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 24) -2016-05-25 10:29:56 INFO Executor:59 - Running task 3.0 in stage 14.0 (TID 26) -2016-05-25 10:29:56 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 23) -2016-05-25 10:29:56 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:29:56 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 24). 936 bytes result sent to driver -2016-05-25 10:29:56 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 25) -2016-05-25 10:29:56 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:29:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 24) in 63 ms on localhost (1/4) -2016-05-25 10:29:56 INFO Executor:59 - Finished task 3.0 in stage 14.0 (TID 26). 1876 bytes result sent to driver -2016-05-25 10:29:57 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:29:57 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 23). 936 bytes result sent to driver -2016-05-25 10:29:57 INFO TaskSetManager:59 - Finished task 3.0 in stage 14.0 (TID 26) in 92 ms on localhost (2/4) -2016-05-25 10:29:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 23) in 98 ms on localhost (3/4) -2016-05-25 10:29:57 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:29:57 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 25). 936 bytes result sent to driver -2016-05-25 10:29:57 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 25) in 110 ms on localhost (4/4) -2016-05-25 10:29:57 INFO DAGScheduler:59 - ResultStage 14 (parquet at NaiveBayes.scala:213) finished in 0.113 s -2016-05-25 10:29:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 10:29:57 INFO DAGScheduler:59 - Job 13 finished: parquet at NaiveBayes.scala:213, took 0.145473 s -2016-05-25 10:29:57 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=601548, maxMem=766075207 -2016-05-25 10:29:57 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 86.4 KB, free 729.9 MB) -2016-05-25 10:29:57 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=690060, maxMem=766075207 -2016-05-25 10:29:57 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-25 10:29:57 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:52526 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 10:29:57 INFO SparkContext:59 - Created broadcast 17 from take at NaiveBayes.scala:216 -2016-05-25 10:29:57 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 10:29:57 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525102953008/result/data/part-r-00000-5923d24c-0b76-40cd-bfce-1f3c385cdd4d.gz.parquet -2016-05-25 10:29:57 INFO SparkContext:59 - Starting job: take at NaiveBayes.scala:216 -2016-05-25 10:29:57 INFO DAGScheduler:59 - Got job 14 (take at NaiveBayes.scala:216) with 1 output partitions -2016-05-25 10:29:57 INFO DAGScheduler:59 - Final stage: ResultStage 15(take at NaiveBayes.scala:216) -2016-05-25 10:29:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:29:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:29:57 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216), which has no missing parents -2016-05-25 10:29:57 INFO MemoryStore:59 - ensureFreeSpace(4280) called with curMem=709848, maxMem=766075207 -2016-05-25 10:29:57 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 4.2 KB, free 729.9 MB) -2016-05-25 10:29:57 INFO MemoryStore:59 - ensureFreeSpace(2466) called with curMem=714128, maxMem=766075207 -2016-05-25 10:29:57 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.4 KB, free 729.9 MB) -2016-05-25 10:29:57 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:52526 (size: 2.4 KB, free: 730.5 MB) -2016-05-25 10:29:57 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:29:57 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 15 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216) -2016-05-25 10:29:57 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 1 tasks -2016-05-25 10:29:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 27, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-25 10:29:57 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 27) -2016-05-25 10:29:57 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525102953008/result/data/part-r-00000-5923d24c-0b76-40cd-bfce-1f3c385cdd4d.gz.parquet start: 0 end: 1477 length: 1477 hosts: []} -2016-05-25 10:29:57 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 10:29:57 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group labels (LIST) { - repeated group bag { - optional double array; - } - } - optional group pi (LIST) { - repeated group bag { - optional double array; - } - } - optional group theta (LIST) { - repeated group bag { - optional group array (LIST) { - repeated group bag { - optional double array; - } - } - } - } - optional binary modelType (UTF8); -} - -Catalyst form: -StructType(StructField(labels,ArrayType(DoubleType,true),true), StructField(pi,ArrayType(DoubleType,true),true), StructField(theta,ArrayType(ArrayType(DoubleType,true),true),true), StructField(modelType,StringType,true)) - -2016-05-25 10:29:57 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 10:29:57 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 10:29:57 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 10:29:57 INFO InternalParquetRecordReader:151 - block read in memory in 26 ms. row count = 1 -2016-05-25 10:29:57 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 27). 2841 bytes result sent to driver -2016-05-25 10:29:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 27) in 242 ms on localhost (1/1) -2016-05-25 10:29:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-25 10:29:57 INFO DAGScheduler:59 - ResultStage 15 (take at NaiveBayes.scala:216) finished in 0.244 s -2016-05-25 10:29:57 INFO DAGScheduler:59 - Job 14 finished: take at NaiveBayes.scala:216, took 0.276853 s -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 10:29:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 10:29:57 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 10:29:57 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 10:29:57 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 10:29:57 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 10:29:57 INFO BlockManager:59 - BlockManager stopped -2016-05-25 10:29:57 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 10:29:57 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 10:29:57 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 10:29:58 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 10:29:58 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-3f75591b-eeba-4605-9860-6f17ad184f8e -2016-05-25 10:30:11 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 10:30:12 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 10:30:12 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 10:30:12 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 10:30:12 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 10:30:13 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 10:30:13 INFO Remoting:74 - Starting remoting -2016-05-25 10:30:13 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:52547] -2016-05-25 10:30:13 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52547. -2016-05-25 10:30:13 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 10:30:13 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 10:30:13 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-ec25a4c9-f072-4947-82db-04ee387b73cb -2016-05-25 10:30:13 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 10:30:14 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-c54906d3-e422-4762-9e76-c7503e8fb124\httpd-104b3674-fb88-4a40-82e4-7226a3509029 -2016-05-25 10:30:14 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 10:30:14 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:30:14 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52548 -2016-05-25 10:30:14 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52548. -2016-05-25 10:30:14 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 10:30:14 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:30:14 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 10:30:14 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 10:30:14 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 10:30:14 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 10:30:14 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 10:30:14 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52567. -2016-05-25 10:30:14 INFO NettyBlockTransferService:59 - Server created on 52567 -2016-05-25 10:30:14 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 10:30:14 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52567 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52567) -2016-05-25 10:30:14 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 10:30:15 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 10:30:15 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 10:30:15 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 10:30:15 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 10:30:15 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52567 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 10:30:15 INFO SparkContext:59 - Created broadcast 0 from textFile at NaiveBayesLearning.scala:26 -2016-05-25 10:30:17 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 10:30:17 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:30:17 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:32 -2016-05-25 10:30:17 INFO DAGScheduler:59 - Got job 0 (foreach at NaiveBayesLearning.scala:32) with 2 output partitions -2016-05-25 10:30:17 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at NaiveBayesLearning.scala:32) -2016-05-25 10:30:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:30:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:30:17 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at NaiveBayesLearning.scala:27), which has no missing parents -2016-05-25 10:30:17 INFO MemoryStore:59 - ensureFreeSpace(3432) called with curMem=171580, maxMem=766075207 -2016-05-25 10:30:17 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-25 10:30:17 INFO MemoryStore:59 - ensureFreeSpace(2015) called with curMem=175012, maxMem=766075207 -2016-05-25 10:30:17 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2015.0 B, free 730.4 MB) -2016-05-25 10:30:17 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52567 (size: 2015.0 B, free: 730.6 MB) -2016-05-25 10:30:17 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:30:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at NaiveBayesLearning.scala:27) -2016-05-25 10:30:17 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 10:30:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:30:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:30:17 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 10:30:17 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 10:30:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:30:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:30:17 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 10:30:17 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 10:30:17 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 10:30:17 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 10:30:17 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 10:30:17 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-25 10:30:17 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-25 10:30:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 124 ms on localhost (1/2) -2016-05-25 10:30:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 102 ms on localhost (2/2) -2016-05-25 10:30:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 10:30:17 INFO DAGScheduler:59 - ResultStage 0 (foreach at NaiveBayesLearning.scala:32) finished in 0.146 s -2016-05-25 10:30:17 INFO DAGScheduler:59 - Job 0 finished: foreach at NaiveBayesLearning.scala:32, took 0.302618 s -2016-05-25 10:30:17 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:30:17 INFO DAGScheduler:59 - Registering RDD 5 (map at NaiveBayes.scala:383) -2016-05-25 10:30:17 INFO DAGScheduler:59 - Got job 1 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:30:17 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at NaiveBayes.scala:401) -2016-05-25 10:30:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1) -2016-05-25 10:30:17 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1) -2016-05-25 10:30:17 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:30:17 INFO MemoryStore:59 - ensureFreeSpace(5104) called with curMem=177027, maxMem=766075207 -2016-05-25 10:30:17 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 10:30:17 INFO MemoryStore:59 - ensureFreeSpace(2695) called with curMem=182131, maxMem=766075207 -2016-05-25 10:30:17 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:30:17 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52567 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:30:17 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:30:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383) -2016-05-25 10:30:17 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-25 10:30:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 10:30:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 10:30:17 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 10:30:17 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-25 10:30:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:30:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:30:18 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2254 bytes result sent to driver -2016-05-25 10:30:18 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2254 bytes result sent to driver -2016-05-25 10:30:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 119 ms on localhost (1/2) -2016-05-25 10:30:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 124 ms on localhost (2/2) -2016-05-25 10:30:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 10:30:18 INFO DAGScheduler:59 - ShuffleMapStage 1 (map at NaiveBayes.scala:383) finished in 0.128 s -2016-05-25 10:30:18 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 10:30:18 INFO DAGScheduler:59 - running: Set() -2016-05-25 10:30:18 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-05-25 10:30:18 INFO DAGScheduler:59 - failed: Set() -2016-05-25 10:30:18 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List() -2016-05-25 10:30:18 INFO DAGScheduler:59 - Submitting ResultStage 2 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 10:30:18 INFO MemoryStore:59 - ensureFreeSpace(2936) called with curMem=184826, maxMem=766075207 -2016-05-25 10:30:18 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-25 10:30:18 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=187762, maxMem=766075207 -2016-05-25 10:30:18 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.4 MB) -2016-05-25 10:30:18 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52567 (size: 1567.0 B, free: 730.6 MB) -2016-05-25 10:30:18 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:30:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383) -2016-05-25 10:30:18 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 10:30:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:30:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:30:18 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-25 10:30:18 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-25 10:30:18 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 10:30:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 10:30:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 35 ms -2016-05-25 10:30:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 38 ms -2016-05-25 10:30:18 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1161 bytes result sent to driver -2016-05-25 10:30:18 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1565 bytes result sent to driver -2016-05-25 10:30:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 87 ms on localhost (1/2) -2016-05-25 10:30:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 90 ms on localhost (2/2) -2016-05-25 10:30:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 10:30:18 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52567 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:30:18 INFO DAGScheduler:59 - ResultStage 2 (collect at NaiveBayes.scala:401) finished in 0.092 s -2016-05-25 10:30:18 INFO DAGScheduler:59 - Job 1 finished: collect at NaiveBayes.scala:401, took 0.300770 s -2016-05-25 10:30:18 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52567 in memory (size: 2015.0 B, free: 730.6 MB) -2016-05-25 10:30:18 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:40 -2016-05-25 10:30:18 INFO DAGScheduler:59 - Got job 2 (count at NaiveBayesLearning.scala:40) with 2 output partitions -2016-05-25 10:30:18 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at NaiveBayesLearning.scala:40) -2016-05-25 10:30:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:30:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:30:18 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:40), which has no missing parents -2016-05-25 10:30:18 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=176083, maxMem=766075207 -2016-05-25 10:30:18 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 10:30:18 INFO MemoryStore:59 - ensureFreeSpace(2627) called with curMem=180907, maxMem=766075207 -2016-05-25 10:30:18 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:30:18 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52567 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:30:18 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:30:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:40) -2016-05-25 10:30:18 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 10:30:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:30:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:30:18 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-25 10:30:18 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-25 10:30:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:30:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:30:18 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 10:30:18 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 10:30:18 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-25 10:30:18 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 2082 bytes result sent to driver -2016-05-25 10:30:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 28 ms on localhost (1/2) -2016-05-25 10:30:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 28 ms on localhost (2/2) -2016-05-25 10:30:18 INFO DAGScheduler:59 - ResultStage 3 (count at NaiveBayesLearning.scala:40) finished in 0.029 s -2016-05-25 10:30:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 10:30:18 INFO DAGScheduler:59 - Job 2 finished: count at NaiveBayesLearning.scala:40, took 0.044848 s -2016-05-25 10:30:18 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:40 -2016-05-25 10:30:18 INFO DAGScheduler:59 - Got job 3 (count at NaiveBayesLearning.scala:40) with 2 output partitions -2016-05-25 10:30:18 INFO DAGScheduler:59 - Final stage: ResultStage 4(count at NaiveBayesLearning.scala:40) -2016-05-25 10:30:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:30:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:30:18 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:30:18 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=183534, maxMem=766075207 -2016-05-25 10:30:18 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:30:18 INFO MemoryStore:59 - ensureFreeSpace(2121) called with curMem=187134, maxMem=766075207 -2016-05-25 10:30:18 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:30:18 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52567 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:30:18 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:30:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:30:18 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 10:30:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:30:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:30:18 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-25 10:30:18 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-25 10:30:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:30:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:30:18 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2082 bytes result sent to driver -2016-05-25 10:30:18 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2082 bytes result sent to driver -2016-05-25 10:30:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 17 ms on localhost (1/2) -2016-05-25 10:30:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 17 ms on localhost (2/2) -2016-05-25 10:30:18 INFO DAGScheduler:59 - ResultStage 4 (count at NaiveBayesLearning.scala:40) finished in 0.019 s -2016-05-25 10:30:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 10:30:18 INFO DAGScheduler:59 - Job 3 finished: count at NaiveBayesLearning.scala:40, took 0.032626 s -2016-05-25 10:30:18 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:44 -2016-05-25 10:30:18 INFO DAGScheduler:59 - Got job 4 (count at NaiveBayesLearning.scala:44) with 2 output partitions -2016-05-25 10:30:18 INFO DAGScheduler:59 - Final stage: ResultStage 5(count at NaiveBayesLearning.scala:44) -2016-05-25 10:30:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:30:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:30:18 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:30:18 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=189255, maxMem=766075207 -2016-05-25 10:30:18 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:30:18 INFO MemoryStore:59 - ensureFreeSpace(2117) called with curMem=192855, maxMem=766075207 -2016-05-25 10:30:18 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:30:18 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52567 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:30:18 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:30:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:30:18 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 10:30:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:30:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:30:18 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-05-25 10:30:18 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-05-25 10:30:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:30:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:30:18 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 2082 bytes result sent to driver -2016-05-25 10:30:18 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 2082 bytes result sent to driver -2016-05-25 10:30:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 19 ms on localhost (1/2) -2016-05-25 10:30:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 19 ms on localhost (2/2) -2016-05-25 10:30:18 INFO DAGScheduler:59 - ResultStage 5 (count at NaiveBayesLearning.scala:44) finished in 0.020 s -2016-05-25 10:30:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 10:30:18 INFO DAGScheduler:59 - Job 4 finished: count at NaiveBayesLearning.scala:44, took 0.033173 s -2016-05-25 10:30:18 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:45 -2016-05-25 10:30:18 INFO DAGScheduler:59 - Got job 5 (foreach at NaiveBayesLearning.scala:45) with 2 output partitions -2016-05-25 10:30:18 INFO DAGScheduler:59 - Final stage: ResultStage 6(foreach at NaiveBayesLearning.scala:45) -2016-05-25 10:30:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:30:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:30:18 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:30:18 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=194972, maxMem=766075207 -2016-05-25 10:30:18 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-25 10:30:18 INFO MemoryStore:59 - ensureFreeSpace(2147) called with curMem=198708, maxMem=766075207 -2016-05-25 10:30:18 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:30:18 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52567 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:30:18 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:30:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:30:18 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 10:30:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:30:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:30:18 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-05-25 10:30:18 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 13) -2016-05-25 10:30:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:30:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:30:18 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 13). 2044 bytes result sent to driver -2016-05-25 10:30:18 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 2044 bytes result sent to driver -2016-05-25 10:30:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 13) in 16 ms on localhost (1/2) -2016-05-25 10:30:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 19 ms on localhost (2/2) -2016-05-25 10:30:18 INFO DAGScheduler:59 - ResultStage 6 (foreach at NaiveBayesLearning.scala:45) finished in 0.019 s -2016-05-25 10:30:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 10:30:18 INFO DAGScheduler:59 - Job 5 finished: foreach at NaiveBayesLearning.scala:45, took 0.032391 s -2016-05-25 10:30:18 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:46 -2016-05-25 10:30:18 INFO DAGScheduler:59 - Got job 6 (count at NaiveBayesLearning.scala:46) with 2 output partitions -2016-05-25 10:30:18 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at NaiveBayesLearning.scala:46) -2016-05-25 10:30:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:30:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:30:18 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:30:18 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=200855, maxMem=766075207 -2016-05-25 10:30:18 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:30:18 INFO MemoryStore:59 - ensureFreeSpace(2121) called with curMem=204455, maxMem=766075207 -2016-05-25 10:30:18 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:30:18 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52567 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:30:18 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:30:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:30:18 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 10:30:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:30:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 15, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:30:18 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 15) -2016-05-25 10:30:18 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 14) -2016-05-25 10:30:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:30:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:30:18 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 14). 2082 bytes result sent to driver -2016-05-25 10:30:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 14) in 18 ms on localhost (1/2) -2016-05-25 10:30:18 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 10:30:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 15) in 20 ms on localhost (2/2) -2016-05-25 10:30:18 INFO DAGScheduler:59 - ResultStage 7 (count at NaiveBayesLearning.scala:46) finished in 0.022 s -2016-05-25 10:30:18 INFO DAGScheduler:59 - Job 6 finished: count at NaiveBayesLearning.scala:46, took 0.040383 s -2016-05-25 10:30:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 10:30:18 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:47 -2016-05-25 10:30:18 INFO DAGScheduler:59 - Got job 7 (foreach at NaiveBayesLearning.scala:47) with 2 output partitions -2016-05-25 10:30:18 INFO DAGScheduler:59 - Final stage: ResultStage 8(foreach at NaiveBayesLearning.scala:47) -2016-05-25 10:30:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:30:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:30:18 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:30:18 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=206576, maxMem=766075207 -2016-05-25 10:30:18 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-25 10:30:18 INFO MemoryStore:59 - ensureFreeSpace(2151) called with curMem=210312, maxMem=766075207 -2016-05-25 10:30:18 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:30:18 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52567 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:30:18 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:30:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:30:18 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 10:30:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:30:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 17, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:30:18 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 16) -2016-05-25 10:30:18 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 17) -2016-05-25 10:30:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:30:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:30:18 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 16). 2044 bytes result sent to driver -2016-05-25 10:30:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 16) in 23 ms on localhost (1/2) -2016-05-25 10:30:18 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 17). 2044 bytes result sent to driver -2016-05-25 10:30:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 17) in 34 ms on localhost (2/2) -2016-05-25 10:30:18 INFO DAGScheduler:59 - ResultStage 8 (foreach at NaiveBayesLearning.scala:47) finished in 0.036 s -2016-05-25 10:30:18 INFO DAGScheduler:59 - Job 7 finished: foreach at NaiveBayesLearning.scala:47, took 0.374577 s -2016-05-25 10:30:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 10:30:18 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:50 -2016-05-25 10:30:18 INFO DAGScheduler:59 - Got job 8 (take at NaiveBayesLearning.scala:50) with 1 output partitions -2016-05-25 10:30:18 INFO DAGScheduler:59 - Final stage: ResultStage 9(take at NaiveBayesLearning.scala:50) -2016-05-25 10:30:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:30:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:30:18 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-25 10:30:18 INFO MemoryStore:59 - ensureFreeSpace(4752) called with curMem=212463, maxMem=766075207 -2016-05-25 10:30:18 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.6 KB, free 730.4 MB) -2016-05-25 10:30:18 INFO MemoryStore:59 - ensureFreeSpace(2626) called with curMem=217215, maxMem=766075207 -2016-05-25 10:30:18 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:30:18 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:52567 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:30:18 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:30:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 9 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39) -2016-05-25 10:30:18 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 1 tasks -2016-05-25 10:30:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:30:18 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 18) -2016-05-25 10:30:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:30:18 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 18). 2281 bytes result sent to driver -2016-05-25 10:30:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 18) in 26 ms on localhost (1/1) -2016-05-25 10:30:18 INFO DAGScheduler:59 - ResultStage 9 (take at NaiveBayesLearning.scala:50) finished in 0.027 s -2016-05-25 10:30:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 10:30:18 INFO DAGScheduler:59 - Job 8 finished: take at NaiveBayesLearning.scala:50, took 0.051062 s -2016-05-25 10:30:18 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:50 -2016-05-25 10:30:18 INFO DAGScheduler:59 - Got job 9 (take at NaiveBayesLearning.scala:50) with 1 output partitions -2016-05-25 10:30:18 INFO DAGScheduler:59 - Final stage: ResultStage 10(take at NaiveBayesLearning.scala:50) -2016-05-25 10:30:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:30:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:30:18 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-25 10:30:18 INFO MemoryStore:59 - ensureFreeSpace(4752) called with curMem=219841, maxMem=766075207 -2016-05-25 10:30:18 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.6 KB, free 730.4 MB) -2016-05-25 10:30:18 INFO MemoryStore:59 - ensureFreeSpace(2626) called with curMem=224593, maxMem=766075207 -2016-05-25 10:30:18 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:30:18 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:52567 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:30:18 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:30:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39) -2016-05-25 10:30:18 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-25 10:30:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:30:19 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 10:30:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:30:19 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2281 bytes result sent to driver -2016-05-25 10:30:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 24 ms on localhost (1/1) -2016-05-25 10:30:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 10:30:19 INFO DAGScheduler:59 - ResultStage 10 (take at NaiveBayesLearning.scala:50) finished in 0.028 s -2016-05-25 10:30:19 INFO DAGScheduler:59 - Job 9 finished: take at NaiveBayesLearning.scala:50, took 0.058573 s -2016-05-25 10:30:19 INFO SparkContext:59 - Starting job: saveAsTextFile at NaiveBayes.scala:202 -2016-05-25 10:30:19 INFO DAGScheduler:59 - Got job 10 (saveAsTextFile at NaiveBayes.scala:202) with 1 output partitions -2016-05-25 10:30:19 INFO DAGScheduler:59 - Final stage: ResultStage 11(saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 10:30:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:30:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:30:19 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202), which has no missing parents -2016-05-25 10:30:19 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=227219, maxMem=766075207 -2016-05-25 10:30:19 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 124.0 KB, free 730.2 MB) -2016-05-25 10:30:19 INFO MemoryStore:59 - ensureFreeSpace(42584) called with curMem=354211, maxMem=766075207 -2016-05-25 10:30:19 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.2 MB) -2016-05-25 10:30:19 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:52567 (size: 41.6 KB, free: 730.5 MB) -2016-05-25 10:30:19 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:30:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 11 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 10:30:19 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 1 tasks -2016-05-25 10:30:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 20, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-25 10:30:19 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 20) -2016-05-25 10:30:19 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251030_0011_m_000000_20' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103019032/result/metadata/_temporary/0/task_201605251030_0011_m_000000 -2016-05-25 10:30:19 INFO SparkHadoopMapRedUtil:59 - attempt_201605251030_0011_m_000000_20: Committed -2016-05-25 10:30:19 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 20). 1864 bytes result sent to driver -2016-05-25 10:30:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 20) in 216 ms on localhost (1/1) -2016-05-25 10:30:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 10:30:19 INFO DAGScheduler:59 - ResultStage 11 (saveAsTextFile at NaiveBayes.scala:202) finished in 0.217 s -2016-05-25 10:30:19 INFO DAGScheduler:59 - Job 10 finished: saveAsTextFile at NaiveBayes.scala:202, took 0.303910 s -2016-05-25 10:30:20 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:52567 in memory (size: 41.6 KB, free: 730.6 MB) -2016-05-25 10:30:20 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-25 10:30:20 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:52567 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:30:20 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 10:30:20 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:52567 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:30:20 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 10:30:20 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:52567 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:30:20 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 10:30:20 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:52567 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:30:20 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 10:30:20 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:52567 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:30:20 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 10:30:20 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:52567 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:30:20 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 10:30:20 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52567 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:30:20 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 10:30:20 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52567 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:30:20 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 10:30:20 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52567 in memory (size: 1567.0 B, free: 730.6 MB) -2016-05-25 10:30:21 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:30:21 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:30:21 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:206 -2016-05-25 10:30:21 INFO DAGScheduler:59 - Got job 11 (parquet at NaiveBayes.scala:206) with 1 output partitions -2016-05-25 10:30:21 INFO DAGScheduler:59 - Final stage: ResultStage 12(parquet at NaiveBayes.scala:206) -2016-05-25 10:30:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:30:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:30:21 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205), which has no missing parents -2016-05-25 10:30:21 INFO MemoryStore:59 - ensureFreeSpace(66936) called with curMem=171580, maxMem=766075207 -2016-05-25 10:30:21 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 65.4 KB, free 730.4 MB) -2016-05-25 10:30:21 INFO MemoryStore:59 - ensureFreeSpace(23362) called with curMem=238516, maxMem=766075207 -2016-05-25 10:30:21 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 22.8 KB, free 730.3 MB) -2016-05-25 10:30:21 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:52567 (size: 22.8 KB, free: 730.6 MB) -2016-05-25 10:30:21 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:30:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205) -2016-05-25 10:30:21 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-25 10:30:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 21, localhost, PROCESS_LOCAL, 2456 bytes) -2016-05-25 10:30:21 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 21) -2016-05-25 10:30:21 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:30:21 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 10:30:21 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 10:30:21 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 10:30:21 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 10:30:21 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 10:30:21 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 10:30:21 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 10:30:21 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 10:30:21 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 115 -2016-05-25 10:30:21 INFO ColumnChunkPageWriteStore:151 - written 74B for [labels, bag, array] DOUBLE: 2 values, 29B raw, 33B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 10:30:21 INFO ColumnChunkPageWriteStore:151 - written 72B for [pi, bag, array] DOUBLE: 2 values, 15B raw, 31B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 1 entries, 8B raw, 1B comp} -2016-05-25 10:30:21 INFO ColumnChunkPageWriteStore:151 - written 78B for [theta, bag, array, bag, array] DOUBLE: 6 values, 19B raw, 37B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 4 entries, 32B raw, 4B comp} -2016-05-25 10:30:21 INFO ColumnChunkPageWriteStore:151 - written 86B for [modelType] BINARY: 1 values, 21B raw, 39B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 10:30:21 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251030_0012_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103019032/result/data/_temporary/0/task_201605251030_0012_m_000000 -2016-05-25 10:30:21 INFO SparkHadoopMapRedUtil:59 - attempt_201605251030_0012_m_000000_0: Committed -2016-05-25 10:30:21 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 21). 915 bytes result sent to driver -2016-05-25 10:30:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 21) in 486 ms on localhost (1/1) -2016-05-25 10:30:21 INFO DAGScheduler:59 - ResultStage 12 (parquet at NaiveBayes.scala:206) finished in 0.488 s -2016-05-25 10:30:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 10:30:21 INFO DAGScheduler:59 - Job 11 finished: parquet at NaiveBayes.scala:206, took 0.551354 s -2016-05-25 10:30:22 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:30:22 INFO DefaultWriterContainer:59 - Job job_201605251030_0000 committed. -2016-05-25 10:30:22 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103019032/result/data on driver -2016-05-25 10:30:22 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103019032/result/data on driver -2016-05-25 10:30:22 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=261878, maxMem=766075207 -2016-05-25 10:30:22 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-25 10:30:22 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=492582, maxMem=766075207 -2016-05-25 10:30:22 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-25 10:30:22 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:52567 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 10:30:22 INFO SparkContext:59 - Created broadcast 14 from textFile at modelSaveLoad.scala:129 -2016-05-25 10:30:22 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:30:22 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 10:30:22 INFO DAGScheduler:59 - Got job 12 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 10:30:22 INFO DAGScheduler:59 - Final stage: ResultStage 13(first at modelSaveLoad.scala:129) -2016-05-25 10:30:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:30:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:30:22 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 10:30:22 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=512370, maxMem=766075207 -2016-05-25 10:30:22 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-25 10:30:22 INFO MemoryStore:59 - ensureFreeSpace(1889) called with curMem=515586, maxMem=766075207 -2016-05-25 10:30:22 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1889.0 B, free 730.1 MB) -2016-05-25 10:30:22 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:52567 (size: 1889.0 B, free: 730.5 MB) -2016-05-25 10:30:22 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:30:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 13 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129) -2016-05-25 10:30:22 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 1 tasks -2016-05-25 10:30:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 22, localhost, PROCESS_LOCAL, 2239 bytes) -2016-05-25 10:30:22 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 22) -2016-05-25 10:30:22 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103019032/result/metadata/part-00000:0+56 -2016-05-25 10:30:22 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 22). 2159 bytes result sent to driver -2016-05-25 10:30:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 22) in 20 ms on localhost (1/1) -2016-05-25 10:30:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 10:30:22 INFO DAGScheduler:59 - ResultStage 13 (first at modelSaveLoad.scala:129) finished in 0.020 s -2016-05-25 10:30:22 INFO DAGScheduler:59 - Job 12 finished: first at modelSaveLoad.scala:129, took 0.033099 s -2016-05-25 10:30:22 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103019032/result/data on driver -2016-05-25 10:30:22 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:213 -2016-05-25 10:30:22 INFO DAGScheduler:59 - Got job 13 (parquet at NaiveBayes.scala:213) with 4 output partitions -2016-05-25 10:30:22 INFO DAGScheduler:59 - Final stage: ResultStage 14(parquet at NaiveBayes.scala:213) -2016-05-25 10:30:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:30:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:30:22 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213), which has no missing parents -2016-05-25 10:30:22 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=517475, maxMem=766075207 -2016-05-25 10:30:22 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-25 10:30:22 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=580475, maxMem=766075207 -2016-05-25 10:30:22 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.0 MB) -2016-05-25 10:30:22 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:52567 (size: 20.6 KB, free: 730.5 MB) -2016-05-25 10:30:22 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:30:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 14 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213) -2016-05-25 10:30:22 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 4 tasks -2016-05-25 10:30:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 23, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:30:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 24, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:30:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 25, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:30:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 14.0 (TID 26, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-25 10:30:22 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 23) -2016-05-25 10:30:22 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 24) -2016-05-25 10:30:22 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 25) -2016-05-25 10:30:22 INFO Executor:59 - Running task 3.0 in stage 14.0 (TID 26) -2016-05-25 10:30:22 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:30:22 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:30:22 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 25). 936 bytes result sent to driver -2016-05-25 10:30:22 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 24). 936 bytes result sent to driver -2016-05-25 10:30:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 25) in 58 ms on localhost (1/4) -2016-05-25 10:30:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 24) in 61 ms on localhost (2/4) -2016-05-25 10:30:22 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:30:22 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 23). 936 bytes result sent to driver -2016-05-25 10:30:22 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:30:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 23) in 75 ms on localhost (3/4) -2016-05-25 10:30:22 INFO Executor:59 - Finished task 3.0 in stage 14.0 (TID 26). 1876 bytes result sent to driver -2016-05-25 10:30:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 14.0 (TID 26) in 96 ms on localhost (4/4) -2016-05-25 10:30:22 INFO DAGScheduler:59 - ResultStage 14 (parquet at NaiveBayes.scala:213) finished in 0.099 s -2016-05-25 10:30:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 10:30:22 INFO DAGScheduler:59 - Job 13 finished: parquet at NaiveBayes.scala:213, took 0.133821 s -2016-05-25 10:30:22 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=601548, maxMem=766075207 -2016-05-25 10:30:22 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 86.4 KB, free 729.9 MB) -2016-05-25 10:30:22 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=690060, maxMem=766075207 -2016-05-25 10:30:22 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-25 10:30:22 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:52567 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 10:30:22 INFO SparkContext:59 - Created broadcast 17 from take at NaiveBayes.scala:216 -2016-05-25 10:30:23 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 10:30:23 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103019032/result/data/part-r-00000-177323ce-a937-4857-8a02-38ddaa1081d3.gz.parquet -2016-05-25 10:30:23 INFO SparkContext:59 - Starting job: take at NaiveBayes.scala:216 -2016-05-25 10:30:23 INFO DAGScheduler:59 - Got job 14 (take at NaiveBayes.scala:216) with 1 output partitions -2016-05-25 10:30:23 INFO DAGScheduler:59 - Final stage: ResultStage 15(take at NaiveBayes.scala:216) -2016-05-25 10:30:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:30:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:30:23 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216), which has no missing parents -2016-05-25 10:30:23 INFO MemoryStore:59 - ensureFreeSpace(4280) called with curMem=709848, maxMem=766075207 -2016-05-25 10:30:23 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 4.2 KB, free 729.9 MB) -2016-05-25 10:30:23 INFO MemoryStore:59 - ensureFreeSpace(2466) called with curMem=714128, maxMem=766075207 -2016-05-25 10:30:23 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.4 KB, free 729.9 MB) -2016-05-25 10:30:23 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:52567 (size: 2.4 KB, free: 730.5 MB) -2016-05-25 10:30:23 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:30:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 15 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216) -2016-05-25 10:30:23 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 1 tasks -2016-05-25 10:30:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 27, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-25 10:30:23 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 27) -2016-05-25 10:30:23 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103019032/result/data/part-r-00000-177323ce-a937-4857-8a02-38ddaa1081d3.gz.parquet start: 0 end: 1441 length: 1441 hosts: []} -2016-05-25 10:30:23 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 10:30:23 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group labels (LIST) { - repeated group bag { - optional double array; - } - } - optional group pi (LIST) { - repeated group bag { - optional double array; - } - } - optional group theta (LIST) { - repeated group bag { - optional group array (LIST) { - repeated group bag { - optional double array; - } - } - } - } - optional binary modelType (UTF8); -} - -Catalyst form: -StructType(StructField(labels,ArrayType(DoubleType,true),true), StructField(pi,ArrayType(DoubleType,true),true), StructField(theta,ArrayType(ArrayType(DoubleType,true),true),true), StructField(modelType,StringType,true)) - -2016-05-25 10:30:23 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 10:30:23 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 10:30:23 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 10:30:23 INFO InternalParquetRecordReader:151 - block read in memory in 11 ms. row count = 1 -2016-05-25 10:30:23 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 27). 2755 bytes result sent to driver -2016-05-25 10:30:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 27) in 113 ms on localhost (1/1) -2016-05-25 10:30:23 INFO DAGScheduler:59 - ResultStage 15 (take at NaiveBayes.scala:216) finished in 0.114 s -2016-05-25 10:30:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-25 10:30:23 INFO DAGScheduler:59 - Job 14 finished: take at NaiveBayes.scala:216, took 0.132535 s -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 10:30:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 10:30:23 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 10:30:23 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 10:30:23 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 10:30:23 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 10:30:23 INFO BlockManager:59 - BlockManager stopped -2016-05-25 10:30:23 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 10:30:23 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 10:30:23 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 10:30:23 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 10:30:23 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-c54906d3-e422-4762-9e76-c7503e8fb124 -2016-05-25 10:31:51 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 10:31:52 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 10:31:52 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 10:31:52 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 10:31:52 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 10:31:53 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 10:31:53 INFO Remoting:74 - Starting remoting -2016-05-25 10:31:53 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:52595] -2016-05-25 10:31:53 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52595. -2016-05-25 10:31:53 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 10:31:53 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 10:31:53 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-33a9931b-8dd0-4649-9a68-3112e8b20cb5 -2016-05-25 10:31:53 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 10:31:53 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-22026809-e67d-4125-a059-df98f3dfcb31\httpd-465eeecf-580a-43ac-a4c9-1cea6b5682ca -2016-05-25 10:31:53 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 10:31:53 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:31:53 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52596 -2016-05-25 10:31:53 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52596. -2016-05-25 10:31:53 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 10:31:54 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:31:54 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 10:31:54 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 10:31:54 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 10:31:54 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 10:31:54 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 10:31:54 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52615. -2016-05-25 10:31:54 INFO NettyBlockTransferService:59 - Server created on 52615 -2016-05-25 10:31:54 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 10:31:54 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52615 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52615) -2016-05-25 10:31:54 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 10:31:55 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 10:31:55 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 10:31:55 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 10:31:55 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 10:31:55 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52615 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 10:31:55 INFO SparkContext:59 - Created broadcast 0 from textFile at NaiveBayesLearning.scala:26 -2016-05-25 10:31:56 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 10:31:56 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:31:56 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:32 -2016-05-25 10:31:56 INFO DAGScheduler:59 - Got job 0 (foreach at NaiveBayesLearning.scala:32) with 2 output partitions -2016-05-25 10:31:56 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at NaiveBayesLearning.scala:32) -2016-05-25 10:31:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:31:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:31:56 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at NaiveBayesLearning.scala:27), which has no missing parents -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(3432) called with curMem=171580, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(2013) called with curMem=175012, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2013.0 B, free 730.4 MB) -2016-05-25 10:31:57 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52615 (size: 2013.0 B, free: 730.6 MB) -2016-05-25 10:31:57 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at NaiveBayesLearning.scala:27) -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 10:31:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:31:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:31:57 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 10:31:57 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 10:31:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:31:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:31:57 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 10:31:57 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 10:31:57 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 10:31:57 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 10:31:57 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 10:31:57 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-25 10:31:57 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-25 10:31:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 100 ms on localhost (1/2) -2016-05-25 10:31:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 126 ms on localhost (2/2) -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 10:31:57 INFO DAGScheduler:59 - ResultStage 0 (foreach at NaiveBayesLearning.scala:32) finished in 0.144 s -2016-05-25 10:31:57 INFO DAGScheduler:59 - Job 0 finished: foreach at NaiveBayesLearning.scala:32, took 0.274884 s -2016-05-25 10:31:57 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:31:57 INFO DAGScheduler:59 - Registering RDD 5 (map at NaiveBayes.scala:383) -2016-05-25 10:31:57 INFO DAGScheduler:59 - Got job 1 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:31:57 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at NaiveBayes.scala:401) -2016-05-25 10:31:57 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1) -2016-05-25 10:31:57 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1) -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(5104) called with curMem=177025, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(2697) called with curMem=182129, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:31:57 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52615 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:31:57 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383) -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-25 10:31:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 10:31:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 10:31:57 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 10:31:57 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-25 10:31:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:31:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:31:57 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 10:31:57 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 10:31:57 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2254 bytes result sent to driver -2016-05-25 10:31:57 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2254 bytes result sent to driver -2016-05-25 10:31:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 112 ms on localhost (1/2) -2016-05-25 10:31:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 111 ms on localhost (2/2) -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 10:31:57 INFO DAGScheduler:59 - ShuffleMapStage 1 (map at NaiveBayes.scala:383) finished in 0.116 s -2016-05-25 10:31:57 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 10:31:57 INFO DAGScheduler:59 - running: Set() -2016-05-25 10:31:57 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-05-25 10:31:57 INFO DAGScheduler:59 - failed: Set() -2016-05-25 10:31:57 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List() -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting ResultStage 2 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(2936) called with curMem=184826, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=187762, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.4 MB) -2016-05-25 10:31:57 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52615 (size: 1567.0 B, free: 730.6 MB) -2016-05-25 10:31:57 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383) -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 10:31:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:31:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:31:57 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-25 10:31:57 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-25 10:31:57 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 10:31:57 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 10:31:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 31 ms -2016-05-25 10:31:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 31 ms -2016-05-25 10:31:57 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52615 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:31:57 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1161 bytes result sent to driver -2016-05-25 10:31:57 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1645 bytes result sent to driver -2016-05-25 10:31:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 65 ms on localhost (1/2) -2016-05-25 10:31:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 71 ms on localhost (2/2) -2016-05-25 10:31:57 INFO DAGScheduler:59 - ResultStage 2 (collect at NaiveBayes.scala:401) finished in 0.072 s -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 10:31:57 INFO DAGScheduler:59 - Job 1 finished: collect at NaiveBayes.scala:401, took 0.233082 s -2016-05-25 10:31:57 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52615 in memory (size: 2013.0 B, free: 730.6 MB) -2016-05-25 10:31:57 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:40 -2016-05-25 10:31:57 INFO DAGScheduler:59 - Got job 2 (count at NaiveBayesLearning.scala:40) with 2 output partitions -2016-05-25 10:31:57 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at NaiveBayesLearning.scala:40) -2016-05-25 10:31:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:31:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:40), which has no missing parents -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=176083, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(2670) called with curMem=180979, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:31:57 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52615 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:31:57 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:40) -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 10:31:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:31:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:31:57 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-25 10:31:57 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-25 10:31:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:31:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:31:57 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-25 10:31:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 20 ms on localhost (1/2) -2016-05-25 10:31:57 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 2082 bytes result sent to driver -2016-05-25 10:31:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 23 ms on localhost (2/2) -2016-05-25 10:31:57 INFO DAGScheduler:59 - ResultStage 3 (count at NaiveBayesLearning.scala:40) finished in 0.026 s -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 10:31:57 INFO DAGScheduler:59 - Job 2 finished: count at NaiveBayesLearning.scala:40, took 0.038341 s -2016-05-25 10:31:57 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:40 -2016-05-25 10:31:57 INFO DAGScheduler:59 - Got job 3 (count at NaiveBayesLearning.scala:40) with 2 output partitions -2016-05-25 10:31:57 INFO DAGScheduler:59 - Final stage: ResultStage 4(count at NaiveBayesLearning.scala:40) -2016-05-25 10:31:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:31:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=183649, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(2116) called with curMem=187249, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:31:57 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52615 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:31:57 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 10:31:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:31:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:31:57 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-25 10:31:57 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-25 10:31:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:31:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:31:57 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2082 bytes result sent to driver -2016-05-25 10:31:57 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2082 bytes result sent to driver -2016-05-25 10:31:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 18 ms on localhost (1/2) -2016-05-25 10:31:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 18 ms on localhost (2/2) -2016-05-25 10:31:57 INFO DAGScheduler:59 - ResultStage 4 (count at NaiveBayesLearning.scala:40) finished in 0.019 s -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 10:31:57 INFO DAGScheduler:59 - Job 3 finished: count at NaiveBayesLearning.scala:40, took 0.032353 s -2016-05-25 10:31:57 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:44 -2016-05-25 10:31:57 INFO DAGScheduler:59 - Got job 4 (count at NaiveBayesLearning.scala:44) with 2 output partitions -2016-05-25 10:31:57 INFO DAGScheduler:59 - Final stage: ResultStage 5(count at NaiveBayesLearning.scala:44) -2016-05-25 10:31:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:31:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=189365, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(2112) called with curMem=192965, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:31:57 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52615 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:31:57 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 10:31:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:31:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:31:57 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-05-25 10:31:57 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-05-25 10:31:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:31:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:31:57 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 2082 bytes result sent to driver -2016-05-25 10:31:57 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 2082 bytes result sent to driver -2016-05-25 10:31:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 18 ms on localhost (1/2) -2016-05-25 10:31:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 18 ms on localhost (2/2) -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 10:31:57 INFO DAGScheduler:59 - ResultStage 5 (count at NaiveBayesLearning.scala:44) finished in 0.019 s -2016-05-25 10:31:57 INFO DAGScheduler:59 - Job 4 finished: count at NaiveBayesLearning.scala:44, took 0.033150 s -2016-05-25 10:31:57 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:45 -2016-05-25 10:31:57 INFO DAGScheduler:59 - Got job 5 (foreach at NaiveBayesLearning.scala:45) with 2 output partitions -2016-05-25 10:31:57 INFO DAGScheduler:59 - Final stage: ResultStage 6(foreach at NaiveBayesLearning.scala:45) -2016-05-25 10:31:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:31:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=195077, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(2142) called with curMem=198813, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:31:57 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52615 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:31:57 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 10:31:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:31:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:31:57 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-05-25 10:31:57 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 13) -2016-05-25 10:31:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:31:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:31:57 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 2044 bytes result sent to driver -2016-05-25 10:31:57 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 13). 2044 bytes result sent to driver -2016-05-25 10:31:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 19 ms on localhost (1/2) -2016-05-25 10:31:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 13) in 19 ms on localhost (2/2) -2016-05-25 10:31:57 INFO DAGScheduler:59 - ResultStage 6 (foreach at NaiveBayesLearning.scala:45) finished in 0.020 s -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 10:31:57 INFO DAGScheduler:59 - Job 5 finished: foreach at NaiveBayesLearning.scala:45, took 0.034358 s -2016-05-25 10:31:57 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:46 -2016-05-25 10:31:57 INFO DAGScheduler:59 - Got job 6 (count at NaiveBayesLearning.scala:46) with 2 output partitions -2016-05-25 10:31:57 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at NaiveBayesLearning.scala:46) -2016-05-25 10:31:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:31:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=200955, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(2116) called with curMem=204555, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:31:57 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52615 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:31:57 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 10:31:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:31:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 15, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:31:57 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 14) -2016-05-25 10:31:57 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 15) -2016-05-25 10:31:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:31:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:31:57 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 14). 2082 bytes result sent to driver -2016-05-25 10:31:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 14) in 17 ms on localhost (1/2) -2016-05-25 10:31:57 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 10:31:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 15) in 20 ms on localhost (2/2) -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 10:31:57 INFO DAGScheduler:59 - ResultStage 7 (count at NaiveBayesLearning.scala:46) finished in 0.022 s -2016-05-25 10:31:57 INFO DAGScheduler:59 - Job 6 finished: count at NaiveBayesLearning.scala:46, took 0.038768 s -2016-05-25 10:31:57 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:47 -2016-05-25 10:31:57 INFO DAGScheduler:59 - Got job 7 (foreach at NaiveBayesLearning.scala:47) with 2 output partitions -2016-05-25 10:31:57 INFO DAGScheduler:59 - Final stage: ResultStage 8(foreach at NaiveBayesLearning.scala:47) -2016-05-25 10:31:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:31:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=206671, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=210407, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:31:57 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52615 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:31:57 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 10:31:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:31:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 17, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:31:57 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 16) -2016-05-25 10:31:57 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 17) -2016-05-25 10:31:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:31:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:31:57 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 16). 2044 bytes result sent to driver -2016-05-25 10:31:57 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 17). 2044 bytes result sent to driver -2016-05-25 10:31:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 17) in 18 ms on localhost (1/2) -2016-05-25 10:31:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 16) in 19 ms on localhost (2/2) -2016-05-25 10:31:57 INFO DAGScheduler:59 - ResultStage 8 (foreach at NaiveBayesLearning.scala:47) finished in 0.020 s -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 10:31:57 INFO DAGScheduler:59 - Job 7 finished: foreach at NaiveBayesLearning.scala:47, took 0.038091 s -2016-05-25 10:31:57 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:50 -2016-05-25 10:31:57 INFO DAGScheduler:59 - Got job 8 (take at NaiveBayesLearning.scala:50) with 1 output partitions -2016-05-25 10:31:57 INFO DAGScheduler:59 - Final stage: ResultStage 9(take at NaiveBayesLearning.scala:50) -2016-05-25 10:31:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:31:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=212553, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(2669) called with curMem=217377, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:31:57 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:52615 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:31:57 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 9 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39) -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 1 tasks -2016-05-25 10:31:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:31:57 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 18) -2016-05-25 10:31:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:31:57 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 18). 2040 bytes result sent to driver -2016-05-25 10:31:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 18) in 22 ms on localhost (1/1) -2016-05-25 10:31:57 INFO DAGScheduler:59 - ResultStage 9 (take at NaiveBayesLearning.scala:50) finished in 0.022 s -2016-05-25 10:31:57 INFO DAGScheduler:59 - Job 8 finished: take at NaiveBayesLearning.scala:50, took 0.037885 s -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 10:31:57 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:50 -2016-05-25 10:31:57 INFO DAGScheduler:59 - Got job 9 (take at NaiveBayesLearning.scala:50) with 1 output partitions -2016-05-25 10:31:57 INFO DAGScheduler:59 - Final stage: ResultStage 10(take at NaiveBayesLearning.scala:50) -2016-05-25 10:31:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:31:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=220046, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 10:31:57 INFO MemoryStore:59 - ensureFreeSpace(2669) called with curMem=224870, maxMem=766075207 -2016-05-25 10:31:57 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:31:57 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:52615 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:31:57 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:31:57 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39) -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-25 10:31:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:31:57 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 10:31:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:31:57 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2206 bytes result sent to driver -2016-05-25 10:31:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 16 ms on localhost (1/1) -2016-05-25 10:31:57 INFO DAGScheduler:59 - ResultStage 10 (take at NaiveBayesLearning.scala:50) finished in 0.018 s -2016-05-25 10:31:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 10:31:57 INFO DAGScheduler:59 - Job 9 finished: take at NaiveBayesLearning.scala:50, took 0.033552 s -2016-05-25 10:31:58 INFO SparkContext:59 - Starting job: saveAsTextFile at NaiveBayes.scala:202 -2016-05-25 10:31:58 INFO DAGScheduler:59 - Got job 10 (saveAsTextFile at NaiveBayes.scala:202) with 1 output partitions -2016-05-25 10:31:58 INFO DAGScheduler:59 - Final stage: ResultStage 11(saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 10:31:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:31:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:31:58 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202), which has no missing parents -2016-05-25 10:31:58 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=227539, maxMem=766075207 -2016-05-25 10:31:58 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 124.0 KB, free 730.2 MB) -2016-05-25 10:31:58 INFO MemoryStore:59 - ensureFreeSpace(42585) called with curMem=354531, maxMem=766075207 -2016-05-25 10:31:58 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.2 MB) -2016-05-25 10:31:58 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:52615 (size: 41.6 KB, free: 730.5 MB) -2016-05-25 10:31:58 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:31:58 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 11 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 10:31:58 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 1 tasks -2016-05-25 10:31:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 20, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-25 10:31:58 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 20) -2016-05-25 10:31:58 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251031_0011_m_000000_20' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103157921/result/metadata/_temporary/0/task_201605251031_0011_m_000000 -2016-05-25 10:31:58 INFO SparkHadoopMapRedUtil:59 - attempt_201605251031_0011_m_000000_20: Committed -2016-05-25 10:31:58 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 20). 1864 bytes result sent to driver -2016-05-25 10:31:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 20) in 233 ms on localhost (1/1) -2016-05-25 10:31:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 10:31:58 INFO DAGScheduler:59 - ResultStage 11 (saveAsTextFile at NaiveBayes.scala:202) finished in 0.233 s -2016-05-25 10:31:58 INFO DAGScheduler:59 - Job 10 finished: saveAsTextFile at NaiveBayes.scala:202, took 0.335680 s -2016-05-25 10:31:58 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:52615 in memory (size: 41.6 KB, free: 730.6 MB) -2016-05-25 10:31:58 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-25 10:31:58 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:52615 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:31:58 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 10:31:58 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:52615 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:31:58 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 10:31:58 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:52615 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:31:58 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 10:31:58 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:52615 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:31:58 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 10:31:58 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:52615 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:31:58 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 10:31:58 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:52615 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:31:58 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 10:31:58 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52615 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:31:58 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 10:31:58 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52615 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:31:58 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 10:31:58 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52615 in memory (size: 1567.0 B, free: 730.6 MB) -2016-05-25 10:32:00 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:32:00 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:32:00 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:206 -2016-05-25 10:32:00 INFO DAGScheduler:59 - Got job 11 (parquet at NaiveBayes.scala:206) with 1 output partitions -2016-05-25 10:32:00 INFO DAGScheduler:59 - Final stage: ResultStage 12(parquet at NaiveBayes.scala:206) -2016-05-25 10:32:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:32:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:32:00 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205), which has no missing parents -2016-05-25 10:32:00 INFO MemoryStore:59 - ensureFreeSpace(66936) called with curMem=171580, maxMem=766075207 -2016-05-25 10:32:00 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 65.4 KB, free 730.4 MB) -2016-05-25 10:32:00 INFO MemoryStore:59 - ensureFreeSpace(23364) called with curMem=238516, maxMem=766075207 -2016-05-25 10:32:00 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 22.8 KB, free 730.3 MB) -2016-05-25 10:32:00 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:52615 (size: 22.8 KB, free: 730.6 MB) -2016-05-25 10:32:00 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:32:00 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205) -2016-05-25 10:32:00 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-25 10:32:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 21, localhost, PROCESS_LOCAL, 2506 bytes) -2016-05-25 10:32:00 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 21) -2016-05-25 10:32:00 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:32:00 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 10:32:00 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 10:32:00 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 10:32:00 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 10:32:00 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 10:32:00 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 10:32:00 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 10:32:00 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 10:32:00 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 231 -2016-05-25 10:32:00 INFO ColumnChunkPageWriteStore:151 - written 77B for [labels, bag, array] DOUBLE: 3 values, 37B raw, 36B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 10:32:00 INFO ColumnChunkPageWriteStore:151 - written 74B for [pi, bag, array] DOUBLE: 3 values, 16B raw, 33B comp, 1 pages, encodings: [RLE, PLAIN_DICTIONARY], dic { 2 entries, 16B raw, 2B comp} -2016-05-25 10:32:00 INFO ColumnChunkPageWriteStore:151 - written 83B for [theta, bag, array, bag, array] DOUBLE: 9 values, 23B raw, 42B comp, 1 pages, encodings: [RLE, PLAIN_DICTIONARY], dic { 6 entries, 48B raw, 6B comp} -2016-05-25 10:32:00 INFO ColumnChunkPageWriteStore:151 - written 86B for [modelType] BINARY: 1 values, 21B raw, 39B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 10:32:00 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251032_0012_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103157921/result/data/_temporary/0/task_201605251032_0012_m_000000 -2016-05-25 10:32:00 INFO SparkHadoopMapRedUtil:59 - attempt_201605251032_0012_m_000000_0: Committed -2016-05-25 10:32:00 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 21). 915 bytes result sent to driver -2016-05-25 10:32:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 21) in 471 ms on localhost (1/1) -2016-05-25 10:32:00 INFO DAGScheduler:59 - ResultStage 12 (parquet at NaiveBayes.scala:206) finished in 0.471 s -2016-05-25 10:32:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 10:32:00 INFO DAGScheduler:59 - Job 11 finished: parquet at NaiveBayes.scala:206, took 0.510718 s -2016-05-25 10:32:00 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:32:00 INFO DefaultWriterContainer:59 - Job job_201605251032_0000 committed. -2016-05-25 10:32:00 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103157921/result/data on driver -2016-05-25 10:32:00 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103157921/result/data on driver -2016-05-25 10:32:01 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=261880, maxMem=766075207 -2016-05-25 10:32:01 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-25 10:32:01 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=492584, maxMem=766075207 -2016-05-25 10:32:01 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-25 10:32:01 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:52615 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 10:32:01 INFO SparkContext:59 - Created broadcast 14 from textFile at modelSaveLoad.scala:129 -2016-05-25 10:32:01 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:32:01 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 10:32:01 INFO DAGScheduler:59 - Got job 12 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 10:32:01 INFO DAGScheduler:59 - Final stage: ResultStage 13(first at modelSaveLoad.scala:129) -2016-05-25 10:32:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:32:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:32:01 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 10:32:01 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=512372, maxMem=766075207 -2016-05-25 10:32:01 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-25 10:32:01 INFO MemoryStore:59 - ensureFreeSpace(1889) called with curMem=515588, maxMem=766075207 -2016-05-25 10:32:01 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1889.0 B, free 730.1 MB) -2016-05-25 10:32:01 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:52615 (size: 1889.0 B, free: 730.5 MB) -2016-05-25 10:32:01 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:32:01 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 13 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129) -2016-05-25 10:32:01 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 1 tasks -2016-05-25 10:32:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 22, localhost, PROCESS_LOCAL, 2239 bytes) -2016-05-25 10:32:01 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 22) -2016-05-25 10:32:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103157921/result/metadata/part-00000:0+56 -2016-05-25 10:32:01 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 22). 2159 bytes result sent to driver -2016-05-25 10:32:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 22) in 13 ms on localhost (1/1) -2016-05-25 10:32:01 INFO DAGScheduler:59 - ResultStage 13 (first at modelSaveLoad.scala:129) finished in 0.014 s -2016-05-25 10:32:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 10:32:01 INFO DAGScheduler:59 - Job 12 finished: first at modelSaveLoad.scala:129, took 0.024811 s -2016-05-25 10:32:01 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103157921/result/data on driver -2016-05-25 10:32:01 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:213 -2016-05-25 10:32:01 INFO DAGScheduler:59 - Got job 13 (parquet at NaiveBayes.scala:213) with 4 output partitions -2016-05-25 10:32:01 INFO DAGScheduler:59 - Final stage: ResultStage 14(parquet at NaiveBayes.scala:213) -2016-05-25 10:32:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:32:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:32:01 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213), which has no missing parents -2016-05-25 10:32:01 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=517477, maxMem=766075207 -2016-05-25 10:32:01 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-25 10:32:01 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=580477, maxMem=766075207 -2016-05-25 10:32:01 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.0 MB) -2016-05-25 10:32:01 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:52615 (size: 20.6 KB, free: 730.5 MB) -2016-05-25 10:32:01 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:32:01 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 14 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213) -2016-05-25 10:32:01 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 4 tasks -2016-05-25 10:32:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 23, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:32:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 24, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:32:01 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 25, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:32:01 INFO TaskSetManager:59 - Starting task 3.0 in stage 14.0 (TID 26, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-25 10:32:01 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 24) -2016-05-25 10:32:01 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 23) -2016-05-25 10:32:01 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 25) -2016-05-25 10:32:01 INFO Executor:59 - Running task 3.0 in stage 14.0 (TID 26) -2016-05-25 10:32:01 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:32:01 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:32:01 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 24). 936 bytes result sent to driver -2016-05-25 10:32:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 24) in 80 ms on localhost (1/4) -2016-05-25 10:32:01 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:32:01 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 25). 936 bytes result sent to driver -2016-05-25 10:32:01 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:32:01 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 23). 936 bytes result sent to driver -2016-05-25 10:32:01 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 25) in 95 ms on localhost (2/4) -2016-05-25 10:32:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 23) in 98 ms on localhost (3/4) -2016-05-25 10:32:01 INFO Executor:59 - Finished task 3.0 in stage 14.0 (TID 26). 1876 bytes result sent to driver -2016-05-25 10:32:01 INFO TaskSetManager:59 - Finished task 3.0 in stage 14.0 (TID 26) in 104 ms on localhost (4/4) -2016-05-25 10:32:01 INFO DAGScheduler:59 - ResultStage 14 (parquet at NaiveBayes.scala:213) finished in 0.108 s -2016-05-25 10:32:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 10:32:01 INFO DAGScheduler:59 - Job 13 finished: parquet at NaiveBayes.scala:213, took 0.142391 s -2016-05-25 10:32:01 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=601550, maxMem=766075207 -2016-05-25 10:32:01 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 86.4 KB, free 729.9 MB) -2016-05-25 10:32:01 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=690062, maxMem=766075207 -2016-05-25 10:32:01 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-25 10:32:01 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:52615 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 10:32:01 INFO SparkContext:59 - Created broadcast 17 from take at NaiveBayes.scala:216 -2016-05-25 10:32:01 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 10:32:01 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103157921/result/data/part-r-00000-8017b33d-a291-414b-94c1-08482214b160.gz.parquet -2016-05-25 10:32:01 INFO SparkContext:59 - Starting job: take at NaiveBayes.scala:216 -2016-05-25 10:32:01 INFO DAGScheduler:59 - Got job 14 (take at NaiveBayes.scala:216) with 1 output partitions -2016-05-25 10:32:01 INFO DAGScheduler:59 - Final stage: ResultStage 15(take at NaiveBayes.scala:216) -2016-05-25 10:32:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:32:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:32:01 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216), which has no missing parents -2016-05-25 10:32:01 INFO MemoryStore:59 - ensureFreeSpace(4280) called with curMem=709850, maxMem=766075207 -2016-05-25 10:32:01 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 4.2 KB, free 729.9 MB) -2016-05-25 10:32:01 INFO MemoryStore:59 - ensureFreeSpace(2466) called with curMem=714130, maxMem=766075207 -2016-05-25 10:32:01 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.4 KB, free 729.9 MB) -2016-05-25 10:32:01 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:52615 (size: 2.4 KB, free: 730.5 MB) -2016-05-25 10:32:01 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:32:01 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 15 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216) -2016-05-25 10:32:01 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 1 tasks -2016-05-25 10:32:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 27, localhost, PROCESS_LOCAL, 2305 bytes) -2016-05-25 10:32:01 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 27) -2016-05-25 10:32:01 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103157921/result/data/part-r-00000-8017b33d-a291-414b-94c1-08482214b160.gz.parquet start: 0 end: 1477 length: 1477 hosts: []} -2016-05-25 10:32:01 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 10:32:01 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group labels (LIST) { - repeated group bag { - optional double array; - } - } - optional group pi (LIST) { - repeated group bag { - optional double array; - } - } - optional group theta (LIST) { - repeated group bag { - optional group array (LIST) { - repeated group bag { - optional double array; - } - } - } - } - optional binary modelType (UTF8); -} - -Catalyst form: -StructType(StructField(labels,ArrayType(DoubleType,true),true), StructField(pi,ArrayType(DoubleType,true),true), StructField(theta,ArrayType(ArrayType(DoubleType,true),true),true), StructField(modelType,StringType,true)) - -2016-05-25 10:32:01 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 10:32:01 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 10:32:01 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 10:32:01 INFO InternalParquetRecordReader:151 - block read in memory in 13 ms. row count = 1 -2016-05-25 10:32:01 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 27). 2841 bytes result sent to driver -2016-05-25 10:32:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 27) in 116 ms on localhost (1/1) -2016-05-25 10:32:01 INFO DAGScheduler:59 - ResultStage 15 (take at NaiveBayes.scala:216) finished in 0.117 s -2016-05-25 10:32:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-25 10:32:01 INFO DAGScheduler:59 - Job 14 finished: take at NaiveBayes.scala:216, took 0.139600 s -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 10:32:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 10:32:02 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 10:32:02 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 10:32:02 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 10:32:02 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 10:32:02 INFO BlockManager:59 - BlockManager stopped -2016-05-25 10:32:02 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 10:32:02 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 10:32:02 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 10:32:02 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 10:32:02 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-22026809-e67d-4125-a059-df98f3dfcb31 -2016-05-25 10:33:03 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 10:33:04 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 10:33:04 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 10:33:04 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 10:33:04 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 10:33:05 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 10:33:05 INFO Remoting:74 - Starting remoting -2016-05-25 10:33:05 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:52639] -2016-05-25 10:33:05 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52639. -2016-05-25 10:33:05 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 10:33:05 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 10:33:05 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-a3794a52-b67d-465c-8985-78e444d03614 -2016-05-25 10:33:05 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 10:33:05 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-ee857718-04ab-482e-885e-ce491671d2a8\httpd-94c52c79-c4f2-43b0-990c-01f476362241 -2016-05-25 10:33:05 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 10:33:06 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:33:06 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52640 -2016-05-25 10:33:06 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52640. -2016-05-25 10:33:06 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 10:33:06 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:33:06 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 10:33:06 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 10:33:06 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 10:33:06 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 10:33:06 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 10:33:06 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52659. -2016-05-25 10:33:06 INFO NettyBlockTransferService:59 - Server created on 52659 -2016-05-25 10:33:06 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 10:33:06 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52659 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52659) -2016-05-25 10:33:06 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 10:33:07 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 10:33:07 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 10:33:07 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 10:33:07 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 10:33:07 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52659 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 10:33:07 INFO SparkContext:59 - Created broadcast 0 from textFile at NaiveBayesLearning.scala:26 -2016-05-25 10:33:08 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 10:33:09 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:33:09 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:32 -2016-05-25 10:33:09 INFO DAGScheduler:59 - Got job 0 (foreach at NaiveBayesLearning.scala:32) with 2 output partitions -2016-05-25 10:33:09 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at NaiveBayesLearning.scala:32) -2016-05-25 10:33:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:09 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at NaiveBayesLearning.scala:27), which has no missing parents -2016-05-25 10:33:09 INFO MemoryStore:59 - ensureFreeSpace(3432) called with curMem=171580, maxMem=766075207 -2016-05-25 10:33:09 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-25 10:33:09 INFO MemoryStore:59 - ensureFreeSpace(2013) called with curMem=175012, maxMem=766075207 -2016-05-25 10:33:09 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2013.0 B, free 730.4 MB) -2016-05-25 10:33:09 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52659 (size: 2013.0 B, free: 730.6 MB) -2016-05-25 10:33:09 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at NaiveBayesLearning.scala:27) -2016-05-25 10:33:09 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 10:33:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:09 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 10:33:09 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 10:33:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:09 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 10:33:09 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 10:33:09 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 10:33:09 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 10:33:09 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 10:33:09 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-25 10:33:09 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-25 10:33:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 100 ms on localhost (1/2) -2016-05-25 10:33:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 124 ms on localhost (2/2) -2016-05-25 10:33:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 10:33:09 INFO DAGScheduler:59 - ResultStage 0 (foreach at NaiveBayesLearning.scala:32) finished in 0.138 s -2016-05-25 10:33:09 INFO DAGScheduler:59 - Job 0 finished: foreach at NaiveBayesLearning.scala:32, took 0.277124 s -2016-05-25 10:33:09 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:33:09 INFO DAGScheduler:59 - Registering RDD 5 (map at NaiveBayes.scala:383) -2016-05-25 10:33:09 INFO DAGScheduler:59 - Got job 1 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:33:09 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at NaiveBayes.scala:401) -2016-05-25 10:33:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1) -2016-05-25 10:33:09 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1) -2016-05-25 10:33:09 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:33:09 INFO MemoryStore:59 - ensureFreeSpace(5104) called with curMem=177025, maxMem=766075207 -2016-05-25 10:33:09 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 10:33:09 INFO MemoryStore:59 - ensureFreeSpace(2697) called with curMem=182129, maxMem=766075207 -2016-05-25 10:33:09 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:33:09 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52659 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:09 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383) -2016-05-25 10:33:09 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-25 10:33:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 10:33:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 10:33:09 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 10:33:09 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-25 10:33:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:09 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 10:33:09 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 10:33:09 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2254 bytes result sent to driver -2016-05-25 10:33:09 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2254 bytes result sent to driver -2016-05-25 10:33:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 92 ms on localhost (1/2) -2016-05-25 10:33:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 91 ms on localhost (2/2) -2016-05-25 10:33:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 10:33:09 INFO DAGScheduler:59 - ShuffleMapStage 1 (map at NaiveBayes.scala:383) finished in 0.094 s -2016-05-25 10:33:09 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 10:33:09 INFO DAGScheduler:59 - running: Set() -2016-05-25 10:33:09 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-05-25 10:33:09 INFO DAGScheduler:59 - failed: Set() -2016-05-25 10:33:09 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List() -2016-05-25 10:33:09 INFO DAGScheduler:59 - Submitting ResultStage 2 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 10:33:09 INFO MemoryStore:59 - ensureFreeSpace(2936) called with curMem=184826, maxMem=766075207 -2016-05-25 10:33:09 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-25 10:33:09 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=187762, maxMem=766075207 -2016-05-25 10:33:09 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.4 MB) -2016-05-25 10:33:09 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52659 (size: 1567.0 B, free: 730.6 MB) -2016-05-25 10:33:09 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383) -2016-05-25 10:33:09 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 10:33:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:33:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:33:09 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-25 10:33:09 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-25 10:33:09 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 10:33:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 10:33:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 10:33:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 10:33:09 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52659 in memory (size: 2013.0 B, free: 730.6 MB) -2016-05-25 10:33:09 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52659 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:09 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1161 bytes result sent to driver -2016-05-25 10:33:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 51 ms on localhost (1/2) -2016-05-25 10:33:09 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1645 bytes result sent to driver -2016-05-25 10:33:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 56 ms on localhost (2/2) -2016-05-25 10:33:09 INFO DAGScheduler:59 - ResultStage 2 (collect at NaiveBayes.scala:401) finished in 0.057 s -2016-05-25 10:33:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 10:33:09 INFO DAGScheduler:59 - Job 1 finished: collect at NaiveBayes.scala:401, took 0.199244 s -2016-05-25 10:33:09 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:40 -2016-05-25 10:33:09 INFO DAGScheduler:59 - Got job 2 (count at NaiveBayesLearning.scala:40) with 2 output partitions -2016-05-25 10:33:09 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at NaiveBayesLearning.scala:40) -2016-05-25 10:33:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:09 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:40), which has no missing parents -2016-05-25 10:33:09 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=176083, maxMem=766075207 -2016-05-25 10:33:09 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-25 10:33:09 INFO MemoryStore:59 - ensureFreeSpace(2670) called with curMem=180979, maxMem=766075207 -2016-05-25 10:33:09 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:33:09 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52659 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:09 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:40) -2016-05-25 10:33:09 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 10:33:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:09 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-25 10:33:09 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-25 10:33:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:09 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-25 10:33:09 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 2082 bytes result sent to driver -2016-05-25 10:33:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 19 ms on localhost (1/2) -2016-05-25 10:33:09 INFO DAGScheduler:59 - ResultStage 3 (count at NaiveBayesLearning.scala:40) finished in 0.021 s -2016-05-25 10:33:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 20 ms on localhost (2/2) -2016-05-25 10:33:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 10:33:09 INFO DAGScheduler:59 - Job 2 finished: count at NaiveBayesLearning.scala:40, took 0.034232 s -2016-05-25 10:33:09 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:40 -2016-05-25 10:33:09 INFO DAGScheduler:59 - Got job 3 (count at NaiveBayesLearning.scala:40) with 2 output partitions -2016-05-25 10:33:09 INFO DAGScheduler:59 - Final stage: ResultStage 4(count at NaiveBayesLearning.scala:40) -2016-05-25 10:33:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:09 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:33:09 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=183649, maxMem=766075207 -2016-05-25 10:33:09 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:33:09 INFO MemoryStore:59 - ensureFreeSpace(2116) called with curMem=187249, maxMem=766075207 -2016-05-25 10:33:09 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:33:09 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52659 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:09 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:33:09 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 10:33:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:09 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-25 10:33:09 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-25 10:33:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:09 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2082 bytes result sent to driver -2016-05-25 10:33:09 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2082 bytes result sent to driver -2016-05-25 10:33:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 22 ms on localhost (1/2) -2016-05-25 10:33:09 INFO DAGScheduler:59 - ResultStage 4 (count at NaiveBayesLearning.scala:40) finished in 0.024 s -2016-05-25 10:33:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 22 ms on localhost (2/2) -2016-05-25 10:33:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 10:33:09 INFO DAGScheduler:59 - Job 3 finished: count at NaiveBayesLearning.scala:40, took 0.036606 s -2016-05-25 10:33:09 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:44 -2016-05-25 10:33:09 INFO DAGScheduler:59 - Got job 4 (count at NaiveBayesLearning.scala:44) with 2 output partitions -2016-05-25 10:33:09 INFO DAGScheduler:59 - Final stage: ResultStage 5(count at NaiveBayesLearning.scala:44) -2016-05-25 10:33:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:09 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:33:09 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=189365, maxMem=766075207 -2016-05-25 10:33:09 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:33:09 INFO MemoryStore:59 - ensureFreeSpace(2112) called with curMem=192965, maxMem=766075207 -2016-05-25 10:33:09 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:33:09 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52659 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:09 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:33:09 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 10:33:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:09 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-05-25 10:33:09 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-05-25 10:33:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:09 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 2082 bytes result sent to driver -2016-05-25 10:33:09 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 2082 bytes result sent to driver -2016-05-25 10:33:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 18 ms on localhost (1/2) -2016-05-25 10:33:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 18 ms on localhost (2/2) -2016-05-25 10:33:09 INFO DAGScheduler:59 - ResultStage 5 (count at NaiveBayesLearning.scala:44) finished in 0.019 s -2016-05-25 10:33:09 INFO DAGScheduler:59 - Job 4 finished: count at NaiveBayesLearning.scala:44, took 0.032962 s -2016-05-25 10:33:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 10:33:09 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:45 -2016-05-25 10:33:09 INFO DAGScheduler:59 - Got job 5 (foreach at NaiveBayesLearning.scala:45) with 2 output partitions -2016-05-25 10:33:09 INFO DAGScheduler:59 - Final stage: ResultStage 6(foreach at NaiveBayesLearning.scala:45) -2016-05-25 10:33:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:09 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:33:09 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=195077, maxMem=766075207 -2016-05-25 10:33:09 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-25 10:33:09 INFO MemoryStore:59 - ensureFreeSpace(2142) called with curMem=198813, maxMem=766075207 -2016-05-25 10:33:09 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:33:09 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52659 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:09 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:33:09 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 10:33:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:09 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-05-25 10:33:09 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 13) -2016-05-25 10:33:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:09 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 13). 2044 bytes result sent to driver -2016-05-25 10:33:09 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 2044 bytes result sent to driver -2016-05-25 10:33:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 19 ms on localhost (1/2) -2016-05-25 10:33:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 13) in 19 ms on localhost (2/2) -2016-05-25 10:33:09 INFO DAGScheduler:59 - ResultStage 6 (foreach at NaiveBayesLearning.scala:45) finished in 0.020 s -2016-05-25 10:33:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 10:33:09 INFO DAGScheduler:59 - Job 5 finished: foreach at NaiveBayesLearning.scala:45, took 0.034764 s -2016-05-25 10:33:09 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:46 -2016-05-25 10:33:09 INFO DAGScheduler:59 - Got job 6 (count at NaiveBayesLearning.scala:46) with 2 output partitions -2016-05-25 10:33:09 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at NaiveBayesLearning.scala:46) -2016-05-25 10:33:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:09 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:33:09 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=200955, maxMem=766075207 -2016-05-25 10:33:09 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:33:09 INFO MemoryStore:59 - ensureFreeSpace(2116) called with curMem=204555, maxMem=766075207 -2016-05-25 10:33:09 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:33:09 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52659 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:09 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:33:10 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 10:33:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 15, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:10 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 14) -2016-05-25 10:33:10 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 15) -2016-05-25 10:33:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:10 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 14). 2082 bytes result sent to driver -2016-05-25 10:33:10 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 10:33:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 14) in 19 ms on localhost (1/2) -2016-05-25 10:33:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 15) in 19 ms on localhost (2/2) -2016-05-25 10:33:10 INFO DAGScheduler:59 - ResultStage 7 (count at NaiveBayesLearning.scala:46) finished in 0.022 s -2016-05-25 10:33:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 10:33:10 INFO DAGScheduler:59 - Job 6 finished: count at NaiveBayesLearning.scala:46, took 0.039061 s -2016-05-25 10:33:10 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:47 -2016-05-25 10:33:10 INFO DAGScheduler:59 - Got job 7 (foreach at NaiveBayesLearning.scala:47) with 2 output partitions -2016-05-25 10:33:10 INFO DAGScheduler:59 - Final stage: ResultStage 8(foreach at NaiveBayesLearning.scala:47) -2016-05-25 10:33:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:10 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:33:10 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=206671, maxMem=766075207 -2016-05-25 10:33:10 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-25 10:33:10 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=210407, maxMem=766075207 -2016-05-25 10:33:10 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:33:10 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52659 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:10 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:33:10 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 10:33:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 17, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:10 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 17) -2016-05-25 10:33:10 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 16) -2016-05-25 10:33:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:10 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 17). 2044 bytes result sent to driver -2016-05-25 10:33:10 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 16). 2044 bytes result sent to driver -2016-05-25 10:33:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 17) in 17 ms on localhost (1/2) -2016-05-25 10:33:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 16) in 19 ms on localhost (2/2) -2016-05-25 10:33:10 INFO DAGScheduler:59 - ResultStage 8 (foreach at NaiveBayesLearning.scala:47) finished in 0.020 s -2016-05-25 10:33:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 10:33:10 INFO DAGScheduler:59 - Job 7 finished: foreach at NaiveBayesLearning.scala:47, took 0.038603 s -2016-05-25 10:33:10 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:50 -2016-05-25 10:33:10 INFO DAGScheduler:59 - Got job 8 (take at NaiveBayesLearning.scala:50) with 1 output partitions -2016-05-25 10:33:10 INFO DAGScheduler:59 - Final stage: ResultStage 9(take at NaiveBayesLearning.scala:50) -2016-05-25 10:33:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:10 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-25 10:33:10 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=212553, maxMem=766075207 -2016-05-25 10:33:10 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 10:33:10 INFO MemoryStore:59 - ensureFreeSpace(2669) called with curMem=217377, maxMem=766075207 -2016-05-25 10:33:10 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:33:10 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:52659 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:10 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:10 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 9 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39) -2016-05-25 10:33:10 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 1 tasks -2016-05-25 10:33:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:10 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 18) -2016-05-25 10:33:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:10 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 18). 2040 bytes result sent to driver -2016-05-25 10:33:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 18) in 21 ms on localhost (1/1) -2016-05-25 10:33:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 10:33:10 INFO DAGScheduler:59 - ResultStage 9 (take at NaiveBayesLearning.scala:50) finished in 0.022 s -2016-05-25 10:33:10 INFO DAGScheduler:59 - Job 8 finished: take at NaiveBayesLearning.scala:50, took 0.038244 s -2016-05-25 10:33:10 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:50 -2016-05-25 10:33:10 INFO DAGScheduler:59 - Got job 9 (take at NaiveBayesLearning.scala:50) with 1 output partitions -2016-05-25 10:33:10 INFO DAGScheduler:59 - Final stage: ResultStage 10(take at NaiveBayesLearning.scala:50) -2016-05-25 10:33:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:10 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-25 10:33:10 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=220046, maxMem=766075207 -2016-05-25 10:33:10 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 10:33:10 INFO MemoryStore:59 - ensureFreeSpace(2669) called with curMem=224870, maxMem=766075207 -2016-05-25 10:33:10 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:33:10 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:52659 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:10 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:10 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39) -2016-05-25 10:33:10 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-25 10:33:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:10 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 10:33:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:10 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2206 bytes result sent to driver -2016-05-25 10:33:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 18 ms on localhost (1/1) -2016-05-25 10:33:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 10:33:10 INFO DAGScheduler:59 - ResultStage 10 (take at NaiveBayesLearning.scala:50) finished in 0.020 s -2016-05-25 10:33:10 INFO DAGScheduler:59 - Job 9 finished: take at NaiveBayesLearning.scala:50, took 0.037517 s -2016-05-25 10:33:10 INFO SparkContext:59 - Starting job: saveAsTextFile at NaiveBayes.scala:202 -2016-05-25 10:33:10 INFO DAGScheduler:59 - Got job 10 (saveAsTextFile at NaiveBayes.scala:202) with 1 output partitions -2016-05-25 10:33:10 INFO DAGScheduler:59 - Final stage: ResultStage 11(saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 10:33:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:10 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202), which has no missing parents -2016-05-25 10:33:10 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=227539, maxMem=766075207 -2016-05-25 10:33:10 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 124.0 KB, free 730.2 MB) -2016-05-25 10:33:10 INFO MemoryStore:59 - ensureFreeSpace(42584) called with curMem=354531, maxMem=766075207 -2016-05-25 10:33:10 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.2 MB) -2016-05-25 10:33:10 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:52659 (size: 41.6 KB, free: 730.5 MB) -2016-05-25 10:33:10 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:10 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 11 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 10:33:10 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 1 tasks -2016-05-25 10:33:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 20, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-25 10:33:10 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 20) -2016-05-25 10:33:10 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251033_0011_m_000000_20' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103310178/result/metadata/_temporary/0/task_201605251033_0011_m_000000 -2016-05-25 10:33:10 INFO SparkHadoopMapRedUtil:59 - attempt_201605251033_0011_m_000000_20: Committed -2016-05-25 10:33:10 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 20). 1864 bytes result sent to driver -2016-05-25 10:33:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 20) in 218 ms on localhost (1/1) -2016-05-25 10:33:10 INFO DAGScheduler:59 - ResultStage 11 (saveAsTextFile at NaiveBayes.scala:202) finished in 0.219 s -2016-05-25 10:33:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 10:33:10 INFO DAGScheduler:59 - Job 10 finished: saveAsTextFile at NaiveBayes.scala:202, took 0.306258 s -2016-05-25 10:33:11 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:52659 in memory (size: 2.1 KB, free: 730.5 MB) -2016-05-25 10:33:11 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 10:33:11 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52659 in memory (size: 2.1 KB, free: 730.5 MB) -2016-05-25 10:33:11 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 10:33:11 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52659 in memory (size: 2.6 KB, free: 730.5 MB) -2016-05-25 10:33:11 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 10:33:11 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52659 in memory (size: 1567.0 B, free: 730.5 MB) -2016-05-25 10:33:11 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:52659 in memory (size: 2.1 KB, free: 730.5 MB) -2016-05-25 10:33:11 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 10:33:11 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:52659 in memory (size: 2.1 KB, free: 730.5 MB) -2016-05-25 10:33:11 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 10:33:11 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:52659 in memory (size: 41.6 KB, free: 730.6 MB) -2016-05-25 10:33:11 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-25 10:33:11 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:52659 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:11 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 10:33:11 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:52659 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:11 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 10:33:11 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:52659 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:11 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 10:33:12 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:33:12 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:33:12 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:206 -2016-05-25 10:33:12 INFO DAGScheduler:59 - Got job 11 (parquet at NaiveBayes.scala:206) with 1 output partitions -2016-05-25 10:33:12 INFO DAGScheduler:59 - Final stage: ResultStage 12(parquet at NaiveBayes.scala:206) -2016-05-25 10:33:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:12 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205), which has no missing parents -2016-05-25 10:33:12 INFO MemoryStore:59 - ensureFreeSpace(66936) called with curMem=171580, maxMem=766075207 -2016-05-25 10:33:12 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 65.4 KB, free 730.4 MB) -2016-05-25 10:33:12 INFO MemoryStore:59 - ensureFreeSpace(23363) called with curMem=238516, maxMem=766075207 -2016-05-25 10:33:12 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 22.8 KB, free 730.3 MB) -2016-05-25 10:33:12 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:52659 (size: 22.8 KB, free: 730.6 MB) -2016-05-25 10:33:12 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:12 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205) -2016-05-25 10:33:12 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-25 10:33:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 21, localhost, PROCESS_LOCAL, 2506 bytes) -2016-05-25 10:33:12 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 21) -2016-05-25 10:33:12 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:33:12 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 10:33:12 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 10:33:12 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 10:33:12 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 10:33:12 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 10:33:12 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 10:33:12 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 10:33:12 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 10:33:12 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 231 -2016-05-25 10:33:12 INFO ColumnChunkPageWriteStore:151 - written 77B for [labels, bag, array] DOUBLE: 3 values, 37B raw, 36B comp, 1 pages, encodings: [RLE, PLAIN] -2016-05-25 10:33:12 INFO ColumnChunkPageWriteStore:151 - written 74B for [pi, bag, array] DOUBLE: 3 values, 16B raw, 33B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 2 entries, 16B raw, 2B comp} -2016-05-25 10:33:12 INFO ColumnChunkPageWriteStore:151 - written 83B for [theta, bag, array, bag, array] DOUBLE: 9 values, 23B raw, 42B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 6 entries, 48B raw, 6B comp} -2016-05-25 10:33:12 INFO ColumnChunkPageWriteStore:151 - written 86B for [modelType] BINARY: 1 values, 21B raw, 39B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 10:33:13 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251033_0012_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103310178/result/data/_temporary/0/task_201605251033_0012_m_000000 -2016-05-25 10:33:13 INFO SparkHadoopMapRedUtil:59 - attempt_201605251033_0012_m_000000_0: Committed -2016-05-25 10:33:13 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 21). 915 bytes result sent to driver -2016-05-25 10:33:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 21) in 472 ms on localhost (1/1) -2016-05-25 10:33:13 INFO DAGScheduler:59 - ResultStage 12 (parquet at NaiveBayes.scala:206) finished in 0.474 s -2016-05-25 10:33:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 10:33:13 INFO DAGScheduler:59 - Job 11 finished: parquet at NaiveBayes.scala:206, took 0.512891 s -2016-05-25 10:33:13 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:33:13 INFO DefaultWriterContainer:59 - Job job_201605251033_0000 committed. -2016-05-25 10:33:13 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103310178/result/data on driver -2016-05-25 10:33:13 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103310178/result/data on driver -2016-05-25 10:33:13 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=261879, maxMem=766075207 -2016-05-25 10:33:13 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-25 10:33:13 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=492583, maxMem=766075207 -2016-05-25 10:33:13 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-25 10:33:13 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:52659 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 10:33:13 INFO SparkContext:59 - Created broadcast 14 from textFile at modelSaveLoad.scala:129 -2016-05-25 10:33:13 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:33:13 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 10:33:13 INFO DAGScheduler:59 - Got job 12 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 10:33:13 INFO DAGScheduler:59 - Final stage: ResultStage 13(first at modelSaveLoad.scala:129) -2016-05-25 10:33:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:13 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 10:33:13 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=512371, maxMem=766075207 -2016-05-25 10:33:13 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-25 10:33:13 INFO MemoryStore:59 - ensureFreeSpace(1889) called with curMem=515587, maxMem=766075207 -2016-05-25 10:33:13 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1889.0 B, free 730.1 MB) -2016-05-25 10:33:13 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:52659 (size: 1889.0 B, free: 730.5 MB) -2016-05-25 10:33:13 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:13 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 13 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129) -2016-05-25 10:33:13 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 1 tasks -2016-05-25 10:33:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 22, localhost, PROCESS_LOCAL, 2239 bytes) -2016-05-25 10:33:13 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 22) -2016-05-25 10:33:13 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103310178/result/metadata/part-00000:0+56 -2016-05-25 10:33:13 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 22). 2159 bytes result sent to driver -2016-05-25 10:33:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 22) in 15 ms on localhost (1/1) -2016-05-25 10:33:13 INFO DAGScheduler:59 - ResultStage 13 (first at modelSaveLoad.scala:129) finished in 0.015 s -2016-05-25 10:33:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 10:33:13 INFO DAGScheduler:59 - Job 12 finished: first at modelSaveLoad.scala:129, took 0.026103 s -2016-05-25 10:33:13 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103310178/result/data on driver -2016-05-25 10:33:13 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:213 -2016-05-25 10:33:13 INFO DAGScheduler:59 - Got job 13 (parquet at NaiveBayes.scala:213) with 4 output partitions -2016-05-25 10:33:13 INFO DAGScheduler:59 - Final stage: ResultStage 14(parquet at NaiveBayes.scala:213) -2016-05-25 10:33:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:13 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213), which has no missing parents -2016-05-25 10:33:13 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=517476, maxMem=766075207 -2016-05-25 10:33:13 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-25 10:33:13 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=580476, maxMem=766075207 -2016-05-25 10:33:13 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.0 MB) -2016-05-25 10:33:13 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:52659 (size: 20.6 KB, free: 730.5 MB) -2016-05-25 10:33:13 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:13 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 14 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213) -2016-05-25 10:33:13 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 4 tasks -2016-05-25 10:33:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 23, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:33:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 24, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:33:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 25, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:33:13 INFO TaskSetManager:59 - Starting task 3.0 in stage 14.0 (TID 26, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-25 10:33:13 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 23) -2016-05-25 10:33:13 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 25) -2016-05-25 10:33:13 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 24) -2016-05-25 10:33:13 INFO Executor:59 - Running task 3.0 in stage 14.0 (TID 26) -2016-05-25 10:33:13 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:33:13 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 23). 936 bytes result sent to driver -2016-05-25 10:33:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 23) in 81 ms on localhost (1/4) -2016-05-25 10:33:13 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:33:13 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 25). 936 bytes result sent to driver -2016-05-25 10:33:13 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:33:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 25) in 84 ms on localhost (2/4) -2016-05-25 10:33:13 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 24). 936 bytes result sent to driver -2016-05-25 10:33:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 24) in 87 ms on localhost (3/4) -2016-05-25 10:33:13 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:33:13 INFO Executor:59 - Finished task 3.0 in stage 14.0 (TID 26). 1876 bytes result sent to driver -2016-05-25 10:33:13 INFO TaskSetManager:59 - Finished task 3.0 in stage 14.0 (TID 26) in 108 ms on localhost (4/4) -2016-05-25 10:33:13 INFO DAGScheduler:59 - ResultStage 14 (parquet at NaiveBayes.scala:213) finished in 0.112 s -2016-05-25 10:33:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 10:33:13 INFO DAGScheduler:59 - Job 13 finished: parquet at NaiveBayes.scala:213, took 0.144334 s -2016-05-25 10:33:13 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=601549, maxMem=766075207 -2016-05-25 10:33:13 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 86.4 KB, free 729.9 MB) -2016-05-25 10:33:13 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=690061, maxMem=766075207 -2016-05-25 10:33:13 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-25 10:33:13 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:52659 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 10:33:13 INFO SparkContext:59 - Created broadcast 17 from take at NaiveBayes.scala:216 -2016-05-25 10:33:14 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 10:33:14 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103310178/result/data/part-r-00000-41d106ba-270e-4f64-bfde-de11e97fc5d0.gz.parquet -2016-05-25 10:33:14 INFO SparkContext:59 - Starting job: take at NaiveBayes.scala:216 -2016-05-25 10:33:14 INFO DAGScheduler:59 - Got job 14 (take at NaiveBayes.scala:216) with 1 output partitions -2016-05-25 10:33:14 INFO DAGScheduler:59 - Final stage: ResultStage 15(take at NaiveBayes.scala:216) -2016-05-25 10:33:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:14 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216), which has no missing parents -2016-05-25 10:33:14 INFO MemoryStore:59 - ensureFreeSpace(4280) called with curMem=709849, maxMem=766075207 -2016-05-25 10:33:14 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 4.2 KB, free 729.9 MB) -2016-05-25 10:33:14 INFO MemoryStore:59 - ensureFreeSpace(2466) called with curMem=714129, maxMem=766075207 -2016-05-25 10:33:14 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.4 KB, free 729.9 MB) -2016-05-25 10:33:14 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:52659 (size: 2.4 KB, free: 730.5 MB) -2016-05-25 10:33:14 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:14 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 15 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216) -2016-05-25 10:33:14 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 1 tasks -2016-05-25 10:33:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 27, localhost, PROCESS_LOCAL, 2303 bytes) -2016-05-25 10:33:14 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 27) -2016-05-25 10:33:14 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103310178/result/data/part-r-00000-41d106ba-270e-4f64-bfde-de11e97fc5d0.gz.parquet start: 0 end: 1477 length: 1477 hosts: []} -2016-05-25 10:33:14 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 10:33:14 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group labels (LIST) { - repeated group bag { - optional double array; - } - } - optional group pi (LIST) { - repeated group bag { - optional double array; - } - } - optional group theta (LIST) { - repeated group bag { - optional group array (LIST) { - repeated group bag { - optional double array; - } - } - } - } - optional binary modelType (UTF8); -} - -Catalyst form: -StructType(StructField(labels,ArrayType(DoubleType,true),true), StructField(pi,ArrayType(DoubleType,true),true), StructField(theta,ArrayType(ArrayType(DoubleType,true),true),true), StructField(modelType,StringType,true)) - -2016-05-25 10:33:14 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 10:33:14 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 10:33:14 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 10:33:14 INFO InternalParquetRecordReader:151 - block read in memory in 13 ms. row count = 1 -2016-05-25 10:33:14 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 27). 2841 bytes result sent to driver -2016-05-25 10:33:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 27) in 121 ms on localhost (1/1) -2016-05-25 10:33:14 INFO DAGScheduler:59 - ResultStage 15 (take at NaiveBayes.scala:216) finished in 0.122 s -2016-05-25 10:33:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-25 10:33:14 INFO DAGScheduler:59 - Job 14 finished: take at NaiveBayes.scala:216, took 0.142200 s -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 10:33:14 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 10:33:14 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 10:33:14 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 10:33:14 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 10:33:14 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 10:33:14 INFO BlockManager:59 - BlockManager stopped -2016-05-25 10:33:14 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 10:33:14 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 10:33:14 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 10:33:14 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 10:33:14 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-ee857718-04ab-482e-885e-ce491671d2a8 -2016-05-25 10:33:14 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 10:33:24 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 10:33:24 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 10:33:25 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 10:33:25 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 10:33:25 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 10:33:25 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 10:33:25 INFO Remoting:74 - Starting remoting -2016-05-25 10:33:26 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:52682] -2016-05-25 10:33:26 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52682. -2016-05-25 10:33:26 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 10:33:26 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 10:33:26 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-aa992189-89c8-4818-812e-9bdb7d3c8652 -2016-05-25 10:33:26 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 10:33:26 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-fa6409fa-4dd3-4b2c-9a3c-292ebeb9938e\httpd-1b0fee7c-b7ba-4e2e-b033-97a56b8b31bd -2016-05-25 10:33:26 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 10:33:26 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:33:26 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52683 -2016-05-25 10:33:26 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52683. -2016-05-25 10:33:26 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 10:33:26 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:33:26 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 10:33:26 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 10:33:26 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 10:33:26 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 10:33:26 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 10:33:27 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52702. -2016-05-25 10:33:27 INFO NettyBlockTransferService:59 - Server created on 52702 -2016-05-25 10:33:27 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 10:33:27 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52702 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52702) -2016-05-25 10:33:27 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 10:33:28 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 10:33:28 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 10:33:28 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 10:33:28 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 10:33:28 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52702 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 10:33:28 INFO SparkContext:59 - Created broadcast 0 from textFile at NaiveBayesLearning.scala:26 -2016-05-25 10:33:29 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 10:33:30 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:33:30 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:32 -2016-05-25 10:33:30 INFO DAGScheduler:59 - Got job 0 (foreach at NaiveBayesLearning.scala:32) with 2 output partitions -2016-05-25 10:33:30 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at NaiveBayesLearning.scala:32) -2016-05-25 10:33:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at NaiveBayesLearning.scala:27), which has no missing parents -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(3432) called with curMem=171580, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(2013) called with curMem=175012, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2013.0 B, free 730.4 MB) -2016-05-25 10:33:30 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52702 (size: 2013.0 B, free: 730.6 MB) -2016-05-25 10:33:30 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at NaiveBayesLearning.scala:27) -2016-05-25 10:33:30 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 10:33:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:30 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 10:33:30 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 10:33:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:30 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 10:33:30 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 10:33:30 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 10:33:30 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 10:33:30 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 10:33:30 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-25 10:33:30 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-25 10:33:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 113 ms on localhost (1/2) -2016-05-25 10:33:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 145 ms on localhost (2/2) -2016-05-25 10:33:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 10:33:30 INFO DAGScheduler:59 - ResultStage 0 (foreach at NaiveBayesLearning.scala:32) finished in 0.164 s -2016-05-25 10:33:30 INFO DAGScheduler:59 - Job 0 finished: foreach at NaiveBayesLearning.scala:32, took 0.284120 s -2016-05-25 10:33:30 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:33:30 INFO DAGScheduler:59 - Registering RDD 5 (map at NaiveBayes.scala:383) -2016-05-25 10:33:30 INFO DAGScheduler:59 - Got job 1 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:33:30 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at NaiveBayes.scala:401) -2016-05-25 10:33:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1) -2016-05-25 10:33:30 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1) -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(5104) called with curMem=177025, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(2695) called with curMem=182129, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:33:30 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52702 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:30 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383) -2016-05-25 10:33:30 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-25 10:33:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 10:33:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 10:33:30 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 10:33:30 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-25 10:33:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:30 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 10:33:30 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 10:33:30 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2254 bytes result sent to driver -2016-05-25 10:33:30 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2254 bytes result sent to driver -2016-05-25 10:33:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 94 ms on localhost (1/2) -2016-05-25 10:33:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 98 ms on localhost (2/2) -2016-05-25 10:33:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 10:33:30 INFO DAGScheduler:59 - ShuffleMapStage 1 (map at NaiveBayes.scala:383) finished in 0.099 s -2016-05-25 10:33:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 10:33:30 INFO DAGScheduler:59 - running: Set() -2016-05-25 10:33:30 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-05-25 10:33:30 INFO DAGScheduler:59 - failed: Set() -2016-05-25 10:33:30 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List() -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting ResultStage 2 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(2936) called with curMem=184824, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=187760, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.4 MB) -2016-05-25 10:33:30 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52702 (size: 1567.0 B, free: 730.6 MB) -2016-05-25 10:33:30 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383) -2016-05-25 10:33:30 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 10:33:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:33:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:33:30 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-25 10:33:30 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-25 10:33:30 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52702 in memory (size: 2013.0 B, free: 730.6 MB) -2016-05-25 10:33:30 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 10:33:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 10:33:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 10:33:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 10:33:30 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1161 bytes result sent to driver -2016-05-25 10:33:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 32 ms on localhost (1/2) -2016-05-25 10:33:30 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1645 bytes result sent to driver -2016-05-25 10:33:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 39 ms on localhost (2/2) -2016-05-25 10:33:30 INFO DAGScheduler:59 - ResultStage 2 (collect at NaiveBayes.scala:401) finished in 0.039 s -2016-05-25 10:33:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 10:33:30 INFO DAGScheduler:59 - Job 1 finished: collect at NaiveBayes.scala:401, took 0.215238 s -2016-05-25 10:33:30 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:40 -2016-05-25 10:33:30 INFO DAGScheduler:59 - Got job 2 (count at NaiveBayesLearning.scala:40) with 2 output partitions -2016-05-25 10:33:30 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at NaiveBayesLearning.scala:40) -2016-05-25 10:33:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:40), which has no missing parents -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=183882, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(2673) called with curMem=188778, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:33:30 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52702 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:30 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:40) -2016-05-25 10:33:30 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 10:33:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:30 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-25 10:33:30 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-25 10:33:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:30 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-25 10:33:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 17 ms on localhost (1/2) -2016-05-25 10:33:30 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 2082 bytes result sent to driver -2016-05-25 10:33:30 INFO DAGScheduler:59 - ResultStage 3 (count at NaiveBayesLearning.scala:40) finished in 0.021 s -2016-05-25 10:33:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 20 ms on localhost (2/2) -2016-05-25 10:33:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 10:33:30 INFO DAGScheduler:59 - Job 2 finished: count at NaiveBayesLearning.scala:40, took 0.035130 s -2016-05-25 10:33:30 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:40 -2016-05-25 10:33:30 INFO DAGScheduler:59 - Got job 3 (count at NaiveBayesLearning.scala:40) with 2 output partitions -2016-05-25 10:33:30 INFO DAGScheduler:59 - Final stage: ResultStage 4(count at NaiveBayesLearning.scala:40) -2016-05-25 10:33:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=191451, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(2121) called with curMem=195051, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:33:30 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52702 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:30 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:33:30 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 10:33:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:30 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-25 10:33:30 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-25 10:33:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:30 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2082 bytes result sent to driver -2016-05-25 10:33:30 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2082 bytes result sent to driver -2016-05-25 10:33:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 17 ms on localhost (1/2) -2016-05-25 10:33:30 INFO DAGScheduler:59 - ResultStage 4 (count at NaiveBayesLearning.scala:40) finished in 0.023 s -2016-05-25 10:33:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 23 ms on localhost (2/2) -2016-05-25 10:33:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 10:33:30 INFO DAGScheduler:59 - Job 3 finished: count at NaiveBayesLearning.scala:40, took 0.036284 s -2016-05-25 10:33:30 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:44 -2016-05-25 10:33:30 INFO DAGScheduler:59 - Got job 4 (count at NaiveBayesLearning.scala:44) with 2 output partitions -2016-05-25 10:33:30 INFO DAGScheduler:59 - Final stage: ResultStage 5(count at NaiveBayesLearning.scala:44) -2016-05-25 10:33:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=197172, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(2116) called with curMem=200772, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:33:30 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52702 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:30 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:33:30 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 10:33:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:30 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-05-25 10:33:30 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-05-25 10:33:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:30 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 2082 bytes result sent to driver -2016-05-25 10:33:30 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 2082 bytes result sent to driver -2016-05-25 10:33:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 18 ms on localhost (1/2) -2016-05-25 10:33:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 17 ms on localhost (2/2) -2016-05-25 10:33:30 INFO DAGScheduler:59 - ResultStage 5 (count at NaiveBayesLearning.scala:44) finished in 0.019 s -2016-05-25 10:33:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 10:33:30 INFO DAGScheduler:59 - Job 4 finished: count at NaiveBayesLearning.scala:44, took 0.032869 s -2016-05-25 10:33:30 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:45 -2016-05-25 10:33:30 INFO DAGScheduler:59 - Got job 5 (foreach at NaiveBayesLearning.scala:45) with 2 output partitions -2016-05-25 10:33:30 INFO DAGScheduler:59 - Final stage: ResultStage 6(foreach at NaiveBayesLearning.scala:45) -2016-05-25 10:33:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=202888, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=206624, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:33:30 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52702 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:30 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:33:30 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 10:33:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:30 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-05-25 10:33:30 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 13) -2016-05-25 10:33:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:30 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 2044 bytes result sent to driver -2016-05-25 10:33:30 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 13). 2044 bytes result sent to driver -2016-05-25 10:33:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 19 ms on localhost (1/2) -2016-05-25 10:33:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 13) in 18 ms on localhost (2/2) -2016-05-25 10:33:30 INFO DAGScheduler:59 - ResultStage 6 (foreach at NaiveBayesLearning.scala:45) finished in 0.020 s -2016-05-25 10:33:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 10:33:30 INFO DAGScheduler:59 - Job 5 finished: foreach at NaiveBayesLearning.scala:45, took 0.033623 s -2016-05-25 10:33:30 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:46 -2016-05-25 10:33:30 INFO DAGScheduler:59 - Got job 6 (count at NaiveBayesLearning.scala:46) with 2 output partitions -2016-05-25 10:33:30 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at NaiveBayesLearning.scala:46) -2016-05-25 10:33:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=208770, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(2121) called with curMem=212370, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:33:30 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52702 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:30 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:33:30 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 10:33:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 15, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:30 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 15) -2016-05-25 10:33:30 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 14) -2016-05-25 10:33:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:30 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 10:33:30 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 14). 2082 bytes result sent to driver -2016-05-25 10:33:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 15) in 17 ms on localhost (1/2) -2016-05-25 10:33:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 14) in 20 ms on localhost (2/2) -2016-05-25 10:33:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 10:33:30 INFO DAGScheduler:59 - ResultStage 7 (count at NaiveBayesLearning.scala:46) finished in 0.021 s -2016-05-25 10:33:30 INFO DAGScheduler:59 - Job 6 finished: count at NaiveBayesLearning.scala:46, took 0.040330 s -2016-05-25 10:33:30 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:47 -2016-05-25 10:33:30 INFO DAGScheduler:59 - Got job 7 (foreach at NaiveBayesLearning.scala:47) with 2 output partitions -2016-05-25 10:33:30 INFO DAGScheduler:59 - Final stage: ResultStage 8(foreach at NaiveBayesLearning.scala:47) -2016-05-25 10:33:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=214491, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(2151) called with curMem=218227, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:33:30 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52702 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:30 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:33:30 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 10:33:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 17, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:30 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 16) -2016-05-25 10:33:30 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 17) -2016-05-25 10:33:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:30 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 16). 2044 bytes result sent to driver -2016-05-25 10:33:30 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 17). 2044 bytes result sent to driver -2016-05-25 10:33:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 16) in 20 ms on localhost (1/2) -2016-05-25 10:33:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 17) in 20 ms on localhost (2/2) -2016-05-25 10:33:30 INFO DAGScheduler:59 - ResultStage 8 (foreach at NaiveBayesLearning.scala:47) finished in 0.022 s -2016-05-25 10:33:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 10:33:30 INFO DAGScheduler:59 - Job 7 finished: foreach at NaiveBayesLearning.scala:47, took 0.042123 s -2016-05-25 10:33:30 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:50 -2016-05-25 10:33:30 INFO DAGScheduler:59 - Got job 8 (take at NaiveBayesLearning.scala:50) with 1 output partitions -2016-05-25 10:33:30 INFO DAGScheduler:59 - Final stage: ResultStage 9(take at NaiveBayesLearning.scala:50) -2016-05-25 10:33:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=220378, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(2672) called with curMem=225202, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:33:30 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:52702 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:30 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 9 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39) -2016-05-25 10:33:30 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 1 tasks -2016-05-25 10:33:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:30 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 18) -2016-05-25 10:33:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:30 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 18). 2040 bytes result sent to driver -2016-05-25 10:33:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 18) in 18 ms on localhost (1/1) -2016-05-25 10:33:30 INFO DAGScheduler:59 - ResultStage 9 (take at NaiveBayesLearning.scala:50) finished in 0.018 s -2016-05-25 10:33:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 10:33:30 INFO DAGScheduler:59 - Job 8 finished: take at NaiveBayesLearning.scala:50, took 0.036722 s -2016-05-25 10:33:30 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:50 -2016-05-25 10:33:30 INFO DAGScheduler:59 - Got job 9 (take at NaiveBayesLearning.scala:50) with 1 output partitions -2016-05-25 10:33:30 INFO DAGScheduler:59 - Final stage: ResultStage 10(take at NaiveBayesLearning.scala:50) -2016-05-25 10:33:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:30 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=227874, maxMem=766075207 -2016-05-25 10:33:30 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 10:33:30 INFO MemoryStore:59 - ensureFreeSpace(2672) called with curMem=232698, maxMem=766075207 -2016-05-25 10:33:31 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:33:31 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:52702 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:31 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39) -2016-05-25 10:33:31 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-25 10:33:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:31 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 10:33:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:31 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2206 bytes result sent to driver -2016-05-25 10:33:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 17 ms on localhost (1/1) -2016-05-25 10:33:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 10:33:31 INFO DAGScheduler:59 - ResultStage 10 (take at NaiveBayesLearning.scala:50) finished in 0.018 s -2016-05-25 10:33:31 INFO DAGScheduler:59 - Job 9 finished: take at NaiveBayesLearning.scala:50, took 0.034899 s -2016-05-25 10:33:31 INFO SparkContext:59 - Starting job: saveAsTextFile at NaiveBayes.scala:202 -2016-05-25 10:33:31 INFO DAGScheduler:59 - Got job 10 (saveAsTextFile at NaiveBayes.scala:202) with 1 output partitions -2016-05-25 10:33:31 INFO DAGScheduler:59 - Final stage: ResultStage 11(saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 10:33:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:31 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202), which has no missing parents -2016-05-25 10:33:31 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=235370, maxMem=766075207 -2016-05-25 10:33:31 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 124.0 KB, free 730.2 MB) -2016-05-25 10:33:31 INFO MemoryStore:59 - ensureFreeSpace(42584) called with curMem=362362, maxMem=766075207 -2016-05-25 10:33:31 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.2 MB) -2016-05-25 10:33:31 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:52702 (size: 41.6 KB, free: 730.5 MB) -2016-05-25 10:33:31 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 11 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 10:33:31 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 1 tasks -2016-05-25 10:33:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 20, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-25 10:33:31 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 20) -2016-05-25 10:33:31 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251033_0011_m_000000_20' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103331026/result/metadata/_temporary/0/task_201605251033_0011_m_000000 -2016-05-25 10:33:31 INFO SparkHadoopMapRedUtil:59 - attempt_201605251033_0011_m_000000_20: Committed -2016-05-25 10:33:31 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 20). 1864 bytes result sent to driver -2016-05-25 10:33:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 20) in 211 ms on localhost (1/1) -2016-05-25 10:33:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 10:33:31 INFO DAGScheduler:59 - ResultStage 11 (saveAsTextFile at NaiveBayes.scala:202) finished in 0.212 s -2016-05-25 10:33:31 INFO DAGScheduler:59 - Job 10 finished: saveAsTextFile at NaiveBayes.scala:202, took 0.297420 s -2016-05-25 10:33:31 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:52702 in memory (size: 41.6 KB, free: 730.6 MB) -2016-05-25 10:33:31 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-25 10:33:31 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:52702 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:31 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 10:33:31 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:52702 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:31 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 10:33:31 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:52702 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:31 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 10:33:31 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:52702 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:31 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 10:33:31 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:52702 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:31 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 10:33:31 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:52702 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:31 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 10:33:31 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52702 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:31 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 10:33:31 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52702 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:31 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 10:33:31 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52702 in memory (size: 1567.0 B, free: 730.6 MB) -2016-05-25 10:33:31 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52702 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:31 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 10:33:33 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:33:33 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:33:33 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:206 -2016-05-25 10:33:33 INFO DAGScheduler:59 - Got job 11 (parquet at NaiveBayes.scala:206) with 1 output partitions -2016-05-25 10:33:33 INFO DAGScheduler:59 - Final stage: ResultStage 12(parquet at NaiveBayes.scala:206) -2016-05-25 10:33:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:33 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205), which has no missing parents -2016-05-25 10:33:33 INFO MemoryStore:59 - ensureFreeSpace(66936) called with curMem=171580, maxMem=766075207 -2016-05-25 10:33:33 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 65.4 KB, free 730.4 MB) -2016-05-25 10:33:33 INFO MemoryStore:59 - ensureFreeSpace(23361) called with curMem=238516, maxMem=766075207 -2016-05-25 10:33:33 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 22.8 KB, free 730.3 MB) -2016-05-25 10:33:33 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:52702 (size: 22.8 KB, free: 730.6 MB) -2016-05-25 10:33:33 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205) -2016-05-25 10:33:33 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-25 10:33:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 21, localhost, PROCESS_LOCAL, 2506 bytes) -2016-05-25 10:33:33 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 21) -2016-05-25 10:33:33 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:33:33 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 10:33:33 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 10:33:33 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 10:33:33 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 10:33:33 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 10:33:33 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 10:33:33 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 10:33:33 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 10:33:33 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 231 -2016-05-25 10:33:33 INFO ColumnChunkPageWriteStore:151 - written 77B for [labels, bag, array] DOUBLE: 3 values, 37B raw, 36B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 10:33:33 INFO ColumnChunkPageWriteStore:151 - written 74B for [pi, bag, array] DOUBLE: 3 values, 16B raw, 33B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 2 entries, 16B raw, 2B comp} -2016-05-25 10:33:33 INFO ColumnChunkPageWriteStore:151 - written 83B for [theta, bag, array, bag, array] DOUBLE: 9 values, 23B raw, 42B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 6 entries, 48B raw, 6B comp} -2016-05-25 10:33:33 INFO ColumnChunkPageWriteStore:151 - written 86B for [modelType] BINARY: 1 values, 21B raw, 39B comp, 1 pages, encodings: [PLAIN, BIT_PACKED, RLE] -2016-05-25 10:33:33 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251033_0012_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103331026/result/data/_temporary/0/task_201605251033_0012_m_000000 -2016-05-25 10:33:33 INFO SparkHadoopMapRedUtil:59 - attempt_201605251033_0012_m_000000_0: Committed -2016-05-25 10:33:33 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 21). 915 bytes result sent to driver -2016-05-25 10:33:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 21) in 486 ms on localhost (1/1) -2016-05-25 10:33:33 INFO DAGScheduler:59 - ResultStage 12 (parquet at NaiveBayes.scala:206) finished in 0.486 s -2016-05-25 10:33:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 10:33:33 INFO DAGScheduler:59 - Job 11 finished: parquet at NaiveBayes.scala:206, took 0.527213 s -2016-05-25 10:33:34 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:33:34 INFO DefaultWriterContainer:59 - Job job_201605251033_0000 committed. -2016-05-25 10:33:34 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103331026/result/data on driver -2016-05-25 10:33:34 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103331026/result/data on driver -2016-05-25 10:33:34 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=261877, maxMem=766075207 -2016-05-25 10:33:34 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-25 10:33:34 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=492581, maxMem=766075207 -2016-05-25 10:33:34 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-25 10:33:34 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:52702 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 10:33:34 INFO SparkContext:59 - Created broadcast 14 from textFile at modelSaveLoad.scala:129 -2016-05-25 10:33:34 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:33:34 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 10:33:34 INFO DAGScheduler:59 - Got job 12 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 10:33:34 INFO DAGScheduler:59 - Final stage: ResultStage 13(first at modelSaveLoad.scala:129) -2016-05-25 10:33:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:34 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 10:33:34 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=512369, maxMem=766075207 -2016-05-25 10:33:34 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-25 10:33:34 INFO MemoryStore:59 - ensureFreeSpace(1889) called with curMem=515585, maxMem=766075207 -2016-05-25 10:33:34 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1889.0 B, free 730.1 MB) -2016-05-25 10:33:34 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:52702 (size: 1889.0 B, free: 730.5 MB) -2016-05-25 10:33:34 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:34 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 13 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129) -2016-05-25 10:33:34 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 1 tasks -2016-05-25 10:33:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 22, localhost, PROCESS_LOCAL, 2239 bytes) -2016-05-25 10:33:34 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 22) -2016-05-25 10:33:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103331026/result/metadata/part-00000:0+56 -2016-05-25 10:33:34 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 22). 2159 bytes result sent to driver -2016-05-25 10:33:34 INFO DAGScheduler:59 - ResultStage 13 (first at modelSaveLoad.scala:129) finished in 0.016 s -2016-05-25 10:33:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 22) in 16 ms on localhost (1/1) -2016-05-25 10:33:34 INFO DAGScheduler:59 - Job 12 finished: first at modelSaveLoad.scala:129, took 0.029215 s -2016-05-25 10:33:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 10:33:34 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103331026/result/data on driver -2016-05-25 10:33:34 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:213 -2016-05-25 10:33:34 INFO DAGScheduler:59 - Got job 13 (parquet at NaiveBayes.scala:213) with 4 output partitions -2016-05-25 10:33:34 INFO DAGScheduler:59 - Final stage: ResultStage 14(parquet at NaiveBayes.scala:213) -2016-05-25 10:33:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:34 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213), which has no missing parents -2016-05-25 10:33:34 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=517474, maxMem=766075207 -2016-05-25 10:33:34 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-25 10:33:34 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=580474, maxMem=766075207 -2016-05-25 10:33:34 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.0 MB) -2016-05-25 10:33:34 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:52702 (size: 20.6 KB, free: 730.5 MB) -2016-05-25 10:33:34 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 14 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213) -2016-05-25 10:33:34 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 4 tasks -2016-05-25 10:33:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 23, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:33:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 24, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:33:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 25, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:33:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 14.0 (TID 26, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-25 10:33:34 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 23) -2016-05-25 10:33:34 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 24) -2016-05-25 10:33:34 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 25) -2016-05-25 10:33:34 INFO Executor:59 - Running task 3.0 in stage 14.0 (TID 26) -2016-05-25 10:33:34 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:33:34 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 23). 936 bytes result sent to driver -2016-05-25 10:33:34 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:33:34 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 25). 936 bytes result sent to driver -2016-05-25 10:33:34 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:33:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 23) in 91 ms on localhost (1/4) -2016-05-25 10:33:34 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 24). 936 bytes result sent to driver -2016-05-25 10:33:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 25) in 89 ms on localhost (2/4) -2016-05-25 10:33:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 24) in 92 ms on localhost (3/4) -2016-05-25 10:33:34 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:33:34 INFO Executor:59 - Finished task 3.0 in stage 14.0 (TID 26). 1876 bytes result sent to driver -2016-05-25 10:33:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 14.0 (TID 26) in 115 ms on localhost (4/4) -2016-05-25 10:33:34 INFO DAGScheduler:59 - ResultStage 14 (parquet at NaiveBayes.scala:213) finished in 0.120 s -2016-05-25 10:33:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 10:33:34 INFO DAGScheduler:59 - Job 13 finished: parquet at NaiveBayes.scala:213, took 0.152856 s -2016-05-25 10:33:34 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=601547, maxMem=766075207 -2016-05-25 10:33:34 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 86.4 KB, free 729.9 MB) -2016-05-25 10:33:34 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=690059, maxMem=766075207 -2016-05-25 10:33:34 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-25 10:33:34 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:52702 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 10:33:34 INFO SparkContext:59 - Created broadcast 17 from take at NaiveBayes.scala:216 -2016-05-25 10:33:34 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 10:33:34 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103331026/result/data/part-r-00000-4be1b18a-5d4e-42b9-8b71-fb8325f0af2d.gz.parquet -2016-05-25 10:33:34 INFO SparkContext:59 - Starting job: take at NaiveBayes.scala:216 -2016-05-25 10:33:34 INFO DAGScheduler:59 - Got job 14 (take at NaiveBayes.scala:216) with 1 output partitions -2016-05-25 10:33:34 INFO DAGScheduler:59 - Final stage: ResultStage 15(take at NaiveBayes.scala:216) -2016-05-25 10:33:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:34 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216), which has no missing parents -2016-05-25 10:33:35 INFO MemoryStore:59 - ensureFreeSpace(4280) called with curMem=709847, maxMem=766075207 -2016-05-25 10:33:35 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 4.2 KB, free 729.9 MB) -2016-05-25 10:33:35 INFO MemoryStore:59 - ensureFreeSpace(2466) called with curMem=714127, maxMem=766075207 -2016-05-25 10:33:35 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.4 KB, free 729.9 MB) -2016-05-25 10:33:35 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:52702 (size: 2.4 KB, free: 730.5 MB) -2016-05-25 10:33:35 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:35 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 15 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216) -2016-05-25 10:33:35 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 1 tasks -2016-05-25 10:33:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 27, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-25 10:33:35 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 27) -2016-05-25 10:33:35 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103331026/result/data/part-r-00000-4be1b18a-5d4e-42b9-8b71-fb8325f0af2d.gz.parquet start: 0 end: 1477 length: 1477 hosts: []} -2016-05-25 10:33:35 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 10:33:35 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group labels (LIST) { - repeated group bag { - optional double array; - } - } - optional group pi (LIST) { - repeated group bag { - optional double array; - } - } - optional group theta (LIST) { - repeated group bag { - optional group array (LIST) { - repeated group bag { - optional double array; - } - } - } - } - optional binary modelType (UTF8); -} - -Catalyst form: -StructType(StructField(labels,ArrayType(DoubleType,true),true), StructField(pi,ArrayType(DoubleType,true),true), StructField(theta,ArrayType(ArrayType(DoubleType,true),true),true), StructField(modelType,StringType,true)) - -2016-05-25 10:33:35 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 10:33:35 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 10:33:35 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 10:33:35 INFO InternalParquetRecordReader:151 - block read in memory in 13 ms. row count = 1 -2016-05-25 10:33:35 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 27). 2841 bytes result sent to driver -2016-05-25 10:33:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 27) in 116 ms on localhost (1/1) -2016-05-25 10:33:35 INFO DAGScheduler:59 - ResultStage 15 (take at NaiveBayes.scala:216) finished in 0.116 s -2016-05-25 10:33:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-25 10:33:35 INFO DAGScheduler:59 - Job 14 finished: take at NaiveBayes.scala:216, took 0.142651 s -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 10:33:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 10:33:35 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 10:33:35 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 10:33:35 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 10:33:35 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 10:33:35 INFO BlockManager:59 - BlockManager stopped -2016-05-25 10:33:35 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 10:33:35 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 10:33:35 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 10:33:35 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 10:33:35 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-fa6409fa-4dd3-4b2c-9a3c-292ebeb9938e -2016-05-25 10:33:47 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 10:33:48 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 10:33:48 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 10:33:48 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 10:33:48 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 10:33:49 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 10:33:49 INFO Remoting:74 - Starting remoting -2016-05-25 10:33:49 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:52724] -2016-05-25 10:33:49 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52724. -2016-05-25 10:33:49 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 10:33:49 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 10:33:49 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-0a189bfc-cfa6-4ad0-a743-0ec98ec7d46e -2016-05-25 10:33:49 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 10:33:49 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-a7b08616-a0dc-48d2-a46a-de2f6f41fa18\httpd-6c6405e6-cb9d-4e62-90e9-44a7710937ba -2016-05-25 10:33:49 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 10:33:49 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:33:49 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52725 -2016-05-25 10:33:49 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52725. -2016-05-25 10:33:49 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 10:33:49 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:33:49 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 10:33:49 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 10:33:49 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 10:33:49 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 10:33:49 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 10:33:50 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52744. -2016-05-25 10:33:50 INFO NettyBlockTransferService:59 - Server created on 52744 -2016-05-25 10:33:50 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 10:33:50 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52744 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52744) -2016-05-25 10:33:50 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 10:33:50 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 10:33:50 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 10:33:51 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 10:33:51 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 10:33:51 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52744 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 10:33:51 INFO SparkContext:59 - Created broadcast 0 from textFile at NaiveBayesLearning.scala:26 -2016-05-25 10:33:52 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 10:33:53 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:33:53 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:32 -2016-05-25 10:33:53 INFO DAGScheduler:59 - Got job 0 (foreach at NaiveBayesLearning.scala:32) with 2 output partitions -2016-05-25 10:33:53 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at NaiveBayesLearning.scala:32) -2016-05-25 10:33:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:53 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at NaiveBayesLearning.scala:27), which has no missing parents -2016-05-25 10:33:53 INFO MemoryStore:59 - ensureFreeSpace(3432) called with curMem=171580, maxMem=766075207 -2016-05-25 10:33:53 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-25 10:33:53 INFO MemoryStore:59 - ensureFreeSpace(2013) called with curMem=175012, maxMem=766075207 -2016-05-25 10:33:53 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2013.0 B, free 730.4 MB) -2016-05-25 10:33:53 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52744 (size: 2013.0 B, free: 730.6 MB) -2016-05-25 10:33:53 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at NaiveBayesLearning.scala:27) -2016-05-25 10:33:53 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 10:33:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:53 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 10:33:53 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 10:33:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:53 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 10:33:53 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 10:33:53 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 10:33:53 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 10:33:53 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 10:33:53 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-25 10:33:53 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-25 10:33:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 124 ms on localhost (1/2) -2016-05-25 10:33:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 151 ms on localhost (2/2) -2016-05-25 10:33:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 10:33:53 INFO DAGScheduler:59 - ResultStage 0 (foreach at NaiveBayesLearning.scala:32) finished in 0.170 s -2016-05-25 10:33:53 INFO DAGScheduler:59 - Job 0 finished: foreach at NaiveBayesLearning.scala:32, took 0.309366 s -2016-05-25 10:33:53 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:33:53 INFO DAGScheduler:59 - Registering RDD 5 (map at NaiveBayes.scala:383) -2016-05-25 10:33:53 INFO DAGScheduler:59 - Got job 1 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:33:53 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at NaiveBayes.scala:401) -2016-05-25 10:33:53 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1) -2016-05-25 10:33:53 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1) -2016-05-25 10:33:53 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:33:53 INFO MemoryStore:59 - ensureFreeSpace(5104) called with curMem=177025, maxMem=766075207 -2016-05-25 10:33:53 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 10:33:53 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=182129, maxMem=766075207 -2016-05-25 10:33:53 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:33:53 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52744 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:53 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383) -2016-05-25 10:33:53 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-25 10:33:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 10:33:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 10:33:53 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-25 10:33:53 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 10:33:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:53 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 10:33:53 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 10:33:53 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2254 bytes result sent to driver -2016-05-25 10:33:53 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2254 bytes result sent to driver -2016-05-25 10:33:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 87 ms on localhost (1/2) -2016-05-25 10:33:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 90 ms on localhost (2/2) -2016-05-25 10:33:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 10:33:53 INFO DAGScheduler:59 - ShuffleMapStage 1 (map at NaiveBayes.scala:383) finished in 0.091 s -2016-05-25 10:33:53 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 10:33:53 INFO DAGScheduler:59 - running: Set() -2016-05-25 10:33:53 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-05-25 10:33:53 INFO DAGScheduler:59 - failed: Set() -2016-05-25 10:33:53 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List() -2016-05-25 10:33:53 INFO DAGScheduler:59 - Submitting ResultStage 2 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 10:33:53 INFO MemoryStore:59 - ensureFreeSpace(2936) called with curMem=184823, maxMem=766075207 -2016-05-25 10:33:53 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-25 10:33:53 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=187759, maxMem=766075207 -2016-05-25 10:33:53 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.4 MB) -2016-05-25 10:33:53 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52744 (size: 1567.0 B, free: 730.6 MB) -2016-05-25 10:33:53 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383) -2016-05-25 10:33:53 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 10:33:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:33:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:33:53 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-25 10:33:53 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-25 10:33:53 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 10:33:53 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 10:33:53 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 10:33:53 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-25 10:33:53 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52744 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:53 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1161 bytes result sent to driver -2016-05-25 10:33:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 61 ms on localhost (1/2) -2016-05-25 10:33:53 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52744 in memory (size: 2013.0 B, free: 730.6 MB) -2016-05-25 10:33:53 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1645 bytes result sent to driver -2016-05-25 10:33:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 67 ms on localhost (2/2) -2016-05-25 10:33:53 INFO DAGScheduler:59 - ResultStage 2 (collect at NaiveBayes.scala:401) finished in 0.068 s -2016-05-25 10:33:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 10:33:53 INFO DAGScheduler:59 - Job 1 finished: collect at NaiveBayes.scala:401, took 0.201523 s -2016-05-25 10:33:53 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:40 -2016-05-25 10:33:53 INFO DAGScheduler:59 - Got job 2 (count at NaiveBayesLearning.scala:40) with 2 output partitions -2016-05-25 10:33:53 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at NaiveBayesLearning.scala:40) -2016-05-25 10:33:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:53 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:40), which has no missing parents -2016-05-25 10:33:53 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=176083, maxMem=766075207 -2016-05-25 10:33:53 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-25 10:33:53 INFO MemoryStore:59 - ensureFreeSpace(2626) called with curMem=180979, maxMem=766075207 -2016-05-25 10:33:53 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:33:53 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52744 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:53 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:40) -2016-05-25 10:33:53 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 10:33:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:53 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-25 10:33:53 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-25 10:33:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:53 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-25 10:33:53 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 2082 bytes result sent to driver -2016-05-25 10:33:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 18 ms on localhost (1/2) -2016-05-25 10:33:53 INFO DAGScheduler:59 - ResultStage 3 (count at NaiveBayesLearning.scala:40) finished in 0.019 s -2016-05-25 10:33:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 18 ms on localhost (2/2) -2016-05-25 10:33:53 INFO DAGScheduler:59 - Job 2 finished: count at NaiveBayesLearning.scala:40, took 0.033043 s -2016-05-25 10:33:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 10:33:53 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:40 -2016-05-25 10:33:53 INFO DAGScheduler:59 - Got job 3 (count at NaiveBayesLearning.scala:40) with 2 output partitions -2016-05-25 10:33:53 INFO DAGScheduler:59 - Final stage: ResultStage 4(count at NaiveBayesLearning.scala:40) -2016-05-25 10:33:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:53 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:33:53 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=183605, maxMem=766075207 -2016-05-25 10:33:53 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:33:53 INFO MemoryStore:59 - ensureFreeSpace(2119) called with curMem=187205, maxMem=766075207 -2016-05-25 10:33:53 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:33:53 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52744 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:53 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:33:53 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 10:33:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:53 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-25 10:33:53 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-25 10:33:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:53 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2082 bytes result sent to driver -2016-05-25 10:33:53 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2082 bytes result sent to driver -2016-05-25 10:33:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 17 ms on localhost (1/2) -2016-05-25 10:33:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 19 ms on localhost (2/2) -2016-05-25 10:33:53 INFO DAGScheduler:59 - ResultStage 4 (count at NaiveBayesLearning.scala:40) finished in 0.019 s -2016-05-25 10:33:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 10:33:53 INFO DAGScheduler:59 - Job 3 finished: count at NaiveBayesLearning.scala:40, took 0.035563 s -2016-05-25 10:33:53 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:44 -2016-05-25 10:33:53 INFO DAGScheduler:59 - Got job 4 (count at NaiveBayesLearning.scala:44) with 2 output partitions -2016-05-25 10:33:53 INFO DAGScheduler:59 - Final stage: ResultStage 5(count at NaiveBayesLearning.scala:44) -2016-05-25 10:33:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:53 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:33:53 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=189324, maxMem=766075207 -2016-05-25 10:33:53 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:33:53 INFO MemoryStore:59 - ensureFreeSpace(2115) called with curMem=192924, maxMem=766075207 -2016-05-25 10:33:53 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:33:53 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52744 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:53 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:33:53 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 10:33:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:53 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-05-25 10:33:53 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-05-25 10:33:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:53 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 2082 bytes result sent to driver -2016-05-25 10:33:53 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 2082 bytes result sent to driver -2016-05-25 10:33:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 18 ms on localhost (1/2) -2016-05-25 10:33:53 INFO DAGScheduler:59 - ResultStage 5 (count at NaiveBayesLearning.scala:44) finished in 0.021 s -2016-05-25 10:33:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 21 ms on localhost (2/2) -2016-05-25 10:33:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 10:33:53 INFO DAGScheduler:59 - Job 4 finished: count at NaiveBayesLearning.scala:44, took 0.035392 s -2016-05-25 10:33:53 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:45 -2016-05-25 10:33:53 INFO DAGScheduler:59 - Got job 5 (foreach at NaiveBayesLearning.scala:45) with 2 output partitions -2016-05-25 10:33:53 INFO DAGScheduler:59 - Final stage: ResultStage 6(foreach at NaiveBayesLearning.scala:45) -2016-05-25 10:33:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:53 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:33:53 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=195039, maxMem=766075207 -2016-05-25 10:33:53 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-25 10:33:53 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=198775, maxMem=766075207 -2016-05-25 10:33:53 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:33:53 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52744 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:53 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:33:53 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 10:33:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:53 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-05-25 10:33:53 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 13) -2016-05-25 10:33:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:54 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 2044 bytes result sent to driver -2016-05-25 10:33:54 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 13). 2044 bytes result sent to driver -2016-05-25 10:33:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 19 ms on localhost (1/2) -2016-05-25 10:33:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 13) in 19 ms on localhost (2/2) -2016-05-25 10:33:54 INFO DAGScheduler:59 - ResultStage 6 (foreach at NaiveBayesLearning.scala:45) finished in 0.021 s -2016-05-25 10:33:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 10:33:54 INFO DAGScheduler:59 - Job 5 finished: foreach at NaiveBayesLearning.scala:45, took 0.035982 s -2016-05-25 10:33:54 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:46 -2016-05-25 10:33:54 INFO DAGScheduler:59 - Got job 6 (count at NaiveBayesLearning.scala:46) with 2 output partitions -2016-05-25 10:33:54 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at NaiveBayesLearning.scala:46) -2016-05-25 10:33:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:54 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:33:54 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=200920, maxMem=766075207 -2016-05-25 10:33:54 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:33:54 INFO MemoryStore:59 - ensureFreeSpace(2119) called with curMem=204520, maxMem=766075207 -2016-05-25 10:33:54 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:33:54 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52744 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:54 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:33:54 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 10:33:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 15, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:54 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 14) -2016-05-25 10:33:54 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 15) -2016-05-25 10:33:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:54 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 14). 2082 bytes result sent to driver -2016-05-25 10:33:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 14) in 18 ms on localhost (1/2) -2016-05-25 10:33:54 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 10:33:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 15) in 22 ms on localhost (2/2) -2016-05-25 10:33:54 INFO DAGScheduler:59 - ResultStage 7 (count at NaiveBayesLearning.scala:46) finished in 0.024 s -2016-05-25 10:33:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 10:33:54 INFO DAGScheduler:59 - Job 6 finished: count at NaiveBayesLearning.scala:46, took 0.041323 s -2016-05-25 10:33:54 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:47 -2016-05-25 10:33:54 INFO DAGScheduler:59 - Got job 7 (foreach at NaiveBayesLearning.scala:47) with 2 output partitions -2016-05-25 10:33:54 INFO DAGScheduler:59 - Final stage: ResultStage 8(foreach at NaiveBayesLearning.scala:47) -2016-05-25 10:33:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:54 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:33:54 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=206639, maxMem=766075207 -2016-05-25 10:33:54 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-25 10:33:54 INFO MemoryStore:59 - ensureFreeSpace(2149) called with curMem=210375, maxMem=766075207 -2016-05-25 10:33:54 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:33:54 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52744 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:54 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:33:54 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 10:33:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 17, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:54 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 16) -2016-05-25 10:33:54 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 17) -2016-05-25 10:33:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:54 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 17). 2044 bytes result sent to driver -2016-05-25 10:33:54 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 16). 2044 bytes result sent to driver -2016-05-25 10:33:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 17) in 17 ms on localhost (1/2) -2016-05-25 10:33:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 16) in 20 ms on localhost (2/2) -2016-05-25 10:33:54 INFO DAGScheduler:59 - ResultStage 8 (foreach at NaiveBayesLearning.scala:47) finished in 0.020 s -2016-05-25 10:33:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 10:33:54 INFO DAGScheduler:59 - Job 7 finished: foreach at NaiveBayesLearning.scala:47, took 0.039516 s -2016-05-25 10:33:54 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:50 -2016-05-25 10:33:54 INFO DAGScheduler:59 - Got job 8 (take at NaiveBayesLearning.scala:50) with 1 output partitions -2016-05-25 10:33:54 INFO DAGScheduler:59 - Final stage: ResultStage 9(take at NaiveBayesLearning.scala:50) -2016-05-25 10:33:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:54 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-25 10:33:54 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=212524, maxMem=766075207 -2016-05-25 10:33:54 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 10:33:54 INFO MemoryStore:59 - ensureFreeSpace(2625) called with curMem=217348, maxMem=766075207 -2016-05-25 10:33:54 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:33:54 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:52744 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:54 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:54 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 9 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39) -2016-05-25 10:33:54 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 1 tasks -2016-05-25 10:33:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:54 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 18) -2016-05-25 10:33:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:33:54 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 18). 2040 bytes result sent to driver -2016-05-25 10:33:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 18) in 18 ms on localhost (1/1) -2016-05-25 10:33:54 INFO DAGScheduler:59 - ResultStage 9 (take at NaiveBayesLearning.scala:50) finished in 0.018 s -2016-05-25 10:33:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 10:33:54 INFO DAGScheduler:59 - Job 8 finished: take at NaiveBayesLearning.scala:50, took 0.035135 s -2016-05-25 10:33:54 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:50 -2016-05-25 10:33:54 INFO DAGScheduler:59 - Got job 9 (take at NaiveBayesLearning.scala:50) with 1 output partitions -2016-05-25 10:33:54 INFO DAGScheduler:59 - Final stage: ResultStage 10(take at NaiveBayesLearning.scala:50) -2016-05-25 10:33:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:54 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-25 10:33:54 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=219973, maxMem=766075207 -2016-05-25 10:33:54 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 10:33:54 INFO MemoryStore:59 - ensureFreeSpace(2625) called with curMem=224797, maxMem=766075207 -2016-05-25 10:33:54 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:33:54 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:52744 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:54 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:54 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39) -2016-05-25 10:33:54 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-25 10:33:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:33:54 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 10:33:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:33:54 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2040 bytes result sent to driver -2016-05-25 10:33:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 16 ms on localhost (1/1) -2016-05-25 10:33:54 INFO DAGScheduler:59 - ResultStage 10 (take at NaiveBayesLearning.scala:50) finished in 0.016 s -2016-05-25 10:33:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 10:33:54 INFO DAGScheduler:59 - Job 9 finished: take at NaiveBayesLearning.scala:50, took 0.033670 s -2016-05-25 10:33:54 INFO SparkContext:59 - Starting job: saveAsTextFile at NaiveBayes.scala:202 -2016-05-25 10:33:54 INFO DAGScheduler:59 - Got job 10 (saveAsTextFile at NaiveBayes.scala:202) with 1 output partitions -2016-05-25 10:33:54 INFO DAGScheduler:59 - Final stage: ResultStage 11(saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 10:33:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:54 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202), which has no missing parents -2016-05-25 10:33:54 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=227422, maxMem=766075207 -2016-05-25 10:33:54 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 124.0 KB, free 730.2 MB) -2016-05-25 10:33:54 INFO MemoryStore:59 - ensureFreeSpace(42587) called with curMem=354414, maxMem=766075207 -2016-05-25 10:33:54 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.2 MB) -2016-05-25 10:33:54 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:52744 (size: 41.6 KB, free: 730.5 MB) -2016-05-25 10:33:54 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:54 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 11 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 10:33:54 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 1 tasks -2016-05-25 10:33:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 20, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-25 10:33:54 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 20) -2016-05-25 10:33:55 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251033_0011_m_000000_20' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103354206/result/metadata/_temporary/0/task_201605251033_0011_m_000000 -2016-05-25 10:33:55 INFO SparkHadoopMapRedUtil:59 - attempt_201605251033_0011_m_000000_20: Committed -2016-05-25 10:33:55 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 20). 1864 bytes result sent to driver -2016-05-25 10:33:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 20) in 250 ms on localhost (1/1) -2016-05-25 10:33:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 10:33:55 INFO DAGScheduler:59 - ResultStage 11 (saveAsTextFile at NaiveBayes.scala:202) finished in 0.250 s -2016-05-25 10:33:55 INFO DAGScheduler:59 - Job 10 finished: saveAsTextFile at NaiveBayes.scala:202, took 0.338587 s -2016-05-25 10:33:55 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:52744 in memory (size: 41.6 KB, free: 730.6 MB) -2016-05-25 10:33:55 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-25 10:33:55 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:52744 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:55 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 10:33:55 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:52744 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:55 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 10:33:55 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:52744 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:55 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 10:33:55 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:52744 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:55 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 10:33:55 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:52744 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:55 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 10:33:55 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:52744 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:55 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 10:33:55 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52744 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:33:55 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 10:33:55 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52744 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:33:55 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 10:33:55 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52744 in memory (size: 1567.0 B, free: 730.6 MB) -2016-05-25 10:33:56 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:33:56 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:33:56 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:206 -2016-05-25 10:33:56 INFO DAGScheduler:59 - Got job 11 (parquet at NaiveBayes.scala:206) with 1 output partitions -2016-05-25 10:33:56 INFO DAGScheduler:59 - Final stage: ResultStage 12(parquet at NaiveBayes.scala:206) -2016-05-25 10:33:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:56 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205), which has no missing parents -2016-05-25 10:33:56 INFO MemoryStore:59 - ensureFreeSpace(66936) called with curMem=171580, maxMem=766075207 -2016-05-25 10:33:56 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 65.4 KB, free 730.4 MB) -2016-05-25 10:33:56 INFO MemoryStore:59 - ensureFreeSpace(23363) called with curMem=238516, maxMem=766075207 -2016-05-25 10:33:56 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 22.8 KB, free 730.3 MB) -2016-05-25 10:33:56 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:52744 (size: 22.8 KB, free: 730.6 MB) -2016-05-25 10:33:56 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:56 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205) -2016-05-25 10:33:56 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-25 10:33:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 21, localhost, PROCESS_LOCAL, 2506 bytes) -2016-05-25 10:33:56 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 21) -2016-05-25 10:33:56 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:33:56 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 10:33:56 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 10:33:56 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 10:33:56 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 10:33:56 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 10:33:56 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 10:33:56 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 10:33:56 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 10:33:57 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 191 -2016-05-25 10:33:57 INFO ColumnChunkPageWriteStore:151 - written 77B for [labels, bag, array] DOUBLE: 3 values, 37B raw, 36B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 10:33:57 INFO ColumnChunkPageWriteStore:151 - written 72B for [pi, bag, array] DOUBLE: 3 values, 15B raw, 31B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 1 entries, 8B raw, 1B comp} -2016-05-25 10:33:57 INFO ColumnChunkPageWriteStore:151 - written 79B for [theta, bag, array, bag, array] DOUBLE: 9 values, 19B raw, 38B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 2 entries, 16B raw, 2B comp} -2016-05-25 10:33:57 INFO ColumnChunkPageWriteStore:151 - written 86B for [modelType] BINARY: 1 values, 21B raw, 39B comp, 1 pages, encodings: [PLAIN, BIT_PACKED, RLE] -2016-05-25 10:33:57 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251033_0012_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103354206/result/data/_temporary/0/task_201605251033_0012_m_000000 -2016-05-25 10:33:57 INFO SparkHadoopMapRedUtil:59 - attempt_201605251033_0012_m_000000_0: Committed -2016-05-25 10:33:57 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 21). 915 bytes result sent to driver -2016-05-25 10:33:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 21) in 490 ms on localhost (1/1) -2016-05-25 10:33:57 INFO DAGScheduler:59 - ResultStage 12 (parquet at NaiveBayes.scala:206) finished in 0.490 s -2016-05-25 10:33:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 10:33:57 INFO DAGScheduler:59 - Job 11 finished: parquet at NaiveBayes.scala:206, took 0.534319 s -2016-05-25 10:33:57 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:33:57 INFO DefaultWriterContainer:59 - Job job_201605251033_0000 committed. -2016-05-25 10:33:57 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103354206/result/data on driver -2016-05-25 10:33:57 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103354206/result/data on driver -2016-05-25 10:33:57 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=261879, maxMem=766075207 -2016-05-25 10:33:57 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-25 10:33:57 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=492583, maxMem=766075207 -2016-05-25 10:33:57 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-25 10:33:57 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:52744 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 10:33:57 INFO SparkContext:59 - Created broadcast 14 from textFile at modelSaveLoad.scala:129 -2016-05-25 10:33:57 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:33:57 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 10:33:57 INFO DAGScheduler:59 - Got job 12 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 10:33:57 INFO DAGScheduler:59 - Final stage: ResultStage 13(first at modelSaveLoad.scala:129) -2016-05-25 10:33:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:57 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 10:33:57 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=512371, maxMem=766075207 -2016-05-25 10:33:57 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-25 10:33:57 INFO MemoryStore:59 - ensureFreeSpace(1889) called with curMem=515587, maxMem=766075207 -2016-05-25 10:33:57 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1889.0 B, free 730.1 MB) -2016-05-25 10:33:57 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:52744 (size: 1889.0 B, free: 730.5 MB) -2016-05-25 10:33:57 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:57 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 13 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129) -2016-05-25 10:33:57 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 1 tasks -2016-05-25 10:33:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 22, localhost, PROCESS_LOCAL, 2239 bytes) -2016-05-25 10:33:57 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 22) -2016-05-25 10:33:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103354206/result/metadata/part-00000:0+56 -2016-05-25 10:33:57 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 22). 2159 bytes result sent to driver -2016-05-25 10:33:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 22) in 15 ms on localhost (1/1) -2016-05-25 10:33:57 INFO DAGScheduler:59 - ResultStage 13 (first at modelSaveLoad.scala:129) finished in 0.015 s -2016-05-25 10:33:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 10:33:57 INFO DAGScheduler:59 - Job 12 finished: first at modelSaveLoad.scala:129, took 0.025784 s -2016-05-25 10:33:57 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103354206/result/data on driver -2016-05-25 10:33:57 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:213 -2016-05-25 10:33:57 INFO DAGScheduler:59 - Got job 13 (parquet at NaiveBayes.scala:213) with 4 output partitions -2016-05-25 10:33:57 INFO DAGScheduler:59 - Final stage: ResultStage 14(parquet at NaiveBayes.scala:213) -2016-05-25 10:33:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:57 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213), which has no missing parents -2016-05-25 10:33:57 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=517476, maxMem=766075207 -2016-05-25 10:33:57 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-25 10:33:57 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=580476, maxMem=766075207 -2016-05-25 10:33:57 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.0 MB) -2016-05-25 10:33:57 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:52744 (size: 20.6 KB, free: 730.5 MB) -2016-05-25 10:33:57 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:57 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 14 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213) -2016-05-25 10:33:57 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 4 tasks -2016-05-25 10:33:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 23, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:33:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 24, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:33:58 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 25, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:33:58 INFO TaskSetManager:59 - Starting task 3.0 in stage 14.0 (TID 26, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-25 10:33:58 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 24) -2016-05-25 10:33:58 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 23) -2016-05-25 10:33:58 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 25) -2016-05-25 10:33:58 INFO Executor:59 - Running task 3.0 in stage 14.0 (TID 26) -2016-05-25 10:33:58 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:33:58 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 24). 936 bytes result sent to driver -2016-05-25 10:33:58 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:33:58 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:33:58 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 23). 936 bytes result sent to driver -2016-05-25 10:33:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 24) in 82 ms on localhost (1/4) -2016-05-25 10:33:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 23) in 84 ms on localhost (2/4) -2016-05-25 10:33:58 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:33:58 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 25). 936 bytes result sent to driver -2016-05-25 10:33:58 INFO Executor:59 - Finished task 3.0 in stage 14.0 (TID 26). 1876 bytes result sent to driver -2016-05-25 10:33:58 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 25) in 106 ms on localhost (3/4) -2016-05-25 10:33:58 INFO TaskSetManager:59 - Finished task 3.0 in stage 14.0 (TID 26) in 106 ms on localhost (4/4) -2016-05-25 10:33:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 10:33:58 INFO DAGScheduler:59 - ResultStage 14 (parquet at NaiveBayes.scala:213) finished in 0.110 s -2016-05-25 10:33:58 INFO DAGScheduler:59 - Job 13 finished: parquet at NaiveBayes.scala:213, took 0.147047 s -2016-05-25 10:33:58 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=601549, maxMem=766075207 -2016-05-25 10:33:58 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 86.4 KB, free 729.9 MB) -2016-05-25 10:33:58 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=690061, maxMem=766075207 -2016-05-25 10:33:58 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-25 10:33:58 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:52744 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 10:33:58 INFO SparkContext:59 - Created broadcast 17 from take at NaiveBayes.scala:216 -2016-05-25 10:33:58 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 10:33:58 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103354206/result/data/part-r-00000-6068bcc0-3576-4216-9ead-2035e35cf3aa.gz.parquet -2016-05-25 10:33:58 INFO SparkContext:59 - Starting job: take at NaiveBayes.scala:216 -2016-05-25 10:33:58 INFO DAGScheduler:59 - Got job 14 (take at NaiveBayes.scala:216) with 1 output partitions -2016-05-25 10:33:58 INFO DAGScheduler:59 - Final stage: ResultStage 15(take at NaiveBayes.scala:216) -2016-05-25 10:33:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:33:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:33:58 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216), which has no missing parents -2016-05-25 10:33:58 INFO MemoryStore:59 - ensureFreeSpace(4280) called with curMem=709849, maxMem=766075207 -2016-05-25 10:33:58 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 4.2 KB, free 729.9 MB) -2016-05-25 10:33:58 INFO MemoryStore:59 - ensureFreeSpace(2466) called with curMem=714129, maxMem=766075207 -2016-05-25 10:33:58 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.4 KB, free 729.9 MB) -2016-05-25 10:33:58 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:52744 (size: 2.4 KB, free: 730.5 MB) -2016-05-25 10:33:58 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:33:58 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 15 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216) -2016-05-25 10:33:58 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 1 tasks -2016-05-25 10:33:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 27, localhost, PROCESS_LOCAL, 2303 bytes) -2016-05-25 10:33:58 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 27) -2016-05-25 10:33:58 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103354206/result/data/part-r-00000-6068bcc0-3576-4216-9ead-2035e35cf3aa.gz.parquet start: 0 end: 1426 length: 1426 hosts: []} -2016-05-25 10:33:58 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 10:33:58 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group labels (LIST) { - repeated group bag { - optional double array; - } - } - optional group pi (LIST) { - repeated group bag { - optional double array; - } - } - optional group theta (LIST) { - repeated group bag { - optional group array (LIST) { - repeated group bag { - optional double array; - } - } - } - } - optional binary modelType (UTF8); -} - -Catalyst form: -StructType(StructField(labels,ArrayType(DoubleType,true),true), StructField(pi,ArrayType(DoubleType,true),true), StructField(theta,ArrayType(ArrayType(DoubleType,true),true),true), StructField(modelType,StringType,true)) - -2016-05-25 10:33:58 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 10:33:58 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 10:33:58 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 10:33:58 INFO InternalParquetRecordReader:151 - block read in memory in 19 ms. row count = 1 -2016-05-25 10:33:58 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 27). 2841 bytes result sent to driver -2016-05-25 10:33:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 27) in 187 ms on localhost (1/1) -2016-05-25 10:33:58 INFO DAGScheduler:59 - ResultStage 15 (take at NaiveBayes.scala:216) finished in 0.188 s -2016-05-25 10:33:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-25 10:33:58 INFO DAGScheduler:59 - Job 14 finished: take at NaiveBayes.scala:216, took 0.210708 s -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 10:33:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 10:33:58 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 10:33:58 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 10:33:58 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 10:33:58 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 10:33:58 INFO BlockManager:59 - BlockManager stopped -2016-05-25 10:33:58 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 10:33:58 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 10:33:59 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 10:33:59 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 10:33:59 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-25 10:33:59 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 10:33:59 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-a7b08616-a0dc-48d2-a46a-de2f6f41fa18 -2016-05-25 10:34:18 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 10:34:19 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 10:34:20 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 10:34:20 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 10:34:20 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 10:34:20 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 10:34:20 INFO Remoting:74 - Starting remoting -2016-05-25 10:34:21 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:52766] -2016-05-25 10:34:21 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52766. -2016-05-25 10:34:21 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 10:34:21 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 10:34:21 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-c6697761-6570-4b35-98ca-82af4a5ad4c7 -2016-05-25 10:34:21 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 10:34:21 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-42c80a3b-c927-4288-9164-48b7d7afeca4\httpd-f6c0290a-6264-4057-ba77-a77d29a4aac1 -2016-05-25 10:34:21 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 10:34:21 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:34:21 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52767 -2016-05-25 10:34:21 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52767. -2016-05-25 10:34:21 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 10:34:21 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:34:21 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 10:34:21 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 10:34:21 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 10:34:21 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 10:34:21 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 10:34:22 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52786. -2016-05-25 10:34:22 INFO NettyBlockTransferService:59 - Server created on 52786 -2016-05-25 10:34:22 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 10:34:22 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52786 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52786) -2016-05-25 10:34:22 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 10:34:22 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 10:34:22 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 10:34:22 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 10:34:22 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 10:34:22 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52786 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 10:34:22 INFO SparkContext:59 - Created broadcast 0 from textFile at NaiveBayesLearning.scala:26 -2016-05-25 10:34:24 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 10:34:25 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:34:25 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:32 -2016-05-25 10:34:25 INFO DAGScheduler:59 - Got job 0 (foreach at NaiveBayesLearning.scala:32) with 2 output partitions -2016-05-25 10:34:25 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at NaiveBayesLearning.scala:32) -2016-05-25 10:34:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:34:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:34:25 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at NaiveBayesLearning.scala:27), which has no missing parents -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(3432) called with curMem=171580, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(2013) called with curMem=175012, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2013.0 B, free 730.4 MB) -2016-05-25 10:34:26 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52786 (size: 2013.0 B, free: 730.6 MB) -2016-05-25 10:34:26 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at NaiveBayesLearning.scala:27) -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 10:34:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:34:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:34:26 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 10:34:26 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 10:34:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:34:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:34:26 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 10:34:26 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 10:34:26 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 10:34:26 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 10:34:26 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 10:34:26 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-25 10:34:26 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-25 10:34:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 134 ms on localhost (1/2) -2016-05-25 10:34:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 114 ms on localhost (2/2) -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 10:34:26 INFO DAGScheduler:59 - ResultStage 0 (foreach at NaiveBayesLearning.scala:32) finished in 0.153 s -2016-05-25 10:34:26 INFO DAGScheduler:59 - Job 0 finished: foreach at NaiveBayesLearning.scala:32, took 0.276683 s -2016-05-25 10:34:26 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:34:26 INFO DAGScheduler:59 - Registering RDD 5 (map at NaiveBayes.scala:383) -2016-05-25 10:34:26 INFO DAGScheduler:59 - Got job 1 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:34:26 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at NaiveBayes.scala:401) -2016-05-25 10:34:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1) -2016-05-25 10:34:26 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1) -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(5104) called with curMem=177025, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(2692) called with curMem=182129, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:34:26 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52786 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:34:26 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383) -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-25 10:34:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 10:34:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 10:34:26 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 10:34:26 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-25 10:34:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:34:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:34:26 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 10:34:26 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 10:34:26 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2254 bytes result sent to driver -2016-05-25 10:34:26 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2254 bytes result sent to driver -2016-05-25 10:34:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 135 ms on localhost (1/2) -2016-05-25 10:34:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 134 ms on localhost (2/2) -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 10:34:26 INFO DAGScheduler:59 - ShuffleMapStage 1 (map at NaiveBayes.scala:383) finished in 0.137 s -2016-05-25 10:34:26 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 10:34:26 INFO DAGScheduler:59 - running: Set() -2016-05-25 10:34:26 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-05-25 10:34:26 INFO DAGScheduler:59 - failed: Set() -2016-05-25 10:34:26 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List() -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting ResultStage 2 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(2936) called with curMem=184821, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=187757, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.4 MB) -2016-05-25 10:34:26 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52786 (size: 1567.0 B, free: 730.6 MB) -2016-05-25 10:34:26 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383) -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 10:34:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:34:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:34:26 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-25 10:34:26 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-25 10:34:26 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 10:34:26 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 10:34:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 10:34:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 10:34:26 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1161 bytes result sent to driver -2016-05-25 10:34:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 29 ms on localhost (1/2) -2016-05-25 10:34:26 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1645 bytes result sent to driver -2016-05-25 10:34:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 34 ms on localhost (2/2) -2016-05-25 10:34:26 INFO DAGScheduler:59 - ResultStage 2 (collect at NaiveBayes.scala:401) finished in 0.035 s -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 10:34:26 INFO DAGScheduler:59 - Job 1 finished: collect at NaiveBayes.scala:401, took 0.225854 s -2016-05-25 10:34:26 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:40 -2016-05-25 10:34:26 INFO DAGScheduler:59 - Got job 2 (count at NaiveBayesLearning.scala:40) with 2 output partitions -2016-05-25 10:34:26 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at NaiveBayesLearning.scala:40) -2016-05-25 10:34:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:34:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:40), which has no missing parents -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=189324, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(2671) called with curMem=194220, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:34:26 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52786 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:34:26 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at filter at NaiveBayesLearning.scala:40) -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 10:34:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:34:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:34:26 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-25 10:34:26 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-25 10:34:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:34:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:34:26 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-25 10:34:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 18 ms on localhost (1/2) -2016-05-25 10:34:26 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 2082 bytes result sent to driver -2016-05-25 10:34:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 20 ms on localhost (2/2) -2016-05-25 10:34:26 INFO DAGScheduler:59 - ResultStage 3 (count at NaiveBayesLearning.scala:40) finished in 0.022 s -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 10:34:26 INFO DAGScheduler:59 - Job 2 finished: count at NaiveBayesLearning.scala:40, took 0.035998 s -2016-05-25 10:34:26 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:40 -2016-05-25 10:34:26 INFO DAGScheduler:59 - Got job 3 (count at NaiveBayesLearning.scala:40) with 2 output partitions -2016-05-25 10:34:26 INFO DAGScheduler:59 - Final stage: ResultStage 4(count at NaiveBayesLearning.scala:40) -2016-05-25 10:34:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:34:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=196891, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(2117) called with curMem=200491, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:34:26 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52786 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:34:26 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 10:34:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:34:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:34:26 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-25 10:34:26 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-25 10:34:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:34:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:34:26 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2082 bytes result sent to driver -2016-05-25 10:34:26 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2082 bytes result sent to driver -2016-05-25 10:34:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 19 ms on localhost (1/2) -2016-05-25 10:34:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 22 ms on localhost (2/2) -2016-05-25 10:34:26 INFO DAGScheduler:59 - ResultStage 4 (count at NaiveBayesLearning.scala:40) finished in 0.022 s -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 10:34:26 INFO DAGScheduler:59 - Job 3 finished: count at NaiveBayesLearning.scala:40, took 0.036871 s -2016-05-25 10:34:26 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:44 -2016-05-25 10:34:26 INFO DAGScheduler:59 - Got job 4 (count at NaiveBayesLearning.scala:44) with 2 output partitions -2016-05-25 10:34:26 INFO DAGScheduler:59 - Final stage: ResultStage 5(count at NaiveBayesLearning.scala:44) -2016-05-25 10:34:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:34:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=202608, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(2113) called with curMem=206208, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:34:26 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52786 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:34:26 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 10:34:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:34:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:34:26 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-05-25 10:34:26 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-05-25 10:34:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:34:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:34:26 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 2082 bytes result sent to driver -2016-05-25 10:34:26 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 2082 bytes result sent to driver -2016-05-25 10:34:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 19 ms on localhost (1/2) -2016-05-25 10:34:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 18 ms on localhost (2/2) -2016-05-25 10:34:26 INFO DAGScheduler:59 - ResultStage 5 (count at NaiveBayesLearning.scala:44) finished in 0.020 s -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 10:34:26 INFO DAGScheduler:59 - Job 4 finished: count at NaiveBayesLearning.scala:44, took 0.034732 s -2016-05-25 10:34:26 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:45 -2016-05-25 10:34:26 INFO DAGScheduler:59 - Got job 5 (foreach at NaiveBayesLearning.scala:45) with 2 output partitions -2016-05-25 10:34:26 INFO DAGScheduler:59 - Final stage: ResultStage 6(foreach at NaiveBayesLearning.scala:45) -2016-05-25 10:34:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:34:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=208321, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(2143) called with curMem=212057, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:34:26 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52786 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:34:26 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[3] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 10:34:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:34:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:34:26 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-05-25 10:34:26 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 13) -2016-05-25 10:34:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:34:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:34:26 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 2044 bytes result sent to driver -2016-05-25 10:34:26 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 13). 2044 bytes result sent to driver -2016-05-25 10:34:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 18 ms on localhost (1/2) -2016-05-25 10:34:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 13) in 18 ms on localhost (2/2) -2016-05-25 10:34:26 INFO DAGScheduler:59 - ResultStage 6 (foreach at NaiveBayesLearning.scala:45) finished in 0.019 s -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 10:34:26 INFO DAGScheduler:59 - Job 5 finished: foreach at NaiveBayesLearning.scala:45, took 0.034904 s -2016-05-25 10:34:26 INFO SparkContext:59 - Starting job: count at NaiveBayesLearning.scala:46 -2016-05-25 10:34:26 INFO DAGScheduler:59 - Got job 6 (count at NaiveBayesLearning.scala:46) with 2 output partitions -2016-05-25 10:34:26 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at NaiveBayesLearning.scala:46) -2016-05-25 10:34:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:34:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(3600) called with curMem=214200, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(2117) called with curMem=217800, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:34:26 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52786 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:34:26 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 10:34:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:34:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 15, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:34:26 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 14) -2016-05-25 10:34:26 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 15) -2016-05-25 10:34:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:34:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:34:26 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 14). 2082 bytes result sent to driver -2016-05-25 10:34:26 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 10:34:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 14) in 19 ms on localhost (1/2) -2016-05-25 10:34:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 15) in 21 ms on localhost (2/2) -2016-05-25 10:34:26 INFO DAGScheduler:59 - ResultStage 7 (count at NaiveBayesLearning.scala:46) finished in 0.023 s -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 10:34:26 INFO DAGScheduler:59 - Job 6 finished: count at NaiveBayesLearning.scala:46, took 0.043869 s -2016-05-25 10:34:26 INFO SparkContext:59 - Starting job: foreach at NaiveBayesLearning.scala:47 -2016-05-25 10:34:26 INFO DAGScheduler:59 - Got job 7 (foreach at NaiveBayesLearning.scala:47) with 2 output partitions -2016-05-25 10:34:26 INFO DAGScheduler:59 - Final stage: ResultStage 8(foreach at NaiveBayesLearning.scala:47) -2016-05-25 10:34:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:34:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(3736) called with curMem=219917, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(2147) called with curMem=223653, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 10:34:26 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52786 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:34:26 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[4] at randomSplit at NaiveBayesLearning.scala:33) -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 10:34:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:34:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 17, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:34:26 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 16) -2016-05-25 10:34:26 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 17) -2016-05-25 10:34:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:34:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:34:26 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 16). 2044 bytes result sent to driver -2016-05-25 10:34:26 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 17). 2044 bytes result sent to driver -2016-05-25 10:34:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 16) in 20 ms on localhost (1/2) -2016-05-25 10:34:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 17) in 19 ms on localhost (2/2) -2016-05-25 10:34:26 INFO DAGScheduler:59 - ResultStage 8 (foreach at NaiveBayesLearning.scala:47) finished in 0.021 s -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 10:34:26 INFO DAGScheduler:59 - Job 7 finished: foreach at NaiveBayesLearning.scala:47, took 0.041565 s -2016-05-25 10:34:26 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:50 -2016-05-25 10:34:26 INFO DAGScheduler:59 - Got job 8 (take at NaiveBayesLearning.scala:50) with 1 output partitions -2016-05-25 10:34:26 INFO DAGScheduler:59 - Final stage: ResultStage 9(take at NaiveBayesLearning.scala:50) -2016-05-25 10:34:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:34:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=225800, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(2669) called with curMem=230624, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:34:26 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:52786 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:34:26 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 9 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39) -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 1 tasks -2016-05-25 10:34:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:34:26 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 18) -2016-05-25 10:34:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 10:34:26 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 18). 2040 bytes result sent to driver -2016-05-25 10:34:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 18) in 20 ms on localhost (1/1) -2016-05-25 10:34:26 INFO DAGScheduler:59 - ResultStage 9 (take at NaiveBayesLearning.scala:50) finished in 0.020 s -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 10:34:26 INFO DAGScheduler:59 - Job 8 finished: take at NaiveBayesLearning.scala:50, took 0.039729 s -2016-05-25 10:34:26 INFO SparkContext:59 - Starting job: take at NaiveBayesLearning.scala:50 -2016-05-25 10:34:26 INFO DAGScheduler:59 - Got job 9 (take at NaiveBayesLearning.scala:50) with 1 output partitions -2016-05-25 10:34:26 INFO DAGScheduler:59 - Final stage: ResultStage 10(take at NaiveBayesLearning.scala:50) -2016-05-25 10:34:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:34:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39), which has no missing parents -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(4824) called with curMem=233293, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 10:34:26 INFO MemoryStore:59 - ensureFreeSpace(2669) called with curMem=238117, maxMem=766075207 -2016-05-25 10:34:26 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 10:34:26 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:52786 (size: 2.6 KB, free: 730.5 MB) -2016-05-25 10:34:26 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:34:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[7] at map at NaiveBayesLearning.scala:39) -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-25 10:34:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 10:34:26 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 10:34:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 10:34:26 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2206 bytes result sent to driver -2016-05-25 10:34:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 50 ms on localhost (1/1) -2016-05-25 10:34:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 10:34:26 INFO DAGScheduler:59 - ResultStage 10 (take at NaiveBayesLearning.scala:50) finished in 0.053 s -2016-05-25 10:34:26 INFO DAGScheduler:59 - Job 9 finished: take at NaiveBayesLearning.scala:50, took 0.073007 s -2016-05-25 10:34:26 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:52786 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:34:26 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 10:34:26 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:52786 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:34:26 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 10:34:26 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:52786 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:34:26 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 10:34:27 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:52786 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:34:27 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 10:34:27 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:52786 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:34:27 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 10:34:27 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52786 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 10:34:27 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 10:34:27 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52786 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:34:27 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 10:34:27 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52786 in memory (size: 1567.0 B, free: 730.6 MB) -2016-05-25 10:34:27 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 10:34:27 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52786 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:34:27 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 10:34:27 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 10:34:27 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52786 in memory (size: 2013.0 B, free: 730.6 MB) -2016-05-25 10:34:27 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 10:34:27 INFO SparkContext:59 - Starting job: saveAsTextFile at NaiveBayes.scala:202 -2016-05-25 10:34:27 INFO DAGScheduler:59 - Got job 10 (saveAsTextFile at NaiveBayes.scala:202) with 1 output partitions -2016-05-25 10:34:27 INFO DAGScheduler:59 - Final stage: ResultStage 11(saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 10:34:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:34:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:34:27 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202), which has no missing parents -2016-05-25 10:34:27 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=179073, maxMem=766075207 -2016-05-25 10:34:27 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 124.0 KB, free 730.3 MB) -2016-05-25 10:34:27 INFO MemoryStore:59 - ensureFreeSpace(42584) called with curMem=306065, maxMem=766075207 -2016-05-25 10:34:27 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.3 MB) -2016-05-25 10:34:27 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:52786 (size: 41.6 KB, free: 730.5 MB) -2016-05-25 10:34:27 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:34:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 11 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 10:34:27 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 1 tasks -2016-05-25 10:34:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 20, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-25 10:34:27 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 20) -2016-05-25 10:34:27 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251034_0011_m_000000_20' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103426963/result/metadata/_temporary/0/task_201605251034_0011_m_000000 -2016-05-25 10:34:27 INFO SparkHadoopMapRedUtil:59 - attempt_201605251034_0011_m_000000_20: Committed -2016-05-25 10:34:27 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 20). 1864 bytes result sent to driver -2016-05-25 10:34:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 20) in 260 ms on localhost (1/1) -2016-05-25 10:34:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 10:34:27 INFO DAGScheduler:59 - ResultStage 11 (saveAsTextFile at NaiveBayes.scala:202) finished in 0.262 s -2016-05-25 10:34:27 INFO DAGScheduler:59 - Job 10 finished: saveAsTextFile at NaiveBayes.scala:202, took 0.345600 s -2016-05-25 10:34:29 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:52786 in memory (size: 41.6 KB, free: 730.6 MB) -2016-05-25 10:34:29 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-25 10:34:29 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:52786 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 10:34:29 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 10:34:29 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:34:29 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:34:29 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:206 -2016-05-25 10:34:29 INFO DAGScheduler:59 - Got job 11 (parquet at NaiveBayes.scala:206) with 1 output partitions -2016-05-25 10:34:29 INFO DAGScheduler:59 - Final stage: ResultStage 12(parquet at NaiveBayes.scala:206) -2016-05-25 10:34:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:34:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:34:29 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205), which has no missing parents -2016-05-25 10:34:29 INFO MemoryStore:59 - ensureFreeSpace(66936) called with curMem=171580, maxMem=766075207 -2016-05-25 10:34:29 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 65.4 KB, free 730.4 MB) -2016-05-25 10:34:29 INFO MemoryStore:59 - ensureFreeSpace(23365) called with curMem=238516, maxMem=766075207 -2016-05-25 10:34:29 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 22.8 KB, free 730.3 MB) -2016-05-25 10:34:29 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:52786 (size: 22.8 KB, free: 730.6 MB) -2016-05-25 10:34:29 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:34:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205) -2016-05-25 10:34:29 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-25 10:34:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 21, localhost, PROCESS_LOCAL, 2506 bytes) -2016-05-25 10:34:29 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 21) -2016-05-25 10:34:29 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 10:34:29 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 10:34:29 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 10:34:29 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 10:34:29 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 10:34:29 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 10:34:29 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 10:34:29 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 10:34:29 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 10:34:29 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 231 -2016-05-25 10:34:29 INFO ColumnChunkPageWriteStore:151 - written 77B for [labels, bag, array] DOUBLE: 3 values, 37B raw, 36B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 10:34:29 INFO ColumnChunkPageWriteStore:151 - written 74B for [pi, bag, array] DOUBLE: 3 values, 16B raw, 33B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 2 entries, 16B raw, 2B comp} -2016-05-25 10:34:29 INFO ColumnChunkPageWriteStore:151 - written 83B for [theta, bag, array, bag, array] DOUBLE: 9 values, 23B raw, 42B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 6 entries, 48B raw, 6B comp} -2016-05-25 10:34:29 INFO ColumnChunkPageWriteStore:151 - written 86B for [modelType] BINARY: 1 values, 21B raw, 39B comp, 1 pages, encodings: [PLAIN, BIT_PACKED, RLE] -2016-05-25 10:34:29 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251034_0012_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103426963/result/data/_temporary/0/task_201605251034_0012_m_000000 -2016-05-25 10:34:29 INFO SparkHadoopMapRedUtil:59 - attempt_201605251034_0012_m_000000_0: Committed -2016-05-25 10:34:29 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 21). 915 bytes result sent to driver -2016-05-25 10:34:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 21) in 495 ms on localhost (1/1) -2016-05-25 10:34:29 INFO DAGScheduler:59 - ResultStage 12 (parquet at NaiveBayes.scala:206) finished in 0.495 s -2016-05-25 10:34:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 10:34:29 INFO DAGScheduler:59 - Job 11 finished: parquet at NaiveBayes.scala:206, took 0.534877 s -2016-05-25 10:34:29 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:34:30 INFO DefaultWriterContainer:59 - Job job_201605251034_0000 committed. -2016-05-25 10:34:30 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103426963/result/data on driver -2016-05-25 10:34:30 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103426963/result/data on driver -2016-05-25 10:34:30 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=261881, maxMem=766075207 -2016-05-25 10:34:30 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-25 10:34:30 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=492585, maxMem=766075207 -2016-05-25 10:34:30 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-25 10:34:30 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:52786 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 10:34:30 INFO SparkContext:59 - Created broadcast 14 from textFile at modelSaveLoad.scala:129 -2016-05-25 10:34:30 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:34:30 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 10:34:30 INFO DAGScheduler:59 - Got job 12 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 10:34:30 INFO DAGScheduler:59 - Final stage: ResultStage 13(first at modelSaveLoad.scala:129) -2016-05-25 10:34:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:34:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:34:30 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 10:34:30 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=512373, maxMem=766075207 -2016-05-25 10:34:30 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-25 10:34:30 INFO MemoryStore:59 - ensureFreeSpace(1889) called with curMem=515589, maxMem=766075207 -2016-05-25 10:34:30 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1889.0 B, free 730.1 MB) -2016-05-25 10:34:30 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:52786 (size: 1889.0 B, free: 730.5 MB) -2016-05-25 10:34:30 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:34:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 13 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129) -2016-05-25 10:34:30 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 1 tasks -2016-05-25 10:34:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 22, localhost, PROCESS_LOCAL, 2239 bytes) -2016-05-25 10:34:30 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 22) -2016-05-25 10:34:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103426963/result/metadata/part-00000:0+56 -2016-05-25 10:34:30 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 22). 2159 bytes result sent to driver -2016-05-25 10:34:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 22) in 16 ms on localhost (1/1) -2016-05-25 10:34:30 INFO DAGScheduler:59 - ResultStage 13 (first at modelSaveLoad.scala:129) finished in 0.017 s -2016-05-25 10:34:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 10:34:30 INFO DAGScheduler:59 - Job 12 finished: first at modelSaveLoad.scala:129, took 0.029210 s -2016-05-25 10:34:30 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103426963/result/data on driver -2016-05-25 10:34:30 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:213 -2016-05-25 10:34:30 INFO DAGScheduler:59 - Got job 13 (parquet at NaiveBayes.scala:213) with 4 output partitions -2016-05-25 10:34:30 INFO DAGScheduler:59 - Final stage: ResultStage 14(parquet at NaiveBayes.scala:213) -2016-05-25 10:34:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:34:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:34:30 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213), which has no missing parents -2016-05-25 10:34:30 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=517478, maxMem=766075207 -2016-05-25 10:34:30 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-25 10:34:30 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=580478, maxMem=766075207 -2016-05-25 10:34:30 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.0 MB) -2016-05-25 10:34:30 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:52786 (size: 20.6 KB, free: 730.5 MB) -2016-05-25 10:34:30 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:34:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 14 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213) -2016-05-25 10:34:30 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 4 tasks -2016-05-25 10:34:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 23, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:34:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 24, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:34:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 25, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 10:34:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 14.0 (TID 26, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-25 10:34:30 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 24) -2016-05-25 10:34:30 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 23) -2016-05-25 10:34:30 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 25) -2016-05-25 10:34:30 INFO Executor:59 - Running task 3.0 in stage 14.0 (TID 26) -2016-05-25 10:34:30 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:34:30 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:34:30 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 25). 936 bytes result sent to driver -2016-05-25 10:34:30 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:34:30 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 24). 936 bytes result sent to driver -2016-05-25 10:34:30 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 23). 936 bytes result sent to driver -2016-05-25 10:34:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 25) in 98 ms on localhost (1/4) -2016-05-25 10:34:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 24) in 100 ms on localhost (2/4) -2016-05-25 10:34:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 23) in 101 ms on localhost (3/4) -2016-05-25 10:34:30 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 10:34:30 INFO Executor:59 - Finished task 3.0 in stage 14.0 (TID 26). 1876 bytes result sent to driver -2016-05-25 10:34:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 14.0 (TID 26) in 141 ms on localhost (4/4) -2016-05-25 10:34:30 INFO DAGScheduler:59 - ResultStage 14 (parquet at NaiveBayes.scala:213) finished in 0.146 s -2016-05-25 10:34:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 10:34:30 INFO DAGScheduler:59 - Job 13 finished: parquet at NaiveBayes.scala:213, took 0.184002 s -2016-05-25 10:34:30 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=601551, maxMem=766075207 -2016-05-25 10:34:30 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 86.4 KB, free 729.9 MB) -2016-05-25 10:34:30 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=690063, maxMem=766075207 -2016-05-25 10:34:30 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-25 10:34:30 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:52786 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 10:34:30 INFO SparkContext:59 - Created broadcast 17 from take at NaiveBayes.scala:216 -2016-05-25 10:34:30 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 10:34:30 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103426963/result/data/part-r-00000-e07a53ff-0932-4342-b41f-ab381a88edda.gz.parquet -2016-05-25 10:34:30 INFO SparkContext:59 - Starting job: take at NaiveBayes.scala:216 -2016-05-25 10:34:30 INFO DAGScheduler:59 - Got job 14 (take at NaiveBayes.scala:216) with 1 output partitions -2016-05-25 10:34:30 INFO DAGScheduler:59 - Final stage: ResultStage 15(take at NaiveBayes.scala:216) -2016-05-25 10:34:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:34:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:34:30 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216), which has no missing parents -2016-05-25 10:34:30 INFO MemoryStore:59 - ensureFreeSpace(4280) called with curMem=709851, maxMem=766075207 -2016-05-25 10:34:30 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 4.2 KB, free 729.9 MB) -2016-05-25 10:34:30 INFO MemoryStore:59 - ensureFreeSpace(2466) called with curMem=714131, maxMem=766075207 -2016-05-25 10:34:30 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.4 KB, free 729.9 MB) -2016-05-25 10:34:30 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:52786 (size: 2.4 KB, free: 730.5 MB) -2016-05-25 10:34:30 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:34:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 15 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216) -2016-05-25 10:34:30 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 1 tasks -2016-05-25 10:34:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 27, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-25 10:34:30 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 27) -2016-05-25 10:34:30 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525103426963/result/data/part-r-00000-e07a53ff-0932-4342-b41f-ab381a88edda.gz.parquet start: 0 end: 1477 length: 1477 hosts: []} -2016-05-25 10:34:31 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 10:34:31 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group labels (LIST) { - repeated group bag { - optional double array; - } - } - optional group pi (LIST) { - repeated group bag { - optional double array; - } - } - optional group theta (LIST) { - repeated group bag { - optional group array (LIST) { - repeated group bag { - optional double array; - } - } - } - } - optional binary modelType (UTF8); -} - -Catalyst form: -StructType(StructField(labels,ArrayType(DoubleType,true),true), StructField(pi,ArrayType(DoubleType,true),true), StructField(theta,ArrayType(ArrayType(DoubleType,true),true),true), StructField(modelType,StringType,true)) - -2016-05-25 10:34:31 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 10:34:31 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 10:34:31 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 10:34:31 INFO InternalParquetRecordReader:151 - block read in memory in 13 ms. row count = 1 -2016-05-25 10:34:31 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 27). 2841 bytes result sent to driver -2016-05-25 10:34:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 27) in 121 ms on localhost (1/1) -2016-05-25 10:34:31 INFO DAGScheduler:59 - ResultStage 15 (take at NaiveBayes.scala:216) finished in 0.121 s -2016-05-25 10:34:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-25 10:34:31 INFO DAGScheduler:59 - Job 14 finished: take at NaiveBayes.scala:216, took 0.140161 s -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 10:34:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 10:34:31 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 10:34:31 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 10:34:31 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 10:34:31 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 10:34:31 INFO BlockManager:59 - BlockManager stopped -2016-05-25 10:34:31 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 10:34:31 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 10:34:31 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 10:34:31 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 10:34:31 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-42c80a3b-c927-4288-9164-48b7d7afeca4 -2016-05-25 10:34:31 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 10:34:31 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-25 10:36:13 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 10:36:13 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 10:36:14 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 10:36:14 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 10:36:14 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 10:36:14 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 10:36:15 INFO Remoting:74 - Starting remoting -2016-05-25 10:36:15 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:52832] -2016-05-25 10:36:15 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52832. -2016-05-25 10:36:15 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 10:36:15 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 10:36:15 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-c51844f8-f934-4e22-98d2-060a84afbcaf -2016-05-25 10:36:15 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 10:36:15 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-a5d97a03-dccb-438b-aaaa-4ebba9b52b97\httpd-2f07062e-cfc6-4527-b1ef-1fcf11df09ce -2016-05-25 10:36:15 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 10:36:15 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:36:15 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52833 -2016-05-25 10:36:15 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52833. -2016-05-25 10:36:15 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 10:36:15 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:36:15 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 10:36:15 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 10:36:15 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 10:36:15 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 10:36:15 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 10:36:16 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52852. -2016-05-25 10:36:16 INFO NettyBlockTransferService:59 - Server created on 52852 -2016-05-25 10:36:16 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 10:36:16 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52852 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52852) -2016-05-25 10:36:16 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 10:36:16 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 10:36:16 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 10:36:16 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 10:36:16 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 10:36:16 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52852 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 10:36:16 INFO SparkContext:59 - Created broadcast 0 from textFile at BernoulliNaiveBayesLearning.scala:26 -2016-05-25 10:36:18 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 10:36:18 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 10:36:18 INFO SparkContext:59 - Starting job: foreach at BernoulliNaiveBayesLearning.scala:32 -2016-05-25 10:36:18 INFO DAGScheduler:59 - Got job 0 (foreach at BernoulliNaiveBayesLearning.scala:32) with 2 output partitions -2016-05-25 10:36:18 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at BernoulliNaiveBayesLearning.scala:32) -2016-05-25 10:36:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:36:18 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:36:18 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at BernoulliNaiveBayesLearning.scala:27), which has no missing parents -2016-05-25 10:36:18 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=171580, maxMem=766075207 -2016-05-25 10:36:18 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-25 10:36:18 INFO MemoryStore:59 - ensureFreeSpace(2021) called with curMem=175036, maxMem=766075207 -2016-05-25 10:36:18 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2021.0 B, free 730.4 MB) -2016-05-25 10:36:18 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52852 (size: 2021.0 B, free: 730.6 MB) -2016-05-25 10:36:18 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:36:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at BernoulliNaiveBayesLearning.scala:27) -2016-05-25 10:36:18 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 10:36:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2206 bytes) -2016-05-25 10:36:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2206 bytes) -2016-05-25 10:36:18 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 10:36:18 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 10:36:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data2.txt:0+31 -2016-05-25 10:36:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data2.txt:31+32 -2016-05-25 10:36:18 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 10:36:18 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 10:36:18 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 10:36:18 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 10:36:18 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 10:36:19 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-25 10:36:19 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-25 10:36:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 97 ms on localhost (1/2) -2016-05-25 10:36:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 252 ms on localhost (2/2) -2016-05-25 10:36:19 INFO DAGScheduler:59 - ResultStage 0 (foreach at BernoulliNaiveBayesLearning.scala:32) finished in 0.269 s -2016-05-25 10:36:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 10:36:19 INFO DAGScheduler:59 - Job 0 finished: foreach at BernoulliNaiveBayesLearning.scala:32, took 0.421685 s -2016-05-25 10:36:19 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 10:36:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 10:36:19 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 10:36:19 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 10:36:19 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 10:36:19 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 10:36:19 INFO BlockManager:59 - BlockManager stopped -2016-05-25 10:36:19 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 10:36:19 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 10:36:19 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 10:36:19 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 10:36:19 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-a5d97a03-dccb-438b-aaaa-4ebba9b52b97 -2016-05-25 10:42:24 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 10:42:25 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 10:42:25 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 10:42:25 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 10:42:25 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 10:42:26 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 10:42:26 INFO Remoting:74 - Starting remoting -2016-05-25 10:42:26 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:52975] -2016-05-25 10:42:26 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52975. -2016-05-25 10:42:26 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 10:42:26 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 10:42:26 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-37e11ad5-ec88-4fd2-bd6d-d761dd306633 -2016-05-25 10:42:26 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 10:42:27 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-07361073-2ce7-473d-8c6c-c32d167ba319\httpd-1a27957b-414f-4108-8484-6d2bc4e54124 -2016-05-25 10:42:27 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 10:42:27 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:42:27 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52976 -2016-05-25 10:42:27 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52976. -2016-05-25 10:42:27 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 10:42:27 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:42:27 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 10:42:27 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 10:42:27 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 10:42:27 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 10:42:27 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 10:42:27 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52995. -2016-05-25 10:42:27 INFO NettyBlockTransferService:59 - Server created on 52995 -2016-05-25 10:42:27 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 10:42:27 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52995 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52995) -2016-05-25 10:42:27 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 10:42:28 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:42:28 INFO DAGScheduler:59 - Registering RDD 1 (map at NaiveBayes.scala:383) -2016-05-25 10:42:28 INFO DAGScheduler:59 - Got job 0 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:42:28 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at NaiveBayes.scala:401) -2016-05-25 10:42:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-05-25 10:42:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-05-25 10:42:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[1] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:42:28 INFO MemoryStore:59 - ensureFreeSpace(3288) called with curMem=0, maxMem=766075207 -2016-05-25 10:42:28 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-05-25 10:42:28 INFO MemoryStore:59 - ensureFreeSpace(1806) called with curMem=3288, maxMem=766075207 -2016-05-25 10:42:28 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1806.0 B, free 730.6 MB) -2016-05-25 10:42:28 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52995 (size: 1806.0 B, free: 730.6 MB) -2016-05-25 10:42:28 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:42:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[1] at map at NaiveBayes.scala:383) -2016-05-25 10:42:28 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 10:42:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2378 bytes) -2016-05-25 10:42:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2378 bytes) -2016-05-25 10:42:28 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 10:42:28 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 10:42:28 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 10:42:28 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 10:42:28 ERROR Executor:96 - Exception in task 0.0 in stage 0.0 (TID 0) -org.apache.spark.SparkException: Bernoulli naive Bayes requires 0 or 1 feature values but found [2.0]. - at org.apache.spark.mllib.classification.NaiveBayes$$anonfun$18.apply(NaiveBayes.scala:375) - at org.apache.spark.mllib.classification.NaiveBayes$$anonfun$18.apply(NaiveBayes.scala:369) - at org.apache.spark.mllib.classification.NaiveBayes$$anonfun$20.apply(NaiveBayes.scala:386) - at org.apache.spark.mllib.classification.NaiveBayes$$anonfun$20.apply(NaiveBayes.scala:384) - at org.apache.spark.util.collection.ExternalSorter$$anonfun$5.apply(ExternalSorter.scala:207) - at org.apache.spark.util.collection.ExternalSorter$$anonfun$5.apply(ExternalSorter.scala:206) - at org.apache.spark.util.collection.AppendOnlyMap.changeValue(AppendOnlyMap.scala:148) - at org.apache.spark.util.collection.SizeTrackingAppendOnlyMap.changeValue(SizeTrackingAppendOnlyMap.scala:32) - at org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:212) - at org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:73) - at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73) - at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-05-25 10:42:28 WARN TaskSetManager:71 - Lost task 0.0 in stage 0.0 (TID 0, localhost): org.apache.spark.SparkException: Bernoulli naive Bayes requires 0 or 1 feature values but found [2.0]. - at org.apache.spark.mllib.classification.NaiveBayes$$anonfun$18.apply(NaiveBayes.scala:375) - at org.apache.spark.mllib.classification.NaiveBayes$$anonfun$18.apply(NaiveBayes.scala:369) - at org.apache.spark.mllib.classification.NaiveBayes$$anonfun$20.apply(NaiveBayes.scala:386) - at org.apache.spark.mllib.classification.NaiveBayes$$anonfun$20.apply(NaiveBayes.scala:384) - at org.apache.spark.util.collection.ExternalSorter$$anonfun$5.apply(ExternalSorter.scala:207) - at org.apache.spark.util.collection.ExternalSorter$$anonfun$5.apply(ExternalSorter.scala:206) - at org.apache.spark.util.collection.AppendOnlyMap.changeValue(AppendOnlyMap.scala:148) - at org.apache.spark.util.collection.SizeTrackingAppendOnlyMap.changeValue(SizeTrackingAppendOnlyMap.scala:32) - at org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:212) - at org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:73) - at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73) - at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) - -2016-05-25 10:42:28 ERROR TaskSetManager:75 - Task 0 in stage 0.0 failed 1 times; aborting job -2016-05-25 10:42:28 INFO TaskSchedulerImpl:59 - Cancelling stage 0 -2016-05-25 10:42:28 INFO Executor:59 - Executor is trying to kill task 1.0 in stage 0.0 (TID 1) -2016-05-25 10:42:28 INFO TaskSchedulerImpl:59 - Stage 0 was cancelled -2016-05-25 10:42:28 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1159 bytes result sent to driver -2016-05-25 10:42:28 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at NaiveBayes.scala:383) failed in 0.171 s -2016-05-25 10:42:28 INFO DAGScheduler:59 - Job 0 failed: collect at NaiveBayes.scala:401, took 0.436874 s -2016-05-25 10:42:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 153 ms on localhost (1/2) -2016-05-25 10:42:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 10:42:28 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 10:42:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 10:42:28 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 10:42:28 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 10:42:28 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 10:42:29 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 10:42:29 INFO BlockManager:59 - BlockManager stopped -2016-05-25 10:42:29 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 10:42:29 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 10:42:29 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 10:42:29 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 10:42:29 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-07361073-2ce7-473d-8c6c-c32d167ba319 -2016-05-25 10:42:29 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 10:42:29 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-25 10:43:23 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 10:43:24 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 10:43:24 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 10:43:24 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 10:43:24 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 10:43:25 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 10:43:25 INFO Remoting:74 - Starting remoting -2016-05-25 10:43:25 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53012] -2016-05-25 10:43:25 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53012. -2016-05-25 10:43:25 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 10:43:25 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 10:43:25 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-d83b17f4-2712-4208-8bc3-2f7aae1bae2e -2016-05-25 10:43:26 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 10:43:26 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-55ca3292-2ba8-4cc4-91a8-f1cd45385fe1\httpd-448878d2-280d-4c7f-9114-783d1ade972e -2016-05-25 10:43:26 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 10:43:26 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:43:26 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53013 -2016-05-25 10:43:26 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53013. -2016-05-25 10:43:26 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 10:43:26 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:43:26 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 10:43:26 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 10:43:26 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 10:43:26 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 10:43:26 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 10:43:27 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53032. -2016-05-25 10:43:27 INFO NettyBlockTransferService:59 - Server created on 53032 -2016-05-25 10:43:27 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 10:43:27 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53032 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53032) -2016-05-25 10:43:27 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 10:43:27 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:43:27 INFO DAGScheduler:59 - Registering RDD 1 (map at NaiveBayes.scala:383) -2016-05-25 10:43:27 INFO DAGScheduler:59 - Got job 0 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:43:27 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at NaiveBayes.scala:401) -2016-05-25 10:43:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-05-25 10:43:27 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-05-25 10:43:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[1] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:43:27 INFO MemoryStore:59 - ensureFreeSpace(3288) called with curMem=0, maxMem=766075207 -2016-05-25 10:43:27 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-05-25 10:43:27 INFO MemoryStore:59 - ensureFreeSpace(1806) called with curMem=3288, maxMem=766075207 -2016-05-25 10:43:27 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1806.0 B, free 730.6 MB) -2016-05-25 10:43:27 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53032 (size: 1806.0 B, free: 730.6 MB) -2016-05-25 10:43:27 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:43:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[1] at map at NaiveBayes.scala:383) -2016-05-25 10:43:27 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 10:43:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2340 bytes) -2016-05-25 10:43:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2378 bytes) -2016-05-25 10:43:27 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 10:43:27 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 10:43:28 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 10:43:28 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 10:43:28 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1159 bytes result sent to driver -2016-05-25 10:43:28 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1159 bytes result sent to driver -2016-05-25 10:43:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 136 ms on localhost (1/2) -2016-05-25 10:43:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 172 ms on localhost (2/2) -2016-05-25 10:43:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 10:43:28 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at NaiveBayes.scala:383) finished in 0.191 s -2016-05-25 10:43:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 10:43:28 INFO DAGScheduler:59 - running: Set() -2016-05-25 10:43:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-05-25 10:43:28 INFO DAGScheduler:59 - failed: Set() -2016-05-25 10:43:28 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-05-25 10:43:28 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[2] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 10:43:28 INFO MemoryStore:59 - ensureFreeSpace(2928) called with curMem=5094, maxMem=766075207 -2016-05-25 10:43:28 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-05-25 10:43:28 INFO MemoryStore:59 - ensureFreeSpace(1559) called with curMem=8022, maxMem=766075207 -2016-05-25 10:43:28 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1559.0 B, free 730.6 MB) -2016-05-25 10:43:28 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53032 (size: 1559.0 B, free: 730.6 MB) -2016-05-25 10:43:28 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:43:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[2] at combineByKey at NaiveBayes.scala:383) -2016-05-25 10:43:28 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-25 10:43:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:43:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:43:28 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 10:43:28 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-25 10:43:28 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 10:43:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 10:43:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 10:43:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 10:43:28 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-05-25 10:43:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 38 ms on localhost (1/2) -2016-05-25 10:43:28 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1469 bytes result sent to driver -2016-05-25 10:43:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 43 ms on localhost (2/2) -2016-05-25 10:43:28 INFO DAGScheduler:59 - ResultStage 1 (collect at NaiveBayes.scala:401) finished in 0.044 s -2016-05-25 10:43:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 10:43:28 INFO DAGScheduler:59 - Job 0 finished: collect at NaiveBayes.scala:401, took 0.473272 s -2016-05-25 10:43:28 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:43:28 INFO DAGScheduler:59 - Registering RDD 4 (map at NaiveBayes.scala:383) -2016-05-25 10:43:28 INFO DAGScheduler:59 - Got job 1 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:43:28 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at NaiveBayes.scala:401) -2016-05-25 10:43:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 2) -2016-05-25 10:43:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 2) -2016-05-25 10:43:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[4] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:43:28 INFO MemoryStore:59 - ensureFreeSpace(3288) called with curMem=9581, maxMem=766075207 -2016-05-25 10:43:28 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-05-25 10:43:28 INFO MemoryStore:59 - ensureFreeSpace(1804) called with curMem=12869, maxMem=766075207 -2016-05-25 10:43:28 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1804.0 B, free 730.6 MB) -2016-05-25 10:43:28 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53032 (size: 1804.0 B, free: 730.6 MB) -2016-05-25 10:43:28 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:43:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[4] at map at NaiveBayes.scala:383) -2016-05-25 10:43:28 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 10:43:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2416 bytes) -2016-05-25 10:43:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2454 bytes) -2016-05-25 10:43:28 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-25 10:43:28 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-25 10:43:28 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1159 bytes result sent to driver -2016-05-25 10:43:28 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1159 bytes result sent to driver -2016-05-25 10:43:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 18 ms on localhost (1/2) -2016-05-25 10:43:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 23 ms on localhost (2/2) -2016-05-25 10:43:28 INFO DAGScheduler:59 - ShuffleMapStage 2 (map at NaiveBayes.scala:383) finished in 0.023 s -2016-05-25 10:43:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 10:43:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 10:43:28 INFO DAGScheduler:59 - running: Set() -2016-05-25 10:43:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 3) -2016-05-25 10:43:28 INFO DAGScheduler:59 - failed: Set() -2016-05-25 10:43:28 INFO DAGScheduler:59 - Missing parents for ResultStage 3: List() -2016-05-25 10:43:28 INFO DAGScheduler:59 - Submitting ResultStage 3 (ShuffledRDD[5] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 10:43:28 INFO MemoryStore:59 - ensureFreeSpace(2928) called with curMem=14673, maxMem=766075207 -2016-05-25 10:43:28 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-05-25 10:43:28 INFO MemoryStore:59 - ensureFreeSpace(1553) called with curMem=17601, maxMem=766075207 -2016-05-25 10:43:28 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1553.0 B, free 730.6 MB) -2016-05-25 10:43:28 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53032 (size: 1553.0 B, free: 730.6 MB) -2016-05-25 10:43:28 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:43:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (ShuffledRDD[5] at combineByKey at NaiveBayes.scala:383) -2016-05-25 10:43:28 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 10:43:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:43:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:43:28 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-25 10:43:28 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-25 10:43:28 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 10:43:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 10:43:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 10:43:28 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 1161 bytes result sent to driver -2016-05-25 10:43:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 10:43:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 10 ms on localhost (1/2) -2016-05-25 10:43:28 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 1533 bytes result sent to driver -2016-05-25 10:43:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 16 ms on localhost (2/2) -2016-05-25 10:43:28 INFO DAGScheduler:59 - ResultStage 3 (collect at NaiveBayes.scala:401) finished in 0.016 s -2016-05-25 10:43:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 10:43:28 INFO DAGScheduler:59 - Job 1 finished: collect at NaiveBayes.scala:401, took 0.090170 s -2016-05-25 10:43:28 INFO MemoryStore:59 - ensureFreeSpace(520) called with curMem=19154, maxMem=766075207 -2016-05-25 10:43:28 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 520.0 B, free 730.6 MB) -2016-05-25 10:43:28 INFO MemoryStore:59 - ensureFreeSpace(679) called with curMem=19674, maxMem=766075207 -2016-05-25 10:43:28 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 679.0 B, free 730.6 MB) -2016-05-25 10:43:28 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53032 (size: 679.0 B, free: 730.6 MB) -2016-05-25 10:43:28 INFO SparkContext:59 - Created broadcast 4 from broadcast at NaiveBayes.scala:89 -2016-05-25 10:43:28 INFO SparkContext:59 - Starting job: collect at BernoulliNaiveBayesLearning.scala:54 -2016-05-25 10:43:28 INFO DAGScheduler:59 - Got job 2 (collect at BernoulliNaiveBayesLearning.scala:54) with 2 output partitions -2016-05-25 10:43:28 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at BernoulliNaiveBayesLearning.scala:54) -2016-05-25 10:43:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:43:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:43:28 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[7] at mapPartitions at NaiveBayes.scala:90), which has no missing parents -2016-05-25 10:43:28 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=20353, maxMem=766075207 -2016-05-25 10:43:28 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-05-25 10:43:28 INFO MemoryStore:59 - ensureFreeSpace(1737) called with curMem=23313, maxMem=766075207 -2016-05-25 10:43:28 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 1737.0 B, free 730.6 MB) -2016-05-25 10:43:28 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53032 (size: 1737.0 B, free: 730.6 MB) -2016-05-25 10:43:28 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:43:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[7] at mapPartitions at NaiveBayes.scala:90) -2016-05-25 10:43:28 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 10:43:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2234 bytes) -2016-05-25 10:43:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2234 bytes) -2016-05-25 10:43:28 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-25 10:43:28 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-25 10:43:28 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 914 bytes result sent to driver -2016-05-25 10:43:28 ERROR Executor:96 - Exception in task 0.0 in stage 4.0 (TID 8) -org.apache.spark.SparkException: Bernoulli naive Bayes requires 0 or 1 feature values but found [2.0]. - at org.apache.spark.mllib.classification.NaiveBayesModel$$anonfun$bernoulliCalculation$1.apply$mcVID$sp(NaiveBayes.scala:148) - at org.apache.spark.mllib.linalg.DenseVector.foreachActive(Vectors.scala:573) - at org.apache.spark.mllib.classification.NaiveBayesModel.bernoulliCalculation(NaiveBayes.scala:146) - at org.apache.spark.mllib.classification.NaiveBayesModel.predict(NaiveBayes.scala:102) - at org.apache.spark.mllib.classification.NaiveBayesModel$$anonfun$predict$1$$anonfun$apply$3.apply(NaiveBayes.scala:92) - at org.apache.spark.mllib.classification.NaiveBayesModel$$anonfun$predict$1$$anonfun$apply$3.apply(NaiveBayes.scala:92) - at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48) - at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:103) - at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:47) - at scala.collection.TraversableOnce$class.to(TraversableOnce.scala:273) - at scala.collection.AbstractIterator.to(Iterator.scala:1157) - at scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:265) - at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1157) - at scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:252) - at scala.collection.AbstractIterator.toArray(Iterator.scala:1157) - at org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$12.apply(RDD.scala:909) - at org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$12.apply(RDD.scala:909) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-05-25 10:43:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 11 ms on localhost (1/2) -2016-05-25 10:43:28 WARN TaskSetManager:71 - Lost task 0.0 in stage 4.0 (TID 8, localhost): org.apache.spark.SparkException: Bernoulli naive Bayes requires 0 or 1 feature values but found [2.0]. - at org.apache.spark.mllib.classification.NaiveBayesModel$$anonfun$bernoulliCalculation$1.apply$mcVID$sp(NaiveBayes.scala:148) - at org.apache.spark.mllib.linalg.DenseVector.foreachActive(Vectors.scala:573) - at org.apache.spark.mllib.classification.NaiveBayesModel.bernoulliCalculation(NaiveBayes.scala:146) - at org.apache.spark.mllib.classification.NaiveBayesModel.predict(NaiveBayes.scala:102) - at org.apache.spark.mllib.classification.NaiveBayesModel$$anonfun$predict$1$$anonfun$apply$3.apply(NaiveBayes.scala:92) - at org.apache.spark.mllib.classification.NaiveBayesModel$$anonfun$predict$1$$anonfun$apply$3.apply(NaiveBayes.scala:92) - at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48) - at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:103) - at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:47) - at scala.collection.TraversableOnce$class.to(TraversableOnce.scala:273) - at scala.collection.AbstractIterator.to(Iterator.scala:1157) - at scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:265) - at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1157) - at scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:252) - at scala.collection.AbstractIterator.toArray(Iterator.scala:1157) - at org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$12.apply(RDD.scala:909) - at org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$12.apply(RDD.scala:909) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) - -2016-05-25 10:43:28 ERROR TaskSetManager:75 - Task 0 in stage 4.0 failed 1 times; aborting job -2016-05-25 10:43:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 10:43:28 INFO TaskSchedulerImpl:59 - Cancelling stage 4 -2016-05-25 10:43:28 INFO DAGScheduler:59 - ResultStage 4 (collect at BernoulliNaiveBayesLearning.scala:54) failed in 0.035 s -2016-05-25 10:43:28 INFO DAGScheduler:59 - Job 2 failed: collect at BernoulliNaiveBayesLearning.scala:54, took 0.071079 s -2016-05-25 10:43:28 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 10:43:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 10:43:28 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 10:43:28 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 10:43:28 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 10:43:28 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 10:43:28 INFO BlockManager:59 - BlockManager stopped -2016-05-25 10:43:28 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 10:43:28 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 10:43:28 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 10:43:28 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-55ca3292-2ba8-4cc4-91a8-f1cd45385fe1 -2016-05-25 10:43:39 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 10:43:40 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 10:43:40 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 10:43:40 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 10:43:40 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 10:43:41 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 10:43:41 INFO Remoting:74 - Starting remoting -2016-05-25 10:43:41 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53049] -2016-05-25 10:43:41 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53049. -2016-05-25 10:43:41 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 10:43:41 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 10:43:41 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-a120bc51-e5d9-49c2-a26d-8721b44c6022 -2016-05-25 10:43:41 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 10:43:41 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-43224fa8-a304-4610-b995-cb5ae69f6b92\httpd-a8caa3fb-34f9-433e-8aa7-b457080daaaf -2016-05-25 10:43:41 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 10:43:41 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:43:41 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53050 -2016-05-25 10:43:41 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53050. -2016-05-25 10:43:41 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 10:43:41 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:43:41 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 10:43:41 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 10:43:41 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 10:43:41 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 10:43:41 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 10:43:42 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53069. -2016-05-25 10:43:42 INFO NettyBlockTransferService:59 - Server created on 53069 -2016-05-25 10:43:42 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 10:43:42 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53069 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53069) -2016-05-25 10:43:42 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 10:43:42 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:43:42 INFO DAGScheduler:59 - Registering RDD 1 (map at NaiveBayes.scala:383) -2016-05-25 10:43:42 INFO DAGScheduler:59 - Got job 0 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:43:42 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at NaiveBayes.scala:401) -2016-05-25 10:43:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-05-25 10:43:42 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-05-25 10:43:42 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[1] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:43:43 INFO MemoryStore:59 - ensureFreeSpace(3288) called with curMem=0, maxMem=766075207 -2016-05-25 10:43:43 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-05-25 10:43:43 INFO MemoryStore:59 - ensureFreeSpace(1806) called with curMem=3288, maxMem=766075207 -2016-05-25 10:43:43 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1806.0 B, free 730.6 MB) -2016-05-25 10:43:43 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53069 (size: 1806.0 B, free: 730.6 MB) -2016-05-25 10:43:43 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:43:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[1] at map at NaiveBayes.scala:383) -2016-05-25 10:43:43 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 10:43:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2340 bytes) -2016-05-25 10:43:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2378 bytes) -2016-05-25 10:43:43 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 10:43:43 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 10:43:43 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 10:43:43 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 10:43:43 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1159 bytes result sent to driver -2016-05-25 10:43:43 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1159 bytes result sent to driver -2016-05-25 10:43:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 150 ms on localhost (1/2) -2016-05-25 10:43:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 120 ms on localhost (2/2) -2016-05-25 10:43:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 10:43:43 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at NaiveBayes.scala:383) finished in 0.166 s -2016-05-25 10:43:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 10:43:43 INFO DAGScheduler:59 - running: Set() -2016-05-25 10:43:43 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-05-25 10:43:43 INFO DAGScheduler:59 - failed: Set() -2016-05-25 10:43:43 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-05-25 10:43:43 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[2] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 10:43:43 INFO MemoryStore:59 - ensureFreeSpace(2928) called with curMem=5094, maxMem=766075207 -2016-05-25 10:43:43 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-05-25 10:43:43 INFO MemoryStore:59 - ensureFreeSpace(1559) called with curMem=8022, maxMem=766075207 -2016-05-25 10:43:43 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1559.0 B, free 730.6 MB) -2016-05-25 10:43:43 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53069 (size: 1559.0 B, free: 730.6 MB) -2016-05-25 10:43:43 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:43:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[2] at combineByKey at NaiveBayes.scala:383) -2016-05-25 10:43:43 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-25 10:43:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:43:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:43:43 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 10:43:43 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-25 10:43:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 10:43:43 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 10:43:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 10:43:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 10:43:43 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-05-25 10:43:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 35 ms on localhost (1/2) -2016-05-25 10:43:43 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1469 bytes result sent to driver -2016-05-25 10:43:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 43 ms on localhost (2/2) -2016-05-25 10:43:43 INFO DAGScheduler:59 - ResultStage 1 (collect at NaiveBayes.scala:401) finished in 0.044 s -2016-05-25 10:43:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 10:43:43 INFO DAGScheduler:59 - Job 0 finished: collect at NaiveBayes.scala:401, took 0.440626 s -2016-05-25 10:43:43 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:43:43 INFO DAGScheduler:59 - Registering RDD 4 (map at NaiveBayes.scala:383) -2016-05-25 10:43:43 INFO DAGScheduler:59 - Got job 1 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:43:43 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at NaiveBayes.scala:401) -2016-05-25 10:43:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 2) -2016-05-25 10:43:43 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 2) -2016-05-25 10:43:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[4] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:43:43 INFO MemoryStore:59 - ensureFreeSpace(3288) called with curMem=9581, maxMem=766075207 -2016-05-25 10:43:43 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-05-25 10:43:43 INFO MemoryStore:59 - ensureFreeSpace(1804) called with curMem=12869, maxMem=766075207 -2016-05-25 10:43:43 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1804.0 B, free 730.6 MB) -2016-05-25 10:43:43 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53069 (size: 1804.0 B, free: 730.6 MB) -2016-05-25 10:43:43 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:43:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[4] at map at NaiveBayes.scala:383) -2016-05-25 10:43:43 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 10:43:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2416 bytes) -2016-05-25 10:43:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2454 bytes) -2016-05-25 10:43:43 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-25 10:43:43 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-25 10:43:43 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1159 bytes result sent to driver -2016-05-25 10:43:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 19 ms on localhost (1/2) -2016-05-25 10:43:43 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1159 bytes result sent to driver -2016-05-25 10:43:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 22 ms on localhost (2/2) -2016-05-25 10:43:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 10:43:43 INFO DAGScheduler:59 - ShuffleMapStage 2 (map at NaiveBayes.scala:383) finished in 0.025 s -2016-05-25 10:43:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 10:43:43 INFO DAGScheduler:59 - running: Set() -2016-05-25 10:43:43 INFO DAGScheduler:59 - waiting: Set(ResultStage 3) -2016-05-25 10:43:43 INFO DAGScheduler:59 - failed: Set() -2016-05-25 10:43:43 INFO DAGScheduler:59 - Missing parents for ResultStage 3: List() -2016-05-25 10:43:43 INFO DAGScheduler:59 - Submitting ResultStage 3 (ShuffledRDD[5] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 10:43:43 INFO MemoryStore:59 - ensureFreeSpace(2928) called with curMem=14673, maxMem=766075207 -2016-05-25 10:43:43 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-05-25 10:43:43 INFO MemoryStore:59 - ensureFreeSpace(1553) called with curMem=17601, maxMem=766075207 -2016-05-25 10:43:43 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1553.0 B, free 730.6 MB) -2016-05-25 10:43:43 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53069 (size: 1553.0 B, free: 730.6 MB) -2016-05-25 10:43:43 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:43:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (ShuffledRDD[5] at combineByKey at NaiveBayes.scala:383) -2016-05-25 10:43:43 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 10:43:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:43:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:43:43 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-25 10:43:43 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-25 10:43:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 10:43:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 10:43:43 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 10:43:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 10:43:43 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 1161 bytes result sent to driver -2016-05-25 10:43:43 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 1533 bytes result sent to driver -2016-05-25 10:43:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 13 ms on localhost (1/2) -2016-05-25 10:43:43 INFO DAGScheduler:59 - ResultStage 3 (collect at NaiveBayes.scala:401) finished in 0.017 s -2016-05-25 10:43:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 16 ms on localhost (2/2) -2016-05-25 10:43:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 10:43:43 INFO DAGScheduler:59 - Job 1 finished: collect at NaiveBayes.scala:401, took 0.072591 s -2016-05-25 10:43:43 INFO MemoryStore:59 - ensureFreeSpace(520) called with curMem=19154, maxMem=766075207 -2016-05-25 10:43:43 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 520.0 B, free 730.6 MB) -2016-05-25 10:43:43 INFO MemoryStore:59 - ensureFreeSpace(679) called with curMem=19674, maxMem=766075207 -2016-05-25 10:43:43 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 679.0 B, free 730.6 MB) -2016-05-25 10:43:43 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53069 (size: 679.0 B, free: 730.6 MB) -2016-05-25 10:43:43 INFO SparkContext:59 - Created broadcast 4 from broadcast at NaiveBayes.scala:89 -2016-05-25 10:43:43 INFO SparkContext:59 - Starting job: collect at BernoulliNaiveBayesLearning.scala:54 -2016-05-25 10:43:43 INFO DAGScheduler:59 - Got job 2 (collect at BernoulliNaiveBayesLearning.scala:54) with 2 output partitions -2016-05-25 10:43:43 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at BernoulliNaiveBayesLearning.scala:54) -2016-05-25 10:43:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:43:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:43:43 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[7] at mapPartitions at NaiveBayes.scala:90), which has no missing parents -2016-05-25 10:43:43 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=20353, maxMem=766075207 -2016-05-25 10:43:43 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-05-25 10:43:43 INFO MemoryStore:59 - ensureFreeSpace(1737) called with curMem=23313, maxMem=766075207 -2016-05-25 10:43:43 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 1737.0 B, free 730.6 MB) -2016-05-25 10:43:43 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53069 (size: 1737.0 B, free: 730.6 MB) -2016-05-25 10:43:43 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:43:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[7] at mapPartitions at NaiveBayes.scala:90) -2016-05-25 10:43:43 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 10:43:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2210 bytes) -2016-05-25 10:43:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2234 bytes) -2016-05-25 10:43:43 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-25 10:43:43 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-25 10:43:43 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 914 bytes result sent to driver -2016-05-25 10:43:43 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 906 bytes result sent to driver -2016-05-25 10:43:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 21 ms on localhost (1/2) -2016-05-25 10:43:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 24 ms on localhost (2/2) -2016-05-25 10:43:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 10:43:43 INFO DAGScheduler:59 - ResultStage 4 (collect at BernoulliNaiveBayesLearning.scala:54) finished in 0.030 s -2016-05-25 10:43:43 INFO DAGScheduler:59 - Job 2 finished: collect at BernoulliNaiveBayesLearning.scala:54, took 0.072339 s -2016-05-25 10:43:43 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53069 in memory (size: 1553.0 B, free: 730.6 MB) -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 10:43:43 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53069 in memory (size: 1804.0 B, free: 730.6 MB) -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 10:43:43 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 10:43:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 10:43:43 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53069 in memory (size: 1559.0 B, free: 730.6 MB) -2016-05-25 10:43:43 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:53069 in memory (size: 1806.0 B, free: 730.6 MB) -2016-05-25 10:43:43 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 10:43:43 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 10:43:43 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 10:43:43 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 10:43:43 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 10:43:43 INFO BlockManager:59 - BlockManager stopped -2016-05-25 10:43:43 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 10:43:43 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 10:43:43 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 10:43:43 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 10:43:43 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-43224fa8-a304-4610-b995-cb5ae69f6b92 -2016-05-25 10:43:43 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 10:43:43 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-25 10:46:58 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 10:46:59 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 10:46:59 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 10:46:59 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 10:46:59 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 10:47:00 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 10:47:00 INFO Remoting:74 - Starting remoting -2016-05-25 10:47:00 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53092] -2016-05-25 10:47:00 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53092. -2016-05-25 10:47:00 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 10:47:00 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 10:47:00 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-abe6e32b-039d-46ba-bad4-c5ba8ea0d41f -2016-05-25 10:47:00 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 10:47:00 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-62fa50ca-f73f-487c-a387-43567efae650\httpd-9a8afaf6-d85c-46a5-90b5-2321a813925a -2016-05-25 10:47:00 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 10:47:00 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:47:00 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53093 -2016-05-25 10:47:00 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53093. -2016-05-25 10:47:00 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 10:47:00 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:47:01 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 10:47:01 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 10:47:01 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 10:47:01 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 10:47:01 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 10:47:01 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53112. -2016-05-25 10:47:01 INFO NettyBlockTransferService:59 - Server created on 53112 -2016-05-25 10:47:01 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 10:47:01 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53112 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53112) -2016-05-25 10:47:01 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 10:47:02 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:47:02 INFO DAGScheduler:59 - Registering RDD 1 (map at NaiveBayes.scala:383) -2016-05-25 10:47:02 INFO DAGScheduler:59 - Got job 0 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:47:02 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at NaiveBayes.scala:401) -2016-05-25 10:47:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-05-25 10:47:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-05-25 10:47:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[1] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:47:02 INFO MemoryStore:59 - ensureFreeSpace(3288) called with curMem=0, maxMem=766075207 -2016-05-25 10:47:02 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-05-25 10:47:02 INFO MemoryStore:59 - ensureFreeSpace(1806) called with curMem=3288, maxMem=766075207 -2016-05-25 10:47:02 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1806.0 B, free 730.6 MB) -2016-05-25 10:47:02 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53112 (size: 1806.0 B, free: 730.6 MB) -2016-05-25 10:47:02 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:47:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[1] at map at NaiveBayes.scala:383) -2016-05-25 10:47:02 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 10:47:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2340 bytes) -2016-05-25 10:47:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2378 bytes) -2016-05-25 10:47:02 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 10:47:02 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 10:47:02 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 10:47:02 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 10:47:02 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1159 bytes result sent to driver -2016-05-25 10:47:02 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1159 bytes result sent to driver -2016-05-25 10:47:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 140 ms on localhost (1/2) -2016-05-25 10:47:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 116 ms on localhost (2/2) -2016-05-25 10:47:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 10:47:02 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at NaiveBayes.scala:383) finished in 0.156 s -2016-05-25 10:47:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 10:47:02 INFO DAGScheduler:59 - running: Set() -2016-05-25 10:47:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-05-25 10:47:02 INFO DAGScheduler:59 - failed: Set() -2016-05-25 10:47:02 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-05-25 10:47:02 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[2] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 10:47:02 INFO MemoryStore:59 - ensureFreeSpace(2928) called with curMem=5094, maxMem=766075207 -2016-05-25 10:47:02 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-05-25 10:47:02 INFO MemoryStore:59 - ensureFreeSpace(1559) called with curMem=8022, maxMem=766075207 -2016-05-25 10:47:02 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1559.0 B, free 730.6 MB) -2016-05-25 10:47:02 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53112 (size: 1559.0 B, free: 730.6 MB) -2016-05-25 10:47:02 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:47:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[2] at combineByKey at NaiveBayes.scala:383) -2016-05-25 10:47:02 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-25 10:47:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:47:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:47:02 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-25 10:47:02 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 10:47:02 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 10:47:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 10:47:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 10:47:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 10:47:02 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-05-25 10:47:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 35 ms on localhost (1/2) -2016-05-25 10:47:02 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1469 bytes result sent to driver -2016-05-25 10:47:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 45 ms on localhost (2/2) -2016-05-25 10:47:02 INFO DAGScheduler:59 - ResultStage 1 (collect at NaiveBayes.scala:401) finished in 0.045 s -2016-05-25 10:47:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 10:47:02 INFO DAGScheduler:59 - Job 0 finished: collect at NaiveBayes.scala:401, took 0.456522 s -2016-05-25 10:47:02 INFO SparkContext:59 - Starting job: foreach at BernoulliNaiveBayesLearning.scala:36 -2016-05-25 10:47:02 INFO DAGScheduler:59 - Got job 1 (foreach at BernoulliNaiveBayesLearning.scala:36) with 2 output partitions -2016-05-25 10:47:02 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at BernoulliNaiveBayesLearning.scala:36) -2016-05-25 10:47:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:47:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:47:02 INFO DAGScheduler:59 - Submitting ResultStage 2 (ParallelCollectionRDD[3] at makeRDD at BernoulliNaiveBayesLearning.scala:36), which has no missing parents -2016-05-25 10:47:02 INFO MemoryStore:59 - ensureFreeSpace(1296) called with curMem=9581, maxMem=766075207 -2016-05-25 10:47:02 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 1296.0 B, free 730.6 MB) -2016-05-25 10:47:02 INFO MemoryStore:59 - ensureFreeSpace(914) called with curMem=10877, maxMem=766075207 -2016-05-25 10:47:02 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 914.0 B, free 730.6 MB) -2016-05-25 10:47:02 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53112 (size: 914.0 B, free: 730.6 MB) -2016-05-25 10:47:02 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:47:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (ParallelCollectionRDD[3] at makeRDD at BernoulliNaiveBayesLearning.scala:36) -2016-05-25 10:47:02 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 10:47:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2351 bytes) -2016-05-25 10:47:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2389 bytes) -2016-05-25 10:47:02 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-25 10:47:02 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-25 10:47:02 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 915 bytes result sent to driver -2016-05-25 10:47:02 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 915 bytes result sent to driver -2016-05-25 10:47:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 18 ms on localhost (1/2) -2016-05-25 10:47:02 INFO DAGScheduler:59 - ResultStage 2 (foreach at BernoulliNaiveBayesLearning.scala:36) finished in 0.021 s -2016-05-25 10:47:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 18 ms on localhost (2/2) -2016-05-25 10:47:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 10:47:02 INFO DAGScheduler:59 - Job 1 finished: foreach at BernoulliNaiveBayesLearning.scala:36, took 0.033939 s -2016-05-25 10:47:02 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:47:02 INFO DAGScheduler:59 - Registering RDD 5 (map at NaiveBayes.scala:383) -2016-05-25 10:47:02 INFO DAGScheduler:59 - Got job 2 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:47:02 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at NaiveBayes.scala:401) -2016-05-25 10:47:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3) -2016-05-25 10:47:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 3) -2016-05-25 10:47:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:47:02 INFO MemoryStore:59 - ensureFreeSpace(3288) called with curMem=11791, maxMem=766075207 -2016-05-25 10:47:02 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-05-25 10:47:02 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=15079, maxMem=766075207 -2016-05-25 10:47:02 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.6 MB) -2016-05-25 10:47:02 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53112 (size: 1803.0 B, free: 730.6 MB) -2016-05-25 10:47:02 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:47:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383) -2016-05-25 10:47:02 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 10:47:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2416 bytes) -2016-05-25 10:47:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2454 bytes) -2016-05-25 10:47:02 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-25 10:47:02 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-25 10:47:02 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 1159 bytes result sent to driver -2016-05-25 10:47:02 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 1159 bytes result sent to driver -2016-05-25 10:47:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 19 ms on localhost (1/2) -2016-05-25 10:47:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 21 ms on localhost (2/2) -2016-05-25 10:47:02 INFO DAGScheduler:59 - ShuffleMapStage 3 (map at NaiveBayes.scala:383) finished in 0.021 s -2016-05-25 10:47:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 10:47:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 10:47:02 INFO DAGScheduler:59 - running: Set() -2016-05-25 10:47:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 4) -2016-05-25 10:47:02 INFO DAGScheduler:59 - failed: Set() -2016-05-25 10:47:02 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List() -2016-05-25 10:47:02 INFO DAGScheduler:59 - Submitting ResultStage 4 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 10:47:02 INFO MemoryStore:59 - ensureFreeSpace(2928) called with curMem=16882, maxMem=766075207 -2016-05-25 10:47:02 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-05-25 10:47:02 INFO MemoryStore:59 - ensureFreeSpace(1554) called with curMem=19810, maxMem=766075207 -2016-05-25 10:47:02 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 1554.0 B, free 730.6 MB) -2016-05-25 10:47:02 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53112 (size: 1554.0 B, free: 730.6 MB) -2016-05-25 10:47:02 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:47:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383) -2016-05-25 10:47:02 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 10:47:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:47:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:47:02 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-25 10:47:02 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-25 10:47:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 10:47:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 10:47:02 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 10:47:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 10:47:02 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 1161 bytes result sent to driver -2016-05-25 10:47:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 11 ms on localhost (1/2) -2016-05-25 10:47:02 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 1533 bytes result sent to driver -2016-05-25 10:47:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 15 ms on localhost (2/2) -2016-05-25 10:47:02 INFO DAGScheduler:59 - ResultStage 4 (collect at NaiveBayes.scala:401) finished in 0.015 s -2016-05-25 10:47:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 10:47:02 INFO DAGScheduler:59 - Job 2 finished: collect at NaiveBayes.scala:401, took 0.059740 s -2016-05-25 10:47:02 INFO MemoryStore:59 - ensureFreeSpace(520) called with curMem=21364, maxMem=766075207 -2016-05-25 10:47:02 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 520.0 B, free 730.6 MB) -2016-05-25 10:47:02 INFO MemoryStore:59 - ensureFreeSpace(679) called with curMem=21884, maxMem=766075207 -2016-05-25 10:47:02 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 679.0 B, free 730.6 MB) -2016-05-25 10:47:02 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53112 (size: 679.0 B, free: 730.6 MB) -2016-05-25 10:47:02 INFO SparkContext:59 - Created broadcast 5 from broadcast at NaiveBayes.scala:89 -2016-05-25 10:47:02 INFO SparkContext:59 - Starting job: collect at BernoulliNaiveBayesLearning.scala:56 -2016-05-25 10:47:02 INFO DAGScheduler:59 - Got job 3 (collect at BernoulliNaiveBayesLearning.scala:56) with 2 output partitions -2016-05-25 10:47:02 INFO DAGScheduler:59 - Final stage: ResultStage 5(collect at BernoulliNaiveBayesLearning.scala:56) -2016-05-25 10:47:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:47:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:47:02 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[8] at mapPartitions at NaiveBayes.scala:90), which has no missing parents -2016-05-25 10:47:02 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=22563, maxMem=766075207 -2016-05-25 10:47:02 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-05-25 10:47:02 INFO MemoryStore:59 - ensureFreeSpace(1737) called with curMem=25523, maxMem=766075207 -2016-05-25 10:47:02 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1737.0 B, free 730.6 MB) -2016-05-25 10:47:02 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53112 (size: 1737.0 B, free: 730.6 MB) -2016-05-25 10:47:02 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:47:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[8] at mapPartitions at NaiveBayes.scala:90) -2016-05-25 10:47:02 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 10:47:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2210 bytes) -2016-05-25 10:47:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2234 bytes) -2016-05-25 10:47:02 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-05-25 10:47:02 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-05-25 10:47:02 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 914 bytes result sent to driver -2016-05-25 10:47:02 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 906 bytes result sent to driver -2016-05-25 10:47:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 11 ms on localhost (1/2) -2016-05-25 10:47:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 13 ms on localhost (2/2) -2016-05-25 10:47:02 INFO DAGScheduler:59 - ResultStage 5 (collect at BernoulliNaiveBayesLearning.scala:56) finished in 0.013 s -2016-05-25 10:47:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 10:47:02 INFO DAGScheduler:59 - Job 3 finished: collect at BernoulliNaiveBayesLearning.scala:56, took 0.026152 s -2016-05-25 10:47:02 INFO SparkContext:59 - Starting job: foreach at BernoulliNaiveBayesLearning.scala:59 -2016-05-25 10:47:02 INFO DAGScheduler:59 - Got job 4 (foreach at BernoulliNaiveBayesLearning.scala:59) with 2 output partitions -2016-05-25 10:47:02 INFO DAGScheduler:59 - Final stage: ResultStage 6(foreach at BernoulliNaiveBayesLearning.scala:59) -2016-05-25 10:47:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:47:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:47:02 INFO DAGScheduler:59 - Submitting ResultStage 6 (ParallelCollectionRDD[9] at makeRDD at BernoulliNaiveBayesLearning.scala:59), which has no missing parents -2016-05-25 10:47:02 INFO MemoryStore:59 - ensureFreeSpace(1296) called with curMem=27260, maxMem=766075207 -2016-05-25 10:47:02 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 1296.0 B, free 730.6 MB) -2016-05-25 10:47:02 INFO MemoryStore:59 - ensureFreeSpace(914) called with curMem=28556, maxMem=766075207 -2016-05-25 10:47:02 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 914.0 B, free 730.6 MB) -2016-05-25 10:47:02 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53112 (size: 914.0 B, free: 730.6 MB) -2016-05-25 10:47:02 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:47:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (ParallelCollectionRDD[9] at makeRDD at BernoulliNaiveBayesLearning.scala:59) -2016-05-25 10:47:02 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 10:47:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2427 bytes) -2016-05-25 10:47:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2465 bytes) -2016-05-25 10:47:02 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-05-25 10:47:02 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 13) -2016-05-25 10:47:02 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 915 bytes result sent to driver -2016-05-25 10:47:02 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 13). 915 bytes result sent to driver -2016-05-25 10:47:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 21 ms on localhost (1/2) -2016-05-25 10:47:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 13) in 19 ms on localhost (2/2) -2016-05-25 10:47:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 10:47:02 INFO DAGScheduler:59 - ResultStage 6 (foreach at BernoulliNaiveBayesLearning.scala:59) finished in 0.023 s -2016-05-25 10:47:02 INFO DAGScheduler:59 - Job 4 finished: foreach at BernoulliNaiveBayesLearning.scala:59, took 0.048733 s -2016-05-25 10:47:02 INFO SparkContext:59 - Starting job: foreach at BernoulliNaiveBayesLearning.scala:61 -2016-05-25 10:47:02 INFO DAGScheduler:59 - Got job 5 (foreach at BernoulliNaiveBayesLearning.scala:61) with 2 output partitions -2016-05-25 10:47:02 INFO DAGScheduler:59 - Final stage: ResultStage 7(foreach at BernoulliNaiveBayesLearning.scala:61) -2016-05-25 10:47:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:47:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:47:02 INFO DAGScheduler:59 - Submitting ResultStage 7 (ParallelCollectionRDD[10] at makeRDD at BernoulliNaiveBayesLearning.scala:61), which has no missing parents -2016-05-25 10:47:02 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=29470, maxMem=766075207 -2016-05-25 10:47:02 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 1224.0 B, free 730.6 MB) -2016-05-25 10:47:02 INFO MemoryStore:59 - ensureFreeSpace(859) called with curMem=30694, maxMem=766075207 -2016-05-25 10:47:02 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 859.0 B, free 730.6 MB) -2016-05-25 10:47:02 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53112 (size: 859.0 B, free: 730.6 MB) -2016-05-25 10:47:02 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:47:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (ParallelCollectionRDD[10] at makeRDD at BernoulliNaiveBayesLearning.scala:61) -2016-05-25 10:47:02 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 10:47:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2210 bytes) -2016-05-25 10:47:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 15, localhost, PROCESS_LOCAL, 2234 bytes) -2016-05-25 10:47:02 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 15) -2016-05-25 10:47:02 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 15). 915 bytes result sent to driver -2016-05-25 10:47:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 15) in 10 ms on localhost (1/2) -2016-05-25 10:47:02 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 14) -2016-05-25 10:47:02 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 14). 915 bytes result sent to driver -2016-05-25 10:47:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 14) in 30 ms on localhost (2/2) -2016-05-25 10:47:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 10:47:02 INFO DAGScheduler:59 - ResultStage 7 (foreach at BernoulliNaiveBayesLearning.scala:61) finished in 0.033 s -2016-05-25 10:47:02 INFO DAGScheduler:59 - Job 5 finished: foreach at BernoulliNaiveBayesLearning.scala:61, took 0.051676 s -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 10:47:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 10:47:03 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 10:47:03 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 10:47:03 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 10:47:03 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 10:47:03 INFO BlockManager:59 - BlockManager stopped -2016-05-25 10:47:03 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 10:47:03 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 10:47:03 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 10:47:03 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 10:47:03 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-62fa50ca-f73f-487c-a387-43567efae650 -2016-05-25 10:49:06 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 10:49:07 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 10:49:07 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 10:49:07 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 10:49:07 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 10:49:08 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 10:49:08 INFO Remoting:74 - Starting remoting -2016-05-25 10:49:08 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53130] -2016-05-25 10:49:08 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53130. -2016-05-25 10:49:08 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 10:49:08 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 10:49:08 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-2bb426d1-d688-4ca5-80a2-2953199e2952 -2016-05-25 10:49:08 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 10:49:08 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-d4484bbe-b09a-4338-a080-4a33af105e32\httpd-4bbd9907-23a4-44b1-b85e-8abb663a859b -2016-05-25 10:49:08 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 10:49:09 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:49:09 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53131 -2016-05-25 10:49:09 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53131. -2016-05-25 10:49:09 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 10:49:09 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:49:09 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 10:49:09 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 10:49:09 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 10:49:09 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 10:49:09 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 10:49:09 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53150. -2016-05-25 10:49:09 INFO NettyBlockTransferService:59 - Server created on 53150 -2016-05-25 10:49:09 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 10:49:09 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53150 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53150) -2016-05-25 10:49:09 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 10:49:10 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:49:10 INFO DAGScheduler:59 - Registering RDD 1 (map at NaiveBayes.scala:383) -2016-05-25 10:49:10 INFO DAGScheduler:59 - Got job 0 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:49:10 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at NaiveBayes.scala:401) -2016-05-25 10:49:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-05-25 10:49:10 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-05-25 10:49:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[1] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:49:10 INFO MemoryStore:59 - ensureFreeSpace(3288) called with curMem=0, maxMem=766075207 -2016-05-25 10:49:10 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-05-25 10:49:10 INFO MemoryStore:59 - ensureFreeSpace(1806) called with curMem=3288, maxMem=766075207 -2016-05-25 10:49:10 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1806.0 B, free 730.6 MB) -2016-05-25 10:49:10 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53150 (size: 1806.0 B, free: 730.6 MB) -2016-05-25 10:49:10 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:49:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[1] at map at NaiveBayes.scala:383) -2016-05-25 10:49:10 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 10:49:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2340 bytes) -2016-05-25 10:49:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2378 bytes) -2016-05-25 10:49:10 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 10:49:10 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 10:49:10 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 10:49:10 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 10:49:10 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1159 bytes result sent to driver -2016-05-25 10:49:10 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1159 bytes result sent to driver -2016-05-25 10:49:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 136 ms on localhost (1/2) -2016-05-25 10:49:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 112 ms on localhost (2/2) -2016-05-25 10:49:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 10:49:10 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at NaiveBayes.scala:383) finished in 0.152 s -2016-05-25 10:49:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 10:49:10 INFO DAGScheduler:59 - running: Set() -2016-05-25 10:49:10 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-05-25 10:49:10 INFO DAGScheduler:59 - failed: Set() -2016-05-25 10:49:10 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-05-25 10:49:10 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[2] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 10:49:10 INFO MemoryStore:59 - ensureFreeSpace(2928) called with curMem=5094, maxMem=766075207 -2016-05-25 10:49:10 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-05-25 10:49:10 INFO MemoryStore:59 - ensureFreeSpace(1559) called with curMem=8022, maxMem=766075207 -2016-05-25 10:49:10 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1559.0 B, free 730.6 MB) -2016-05-25 10:49:10 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53150 (size: 1559.0 B, free: 730.6 MB) -2016-05-25 10:49:10 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:49:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[2] at combineByKey at NaiveBayes.scala:383) -2016-05-25 10:49:10 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-25 10:49:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:49:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:49:10 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-25 10:49:10 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 10:49:10 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 10:49:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 10:49:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 10:49:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 10:49:10 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-05-25 10:49:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 33 ms on localhost (1/2) -2016-05-25 10:49:10 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1469 bytes result sent to driver -2016-05-25 10:49:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 40 ms on localhost (2/2) -2016-05-25 10:49:10 INFO DAGScheduler:59 - ResultStage 1 (collect at NaiveBayes.scala:401) finished in 0.040 s -2016-05-25 10:49:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 10:49:10 INFO DAGScheduler:59 - Job 0 finished: collect at NaiveBayes.scala:401, took 0.445369 s -2016-05-25 10:49:10 INFO SparkContext:59 - Starting job: foreach at BernoulliNaiveBayesLearning.scala:37 -2016-05-25 10:49:10 INFO DAGScheduler:59 - Got job 1 (foreach at BernoulliNaiveBayesLearning.scala:37) with 2 output partitions -2016-05-25 10:49:10 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at BernoulliNaiveBayesLearning.scala:37) -2016-05-25 10:49:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:49:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:49:10 INFO DAGScheduler:59 - Submitting ResultStage 2 (ParallelCollectionRDD[3] at makeRDD at BernoulliNaiveBayesLearning.scala:37), which has no missing parents -2016-05-25 10:49:10 INFO MemoryStore:59 - ensureFreeSpace(1296) called with curMem=9581, maxMem=766075207 -2016-05-25 10:49:10 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 1296.0 B, free 730.6 MB) -2016-05-25 10:49:10 INFO MemoryStore:59 - ensureFreeSpace(914) called with curMem=10877, maxMem=766075207 -2016-05-25 10:49:10 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 914.0 B, free 730.6 MB) -2016-05-25 10:49:10 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53150 (size: 914.0 B, free: 730.6 MB) -2016-05-25 10:49:10 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:49:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (ParallelCollectionRDD[3] at makeRDD at BernoulliNaiveBayesLearning.scala:37) -2016-05-25 10:49:10 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 10:49:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2351 bytes) -2016-05-25 10:49:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2389 bytes) -2016-05-25 10:49:10 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-25 10:49:10 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 915 bytes result sent to driver -2016-05-25 10:49:10 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-25 10:49:10 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 915 bytes result sent to driver -2016-05-25 10:49:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 14 ms on localhost (1/2) -2016-05-25 10:49:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 16 ms on localhost (2/2) -2016-05-25 10:49:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 10:49:10 INFO DAGScheduler:59 - ResultStage 2 (foreach at BernoulliNaiveBayesLearning.scala:37) finished in 0.017 s -2016-05-25 10:49:10 INFO DAGScheduler:59 - Job 1 finished: foreach at BernoulliNaiveBayesLearning.scala:37, took 0.029219 s -2016-05-25 10:49:10 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:49:10 INFO DAGScheduler:59 - Registering RDD 5 (map at NaiveBayes.scala:383) -2016-05-25 10:49:10 INFO DAGScheduler:59 - Got job 2 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:49:10 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at NaiveBayes.scala:401) -2016-05-25 10:49:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3) -2016-05-25 10:49:10 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 3) -2016-05-25 10:49:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:49:10 INFO MemoryStore:59 - ensureFreeSpace(3288) called with curMem=11791, maxMem=766075207 -2016-05-25 10:49:10 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-05-25 10:49:10 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=15079, maxMem=766075207 -2016-05-25 10:49:10 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.6 MB) -2016-05-25 10:49:10 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53150 (size: 1803.0 B, free: 730.6 MB) -2016-05-25 10:49:10 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:49:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383) -2016-05-25 10:49:10 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 10:49:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2416 bytes) -2016-05-25 10:49:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2454 bytes) -2016-05-25 10:49:10 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-25 10:49:10 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-25 10:49:10 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 1159 bytes result sent to driver -2016-05-25 10:49:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 34 ms on localhost (1/2) -2016-05-25 10:49:10 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 1159 bytes result sent to driver -2016-05-25 10:49:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 33 ms on localhost (2/2) -2016-05-25 10:49:10 INFO DAGScheduler:59 - ShuffleMapStage 3 (map at NaiveBayes.scala:383) finished in 0.034 s -2016-05-25 10:49:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 10:49:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 10:49:10 INFO DAGScheduler:59 - running: Set() -2016-05-25 10:49:10 INFO DAGScheduler:59 - waiting: Set(ResultStage 4) -2016-05-25 10:49:10 INFO DAGScheduler:59 - failed: Set() -2016-05-25 10:49:10 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List() -2016-05-25 10:49:10 INFO DAGScheduler:59 - Submitting ResultStage 4 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 10:49:10 INFO MemoryStore:59 - ensureFreeSpace(2928) called with curMem=16882, maxMem=766075207 -2016-05-25 10:49:10 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-05-25 10:49:10 INFO MemoryStore:59 - ensureFreeSpace(1554) called with curMem=19810, maxMem=766075207 -2016-05-25 10:49:10 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 1554.0 B, free 730.6 MB) -2016-05-25 10:49:10 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53150 (size: 1554.0 B, free: 730.6 MB) -2016-05-25 10:49:10 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:49:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383) -2016-05-25 10:49:10 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 10:49:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:49:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:49:10 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-25 10:49:10 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-25 10:49:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 10:49:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 10:49:10 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 1533 bytes result sent to driver -2016-05-25 10:49:10 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 10:49:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 10:49:10 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 1161 bytes result sent to driver -2016-05-25 10:49:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 17 ms on localhost (1/2) -2016-05-25 10:49:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 19 ms on localhost (2/2) -2016-05-25 10:49:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 10:49:10 INFO DAGScheduler:59 - ResultStage 4 (collect at NaiveBayes.scala:401) finished in 0.021 s -2016-05-25 10:49:10 INFO DAGScheduler:59 - Job 2 finished: collect at NaiveBayes.scala:401, took 0.092404 s -2016-05-25 10:49:10 INFO MemoryStore:59 - ensureFreeSpace(520) called with curMem=21364, maxMem=766075207 -2016-05-25 10:49:10 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 520.0 B, free 730.6 MB) -2016-05-25 10:49:10 INFO MemoryStore:59 - ensureFreeSpace(679) called with curMem=21884, maxMem=766075207 -2016-05-25 10:49:10 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 679.0 B, free 730.6 MB) -2016-05-25 10:49:10 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53150 (size: 679.0 B, free: 730.6 MB) -2016-05-25 10:49:10 INFO SparkContext:59 - Created broadcast 5 from broadcast at NaiveBayes.scala:89 -2016-05-25 10:49:10 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53150 in memory (size: 1554.0 B, free: 730.6 MB) -2016-05-25 10:49:10 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 10:49:11 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53150 in memory (size: 1803.0 B, free: 730.6 MB) -2016-05-25 10:49:11 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 10:49:11 INFO SparkContext:59 - Starting job: collect at BernoulliNaiveBayesLearning.scala:57 -2016-05-25 10:49:11 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-25 10:49:11 INFO DAGScheduler:59 - Got job 3 (collect at BernoulliNaiveBayesLearning.scala:57) with 2 output partitions -2016-05-25 10:49:11 INFO DAGScheduler:59 - Final stage: ResultStage 5(collect at BernoulliNaiveBayesLearning.scala:57) -2016-05-25 10:49:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:49:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:49:11 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[8] at mapPartitions at NaiveBayes.scala:90), which has no missing parents -2016-05-25 10:49:11 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=10780, maxMem=766075207 -2016-05-25 10:49:11 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-05-25 10:49:11 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53150 in memory (size: 914.0 B, free: 730.6 MB) -2016-05-25 10:49:11 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 10:49:11 INFO MemoryStore:59 - ensureFreeSpace(1737) called with curMem=13740, maxMem=766075207 -2016-05-25 10:49:11 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1737.0 B, free 730.6 MB) -2016-05-25 10:49:11 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53150 (size: 1737.0 B, free: 730.6 MB) -2016-05-25 10:49:11 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53150 in memory (size: 1559.0 B, free: 730.6 MB) -2016-05-25 10:49:11 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:49:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[8] at mapPartitions at NaiveBayes.scala:90) -2016-05-25 10:49:11 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 10:49:11 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 10:49:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2210 bytes) -2016-05-25 10:49:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2234 bytes) -2016-05-25 10:49:11 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-05-25 10:49:11 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-05-25 10:49:11 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:53150 in memory (size: 1806.0 B, free: 730.6 MB) -2016-05-25 10:49:11 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 10:49:11 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 10:49:11 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 906 bytes result sent to driver -2016-05-25 10:49:11 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 914 bytes result sent to driver -2016-05-25 10:49:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 20 ms on localhost (1/2) -2016-05-25 10:49:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 19 ms on localhost (2/2) -2016-05-25 10:49:11 INFO DAGScheduler:59 - ResultStage 5 (collect at BernoulliNaiveBayesLearning.scala:57) finished in 0.022 s -2016-05-25 10:49:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 10:49:11 INFO DAGScheduler:59 - Job 3 finished: collect at BernoulliNaiveBayesLearning.scala:57, took 0.042382 s -2016-05-25 10:49:11 INFO SparkContext:59 - Starting job: foreach at BernoulliNaiveBayesLearning.scala:60 -2016-05-25 10:49:11 INFO DAGScheduler:59 - Got job 4 (foreach at BernoulliNaiveBayesLearning.scala:60) with 2 output partitions -2016-05-25 10:49:11 INFO DAGScheduler:59 - Final stage: ResultStage 6(foreach at BernoulliNaiveBayesLearning.scala:60) -2016-05-25 10:49:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:49:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:49:11 INFO DAGScheduler:59 - Submitting ResultStage 6 (ParallelCollectionRDD[9] at makeRDD at BernoulliNaiveBayesLearning.scala:60), which has no missing parents -2016-05-25 10:49:11 INFO MemoryStore:59 - ensureFreeSpace(1296) called with curMem=5896, maxMem=766075207 -2016-05-25 10:49:11 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 1296.0 B, free 730.6 MB) -2016-05-25 10:49:11 INFO MemoryStore:59 - ensureFreeSpace(914) called with curMem=7192, maxMem=766075207 -2016-05-25 10:49:11 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 914.0 B, free 730.6 MB) -2016-05-25 10:49:11 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53150 (size: 914.0 B, free: 730.6 MB) -2016-05-25 10:49:11 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:49:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (ParallelCollectionRDD[9] at makeRDD at BernoulliNaiveBayesLearning.scala:60) -2016-05-25 10:49:11 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 10:49:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2427 bytes) -2016-05-25 10:49:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2465 bytes) -2016-05-25 10:49:11 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-05-25 10:49:11 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 13) -2016-05-25 10:49:11 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 915 bytes result sent to driver -2016-05-25 10:49:11 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 13). 915 bytes result sent to driver -2016-05-25 10:49:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 10 ms on localhost (1/2) -2016-05-25 10:49:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 13) in 10 ms on localhost (2/2) -2016-05-25 10:49:11 INFO DAGScheduler:59 - ResultStage 6 (foreach at BernoulliNaiveBayesLearning.scala:60) finished in 0.012 s -2016-05-25 10:49:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 10:49:11 INFO DAGScheduler:59 - Job 4 finished: foreach at BernoulliNaiveBayesLearning.scala:60, took 0.026692 s -2016-05-25 10:49:11 INFO SparkContext:59 - Starting job: foreach at BernoulliNaiveBayesLearning.scala:62 -2016-05-25 10:49:11 INFO DAGScheduler:59 - Got job 5 (foreach at BernoulliNaiveBayesLearning.scala:62) with 2 output partitions -2016-05-25 10:49:11 INFO DAGScheduler:59 - Final stage: ResultStage 7(foreach at BernoulliNaiveBayesLearning.scala:62) -2016-05-25 10:49:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:49:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:49:11 INFO DAGScheduler:59 - Submitting ResultStage 7 (ParallelCollectionRDD[10] at makeRDD at BernoulliNaiveBayesLearning.scala:62), which has no missing parents -2016-05-25 10:49:11 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=8106, maxMem=766075207 -2016-05-25 10:49:11 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 1224.0 B, free 730.6 MB) -2016-05-25 10:49:11 INFO MemoryStore:59 - ensureFreeSpace(859) called with curMem=9330, maxMem=766075207 -2016-05-25 10:49:11 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 859.0 B, free 730.6 MB) -2016-05-25 10:49:11 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53150 (size: 859.0 B, free: 730.6 MB) -2016-05-25 10:49:11 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:49:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (ParallelCollectionRDD[10] at makeRDD at BernoulliNaiveBayesLearning.scala:62) -2016-05-25 10:49:11 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 10:49:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2210 bytes) -2016-05-25 10:49:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 15, localhost, PROCESS_LOCAL, 2234 bytes) -2016-05-25 10:49:11 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 14) -2016-05-25 10:49:11 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 15) -2016-05-25 10:49:11 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 14). 915 bytes result sent to driver -2016-05-25 10:49:11 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 15). 915 bytes result sent to driver -2016-05-25 10:49:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 14) in 10 ms on localhost (1/2) -2016-05-25 10:49:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 15) in 10 ms on localhost (2/2) -2016-05-25 10:49:11 INFO DAGScheduler:59 - ResultStage 7 (foreach at BernoulliNaiveBayesLearning.scala:62) finished in 0.012 s -2016-05-25 10:49:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 10:49:11 INFO DAGScheduler:59 - Job 5 finished: foreach at BernoulliNaiveBayesLearning.scala:62, took 0.028826 s -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 10:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 10:49:11 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 10:49:11 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 10:49:11 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 10:49:11 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 10:49:11 INFO BlockManager:59 - BlockManager stopped -2016-05-25 10:49:11 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 10:49:11 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 10:49:11 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 10:49:11 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 10:49:11 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-d4484bbe-b09a-4338-a080-4a33af105e32 -2016-05-25 10:49:45 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 10:49:46 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 10:49:46 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 10:49:46 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 10:49:46 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 10:49:47 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 10:49:47 INFO Remoting:74 - Starting remoting -2016-05-25 10:49:47 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53167] -2016-05-25 10:49:47 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53167. -2016-05-25 10:49:47 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 10:49:47 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 10:49:47 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-cfa99201-f7a7-480e-982a-76809484fc22 -2016-05-25 10:49:47 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 10:49:47 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-1444d0b1-def0-4632-a59c-f172973d04e3\httpd-564b6091-6a4f-411b-abfe-8c2b723caffa -2016-05-25 10:49:48 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 10:49:48 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:49:48 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53168 -2016-05-25 10:49:48 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53168. -2016-05-25 10:49:48 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 10:49:48 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 10:49:48 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 10:49:48 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 10:49:48 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 10:49:48 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 10:49:48 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 10:49:48 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53187. -2016-05-25 10:49:48 INFO NettyBlockTransferService:59 - Server created on 53187 -2016-05-25 10:49:48 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 10:49:48 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53187 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53187) -2016-05-25 10:49:48 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 10:49:49 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:49:49 INFO DAGScheduler:59 - Registering RDD 1 (map at NaiveBayes.scala:383) -2016-05-25 10:49:49 INFO DAGScheduler:59 - Got job 0 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:49:49 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at NaiveBayes.scala:401) -2016-05-25 10:49:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-05-25 10:49:49 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-05-25 10:49:49 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[1] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:49:49 INFO MemoryStore:59 - ensureFreeSpace(3288) called with curMem=0, maxMem=766075207 -2016-05-25 10:49:49 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-05-25 10:49:49 INFO MemoryStore:59 - ensureFreeSpace(1806) called with curMem=3288, maxMem=766075207 -2016-05-25 10:49:49 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1806.0 B, free 730.6 MB) -2016-05-25 10:49:49 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53187 (size: 1806.0 B, free: 730.6 MB) -2016-05-25 10:49:49 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:49:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[1] at map at NaiveBayes.scala:383) -2016-05-25 10:49:49 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 10:49:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2340 bytes) -2016-05-25 10:49:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2378 bytes) -2016-05-25 10:49:49 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 10:49:49 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 10:49:49 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 10:49:49 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 10:49:49 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1159 bytes result sent to driver -2016-05-25 10:49:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 114 ms on localhost (1/2) -2016-05-25 10:49:49 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1159 bytes result sent to driver -2016-05-25 10:49:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 146 ms on localhost (2/2) -2016-05-25 10:49:49 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at NaiveBayes.scala:383) finished in 0.161 s -2016-05-25 10:49:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 10:49:49 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 10:49:49 INFO DAGScheduler:59 - running: Set() -2016-05-25 10:49:49 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-05-25 10:49:49 INFO DAGScheduler:59 - failed: Set() -2016-05-25 10:49:49 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-05-25 10:49:49 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[2] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 10:49:49 INFO MemoryStore:59 - ensureFreeSpace(2928) called with curMem=5094, maxMem=766075207 -2016-05-25 10:49:49 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-05-25 10:49:49 INFO MemoryStore:59 - ensureFreeSpace(1559) called with curMem=8022, maxMem=766075207 -2016-05-25 10:49:49 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1559.0 B, free 730.6 MB) -2016-05-25 10:49:49 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53187 (size: 1559.0 B, free: 730.6 MB) -2016-05-25 10:49:49 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:49:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[2] at combineByKey at NaiveBayes.scala:383) -2016-05-25 10:49:49 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-25 10:49:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:49:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:49:49 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 10:49:49 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-25 10:49:49 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 10:49:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 10:49:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 10:49:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 10:49:49 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-05-25 10:49:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 35 ms on localhost (1/2) -2016-05-25 10:49:49 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1469 bytes result sent to driver -2016-05-25 10:49:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 41 ms on localhost (2/2) -2016-05-25 10:49:49 INFO DAGScheduler:59 - ResultStage 1 (collect at NaiveBayes.scala:401) finished in 0.042 s -2016-05-25 10:49:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 10:49:49 INFO DAGScheduler:59 - Job 0 finished: collect at NaiveBayes.scala:401, took 0.527815 s -2016-05-25 10:49:49 INFO SparkContext:59 - Starting job: foreach at BernoulliNaiveBayesLearning.scala:37 -2016-05-25 10:49:49 INFO DAGScheduler:59 - Got job 1 (foreach at BernoulliNaiveBayesLearning.scala:37) with 2 output partitions -2016-05-25 10:49:49 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at BernoulliNaiveBayesLearning.scala:37) -2016-05-25 10:49:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:49:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:49:49 INFO DAGScheduler:59 - Submitting ResultStage 2 (ParallelCollectionRDD[3] at makeRDD at BernoulliNaiveBayesLearning.scala:37), which has no missing parents -2016-05-25 10:49:49 INFO MemoryStore:59 - ensureFreeSpace(1296) called with curMem=9581, maxMem=766075207 -2016-05-25 10:49:49 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 1296.0 B, free 730.6 MB) -2016-05-25 10:49:49 INFO MemoryStore:59 - ensureFreeSpace(914) called with curMem=10877, maxMem=766075207 -2016-05-25 10:49:49 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 914.0 B, free 730.6 MB) -2016-05-25 10:49:49 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53187 (size: 914.0 B, free: 730.6 MB) -2016-05-25 10:49:49 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:49:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (ParallelCollectionRDD[3] at makeRDD at BernoulliNaiveBayesLearning.scala:37) -2016-05-25 10:49:49 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 10:49:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2351 bytes) -2016-05-25 10:49:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2389 bytes) -2016-05-25 10:49:49 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-25 10:49:49 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-25 10:49:49 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 915 bytes result sent to driver -2016-05-25 10:49:49 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 915 bytes result sent to driver -2016-05-25 10:49:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 12 ms on localhost (1/2) -2016-05-25 10:49:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 11 ms on localhost (2/2) -2016-05-25 10:49:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 10:49:49 INFO DAGScheduler:59 - ResultStage 2 (foreach at BernoulliNaiveBayesLearning.scala:37) finished in 0.013 s -2016-05-25 10:49:49 INFO DAGScheduler:59 - Job 1 finished: foreach at BernoulliNaiveBayesLearning.scala:37, took 0.024679 s -2016-05-25 10:49:49 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 10:49:49 INFO DAGScheduler:59 - Registering RDD 5 (map at NaiveBayes.scala:383) -2016-05-25 10:49:49 INFO DAGScheduler:59 - Got job 2 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 10:49:49 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at NaiveBayes.scala:401) -2016-05-25 10:49:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3) -2016-05-25 10:49:49 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 3) -2016-05-25 10:49:49 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 10:49:49 INFO MemoryStore:59 - ensureFreeSpace(3288) called with curMem=11791, maxMem=766075207 -2016-05-25 10:49:49 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-05-25 10:49:49 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=15079, maxMem=766075207 -2016-05-25 10:49:49 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.6 MB) -2016-05-25 10:49:49 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53187 (size: 1803.0 B, free: 730.6 MB) -2016-05-25 10:49:49 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:49:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383) -2016-05-25 10:49:49 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 10:49:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2416 bytes) -2016-05-25 10:49:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2454 bytes) -2016-05-25 10:49:49 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-25 10:49:49 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-25 10:49:49 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 1159 bytes result sent to driver -2016-05-25 10:49:49 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 1159 bytes result sent to driver -2016-05-25 10:49:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 17 ms on localhost (1/2) -2016-05-25 10:49:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 20 ms on localhost (2/2) -2016-05-25 10:49:49 INFO DAGScheduler:59 - ShuffleMapStage 3 (map at NaiveBayes.scala:383) finished in 0.020 s -2016-05-25 10:49:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 10:49:49 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 10:49:49 INFO DAGScheduler:59 - running: Set() -2016-05-25 10:49:49 INFO DAGScheduler:59 - waiting: Set(ResultStage 4) -2016-05-25 10:49:49 INFO DAGScheduler:59 - failed: Set() -2016-05-25 10:49:49 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List() -2016-05-25 10:49:50 INFO DAGScheduler:59 - Submitting ResultStage 4 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 10:49:50 INFO MemoryStore:59 - ensureFreeSpace(2928) called with curMem=16882, maxMem=766075207 -2016-05-25 10:49:50 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-05-25 10:49:50 INFO MemoryStore:59 - ensureFreeSpace(1554) called with curMem=19810, maxMem=766075207 -2016-05-25 10:49:50 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 1554.0 B, free 730.6 MB) -2016-05-25 10:49:50 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53187 (size: 1554.0 B, free: 730.6 MB) -2016-05-25 10:49:50 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:49:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383) -2016-05-25 10:49:50 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 10:49:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:49:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 10:49:50 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-25 10:49:50 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-25 10:49:50 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 10:49:50 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 10:49:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 10:49:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 10:49:50 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 1161 bytes result sent to driver -2016-05-25 10:49:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 11 ms on localhost (1/2) -2016-05-25 10:49:50 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 1533 bytes result sent to driver -2016-05-25 10:49:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 16 ms on localhost (2/2) -2016-05-25 10:49:50 INFO DAGScheduler:59 - ResultStage 4 (collect at NaiveBayes.scala:401) finished in 0.016 s -2016-05-25 10:49:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 10:49:50 INFO DAGScheduler:59 - Job 2 finished: collect at NaiveBayes.scala:401, took 0.059864 s -2016-05-25 10:49:50 INFO MemoryStore:59 - ensureFreeSpace(520) called with curMem=21364, maxMem=766075207 -2016-05-25 10:49:50 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 520.0 B, free 730.6 MB) -2016-05-25 10:49:50 INFO MemoryStore:59 - ensureFreeSpace(679) called with curMem=21884, maxMem=766075207 -2016-05-25 10:49:50 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 679.0 B, free 730.6 MB) -2016-05-25 10:49:50 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53187 (size: 679.0 B, free: 730.6 MB) -2016-05-25 10:49:50 INFO SparkContext:59 - Created broadcast 5 from broadcast at NaiveBayes.scala:89 -2016-05-25 10:49:50 INFO SparkContext:59 - Starting job: collect at BernoulliNaiveBayesLearning.scala:57 -2016-05-25 10:49:50 INFO DAGScheduler:59 - Got job 3 (collect at BernoulliNaiveBayesLearning.scala:57) with 2 output partitions -2016-05-25 10:49:50 INFO DAGScheduler:59 - Final stage: ResultStage 5(collect at BernoulliNaiveBayesLearning.scala:57) -2016-05-25 10:49:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:49:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:49:50 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[8] at mapPartitions at NaiveBayes.scala:90), which has no missing parents -2016-05-25 10:49:50 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=22563, maxMem=766075207 -2016-05-25 10:49:50 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-05-25 10:49:50 INFO MemoryStore:59 - ensureFreeSpace(1737) called with curMem=25523, maxMem=766075207 -2016-05-25 10:49:50 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1737.0 B, free 730.6 MB) -2016-05-25 10:49:50 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53187 (size: 1737.0 B, free: 730.6 MB) -2016-05-25 10:49:50 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:49:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[8] at mapPartitions at NaiveBayes.scala:90) -2016-05-25 10:49:50 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 10:49:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2210 bytes) -2016-05-25 10:49:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2234 bytes) -2016-05-25 10:49:50 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-05-25 10:49:50 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-05-25 10:49:50 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 914 bytes result sent to driver -2016-05-25 10:49:50 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 906 bytes result sent to driver -2016-05-25 10:49:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 11 ms on localhost (1/2) -2016-05-25 10:49:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 14 ms on localhost (2/2) -2016-05-25 10:49:50 INFO DAGScheduler:59 - ResultStage 5 (collect at BernoulliNaiveBayesLearning.scala:57) finished in 0.016 s -2016-05-25 10:49:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 10:49:50 INFO DAGScheduler:59 - Job 3 finished: collect at BernoulliNaiveBayesLearning.scala:57, took 0.054595 s -2016-05-25 10:49:50 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53187 in memory (size: 1554.0 B, free: 730.6 MB) -2016-05-25 10:49:50 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 10:49:50 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53187 in memory (size: 1803.0 B, free: 730.6 MB) -2016-05-25 10:49:50 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 10:49:50 INFO SparkContext:59 - Starting job: foreach at BernoulliNaiveBayesLearning.scala:60 -2016-05-25 10:49:50 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-25 10:49:50 INFO DAGScheduler:59 - Got job 4 (foreach at BernoulliNaiveBayesLearning.scala:60) with 2 output partitions -2016-05-25 10:49:50 INFO DAGScheduler:59 - Final stage: ResultStage 6(foreach at BernoulliNaiveBayesLearning.scala:60) -2016-05-25 10:49:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:49:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:49:50 INFO DAGScheduler:59 - Submitting ResultStage 6 (ParallelCollectionRDD[9] at makeRDD at BernoulliNaiveBayesLearning.scala:60), which has no missing parents -2016-05-25 10:49:50 INFO MemoryStore:59 - ensureFreeSpace(1296) called with curMem=15477, maxMem=766075207 -2016-05-25 10:49:50 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 1296.0 B, free 730.6 MB) -2016-05-25 10:49:50 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53187 in memory (size: 914.0 B, free: 730.6 MB) -2016-05-25 10:49:50 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 10:49:50 INFO MemoryStore:59 - ensureFreeSpace(914) called with curMem=15214, maxMem=766075207 -2016-05-25 10:49:50 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 914.0 B, free 730.6 MB) -2016-05-25 10:49:50 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53187 in memory (size: 1559.0 B, free: 730.6 MB) -2016-05-25 10:49:50 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 10:49:50 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53187 (size: 914.0 B, free: 730.6 MB) -2016-05-25 10:49:50 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:49:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (ParallelCollectionRDD[9] at makeRDD at BernoulliNaiveBayesLearning.scala:60) -2016-05-25 10:49:50 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 10:49:50 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:53187 in memory (size: 1806.0 B, free: 730.6 MB) -2016-05-25 10:49:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2427 bytes) -2016-05-25 10:49:50 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 10:49:50 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 10:49:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2465 bytes) -2016-05-25 10:49:50 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-05-25 10:49:50 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 13) -2016-05-25 10:49:50 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 915 bytes result sent to driver -2016-05-25 10:49:50 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 13). 915 bytes result sent to driver -2016-05-25 10:49:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 11 ms on localhost (1/2) -2016-05-25 10:49:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 13) in 11 ms on localhost (2/2) -2016-05-25 10:49:50 INFO DAGScheduler:59 - ResultStage 6 (foreach at BernoulliNaiveBayesLearning.scala:60) finished in 0.013 s -2016-05-25 10:49:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 10:49:50 INFO DAGScheduler:59 - Job 4 finished: foreach at BernoulliNaiveBayesLearning.scala:60, took 0.029415 s -2016-05-25 10:49:50 INFO SparkContext:59 - Starting job: foreach at BernoulliNaiveBayesLearning.scala:62 -2016-05-25 10:49:50 INFO DAGScheduler:59 - Got job 5 (foreach at BernoulliNaiveBayesLearning.scala:62) with 2 output partitions -2016-05-25 10:49:50 INFO DAGScheduler:59 - Final stage: ResultStage 7(foreach at BernoulliNaiveBayesLearning.scala:62) -2016-05-25 10:49:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 10:49:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 10:49:50 INFO DAGScheduler:59 - Submitting ResultStage 7 (ParallelCollectionRDD[10] at makeRDD at BernoulliNaiveBayesLearning.scala:62), which has no missing parents -2016-05-25 10:49:50 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=8106, maxMem=766075207 -2016-05-25 10:49:50 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 1224.0 B, free 730.6 MB) -2016-05-25 10:49:50 INFO MemoryStore:59 - ensureFreeSpace(859) called with curMem=9330, maxMem=766075207 -2016-05-25 10:49:50 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 859.0 B, free 730.6 MB) -2016-05-25 10:49:50 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53187 (size: 859.0 B, free: 730.6 MB) -2016-05-25 10:49:50 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 10:49:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (ParallelCollectionRDD[10] at makeRDD at BernoulliNaiveBayesLearning.scala:62) -2016-05-25 10:49:50 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 10:49:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2210 bytes) -2016-05-25 10:49:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 15, localhost, PROCESS_LOCAL, 2234 bytes) -2016-05-25 10:49:50 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 14) -2016-05-25 10:49:50 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 15) -2016-05-25 10:49:50 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 15). 915 bytes result sent to driver -2016-05-25 10:49:50 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 14). 915 bytes result sent to driver -2016-05-25 10:49:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 15) in 9 ms on localhost (1/2) -2016-05-25 10:49:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 14) in 12 ms on localhost (2/2) -2016-05-25 10:49:50 INFO DAGScheduler:59 - ResultStage 7 (foreach at BernoulliNaiveBayesLearning.scala:62) finished in 0.012 s -2016-05-25 10:49:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 10:49:50 INFO DAGScheduler:59 - Job 5 finished: foreach at BernoulliNaiveBayesLearning.scala:62, took 0.029650 s -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 10:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 10:49:50 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 10:49:50 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 10:49:50 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 10:49:50 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 10:49:50 INFO BlockManager:59 - BlockManager stopped -2016-05-25 10:49:50 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 10:49:50 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 10:49:50 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 10:49:50 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 10:49:50 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-1444d0b1-def0-4632-a59c-f172973d04e3 -2016-05-25 11:04:15 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 11:04:16 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 11:04:16 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 11:04:16 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 11:04:16 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 11:04:17 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 11:04:17 INFO Remoting:74 - Starting remoting -2016-05-25 11:04:17 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53294] -2016-05-25 11:04:17 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53294. -2016-05-25 11:04:17 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 11:04:17 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 11:04:17 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-6f68ce59-a0c9-414d-a70d-c142d267392b -2016-05-25 11:04:17 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 11:04:17 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-63d75282-e81c-4563-a236-d417948f386b\httpd-8cc655eb-5985-44dc-b000-6c73abbe1a96 -2016-05-25 11:04:17 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 11:04:18 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:04:18 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53295 -2016-05-25 11:04:18 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53295. -2016-05-25 11:04:18 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 11:04:18 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:04:18 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 11:04:18 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 11:04:18 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 11:04:18 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 11:04:18 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 11:04:18 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53314. -2016-05-25 11:04:18 INFO NettyBlockTransferService:59 - Server created on 53314 -2016-05-25 11:04:18 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 11:04:18 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53314 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53314) -2016-05-25 11:04:18 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 11:04:19 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 11:04:19 INFO DAGScheduler:59 - Registering RDD 1 (map at NaiveBayes.scala:383) -2016-05-25 11:04:19 INFO DAGScheduler:59 - Got job 0 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 11:04:19 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at NaiveBayes.scala:401) -2016-05-25 11:04:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-05-25 11:04:19 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-05-25 11:04:19 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[1] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 11:04:19 INFO MemoryStore:59 - ensureFreeSpace(3288) called with curMem=0, maxMem=766075207 -2016-05-25 11:04:19 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-05-25 11:04:19 INFO MemoryStore:59 - ensureFreeSpace(1806) called with curMem=3288, maxMem=766075207 -2016-05-25 11:04:19 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1806.0 B, free 730.6 MB) -2016-05-25 11:04:19 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53314 (size: 1806.0 B, free: 730.6 MB) -2016-05-25 11:04:19 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:04:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[1] at map at NaiveBayes.scala:383) -2016-05-25 11:04:19 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 11:04:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2340 bytes) -2016-05-25 11:04:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2378 bytes) -2016-05-25 11:04:19 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 11:04:19 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 11:04:19 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 11:04:19 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 11:04:19 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1159 bytes result sent to driver -2016-05-25 11:04:19 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1159 bytes result sent to driver -2016-05-25 11:04:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 151 ms on localhost (1/2) -2016-05-25 11:04:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 125 ms on localhost (2/2) -2016-05-25 11:04:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 11:04:19 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at NaiveBayes.scala:383) finished in 0.169 s -2016-05-25 11:04:19 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:04:19 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:04:19 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-05-25 11:04:19 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:04:19 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-05-25 11:04:19 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[2] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 11:04:19 INFO MemoryStore:59 - ensureFreeSpace(2928) called with curMem=5094, maxMem=766075207 -2016-05-25 11:04:19 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-05-25 11:04:19 INFO MemoryStore:59 - ensureFreeSpace(1559) called with curMem=8022, maxMem=766075207 -2016-05-25 11:04:19 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1559.0 B, free 730.6 MB) -2016-05-25 11:04:19 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53314 (size: 1559.0 B, free: 730.6 MB) -2016-05-25 11:04:19 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:04:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[2] at combineByKey at NaiveBayes.scala:383) -2016-05-25 11:04:19 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-25 11:04:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:04:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:04:19 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 11:04:19 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-25 11:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-25 11:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-25 11:04:19 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-05-25 11:04:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 36 ms on localhost (1/2) -2016-05-25 11:04:19 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1469 bytes result sent to driver -2016-05-25 11:04:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 43 ms on localhost (2/2) -2016-05-25 11:04:19 INFO DAGScheduler:59 - ResultStage 1 (collect at NaiveBayes.scala:401) finished in 0.044 s -2016-05-25 11:04:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 11:04:19 INFO DAGScheduler:59 - Job 0 finished: collect at NaiveBayes.scala:401, took 0.447783 s -2016-05-25 11:04:19 INFO SparkContext:59 - Starting job: foreach at BernoulliNaiveBayesLearning.scala:38 -2016-05-25 11:04:19 INFO DAGScheduler:59 - Got job 1 (foreach at BernoulliNaiveBayesLearning.scala:38) with 2 output partitions -2016-05-25 11:04:19 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at BernoulliNaiveBayesLearning.scala:38) -2016-05-25 11:04:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:04:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:04:19 INFO DAGScheduler:59 - Submitting ResultStage 2 (ParallelCollectionRDD[3] at makeRDD at BernoulliNaiveBayesLearning.scala:38), which has no missing parents -2016-05-25 11:04:19 INFO MemoryStore:59 - ensureFreeSpace(1296) called with curMem=9581, maxMem=766075207 -2016-05-25 11:04:19 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 1296.0 B, free 730.6 MB) -2016-05-25 11:04:19 INFO MemoryStore:59 - ensureFreeSpace(914) called with curMem=10877, maxMem=766075207 -2016-05-25 11:04:19 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 914.0 B, free 730.6 MB) -2016-05-25 11:04:19 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53314 (size: 914.0 B, free: 730.6 MB) -2016-05-25 11:04:19 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:04:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (ParallelCollectionRDD[3] at makeRDD at BernoulliNaiveBayesLearning.scala:38) -2016-05-25 11:04:19 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 11:04:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2351 bytes) -2016-05-25 11:04:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2389 bytes) -2016-05-25 11:04:19 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-25 11:04:19 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-25 11:04:19 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 915 bytes result sent to driver -2016-05-25 11:04:19 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 915 bytes result sent to driver -2016-05-25 11:04:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 14 ms on localhost (1/2) -2016-05-25 11:04:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 14 ms on localhost (2/2) -2016-05-25 11:04:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 11:04:19 INFO DAGScheduler:59 - ResultStage 2 (foreach at BernoulliNaiveBayesLearning.scala:38) finished in 0.017 s -2016-05-25 11:04:19 INFO DAGScheduler:59 - Job 1 finished: foreach at BernoulliNaiveBayesLearning.scala:38, took 0.029800 s -2016-05-25 11:04:19 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 11:04:19 INFO DAGScheduler:59 - Registering RDD 5 (map at NaiveBayes.scala:383) -2016-05-25 11:04:19 INFO DAGScheduler:59 - Got job 2 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 11:04:19 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at NaiveBayes.scala:401) -2016-05-25 11:04:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3) -2016-05-25 11:04:19 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 3) -2016-05-25 11:04:19 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 11:04:19 INFO MemoryStore:59 - ensureFreeSpace(3288) called with curMem=11791, maxMem=766075207 -2016-05-25 11:04:19 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-05-25 11:04:19 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=15079, maxMem=766075207 -2016-05-25 11:04:19 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.6 MB) -2016-05-25 11:04:19 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53314 (size: 1803.0 B, free: 730.6 MB) -2016-05-25 11:04:19 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:04:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383) -2016-05-25 11:04:19 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 11:04:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2416 bytes) -2016-05-25 11:04:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2454 bytes) -2016-05-25 11:04:19 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-25 11:04:19 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-25 11:04:19 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 1159 bytes result sent to driver -2016-05-25 11:04:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 19 ms on localhost (1/2) -2016-05-25 11:04:19 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 1159 bytes result sent to driver -2016-05-25 11:04:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 21 ms on localhost (2/2) -2016-05-25 11:04:19 INFO DAGScheduler:59 - ShuffleMapStage 3 (map at NaiveBayes.scala:383) finished in 0.022 s -2016-05-25 11:04:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 11:04:19 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:04:19 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:04:19 INFO DAGScheduler:59 - waiting: Set(ResultStage 4) -2016-05-25 11:04:19 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:04:19 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List() -2016-05-25 11:04:19 INFO DAGScheduler:59 - Submitting ResultStage 4 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 11:04:19 INFO MemoryStore:59 - ensureFreeSpace(2928) called with curMem=16882, maxMem=766075207 -2016-05-25 11:04:19 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-05-25 11:04:19 INFO MemoryStore:59 - ensureFreeSpace(1554) called with curMem=19810, maxMem=766075207 -2016-05-25 11:04:19 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 1554.0 B, free 730.6 MB) -2016-05-25 11:04:19 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53314 (size: 1554.0 B, free: 730.6 MB) -2016-05-25 11:04:19 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:04:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383) -2016-05-25 11:04:19 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 11:04:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:04:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:04:19 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-25 11:04:19 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-25 11:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:04:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 11:04:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 11:04:19 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 1161 bytes result sent to driver -2016-05-25 11:04:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 11 ms on localhost (1/2) -2016-05-25 11:04:19 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 1533 bytes result sent to driver -2016-05-25 11:04:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 17 ms on localhost (2/2) -2016-05-25 11:04:19 INFO DAGScheduler:59 - ResultStage 4 (collect at NaiveBayes.scala:401) finished in 0.017 s -2016-05-25 11:04:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 11:04:19 INFO DAGScheduler:59 - Job 2 finished: collect at NaiveBayes.scala:401, took 0.064992 s -2016-05-25 11:04:19 INFO MemoryStore:59 - ensureFreeSpace(520) called with curMem=21364, maxMem=766075207 -2016-05-25 11:04:20 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 520.0 B, free 730.6 MB) -2016-05-25 11:04:20 INFO MemoryStore:59 - ensureFreeSpace(679) called with curMem=21884, maxMem=766075207 -2016-05-25 11:04:20 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 679.0 B, free 730.6 MB) -2016-05-25 11:04:20 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53314 (size: 679.0 B, free: 730.6 MB) -2016-05-25 11:04:20 INFO SparkContext:59 - Created broadcast 5 from broadcast at NaiveBayes.scala:89 -2016-05-25 11:04:20 INFO SparkContext:59 - Starting job: collect at BernoulliNaiveBayesLearning.scala:58 -2016-05-25 11:04:20 INFO DAGScheduler:59 - Got job 3 (collect at BernoulliNaiveBayesLearning.scala:58) with 2 output partitions -2016-05-25 11:04:20 INFO DAGScheduler:59 - Final stage: ResultStage 5(collect at BernoulliNaiveBayesLearning.scala:58) -2016-05-25 11:04:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:04:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:04:20 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[8] at mapPartitions at NaiveBayes.scala:90), which has no missing parents -2016-05-25 11:04:20 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=22563, maxMem=766075207 -2016-05-25 11:04:20 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-05-25 11:04:20 INFO MemoryStore:59 - ensureFreeSpace(1737) called with curMem=25523, maxMem=766075207 -2016-05-25 11:04:20 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1737.0 B, free 730.6 MB) -2016-05-25 11:04:20 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53314 (size: 1737.0 B, free: 730.6 MB) -2016-05-25 11:04:20 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:04:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[8] at mapPartitions at NaiveBayes.scala:90) -2016-05-25 11:04:20 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 11:04:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2210 bytes) -2016-05-25 11:04:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2234 bytes) -2016-05-25 11:04:20 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-05-25 11:04:20 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-05-25 11:04:20 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 914 bytes result sent to driver -2016-05-25 11:04:20 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 906 bytes result sent to driver -2016-05-25 11:04:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 11 ms on localhost (1/2) -2016-05-25 11:04:20 INFO DAGScheduler:59 - ResultStage 5 (collect at BernoulliNaiveBayesLearning.scala:58) finished in 0.014 s -2016-05-25 11:04:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 13 ms on localhost (2/2) -2016-05-25 11:04:20 INFO DAGScheduler:59 - Job 3 finished: collect at BernoulliNaiveBayesLearning.scala:58, took 0.026227 s -2016-05-25 11:04:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 11:04:20 INFO SparkContext:59 - Starting job: foreach at BernoulliNaiveBayesLearning.scala:61 -2016-05-25 11:04:20 INFO DAGScheduler:59 - Got job 4 (foreach at BernoulliNaiveBayesLearning.scala:61) with 2 output partitions -2016-05-25 11:04:20 INFO DAGScheduler:59 - Final stage: ResultStage 6(foreach at BernoulliNaiveBayesLearning.scala:61) -2016-05-25 11:04:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:04:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:04:20 INFO DAGScheduler:59 - Submitting ResultStage 6 (ParallelCollectionRDD[9] at makeRDD at BernoulliNaiveBayesLearning.scala:61), which has no missing parents -2016-05-25 11:04:20 INFO MemoryStore:59 - ensureFreeSpace(1296) called with curMem=27260, maxMem=766075207 -2016-05-25 11:04:20 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 1296.0 B, free 730.6 MB) -2016-05-25 11:04:20 INFO MemoryStore:59 - ensureFreeSpace(914) called with curMem=28556, maxMem=766075207 -2016-05-25 11:04:20 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 914.0 B, free 730.6 MB) -2016-05-25 11:04:20 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53314 (size: 914.0 B, free: 730.6 MB) -2016-05-25 11:04:20 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:04:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (ParallelCollectionRDD[9] at makeRDD at BernoulliNaiveBayesLearning.scala:61) -2016-05-25 11:04:20 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 11:04:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2427 bytes) -2016-05-25 11:04:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2465 bytes) -2016-05-25 11:04:20 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-05-25 11:04:20 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 13) -2016-05-25 11:04:20 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 13). 915 bytes result sent to driver -2016-05-25 11:04:20 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 915 bytes result sent to driver -2016-05-25 11:04:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 13) in 9 ms on localhost (1/2) -2016-05-25 11:04:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 11 ms on localhost (2/2) -2016-05-25 11:04:20 INFO DAGScheduler:59 - ResultStage 6 (foreach at BernoulliNaiveBayesLearning.scala:61) finished in 0.011 s -2016-05-25 11:04:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 11:04:20 INFO DAGScheduler:59 - Job 4 finished: foreach at BernoulliNaiveBayesLearning.scala:61, took 0.023251 s -2016-05-25 11:04:20 INFO SparkContext:59 - Starting job: foreach at BernoulliNaiveBayesLearning.scala:63 -2016-05-25 11:04:20 INFO DAGScheduler:59 - Got job 5 (foreach at BernoulliNaiveBayesLearning.scala:63) with 2 output partitions -2016-05-25 11:04:20 INFO DAGScheduler:59 - Final stage: ResultStage 7(foreach at BernoulliNaiveBayesLearning.scala:63) -2016-05-25 11:04:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:04:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:04:20 INFO DAGScheduler:59 - Submitting ResultStage 7 (ParallelCollectionRDD[10] at makeRDD at BernoulliNaiveBayesLearning.scala:63), which has no missing parents -2016-05-25 11:04:20 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=29470, maxMem=766075207 -2016-05-25 11:04:20 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 1224.0 B, free 730.6 MB) -2016-05-25 11:04:20 INFO MemoryStore:59 - ensureFreeSpace(859) called with curMem=30694, maxMem=766075207 -2016-05-25 11:04:20 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 859.0 B, free 730.6 MB) -2016-05-25 11:04:20 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53314 (size: 859.0 B, free: 730.6 MB) -2016-05-25 11:04:20 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:04:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (ParallelCollectionRDD[10] at makeRDD at BernoulliNaiveBayesLearning.scala:63) -2016-05-25 11:04:20 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 11:04:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2210 bytes) -2016-05-25 11:04:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 15, localhost, PROCESS_LOCAL, 2234 bytes) -2016-05-25 11:04:20 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 15) -2016-05-25 11:04:20 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 14) -2016-05-25 11:04:20 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 15). 915 bytes result sent to driver -2016-05-25 11:04:20 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 14). 915 bytes result sent to driver -2016-05-25 11:04:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 15) in 18 ms on localhost (1/2) -2016-05-25 11:04:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 14) in 23 ms on localhost (2/2) -2016-05-25 11:04:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 11:04:20 INFO DAGScheduler:59 - ResultStage 7 (foreach at BernoulliNaiveBayesLearning.scala:63) finished in 0.025 s -2016-05-25 11:04:20 INFO DAGScheduler:59 - Job 5 finished: foreach at BernoulliNaiveBayesLearning.scala:63, took 0.044001 s -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 11:04:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 11:04:20 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 11:04:20 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 11:04:20 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 11:04:20 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 11:04:20 INFO BlockManager:59 - BlockManager stopped -2016-05-25 11:04:20 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 11:04:20 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 11:04:20 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 11:04:20 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 11:04:20 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-63d75282-e81c-4563-a236-d417948f386b -2016-05-25 11:04:20 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 11:04:20 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-25 11:09:35 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 11:09:36 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 11:09:36 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 11:09:36 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 11:09:36 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 11:09:37 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 11:09:37 INFO Remoting:74 - Starting remoting -2016-05-25 11:09:37 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53444] -2016-05-25 11:09:37 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53444. -2016-05-25 11:09:37 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 11:09:37 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 11:09:37 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-08508570-e499-4e4d-8216-b4779eae701c -2016-05-25 11:09:37 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 11:09:37 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-6a36f9cc-5957-49f5-af23-181e89f591f3\httpd-7d268d91-2925-4c06-ae1f-d35ab8145191 -2016-05-25 11:09:37 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 11:09:37 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:09:37 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53445 -2016-05-25 11:09:37 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53445. -2016-05-25 11:09:37 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 11:09:37 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:09:37 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 11:09:37 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 11:09:37 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 11:09:38 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 11:09:38 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 11:09:38 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53464. -2016-05-25 11:09:38 INFO NettyBlockTransferService:59 - Server created on 53464 -2016-05-25 11:09:38 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 11:09:38 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53464 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53464) -2016-05-25 11:09:38 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 11:09:39 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 11:09:39 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 11:09:39 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 11:09:39 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 11:09:39 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53464 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 11:09:39 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 11:09:40 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 11:09:41 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 11:09:41 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 11:09:41 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 11:09:41 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 11:09:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:09:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:09:41 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 11:09:41 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 11:09:41 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 11:09:41 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 11:09:41 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 11:09:41 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53464 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:09:41 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:09:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 11:09:41 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 11:09:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:09:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:09:41 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 11:09:41 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 11:09:41 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 11:09:41 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 11:09:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 11:09:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 11:09:41 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 11:09:41 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 11:09:41 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 11:09:41 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 11:09:41 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 11:09:41 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177630, maxMem=766075207 -2016-05-25 11:09:41 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 11:09:41 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263278, maxMem=766075207 -2016-05-25 11:09:41 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:53464 (size: 83.6 KB, free: 730.5 MB) -2016-05-25 11:09:41 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 11:09:41 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:53464 (size: 83.1 KB, free: 730.4 MB) -2016-05-25 11:09:41 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 11:09:41 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 11:09:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 367 ms on localhost (1/2) -2016-05-25 11:09:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 351 ms on localhost (2/2) -2016-05-25 11:09:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 11:09:41 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.391 s -2016-05-25 11:09:41 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.594599 s -2016-05-25 11:09:42 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 11:09:42 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 11:09:42 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 11:09:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:09:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:09:42 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 11:09:42 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348358, maxMem=766075207 -2016-05-25 11:09:42 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 11:09:42 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=353126, maxMem=766075207 -2016-05-25 11:09:42 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 11:09:42 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53464 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:09:42 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:09:42 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 11:09:42 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 11:09:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:09:42 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 11:09:42 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:09:42 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 11:09:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 45 ms on localhost (1/1) -2016-05-25 11:09:42 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.045 s -2016-05-25 11:09:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 11:09:42 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.074181 s -2016-05-25 11:09:42 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 11:09:42 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 11:09:42 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 11:09:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:09:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:09:42 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 11:09:42 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355673, maxMem=766075207 -2016-05-25 11:09:42 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 11:09:42 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=360009, maxMem=766075207 -2016-05-25 11:09:42 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 11:09:42 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53464 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:09:42 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:09:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 11:09:42 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 11:09:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:09:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:09:42 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 11:09:42 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 11:09:42 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:09:42 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:09:42 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 11:09:42 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 11:09:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 14 ms on localhost (1/2) -2016-05-25 11:09:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 12 ms on localhost (2/2) -2016-05-25 11:09:42 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.014 s -2016-05-25 11:09:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 11:09:42 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.026638 s -2016-05-25 11:09:42 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 11:09:42 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 11:09:42 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 11:09:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:09:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:09:42 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 11:09:42 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362328, maxMem=766075207 -2016-05-25 11:09:42 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 11:09:42 INFO MemoryStore:59 - ensureFreeSpace(2630) called with curMem=367392, maxMem=766075207 -2016-05-25 11:09:42 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 11:09:42 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53464 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:09:42 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:09:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 11:09:42 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 11:09:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:09:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:09:42 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 11:09:42 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 11:09:42 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:09:42 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 71652 bytes result sent to driver -2016-05-25 11:09:42 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:09:42 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 54219 bytes result sent to driver -2016-05-25 11:09:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 28 ms on localhost (1/2) -2016-05-25 11:09:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 30 ms on localhost (2/2) -2016-05-25 11:09:42 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.034 s -2016-05-25 11:09:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 11:09:42 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.051100 s -2016-05-25 11:09:49 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53464 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:09:49 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 11:09:49 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53464 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:09:49 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 11:09:49 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53464 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:09:49 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 11:09:49 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=348358, maxMem=766075207 -2016-05-25 11:09:49 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 11:09:49 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=348398, maxMem=766075207 -2016-05-25 11:09:49 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 11:09:49 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53464 (size: 81.0 B, free: 730.4 MB) -2016-05-25 11:09:49 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 11:09:49 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 11:09:49 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 11:09:49 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 11:09:49 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 11:09:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 11:09:49 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 11:09:49 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 11:09:50 INFO MemoryStore:59 - ensureFreeSpace(497512) called with curMem=348479, maxMem=766075207 -2016-05-25 11:09:50 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 485.9 KB, free 729.8 MB) -2016-05-25 11:09:50 INFO MemoryStore:59 - ensureFreeSpace(122193) called with curMem=845991, maxMem=766075207 -2016-05-25 11:09:50 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 119.3 KB, free 729.7 MB) -2016-05-25 11:09:50 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53464 (size: 119.3 KB, free: 730.3 MB) -2016-05-25 11:09:50 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:09:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 11:09:50 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 11:09:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:09:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:09:50 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 11:09:50 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 11:09:50 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 11:09:50 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 11:09:50 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:09:50 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:09:50 INFO MemoryStore:59 - ensureFreeSpace(85816) called with curMem=968184, maxMem=766075207 -2016-05-25 11:09:50 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 83.8 KB, free 729.6 MB) -2016-05-25 11:09:50 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:53464 (size: 83.8 KB, free: 730.2 MB) -2016-05-25 11:09:50 INFO MemoryStore:59 - ensureFreeSpace(114416) called with curMem=1054000, maxMem=766075207 -2016-05-25 11:09:50 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 111.7 KB, free 729.5 MB) -2016-05-25 11:09:50 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:53464 (size: 111.7 KB, free: 730.1 MB) -2016-05-25 11:09:50 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 11:09:50 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 11:09:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 260 ms on localhost (1/2) -2016-05-25 11:09:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 259 ms on localhost (2/2) -2016-05-25 11:09:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 11:09:50 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.262 s -2016-05-25 11:09:50 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:09:50 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:09:50 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 11:09:50 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:09:50 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 11:09:50 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 11:09:50 INFO MemoryStore:59 - ensureFreeSpace(216760) called with curMem=1168416, maxMem=766075207 -2016-05-25 11:09:50 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 211.7 KB, free 729.3 MB) -2016-05-25 11:09:50 INFO MemoryStore:59 - ensureFreeSpace(45995) called with curMem=1385176, maxMem=766075207 -2016-05-25 11:09:50 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 44.9 KB, free 729.2 MB) -2016-05-25 11:09:50 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53464 (size: 44.9 KB, free: 730.1 MB) -2016-05-25 11:09:50 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:09:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 11:09:50 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 11:09:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:09:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:09:50 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 11:09:50 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 11:09:50 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:09:50 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:09:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 11:09:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 11:09:50 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 11:09:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 162 ms on localhost (1/2) -2016-05-25 11:09:50 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 11:09:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 478 ms on localhost (2/2) -2016-05-25 11:09:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 11:09:50 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.478 s -2016-05-25 11:09:50 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.798112 s -2016-05-25 11:09:50 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 11:09:50 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 11:09:50 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 11:09:50 INFO RandomForest:59 - init: 7.685715791 - total: 8.75478329 - findSplitsBins: 7.421105778 - findBestSplits: 1.045272754 - chooseSplits: 1.038776342 -2016-05-25 11:09:50 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:43 -2016-05-25 11:09:50 INFO DAGScheduler:59 - Got job 5 (count at DecisionTreesLearning.scala:43) with 2 output partitions -2016-05-25 11:09:50 INFO DAGScheduler:59 - Final stage: ResultStage 6(count at DecisionTreesLearning.scala:43) -2016-05-25 11:09:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:09:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:09:50 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[18] at filter at DecisionTreesLearning.scala:43), which has no missing parents -2016-05-25 11:09:50 INFO MemoryStore:59 - ensureFreeSpace(6520) called with curMem=1230939, maxMem=766075207 -2016-05-25 11:09:50 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 6.4 KB, free 729.4 MB) -2016-05-25 11:09:50 INFO MemoryStore:59 - ensureFreeSpace(3492) called with curMem=1237459, maxMem=766075207 -2016-05-25 11:09:50 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-25 11:09:50 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53464 (size: 3.4 KB, free: 730.2 MB) -2016-05-25 11:09:50 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:09:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[18] at filter at DecisionTreesLearning.scala:43) -2016-05-25 11:09:50 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 11:09:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:09:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:09:50 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 11:09:50 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 11:09:50 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:09:50 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:09:50 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2082 bytes result sent to driver -2016-05-25 11:09:50 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2082 bytes result sent to driver -2016-05-25 11:09:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 10 ms on localhost (1/2) -2016-05-25 11:09:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 12 ms on localhost (2/2) -2016-05-25 11:09:50 INFO DAGScheduler:59 - ResultStage 6 (count at DecisionTreesLearning.scala:43) finished in 0.012 s -2016-05-25 11:09:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 11:09:50 INFO DAGScheduler:59 - Job 5 finished: count at DecisionTreesLearning.scala:43, took 0.029293 s -2016-05-25 11:09:50 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:43 -2016-05-25 11:09:50 INFO DAGScheduler:59 - Got job 6 (count at DecisionTreesLearning.scala:43) with 2 output partitions -2016-05-25 11:09:50 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at DecisionTreesLearning.scala:43) -2016-05-25 11:09:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:09:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:09:50 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:24), which has no missing parents -2016-05-25 11:09:50 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1240951, maxMem=766075207 -2016-05-25 11:09:50 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-05-25 11:09:50 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=1245063, maxMem=766075207 -2016-05-25 11:09:50 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.4 MB) -2016-05-25 11:09:50 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53464 (size: 2.2 KB, free: 730.2 MB) -2016-05-25 11:09:50 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:09:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:24) -2016-05-25 11:09:50 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 11:09:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:09:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:09:50 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 11:09:50 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 11:09:50 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:09:50 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:09:50 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2082 bytes result sent to driver -2016-05-25 11:09:50 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2082 bytes result sent to driver -2016-05-25 11:09:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 9 ms on localhost (1/2) -2016-05-25 11:09:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 11 ms on localhost (2/2) -2016-05-25 11:09:50 INFO DAGScheduler:59 - ResultStage 7 (count at DecisionTreesLearning.scala:43) finished in 0.011 s -2016-05-25 11:09:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 11:09:50 INFO DAGScheduler:59 - Job 6 finished: count at DecisionTreesLearning.scala:43, took 0.027195 s -2016-05-25 11:09:51 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53464 in memory (size: 2.2 KB, free: 730.2 MB) -2016-05-25 11:09:51 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 11:09:51 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53464 in memory (size: 3.4 KB, free: 730.2 MB) -2016-05-25 11:09:51 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 11:09:51 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53464 in memory (size: 44.9 KB, free: 730.3 MB) -2016-05-25 11:09:51 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 11:09:51 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53464 in memory (size: 119.3 KB, free: 730.4 MB) -2016-05-25 11:09:51 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 11:09:51 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 11:09:51 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53464 in memory (size: 81.0 B, free: 730.4 MB) -2016-05-25 11:09:51 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 11:09:51 INFO ContextCleaner:59 - Cleaned RDD 13 -2016-05-25 11:09:51 INFO SparkContext:59 - Starting job: saveAsTextFile at DecisionTreeModel.scala:235 -2016-05-25 11:09:51 INFO DAGScheduler:59 - Got job 7 (saveAsTextFile at DecisionTreeModel.scala:235) with 1 output partitions -2016-05-25 11:09:51 INFO DAGScheduler:59 - Final stage: ResultStage 8(saveAsTextFile at DecisionTreeModel.scala:235) -2016-05-25 11:09:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:09:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:09:51 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[20] at saveAsTextFile at DecisionTreeModel.scala:235), which has no missing parents -2016-05-25 11:09:51 INFO MemoryStore:59 - ensureFreeSpace(126856) called with curMem=348358, maxMem=766075207 -2016-05-25 11:09:51 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 123.9 KB, free 730.1 MB) -2016-05-25 11:09:51 INFO MemoryStore:59 - ensureFreeSpace(42459) called with curMem=475214, maxMem=766075207 -2016-05-25 11:09:51 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 41.5 KB, free 730.1 MB) -2016-05-25 11:09:51 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53464 (size: 41.5 KB, free: 730.4 MB) -2016-05-25 11:09:51 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:09:51 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[20] at saveAsTextFile at DecisionTreeModel.scala:235) -2016-05-25 11:09:51 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-05-25 11:09:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:09:51 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 11:09:52 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251109_0008_m_000000_15' to file:/D:/all/idea/SparkLearning/myModelPath/metadata/_temporary/0/task_201605251109_0008_m_000000 -2016-05-25 11:09:52 INFO SparkHadoopMapRedUtil:59 - attempt_201605251109_0008_m_000000_15: Committed -2016-05-25 11:09:52 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 1864 bytes result sent to driver -2016-05-25 11:09:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 515 ms on localhost (1/1) -2016-05-25 11:09:52 INFO DAGScheduler:59 - ResultStage 8 (saveAsTextFile at DecisionTreeModel.scala:235) finished in 0.515 s -2016-05-25 11:09:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 11:09:52 INFO DAGScheduler:59 - Job 7 finished: saveAsTextFile at DecisionTreeModel.scala:235, took 0.600420 s -2016-05-25 11:09:53 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:09:53 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:09:53 INFO SparkContext:59 - Starting job: parquet at DecisionTreeModel.scala:242 -2016-05-25 11:09:53 INFO DAGScheduler:59 - Got job 8 (parquet at DecisionTreeModel.scala:242) with 4 output partitions -2016-05-25 11:09:53 INFO DAGScheduler:59 - Final stage: ResultStage 9(parquet at DecisionTreeModel.scala:242) -2016-05-25 11:09:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:09:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:09:53 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[23] at rddToDataFrameHolder at DecisionTreeModel.scala:240), which has no missing parents -2016-05-25 11:09:53 INFO MemoryStore:59 - ensureFreeSpace(68616) called with curMem=517673, maxMem=766075207 -2016-05-25 11:09:53 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 67.0 KB, free 730.0 MB) -2016-05-25 11:09:53 INFO MemoryStore:59 - ensureFreeSpace(24060) called with curMem=586289, maxMem=766075207 -2016-05-25 11:09:53 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 23.5 KB, free 730.0 MB) -2016-05-25 11:09:53 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53464 (size: 23.5 KB, free: 730.3 MB) -2016-05-25 11:09:53 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:09:53 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 9 (MapPartitionsRDD[23] at rddToDataFrameHolder at DecisionTreeModel.scala:240) -2016-05-25 11:09:53 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 4 tasks -2016-05-25 11:09:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 16, localhost, PROCESS_LOCAL, 2109 bytes) -2016-05-25 11:09:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 3790 bytes) -2016-05-25 11:09:53 INFO TaskSetManager:59 - Starting task 2.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2428 bytes) -2016-05-25 11:09:53 INFO TaskSetManager:59 - Starting task 3.0 in stage 9.0 (TID 19, localhost, PROCESS_LOCAL, 2428 bytes) -2016-05-25 11:09:53 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 16) -2016-05-25 11:09:53 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 17) -2016-05-25 11:09:53 INFO Executor:59 - Running task 2.0 in stage 9.0 (TID 18) -2016-05-25 11:09:53 INFO Executor:59 - Running task 3.0 in stage 9.0 (TID 19) -2016-05-25 11:09:53 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:09:53 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 11:09:53 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:09:53 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 11:09:53 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:09:53 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 11:09:53 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:09:53 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 11:09:53 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 11:09:53 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 11:09:53 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 11:09:53 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 11:09:53 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 11:09:54 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,568 -2016-05-25 11:09:54 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,620 -2016-05-25 11:09:54 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,620 -2016-05-25 11:09:54 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,676 -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 61B for [nodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, predict] DOUBLE: 1 values, 15B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 74B for [predict, prob] DOUBLE: 1 values, 15B raw, 33B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 75B for [impurity] DOUBLE: 1 values, 14B raw, 34B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 64B for [split, feature] INT32: 1 values, 11B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 61B for [nodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, predict] DOUBLE: 1 values, 15B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, prob] DOUBLE: 1 values, 15B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 69B for [impurity] DOUBLE: 1 values, 14B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 61B for [nodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, feature] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 69B for [predict, predict] DOUBLE: 1 values, 15B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, threshold] DOUBLE: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, prob] DOUBLE: 1 values, 15B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, featureType] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 69B for [impurity] DOUBLE: 1 values, 14B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 51B for [split, categories, bag, array] DOUBLE: 1 values, 14B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 69B for [split, threshold] DOUBLE: 1 values, 15B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, feature] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 61B for [split, featureType] INT32: 1 values, 11B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, threshold] DOUBLE: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 52B for [split, categories, bag, array] DOUBLE: 1 values, 14B raw, 31B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, featureType] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 61B for [leftNodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 47B for [leftNodeId] INT32: 1 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 51B for [split, categories, bag, array] DOUBLE: 1 values, 14B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 61B for [rightNodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 47B for [leftNodeId] INT32: 1 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 75B for [infoGain] DOUBLE: 1 values, 14B raw, 34B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 47B for [rightNodeId] INT32: 1 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 47B for [infoGain] DOUBLE: 1 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 47B for [rightNodeId] INT32: 1 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO ColumnChunkPageWriteStore:151 - written 47B for [infoGain] DOUBLE: 1 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:09:54 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251109_0009_m_000002_0' to file:/D:/all/idea/SparkLearning/myModelPath/data/_temporary/0/task_201605251109_0009_m_000002 -2016-05-25 11:09:54 INFO SparkHadoopMapRedUtil:59 - attempt_201605251109_0009_m_000002_0: Committed -2016-05-25 11:09:54 INFO Executor:59 - Finished task 2.0 in stage 9.0 (TID 18). 915 bytes result sent to driver -2016-05-25 11:09:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 9.0 (TID 18) in 636 ms on localhost (1/4) -2016-05-25 11:09:54 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251109_0009_m_000001_0' to file:/D:/all/idea/SparkLearning/myModelPath/data/_temporary/0/task_201605251109_0009_m_000001 -2016-05-25 11:09:54 INFO SparkHadoopMapRedUtil:59 - attempt_201605251109_0009_m_000001_0: Committed -2016-05-25 11:09:54 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251109_0009_m_000003_0' to file:/D:/all/idea/SparkLearning/myModelPath/data/_temporary/0/task_201605251109_0009_m_000003 -2016-05-25 11:09:54 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 17). 915 bytes result sent to driver -2016-05-25 11:09:54 INFO SparkHadoopMapRedUtil:59 - attempt_201605251109_0009_m_000003_0: Committed -2016-05-25 11:09:54 INFO Executor:59 - Finished task 3.0 in stage 9.0 (TID 19). 915 bytes result sent to driver -2016-05-25 11:09:54 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251109_0009_m_000000_0' to file:/D:/all/idea/SparkLearning/myModelPath/data/_temporary/0/task_201605251109_0009_m_000000 -2016-05-25 11:09:54 INFO SparkHadoopMapRedUtil:59 - attempt_201605251109_0009_m_000000_0: Committed -2016-05-25 11:09:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 17) in 643 ms on localhost (2/4) -2016-05-25 11:09:54 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 16). 915 bytes result sent to driver -2016-05-25 11:09:54 INFO TaskSetManager:59 - Finished task 3.0 in stage 9.0 (TID 19) in 644 ms on localhost (3/4) -2016-05-25 11:09:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 16) in 647 ms on localhost (4/4) -2016-05-25 11:09:54 INFO DAGScheduler:59 - ResultStage 9 (parquet at DecisionTreeModel.scala:242) finished in 0.647 s -2016-05-25 11:09:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 11:09:54 INFO DAGScheduler:59 - Job 8 finished: parquet at DecisionTreeModel.scala:242, took 0.696338 s -2016-05-25 11:09:54 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:09:54 INFO DefaultWriterContainer:59 - Job job_201605251109_0000 committed. -2016-05-25 11:09:54 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/myModelPath/data on driver -2016-05-25 11:09:54 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/myModelPath/data on driver -2016-05-25 11:09:54 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=610349, maxMem=766075207 -2016-05-25 11:09:54 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 225.3 KB, free 729.8 MB) -2016-05-25 11:09:54 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=772437, maxMem=766075207 -2016-05-25 11:09:54 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-25 11:09:54 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:53464 in memory (size: 23.5 KB, free: 730.4 MB) -2016-05-25 11:09:54 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 11:09:54 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53464 (size: 19.3 KB, free: 730.3 MB) -2016-05-25 11:09:54 INFO SparkContext:59 - Created broadcast 12 from textFile at modelSaveLoad.scala:129 -2016-05-25 11:09:54 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:53464 in memory (size: 41.5 KB, free: 730.4 MB) -2016-05-25 11:09:54 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 11:09:55 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 11:09:55 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 11:09:55 INFO DAGScheduler:59 - Got job 9 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 11:09:55 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at modelSaveLoad.scala:129) -2016-05-25 11:09:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:09:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:09:55 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[26] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 11:09:55 INFO MemoryStore:59 - ensureFreeSpace(3152) called with curMem=598850, maxMem=766075207 -2016-05-25 11:09:55 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 3.1 KB, free 730.0 MB) -2016-05-25 11:09:55 INFO MemoryStore:59 - ensureFreeSpace(1826) called with curMem=602002, maxMem=766075207 -2016-05-25 11:09:55 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 1826.0 B, free 730.0 MB) -2016-05-25 11:09:55 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53464 (size: 1826.0 B, free: 730.4 MB) -2016-05-25 11:09:55 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:09:55 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[26] at textFile at modelSaveLoad.scala:129) -2016-05-25 11:09:55 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-25 11:09:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2172 bytes) -2016-05-25 11:09:55 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 20) -2016-05-25 11:09:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/myModelPath/metadata/part-00000:0+55 -2016-05-25 11:09:55 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 20). 2157 bytes result sent to driver -2016-05-25 11:09:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 20) in 10 ms on localhost (1/1) -2016-05-25 11:09:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 11:09:55 INFO DAGScheduler:59 - ResultStage 10 (first at modelSaveLoad.scala:129) finished in 0.011 s -2016-05-25 11:09:55 INFO DAGScheduler:59 - Job 9 finished: first at modelSaveLoad.scala:129, took 0.026005 s -2016-05-25 11:09:55 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/myModelPath/data on driver -2016-05-25 11:09:55 INFO SparkContext:59 - Starting job: parquet at DecisionTreeModel.scala:249 -2016-05-25 11:09:55 INFO DAGScheduler:59 - Got job 10 (parquet at DecisionTreeModel.scala:249) with 4 output partitions -2016-05-25 11:09:55 INFO DAGScheduler:59 - Final stage: ResultStage 11(parquet at DecisionTreeModel.scala:249) -2016-05-25 11:09:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:09:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:09:55 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[28] at parquet at DecisionTreeModel.scala:249), which has no missing parents -2016-05-25 11:09:55 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=603828, maxMem=766075207 -2016-05-25 11:09:55 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-25 11:09:55 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=666828, maxMem=766075207 -2016-05-25 11:09:55 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 20.6 KB, free 729.9 MB) -2016-05-25 11:09:55 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53464 (size: 20.6 KB, free: 730.4 MB) -2016-05-25 11:09:55 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:09:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 11 (MapPartitionsRDD[28] at parquet at DecisionTreeModel.scala:249) -2016-05-25 11:09:55 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 4 tasks -2016-05-25 11:09:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 11:09:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 11:09:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 11.0 (TID 23, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 11:09:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 11.0 (TID 24, localhost, PROCESS_LOCAL, 2214 bytes) -2016-05-25 11:09:55 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-25 11:09:55 INFO Executor:59 - Running task 2.0 in stage 11.0 (TID 23) -2016-05-25 11:09:55 INFO Executor:59 - Running task 3.0 in stage 11.0 (TID 24) -2016-05-25 11:09:55 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:09:55 INFO Executor:59 - Finished task 2.0 in stage 11.0 (TID 23). 936 bytes result sent to driver -2016-05-25 11:09:55 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:09:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 11.0 (TID 23) in 43 ms on localhost (1/4) -2016-05-25 11:09:55 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-25 11:09:55 INFO Executor:59 - Finished task 3.0 in stage 11.0 (TID 24). 2321 bytes result sent to driver -2016-05-25 11:09:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 11.0 (TID 24) in 87 ms on localhost (2/4) -2016-05-25 11:09:55 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:09:55 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 936 bytes result sent to driver -2016-05-25 11:09:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 100 ms on localhost (3/4) -2016-05-25 11:09:55 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:09:55 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 936 bytes result sent to driver -2016-05-25 11:09:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 117 ms on localhost (4/4) -2016-05-25 11:09:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 11:09:55 INFO DAGScheduler:59 - ResultStage 11 (parquet at DecisionTreeModel.scala:249) finished in 0.117 s -2016-05-25 11:09:55 INFO DAGScheduler:59 - Job 10 finished: parquet at DecisionTreeModel.scala:249, took 0.155978 s -2016-05-25 11:09:55 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=687901, maxMem=766075207 -2016-05-25 11:09:55 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 86.4 KB, free 729.8 MB) -2016-05-25 11:09:55 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=776413, maxMem=766075207 -2016-05-25 11:09:55 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.8 MB) -2016-05-25 11:09:55 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53464 (size: 19.3 KB, free: 730.3 MB) -2016-05-25 11:09:55 INFO SparkContext:59 - Created broadcast 15 from map at DecisionTreeModel.scala:252 -2016-05-25 11:09:55 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 11:09:55 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/myModelPath/data/part-r-00000-832053d3-ca45-4c24-abd2-cd3b0065a7c0.gz.parquet, file:/D:/all/idea/SparkLearning/myModelPath/data/part-r-00001-832053d3-ca45-4c24-abd2-cd3b0065a7c0.gz.parquet, file:/D:/all/idea/SparkLearning/myModelPath/data/part-r-00003-832053d3-ca45-4c24-abd2-cd3b0065a7c0.gz.parquet, file:/D:/all/idea/SparkLearning/myModelPath/data/part-r-00002-832053d3-ca45-4c24-abd2-cd3b0065a7c0.gz.parquet -2016-05-25 11:09:55 INFO SparkContext:59 - Starting job: collect at DecisionTreeModel.scala:267 -2016-05-25 11:09:55 INFO DAGScheduler:59 - Registering RDD 32 (groupBy at DecisionTreeModel.scala:265) -2016-05-25 11:09:55 INFO DAGScheduler:59 - Got job 11 (collect at DecisionTreeModel.scala:267) with 4 output partitions -2016-05-25 11:09:55 INFO DAGScheduler:59 - Final stage: ResultStage 13(collect at DecisionTreeModel.scala:267) -2016-05-25 11:09:55 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-05-25 11:09:55 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-05-25 11:09:55 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (MapPartitionsRDD[32] at groupBy at DecisionTreeModel.scala:265), which has no missing parents -2016-05-25 11:09:55 INFO MemoryStore:59 - ensureFreeSpace(7248) called with curMem=796201, maxMem=766075207 -2016-05-25 11:09:55 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 7.1 KB, free 729.8 MB) -2016-05-25 11:09:55 INFO MemoryStore:59 - ensureFreeSpace(3790) called with curMem=803449, maxMem=766075207 -2016-05-25 11:09:55 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.7 KB, free 729.8 MB) -2016-05-25 11:09:55 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53464 (size: 3.7 KB, free: 730.3 MB) -2016-05-25 11:09:55 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:09:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 12 (MapPartitionsRDD[32] at groupBy at DecisionTreeModel.scala:265) -2016-05-25 11:09:55 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 4 tasks -2016-05-25 11:09:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 25, localhost, PROCESS_LOCAL, 2243 bytes) -2016-05-25 11:09:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 26, localhost, PROCESS_LOCAL, 2243 bytes) -2016-05-25 11:09:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 12.0 (TID 27, localhost, PROCESS_LOCAL, 2242 bytes) -2016-05-25 11:09:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 12.0 (TID 28, localhost, PROCESS_LOCAL, 2242 bytes) -2016-05-25 11:09:55 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 26) -2016-05-25 11:09:55 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/myModelPath/data/part-r-00001-832053d3-ca45-4c24-abd2-cd3b0065a7c0.gz.parquet start: 0 end: 3045 length: 3045 hosts: []} -2016-05-25 11:09:55 INFO Executor:59 - Running task 2.0 in stage 12.0 (TID 27) -2016-05-25 11:09:55 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 11:09:55 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/myModelPath/data/part-r-00003-832053d3-ca45-4c24-abd2-cd3b0065a7c0.gz.parquet start: 0 end: 2838 length: 2838 hosts: []} -2016-05-25 11:09:55 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 11:09:55 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 25) -2016-05-25 11:09:56 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 treeId; - optional int32 nodeId; - optional group predict { - optional double predict; - optional double prob; - } - optional double impurity; - optional boolean isLeaf; - optional group split { - optional int32 feature; - optional double threshold; - optional int32 featureType; - optional group categories (LIST) { - repeated group bag { - optional double array; - } - } - } - optional int32 leftNodeId; - optional int32 rightNodeId; - optional double infoGain; -} - -Catalyst form: -StructType(StructField(treeId,IntegerType,true), StructField(nodeId,IntegerType,true), StructField(predict,StructType(StructField(predict,DoubleType,true), StructField(prob,DoubleType,true)),true), StructField(impurity,DoubleType,true), StructField(isLeaf,BooleanType,true), StructField(split,StructType(StructField(feature,IntegerType,true), StructField(threshold,DoubleType,true), StructField(featureType,IntegerType,true), StructField(categories,ArrayType(DoubleType,true),true)),true), StructField(leftNodeId,IntegerType,true), StructField(rightNodeId,IntegerType,true), StructField(infoGain,DoubleType,true)) - -2016-05-25 11:09:56 INFO Executor:59 - Running task 3.0 in stage 12.0 (TID 28) -2016-05-25 11:09:56 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 treeId; - optional int32 nodeId; - optional group predict { - optional double predict; - optional double prob; - } - optional double impurity; - optional boolean isLeaf; - optional group split { - optional int32 feature; - optional double threshold; - optional int32 featureType; - optional group categories (LIST) { - repeated group bag { - optional double array; - } - } - } - optional int32 leftNodeId; - optional int32 rightNodeId; - optional double infoGain; -} - -Catalyst form: -StructType(StructField(treeId,IntegerType,true), StructField(nodeId,IntegerType,true), StructField(predict,StructType(StructField(predict,DoubleType,true), StructField(prob,DoubleType,true)),true), StructField(impurity,DoubleType,true), StructField(isLeaf,BooleanType,true), StructField(split,StructType(StructField(feature,IntegerType,true), StructField(threshold,DoubleType,true), StructField(featureType,IntegerType,true), StructField(categories,ArrayType(DoubleType,true),true)),true), StructField(leftNodeId,IntegerType,true), StructField(rightNodeId,IntegerType,true), StructField(infoGain,DoubleType,true)) - -2016-05-25 11:09:56 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53464 in memory (size: 2.1 KB, free: 730.3 MB) -2016-05-25 11:09:56 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/myModelPath/data/part-r-00002-832053d3-ca45-4c24-abd2-cd3b0065a7c0.gz.parquet start: 0 end: 2836 length: 2836 hosts: []} -2016-05-25 11:09:56 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/myModelPath/data/part-r-00000-832053d3-ca45-4c24-abd2-cd3b0065a7c0.gz.parquet start: 0 end: 1461 length: 1461 hosts: []} -2016-05-25 11:09:56 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 11:09:56 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 11:09:56 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 11:09:56 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 11:09:56 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 treeId; - optional int32 nodeId; - optional group predict { - optional double predict; - optional double prob; - } - optional double impurity; - optional boolean isLeaf; - optional group split { - optional int32 feature; - optional double threshold; - optional int32 featureType; - optional group categories (LIST) { - repeated group bag { - optional double array; - } - } - } - optional int32 leftNodeId; - optional int32 rightNodeId; - optional double infoGain; -} - -Catalyst form: -StructType(StructField(treeId,IntegerType,true), StructField(nodeId,IntegerType,true), StructField(predict,StructType(StructField(predict,DoubleType,true), StructField(prob,DoubleType,true)),true), StructField(impurity,DoubleType,true), StructField(isLeaf,BooleanType,true), StructField(split,StructType(StructField(feature,IntegerType,true), StructField(threshold,DoubleType,true), StructField(featureType,IntegerType,true), StructField(categories,ArrayType(DoubleType,true),true)),true), StructField(leftNodeId,IntegerType,true), StructField(rightNodeId,IntegerType,true), StructField(infoGain,DoubleType,true)) - -2016-05-25 11:09:56 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:53464 in memory (size: 20.6 KB, free: 730.4 MB) -2016-05-25 11:09:56 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 11:09:56 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 0 records. -2016-05-25 11:09:56 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 11:09:56 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 treeId; - optional int32 nodeId; - optional group predict { - optional double predict; - optional double prob; - } - optional double impurity; - optional boolean isLeaf; - optional group split { - optional int32 feature; - optional double threshold; - optional int32 featureType; - optional group categories (LIST) { - repeated group bag { - optional double array; - } - } - } - optional int32 leftNodeId; - optional int32 rightNodeId; - optional double infoGain; -} - -Catalyst form: -StructType(StructField(treeId,IntegerType,true), StructField(nodeId,IntegerType,true), StructField(predict,StructType(StructField(predict,DoubleType,true), StructField(prob,DoubleType,true)),true), StructField(impurity,DoubleType,true), StructField(isLeaf,BooleanType,true), StructField(split,StructType(StructField(feature,IntegerType,true), StructField(threshold,DoubleType,true), StructField(featureType,IntegerType,true), StructField(categories,ArrayType(DoubleType,true),true)),true), StructField(leftNodeId,IntegerType,true), StructField(rightNodeId,IntegerType,true), StructField(infoGain,DoubleType,true)) - -2016-05-25 11:09:56 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-25 11:09:56 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 11:09:56 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 11:09:56 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 11:09:56 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 11:09:56 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:53464 in memory (size: 1826.0 B, free: 730.4 MB) -2016-05-25 11:09:56 INFO InternalParquetRecordReader:151 - block read in memory in 13 ms. row count = 1 -2016-05-25 11:09:56 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 11:09:56 INFO InternalParquetRecordReader:151 - block read in memory in 8 ms. row count = 1 -2016-05-25 11:09:56 INFO InternalParquetRecordReader:151 - block read in memory in 3 ms. row count = 1 -2016-05-25 11:09:56 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 11:09:56 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:53464 in memory (size: 19.3 KB, free: 730.4 MB) -2016-05-25 11:09:56 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 11:09:56 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 25). 2256 bytes result sent to driver -2016-05-25 11:09:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 25) in 701 ms on localhost (1/4) -2016-05-25 11:09:56 INFO Executor:59 - Finished task 3.0 in stage 12.0 (TID 28). 2256 bytes result sent to driver -2016-05-25 11:09:56 INFO Executor:59 - Finished task 2.0 in stage 12.0 (TID 27). 2256 bytes result sent to driver -2016-05-25 11:09:56 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 26). 2256 bytes result sent to driver -2016-05-25 11:09:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 12.0 (TID 28) in 831 ms on localhost (2/4) -2016-05-25 11:09:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 12.0 (TID 27) in 833 ms on localhost (3/4) -2016-05-25 11:09:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 26) in 835 ms on localhost (4/4) -2016-05-25 11:09:56 INFO DAGScheduler:59 - ShuffleMapStage 12 (groupBy at DecisionTreeModel.scala:265) finished in 0.838 s -2016-05-25 11:09:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 11:09:56 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:09:56 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:09:56 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-05-25 11:09:56 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:09:56 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-05-25 11:09:56 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[34] at mapValues at DecisionTreeModel.scala:266), which is now runnable -2016-05-25 11:09:57 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:53464 in memory (size: 3.7 KB, free: 730.4 MB) -2016-05-25 11:10:11 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 11:10:12 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 11:10:12 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 11:10:12 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 11:10:12 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 11:10:13 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 11:10:13 INFO Remoting:74 - Starting remoting -2016-05-25 11:10:13 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53484] -2016-05-25 11:10:13 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53484. -2016-05-25 11:10:13 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 11:10:13 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 11:10:13 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-2fd8dcf6-8ed4-4074-8252-49cc8cb82b4a -2016-05-25 11:10:13 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 11:10:13 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-02c6f5cf-89eb-4d5a-9ff7-9fbccb9ce1f5\httpd-7c6c5f68-e866-4852-b209-18e3d4b94ca3 -2016-05-25 11:10:13 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 11:10:13 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:10:13 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53485 -2016-05-25 11:10:13 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53485. -2016-05-25 11:10:13 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 11:10:13 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:10:13 WARN AbstractLifeCycle:204 - FAILED SelectChannelConnector@0.0.0.0:4040: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.mllib.learning.classification.DecisionTreesLearning$.main(DecisionTreesLearning.scala:19) - at org.apache.spark.mllib.learning.classification.DecisionTreesLearning.main(DecisionTreesLearning.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-05-25 11:10:13 WARN AbstractLifeCycle:204 - FAILED org.spark-project.jetty.server.Server@71a12d23: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.mllib.learning.classification.DecisionTreesLearning$.main(DecisionTreesLearning.scala:19) - at org.apache.spark.mllib.learning.classification.DecisionTreesLearning.main(DecisionTreesLearning.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 11:10:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 11:10:14 WARN Utils:71 - Service 'SparkUI' could not bind on port 4040. Attempting port 4041. -2016-05-25 11:10:14 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:10:14 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4041 -2016-05-25 11:10:14 INFO Utils:59 - Successfully started service 'SparkUI' on port 4041. -2016-05-25 11:10:14 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4041 -2016-05-25 11:10:14 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 11:10:14 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 11:10:14 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53504. -2016-05-25 11:10:14 INFO NettyBlockTransferService:59 - Server created on 53504 -2016-05-25 11:10:14 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 11:10:14 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53504 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53504) -2016-05-25 11:10:14 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 11:10:15 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 11:10:15 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 11:10:15 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 11:10:15 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 11:10:15 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53504 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 11:10:15 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 11:10:16 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 11:10:17 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 11:10:17 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 11:10:17 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 11:10:17 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 11:10:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:10:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:10:17 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 11:10:17 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 11:10:17 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 11:10:17 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 11:10:17 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 11:10:17 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53504 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:10:17 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:10:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 11:10:17 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 11:10:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:10:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:10:17 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 11:10:17 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 11:10:17 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 11:10:17 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 11:10:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 11:10:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 11:10:17 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 11:10:17 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 11:10:17 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 11:10:17 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 11:10:17 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 11:10:17 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177630, maxMem=766075207 -2016-05-25 11:10:17 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 11:10:17 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262710, maxMem=766075207 -2016-05-25 11:10:17 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:53504 (size: 83.1 KB, free: 730.5 MB) -2016-05-25 11:10:17 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 11:10:17 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:53504 (size: 83.6 KB, free: 730.4 MB) -2016-05-25 11:10:17 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 11:10:17 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 11:10:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 430 ms on localhost (1/2) -2016-05-25 11:10:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 414 ms on localhost (2/2) -2016-05-25 11:10:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 11:10:17 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.451 s -2016-05-25 11:10:17 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.599697 s -2016-05-25 11:10:17 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 11:10:17 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 11:10:17 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 11:10:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:10:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:10:17 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 11:10:17 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348358, maxMem=766075207 -2016-05-25 11:10:17 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 11:10:17 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=353126, maxMem=766075207 -2016-05-25 11:10:17 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 11:10:17 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53504 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:10:17 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:10:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 11:10:17 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 11:10:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:10:17 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 11:10:17 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:10:17 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 11:10:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 26 ms on localhost (1/1) -2016-05-25 11:10:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 11:10:17 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.027 s -2016-05-25 11:10:17 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.045883 s -2016-05-25 11:10:17 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 11:10:17 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 11:10:17 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 11:10:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:10:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:10:17 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 11:10:17 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355673, maxMem=766075207 -2016-05-25 11:10:17 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 11:10:17 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=360009, maxMem=766075207 -2016-05-25 11:10:17 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 11:10:17 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53504 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:10:17 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:10:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 11:10:17 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 11:10:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:10:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:10:17 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 11:10:17 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 11:10:17 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:10:17 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:10:17 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 11:10:17 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 11:10:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 13 ms on localhost (1/2) -2016-05-25 11:10:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 14 ms on localhost (2/2) -2016-05-25 11:10:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 11:10:17 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.014 s -2016-05-25 11:10:17 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.029089 s -2016-05-25 11:10:17 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 11:10:17 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 11:10:17 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 11:10:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:10:17 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:10:17 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 11:10:17 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362328, maxMem=766075207 -2016-05-25 11:10:17 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 11:10:17 INFO MemoryStore:59 - ensureFreeSpace(2630) called with curMem=367392, maxMem=766075207 -2016-05-25 11:10:17 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 11:10:17 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53504 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:10:17 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:10:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 11:10:17 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 11:10:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:10:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:10:17 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 11:10:17 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 11:10:17 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:10:17 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 68689 bytes result sent to driver -2016-05-25 11:10:17 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:10:17 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 61975 bytes result sent to driver -2016-05-25 11:10:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 32 ms on localhost (1/2) -2016-05-25 11:10:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 38 ms on localhost (2/2) -2016-05-25 11:10:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 11:10:17 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.039 s -2016-05-25 11:10:17 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.065002 s -2016-05-25 11:10:19 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53504 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:10:19 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 11:10:19 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53504 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:10:19 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 11:10:19 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53504 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:10:19 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 11:10:19 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53504 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-25 11:10:19 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=342308, maxMem=766075207 -2016-05-25 11:10:19 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 11:10:19 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=342348, maxMem=766075207 -2016-05-25 11:10:19 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 11:10:19 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53504 (size: 81.0 B, free: 730.4 MB) -2016-05-25 11:10:19 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 11:10:19 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 11:10:19 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 11:10:19 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 11:10:19 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 11:10:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 11:10:19 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 11:10:19 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 11:10:19 INFO MemoryStore:59 - ensureFreeSpace(493576) called with curMem=342429, maxMem=766075207 -2016-05-25 11:10:19 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 482.0 KB, free 729.8 MB) -2016-05-25 11:10:19 INFO MemoryStore:59 - ensureFreeSpace(121051) called with curMem=836005, maxMem=766075207 -2016-05-25 11:10:19 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 118.2 KB, free 729.7 MB) -2016-05-25 11:10:19 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53504 (size: 118.2 KB, free: 730.3 MB) -2016-05-25 11:10:19 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:10:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 11:10:19 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 11:10:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:10:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:10:19 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 11:10:19 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 11:10:19 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 11:10:19 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:10:19 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 11:10:19 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:10:19 INFO MemoryStore:59 - ensureFreeSpace(111560) called with curMem=957056, maxMem=766075207 -2016-05-25 11:10:19 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 108.9 KB, free 729.6 MB) -2016-05-25 11:10:19 INFO MemoryStore:59 - ensureFreeSpace(102976) called with curMem=1068616, maxMem=766075207 -2016-05-25 11:10:19 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 100.6 KB, free 729.5 MB) -2016-05-25 11:10:19 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:53504 (size: 108.9 KB, free: 730.2 MB) -2016-05-25 11:10:19 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:53504 (size: 100.6 KB, free: 730.1 MB) -2016-05-25 11:10:19 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 11:10:19 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 11:10:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 264 ms on localhost (1/2) -2016-05-25 11:10:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 263 ms on localhost (2/2) -2016-05-25 11:10:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 11:10:19 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.266 s -2016-05-25 11:10:19 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:10:19 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:10:19 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 11:10:19 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:10:19 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 11:10:19 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 11:10:19 INFO MemoryStore:59 - ensureFreeSpace(214832) called with curMem=1171592, maxMem=766075207 -2016-05-25 11:10:19 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 209.8 KB, free 729.3 MB) -2016-05-25 11:10:19 INFO MemoryStore:59 - ensureFreeSpace(45685) called with curMem=1386424, maxMem=766075207 -2016-05-25 11:10:19 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 44.6 KB, free 729.2 MB) -2016-05-25 11:10:19 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53504 (size: 44.6 KB, free: 730.0 MB) -2016-05-25 11:10:19 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:10:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 11:10:19 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 11:10:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:10:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:10:19 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 11:10:19 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 11:10:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:10:20 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:10:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 11:10:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 11:10:20 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 11:10:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 150 ms on localhost (1/2) -2016-05-25 11:10:20 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 11:10:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 454 ms on localhost (2/2) -2016-05-25 11:10:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 11:10:20 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.455 s -2016-05-25 11:10:20 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.784663 s -2016-05-25 11:10:20 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 11:10:20 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 11:10:20 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 11:10:20 INFO RandomForest:59 - init: 1.594254206 - total: 2.612448088 - findSplitsBins: 1.377079296 - findBestSplits: 0.984302679 - chooseSplits: 0.977169074 -2016-05-25 11:10:20 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:43 -2016-05-25 11:10:20 INFO DAGScheduler:59 - Got job 5 (count at DecisionTreesLearning.scala:43) with 2 output partitions -2016-05-25 11:10:20 INFO DAGScheduler:59 - Final stage: ResultStage 6(count at DecisionTreesLearning.scala:43) -2016-05-25 11:10:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:10:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:10:20 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[18] at filter at DecisionTreesLearning.scala:43), which has no missing parents -2016-05-25 11:10:20 INFO MemoryStore:59 - ensureFreeSpace(6520) called with curMem=1217573, maxMem=766075207 -2016-05-25 11:10:20 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 6.4 KB, free 729.4 MB) -2016-05-25 11:10:20 INFO MemoryStore:59 - ensureFreeSpace(3484) called with curMem=1224093, maxMem=766075207 -2016-05-25 11:10:20 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-25 11:10:20 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53504 (size: 3.4 KB, free: 730.2 MB) -2016-05-25 11:10:20 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:10:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[18] at filter at DecisionTreesLearning.scala:43) -2016-05-25 11:10:20 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 11:10:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:10:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:10:20 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 11:10:20 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 11:10:20 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:10:20 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2082 bytes result sent to driver -2016-05-25 11:10:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 13 ms on localhost (1/2) -2016-05-25 11:10:20 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:10:20 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2082 bytes result sent to driver -2016-05-25 11:10:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 23 ms on localhost (2/2) -2016-05-25 11:10:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 11:10:20 INFO DAGScheduler:59 - ResultStage 6 (count at DecisionTreesLearning.scala:43) finished in 0.023 s -2016-05-25 11:10:20 INFO DAGScheduler:59 - Job 5 finished: count at DecisionTreesLearning.scala:43, took 0.045010 s -2016-05-25 11:10:20 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:43 -2016-05-25 11:10:20 INFO DAGScheduler:59 - Got job 6 (count at DecisionTreesLearning.scala:43) with 2 output partitions -2016-05-25 11:10:20 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at DecisionTreesLearning.scala:43) -2016-05-25 11:10:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:10:20 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:10:20 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:24), which has no missing parents -2016-05-25 11:10:20 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1227577, maxMem=766075207 -2016-05-25 11:10:20 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-05-25 11:10:20 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=1231689, maxMem=766075207 -2016-05-25 11:10:20 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.4 MB) -2016-05-25 11:10:20 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53504 (size: 2.2 KB, free: 730.2 MB) -2016-05-25 11:10:20 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:10:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:24) -2016-05-25 11:10:20 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 11:10:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:10:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:10:20 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 11:10:20 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 11:10:20 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:10:20 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:10:20 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2082 bytes result sent to driver -2016-05-25 11:10:20 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2082 bytes result sent to driver -2016-05-25 11:10:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 9 ms on localhost (1/2) -2016-05-25 11:10:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 11 ms on localhost (2/2) -2016-05-25 11:10:20 INFO DAGScheduler:59 - ResultStage 7 (count at DecisionTreesLearning.scala:43) finished in 0.013 s -2016-05-25 11:10:20 INFO DAGScheduler:59 - Job 6 finished: count at DecisionTreesLearning.scala:43, took 0.030759 s -2016-05-25 11:10:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 11:10:20 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 11:10:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 11:10:20 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4041 -2016-05-25 11:10:20 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 11:10:21 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 11:10:21 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 11:10:21 INFO BlockManager:59 - BlockManager stopped -2016-05-25 11:10:21 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 11:10:21 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 11:10:21 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 11:10:21 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 11:10:21 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-02c6f5cf-89eb-4d5a-9ff7-9fbccb9ce1f5 -2016-05-25 11:10:21 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 11:10:28 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 11:10:29 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 11:10:29 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 11:10:29 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 11:10:29 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 11:10:30 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 11:10:30 INFO Remoting:74 - Starting remoting -2016-05-25 11:10:30 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53524] -2016-05-25 11:10:30 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53524. -2016-05-25 11:10:30 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 11:10:30 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 11:10:30 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-273a0227-7d0a-4efc-8d44-51ff03973f41 -2016-05-25 11:10:30 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 11:10:31 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-60e06cb3-b782-4d47-aacc-b0040d7e19a8\httpd-ce86c082-9b34-47a5-b04a-1b0874f150a4 -2016-05-25 11:10:31 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 11:10:31 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:10:31 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53525 -2016-05-25 11:10:31 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53525. -2016-05-25 11:10:31 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 11:10:31 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:10:31 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 11:10:31 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 11:10:31 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 11:10:31 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 11:10:31 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 11:10:31 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53544. -2016-05-25 11:10:31 INFO NettyBlockTransferService:59 - Server created on 53544 -2016-05-25 11:10:31 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 11:10:31 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53544 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53544) -2016-05-25 11:10:31 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 11:10:32 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 11:10:32 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 11:10:32 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 11:10:32 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 11:10:32 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53544 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 11:10:32 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 11:10:33 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 11:10:34 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 11:10:34 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 11:10:34 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 11:10:34 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 11:10:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:10:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:10:34 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 11:10:34 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 11:10:34 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 11:10:34 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 11:10:34 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 11:10:34 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53544 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:10:34 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:10:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 11:10:34 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 11:10:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:10:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:10:34 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 11:10:34 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 11:10:34 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 11:10:34 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 11:10:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 11:10:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 11:10:34 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 11:10:34 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 11:10:34 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 11:10:34 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 11:10:34 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 11:10:34 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177630, maxMem=766075207 -2016-05-25 11:10:34 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 11:10:34 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:53544 (size: 83.6 KB, free: 730.5 MB) -2016-05-25 11:10:34 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263278, maxMem=766075207 -2016-05-25 11:10:34 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 11:10:34 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:53544 (size: 83.1 KB, free: 730.4 MB) -2016-05-25 11:10:34 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 11:10:34 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 11:10:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 370 ms on localhost (1/2) -2016-05-25 11:10:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 355 ms on localhost (2/2) -2016-05-25 11:10:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 11:10:34 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.396 s -2016-05-25 11:10:34 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.579461 s -2016-05-25 11:10:35 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 11:10:35 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 11:10:35 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 11:10:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:10:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:10:35 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 11:10:35 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348358, maxMem=766075207 -2016-05-25 11:10:35 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 11:10:35 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=353126, maxMem=766075207 -2016-05-25 11:10:35 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 11:10:35 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53544 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:10:35 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:10:35 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 11:10:35 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 11:10:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:10:35 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 11:10:35 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:10:35 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 11:10:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 34 ms on localhost (1/1) -2016-05-25 11:10:35 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.034 s -2016-05-25 11:10:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 11:10:35 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.057255 s -2016-05-25 11:10:35 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 11:10:35 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 11:10:35 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 11:10:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:10:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:10:35 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 11:10:35 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355673, maxMem=766075207 -2016-05-25 11:10:35 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 11:10:35 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=360009, maxMem=766075207 -2016-05-25 11:10:35 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 11:10:35 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53544 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:10:35 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:10:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 11:10:35 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 11:10:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:10:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:10:35 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 11:10:35 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 11:10:35 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:10:35 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:10:35 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 11:10:35 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 11:10:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 19 ms on localhost (1/2) -2016-05-25 11:10:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 21 ms on localhost (2/2) -2016-05-25 11:10:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 11:10:35 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.022 s -2016-05-25 11:10:35 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.040847 s -2016-05-25 11:10:35 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 11:10:35 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 11:10:35 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 11:10:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:10:35 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:10:35 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 11:10:35 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362328, maxMem=766075207 -2016-05-25 11:10:35 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 11:10:35 INFO MemoryStore:59 - ensureFreeSpace(2630) called with curMem=367392, maxMem=766075207 -2016-05-25 11:10:35 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 11:10:35 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53544 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:10:35 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:10:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 11:10:35 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 11:10:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:10:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:10:35 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 11:10:35 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 11:10:35 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:10:35 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:10:35 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 62023 bytes result sent to driver -2016-05-25 11:10:35 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 59360 bytes result sent to driver -2016-05-25 11:10:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 31 ms on localhost (1/2) -2016-05-25 11:10:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 28 ms on localhost (2/2) -2016-05-25 11:10:35 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.031 s -2016-05-25 11:10:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 11:10:35 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.048134 s -2016-05-25 11:10:36 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53544 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:10:36 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 11:10:36 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53544 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:10:36 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 11:10:36 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53544 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:10:36 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 11:10:36 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=348358, maxMem=766075207 -2016-05-25 11:10:36 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 11:10:36 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=348398, maxMem=766075207 -2016-05-25 11:10:36 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 11:10:36 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53544 (size: 81.0 B, free: 730.4 MB) -2016-05-25 11:10:36 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 11:10:36 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 11:10:36 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 11:10:36 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 11:10:36 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 11:10:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 11:10:36 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 11:10:36 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 11:10:36 INFO MemoryStore:59 - ensureFreeSpace(476304) called with curMem=348479, maxMem=766075207 -2016-05-25 11:10:36 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 465.1 KB, free 729.8 MB) -2016-05-25 11:10:36 INFO MemoryStore:59 - ensureFreeSpace(117081) called with curMem=824783, maxMem=766075207 -2016-05-25 11:10:36 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 114.3 KB, free 729.7 MB) -2016-05-25 11:10:36 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53544 (size: 114.3 KB, free: 730.3 MB) -2016-05-25 11:10:36 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:10:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 11:10:36 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 11:10:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:10:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:10:36 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 11:10:36 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 11:10:37 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 11:10:37 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:10:37 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 11:10:37 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:10:37 INFO MemoryStore:59 - ensureFreeSpace(100120) called with curMem=941864, maxMem=766075207 -2016-05-25 11:10:37 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 97.8 KB, free 729.6 MB) -2016-05-25 11:10:37 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:53544 (size: 97.8 KB, free: 730.2 MB) -2016-05-25 11:10:37 INFO MemoryStore:59 - ensureFreeSpace(102976) called with curMem=1041984, maxMem=766075207 -2016-05-25 11:10:37 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 100.6 KB, free 729.5 MB) -2016-05-25 11:10:37 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:53544 (size: 100.6 KB, free: 730.1 MB) -2016-05-25 11:10:37 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 11:10:37 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 11:10:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 254 ms on localhost (1/2) -2016-05-25 11:10:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 252 ms on localhost (2/2) -2016-05-25 11:10:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 11:10:37 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.255 s -2016-05-25 11:10:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:10:37 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:10:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 11:10:37 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:10:37 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 11:10:37 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 11:10:37 INFO MemoryStore:59 - ensureFreeSpace(206344) called with curMem=1144960, maxMem=766075207 -2016-05-25 11:10:37 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 201.5 KB, free 729.3 MB) -2016-05-25 11:10:37 INFO MemoryStore:59 - ensureFreeSpace(44020) called with curMem=1351304, maxMem=766075207 -2016-05-25 11:10:37 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 43.0 KB, free 729.3 MB) -2016-05-25 11:10:37 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53544 (size: 43.0 KB, free: 730.1 MB) -2016-05-25 11:10:37 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:10:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 11:10:37 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 11:10:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:10:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:10:37 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 11:10:37 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 11:10:37 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:10:37 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:10:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-25 11:10:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-25 11:10:37 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 11:10:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 140 ms on localhost (1/2) -2016-05-25 11:10:37 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 11:10:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 413 ms on localhost (2/2) -2016-05-25 11:10:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 11:10:37 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.413 s -2016-05-25 11:10:37 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.727794 s -2016-05-25 11:10:37 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 11:10:37 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 11:10:37 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 11:10:37 INFO RandomForest:59 - init: 1.619284962 - total: 2.589768875 - findSplitsBins: 1.331196569 - findBestSplits: 0.954984711 - chooseSplits: 0.948899706 -2016-05-25 11:10:37 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:43 -2016-05-25 11:10:37 INFO DAGScheduler:59 - Got job 5 (count at DecisionTreesLearning.scala:43) with 2 output partitions -2016-05-25 11:10:37 INFO DAGScheduler:59 - Final stage: ResultStage 6(count at DecisionTreesLearning.scala:43) -2016-05-25 11:10:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:10:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:10:37 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[18] at filter at DecisionTreesLearning.scala:43), which has no missing parents -2016-05-25 11:10:37 INFO MemoryStore:59 - ensureFreeSpace(6520) called with curMem=1192228, maxMem=766075207 -2016-05-25 11:10:37 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 6.4 KB, free 729.4 MB) -2016-05-25 11:10:37 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=1198748, maxMem=766075207 -2016-05-25 11:10:37 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-25 11:10:37 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53544 (size: 3.4 KB, free: 730.3 MB) -2016-05-25 11:10:37 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:10:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[18] at filter at DecisionTreesLearning.scala:43) -2016-05-25 11:10:37 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 11:10:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:10:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:10:37 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 11:10:37 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 11:10:37 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:10:37 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:10:37 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2082 bytes result sent to driver -2016-05-25 11:10:37 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2082 bytes result sent to driver -2016-05-25 11:10:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 12 ms on localhost (1/2) -2016-05-25 11:10:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 11 ms on localhost (2/2) -2016-05-25 11:10:37 INFO DAGScheduler:59 - ResultStage 6 (count at DecisionTreesLearning.scala:43) finished in 0.013 s -2016-05-25 11:10:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 11:10:37 INFO DAGScheduler:59 - Job 5 finished: count at DecisionTreesLearning.scala:43, took 0.033840 s -2016-05-25 11:10:37 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:43 -2016-05-25 11:10:37 INFO DAGScheduler:59 - Got job 6 (count at DecisionTreesLearning.scala:43) with 2 output partitions -2016-05-25 11:10:37 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at DecisionTreesLearning.scala:43) -2016-05-25 11:10:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:10:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:10:37 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:24), which has no missing parents -2016-05-25 11:10:37 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1202235, maxMem=766075207 -2016-05-25 11:10:37 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-05-25 11:10:37 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=1206347, maxMem=766075207 -2016-05-25 11:10:37 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.4 MB) -2016-05-25 11:10:37 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53544 (size: 2.2 KB, free: 730.2 MB) -2016-05-25 11:10:37 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:10:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:24) -2016-05-25 11:10:37 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 11:10:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:10:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:10:37 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 11:10:37 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 11:10:37 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:10:37 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2082 bytes result sent to driver -2016-05-25 11:10:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 11 ms on localhost (1/2) -2016-05-25 11:10:37 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:10:37 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2082 bytes result sent to driver -2016-05-25 11:10:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 20 ms on localhost (2/2) -2016-05-25 11:10:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 11:10:37 INFO DAGScheduler:59 - ResultStage 7 (count at DecisionTreesLearning.scala:43) finished in 0.021 s -2016-05-25 11:10:37 INFO DAGScheduler:59 - Job 6 finished: count at DecisionTreesLearning.scala:43, took 0.038866 s -2016-05-25 11:10:38 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 11:10:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 11:10:38 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 11:10:38 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 11:10:38 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 11:10:38 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 11:10:38 INFO BlockManager:59 - BlockManager stopped -2016-05-25 11:10:38 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 11:10:38 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 11:10:38 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 11:10:38 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 11:10:38 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-60e06cb3-b782-4d47-aacc-b0040d7e19a8 -2016-05-25 11:12:28 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 11:12:30 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 11:12:30 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 11:12:30 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 11:12:30 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 11:12:31 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 11:12:31 INFO Remoting:74 - Starting remoting -2016-05-25 11:12:32 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53572] -2016-05-25 11:12:32 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53572. -2016-05-25 11:12:32 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 11:12:32 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 11:12:32 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-7d21473a-b550-4ee9-8667-b7fd7f6247d4 -2016-05-25 11:12:32 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 11:12:32 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-506e8042-4e64-45fa-8cd8-563770fe4bb2\httpd-3736023f-3932-4ac1-9794-8cb932a59442 -2016-05-25 11:12:32 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 11:12:32 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:12:32 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53574 -2016-05-25 11:12:32 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53574. -2016-05-25 11:12:32 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 11:12:33 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:12:33 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 11:12:33 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 11:12:33 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 11:12:33 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 11:12:33 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 11:12:34 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53594. -2016-05-25 11:12:34 INFO NettyBlockTransferService:59 - Server created on 53594 -2016-05-25 11:12:34 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 11:12:34 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53594 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53594) -2016-05-25 11:12:34 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 11:12:35 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 11:12:35 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 11:12:35 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 11:12:35 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 11:12:35 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53594 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 11:12:35 INFO SparkContext:59 - Created broadcast 0 from textFile at MultinomialNaiveBayesLearning.scala:26 -2016-05-25 11:12:37 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 11:12:37 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 11:12:37 INFO SparkContext:59 - Starting job: foreach at MultinomialNaiveBayesLearning.scala:32 -2016-05-25 11:12:37 INFO DAGScheduler:59 - Got job 0 (foreach at MultinomialNaiveBayesLearning.scala:32) with 2 output partitions -2016-05-25 11:12:37 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at MultinomialNaiveBayesLearning.scala:32) -2016-05-25 11:12:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:12:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:12:37 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at MultinomialNaiveBayesLearning.scala:27), which has no missing parents -2016-05-25 11:12:37 INFO MemoryStore:59 - ensureFreeSpace(3456) called with curMem=171580, maxMem=766075207 -2016-05-25 11:12:37 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-05-25 11:12:37 INFO MemoryStore:59 - ensureFreeSpace(2022) called with curMem=175036, maxMem=766075207 -2016-05-25 11:12:37 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2022.0 B, free 730.4 MB) -2016-05-25 11:12:37 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53594 (size: 2022.0 B, free: 730.6 MB) -2016-05-25 11:12:37 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:12:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at MultinomialNaiveBayesLearning.scala:27) -2016-05-25 11:12:37 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 11:12:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 11:12:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 11:12:37 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 11:12:37 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 11:12:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 11:12:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 11:12:37 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 11:12:37 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 11:12:37 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 11:12:37 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 11:12:37 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 11:12:37 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-05-25 11:12:37 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-05-25 11:12:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 122 ms on localhost (1/2) -2016-05-25 11:12:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 103 ms on localhost (2/2) -2016-05-25 11:12:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 11:12:37 INFO DAGScheduler:59 - ResultStage 0 (foreach at MultinomialNaiveBayesLearning.scala:32) finished in 0.150 s -2016-05-25 11:12:37 INFO DAGScheduler:59 - Job 0 finished: foreach at MultinomialNaiveBayesLearning.scala:32, took 0.398743 s -2016-05-25 11:12:38 INFO SparkContext:59 - Starting job: collect at NaiveBayes.scala:401 -2016-05-25 11:12:38 INFO DAGScheduler:59 - Registering RDD 5 (map at NaiveBayes.scala:383) -2016-05-25 11:12:38 INFO DAGScheduler:59 - Got job 1 (collect at NaiveBayes.scala:401) with 2 output partitions -2016-05-25 11:12:38 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at NaiveBayes.scala:401) -2016-05-25 11:12:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1) -2016-05-25 11:12:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1) -2016-05-25 11:12:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383), which has no missing parents -2016-05-25 11:12:38 INFO MemoryStore:59 - ensureFreeSpace(5112) called with curMem=177058, maxMem=766075207 -2016-05-25 11:12:38 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 11:12:38 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=182170, maxMem=766075207 -2016-05-25 11:12:38 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 11:12:38 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53594 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 11:12:38 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:12:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[5] at map at NaiveBayes.scala:383) -2016-05-25 11:12:38 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-25 11:12:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 11:12:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2194 bytes) -2016-05-25 11:12:38 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 11:12:38 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-25 11:12:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 11:12:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 11:12:38 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 11:12:38 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 11:12:38 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2254 bytes result sent to driver -2016-05-25 11:12:38 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2254 bytes result sent to driver -2016-05-25 11:12:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 96 ms on localhost (1/2) -2016-05-25 11:12:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 94 ms on localhost (2/2) -2016-05-25 11:12:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 11:12:38 INFO DAGScheduler:59 - ShuffleMapStage 1 (map at NaiveBayes.scala:383) finished in 0.099 s -2016-05-25 11:12:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:12:38 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:12:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-05-25 11:12:38 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:12:38 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List() -2016-05-25 11:12:38 INFO DAGScheduler:59 - Submitting ResultStage 2 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383), which is now runnable -2016-05-25 11:12:38 INFO MemoryStore:59 - ensureFreeSpace(2936) called with curMem=184866, maxMem=766075207 -2016-05-25 11:12:38 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-05-25 11:12:38 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=187802, maxMem=766075207 -2016-05-25 11:12:38 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.4 MB) -2016-05-25 11:12:38 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53594 (size: 1567.0 B, free: 730.6 MB) -2016-05-25 11:12:38 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:12:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (ShuffledRDD[6] at combineByKey at NaiveBayes.scala:383) -2016-05-25 11:12:38 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 11:12:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:12:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:12:38 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-25 11:12:38 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-25 11:12:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:12:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:12:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 11:12:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 11:12:38 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1161 bytes result sent to driver -2016-05-25 11:12:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 34 ms on localhost (1/2) -2016-05-25 11:12:38 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1645 bytes result sent to driver -2016-05-25 11:12:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 40 ms on localhost (2/2) -2016-05-25 11:12:38 INFO DAGScheduler:59 - ResultStage 2 (collect at NaiveBayes.scala:401) finished in 0.040 s -2016-05-25 11:12:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 11:12:38 INFO DAGScheduler:59 - Job 1 finished: collect at NaiveBayes.scala:401, took 0.194335 s -2016-05-25 11:12:38 INFO SparkContext:59 - Starting job: count at MultinomialNaiveBayesLearning.scala:40 -2016-05-25 11:12:38 INFO DAGScheduler:59 - Got job 2 (count at MultinomialNaiveBayesLearning.scala:40) with 2 output partitions -2016-05-25 11:12:38 INFO DAGScheduler:59 - Final stage: ResultStage 3(count at MultinomialNaiveBayesLearning.scala:40) -2016-05-25 11:12:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:12:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:12:38 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at filter at MultinomialNaiveBayesLearning.scala:40), which has no missing parents -2016-05-25 11:12:38 INFO MemoryStore:59 - ensureFreeSpace(4928) called with curMem=189369, maxMem=766075207 -2016-05-25 11:12:38 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-25 11:12:38 INFO MemoryStore:59 - ensureFreeSpace(2674) called with curMem=194297, maxMem=766075207 -2016-05-25 11:12:38 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 11:12:38 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53594 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 11:12:38 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:12:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at filter at MultinomialNaiveBayesLearning.scala:40) -2016-05-25 11:12:38 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 11:12:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 11:12:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 11:12:38 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-25 11:12:38 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-25 11:12:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 11:12:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 11:12:38 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 2082 bytes result sent to driver -2016-05-25 11:12:38 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 2082 bytes result sent to driver -2016-05-25 11:12:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 21 ms on localhost (1/2) -2016-05-25 11:12:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 21 ms on localhost (2/2) -2016-05-25 11:12:38 INFO DAGScheduler:59 - ResultStage 3 (count at MultinomialNaiveBayesLearning.scala:40) finished in 0.022 s -2016-05-25 11:12:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 11:12:38 INFO DAGScheduler:59 - Job 2 finished: count at MultinomialNaiveBayesLearning.scala:40, took 0.035535 s -2016-05-25 11:12:38 INFO SparkContext:59 - Starting job: count at MultinomialNaiveBayesLearning.scala:40 -2016-05-25 11:12:38 INFO DAGScheduler:59 - Got job 3 (count at MultinomialNaiveBayesLearning.scala:40) with 2 output partitions -2016-05-25 11:12:38 INFO DAGScheduler:59 - Final stage: ResultStage 4(count at MultinomialNaiveBayesLearning.scala:40) -2016-05-25 11:12:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:12:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:12:38 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[4] at randomSplit at MultinomialNaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 11:12:38 INFO MemoryStore:59 - ensureFreeSpace(3608) called with curMem=196971, maxMem=766075207 -2016-05-25 11:12:38 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 11:12:38 INFO MemoryStore:59 - ensureFreeSpace(2126) called with curMem=200579, maxMem=766075207 -2016-05-25 11:12:38 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 11:12:38 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53594 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:12:38 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:12:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[4] at randomSplit at MultinomialNaiveBayesLearning.scala:33) -2016-05-25 11:12:38 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 11:12:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 11:12:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 11:12:38 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-25 11:12:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 11:12:38 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2082 bytes result sent to driver -2016-05-25 11:12:38 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-25 11:12:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 11:12:38 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2082 bytes result sent to driver -2016-05-25 11:12:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 44 ms on localhost (1/2) -2016-05-25 11:12:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 43 ms on localhost (2/2) -2016-05-25 11:12:38 INFO DAGScheduler:59 - ResultStage 4 (count at MultinomialNaiveBayesLearning.scala:40) finished in 0.045 s -2016-05-25 11:12:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 11:12:38 INFO DAGScheduler:59 - Job 3 finished: count at MultinomialNaiveBayesLearning.scala:40, took 0.062609 s -2016-05-25 11:12:38 INFO SparkContext:59 - Starting job: count at MultinomialNaiveBayesLearning.scala:44 -2016-05-25 11:12:38 INFO DAGScheduler:59 - Got job 4 (count at MultinomialNaiveBayesLearning.scala:44) with 2 output partitions -2016-05-25 11:12:38 INFO DAGScheduler:59 - Final stage: ResultStage 5(count at MultinomialNaiveBayesLearning.scala:44) -2016-05-25 11:12:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:12:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:12:38 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[3] at randomSplit at MultinomialNaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 11:12:38 INFO MemoryStore:59 - ensureFreeSpace(3608) called with curMem=202705, maxMem=766075207 -2016-05-25 11:12:38 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 11:12:38 INFO MemoryStore:59 - ensureFreeSpace(2122) called with curMem=206313, maxMem=766075207 -2016-05-25 11:12:38 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 11:12:38 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53594 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:12:38 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:12:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[3] at randomSplit at MultinomialNaiveBayesLearning.scala:33) -2016-05-25 11:12:38 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 11:12:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 11:12:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 11:12:38 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-05-25 11:12:38 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-05-25 11:12:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 11:12:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 11:12:38 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 2082 bytes result sent to driver -2016-05-25 11:12:38 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 2082 bytes result sent to driver -2016-05-25 11:12:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 25 ms on localhost (1/2) -2016-05-25 11:12:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 25 ms on localhost (2/2) -2016-05-25 11:12:38 INFO DAGScheduler:59 - ResultStage 5 (count at MultinomialNaiveBayesLearning.scala:44) finished in 0.028 s -2016-05-25 11:12:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 11:12:38 INFO DAGScheduler:59 - Job 4 finished: count at MultinomialNaiveBayesLearning.scala:44, took 0.042008 s -2016-05-25 11:12:38 INFO SparkContext:59 - Starting job: foreach at MultinomialNaiveBayesLearning.scala:45 -2016-05-25 11:12:38 INFO DAGScheduler:59 - Got job 5 (foreach at MultinomialNaiveBayesLearning.scala:45) with 2 output partitions -2016-05-25 11:12:38 INFO DAGScheduler:59 - Final stage: ResultStage 6(foreach at MultinomialNaiveBayesLearning.scala:45) -2016-05-25 11:12:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:12:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:12:38 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[3] at randomSplit at MultinomialNaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 11:12:38 INFO MemoryStore:59 - ensureFreeSpace(3760) called with curMem=208435, maxMem=766075207 -2016-05-25 11:12:38 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-25 11:12:38 INFO MemoryStore:59 - ensureFreeSpace(2154) called with curMem=212195, maxMem=766075207 -2016-05-25 11:12:38 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 11:12:38 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53594 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:12:38 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:12:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[3] at randomSplit at MultinomialNaiveBayesLearning.scala:33) -2016-05-25 11:12:38 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 11:12:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 11:12:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 11:12:38 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-05-25 11:12:38 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 13) -2016-05-25 11:12:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 11:12:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 11:12:38 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 2044 bytes result sent to driver -2016-05-25 11:12:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 15 ms on localhost (1/2) -2016-05-25 11:12:38 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 13). 2044 bytes result sent to driver -2016-05-25 11:12:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 13) in 20 ms on localhost (2/2) -2016-05-25 11:12:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 11:12:38 INFO DAGScheduler:59 - ResultStage 6 (foreach at MultinomialNaiveBayesLearning.scala:45) finished in 0.024 s -2016-05-25 11:12:38 INFO DAGScheduler:59 - Job 5 finished: foreach at MultinomialNaiveBayesLearning.scala:45, took 0.039874 s -2016-05-25 11:12:38 INFO SparkContext:59 - Starting job: count at MultinomialNaiveBayesLearning.scala:46 -2016-05-25 11:12:38 INFO DAGScheduler:59 - Got job 6 (count at MultinomialNaiveBayesLearning.scala:46) with 2 output partitions -2016-05-25 11:12:38 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at MultinomialNaiveBayesLearning.scala:46) -2016-05-25 11:12:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:12:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:12:38 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[4] at randomSplit at MultinomialNaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 11:12:38 INFO MemoryStore:59 - ensureFreeSpace(3608) called with curMem=214349, maxMem=766075207 -2016-05-25 11:12:38 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 11:12:38 INFO MemoryStore:59 - ensureFreeSpace(2126) called with curMem=217957, maxMem=766075207 -2016-05-25 11:12:38 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 11:12:38 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53594 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:12:38 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:12:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[4] at randomSplit at MultinomialNaiveBayesLearning.scala:33) -2016-05-25 11:12:38 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 11:12:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 11:12:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 15, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 11:12:38 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 14) -2016-05-25 11:12:38 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 15) -2016-05-25 11:12:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 11:12:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 11:12:38 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 14). 2082 bytes result sent to driver -2016-05-25 11:12:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 14) in 21 ms on localhost (1/2) -2016-05-25 11:12:38 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 11:12:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 15) in 36 ms on localhost (2/2) -2016-05-25 11:12:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 11:12:38 INFO DAGScheduler:59 - ResultStage 7 (count at MultinomialNaiveBayesLearning.scala:46) finished in 0.040 s -2016-05-25 11:12:38 INFO DAGScheduler:59 - Job 6 finished: count at MultinomialNaiveBayesLearning.scala:46, took 0.062124 s -2016-05-25 11:12:38 INFO SparkContext:59 - Starting job: foreach at MultinomialNaiveBayesLearning.scala:47 -2016-05-25 11:12:38 INFO DAGScheduler:59 - Got job 7 (foreach at MultinomialNaiveBayesLearning.scala:47) with 2 output partitions -2016-05-25 11:12:38 INFO DAGScheduler:59 - Final stage: ResultStage 8(foreach at MultinomialNaiveBayesLearning.scala:47) -2016-05-25 11:12:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:12:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:12:38 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[4] at randomSplit at MultinomialNaiveBayesLearning.scala:33), which has no missing parents -2016-05-25 11:12:38 INFO MemoryStore:59 - ensureFreeSpace(3760) called with curMem=220083, maxMem=766075207 -2016-05-25 11:12:38 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-25 11:12:38 INFO MemoryStore:59 - ensureFreeSpace(2158) called with curMem=223843, maxMem=766075207 -2016-05-25 11:12:38 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 11:12:38 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53594 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:12:38 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:12:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[4] at randomSplit at MultinomialNaiveBayesLearning.scala:33) -2016-05-25 11:12:38 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 11:12:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 11:12:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 17, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 11:12:38 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 16) -2016-05-25 11:12:38 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 17) -2016-05-25 11:12:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 11:12:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 11:12:38 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 17). 2044 bytes result sent to driver -2016-05-25 11:12:38 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 16). 2044 bytes result sent to driver -2016-05-25 11:12:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 17) in 26 ms on localhost (1/2) -2016-05-25 11:12:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 16) in 30 ms on localhost (2/2) -2016-05-25 11:12:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 11:12:38 INFO DAGScheduler:59 - ResultStage 8 (foreach at MultinomialNaiveBayesLearning.scala:47) finished in 0.033 s -2016-05-25 11:12:38 INFO DAGScheduler:59 - Job 7 finished: foreach at MultinomialNaiveBayesLearning.scala:47, took 0.061272 s -2016-05-25 11:12:38 INFO SparkContext:59 - Starting job: take at MultinomialNaiveBayesLearning.scala:50 -2016-05-25 11:12:38 INFO DAGScheduler:59 - Got job 8 (take at MultinomialNaiveBayesLearning.scala:50) with 1 output partitions -2016-05-25 11:12:38 INFO DAGScheduler:59 - Final stage: ResultStage 9(take at MultinomialNaiveBayesLearning.scala:50) -2016-05-25 11:12:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:12:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:12:38 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[7] at map at MultinomialNaiveBayesLearning.scala:39), which has no missing parents -2016-05-25 11:12:38 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=226001, maxMem=766075207 -2016-05-25 11:12:38 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 11:12:38 INFO MemoryStore:59 - ensureFreeSpace(2673) called with curMem=230849, maxMem=766075207 -2016-05-25 11:12:38 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 11:12:38 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53594 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 11:12:38 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:12:38 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 9 (MapPartitionsRDD[7] at map at MultinomialNaiveBayesLearning.scala:39) -2016-05-25 11:12:38 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 1 tasks -2016-05-25 11:12:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 11:12:38 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 18) -2016-05-25 11:12:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:0+47 -2016-05-25 11:12:38 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 18). 2040 bytes result sent to driver -2016-05-25 11:12:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 18) in 16 ms on localhost (1/1) -2016-05-25 11:12:38 INFO DAGScheduler:59 - ResultStage 9 (take at MultinomialNaiveBayesLearning.scala:50) finished in 0.016 s -2016-05-25 11:12:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 11:12:38 INFO DAGScheduler:59 - Job 8 finished: take at MultinomialNaiveBayesLearning.scala:50, took 0.035383 s -2016-05-25 11:12:38 INFO SparkContext:59 - Starting job: take at MultinomialNaiveBayesLearning.scala:50 -2016-05-25 11:12:38 INFO DAGScheduler:59 - Got job 9 (take at MultinomialNaiveBayesLearning.scala:50) with 1 output partitions -2016-05-25 11:12:38 INFO DAGScheduler:59 - Final stage: ResultStage 10(take at MultinomialNaiveBayesLearning.scala:50) -2016-05-25 11:12:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:12:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:12:38 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[7] at map at MultinomialNaiveBayesLearning.scala:39), which has no missing parents -2016-05-25 11:12:38 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=233522, maxMem=766075207 -2016-05-25 11:12:38 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-05-25 11:12:38 INFO MemoryStore:59 - ensureFreeSpace(2673) called with curMem=238370, maxMem=766075207 -2016-05-25 11:12:38 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 11:12:38 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53594 (size: 2.6 KB, free: 730.5 MB) -2016-05-25 11:12:38 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:12:38 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[7] at map at MultinomialNaiveBayesLearning.scala:39) -2016-05-25 11:12:38 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-25 11:12:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2205 bytes) -2016-05-25 11:12:38 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 11:12:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_naive_bayes_data.txt:47+48 -2016-05-25 11:12:38 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2206 bytes result sent to driver -2016-05-25 11:12:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 21 ms on localhost (1/1) -2016-05-25 11:12:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 11:12:38 INFO DAGScheduler:59 - ResultStage 10 (take at MultinomialNaiveBayesLearning.scala:50) finished in 0.025 s -2016-05-25 11:12:38 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:53594 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 11:12:38 INFO DAGScheduler:59 - Job 9 finished: take at MultinomialNaiveBayesLearning.scala:50, took 0.078534 s -2016-05-25 11:12:38 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 11:12:38 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53594 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:12:38 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 11:12:38 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53594 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:12:38 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 11:12:38 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53594 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:12:38 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 11:12:38 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53594 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:12:38 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 11:12:38 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53594 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:12:38 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 11:12:38 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53594 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 11:12:38 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 11:12:38 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53594 in memory (size: 1567.0 B, free: 730.6 MB) -2016-05-25 11:12:38 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 11:12:38 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53594 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 11:12:38 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 11:12:38 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 11:12:38 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53594 in memory (size: 2022.0 B, free: 730.6 MB) -2016-05-25 11:12:38 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 11:12:39 INFO SparkContext:59 - Starting job: saveAsTextFile at NaiveBayes.scala:202 -2016-05-25 11:12:39 INFO DAGScheduler:59 - Got job 10 (saveAsTextFile at NaiveBayes.scala:202) with 1 output partitions -2016-05-25 11:12:39 INFO DAGScheduler:59 - Final stage: ResultStage 11(saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 11:12:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:12:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:12:39 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202), which has no missing parents -2016-05-25 11:12:39 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=179101, maxMem=766075207 -2016-05-25 11:12:39 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 124.0 KB, free 730.3 MB) -2016-05-25 11:12:39 INFO MemoryStore:59 - ensureFreeSpace(42584) called with curMem=306093, maxMem=766075207 -2016-05-25 11:12:39 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.3 MB) -2016-05-25 11:12:39 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53594 (size: 41.6 KB, free: 730.5 MB) -2016-05-25 11:12:39 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:12:39 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 11 (MapPartitionsRDD[10] at saveAsTextFile at NaiveBayes.scala:202) -2016-05-25 11:12:39 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 1 tasks -2016-05-25 11:12:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 20, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-25 11:12:39 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 20) -2016-05-25 11:12:39 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251112_0011_m_000000_20' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525111238790/result/metadata/_temporary/0/task_201605251112_0011_m_000000 -2016-05-25 11:12:39 INFO SparkHadoopMapRedUtil:59 - attempt_201605251112_0011_m_000000_20: Committed -2016-05-25 11:12:39 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 20). 1864 bytes result sent to driver -2016-05-25 11:12:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 20) in 314 ms on localhost (1/1) -2016-05-25 11:12:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 11:12:39 INFO DAGScheduler:59 - ResultStage 11 (saveAsTextFile at NaiveBayes.scala:202) finished in 0.314 s -2016-05-25 11:12:39 INFO DAGScheduler:59 - Job 10 finished: saveAsTextFile at NaiveBayes.scala:202, took 0.401949 s -2016-05-25 11:12:41 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:53594 in memory (size: 41.6 KB, free: 730.6 MB) -2016-05-25 11:12:41 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-25 11:12:41 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:53594 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 11:12:41 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 11:12:41 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:12:41 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:12:41 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:206 -2016-05-25 11:12:41 INFO DAGScheduler:59 - Got job 11 (parquet at NaiveBayes.scala:206) with 1 output partitions -2016-05-25 11:12:41 INFO DAGScheduler:59 - Final stage: ResultStage 12(parquet at NaiveBayes.scala:206) -2016-05-25 11:12:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:12:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:12:41 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205), which has no missing parents -2016-05-25 11:12:41 INFO MemoryStore:59 - ensureFreeSpace(66936) called with curMem=171580, maxMem=766075207 -2016-05-25 11:12:41 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 65.4 KB, free 730.4 MB) -2016-05-25 11:12:41 INFO MemoryStore:59 - ensureFreeSpace(23362) called with curMem=238516, maxMem=766075207 -2016-05-25 11:12:41 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 22.8 KB, free 730.3 MB) -2016-05-25 11:12:41 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53594 (size: 22.8 KB, free: 730.6 MB) -2016-05-25 11:12:41 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:12:41 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[12] at rddToDataFrameHolder at NaiveBayes.scala:205) -2016-05-25 11:12:41 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-25 11:12:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 21, localhost, PROCESS_LOCAL, 2506 bytes) -2016-05-25 11:12:41 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 21) -2016-05-25 11:12:41 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:12:41 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 11:12:41 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 11:12:41 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 11:12:41 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 11:12:41 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 11:12:41 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 11:12:41 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 11:12:41 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 11:12:41 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 231 -2016-05-25 11:12:41 INFO ColumnChunkPageWriteStore:151 - written 77B for [labels, bag, array] DOUBLE: 3 values, 37B raw, 36B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 11:12:41 INFO ColumnChunkPageWriteStore:151 - written 74B for [pi, bag, array] DOUBLE: 3 values, 16B raw, 33B comp, 1 pages, encodings: [RLE, PLAIN_DICTIONARY], dic { 2 entries, 16B raw, 2B comp} -2016-05-25 11:12:41 INFO ColumnChunkPageWriteStore:151 - written 83B for [theta, bag, array, bag, array] DOUBLE: 9 values, 23B raw, 42B comp, 1 pages, encodings: [RLE, PLAIN_DICTIONARY], dic { 6 entries, 48B raw, 6B comp} -2016-05-25 11:12:41 INFO ColumnChunkPageWriteStore:151 - written 86B for [modelType] BINARY: 1 values, 21B raw, 39B comp, 1 pages, encodings: [PLAIN, BIT_PACKED, RLE] -2016-05-25 11:12:41 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251112_0012_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525111238790/result/data/_temporary/0/task_201605251112_0012_m_000000 -2016-05-25 11:12:41 INFO SparkHadoopMapRedUtil:59 - attempt_201605251112_0012_m_000000_0: Committed -2016-05-25 11:12:41 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 21). 915 bytes result sent to driver -2016-05-25 11:12:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 21) in 552 ms on localhost (1/1) -2016-05-25 11:12:41 INFO DAGScheduler:59 - ResultStage 12 (parquet at NaiveBayes.scala:206) finished in 0.554 s -2016-05-25 11:12:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 11:12:41 INFO DAGScheduler:59 - Job 11 finished: parquet at NaiveBayes.scala:206, took 0.597572 s -2016-05-25 11:12:42 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:12:42 INFO DefaultWriterContainer:59 - Job job_201605251112_0000 committed. -2016-05-25 11:12:42 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525111238790/result/data on driver -2016-05-25 11:12:42 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525111238790/result/data on driver -2016-05-25 11:12:42 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=261878, maxMem=766075207 -2016-05-25 11:12:42 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-25 11:12:42 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=492582, maxMem=766075207 -2016-05-25 11:12:42 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-25 11:12:42 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53594 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 11:12:42 INFO SparkContext:59 - Created broadcast 14 from textFile at modelSaveLoad.scala:129 -2016-05-25 11:12:42 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 11:12:42 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 11:12:42 INFO DAGScheduler:59 - Got job 12 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 11:12:42 INFO DAGScheduler:59 - Final stage: ResultStage 13(first at modelSaveLoad.scala:129) -2016-05-25 11:12:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:12:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:12:42 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 11:12:42 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=512370, maxMem=766075207 -2016-05-25 11:12:42 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-25 11:12:42 INFO MemoryStore:59 - ensureFreeSpace(1889) called with curMem=515586, maxMem=766075207 -2016-05-25 11:12:42 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1889.0 B, free 730.1 MB) -2016-05-25 11:12:42 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53594 (size: 1889.0 B, free: 730.5 MB) -2016-05-25 11:12:42 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:12:42 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 13 (MapPartitionsRDD[15] at textFile at modelSaveLoad.scala:129) -2016-05-25 11:12:42 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 1 tasks -2016-05-25 11:12:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 22, localhost, PROCESS_LOCAL, 2239 bytes) -2016-05-25 11:12:42 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 22) -2016-05-25 11:12:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525111238790/result/metadata/part-00000:0+56 -2016-05-25 11:12:42 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 22). 2159 bytes result sent to driver -2016-05-25 11:12:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 22) in 15 ms on localhost (1/1) -2016-05-25 11:12:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 11:12:42 INFO DAGScheduler:59 - ResultStage 13 (first at modelSaveLoad.scala:129) finished in 0.016 s -2016-05-25 11:12:42 INFO DAGScheduler:59 - Job 12 finished: first at modelSaveLoad.scala:129, took 0.029308 s -2016-05-25 11:12:42 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525111238790/result/data on driver -2016-05-25 11:12:42 INFO SparkContext:59 - Starting job: parquet at NaiveBayes.scala:213 -2016-05-25 11:12:42 INFO DAGScheduler:59 - Got job 13 (parquet at NaiveBayes.scala:213) with 4 output partitions -2016-05-25 11:12:42 INFO DAGScheduler:59 - Final stage: ResultStage 14(parquet at NaiveBayes.scala:213) -2016-05-25 11:12:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:12:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:12:42 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213), which has no missing parents -2016-05-25 11:12:42 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=517475, maxMem=766075207 -2016-05-25 11:12:42 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-25 11:12:42 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=580475, maxMem=766075207 -2016-05-25 11:12:42 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.0 MB) -2016-05-25 11:12:42 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53594 (size: 20.6 KB, free: 730.5 MB) -2016-05-25 11:12:42 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:12:42 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 14 (MapPartitionsRDD[17] at parquet at NaiveBayes.scala:213) -2016-05-25 11:12:42 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 4 tasks -2016-05-25 11:12:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 23, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 11:12:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 24, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 11:12:42 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 25, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 11:12:42 INFO TaskSetManager:59 - Starting task 3.0 in stage 14.0 (TID 26, localhost, PROCESS_LOCAL, 2280 bytes) -2016-05-25 11:12:42 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 24) -2016-05-25 11:12:42 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 23) -2016-05-25 11:12:42 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 25) -2016-05-25 11:12:42 INFO Executor:59 - Running task 3.0 in stage 14.0 (TID 26) -2016-05-25 11:12:42 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:12:42 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 24). 936 bytes result sent to driver -2016-05-25 11:12:42 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:12:42 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 25). 936 bytes result sent to driver -2016-05-25 11:12:42 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:12:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 24) in 147 ms on localhost (1/4) -2016-05-25 11:12:42 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 25) in 147 ms on localhost (2/4) -2016-05-25 11:12:42 INFO Executor:59 - Finished task 3.0 in stage 14.0 (TID 26). 1876 bytes result sent to driver -2016-05-25 11:12:42 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:12:42 INFO TaskSetManager:59 - Finished task 3.0 in stage 14.0 (TID 26) in 162 ms on localhost (3/4) -2016-05-25 11:12:42 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 23). 936 bytes result sent to driver -2016-05-25 11:12:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 23) in 167 ms on localhost (4/4) -2016-05-25 11:12:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 11:12:42 INFO DAGScheduler:59 - ResultStage 14 (parquet at NaiveBayes.scala:213) finished in 0.169 s -2016-05-25 11:12:42 INFO DAGScheduler:59 - Job 13 finished: parquet at NaiveBayes.scala:213, took 0.199234 s -2016-05-25 11:12:42 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=601548, maxMem=766075207 -2016-05-25 11:12:42 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 86.4 KB, free 729.9 MB) -2016-05-25 11:12:43 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=690060, maxMem=766075207 -2016-05-25 11:12:43 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-25 11:12:43 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53594 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 11:12:43 INFO SparkContext:59 - Created broadcast 17 from take at NaiveBayes.scala:216 -2016-05-25 11:12:43 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 11:12:43 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525111238790/result/data/part-r-00000-611c23e5-5969-4fe6-9ce1-cd51d8b19edc.gz.parquet -2016-05-25 11:12:43 INFO SparkContext:59 - Starting job: take at NaiveBayes.scala:216 -2016-05-25 11:12:43 INFO DAGScheduler:59 - Got job 14 (take at NaiveBayes.scala:216) with 1 output partitions -2016-05-25 11:12:43 INFO DAGScheduler:59 - Final stage: ResultStage 15(take at NaiveBayes.scala:216) -2016-05-25 11:12:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:12:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:12:43 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216), which has no missing parents -2016-05-25 11:12:43 INFO MemoryStore:59 - ensureFreeSpace(4280) called with curMem=709848, maxMem=766075207 -2016-05-25 11:12:43 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 4.2 KB, free 729.9 MB) -2016-05-25 11:12:43 INFO MemoryStore:59 - ensureFreeSpace(2466) called with curMem=714128, maxMem=766075207 -2016-05-25 11:12:43 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.4 KB, free 729.9 MB) -2016-05-25 11:12:43 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53594 (size: 2.4 KB, free: 730.5 MB) -2016-05-25 11:12:43 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:12:43 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 15 (MapPartitionsRDD[19] at take at NaiveBayes.scala:216) -2016-05-25 11:12:43 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 1 tasks -2016-05-25 11:12:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 27, localhost, PROCESS_LOCAL, 2304 bytes) -2016-05-25 11:12:43 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 27) -2016-05-25 11:12:43 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/NaiveBayesModel20160525111238790/result/data/part-r-00000-611c23e5-5969-4fe6-9ce1-cd51d8b19edc.gz.parquet start: 0 end: 1477 length: 1477 hosts: []} -2016-05-25 11:12:43 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 11:12:43 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional group labels (LIST) { - repeated group bag { - optional double array; - } - } - optional group pi (LIST) { - repeated group bag { - optional double array; - } - } - optional group theta (LIST) { - repeated group bag { - optional group array (LIST) { - repeated group bag { - optional double array; - } - } - } - } - optional binary modelType (UTF8); -} - -Catalyst form: -StructType(StructField(labels,ArrayType(DoubleType,true),true), StructField(pi,ArrayType(DoubleType,true),true), StructField(theta,ArrayType(ArrayType(DoubleType,true),true),true), StructField(modelType,StringType,true)) - -2016-05-25 11:12:43 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 11:12:43 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 11:12:43 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 11:12:43 INFO InternalParquetRecordReader:151 - block read in memory in 14 ms. row count = 1 -2016-05-25 11:12:43 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 27). 2841 bytes result sent to driver -2016-05-25 11:12:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 27) in 138 ms on localhost (1/1) -2016-05-25 11:12:43 INFO DAGScheduler:59 - ResultStage 15 (take at NaiveBayes.scala:216) finished in 0.139 s -2016-05-25 11:12:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-25 11:12:43 INFO DAGScheduler:59 - Job 14 finished: take at NaiveBayes.scala:216, took 0.172798 s -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 11:12:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 11:12:43 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 11:12:43 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 11:12:43 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 11:12:43 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 11:12:43 INFO BlockManager:59 - BlockManager stopped -2016-05-25 11:12:43 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 11:12:43 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 11:12:43 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 11:12:43 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 11:12:43 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-506e8042-4e64-45fa-8cd8-563770fe4bb2 -2016-05-25 11:12:43 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 11:13:49 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 11:13:50 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 11:13:50 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 11:13:50 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 11:13:50 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 11:13:51 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 11:13:51 INFO Remoting:74 - Starting remoting -2016-05-25 11:13:51 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53615] -2016-05-25 11:13:51 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53615. -2016-05-25 11:13:51 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 11:13:51 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 11:13:51 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-2062c149-92b9-4660-b9f3-742f0c27d324 -2016-05-25 11:13:51 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 11:13:51 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-ff35d43a-0a81-42c1-a2f4-6f2deb39df77\httpd-08dd3efb-70d6-4267-93ab-9325219484bc -2016-05-25 11:13:51 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 11:13:51 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:13:51 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53616 -2016-05-25 11:13:51 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53616. -2016-05-25 11:13:51 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 11:13:51 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:13:51 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 11:13:51 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 11:13:51 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 11:13:52 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 11:13:52 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 11:13:52 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53635. -2016-05-25 11:13:52 INFO NettyBlockTransferService:59 - Server created on 53635 -2016-05-25 11:13:52 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 11:13:52 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53635 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53635) -2016-05-25 11:13:52 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 11:13:53 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 11:13:53 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 11:13:53 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 11:13:53 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 11:13:53 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53635 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 11:13:53 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 11:13:54 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 11:13:55 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 11:13:55 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 11:13:55 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 11:13:55 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 11:13:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:13:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:13:55 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 11:13:55 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 11:13:55 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 11:13:55 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 11:13:55 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 11:13:55 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53635 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:13:55 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:13:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 11:13:55 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 11:13:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:13:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:13:55 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 11:13:55 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 11:13:55 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 11:13:55 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 11:13:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 11:13:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 11:13:55 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 11:13:55 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 11:13:55 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 11:13:55 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 11:13:55 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 11:13:55 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177630, maxMem=766075207 -2016-05-25 11:13:55 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 11:13:55 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262710, maxMem=766075207 -2016-05-25 11:13:55 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 11:13:55 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:53635 (size: 83.1 KB, free: 730.5 MB) -2016-05-25 11:13:55 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:53635 (size: 83.6 KB, free: 730.4 MB) -2016-05-25 11:13:55 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 11:13:55 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 11:13:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 497 ms on localhost (1/2) -2016-05-25 11:13:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 525 ms on localhost (2/2) -2016-05-25 11:13:55 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.544 s -2016-05-25 11:13:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 11:13:55 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.680310 s -2016-05-25 11:13:55 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 11:13:55 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 11:13:55 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 11:13:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:13:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:13:55 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 11:13:55 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348358, maxMem=766075207 -2016-05-25 11:13:55 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 11:13:55 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=353126, maxMem=766075207 -2016-05-25 11:13:55 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 11:13:55 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53635 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:13:55 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:13:55 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 11:13:55 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 11:13:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:13:55 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 11:13:55 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:13:55 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 11:13:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 28 ms on localhost (1/1) -2016-05-25 11:13:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 11:13:56 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.029 s -2016-05-25 11:13:56 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.048030 s -2016-05-25 11:13:56 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 11:13:56 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 11:13:56 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 11:13:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:13:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:13:56 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 11:13:56 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355673, maxMem=766075207 -2016-05-25 11:13:56 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 11:13:56 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=360009, maxMem=766075207 -2016-05-25 11:13:56 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 11:13:56 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53635 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:13:56 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:13:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 11:13:56 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 11:13:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:13:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:13:56 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 11:13:56 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 11:13:56 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:13:56 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:13:56 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 11:13:56 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 11:13:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 14 ms on localhost (1/2) -2016-05-25 11:13:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 17 ms on localhost (2/2) -2016-05-25 11:13:56 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.018 s -2016-05-25 11:13:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 11:13:56 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.031152 s -2016-05-25 11:13:56 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 11:13:56 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 11:13:56 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 11:13:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:13:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:13:56 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 11:13:56 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362328, maxMem=766075207 -2016-05-25 11:13:56 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 11:13:56 INFO MemoryStore:59 - ensureFreeSpace(2631) called with curMem=367392, maxMem=766075207 -2016-05-25 11:13:56 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 11:13:56 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53635 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:13:56 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:13:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 11:13:56 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 11:13:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:13:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:13:56 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 11:13:56 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 11:13:56 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:13:56 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:13:56 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 64301 bytes result sent to driver -2016-05-25 11:13:56 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 61704 bytes result sent to driver -2016-05-25 11:13:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 34 ms on localhost (1/2) -2016-05-25 11:13:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 35 ms on localhost (2/2) -2016-05-25 11:13:56 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.037 s -2016-05-25 11:13:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 11:13:56 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.063877 s -2016-05-25 11:13:57 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53635 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:13:57 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 11:13:57 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53635 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:13:57 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 11:13:57 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53635 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:13:57 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 11:13:57 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53635 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-25 11:13:57 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=342308, maxMem=766075207 -2016-05-25 11:13:57 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 11:13:57 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=342348, maxMem=766075207 -2016-05-25 11:13:57 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 11:13:57 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53635 (size: 81.0 B, free: 730.4 MB) -2016-05-25 11:13:57 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 11:13:57 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 11:13:57 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 11:13:57 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 11:13:57 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 11:13:57 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 11:13:57 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 11:13:57 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 11:13:57 INFO MemoryStore:59 - ensureFreeSpace(490784) called with curMem=342429, maxMem=766075207 -2016-05-25 11:13:57 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 479.3 KB, free 729.8 MB) -2016-05-25 11:13:57 INFO MemoryStore:59 - ensureFreeSpace(120336) called with curMem=833213, maxMem=766075207 -2016-05-25 11:13:57 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 117.5 KB, free 729.7 MB) -2016-05-25 11:13:57 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53635 (size: 117.5 KB, free: 730.3 MB) -2016-05-25 11:13:57 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:13:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 11:13:57 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 11:13:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:13:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:13:57 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 11:13:57 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 11:13:57 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 11:13:57 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:13:57 INFO MemoryStore:59 - ensureFreeSpace(102976) called with curMem=953549, maxMem=766075207 -2016-05-25 11:13:57 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 100.6 KB, free 729.6 MB) -2016-05-25 11:13:57 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:53635 (size: 100.6 KB, free: 730.2 MB) -2016-05-25 11:13:57 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 11:13:57 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:13:57 INFO MemoryStore:59 - ensureFreeSpace(105840) called with curMem=1056525, maxMem=766075207 -2016-05-25 11:13:57 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 103.4 KB, free 729.5 MB) -2016-05-25 11:13:57 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:53635 (size: 103.4 KB, free: 730.1 MB) -2016-05-25 11:13:57 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 11:13:57 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 11:13:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 265 ms on localhost (1/2) -2016-05-25 11:13:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 264 ms on localhost (2/2) -2016-05-25 11:13:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 11:13:57 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.267 s -2016-05-25 11:13:57 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:13:57 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:13:57 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 11:13:57 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:13:57 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 11:13:57 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 11:13:58 INFO MemoryStore:59 - ensureFreeSpace(213456) called with curMem=1162365, maxMem=766075207 -2016-05-25 11:13:58 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 208.5 KB, free 729.3 MB) -2016-05-25 11:13:58 INFO MemoryStore:59 - ensureFreeSpace(45354) called with curMem=1375821, maxMem=766075207 -2016-05-25 11:13:58 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 44.3 KB, free 729.2 MB) -2016-05-25 11:13:58 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53635 (size: 44.3 KB, free: 730.1 MB) -2016-05-25 11:13:58 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:13:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 11:13:58 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 11:13:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:13:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:13:58 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 11:13:58 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 11:13:58 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:13:58 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:13:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 11:13:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 11:13:58 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 11:13:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 115 ms on localhost (1/2) -2016-05-25 11:13:58 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 11:13:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 379 ms on localhost (2/2) -2016-05-25 11:13:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 11:13:58 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.379 s -2016-05-25 11:13:58 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.704754 s -2016-05-25 11:13:58 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=1421175, maxMem=766075207 -2016-05-25 11:13:58 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 40.0 B, free 729.2 MB) -2016-05-25 11:13:58 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=1421215, maxMem=766075207 -2016-05-25 11:13:58 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 81.0 B, free 729.2 MB) -2016-05-25 11:13:58 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53635 (size: 81.0 B, free: 730.1 MB) -2016-05-25 11:13:58 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 11:13:58 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 11:13:58 INFO DAGScheduler:59 - Registering RDD 17 (mapPartitions at DecisionTree.scala:625) -2016-05-25 11:13:58 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 11:13:58 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 11:13:58 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 11:13:58 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 11:13:58 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 11:13:58 INFO MemoryStore:59 - ensureFreeSpace(491136) called with curMem=1421296, maxMem=766075207 -2016-05-25 11:13:58 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 479.6 KB, free 728.8 MB) -2016-05-25 11:13:58 INFO MemoryStore:59 - ensureFreeSpace(120519) called with curMem=1912432, maxMem=766075207 -2016-05-25 11:13:58 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 117.7 KB, free 728.6 MB) -2016-05-25 11:13:58 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53635 (size: 117.7 KB, free: 729.9 MB) -2016-05-25 11:13:58 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:13:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625) -2016-05-25 11:13:58 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 11:13:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:13:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:13:58 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 11:13:58 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 11:13:58 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-25 11:13:58 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-25 11:13:58 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 11:13:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 126 ms on localhost (1/2) -2016-05-25 11:13:58 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 11:13:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 132 ms on localhost (2/2) -2016-05-25 11:13:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 11:13:58 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.134 s -2016-05-25 11:13:58 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:13:58 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:13:58 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 11:13:58 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:13:58 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 11:13:58 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 11:13:58 INFO MemoryStore:59 - ensureFreeSpace(213456) called with curMem=2032951, maxMem=766075207 -2016-05-25 11:13:58 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 208.5 KB, free 728.4 MB) -2016-05-25 11:13:58 INFO MemoryStore:59 - ensureFreeSpace(45340) called with curMem=2246407, maxMem=766075207 -2016-05-25 11:13:58 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 44.3 KB, free 728.4 MB) -2016-05-25 11:13:58 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53635 (size: 44.3 KB, free: 729.9 MB) -2016-05-25 11:13:58 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:13:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645) -2016-05-25 11:13:58 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 11:13:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:13:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:13:58 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 11:13:58 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 11:13:58 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:13:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 11:13:58 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 1161 bytes result sent to driver -2016-05-25 11:13:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 42 ms on localhost (1/2) -2016-05-25 11:13:58 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:13:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 11:13:58 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 11:13:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 120 ms on localhost (2/2) -2016-05-25 11:13:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 11:13:58 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.121 s -2016-05-25 11:13:58 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.316145 s -2016-05-25 11:13:58 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 11:13:58 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 11:13:58 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 11:13:58 INFO RandomForest:59 - init: 1.546010567 - total: 2.868644312 - findSplitsBins: 1.306059615 - findBestSplits: 1.29981085 - chooseSplits: 1.291152536 -2016-05-25 11:13:58 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:47 -2016-05-25 11:13:58 INFO DAGScheduler:59 - Got job 6 (count at DecisionTreesLearning.scala:47) with 2 output partitions -2016-05-25 11:13:58 INFO DAGScheduler:59 - Final stage: ResultStage 8(count at DecisionTreesLearning.scala:47) -2016-05-25 11:13:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:13:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:13:58 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at filter at DecisionTreesLearning.scala:47), which has no missing parents -2016-05-25 11:13:58 INFO MemoryStore:59 - ensureFreeSpace(6744) called with curMem=2082931, maxMem=766075207 -2016-05-25 11:13:58 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 6.6 KB, free 728.6 MB) -2016-05-25 11:13:58 INFO MemoryStore:59 - ensureFreeSpace(3590) called with curMem=2089675, maxMem=766075207 -2016-05-25 11:13:58 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.5 KB, free 728.6 MB) -2016-05-25 11:13:58 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53635 (size: 3.5 KB, free: 730.1 MB) -2016-05-25 11:13:58 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:13:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at filter at DecisionTreesLearning.scala:47) -2016-05-25 11:13:58 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 11:13:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:13:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:13:58 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 11:13:58 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 11:13:58 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:13:58 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:13:58 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 11:13:58 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 11:13:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 13 ms on localhost (1/2) -2016-05-25 11:13:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 13 ms on localhost (2/2) -2016-05-25 11:13:58 INFO DAGScheduler:59 - ResultStage 8 (count at DecisionTreesLearning.scala:47) finished in 0.014 s -2016-05-25 11:13:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 11:13:58 INFO DAGScheduler:59 - Job 6 finished: count at DecisionTreesLearning.scala:47, took 0.034993 s -2016-05-25 11:13:58 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:47 -2016-05-25 11:13:58 INFO DAGScheduler:59 - Got job 7 (count at DecisionTreesLearning.scala:47) with 2 output partitions -2016-05-25 11:13:58 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at DecisionTreesLearning.scala:47) -2016-05-25 11:13:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:13:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:13:58 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:13:58 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2093265, maxMem=766075207 -2016-05-25 11:13:58 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.0 KB, free 728.6 MB) -2016-05-25 11:13:58 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=2097377, maxMem=766075207 -2016-05-25 11:13:58 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.6 MB) -2016-05-25 11:13:58 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53635 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 11:13:58 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:13:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:13:58 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 11:13:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:13:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:13:58 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 11:13:58 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-25 11:13:58 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:13:58 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:13:58 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2082 bytes result sent to driver -2016-05-25 11:13:58 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2082 bytes result sent to driver -2016-05-25 11:13:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 9 ms on localhost (1/2) -2016-05-25 11:13:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 11 ms on localhost (2/2) -2016-05-25 11:13:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 11:13:58 INFO DAGScheduler:59 - ResultStage 9 (count at DecisionTreesLearning.scala:47) finished in 0.013 s -2016-05-25 11:13:58 INFO DAGScheduler:59 - Job 7 finished: count at DecisionTreesLearning.scala:47, took 0.029576 s -2016-05-25 11:13:59 INFO SparkContext:59 - Starting job: saveAsTextFile at DecisionTreeModel.scala:235 -2016-05-25 11:13:59 INFO DAGScheduler:59 - Got job 8 (saveAsTextFile at DecisionTreeModel.scala:235) with 1 output partitions -2016-05-25 11:13:59 INFO DAGScheduler:59 - Final stage: ResultStage 10(saveAsTextFile at DecisionTreeModel.scala:235) -2016-05-25 11:13:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:13:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:13:59 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[23] at saveAsTextFile at DecisionTreeModel.scala:235), which has no missing parents -2016-05-25 11:13:59 INFO MemoryStore:59 - ensureFreeSpace(127000) called with curMem=2099645, maxMem=766075207 -2016-05-25 11:13:59 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 124.0 KB, free 728.5 MB) -2016-05-25 11:13:59 INFO MemoryStore:59 - ensureFreeSpace(42609) called with curMem=2226645, maxMem=766075207 -2016-05-25 11:13:59 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 41.6 KB, free 728.4 MB) -2016-05-25 11:13:59 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53635 (size: 41.6 KB, free: 730.0 MB) -2016-05-25 11:13:59 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:13:59 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[23] at saveAsTextFile at DecisionTreeModel.scala:235) -2016-05-25 11:13:59 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-25 11:13:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:13:59 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 11:13:59 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251113_0010_m_000000_19' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111358897/result/metadata/_temporary/0/task_201605251113_0010_m_000000 -2016-05-25 11:13:59 INFO SparkHadoopMapRedUtil:59 - attempt_201605251113_0010_m_000000_19: Committed -2016-05-25 11:13:59 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 1864 bytes result sent to driver -2016-05-25 11:13:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 181 ms on localhost (1/1) -2016-05-25 11:13:59 INFO DAGScheduler:59 - ResultStage 10 (saveAsTextFile at DecisionTreeModel.scala:235) finished in 0.181 s -2016-05-25 11:13:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 11:13:59 INFO DAGScheduler:59 - Job 8 finished: saveAsTextFile at DecisionTreeModel.scala:235, took 0.258661 s -2016-05-25 11:13:59 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:53635 in memory (size: 41.6 KB, free: 730.1 MB) -2016-05-25 11:13:59 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 11:13:59 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:53635 in memory (size: 2.2 KB, free: 730.1 MB) -2016-05-25 11:13:59 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 11:13:59 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:53635 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-25 11:13:59 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 11:13:59 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:53635 in memory (size: 44.3 KB, free: 730.1 MB) -2016-05-25 11:13:59 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 11:13:59 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53635 in memory (size: 117.7 KB, free: 730.3 MB) -2016-05-25 11:13:59 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 11:13:59 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-25 11:13:59 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53635 in memory (size: 81.0 B, free: 730.3 MB) -2016-05-25 11:13:59 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53635 in memory (size: 44.3 KB, free: 730.3 MB) -2016-05-25 11:13:59 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 11:13:59 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53635 in memory (size: 117.5 KB, free: 730.4 MB) -2016-05-25 11:13:59 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 11:13:59 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 11:13:59 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53635 in memory (size: 81.0 B, free: 730.4 MB) -2016-05-25 11:13:59 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 11:13:59 INFO ContextCleaner:59 - Cleaned RDD 13 -2016-05-25 11:14:01 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:14:01 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:14:01 INFO SparkContext:59 - Starting job: parquet at DecisionTreeModel.scala:242 -2016-05-25 11:14:01 INFO DAGScheduler:59 - Got job 9 (parquet at DecisionTreeModel.scala:242) with 4 output partitions -2016-05-25 11:14:01 INFO DAGScheduler:59 - Final stage: ResultStage 11(parquet at DecisionTreeModel.scala:242) -2016-05-25 11:14:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:14:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:14:01 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[26] at rddToDataFrameHolder at DecisionTreeModel.scala:240), which has no missing parents -2016-05-25 11:14:01 INFO MemoryStore:59 - ensureFreeSpace(68760) called with curMem=342308, maxMem=766075207 -2016-05-25 11:14:01 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 67.1 KB, free 730.2 MB) -2016-05-25 11:14:01 INFO MemoryStore:59 - ensureFreeSpace(24134) called with curMem=411068, maxMem=766075207 -2016-05-25 11:14:01 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 23.6 KB, free 730.2 MB) -2016-05-25 11:14:01 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53635 (size: 23.6 KB, free: 730.4 MB) -2016-05-25 11:14:01 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:01 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 11 (MapPartitionsRDD[26] at rddToDataFrameHolder at DecisionTreeModel.scala:240) -2016-05-25 11:14:01 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 4 tasks -2016-05-25 11:14:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 20, localhost, PROCESS_LOCAL, 4009 bytes) -2016-05-25 11:14:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 3790 bytes) -2016-05-25 11:14:01 INFO TaskSetManager:59 - Starting task 2.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2428 bytes) -2016-05-25 11:14:01 INFO TaskSetManager:59 - Starting task 3.0 in stage 11.0 (TID 23, localhost, PROCESS_LOCAL, 2489 bytes) -2016-05-25 11:14:01 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 20) -2016-05-25 11:14:01 INFO Executor:59 - Running task 2.0 in stage 11.0 (TID 22) -2016-05-25 11:14:01 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 21) -2016-05-25 11:14:01 INFO Executor:59 - Running task 3.0 in stage 11.0 (TID 23) -2016-05-25 11:14:01 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:14:01 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 11:14:01 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:14:01 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 11:14:01 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:14:01 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 11:14:01 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:14:01 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 11:14:01 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 11:14:01 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 11:14:01 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 11:14:01 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 11:14:01 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 11:14:01 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,620 -2016-05-25 11:14:01 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,644 -2016-05-25 11:14:01 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,676 -2016-05-25 11:14:01 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,676 -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [nodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, predict] DOUBLE: 1 values, 15B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 76B for [predict, prob] DOUBLE: 1 values, 15B raw, 35B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 75B for [impurity] DOUBLE: 1 values, 14B raw, 34B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 64B for [split, feature] INT32: 1 values, 11B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 71B for [split, threshold] DOUBLE: 1 values, 15B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [split, featureType] INT32: 1 values, 11B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 52B for [split, categories, bag, array] DOUBLE: 1 values, 14B raw, 31B comp, 1 pages, encodings: [RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [leftNodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [rightNodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 75B for [infoGain] DOUBLE: 1 values, 14B raw, 34B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [nodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [nodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 69B for [predict, predict] DOUBLE: 1 values, 15B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, prob] DOUBLE: 1 values, 15B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 69B for [predict, predict] DOUBLE: 1 values, 15B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 69B for [impurity] DOUBLE: 1 values, 14B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, prob] DOUBLE: 1 values, 15B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 71B for [impurity] DOUBLE: 1 values, 14B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, feature] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 2 values, 8B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN_DICTIONARY, RLE], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, threshold] DOUBLE: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 63B for [nodeId] INT32: 2 values, 14B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 62B for [split, feature] INT32: 1 values, 11B raw, 29B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, featureType] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 72B for [split, threshold] DOUBLE: 1 values, 15B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 68B for [predict, predict] DOUBLE: 2 values, 9B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN_DICTIONARY, RLE], dic { 1 entries, 8B raw, 1B comp} -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 51B for [split, categories, bag, array] DOUBLE: 1 values, 14B raw, 30B comp, 1 pages, encodings: [RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [split, featureType] INT32: 1 values, 11B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 47B for [leftNodeId] INT32: 1 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 68B for [predict, prob] DOUBLE: 2 values, 9B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN_DICTIONARY, RLE], dic { 1 entries, 8B raw, 1B comp} -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 52B for [split, categories, bag, array] DOUBLE: 1 values, 14B raw, 31B comp, 1 pages, encodings: [RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 47B for [rightNodeId] INT32: 1 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [leftNodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 47B for [infoGain] DOUBLE: 1 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 69B for [impurity] DOUBLE: 2 values, 8B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN_DICTIONARY, RLE], dic { 1 entries, 8B raw, 1B comp} -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [rightNodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 2 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 71B for [infoGain] DOUBLE: 1 values, 14B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, feature] INT32: 2 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, threshold] DOUBLE: 2 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, featureType] INT32: 2 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 51B for [split, categories, bag, array] DOUBLE: 2 values, 14B raw, 30B comp, 1 pages, encodings: [RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 47B for [leftNodeId] INT32: 2 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 47B for [rightNodeId] INT32: 2 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO ColumnChunkPageWriteStore:151 - written 47B for [infoGain] DOUBLE: 2 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 11:14:01 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251114_0011_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111358897/result/data/_temporary/0/task_201605251114_0011_m_000000 -2016-05-25 11:14:01 INFO SparkHadoopMapRedUtil:59 - attempt_201605251114_0011_m_000000_0: Committed -2016-05-25 11:14:01 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 20). 915 bytes result sent to driver -2016-05-25 11:14:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 20) in 638 ms on localhost (1/4) -2016-05-25 11:14:01 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251114_0011_m_000003_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111358897/result/data/_temporary/0/task_201605251114_0011_m_000003 -2016-05-25 11:14:01 INFO SparkHadoopMapRedUtil:59 - attempt_201605251114_0011_m_000003_0: Committed -2016-05-25 11:14:01 INFO Executor:59 - Finished task 3.0 in stage 11.0 (TID 23). 915 bytes result sent to driver -2016-05-25 11:14:01 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251114_0011_m_000001_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111358897/result/data/_temporary/0/task_201605251114_0011_m_000001 -2016-05-25 11:14:01 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251114_0011_m_000002_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111358897/result/data/_temporary/0/task_201605251114_0011_m_000002 -2016-05-25 11:14:01 INFO SparkHadoopMapRedUtil:59 - attempt_201605251114_0011_m_000001_0: Committed -2016-05-25 11:14:01 INFO SparkHadoopMapRedUtil:59 - attempt_201605251114_0011_m_000002_0: Committed -2016-05-25 11:14:01 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 21). 915 bytes result sent to driver -2016-05-25 11:14:01 INFO Executor:59 - Finished task 2.0 in stage 11.0 (TID 22). 915 bytes result sent to driver -2016-05-25 11:14:01 INFO TaskSetManager:59 - Finished task 3.0 in stage 11.0 (TID 23) in 642 ms on localhost (2/4) -2016-05-25 11:14:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 21) in 645 ms on localhost (3/4) -2016-05-25 11:14:01 INFO TaskSetManager:59 - Finished task 2.0 in stage 11.0 (TID 22) in 644 ms on localhost (4/4) -2016-05-25 11:14:01 INFO DAGScheduler:59 - ResultStage 11 (parquet at DecisionTreeModel.scala:242) finished in 0.642 s -2016-05-25 11:14:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 11:14:01 INFO DAGScheduler:59 - Job 9 finished: parquet at DecisionTreeModel.scala:242, took 0.683517 s -2016-05-25 11:14:02 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:14:02 INFO DefaultWriterContainer:59 - Job job_201605251114_0000 committed. -2016-05-25 11:14:02 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111358897/result/data on driver -2016-05-25 11:14:02 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111358897/result/data on driver -2016-05-25 11:14:02 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=435202, maxMem=766075207 -2016-05-25 11:14:02 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 225.3 KB, free 730.0 MB) -2016-05-25 11:14:02 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=665906, maxMem=766075207 -2016-05-25 11:14:02 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-25 11:14:02 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53635 (size: 19.3 KB, free: 730.4 MB) -2016-05-25 11:14:02 INFO SparkContext:59 - Created broadcast 15 from textFile at modelSaveLoad.scala:129 -2016-05-25 11:14:02 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 11:14:02 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 11:14:02 INFO DAGScheduler:59 - Got job 10 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 11:14:02 INFO DAGScheduler:59 - Final stage: ResultStage 12(first at modelSaveLoad.scala:129) -2016-05-25 11:14:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:14:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:14:02 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[29] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 11:14:02 INFO MemoryStore:59 - ensureFreeSpace(3224) called with curMem=685694, maxMem=766075207 -2016-05-25 11:14:02 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 3.1 KB, free 729.9 MB) -2016-05-25 11:14:02 INFO MemoryStore:59 - ensureFreeSpace(1898) called with curMem=688918, maxMem=766075207 -2016-05-25 11:14:02 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1898.0 B, free 729.9 MB) -2016-05-25 11:14:02 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53635 (size: 1898.0 B, free: 730.4 MB) -2016-05-25 11:14:02 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:02 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[29] at textFile at modelSaveLoad.scala:129) -2016-05-25 11:14:02 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-25 11:14:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2245 bytes) -2016-05-25 11:14:02 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 24) -2016-05-25 11:14:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111358897/result/metadata/part-00000:0+55 -2016-05-25 11:14:02 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 24). 2157 bytes result sent to driver -2016-05-25 11:14:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 24) in 13 ms on localhost (1/1) -2016-05-25 11:14:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 11:14:02 INFO DAGScheduler:59 - ResultStage 12 (first at modelSaveLoad.scala:129) finished in 0.015 s -2016-05-25 11:14:02 INFO DAGScheduler:59 - Job 10 finished: first at modelSaveLoad.scala:129, took 0.027287 s -2016-05-25 11:14:02 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111358897/result/data on driver -2016-05-25 11:14:02 INFO SparkContext:59 - Starting job: parquet at DecisionTreeModel.scala:249 -2016-05-25 11:14:02 INFO DAGScheduler:59 - Got job 11 (parquet at DecisionTreeModel.scala:249) with 4 output partitions -2016-05-25 11:14:02 INFO DAGScheduler:59 - Final stage: ResultStage 13(parquet at DecisionTreeModel.scala:249) -2016-05-25 11:14:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:14:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:14:02 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[31] at parquet at DecisionTreeModel.scala:249), which has no missing parents -2016-05-25 11:14:02 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=690816, maxMem=766075207 -2016-05-25 11:14:02 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 61.5 KB, free 729.9 MB) -2016-05-25 11:14:02 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=753816, maxMem=766075207 -2016-05-25 11:14:02 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 20.6 KB, free 729.8 MB) -2016-05-25 11:14:02 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53635 (size: 20.6 KB, free: 730.3 MB) -2016-05-25 11:14:02 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:02 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 13 (MapPartitionsRDD[31] at parquet at DecisionTreeModel.scala:249) -2016-05-25 11:14:02 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 4 tasks -2016-05-25 11:14:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 11:14:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 11:14:02 INFO TaskSetManager:59 - Starting task 2.0 in stage 13.0 (TID 27, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 11:14:02 INFO TaskSetManager:59 - Starting task 3.0 in stage 13.0 (TID 28, localhost, PROCESS_LOCAL, 2286 bytes) -2016-05-25 11:14:02 INFO Executor:59 - Running task 3.0 in stage 13.0 (TID 28) -2016-05-25 11:14:02 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-25 11:14:02 INFO Executor:59 - Running task 2.0 in stage 13.0 (TID 27) -2016-05-25 11:14:02 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-25 11:14:02 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:14:02 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:14:02 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:14:02 INFO Executor:59 - Finished task 2.0 in stage 13.0 (TID 27). 936 bytes result sent to driver -2016-05-25 11:14:02 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 936 bytes result sent to driver -2016-05-25 11:14:02 INFO TaskSetManager:59 - Finished task 2.0 in stage 13.0 (TID 27) in 72 ms on localhost (1/4) -2016-05-25 11:14:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 74 ms on localhost (2/4) -2016-05-25 11:14:02 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:14:02 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 936 bytes result sent to driver -2016-05-25 11:14:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 89 ms on localhost (3/4) -2016-05-25 11:14:02 INFO Executor:59 - Finished task 3.0 in stage 13.0 (TID 28). 2321 bytes result sent to driver -2016-05-25 11:14:02 INFO TaskSetManager:59 - Finished task 3.0 in stage 13.0 (TID 28) in 97 ms on localhost (4/4) -2016-05-25 11:14:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 11:14:02 INFO DAGScheduler:59 - ResultStage 13 (parquet at DecisionTreeModel.scala:249) finished in 0.100 s -2016-05-25 11:14:02 INFO DAGScheduler:59 - Job 11 finished: parquet at DecisionTreeModel.scala:249, took 0.138410 s -2016-05-25 11:14:02 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=774889, maxMem=766075207 -2016-05-25 11:14:02 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 86.4 KB, free 729.8 MB) -2016-05-25 11:14:03 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=863401, maxMem=766075207 -2016-05-25 11:14:03 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.7 MB) -2016-05-25 11:14:03 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53635 (size: 19.3 KB, free: 730.3 MB) -2016-05-25 11:14:03 INFO SparkContext:59 - Created broadcast 18 from map at DecisionTreeModel.scala:252 -2016-05-25 11:14:03 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:53635 in memory (size: 20.6 KB, free: 730.3 MB) -2016-05-25 11:14:03 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-25 11:14:03 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:53635 in memory (size: 1898.0 B, free: 730.3 MB) -2016-05-25 11:14:03 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-25 11:14:03 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:53635 in memory (size: 19.3 KB, free: 730.4 MB) -2016-05-25 11:14:03 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:53635 in memory (size: 23.6 KB, free: 730.4 MB) -2016-05-25 11:14:03 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-25 11:14:03 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 11:14:03 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111358897/result/data/part-r-00001-3ed1398f-ad3e-4878-9f60-7e3d4817b219.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111358897/result/data/part-r-00002-3ed1398f-ad3e-4878-9f60-7e3d4817b219.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111358897/result/data/part-r-00003-3ed1398f-ad3e-4878-9f60-7e3d4817b219.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111358897/result/data/part-r-00000-3ed1398f-ad3e-4878-9f60-7e3d4817b219.gz.parquet -2016-05-25 11:14:03 INFO SparkContext:59 - Starting job: collect at DecisionTreeModel.scala:267 -2016-05-25 11:14:03 INFO DAGScheduler:59 - Registering RDD 35 (groupBy at DecisionTreeModel.scala:265) -2016-05-25 11:14:03 INFO DAGScheduler:59 - Got job 12 (collect at DecisionTreeModel.scala:267) with 4 output partitions -2016-05-25 11:14:03 INFO DAGScheduler:59 - Final stage: ResultStage 15(collect at DecisionTreeModel.scala:267) -2016-05-25 11:14:03 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 14) -2016-05-25 11:14:03 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 14) -2016-05-25 11:14:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 14 (MapPartitionsRDD[35] at groupBy at DecisionTreeModel.scala:265), which has no missing parents -2016-05-25 11:14:03 INFO MemoryStore:59 - ensureFreeSpace(7248) called with curMem=450608, maxMem=766075207 -2016-05-25 11:14:03 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 7.1 KB, free 730.1 MB) -2016-05-25 11:14:03 INFO MemoryStore:59 - ensureFreeSpace(3789) called with curMem=457856, maxMem=766075207 -2016-05-25 11:14:03 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.7 KB, free 730.1 MB) -2016-05-25 11:14:03 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53635 (size: 3.7 KB, free: 730.4 MB) -2016-05-25 11:14:03 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:03 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 14 (MapPartitionsRDD[35] at groupBy at DecisionTreeModel.scala:265) -2016-05-25 11:14:03 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 4 tasks -2016-05-25 11:14:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 29, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 11:14:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 30, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 11:14:03 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 31, localhost, PROCESS_LOCAL, 2292 bytes) -2016-05-25 11:14:03 INFO TaskSetManager:59 - Starting task 3.0 in stage 14.0 (TID 32, localhost, PROCESS_LOCAL, 2292 bytes) -2016-05-25 11:14:03 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 30) -2016-05-25 11:14:03 INFO Executor:59 - Running task 3.0 in stage 14.0 (TID 32) -2016-05-25 11:14:03 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 29) -2016-05-25 11:14:03 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 31) -2016-05-25 11:14:03 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111358897/result/data/part-r-00000-3ed1398f-ad3e-4878-9f60-7e3d4817b219.gz.parquet start: 0 end: 3049 length: 3049 hosts: []} -2016-05-25 11:14:03 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111358897/result/data/part-r-00003-3ed1398f-ad3e-4878-9f60-7e3d4817b219.gz.parquet start: 0 end: 2987 length: 2987 hosts: []} -2016-05-25 11:14:03 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111358897/result/data/part-r-00002-3ed1398f-ad3e-4878-9f60-7e3d4817b219.gz.parquet start: 0 end: 2836 length: 2836 hosts: []} -2016-05-25 11:14:03 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111358897/result/data/part-r-00001-3ed1398f-ad3e-4878-9f60-7e3d4817b219.gz.parquet start: 0 end: 3032 length: 3032 hosts: []} -2016-05-25 11:14:03 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 11:14:03 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 11:14:03 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 11:14:03 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 11:14:03 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 11:14:03 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 treeId; - optional int32 nodeId; - optional group predict { - optional double predict; - optional double prob; - } - optional double impurity; - optional boolean isLeaf; - optional group split { - optional int32 feature; - optional double threshold; - optional int32 featureType; - optional group categories (LIST) { - repeated group bag { - optional double array; - } - } - } - optional int32 leftNodeId; - optional int32 rightNodeId; - optional double infoGain; -} - -Catalyst form: -StructType(StructField(treeId,IntegerType,true), StructField(nodeId,IntegerType,true), StructField(predict,StructType(StructField(predict,DoubleType,true), StructField(prob,DoubleType,true)),true), StructField(impurity,DoubleType,true), StructField(isLeaf,BooleanType,true), StructField(split,StructType(StructField(feature,IntegerType,true), StructField(threshold,DoubleType,true), StructField(featureType,IntegerType,true), StructField(categories,ArrayType(DoubleType,true),true)),true), StructField(leftNodeId,IntegerType,true), StructField(rightNodeId,IntegerType,true), StructField(infoGain,DoubleType,true)) - -2016-05-25 11:14:04 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 treeId; - optional int32 nodeId; - optional group predict { - optional double predict; - optional double prob; - } - optional double impurity; - optional boolean isLeaf; - optional group split { - optional int32 feature; - optional double threshold; - optional int32 featureType; - optional group categories (LIST) { - repeated group bag { - optional double array; - } - } - } - optional int32 leftNodeId; - optional int32 rightNodeId; - optional double infoGain; -} - -Catalyst form: -StructType(StructField(treeId,IntegerType,true), StructField(nodeId,IntegerType,true), StructField(predict,StructType(StructField(predict,DoubleType,true), StructField(prob,DoubleType,true)),true), StructField(impurity,DoubleType,true), StructField(isLeaf,BooleanType,true), StructField(split,StructType(StructField(feature,IntegerType,true), StructField(threshold,DoubleType,true), StructField(featureType,IntegerType,true), StructField(categories,ArrayType(DoubleType,true),true)),true), StructField(leftNodeId,IntegerType,true), StructField(rightNodeId,IntegerType,true), StructField(infoGain,DoubleType,true)) - -2016-05-25 11:14:04 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 treeId; - optional int32 nodeId; - optional group predict { - optional double predict; - optional double prob; - } - optional double impurity; - optional boolean isLeaf; - optional group split { - optional int32 feature; - optional double threshold; - optional int32 featureType; - optional group categories (LIST) { - repeated group bag { - optional double array; - } - } - } - optional int32 leftNodeId; - optional int32 rightNodeId; - optional double infoGain; -} - -Catalyst form: -StructType(StructField(treeId,IntegerType,true), StructField(nodeId,IntegerType,true), StructField(predict,StructType(StructField(predict,DoubleType,true), StructField(prob,DoubleType,true)),true), StructField(impurity,DoubleType,true), StructField(isLeaf,BooleanType,true), StructField(split,StructType(StructField(feature,IntegerType,true), StructField(threshold,DoubleType,true), StructField(featureType,IntegerType,true), StructField(categories,ArrayType(DoubleType,true),true)),true), StructField(leftNodeId,IntegerType,true), StructField(rightNodeId,IntegerType,true), StructField(infoGain,DoubleType,true)) - -2016-05-25 11:14:04 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 treeId; - optional int32 nodeId; - optional group predict { - optional double predict; - optional double prob; - } - optional double impurity; - optional boolean isLeaf; - optional group split { - optional int32 feature; - optional double threshold; - optional int32 featureType; - optional group categories (LIST) { - repeated group bag { - optional double array; - } - } - } - optional int32 leftNodeId; - optional int32 rightNodeId; - optional double infoGain; -} - -Catalyst form: -StructType(StructField(treeId,IntegerType,true), StructField(nodeId,IntegerType,true), StructField(predict,StructType(StructField(predict,DoubleType,true), StructField(prob,DoubleType,true)),true), StructField(impurity,DoubleType,true), StructField(isLeaf,BooleanType,true), StructField(split,StructType(StructField(feature,IntegerType,true), StructField(threshold,DoubleType,true), StructField(featureType,IntegerType,true), StructField(categories,ArrayType(DoubleType,true),true)),true), StructField(leftNodeId,IntegerType,true), StructField(rightNodeId,IntegerType,true), StructField(infoGain,DoubleType,true)) - -2016-05-25 11:14:04 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 11:14:04 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 11:14:04 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 2 records. -2016-05-25 11:14:04 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 11:14:04 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 11:14:04 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 11:14:04 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 11:14:04 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 11:14:04 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 11:14:04 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 11:14:04 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 11:14:04 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 11:14:04 INFO InternalParquetRecordReader:151 - block read in memory in 12 ms. row count = 1 -2016-05-25 11:14:04 INFO InternalParquetRecordReader:151 - block read in memory in 11 ms. row count = 1 -2016-05-25 11:14:04 INFO InternalParquetRecordReader:151 - block read in memory in 12 ms. row count = 2 -2016-05-25 11:14:04 INFO InternalParquetRecordReader:151 - block read in memory in 13 ms. row count = 1 -2016-05-25 11:14:04 INFO Executor:59 - Finished task 3.0 in stage 14.0 (TID 32). 2256 bytes result sent to driver -2016-05-25 11:14:04 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 29). 2256 bytes result sent to driver -2016-05-25 11:14:04 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 30). 2256 bytes result sent to driver -2016-05-25 11:14:04 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 31). 2256 bytes result sent to driver -2016-05-25 11:14:04 INFO TaskSetManager:59 - Finished task 3.0 in stage 14.0 (TID 32) in 732 ms on localhost (1/4) -2016-05-25 11:14:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 29) in 736 ms on localhost (2/4) -2016-05-25 11:14:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 30) in 735 ms on localhost (3/4) -2016-05-25 11:14:04 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 31) in 735 ms on localhost (4/4) -2016-05-25 11:14:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 11:14:04 INFO DAGScheduler:59 - ShuffleMapStage 14 (groupBy at DecisionTreeModel.scala:265) finished in 0.738 s -2016-05-25 11:14:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:14:04 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:14:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 15) -2016-05-25 11:14:04 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:14:04 INFO DAGScheduler:59 - Missing parents for ResultStage 15: List() -2016-05-25 11:14:04 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[37] at mapValues at DecisionTreeModel.scala:266), which is now runnable -2016-05-25 11:14:05 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:53635 in memory (size: 3.7 KB, free: 730.4 MB) -2016-05-25 11:14:30 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 11:14:31 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 11:14:31 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 11:14:31 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 11:14:31 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 11:14:32 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 11:14:32 INFO Remoting:74 - Starting remoting -2016-05-25 11:14:32 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53656] -2016-05-25 11:14:32 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53656. -2016-05-25 11:14:32 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 11:14:32 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 11:14:32 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-5888ccd7-03f4-40c1-aace-27e1c6c12ce5 -2016-05-25 11:14:32 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 11:14:32 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-7a3bff2b-e5b6-4ba8-a0d0-0777f5ae7633\httpd-231f4bf8-6c5c-4996-acbb-42825b81c4b6 -2016-05-25 11:14:32 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 11:14:33 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:14:33 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53657 -2016-05-25 11:14:33 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53657. -2016-05-25 11:14:33 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 11:14:33 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:14:33 WARN AbstractLifeCycle:204 - FAILED SelectChannelConnector@0.0.0.0:4040: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.mllib.learning.classification.DecisionTreesLearning$.main(DecisionTreesLearning.scala:23) - at org.apache.spark.mllib.learning.classification.DecisionTreesLearning.main(DecisionTreesLearning.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-05-25 11:14:33 WARN AbstractLifeCycle:204 - FAILED org.spark-project.jetty.server.Server@71d11087: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.mllib.learning.classification.DecisionTreesLearning$.main(DecisionTreesLearning.scala:23) - at org.apache.spark.mllib.learning.classification.DecisionTreesLearning.main(DecisionTreesLearning.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 11:14:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 11:14:33 WARN Utils:71 - Service 'SparkUI' could not bind on port 4040. Attempting port 4041. -2016-05-25 11:14:33 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:14:33 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4041 -2016-05-25 11:14:33 INFO Utils:59 - Successfully started service 'SparkUI' on port 4041. -2016-05-25 11:14:33 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4041 -2016-05-25 11:14:33 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 11:14:33 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 11:14:33 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53676. -2016-05-25 11:14:33 INFO NettyBlockTransferService:59 - Server created on 53676 -2016-05-25 11:14:33 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 11:14:33 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53676 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53676) -2016-05-25 11:14:33 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 11:14:34 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 11:14:34 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 11:14:34 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 11:14:34 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 11:14:34 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53676 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 11:14:34 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 11:14:36 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 11:14:36 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 11:14:36 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 11:14:36 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 11:14:36 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 11:14:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:14:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:14:36 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 11:14:36 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 11:14:36 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 11:14:36 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 11:14:36 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 11:14:36 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53676 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:14:36 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 11:14:36 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 11:14:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:14:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:14:36 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 11:14:36 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 11:14:36 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 11:14:36 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 11:14:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 11:14:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 11:14:36 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 11:14:36 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 11:14:36 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 11:14:36 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 11:14:36 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 11:14:37 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177630, maxMem=766075207 -2016-05-25 11:14:37 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 11:14:37 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263278, maxMem=766075207 -2016-05-25 11:14:37 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 11:14:37 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:53676 (size: 83.6 KB, free: 730.5 MB) -2016-05-25 11:14:37 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:53676 (size: 83.1 KB, free: 730.4 MB) -2016-05-25 11:14:37 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 11:14:37 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 11:14:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 430 ms on localhost (1/2) -2016-05-25 11:14:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 414 ms on localhost (2/2) -2016-05-25 11:14:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 11:14:37 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.457 s -2016-05-25 11:14:37 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.588128 s -2016-05-25 11:14:37 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 11:14:37 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 11:14:37 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 11:14:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:14:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:14:37 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 11:14:37 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348358, maxMem=766075207 -2016-05-25 11:14:37 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 11:14:37 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=353126, maxMem=766075207 -2016-05-25 11:14:37 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 11:14:37 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53676 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:14:37 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:37 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 11:14:37 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 11:14:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:14:37 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 11:14:37 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:14:37 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 11:14:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 26 ms on localhost (1/1) -2016-05-25 11:14:37 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.026 s -2016-05-25 11:14:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 11:14:37 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.047831 s -2016-05-25 11:14:37 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 11:14:37 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 11:14:37 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 11:14:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:14:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:14:37 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 11:14:37 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355673, maxMem=766075207 -2016-05-25 11:14:37 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 11:14:37 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=360009, maxMem=766075207 -2016-05-25 11:14:37 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 11:14:37 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53676 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:14:37 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 11:14:37 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 11:14:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:14:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:14:37 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 11:14:37 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 11:14:37 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:14:37 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:14:37 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 11:14:37 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 11:14:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 15 ms on localhost (1/2) -2016-05-25 11:14:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 15 ms on localhost (2/2) -2016-05-25 11:14:37 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.017 s -2016-05-25 11:14:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 11:14:37 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.030774 s -2016-05-25 11:14:37 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 11:14:37 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 11:14:37 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 11:14:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:14:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:14:37 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 11:14:37 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362328, maxMem=766075207 -2016-05-25 11:14:37 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 11:14:37 INFO MemoryStore:59 - ensureFreeSpace(2631) called with curMem=367392, maxMem=766075207 -2016-05-25 11:14:37 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 11:14:37 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53676 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:14:37 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 11:14:37 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 11:14:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:14:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:14:37 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 11:14:37 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 11:14:37 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:14:37 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:14:37 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 63496 bytes result sent to driver -2016-05-25 11:14:37 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 68685 bytes result sent to driver -2016-05-25 11:14:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 29 ms on localhost (1/2) -2016-05-25 11:14:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 35 ms on localhost (2/2) -2016-05-25 11:14:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 11:14:37 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.035 s -2016-05-25 11:14:37 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.054267 s -2016-05-25 11:14:37 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53676 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:14:37 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 11:14:37 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53676 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:14:37 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 11:14:37 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53676 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:14:37 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 11:14:37 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53676 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-25 11:14:37 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 11:14:39 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=342308, maxMem=766075207 -2016-05-25 11:14:39 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 11:14:39 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=342348, maxMem=766075207 -2016-05-25 11:14:39 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 11:14:39 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53676 (size: 81.0 B, free: 730.4 MB) -2016-05-25 11:14:39 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 11:14:39 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 11:14:39 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 11:14:39 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 11:14:39 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 11:14:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 11:14:39 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 11:14:39 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 11:14:39 INFO MemoryStore:59 - ensureFreeSpace(505032) called with curMem=342429, maxMem=766075207 -2016-05-25 11:14:39 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 493.2 KB, free 729.8 MB) -2016-05-25 11:14:39 INFO MemoryStore:59 - ensureFreeSpace(123835) called with curMem=847461, maxMem=766075207 -2016-05-25 11:14:39 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 120.9 KB, free 729.7 MB) -2016-05-25 11:14:39 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53676 (size: 120.9 KB, free: 730.3 MB) -2016-05-25 11:14:39 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 11:14:39 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 11:14:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:14:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:14:39 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 11:14:39 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 11:14:39 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 11:14:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:14:39 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 11:14:39 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:14:39 INFO MemoryStore:59 - ensureFreeSpace(105840) called with curMem=971296, maxMem=766075207 -2016-05-25 11:14:39 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 103.4 KB, free 729.6 MB) -2016-05-25 11:14:39 INFO MemoryStore:59 - ensureFreeSpace(100120) called with curMem=1077136, maxMem=766075207 -2016-05-25 11:14:39 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 97.8 KB, free 729.5 MB) -2016-05-25 11:14:39 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:53676 (size: 103.4 KB, free: 730.2 MB) -2016-05-25 11:14:39 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:53676 (size: 97.8 KB, free: 730.1 MB) -2016-05-25 11:14:39 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 11:14:39 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 11:14:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 258 ms on localhost (1/2) -2016-05-25 11:14:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 258 ms on localhost (2/2) -2016-05-25 11:14:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 11:14:39 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.261 s -2016-05-25 11:14:39 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:14:39 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:14:39 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 11:14:39 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:14:39 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 11:14:39 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 11:14:39 INFO MemoryStore:59 - ensureFreeSpace(220456) called with curMem=1177256, maxMem=766075207 -2016-05-25 11:14:39 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 215.3 KB, free 729.3 MB) -2016-05-25 11:14:39 INFO MemoryStore:59 - ensureFreeSpace(46670) called with curMem=1397712, maxMem=766075207 -2016-05-25 11:14:39 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 45.6 KB, free 729.2 MB) -2016-05-25 11:14:39 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53676 (size: 45.6 KB, free: 730.1 MB) -2016-05-25 11:14:39 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 11:14:39 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 11:14:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:14:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:14:39 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 11:14:39 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 11:14:39 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:14:39 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:14:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-25 11:14:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-25 11:14:39 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 11:14:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 154 ms on localhost (1/2) -2016-05-25 11:14:40 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 11:14:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 440 ms on localhost (2/2) -2016-05-25 11:14:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 11:14:40 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.442 s -2016-05-25 11:14:40 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.773868 s -2016-05-25 11:14:40 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=1444382, maxMem=766075207 -2016-05-25 11:14:40 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 40.0 B, free 729.2 MB) -2016-05-25 11:14:40 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=1444422, maxMem=766075207 -2016-05-25 11:14:40 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 81.0 B, free 729.2 MB) -2016-05-25 11:14:40 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53676 (size: 81.0 B, free: 730.1 MB) -2016-05-25 11:14:40 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 11:14:40 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 11:14:40 INFO DAGScheduler:59 - Registering RDD 17 (mapPartitions at DecisionTree.scala:625) -2016-05-25 11:14:40 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 11:14:40 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 11:14:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 11:14:40 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 11:14:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 11:14:40 INFO MemoryStore:59 - ensureFreeSpace(505392) called with curMem=1444503, maxMem=766075207 -2016-05-25 11:14:40 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 493.5 KB, free 728.7 MB) -2016-05-25 11:14:40 INFO MemoryStore:59 - ensureFreeSpace(124020) called with curMem=1949895, maxMem=766075207 -2016-05-25 11:14:40 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 121.1 KB, free 728.6 MB) -2016-05-25 11:14:40 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53676 (size: 121.1 KB, free: 729.9 MB) -2016-05-25 11:14:40 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625) -2016-05-25 11:14:40 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 11:14:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:14:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:14:40 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 11:14:40 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 11:14:40 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-25 11:14:40 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-25 11:14:40 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 11:14:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 118 ms on localhost (1/2) -2016-05-25 11:14:40 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 11:14:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 121 ms on localhost (2/2) -2016-05-25 11:14:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 11:14:40 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.124 s -2016-05-25 11:14:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:14:40 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:14:40 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 11:14:40 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:14:40 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 11:14:40 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 11:14:40 INFO MemoryStore:59 - ensureFreeSpace(220456) called with curMem=2073915, maxMem=766075207 -2016-05-25 11:14:40 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 215.3 KB, free 728.4 MB) -2016-05-25 11:14:40 INFO MemoryStore:59 - ensureFreeSpace(46667) called with curMem=2294371, maxMem=766075207 -2016-05-25 11:14:40 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 45.6 KB, free 728.4 MB) -2016-05-25 11:14:40 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53676 (size: 45.6 KB, free: 729.9 MB) -2016-05-25 11:14:40 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645) -2016-05-25 11:14:40 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 11:14:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:14:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:14:40 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 11:14:40 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 11:14:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:14:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 11:14:40 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:14:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 11:14:40 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 1161 bytes result sent to driver -2016-05-25 11:14:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 29 ms on localhost (1/2) -2016-05-25 11:14:40 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 11:14:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 100 ms on localhost (2/2) -2016-05-25 11:14:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 11:14:40 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.100 s -2016-05-25 11:14:40 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.272800 s -2016-05-25 11:14:40 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 11:14:40 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 11:14:40 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 11:14:40 INFO RandomForest:59 - init: 1.96018309 - total: 3.377364426 - findSplitsBins: 1.676910226 - findBestSplits: 1.39257359 - chooseSplits: 1.384596197 -2016-05-25 11:14:40 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:47 -2016-05-25 11:14:40 INFO DAGScheduler:59 - Got job 6 (count at DecisionTreesLearning.scala:47) with 2 output partitions -2016-05-25 11:14:40 INFO DAGScheduler:59 - Final stage: ResultStage 8(count at DecisionTreesLearning.scala:47) -2016-05-25 11:14:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:14:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:14:40 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at filter at DecisionTreesLearning.scala:47), which has no missing parents -2016-05-25 11:14:40 INFO MemoryStore:59 - ensureFreeSpace(6744) called with curMem=2135078, maxMem=766075207 -2016-05-25 11:14:40 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 6.6 KB, free 728.5 MB) -2016-05-25 11:14:40 INFO MemoryStore:59 - ensureFreeSpace(3589) called with curMem=2141822, maxMem=766075207 -2016-05-25 11:14:40 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.5 KB, free 728.5 MB) -2016-05-25 11:14:40 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53676 (size: 3.5 KB, free: 730.1 MB) -2016-05-25 11:14:40 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at filter at DecisionTreesLearning.scala:47) -2016-05-25 11:14:40 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 11:14:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:14:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:14:40 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 11:14:40 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 11:14:40 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:14:40 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:14:40 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 11:14:40 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 11:14:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 13 ms on localhost (1/2) -2016-05-25 11:14:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 13 ms on localhost (2/2) -2016-05-25 11:14:40 INFO DAGScheduler:59 - ResultStage 8 (count at DecisionTreesLearning.scala:47) finished in 0.014 s -2016-05-25 11:14:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 11:14:40 INFO DAGScheduler:59 - Job 6 finished: count at DecisionTreesLearning.scala:47, took 0.034055 s -2016-05-25 11:14:40 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:47 -2016-05-25 11:14:40 INFO DAGScheduler:59 - Got job 7 (count at DecisionTreesLearning.scala:47) with 2 output partitions -2016-05-25 11:14:40 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at DecisionTreesLearning.scala:47) -2016-05-25 11:14:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:14:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:14:40 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:14:40 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2145411, maxMem=766075207 -2016-05-25 11:14:40 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.0 KB, free 728.5 MB) -2016-05-25 11:14:40 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=2149523, maxMem=766075207 -2016-05-25 11:14:40 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.5 MB) -2016-05-25 11:14:40 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53676 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 11:14:40 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:14:40 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 11:14:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:14:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:14:40 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 11:14:40 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-25 11:14:40 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:14:40 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:14:40 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2082 bytes result sent to driver -2016-05-25 11:14:40 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2082 bytes result sent to driver -2016-05-25 11:14:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 9 ms on localhost (1/2) -2016-05-25 11:14:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 11 ms on localhost (2/2) -2016-05-25 11:14:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 11:14:40 INFO DAGScheduler:59 - ResultStage 9 (count at DecisionTreesLearning.scala:47) finished in 0.011 s -2016-05-25 11:14:40 INFO DAGScheduler:59 - Job 7 finished: count at DecisionTreesLearning.scala:47, took 0.029089 s -2016-05-25 11:14:41 INFO SparkContext:59 - Starting job: saveAsTextFile at DecisionTreeModel.scala:235 -2016-05-25 11:14:41 INFO DAGScheduler:59 - Got job 8 (saveAsTextFile at DecisionTreeModel.scala:235) with 1 output partitions -2016-05-25 11:14:41 INFO DAGScheduler:59 - Final stage: ResultStage 10(saveAsTextFile at DecisionTreeModel.scala:235) -2016-05-25 11:14:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:14:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:14:41 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[23] at saveAsTextFile at DecisionTreeModel.scala:235), which has no missing parents -2016-05-25 11:14:41 INFO MemoryStore:59 - ensureFreeSpace(127000) called with curMem=2151791, maxMem=766075207 -2016-05-25 11:14:41 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 124.0 KB, free 728.4 MB) -2016-05-25 11:14:41 INFO MemoryStore:59 - ensureFreeSpace(42613) called with curMem=2278791, maxMem=766075207 -2016-05-25 11:14:41 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 41.6 KB, free 728.4 MB) -2016-05-25 11:14:41 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53676 (size: 41.6 KB, free: 730.0 MB) -2016-05-25 11:14:41 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:41 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[23] at saveAsTextFile at DecisionTreeModel.scala:235) -2016-05-25 11:14:41 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-25 11:14:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:14:41 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 11:14:49 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 11:14:50 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 11:14:50 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 11:14:50 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 11:14:50 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 11:14:51 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 11:14:51 INFO Remoting:74 - Starting remoting -2016-05-25 11:14:51 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53697] -2016-05-25 11:14:51 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53697. -2016-05-25 11:14:51 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 11:14:51 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 11:14:51 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-1e22b620-a272-4d0e-be88-391b268ced93 -2016-05-25 11:14:51 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 11:14:51 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-72afe40d-d9e1-427a-89d4-67d95731bd6e\httpd-32abdbb0-4df0-4328-be81-224889dbcc28 -2016-05-25 11:14:51 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 11:14:51 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:14:52 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53712 -2016-05-25 11:14:52 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53712. -2016-05-25 11:14:52 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 11:14:52 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:14:52 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 11:14:52 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 11:14:52 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 11:14:52 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 11:14:52 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 11:14:52 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53736. -2016-05-25 11:14:52 INFO NettyBlockTransferService:59 - Server created on 53736 -2016-05-25 11:14:52 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 11:14:52 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53736 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53736) -2016-05-25 11:14:52 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 11:14:53 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 11:14:53 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 11:14:53 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 11:14:53 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 11:14:53 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53736 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 11:14:53 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 11:14:54 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 11:14:55 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 11:14:55 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 11:14:55 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 11:14:55 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 11:14:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:14:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:14:55 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 11:14:55 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 11:14:55 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 11:14:55 INFO MemoryStore:59 - ensureFreeSpace(2147) called with curMem=175484, maxMem=766075207 -2016-05-25 11:14:55 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 11:14:55 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53736 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:14:55 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 11:14:55 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 11:14:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:14:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:14:55 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 11:14:55 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 11:14:55 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 11:14:55 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 11:14:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 11:14:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 11:14:55 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 11:14:55 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 11:14:55 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 11:14:55 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 11:14:55 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 11:14:55 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177631, maxMem=766075207 -2016-05-25 11:14:55 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 11:14:55 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:53736 (size: 83.1 KB, free: 730.5 MB) -2016-05-25 11:14:55 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262711, maxMem=766075207 -2016-05-25 11:14:55 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 11:14:55 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:53736 (size: 83.6 KB, free: 730.4 MB) -2016-05-25 11:14:55 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 11:14:55 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 11:14:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 371 ms on localhost (1/2) -2016-05-25 11:14:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 398 ms on localhost (2/2) -2016-05-25 11:14:55 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.417 s -2016-05-25 11:14:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 11:14:55 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.590095 s -2016-05-25 11:14:56 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 11:14:56 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 11:14:56 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 11:14:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:14:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:14:56 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 11:14:56 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348359, maxMem=766075207 -2016-05-25 11:14:56 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 11:14:56 INFO MemoryStore:59 - ensureFreeSpace(2548) called with curMem=353127, maxMem=766075207 -2016-05-25 11:14:56 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 11:14:56 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53736 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:14:56 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:56 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 11:14:56 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 11:14:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:14:56 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 11:14:56 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:14:56 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 11:14:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 31 ms on localhost (1/1) -2016-05-25 11:14:56 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.031 s -2016-05-25 11:14:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 11:14:56 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.054423 s -2016-05-25 11:14:56 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 11:14:56 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 11:14:56 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 11:14:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:14:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:14:56 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 11:14:56 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355675, maxMem=766075207 -2016-05-25 11:14:56 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 11:14:56 INFO MemoryStore:59 - ensureFreeSpace(2320) called with curMem=360011, maxMem=766075207 -2016-05-25 11:14:56 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 11:14:56 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53736 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:14:56 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 11:14:56 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 11:14:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:14:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:14:56 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 11:14:56 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 11:14:56 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:14:56 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:14:56 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 11:14:56 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 11:14:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 14 ms on localhost (1/2) -2016-05-25 11:14:56 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.014 s -2016-05-25 11:14:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 13 ms on localhost (2/2) -2016-05-25 11:14:56 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.077392 s -2016-05-25 11:14:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 11:14:56 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 11:14:56 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 11:14:56 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 11:14:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:14:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:14:56 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 11:14:56 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362331, maxMem=766075207 -2016-05-25 11:14:56 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 11:14:56 INFO MemoryStore:59 - ensureFreeSpace(2631) called with curMem=367395, maxMem=766075207 -2016-05-25 11:14:56 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 11:14:56 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53736 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:14:56 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 11:14:56 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 11:14:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:14:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:14:56 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 11:14:56 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 11:14:56 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:14:56 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 60157 bytes result sent to driver -2016-05-25 11:14:56 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:14:56 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 60233 bytes result sent to driver -2016-05-25 11:14:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 32 ms on localhost (1/2) -2016-05-25 11:14:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 35 ms on localhost (2/2) -2016-05-25 11:14:56 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.037 s -2016-05-25 11:14:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 11:14:56 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.055903 s -2016-05-25 11:14:56 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53736 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:14:56 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 11:14:56 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53736 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:14:56 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 11:14:56 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53736 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:14:56 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 11:14:56 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53736 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-25 11:14:56 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 11:14:57 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=342308, maxMem=766075207 -2016-05-25 11:14:57 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 11:14:57 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=342348, maxMem=766075207 -2016-05-25 11:14:57 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 11:14:57 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53736 (size: 81.0 B, free: 730.4 MB) -2016-05-25 11:14:57 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 11:14:57 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 11:14:57 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 11:14:57 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 11:14:57 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 11:14:57 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 11:14:57 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 11:14:57 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 11:14:57 INFO MemoryStore:59 - ensureFreeSpace(477336) called with curMem=342429, maxMem=766075207 -2016-05-25 11:14:57 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 466.1 KB, free 729.8 MB) -2016-05-25 11:14:57 INFO MemoryStore:59 - ensureFreeSpace(117228) called with curMem=819765, maxMem=766075207 -2016-05-25 11:14:57 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 114.5 KB, free 729.7 MB) -2016-05-25 11:14:57 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53736 (size: 114.5 KB, free: 730.3 MB) -2016-05-25 11:14:57 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 11:14:57 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 11:14:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:14:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:14:57 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 11:14:57 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 11:14:57 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 11:14:57 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:14:57 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 11:14:57 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:14:58 INFO MemoryStore:59 - ensureFreeSpace(97256) called with curMem=936993, maxMem=766075207 -2016-05-25 11:14:58 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 95.0 KB, free 729.6 MB) -2016-05-25 11:14:58 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:53736 (size: 95.0 KB, free: 730.2 MB) -2016-05-25 11:14:58 INFO MemoryStore:59 - ensureFreeSpace(100120) called with curMem=1034249, maxMem=766075207 -2016-05-25 11:14:58 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 97.8 KB, free 729.5 MB) -2016-05-25 11:14:58 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:53736 (size: 97.8 KB, free: 730.1 MB) -2016-05-25 11:14:58 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 11:14:58 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 11:14:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 297 ms on localhost (1/2) -2016-05-25 11:14:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 300 ms on localhost (2/2) -2016-05-25 11:14:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 11:14:58 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.302 s -2016-05-25 11:14:58 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:14:58 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:14:58 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 11:14:58 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:14:58 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 11:14:58 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 11:14:58 INFO MemoryStore:59 - ensureFreeSpace(206848) called with curMem=1134369, maxMem=766075207 -2016-05-25 11:14:58 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 202.0 KB, free 729.3 MB) -2016-05-25 11:14:58 INFO MemoryStore:59 - ensureFreeSpace(44204) called with curMem=1341217, maxMem=766075207 -2016-05-25 11:14:58 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 43.2 KB, free 729.3 MB) -2016-05-25 11:14:58 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53736 (size: 43.2 KB, free: 730.1 MB) -2016-05-25 11:14:58 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 11:14:58 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 11:14:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:14:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:14:58 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 11:14:58 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 11:14:58 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:14:58 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:14:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 11:14:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 11:14:58 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 11:14:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 135 ms on localhost (1/2) -2016-05-25 11:14:58 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 11:14:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 430 ms on localhost (2/2) -2016-05-25 11:14:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 11:14:58 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.431 s -2016-05-25 11:14:58 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.791911 s -2016-05-25 11:14:58 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=1385421, maxMem=766075207 -2016-05-25 11:14:58 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 40.0 B, free 729.3 MB) -2016-05-25 11:14:58 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=1385461, maxMem=766075207 -2016-05-25 11:14:58 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 81.0 B, free 729.3 MB) -2016-05-25 11:14:58 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53736 (size: 81.0 B, free: 730.1 MB) -2016-05-25 11:14:58 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 11:14:58 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 11:14:58 INFO DAGScheduler:59 - Registering RDD 17 (mapPartitions at DecisionTree.scala:625) -2016-05-25 11:14:58 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 11:14:58 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 11:14:58 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 11:14:58 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 11:14:58 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 11:14:58 INFO MemoryStore:59 - ensureFreeSpace(477688) called with curMem=1385542, maxMem=766075207 -2016-05-25 11:14:58 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 466.5 KB, free 728.8 MB) -2016-05-25 11:14:58 INFO MemoryStore:59 - ensureFreeSpace(117409) called with curMem=1863230, maxMem=766075207 -2016-05-25 11:14:58 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 114.7 KB, free 728.7 MB) -2016-05-25 11:14:58 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53736 (size: 114.7 KB, free: 730.0 MB) -2016-05-25 11:14:58 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625) -2016-05-25 11:14:58 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 11:14:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:14:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:14:58 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 11:14:58 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 11:14:58 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-25 11:14:58 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-25 11:14:58 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 11:14:58 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 11:14:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 120 ms on localhost (1/2) -2016-05-25 11:14:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 119 ms on localhost (2/2) -2016-05-25 11:14:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 11:14:58 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.121 s -2016-05-25 11:14:58 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:14:58 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:14:58 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 11:14:58 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:14:58 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 11:14:58 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 11:14:58 INFO MemoryStore:59 - ensureFreeSpace(206848) called with curMem=1980639, maxMem=766075207 -2016-05-25 11:14:58 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 202.0 KB, free 728.5 MB) -2016-05-25 11:14:58 INFO MemoryStore:59 - ensureFreeSpace(44193) called with curMem=2187487, maxMem=766075207 -2016-05-25 11:14:58 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 43.2 KB, free 728.5 MB) -2016-05-25 11:14:58 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53736 (size: 43.2 KB, free: 729.9 MB) -2016-05-25 11:14:58 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645) -2016-05-25 11:14:58 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 11:14:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:14:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:14:58 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 11:14:58 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 11:14:58 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:14:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 11:14:58 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 1161 bytes result sent to driver -2016-05-25 11:14:58 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:14:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 11:14:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 32 ms on localhost (1/2) -2016-05-25 11:14:58 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 11:14:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 93 ms on localhost (2/2) -2016-05-25 11:14:58 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.093 s -2016-05-25 11:14:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 11:14:58 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.267842 s -2016-05-25 11:14:58 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 11:14:58 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 11:14:58 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 11:14:58 INFO RandomForest:59 - init: 1.55687585 - total: 2.898397338 - findSplitsBins: 1.270343401 - findBestSplits: 1.322256248 - chooseSplits: 1.316486717 -2016-05-25 11:14:58 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:47 -2016-05-25 11:14:58 INFO DAGScheduler:59 - Got job 6 (count at DecisionTreesLearning.scala:47) with 2 output partitions -2016-05-25 11:14:58 INFO DAGScheduler:59 - Final stage: ResultStage 8(count at DecisionTreesLearning.scala:47) -2016-05-25 11:14:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:14:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:14:58 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at filter at DecisionTreesLearning.scala:47), which has no missing parents -2016-05-25 11:14:58 INFO MemoryStore:59 - ensureFreeSpace(6744) called with curMem=2034304, maxMem=766075207 -2016-05-25 11:14:58 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 6.6 KB, free 728.6 MB) -2016-05-25 11:14:58 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=2041048, maxMem=766075207 -2016-05-25 11:14:58 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.5 KB, free 728.6 MB) -2016-05-25 11:14:58 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53736 (size: 3.5 KB, free: 730.1 MB) -2016-05-25 11:14:58 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at filter at DecisionTreesLearning.scala:47) -2016-05-25 11:14:58 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 11:14:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:14:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:14:58 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 11:14:58 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 11:14:58 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:14:58 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:14:58 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 11:14:58 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 11:14:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 13 ms on localhost (1/2) -2016-05-25 11:14:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 17 ms on localhost (2/2) -2016-05-25 11:14:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 11:14:58 INFO DAGScheduler:59 - ResultStage 8 (count at DecisionTreesLearning.scala:47) finished in 0.018 s -2016-05-25 11:14:58 INFO DAGScheduler:59 - Job 6 finished: count at DecisionTreesLearning.scala:47, took 0.035351 s -2016-05-25 11:14:58 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:47 -2016-05-25 11:14:58 INFO DAGScheduler:59 - Got job 7 (count at DecisionTreesLearning.scala:47) with 2 output partitions -2016-05-25 11:14:58 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at DecisionTreesLearning.scala:47) -2016-05-25 11:14:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:14:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:14:58 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:14:58 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2044640, maxMem=766075207 -2016-05-25 11:14:58 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.0 KB, free 728.6 MB) -2016-05-25 11:14:58 INFO MemoryStore:59 - ensureFreeSpace(2269) called with curMem=2048752, maxMem=766075207 -2016-05-25 11:14:59 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.6 MB) -2016-05-25 11:14:59 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53736 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 11:14:59 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:14:59 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 11:14:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:14:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:14:59 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 11:14:59 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-25 11:14:59 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:14:59 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:14:59 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2082 bytes result sent to driver -2016-05-25 11:14:59 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2082 bytes result sent to driver -2016-05-25 11:14:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 12 ms on localhost (1/2) -2016-05-25 11:14:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 15 ms on localhost (2/2) -2016-05-25 11:14:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 11:14:59 INFO DAGScheduler:59 - ResultStage 9 (count at DecisionTreesLearning.scala:47) finished in 0.015 s -2016-05-25 11:14:59 INFO DAGScheduler:59 - Job 7 finished: count at DecisionTreesLearning.scala:47, took 0.033321 s -2016-05-25 11:14:59 INFO SparkContext:59 - Starting job: saveAsTextFile at DecisionTreeModel.scala:235 -2016-05-25 11:14:59 INFO DAGScheduler:59 - Got job 8 (saveAsTextFile at DecisionTreeModel.scala:235) with 1 output partitions -2016-05-25 11:14:59 INFO DAGScheduler:59 - Final stage: ResultStage 10(saveAsTextFile at DecisionTreeModel.scala:235) -2016-05-25 11:14:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:14:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:14:59 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[23] at saveAsTextFile at DecisionTreeModel.scala:235), which has no missing parents -2016-05-25 11:14:59 INFO MemoryStore:59 - ensureFreeSpace(127000) called with curMem=2051021, maxMem=766075207 -2016-05-25 11:14:59 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 124.0 KB, free 728.5 MB) -2016-05-25 11:14:59 INFO MemoryStore:59 - ensureFreeSpace(42613) called with curMem=2178021, maxMem=766075207 -2016-05-25 11:14:59 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 41.6 KB, free 728.5 MB) -2016-05-25 11:14:59 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53736 (size: 41.6 KB, free: 730.1 MB) -2016-05-25 11:14:59 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:14:59 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[23] at saveAsTextFile at DecisionTreeModel.scala:235) -2016-05-25 11:14:59 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-25 11:14:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:14:59 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 11:14:59 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251114_0010_m_000000_19' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111459026/result/metadata/_temporary/0/task_201605251114_0010_m_000000 -2016-05-25 11:14:59 INFO SparkHadoopMapRedUtil:59 - attempt_201605251114_0010_m_000000_19: Committed -2016-05-25 11:14:59 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 1864 bytes result sent to driver -2016-05-25 11:14:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 189 ms on localhost (1/1) -2016-05-25 11:14:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 11:14:59 INFO DAGScheduler:59 - ResultStage 10 (saveAsTextFile at DecisionTreeModel.scala:235) finished in 0.189 s -2016-05-25 11:14:59 INFO DAGScheduler:59 - Job 8 finished: saveAsTextFile at DecisionTreeModel.scala:235, took 0.270665 s -2016-05-25 11:15:01 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:53736 in memory (size: 41.6 KB, free: 730.1 MB) -2016-05-25 11:15:01 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 11:15:01 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:53736 in memory (size: 2.2 KB, free: 730.1 MB) -2016-05-25 11:15:01 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 11:15:01 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:53736 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-25 11:15:01 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 11:15:01 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:53736 in memory (size: 43.2 KB, free: 730.1 MB) -2016-05-25 11:15:01 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 11:15:01 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53736 in memory (size: 114.7 KB, free: 730.3 MB) -2016-05-25 11:15:01 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 11:15:01 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-25 11:15:01 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53736 in memory (size: 81.0 B, free: 730.3 MB) -2016-05-25 11:15:01 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53736 in memory (size: 43.2 KB, free: 730.3 MB) -2016-05-25 11:15:01 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 11:15:01 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53736 in memory (size: 114.5 KB, free: 730.4 MB) -2016-05-25 11:15:01 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 11:15:01 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 11:15:01 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53736 in memory (size: 81.0 B, free: 730.4 MB) -2016-05-25 11:15:01 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 11:15:01 INFO ContextCleaner:59 - Cleaned RDD 13 -2016-05-25 11:15:01 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:15:01 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:15:01 INFO SparkContext:59 - Starting job: parquet at DecisionTreeModel.scala:242 -2016-05-25 11:15:01 INFO DAGScheduler:59 - Got job 9 (parquet at DecisionTreeModel.scala:242) with 4 output partitions -2016-05-25 11:15:01 INFO DAGScheduler:59 - Final stage: ResultStage 11(parquet at DecisionTreeModel.scala:242) -2016-05-25 11:15:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:15:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:15:01 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[26] at rddToDataFrameHolder at DecisionTreeModel.scala:240), which has no missing parents -2016-05-25 11:15:01 INFO MemoryStore:59 - ensureFreeSpace(68760) called with curMem=342308, maxMem=766075207 -2016-05-25 11:15:01 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 67.1 KB, free 730.2 MB) -2016-05-25 11:15:01 INFO MemoryStore:59 - ensureFreeSpace(24136) called with curMem=411068, maxMem=766075207 -2016-05-25 11:15:01 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 23.6 KB, free 730.2 MB) -2016-05-25 11:15:01 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53736 (size: 23.6 KB, free: 730.4 MB) -2016-05-25 11:15:01 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:15:01 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 11 (MapPartitionsRDD[26] at rddToDataFrameHolder at DecisionTreeModel.scala:240) -2016-05-25 11:15:01 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 4 tasks -2016-05-25 11:15:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 20, localhost, PROCESS_LOCAL, 4009 bytes) -2016-05-25 11:15:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 3790 bytes) -2016-05-25 11:15:01 INFO TaskSetManager:59 - Starting task 2.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2428 bytes) -2016-05-25 11:15:01 INFO TaskSetManager:59 - Starting task 3.0 in stage 11.0 (TID 23, localhost, PROCESS_LOCAL, 2489 bytes) -2016-05-25 11:15:01 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 21) -2016-05-25 11:15:01 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 20) -2016-05-25 11:15:01 INFO Executor:59 - Running task 2.0 in stage 11.0 (TID 22) -2016-05-25 11:15:01 INFO Executor:59 - Running task 3.0 in stage 11.0 (TID 23) -2016-05-25 11:15:01 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:15:01 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 11:15:01 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:15:01 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 11:15:01 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:15:01 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 11:15:01 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:15:01 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 11:15:01 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 11:15:01 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 11:15:01 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 11:15:01 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 11:15:01 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 11:15:01 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,676 -2016-05-25 11:15:01 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,676 -2016-05-25 11:15:01 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,644 -2016-05-25 11:15:01 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,620 -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 2 values, 8B raw, 28B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 63B for [nodeId] INT32: 2 values, 14B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [nodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [nodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, predict] DOUBLE: 1 values, 15B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 68B for [predict, predict] DOUBLE: 2 values, 9B raw, 27B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 1 entries, 8B raw, 1B comp} -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 69B for [predict, predict] DOUBLE: 1 values, 15B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 76B for [predict, prob] DOUBLE: 1 values, 15B raw, 35B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, prob] DOUBLE: 1 values, 15B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 68B for [predict, prob] DOUBLE: 2 values, 9B raw, 27B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 1 entries, 8B raw, 1B comp} -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 69B for [impurity] DOUBLE: 1 values, 14B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 69B for [impurity] DOUBLE: 2 values, 8B raw, 28B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 1 entries, 8B raw, 1B comp} -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 2 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, feature] INT32: 2 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, threshold] DOUBLE: 2 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, featureType] INT32: 2 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 51B for [split, categories, bag, array] DOUBLE: 2 values, 14B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 47B for [leftNodeId] INT32: 2 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 47B for [rightNodeId] INT32: 2 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 47B for [infoGain] DOUBLE: 2 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [nodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 75B for [impurity] DOUBLE: 1 values, 14B raw, 34B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, feature] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 64B for [split, feature] INT32: 1 values, 11B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, threshold] DOUBLE: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 69B for [split, threshold] DOUBLE: 1 values, 15B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [split, featureType] INT32: 1 values, 11B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 52B for [split, categories, bag, array] DOUBLE: 1 values, 14B raw, 31B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [leftNodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [rightNodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 75B for [infoGain] DOUBLE: 1 values, 14B raw, 34B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 69B for [predict, predict] DOUBLE: 1 values, 15B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, featureType] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, prob] DOUBLE: 1 values, 15B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 51B for [split, categories, bag, array] DOUBLE: 1 values, 14B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 71B for [impurity] DOUBLE: 1 values, 14B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 47B for [leftNodeId] INT32: 1 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 47B for [rightNodeId] INT32: 1 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 62B for [split, feature] INT32: 1 values, 11B raw, 29B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 47B for [infoGain] DOUBLE: 1 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 72B for [split, threshold] DOUBLE: 1 values, 15B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [split, featureType] INT32: 1 values, 11B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 52B for [split, categories, bag, array] DOUBLE: 1 values, 14B raw, 31B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [leftNodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [rightNodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:01 INFO ColumnChunkPageWriteStore:151 - written 71B for [infoGain] DOUBLE: 1 values, 14B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 11:15:02 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251115_0011_m_000001_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111459026/result/data/_temporary/0/task_201605251115_0011_m_000001 -2016-05-25 11:15:02 INFO SparkHadoopMapRedUtil:59 - attempt_201605251115_0011_m_000001_0: Committed -2016-05-25 11:15:02 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 21). 915 bytes result sent to driver -2016-05-25 11:15:02 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251115_0011_m_000003_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111459026/result/data/_temporary/0/task_201605251115_0011_m_000003 -2016-05-25 11:15:02 INFO SparkHadoopMapRedUtil:59 - attempt_201605251115_0011_m_000003_0: Committed -2016-05-25 11:15:02 INFO Executor:59 - Finished task 3.0 in stage 11.0 (TID 23). 915 bytes result sent to driver -2016-05-25 11:15:02 INFO TaskSetManager:59 - Finished task 3.0 in stage 11.0 (TID 23) in 620 ms on localhost (1/4) -2016-05-25 11:15:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 21) in 622 ms on localhost (2/4) -2016-05-25 11:15:02 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251115_0011_m_000002_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111459026/result/data/_temporary/0/task_201605251115_0011_m_000002 -2016-05-25 11:15:02 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251115_0011_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111459026/result/data/_temporary/0/task_201605251115_0011_m_000000 -2016-05-25 11:15:02 INFO SparkHadoopMapRedUtil:59 - attempt_201605251115_0011_m_000002_0: Committed -2016-05-25 11:15:02 INFO SparkHadoopMapRedUtil:59 - attempt_201605251115_0011_m_000000_0: Committed -2016-05-25 11:15:02 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 20). 915 bytes result sent to driver -2016-05-25 11:15:02 INFO Executor:59 - Finished task 2.0 in stage 11.0 (TID 22). 915 bytes result sent to driver -2016-05-25 11:15:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 20) in 629 ms on localhost (3/4) -2016-05-25 11:15:02 INFO TaskSetManager:59 - Finished task 2.0 in stage 11.0 (TID 22) in 629 ms on localhost (4/4) -2016-05-25 11:15:02 INFO DAGScheduler:59 - ResultStage 11 (parquet at DecisionTreeModel.scala:242) finished in 0.631 s -2016-05-25 11:15:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 11:15:02 INFO DAGScheduler:59 - Job 9 finished: parquet at DecisionTreeModel.scala:242, took 0.670513 s -2016-05-25 11:15:02 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:15:02 INFO DefaultWriterContainer:59 - Job job_201605251115_0000 committed. -2016-05-25 11:15:02 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111459026/result/data on driver -2016-05-25 11:15:02 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111459026/result/data on driver -2016-05-25 11:15:02 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=435204, maxMem=766075207 -2016-05-25 11:15:02 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 225.3 KB, free 730.0 MB) -2016-05-25 11:15:02 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=665908, maxMem=766075207 -2016-05-25 11:15:02 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-25 11:15:02 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53736 (size: 19.3 KB, free: 730.4 MB) -2016-05-25 11:15:02 INFO SparkContext:59 - Created broadcast 15 from textFile at modelSaveLoad.scala:129 -2016-05-25 11:15:02 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 11:15:02 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 11:15:02 INFO DAGScheduler:59 - Got job 10 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 11:15:02 INFO DAGScheduler:59 - Final stage: ResultStage 12(first at modelSaveLoad.scala:129) -2016-05-25 11:15:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:15:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:15:02 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[29] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 11:15:02 INFO MemoryStore:59 - ensureFreeSpace(3224) called with curMem=685696, maxMem=766075207 -2016-05-25 11:15:02 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 3.1 KB, free 729.9 MB) -2016-05-25 11:15:02 INFO MemoryStore:59 - ensureFreeSpace(1899) called with curMem=688920, maxMem=766075207 -2016-05-25 11:15:02 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1899.0 B, free 729.9 MB) -2016-05-25 11:15:02 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53736 (size: 1899.0 B, free: 730.4 MB) -2016-05-25 11:15:02 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:15:02 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[29] at textFile at modelSaveLoad.scala:129) -2016-05-25 11:15:02 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-25 11:15:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2245 bytes) -2016-05-25 11:15:02 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 24) -2016-05-25 11:15:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111459026/result/metadata/part-00000:0+55 -2016-05-25 11:15:02 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 24). 2157 bytes result sent to driver -2016-05-25 11:15:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 24) in 14 ms on localhost (1/1) -2016-05-25 11:15:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 11:15:02 INFO DAGScheduler:59 - ResultStage 12 (first at modelSaveLoad.scala:129) finished in 0.013 s -2016-05-25 11:15:02 INFO DAGScheduler:59 - Job 10 finished: first at modelSaveLoad.scala:129, took 0.030180 s -2016-05-25 11:15:02 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111459026/result/data on driver -2016-05-25 11:15:02 INFO SparkContext:59 - Starting job: parquet at DecisionTreeModel.scala:249 -2016-05-25 11:15:02 INFO DAGScheduler:59 - Got job 11 (parquet at DecisionTreeModel.scala:249) with 4 output partitions -2016-05-25 11:15:02 INFO DAGScheduler:59 - Final stage: ResultStage 13(parquet at DecisionTreeModel.scala:249) -2016-05-25 11:15:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:15:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:15:02 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[31] at parquet at DecisionTreeModel.scala:249), which has no missing parents -2016-05-25 11:15:02 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=690819, maxMem=766075207 -2016-05-25 11:15:02 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 61.5 KB, free 729.9 MB) -2016-05-25 11:15:02 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=753819, maxMem=766075207 -2016-05-25 11:15:02 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 20.6 KB, free 729.8 MB) -2016-05-25 11:15:02 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53736 (size: 20.6 KB, free: 730.3 MB) -2016-05-25 11:15:02 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:15:02 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 13 (MapPartitionsRDD[31] at parquet at DecisionTreeModel.scala:249) -2016-05-25 11:15:02 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 4 tasks -2016-05-25 11:15:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 11:15:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 11:15:02 INFO TaskSetManager:59 - Starting task 2.0 in stage 13.0 (TID 27, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 11:15:02 INFO TaskSetManager:59 - Starting task 3.0 in stage 13.0 (TID 28, localhost, PROCESS_LOCAL, 2286 bytes) -2016-05-25 11:15:02 INFO Executor:59 - Running task 2.0 in stage 13.0 (TID 27) -2016-05-25 11:15:02 INFO Executor:59 - Running task 3.0 in stage 13.0 (TID 28) -2016-05-25 11:15:02 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-25 11:15:02 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-25 11:15:02 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:15:02 INFO Executor:59 - Finished task 2.0 in stage 13.0 (TID 27). 936 bytes result sent to driver -2016-05-25 11:15:02 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:15:02 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 936 bytes result sent to driver -2016-05-25 11:15:02 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:15:02 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 936 bytes result sent to driver -2016-05-25 11:15:02 INFO TaskSetManager:59 - Finished task 2.0 in stage 13.0 (TID 27) in 67 ms on localhost (1/4) -2016-05-25 11:15:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 68 ms on localhost (2/4) -2016-05-25 11:15:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 69 ms on localhost (3/4) -2016-05-25 11:15:02 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:15:02 INFO Executor:59 - Finished task 3.0 in stage 13.0 (TID 28). 2321 bytes result sent to driver -2016-05-25 11:15:02 INFO TaskSetManager:59 - Finished task 3.0 in stage 13.0 (TID 28) in 91 ms on localhost (4/4) -2016-05-25 11:15:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 11:15:02 INFO DAGScheduler:59 - ResultStage 13 (parquet at DecisionTreeModel.scala:249) finished in 0.092 s -2016-05-25 11:15:02 INFO DAGScheduler:59 - Job 11 finished: parquet at DecisionTreeModel.scala:249, took 0.119888 s -2016-05-25 11:15:03 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=774892, maxMem=766075207 -2016-05-25 11:15:03 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 86.4 KB, free 729.8 MB) -2016-05-25 11:15:03 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=863404, maxMem=766075207 -2016-05-25 11:15:03 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.7 MB) -2016-05-25 11:15:03 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53736 (size: 19.3 KB, free: 730.3 MB) -2016-05-25 11:15:03 INFO SparkContext:59 - Created broadcast 18 from map at DecisionTreeModel.scala:252 -2016-05-25 11:15:03 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 11:15:03 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111459026/result/data/part-r-00002-661435ef-5a63-4bcf-a112-93f52e3604f1.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111459026/result/data/part-r-00000-661435ef-5a63-4bcf-a112-93f52e3604f1.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111459026/result/data/part-r-00001-661435ef-5a63-4bcf-a112-93f52e3604f1.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111459026/result/data/part-r-00003-661435ef-5a63-4bcf-a112-93f52e3604f1.gz.parquet -2016-05-25 11:15:03 INFO SparkContext:59 - Starting job: collect at DecisionTreeModel.scala:267 -2016-05-25 11:15:03 INFO DAGScheduler:59 - Registering RDD 35 (groupBy at DecisionTreeModel.scala:265) -2016-05-25 11:15:03 INFO DAGScheduler:59 - Got job 12 (collect at DecisionTreeModel.scala:267) with 4 output partitions -2016-05-25 11:15:03 INFO DAGScheduler:59 - Final stage: ResultStage 15(collect at DecisionTreeModel.scala:267) -2016-05-25 11:15:03 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 14) -2016-05-25 11:15:03 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 14) -2016-05-25 11:15:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 14 (MapPartitionsRDD[35] at groupBy at DecisionTreeModel.scala:265), which has no missing parents -2016-05-25 11:15:03 INFO MemoryStore:59 - ensureFreeSpace(7248) called with curMem=883192, maxMem=766075207 -2016-05-25 11:15:03 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 7.1 KB, free 729.7 MB) -2016-05-25 11:15:03 INFO MemoryStore:59 - ensureFreeSpace(3789) called with curMem=890440, maxMem=766075207 -2016-05-25 11:15:03 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.7 KB, free 729.7 MB) -2016-05-25 11:15:03 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53736 (size: 3.7 KB, free: 730.3 MB) -2016-05-25 11:15:03 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:15:03 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 14 (MapPartitionsRDD[35] at groupBy at DecisionTreeModel.scala:265) -2016-05-25 11:15:03 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 4 tasks -2016-05-25 11:15:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 29, localhost, PROCESS_LOCAL, 2290 bytes) -2016-05-25 11:15:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 30, localhost, PROCESS_LOCAL, 2290 bytes) -2016-05-25 11:15:03 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 31, localhost, PROCESS_LOCAL, 2292 bytes) -2016-05-25 11:15:03 INFO TaskSetManager:59 - Starting task 3.0 in stage 14.0 (TID 32, localhost, PROCESS_LOCAL, 2290 bytes) -2016-05-25 11:15:03 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 30) -2016-05-25 11:15:03 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 31) -2016-05-25 11:15:03 INFO Executor:59 - Running task 3.0 in stage 14.0 (TID 32) -2016-05-25 11:15:03 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 29) -2016-05-25 11:15:03 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111459026/result/data/part-r-00001-661435ef-5a63-4bcf-a112-93f52e3604f1.gz.parquet start: 0 end: 3032 length: 3032 hosts: []} -2016-05-25 11:15:03 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111459026/result/data/part-r-00002-661435ef-5a63-4bcf-a112-93f52e3604f1.gz.parquet start: 0 end: 2836 length: 2836 hosts: []} -2016-05-25 11:15:03 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111459026/result/data/part-r-00003-661435ef-5a63-4bcf-a112-93f52e3604f1.gz.parquet start: 0 end: 2987 length: 2987 hosts: []} -2016-05-25 11:15:03 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111459026/result/data/part-r-00000-661435ef-5a63-4bcf-a112-93f52e3604f1.gz.parquet start: 0 end: 3047 length: 3047 hosts: []} -2016-05-25 11:15:03 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 11:15:03 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 11:15:03 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 11:15:03 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 11:15:03 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:53736 in memory (size: 20.6 KB, free: 730.3 MB) -2016-05-25 11:15:03 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 treeId; - optional int32 nodeId; - optional group predict { - optional double predict; - optional double prob; - } - optional double impurity; - optional boolean isLeaf; - optional group split { - optional int32 feature; - optional double threshold; - optional int32 featureType; - optional group categories (LIST) { - repeated group bag { - optional double array; - } - } - } - optional int32 leftNodeId; - optional int32 rightNodeId; - optional double infoGain; -} - -Catalyst form: -StructType(StructField(treeId,IntegerType,true), StructField(nodeId,IntegerType,true), StructField(predict,StructType(StructField(predict,DoubleType,true), StructField(prob,DoubleType,true)),true), StructField(impurity,DoubleType,true), StructField(isLeaf,BooleanType,true), StructField(split,StructType(StructField(feature,IntegerType,true), StructField(threshold,DoubleType,true), StructField(featureType,IntegerType,true), StructField(categories,ArrayType(DoubleType,true),true)),true), StructField(leftNodeId,IntegerType,true), StructField(rightNodeId,IntegerType,true), StructField(infoGain,DoubleType,true)) - -2016-05-25 11:15:03 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 treeId; - optional int32 nodeId; - optional group predict { - optional double predict; - optional double prob; - } - optional double impurity; - optional boolean isLeaf; - optional group split { - optional int32 feature; - optional double threshold; - optional int32 featureType; - optional group categories (LIST) { - repeated group bag { - optional double array; - } - } - } - optional int32 leftNodeId; - optional int32 rightNodeId; - optional double infoGain; -} - -Catalyst form: -StructType(StructField(treeId,IntegerType,true), StructField(nodeId,IntegerType,true), StructField(predict,StructType(StructField(predict,DoubleType,true), StructField(prob,DoubleType,true)),true), StructField(impurity,DoubleType,true), StructField(isLeaf,BooleanType,true), StructField(split,StructType(StructField(feature,IntegerType,true), StructField(threshold,DoubleType,true), StructField(featureType,IntegerType,true), StructField(categories,ArrayType(DoubleType,true),true)),true), StructField(leftNodeId,IntegerType,true), StructField(rightNodeId,IntegerType,true), StructField(infoGain,DoubleType,true)) - -2016-05-25 11:15:03 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 treeId; - optional int32 nodeId; - optional group predict { - optional double predict; - optional double prob; - } - optional double impurity; - optional boolean isLeaf; - optional group split { - optional int32 feature; - optional double threshold; - optional int32 featureType; - optional group categories (LIST) { - repeated group bag { - optional double array; - } - } - } - optional int32 leftNodeId; - optional int32 rightNodeId; - optional double infoGain; -} - -Catalyst form: -StructType(StructField(treeId,IntegerType,true), StructField(nodeId,IntegerType,true), StructField(predict,StructType(StructField(predict,DoubleType,true), StructField(prob,DoubleType,true)),true), StructField(impurity,DoubleType,true), StructField(isLeaf,BooleanType,true), StructField(split,StructType(StructField(feature,IntegerType,true), StructField(threshold,DoubleType,true), StructField(featureType,IntegerType,true), StructField(categories,ArrayType(DoubleType,true),true)),true), StructField(leftNodeId,IntegerType,true), StructField(rightNodeId,IntegerType,true), StructField(infoGain,DoubleType,true)) - -2016-05-25 11:15:03 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-25 11:15:04 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 treeId; - optional int32 nodeId; - optional group predict { - optional double predict; - optional double prob; - } - optional double impurity; - optional boolean isLeaf; - optional group split { - optional int32 feature; - optional double threshold; - optional int32 featureType; - optional group categories (LIST) { - repeated group bag { - optional double array; - } - } - } - optional int32 leftNodeId; - optional int32 rightNodeId; - optional double infoGain; -} - -Catalyst form: -StructType(StructField(treeId,IntegerType,true), StructField(nodeId,IntegerType,true), StructField(predict,StructType(StructField(predict,DoubleType,true), StructField(prob,DoubleType,true)),true), StructField(impurity,DoubleType,true), StructField(isLeaf,BooleanType,true), StructField(split,StructType(StructField(feature,IntegerType,true), StructField(threshold,DoubleType,true), StructField(featureType,IntegerType,true), StructField(categories,ArrayType(DoubleType,true),true)),true), StructField(leftNodeId,IntegerType,true), StructField(rightNodeId,IntegerType,true), StructField(infoGain,DoubleType,true)) - -2016-05-25 11:15:04 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:53736 in memory (size: 1899.0 B, free: 730.3 MB) -2016-05-25 11:15:04 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-25 11:15:04 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:53736 in memory (size: 19.3 KB, free: 730.4 MB) -2016-05-25 11:15:04 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:53736 in memory (size: 23.6 KB, free: 730.4 MB) -2016-05-25 11:15:04 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-25 11:15:04 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 11:15:04 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 11:15:04 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 2 records. -2016-05-25 11:15:04 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 11:15:04 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 11:15:04 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 11:15:04 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 11:15:04 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 11:15:04 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 11:15:04 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 11:15:04 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 11:15:04 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 11:15:04 INFO InternalParquetRecordReader:151 - block read in memory in 10 ms. row count = 1 -2016-05-25 11:15:04 INFO InternalParquetRecordReader:151 - block read in memory in 10 ms. row count = 2 -2016-05-25 11:15:04 INFO InternalParquetRecordReader:151 - block read in memory in 10 ms. row count = 1 -2016-05-25 11:15:04 INFO InternalParquetRecordReader:151 - block read in memory in 10 ms. row count = 1 -2016-05-25 11:15:04 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 31). 2256 bytes result sent to driver -2016-05-25 11:15:04 INFO Executor:59 - Finished task 3.0 in stage 14.0 (TID 32). 2256 bytes result sent to driver -2016-05-25 11:15:04 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 30). 2256 bytes result sent to driver -2016-05-25 11:15:04 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 29). 2256 bytes result sent to driver -2016-05-25 11:15:04 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 31) in 752 ms on localhost (1/4) -2016-05-25 11:15:04 INFO TaskSetManager:59 - Finished task 3.0 in stage 14.0 (TID 32) in 751 ms on localhost (2/4) -2016-05-25 11:15:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 30) in 754 ms on localhost (3/4) -2016-05-25 11:15:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 29) in 755 ms on localhost (4/4) -2016-05-25 11:15:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 11:15:04 INFO DAGScheduler:59 - ShuffleMapStage 14 (groupBy at DecisionTreeModel.scala:265) finished in 0.756 s -2016-05-25 11:15:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:15:04 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:15:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 15) -2016-05-25 11:15:04 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:15:04 INFO DAGScheduler:59 - Missing parents for ResultStage 15: List() -2016-05-25 11:15:04 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[37] at mapValues at DecisionTreeModel.scala:266), which is now runnable -2016-05-25 11:15:05 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:53736 in memory (size: 3.7 KB, free: 730.4 MB) -2016-05-25 11:15:54 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 11:15:55 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 11:15:55 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 11:15:55 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 11:15:55 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 11:16:21 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 11:16:21 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 11:16:21 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 11:16:21 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 11:16:21 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 11:16:22 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 11:16:22 INFO Remoting:74 - Starting remoting -2016-05-25 11:16:23 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53760] -2016-05-25 11:16:23 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53760. -2016-05-25 11:16:23 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 11:16:23 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 11:16:23 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-91d2d3f1-fb92-4634-afea-e3807b6b641f -2016-05-25 11:16:23 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 11:16:23 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-e75c3028-cc11-4b88-9c2a-98c42b4caa8f\httpd-d0d96a4f-49c1-42ee-9d8a-ccca64bca604 -2016-05-25 11:16:23 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 11:16:23 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:16:23 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53761 -2016-05-25 11:16:23 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53761. -2016-05-25 11:16:23 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 11:16:23 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:16:23 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 11:16:23 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 11:16:23 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 11:16:23 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 11:16:23 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 11:16:24 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53780. -2016-05-25 11:16:24 INFO NettyBlockTransferService:59 - Server created on 53780 -2016-05-25 11:16:24 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 11:16:24 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53780 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53780) -2016-05-25 11:16:24 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 11:16:24 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 11:16:24 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 11:16:24 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 11:16:24 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 11:16:24 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53780 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 11:16:24 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 11:16:26 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 11:16:26 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 11:16:26 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 11:16:26 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 11:16:26 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 11:16:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:16:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:16:26 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 11:16:26 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 11:16:26 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 11:16:26 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 11:16:26 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 11:16:26 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53780 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:16:26 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:16:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 11:16:26 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 11:16:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:16:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:16:26 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 11:16:26 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 11:16:26 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 11:16:26 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 11:16:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 11:16:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 11:16:26 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 11:16:26 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 11:16:26 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 11:16:26 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 11:16:26 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 11:16:27 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177630, maxMem=766075207 -2016-05-25 11:16:27 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 11:16:27 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263278, maxMem=766075207 -2016-05-25 11:16:27 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 11:16:27 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:53780 (size: 83.6 KB, free: 730.5 MB) -2016-05-25 11:16:27 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:53780 (size: 83.1 KB, free: 730.4 MB) -2016-05-25 11:16:27 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 11:16:27 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 11:16:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 394 ms on localhost (1/2) -2016-05-25 11:16:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 418 ms on localhost (2/2) -2016-05-25 11:16:27 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.437 s -2016-05-25 11:16:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 11:16:27 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.597396 s -2016-05-25 11:16:27 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 11:16:27 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 11:16:27 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 11:16:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:16:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:16:27 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 11:16:27 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348358, maxMem=766075207 -2016-05-25 11:16:27 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 11:16:27 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=353126, maxMem=766075207 -2016-05-25 11:16:27 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 11:16:27 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53780 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:16:27 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:16:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 11:16:27 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 11:16:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:16:27 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 11:16:27 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:16:27 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 11:16:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 28 ms on localhost (1/1) -2016-05-25 11:16:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 11:16:27 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.028 s -2016-05-25 11:16:27 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.047315 s -2016-05-25 11:16:27 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 11:16:27 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 11:16:27 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 11:16:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:16:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:16:27 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 11:16:27 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355673, maxMem=766075207 -2016-05-25 11:16:27 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 11:16:27 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=360009, maxMem=766075207 -2016-05-25 11:16:27 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 11:16:27 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53780 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:16:27 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:16:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 11:16:27 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 11:16:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:16:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:16:27 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 11:16:27 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 11:16:27 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:16:27 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:16:27 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 11:16:27 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 11:16:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 14 ms on localhost (1/2) -2016-05-25 11:16:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 14 ms on localhost (2/2) -2016-05-25 11:16:27 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.015 s -2016-05-25 11:16:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 11:16:27 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.028251 s -2016-05-25 11:16:27 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 11:16:27 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 11:16:27 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 11:16:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:16:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:16:27 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 11:16:27 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362328, maxMem=766075207 -2016-05-25 11:16:27 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 11:16:27 INFO MemoryStore:59 - ensureFreeSpace(2631) called with curMem=367392, maxMem=766075207 -2016-05-25 11:16:27 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 11:16:27 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53780 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:16:27 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:16:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 11:16:27 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 11:16:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:16:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:16:27 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 11:16:27 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 11:16:27 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:16:27 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:16:27 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 55132 bytes result sent to driver -2016-05-25 11:16:27 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 57006 bytes result sent to driver -2016-05-25 11:16:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 26 ms on localhost (1/2) -2016-05-25 11:16:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 30 ms on localhost (2/2) -2016-05-25 11:16:27 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.031 s -2016-05-25 11:16:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 11:16:27 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.049888 s -2016-05-25 11:16:28 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53780 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:16:28 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53780 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:16:28 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 11:16:28 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 11:16:28 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53780 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:16:28 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 11:16:28 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=348358, maxMem=766075207 -2016-05-25 11:16:28 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 11:16:28 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=348398, maxMem=766075207 -2016-05-25 11:16:28 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 11:16:28 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53780 (size: 81.0 B, free: 730.4 MB) -2016-05-25 11:16:28 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 11:16:29 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 11:16:29 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 11:16:29 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 11:16:29 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 11:16:29 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 11:16:29 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 11:16:29 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 11:16:29 INFO MemoryStore:59 - ensureFreeSpace(450144) called with curMem=348479, maxMem=766075207 -2016-05-25 11:16:29 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 439.6 KB, free 729.8 MB) -2016-05-25 11:16:29 INFO MemoryStore:59 - ensureFreeSpace(110354) called with curMem=798623, maxMem=766075207 -2016-05-25 11:16:29 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 107.8 KB, free 729.7 MB) -2016-05-25 11:16:29 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53780 (size: 107.8 KB, free: 730.3 MB) -2016-05-25 11:16:29 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:16:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 11:16:29 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 11:16:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:16:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:16:29 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 11:16:29 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 11:16:29 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 11:16:29 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:16:29 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 11:16:29 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:16:29 INFO MemoryStore:59 - ensureFreeSpace(88680) called with curMem=908977, maxMem=766075207 -2016-05-25 11:16:29 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 86.6 KB, free 729.6 MB) -2016-05-25 11:16:29 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:53780 (size: 86.6 KB, free: 730.2 MB) -2016-05-25 11:16:29 INFO MemoryStore:59 - ensureFreeSpace(94400) called with curMem=997657, maxMem=766075207 -2016-05-25 11:16:29 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 92.2 KB, free 729.5 MB) -2016-05-25 11:16:29 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:53780 (size: 92.2 KB, free: 730.1 MB) -2016-05-25 11:16:29 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 11:16:29 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 11:16:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 260 ms on localhost (1/2) -2016-05-25 11:16:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 262 ms on localhost (2/2) -2016-05-25 11:16:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 11:16:29 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.264 s -2016-05-25 11:16:29 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:16:29 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:16:29 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 11:16:29 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:16:29 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 11:16:29 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 11:16:29 INFO MemoryStore:59 - ensureFreeSpace(193496) called with curMem=1092057, maxMem=766075207 -2016-05-25 11:16:29 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 189.0 KB, free 729.4 MB) -2016-05-25 11:16:29 INFO MemoryStore:59 - ensureFreeSpace(41492) called with curMem=1285553, maxMem=766075207 -2016-05-25 11:16:29 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 40.5 KB, free 729.3 MB) -2016-05-25 11:16:29 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53780 (size: 40.5 KB, free: 730.1 MB) -2016-05-25 11:16:29 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:16:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 11:16:29 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 11:16:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:16:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:16:29 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 11:16:29 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 11:16:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:16:29 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:16:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-25 11:16:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-25 11:16:29 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 11:16:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 142 ms on localhost (1/2) -2016-05-25 11:16:29 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 11:16:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 399 ms on localhost (2/2) -2016-05-25 11:16:29 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.401 s -2016-05-25 11:16:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 11:16:29 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.731854 s -2016-05-25 11:16:29 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=1327045, maxMem=766075207 -2016-05-25 11:16:29 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 40.0 B, free 729.3 MB) -2016-05-25 11:16:29 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=1327085, maxMem=766075207 -2016-05-25 11:16:29 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 81.0 B, free 729.3 MB) -2016-05-25 11:16:29 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53780 (size: 81.0 B, free: 730.1 MB) -2016-05-25 11:16:29 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 11:16:29 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 11:16:29 INFO DAGScheduler:59 - Registering RDD 17 (mapPartitions at DecisionTree.scala:625) -2016-05-25 11:16:29 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 11:16:29 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 11:16:29 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 11:16:29 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 11:16:29 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 11:16:29 INFO MemoryStore:59 - ensureFreeSpace(450496) called with curMem=1327166, maxMem=766075207 -2016-05-25 11:16:29 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 439.9 KB, free 728.9 MB) -2016-05-25 11:16:29 INFO MemoryStore:59 - ensureFreeSpace(110537) called with curMem=1777662, maxMem=766075207 -2016-05-25 11:16:29 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 107.9 KB, free 728.8 MB) -2016-05-25 11:16:29 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53780 (size: 107.9 KB, free: 730.0 MB) -2016-05-25 11:16:29 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:16:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625) -2016-05-25 11:16:29 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 11:16:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:16:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:16:29 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 11:16:29 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 11:16:30 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-25 11:16:30 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 11:16:30 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-25 11:16:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 149 ms on localhost (1/2) -2016-05-25 11:16:30 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 11:16:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 160 ms on localhost (2/2) -2016-05-25 11:16:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 11:16:30 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.163 s -2016-05-25 11:16:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:16:30 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:16:30 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 11:16:30 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:16:30 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 11:16:30 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 11:16:30 INFO MemoryStore:59 - ensureFreeSpace(193496) called with curMem=1888199, maxMem=766075207 -2016-05-25 11:16:30 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 189.0 KB, free 728.6 MB) -2016-05-25 11:16:30 INFO MemoryStore:59 - ensureFreeSpace(41485) called with curMem=2081695, maxMem=766075207 -2016-05-25 11:16:30 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 40.5 KB, free 728.6 MB) -2016-05-25 11:16:30 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53780 (size: 40.5 KB, free: 729.9 MB) -2016-05-25 11:16:30 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:16:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645) -2016-05-25 11:16:30 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 11:16:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:16:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:16:30 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 11:16:30 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 11:16:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:16:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 11:16:30 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:16:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 11:16:30 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 1161 bytes result sent to driver -2016-05-25 11:16:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 33 ms on localhost (1/2) -2016-05-25 11:16:30 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 11:16:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 97 ms on localhost (2/2) -2016-05-25 11:16:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 11:16:30 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.098 s -2016-05-25 11:16:30 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.313079 s -2016-05-25 11:16:30 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 11:16:30 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 11:16:30 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 11:16:30 INFO RandomForest:59 - init: 1.530818358 - total: 2.902901409 - findSplitsBins: 1.286696655 - findBestSplits: 1.350966921 - chooseSplits: 1.343232268 -2016-05-25 11:16:30 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:47 -2016-05-25 11:16:30 INFO DAGScheduler:59 - Got job 6 (count at DecisionTreesLearning.scala:47) with 2 output partitions -2016-05-25 11:16:30 INFO DAGScheduler:59 - Final stage: ResultStage 8(count at DecisionTreesLearning.scala:47) -2016-05-25 11:16:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:16:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:16:30 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at filter at DecisionTreesLearning.scala:47), which has no missing parents -2016-05-25 11:16:30 INFO MemoryStore:59 - ensureFreeSpace(6744) called with curMem=1940100, maxMem=766075207 -2016-05-25 11:16:30 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 6.6 KB, free 728.7 MB) -2016-05-25 11:16:30 INFO MemoryStore:59 - ensureFreeSpace(3582) called with curMem=1946844, maxMem=766075207 -2016-05-25 11:16:30 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.5 KB, free 728.7 MB) -2016-05-25 11:16:30 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53780 (size: 3.5 KB, free: 730.1 MB) -2016-05-25 11:16:30 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:16:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at filter at DecisionTreesLearning.scala:47) -2016-05-25 11:16:30 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 11:16:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:16:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:16:30 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 11:16:30 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 11:16:30 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:16:30 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:16:30 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 11:16:30 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 11:16:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 11 ms on localhost (1/2) -2016-05-25 11:16:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 14 ms on localhost (2/2) -2016-05-25 11:16:30 INFO DAGScheduler:59 - ResultStage 8 (count at DecisionTreesLearning.scala:47) finished in 0.014 s -2016-05-25 11:16:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 11:16:30 INFO DAGScheduler:59 - Job 6 finished: count at DecisionTreesLearning.scala:47, took 0.035118 s -2016-05-25 11:16:30 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:47 -2016-05-25 11:16:30 INFO DAGScheduler:59 - Got job 7 (count at DecisionTreesLearning.scala:47) with 2 output partitions -2016-05-25 11:16:30 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at DecisionTreesLearning.scala:47) -2016-05-25 11:16:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:16:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:16:30 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:16:30 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1950426, maxMem=766075207 -2016-05-25 11:16:30 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.0 KB, free 728.7 MB) -2016-05-25 11:16:30 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=1954538, maxMem=766075207 -2016-05-25 11:16:30 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.7 MB) -2016-05-25 11:16:30 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53780 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 11:16:30 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:16:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:16:30 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 11:16:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:16:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:16:30 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 11:16:30 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-25 11:16:30 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:16:30 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:16:30 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2082 bytes result sent to driver -2016-05-25 11:16:30 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2082 bytes result sent to driver -2016-05-25 11:16:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 11 ms on localhost (1/2) -2016-05-25 11:16:30 INFO DAGScheduler:59 - ResultStage 9 (count at DecisionTreesLearning.scala:47) finished in 0.012 s -2016-05-25 11:16:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 11 ms on localhost (2/2) -2016-05-25 11:16:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 11:16:30 INFO DAGScheduler:59 - Job 7 finished: count at DecisionTreesLearning.scala:47, took 0.031519 s -2016-05-25 11:16:30 INFO SparkContext:59 - Starting job: saveAsTextFile at DecisionTreeModel.scala:235 -2016-05-25 11:16:30 INFO DAGScheduler:59 - Got job 8 (saveAsTextFile at DecisionTreeModel.scala:235) with 1 output partitions -2016-05-25 11:16:30 INFO DAGScheduler:59 - Final stage: ResultStage 10(saveAsTextFile at DecisionTreeModel.scala:235) -2016-05-25 11:16:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:16:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:16:30 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[23] at saveAsTextFile at DecisionTreeModel.scala:235), which has no missing parents -2016-05-25 11:16:30 INFO MemoryStore:59 - ensureFreeSpace(127000) called with curMem=1956806, maxMem=766075207 -2016-05-25 11:16:30 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 124.0 KB, free 728.6 MB) -2016-05-25 11:16:30 INFO MemoryStore:59 - ensureFreeSpace(42613) called with curMem=2083806, maxMem=766075207 -2016-05-25 11:16:30 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 41.6 KB, free 728.6 MB) -2016-05-25 11:16:30 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53780 (size: 41.6 KB, free: 730.1 MB) -2016-05-25 11:16:30 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:16:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[23] at saveAsTextFile at DecisionTreeModel.scala:235) -2016-05-25 11:16:30 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-25 11:16:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:16:30 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 11:16:31 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251116_0010_m_000000_19' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111630273/result/metadata/_temporary/0/task_201605251116_0010_m_000000 -2016-05-25 11:16:31 INFO SparkHadoopMapRedUtil:59 - attempt_201605251116_0010_m_000000_19: Committed -2016-05-25 11:16:31 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 1864 bytes result sent to driver -2016-05-25 11:16:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 182 ms on localhost (1/1) -2016-05-25 11:16:31 INFO DAGScheduler:59 - ResultStage 10 (saveAsTextFile at DecisionTreeModel.scala:235) finished in 0.183 s -2016-05-25 11:16:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 11:16:31 INFO DAGScheduler:59 - Job 8 finished: saveAsTextFile at DecisionTreeModel.scala:235, took 0.261758 s -2016-05-25 11:16:31 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:53780 in memory (size: 41.6 KB, free: 730.1 MB) -2016-05-25 11:16:31 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 11:16:31 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:53780 in memory (size: 2.2 KB, free: 730.1 MB) -2016-05-25 11:16:31 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 11:16:31 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:53780 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-25 11:16:31 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 11:16:31 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:53780 in memory (size: 40.5 KB, free: 730.2 MB) -2016-05-25 11:16:31 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 11:16:31 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53780 in memory (size: 107.9 KB, free: 730.3 MB) -2016-05-25 11:16:31 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 11:16:31 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-25 11:16:31 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53780 in memory (size: 81.0 B, free: 730.3 MB) -2016-05-25 11:16:31 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53780 in memory (size: 40.5 KB, free: 730.3 MB) -2016-05-25 11:16:31 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 11:16:31 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53780 in memory (size: 107.8 KB, free: 730.4 MB) -2016-05-25 11:16:31 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 11:16:31 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 11:16:31 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53780 in memory (size: 81.0 B, free: 730.4 MB) -2016-05-25 11:16:31 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 11:16:31 INFO ContextCleaner:59 - Cleaned RDD 13 -2016-05-25 11:16:32 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:16:32 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:16:32 INFO SparkContext:59 - Starting job: parquet at DecisionTreeModel.scala:242 -2016-05-25 11:16:32 INFO DAGScheduler:59 - Got job 9 (parquet at DecisionTreeModel.scala:242) with 4 output partitions -2016-05-25 11:16:32 INFO DAGScheduler:59 - Final stage: ResultStage 11(parquet at DecisionTreeModel.scala:242) -2016-05-25 11:16:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:16:32 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:16:32 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[26] at rddToDataFrameHolder at DecisionTreeModel.scala:240), which has no missing parents -2016-05-25 11:16:32 INFO MemoryStore:59 - ensureFreeSpace(68760) called with curMem=348358, maxMem=766075207 -2016-05-25 11:16:32 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 67.1 KB, free 730.2 MB) -2016-05-25 11:16:32 INFO MemoryStore:59 - ensureFreeSpace(24134) called with curMem=417118, maxMem=766075207 -2016-05-25 11:16:32 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 23.6 KB, free 730.2 MB) -2016-05-25 11:16:32 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53780 (size: 23.6 KB, free: 730.4 MB) -2016-05-25 11:16:32 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:16:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 11 (MapPartitionsRDD[26] at rddToDataFrameHolder at DecisionTreeModel.scala:240) -2016-05-25 11:16:32 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 4 tasks -2016-05-25 11:16:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 20, localhost, PROCESS_LOCAL, 4009 bytes) -2016-05-25 11:16:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 3790 bytes) -2016-05-25 11:16:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2428 bytes) -2016-05-25 11:16:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 11.0 (TID 23, localhost, PROCESS_LOCAL, 2489 bytes) -2016-05-25 11:16:32 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 20) -2016-05-25 11:16:32 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 21) -2016-05-25 11:16:32 INFO Executor:59 - Running task 2.0 in stage 11.0 (TID 22) -2016-05-25 11:16:32 INFO Executor:59 - Running task 3.0 in stage 11.0 (TID 23) -2016-05-25 11:16:32 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:16:32 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 11:16:32 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 11:16:32 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:16:32 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 11:16:32 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 11:16:32 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 11:16:32 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 11:16:32 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 11:16:32 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 11:16:32 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 11:16:32 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 11:16:32 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 11:16:33 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,676 -2016-05-25 11:16:33 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,644 -2016-05-25 11:16:33 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,620 -2016-05-25 11:16:33 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,676 -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 61B for [nodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 69B for [predict, predict] DOUBLE: 1 values, 15B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 76B for [predict, prob] DOUBLE: 1 values, 15B raw, 35B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 75B for [impurity] DOUBLE: 1 values, 14B raw, 34B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 1 values, 7B raw, 27B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 62B for [split, feature] INT32: 1 values, 11B raw, 29B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 72B for [split, threshold] DOUBLE: 1 values, 15B raw, 31B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 61B for [split, featureType] INT32: 1 values, 11B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 52B for [split, categories, bag, array] DOUBLE: 1 values, 14B raw, 31B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 61B for [leftNodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 61B for [rightNodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 75B for [infoGain] DOUBLE: 1 values, 14B raw, 34B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 61B for [nodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 69B for [predict, predict] DOUBLE: 1 values, 15B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 2 values, 8B raw, 28B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, prob] DOUBLE: 1 values, 15B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 63B for [nodeId] INT32: 2 values, 14B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 68B for [predict, predict] DOUBLE: 2 values, 9B raw, 27B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 1 entries, 8B raw, 1B comp} -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 69B for [impurity] DOUBLE: 1 values, 14B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 68B for [predict, prob] DOUBLE: 2 values, 9B raw, 27B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 1 entries, 8B raw, 1B comp} -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 1 values, 7B raw, 27B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 61B for [nodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, feature] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 69B for [impurity] DOUBLE: 2 values, 8B raw, 28B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 1 entries, 8B raw, 1B comp} -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, predict] DOUBLE: 1 values, 15B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 2 values, 7B raw, 27B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, prob] DOUBLE: 1 values, 15B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, feature] INT32: 2 values, 7B raw, 27B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 72B for [impurity] DOUBLE: 1 values, 14B raw, 31B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, threshold] DOUBLE: 2 values, 7B raw, 27B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 1 values, 7B raw, 27B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, featureType] INT32: 2 values, 7B raw, 27B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 64B for [split, feature] INT32: 1 values, 11B raw, 31B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 51B for [split, categories, bag, array] DOUBLE: 2 values, 14B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 71B for [split, threshold] DOUBLE: 1 values, 15B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 47B for [leftNodeId] INT32: 2 values, 6B raw, 26B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 61B for [split, featureType] INT32: 1 values, 11B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 47B for [rightNodeId] INT32: 2 values, 6B raw, 26B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 52B for [split, categories, bag, array] DOUBLE: 1 values, 14B raw, 31B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 47B for [infoGain] DOUBLE: 2 values, 6B raw, 26B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, threshold] DOUBLE: 1 values, 7B raw, 27B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 61B for [leftNodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, featureType] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 61B for [rightNodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 51B for [split, categories, bag, array] DOUBLE: 1 values, 14B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 75B for [infoGain] DOUBLE: 1 values, 14B raw, 34B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 47B for [leftNodeId] INT32: 1 values, 6B raw, 26B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 47B for [rightNodeId] INT32: 1 values, 6B raw, 26B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO ColumnChunkPageWriteStore:151 - written 47B for [infoGain] DOUBLE: 1 values, 6B raw, 26B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 11:16:33 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251116_0011_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111630273/result/data/_temporary/0/task_201605251116_0011_m_000000 -2016-05-25 11:16:33 INFO SparkHadoopMapRedUtil:59 - attempt_201605251116_0011_m_000000_0: Committed -2016-05-25 11:16:33 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 20). 915 bytes result sent to driver -2016-05-25 11:16:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 20) in 631 ms on localhost (1/4) -2016-05-25 11:16:33 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251116_0011_m_000001_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111630273/result/data/_temporary/0/task_201605251116_0011_m_000001 -2016-05-25 11:16:33 INFO SparkHadoopMapRedUtil:59 - attempt_201605251116_0011_m_000001_0: Committed -2016-05-25 11:16:33 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 21). 915 bytes result sent to driver -2016-05-25 11:16:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 21) in 635 ms on localhost (2/4) -2016-05-25 11:16:33 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251116_0011_m_000002_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111630273/result/data/_temporary/0/task_201605251116_0011_m_000002 -2016-05-25 11:16:33 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251116_0011_m_000003_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111630273/result/data/_temporary/0/task_201605251116_0011_m_000003 -2016-05-25 11:16:33 INFO SparkHadoopMapRedUtil:59 - attempt_201605251116_0011_m_000002_0: Committed -2016-05-25 11:16:33 INFO SparkHadoopMapRedUtil:59 - attempt_201605251116_0011_m_000003_0: Committed -2016-05-25 11:16:33 INFO Executor:59 - Finished task 2.0 in stage 11.0 (TID 22). 915 bytes result sent to driver -2016-05-25 11:16:33 INFO Executor:59 - Finished task 3.0 in stage 11.0 (TID 23). 915 bytes result sent to driver -2016-05-25 11:16:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 11.0 (TID 22) in 637 ms on localhost (3/4) -2016-05-25 11:16:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 11.0 (TID 23) in 637 ms on localhost (4/4) -2016-05-25 11:16:33 INFO DAGScheduler:59 - ResultStage 11 (parquet at DecisionTreeModel.scala:242) finished in 0.640 s -2016-05-25 11:16:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 11:16:33 INFO DAGScheduler:59 - Job 9 finished: parquet at DecisionTreeModel.scala:242, took 0.677368 s -2016-05-25 11:16:33 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:16:33 INFO DefaultWriterContainer:59 - Job job_201605251116_0000 committed. -2016-05-25 11:16:33 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111630273/result/data on driver -2016-05-25 11:16:33 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111630273/result/data on driver -2016-05-25 11:16:33 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=441252, maxMem=766075207 -2016-05-25 11:16:33 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 225.3 KB, free 729.9 MB) -2016-05-25 11:16:33 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=671956, maxMem=766075207 -2016-05-25 11:16:33 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-25 11:16:33 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53780 (size: 19.3 KB, free: 730.4 MB) -2016-05-25 11:16:33 INFO SparkContext:59 - Created broadcast 15 from textFile at modelSaveLoad.scala:129 -2016-05-25 11:16:34 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 11:16:34 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 11:16:34 INFO DAGScheduler:59 - Got job 10 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 11:16:34 INFO DAGScheduler:59 - Final stage: ResultStage 12(first at modelSaveLoad.scala:129) -2016-05-25 11:16:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:16:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:16:34 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[29] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 11:16:34 INFO MemoryStore:59 - ensureFreeSpace(3224) called with curMem=691744, maxMem=766075207 -2016-05-25 11:16:34 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 3.1 KB, free 729.9 MB) -2016-05-25 11:16:34 INFO MemoryStore:59 - ensureFreeSpace(1898) called with curMem=694968, maxMem=766075207 -2016-05-25 11:16:34 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1898.0 B, free 729.9 MB) -2016-05-25 11:16:34 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53780 (size: 1898.0 B, free: 730.4 MB) -2016-05-25 11:16:34 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:16:34 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[29] at textFile at modelSaveLoad.scala:129) -2016-05-25 11:16:34 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-25 11:16:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2245 bytes) -2016-05-25 11:16:34 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 24) -2016-05-25 11:16:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111630273/result/metadata/part-00000:0+55 -2016-05-25 11:16:34 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 24). 2157 bytes result sent to driver -2016-05-25 11:16:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 24) in 20 ms on localhost (1/1) -2016-05-25 11:16:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 11:16:34 INFO DAGScheduler:59 - ResultStage 12 (first at modelSaveLoad.scala:129) finished in 0.020 s -2016-05-25 11:16:34 INFO DAGScheduler:59 - Job 10 finished: first at modelSaveLoad.scala:129, took 0.034406 s -2016-05-25 11:16:34 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111630273/result/data on driver -2016-05-25 11:16:34 INFO SparkContext:59 - Starting job: parquet at DecisionTreeModel.scala:249 -2016-05-25 11:16:34 INFO DAGScheduler:59 - Got job 11 (parquet at DecisionTreeModel.scala:249) with 4 output partitions -2016-05-25 11:16:34 INFO DAGScheduler:59 - Final stage: ResultStage 13(parquet at DecisionTreeModel.scala:249) -2016-05-25 11:16:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:16:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:16:34 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[31] at parquet at DecisionTreeModel.scala:249), which has no missing parents -2016-05-25 11:16:34 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=696866, maxMem=766075207 -2016-05-25 11:16:34 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 61.5 KB, free 729.9 MB) -2016-05-25 11:16:34 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=759866, maxMem=766075207 -2016-05-25 11:16:34 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 20.6 KB, free 729.8 MB) -2016-05-25 11:16:34 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53780 (size: 20.6 KB, free: 730.3 MB) -2016-05-25 11:16:34 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:16:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 13 (MapPartitionsRDD[31] at parquet at DecisionTreeModel.scala:249) -2016-05-25 11:16:34 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 4 tasks -2016-05-25 11:16:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 11:16:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 11:16:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 13.0 (TID 27, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 11:16:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 13.0 (TID 28, localhost, PROCESS_LOCAL, 2286 bytes) -2016-05-25 11:16:34 INFO Executor:59 - Running task 3.0 in stage 13.0 (TID 28) -2016-05-25 11:16:34 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-25 11:16:34 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-25 11:16:34 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:16:34 INFO Executor:59 - Running task 2.0 in stage 13.0 (TID 27) -2016-05-25 11:16:34 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:16:34 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 936 bytes result sent to driver -2016-05-25 11:16:34 INFO Executor:59 - Finished task 3.0 in stage 13.0 (TID 28). 2321 bytes result sent to driver -2016-05-25 11:16:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 75 ms on localhost (1/4) -2016-05-25 11:16:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 13.0 (TID 28) in 76 ms on localhost (2/4) -2016-05-25 11:16:34 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:16:34 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 936 bytes result sent to driver -2016-05-25 11:16:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 94 ms on localhost (3/4) -2016-05-25 11:16:34 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 11:16:34 INFO Executor:59 - Finished task 2.0 in stage 13.0 (TID 27). 936 bytes result sent to driver -2016-05-25 11:16:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 13.0 (TID 27) in 98 ms on localhost (4/4) -2016-05-25 11:16:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 11:16:34 INFO DAGScheduler:59 - ResultStage 13 (parquet at DecisionTreeModel.scala:249) finished in 0.100 s -2016-05-25 11:16:34 INFO DAGScheduler:59 - Job 11 finished: parquet at DecisionTreeModel.scala:249, took 0.136415 s -2016-05-25 11:16:34 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=780939, maxMem=766075207 -2016-05-25 11:16:34 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 86.4 KB, free 729.8 MB) -2016-05-25 11:16:34 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=869451, maxMem=766075207 -2016-05-25 11:16:34 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.7 MB) -2016-05-25 11:16:34 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53780 (size: 19.3 KB, free: 730.3 MB) -2016-05-25 11:16:34 INFO SparkContext:59 - Created broadcast 18 from map at DecisionTreeModel.scala:252 -2016-05-25 11:16:34 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 11:16:34 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111630273/result/data/part-r-00001-dd18089f-28c7-492e-a242-a9bb43261c6b.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111630273/result/data/part-r-00002-dd18089f-28c7-492e-a242-a9bb43261c6b.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111630273/result/data/part-r-00003-dd18089f-28c7-492e-a242-a9bb43261c6b.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111630273/result/data/part-r-00000-dd18089f-28c7-492e-a242-a9bb43261c6b.gz.parquet -2016-05-25 11:16:34 INFO SparkContext:59 - Starting job: collect at DecisionTreeModel.scala:267 -2016-05-25 11:16:34 INFO DAGScheduler:59 - Registering RDD 35 (groupBy at DecisionTreeModel.scala:265) -2016-05-25 11:16:34 INFO DAGScheduler:59 - Got job 12 (collect at DecisionTreeModel.scala:267) with 4 output partitions -2016-05-25 11:16:34 INFO DAGScheduler:59 - Final stage: ResultStage 15(collect at DecisionTreeModel.scala:267) -2016-05-25 11:16:34 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 14) -2016-05-25 11:16:34 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 14) -2016-05-25 11:16:34 INFO DAGScheduler:59 - Submitting ShuffleMapStage 14 (MapPartitionsRDD[35] at groupBy at DecisionTreeModel.scala:265), which has no missing parents -2016-05-25 11:16:34 INFO MemoryStore:59 - ensureFreeSpace(7248) called with curMem=889239, maxMem=766075207 -2016-05-25 11:16:34 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 7.1 KB, free 729.7 MB) -2016-05-25 11:16:34 INFO MemoryStore:59 - ensureFreeSpace(3789) called with curMem=896487, maxMem=766075207 -2016-05-25 11:16:34 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.7 KB, free 729.7 MB) -2016-05-25 11:16:34 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53780 (size: 3.7 KB, free: 730.3 MB) -2016-05-25 11:16:34 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:16:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 14 (MapPartitionsRDD[35] at groupBy at DecisionTreeModel.scala:265) -2016-05-25 11:16:34 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 4 tasks -2016-05-25 11:16:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 29, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 11:16:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 30, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 11:16:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 31, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 11:16:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 14.0 (TID 32, localhost, PROCESS_LOCAL, 2292 bytes) -2016-05-25 11:16:34 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 31) -2016-05-25 11:16:34 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 30) -2016-05-25 11:16:34 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 29) -2016-05-25 11:16:34 INFO Executor:59 - Running task 3.0 in stage 14.0 (TID 32) -2016-05-25 11:16:34 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111630273/result/data/part-r-00003-dd18089f-28c7-492e-a242-a9bb43261c6b.gz.parquet start: 0 end: 2987 length: 2987 hosts: []} -2016-05-25 11:16:34 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111630273/result/data/part-r-00000-dd18089f-28c7-492e-a242-a9bb43261c6b.gz.parquet start: 0 end: 3041 length: 3041 hosts: []} -2016-05-25 11:16:34 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111630273/result/data/part-r-00002-dd18089f-28c7-492e-a242-a9bb43261c6b.gz.parquet start: 0 end: 2836 length: 2836 hosts: []} -2016-05-25 11:16:34 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525111630273/result/data/part-r-00001-dd18089f-28c7-492e-a242-a9bb43261c6b.gz.parquet start: 0 end: 3045 length: 3045 hosts: []} -2016-05-25 11:16:35 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 11:16:35 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 11:16:35 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 11:16:35 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53780 in memory (size: 2.1 KB, free: 730.3 MB) -2016-05-25 11:16:35 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 11:16:35 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 treeId; - optional int32 nodeId; - optional group predict { - optional double predict; - optional double prob; - } - optional double impurity; - optional boolean isLeaf; - optional group split { - optional int32 feature; - optional double threshold; - optional int32 featureType; - optional group categories (LIST) { - repeated group bag { - optional double array; - } - } - } - optional int32 leftNodeId; - optional int32 rightNodeId; - optional double infoGain; -} - -Catalyst form: -StructType(StructField(treeId,IntegerType,true), StructField(nodeId,IntegerType,true), StructField(predict,StructType(StructField(predict,DoubleType,true), StructField(prob,DoubleType,true)),true), StructField(impurity,DoubleType,true), StructField(isLeaf,BooleanType,true), StructField(split,StructType(StructField(feature,IntegerType,true), StructField(threshold,DoubleType,true), StructField(featureType,IntegerType,true), StructField(categories,ArrayType(DoubleType,true),true)),true), StructField(leftNodeId,IntegerType,true), StructField(rightNodeId,IntegerType,true), StructField(infoGain,DoubleType,true)) - -2016-05-25 11:16:35 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 treeId; - optional int32 nodeId; - optional group predict { - optional double predict; - optional double prob; - } - optional double impurity; - optional boolean isLeaf; - optional group split { - optional int32 feature; - optional double threshold; - optional int32 featureType; - optional group categories (LIST) { - repeated group bag { - optional double array; - } - } - } - optional int32 leftNodeId; - optional int32 rightNodeId; - optional double infoGain; -} - -Catalyst form: -StructType(StructField(treeId,IntegerType,true), StructField(nodeId,IntegerType,true), StructField(predict,StructType(StructField(predict,DoubleType,true), StructField(prob,DoubleType,true)),true), StructField(impurity,DoubleType,true), StructField(isLeaf,BooleanType,true), StructField(split,StructType(StructField(feature,IntegerType,true), StructField(threshold,DoubleType,true), StructField(featureType,IntegerType,true), StructField(categories,ArrayType(DoubleType,true),true)),true), StructField(leftNodeId,IntegerType,true), StructField(rightNodeId,IntegerType,true), StructField(infoGain,DoubleType,true)) - -2016-05-25 11:16:35 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 treeId; - optional int32 nodeId; - optional group predict { - optional double predict; - optional double prob; - } - optional double impurity; - optional boolean isLeaf; - optional group split { - optional int32 feature; - optional double threshold; - optional int32 featureType; - optional group categories (LIST) { - repeated group bag { - optional double array; - } - } - } - optional int32 leftNodeId; - optional int32 rightNodeId; - optional double infoGain; -} - -Catalyst form: -StructType(StructField(treeId,IntegerType,true), StructField(nodeId,IntegerType,true), StructField(predict,StructType(StructField(predict,DoubleType,true), StructField(prob,DoubleType,true)),true), StructField(impurity,DoubleType,true), StructField(isLeaf,BooleanType,true), StructField(split,StructType(StructField(feature,IntegerType,true), StructField(threshold,DoubleType,true), StructField(featureType,IntegerType,true), StructField(categories,ArrayType(DoubleType,true),true)),true), StructField(leftNodeId,IntegerType,true), StructField(rightNodeId,IntegerType,true), StructField(infoGain,DoubleType,true)) - -2016-05-25 11:16:35 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 treeId; - optional int32 nodeId; - optional group predict { - optional double predict; - optional double prob; - } - optional double impurity; - optional boolean isLeaf; - optional group split { - optional int32 feature; - optional double threshold; - optional int32 featureType; - optional group categories (LIST) { - repeated group bag { - optional double array; - } - } - } - optional int32 leftNodeId; - optional int32 rightNodeId; - optional double infoGain; -} - -Catalyst form: -StructType(StructField(treeId,IntegerType,true), StructField(nodeId,IntegerType,true), StructField(predict,StructType(StructField(predict,DoubleType,true), StructField(prob,DoubleType,true)),true), StructField(impurity,DoubleType,true), StructField(isLeaf,BooleanType,true), StructField(split,StructType(StructField(feature,IntegerType,true), StructField(threshold,DoubleType,true), StructField(featureType,IntegerType,true), StructField(categories,ArrayType(DoubleType,true),true)),true), StructField(leftNodeId,IntegerType,true), StructField(rightNodeId,IntegerType,true), StructField(infoGain,DoubleType,true)) - -2016-05-25 11:16:35 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 2 records. -2016-05-25 11:16:35 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 11:16:35 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 11:16:35 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 11:16:35 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 11:16:35 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 11:16:35 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 11:16:35 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 11:16:35 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 11:16:35 INFO InternalParquetRecordReader:151 - block read in memory in 9 ms. row count = 2 -2016-05-25 11:16:35 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 11:16:35 INFO InternalParquetRecordReader:151 - block read in memory in 8 ms. row count = 1 -2016-05-25 11:16:35 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 11:16:35 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 11:16:35 INFO InternalParquetRecordReader:151 - block read in memory in 2 ms. row count = 1 -2016-05-25 11:16:35 INFO InternalParquetRecordReader:151 - block read in memory in 7 ms. row count = 1 -2016-05-25 11:16:35 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:53780 in memory (size: 20.6 KB, free: 730.3 MB) -2016-05-25 11:16:35 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-25 11:16:35 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:53780 in memory (size: 1898.0 B, free: 730.3 MB) -2016-05-25 11:16:35 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-25 11:16:35 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:53780 in memory (size: 19.3 KB, free: 730.4 MB) -2016-05-25 11:16:35 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:53780 in memory (size: 23.6 KB, free: 730.4 MB) -2016-05-25 11:16:35 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-25 11:16:35 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 11:16:35 INFO Executor:59 - Finished task 3.0 in stage 14.0 (TID 32). 2256 bytes result sent to driver -2016-05-25 11:16:35 INFO TaskSetManager:59 - Finished task 3.0 in stage 14.0 (TID 32) in 880 ms on localhost (1/4) -2016-05-25 11:16:35 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 31). 2256 bytes result sent to driver -2016-05-25 11:16:35 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 29). 2256 bytes result sent to driver -2016-05-25 11:16:35 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 31) in 887 ms on localhost (2/4) -2016-05-25 11:16:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 29) in 892 ms on localhost (3/4) -2016-05-25 11:16:35 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 30). 2256 bytes result sent to driver -2016-05-25 11:16:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 30) in 892 ms on localhost (4/4) -2016-05-25 11:16:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 11:16:35 INFO DAGScheduler:59 - ShuffleMapStage 14 (groupBy at DecisionTreeModel.scala:265) finished in 0.896 s -2016-05-25 11:16:35 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:16:35 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:16:35 INFO DAGScheduler:59 - waiting: Set(ResultStage 15) -2016-05-25 11:16:35 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:16:35 INFO DAGScheduler:59 - Missing parents for ResultStage 15: List() -2016-05-25 11:16:35 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[37] at mapValues at DecisionTreeModel.scala:266), which is now runnable -2016-05-25 11:16:36 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:53780 in memory (size: 3.7 KB, free: 730.4 MB) -2016-05-25 11:17:24 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 11:17:25 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 11:17:25 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 11:17:25 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 11:17:25 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 11:17:26 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 11:17:26 INFO Remoting:74 - Starting remoting -2016-05-25 11:17:26 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53801] -2016-05-25 11:17:26 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53801. -2016-05-25 11:17:26 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 11:17:26 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 11:17:26 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-1977f359-bdbd-45a0-80b0-9d05d7122adb -2016-05-25 11:17:26 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 11:17:26 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-b207a271-9140-4380-9706-c80436405b3f\httpd-020a5d22-1ea9-40c9-bfbe-d1f35a57a301 -2016-05-25 11:17:26 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 11:17:26 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:17:26 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53802 -2016-05-25 11:17:26 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53802. -2016-05-25 11:17:26 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 11:17:26 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:17:26 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 11:17:26 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 11:17:26 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 11:17:27 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 11:17:27 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 11:17:27 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53821. -2016-05-25 11:17:27 INFO NettyBlockTransferService:59 - Server created on 53821 -2016-05-25 11:17:27 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 11:17:27 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53821 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53821) -2016-05-25 11:17:27 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 11:17:28 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 11:17:28 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 11:17:28 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 11:17:28 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 11:17:28 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53821 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 11:17:28 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 11:17:29 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 11:17:29 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 11:17:29 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 11:17:29 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 11:17:29 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 11:17:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:17:29 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:17:29 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 11:17:29 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 11:17:29 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 11:17:30 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 11:17:30 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 11:17:30 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53821 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:17:30 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:17:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 11:17:30 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 11:17:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:17:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:17:30 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 11:17:30 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 11:17:30 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 11:17:30 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 11:17:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 11:17:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 11:17:30 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 11:17:30 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 11:17:30 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 11:17:30 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 11:17:30 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 11:17:30 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177630, maxMem=766075207 -2016-05-25 11:17:30 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 11:17:30 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263278, maxMem=766075207 -2016-05-25 11:17:30 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 11:17:30 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:53821 (size: 83.6 KB, free: 730.5 MB) -2016-05-25 11:17:30 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:53821 (size: 83.1 KB, free: 730.4 MB) -2016-05-25 11:17:30 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 11:17:30 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 11:17:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 386 ms on localhost (1/2) -2016-05-25 11:17:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 409 ms on localhost (2/2) -2016-05-25 11:17:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 11:17:30 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.432 s -2016-05-25 11:17:30 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.640933 s -2016-05-25 11:17:30 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 11:17:30 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 11:17:30 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 11:17:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:17:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:17:30 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 11:17:30 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348358, maxMem=766075207 -2016-05-25 11:17:30 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 11:17:30 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=353126, maxMem=766075207 -2016-05-25 11:17:30 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 11:17:30 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53821 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:17:30 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:17:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 11:17:30 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 11:17:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:17:30 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 11:17:30 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:17:30 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 11:17:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 32 ms on localhost (1/1) -2016-05-25 11:17:30 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.032 s -2016-05-25 11:17:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 11:17:30 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.056096 s -2016-05-25 11:17:30 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 11:17:30 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 11:17:30 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 11:17:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:17:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:17:30 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 11:17:30 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355673, maxMem=766075207 -2016-05-25 11:17:30 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 11:17:30 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=360009, maxMem=766075207 -2016-05-25 11:17:30 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 11:17:30 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53821 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:17:30 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:17:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 11:17:30 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 11:17:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:17:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:17:30 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 11:17:30 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 11:17:30 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:17:30 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 11:17:30 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:17:30 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 11:17:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 15 ms on localhost (1/2) -2016-05-25 11:17:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 17 ms on localhost (2/2) -2016-05-25 11:17:30 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.017 s -2016-05-25 11:17:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 11:17:30 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.032934 s -2016-05-25 11:17:30 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 11:17:30 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 11:17:30 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 11:17:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:17:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:17:30 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 11:17:30 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362328, maxMem=766075207 -2016-05-25 11:17:30 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 11:17:30 INFO MemoryStore:59 - ensureFreeSpace(2631) called with curMem=367392, maxMem=766075207 -2016-05-25 11:17:30 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 11:17:30 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53821 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:17:30 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:17:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 11:17:30 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 11:17:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:17:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:17:30 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 11:17:30 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 11:17:30 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:17:30 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:17:30 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 63802 bytes result sent to driver -2016-05-25 11:17:30 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 62889 bytes result sent to driver -2016-05-25 11:17:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 32 ms on localhost (1/2) -2016-05-25 11:17:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 35 ms on localhost (2/2) -2016-05-25 11:17:30 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.037 s -2016-05-25 11:17:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 11:17:30 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.082803 s -2016-05-25 11:17:32 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53821 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:17:32 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 11:17:32 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53821 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:17:32 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 11:17:32 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53821 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:17:32 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 11:17:32 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=348358, maxMem=766075207 -2016-05-25 11:17:32 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 11:17:32 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=348398, maxMem=766075207 -2016-05-25 11:17:32 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 11:17:32 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53821 (size: 81.0 B, free: 730.4 MB) -2016-05-25 11:17:32 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 11:17:32 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 11:17:32 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 11:17:32 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 11:17:32 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 11:17:32 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 11:17:32 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 11:17:32 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 11:17:32 INFO MemoryStore:59 - ensureFreeSpace(492552) called with curMem=348479, maxMem=766075207 -2016-05-25 11:17:32 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 481.0 KB, free 729.8 MB) -2016-05-25 11:17:32 INFO MemoryStore:59 - ensureFreeSpace(120910) called with curMem=841031, maxMem=766075207 -2016-05-25 11:17:32 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 118.1 KB, free 729.7 MB) -2016-05-25 11:17:32 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53821 (size: 118.1 KB, free: 730.3 MB) -2016-05-25 11:17:32 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:17:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 11:17:32 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 11:17:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:17:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:17:32 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 11:17:32 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 11:17:32 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 11:17:32 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:17:32 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 11:17:32 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:17:32 INFO MemoryStore:59 - ensureFreeSpace(105840) called with curMem=961941, maxMem=766075207 -2016-05-25 11:17:32 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 103.4 KB, free 729.6 MB) -2016-05-25 11:17:32 INFO MemoryStore:59 - ensureFreeSpace(94400) called with curMem=1067781, maxMem=766075207 -2016-05-25 11:17:32 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 92.2 KB, free 729.5 MB) -2016-05-25 11:17:32 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:53821 (size: 103.4 KB, free: 730.2 MB) -2016-05-25 11:17:32 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:53821 (size: 92.2 KB, free: 730.1 MB) -2016-05-25 11:17:32 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 11:17:32 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 11:17:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 221 ms on localhost (1/2) -2016-05-25 11:17:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 223 ms on localhost (2/2) -2016-05-25 11:17:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 11:17:32 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.225 s -2016-05-25 11:17:32 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:17:32 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:17:32 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 11:17:32 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:17:32 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 11:17:32 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 11:17:32 INFO MemoryStore:59 - ensureFreeSpace(214328) called with curMem=1162181, maxMem=766075207 -2016-05-25 11:17:32 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 209.3 KB, free 729.3 MB) -2016-05-25 11:17:32 INFO MemoryStore:59 - ensureFreeSpace(45502) called with curMem=1376509, maxMem=766075207 -2016-05-25 11:17:32 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 44.4 KB, free 729.2 MB) -2016-05-25 11:17:32 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53821 (size: 44.4 KB, free: 730.1 MB) -2016-05-25 11:17:32 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:17:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 11:17:32 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 11:17:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:17:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:17:32 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 11:17:32 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 11:17:32 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:17:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:17:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-25 11:17:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-25 11:17:32 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 11:17:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 133 ms on localhost (1/2) -2016-05-25 11:17:33 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 11:17:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 402 ms on localhost (2/2) -2016-05-25 11:17:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 11:17:33 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.402 s -2016-05-25 11:17:33 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.686077 s -2016-05-25 11:17:33 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=1422011, maxMem=766075207 -2016-05-25 11:17:33 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 40.0 B, free 729.2 MB) -2016-05-25 11:17:33 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=1422051, maxMem=766075207 -2016-05-25 11:17:33 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 81.0 B, free 729.2 MB) -2016-05-25 11:17:33 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53821 (size: 81.0 B, free: 730.1 MB) -2016-05-25 11:17:33 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 11:17:33 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 11:17:33 INFO DAGScheduler:59 - Registering RDD 17 (mapPartitions at DecisionTree.scala:625) -2016-05-25 11:17:33 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 11:17:33 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 11:17:33 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 11:17:33 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 11:17:33 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 11:17:33 INFO MemoryStore:59 - ensureFreeSpace(492904) called with curMem=1422132, maxMem=766075207 -2016-05-25 11:17:33 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 481.4 KB, free 728.8 MB) -2016-05-25 11:17:33 INFO MemoryStore:59 - ensureFreeSpace(121125) called with curMem=1915036, maxMem=766075207 -2016-05-25 11:17:33 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 118.3 KB, free 728.6 MB) -2016-05-25 11:17:33 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53821 (size: 118.3 KB, free: 729.9 MB) -2016-05-25 11:17:33 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:17:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625) -2016-05-25 11:17:33 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 11:17:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:17:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:17:33 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 11:17:33 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 11:17:33 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-25 11:17:33 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-25 11:17:33 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 11:17:33 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 11:17:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 60 ms on localhost (1/2) -2016-05-25 11:17:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 64 ms on localhost (2/2) -2016-05-25 11:17:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 11:17:33 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.064 s -2016-05-25 11:17:33 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:17:33 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:17:33 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 11:17:33 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:17:33 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 11:17:33 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 11:17:33 INFO MemoryStore:59 - ensureFreeSpace(214328) called with curMem=2036161, maxMem=766075207 -2016-05-25 11:17:33 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 209.3 KB, free 728.4 MB) -2016-05-25 11:17:33 INFO MemoryStore:59 - ensureFreeSpace(45498) called with curMem=2250489, maxMem=766075207 -2016-05-25 11:17:33 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 44.4 KB, free 728.4 MB) -2016-05-25 11:17:33 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53821 (size: 44.4 KB, free: 729.9 MB) -2016-05-25 11:17:33 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:17:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645) -2016-05-25 11:17:33 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 11:17:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:17:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:17:33 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 11:17:33 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 11:17:33 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:17:33 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:17:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 11:17:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 11:17:33 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 1161 bytes result sent to driver -2016-05-25 11:17:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 22 ms on localhost (1/2) -2016-05-25 11:17:33 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 11:17:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 89 ms on localhost (2/2) -2016-05-25 11:17:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 11:17:33 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.090 s -2016-05-25 11:17:33 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.207840 s -2016-05-25 11:17:33 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 11:17:33 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 11:17:33 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 11:17:33 INFO RandomForest:59 - init: 1.604055717 - total: 2.800691898 - findSplitsBins: 1.36141544 - findBestSplits: 1.177072425 - chooseSplits: 1.169779522 -2016-05-25 11:17:33 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:47 -2016-05-25 11:17:33 INFO DAGScheduler:59 - Got job 6 (count at DecisionTreesLearning.scala:47) with 2 output partitions -2016-05-25 11:17:33 INFO DAGScheduler:59 - Final stage: ResultStage 8(count at DecisionTreesLearning.scala:47) -2016-05-25 11:17:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:17:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:17:33 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at filter at DecisionTreesLearning.scala:47), which has no missing parents -2016-05-25 11:17:33 INFO MemoryStore:59 - ensureFreeSpace(6744) called with curMem=2095747, maxMem=766075207 -2016-05-25 11:17:33 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 6.6 KB, free 728.6 MB) -2016-05-25 11:17:33 INFO MemoryStore:59 - ensureFreeSpace(3587) called with curMem=2102491, maxMem=766075207 -2016-05-25 11:17:33 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.5 KB, free 728.6 MB) -2016-05-25 11:17:33 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53821 (size: 3.5 KB, free: 730.1 MB) -2016-05-25 11:17:33 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:17:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at filter at DecisionTreesLearning.scala:47) -2016-05-25 11:17:33 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 11:17:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:17:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:17:33 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 11:17:33 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 11:17:33 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:17:33 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:17:33 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 11:17:33 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 11:17:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 12 ms on localhost (1/2) -2016-05-25 11:17:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 14 ms on localhost (2/2) -2016-05-25 11:17:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 11:17:33 INFO DAGScheduler:59 - ResultStage 8 (count at DecisionTreesLearning.scala:47) finished in 0.014 s -2016-05-25 11:17:33 INFO DAGScheduler:59 - Job 6 finished: count at DecisionTreesLearning.scala:47, took 0.035253 s -2016-05-25 11:17:33 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:47 -2016-05-25 11:17:33 INFO DAGScheduler:59 - Got job 7 (count at DecisionTreesLearning.scala:47) with 2 output partitions -2016-05-25 11:17:33 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at DecisionTreesLearning.scala:47) -2016-05-25 11:17:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:17:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:17:33 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:17:33 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2106078, maxMem=766075207 -2016-05-25 11:17:33 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.0 KB, free 728.6 MB) -2016-05-25 11:17:33 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=2110190, maxMem=766075207 -2016-05-25 11:17:33 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.6 MB) -2016-05-25 11:17:33 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53821 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 11:17:33 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:17:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:17:33 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 11:17:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:17:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:17:33 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 11:17:33 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-25 11:17:33 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:17:33 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:17:33 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2082 bytes result sent to driver -2016-05-25 11:17:33 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2082 bytes result sent to driver -2016-05-25 11:17:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 10 ms on localhost (1/2) -2016-05-25 11:17:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 12 ms on localhost (2/2) -2016-05-25 11:17:33 INFO DAGScheduler:59 - ResultStage 9 (count at DecisionTreesLearning.scala:47) finished in 0.012 s -2016-05-25 11:17:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 11:17:33 INFO DAGScheduler:59 - Job 7 finished: count at DecisionTreesLearning.scala:47, took 0.041344 s -2016-05-25 11:17:33 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:51 -2016-05-25 11:17:33 INFO DAGScheduler:59 - Got job 8 (count at DecisionTreesLearning.scala:51) with 2 output partitions -2016-05-25 11:17:33 INFO DAGScheduler:59 - Final stage: ResultStage 10(count at DecisionTreesLearning.scala:51) -2016-05-25 11:17:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:17:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:17:33 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[7] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:17:33 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2112458, maxMem=766075207 -2016-05-25 11:17:33 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 4.0 KB, free 728.6 MB) -2016-05-25 11:17:33 INFO MemoryStore:59 - ensureFreeSpace(2265) called with curMem=2116570, maxMem=766075207 -2016-05-25 11:17:33 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.6 MB) -2016-05-25 11:17:33 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53821 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 11:17:33 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:17:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[7] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:17:33 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-25 11:17:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:17:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:17:33 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 11:17:33 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-25 11:17:33 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:17:33 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:17:33 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2082 bytes result sent to driver -2016-05-25 11:17:33 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2082 bytes result sent to driver -2016-05-25 11:17:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 11 ms on localhost (1/2) -2016-05-25 11:17:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 10 ms on localhost (2/2) -2016-05-25 11:17:33 INFO DAGScheduler:59 - ResultStage 10 (count at DecisionTreesLearning.scala:51) finished in 0.012 s -2016-05-25 11:17:33 INFO DAGScheduler:59 - Job 8 finished: count at DecisionTreesLearning.scala:51, took 0.028436 s -2016-05-25 11:17:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 11:17:33 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:52 -2016-05-25 11:17:33 INFO DAGScheduler:59 - Got job 9 (count at DecisionTreesLearning.scala:52) with 2 output partitions -2016-05-25 11:17:33 INFO DAGScheduler:59 - Final stage: ResultStage 11(count at DecisionTreesLearning.scala:52) -2016-05-25 11:17:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:17:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:17:33 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:17:33 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2118835, maxMem=766075207 -2016-05-25 11:17:33 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 4.0 KB, free 728.6 MB) -2016-05-25 11:17:33 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=2122947, maxMem=766075207 -2016-05-25 11:17:33 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.6 MB) -2016-05-25 11:17:33 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53821 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 11:17:33 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:17:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:17:33 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-25 11:17:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:17:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:17:33 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-25 11:17:33 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-25 11:17:33 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:17:33 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:17:33 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2082 bytes result sent to driver -2016-05-25 11:17:33 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2082 bytes result sent to driver -2016-05-25 11:17:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 8 ms on localhost (1/2) -2016-05-25 11:17:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 10 ms on localhost (2/2) -2016-05-25 11:17:33 INFO DAGScheduler:59 - ResultStage 11 (count at DecisionTreesLearning.scala:52) finished in 0.012 s -2016-05-25 11:17:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 11:17:33 INFO DAGScheduler:59 - Job 9 finished: count at DecisionTreesLearning.scala:52, took 0.026140 s -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 11:17:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 11:17:33 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 11:17:33 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 11:17:33 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 11:17:33 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 11:17:33 INFO BlockManager:59 - BlockManager stopped -2016-05-25 11:17:33 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 11:17:33 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 11:17:33 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 11:17:33 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 11:17:33 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-b207a271-9140-4380-9706-c80436405b3f -2016-05-25 11:18:06 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 11:18:07 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 11:18:07 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 11:18:07 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 11:18:07 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 11:18:08 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 11:18:08 INFO Remoting:74 - Starting remoting -2016-05-25 11:18:08 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53842] -2016-05-25 11:18:08 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53842. -2016-05-25 11:18:08 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 11:18:08 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 11:18:08 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-27e360c6-4fe9-4210-93cf-ac3d2ef9ba70 -2016-05-25 11:18:08 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 11:18:08 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-b082cae9-de5a-4def-b123-c7979b5b8fcb\httpd-e2ecd2ef-8496-4a68-b6a9-23fa9145364e -2016-05-25 11:18:08 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 11:18:08 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:18:08 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53843 -2016-05-25 11:18:08 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53843. -2016-05-25 11:18:08 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 11:18:08 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:18:08 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 11:18:08 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 11:18:08 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 11:18:08 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 11:18:08 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 11:18:09 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53862. -2016-05-25 11:18:09 INFO NettyBlockTransferService:59 - Server created on 53862 -2016-05-25 11:18:09 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 11:18:09 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53862 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53862) -2016-05-25 11:18:09 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 11:18:09 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 11:18:09 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 11:18:09 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 11:18:09 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 11:18:09 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53862 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 11:18:09 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 11:18:11 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 11:18:11 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 11:18:11 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 11:18:11 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 11:18:11 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 11:18:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:18:11 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:18:11 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 11:18:11 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 11:18:11 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 11:18:11 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 11:18:11 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 11:18:11 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53862 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:18:11 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:18:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 11:18:11 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 11:18:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:18:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:18:11 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 11:18:11 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 11:18:11 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 11:18:11 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 11:18:11 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 11:18:11 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 11:18:12 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 11:18:12 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 11:18:12 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 11:18:12 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 11:18:12 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 11:18:12 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177630, maxMem=766075207 -2016-05-25 11:18:12 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 11:18:12 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262710, maxMem=766075207 -2016-05-25 11:18:12 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 11:18:12 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:53862 (size: 83.1 KB, free: 730.5 MB) -2016-05-25 11:18:12 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:53862 (size: 83.6 KB, free: 730.4 MB) -2016-05-25 11:18:12 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 11:18:12 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 11:18:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 401 ms on localhost (1/2) -2016-05-25 11:18:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 385 ms on localhost (2/2) -2016-05-25 11:18:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 11:18:12 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.423 s -2016-05-25 11:18:12 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.568236 s -2016-05-25 11:18:12 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 11:18:12 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 11:18:12 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 11:18:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:18:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:18:12 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 11:18:12 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348358, maxMem=766075207 -2016-05-25 11:18:12 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 11:18:12 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=353126, maxMem=766075207 -2016-05-25 11:18:12 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 11:18:12 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53862 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:18:12 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:18:12 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 11:18:12 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 11:18:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:18:12 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 11:18:12 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:18:12 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 11:18:12 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.027 s -2016-05-25 11:18:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 26 ms on localhost (1/1) -2016-05-25 11:18:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 11:18:12 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.045597 s -2016-05-25 11:18:12 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 11:18:12 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 11:18:12 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 11:18:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:18:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:18:12 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 11:18:12 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355673, maxMem=766075207 -2016-05-25 11:18:12 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 11:18:12 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=360009, maxMem=766075207 -2016-05-25 11:18:12 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 11:18:12 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53862 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:18:12 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:18:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 11:18:12 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 11:18:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:18:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:18:12 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 11:18:12 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:18:12 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 11:18:12 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 11:18:12 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:18:12 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 11:18:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 24 ms on localhost (1/2) -2016-05-25 11:18:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 29 ms on localhost (2/2) -2016-05-25 11:18:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 11:18:12 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.011 s -2016-05-25 11:18:12 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.064751 s -2016-05-25 11:18:12 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 11:18:12 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 11:18:12 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 11:18:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:18:12 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:18:12 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 11:18:12 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362328, maxMem=766075207 -2016-05-25 11:18:12 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 11:18:12 INFO MemoryStore:59 - ensureFreeSpace(2632) called with curMem=367392, maxMem=766075207 -2016-05-25 11:18:12 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 11:18:12 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53862 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:18:12 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:18:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 11:18:12 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 11:18:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:18:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:18:12 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 11:18:12 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 11:18:12 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:18:12 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:18:12 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 66603 bytes result sent to driver -2016-05-25 11:18:12 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 52389 bytes result sent to driver -2016-05-25 11:18:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 45 ms on localhost (1/2) -2016-05-25 11:18:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 43 ms on localhost (2/2) -2016-05-25 11:18:12 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.041 s -2016-05-25 11:18:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 11:18:12 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.068240 s -2016-05-25 11:18:13 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53862 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:18:13 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 11:18:13 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53862 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:18:13 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 11:18:13 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53862 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:18:13 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 11:18:13 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=348358, maxMem=766075207 -2016-05-25 11:18:13 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 11:18:13 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=348398, maxMem=766075207 -2016-05-25 11:18:13 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 11:18:13 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53862 (size: 81.0 B, free: 730.4 MB) -2016-05-25 11:18:13 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 11:18:14 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 11:18:14 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 11:18:14 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 11:18:14 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 11:18:14 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 11:18:14 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 11:18:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 11:18:14 INFO MemoryStore:59 - ensureFreeSpace(469976) called with curMem=348479, maxMem=766075207 -2016-05-25 11:18:14 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 459.0 KB, free 729.8 MB) -2016-05-25 11:18:14 INFO MemoryStore:59 - ensureFreeSpace(115283) called with curMem=818455, maxMem=766075207 -2016-05-25 11:18:14 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 112.6 KB, free 729.7 MB) -2016-05-25 11:18:14 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53862 (size: 112.6 KB, free: 730.3 MB) -2016-05-25 11:18:14 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:18:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 11:18:14 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 11:18:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:18:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:18:14 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 11:18:14 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 11:18:14 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 11:18:14 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 11:18:14 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:18:14 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:18:14 INFO MemoryStore:59 - ensureFreeSpace(91536) called with curMem=933738, maxMem=766075207 -2016-05-25 11:18:14 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 89.4 KB, free 729.6 MB) -2016-05-25 11:18:14 INFO MemoryStore:59 - ensureFreeSpace(111560) called with curMem=1025274, maxMem=766075207 -2016-05-25 11:18:14 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 108.9 KB, free 729.5 MB) -2016-05-25 11:18:14 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:53862 (size: 89.4 KB, free: 730.2 MB) -2016-05-25 11:18:14 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:53862 (size: 108.9 KB, free: 730.1 MB) -2016-05-25 11:18:14 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 11:18:14 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 11:18:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 235 ms on localhost (1/2) -2016-05-25 11:18:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 232 ms on localhost (2/2) -2016-05-25 11:18:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 11:18:14 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.237 s -2016-05-25 11:18:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:18:14 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:18:14 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 11:18:14 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:18:14 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 11:18:14 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 11:18:14 INFO MemoryStore:59 - ensureFreeSpace(203240) called with curMem=1136834, maxMem=766075207 -2016-05-25 11:18:14 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 198.5 KB, free 729.3 MB) -2016-05-25 11:18:14 INFO MemoryStore:59 - ensureFreeSpace(43354) called with curMem=1340074, maxMem=766075207 -2016-05-25 11:18:14 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 42.3 KB, free 729.3 MB) -2016-05-25 11:18:14 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53862 (size: 42.3 KB, free: 730.1 MB) -2016-05-25 11:18:14 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:18:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 11:18:14 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 11:18:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:18:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:18:14 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 11:18:14 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 11:18:14 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:18:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:18:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 11:18:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 11:18:14 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 11:18:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 165 ms on localhost (1/2) -2016-05-25 11:18:14 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 11:18:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 436 ms on localhost (2/2) -2016-05-25 11:18:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 11:18:14 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.437 s -2016-05-25 11:18:14 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.743798 s -2016-05-25 11:18:14 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 11:18:14 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 11:18:14 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 11:18:14 INFO RandomForest:59 - init: 1.522116762 - total: 2.488226005 - findSplitsBins: 1.263079056 - findBestSplits: 0.947698055 - chooseSplits: 0.940068262 -2016-05-25 11:18:14 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:47 -2016-05-25 11:18:14 INFO DAGScheduler:59 - Got job 5 (count at DecisionTreesLearning.scala:47) with 2 output partitions -2016-05-25 11:18:14 INFO DAGScheduler:59 - Final stage: ResultStage 6(count at DecisionTreesLearning.scala:47) -2016-05-25 11:18:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:18:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:18:14 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[18] at filter at DecisionTreesLearning.scala:47), which has no missing parents -2016-05-25 11:18:14 INFO MemoryStore:59 - ensureFreeSpace(6520) called with curMem=1180332, maxMem=766075207 -2016-05-25 11:18:14 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 6.4 KB, free 729.5 MB) -2016-05-25 11:18:14 INFO MemoryStore:59 - ensureFreeSpace(3489) called with curMem=1186852, maxMem=766075207 -2016-05-25 11:18:14 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.5 MB) -2016-05-25 11:18:14 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53862 (size: 3.4 KB, free: 730.3 MB) -2016-05-25 11:18:14 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:18:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[18] at filter at DecisionTreesLearning.scala:47) -2016-05-25 11:18:14 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 11:18:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:18:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:18:14 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 11:18:14 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 11:18:14 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:18:14 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:18:14 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2082 bytes result sent to driver -2016-05-25 11:18:14 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2082 bytes result sent to driver -2016-05-25 11:18:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 15 ms on localhost (1/2) -2016-05-25 11:18:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 16 ms on localhost (2/2) -2016-05-25 11:18:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 11:18:14 INFO DAGScheduler:59 - ResultStage 6 (count at DecisionTreesLearning.scala:47) finished in 0.018 s -2016-05-25 11:18:14 INFO DAGScheduler:59 - Job 5 finished: count at DecisionTreesLearning.scala:47, took 0.038043 s -2016-05-25 11:18:14 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:47 -2016-05-25 11:18:14 INFO DAGScheduler:59 - Got job 6 (count at DecisionTreesLearning.scala:47) with 2 output partitions -2016-05-25 11:18:14 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at DecisionTreesLearning.scala:47) -2016-05-25 11:18:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:18:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:18:14 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:18:14 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1190341, maxMem=766075207 -2016-05-25 11:18:14 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-05-25 11:18:14 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=1194453, maxMem=766075207 -2016-05-25 11:18:14 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.4 MB) -2016-05-25 11:18:14 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53862 (size: 2.2 KB, free: 730.3 MB) -2016-05-25 11:18:14 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:18:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:18:14 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 11:18:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:18:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:18:15 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 11:18:15 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 11:18:15 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:18:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:18:15 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2082 bytes result sent to driver -2016-05-25 11:18:15 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2082 bytes result sent to driver -2016-05-25 11:18:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 11 ms on localhost (1/2) -2016-05-25 11:18:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 13 ms on localhost (2/2) -2016-05-25 11:18:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 11:18:15 INFO DAGScheduler:59 - ResultStage 7 (count at DecisionTreesLearning.scala:47) finished in 0.013 s -2016-05-25 11:18:15 INFO DAGScheduler:59 - Job 6 finished: count at DecisionTreesLearning.scala:47, took 0.032622 s -2016-05-25 11:18:15 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:50 -2016-05-25 11:18:15 INFO DAGScheduler:59 - Got job 7 (count at DecisionTreesLearning.scala:50) with 2 output partitions -2016-05-25 11:18:15 INFO DAGScheduler:59 - Final stage: ResultStage 8(count at DecisionTreesLearning.scala:50) -2016-05-25 11:18:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:18:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:18:15 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 11:18:15 INFO MemoryStore:59 - ensureFreeSpace(3816) called with curMem=1196721, maxMem=766075207 -2016-05-25 11:18:15 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 3.7 KB, free 729.4 MB) -2016-05-25 11:18:15 INFO MemoryStore:59 - ensureFreeSpace(2142) called with curMem=1200537, maxMem=766075207 -2016-05-25 11:18:15 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.4 MB) -2016-05-25 11:18:15 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53862 (size: 2.1 KB, free: 730.2 MB) -2016-05-25 11:18:15 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:18:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 11:18:15 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 11:18:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:18:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:18:15 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 11:18:15 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 11:18:15 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:18:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:18:15 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 11:18:15 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 11:18:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 10 ms on localhost (1/2) -2016-05-25 11:18:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 10 ms on localhost (2/2) -2016-05-25 11:18:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 11:18:15 INFO DAGScheduler:59 - ResultStage 8 (count at DecisionTreesLearning.scala:50) finished in 0.011 s -2016-05-25 11:18:15 INFO DAGScheduler:59 - Job 7 finished: count at DecisionTreesLearning.scala:50, took 0.027543 s -2016-05-25 11:18:15 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:51 -2016-05-25 11:18:15 INFO DAGScheduler:59 - Got job 8 (count at DecisionTreesLearning.scala:51) with 2 output partitions -2016-05-25 11:18:15 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at DecisionTreesLearning.scala:51) -2016-05-25 11:18:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:18:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:18:15 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[7] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:18:15 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1202679, maxMem=766075207 -2016-05-25 11:18:15 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-05-25 11:18:15 INFO MemoryStore:59 - ensureFreeSpace(2265) called with curMem=1206791, maxMem=766075207 -2016-05-25 11:18:15 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.4 MB) -2016-05-25 11:18:15 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53862 (size: 2.2 KB, free: 730.2 MB) -2016-05-25 11:18:15 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:18:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[7] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:18:15 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 11:18:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:18:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:18:15 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-25 11:18:15 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 11:18:15 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:18:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:18:15 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2082 bytes result sent to driver -2016-05-25 11:18:15 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2082 bytes result sent to driver -2016-05-25 11:18:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 13 ms on localhost (1/2) -2016-05-25 11:18:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 15 ms on localhost (2/2) -2016-05-25 11:18:15 INFO DAGScheduler:59 - ResultStage 9 (count at DecisionTreesLearning.scala:51) finished in 0.016 s -2016-05-25 11:18:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 11:18:15 INFO DAGScheduler:59 - Job 8 finished: count at DecisionTreesLearning.scala:51, took 0.034886 s -2016-05-25 11:18:15 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:52 -2016-05-25 11:18:15 INFO DAGScheduler:59 - Got job 9 (count at DecisionTreesLearning.scala:52) with 2 output partitions -2016-05-25 11:18:15 INFO DAGScheduler:59 - Final stage: ResultStage 10(count at DecisionTreesLearning.scala:52) -2016-05-25 11:18:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:18:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:18:15 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:18:15 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1209056, maxMem=766075207 -2016-05-25 11:18:15 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-05-25 11:18:15 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=1213168, maxMem=766075207 -2016-05-25 11:18:15 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.4 MB) -2016-05-25 11:18:15 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53862 (size: 2.2 KB, free: 730.2 MB) -2016-05-25 11:18:15 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:18:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:18:15 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-25 11:18:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:18:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:18:15 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 11:18:15 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-25 11:18:15 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:18:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:18:15 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2082 bytes result sent to driver -2016-05-25 11:18:15 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2082 bytes result sent to driver -2016-05-25 11:18:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 10 ms on localhost (1/2) -2016-05-25 11:18:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 13 ms on localhost (2/2) -2016-05-25 11:18:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 11:18:15 INFO DAGScheduler:59 - ResultStage 10 (count at DecisionTreesLearning.scala:52) finished in 0.013 s -2016-05-25 11:18:15 INFO DAGScheduler:59 - Job 9 finished: count at DecisionTreesLearning.scala:52, took 0.031415 s -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 11:18:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 11:18:15 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 11:18:15 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 11:18:15 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 11:18:15 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 11:18:15 INFO BlockManager:59 - BlockManager stopped -2016-05-25 11:18:15 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 11:18:15 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 11:18:15 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 11:18:15 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 11:18:15 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-b082cae9-de5a-4def-b123-c7979b5b8fcb -2016-05-25 11:18:51 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 11:18:52 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 11:18:52 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 11:18:52 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 11:18:52 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 11:18:53 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 11:18:53 INFO Remoting:74 - Starting remoting -2016-05-25 11:18:53 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53884] -2016-05-25 11:18:53 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53884. -2016-05-25 11:18:53 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 11:18:53 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 11:18:53 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-daef8c17-49ec-4f49-b0b8-5219ff6571c6 -2016-05-25 11:18:53 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 11:18:53 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-2103fc9e-555b-4fb8-96e1-b0a732096fd9\httpd-a5fda95c-309b-4f91-a29a-607daa256013 -2016-05-25 11:18:53 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 11:18:53 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:18:53 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53885 -2016-05-25 11:18:53 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53885. -2016-05-25 11:18:53 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 11:18:53 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:18:53 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 11:18:53 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 11:18:53 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 11:18:53 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 11:18:53 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 11:18:54 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53904. -2016-05-25 11:18:54 INFO NettyBlockTransferService:59 - Server created on 53904 -2016-05-25 11:18:54 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 11:18:54 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53904 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53904) -2016-05-25 11:18:54 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 11:18:54 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 11:18:54 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 11:18:54 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 11:18:54 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 11:18:55 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53904 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 11:18:55 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 11:18:56 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 11:18:56 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 11:18:56 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 11:18:56 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 11:18:56 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 11:18:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:18:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:18:56 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 11:18:57 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 11:18:57 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 11:18:57 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 11:18:57 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 11:18:57 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53904 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:18:57 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:18:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 11:18:57 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 11:18:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:18:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:18:57 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 11:18:57 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 11:18:57 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 11:18:57 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 11:18:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 11:18:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 11:18:57 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 11:18:57 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 11:18:57 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 11:18:57 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 11:18:57 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 11:18:57 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177630, maxMem=766075207 -2016-05-25 11:18:57 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 11:18:57 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262710, maxMem=766075207 -2016-05-25 11:18:57 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 11:18:57 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:53904 (size: 83.1 KB, free: 730.5 MB) -2016-05-25 11:18:57 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:53904 (size: 83.6 KB, free: 730.4 MB) -2016-05-25 11:18:57 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 11:18:57 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 11:18:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 341 ms on localhost (1/2) -2016-05-25 11:18:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 364 ms on localhost (2/2) -2016-05-25 11:18:57 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.382 s -2016-05-25 11:18:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 11:18:57 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.532949 s -2016-05-25 11:18:57 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 11:18:57 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 11:18:57 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 11:18:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:18:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:18:57 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 11:18:57 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348358, maxMem=766075207 -2016-05-25 11:18:57 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 11:18:57 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=353126, maxMem=766075207 -2016-05-25 11:18:57 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 11:18:57 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53904 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:18:57 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:18:57 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 11:18:57 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 11:18:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:18:57 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 11:18:57 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:18:57 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 11:18:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 31 ms on localhost (1/1) -2016-05-25 11:18:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 11:18:57 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.031 s -2016-05-25 11:18:57 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.057495 s -2016-05-25 11:18:57 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 11:18:57 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 11:18:57 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 11:18:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:18:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:18:57 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 11:18:57 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355673, maxMem=766075207 -2016-05-25 11:18:57 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 11:18:57 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=360009, maxMem=766075207 -2016-05-25 11:18:57 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 11:18:57 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53904 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:18:57 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:18:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 11:18:57 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 11:18:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:18:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:18:57 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 11:18:57 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 11:18:57 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:18:57 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 11:18:57 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:18:57 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 11:18:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 15 ms on localhost (1/2) -2016-05-25 11:18:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 16 ms on localhost (2/2) -2016-05-25 11:18:57 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.017 s -2016-05-25 11:18:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 11:18:57 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.035964 s -2016-05-25 11:18:57 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 11:18:57 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 11:18:57 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 11:18:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:18:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:18:57 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 11:18:57 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362328, maxMem=766075207 -2016-05-25 11:18:57 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 11:18:57 INFO MemoryStore:59 - ensureFreeSpace(2630) called with curMem=367392, maxMem=766075207 -2016-05-25 11:18:57 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 11:18:57 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53904 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:18:57 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:18:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 11:18:57 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 11:18:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:18:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:18:57 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 11:18:57 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 11:18:57 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:18:57 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:18:57 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 50585 bytes result sent to driver -2016-05-25 11:18:57 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 67832 bytes result sent to driver -2016-05-25 11:18:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 29 ms on localhost (1/2) -2016-05-25 11:18:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 35 ms on localhost (2/2) -2016-05-25 11:18:57 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.036 s -2016-05-25 11:18:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 11:18:57 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.057330 s -2016-05-25 11:18:58 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53904 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:18:58 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 11:18:58 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53904 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:18:58 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 11:18:58 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53904 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:18:58 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 11:18:58 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53904 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-25 11:18:58 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 11:18:58 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=342308, maxMem=766075207 -2016-05-25 11:18:58 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 11:18:59 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=342348, maxMem=766075207 -2016-05-25 11:18:59 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 11:18:59 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53904 (size: 81.0 B, free: 730.4 MB) -2016-05-25 11:18:59 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 11:18:59 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 11:18:59 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 11:18:59 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 11:18:59 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 11:18:59 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 11:18:59 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 11:18:59 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 11:18:59 INFO MemoryStore:59 - ensureFreeSpace(467760) called with curMem=342429, maxMem=766075207 -2016-05-25 11:18:59 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 456.8 KB, free 729.8 MB) -2016-05-25 11:18:59 INFO MemoryStore:59 - ensureFreeSpace(114634) called with curMem=810189, maxMem=766075207 -2016-05-25 11:18:59 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 111.9 KB, free 729.7 MB) -2016-05-25 11:18:59 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53904 (size: 111.9 KB, free: 730.3 MB) -2016-05-25 11:18:59 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:18:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 11:18:59 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 11:18:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:18:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:18:59 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 11:18:59 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 11:18:59 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 11:18:59 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:18:59 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 11:18:59 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:18:59 INFO MemoryStore:59 - ensureFreeSpace(88680) called with curMem=924823, maxMem=766075207 -2016-05-25 11:18:59 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 86.6 KB, free 729.6 MB) -2016-05-25 11:18:59 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:53904 (size: 86.6 KB, free: 730.2 MB) -2016-05-25 11:18:59 INFO MemoryStore:59 - ensureFreeSpace(102976) called with curMem=1013503, maxMem=766075207 -2016-05-25 11:18:59 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 100.6 KB, free 729.5 MB) -2016-05-25 11:18:59 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:53904 (size: 100.6 KB, free: 730.1 MB) -2016-05-25 11:18:59 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 11:18:59 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 11:18:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 232 ms on localhost (1/2) -2016-05-25 11:18:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 231 ms on localhost (2/2) -2016-05-25 11:18:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 11:18:59 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.234 s -2016-05-25 11:18:59 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:18:59 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:18:59 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 11:18:59 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:18:59 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 11:18:59 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 11:18:59 INFO MemoryStore:59 - ensureFreeSpace(202144) called with curMem=1116479, maxMem=766075207 -2016-05-25 11:18:59 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 197.4 KB, free 729.3 MB) -2016-05-25 11:18:59 INFO MemoryStore:59 - ensureFreeSpace(43177) called with curMem=1318623, maxMem=766075207 -2016-05-25 11:18:59 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 42.2 KB, free 729.3 MB) -2016-05-25 11:18:59 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53904 (size: 42.2 KB, free: 730.1 MB) -2016-05-25 11:18:59 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:18:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 11:18:59 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 11:18:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:18:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:18:59 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 11:18:59 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 11:18:59 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:18:59 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:18:59 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 11:18:59 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 11:18:59 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 11:18:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 160 ms on localhost (1/2) -2016-05-25 11:18:59 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 11:18:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 434 ms on localhost (2/2) -2016-05-25 11:18:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 11:18:59 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.434 s -2016-05-25 11:18:59 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.751245 s -2016-05-25 11:18:59 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=1361800, maxMem=766075207 -2016-05-25 11:18:59 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 40.0 B, free 729.3 MB) -2016-05-25 11:18:59 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=1361840, maxMem=766075207 -2016-05-25 11:18:59 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 81.0 B, free 729.3 MB) -2016-05-25 11:18:59 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53904 (size: 81.0 B, free: 730.1 MB) -2016-05-25 11:18:59 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 11:19:00 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 11:19:00 INFO DAGScheduler:59 - Registering RDD 17 (mapPartitions at DecisionTree.scala:625) -2016-05-25 11:19:00 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 11:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 11:19:00 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 11:19:00 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 11:19:00 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 11:19:00 INFO MemoryStore:59 - ensureFreeSpace(468112) called with curMem=1361921, maxMem=766075207 -2016-05-25 11:19:00 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 457.1 KB, free 728.8 MB) -2016-05-25 11:19:00 INFO MemoryStore:59 - ensureFreeSpace(114821) called with curMem=1830033, maxMem=766075207 -2016-05-25 11:19:00 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 112.1 KB, free 728.7 MB) -2016-05-25 11:19:00 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53904 (size: 112.1 KB, free: 730.0 MB) -2016-05-25 11:19:00 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625) -2016-05-25 11:19:00 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 11:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:19:00 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 11:19:00 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 11:19:00 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-25 11:19:00 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-25 11:19:00 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 11:19:00 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 11:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 46 ms on localhost (1/2) -2016-05-25 11:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 48 ms on localhost (2/2) -2016-05-25 11:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 11:19:00 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.048 s -2016-05-25 11:19:00 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:19:00 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:19:00 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 11:19:00 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:19:00 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 11:19:00 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 11:19:00 INFO MemoryStore:59 - ensureFreeSpace(202144) called with curMem=1944854, maxMem=766075207 -2016-05-25 11:19:00 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 197.4 KB, free 728.5 MB) -2016-05-25 11:19:00 INFO MemoryStore:59 - ensureFreeSpace(43169) called with curMem=2146998, maxMem=766075207 -2016-05-25 11:19:00 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 42.2 KB, free 728.5 MB) -2016-05-25 11:19:00 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53904 (size: 42.2 KB, free: 729.9 MB) -2016-05-25 11:19:00 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645) -2016-05-25 11:19:00 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 11:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:19:00 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 11:19:00 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 11:19:00 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:19:00 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 11:19:00 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 1161 bytes result sent to driver -2016-05-25 11:19:00 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 10 ms on localhost (1/2) -2016-05-25 11:19:00 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 11:19:00 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 11:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 66 ms on localhost (2/2) -2016-05-25 11:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 11:19:00 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.066 s -2016-05-25 11:19:00 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.169262 s -2016-05-25 11:19:00 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 11:19:00 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 11:19:00 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 11:19:00 INFO RandomForest:59 - init: 1.499099319 - total: 2.703977943 - findSplitsBins: 1.258327335 - findBestSplits: 1.183960612 - chooseSplits: 1.176903756 -2016-05-25 11:19:00 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:47 -2016-05-25 11:19:00 INFO DAGScheduler:59 - Got job 6 (count at DecisionTreesLearning.scala:47) with 2 output partitions -2016-05-25 11:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 8(count at DecisionTreesLearning.scala:47) -2016-05-25 11:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:19:00 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at filter at DecisionTreesLearning.scala:47), which has no missing parents -2016-05-25 11:19:00 INFO MemoryStore:59 - ensureFreeSpace(6744) called with curMem=1998511, maxMem=766075207 -2016-05-25 11:19:00 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 6.6 KB, free 728.7 MB) -2016-05-25 11:19:00 INFO MemoryStore:59 - ensureFreeSpace(3587) called with curMem=2005255, maxMem=766075207 -2016-05-25 11:19:00 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.5 KB, free 728.7 MB) -2016-05-25 11:19:00 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53904 (size: 3.5 KB, free: 730.1 MB) -2016-05-25 11:19:00 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at filter at DecisionTreesLearning.scala:47) -2016-05-25 11:19:00 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 11:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:00 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 11:19:00 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 11:19:00 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:19:00 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 11:19:00 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:19:00 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 11:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 11 ms on localhost (1/2) -2016-05-25 11:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 15 ms on localhost (2/2) -2016-05-25 11:19:00 INFO DAGScheduler:59 - ResultStage 8 (count at DecisionTreesLearning.scala:47) finished in 0.015 s -2016-05-25 11:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 11:19:00 INFO DAGScheduler:59 - Job 6 finished: count at DecisionTreesLearning.scala:47, took 0.031562 s -2016-05-25 11:19:00 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:47 -2016-05-25 11:19:00 INFO DAGScheduler:59 - Got job 7 (count at DecisionTreesLearning.scala:47) with 2 output partitions -2016-05-25 11:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at DecisionTreesLearning.scala:47) -2016-05-25 11:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:19:00 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:19:00 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2008842, maxMem=766075207 -2016-05-25 11:19:00 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.0 KB, free 728.7 MB) -2016-05-25 11:19:00 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=2012954, maxMem=766075207 -2016-05-25 11:19:00 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.7 MB) -2016-05-25 11:19:00 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53904 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 11:19:00 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:19:00 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 11:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:00 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-25 11:19:00 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 11:19:00 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:19:00 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:19:00 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2082 bytes result sent to driver -2016-05-25 11:19:00 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2082 bytes result sent to driver -2016-05-25 11:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 11 ms on localhost (1/2) -2016-05-25 11:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 11 ms on localhost (2/2) -2016-05-25 11:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 11:19:00 INFO DAGScheduler:59 - ResultStage 9 (count at DecisionTreesLearning.scala:47) finished in 0.012 s -2016-05-25 11:19:00 INFO DAGScheduler:59 - Job 7 finished: count at DecisionTreesLearning.scala:47, took 0.028890 s -2016-05-25 11:19:00 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:50 -2016-05-25 11:19:00 INFO DAGScheduler:59 - Got job 8 (count at DecisionTreesLearning.scala:50) with 2 output partitions -2016-05-25 11:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 10(count at DecisionTreesLearning.scala:50) -2016-05-25 11:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:19:00 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 11:19:00 INFO MemoryStore:59 - ensureFreeSpace(3816) called with curMem=2015222, maxMem=766075207 -2016-05-25 11:19:00 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 3.7 KB, free 728.7 MB) -2016-05-25 11:19:00 INFO MemoryStore:59 - ensureFreeSpace(2142) called with curMem=2019038, maxMem=766075207 -2016-05-25 11:19:00 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.1 KB, free 728.7 MB) -2016-05-25 11:19:00 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53904 (size: 2.1 KB, free: 730.1 MB) -2016-05-25 11:19:00 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 11:19:00 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-25 11:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:00 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 11:19:00 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-25 11:19:00 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:19:00 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:19:00 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2082 bytes result sent to driver -2016-05-25 11:19:00 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2082 bytes result sent to driver -2016-05-25 11:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 9 ms on localhost (1/2) -2016-05-25 11:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 12 ms on localhost (2/2) -2016-05-25 11:19:00 INFO DAGScheduler:59 - ResultStage 10 (count at DecisionTreesLearning.scala:50) finished in 0.012 s -2016-05-25 11:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 11:19:00 INFO DAGScheduler:59 - Job 8 finished: count at DecisionTreesLearning.scala:50, took 0.026422 s -2016-05-25 11:19:00 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:51 -2016-05-25 11:19:00 INFO DAGScheduler:59 - Got job 9 (count at DecisionTreesLearning.scala:51) with 2 output partitions -2016-05-25 11:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 11(count at DecisionTreesLearning.scala:51) -2016-05-25 11:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:19:00 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[7] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:19:00 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2021180, maxMem=766075207 -2016-05-25 11:19:00 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 4.0 KB, free 728.7 MB) -2016-05-25 11:19:00 INFO MemoryStore:59 - ensureFreeSpace(2265) called with curMem=2025292, maxMem=766075207 -2016-05-25 11:19:00 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.7 MB) -2016-05-25 11:19:00 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53904 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 11:19:00 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[7] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:19:00 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-25 11:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:00 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-25 11:19:00 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-25 11:19:00 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:19:00 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:19:00 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2082 bytes result sent to driver -2016-05-25 11:19:00 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2082 bytes result sent to driver -2016-05-25 11:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 10 ms on localhost (1/2) -2016-05-25 11:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 12 ms on localhost (2/2) -2016-05-25 11:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 11:19:00 INFO DAGScheduler:59 - ResultStage 11 (count at DecisionTreesLearning.scala:51) finished in 0.013 s -2016-05-25 11:19:00 INFO DAGScheduler:59 - Job 9 finished: count at DecisionTreesLearning.scala:51, took 0.027163 s -2016-05-25 11:19:00 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:52 -2016-05-25 11:19:00 INFO DAGScheduler:59 - Got job 10 (count at DecisionTreesLearning.scala:52) with 2 output partitions -2016-05-25 11:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 12(count at DecisionTreesLearning.scala:52) -2016-05-25 11:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:19:00 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:19:00 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2027557, maxMem=766075207 -2016-05-25 11:19:00 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 4.0 KB, free 728.6 MB) -2016-05-25 11:19:00 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=2031669, maxMem=766075207 -2016-05-25 11:19:00 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.6 MB) -2016-05-25 11:19:00 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53904 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 11:19:00 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:19:00 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-25 11:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:00 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-25 11:19:00 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-25 11:19:00 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:19:00 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:19:00 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2082 bytes result sent to driver -2016-05-25 11:19:00 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2082 bytes result sent to driver -2016-05-25 11:19:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 9 ms on localhost (1/2) -2016-05-25 11:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 11 ms on localhost (2/2) -2016-05-25 11:19:00 INFO DAGScheduler:59 - ResultStage 12 (count at DecisionTreesLearning.scala:52) finished in 0.012 s -2016-05-25 11:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 11:19:00 INFO DAGScheduler:59 - Job 10 finished: count at DecisionTreesLearning.scala:52, took 0.023495 s -2016-05-25 11:19:00 INFO SparkContext:59 - Starting job: take at DecisionTreesLearning.scala:54 -2016-05-25 11:19:00 INFO DAGScheduler:59 - Got job 11 (take at DecisionTreesLearning.scala:54) with 1 output partitions -2016-05-25 11:19:00 INFO DAGScheduler:59 - Final stage: ResultStage 13(take at DecisionTreesLearning.scala:54) -2016-05-25 11:19:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:19:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:19:00 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[20] at map at DecisionTreesLearning.scala:43), which has no missing parents -2016-05-25 11:19:00 INFO MemoryStore:59 - ensureFreeSpace(6752) called with curMem=2033937, maxMem=766075207 -2016-05-25 11:19:00 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 6.6 KB, free 728.6 MB) -2016-05-25 11:19:00 INFO MemoryStore:59 - ensureFreeSpace(3621) called with curMem=2040689, maxMem=766075207 -2016-05-25 11:19:00 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.5 KB, free 728.6 MB) -2016-05-25 11:19:00 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53904 (size: 3.5 KB, free: 730.1 MB) -2016-05-25 11:19:00 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:00 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 13 (MapPartitionsRDD[20] at map at DecisionTreesLearning.scala:43) -2016-05-25 11:19:00 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 1 tasks -2016-05-25 11:19:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:00 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-25 11:19:00 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:19:00 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2401 bytes result sent to driver -2016-05-25 11:19:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 12 ms on localhost (1/1) -2016-05-25 11:19:00 INFO DAGScheduler:59 - ResultStage 13 (take at DecisionTreesLearning.scala:54) finished in 0.012 s -2016-05-25 11:19:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 11:19:00 INFO DAGScheduler:59 - Job 11 finished: take at DecisionTreesLearning.scala:54, took 0.028144 s -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 11:19:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 11:19:00 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 11:19:00 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 11:19:00 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 11:19:00 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 11:19:00 INFO BlockManager:59 - BlockManager stopped -2016-05-25 11:19:00 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 11:19:00 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 11:19:00 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 11:19:00 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 11:19:00 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-2103fc9e-555b-4fb8-96e1-b0a732096fd9 -2016-05-25 11:19:00 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 11:19:33 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 11:19:33 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 11:19:34 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 11:19:34 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 11:19:34 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 11:19:34 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 11:19:34 INFO Remoting:74 - Starting remoting -2016-05-25 11:19:35 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53925] -2016-05-25 11:19:35 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53925. -2016-05-25 11:19:35 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 11:19:35 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 11:19:35 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-1da12e99-c119-4058-9793-66cf0d0cacc4 -2016-05-25 11:19:35 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 11:19:35 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-d995cd89-d616-415f-96bb-0d9c0ae36a0e\httpd-7a362e59-0002-402a-9790-721fbe78bdbe -2016-05-25 11:19:35 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 11:19:35 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:19:35 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53926 -2016-05-25 11:19:35 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53926. -2016-05-25 11:19:35 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 11:19:35 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:19:35 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 11:19:35 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 11:19:35 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 11:19:35 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 11:19:35 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 11:19:36 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53945. -2016-05-25 11:19:36 INFO NettyBlockTransferService:59 - Server created on 53945 -2016-05-25 11:19:36 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 11:19:36 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53945 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53945) -2016-05-25 11:19:36 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 11:19:36 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 11:19:36 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 11:19:36 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 11:19:36 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 11:19:36 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53945 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 11:19:36 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 11:19:38 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 11:19:38 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 11:19:38 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 11:19:38 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 11:19:38 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 11:19:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:19:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:19:38 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 11:19:38 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 11:19:38 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 11:19:38 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 11:19:38 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 11:19:38 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53945 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:19:38 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 11:19:38 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 11:19:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:38 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 11:19:38 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 11:19:38 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 11:19:38 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 11:19:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 11:19:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 11:19:38 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 11:19:38 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 11:19:38 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 11:19:38 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 11:19:38 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 11:19:39 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177630, maxMem=766075207 -2016-05-25 11:19:39 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 11:19:39 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263278, maxMem=766075207 -2016-05-25 11:19:39 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 11:19:39 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:53945 (size: 83.6 KB, free: 730.5 MB) -2016-05-25 11:19:39 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:53945 (size: 83.1 KB, free: 730.4 MB) -2016-05-25 11:19:39 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 11:19:39 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 11:19:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 363 ms on localhost (1/2) -2016-05-25 11:19:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 390 ms on localhost (2/2) -2016-05-25 11:19:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 11:19:39 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.407 s -2016-05-25 11:19:39 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.552843 s -2016-05-25 11:19:39 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 11:19:39 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 11:19:39 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 11:19:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:19:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:19:39 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 11:19:39 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348358, maxMem=766075207 -2016-05-25 11:19:39 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 11:19:39 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=353126, maxMem=766075207 -2016-05-25 11:19:39 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 11:19:39 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53945 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:19:39 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:39 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 11:19:39 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 11:19:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:39 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 11:19:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:19:39 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 11:19:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 32 ms on localhost (1/1) -2016-05-25 11:19:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 11:19:39 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.032 s -2016-05-25 11:19:39 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.058229 s -2016-05-25 11:19:39 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 11:19:39 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 11:19:39 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 11:19:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:19:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:19:39 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 11:19:39 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355673, maxMem=766075207 -2016-05-25 11:19:39 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 11:19:39 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=360009, maxMem=766075207 -2016-05-25 11:19:39 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 11:19:39 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53945 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:19:39 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 11:19:39 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 11:19:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:39 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 11:19:39 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 11:19:39 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:19:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:19:39 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 11:19:39 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 11:19:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 17 ms on localhost (1/2) -2016-05-25 11:19:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 17 ms on localhost (2/2) -2016-05-25 11:19:39 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.018 s -2016-05-25 11:19:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 11:19:39 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.035910 s -2016-05-25 11:19:39 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 11:19:39 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 11:19:39 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 11:19:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:19:39 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:19:39 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 11:19:39 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362328, maxMem=766075207 -2016-05-25 11:19:39 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 11:19:39 INFO MemoryStore:59 - ensureFreeSpace(2630) called with curMem=367392, maxMem=766075207 -2016-05-25 11:19:39 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 11:19:39 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53945 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:19:39 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 11:19:39 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 11:19:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:19:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:19:39 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 11:19:39 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 11:19:39 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:19:39 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 56065 bytes result sent to driver -2016-05-25 11:19:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:19:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 32 ms on localhost (1/2) -2016-05-25 11:19:39 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 61943 bytes result sent to driver -2016-05-25 11:19:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 45 ms on localhost (2/2) -2016-05-25 11:19:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 11:19:39 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.043 s -2016-05-25 11:19:39 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.065393 s -2016-05-25 11:19:40 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53945 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:19:40 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 11:19:40 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53945 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:19:40 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 11:19:40 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53945 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:19:40 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 11:19:40 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53945 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-25 11:19:40 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=342308, maxMem=766075207 -2016-05-25 11:19:40 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 11:19:40 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=342348, maxMem=766075207 -2016-05-25 11:19:40 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 11:19:40 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53945 (size: 81.0 B, free: 730.4 MB) -2016-05-25 11:19:40 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 11:19:40 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 11:19:40 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 11:19:40 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 11:19:40 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 11:19:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 11:19:40 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 11:19:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 11:19:40 INFO MemoryStore:59 - ensureFreeSpace(477336) called with curMem=342429, maxMem=766075207 -2016-05-25 11:19:40 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 466.1 KB, free 729.8 MB) -2016-05-25 11:19:40 INFO MemoryStore:59 - ensureFreeSpace(116906) called with curMem=819765, maxMem=766075207 -2016-05-25 11:19:40 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 114.2 KB, free 729.7 MB) -2016-05-25 11:19:40 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53945 (size: 114.2 KB, free: 730.3 MB) -2016-05-25 11:19:40 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 11:19:40 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 11:19:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:19:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:19:40 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 11:19:40 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 11:19:41 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 11:19:41 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 11:19:41 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:19:41 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:19:41 INFO MemoryStore:59 - ensureFreeSpace(85816) called with curMem=936671, maxMem=766075207 -2016-05-25 11:19:41 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 83.8 KB, free 729.6 MB) -2016-05-25 11:19:41 INFO MemoryStore:59 - ensureFreeSpace(100120) called with curMem=1022487, maxMem=766075207 -2016-05-25 11:19:41 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 97.8 KB, free 729.5 MB) -2016-05-25 11:19:41 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:53945 (size: 83.8 KB, free: 730.2 MB) -2016-05-25 11:19:41 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:53945 (size: 97.8 KB, free: 730.1 MB) -2016-05-25 11:19:41 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 11:19:41 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 11:19:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 229 ms on localhost (1/2) -2016-05-25 11:19:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 226 ms on localhost (2/2) -2016-05-25 11:19:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 11:19:41 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.230 s -2016-05-25 11:19:41 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:19:41 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:19:41 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 11:19:41 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:19:41 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 11:19:41 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 11:19:41 INFO MemoryStore:59 - ensureFreeSpace(206848) called with curMem=1122607, maxMem=766075207 -2016-05-25 11:19:41 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 202.0 KB, free 729.3 MB) -2016-05-25 11:19:41 INFO MemoryStore:59 - ensureFreeSpace(44100) called with curMem=1329455, maxMem=766075207 -2016-05-25 11:19:41 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 43.1 KB, free 729.3 MB) -2016-05-25 11:19:41 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53945 (size: 43.1 KB, free: 730.1 MB) -2016-05-25 11:19:41 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 11:19:41 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 11:19:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:19:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:19:41 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 11:19:41 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 11:19:41 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:19:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:19:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 11:19:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 11:19:41 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 11:19:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 154 ms on localhost (1/2) -2016-05-25 11:19:41 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 11:19:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 425 ms on localhost (2/2) -2016-05-25 11:19:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 11:19:41 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.426 s -2016-05-25 11:19:41 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.713270 s -2016-05-25 11:19:41 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 11:19:41 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 11:19:41 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 11:19:41 INFO RandomForest:59 - init: 1.542191878 - total: 2.470562224 - findSplitsBins: 1.295213519 - findBestSplits: 0.910358519 - chooseSplits: 0.903791605 -2016-05-25 11:19:41 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:47 -2016-05-25 11:19:41 INFO DAGScheduler:59 - Got job 5 (count at DecisionTreesLearning.scala:47) with 2 output partitions -2016-05-25 11:19:41 INFO DAGScheduler:59 - Final stage: ResultStage 6(count at DecisionTreesLearning.scala:47) -2016-05-25 11:19:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:19:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:19:41 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[18] at filter at DecisionTreesLearning.scala:47), which has no missing parents -2016-05-25 11:19:41 INFO MemoryStore:59 - ensureFreeSpace(6520) called with curMem=1187619, maxMem=766075207 -2016-05-25 11:19:41 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 6.4 KB, free 729.4 MB) -2016-05-25 11:19:41 INFO MemoryStore:59 - ensureFreeSpace(3488) called with curMem=1194139, maxMem=766075207 -2016-05-25 11:19:41 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-25 11:19:41 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53945 (size: 3.4 KB, free: 730.3 MB) -2016-05-25 11:19:41 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[18] at filter at DecisionTreesLearning.scala:47) -2016-05-25 11:19:41 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 11:19:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:41 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 11:19:41 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 11:19:41 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:19:41 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:19:41 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2082 bytes result sent to driver -2016-05-25 11:19:41 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2082 bytes result sent to driver -2016-05-25 11:19:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 12 ms on localhost (1/2) -2016-05-25 11:19:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 13 ms on localhost (2/2) -2016-05-25 11:19:41 INFO DAGScheduler:59 - ResultStage 6 (count at DecisionTreesLearning.scala:47) finished in 0.014 s -2016-05-25 11:19:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 11:19:41 INFO DAGScheduler:59 - Job 5 finished: count at DecisionTreesLearning.scala:47, took 0.032038 s -2016-05-25 11:19:41 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:47 -2016-05-25 11:19:41 INFO DAGScheduler:59 - Got job 6 (count at DecisionTreesLearning.scala:47) with 2 output partitions -2016-05-25 11:19:41 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at DecisionTreesLearning.scala:47) -2016-05-25 11:19:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:19:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:19:41 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:19:41 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1197627, maxMem=766075207 -2016-05-25 11:19:41 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-05-25 11:19:41 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=1201739, maxMem=766075207 -2016-05-25 11:19:41 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.4 MB) -2016-05-25 11:19:41 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53945 (size: 2.2 KB, free: 730.3 MB) -2016-05-25 11:19:41 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:19:41 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 11:19:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:41 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 11:19:41 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 11:19:41 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:19:41 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:19:41 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2082 bytes result sent to driver -2016-05-25 11:19:41 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2082 bytes result sent to driver -2016-05-25 11:19:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 12 ms on localhost (1/2) -2016-05-25 11:19:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 11 ms on localhost (2/2) -2016-05-25 11:19:41 INFO DAGScheduler:59 - ResultStage 7 (count at DecisionTreesLearning.scala:47) finished in 0.013 s -2016-05-25 11:19:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 11:19:41 INFO DAGScheduler:59 - Job 6 finished: count at DecisionTreesLearning.scala:47, took 0.031050 s -2016-05-25 11:19:41 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:50 -2016-05-25 11:19:41 INFO DAGScheduler:59 - Got job 7 (count at DecisionTreesLearning.scala:50) with 2 output partitions -2016-05-25 11:19:41 INFO DAGScheduler:59 - Final stage: ResultStage 8(count at DecisionTreesLearning.scala:50) -2016-05-25 11:19:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:19:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:19:41 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 11:19:41 INFO MemoryStore:59 - ensureFreeSpace(3816) called with curMem=1204007, maxMem=766075207 -2016-05-25 11:19:41 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 3.7 KB, free 729.4 MB) -2016-05-25 11:19:41 INFO MemoryStore:59 - ensureFreeSpace(2142) called with curMem=1207823, maxMem=766075207 -2016-05-25 11:19:41 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.4 MB) -2016-05-25 11:19:41 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53945 (size: 2.1 KB, free: 730.2 MB) -2016-05-25 11:19:41 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 11:19:41 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 11:19:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:41 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 11:19:41 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 11:19:41 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:19:41 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 11:19:41 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:19:41 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 11:19:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 8 ms on localhost (1/2) -2016-05-25 11:19:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 11 ms on localhost (2/2) -2016-05-25 11:19:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 11:19:41 INFO DAGScheduler:59 - ResultStage 8 (count at DecisionTreesLearning.scala:50) finished in 0.012 s -2016-05-25 11:19:41 INFO DAGScheduler:59 - Job 7 finished: count at DecisionTreesLearning.scala:50, took 0.027871 s -2016-05-25 11:19:41 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:51 -2016-05-25 11:19:41 INFO DAGScheduler:59 - Got job 8 (count at DecisionTreesLearning.scala:51) with 2 output partitions -2016-05-25 11:19:41 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at DecisionTreesLearning.scala:51) -2016-05-25 11:19:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:19:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:19:41 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[7] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:19:41 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1209965, maxMem=766075207 -2016-05-25 11:19:41 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-05-25 11:19:41 INFO MemoryStore:59 - ensureFreeSpace(2265) called with curMem=1214077, maxMem=766075207 -2016-05-25 11:19:41 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.4 MB) -2016-05-25 11:19:41 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53945 (size: 2.2 KB, free: 730.2 MB) -2016-05-25 11:19:41 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[7] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:19:41 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 11:19:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:41 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-25 11:19:41 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 11:19:41 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:19:41 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:19:41 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2082 bytes result sent to driver -2016-05-25 11:19:41 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2082 bytes result sent to driver -2016-05-25 11:19:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 8 ms on localhost (1/2) -2016-05-25 11:19:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 11 ms on localhost (2/2) -2016-05-25 11:19:41 INFO DAGScheduler:59 - ResultStage 9 (count at DecisionTreesLearning.scala:51) finished in 0.011 s -2016-05-25 11:19:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 11:19:41 INFO DAGScheduler:59 - Job 8 finished: count at DecisionTreesLearning.scala:51, took 0.026078 s -2016-05-25 11:19:41 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:52 -2016-05-25 11:19:41 INFO DAGScheduler:59 - Got job 9 (count at DecisionTreesLearning.scala:52) with 2 output partitions -2016-05-25 11:19:41 INFO DAGScheduler:59 - Final stage: ResultStage 10(count at DecisionTreesLearning.scala:52) -2016-05-25 11:19:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:19:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:19:41 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:19:41 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1216342, maxMem=766075207 -2016-05-25 11:19:41 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-05-25 11:19:41 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=1220454, maxMem=766075207 -2016-05-25 11:19:41 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.4 MB) -2016-05-25 11:19:41 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53945 (size: 2.2 KB, free: 730.2 MB) -2016-05-25 11:19:41 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:19:41 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-25 11:19:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:41 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-25 11:19:41 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 11:19:41 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:19:41 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2082 bytes result sent to driver -2016-05-25 11:19:41 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:19:41 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2082 bytes result sent to driver -2016-05-25 11:19:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 8 ms on localhost (1/2) -2016-05-25 11:19:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 10 ms on localhost (2/2) -2016-05-25 11:19:41 INFO DAGScheduler:59 - ResultStage 10 (count at DecisionTreesLearning.scala:52) finished in 0.011 s -2016-05-25 11:19:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 11:19:41 INFO DAGScheduler:59 - Job 9 finished: count at DecisionTreesLearning.scala:52, took 0.026662 s -2016-05-25 11:19:41 INFO SparkContext:59 - Starting job: take at DecisionTreesLearning.scala:54 -2016-05-25 11:19:41 INFO DAGScheduler:59 - Got job 10 (take at DecisionTreesLearning.scala:54) with 1 output partitions -2016-05-25 11:19:41 INFO DAGScheduler:59 - Final stage: ResultStage 11(take at DecisionTreesLearning.scala:54) -2016-05-25 11:19:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:19:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:19:41 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[17] at map at DecisionTreesLearning.scala:43), which has no missing parents -2016-05-25 11:19:41 INFO MemoryStore:59 - ensureFreeSpace(6536) called with curMem=1222722, maxMem=766075207 -2016-05-25 11:19:41 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 6.4 KB, free 729.4 MB) -2016-05-25 11:19:41 INFO MemoryStore:59 - ensureFreeSpace(3522) called with curMem=1229258, maxMem=766075207 -2016-05-25 11:19:41 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-25 11:19:41 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53945 (size: 3.4 KB, free: 730.2 MB) -2016-05-25 11:19:41 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:41 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 11 (MapPartitionsRDD[17] at map at DecisionTreesLearning.scala:43) -2016-05-25 11:19:41 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 1 tasks -2016-05-25 11:19:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:41 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-25 11:19:41 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:19:41 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2521 bytes result sent to driver -2016-05-25 11:19:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 8 ms on localhost (1/1) -2016-05-25 11:19:41 INFO DAGScheduler:59 - ResultStage 11 (take at DecisionTreesLearning.scala:54) finished in 0.008 s -2016-05-25 11:19:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 11:19:41 INFO DAGScheduler:59 - Job 10 finished: take at DecisionTreesLearning.scala:54, took 0.022570 s -2016-05-25 11:19:41 INFO SparkContext:59 - Starting job: take at DecisionTreesLearning.scala:54 -2016-05-25 11:19:41 INFO DAGScheduler:59 - Got job 11 (take at DecisionTreesLearning.scala:54) with 1 output partitions -2016-05-25 11:19:41 INFO DAGScheduler:59 - Final stage: ResultStage 12(take at DecisionTreesLearning.scala:54) -2016-05-25 11:19:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:19:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:19:41 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[17] at map at DecisionTreesLearning.scala:43), which has no missing parents -2016-05-25 11:19:41 INFO MemoryStore:59 - ensureFreeSpace(6536) called with curMem=1232780, maxMem=766075207 -2016-05-25 11:19:41 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 6.4 KB, free 729.4 MB) -2016-05-25 11:19:41 INFO MemoryStore:59 - ensureFreeSpace(3522) called with curMem=1239316, maxMem=766075207 -2016-05-25 11:19:41 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-25 11:19:41 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53945 (size: 3.4 KB, free: 730.2 MB) -2016-05-25 11:19:41 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:19:41 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[17] at map at DecisionTreesLearning.scala:43) -2016-05-25 11:19:41 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-25 11:19:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 22, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:19:41 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 22) -2016-05-25 11:19:41 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:19:41 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 22). 2521 bytes result sent to driver -2016-05-25 11:19:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 22) in 9 ms on localhost (1/1) -2016-05-25 11:19:41 INFO DAGScheduler:59 - ResultStage 12 (take at DecisionTreesLearning.scala:54) finished in 0.009 s -2016-05-25 11:19:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 11:19:41 INFO DAGScheduler:59 - Job 11 finished: take at DecisionTreesLearning.scala:54, took 0.026753 s -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 11:19:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 11:19:41 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 11:19:41 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 11:19:42 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 11:19:42 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 11:19:42 INFO BlockManager:59 - BlockManager stopped -2016-05-25 11:19:42 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 11:19:42 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 11:19:42 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 11:19:42 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 11:19:42 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-d995cd89-d616-415f-96bb-0d9c0ae36a0e -2016-05-25 11:21:21 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 11:21:22 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 11:21:22 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 11:21:22 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 11:21:22 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 11:21:23 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 11:21:23 INFO Remoting:74 - Starting remoting -2016-05-25 11:21:23 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53967] -2016-05-25 11:21:23 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53967. -2016-05-25 11:21:23 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 11:21:23 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 11:21:24 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-2a4f076e-08a0-4c9a-b8c2-27ad3dc5499c -2016-05-25 11:21:24 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 11:21:24 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-256bee8c-d80f-499c-8f24-f93ce3cf54ff\httpd-6d0de9b4-6b44-4ca8-adc4-c24b43c5912e -2016-05-25 11:21:24 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 11:21:24 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:21:24 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53968 -2016-05-25 11:21:24 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53968. -2016-05-25 11:21:24 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 11:21:24 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:21:24 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 11:21:24 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 11:21:24 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 11:21:24 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 11:21:24 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 11:21:24 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53987. -2016-05-25 11:21:24 INFO NettyBlockTransferService:59 - Server created on 53987 -2016-05-25 11:21:24 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 11:21:24 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53987 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53987) -2016-05-25 11:21:24 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 11:21:25 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 11:21:25 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 11:21:25 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 11:21:25 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 11:21:25 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53987 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 11:21:25 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 11:21:26 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 11:21:27 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 11:21:27 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 11:21:27 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 11:21:27 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 11:21:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:21:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:21:27 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 11:21:27 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 11:21:27 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 11:21:27 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 11:21:27 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 11:21:27 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53987 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:21:27 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:21:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 11:21:27 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 11:21:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:21:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:21:27 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 11:21:27 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 11:21:27 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 11:21:27 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 11:21:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 11:21:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 11:21:27 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 11:21:27 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 11:21:27 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 11:21:27 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 11:21:27 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 11:21:27 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177630, maxMem=766075207 -2016-05-25 11:21:27 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 11:21:27 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:53987 (size: 83.6 KB, free: 730.5 MB) -2016-05-25 11:21:27 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263278, maxMem=766075207 -2016-05-25 11:21:27 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 11:21:27 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:53987 (size: 83.1 KB, free: 730.4 MB) -2016-05-25 11:21:28 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 11:21:28 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 11:21:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 443 ms on localhost (1/2) -2016-05-25 11:21:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 481 ms on localhost (2/2) -2016-05-25 11:21:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 11:21:28 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.514 s -2016-05-25 11:21:28 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.680019 s -2016-05-25 11:21:28 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 11:21:28 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 11:21:28 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 11:21:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:21:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:21:28 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 11:21:28 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348358, maxMem=766075207 -2016-05-25 11:21:28 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 11:21:28 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=353126, maxMem=766075207 -2016-05-25 11:21:28 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 11:21:28 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53987 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:21:28 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:21:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 11:21:28 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 11:21:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:21:28 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 11:21:28 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:21:28 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 11:21:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 24 ms on localhost (1/1) -2016-05-25 11:21:28 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.026 s -2016-05-25 11:21:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 11:21:28 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.044079 s -2016-05-25 11:21:28 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 11:21:28 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 11:21:28 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 11:21:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:21:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:21:28 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 11:21:28 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355673, maxMem=766075207 -2016-05-25 11:21:28 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 11:21:28 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=360009, maxMem=766075207 -2016-05-25 11:21:28 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 11:21:28 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53987 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:21:28 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:21:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 11:21:28 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 11:21:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:21:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:21:28 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 11:21:28 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 11:21:28 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:21:28 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:21:28 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 11:21:28 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 11:21:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 13 ms on localhost (1/2) -2016-05-25 11:21:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 12 ms on localhost (2/2) -2016-05-25 11:21:28 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.014 s -2016-05-25 11:21:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 11:21:28 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.028028 s -2016-05-25 11:21:28 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 11:21:28 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 11:21:28 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 11:21:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:21:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:21:28 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 11:21:28 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362328, maxMem=766075207 -2016-05-25 11:21:28 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 11:21:28 INFO MemoryStore:59 - ensureFreeSpace(2631) called with curMem=367392, maxMem=766075207 -2016-05-25 11:21:28 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 11:21:28 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53987 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:21:28 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:21:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 11:21:28 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 11:21:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:21:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:21:28 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 11:21:28 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 11:21:28 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:21:28 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:21:28 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 66679 bytes result sent to driver -2016-05-25 11:21:28 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 72497 bytes result sent to driver -2016-05-25 11:21:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 32 ms on localhost (1/2) -2016-05-25 11:21:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 32 ms on localhost (2/2) -2016-05-25 11:21:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 11:21:28 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.036 s -2016-05-25 11:21:28 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.052006 s -2016-05-25 11:21:29 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53987 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:21:29 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 11:21:29 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53987 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:21:29 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 11:21:29 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53987 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:21:29 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 11:21:29 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53987 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-25 11:21:29 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=342308, maxMem=766075207 -2016-05-25 11:21:29 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 11:21:29 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=342348, maxMem=766075207 -2016-05-25 11:21:29 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 11:21:29 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53987 (size: 81.0 B, free: 730.4 MB) -2016-05-25 11:21:29 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 11:21:29 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 11:21:29 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 11:21:29 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 11:21:29 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 11:21:29 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 11:21:29 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 11:21:29 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 11:21:29 INFO MemoryStore:59 - ensureFreeSpace(516208) called with curMem=342429, maxMem=766075207 -2016-05-25 11:21:29 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 504.1 KB, free 729.8 MB) -2016-05-25 11:21:29 INFO MemoryStore:59 - ensureFreeSpace(126632) called with curMem=858637, maxMem=766075207 -2016-05-25 11:21:29 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 123.7 KB, free 729.6 MB) -2016-05-25 11:21:29 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53987 (size: 123.7 KB, free: 730.3 MB) -2016-05-25 11:21:29 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:21:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 11:21:29 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 11:21:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:21:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:21:29 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 11:21:29 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 11:21:30 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 11:21:30 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:21:30 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 11:21:30 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:21:30 INFO MemoryStore:59 - ensureFreeSpace(114416) called with curMem=985269, maxMem=766075207 -2016-05-25 11:21:30 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 111.7 KB, free 729.5 MB) -2016-05-25 11:21:30 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:53987 (size: 111.7 KB, free: 730.2 MB) -2016-05-25 11:21:30 INFO MemoryStore:59 - ensureFreeSpace(108696) called with curMem=1099685, maxMem=766075207 -2016-05-25 11:21:30 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 106.1 KB, free 729.4 MB) -2016-05-25 11:21:30 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:53987 (size: 106.1 KB, free: 730.1 MB) -2016-05-25 11:21:30 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 11:21:30 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 11:21:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 249 ms on localhost (1/2) -2016-05-25 11:21:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 247 ms on localhost (2/2) -2016-05-25 11:21:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 11:21:30 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.251 s -2016-05-25 11:21:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:21:30 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:21:30 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 11:21:30 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:21:30 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 11:21:30 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 11:21:30 INFO MemoryStore:59 - ensureFreeSpace(225944) called with curMem=1208381, maxMem=766075207 -2016-05-25 11:21:30 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 220.6 KB, free 729.2 MB) -2016-05-25 11:21:30 INFO MemoryStore:59 - ensureFreeSpace(47696) called with curMem=1434325, maxMem=766075207 -2016-05-25 11:21:30 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 46.6 KB, free 729.2 MB) -2016-05-25 11:21:30 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53987 (size: 46.6 KB, free: 730.0 MB) -2016-05-25 11:21:30 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:21:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 11:21:30 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 11:21:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:21:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:21:30 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 11:21:30 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 11:21:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:21:30 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:21:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 11:21:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 11:21:30 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 11:21:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 160 ms on localhost (1/2) -2016-05-25 11:21:30 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 11:21:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 411 ms on localhost (2/2) -2016-05-25 11:21:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 11:21:30 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.412 s -2016-05-25 11:21:30 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.724930 s -2016-05-25 11:21:30 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=1482021, maxMem=766075207 -2016-05-25 11:21:30 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 40.0 B, free 729.2 MB) -2016-05-25 11:21:30 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=1482061, maxMem=766075207 -2016-05-25 11:21:30 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 81.0 B, free 729.2 MB) -2016-05-25 11:21:30 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53987 (size: 81.0 B, free: 730.0 MB) -2016-05-25 11:21:30 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 11:21:30 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 11:21:30 INFO DAGScheduler:59 - Registering RDD 17 (mapPartitions at DecisionTree.scala:625) -2016-05-25 11:21:30 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 11:21:30 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 11:21:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 11:21:30 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 11:21:30 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 11:21:30 INFO MemoryStore:59 - ensureFreeSpace(516560) called with curMem=1482142, maxMem=766075207 -2016-05-25 11:21:30 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 504.5 KB, free 728.7 MB) -2016-05-25 11:21:30 INFO MemoryStore:59 - ensureFreeSpace(126823) called with curMem=1998702, maxMem=766075207 -2016-05-25 11:21:30 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 123.9 KB, free 728.6 MB) -2016-05-25 11:21:30 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53987 (size: 123.9 KB, free: 729.9 MB) -2016-05-25 11:21:30 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:21:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625) -2016-05-25 11:21:30 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 11:21:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:21:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:21:30 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 11:21:30 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 11:21:30 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-25 11:21:30 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-25 11:21:30 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 11:21:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 39 ms on localhost (1/2) -2016-05-25 11:21:30 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 11:21:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 44 ms on localhost (2/2) -2016-05-25 11:21:30 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.044 s -2016-05-25 11:21:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 11:21:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:21:30 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:21:30 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 11:21:30 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:21:30 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 11:21:30 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 11:21:30 INFO MemoryStore:59 - ensureFreeSpace(225944) called with curMem=2125525, maxMem=766075207 -2016-05-25 11:21:30 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 220.6 KB, free 728.3 MB) -2016-05-25 11:21:30 INFO MemoryStore:59 - ensureFreeSpace(47689) called with curMem=2351469, maxMem=766075207 -2016-05-25 11:21:30 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 46.6 KB, free 728.3 MB) -2016-05-25 11:21:30 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53987 (size: 46.6 KB, free: 729.9 MB) -2016-05-25 11:21:30 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:21:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645) -2016-05-25 11:21:30 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 11:21:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:21:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:21:30 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 11:21:30 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 11:21:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:21:30 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:21:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 11:21:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 11:21:30 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 1161 bytes result sent to driver -2016-05-25 11:21:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 15 ms on localhost (1/2) -2016-05-25 11:21:30 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 11:21:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 80 ms on localhost (2/2) -2016-05-25 11:21:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 11:21:30 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.080 s -2016-05-25 11:21:30 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.177263 s -2016-05-25 11:21:30 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 11:21:30 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 11:21:30 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 11:21:30 INFO RandomForest:59 - init: 1.597002431 - total: 2.779630652 - findSplitsBins: 1.3721067 - findBestSplits: 1.160573704 - chooseSplits: 1.153024676 -2016-05-25 11:21:30 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:47 -2016-05-25 11:21:30 INFO DAGScheduler:59 - Got job 6 (count at DecisionTreesLearning.scala:47) with 2 output partitions -2016-05-25 11:21:30 INFO DAGScheduler:59 - Final stage: ResultStage 8(count at DecisionTreesLearning.scala:47) -2016-05-25 11:21:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:21:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:21:30 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at filter at DecisionTreesLearning.scala:47), which has no missing parents -2016-05-25 11:21:30 INFO MemoryStore:59 - ensureFreeSpace(6744) called with curMem=2176046, maxMem=766075207 -2016-05-25 11:21:30 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 6.6 KB, free 728.5 MB) -2016-05-25 11:21:30 INFO MemoryStore:59 - ensureFreeSpace(3588) called with curMem=2182790, maxMem=766075207 -2016-05-25 11:21:30 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.5 KB, free 728.5 MB) -2016-05-25 11:21:30 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53987 (size: 3.5 KB, free: 730.1 MB) -2016-05-25 11:21:30 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:21:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at filter at DecisionTreesLearning.scala:47) -2016-05-25 11:21:30 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 11:21:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:21:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:21:30 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 11:21:30 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 11:21:30 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:21:30 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:21:30 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 11:21:30 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 11:21:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 11 ms on localhost (1/2) -2016-05-25 11:21:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 13 ms on localhost (2/2) -2016-05-25 11:21:30 INFO DAGScheduler:59 - ResultStage 8 (count at DecisionTreesLearning.scala:47) finished in 0.013 s -2016-05-25 11:21:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 11:21:30 INFO DAGScheduler:59 - Job 6 finished: count at DecisionTreesLearning.scala:47, took 0.031837 s -2016-05-25 11:21:30 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:47 -2016-05-25 11:21:30 INFO DAGScheduler:59 - Got job 7 (count at DecisionTreesLearning.scala:47) with 2 output partitions -2016-05-25 11:21:30 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at DecisionTreesLearning.scala:47) -2016-05-25 11:21:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:21:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:21:30 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:21:30 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2186378, maxMem=766075207 -2016-05-25 11:21:30 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.0 KB, free 728.5 MB) -2016-05-25 11:21:30 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=2190490, maxMem=766075207 -2016-05-25 11:21:30 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.5 MB) -2016-05-25 11:21:30 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53987 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 11:21:30 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:21:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:21:30 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 11:21:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:21:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:21:30 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 11:21:30 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-25 11:21:30 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:21:30 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:21:30 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2082 bytes result sent to driver -2016-05-25 11:21:30 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2082 bytes result sent to driver -2016-05-25 11:21:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 9 ms on localhost (1/2) -2016-05-25 11:21:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 11 ms on localhost (2/2) -2016-05-25 11:21:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 11:21:30 INFO DAGScheduler:59 - ResultStage 9 (count at DecisionTreesLearning.scala:47) finished in 0.011 s -2016-05-25 11:21:30 INFO DAGScheduler:59 - Job 7 finished: count at DecisionTreesLearning.scala:47, took 0.027851 s -2016-05-25 11:21:30 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:50 -2016-05-25 11:21:30 INFO DAGScheduler:59 - Got job 8 (count at DecisionTreesLearning.scala:50) with 2 output partitions -2016-05-25 11:21:30 INFO DAGScheduler:59 - Final stage: ResultStage 10(count at DecisionTreesLearning.scala:50) -2016-05-25 11:21:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:21:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:21:30 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 11:21:31 INFO MemoryStore:59 - ensureFreeSpace(3816) called with curMem=2192758, maxMem=766075207 -2016-05-25 11:21:31 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 3.7 KB, free 728.5 MB) -2016-05-25 11:21:31 INFO MemoryStore:59 - ensureFreeSpace(2142) called with curMem=2196574, maxMem=766075207 -2016-05-25 11:21:31 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.1 KB, free 728.5 MB) -2016-05-25 11:21:31 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53987 (size: 2.1 KB, free: 730.1 MB) -2016-05-25 11:21:31 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:21:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 11:21:31 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-25 11:21:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:21:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:21:31 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-25 11:21:31 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 11:21:31 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:21:31 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:21:31 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2082 bytes result sent to driver -2016-05-25 11:21:31 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2082 bytes result sent to driver -2016-05-25 11:21:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 10 ms on localhost (1/2) -2016-05-25 11:21:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 13 ms on localhost (2/2) -2016-05-25 11:21:31 INFO DAGScheduler:59 - ResultStage 10 (count at DecisionTreesLearning.scala:50) finished in 0.013 s -2016-05-25 11:21:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 11:21:31 INFO DAGScheduler:59 - Job 8 finished: count at DecisionTreesLearning.scala:50, took 0.029758 s -2016-05-25 11:21:31 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:51 -2016-05-25 11:21:31 INFO DAGScheduler:59 - Got job 9 (count at DecisionTreesLearning.scala:51) with 2 output partitions -2016-05-25 11:21:31 INFO DAGScheduler:59 - Final stage: ResultStage 11(count at DecisionTreesLearning.scala:51) -2016-05-25 11:21:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:21:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:21:31 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[7] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:21:31 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2198716, maxMem=766075207 -2016-05-25 11:21:31 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 4.0 KB, free 728.5 MB) -2016-05-25 11:21:31 INFO MemoryStore:59 - ensureFreeSpace(2265) called with curMem=2202828, maxMem=766075207 -2016-05-25 11:21:31 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.5 MB) -2016-05-25 11:21:31 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53987 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 11:21:31 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:21:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[7] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:21:31 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-25 11:21:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:21:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:21:31 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-25 11:21:31 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-25 11:21:31 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:21:31 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:21:31 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2082 bytes result sent to driver -2016-05-25 11:21:31 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2082 bytes result sent to driver -2016-05-25 11:21:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 10 ms on localhost (1/2) -2016-05-25 11:21:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 11 ms on localhost (2/2) -2016-05-25 11:21:31 INFO DAGScheduler:59 - ResultStage 11 (count at DecisionTreesLearning.scala:51) finished in 0.011 s -2016-05-25 11:21:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 11:21:31 INFO DAGScheduler:59 - Job 9 finished: count at DecisionTreesLearning.scala:51, took 0.030323 s -2016-05-25 11:21:31 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:52 -2016-05-25 11:21:31 INFO DAGScheduler:59 - Got job 10 (count at DecisionTreesLearning.scala:52) with 2 output partitions -2016-05-25 11:21:31 INFO DAGScheduler:59 - Final stage: ResultStage 12(count at DecisionTreesLearning.scala:52) -2016-05-25 11:21:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:21:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:21:31 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:21:31 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2205093, maxMem=766075207 -2016-05-25 11:21:31 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 4.0 KB, free 728.5 MB) -2016-05-25 11:21:31 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=2209205, maxMem=766075207 -2016-05-25 11:21:31 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.5 MB) -2016-05-25 11:21:31 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53987 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 11:21:31 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:21:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:21:31 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-25 11:21:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:21:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:21:31 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-25 11:21:31 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-25 11:21:31 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:21:31 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:21:31 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2082 bytes result sent to driver -2016-05-25 11:21:31 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2082 bytes result sent to driver -2016-05-25 11:21:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 9 ms on localhost (1/2) -2016-05-25 11:21:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 8 ms on localhost (2/2) -2016-05-25 11:21:31 INFO DAGScheduler:59 - ResultStage 12 (count at DecisionTreesLearning.scala:52) finished in 0.009 s -2016-05-25 11:21:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 11:21:31 INFO DAGScheduler:59 - Job 10 finished: count at DecisionTreesLearning.scala:52, took 0.023855 s -2016-05-25 11:21:31 INFO SparkContext:59 - Starting job: take at DecisionTreesLearning.scala:58 -2016-05-25 11:21:31 INFO DAGScheduler:59 - Got job 11 (take at DecisionTreesLearning.scala:58) with 1 output partitions -2016-05-25 11:21:31 INFO DAGScheduler:59 - Final stage: ResultStage 13(take at DecisionTreesLearning.scala:58) -2016-05-25 11:21:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:21:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:21:31 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[20] at map at DecisionTreesLearning.scala:43), which has no missing parents -2016-05-25 11:21:31 INFO MemoryStore:59 - ensureFreeSpace(6792) called with curMem=2211473, maxMem=766075207 -2016-05-25 11:21:31 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 6.6 KB, free 728.5 MB) -2016-05-25 11:21:31 INFO MemoryStore:59 - ensureFreeSpace(3631) called with curMem=2218265, maxMem=766075207 -2016-05-25 11:21:31 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.5 KB, free 728.5 MB) -2016-05-25 11:21:31 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53987 (size: 3.5 KB, free: 730.1 MB) -2016-05-25 11:21:31 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:21:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 13 (MapPartitionsRDD[20] at map at DecisionTreesLearning.scala:43) -2016-05-25 11:21:31 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 1 tasks -2016-05-25 11:21:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:21:31 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-25 11:21:31 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:21:31 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2449 bytes result sent to driver -2016-05-25 11:21:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 9 ms on localhost (1/1) -2016-05-25 11:21:31 INFO DAGScheduler:59 - ResultStage 13 (take at DecisionTreesLearning.scala:58) finished in 0.010 s -2016-05-25 11:21:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 11:21:31 INFO DAGScheduler:59 - Job 11 finished: take at DecisionTreesLearning.scala:58, took 0.022002 s -2016-05-25 11:21:31 INFO SparkContext:59 - Starting job: take at DecisionTreesLearning.scala:58 -2016-05-25 11:21:31 INFO DAGScheduler:59 - Got job 12 (take at DecisionTreesLearning.scala:58) with 1 output partitions -2016-05-25 11:21:31 INFO DAGScheduler:59 - Final stage: ResultStage 14(take at DecisionTreesLearning.scala:58) -2016-05-25 11:21:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:21:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:21:31 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[20] at map at DecisionTreesLearning.scala:43), which has no missing parents -2016-05-25 11:21:31 INFO MemoryStore:59 - ensureFreeSpace(6792) called with curMem=2221896, maxMem=766075207 -2016-05-25 11:21:31 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 6.6 KB, free 728.5 MB) -2016-05-25 11:21:31 INFO MemoryStore:59 - ensureFreeSpace(3631) called with curMem=2228688, maxMem=766075207 -2016-05-25 11:21:31 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.5 KB, free 728.5 MB) -2016-05-25 11:21:31 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53987 (size: 3.5 KB, free: 730.1 MB) -2016-05-25 11:21:31 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:21:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 14 (MapPartitionsRDD[20] at map at DecisionTreesLearning.scala:43) -2016-05-25 11:21:31 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 1 tasks -2016-05-25 11:21:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 26, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:21:31 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 26) -2016-05-25 11:21:31 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:21:31 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 26). 2401 bytes result sent to driver -2016-05-25 11:21:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 26) in 9 ms on localhost (1/1) -2016-05-25 11:21:31 INFO DAGScheduler:59 - ResultStage 14 (take at DecisionTreesLearning.scala:58) finished in 0.009 s -2016-05-25 11:21:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 11:21:31 INFO DAGScheduler:59 - Job 12 finished: take at DecisionTreesLearning.scala:58, took 0.025123 s -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 11:21:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 11:21:31 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 11:21:31 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 11:21:31 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 11:21:31 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 11:21:31 INFO BlockManager:59 - BlockManager stopped -2016-05-25 11:21:31 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 11:21:31 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 11:21:31 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 11:21:31 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 11:21:31 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-256bee8c-d80f-499c-8f24-f93ce3cf54ff -2016-05-25 11:21:31 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 11:24:21 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 11:24:22 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 11:24:22 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 11:24:22 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 11:24:22 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 11:24:23 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 11:24:23 INFO Remoting:74 - Starting remoting -2016-05-25 11:24:23 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:54168] -2016-05-25 11:24:23 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54168. -2016-05-25 11:24:23 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 11:24:23 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 11:24:23 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-539eaee6-05b3-471d-a90f-df5ea15c4a8f -2016-05-25 11:24:23 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 11:24:23 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-26859fb1-4e39-46c4-8fe8-3089144d9b0c\httpd-07a43a86-a709-4e3c-833b-c6f776bb30c4 -2016-05-25 11:24:23 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 11:24:23 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:24:24 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54169 -2016-05-25 11:24:24 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54169. -2016-05-25 11:24:24 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 11:24:24 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 11:24:24 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 11:24:24 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 11:24:24 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 11:24:24 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 11:24:24 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 11:24:24 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54188. -2016-05-25 11:24:24 INFO NettyBlockTransferService:59 - Server created on 54188 -2016-05-25 11:24:24 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 11:24:24 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54188 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54188) -2016-05-25 11:24:24 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 11:24:25 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 11:24:25 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 11:24:25 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 11:24:25 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 11:24:25 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54188 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 11:24:25 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 11:24:26 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 11:24:27 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 11:24:27 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 11:24:27 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 11:24:27 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 11:24:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:24:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:24:27 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 11:24:27 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 11:24:27 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 11:24:27 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 11:24:27 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 11:24:27 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54188 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 11:24:27 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:24:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 11:24:27 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 11:24:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:24:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:24:27 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 11:24:27 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 11:24:27 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 11:24:27 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 11:24:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 11:24:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 11:24:27 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 11:24:27 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 11:24:27 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 11:24:27 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 11:24:27 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 11:24:27 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177630, maxMem=766075207 -2016-05-25 11:24:27 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 11:24:27 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:54188 (size: 83.1 KB, free: 730.5 MB) -2016-05-25 11:24:27 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262710, maxMem=766075207 -2016-05-25 11:24:27 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 11:24:27 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:54188 (size: 83.6 KB, free: 730.4 MB) -2016-05-25 11:24:27 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 11:24:27 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 11:24:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 437 ms on localhost (1/2) -2016-05-25 11:24:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 460 ms on localhost (2/2) -2016-05-25 11:24:27 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.477 s -2016-05-25 11:24:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 11:24:27 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.642814 s -2016-05-25 11:24:27 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 11:24:27 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 11:24:27 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 11:24:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:24:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:24:27 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 11:24:27 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348358, maxMem=766075207 -2016-05-25 11:24:27 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 11:24:27 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=353126, maxMem=766075207 -2016-05-25 11:24:27 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 11:24:27 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54188 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:24:27 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:24:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 11:24:27 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 11:24:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:24:27 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 11:24:27 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:24:27 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 11:24:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 30 ms on localhost (1/1) -2016-05-25 11:24:27 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.030 s -2016-05-25 11:24:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 11:24:27 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.046766 s -2016-05-25 11:24:27 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 11:24:27 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 11:24:27 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 11:24:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:24:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:24:27 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 11:24:27 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355673, maxMem=766075207 -2016-05-25 11:24:27 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 11:24:27 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=360009, maxMem=766075207 -2016-05-25 11:24:27 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 11:24:27 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54188 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:24:27 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:24:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 11:24:27 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 11:24:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:24:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:24:27 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 11:24:27 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 11:24:27 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:24:27 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:24:27 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 11:24:27 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 11:24:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 12 ms on localhost (1/2) -2016-05-25 11:24:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 13 ms on localhost (2/2) -2016-05-25 11:24:27 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.014 s -2016-05-25 11:24:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 11:24:27 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.026481 s -2016-05-25 11:24:28 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 11:24:28 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 11:24:28 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 11:24:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:24:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:24:28 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 11:24:28 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362328, maxMem=766075207 -2016-05-25 11:24:28 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 11:24:28 INFO MemoryStore:59 - ensureFreeSpace(2631) called with curMem=367392, maxMem=766075207 -2016-05-25 11:24:28 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 11:24:28 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54188 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:24:28 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:24:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 11:24:28 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 11:24:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:24:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 11:24:28 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 11:24:28 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 11:24:28 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:24:28 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:24:28 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 54395 bytes result sent to driver -2016-05-25 11:24:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 29 ms on localhost (1/2) -2016-05-25 11:24:28 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 61314 bytes result sent to driver -2016-05-25 11:24:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 39 ms on localhost (2/2) -2016-05-25 11:24:28 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.039 s -2016-05-25 11:24:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 11:24:28 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.055592 s -2016-05-25 11:24:29 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:54188 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 11:24:29 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 11:24:29 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:54188 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 11:24:29 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 11:24:29 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:54188 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 11:24:29 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 11:24:29 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:54188 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-25 11:24:29 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=342308, maxMem=766075207 -2016-05-25 11:24:29 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 11:24:29 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=342348, maxMem=766075207 -2016-05-25 11:24:29 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 11:24:29 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:54188 (size: 81.0 B, free: 730.4 MB) -2016-05-25 11:24:29 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 11:24:29 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 11:24:29 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 11:24:29 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 11:24:29 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 11:24:29 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 11:24:29 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 11:24:29 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 11:24:29 INFO MemoryStore:59 - ensureFreeSpace(475792) called with curMem=342429, maxMem=766075207 -2016-05-25 11:24:29 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 464.6 KB, free 729.8 MB) -2016-05-25 11:24:29 INFO MemoryStore:59 - ensureFreeSpace(116697) called with curMem=818221, maxMem=766075207 -2016-05-25 11:24:29 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 114.0 KB, free 729.7 MB) -2016-05-25 11:24:29 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:54188 (size: 114.0 KB, free: 730.3 MB) -2016-05-25 11:24:29 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:24:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 11:24:29 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 11:24:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:24:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 11:24:29 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 11:24:29 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 11:24:29 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 11:24:29 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:24:29 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 11:24:29 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:24:29 INFO MemoryStore:59 - ensureFreeSpace(100120) called with curMem=934918, maxMem=766075207 -2016-05-25 11:24:29 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 97.8 KB, free 729.6 MB) -2016-05-25 11:24:29 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:54188 (size: 97.8 KB, free: 730.2 MB) -2016-05-25 11:24:29 INFO MemoryStore:59 - ensureFreeSpace(88680) called with curMem=1035038, maxMem=766075207 -2016-05-25 11:24:29 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 86.6 KB, free 729.5 MB) -2016-05-25 11:24:29 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:54188 (size: 86.6 KB, free: 730.1 MB) -2016-05-25 11:24:29 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 11:24:29 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 11:24:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 244 ms on localhost (1/2) -2016-05-25 11:24:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 243 ms on localhost (2/2) -2016-05-25 11:24:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 11:24:29 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.247 s -2016-05-25 11:24:29 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 11:24:29 INFO DAGScheduler:59 - running: Set() -2016-05-25 11:24:29 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 11:24:29 INFO DAGScheduler:59 - failed: Set() -2016-05-25 11:24:29 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 11:24:29 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 11:24:29 INFO MemoryStore:59 - ensureFreeSpace(206096) called with curMem=1123718, maxMem=766075207 -2016-05-25 11:24:29 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 201.3 KB, free 729.3 MB) -2016-05-25 11:24:29 INFO MemoryStore:59 - ensureFreeSpace(44004) called with curMem=1329814, maxMem=766075207 -2016-05-25 11:24:29 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 43.0 KB, free 729.3 MB) -2016-05-25 11:24:29 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:54188 (size: 43.0 KB, free: 730.1 MB) -2016-05-25 11:24:29 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:24:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 11:24:29 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 11:24:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:24:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 11:24:29 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 11:24:29 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 11:24:29 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 11:24:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 11:24:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-25 11:24:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-25 11:24:29 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 11:24:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 113 ms on localhost (1/2) -2016-05-25 11:24:30 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 11:24:30 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.381 s -2016-05-25 11:24:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 381 ms on localhost (2/2) -2016-05-25 11:24:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 11:24:30 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.682541 s -2016-05-25 11:24:30 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 11:24:30 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 11:24:30 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 11:24:30 INFO RandomForest:59 - init: 1.419218744 - total: 2.342580351 - findSplitsBins: 1.193825486 - findBestSplits: 0.906837899 - chooseSplits: 0.899951051 -2016-05-25 11:24:30 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:47 -2016-05-25 11:24:30 INFO DAGScheduler:59 - Got job 5 (count at DecisionTreesLearning.scala:47) with 2 output partitions -2016-05-25 11:24:30 INFO DAGScheduler:59 - Final stage: ResultStage 6(count at DecisionTreesLearning.scala:47) -2016-05-25 11:24:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:24:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:24:30 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[18] at filter at DecisionTreesLearning.scala:47), which has no missing parents -2016-05-25 11:24:30 INFO MemoryStore:59 - ensureFreeSpace(6520) called with curMem=1185018, maxMem=766075207 -2016-05-25 11:24:30 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 6.4 KB, free 729.4 MB) -2016-05-25 11:24:30 INFO MemoryStore:59 - ensureFreeSpace(3487) called with curMem=1191538, maxMem=766075207 -2016-05-25 11:24:30 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-25 11:24:30 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:54188 (size: 3.4 KB, free: 730.3 MB) -2016-05-25 11:24:30 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:24:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[18] at filter at DecisionTreesLearning.scala:47) -2016-05-25 11:24:30 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 11:24:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:24:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:24:30 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 11:24:30 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 11:24:30 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:24:30 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:24:30 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2082 bytes result sent to driver -2016-05-25 11:24:30 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2082 bytes result sent to driver -2016-05-25 11:24:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 12 ms on localhost (1/2) -2016-05-25 11:24:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 15 ms on localhost (2/2) -2016-05-25 11:24:30 INFO DAGScheduler:59 - ResultStage 6 (count at DecisionTreesLearning.scala:47) finished in 0.015 s -2016-05-25 11:24:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 11:24:30 INFO DAGScheduler:59 - Job 5 finished: count at DecisionTreesLearning.scala:47, took 0.034650 s -2016-05-25 11:24:30 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:47 -2016-05-25 11:24:30 INFO DAGScheduler:59 - Got job 6 (count at DecisionTreesLearning.scala:47) with 2 output partitions -2016-05-25 11:24:30 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at DecisionTreesLearning.scala:47) -2016-05-25 11:24:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:24:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:24:30 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:24:30 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1195025, maxMem=766075207 -2016-05-25 11:24:30 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-05-25 11:24:30 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=1199137, maxMem=766075207 -2016-05-25 11:24:30 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.4 MB) -2016-05-25 11:24:30 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:54188 (size: 2.2 KB, free: 730.3 MB) -2016-05-25 11:24:30 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:24:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:24:30 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 11:24:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:24:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:24:30 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 11:24:30 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 11:24:30 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:24:30 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:24:30 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2082 bytes result sent to driver -2016-05-25 11:24:30 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2082 bytes result sent to driver -2016-05-25 11:24:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 9 ms on localhost (1/2) -2016-05-25 11:24:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 11 ms on localhost (2/2) -2016-05-25 11:24:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 11:24:30 INFO DAGScheduler:59 - ResultStage 7 (count at DecisionTreesLearning.scala:47) finished in 0.012 s -2016-05-25 11:24:30 INFO DAGScheduler:59 - Job 6 finished: count at DecisionTreesLearning.scala:47, took 0.030987 s -2016-05-25 11:24:30 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:50 -2016-05-25 11:24:30 INFO DAGScheduler:59 - Got job 7 (count at DecisionTreesLearning.scala:50) with 2 output partitions -2016-05-25 11:24:30 INFO DAGScheduler:59 - Final stage: ResultStage 8(count at DecisionTreesLearning.scala:50) -2016-05-25 11:24:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:24:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:24:30 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 11:24:30 INFO MemoryStore:59 - ensureFreeSpace(3816) called with curMem=1201405, maxMem=766075207 -2016-05-25 11:24:30 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 3.7 KB, free 729.4 MB) -2016-05-25 11:24:30 INFO MemoryStore:59 - ensureFreeSpace(2142) called with curMem=1205221, maxMem=766075207 -2016-05-25 11:24:30 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.4 MB) -2016-05-25 11:24:30 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:54188 (size: 2.1 KB, free: 730.2 MB) -2016-05-25 11:24:30 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:24:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 11:24:30 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 11:24:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:24:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:24:30 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 11:24:30 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 11:24:30 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:24:30 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 11:24:30 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:24:30 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 11:24:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 7 ms on localhost (1/2) -2016-05-25 11:24:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 9 ms on localhost (2/2) -2016-05-25 11:24:30 INFO DAGScheduler:59 - ResultStage 8 (count at DecisionTreesLearning.scala:50) finished in 0.009 s -2016-05-25 11:24:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 11:24:30 INFO DAGScheduler:59 - Job 7 finished: count at DecisionTreesLearning.scala:50, took 0.025238 s -2016-05-25 11:24:30 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:51 -2016-05-25 11:24:30 INFO DAGScheduler:59 - Got job 8 (count at DecisionTreesLearning.scala:51) with 2 output partitions -2016-05-25 11:24:30 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at DecisionTreesLearning.scala:51) -2016-05-25 11:24:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:24:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:24:30 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[7] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:24:30 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1207363, maxMem=766075207 -2016-05-25 11:24:30 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-05-25 11:24:30 INFO MemoryStore:59 - ensureFreeSpace(2265) called with curMem=1211475, maxMem=766075207 -2016-05-25 11:24:30 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.4 MB) -2016-05-25 11:24:30 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:54188 (size: 2.2 KB, free: 730.2 MB) -2016-05-25 11:24:30 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:24:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[7] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:24:30 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 11:24:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:24:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:24:30 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-25 11:24:30 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 11:24:30 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:24:30 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:24:30 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2082 bytes result sent to driver -2016-05-25 11:24:30 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2082 bytes result sent to driver -2016-05-25 11:24:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 11 ms on localhost (1/2) -2016-05-25 11:24:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 13 ms on localhost (2/2) -2016-05-25 11:24:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 11:24:30 INFO DAGScheduler:59 - ResultStage 9 (count at DecisionTreesLearning.scala:51) finished in 0.014 s -2016-05-25 11:24:30 INFO DAGScheduler:59 - Job 8 finished: count at DecisionTreesLearning.scala:51, took 0.030387 s -2016-05-25 11:24:30 INFO SparkContext:59 - Starting job: count at DecisionTreesLearning.scala:52 -2016-05-25 11:24:30 INFO DAGScheduler:59 - Got job 9 (count at DecisionTreesLearning.scala:52) with 2 output partitions -2016-05-25 11:24:30 INFO DAGScheduler:59 - Final stage: ResultStage 10(count at DecisionTreesLearning.scala:52) -2016-05-25 11:24:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:24:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:24:30 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28), which has no missing parents -2016-05-25 11:24:30 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1213740, maxMem=766075207 -2016-05-25 11:24:30 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-05-25 11:24:30 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=1217852, maxMem=766075207 -2016-05-25 11:24:30 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.4 MB) -2016-05-25 11:24:30 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:54188 (size: 2.2 KB, free: 730.2 MB) -2016-05-25 11:24:30 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:24:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[8] at randomSplit at DecisionTreesLearning.scala:28) -2016-05-25 11:24:30 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-25 11:24:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:24:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:24:30 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 11:24:30 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-25 11:24:30 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:24:30 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:24:30 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2082 bytes result sent to driver -2016-05-25 11:24:30 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2082 bytes result sent to driver -2016-05-25 11:24:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 9 ms on localhost (1/2) -2016-05-25 11:24:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 11 ms on localhost (2/2) -2016-05-25 11:24:30 INFO DAGScheduler:59 - ResultStage 10 (count at DecisionTreesLearning.scala:52) finished in 0.012 s -2016-05-25 11:24:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 11:24:30 INFO DAGScheduler:59 - Job 9 finished: count at DecisionTreesLearning.scala:52, took 0.025962 s -2016-05-25 11:24:30 INFO SparkContext:59 - Starting job: take at DecisionTreesLearning.scala:58 -2016-05-25 11:24:30 INFO DAGScheduler:59 - Got job 10 (take at DecisionTreesLearning.scala:58) with 1 output partitions -2016-05-25 11:24:30 INFO DAGScheduler:59 - Final stage: ResultStage 11(take at DecisionTreesLearning.scala:58) -2016-05-25 11:24:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:24:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:24:30 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[17] at map at DecisionTreesLearning.scala:43), which has no missing parents -2016-05-25 11:24:30 INFO MemoryStore:59 - ensureFreeSpace(6568) called with curMem=1220120, maxMem=766075207 -2016-05-25 11:24:30 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 6.4 KB, free 729.4 MB) -2016-05-25 11:24:30 INFO MemoryStore:59 - ensureFreeSpace(3532) called with curMem=1226688, maxMem=766075207 -2016-05-25 11:24:30 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-25 11:24:30 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:54188 (size: 3.4 KB, free: 730.2 MB) -2016-05-25 11:24:30 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:24:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 11 (MapPartitionsRDD[17] at map at DecisionTreesLearning.scala:43) -2016-05-25 11:24:30 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 1 tasks -2016-05-25 11:24:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:24:30 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-25 11:24:30 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 11:24:30 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2521 bytes result sent to driver -2016-05-25 11:24:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 8 ms on localhost (1/1) -2016-05-25 11:24:30 INFO DAGScheduler:59 - ResultStage 11 (take at DecisionTreesLearning.scala:58) finished in 0.009 s -2016-05-25 11:24:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 11:24:30 INFO DAGScheduler:59 - Job 10 finished: take at DecisionTreesLearning.scala:58, took 0.024592 s -2016-05-25 11:24:30 INFO SparkContext:59 - Starting job: take at DecisionTreesLearning.scala:58 -2016-05-25 11:24:30 INFO DAGScheduler:59 - Got job 11 (take at DecisionTreesLearning.scala:58) with 1 output partitions -2016-05-25 11:24:30 INFO DAGScheduler:59 - Final stage: ResultStage 12(take at DecisionTreesLearning.scala:58) -2016-05-25 11:24:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 11:24:30 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 11:24:30 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[17] at map at DecisionTreesLearning.scala:43), which has no missing parents -2016-05-25 11:24:30 INFO MemoryStore:59 - ensureFreeSpace(6568) called with curMem=1230220, maxMem=766075207 -2016-05-25 11:24:30 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 6.4 KB, free 729.4 MB) -2016-05-25 11:24:30 INFO MemoryStore:59 - ensureFreeSpace(3532) called with curMem=1236788, maxMem=766075207 -2016-05-25 11:24:30 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.4 KB, free 729.4 MB) -2016-05-25 11:24:30 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:54188 (size: 3.4 KB, free: 730.2 MB) -2016-05-25 11:24:30 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-25 11:24:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[17] at map at DecisionTreesLearning.scala:43) -2016-05-25 11:24:30 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-25 11:24:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 22, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 11:24:30 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 22) -2016-05-25 11:24:30 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 11:24:30 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 22). 2521 bytes result sent to driver -2016-05-25 11:24:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 22) in 8 ms on localhost (1/1) -2016-05-25 11:24:30 INFO DAGScheduler:59 - ResultStage 12 (take at DecisionTreesLearning.scala:58) finished in 0.008 s -2016-05-25 11:24:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 11:24:30 INFO DAGScheduler:59 - Job 11 finished: take at DecisionTreesLearning.scala:58, took 0.023507 s -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 11:24:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 11:24:30 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 11:24:30 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 11:24:30 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 11:24:30 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 11:24:30 INFO BlockManager:59 - BlockManager stopped -2016-05-25 11:24:30 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 11:24:30 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 11:24:30 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 11:24:30 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 11:24:30 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-26859fb1-4e39-46c4-8fe8-3089144d9b0c -2016-05-25 11:24:30 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 14:49:32 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 14:49:35 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 14:49:36 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 14:49:36 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 14:49:36 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 14:49:40 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 14:49:40 INFO Remoting:74 - Starting remoting -2016-05-25 14:49:41 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:59218] -2016-05-25 14:49:41 INFO Utils:59 - Successfully started service 'sparkDriver' on port 59218. -2016-05-25 14:49:41 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 14:49:41 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 14:49:42 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-b41dc145-5da5-4603-b094-8e849fdb1e6f -2016-05-25 14:49:42 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 14:49:42 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-160b7720-54a5-42ac-9d5b-343f0a569b00\httpd-b21bd5dc-266a-4a7c-81ee-2cea8cd49d44 -2016-05-25 14:49:42 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 14:49:42 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 14:49:42 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:59219 -2016-05-25 14:49:42 INFO Utils:59 - Successfully started service 'HTTP file server' on port 59219. -2016-05-25 14:49:43 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 14:49:43 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 14:49:43 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 14:49:43 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 14:49:44 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 14:49:44 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 14:49:44 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 14:49:44 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 59238. -2016-05-25 14:49:44 INFO NettyBlockTransferService:59 - Server created on 59238 -2016-05-25 14:49:44 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 14:49:44 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:59238 with 730.6 MB RAM, BlockManagerId(driver, localhost, 59238) -2016-05-25 14:49:45 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 14:49:47 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 14:49:47 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 14:49:47 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 14:49:47 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 14:49:47 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:59238 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 14:49:47 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 14:49:50 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 14:49:52 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 14:49:53 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 14:49:53 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 14:49:53 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 14:49:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 14:49:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 14:49:53 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 14:49:53 INFO MemoryStore:59 - ensureFreeSpace(3888) called with curMem=171580, maxMem=766075207 -2016-05-25 14:49:53 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 14:49:53 INFO MemoryStore:59 - ensureFreeSpace(2131) called with curMem=175468, maxMem=766075207 -2016-05-25 14:49:53 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 14:49:53 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:59238 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 14:49:53 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:49:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 14:49:53 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 14:49:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 14:49:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 14:49:53 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 14:49:53 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 14:49:53 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 14:49:53 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 14:49:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/dt.txt:0+57 -2016-05-25 14:49:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/dt.txt:57+57 -2016-05-25 14:49:53 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 14:49:53 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 14:49:53 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 14:49:53 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 14:49:53 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 14:49:53 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=177599, maxMem=766075207 -2016-05-25 14:49:53 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 280.0 B, free 730.4 MB) -2016-05-25 14:49:53 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:59238 (size: 280.0 B, free: 730.6 MB) -2016-05-25 14:49:53 INFO MemoryStore:59 - ensureFreeSpace(544) called with curMem=177879, maxMem=766075207 -2016-05-25 14:49:53 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 544.0 B, free 730.4 MB) -2016-05-25 14:49:53 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:59238 (size: 544.0 B, free: 730.6 MB) -2016-05-25 14:49:53 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 14:49:53 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 14:49:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 365 ms on localhost (1/2) -2016-05-25 14:49:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 468 ms on localhost (2/2) -2016-05-25 14:49:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 14:49:53 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.554 s -2016-05-25 14:49:54 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.929142 s -2016-05-25 14:49:54 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 14:49:54 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 14:49:54 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 14:49:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 14:49:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 14:49:54 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[8] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 14:49:54 INFO MemoryStore:59 - ensureFreeSpace(4456) called with curMem=178423, maxMem=766075207 -2016-05-25 14:49:54 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-25 14:49:54 INFO MemoryStore:59 - ensureFreeSpace(2401) called with curMem=182879, maxMem=766075207 -2016-05-25 14:49:54 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-25 14:49:54 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:59238 (size: 2.3 KB, free: 730.6 MB) -2016-05-25 14:49:54 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:49:54 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[8] at map at DecisionTreeMetadata.scala:110) -2016-05-25 14:49:54 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 14:49:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 14:49:54 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 14:49:54 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 14:49:54 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 14:49:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 125 ms on localhost (1/1) -2016-05-25 14:49:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 14:49:54 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.127 s -2016-05-25 14:49:54 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.147452 s -2016-05-25 14:49:54 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 14:49:54 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 14:49:54 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 14:49:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 14:49:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 14:49:54 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 14:49:54 INFO MemoryStore:59 - ensureFreeSpace(4016) called with curMem=185280, maxMem=766075207 -2016-05-25 14:49:54 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-05-25 14:49:54 INFO MemoryStore:59 - ensureFreeSpace(2184) called with curMem=189296, maxMem=766075207 -2016-05-25 14:49:54 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 14:49:54 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:59238 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 14:49:54 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:49:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at retag at RandomForest.scala:137) -2016-05-25 14:49:54 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 14:49:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 14:49:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 14:49:54 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 14:49:54 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 14:49:54 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 14:49:54 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 14:49:54 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 14:49:54 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 14:49:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 24 ms on localhost (1/2) -2016-05-25 14:49:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 25 ms on localhost (2/2) -2016-05-25 14:49:54 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.025 s -2016-05-25 14:49:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 14:49:54 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.042266 s -2016-05-25 14:49:54 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:59238 in memory (size: 2.3 KB, free: 730.6 MB) -2016-05-25 14:49:54 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 14:49:54 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:59238 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 14:49:54 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 14:49:54 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 14:49:54 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 14:49:54 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 14:49:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 14:49:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 14:49:54 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[9] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 14:49:54 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=178604, maxMem=766075207 -2016-05-25 14:49:54 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.6 KB, free 730.4 MB) -2016-05-25 14:49:54 INFO MemoryStore:59 - ensureFreeSpace(2491) called with curMem=183348, maxMem=766075207 -2016-05-25 14:49:54 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-05-25 14:49:54 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:59238 (size: 2.4 KB, free: 730.6 MB) -2016-05-25 14:49:54 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:49:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[9] at sample at DecisionTree.scala:989) -2016-05-25 14:49:54 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 14:49:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 14:49:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 14:49:54 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 14:49:54 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 14:49:54 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 14:49:54 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2677 bytes result sent to driver -2016-05-25 14:49:54 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 14:49:54 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2493 bytes result sent to driver -2016-05-25 14:49:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 22 ms on localhost (1/2) -2016-05-25 14:49:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 19 ms on localhost (2/2) -2016-05-25 14:49:54 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.023 s -2016-05-25 14:49:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 14:49:54 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.045897 s -2016-05-25 14:49:56 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:59238 in memory (size: 2.4 KB, free: 730.6 MB) -2016-05-25 14:49:56 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 14:50:02 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=178604, maxMem=766075207 -2016-05-25 14:50:02 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.4 MB) -2016-05-25 14:50:02 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=178644, maxMem=766075207 -2016-05-25 14:50:02 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.4 MB) -2016-05-25 14:50:02 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:59238 (size: 81.0 B, free: 730.6 MB) -2016-05-25 14:50:02 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 14:50:02 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 14:50:02 INFO DAGScheduler:59 - Registering RDD 12 (mapPartitions at DecisionTree.scala:625) -2016-05-25 14:50:02 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 14:50:02 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 14:50:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 14:50:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 14:50:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[12] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 14:50:02 INFO MemoryStore:59 - ensureFreeSpace(9736) called with curMem=178725, maxMem=766075207 -2016-05-25 14:50:02 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 9.5 KB, free 730.4 MB) -2016-05-25 14:50:02 INFO MemoryStore:59 - ensureFreeSpace(4926) called with curMem=188461, maxMem=766075207 -2016-05-25 14:50:02 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-25 14:50:02 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:59238 (size: 4.8 KB, free: 730.6 MB) -2016-05-25 14:50:02 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:50:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[12] at mapPartitions at DecisionTree.scala:625) -2016-05-25 14:50:02 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 14:50:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 14:50:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 14:50:02 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 14:50:02 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 14:50:02 INFO CacheManager:59 - Partition rdd_11_0 not found, computing it -2016-05-25 14:50:02 INFO CacheManager:59 - Partition rdd_11_1 not found, computing it -2016-05-25 14:50:02 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 14:50:02 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 14:50:02 INFO MemoryStore:59 - ensureFreeSpace(232) called with curMem=193387, maxMem=766075207 -2016-05-25 14:50:02 INFO MemoryStore:59 - Block rdd_11_1 stored as values in memory (estimated size 232.0 B, free 730.4 MB) -2016-05-25 14:50:02 INFO MemoryStore:59 - ensureFreeSpace(448) called with curMem=193619, maxMem=766075207 -2016-05-25 14:50:02 INFO MemoryStore:59 - Block rdd_11_0 stored as values in memory (estimated size 448.0 B, free 730.4 MB) -2016-05-25 14:50:02 INFO BlockManagerInfo:59 - Added rdd_11_1 in memory on localhost:59238 (size: 232.0 B, free: 730.6 MB) -2016-05-25 14:50:02 INFO BlockManagerInfo:59 - Added rdd_11_0 in memory on localhost:59238 (size: 448.0 B, free: 730.6 MB) -2016-05-25 14:50:02 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 14:50:02 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 14:50:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 207 ms on localhost (1/2) -2016-05-25 14:50:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 210 ms on localhost (2/2) -2016-05-25 14:50:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 14:50:02 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.213 s -2016-05-25 14:50:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 14:50:02 INFO DAGScheduler:59 - running: Set() -2016-05-25 14:50:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 14:50:02 INFO DAGScheduler:59 - failed: Set() -2016-05-25 14:50:02 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 14:50:02 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[14] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 14:50:02 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=194067, maxMem=766075207 -2016-05-25 14:50:02 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-05-25 14:50:02 INFO MemoryStore:59 - ensureFreeSpace(2871) called with curMem=199299, maxMem=766075207 -2016-05-25 14:50:02 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-25 14:50:02 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:59238 (size: 2.8 KB, free: 730.6 MB) -2016-05-25 14:50:02 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:50:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[14] at map at DecisionTree.scala:645) -2016-05-25 14:50:02 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 14:50:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 14:50:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 14:50:02 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 14:50:02 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 14:50:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 14:50:02 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 14:50:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 19 ms -2016-05-25 14:50:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 19 ms -2016-05-25 14:50:02 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 14:50:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 79 ms on localhost (1/2) -2016-05-25 14:50:02 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 14:50:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 154 ms on localhost (2/2) -2016-05-25 14:50:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 14:50:02 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.155 s -2016-05-25 14:50:02 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.428487 s -2016-05-25 14:50:02 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=202170, maxMem=766075207 -2016-05-25 14:50:02 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 40.0 B, free 730.4 MB) -2016-05-25 14:50:02 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=202210, maxMem=766075207 -2016-05-25 14:50:02 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.4 MB) -2016-05-25 14:50:02 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:59238 (size: 81.0 B, free: 730.6 MB) -2016-05-25 14:50:02 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 14:50:02 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 14:50:02 INFO DAGScheduler:59 - Registering RDD 15 (mapPartitions at DecisionTree.scala:625) -2016-05-25 14:50:02 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 14:50:02 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 14:50:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 14:50:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 14:50:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[15] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 14:50:02 INFO MemoryStore:59 - ensureFreeSpace(10096) called with curMem=202291, maxMem=766075207 -2016-05-25 14:50:02 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 9.9 KB, free 730.4 MB) -2016-05-25 14:50:02 INFO MemoryStore:59 - ensureFreeSpace(5114) called with curMem=212387, maxMem=766075207 -2016-05-25 14:50:02 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 14:50:02 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:59238 (size: 5.0 KB, free: 730.6 MB) -2016-05-25 14:50:02 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:50:02 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:59238 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-25 14:50:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[15] at mapPartitions at DecisionTree.scala:625) -2016-05-25 14:50:02 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 14:50:02 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 14:50:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 14:50:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 14:50:02 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 14:50:02 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 14:50:03 INFO BlockManager:59 - Found block rdd_11_0 locally -2016-05-25 14:50:03 INFO BlockManager:59 - Found block rdd_11_1 locally -2016-05-25 14:50:03 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 14:50:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 33 ms on localhost (1/2) -2016-05-25 14:50:03 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 14:50:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 36 ms on localhost (2/2) -2016-05-25 14:50:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 14:50:03 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.039 s -2016-05-25 14:50:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 14:50:03 INFO DAGScheduler:59 - running: Set() -2016-05-25 14:50:03 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 14:50:03 INFO DAGScheduler:59 - failed: Set() -2016-05-25 14:50:03 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 14:50:03 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[17] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 14:50:03 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=209398, maxMem=766075207 -2016-05-25 14:50:03 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-05-25 14:50:03 INFO MemoryStore:59 - ensureFreeSpace(2868) called with curMem=214630, maxMem=766075207 -2016-05-25 14:50:03 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-25 14:50:03 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:59238 (size: 2.8 KB, free: 730.6 MB) -2016-05-25 14:50:03 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:50:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[17] at map at DecisionTree.scala:645) -2016-05-25 14:50:03 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 14:50:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 14:50:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 14:50:03 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 14:50:03 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 14:50:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 14:50:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 14:50:03 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 14:50:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 14:50:03 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 1161 bytes result sent to driver -2016-05-25 14:50:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 12 ms on localhost (1/2) -2016-05-25 14:50:03 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 14:50:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 22 ms on localhost (2/2) -2016-05-25 14:50:03 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.023 s -2016-05-25 14:50:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 14:50:03 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.119202 s -2016-05-25 14:50:03 INFO MapPartitionsRDD:59 - Removing RDD 11 from persistence list -2016-05-25 14:50:03 INFO BlockManager:59 - Removing RDD 11 -2016-05-25 14:50:03 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 14:50:03 INFO RandomForest:59 - init: 8.208871782 - total: 8.964527418 - findSplitsBins: 7.816235722 - findBestSplits: 0.735857618 - chooseSplits: 0.728425497 -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 14:50:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 14:50:03 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 14:50:03 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 14:50:03 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 14:50:03 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 14:50:03 INFO BlockManager:59 - BlockManager stopped -2016-05-25 14:50:03 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 14:50:03 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 14:50:03 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 14:50:03 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 14:50:03 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-160b7720-54a5-42ac-9d5b-343f0a569b00 -2016-05-25 14:53:33 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 14:53:34 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 14:53:35 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 14:53:35 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 14:53:35 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 14:53:35 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 14:53:35 INFO Remoting:74 - Starting remoting -2016-05-25 14:53:36 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:59269] -2016-05-25 14:53:36 INFO Utils:59 - Successfully started service 'sparkDriver' on port 59269. -2016-05-25 14:53:36 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 14:53:36 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 14:53:36 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-4145c28d-d73e-4a50-a2f2-3f177a68b8bb -2016-05-25 14:53:36 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 14:53:36 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-ceb557ce-cf97-40f6-a50c-1647abcf2e14\httpd-2eef488e-3dfd-48c2-ab31-8664291c6b96 -2016-05-25 14:53:36 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 14:53:36 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 14:53:37 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:59270 -2016-05-25 14:53:37 INFO Utils:59 - Successfully started service 'HTTP file server' on port 59270. -2016-05-25 14:53:37 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 14:53:37 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 14:53:37 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 14:53:37 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 14:53:37 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 14:53:37 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 14:53:37 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 14:53:37 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 59289. -2016-05-25 14:53:37 INFO NettyBlockTransferService:59 - Server created on 59289 -2016-05-25 14:53:37 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 14:53:37 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:59289 with 730.6 MB RAM, BlockManagerId(driver, localhost, 59289) -2016-05-25 14:53:37 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 14:53:38 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 14:53:38 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 14:53:38 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 14:53:38 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 14:53:38 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:59289 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 14:53:38 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 14:53:39 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 14:53:40 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 14:53:40 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 14:53:40 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 14:53:40 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 14:53:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 14:53:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 14:53:40 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 14:53:40 INFO MemoryStore:59 - ensureFreeSpace(3888) called with curMem=171580, maxMem=766075207 -2016-05-25 14:53:40 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 14:53:40 INFO MemoryStore:59 - ensureFreeSpace(2133) called with curMem=175468, maxMem=766075207 -2016-05-25 14:53:40 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 14:53:40 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:59289 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 14:53:40 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:53:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 14:53:40 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 14:53:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 14:53:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 14:53:40 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 14:53:40 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 14:53:40 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 14:53:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/dt.txt:0+57 -2016-05-25 14:53:40 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 14:53:40 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 14:53:40 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 14:53:40 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 14:53:40 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 14:53:40 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 14:53:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/dt.txt:57+57 -2016-05-25 14:53:40 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=177601, maxMem=766075207 -2016-05-25 14:53:40 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 280.0 B, free 730.4 MB) -2016-05-25 14:53:40 INFO MemoryStore:59 - ensureFreeSpace(544) called with curMem=177881, maxMem=766075207 -2016-05-25 14:53:40 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 544.0 B, free 730.4 MB) -2016-05-25 14:53:40 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:59289 (size: 280.0 B, free: 730.6 MB) -2016-05-25 14:53:40 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:59289 (size: 544.0 B, free: 730.6 MB) -2016-05-25 14:53:40 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 14:53:40 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 14:53:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 121 ms on localhost (1/2) -2016-05-25 14:53:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 147 ms on localhost (2/2) -2016-05-25 14:53:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 14:53:40 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.165 s -2016-05-25 14:53:40 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.303143 s -2016-05-25 14:53:40 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 14:53:40 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 14:53:40 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 14:53:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 14:53:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 14:53:40 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[8] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 14:53:40 INFO MemoryStore:59 - ensureFreeSpace(4456) called with curMem=178425, maxMem=766075207 -2016-05-25 14:53:40 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-25 14:53:40 INFO MemoryStore:59 - ensureFreeSpace(2402) called with curMem=182881, maxMem=766075207 -2016-05-25 14:53:40 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-25 14:53:40 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:59289 (size: 2.3 KB, free: 730.6 MB) -2016-05-25 14:53:40 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:53:40 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[8] at map at DecisionTreeMetadata.scala:110) -2016-05-25 14:53:40 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 14:53:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 14:53:40 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 14:53:40 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 14:53:40 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 14:53:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 23 ms on localhost (1/1) -2016-05-25 14:53:40 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.024 s -2016-05-25 14:53:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 14:53:40 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.049108 s -2016-05-25 14:53:40 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 14:53:40 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 14:53:40 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 14:53:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 14:53:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 14:53:40 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 14:53:40 INFO MemoryStore:59 - ensureFreeSpace(4016) called with curMem=185283, maxMem=766075207 -2016-05-25 14:53:40 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-05-25 14:53:40 INFO MemoryStore:59 - ensureFreeSpace(2185) called with curMem=189299, maxMem=766075207 -2016-05-25 14:53:40 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 14:53:40 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:59289 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 14:53:40 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:53:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at retag at RandomForest.scala:137) -2016-05-25 14:53:40 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 14:53:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 14:53:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 14:53:40 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 14:53:40 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 14:53:40 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 14:53:40 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 14:53:40 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 14:53:40 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 14:53:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 13 ms on localhost (1/2) -2016-05-25 14:53:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 15 ms on localhost (2/2) -2016-05-25 14:53:40 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.015 s -2016-05-25 14:53:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 14:53:40 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.029769 s -2016-05-25 14:53:40 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 14:53:40 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 14:53:40 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 14:53:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 14:53:40 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 14:53:40 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[9] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 14:53:40 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=191484, maxMem=766075207 -2016-05-25 14:53:40 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.6 KB, free 730.4 MB) -2016-05-25 14:53:40 INFO MemoryStore:59 - ensureFreeSpace(2493) called with curMem=196228, maxMem=766075207 -2016-05-25 14:53:40 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-05-25 14:53:40 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:59289 (size: 2.4 KB, free: 730.6 MB) -2016-05-25 14:53:40 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:53:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[9] at sample at DecisionTree.scala:989) -2016-05-25 14:53:40 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 14:53:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 14:53:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 14:53:41 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 14:53:41 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 14:53:41 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 14:53:41 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 14:53:41 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2677 bytes result sent to driver -2016-05-25 14:53:41 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2493 bytes result sent to driver -2016-05-25 14:53:41 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:59289 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 14:53:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 28 ms on localhost (1/2) -2016-05-25 14:53:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 28 ms on localhost (2/2) -2016-05-25 14:53:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 14:53:41 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.038 s -2016-05-25 14:53:41 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.079122 s -2016-05-25 14:53:41 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:59289 in memory (size: 2.3 KB, free: 730.6 MB) -2016-05-25 14:53:41 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:59289 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 14:53:41 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=179641, maxMem=766075207 -2016-05-25 14:53:41 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.4 MB) -2016-05-25 14:53:41 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=179681, maxMem=766075207 -2016-05-25 14:53:41 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.4 MB) -2016-05-25 14:53:41 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:59289 (size: 81.0 B, free: 730.6 MB) -2016-05-25 14:53:41 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 14:53:42 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 14:53:42 INFO DAGScheduler:59 - Registering RDD 12 (mapPartitions at DecisionTree.scala:625) -2016-05-25 14:53:42 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 14:53:42 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 14:53:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 14:53:42 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 14:53:42 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[12] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 14:53:42 INFO MemoryStore:59 - ensureFreeSpace(9736) called with curMem=179762, maxMem=766075207 -2016-05-25 14:53:42 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 9.5 KB, free 730.4 MB) -2016-05-25 14:53:42 INFO MemoryStore:59 - ensureFreeSpace(4927) called with curMem=189498, maxMem=766075207 -2016-05-25 14:53:42 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-25 14:53:42 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:59289 (size: 4.8 KB, free: 730.6 MB) -2016-05-25 14:53:42 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:53:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[12] at mapPartitions at DecisionTree.scala:625) -2016-05-25 14:53:42 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 14:53:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 14:53:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 14:53:42 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 14:53:42 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 14:53:42 INFO CacheManager:59 - Partition rdd_11_0 not found, computing it -2016-05-25 14:53:42 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 14:53:42 INFO CacheManager:59 - Partition rdd_11_1 not found, computing it -2016-05-25 14:53:42 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 14:53:42 INFO MemoryStore:59 - ensureFreeSpace(448) called with curMem=194425, maxMem=766075207 -2016-05-25 14:53:42 INFO MemoryStore:59 - Block rdd_11_0 stored as values in memory (estimated size 448.0 B, free 730.4 MB) -2016-05-25 14:53:42 INFO MemoryStore:59 - ensureFreeSpace(232) called with curMem=194873, maxMem=766075207 -2016-05-25 14:53:42 INFO MemoryStore:59 - Block rdd_11_1 stored as values in memory (estimated size 232.0 B, free 730.4 MB) -2016-05-25 14:53:42 INFO BlockManagerInfo:59 - Added rdd_11_0 in memory on localhost:59289 (size: 448.0 B, free: 730.6 MB) -2016-05-25 14:53:42 INFO BlockManagerInfo:59 - Added rdd_11_1 in memory on localhost:59289 (size: 232.0 B, free: 730.6 MB) -2016-05-25 14:53:42 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 14:53:42 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 14:53:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 114 ms on localhost (1/2) -2016-05-25 14:53:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 111 ms on localhost (2/2) -2016-05-25 14:53:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 14:53:42 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.117 s -2016-05-25 14:53:42 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 14:53:42 INFO DAGScheduler:59 - running: Set() -2016-05-25 14:53:42 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 14:53:42 INFO DAGScheduler:59 - failed: Set() -2016-05-25 14:53:42 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 14:53:42 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[14] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 14:53:42 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=195105, maxMem=766075207 -2016-05-25 14:53:42 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-05-25 14:53:42 INFO MemoryStore:59 - ensureFreeSpace(2871) called with curMem=200337, maxMem=766075207 -2016-05-25 14:53:42 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-25 14:53:42 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:59289 (size: 2.8 KB, free: 730.6 MB) -2016-05-25 14:53:42 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:53:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[14] at map at DecisionTree.scala:645) -2016-05-25 14:53:42 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 14:53:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 14:53:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 14:53:42 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 14:53:42 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 14:53:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 14:53:42 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 14:53:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 14:53:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 14:53:42 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 14:53:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 40 ms on localhost (1/2) -2016-05-25 14:53:42 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 14:53:42 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.074 s -2016-05-25 14:53:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 73 ms on localhost (2/2) -2016-05-25 14:53:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 14:53:42 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.251899 s -2016-05-25 14:53:42 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=203208, maxMem=766075207 -2016-05-25 14:53:42 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 40.0 B, free 730.4 MB) -2016-05-25 14:53:42 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=203248, maxMem=766075207 -2016-05-25 14:53:42 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.4 MB) -2016-05-25 14:53:42 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:59289 (size: 81.0 B, free: 730.6 MB) -2016-05-25 14:53:42 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 14:53:42 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 14:53:42 INFO DAGScheduler:59 - Registering RDD 15 (mapPartitions at DecisionTree.scala:625) -2016-05-25 14:53:42 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 14:53:42 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 14:53:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 14:53:42 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 14:53:42 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[15] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 14:53:42 INFO MemoryStore:59 - ensureFreeSpace(10096) called with curMem=203329, maxMem=766075207 -2016-05-25 14:53:42 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 9.9 KB, free 730.4 MB) -2016-05-25 14:53:42 INFO MemoryStore:59 - ensureFreeSpace(5115) called with curMem=213425, maxMem=766075207 -2016-05-25 14:53:42 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 14:53:42 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:59289 (size: 5.0 KB, free: 730.6 MB) -2016-05-25 14:53:42 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:53:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[15] at mapPartitions at DecisionTree.scala:625) -2016-05-25 14:53:42 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 14:53:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 14:53:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 14:53:42 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 14:53:42 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 14:53:42 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:59289 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-25 14:53:42 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 14:53:42 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:59289 in memory (size: 4.8 KB, free: 730.6 MB) -2016-05-25 14:53:42 INFO BlockManager:59 - Found block rdd_11_0 locally -2016-05-25 14:53:42 INFO BlockManager:59 - Found block rdd_11_1 locally -2016-05-25 14:53:42 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 14:53:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 30 ms on localhost (1/2) -2016-05-25 14:53:42 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 14:53:42 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 14:53:42 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 14:53:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 45 ms on localhost (2/2) -2016-05-25 14:53:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 14:53:42 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.048 s -2016-05-25 14:53:42 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:59289 in memory (size: 81.0 B, free: 730.6 MB) -2016-05-25 14:53:42 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 14:53:42 INFO DAGScheduler:59 - running: Set() -2016-05-25 14:53:42 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 14:53:42 INFO DAGScheduler:59 - failed: Set() -2016-05-25 14:53:42 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 14:53:42 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[17] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 14:53:42 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:59289 in memory (size: 2.4 KB, free: 730.6 MB) -2016-05-25 14:53:42 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=188416, maxMem=766075207 -2016-05-25 14:53:42 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-05-25 14:53:42 INFO MemoryStore:59 - ensureFreeSpace(2868) called with curMem=193648, maxMem=766075207 -2016-05-25 14:53:42 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-25 14:53:42 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:59289 (size: 2.8 KB, free: 730.6 MB) -2016-05-25 14:53:42 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:53:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[17] at map at DecisionTree.scala:645) -2016-05-25 14:53:42 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 14:53:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 14:53:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 14:53:42 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 14:53:42 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 14:53:42 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 14:53:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 14:53:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 14:53:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 14:53:42 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 1161 bytes result sent to driver -2016-05-25 14:53:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 11 ms on localhost (1/2) -2016-05-25 14:53:42 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 14:53:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 23 ms on localhost (2/2) -2016-05-25 14:53:42 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.023 s -2016-05-25 14:53:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 14:53:42 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.138798 s -2016-05-25 14:53:42 INFO MapPartitionsRDD:59 - Removing RDD 11 from persistence list -2016-05-25 14:53:42 INFO BlockManager:59 - Removing RDD 11 -2016-05-25 14:53:42 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 14:53:42 INFO RandomForest:59 - init: 1.157776843 - total: 1.691904763 - findSplitsBins: 1.016938685 - findBestSplits: 0.51487919 - chooseSplits: 0.509209611 -2016-05-25 14:53:42 INFO SparkContext:59 - Starting job: take at DecisionTrees2Learning.scala:35 -2016-05-25 14:53:42 INFO DAGScheduler:59 - Got job 6 (take at DecisionTrees2Learning.scala:35) with 1 output partitions -2016-05-25 14:53:42 INFO DAGScheduler:59 - Final stage: ResultStage 8(take at DecisionTrees2Learning.scala:35) -2016-05-25 14:53:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 14:53:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 14:53:42 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 14:53:42 INFO MemoryStore:59 - ensureFreeSpace(3968) called with curMem=195836, maxMem=766075207 -2016-05-25 14:53:42 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-05-25 14:53:42 INFO MemoryStore:59 - ensureFreeSpace(2188) called with curMem=199804, maxMem=766075207 -2016-05-25 14:53:42 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 14:53:42 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:59289 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 14:53:42 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:53:42 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 14:53:42 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-05-25 14:53:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 14:53:42 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 14:53:42 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 14:53:42 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2493 bytes result sent to driver -2016-05-25 14:53:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 18 ms on localhost (1/1) -2016-05-25 14:53:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 14:53:42 INFO DAGScheduler:59 - ResultStage 8 (take at DecisionTrees2Learning.scala:35) finished in 0.020 s -2016-05-25 14:53:42 INFO DAGScheduler:59 - Job 6 finished: take at DecisionTrees2Learning.scala:35, took 0.046398 s -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 14:53:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 14:53:42 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 14:53:42 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 14:53:42 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 14:53:42 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 14:53:42 INFO BlockManager:59 - BlockManager stopped -2016-05-25 14:53:42 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 14:53:42 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 14:53:42 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 14:53:42 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 14:53:42 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-ceb557ce-cf97-40f6-a50c-1647abcf2e14 -2016-05-25 14:53:52 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 14:53:52 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 14:53:52 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 14:53:52 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 14:53:52 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 14:53:53 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 14:53:53 INFO Remoting:74 - Starting remoting -2016-05-25 14:53:53 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:59310] -2016-05-25 14:53:53 INFO Utils:59 - Successfully started service 'sparkDriver' on port 59310. -2016-05-25 14:53:54 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 14:53:54 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 14:53:54 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-e1e89431-1b64-48e6-b1f1-bfee1f46bf77 -2016-05-25 14:53:54 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 14:53:54 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-573de7b6-2219-49f3-bf4f-fa90ad33e901\httpd-cfb7fcd3-b3c7-42fa-9862-de3f91ba7bf1 -2016-05-25 14:53:54 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 14:53:54 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 14:53:54 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:59311 -2016-05-25 14:53:54 INFO Utils:59 - Successfully started service 'HTTP file server' on port 59311. -2016-05-25 14:53:54 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 14:53:54 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 14:53:54 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 14:53:54 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 14:53:54 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 14:53:54 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 14:53:54 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 14:53:54 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 59330. -2016-05-25 14:53:54 INFO NettyBlockTransferService:59 - Server created on 59330 -2016-05-25 14:53:54 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 14:53:54 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:59330 with 730.6 MB RAM, BlockManagerId(driver, localhost, 59330) -2016-05-25 14:53:54 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 14:53:55 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 14:53:55 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 14:53:55 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 14:53:55 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 14:53:55 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:59330 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 14:53:55 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 14:53:56 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 14:53:57 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 14:53:57 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 14:53:57 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 14:53:57 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 14:53:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 14:53:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 14:53:57 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 14:53:57 INFO MemoryStore:59 - ensureFreeSpace(3888) called with curMem=171580, maxMem=766075207 -2016-05-25 14:53:57 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 14:53:57 INFO MemoryStore:59 - ensureFreeSpace(2131) called with curMem=175468, maxMem=766075207 -2016-05-25 14:53:57 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 14:53:57 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:59330 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 14:53:57 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:53:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 14:53:57 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 14:53:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 14:53:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 14:53:57 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 14:53:57 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 14:53:57 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 14:53:57 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 14:53:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/dt.txt:57+57 -2016-05-25 14:53:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/dt.txt:0+57 -2016-05-25 14:53:57 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 14:53:57 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 14:53:57 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 14:53:57 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 14:53:57 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 14:53:57 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=177599, maxMem=766075207 -2016-05-25 14:53:57 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 280.0 B, free 730.4 MB) -2016-05-25 14:53:57 INFO MemoryStore:59 - ensureFreeSpace(544) called with curMem=177879, maxMem=766075207 -2016-05-25 14:53:57 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:59330 (size: 280.0 B, free: 730.6 MB) -2016-05-25 14:53:57 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 544.0 B, free 730.4 MB) -2016-05-25 14:53:57 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:59330 (size: 544.0 B, free: 730.6 MB) -2016-05-25 14:53:57 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 14:53:57 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 14:53:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 121 ms on localhost (1/2) -2016-05-25 14:53:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 145 ms on localhost (2/2) -2016-05-25 14:53:57 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.162 s -2016-05-25 14:53:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 14:53:57 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.319457 s -2016-05-25 14:53:57 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 14:53:57 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 14:53:57 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 14:53:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 14:53:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 14:53:57 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[8] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 14:53:57 INFO MemoryStore:59 - ensureFreeSpace(4456) called with curMem=178423, maxMem=766075207 -2016-05-25 14:53:57 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-25 14:53:57 INFO MemoryStore:59 - ensureFreeSpace(2401) called with curMem=182879, maxMem=766075207 -2016-05-25 14:53:57 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-25 14:53:57 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:59330 (size: 2.3 KB, free: 730.6 MB) -2016-05-25 14:53:57 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:53:57 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[8] at map at DecisionTreeMetadata.scala:110) -2016-05-25 14:53:57 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 14:53:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 14:53:57 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 14:53:57 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 14:53:57 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 14:53:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 20 ms on localhost (1/1) -2016-05-25 14:53:57 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.021 s -2016-05-25 14:53:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 14:53:57 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.040397 s -2016-05-25 14:53:57 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 14:53:57 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 14:53:57 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 14:53:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 14:53:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 14:53:57 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 14:53:57 INFO MemoryStore:59 - ensureFreeSpace(4016) called with curMem=185280, maxMem=766075207 -2016-05-25 14:53:57 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-05-25 14:53:57 INFO MemoryStore:59 - ensureFreeSpace(2184) called with curMem=189296, maxMem=766075207 -2016-05-25 14:53:57 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 14:53:57 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:59330 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 14:53:57 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:53:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at retag at RandomForest.scala:137) -2016-05-25 14:53:57 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 14:53:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 14:53:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 14:53:57 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 14:53:57 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 14:53:57 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 14:53:57 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 14:53:57 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 14:53:57 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 14:53:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 15 ms on localhost (1/2) -2016-05-25 14:53:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 17 ms on localhost (2/2) -2016-05-25 14:53:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 14:53:57 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.017 s -2016-05-25 14:53:57 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.035526 s -2016-05-25 14:53:57 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 14:53:57 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 14:53:57 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 14:53:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 14:53:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 14:53:57 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[9] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 14:53:57 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=191480, maxMem=766075207 -2016-05-25 14:53:57 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.6 KB, free 730.4 MB) -2016-05-25 14:53:57 INFO MemoryStore:59 - ensureFreeSpace(2492) called with curMem=196224, maxMem=766075207 -2016-05-25 14:53:57 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-05-25 14:53:57 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:59330 (size: 2.4 KB, free: 730.6 MB) -2016-05-25 14:53:57 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:53:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[9] at sample at DecisionTree.scala:989) -2016-05-25 14:53:57 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 14:53:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 14:53:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 14:53:57 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 14:53:57 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 14:53:57 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:59330 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 14:53:58 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 14:53:58 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 14:53:58 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2677 bytes result sent to driver -2016-05-25 14:53:58 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2493 bytes result sent to driver -2016-05-25 14:53:58 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:59330 in memory (size: 2.3 KB, free: 730.6 MB) -2016-05-25 14:53:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 27 ms on localhost (1/2) -2016-05-25 14:53:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 25 ms on localhost (2/2) -2016-05-25 14:53:58 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.028 s -2016-05-25 14:53:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 14:53:58 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.072295 s -2016-05-25 14:53:58 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:59330 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 14:53:58 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=179640, maxMem=766075207 -2016-05-25 14:53:58 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.4 MB) -2016-05-25 14:53:58 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=179680, maxMem=766075207 -2016-05-25 14:53:58 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.4 MB) -2016-05-25 14:53:58 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:59330 (size: 81.0 B, free: 730.6 MB) -2016-05-25 14:53:58 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 14:53:58 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 14:53:58 INFO DAGScheduler:59 - Registering RDD 12 (mapPartitions at DecisionTree.scala:625) -2016-05-25 14:53:58 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 14:53:58 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 14:53:58 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 14:53:58 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 14:53:58 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[12] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 14:53:58 INFO MemoryStore:59 - ensureFreeSpace(9736) called with curMem=179761, maxMem=766075207 -2016-05-25 14:53:58 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 9.5 KB, free 730.4 MB) -2016-05-25 14:53:58 INFO MemoryStore:59 - ensureFreeSpace(4926) called with curMem=189497, maxMem=766075207 -2016-05-25 14:53:58 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-25 14:53:58 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:59330 (size: 4.8 KB, free: 730.6 MB) -2016-05-25 14:53:58 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:53:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[12] at mapPartitions at DecisionTree.scala:625) -2016-05-25 14:53:58 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 14:53:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 14:53:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 14:53:58 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 14:53:58 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 14:53:58 INFO CacheManager:59 - Partition rdd_11_0 not found, computing it -2016-05-25 14:53:58 INFO CacheManager:59 - Partition rdd_11_1 not found, computing it -2016-05-25 14:53:58 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 14:53:58 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 14:53:58 INFO MemoryStore:59 - ensureFreeSpace(232) called with curMem=194423, maxMem=766075207 -2016-05-25 14:53:58 INFO MemoryStore:59 - Block rdd_11_1 stored as values in memory (estimated size 232.0 B, free 730.4 MB) -2016-05-25 14:53:58 INFO MemoryStore:59 - ensureFreeSpace(448) called with curMem=194655, maxMem=766075207 -2016-05-25 14:53:58 INFO MemoryStore:59 - Block rdd_11_0 stored as values in memory (estimated size 448.0 B, free 730.4 MB) -2016-05-25 14:53:58 INFO BlockManagerInfo:59 - Added rdd_11_1 in memory on localhost:59330 (size: 232.0 B, free: 730.6 MB) -2016-05-25 14:53:58 INFO BlockManagerInfo:59 - Added rdd_11_0 in memory on localhost:59330 (size: 448.0 B, free: 730.6 MB) -2016-05-25 14:53:58 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 14:53:58 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 14:53:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 90 ms on localhost (1/2) -2016-05-25 14:53:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 93 ms on localhost (2/2) -2016-05-25 14:53:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 14:53:58 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.094 s -2016-05-25 14:53:58 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 14:53:58 INFO DAGScheduler:59 - running: Set() -2016-05-25 14:53:58 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 14:53:58 INFO DAGScheduler:59 - failed: Set() -2016-05-25 14:53:58 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 14:53:58 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[14] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 14:53:58 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=195103, maxMem=766075207 -2016-05-25 14:53:58 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-05-25 14:53:58 INFO MemoryStore:59 - ensureFreeSpace(2871) called with curMem=200335, maxMem=766075207 -2016-05-25 14:53:58 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-25 14:53:59 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:59330 (size: 2.8 KB, free: 730.6 MB) -2016-05-25 14:53:59 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:53:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[14] at map at DecisionTree.scala:645) -2016-05-25 14:53:59 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 14:53:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 14:53:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 14:53:59 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 14:53:59 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 14:53:59 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 14:53:59 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 14:53:59 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 14:53:59 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 14:53:59 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 14:53:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 37 ms on localhost (1/2) -2016-05-25 14:53:59 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 14:53:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 69 ms on localhost (2/2) -2016-05-25 14:53:59 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.070 s -2016-05-25 14:53:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 14:53:59 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.225479 s -2016-05-25 14:53:59 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=203206, maxMem=766075207 -2016-05-25 14:53:59 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 40.0 B, free 730.4 MB) -2016-05-25 14:53:59 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=203246, maxMem=766075207 -2016-05-25 14:53:59 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.4 MB) -2016-05-25 14:53:59 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:59330 (size: 81.0 B, free: 730.6 MB) -2016-05-25 14:53:59 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 14:53:59 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 14:53:59 INFO DAGScheduler:59 - Registering RDD 15 (mapPartitions at DecisionTree.scala:625) -2016-05-25 14:53:59 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 14:53:59 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 14:53:59 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 14:53:59 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 14:53:59 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[15] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 14:53:59 INFO MemoryStore:59 - ensureFreeSpace(10096) called with curMem=203327, maxMem=766075207 -2016-05-25 14:53:59 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 9.9 KB, free 730.4 MB) -2016-05-25 14:53:59 INFO MemoryStore:59 - ensureFreeSpace(5114) called with curMem=213423, maxMem=766075207 -2016-05-25 14:53:59 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 14:53:59 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:59330 (size: 5.0 KB, free: 730.6 MB) -2016-05-25 14:53:59 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:53:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[15] at mapPartitions at DecisionTree.scala:625) -2016-05-25 14:53:59 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 14:53:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 14:53:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 14:53:59 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 14:53:59 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 14:53:59 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:59330 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-25 14:53:59 INFO BlockManager:59 - Found block rdd_11_0 locally -2016-05-25 14:53:59 INFO BlockManager:59 - Found block rdd_11_1 locally -2016-05-25 14:53:59 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 14:53:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 34 ms on localhost (1/2) -2016-05-25 14:53:59 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 14:53:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 38 ms on localhost (2/2) -2016-05-25 14:53:59 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.041 s -2016-05-25 14:53:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 14:53:59 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 14:53:59 INFO DAGScheduler:59 - running: Set() -2016-05-25 14:53:59 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 14:53:59 INFO DAGScheduler:59 - failed: Set() -2016-05-25 14:53:59 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 14:53:59 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[17] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 14:53:59 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 14:53:59 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=210434, maxMem=766075207 -2016-05-25 14:53:59 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-05-25 14:53:59 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:59330 in memory (size: 4.8 KB, free: 730.6 MB) -2016-05-25 14:53:59 INFO MemoryStore:59 - ensureFreeSpace(2868) called with curMem=201004, maxMem=766075207 -2016-05-25 14:53:59 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 14:53:59 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-25 14:53:59 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:59330 (size: 2.8 KB, free: 730.6 MB) -2016-05-25 14:53:59 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:53:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[17] at map at DecisionTree.scala:645) -2016-05-25 14:53:59 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 14:53:59 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 14:53:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 14:53:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 14:53:59 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 14:53:59 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 14:53:59 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:59330 in memory (size: 81.0 B, free: 730.6 MB) -2016-05-25 14:53:59 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 14:53:59 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 14:53:59 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 14:53:59 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 14:53:59 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 1161 bytes result sent to driver -2016-05-25 14:53:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 18 ms on localhost (1/2) -2016-05-25 14:53:59 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:59330 in memory (size: 2.4 KB, free: 730.6 MB) -2016-05-25 14:53:59 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 14:53:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 32 ms on localhost (2/2) -2016-05-25 14:53:59 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.032 s -2016-05-25 14:53:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 14:53:59 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.154843 s -2016-05-25 14:53:59 INFO MapPartitionsRDD:59 - Removing RDD 11 from persistence list -2016-05-25 14:53:59 INFO BlockManager:59 - Removing RDD 11 -2016-05-25 14:53:59 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 14:53:59 INFO RandomForest:59 - init: 0.948919785 - total: 1.486665598 - findSplitsBins: 0.815187121 - findBestSplits: 0.519113749 - chooseSplits: 0.510836056 -2016-05-25 14:53:59 INFO SparkContext:59 - Starting job: take at DecisionTrees2Learning.scala:37 -2016-05-25 14:53:59 INFO DAGScheduler:59 - Got job 6 (take at DecisionTrees2Learning.scala:37) with 1 output partitions -2016-05-25 14:53:59 INFO DAGScheduler:59 - Final stage: ResultStage 8(take at DecisionTrees2Learning.scala:37) -2016-05-25 14:53:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 14:53:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 14:53:59 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 14:53:59 INFO MemoryStore:59 - ensureFreeSpace(3968) called with curMem=195835, maxMem=766075207 -2016-05-25 14:53:59 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-05-25 14:53:59 INFO MemoryStore:59 - ensureFreeSpace(2186) called with curMem=199803, maxMem=766075207 -2016-05-25 14:53:59 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 14:53:59 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:59330 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 14:53:59 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 14:53:59 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 14:53:59 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-05-25 14:53:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 14:53:59 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 14:53:59 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 14:53:59 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2493 bytes result sent to driver -2016-05-25 14:53:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 10 ms on localhost (1/1) -2016-05-25 14:53:59 INFO DAGScheduler:59 - ResultStage 8 (take at DecisionTrees2Learning.scala:37) finished in 0.010 s -2016-05-25 14:53:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 14:53:59 INFO DAGScheduler:59 - Job 6 finished: take at DecisionTrees2Learning.scala:37, took 0.030537 s -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 14:53:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 14:53:59 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 14:53:59 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 14:53:59 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 14:53:59 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 14:53:59 INFO BlockManager:59 - BlockManager stopped -2016-05-25 14:53:59 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 14:53:59 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 14:53:59 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 14:53:59 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 14:53:59 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-573de7b6-2219-49f3-bf4f-fa90ad33e901 -2016-05-25 15:13:41 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 15:13:42 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 15:13:42 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 15:13:42 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 15:13:42 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 15:13:43 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 15:13:43 INFO Remoting:74 - Starting remoting -2016-05-25 15:13:43 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:52368] -2016-05-25 15:13:43 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52368. -2016-05-25 15:13:43 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 15:13:43 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 15:13:43 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-566ff2d0-8889-44c3-a1df-ae537b8d9bee -2016-05-25 15:13:43 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 15:13:43 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-0e9d0570-97bc-4a1b-9781-a3a36ce81418\httpd-aab2e99e-7a84-45fc-af31-77cb6dad6480 -2016-05-25 15:13:43 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 15:13:43 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:13:43 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52369 -2016-05-25 15:13:43 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52369. -2016-05-25 15:13:43 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 15:13:43 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:13:43 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 15:13:43 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 15:13:43 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 15:13:44 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 15:13:44 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 15:13:44 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52388. -2016-05-25 15:13:44 INFO NettyBlockTransferService:59 - Server created on 52388 -2016-05-25 15:13:44 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 15:13:44 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52388 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52388) -2016-05-25 15:13:44 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 15:13:45 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 15:13:45 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 15:13:45 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 15:13:45 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 15:13:45 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52388 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 15:13:45 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 15:13:46 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 15:13:47 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 15:13:47 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 15:13:47 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 15:13:47 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 15:13:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:13:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:13:47 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 15:13:47 INFO MemoryStore:59 - ensureFreeSpace(3888) called with curMem=171580, maxMem=766075207 -2016-05-25 15:13:47 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 15:13:47 INFO MemoryStore:59 - ensureFreeSpace(2131) called with curMem=175468, maxMem=766075207 -2016-05-25 15:13:47 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 15:13:47 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52388 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:13:47 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:13:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 15:13:47 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 15:13:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 15:13:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 15:13:47 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 15:13:47 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 15:13:47 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 15:13:47 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 15:13:47 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/dt.txt:57+57 -2016-05-25 15:13:47 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/dt.txt:0+57 -2016-05-25 15:13:47 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 15:13:47 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 15:13:47 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 15:13:47 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 15:13:47 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 15:13:47 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=177599, maxMem=766075207 -2016-05-25 15:13:47 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 280.0 B, free 730.4 MB) -2016-05-25 15:13:47 INFO MemoryStore:59 - ensureFreeSpace(544) called with curMem=177879, maxMem=766075207 -2016-05-25 15:13:47 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 544.0 B, free 730.4 MB) -2016-05-25 15:13:47 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:52388 (size: 280.0 B, free: 730.6 MB) -2016-05-25 15:13:47 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:52388 (size: 544.0 B, free: 730.6 MB) -2016-05-25 15:13:47 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 15:13:47 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 15:13:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 137 ms on localhost (1/2) -2016-05-25 15:13:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 120 ms on localhost (2/2) -2016-05-25 15:13:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 15:13:47 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.165 s -2016-05-25 15:13:47 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.302672 s -2016-05-25 15:13:47 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 15:13:47 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 15:13:47 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 15:13:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:13:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:13:47 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[8] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 15:13:47 INFO MemoryStore:59 - ensureFreeSpace(4456) called with curMem=178423, maxMem=766075207 -2016-05-25 15:13:47 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-25 15:13:47 INFO MemoryStore:59 - ensureFreeSpace(2401) called with curMem=182879, maxMem=766075207 -2016-05-25 15:13:47 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-25 15:13:47 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52388 (size: 2.3 KB, free: 730.6 MB) -2016-05-25 15:13:47 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:13:47 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[8] at map at DecisionTreeMetadata.scala:110) -2016-05-25 15:13:47 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 15:13:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 15:13:47 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 15:13:47 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:13:47 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 15:13:47 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.026 s -2016-05-25 15:13:47 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.045433 s -2016-05-25 15:13:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 26 ms on localhost (1/1) -2016-05-25 15:13:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 15:13:47 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 15:13:47 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 15:13:47 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 15:13:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:13:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:13:47 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 15:13:47 INFO MemoryStore:59 - ensureFreeSpace(4016) called with curMem=185280, maxMem=766075207 -2016-05-25 15:13:47 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-05-25 15:13:47 INFO MemoryStore:59 - ensureFreeSpace(2184) called with curMem=189296, maxMem=766075207 -2016-05-25 15:13:47 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 15:13:47 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52388 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:13:47 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:13:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at retag at RandomForest.scala:137) -2016-05-25 15:13:47 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 15:13:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 15:13:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 15:13:47 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 15:13:47 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 15:13:47 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:13:47 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 15:13:47 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:13:47 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 15:13:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 17 ms on localhost (1/2) -2016-05-25 15:13:47 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.018 s -2016-05-25 15:13:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 17 ms on localhost (2/2) -2016-05-25 15:13:47 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.039019 s -2016-05-25 15:13:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 15:13:47 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 15:13:47 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 15:13:47 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 15:13:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:13:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:13:47 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[9] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 15:13:47 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=191480, maxMem=766075207 -2016-05-25 15:13:47 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.6 KB, free 730.4 MB) -2016-05-25 15:13:47 INFO MemoryStore:59 - ensureFreeSpace(2492) called with curMem=196224, maxMem=766075207 -2016-05-25 15:13:47 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-05-25 15:13:47 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52388 (size: 2.4 KB, free: 730.6 MB) -2016-05-25 15:13:47 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:13:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[9] at sample at DecisionTree.scala:989) -2016-05-25 15:13:47 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 15:13:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 15:13:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 15:13:47 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 15:13:47 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 15:13:47 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:13:47 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:13:47 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2677 bytes result sent to driver -2016-05-25 15:13:47 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2493 bytes result sent to driver -2016-05-25 15:13:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 18 ms on localhost (1/2) -2016-05-25 15:13:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 17 ms on localhost (2/2) -2016-05-25 15:13:47 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.020 s -2016-05-25 15:13:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 15:13:47 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.035133 s -2016-05-25 15:13:48 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52388 in memory (size: 2.4 KB, free: 730.6 MB) -2016-05-25 15:13:48 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 15:13:48 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52388 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:13:48 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 15:13:48 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52388 in memory (size: 2.3 KB, free: 730.6 MB) -2016-05-25 15:13:48 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 15:13:48 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52388 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:13:48 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 15:13:48 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=172404, maxMem=766075207 -2016-05-25 15:13:48 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 200.0 B, free 730.4 MB) -2016-05-25 15:13:48 INFO MemoryStore:59 - ensureFreeSpace(313) called with curMem=172604, maxMem=766075207 -2016-05-25 15:13:48 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 313.0 B, free 730.4 MB) -2016-05-25 15:13:48 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52388 (size: 313.0 B, free: 730.6 MB) -2016-05-25 15:13:48 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 15:13:49 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:13:49 INFO DAGScheduler:59 - Registering RDD 12 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:13:49 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:13:49 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:13:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 15:13:49 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 15:13:49 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[12] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:13:49 INFO MemoryStore:59 - ensureFreeSpace(10272) called with curMem=172917, maxMem=766075207 -2016-05-25 15:13:49 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 10.0 KB, free 730.4 MB) -2016-05-25 15:13:49 INFO MemoryStore:59 - ensureFreeSpace(5151) called with curMem=183189, maxMem=766075207 -2016-05-25 15:13:49 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 15:13:49 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52388 (size: 5.0 KB, free: 730.6 MB) -2016-05-25 15:13:49 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:13:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[12] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:13:49 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 15:13:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 15:13:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 15:13:49 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 15:13:49 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 15:13:49 INFO CacheManager:59 - Partition rdd_11_1 not found, computing it -2016-05-25 15:13:49 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:13:49 INFO CacheManager:59 - Partition rdd_11_0 not found, computing it -2016-05-25 15:13:49 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:13:49 INFO MemoryStore:59 - ensureFreeSpace(264) called with curMem=188340, maxMem=766075207 -2016-05-25 15:13:49 INFO MemoryStore:59 - Block rdd_11_1 stored as values in memory (estimated size 264.0 B, free 730.4 MB) -2016-05-25 15:13:49 INFO MemoryStore:59 - ensureFreeSpace(512) called with curMem=188604, maxMem=766075207 -2016-05-25 15:13:49 INFO MemoryStore:59 - Block rdd_11_0 stored as values in memory (estimated size 512.0 B, free 730.4 MB) -2016-05-25 15:13:49 INFO BlockManagerInfo:59 - Added rdd_11_1 in memory on localhost:52388 (size: 264.0 B, free: 730.6 MB) -2016-05-25 15:13:49 INFO BlockManagerInfo:59 - Added rdd_11_0 in memory on localhost:52388 (size: 512.0 B, free: 730.6 MB) -2016-05-25 15:13:49 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 15:13:49 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 15:13:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 257 ms on localhost (1/2) -2016-05-25 15:13:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 256 ms on localhost (2/2) -2016-05-25 15:13:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 15:13:49 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.259 s -2016-05-25 15:13:49 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:13:49 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:13:49 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 15:13:49 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:13:49 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 15:13:49 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[14] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:13:49 INFO MemoryStore:59 - ensureFreeSpace(5352) called with curMem=189116, maxMem=766075207 -2016-05-25 15:13:49 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.4 MB) -2016-05-25 15:13:49 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=194468, maxMem=766075207 -2016-05-25 15:13:49 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-25 15:13:49 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52388 (size: 2.8 KB, free: 730.6 MB) -2016-05-25 15:13:49 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:13:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[14] at map at DecisionTree.scala:645) -2016-05-25 15:13:49 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 15:13:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:13:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:13:49 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 15:13:49 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 15:13:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:13:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:13:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-25 15:13:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-25 15:13:49 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2928 bytes result sent to driver -2016-05-25 15:13:49 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2774 bytes result sent to driver -2016-05-25 15:13:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 108 ms on localhost (1/2) -2016-05-25 15:13:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 106 ms on localhost (2/2) -2016-05-25 15:13:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 15:13:49 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.110 s -2016-05-25 15:13:49 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.447296 s -2016-05-25 15:13:49 INFO MemoryStore:59 - ensureFreeSpace(152) called with curMem=197364, maxMem=766075207 -2016-05-25 15:13:49 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 152.0 B, free 730.4 MB) -2016-05-25 15:13:49 INFO MemoryStore:59 - ensureFreeSpace(297) called with curMem=197516, maxMem=766075207 -2016-05-25 15:13:49 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 297.0 B, free 730.4 MB) -2016-05-25 15:13:49 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52388 (size: 297.0 B, free: 730.6 MB) -2016-05-25 15:13:49 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 15:13:49 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:13:49 INFO DAGScheduler:59 - Registering RDD 15 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:13:49 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:13:49 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:13:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 15:13:49 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 15:13:49 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[15] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:13:49 INFO MemoryStore:59 - ensureFreeSpace(10960) called with curMem=197813, maxMem=766075207 -2016-05-25 15:13:49 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 10.7 KB, free 730.4 MB) -2016-05-25 15:13:49 INFO MemoryStore:59 - ensureFreeSpace(5442) called with curMem=208773, maxMem=766075207 -2016-05-25 15:13:49 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 5.3 KB, free 730.4 MB) -2016-05-25 15:13:49 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52388 (size: 5.3 KB, free: 730.6 MB) -2016-05-25 15:13:49 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:13:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[15] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:13:49 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 15:13:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 15:13:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 15:13:49 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 15:13:49 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 15:13:49 INFO BlockManager:59 - Found block rdd_11_0 locally -2016-05-25 15:13:49 INFO BlockManager:59 - Found block rdd_11_1 locally -2016-05-25 15:13:49 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 15:13:49 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 15:13:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 39 ms on localhost (1/2) -2016-05-25 15:13:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 44 ms on localhost (2/2) -2016-05-25 15:13:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 15:13:49 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.045 s -2016-05-25 15:13:49 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:13:49 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:13:49 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 15:13:49 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:13:49 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 15:13:49 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[17] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:13:49 INFO MemoryStore:59 - ensureFreeSpace(5288) called with curMem=214215, maxMem=766075207 -2016-05-25 15:13:49 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.2 KB, free 730.4 MB) -2016-05-25 15:13:49 INFO MemoryStore:59 - ensureFreeSpace(2899) called with curMem=219503, maxMem=766075207 -2016-05-25 15:13:49 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-25 15:13:49 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:52388 (size: 2.8 KB, free: 730.6 MB) -2016-05-25 15:13:49 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:13:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[17] at map at DecisionTree.scala:645) -2016-05-25 15:13:49 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 15:13:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:13:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:13:49 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 15:13:49 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 15:13:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:13:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 15:13:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:13:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-25 15:13:49 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2774 bytes result sent to driver -2016-05-25 15:13:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 23 ms on localhost (1/2) -2016-05-25 15:13:49 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 15:13:49 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.031 s -2016-05-25 15:13:49 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.124012 s -2016-05-25 15:13:49 INFO MemoryStore:59 - ensureFreeSpace(104) called with curMem=222402, maxMem=766075207 -2016-05-25 15:13:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 30 ms on localhost (2/2) -2016-05-25 15:13:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 15:13:49 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 104.0 B, free 730.4 MB) -2016-05-25 15:13:49 INFO MemoryStore:59 - ensureFreeSpace(264) called with curMem=222506, maxMem=766075207 -2016-05-25 15:13:49 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 264.0 B, free 730.4 MB) -2016-05-25 15:13:49 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:52388 (size: 264.0 B, free: 730.6 MB) -2016-05-25 15:13:49 INFO SparkContext:59 - Created broadcast 11 from broadcast at DecisionTree.scala:604 -2016-05-25 15:13:49 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:13:49 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:13:49 INFO DAGScheduler:59 - Got job 6 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:13:49 INFO DAGScheduler:59 - Final stage: ResultStage 9(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:13:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 8) -2016-05-25 15:13:49 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 8) -2016-05-25 15:13:49 INFO DAGScheduler:59 - Submitting ShuffleMapStage 8 (MapPartitionsRDD[18] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:13:49 INFO MemoryStore:59 - ensureFreeSpace(11256) called with curMem=222770, maxMem=766075207 -2016-05-25 15:13:49 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 11.0 KB, free 730.4 MB) -2016-05-25 15:13:49 INFO MemoryStore:59 - ensureFreeSpace(5498) called with curMem=234026, maxMem=766075207 -2016-05-25 15:13:49 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 5.4 KB, free 730.4 MB) -2016-05-25 15:13:49 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:52388 (size: 5.4 KB, free: 730.5 MB) -2016-05-25 15:13:49 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:13:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 8 (MapPartitionsRDD[18] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:13:49 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 15:13:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 15:13:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 15:13:49 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 15:13:49 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 15:13:49 INFO BlockManager:59 - Found block rdd_11_0 locally -2016-05-25 15:13:49 INFO BlockManager:59 - Found block rdd_11_1 locally -2016-05-25 15:13:49 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2254 bytes result sent to driver -2016-05-25 15:13:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 38 ms on localhost (1/2) -2016-05-25 15:13:49 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2254 bytes result sent to driver -2016-05-25 15:13:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 47 ms on localhost (2/2) -2016-05-25 15:13:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 15:13:49 INFO DAGScheduler:59 - ShuffleMapStage 8 (mapPartitions at DecisionTree.scala:625) finished in 0.050 s -2016-05-25 15:13:49 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:13:49 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:13:49 INFO DAGScheduler:59 - waiting: Set(ResultStage 9) -2016-05-25 15:13:49 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:13:49 INFO DAGScheduler:59 - Missing parents for ResultStage 9: List() -2016-05-25 15:13:49 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[20] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:13:49 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=239524, maxMem=766075207 -2016-05-25 15:13:49 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-05-25 15:13:49 INFO MemoryStore:59 - ensureFreeSpace(2869) called with curMem=244756, maxMem=766075207 -2016-05-25 15:13:49 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-25 15:13:49 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:52388 (size: 2.8 KB, free: 730.5 MB) -2016-05-25 15:13:49 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:13:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[20] at map at DecisionTree.scala:645) -2016-05-25 15:13:49 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 15:13:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:13:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:13:49 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 15:13:49 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-25 15:13:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:13:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 15:13:49 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 15:13:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 15:13:49 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 1161 bytes result sent to driver -2016-05-25 15:13:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 17 ms on localhost (1/2) -2016-05-25 15:13:49 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2774 bytes result sent to driver -2016-05-25 15:13:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 28 ms on localhost (2/2) -2016-05-25 15:13:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 15:13:49 INFO DAGScheduler:59 - ResultStage 9 (collectAsMap at DecisionTree.scala:654) finished in 0.030 s -2016-05-25 15:13:49 INFO DAGScheduler:59 - Job 6 finished: collectAsMap at DecisionTree.scala:654, took 0.130370 s -2016-05-25 15:13:49 INFO MapPartitionsRDD:59 - Removing RDD 11 from persistence list -2016-05-25 15:13:49 INFO BlockManager:59 - Removing RDD 11 -2016-05-25 15:13:49 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 15:13:49 INFO RandomForest:59 - init: 1.374391757 - total: 2.345200066 - findSplitsBins: 1.120057579 - findBestSplits: 0.945666894 - chooseSplits: 0.935584269 -2016-05-25 15:13:49 INFO SparkContext:59 - Starting job: take at DecisionTrees3GBT.scala:37 -2016-05-25 15:13:49 INFO DAGScheduler:59 - Got job 7 (take at DecisionTrees3GBT.scala:37) with 1 output partitions -2016-05-25 15:13:49 INFO DAGScheduler:59 - Final stage: ResultStage 10(take at DecisionTrees3GBT.scala:37) -2016-05-25 15:13:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:13:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:13:49 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 15:13:49 INFO MemoryStore:59 - ensureFreeSpace(3968) called with curMem=246849, maxMem=766075207 -2016-05-25 15:13:49 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 3.9 KB, free 730.3 MB) -2016-05-25 15:13:49 INFO MemoryStore:59 - ensureFreeSpace(2186) called with curMem=250817, maxMem=766075207 -2016-05-25 15:13:49 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.3 MB) -2016-05-25 15:13:49 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:52388 (size: 2.1 KB, free: 730.5 MB) -2016-05-25 15:13:49 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:13:49 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 15:13:49 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-25 15:13:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 15:13:49 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 15:13:49 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:13:49 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2493 bytes result sent to driver -2016-05-25 15:13:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 13 ms on localhost (1/1) -2016-05-25 15:13:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 15:13:49 INFO DAGScheduler:59 - ResultStage 10 (take at DecisionTrees3GBT.scala:37) finished in 0.014 s -2016-05-25 15:13:49 INFO DAGScheduler:59 - Job 7 finished: take at DecisionTrees3GBT.scala:37, took 0.034614 s -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 15:13:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 15:13:50 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 15:13:50 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 15:13:50 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 15:13:50 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 15:13:50 INFO BlockManager:59 - BlockManager stopped -2016-05-25 15:13:50 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 15:13:50 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 15:13:50 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 15:13:50 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 15:13:50 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-0e9d0570-97bc-4a1b-9781-a3a36ce81418 -2016-05-25 15:13:50 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 15:21:45 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 15:21:46 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 15:21:46 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 15:21:46 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 15:21:46 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 15:21:47 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 15:21:47 INFO Remoting:74 - Starting remoting -2016-05-25 15:21:47 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:52648] -2016-05-25 15:21:47 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52648. -2016-05-25 15:21:47 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 15:21:47 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 15:21:47 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-bba74332-2b16-437b-b87a-33e1cb67769f -2016-05-25 15:21:47 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 15:21:47 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-7f0f9c89-db9f-4e05-943f-6428e534310b\httpd-cef40b4d-2077-4d7c-9759-74f2b85cf08a -2016-05-25 15:21:47 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 15:21:47 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:21:47 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52649 -2016-05-25 15:21:47 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52649. -2016-05-25 15:21:47 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 15:21:47 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:21:47 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 15:21:47 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 15:21:47 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 15:21:47 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 15:21:47 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 15:21:48 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52668. -2016-05-25 15:21:48 INFO NettyBlockTransferService:59 - Server created on 52668 -2016-05-25 15:21:48 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 15:21:48 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52668 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52668) -2016-05-25 15:21:48 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 15:21:49 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 15:21:49 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 15:21:49 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 15:21:49 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 15:21:49 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52668 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 15:21:49 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 15:21:50 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 15:21:51 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 15:21:51 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 15:21:51 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 15:21:51 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 15:21:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:21:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:21:51 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 15:21:51 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 15:21:51 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 15:21:51 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 15:21:51 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 15:21:51 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52668 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:21:51 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:21:51 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 15:21:51 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 15:21:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:21:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:21:51 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 15:21:51 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 15:21:51 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 15:21:51 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 15:21:51 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 15:21:51 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 15:21:51 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 15:21:51 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 15:21:51 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 15:21:51 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 15:21:51 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 15:21:51 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177630, maxMem=766075207 -2016-05-25 15:21:51 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 15:21:51 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262710, maxMem=766075207 -2016-05-25 15:21:51 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 15:21:51 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:52668 (size: 83.1 KB, free: 730.5 MB) -2016-05-25 15:21:51 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:52668 (size: 83.6 KB, free: 730.4 MB) -2016-05-25 15:21:51 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 15:21:51 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 15:21:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 456 ms on localhost (1/2) -2016-05-25 15:21:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 486 ms on localhost (2/2) -2016-05-25 15:21:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 15:21:51 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.502 s -2016-05-25 15:21:51 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.656024 s -2016-05-25 15:21:51 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 15:21:51 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 15:21:51 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 15:21:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:21:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:21:51 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 15:21:51 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348358, maxMem=766075207 -2016-05-25 15:21:51 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 15:21:52 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=353126, maxMem=766075207 -2016-05-25 15:21:52 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 15:21:52 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52668 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 15:21:52 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:21:52 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 15:21:52 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 15:21:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:21:52 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 15:21:52 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:21:52 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 15:21:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 44 ms on localhost (1/1) -2016-05-25 15:21:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 15:21:52 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.045 s -2016-05-25 15:21:52 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.064741 s -2016-05-25 15:21:52 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 15:21:52 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 15:21:52 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 15:21:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:21:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:21:52 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 15:21:52 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355673, maxMem=766075207 -2016-05-25 15:21:52 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 15:21:52 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=360009, maxMem=766075207 -2016-05-25 15:21:52 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 15:21:52 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52668 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 15:21:52 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:21:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 15:21:52 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 15:21:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:21:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:21:52 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 15:21:52 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 15:21:52 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:21:52 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 15:21:52 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:21:52 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 15:21:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 16 ms on localhost (1/2) -2016-05-25 15:21:52 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.020 s -2016-05-25 15:21:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 20 ms on localhost (2/2) -2016-05-25 15:21:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 15:21:52 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.034455 s -2016-05-25 15:21:52 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 15:21:52 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 15:21:52 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 15:21:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:21:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:21:52 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 15:21:52 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362328, maxMem=766075207 -2016-05-25 15:21:52 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 15:21:52 INFO MemoryStore:59 - ensureFreeSpace(2630) called with curMem=367392, maxMem=766075207 -2016-05-25 15:21:52 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 15:21:52 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52668 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:21:52 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:21:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 15:21:52 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 15:21:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 15:21:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 15:21:52 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 15:21:52 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 15:21:52 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:21:52 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:21:52 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 75045 bytes result sent to driver -2016-05-25 15:21:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 38 ms on localhost (1/2) -2016-05-25 15:21:52 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 64145 bytes result sent to driver -2016-05-25 15:21:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 46 ms on localhost (2/2) -2016-05-25 15:21:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 15:21:52 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.051 s -2016-05-25 15:21:52 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.083383 s -2016-05-25 15:21:53 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52668 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:21:53 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 15:21:53 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52668 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 15:21:53 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 15:21:53 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52668 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 15:21:53 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 15:21:53 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52668 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-25 15:21:53 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=342308, maxMem=766075207 -2016-05-25 15:21:53 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 15:21:53 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=342348, maxMem=766075207 -2016-05-25 15:21:53 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 15:21:53 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52668 (size: 81.0 B, free: 730.4 MB) -2016-05-25 15:21:53 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 15:21:54 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:21:54 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:21:54 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:21:54 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:21:54 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 15:21:54 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 15:21:54 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:21:54 INFO MemoryStore:59 - ensureFreeSpace(513360) called with curMem=342429, maxMem=766075207 -2016-05-25 15:21:54 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 501.3 KB, free 729.8 MB) -2016-05-25 15:21:54 INFO MemoryStore:59 - ensureFreeSpace(125973) called with curMem=855789, maxMem=766075207 -2016-05-25 15:21:54 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 123.0 KB, free 729.6 MB) -2016-05-25 15:21:54 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52668 (size: 123.0 KB, free: 730.3 MB) -2016-05-25 15:21:54 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:21:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:21:54 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 15:21:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:21:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:21:54 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 15:21:54 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 15:21:54 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 15:21:54 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:21:54 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 15:21:54 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:21:54 INFO MemoryStore:59 - ensureFreeSpace(102976) called with curMem=981762, maxMem=766075207 -2016-05-25 15:21:54 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 100.6 KB, free 729.6 MB) -2016-05-25 15:21:54 INFO MemoryStore:59 - ensureFreeSpace(120136) called with curMem=1084738, maxMem=766075207 -2016-05-25 15:21:54 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 117.3 KB, free 729.4 MB) -2016-05-25 15:21:54 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:52668 (size: 100.6 KB, free: 730.2 MB) -2016-05-25 15:21:54 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:52668 (size: 117.3 KB, free: 730.1 MB) -2016-05-25 15:21:54 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 15:21:54 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 15:21:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 289 ms on localhost (1/2) -2016-05-25 15:21:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 292 ms on localhost (2/2) -2016-05-25 15:21:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 15:21:54 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.294 s -2016-05-25 15:21:54 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:21:54 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:21:54 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 15:21:54 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:21:54 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 15:21:54 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:21:54 INFO MemoryStore:59 - ensureFreeSpace(224544) called with curMem=1204874, maxMem=766075207 -2016-05-25 15:21:54 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 219.3 KB, free 729.2 MB) -2016-05-25 15:21:54 INFO MemoryStore:59 - ensureFreeSpace(47445) called with curMem=1429418, maxMem=766075207 -2016-05-25 15:21:54 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 46.3 KB, free 729.2 MB) -2016-05-25 15:21:54 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52668 (size: 46.3 KB, free: 730.0 MB) -2016-05-25 15:21:54 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:21:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 15:21:54 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 15:21:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:21:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:21:54 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 15:21:54 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 15:21:54 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 15:21:54 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:21:54 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 8 ms -2016-05-25 15:21:54 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 8 ms -2016-05-25 15:21:54 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 15:21:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 168 ms on localhost (1/2) -2016-05-25 15:21:54 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 15:21:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 391 ms on localhost (2/2) -2016-05-25 15:21:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 15:21:54 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.392 s -2016-05-25 15:21:54 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.757738 s -2016-05-25 15:21:54 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=1476863, maxMem=766075207 -2016-05-25 15:21:54 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 40.0 B, free 729.2 MB) -2016-05-25 15:21:54 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=1476903, maxMem=766075207 -2016-05-25 15:21:54 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 81.0 B, free 729.2 MB) -2016-05-25 15:21:54 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52668 (size: 81.0 B, free: 730.0 MB) -2016-05-25 15:21:54 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 15:21:54 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:21:54 INFO DAGScheduler:59 - Registering RDD 17 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:21:54 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:21:54 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:21:54 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 15:21:54 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 15:21:54 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:21:54 INFO MemoryStore:59 - ensureFreeSpace(513712) called with curMem=1476984, maxMem=766075207 -2016-05-25 15:21:54 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 501.7 KB, free 728.7 MB) -2016-05-25 15:21:54 INFO MemoryStore:59 - ensureFreeSpace(126164) called with curMem=1990696, maxMem=766075207 -2016-05-25 15:21:54 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 123.2 KB, free 728.6 MB) -2016-05-25 15:21:54 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52668 (size: 123.2 KB, free: 729.9 MB) -2016-05-25 15:21:54 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:21:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:21:54 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 15:21:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:21:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:21:54 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 15:21:54 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 15:21:55 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-25 15:21:55 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-25 15:21:55 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 15:21:55 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 15:21:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 160 ms on localhost (1/2) -2016-05-25 15:21:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 165 ms on localhost (2/2) -2016-05-25 15:21:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 15:21:55 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.167 s -2016-05-25 15:21:55 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:21:55 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:21:55 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 15:21:55 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:21:55 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 15:21:55 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:21:55 INFO MemoryStore:59 - ensureFreeSpace(224544) called with curMem=2116860, maxMem=766075207 -2016-05-25 15:21:55 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 219.3 KB, free 728.4 MB) -2016-05-25 15:21:55 INFO MemoryStore:59 - ensureFreeSpace(47445) called with curMem=2341404, maxMem=766075207 -2016-05-25 15:21:55 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 46.3 KB, free 728.3 MB) -2016-05-25 15:21:55 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:52668 (size: 46.3 KB, free: 729.9 MB) -2016-05-25 15:21:55 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:21:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645) -2016-05-25 15:21:55 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 15:21:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:21:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:21:55 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 15:21:55 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 15:21:55 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:21:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 15:21:55 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 15:21:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-05-25 15:21:55 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 1161 bytes result sent to driver -2016-05-25 15:21:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 74 ms on localhost (1/2) -2016-05-25 15:21:55 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 15:21:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 113 ms on localhost (2/2) -2016-05-25 15:21:55 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.113 s -2016-05-25 15:21:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 15:21:55 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.336558 s -2016-05-25 15:21:55 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 15:21:55 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 15:21:55 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 15:21:55 INFO RandomForest:59 - init: 1.897380418 - total: 3.266381478 - findSplitsBins: 1.617901256 - findBestSplits: 1.348117852 - chooseSplits: 1.341356389 -2016-05-25 15:21:55 INFO SparkContext:59 - Starting job: count at DecisionTrees4Regression.scala:49 -2016-05-25 15:21:55 INFO DAGScheduler:59 - Got job 6 (count at DecisionTrees4Regression.scala:49) with 2 output partitions -2016-05-25 15:21:55 INFO DAGScheduler:59 - Final stage: ResultStage 8(count at DecisionTrees4Regression.scala:49) -2016-05-25 15:21:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:21:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:21:55 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 15:21:55 INFO MemoryStore:59 - ensureFreeSpace(3816) called with curMem=2165737, maxMem=766075207 -2016-05-25 15:21:55 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 3.7 KB, free 728.5 MB) -2016-05-25 15:21:55 INFO MemoryStore:59 - ensureFreeSpace(2142) called with curMem=2169553, maxMem=766075207 -2016-05-25 15:21:55 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.1 KB, free 728.5 MB) -2016-05-25 15:21:55 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:52668 (size: 2.1 KB, free: 730.1 MB) -2016-05-25 15:21:55 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:21:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 15:21:55 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 15:21:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:21:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:21:55 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 15:21:55 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 15:21:55 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:21:55 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:21:55 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 15:21:55 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 15:21:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 10 ms on localhost (1/2) -2016-05-25 15:21:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 10 ms on localhost (2/2) -2016-05-25 15:21:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 15:21:55 INFO DAGScheduler:59 - ResultStage 8 (count at DecisionTrees4Regression.scala:49) finished in 0.011 s -2016-05-25 15:21:55 INFO DAGScheduler:59 - Job 6 finished: count at DecisionTrees4Regression.scala:49, took 0.030169 s -2016-05-25 15:21:55 INFO SparkContext:59 - Starting job: count at DecisionTrees4Regression.scala:50 -2016-05-25 15:21:55 INFO DAGScheduler:59 - Got job 7 (count at DecisionTrees4Regression.scala:50) with 2 output partitions -2016-05-25 15:21:55 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at DecisionTrees4Regression.scala:50) -2016-05-25 15:21:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:21:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:21:55 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[7] at randomSplit at DecisionTrees4Regression.scala:28), which has no missing parents -2016-05-25 15:21:55 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2171695, maxMem=766075207 -2016-05-25 15:21:55 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.0 KB, free 728.5 MB) -2016-05-25 15:21:55 INFO MemoryStore:59 - ensureFreeSpace(2265) called with curMem=2175807, maxMem=766075207 -2016-05-25 15:21:55 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.5 MB) -2016-05-25 15:21:55 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:52668 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 15:21:55 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:21:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[7] at randomSplit at DecisionTrees4Regression.scala:28) -2016-05-25 15:21:55 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 15:21:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:21:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:21:55 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-25 15:21:55 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 15:21:55 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:21:55 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:21:55 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2082 bytes result sent to driver -2016-05-25 15:21:55 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2082 bytes result sent to driver -2016-05-25 15:21:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 10 ms on localhost (1/2) -2016-05-25 15:21:55 INFO DAGScheduler:59 - ResultStage 9 (count at DecisionTrees4Regression.scala:50) finished in 0.014 s -2016-05-25 15:21:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 12 ms on localhost (2/2) -2016-05-25 15:21:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 15:21:55 INFO DAGScheduler:59 - Job 7 finished: count at DecisionTrees4Regression.scala:50, took 0.031933 s -2016-05-25 15:21:55 INFO SparkContext:59 - Starting job: count at DecisionTrees4Regression.scala:51 -2016-05-25 15:21:55 INFO DAGScheduler:59 - Got job 8 (count at DecisionTrees4Regression.scala:51) with 2 output partitions -2016-05-25 15:21:55 INFO DAGScheduler:59 - Final stage: ResultStage 10(count at DecisionTrees4Regression.scala:51) -2016-05-25 15:21:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:21:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:21:55 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[8] at randomSplit at DecisionTrees4Regression.scala:28), which has no missing parents -2016-05-25 15:21:55 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2178072, maxMem=766075207 -2016-05-25 15:21:55 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 4.0 KB, free 728.5 MB) -2016-05-25 15:21:55 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=2182184, maxMem=766075207 -2016-05-25 15:21:55 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.5 MB) -2016-05-25 15:21:55 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:52668 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 15:21:55 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:21:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[8] at randomSplit at DecisionTrees4Regression.scala:28) -2016-05-25 15:21:55 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-25 15:21:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:21:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:21:55 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-25 15:21:55 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 15:21:55 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:21:55 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2082 bytes result sent to driver -2016-05-25 15:21:55 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:21:55 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2082 bytes result sent to driver -2016-05-25 15:21:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 8 ms on localhost (1/2) -2016-05-25 15:21:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 11 ms on localhost (2/2) -2016-05-25 15:21:55 INFO DAGScheduler:59 - ResultStage 10 (count at DecisionTrees4Regression.scala:51) finished in 0.011 s -2016-05-25 15:21:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 15:21:55 INFO DAGScheduler:59 - Job 8 finished: count at DecisionTrees4Regression.scala:51, took 0.027677 s -2016-05-25 15:21:55 INFO SparkContext:59 - Starting job: take at DecisionTrees4Regression.scala:57 -2016-05-25 15:21:55 INFO DAGScheduler:59 - Got job 9 (take at DecisionTrees4Regression.scala:57) with 1 output partitions -2016-05-25 15:21:55 INFO DAGScheduler:59 - Final stage: ResultStage 11(take at DecisionTrees4Regression.scala:57) -2016-05-25 15:21:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:21:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:21:55 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[20] at map at DecisionTrees4Regression.scala:42), which has no missing parents -2016-05-25 15:21:55 INFO MemoryStore:59 - ensureFreeSpace(6792) called with curMem=2184452, maxMem=766075207 -2016-05-25 15:21:55 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 6.6 KB, free 728.5 MB) -2016-05-25 15:21:55 INFO MemoryStore:59 - ensureFreeSpace(3634) called with curMem=2191244, maxMem=766075207 -2016-05-25 15:21:55 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.5 KB, free 728.5 MB) -2016-05-25 15:21:55 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:52668 (size: 3.5 KB, free: 730.1 MB) -2016-05-25 15:21:55 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:21:55 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 11 (MapPartitionsRDD[20] at map at DecisionTrees4Regression.scala:42) -2016-05-25 15:21:55 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 1 tasks -2016-05-25 15:21:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:21:55 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-25 15:21:55 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:21:55 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2353 bytes result sent to driver -2016-05-25 15:21:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 11 ms on localhost (1/1) -2016-05-25 15:21:55 INFO DAGScheduler:59 - ResultStage 11 (take at DecisionTrees4Regression.scala:57) finished in 0.011 s -2016-05-25 15:21:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 15:21:55 INFO DAGScheduler:59 - Job 9 finished: take at DecisionTrees4Regression.scala:57, took 0.025763 s -2016-05-25 15:21:55 INFO SparkContext:59 - Starting job: take at DecisionTrees4Regression.scala:57 -2016-05-25 15:21:55 INFO DAGScheduler:59 - Got job 10 (take at DecisionTrees4Regression.scala:57) with 1 output partitions -2016-05-25 15:21:55 INFO DAGScheduler:59 - Final stage: ResultStage 12(take at DecisionTrees4Regression.scala:57) -2016-05-25 15:21:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:21:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:21:55 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[20] at map at DecisionTrees4Regression.scala:42), which has no missing parents -2016-05-25 15:21:55 INFO MemoryStore:59 - ensureFreeSpace(6792) called with curMem=2194878, maxMem=766075207 -2016-05-25 15:21:55 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 6.6 KB, free 728.5 MB) -2016-05-25 15:21:55 INFO MemoryStore:59 - ensureFreeSpace(3634) called with curMem=2201670, maxMem=766075207 -2016-05-25 15:21:55 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.5 KB, free 728.5 MB) -2016-05-25 15:21:55 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:52668 (size: 3.5 KB, free: 730.1 MB) -2016-05-25 15:21:55 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:21:55 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[20] at map at DecisionTrees4Regression.scala:42) -2016-05-25 15:21:55 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-25 15:21:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 22, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:21:55 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 22) -2016-05-25 15:21:55 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:21:55 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 22). 2497 bytes result sent to driver -2016-05-25 15:21:55 INFO DAGScheduler:59 - ResultStage 12 (take at DecisionTrees4Regression.scala:57) finished in 0.010 s -2016-05-25 15:21:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 22) in 10 ms on localhost (1/1) -2016-05-25 15:21:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 15:21:55 INFO DAGScheduler:59 - Job 10 finished: take at DecisionTrees4Regression.scala:57, took 0.025022 s -2016-05-25 15:21:55 INFO SparkContext:59 - Starting job: mean at DecisionTrees4Regression.scala:59 -2016-05-25 15:21:55 INFO DAGScheduler:59 - Got job 11 (mean at DecisionTrees4Regression.scala:59) with 2 output partitions -2016-05-25 15:21:55 INFO DAGScheduler:59 - Final stage: ResultStage 13(mean at DecisionTrees4Regression.scala:59) -2016-05-25 15:21:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:21:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:21:55 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[22] at mean at DecisionTrees4Regression.scala:59), which has no missing parents -2016-05-25 15:21:55 INFO MemoryStore:59 - ensureFreeSpace(7368) called with curMem=2205304, maxMem=766075207 -2016-05-25 15:21:55 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 7.2 KB, free 728.5 MB) -2016-05-25 15:21:55 INFO MemoryStore:59 - ensureFreeSpace(3884) called with curMem=2212672, maxMem=766075207 -2016-05-25 15:21:55 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.8 KB, free 728.5 MB) -2016-05-25 15:21:55 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:52668 (size: 3.8 KB, free: 730.1 MB) -2016-05-25 15:21:55 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:21:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[22] at mean at DecisionTrees4Regression.scala:59) -2016-05-25 15:21:55 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-25 15:21:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 23, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:21:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 24, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:21:55 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 24) -2016-05-25 15:21:55 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 23) -2016-05-25 15:21:55 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:21:55 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:21:55 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 24). 2209 bytes result sent to driver -2016-05-25 15:21:55 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 23). 2209 bytes result sent to driver -2016-05-25 15:21:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 24) in 13 ms on localhost (1/2) -2016-05-25 15:21:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 23) in 14 ms on localhost (2/2) -2016-05-25 15:21:55 INFO DAGScheduler:59 - ResultStage 13 (mean at DecisionTrees4Regression.scala:59) finished in 0.015 s -2016-05-25 15:21:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 15:21:55 INFO DAGScheduler:59 - Job 11 finished: mean at DecisionTrees4Regression.scala:59, took 0.038909 s -2016-05-25 15:21:56 INFO SparkContext:59 - Starting job: saveAsTextFile at DecisionTreeModel.scala:235 -2016-05-25 15:21:56 INFO DAGScheduler:59 - Got job 12 (saveAsTextFile at DecisionTreeModel.scala:235) with 1 output partitions -2016-05-25 15:21:56 INFO DAGScheduler:59 - Final stage: ResultStage 14(saveAsTextFile at DecisionTreeModel.scala:235) -2016-05-25 15:21:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:21:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:21:56 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[24] at saveAsTextFile at DecisionTreeModel.scala:235), which has no missing parents -2016-05-25 15:21:56 INFO MemoryStore:59 - ensureFreeSpace(127000) called with curMem=2216556, maxMem=766075207 -2016-05-25 15:21:56 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 124.0 KB, free 728.4 MB) -2016-05-25 15:21:56 INFO MemoryStore:59 - ensureFreeSpace(42614) called with curMem=2343556, maxMem=766075207 -2016-05-25 15:21:56 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 41.6 KB, free 728.3 MB) -2016-05-25 15:21:56 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:52668 (size: 41.6 KB, free: 730.0 MB) -2016-05-25 15:21:56 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:21:56 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 14 (MapPartitionsRDD[24] at saveAsTextFile at DecisionTreeModel.scala:235) -2016-05-25 15:21:56 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 1 tasks -2016-05-25 15:21:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 25, localhost, PROCESS_LOCAL, 2196 bytes) -2016-05-25 15:21:56 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 25) -2016-05-25 15:21:57 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251521_0014_m_000000_25' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525152155496/result/metadata/_temporary/0/task_201605251521_0014_m_000000 -2016-05-25 15:21:57 INFO SparkHadoopMapRedUtil:59 - attempt_201605251521_0014_m_000000_25: Committed -2016-05-25 15:21:57 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 25). 1864 bytes result sent to driver -2016-05-25 15:21:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 25) in 365 ms on localhost (1/1) -2016-05-25 15:21:57 INFO DAGScheduler:59 - ResultStage 14 (saveAsTextFile at DecisionTreeModel.scala:235) finished in 0.365 s -2016-05-25 15:21:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 15:21:57 INFO DAGScheduler:59 - Job 12 finished: saveAsTextFile at DecisionTreeModel.scala:235, took 0.433057 s -2016-05-25 15:21:57 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:52668 in memory (size: 3.5 KB, free: 730.0 MB) -2016-05-25 15:21:57 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:52668 in memory (size: 41.6 KB, free: 730.1 MB) -2016-05-25 15:21:57 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-25 15:21:57 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:52668 in memory (size: 3.8 KB, free: 730.1 MB) -2016-05-25 15:21:57 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-25 15:21:57 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-25 15:21:57 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:52668 in memory (size: 3.5 KB, free: 730.1 MB) -2016-05-25 15:21:57 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-25 15:21:57 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:52668 in memory (size: 2.2 KB, free: 730.1 MB) -2016-05-25 15:21:57 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 15:21:57 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:52668 in memory (size: 2.2 KB, free: 730.1 MB) -2016-05-25 15:21:57 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 15:21:57 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:52668 in memory (size: 2.1 KB, free: 730.1 MB) -2016-05-25 15:21:57 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 15:21:57 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:52668 in memory (size: 46.3 KB, free: 730.1 MB) -2016-05-25 15:21:57 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 15:21:57 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:52668 in memory (size: 123.2 KB, free: 730.2 MB) -2016-05-25 15:21:57 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 15:21:57 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-25 15:21:57 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:52668 in memory (size: 81.0 B, free: 730.2 MB) -2016-05-25 15:21:57 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:52668 in memory (size: 46.3 KB, free: 730.3 MB) -2016-05-25 15:21:57 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 15:21:57 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:52668 in memory (size: 123.0 KB, free: 730.4 MB) -2016-05-25 15:21:57 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 15:21:57 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 15:21:57 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52668 in memory (size: 81.0 B, free: 730.4 MB) -2016-05-25 15:21:57 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 15:21:57 INFO ContextCleaner:59 - Cleaned RDD 13 -2016-05-25 15:22:01 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 15:22:01 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 15:22:01 INFO SparkContext:59 - Starting job: parquet at DecisionTreeModel.scala:242 -2016-05-25 15:22:01 INFO DAGScheduler:59 - Got job 13 (parquet at DecisionTreeModel.scala:242) with 4 output partitions -2016-05-25 15:22:01 INFO DAGScheduler:59 - Final stage: ResultStage 15(parquet at DecisionTreeModel.scala:242) -2016-05-25 15:22:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:22:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:22:01 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[27] at rddToDataFrameHolder at DecisionTreeModel.scala:240), which has no missing parents -2016-05-25 15:22:01 INFO MemoryStore:59 - ensureFreeSpace(68760) called with curMem=342308, maxMem=766075207 -2016-05-25 15:22:01 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 67.1 KB, free 730.2 MB) -2016-05-25 15:22:01 INFO MemoryStore:59 - ensureFreeSpace(24138) called with curMem=411068, maxMem=766075207 -2016-05-25 15:22:01 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 23.6 KB, free 730.2 MB) -2016-05-25 15:22:01 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:52668 (size: 23.6 KB, free: 730.4 MB) -2016-05-25 15:22:01 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:22:01 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 15 (MapPartitionsRDD[27] at rddToDataFrameHolder at DecisionTreeModel.scala:240) -2016-05-25 15:22:01 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 4 tasks -2016-05-25 15:22:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 26, localhost, PROCESS_LOCAL, 4046 bytes) -2016-05-25 15:22:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 27, localhost, PROCESS_LOCAL, 3827 bytes) -2016-05-25 15:22:01 INFO TaskSetManager:59 - Starting task 2.0 in stage 15.0 (TID 28, localhost, PROCESS_LOCAL, 2428 bytes) -2016-05-25 15:22:01 INFO TaskSetManager:59 - Starting task 3.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2489 bytes) -2016-05-25 15:22:01 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 26) -2016-05-25 15:22:01 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 27) -2016-05-25 15:22:01 INFO Executor:59 - Running task 2.0 in stage 15.0 (TID 28) -2016-05-25 15:22:01 INFO Executor:59 - Running task 3.0 in stage 15.0 (TID 29) -2016-05-25 15:22:01 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 15:22:01 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 15:22:01 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 15:22:01 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 15:22:01 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 15:22:01 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 15:22:01 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 15:22:01 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 15:22:01 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 15:22:01 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 15:22:01 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 15:22:01 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 15:22:01 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 15:22:01 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,676 -2016-05-25 15:22:01 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,676 -2016-05-25 15:22:01 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,644 -2016-05-25 15:22:01 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,620 -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 61B for [nodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 61B for [nodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 61B for [nodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 74B for [predict, predict] DOUBLE: 1 values, 15B raw, 33B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 69B for [predict, predict] DOUBLE: 1 values, 15B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 74B for [predict, predict] DOUBLE: 1 values, 15B raw, 33B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, prob] DOUBLE: 1 values, 15B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, prob] DOUBLE: 1 values, 15B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, prob] DOUBLE: 1 values, 15B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 75B for [impurity] DOUBLE: 1 values, 14B raw, 34B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 69B for [impurity] DOUBLE: 1 values, 14B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 75B for [impurity] DOUBLE: 1 values, 14B raw, 34B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 62B for [split, feature] INT32: 1 values, 11B raw, 29B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, feature] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 72B for [split, threshold] DOUBLE: 1 values, 15B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 61B for [split, featureType] INT32: 1 values, 11B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, threshold] DOUBLE: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 64B for [split, feature] INT32: 1 values, 11B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 52B for [split, categories, bag, array] DOUBLE: 1 values, 14B raw, 31B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, featureType] INT32: 1 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 69B for [split, threshold] DOUBLE: 1 values, 15B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 61B for [split, featureType] INT32: 1 values, 11B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 51B for [split, categories, bag, array] DOUBLE: 1 values, 14B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 52B for [split, categories, bag, array] DOUBLE: 1 values, 14B raw, 31B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 47B for [leftNodeId] INT32: 1 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 61B for [leftNodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 47B for [rightNodeId] INT32: 1 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 2 values, 8B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN_DICTIONARY, RLE], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 61B for [leftNodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 47B for [infoGain] DOUBLE: 1 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 61B for [rightNodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 61B for [rightNodeId] INT32: 1 values, 10B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 63B for [nodeId] INT32: 2 values, 14B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 75B for [infoGain] DOUBLE: 1 values, 14B raw, 34B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 75B for [infoGain] DOUBLE: 1 values, 14B raw, 34B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 68B for [predict, predict] DOUBLE: 2 values, 9B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN_DICTIONARY, RLE], dic { 1 entries, 8B raw, 1B comp} -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 68B for [predict, prob] DOUBLE: 2 values, 9B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN_DICTIONARY, RLE], dic { 1 entries, 8B raw, 1B comp} -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 69B for [impurity] DOUBLE: 2 values, 8B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN_DICTIONARY, RLE], dic { 1 entries, 8B raw, 1B comp} -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 2 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, feature] INT32: 2 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, threshold] DOUBLE: 2 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 48B for [split, featureType] INT32: 2 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 51B for [split, categories, bag, array] DOUBLE: 2 values, 14B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 47B for [leftNodeId] INT32: 2 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 47B for [rightNodeId] INT32: 2 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:02 INFO ColumnChunkPageWriteStore:151 - written 47B for [infoGain] DOUBLE: 2 values, 6B raw, 26B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-05-25 15:22:03 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251522_0015_m_000003_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525152155496/result/data/_temporary/0/task_201605251522_0015_m_000003 -2016-05-25 15:22:03 INFO SparkHadoopMapRedUtil:59 - attempt_201605251522_0015_m_000003_0: Committed -2016-05-25 15:22:03 INFO Executor:59 - Finished task 3.0 in stage 15.0 (TID 29). 915 bytes result sent to driver -2016-05-25 15:22:03 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251522_0015_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525152155496/result/data/_temporary/0/task_201605251522_0015_m_000000 -2016-05-25 15:22:03 INFO SparkHadoopMapRedUtil:59 - attempt_201605251522_0015_m_000000_0: Committed -2016-05-25 15:22:03 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 26). 915 bytes result sent to driver -2016-05-25 15:22:03 INFO TaskSetManager:59 - Finished task 3.0 in stage 15.0 (TID 29) in 1707 ms on localhost (1/4) -2016-05-25 15:22:03 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251522_0015_m_000001_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525152155496/result/data/_temporary/0/task_201605251522_0015_m_000001 -2016-05-25 15:22:03 INFO SparkHadoopMapRedUtil:59 - attempt_201605251522_0015_m_000001_0: Committed -2016-05-25 15:22:03 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 27). 915 bytes result sent to driver -2016-05-25 15:22:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 26) in 1713 ms on localhost (2/4) -2016-05-25 15:22:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 27) in 1713 ms on localhost (3/4) -2016-05-25 15:22:03 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251522_0015_m_000002_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525152155496/result/data/_temporary/0/task_201605251522_0015_m_000002 -2016-05-25 15:22:03 INFO SparkHadoopMapRedUtil:59 - attempt_201605251522_0015_m_000002_0: Committed -2016-05-25 15:22:03 INFO Executor:59 - Finished task 2.0 in stage 15.0 (TID 28). 915 bytes result sent to driver -2016-05-25 15:22:03 INFO TaskSetManager:59 - Finished task 2.0 in stage 15.0 (TID 28) in 1718 ms on localhost (4/4) -2016-05-25 15:22:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-25 15:22:03 INFO DAGScheduler:59 - ResultStage 15 (parquet at DecisionTreeModel.scala:242) finished in 1.720 s -2016-05-25 15:22:03 INFO DAGScheduler:59 - Job 13 finished: parquet at DecisionTreeModel.scala:242, took 1.786487 s -2016-05-25 15:22:03 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 15:22:03 INFO DefaultWriterContainer:59 - Job job_201605251522_0000 committed. -2016-05-25 15:22:03 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525152155496/result/data on driver -2016-05-25 15:22:03 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525152155496/result/data on driver -2016-05-25 15:22:03 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=435206, maxMem=766075207 -2016-05-25 15:22:03 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 225.3 KB, free 730.0 MB) -2016-05-25 15:22:03 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=665910, maxMem=766075207 -2016-05-25 15:22:03 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-25 15:22:03 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:52668 (size: 19.3 KB, free: 730.4 MB) -2016-05-25 15:22:03 INFO SparkContext:59 - Created broadcast 19 from textFile at modelSaveLoad.scala:129 -2016-05-25 15:22:03 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 15:22:03 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 15:22:03 INFO DAGScheduler:59 - Got job 14 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 15:22:03 INFO DAGScheduler:59 - Final stage: ResultStage 16(first at modelSaveLoad.scala:129) -2016-05-25 15:22:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:22:03 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:22:03 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[30] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 15:22:03 INFO MemoryStore:59 - ensureFreeSpace(3224) called with curMem=685698, maxMem=766075207 -2016-05-25 15:22:03 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.1 KB, free 729.9 MB) -2016-05-25 15:22:03 INFO MemoryStore:59 - ensureFreeSpace(1897) called with curMem=688922, maxMem=766075207 -2016-05-25 15:22:03 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 1897.0 B, free 729.9 MB) -2016-05-25 15:22:03 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:52668 (size: 1897.0 B, free: 730.4 MB) -2016-05-25 15:22:03 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:22:03 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 16 (MapPartitionsRDD[30] at textFile at modelSaveLoad.scala:129) -2016-05-25 15:22:03 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 1 tasks -2016-05-25 15:22:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 30, localhost, PROCESS_LOCAL, 2245 bytes) -2016-05-25 15:22:03 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 30) -2016-05-25 15:22:03 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525152155496/result/metadata/part-00000:0+53 -2016-05-25 15:22:03 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 30). 2153 bytes result sent to driver -2016-05-25 15:22:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 30) in 10 ms on localhost (1/1) -2016-05-25 15:22:03 INFO DAGScheduler:59 - ResultStage 16 (first at modelSaveLoad.scala:129) finished in 0.011 s -2016-05-25 15:22:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-25 15:22:03 INFO DAGScheduler:59 - Job 14 finished: first at modelSaveLoad.scala:129, took 0.021102 s -2016-05-25 15:22:04 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525152155496/result/data on driver -2016-05-25 15:22:04 INFO SparkContext:59 - Starting job: parquet at DecisionTreeModel.scala:249 -2016-05-25 15:22:04 INFO DAGScheduler:59 - Got job 15 (parquet at DecisionTreeModel.scala:249) with 4 output partitions -2016-05-25 15:22:04 INFO DAGScheduler:59 - Final stage: ResultStage 17(parquet at DecisionTreeModel.scala:249) -2016-05-25 15:22:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:22:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:22:04 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[32] at parquet at DecisionTreeModel.scala:249), which has no missing parents -2016-05-25 15:22:04 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=690819, maxMem=766075207 -2016-05-25 15:22:04 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 61.5 KB, free 729.9 MB) -2016-05-25 15:22:04 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=753819, maxMem=766075207 -2016-05-25 15:22:04 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 20.6 KB, free 729.8 MB) -2016-05-25 15:22:04 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:52668 (size: 20.6 KB, free: 730.3 MB) -2016-05-25 15:22:04 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:22:04 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (MapPartitionsRDD[32] at parquet at DecisionTreeModel.scala:249) -2016-05-25 15:22:04 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-25 15:22:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 31, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 15:22:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 32, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 15:22:04 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 15:22:04 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2286 bytes) -2016-05-25 15:22:04 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 31) -2016-05-25 15:22:04 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 32) -2016-05-25 15:22:04 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 33) -2016-05-25 15:22:04 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 34) -2016-05-25 15:22:04 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 15:22:04 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 31). 936 bytes result sent to driver -2016-05-25 15:22:04 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 15:22:04 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 32). 936 bytes result sent to driver -2016-05-25 15:22:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 31) in 56 ms on localhost (1/4) -2016-05-25 15:22:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 32) in 56 ms on localhost (2/4) -2016-05-25 15:22:04 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 15:22:04 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 15:22:04 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 33). 936 bytes result sent to driver -2016-05-25 15:22:04 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 33) in 65 ms on localhost (3/4) -2016-05-25 15:22:04 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 34). 2321 bytes result sent to driver -2016-05-25 15:22:04 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 34) in 103 ms on localhost (4/4) -2016-05-25 15:22:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-25 15:22:04 INFO DAGScheduler:59 - ResultStage 17 (parquet at DecisionTreeModel.scala:249) finished in 0.105 s -2016-05-25 15:22:04 INFO DAGScheduler:59 - Job 15 finished: parquet at DecisionTreeModel.scala:249, took 0.137092 s -2016-05-25 15:22:04 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=774892, maxMem=766075207 -2016-05-25 15:22:04 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 86.4 KB, free 729.8 MB) -2016-05-25 15:22:04 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:52668 in memory (size: 20.6 KB, free: 730.4 MB) -2016-05-25 15:22:04 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-25 15:22:04 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:52668 in memory (size: 1897.0 B, free: 730.4 MB) -2016-05-25 15:22:04 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=774210, maxMem=766075207 -2016-05-25 15:22:04 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-25 15:22:04 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.8 MB) -2016-05-25 15:22:04 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:52668 (size: 19.3 KB, free: 730.3 MB) -2016-05-25 15:22:04 INFO SparkContext:59 - Created broadcast 22 from map at DecisionTreeModel.scala:252 -2016-05-25 15:22:04 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:52668 in memory (size: 19.3 KB, free: 730.4 MB) -2016-05-25 15:22:04 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:52668 in memory (size: 23.6 KB, free: 730.4 MB) -2016-05-25 15:22:04 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-25 15:22:05 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 15:22:05 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525152155496/result/data/part-r-00003-eee96a6e-765e-48c1-b8d3-d3505abd0c56.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525152155496/result/data/part-r-00000-eee96a6e-765e-48c1-b8d3-d3505abd0c56.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525152155496/result/data/part-r-00002-eee96a6e-765e-48c1-b8d3-d3505abd0c56.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525152155496/result/data/part-r-00001-eee96a6e-765e-48c1-b8d3-d3505abd0c56.gz.parquet -2016-05-25 15:22:05 INFO SparkContext:59 - Starting job: collect at DecisionTreeModel.scala:267 -2016-05-25 15:22:05 INFO DAGScheduler:59 - Registering RDD 36 (groupBy at DecisionTreeModel.scala:265) -2016-05-25 15:22:05 INFO DAGScheduler:59 - Got job 16 (collect at DecisionTreeModel.scala:267) with 4 output partitions -2016-05-25 15:22:05 INFO DAGScheduler:59 - Final stage: ResultStage 19(collect at DecisionTreeModel.scala:267) -2016-05-25 15:22:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 18) -2016-05-25 15:22:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 18) -2016-05-25 15:22:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 18 (MapPartitionsRDD[36] at groupBy at DecisionTreeModel.scala:265), which has no missing parents -2016-05-25 15:22:05 INFO MemoryStore:59 - ensureFreeSpace(7248) called with curMem=450608, maxMem=766075207 -2016-05-25 15:22:05 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 7.1 KB, free 730.1 MB) -2016-05-25 15:22:05 INFO MemoryStore:59 - ensureFreeSpace(3789) called with curMem=457856, maxMem=766075207 -2016-05-25 15:22:05 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.7 KB, free 730.1 MB) -2016-05-25 15:22:05 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:52668 (size: 3.7 KB, free: 730.4 MB) -2016-05-25 15:22:05 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:22:05 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 18 (MapPartitionsRDD[36] at groupBy at DecisionTreeModel.scala:265) -2016-05-25 15:22:05 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-25 15:22:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2291 bytes) -2016-05-25 15:22:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2291 bytes) -2016-05-25 15:22:05 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 37, localhost, PROCESS_LOCAL, 2291 bytes) -2016-05-25 15:22:05 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 38, localhost, PROCESS_LOCAL, 2292 bytes) -2016-05-25 15:22:05 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 37) -2016-05-25 15:22:05 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-05-25 15:22:05 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-05-25 15:22:05 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 38) -2016-05-25 15:22:05 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 15:22:05 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525152155496/result/data/part-r-00001-eee96a6e-765e-48c1-b8d3-d3505abd0c56.gz.parquet start: 0 end: 3045 length: 3045 hosts: []} -2016-05-25 15:22:05 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525152155496/result/data/part-r-00000-eee96a6e-765e-48c1-b8d3-d3505abd0c56.gz.parquet start: 0 end: 3045 length: 3045 hosts: []} -2016-05-25 15:22:05 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525152155496/result/data/part-r-00003-eee96a6e-765e-48c1-b8d3-d3505abd0c56.gz.parquet start: 0 end: 2987 length: 2987 hosts: []} -2016-05-25 15:22:05 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/DecisionTreesLearning20160525152155496/result/data/part-r-00002-eee96a6e-765e-48c1-b8d3-d3505abd0c56.gz.parquet start: 0 end: 2836 length: 2836 hosts: []} -2016-05-25 15:22:05 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 15:22:05 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 15:22:05 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 15:22:05 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 15:22:06 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 treeId; - optional int32 nodeId; - optional group predict { - optional double predict; - optional double prob; - } - optional double impurity; - optional boolean isLeaf; - optional group split { - optional int32 feature; - optional double threshold; - optional int32 featureType; - optional group categories (LIST) { - repeated group bag { - optional double array; - } - } - } - optional int32 leftNodeId; - optional int32 rightNodeId; - optional double infoGain; -} - -Catalyst form: -StructType(StructField(treeId,IntegerType,true), StructField(nodeId,IntegerType,true), StructField(predict,StructType(StructField(predict,DoubleType,true), StructField(prob,DoubleType,true)),true), StructField(impurity,DoubleType,true), StructField(isLeaf,BooleanType,true), StructField(split,StructType(StructField(feature,IntegerType,true), StructField(threshold,DoubleType,true), StructField(featureType,IntegerType,true), StructField(categories,ArrayType(DoubleType,true),true)),true), StructField(leftNodeId,IntegerType,true), StructField(rightNodeId,IntegerType,true), StructField(infoGain,DoubleType,true)) - -2016-05-25 15:22:06 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 treeId; - optional int32 nodeId; - optional group predict { - optional double predict; - optional double prob; - } - optional double impurity; - optional boolean isLeaf; - optional group split { - optional int32 feature; - optional double threshold; - optional int32 featureType; - optional group categories (LIST) { - repeated group bag { - optional double array; - } - } - } - optional int32 leftNodeId; - optional int32 rightNodeId; - optional double infoGain; -} - -Catalyst form: -StructType(StructField(treeId,IntegerType,true), StructField(nodeId,IntegerType,true), StructField(predict,StructType(StructField(predict,DoubleType,true), StructField(prob,DoubleType,true)),true), StructField(impurity,DoubleType,true), StructField(isLeaf,BooleanType,true), StructField(split,StructType(StructField(feature,IntegerType,true), StructField(threshold,DoubleType,true), StructField(featureType,IntegerType,true), StructField(categories,ArrayType(DoubleType,true),true)),true), StructField(leftNodeId,IntegerType,true), StructField(rightNodeId,IntegerType,true), StructField(infoGain,DoubleType,true)) - -2016-05-25 15:22:06 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 treeId; - optional int32 nodeId; - optional group predict { - optional double predict; - optional double prob; - } - optional double impurity; - optional boolean isLeaf; - optional group split { - optional int32 feature; - optional double threshold; - optional int32 featureType; - optional group categories (LIST) { - repeated group bag { - optional double array; - } - } - } - optional int32 leftNodeId; - optional int32 rightNodeId; - optional double infoGain; -} - -Catalyst form: -StructType(StructField(treeId,IntegerType,true), StructField(nodeId,IntegerType,true), StructField(predict,StructType(StructField(predict,DoubleType,true), StructField(prob,DoubleType,true)),true), StructField(impurity,DoubleType,true), StructField(isLeaf,BooleanType,true), StructField(split,StructType(StructField(feature,IntegerType,true), StructField(threshold,DoubleType,true), StructField(featureType,IntegerType,true), StructField(categories,ArrayType(DoubleType,true),true)),true), StructField(leftNodeId,IntegerType,true), StructField(rightNodeId,IntegerType,true), StructField(infoGain,DoubleType,true)) - -2016-05-25 15:22:06 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 treeId; - optional int32 nodeId; - optional group predict { - optional double predict; - optional double prob; - } - optional double impurity; - optional boolean isLeaf; - optional group split { - optional int32 feature; - optional double threshold; - optional int32 featureType; - optional group categories (LIST) { - repeated group bag { - optional double array; - } - } - } - optional int32 leftNodeId; - optional int32 rightNodeId; - optional double infoGain; -} - -Catalyst form: -StructType(StructField(treeId,IntegerType,true), StructField(nodeId,IntegerType,true), StructField(predict,StructType(StructField(predict,DoubleType,true), StructField(prob,DoubleType,true)),true), StructField(impurity,DoubleType,true), StructField(isLeaf,BooleanType,true), StructField(split,StructType(StructField(feature,IntegerType,true), StructField(threshold,DoubleType,true), StructField(featureType,IntegerType,true), StructField(categories,ArrayType(DoubleType,true),true)),true), StructField(leftNodeId,IntegerType,true), StructField(rightNodeId,IntegerType,true), StructField(infoGain,DoubleType,true)) - -2016-05-25 15:22:06 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 15:22:06 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 15:22:06 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 15:22:06 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 15:22:06 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 1 records. -2016-05-25 15:22:06 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 15:22:06 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 2 records. -2016-05-25 15:22:06 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 15:22:06 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 15:22:06 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 15:22:06 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 15:22:06 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 15:22:06 INFO InternalParquetRecordReader:151 - block read in memory in 30 ms. row count = 1 -2016-05-25 15:22:06 INFO InternalParquetRecordReader:151 - block read in memory in 30 ms. row count = 2 -2016-05-25 15:22:06 INFO InternalParquetRecordReader:151 - block read in memory in 37 ms. row count = 1 -2016-05-25 15:22:06 INFO InternalParquetRecordReader:151 - block read in memory in 35 ms. row count = 1 -2016-05-25 15:22:07 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 38). 2256 bytes result sent to driver -2016-05-25 15:22:07 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 37). 2256 bytes result sent to driver -2016-05-25 15:22:07 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2256 bytes result sent to driver -2016-05-25 15:22:07 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2256 bytes result sent to driver -2016-05-25 15:22:07 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 38) in 1584 ms on localhost (1/4) -2016-05-25 15:22:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 1586 ms on localhost (2/4) -2016-05-25 15:22:07 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 37) in 1585 ms on localhost (3/4) -2016-05-25 15:22:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 1590 ms on localhost (4/4) -2016-05-25 15:22:07 INFO DAGScheduler:59 - ShuffleMapStage 18 (groupBy at DecisionTreeModel.scala:265) finished in 1.590 s -2016-05-25 15:22:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-25 15:22:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:22:07 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:22:07 INFO DAGScheduler:59 - waiting: Set(ResultStage 19) -2016-05-25 15:22:07 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:22:07 INFO DAGScheduler:59 - Missing parents for ResultStage 19: List() -2016-05-25 15:22:07 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[38] at mapValues at DecisionTreeModel.scala:266), which is now runnable -2016-05-25 15:22:08 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:52668 in memory (size: 3.7 KB, free: 730.4 MB) -2016-05-25 15:22:32 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 15:22:33 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 15:22:33 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 15:22:33 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 15:22:33 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 15:22:34 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 15:22:34 INFO Remoting:74 - Starting remoting -2016-05-25 15:22:34 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:52689] -2016-05-25 15:22:34 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52689. -2016-05-25 15:22:34 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 15:22:34 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 15:22:34 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-af6b6e90-81e5-4cac-81e1-17913655876f -2016-05-25 15:22:34 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 15:22:34 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-cb174115-28b3-41a1-aad2-f704a9b466ef\httpd-48d6597d-7488-48a7-9272-2d31f89d6c15 -2016-05-25 15:22:34 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 15:22:34 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:22:34 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52690 -2016-05-25 15:22:34 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52690. -2016-05-25 15:22:34 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 15:22:35 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:22:35 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 15:22:35 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 15:22:35 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 15:22:35 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 15:22:35 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 15:22:35 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52709. -2016-05-25 15:22:35 INFO NettyBlockTransferService:59 - Server created on 52709 -2016-05-25 15:22:35 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 15:22:35 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52709 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52709) -2016-05-25 15:22:35 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 15:22:36 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 15:22:36 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 15:22:36 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 15:22:36 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 15:22:36 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52709 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 15:22:36 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 15:22:37 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 15:22:38 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 15:22:38 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 15:22:38 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 15:22:38 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 15:22:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:22:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:22:38 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 15:22:38 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 15:22:38 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 15:22:38 INFO MemoryStore:59 - ensureFreeSpace(2148) called with curMem=175484, maxMem=766075207 -2016-05-25 15:22:38 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 15:22:38 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52709 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:22:38 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:22:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 15:22:38 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 15:22:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:22:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:22:38 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 15:22:38 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 15:22:38 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 15:22:38 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 15:22:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 15:22:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 15:22:38 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 15:22:38 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 15:22:38 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 15:22:38 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 15:22:38 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 15:22:38 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177632, maxMem=766075207 -2016-05-25 15:22:38 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 15:22:38 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262712, maxMem=766075207 -2016-05-25 15:22:38 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 15:22:38 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:52709 (size: 83.1 KB, free: 730.5 MB) -2016-05-25 15:22:38 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:52709 (size: 83.6 KB, free: 730.4 MB) -2016-05-25 15:22:38 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 15:22:38 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 15:22:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 458 ms on localhost (1/2) -2016-05-25 15:22:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 482 ms on localhost (2/2) -2016-05-25 15:22:38 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.498 s -2016-05-25 15:22:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 15:22:38 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.679319 s -2016-05-25 15:22:38 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 15:22:38 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 15:22:38 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 15:22:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:22:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:22:38 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 15:22:38 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348360, maxMem=766075207 -2016-05-25 15:22:38 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 15:22:38 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=353128, maxMem=766075207 -2016-05-25 15:22:38 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 15:22:38 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52709 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 15:22:38 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:22:38 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 15:22:38 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 15:22:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:22:38 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 15:22:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:22:38 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 15:22:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 24 ms on localhost (1/1) -2016-05-25 15:22:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 15:22:38 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.026 s -2016-05-25 15:22:38 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.044168 s -2016-05-25 15:22:38 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 15:22:38 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 15:22:38 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 15:22:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:22:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:22:38 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 15:22:38 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355675, maxMem=766075207 -2016-05-25 15:22:38 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 15:22:38 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=360011, maxMem=766075207 -2016-05-25 15:22:38 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 15:22:38 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52709 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 15:22:38 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:22:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 15:22:38 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 15:22:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:22:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:22:38 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 15:22:38 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 15:22:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:22:38 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:22:38 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 15:22:38 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 15:22:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 12 ms on localhost (1/2) -2016-05-25 15:22:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 14 ms on localhost (2/2) -2016-05-25 15:22:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 15:22:38 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.015 s -2016-05-25 15:22:38 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.027504 s -2016-05-25 15:22:38 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 15:22:38 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 15:22:38 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 15:22:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:22:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:22:38 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 15:22:38 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362330, maxMem=766075207 -2016-05-25 15:22:38 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 15:22:38 INFO MemoryStore:59 - ensureFreeSpace(2630) called with curMem=367394, maxMem=766075207 -2016-05-25 15:22:38 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 15:22:38 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52709 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:22:38 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:22:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 15:22:38 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 15:22:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 15:22:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 15:22:38 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 15:22:38 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 15:22:38 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:22:38 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 56874 bytes result sent to driver -2016-05-25 15:22:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:22:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 29 ms on localhost (1/2) -2016-05-25 15:22:39 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 60695 bytes result sent to driver -2016-05-25 15:22:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 44 ms on localhost (2/2) -2016-05-25 15:22:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 15:22:39 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.044 s -2016-05-25 15:22:39 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.072056 s -2016-05-25 15:22:40 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52709 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:22:40 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 15:22:40 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52709 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 15:22:40 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 15:22:40 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52709 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 15:22:40 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 15:22:40 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52709 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-25 15:22:40 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=342308, maxMem=766075207 -2016-05-25 15:22:40 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 15:22:40 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=342348, maxMem=766075207 -2016-05-25 15:22:40 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 15:22:40 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52709 (size: 81.0 B, free: 730.4 MB) -2016-05-25 15:22:40 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 15:22:40 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:22:40 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:22:40 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:22:40 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:22:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 15:22:40 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 15:22:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:22:40 INFO MemoryStore:59 - ensureFreeSpace(470272) called with curMem=342429, maxMem=766075207 -2016-05-25 15:22:40 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 459.3 KB, free 729.8 MB) -2016-05-25 15:22:40 INFO MemoryStore:59 - ensureFreeSpace(115521) called with curMem=812701, maxMem=766075207 -2016-05-25 15:22:40 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 112.8 KB, free 729.7 MB) -2016-05-25 15:22:40 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52709 (size: 112.8 KB, free: 730.3 MB) -2016-05-25 15:22:40 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:22:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:22:40 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 15:22:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:22:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:22:40 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 15:22:40 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 15:22:40 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 15:22:40 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:22:40 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 15:22:40 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:22:40 INFO MemoryStore:59 - ensureFreeSpace(85816) called with curMem=928222, maxMem=766075207 -2016-05-25 15:22:40 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 83.8 KB, free 729.6 MB) -2016-05-25 15:22:40 INFO MemoryStore:59 - ensureFreeSpace(94400) called with curMem=1014038, maxMem=766075207 -2016-05-25 15:22:40 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 92.2 KB, free 729.5 MB) -2016-05-25 15:22:40 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:52709 (size: 83.8 KB, free: 730.2 MB) -2016-05-25 15:22:40 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:52709 (size: 92.2 KB, free: 730.1 MB) -2016-05-25 15:22:40 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 15:22:40 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 15:22:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 252 ms on localhost (1/2) -2016-05-25 15:22:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 250 ms on localhost (2/2) -2016-05-25 15:22:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 15:22:40 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.254 s -2016-05-25 15:22:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:22:40 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:22:40 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 15:22:40 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:22:40 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 15:22:40 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:22:40 INFO MemoryStore:59 - ensureFreeSpace(203376) called with curMem=1108438, maxMem=766075207 -2016-05-25 15:22:40 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 198.6 KB, free 729.3 MB) -2016-05-25 15:22:40 INFO MemoryStore:59 - ensureFreeSpace(43530) called with curMem=1311814, maxMem=766075207 -2016-05-25 15:22:40 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 42.5 KB, free 729.3 MB) -2016-05-25 15:22:40 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52709 (size: 42.5 KB, free: 730.1 MB) -2016-05-25 15:22:40 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:22:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 15:22:40 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 15:22:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:22:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:22:40 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 15:22:40 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 15:22:41 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 15:22:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:22:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-25 15:22:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-25 15:22:41 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 15:22:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 156 ms on localhost (1/2) -2016-05-25 15:22:41 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 15:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 374 ms on localhost (2/2) -2016-05-25 15:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 15:22:41 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.375 s -2016-05-25 15:22:41 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.699179 s -2016-05-25 15:22:41 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 15:22:41 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 15:22:41 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 15:22:41 INFO RandomForest:59 - init: 1.489217936 - total: 2.43495087 - findSplitsBins: 1.259045738 - findBestSplits: 0.923212126 - chooseSplits: 0.915162 -2016-05-25 15:22:41 INFO SparkContext:59 - Starting job: count at DecisionTrees4Regression.scala:50 -2016-05-25 15:22:41 INFO DAGScheduler:59 - Got job 5 (count at DecisionTrees4Regression.scala:50) with 2 output partitions -2016-05-25 15:22:41 INFO DAGScheduler:59 - Final stage: ResultStage 6(count at DecisionTrees4Regression.scala:50) -2016-05-25 15:22:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:22:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:22:41 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 15:22:41 INFO MemoryStore:59 - ensureFreeSpace(3816) called with curMem=1175128, maxMem=766075207 -2016-05-25 15:22:41 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 3.7 KB, free 729.5 MB) -2016-05-25 15:22:41 INFO MemoryStore:59 - ensureFreeSpace(2142) called with curMem=1178944, maxMem=766075207 -2016-05-25 15:22:41 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.5 MB) -2016-05-25 15:22:41 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52709 (size: 2.1 KB, free: 730.3 MB) -2016-05-25 15:22:41 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:22:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 15:22:41 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 15:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:22:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:22:41 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 15:22:41 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 15:22:41 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:22:41 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:22:41 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2082 bytes result sent to driver -2016-05-25 15:22:41 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2082 bytes result sent to driver -2016-05-25 15:22:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 11 ms on localhost (1/2) -2016-05-25 15:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 13 ms on localhost (2/2) -2016-05-25 15:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 15:22:41 INFO DAGScheduler:59 - ResultStage 6 (count at DecisionTrees4Regression.scala:50) finished in 0.013 s -2016-05-25 15:22:41 INFO DAGScheduler:59 - Job 5 finished: count at DecisionTrees4Regression.scala:50, took 0.033829 s -2016-05-25 15:22:41 INFO SparkContext:59 - Starting job: count at DecisionTrees4Regression.scala:51 -2016-05-25 15:22:41 INFO DAGScheduler:59 - Got job 6 (count at DecisionTrees4Regression.scala:51) with 2 output partitions -2016-05-25 15:22:41 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at DecisionTrees4Regression.scala:51) -2016-05-25 15:22:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:22:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:22:41 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[7] at randomSplit at DecisionTrees4Regression.scala:29), which has no missing parents -2016-05-25 15:22:41 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1181086, maxMem=766075207 -2016-05-25 15:22:41 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 4.0 KB, free 729.5 MB) -2016-05-25 15:22:41 INFO MemoryStore:59 - ensureFreeSpace(2265) called with curMem=1185198, maxMem=766075207 -2016-05-25 15:22:41 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.5 MB) -2016-05-25 15:22:41 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52709 (size: 2.2 KB, free: 730.3 MB) -2016-05-25 15:22:41 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:22:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[7] at randomSplit at DecisionTrees4Regression.scala:29) -2016-05-25 15:22:41 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 15:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:22:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:22:41 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 15:22:41 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 15:22:41 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:22:41 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:22:41 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2082 bytes result sent to driver -2016-05-25 15:22:41 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2082 bytes result sent to driver -2016-05-25 15:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 12 ms on localhost (1/2) -2016-05-25 15:22:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 11 ms on localhost (2/2) -2016-05-25 15:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 15:22:41 INFO DAGScheduler:59 - ResultStage 7 (count at DecisionTrees4Regression.scala:51) finished in 0.013 s -2016-05-25 15:22:41 INFO DAGScheduler:59 - Job 6 finished: count at DecisionTrees4Regression.scala:51, took 0.032452 s -2016-05-25 15:22:41 INFO SparkContext:59 - Starting job: count at DecisionTrees4Regression.scala:52 -2016-05-25 15:22:41 INFO DAGScheduler:59 - Got job 7 (count at DecisionTrees4Regression.scala:52) with 2 output partitions -2016-05-25 15:22:41 INFO DAGScheduler:59 - Final stage: ResultStage 8(count at DecisionTrees4Regression.scala:52) -2016-05-25 15:22:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:22:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:22:41 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[8] at randomSplit at DecisionTrees4Regression.scala:29), which has no missing parents -2016-05-25 15:22:41 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1187463, maxMem=766075207 -2016-05-25 15:22:41 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-05-25 15:22:41 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=1191575, maxMem=766075207 -2016-05-25 15:22:41 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.4 MB) -2016-05-25 15:22:41 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:52709 (size: 2.2 KB, free: 730.3 MB) -2016-05-25 15:22:41 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:22:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[8] at randomSplit at DecisionTrees4Regression.scala:29) -2016-05-25 15:22:41 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 15:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:22:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:22:41 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 15:22:41 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 15:22:41 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:22:41 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:22:41 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 15:22:41 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 15:22:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 11 ms on localhost (1/2) -2016-05-25 15:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 13 ms on localhost (2/2) -2016-05-25 15:22:41 INFO DAGScheduler:59 - ResultStage 8 (count at DecisionTrees4Regression.scala:52) finished in 0.013 s -2016-05-25 15:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 15:22:41 INFO DAGScheduler:59 - Job 7 finished: count at DecisionTrees4Regression.scala:52, took 0.031640 s -2016-05-25 15:22:41 INFO SparkContext:59 - Starting job: take at DecisionTrees4Regression.scala:58 -2016-05-25 15:22:41 INFO DAGScheduler:59 - Got job 8 (take at DecisionTrees4Regression.scala:58) with 1 output partitions -2016-05-25 15:22:41 INFO DAGScheduler:59 - Final stage: ResultStage 9(take at DecisionTrees4Regression.scala:58) -2016-05-25 15:22:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:22:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:22:41 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[17] at map at DecisionTrees4Regression.scala:43), which has no missing parents -2016-05-25 15:22:41 INFO MemoryStore:59 - ensureFreeSpace(6576) called with curMem=1193843, maxMem=766075207 -2016-05-25 15:22:41 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 6.4 KB, free 729.4 MB) -2016-05-25 15:22:41 INFO MemoryStore:59 - ensureFreeSpace(3537) called with curMem=1200419, maxMem=766075207 -2016-05-25 15:22:41 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-25 15:22:41 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:52709 (size: 3.5 KB, free: 730.2 MB) -2016-05-25 15:22:41 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:22:41 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 9 (MapPartitionsRDD[17] at map at DecisionTrees4Regression.scala:43) -2016-05-25 15:22:41 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 1 tasks -2016-05-25 15:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:22:41 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 15:22:41 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:22:41 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2569 bytes result sent to driver -2016-05-25 15:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 11 ms on localhost (1/1) -2016-05-25 15:22:41 INFO DAGScheduler:59 - ResultStage 9 (take at DecisionTrees4Regression.scala:58) finished in 0.011 s -2016-05-25 15:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 15:22:41 INFO DAGScheduler:59 - Job 8 finished: take at DecisionTrees4Regression.scala:58, took 0.027147 s -2016-05-25 15:22:41 INFO SparkContext:59 - Starting job: take at DecisionTrees4Regression.scala:58 -2016-05-25 15:22:41 INFO DAGScheduler:59 - Got job 9 (take at DecisionTrees4Regression.scala:58) with 1 output partitions -2016-05-25 15:22:41 INFO DAGScheduler:59 - Final stage: ResultStage 10(take at DecisionTrees4Regression.scala:58) -2016-05-25 15:22:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:22:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:22:41 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[17] at map at DecisionTrees4Regression.scala:43), which has no missing parents -2016-05-25 15:22:41 INFO MemoryStore:59 - ensureFreeSpace(6576) called with curMem=1203956, maxMem=766075207 -2016-05-25 15:22:41 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 6.4 KB, free 729.4 MB) -2016-05-25 15:22:41 INFO MemoryStore:59 - ensureFreeSpace(3536) called with curMem=1210532, maxMem=766075207 -2016-05-25 15:22:41 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.4 MB) -2016-05-25 15:22:41 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:52709 (size: 3.5 KB, free: 730.2 MB) -2016-05-25 15:22:41 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:22:41 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[17] at map at DecisionTrees4Regression.scala:43) -2016-05-25 15:22:41 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-25 15:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 18, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:22:41 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 18) -2016-05-25 15:22:41 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:22:41 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 18). 2473 bytes result sent to driver -2016-05-25 15:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 18) in 8 ms on localhost (1/1) -2016-05-25 15:22:41 INFO DAGScheduler:59 - ResultStage 10 (take at DecisionTrees4Regression.scala:58) finished in 0.009 s -2016-05-25 15:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 15:22:41 INFO DAGScheduler:59 - Job 9 finished: take at DecisionTrees4Regression.scala:58, took 0.025155 s -2016-05-25 15:22:41 INFO SparkContext:59 - Starting job: mean at DecisionTrees4Regression.scala:60 -2016-05-25 15:22:41 INFO DAGScheduler:59 - Got job 10 (mean at DecisionTrees4Regression.scala:60) with 2 output partitions -2016-05-25 15:22:41 INFO DAGScheduler:59 - Final stage: ResultStage 11(mean at DecisionTrees4Regression.scala:60) -2016-05-25 15:22:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:22:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:22:41 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[19] at mean at DecisionTrees4Regression.scala:60), which has no missing parents -2016-05-25 15:22:41 INFO MemoryStore:59 - ensureFreeSpace(7152) called with curMem=1214068, maxMem=766075207 -2016-05-25 15:22:41 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 7.0 KB, free 729.4 MB) -2016-05-25 15:22:41 INFO MemoryStore:59 - ensureFreeSpace(3788) called with curMem=1221220, maxMem=766075207 -2016-05-25 15:22:41 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.7 KB, free 729.4 MB) -2016-05-25 15:22:41 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:52709 (size: 3.7 KB, free: 730.2 MB) -2016-05-25 15:22:41 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:22:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[19] at mean at DecisionTrees4Regression.scala:60) -2016-05-25 15:22:41 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-25 15:22:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 19, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:22:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 20, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:22:41 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 19) -2016-05-25 15:22:41 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 20) -2016-05-25 15:22:41 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:22:41 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:22:41 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 20). 2209 bytes result sent to driver -2016-05-25 15:22:41 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 19). 2209 bytes result sent to driver -2016-05-25 15:22:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 19) in 15 ms on localhost (1/2) -2016-05-25 15:22:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 20) in 15 ms on localhost (2/2) -2016-05-25 15:22:41 INFO DAGScheduler:59 - ResultStage 11 (mean at DecisionTrees4Regression.scala:60) finished in 0.016 s -2016-05-25 15:22:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 15:22:41 INFO DAGScheduler:59 - Job 10 finished: mean at DecisionTrees4Regression.scala:60, took 0.035382 s -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 15:22:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 15:22:41 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 15:22:41 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 15:22:41 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 15:22:41 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 15:22:41 INFO BlockManager:59 - BlockManager stopped -2016-05-25 15:22:41 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 15:22:41 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 15:22:41 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 15:22:41 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 15:22:41 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-cb174115-28b3-41a1-aad2-f704a9b466ef -2016-05-25 15:22:41 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 15:22:41 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-25 15:22:58 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 15:22:59 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 15:22:59 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 15:22:59 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 15:22:59 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 15:23:00 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 15:23:00 INFO Remoting:74 - Starting remoting -2016-05-25 15:23:00 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:52732] -2016-05-25 15:23:00 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52732. -2016-05-25 15:23:00 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 15:23:00 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 15:23:00 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-9835a5fb-e83c-496a-9cc3-278bec3cd8aa -2016-05-25 15:23:00 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 15:23:00 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-2e34a8e0-e315-466c-9e00-32492aa88302\httpd-a8a39627-81f3-4f9d-96da-6d14e9fe5b63 -2016-05-25 15:23:00 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 15:23:00 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:23:00 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52733 -2016-05-25 15:23:00 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52733. -2016-05-25 15:23:00 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 15:23:01 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:23:01 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 15:23:01 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 15:23:01 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 15:23:01 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 15:23:01 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 15:23:01 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52752. -2016-05-25 15:23:01 INFO NettyBlockTransferService:59 - Server created on 52752 -2016-05-25 15:23:01 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 15:23:01 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52752 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52752) -2016-05-25 15:23:01 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 15:23:02 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 15:23:02 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 15:23:02 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 15:23:02 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 15:23:02 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52752 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 15:23:02 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 15:23:03 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 15:23:03 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 15:23:03 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 15:23:03 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 15:23:03 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 15:23:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:23:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:23:04 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 15:23:04 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 15:23:04 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 15:23:04 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 15:23:04 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 15:23:04 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52752 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:23:04 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:23:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 15:23:04 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 15:23:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:23:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:23:04 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 15:23:04 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 15:23:04 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 15:23:04 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 15:23:04 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 15:23:04 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 15:23:04 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 15:23:04 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 15:23:04 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 15:23:04 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 15:23:04 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 15:23:04 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177630, maxMem=766075207 -2016-05-25 15:23:04 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 15:23:04 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263278, maxMem=766075207 -2016-05-25 15:23:04 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 15:23:04 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:52752 (size: 83.6 KB, free: 730.5 MB) -2016-05-25 15:23:04 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:52752 (size: 83.1 KB, free: 730.4 MB) -2016-05-25 15:23:04 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 15:23:04 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 15:23:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 452 ms on localhost (1/2) -2016-05-25 15:23:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 429 ms on localhost (2/2) -2016-05-25 15:23:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 15:23:04 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.476 s -2016-05-25 15:23:04 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.640892 s -2016-05-25 15:23:04 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 15:23:04 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 15:23:04 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 15:23:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:23:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:23:04 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 15:23:04 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348358, maxMem=766075207 -2016-05-25 15:23:04 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 15:23:04 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=353126, maxMem=766075207 -2016-05-25 15:23:04 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 15:23:04 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52752 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 15:23:04 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:23:04 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 15:23:04 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 15:23:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:23:04 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 15:23:04 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:23:04 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 15:23:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 25 ms on localhost (1/1) -2016-05-25 15:23:04 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.025 s -2016-05-25 15:23:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 15:23:04 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.043142 s -2016-05-25 15:23:04 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 15:23:04 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 15:23:04 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 15:23:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:23:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:23:04 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 15:23:04 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355673, maxMem=766075207 -2016-05-25 15:23:04 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 15:23:04 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=360009, maxMem=766075207 -2016-05-25 15:23:04 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 15:23:04 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52752 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 15:23:04 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:23:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 15:23:04 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 15:23:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:23:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:23:04 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 15:23:04 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 15:23:04 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:23:04 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:23:04 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 15:23:04 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 15:23:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 14 ms on localhost (1/2) -2016-05-25 15:23:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 13 ms on localhost (2/2) -2016-05-25 15:23:04 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.016 s -2016-05-25 15:23:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 15:23:04 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.028707 s -2016-05-25 15:23:04 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 15:23:04 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 15:23:04 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 15:23:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:23:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:23:04 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 15:23:04 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362328, maxMem=766075207 -2016-05-25 15:23:04 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 15:23:04 INFO MemoryStore:59 - ensureFreeSpace(2631) called with curMem=367392, maxMem=766075207 -2016-05-25 15:23:04 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 15:23:04 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52752 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:23:04 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:23:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 15:23:04 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 15:23:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 15:23:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 15:23:04 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 15:23:04 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 15:23:04 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:23:04 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:23:04 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 67808 bytes result sent to driver -2016-05-25 15:23:04 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 75864 bytes result sent to driver -2016-05-25 15:23:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 33 ms on localhost (1/2) -2016-05-25 15:23:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 36 ms on localhost (2/2) -2016-05-25 15:23:04 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.039 s -2016-05-25 15:23:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 15:23:04 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.054497 s -2016-05-25 15:23:06 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52752 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:23:06 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 15:23:06 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52752 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 15:23:06 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 15:23:06 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52752 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 15:23:06 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 15:23:06 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=348358, maxMem=766075207 -2016-05-25 15:23:06 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 15:23:06 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=348398, maxMem=766075207 -2016-05-25 15:23:06 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 15:23:06 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52752 (size: 81.0 B, free: 730.4 MB) -2016-05-25 15:23:06 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 15:23:06 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:23:06 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:23:06 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:23:06 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:23:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 15:23:06 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 15:23:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:23:06 INFO MemoryStore:59 - ensureFreeSpace(522712) called with curMem=348479, maxMem=766075207 -2016-05-25 15:23:06 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 510.5 KB, free 729.8 MB) -2016-05-25 15:23:06 INFO MemoryStore:59 - ensureFreeSpace(128249) called with curMem=871191, maxMem=766075207 -2016-05-25 15:23:06 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 125.2 KB, free 729.6 MB) -2016-05-25 15:23:06 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52752 (size: 125.2 KB, free: 730.3 MB) -2016-05-25 15:23:06 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:23:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:23:06 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 15:23:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:23:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:23:06 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 15:23:06 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 15:23:06 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 15:23:06 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:23:06 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 15:23:06 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:23:06 INFO MemoryStore:59 - ensureFreeSpace(111560) called with curMem=999440, maxMem=766075207 -2016-05-25 15:23:06 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 108.9 KB, free 729.5 MB) -2016-05-25 15:23:06 INFO MemoryStore:59 - ensureFreeSpace(120136) called with curMem=1111000, maxMem=766075207 -2016-05-25 15:23:06 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 117.3 KB, free 729.4 MB) -2016-05-25 15:23:06 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:52752 (size: 108.9 KB, free: 730.2 MB) -2016-05-25 15:23:06 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:52752 (size: 117.3 KB, free: 730.1 MB) -2016-05-25 15:23:06 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 15:23:06 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 15:23:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 325 ms on localhost (1/2) -2016-05-25 15:23:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 327 ms on localhost (2/2) -2016-05-25 15:23:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 15:23:06 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.330 s -2016-05-25 15:23:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:23:06 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:23:06 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 15:23:06 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:23:06 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 15:23:06 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:23:06 INFO MemoryStore:59 - ensureFreeSpace(229136) called with curMem=1231136, maxMem=766075207 -2016-05-25 15:23:06 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 223.8 KB, free 729.2 MB) -2016-05-25 15:23:06 INFO MemoryStore:59 - ensureFreeSpace(48197) called with curMem=1460272, maxMem=766075207 -2016-05-25 15:23:06 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 47.1 KB, free 729.1 MB) -2016-05-25 15:23:06 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52752 (size: 47.1 KB, free: 730.0 MB) -2016-05-25 15:23:06 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:23:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 15:23:06 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 15:23:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:23:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:23:06 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 15:23:06 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 15:23:06 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 15:23:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:23:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 9 ms -2016-05-25 15:23:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 8 ms -2016-05-25 15:23:06 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 15:23:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 226 ms on localhost (1/2) -2016-05-25 15:23:07 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 15:23:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 447 ms on localhost (2/2) -2016-05-25 15:23:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 15:23:07 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.449 s -2016-05-25 15:23:07 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.839603 s -2016-05-25 15:23:07 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=1508469, maxMem=766075207 -2016-05-25 15:23:07 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 40.0 B, free 729.1 MB) -2016-05-25 15:23:07 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=1508509, maxMem=766075207 -2016-05-25 15:23:07 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 81.0 B, free 729.1 MB) -2016-05-25 15:23:07 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52752 (size: 81.0 B, free: 730.0 MB) -2016-05-25 15:23:07 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 15:23:07 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:23:07 INFO DAGScheduler:59 - Registering RDD 17 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:23:07 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:23:07 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:23:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 15:23:07 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 15:23:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:23:07 INFO MemoryStore:59 - ensureFreeSpace(523064) called with curMem=1508590, maxMem=766075207 -2016-05-25 15:23:07 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 510.8 KB, free 728.6 MB) -2016-05-25 15:23:07 INFO MemoryStore:59 - ensureFreeSpace(128442) called with curMem=2031654, maxMem=766075207 -2016-05-25 15:23:07 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 125.4 KB, free 728.5 MB) -2016-05-25 15:23:07 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52752 (size: 125.4 KB, free: 729.9 MB) -2016-05-25 15:23:07 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:23:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:23:07 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 15:23:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:23:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:23:07 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 15:23:07 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 15:23:07 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-25 15:23:07 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-25 15:23:07 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 15:23:07 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 15:23:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 68 ms on localhost (1/2) -2016-05-25 15:23:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 71 ms on localhost (2/2) -2016-05-25 15:23:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 15:23:07 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.073 s -2016-05-25 15:23:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:23:07 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:23:07 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 15:23:07 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:23:07 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 15:23:07 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:23:07 INFO MemoryStore:59 - ensureFreeSpace(229136) called with curMem=2160096, maxMem=766075207 -2016-05-25 15:23:07 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 223.8 KB, free 728.3 MB) -2016-05-25 15:23:07 INFO MemoryStore:59 - ensureFreeSpace(48197) called with curMem=2389232, maxMem=766075207 -2016-05-25 15:23:07 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 47.1 KB, free 728.3 MB) -2016-05-25 15:23:07 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:52752 (size: 47.1 KB, free: 729.8 MB) -2016-05-25 15:23:07 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:23:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645) -2016-05-25 15:23:07 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 15:23:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:23:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:23:07 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 15:23:07 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 15:23:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:23:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 15:23:07 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 15:23:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 15:23:07 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 1161 bytes result sent to driver -2016-05-25 15:23:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 15 ms on localhost (1/2) -2016-05-25 15:23:07 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 15:23:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 85 ms on localhost (2/2) -2016-05-25 15:23:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 15:23:07 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.086 s -2016-05-25 15:23:07 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.205365 s -2016-05-25 15:23:07 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 15:23:07 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 15:23:07 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 15:23:07 INFO RandomForest:59 - init: 1.498482653 - total: 2.828858636 - findSplitsBins: 1.277367237 - findBestSplits: 1.307656611 - chooseSplits: 1.299718485 -2016-05-25 15:23:07 INFO SparkContext:59 - Starting job: count at DecisionTrees4Regression.scala:50 -2016-05-25 15:23:07 INFO DAGScheduler:59 - Got job 6 (count at DecisionTrees4Regression.scala:50) with 2 output partitions -2016-05-25 15:23:07 INFO DAGScheduler:59 - Final stage: ResultStage 8(count at DecisionTrees4Regression.scala:50) -2016-05-25 15:23:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:23:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:23:07 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 15:23:07 INFO MemoryStore:59 - ensureFreeSpace(3816) called with curMem=2205733, maxMem=766075207 -2016-05-25 15:23:07 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 3.7 KB, free 728.5 MB) -2016-05-25 15:23:07 INFO MemoryStore:59 - ensureFreeSpace(2142) called with curMem=2209549, maxMem=766075207 -2016-05-25 15:23:07 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.1 KB, free 728.5 MB) -2016-05-25 15:23:07 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:52752 (size: 2.1 KB, free: 730.1 MB) -2016-05-25 15:23:07 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:23:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 15:23:07 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 15:23:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:23:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:23:07 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 15:23:07 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 15:23:07 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:23:07 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:23:07 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 15:23:07 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 15:23:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 9 ms on localhost (1/2) -2016-05-25 15:23:07 INFO DAGScheduler:59 - ResultStage 8 (count at DecisionTrees4Regression.scala:50) finished in 0.011 s -2016-05-25 15:23:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 11 ms on localhost (2/2) -2016-05-25 15:23:07 INFO DAGScheduler:59 - Job 6 finished: count at DecisionTrees4Regression.scala:50, took 0.030911 s -2016-05-25 15:23:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 15:23:07 INFO SparkContext:59 - Starting job: count at DecisionTrees4Regression.scala:51 -2016-05-25 15:23:07 INFO DAGScheduler:59 - Got job 7 (count at DecisionTrees4Regression.scala:51) with 2 output partitions -2016-05-25 15:23:07 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at DecisionTrees4Regression.scala:51) -2016-05-25 15:23:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:23:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:23:07 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[7] at randomSplit at DecisionTrees4Regression.scala:29), which has no missing parents -2016-05-25 15:23:07 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2211691, maxMem=766075207 -2016-05-25 15:23:07 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.0 KB, free 728.5 MB) -2016-05-25 15:23:07 INFO MemoryStore:59 - ensureFreeSpace(2265) called with curMem=2215803, maxMem=766075207 -2016-05-25 15:23:07 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.5 MB) -2016-05-25 15:23:07 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:52752 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 15:23:07 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:23:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[7] at randomSplit at DecisionTrees4Regression.scala:29) -2016-05-25 15:23:07 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 15:23:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:23:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:23:07 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 15:23:07 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-25 15:23:07 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:23:07 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:23:07 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2082 bytes result sent to driver -2016-05-25 15:23:07 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2082 bytes result sent to driver -2016-05-25 15:23:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 10 ms on localhost (1/2) -2016-05-25 15:23:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 10 ms on localhost (2/2) -2016-05-25 15:23:07 INFO DAGScheduler:59 - ResultStage 9 (count at DecisionTrees4Regression.scala:51) finished in 0.011 s -2016-05-25 15:23:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 15:23:07 INFO DAGScheduler:59 - Job 7 finished: count at DecisionTrees4Regression.scala:51, took 0.028519 s -2016-05-25 15:23:07 INFO SparkContext:59 - Starting job: count at DecisionTrees4Regression.scala:52 -2016-05-25 15:23:07 INFO DAGScheduler:59 - Got job 8 (count at DecisionTrees4Regression.scala:52) with 2 output partitions -2016-05-25 15:23:07 INFO DAGScheduler:59 - Final stage: ResultStage 10(count at DecisionTrees4Regression.scala:52) -2016-05-25 15:23:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:23:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:23:07 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[8] at randomSplit at DecisionTrees4Regression.scala:29), which has no missing parents -2016-05-25 15:23:07 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2218068, maxMem=766075207 -2016-05-25 15:23:07 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 4.0 KB, free 728.5 MB) -2016-05-25 15:23:07 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=2222180, maxMem=766075207 -2016-05-25 15:23:07 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.5 MB) -2016-05-25 15:23:07 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:52752 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 15:23:07 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:23:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[8] at randomSplit at DecisionTrees4Regression.scala:29) -2016-05-25 15:23:07 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-25 15:23:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:23:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:23:07 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 15:23:07 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-25 15:23:07 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:23:07 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:23:07 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2082 bytes result sent to driver -2016-05-25 15:23:07 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2082 bytes result sent to driver -2016-05-25 15:23:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 11 ms on localhost (1/2) -2016-05-25 15:23:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 10 ms on localhost (2/2) -2016-05-25 15:23:07 INFO DAGScheduler:59 - ResultStage 10 (count at DecisionTrees4Regression.scala:52) finished in 0.012 s -2016-05-25 15:23:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 15:23:07 INFO DAGScheduler:59 - Job 8 finished: count at DecisionTrees4Regression.scala:52, took 0.025306 s -2016-05-25 15:23:07 INFO SparkContext:59 - Starting job: take at DecisionTrees4Regression.scala:58 -2016-05-25 15:23:07 INFO DAGScheduler:59 - Got job 9 (take at DecisionTrees4Regression.scala:58) with 1 output partitions -2016-05-25 15:23:07 INFO DAGScheduler:59 - Final stage: ResultStage 11(take at DecisionTrees4Regression.scala:58) -2016-05-25 15:23:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:23:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:23:07 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[20] at map at DecisionTrees4Regression.scala:43), which has no missing parents -2016-05-25 15:23:07 INFO MemoryStore:59 - ensureFreeSpace(6792) called with curMem=2224448, maxMem=766075207 -2016-05-25 15:23:07 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 6.6 KB, free 728.5 MB) -2016-05-25 15:23:07 INFO MemoryStore:59 - ensureFreeSpace(3634) called with curMem=2231240, maxMem=766075207 -2016-05-25 15:23:07 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.5 KB, free 728.5 MB) -2016-05-25 15:23:07 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:52752 (size: 3.5 KB, free: 730.1 MB) -2016-05-25 15:23:07 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:23:07 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 11 (MapPartitionsRDD[20] at map at DecisionTrees4Regression.scala:43) -2016-05-25 15:23:07 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 1 tasks -2016-05-25 15:23:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:23:07 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-25 15:23:07 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:23:07 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2353 bytes result sent to driver -2016-05-25 15:23:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 10 ms on localhost (1/1) -2016-05-25 15:23:07 INFO DAGScheduler:59 - ResultStage 11 (take at DecisionTrees4Regression.scala:58) finished in 0.010 s -2016-05-25 15:23:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 15:23:07 INFO DAGScheduler:59 - Job 9 finished: take at DecisionTrees4Regression.scala:58, took 0.024996 s -2016-05-25 15:23:07 INFO SparkContext:59 - Starting job: take at DecisionTrees4Regression.scala:58 -2016-05-25 15:23:07 INFO DAGScheduler:59 - Got job 10 (take at DecisionTrees4Regression.scala:58) with 1 output partitions -2016-05-25 15:23:07 INFO DAGScheduler:59 - Final stage: ResultStage 12(take at DecisionTrees4Regression.scala:58) -2016-05-25 15:23:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:23:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:23:07 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[20] at map at DecisionTrees4Regression.scala:43), which has no missing parents -2016-05-25 15:23:07 INFO MemoryStore:59 - ensureFreeSpace(6792) called with curMem=2234874, maxMem=766075207 -2016-05-25 15:23:07 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 6.6 KB, free 728.4 MB) -2016-05-25 15:23:07 INFO MemoryStore:59 - ensureFreeSpace(3634) called with curMem=2241666, maxMem=766075207 -2016-05-25 15:23:07 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.5 KB, free 728.4 MB) -2016-05-25 15:23:07 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:52752 (size: 3.5 KB, free: 730.1 MB) -2016-05-25 15:23:07 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:23:07 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[20] at map at DecisionTrees4Regression.scala:43) -2016-05-25 15:23:07 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-25 15:23:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 22, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:23:07 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 22) -2016-05-25 15:23:07 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:23:07 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 22). 2206 bytes result sent to driver -2016-05-25 15:23:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 22) in 7 ms on localhost (1/1) -2016-05-25 15:23:07 INFO DAGScheduler:59 - ResultStage 12 (take at DecisionTrees4Regression.scala:58) finished in 0.007 s -2016-05-25 15:23:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 15:23:07 INFO DAGScheduler:59 - Job 10 finished: take at DecisionTrees4Regression.scala:58, took 0.020494 s -2016-05-25 15:23:07 INFO SparkContext:59 - Starting job: mean at DecisionTrees4Regression.scala:60 -2016-05-25 15:23:07 INFO DAGScheduler:59 - Got job 11 (mean at DecisionTrees4Regression.scala:60) with 2 output partitions -2016-05-25 15:23:07 INFO DAGScheduler:59 - Final stage: ResultStage 13(mean at DecisionTrees4Regression.scala:60) -2016-05-25 15:23:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:23:07 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:23:07 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[22] at mean at DecisionTrees4Regression.scala:60), which has no missing parents -2016-05-25 15:23:07 INFO MemoryStore:59 - ensureFreeSpace(7368) called with curMem=2245300, maxMem=766075207 -2016-05-25 15:23:07 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 7.2 KB, free 728.4 MB) -2016-05-25 15:23:07 INFO MemoryStore:59 - ensureFreeSpace(3884) called with curMem=2252668, maxMem=766075207 -2016-05-25 15:23:07 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.8 KB, free 728.4 MB) -2016-05-25 15:23:07 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:52752 (size: 3.8 KB, free: 730.1 MB) -2016-05-25 15:23:07 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:23:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[22] at mean at DecisionTrees4Regression.scala:60) -2016-05-25 15:23:07 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-25 15:23:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 23, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:23:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 24, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:23:07 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 23) -2016-05-25 15:23:07 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 24) -2016-05-25 15:23:07 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:23:07 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:23:07 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 24). 2209 bytes result sent to driver -2016-05-25 15:23:07 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 23). 2209 bytes result sent to driver -2016-05-25 15:23:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 24) in 12 ms on localhost (1/2) -2016-05-25 15:23:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 23) in 14 ms on localhost (2/2) -2016-05-25 15:23:07 INFO DAGScheduler:59 - ResultStage 13 (mean at DecisionTrees4Regression.scala:60) finished in 0.014 s -2016-05-25 15:23:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 15:23:07 INFO DAGScheduler:59 - Job 11 finished: mean at DecisionTrees4Regression.scala:60, took 0.031429 s -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 15:23:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 15:23:07 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 15:23:07 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 15:23:07 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 15:23:08 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 15:23:08 INFO BlockManager:59 - BlockManager stopped -2016-05-25 15:23:08 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 15:23:08 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 15:23:08 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 15:23:08 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 15:23:08 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-2e34a8e0-e315-466c-9e00-32492aa88302 -2016-05-25 15:24:55 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 15:24:56 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 15:24:56 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 15:24:56 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 15:24:56 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 15:24:57 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 15:24:57 INFO Remoting:74 - Starting remoting -2016-05-25 15:24:57 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:52773] -2016-05-25 15:24:57 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52773. -2016-05-25 15:24:57 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 15:24:57 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 15:24:57 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-57c82143-8e60-4c50-b8c6-e83b8b4d76e4 -2016-05-25 15:24:57 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 15:24:57 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-7d5f4812-09a0-43d2-bf63-c41579ad0054\httpd-c997fb98-84f8-49e8-866f-72f4baddc216 -2016-05-25 15:24:57 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 15:24:57 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:24:57 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52774 -2016-05-25 15:24:57 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52774. -2016-05-25 15:24:57 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 15:24:57 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:24:57 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 15:24:57 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 15:24:57 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 15:24:58 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 15:24:58 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 15:24:58 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52793. -2016-05-25 15:24:58 INFO NettyBlockTransferService:59 - Server created on 52793 -2016-05-25 15:24:58 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 15:24:58 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52793 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52793) -2016-05-25 15:24:58 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 15:24:58 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 15:24:58 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 15:24:59 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 15:24:59 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 15:24:59 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52793 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 15:24:59 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 15:25:00 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 15:25:00 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 15:25:00 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 15:25:00 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 15:25:00 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 15:25:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:25:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:25:00 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 15:25:00 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 15:25:00 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 15:25:00 INFO MemoryStore:59 - ensureFreeSpace(2148) called with curMem=175484, maxMem=766075207 -2016-05-25 15:25:00 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 15:25:01 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52793 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:25:01 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 15:25:01 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 15:25:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:01 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 15:25:01 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 15:25:01 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 15:25:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 15:25:01 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 15:25:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 15:25:01 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 15:25:01 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 15:25:01 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 15:25:01 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 15:25:01 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 15:25:01 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177632, maxMem=766075207 -2016-05-25 15:25:01 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 15:25:01 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263280, maxMem=766075207 -2016-05-25 15:25:01 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 15:25:01 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:52793 (size: 83.6 KB, free: 730.5 MB) -2016-05-25 15:25:01 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:52793 (size: 83.1 KB, free: 730.4 MB) -2016-05-25 15:25:01 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 15:25:01 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 15:25:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 431 ms on localhost (1/2) -2016-05-25 15:25:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 418 ms on localhost (2/2) -2016-05-25 15:25:01 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.454 s -2016-05-25 15:25:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 15:25:01 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.680228 s -2016-05-25 15:25:01 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 15:25:01 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 15:25:01 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 15:25:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:25:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:25:01 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 15:25:01 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348360, maxMem=766075207 -2016-05-25 15:25:01 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 15:25:01 INFO MemoryStore:59 - ensureFreeSpace(2549) called with curMem=353128, maxMem=766075207 -2016-05-25 15:25:01 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 15:25:01 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52793 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 15:25:01 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:01 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 15:25:01 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 15:25:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:01 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 15:25:01 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:25:01 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 15:25:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 28 ms on localhost (1/1) -2016-05-25 15:25:01 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.028 s -2016-05-25 15:25:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 15:25:01 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.046333 s -2016-05-25 15:25:01 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 15:25:01 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 15:25:01 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 15:25:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:25:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:25:01 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 15:25:01 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355677, maxMem=766075207 -2016-05-25 15:25:01 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 15:25:01 INFO MemoryStore:59 - ensureFreeSpace(2320) called with curMem=360013, maxMem=766075207 -2016-05-25 15:25:01 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 15:25:01 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52793 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 15:25:01 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 15:25:01 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 15:25:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:01 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 15:25:01 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 15:25:01 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:25:01 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:25:01 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 15:25:01 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 15:25:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 12 ms on localhost (1/2) -2016-05-25 15:25:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 12 ms on localhost (2/2) -2016-05-25 15:25:01 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.013 s -2016-05-25 15:25:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 15:25:01 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.026481 s -2016-05-25 15:25:01 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 15:25:01 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 15:25:01 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 15:25:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:25:01 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:25:01 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 15:25:01 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362333, maxMem=766075207 -2016-05-25 15:25:01 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 15:25:01 INFO MemoryStore:59 - ensureFreeSpace(2630) called with curMem=367397, maxMem=766075207 -2016-05-25 15:25:01 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 15:25:01 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52793 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:25:01 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 15:25:01 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 15:25:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 15:25:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 15:25:01 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 15:25:01 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 15:25:01 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:25:01 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:25:01 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 71600 bytes result sent to driver -2016-05-25 15:25:01 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 64189 bytes result sent to driver -2016-05-25 15:25:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 31 ms on localhost (1/2) -2016-05-25 15:25:01 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.036 s -2016-05-25 15:25:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 34 ms on localhost (2/2) -2016-05-25 15:25:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 15:25:01 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.057949 s -2016-05-25 15:25:03 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52793 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:25:03 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 15:25:03 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52793 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 15:25:03 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 15:25:03 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52793 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 15:25:03 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 15:25:03 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=348360, maxMem=766075207 -2016-05-25 15:25:03 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 15:25:03 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=348400, maxMem=766075207 -2016-05-25 15:25:03 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 15:25:03 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52793 (size: 81.0 B, free: 730.4 MB) -2016-05-25 15:25:03 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 15:25:03 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:25:03 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:25:03 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:25:03 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:25:03 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 15:25:03 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 15:25:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:25:03 INFO MemoryStore:59 - ensureFreeSpace(506296) called with curMem=348481, maxMem=766075207 -2016-05-25 15:25:03 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 494.4 KB, free 729.8 MB) -2016-05-25 15:25:03 INFO MemoryStore:59 - ensureFreeSpace(124191) called with curMem=854777, maxMem=766075207 -2016-05-25 15:25:03 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 121.3 KB, free 729.7 MB) -2016-05-25 15:25:03 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52793 (size: 121.3 KB, free: 730.3 MB) -2016-05-25 15:25:03 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:25:03 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 15:25:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:25:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:25:03 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 15:25:03 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 15:25:03 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 15:25:03 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:25:03 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 15:25:03 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:25:03 INFO MemoryStore:59 - ensureFreeSpace(105840) called with curMem=978968, maxMem=766075207 -2016-05-25 15:25:03 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 103.4 KB, free 729.6 MB) -2016-05-25 15:25:03 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:52793 (size: 103.4 KB, free: 730.2 MB) -2016-05-25 15:25:03 INFO MemoryStore:59 - ensureFreeSpace(117280) called with curMem=1084808, maxMem=766075207 -2016-05-25 15:25:03 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 114.5 KB, free 729.4 MB) -2016-05-25 15:25:03 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:52793 (size: 114.5 KB, free: 730.1 MB) -2016-05-25 15:25:03 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 15:25:03 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 15:25:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 285 ms on localhost (1/2) -2016-05-25 15:25:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 288 ms on localhost (2/2) -2016-05-25 15:25:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 15:25:03 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.290 s -2016-05-25 15:25:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:25:03 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:25:03 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 15:25:03 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:25:03 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 15:25:03 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:25:03 INFO MemoryStore:59 - ensureFreeSpace(221072) called with curMem=1202088, maxMem=766075207 -2016-05-25 15:25:03 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 215.9 KB, free 729.2 MB) -2016-05-25 15:25:03 INFO MemoryStore:59 - ensureFreeSpace(46729) called with curMem=1423160, maxMem=766075207 -2016-05-25 15:25:03 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 45.6 KB, free 729.2 MB) -2016-05-25 15:25:03 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52793 (size: 45.6 KB, free: 730.0 MB) -2016-05-25 15:25:03 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 15:25:03 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 15:25:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:25:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:25:03 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 15:25:03 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 15:25:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:25:03 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 15:25:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 9 ms -2016-05-25 15:25:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 13 ms -2016-05-25 15:25:03 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 15:25:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 179 ms on localhost (1/2) -2016-05-25 15:25:04 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 15:25:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 419 ms on localhost (2/2) -2016-05-25 15:25:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 15:25:04 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.420 s -2016-05-25 15:25:04 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.776273 s -2016-05-25 15:25:04 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=1469889, maxMem=766075207 -2016-05-25 15:25:04 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 40.0 B, free 729.2 MB) -2016-05-25 15:25:04 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=1469929, maxMem=766075207 -2016-05-25 15:25:04 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 81.0 B, free 729.2 MB) -2016-05-25 15:25:04 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52793 (size: 81.0 B, free: 730.0 MB) -2016-05-25 15:25:04 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 15:25:04 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:25:04 INFO DAGScheduler:59 - Registering RDD 17 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:25:04 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:25:04 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:25:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 15:25:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 15:25:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:25:04 INFO MemoryStore:59 - ensureFreeSpace(506648) called with curMem=1470010, maxMem=766075207 -2016-05-25 15:25:04 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 494.8 KB, free 728.7 MB) -2016-05-25 15:25:04 INFO MemoryStore:59 - ensureFreeSpace(124385) called with curMem=1976658, maxMem=766075207 -2016-05-25 15:25:04 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 121.5 KB, free 728.6 MB) -2016-05-25 15:25:04 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52793 (size: 121.5 KB, free: 729.9 MB) -2016-05-25 15:25:04 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:25:04 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 15:25:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:25:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:25:04 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 15:25:04 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-25 15:25:04 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 15:25:04 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 15:25:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 54 ms on localhost (1/2) -2016-05-25 15:25:04 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-25 15:25:04 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 15:25:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 75 ms on localhost (2/2) -2016-05-25 15:25:04 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.076 s -2016-05-25 15:25:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 15:25:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:25:04 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:25:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 15:25:04 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:25:04 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 15:25:04 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:25:04 INFO MemoryStore:59 - ensureFreeSpace(221072) called with curMem=2101043, maxMem=766075207 -2016-05-25 15:25:04 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 215.9 KB, free 728.4 MB) -2016-05-25 15:25:04 INFO MemoryStore:59 - ensureFreeSpace(46729) called with curMem=2322115, maxMem=766075207 -2016-05-25 15:25:04 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 45.6 KB, free 728.3 MB) -2016-05-25 15:25:04 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:52793 (size: 45.6 KB, free: 729.9 MB) -2016-05-25 15:25:04 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645) -2016-05-25 15:25:04 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 15:25:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:25:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:25:04 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 15:25:04 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 15:25:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:25:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 15:25:04 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 15:25:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 15:25:04 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 1161 bytes result sent to driver -2016-05-25 15:25:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 14 ms on localhost (1/2) -2016-05-25 15:25:04 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 15:25:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 87 ms on localhost (2/2) -2016-05-25 15:25:04 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.089 s -2016-05-25 15:25:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 15:25:04 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.221491 s -2016-05-25 15:25:04 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 15:25:04 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 15:25:04 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 15:25:04 INFO RandomForest:59 - init: 1.541976803 - total: 2.819199467 - findSplitsBins: 1.322541378 - findBestSplits: 1.253995502 - chooseSplits: 1.24662942 -2016-05-25 15:25:04 INFO SparkContext:59 - Starting job: count at DecisionTrees4ByVarianceRegression.scala:50 -2016-05-25 15:25:04 INFO DAGScheduler:59 - Got job 6 (count at DecisionTrees4ByVarianceRegression.scala:50) with 2 output partitions -2016-05-25 15:25:04 INFO DAGScheduler:59 - Final stage: ResultStage 8(count at DecisionTrees4ByVarianceRegression.scala:50) -2016-05-25 15:25:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:25:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:25:04 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 15:25:04 INFO MemoryStore:59 - ensureFreeSpace(3816) called with curMem=2145724, maxMem=766075207 -2016-05-25 15:25:04 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 3.7 KB, free 728.5 MB) -2016-05-25 15:25:04 INFO MemoryStore:59 - ensureFreeSpace(2142) called with curMem=2149540, maxMem=766075207 -2016-05-25 15:25:04 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.1 KB, free 728.5 MB) -2016-05-25 15:25:04 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:52793 (size: 2.1 KB, free: 730.1 MB) -2016-05-25 15:25:04 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 15:25:04 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 15:25:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:04 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 15:25:04 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 15:25:04 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:25:04 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:25:04 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 15:25:04 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 15:25:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 26 ms on localhost (1/2) -2016-05-25 15:25:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 26 ms on localhost (2/2) -2016-05-25 15:25:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 15:25:04 INFO DAGScheduler:59 - ResultStage 8 (count at DecisionTrees4ByVarianceRegression.scala:50) finished in 0.029 s -2016-05-25 15:25:04 INFO DAGScheduler:59 - Job 6 finished: count at DecisionTrees4ByVarianceRegression.scala:50, took 0.050579 s -2016-05-25 15:25:04 INFO SparkContext:59 - Starting job: count at DecisionTrees4ByVarianceRegression.scala:51 -2016-05-25 15:25:04 INFO DAGScheduler:59 - Got job 7 (count at DecisionTrees4ByVarianceRegression.scala:51) with 2 output partitions -2016-05-25 15:25:04 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at DecisionTrees4ByVarianceRegression.scala:51) -2016-05-25 15:25:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:25:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:25:04 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[7] at randomSplit at DecisionTrees4ByVarianceRegression.scala:29), which has no missing parents -2016-05-25 15:25:04 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2151682, maxMem=766075207 -2016-05-25 15:25:04 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.0 KB, free 728.5 MB) -2016-05-25 15:25:04 INFO MemoryStore:59 - ensureFreeSpace(2265) called with curMem=2155794, maxMem=766075207 -2016-05-25 15:25:04 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.5 MB) -2016-05-25 15:25:04 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:52793 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 15:25:04 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[7] at randomSplit at DecisionTrees4ByVarianceRegression.scala:29) -2016-05-25 15:25:04 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 15:25:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:04 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 15:25:04 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-25 15:25:04 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:25:04 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2082 bytes result sent to driver -2016-05-25 15:25:04 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:25:04 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2082 bytes result sent to driver -2016-05-25 15:25:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 12 ms on localhost (1/2) -2016-05-25 15:25:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 13 ms on localhost (2/2) -2016-05-25 15:25:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 15:25:04 INFO DAGScheduler:59 - ResultStage 9 (count at DecisionTrees4ByVarianceRegression.scala:51) finished in 0.015 s -2016-05-25 15:25:04 INFO DAGScheduler:59 - Job 7 finished: count at DecisionTrees4ByVarianceRegression.scala:51, took 0.037227 s -2016-05-25 15:25:04 INFO SparkContext:59 - Starting job: count at DecisionTrees4ByVarianceRegression.scala:52 -2016-05-25 15:25:04 INFO DAGScheduler:59 - Got job 8 (count at DecisionTrees4ByVarianceRegression.scala:52) with 2 output partitions -2016-05-25 15:25:04 INFO DAGScheduler:59 - Final stage: ResultStage 10(count at DecisionTrees4ByVarianceRegression.scala:52) -2016-05-25 15:25:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:25:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:25:04 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[8] at randomSplit at DecisionTrees4ByVarianceRegression.scala:29), which has no missing parents -2016-05-25 15:25:04 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2158059, maxMem=766075207 -2016-05-25 15:25:04 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 4.0 KB, free 728.5 MB) -2016-05-25 15:25:04 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=2162171, maxMem=766075207 -2016-05-25 15:25:04 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.5 MB) -2016-05-25 15:25:04 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:52793 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 15:25:04 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[8] at randomSplit at DecisionTrees4ByVarianceRegression.scala:29) -2016-05-25 15:25:04 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-25 15:25:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:04 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 15:25:04 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-25 15:25:04 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:25:04 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:25:04 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2082 bytes result sent to driver -2016-05-25 15:25:04 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2082 bytes result sent to driver -2016-05-25 15:25:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 11 ms on localhost (1/2) -2016-05-25 15:25:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 10 ms on localhost (2/2) -2016-05-25 15:25:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 15:25:04 INFO DAGScheduler:59 - ResultStage 10 (count at DecisionTrees4ByVarianceRegression.scala:52) finished in 0.013 s -2016-05-25 15:25:04 INFO DAGScheduler:59 - Job 8 finished: count at DecisionTrees4ByVarianceRegression.scala:52, took 0.030757 s -2016-05-25 15:25:04 INFO SparkContext:59 - Starting job: take at DecisionTrees4ByVarianceRegression.scala:58 -2016-05-25 15:25:04 INFO DAGScheduler:59 - Got job 9 (take at DecisionTrees4ByVarianceRegression.scala:58) with 1 output partitions -2016-05-25 15:25:04 INFO DAGScheduler:59 - Final stage: ResultStage 11(take at DecisionTrees4ByVarianceRegression.scala:58) -2016-05-25 15:25:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:25:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:25:04 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[20] at map at DecisionTrees4ByVarianceRegression.scala:43), which has no missing parents -2016-05-25 15:25:04 INFO MemoryStore:59 - ensureFreeSpace(6800) called with curMem=2164439, maxMem=766075207 -2016-05-25 15:25:04 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 6.6 KB, free 728.5 MB) -2016-05-25 15:25:04 INFO MemoryStore:59 - ensureFreeSpace(3647) called with curMem=2171239, maxMem=766075207 -2016-05-25 15:25:04 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.6 KB, free 728.5 MB) -2016-05-25 15:25:04 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:52793 (size: 3.6 KB, free: 730.1 MB) -2016-05-25 15:25:04 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:04 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 11 (MapPartitionsRDD[20] at map at DecisionTrees4ByVarianceRegression.scala:43) -2016-05-25 15:25:04 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 1 tasks -2016-05-25 15:25:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:04 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-25 15:25:04 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:25:04 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2377 bytes result sent to driver -2016-05-25 15:25:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 12 ms on localhost (1/1) -2016-05-25 15:25:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 15:25:04 INFO DAGScheduler:59 - ResultStage 11 (take at DecisionTrees4ByVarianceRegression.scala:58) finished in 0.014 s -2016-05-25 15:25:04 INFO DAGScheduler:59 - Job 9 finished: take at DecisionTrees4ByVarianceRegression.scala:58, took 0.031333 s -2016-05-25 15:25:04 INFO SparkContext:59 - Starting job: take at DecisionTrees4ByVarianceRegression.scala:58 -2016-05-25 15:25:04 INFO DAGScheduler:59 - Got job 10 (take at DecisionTrees4ByVarianceRegression.scala:58) with 1 output partitions -2016-05-25 15:25:04 INFO DAGScheduler:59 - Final stage: ResultStage 12(take at DecisionTrees4ByVarianceRegression.scala:58) -2016-05-25 15:25:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:25:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:25:04 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[20] at map at DecisionTrees4ByVarianceRegression.scala:43), which has no missing parents -2016-05-25 15:25:04 INFO MemoryStore:59 - ensureFreeSpace(6800) called with curMem=2174886, maxMem=766075207 -2016-05-25 15:25:04 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 6.6 KB, free 728.5 MB) -2016-05-25 15:25:04 INFO MemoryStore:59 - ensureFreeSpace(3647) called with curMem=2181686, maxMem=766075207 -2016-05-25 15:25:04 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.6 KB, free 728.5 MB) -2016-05-25 15:25:04 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:52793 (size: 3.6 KB, free: 730.1 MB) -2016-05-25 15:25:04 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:04 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 12 (MapPartitionsRDD[20] at map at DecisionTrees4ByVarianceRegression.scala:43) -2016-05-25 15:25:04 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 1 tasks -2016-05-25 15:25:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 22, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:04 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 22) -2016-05-25 15:25:04 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:25:04 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 22). 2182 bytes result sent to driver -2016-05-25 15:25:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 22) in 23 ms on localhost (1/1) -2016-05-25 15:25:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 15:25:04 INFO DAGScheduler:59 - ResultStage 12 (take at DecisionTrees4ByVarianceRegression.scala:58) finished in 0.023 s -2016-05-25 15:25:04 INFO DAGScheduler:59 - Job 10 finished: take at DecisionTrees4ByVarianceRegression.scala:58, took 0.041073 s -2016-05-25 15:25:04 INFO SparkContext:59 - Starting job: mean at DecisionTrees4ByVarianceRegression.scala:60 -2016-05-25 15:25:04 INFO DAGScheduler:59 - Got job 11 (mean at DecisionTrees4ByVarianceRegression.scala:60) with 2 output partitions -2016-05-25 15:25:04 INFO DAGScheduler:59 - Final stage: ResultStage 13(mean at DecisionTrees4ByVarianceRegression.scala:60) -2016-05-25 15:25:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:25:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:25:04 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[22] at mean at DecisionTrees4ByVarianceRegression.scala:60), which has no missing parents -2016-05-25 15:25:04 INFO MemoryStore:59 - ensureFreeSpace(7392) called with curMem=2185333, maxMem=766075207 -2016-05-25 15:25:04 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 7.2 KB, free 728.5 MB) -2016-05-25 15:25:04 INFO MemoryStore:59 - ensureFreeSpace(3889) called with curMem=2192725, maxMem=766075207 -2016-05-25 15:25:04 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.8 KB, free 728.5 MB) -2016-05-25 15:25:04 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:52793 (size: 3.8 KB, free: 730.1 MB) -2016-05-25 15:25:04 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[22] at mean at DecisionTrees4ByVarianceRegression.scala:60) -2016-05-25 15:25:04 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-25 15:25:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 23, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 24, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:04 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 24) -2016-05-25 15:25:04 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 23) -2016-05-25 15:25:04 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:25:04 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:25:04 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 23). 2209 bytes result sent to driver -2016-05-25 15:25:04 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 24). 2209 bytes result sent to driver -2016-05-25 15:25:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 23) in 13 ms on localhost (1/2) -2016-05-25 15:25:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 24) in 13 ms on localhost (2/2) -2016-05-25 15:25:04 INFO DAGScheduler:59 - ResultStage 13 (mean at DecisionTrees4ByVarianceRegression.scala:60) finished in 0.015 s -2016-05-25 15:25:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 15:25:04 INFO DAGScheduler:59 - Job 11 finished: mean at DecisionTrees4ByVarianceRegression.scala:60, took 0.034941 s -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 15:25:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 15:25:04 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 15:25:04 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 15:25:04 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 15:25:05 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 15:25:05 INFO BlockManager:59 - BlockManager stopped -2016-05-25 15:25:05 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 15:25:05 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 15:25:05 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 15:25:05 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 15:25:05 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-7d5f4812-09a0-43d2-bf63-c41579ad0054 -2016-05-25 15:25:05 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 15:25:05 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-25 15:25:27 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 15:25:28 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 15:25:29 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 15:25:29 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 15:25:29 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 15:25:29 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 15:25:29 INFO Remoting:74 - Starting remoting -2016-05-25 15:25:30 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:52814] -2016-05-25 15:25:30 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52814. -2016-05-25 15:25:30 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 15:25:30 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 15:25:30 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-ef7a1e1d-1321-4fce-8300-a4462b8edcba -2016-05-25 15:25:30 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 15:25:30 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-5afda8bb-b9ec-46ee-90d6-00a1aec04b11\httpd-71bdb98c-1885-4069-b2a3-5c74143cf37e -2016-05-25 15:25:30 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 15:25:30 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:25:30 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52815 -2016-05-25 15:25:30 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52815. -2016-05-25 15:25:30 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 15:25:30 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:25:30 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 15:25:30 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 15:25:30 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 15:25:30 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 15:25:30 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 15:25:31 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52834. -2016-05-25 15:25:31 INFO NettyBlockTransferService:59 - Server created on 52834 -2016-05-25 15:25:31 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 15:25:31 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52834 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52834) -2016-05-25 15:25:31 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 15:25:31 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 15:25:31 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 15:25:31 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 15:25:31 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 15:25:31 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52834 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 15:25:31 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 15:25:33 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 15:25:33 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 15:25:33 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 15:25:33 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 15:25:33 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 15:25:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:25:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:25:33 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 15:25:33 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 15:25:33 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 15:25:33 INFO MemoryStore:59 - ensureFreeSpace(2148) called with curMem=175484, maxMem=766075207 -2016-05-25 15:25:33 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 15:25:33 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52834 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:25:33 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 15:25:33 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 15:25:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:33 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 15:25:33 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 15:25:33 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 15:25:33 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 15:25:33 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 15:25:33 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 15:25:33 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 15:25:33 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 15:25:33 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 15:25:33 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 15:25:33 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 15:25:34 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177632, maxMem=766075207 -2016-05-25 15:25:34 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 15:25:34 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:52834 (size: 83.1 KB, free: 730.5 MB) -2016-05-25 15:25:34 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262712, maxMem=766075207 -2016-05-25 15:25:34 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 15:25:34 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:52834 (size: 83.6 KB, free: 730.4 MB) -2016-05-25 15:25:34 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 15:25:34 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 15:25:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 453 ms on localhost (1/2) -2016-05-25 15:25:34 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.498 s -2016-05-25 15:25:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 483 ms on localhost (2/2) -2016-05-25 15:25:34 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.697180 s -2016-05-25 15:25:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 15:25:34 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 15:25:34 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 15:25:34 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 15:25:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:25:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:25:34 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 15:25:34 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348360, maxMem=766075207 -2016-05-25 15:25:34 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 15:25:34 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=353128, maxMem=766075207 -2016-05-25 15:25:34 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 15:25:34 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52834 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 15:25:34 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:34 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 15:25:34 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 15:25:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:34 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 15:25:34 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:25:34 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 15:25:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 32 ms on localhost (1/1) -2016-05-25 15:25:34 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.033 s -2016-05-25 15:25:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 15:25:34 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.056312 s -2016-05-25 15:25:34 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 15:25:34 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 15:25:34 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 15:25:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:25:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:25:34 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 15:25:34 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355675, maxMem=766075207 -2016-05-25 15:25:34 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 15:25:34 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=360011, maxMem=766075207 -2016-05-25 15:25:34 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 15:25:34 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52834 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 15:25:34 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 15:25:34 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 15:25:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:34 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 15:25:34 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 15:25:34 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:25:34 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:25:34 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 15:25:34 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 15:25:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 14 ms on localhost (1/2) -2016-05-25 15:25:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 17 ms on localhost (2/2) -2016-05-25 15:25:34 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.017 s -2016-05-25 15:25:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 15:25:34 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.034921 s -2016-05-25 15:25:34 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 15:25:34 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 15:25:34 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 15:25:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:25:34 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:25:34 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 15:25:34 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362330, maxMem=766075207 -2016-05-25 15:25:34 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 15:25:34 INFO MemoryStore:59 - ensureFreeSpace(2633) called with curMem=367394, maxMem=766075207 -2016-05-25 15:25:34 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 15:25:34 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52834 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:25:34 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 15:25:34 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 15:25:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 15:25:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 15:25:34 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 15:25:34 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 15:25:34 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:25:34 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:25:34 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 54848 bytes result sent to driver -2016-05-25 15:25:34 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 54780 bytes result sent to driver -2016-05-25 15:25:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 33 ms on localhost (1/2) -2016-05-25 15:25:34 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.039 s -2016-05-25 15:25:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 35 ms on localhost (2/2) -2016-05-25 15:25:34 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.058225 s -2016-05-25 15:25:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 15:25:35 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52834 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:25:35 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 15:25:35 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52834 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 15:25:35 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 15:25:35 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52834 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 15:25:35 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 15:25:35 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52834 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-25 15:25:36 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=342308, maxMem=766075207 -2016-05-25 15:25:36 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 15:25:36 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=342348, maxMem=766075207 -2016-05-25 15:25:36 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 15:25:36 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52834 (size: 81.0 B, free: 730.4 MB) -2016-05-25 15:25:36 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 15:25:36 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:25:36 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:25:36 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:25:36 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:25:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 15:25:36 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 15:25:36 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:25:36 INFO MemoryStore:59 - ensureFreeSpace(446104) called with curMem=342429, maxMem=766075207 -2016-05-25 15:25:36 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 435.6 KB, free 729.8 MB) -2016-05-25 15:25:36 INFO MemoryStore:59 - ensureFreeSpace(109449) called with curMem=788533, maxMem=766075207 -2016-05-25 15:25:36 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 106.9 KB, free 729.7 MB) -2016-05-25 15:25:36 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52834 (size: 106.9 KB, free: 730.3 MB) -2016-05-25 15:25:36 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:25:36 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 15:25:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:25:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:25:36 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 15:25:36 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 15:25:36 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 15:25:36 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:25:36 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 15:25:36 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:25:36 INFO MemoryStore:59 - ensureFreeSpace(94400) called with curMem=897982, maxMem=766075207 -2016-05-25 15:25:36 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 92.2 KB, free 729.6 MB) -2016-05-25 15:25:36 INFO MemoryStore:59 - ensureFreeSpace(91536) called with curMem=992382, maxMem=766075207 -2016-05-25 15:25:36 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 89.4 KB, free 729.6 MB) -2016-05-25 15:25:36 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:52834 (size: 92.2 KB, free: 730.2 MB) -2016-05-25 15:25:36 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:52834 (size: 89.4 KB, free: 730.1 MB) -2016-05-25 15:25:36 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 15:25:36 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 15:25:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 255 ms on localhost (1/2) -2016-05-25 15:25:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 258 ms on localhost (2/2) -2016-05-25 15:25:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 15:25:36 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.259 s -2016-05-25 15:25:36 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:25:36 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:25:36 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 15:25:36 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:25:36 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 15:25:36 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:25:36 INFO MemoryStore:59 - ensureFreeSpace(191504) called with curMem=1083918, maxMem=766075207 -2016-05-25 15:25:36 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 187.0 KB, free 729.4 MB) -2016-05-25 15:25:36 INFO MemoryStore:59 - ensureFreeSpace(41109) called with curMem=1275422, maxMem=766075207 -2016-05-25 15:25:36 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 40.1 KB, free 729.3 MB) -2016-05-25 15:25:36 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52834 (size: 40.1 KB, free: 730.1 MB) -2016-05-25 15:25:36 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 15:25:36 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 15:25:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:25:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:25:36 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 15:25:36 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 15:25:36 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 15:25:36 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:25:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 9 ms -2016-05-25 15:25:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 9 ms -2016-05-25 15:25:36 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 15:25:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 142 ms on localhost (1/2) -2016-05-25 15:25:36 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 15:25:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 389 ms on localhost (2/2) -2016-05-25 15:25:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 15:25:36 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.390 s -2016-05-25 15:25:36 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.715865 s -2016-05-25 15:25:36 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=1316531, maxMem=766075207 -2016-05-25 15:25:36 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 40.0 B, free 729.3 MB) -2016-05-25 15:25:36 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=1316571, maxMem=766075207 -2016-05-25 15:25:36 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 81.0 B, free 729.3 MB) -2016-05-25 15:25:36 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52834 (size: 81.0 B, free: 730.1 MB) -2016-05-25 15:25:36 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 15:25:36 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:25:36 INFO DAGScheduler:59 - Registering RDD 17 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:25:36 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:25:36 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:25:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 15:25:36 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 15:25:36 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:25:36 INFO MemoryStore:59 - ensureFreeSpace(446456) called with curMem=1316652, maxMem=766075207 -2016-05-25 15:25:36 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 436.0 KB, free 728.9 MB) -2016-05-25 15:25:36 INFO MemoryStore:59 - ensureFreeSpace(109645) called with curMem=1763108, maxMem=766075207 -2016-05-25 15:25:36 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 107.1 KB, free 728.8 MB) -2016-05-25 15:25:37 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52834 (size: 107.1 KB, free: 730.0 MB) -2016-05-25 15:25:37 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:25:37 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 15:25:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:25:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:25:37 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 15:25:37 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 15:25:37 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-25 15:25:37 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-25 15:25:37 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 15:25:37 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 15:25:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 68 ms on localhost (1/2) -2016-05-25 15:25:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 69 ms on localhost (2/2) -2016-05-25 15:25:37 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.070 s -2016-05-25 15:25:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:25:37 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:25:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 15:25:37 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:25:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 15:25:37 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 15:25:37 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:25:37 INFO MemoryStore:59 - ensureFreeSpace(191504) called with curMem=1872753, maxMem=766075207 -2016-05-25 15:25:37 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 187.0 KB, free 728.6 MB) -2016-05-25 15:25:37 INFO MemoryStore:59 - ensureFreeSpace(41109) called with curMem=2064257, maxMem=766075207 -2016-05-25 15:25:37 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 40.1 KB, free 728.6 MB) -2016-05-25 15:25:37 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:52834 (size: 40.1 KB, free: 729.9 MB) -2016-05-25 15:25:37 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645) -2016-05-25 15:25:37 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 15:25:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:25:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:25:37 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 15:25:37 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 15:25:37 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 15:25:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 15:25:37 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:25:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 15:25:37 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 1161 bytes result sent to driver -2016-05-25 15:25:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 16 ms on localhost (1/2) -2016-05-25 15:25:37 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 15:25:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 77 ms on localhost (2/2) -2016-05-25 15:25:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 15:25:37 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.078 s -2016-05-25 15:25:37 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.196007 s -2016-05-25 15:25:37 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 15:25:37 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 15:25:37 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 15:25:37 INFO RandomForest:59 - init: 1.672517261 - total: 2.860578122 - findSplitsBins: 1.403603972 - findBestSplits: 1.166709578 - chooseSplits: 1.160922643 -2016-05-25 15:25:37 INFO SparkContext:59 - Starting job: count at DecisionTrees4ByVarianceRegression.scala:50 -2016-05-25 15:25:37 INFO DAGScheduler:59 - Got job 6 (count at DecisionTrees4ByVarianceRegression.scala:50) with 2 output partitions -2016-05-25 15:25:37 INFO DAGScheduler:59 - Final stage: ResultStage 8(count at DecisionTrees4ByVarianceRegression.scala:50) -2016-05-25 15:25:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:25:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:25:37 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 15:25:37 INFO MemoryStore:59 - ensureFreeSpace(3816) called with curMem=1919430, maxMem=766075207 -2016-05-25 15:25:37 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 3.7 KB, free 728.8 MB) -2016-05-25 15:25:37 INFO MemoryStore:59 - ensureFreeSpace(2142) called with curMem=1923246, maxMem=766075207 -2016-05-25 15:25:37 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.1 KB, free 728.8 MB) -2016-05-25 15:25:37 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:52834 (size: 2.1 KB, free: 730.1 MB) -2016-05-25 15:25:37 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 15:25:37 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 15:25:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:37 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 15:25:37 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 15:25:37 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:25:37 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:25:37 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 15:25:37 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 15:25:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 11 ms on localhost (1/2) -2016-05-25 15:25:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 11 ms on localhost (2/2) -2016-05-25 15:25:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 15:25:37 INFO DAGScheduler:59 - ResultStage 8 (count at DecisionTrees4ByVarianceRegression.scala:50) finished in 0.013 s -2016-05-25 15:25:37 INFO DAGScheduler:59 - Job 6 finished: count at DecisionTrees4ByVarianceRegression.scala:50, took 0.031253 s -2016-05-25 15:25:37 INFO SparkContext:59 - Starting job: count at DecisionTrees4ByVarianceRegression.scala:51 -2016-05-25 15:25:37 INFO DAGScheduler:59 - Got job 7 (count at DecisionTrees4ByVarianceRegression.scala:51) with 2 output partitions -2016-05-25 15:25:37 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at DecisionTrees4ByVarianceRegression.scala:51) -2016-05-25 15:25:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:25:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:25:37 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[7] at randomSplit at DecisionTrees4ByVarianceRegression.scala:29), which has no missing parents -2016-05-25 15:25:37 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1925388, maxMem=766075207 -2016-05-25 15:25:37 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.0 KB, free 728.7 MB) -2016-05-25 15:25:37 INFO MemoryStore:59 - ensureFreeSpace(2265) called with curMem=1929500, maxMem=766075207 -2016-05-25 15:25:37 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.7 MB) -2016-05-25 15:25:37 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:52834 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 15:25:37 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[7] at randomSplit at DecisionTrees4ByVarianceRegression.scala:29) -2016-05-25 15:25:37 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 15:25:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:37 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 15:25:37 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-25 15:25:37 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:25:37 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2082 bytes result sent to driver -2016-05-25 15:25:37 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:25:37 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2082 bytes result sent to driver -2016-05-25 15:25:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 10 ms on localhost (1/2) -2016-05-25 15:25:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 10 ms on localhost (2/2) -2016-05-25 15:25:37 INFO DAGScheduler:59 - ResultStage 9 (count at DecisionTrees4ByVarianceRegression.scala:51) finished in 0.011 s -2016-05-25 15:25:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 15:25:37 INFO DAGScheduler:59 - Job 7 finished: count at DecisionTrees4ByVarianceRegression.scala:51, took 0.030741 s -2016-05-25 15:25:37 INFO SparkContext:59 - Starting job: count at DecisionTrees4ByVarianceRegression.scala:52 -2016-05-25 15:25:37 INFO DAGScheduler:59 - Got job 8 (count at DecisionTrees4ByVarianceRegression.scala:52) with 2 output partitions -2016-05-25 15:25:37 INFO DAGScheduler:59 - Final stage: ResultStage 10(count at DecisionTrees4ByVarianceRegression.scala:52) -2016-05-25 15:25:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:25:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:25:37 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[8] at randomSplit at DecisionTrees4ByVarianceRegression.scala:29), which has no missing parents -2016-05-25 15:25:37 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1931765, maxMem=766075207 -2016-05-25 15:25:37 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 4.0 KB, free 728.7 MB) -2016-05-25 15:25:37 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=1935877, maxMem=766075207 -2016-05-25 15:25:37 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.7 MB) -2016-05-25 15:25:37 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:52834 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 15:25:37 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[8] at randomSplit at DecisionTrees4ByVarianceRegression.scala:29) -2016-05-25 15:25:37 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-25 15:25:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:37 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 15:25:37 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-25 15:25:37 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:25:37 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:25:37 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2082 bytes result sent to driver -2016-05-25 15:25:37 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2082 bytes result sent to driver -2016-05-25 15:25:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 9 ms on localhost (1/2) -2016-05-25 15:25:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 12 ms on localhost (2/2) -2016-05-25 15:25:37 INFO DAGScheduler:59 - ResultStage 10 (count at DecisionTrees4ByVarianceRegression.scala:52) finished in 0.013 s -2016-05-25 15:25:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 15:25:37 INFO DAGScheduler:59 - Job 8 finished: count at DecisionTrees4ByVarianceRegression.scala:52, took 0.026704 s -2016-05-25 15:25:37 INFO SparkContext:59 - Starting job: take at DecisionTrees4ByVarianceRegression.scala:58 -2016-05-25 15:25:37 INFO DAGScheduler:59 - Got job 9 (take at DecisionTrees4ByVarianceRegression.scala:58) with 1 output partitions -2016-05-25 15:25:37 INFO DAGScheduler:59 - Final stage: ResultStage 11(take at DecisionTrees4ByVarianceRegression.scala:58) -2016-05-25 15:25:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:25:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:25:37 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[20] at map at DecisionTrees4ByVarianceRegression.scala:43), which has no missing parents -2016-05-25 15:25:37 INFO MemoryStore:59 - ensureFreeSpace(6800) called with curMem=1938145, maxMem=766075207 -2016-05-25 15:25:37 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 6.6 KB, free 728.7 MB) -2016-05-25 15:25:37 INFO MemoryStore:59 - ensureFreeSpace(3644) called with curMem=1944945, maxMem=766075207 -2016-05-25 15:25:37 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.6 KB, free 728.7 MB) -2016-05-25 15:25:37 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:52834 (size: 3.6 KB, free: 730.1 MB) -2016-05-25 15:25:37 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:37 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 11 (MapPartitionsRDD[20] at map at DecisionTrees4ByVarianceRegression.scala:43) -2016-05-25 15:25:37 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 1 tasks -2016-05-25 15:25:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:37 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-25 15:25:37 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:25:37 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2401 bytes result sent to driver -2016-05-25 15:25:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 12 ms on localhost (1/1) -2016-05-25 15:25:37 INFO DAGScheduler:59 - ResultStage 11 (take at DecisionTrees4ByVarianceRegression.scala:58) finished in 0.013 s -2016-05-25 15:25:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 15:25:37 INFO DAGScheduler:59 - Job 9 finished: take at DecisionTrees4ByVarianceRegression.scala:58, took 0.029375 s -2016-05-25 15:25:37 INFO SparkContext:59 - Starting job: mean at DecisionTrees4ByVarianceRegression.scala:60 -2016-05-25 15:25:37 INFO DAGScheduler:59 - Got job 10 (mean at DecisionTrees4ByVarianceRegression.scala:60) with 2 output partitions -2016-05-25 15:25:37 INFO DAGScheduler:59 - Final stage: ResultStage 12(mean at DecisionTrees4ByVarianceRegression.scala:60) -2016-05-25 15:25:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:25:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:25:37 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[22] at mean at DecisionTrees4ByVarianceRegression.scala:60), which has no missing parents -2016-05-25 15:25:37 INFO MemoryStore:59 - ensureFreeSpace(7392) called with curMem=1948589, maxMem=766075207 -2016-05-25 15:25:37 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 7.2 KB, free 728.7 MB) -2016-05-25 15:25:37 INFO MemoryStore:59 - ensureFreeSpace(3894) called with curMem=1955981, maxMem=766075207 -2016-05-25 15:25:37 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.8 KB, free 728.7 MB) -2016-05-25 15:25:37 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:52834 (size: 3.8 KB, free: 730.1 MB) -2016-05-25 15:25:37 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:25:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[22] at mean at DecisionTrees4ByVarianceRegression.scala:60) -2016-05-25 15:25:37 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-25 15:25:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 22, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:25:37 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 22) -2016-05-25 15:25:37 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 23) -2016-05-25 15:25:37 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:25:37 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:25:37 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 23). 2209 bytes result sent to driver -2016-05-25 15:25:37 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 22). 2209 bytes result sent to driver -2016-05-25 15:25:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 23) in 12 ms on localhost (1/2) -2016-05-25 15:25:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 22) in 13 ms on localhost (2/2) -2016-05-25 15:25:37 INFO DAGScheduler:59 - ResultStage 12 (mean at DecisionTrees4ByVarianceRegression.scala:60) finished in 0.013 s -2016-05-25 15:25:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 15:25:37 INFO DAGScheduler:59 - Job 10 finished: mean at DecisionTrees4ByVarianceRegression.scala:60, took 0.030367 s -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 15:25:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 15:25:37 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 15:25:37 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 15:25:37 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 15:25:37 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 15:25:37 INFO BlockManager:59 - BlockManager stopped -2016-05-25 15:25:37 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 15:25:37 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 15:25:37 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 15:25:37 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 15:25:37 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-5afda8bb-b9ec-46ee-90d6-00a1aec04b11 -2016-05-25 15:30:08 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 15:30:09 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 15:30:09 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 15:30:09 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 15:30:09 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 15:30:10 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 15:30:10 INFO Remoting:74 - Starting remoting -2016-05-25 15:30:10 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53071] -2016-05-25 15:30:10 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53071. -2016-05-25 15:30:10 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 15:30:10 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 15:30:10 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-746a90d0-e9cb-4e9f-8936-0fa8b5a68aec -2016-05-25 15:30:10 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 15:30:11 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-aa388218-e1bc-4af0-9f69-f7004daf6426\httpd-ac4619f7-17e3-471c-9c42-b5bd87b4555c -2016-05-25 15:30:11 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 15:30:11 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:30:11 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53072 -2016-05-25 15:30:11 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53072. -2016-05-25 15:30:11 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 15:30:11 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:30:11 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 15:30:11 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 15:30:11 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 15:30:11 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 15:30:11 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 15:30:11 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53091. -2016-05-25 15:30:11 INFO NettyBlockTransferService:59 - Server created on 53091 -2016-05-25 15:30:11 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 15:30:11 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53091 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53091) -2016-05-25 15:30:11 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 15:30:12 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 15:30:12 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 15:30:12 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 15:30:12 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 15:30:12 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53091 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 15:30:12 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 15:30:14 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 15:30:14 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 15:30:14 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 15:30:14 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 15:30:14 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 15:30:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:30:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:30:14 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 15:30:14 INFO MemoryStore:59 - ensureFreeSpace(3888) called with curMem=171580, maxMem=766075207 -2016-05-25 15:30:14 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 15:30:14 INFO MemoryStore:59 - ensureFreeSpace(2131) called with curMem=175468, maxMem=766075207 -2016-05-25 15:30:14 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 15:30:14 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53091 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:30:14 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:30:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 15:30:14 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 15:30:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 15:30:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 15:30:14 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 15:30:14 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 15:30:14 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 15:30:14 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 15:30:14 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/dt.txt:0+57 -2016-05-25 15:30:14 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/dt.txt:57+57 -2016-05-25 15:30:14 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 15:30:14 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 15:30:14 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 15:30:14 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 15:30:14 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 15:30:14 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=177599, maxMem=766075207 -2016-05-25 15:30:14 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 280.0 B, free 730.4 MB) -2016-05-25 15:30:14 INFO MemoryStore:59 - ensureFreeSpace(544) called with curMem=177879, maxMem=766075207 -2016-05-25 15:30:14 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:53091 (size: 280.0 B, free: 730.6 MB) -2016-05-25 15:30:14 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 544.0 B, free 730.4 MB) -2016-05-25 15:30:14 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:53091 (size: 544.0 B, free: 730.6 MB) -2016-05-25 15:30:14 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 15:30:14 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 15:30:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 123 ms on localhost (1/2) -2016-05-25 15:30:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 107 ms on localhost (2/2) -2016-05-25 15:30:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 15:30:14 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.145 s -2016-05-25 15:30:14 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.321634 s -2016-05-25 15:30:15 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 15:30:15 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 15:30:15 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 15:30:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:30:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:30:15 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[8] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 15:30:15 INFO MemoryStore:59 - ensureFreeSpace(4456) called with curMem=178423, maxMem=766075207 -2016-05-25 15:30:15 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-25 15:30:15 INFO MemoryStore:59 - ensureFreeSpace(2401) called with curMem=182879, maxMem=766075207 -2016-05-25 15:30:15 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-25 15:30:15 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53091 (size: 2.3 KB, free: 730.6 MB) -2016-05-25 15:30:15 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:30:15 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[8] at map at DecisionTreeMetadata.scala:110) -2016-05-25 15:30:15 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 15:30:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 15:30:15 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 15:30:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:30:15 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 15:30:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 56 ms on localhost (1/1) -2016-05-25 15:30:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 15:30:15 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.061 s -2016-05-25 15:30:15 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.083671 s -2016-05-25 15:30:15 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 15:30:15 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 15:30:15 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 15:30:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:30:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:30:15 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 15:30:15 INFO MemoryStore:59 - ensureFreeSpace(4016) called with curMem=185280, maxMem=766075207 -2016-05-25 15:30:15 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-05-25 15:30:15 INFO MemoryStore:59 - ensureFreeSpace(2184) called with curMem=189296, maxMem=766075207 -2016-05-25 15:30:15 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 15:30:15 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53091 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:30:15 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:30:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at retag at RandomForest.scala:137) -2016-05-25 15:30:15 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 15:30:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 15:30:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 15:30:15 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 15:30:15 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 15:30:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:30:15 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 15:30:15 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:30:15 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 15:30:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 22 ms on localhost (1/2) -2016-05-25 15:30:15 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.027 s -2016-05-25 15:30:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 25 ms on localhost (2/2) -2016-05-25 15:30:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 15:30:15 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.058503 s -2016-05-25 15:30:15 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 15:30:15 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 15:30:15 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 15:30:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:30:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:30:15 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[9] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 15:30:15 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=191480, maxMem=766075207 -2016-05-25 15:30:15 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.6 KB, free 730.4 MB) -2016-05-25 15:30:15 INFO MemoryStore:59 - ensureFreeSpace(2491) called with curMem=196224, maxMem=766075207 -2016-05-25 15:30:15 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-05-25 15:30:15 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53091 (size: 2.4 KB, free: 730.6 MB) -2016-05-25 15:30:15 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:30:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[9] at sample at DecisionTree.scala:989) -2016-05-25 15:30:15 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 15:30:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 15:30:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 15:30:15 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 15:30:15 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 15:30:15 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:30:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:30:15 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2493 bytes result sent to driver -2016-05-25 15:30:15 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2677 bytes result sent to driver -2016-05-25 15:30:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 20 ms on localhost (1/2) -2016-05-25 15:30:15 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.024 s -2016-05-25 15:30:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 23 ms on localhost (2/2) -2016-05-25 15:30:15 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.055537 s -2016-05-25 15:30:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 15:30:16 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53091 in memory (size: 2.4 KB, free: 730.6 MB) -2016-05-25 15:30:16 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 15:30:16 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53091 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:30:16 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 15:30:16 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53091 in memory (size: 2.3 KB, free: 730.6 MB) -2016-05-25 15:30:16 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 15:30:16 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53091 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:30:16 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 15:30:16 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=172404, maxMem=766075207 -2016-05-25 15:30:16 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 200.0 B, free 730.4 MB) -2016-05-25 15:30:16 INFO MemoryStore:59 - ensureFreeSpace(303) called with curMem=172604, maxMem=766075207 -2016-05-25 15:30:16 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 303.0 B, free 730.4 MB) -2016-05-25 15:30:16 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53091 (size: 303.0 B, free: 730.6 MB) -2016-05-25 15:30:16 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 15:30:16 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:30:16 INFO DAGScheduler:59 - Registering RDD 12 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:30:16 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:30:16 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:30:16 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 15:30:16 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 15:30:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[12] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:30:16 INFO MemoryStore:59 - ensureFreeSpace(10272) called with curMem=172907, maxMem=766075207 -2016-05-25 15:30:16 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 10.0 KB, free 730.4 MB) -2016-05-25 15:30:16 INFO MemoryStore:59 - ensureFreeSpace(5144) called with curMem=183179, maxMem=766075207 -2016-05-25 15:30:16 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 15:30:16 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53091 (size: 5.0 KB, free: 730.6 MB) -2016-05-25 15:30:16 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:30:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[12] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:30:16 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 15:30:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 15:30:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 15:30:16 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 15:30:16 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 15:30:16 INFO CacheManager:59 - Partition rdd_11_1 not found, computing it -2016-05-25 15:30:16 INFO CacheManager:59 - Partition rdd_11_0 not found, computing it -2016-05-25 15:30:16 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:30:16 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:30:16 INFO MemoryStore:59 - ensureFreeSpace(264) called with curMem=188323, maxMem=766075207 -2016-05-25 15:30:16 INFO MemoryStore:59 - Block rdd_11_1 stored as values in memory (estimated size 264.0 B, free 730.4 MB) -2016-05-25 15:30:16 INFO MemoryStore:59 - ensureFreeSpace(512) called with curMem=188587, maxMem=766075207 -2016-05-25 15:30:16 INFO MemoryStore:59 - Block rdd_11_0 stored as values in memory (estimated size 512.0 B, free 730.4 MB) -2016-05-25 15:30:16 INFO BlockManagerInfo:59 - Added rdd_11_1 in memory on localhost:53091 (size: 264.0 B, free: 730.6 MB) -2016-05-25 15:30:16 INFO BlockManagerInfo:59 - Added rdd_11_0 in memory on localhost:53091 (size: 512.0 B, free: 730.6 MB) -2016-05-25 15:30:16 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 15:30:16 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 15:30:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 197 ms on localhost (1/2) -2016-05-25 15:30:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 196 ms on localhost (2/2) -2016-05-25 15:30:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 15:30:16 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.201 s -2016-05-25 15:30:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:30:16 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:30:16 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 15:30:16 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:30:16 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 15:30:16 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[14] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:30:16 INFO MemoryStore:59 - ensureFreeSpace(5352) called with curMem=189099, maxMem=766075207 -2016-05-25 15:30:16 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.2 KB, free 730.4 MB) -2016-05-25 15:30:16 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=194451, maxMem=766075207 -2016-05-25 15:30:16 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-25 15:30:16 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53091 (size: 2.8 KB, free: 730.6 MB) -2016-05-25 15:30:16 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:30:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[14] at map at DecisionTree.scala:645) -2016-05-25 15:30:16 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 15:30:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:30:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:30:16 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 15:30:16 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 15:30:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:30:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:30:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 8 ms -2016-05-25 15:30:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 8 ms -2016-05-25 15:30:16 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2905 bytes result sent to driver -2016-05-25 15:30:16 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2774 bytes result sent to driver -2016-05-25 15:30:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 115 ms on localhost (1/2) -2016-05-25 15:30:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 114 ms on localhost (2/2) -2016-05-25 15:30:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 15:30:16 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.118 s -2016-05-25 15:30:16 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.379815 s -2016-05-25 15:30:16 INFO MemoryStore:59 - ensureFreeSpace(104) called with curMem=197347, maxMem=766075207 -2016-05-25 15:30:16 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 104.0 B, free 730.4 MB) -2016-05-25 15:30:16 INFO MemoryStore:59 - ensureFreeSpace(264) called with curMem=197451, maxMem=766075207 -2016-05-25 15:30:16 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 264.0 B, free 730.4 MB) -2016-05-25 15:30:16 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53091 (size: 264.0 B, free: 730.6 MB) -2016-05-25 15:30:16 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 15:30:16 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:30:16 INFO DAGScheduler:59 - Registering RDD 15 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:30:16 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:30:16 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:30:16 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 15:30:16 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 15:30:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[15] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:30:16 INFO MemoryStore:59 - ensureFreeSpace(10688) called with curMem=197715, maxMem=766075207 -2016-05-25 15:30:16 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 10.4 KB, free 730.4 MB) -2016-05-25 15:30:16 INFO MemoryStore:59 - ensureFreeSpace(5328) called with curMem=208403, maxMem=766075207 -2016-05-25 15:30:16 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 5.2 KB, free 730.4 MB) -2016-05-25 15:30:16 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53091 (size: 5.2 KB, free: 730.6 MB) -2016-05-25 15:30:16 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:30:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[15] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:30:16 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 15:30:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 15:30:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 15:30:16 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 15:30:16 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 15:30:16 INFO BlockManager:59 - Found block rdd_11_1 locally -2016-05-25 15:30:16 INFO BlockManager:59 - Found block rdd_11_0 locally -2016-05-25 15:30:16 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 15:30:16 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 15:30:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 27 ms on localhost (1/2) -2016-05-25 15:30:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 31 ms on localhost (2/2) -2016-05-25 15:30:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 15:30:16 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.031 s -2016-05-25 15:30:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:30:16 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:30:16 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 15:30:16 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:30:16 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 15:30:16 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[17] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:30:16 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=213731, maxMem=766075207 -2016-05-25 15:30:16 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-05-25 15:30:16 INFO MemoryStore:59 - ensureFreeSpace(2866) called with curMem=218963, maxMem=766075207 -2016-05-25 15:30:16 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-25 15:30:16 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53091 (size: 2.8 KB, free: 730.6 MB) -2016-05-25 15:30:16 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:30:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[17] at map at DecisionTree.scala:645) -2016-05-25 15:30:16 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 15:30:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:30:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:30:16 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 15:30:16 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 15:30:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:30:16 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 15:30:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 15:30:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 15:30:16 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 1161 bytes result sent to driver -2016-05-25 15:30:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 15 ms on localhost (1/2) -2016-05-25 15:30:16 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 15:30:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 26 ms on localhost (2/2) -2016-05-25 15:30:16 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.026 s -2016-05-25 15:30:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 15:30:16 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.096768 s -2016-05-25 15:30:16 INFO MapPartitionsRDD:59 - Removing RDD 11 from persistence list -2016-05-25 15:30:16 INFO BlockManager:59 - Removing RDD 11 -2016-05-25 15:30:16 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 15:30:16 INFO RandomForest:59 - init: 1.181406936 - total: 1.826250077 - findSplitsBins: 0.973028219 - findBestSplits: 0.623823394 - chooseSplits: 0.61465952 -2016-05-25 15:30:16 INFO SparkContext:59 - Starting job: take at RandomForests1.scala:37 -2016-05-25 15:30:16 INFO DAGScheduler:59 - Got job 6 (take at RandomForests1.scala:37) with 1 output partitions -2016-05-25 15:30:16 INFO DAGScheduler:59 - Final stage: ResultStage 8(take at RandomForests1.scala:37) -2016-05-25 15:30:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:30:16 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:30:16 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 15:30:16 INFO MemoryStore:59 - ensureFreeSpace(3968) called with curMem=221053, maxMem=766075207 -2016-05-25 15:30:16 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-05-25 15:30:16 INFO MemoryStore:59 - ensureFreeSpace(2186) called with curMem=225021, maxMem=766075207 -2016-05-25 15:30:16 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 15:30:16 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53091 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:30:16 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:30:16 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 15:30:16 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-05-25 15:30:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 15:30:16 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 15:30:16 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:30:16 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2493 bytes result sent to driver -2016-05-25 15:30:16 INFO DAGScheduler:59 - ResultStage 8 (take at RandomForests1.scala:37) finished in 0.011 s -2016-05-25 15:30:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 11 ms on localhost (1/1) -2016-05-25 15:30:16 INFO DAGScheduler:59 - Job 6 finished: take at RandomForests1.scala:37, took 0.031295 s -2016-05-25 15:30:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 15:30:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 15:30:16 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 15:30:16 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 15:30:17 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 15:30:17 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 15:30:17 INFO BlockManager:59 - BlockManager stopped -2016-05-25 15:30:17 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 15:30:17 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 15:30:17 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 15:30:17 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 15:30:17 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-aa388218-e1bc-4af0-9f69-f7004daf6426 -2016-05-25 15:30:17 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 15:31:48 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 15:31:48 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 15:31:49 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 15:31:49 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 15:31:49 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 15:31:50 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 15:31:50 INFO Remoting:74 - Starting remoting -2016-05-25 15:31:50 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53161] -2016-05-25 15:31:50 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53161. -2016-05-25 15:31:50 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 15:31:50 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 15:31:50 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-23e63ed2-bf70-4901-81ae-4fcf5cd86650 -2016-05-25 15:31:50 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 15:31:50 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-49b0503d-54b0-45cb-bfde-8e948c7a9d19\httpd-a02df7dc-7bfe-421d-86e0-690ec4014405 -2016-05-25 15:31:50 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 15:31:50 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:31:50 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53162 -2016-05-25 15:31:50 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53162. -2016-05-25 15:31:50 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 15:31:50 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:31:50 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 15:31:50 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 15:31:50 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 15:31:50 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 15:31:50 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 15:31:51 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53181. -2016-05-25 15:31:51 INFO NettyBlockTransferService:59 - Server created on 53181 -2016-05-25 15:31:51 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 15:31:51 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53181 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53181) -2016-05-25 15:31:51 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 15:31:51 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 15:31:51 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 15:31:51 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 15:31:51 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 15:31:51 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53181 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 15:31:51 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 15:31:53 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 15:31:53 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 15:31:53 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 15:31:53 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 15:31:53 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 15:31:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:31:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:31:53 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 15:31:53 INFO MemoryStore:59 - ensureFreeSpace(3888) called with curMem=171580, maxMem=766075207 -2016-05-25 15:31:53 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 15:31:53 INFO MemoryStore:59 - ensureFreeSpace(2131) called with curMem=175468, maxMem=766075207 -2016-05-25 15:31:53 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 15:31:53 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53181 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:31:53 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:31:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 15:31:53 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 15:31:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 15:31:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 15:31:53 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 15:31:53 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 15:31:53 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 15:31:53 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 15:31:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/dt.txt:0+57 -2016-05-25 15:31:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/dt.txt:57+57 -2016-05-25 15:31:53 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 15:31:53 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 15:31:53 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 15:31:53 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 15:31:53 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 15:31:53 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=177599, maxMem=766075207 -2016-05-25 15:31:53 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 280.0 B, free 730.4 MB) -2016-05-25 15:31:53 INFO MemoryStore:59 - ensureFreeSpace(544) called with curMem=177879, maxMem=766075207 -2016-05-25 15:31:53 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 544.0 B, free 730.4 MB) -2016-05-25 15:31:53 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:53181 (size: 280.0 B, free: 730.6 MB) -2016-05-25 15:31:53 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:53181 (size: 544.0 B, free: 730.6 MB) -2016-05-25 15:31:54 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 15:31:54 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 15:31:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 101 ms on localhost (1/2) -2016-05-25 15:31:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 123 ms on localhost (2/2) -2016-05-25 15:31:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 15:31:54 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.141 s -2016-05-25 15:31:54 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.284309 s -2016-05-25 15:31:54 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 15:31:54 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 15:31:54 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 15:31:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:31:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:31:54 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[8] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 15:31:54 INFO MemoryStore:59 - ensureFreeSpace(4456) called with curMem=178423, maxMem=766075207 -2016-05-25 15:31:54 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-25 15:31:54 INFO MemoryStore:59 - ensureFreeSpace(2401) called with curMem=182879, maxMem=766075207 -2016-05-25 15:31:54 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-05-25 15:31:54 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53181 (size: 2.3 KB, free: 730.6 MB) -2016-05-25 15:31:54 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:31:54 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[8] at map at DecisionTreeMetadata.scala:110) -2016-05-25 15:31:54 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 15:31:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 15:31:54 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 15:31:54 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:31:54 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 15:31:54 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.021 s -2016-05-25 15:31:54 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.039979 s -2016-05-25 15:31:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 21 ms on localhost (1/1) -2016-05-25 15:31:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 15:31:54 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 15:31:54 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 15:31:54 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 15:31:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:31:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:31:54 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 15:31:54 INFO MemoryStore:59 - ensureFreeSpace(4016) called with curMem=185280, maxMem=766075207 -2016-05-25 15:31:54 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-05-25 15:31:54 INFO MemoryStore:59 - ensureFreeSpace(2184) called with curMem=189296, maxMem=766075207 -2016-05-25 15:31:54 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 15:31:54 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53181 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:31:54 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:31:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at retag at RandomForest.scala:137) -2016-05-25 15:31:54 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 15:31:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 15:31:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 15:31:54 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 15:31:54 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 15:31:54 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:31:54 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:31:54 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 15:31:54 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 15:31:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 12 ms on localhost (1/2) -2016-05-25 15:31:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 14 ms on localhost (2/2) -2016-05-25 15:31:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 15:31:54 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.014 s -2016-05-25 15:31:54 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.028947 s -2016-05-25 15:31:54 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 15:31:54 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 15:31:54 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 15:31:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:31:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:31:54 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[9] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 15:31:54 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=191480, maxMem=766075207 -2016-05-25 15:31:54 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.6 KB, free 730.4 MB) -2016-05-25 15:31:54 INFO MemoryStore:59 - ensureFreeSpace(2492) called with curMem=196224, maxMem=766075207 -2016-05-25 15:31:54 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-05-25 15:31:54 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53181 (size: 2.4 KB, free: 730.6 MB) -2016-05-25 15:31:54 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:31:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[9] at sample at DecisionTree.scala:989) -2016-05-25 15:31:54 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 15:31:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 15:31:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 15:31:54 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 15:31:54 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 15:31:54 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:31:54 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:31:54 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2493 bytes result sent to driver -2016-05-25 15:31:54 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2677 bytes result sent to driver -2016-05-25 15:31:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 17 ms on localhost (1/2) -2016-05-25 15:31:54 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.019 s -2016-05-25 15:31:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 19 ms on localhost (2/2) -2016-05-25 15:31:54 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.033946 s -2016-05-25 15:31:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 15:31:54 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53181 in memory (size: 2.4 KB, free: 730.6 MB) -2016-05-25 15:31:54 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 15:31:54 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53181 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:31:54 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 15:31:54 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53181 in memory (size: 2.3 KB, free: 730.6 MB) -2016-05-25 15:31:54 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 15:31:54 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53181 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:31:54 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 15:31:55 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=172404, maxMem=766075207 -2016-05-25 15:31:55 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.4 MB) -2016-05-25 15:31:55 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=172444, maxMem=766075207 -2016-05-25 15:31:55 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.4 MB) -2016-05-25 15:31:55 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53181 (size: 81.0 B, free: 730.6 MB) -2016-05-25 15:31:55 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 15:31:55 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:31:55 INFO DAGScheduler:59 - Registering RDD 12 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:31:55 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:31:55 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:31:55 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 15:31:55 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 15:31:55 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[12] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:31:55 INFO MemoryStore:59 - ensureFreeSpace(9736) called with curMem=172525, maxMem=766075207 -2016-05-25 15:31:55 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 9.5 KB, free 730.4 MB) -2016-05-25 15:31:55 INFO MemoryStore:59 - ensureFreeSpace(4926) called with curMem=182261, maxMem=766075207 -2016-05-25 15:31:55 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-25 15:31:55 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53181 (size: 4.8 KB, free: 730.6 MB) -2016-05-25 15:31:55 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:31:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[12] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:31:55 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 15:31:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 15:31:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 15:31:55 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 15:31:55 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 15:31:55 INFO CacheManager:59 - Partition rdd_11_1 not found, computing it -2016-05-25 15:31:55 INFO CacheManager:59 - Partition rdd_11_0 not found, computing it -2016-05-25 15:31:55 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:31:55 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:31:55 INFO MemoryStore:59 - ensureFreeSpace(232) called with curMem=187187, maxMem=766075207 -2016-05-25 15:31:55 INFO MemoryStore:59 - Block rdd_11_1 stored as values in memory (estimated size 232.0 B, free 730.4 MB) -2016-05-25 15:31:55 INFO MemoryStore:59 - ensureFreeSpace(448) called with curMem=187419, maxMem=766075207 -2016-05-25 15:31:55 INFO MemoryStore:59 - Block rdd_11_0 stored as values in memory (estimated size 448.0 B, free 730.4 MB) -2016-05-25 15:31:55 INFO BlockManagerInfo:59 - Added rdd_11_1 in memory on localhost:53181 (size: 232.0 B, free: 730.6 MB) -2016-05-25 15:31:55 INFO BlockManagerInfo:59 - Added rdd_11_0 in memory on localhost:53181 (size: 448.0 B, free: 730.6 MB) -2016-05-25 15:31:55 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 15:31:55 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 15:31:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 96 ms on localhost (1/2) -2016-05-25 15:31:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 94 ms on localhost (2/2) -2016-05-25 15:31:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 15:31:55 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.098 s -2016-05-25 15:31:55 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:31:55 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:31:55 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 15:31:55 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:31:55 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 15:31:55 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[14] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:31:55 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=187867, maxMem=766075207 -2016-05-25 15:31:55 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-05-25 15:31:55 INFO MemoryStore:59 - ensureFreeSpace(2871) called with curMem=193099, maxMem=766075207 -2016-05-25 15:31:55 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-25 15:31:55 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53181 (size: 2.8 KB, free: 730.6 MB) -2016-05-25 15:31:55 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:31:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[14] at map at DecisionTree.scala:645) -2016-05-25 15:31:55 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 15:31:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:31:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:31:55 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 15:31:55 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 15:31:55 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 15:31:55 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:31:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 15:31:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 15:31:55 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 15:31:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 40 ms on localhost (1/2) -2016-05-25 15:31:55 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 15:31:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 76 ms on localhost (2/2) -2016-05-25 15:31:55 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.076 s -2016-05-25 15:31:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 15:31:55 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.234878 s -2016-05-25 15:31:55 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=195970, maxMem=766075207 -2016-05-25 15:31:55 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 40.0 B, free 730.4 MB) -2016-05-25 15:31:55 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=196010, maxMem=766075207 -2016-05-25 15:31:55 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.4 MB) -2016-05-25 15:31:55 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53181 (size: 81.0 B, free: 730.6 MB) -2016-05-25 15:31:55 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 15:31:55 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:31:55 INFO DAGScheduler:59 - Registering RDD 15 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:31:55 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:31:55 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:31:55 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 15:31:55 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 15:31:55 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[15] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:31:55 INFO MemoryStore:59 - ensureFreeSpace(10096) called with curMem=196091, maxMem=766075207 -2016-05-25 15:31:55 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 9.9 KB, free 730.4 MB) -2016-05-25 15:31:55 INFO MemoryStore:59 - ensureFreeSpace(5114) called with curMem=206187, maxMem=766075207 -2016-05-25 15:31:55 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 15:31:55 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53181 (size: 5.0 KB, free: 730.6 MB) -2016-05-25 15:31:55 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:31:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[15] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:31:55 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 15:31:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 15:31:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 15:31:55 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 15:31:55 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 15:31:55 INFO BlockManager:59 - Found block rdd_11_1 locally -2016-05-25 15:31:55 INFO BlockManager:59 - Found block rdd_11_0 locally -2016-05-25 15:31:55 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 15:31:55 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 15:31:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 28 ms on localhost (1/2) -2016-05-25 15:31:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 29 ms on localhost (2/2) -2016-05-25 15:31:55 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.030 s -2016-05-25 15:31:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 15:31:55 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:31:55 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:31:55 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 15:31:55 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:31:55 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 15:31:55 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[17] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:31:55 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=211301, maxMem=766075207 -2016-05-25 15:31:55 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-05-25 15:31:55 INFO MemoryStore:59 - ensureFreeSpace(2868) called with curMem=216533, maxMem=766075207 -2016-05-25 15:31:55 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-25 15:31:55 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53181 (size: 2.8 KB, free: 730.6 MB) -2016-05-25 15:31:55 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:31:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[17] at map at DecisionTree.scala:645) -2016-05-25 15:31:55 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 15:31:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:31:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:31:55 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 15:31:55 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 15:31:55 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:31:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 15:31:55 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 15:31:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 15:31:55 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 1161 bytes result sent to driver -2016-05-25 15:31:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 11 ms on localhost (1/2) -2016-05-25 15:31:55 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 15:31:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 22 ms on localhost (2/2) -2016-05-25 15:31:55 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.022 s -2016-05-25 15:31:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 15:31:55 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.087829 s -2016-05-25 15:31:55 INFO MapPartitionsRDD:59 - Removing RDD 11 from persistence list -2016-05-25 15:31:55 INFO BlockManager:59 - Removing RDD 11 -2016-05-25 15:31:55 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 15:31:55 INFO RandomForest:59 - init: 0.941823663 - total: 1.415110463 - findSplitsBins: 0.816295068 - findBestSplits: 0.454714975 - chooseSplits: 0.447476064 -2016-05-25 15:31:55 INFO SparkContext:59 - Starting job: take at DecisionTrees2ByEntropy.scala:36 -2016-05-25 15:31:55 INFO DAGScheduler:59 - Got job 6 (take at DecisionTrees2ByEntropy.scala:36) with 1 output partitions -2016-05-25 15:31:55 INFO DAGScheduler:59 - Final stage: ResultStage 8(take at DecisionTrees2ByEntropy.scala:36) -2016-05-25 15:31:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:31:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:31:55 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 15:31:55 INFO MemoryStore:59 - ensureFreeSpace(3968) called with curMem=218721, maxMem=766075207 -2016-05-25 15:31:55 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-05-25 15:31:55 INFO MemoryStore:59 - ensureFreeSpace(2186) called with curMem=222689, maxMem=766075207 -2016-05-25 15:31:55 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 15:31:55 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53181 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:31:55 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:31:55 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 15:31:55 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-05-25 15:31:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 15:31:55 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 15:31:55 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:31:55 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2493 bytes result sent to driver -2016-05-25 15:31:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 14 ms on localhost (1/1) -2016-05-25 15:31:55 INFO DAGScheduler:59 - ResultStage 8 (take at DecisionTrees2ByEntropy.scala:36) finished in 0.014 s -2016-05-25 15:31:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 15:31:55 INFO DAGScheduler:59 - Job 6 finished: take at DecisionTrees2ByEntropy.scala:36, took 0.036182 s -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 15:31:55 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 15:31:55 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 15:31:55 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 15:31:55 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 15:31:55 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 15:31:55 INFO BlockManager:59 - BlockManager stopped -2016-05-25 15:31:55 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 15:31:55 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 15:31:55 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 15:31:55 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 15:31:55 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-49b0503d-54b0-45cb-bfde-8e948c7a9d19 -2016-05-25 15:39:43 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 15:39:43 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 15:39:44 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 15:39:44 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 15:39:44 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 15:39:44 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 15:39:45 INFO Remoting:74 - Starting remoting -2016-05-25 15:39:45 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53429] -2016-05-25 15:39:45 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53429. -2016-05-25 15:39:45 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 15:39:45 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 15:39:45 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-39d4d131-77ca-4b08-a837-28baca4718a9 -2016-05-25 15:39:45 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 15:39:45 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-0e7c0f22-6406-48ba-b698-18a54113dfde\httpd-35d8c40a-7418-4f1c-8e6b-589de84a173c -2016-05-25 15:39:45 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 15:39:45 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:39:45 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53430 -2016-05-25 15:39:45 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53430. -2016-05-25 15:39:45 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 15:39:45 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:39:45 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 15:39:45 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 15:39:45 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 15:39:45 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 15:39:45 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 15:39:46 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53449. -2016-05-25 15:39:46 INFO NettyBlockTransferService:59 - Server created on 53449 -2016-05-25 15:39:46 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 15:39:46 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53449 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53449) -2016-05-25 15:39:46 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 15:39:46 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 15:39:46 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 15:39:46 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 15:39:46 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 15:39:46 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53449 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 15:39:46 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 15:39:48 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 15:39:48 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 15:39:48 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 15:39:48 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 15:39:48 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 15:39:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:39:48 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:39:48 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 15:39:48 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 15:39:48 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 15:39:48 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 15:39:48 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 15:39:48 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53449 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:39:48 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:39:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 15:39:48 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 15:39:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:39:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:39:49 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 15:39:49 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 15:39:49 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 15:39:49 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 15:39:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 15:39:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 15:39:49 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 15:39:49 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 15:39:49 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 15:39:49 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 15:39:49 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 15:39:49 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177630, maxMem=766075207 -2016-05-25 15:39:49 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 15:39:49 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:53449 (size: 83.1 KB, free: 730.5 MB) -2016-05-25 15:39:49 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262710, maxMem=766075207 -2016-05-25 15:39:49 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 15:39:49 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:53449 (size: 83.6 KB, free: 730.4 MB) -2016-05-25 15:39:49 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 15:39:49 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 15:39:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 442 ms on localhost (1/2) -2016-05-25 15:39:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 492 ms on localhost (2/2) -2016-05-25 15:39:49 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.513 s -2016-05-25 15:39:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 15:39:49 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.755228 s -2016-05-25 15:39:49 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 15:39:49 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 15:39:49 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 15:39:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:39:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:39:49 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 15:39:49 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348358, maxMem=766075207 -2016-05-25 15:39:49 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 15:39:49 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=353126, maxMem=766075207 -2016-05-25 15:39:49 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 15:39:49 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53449 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 15:39:49 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:39:49 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 15:39:49 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 15:39:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:39:49 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 15:39:49 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:39:49 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 15:39:49 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.027 s -2016-05-25 15:39:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 27 ms on localhost (1/1) -2016-05-25 15:39:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 15:39:49 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.046843 s -2016-05-25 15:39:49 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 15:39:49 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 15:39:49 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 15:39:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:39:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:39:49 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 15:39:49 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355673, maxMem=766075207 -2016-05-25 15:39:49 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 15:39:49 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=360009, maxMem=766075207 -2016-05-25 15:39:49 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 15:39:49 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53449 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 15:39:49 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:39:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 15:39:49 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 15:39:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:39:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:39:49 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 15:39:49 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 15:39:49 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:39:49 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:39:49 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 15:39:49 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 15:39:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 12 ms on localhost (1/2) -2016-05-25 15:39:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 12 ms on localhost (2/2) -2016-05-25 15:39:49 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.013 s -2016-05-25 15:39:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 15:39:49 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.027517 s -2016-05-25 15:39:49 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 15:39:49 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 15:39:49 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 15:39:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:39:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:39:49 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 15:39:49 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362328, maxMem=766075207 -2016-05-25 15:39:49 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 15:39:49 INFO MemoryStore:59 - ensureFreeSpace(2631) called with curMem=367392, maxMem=766075207 -2016-05-25 15:39:49 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 15:39:49 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53449 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:39:49 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:39:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 15:39:49 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 15:39:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 15:39:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 15:39:49 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 15:39:49 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 15:39:49 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:39:49 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:39:49 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 51180 bytes result sent to driver -2016-05-25 15:39:49 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 49512 bytes result sent to driver -2016-05-25 15:39:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 31 ms on localhost (1/2) -2016-05-25 15:39:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 31 ms on localhost (2/2) -2016-05-25 15:39:49 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.033 s -2016-05-25 15:39:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 15:39:49 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.049794 s -2016-05-25 15:39:51 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 15:39:51 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53449 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:39:51 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 15:39:51 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53449 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 15:39:51 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 15:39:51 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53449 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 15:39:51 INFO MemoryStore:59 - ensureFreeSpace(512) called with curMem=348358, maxMem=766075207 -2016-05-25 15:39:51 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 512.0 B, free 730.3 MB) -2016-05-25 15:39:51 INFO MemoryStore:59 - ensureFreeSpace(617) called with curMem=348870, maxMem=766075207 -2016-05-25 15:39:51 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 617.0 B, free 730.3 MB) -2016-05-25 15:39:51 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53449 (size: 617.0 B, free: 730.4 MB) -2016-05-25 15:39:51 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 15:39:51 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:39:51 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:39:51 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:39:51 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:39:51 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 15:39:51 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 15:39:51 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:39:51 INFO MemoryStore:59 - ensureFreeSpace(433888) called with curMem=349487, maxMem=766075207 -2016-05-25 15:39:51 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 423.7 KB, free 729.8 MB) -2016-05-25 15:39:51 INFO MemoryStore:59 - ensureFreeSpace(106639) called with curMem=783375, maxMem=766075207 -2016-05-25 15:39:51 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 104.1 KB, free 729.7 MB) -2016-05-25 15:39:51 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53449 (size: 104.1 KB, free: 730.3 MB) -2016-05-25 15:39:51 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:39:51 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:39:51 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 15:39:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:39:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:39:51 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 15:39:51 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 15:39:51 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 15:39:51 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:39:51 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 15:39:51 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:39:51 INFO MemoryStore:59 - ensureFreeSpace(77672) called with curMem=890014, maxMem=766075207 -2016-05-25 15:39:51 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 75.9 KB, free 729.7 MB) -2016-05-25 15:39:51 INFO MemoryStore:59 - ensureFreeSpace(89176) called with curMem=967686, maxMem=766075207 -2016-05-25 15:39:51 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 87.1 KB, free 729.6 MB) -2016-05-25 15:39:51 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:53449 (size: 75.9 KB, free: 730.2 MB) -2016-05-25 15:39:51 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:53449 (size: 87.1 KB, free: 730.1 MB) -2016-05-25 15:39:51 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 15:39:51 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 15:39:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 251 ms on localhost (1/2) -2016-05-25 15:39:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 254 ms on localhost (2/2) -2016-05-25 15:39:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 15:39:51 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.255 s -2016-05-25 15:39:51 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:39:51 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:39:51 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 15:39:51 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:39:51 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 15:39:51 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:39:51 INFO MemoryStore:59 - ensureFreeSpace(185272) called with curMem=1056862, maxMem=766075207 -2016-05-25 15:39:51 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 180.9 KB, free 729.4 MB) -2016-05-25 15:39:51 INFO MemoryStore:59 - ensureFreeSpace(39901) called with curMem=1242134, maxMem=766075207 -2016-05-25 15:39:51 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 39.0 KB, free 729.4 MB) -2016-05-25 15:39:51 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53449 (size: 39.0 KB, free: 730.1 MB) -2016-05-25 15:39:51 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:39:51 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 15:39:51 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 15:39:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:39:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:39:51 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 15:39:51 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 15:39:51 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:39:51 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:39:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-25 15:39:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-25 15:39:51 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2774 bytes result sent to driver -2016-05-25 15:39:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 177 ms on localhost (1/2) -2016-05-25 15:39:51 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2928 bytes result sent to driver -2016-05-25 15:39:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 193 ms on localhost (2/2) -2016-05-25 15:39:51 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.194 s -2016-05-25 15:39:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 15:39:51 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.505460 s -2016-05-25 15:39:51 INFO MemoryStore:59 - ensureFreeSpace(512) called with curMem=1282035, maxMem=766075207 -2016-05-25 15:39:51 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 512.0 B, free 729.4 MB) -2016-05-25 15:39:51 INFO MemoryStore:59 - ensureFreeSpace(620) called with curMem=1282547, maxMem=766075207 -2016-05-25 15:39:51 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 620.0 B, free 729.4 MB) -2016-05-25 15:39:51 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53449 (size: 620.0 B, free: 730.1 MB) -2016-05-25 15:39:51 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 15:39:52 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:39:52 INFO DAGScheduler:59 - Registering RDD 17 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:39:52 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:39:52 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:39:52 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 15:39:52 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 15:39:52 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:39:52 INFO MemoryStore:59 - ensureFreeSpace(434648) called with curMem=1283167, maxMem=766075207 -2016-05-25 15:39:52 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 424.5 KB, free 728.9 MB) -2016-05-25 15:39:52 INFO MemoryStore:59 - ensureFreeSpace(107019) called with curMem=1717815, maxMem=766075207 -2016-05-25 15:39:52 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 104.5 KB, free 728.8 MB) -2016-05-25 15:39:52 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53449 (size: 104.5 KB, free: 730.0 MB) -2016-05-25 15:39:52 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:39:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:39:52 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 15:39:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:39:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:39:52 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 15:39:52 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 15:39:52 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-25 15:39:52 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-25 15:39:52 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 15:39:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 41 ms on localhost (1/2) -2016-05-25 15:39:52 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 15:39:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 42 ms on localhost (2/2) -2016-05-25 15:39:52 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.044 s -2016-05-25 15:39:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 15:39:52 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:39:52 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:39:52 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 15:39:52 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:39:52 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 15:39:52 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:39:52 INFO MemoryStore:59 - ensureFreeSpace(185272) called with curMem=1824834, maxMem=766075207 -2016-05-25 15:39:52 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 180.9 KB, free 728.7 MB) -2016-05-25 15:39:52 INFO MemoryStore:59 - ensureFreeSpace(39949) called with curMem=2010106, maxMem=766075207 -2016-05-25 15:39:52 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 39.0 KB, free 728.6 MB) -2016-05-25 15:39:52 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53449 (size: 39.0 KB, free: 730.0 MB) -2016-05-25 15:39:52 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:39:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645) -2016-05-25 15:39:52 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 15:39:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:39:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:39:52 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 15:39:52 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 15:39:52 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:39:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 15:39:52 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:39:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 15:39:52 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2774 bytes result sent to driver -2016-05-25 15:39:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 36 ms on localhost (1/2) -2016-05-25 15:39:52 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2928 bytes result sent to driver -2016-05-25 15:39:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 54 ms on localhost (2/2) -2016-05-25 15:39:52 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.054 s -2016-05-25 15:39:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 15:39:52 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.144084 s -2016-05-25 15:39:52 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 15:39:52 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 15:39:52 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 15:39:52 INFO RandomForest:59 - init: 1.709045582 - total: 2.631969454 - findSplitsBins: 1.465891714 - findBestSplits: 0.896246591 - chooseSplits: 0.889867979 -2016-05-25 15:39:52 INFO SparkContext:59 - Starting job: count at RandomForest2Spark.scala:50 -2016-05-25 15:39:52 INFO DAGScheduler:59 - Got job 6 (count at RandomForest2Spark.scala:50) with 2 output partitions -2016-05-25 15:39:52 INFO DAGScheduler:59 - Final stage: ResultStage 8(count at RandomForest2Spark.scala:50) -2016-05-25 15:39:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:39:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:39:52 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at filter at RandomForest2Spark.scala:50), which has no missing parents -2016-05-25 15:39:52 INFO MemoryStore:59 - ensureFreeSpace(8408) called with curMem=1883207, maxMem=766075207 -2016-05-25 15:39:52 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 8.2 KB, free 728.8 MB) -2016-05-25 15:39:52 INFO MemoryStore:59 - ensureFreeSpace(4196) called with curMem=1891615, maxMem=766075207 -2016-05-25 15:39:52 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 4.1 KB, free 728.8 MB) -2016-05-25 15:39:52 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53449 (size: 4.1 KB, free: 730.1 MB) -2016-05-25 15:39:52 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:39:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at filter at RandomForest2Spark.scala:50) -2016-05-25 15:39:52 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 15:39:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:39:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:39:52 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 15:39:52 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 15:39:52 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:39:52 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:39:52 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 15:39:52 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 15:39:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 12 ms on localhost (1/2) -2016-05-25 15:39:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 14 ms on localhost (2/2) -2016-05-25 15:39:52 INFO DAGScheduler:59 - ResultStage 8 (count at RandomForest2Spark.scala:50) finished in 0.014 s -2016-05-25 15:39:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 15:39:52 INFO DAGScheduler:59 - Job 6 finished: count at RandomForest2Spark.scala:50, took 0.032591 s -2016-05-25 15:39:52 INFO SparkContext:59 - Starting job: count at RandomForest2Spark.scala:50 -2016-05-25 15:39:52 INFO DAGScheduler:59 - Got job 7 (count at RandomForest2Spark.scala:50) with 2 output partitions -2016-05-25 15:39:52 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at RandomForest2Spark.scala:50) -2016-05-25 15:39:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:39:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:39:52 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[8] at randomSplit at RandomForest2Spark.scala:29), which has no missing parents -2016-05-25 15:39:52 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1895811, maxMem=766075207 -2016-05-25 15:39:52 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.0 KB, free 728.8 MB) -2016-05-25 15:39:52 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=1899923, maxMem=766075207 -2016-05-25 15:39:52 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.8 MB) -2016-05-25 15:39:52 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53449 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 15:39:52 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:39:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[8] at randomSplit at RandomForest2Spark.scala:29) -2016-05-25 15:39:52 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 15:39:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:39:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:39:52 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 15:39:52 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-25 15:39:52 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:39:52 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2082 bytes result sent to driver -2016-05-25 15:39:52 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:39:52 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2082 bytes result sent to driver -2016-05-25 15:39:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 9 ms on localhost (1/2) -2016-05-25 15:39:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 10 ms on localhost (2/2) -2016-05-25 15:39:52 INFO DAGScheduler:59 - ResultStage 9 (count at RandomForest2Spark.scala:50) finished in 0.011 s -2016-05-25 15:39:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 15:39:52 INFO DAGScheduler:59 - Job 7 finished: count at RandomForest2Spark.scala:50, took 0.027670 s -2016-05-25 15:39:52 INFO SparkContext:59 - Starting job: count at RandomForest2Spark.scala:56 -2016-05-25 15:39:52 INFO DAGScheduler:59 - Got job 8 (count at RandomForest2Spark.scala:56) with 2 output partitions -2016-05-25 15:39:52 INFO DAGScheduler:59 - Final stage: ResultStage 10(count at RandomForest2Spark.scala:56) -2016-05-25 15:39:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:39:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:39:52 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 15:39:52 INFO MemoryStore:59 - ensureFreeSpace(3816) called with curMem=1902191, maxMem=766075207 -2016-05-25 15:39:52 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 3.7 KB, free 728.8 MB) -2016-05-25 15:39:52 INFO MemoryStore:59 - ensureFreeSpace(2142) called with curMem=1906007, maxMem=766075207 -2016-05-25 15:39:52 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.1 KB, free 728.8 MB) -2016-05-25 15:39:52 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53449 (size: 2.1 KB, free: 730.1 MB) -2016-05-25 15:39:52 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:39:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 15:39:52 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-25 15:39:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:39:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:39:52 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 15:39:52 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-25 15:39:52 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:39:52 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2082 bytes result sent to driver -2016-05-25 15:39:52 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:39:52 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2082 bytes result sent to driver -2016-05-25 15:39:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 9 ms on localhost (1/2) -2016-05-25 15:39:52 INFO DAGScheduler:59 - ResultStage 10 (count at RandomForest2Spark.scala:56) finished in 0.012 s -2016-05-25 15:39:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 11 ms on localhost (2/2) -2016-05-25 15:39:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 15:39:52 INFO DAGScheduler:59 - Job 8 finished: count at RandomForest2Spark.scala:56, took 0.025738 s -2016-05-25 15:39:52 INFO SparkContext:59 - Starting job: count at RandomForest2Spark.scala:57 -2016-05-25 15:39:52 INFO DAGScheduler:59 - Got job 9 (count at RandomForest2Spark.scala:57) with 2 output partitions -2016-05-25 15:39:52 INFO DAGScheduler:59 - Final stage: ResultStage 11(count at RandomForest2Spark.scala:57) -2016-05-25 15:39:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:39:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:39:52 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[7] at randomSplit at RandomForest2Spark.scala:29), which has no missing parents -2016-05-25 15:39:52 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1908149, maxMem=766075207 -2016-05-25 15:39:52 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 4.0 KB, free 728.8 MB) -2016-05-25 15:39:52 INFO MemoryStore:59 - ensureFreeSpace(2265) called with curMem=1912261, maxMem=766075207 -2016-05-25 15:39:52 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.8 MB) -2016-05-25 15:39:52 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53449 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 15:39:52 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:39:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[7] at randomSplit at RandomForest2Spark.scala:29) -2016-05-25 15:39:52 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-25 15:39:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:39:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:39:52 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-25 15:39:52 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-25 15:39:52 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:39:52 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2082 bytes result sent to driver -2016-05-25 15:39:52 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:39:52 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2082 bytes result sent to driver -2016-05-25 15:39:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 7 ms on localhost (1/2) -2016-05-25 15:39:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 10 ms on localhost (2/2) -2016-05-25 15:39:52 INFO DAGScheduler:59 - ResultStage 11 (count at RandomForest2Spark.scala:57) finished in 0.010 s -2016-05-25 15:39:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 15:39:52 INFO DAGScheduler:59 - Job 9 finished: count at RandomForest2Spark.scala:57, took 0.024622 s -2016-05-25 15:39:52 INFO SparkContext:59 - Starting job: count at RandomForest2Spark.scala:58 -2016-05-25 15:39:52 INFO DAGScheduler:59 - Got job 10 (count at RandomForest2Spark.scala:58) with 2 output partitions -2016-05-25 15:39:52 INFO DAGScheduler:59 - Final stage: ResultStage 12(count at RandomForest2Spark.scala:58) -2016-05-25 15:39:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:39:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:39:52 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[8] at randomSplit at RandomForest2Spark.scala:29), which has no missing parents -2016-05-25 15:39:52 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1914526, maxMem=766075207 -2016-05-25 15:39:52 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 4.0 KB, free 728.8 MB) -2016-05-25 15:39:52 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=1918638, maxMem=766075207 -2016-05-25 15:39:52 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.8 MB) -2016-05-25 15:39:52 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53449 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 15:39:52 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:39:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[8] at randomSplit at RandomForest2Spark.scala:29) -2016-05-25 15:39:52 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-25 15:39:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:39:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:39:52 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-25 15:39:52 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-25 15:39:52 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:39:52 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2082 bytes result sent to driver -2016-05-25 15:39:52 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:39:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 10 ms on localhost (1/2) -2016-05-25 15:39:52 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2082 bytes result sent to driver -2016-05-25 15:39:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 12 ms on localhost (2/2) -2016-05-25 15:39:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 15:39:52 INFO DAGScheduler:59 - ResultStage 12 (count at RandomForest2Spark.scala:58) finished in 0.013 s -2016-05-25 15:39:52 INFO DAGScheduler:59 - Job 10 finished: count at RandomForest2Spark.scala:58, took 0.025045 s -2016-05-25 15:39:52 INFO SparkContext:59 - Starting job: take at RandomForest2Spark.scala:63 -2016-05-25 15:39:52 INFO DAGScheduler:59 - Got job 11 (take at RandomForest2Spark.scala:63) with 1 output partitions -2016-05-25 15:39:52 INFO DAGScheduler:59 - Final stage: ResultStage 13(take at RandomForest2Spark.scala:63) -2016-05-25 15:39:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:39:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:39:52 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[20] at map at RandomForest2Spark.scala:46), which has no missing parents -2016-05-25 15:39:52 INFO MemoryStore:59 - ensureFreeSpace(8464) called with curMem=1920906, maxMem=766075207 -2016-05-25 15:39:52 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 8.3 KB, free 728.7 MB) -2016-05-25 15:39:52 INFO MemoryStore:59 - ensureFreeSpace(4238) called with curMem=1929370, maxMem=766075207 -2016-05-25 15:39:52 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 4.1 KB, free 728.7 MB) -2016-05-25 15:39:52 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53449 (size: 4.1 KB, free: 730.1 MB) -2016-05-25 15:39:52 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:39:52 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 13 (MapPartitionsRDD[20] at map at RandomForest2Spark.scala:46) -2016-05-25 15:39:52 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 1 tasks -2016-05-25 15:39:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:39:52 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-25 15:39:52 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:39:52 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2713 bytes result sent to driver -2016-05-25 15:39:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 13 ms on localhost (1/1) -2016-05-25 15:39:52 INFO DAGScheduler:59 - ResultStage 13 (take at RandomForest2Spark.scala:63) finished in 0.013 s -2016-05-25 15:39:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 15:39:52 INFO DAGScheduler:59 - Job 11 finished: take at RandomForest2Spark.scala:63, took 0.028239 s -2016-05-25 15:39:52 INFO SparkContext:59 - Starting job: take at RandomForest2Spark.scala:63 -2016-05-25 15:39:52 INFO DAGScheduler:59 - Got job 12 (take at RandomForest2Spark.scala:63) with 1 output partitions -2016-05-25 15:39:52 INFO DAGScheduler:59 - Final stage: ResultStage 14(take at RandomForest2Spark.scala:63) -2016-05-25 15:39:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:39:52 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:39:52 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[20] at map at RandomForest2Spark.scala:46), which has no missing parents -2016-05-25 15:39:52 INFO MemoryStore:59 - ensureFreeSpace(8464) called with curMem=1933608, maxMem=766075207 -2016-05-25 15:39:52 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 8.3 KB, free 728.7 MB) -2016-05-25 15:39:52 INFO MemoryStore:59 - ensureFreeSpace(4238) called with curMem=1942072, maxMem=766075207 -2016-05-25 15:39:52 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 4.1 KB, free 728.7 MB) -2016-05-25 15:39:52 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53449 (size: 4.1 KB, free: 730.1 MB) -2016-05-25 15:39:52 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:39:52 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 14 (MapPartitionsRDD[20] at map at RandomForest2Spark.scala:46) -2016-05-25 15:39:52 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 1 tasks -2016-05-25 15:39:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 26, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:39:52 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 26) -2016-05-25 15:39:52 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:39:52 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 26). 2329 bytes result sent to driver -2016-05-25 15:39:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 26) in 8 ms on localhost (1/1) -2016-05-25 15:39:52 INFO DAGScheduler:59 - ResultStage 14 (take at RandomForest2Spark.scala:63) finished in 0.009 s -2016-05-25 15:39:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 15:39:52 INFO DAGScheduler:59 - Job 12 finished: take at RandomForest2Spark.scala:63, took 0.025252 s -2016-05-25 15:39:53 INFO SparkContext:59 - Starting job: saveAsTextFile at treeEnsembleModels.scala:448 -2016-05-25 15:39:53 INFO DAGScheduler:59 - Got job 13 (saveAsTextFile at treeEnsembleModels.scala:448) with 1 output partitions -2016-05-25 15:39:53 INFO DAGScheduler:59 - Final stage: ResultStage 15(saveAsTextFile at treeEnsembleModels.scala:448) -2016-05-25 15:39:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:39:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:39:53 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[23] at saveAsTextFile at treeEnsembleModels.scala:448), which has no missing parents -2016-05-25 15:39:53 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=1946310, maxMem=766075207 -2016-05-25 15:39:53 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 124.0 KB, free 728.6 MB) -2016-05-25 15:39:53 INFO MemoryStore:59 - ensureFreeSpace(42621) called with curMem=2073302, maxMem=766075207 -2016-05-25 15:39:53 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 41.6 KB, free 728.6 MB) -2016-05-25 15:39:53 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53449 (size: 41.6 KB, free: 730.1 MB) -2016-05-25 15:39:53 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:39:53 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 15 (MapPartitionsRDD[23] at saveAsTextFile at treeEnsembleModels.scala:448) -2016-05-25 15:39:53 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 1 tasks -2016-05-25 15:39:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 27, localhost, PROCESS_LOCAL, 2289 bytes) -2016-05-25 15:39:53 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 27) -2016-05-25 15:39:53 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251539_0015_m_000000_27' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525153952435/result/metadata/_temporary/0/task_201605251539_0015_m_000000 -2016-05-25 15:39:53 INFO SparkHadoopMapRedUtil:59 - attempt_201605251539_0015_m_000000_27: Committed -2016-05-25 15:39:53 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 27). 1864 bytes result sent to driver -2016-05-25 15:39:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 27) in 219 ms on localhost (1/1) -2016-05-25 15:39:53 INFO DAGScheduler:59 - ResultStage 15 (saveAsTextFile at treeEnsembleModels.scala:448) finished in 0.219 s -2016-05-25 15:39:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-25 15:39:53 INFO DAGScheduler:59 - Job 13 finished: saveAsTextFile at treeEnsembleModels.scala:448, took 0.295445 s -2016-05-25 15:39:53 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 15:39:53 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53449 in memory (size: 104.5 KB, free: 730.2 MB) -2016-05-25 15:39:53 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 15:39:53 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-25 15:39:53 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53449 in memory (size: 620.0 B, free: 730.2 MB) -2016-05-25 15:39:53 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53449 in memory (size: 39.0 KB, free: 730.2 MB) -2016-05-25 15:39:53 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 15:39:53 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53449 in memory (size: 104.1 KB, free: 730.3 MB) -2016-05-25 15:39:53 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 15:39:53 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 15:39:53 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53449 in memory (size: 617.0 B, free: 730.3 MB) -2016-05-25 15:39:53 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 15:39:53 INFO ContextCleaner:59 - Cleaned RDD 13 -2016-05-25 15:39:53 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:53449 in memory (size: 41.6 KB, free: 730.3 MB) -2016-05-25 15:39:53 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-25 15:39:53 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:53449 in memory (size: 4.1 KB, free: 730.4 MB) -2016-05-25 15:39:53 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-25 15:39:53 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:53449 in memory (size: 4.1 KB, free: 730.4 MB) -2016-05-25 15:39:53 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-25 15:39:53 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:53449 in memory (size: 2.2 KB, free: 730.4 MB) -2016-05-25 15:39:53 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-25 15:39:53 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:53449 in memory (size: 2.2 KB, free: 730.4 MB) -2016-05-25 15:39:53 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-25 15:39:53 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:53449 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-25 15:39:53 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 15:39:53 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:53449 in memory (size: 2.2 KB, free: 730.4 MB) -2016-05-25 15:39:53 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 15:39:53 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:53449 in memory (size: 4.1 KB, free: 730.4 MB) -2016-05-25 15:39:53 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 15:39:53 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:53449 in memory (size: 39.0 KB, free: 730.4 MB) -2016-05-25 15:39:55 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 15:39:55 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 15:39:55 INFO SparkContext:59 - Starting job: parquet at treeEnsembleModels.scala:454 -2016-05-25 15:39:55 INFO DAGScheduler:59 - Got job 14 (parquet at treeEnsembleModels.scala:454) with 4 output partitions -2016-05-25 15:39:55 INFO DAGScheduler:59 - Final stage: ResultStage 16(parquet at treeEnsembleModels.scala:454) -2016-05-25 15:39:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:39:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:39:55 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[26] at rddToDataFrameHolder at treeEnsembleModels.scala:451), which has no missing parents -2016-05-25 15:39:55 INFO MemoryStore:59 - ensureFreeSpace(68568) called with curMem=348358, maxMem=766075207 -2016-05-25 15:39:55 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 67.0 KB, free 730.2 MB) -2016-05-25 15:39:55 INFO MemoryStore:59 - ensureFreeSpace(24139) called with curMem=416926, maxMem=766075207 -2016-05-25 15:39:55 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 23.6 KB, free 730.2 MB) -2016-05-25 15:39:55 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53449 (size: 23.6 KB, free: 730.4 MB) -2016-05-25 15:39:55 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:39:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 16 (MapPartitionsRDD[26] at rddToDataFrameHolder at treeEnsembleModels.scala:451) -2016-05-25 15:39:55 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 4 tasks -2016-05-25 15:39:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 28, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 15:39:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 29, localhost, PROCESS_LOCAL, 4439 bytes) -2016-05-25 15:39:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 16.0 (TID 30, localhost, PROCESS_LOCAL, 4439 bytes) -2016-05-25 15:39:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 4444 bytes) -2016-05-25 15:39:55 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 28) -2016-05-25 15:39:55 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 29) -2016-05-25 15:39:55 INFO Executor:59 - Running task 3.0 in stage 16.0 (TID 31) -2016-05-25 15:39:55 INFO Executor:59 - Running task 2.0 in stage 16.0 (TID 30) -2016-05-25 15:39:55 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 15:39:55 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 15:39:55 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 15:39:55 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 15:39:55 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 15:39:55 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 15:39:55 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 15:39:55 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 15:39:55 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 15:39:55 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 15:39:55 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 15:39:55 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 15:39:55 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 15:39:55 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,568 -2016-05-25 15:39:55 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,856 -2016-05-25 15:39:55 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,864 -2016-05-25 15:39:55 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,864 -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 5 values, 8B raw, 28B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 71B for [nodeId] INT32: 5 values, 26B raw, 38B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, predict] DOUBLE: 5 values, 10B raw, 30B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 2 entries, 16B raw, 2B comp} -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 72B for [predict, prob] DOUBLE: 5 values, 11B raw, 31B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 3 entries, 24B raw, 3B comp} -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 71B for [impurity] DOUBLE: 5 values, 10B raw, 30B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 3 entries, 24B raw, 3B comp} -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 5 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 68B for [split, feature] INT32: 5 values, 15B raw, 35B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 68B for [split, threshold] DOUBLE: 5 values, 9B raw, 27B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 1 entries, 8B raw, 1B comp} -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 60B for [split, featureType] INT32: 5 values, 9B raw, 27B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 52B for [split, categories, bag, array] DOUBLE: 5 values, 14B raw, 31B comp, 1 pages, encodings: [RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 64B for [leftNodeId] INT32: 5 values, 14B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 64B for [rightNodeId] INT32: 5 values, 14B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 83B for [infoGain] DOUBLE: 5 values, 22B raw, 42B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 5 values, 8B raw, 28B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 5 values, 8B raw, 28B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 71B for [nodeId] INT32: 5 values, 26B raw, 38B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 71B for [nodeId] INT32: 5 values, 26B raw, 38B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, predict] DOUBLE: 5 values, 10B raw, 30B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 2 entries, 16B raw, 2B comp} -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 72B for [predict, prob] DOUBLE: 5 values, 11B raw, 31B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 3 entries, 24B raw, 3B comp} -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 71B for [impurity] DOUBLE: 5 values, 10B raw, 30B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 3 entries, 24B raw, 3B comp} -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 5 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 66B for [split, feature] INT32: 5 values, 15B raw, 33B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 72B for [split, threshold] DOUBLE: 5 values, 23B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 60B for [split, featureType] INT32: 5 values, 9B raw, 27B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 52B for [split, categories, bag, array] DOUBLE: 5 values, 14B raw, 31B comp, 1 pages, encodings: [RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 64B for [leftNodeId] INT32: 5 values, 14B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 64B for [rightNodeId] INT32: 5 values, 14B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 84B for [infoGain] DOUBLE: 5 values, 22B raw, 43B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, predict] DOUBLE: 5 values, 10B raw, 30B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 2 entries, 16B raw, 2B comp} -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 72B for [predict, prob] DOUBLE: 5 values, 11B raw, 31B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 3 entries, 24B raw, 3B comp} -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 71B for [impurity] DOUBLE: 5 values, 10B raw, 30B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 3 entries, 24B raw, 3B comp} -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 5 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 67B for [split, feature] INT32: 5 values, 15B raw, 34B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 72B for [split, threshold] DOUBLE: 5 values, 23B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 60B for [split, featureType] INT32: 5 values, 9B raw, 27B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 52B for [split, categories, bag, array] DOUBLE: 5 values, 14B raw, 31B comp, 1 pages, encodings: [RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 64B for [leftNodeId] INT32: 5 values, 14B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 64B for [rightNodeId] INT32: 5 values, 14B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:39:56 INFO ColumnChunkPageWriteStore:151 - written 84B for [infoGain] DOUBLE: 5 values, 22B raw, 43B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:39:56 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251539_0016_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525153952435/result/data/_temporary/0/task_201605251539_0016_m_000000 -2016-05-25 15:39:56 INFO SparkHadoopMapRedUtil:59 - attempt_201605251539_0016_m_000000_0: Committed -2016-05-25 15:39:56 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 28). 915 bytes result sent to driver -2016-05-25 15:39:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 28) in 779 ms on localhost (1/4) -2016-05-25 15:39:56 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251539_0016_m_000003_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525153952435/result/data/_temporary/0/task_201605251539_0016_m_000003 -2016-05-25 15:39:56 INFO SparkHadoopMapRedUtil:59 - attempt_201605251539_0016_m_000003_0: Committed -2016-05-25 15:39:56 INFO Executor:59 - Finished task 3.0 in stage 16.0 (TID 31). 915 bytes result sent to driver -2016-05-25 15:39:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 16.0 (TID 31) in 825 ms on localhost (2/4) -2016-05-25 15:39:56 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251539_0016_m_000001_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525153952435/result/data/_temporary/0/task_201605251539_0016_m_000001 -2016-05-25 15:39:56 INFO SparkHadoopMapRedUtil:59 - attempt_201605251539_0016_m_000001_0: Committed -2016-05-25 15:39:56 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 29). 915 bytes result sent to driver -2016-05-25 15:39:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 29) in 836 ms on localhost (3/4) -2016-05-25 15:39:56 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251539_0016_m_000002_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525153952435/result/data/_temporary/0/task_201605251539_0016_m_000002 -2016-05-25 15:39:56 INFO SparkHadoopMapRedUtil:59 - attempt_201605251539_0016_m_000002_0: Committed -2016-05-25 15:39:56 INFO Executor:59 - Finished task 2.0 in stage 16.0 (TID 30). 915 bytes result sent to driver -2016-05-25 15:39:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 16.0 (TID 30) in 837 ms on localhost (4/4) -2016-05-25 15:39:56 INFO DAGScheduler:59 - ResultStage 16 (parquet at treeEnsembleModels.scala:454) finished in 0.840 s -2016-05-25 15:39:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-25 15:39:56 INFO DAGScheduler:59 - Job 14 finished: parquet at treeEnsembleModels.scala:454, took 0.897039 s -2016-05-25 15:39:56 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 15:39:56 INFO DefaultWriterContainer:59 - Job job_201605251539_0000 committed. -2016-05-25 15:39:56 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525153952435/result/data on driver -2016-05-25 15:39:56 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525153952435/result/data on driver -2016-05-25 15:39:56 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=441065, maxMem=766075207 -2016-05-25 15:39:56 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 225.3 KB, free 729.9 MB) -2016-05-25 15:39:56 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=671769, maxMem=766075207 -2016-05-25 15:39:56 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-25 15:39:56 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:53449 (size: 19.3 KB, free: 730.4 MB) -2016-05-25 15:39:56 INFO SparkContext:59 - Created broadcast 20 from textFile at modelSaveLoad.scala:129 -2016-05-25 15:39:56 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 15:39:56 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 15:39:56 INFO DAGScheduler:59 - Got job 15 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 15:39:56 INFO DAGScheduler:59 - Final stage: ResultStage 17(first at modelSaveLoad.scala:129) -2016-05-25 15:39:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:39:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:39:56 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[29] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 15:39:56 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=691557, maxMem=766075207 -2016-05-25 15:39:56 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 3.1 KB, free 729.9 MB) -2016-05-25 15:39:56 INFO MemoryStore:59 - ensureFreeSpace(1893) called with curMem=694773, maxMem=766075207 -2016-05-25 15:39:56 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1893.0 B, free 729.9 MB) -2016-05-25 15:39:56 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:53449 (size: 1893.0 B, free: 730.4 MB) -2016-05-25 15:39:56 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:39:56 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 17 (MapPartitionsRDD[29] at textFile at modelSaveLoad.scala:129) -2016-05-25 15:39:56 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 1 tasks -2016-05-25 15:39:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 32, localhost, PROCESS_LOCAL, 2241 bytes) -2016-05-25 15:39:56 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 32) -2016-05-25 15:39:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525153952435/result/metadata/part-00000:0+100 -2016-05-25 15:39:56 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 32). 2249 bytes result sent to driver -2016-05-25 15:39:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 32) in 12 ms on localhost (1/1) -2016-05-25 15:39:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-25 15:39:56 INFO DAGScheduler:59 - ResultStage 17 (first at modelSaveLoad.scala:129) finished in 0.013 s -2016-05-25 15:39:56 INFO DAGScheduler:59 - Job 15 finished: first at modelSaveLoad.scala:129, took 0.023920 s -2016-05-25 15:39:57 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53449 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-25 15:39:57 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:53449 in memory (size: 1893.0 B, free: 730.4 MB) -2016-05-25 15:39:57 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-25 15:39:57 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:53449 in memory (size: 19.3 KB, free: 730.4 MB) -2016-05-25 15:39:57 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:53449 in memory (size: 23.6 KB, free: 730.4 MB) -2016-05-25 15:39:57 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-25 15:39:57 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 15:39:57 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525153952435/result/data on driver -2016-05-25 15:39:57 INFO SparkContext:59 - Starting job: parquet at treeEnsembleModels.scala:477 -2016-05-25 15:39:57 INFO DAGScheduler:59 - Got job 16 (parquet at treeEnsembleModels.scala:477) with 4 output partitions -2016-05-25 15:39:57 INFO DAGScheduler:59 - Final stage: ResultStage 18(parquet at treeEnsembleModels.scala:477) -2016-05-25 15:39:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:39:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:39:57 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[31] at parquet at treeEnsembleModels.scala:477), which has no missing parents -2016-05-25 15:39:57 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=342308, maxMem=766075207 -2016-05-25 15:39:57 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 61.5 KB, free 730.2 MB) -2016-05-25 15:39:57 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=405308, maxMem=766075207 -2016-05-25 15:39:57 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.2 MB) -2016-05-25 15:39:57 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:53449 (size: 20.6 KB, free: 730.4 MB) -2016-05-25 15:39:57 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:39:57 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (MapPartitionsRDD[31] at parquet at treeEnsembleModels.scala:477) -2016-05-25 15:39:57 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-05-25 15:39:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 33, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 15:39:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 34, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 15:39:57 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 15:39:57 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2282 bytes) -2016-05-25 15:39:57 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 33) -2016-05-25 15:39:57 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 35) -2016-05-25 15:39:57 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 36) -2016-05-25 15:39:57 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 34) -2016-05-25 15:39:57 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 15:39:57 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 15:39:57 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 33). 936 bytes result sent to driver -2016-05-25 15:39:57 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 15:39:57 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 34). 936 bytes result sent to driver -2016-05-25 15:39:57 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 15:39:57 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 35). 936 bytes result sent to driver -2016-05-25 15:39:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 33) in 87 ms on localhost (1/4) -2016-05-25 15:39:57 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 35) in 77 ms on localhost (2/4) -2016-05-25 15:39:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 34) in 79 ms on localhost (3/4) -2016-05-25 15:39:57 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 36). 2321 bytes result sent to driver -2016-05-25 15:39:57 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 36) in 81 ms on localhost (4/4) -2016-05-25 15:39:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-25 15:39:57 INFO DAGScheduler:59 - ResultStage 18 (parquet at treeEnsembleModels.scala:477) finished in 0.094 s -2016-05-25 15:39:57 INFO DAGScheduler:59 - Job 16 finished: parquet at treeEnsembleModels.scala:477, took 0.135169 s -2016-05-25 15:39:57 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=426381, maxMem=766075207 -2016-05-25 15:39:57 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 86.4 KB, free 730.1 MB) -2016-05-25 15:39:57 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=514893, maxMem=766075207 -2016-05-25 15:39:57 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-25 15:39:57 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:53449 (size: 19.3 KB, free: 730.4 MB) -2016-05-25 15:39:57 INFO SparkContext:59 - Created broadcast 23 from map at treeEnsembleModels.scala:477 -2016-05-25 15:39:58 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:53449 in memory (size: 20.6 KB, free: 730.4 MB) -2016-05-25 15:39:58 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-25 15:39:58 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 15:39:58 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525153952435/result/data/part-r-00001-16b3e16e-2691-4725-ba4d-3f87486de337.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525153952435/result/data/part-r-00002-16b3e16e-2691-4725-ba4d-3f87486de337.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525153952435/result/data/part-r-00000-16b3e16e-2691-4725-ba4d-3f87486de337.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525153952435/result/data/part-r-00003-16b3e16e-2691-4725-ba4d-3f87486de337.gz.parquet -2016-05-25 15:40:37 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 15:40:38 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 15:40:39 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 15:40:39 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 15:40:39 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 15:40:39 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 15:40:39 INFO Remoting:74 - Starting remoting -2016-05-25 15:40:40 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53471] -2016-05-25 15:40:40 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53471. -2016-05-25 15:40:40 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 15:40:40 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 15:40:40 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-c1c375bf-727e-4657-acb7-a0d096caba82 -2016-05-25 15:40:40 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 15:40:40 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-65175067-1a15-4f73-b67e-5c343d6d59be\httpd-b6438ce9-1066-4177-921c-a3340754cc71 -2016-05-25 15:40:40 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 15:40:40 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:40:40 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53472 -2016-05-25 15:40:40 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53472. -2016-05-25 15:40:40 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 15:40:40 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:40:40 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 15:40:40 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 15:40:40 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 15:40:40 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 15:40:40 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 15:40:41 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53491. -2016-05-25 15:40:41 INFO NettyBlockTransferService:59 - Server created on 53491 -2016-05-25 15:40:41 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 15:40:41 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53491 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53491) -2016-05-25 15:40:41 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 15:40:41 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 15:40:41 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 15:40:41 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 15:40:41 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 15:40:41 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53491 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 15:40:41 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 15:40:43 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 15:40:43 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 15:40:43 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 15:40:43 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 15:40:43 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 15:40:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:40:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:40:43 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 15:40:43 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 15:40:43 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 15:40:43 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 15:40:43 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 15:40:43 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53491 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:40:43 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:40:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 15:40:43 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 15:40:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:40:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:40:43 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 15:40:43 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 15:40:43 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 15:40:43 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 15:40:43 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 15:40:43 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 15:40:43 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 15:40:43 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 15:40:43 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 15:40:43 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 15:40:43 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 15:40:44 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177630, maxMem=766075207 -2016-05-25 15:40:44 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 15:40:44 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263278, maxMem=766075207 -2016-05-25 15:40:44 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 15:40:44 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:53491 (size: 83.6 KB, free: 730.5 MB) -2016-05-25 15:40:44 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:53491 (size: 83.1 KB, free: 730.4 MB) -2016-05-25 15:40:44 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 15:40:44 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 15:40:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 381 ms on localhost (1/2) -2016-05-25 15:40:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 415 ms on localhost (2/2) -2016-05-25 15:40:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 15:40:44 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.435 s -2016-05-25 15:40:44 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.580082 s -2016-05-25 15:40:44 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53491 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-25 15:40:44 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 15:40:44 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 15:40:44 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 15:40:44 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 15:40:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:40:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:40:44 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 15:40:44 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=342308, maxMem=766075207 -2016-05-25 15:40:44 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.3 MB) -2016-05-25 15:40:44 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=347076, maxMem=766075207 -2016-05-25 15:40:44 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.3 MB) -2016-05-25 15:40:44 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53491 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 15:40:44 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:40:44 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 15:40:44 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 15:40:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:40:44 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 15:40:44 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:40:44 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 15:40:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 31 ms on localhost (1/1) -2016-05-25 15:40:44 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.032 s -2016-05-25 15:40:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 15:40:44 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.049742 s -2016-05-25 15:40:44 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 15:40:44 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 15:40:44 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 15:40:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:40:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:40:44 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 15:40:44 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=349623, maxMem=766075207 -2016-05-25 15:40:44 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 15:40:44 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=353959, maxMem=766075207 -2016-05-25 15:40:44 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 15:40:44 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53491 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 15:40:44 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:40:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 15:40:44 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 15:40:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:40:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:40:44 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 15:40:44 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 15:40:44 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:40:44 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:40:44 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 15:40:44 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 15:40:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 16 ms on localhost (1/2) -2016-05-25 15:40:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 15 ms on localhost (2/2) -2016-05-25 15:40:44 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.016 s -2016-05-25 15:40:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 15:40:44 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.030177 s -2016-05-25 15:40:44 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 15:40:44 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 15:40:44 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 15:40:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:40:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:40:44 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 15:40:44 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=356278, maxMem=766075207 -2016-05-25 15:40:44 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 15:40:44 INFO MemoryStore:59 - ensureFreeSpace(2631) called with curMem=361342, maxMem=766075207 -2016-05-25 15:40:44 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 15:40:44 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53491 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:40:44 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:40:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 15:40:44 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 15:40:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 15:40:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 15:40:44 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 15:40:44 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 15:40:44 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:40:44 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:40:44 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 59464 bytes result sent to driver -2016-05-25 15:40:44 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 67918 bytes result sent to driver -2016-05-25 15:40:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 37 ms on localhost (1/2) -2016-05-25 15:40:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 42 ms on localhost (2/2) -2016-05-25 15:40:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 15:40:44 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.042 s -2016-05-25 15:40:44 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.065638 s -2016-05-25 15:40:45 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53491 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:40:45 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 15:40:45 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53491 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 15:40:45 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 15:40:45 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53491 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 15:40:45 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 15:40:45 INFO MemoryStore:59 - ensureFreeSpace(512) called with curMem=342308, maxMem=766075207 -2016-05-25 15:40:45 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 512.0 B, free 730.3 MB) -2016-05-25 15:40:45 INFO MemoryStore:59 - ensureFreeSpace(622) called with curMem=342820, maxMem=766075207 -2016-05-25 15:40:45 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 622.0 B, free 730.3 MB) -2016-05-25 15:40:45 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53491 (size: 622.0 B, free: 730.4 MB) -2016-05-25 15:40:45 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 15:40:46 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:40:46 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:40:46 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:40:46 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:40:46 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 15:40:46 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 15:40:46 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:40:46 INFO MemoryStore:59 - ensureFreeSpace(498072) called with curMem=343442, maxMem=766075207 -2016-05-25 15:40:46 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 486.4 KB, free 729.8 MB) -2016-05-25 15:40:46 INFO MemoryStore:59 - ensureFreeSpace(122577) called with curMem=841514, maxMem=766075207 -2016-05-25 15:40:46 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 119.7 KB, free 729.7 MB) -2016-05-25 15:40:46 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53491 (size: 119.7 KB, free: 730.3 MB) -2016-05-25 15:40:46 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:40:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:40:46 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 15:40:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:40:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:40:46 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 15:40:46 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 15:40:46 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 15:40:46 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:40:46 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 15:40:46 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:40:46 INFO MemoryStore:59 - ensureFreeSpace(94928) called with curMem=964091, maxMem=766075207 -2016-05-25 15:40:46 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 92.7 KB, free 729.6 MB) -2016-05-25 15:40:46 INFO MemoryStore:59 - ensureFreeSpace(112184) called with curMem=1059019, maxMem=766075207 -2016-05-25 15:40:46 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 109.6 KB, free 729.5 MB) -2016-05-25 15:40:46 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:53491 (size: 92.7 KB, free: 730.2 MB) -2016-05-25 15:40:46 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:53491 (size: 109.6 KB, free: 730.1 MB) -2016-05-25 15:40:46 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 15:40:46 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 15:40:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 289 ms on localhost (1/2) -2016-05-25 15:40:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 291 ms on localhost (2/2) -2016-05-25 15:40:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 15:40:46 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.293 s -2016-05-25 15:40:46 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:40:46 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:40:46 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 15:40:46 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:40:46 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 15:40:46 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:40:46 INFO MemoryStore:59 - ensureFreeSpace(216800) called with curMem=1171203, maxMem=766075207 -2016-05-25 15:40:46 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 211.7 KB, free 729.3 MB) -2016-05-25 15:40:46 INFO MemoryStore:59 - ensureFreeSpace(46032) called with curMem=1388003, maxMem=766075207 -2016-05-25 15:40:46 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 45.0 KB, free 729.2 MB) -2016-05-25 15:40:46 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53491 (size: 45.0 KB, free: 730.1 MB) -2016-05-25 15:40:46 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:40:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 15:40:46 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 15:40:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:40:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:40:46 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 15:40:46 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 15:40:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:40:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:40:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 15:40:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 15:40:46 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2774 bytes result sent to driver -2016-05-25 15:40:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 208 ms on localhost (1/2) -2016-05-25 15:40:46 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2928 bytes result sent to driver -2016-05-25 15:40:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 223 ms on localhost (2/2) -2016-05-25 15:40:46 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.224 s -2016-05-25 15:40:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 15:40:46 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.581832 s -2016-05-25 15:40:46 INFO MemoryStore:59 - ensureFreeSpace(512) called with curMem=1434035, maxMem=766075207 -2016-05-25 15:40:46 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 512.0 B, free 729.2 MB) -2016-05-25 15:40:46 INFO MemoryStore:59 - ensureFreeSpace(619) called with curMem=1434547, maxMem=766075207 -2016-05-25 15:40:46 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 619.0 B, free 729.2 MB) -2016-05-25 15:40:46 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53491 (size: 619.0 B, free: 730.1 MB) -2016-05-25 15:40:46 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 15:40:46 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:40:46 INFO DAGScheduler:59 - Registering RDD 17 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:40:46 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:40:46 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:40:46 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 15:40:46 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 15:40:46 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:40:46 INFO MemoryStore:59 - ensureFreeSpace(498832) called with curMem=1435166, maxMem=766075207 -2016-05-25 15:40:46 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 487.1 KB, free 728.7 MB) -2016-05-25 15:40:46 INFO MemoryStore:59 - ensureFreeSpace(122948) called with curMem=1933998, maxMem=766075207 -2016-05-25 15:40:46 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 120.1 KB, free 728.6 MB) -2016-05-25 15:40:46 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53491 (size: 120.1 KB, free: 729.9 MB) -2016-05-25 15:40:46 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:40:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:40:46 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 15:40:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:40:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:40:46 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 15:40:46 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 15:40:46 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-25 15:40:46 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-25 15:40:46 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 15:40:46 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 15:40:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 47 ms on localhost (1/2) -2016-05-25 15:40:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 49 ms on localhost (2/2) -2016-05-25 15:40:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 15:40:46 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.051 s -2016-05-25 15:40:46 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:40:46 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:40:46 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 15:40:46 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:40:46 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 15:40:46 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:40:46 INFO MemoryStore:59 - ensureFreeSpace(216800) called with curMem=2056946, maxMem=766075207 -2016-05-25 15:40:46 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 211.7 KB, free 728.4 MB) -2016-05-25 15:40:46 INFO MemoryStore:59 - ensureFreeSpace(46069) called with curMem=2273746, maxMem=766075207 -2016-05-25 15:40:46 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 45.0 KB, free 728.6 MB) -2016-05-25 15:40:46 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53491 (size: 45.0 KB, free: 729.9 MB) -2016-05-25 15:40:46 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:40:46 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53491 in memory (size: 45.0 KB, free: 729.9 MB) -2016-05-25 15:40:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645) -2016-05-25 15:40:46 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 15:40:46 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 15:40:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:40:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:40:46 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53491 in memory (size: 119.7 KB, free: 730.0 MB) -2016-05-25 15:40:46 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 15:40:46 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 15:40:46 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 15:40:46 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 15:40:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:40:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:40:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 15:40:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 15:40:46 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53491 in memory (size: 622.0 B, free: 730.1 MB) -2016-05-25 15:40:46 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2774 bytes result sent to driver -2016-05-25 15:40:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 43 ms on localhost (1/2) -2016-05-25 15:40:46 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2928 bytes result sent to driver -2016-05-25 15:40:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 60 ms on localhost (2/2) -2016-05-25 15:40:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 15:40:46 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.060 s -2016-05-25 15:40:46 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.176756 s -2016-05-25 15:40:46 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 15:40:46 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 15:40:46 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 15:40:46 INFO RandomForest:59 - init: 1.659424483 - total: 2.696808641 - findSplitsBins: 1.41670478 - findBestSplits: 1.015401483 - chooseSplits: 1.00951415 -2016-05-25 15:40:46 INFO SparkContext:59 - Starting job: count at RandomForest2Spark.scala:50 -2016-05-25 15:40:46 INFO DAGScheduler:59 - Got job 6 (count at RandomForest2Spark.scala:50) with 2 output partitions -2016-05-25 15:40:46 INFO DAGScheduler:59 - Final stage: ResultStage 8(count at RandomForest2Spark.scala:50) -2016-05-25 15:40:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:40:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:40:46 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[21] at filter at RandomForest2Spark.scala:50), which has no missing parents -2016-05-25 15:40:46 INFO MemoryStore:59 - ensureFreeSpace(8408) called with curMem=1228088, maxMem=766075207 -2016-05-25 15:40:46 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 8.2 KB, free 729.4 MB) -2016-05-25 15:40:46 INFO MemoryStore:59 - ensureFreeSpace(4233) called with curMem=1236496, maxMem=766075207 -2016-05-25 15:40:46 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 4.1 KB, free 729.4 MB) -2016-05-25 15:40:46 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53491 (size: 4.1 KB, free: 730.2 MB) -2016-05-25 15:40:46 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:40:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[21] at filter at RandomForest2Spark.scala:50) -2016-05-25 15:40:46 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 15:40:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:40:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:40:46 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 15:40:46 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 15:40:46 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:40:46 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:40:47 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 15:40:47 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 15:40:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 16 ms on localhost (1/2) -2016-05-25 15:40:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 18 ms on localhost (2/2) -2016-05-25 15:40:47 INFO DAGScheduler:59 - ResultStage 8 (count at RandomForest2Spark.scala:50) finished in 0.018 s -2016-05-25 15:40:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 15:40:47 INFO DAGScheduler:59 - Job 6 finished: count at RandomForest2Spark.scala:50, took 0.037965 s -2016-05-25 15:40:47 INFO SparkContext:59 - Starting job: count at RandomForest2Spark.scala:50 -2016-05-25 15:40:47 INFO DAGScheduler:59 - Got job 7 (count at RandomForest2Spark.scala:50) with 2 output partitions -2016-05-25 15:40:47 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at RandomForest2Spark.scala:50) -2016-05-25 15:40:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:40:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:40:47 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[8] at randomSplit at RandomForest2Spark.scala:29), which has no missing parents -2016-05-25 15:40:47 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1240729, maxMem=766075207 -2016-05-25 15:40:47 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-05-25 15:40:47 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=1244841, maxMem=766075207 -2016-05-25 15:40:47 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.4 MB) -2016-05-25 15:40:47 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53491 (size: 2.2 KB, free: 730.2 MB) -2016-05-25 15:40:47 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:40:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[8] at randomSplit at RandomForest2Spark.scala:29) -2016-05-25 15:40:47 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 15:40:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:40:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:40:47 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 15:40:47 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-25 15:40:47 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:40:47 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:40:47 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2082 bytes result sent to driver -2016-05-25 15:40:47 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2082 bytes result sent to driver -2016-05-25 15:40:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 11 ms on localhost (1/2) -2016-05-25 15:40:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 11 ms on localhost (2/2) -2016-05-25 15:40:47 INFO DAGScheduler:59 - ResultStage 9 (count at RandomForest2Spark.scala:50) finished in 0.013 s -2016-05-25 15:40:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 15:40:47 INFO DAGScheduler:59 - Job 7 finished: count at RandomForest2Spark.scala:50, took 0.027310 s -2016-05-25 15:40:47 INFO SparkContext:59 - Starting job: count at RandomForest2Spark.scala:56 -2016-05-25 15:40:47 INFO DAGScheduler:59 - Got job 8 (count at RandomForest2Spark.scala:56) with 2 output partitions -2016-05-25 15:40:47 INFO DAGScheduler:59 - Final stage: ResultStage 10(count at RandomForest2Spark.scala:56) -2016-05-25 15:40:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:40:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:40:47 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 15:40:47 INFO MemoryStore:59 - ensureFreeSpace(3816) called with curMem=1247109, maxMem=766075207 -2016-05-25 15:40:47 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 3.7 KB, free 729.4 MB) -2016-05-25 15:40:47 INFO MemoryStore:59 - ensureFreeSpace(2142) called with curMem=1250925, maxMem=766075207 -2016-05-25 15:40:47 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.4 MB) -2016-05-25 15:40:47 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53491 (size: 2.1 KB, free: 730.2 MB) -2016-05-25 15:40:47 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:40:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 15:40:47 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-25 15:40:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:40:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:40:47 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 15:40:47 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-25 15:40:47 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:40:47 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2082 bytes result sent to driver -2016-05-25 15:40:47 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:40:47 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2082 bytes result sent to driver -2016-05-25 15:40:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 9 ms on localhost (1/2) -2016-05-25 15:40:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 8 ms on localhost (2/2) -2016-05-25 15:40:47 INFO DAGScheduler:59 - ResultStage 10 (count at RandomForest2Spark.scala:56) finished in 0.011 s -2016-05-25 15:40:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 15:40:47 INFO DAGScheduler:59 - Job 8 finished: count at RandomForest2Spark.scala:56, took 0.023814 s -2016-05-25 15:40:47 INFO SparkContext:59 - Starting job: count at RandomForest2Spark.scala:57 -2016-05-25 15:40:47 INFO DAGScheduler:59 - Got job 9 (count at RandomForest2Spark.scala:57) with 2 output partitions -2016-05-25 15:40:47 INFO DAGScheduler:59 - Final stage: ResultStage 11(count at RandomForest2Spark.scala:57) -2016-05-25 15:40:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:40:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:40:47 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[7] at randomSplit at RandomForest2Spark.scala:29), which has no missing parents -2016-05-25 15:40:47 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1253067, maxMem=766075207 -2016-05-25 15:40:47 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-05-25 15:40:47 INFO MemoryStore:59 - ensureFreeSpace(2265) called with curMem=1257179, maxMem=766075207 -2016-05-25 15:40:47 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.4 MB) -2016-05-25 15:40:47 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53491 (size: 2.2 KB, free: 730.2 MB) -2016-05-25 15:40:47 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:40:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[7] at randomSplit at RandomForest2Spark.scala:29) -2016-05-25 15:40:47 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-25 15:40:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:40:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:40:47 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-25 15:40:47 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-25 15:40:47 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:40:47 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:40:47 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2082 bytes result sent to driver -2016-05-25 15:40:47 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2082 bytes result sent to driver -2016-05-25 15:40:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 8 ms on localhost (1/2) -2016-05-25 15:40:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 9 ms on localhost (2/2) -2016-05-25 15:40:47 INFO DAGScheduler:59 - ResultStage 11 (count at RandomForest2Spark.scala:57) finished in 0.010 s -2016-05-25 15:40:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 15:40:47 INFO DAGScheduler:59 - Job 9 finished: count at RandomForest2Spark.scala:57, took 0.023594 s -2016-05-25 15:40:47 INFO SparkContext:59 - Starting job: count at RandomForest2Spark.scala:58 -2016-05-25 15:40:47 INFO DAGScheduler:59 - Got job 10 (count at RandomForest2Spark.scala:58) with 2 output partitions -2016-05-25 15:40:47 INFO DAGScheduler:59 - Final stage: ResultStage 12(count at RandomForest2Spark.scala:58) -2016-05-25 15:40:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:40:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:40:47 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[8] at randomSplit at RandomForest2Spark.scala:29), which has no missing parents -2016-05-25 15:40:47 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1259444, maxMem=766075207 -2016-05-25 15:40:47 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-05-25 15:40:47 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=1263556, maxMem=766075207 -2016-05-25 15:40:47 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.4 MB) -2016-05-25 15:40:47 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53491 (size: 2.2 KB, free: 730.2 MB) -2016-05-25 15:40:47 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:40:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[8] at randomSplit at RandomForest2Spark.scala:29) -2016-05-25 15:40:47 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-25 15:40:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:40:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:40:47 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-25 15:40:47 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-25 15:40:47 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:40:47 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:40:47 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2082 bytes result sent to driver -2016-05-25 15:40:47 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2082 bytes result sent to driver -2016-05-25 15:40:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 9 ms on localhost (1/2) -2016-05-25 15:40:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 9 ms on localhost (2/2) -2016-05-25 15:40:47 INFO DAGScheduler:59 - ResultStage 12 (count at RandomForest2Spark.scala:58) finished in 0.010 s -2016-05-25 15:40:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 15:40:47 INFO DAGScheduler:59 - Job 10 finished: count at RandomForest2Spark.scala:58, took 0.020933 s -2016-05-25 15:40:47 INFO SparkContext:59 - Starting job: take at RandomForest2Spark.scala:63 -2016-05-25 15:40:47 INFO DAGScheduler:59 - Got job 11 (take at RandomForest2Spark.scala:63) with 1 output partitions -2016-05-25 15:40:47 INFO DAGScheduler:59 - Final stage: ResultStage 13(take at RandomForest2Spark.scala:63) -2016-05-25 15:40:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:40:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:40:47 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[20] at map at RandomForest2Spark.scala:46), which has no missing parents -2016-05-25 15:40:47 INFO MemoryStore:59 - ensureFreeSpace(8464) called with curMem=1265824, maxMem=766075207 -2016-05-25 15:40:47 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 8.3 KB, free 729.4 MB) -2016-05-25 15:40:47 INFO MemoryStore:59 - ensureFreeSpace(4279) called with curMem=1274288, maxMem=766075207 -2016-05-25 15:40:47 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 4.2 KB, free 729.4 MB) -2016-05-25 15:40:47 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53491 (size: 4.2 KB, free: 730.2 MB) -2016-05-25 15:40:47 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:40:47 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 13 (MapPartitionsRDD[20] at map at RandomForest2Spark.scala:46) -2016-05-25 15:40:47 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 1 tasks -2016-05-25 15:40:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:40:47 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-25 15:40:47 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:40:47 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2401 bytes result sent to driver -2016-05-25 15:40:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 12 ms on localhost (1/1) -2016-05-25 15:40:47 INFO DAGScheduler:59 - ResultStage 13 (take at RandomForest2Spark.scala:63) finished in 0.012 s -2016-05-25 15:40:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 15:40:47 INFO DAGScheduler:59 - Job 11 finished: take at RandomForest2Spark.scala:63, took 0.026064 s -2016-05-25 15:40:47 INFO SparkContext:59 - Starting job: saveAsTextFile at treeEnsembleModels.scala:448 -2016-05-25 15:40:47 INFO DAGScheduler:59 - Got job 12 (saveAsTextFile at treeEnsembleModels.scala:448) with 1 output partitions -2016-05-25 15:40:47 INFO DAGScheduler:59 - Final stage: ResultStage 14(saveAsTextFile at treeEnsembleModels.scala:448) -2016-05-25 15:40:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:40:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:40:47 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[23] at saveAsTextFile at treeEnsembleModels.scala:448), which has no missing parents -2016-05-25 15:40:47 INFO MemoryStore:59 - ensureFreeSpace(126992) called with curMem=1278567, maxMem=766075207 -2016-05-25 15:40:47 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 124.0 KB, free 729.2 MB) -2016-05-25 15:40:47 INFO MemoryStore:59 - ensureFreeSpace(42626) called with curMem=1405559, maxMem=766075207 -2016-05-25 15:40:47 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 41.6 KB, free 729.2 MB) -2016-05-25 15:40:47 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53491 (size: 41.6 KB, free: 730.2 MB) -2016-05-25 15:40:47 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:40:47 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 14 (MapPartitionsRDD[23] at saveAsTextFile at treeEnsembleModels.scala:448) -2016-05-25 15:40:47 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 1 tasks -2016-05-25 15:40:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 26, localhost, PROCESS_LOCAL, 2289 bytes) -2016-05-25 15:40:47 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 26) -2016-05-25 15:40:47 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251540_0014_m_000000_26' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525154047164/result/metadata/_temporary/0/task_201605251540_0014_m_000000 -2016-05-25 15:40:47 INFO SparkHadoopMapRedUtil:59 - attempt_201605251540_0014_m_000000_26: Committed -2016-05-25 15:40:47 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 26). 1864 bytes result sent to driver -2016-05-25 15:40:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 26) in 151 ms on localhost (1/1) -2016-05-25 15:40:47 INFO DAGScheduler:59 - ResultStage 14 (saveAsTextFile at treeEnsembleModels.scala:448) finished in 0.152 s -2016-05-25 15:40:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 15:40:47 INFO DAGScheduler:59 - Job 12 finished: saveAsTextFile at treeEnsembleModels.scala:448, took 0.223434 s -2016-05-25 15:40:49 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-25 15:40:49 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:53491 in memory (size: 2.1 KB, free: 730.2 MB) -2016-05-25 15:40:49 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 15:40:49 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:53491 in memory (size: 2.2 KB, free: 730.2 MB) -2016-05-25 15:40:49 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 15:40:49 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:53491 in memory (size: 4.1 KB, free: 730.2 MB) -2016-05-25 15:40:49 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 15:40:49 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:53491 in memory (size: 45.0 KB, free: 730.2 MB) -2016-05-25 15:40:49 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:53491 in memory (size: 41.6 KB, free: 730.3 MB) -2016-05-25 15:40:49 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-25 15:40:49 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:53491 in memory (size: 4.2 KB, free: 730.3 MB) -2016-05-25 15:40:49 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-25 15:40:49 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:53491 in memory (size: 2.2 KB, free: 730.3 MB) -2016-05-25 15:40:49 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-25 15:40:49 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:53491 in memory (size: 2.2 KB, free: 730.3 MB) -2016-05-25 15:40:49 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 15:40:49 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 15:40:49 INFO SparkContext:59 - Starting job: parquet at treeEnsembleModels.scala:454 -2016-05-25 15:40:49 INFO DAGScheduler:59 - Got job 13 (parquet at treeEnsembleModels.scala:454) with 4 output partitions -2016-05-25 15:40:49 INFO DAGScheduler:59 - Final stage: ResultStage 15(parquet at treeEnsembleModels.scala:454) -2016-05-25 15:40:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:40:49 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:40:49 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[26] at rddToDataFrameHolder at treeEnsembleModels.scala:451), which has no missing parents -2016-05-25 15:40:49 INFO MemoryStore:59 - ensureFreeSpace(68568) called with curMem=965219, maxMem=766075207 -2016-05-25 15:40:49 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 67.0 KB, free 729.6 MB) -2016-05-25 15:40:49 INFO MemoryStore:59 - ensureFreeSpace(24138) called with curMem=1033787, maxMem=766075207 -2016-05-25 15:40:49 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 23.6 KB, free 729.6 MB) -2016-05-25 15:40:49 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53491 (size: 23.6 KB, free: 730.3 MB) -2016-05-25 15:40:49 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:40:49 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 15 (MapPartitionsRDD[26] at rddToDataFrameHolder at treeEnsembleModels.scala:451) -2016-05-25 15:40:49 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 4 tasks -2016-05-25 15:40:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 27, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 15:40:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 28, localhost, PROCESS_LOCAL, 4439 bytes) -2016-05-25 15:40:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 4439 bytes) -2016-05-25 15:40:49 INFO TaskSetManager:59 - Starting task 3.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 4444 bytes) -2016-05-25 15:40:49 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 27) -2016-05-25 15:40:49 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 28) -2016-05-25 15:40:49 INFO Executor:59 - Running task 2.0 in stage 15.0 (TID 29) -2016-05-25 15:40:49 INFO Executor:59 - Running task 3.0 in stage 15.0 (TID 30) -2016-05-25 15:40:49 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 15:40:49 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 15:40:49 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 15:40:49 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 15:40:49 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 15:40:49 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 15:40:49 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 15:40:49 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 15:40:49 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 15:40:49 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 15:40:49 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 15:40:49 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 15:40:49 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 15:40:50 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,568 -2016-05-25 15:40:50 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,864 -2016-05-25 15:40:50 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,864 -2016-05-25 15:40:50 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,856 -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 5 values, 8B raw, 28B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 5 values, 8B raw, 28B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 71B for [nodeId] INT32: 5 values, 26B raw, 38B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 71B for [nodeId] INT32: 5 values, 26B raw, 38B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, predict] DOUBLE: 5 values, 10B raw, 30B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 2 entries, 16B raw, 2B comp} -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, predict] DOUBLE: 5 values, 10B raw, 30B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 2 entries, 16B raw, 2B comp} -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 72B for [predict, prob] DOUBLE: 5 values, 11B raw, 31B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 3 entries, 24B raw, 3B comp} -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 72B for [predict, prob] DOUBLE: 5 values, 11B raw, 31B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 3 entries, 24B raw, 3B comp} -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 71B for [impurity] DOUBLE: 5 values, 10B raw, 30B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 3 entries, 24B raw, 3B comp} -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 5 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 66B for [split, feature] INT32: 5 values, 15B raw, 33B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 71B for [split, threshold] DOUBLE: 5 values, 23B raw, 30B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 60B for [split, featureType] INT32: 5 values, 9B raw, 27B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 52B for [split, categories, bag, array] DOUBLE: 5 values, 14B raw, 31B comp, 1 pages, encodings: [RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 64B for [leftNodeId] INT32: 5 values, 14B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 64B for [rightNodeId] INT32: 5 values, 14B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 84B for [infoGain] DOUBLE: 5 values, 22B raw, 43B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 5 values, 8B raw, 28B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 71B for [nodeId] INT32: 5 values, 26B raw, 38B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 71B for [predict, predict] DOUBLE: 5 values, 10B raw, 30B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 2 entries, 16B raw, 2B comp} -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 72B for [predict, prob] DOUBLE: 5 values, 11B raw, 31B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 3 entries, 24B raw, 3B comp} -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 71B for [impurity] DOUBLE: 5 values, 10B raw, 30B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 3 entries, 24B raw, 3B comp} -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 5 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 66B for [split, feature] INT32: 5 values, 15B raw, 33B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 74B for [split, threshold] DOUBLE: 5 values, 23B raw, 33B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 60B for [split, featureType] INT32: 5 values, 9B raw, 27B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 52B for [split, categories, bag, array] DOUBLE: 5 values, 14B raw, 31B comp, 1 pages, encodings: [RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 64B for [leftNodeId] INT32: 5 values, 14B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 64B for [rightNodeId] INT32: 5 values, 14B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 84B for [infoGain] DOUBLE: 5 values, 22B raw, 43B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 71B for [impurity] DOUBLE: 5 values, 10B raw, 30B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 3 entries, 24B raw, 3B comp} -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 5 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 66B for [split, feature] INT32: 5 values, 15B raw, 33B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 68B for [split, threshold] DOUBLE: 5 values, 9B raw, 27B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 1 entries, 8B raw, 1B comp} -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 60B for [split, featureType] INT32: 5 values, 9B raw, 27B comp, 1 pages, encodings: [PLAIN_DICTIONARY, BIT_PACKED, RLE], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 52B for [split, categories, bag, array] DOUBLE: 5 values, 14B raw, 31B comp, 1 pages, encodings: [RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 64B for [leftNodeId] INT32: 5 values, 14B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 64B for [rightNodeId] INT32: 5 values, 14B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:40:50 INFO ColumnChunkPageWriteStore:151 - written 83B for [infoGain] DOUBLE: 5 values, 22B raw, 42B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-05-25 15:40:50 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251540_0015_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525154047164/result/data/_temporary/0/task_201605251540_0015_m_000000 -2016-05-25 15:40:50 INFO SparkHadoopMapRedUtil:59 - attempt_201605251540_0015_m_000000_0: Committed -2016-05-25 15:40:50 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 27). 915 bytes result sent to driver -2016-05-25 15:40:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 27) in 608 ms on localhost (1/4) -2016-05-25 15:40:50 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251540_0015_m_000001_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525154047164/result/data/_temporary/0/task_201605251540_0015_m_000001 -2016-05-25 15:40:50 INFO SparkHadoopMapRedUtil:59 - attempt_201605251540_0015_m_000001_0: Committed -2016-05-25 15:40:50 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 28). 915 bytes result sent to driver -2016-05-25 15:40:50 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251540_0015_m_000003_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525154047164/result/data/_temporary/0/task_201605251540_0015_m_000003 -2016-05-25 15:40:50 INFO SparkHadoopMapRedUtil:59 - attempt_201605251540_0015_m_000003_0: Committed -2016-05-25 15:40:50 INFO Executor:59 - Finished task 3.0 in stage 15.0 (TID 30). 915 bytes result sent to driver -2016-05-25 15:40:50 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251540_0015_m_000002_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525154047164/result/data/_temporary/0/task_201605251540_0015_m_000002 -2016-05-25 15:40:50 INFO SparkHadoopMapRedUtil:59 - attempt_201605251540_0015_m_000002_0: Committed -2016-05-25 15:40:50 INFO Executor:59 - Finished task 2.0 in stage 15.0 (TID 29). 915 bytes result sent to driver -2016-05-25 15:40:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 28) in 658 ms on localhost (2/4) -2016-05-25 15:40:50 INFO TaskSetManager:59 - Finished task 2.0 in stage 15.0 (TID 29) in 658 ms on localhost (3/4) -2016-05-25 15:40:50 INFO DAGScheduler:59 - ResultStage 15 (parquet at treeEnsembleModels.scala:454) finished in 0.661 s -2016-05-25 15:40:50 INFO TaskSetManager:59 - Finished task 3.0 in stage 15.0 (TID 30) in 658 ms on localhost (4/4) -2016-05-25 15:40:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-25 15:40:50 INFO DAGScheduler:59 - Job 13 finished: parquet at treeEnsembleModels.scala:454, took 0.704328 s -2016-05-25 15:40:50 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 15:40:50 INFO DefaultWriterContainer:59 - Job job_201605251540_0000 committed. -2016-05-25 15:40:50 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525154047164/result/data on driver -2016-05-25 15:40:50 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525154047164/result/data on driver -2016-05-25 15:40:50 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=1057925, maxMem=766075207 -2016-05-25 15:40:50 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 225.3 KB, free 729.4 MB) -2016-05-25 15:40:50 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1288629, maxMem=766075207 -2016-05-25 15:40:50 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.3 MB) -2016-05-25 15:40:50 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53491 (size: 19.3 KB, free: 730.3 MB) -2016-05-25 15:40:50 INFO SparkContext:59 - Created broadcast 19 from textFile at modelSaveLoad.scala:129 -2016-05-25 15:40:50 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 15:40:50 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 15:40:50 INFO DAGScheduler:59 - Got job 14 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 15:40:50 INFO DAGScheduler:59 - Final stage: ResultStage 16(first at modelSaveLoad.scala:129) -2016-05-25 15:40:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:40:50 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:40:50 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[29] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 15:40:50 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=1308417, maxMem=766075207 -2016-05-25 15:40:50 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.1 KB, free 729.3 MB) -2016-05-25 15:40:50 INFO MemoryStore:59 - ensureFreeSpace(1893) called with curMem=1311633, maxMem=766075207 -2016-05-25 15:40:50 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 1893.0 B, free 729.3 MB) -2016-05-25 15:40:50 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:53491 (size: 1893.0 B, free: 730.2 MB) -2016-05-25 15:40:50 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:40:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 16 (MapPartitionsRDD[29] at textFile at modelSaveLoad.scala:129) -2016-05-25 15:40:50 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 1 tasks -2016-05-25 15:40:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2241 bytes) -2016-05-25 15:40:50 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-25 15:40:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525154047164/result/metadata/part-00000:0+100 -2016-05-25 15:40:50 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2249 bytes result sent to driver -2016-05-25 15:40:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 9 ms on localhost (1/1) -2016-05-25 15:40:50 INFO DAGScheduler:59 - ResultStage 16 (first at modelSaveLoad.scala:129) finished in 0.011 s -2016-05-25 15:40:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-25 15:40:50 INFO DAGScheduler:59 - Job 14 finished: first at modelSaveLoad.scala:129, took 0.021126 s -2016-05-25 15:40:51 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 15:40:51 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53491 in memory (size: 120.1 KB, free: 730.4 MB) -2016-05-25 15:40:51 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:53491 in memory (size: 1893.0 B, free: 730.4 MB) -2016-05-25 15:40:51 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-25 15:40:51 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:53491 in memory (size: 19.3 KB, free: 730.4 MB) -2016-05-25 15:40:51 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525154047164/result/data on driver -2016-05-25 15:40:51 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:53491 in memory (size: 23.6 KB, free: 730.4 MB) -2016-05-25 15:40:51 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-25 15:40:51 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 15:40:51 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-25 15:40:51 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53491 in memory (size: 619.0 B, free: 730.4 MB) -2016-05-25 15:40:51 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 15:40:51 INFO ContextCleaner:59 - Cleaned RDD 13 -2016-05-25 15:40:51 INFO SparkContext:59 - Starting job: parquet at treeEnsembleModels.scala:477 -2016-05-25 15:40:51 INFO DAGScheduler:59 - Got job 15 (parquet at treeEnsembleModels.scala:477) with 4 output partitions -2016-05-25 15:40:51 INFO DAGScheduler:59 - Final stage: ResultStage 17(parquet at treeEnsembleModels.scala:477) -2016-05-25 15:40:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:40:51 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:40:51 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[31] at parquet at treeEnsembleModels.scala:477), which has no missing parents -2016-05-25 15:40:51 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=342308, maxMem=766075207 -2016-05-25 15:40:51 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 61.5 KB, free 730.2 MB) -2016-05-25 15:40:51 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=405308, maxMem=766075207 -2016-05-25 15:40:51 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.2 MB) -2016-05-25 15:40:51 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:53491 (size: 20.6 KB, free: 730.4 MB) -2016-05-25 15:40:51 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:40:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (MapPartitionsRDD[31] at parquet at treeEnsembleModels.scala:477) -2016-05-25 15:40:51 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-05-25 15:40:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 32, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 15:40:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 15:40:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 15:40:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 35, localhost, PROCESS_LOCAL, 2282 bytes) -2016-05-25 15:40:51 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 33) -2016-05-25 15:40:51 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 34) -2016-05-25 15:40:51 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 32) -2016-05-25 15:40:51 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 35) -2016-05-25 15:40:51 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 15:40:51 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 32). 936 bytes result sent to driver -2016-05-25 15:40:51 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 15:40:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 32) in 70 ms on localhost (1/4) -2016-05-25 15:40:51 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 15:40:51 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 33). 936 bytes result sent to driver -2016-05-25 15:40:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 33) in 70 ms on localhost (2/4) -2016-05-25 15:40:51 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 15:40:51 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 34). 936 bytes result sent to driver -2016-05-25 15:40:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 34) in 94 ms on localhost (3/4) -2016-05-25 15:40:51 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 35). 2321 bytes result sent to driver -2016-05-25 15:40:51 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 35) in 118 ms on localhost (4/4) -2016-05-25 15:40:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-25 15:40:51 INFO DAGScheduler:59 - ResultStage 17 (parquet at treeEnsembleModels.scala:477) finished in 0.130 s -2016-05-25 15:40:51 INFO DAGScheduler:59 - Job 15 finished: parquet at treeEnsembleModels.scala:477, took 0.160652 s -2016-05-25 15:40:51 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=426381, maxMem=766075207 -2016-05-25 15:40:51 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 86.4 KB, free 730.1 MB) -2016-05-25 15:40:51 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=514893, maxMem=766075207 -2016-05-25 15:40:51 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-25 15:40:51 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:53491 (size: 19.3 KB, free: 730.4 MB) -2016-05-25 15:40:51 INFO SparkContext:59 - Created broadcast 22 from map at treeEnsembleModels.scala:477 -2016-05-25 15:40:52 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:53491 in memory (size: 20.6 KB, free: 730.4 MB) -2016-05-25 15:40:52 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-25 15:40:52 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 15:40:52 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525154047164/result/data/part-r-00003-d1371e2b-8fd4-480b-9114-9a4e2cc89b61.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525154047164/result/data/part-r-00001-d1371e2b-8fd4-480b-9114-9a4e2cc89b61.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525154047164/result/data/part-r-00000-d1371e2b-8fd4-480b-9114-9a4e2cc89b61.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/RandomForestModel20160525154047164/result/data/part-r-00002-d1371e2b-8fd4-480b-9114-9a4e2cc89b61.gz.parquet -2016-05-25 15:41:15 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 15:41:16 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 15:41:16 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 15:41:16 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 15:41:16 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 15:41:17 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 15:41:17 INFO Remoting:74 - Starting remoting -2016-05-25 15:41:17 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53512] -2016-05-25 15:41:17 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53512. -2016-05-25 15:41:17 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 15:41:17 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 15:41:17 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-717e6899-4603-454e-812e-393e8ddcace8 -2016-05-25 15:41:17 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 15:41:17 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-889a6213-7a03-42df-b5f9-d4f127825b3e\httpd-3c9f6131-3058-46d8-ac4d-b0502db350b1 -2016-05-25 15:41:17 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 15:41:18 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:41:18 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53513 -2016-05-25 15:41:18 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53513. -2016-05-25 15:41:18 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 15:41:18 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:41:18 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 15:41:18 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 15:41:18 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 15:41:18 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 15:41:18 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 15:41:18 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53532. -2016-05-25 15:41:18 INFO NettyBlockTransferService:59 - Server created on 53532 -2016-05-25 15:41:18 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 15:41:18 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53532 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53532) -2016-05-25 15:41:18 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 15:41:19 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 15:41:19 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 15:41:19 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 15:41:19 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 15:41:19 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53532 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 15:41:19 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 15:41:20 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 15:41:21 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 15:41:21 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 15:41:21 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 15:41:21 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 15:41:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:41:21 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:41:21 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 15:41:21 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 15:41:21 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 15:41:21 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 15:41:21 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 15:41:21 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53532 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:41:21 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:41:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 15:41:21 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 15:41:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:41:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:41:21 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 15:41:21 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 15:41:21 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 15:41:21 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 15:41:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 15:41:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 15:41:21 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 15:41:21 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 15:41:21 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 15:41:21 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 15:41:21 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 15:41:22 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177630, maxMem=766075207 -2016-05-25 15:41:22 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 15:41:22 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:53532 (size: 83.6 KB, free: 730.5 MB) -2016-05-25 15:41:22 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263278, maxMem=766075207 -2016-05-25 15:41:22 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 15:41:22 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:53532 (size: 83.1 KB, free: 730.4 MB) -2016-05-25 15:41:22 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 15:41:22 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 15:41:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 423 ms on localhost (1/2) -2016-05-25 15:41:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 453 ms on localhost (2/2) -2016-05-25 15:41:22 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.475 s -2016-05-25 15:41:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 15:41:22 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.683550 s -2016-05-25 15:41:22 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 15:41:22 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 15:41:22 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 15:41:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:41:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:41:22 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 15:41:22 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348358, maxMem=766075207 -2016-05-25 15:41:22 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 15:41:22 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=353126, maxMem=766075207 -2016-05-25 15:41:22 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 15:41:22 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53532 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 15:41:22 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:41:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 15:41:22 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 15:41:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:41:22 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 15:41:22 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:41:22 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 15:41:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 28 ms on localhost (1/1) -2016-05-25 15:41:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 15:41:22 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.028 s -2016-05-25 15:41:22 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.048954 s -2016-05-25 15:41:22 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 15:41:22 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 15:41:22 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 15:41:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:41:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:41:22 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 15:41:22 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355673, maxMem=766075207 -2016-05-25 15:41:22 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 15:41:22 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=360009, maxMem=766075207 -2016-05-25 15:41:22 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 15:41:22 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53532 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 15:41:22 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:41:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 15:41:22 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 15:41:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:41:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:41:22 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 15:41:22 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 15:41:22 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:41:22 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 15:41:22 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:41:22 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 15:41:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 16 ms on localhost (1/2) -2016-05-25 15:41:22 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.018 s -2016-05-25 15:41:22 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.035558 s -2016-05-25 15:41:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 18 ms on localhost (2/2) -2016-05-25 15:41:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 15:41:22 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 15:41:22 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 15:41:22 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 15:41:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:41:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:41:22 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 15:41:22 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362328, maxMem=766075207 -2016-05-25 15:41:22 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 15:41:22 INFO MemoryStore:59 - ensureFreeSpace(2631) called with curMem=367392, maxMem=766075207 -2016-05-25 15:41:22 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 15:41:22 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53532 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:41:22 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:41:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 15:41:22 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 15:41:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 15:41:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 15:41:22 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 15:41:22 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 15:41:22 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:41:22 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:41:22 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 67088 bytes result sent to driver -2016-05-25 15:41:22 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 70783 bytes result sent to driver -2016-05-25 15:41:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 31 ms on localhost (1/2) -2016-05-25 15:41:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 34 ms on localhost (2/2) -2016-05-25 15:41:22 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.036 s -2016-05-25 15:41:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 15:41:22 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.056366 s -2016-05-25 15:41:23 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53532 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:41:23 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 15:41:23 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53532 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 15:41:23 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 15:41:23 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53532 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 15:41:23 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 15:41:24 INFO MemoryStore:59 - ensureFreeSpace(512) called with curMem=348358, maxMem=766075207 -2016-05-25 15:41:24 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 512.0 B, free 730.3 MB) -2016-05-25 15:41:24 INFO MemoryStore:59 - ensureFreeSpace(619) called with curMem=348870, maxMem=766075207 -2016-05-25 15:41:24 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 619.0 B, free 730.3 MB) -2016-05-25 15:41:24 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53532 (size: 619.0 B, free: 730.4 MB) -2016-05-25 15:41:24 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 15:41:24 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:41:24 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:41:24 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:41:24 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:41:24 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 15:41:24 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 15:41:24 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:41:24 INFO MemoryStore:59 - ensureFreeSpace(516312) called with curMem=349489, maxMem=766075207 -2016-05-25 15:41:24 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 504.2 KB, free 729.8 MB) -2016-05-25 15:41:24 INFO MemoryStore:59 - ensureFreeSpace(127118) called with curMem=865801, maxMem=766075207 -2016-05-25 15:41:24 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 124.1 KB, free 729.6 MB) -2016-05-25 15:41:24 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53532 (size: 124.1 KB, free: 730.3 MB) -2016-05-25 15:41:24 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:41:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:41:24 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 15:41:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:41:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:41:24 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 15:41:24 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 15:41:24 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 15:41:24 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:41:24 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 15:41:24 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:41:24 INFO MemoryStore:59 - ensureFreeSpace(112184) called with curMem=992919, maxMem=766075207 -2016-05-25 15:41:24 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 109.6 KB, free 729.5 MB) -2016-05-25 15:41:24 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:53532 (size: 109.6 KB, free: 730.2 MB) -2016-05-25 15:41:24 INFO MemoryStore:59 - ensureFreeSpace(115056) called with curMem=1105103, maxMem=766075207 -2016-05-25 15:41:24 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 112.4 KB, free 729.4 MB) -2016-05-25 15:41:24 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:53532 (size: 112.4 KB, free: 730.1 MB) -2016-05-25 15:41:24 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 15:41:24 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 15:41:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 367 ms on localhost (1/2) -2016-05-25 15:41:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 366 ms on localhost (2/2) -2016-05-25 15:41:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 15:41:24 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.371 s -2016-05-25 15:41:24 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:41:24 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:41:24 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 15:41:24 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:41:24 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 15:41:24 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:41:24 INFO MemoryStore:59 - ensureFreeSpace(225760) called with curMem=1220159, maxMem=766075207 -2016-05-25 15:41:24 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 220.5 KB, free 729.2 MB) -2016-05-25 15:41:24 INFO MemoryStore:59 - ensureFreeSpace(47524) called with curMem=1445919, maxMem=766075207 -2016-05-25 15:41:24 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 46.4 KB, free 729.2 MB) -2016-05-25 15:41:24 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53532 (size: 46.4 KB, free: 730.0 MB) -2016-05-25 15:41:24 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:41:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 15:41:24 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 15:41:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:41:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:41:24 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 15:41:24 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 15:41:24 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:41:24 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:41:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 8 ms -2016-05-25 15:41:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-05-25 15:41:25 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2774 bytes result sent to driver -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 337 ms on localhost (1/2) -2016-05-25 15:41:25 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2928 bytes result sent to driver -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 348 ms on localhost (2/2) -2016-05-25 15:41:25 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.349 s -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 15:41:25 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.783612 s -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(512) called with curMem=1493443, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 512.0 B, free 729.2 MB) -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(609) called with curMem=1493955, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 609.0 B, free 729.2 MB) -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53532 (size: 609.0 B, free: 730.0 MB) -2016-05-25 15:41:25 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 15:41:25 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Registering RDD 17 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:41:25 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:41:25 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:41:25 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 15:41:25 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(517072) called with curMem=1494564, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 505.0 KB, free 728.7 MB) -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(127471) called with curMem=2011636, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 124.5 KB, free 728.5 MB) -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53532 (size: 124.5 KB, free: 729.9 MB) -2016-05-25 15:41:25 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:41:25 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 15:41:25 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 15:41:25 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-25 15:41:25 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 37 ms on localhost (1/2) -2016-05-25 15:41:25 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-25 15:41:25 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 57 ms on localhost (2/2) -2016-05-25 15:41:25 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.058 s -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 15:41:25 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:41:25 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:41:25 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 15:41:25 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:41:25 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(225760) called with curMem=2139107, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 220.5 KB, free 728.3 MB) -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(47558) called with curMem=2364867, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 46.4 KB, free 728.3 MB) -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53532 (size: 46.4 KB, free: 729.9 MB) -2016-05-25 15:41:25 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645) -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:41:25 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 15:41:25 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 15:41:25 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:41:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 15:41:25 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:41:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 15:41:25 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2774 bytes result sent to driver -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 44 ms on localhost (1/2) -2016-05-25 15:41:25 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2928 bytes result sent to driver -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 71 ms on localhost (2/2) -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 15:41:25 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.072 s -2016-05-25 15:41:25 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.172109 s -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(360) called with curMem=2412425, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 360.0 B, free 728.3 MB) -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(493) called with curMem=2412785, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 493.0 B, free 728.3 MB) -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53532 (size: 493.0 B, free: 729.9 MB) -2016-05-25 15:41:25 INFO SparkContext:59 - Created broadcast 11 from broadcast at DecisionTree.scala:604 -2016-05-25 15:41:25 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Registering RDD 20 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:41:25 INFO DAGScheduler:59 - Got job 6 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:41:25 INFO DAGScheduler:59 - Final stage: ResultStage 9(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:41:25 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 8) -2016-05-25 15:41:25 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 8) -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting ShuffleMapStage 8 (MapPartitionsRDD[20] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(517552) called with curMem=2413278, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 505.4 KB, free 727.8 MB) -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(127552) called with curMem=2930830, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 124.6 KB, free 727.7 MB) -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53532 (size: 124.6 KB, free: 729.7 MB) -2016-05-25 15:41:25 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 8 (MapPartitionsRDD[20] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:41:25 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 15:41:25 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 15:41:25 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-25 15:41:25 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-25 15:41:25 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2254 bytes result sent to driver -2016-05-25 15:41:25 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2254 bytes result sent to driver -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 37 ms on localhost (1/2) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 39 ms on localhost (2/2) -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 15:41:25 INFO DAGScheduler:59 - ShuffleMapStage 8 (mapPartitions at DecisionTree.scala:625) finished in 0.040 s -2016-05-25 15:41:25 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:41:25 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:41:25 INFO DAGScheduler:59 - waiting: Set(ResultStage 9) -2016-05-25 15:41:25 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:41:25 INFO DAGScheduler:59 - Missing parents for ResultStage 9: List() -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[22] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(225696) called with curMem=3058382, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 220.4 KB, free 727.5 MB) -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(47537) called with curMem=3284078, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 46.4 KB, free 727.4 MB) -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53532 (size: 46.4 KB, free: 729.7 MB) -2016-05-25 15:41:25 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[22] at map at DecisionTree.scala:645) -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:41:25 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 15:41:25 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-25 15:41:25 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:41:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 15:41:25 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:41:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 15:41:25 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2774 bytes result sent to driver -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 28 ms on localhost (1/2) -2016-05-25 15:41:25 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2774 bytes result sent to driver -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 33 ms on localhost (2/2) -2016-05-25 15:41:25 INFO DAGScheduler:59 - ResultStage 9 (collectAsMap at DecisionTree.scala:654) finished in 0.035 s -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 15:41:25 INFO DAGScheduler:59 - Job 6 finished: collectAsMap at DecisionTree.scala:654, took 0.124834 s -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(208) called with curMem=3331615, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 208.0 B, free 727.4 MB) -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(365) called with curMem=3331823, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 365.0 B, free 727.4 MB) -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53532 (size: 365.0 B, free: 729.7 MB) -2016-05-25 15:41:25 INFO SparkContext:59 - Created broadcast 14 from broadcast at DecisionTree.scala:604 -2016-05-25 15:41:25 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Registering RDD 23 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:41:25 INFO DAGScheduler:59 - Got job 7 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:41:25 INFO DAGScheduler:59 - Final stage: ResultStage 11(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:41:25 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 10) -2016-05-25 15:41:25 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 10) -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting ShuffleMapStage 10 (MapPartitionsRDD[23] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(517752) called with curMem=3332188, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 505.6 KB, free 726.9 MB) -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(127521) called with curMem=3849940, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 124.5 KB, free 726.8 MB) -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53532 (size: 124.5 KB, free: 729.6 MB) -2016-05-25 15:41:25 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 10 (MapPartitionsRDD[23] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:41:25 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 15:41:25 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-25 15:41:25 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-25 15:41:25 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-25 15:41:25 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2254 bytes result sent to driver -2016-05-25 15:41:25 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2254 bytes result sent to driver -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 30 ms on localhost (1/2) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 30 ms on localhost (2/2) -2016-05-25 15:41:25 INFO DAGScheduler:59 - ShuffleMapStage 10 (mapPartitions at DecisionTree.scala:625) finished in 0.031 s -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 15:41:25 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:41:25 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:41:25 INFO DAGScheduler:59 - waiting: Set(ResultStage 11) -2016-05-25 15:41:25 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:41:25 INFO DAGScheduler:59 - Missing parents for ResultStage 11: List() -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[25] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(225640) called with curMem=3977461, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 220.4 KB, free 726.6 MB) -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(47518) called with curMem=4203101, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 46.4 KB, free 726.5 MB) -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53532 (size: 46.4 KB, free: 729.5 MB) -2016-05-25 15:41:25 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[25] at map at DecisionTree.scala:645) -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:41:25 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-25 15:41:25 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-25 15:41:25 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 15:41:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 15:41:25 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:41:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 15:41:25 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 1161 bytes result sent to driver -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 11 ms on localhost (1/2) -2016-05-25 15:41:25 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2774 bytes result sent to driver -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 35 ms on localhost (2/2) -2016-05-25 15:41:25 INFO DAGScheduler:59 - ResultStage 11 (collectAsMap at DecisionTree.scala:654) finished in 0.035 s -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 15:41:25 INFO DAGScheduler:59 - Job 7 finished: collectAsMap at DecisionTree.scala:654, took 0.111777 s -2016-05-25 15:41:25 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 15:41:25 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 15:41:25 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 15:41:25 INFO RandomForest:59 - init: 1.647013074 - total: 3.248695385 - findSplitsBins: 1.400938743 - findBestSplits: 1.573984096 - chooseSplits: 1.567696957 -2016-05-25 15:41:25 INFO SparkContext:59 - Starting job: count at RandomForest2Spark.scala:50 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Got job 8 (count at RandomForest2Spark.scala:50) with 2 output partitions -2016-05-25 15:41:25 INFO DAGScheduler:59 - Final stage: ResultStage 12(count at RandomForest2Spark.scala:50) -2016-05-25 15:41:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:41:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[27] at filter at RandomForest2Spark.scala:50), which has no missing parents -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(9064) called with curMem=4023379, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 8.9 KB, free 726.7 MB) -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(4458) called with curMem=4032443, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 4.4 KB, free 726.7 MB) -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53532 (size: 4.4 KB, free: 729.7 MB) -2016-05-25 15:41:25 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[27] at filter at RandomForest2Spark.scala:50) -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:41:25 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-25 15:41:25 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-25 15:41:25 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:41:25 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:41:25 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2082 bytes result sent to driver -2016-05-25 15:41:25 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2082 bytes result sent to driver -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 17 ms on localhost (1/2) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 17 ms on localhost (2/2) -2016-05-25 15:41:25 INFO DAGScheduler:59 - ResultStage 12 (count at RandomForest2Spark.scala:50) finished in 0.018 s -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 15:41:25 INFO DAGScheduler:59 - Job 8 finished: count at RandomForest2Spark.scala:50, took 0.030564 s -2016-05-25 15:41:25 INFO SparkContext:59 - Starting job: count at RandomForest2Spark.scala:50 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Got job 9 (count at RandomForest2Spark.scala:50) with 2 output partitions -2016-05-25 15:41:25 INFO DAGScheduler:59 - Final stage: ResultStage 13(count at RandomForest2Spark.scala:50) -2016-05-25 15:41:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:41:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[8] at randomSplit at RandomForest2Spark.scala:29), which has no missing parents -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=4036901, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 4.0 KB, free 726.7 MB) -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=4041013, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.2 KB, free 726.7 MB) -2016-05-25 15:41:25 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53532 (size: 2.2 KB, free: 729.7 MB) -2016-05-25 15:41:25 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[8] at randomSplit at RandomForest2Spark.scala:29) -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:53532 in memory (size: 124.5 KB, free: 729.9 MB) -2016-05-25 15:41:25 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:41:25 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-25 15:41:25 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-25 15:41:25 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:41:25 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 2082 bytes result sent to driver -2016-05-25 15:41:25 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-25 15:41:25 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:41:25 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2082 bytes result sent to driver -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:53532 in memory (size: 609.0 B, free: 729.9 MB) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 23 ms on localhost (1/2) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 25 ms on localhost (2/2) -2016-05-25 15:41:25 INFO DAGScheduler:59 - ResultStage 13 (count at RandomForest2Spark.scala:50) finished in 0.026 s -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 15:41:25 INFO DAGScheduler:59 - Job 9 finished: count at RandomForest2Spark.scala:50, took 0.075829 s -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:53532 in memory (size: 46.4 KB, free: 729.9 MB) -2016-05-25 15:41:25 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:53532 in memory (size: 124.1 KB, free: 730.0 MB) -2016-05-25 15:41:25 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 15:41:25 INFO SparkContext:59 - Starting job: count at RandomForest2Spark.scala:56 -2016-05-25 15:41:25 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Got job 10 (count at RandomForest2Spark.scala:56) with 2 output partitions -2016-05-25 15:41:25 INFO DAGScheduler:59 - Final stage: ResultStage 14(count at RandomForest2Spark.scala:56) -2016-05-25 15:41:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:41:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:53532 in memory (size: 619.0 B, free: 730.0 MB) -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(3816) called with curMem=2479772, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.7 KB, free 728.2 MB) -2016-05-25 15:41:25 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(2142) called with curMem=2483588, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 2.1 KB, free 728.2 MB) -2016-05-25 15:41:25 INFO ContextCleaner:59 - Cleaned RDD 13 -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53532 (size: 2.1 KB, free: 730.0 MB) -2016-05-25 15:41:25 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:53532 in memory (size: 4.4 KB, free: 730.0 MB) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:41:25 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-25 15:41:25 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:41:25 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-25 15:41:25 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2082 bytes result sent to driver -2016-05-25 15:41:25 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-25 15:41:25 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:41:25 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2082 bytes result sent to driver -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:53532 in memory (size: 46.4 KB, free: 730.1 MB) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 15 ms on localhost (1/2) -2016-05-25 15:41:25 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-25 15:41:25 INFO DAGScheduler:59 - ResultStage 14 (count at RandomForest2Spark.scala:56) finished in 0.019 s -2016-05-25 15:41:25 INFO DAGScheduler:59 - Job 10 finished: count at RandomForest2Spark.scala:56, took 0.039002 s -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:53532 in memory (size: 124.5 KB, free: 730.2 MB) -2016-05-25 15:41:25 INFO SparkContext:59 - Starting job: count at RandomForest2Spark.scala:57 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Got job 11 (count at RandomForest2Spark.scala:57) with 2 output partitions -2016-05-25 15:41:25 INFO DAGScheduler:59 - Final stage: ResultStage 15(count at RandomForest2Spark.scala:57) -2016-05-25 15:41:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 17 ms on localhost (2/2) -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 15:41:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:41:25 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[7] at randomSplit at RandomForest2Spark.scala:29), which has no missing parents -2016-05-25 15:41:25 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1553777, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 4.0 KB, free 729.1 MB) -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(2265) called with curMem=1557524, maxMem=766075207 -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:53532 in memory (size: 365.0 B, free: 730.2 MB) -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.1 MB) -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:53532 (size: 2.2 KB, free: 730.2 MB) -2016-05-25 15:41:25 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[7] at randomSplit at RandomForest2Spark.scala:29) -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:41:25 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-05-25 15:41:25 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-25 15:41:25 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:41:25 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:41:25 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 2082 bytes result sent to driver -2016-05-25 15:41:25 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 2082 bytes result sent to driver -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:53532 in memory (size: 46.4 KB, free: 730.2 MB) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 11 ms on localhost (1/2) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 13 ms on localhost (2/2) -2016-05-25 15:41:25 INFO DAGScheduler:59 - ResultStage 15 (count at RandomForest2Spark.scala:57) finished in 0.014 s -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-25 15:41:25 INFO DAGScheduler:59 - Job 11 finished: count at RandomForest2Spark.scala:57, took 0.034790 s -2016-05-25 15:41:25 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:53532 in memory (size: 124.6 KB, free: 730.4 MB) -2016-05-25 15:41:25 INFO SparkContext:59 - Starting job: count at RandomForest2Spark.scala:58 -2016-05-25 15:41:25 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 15:41:25 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Got job 12 (count at RandomForest2Spark.scala:58) with 2 output partitions -2016-05-25 15:41:25 INFO DAGScheduler:59 - Final stage: ResultStage 16(count at RandomForest2Spark.scala:58) -2016-05-25 15:41:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:41:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[8] at randomSplit at RandomForest2Spark.scala:29), which has no missing parents -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=640391, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 4.0 KB, free 730.0 MB) -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=644503, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.0 MB) -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:53532 in memory (size: 493.0 B, free: 730.4 MB) -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:53532 (size: 2.2 KB, free: 730.4 MB) -2016-05-25 15:41:25 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[8] at randomSplit at RandomForest2Spark.scala:29) -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:41:25 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-05-25 15:41:25 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-05-25 15:41:25 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:41:25 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:41:25 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2082 bytes result sent to driver -2016-05-25 15:41:25 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2082 bytes result sent to driver -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 9 ms on localhost (1/2) -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:53532 in memory (size: 46.4 KB, free: 730.4 MB) -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 14 ms on localhost (2/2) -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-25 15:41:25 INFO DAGScheduler:59 - ResultStage 16 (count at RandomForest2Spark.scala:58) finished in 0.016 s -2016-05-25 15:41:25 INFO DAGScheduler:59 - Job 12 finished: count at RandomForest2Spark.scala:58, took 0.034533 s -2016-05-25 15:41:25 INFO SparkContext:59 - Starting job: take at RandomForest2Spark.scala:63 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Got job 13 (take at RandomForest2Spark.scala:63) with 1 output partitions -2016-05-25 15:41:25 INFO DAGScheduler:59 - Final stage: ResultStage 17(take at RandomForest2Spark.scala:63) -2016-05-25 15:41:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:41:25 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[26] at map at RandomForest2Spark.scala:46), which has no missing parents -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(9120) called with curMem=373453, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 8.9 KB, free 730.2 MB) -2016-05-25 15:41:25 INFO MemoryStore:59 - ensureFreeSpace(4498) called with curMem=382573, maxMem=766075207 -2016-05-25 15:41:25 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.2 MB) -2016-05-25 15:41:25 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:53532 (size: 4.4 KB, free: 730.4 MB) -2016-05-25 15:41:25 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:41:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 17 (MapPartitionsRDD[26] at map at RandomForest2Spark.scala:46) -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 1 tasks -2016-05-25 15:41:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:41:25 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-05-25 15:41:25 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:41:25 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2401 bytes result sent to driver -2016-05-25 15:41:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 11 ms on localhost (1/1) -2016-05-25 15:41:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-25 15:41:25 INFO DAGScheduler:59 - ResultStage 17 (take at RandomForest2Spark.scala:63) finished in 0.011 s -2016-05-25 15:41:25 INFO DAGScheduler:59 - Job 13 finished: take at RandomForest2Spark.scala:63, took 0.026458 s -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 15:41:25 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 15:41:26 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 15:41:26 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 15:41:26 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 15:41:26 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 15:41:26 INFO BlockManager:59 - BlockManager stopped -2016-05-25 15:41:26 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 15:41:26 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 15:41:26 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 15:41:26 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 15:41:26 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-889a6213-7a03-42df-b5f9-d4f127825b3e -2016-05-25 15:41:26 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 15:42:22 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 15:42:23 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 15:42:23 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 15:42:23 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 15:42:23 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 15:42:24 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 15:42:24 INFO Remoting:74 - Starting remoting -2016-05-25 15:42:24 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53612] -2016-05-25 15:42:24 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53612. -2016-05-25 15:42:24 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 15:42:24 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 15:42:24 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-7f7e79d7-3da2-49be-a8a9-61e463231bf5 -2016-05-25 15:42:24 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 15:42:24 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-1e678487-1573-4e6b-9ace-5b6f29b839de\httpd-7af5f1b8-32f0-4f0c-b886-b8bf04883cb5 -2016-05-25 15:42:24 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 15:42:24 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:42:24 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53613 -2016-05-25 15:42:24 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53613. -2016-05-25 15:42:25 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 15:42:25 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:42:25 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 15:42:25 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 15:42:25 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 15:42:25 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 15:42:25 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 15:42:25 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53632. -2016-05-25 15:42:25 INFO NettyBlockTransferService:59 - Server created on 53632 -2016-05-25 15:42:25 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 15:42:25 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53632 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53632) -2016-05-25 15:42:25 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 15:42:26 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 15:42:26 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 15:42:26 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 15:42:26 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 15:42:26 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53632 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 15:42:26 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 15:42:27 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 15:42:28 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 15:42:28 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 15:42:28 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 15:42:28 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 15:42:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:42:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:42:28 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 15:42:28 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 15:42:28 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 15:42:28 INFO MemoryStore:59 - ensureFreeSpace(2148) called with curMem=175484, maxMem=766075207 -2016-05-25 15:42:28 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 15:42:28 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53632 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:42:28 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:42:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 15:42:28 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 15:42:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:42:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:42:28 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 15:42:28 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 15:42:28 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 15:42:28 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 15:42:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 15:42:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 15:42:28 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 15:42:28 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 15:42:28 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 15:42:28 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 15:42:28 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 15:42:28 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177632, maxMem=766075207 -2016-05-25 15:42:28 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 15:42:28 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:53632 (size: 83.1 KB, free: 730.5 MB) -2016-05-25 15:42:28 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262712, maxMem=766075207 -2016-05-25 15:42:28 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 15:42:28 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:53632 (size: 83.6 KB, free: 730.4 MB) -2016-05-25 15:42:28 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 15:42:28 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 15:42:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 432 ms on localhost (1/2) -2016-05-25 15:42:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 416 ms on localhost (2/2) -2016-05-25 15:42:28 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.456 s -2016-05-25 15:42:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 15:42:28 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.626917 s -2016-05-25 15:42:28 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 15:42:28 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 15:42:28 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 15:42:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:42:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:42:28 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 15:42:28 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348360, maxMem=766075207 -2016-05-25 15:42:28 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 15:42:28 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=353128, maxMem=766075207 -2016-05-25 15:42:28 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 15:42:28 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53632 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 15:42:28 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:42:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 15:42:28 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 15:42:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:42:28 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 15:42:28 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:42:28 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 15:42:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 25 ms on localhost (1/1) -2016-05-25 15:42:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 15:42:28 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.025 s -2016-05-25 15:42:28 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.043628 s -2016-05-25 15:42:28 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 15:42:28 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 15:42:28 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 15:42:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:42:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:42:28 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 15:42:28 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355675, maxMem=766075207 -2016-05-25 15:42:28 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 15:42:28 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=360011, maxMem=766075207 -2016-05-25 15:42:28 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 15:42:28 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53632 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 15:42:28 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:42:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 15:42:28 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 15:42:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:42:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:42:28 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 15:42:28 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 15:42:28 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:42:28 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:42:28 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 15:42:28 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 15:42:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 14 ms on localhost (1/2) -2016-05-25 15:42:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 14 ms on localhost (2/2) -2016-05-25 15:42:28 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.015 s -2016-05-25 15:42:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 15:42:28 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.028263 s -2016-05-25 15:42:28 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 15:42:28 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 15:42:28 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 15:42:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:42:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:42:28 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 15:42:28 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362330, maxMem=766075207 -2016-05-25 15:42:28 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 15:42:28 INFO MemoryStore:59 - ensureFreeSpace(2630) called with curMem=367394, maxMem=766075207 -2016-05-25 15:42:28 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 15:42:28 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53632 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:42:28 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:42:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 15:42:28 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 15:42:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 15:42:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 15:42:28 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 15:42:28 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 15:42:28 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:42:28 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:42:28 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 67796 bytes result sent to driver -2016-05-25 15:42:28 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 60381 bytes result sent to driver -2016-05-25 15:42:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 29 ms on localhost (1/2) -2016-05-25 15:42:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 33 ms on localhost (2/2) -2016-05-25 15:42:28 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.033 s -2016-05-25 15:42:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 15:42:28 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.052982 s -2016-05-25 15:42:30 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53632 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:42:30 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 15:42:30 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53632 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 15:42:30 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 15:42:30 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53632 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 15:42:30 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 15:42:30 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53632 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-25 15:42:30 INFO MemoryStore:59 - ensureFreeSpace(512) called with curMem=342308, maxMem=766075207 -2016-05-25 15:42:30 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 512.0 B, free 730.3 MB) -2016-05-25 15:42:30 INFO MemoryStore:59 - ensureFreeSpace(615) called with curMem=342820, maxMem=766075207 -2016-05-25 15:42:30 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 615.0 B, free 730.3 MB) -2016-05-25 15:42:30 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53632 (size: 615.0 B, free: 730.4 MB) -2016-05-25 15:42:30 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 15:42:30 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:42:30 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:42:30 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:42:30 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:42:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 15:42:30 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 15:42:30 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:42:30 INFO MemoryStore:59 - ensureFreeSpace(486728) called with curMem=343435, maxMem=766075207 -2016-05-25 15:42:30 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 475.3 KB, free 729.8 MB) -2016-05-25 15:42:30 INFO MemoryStore:59 - ensureFreeSpace(119676) called with curMem=830163, maxMem=766075207 -2016-05-25 15:42:30 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 116.9 KB, free 729.7 MB) -2016-05-25 15:42:30 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53632 (size: 116.9 KB, free: 730.3 MB) -2016-05-25 15:42:30 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:42:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:42:30 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 15:42:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:42:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:42:30 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 15:42:30 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 15:42:30 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 15:42:30 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 15:42:30 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:42:30 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:42:30 INFO MemoryStore:59 - ensureFreeSpace(112184) called with curMem=949839, maxMem=766075207 -2016-05-25 15:42:30 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 109.6 KB, free 729.6 MB) -2016-05-25 15:42:30 INFO MemoryStore:59 - ensureFreeSpace(103552) called with curMem=1062023, maxMem=766075207 -2016-05-25 15:42:30 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 101.1 KB, free 729.5 MB) -2016-05-25 15:42:30 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:53632 (size: 109.6 KB, free: 730.2 MB) -2016-05-25 15:42:30 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:53632 (size: 101.1 KB, free: 730.1 MB) -2016-05-25 15:42:30 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 15:42:30 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 15:42:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 311 ms on localhost (1/2) -2016-05-25 15:42:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 310 ms on localhost (2/2) -2016-05-25 15:42:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 15:42:30 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.313 s -2016-05-25 15:42:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:42:30 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:42:30 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 15:42:30 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:42:30 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 15:42:30 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:42:30 INFO MemoryStore:59 - ensureFreeSpace(211224) called with curMem=1165575, maxMem=766075207 -2016-05-25 15:42:30 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 206.3 KB, free 729.3 MB) -2016-05-25 15:42:30 INFO MemoryStore:59 - ensureFreeSpace(45045) called with curMem=1376799, maxMem=766075207 -2016-05-25 15:42:30 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 44.0 KB, free 729.2 MB) -2016-05-25 15:42:30 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53632 (size: 44.0 KB, free: 730.0 MB) -2016-05-25 15:42:30 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:42:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 15:42:30 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 15:42:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:42:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:42:30 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 15:42:30 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 15:42:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:42:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:42:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 9 ms -2016-05-25 15:42:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 9 ms -2016-05-25 15:42:31 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2774 bytes result sent to driver -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 262 ms on localhost (1/2) -2016-05-25 15:42:31 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2928 bytes result sent to driver -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 275 ms on localhost (2/2) -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 15:42:31 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.276 s -2016-05-25 15:42:31 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.646389 s -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(360) called with curMem=1421844, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 360.0 B, free 729.2 MB) -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(491) called with curMem=1422204, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 491.0 B, free 729.2 MB) -2016-05-25 15:42:31 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53632 (size: 491.0 B, free: 730.0 MB) -2016-05-25 15:42:31 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 15:42:31 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:42:31 INFO DAGScheduler:59 - Registering RDD 17 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:42:31 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:42:31 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:42:31 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 15:42:31 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 15:42:31 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(487312) called with curMem=1422695, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 475.9 KB, free 728.8 MB) -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(119901) called with curMem=1910007, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 117.1 KB, free 728.7 MB) -2016-05-25 15:42:31 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53632 (size: 117.1 KB, free: 729.9 MB) -2016-05-25 15:42:31 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:42:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 15:42:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:42:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:42:31 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 15:42:31 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 15:42:31 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-25 15:42:31 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 89 ms on localhost (1/2) -2016-05-25 15:42:31 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-25 15:42:31 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 108 ms on localhost (2/2) -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 15:42:31 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.110 s -2016-05-25 15:42:31 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:42:31 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:42:31 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 15:42:31 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:42:31 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 15:42:31 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(211168) called with curMem=2029908, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 206.2 KB, free 728.4 MB) -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(44975) called with curMem=2241076, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 43.9 KB, free 728.4 MB) -2016-05-25 15:42:31 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53632 (size: 43.9 KB, free: 729.9 MB) -2016-05-25 15:42:31 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:42:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645) -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 15:42:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:42:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:42:31 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 15:42:31 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 15:42:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:42:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:42:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 15:42:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 15:42:31 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2774 bytes result sent to driver -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 33 ms on localhost (1/2) -2016-05-25 15:42:31 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 36 ms on localhost (2/2) -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 15:42:31 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.037 s -2016-05-25 15:42:31 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.196623 s -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(208) called with curMem=2286051, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 208.0 B, free 728.4 MB) -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(364) called with curMem=2286259, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 364.0 B, free 728.4 MB) -2016-05-25 15:42:31 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53632 (size: 364.0 B, free: 729.9 MB) -2016-05-25 15:42:31 INFO SparkContext:59 - Created broadcast 11 from broadcast at DecisionTree.scala:604 -2016-05-25 15:42:31 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:42:31 INFO DAGScheduler:59 - Registering RDD 20 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:42:31 INFO DAGScheduler:59 - Got job 6 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:42:31 INFO DAGScheduler:59 - Final stage: ResultStage 9(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:42:31 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 8) -2016-05-25 15:42:31 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 8) -2016-05-25 15:42:31 INFO DAGScheduler:59 - Submitting ShuffleMapStage 8 (MapPartitionsRDD[20] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(487512) called with curMem=2286623, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 476.1 KB, free 727.9 MB) -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(119885) called with curMem=2774135, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 117.1 KB, free 727.8 MB) -2016-05-25 15:42:31 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:53632 (size: 117.1 KB, free: 729.8 MB) -2016-05-25 15:42:31 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:42:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 8 (MapPartitionsRDD[20] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 15:42:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:42:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:42:31 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 15:42:31 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 15:42:31 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-25 15:42:31 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-25 15:42:31 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2254 bytes result sent to driver -2016-05-25 15:42:31 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2254 bytes result sent to driver -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 28 ms on localhost (1/2) -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 31 ms on localhost (2/2) -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 15:42:31 INFO DAGScheduler:59 - ShuffleMapStage 8 (mapPartitions at DecisionTree.scala:625) finished in 0.033 s -2016-05-25 15:42:31 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:42:31 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:42:31 INFO DAGScheduler:59 - waiting: Set(ResultStage 9) -2016-05-25 15:42:31 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:42:31 INFO DAGScheduler:59 - Missing parents for ResultStage 9: List() -2016-05-25 15:42:31 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[22] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(211104) called with curMem=2894020, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 206.2 KB, free 727.6 MB) -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(44959) called with curMem=3105124, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 43.9 KB, free 727.6 MB) -2016-05-25 15:42:31 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:53632 (size: 43.9 KB, free: 729.7 MB) -2016-05-25 15:42:31 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:42:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[22] at map at DecisionTree.scala:645) -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 15:42:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:42:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:42:31 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-25 15:42:31 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 15:42:31 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 15:42:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 15:42:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:42:31 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 1161 bytes result sent to driver -2016-05-25 15:42:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 12 ms on localhost (1/2) -2016-05-25 15:42:31 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2774 bytes result sent to driver -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 35 ms on localhost (2/2) -2016-05-25 15:42:31 INFO DAGScheduler:59 - ResultStage 9 (collectAsMap at DecisionTree.scala:654) finished in 0.035 s -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 15:42:31 INFO DAGScheduler:59 - Job 6 finished: collectAsMap at DecisionTree.scala:654, took 0.123756 s -2016-05-25 15:42:31 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 15:42:31 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 15:42:31 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 15:42:31 INFO RandomForest:59 - init: 1.550064411 - total: 2.840640887 - findSplitsBins: 1.324543535 - findBestSplits: 1.269241701 - chooseSplits: 1.263528392 -2016-05-25 15:42:31 INFO SparkContext:59 - Starting job: count at RandomForest2Spark.scala:50 -2016-05-25 15:42:31 INFO DAGScheduler:59 - Got job 7 (count at RandomForest2Spark.scala:50) with 2 output partitions -2016-05-25 15:42:31 INFO DAGScheduler:59 - Final stage: ResultStage 10(count at RandomForest2Spark.scala:50) -2016-05-25 15:42:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:42:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:42:31 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[24] at filter at RandomForest2Spark.scala:50), which has no missing parents -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(8408) called with curMem=2934347, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 8.2 KB, free 727.8 MB) -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(4234) called with curMem=2942755, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 4.1 KB, free 727.8 MB) -2016-05-25 15:42:31 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:53632 (size: 4.1 KB, free: 729.9 MB) -2016-05-25 15:42:31 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:42:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[24] at filter at RandomForest2Spark.scala:50) -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-25 15:42:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:42:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:42:31 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-05-25 15:42:31 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-05-25 15:42:31 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:42:31 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:42:31 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2082 bytes result sent to driver -2016-05-25 15:42:31 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2082 bytes result sent to driver -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 16 ms on localhost (1/2) -2016-05-25 15:42:31 INFO DAGScheduler:59 - ResultStage 10 (count at RandomForest2Spark.scala:50) finished in 0.017 s -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 16 ms on localhost (2/2) -2016-05-25 15:42:31 INFO DAGScheduler:59 - Job 7 finished: count at RandomForest2Spark.scala:50, took 0.032172 s -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 15:42:31 INFO SparkContext:59 - Starting job: count at RandomForest2Spark.scala:50 -2016-05-25 15:42:31 INFO DAGScheduler:59 - Got job 8 (count at RandomForest2Spark.scala:50) with 2 output partitions -2016-05-25 15:42:31 INFO DAGScheduler:59 - Final stage: ResultStage 11(count at RandomForest2Spark.scala:50) -2016-05-25 15:42:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:42:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:42:31 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[8] at randomSplit at RandomForest2Spark.scala:29), which has no missing parents -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2946989, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 4.0 KB, free 727.8 MB) -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=2951101, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.2 KB, free 727.8 MB) -2016-05-25 15:42:31 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:53632 (size: 2.2 KB, free: 729.9 MB) -2016-05-25 15:42:31 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:42:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[8] at randomSplit at RandomForest2Spark.scala:29) -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-25 15:42:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:42:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:42:31 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-25 15:42:31 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-25 15:42:31 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:42:31 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2082 bytes result sent to driver -2016-05-25 15:42:31 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:42:31 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2082 bytes result sent to driver -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 8 ms on localhost (1/2) -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 9 ms on localhost (2/2) -2016-05-25 15:42:31 INFO DAGScheduler:59 - ResultStage 11 (count at RandomForest2Spark.scala:50) finished in 0.011 s -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 15:42:31 INFO DAGScheduler:59 - Job 8 finished: count at RandomForest2Spark.scala:50, took 0.022822 s -2016-05-25 15:42:31 INFO SparkContext:59 - Starting job: count at RandomForest2Spark.scala:56 -2016-05-25 15:42:31 INFO DAGScheduler:59 - Got job 9 (count at RandomForest2Spark.scala:56) with 2 output partitions -2016-05-25 15:42:31 INFO DAGScheduler:59 - Final stage: ResultStage 12(count at RandomForest2Spark.scala:56) -2016-05-25 15:42:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:42:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:42:31 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(3816) called with curMem=2953369, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 3.7 KB, free 727.8 MB) -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(2142) called with curMem=2957185, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.1 KB, free 727.8 MB) -2016-05-25 15:42:31 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:53632 (size: 2.1 KB, free: 729.9 MB) -2016-05-25 15:42:31 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:42:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-25 15:42:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:42:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:42:31 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-05-25 15:42:31 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-05-25 15:42:31 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:42:31 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:42:31 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2082 bytes result sent to driver -2016-05-25 15:42:31 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2082 bytes result sent to driver -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 8 ms on localhost (1/2) -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 8 ms on localhost (2/2) -2016-05-25 15:42:31 INFO DAGScheduler:59 - ResultStage 12 (count at RandomForest2Spark.scala:56) finished in 0.009 s -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 15:42:31 INFO DAGScheduler:59 - Job 9 finished: count at RandomForest2Spark.scala:56, took 0.024035 s -2016-05-25 15:42:31 INFO SparkContext:59 - Starting job: count at RandomForest2Spark.scala:57 -2016-05-25 15:42:31 INFO DAGScheduler:59 - Got job 10 (count at RandomForest2Spark.scala:57) with 2 output partitions -2016-05-25 15:42:31 INFO DAGScheduler:59 - Final stage: ResultStage 13(count at RandomForest2Spark.scala:57) -2016-05-25 15:42:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:42:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:42:31 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[7] at randomSplit at RandomForest2Spark.scala:29), which has no missing parents -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2959327, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 4.0 KB, free 727.8 MB) -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(2265) called with curMem=2963439, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.2 KB, free 727.8 MB) -2016-05-25 15:42:31 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:53632 (size: 2.2 KB, free: 729.9 MB) -2016-05-25 15:42:31 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:42:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[7] at randomSplit at RandomForest2Spark.scala:29) -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-25 15:42:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:42:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:42:31 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-05-25 15:42:31 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-05-25 15:42:31 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:42:31 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2082 bytes result sent to driver -2016-05-25 15:42:31 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:42:31 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 2082 bytes result sent to driver -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 8 ms on localhost (1/2) -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 9 ms on localhost (2/2) -2016-05-25 15:42:31 INFO DAGScheduler:59 - ResultStage 13 (count at RandomForest2Spark.scala:57) finished in 0.010 s -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 15:42:31 INFO DAGScheduler:59 - Job 10 finished: count at RandomForest2Spark.scala:57, took 0.022541 s -2016-05-25 15:42:31 INFO SparkContext:59 - Starting job: count at RandomForest2Spark.scala:58 -2016-05-25 15:42:31 INFO DAGScheduler:59 - Got job 11 (count at RandomForest2Spark.scala:58) with 2 output partitions -2016-05-25 15:42:31 INFO DAGScheduler:59 - Final stage: ResultStage 14(count at RandomForest2Spark.scala:58) -2016-05-25 15:42:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:42:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:42:31 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[8] at randomSplit at RandomForest2Spark.scala:29), which has no missing parents -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2965704, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 4.0 KB, free 727.8 MB) -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=2969816, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.2 KB, free 727.8 MB) -2016-05-25 15:42:31 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:53632 (size: 2.2 KB, free: 729.9 MB) -2016-05-25 15:42:31 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:42:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[8] at randomSplit at RandomForest2Spark.scala:29) -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-25 15:42:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:42:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:42:31 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-05-25 15:42:31 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-05-25 15:42:31 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:42:31 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:42:31 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2082 bytes result sent to driver -2016-05-25 15:42:31 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2082 bytes result sent to driver -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 7 ms on localhost (1/2) -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 9 ms on localhost (2/2) -2016-05-25 15:42:31 INFO DAGScheduler:59 - ResultStage 14 (count at RandomForest2Spark.scala:58) finished in 0.009 s -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 15:42:31 INFO DAGScheduler:59 - Job 11 finished: count at RandomForest2Spark.scala:58, took 0.023624 s -2016-05-25 15:42:31 INFO SparkContext:59 - Starting job: take at RandomForest2Spark.scala:63 -2016-05-25 15:42:31 INFO DAGScheduler:59 - Got job 12 (take at RandomForest2Spark.scala:63) with 1 output partitions -2016-05-25 15:42:31 INFO DAGScheduler:59 - Final stage: ResultStage 15(take at RandomForest2Spark.scala:63) -2016-05-25 15:42:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:42:31 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:42:31 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[23] at map at RandomForest2Spark.scala:46), which has no missing parents -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(8464) called with curMem=2972084, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 8.3 KB, free 727.7 MB) -2016-05-25 15:42:31 INFO MemoryStore:59 - ensureFreeSpace(4277) called with curMem=2980548, maxMem=766075207 -2016-05-25 15:42:31 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 4.2 KB, free 727.7 MB) -2016-05-25 15:42:31 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:53632 (size: 4.2 KB, free: 729.9 MB) -2016-05-25 15:42:31 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:42:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 15 (MapPartitionsRDD[23] at map at RandomForest2Spark.scala:46) -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 1 tasks -2016-05-25 15:42:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:42:31 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-05-25 15:42:31 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:42:31 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 2401 bytes result sent to driver -2016-05-25 15:42:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 12 ms on localhost (1/1) -2016-05-25 15:42:31 INFO DAGScheduler:59 - ResultStage 15 (take at RandomForest2Spark.scala:63) finished in 0.012 s -2016-05-25 15:42:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-25 15:42:31 INFO DAGScheduler:59 - Job 12 finished: take at RandomForest2Spark.scala:63, took 0.032385 s -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 15:42:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 15:42:31 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 15:42:31 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 15:42:31 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 15:42:32 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 15:42:32 INFO BlockManager:59 - BlockManager stopped -2016-05-25 15:42:32 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 15:42:32 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 15:42:32 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 15:42:32 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 15:42:32 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-1e678487-1573-4e6b-9ace-5b6f29b839de -2016-05-25 15:47:13 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 15:47:14 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 15:47:14 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 15:47:14 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 15:47:14 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 15:47:15 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 15:47:15 INFO Remoting:74 - Starting remoting -2016-05-25 15:47:15 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:53775] -2016-05-25 15:47:15 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53775. -2016-05-25 15:47:15 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 15:47:15 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 15:47:15 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-77dd71a4-4e9a-404e-9f82-3a783078805a -2016-05-25 15:47:15 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 15:47:15 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-49f872e6-7e4b-4dbc-b05f-e8ed99897309\httpd-f4eb503e-14d8-4583-b7e8-735bbb61f492 -2016-05-25 15:47:15 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 15:47:15 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:47:15 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53776 -2016-05-25 15:47:15 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53776. -2016-05-25 15:47:15 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 15:47:15 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:47:16 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 15:47:16 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 15:47:16 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 15:47:16 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 15:47:16 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 15:47:16 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53795. -2016-05-25 15:47:16 INFO NettyBlockTransferService:59 - Server created on 53795 -2016-05-25 15:47:16 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 15:47:16 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53795 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53795) -2016-05-25 15:47:16 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 15:47:17 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 15:47:17 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 15:47:17 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 15:47:17 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 15:47:17 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53795 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 15:47:17 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 15:47:18 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 15:47:18 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 15:47:19 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 15:47:19 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 15:47:19 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 15:47:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:47:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:47:19 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 15:47:19 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 15:47:19 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 15:47:19 INFO MemoryStore:59 - ensureFreeSpace(2147) called with curMem=175484, maxMem=766075207 -2016-05-25 15:47:19 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 15:47:19 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53795 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 15:47:19 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:47:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 15:47:19 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 15:47:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:47:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:47:19 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 15:47:19 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 15:47:19 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 15:47:19 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 15:47:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 15:47:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 15:47:19 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 15:47:19 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 15:47:19 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 15:47:19 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 15:47:19 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 15:47:19 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177631, maxMem=766075207 -2016-05-25 15:47:19 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 15:47:19 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:53795 (size: 83.1 KB, free: 730.5 MB) -2016-05-25 15:47:19 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262711, maxMem=766075207 -2016-05-25 15:47:19 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 15:47:19 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:53795 (size: 83.6 KB, free: 730.4 MB) -2016-05-25 15:47:19 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 15:47:19 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 15:47:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 413 ms on localhost (1/2) -2016-05-25 15:47:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 441 ms on localhost (2/2) -2016-05-25 15:47:19 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.468 s -2016-05-25 15:47:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 15:47:19 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.658195 s -2016-05-25 15:47:19 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 15:47:19 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 15:47:19 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 15:47:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:47:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:47:19 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 15:47:19 INFO MemoryStore:59 - ensureFreeSpace(4768) called with curMem=348359, maxMem=766075207 -2016-05-25 15:47:19 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 15:47:19 INFO MemoryStore:59 - ensureFreeSpace(2547) called with curMem=353127, maxMem=766075207 -2016-05-25 15:47:19 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 15:47:19 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53795 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 15:47:19 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:47:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 15:47:19 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 15:47:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:47:19 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 15:47:19 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:47:19 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2031 bytes result sent to driver -2016-05-25 15:47:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 35 ms on localhost (1/1) -2016-05-25 15:47:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 15:47:19 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.035 s -2016-05-25 15:47:19 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.059105 s -2016-05-25 15:47:19 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 15:47:19 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 15:47:19 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 15:47:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:47:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:47:19 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 15:47:19 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=355674, maxMem=766075207 -2016-05-25 15:47:19 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 15:47:19 INFO MemoryStore:59 - ensureFreeSpace(2319) called with curMem=360010, maxMem=766075207 -2016-05-25 15:47:19 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 15:47:19 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53795 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 15:47:19 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:47:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 15:47:19 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 15:47:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:47:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:47:19 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 15:47:19 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 15:47:19 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:47:19 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:47:19 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 15:47:19 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-05-25 15:47:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 16 ms on localhost (1/2) -2016-05-25 15:47:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 16 ms on localhost (2/2) -2016-05-25 15:47:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 15:47:19 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.018 s -2016-05-25 15:47:19 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.034164 s -2016-05-25 15:47:19 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 15:47:19 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 15:47:19 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 15:47:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:47:19 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:47:19 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 15:47:19 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=362329, maxMem=766075207 -2016-05-25 15:47:19 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.2 MB) -2016-05-25 15:47:19 INFO MemoryStore:59 - ensureFreeSpace(2631) called with curMem=367393, maxMem=766075207 -2016-05-25 15:47:19 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-05-25 15:47:19 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53795 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:47:19 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:47:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 15:47:19 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 15:47:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 15:47:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 15:47:19 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 15:47:19 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 15:47:19 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:47:19 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:47:19 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 59024 bytes result sent to driver -2016-05-25 15:47:19 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 64265 bytes result sent to driver -2016-05-25 15:47:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 29 ms on localhost (1/2) -2016-05-25 15:47:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 27 ms on localhost (2/2) -2016-05-25 15:47:19 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.031 s -2016-05-25 15:47:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 15:47:19 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.048344 s -2016-05-25 15:47:21 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53795 in memory (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:47:21 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 15:47:21 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:53795 in memory (size: 2.3 KB, free: 730.4 MB) -2016-05-25 15:47:21 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 15:47:21 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53795 in memory (size: 2.5 KB, free: 730.4 MB) -2016-05-25 15:47:21 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 15:47:21 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53795 in memory (size: 2.1 KB, free: 730.4 MB) -2016-05-25 15:47:21 INFO MemoryStore:59 - ensureFreeSpace(2960) called with curMem=342308, maxMem=766075207 -2016-05-25 15:47:21 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 2.9 KB, free 730.3 MB) -2016-05-25 15:47:21 INFO MemoryStore:59 - ensureFreeSpace(2713) called with curMem=345268, maxMem=766075207 -2016-05-25 15:47:21 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.3 MB) -2016-05-25 15:47:21 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53795 (size: 2.6 KB, free: 730.4 MB) -2016-05-25 15:47:21 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 15:47:21 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 15:47:21 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 15:47:21 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 15:47:21 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 15:47:21 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 15:47:21 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 15:47:21 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 15:47:21 INFO MemoryStore:59 - ensureFreeSpace(487360) called with curMem=347981, maxMem=766075207 -2016-05-25 15:47:21 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 475.9 KB, free 729.8 MB) -2016-05-25 15:47:21 INFO MemoryStore:59 - ensureFreeSpace(121260) called with curMem=835341, maxMem=766075207 -2016-05-25 15:47:21 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 118.4 KB, free 729.7 MB) -2016-05-25 15:47:21 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53795 (size: 118.4 KB, free: 730.3 MB) -2016-05-25 15:47:21 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:47:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 15:47:21 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 15:47:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:47:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 15:47:21 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 15:47:21 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 15:47:21 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 15:47:21 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:47:21 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 15:47:21 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:47:21 INFO MemoryStore:59 - ensureFreeSpace(92048) called with curMem=956601, maxMem=766075207 -2016-05-25 15:47:21 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 89.9 KB, free 729.6 MB) -2016-05-25 15:47:21 INFO MemoryStore:59 - ensureFreeSpace(103552) called with curMem=1048649, maxMem=766075207 -2016-05-25 15:47:21 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 101.1 KB, free 729.5 MB) -2016-05-25 15:47:21 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:53795 (size: 89.9 KB, free: 730.2 MB) -2016-05-25 15:47:21 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:53795 (size: 101.1 KB, free: 730.1 MB) -2016-05-25 15:47:21 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 15:47:21 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 15:47:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 337 ms on localhost (1/2) -2016-05-25 15:47:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 336 ms on localhost (2/2) -2016-05-25 15:47:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 15:47:21 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.340 s -2016-05-25 15:47:21 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 15:47:21 INFO DAGScheduler:59 - running: Set() -2016-05-25 15:47:21 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 15:47:21 INFO DAGScheduler:59 - failed: Set() -2016-05-25 15:47:21 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 15:47:21 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 15:47:21 INFO MemoryStore:59 - ensureFreeSpace(210328) called with curMem=1152201, maxMem=766075207 -2016-05-25 15:47:21 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 205.4 KB, free 729.3 MB) -2016-05-25 15:47:21 INFO MemoryStore:59 - ensureFreeSpace(44848) called with curMem=1362529, maxMem=766075207 -2016-05-25 15:47:21 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 43.8 KB, free 729.2 MB) -2016-05-25 15:47:21 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53795 (size: 43.8 KB, free: 730.1 MB) -2016-05-25 15:47:21 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:47:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 15:47:21 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 15:47:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:47:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 15:47:21 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 15:47:21 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 15:47:22 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:47:22 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 15:47:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 15:47:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 15:47:22 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2774 bytes result sent to driver -2016-05-25 15:47:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 290 ms on localhost (1/2) -2016-05-25 15:47:22 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2928 bytes result sent to driver -2016-05-25 15:47:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 316 ms on localhost (2/2) -2016-05-25 15:47:22 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.317 s -2016-05-25 15:47:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 15:47:22 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.718274 s -2016-05-25 15:47:22 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 15:47:22 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 15:47:22 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 15:47:22 INFO RandomForest:59 - init: 1.597480325 - total: 2.529950475 - findSplitsBins: 1.326648322 - findBestSplits: 0.912190222 - chooseSplits: 0.906156532 -2016-05-25 15:47:22 INFO SparkContext:59 - Starting job: mean at RandomForest3VarianceRegression.scala:48 -2016-05-25 15:47:22 INFO DAGScheduler:59 - Got job 5 (mean at RandomForest3VarianceRegression.scala:48) with 2 output partitions -2016-05-25 15:47:22 INFO DAGScheduler:59 - Final stage: ResultStage 6(mean at RandomForest3VarianceRegression.scala:48) -2016-05-25 15:47:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:47:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:47:22 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[19] at mean at RandomForest3VarianceRegression.scala:48), which has no missing parents -2016-05-25 15:47:22 INFO MemoryStore:59 - ensureFreeSpace(8360) called with curMem=1211777, maxMem=766075207 -2016-05-25 15:47:22 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 8.2 KB, free 729.4 MB) -2016-05-25 15:47:22 INFO MemoryStore:59 - ensureFreeSpace(4265) called with curMem=1220137, maxMem=766075207 -2016-05-25 15:47:22 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 4.2 KB, free 729.4 MB) -2016-05-25 15:47:22 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:53795 (size: 4.2 KB, free: 730.2 MB) -2016-05-25 15:47:22 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:47:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[19] at mean at RandomForest3VarianceRegression.scala:48) -2016-05-25 15:47:22 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 15:47:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:47:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:47:22 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 15:47:22 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 15:47:22 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:47:22 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:47:22 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 15:47:22 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 15:47:22 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2209 bytes result sent to driver -2016-05-25 15:47:22 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2209 bytes result sent to driver -2016-05-25 15:47:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 156 ms on localhost (1/2) -2016-05-25 15:47:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 159 ms on localhost (2/2) -2016-05-25 15:47:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 15:47:22 INFO DAGScheduler:59 - ResultStage 6 (mean at RandomForest3VarianceRegression.scala:48) finished in 0.164 s -2016-05-25 15:47:22 INFO DAGScheduler:59 - Job 5 finished: mean at RandomForest3VarianceRegression.scala:48, took 0.180780 s -2016-05-25 15:47:22 INFO SparkContext:59 - Starting job: count at RandomForest3VarianceRegression.scala:52 -2016-05-25 15:47:22 INFO DAGScheduler:59 - Got job 6 (count at RandomForest3VarianceRegression.scala:52) with 2 output partitions -2016-05-25 15:47:22 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at RandomForest3VarianceRegression.scala:52) -2016-05-25 15:47:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:47:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:47:22 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 15:47:22 INFO MemoryStore:59 - ensureFreeSpace(3816) called with curMem=1224402, maxMem=766075207 -2016-05-25 15:47:22 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 3.7 KB, free 729.4 MB) -2016-05-25 15:47:22 INFO MemoryStore:59 - ensureFreeSpace(2143) called with curMem=1228218, maxMem=766075207 -2016-05-25 15:47:22 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.4 MB) -2016-05-25 15:47:22 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:53795 (size: 2.1 KB, free: 730.2 MB) -2016-05-25 15:47:22 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:47:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 15:47:22 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 15:47:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:47:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:47:22 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 15:47:22 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 15:47:22 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:47:22 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2082 bytes result sent to driver -2016-05-25 15:47:22 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:47:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 10 ms on localhost (1/2) -2016-05-25 15:47:22 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2082 bytes result sent to driver -2016-05-25 15:47:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 11 ms on localhost (2/2) -2016-05-25 15:47:22 INFO DAGScheduler:59 - ResultStage 7 (count at RandomForest3VarianceRegression.scala:52) finished in 0.013 s -2016-05-25 15:47:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 15:47:22 INFO DAGScheduler:59 - Job 6 finished: count at RandomForest3VarianceRegression.scala:52, took 0.026170 s -2016-05-25 15:47:22 INFO SparkContext:59 - Starting job: count at RandomForest3VarianceRegression.scala:53 -2016-05-25 15:47:22 INFO DAGScheduler:59 - Got job 7 (count at RandomForest3VarianceRegression.scala:53) with 2 output partitions -2016-05-25 15:47:22 INFO DAGScheduler:59 - Final stage: ResultStage 8(count at RandomForest3VarianceRegression.scala:53) -2016-05-25 15:47:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:47:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:47:22 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[7] at randomSplit at RandomForest3VarianceRegression.scala:27), which has no missing parents -2016-05-25 15:47:22 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1230361, maxMem=766075207 -2016-05-25 15:47:22 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-05-25 15:47:22 INFO MemoryStore:59 - ensureFreeSpace(2265) called with curMem=1234473, maxMem=766075207 -2016-05-25 15:47:22 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.4 MB) -2016-05-25 15:47:22 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:53795 (size: 2.2 KB, free: 730.2 MB) -2016-05-25 15:47:22 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:47:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[7] at randomSplit at RandomForest3VarianceRegression.scala:27) -2016-05-25 15:47:22 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 15:47:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:47:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:47:22 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-05-25 15:47:22 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 15:47:22 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:47:22 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:47:22 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 15:47:22 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2082 bytes result sent to driver -2016-05-25 15:47:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 8 ms on localhost (1/2) -2016-05-25 15:47:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 9 ms on localhost (2/2) -2016-05-25 15:47:22 INFO DAGScheduler:59 - ResultStage 8 (count at RandomForest3VarianceRegression.scala:53) finished in 0.009 s -2016-05-25 15:47:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 15:47:22 INFO DAGScheduler:59 - Job 7 finished: count at RandomForest3VarianceRegression.scala:53, took 0.021872 s -2016-05-25 15:47:22 INFO SparkContext:59 - Starting job: count at RandomForest3VarianceRegression.scala:54 -2016-05-25 15:47:22 INFO DAGScheduler:59 - Got job 8 (count at RandomForest3VarianceRegression.scala:54) with 2 output partitions -2016-05-25 15:47:22 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at RandomForest3VarianceRegression.scala:54) -2016-05-25 15:47:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 15:47:22 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 15:47:22 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[8] at randomSplit at RandomForest3VarianceRegression.scala:27), which has no missing parents -2016-05-25 15:47:22 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1236738, maxMem=766075207 -2016-05-25 15:47:22 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-05-25 15:47:22 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=1240850, maxMem=766075207 -2016-05-25 15:47:22 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.4 MB) -2016-05-25 15:47:22 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:53795 (size: 2.2 KB, free: 730.2 MB) -2016-05-25 15:47:22 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 15:47:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[8] at randomSplit at RandomForest3VarianceRegression.scala:27) -2016-05-25 15:47:22 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 15:47:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:47:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 15:47:22 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-05-25 15:47:22 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-05-25 15:47:22 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 15:47:22 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 15:47:22 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2082 bytes result sent to driver -2016-05-25 15:47:22 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2082 bytes result sent to driver -2016-05-25 15:47:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 9 ms on localhost (1/2) -2016-05-25 15:47:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 11 ms on localhost (2/2) -2016-05-25 15:47:22 INFO DAGScheduler:59 - ResultStage 9 (count at RandomForest3VarianceRegression.scala:54) finished in 0.011 s -2016-05-25 15:47:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 15:47:22 INFO DAGScheduler:59 - Job 8 finished: count at RandomForest3VarianceRegression.scala:54, took 0.026133 s -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 15:47:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 15:47:22 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 15:47:22 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 15:47:22 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 15:47:22 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 15:47:22 INFO BlockManager:59 - BlockManager stopped -2016-05-25 15:47:22 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 15:47:22 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 15:47:22 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 15:47:22 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 15:47:22 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-49f872e6-7e4b-4dbc-b05f-e8ed99897309 -2016-05-25 15:47:22 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 15:59:54 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 15:59:55 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 15:59:55 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 15:59:55 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 15:59:55 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 15:59:56 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 15:59:56 INFO Remoting:74 - Starting remoting -2016-05-25 15:59:56 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:54026] -2016-05-25 15:59:56 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54026. -2016-05-25 15:59:56 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 15:59:56 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 15:59:56 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-cfbb565c-8b62-4231-b22f-33201d34419b -2016-05-25 15:59:56 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 15:59:56 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-4d15702a-3b73-44f1-a8cb-61be9d2e20e0\httpd-12a0da2e-d4f7-455f-9ba6-4bfb35f50d8e -2016-05-25 15:59:56 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 15:59:56 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:59:56 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54027 -2016-05-25 15:59:56 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54027. -2016-05-25 15:59:57 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 15:59:57 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 15:59:57 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 15:59:57 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 15:59:57 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 15:59:57 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 15:59:57 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 15:59:57 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54046. -2016-05-25 15:59:57 INFO NettyBlockTransferService:59 - Server created on 54046 -2016-05-25 15:59:57 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 15:59:57 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54046 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54046) -2016-05-25 15:59:57 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 15:59:58 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 15:59:58 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 15:59:58 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 15:59:58 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 15:59:58 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54046 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 15:59:58 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 15:59:59 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 16:00:00 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 16:00:00 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 16:00:00 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 16:00:00 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 16:00:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:00:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:00:00 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 16:00:00 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 16:00:00 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 16:00:00 INFO MemoryStore:59 - ensureFreeSpace(2147) called with curMem=175484, maxMem=766075207 -2016-05-25 16:00:00 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 16:00:00 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54046 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 16:00:00 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:00:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 16:00:00 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 16:00:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:00:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:00:00 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 16:00:00 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 16:00:00 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 16:00:00 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 16:00:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 16:00:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 16:00:00 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 16:00:00 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 16:00:00 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 16:00:00 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 16:00:00 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 16:00:01 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177631, maxMem=766075207 -2016-05-25 16:00:01 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 16:00:01 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:54046 (size: 83.6 KB, free: 730.5 MB) -2016-05-25 16:00:01 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263279, maxMem=766075207 -2016-05-25 16:00:01 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 16:00:01 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:54046 (size: 83.1 KB, free: 730.4 MB) -2016-05-25 16:00:01 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 16:00:01 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 16:00:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 459 ms on localhost (1/2) -2016-05-25 16:00:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 484 ms on localhost (2/2) -2016-05-25 16:00:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 16:00:01 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.505 s -2016-05-25 16:00:01 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.663164 s -2016-05-25 16:00:01 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 16:00:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 16:00:01 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 16:00:01 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 16:00:01 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 16:00:01 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 16:00:01 INFO BlockManager:59 - BlockManager stopped -2016-05-25 16:00:01 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 16:00:01 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 16:00:01 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 16:00:01 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-4d15702a-3b73-44f1-a8cb-61be9d2e20e0 -2016-05-25 16:00:01 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 16:09:31 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 16:09:32 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 16:09:32 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 16:09:32 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 16:09:32 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 16:09:33 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 16:09:33 INFO Remoting:74 - Starting remoting -2016-05-25 16:09:33 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:54100] -2016-05-25 16:09:33 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54100. -2016-05-25 16:09:33 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 16:09:33 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 16:09:33 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-d368e58e-02b8-461c-848d-20703539fd6f -2016-05-25 16:09:33 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 16:09:33 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-913aff90-9604-490c-b343-94625169801b\httpd-3085f650-e687-4806-a0e5-622881bfb98e -2016-05-25 16:09:33 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 16:09:34 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 16:09:34 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54101 -2016-05-25 16:09:34 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54101. -2016-05-25 16:09:34 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 16:09:34 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 16:09:34 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 16:09:34 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 16:09:34 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 16:09:34 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 16:09:34 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 16:09:34 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54120. -2016-05-25 16:09:34 INFO NettyBlockTransferService:59 - Server created on 54120 -2016-05-25 16:09:34 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 16:09:34 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54120 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54120) -2016-05-25 16:09:34 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 16:09:35 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 16:09:35 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 16:09:35 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 16:09:35 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 16:09:35 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54120 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 16:09:35 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 16:09:36 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 16:09:37 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 16:09:37 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 16:09:37 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 16:09:37 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 16:09:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:09:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:09:37 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 16:09:37 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 16:09:37 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 16:09:37 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 16:09:37 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 16:09:37 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54120 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 16:09:37 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:09:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 16:09:37 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 16:09:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:09:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:09:37 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 16:09:37 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 16:09:37 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 16:09:37 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 16:09:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 16:09:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 16:09:37 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 16:09:37 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 16:09:37 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 16:09:37 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 16:09:37 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 16:09:37 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177630, maxMem=766075207 -2016-05-25 16:09:37 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 16:09:37 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:54120 (size: 83.1 KB, free: 730.5 MB) -2016-05-25 16:09:37 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262710, maxMem=766075207 -2016-05-25 16:09:37 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 16:09:37 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:54120 (size: 83.6 KB, free: 730.4 MB) -2016-05-25 16:09:37 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 16:09:37 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 16:09:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 369 ms on localhost (1/2) -2016-05-25 16:09:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 351 ms on localhost (2/2) -2016-05-25 16:09:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 16:09:37 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.393 s -2016-05-25 16:09:37 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.551139 s -2016-05-25 16:09:37 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 16:09:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 16:09:37 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 16:09:37 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 16:09:37 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 16:09:37 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 16:09:37 INFO BlockManager:59 - BlockManager stopped -2016-05-25 16:09:37 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 16:09:37 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 16:09:37 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 16:09:37 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 16:09:37 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-913aff90-9604-490c-b343-94625169801b -2016-05-25 16:14:26 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 16:14:27 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 16:14:27 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 16:14:27 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 16:14:27 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 16:14:28 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 16:14:28 INFO Remoting:74 - Starting remoting -2016-05-25 16:14:28 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:54150] -2016-05-25 16:14:28 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54150. -2016-05-25 16:14:28 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 16:14:28 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 16:14:28 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-2d458e52-f75e-4566-bbea-f96ac790763a -2016-05-25 16:14:28 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 16:14:28 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-cac856dd-f963-4523-b240-5514c7b422c3\httpd-f59425d6-3843-49c0-bff9-1cc04cbccf9c -2016-05-25 16:14:28 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 16:14:28 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 16:14:28 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54151 -2016-05-25 16:14:28 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54151. -2016-05-25 16:14:28 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 16:14:29 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 16:14:29 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 16:14:29 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 16:14:29 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 16:14:29 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 16:14:29 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 16:14:29 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54170. -2016-05-25 16:14:29 INFO NettyBlockTransferService:59 - Server created on 54170 -2016-05-25 16:14:29 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 16:14:29 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54170 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54170) -2016-05-25 16:14:29 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 16:14:30 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 16:14:30 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 16:14:30 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 16:14:30 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 16:14:30 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54170 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 16:14:30 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 16:14:31 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 16:14:32 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 16:14:32 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 16:14:32 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 16:14:32 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 16:14:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:14:32 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:14:32 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 16:14:32 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 16:14:32 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 16:14:32 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 16:14:32 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 16:14:32 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54170 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 16:14:32 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 16:14:32 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 16:14:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:14:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:14:32 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 16:14:32 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 16:14:32 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 16:14:32 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 16:14:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 16:14:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 16:14:32 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 16:14:32 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 16:14:32 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 16:14:32 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 16:14:32 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 16:14:32 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177630, maxMem=766075207 -2016-05-25 16:14:32 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 16:14:32 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:54170 (size: 83.6 KB, free: 730.5 MB) -2016-05-25 16:14:32 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263278, maxMem=766075207 -2016-05-25 16:14:32 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 16:14:32 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:54170 (size: 83.1 KB, free: 730.4 MB) -2016-05-25 16:14:32 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 16:14:32 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 16:14:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 354 ms on localhost (1/2) -2016-05-25 16:14:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 377 ms on localhost (2/2) -2016-05-25 16:14:32 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.395 s -2016-05-25 16:14:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 16:14:32 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.504815 s -2016-05-25 16:14:32 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 16:14:32 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 16:14:32 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 16:14:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:14:32 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:14:32 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[11] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 16:14:32 INFO MemoryStore:59 - ensureFreeSpace(4912) called with curMem=348358, maxMem=766075207 -2016-05-25 16:14:32 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-25 16:14:32 INFO MemoryStore:59 - ensureFreeSpace(2601) called with curMem=353270, maxMem=766075207 -2016-05-25 16:14:32 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 16:14:32 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54170 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 16:14:32 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[11] at map at DecisionTreeMetadata.scala:110) -2016-05-25 16:14:32 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 16:14:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:14:32 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 16:14:32 INFO CacheManager:59 - Partition rdd_9_0 not found, computing it -2016-05-25 16:14:32 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:14:32 INFO MemoryStore:59 - ensureFreeSpace(53640) called with curMem=355871, maxMem=766075207 -2016-05-25 16:14:32 INFO MemoryStore:59 - Block rdd_9_0 stored as values in memory (estimated size 52.4 KB, free 730.2 MB) -2016-05-25 16:14:32 INFO BlockManagerInfo:59 - Added rdd_9_0 in memory on localhost:54170 (size: 52.4 KB, free: 730.4 MB) -2016-05-25 16:14:32 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2611 bytes result sent to driver -2016-05-25 16:14:32 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.046 s -2016-05-25 16:14:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 46 ms on localhost (1/1) -2016-05-25 16:14:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 16:14:32 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.068423 s -2016-05-25 16:14:32 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 16:14:32 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 16:14:32 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 16:14:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:14:32 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:14:32 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 16:14:32 INFO MemoryStore:59 - ensureFreeSpace(4480) called with curMem=409511, maxMem=766075207 -2016-05-25 16:14:32 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.4 KB, free 730.2 MB) -2016-05-25 16:14:32 INFO MemoryStore:59 - ensureFreeSpace(2382) called with curMem=413991, maxMem=766075207 -2016-05-25 16:14:32 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 16:14:32 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54170 (size: 2.3 KB, free: 730.4 MB) -2016-05-25 16:14:32 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at retag at RandomForest.scala:137) -2016-05-25 16:14:32 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 16:14:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:14:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:14:32 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 16:14:32 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 16:14:32 INFO CacheManager:59 - Partition rdd_9_1 not found, computing it -2016-05-25 16:14:32 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:14:32 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 16:14:32 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 16:14:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 19 ms on localhost (1/2) -2016-05-25 16:14:32 INFO MemoryStore:59 - ensureFreeSpace(63648) called with curMem=416373, maxMem=766075207 -2016-05-25 16:14:32 INFO MemoryStore:59 - Block rdd_9_1 stored as values in memory (estimated size 62.2 KB, free 730.1 MB) -2016-05-25 16:14:32 INFO BlockManagerInfo:59 - Added rdd_9_1 in memory on localhost:54170 (size: 62.2 KB, free: 730.3 MB) -2016-05-25 16:14:33 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2662 bytes result sent to driver -2016-05-25 16:14:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 32 ms on localhost (2/2) -2016-05-25 16:14:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 16:14:33 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.036 s -2016-05-25 16:14:33 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.050812 s -2016-05-25 16:14:33 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 16:14:33 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 16:14:33 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 16:14:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:14:33 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:14:33 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[12] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 16:14:33 INFO MemoryStore:59 - ensureFreeSpace(5208) called with curMem=480021, maxMem=766075207 -2016-05-25 16:14:33 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.1 KB, free 730.1 MB) -2016-05-25 16:14:33 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=485229, maxMem=766075207 -2016-05-25 16:14:33 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.1 MB) -2016-05-25 16:14:33 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54170 (size: 2.6 KB, free: 730.3 MB) -2016-05-25 16:14:33 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[12] at sample at DecisionTree.scala:989) -2016-05-25 16:14:33 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 16:14:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 16:14:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 16:14:33 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 16:14:33 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 16:14:33 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-25 16:14:33 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:14:33 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 54824 bytes result sent to driver -2016-05-25 16:14:33 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 64549 bytes result sent to driver -2016-05-25 16:14:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 34 ms on localhost (1/2) -2016-05-25 16:14:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 33 ms on localhost (2/2) -2016-05-25 16:14:33 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.036 s -2016-05-25 16:14:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 16:14:33 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.060108 s -2016-05-25 16:14:33 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:54170 in memory (size: 2.6 KB, free: 730.3 MB) -2016-05-25 16:14:33 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 16:14:33 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:54170 in memory (size: 2.3 KB, free: 730.3 MB) -2016-05-25 16:14:33 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 16:14:33 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:54170 in memory (size: 2.5 KB, free: 730.3 MB) -2016-05-25 16:14:33 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 16:14:33 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:54170 in memory (size: 2.1 KB, free: 730.3 MB) -2016-05-25 16:14:33 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 16:14:34 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=459596, maxMem=766075207 -2016-05-25 16:14:34 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.1 MB) -2016-05-25 16:14:34 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=459636, maxMem=766075207 -2016-05-25 16:14:34 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.1 MB) -2016-05-25 16:14:34 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:54170 (size: 81.0 B, free: 730.3 MB) -2016-05-25 16:14:34 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 16:14:34 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:14:34 INFO DAGScheduler:59 - Registering RDD 15 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:14:34 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:14:34 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:14:34 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 16:14:34 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 16:14:34 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[15] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:14:34 INFO MemoryStore:59 - ensureFreeSpace(475936) called with curMem=459717, maxMem=766075207 -2016-05-25 16:14:34 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 464.8 KB, free 729.7 MB) -2016-05-25 16:14:34 INFO MemoryStore:59 - ensureFreeSpace(116672) called with curMem=935653, maxMem=766075207 -2016-05-25 16:14:34 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 113.9 KB, free 729.6 MB) -2016-05-25 16:14:34 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:54170 (size: 113.9 KB, free: 730.2 MB) -2016-05-25 16:14:34 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[15] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:14:34 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 16:14:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 16:14:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 16:14:34 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 16:14:34 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 16:14:34 INFO CacheManager:59 - Partition rdd_14_0 not found, computing it -2016-05-25 16:14:34 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:14:34 INFO CacheManager:59 - Partition rdd_14_1 not found, computing it -2016-05-25 16:14:34 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-25 16:14:35 INFO MemoryStore:59 - ensureFreeSpace(94400) called with curMem=1052325, maxMem=766075207 -2016-05-25 16:14:35 INFO MemoryStore:59 - Block rdd_14_0 stored as values in memory (estimated size 92.2 KB, free 729.5 MB) -2016-05-25 16:14:35 INFO BlockManagerInfo:59 - Added rdd_14_0 in memory on localhost:54170 (size: 92.2 KB, free: 730.1 MB) -2016-05-25 16:14:35 INFO MemoryStore:59 - ensureFreeSpace(114416) called with curMem=1146725, maxMem=766075207 -2016-05-25 16:14:35 INFO MemoryStore:59 - Block rdd_14_1 stored as values in memory (estimated size 111.7 KB, free 729.4 MB) -2016-05-25 16:14:35 INFO BlockManagerInfo:59 - Added rdd_14_1 in memory on localhost:54170 (size: 111.7 KB, free: 730.0 MB) -2016-05-25 16:14:35 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 16:14:35 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 16:14:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 584 ms on localhost (1/2) -2016-05-25 16:14:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 582 ms on localhost (2/2) -2016-05-25 16:14:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 16:14:35 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.587 s -2016-05-25 16:14:35 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:14:35 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:14:35 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 16:14:35 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:14:35 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 16:14:35 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[17] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:14:35 INFO MemoryStore:59 - ensureFreeSpace(206096) called with curMem=1261141, maxMem=766075207 -2016-05-25 16:14:35 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 201.3 KB, free 729.2 MB) -2016-05-25 16:14:35 INFO MemoryStore:59 - ensureFreeSpace(43961) called with curMem=1467237, maxMem=766075207 -2016-05-25 16:14:35 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 42.9 KB, free 729.1 MB) -2016-05-25 16:14:35 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:54170 (size: 42.9 KB, free: 729.9 MB) -2016-05-25 16:14:35 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[17] at map at DecisionTree.scala:645) -2016-05-25 16:14:35 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 16:14:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:14:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:14:35 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 16:14:35 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 16:14:35 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:14:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 8 ms -2016-05-25 16:14:35 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:14:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:14:35 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 16:14:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 322 ms on localhost (1/2) -2016-05-25 16:14:35 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 16:14:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 570 ms on localhost (2/2) -2016-05-25 16:14:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 16:14:35 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.573 s -2016-05-25 16:14:35 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 1.235570 s -2016-05-25 16:14:35 INFO MapPartitionsRDD:59 - Removing RDD 14 from persistence list -2016-05-25 16:14:35 INFO BlockManager:59 - Removing RDD 14 -2016-05-25 16:14:35 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 16:14:35 INFO RandomForest:59 - init: 1.625126334 - total: 3.07286968 - findSplitsBins: 1.343334897 - findBestSplits: 1.429461559 - chooseSplits: 1.422796924 -2016-05-25 16:14:36 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 16:14:36 INFO DAGScheduler:59 - Got job 5 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 16:14:36 INFO DAGScheduler:59 - Final stage: ResultStage 6(take at DecisionTreeMetadata.scala:110) -2016-05-25 16:14:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:14:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:14:36 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[23] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(7840) called with curMem=1302382, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 7.7 KB, free 729.3 MB) -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(4076) called with curMem=1310222, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 4.0 KB, free 729.3 MB) -2016-05-25 16:14:36 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:54170 (size: 4.0 KB, free: 730.1 MB) -2016-05-25 16:14:36 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:36 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 6 (MapPartitionsRDD[23] at map at DecisionTreeMetadata.scala:110) -2016-05-25 16:14:36 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 1 tasks -2016-05-25 16:14:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2374 bytes) -2016-05-25 16:14:36 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 16:14:36 INFO CacheManager:59 - Partition rdd_18_0 not found, computing it -2016-05-25 16:14:36 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(1472) called with curMem=1314298, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block rdd_18_0 stored as values in memory (estimated size 1472.0 B, free 729.3 MB) -2016-05-25 16:14:36 INFO BlockManagerInfo:59 - Added rdd_18_0 in memory on localhost:54170 (size: 1472.0 B, free: 730.1 MB) -2016-05-25 16:14:36 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:14:36 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2611 bytes result sent to driver -2016-05-25 16:14:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 38 ms on localhost (1/1) -2016-05-25 16:14:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 16:14:36 INFO DAGScheduler:59 - ResultStage 6 (take at DecisionTreeMetadata.scala:110) finished in 0.041 s -2016-05-25 16:14:36 INFO DAGScheduler:59 - Job 5 finished: take at DecisionTreeMetadata.scala:110, took 0.100237 s -2016-05-25 16:14:36 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 16:14:36 INFO DAGScheduler:59 - Got job 6 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 16:14:36 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at DecisionTreeMetadata.scala:114) -2016-05-25 16:14:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:14:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:14:36 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[22] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(7408) called with curMem=1315770, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 7.2 KB, free 729.3 MB) -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(3858) called with curMem=1323178, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.8 KB, free 729.3 MB) -2016-05-25 16:14:36 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:54170 (size: 3.8 KB, free: 730.1 MB) -2016-05-25 16:14:36 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[22] at retag at RandomForest.scala:137) -2016-05-25 16:14:36 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 16:14:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 12, localhost, PROCESS_LOCAL, 2374 bytes) -2016-05-25 16:14:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2374 bytes) -2016-05-25 16:14:36 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 12) -2016-05-25 16:14:36 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 13) -2016-05-25 16:14:36 INFO BlockManager:59 - Found block rdd_18_0 locally -2016-05-25 16:14:36 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:14:36 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 12). 2082 bytes result sent to driver -2016-05-25 16:14:36 INFO CacheManager:59 - Partition rdd_18_1 not found, computing it -2016-05-25 16:14:36 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-25 16:14:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 12) in 20 ms on localhost (1/2) -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(1776) called with curMem=1327036, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block rdd_18_1 stored as values in memory (estimated size 1776.0 B, free 729.3 MB) -2016-05-25 16:14:36 INFO BlockManagerInfo:59 - Added rdd_18_1 in memory on localhost:54170 (size: 1776.0 B, free: 730.1 MB) -2016-05-25 16:14:36 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-25 16:14:36 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 13). 2662 bytes result sent to driver -2016-05-25 16:14:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 13) in 26 ms on localhost (2/2) -2016-05-25 16:14:36 INFO DAGScheduler:59 - ResultStage 7 (count at DecisionTreeMetadata.scala:114) finished in 0.027 s -2016-05-25 16:14:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 16:14:36 INFO DAGScheduler:59 - Job 6 finished: count at DecisionTreeMetadata.scala:114, took 0.051887 s -2016-05-25 16:14:36 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 16:14:36 INFO DAGScheduler:59 - Got job 7 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 16:14:36 INFO DAGScheduler:59 - Final stage: ResultStage 8(collect at DecisionTree.scala:989) -2016-05-25 16:14:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:14:36 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:14:36 INFO DAGScheduler:59 - Submitting ResultStage 8 (PartitionwiseSampledRDD[24] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(8136) called with curMem=1328812, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 7.9 KB, free 729.3 MB) -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(4175) called with curMem=1336948, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 4.1 KB, free 729.3 MB) -2016-05-25 16:14:36 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:54170 (size: 4.1 KB, free: 730.1 MB) -2016-05-25 16:14:36 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (PartitionwiseSampledRDD[24] at sample at DecisionTree.scala:989) -2016-05-25 16:14:36 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-05-25 16:14:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 14, localhost, PROCESS_LOCAL, 2483 bytes) -2016-05-25 16:14:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2483 bytes) -2016-05-25 16:14:36 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 15) -2016-05-25 16:14:36 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 14) -2016-05-25 16:14:36 INFO BlockManager:59 - Found block rdd_18_0 locally -2016-05-25 16:14:36 INFO BlockManager:59 - Found block rdd_18_1 locally -2016-05-25 16:14:36 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:14:36 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-25 16:14:36 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 14). 54824 bytes result sent to driver -2016-05-25 16:14:36 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 15). 64549 bytes result sent to driver -2016-05-25 16:14:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 14) in 14 ms on localhost (1/2) -2016-05-25 16:14:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 15) in 14 ms on localhost (2/2) -2016-05-25 16:14:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 16:14:36 INFO DAGScheduler:59 - ResultStage 8 (collect at DecisionTree.scala:989) finished in 0.016 s -2016-05-25 16:14:36 INFO DAGScheduler:59 - Job 7 finished: collect at DecisionTree.scala:989, took 0.032280 s -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=1341123, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 40.0 B, free 729.3 MB) -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=1341163, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 81.0 B, free 729.3 MB) -2016-05-25 16:14:36 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:54170 (size: 81.0 B, free: 730.1 MB) -2016-05-25 16:14:36 INFO SparkContext:59 - Created broadcast 11 from broadcast at DecisionTree.scala:604 -2016-05-25 16:14:36 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:14:36 INFO DAGScheduler:59 - Registering RDD 27 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:14:36 INFO DAGScheduler:59 - Got job 8 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:14:36 INFO DAGScheduler:59 - Final stage: ResultStage 10(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:14:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9) -2016-05-25 16:14:36 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9) -2016-05-25 16:14:36 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (MapPartitionsRDD[27] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(477616) called with curMem=1341244, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 466.4 KB, free 728.9 MB) -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(117301) called with curMem=1818860, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 114.6 KB, free 728.7 MB) -2016-05-25 16:14:36 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:54170 (size: 114.6 KB, free: 730.0 MB) -2016-05-25 16:14:36 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 9 (MapPartitionsRDD[27] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:14:36 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 16:14:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 16, localhost, PROCESS_LOCAL, 2363 bytes) -2016-05-25 16:14:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2363 bytes) -2016-05-25 16:14:36 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 16) -2016-05-25 16:14:36 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 17) -2016-05-25 16:14:36 INFO CacheManager:59 - Partition rdd_26_0 not found, computing it -2016-05-25 16:14:36 INFO BlockManager:59 - Found block rdd_18_0 locally -2016-05-25 16:14:36 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(94400) called with curMem=1936161, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block rdd_26_0 stored as values in memory (estimated size 92.2 KB, free 728.6 MB) -2016-05-25 16:14:36 INFO BlockManagerInfo:59 - Added rdd_26_0 in memory on localhost:54170 (size: 92.2 KB, free: 729.9 MB) -2016-05-25 16:14:36 INFO CacheManager:59 - Partition rdd_26_1 not found, computing it -2016-05-25 16:14:36 INFO BlockManager:59 - Found block rdd_18_1 locally -2016-05-25 16:14:36 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(114416) called with curMem=2030561, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block rdd_26_1 stored as values in memory (estimated size 111.7 KB, free 728.5 MB) -2016-05-25 16:14:36 INFO BlockManagerInfo:59 - Added rdd_26_1 in memory on localhost:54170 (size: 111.7 KB, free: 729.8 MB) -2016-05-25 16:14:36 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 16). 2834 bytes result sent to driver -2016-05-25 16:14:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 16) in 76 ms on localhost (1/2) -2016-05-25 16:14:36 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 17). 2834 bytes result sent to driver -2016-05-25 16:14:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 17) in 84 ms on localhost (2/2) -2016-05-25 16:14:36 INFO DAGScheduler:59 - ShuffleMapStage 9 (mapPartitions at DecisionTree.scala:625) finished in 0.086 s -2016-05-25 16:14:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 16:14:36 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:14:36 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:14:36 INFO DAGScheduler:59 - waiting: Set(ResultStage 10) -2016-05-25 16:14:36 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:14:36 INFO DAGScheduler:59 - Missing parents for ResultStage 10: List() -2016-05-25 16:14:36 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[29] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(206096) called with curMem=2144977, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 201.3 KB, free 728.3 MB) -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(43950) called with curMem=2351073, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 42.9 KB, free 728.3 MB) -2016-05-25 16:14:36 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:54170 (size: 42.9 KB, free: 729.8 MB) -2016-05-25 16:14:36 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[29] at map at DecisionTree.scala:645) -2016-05-25 16:14:36 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-25 16:14:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:14:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:14:36 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 19) -2016-05-25 16:14:36 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 18) -2016-05-25 16:14:36 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:14:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:14:36 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:14:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:14:36 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 19). 1161 bytes result sent to driver -2016-05-25 16:14:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 19) in 8 ms on localhost (1/2) -2016-05-25 16:14:36 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 18). 2774 bytes result sent to driver -2016-05-25 16:14:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 18) in 54 ms on localhost (2/2) -2016-05-25 16:14:36 INFO DAGScheduler:59 - ResultStage 10 (collectAsMap at DecisionTree.scala:654) finished in 0.054 s -2016-05-25 16:14:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 16:14:36 INFO DAGScheduler:59 - Job 8 finished: collectAsMap at DecisionTree.scala:654, took 0.179030 s -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=2395023, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 40.0 B, free 728.3 MB) -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=2395063, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 81.0 B, free 728.3 MB) -2016-05-25 16:14:36 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:54170 (size: 81.0 B, free: 729.8 MB) -2016-05-25 16:14:36 INFO SparkContext:59 - Created broadcast 14 from broadcast at DecisionTree.scala:604 -2016-05-25 16:14:36 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:14:36 INFO DAGScheduler:59 - Registering RDD 30 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:14:36 INFO DAGScheduler:59 - Got job 9 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:14:36 INFO DAGScheduler:59 - Final stage: ResultStage 12(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:14:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 11) -2016-05-25 16:14:36 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 11) -2016-05-25 16:14:36 INFO DAGScheduler:59 - Submitting ShuffleMapStage 11 (MapPartitionsRDD[30] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(477816) called with curMem=2395144, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 466.6 KB, free 727.8 MB) -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(117414) called with curMem=2872960, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 114.7 KB, free 727.7 MB) -2016-05-25 16:14:36 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:54170 (size: 114.7 KB, free: 729.7 MB) -2016-05-25 16:14:36 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 11 (MapPartitionsRDD[30] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:14:36 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-25 16:14:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 20, localhost, PROCESS_LOCAL, 2363 bytes) -2016-05-25 16:14:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2363 bytes) -2016-05-25 16:14:36 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 20) -2016-05-25 16:14:36 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 21) -2016-05-25 16:14:36 INFO BlockManager:59 - Found block rdd_26_1 locally -2016-05-25 16:14:36 INFO BlockManager:59 - Found block rdd_26_0 locally -2016-05-25 16:14:36 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 20). 2254 bytes result sent to driver -2016-05-25 16:14:36 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 21). 2254 bytes result sent to driver -2016-05-25 16:14:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 21) in 22 ms on localhost (1/2) -2016-05-25 16:14:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 20) in 24 ms on localhost (2/2) -2016-05-25 16:14:36 INFO DAGScheduler:59 - ShuffleMapStage 11 (mapPartitions at DecisionTree.scala:625) finished in 0.024 s -2016-05-25 16:14:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 16:14:36 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:14:36 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:14:36 INFO DAGScheduler:59 - waiting: Set(ResultStage 12) -2016-05-25 16:14:36 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:14:36 INFO DAGScheduler:59 - Missing parents for ResultStage 12: List() -2016-05-25 16:14:36 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[32] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(206096) called with curMem=2990374, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 201.3 KB, free 727.5 MB) -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(43958) called with curMem=3196470, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 42.9 KB, free 727.5 MB) -2016-05-25 16:14:36 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:54170 (size: 42.9 KB, free: 729.6 MB) -2016-05-25 16:14:36 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[32] at map at DecisionTree.scala:645) -2016-05-25 16:14:36 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-25 16:14:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 22, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:14:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:14:36 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 22) -2016-05-25 16:14:36 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:14:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:14:36 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 22). 2774 bytes result sent to driver -2016-05-25 16:14:36 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 23) -2016-05-25 16:14:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 22) in 58 ms on localhost (1/2) -2016-05-25 16:14:36 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:14:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:14:36 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 23). 1161 bytes result sent to driver -2016-05-25 16:14:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 23) in 70 ms on localhost (2/2) -2016-05-25 16:14:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 16:14:36 INFO DAGScheduler:59 - ResultStage 12 (collectAsMap at DecisionTree.scala:654) finished in 0.074 s -2016-05-25 16:14:36 INFO DAGScheduler:59 - Job 9 finished: collectAsMap at DecisionTree.scala:654, took 0.143206 s -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=3240428, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 40.0 B, free 727.5 MB) -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=3240468, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 81.0 B, free 727.5 MB) -2016-05-25 16:14:36 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:54170 (size: 81.0 B, free: 729.6 MB) -2016-05-25 16:14:36 INFO SparkContext:59 - Created broadcast 17 from broadcast at DecisionTree.scala:604 -2016-05-25 16:14:36 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:14:36 INFO DAGScheduler:59 - Registering RDD 33 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:14:36 INFO DAGScheduler:59 - Got job 10 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:14:36 INFO DAGScheduler:59 - Final stage: ResultStage 14(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:14:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 13) -2016-05-25 16:14:36 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 13) -2016-05-25 16:14:36 INFO DAGScheduler:59 - Submitting ShuffleMapStage 13 (MapPartitionsRDD[33] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(478040) called with curMem=3240549, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 466.8 KB, free 727.0 MB) -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(117491) called with curMem=3718589, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 114.7 KB, free 726.9 MB) -2016-05-25 16:14:36 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:54170 (size: 114.7 KB, free: 729.5 MB) -2016-05-25 16:14:36 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 13 (MapPartitionsRDD[33] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:14:36 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-25 16:14:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 24, localhost, PROCESS_LOCAL, 2363 bytes) -2016-05-25 16:14:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2363 bytes) -2016-05-25 16:14:36 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 25) -2016-05-25 16:14:36 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 24) -2016-05-25 16:14:36 INFO BlockManager:59 - Found block rdd_26_1 locally -2016-05-25 16:14:36 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 25). 2254 bytes result sent to driver -2016-05-25 16:14:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 25) in 33 ms on localhost (1/2) -2016-05-25 16:14:36 INFO BlockManager:59 - Found block rdd_26_0 locally -2016-05-25 16:14:36 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 24). 2254 bytes result sent to driver -2016-05-25 16:14:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 24) in 56 ms on localhost (2/2) -2016-05-25 16:14:36 INFO DAGScheduler:59 - ShuffleMapStage 13 (mapPartitions at DecisionTree.scala:625) finished in 0.056 s -2016-05-25 16:14:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 16:14:36 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:14:36 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:14:36 INFO DAGScheduler:59 - waiting: Set(ResultStage 14) -2016-05-25 16:14:36 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:14:36 INFO DAGScheduler:59 - Missing parents for ResultStage 14: List() -2016-05-25 16:14:36 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[35] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(206096) called with curMem=3836080, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 201.3 KB, free 726.7 MB) -2016-05-25 16:14:36 INFO MemoryStore:59 - ensureFreeSpace(43965) called with curMem=4042176, maxMem=766075207 -2016-05-25 16:14:36 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 42.9 KB, free 726.7 MB) -2016-05-25 16:14:36 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:54170 (size: 42.9 KB, free: 729.5 MB) -2016-05-25 16:14:36 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[35] at map at DecisionTree.scala:645) -2016-05-25 16:14:36 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-25 16:14:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 26, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:14:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:14:36 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 26) -2016-05-25 16:14:36 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 27) -2016-05-25 16:14:36 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:14:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:14:36 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:14:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 16:14:36 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 27). 1161 bytes result sent to driver -2016-05-25 16:14:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 27) in 18 ms on localhost (1/2) -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:54170 in memory (size: 4.1 KB, free: 729.5 MB) -2016-05-25 16:14:37 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:54170 in memory (size: 3.8 KB, free: 729.5 MB) -2016-05-25 16:14:37 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 16:14:37 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 26). 2774 bytes result sent to driver -2016-05-25 16:14:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 26) in 59 ms on localhost (2/2) -2016-05-25 16:14:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:54170 in memory (size: 4.0 KB, free: 729.5 MB) -2016-05-25 16:14:37 INFO DAGScheduler:59 - ResultStage 14 (collectAsMap at DecisionTree.scala:654) finished in 0.080 s -2016-05-25 16:14:37 INFO DAGScheduler:59 - Job 10 finished: collectAsMap at DecisionTree.scala:654, took 0.210398 s -2016-05-25 16:14:37 INFO MapPartitionsRDD:59 - Removing RDD 26 from persistence list -2016-05-25 16:14:37 INFO BlockManager:59 - Removing RDD 26 -2016-05-25 16:14:37 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 16:14:37 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 16:14:37 INFO RandomForest:59 - init: 0.337726001 - total: 1.084942414 - findSplitsBins: 0.145240047 - findBestSplits: 0.702779414 - chooseSplits: 0.702233697 -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:54170 in memory (size: 42.9 KB, free: 729.7 MB) -2016-05-25 16:14:37 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 16:14:37 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:54170 in memory (size: 113.9 KB, free: 729.8 MB) -2016-05-25 16:14:37 INFO DAGScheduler:59 - Got job 11 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 16:14:37 INFO DAGScheduler:59 - Final stage: ResultStage 15(take at DecisionTreeMetadata.scala:110) -2016-05-25 16:14:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:14:37 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 16:14:37 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 16:14:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:14:37 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[41] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(8744) called with curMem=2999167, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 8.5 KB, free 727.7 MB) -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(4421) called with curMem=3007911, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 4.3 KB, free 727.7 MB) -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:54170 (size: 4.3 KB, free: 729.8 MB) -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:54170 in memory (size: 81.0 B, free: 729.8 MB) -2016-05-25 16:14:37 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:37 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 15 (MapPartitionsRDD[41] at map at DecisionTreeMetadata.scala:110) -2016-05-25 16:14:37 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 1 tasks -2016-05-25 16:14:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 28, localhost, PROCESS_LOCAL, 2406 bytes) -2016-05-25 16:14:37 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 28) -2016-05-25 16:14:37 INFO CacheManager:59 - Partition rdd_37_0 not found, computing it -2016-05-25 16:14:37 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:14:37 INFO BlockManager:59 - Found block rdd_18_0 locally -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(1472) called with curMem=3012211, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block rdd_37_0 stored as values in memory (estimated size 1472.0 B, free 727.7 MB) -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Added rdd_37_0 in memory on localhost:54170 (size: 1472.0 B, free: 729.8 MB) -2016-05-25 16:14:37 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:14:37 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 28). 2611 bytes result sent to driver -2016-05-25 16:14:37 INFO BlockManager:59 - Removing RDD 14 -2016-05-25 16:14:37 INFO ContextCleaner:59 - Cleaned RDD 14 -2016-05-25 16:14:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 28) in 47 ms on localhost (1/1) -2016-05-25 16:14:37 INFO DAGScheduler:59 - ResultStage 15 (take at DecisionTreeMetadata.scala:110) finished in 0.050 s -2016-05-25 16:14:37 INFO DAGScheduler:59 - Job 11 finished: take at DecisionTreeMetadata.scala:110, took 0.124855 s -2016-05-25 16:14:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:54170 in memory (size: 114.7 KB, free: 729.9 MB) -2016-05-25 16:14:37 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 16:14:37 INFO DAGScheduler:59 - Got job 12 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 16:14:37 INFO DAGScheduler:59 - Final stage: ResultStage 16(count at DecisionTreeMetadata.scala:114) -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:54170 in memory (size: 42.9 KB, free: 730.0 MB) -2016-05-25 16:14:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:14:37 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-25 16:14:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:14:37 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[40] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:54170 in memory (size: 114.7 KB, free: 730.1 MB) -2016-05-25 16:14:37 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-25 16:14:37 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(8312) called with curMem=1572868, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 8.1 KB, free 729.1 MB) -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:54170 in memory (size: 81.0 B, free: 730.1 MB) -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(4213) called with curMem=1581099, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 4.1 KB, free 729.1 MB) -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:54170 (size: 4.1 KB, free: 730.1 MB) -2016-05-25 16:14:37 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[40] at retag at RandomForest.scala:137) -2016-05-25 16:14:37 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:54170 in memory (size: 42.9 KB, free: 730.1 MB) -2016-05-25 16:14:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 29, localhost, PROCESS_LOCAL, 2406 bytes) -2016-05-25 16:14:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 30, localhost, PROCESS_LOCAL, 2406 bytes) -2016-05-25 16:14:37 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 30) -2016-05-25 16:14:37 INFO CacheManager:59 - Partition rdd_37_1 not found, computing it -2016-05-25 16:14:37 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-25 16:14:37 INFO BlockManager:59 - Found block rdd_18_1 locally -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(1776) called with curMem=1335226, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block rdd_37_1 stored as values in memory (estimated size 1776.0 B, free 729.3 MB) -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Added rdd_37_1 in memory on localhost:54170 (size: 1776.0 B, free: 730.1 MB) -2016-05-25 16:14:37 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-25 16:14:37 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 30). 2662 bytes result sent to driver -2016-05-25 16:14:37 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 29) -2016-05-25 16:14:37 INFO BlockManager:59 - Found block rdd_37_0 locally -2016-05-25 16:14:37 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:14:37 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 29). 2082 bytes result sent to driver -2016-05-25 16:14:37 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 16:14:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 30) in 33 ms on localhost (1/2) -2016-05-25 16:14:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 29) in 35 ms on localhost (2/2) -2016-05-25 16:14:37 INFO DAGScheduler:59 - ResultStage 16 (count at DecisionTreeMetadata.scala:114) finished in 0.035 s -2016-05-25 16:14:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-25 16:14:37 INFO DAGScheduler:59 - Job 12 finished: count at DecisionTreeMetadata.scala:114, took 0.057838 s -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:54170 in memory (size: 114.6 KB, free: 730.2 MB) -2016-05-25 16:14:37 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 16:14:37 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:54170 in memory (size: 81.0 B, free: 730.2 MB) -2016-05-25 16:14:37 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 16:14:37 INFO DAGScheduler:59 - Got job 13 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 16:14:37 INFO DAGScheduler:59 - Final stage: ResultStage 17(collect at DecisionTree.scala:989) -2016-05-25 16:14:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:14:37 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:14:37 INFO DAGScheduler:59 - Submitting ResultStage 17 (PartitionwiseSampledRDD[42] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(9040) called with curMem=741964, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 8.8 KB, free 729.9 MB) -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(4515) called with curMem=751004, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 4.4 KB, free 729.9 MB) -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:54170 (size: 4.4 KB, free: 730.2 MB) -2016-05-25 16:14:37 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (PartitionwiseSampledRDD[42] at sample at DecisionTree.scala:989) -2016-05-25 16:14:37 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-25 16:14:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 31, localhost, PROCESS_LOCAL, 2515 bytes) -2016-05-25 16:14:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 32, localhost, PROCESS_LOCAL, 2515 bytes) -2016-05-25 16:14:37 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 31) -2016-05-25 16:14:37 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 32) -2016-05-25 16:14:37 INFO BlockManager:59 - Found block rdd_37_1 locally -2016-05-25 16:14:37 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-25 16:14:37 INFO BlockManager:59 - Found block rdd_37_0 locally -2016-05-25 16:14:37 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:14:37 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 32). 64549 bytes result sent to driver -2016-05-25 16:14:37 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 31). 54824 bytes result sent to driver -2016-05-25 16:14:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 32) in 9 ms on localhost (1/2) -2016-05-25 16:14:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 31) in 11 ms on localhost (2/2) -2016-05-25 16:14:37 INFO DAGScheduler:59 - ResultStage 17 (collect at DecisionTree.scala:989) finished in 0.012 s -2016-05-25 16:14:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-25 16:14:37 INFO DAGScheduler:59 - Job 13 finished: collect at DecisionTree.scala:989, took 0.028332 s -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=755519, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 40.0 B, free 729.9 MB) -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=755559, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 81.0 B, free 729.9 MB) -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:54170 (size: 81.0 B, free: 730.2 MB) -2016-05-25 16:14:37 INFO SparkContext:59 - Created broadcast 23 from broadcast at DecisionTree.scala:604 -2016-05-25 16:14:37 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:14:37 INFO DAGScheduler:59 - Registering RDD 45 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:14:37 INFO DAGScheduler:59 - Got job 14 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:14:37 INFO DAGScheduler:59 - Final stage: ResultStage 19(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:14:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 18) -2016-05-25 16:14:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 18) -2016-05-25 16:14:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 18 (MapPartitionsRDD[45] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(478520) called with curMem=755640, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 467.3 KB, free 729.4 MB) -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(117928) called with curMem=1234160, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 115.2 KB, free 729.3 MB) -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:54170 (size: 115.2 KB, free: 730.1 MB) -2016-05-25 16:14:37 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 18 (MapPartitionsRDD[45] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:14:37 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-25 16:14:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 33, localhost, PROCESS_LOCAL, 2395 bytes) -2016-05-25 16:14:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 34, localhost, PROCESS_LOCAL, 2395 bytes) -2016-05-25 16:14:37 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 33) -2016-05-25 16:14:37 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 34) -2016-05-25 16:14:37 INFO CacheManager:59 - Partition rdd_44_0 not found, computing it -2016-05-25 16:14:37 INFO BlockManager:59 - Found block rdd_37_0 locally -2016-05-25 16:14:37 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:14:37 INFO CacheManager:59 - Partition rdd_44_1 not found, computing it -2016-05-25 16:14:37 INFO BlockManager:59 - Found block rdd_37_1 locally -2016-05-25 16:14:37 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(94400) called with curMem=1352088, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block rdd_44_0 stored as values in memory (estimated size 92.2 KB, free 729.2 MB) -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Added rdd_44_0 in memory on localhost:54170 (size: 92.2 KB, free: 730.0 MB) -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(114416) called with curMem=1446488, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block rdd_44_1 stored as values in memory (estimated size 111.7 KB, free 729.1 MB) -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Added rdd_44_1 in memory on localhost:54170 (size: 111.7 KB, free: 729.9 MB) -2016-05-25 16:14:37 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 33). 2834 bytes result sent to driver -2016-05-25 16:14:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 33) in 58 ms on localhost (1/2) -2016-05-25 16:14:37 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 34). 2834 bytes result sent to driver -2016-05-25 16:14:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 34) in 61 ms on localhost (2/2) -2016-05-25 16:14:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-25 16:14:37 INFO DAGScheduler:59 - ShuffleMapStage 18 (mapPartitions at DecisionTree.scala:625) finished in 0.062 s -2016-05-25 16:14:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:14:37 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:14:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 19) -2016-05-25 16:14:37 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:14:37 INFO DAGScheduler:59 - Missing parents for ResultStage 19: List() -2016-05-25 16:14:37 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[47] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(206096) called with curMem=1560904, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 201.3 KB, free 728.9 MB) -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(43954) called with curMem=1767000, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 42.9 KB, free 728.9 MB) -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:54170 (size: 42.9 KB, free: 729.9 MB) -2016-05-25 16:14:37 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[47] at map at DecisionTree.scala:645) -2016-05-25 16:14:37 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-25 16:14:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 35, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:14:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 36, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:14:37 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 35) -2016-05-25 16:14:37 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 36) -2016-05-25 16:14:37 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:14:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:14:37 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:14:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:14:37 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 36). 1161 bytes result sent to driver -2016-05-25 16:14:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 36) in 10 ms on localhost (1/2) -2016-05-25 16:14:37 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 35). 2774 bytes result sent to driver -2016-05-25 16:14:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 35) in 23 ms on localhost (2/2) -2016-05-25 16:14:37 INFO DAGScheduler:59 - ResultStage 19 (collectAsMap at DecisionTree.scala:654) finished in 0.025 s -2016-05-25 16:14:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-25 16:14:37 INFO DAGScheduler:59 - Job 14 finished: collectAsMap at DecisionTree.scala:654, took 0.128305 s -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=1810954, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 40.0 B, free 728.9 MB) -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=1810994, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 81.0 B, free 728.9 MB) -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:54170 (size: 81.0 B, free: 729.9 MB) -2016-05-25 16:14:37 INFO SparkContext:59 - Created broadcast 26 from broadcast at DecisionTree.scala:604 -2016-05-25 16:14:37 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:14:37 INFO DAGScheduler:59 - Registering RDD 48 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:14:37 INFO DAGScheduler:59 - Got job 15 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:14:37 INFO DAGScheduler:59 - Final stage: ResultStage 21(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:14:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 20) -2016-05-25 16:14:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 20) -2016-05-25 16:14:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 20 (MapPartitionsRDD[48] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(478848) called with curMem=1811075, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 467.6 KB, free 728.4 MB) -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(118102) called with curMem=2289923, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 115.3 KB, free 728.3 MB) -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:54170 (size: 115.3 KB, free: 729.8 MB) -2016-05-25 16:14:37 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 20 (MapPartitionsRDD[48] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:14:37 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-25 16:14:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 37, localhost, PROCESS_LOCAL, 2395 bytes) -2016-05-25 16:14:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 38, localhost, PROCESS_LOCAL, 2395 bytes) -2016-05-25 16:14:37 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 37) -2016-05-25 16:14:37 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 38) -2016-05-25 16:14:37 INFO BlockManager:59 - Found block rdd_44_1 locally -2016-05-25 16:14:37 INFO BlockManager:59 - Found block rdd_44_0 locally -2016-05-25 16:14:37 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 38). 2254 bytes result sent to driver -2016-05-25 16:14:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 38) in 32 ms on localhost (1/2) -2016-05-25 16:14:37 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 37). 2254 bytes result sent to driver -2016-05-25 16:14:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 37) in 40 ms on localhost (2/2) -2016-05-25 16:14:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-25 16:14:37 INFO DAGScheduler:59 - ShuffleMapStage 20 (mapPartitions at DecisionTree.scala:625) finished in 0.040 s -2016-05-25 16:14:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:14:37 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:14:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 21) -2016-05-25 16:14:37 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:14:37 INFO DAGScheduler:59 - Missing parents for ResultStage 21: List() -2016-05-25 16:14:37 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[50] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(206152) called with curMem=2408025, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 201.3 KB, free 728.1 MB) -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(43995) called with curMem=2614177, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 43.0 KB, free 728.1 MB) -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:54170 (size: 43.0 KB, free: 729.7 MB) -2016-05-25 16:14:37 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[50] at map at DecisionTree.scala:645) -2016-05-25 16:14:37 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-25 16:14:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 39, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:14:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 40, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:14:37 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 39) -2016-05-25 16:14:37 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 40) -2016-05-25 16:14:37 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:14:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:14:37 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:14:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:14:37 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 39). 2774 bytes result sent to driver -2016-05-25 16:14:37 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 40). 2774 bytes result sent to driver -2016-05-25 16:14:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 39) in 39 ms on localhost (1/2) -2016-05-25 16:14:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 40) in 39 ms on localhost (2/2) -2016-05-25 16:14:37 INFO DAGScheduler:59 - ResultStage 21 (collectAsMap at DecisionTree.scala:654) finished in 0.040 s -2016-05-25 16:14:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-25 16:14:37 INFO DAGScheduler:59 - Job 15 finished: collectAsMap at DecisionTree.scala:654, took 0.125566 s -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=2658172, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 40.0 B, free 728.1 MB) -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=2658212, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 81.0 B, free 728.1 MB) -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:54170 (size: 81.0 B, free: 729.7 MB) -2016-05-25 16:14:37 INFO SparkContext:59 - Created broadcast 29 from broadcast at DecisionTree.scala:604 -2016-05-25 16:14:37 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:14:37 INFO DAGScheduler:59 - Registering RDD 51 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:14:37 INFO DAGScheduler:59 - Got job 16 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:14:37 INFO DAGScheduler:59 - Final stage: ResultStage 23(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:14:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 22) -2016-05-25 16:14:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 22) -2016-05-25 16:14:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 22 (MapPartitionsRDD[51] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(479392) called with curMem=2658293, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 468.2 KB, free 727.6 MB) -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(118302) called with curMem=3137685, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 115.5 KB, free 727.5 MB) -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:54170 (size: 115.5 KB, free: 729.6 MB) -2016-05-25 16:14:37 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 22 (MapPartitionsRDD[51] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:14:37 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-25 16:14:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2395 bytes) -2016-05-25 16:14:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 42, localhost, PROCESS_LOCAL, 2395 bytes) -2016-05-25 16:14:37 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 42) -2016-05-25 16:14:37 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 41) -2016-05-25 16:14:37 INFO BlockManager:59 - Found block rdd_44_1 locally -2016-05-25 16:14:37 INFO BlockManager:59 - Found block rdd_44_0 locally -2016-05-25 16:14:37 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 41). 2254 bytes result sent to driver -2016-05-25 16:14:37 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 42). 2254 bytes result sent to driver -2016-05-25 16:14:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 41) in 39 ms on localhost (1/2) -2016-05-25 16:14:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 42) in 39 ms on localhost (2/2) -2016-05-25 16:14:37 INFO DAGScheduler:59 - ShuffleMapStage 22 (mapPartitions at DecisionTree.scala:625) finished in 0.040 s -2016-05-25 16:14:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-25 16:14:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:14:37 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:14:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 23) -2016-05-25 16:14:37 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:14:37 INFO DAGScheduler:59 - Missing parents for ResultStage 23: List() -2016-05-25 16:14:37 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[53] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(206280) called with curMem=3255987, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 201.4 KB, free 727.3 MB) -2016-05-25 16:14:37 INFO MemoryStore:59 - ensureFreeSpace(44075) called with curMem=3462267, maxMem=766075207 -2016-05-25 16:14:37 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 43.0 KB, free 727.2 MB) -2016-05-25 16:14:37 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:54170 (size: 43.0 KB, free: 729.6 MB) -2016-05-25 16:14:37 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[53] at map at DecisionTree.scala:645) -2016-05-25 16:14:37 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-25 16:14:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:14:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:14:37 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 44) -2016-05-25 16:14:37 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 43) -2016-05-25 16:14:37 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:14:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:14:37 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:14:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:54170 in memory (size: 115.3 KB, free: 729.7 MB) -2016-05-25 16:14:38 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-25 16:14:38 INFO ContextCleaner:59 - Cleaned shuffle 5 -2016-05-25 16:14:38 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 44). 2828 bytes result sent to driver -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:54170 in memory (size: 81.0 B, free: 729.7 MB) -2016-05-25 16:14:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 44) in 51 ms on localhost (1/2) -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:54170 in memory (size: 42.9 KB, free: 729.7 MB) -2016-05-25 16:14:38 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 43). 2928 bytes result sent to driver -2016-05-25 16:14:38 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-25 16:14:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 43) in 56 ms on localhost (2/2) -2016-05-25 16:14:38 INFO DAGScheduler:59 - ResultStage 23 (collectAsMap at DecisionTree.scala:654) finished in 0.057 s -2016-05-25 16:14:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-25 16:14:38 INFO DAGScheduler:59 - Job 16 finished: collectAsMap at DecisionTree.scala:654, took 0.138762 s -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:54170 in memory (size: 115.2 KB, free: 729.8 MB) -2016-05-25 16:14:38 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-25 16:14:38 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=2062773, maxMem=766075207 -2016-05-25 16:14:38 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 40.0 B, free 728.6 MB) -2016-05-25 16:14:38 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=2062813, maxMem=766075207 -2016-05-25 16:14:38 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 81.0 B, free 728.6 MB) -2016-05-25 16:14:38 INFO ContextCleaner:59 - Cleaned shuffle 4 -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:54170 (size: 81.0 B, free: 729.8 MB) -2016-05-25 16:14:38 INFO SparkContext:59 - Created broadcast 32 from broadcast at DecisionTree.scala:604 -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:54170 in memory (size: 81.0 B, free: 729.8 MB) -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:54170 in memory (size: 4.4 KB, free: 729.8 MB) -2016-05-25 16:14:38 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:54170 in memory (size: 4.1 KB, free: 729.8 MB) -2016-05-25 16:14:38 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:54170 in memory (size: 4.3 KB, free: 729.9 MB) -2016-05-25 16:14:38 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:54170 in memory (size: 42.9 KB, free: 729.9 MB) -2016-05-25 16:14:38 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-25 16:14:38 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-25 16:14:38 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:54170 in memory (size: 81.0 B, free: 729.9 MB) -2016-05-25 16:14:38 INFO BlockManager:59 - Removing RDD 26 -2016-05-25 16:14:38 INFO ContextCleaner:59 - Cleaned RDD 26 -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:54170 in memory (size: 115.5 KB, free: 730.0 MB) -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:54170 in memory (size: 43.0 KB, free: 730.1 MB) -2016-05-25 16:14:38 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-25 16:14:38 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:14:38 INFO DAGScheduler:59 - Registering RDD 54 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:14:38 INFO DAGScheduler:59 - Got job 17 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:14:38 INFO DAGScheduler:59 - Final stage: ResultStage 25(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:14:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 24) -2016-05-25 16:14:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 24) -2016-05-25 16:14:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 24 (MapPartitionsRDD[54] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:14:38 INFO MemoryStore:59 - ensureFreeSpace(479608) called with curMem=925505, maxMem=766075207 -2016-05-25 16:14:38 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 468.4 KB, free 729.2 MB) -2016-05-25 16:14:38 INFO MemoryStore:59 - ensureFreeSpace(118305) called with curMem=1405113, maxMem=766075207 -2016-05-25 16:14:38 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 115.5 KB, free 729.1 MB) -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:54170 (size: 115.5 KB, free: 729.9 MB) -2016-05-25 16:14:38 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 24 (MapPartitionsRDD[54] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:14:38 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-25 16:14:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 45, localhost, PROCESS_LOCAL, 2395 bytes) -2016-05-25 16:14:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 2395 bytes) -2016-05-25 16:14:38 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 46) -2016-05-25 16:14:38 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 45) -2016-05-25 16:14:38 INFO BlockManager:59 - Found block rdd_44_1 locally -2016-05-25 16:14:38 INFO BlockManager:59 - Found block rdd_44_0 locally -2016-05-25 16:14:38 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 45). 2254 bytes result sent to driver -2016-05-25 16:14:38 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 46). 2254 bytes result sent to driver -2016-05-25 16:14:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 45) in 21 ms on localhost (1/2) -2016-05-25 16:14:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 46) in 21 ms on localhost (2/2) -2016-05-25 16:14:38 INFO DAGScheduler:59 - ShuffleMapStage 24 (mapPartitions at DecisionTree.scala:625) finished in 0.022 s -2016-05-25 16:14:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-25 16:14:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:14:38 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:14:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 25) -2016-05-25 16:14:38 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:14:38 INFO DAGScheduler:59 - Missing parents for ResultStage 25: List() -2016-05-25 16:14:38 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[56] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:14:38 INFO MemoryStore:59 - ensureFreeSpace(206096) called with curMem=1523418, maxMem=766075207 -2016-05-25 16:14:38 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 201.3 KB, free 728.9 MB) -2016-05-25 16:14:38 INFO MemoryStore:59 - ensureFreeSpace(43967) called with curMem=1729514, maxMem=766075207 -2016-05-25 16:14:38 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 42.9 KB, free 728.9 MB) -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:54170 (size: 42.9 KB, free: 729.9 MB) -2016-05-25 16:14:38 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[56] at map at DecisionTree.scala:645) -2016-05-25 16:14:38 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-25 16:14:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:14:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:14:38 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 47) -2016-05-25 16:14:38 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 48) -2016-05-25 16:14:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:14:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:14:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:14:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:14:38 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 48). 1161 bytes result sent to driver -2016-05-25 16:14:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 48) in 7 ms on localhost (1/2) -2016-05-25 16:14:38 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 47). 2774 bytes result sent to driver -2016-05-25 16:14:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 47) in 19 ms on localhost (2/2) -2016-05-25 16:14:38 INFO DAGScheduler:59 - ResultStage 25 (collectAsMap at DecisionTree.scala:654) finished in 0.020 s -2016-05-25 16:14:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-25 16:14:38 INFO DAGScheduler:59 - Job 17 finished: collectAsMap at DecisionTree.scala:654, took 0.074038 s -2016-05-25 16:14:38 INFO MapPartitionsRDD:59 - Removing RDD 44 from persistence list -2016-05-25 16:14:38 INFO BlockManager:59 - Removing RDD 44 -2016-05-25 16:14:38 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 16:14:38 INFO RandomForest:59 - init: 0.3438235 - total: 1.037297593 - findSplitsBins: 0.104597643 - findBestSplits: 0.685172302 - chooseSplits: 0.684585978 -2016-05-25 16:14:38 INFO GradientBoostedTrees:59 - Internal timing for DecisionTree: -2016-05-25 16:14:38 INFO GradientBoostedTrees:59 - init: 0.081069287 - total: 5.420706864 - building tree 0: 3.099587119 - building tree 2: 1.03798476 - building tree 1: 1.087151614 -2016-05-25 16:14:38 INFO MapPartitionsRDD:59 - Removing RDD 9 from persistence list -2016-05-25 16:14:38 INFO BlockManager:59 - Removing RDD 9 -2016-05-25 16:14:38 INFO SparkContext:59 - Starting job: count at GBTs1.scala:57 -2016-05-25 16:14:38 INFO DAGScheduler:59 - Got job 18 (count at GBTs1.scala:57) with 2 output partitions -2016-05-25 16:14:38 INFO DAGScheduler:59 - Final stage: ResultStage 26(count at GBTs1.scala:57) -2016-05-25 16:14:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:14:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:14:38 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[60] at filter at GBTs1.scala:57), which has no missing parents -2016-05-25 16:14:38 INFO MemoryStore:59 - ensureFreeSpace(9040) called with curMem=1447377, maxMem=766075207 -2016-05-25 16:14:38 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 8.8 KB, free 729.2 MB) -2016-05-25 16:14:38 INFO MemoryStore:59 - ensureFreeSpace(4418) called with curMem=1456417, maxMem=766075207 -2016-05-25 16:14:38 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 4.3 KB, free 729.2 MB) -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:54170 (size: 4.3 KB, free: 730.2 MB) -2016-05-25 16:14:38 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[60] at filter at GBTs1.scala:57) -2016-05-25 16:14:38 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-25 16:14:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:14:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:14:38 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-25 16:14:38 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-25 16:14:38 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 16:14:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:14:38 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 16:14:38 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 16:14:38 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 2082 bytes result sent to driver -2016-05-25 16:14:38 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 2082 bytes result sent to driver -2016-05-25 16:14:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 12 ms on localhost (1/2) -2016-05-25 16:14:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 13 ms on localhost (2/2) -2016-05-25 16:14:38 INFO DAGScheduler:59 - ResultStage 26 (count at GBTs1.scala:57) finished in 0.014 s -2016-05-25 16:14:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-25 16:14:38 INFO DAGScheduler:59 - Job 18 finished: count at GBTs1.scala:57, took 0.022698 s -2016-05-25 16:14:38 INFO SparkContext:59 - Starting job: count at GBTs1.scala:57 -2016-05-25 16:14:38 INFO DAGScheduler:59 - Got job 19 (count at GBTs1.scala:57) with 2 output partitions -2016-05-25 16:14:38 INFO DAGScheduler:59 - Final stage: ResultStage 27(count at GBTs1.scala:57) -2016-05-25 16:14:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:14:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:14:38 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[8] at randomSplit at GBTs1.scala:32), which has no missing parents -2016-05-25 16:14:38 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1460835, maxMem=766075207 -2016-05-25 16:14:38 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 4.0 KB, free 729.2 MB) -2016-05-25 16:14:38 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=1464947, maxMem=766075207 -2016-05-25 16:14:38 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.2 MB) -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:54170 (size: 2.2 KB, free: 730.2 MB) -2016-05-25 16:14:38 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[8] at randomSplit at GBTs1.scala:32) -2016-05-25 16:14:38 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-25 16:14:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 51, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:14:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 52, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:14:38 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 51) -2016-05-25 16:14:38 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 52) -2016-05-25 16:14:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:14:38 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 16:14:38 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 51). 2082 bytes result sent to driver -2016-05-25 16:14:38 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 52). 2082 bytes result sent to driver -2016-05-25 16:14:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 51) in 6 ms on localhost (1/2) -2016-05-25 16:14:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 52) in 5 ms on localhost (2/2) -2016-05-25 16:14:38 INFO DAGScheduler:59 - ResultStage 27 (count at GBTs1.scala:57) finished in 0.006 s -2016-05-25 16:14:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-25 16:14:38 INFO DAGScheduler:59 - Job 19 finished: count at GBTs1.scala:57, took 0.014616 s -2016-05-25 16:14:38 INFO SparkContext:59 - Starting job: count at GBTs1.scala:63 -2016-05-25 16:14:38 INFO DAGScheduler:59 - Got job 20 (count at GBTs1.scala:63) with 2 output partitions -2016-05-25 16:14:38 INFO DAGScheduler:59 - Final stage: ResultStage 28(count at GBTs1.scala:63) -2016-05-25 16:14:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:14:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:14:38 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 16:14:38 INFO MemoryStore:59 - ensureFreeSpace(3816) called with curMem=1467215, maxMem=766075207 -2016-05-25 16:14:38 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 3.7 KB, free 729.2 MB) -2016-05-25 16:14:38 INFO MemoryStore:59 - ensureFreeSpace(2142) called with curMem=1471031, maxMem=766075207 -2016-05-25 16:14:38 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.2 MB) -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:54170 (size: 2.1 KB, free: 730.2 MB) -2016-05-25 16:14:38 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 16:14:38 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-25 16:14:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 53, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:14:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 54, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:14:38 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 53) -2016-05-25 16:14:38 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 54) -2016-05-25 16:14:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:14:38 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 16:14:38 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 53). 2082 bytes result sent to driver -2016-05-25 16:14:38 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 54). 2082 bytes result sent to driver -2016-05-25 16:14:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 53) in 5 ms on localhost (1/2) -2016-05-25 16:14:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 54) in 5 ms on localhost (2/2) -2016-05-25 16:14:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-25 16:14:38 INFO DAGScheduler:59 - ResultStage 28 (count at GBTs1.scala:63) finished in 0.006 s -2016-05-25 16:14:38 INFO DAGScheduler:59 - Job 20 finished: count at GBTs1.scala:63, took 0.014991 s -2016-05-25 16:14:38 INFO SparkContext:59 - Starting job: count at GBTs1.scala:64 -2016-05-25 16:14:38 INFO DAGScheduler:59 - Got job 21 (count at GBTs1.scala:64) with 2 output partitions -2016-05-25 16:14:38 INFO DAGScheduler:59 - Final stage: ResultStage 29(count at GBTs1.scala:64) -2016-05-25 16:14:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:14:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:14:38 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[7] at randomSplit at GBTs1.scala:32), which has no missing parents -2016-05-25 16:14:38 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1473173, maxMem=766075207 -2016-05-25 16:14:38 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 4.0 KB, free 729.2 MB) -2016-05-25 16:14:38 INFO MemoryStore:59 - ensureFreeSpace(2265) called with curMem=1477285, maxMem=766075207 -2016-05-25 16:14:38 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.2 MB) -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:54170 (size: 2.2 KB, free: 730.2 MB) -2016-05-25 16:14:38 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[7] at randomSplit at GBTs1.scala:32) -2016-05-25 16:14:38 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-25 16:14:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 55, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:14:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 56, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:14:38 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 56) -2016-05-25 16:14:38 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 55) -2016-05-25 16:14:38 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 16:14:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:14:38 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 56). 2082 bytes result sent to driver -2016-05-25 16:14:38 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 55). 2082 bytes result sent to driver -2016-05-25 16:14:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 56) in 4 ms on localhost (1/2) -2016-05-25 16:14:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 55) in 6 ms on localhost (2/2) -2016-05-25 16:14:38 INFO DAGScheduler:59 - ResultStage 29 (count at GBTs1.scala:64) finished in 0.006 s -2016-05-25 16:14:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-25 16:14:38 INFO DAGScheduler:59 - Job 21 finished: count at GBTs1.scala:64, took 0.014332 s -2016-05-25 16:14:38 INFO SparkContext:59 - Starting job: count at GBTs1.scala:65 -2016-05-25 16:14:38 INFO DAGScheduler:59 - Got job 22 (count at GBTs1.scala:65) with 2 output partitions -2016-05-25 16:14:38 INFO DAGScheduler:59 - Final stage: ResultStage 30(count at GBTs1.scala:65) -2016-05-25 16:14:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:14:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:14:38 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[8] at randomSplit at GBTs1.scala:32), which has no missing parents -2016-05-25 16:14:38 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=1479550, maxMem=766075207 -2016-05-25 16:14:38 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 4.0 KB, free 729.2 MB) -2016-05-25 16:14:38 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=1483662, maxMem=766075207 -2016-05-25 16:14:38 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.2 MB) -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:54170 (size: 2.2 KB, free: 730.2 MB) -2016-05-25 16:14:38 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[8] at randomSplit at GBTs1.scala:32) -2016-05-25 16:14:38 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-25 16:14:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:14:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 58, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:14:38 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 57) -2016-05-25 16:14:38 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 58) -2016-05-25 16:14:38 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 16:14:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:14:38 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 58). 2082 bytes result sent to driver -2016-05-25 16:14:38 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 57). 2082 bytes result sent to driver -2016-05-25 16:14:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 58) in 7 ms on localhost (1/2) -2016-05-25 16:14:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 57) in 8 ms on localhost (2/2) -2016-05-25 16:14:38 INFO DAGScheduler:59 - ResultStage 30 (count at GBTs1.scala:65) finished in 0.009 s -2016-05-25 16:14:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-25 16:14:38 INFO DAGScheduler:59 - Job 22 finished: count at GBTs1.scala:65, took 0.021470 s -2016-05-25 16:14:38 INFO SparkContext:59 - Starting job: take at GBTs1.scala:71 -2016-05-25 16:14:38 INFO DAGScheduler:59 - Got job 23 (take at GBTs1.scala:71) with 1 output partitions -2016-05-25 16:14:38 INFO DAGScheduler:59 - Final stage: ResultStage 31(take at GBTs1.scala:71) -2016-05-25 16:14:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:14:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:14:38 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[59] at map at GBTs1.scala:53), which has no missing parents -2016-05-25 16:14:38 INFO MemoryStore:59 - ensureFreeSpace(9112) called with curMem=1485930, maxMem=766075207 -2016-05-25 16:14:38 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 8.9 KB, free 729.2 MB) -2016-05-25 16:14:38 INFO MemoryStore:59 - ensureFreeSpace(4460) called with curMem=1495042, maxMem=766075207 -2016-05-25 16:14:38 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 4.4 KB, free 729.2 MB) -2016-05-25 16:14:38 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:54170 (size: 4.4 KB, free: 730.2 MB) -2016-05-25 16:14:38 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:38 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 31 (MapPartitionsRDD[59] at map at GBTs1.scala:53) -2016-05-25 16:14:38 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 1 tasks -2016-05-25 16:14:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 59, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:14:38 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 59) -2016-05-25 16:14:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:14:38 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 59). 2401 bytes result sent to driver -2016-05-25 16:14:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 59) in 7 ms on localhost (1/1) -2016-05-25 16:14:38 INFO DAGScheduler:59 - ResultStage 31 (take at GBTs1.scala:71) finished in 0.007 s -2016-05-25 16:14:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-25 16:14:38 INFO DAGScheduler:59 - Job 23 finished: take at GBTs1.scala:71, took 0.018674 s -2016-05-25 16:14:38 INFO SparkContext:59 - Starting job: saveAsTextFile at treeEnsembleModels.scala:448 -2016-05-25 16:14:38 INFO DAGScheduler:59 - Got job 24 (saveAsTextFile at treeEnsembleModels.scala:448) with 1 output partitions -2016-05-25 16:14:38 INFO DAGScheduler:59 - Final stage: ResultStage 32(saveAsTextFile at treeEnsembleModels.scala:448) -2016-05-25 16:14:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:14:38 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:14:38 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[62] at saveAsTextFile at treeEnsembleModels.scala:448), which has no missing parents -2016-05-25 16:14:39 INFO MemoryStore:59 - ensureFreeSpace(127016) called with curMem=1499502, maxMem=766075207 -2016-05-25 16:14:39 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 124.0 KB, free 729.0 MB) -2016-05-25 16:14:39 INFO MemoryStore:59 - ensureFreeSpace(42641) called with curMem=1626518, maxMem=766075207 -2016-05-25 16:14:39 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 41.6 KB, free 729.0 MB) -2016-05-25 16:14:39 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:54170 (size: 41.6 KB, free: 730.1 MB) -2016-05-25 16:14:39 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:39 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 32 (MapPartitionsRDD[62] at saveAsTextFile at treeEnsembleModels.scala:448) -2016-05-25 16:14:39 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 1 tasks -2016-05-25 16:14:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 60, localhost, PROCESS_LOCAL, 2292 bytes) -2016-05-25 16:14:39 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 60) -2016-05-25 16:14:39 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251614_0032_m_000000_60' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/GradientBoostedTreesModel20160525161438316/result/metadata/_temporary/0/task_201605251614_0032_m_000000 -2016-05-25 16:14:39 INFO SparkHadoopMapRedUtil:59 - attempt_201605251614_0032_m_000000_60: Committed -2016-05-25 16:14:39 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 60). 1864 bytes result sent to driver -2016-05-25 16:14:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 60) in 305 ms on localhost (1/1) -2016-05-25 16:14:39 INFO DAGScheduler:59 - ResultStage 32 (saveAsTextFile at treeEnsembleModels.scala:448) finished in 0.305 s -2016-05-25 16:14:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-25 16:14:39 INFO DAGScheduler:59 - Job 24 finished: saveAsTextFile at treeEnsembleModels.scala:448, took 0.399406 s -2016-05-25 16:14:40 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:14:40 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:14:41 INFO SparkContext:59 - Starting job: parquet at treeEnsembleModels.scala:454 -2016-05-25 16:14:41 INFO DAGScheduler:59 - Got job 25 (parquet at treeEnsembleModels.scala:454) with 4 output partitions -2016-05-25 16:14:41 INFO DAGScheduler:59 - Final stage: ResultStage 33(parquet at treeEnsembleModels.scala:454) -2016-05-25 16:14:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:14:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:14:41 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[65] at rddToDataFrameHolder at treeEnsembleModels.scala:451), which has no missing parents -2016-05-25 16:14:41 INFO MemoryStore:59 - ensureFreeSpace(68584) called with curMem=1669159, maxMem=766075207 -2016-05-25 16:14:41 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 67.0 KB, free 728.9 MB) -2016-05-25 16:14:41 INFO MemoryStore:59 - ensureFreeSpace(24137) called with curMem=1737743, maxMem=766075207 -2016-05-25 16:14:41 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 23.6 KB, free 728.9 MB) -2016-05-25 16:14:41 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:54170 (size: 23.6 KB, free: 730.1 MB) -2016-05-25 16:14:41 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:41 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[65] at rddToDataFrameHolder at treeEnsembleModels.scala:451) -2016-05-25 16:14:41 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-05-25 16:14:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 61, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 16:14:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 62, localhost, PROCESS_LOCAL, 4220 bytes) -2016-05-25 16:14:41 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 63, localhost, PROCESS_LOCAL, 4539 bytes) -2016-05-25 16:14:41 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 64, localhost, PROCESS_LOCAL, 5205 bytes) -2016-05-25 16:14:41 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 62) -2016-05-25 16:14:41 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 61) -2016-05-25 16:14:41 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 63) -2016-05-25 16:14:41 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 64) -2016-05-25 16:14:41 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:14:41 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 16:14:41 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:14:41 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 16:14:41 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:14:41 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 16:14:41 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:14:41 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 16:14:41 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 16:14:41 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 16:14:41 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 16:14:41 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 16:14:41 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 16:14:41 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,568 -2016-05-25 16:14:41 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,748 -2016-05-25 16:14:41 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,892 -2016-05-25 16:14:41 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 66,568 -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 3 values, 8B raw, 28B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 9 values, 8B raw, 28B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 61B for [treeId] INT32: 5 values, 8B raw, 28B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 66B for [nodeId] INT32: 3 values, 18B raw, 33B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 80B for [nodeId] INT32: 9 values, 42B raw, 47B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 84B for [predict, predict] DOUBLE: 3 values, 31B raw, 43B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 75B for [predict, predict] DOUBLE: 9 values, 14B raw, 34B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 6 entries, 48B raw, 6B comp} -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 68B for [predict, prob] DOUBLE: 3 values, 9B raw, 27B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 1 entries, 8B raw, 1B comp} -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 69B for [predict, prob] DOUBLE: 9 values, 8B raw, 28B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 1 entries, 8B raw, 1B comp} -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 75B for [impurity] DOUBLE: 9 values, 14B raw, 34B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 6 entries, 48B raw, 6B comp} -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 70B for [impurity] DOUBLE: 3 values, 9B raw, 29B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 2 entries, 16B raw, 2B comp} -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 3 values, 7B raw, 27B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 64B for [split, feature] INT32: 3 values, 11B raw, 31B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 69B for [split, threshold] DOUBLE: 3 values, 15B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 61B for [split, featureType] INT32: 3 values, 11B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 52B for [split, categories, bag, array] DOUBLE: 3 values, 14B raw, 31B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 61B for [leftNodeId] INT32: 3 values, 10B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 61B for [rightNodeId] INT32: 3 values, 10B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 75B for [infoGain] DOUBLE: 3 values, 14B raw, 34B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 71B for [nodeId] INT32: 5 values, 26B raw, 38B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 55B for [isLeaf] BOOLEAN: 9 values, 8B raw, 28B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 94B for [predict, predict] DOUBLE: 5 values, 47B raw, 53B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 74B for [split, feature] INT32: 9 values, 25B raw, 41B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 74B for [split, threshold] DOUBLE: 9 values, 13B raw, 33B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 3 entries, 24B raw, 3B comp} -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 62B for [split, featureType] INT32: 9 values, 11B raw, 29B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 55B for [split, categories, bag, array] DOUBLE: 9 values, 17B raw, 34B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 70B for [predict, prob] DOUBLE: 5 values, 9B raw, 29B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 1 entries, 8B raw, 1B comp} -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 70B for [leftNodeId] INT32: 9 values, 23B raw, 37B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 69B for [rightNodeId] INT32: 9 values, 23B raw, 36B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 71B for [impurity] DOUBLE: 5 values, 10B raw, 30B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 4 entries, 32B raw, 4B comp} -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 54B for [isLeaf] BOOLEAN: 5 values, 7B raw, 27B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 73B for [infoGain] DOUBLE: 9 values, 12B raw, 32B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 5 entries, 40B raw, 5B comp} -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 66B for [split, feature] INT32: 5 values, 15B raw, 33B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 76B for [split, threshold] DOUBLE: 5 values, 23B raw, 35B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 60B for [split, featureType] INT32: 5 values, 9B raw, 27B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 1 entries, 4B raw, 1B comp} -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 52B for [split, categories, bag, array] DOUBLE: 5 values, 14B raw, 31B comp, 1 pages, encodings: [PLAIN, RLE] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 64B for [leftNodeId] INT32: 5 values, 14B raw, 31B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 64B for [rightNodeId] INT32: 5 values, 14B raw, 31B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO ColumnChunkPageWriteStore:151 - written 87B for [infoGain] DOUBLE: 5 values, 30B raw, 46B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:14:41 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251614_0033_m_000002_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/GradientBoostedTreesModel20160525161438316/result/data/_temporary/0/task_201605251614_0033_m_000002 -2016-05-25 16:14:41 INFO SparkHadoopMapRedUtil:59 - attempt_201605251614_0033_m_000002_0: Committed -2016-05-25 16:14:41 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 63). 915 bytes result sent to driver -2016-05-25 16:14:41 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251614_0033_m_000001_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/GradientBoostedTreesModel20160525161438316/result/data/_temporary/0/task_201605251614_0033_m_000001 -2016-05-25 16:14:41 INFO SparkHadoopMapRedUtil:59 - attempt_201605251614_0033_m_000001_0: Committed -2016-05-25 16:14:41 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 62). 915 bytes result sent to driver -2016-05-25 16:14:41 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 63) in 608 ms on localhost (1/4) -2016-05-25 16:14:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 62) in 609 ms on localhost (2/4) -2016-05-25 16:14:41 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251614_0033_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/GradientBoostedTreesModel20160525161438316/result/data/_temporary/0/task_201605251614_0033_m_000000 -2016-05-25 16:14:41 INFO SparkHadoopMapRedUtil:59 - attempt_201605251614_0033_m_000000_0: Committed -2016-05-25 16:14:41 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 61). 915 bytes result sent to driver -2016-05-25 16:14:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 61) in 622 ms on localhost (3/4) -2016-05-25 16:14:41 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251614_0033_m_000003_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/GradientBoostedTreesModel20160525161438316/result/data/_temporary/0/task_201605251614_0033_m_000003 -2016-05-25 16:14:41 INFO SparkHadoopMapRedUtil:59 - attempt_201605251614_0033_m_000003_0: Committed -2016-05-25 16:14:41 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 64). 915 bytes result sent to driver -2016-05-25 16:14:41 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 64) in 629 ms on localhost (4/4) -2016-05-25 16:14:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-25 16:14:41 INFO DAGScheduler:59 - ResultStage 33 (parquet at treeEnsembleModels.scala:454) finished in 0.632 s -2016-05-25 16:14:41 INFO DAGScheduler:59 - Job 25 finished: parquet at treeEnsembleModels.scala:454, took 0.675897 s -2016-05-25 16:14:41 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 16:14:41 INFO DefaultWriterContainer:59 - Job job_201605251614_0000 committed. -2016-05-25 16:14:41 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/GradientBoostedTreesModel20160525161438316/result/data on driver -2016-05-25 16:14:41 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/GradientBoostedTreesModel20160525161438316/result/data on driver -2016-05-25 16:14:41 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=1761880, maxMem=766075207 -2016-05-25 16:14:41 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 225.3 KB, free 728.7 MB) -2016-05-25 16:14:42 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1992584, maxMem=766075207 -2016-05-25 16:14:42 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 19.3 KB, free 728.7 MB) -2016-05-25 16:14:42 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:54170 (size: 19.3 KB, free: 730.1 MB) -2016-05-25 16:14:42 INFO SparkContext:59 - Created broadcast 43 from textFile at modelSaveLoad.scala:129 -2016-05-25 16:14:42 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:54170 in memory (size: 23.6 KB, free: 730.1 MB) -2016-05-25 16:14:42 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-05-25 16:14:42 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:54170 in memory (size: 41.6 KB, free: 730.2 MB) -2016-05-25 16:14:42 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-05-25 16:14:42 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:54170 in memory (size: 4.4 KB, free: 730.2 MB) -2016-05-25 16:14:42 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-05-25 16:14:42 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:54170 in memory (size: 2.2 KB, free: 730.2 MB) -2016-05-25 16:14:42 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-05-25 16:14:42 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:54170 in memory (size: 2.2 KB, free: 730.2 MB) -2016-05-25 16:14:42 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-05-25 16:14:42 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:54170 in memory (size: 2.1 KB, free: 730.2 MB) -2016-05-25 16:14:42 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-05-25 16:14:42 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:54170 in memory (size: 2.2 KB, free: 730.2 MB) -2016-05-25 16:14:42 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-05-25 16:14:42 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:54170 in memory (size: 4.3 KB, free: 730.2 MB) -2016-05-25 16:14:42 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-05-25 16:14:42 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:54170 in memory (size: 42.9 KB, free: 730.2 MB) -2016-05-25 16:14:42 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-05-25 16:14:42 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:54170 in memory (size: 115.5 KB, free: 730.3 MB) -2016-05-25 16:14:42 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-05-25 16:14:42 INFO ContextCleaner:59 - Cleaned shuffle 7 -2016-05-25 16:14:42 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:54170 in memory (size: 81.0 B, free: 730.3 MB) -2016-05-25 16:14:42 INFO BlockManager:59 - Removing RDD 58 -2016-05-25 16:14:42 INFO ContextCleaner:59 - Cleaned RDD 58 -2016-05-25 16:14:42 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:54170 in memory (size: 43.0 KB, free: 730.4 MB) -2016-05-25 16:14:42 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-25 16:14:42 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-25 16:14:42 INFO ContextCleaner:59 - Cleaned shuffle 6 -2016-05-25 16:14:42 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:54170 in memory (size: 81.0 B, free: 730.4 MB) -2016-05-25 16:14:42 INFO BlockManager:59 - Removing RDD 44 -2016-05-25 16:14:42 INFO ContextCleaner:59 - Cleaned RDD 44 -2016-05-25 16:14:42 INFO BlockManager:59 - Removing RDD 37 -2016-05-25 16:14:42 INFO ContextCleaner:59 - Cleaned RDD 37 -2016-05-25 16:14:42 INFO BlockManager:59 - Removing RDD 18 -2016-05-25 16:14:42 INFO ContextCleaner:59 - Cleaned RDD 18 -2016-05-25 16:14:42 INFO BlockManager:59 - Removing RDD 9 -2016-05-25 16:14:42 INFO ContextCleaner:59 - Cleaned RDD 9 -2016-05-25 16:14:42 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 16:14:42 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 16:14:42 INFO DAGScheduler:59 - Got job 26 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 16:14:42 INFO DAGScheduler:59 - Final stage: ResultStage 34(first at modelSaveLoad.scala:129) -2016-05-25 16:14:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:14:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:14:42 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[68] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 16:14:42 INFO MemoryStore:59 - ensureFreeSpace(3224) called with curMem=592800, maxMem=766075207 -2016-05-25 16:14:42 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 3.1 KB, free 730.0 MB) -2016-05-25 16:14:42 INFO MemoryStore:59 - ensureFreeSpace(1899) called with curMem=596024, maxMem=766075207 -2016-05-25 16:14:42 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 1899.0 B, free 730.0 MB) -2016-05-25 16:14:42 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:54170 (size: 1899.0 B, free: 730.4 MB) -2016-05-25 16:14:42 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:14:42 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 34 (MapPartitionsRDD[68] at textFile at modelSaveLoad.scala:129) -2016-05-25 16:14:42 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 1 tasks -2016-05-25 16:14:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2249 bytes) -2016-05-25 16:14:42 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 65) -2016-05-25 16:14:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/GradientBoostedTreesModel20160525161438316/result/metadata/part-00000:0+101 -2016-05-25 16:14:42 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 65). 2252 bytes result sent to driver -2016-05-25 16:14:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 65) in 9 ms on localhost (1/1) -2016-05-25 16:14:42 INFO DAGScheduler:59 - ResultStage 34 (first at modelSaveLoad.scala:129) finished in 0.009 s -2016-05-25 16:14:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-25 16:14:42 INFO DAGScheduler:59 - Job 26 finished: first at modelSaveLoad.scala:129, took 0.017773 s -2016-05-25 16:14:42 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 16:14:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 16:14:42 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 16:14:43 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:54170 in memory (size: 1899.0 B, free: 730.4 MB) -2016-05-25 16:14:43 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 16:14:43 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 16:14:43 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 16:14:43 INFO BlockManager:59 - BlockManager stopped -2016-05-25 16:14:43 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 16:14:43 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 16:14:43 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 16:14:43 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 16:14:43 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-cac856dd-f963-4523-b240-5514c7b422c3 -2016-05-25 16:14:43 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 16:15:03 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 16:15:04 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 16:15:04 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 16:15:04 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 16:15:04 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 16:15:05 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 16:15:05 INFO Remoting:74 - Starting remoting -2016-05-25 16:15:05 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:54224] -2016-05-25 16:15:05 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54224. -2016-05-25 16:15:05 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 16:15:05 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 16:15:05 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-71a1baac-4f82-468a-a9bf-10052de6b93a -2016-05-25 16:15:05 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 16:15:05 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-9778b023-539b-499f-a1ed-289d549f94a4\httpd-635bffdf-24d3-4c88-b350-0cd8674f40a3 -2016-05-25 16:15:05 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 16:15:06 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 16:15:06 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54225 -2016-05-25 16:15:06 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54225. -2016-05-25 16:15:06 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 16:15:06 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 16:15:06 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 16:15:06 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 16:15:06 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 16:15:06 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 16:15:06 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 16:15:06 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54244. -2016-05-25 16:15:06 INFO NettyBlockTransferService:59 - Server created on 54244 -2016-05-25 16:15:06 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 16:15:06 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54244 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54244) -2016-05-25 16:15:06 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 16:15:07 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 16:15:07 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 16:15:07 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 16:15:07 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 16:15:07 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54244 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 16:15:07 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 16:15:08 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 16:15:09 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 16:15:09 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 16:15:09 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 16:15:09 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 16:15:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:15:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:15:09 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 16:15:09 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 16:15:09 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 16:15:09 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 16:15:09 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 16:15:09 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54244 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 16:15:09 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 16:15:09 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 16:15:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:15:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:15:09 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 16:15:09 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 16:15:09 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 16:15:09 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 16:15:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 16:15:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 16:15:09 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 16:15:09 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 16:15:09 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 16:15:09 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 16:15:09 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 16:15:09 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177630, maxMem=766075207 -2016-05-25 16:15:09 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 16:15:09 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263278, maxMem=766075207 -2016-05-25 16:15:09 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:54244 (size: 83.6 KB, free: 730.5 MB) -2016-05-25 16:15:09 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 16:15:09 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:54244 (size: 83.1 KB, free: 730.4 MB) -2016-05-25 16:15:09 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 16:15:09 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 16:15:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 414 ms on localhost (1/2) -2016-05-25 16:15:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 403 ms on localhost (2/2) -2016-05-25 16:15:09 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.443 s -2016-05-25 16:15:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 16:15:09 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.551623 s -2016-05-25 16:15:09 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 16:15:09 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 16:15:09 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 16:15:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:15:09 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:15:09 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[11] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 16:15:09 INFO MemoryStore:59 - ensureFreeSpace(4912) called with curMem=348358, maxMem=766075207 -2016-05-25 16:15:09 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-05-25 16:15:09 INFO MemoryStore:59 - ensureFreeSpace(2601) called with curMem=353270, maxMem=766075207 -2016-05-25 16:15:09 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 16:15:09 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54244 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 16:15:09 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:09 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[11] at map at DecisionTreeMetadata.scala:110) -2016-05-25 16:15:09 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 16:15:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:15:09 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 16:15:10 INFO CacheManager:59 - Partition rdd_9_0 not found, computing it -2016-05-25 16:15:10 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:15:10 INFO MemoryStore:59 - ensureFreeSpace(61528) called with curMem=355871, maxMem=766075207 -2016-05-25 16:15:10 INFO MemoryStore:59 - Block rdd_9_0 stored as values in memory (estimated size 60.1 KB, free 730.2 MB) -2016-05-25 16:15:10 INFO BlockManagerInfo:59 - Added rdd_9_0 in memory on localhost:54244 (size: 60.1 KB, free: 730.3 MB) -2016-05-25 16:15:10 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2611 bytes result sent to driver -2016-05-25 16:15:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 50 ms on localhost (1/1) -2016-05-25 16:15:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 16:15:10 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.051 s -2016-05-25 16:15:10 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.078061 s -2016-05-25 16:15:10 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 16:15:10 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 16:15:10 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 16:15:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:15:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:15:10 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[10] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 16:15:10 INFO MemoryStore:59 - ensureFreeSpace(4480) called with curMem=417399, maxMem=766075207 -2016-05-25 16:15:10 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.4 KB, free 730.2 MB) -2016-05-25 16:15:10 INFO MemoryStore:59 - ensureFreeSpace(2382) called with curMem=421879, maxMem=766075207 -2016-05-25 16:15:10 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 16:15:10 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54244 (size: 2.3 KB, free: 730.3 MB) -2016-05-25 16:15:10 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[10] at retag at RandomForest.scala:137) -2016-05-25 16:15:10 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 16:15:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:15:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:15:10 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 16:15:10 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 16:15:10 INFO CacheManager:59 - Partition rdd_9_1 not found, computing it -2016-05-25 16:15:10 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 16:15:10 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:15:10 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 16:15:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 19 ms on localhost (1/2) -2016-05-25 16:15:10 INFO MemoryStore:59 - ensureFreeSpace(60744) called with curMem=424261, maxMem=766075207 -2016-05-25 16:15:10 INFO MemoryStore:59 - Block rdd_9_1 stored as values in memory (estimated size 59.3 KB, free 730.1 MB) -2016-05-25 16:15:10 INFO BlockManagerInfo:59 - Added rdd_9_1 in memory on localhost:54244 (size: 59.3 KB, free: 730.3 MB) -2016-05-25 16:15:10 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2662 bytes result sent to driver -2016-05-25 16:15:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 31 ms on localhost (2/2) -2016-05-25 16:15:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 16:15:10 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.036 s -2016-05-25 16:15:10 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.053938 s -2016-05-25 16:15:10 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 16:15:10 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 16:15:10 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 16:15:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:15:10 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:15:10 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[12] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 16:15:10 INFO MemoryStore:59 - ensureFreeSpace(5208) called with curMem=485005, maxMem=766075207 -2016-05-25 16:15:10 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.1 KB, free 730.1 MB) -2016-05-25 16:15:10 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=490213, maxMem=766075207 -2016-05-25 16:15:10 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.1 MB) -2016-05-25 16:15:10 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54244 (size: 2.6 KB, free: 730.3 MB) -2016-05-25 16:15:10 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[12] at sample at DecisionTree.scala:989) -2016-05-25 16:15:10 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 16:15:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 16:15:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 16:15:10 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 16:15:10 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 16:15:10 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-25 16:15:10 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:15:10 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 62623 bytes result sent to driver -2016-05-25 16:15:10 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 61832 bytes result sent to driver -2016-05-25 16:15:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 29 ms on localhost (1/2) -2016-05-25 16:15:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 32 ms on localhost (2/2) -2016-05-25 16:15:10 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.032 s -2016-05-25 16:15:10 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.051783 s -2016-05-25 16:15:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 16:15:10 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:54244 in memory (size: 2.6 KB, free: 730.3 MB) -2016-05-25 16:15:10 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 16:15:10 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:54244 in memory (size: 2.3 KB, free: 730.3 MB) -2016-05-25 16:15:10 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 16:15:10 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:54244 in memory (size: 2.5 KB, free: 730.3 MB) -2016-05-25 16:15:10 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 16:15:10 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:54244 in memory (size: 2.1 KB, free: 730.3 MB) -2016-05-25 16:15:10 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-05-25 16:15:11 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=464580, maxMem=766075207 -2016-05-25 16:15:11 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.1 MB) -2016-05-25 16:15:11 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=464620, maxMem=766075207 -2016-05-25 16:15:11 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.1 MB) -2016-05-25 16:15:11 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:54244 (size: 81.0 B, free: 730.3 MB) -2016-05-25 16:15:11 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 16:15:11 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:15:11 INFO DAGScheduler:59 - Registering RDD 15 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:11 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:15:11 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:15:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 16:15:11 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 16:15:11 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[15] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:15:11 INFO MemoryStore:59 - ensureFreeSpace(481240) called with curMem=464701, maxMem=766075207 -2016-05-25 16:15:11 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 470.0 KB, free 729.7 MB) -2016-05-25 16:15:11 INFO MemoryStore:59 - ensureFreeSpace(117983) called with curMem=945941, maxMem=766075207 -2016-05-25 16:15:11 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 115.2 KB, free 729.6 MB) -2016-05-25 16:15:11 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:54244 (size: 115.2 KB, free: 730.2 MB) -2016-05-25 16:15:11 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[15] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:11 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 16:15:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 16:15:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 16:15:11 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 16:15:11 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 16:15:12 INFO CacheManager:59 - Partition rdd_14_0 not found, computing it -2016-05-25 16:15:12 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:15:12 INFO CacheManager:59 - Partition rdd_14_1 not found, computing it -2016-05-25 16:15:12 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-25 16:15:12 INFO MemoryStore:59 - ensureFreeSpace(100120) called with curMem=1063924, maxMem=766075207 -2016-05-25 16:15:12 INFO MemoryStore:59 - Block rdd_14_1 stored as values in memory (estimated size 97.8 KB, free 729.5 MB) -2016-05-25 16:15:12 INFO BlockManagerInfo:59 - Added rdd_14_1 in memory on localhost:54244 (size: 97.8 KB, free: 730.1 MB) -2016-05-25 16:15:12 INFO MemoryStore:59 - ensureFreeSpace(102976) called with curMem=1164044, maxMem=766075207 -2016-05-25 16:15:12 INFO MemoryStore:59 - Block rdd_14_0 stored as values in memory (estimated size 100.6 KB, free 729.4 MB) -2016-05-25 16:15:12 INFO BlockManagerInfo:59 - Added rdd_14_0 in memory on localhost:54244 (size: 100.6 KB, free: 730.0 MB) -2016-05-25 16:15:12 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 16:15:12 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 16:15:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 541 ms on localhost (1/2) -2016-05-25 16:15:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 545 ms on localhost (2/2) -2016-05-25 16:15:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 16:15:12 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.545 s -2016-05-25 16:15:12 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:15:12 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:15:12 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 16:15:12 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:15:12 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 16:15:12 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[17] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:15:12 INFO MemoryStore:59 - ensureFreeSpace(208696) called with curMem=1267020, maxMem=766075207 -2016-05-25 16:15:12 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 203.8 KB, free 729.2 MB) -2016-05-25 16:15:12 INFO MemoryStore:59 - ensureFreeSpace(44514) called with curMem=1475716, maxMem=766075207 -2016-05-25 16:15:12 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 43.5 KB, free 729.1 MB) -2016-05-25 16:15:12 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:54244 (size: 43.5 KB, free: 729.9 MB) -2016-05-25 16:15:12 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[17] at map at DecisionTree.scala:645) -2016-05-25 16:15:12 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 16:15:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:12 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 16:15:12 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 16:15:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:15:12 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:15:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 9 ms -2016-05-25 16:15:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 10 ms -2016-05-25 16:15:12 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 16:15:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 265 ms on localhost (1/2) -2016-05-25 16:15:12 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 16:15:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 514 ms on localhost (2/2) -2016-05-25 16:15:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 16:15:12 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.515 s -2016-05-25 16:15:12 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 1.140017 s -2016-05-25 16:15:12 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=1520230, maxMem=766075207 -2016-05-25 16:15:12 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 40.0 B, free 729.1 MB) -2016-05-25 16:15:12 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=1520270, maxMem=766075207 -2016-05-25 16:15:12 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 81.0 B, free 729.1 MB) -2016-05-25 16:15:12 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:54244 (size: 81.0 B, free: 729.9 MB) -2016-05-25 16:15:12 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 16:15:13 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:15:13 INFO DAGScheduler:59 - Registering RDD 18 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:13 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:15:13 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:15:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 16:15:13 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[18] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(481592) called with curMem=1520351, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 470.3 KB, free 728.7 MB) -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(118182) called with curMem=2001943, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 115.4 KB, free 728.6 MB) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:54244 (size: 115.4 KB, free: 729.8 MB) -2016-05-25 16:15:13 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[18] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:13 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 16:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 16:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 16:15:13 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 16:15:13 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 16:15:13 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-05-25 16:15:13 INFO BlockManager:59 - Found block rdd_14_1 locally -2016-05-25 16:15:13 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 16:15:13 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 16:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 73 ms on localhost (1/2) -2016-05-25 16:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 72 ms on localhost (2/2) -2016-05-25 16:15:13 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.074 s -2016-05-25 16:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 16:15:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:15:13 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:15:13 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 16:15:13 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:15:13 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[20] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(208696) called with curMem=2120125, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 203.8 KB, free 728.4 MB) -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(44513) called with curMem=2328821, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 43.5 KB, free 728.3 MB) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:54244 (size: 43.5 KB, free: 729.8 MB) -2016-05-25 16:15:13 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[20] at map at DecisionTree.scala:645) -2016-05-25 16:15:13 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 16:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:13 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 16:15:13 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 16:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:15:13 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 1161 bytes result sent to driver -2016-05-25 16:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 10 ms on localhost (1/2) -2016-05-25 16:15:13 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 16:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 53 ms on localhost (2/2) -2016-05-25 16:15:13 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.054 s -2016-05-25 16:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 16:15:13 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.165375 s -2016-05-25 16:15:13 INFO MapPartitionsRDD:59 - Removing RDD 14 from persistence list -2016-05-25 16:15:13 INFO BlockManager:59 - Removing RDD 14 -2016-05-25 16:15:13 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 16:15:13 INFO RandomForest:59 - init: 1.649195946 - total: 3.262024655 - findSplitsBins: 1.350168646 - findBestSplits: 1.591167305 - chooseSplits: 1.584723993 -2016-05-25 16:15:13 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 16:15:13 INFO DAGScheduler:59 - Got job 6 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 16:15:13 INFO DAGScheduler:59 - Final stage: ResultStage 8(take at DecisionTreeMetadata.scala:110) -2016-05-25 16:15:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:15:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[26] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(8064) called with curMem=2170238, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 7.9 KB, free 728.5 MB) -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(4172) called with curMem=2178302, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 4.1 KB, free 728.5 MB) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:54244 (size: 4.1 KB, free: 730.0 MB) -2016-05-25 16:15:13 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[26] at map at DecisionTreeMetadata.scala:110) -2016-05-25 16:15:13 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-05-25 16:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2374 bytes) -2016-05-25 16:15:13 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 16:15:13 INFO CacheManager:59 - Partition rdd_21_0 not found, computing it -2016-05-25 16:15:13 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(1600) called with curMem=2182474, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block rdd_21_0 stored as values in memory (estimated size 1600.0 B, free 728.5 MB) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Added rdd_21_0 in memory on localhost:54244 (size: 1600.0 B, free: 730.0 MB) -2016-05-25 16:15:13 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:15:13 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2611 bytes result sent to driver -2016-05-25 16:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 15 ms on localhost (1/1) -2016-05-25 16:15:13 INFO DAGScheduler:59 - ResultStage 8 (take at DecisionTreeMetadata.scala:110) finished in 0.016 s -2016-05-25 16:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 16:15:13 INFO DAGScheduler:59 - Job 6 finished: take at DecisionTreeMetadata.scala:110, took 0.027805 s -2016-05-25 16:15:13 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 16:15:13 INFO DAGScheduler:59 - Got job 7 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 16:15:13 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at DecisionTreeMetadata.scala:114) -2016-05-25 16:15:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:15:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[25] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(7624) called with curMem=2184074, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 7.4 KB, free 728.5 MB) -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(3951) called with curMem=2191698, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.9 KB, free 728.5 MB) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:54244 (size: 3.9 KB, free: 730.0 MB) -2016-05-25 16:15:13 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[25] at retag at RandomForest.scala:137) -2016-05-25 16:15:13 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 16:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 16, localhost, PROCESS_LOCAL, 2374 bytes) -2016-05-25 16:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2374 bytes) -2016-05-25 16:15:13 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 16) -2016-05-25 16:15:13 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 17) -2016-05-25 16:15:13 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-05-25 16:15:13 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:15:13 INFO CacheManager:59 - Partition rdd_21_1 not found, computing it -2016-05-25 16:15:13 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 16:15:13 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(1560) called with curMem=2195649, maxMem=766075207 -2016-05-25 16:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 16) in 10 ms on localhost (1/2) -2016-05-25 16:15:13 INFO MemoryStore:59 - Block rdd_21_1 stored as values in memory (estimated size 1560.0 B, free 728.5 MB) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Added rdd_21_1 in memory on localhost:54244 (size: 1560.0 B, free: 730.0 MB) -2016-05-25 16:15:13 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-25 16:15:13 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 17). 2662 bytes result sent to driver -2016-05-25 16:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 17) in 12 ms on localhost (2/2) -2016-05-25 16:15:13 INFO DAGScheduler:59 - ResultStage 9 (count at DecisionTreeMetadata.scala:114) finished in 0.013 s -2016-05-25 16:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 16:15:13 INFO DAGScheduler:59 - Job 7 finished: count at DecisionTreeMetadata.scala:114, took 0.027079 s -2016-05-25 16:15:13 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 16:15:13 INFO DAGScheduler:59 - Got job 8 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 16:15:13 INFO DAGScheduler:59 - Final stage: ResultStage 10(collect at DecisionTree.scala:989) -2016-05-25 16:15:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:15:13 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting ResultStage 10 (PartitionwiseSampledRDD[27] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(8360) called with curMem=2197209, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 8.2 KB, free 728.5 MB) -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(4264) called with curMem=2205569, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 4.2 KB, free 728.5 MB) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:54244 (size: 4.2 KB, free: 730.0 MB) -2016-05-25 16:15:13 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (PartitionwiseSampledRDD[27] at sample at DecisionTree.scala:989) -2016-05-25 16:15:13 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-25 16:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 18, localhost, PROCESS_LOCAL, 2483 bytes) -2016-05-25 16:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2483 bytes) -2016-05-25 16:15:13 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 19) -2016-05-25 16:15:13 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 18) -2016-05-25 16:15:13 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-05-25 16:15:13 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-25 16:15:13 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 19). 61832 bytes result sent to driver -2016-05-25 16:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 19) in 9 ms on localhost (1/2) -2016-05-25 16:15:13 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-05-25 16:15:13 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:15:13 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 18). 62623 bytes result sent to driver -2016-05-25 16:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 18) in 16 ms on localhost (2/2) -2016-05-25 16:15:13 INFO DAGScheduler:59 - ResultStage 10 (collect at DecisionTree.scala:989) finished in 0.015 s -2016-05-25 16:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 16:15:13 INFO DAGScheduler:59 - Job 8 finished: collect at DecisionTree.scala:989, took 0.031786 s -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=2209833, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 40.0 B, free 728.5 MB) -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=2209873, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 81.0 B, free 728.5 MB) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:54244 (size: 81.0 B, free: 730.0 MB) -2016-05-25 16:15:13 INFO SparkContext:59 - Created broadcast 14 from broadcast at DecisionTree.scala:604 -2016-05-25 16:15:13 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:15:13 INFO DAGScheduler:59 - Registering RDD 30 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:13 INFO DAGScheduler:59 - Got job 9 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:15:13 INFO DAGScheduler:59 - Final stage: ResultStage 12(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:15:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 11) -2016-05-25 16:15:13 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 11) -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 11 (MapPartitionsRDD[30] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(483136) called with curMem=2209954, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 471.8 KB, free 728.0 MB) -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(118828) called with curMem=2693090, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 116.0 KB, free 727.9 MB) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:54244 (size: 116.0 KB, free: 729.9 MB) -2016-05-25 16:15:13 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 11 (MapPartitionsRDD[30] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:13 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-25 16:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 20, localhost, PROCESS_LOCAL, 2363 bytes) -2016-05-25 16:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2363 bytes) -2016-05-25 16:15:13 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 20) -2016-05-25 16:15:13 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 21) -2016-05-25 16:15:13 INFO CacheManager:59 - Partition rdd_29_1 not found, computing it -2016-05-25 16:15:13 INFO CacheManager:59 - Partition rdd_29_0 not found, computing it -2016-05-25 16:15:13 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-05-25 16:15:13 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-05-25 16:15:13 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-25 16:15:13 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(100120) called with curMem=2811918, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block rdd_29_1 stored as values in memory (estimated size 97.8 KB, free 727.8 MB) -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(102976) called with curMem=2912038, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block rdd_29_0 stored as values in memory (estimated size 100.6 KB, free 727.7 MB) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Added rdd_29_1 in memory on localhost:54244 (size: 97.8 KB, free: 729.8 MB) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Added rdd_29_0 in memory on localhost:54244 (size: 100.6 KB, free: 729.7 MB) -2016-05-25 16:15:13 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 21). 2834 bytes result sent to driver -2016-05-25 16:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 21) in 50 ms on localhost (1/2) -2016-05-25 16:15:13 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 20). 2834 bytes result sent to driver -2016-05-25 16:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 20) in 56 ms on localhost (2/2) -2016-05-25 16:15:13 INFO DAGScheduler:59 - ShuffleMapStage 11 (mapPartitions at DecisionTree.scala:625) finished in 0.058 s -2016-05-25 16:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 16:15:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:15:13 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:15:13 INFO DAGScheduler:59 - waiting: Set(ResultStage 12) -2016-05-25 16:15:13 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:15:13 INFO DAGScheduler:59 - Missing parents for ResultStage 12: List() -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[32] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(208696) called with curMem=3015014, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 203.8 KB, free 727.5 MB) -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(44504) called with curMem=3223710, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 43.5 KB, free 727.5 MB) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:54244 (size: 43.5 KB, free: 729.6 MB) -2016-05-25 16:15:13 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[32] at map at DecisionTree.scala:645) -2016-05-25 16:15:13 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-25 16:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 22, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:13 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 23) -2016-05-25 16:15:13 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 22) -2016-05-25 16:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:15:13 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 23). 1161 bytes result sent to driver -2016-05-25 16:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 23) in 11 ms on localhost (1/2) -2016-05-25 16:15:13 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 22). 2774 bytes result sent to driver -2016-05-25 16:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 22) in 31 ms on localhost (2/2) -2016-05-25 16:15:13 INFO DAGScheduler:59 - ResultStage 12 (collectAsMap at DecisionTree.scala:654) finished in 0.031 s -2016-05-25 16:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 16:15:13 INFO DAGScheduler:59 - Job 9 finished: collectAsMap at DecisionTree.scala:654, took 0.132436 s -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=3268214, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 40.0 B, free 727.5 MB) -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=3268254, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 81.0 B, free 727.5 MB) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:54244 (size: 81.0 B, free: 729.6 MB) -2016-05-25 16:15:13 INFO SparkContext:59 - Created broadcast 17 from broadcast at DecisionTree.scala:604 -2016-05-25 16:15:13 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:15:13 INFO DAGScheduler:59 - Registering RDD 33 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:13 INFO DAGScheduler:59 - Got job 10 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:15:13 INFO DAGScheduler:59 - Final stage: ResultStage 14(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:15:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 13) -2016-05-25 16:15:13 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 13) -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 13 (MapPartitionsRDD[33] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(483336) called with curMem=3268335, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 472.0 KB, free 727.0 MB) -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(118953) called with curMem=3751671, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 116.2 KB, free 726.9 MB) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:54244 (size: 116.2 KB, free: 729.5 MB) -2016-05-25 16:15:13 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 13 (MapPartitionsRDD[33] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:13 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-25 16:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 24, localhost, PROCESS_LOCAL, 2363 bytes) -2016-05-25 16:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2363 bytes) -2016-05-25 16:15:13 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 24) -2016-05-25 16:15:13 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 25) -2016-05-25 16:15:13 INFO BlockManager:59 - Found block rdd_29_1 locally -2016-05-25 16:15:13 INFO BlockManager:59 - Found block rdd_29_0 locally -2016-05-25 16:15:13 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 25). 2254 bytes result sent to driver -2016-05-25 16:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 25) in 29 ms on localhost (1/2) -2016-05-25 16:15:13 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 24). 2254 bytes result sent to driver -2016-05-25 16:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 24) in 34 ms on localhost (2/2) -2016-05-25 16:15:13 INFO DAGScheduler:59 - ShuffleMapStage 13 (mapPartitions at DecisionTree.scala:625) finished in 0.034 s -2016-05-25 16:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 16:15:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:15:13 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:15:13 INFO DAGScheduler:59 - waiting: Set(ResultStage 14) -2016-05-25 16:15:13 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:15:13 INFO DAGScheduler:59 - Missing parents for ResultStage 14: List() -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[35] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(208696) called with curMem=3870624, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 203.8 KB, free 726.7 MB) -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(44517) called with curMem=4079320, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 43.5 KB, free 726.7 MB) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:54244 (size: 43.5 KB, free: 729.5 MB) -2016-05-25 16:15:13 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[35] at map at DecisionTree.scala:645) -2016-05-25 16:15:13 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-25 16:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 26, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:13 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 26) -2016-05-25 16:15:13 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 27) -2016-05-25 16:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:15:13 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 27). 1161 bytes result sent to driver -2016-05-25 16:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 27) in 12 ms on localhost (1/2) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:54244 in memory (size: 116.2 KB, free: 729.6 MB) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:54244 in memory (size: 43.5 KB, free: 729.6 MB) -2016-05-25 16:15:13 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 26). 2774 bytes result sent to driver -2016-05-25 16:15:13 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-05-25 16:15:13 INFO DAGScheduler:59 - ResultStage 14 (collectAsMap at DecisionTree.scala:654) finished in 0.047 s -2016-05-25 16:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 26) in 47 ms on localhost (2/2) -2016-05-25 16:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 16:15:13 INFO DAGScheduler:59 - Job 10 finished: collectAsMap at DecisionTree.scala:654, took 0.129816 s -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:54244 in memory (size: 116.0 KB, free: 729.7 MB) -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=3149520, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 40.0 B, free 728.0 MB) -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=2666424, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 81.0 B, free 728.0 MB) -2016-05-25 16:15:13 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:54244 (size: 81.0 B, free: 729.7 MB) -2016-05-25 16:15:13 INFO SparkContext:59 - Created broadcast 20 from broadcast at DecisionTree.scala:604 -2016-05-25 16:15:13 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:54244 in memory (size: 81.0 B, free: 729.7 MB) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:54244 in memory (size: 4.2 KB, free: 729.7 MB) -2016-05-25 16:15:13 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:54244 in memory (size: 3.9 KB, free: 729.7 MB) -2016-05-25 16:15:13 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:54244 in memory (size: 4.1 KB, free: 729.7 MB) -2016-05-25 16:15:13 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:54244 in memory (size: 43.5 KB, free: 729.8 MB) -2016-05-25 16:15:13 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 16:15:13 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:54244 in memory (size: 115.4 KB, free: 729.9 MB) -2016-05-25 16:15:13 INFO DAGScheduler:59 - Registering RDD 36 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:13 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 16:15:13 INFO DAGScheduler:59 - Got job 11 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:15:13 INFO DAGScheduler:59 - Final stage: ResultStage 16(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:15:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-25 16:15:13 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 15) -2016-05-25 16:15:13 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 15 (MapPartitionsRDD[36] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:54244 in memory (size: 81.0 B, free: 729.9 MB) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:54244 in memory (size: 43.5 KB, free: 729.9 MB) -2016-05-25 16:15:13 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:54244 in memory (size: 115.2 KB, free: 730.1 MB) -2016-05-25 16:15:13 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 16:15:13 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(483688) called with curMem=924412, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 472.4 KB, free 729.2 MB) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:54244 in memory (size: 81.0 B, free: 730.1 MB) -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(119089) called with curMem=1407979, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 116.3 KB, free 729.1 MB) -2016-05-25 16:15:13 INFO BlockManager:59 - Removing RDD 14 -2016-05-25 16:15:13 INFO ContextCleaner:59 - Cleaned RDD 14 -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:54244 (size: 116.3 KB, free: 729.9 MB) -2016-05-25 16:15:13 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 15 (MapPartitionsRDD[36] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:13 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-25 16:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 28, localhost, PROCESS_LOCAL, 2363 bytes) -2016-05-25 16:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2363 bytes) -2016-05-25 16:15:13 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 28) -2016-05-25 16:15:13 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 29) -2016-05-25 16:15:13 INFO BlockManager:59 - Found block rdd_29_0 locally -2016-05-25 16:15:13 INFO BlockManager:59 - Found block rdd_29_1 locally -2016-05-25 16:15:13 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 28). 2254 bytes result sent to driver -2016-05-25 16:15:13 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 29). 2254 bytes result sent to driver -2016-05-25 16:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 28) in 37 ms on localhost (1/2) -2016-05-25 16:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 29) in 38 ms on localhost (2/2) -2016-05-25 16:15:13 INFO DAGScheduler:59 - ShuffleMapStage 15 (mapPartitions at DecisionTree.scala:625) finished in 0.040 s -2016-05-25 16:15:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:15:13 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:15:13 INFO DAGScheduler:59 - waiting: Set(ResultStage 16) -2016-05-25 16:15:13 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-25 16:15:13 INFO DAGScheduler:59 - Missing parents for ResultStage 16: List() -2016-05-25 16:15:13 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[38] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(208760) called with curMem=1527068, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 203.9 KB, free 728.9 MB) -2016-05-25 16:15:13 INFO MemoryStore:59 - ensureFreeSpace(44529) called with curMem=1735828, maxMem=766075207 -2016-05-25 16:15:13 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 43.5 KB, free 728.9 MB) -2016-05-25 16:15:13 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:54244 (size: 43.5 KB, free: 729.9 MB) -2016-05-25 16:15:13 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[38] at map at DecisionTree.scala:645) -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 30, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:14 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 31) -2016-05-25 16:15:14 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 30) -2016-05-25 16:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:15:14 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 31). 2774 bytes result sent to driver -2016-05-25 16:15:14 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 30). 2774 bytes result sent to driver -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 31) in 30 ms on localhost (1/2) -2016-05-25 16:15:14 INFO DAGScheduler:59 - ResultStage 16 (collectAsMap at DecisionTree.scala:654) finished in 0.033 s -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 30) in 33 ms on localhost (2/2) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Job 11 finished: collectAsMap at DecisionTree.scala:654, took 0.122841 s -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=1780357, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 40.0 B, free 728.9 MB) -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=1780397, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 81.0 B, free 728.9 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:54244 (size: 81.0 B, free: 729.9 MB) -2016-05-25 16:15:14 INFO SparkContext:59 - Created broadcast 23 from broadcast at DecisionTree.scala:604 -2016-05-25 16:15:14 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:15:14 INFO DAGScheduler:59 - Registering RDD 39 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Got job 12 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:15:14 INFO DAGScheduler:59 - Final stage: ResultStage 18(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 17) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 17) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 17 (MapPartitionsRDD[39] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(483880) called with curMem=1780478, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 472.5 KB, free 728.4 MB) -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(119144) called with curMem=2264358, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 116.4 KB, free 728.3 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:54244 (size: 116.4 KB, free: 729.8 MB) -2016-05-25 16:15:14 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 17 (MapPartitionsRDD[39] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 32, localhost, PROCESS_LOCAL, 2363 bytes) -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2363 bytes) -2016-05-25 16:15:14 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 32) -2016-05-25 16:15:14 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 33) -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_29_0 locally -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_29_1 locally -2016-05-25 16:15:14 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 32). 2254 bytes result sent to driver -2016-05-25 16:15:14 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 33). 2254 bytes result sent to driver -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 32) in 38 ms on localhost (1/2) -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 33) in 39 ms on localhost (2/2) -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-25 16:15:14 INFO DAGScheduler:59 - ShuffleMapStage 17 (mapPartitions at DecisionTree.scala:625) finished in 0.040 s -2016-05-25 16:15:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:15:14 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:15:14 INFO DAGScheduler:59 - waiting: Set(ResultStage 18) -2016-05-25 16:15:14 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:15:14 INFO DAGScheduler:59 - Missing parents for ResultStage 18: List() -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[41] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(208696) called with curMem=2383502, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 203.8 KB, free 728.1 MB) -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(44516) called with curMem=2592198, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 43.5 KB, free 728.1 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:54244 (size: 43.5 KB, free: 729.7 MB) -2016-05-25 16:15:14 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[41] at map at DecisionTree.scala:645) -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 34, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:14 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 34) -2016-05-25 16:15:14 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 35) -2016-05-25 16:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:15:14 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 35). 1161 bytes result sent to driver -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 35) in 11 ms on localhost (1/2) -2016-05-25 16:15:14 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 34). 2774 bytes result sent to driver -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 34) in 30 ms on localhost (2/2) -2016-05-25 16:15:14 INFO DAGScheduler:59 - ResultStage 18 (collectAsMap at DecisionTree.scala:654) finished in 0.030 s -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-25 16:15:14 INFO DAGScheduler:59 - Job 12 finished: collectAsMap at DecisionTree.scala:654, took 0.117991 s -2016-05-25 16:15:14 INFO MapPartitionsRDD:59 - Removing RDD 29 from persistence list -2016-05-25 16:15:14 INFO BlockManager:59 - Removing RDD 29 -2016-05-25 16:15:14 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 16:15:14 INFO RandomForest:59 - init: 0.241441749 - total: 0.982388872 - findSplitsBins: 0.145664395 - findBestSplits: 0.731993415 - chooseSplits: 0.731336144 -2016-05-25 16:15:14 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 16:15:14 INFO DAGScheduler:59 - Got job 13 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 16:15:14 INFO DAGScheduler:59 - Final stage: ResultStage 19(take at DecisionTreeMetadata.scala:110) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:15:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[47] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(9288) called with curMem=2433618, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 9.1 KB, free 728.3 MB) -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(4607) called with curMem=2442906, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 4.5 KB, free 728.3 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:54244 (size: 4.5 KB, free: 729.9 MB) -2016-05-25 16:15:14 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 19 (MapPartitionsRDD[47] at map at DecisionTreeMetadata.scala:110) -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 1 tasks -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 36, localhost, PROCESS_LOCAL, 2406 bytes) -2016-05-25 16:15:14 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 36) -2016-05-25 16:15:14 INFO CacheManager:59 - Partition rdd_43_0 not found, computing it -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(1600) called with curMem=2447513, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block rdd_43_0 stored as values in memory (estimated size 1600.0 B, free 728.3 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Added rdd_43_0 in memory on localhost:54244 (size: 1600.0 B, free: 729.9 MB) -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:15:14 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 36). 2611 bytes result sent to driver -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 36) in 11 ms on localhost (1/1) -2016-05-25 16:15:14 INFO DAGScheduler:59 - ResultStage 19 (take at DecisionTreeMetadata.scala:110) finished in 0.011 s -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-25 16:15:14 INFO DAGScheduler:59 - Job 13 finished: take at DecisionTreeMetadata.scala:110, took 0.024108 s -2016-05-25 16:15:14 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 16:15:14 INFO DAGScheduler:59 - Got job 14 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 16:15:14 INFO DAGScheduler:59 - Final stage: ResultStage 20(count at DecisionTreeMetadata.scala:114) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:15:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[46] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(8848) called with curMem=2449113, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 8.6 KB, free 728.2 MB) -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(4402) called with curMem=2457961, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 4.3 KB, free 728.2 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:54244 (size: 4.3 KB, free: 729.9 MB) -2016-05-25 16:15:14 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[46] at retag at RandomForest.scala:137) -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 37, localhost, PROCESS_LOCAL, 2406 bytes) -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 38, localhost, PROCESS_LOCAL, 2406 bytes) -2016-05-25 16:15:14 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 37) -2016-05-25 16:15:14 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 38) -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_43_0 locally -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:15:14 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 37). 2082 bytes result sent to driver -2016-05-25 16:15:14 INFO CacheManager:59 - Partition rdd_43_1 not found, computing it -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 37) in 9 ms on localhost (1/2) -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(1560) called with curMem=2462363, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block rdd_43_1 stored as values in memory (estimated size 1560.0 B, free 728.2 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Added rdd_43_1 in memory on localhost:54244 (size: 1560.0 B, free: 729.9 MB) -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-25 16:15:14 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 38). 2662 bytes result sent to driver -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 38) in 11 ms on localhost (2/2) -2016-05-25 16:15:14 INFO DAGScheduler:59 - ResultStage 20 (count at DecisionTreeMetadata.scala:114) finished in 0.013 s -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-25 16:15:14 INFO DAGScheduler:59 - Job 14 finished: count at DecisionTreeMetadata.scala:114, took 0.027624 s -2016-05-25 16:15:14 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 16:15:14 INFO DAGScheduler:59 - Got job 15 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 16:15:14 INFO DAGScheduler:59 - Final stage: ResultStage 21(collect at DecisionTree.scala:989) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:15:14 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting ResultStage 21 (PartitionwiseSampledRDD[48] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(9576) called with curMem=2463923, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 9.4 KB, free 728.2 MB) -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(4701) called with curMem=2473499, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 4.6 KB, free 728.2 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:54244 (size: 4.6 KB, free: 729.9 MB) -2016-05-25 16:15:14 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (PartitionwiseSampledRDD[48] at sample at DecisionTree.scala:989) -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 39, localhost, PROCESS_LOCAL, 2515 bytes) -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 40, localhost, PROCESS_LOCAL, 2515 bytes) -2016-05-25 16:15:14 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 39) -2016-05-25 16:15:14 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 40) -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_43_0 locally -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_43_1 locally -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-25 16:15:14 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 39). 62623 bytes result sent to driver -2016-05-25 16:15:14 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 40). 61832 bytes result sent to driver -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 39) in 10 ms on localhost (1/2) -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 40) in 10 ms on localhost (2/2) -2016-05-25 16:15:14 INFO DAGScheduler:59 - ResultStage 21 (collect at DecisionTree.scala:989) finished in 0.011 s -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-25 16:15:14 INFO DAGScheduler:59 - Job 15 finished: collect at DecisionTree.scala:989, took 0.025176 s -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=2478200, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 40.0 B, free 728.2 MB) -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=2478240, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 81.0 B, free 728.2 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:54244 (size: 81.0 B, free: 729.9 MB) -2016-05-25 16:15:14 INFO SparkContext:59 - Created broadcast 29 from broadcast at DecisionTree.scala:604 -2016-05-25 16:15:14 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:15:14 INFO DAGScheduler:59 - Registering RDD 51 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Got job 16 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:15:14 INFO DAGScheduler:59 - Final stage: ResultStage 23(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 22) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 22) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 22 (MapPartitionsRDD[51] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(484360) called with curMem=2478321, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 473.0 KB, free 727.8 MB) -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(119412) called with curMem=2962681, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 116.6 KB, free 727.6 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:54244 (size: 116.6 KB, free: 729.8 MB) -2016-05-25 16:15:14 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 22 (MapPartitionsRDD[51] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2395 bytes) -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 42, localhost, PROCESS_LOCAL, 2395 bytes) -2016-05-25 16:15:14 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 41) -2016-05-25 16:15:14 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 42) -2016-05-25 16:15:14 INFO CacheManager:59 - Partition rdd_50_1 not found, computing it -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_43_1 locally -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-05-25 16:15:14 INFO CacheManager:59 - Partition rdd_50_0 not found, computing it -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_43_0 locally -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(102976) called with curMem=3082093, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block rdd_50_0 stored as values in memory (estimated size 100.6 KB, free 727.5 MB) -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(100120) called with curMem=3185069, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block rdd_50_1 stored as values in memory (estimated size 97.8 KB, free 727.5 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Added rdd_50_0 in memory on localhost:54244 (size: 100.6 KB, free: 729.7 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Added rdd_50_1 in memory on localhost:54244 (size: 97.8 KB, free: 729.6 MB) -2016-05-25 16:15:14 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 41). 2834 bytes result sent to driver -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 41) in 41 ms on localhost (1/2) -2016-05-25 16:15:14 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 42). 2834 bytes result sent to driver -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 42) in 47 ms on localhost (2/2) -2016-05-25 16:15:14 INFO DAGScheduler:59 - ShuffleMapStage 22 (mapPartitions at DecisionTree.scala:625) finished in 0.048 s -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-25 16:15:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:15:14 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:15:14 INFO DAGScheduler:59 - waiting: Set(ResultStage 23) -2016-05-25 16:15:14 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:15:14 INFO DAGScheduler:59 - Missing parents for ResultStage 23: List() -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[53] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(208696) called with curMem=3285189, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 203.8 KB, free 727.3 MB) -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(44507) called with curMem=3493885, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 43.5 KB, free 727.2 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:54244 (size: 43.5 KB, free: 729.6 MB) -2016-05-25 16:15:14 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[53] at map at DecisionTree.scala:645) -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:14 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 43) -2016-05-25 16:15:14 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 44) -2016-05-25 16:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:15:14 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 44). 1161 bytes result sent to driver -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 44) in 9 ms on localhost (1/2) -2016-05-25 16:15:14 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 43). 2774 bytes result sent to driver -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 43) in 24 ms on localhost (2/2) -2016-05-25 16:15:14 INFO DAGScheduler:59 - ResultStage 23 (collectAsMap at DecisionTree.scala:654) finished in 0.024 s -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-25 16:15:14 INFO DAGScheduler:59 - Job 16 finished: collectAsMap at DecisionTree.scala:654, took 0.113225 s -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=3538392, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 40.0 B, free 727.2 MB) -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=3538432, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 81.0 B, free 727.2 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:54244 (size: 81.0 B, free: 729.6 MB) -2016-05-25 16:15:14 INFO SparkContext:59 - Created broadcast 32 from broadcast at DecisionTree.scala:604 -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:54244 in memory (size: 43.5 KB, free: 729.6 MB) -2016-05-25 16:15:14 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:54244 in memory (size: 116.6 KB, free: 729.7 MB) -2016-05-25 16:15:14 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-05-25 16:15:14 INFO ContextCleaner:59 - Cleaned shuffle 6 -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:54244 in memory (size: 81.0 B, free: 729.7 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:54244 in memory (size: 4.6 KB, free: 729.7 MB) -2016-05-25 16:15:14 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:54244 in memory (size: 4.3 KB, free: 729.7 MB) -2016-05-25 16:15:14 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:54244 in memory (size: 4.5 KB, free: 729.7 MB) -2016-05-25 16:15:14 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:15:14 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-05-25 16:15:14 INFO DAGScheduler:59 - Registering RDD 54 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Got job 17 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:15:14 INFO DAGScheduler:59 - Final stage: ResultStage 25(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 24) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 24) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:54244 in memory (size: 43.5 KB, free: 729.8 MB) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 24 (MapPartitionsRDD[54] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:15:14 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:54244 in memory (size: 116.4 KB, free: 729.9 MB) -2016-05-25 16:15:14 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-05-25 16:15:14 INFO ContextCleaner:59 - Cleaned shuffle 5 -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:54244 in memory (size: 81.0 B, free: 729.9 MB) -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(484688) called with curMem=1783638, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 473.3 KB, free 728.4 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:54244 in memory (size: 43.5 KB, free: 729.9 MB) -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(119586) called with curMem=2015037, maxMem=766075207 -2016-05-25 16:15:14 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 116.8 KB, free 728.6 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:54244 (size: 116.8 KB, free: 729.8 MB) -2016-05-25 16:15:14 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:54244 in memory (size: 116.3 KB, free: 729.9 MB) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 24 (MapPartitionsRDD[54] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-05-25 16:15:14 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 45, localhost, PROCESS_LOCAL, 2395 bytes) -2016-05-25 16:15:14 INFO ContextCleaner:59 - Cleaned shuffle 4 -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 46, localhost, PROCESS_LOCAL, 2395 bytes) -2016-05-25 16:15:14 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 45) -2016-05-25 16:15:14 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 46) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:54244 in memory (size: 81.0 B, free: 729.9 MB) -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_50_0 locally -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:54244 in memory (size: 43.5 KB, free: 730.0 MB) -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_50_1 locally -2016-05-25 16:15:14 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-05-25 16:15:14 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-05-25 16:15:14 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:54244 in memory (size: 81.0 B, free: 730.0 MB) -2016-05-25 16:15:14 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 45). 2254 bytes result sent to driver -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 45) in 36 ms on localhost (1/2) -2016-05-25 16:15:14 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 46). 2254 bytes result sent to driver -2016-05-25 16:15:14 INFO BlockManager:59 - Removing RDD 29 -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 46) in 39 ms on localhost (2/2) -2016-05-25 16:15:14 INFO DAGScheduler:59 - ShuffleMapStage 24 (mapPartitions at DecisionTree.scala:625) finished in 0.041 s -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-05-25 16:15:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:15:14 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:15:14 INFO DAGScheduler:59 - waiting: Set(ResultStage 25) -2016-05-25 16:15:14 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:15:14 INFO DAGScheduler:59 - Missing parents for ResultStage 25: List() -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[56] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:15:14 INFO ContextCleaner:59 - Cleaned RDD 29 -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(208760) called with curMem=1278391, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 203.9 KB, free 729.2 MB) -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(44538) called with curMem=1487151, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 43.5 KB, free 729.1 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:54244 (size: 43.5 KB, free: 729.9 MB) -2016-05-25 16:15:14 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[56] at map at DecisionTree.scala:645) -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:14 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 48) -2016-05-25 16:15:14 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 47) -2016-05-25 16:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:15:14 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 47). 2774 bytes result sent to driver -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 47) in 39 ms on localhost (1/2) -2016-05-25 16:15:14 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 48). 2774 bytes result sent to driver -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 48) in 40 ms on localhost (2/2) -2016-05-25 16:15:14 INFO DAGScheduler:59 - ResultStage 25 (collectAsMap at DecisionTree.scala:654) finished in 0.042 s -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-05-25 16:15:14 INFO DAGScheduler:59 - Job 17 finished: collectAsMap at DecisionTree.scala:654, took 0.133285 s -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=1531689, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 40.0 B, free 729.1 MB) -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=1531729, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 81.0 B, free 729.1 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:54244 (size: 81.0 B, free: 729.9 MB) -2016-05-25 16:15:14 INFO SparkContext:59 - Created broadcast 35 from broadcast at DecisionTree.scala:604 -2016-05-25 16:15:14 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:15:14 INFO DAGScheduler:59 - Registering RDD 57 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Got job 18 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:15:14 INFO DAGScheduler:59 - Final stage: ResultStage 27(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[57] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(485232) called with curMem=1531810, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 473.9 KB, free 728.7 MB) -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(119769) called with curMem=2017042, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 117.0 KB, free 728.5 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:54244 (size: 117.0 KB, free: 729.8 MB) -2016-05-25 16:15:14 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[57] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 49, localhost, PROCESS_LOCAL, 2395 bytes) -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 50, localhost, PROCESS_LOCAL, 2395 bytes) -2016-05-25 16:15:14 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 49) -2016-05-25 16:15:14 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 50) -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_50_0 locally -2016-05-25 16:15:14 INFO BlockManager:59 - Found block rdd_50_1 locally -2016-05-25 16:15:14 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 49). 2254 bytes result sent to driver -2016-05-25 16:15:14 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 50). 2254 bytes result sent to driver -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 49) in 29 ms on localhost (1/2) -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 50) in 30 ms on localhost (2/2) -2016-05-25 16:15:14 INFO DAGScheduler:59 - ShuffleMapStage 26 (mapPartitions at DecisionTree.scala:625) finished in 0.031 s -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-05-25 16:15:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:15:14 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:15:14 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-05-25 16:15:14 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:15:14 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[59] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(208880) called with curMem=2136811, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 204.0 KB, free 728.3 MB) -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(44528) called with curMem=2345691, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 43.5 KB, free 728.3 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:54244 (size: 43.5 KB, free: 729.8 MB) -2016-05-25 16:15:14 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[59] at map at DecisionTree.scala:645) -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 51, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 52, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:14 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 52) -2016-05-25 16:15:14 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 51) -2016-05-25 16:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:15:14 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 52). 2828 bytes result sent to driver -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 52) in 37 ms on localhost (1/2) -2016-05-25 16:15:14 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 51). 2928 bytes result sent to driver -2016-05-25 16:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 51) in 40 ms on localhost (2/2) -2016-05-25 16:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-05-25 16:15:14 INFO DAGScheduler:59 - ResultStage 27 (collectAsMap at DecisionTree.scala:654) finished in 0.042 s -2016-05-25 16:15:14 INFO DAGScheduler:59 - Job 18 finished: collectAsMap at DecisionTree.scala:654, took 0.109560 s -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=2390219, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 40.0 B, free 728.3 MB) -2016-05-25 16:15:14 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=2390259, maxMem=766075207 -2016-05-25 16:15:14 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 81.0 B, free 728.3 MB) -2016-05-25 16:15:14 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:54244 (size: 81.0 B, free: 729.8 MB) -2016-05-25 16:15:14 INFO SparkContext:59 - Created broadcast 38 from broadcast at DecisionTree.scala:604 -2016-05-25 16:15:14 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:15:14 INFO DAGScheduler:59 - Registering RDD 60 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Got job 19 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:15:14 INFO DAGScheduler:59 - Final stage: ResultStage 29(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 28) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 28) -2016-05-25 16:15:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 28 (MapPartitionsRDD[60] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(485736) called with curMem=2390340, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 474.4 KB, free 727.8 MB) -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(119938) called with curMem=2876076, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 117.1 KB, free 727.7 MB) -2016-05-25 16:15:15 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:54244 (size: 117.1 KB, free: 729.7 MB) -2016-05-25 16:15:15 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 28 (MapPartitionsRDD[60] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:15 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-05-25 16:15:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 53, localhost, PROCESS_LOCAL, 2395 bytes) -2016-05-25 16:15:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 54, localhost, PROCESS_LOCAL, 2395 bytes) -2016-05-25 16:15:15 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 53) -2016-05-25 16:15:15 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 54) -2016-05-25 16:15:15 INFO BlockManager:59 - Found block rdd_50_1 locally -2016-05-25 16:15:15 INFO BlockManager:59 - Found block rdd_50_0 locally -2016-05-25 16:15:15 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 53). 2254 bytes result sent to driver -2016-05-25 16:15:15 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 54). 2254 bytes result sent to driver -2016-05-25 16:15:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 54) in 32 ms on localhost (1/2) -2016-05-25 16:15:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 53) in 34 ms on localhost (2/2) -2016-05-25 16:15:15 INFO DAGScheduler:59 - ShuffleMapStage 28 (mapPartitions at DecisionTree.scala:625) finished in 0.034 s -2016-05-25 16:15:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-05-25 16:15:15 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:15:15 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:15:15 INFO DAGScheduler:59 - waiting: Set(ResultStage 29) -2016-05-25 16:15:15 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:15:15 INFO DAGScheduler:59 - Missing parents for ResultStage 29: List() -2016-05-25 16:15:15 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[62] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(208816) called with curMem=2996014, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 203.9 KB, free 727.5 MB) -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(44532) called with curMem=3204830, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 43.5 KB, free 727.5 MB) -2016-05-25 16:15:15 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:54244 (size: 43.5 KB, free: 729.6 MB) -2016-05-25 16:15:15 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[62] at map at DecisionTree.scala:645) -2016-05-25 16:15:15 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-05-25 16:15:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 55, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:15 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 55) -2016-05-25 16:15:15 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 56) -2016-05-25 16:15:15 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:15:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:15:15 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:15:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:15:15 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 55). 2928 bytes result sent to driver -2016-05-25 16:15:15 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 56). 2774 bytes result sent to driver -2016-05-25 16:15:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 55) in 36 ms on localhost (1/2) -2016-05-25 16:15:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 56) in 37 ms on localhost (2/2) -2016-05-25 16:15:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-05-25 16:15:15 INFO DAGScheduler:59 - ResultStage 29 (collectAsMap at DecisionTree.scala:654) finished in 0.037 s -2016-05-25 16:15:15 INFO DAGScheduler:59 - Job 19 finished: collectAsMap at DecisionTree.scala:654, took 0.111864 s -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=3249362, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 40.0 B, free 727.5 MB) -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=3249402, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 81.0 B, free 727.5 MB) -2016-05-25 16:15:15 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:54244 (size: 81.0 B, free: 729.6 MB) -2016-05-25 16:15:15 INFO SparkContext:59 - Created broadcast 41 from broadcast at DecisionTree.scala:604 -2016-05-25 16:15:15 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:15:15 INFO DAGScheduler:59 - Registering RDD 63 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:15 INFO DAGScheduler:59 - Got job 20 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:15:15 INFO DAGScheduler:59 - Final stage: ResultStage 31(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:15:15 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30) -2016-05-25 16:15:15 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 30) -2016-05-25 16:15:15 INFO DAGScheduler:59 - Submitting ShuffleMapStage 30 (MapPartitionsRDD[63] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(485984) called with curMem=3249483, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 474.6 KB, free 727.0 MB) -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(119945) called with curMem=3735467, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 117.1 KB, free 726.9 MB) -2016-05-25 16:15:15 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:54244 (size: 117.1 KB, free: 729.5 MB) -2016-05-25 16:15:15 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 30 (MapPartitionsRDD[63] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:15:15 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-05-25 16:15:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 57, localhost, PROCESS_LOCAL, 2395 bytes) -2016-05-25 16:15:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 58, localhost, PROCESS_LOCAL, 2395 bytes) -2016-05-25 16:15:15 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 57) -2016-05-25 16:15:15 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 58) -2016-05-25 16:15:15 INFO BlockManager:59 - Found block rdd_50_0 locally -2016-05-25 16:15:15 INFO BlockManager:59 - Found block rdd_50_1 locally -2016-05-25 16:15:15 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 57). 2254 bytes result sent to driver -2016-05-25 16:15:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 57) in 24 ms on localhost (1/2) -2016-05-25 16:15:15 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 58). 2254 bytes result sent to driver -2016-05-25 16:15:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 58) in 30 ms on localhost (2/2) -2016-05-25 16:15:15 INFO DAGScheduler:59 - ShuffleMapStage 30 (mapPartitions at DecisionTree.scala:625) finished in 0.031 s -2016-05-25 16:15:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-05-25 16:15:15 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:15:15 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:15:15 INFO DAGScheduler:59 - waiting: Set(ResultStage 31) -2016-05-25 16:15:15 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:15:15 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List() -2016-05-25 16:15:15 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[65] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(208696) called with curMem=3855412, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 203.8 KB, free 726.7 MB) -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(44515) called with curMem=4064108, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 43.5 KB, free 726.7 MB) -2016-05-25 16:15:15 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:54244 (size: 43.5 KB, free: 729.5 MB) -2016-05-25 16:15:15 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[65] at map at DecisionTree.scala:645) -2016-05-25 16:15:15 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-05-25 16:15:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 59, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:15:15 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 60) -2016-05-25 16:15:15 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 59) -2016-05-25 16:15:15 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:15:15 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:15:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:15:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:15:15 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 60). 1161 bytes result sent to driver -2016-05-25 16:15:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 60) in 10 ms on localhost (1/2) -2016-05-25 16:15:15 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 59). 2774 bytes result sent to driver -2016-05-25 16:15:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 59) in 24 ms on localhost (2/2) -2016-05-25 16:15:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-05-25 16:15:15 INFO DAGScheduler:59 - ResultStage 31 (collectAsMap at DecisionTree.scala:654) finished in 0.025 s -2016-05-25 16:15:15 INFO DAGScheduler:59 - Job 20 finished: collectAsMap at DecisionTree.scala:654, took 0.092158 s -2016-05-25 16:15:15 INFO MapPartitionsRDD:59 - Removing RDD 50 from persistence list -2016-05-25 16:15:15 INFO BlockManager:59 - Removing RDD 50 -2016-05-25 16:15:15 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 16:15:15 INFO RandomForest:59 - init: 0.190027322 - total: 1.004803035 - findSplitsBins: 0.104787284 - findBestSplits: 0.803961746 - chooseSplits: 0.803257175 -2016-05-25 16:15:15 INFO GradientBoostedTrees:59 - Internal timing for DecisionTree: -2016-05-25 16:15:15 INFO GradientBoostedTrees:59 - init: 0.009598038 - total: 5.329443845 - building tree 0: 3.283937275 - building tree 2: 1.005574983 - building tree 1: 0.984165692 -2016-05-25 16:15:15 INFO MapPartitionsRDD:59 - Removing RDD 9 from persistence list -2016-05-25 16:15:15 INFO BlockManager:59 - Removing RDD 9 -2016-05-25 16:15:15 INFO SparkContext:59 - Starting job: count at GBTs1.scala:58 -2016-05-25 16:15:15 INFO DAGScheduler:59 - Got job 21 (count at GBTs1.scala:58) with 2 output partitions -2016-05-25 16:15:15 INFO DAGScheduler:59 - Final stage: ResultStage 32(count at GBTs1.scala:58) -2016-05-25 16:15:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:15:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:15:15 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[69] at filter at GBTs1.scala:58), which has no missing parents -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(10120) called with curMem=3783255, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 9.9 KB, free 727.0 MB) -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(4730) called with curMem=3793375, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 4.6 KB, free 727.0 MB) -2016-05-25 16:15:15 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:54244 (size: 4.6 KB, free: 729.8 MB) -2016-05-25 16:15:15 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[69] at filter at GBTs1.scala:58) -2016-05-25 16:15:15 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-05-25 16:15:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 61, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:15:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 62, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:15:15 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 62) -2016-05-25 16:15:15 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 61) -2016-05-25 16:15:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:15:15 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 16:15:15 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 16:15:15 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 16:15:15 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 62). 2082 bytes result sent to driver -2016-05-25 16:15:15 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 61). 2082 bytes result sent to driver -2016-05-25 16:15:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 62) in 15 ms on localhost (1/2) -2016-05-25 16:15:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 61) in 17 ms on localhost (2/2) -2016-05-25 16:15:15 INFO DAGScheduler:59 - ResultStage 32 (count at GBTs1.scala:58) finished in 0.018 s -2016-05-25 16:15:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-05-25 16:15:15 INFO DAGScheduler:59 - Job 21 finished: count at GBTs1.scala:58, took 0.028886 s -2016-05-25 16:15:15 INFO SparkContext:59 - Starting job: count at GBTs1.scala:58 -2016-05-25 16:15:15 INFO DAGScheduler:59 - Got job 22 (count at GBTs1.scala:58) with 2 output partitions -2016-05-25 16:15:15 INFO DAGScheduler:59 - Final stage: ResultStage 33(count at GBTs1.scala:58) -2016-05-25 16:15:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:15:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:15:15 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[8] at randomSplit at GBTs1.scala:33), which has no missing parents -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=3798105, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 4.0 KB, free 727.0 MB) -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=3802217, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.2 KB, free 727.0 MB) -2016-05-25 16:15:15 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:54244 (size: 2.2 KB, free: 729.8 MB) -2016-05-25 16:15:15 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[8] at randomSplit at GBTs1.scala:33) -2016-05-25 16:15:15 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-05-25 16:15:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 63, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:15:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 64, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:15:15 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 63) -2016-05-25 16:15:15 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 64) -2016-05-25 16:15:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:15:15 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 16:15:15 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 63). 2082 bytes result sent to driver -2016-05-25 16:15:15 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 64). 2082 bytes result sent to driver -2016-05-25 16:15:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 63) in 9 ms on localhost (1/2) -2016-05-25 16:15:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 64) in 9 ms on localhost (2/2) -2016-05-25 16:15:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-05-25 16:15:15 INFO DAGScheduler:59 - ResultStage 33 (count at GBTs1.scala:58) finished in 0.011 s -2016-05-25 16:15:15 INFO DAGScheduler:59 - Job 22 finished: count at GBTs1.scala:58, took 0.022715 s -2016-05-25 16:15:15 INFO SparkContext:59 - Starting job: count at GBTs1.scala:63 -2016-05-25 16:15:15 INFO DAGScheduler:59 - Got job 23 (count at GBTs1.scala:63) with 2 output partitions -2016-05-25 16:15:15 INFO DAGScheduler:59 - Final stage: ResultStage 34(count at GBTs1.scala:63) -2016-05-25 16:15:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:15:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:15:15 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(3816) called with curMem=3804485, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 3.7 KB, free 727.0 MB) -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(2142) called with curMem=3808301, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 2.1 KB, free 727.0 MB) -2016-05-25 16:15:15 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:54244 (size: 2.1 KB, free: 729.8 MB) -2016-05-25 16:15:15 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 16:15:15 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-05-25 16:15:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 65, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:15:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 66, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:15:15 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 66) -2016-05-25 16:15:15 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 65) -2016-05-25 16:15:15 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 16:15:15 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 66). 2082 bytes result sent to driver -2016-05-25 16:15:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:15:15 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 65). 2082 bytes result sent to driver -2016-05-25 16:15:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 66) in 6 ms on localhost (1/2) -2016-05-25 16:15:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 65) in 8 ms on localhost (2/2) -2016-05-25 16:15:15 INFO DAGScheduler:59 - ResultStage 34 (count at GBTs1.scala:63) finished in 0.008 s -2016-05-25 16:15:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-05-25 16:15:15 INFO DAGScheduler:59 - Job 23 finished: count at GBTs1.scala:63, took 0.018420 s -2016-05-25 16:15:15 INFO SparkContext:59 - Starting job: count at GBTs1.scala:64 -2016-05-25 16:15:15 INFO DAGScheduler:59 - Got job 24 (count at GBTs1.scala:64) with 2 output partitions -2016-05-25 16:15:15 INFO DAGScheduler:59 - Final stage: ResultStage 35(count at GBTs1.scala:64) -2016-05-25 16:15:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:15:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:15:15 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[7] at randomSplit at GBTs1.scala:33), which has no missing parents -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=3810443, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 4.0 KB, free 726.9 MB) -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(2265) called with curMem=3814555, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.2 KB, free 726.9 MB) -2016-05-25 16:15:15 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:54244 (size: 2.2 KB, free: 729.8 MB) -2016-05-25 16:15:15 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[7] at randomSplit at GBTs1.scala:33) -2016-05-25 16:15:15 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-05-25 16:15:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 67, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:15:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 68, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:15:15 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 68) -2016-05-25 16:15:15 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 67) -2016-05-25 16:15:15 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 16:15:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:15:15 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 68). 2082 bytes result sent to driver -2016-05-25 16:15:15 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 67). 2082 bytes result sent to driver -2016-05-25 16:15:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 68) in 6 ms on localhost (1/2) -2016-05-25 16:15:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 67) in 8 ms on localhost (2/2) -2016-05-25 16:15:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-05-25 16:15:15 INFO DAGScheduler:59 - ResultStage 35 (count at GBTs1.scala:64) finished in 0.008 s -2016-05-25 16:15:15 INFO DAGScheduler:59 - Job 24 finished: count at GBTs1.scala:64, took 0.017190 s -2016-05-25 16:15:15 INFO SparkContext:59 - Starting job: count at GBTs1.scala:65 -2016-05-25 16:15:15 INFO DAGScheduler:59 - Got job 25 (count at GBTs1.scala:65) with 2 output partitions -2016-05-25 16:15:15 INFO DAGScheduler:59 - Final stage: ResultStage 36(count at GBTs1.scala:65) -2016-05-25 16:15:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:15:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:15:15 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[8] at randomSplit at GBTs1.scala:33), which has no missing parents -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=3816820, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 4.0 KB, free 726.9 MB) -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=3820932, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.2 KB, free 726.9 MB) -2016-05-25 16:15:15 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:54244 (size: 2.2 KB, free: 729.8 MB) -2016-05-25 16:15:15 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[8] at randomSplit at GBTs1.scala:33) -2016-05-25 16:15:15 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-05-25 16:15:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 69, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:15:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 70, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:15:15 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 70) -2016-05-25 16:15:15 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 69) -2016-05-25 16:15:15 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 16:15:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:15:15 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 70). 2082 bytes result sent to driver -2016-05-25 16:15:15 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 69). 2082 bytes result sent to driver -2016-05-25 16:15:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 70) in 6 ms on localhost (1/2) -2016-05-25 16:15:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 69) in 7 ms on localhost (2/2) -2016-05-25 16:15:15 INFO DAGScheduler:59 - ResultStage 36 (count at GBTs1.scala:65) finished in 0.007 s -2016-05-25 16:15:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-05-25 16:15:15 INFO DAGScheduler:59 - Job 25 finished: count at GBTs1.scala:65, took 0.016977 s -2016-05-25 16:15:15 INFO SparkContext:59 - Starting job: take at GBTs1.scala:71 -2016-05-25 16:15:15 INFO DAGScheduler:59 - Got job 26 (take at GBTs1.scala:71) with 1 output partitions -2016-05-25 16:15:15 INFO DAGScheduler:59 - Final stage: ResultStage 37(take at GBTs1.scala:71) -2016-05-25 16:15:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:15:15 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:15:15 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[68] at map at GBTs1.scala:54), which has no missing parents -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(10184) called with curMem=3823200, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 9.9 KB, free 726.9 MB) -2016-05-25 16:15:15 INFO MemoryStore:59 - ensureFreeSpace(4760) called with curMem=3833384, maxMem=766075207 -2016-05-25 16:15:15 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 4.6 KB, free 726.9 MB) -2016-05-25 16:15:15 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:54244 (size: 4.6 KB, free: 729.8 MB) -2016-05-25 16:15:15 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:15:15 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 37 (MapPartitionsRDD[68] at map at GBTs1.scala:54) -2016-05-25 16:15:15 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 1 tasks -2016-05-25 16:15:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 71, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:15:15 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 71) -2016-05-25 16:15:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:15:15 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 71). 2401 bytes result sent to driver -2016-05-25 16:15:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 71) in 6 ms on localhost (1/1) -2016-05-25 16:15:15 INFO DAGScheduler:59 - ResultStage 37 (take at GBTs1.scala:71) finished in 0.007 s -2016-05-25 16:15:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-05-25 16:15:15 INFO DAGScheduler:59 - Job 26 finished: take at GBTs1.scala:71, took 0.016842 s -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 16:15:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 16:15:15 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 16:15:15 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 16:15:15 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 16:15:15 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 16:15:15 INFO BlockManager:59 - BlockManager stopped -2016-05-25 16:15:15 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 16:15:15 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 16:15:15 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 16:15:15 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 16:15:15 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-9778b023-539b-499f-a1ed-289d549f94a4 -2016-05-25 16:15:15 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 16:15:15 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-05-25 16:37:48 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 16:37:48 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 16:37:49 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 16:37:49 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 16:37:49 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 16:37:50 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 16:37:50 INFO Remoting:74 - Starting remoting -2016-05-25 16:37:50 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:55301] -2016-05-25 16:37:50 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55301. -2016-05-25 16:37:50 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 16:37:50 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 16:37:50 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-1821860b-b6fb-4362-afeb-751e6097cbea -2016-05-25 16:37:50 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 16:37:50 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-56442cfe-6a46-4510-8fe5-616b9522900e\httpd-c3684fcb-b415-459a-bd27-0847d880e570 -2016-05-25 16:37:50 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 16:37:50 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 16:37:50 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55302 -2016-05-25 16:37:50 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55302. -2016-05-25 16:37:50 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 16:37:50 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 16:37:50 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 16:37:50 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 16:37:50 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 16:37:50 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 16:37:50 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 16:37:51 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55321. -2016-05-25 16:37:51 INFO NettyBlockTransferService:59 - Server created on 55321 -2016-05-25 16:37:51 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 16:37:51 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55321 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55321) -2016-05-25 16:37:51 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 16:37:51 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 16:37:51 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 16:37:51 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 16:37:51 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 16:37:51 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55321 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 16:37:51 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 16:37:53 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 16:37:53 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 16:37:53 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 16:37:53 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 16:37:53 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 16:37:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:37:53 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:37:53 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 16:37:53 INFO MemoryStore:59 - ensureFreeSpace(3904) called with curMem=171580, maxMem=766075207 -2016-05-25 16:37:53 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 16:37:53 INFO MemoryStore:59 - ensureFreeSpace(2146) called with curMem=175484, maxMem=766075207 -2016-05-25 16:37:53 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 16:37:53 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55321 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 16:37:53 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 16:37:53 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 16:37:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:37:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:37:53 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 16:37:53 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 16:37:53 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 16:37:53 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 16:37:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:52368+52368 -2016-05-25 16:37:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_libsvm_data.txt:0+52368 -2016-05-25 16:37:54 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 16:37:54 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 16:37:54 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 16:37:54 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 16:37:54 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 16:37:54 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177630, maxMem=766075207 -2016-05-25 16:37:54 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-05-25 16:37:54 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:55321 (size: 83.1 KB, free: 730.5 MB) -2016-05-25 16:37:54 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262710, maxMem=766075207 -2016-05-25 16:37:54 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-05-25 16:37:54 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:55321 (size: 83.6 KB, free: 730.4 MB) -2016-05-25 16:37:54 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 16:37:54 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 16:37:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 414 ms on localhost (1/2) -2016-05-25 16:37:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 396 ms on localhost (2/2) -2016-05-25 16:37:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 16:37:54 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.437 s -2016-05-25 16:37:54 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.587685 s -2016-05-25 16:37:54 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 16:37:54 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 16:37:54 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 16:37:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:37:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:37:54 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 16:37:54 INFO MemoryStore:59 - ensureFreeSpace(4776) called with curMem=348358, maxMem=766075207 -2016-05-25 16:37:54 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-05-25 16:37:54 INFO MemoryStore:59 - ensureFreeSpace(2554) called with curMem=353134, maxMem=766075207 -2016-05-25 16:37:54 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-05-25 16:37:54 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55321 (size: 2.5 KB, free: 730.4 MB) -2016-05-25 16:37:54 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:54 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at map at DecisionTreeMetadata.scala:110) -2016-05-25 16:37:54 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 16:37:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:37:54 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 16:37:54 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-05-25 16:37:54 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:37:54 INFO MemoryStore:59 - ensureFreeSpace(59104) called with curMem=355688, maxMem=766075207 -2016-05-25 16:37:54 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 57.7 KB, free 730.2 MB) -2016-05-25 16:37:54 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:55321 (size: 57.7 KB, free: 730.3 MB) -2016-05-25 16:37:54 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2611 bytes result sent to driver -2016-05-25 16:37:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 69 ms on localhost (1/1) -2016-05-25 16:37:54 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.069 s -2016-05-25 16:37:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 16:37:54 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.094435 s -2016-05-25 16:37:54 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 16:37:54 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 16:37:54 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 16:37:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:37:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:37:54 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 16:37:54 INFO MemoryStore:59 - ensureFreeSpace(4336) called with curMem=414792, maxMem=766075207 -2016-05-25 16:37:54 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.2 KB, free 730.2 MB) -2016-05-25 16:37:54 INFO MemoryStore:59 - ensureFreeSpace(2326) called with curMem=419128, maxMem=766075207 -2016-05-25 16:37:54 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-05-25 16:37:54 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55321 (size: 2.3 KB, free: 730.3 MB) -2016-05-25 16:37:54 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[9] at retag at RandomForest.scala:137) -2016-05-25 16:37:54 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 16:37:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:37:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:37:54 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 16:37:54 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 16:37:54 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:37:54 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-05-25 16:37:54 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 16:37:54 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 16:37:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 14 ms on localhost (1/2) -2016-05-25 16:37:54 INFO MemoryStore:59 - ensureFreeSpace(58048) called with curMem=421454, maxMem=766075207 -2016-05-25 16:37:54 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 56.7 KB, free 730.1 MB) -2016-05-25 16:37:54 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:55321 (size: 56.7 KB, free: 730.3 MB) -2016-05-25 16:37:54 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2662 bytes result sent to driver -2016-05-25 16:37:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 26 ms on localhost (2/2) -2016-05-25 16:37:54 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.027 s -2016-05-25 16:37:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 16:37:54 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.040750 s -2016-05-25 16:37:54 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 16:37:54 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 16:37:54 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 16:37:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:37:54 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:37:54 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 16:37:54 INFO MemoryStore:59 - ensureFreeSpace(5072) called with curMem=479502, maxMem=766075207 -2016-05-25 16:37:54 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.0 KB, free 730.1 MB) -2016-05-25 16:37:54 INFO MemoryStore:59 - ensureFreeSpace(2639) called with curMem=484574, maxMem=766075207 -2016-05-25 16:37:54 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.1 MB) -2016-05-25 16:37:54 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55321 (size: 2.6 KB, free: 730.3 MB) -2016-05-25 16:37:54 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[11] at sample at DecisionTree.scala:989) -2016-05-25 16:37:54 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 16:37:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 16:37:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 16:37:54 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 16:37:54 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 16:37:54 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:37:54 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:37:54 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 60189 bytes result sent to driver -2016-05-25 16:37:54 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 59120 bytes result sent to driver -2016-05-25 16:37:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 33 ms on localhost (1/2) -2016-05-25 16:37:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 34 ms on localhost (2/2) -2016-05-25 16:37:54 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.036 s -2016-05-25 16:37:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 16:37:54 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.053652 s -2016-05-25 16:37:56 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:55321 in memory (size: 2.6 KB, free: 730.3 MB) -2016-05-25 16:37:56 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 16:37:56 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:55321 in memory (size: 2.3 KB, free: 730.3 MB) -2016-05-25 16:37:56 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-05-25 16:37:56 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:55321 in memory (size: 2.5 KB, free: 730.3 MB) -2016-05-25 16:37:56 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-05-25 16:37:56 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:55321 in memory (size: 2.1 KB, free: 730.3 MB) -2016-05-25 16:37:56 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=459460, maxMem=766075207 -2016-05-25 16:37:56 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.1 MB) -2016-05-25 16:37:56 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=459500, maxMem=766075207 -2016-05-25 16:37:56 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.1 MB) -2016-05-25 16:37:56 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55321 (size: 81.0 B, free: 730.3 MB) -2016-05-25 16:37:56 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 16:37:56 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:37:56 INFO DAGScheduler:59 - Registering RDD 14 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:37:56 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:37:56 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:37:56 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 16:37:56 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 16:37:56 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:37:56 INFO MemoryStore:59 - ensureFreeSpace(470496) called with curMem=459581, maxMem=766075207 -2016-05-25 16:37:56 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 459.5 KB, free 729.7 MB) -2016-05-25 16:37:56 INFO MemoryStore:59 - ensureFreeSpace(115439) called with curMem=930077, maxMem=766075207 -2016-05-25 16:37:56 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 112.7 KB, free 729.6 MB) -2016-05-25 16:37:56 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55321 (size: 112.7 KB, free: 730.2 MB) -2016-05-25 16:37:56 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[14] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:37:56 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 16:37:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 16:37:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 16:37:56 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 16:37:56 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 16:37:56 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-05-25 16:37:56 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:37:56 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-05-25 16:37:56 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:37:56 INFO MemoryStore:59 - ensureFreeSpace(100120) called with curMem=1045516, maxMem=766075207 -2016-05-25 16:37:56 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 97.8 KB, free 729.5 MB) -2016-05-25 16:37:56 INFO MemoryStore:59 - ensureFreeSpace(102976) called with curMem=1145636, maxMem=766075207 -2016-05-25 16:37:56 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 100.6 KB, free 729.4 MB) -2016-05-25 16:37:56 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:55321 (size: 97.8 KB, free: 730.1 MB) -2016-05-25 16:37:56 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:55321 (size: 100.6 KB, free: 730.0 MB) -2016-05-25 16:37:56 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 16:37:56 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 16:37:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 251 ms on localhost (1/2) -2016-05-25 16:37:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 253 ms on localhost (2/2) -2016-05-25 16:37:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 16:37:56 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.254 s -2016-05-25 16:37:56 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:37:56 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:37:56 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 16:37:56 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:37:56 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 16:37:56 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:37:56 INFO MemoryStore:59 - ensureFreeSpace(203488) called with curMem=1248612, maxMem=766075207 -2016-05-25 16:37:56 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 198.7 KB, free 729.2 MB) -2016-05-25 16:37:56 INFO MemoryStore:59 - ensureFreeSpace(43429) called with curMem=1452100, maxMem=766075207 -2016-05-25 16:37:56 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 42.4 KB, free 729.2 MB) -2016-05-25 16:37:56 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55321 (size: 42.4 KB, free: 730.0 MB) -2016-05-25 16:37:56 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[16] at map at DecisionTree.scala:645) -2016-05-25 16:37:56 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 16:37:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:37:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:37:56 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 16:37:56 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 16:37:56 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:37:56 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:37:56 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 16:37:56 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-05-25 16:37:56 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 16:37:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 146 ms on localhost (1/2) -2016-05-25 16:37:57 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 16:37:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 374 ms on localhost (2/2) -2016-05-25 16:37:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 16:37:57 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.375 s -2016-05-25 16:37:57 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.690099 s -2016-05-25 16:37:57 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=1495529, maxMem=766075207 -2016-05-25 16:37:57 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 40.0 B, free 729.2 MB) -2016-05-25 16:37:57 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=1495569, maxMem=766075207 -2016-05-25 16:37:57 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 81.0 B, free 729.2 MB) -2016-05-25 16:37:57 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55321 (size: 81.0 B, free: 730.0 MB) -2016-05-25 16:37:57 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 16:37:57 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:37:57 INFO DAGScheduler:59 - Registering RDD 17 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:37:57 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:37:57 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:37:57 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 16:37:57 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 16:37:57 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:37:57 INFO MemoryStore:59 - ensureFreeSpace(470848) called with curMem=1495650, maxMem=766075207 -2016-05-25 16:37:57 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 459.8 KB, free 728.7 MB) -2016-05-25 16:37:57 INFO MemoryStore:59 - ensureFreeSpace(115632) called with curMem=1966498, maxMem=766075207 -2016-05-25 16:37:57 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 112.9 KB, free 728.6 MB) -2016-05-25 16:37:57 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55321 (size: 112.9 KB, free: 729.8 MB) -2016-05-25 16:37:57 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[17] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:37:57 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 16:37:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 16:37:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2189 bytes) -2016-05-25 16:37:57 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 16:37:57 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 16:37:57 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-05-25 16:37:57 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-05-25 16:37:57 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 16:37:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 50 ms on localhost (1/2) -2016-05-25 16:37:57 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 16:37:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 67 ms on localhost (2/2) -2016-05-25 16:37:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 16:37:57 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.069 s -2016-05-25 16:37:57 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:37:57 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:37:57 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 16:37:57 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:37:57 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 16:37:57 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:37:57 INFO MemoryStore:59 - ensureFreeSpace(203488) called with curMem=2082130, maxMem=766075207 -2016-05-25 16:37:57 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 198.7 KB, free 728.4 MB) -2016-05-25 16:37:57 INFO MemoryStore:59 - ensureFreeSpace(43428) called with curMem=2285618, maxMem=766075207 -2016-05-25 16:37:57 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 42.4 KB, free 728.4 MB) -2016-05-25 16:37:57 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55321 (size: 42.4 KB, free: 729.8 MB) -2016-05-25 16:37:57 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[19] at map at DecisionTree.scala:645) -2016-05-25 16:37:57 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 16:37:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:37:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:37:57 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 16:37:57 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 16:37:57 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:37:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:37:57 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 1161 bytes result sent to driver -2016-05-25 16:37:57 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:37:57 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:37:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 23 ms on localhost (1/2) -2016-05-25 16:37:57 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 16:37:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 97 ms on localhost (2/2) -2016-05-25 16:37:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 16:37:57 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.099 s -2016-05-25 16:37:57 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.220907 s -2016-05-25 16:37:57 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-05-25 16:37:57 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 16:37:57 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 16:37:57 INFO RandomForest:59 - init: 1.825187197 - total: 3.025866512 - findSplitsBins: 1.530571157 - findBestSplits: 1.178655587 - chooseSplits: 1.171853074 -2016-05-25 16:37:57 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 16:37:57 INFO DAGScheduler:59 - Got job 6 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 16:37:57 INFO DAGScheduler:59 - Final stage: ResultStage 8(take at DecisionTreeMetadata.scala:110) -2016-05-25 16:37:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:37:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:37:57 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[25] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 16:37:57 INFO MemoryStore:59 - ensureFreeSpace(7928) called with curMem=2125950, maxMem=766075207 -2016-05-25 16:37:57 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 7.7 KB, free 728.6 MB) -2016-05-25 16:37:57 INFO MemoryStore:59 - ensureFreeSpace(4146) called with curMem=2133878, maxMem=766075207 -2016-05-25 16:37:57 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 4.0 KB, free 728.5 MB) -2016-05-25 16:37:57 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55321 (size: 4.0 KB, free: 730.0 MB) -2016-05-25 16:37:57 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:57 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[25] at map at DecisionTreeMetadata.scala:110) -2016-05-25 16:37:57 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-05-25 16:37:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2374 bytes) -2016-05-25 16:37:57 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 16:37:57 INFO CacheManager:59 - Partition rdd_20_0 not found, computing it -2016-05-25 16:37:57 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:37:57 INFO MemoryStore:59 - ensureFreeSpace(1600) called with curMem=2138024, maxMem=766075207 -2016-05-25 16:37:57 INFO MemoryStore:59 - Block rdd_20_0 stored as values in memory (estimated size 1600.0 B, free 728.5 MB) -2016-05-25 16:37:57 INFO BlockManagerInfo:59 - Added rdd_20_0 in memory on localhost:55321 (size: 1600.0 B, free: 730.0 MB) -2016-05-25 16:37:57 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:37:57 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2611 bytes result sent to driver -2016-05-25 16:37:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 22 ms on localhost (1/1) -2016-05-25 16:37:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 16:37:57 INFO DAGScheduler:59 - ResultStage 8 (take at DecisionTreeMetadata.scala:110) finished in 0.022 s -2016-05-25 16:37:57 INFO DAGScheduler:59 - Job 6 finished: take at DecisionTreeMetadata.scala:110, took 0.046624 s -2016-05-25 16:37:57 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 16:37:57 INFO DAGScheduler:59 - Got job 7 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 16:37:57 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at DecisionTreeMetadata.scala:114) -2016-05-25 16:37:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:37:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:37:57 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[24] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 16:37:57 INFO MemoryStore:59 - ensureFreeSpace(7488) called with curMem=2139624, maxMem=766075207 -2016-05-25 16:37:57 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 7.3 KB, free 728.5 MB) -2016-05-25 16:37:57 INFO MemoryStore:59 - ensureFreeSpace(3931) called with curMem=2147112, maxMem=766075207 -2016-05-25 16:37:57 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.8 KB, free 728.5 MB) -2016-05-25 16:37:57 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:55321 (size: 3.8 KB, free: 730.0 MB) -2016-05-25 16:37:57 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[24] at retag at RandomForest.scala:137) -2016-05-25 16:37:57 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 16:37:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 16, localhost, PROCESS_LOCAL, 2374 bytes) -2016-05-25 16:37:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2374 bytes) -2016-05-25 16:37:57 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 16) -2016-05-25 16:37:57 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 17) -2016-05-25 16:37:57 INFO BlockManager:59 - Found block rdd_20_0 locally -2016-05-25 16:37:57 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:37:57 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 16:37:57 INFO CacheManager:59 - Partition rdd_20_1 not found, computing it -2016-05-25 16:37:57 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:37:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 16) in 13 ms on localhost (1/2) -2016-05-25 16:37:57 INFO MemoryStore:59 - ensureFreeSpace(1560) called with curMem=2151043, maxMem=766075207 -2016-05-25 16:37:57 INFO MemoryStore:59 - Block rdd_20_1 stored as values in memory (estimated size 1560.0 B, free 728.5 MB) -2016-05-25 16:37:57 INFO BlockManagerInfo:59 - Added rdd_20_1 in memory on localhost:55321 (size: 1560.0 B, free: 730.0 MB) -2016-05-25 16:37:57 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:37:57 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 17). 2662 bytes result sent to driver -2016-05-25 16:37:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 17) in 18 ms on localhost (2/2) -2016-05-25 16:37:57 INFO DAGScheduler:59 - ResultStage 9 (count at DecisionTreeMetadata.scala:114) finished in 0.019 s -2016-05-25 16:37:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 16:37:57 INFO DAGScheduler:59 - Job 7 finished: count at DecisionTreeMetadata.scala:114, took 0.041088 s -2016-05-25 16:37:57 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 16:37:57 INFO DAGScheduler:59 - Got job 8 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 16:37:57 INFO DAGScheduler:59 - Final stage: ResultStage 10(collect at DecisionTree.scala:989) -2016-05-25 16:37:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:37:57 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:37:57 INFO DAGScheduler:59 - Submitting ResultStage 10 (PartitionwiseSampledRDD[26] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 16:37:57 INFO MemoryStore:59 - ensureFreeSpace(8224) called with curMem=2152603, maxMem=766075207 -2016-05-25 16:37:57 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 8.0 KB, free 728.5 MB) -2016-05-25 16:37:57 INFO MemoryStore:59 - ensureFreeSpace(4236) called with curMem=2160827, maxMem=766075207 -2016-05-25 16:37:57 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 4.1 KB, free 728.5 MB) -2016-05-25 16:37:57 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:55321 (size: 4.1 KB, free: 730.0 MB) -2016-05-25 16:37:57 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (PartitionwiseSampledRDD[26] at sample at DecisionTree.scala:989) -2016-05-25 16:37:57 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-25 16:37:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 18, localhost, PROCESS_LOCAL, 2483 bytes) -2016-05-25 16:37:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2483 bytes) -2016-05-25 16:37:57 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 18) -2016-05-25 16:37:57 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 19) -2016-05-25 16:37:57 INFO BlockManager:59 - Found block rdd_20_0 locally -2016-05-25 16:37:57 INFO BlockManager:59 - Found block rdd_20_1 locally -2016-05-25 16:37:57 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:37:57 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:37:57 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 18). 60189 bytes result sent to driver -2016-05-25 16:37:57 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 19). 59120 bytes result sent to driver -2016-05-25 16:37:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 18) in 15 ms on localhost (1/2) -2016-05-25 16:37:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 19) in 16 ms on localhost (2/2) -2016-05-25 16:37:57 INFO DAGScheduler:59 - ResultStage 10 (collect at DecisionTree.scala:989) finished in 0.017 s -2016-05-25 16:37:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 16:37:57 INFO DAGScheduler:59 - Job 8 finished: collect at DecisionTree.scala:989, took 0.038140 s -2016-05-25 16:37:57 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=2165063, maxMem=766075207 -2016-05-25 16:37:57 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 40.0 B, free 728.5 MB) -2016-05-25 16:37:57 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=2165103, maxMem=766075207 -2016-05-25 16:37:57 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 81.0 B, free 728.5 MB) -2016-05-25 16:37:57 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:55321 (size: 81.0 B, free: 730.0 MB) -2016-05-25 16:37:57 INFO SparkContext:59 - Created broadcast 14 from broadcast at DecisionTree.scala:604 -2016-05-25 16:37:57 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:37:57 INFO DAGScheduler:59 - Registering RDD 29 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:37:57 INFO DAGScheduler:59 - Got job 9 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:37:57 INFO DAGScheduler:59 - Final stage: ResultStage 12(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:37:57 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 11) -2016-05-25 16:37:57 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 11) -2016-05-25 16:37:57 INFO DAGScheduler:59 - Submitting ShuffleMapStage 11 (MapPartitionsRDD[29] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:37:57 INFO MemoryStore:59 - ensureFreeSpace(472400) called with curMem=2165184, maxMem=766075207 -2016-05-25 16:37:57 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 461.3 KB, free 728.1 MB) -2016-05-25 16:37:57 INFO MemoryStore:59 - ensureFreeSpace(116257) called with curMem=2637584, maxMem=766075207 -2016-05-25 16:37:57 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 113.5 KB, free 728.0 MB) -2016-05-25 16:37:57 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:55321 (size: 113.5 KB, free: 729.9 MB) -2016-05-25 16:37:57 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 11 (MapPartitionsRDD[29] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:37:57 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-25 16:37:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 20, localhost, PROCESS_LOCAL, 2363 bytes) -2016-05-25 16:37:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2363 bytes) -2016-05-25 16:37:57 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 21) -2016-05-25 16:37:57 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 20) -2016-05-25 16:37:57 INFO CacheManager:59 - Partition rdd_28_1 not found, computing it -2016-05-25 16:37:57 INFO CacheManager:59 - Partition rdd_28_0 not found, computing it -2016-05-25 16:37:57 INFO BlockManager:59 - Found block rdd_20_0 locally -2016-05-25 16:37:57 INFO BlockManager:59 - Found block rdd_20_1 locally -2016-05-25 16:37:57 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:37:57 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(100120) called with curMem=2753841, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block rdd_28_1 stored as values in memory (estimated size 97.8 KB, free 727.9 MB) -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(102976) called with curMem=2853961, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block rdd_28_0 stored as values in memory (estimated size 100.6 KB, free 727.8 MB) -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Added rdd_28_1 in memory on localhost:55321 (size: 97.8 KB, free: 729.8 MB) -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Added rdd_28_0 in memory on localhost:55321 (size: 100.6 KB, free: 729.7 MB) -2016-05-25 16:37:58 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 21). 2834 bytes result sent to driver -2016-05-25 16:37:58 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 20). 2834 bytes result sent to driver -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 21) in 62 ms on localhost (1/2) -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 20) in 64 ms on localhost (2/2) -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 16:37:58 INFO DAGScheduler:59 - ShuffleMapStage 11 (mapPartitions at DecisionTree.scala:625) finished in 0.065 s -2016-05-25 16:37:58 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:37:58 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:37:58 INFO DAGScheduler:59 - waiting: Set(ResultStage 12) -2016-05-25 16:37:58 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:37:58 INFO DAGScheduler:59 - Missing parents for ResultStage 12: List() -2016-05-25 16:37:58 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[31] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(203488) called with curMem=2956937, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 198.7 KB, free 727.6 MB) -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(43423) called with curMem=3160425, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 42.4 KB, free 727.5 MB) -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:55321 (size: 42.4 KB, free: 729.6 MB) -2016-05-25 16:37:58 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[31] at map at DecisionTree.scala:645) -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-25 16:37:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 22, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:55321 in memory (size: 4.1 KB, free: 729.6 MB) -2016-05-25 16:37:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:37:58 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 22) -2016-05-25 16:37:58 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 23) -2016-05-25 16:37:58 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:55321 in memory (size: 3.8 KB, free: 729.6 MB) -2016-05-25 16:37:58 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-05-25 16:37:58 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:37:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:37:58 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:37:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:37:58 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 23). 1161 bytes result sent to driver -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:55321 in memory (size: 4.0 KB, free: 729.6 MB) -2016-05-25 16:37:58 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 23) in 15 ms on localhost (1/2) -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:55321 in memory (size: 42.4 KB, free: 729.7 MB) -2016-05-25 16:37:58 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:55321 in memory (size: 112.9 KB, free: 729.8 MB) -2016-05-25 16:37:58 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 16:37:58 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-05-25 16:37:58 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 22). 2774 bytes result sent to driver -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 22) in 39 ms on localhost (2/2) -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:55321 in memory (size: 81.0 B, free: 729.8 MB) -2016-05-25 16:37:58 INFO DAGScheduler:59 - ResultStage 12 (collectAsMap at DecisionTree.scala:654) finished in 0.048 s -2016-05-25 16:37:58 INFO DAGScheduler:59 - Job 9 finished: collectAsMap at DecisionTree.scala:654, took 0.200269 s -2016-05-25 16:37:58 INFO MapPartitionsRDD:59 - Removing RDD 28 from persistence list -2016-05-25 16:37:58 INFO BlockManager:59 - Removing RDD 28 -2016-05-25 16:37:58 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 16:37:58 INFO RandomForest:59 - init: 0.343678927 - total: 0.618418416 - findSplitsBins: 0.199371018 - findBestSplits: 0.26836534 - chooseSplits: 0.268215412 -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:55321 in memory (size: 42.4 KB, free: 730.0 MB) -2016-05-25 16:37:58 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:55321 in memory (size: 112.7 KB, free: 730.1 MB) -2016-05-25 16:37:58 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 16:37:58 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55321 in memory (size: 81.0 B, free: 730.1 MB) -2016-05-25 16:37:58 INFO BlockManager:59 - Removing RDD 13 -2016-05-25 16:37:58 INFO ContextCleaner:59 - Cleaned RDD 13 -2016-05-25 16:37:58 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 16:37:58 INFO DAGScheduler:59 - Got job 10 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 16:37:58 INFO DAGScheduler:59 - Final stage: ResultStage 13(take at DecisionTreeMetadata.scala:110) -2016-05-25 16:37:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:37:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:37:58 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[37] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(8392) called with curMem=1298309, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 8.2 KB, free 729.3 MB) -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(4289) called with curMem=1306701, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 4.2 KB, free 729.3 MB) -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:55321 (size: 4.2 KB, free: 730.1 MB) -2016-05-25 16:37:58 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:58 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 13 (MapPartitionsRDD[37] at map at DecisionTreeMetadata.scala:110) -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 1 tasks -2016-05-25 16:37:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 24, localhost, PROCESS_LOCAL, 2406 bytes) -2016-05-25 16:37:58 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 24) -2016-05-25 16:37:58 INFO CacheManager:59 - Partition rdd_33_0 not found, computing it -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_20_0 locally -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(1600) called with curMem=1310990, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block rdd_33_0 stored as values in memory (estimated size 1600.0 B, free 729.3 MB) -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Added rdd_33_0 in memory on localhost:55321 (size: 1600.0 B, free: 730.1 MB) -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:37:58 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 24). 2611 bytes result sent to driver -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 24) in 15 ms on localhost (1/1) -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 16:37:58 INFO DAGScheduler:59 - ResultStage 13 (take at DecisionTreeMetadata.scala:110) finished in 0.016 s -2016-05-25 16:37:58 INFO DAGScheduler:59 - Job 10 finished: take at DecisionTreeMetadata.scala:110, took 0.033018 s -2016-05-25 16:37:58 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 16:37:58 INFO DAGScheduler:59 - Got job 11 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 16:37:58 INFO DAGScheduler:59 - Final stage: ResultStage 14(count at DecisionTreeMetadata.scala:114) -2016-05-25 16:37:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:37:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:37:58 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[36] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(7960) called with curMem=1312590, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 7.8 KB, free 729.3 MB) -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(4090) called with curMem=1320550, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 4.0 KB, free 729.3 MB) -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:55321 (size: 4.0 KB, free: 730.1 MB) -2016-05-25 16:37:58 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[36] at retag at RandomForest.scala:137) -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-25 16:37:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 25, localhost, PROCESS_LOCAL, 2406 bytes) -2016-05-25 16:37:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 26, localhost, PROCESS_LOCAL, 2406 bytes) -2016-05-25 16:37:58 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 25) -2016-05-25 16:37:58 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 26) -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_33_0 locally -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:37:58 INFO CacheManager:59 - Partition rdd_33_1 not found, computing it -2016-05-25 16:37:58 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 25). 2082 bytes result sent to driver -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_20_1 locally -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 25) in 11 ms on localhost (1/2) -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(1560) called with curMem=1324640, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block rdd_33_1 stored as values in memory (estimated size 1560.0 B, free 729.3 MB) -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Added rdd_33_1 in memory on localhost:55321 (size: 1560.0 B, free: 730.1 MB) -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:37:58 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 26). 2662 bytes result sent to driver -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 26) in 17 ms on localhost (2/2) -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 16:37:58 INFO DAGScheduler:59 - ResultStage 14 (count at DecisionTreeMetadata.scala:114) finished in 0.018 s -2016-05-25 16:37:58 INFO DAGScheduler:59 - Job 11 finished: count at DecisionTreeMetadata.scala:114, took 0.036104 s -2016-05-25 16:37:58 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 16:37:58 INFO DAGScheduler:59 - Got job 12 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 16:37:58 INFO DAGScheduler:59 - Final stage: ResultStage 15(collect at DecisionTree.scala:989) -2016-05-25 16:37:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:37:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:37:58 INFO DAGScheduler:59 - Submitting ResultStage 15 (PartitionwiseSampledRDD[38] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(8688) called with curMem=1326200, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 8.5 KB, free 729.3 MB) -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(4381) called with curMem=1334888, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 4.3 KB, free 729.3 MB) -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:55321 (size: 4.3 KB, free: 730.1 MB) -2016-05-25 16:37:58 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (PartitionwiseSampledRDD[38] at sample at DecisionTree.scala:989) -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-25 16:37:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 27, localhost, PROCESS_LOCAL, 2515 bytes) -2016-05-25 16:37:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 28, localhost, PROCESS_LOCAL, 2515 bytes) -2016-05-25 16:37:58 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 27) -2016-05-25 16:37:58 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 28) -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_33_1 locally -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_33_0 locally -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:37:58 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 28). 59120 bytes result sent to driver -2016-05-25 16:37:58 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 27). 60189 bytes result sent to driver -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 28) in 13 ms on localhost (1/2) -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 27) in 15 ms on localhost (2/2) -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-25 16:37:58 INFO DAGScheduler:59 - ResultStage 15 (collect at DecisionTree.scala:989) finished in 0.016 s -2016-05-25 16:37:58 INFO DAGScheduler:59 - Job 12 finished: collect at DecisionTree.scala:989, took 0.038190 s -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=1339269, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 40.0 B, free 729.3 MB) -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=1339309, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 81.0 B, free 729.3 MB) -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:55321 (size: 81.0 B, free: 730.1 MB) -2016-05-25 16:37:58 INFO SparkContext:59 - Created broadcast 20 from broadcast at DecisionTree.scala:604 -2016-05-25 16:37:58 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:37:58 INFO DAGScheduler:59 - Registering RDD 41 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:37:58 INFO DAGScheduler:59 - Got job 13 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:37:58 INFO DAGScheduler:59 - Final stage: ResultStage 17(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:37:58 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 16) -2016-05-25 16:37:58 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 16) -2016-05-25 16:37:58 INFO DAGScheduler:59 - Submitting ShuffleMapStage 16 (MapPartitionsRDD[41] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(472864) called with curMem=1339390, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 461.8 KB, free 728.9 MB) -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(116173) called with curMem=1812254, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 113.5 KB, free 728.7 MB) -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:55321 (size: 113.5 KB, free: 730.0 MB) -2016-05-25 16:37:58 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 16 (MapPartitionsRDD[41] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-25 16:37:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 29, localhost, PROCESS_LOCAL, 2395 bytes) -2016-05-25 16:37:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 30, localhost, PROCESS_LOCAL, 2395 bytes) -2016-05-25 16:37:58 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 29) -2016-05-25 16:37:58 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 30) -2016-05-25 16:37:58 INFO CacheManager:59 - Partition rdd_40_0 not found, computing it -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_33_0 locally -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:37:58 INFO CacheManager:59 - Partition rdd_40_1 not found, computing it -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(102976) called with curMem=1928427, maxMem=766075207 -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_33_1 locally -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:37:58 INFO MemoryStore:59 - Block rdd_40_0 stored as values in memory (estimated size 100.6 KB, free 728.6 MB) -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Added rdd_40_0 in memory on localhost:55321 (size: 100.6 KB, free: 729.9 MB) -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(100120) called with curMem=2031403, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block rdd_40_1 stored as values in memory (estimated size 97.8 KB, free 728.6 MB) -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Added rdd_40_1 in memory on localhost:55321 (size: 97.8 KB, free: 729.8 MB) -2016-05-25 16:37:58 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 29). 2834 bytes result sent to driver -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 29) in 46 ms on localhost (1/2) -2016-05-25 16:37:58 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 30). 2834 bytes result sent to driver -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 30) in 51 ms on localhost (2/2) -2016-05-25 16:37:58 INFO DAGScheduler:59 - ShuffleMapStage 16 (mapPartitions at DecisionTree.scala:625) finished in 0.052 s -2016-05-25 16:37:58 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-25 16:37:58 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:37:58 INFO DAGScheduler:59 - waiting: Set(ResultStage 17) -2016-05-25 16:37:58 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:37:58 INFO DAGScheduler:59 - Missing parents for ResultStage 17: List() -2016-05-25 16:37:58 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[43] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(203488) called with curMem=2131523, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 198.7 KB, free 728.4 MB) -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(43424) called with curMem=2335011, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 42.4 KB, free 728.3 MB) -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:55321 (size: 42.4 KB, free: 729.8 MB) -2016-05-25 16:37:58 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[43] at map at DecisionTree.scala:645) -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-05-25 16:37:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 31, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:37:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 32, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:37:58 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 31) -2016-05-25 16:37:58 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 32) -2016-05-25 16:37:58 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:37:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:37:58 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:37:58 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:37:58 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 32). 1161 bytes result sent to driver -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 32) in 10 ms on localhost (1/2) -2016-05-25 16:37:58 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 31). 2774 bytes result sent to driver -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 31) in 24 ms on localhost (2/2) -2016-05-25 16:37:58 INFO DAGScheduler:59 - ResultStage 17 (collectAsMap at DecisionTree.scala:654) finished in 0.025 s -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-25 16:37:58 INFO DAGScheduler:59 - Job 13 finished: collectAsMap at DecisionTree.scala:654, took 0.123432 s -2016-05-25 16:37:58 INFO MapPartitionsRDD:59 - Removing RDD 40 from persistence list -2016-05-25 16:37:58 INFO BlockManager:59 - Removing RDD 40 -2016-05-25 16:37:58 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 16:37:58 INFO RandomForest:59 - init: 0.204316395 - total: 0.381465568 - findSplitsBins: 0.099841015 - findBestSplits: 0.173215361 - chooseSplits: 0.173106038 -2016-05-25 16:37:58 INFO GradientBoostedTrees:59 - Internal timing for DecisionTree: -2016-05-25 16:37:58 INFO GradientBoostedTrees:59 - init: 0.006987693 - total: 4.116649406 - building tree 0: 3.045207161 - building tree 2: 0.38204966 - building tree 1: 0.620484382 -2016-05-25 16:37:58 INFO MapPartitionsRDD:59 - Removing RDD 7 from persistence list -2016-05-25 16:37:58 INFO BlockManager:59 - Removing RDD 7 -2016-05-25 16:37:58 INFO SparkContext:59 - Starting job: mean at GBTs2Regression.scala:63 -2016-05-25 16:37:58 INFO DAGScheduler:59 - Got job 14 (mean at GBTs2Regression.scala:63) with 2 output partitions -2016-05-25 16:37:58 INFO DAGScheduler:59 - Final stage: ResultStage 18(mean at GBTs2Regression.scala:63) -2016-05-25 16:37:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:37:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:37:58 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[48] at mean at GBTs2Regression.scala:63), which has no missing parents -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(8328) called with curMem=2058187, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 8.1 KB, free 728.6 MB) -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(4253) called with curMem=2066515, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 4.2 KB, free 728.6 MB) -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:55321 (size: 4.2 KB, free: 730.1 MB) -2016-05-25 16:37:58 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[48] at mean at GBTs2Regression.scala:63) -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-25 16:37:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 33, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:37:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 34, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:37:58 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 33) -2016-05-25 16:37:58 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 34) -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:37:58 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-05-25 16:37:58 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-05-25 16:37:58 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 33). 2209 bytes result sent to driver -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 16:37:58 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 34). 2209 bytes result sent to driver -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 34) in 20 ms on localhost (1/2) -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 33) in 23 ms on localhost (2/2) -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-25 16:37:58 INFO DAGScheduler:59 - ResultStage 18 (mean at GBTs2Regression.scala:63) finished in 0.025 s -2016-05-25 16:37:58 INFO DAGScheduler:59 - Job 14 finished: mean at GBTs2Regression.scala:63, took 0.040300 s -2016-05-25 16:37:58 INFO SparkContext:59 - Starting job: count at GBTs2Regression.scala:69 -2016-05-25 16:37:58 INFO DAGScheduler:59 - Got job 15 (count at GBTs2Regression.scala:69) with 2 output partitions -2016-05-25 16:37:58 INFO DAGScheduler:59 - Final stage: ResultStage 19(count at GBTs2Regression.scala:69) -2016-05-25 16:37:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:37:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:37:58 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(3816) called with curMem=2070768, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 3.7 KB, free 728.6 MB) -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(2142) called with curMem=2074584, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.1 KB, free 728.6 MB) -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:55321 (size: 2.1 KB, free: 730.1 MB) -2016-05-25 16:37:58 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-25 16:37:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 35, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:37:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 36, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:37:58 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 36) -2016-05-25 16:37:58 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 35) -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:37:58 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 36). 2082 bytes result sent to driver -2016-05-25 16:37:58 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 35). 2082 bytes result sent to driver -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 36) in 8 ms on localhost (1/2) -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 35) in 11 ms on localhost (2/2) -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-25 16:37:58 INFO DAGScheduler:59 - ResultStage 19 (count at GBTs2Regression.scala:69) finished in 0.013 s -2016-05-25 16:37:58 INFO DAGScheduler:59 - Job 15 finished: count at GBTs2Regression.scala:69, took 0.026025 s -2016-05-25 16:37:58 INFO SparkContext:59 - Starting job: count at GBTs2Regression.scala:70 -2016-05-25 16:37:58 INFO DAGScheduler:59 - Got job 16 (count at GBTs2Regression.scala:70) with 2 output partitions -2016-05-25 16:37:58 INFO DAGScheduler:59 - Final stage: ResultStage 20(count at GBTs2Regression.scala:70) -2016-05-25 16:37:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:37:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:37:58 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[7] at randomSplit at GBTs2Regression.scala:37), which has no missing parents -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2076726, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 4.0 KB, free 728.6 MB) -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(2265) called with curMem=2080838, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.6 MB) -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:55321 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 16:37:58 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[7] at randomSplit at GBTs2Regression.scala:37) -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-25 16:37:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 37, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:37:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 38, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:37:58 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 38) -2016-05-25 16:37:58 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 37) -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 16:37:58 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 37). 2082 bytes result sent to driver -2016-05-25 16:37:58 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 38). 2082 bytes result sent to driver -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 37) in 8 ms on localhost (1/2) -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 38) in 7 ms on localhost (2/2) -2016-05-25 16:37:58 INFO DAGScheduler:59 - ResultStage 20 (count at GBTs2Regression.scala:70) finished in 0.008 s -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-25 16:37:58 INFO DAGScheduler:59 - Job 16 finished: count at GBTs2Regression.scala:70, took 0.020946 s -2016-05-25 16:37:58 INFO SparkContext:59 - Starting job: count at GBTs2Regression.scala:71 -2016-05-25 16:37:58 INFO DAGScheduler:59 - Got job 17 (count at GBTs2Regression.scala:71) with 2 output partitions -2016-05-25 16:37:58 INFO DAGScheduler:59 - Final stage: ResultStage 21(count at GBTs2Regression.scala:71) -2016-05-25 16:37:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:37:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:37:58 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[8] at randomSplit at GBTs2Regression.scala:37), which has no missing parents -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(4112) called with curMem=2083103, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 4.0 KB, free 728.6 MB) -2016-05-25 16:37:58 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=2087215, maxMem=766075207 -2016-05-25 16:37:58 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.6 MB) -2016-05-25 16:37:58 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:55321 (size: 2.2 KB, free: 730.1 MB) -2016-05-25 16:37:58 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:37:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[8] at randomSplit at GBTs2Regression.scala:37) -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-25 16:37:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 39, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:37:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 40, localhost, PROCESS_LOCAL, 2200 bytes) -2016-05-25 16:37:58 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 39) -2016-05-25 16:37:58 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 40) -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:37:58 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 16:37:58 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 40). 2082 bytes result sent to driver -2016-05-25 16:37:58 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 39). 2082 bytes result sent to driver -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 40) in 6 ms on localhost (1/2) -2016-05-25 16:37:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 39) in 7 ms on localhost (2/2) -2016-05-25 16:37:58 INFO DAGScheduler:59 - ResultStage 21 (count at GBTs2Regression.scala:71) finished in 0.008 s -2016-05-25 16:37:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-25 16:37:58 INFO DAGScheduler:59 - Job 17 finished: count at GBTs2Regression.scala:71, took 0.016746 s -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 16:37:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 16:37:58 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 16:37:58 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 16:37:58 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 16:37:58 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 16:37:58 INFO BlockManager:59 - BlockManager stopped -2016-05-25 16:37:58 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 16:37:59 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 16:37:59 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 16:37:59 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 16:37:59 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-56442cfe-6a46-4510-8fe5-616b9522900e -2016-05-25 16:37:59 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 16:42:36 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 16:42:36 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 16:42:37 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 16:42:37 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 16:42:37 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 16:42:37 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 16:42:38 INFO Remoting:74 - Starting remoting -2016-05-25 16:42:38 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:55567] -2016-05-25 16:42:38 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55567. -2016-05-25 16:42:38 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 16:42:38 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 16:42:38 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-5dd6c488-7201-484b-84ab-54b63861c844 -2016-05-25 16:42:38 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 16:42:38 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-37658118-9024-4d92-92d2-6252dfa4a9a9\httpd-3a9af64e-8f77-47c6-9fa0-29abbcd1639d -2016-05-25 16:42:38 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 16:42:38 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 16:42:38 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55568 -2016-05-25 16:42:38 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55568. -2016-05-25 16:42:38 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 16:42:38 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 16:42:38 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 16:42:38 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 16:42:38 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 16:42:38 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 16:42:38 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 16:42:39 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55587. -2016-05-25 16:42:39 INFO NettyBlockTransferService:59 - Server created on 55587 -2016-05-25 16:42:39 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 16:42:39 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55587 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55587) -2016-05-25 16:42:39 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 16:42:39 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 16:42:39 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 16:42:39 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 16:42:39 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 16:42:39 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55587 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 16:42:39 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 16:42:41 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 16:42:41 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 16:42:41 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 16:42:41 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 16:42:41 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 16:42:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:42:41 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:42:41 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 16:42:41 INFO MemoryStore:59 - ensureFreeSpace(3888) called with curMem=171580, maxMem=766075207 -2016-05-25 16:42:41 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 16:42:41 INFO MemoryStore:59 - ensureFreeSpace(2129) called with curMem=175468, maxMem=766075207 -2016-05-25 16:42:41 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 16:42:41 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55587 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 16:42:41 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 16:42:41 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 16:42:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 16:42:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 16:42:41 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 16:42:41 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 16:42:41 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 16:42:41 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 16:42:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/dt.txt:0+57 -2016-05-25 16:42:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/dt.txt:57+57 -2016-05-25 16:42:41 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 16:42:41 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 16:42:41 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 16:42:41 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 16:42:41 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 16:42:41 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=177597, maxMem=766075207 -2016-05-25 16:42:41 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 280.0 B, free 730.4 MB) -2016-05-25 16:42:41 INFO MemoryStore:59 - ensureFreeSpace(544) called with curMem=177877, maxMem=766075207 -2016-05-25 16:42:41 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:55587 (size: 280.0 B, free: 730.6 MB) -2016-05-25 16:42:41 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 544.0 B, free 730.4 MB) -2016-05-25 16:42:41 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:55587 (size: 544.0 B, free: 730.6 MB) -2016-05-25 16:42:41 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 16:42:41 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 16:42:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 143 ms on localhost (1/2) -2016-05-25 16:42:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 124 ms on localhost (2/2) -2016-05-25 16:42:41 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.163 s -2016-05-25 16:42:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 16:42:41 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.314434 s -2016-05-25 16:42:42 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 16:42:42 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 16:42:42 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 16:42:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:42:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:42:42 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 16:42:42 INFO MemoryStore:59 - ensureFreeSpace(4600) called with curMem=178421, maxMem=766075207 -2016-05-25 16:42:42 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.4 MB) -2016-05-25 16:42:42 INFO MemoryStore:59 - ensureFreeSpace(2451) called with curMem=183021, maxMem=766075207 -2016-05-25 16:42:42 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-05-25 16:42:42 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55587 (size: 2.4 KB, free: 730.6 MB) -2016-05-25 16:42:42 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:42 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at DecisionTreeMetadata.scala:110) -2016-05-25 16:42:42 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 16:42:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 16:42:42 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 16:42:42 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-05-25 16:42:42 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:42:42 INFO MemoryStore:59 - ensureFreeSpace(544) called with curMem=185472, maxMem=766075207 -2016-05-25 16:42:42 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 544.0 B, free 730.4 MB) -2016-05-25 16:42:42 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:55587 (size: 544.0 B, free: 730.6 MB) -2016-05-25 16:42:42 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2611 bytes result sent to driver -2016-05-25 16:42:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 29 ms on localhost (1/1) -2016-05-25 16:42:42 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.030 s -2016-05-25 16:42:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 16:42:42 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.051410 s -2016-05-25 16:42:42 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 16:42:42 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 16:42:42 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 16:42:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:42:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:42:42 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[8] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 16:42:42 INFO MemoryStore:59 - ensureFreeSpace(4168) called with curMem=186016, maxMem=766075207 -2016-05-25 16:42:42 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.4 MB) -2016-05-25 16:42:42 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=190184, maxMem=766075207 -2016-05-25 16:42:42 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-05-25 16:42:42 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55587 (size: 2.2 KB, free: 730.6 MB) -2016-05-25 16:42:42 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[8] at retag at RandomForest.scala:137) -2016-05-25 16:42:42 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 16:42:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 16:42:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 16:42:42 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 16:42:42 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 16:42:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:42:42 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 16:42:42 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-05-25 16:42:42 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 16:42:42 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=192422, maxMem=766075207 -2016-05-25 16:42:42 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 280.0 B, free 730.4 MB) -2016-05-25 16:42:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 15 ms on localhost (1/2) -2016-05-25 16:42:42 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:55587 (size: 280.0 B, free: 730.6 MB) -2016-05-25 16:42:42 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2662 bytes result sent to driver -2016-05-25 16:42:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 21 ms on localhost (2/2) -2016-05-25 16:42:42 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.022 s -2016-05-25 16:42:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 16:42:42 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.039055 s -2016-05-25 16:42:42 WARN DecisionTreeMetadata:71 - DecisionTree reducing maxBins from 32 to 6 (= number of training instances) -2016-05-25 16:42:42 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 16:42:42 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 16:42:42 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 16:42:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:42:42 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:42:42 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[10] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 16:42:42 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=192702, maxMem=766075207 -2016-05-25 16:42:42 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-25 16:42:42 INFO MemoryStore:59 - ensureFreeSpace(2552) called with curMem=197598, maxMem=766075207 -2016-05-25 16:42:42 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.4 MB) -2016-05-25 16:42:42 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55587 (size: 2.5 KB, free: 730.6 MB) -2016-05-25 16:42:42 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[10] at sample at DecisionTree.scala:989) -2016-05-25 16:42:42 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 16:42:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 16:42:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 16:42:42 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 16:42:42 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 16:42:42 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:42:42 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:42:42 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2493 bytes result sent to driver -2016-05-25 16:42:42 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2677 bytes result sent to driver -2016-05-25 16:42:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 20 ms on localhost (1/2) -2016-05-25 16:42:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 23 ms on localhost (2/2) -2016-05-25 16:42:42 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.024 s -2016-05-25 16:42:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 16:42:42 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.040764 s -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=200150, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.4 MB) -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=200190, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.4 MB) -2016-05-25 16:42:43 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55587 (size: 81.0 B, free: 730.6 MB) -2016-05-25 16:42:43 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 16:42:43 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:42:43 INFO DAGScheduler:59 - Registering RDD 13 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:42:43 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:42:43 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:42:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 16:42:43 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 16:42:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[13] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(9880) called with curMem=200271, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 9.6 KB, free 730.4 MB) -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(4976) called with curMem=210151, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 4.9 KB, free 730.4 MB) -2016-05-25 16:42:43 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55587 (size: 4.9 KB, free: 730.6 MB) -2016-05-25 16:42:43 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[13] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:42:43 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 16:42:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 16:42:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 16:42:43 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 16:42:43 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 16:42:43 INFO CacheManager:59 - Partition rdd_12_0 not found, computing it -2016-05-25 16:42:43 INFO CacheManager:59 - Partition rdd_12_1 not found, computing it -2016-05-25 16:42:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:42:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(232) called with curMem=215127, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block rdd_12_1 stored as values in memory (estimated size 232.0 B, free 730.4 MB) -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(448) called with curMem=215359, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block rdd_12_0 stored as values in memory (estimated size 448.0 B, free 730.4 MB) -2016-05-25 16:42:43 INFO BlockManagerInfo:59 - Added rdd_12_1 in memory on localhost:55587 (size: 232.0 B, free: 730.6 MB) -2016-05-25 16:42:43 INFO BlockManagerInfo:59 - Added rdd_12_0 in memory on localhost:55587 (size: 448.0 B, free: 730.6 MB) -2016-05-25 16:42:43 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 16:42:43 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 16:42:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 83 ms on localhost (1/2) -2016-05-25 16:42:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 81 ms on localhost (2/2) -2016-05-25 16:42:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 16:42:43 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.085 s -2016-05-25 16:42:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:42:43 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:42:43 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 16:42:43 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:42:43 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 16:42:43 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=215807, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(2873) called with curMem=221039, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-25 16:42:43 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55587 (size: 2.8 KB, free: 730.6 MB) -2016-05-25 16:42:43 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at map at DecisionTree.scala:645) -2016-05-25 16:42:43 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 16:42:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:42:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:42:43 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 16:42:43 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 16:42:43 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:42:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:42:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 16:42:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 16:42:43 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 16:42:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 32 ms on localhost (1/2) -2016-05-25 16:42:43 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 16:42:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 59 ms on localhost (2/2) -2016-05-25 16:42:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 16:42:43 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.060 s -2016-05-25 16:42:43 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.189299 s -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=223912, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 40.0 B, free 730.4 MB) -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=223952, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.4 MB) -2016-05-25 16:42:43 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55587 (size: 81.0 B, free: 730.6 MB) -2016-05-25 16:42:43 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 16:42:43 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:42:43 INFO DAGScheduler:59 - Registering RDD 16 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:42:43 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:42:43 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:42:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 16:42:43 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 16:42:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[16] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(10240) called with curMem=224033, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 10.0 KB, free 730.4 MB) -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(5159) called with curMem=234273, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 16:42:43 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55587 (size: 5.0 KB, free: 730.5 MB) -2016-05-25 16:42:43 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[16] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:42:43 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 16:42:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 16:42:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 16:42:43 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 16:42:43 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 16:42:43 INFO BlockManager:59 - Found block rdd_12_1 locally -2016-05-25 16:42:43 INFO BlockManager:59 - Found block rdd_12_0 locally -2016-05-25 16:42:43 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 16:42:43 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 16:42:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 52 ms on localhost (1/2) -2016-05-25 16:42:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 56 ms on localhost (2/2) -2016-05-25 16:42:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 16:42:43 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.058 s -2016-05-25 16:42:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:42:43 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:42:43 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 16:42:43 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:42:43 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 16:42:43 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[18] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=239432, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(2869) called with curMem=244664, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-25 16:42:43 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55587 (size: 2.8 KB, free: 730.5 MB) -2016-05-25 16:42:43 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[18] at map at DecisionTree.scala:645) -2016-05-25 16:42:43 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 16:42:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:42:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:42:43 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 16:42:43 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:55587 in memory (size: 4.9 KB, free: 730.6 MB) -2016-05-25 16:42:43 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 16:42:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:42:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:42:43 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:42:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:42:43 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 1161 bytes result sent to driver -2016-05-25 16:42:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 18 ms on localhost (1/2) -2016-05-25 16:42:43 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 16:42:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 27 ms on localhost (2/2) -2016-05-25 16:42:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 16:42:43 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 16:42:43 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.029 s -2016-05-25 16:42:43 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.132474 s -2016-05-25 16:42:43 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-05-25 16:42:43 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 16:42:43 INFO BlockManager:59 - Removing RDD 12 -2016-05-25 16:42:43 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55587 in memory (size: 81.0 B, free: 730.6 MB) -2016-05-25 16:42:43 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 16:42:43 INFO RandomForest:59 - init: 1.196611638 - total: 1.659771637 - findSplitsBins: 1.049987439 - findBestSplits: 0.435136494 - chooseSplits: 0.42964763 -2016-05-25 16:42:43 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:55587 in memory (size: 2.5 KB, free: 730.6 MB) -2016-05-25 16:42:43 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 16:42:43 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:55587 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-25 16:42:43 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 16:42:43 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 16:42:43 INFO DAGScheduler:59 - Got job 6 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 16:42:43 INFO DAGScheduler:59 - Final stage: ResultStage 8(take at DecisionTreeMetadata.scala:110) -2016-05-25 16:42:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:42:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:42:43 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[24] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(7760) called with curMem=216323, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 7.6 KB, free 730.4 MB) -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(3994) called with curMem=224083, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.9 KB, free 730.4 MB) -2016-05-25 16:42:43 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55587 (size: 3.9 KB, free: 730.6 MB) -2016-05-25 16:42:43 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:43 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[24] at map at DecisionTreeMetadata.scala:110) -2016-05-25 16:42:43 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-05-25 16:42:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2358 bytes) -2016-05-25 16:42:43 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 16:42:43 INFO CacheManager:59 - Partition rdd_19_0 not found, computing it -2016-05-25 16:42:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=228077, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block rdd_19_0 stored as values in memory (estimated size 192.0 B, free 730.4 MB) -2016-05-25 16:42:43 INFO BlockManagerInfo:59 - Added rdd_19_0 in memory on localhost:55587 (size: 192.0 B, free: 730.6 MB) -2016-05-25 16:42:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:42:43 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2611 bytes result sent to driver -2016-05-25 16:42:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 30 ms on localhost (1/1) -2016-05-25 16:42:43 INFO DAGScheduler:59 - ResultStage 8 (take at DecisionTreeMetadata.scala:110) finished in 0.030 s -2016-05-25 16:42:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 16:42:43 INFO DAGScheduler:59 - Job 6 finished: take at DecisionTreeMetadata.scala:110, took 0.055086 s -2016-05-25 16:42:43 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 16:42:43 INFO DAGScheduler:59 - Got job 7 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 16:42:43 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at DecisionTreeMetadata.scala:114) -2016-05-25 16:42:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:42:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:42:43 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[23] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(7328) called with curMem=228269, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 7.2 KB, free 730.4 MB) -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(3781) called with curMem=235597, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-25 16:42:43 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:55587 (size: 3.7 KB, free: 730.5 MB) -2016-05-25 16:42:43 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[23] at retag at RandomForest.scala:137) -2016-05-25 16:42:43 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 16:42:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 16, localhost, PROCESS_LOCAL, 2358 bytes) -2016-05-25 16:42:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2358 bytes) -2016-05-25 16:42:43 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 17) -2016-05-25 16:42:43 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 16) -2016-05-25 16:42:43 INFO BlockManager:59 - Found block rdd_19_0 locally -2016-05-25 16:42:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:42:43 INFO CacheManager:59 - Partition rdd_19_1 not found, computing it -2016-05-25 16:42:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:42:43 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(104) called with curMem=239378, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block rdd_19_1 stored as values in memory (estimated size 104.0 B, free 730.4 MB) -2016-05-25 16:42:43 INFO BlockManagerInfo:59 - Added rdd_19_1 in memory on localhost:55587 (size: 104.0 B, free: 730.5 MB) -2016-05-25 16:42:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 16) in 19 ms on localhost (1/2) -2016-05-25 16:42:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:42:43 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 17). 2662 bytes result sent to driver -2016-05-25 16:42:43 INFO DAGScheduler:59 - ResultStage 9 (count at DecisionTreeMetadata.scala:114) finished in 0.025 s -2016-05-25 16:42:43 INFO DAGScheduler:59 - Job 7 finished: count at DecisionTreeMetadata.scala:114, took 0.047734 s -2016-05-25 16:42:43 WARN DecisionTreeMetadata:71 - DecisionTree reducing maxBins from 32 to 6 (= number of training instances) -2016-05-25 16:42:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 17) in 22 ms on localhost (2/2) -2016-05-25 16:42:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 16:42:43 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 16:42:43 INFO DAGScheduler:59 - Got job 8 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 16:42:43 INFO DAGScheduler:59 - Final stage: ResultStage 10(collect at DecisionTree.scala:989) -2016-05-25 16:42:43 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:42:43 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:42:43 INFO DAGScheduler:59 - Submitting ResultStage 10 (PartitionwiseSampledRDD[25] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(8056) called with curMem=239482, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 7.9 KB, free 730.4 MB) -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(4117) called with curMem=247538, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.3 MB) -2016-05-25 16:42:43 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:55587 (size: 4.0 KB, free: 730.5 MB) -2016-05-25 16:42:43 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (PartitionwiseSampledRDD[25] at sample at DecisionTree.scala:989) -2016-05-25 16:42:43 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-25 16:42:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 18, localhost, PROCESS_LOCAL, 2467 bytes) -2016-05-25 16:42:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2467 bytes) -2016-05-25 16:42:43 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 18) -2016-05-25 16:42:43 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 19) -2016-05-25 16:42:43 INFO BlockManager:59 - Found block rdd_19_0 locally -2016-05-25 16:42:43 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:42:43 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 18). 2677 bytes result sent to driver -2016-05-25 16:42:43 INFO BlockManager:59 - Found block rdd_19_1 locally -2016-05-25 16:42:43 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:42:43 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 19). 2493 bytes result sent to driver -2016-05-25 16:42:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 18) in 15 ms on localhost (1/2) -2016-05-25 16:42:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 19) in 16 ms on localhost (2/2) -2016-05-25 16:42:43 INFO DAGScheduler:59 - ResultStage 10 (collect at DecisionTree.scala:989) finished in 0.018 s -2016-05-25 16:42:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 16:42:43 INFO DAGScheduler:59 - Job 8 finished: collect at DecisionTree.scala:989, took 0.038091 s -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=251655, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=251695, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 16:42:43 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:55587 (size: 81.0 B, free: 730.5 MB) -2016-05-25 16:42:43 INFO SparkContext:59 - Created broadcast 14 from broadcast at DecisionTree.scala:604 -2016-05-25 16:42:43 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:42:43 INFO DAGScheduler:59 - Registering RDD 28 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:42:43 INFO DAGScheduler:59 - Got job 9 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:42:43 INFO DAGScheduler:59 - Final stage: ResultStage 12(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:42:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 11) -2016-05-25 16:42:43 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 11) -2016-05-25 16:42:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 11 (MapPartitionsRDD[28] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(11776) called with curMem=251776, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 11.5 KB, free 730.3 MB) -2016-05-25 16:42:43 INFO MemoryStore:59 - ensureFreeSpace(5748) called with curMem=263552, maxMem=766075207 -2016-05-25 16:42:43 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 5.6 KB, free 730.3 MB) -2016-05-25 16:42:43 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:55587 (size: 5.6 KB, free: 730.5 MB) -2016-05-25 16:42:43 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 11 (MapPartitionsRDD[28] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:42:43 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-25 16:42:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 20, localhost, PROCESS_LOCAL, 2347 bytes) -2016-05-25 16:42:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2347 bytes) -2016-05-25 16:42:43 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 21) -2016-05-25 16:42:43 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 20) -2016-05-25 16:42:44 INFO CacheManager:59 - Partition rdd_27_1 not found, computing it -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_19_1 locally -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:42:44 INFO CacheManager:59 - Partition rdd_27_0 not found, computing it -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_19_0 locally -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(232) called with curMem=269300, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block rdd_27_1 stored as values in memory (estimated size 232.0 B, free 730.3 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added rdd_27_1 in memory on localhost:55587 (size: 232.0 B, free: 730.5 MB) -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(448) called with curMem=269532, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block rdd_27_0 stored as values in memory (estimated size 448.0 B, free 730.3 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added rdd_27_0 in memory on localhost:55587 (size: 448.0 B, free: 730.5 MB) -2016-05-25 16:42:44 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 21). 2834 bytes result sent to driver -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 21) in 36 ms on localhost (1/2) -2016-05-25 16:42:44 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 20). 2834 bytes result sent to driver -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 20) in 43 ms on localhost (2/2) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 16:42:44 INFO DAGScheduler:59 - ShuffleMapStage 11 (mapPartitions at DecisionTree.scala:625) finished in 0.044 s -2016-05-25 16:42:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:42:44 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:42:44 INFO DAGScheduler:59 - waiting: Set(ResultStage 12) -2016-05-25 16:42:44 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:42:44 INFO DAGScheduler:59 - Missing parents for ResultStage 12: List() -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[30] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=269980, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 5.1 KB, free 730.3 MB) -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(2865) called with curMem=275212, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:55587 (size: 2.8 KB, free: 730.5 MB) -2016-05-25 16:42:44 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[30] at map at DecisionTree.scala:645) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 22, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:42:44 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 22) -2016-05-25 16:42:44 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 23) -2016-05-25 16:42:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:42:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:42:44 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:42:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:42:44 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 23). 1161 bytes result sent to driver -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 23) in 13 ms on localhost (1/2) -2016-05-25 16:42:44 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 22). 2774 bytes result sent to driver -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 22) in 28 ms on localhost (2/2) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 16:42:44 INFO DAGScheduler:59 - ResultStage 12 (collectAsMap at DecisionTree.scala:654) finished in 0.030 s -2016-05-25 16:42:44 INFO DAGScheduler:59 - Job 9 finished: collectAsMap at DecisionTree.scala:654, took 0.113856 s -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=278077, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=278117, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:55587 (size: 81.0 B, free: 730.5 MB) -2016-05-25 16:42:44 INFO SparkContext:59 - Created broadcast 17 from broadcast at DecisionTree.scala:604 -2016-05-25 16:42:44 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:42:44 INFO DAGScheduler:59 - Registering RDD 31 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:42:44 INFO DAGScheduler:59 - Got job 10 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:42:44 INFO DAGScheduler:59 - Final stage: ResultStage 14(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:42:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 13) -2016-05-25 16:42:44 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 13) -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 13 (MapPartitionsRDD[31] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(11984) called with curMem=278198, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.7 KB, free 730.3 MB) -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(5851) called with curMem=290182, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 5.7 KB, free 730.3 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:55587 (size: 5.7 KB, free: 730.5 MB) -2016-05-25 16:42:44 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 13 (MapPartitionsRDD[31] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 24, localhost, PROCESS_LOCAL, 2347 bytes) -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2347 bytes) -2016-05-25 16:42:44 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 24) -2016-05-25 16:42:44 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 25) -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-05-25 16:42:44 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 24). 2254 bytes result sent to driver -2016-05-25 16:42:44 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 25). 2254 bytes result sent to driver -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 25) in 33 ms on localhost (1/2) -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 24) in 36 ms on localhost (2/2) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 16:42:44 INFO DAGScheduler:59 - ShuffleMapStage 13 (mapPartitions at DecisionTree.scala:625) finished in 0.038 s -2016-05-25 16:42:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:42:44 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:42:44 INFO DAGScheduler:59 - waiting: Set(ResultStage 14) -2016-05-25 16:42:44 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:42:44 INFO DAGScheduler:59 - Missing parents for ResultStage 14: List() -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[33] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=296033, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.1 KB, free 730.3 MB) -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(2878) called with curMem=301265, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:55587 (size: 2.8 KB, free: 730.5 MB) -2016-05-25 16:42:44 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[33] at map at DecisionTree.scala:645) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 26, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:42:44 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 27) -2016-05-25 16:42:44 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 26) -2016-05-25 16:42:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:42:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:42:44 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:42:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:42:44 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 27). 1161 bytes result sent to driver -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 27) in 13 ms on localhost (1/2) -2016-05-25 16:42:44 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 26). 2774 bytes result sent to driver -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 26) in 26 ms on localhost (2/2) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 16:42:44 INFO DAGScheduler:59 - ResultStage 14 (collectAsMap at DecisionTree.scala:654) finished in 0.029 s -2016-05-25 16:42:44 INFO DAGScheduler:59 - Job 10 finished: collectAsMap at DecisionTree.scala:654, took 0.111447 s -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=304143, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=304183, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:55587 (size: 81.0 B, free: 730.5 MB) -2016-05-25 16:42:44 INFO SparkContext:59 - Created broadcast 20 from broadcast at DecisionTree.scala:604 -2016-05-25 16:42:44 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:42:44 INFO DAGScheduler:59 - Registering RDD 34 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:42:44 INFO DAGScheduler:59 - Got job 11 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:42:44 INFO DAGScheduler:59 - Final stage: ResultStage 16(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:42:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-25 16:42:44 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 15) -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 15 (MapPartitionsRDD[34] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(12200) called with curMem=304264, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 11.9 KB, free 730.3 MB) -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(5915) called with curMem=316464, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 5.8 KB, free 730.3 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:55587 (size: 5.8 KB, free: 730.5 MB) -2016-05-25 16:42:44 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 15 (MapPartitionsRDD[34] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 28, localhost, PROCESS_LOCAL, 2347 bytes) -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2347 bytes) -2016-05-25 16:42:44 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 28) -2016-05-25 16:42:44 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 29) -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-05-25 16:42:44 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 28). 2254 bytes result sent to driver -2016-05-25 16:42:44 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 29). 2254 bytes result sent to driver -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 28) in 26 ms on localhost (1/2) -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 29) in 26 ms on localhost (2/2) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-25 16:42:44 INFO DAGScheduler:59 - ShuffleMapStage 15 (mapPartitions at DecisionTree.scala:625) finished in 0.027 s -2016-05-25 16:42:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:42:44 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:42:44 INFO DAGScheduler:59 - waiting: Set(ResultStage 16) -2016-05-25 16:42:44 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:42:44 INFO DAGScheduler:59 - Missing parents for ResultStage 16: List() -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[36] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=322379, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 5.1 KB, free 730.3 MB) -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(2873) called with curMem=327611, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:55587 (size: 2.8 KB, free: 730.5 MB) -2016-05-25 16:42:44 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[36] at map at DecisionTree.scala:645) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 30, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:42:44 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 30) -2016-05-25 16:42:44 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 31) -2016-05-25 16:42:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:42:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:42:44 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:42:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:42:44 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 31). 1161 bytes result sent to driver -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 31) in 11 ms on localhost (1/2) -2016-05-25 16:42:44 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 30). 2774 bytes result sent to driver -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 30) in 21 ms on localhost (2/2) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-25 16:42:44 INFO DAGScheduler:59 - ResultStage 16 (collectAsMap at DecisionTree.scala:654) finished in 0.022 s -2016-05-25 16:42:44 INFO DAGScheduler:59 - Job 11 finished: collectAsMap at DecisionTree.scala:654, took 0.090080 s -2016-05-25 16:42:44 INFO MapPartitionsRDD:59 - Removing RDD 27 from persistence list -2016-05-25 16:42:44 INFO BlockManager:59 - Removing RDD 27 -2016-05-25 16:42:44 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 16:42:44 INFO RandomForest:59 - init: 0.200158138 - total: 0.65642281 - findSplitsBins: 0.052042249 - findBestSplits: 0.451239424 - chooseSplits: 0.450701736 -2016-05-25 16:42:44 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 16:42:44 INFO DAGScheduler:59 - Got job 12 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 16:42:44 INFO DAGScheduler:59 - Final stage: ResultStage 17(take at DecisionTreeMetadata.scala:110) -2016-05-25 16:42:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:42:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[42] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(8664) called with curMem=329804, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 8.5 KB, free 730.3 MB) -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(4313) called with curMem=338468, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.3 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:55587 (size: 4.2 KB, free: 730.5 MB) -2016-05-25 16:42:44 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 17 (MapPartitionsRDD[42] at map at DecisionTreeMetadata.scala:110) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 1 tasks -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 32, localhost, PROCESS_LOCAL, 2390 bytes) -2016-05-25 16:42:44 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 32) -2016-05-25 16:42:44 INFO CacheManager:59 - Partition rdd_38_0 not found, computing it -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_19_0 locally -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=342781, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block rdd_38_0 stored as values in memory (estimated size 192.0 B, free 730.3 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added rdd_38_0 in memory on localhost:55587 (size: 192.0 B, free: 730.5 MB) -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:42:44 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 32). 2611 bytes result sent to driver -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 32) in 20 ms on localhost (1/1) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-25 16:42:44 INFO DAGScheduler:59 - ResultStage 17 (take at DecisionTreeMetadata.scala:110) finished in 0.020 s -2016-05-25 16:42:44 INFO DAGScheduler:59 - Job 12 finished: take at DecisionTreeMetadata.scala:110, took 0.044426 s -2016-05-25 16:42:44 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 16:42:44 INFO DAGScheduler:59 - Got job 13 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 16:42:44 INFO DAGScheduler:59 - Final stage: ResultStage 18(count at DecisionTreeMetadata.scala:114) -2016-05-25 16:42:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:42:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[41] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(8232) called with curMem=342973, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 8.0 KB, free 730.3 MB) -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(4118) called with curMem=351205, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.2 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:55587 (size: 4.0 KB, free: 730.5 MB) -2016-05-25 16:42:44 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[41] at retag at RandomForest.scala:137) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 33, localhost, PROCESS_LOCAL, 2390 bytes) -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 34, localhost, PROCESS_LOCAL, 2390 bytes) -2016-05-25 16:42:44 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 33) -2016-05-25 16:42:44 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 34) -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_38_0 locally -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:42:44 INFO CacheManager:59 - Partition rdd_38_1 not found, computing it -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_19_1 locally -2016-05-25 16:42:44 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 33). 2082 bytes result sent to driver -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(104) called with curMem=355323, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block rdd_38_1 stored as values in memory (estimated size 104.0 B, free 730.2 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added rdd_38_1 in memory on localhost:55587 (size: 104.0 B, free: 730.5 MB) -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 33) in 16 ms on localhost (1/2) -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:42:44 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 34). 2662 bytes result sent to driver -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 34) in 19 ms on localhost (2/2) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-25 16:42:44 INFO DAGScheduler:59 - ResultStage 18 (count at DecisionTreeMetadata.scala:114) finished in 0.021 s -2016-05-25 16:42:44 INFO DAGScheduler:59 - Job 13 finished: count at DecisionTreeMetadata.scala:114, took 0.039905 s -2016-05-25 16:42:44 WARN DecisionTreeMetadata:71 - DecisionTree reducing maxBins from 32 to 6 (= number of training instances) -2016-05-25 16:42:44 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 16:42:44 INFO DAGScheduler:59 - Got job 14 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 16:42:44 INFO DAGScheduler:59 - Final stage: ResultStage 19(collect at DecisionTree.scala:989) -2016-05-25 16:42:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:42:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting ResultStage 19 (PartitionwiseSampledRDD[43] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(8960) called with curMem=355427, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 8.8 KB, free 730.2 MB) -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(4420) called with curMem=364387, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.2 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:55587 (size: 4.3 KB, free: 730.5 MB) -2016-05-25 16:42:44 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (PartitionwiseSampledRDD[43] at sample at DecisionTree.scala:989) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 35, localhost, PROCESS_LOCAL, 2499 bytes) -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 36, localhost, PROCESS_LOCAL, 2499 bytes) -2016-05-25 16:42:44 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 35) -2016-05-25 16:42:44 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 36) -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_38_0 locally -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:42:44 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 35). 2677 bytes result sent to driver -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_38_1 locally -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:42:44 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 36). 2493 bytes result sent to driver -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 35) in 17 ms on localhost (1/2) -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 36) in 18 ms on localhost (2/2) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-25 16:42:44 INFO DAGScheduler:59 - ResultStage 19 (collect at DecisionTree.scala:989) finished in 0.019 s -2016-05-25 16:42:44 INFO DAGScheduler:59 - Job 14 finished: collect at DecisionTree.scala:989, took 0.039261 s -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=368807, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 40.0 B, free 730.2 MB) -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=368847, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.2 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:55587 (size: 81.0 B, free: 730.5 MB) -2016-05-25 16:42:44 INFO SparkContext:59 - Created broadcast 26 from broadcast at DecisionTree.scala:604 -2016-05-25 16:42:44 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:42:44 INFO DAGScheduler:59 - Registering RDD 46 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:42:44 INFO DAGScheduler:59 - Got job 15 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:42:44 INFO DAGScheduler:59 - Final stage: ResultStage 21(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:42:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 20) -2016-05-25 16:42:44 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 20) -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 20 (MapPartitionsRDD[46] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(12680) called with curMem=368928, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 12.4 KB, free 730.2 MB) -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(6059) called with curMem=381608, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 5.9 KB, free 730.2 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:55587 (size: 5.9 KB, free: 730.5 MB) -2016-05-25 16:42:44 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 20 (MapPartitionsRDD[46] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 37, localhost, PROCESS_LOCAL, 2379 bytes) -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 38, localhost, PROCESS_LOCAL, 2379 bytes) -2016-05-25 16:42:44 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 37) -2016-05-25 16:42:44 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 38) -2016-05-25 16:42:44 INFO CacheManager:59 - Partition rdd_45_1 not found, computing it -2016-05-25 16:42:44 INFO CacheManager:59 - Partition rdd_45_0 not found, computing it -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_38_1 locally -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(232) called with curMem=387667, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block rdd_45_1 stored as values in memory (estimated size 232.0 B, free 730.2 MB) -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_38_0 locally -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added rdd_45_1 in memory on localhost:55587 (size: 232.0 B, free: 730.5 MB) -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(448) called with curMem=387899, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block rdd_45_0 stored as values in memory (estimated size 448.0 B, free 730.2 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added rdd_45_0 in memory on localhost:55587 (size: 448.0 B, free: 730.5 MB) -2016-05-25 16:42:44 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 38). 2834 bytes result sent to driver -2016-05-25 16:42:44 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 37). 2834 bytes result sent to driver -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 38) in 38 ms on localhost (1/2) -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 37) in 41 ms on localhost (2/2) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-25 16:42:44 INFO DAGScheduler:59 - ShuffleMapStage 20 (mapPartitions at DecisionTree.scala:625) finished in 0.042 s -2016-05-25 16:42:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:42:44 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:42:44 INFO DAGScheduler:59 - waiting: Set(ResultStage 21) -2016-05-25 16:42:44 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:42:44 INFO DAGScheduler:59 - Missing parents for ResultStage 21: List() -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[48] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=388347, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 5.1 KB, free 730.2 MB) -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(2868) called with curMem=393579, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:55587 (size: 2.8 KB, free: 730.5 MB) -2016-05-25 16:42:44 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[48] at map at DecisionTree.scala:645) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 39, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 40, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:42:44 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 40) -2016-05-25 16:42:44 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 39) -2016-05-25 16:42:44 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:42:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:42:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:42:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:42:44 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 40). 1161 bytes result sent to driver -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 40) in 11 ms on localhost (1/2) -2016-05-25 16:42:44 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 39). 2774 bytes result sent to driver -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 39) in 22 ms on localhost (2/2) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-25 16:42:44 INFO DAGScheduler:59 - ResultStage 21 (collectAsMap at DecisionTree.scala:654) finished in 0.022 s -2016-05-25 16:42:44 INFO DAGScheduler:59 - Job 15 finished: collectAsMap at DecisionTree.scala:654, took 0.104219 s -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=396447, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 40.0 B, free 730.2 MB) -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=396487, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.2 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:55587 (size: 81.0 B, free: 730.5 MB) -2016-05-25 16:42:44 INFO SparkContext:59 - Created broadcast 29 from broadcast at DecisionTree.scala:604 -2016-05-25 16:42:44 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:42:44 INFO DAGScheduler:59 - Registering RDD 49 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:42:44 INFO DAGScheduler:59 - Got job 16 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:42:44 INFO DAGScheduler:59 - Final stage: ResultStage 23(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:42:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 22) -2016-05-25 16:42:44 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 22) -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 22 (MapPartitionsRDD[49] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(13000) called with curMem=396568, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 12.7 KB, free 730.2 MB) -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(6210) called with curMem=409568, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 6.1 KB, free 730.2 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:55587 (size: 6.1 KB, free: 730.5 MB) -2016-05-25 16:42:44 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 22 (MapPartitionsRDD[49] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2379 bytes) -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 42, localhost, PROCESS_LOCAL, 2379 bytes) -2016-05-25 16:42:44 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 41) -2016-05-25 16:42:44 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 42) -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_45_0 locally -2016-05-25 16:42:44 INFO BlockManager:59 - Found block rdd_45_1 locally -2016-05-25 16:42:44 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 41). 2254 bytes result sent to driver -2016-05-25 16:42:44 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 42). 2254 bytes result sent to driver -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 41) in 28 ms on localhost (1/2) -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 42) in 31 ms on localhost (2/2) -2016-05-25 16:42:44 INFO DAGScheduler:59 - ShuffleMapStage 22 (mapPartitions at DecisionTree.scala:625) finished in 0.032 s -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-25 16:42:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:42:44 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:42:44 INFO DAGScheduler:59 - waiting: Set(ResultStage 23) -2016-05-25 16:42:44 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:42:44 INFO DAGScheduler:59 - Missing parents for ResultStage 23: List() -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[51] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(5288) called with curMem=415778, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-25 16:42:44 INFO MemoryStore:59 - ensureFreeSpace(2901) called with curMem=421066, maxMem=766075207 -2016-05-25 16:42:44 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-25 16:42:44 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:55587 (size: 2.8 KB, free: 730.5 MB) -2016-05-25 16:42:44 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:42:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[51] at map at DecisionTree.scala:645) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:42:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:42:44 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 43) -2016-05-25 16:42:44 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 44) -2016-05-25 16:42:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:42:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:42:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:42:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:42:44 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 43). 2774 bytes result sent to driver -2016-05-25 16:42:44 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 44). 2774 bytes result sent to driver -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 43) in 21 ms on localhost (1/2) -2016-05-25 16:42:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 44) in 21 ms on localhost (2/2) -2016-05-25 16:42:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-25 16:42:44 INFO DAGScheduler:59 - ResultStage 23 (collectAsMap at DecisionTree.scala:654) finished in 0.024 s -2016-05-25 16:42:44 INFO DAGScheduler:59 - Job 16 finished: collectAsMap at DecisionTree.scala:654, took 0.094159 s -2016-05-25 16:42:44 INFO MapPartitionsRDD:59 - Removing RDD 45 from persistence list -2016-05-25 16:42:44 INFO BlockManager:59 - Removing RDD 45 -2016-05-25 16:42:44 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 16:42:44 INFO RandomForest:59 - init: 0.172035574 - total: 0.461006575 - findSplitsBins: 0.053812375 - findBestSplits: 0.284005546 - chooseSplits: 0.283668209 -2016-05-25 16:42:44 INFO GradientBoostedTrees:59 - Internal timing for DecisionTree: -2016-05-25 16:42:44 INFO GradientBoostedTrees:59 - init: 0.005273789 - total: 2.875275035 - building tree 0: 1.692020777 - building tree 2: 0.4618285 - building tree 1: 0.658344649 -2016-05-25 16:42:44 INFO MapPartitionsRDD:59 - Removing RDD 7 from persistence list -2016-05-25 16:42:44 INFO BlockManager:59 - Removing RDD 7 -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 16:42:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 16:42:44 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 16:42:44 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 16:42:45 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 16:42:45 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 16:42:45 INFO BlockManager:59 - BlockManager stopped -2016-05-25 16:42:45 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 16:42:45 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 16:42:45 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 16:42:45 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 16:42:45 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-37658118-9024-4d92-92d2-6252dfa4a9a9 -2016-05-25 16:42:45 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 16:44:38 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 16:44:39 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 16:44:39 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 16:44:39 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 16:44:39 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 16:44:40 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 16:44:40 INFO Remoting:74 - Starting remoting -2016-05-25 16:44:40 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:55613] -2016-05-25 16:44:40 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55613. -2016-05-25 16:44:40 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 16:44:40 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 16:44:40 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-b7dbae17-26b2-407e-9746-8fd47a228138 -2016-05-25 16:44:40 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 16:44:41 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-a6333a8d-7c41-4666-8d35-2f62d3b69edd\httpd-f2fe9da6-0736-4c7f-aaf3-6b1e53c8a7f2 -2016-05-25 16:44:41 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 16:44:41 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 16:44:41 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55614 -2016-05-25 16:44:41 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55614. -2016-05-25 16:44:41 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 16:44:41 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 16:44:41 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 16:44:41 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 16:44:41 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 16:44:41 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 16:44:41 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 16:44:41 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55633. -2016-05-25 16:44:41 INFO NettyBlockTransferService:59 - Server created on 55633 -2016-05-25 16:44:41 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 16:44:41 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55633 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55633) -2016-05-25 16:44:41 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 16:44:42 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 16:44:42 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 16:44:42 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 16:44:42 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 16:44:42 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55633 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 16:44:42 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-05-25 16:44:43 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 16:44:44 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 16:44:44 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-05-25 16:44:44 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-05-25 16:44:44 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-05-25 16:44:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:44:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:44:44 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-05-25 16:44:44 INFO MemoryStore:59 - ensureFreeSpace(3888) called with curMem=171580, maxMem=766075207 -2016-05-25 16:44:44 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-05-25 16:44:44 INFO MemoryStore:59 - ensureFreeSpace(2131) called with curMem=175468, maxMem=766075207 -2016-05-25 16:44:44 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-05-25 16:44:44 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55633 (size: 2.1 KB, free: 730.6 MB) -2016-05-25 16:44:44 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-05-25 16:44:44 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 16:44:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 16:44:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 16:44:44 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 16:44:44 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 16:44:44 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-05-25 16:44:44 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-05-25 16:44:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/dt.txt:57+57 -2016-05-25 16:44:44 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/dt.txt:0+57 -2016-05-25 16:44:44 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 16:44:44 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 16:44:44 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 16:44:44 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 16:44:44 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 16:44:44 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=177599, maxMem=766075207 -2016-05-25 16:44:44 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 280.0 B, free 730.4 MB) -2016-05-25 16:44:44 INFO MemoryStore:59 - ensureFreeSpace(544) called with curMem=177879, maxMem=766075207 -2016-05-25 16:44:44 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 544.0 B, free 730.4 MB) -2016-05-25 16:44:44 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:55633 (size: 280.0 B, free: 730.6 MB) -2016-05-25 16:44:44 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:55633 (size: 544.0 B, free: 730.6 MB) -2016-05-25 16:44:44 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-05-25 16:44:44 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-05-25 16:44:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 124 ms on localhost (1/2) -2016-05-25 16:44:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 108 ms on localhost (2/2) -2016-05-25 16:44:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 16:44:44 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.147 s -2016-05-25 16:44:44 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.299413 s -2016-05-25 16:44:44 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 16:44:44 INFO DAGScheduler:59 - Got job 1 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 16:44:44 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at DecisionTreeMetadata.scala:110) -2016-05-25 16:44:44 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:44:44 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:44:44 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[9] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 16:44:44 INFO MemoryStore:59 - ensureFreeSpace(4600) called with curMem=178423, maxMem=766075207 -2016-05-25 16:44:44 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.5 KB, free 730.4 MB) -2016-05-25 16:44:44 INFO MemoryStore:59 - ensureFreeSpace(2452) called with curMem=183023, maxMem=766075207 -2016-05-25 16:44:44 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-05-25 16:44:44 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55633 (size: 2.4 KB, free: 730.6 MB) -2016-05-25 16:44:44 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:44 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[9] at map at DecisionTreeMetadata.scala:110) -2016-05-25 16:44:44 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-05-25 16:44:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 16:44:44 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 16:44:44 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-05-25 16:44:44 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-05-25 16:44:44 INFO MemoryStore:59 - ensureFreeSpace(544) called with curMem=185475, maxMem=766075207 -2016-05-25 16:44:44 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 544.0 B, free 730.4 MB) -2016-05-25 16:44:45 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:55633 (size: 544.0 B, free: 730.6 MB) -2016-05-25 16:44:45 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2611 bytes result sent to driver -2016-05-25 16:44:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 387 ms on localhost (1/1) -2016-05-25 16:44:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 16:44:45 INFO DAGScheduler:59 - ResultStage 1 (take at DecisionTreeMetadata.scala:110) finished in 0.392 s -2016-05-25 16:44:45 INFO DAGScheduler:59 - Job 1 finished: take at DecisionTreeMetadata.scala:110, took 0.415051 s -2016-05-25 16:44:45 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 16:44:45 INFO DAGScheduler:59 - Got job 2 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 16:44:45 INFO DAGScheduler:59 - Final stage: ResultStage 2(count at DecisionTreeMetadata.scala:114) -2016-05-25 16:44:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:44:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:44:45 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[8] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 16:44:45 INFO MemoryStore:59 - ensureFreeSpace(4168) called with curMem=186019, maxMem=766075207 -2016-05-25 16:44:45 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.4 MB) -2016-05-25 16:44:45 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=190187, maxMem=766075207 -2016-05-25 16:44:45 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-05-25 16:44:45 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55633 (size: 2.2 KB, free: 730.6 MB) -2016-05-25 16:44:45 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[8] at retag at RandomForest.scala:137) -2016-05-25 16:44:45 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 16:44:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 16:44:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-05-25 16:44:45 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-05-25 16:44:45 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-05-25 16:44:45 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-05-25 16:44:45 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-05-25 16:44:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:44:45 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=192425, maxMem=766075207 -2016-05-25 16:44:45 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 280.0 B, free 730.4 MB) -2016-05-25 16:44:45 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2082 bytes result sent to driver -2016-05-25 16:44:45 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:55633 (size: 280.0 B, free: 730.6 MB) -2016-05-25 16:44:45 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2662 bytes result sent to driver -2016-05-25 16:44:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 25 ms on localhost (1/2) -2016-05-25 16:44:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 28 ms on localhost (2/2) -2016-05-25 16:44:45 INFO DAGScheduler:59 - ResultStage 2 (count at DecisionTreeMetadata.scala:114) finished in 0.030 s -2016-05-25 16:44:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 16:44:45 INFO DAGScheduler:59 - Job 2 finished: count at DecisionTreeMetadata.scala:114, took 0.052804 s -2016-05-25 16:44:45 WARN DecisionTreeMetadata:71 - DecisionTree reducing maxBins from 32 to 6 (= number of training instances) -2016-05-25 16:44:45 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 16:44:45 INFO DAGScheduler:59 - Got job 3 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 16:44:45 INFO DAGScheduler:59 - Final stage: ResultStage 3(collect at DecisionTree.scala:989) -2016-05-25 16:44:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:44:45 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:44:45 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[10] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 16:44:45 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=192705, maxMem=766075207 -2016-05-25 16:44:45 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-25 16:44:45 INFO MemoryStore:59 - ensureFreeSpace(2551) called with curMem=197601, maxMem=766075207 -2016-05-25 16:44:45 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.4 MB) -2016-05-25 16:44:45 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55633 (size: 2.5 KB, free: 730.6 MB) -2016-05-25 16:44:45 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[10] at sample at DecisionTree.scala:989) -2016-05-25 16:44:45 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 16:44:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 16:44:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2293 bytes) -2016-05-25 16:44:45 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-05-25 16:44:45 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-05-25 16:44:45 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:44:45 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:44:45 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2493 bytes result sent to driver -2016-05-25 16:44:45 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2677 bytes result sent to driver -2016-05-25 16:44:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 38 ms on localhost (1/2) -2016-05-25 16:44:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 35 ms on localhost (2/2) -2016-05-25 16:44:45 INFO DAGScheduler:59 - ResultStage 3 (collect at DecisionTree.scala:989) finished in 0.040 s -2016-05-25 16:44:45 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:55633 in memory (size: 2.2 KB, free: 730.6 MB) -2016-05-25 16:44:45 INFO DAGScheduler:59 - Job 3 finished: collect at DecisionTree.scala:989, took 0.087952 s -2016-05-25 16:44:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 16:44:45 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:55633 in memory (size: 2.4 KB, free: 730.6 MB) -2016-05-25 16:44:45 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:55633 in memory (size: 2.1 KB, free: 730.6 MB) -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=180675, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 40.0 B, free 730.4 MB) -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=180715, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.4 MB) -2016-05-25 16:44:46 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55633 (size: 81.0 B, free: 730.6 MB) -2016-05-25 16:44:46 INFO SparkContext:59 - Created broadcast 5 from broadcast at DecisionTree.scala:604 -2016-05-25 16:44:46 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:44:46 INFO DAGScheduler:59 - Registering RDD 13 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:44:46 INFO DAGScheduler:59 - Got job 4 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:44:46 INFO DAGScheduler:59 - Final stage: ResultStage 5(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:44:46 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-05-25 16:44:46 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-05-25 16:44:46 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[13] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(9880) called with curMem=180796, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 9.6 KB, free 730.4 MB) -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(4976) called with curMem=190676, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 4.9 KB, free 730.4 MB) -2016-05-25 16:44:46 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55633 (size: 4.9 KB, free: 730.6 MB) -2016-05-25 16:44:46 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[13] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:44:46 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 16:44:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 16:44:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 16:44:46 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-05-25 16:44:46 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-05-25 16:44:46 INFO CacheManager:59 - Partition rdd_12_0 not found, computing it -2016-05-25 16:44:46 INFO CacheManager:59 - Partition rdd_12_1 not found, computing it -2016-05-25 16:44:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:44:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(232) called with curMem=195652, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block rdd_12_1 stored as values in memory (estimated size 232.0 B, free 730.4 MB) -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(448) called with curMem=195884, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block rdd_12_0 stored as values in memory (estimated size 448.0 B, free 730.4 MB) -2016-05-25 16:44:46 INFO BlockManagerInfo:59 - Added rdd_12_1 in memory on localhost:55633 (size: 232.0 B, free: 730.6 MB) -2016-05-25 16:44:46 INFO BlockManagerInfo:59 - Added rdd_12_0 in memory on localhost:55633 (size: 448.0 B, free: 730.6 MB) -2016-05-25 16:44:46 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2834 bytes result sent to driver -2016-05-25 16:44:46 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2834 bytes result sent to driver -2016-05-25 16:44:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 102 ms on localhost (1/2) -2016-05-25 16:44:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 105 ms on localhost (2/2) -2016-05-25 16:44:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 16:44:46 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at DecisionTree.scala:625) finished in 0.106 s -2016-05-25 16:44:46 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:44:46 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:44:46 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-05-25 16:44:46 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:44:46 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-05-25 16:44:46 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[15] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=196332, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(2873) called with curMem=201564, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-25 16:44:46 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55633 (size: 2.8 KB, free: 730.6 MB) -2016-05-25 16:44:46 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[15] at map at DecisionTree.scala:645) -2016-05-25 16:44:46 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 16:44:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:44:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:44:46 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 16:44:46 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 16:44:46 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:44:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:44:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-25 16:44:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-05-25 16:44:46 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1161 bytes result sent to driver -2016-05-25 16:44:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 33 ms on localhost (1/2) -2016-05-25 16:44:46 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2774 bytes result sent to driver -2016-05-25 16:44:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 65 ms on localhost (2/2) -2016-05-25 16:44:46 INFO DAGScheduler:59 - ResultStage 5 (collectAsMap at DecisionTree.scala:654) finished in 0.065 s -2016-05-25 16:44:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 16:44:46 INFO DAGScheduler:59 - Job 4 finished: collectAsMap at DecisionTree.scala:654, took 0.219099 s -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=204437, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 40.0 B, free 730.4 MB) -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=204477, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.4 MB) -2016-05-25 16:44:46 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55633 (size: 81.0 B, free: 730.6 MB) -2016-05-25 16:44:46 INFO SparkContext:59 - Created broadcast 8 from broadcast at DecisionTree.scala:604 -2016-05-25 16:44:46 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:44:46 INFO DAGScheduler:59 - Registering RDD 16 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:44:46 INFO DAGScheduler:59 - Got job 5 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:44:46 INFO DAGScheduler:59 - Final stage: ResultStage 7(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:44:46 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-05-25 16:44:46 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-05-25 16:44:46 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (MapPartitionsRDD[16] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(10240) called with curMem=204558, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 10.0 KB, free 730.4 MB) -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(5159) called with curMem=214798, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 16:44:46 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55633 (size: 5.0 KB, free: 730.6 MB) -2016-05-25 16:44:46 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[16] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:44:46 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 16:44:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 16:44:46 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:55633 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-25 16:44:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2173 bytes) -2016-05-25 16:44:46 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-05-25 16:44:46 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-05-25 16:44:46 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 16:44:46 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:55633 in memory (size: 4.9 KB, free: 730.6 MB) -2016-05-25 16:44:46 INFO BlockManager:59 - Found block rdd_12_1 locally -2016-05-25 16:44:46 INFO BlockManager:59 - Found block rdd_12_0 locally -2016-05-25 16:44:46 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 16:44:46 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2254 bytes result sent to driver -2016-05-25 16:44:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 38 ms on localhost (1/2) -2016-05-25 16:44:46 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-05-25 16:44:46 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2254 bytes result sent to driver -2016-05-25 16:44:46 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55633 in memory (size: 81.0 B, free: 730.6 MB) -2016-05-25 16:44:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 49 ms on localhost (2/2) -2016-05-25 16:44:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 16:44:46 INFO DAGScheduler:59 - ShuffleMapStage 6 (mapPartitions at DecisionTree.scala:625) finished in 0.050 s -2016-05-25 16:44:46 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:44:46 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:44:46 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-05-25 16:44:46 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:44:46 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-05-25 16:44:46 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[18] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=196875, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(2869) called with curMem=202107, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-25 16:44:46 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55633 (size: 2.8 KB, free: 730.6 MB) -2016-05-25 16:44:46 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[18] at map at DecisionTree.scala:645) -2016-05-25 16:44:46 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-05-25 16:44:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:44:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:44:46 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-05-25 16:44:46 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-05-25 16:44:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:44:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:44:46 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:44:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:44:46 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 1161 bytes result sent to driver -2016-05-25 16:44:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 15 ms on localhost (1/2) -2016-05-25 16:44:46 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2774 bytes result sent to driver -2016-05-25 16:44:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 25 ms on localhost (2/2) -2016-05-25 16:44:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 16:44:46 INFO DAGScheduler:59 - ResultStage 7 (collectAsMap at DecisionTree.scala:654) finished in 0.027 s -2016-05-25 16:44:46 INFO DAGScheduler:59 - Job 5 finished: collectAsMap at DecisionTree.scala:654, took 0.154471 s -2016-05-25 16:44:46 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-05-25 16:44:46 INFO BlockManager:59 - Removing RDD 12 -2016-05-25 16:44:46 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 16:44:46 INFO RandomForest:59 - init: 1.614710388 - total: 2.119356133 - findSplitsBins: 1.086500143 - findBestSplits: 0.486289441 - chooseSplits: 0.480378012 -2016-05-25 16:44:46 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 16:44:46 INFO DAGScheduler:59 - Got job 6 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 16:44:46 INFO DAGScheduler:59 - Final stage: ResultStage 8(take at DecisionTreeMetadata.scala:110) -2016-05-25 16:44:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:44:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:44:46 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[24] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(7760) called with curMem=204296, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 7.6 KB, free 730.4 MB) -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(3995) called with curMem=212056, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.9 KB, free 730.4 MB) -2016-05-25 16:44:46 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55633 (size: 3.9 KB, free: 730.6 MB) -2016-05-25 16:44:46 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:46 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[24] at map at DecisionTreeMetadata.scala:110) -2016-05-25 16:44:46 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-05-25 16:44:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2358 bytes) -2016-05-25 16:44:46 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 16:44:46 INFO CacheManager:59 - Partition rdd_19_0 not found, computing it -2016-05-25 16:44:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=216051, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block rdd_19_0 stored as values in memory (estimated size 192.0 B, free 730.4 MB) -2016-05-25 16:44:46 INFO BlockManagerInfo:59 - Added rdd_19_0 in memory on localhost:55633 (size: 192.0 B, free: 730.6 MB) -2016-05-25 16:44:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:44:46 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2611 bytes result sent to driver -2016-05-25 16:44:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 21 ms on localhost (1/1) -2016-05-25 16:44:46 INFO DAGScheduler:59 - ResultStage 8 (take at DecisionTreeMetadata.scala:110) finished in 0.023 s -2016-05-25 16:44:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 16:44:46 INFO DAGScheduler:59 - Job 6 finished: take at DecisionTreeMetadata.scala:110, took 0.039824 s -2016-05-25 16:44:46 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 16:44:46 INFO DAGScheduler:59 - Got job 7 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 16:44:46 INFO DAGScheduler:59 - Final stage: ResultStage 9(count at DecisionTreeMetadata.scala:114) -2016-05-25 16:44:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:44:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:44:46 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[23] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(7328) called with curMem=216243, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 7.2 KB, free 730.4 MB) -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(3781) called with curMem=223571, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-25 16:44:46 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:55633 (size: 3.7 KB, free: 730.6 MB) -2016-05-25 16:44:46 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[23] at retag at RandomForest.scala:137) -2016-05-25 16:44:46 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-05-25 16:44:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 16, localhost, PROCESS_LOCAL, 2358 bytes) -2016-05-25 16:44:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2358 bytes) -2016-05-25 16:44:46 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 17) -2016-05-25 16:44:46 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 16) -2016-05-25 16:44:46 INFO BlockManager:59 - Found block rdd_19_0 locally -2016-05-25 16:44:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:44:46 INFO CacheManager:59 - Partition rdd_19_1 not found, computing it -2016-05-25 16:44:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:44:46 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 16). 2082 bytes result sent to driver -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(104) called with curMem=227352, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block rdd_19_1 stored as values in memory (estimated size 104.0 B, free 730.4 MB) -2016-05-25 16:44:46 INFO BlockManagerInfo:59 - Added rdd_19_1 in memory on localhost:55633 (size: 104.0 B, free: 730.6 MB) -2016-05-25 16:44:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:44:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 16) in 18 ms on localhost (1/2) -2016-05-25 16:44:46 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 17). 2662 bytes result sent to driver -2016-05-25 16:44:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 17) in 21 ms on localhost (2/2) -2016-05-25 16:44:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 16:44:46 INFO DAGScheduler:59 - ResultStage 9 (count at DecisionTreeMetadata.scala:114) finished in 0.022 s -2016-05-25 16:44:46 INFO DAGScheduler:59 - Job 7 finished: count at DecisionTreeMetadata.scala:114, took 0.041159 s -2016-05-25 16:44:46 WARN DecisionTreeMetadata:71 - DecisionTree reducing maxBins from 32 to 6 (= number of training instances) -2016-05-25 16:44:46 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 16:44:46 INFO DAGScheduler:59 - Got job 8 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 16:44:46 INFO DAGScheduler:59 - Final stage: ResultStage 10(collect at DecisionTree.scala:989) -2016-05-25 16:44:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:44:46 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:44:46 INFO DAGScheduler:59 - Submitting ResultStage 10 (PartitionwiseSampledRDD[25] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(8056) called with curMem=227456, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 7.9 KB, free 730.4 MB) -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(4116) called with curMem=235512, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.4 MB) -2016-05-25 16:44:46 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:55633 (size: 4.0 KB, free: 730.5 MB) -2016-05-25 16:44:46 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (PartitionwiseSampledRDD[25] at sample at DecisionTree.scala:989) -2016-05-25 16:44:46 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-05-25 16:44:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 18, localhost, PROCESS_LOCAL, 2467 bytes) -2016-05-25 16:44:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2467 bytes) -2016-05-25 16:44:46 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 19) -2016-05-25 16:44:46 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 18) -2016-05-25 16:44:46 INFO BlockManager:59 - Found block rdd_19_0 locally -2016-05-25 16:44:46 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:44:46 INFO BlockManager:59 - Found block rdd_19_1 locally -2016-05-25 16:44:46 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:44:46 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 18). 2677 bytes result sent to driver -2016-05-25 16:44:46 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 19). 2493 bytes result sent to driver -2016-05-25 16:44:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 18) in 17 ms on localhost (1/2) -2016-05-25 16:44:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 19) in 17 ms on localhost (2/2) -2016-05-25 16:44:46 INFO DAGScheduler:59 - ResultStage 10 (collect at DecisionTree.scala:989) finished in 0.018 s -2016-05-25 16:44:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 16:44:46 INFO DAGScheduler:59 - Job 8 finished: collect at DecisionTree.scala:989, took 0.035332 s -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=239628, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 40.0 B, free 730.4 MB) -2016-05-25 16:44:46 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=239668, maxMem=766075207 -2016-05-25 16:44:46 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.4 MB) -2016-05-25 16:44:46 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:55633 (size: 81.0 B, free: 730.5 MB) -2016-05-25 16:44:46 INFO SparkContext:59 - Created broadcast 14 from broadcast at DecisionTree.scala:604 -2016-05-25 16:44:46 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:44:46 INFO DAGScheduler:59 - Registering RDD 28 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:44:46 INFO DAGScheduler:59 - Got job 9 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:44:46 INFO DAGScheduler:59 - Final stage: ResultStage 12(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:44:46 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 11) -2016-05-25 16:44:46 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 11) -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 11 (MapPartitionsRDD[28] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(11776) called with curMem=239749, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 11.5 KB, free 730.3 MB) -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(5748) called with curMem=251525, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 5.6 KB, free 730.3 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:55633 (size: 5.6 KB, free: 730.5 MB) -2016-05-25 16:44:47 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 11 (MapPartitionsRDD[28] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 20, localhost, PROCESS_LOCAL, 2347 bytes) -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2347 bytes) -2016-05-25 16:44:47 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 21) -2016-05-25 16:44:47 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 20) -2016-05-25 16:44:47 INFO CacheManager:59 - Partition rdd_27_1 not found, computing it -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_19_1 locally -2016-05-25 16:44:47 INFO CacheManager:59 - Partition rdd_27_0 not found, computing it -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_19_0 locally -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(232) called with curMem=257273, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block rdd_27_1 stored as values in memory (estimated size 232.0 B, free 730.3 MB) -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(448) called with curMem=257505, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block rdd_27_0 stored as values in memory (estimated size 448.0 B, free 730.3 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added rdd_27_1 in memory on localhost:55633 (size: 232.0 B, free: 730.5 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added rdd_27_0 in memory on localhost:55633 (size: 448.0 B, free: 730.5 MB) -2016-05-25 16:44:47 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 21). 2834 bytes result sent to driver -2016-05-25 16:44:47 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 20). 2834 bytes result sent to driver -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 21) in 32 ms on localhost (1/2) -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 20) in 34 ms on localhost (2/2) -2016-05-25 16:44:47 INFO DAGScheduler:59 - ShuffleMapStage 11 (mapPartitions at DecisionTree.scala:625) finished in 0.035 s -2016-05-25 16:44:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 16:44:47 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:44:47 INFO DAGScheduler:59 - waiting: Set(ResultStage 12) -2016-05-25 16:44:47 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:44:47 INFO DAGScheduler:59 - Missing parents for ResultStage 12: List() -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[30] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=257953, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 5.1 KB, free 730.3 MB) -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(2865) called with curMem=263185, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:55633 (size: 2.8 KB, free: 730.5 MB) -2016-05-25 16:44:47 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[30] at map at DecisionTree.scala:645) -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 22, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:44:47 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 23) -2016-05-25 16:44:47 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 22) -2016-05-25 16:44:47 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:44:47 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:44:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:44:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:44:47 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 23). 1161 bytes result sent to driver -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 23) in 11 ms on localhost (1/2) -2016-05-25 16:44:47 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 22). 2774 bytes result sent to driver -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 22) in 23 ms on localhost (2/2) -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 16:44:47 INFO DAGScheduler:59 - ResultStage 12 (collectAsMap at DecisionTree.scala:654) finished in 0.024 s -2016-05-25 16:44:47 INFO DAGScheduler:59 - Job 9 finished: collectAsMap at DecisionTree.scala:654, took 0.092452 s -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=266050, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=266090, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:55633 (size: 81.0 B, free: 730.5 MB) -2016-05-25 16:44:47 INFO SparkContext:59 - Created broadcast 17 from broadcast at DecisionTree.scala:604 -2016-05-25 16:44:47 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Registering RDD 31 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:44:47 INFO DAGScheduler:59 - Got job 10 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:44:47 INFO DAGScheduler:59 - Final stage: ResultStage 14(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:44:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 13) -2016-05-25 16:44:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 13) -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 13 (MapPartitionsRDD[31] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(11984) called with curMem=266171, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.7 KB, free 730.3 MB) -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(5851) called with curMem=278155, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 5.7 KB, free 730.3 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:55633 (size: 5.7 KB, free: 730.5 MB) -2016-05-25 16:44:47 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 13 (MapPartitionsRDD[31] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 24, localhost, PROCESS_LOCAL, 2347 bytes) -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2347 bytes) -2016-05-25 16:44:47 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 25) -2016-05-25 16:44:47 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 24) -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-05-25 16:44:47 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 25). 2254 bytes result sent to driver -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 25) in 29 ms on localhost (1/2) -2016-05-25 16:44:47 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 24). 2254 bytes result sent to driver -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 24) in 37 ms on localhost (2/2) -2016-05-25 16:44:47 INFO DAGScheduler:59 - ShuffleMapStage 13 (mapPartitions at DecisionTree.scala:625) finished in 0.038 s -2016-05-25 16:44:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:44:47 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:44:47 INFO DAGScheduler:59 - waiting: Set(ResultStage 14) -2016-05-25 16:44:47 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-05-25 16:44:47 INFO DAGScheduler:59 - Missing parents for ResultStage 14: List() -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[33] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=284006, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.1 KB, free 730.3 MB) -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(2878) called with curMem=289238, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:55633 (size: 2.8 KB, free: 730.5 MB) -2016-05-25 16:44:47 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[33] at map at DecisionTree.scala:645) -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 26, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:44:47 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 27) -2016-05-25 16:44:47 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 26) -2016-05-25 16:44:47 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:44:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:44:47 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:44:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:44:47 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 27). 1161 bytes result sent to driver -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 27) in 12 ms on localhost (1/2) -2016-05-25 16:44:47 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 26). 2774 bytes result sent to driver -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 26) in 26 ms on localhost (2/2) -2016-05-25 16:44:47 INFO DAGScheduler:59 - ResultStage 14 (collectAsMap at DecisionTree.scala:654) finished in 0.028 s -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-05-25 16:44:47 INFO DAGScheduler:59 - Job 10 finished: collectAsMap at DecisionTree.scala:654, took 0.114221 s -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=292116, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 40.0 B, free 730.3 MB) -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=292156, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.3 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:55633 (size: 81.0 B, free: 730.5 MB) -2016-05-25 16:44:47 INFO SparkContext:59 - Created broadcast 20 from broadcast at DecisionTree.scala:604 -2016-05-25 16:44:47 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Registering RDD 34 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:44:47 INFO DAGScheduler:59 - Got job 11 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:44:47 INFO DAGScheduler:59 - Final stage: ResultStage 16(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:44:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-05-25 16:44:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 15) -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 15 (MapPartitionsRDD[34] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(12200) called with curMem=292237, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 11.9 KB, free 730.3 MB) -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(5915) called with curMem=304437, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 5.8 KB, free 730.3 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:55633 (size: 5.8 KB, free: 730.5 MB) -2016-05-25 16:44:47 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 15 (MapPartitionsRDD[34] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 28, localhost, PROCESS_LOCAL, 2347 bytes) -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2347 bytes) -2016-05-25 16:44:47 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 29) -2016-05-25 16:44:47 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 28) -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-05-25 16:44:47 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 29). 2254 bytes result sent to driver -2016-05-25 16:44:47 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 28). 2254 bytes result sent to driver -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 29) in 25 ms on localhost (1/2) -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 28) in 27 ms on localhost (2/2) -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-05-25 16:44:47 INFO DAGScheduler:59 - ShuffleMapStage 15 (mapPartitions at DecisionTree.scala:625) finished in 0.028 s -2016-05-25 16:44:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:44:47 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:44:47 INFO DAGScheduler:59 - waiting: Set(ResultStage 16) -2016-05-25 16:44:47 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:44:47 INFO DAGScheduler:59 - Missing parents for ResultStage 16: List() -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[36] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=310352, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 5.1 KB, free 730.3 MB) -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(2873) called with curMem=315584, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:55633 (size: 2.8 KB, free: 730.5 MB) -2016-05-25 16:44:47 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[36] at map at DecisionTree.scala:645) -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 30, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:44:47 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 31) -2016-05-25 16:44:47 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 30) -2016-05-25 16:44:47 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:44:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:44:47 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:44:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:44:47 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 31). 1161 bytes result sent to driver -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 31) in 11 ms on localhost (1/2) -2016-05-25 16:44:47 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 30). 2774 bytes result sent to driver -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 30) in 22 ms on localhost (2/2) -2016-05-25 16:44:47 INFO DAGScheduler:59 - ResultStage 16 (collectAsMap at DecisionTree.scala:654) finished in 0.023 s -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-05-25 16:44:47 INFO DAGScheduler:59 - Job 11 finished: collectAsMap at DecisionTree.scala:654, took 0.090857 s -2016-05-25 16:44:47 INFO MapPartitionsRDD:59 - Removing RDD 27 from persistence list -2016-05-25 16:44:47 INFO BlockManager:59 - Removing RDD 27 -2016-05-25 16:44:47 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 16:44:47 INFO RandomForest:59 - init: 0.170000843 - total: 0.619139942 - findSplitsBins: 0.050600089 - findBestSplits: 0.440630712 - chooseSplits: 0.440043497 -2016-05-25 16:44:47 INFO SparkContext:59 - Starting job: take at DecisionTreeMetadata.scala:110 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Got job 12 (take at DecisionTreeMetadata.scala:110) with 1 output partitions -2016-05-25 16:44:47 INFO DAGScheduler:59 - Final stage: ResultStage 17(take at DecisionTreeMetadata.scala:110) -2016-05-25 16:44:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:44:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[42] at map at DecisionTreeMetadata.scala:110), which has no missing parents -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(8664) called with curMem=317777, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 8.5 KB, free 730.3 MB) -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(4314) called with curMem=326441, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.3 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:55633 (size: 4.2 KB, free: 730.5 MB) -2016-05-25 16:44:47 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 17 (MapPartitionsRDD[42] at map at DecisionTreeMetadata.scala:110) -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 1 tasks -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 32, localhost, PROCESS_LOCAL, 2390 bytes) -2016-05-25 16:44:47 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 32) -2016-05-25 16:44:47 INFO CacheManager:59 - Partition rdd_38_0 not found, computing it -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_19_0 locally -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=330755, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block rdd_38_0 stored as values in memory (estimated size 192.0 B, free 730.3 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added rdd_38_0 in memory on localhost:55633 (size: 192.0 B, free: 730.5 MB) -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:44:47 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 32). 2611 bytes result sent to driver -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 32) in 20 ms on localhost (1/1) -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-05-25 16:44:47 INFO DAGScheduler:59 - ResultStage 17 (take at DecisionTreeMetadata.scala:110) finished in 0.020 s -2016-05-25 16:44:47 INFO DAGScheduler:59 - Job 12 finished: take at DecisionTreeMetadata.scala:110, took 0.043800 s -2016-05-25 16:44:47 INFO SparkContext:59 - Starting job: count at DecisionTreeMetadata.scala:114 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Got job 13 (count at DecisionTreeMetadata.scala:114) with 2 output partitions -2016-05-25 16:44:47 INFO DAGScheduler:59 - Final stage: ResultStage 18(count at DecisionTreeMetadata.scala:114) -2016-05-25 16:44:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:44:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[41] at retag at RandomForest.scala:137), which has no missing parents -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(8232) called with curMem=330947, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 8.0 KB, free 730.3 MB) -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(4118) called with curMem=339179, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.3 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:55633 (size: 4.0 KB, free: 730.5 MB) -2016-05-25 16:44:47 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[41] at retag at RandomForest.scala:137) -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 33, localhost, PROCESS_LOCAL, 2390 bytes) -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 34, localhost, PROCESS_LOCAL, 2390 bytes) -2016-05-25 16:44:47 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 34) -2016-05-25 16:44:47 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 33) -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_38_0 locally -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:44:47 INFO CacheManager:59 - Partition rdd_38_1 not found, computing it -2016-05-25 16:44:47 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 33). 2082 bytes result sent to driver -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_19_1 locally -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(104) called with curMem=343297, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block rdd_38_1 stored as values in memory (estimated size 104.0 B, free 730.3 MB) -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 33) in 16 ms on localhost (1/2) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added rdd_38_1 in memory on localhost:55633 (size: 104.0 B, free: 730.5 MB) -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:44:47 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 34). 2662 bytes result sent to driver -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 34) in 20 ms on localhost (2/2) -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-05-25 16:44:47 INFO DAGScheduler:59 - ResultStage 18 (count at DecisionTreeMetadata.scala:114) finished in 0.021 s -2016-05-25 16:44:47 INFO DAGScheduler:59 - Job 13 finished: count at DecisionTreeMetadata.scala:114, took 0.037448 s -2016-05-25 16:44:47 WARN DecisionTreeMetadata:71 - DecisionTree reducing maxBins from 32 to 6 (= number of training instances) -2016-05-25 16:44:47 INFO SparkContext:59 - Starting job: collect at DecisionTree.scala:989 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Got job 14 (collect at DecisionTree.scala:989) with 2 output partitions -2016-05-25 16:44:47 INFO DAGScheduler:59 - Final stage: ResultStage 19(collect at DecisionTree.scala:989) -2016-05-25 16:44:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:44:47 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting ResultStage 19 (PartitionwiseSampledRDD[43] at sample at DecisionTree.scala:989), which has no missing parents -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(8960) called with curMem=343401, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 8.8 KB, free 730.3 MB) -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(4420) called with curMem=352361, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.2 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:55633 (size: 4.3 KB, free: 730.5 MB) -2016-05-25 16:44:47 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (PartitionwiseSampledRDD[43] at sample at DecisionTree.scala:989) -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 35, localhost, PROCESS_LOCAL, 2499 bytes) -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 36, localhost, PROCESS_LOCAL, 2499 bytes) -2016-05-25 16:44:47 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 35) -2016-05-25 16:44:47 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 36) -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_38_1 locally -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_38_0 locally -2016-05-25 16:44:47 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 36). 2493 bytes result sent to driver -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:44:47 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 35). 2677 bytes result sent to driver -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 36) in 17 ms on localhost (1/2) -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 35) in 20 ms on localhost (2/2) -2016-05-25 16:44:47 INFO DAGScheduler:59 - ResultStage 19 (collect at DecisionTree.scala:989) finished in 0.020 s -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-05-25 16:44:47 INFO DAGScheduler:59 - Job 14 finished: collect at DecisionTree.scala:989, took 0.040332 s -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=356781, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 40.0 B, free 730.2 MB) -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=356821, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.2 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:55633 (size: 81.0 B, free: 730.5 MB) -2016-05-25 16:44:47 INFO SparkContext:59 - Created broadcast 26 from broadcast at DecisionTree.scala:604 -2016-05-25 16:44:47 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Registering RDD 46 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:44:47 INFO DAGScheduler:59 - Got job 15 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:44:47 INFO DAGScheduler:59 - Final stage: ResultStage 21(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:44:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 20) -2016-05-25 16:44:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 20) -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 20 (MapPartitionsRDD[46] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(12680) called with curMem=356902, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 12.4 KB, free 730.2 MB) -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(6059) called with curMem=369582, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 5.9 KB, free 730.2 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:55633 (size: 5.9 KB, free: 730.5 MB) -2016-05-25 16:44:47 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 20 (MapPartitionsRDD[46] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 37, localhost, PROCESS_LOCAL, 2379 bytes) -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 38, localhost, PROCESS_LOCAL, 2379 bytes) -2016-05-25 16:44:47 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 37) -2016-05-25 16:44:47 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 38) -2016-05-25 16:44:47 INFO CacheManager:59 - Partition rdd_45_1 not found, computing it -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_38_1 locally -2016-05-25 16:44:47 INFO CacheManager:59 - Partition rdd_45_0 not found, computing it -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_38_0 locally -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(232) called with curMem=375641, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block rdd_45_1 stored as values in memory (estimated size 232.0 B, free 730.2 MB) -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(448) called with curMem=375873, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block rdd_45_0 stored as values in memory (estimated size 448.0 B, free 730.2 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added rdd_45_1 in memory on localhost:55633 (size: 232.0 B, free: 730.5 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added rdd_45_0 in memory on localhost:55633 (size: 448.0 B, free: 730.5 MB) -2016-05-25 16:44:47 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 38). 2834 bytes result sent to driver -2016-05-25 16:44:47 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 37). 2834 bytes result sent to driver -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 38) in 33 ms on localhost (1/2) -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 37) in 39 ms on localhost (2/2) -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-05-25 16:44:47 INFO DAGScheduler:59 - ShuffleMapStage 20 (mapPartitions at DecisionTree.scala:625) finished in 0.039 s -2016-05-25 16:44:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:44:47 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:44:47 INFO DAGScheduler:59 - waiting: Set(ResultStage 21) -2016-05-25 16:44:47 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:44:47 INFO DAGScheduler:59 - Missing parents for ResultStage 21: List() -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[48] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=376321, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 5.1 KB, free 730.2 MB) -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(2868) called with curMem=381553, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:55633 (size: 2.8 KB, free: 730.5 MB) -2016-05-25 16:44:47 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[48] at map at DecisionTree.scala:645) -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 39, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 40, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:44:47 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 39) -2016-05-25 16:44:47 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 40) -2016-05-25 16:44:47 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:44:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:44:47 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-05-25 16:44:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-05-25 16:44:47 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 40). 1161 bytes result sent to driver -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 40) in 12 ms on localhost (1/2) -2016-05-25 16:44:47 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 39). 2774 bytes result sent to driver -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 39) in 22 ms on localhost (2/2) -2016-05-25 16:44:47 INFO DAGScheduler:59 - ResultStage 21 (collectAsMap at DecisionTree.scala:654) finished in 0.023 s -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-05-25 16:44:47 INFO DAGScheduler:59 - Job 15 finished: collectAsMap at DecisionTree.scala:654, took 0.102464 s -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(40) called with curMem=384421, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 40.0 B, free 730.2 MB) -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=384461, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.2 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:55633 (size: 81.0 B, free: 730.5 MB) -2016-05-25 16:44:47 INFO SparkContext:59 - Created broadcast 29 from broadcast at DecisionTree.scala:604 -2016-05-25 16:44:47 INFO SparkContext:59 - Starting job: collectAsMap at DecisionTree.scala:654 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Registering RDD 49 (mapPartitions at DecisionTree.scala:625) -2016-05-25 16:44:47 INFO DAGScheduler:59 - Got job 16 (collectAsMap at DecisionTree.scala:654) with 2 output partitions -2016-05-25 16:44:47 INFO DAGScheduler:59 - Final stage: ResultStage 23(collectAsMap at DecisionTree.scala:654) -2016-05-25 16:44:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 22) -2016-05-25 16:44:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 22) -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 22 (MapPartitionsRDD[49] at mapPartitions at DecisionTree.scala:625), which has no missing parents -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(13000) called with curMem=384542, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 12.7 KB, free 730.2 MB) -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(6210) called with curMem=397542, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 6.1 KB, free 730.2 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:55633 (size: 6.1 KB, free: 730.5 MB) -2016-05-25 16:44:47 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 22 (MapPartitionsRDD[49] at mapPartitions at DecisionTree.scala:625) -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 41, localhost, PROCESS_LOCAL, 2379 bytes) -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 42, localhost, PROCESS_LOCAL, 2379 bytes) -2016-05-25 16:44:47 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 41) -2016-05-25 16:44:47 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 42) -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_45_1 locally -2016-05-25 16:44:47 INFO BlockManager:59 - Found block rdd_45_0 locally -2016-05-25 16:44:47 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 41). 2254 bytes result sent to driver -2016-05-25 16:44:47 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 42). 2254 bytes result sent to driver -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 41) in 28 ms on localhost (1/2) -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 42) in 28 ms on localhost (2/2) -2016-05-25 16:44:47 INFO DAGScheduler:59 - ShuffleMapStage 22 (mapPartitions at DecisionTree.scala:625) finished in 0.029 s -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-05-25 16:44:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:44:47 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:44:47 INFO DAGScheduler:59 - waiting: Set(ResultStage 23) -2016-05-25 16:44:47 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:44:47 INFO DAGScheduler:59 - Missing parents for ResultStage 23: List() -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[51] at map at DecisionTree.scala:645), which is now runnable -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(5288) called with curMem=403752, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-05-25 16:44:47 INFO MemoryStore:59 - ensureFreeSpace(2901) called with curMem=409040, maxMem=766075207 -2016-05-25 16:44:47 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-05-25 16:44:47 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:55633 (size: 2.8 KB, free: 730.5 MB) -2016-05-25 16:44:47 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[51] at map at DecisionTree.scala:645) -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 43, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:44:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 44, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:44:47 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 44) -2016-05-25 16:44:47 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 43) -2016-05-25 16:44:47 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:44:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:44:47 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:44:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-05-25 16:44:47 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 44). 2774 bytes result sent to driver -2016-05-25 16:44:47 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 43). 2774 bytes result sent to driver -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 44) in 19 ms on localhost (1/2) -2016-05-25 16:44:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 43) in 20 ms on localhost (2/2) -2016-05-25 16:44:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-05-25 16:44:47 INFO DAGScheduler:59 - ResultStage 23 (collectAsMap at DecisionTree.scala:654) finished in 0.021 s -2016-05-25 16:44:47 INFO DAGScheduler:59 - Job 16 finished: collectAsMap at DecisionTree.scala:654, took 0.087277 s -2016-05-25 16:44:47 INFO MapPartitionsRDD:59 - Removing RDD 45 from persistence list -2016-05-25 16:44:47 INFO BlockManager:59 - Removing RDD 45 -2016-05-25 16:44:47 INFO RandomForest:59 - Internal timing for DecisionTree: -2016-05-25 16:44:47 INFO RandomForest:59 - init: 0.166385181 - total: 0.449401473 - findSplitsBins: 0.052930659 - findBestSplits: 0.277709037 - chooseSplits: 0.277331094 -2016-05-25 16:44:47 INFO GradientBoostedTrees:59 - Internal timing for DecisionTree: -2016-05-25 16:44:47 INFO GradientBoostedTrees:59 - init: 0.005796751 - total: 3.28609873 - building tree 0: 2.147037392 - building tree 2: 0.450313532 - building tree 1: 0.621641858 -2016-05-25 16:44:47 INFO MapPartitionsRDD:59 - Removing RDD 7 from persistence list -2016-05-25 16:44:47 INFO BlockManager:59 - Removing RDD 7 -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 16:44:47 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 16:44:47 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 16:44:47 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 16:44:48 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 16:44:48 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 16:44:48 INFO BlockManager:59 - BlockManager stopped -2016-05-25 16:44:48 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 16:44:48 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 16:44:48 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 16:44:48 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 16:44:48 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-a6333a8d-7c41-4666-8d35-2f62d3b69edd -2016-05-25 16:44:48 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 16:49:49 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 16:49:50 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 16:49:50 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 16:49:50 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 16:49:50 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 16:49:51 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 16:49:51 INFO Remoting:74 - Starting remoting -2016-05-25 16:49:51 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:55684] -2016-05-25 16:49:51 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55684. -2016-05-25 16:49:51 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 16:49:51 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 16:49:51 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-a57cd439-5121-4eb4-8dd8-aac2f7be2be0 -2016-05-25 16:49:51 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 16:49:51 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-b5437c77-05f7-4bb5-8390-0999b33419a1\httpd-0015cd32-eb1e-49b5-b9a7-cdf91b98a328 -2016-05-25 16:49:51 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 16:49:51 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 16:49:51 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55685 -2016-05-25 16:49:51 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55685. -2016-05-25 16:49:51 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 16:49:51 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 16:49:51 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 16:49:51 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 16:49:51 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 16:49:51 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 16:49:51 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 16:49:52 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55704. -2016-05-25 16:49:52 INFO NettyBlockTransferService:59 - Server created on 55704 -2016-05-25 16:49:52 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 16:49:52 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55704 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55704) -2016-05-25 16:49:52 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 16:49:52 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 16:49:52 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 16:49:53 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 16:49:53 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 16:49:53 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55704 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 16:49:53 INFO SparkContext:59 - Created broadcast 0 from textFile at IsotonicRegression1.scala:33 -2016-05-25 16:49:54 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 16:49:54 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 16:49:55 INFO SparkContext:59 - Starting job: sortBy at IsotonicRegression.scala:418 -2016-05-25 16:49:55 INFO DAGScheduler:59 - Got job 0 (sortBy at IsotonicRegression.scala:418) with 2 output partitions -2016-05-25 16:49:55 INFO DAGScheduler:59 - Final stage: ResultStage 0(sortBy at IsotonicRegression.scala:418) -2016-05-25 16:49:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:49:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:49:55 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[7] at sortBy at IsotonicRegression.scala:418), which has no missing parents -2016-05-25 16:49:55 INFO MemoryStore:59 - ensureFreeSpace(4256) called with curMem=171580, maxMem=766075207 -2016-05-25 16:49:55 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.2 KB, free 730.4 MB) -2016-05-25 16:49:55 INFO MemoryStore:59 - ensureFreeSpace(2276) called with curMem=175836, maxMem=766075207 -2016-05-25 16:49:55 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-05-25 16:49:55 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55704 (size: 2.2 KB, free: 730.6 MB) -2016-05-25 16:49:55 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:49:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[7] at sortBy at IsotonicRegression.scala:418) -2016-05-25 16:49:55 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 16:49:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:49:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:49:55 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 16:49:55 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 16:49:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:799+799 -2016-05-25 16:49:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:0+799 -2016-05-25 16:49:55 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 16:49:55 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 16:49:55 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 16:49:55 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 16:49:55 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 16:49:55 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 3161 bytes result sent to driver -2016-05-25 16:49:55 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 3065 bytes result sent to driver -2016-05-25 16:49:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 136 ms on localhost (1/2) -2016-05-25 16:49:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 119 ms on localhost (2/2) -2016-05-25 16:49:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 16:49:55 INFO DAGScheduler:59 - ResultStage 0 (sortBy at IsotonicRegression.scala:418) finished in 0.155 s -2016-05-25 16:49:55 INFO DAGScheduler:59 - Job 0 finished: sortBy at IsotonicRegression.scala:418, took 0.248205 s -2016-05-25 16:49:55 INFO SparkContext:59 - Starting job: collect at IsotonicRegression.scala:421 -2016-05-25 16:49:55 INFO DAGScheduler:59 - Registering RDD 5 (sortBy at IsotonicRegression.scala:418) -2016-05-25 16:49:55 INFO DAGScheduler:59 - Got job 1 (collect at IsotonicRegression.scala:421) with 2 output partitions -2016-05-25 16:49:55 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at IsotonicRegression.scala:421) -2016-05-25 16:49:55 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1) -2016-05-25 16:49:55 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1) -2016-05-25 16:49:55 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1 (MapPartitionsRDD[5] at sortBy at IsotonicRegression.scala:418), which has no missing parents -2016-05-25 16:49:55 INFO MemoryStore:59 - ensureFreeSpace(4864) called with curMem=178112, maxMem=766075207 -2016-05-25 16:49:55 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-25 16:49:55 INFO MemoryStore:59 - ensureFreeSpace(2714) called with curMem=182976, maxMem=766075207 -2016-05-25 16:49:55 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-05-25 16:49:55 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55704 (size: 2.7 KB, free: 730.6 MB) -2016-05-25 16:49:55 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:49:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[5] at sortBy at IsotonicRegression.scala:418) -2016-05-25 16:49:55 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-25 16:49:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-25 16:49:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-25 16:49:55 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 16:49:55 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-25 16:49:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:799+799 -2016-05-25 16:49:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:0+799 -2016-05-25 16:49:55 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2254 bytes result sent to driver -2016-05-25 16:49:55 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2254 bytes result sent to driver -2016-05-25 16:49:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 75 ms on localhost (1/2) -2016-05-25 16:49:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 78 ms on localhost (2/2) -2016-05-25 16:49:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 16:49:55 INFO DAGScheduler:59 - ShuffleMapStage 1 (sortBy at IsotonicRegression.scala:418) finished in 0.079 s -2016-05-25 16:49:55 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:49:55 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:49:55 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-05-25 16:49:55 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:49:55 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List() -2016-05-25 16:49:55 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[11] at flatMap at IsotonicRegression.scala:420), which is now runnable -2016-05-25 16:49:55 INFO MemoryStore:59 - ensureFreeSpace(3808) called with curMem=185690, maxMem=766075207 -2016-05-25 16:49:55 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-25 16:49:55 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=189498, maxMem=766075207 -2016-05-25 16:49:55 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-05-25 16:49:55 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55704 (size: 2.0 KB, free: 730.6 MB) -2016-05-25 16:49:55 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:49:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[11] at flatMap at IsotonicRegression.scala:420) -2016-05-25 16:49:55 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 16:49:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:49:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:49:55 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-25 16:49:55 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-25 16:49:55 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:49:55 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:49:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 33 ms -2016-05-25 16:49:55 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 33 ms -2016-05-25 16:49:55 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:55704 in memory (size: 2.7 KB, free: 730.6 MB) -2016-05-25 16:49:55 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:55704 in memory (size: 2.2 KB, free: 730.6 MB) -2016-05-25 16:49:55 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2115 bytes result sent to driver -2016-05-25 16:49:55 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1971 bytes result sent to driver -2016-05-25 16:49:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 171 ms on localhost (1/2) -2016-05-25 16:49:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 174 ms on localhost (2/2) -2016-05-25 16:49:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 16:49:55 INFO DAGScheduler:59 - ResultStage 2 (collect at IsotonicRegression.scala:421) finished in 0.175 s -2016-05-25 16:49:55 INFO DAGScheduler:59 - Job 1 finished: collect at IsotonicRegression.scala:421, took 0.297524 s -2016-05-25 16:49:55 INFO SparkContext:59 - Starting job: mean at IsotonicRegression1.scala:57 -2016-05-25 16:49:55 INFO DAGScheduler:59 - Got job 2 (mean at IsotonicRegression1.scala:57) with 2 output partitions -2016-05-25 16:49:55 INFO DAGScheduler:59 - Final stage: ResultStage 3(mean at IsotonicRegression1.scala:57) -2016-05-25 16:49:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:49:55 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:49:55 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[14] at mean at IsotonicRegression1.scala:57), which has no missing parents -2016-05-25 16:49:55 INFO MemoryStore:59 - ensureFreeSpace(5120) called with curMem=177472, maxMem=766075207 -2016-05-25 16:49:55 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 16:49:55 INFO MemoryStore:59 - ensureFreeSpace(2865) called with curMem=182592, maxMem=766075207 -2016-05-25 16:49:55 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-25 16:49:55 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55704 (size: 2.8 KB, free: 730.6 MB) -2016-05-25 16:49:55 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:49:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[14] at mean at IsotonicRegression1.scala:57) -2016-05-25 16:49:55 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 16:49:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:49:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:49:55 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-25 16:49:55 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-25 16:49:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:799+799 -2016-05-25 16:49:55 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:0+799 -2016-05-25 16:49:55 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 2209 bytes result sent to driver -2016-05-25 16:49:55 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 2209 bytes result sent to driver -2016-05-25 16:49:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 25 ms on localhost (1/2) -2016-05-25 16:49:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 28 ms on localhost (2/2) -2016-05-25 16:49:55 INFO DAGScheduler:59 - ResultStage 3 (mean at IsotonicRegression1.scala:57) finished in 0.028 s -2016-05-25 16:49:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 16:49:55 INFO DAGScheduler:59 - Job 2 finished: mean at IsotonicRegression1.scala:57, took 0.055496 s -2016-05-25 16:49:56 INFO SparkContext:59 - Starting job: saveAsTextFile at IsotonicRegression.scala:196 -2016-05-25 16:49:56 INFO DAGScheduler:59 - Got job 3 (saveAsTextFile at IsotonicRegression.scala:196) with 1 output partitions -2016-05-25 16:49:56 INFO DAGScheduler:59 - Final stage: ResultStage 4(saveAsTextFile at IsotonicRegression.scala:196) -2016-05-25 16:49:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:49:56 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:49:56 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[16] at saveAsTextFile at IsotonicRegression.scala:196), which has no missing parents -2016-05-25 16:49:56 INFO MemoryStore:59 - ensureFreeSpace(127008) called with curMem=185457, maxMem=766075207 -2016-05-25 16:49:56 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 124.0 KB, free 730.3 MB) -2016-05-25 16:49:56 INFO MemoryStore:59 - ensureFreeSpace(42649) called with curMem=312465, maxMem=766075207 -2016-05-25 16:49:56 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.2 MB) -2016-05-25 16:49:56 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55704 (size: 41.6 KB, free: 730.5 MB) -2016-05-25 16:49:56 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:49:56 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (MapPartitionsRDD[16] at saveAsTextFile at IsotonicRegression.scala:196) -2016-05-25 16:49:56 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-05-25 16:49:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-25 16:49:56 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-25 16:49:56 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251649_0004_m_000000_8' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525164955818/result/metadata/_temporary/0/task_201605251649_0004_m_000000 -2016-05-25 16:49:56 INFO SparkHadoopMapRedUtil:59 - attempt_201605251649_0004_m_000000_8: Committed -2016-05-25 16:49:56 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 1864 bytes result sent to driver -2016-05-25 16:49:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 332 ms on localhost (1/1) -2016-05-25 16:49:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 16:49:56 INFO DAGScheduler:59 - ResultStage 4 (saveAsTextFile at IsotonicRegression.scala:196) finished in 0.332 s -2016-05-25 16:49:56 INFO DAGScheduler:59 - Job 3 finished: saveAsTextFile at IsotonicRegression.scala:196, took 0.426135 s -2016-05-25 16:49:58 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55704 in memory (size: 41.6 KB, free: 730.6 MB) -2016-05-25 16:49:58 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 16:49:58 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:55704 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-25 16:49:58 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 16:49:58 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:55704 in memory (size: 2.0 KB, free: 730.6 MB) -2016-05-25 16:49:58 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:49:58 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:49:58 INFO SparkContext:59 - Starting job: parquet at IsotonicRegression.scala:200 -2016-05-25 16:49:58 INFO DAGScheduler:59 - Got job 4 (parquet at IsotonicRegression.scala:200) with 4 output partitions -2016-05-25 16:49:58 INFO DAGScheduler:59 - Final stage: ResultStage 5(parquet at IsotonicRegression.scala:200) -2016-05-25 16:49:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:49:58 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:49:58 INFO DAGScheduler:59 - Submitting ResultStage 5 (ParallelCollectionRDD[17] at parquet at IsotonicRegression.scala:200), which has no missing parents -2016-05-25 16:49:58 INFO MemoryStore:59 - ensureFreeSpace(65456) called with curMem=171580, maxMem=766075207 -2016-05-25 16:49:58 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 63.9 KB, free 730.4 MB) -2016-05-25 16:49:58 INFO MemoryStore:59 - ensureFreeSpace(22318) called with curMem=237036, maxMem=766075207 -2016-05-25 16:49:58 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 21.8 KB, free 730.3 MB) -2016-05-25 16:49:58 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55704 (size: 21.8 KB, free: 730.6 MB) -2016-05-25 16:49:58 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:49:58 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 5 (ParallelCollectionRDD[17] at parquet at IsotonicRegression.scala:200) -2016-05-25 16:49:58 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 4 tasks -2016-05-25 16:49:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2542 bytes) -2016-05-25 16:49:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2586 bytes) -2016-05-25 16:49:58 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2586 bytes) -2016-05-25 16:49:58 INFO TaskSetManager:59 - Starting task 3.0 in stage 5.0 (TID 12, localhost, PROCESS_LOCAL, 2586 bytes) -2016-05-25 16:49:58 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-05-25 16:49:58 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-05-25 16:49:58 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 11) -2016-05-25 16:49:58 INFO Executor:59 - Running task 3.0 in stage 5.0 (TID 12) -2016-05-25 16:49:58 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:49:58 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 16:49:58 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:49:58 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 16:49:58 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:49:58 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 16:49:58 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:49:58 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 16:49:58 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 16:49:58 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 16:49:58 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 16:49:58 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 16:49:58 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 16:49:58 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 144 -2016-05-25 16:49:58 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 128 -2016-05-25 16:49:58 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 120 -2016-05-25 16:49:58 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 104 -2016-05-25 16:49:58 INFO ColumnChunkPageWriteStore:151 - written 110B for [boundary] DOUBLE: 6 values, 54B raw, 68B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN] -2016-05-25 16:49:58 INFO ColumnChunkPageWriteStore:151 - written 119B for [prediction] DOUBLE: 6 values, 54B raw, 77B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN] -2016-05-25 16:49:58 INFO ColumnChunkPageWriteStore:151 - written 103B for [boundary] DOUBLE: 5 values, 46B raw, 62B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN] -2016-05-25 16:49:58 INFO ColumnChunkPageWriteStore:151 - written 111B for [boundary] DOUBLE: 6 values, 54B raw, 69B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN] -2016-05-25 16:49:58 INFO ColumnChunkPageWriteStore:151 - written 106B for [boundary] DOUBLE: 6 values, 54B raw, 64B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN] -2016-05-25 16:49:58 INFO ColumnChunkPageWriteStore:151 - written 71B for [prediction] DOUBLE: 6 values, 10B raw, 30B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 3 entries, 24B raw, 3B comp} -2016-05-25 16:49:58 INFO ColumnChunkPageWriteStore:151 - written 71B for [prediction] DOUBLE: 5 values, 10B raw, 30B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 3 entries, 24B raw, 3B comp} -2016-05-25 16:49:58 INFO ColumnChunkPageWriteStore:151 - written 71B for [prediction] DOUBLE: 6 values, 10B raw, 30B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE, BIT_PACKED], dic { 4 entries, 32B raw, 4B comp} -2016-05-25 16:49:59 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251649_0005_m_000003_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525164955818/result/data/_temporary/0/task_201605251649_0005_m_000003 -2016-05-25 16:49:59 INFO SparkHadoopMapRedUtil:59 - attempt_201605251649_0005_m_000003_0: Committed -2016-05-25 16:49:59 INFO Executor:59 - Finished task 3.0 in stage 5.0 (TID 12). 915 bytes result sent to driver -2016-05-25 16:49:59 INFO TaskSetManager:59 - Finished task 3.0 in stage 5.0 (TID 12) in 536 ms on localhost (1/4) -2016-05-25 16:49:59 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251649_0005_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525164955818/result/data/_temporary/0/task_201605251649_0005_m_000000 -2016-05-25 16:49:59 INFO SparkHadoopMapRedUtil:59 - attempt_201605251649_0005_m_000000_0: Committed -2016-05-25 16:49:59 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 915 bytes result sent to driver -2016-05-25 16:49:59 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251649_0005_m_000002_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525164955818/result/data/_temporary/0/task_201605251649_0005_m_000002 -2016-05-25 16:49:59 INFO SparkHadoopMapRedUtil:59 - attempt_201605251649_0005_m_000002_0: Committed -2016-05-25 16:49:59 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 11). 915 bytes result sent to driver -2016-05-25 16:49:59 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251649_0005_m_000001_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525164955818/result/data/_temporary/0/task_201605251649_0005_m_000001 -2016-05-25 16:49:59 INFO SparkHadoopMapRedUtil:59 - attempt_201605251649_0005_m_000001_0: Committed -2016-05-25 16:49:59 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 915 bytes result sent to driver -2016-05-25 16:49:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 559 ms on localhost (2/4) -2016-05-25 16:49:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 589 ms on localhost (3/4) -2016-05-25 16:49:59 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 11) in 559 ms on localhost (4/4) -2016-05-25 16:49:59 INFO DAGScheduler:59 - ResultStage 5 (parquet at IsotonicRegression.scala:200) finished in 0.590 s -2016-05-25 16:49:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 16:49:59 INFO DAGScheduler:59 - Job 4 finished: parquet at IsotonicRegression.scala:200, took 0.633666 s -2016-05-25 16:49:59 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 16:49:59 INFO DefaultWriterContainer:59 - Job job_201605251649_0000 committed. -2016-05-25 16:49:59 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525164955818/result/data on driver -2016-05-25 16:49:59 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525164955818/result/data on driver -2016-05-25 16:49:59 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=259354, maxMem=766075207 -2016-05-25 16:49:59 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-25 16:49:59 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=490058, maxMem=766075207 -2016-05-25 16:49:59 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-25 16:49:59 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55704 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 16:49:59 INFO SparkContext:59 - Created broadcast 7 from textFile at modelSaveLoad.scala:129 -2016-05-25 16:49:59 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 16:49:59 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 16:49:59 INFO DAGScheduler:59 - Got job 5 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 16:49:59 INFO DAGScheduler:59 - Final stage: ResultStage 6(first at modelSaveLoad.scala:129) -2016-05-25 16:49:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:49:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:49:59 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[20] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 16:49:59 INFO MemoryStore:59 - ensureFreeSpace(3224) called with curMem=509846, maxMem=766075207 -2016-05-25 16:49:59 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-25 16:49:59 INFO MemoryStore:59 - ensureFreeSpace(1901) called with curMem=513070, maxMem=766075207 -2016-05-25 16:49:59 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 1901.0 B, free 730.1 MB) -2016-05-25 16:49:59 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55704 (size: 1901.0 B, free: 730.5 MB) -2016-05-25 16:49:59 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:49:59 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 6 (MapPartitionsRDD[20] at textFile at modelSaveLoad.scala:129) -2016-05-25 16:49:59 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 1 tasks -2016-05-25 16:49:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2247 bytes) -2016-05-25 16:49:59 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 13) -2016-05-25 16:49:59 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525164955818/result/metadata/part-00000:0+51 -2016-05-25 16:49:59 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 13). 2148 bytes result sent to driver -2016-05-25 16:49:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 13) in 15 ms on localhost (1/1) -2016-05-25 16:49:59 INFO DAGScheduler:59 - ResultStage 6 (first at modelSaveLoad.scala:129) finished in 0.015 s -2016-05-25 16:49:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 16:49:59 INFO DAGScheduler:59 - Job 5 finished: first at modelSaveLoad.scala:129, took 0.032121 s -2016-05-25 16:49:59 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525164955818/result/data on driver -2016-05-25 16:49:59 INFO SparkContext:59 - Starting job: parquet at IsotonicRegression.scala:205 -2016-05-25 16:49:59 INFO DAGScheduler:59 - Got job 6 (parquet at IsotonicRegression.scala:205) with 4 output partitions -2016-05-25 16:49:59 INFO DAGScheduler:59 - Final stage: ResultStage 7(parquet at IsotonicRegression.scala:205) -2016-05-25 16:49:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:49:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:49:59 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[22] at parquet at IsotonicRegression.scala:205), which has no missing parents -2016-05-25 16:49:59 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=514971, maxMem=766075207 -2016-05-25 16:49:59 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-25 16:49:59 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=577971, maxMem=766075207 -2016-05-25 16:49:59 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.0 MB) -2016-05-25 16:49:59 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55704 (size: 20.6 KB, free: 730.5 MB) -2016-05-25 16:49:59 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:49:59 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 7 (MapPartitionsRDD[22] at parquet at IsotonicRegression.scala:205) -2016-05-25 16:49:59 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 4 tasks -2016-05-25 16:49:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 16:49:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 15, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 16:49:59 INFO TaskSetManager:59 - Starting task 2.0 in stage 7.0 (TID 16, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 16:49:59 INFO TaskSetManager:59 - Starting task 3.0 in stage 7.0 (TID 17, localhost, PROCESS_LOCAL, 2288 bytes) -2016-05-25 16:49:59 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 14) -2016-05-25 16:49:59 INFO Executor:59 - Running task 3.0 in stage 7.0 (TID 17) -2016-05-25 16:49:59 INFO Executor:59 - Running task 2.0 in stage 7.0 (TID 16) -2016-05-25 16:49:59 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 15) -2016-05-25 16:49:59 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 16:49:59 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 14). 936 bytes result sent to driver -2016-05-25 16:49:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 14) in 46 ms on localhost (1/4) -2016-05-25 16:49:59 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 16:49:59 INFO Executor:59 - Finished task 2.0 in stage 7.0 (TID 16). 936 bytes result sent to driver -2016-05-25 16:49:59 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 16:49:59 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 15). 936 bytes result sent to driver -2016-05-25 16:49:59 INFO TaskSetManager:59 - Finished task 2.0 in stage 7.0 (TID 16) in 118 ms on localhost (2/4) -2016-05-25 16:49:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 15) in 121 ms on localhost (3/4) -2016-05-25 16:49:59 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 16:49:59 INFO Executor:59 - Finished task 3.0 in stage 7.0 (TID 17). 1661 bytes result sent to driver -2016-05-25 16:49:59 INFO TaskSetManager:59 - Finished task 3.0 in stage 7.0 (TID 17) in 146 ms on localhost (4/4) -2016-05-25 16:49:59 INFO DAGScheduler:59 - ResultStage 7 (parquet at IsotonicRegression.scala:205) finished in 0.149 s -2016-05-25 16:49:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 16:49:59 INFO DAGScheduler:59 - Job 6 finished: parquet at IsotonicRegression.scala:205, took 0.176153 s -2016-05-25 16:50:00 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=599044, maxMem=766075207 -2016-05-25 16:50:00 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 86.4 KB, free 729.9 MB) -2016-05-25 16:50:00 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=687556, maxMem=766075207 -2016-05-25 16:50:00 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-25 16:50:00 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55704 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 16:50:00 INFO SparkContext:59 - Created broadcast 10 from collect at IsotonicRegression.scala:208 -2016-05-25 16:50:00 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 16:50:00 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525164955818/result/data/part-r-00001-db0e3d18-cb1c-47dd-abd2-4dd44122276e.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525164955818/result/data/part-r-00002-db0e3d18-cb1c-47dd-abd2-4dd44122276e.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525164955818/result/data/part-r-00000-db0e3d18-cb1c-47dd-abd2-4dd44122276e.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525164955818/result/data/part-r-00003-db0e3d18-cb1c-47dd-abd2-4dd44122276e.gz.parquet -2016-05-25 16:50:00 INFO SparkContext:59 - Starting job: collect at IsotonicRegression.scala:208 -2016-05-25 16:50:00 INFO DAGScheduler:59 - Got job 7 (collect at IsotonicRegression.scala:208) with 4 output partitions -2016-05-25 16:50:00 INFO DAGScheduler:59 - Final stage: ResultStage 8(collect at IsotonicRegression.scala:208) -2016-05-25 16:50:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:50:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:50:00 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[24] at collect at IsotonicRegression.scala:208), which has no missing parents -2016-05-25 16:50:00 INFO MemoryStore:59 - ensureFreeSpace(5056) called with curMem=707344, maxMem=766075207 -2016-05-25 16:50:00 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.9 KB, free 729.9 MB) -2016-05-25 16:50:00 INFO MemoryStore:59 - ensureFreeSpace(2938) called with curMem=712400, maxMem=766075207 -2016-05-25 16:50:00 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.9 KB, free 729.9 MB) -2016-05-25 16:50:00 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55704 (size: 2.9 KB, free: 730.5 MB) -2016-05-25 16:50:00 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:50:00 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 8 (MapPartitionsRDD[24] at collect at IsotonicRegression.scala:208) -2016-05-25 16:50:00 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 4 tasks -2016-05-25 16:50:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 18, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 16:50:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 19, localhost, PROCESS_LOCAL, 2308 bytes) -2016-05-25 16:50:00 INFO TaskSetManager:59 - Starting task 2.0 in stage 8.0 (TID 20, localhost, PROCESS_LOCAL, 2308 bytes) -2016-05-25 16:50:00 INFO TaskSetManager:59 - Starting task 3.0 in stage 8.0 (TID 21, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 16:50:00 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 18) -2016-05-25 16:50:00 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 19) -2016-05-25 16:50:00 INFO Executor:59 - Running task 2.0 in stage 8.0 (TID 20) -2016-05-25 16:50:00 INFO Executor:59 - Running task 3.0 in stage 8.0 (TID 21) -2016-05-25 16:50:00 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525164955818/result/data/part-r-00003-db0e3d18-cb1c-47dd-abd2-4dd44122276e.gz.parquet start: 0 end: 669 length: 669 hosts: []} -2016-05-25 16:50:00 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525164955818/result/data/part-r-00000-db0e3d18-cb1c-47dd-abd2-4dd44122276e.gz.parquet start: 0 end: 661 length: 661 hosts: []} -2016-05-25 16:50:00 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525164955818/result/data/part-r-00001-db0e3d18-cb1c-47dd-abd2-4dd44122276e.gz.parquet start: 0 end: 674 length: 674 hosts: []} -2016-05-25 16:50:00 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 16:50:00 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525164955818/result/data/part-r-00002-db0e3d18-cb1c-47dd-abd2-4dd44122276e.gz.parquet start: 0 end: 658 length: 658 hosts: []} -2016-05-25 16:50:00 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 16:50:00 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 16:50:00 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 16:50:00 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional double boundary; - optional double prediction; -} - -Catalyst form: -StructType(StructField(boundary,DoubleType,true), StructField(prediction,DoubleType,true)) - -2016-05-25 16:50:00 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional double boundary; - optional double prediction; -} - -Catalyst form: -StructType(StructField(boundary,DoubleType,true), StructField(prediction,DoubleType,true)) - -2016-05-25 16:50:00 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional double boundary; - optional double prediction; -} - -Catalyst form: -StructType(StructField(boundary,DoubleType,true), StructField(prediction,DoubleType,true)) - -2016-05-25 16:50:00 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional double boundary; - optional double prediction; -} - -Catalyst form: -StructType(StructField(boundary,DoubleType,true), StructField(prediction,DoubleType,true)) - -2016-05-25 16:50:00 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 6 records. -2016-05-25 16:50:00 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 6 records. -2016-05-25 16:50:00 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 5 records. -2016-05-25 16:50:00 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 6 records. -2016-05-25 16:50:00 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 16:50:00 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 16:50:00 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 16:50:00 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 16:50:00 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 16:50:00 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 16:50:00 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 16:50:00 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 16:50:00 INFO InternalParquetRecordReader:151 - block read in memory in 11 ms. row count = 6 -2016-05-25 16:50:00 INFO InternalParquetRecordReader:151 - block read in memory in 10 ms. row count = 6 -2016-05-25 16:50:00 INFO InternalParquetRecordReader:151 - block read in memory in 10 ms. row count = 6 -2016-05-25 16:50:00 INFO InternalParquetRecordReader:151 - block read in memory in 11 ms. row count = 5 -2016-05-25 16:50:00 INFO Executor:59 - Finished task 2.0 in stage 8.0 (TID 20). 3340 bytes result sent to driver -2016-05-25 16:50:00 INFO Executor:59 - Finished task 3.0 in stage 8.0 (TID 21). 3389 bytes result sent to driver -2016-05-25 16:50:00 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 19). 3389 bytes result sent to driver -2016-05-25 16:50:00 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 18). 3389 bytes result sent to driver -2016-05-25 16:50:00 INFO TaskSetManager:59 - Finished task 2.0 in stage 8.0 (TID 20) in 113 ms on localhost (1/4) -2016-05-25 16:50:00 INFO TaskSetManager:59 - Finished task 3.0 in stage 8.0 (TID 21) in 112 ms on localhost (2/4) -2016-05-25 16:50:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 18) in 119 ms on localhost (3/4) -2016-05-25 16:50:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 19) in 118 ms on localhost (4/4) -2016-05-25 16:50:00 INFO DAGScheduler:59 - ResultStage 8 (collect at IsotonicRegression.scala:208) finished in 0.121 s -2016-05-25 16:50:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 16:50:00 INFO DAGScheduler:59 - Job 7 finished: collect at IsotonicRegression.scala:208, took 0.136161 s -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 16:50:00 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 16:50:00 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 16:50:00 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 16:50:00 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 16:50:00 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 16:50:00 INFO BlockManager:59 - BlockManager stopped -2016-05-25 16:50:00 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 16:50:00 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 16:50:00 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 16:50:00 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 16:50:00 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-b5437c77-05f7-4bb5-8390-0999b33419a1 -2016-05-25 16:50:00 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 16:57:17 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 16:57:18 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 16:57:18 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 16:57:18 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 16:57:18 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 16:57:19 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 16:57:19 INFO Remoting:74 - Starting remoting -2016-05-25 16:57:19 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:55908] -2016-05-25 16:57:19 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55908. -2016-05-25 16:57:19 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 16:57:19 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 16:57:19 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-de771166-e621-466f-a437-502307bdaedc -2016-05-25 16:57:19 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 16:57:19 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-02daeafd-0924-4f4e-aca1-5c35565d039a\httpd-085eece5-254e-4244-a73f-88638e3267ce -2016-05-25 16:57:19 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 16:57:19 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 16:57:19 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55909 -2016-05-25 16:57:19 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55909. -2016-05-25 16:57:19 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 16:57:20 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 16:57:20 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 16:57:20 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 16:57:20 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 16:57:20 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 16:57:20 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 16:57:20 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55928. -2016-05-25 16:57:20 INFO NettyBlockTransferService:59 - Server created on 55928 -2016-05-25 16:57:20 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 16:57:20 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55928 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55928) -2016-05-25 16:57:20 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 16:57:21 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 16:57:21 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 16:57:21 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 16:57:21 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 16:57:21 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55928 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 16:57:21 INFO SparkContext:59 - Created broadcast 0 from textFile at IsotonicRegression1.scala:33 -2016-05-25 16:57:22 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 16:57:23 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 16:57:23 INFO SparkContext:59 - Starting job: sortBy at IsotonicRegression.scala:418 -2016-05-25 16:57:23 INFO DAGScheduler:59 - Got job 0 (sortBy at IsotonicRegression.scala:418) with 2 output partitions -2016-05-25 16:57:23 INFO DAGScheduler:59 - Final stage: ResultStage 0(sortBy at IsotonicRegression.scala:418) -2016-05-25 16:57:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:57:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:57:23 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[7] at sortBy at IsotonicRegression.scala:418), which has no missing parents -2016-05-25 16:57:23 INFO MemoryStore:59 - ensureFreeSpace(4256) called with curMem=171580, maxMem=766075207 -2016-05-25 16:57:23 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.2 KB, free 730.4 MB) -2016-05-25 16:57:23 INFO MemoryStore:59 - ensureFreeSpace(2276) called with curMem=175836, maxMem=766075207 -2016-05-25 16:57:23 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-05-25 16:57:23 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55928 (size: 2.2 KB, free: 730.6 MB) -2016-05-25 16:57:23 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:57:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[7] at sortBy at IsotonicRegression.scala:418) -2016-05-25 16:57:23 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 16:57:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:57:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:57:23 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 16:57:23 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 16:57:23 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:799+799 -2016-05-25 16:57:23 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:0+799 -2016-05-25 16:57:23 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 16:57:23 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 16:57:23 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 16:57:23 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 16:57:23 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 16:57:23 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 3161 bytes result sent to driver -2016-05-25 16:57:23 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 3065 bytes result sent to driver -2016-05-25 16:57:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 117 ms on localhost (1/2) -2016-05-25 16:57:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 101 ms on localhost (2/2) -2016-05-25 16:57:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 16:57:23 INFO DAGScheduler:59 - ResultStage 0 (sortBy at IsotonicRegression.scala:418) finished in 0.138 s -2016-05-25 16:57:23 INFO DAGScheduler:59 - Job 0 finished: sortBy at IsotonicRegression.scala:418, took 0.223131 s -2016-05-25 16:57:23 INFO SparkContext:59 - Starting job: collect at IsotonicRegression.scala:421 -2016-05-25 16:57:23 INFO DAGScheduler:59 - Registering RDD 5 (sortBy at IsotonicRegression.scala:418) -2016-05-25 16:57:23 INFO DAGScheduler:59 - Got job 1 (collect at IsotonicRegression.scala:421) with 2 output partitions -2016-05-25 16:57:23 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at IsotonicRegression.scala:421) -2016-05-25 16:57:23 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1) -2016-05-25 16:57:23 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1) -2016-05-25 16:57:23 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1 (MapPartitionsRDD[5] at sortBy at IsotonicRegression.scala:418), which has no missing parents -2016-05-25 16:57:23 INFO MemoryStore:59 - ensureFreeSpace(4864) called with curMem=178112, maxMem=766075207 -2016-05-25 16:57:23 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-25 16:57:23 INFO MemoryStore:59 - ensureFreeSpace(2714) called with curMem=182976, maxMem=766075207 -2016-05-25 16:57:23 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-05-25 16:57:23 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55928 (size: 2.7 KB, free: 730.6 MB) -2016-05-25 16:57:23 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:57:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[5] at sortBy at IsotonicRegression.scala:418) -2016-05-25 16:57:23 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-25 16:57:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-25 16:57:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-25 16:57:23 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-25 16:57:23 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 16:57:23 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:799+799 -2016-05-25 16:57:23 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:0+799 -2016-05-25 16:57:23 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2254 bytes result sent to driver -2016-05-25 16:57:23 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2254 bytes result sent to driver -2016-05-25 16:57:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 85 ms on localhost (1/2) -2016-05-25 16:57:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 83 ms on localhost (2/2) -2016-05-25 16:57:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 16:57:23 INFO DAGScheduler:59 - ShuffleMapStage 1 (sortBy at IsotonicRegression.scala:418) finished in 0.087 s -2016-05-25 16:57:23 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:57:23 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:57:23 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-05-25 16:57:23 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:57:23 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List() -2016-05-25 16:57:23 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[11] at flatMap at IsotonicRegression.scala:420), which is now runnable -2016-05-25 16:57:23 INFO MemoryStore:59 - ensureFreeSpace(3808) called with curMem=185690, maxMem=766075207 -2016-05-25 16:57:23 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-25 16:57:23 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=189498, maxMem=766075207 -2016-05-25 16:57:23 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-05-25 16:57:23 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55928 (size: 2.0 KB, free: 730.6 MB) -2016-05-25 16:57:23 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:57:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[11] at flatMap at IsotonicRegression.scala:420) -2016-05-25 16:57:23 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 16:57:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:57:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:57:23 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-25 16:57:23 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-25 16:57:23 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:55928 in memory (size: 2.2 KB, free: 730.6 MB) -2016-05-25 16:57:23 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:57:23 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:57:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 16:57:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 16:57:23 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2115 bytes result sent to driver -2016-05-25 16:57:23 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1971 bytes result sent to driver -2016-05-25 16:57:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 91 ms on localhost (1/2) -2016-05-25 16:57:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 93 ms on localhost (2/2) -2016-05-25 16:57:23 INFO DAGScheduler:59 - ResultStage 2 (collect at IsotonicRegression.scala:421) finished in 0.094 s -2016-05-25 16:57:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 16:57:23 INFO DAGScheduler:59 - Job 1 finished: collect at IsotonicRegression.scala:421, took 0.261937 s -2016-05-25 16:57:23 INFO SparkContext:59 - Starting job: mean at IsotonicRegression1.scala:57 -2016-05-25 16:57:23 INFO DAGScheduler:59 - Got job 2 (mean at IsotonicRegression1.scala:57) with 2 output partitions -2016-05-25 16:57:23 INFO DAGScheduler:59 - Final stage: ResultStage 3(mean at IsotonicRegression1.scala:57) -2016-05-25 16:57:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:57:23 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:57:23 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[14] at mean at IsotonicRegression1.scala:57), which has no missing parents -2016-05-25 16:57:23 INFO MemoryStore:59 - ensureFreeSpace(5120) called with curMem=185050, maxMem=766075207 -2016-05-25 16:57:23 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 16:57:23 INFO MemoryStore:59 - ensureFreeSpace(2865) called with curMem=190170, maxMem=766075207 -2016-05-25 16:57:23 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-25 16:57:23 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55928 (size: 2.8 KB, free: 730.6 MB) -2016-05-25 16:57:23 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:57:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[14] at mean at IsotonicRegression1.scala:57) -2016-05-25 16:57:23 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 16:57:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:57:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:57:23 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-25 16:57:23 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-25 16:57:23 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:799+799 -2016-05-25 16:57:23 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:0+799 -2016-05-25 16:57:23 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 2209 bytes result sent to driver -2016-05-25 16:57:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 23 ms on localhost (1/2) -2016-05-25 16:57:23 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 2209 bytes result sent to driver -2016-05-25 16:57:24 INFO DAGScheduler:59 - ResultStage 3 (mean at IsotonicRegression1.scala:57) finished in 0.033 s -2016-05-25 16:57:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 33 ms on localhost (2/2) -2016-05-25 16:57:24 INFO DAGScheduler:59 - Job 2 finished: mean at IsotonicRegression1.scala:57, took 0.052477 s -2016-05-25 16:57:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 16:57:24 INFO SparkContext:59 - Starting job: count at IsotonicRegression1.scala:59 -2016-05-25 16:57:24 INFO DAGScheduler:59 - Got job 3 (count at IsotonicRegression1.scala:59) with 2 output partitions -2016-05-25 16:57:24 INFO DAGScheduler:59 - Final stage: ResultStage 4(count at IsotonicRegression1.scala:59) -2016-05-25 16:57:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:57:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:57:24 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[1] at textFile at IsotonicRegression1.scala:33), which has no missing parents -2016-05-25 16:57:24 INFO MemoryStore:59 - ensureFreeSpace(3032) called with curMem=193035, maxMem=766075207 -2016-05-25 16:57:24 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.0 KB, free 730.4 MB) -2016-05-25 16:57:24 INFO MemoryStore:59 - ensureFreeSpace(1810) called with curMem=196067, maxMem=766075207 -2016-05-25 16:57:24 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 1810.0 B, free 730.4 MB) -2016-05-25 16:57:24 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55928 (size: 1810.0 B, free: 730.6 MB) -2016-05-25 16:57:24 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:57:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[1] at textFile at IsotonicRegression1.scala:33) -2016-05-25 16:57:24 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 16:57:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:57:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:57:24 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-25 16:57:24 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-25 16:57:24 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:799+799 -2016-05-25 16:57:24 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:0+799 -2016-05-25 16:57:24 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2082 bytes result sent to driver -2016-05-25 16:57:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 18 ms on localhost (1/2) -2016-05-25 16:57:24 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2082 bytes result sent to driver -2016-05-25 16:57:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 24 ms on localhost (2/2) -2016-05-25 16:57:24 INFO DAGScheduler:59 - ResultStage 4 (count at IsotonicRegression1.scala:59) finished in 0.025 s -2016-05-25 16:57:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 16:57:24 INFO DAGScheduler:59 - Job 3 finished: count at IsotonicRegression1.scala:59, took 0.037804 s -2016-05-25 16:57:24 INFO SparkContext:59 - Starting job: count at IsotonicRegression1.scala:60 -2016-05-25 16:57:24 INFO DAGScheduler:59 - Got job 4 (count at IsotonicRegression1.scala:60) with 2 output partitions -2016-05-25 16:57:24 INFO DAGScheduler:59 - Final stage: ResultStage 5(count at IsotonicRegression1.scala:60) -2016-05-25 16:57:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:57:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:57:24 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[3] at randomSplit at IsotonicRegression1.scala:42), which has no missing parents -2016-05-25 16:57:24 INFO MemoryStore:59 - ensureFreeSpace(3544) called with curMem=197877, maxMem=766075207 -2016-05-25 16:57:24 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 16:57:24 INFO MemoryStore:59 - ensureFreeSpace(2071) called with curMem=201421, maxMem=766075207 -2016-05-25 16:57:24 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-05-25 16:57:24 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55928 (size: 2.0 KB, free: 730.6 MB) -2016-05-25 16:57:24 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:57:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[3] at randomSplit at IsotonicRegression1.scala:42) -2016-05-25 16:57:24 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 16:57:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:57:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:57:24 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-05-25 16:57:24 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-05-25 16:57:24 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:0+799 -2016-05-25 16:57:24 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:799+799 -2016-05-25 16:57:24 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 2082 bytes result sent to driver -2016-05-25 16:57:24 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 2082 bytes result sent to driver -2016-05-25 16:57:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 25 ms on localhost (1/2) -2016-05-25 16:57:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 24 ms on localhost (2/2) -2016-05-25 16:57:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 16:57:24 INFO DAGScheduler:59 - ResultStage 5 (count at IsotonicRegression1.scala:60) finished in 0.026 s -2016-05-25 16:57:24 INFO DAGScheduler:59 - Job 4 finished: count at IsotonicRegression1.scala:60, took 0.044501 s -2016-05-25 16:57:24 INFO SparkContext:59 - Starting job: count at IsotonicRegression1.scala:61 -2016-05-25 16:57:24 INFO DAGScheduler:59 - Got job 5 (count at IsotonicRegression1.scala:61) with 2 output partitions -2016-05-25 16:57:24 INFO DAGScheduler:59 - Final stage: ResultStage 6(count at IsotonicRegression1.scala:61) -2016-05-25 16:57:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:57:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:57:24 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[4] at randomSplit at IsotonicRegression1.scala:42), which has no missing parents -2016-05-25 16:57:24 INFO MemoryStore:59 - ensureFreeSpace(3544) called with curMem=203492, maxMem=766075207 -2016-05-25 16:57:24 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 16:57:24 INFO MemoryStore:59 - ensureFreeSpace(2075) called with curMem=207036, maxMem=766075207 -2016-05-25 16:57:24 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-05-25 16:57:24 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55928 (size: 2.0 KB, free: 730.6 MB) -2016-05-25 16:57:24 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:57:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[4] at randomSplit at IsotonicRegression1.scala:42) -2016-05-25 16:57:24 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 16:57:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:57:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:57:24 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-05-25 16:57:24 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 13) -2016-05-25 16:57:24 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:0+799 -2016-05-25 16:57:24 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:799+799 -2016-05-25 16:57:24 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 2082 bytes result sent to driver -2016-05-25 16:57:24 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 13). 2082 bytes result sent to driver -2016-05-25 16:57:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 13) in 22 ms on localhost (1/2) -2016-05-25 16:57:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 23 ms on localhost (2/2) -2016-05-25 16:57:24 INFO DAGScheduler:59 - ResultStage 6 (count at IsotonicRegression1.scala:61) finished in 0.025 s -2016-05-25 16:57:24 INFO DAGScheduler:59 - Job 5 finished: count at IsotonicRegression1.scala:61, took 0.039853 s -2016-05-25 16:57:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 16:57:24 INFO SparkContext:59 - Starting job: take at IsotonicRegression1.scala:66 -2016-05-25 16:57:24 INFO DAGScheduler:59 - Got job 6 (take at IsotonicRegression1.scala:66) with 1 output partitions -2016-05-25 16:57:24 INFO DAGScheduler:59 - Final stage: ResultStage 7(take at IsotonicRegression1.scala:66) -2016-05-25 16:57:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:57:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:57:24 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[12] at map at IsotonicRegression1.scala:51), which has no missing parents -2016-05-25 16:57:24 INFO MemoryStore:59 - ensureFreeSpace(4552) called with curMem=209111, maxMem=766075207 -2016-05-25 16:57:24 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-25 16:57:24 INFO MemoryStore:59 - ensureFreeSpace(2634) called with curMem=213663, maxMem=766075207 -2016-05-25 16:57:24 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 16:57:24 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55928 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 16:57:24 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:57:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (MapPartitionsRDD[12] at map at IsotonicRegression1.scala:51) -2016-05-25 16:57:24 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-25 16:57:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:57:24 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 14) -2016-05-25 16:57:24 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:0+799 -2016-05-25 16:57:24 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 14). 2401 bytes result sent to driver -2016-05-25 16:57:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 14) in 19 ms on localhost (1/1) -2016-05-25 16:57:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 16:57:24 INFO DAGScheduler:59 - ResultStage 7 (take at IsotonicRegression1.scala:66) finished in 0.020 s -2016-05-25 16:57:24 INFO DAGScheduler:59 - Job 6 finished: take at IsotonicRegression1.scala:66, took 0.039210 s -2016-05-25 16:57:24 INFO SparkContext:59 - Starting job: saveAsTextFile at IsotonicRegression.scala:196 -2016-05-25 16:57:24 INFO DAGScheduler:59 - Got job 7 (saveAsTextFile at IsotonicRegression.scala:196) with 1 output partitions -2016-05-25 16:57:24 INFO DAGScheduler:59 - Final stage: ResultStage 8(saveAsTextFile at IsotonicRegression.scala:196) -2016-05-25 16:57:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:57:24 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:57:24 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[16] at saveAsTextFile at IsotonicRegression.scala:196), which has no missing parents -2016-05-25 16:57:24 INFO MemoryStore:59 - ensureFreeSpace(127008) called with curMem=216297, maxMem=766075207 -2016-05-25 16:57:24 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 124.0 KB, free 730.3 MB) -2016-05-25 16:57:24 INFO MemoryStore:59 - ensureFreeSpace(42645) called with curMem=343305, maxMem=766075207 -2016-05-25 16:57:24 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.2 MB) -2016-05-25 16:57:24 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55928 (size: 41.6 KB, free: 730.5 MB) -2016-05-25 16:57:24 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:57:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[16] at saveAsTextFile at IsotonicRegression.scala:196) -2016-05-25 16:57:24 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-05-25 16:57:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-25 16:57:24 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 16:57:24 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251657_0008_m_000000_15' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165724203/result/metadata/_temporary/0/task_201605251657_0008_m_000000 -2016-05-25 16:57:24 INFO SparkHadoopMapRedUtil:59 - attempt_201605251657_0008_m_000000_15: Committed -2016-05-25 16:57:24 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 1864 bytes result sent to driver -2016-05-25 16:57:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 199 ms on localhost (1/1) -2016-05-25 16:57:24 INFO DAGScheduler:59 - ResultStage 8 (saveAsTextFile at IsotonicRegression.scala:196) finished in 0.200 s -2016-05-25 16:57:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 16:57:24 INFO DAGScheduler:59 - Job 7 finished: saveAsTextFile at IsotonicRegression.scala:196, took 0.284049 s -2016-05-25 16:57:25 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:55928 in memory (size: 41.6 KB, free: 730.6 MB) -2016-05-25 16:57:25 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 16:57:25 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:55928 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 16:57:25 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 16:57:25 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:55928 in memory (size: 2.0 KB, free: 730.6 MB) -2016-05-25 16:57:25 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 16:57:25 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:55928 in memory (size: 2.0 KB, free: 730.6 MB) -2016-05-25 16:57:25 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 16:57:25 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55928 in memory (size: 1810.0 B, free: 730.6 MB) -2016-05-25 16:57:25 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 16:57:25 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:55928 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-25 16:57:25 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 16:57:26 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:57:26 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:57:26 INFO SparkContext:59 - Starting job: parquet at IsotonicRegression.scala:200 -2016-05-25 16:57:26 INFO DAGScheduler:59 - Got job 8 (parquet at IsotonicRegression.scala:200) with 4 output partitions -2016-05-25 16:57:26 INFO DAGScheduler:59 - Final stage: ResultStage 9(parquet at IsotonicRegression.scala:200) -2016-05-25 16:57:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:57:26 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:57:26 INFO DAGScheduler:59 - Submitting ResultStage 9 (ParallelCollectionRDD[17] at parquet at IsotonicRegression.scala:200), which has no missing parents -2016-05-25 16:57:26 INFO MemoryStore:59 - ensureFreeSpace(65456) called with curMem=185050, maxMem=766075207 -2016-05-25 16:57:26 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 63.9 KB, free 730.3 MB) -2016-05-25 16:57:26 INFO MemoryStore:59 - ensureFreeSpace(22322) called with curMem=250506, maxMem=766075207 -2016-05-25 16:57:26 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 21.8 KB, free 730.3 MB) -2016-05-25 16:57:26 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55928 (size: 21.8 KB, free: 730.5 MB) -2016-05-25 16:57:26 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:57:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 9 (ParallelCollectionRDD[17] at parquet at IsotonicRegression.scala:200) -2016-05-25 16:57:26 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 4 tasks -2016-05-25 16:57:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 16, localhost, PROCESS_LOCAL, 2542 bytes) -2016-05-25 16:57:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2586 bytes) -2016-05-25 16:57:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2586 bytes) -2016-05-25 16:57:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 9.0 (TID 19, localhost, PROCESS_LOCAL, 2586 bytes) -2016-05-25 16:57:26 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 16) -2016-05-25 16:57:26 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 17) -2016-05-25 16:57:26 INFO Executor:59 - Running task 2.0 in stage 9.0 (TID 18) -2016-05-25 16:57:26 INFO Executor:59 - Running task 3.0 in stage 9.0 (TID 19) -2016-05-25 16:57:26 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:57:26 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 16:57:26 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:57:26 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 16:57:26 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:57:26 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 16:57:26 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:57:26 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 16:57:26 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 16:57:26 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 16:57:26 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 16:57:26 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 16:57:26 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 16:57:26 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 104 -2016-05-25 16:57:26 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 128 -2016-05-25 16:57:26 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 120 -2016-05-25 16:57:26 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 144 -2016-05-25 16:57:27 INFO ColumnChunkPageWriteStore:151 - written 110B for [boundary] DOUBLE: 6 values, 54B raw, 68B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN] -2016-05-25 16:57:27 INFO ColumnChunkPageWriteStore:151 - written 119B for [prediction] DOUBLE: 6 values, 54B raw, 77B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN] -2016-05-25 16:57:27 INFO ColumnChunkPageWriteStore:151 - written 111B for [boundary] DOUBLE: 6 values, 54B raw, 69B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN] -2016-05-25 16:57:27 INFO ColumnChunkPageWriteStore:151 - written 103B for [boundary] DOUBLE: 5 values, 46B raw, 62B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN] -2016-05-25 16:57:27 INFO ColumnChunkPageWriteStore:151 - written 106B for [boundary] DOUBLE: 6 values, 54B raw, 64B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN] -2016-05-25 16:57:27 INFO ColumnChunkPageWriteStore:151 - written 71B for [prediction] DOUBLE: 5 values, 10B raw, 30B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN_DICTIONARY], dic { 3 entries, 24B raw, 3B comp} -2016-05-25 16:57:27 INFO ColumnChunkPageWriteStore:151 - written 71B for [prediction] DOUBLE: 6 values, 10B raw, 30B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN_DICTIONARY], dic { 3 entries, 24B raw, 3B comp} -2016-05-25 16:57:27 INFO ColumnChunkPageWriteStore:151 - written 71B for [prediction] DOUBLE: 6 values, 10B raw, 30B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN_DICTIONARY], dic { 4 entries, 32B raw, 4B comp} -2016-05-25 16:57:27 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251657_0009_m_000003_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165724203/result/data/_temporary/0/task_201605251657_0009_m_000003 -2016-05-25 16:57:27 INFO SparkHadoopMapRedUtil:59 - attempt_201605251657_0009_m_000003_0: Committed -2016-05-25 16:57:27 INFO Executor:59 - Finished task 3.0 in stage 9.0 (TID 19). 915 bytes result sent to driver -2016-05-25 16:57:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 9.0 (TID 19) in 601 ms on localhost (1/4) -2016-05-25 16:57:27 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251657_0009_m_000002_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165724203/result/data/_temporary/0/task_201605251657_0009_m_000002 -2016-05-25 16:57:27 INFO SparkHadoopMapRedUtil:59 - attempt_201605251657_0009_m_000002_0: Committed -2016-05-25 16:57:27 INFO Executor:59 - Finished task 2.0 in stage 9.0 (TID 18). 915 bytes result sent to driver -2016-05-25 16:57:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 9.0 (TID 18) in 610 ms on localhost (2/4) -2016-05-25 16:57:27 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251657_0009_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165724203/result/data/_temporary/0/task_201605251657_0009_m_000000 -2016-05-25 16:57:27 INFO SparkHadoopMapRedUtil:59 - attempt_201605251657_0009_m_000000_0: Committed -2016-05-25 16:57:27 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251657_0009_m_000001_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165724203/result/data/_temporary/0/task_201605251657_0009_m_000001 -2016-05-25 16:57:27 INFO SparkHadoopMapRedUtil:59 - attempt_201605251657_0009_m_000001_0: Committed -2016-05-25 16:57:27 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 17). 915 bytes result sent to driver -2016-05-25 16:57:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 17) in 618 ms on localhost (3/4) -2016-05-25 16:57:27 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 16). 915 bytes result sent to driver -2016-05-25 16:57:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 16) in 626 ms on localhost (4/4) -2016-05-25 16:57:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 16:57:27 INFO DAGScheduler:59 - ResultStage 9 (parquet at IsotonicRegression.scala:200) finished in 0.626 s -2016-05-25 16:57:27 INFO DAGScheduler:59 - Job 8 finished: parquet at IsotonicRegression.scala:200, took 0.664564 s -2016-05-25 16:57:27 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 16:57:27 INFO DefaultWriterContainer:59 - Job job_201605251657_0000 committed. -2016-05-25 16:57:27 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165724203/result/data on driver -2016-05-25 16:57:27 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165724203/result/data on driver -2016-05-25 16:57:27 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=272828, maxMem=766075207 -2016-05-25 16:57:27 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-25 16:57:27 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=503532, maxMem=766075207 -2016-05-25 16:57:27 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-25 16:57:27 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55928 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 16:57:27 INFO SparkContext:59 - Created broadcast 11 from textFile at modelSaveLoad.scala:129 -2016-05-25 16:57:27 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 16:57:27 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 16:57:27 INFO DAGScheduler:59 - Got job 9 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 16:57:27 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at modelSaveLoad.scala:129) -2016-05-25 16:57:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:57:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:57:27 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 16:57:27 INFO MemoryStore:59 - ensureFreeSpace(3224) called with curMem=523320, maxMem=766075207 -2016-05-25 16:57:27 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-25 16:57:27 INFO MemoryStore:59 - ensureFreeSpace(1901) called with curMem=526544, maxMem=766075207 -2016-05-25 16:57:27 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 1901.0 B, free 730.1 MB) -2016-05-25 16:57:27 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:55928 (size: 1901.0 B, free: 730.5 MB) -2016-05-25 16:57:27 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:57:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at textFile at modelSaveLoad.scala:129) -2016-05-25 16:57:27 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-25 16:57:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2247 bytes) -2016-05-25 16:57:27 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 20) -2016-05-25 16:57:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165724203/result/metadata/part-00000:0+51 -2016-05-25 16:57:27 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 20). 2148 bytes result sent to driver -2016-05-25 16:57:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 20) in 13 ms on localhost (1/1) -2016-05-25 16:57:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 16:57:27 INFO DAGScheduler:59 - ResultStage 10 (first at modelSaveLoad.scala:129) finished in 0.013 s -2016-05-25 16:57:27 INFO DAGScheduler:59 - Job 9 finished: first at modelSaveLoad.scala:129, took 0.033175 s -2016-05-25 16:57:27 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165724203/result/data on driver -2016-05-25 16:57:27 INFO SparkContext:59 - Starting job: parquet at IsotonicRegression.scala:205 -2016-05-25 16:57:27 INFO DAGScheduler:59 - Got job 10 (parquet at IsotonicRegression.scala:205) with 4 output partitions -2016-05-25 16:57:27 INFO DAGScheduler:59 - Final stage: ResultStage 11(parquet at IsotonicRegression.scala:205) -2016-05-25 16:57:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:57:27 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:57:27 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[22] at parquet at IsotonicRegression.scala:205), which has no missing parents -2016-05-25 16:57:27 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=528445, maxMem=766075207 -2016-05-25 16:57:27 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-25 16:57:27 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=591445, maxMem=766075207 -2016-05-25 16:57:27 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.0 MB) -2016-05-25 16:57:27 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:55928 (size: 20.6 KB, free: 730.5 MB) -2016-05-25 16:57:27 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:57:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 11 (MapPartitionsRDD[22] at parquet at IsotonicRegression.scala:205) -2016-05-25 16:57:27 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 4 tasks -2016-05-25 16:57:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 16:57:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 16:57:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 11.0 (TID 23, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 16:57:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 11.0 (TID 24, localhost, PROCESS_LOCAL, 2288 bytes) -2016-05-25 16:57:27 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-25 16:57:27 INFO Executor:59 - Running task 3.0 in stage 11.0 (TID 24) -2016-05-25 16:57:27 INFO Executor:59 - Running task 2.0 in stage 11.0 (TID 23) -2016-05-25 16:57:27 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-25 16:57:27 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 16:57:27 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 936 bytes result sent to driver -2016-05-25 16:57:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 39 ms on localhost (1/4) -2016-05-25 16:57:27 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 16:57:27 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 16:57:27 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 936 bytes result sent to driver -2016-05-25 16:57:27 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 16:57:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 95 ms on localhost (2/4) -2016-05-25 16:57:27 INFO Executor:59 - Finished task 2.0 in stage 11.0 (TID 23). 936 bytes result sent to driver -2016-05-25 16:57:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 11.0 (TID 23) in 97 ms on localhost (3/4) -2016-05-25 16:57:27 INFO Executor:59 - Finished task 3.0 in stage 11.0 (TID 24). 1661 bytes result sent to driver -2016-05-25 16:57:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 11.0 (TID 24) in 117 ms on localhost (4/4) -2016-05-25 16:57:27 INFO DAGScheduler:59 - ResultStage 11 (parquet at IsotonicRegression.scala:205) finished in 0.120 s -2016-05-25 16:57:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 16:57:27 INFO DAGScheduler:59 - Job 10 finished: parquet at IsotonicRegression.scala:205, took 0.148628 s -2016-05-25 16:57:28 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=612518, maxMem=766075207 -2016-05-25 16:57:28 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 86.4 KB, free 729.9 MB) -2016-05-25 16:57:28 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=701030, maxMem=766075207 -2016-05-25 16:57:28 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-25 16:57:28 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:55928 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 16:57:28 INFO SparkContext:59 - Created broadcast 14 from collect at IsotonicRegression.scala:208 -2016-05-25 16:57:28 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 16:57:28 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165724203/result/data/part-r-00002-aea4a9ec-68ad-4e32-9abf-54badfb2ff50.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165724203/result/data/part-r-00003-aea4a9ec-68ad-4e32-9abf-54badfb2ff50.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165724203/result/data/part-r-00001-aea4a9ec-68ad-4e32-9abf-54badfb2ff50.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165724203/result/data/part-r-00000-aea4a9ec-68ad-4e32-9abf-54badfb2ff50.gz.parquet -2016-05-25 16:57:28 INFO SparkContext:59 - Starting job: collect at IsotonicRegression.scala:208 -2016-05-25 16:57:28 INFO DAGScheduler:59 - Got job 11 (collect at IsotonicRegression.scala:208) with 4 output partitions -2016-05-25 16:57:28 INFO DAGScheduler:59 - Final stage: ResultStage 12(collect at IsotonicRegression.scala:208) -2016-05-25 16:57:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:57:28 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:57:28 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[24] at collect at IsotonicRegression.scala:208), which has no missing parents -2016-05-25 16:57:28 INFO MemoryStore:59 - ensureFreeSpace(5056) called with curMem=720818, maxMem=766075207 -2016-05-25 16:57:28 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 4.9 KB, free 729.9 MB) -2016-05-25 16:57:28 INFO MemoryStore:59 - ensureFreeSpace(2936) called with curMem=725874, maxMem=766075207 -2016-05-25 16:57:28 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.9 KB, free 729.9 MB) -2016-05-25 16:57:28 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:55928 (size: 2.9 KB, free: 730.5 MB) -2016-05-25 16:57:28 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:57:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 12 (MapPartitionsRDD[24] at collect at IsotonicRegression.scala:208) -2016-05-25 16:57:28 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 4 tasks -2016-05-25 16:57:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 25, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-25 16:57:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 26, localhost, PROCESS_LOCAL, 2308 bytes) -2016-05-25 16:57:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 12.0 (TID 27, localhost, PROCESS_LOCAL, 2308 bytes) -2016-05-25 16:57:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 12.0 (TID 28, localhost, PROCESS_LOCAL, 2307 bytes) -2016-05-25 16:57:28 INFO Executor:59 - Running task 2.0 in stage 12.0 (TID 27) -2016-05-25 16:57:28 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 26) -2016-05-25 16:57:28 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 25) -2016-05-25 16:57:28 INFO Executor:59 - Running task 3.0 in stage 12.0 (TID 28) -2016-05-25 16:57:28 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165724203/result/data/part-r-00002-aea4a9ec-68ad-4e32-9abf-54badfb2ff50.gz.parquet start: 0 end: 658 length: 658 hosts: []} -2016-05-25 16:57:28 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165724203/result/data/part-r-00001-aea4a9ec-68ad-4e32-9abf-54badfb2ff50.gz.parquet start: 0 end: 674 length: 674 hosts: []} -2016-05-25 16:57:28 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165724203/result/data/part-r-00003-aea4a9ec-68ad-4e32-9abf-54badfb2ff50.gz.parquet start: 0 end: 669 length: 669 hosts: []} -2016-05-25 16:57:28 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165724203/result/data/part-r-00000-aea4a9ec-68ad-4e32-9abf-54badfb2ff50.gz.parquet start: 0 end: 661 length: 661 hosts: []} -2016-05-25 16:57:28 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 16:57:28 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 16:57:28 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 16:57:28 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 16:57:28 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional double boundary; - optional double prediction; -} - -Catalyst form: -StructType(StructField(boundary,DoubleType,true), StructField(prediction,DoubleType,true)) - -2016-05-25 16:57:28 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional double boundary; - optional double prediction; -} - -Catalyst form: -StructType(StructField(boundary,DoubleType,true), StructField(prediction,DoubleType,true)) - -2016-05-25 16:57:28 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional double boundary; - optional double prediction; -} - -Catalyst form: -StructType(StructField(boundary,DoubleType,true), StructField(prediction,DoubleType,true)) - -2016-05-25 16:57:28 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional double boundary; - optional double prediction; -} - -Catalyst form: -StructType(StructField(boundary,DoubleType,true), StructField(prediction,DoubleType,true)) - -2016-05-25 16:57:28 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 6 records. -2016-05-25 16:57:28 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 6 records. -2016-05-25 16:57:28 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 6 records. -2016-05-25 16:57:28 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 5 records. -2016-05-25 16:57:28 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 16:57:28 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 16:57:28 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 16:57:28 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 16:57:28 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 16:57:28 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 16:57:28 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 16:57:28 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 16:57:28 INFO InternalParquetRecordReader:151 - block read in memory in 12 ms. row count = 6 -2016-05-25 16:57:28 INFO InternalParquetRecordReader:151 - block read in memory in 13 ms. row count = 6 -2016-05-25 16:57:28 INFO InternalParquetRecordReader:151 - block read in memory in 12 ms. row count = 5 -2016-05-25 16:57:28 INFO InternalParquetRecordReader:151 - block read in memory in 12 ms. row count = 6 -2016-05-25 16:57:28 INFO Executor:59 - Finished task 2.0 in stage 12.0 (TID 27). 3389 bytes result sent to driver -2016-05-25 16:57:28 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 25). 3389 bytes result sent to driver -2016-05-25 16:57:28 INFO Executor:59 - Finished task 3.0 in stage 12.0 (TID 28). 3340 bytes result sent to driver -2016-05-25 16:57:28 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 26). 3389 bytes result sent to driver -2016-05-25 16:57:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 12.0 (TID 27) in 118 ms on localhost (1/4) -2016-05-25 16:57:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 25) in 123 ms on localhost (2/4) -2016-05-25 16:57:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 26) in 121 ms on localhost (3/4) -2016-05-25 16:57:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 12.0 (TID 28) in 118 ms on localhost (4/4) -2016-05-25 16:57:28 INFO DAGScheduler:59 - ResultStage 12 (collect at IsotonicRegression.scala:208) finished in 0.127 s -2016-05-25 16:57:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 16:57:28 INFO DAGScheduler:59 - Job 11 finished: collect at IsotonicRegression.scala:208, took 0.144149 s -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 16:57:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 16:57:28 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 16:57:28 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 16:57:28 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 16:57:28 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 16:57:28 INFO BlockManager:59 - BlockManager stopped -2016-05-25 16:57:28 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 16:57:28 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 16:57:28 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 16:57:28 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 16:57:28 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-02daeafd-0924-4f4e-aca1-5c35565d039a -2016-05-25 16:57:53 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-05-25 16:57:54 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-05-25 16:57:54 INFO SecurityManager:59 - Changing view acls to: xubo -2016-05-25 16:57:54 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-05-25 16:57:54 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-05-25 16:57:55 INFO Slf4jLogger:80 - Slf4jLogger started -2016-05-25 16:57:55 INFO Remoting:74 - Starting remoting -2016-05-25 16:57:55 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.130:55949] -2016-05-25 16:57:55 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55949. -2016-05-25 16:57:55 INFO SparkEnv:59 - Registering MapOutputTracker -2016-05-25 16:57:55 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-05-25 16:57:56 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-dc2bb404-da8e-415f-97c2-5373d63aced9 -2016-05-25 16:57:56 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-05-25 16:57:56 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-2ea675cb-7d1f-4ef1-ad09-8b6515ed4b56\httpd-d963baef-cbb8-4c0d-8ab5-8b0d6d6bbe28 -2016-05-25 16:57:56 INFO HttpServer:59 - Starting HTTP Server -2016-05-25 16:57:56 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 16:57:56 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55950 -2016-05-25 16:57:56 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55950. -2016-05-25 16:57:56 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-05-25 16:57:56 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-05-25 16:57:56 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-05-25 16:57:56 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-05-25 16:57:56 INFO SparkUI:59 - Started SparkUI at http://211.86.159.130:4040 -2016-05-25 16:57:56 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-05-25 16:57:56 INFO Executor:59 - Starting executor ID driver on host localhost -2016-05-25 16:57:56 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55969. -2016-05-25 16:57:56 INFO NettyBlockTransferService:59 - Server created on 55969 -2016-05-25 16:57:56 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-05-25 16:57:56 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55969 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55969) -2016-05-25 16:57:56 INFO BlockManagerMaster:59 - Registered BlockManager -2016-05-25 16:57:57 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-05-25 16:57:57 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-05-25 16:57:57 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-05-25 16:57:57 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-05-25 16:57:57 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55969 (size: 13.9 KB, free: 730.6 MB) -2016-05-25 16:57:57 INFO SparkContext:59 - Created broadcast 0 from textFile at IsotonicRegression1.scala:33 -2016-05-25 16:57:58 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:d356:9f82%20, but we couldn't find any external IP address! -2016-05-25 16:57:59 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 16:57:59 INFO SparkContext:59 - Starting job: sortBy at IsotonicRegression.scala:418 -2016-05-25 16:57:59 INFO DAGScheduler:59 - Got job 0 (sortBy at IsotonicRegression.scala:418) with 2 output partitions -2016-05-25 16:57:59 INFO DAGScheduler:59 - Final stage: ResultStage 0(sortBy at IsotonicRegression.scala:418) -2016-05-25 16:57:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:57:59 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:57:59 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[7] at sortBy at IsotonicRegression.scala:418), which has no missing parents -2016-05-25 16:57:59 INFO MemoryStore:59 - ensureFreeSpace(4256) called with curMem=171580, maxMem=766075207 -2016-05-25 16:57:59 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.2 KB, free 730.4 MB) -2016-05-25 16:57:59 INFO MemoryStore:59 - ensureFreeSpace(2276) called with curMem=175836, maxMem=766075207 -2016-05-25 16:57:59 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-05-25 16:57:59 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55969 (size: 2.2 KB, free: 730.6 MB) -2016-05-25 16:57:59 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:57:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[7] at sortBy at IsotonicRegression.scala:418) -2016-05-25 16:57:59 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-05-25 16:57:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:57:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:57:59 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-05-25 16:57:59 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-05-25 16:57:59 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:0+799 -2016-05-25 16:57:59 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:799+799 -2016-05-25 16:57:59 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-05-25 16:57:59 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-05-25 16:57:59 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-05-25 16:57:59 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-05-25 16:57:59 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-05-25 16:57:59 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 3161 bytes result sent to driver -2016-05-25 16:57:59 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 3065 bytes result sent to driver -2016-05-25 16:57:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 139 ms on localhost (1/2) -2016-05-25 16:57:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 124 ms on localhost (2/2) -2016-05-25 16:57:59 INFO DAGScheduler:59 - ResultStage 0 (sortBy at IsotonicRegression.scala:418) finished in 0.161 s -2016-05-25 16:57:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-05-25 16:57:59 INFO DAGScheduler:59 - Job 0 finished: sortBy at IsotonicRegression.scala:418, took 0.242014 s -2016-05-25 16:57:59 INFO SparkContext:59 - Starting job: collect at IsotonicRegression.scala:421 -2016-05-25 16:57:59 INFO DAGScheduler:59 - Registering RDD 5 (sortBy at IsotonicRegression.scala:418) -2016-05-25 16:57:59 INFO DAGScheduler:59 - Got job 1 (collect at IsotonicRegression.scala:421) with 2 output partitions -2016-05-25 16:57:59 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at IsotonicRegression.scala:421) -2016-05-25 16:57:59 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1) -2016-05-25 16:57:59 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1) -2016-05-25 16:57:59 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1 (MapPartitionsRDD[5] at sortBy at IsotonicRegression.scala:418), which has no missing parents -2016-05-25 16:57:59 INFO MemoryStore:59 - ensureFreeSpace(4864) called with curMem=178112, maxMem=766075207 -2016-05-25 16:57:59 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-05-25 16:57:59 INFO MemoryStore:59 - ensureFreeSpace(2714) called with curMem=182976, maxMem=766075207 -2016-05-25 16:57:59 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-05-25 16:57:59 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55969 (size: 2.7 KB, free: 730.6 MB) -2016-05-25 16:57:59 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:57:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[5] at sortBy at IsotonicRegression.scala:418) -2016-05-25 16:57:59 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-05-25 16:57:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-25 16:57:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2202 bytes) -2016-05-25 16:57:59 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-05-25 16:57:59 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-05-25 16:57:59 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:799+799 -2016-05-25 16:57:59 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:0+799 -2016-05-25 16:57:59 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2254 bytes result sent to driver -2016-05-25 16:57:59 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2254 bytes result sent to driver -2016-05-25 16:57:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 67 ms on localhost (1/2) -2016-05-25 16:57:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 65 ms on localhost (2/2) -2016-05-25 16:57:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-05-25 16:57:59 INFO DAGScheduler:59 - ShuffleMapStage 1 (sortBy at IsotonicRegression.scala:418) finished in 0.069 s -2016-05-25 16:57:59 INFO DAGScheduler:59 - looking for newly runnable stages -2016-05-25 16:57:59 INFO DAGScheduler:59 - running: Set() -2016-05-25 16:57:59 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-05-25 16:57:59 INFO DAGScheduler:59 - failed: Set() -2016-05-25 16:57:59 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List() -2016-05-25 16:57:59 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[11] at flatMap at IsotonicRegression.scala:420), which is now runnable -2016-05-25 16:57:59 INFO MemoryStore:59 - ensureFreeSpace(3808) called with curMem=185690, maxMem=766075207 -2016-05-25 16:57:59 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.7 KB, free 730.4 MB) -2016-05-25 16:57:59 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=189498, maxMem=766075207 -2016-05-25 16:57:59 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-05-25 16:57:59 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55969 (size: 2.0 KB, free: 730.6 MB) -2016-05-25 16:57:59 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:57:59 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[11] at flatMap at IsotonicRegression.scala:420) -2016-05-25 16:57:59 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-05-25 16:57:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:57:59 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 1901 bytes) -2016-05-25 16:57:59 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-05-25 16:57:59 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-05-25 16:57:59 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:57:59 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-05-25 16:57:59 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 16:57:59 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-05-25 16:58:00 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:55969 in memory (size: 2.7 KB, free: 730.6 MB) -2016-05-25 16:58:00 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:55969 in memory (size: 2.2 KB, free: 730.6 MB) -2016-05-25 16:58:00 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1971 bytes result sent to driver -2016-05-25 16:58:00 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2115 bytes result sent to driver -2016-05-25 16:58:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 125 ms on localhost (1/2) -2016-05-25 16:58:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 123 ms on localhost (2/2) -2016-05-25 16:58:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-05-25 16:58:00 INFO DAGScheduler:59 - ResultStage 2 (collect at IsotonicRegression.scala:421) finished in 0.126 s -2016-05-25 16:58:00 INFO DAGScheduler:59 - Job 1 finished: collect at IsotonicRegression.scala:421, took 0.235588 s -2016-05-25 16:58:00 INFO SparkContext:59 - Starting job: mean at IsotonicRegression1.scala:57 -2016-05-25 16:58:00 INFO DAGScheduler:59 - Got job 2 (mean at IsotonicRegression1.scala:57) with 2 output partitions -2016-05-25 16:58:00 INFO DAGScheduler:59 - Final stage: ResultStage 3(mean at IsotonicRegression1.scala:57) -2016-05-25 16:58:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:58:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:58:00 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[14] at mean at IsotonicRegression1.scala:57), which has no missing parents -2016-05-25 16:58:00 INFO MemoryStore:59 - ensureFreeSpace(5120) called with curMem=177472, maxMem=766075207 -2016-05-25 16:58:00 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-05-25 16:58:00 INFO MemoryStore:59 - ensureFreeSpace(2865) called with curMem=182592, maxMem=766075207 -2016-05-25 16:58:00 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-05-25 16:58:00 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55969 (size: 2.8 KB, free: 730.6 MB) -2016-05-25 16:58:00 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:58:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[14] at mean at IsotonicRegression1.scala:57) -2016-05-25 16:58:00 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-05-25 16:58:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:58:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:58:00 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-05-25 16:58:00 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-05-25 16:58:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:0+799 -2016-05-25 16:58:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:799+799 -2016-05-25 16:58:00 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 2209 bytes result sent to driver -2016-05-25 16:58:00 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 2209 bytes result sent to driver -2016-05-25 16:58:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 26 ms on localhost (1/2) -2016-05-25 16:58:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 26 ms on localhost (2/2) -2016-05-25 16:58:00 INFO DAGScheduler:59 - ResultStage 3 (mean at IsotonicRegression1.scala:57) finished in 0.027 s -2016-05-25 16:58:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-05-25 16:58:00 INFO DAGScheduler:59 - Job 2 finished: mean at IsotonicRegression1.scala:57, took 0.045777 s -2016-05-25 16:58:00 INFO SparkContext:59 - Starting job: count at IsotonicRegression1.scala:59 -2016-05-25 16:58:00 INFO DAGScheduler:59 - Got job 3 (count at IsotonicRegression1.scala:59) with 2 output partitions -2016-05-25 16:58:00 INFO DAGScheduler:59 - Final stage: ResultStage 4(count at IsotonicRegression1.scala:59) -2016-05-25 16:58:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:58:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:58:00 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[1] at textFile at IsotonicRegression1.scala:33), which has no missing parents -2016-05-25 16:58:00 INFO MemoryStore:59 - ensureFreeSpace(3032) called with curMem=185457, maxMem=766075207 -2016-05-25 16:58:00 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.0 KB, free 730.4 MB) -2016-05-25 16:58:00 INFO MemoryStore:59 - ensureFreeSpace(1815) called with curMem=188489, maxMem=766075207 -2016-05-25 16:58:00 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 1815.0 B, free 730.4 MB) -2016-05-25 16:58:00 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55969 (size: 1815.0 B, free: 730.6 MB) -2016-05-25 16:58:00 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:58:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[1] at textFile at IsotonicRegression1.scala:33) -2016-05-25 16:58:00 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-05-25 16:58:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:58:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:58:00 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-05-25 16:58:00 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-05-25 16:58:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:799+799 -2016-05-25 16:58:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:0+799 -2016-05-25 16:58:00 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2082 bytes result sent to driver -2016-05-25 16:58:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 20 ms on localhost (1/2) -2016-05-25 16:58:00 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2082 bytes result sent to driver -2016-05-25 16:58:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 24 ms on localhost (2/2) -2016-05-25 16:58:00 INFO DAGScheduler:59 - ResultStage 4 (count at IsotonicRegression1.scala:59) finished in 0.024 s -2016-05-25 16:58:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-05-25 16:58:00 INFO DAGScheduler:59 - Job 3 finished: count at IsotonicRegression1.scala:59, took 0.037578 s -2016-05-25 16:58:00 INFO SparkContext:59 - Starting job: count at IsotonicRegression1.scala:60 -2016-05-25 16:58:00 INFO DAGScheduler:59 - Got job 4 (count at IsotonicRegression1.scala:60) with 2 output partitions -2016-05-25 16:58:00 INFO DAGScheduler:59 - Final stage: ResultStage 5(count at IsotonicRegression1.scala:60) -2016-05-25 16:58:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:58:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:58:00 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[3] at randomSplit at IsotonicRegression1.scala:42), which has no missing parents -2016-05-25 16:58:00 INFO MemoryStore:59 - ensureFreeSpace(3544) called with curMem=190304, maxMem=766075207 -2016-05-25 16:58:00 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 16:58:00 INFO MemoryStore:59 - ensureFreeSpace(2074) called with curMem=193848, maxMem=766075207 -2016-05-25 16:58:00 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-05-25 16:58:00 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55969 (size: 2.0 KB, free: 730.6 MB) -2016-05-25 16:58:00 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:58:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[3] at randomSplit at IsotonicRegression1.scala:42) -2016-05-25 16:58:00 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-05-25 16:58:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:58:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:58:00 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-05-25 16:58:00 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-05-25 16:58:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:0+799 -2016-05-25 16:58:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:799+799 -2016-05-25 16:58:00 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 2082 bytes result sent to driver -2016-05-25 16:58:00 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 2082 bytes result sent to driver -2016-05-25 16:58:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 24 ms on localhost (1/2) -2016-05-25 16:58:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 23 ms on localhost (2/2) -2016-05-25 16:58:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-05-25 16:58:00 INFO DAGScheduler:59 - ResultStage 5 (count at IsotonicRegression1.scala:60) finished in 0.026 s -2016-05-25 16:58:00 INFO DAGScheduler:59 - Job 4 finished: count at IsotonicRegression1.scala:60, took 0.043305 s -2016-05-25 16:58:00 INFO SparkContext:59 - Starting job: count at IsotonicRegression1.scala:61 -2016-05-25 16:58:00 INFO DAGScheduler:59 - Got job 5 (count at IsotonicRegression1.scala:61) with 2 output partitions -2016-05-25 16:58:00 INFO DAGScheduler:59 - Final stage: ResultStage 6(count at IsotonicRegression1.scala:61) -2016-05-25 16:58:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:58:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:58:00 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[4] at randomSplit at IsotonicRegression1.scala:42), which has no missing parents -2016-05-25 16:58:00 INFO MemoryStore:59 - ensureFreeSpace(3544) called with curMem=195922, maxMem=766075207 -2016-05-25 16:58:00 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-05-25 16:58:00 INFO MemoryStore:59 - ensureFreeSpace(2078) called with curMem=199466, maxMem=766075207 -2016-05-25 16:58:00 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-05-25 16:58:00 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55969 (size: 2.0 KB, free: 730.6 MB) -2016-05-25 16:58:00 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:58:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[4] at randomSplit at IsotonicRegression1.scala:42) -2016-05-25 16:58:00 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-05-25 16:58:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:58:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:58:00 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-05-25 16:58:00 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 13) -2016-05-25 16:58:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:0+799 -2016-05-25 16:58:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:799+799 -2016-05-25 16:58:00 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 2082 bytes result sent to driver -2016-05-25 16:58:00 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 13). 2082 bytes result sent to driver -2016-05-25 16:58:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 21 ms on localhost (1/2) -2016-05-25 16:58:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 13) in 22 ms on localhost (2/2) -2016-05-25 16:58:00 INFO DAGScheduler:59 - ResultStage 6 (count at IsotonicRegression1.scala:61) finished in 0.024 s -2016-05-25 16:58:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-05-25 16:58:00 INFO DAGScheduler:59 - Job 5 finished: count at IsotonicRegression1.scala:61, took 0.039559 s -2016-05-25 16:58:00 INFO SparkContext:59 - Starting job: take at IsotonicRegression1.scala:66 -2016-05-25 16:58:00 INFO DAGScheduler:59 - Got job 6 (take at IsotonicRegression1.scala:66) with 1 output partitions -2016-05-25 16:58:00 INFO DAGScheduler:59 - Final stage: ResultStage 7(take at IsotonicRegression1.scala:66) -2016-05-25 16:58:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:58:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:58:00 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[12] at map at IsotonicRegression1.scala:51), which has no missing parents -2016-05-25 16:58:00 INFO MemoryStore:59 - ensureFreeSpace(4552) called with curMem=201544, maxMem=766075207 -2016-05-25 16:58:00 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-05-25 16:58:00 INFO MemoryStore:59 - ensureFreeSpace(2634) called with curMem=206096, maxMem=766075207 -2016-05-25 16:58:00 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-05-25 16:58:00 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55969 (size: 2.6 KB, free: 730.6 MB) -2016-05-25 16:58:00 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:58:00 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (MapPartitionsRDD[12] at map at IsotonicRegression1.scala:51) -2016-05-25 16:58:00 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-05-25 16:58:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2213 bytes) -2016-05-25 16:58:00 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 14) -2016-05-25 16:58:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/classification/sample_isotonic_regression_data.txt:0+799 -2016-05-25 16:58:00 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 14). 2401 bytes result sent to driver -2016-05-25 16:58:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 14) in 20 ms on localhost (1/1) -2016-05-25 16:58:00 INFO DAGScheduler:59 - ResultStage 7 (take at IsotonicRegression1.scala:66) finished in 0.020 s -2016-05-25 16:58:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-05-25 16:58:00 INFO DAGScheduler:59 - Job 6 finished: take at IsotonicRegression1.scala:66, took 0.036845 s -2016-05-25 16:58:00 INFO SparkContext:59 - Starting job: saveAsTextFile at IsotonicRegression.scala:196 -2016-05-25 16:58:00 INFO DAGScheduler:59 - Got job 7 (saveAsTextFile at IsotonicRegression.scala:196) with 1 output partitions -2016-05-25 16:58:00 INFO DAGScheduler:59 - Final stage: ResultStage 8(saveAsTextFile at IsotonicRegression.scala:196) -2016-05-25 16:58:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:58:00 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:58:00 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[16] at saveAsTextFile at IsotonicRegression.scala:196), which has no missing parents -2016-05-25 16:58:00 INFO MemoryStore:59 - ensureFreeSpace(127008) called with curMem=208730, maxMem=766075207 -2016-05-25 16:58:00 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 124.0 KB, free 730.3 MB) -2016-05-25 16:58:00 INFO MemoryStore:59 - ensureFreeSpace(42645) called with curMem=335738, maxMem=766075207 -2016-05-25 16:58:00 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.2 MB) -2016-05-25 16:58:00 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:55969 (size: 41.6 KB, free: 730.5 MB) -2016-05-25 16:58:00 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:58:00 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[16] at saveAsTextFile at IsotonicRegression.scala:196) -2016-05-25 16:58:00 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-05-25 16:58:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2191 bytes) -2016-05-25 16:58:00 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-05-25 16:58:01 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251658_0008_m_000000_15' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165800365/result/metadata/_temporary/0/task_201605251658_0008_m_000000 -2016-05-25 16:58:01 INFO SparkHadoopMapRedUtil:59 - attempt_201605251658_0008_m_000000_15: Committed -2016-05-25 16:58:01 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 1864 bytes result sent to driver -2016-05-25 16:58:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 191 ms on localhost (1/1) -2016-05-25 16:58:01 INFO DAGScheduler:59 - ResultStage 8 (saveAsTextFile at IsotonicRegression.scala:196) finished in 0.191 s -2016-05-25 16:58:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-05-25 16:58:01 INFO DAGScheduler:59 - Job 7 finished: saveAsTextFile at IsotonicRegression.scala:196, took 0.274006 s -2016-05-25 16:58:01 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:55969 in memory (size: 1815.0 B, free: 730.5 MB) -2016-05-25 16:58:01 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:55969 in memory (size: 41.6 KB, free: 730.6 MB) -2016-05-25 16:58:01 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-05-25 16:58:01 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:55969 in memory (size: 2.6 KB, free: 730.6 MB) -2016-05-25 16:58:01 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-05-25 16:58:01 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:55969 in memory (size: 2.0 KB, free: 730.6 MB) -2016-05-25 16:58:01 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-05-25 16:58:01 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:55969 in memory (size: 2.0 KB, free: 730.6 MB) -2016-05-25 16:58:01 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-05-25 16:58:01 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-05-25 16:58:01 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:55969 in memory (size: 2.8 KB, free: 730.6 MB) -2016-05-25 16:58:01 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-05-25 16:58:01 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:55969 in memory (size: 2.0 KB, free: 730.6 MB) -2016-05-25 16:58:02 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:58:02 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:58:02 INFO SparkContext:59 - Starting job: parquet at IsotonicRegression.scala:200 -2016-05-25 16:58:02 INFO DAGScheduler:59 - Got job 8 (parquet at IsotonicRegression.scala:200) with 4 output partitions -2016-05-25 16:58:02 INFO DAGScheduler:59 - Final stage: ResultStage 9(parquet at IsotonicRegression.scala:200) -2016-05-25 16:58:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:58:02 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:58:02 INFO DAGScheduler:59 - Submitting ResultStage 9 (ParallelCollectionRDD[17] at parquet at IsotonicRegression.scala:200), which has no missing parents -2016-05-25 16:58:02 INFO MemoryStore:59 - ensureFreeSpace(65456) called with curMem=171580, maxMem=766075207 -2016-05-25 16:58:02 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 63.9 KB, free 730.4 MB) -2016-05-25 16:58:02 INFO MemoryStore:59 - ensureFreeSpace(22312) called with curMem=237036, maxMem=766075207 -2016-05-25 16:58:02 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 21.8 KB, free 730.3 MB) -2016-05-25 16:58:02 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:55969 (size: 21.8 KB, free: 730.6 MB) -2016-05-25 16:58:02 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:58:02 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 9 (ParallelCollectionRDD[17] at parquet at IsotonicRegression.scala:200) -2016-05-25 16:58:02 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 4 tasks -2016-05-25 16:58:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 16, localhost, PROCESS_LOCAL, 2542 bytes) -2016-05-25 16:58:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2586 bytes) -2016-05-25 16:58:02 INFO TaskSetManager:59 - Starting task 2.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2586 bytes) -2016-05-25 16:58:02 INFO TaskSetManager:59 - Starting task 3.0 in stage 9.0 (TID 19, localhost, PROCESS_LOCAL, 2586 bytes) -2016-05-25 16:58:02 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 16) -2016-05-25 16:58:02 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 17) -2016-05-25 16:58:02 INFO Executor:59 - Running task 2.0 in stage 9.0 (TID 18) -2016-05-25 16:58:02 INFO Executor:59 - Running task 3.0 in stage 9.0 (TID 19) -2016-05-25 16:58:02 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:58:02 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 16:58:02 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:58:02 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 16:58:02 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:58:02 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 16:58:02 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-05-25 16:58:02 INFO CodecConfig:151 - Compression: GZIP -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Dictionary is on -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Validation is off -2016-05-25 16:58:02 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-05-25 16:58:02 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 16:58:02 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 16:58:02 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 16:58:02 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-05-25 16:58:02 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 120 -2016-05-25 16:58:02 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 104 -2016-05-25 16:58:02 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 144 -2016-05-25 16:58:02 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 128 -2016-05-25 16:58:02 INFO ColumnChunkPageWriteStore:151 - written 110B for [boundary] DOUBLE: 6 values, 54B raw, 68B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:58:02 INFO ColumnChunkPageWriteStore:151 - written 119B for [prediction] DOUBLE: 6 values, 54B raw, 77B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:58:02 INFO ColumnChunkPageWriteStore:151 - written 111B for [boundary] DOUBLE: 6 values, 54B raw, 69B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:58:02 INFO ColumnChunkPageWriteStore:151 - written 106B for [boundary] DOUBLE: 6 values, 54B raw, 64B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:58:02 INFO ColumnChunkPageWriteStore:151 - written 103B for [boundary] DOUBLE: 5 values, 46B raw, 62B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-05-25 16:58:03 INFO ColumnChunkPageWriteStore:151 - written 71B for [prediction] DOUBLE: 6 values, 10B raw, 30B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN_DICTIONARY], dic { 3 entries, 24B raw, 3B comp} -2016-05-25 16:58:03 INFO ColumnChunkPageWriteStore:151 - written 71B for [prediction] DOUBLE: 6 values, 10B raw, 30B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN_DICTIONARY], dic { 4 entries, 32B raw, 4B comp} -2016-05-25 16:58:03 INFO ColumnChunkPageWriteStore:151 - written 71B for [prediction] DOUBLE: 5 values, 10B raw, 30B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN_DICTIONARY], dic { 3 entries, 24B raw, 3B comp} -2016-05-25 16:58:03 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251658_0009_m_000002_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165800365/result/data/_temporary/0/task_201605251658_0009_m_000002 -2016-05-25 16:58:03 INFO SparkHadoopMapRedUtil:59 - attempt_201605251658_0009_m_000002_0: Committed -2016-05-25 16:58:03 INFO Executor:59 - Finished task 2.0 in stage 9.0 (TID 18). 915 bytes result sent to driver -2016-05-25 16:58:03 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251658_0009_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165800365/result/data/_temporary/0/task_201605251658_0009_m_000000 -2016-05-25 16:58:03 INFO TaskSetManager:59 - Finished task 2.0 in stage 9.0 (TID 18) in 621 ms on localhost (1/4) -2016-05-25 16:58:03 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251658_0009_m_000001_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165800365/result/data/_temporary/0/task_201605251658_0009_m_000001 -2016-05-25 16:58:03 INFO SparkHadoopMapRedUtil:59 - attempt_201605251658_0009_m_000001_0: Committed -2016-05-25 16:58:03 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 17). 915 bytes result sent to driver -2016-05-25 16:58:03 INFO SparkHadoopMapRedUtil:59 - attempt_201605251658_0009_m_000000_0: Committed -2016-05-25 16:58:03 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 16). 915 bytes result sent to driver -2016-05-25 16:58:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 17) in 626 ms on localhost (2/4) -2016-05-25 16:58:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 16) in 631 ms on localhost (3/4) -2016-05-25 16:58:03 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201605251658_0009_m_000003_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165800365/result/data/_temporary/0/task_201605251658_0009_m_000003 -2016-05-25 16:58:03 INFO SparkHadoopMapRedUtil:59 - attempt_201605251658_0009_m_000003_0: Committed -2016-05-25 16:58:03 INFO Executor:59 - Finished task 3.0 in stage 9.0 (TID 19). 915 bytes result sent to driver -2016-05-25 16:58:03 INFO TaskSetManager:59 - Finished task 3.0 in stage 9.0 (TID 19) in 632 ms on localhost (4/4) -2016-05-25 16:58:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-05-25 16:58:03 INFO DAGScheduler:59 - ResultStage 9 (parquet at IsotonicRegression.scala:200) finished in 0.638 s -2016-05-25 16:58:03 INFO DAGScheduler:59 - Job 8 finished: parquet at IsotonicRegression.scala:200, took 0.678989 s -2016-05-25 16:58:03 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 16:58:03 INFO DefaultWriterContainer:59 - Job job_201605251658_0000 committed. -2016-05-25 16:58:03 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165800365/result/data on driver -2016-05-25 16:58:03 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165800365/result/data on driver -2016-05-25 16:58:03 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=259348, maxMem=766075207 -2016-05-25 16:58:03 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-05-25 16:58:03 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=490052, maxMem=766075207 -2016-05-25 16:58:03 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-05-25 16:58:03 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:55969 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 16:58:03 INFO SparkContext:59 - Created broadcast 11 from textFile at modelSaveLoad.scala:129 -2016-05-25 16:58:03 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-05-25 16:58:03 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-05-25 16:58:03 INFO DAGScheduler:59 - Got job 9 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-05-25 16:58:03 INFO DAGScheduler:59 - Final stage: ResultStage 10(first at modelSaveLoad.scala:129) -2016-05-25 16:58:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:58:03 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:58:03 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[20] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-05-25 16:58:03 INFO MemoryStore:59 - ensureFreeSpace(3224) called with curMem=509840, maxMem=766075207 -2016-05-25 16:58:03 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 3.1 KB, free 730.1 MB) -2016-05-25 16:58:03 INFO MemoryStore:59 - ensureFreeSpace(1901) called with curMem=513064, maxMem=766075207 -2016-05-25 16:58:03 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 1901.0 B, free 730.1 MB) -2016-05-25 16:58:03 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:55969 (size: 1901.0 B, free: 730.5 MB) -2016-05-25 16:58:03 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:58:03 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 10 (MapPartitionsRDD[20] at textFile at modelSaveLoad.scala:129) -2016-05-25 16:58:03 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 1 tasks -2016-05-25 16:58:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2247 bytes) -2016-05-25 16:58:03 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 20) -2016-05-25 16:58:03 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165800365/result/metadata/part-00000:0+51 -2016-05-25 16:58:03 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 20). 2148 bytes result sent to driver -2016-05-25 16:58:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 20) in 16 ms on localhost (1/1) -2016-05-25 16:58:03 INFO DAGScheduler:59 - ResultStage 10 (first at modelSaveLoad.scala:129) finished in 0.016 s -2016-05-25 16:58:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-05-25 16:58:03 INFO DAGScheduler:59 - Job 9 finished: first at modelSaveLoad.scala:129, took 0.030880 s -2016-05-25 16:58:03 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165800365/result/data on driver -2016-05-25 16:58:03 INFO SparkContext:59 - Starting job: parquet at IsotonicRegression.scala:205 -2016-05-25 16:58:03 INFO DAGScheduler:59 - Got job 10 (parquet at IsotonicRegression.scala:205) with 4 output partitions -2016-05-25 16:58:03 INFO DAGScheduler:59 - Final stage: ResultStage 11(parquet at IsotonicRegression.scala:205) -2016-05-25 16:58:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:58:03 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:58:03 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[22] at parquet at IsotonicRegression.scala:205), which has no missing parents -2016-05-25 16:58:03 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=514965, maxMem=766075207 -2016-05-25 16:58:03 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 61.5 KB, free 730.0 MB) -2016-05-25 16:58:03 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=577965, maxMem=766075207 -2016-05-25 16:58:03 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.0 MB) -2016-05-25 16:58:03 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:55969 (size: 20.6 KB, free: 730.5 MB) -2016-05-25 16:58:03 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:58:03 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 11 (MapPartitionsRDD[22] at parquet at IsotonicRegression.scala:205) -2016-05-25 16:58:03 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 4 tasks -2016-05-25 16:58:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 16:58:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 16:58:03 INFO TaskSetManager:59 - Starting task 2.0 in stage 11.0 (TID 23, localhost, PROCESS_LOCAL, 2083 bytes) -2016-05-25 16:58:03 INFO TaskSetManager:59 - Starting task 3.0 in stage 11.0 (TID 24, localhost, PROCESS_LOCAL, 2288 bytes) -2016-05-25 16:58:03 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-05-25 16:58:03 INFO Executor:59 - Running task 3.0 in stage 11.0 (TID 24) -2016-05-25 16:58:03 INFO Executor:59 - Running task 2.0 in stage 11.0 (TID 23) -2016-05-25 16:58:03 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-05-25 16:58:03 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 16:58:03 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 16:58:03 INFO Executor:59 - Finished task 2.0 in stage 11.0 (TID 23). 936 bytes result sent to driver -2016-05-25 16:58:04 INFO Executor:59 - Finished task 3.0 in stage 11.0 (TID 24). 1661 bytes result sent to driver -2016-05-25 16:58:04 INFO TaskSetManager:59 - Finished task 2.0 in stage 11.0 (TID 23) in 91 ms on localhost (1/4) -2016-05-25 16:58:04 INFO TaskSetManager:59 - Finished task 3.0 in stage 11.0 (TID 24) in 93 ms on localhost (2/4) -2016-05-25 16:58:04 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 16:58:04 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 936 bytes result sent to driver -2016-05-25 16:58:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 107 ms on localhost (3/4) -2016-05-25 16:58:04 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-05-25 16:58:04 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 936 bytes result sent to driver -2016-05-25 16:58:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 121 ms on localhost (4/4) -2016-05-25 16:58:04 INFO DAGScheduler:59 - ResultStage 11 (parquet at IsotonicRegression.scala:205) finished in 0.122 s -2016-05-25 16:58:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-05-25 16:58:04 INFO DAGScheduler:59 - Job 10 finished: parquet at IsotonicRegression.scala:205, took 0.152306 s -2016-05-25 16:58:04 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=599038, maxMem=766075207 -2016-05-25 16:58:04 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 86.4 KB, free 729.9 MB) -2016-05-25 16:58:04 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=687550, maxMem=766075207 -2016-05-25 16:58:04 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-05-25 16:58:04 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:55969 (size: 19.3 KB, free: 730.5 MB) -2016-05-25 16:58:04 INFO SparkContext:59 - Created broadcast 14 from collect at IsotonicRegression.scala:208 -2016-05-25 16:58:04 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-05-25 16:58:04 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165800365/result/data/part-r-00002-5bd993a3-be34-4c4b-aca7-109981634908.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165800365/result/data/part-r-00001-5bd993a3-be34-4c4b-aca7-109981634908.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165800365/result/data/part-r-00003-5bd993a3-be34-4c4b-aca7-109981634908.gz.parquet, file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165800365/result/data/part-r-00000-5bd993a3-be34-4c4b-aca7-109981634908.gz.parquet -2016-05-25 16:58:04 INFO SparkContext:59 - Starting job: collect at IsotonicRegression.scala:208 -2016-05-25 16:58:04 INFO DAGScheduler:59 - Got job 11 (collect at IsotonicRegression.scala:208) with 4 output partitions -2016-05-25 16:58:04 INFO DAGScheduler:59 - Final stage: ResultStage 12(collect at IsotonicRegression.scala:208) -2016-05-25 16:58:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-05-25 16:58:04 INFO DAGScheduler:59 - Missing parents: List() -2016-05-25 16:58:04 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[24] at collect at IsotonicRegression.scala:208), which has no missing parents -2016-05-25 16:58:04 INFO MemoryStore:59 - ensureFreeSpace(5056) called with curMem=707338, maxMem=766075207 -2016-05-25 16:58:04 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 4.9 KB, free 729.9 MB) -2016-05-25 16:58:04 INFO MemoryStore:59 - ensureFreeSpace(2936) called with curMem=712394, maxMem=766075207 -2016-05-25 16:58:04 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.9 KB, free 729.9 MB) -2016-05-25 16:58:04 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:55969 (size: 2.9 KB, free: 730.5 MB) -2016-05-25 16:58:04 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-05-25 16:58:04 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 12 (MapPartitionsRDD[24] at collect at IsotonicRegression.scala:208) -2016-05-25 16:58:04 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 4 tasks -2016-05-25 16:58:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 25, localhost, PROCESS_LOCAL, 2308 bytes) -2016-05-25 16:58:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 26, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 16:58:04 INFO TaskSetManager:59 - Starting task 2.0 in stage 12.0 (TID 27, localhost, PROCESS_LOCAL, 2309 bytes) -2016-05-25 16:58:04 INFO TaskSetManager:59 - Starting task 3.0 in stage 12.0 (TID 28, localhost, PROCESS_LOCAL, 2308 bytes) -2016-05-25 16:58:04 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 25) -2016-05-25 16:58:04 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 26) -2016-05-25 16:58:04 INFO Executor:59 - Running task 2.0 in stage 12.0 (TID 27) -2016-05-25 16:58:04 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165800365/result/data/part-r-00002-5bd993a3-be34-4c4b-aca7-109981634908.gz.parquet start: 0 end: 658 length: 658 hosts: []} -2016-05-25 16:58:04 INFO Executor:59 - Running task 3.0 in stage 12.0 (TID 28) -2016-05-25 16:58:04 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165800365/result/data/part-r-00000-5bd993a3-be34-4c4b-aca7-109981634908.gz.parquet start: 0 end: 661 length: 661 hosts: []} -2016-05-25 16:58:04 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 16:58:04 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165800365/result/data/part-r-00001-5bd993a3-be34-4c4b-aca7-109981634908.gz.parquet start: 0 end: 674 length: 674 hosts: []} -2016-05-25 16:58:04 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/classification/IsotonicRegressionModel20160525165800365/result/data/part-r-00003-5bd993a3-be34-4c4b-aca7-109981634908.gz.parquet start: 0 end: 669 length: 669 hosts: []} -2016-05-25 16:58:04 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 16:58:04 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 16:58:04 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-05-25 16:58:04 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional double boundary; - optional double prediction; -} - -Catalyst form: -StructType(StructField(boundary,DoubleType,true), StructField(prediction,DoubleType,true)) - -2016-05-25 16:58:04 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional double boundary; - optional double prediction; -} - -Catalyst form: -StructType(StructField(boundary,DoubleType,true), StructField(prediction,DoubleType,true)) - -2016-05-25 16:58:04 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional double boundary; - optional double prediction; -} - -Catalyst form: -StructType(StructField(boundary,DoubleType,true), StructField(prediction,DoubleType,true)) - -2016-05-25 16:58:04 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional double boundary; - optional double prediction; -} - -Catalyst form: -StructType(StructField(boundary,DoubleType,true), StructField(prediction,DoubleType,true)) - -2016-05-25 16:58:04 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 6 records. -2016-05-25 16:58:04 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 6 records. -2016-05-25 16:58:04 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 6 records. -2016-05-25 16:58:04 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 5 records. -2016-05-25 16:58:04 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 16:58:04 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 16:58:04 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 16:58:04 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-05-25 16:58:04 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 16:58:04 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 16:58:04 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 16:58:04 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-05-25 16:58:04 INFO InternalParquetRecordReader:151 - block read in memory in 11 ms. row count = 6 -2016-05-25 16:58:04 INFO InternalParquetRecordReader:151 - block read in memory in 10 ms. row count = 5 -2016-05-25 16:58:04 INFO InternalParquetRecordReader:151 - block read in memory in 11 ms. row count = 6 -2016-05-25 16:58:04 INFO InternalParquetRecordReader:151 - block read in memory in 10 ms. row count = 6 -2016-05-25 16:58:04 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 25). 3389 bytes result sent to driver -2016-05-25 16:58:04 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 26). 3389 bytes result sent to driver -2016-05-25 16:58:04 INFO Executor:59 - Finished task 2.0 in stage 12.0 (TID 27). 3389 bytes result sent to driver -2016-05-25 16:58:04 INFO Executor:59 - Finished task 3.0 in stage 12.0 (TID 28). 3340 bytes result sent to driver -2016-05-25 16:58:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 25) in 117 ms on localhost (1/4) -2016-05-25 16:58:04 INFO TaskSetManager:59 - Finished task 3.0 in stage 12.0 (TID 28) in 114 ms on localhost (2/4) -2016-05-25 16:58:04 INFO TaskSetManager:59 - Finished task 2.0 in stage 12.0 (TID 27) in 115 ms on localhost (3/4) -2016-05-25 16:58:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 26) in 118 ms on localhost (4/4) -2016-05-25 16:58:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-05-25 16:58:04 INFO DAGScheduler:59 - ResultStage 12 (collect at IsotonicRegression.scala:208) finished in 0.121 s -2016-05-25 16:58:04 INFO DAGScheduler:59 - Job 11 finished: collect at IsotonicRegression.scala:208, took 0.138177 s -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-05-25 16:58:04 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-05-25 16:58:04 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.130:4040 -2016-05-25 16:58:04 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-05-25 16:58:04 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-05-25 16:58:04 INFO MemoryStore:59 - MemoryStore cleared -2016-05-25 16:58:04 INFO BlockManager:59 - BlockManager stopped -2016-05-25 16:58:04 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-05-25 16:58:04 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-05-25 16:58:04 INFO SparkContext:59 - Successfully stopped SparkContext -2016-05-25 16:58:04 INFO ShutdownHookManager:59 - Shutdown hook called -2016-05-25 16:58:04 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-2ea675cb-7d1f-4ef1-ad09-8b6515ed4b56 -2016-05-25 16:58:04 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-05-25 16:58:04 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-03 13:29:22 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-03 13:29:23 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-03 13:29:23 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-03 13:29:23 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-03 13:29:23 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-03 13:29:24 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-03 13:29:24 INFO Remoting:74 - Starting remoting -2016-06-03 13:29:24 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@211.86.159.79:64546] -2016-06-03 13:29:24 INFO Utils:59 - Successfully started service 'sparkDriver' on port 64546. -2016-06-03 13:29:24 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-03 13:29:24 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-03 13:29:24 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-f687b0dc-ed0a-4afc-9e5e-ef87cee9da67 -2016-06-03 13:29:24 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-03 13:29:25 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-32d2733c-2d3a-4c3a-8f3e-9a69834e156a\httpd-2db3f193-e1d6-49d4-b67c-3efafee4f747 -2016-06-03 13:29:25 INFO HttpServer:59 - Starting HTTP Server -2016-06-03 13:29:25 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-03 13:29:25 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:64547 -2016-06-03 13:29:25 INFO Utils:59 - Successfully started service 'HTTP file server' on port 64547. -2016-06-03 13:29:25 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-03 13:29:25 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-03 13:29:25 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-03 13:29:25 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-03 13:29:25 INFO SparkUI:59 - Started SparkUI at http://211.86.159.79:4040 -2016-06-03 13:29:25 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-03 13:29:25 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-03 13:29:25 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 64554. -2016-06-03 13:29:25 INFO NettyBlockTransferService:59 - Server created on 64554 -2016-06-03 13:29:25 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-03 13:29:25 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:64554 with 730.6 MB RAM, BlockManagerId(driver, localhost, 64554) -2016-06-03 13:29:25 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-03 13:29:30 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-03 13:29:31 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-03 13:29:31 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-03 13:29:31 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-03 13:29:31 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-03 13:29:31 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-03 13:29:31 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-03 13:29:31 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-03 13:29:31 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-03 13:29:31 INFO DAGScheduler:59 - Got job 0 (main at null:-2) with 1 output partitions -2016-06-03 13:29:31 INFO DAGScheduler:59 - Final stage: ResultStage 0(main at null:-2) -2016-06-03 13:29:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-03 13:29:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-03 13:29:31 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at main at null:-2), which has no missing parents -2016-06-03 13:29:32 INFO MemoryStore:59 - ensureFreeSpace(68744) called with curMem=0, maxMem=766075207 -2016-06-03 13:29:32 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 67.1 KB, free 730.5 MB) -2016-06-03 13:29:32 INFO MemoryStore:59 - ensureFreeSpace(24474) called with curMem=68744, maxMem=766075207 -2016-06-03 13:29:32 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 23.9 KB, free 730.5 MB) -2016-06-03 13:29:32 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:64554 (size: 23.9 KB, free: 730.6 MB) -2016-06-03 13:29:32 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-03 13:29:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at main at null:-2) -2016-06-03 13:29:32 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-03 13:29:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2142 bytes) -2016-06-03 13:29:32 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-03 13:29:33 INFO GenerateUnsafeProjection:59 - Code generated in 600.136631 ms -2016-06-03 13:29:33 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-03 13:29:33 INFO CodecConfig:151 - Compression: GZIP -2016-06-03 13:29:33 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-06-03 13:29:33 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-06-03 13:29:33 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-06-03 13:29:33 INFO ParquetOutputFormat:151 - Dictionary is on -2016-06-03 13:29:33 INFO ParquetOutputFormat:151 - Validation is off -2016-06-03 13:29:33 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-06-03 13:29:33 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-06-03 13:29:33 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 80 -2016-06-03 13:29:33 INFO ColumnChunkPageWriteStore:151 - written 71B for [single] INT32: 5 values, 26B raw, 38B comp, 1 pages, encodings: [RLE, PLAIN, BIT_PACKED] -2016-06-03 13:29:33 INFO ColumnChunkPageWriteStore:151 - written 71B for [double] INT32: 5 values, 26B raw, 38B comp, 1 pages, encodings: [RLE, PLAIN, BIT_PACKED] -2016-06-03 13:29:33 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201606031329_0000_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/sql/output/20160603132926436/test_table/key=1/_temporary/0/task_201606031329_0000_m_000000 -2016-06-03 13:29:33 INFO SparkHadoopMapRedUtil:59 - attempt_201606031329_0000_m_000000_0: Committed -2016-06-03 13:29:34 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1060 bytes result sent to driver -2016-06-03 13:29:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 1517 ms on localhost (1/1) -2016-06-03 13:29:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-03 13:29:34 INFO DAGScheduler:59 - ResultStage 0 (main at null:-2) finished in 1.694 s -2016-06-03 13:29:34 INFO DAGScheduler:59 - Job 0 finished: main at null:-2, took 2.559511 s -2016-06-03 13:29:34 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-06-03 13:29:34 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:64554 in memory (size: 23.9 KB, free: 730.6 MB) -2016-06-03 13:29:34 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-03 13:29:34 INFO DefaultWriterContainer:59 - Job job_201606031329_0000 committed. -2016-06-03 13:29:34 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/sql/output/20160603132926436/test_table/key=1 on driver -2016-06-03 13:29:34 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/sql/output/20160603132926436/test_table/key=1 on driver -2016-06-03 13:29:34 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-03 13:29:34 INFO DAGScheduler:59 - Got job 1 (main at null:-2) with 1 output partitions -2016-06-03 13:29:34 INFO DAGScheduler:59 - Final stage: ResultStage 1(main at null:-2) -2016-06-03 13:29:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-03 13:29:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-03 13:29:34 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at main at null:-2), which has no missing parents -2016-06-03 13:29:34 INFO MemoryStore:59 - ensureFreeSpace(5128) called with curMem=0, maxMem=766075207 -2016-06-03 13:29:34 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 5.0 KB, free 730.6 MB) -2016-06-03 13:29:34 INFO MemoryStore:59 - ensureFreeSpace(2804) called with curMem=5128, maxMem=766075207 -2016-06-03 13:29:34 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.6 MB) -2016-06-03 13:29:34 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:64554 (size: 2.7 KB, free: 730.6 MB) -2016-06-03 13:29:34 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-03 13:29:34 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at main at null:-2) -2016-06-03 13:29:34 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-03 13:29:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2142 bytes) -2016-06-03 13:29:34 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-03 13:29:34 INFO GenerateSafeProjection:59 - Code generated in 11.458299 ms -2016-06-03 13:29:34 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 1513 bytes result sent to driver -2016-06-03 13:29:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 72 ms on localhost (1/1) -2016-06-03 13:29:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-03 13:29:34 INFO DAGScheduler:59 - ResultStage 1 (main at null:-2) finished in 0.073 s -2016-06-03 13:29:34 INFO DAGScheduler:59 - Job 1 finished: main at null:-2, took 0.087693 s -2016-06-03 13:29:35 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-03 13:29:35 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-03 13:29:35 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-03 13:29:35 INFO DAGScheduler:59 - Got job 2 (main at null:-2) with 1 output partitions -2016-06-03 13:29:35 INFO DAGScheduler:59 - Final stage: ResultStage 2(main at null:-2) -2016-06-03 13:29:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-03 13:29:35 INFO DAGScheduler:59 - Missing parents: List() -2016-06-03 13:29:35 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[11] at main at null:-2), which has no missing parents -2016-06-03 13:29:35 INFO MemoryStore:59 - ensureFreeSpace(68744) called with curMem=7932, maxMem=766075207 -2016-06-03 13:29:35 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 67.1 KB, free 730.5 MB) -2016-06-03 13:29:35 INFO MemoryStore:59 - ensureFreeSpace(24473) called with curMem=76676, maxMem=766075207 -2016-06-03 13:29:35 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 23.9 KB, free 730.5 MB) -2016-06-03 13:29:35 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:64554 (size: 23.9 KB, free: 730.6 MB) -2016-06-03 13:29:35 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-03 13:29:35 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[11] at main at null:-2) -2016-06-03 13:29:35 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-06-03 13:29:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2142 bytes) -2016-06-03 13:29:35 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-06-03 13:29:35 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-03 13:29:35 INFO CodecConfig:151 - Compression: GZIP -2016-06-03 13:29:35 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-06-03 13:29:35 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-06-03 13:29:35 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-06-03 13:29:35 INFO ParquetOutputFormat:151 - Dictionary is on -2016-06-03 13:29:35 INFO ParquetOutputFormat:151 - Validation is off -2016-06-03 13:29:35 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-06-03 13:29:35 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-06-03 13:29:35 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 80 -2016-06-03 13:29:35 INFO ColumnChunkPageWriteStore:151 - written 71B for [single] INT32: 5 values, 26B raw, 38B comp, 1 pages, encodings: [RLE, PLAIN, BIT_PACKED] -2016-06-03 13:29:35 INFO ColumnChunkPageWriteStore:151 - written 71B for [triple] INT32: 5 values, 26B raw, 38B comp, 1 pages, encodings: [RLE, PLAIN, BIT_PACKED] -2016-06-03 13:29:35 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201606031329_0002_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/sql/output/20160603132926436/test_table/key=2/_temporary/0/task_201606031329_0002_m_000000 -2016-06-03 13:29:35 INFO SparkHadoopMapRedUtil:59 - attempt_201606031329_0002_m_000000_0: Committed -2016-06-03 13:29:35 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 1060 bytes result sent to driver -2016-06-03 13:29:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 155 ms on localhost (1/1) -2016-06-03 13:29:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-03 13:29:35 INFO DAGScheduler:59 - ResultStage 2 (main at null:-2) finished in 0.157 s -2016-06-03 13:29:35 INFO DAGScheduler:59 - Job 2 finished: main at null:-2, took 0.209376 s -2016-06-03 13:29:35 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-06-03 13:29:35 INFO DefaultWriterContainer:59 - Job job_201606031329_0000 committed. -2016-06-03 13:29:35 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/sql/output/20160603132926436/test_table/key=2 on driver -2016-06-03 13:29:35 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/sql/output/20160603132926436/test_table/key=2 on driver -2016-06-03 13:29:35 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-03 13:29:35 INFO DAGScheduler:59 - Got job 3 (main at null:-2) with 1 output partitions -2016-06-03 13:29:35 INFO DAGScheduler:59 - Final stage: ResultStage 3(main at null:-2) -2016-06-03 13:29:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-03 13:29:35 INFO DAGScheduler:59 - Missing parents: List() -2016-06-03 13:29:35 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[15] at main at null:-2), which has no missing parents -2016-06-03 13:29:35 INFO MemoryStore:59 - ensureFreeSpace(5128) called with curMem=101149, maxMem=766075207 -2016-06-03 13:29:35 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.0 KB, free 730.5 MB) -2016-06-03 13:29:35 INFO MemoryStore:59 - ensureFreeSpace(2810) called with curMem=106277, maxMem=766075207 -2016-06-03 13:29:35 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.5 MB) -2016-06-03 13:29:35 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:64554 (size: 2.7 KB, free: 730.6 MB) -2016-06-03 13:29:35 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-03 13:29:35 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 3 (MapPartitionsRDD[15] at main at null:-2) -2016-06-03 13:29:35 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-06-03 13:29:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 3, localhost, PROCESS_LOCAL, 2142 bytes) -2016-06-03 13:29:35 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 3) -2016-06-03 13:29:35 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 3). 1523 bytes result sent to driver -2016-06-03 13:29:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 3) in 14 ms on localhost (1/1) -2016-06-03 13:29:35 INFO DAGScheduler:59 - ResultStage 3 (main at null:-2) finished in 0.016 s -2016-06-03 13:29:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-03 13:29:35 INFO DAGScheduler:59 - Job 3 finished: main at null:-2, took 0.044177 s -2016-06-03 13:29:35 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/sql/output/20160603132926436/test_table on driver -2016-06-03 13:29:35 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/sql/output/20160603132926436/test_table/key=1 on driver -2016-06-03 13:29:35 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/sql/output/20160603132926436/test_table/key=2 on driver -2016-06-03 13:29:35 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-03 13:29:35 INFO DAGScheduler:59 - Got job 4 (main at null:-2) with 1 output partitions -2016-06-03 13:29:35 INFO DAGScheduler:59 - Final stage: ResultStage 4(main at null:-2) -2016-06-03 13:29:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-03 13:29:35 INFO DAGScheduler:59 - Missing parents: List() -2016-06-03 13:29:35 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[17] at main at null:-2), which has no missing parents -2016-06-03 13:29:35 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=109087, maxMem=766075207 -2016-06-03 13:29:35 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 61.5 KB, free 730.4 MB) -2016-06-03 13:29:35 INFO MemoryStore:59 - ensureFreeSpace(21072) called with curMem=172087, maxMem=766075207 -2016-06-03 13:29:35 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.4 MB) -2016-06-03 13:29:35 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:64554 (size: 20.6 KB, free: 730.5 MB) -2016-06-03 13:29:35 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-03 13:29:35 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (MapPartitionsRDD[17] at main at null:-2) -2016-06-03 13:29:35 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-06-03 13:29:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 4, localhost, PROCESS_LOCAL, 2962 bytes) -2016-06-03 13:29:35 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 4) -2016-06-03 13:29:35 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-06-03 13:29:35 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 4). 1688 bytes result sent to driver -2016-06-03 13:29:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 4) in 98 ms on localhost (1/1) -2016-06-03 13:29:35 INFO DAGScheduler:59 - ResultStage 4 (main at null:-2) finished in 0.098 s -2016-06-03 13:29:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-03 13:29:35 INFO DAGScheduler:59 - Job 4 finished: main at null:-2, took 0.187607 s -2016-06-03 13:29:35 INFO DataSourceStrategy:59 - Selected 2 partitions out of 2, pruned 0.0% partitions. -2016-06-03 13:29:36 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=193159, maxMem=766075207 -2016-06-03 13:29:36 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 86.4 KB, free 730.3 MB) -2016-06-03 13:29:36 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=281671, maxMem=766075207 -2016-06-03 13:29:36 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.3 MB) -2016-06-03 13:29:36 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:64554 (size: 19.3 KB, free: 730.5 MB) -2016-06-03 13:29:36 INFO SparkContext:59 - Created broadcast 5 from main at null:-2 -2016-06-03 13:29:36 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-06-03 13:29:36 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/sql/output/20160603132926436/test_table/key=2/part-r-00000-66921cbc-6fa9-46d6-82ad-aa59ba1fd4a3.gz.parquet -2016-06-03 13:29:36 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/sql/output/20160603132926436/test_table/key=1/part-r-00000-530b6461-41a6-48a3-92c3-ffc182222e45.gz.parquet -2016-06-03 13:29:36 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-03 13:29:36 INFO DAGScheduler:59 - Got job 5 (main at null:-2) with 1 output partitions -2016-06-03 13:29:36 INFO DAGScheduler:59 - Final stage: ResultStage 5(main at null:-2) -2016-06-03 13:29:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-03 13:29:36 INFO DAGScheduler:59 - Missing parents: List() -2016-06-03 13:29:36 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[23] at main at null:-2), which has no missing parents -2016-06-03 13:29:36 INFO MemoryStore:59 - ensureFreeSpace(6400) called with curMem=301459, maxMem=766075207 -2016-06-03 13:29:36 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 6.3 KB, free 730.3 MB) -2016-06-03 13:29:36 INFO MemoryStore:59 - ensureFreeSpace(3422) called with curMem=307859, maxMem=766075207 -2016-06-03 13:29:36 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.3 MB) -2016-06-03 13:29:36 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:64554 (size: 3.3 KB, free: 730.5 MB) -2016-06-03 13:29:36 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-03 13:29:36 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 5 (MapPartitionsRDD[23] at main at null:-2) -2016-06-03 13:29:36 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 1 tasks -2016-06-03 13:29:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 5, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-03 13:29:36 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 5) -2016-06-03 13:29:36 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/sql/output/20160603132926436/test_table/key=2/part-r-00000-66921cbc-6fa9-46d6-82ad-aa59ba1fd4a3.gz.parquet start: 0 end: 537 length: 537 hosts: []} -2016-06-03 13:29:36 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-06-03 13:29:36 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 single; - optional int32 triple; - optional int32 double; -} - -Catalyst form: -StructType(StructField(single,IntegerType,true), StructField(triple,IntegerType,true), StructField(double,IntegerType,true)) - -2016-06-03 13:29:36 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 5 records. -2016-06-03 13:29:36 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-06-03 13:29:36 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-06-03 13:29:36 INFO InternalParquetRecordReader:151 - block read in memory in 23 ms. row count = 5 -2016-06-03 13:29:36 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 5). 2542 bytes result sent to driver -2016-06-03 13:29:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 5) in 268 ms on localhost (1/1) -2016-06-03 13:29:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-03 13:29:36 INFO DAGScheduler:59 - ResultStage 5 (main at null:-2) finished in 0.269 s -2016-06-03 13:29:36 INFO DAGScheduler:59 - Job 5 finished: main at null:-2, took 0.312776 s -2016-06-03 13:29:36 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-03 13:29:36 INFO DAGScheduler:59 - Got job 6 (main at null:-2) with 1 output partitions -2016-06-03 13:29:36 INFO DAGScheduler:59 - Final stage: ResultStage 6(main at null:-2) -2016-06-03 13:29:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-03 13:29:36 INFO DAGScheduler:59 - Missing parents: List() -2016-06-03 13:29:36 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[23] at main at null:-2), which has no missing parents -2016-06-03 13:29:36 INFO MemoryStore:59 - ensureFreeSpace(6400) called with curMem=311281, maxMem=766075207 -2016-06-03 13:29:36 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 6.3 KB, free 730.3 MB) -2016-06-03 13:29:36 INFO MemoryStore:59 - ensureFreeSpace(3422) called with curMem=317681, maxMem=766075207 -2016-06-03 13:29:36 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.3 MB) -2016-06-03 13:29:36 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:64554 (size: 3.3 KB, free: 730.5 MB) -2016-06-03 13:29:36 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-03 13:29:36 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 6 (MapPartitionsRDD[23] at main at null:-2) -2016-06-03 13:29:36 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 1 tasks -2016-06-03 13:29:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 6, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-03 13:29:36 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 6) -2016-06-03 13:29:36 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/sql/output/20160603132926436/test_table/key=1/part-r-00000-530b6461-41a6-48a3-92c3-ffc182222e45.gz.parquet start: 0 end: 537 length: 537 hosts: []} -2016-06-03 13:29:36 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-06-03 13:29:36 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int32 single; - optional int32 triple; - optional int32 double; -} - -Catalyst form: -StructType(StructField(single,IntegerType,true), StructField(triple,IntegerType,true), StructField(double,IntegerType,true)) - -2016-06-03 13:29:36 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 5 records. -2016-06-03 13:29:36 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-06-03 13:29:36 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-06-03 13:29:36 INFO InternalParquetRecordReader:151 - block read in memory in 1 ms. row count = 5 -2016-06-03 13:29:36 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 6). 2527 bytes result sent to driver -2016-06-03 13:29:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 6) in 28 ms on localhost (1/1) -2016-06-03 13:29:36 INFO DAGScheduler:59 - ResultStage 6 (main at null:-2) finished in 0.029 s -2016-06-03 13:29:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-03 13:29:36 INFO DAGScheduler:59 - Job 6 finished: main at null:-2, took 0.045798 s -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-03 13:29:36 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-03 13:29:36 INFO SparkUI:59 - Stopped Spark web UI at http://211.86.159.79:4040 -2016-06-03 13:29:36 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-03 13:29:37 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-03 13:29:37 INFO MemoryStore:59 - MemoryStore cleared -2016-06-03 13:29:37 INFO BlockManager:59 - BlockManager stopped -2016-06-03 13:29:37 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-03 13:29:37 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-03 13:29:37 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-03 13:29:37 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-03 13:29:37 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-32d2733c-2d3a-4c3a-8f3e-9a69834e156a -2016-06-03 13:29:37 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-03 13:29:37 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 13:36:52 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 13:37:00 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 13:37:03 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 13:37:03 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 13:37:03 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 13:37:20 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 13:37:22 INFO Remoting:74 - Starting remoting -2016-06-13 13:37:26 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@219.219.220.162:54410] -2016-06-13 13:37:26 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54410. -2016-06-13 13:37:27 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 13:37:29 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 13:37:30 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-8d9b0044-c359-4607-8cca-52adc572cfb3 -2016-06-13 13:37:30 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 13:37:31 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-e9b3bc5c-60ec-4b92-ac17-03f7480e7e3c\httpd-9c6a5bbb-e872-48ac-b6c3-9a85ca4f1e83 -2016-06-13 13:37:31 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 13:37:33 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 13:37:33 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54412 -2016-06-13 13:37:33 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54412. -2016-06-13 13:37:34 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 13:37:37 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 13:37:37 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 13:37:37 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 13:37:37 INFO SparkUI:59 - Started SparkUI at http://219.219.220.162:4040 -2016-06-13 13:37:39 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 13:37:39 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 13:37:43 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54419. -2016-06-13 13:37:43 INFO NettyBlockTransferService:59 - Server created on 54419 -2016-06-13 13:37:43 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 13:37:43 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54419 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54419) -2016-06-13 13:37:44 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 13:38:03 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: 2002:dbdb:dca2:0:0:0:dbdb:dca2, but we couldn't find any external IP address! -2016-06-13 13:38:05 INFO JSONRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/examples/src/main/resources/people.json on driver -2016-06-13 13:38:24 INFO MemoryStore:59 - ensureFreeSpace(231288) called with curMem=0, maxMem=766075207 -2016-06-13 13:38:24 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 225.9 KB, free 730.4 MB) -2016-06-13 13:38:25 INFO MemoryStore:59 - ensureFreeSpace(19919) called with curMem=231288, maxMem=766075207 -2016-06-13 13:38:25 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 19.5 KB, free 730.3 MB) -2016-06-13 13:38:25 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54419 (size: 19.5 KB, free: 730.6 MB) -2016-06-13 13:38:25 INFO SparkContext:59 - Created broadcast 0 from main at null:-2 -2016-06-13 13:38:28 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 13:38:29 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-13 13:38:31 INFO DAGScheduler:59 - Got job 0 (main at null:-2) with 1 output partitions -2016-06-13 13:38:31 INFO DAGScheduler:59 - Final stage: ResultStage 0(main at null:-2) -2016-06-13 13:38:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 13:38:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 13:38:31 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at main at null:-2), which has no missing parents -2016-06-13 13:38:32 INFO MemoryStore:59 - ensureFreeSpace(4056) called with curMem=251207, maxMem=766075207 -2016-06-13 13:38:32 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 13:38:32 INFO MemoryStore:59 - ensureFreeSpace(2295) called with curMem=255263, maxMem=766075207 -2016-06-13 13:38:32 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 13:38:32 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54419 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 13:38:32 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 13:38:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at main at null:-2) -2016-06-13 13:38:33 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 13:38:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 13:38:35 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 13:38:36 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/examples/src/main/resources/people.json:0+73 -2016-06-13 13:38:37 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 13:38:37 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 13:38:37 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 13:38:37 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 13:38:37 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 13:38:59 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2845 bytes result sent to driver -2016-06-13 13:38:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 24115 ms on localhost (1/1) -2016-06-13 13:38:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 13:38:59 INFO DAGScheduler:59 - ResultStage 0 (main at null:-2) finished in 25.184 s -2016-06-13 13:38:59 INFO DAGScheduler:59 - Job 0 finished: main at null:-2, took 29.981676 s -2016-06-13 13:39:09 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=257558, maxMem=766075207 -2016-06-13 13:39:09 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 86.4 KB, free 730.3 MB) -2016-06-13 13:39:09 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=346070, maxMem=766075207 -2016-06-13 13:39:09 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.2 MB) -2016-06-13 13:39:09 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54419 (size: 19.3 KB, free: 730.5 MB) -2016-06-13 13:39:09 INFO SparkContext:59 - Created broadcast 2 from main at null:-2 -2016-06-13 13:39:10 INFO MemoryStore:59 - ensureFreeSpace(231288) called with curMem=365858, maxMem=766075207 -2016-06-13 13:39:10 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 225.9 KB, free 730.0 MB) -2016-06-13 13:39:11 INFO MemoryStore:59 - ensureFreeSpace(19919) called with curMem=597146, maxMem=766075207 -2016-06-13 13:39:11 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 19.5 KB, free 730.0 MB) -2016-06-13 13:39:11 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54419 (size: 19.5 KB, free: 730.5 MB) -2016-06-13 13:39:11 INFO SparkContext:59 - Created broadcast 3 from main at null:-2 -2016-06-13 13:39:13 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 13:39:13 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-13 13:39:13 INFO DAGScheduler:59 - Got job 1 (main at null:-2) with 1 output partitions -2016-06-13 13:39:13 INFO DAGScheduler:59 - Final stage: ResultStage 1(main at null:-2) -2016-06-13 13:39:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 13:39:13 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 13:39:13 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at main at null:-2), which has no missing parents -2016-06-13 13:39:13 INFO MemoryStore:59 - ensureFreeSpace(3960) called with curMem=617065, maxMem=766075207 -2016-06-13 13:39:13 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.9 KB, free 730.0 MB) -2016-06-13 13:39:13 INFO MemoryStore:59 - ensureFreeSpace(2299) called with curMem=621025, maxMem=766075207 -2016-06-13 13:39:13 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.0 MB) -2016-06-13 13:39:13 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54419 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 13:39:13 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 13:39:13 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at main at null:-2) -2016-06-13 13:39:13 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 13:39:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 13:39:13 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 13:39:13 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/examples/src/main/resources/people.json:0+73 -2016-06-13 13:39:13 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2508 bytes result sent to driver -2016-06-13 13:39:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 110 ms on localhost (1/1) -2016-06-13 13:39:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 13:39:13 INFO DAGScheduler:59 - ResultStage 1 (main at null:-2) finished in 0.111 s -2016-06-13 13:39:13 INFO DAGScheduler:59 - Job 1 finished: main at null:-2, took 0.173575 s -2016-06-13 13:39:13 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:54419 in memory (size: 19.3 KB, free: 730.5 MB) -2016-06-13 13:39:16 INFO MemoryStore:59 - ensureFreeSpace(230728) called with curMem=515024, maxMem=766075207 -2016-06-13 13:39:16 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 225.3 KB, free 729.9 MB) -2016-06-13 13:39:17 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=745752, maxMem=766075207 -2016-06-13 13:39:17 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.9 MB) -2016-06-13 13:39:17 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:54419 (size: 19.3 KB, free: 730.5 MB) -2016-06-13 13:39:17 INFO SparkContext:59 - Created broadcast 5 from main at null:-2 -2016-06-13 13:39:17 INFO MemoryStore:59 - ensureFreeSpace(231288) called with curMem=765540, maxMem=766075207 -2016-06-13 13:39:17 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 225.9 KB, free 729.6 MB) -2016-06-13 13:39:17 INFO MemoryStore:59 - ensureFreeSpace(19919) called with curMem=996828, maxMem=766075207 -2016-06-13 13:39:17 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 19.5 KB, free 729.6 MB) -2016-06-13 13:39:17 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:54419 (size: 19.5 KB, free: 730.5 MB) -2016-06-13 13:39:17 INFO SparkContext:59 - Created broadcast 6 from main at null:-2 -2016-06-13 13:39:17 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 13:39:17 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-13 13:39:17 INFO DAGScheduler:59 - Got job 2 (main at null:-2) with 1 output partitions -2016-06-13 13:39:17 INFO DAGScheduler:59 - Final stage: ResultStage 2(main at null:-2) -2016-06-13 13:39:17 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:54419 in memory (size: 19.3 KB, free: 730.5 MB) -2016-06-13 13:39:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 13:39:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 13:39:17 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[11] at main at null:-2), which has no missing parents -2016-06-13 13:39:17 INFO MemoryStore:59 - ensureFreeSpace(3888) called with curMem=766231, maxMem=766075207 -2016-06-13 13:39:17 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.8 KB, free 729.9 MB) -2016-06-13 13:39:17 INFO MemoryStore:59 - ensureFreeSpace(2254) called with curMem=770119, maxMem=766075207 -2016-06-13 13:39:17 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.8 MB) -2016-06-13 13:39:17 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:54419 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 13:39:17 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 13:39:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[11] at main at null:-2) -2016-06-13 13:39:17 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-06-13 13:39:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 13:39:17 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-06-13 13:39:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/examples/src/main/resources/people.json:0+73 -2016-06-13 13:39:17 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 2415 bytes result sent to driver -2016-06-13 13:39:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 17 ms on localhost (1/1) -2016-06-13 13:39:17 INFO DAGScheduler:59 - ResultStage 2 (main at null:-2) finished in 0.017 s -2016-06-13 13:39:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 13:39:17 INFO DAGScheduler:59 - Job 2 finished: main at null:-2, took 0.031530 s -2016-06-13 13:39:20 INFO MemoryStore:59 - ensureFreeSpace(230728) called with curMem=772373, maxMem=766075207 -2016-06-13 13:39:20 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 225.3 KB, free 729.6 MB) -2016-06-13 13:39:20 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1003101, maxMem=766075207 -2016-06-13 13:39:20 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.6 MB) -2016-06-13 13:39:20 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:54419 (size: 19.3 KB, free: 730.5 MB) -2016-06-13 13:39:20 INFO SparkContext:59 - Created broadcast 8 from main at null:-2 -2016-06-13 13:39:20 INFO MemoryStore:59 - ensureFreeSpace(231288) called with curMem=1022889, maxMem=766075207 -2016-06-13 13:39:20 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 225.9 KB, free 729.4 MB) -2016-06-13 13:39:20 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:54419 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 13:39:20 INFO MemoryStore:59 - ensureFreeSpace(19919) called with curMem=1248035, maxMem=766075207 -2016-06-13 13:39:20 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 19.5 KB, free 729.4 MB) -2016-06-13 13:39:20 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:54419 (size: 19.5 KB, free: 730.5 MB) -2016-06-13 13:39:20 INFO SparkContext:59 - Created broadcast 9 from main at null:-2 -2016-06-13 13:39:20 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 13:39:21 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 13:39:21 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-13 13:39:21 INFO DAGScheduler:59 - Got job 3 (main at null:-2) with 1 output partitions -2016-06-13 13:39:21 INFO DAGScheduler:59 - Final stage: ResultStage 3(main at null:-2) -2016-06-13 13:39:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 13:39:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 13:39:21 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[16] at main at null:-2), which has no missing parents -2016-06-13 13:39:21 INFO MemoryStore:59 - ensureFreeSpace(6608) called with curMem=1267954, maxMem=766075207 -2016-06-13 13:39:21 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 6.5 KB, free 729.4 MB) -2016-06-13 13:39:21 INFO MemoryStore:59 - ensureFreeSpace(3682) called with curMem=1274562, maxMem=766075207 -2016-06-13 13:39:21 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.6 KB, free 729.4 MB) -2016-06-13 13:39:21 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:54419 (size: 3.6 KB, free: 730.5 MB) -2016-06-13 13:39:21 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 13:39:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 3 (MapPartitionsRDD[16] at main at null:-2) -2016-06-13 13:39:21 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-06-13 13:39:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 3, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 13:39:21 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 3) -2016-06-13 13:39:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/examples/src/main/resources/people.json:0+73 -2016-06-13 13:39:24 INFO GenerateMutableProjection:59 - Code generated in 2758.02227 ms -2016-06-13 13:39:24 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 3). 2653 bytes result sent to driver -2016-06-13 13:39:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 3) in 3514 ms on localhost (1/1) -2016-06-13 13:39:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 13:39:25 INFO DAGScheduler:59 - ResultStage 3 (main at null:-2) finished in 3.515 s -2016-06-13 13:39:25 INFO DAGScheduler:59 - Job 3 finished: main at null:-2, took 3.538772 s -2016-06-13 13:39:25 INFO MemoryStore:59 - ensureFreeSpace(230728) called with curMem=1278244, maxMem=766075207 -2016-06-13 13:39:25 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 225.3 KB, free 729.1 MB) -2016-06-13 13:39:25 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1508972, maxMem=766075207 -2016-06-13 13:39:25 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.1 MB) -2016-06-13 13:39:25 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:54419 (size: 19.3 KB, free: 730.5 MB) -2016-06-13 13:39:25 INFO SparkContext:59 - Created broadcast 11 from main at null:-2 -2016-06-13 13:39:25 INFO MemoryStore:59 - ensureFreeSpace(231288) called with curMem=1528760, maxMem=766075207 -2016-06-13 13:39:25 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 225.9 KB, free 728.9 MB) -2016-06-13 13:39:25 INFO MemoryStore:59 - ensureFreeSpace(19919) called with curMem=1760048, maxMem=766075207 -2016-06-13 13:39:25 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 19.5 KB, free 728.9 MB) -2016-06-13 13:39:25 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:54419 (size: 19.5 KB, free: 730.4 MB) -2016-06-13 13:39:25 INFO SparkContext:59 - Created broadcast 12 from main at null:-2 -2016-06-13 13:39:25 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 13:39:25 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-13 13:39:25 INFO DAGScheduler:59 - Got job 4 (main at null:-2) with 1 output partitions -2016-06-13 13:39:25 INFO DAGScheduler:59 - Final stage: ResultStage 4(main at null:-2) -2016-06-13 13:39:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 13:39:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 13:39:25 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[21] at main at null:-2), which has no missing parents -2016-06-13 13:39:25 INFO MemoryStore:59 - ensureFreeSpace(6408) called with curMem=1779967, maxMem=766075207 -2016-06-13 13:39:25 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 6.3 KB, free 728.9 MB) -2016-06-13 13:39:25 INFO MemoryStore:59 - ensureFreeSpace(3558) called with curMem=1786375, maxMem=766075207 -2016-06-13 13:39:25 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.5 KB, free 728.9 MB) -2016-06-13 13:39:25 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:54419 (size: 3.5 KB, free: 730.4 MB) -2016-06-13 13:39:25 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 13:39:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (MapPartitionsRDD[21] at main at null:-2) -2016-06-13 13:39:25 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-06-13 13:39:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 4, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 13:39:25 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 4) -2016-06-13 13:39:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/examples/src/main/resources/people.json:0+73 -2016-06-13 13:39:25 INFO GeneratePredicate:59 - Code generated in 5.59687 ms -2016-06-13 13:39:25 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 4). 2636 bytes result sent to driver -2016-06-13 13:39:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 4) in 78 ms on localhost (1/1) -2016-06-13 13:39:25 INFO DAGScheduler:59 - ResultStage 4 (main at null:-2) finished in 0.079 s -2016-06-13 13:39:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 13:39:25 INFO DAGScheduler:59 - Job 4 finished: main at null:-2, took 0.095565 s -2016-06-13 13:39:26 INFO MemoryStore:59 - ensureFreeSpace(230728) called with curMem=1789933, maxMem=766075207 -2016-06-13 13:39:26 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 225.3 KB, free 728.7 MB) -2016-06-13 13:39:26 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=2020661, maxMem=766075207 -2016-06-13 13:39:26 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 19.3 KB, free 728.6 MB) -2016-06-13 13:39:26 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:54419 (size: 19.3 KB, free: 730.4 MB) -2016-06-13 13:39:26 INFO SparkContext:59 - Created broadcast 14 from main at null:-2 -2016-06-13 13:39:26 INFO MemoryStore:59 - ensureFreeSpace(231288) called with curMem=2040449, maxMem=766075207 -2016-06-13 13:39:26 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 225.9 KB, free 728.4 MB) -2016-06-13 13:39:26 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 13:39:26 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 13:39:26 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:54419 in memory (size: 3.5 KB, free: 730.4 MB) -2016-06-13 13:39:26 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 13:39:26 INFO MemoryStore:59 - ensureFreeSpace(19919) called with curMem=2261771, maxMem=766075207 -2016-06-13 13:39:26 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 19.5 KB, free 728.4 MB) -2016-06-13 13:39:26 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:54419 (size: 19.5 KB, free: 730.4 MB) -2016-06-13 13:39:26 INFO SparkContext:59 - Created broadcast 15 from main at null:-2 -2016-06-13 13:39:28 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-13 13:39:28 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 13:39:28 INFO DAGScheduler:59 - Registering RDD 26 (main at null:-2) -2016-06-13 13:39:28 INFO DAGScheduler:59 - Got job 5 (main at null:-2) with 1 output partitions -2016-06-13 13:39:28 INFO DAGScheduler:59 - Final stage: ResultStage 6(main at null:-2) -2016-06-13 13:39:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 5) -2016-06-13 13:39:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 5) -2016-06-13 13:39:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 5 (MapPartitionsRDD[26] at main at null:-2), which has no missing parents -2016-06-13 13:39:28 INFO MemoryStore:59 - ensureFreeSpace(10424) called with curMem=2281690, maxMem=766075207 -2016-06-13 13:39:28 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 10.2 KB, free 728.4 MB) -2016-06-13 13:39:28 INFO MemoryStore:59 - ensureFreeSpace(5241) called with curMem=2292114, maxMem=766075207 -2016-06-13 13:39:28 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 5.1 KB, free 728.4 MB) -2016-06-13 13:39:28 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:54419 (size: 5.1 KB, free: 730.4 MB) -2016-06-13 13:39:28 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 13:39:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 5 (MapPartitionsRDD[26] at main at null:-2) -2016-06-13 13:39:28 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 1 tasks -2016-06-13 13:39:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 5, localhost, PROCESS_LOCAL, 2179 bytes) -2016-06-13 13:39:28 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 5) -2016-06-13 13:39:29 INFO GenerateMutableProjection:59 - Code generated in 18.908345 ms -2016-06-13 13:39:29 INFO GenerateUnsafeProjection:59 - Code generated in 13.114694 ms -2016-06-13 13:39:29 INFO GenerateMutableProjection:59 - Code generated in 25.269134 ms -2016-06-13 13:39:29 INFO GenerateUnsafeRowJoiner:59 - Code generated in 9.244675 ms -2016-06-13 13:39:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/examples/src/main/resources/people.json:0+73 -2016-06-13 13:39:29 INFO GenerateMutableProjection:59 - Code generated in 6.540614 ms -2016-06-13 13:39:33 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 5). 2669 bytes result sent to driver -2016-06-13 13:39:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 5) in 4300 ms on localhost (1/1) -2016-06-13 13:39:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 13:39:33 INFO DAGScheduler:59 - ShuffleMapStage 5 (main at null:-2) finished in 4.301 s -2016-06-13 13:39:33 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 13:39:33 INFO DAGScheduler:59 - running: Set() -2016-06-13 13:39:33 INFO DAGScheduler:59 - waiting: Set(ResultStage 6) -2016-06-13 13:39:33 INFO DAGScheduler:59 - failed: Set() -2016-06-13 13:39:33 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List() -2016-06-13 13:39:33 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[30] at main at null:-2), which is now runnable -2016-06-13 13:39:33 INFO MemoryStore:59 - ensureFreeSpace(11224) called with curMem=2297355, maxMem=766075207 -2016-06-13 13:39:33 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 11.0 KB, free 728.4 MB) -2016-06-13 13:39:33 INFO MemoryStore:59 - ensureFreeSpace(5645) called with curMem=2308579, maxMem=766075207 -2016-06-13 13:39:33 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 5.5 KB, free 728.4 MB) -2016-06-13 13:39:33 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:54419 (size: 5.5 KB, free: 730.4 MB) -2016-06-13 13:39:33 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 13:39:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 6 (MapPartitionsRDD[30] at main at null:-2) -2016-06-13 13:39:33 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 1 tasks -2016-06-13 13:39:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 6, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:33 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 6) -2016-06-13 13:39:33 INFO GenerateMutableProjection:59 - Code generated in 7.01137 ms -2016-06-13 13:39:33 INFO GenerateUnsafeProjection:59 - Code generated in 9.821184 ms -2016-06-13 13:39:33 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:54419 in memory (size: 5.1 KB, free: 730.4 MB) -2016-06-13 13:39:33 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:54419 in memory (size: 19.3 KB, free: 730.4 MB) -2016-06-13 13:39:33 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:54419 in memory (size: 19.5 KB, free: 730.4 MB) -2016-06-13 13:39:33 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:54419 in memory (size: 19.3 KB, free: 730.5 MB) -2016-06-13 13:39:33 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:54419 in memory (size: 3.6 KB, free: 730.5 MB) -2016-06-13 13:39:33 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 13:39:33 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 13:39:33 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:54419 in memory (size: 19.5 KB, free: 730.5 MB) -2016-06-13 13:39:33 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:54419 in memory (size: 19.3 KB, free: 730.5 MB) -2016-06-13 13:39:33 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:54419 in memory (size: 19.5 KB, free: 730.5 MB) -2016-06-13 13:39:33 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:54419 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 13:39:33 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 13:39:33 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:54419 in memory (size: 19.5 KB, free: 730.5 MB) -2016-06-13 13:39:33 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:54419 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 13:39:33 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 13:39:33 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:54419 in memory (size: 19.5 KB, free: 730.6 MB) -2016-06-13 13:39:33 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 20 ms -2016-06-13 13:39:34 INFO GenerateSafeProjection:59 - Code generated in 6.762383 ms -2016-06-13 13:39:34 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 6). 1437 bytes result sent to driver -2016-06-13 13:39:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 6) in 1249 ms on localhost (1/1) -2016-06-13 13:39:34 INFO DAGScheduler:59 - ResultStage 6 (main at null:-2) finished in 1.250 s -2016-06-13 13:39:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 13:39:34 INFO DAGScheduler:59 - Job 5 finished: main at null:-2, took 5.975696 s -2016-06-13 13:39:34 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-13 13:39:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 150 bytes -2016-06-13 13:39:34 INFO DAGScheduler:59 - Got job 6 (main at null:-2) with 199 output partitions -2016-06-13 13:39:34 INFO DAGScheduler:59 - Final stage: ResultStage 8(main at null:-2) -2016-06-13 13:39:34 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 7) -2016-06-13 13:39:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 13:39:34 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[30] at main at null:-2), which has no missing parents -2016-06-13 13:39:34 INFO MemoryStore:59 - ensureFreeSpace(11224) called with curMem=268076, maxMem=766075207 -2016-06-13 13:39:34 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.0 KB, free 730.3 MB) -2016-06-13 13:39:34 INFO MemoryStore:59 - ensureFreeSpace(5645) called with curMem=279300, maxMem=766075207 -2016-06-13 13:39:34 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 5.5 KB, free 730.3 MB) -2016-06-13 13:39:34 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:54419 (size: 5.5 KB, free: 730.6 MB) -2016-06-13 13:39:34 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 13:39:34 INFO DAGScheduler:59 - Submitting 199 missing tasks from ResultStage 8 (MapPartitionsRDD[30] at main at null:-2) -2016-06-13 13:39:34 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 199 tasks -2016-06-13 13:39:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 7, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:34 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 7) -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:34 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 7). 1437 bytes result sent to driver -2016-06-13 13:39:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 8, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 7) in 21 ms on localhost (1/199) -2016-06-13 13:39:34 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 8) -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:34 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 8). 1437 bytes result sent to driver -2016-06-13 13:39:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 8.0 (TID 9, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 8) in 22 ms on localhost (2/199) -2016-06-13 13:39:34 INFO Executor:59 - Running task 2.0 in stage 8.0 (TID 9) -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:34 INFO Executor:59 - Finished task 2.0 in stage 8.0 (TID 9). 1437 bytes result sent to driver -2016-06-13 13:39:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 8.0 (TID 10, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:34 INFO Executor:59 - Running task 3.0 in stage 8.0 (TID 10) -2016-06-13 13:39:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 8.0 (TID 9) in 24 ms on localhost (3/199) -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:34 INFO Executor:59 - Finished task 3.0 in stage 8.0 (TID 10). 1437 bytes result sent to driver -2016-06-13 13:39:34 INFO TaskSetManager:59 - Starting task 4.0 in stage 8.0 (TID 11, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:34 INFO Executor:59 - Running task 4.0 in stage 8.0 (TID 11) -2016-06-13 13:39:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 8.0 (TID 10) in 18 ms on localhost (4/199) -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:34 INFO Executor:59 - Finished task 4.0 in stage 8.0 (TID 11). 1437 bytes result sent to driver -2016-06-13 13:39:34 INFO TaskSetManager:59 - Starting task 5.0 in stage 8.0 (TID 12, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:34 INFO Executor:59 - Running task 5.0 in stage 8.0 (TID 12) -2016-06-13 13:39:34 INFO TaskSetManager:59 - Finished task 4.0 in stage 8.0 (TID 11) in 20 ms on localhost (5/199) -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:34 INFO Executor:59 - Finished task 5.0 in stage 8.0 (TID 12). 1437 bytes result sent to driver -2016-06-13 13:39:34 INFO TaskSetManager:59 - Starting task 6.0 in stage 8.0 (TID 13, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:34 INFO Executor:59 - Running task 6.0 in stage 8.0 (TID 13) -2016-06-13 13:39:34 INFO TaskSetManager:59 - Finished task 5.0 in stage 8.0 (TID 12) in 24 ms on localhost (6/199) -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:34 INFO Executor:59 - Finished task 6.0 in stage 8.0 (TID 13). 1437 bytes result sent to driver -2016-06-13 13:39:34 INFO TaskSetManager:59 - Starting task 7.0 in stage 8.0 (TID 14, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:34 INFO Executor:59 - Running task 7.0 in stage 8.0 (TID 14) -2016-06-13 13:39:34 INFO TaskSetManager:59 - Finished task 6.0 in stage 8.0 (TID 13) in 20 ms on localhost (7/199) -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:34 INFO Executor:59 - Finished task 7.0 in stage 8.0 (TID 14). 1437 bytes result sent to driver -2016-06-13 13:39:34 INFO TaskSetManager:59 - Starting task 8.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:34 INFO Executor:59 - Running task 8.0 in stage 8.0 (TID 15) -2016-06-13 13:39:34 INFO TaskSetManager:59 - Finished task 7.0 in stage 8.0 (TID 14) in 22 ms on localhost (8/199) -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:34 INFO Executor:59 - Finished task 8.0 in stage 8.0 (TID 15). 1437 bytes result sent to driver -2016-06-13 13:39:34 INFO TaskSetManager:59 - Starting task 9.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:34 INFO Executor:59 - Running task 9.0 in stage 8.0 (TID 16) -2016-06-13 13:39:34 INFO TaskSetManager:59 - Finished task 8.0 in stage 8.0 (TID 15) in 21 ms on localhost (9/199) -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:34 INFO Executor:59 - Finished task 9.0 in stage 8.0 (TID 16). 1437 bytes result sent to driver -2016-06-13 13:39:34 INFO TaskSetManager:59 - Starting task 10.0 in stage 8.0 (TID 17, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:34 INFO Executor:59 - Running task 10.0 in stage 8.0 (TID 17) -2016-06-13 13:39:34 INFO TaskSetManager:59 - Finished task 9.0 in stage 8.0 (TID 16) in 17 ms on localhost (10/199) -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:34 INFO Executor:59 - Finished task 10.0 in stage 8.0 (TID 17). 1437 bytes result sent to driver -2016-06-13 13:39:34 INFO TaskSetManager:59 - Starting task 11.0 in stage 8.0 (TID 18, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:34 INFO Executor:59 - Running task 11.0 in stage 8.0 (TID 18) -2016-06-13 13:39:34 INFO TaskSetManager:59 - Finished task 10.0 in stage 8.0 (TID 17) in 18 ms on localhost (11/199) -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:34 INFO Executor:59 - Finished task 11.0 in stage 8.0 (TID 18). 1437 bytes result sent to driver -2016-06-13 13:39:34 INFO TaskSetManager:59 - Starting task 12.0 in stage 8.0 (TID 19, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:34 INFO Executor:59 - Running task 12.0 in stage 8.0 (TID 19) -2016-06-13 13:39:34 INFO TaskSetManager:59 - Finished task 11.0 in stage 8.0 (TID 18) in 88 ms on localhost (12/199) -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:34 INFO Executor:59 - Finished task 12.0 in stage 8.0 (TID 19). 1437 bytes result sent to driver -2016-06-13 13:39:34 INFO TaskSetManager:59 - Starting task 13.0 in stage 8.0 (TID 20, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:34 INFO Executor:59 - Running task 13.0 in stage 8.0 (TID 20) -2016-06-13 13:39:34 INFO TaskSetManager:59 - Finished task 12.0 in stage 8.0 (TID 19) in 24 ms on localhost (13/199) -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:34 INFO Executor:59 - Finished task 13.0 in stage 8.0 (TID 20). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 14.0 in stage 8.0 (TID 21, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 13.0 in stage 8.0 (TID 20) in 16 ms on localhost (14/199) -2016-06-13 13:39:35 INFO Executor:59 - Running task 14.0 in stage 8.0 (TID 21) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 14.0 in stage 8.0 (TID 21). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 15.0 in stage 8.0 (TID 22, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 15.0 in stage 8.0 (TID 22) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 14.0 in stage 8.0 (TID 21) in 19 ms on localhost (15/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 15.0 in stage 8.0 (TID 22). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 16.0 in stage 8.0 (TID 23, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 16.0 in stage 8.0 (TID 23) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 15.0 in stage 8.0 (TID 22) in 18 ms on localhost (16/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 16.0 in stage 8.0 (TID 23). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 17.0 in stage 8.0 (TID 24, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 17.0 in stage 8.0 (TID 24) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 16.0 in stage 8.0 (TID 23) in 16 ms on localhost (17/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 17.0 in stage 8.0 (TID 24). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 18.0 in stage 8.0 (TID 25, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 18.0 in stage 8.0 (TID 25) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 17.0 in stage 8.0 (TID 24) in 19 ms on localhost (18/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 18.0 in stage 8.0 (TID 25). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 19.0 in stage 8.0 (TID 26, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 19.0 in stage 8.0 (TID 26) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 18.0 in stage 8.0 (TID 25) in 19 ms on localhost (19/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 19.0 in stage 8.0 (TID 26). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 20.0 in stage 8.0 (TID 27, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 20.0 in stage 8.0 (TID 27) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 19.0 in stage 8.0 (TID 26) in 15 ms on localhost (20/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 20.0 in stage 8.0 (TID 27). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 21.0 in stage 8.0 (TID 28, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 21.0 in stage 8.0 (TID 28) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 20.0 in stage 8.0 (TID 27) in 17 ms on localhost (21/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 21.0 in stage 8.0 (TID 28). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 22.0 in stage 8.0 (TID 29, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 22.0 in stage 8.0 (TID 29) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 21.0 in stage 8.0 (TID 28) in 15 ms on localhost (22/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 22.0 in stage 8.0 (TID 29). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 23.0 in stage 8.0 (TID 30, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 23.0 in stage 8.0 (TID 30) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 22.0 in stage 8.0 (TID 29) in 21 ms on localhost (23/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 23.0 in stage 8.0 (TID 30). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 24.0 in stage 8.0 (TID 31, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 24.0 in stage 8.0 (TID 31) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 23.0 in stage 8.0 (TID 30) in 41 ms on localhost (24/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 24.0 in stage 8.0 (TID 31). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 25.0 in stage 8.0 (TID 32, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 25.0 in stage 8.0 (TID 32) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 24.0 in stage 8.0 (TID 31) in 41 ms on localhost (25/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 25.0 in stage 8.0 (TID 32). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 26.0 in stage 8.0 (TID 33, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 26.0 in stage 8.0 (TID 33) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 25.0 in stage 8.0 (TID 32) in 17 ms on localhost (26/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 26.0 in stage 8.0 (TID 33). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 27.0 in stage 8.0 (TID 34, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 27.0 in stage 8.0 (TID 34) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 26.0 in stage 8.0 (TID 33) in 18 ms on localhost (27/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 27.0 in stage 8.0 (TID 34). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 28.0 in stage 8.0 (TID 35, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 28.0 in stage 8.0 (TID 35) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 27.0 in stage 8.0 (TID 34) in 17 ms on localhost (28/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 28.0 in stage 8.0 (TID 35). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 29.0 in stage 8.0 (TID 36, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 29.0 in stage 8.0 (TID 36) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 28.0 in stage 8.0 (TID 35) in 19 ms on localhost (29/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 29.0 in stage 8.0 (TID 36). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 30.0 in stage 8.0 (TID 37, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 30.0 in stage 8.0 (TID 37) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 29.0 in stage 8.0 (TID 36) in 32 ms on localhost (30/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 30.0 in stage 8.0 (TID 37). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 31.0 in stage 8.0 (TID 38, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 30.0 in stage 8.0 (TID 37) in 16 ms on localhost (31/199) -2016-06-13 13:39:35 INFO Executor:59 - Running task 31.0 in stage 8.0 (TID 38) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 31.0 in stage 8.0 (TID 38). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 32.0 in stage 8.0 (TID 39, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 32.0 in stage 8.0 (TID 39) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 31.0 in stage 8.0 (TID 38) in 18 ms on localhost (32/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 32.0 in stage 8.0 (TID 39). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 33.0 in stage 8.0 (TID 40, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 33.0 in stage 8.0 (TID 40) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 32.0 in stage 8.0 (TID 39) in 18 ms on localhost (33/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 33.0 in stage 8.0 (TID 40). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 34.0 in stage 8.0 (TID 41, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 34.0 in stage 8.0 (TID 41) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 33.0 in stage 8.0 (TID 40) in 12 ms on localhost (34/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 34.0 in stage 8.0 (TID 41). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 35.0 in stage 8.0 (TID 42, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 35.0 in stage 8.0 (TID 42) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 34.0 in stage 8.0 (TID 41) in 16 ms on localhost (35/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 35.0 in stage 8.0 (TID 42). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 36.0 in stage 8.0 (TID 43, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 36.0 in stage 8.0 (TID 43) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 35.0 in stage 8.0 (TID 42) in 14 ms on localhost (36/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 36.0 in stage 8.0 (TID 43). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 37.0 in stage 8.0 (TID 44, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 37.0 in stage 8.0 (TID 44) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 36.0 in stage 8.0 (TID 43) in 11 ms on localhost (37/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 37.0 in stage 8.0 (TID 44). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 38.0 in stage 8.0 (TID 45, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 38.0 in stage 8.0 (TID 45) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 37.0 in stage 8.0 (TID 44) in 12 ms on localhost (38/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 38.0 in stage 8.0 (TID 45). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 39.0 in stage 8.0 (TID 46, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 39.0 in stage 8.0 (TID 46) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 38.0 in stage 8.0 (TID 45) in 11 ms on localhost (39/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 39.0 in stage 8.0 (TID 46). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 40.0 in stage 8.0 (TID 47, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 40.0 in stage 8.0 (TID 47) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 39.0 in stage 8.0 (TID 46) in 13 ms on localhost (40/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 40.0 in stage 8.0 (TID 47). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 41.0 in stage 8.0 (TID 48, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 41.0 in stage 8.0 (TID 48) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 40.0 in stage 8.0 (TID 47) in 12 ms on localhost (41/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 41.0 in stage 8.0 (TID 48). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 42.0 in stage 8.0 (TID 49, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 42.0 in stage 8.0 (TID 49) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 41.0 in stage 8.0 (TID 48) in 12 ms on localhost (42/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 42.0 in stage 8.0 (TID 49). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 43.0 in stage 8.0 (TID 50, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 43.0 in stage 8.0 (TID 50) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 42.0 in stage 8.0 (TID 49) in 11 ms on localhost (43/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 43.0 in stage 8.0 (TID 50). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 44.0 in stage 8.0 (TID 51, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 43.0 in stage 8.0 (TID 50) in 16 ms on localhost (44/199) -2016-06-13 13:39:35 INFO Executor:59 - Running task 44.0 in stage 8.0 (TID 51) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 44.0 in stage 8.0 (TID 51). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 45.0 in stage 8.0 (TID 52, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 45.0 in stage 8.0 (TID 52) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 44.0 in stage 8.0 (TID 51) in 13 ms on localhost (45/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 45.0 in stage 8.0 (TID 52). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 46.0 in stage 8.0 (TID 53, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 46.0 in stage 8.0 (TID 53) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 45.0 in stage 8.0 (TID 52) in 10 ms on localhost (46/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 46.0 in stage 8.0 (TID 53). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 47.0 in stage 8.0 (TID 54, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 47.0 in stage 8.0 (TID 54) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 46.0 in stage 8.0 (TID 53) in 10 ms on localhost (47/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 47.0 in stage 8.0 (TID 54). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 48.0 in stage 8.0 (TID 55, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 47.0 in stage 8.0 (TID 54) in 11 ms on localhost (48/199) -2016-06-13 13:39:35 INFO Executor:59 - Running task 48.0 in stage 8.0 (TID 55) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 48.0 in stage 8.0 (TID 55). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 49.0 in stage 8.0 (TID 56, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 49.0 in stage 8.0 (TID 56) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 48.0 in stage 8.0 (TID 55) in 11 ms on localhost (49/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 49.0 in stage 8.0 (TID 56). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 50.0 in stage 8.0 (TID 57, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 50.0 in stage 8.0 (TID 57) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 49.0 in stage 8.0 (TID 56) in 11 ms on localhost (50/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 50.0 in stage 8.0 (TID 57). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 51.0 in stage 8.0 (TID 58, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 51.0 in stage 8.0 (TID 58) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 50.0 in stage 8.0 (TID 57) in 11 ms on localhost (51/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 51.0 in stage 8.0 (TID 58). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 52.0 in stage 8.0 (TID 59, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 51.0 in stage 8.0 (TID 58) in 14 ms on localhost (52/199) -2016-06-13 13:39:35 INFO Executor:59 - Running task 52.0 in stage 8.0 (TID 59) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 52.0 in stage 8.0 (TID 59). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 53.0 in stage 8.0 (TID 60, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 52.0 in stage 8.0 (TID 59) in 13 ms on localhost (53/199) -2016-06-13 13:39:35 INFO Executor:59 - Running task 53.0 in stage 8.0 (TID 60) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 53.0 in stage 8.0 (TID 60). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 54.0 in stage 8.0 (TID 61, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 54.0 in stage 8.0 (TID 61) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 53.0 in stage 8.0 (TID 60) in 12 ms on localhost (54/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 54.0 in stage 8.0 (TID 61). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 55.0 in stage 8.0 (TID 62, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 55.0 in stage 8.0 (TID 62) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 54.0 in stage 8.0 (TID 61) in 11 ms on localhost (55/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 55.0 in stage 8.0 (TID 62). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 56.0 in stage 8.0 (TID 63, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 56.0 in stage 8.0 (TID 63) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 55.0 in stage 8.0 (TID 62) in 10 ms on localhost (56/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 56.0 in stage 8.0 (TID 63). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 57.0 in stage 8.0 (TID 64, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 57.0 in stage 8.0 (TID 64) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 56.0 in stage 8.0 (TID 63) in 12 ms on localhost (57/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 57.0 in stage 8.0 (TID 64). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 58.0 in stage 8.0 (TID 65, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 57.0 in stage 8.0 (TID 64) in 11 ms on localhost (58/199) -2016-06-13 13:39:35 INFO Executor:59 - Running task 58.0 in stage 8.0 (TID 65) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 58.0 in stage 8.0 (TID 65). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 59.0 in stage 8.0 (TID 66, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 59.0 in stage 8.0 (TID 66) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 58.0 in stage 8.0 (TID 65) in 12 ms on localhost (59/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 59.0 in stage 8.0 (TID 66). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 60.0 in stage 8.0 (TID 67, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 59.0 in stage 8.0 (TID 66) in 12 ms on localhost (60/199) -2016-06-13 13:39:35 INFO Executor:59 - Running task 60.0 in stage 8.0 (TID 67) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 60.0 in stage 8.0 (TID 67). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 61.0 in stage 8.0 (TID 68, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 61.0 in stage 8.0 (TID 68) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 60.0 in stage 8.0 (TID 67) in 14 ms on localhost (61/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 61.0 in stage 8.0 (TID 68). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 62.0 in stage 8.0 (TID 69, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 62.0 in stage 8.0 (TID 69) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 61.0 in stage 8.0 (TID 68) in 14 ms on localhost (62/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 62.0 in stage 8.0 (TID 69). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 63.0 in stage 8.0 (TID 70, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 63.0 in stage 8.0 (TID 70) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 62.0 in stage 8.0 (TID 69) in 16 ms on localhost (63/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 63.0 in stage 8.0 (TID 70). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 64.0 in stage 8.0 (TID 71, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 64.0 in stage 8.0 (TID 71) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 63.0 in stage 8.0 (TID 70) in 13 ms on localhost (64/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 64.0 in stage 8.0 (TID 71). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 65.0 in stage 8.0 (TID 72, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 65.0 in stage 8.0 (TID 72) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 64.0 in stage 8.0 (TID 71) in 10 ms on localhost (65/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 65.0 in stage 8.0 (TID 72). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 66.0 in stage 8.0 (TID 73, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 66.0 in stage 8.0 (TID 73) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 65.0 in stage 8.0 (TID 72) in 13 ms on localhost (66/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 66.0 in stage 8.0 (TID 73). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 67.0 in stage 8.0 (TID 74, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 67.0 in stage 8.0 (TID 74) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 66.0 in stage 8.0 (TID 73) in 14 ms on localhost (67/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 67.0 in stage 8.0 (TID 74). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 68.0 in stage 8.0 (TID 75, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 68.0 in stage 8.0 (TID 75) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 67.0 in stage 8.0 (TID 74) in 13 ms on localhost (68/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 68.0 in stage 8.0 (TID 75). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 69.0 in stage 8.0 (TID 76, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 69.0 in stage 8.0 (TID 76) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 68.0 in stage 8.0 (TID 75) in 10 ms on localhost (69/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 69.0 in stage 8.0 (TID 76). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 70.0 in stage 8.0 (TID 77, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 70.0 in stage 8.0 (TID 77) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 69.0 in stage 8.0 (TID 76) in 12 ms on localhost (70/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 70.0 in stage 8.0 (TID 77). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 71.0 in stage 8.0 (TID 78, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 71.0 in stage 8.0 (TID 78) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 70.0 in stage 8.0 (TID 77) in 12 ms on localhost (71/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 71.0 in stage 8.0 (TID 78). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 72.0 in stage 8.0 (TID 79, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 72.0 in stage 8.0 (TID 79) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 71.0 in stage 8.0 (TID 78) in 13 ms on localhost (72/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 72.0 in stage 8.0 (TID 79). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 73.0 in stage 8.0 (TID 80, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 73.0 in stage 8.0 (TID 80) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 72.0 in stage 8.0 (TID 79) in 13 ms on localhost (73/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 73.0 in stage 8.0 (TID 80). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 74.0 in stage 8.0 (TID 81, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 74.0 in stage 8.0 (TID 81) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 73.0 in stage 8.0 (TID 80) in 15 ms on localhost (74/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 74.0 in stage 8.0 (TID 81). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 75.0 in stage 8.0 (TID 82, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 75.0 in stage 8.0 (TID 82) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 74.0 in stage 8.0 (TID 81) in 13 ms on localhost (75/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 75.0 in stage 8.0 (TID 82). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 76.0 in stage 8.0 (TID 83, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 76.0 in stage 8.0 (TID 83) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 75.0 in stage 8.0 (TID 82) in 10 ms on localhost (76/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 76.0 in stage 8.0 (TID 83). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 77.0 in stage 8.0 (TID 84, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 77.0 in stage 8.0 (TID 84) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 76.0 in stage 8.0 (TID 83) in 11 ms on localhost (77/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 77.0 in stage 8.0 (TID 84). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 78.0 in stage 8.0 (TID 85, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 78.0 in stage 8.0 (TID 85) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 77.0 in stage 8.0 (TID 84) in 14 ms on localhost (78/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 78.0 in stage 8.0 (TID 85). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 79.0 in stage 8.0 (TID 86, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 79.0 in stage 8.0 (TID 86) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 78.0 in stage 8.0 (TID 85) in 12 ms on localhost (79/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 79.0 in stage 8.0 (TID 86). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 80.0 in stage 8.0 (TID 87, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 80.0 in stage 8.0 (TID 87) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 79.0 in stage 8.0 (TID 86) in 13 ms on localhost (80/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 80.0 in stage 8.0 (TID 87). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 81.0 in stage 8.0 (TID 88, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 81.0 in stage 8.0 (TID 88) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 80.0 in stage 8.0 (TID 87) in 11 ms on localhost (81/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 81.0 in stage 8.0 (TID 88). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 82.0 in stage 8.0 (TID 89, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 82.0 in stage 8.0 (TID 89) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 81.0 in stage 8.0 (TID 88) in 16 ms on localhost (82/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 82.0 in stage 8.0 (TID 89). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 83.0 in stage 8.0 (TID 90, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 83.0 in stage 8.0 (TID 90) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 82.0 in stage 8.0 (TID 89) in 14 ms on localhost (83/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 83.0 in stage 8.0 (TID 90). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 84.0 in stage 8.0 (TID 91, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 84.0 in stage 8.0 (TID 91) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 83.0 in stage 8.0 (TID 90) in 13 ms on localhost (84/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 84.0 in stage 8.0 (TID 91). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 85.0 in stage 8.0 (TID 92, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 85.0 in stage 8.0 (TID 92) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 84.0 in stage 8.0 (TID 91) in 10 ms on localhost (85/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 85.0 in stage 8.0 (TID 92). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 86.0 in stage 8.0 (TID 93, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 86.0 in stage 8.0 (TID 93) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 85.0 in stage 8.0 (TID 92) in 11 ms on localhost (86/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 86.0 in stage 8.0 (TID 93). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 87.0 in stage 8.0 (TID 94, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 87.0 in stage 8.0 (TID 94) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 86.0 in stage 8.0 (TID 93) in 11 ms on localhost (87/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 87.0 in stage 8.0 (TID 94). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 88.0 in stage 8.0 (TID 95, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 88.0 in stage 8.0 (TID 95) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 87.0 in stage 8.0 (TID 94) in 11 ms on localhost (88/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 88.0 in stage 8.0 (TID 95). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 89.0 in stage 8.0 (TID 96, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 89.0 in stage 8.0 (TID 96) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 88.0 in stage 8.0 (TID 95) in 19 ms on localhost (89/199) -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:35 INFO Executor:59 - Finished task 89.0 in stage 8.0 (TID 96). 1437 bytes result sent to driver -2016-06-13 13:39:35 INFO TaskSetManager:59 - Starting task 90.0 in stage 8.0 (TID 97, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:35 INFO Executor:59 - Running task 90.0 in stage 8.0 (TID 97) -2016-06-13 13:39:35 INFO TaskSetManager:59 - Finished task 89.0 in stage 8.0 (TID 96) in 17 ms on localhost (90/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 90.0 in stage 8.0 (TID 97). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 91.0 in stage 8.0 (TID 98, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 91.0 in stage 8.0 (TID 98) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 90.0 in stage 8.0 (TID 97) in 12 ms on localhost (91/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 91.0 in stage 8.0 (TID 98). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 92.0 in stage 8.0 (TID 99, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 92.0 in stage 8.0 (TID 99) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 91.0 in stage 8.0 (TID 98) in 13 ms on localhost (92/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 92.0 in stage 8.0 (TID 99). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 93.0 in stage 8.0 (TID 100, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 93.0 in stage 8.0 (TID 100) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 92.0 in stage 8.0 (TID 99) in 9 ms on localhost (93/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 93.0 in stage 8.0 (TID 100). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 94.0 in stage 8.0 (TID 101, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 94.0 in stage 8.0 (TID 101) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 93.0 in stage 8.0 (TID 100) in 9 ms on localhost (94/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 94.0 in stage 8.0 (TID 101). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 95.0 in stage 8.0 (TID 102, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 95.0 in stage 8.0 (TID 102) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 94.0 in stage 8.0 (TID 101) in 11 ms on localhost (95/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 95.0 in stage 8.0 (TID 102). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 96.0 in stage 8.0 (TID 103, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 96.0 in stage 8.0 (TID 103) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 95.0 in stage 8.0 (TID 102) in 9 ms on localhost (96/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 96.0 in stage 8.0 (TID 103). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 97.0 in stage 8.0 (TID 104, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 97.0 in stage 8.0 (TID 104) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 96.0 in stage 8.0 (TID 103) in 9 ms on localhost (97/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 97.0 in stage 8.0 (TID 104). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 98.0 in stage 8.0 (TID 105, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 98.0 in stage 8.0 (TID 105) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 97.0 in stage 8.0 (TID 104) in 10 ms on localhost (98/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 98.0 in stage 8.0 (TID 105). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 99.0 in stage 8.0 (TID 106, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 99.0 in stage 8.0 (TID 106) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 98.0 in stage 8.0 (TID 105) in 12 ms on localhost (99/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 99.0 in stage 8.0 (TID 106). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 100.0 in stage 8.0 (TID 107, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 100.0 in stage 8.0 (TID 107) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 99.0 in stage 8.0 (TID 106) in 13 ms on localhost (100/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 100.0 in stage 8.0 (TID 107). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 101.0 in stage 8.0 (TID 108, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 101.0 in stage 8.0 (TID 108) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 100.0 in stage 8.0 (TID 107) in 13 ms on localhost (101/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 101.0 in stage 8.0 (TID 108). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 102.0 in stage 8.0 (TID 109, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 101.0 in stage 8.0 (TID 108) in 61 ms on localhost (102/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 102.0 in stage 8.0 (TID 109) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 102.0 in stage 8.0 (TID 109). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 103.0 in stage 8.0 (TID 110, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 103.0 in stage 8.0 (TID 110) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 103.0 in stage 8.0 (TID 110). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 102.0 in stage 8.0 (TID 109) in 20 ms on localhost (103/199) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 104.0 in stage 8.0 (TID 111, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 103.0 in stage 8.0 (TID 110) in 14 ms on localhost (104/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 104.0 in stage 8.0 (TID 111) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 104.0 in stage 8.0 (TID 111). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 105.0 in stage 8.0 (TID 112, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 104.0 in stage 8.0 (TID 111) in 10 ms on localhost (105/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 105.0 in stage 8.0 (TID 112) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 105.0 in stage 8.0 (TID 112). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 106.0 in stage 8.0 (TID 113, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 105.0 in stage 8.0 (TID 112) in 10 ms on localhost (106/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 106.0 in stage 8.0 (TID 113) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 106.0 in stage 8.0 (TID 113). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 107.0 in stage 8.0 (TID 114, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 106.0 in stage 8.0 (TID 113) in 14 ms on localhost (107/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 107.0 in stage 8.0 (TID 114) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 107.0 in stage 8.0 (TID 114). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 108.0 in stage 8.0 (TID 115, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 108.0 in stage 8.0 (TID 115) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 107.0 in stage 8.0 (TID 114) in 11 ms on localhost (108/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 108.0 in stage 8.0 (TID 115). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 109.0 in stage 8.0 (TID 116, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 108.0 in stage 8.0 (TID 115) in 13 ms on localhost (109/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 109.0 in stage 8.0 (TID 116) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 109.0 in stage 8.0 (TID 116). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 110.0 in stage 8.0 (TID 117, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 110.0 in stage 8.0 (TID 117) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 109.0 in stage 8.0 (TID 116) in 13 ms on localhost (110/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 110.0 in stage 8.0 (TID 117). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 111.0 in stage 8.0 (TID 118, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 111.0 in stage 8.0 (TID 118) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 110.0 in stage 8.0 (TID 117) in 9 ms on localhost (111/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 111.0 in stage 8.0 (TID 118). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 112.0 in stage 8.0 (TID 119, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 112.0 in stage 8.0 (TID 119) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 111.0 in stage 8.0 (TID 118) in 9 ms on localhost (112/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 112.0 in stage 8.0 (TID 119). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 113.0 in stage 8.0 (TID 120, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 113.0 in stage 8.0 (TID 120) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 112.0 in stage 8.0 (TID 119) in 10 ms on localhost (113/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 113.0 in stage 8.0 (TID 120). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 114.0 in stage 8.0 (TID 121, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 114.0 in stage 8.0 (TID 121) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 113.0 in stage 8.0 (TID 120) in 9 ms on localhost (114/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 114.0 in stage 8.0 (TID 121). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 115.0 in stage 8.0 (TID 122, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 115.0 in stage 8.0 (TID 122) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 114.0 in stage 8.0 (TID 121) in 10 ms on localhost (115/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 115.0 in stage 8.0 (TID 122). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 116.0 in stage 8.0 (TID 123, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 116.0 in stage 8.0 (TID 123) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 115.0 in stage 8.0 (TID 122) in 10 ms on localhost (116/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 116.0 in stage 8.0 (TID 123). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 117.0 in stage 8.0 (TID 124, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 117.0 in stage 8.0 (TID 124) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 116.0 in stage 8.0 (TID 123) in 8 ms on localhost (117/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 117.0 in stage 8.0 (TID 124). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 118.0 in stage 8.0 (TID 125, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 118.0 in stage 8.0 (TID 125) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 117.0 in stage 8.0 (TID 124) in 8 ms on localhost (118/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 118.0 in stage 8.0 (TID 125). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 119.0 in stage 8.0 (TID 126, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 119.0 in stage 8.0 (TID 126) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 118.0 in stage 8.0 (TID 125) in 8 ms on localhost (119/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 119.0 in stage 8.0 (TID 126). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 120.0 in stage 8.0 (TID 127, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 120.0 in stage 8.0 (TID 127) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 119.0 in stage 8.0 (TID 126) in 8 ms on localhost (120/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 120.0 in stage 8.0 (TID 127). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 121.0 in stage 8.0 (TID 128, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 120.0 in stage 8.0 (TID 127) in 8 ms on localhost (121/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 121.0 in stage 8.0 (TID 128) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 121.0 in stage 8.0 (TID 128). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 122.0 in stage 8.0 (TID 129, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 121.0 in stage 8.0 (TID 128) in 8 ms on localhost (122/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 122.0 in stage 8.0 (TID 129) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 122.0 in stage 8.0 (TID 129). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 123.0 in stage 8.0 (TID 130, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 123.0 in stage 8.0 (TID 130) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 122.0 in stage 8.0 (TID 129) in 8 ms on localhost (123/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 123.0 in stage 8.0 (TID 130). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 124.0 in stage 8.0 (TID 131, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 123.0 in stage 8.0 (TID 130) in 8 ms on localhost (124/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 124.0 in stage 8.0 (TID 131) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 124.0 in stage 8.0 (TID 131). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 125.0 in stage 8.0 (TID 132, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 125.0 in stage 8.0 (TID 132) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 124.0 in stage 8.0 (TID 131) in 10 ms on localhost (125/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 125.0 in stage 8.0 (TID 132). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 126.0 in stage 8.0 (TID 133, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 126.0 in stage 8.0 (TID 133) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 125.0 in stage 8.0 (TID 132) in 9 ms on localhost (126/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 126.0 in stage 8.0 (TID 133). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 127.0 in stage 8.0 (TID 134, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 126.0 in stage 8.0 (TID 133) in 9 ms on localhost (127/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 127.0 in stage 8.0 (TID 134) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 127.0 in stage 8.0 (TID 134). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 128.0 in stage 8.0 (TID 135, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 127.0 in stage 8.0 (TID 134) in 8 ms on localhost (128/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 128.0 in stage 8.0 (TID 135) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 128.0 in stage 8.0 (TID 135). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 129.0 in stage 8.0 (TID 136, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 129.0 in stage 8.0 (TID 136) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 128.0 in stage 8.0 (TID 135) in 8 ms on localhost (129/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 129.0 in stage 8.0 (TID 136). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 130.0 in stage 8.0 (TID 137, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 129.0 in stage 8.0 (TID 136) in 8 ms on localhost (130/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 130.0 in stage 8.0 (TID 137) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 130.0 in stage 8.0 (TID 137). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 131.0 in stage 8.0 (TID 138, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 131.0 in stage 8.0 (TID 138) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 130.0 in stage 8.0 (TID 137) in 8 ms on localhost (131/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 131.0 in stage 8.0 (TID 138). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 132.0 in stage 8.0 (TID 139, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 132.0 in stage 8.0 (TID 139) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 131.0 in stage 8.0 (TID 138) in 11 ms on localhost (132/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 132.0 in stage 8.0 (TID 139). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 133.0 in stage 8.0 (TID 140, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 133.0 in stage 8.0 (TID 140) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 132.0 in stage 8.0 (TID 139) in 8 ms on localhost (133/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 133.0 in stage 8.0 (TID 140). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 134.0 in stage 8.0 (TID 141, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 134.0 in stage 8.0 (TID 141) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 133.0 in stage 8.0 (TID 140) in 8 ms on localhost (134/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 134.0 in stage 8.0 (TID 141). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 135.0 in stage 8.0 (TID 142, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 134.0 in stage 8.0 (TID 141) in 8 ms on localhost (135/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 135.0 in stage 8.0 (TID 142) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 135.0 in stage 8.0 (TID 142). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 136.0 in stage 8.0 (TID 143, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 136.0 in stage 8.0 (TID 143) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 135.0 in stage 8.0 (TID 142) in 11 ms on localhost (136/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 136.0 in stage 8.0 (TID 143). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 137.0 in stage 8.0 (TID 144, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 136.0 in stage 8.0 (TID 143) in 8 ms on localhost (137/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 137.0 in stage 8.0 (TID 144) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 137.0 in stage 8.0 (TID 144). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 138.0 in stage 8.0 (TID 145, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 138.0 in stage 8.0 (TID 145) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 137.0 in stage 8.0 (TID 144) in 11 ms on localhost (138/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 138.0 in stage 8.0 (TID 145). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 139.0 in stage 8.0 (TID 146, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 139.0 in stage 8.0 (TID 146) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 138.0 in stage 8.0 (TID 145) in 8 ms on localhost (139/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 139.0 in stage 8.0 (TID 146). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 140.0 in stage 8.0 (TID 147, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 140.0 in stage 8.0 (TID 147) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 139.0 in stage 8.0 (TID 146) in 9 ms on localhost (140/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 140.0 in stage 8.0 (TID 147). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 141.0 in stage 8.0 (TID 148, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 141.0 in stage 8.0 (TID 148) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 140.0 in stage 8.0 (TID 147) in 8 ms on localhost (141/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 141.0 in stage 8.0 (TID 148). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 142.0 in stage 8.0 (TID 149, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 142.0 in stage 8.0 (TID 149) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 141.0 in stage 8.0 (TID 148) in 8 ms on localhost (142/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 142.0 in stage 8.0 (TID 149). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 143.0 in stage 8.0 (TID 150, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 143.0 in stage 8.0 (TID 150) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 142.0 in stage 8.0 (TID 149) in 8 ms on localhost (143/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 143.0 in stage 8.0 (TID 150). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 144.0 in stage 8.0 (TID 151, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 143.0 in stage 8.0 (TID 150) in 8 ms on localhost (144/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 144.0 in stage 8.0 (TID 151) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 144.0 in stage 8.0 (TID 151). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 145.0 in stage 8.0 (TID 152, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 145.0 in stage 8.0 (TID 152) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 144.0 in stage 8.0 (TID 151) in 9 ms on localhost (145/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 145.0 in stage 8.0 (TID 152). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 146.0 in stage 8.0 (TID 153, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 146.0 in stage 8.0 (TID 153) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 145.0 in stage 8.0 (TID 152) in 9 ms on localhost (146/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 146.0 in stage 8.0 (TID 153). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 147.0 in stage 8.0 (TID 154, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 146.0 in stage 8.0 (TID 153) in 8 ms on localhost (147/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 147.0 in stage 8.0 (TID 154) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 147.0 in stage 8.0 (TID 154). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 148.0 in stage 8.0 (TID 155, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 147.0 in stage 8.0 (TID 154) in 9 ms on localhost (148/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 148.0 in stage 8.0 (TID 155) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 148.0 in stage 8.0 (TID 155). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 149.0 in stage 8.0 (TID 156, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 149.0 in stage 8.0 (TID 156) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 148.0 in stage 8.0 (TID 155) in 9 ms on localhost (149/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 149.0 in stage 8.0 (TID 156). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 150.0 in stage 8.0 (TID 157, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 150.0 in stage 8.0 (TID 157) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 149.0 in stage 8.0 (TID 156) in 10 ms on localhost (150/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 150.0 in stage 8.0 (TID 157). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 151.0 in stage 8.0 (TID 158, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 150.0 in stage 8.0 (TID 157) in 8 ms on localhost (151/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 151.0 in stage 8.0 (TID 158) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 151.0 in stage 8.0 (TID 158). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 152.0 in stage 8.0 (TID 159, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 152.0 in stage 8.0 (TID 159) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 151.0 in stage 8.0 (TID 158) in 9 ms on localhost (152/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 152.0 in stage 8.0 (TID 159). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 153.0 in stage 8.0 (TID 160, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 152.0 in stage 8.0 (TID 159) in 8 ms on localhost (153/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 153.0 in stage 8.0 (TID 160) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 153.0 in stage 8.0 (TID 160). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 154.0 in stage 8.0 (TID 161, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 154.0 in stage 8.0 (TID 161) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 153.0 in stage 8.0 (TID 160) in 7 ms on localhost (154/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 154.0 in stage 8.0 (TID 161). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 155.0 in stage 8.0 (TID 162, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 155.0 in stage 8.0 (TID 162) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 154.0 in stage 8.0 (TID 161) in 9 ms on localhost (155/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 155.0 in stage 8.0 (TID 162). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 156.0 in stage 8.0 (TID 163, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 156.0 in stage 8.0 (TID 163) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 155.0 in stage 8.0 (TID 162) in 8 ms on localhost (156/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 156.0 in stage 8.0 (TID 163). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 157.0 in stage 8.0 (TID 164, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 157.0 in stage 8.0 (TID 164) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 156.0 in stage 8.0 (TID 163) in 8 ms on localhost (157/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 157.0 in stage 8.0 (TID 164). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 158.0 in stage 8.0 (TID 165, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 158.0 in stage 8.0 (TID 165) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 157.0 in stage 8.0 (TID 164) in 8 ms on localhost (158/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 158.0 in stage 8.0 (TID 165). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 159.0 in stage 8.0 (TID 166, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 158.0 in stage 8.0 (TID 165) in 8 ms on localhost (159/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 159.0 in stage 8.0 (TID 166) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 159.0 in stage 8.0 (TID 166). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 160.0 in stage 8.0 (TID 167, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 159.0 in stage 8.0 (TID 166) in 11 ms on localhost (160/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 160.0 in stage 8.0 (TID 167) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 160.0 in stage 8.0 (TID 167). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 161.0 in stage 8.0 (TID 168, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 161.0 in stage 8.0 (TID 168) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 160.0 in stage 8.0 (TID 167) in 9 ms on localhost (161/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 161.0 in stage 8.0 (TID 168). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 162.0 in stage 8.0 (TID 169, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 161.0 in stage 8.0 (TID 168) in 8 ms on localhost (162/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 162.0 in stage 8.0 (TID 169) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 162.0 in stage 8.0 (TID 169). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 163.0 in stage 8.0 (TID 170, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 162.0 in stage 8.0 (TID 169) in 9 ms on localhost (163/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 163.0 in stage 8.0 (TID 170) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 163.0 in stage 8.0 (TID 170). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 164.0 in stage 8.0 (TID 171, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 163.0 in stage 8.0 (TID 170) in 9 ms on localhost (164/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 164.0 in stage 8.0 (TID 171) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 164.0 in stage 8.0 (TID 171). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 165.0 in stage 8.0 (TID 172, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 165.0 in stage 8.0 (TID 172) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 164.0 in stage 8.0 (TID 171) in 9 ms on localhost (165/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 165.0 in stage 8.0 (TID 172). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 166.0 in stage 8.0 (TID 173, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 165.0 in stage 8.0 (TID 172) in 8 ms on localhost (166/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 166.0 in stage 8.0 (TID 173) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 166.0 in stage 8.0 (TID 173). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 167.0 in stage 8.0 (TID 174, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 166.0 in stage 8.0 (TID 173) in 8 ms on localhost (167/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 167.0 in stage 8.0 (TID 174) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 167.0 in stage 8.0 (TID 174). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 168.0 in stage 8.0 (TID 175, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 168.0 in stage 8.0 (TID 175) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 167.0 in stage 8.0 (TID 174) in 8 ms on localhost (168/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 168.0 in stage 8.0 (TID 175). 1723 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 169.0 in stage 8.0 (TID 176, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 169.0 in stage 8.0 (TID 176) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 168.0 in stage 8.0 (TID 175) in 8 ms on localhost (169/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 169.0 in stage 8.0 (TID 176). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 170.0 in stage 8.0 (TID 177, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 169.0 in stage 8.0 (TID 176) in 7 ms on localhost (170/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 170.0 in stage 8.0 (TID 177) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 170.0 in stage 8.0 (TID 177). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 171.0 in stage 8.0 (TID 178, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 171.0 in stage 8.0 (TID 178) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 170.0 in stage 8.0 (TID 177) in 8 ms on localhost (171/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 171.0 in stage 8.0 (TID 178). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 172.0 in stage 8.0 (TID 179, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 172.0 in stage 8.0 (TID 179) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 171.0 in stage 8.0 (TID 178) in 8 ms on localhost (172/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 172.0 in stage 8.0 (TID 179). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 173.0 in stage 8.0 (TID 180, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 172.0 in stage 8.0 (TID 179) in 8 ms on localhost (173/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 173.0 in stage 8.0 (TID 180) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 173.0 in stage 8.0 (TID 180). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 174.0 in stage 8.0 (TID 181, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 174.0 in stage 8.0 (TID 181) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 173.0 in stage 8.0 (TID 180) in 8 ms on localhost (174/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 174.0 in stage 8.0 (TID 181). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 175.0 in stage 8.0 (TID 182, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 175.0 in stage 8.0 (TID 182) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 174.0 in stage 8.0 (TID 181) in 9 ms on localhost (175/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 175.0 in stage 8.0 (TID 182). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 176.0 in stage 8.0 (TID 183, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 175.0 in stage 8.0 (TID 182) in 8 ms on localhost (176/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 176.0 in stage 8.0 (TID 183) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 176.0 in stage 8.0 (TID 183). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 177.0 in stage 8.0 (TID 184, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 177.0 in stage 8.0 (TID 184) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 176.0 in stage 8.0 (TID 183) in 113 ms on localhost (177/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 177.0 in stage 8.0 (TID 184). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 178.0 in stage 8.0 (TID 185, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 177.0 in stage 8.0 (TID 184) in 126 ms on localhost (178/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 178.0 in stage 8.0 (TID 185) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 178.0 in stage 8.0 (TID 185). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 179.0 in stage 8.0 (TID 186, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 178.0 in stage 8.0 (TID 185) in 13 ms on localhost (179/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 179.0 in stage 8.0 (TID 186) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 179.0 in stage 8.0 (TID 186). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 180.0 in stage 8.0 (TID 187, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 180.0 in stage 8.0 (TID 187) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 179.0 in stage 8.0 (TID 186) in 18 ms on localhost (180/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 180.0 in stage 8.0 (TID 187). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 181.0 in stage 8.0 (TID 188, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 181.0 in stage 8.0 (TID 188) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 180.0 in stage 8.0 (TID 187) in 9 ms on localhost (181/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 181.0 in stage 8.0 (TID 188). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 182.0 in stage 8.0 (TID 189, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 182.0 in stage 8.0 (TID 189) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 181.0 in stage 8.0 (TID 188) in 10 ms on localhost (182/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 182.0 in stage 8.0 (TID 189). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 183.0 in stage 8.0 (TID 190, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 183.0 in stage 8.0 (TID 190) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 182.0 in stage 8.0 (TID 189) in 9 ms on localhost (183/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 183.0 in stage 8.0 (TID 190). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 184.0 in stage 8.0 (TID 191, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 183.0 in stage 8.0 (TID 190) in 13 ms on localhost (184/199) -2016-06-13 13:39:36 INFO Executor:59 - Running task 184.0 in stage 8.0 (TID 191) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 184.0 in stage 8.0 (TID 191). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 185.0 in stage 8.0 (TID 192, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 185.0 in stage 8.0 (TID 192) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 184.0 in stage 8.0 (TID 191) in 9 ms on localhost (185/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 185.0 in stage 8.0 (TID 192). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 186.0 in stage 8.0 (TID 193, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 186.0 in stage 8.0 (TID 193) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 185.0 in stage 8.0 (TID 192) in 9 ms on localhost (186/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 186.0 in stage 8.0 (TID 193). 1437 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 187.0 in stage 8.0 (TID 194, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 187.0 in stage 8.0 (TID 194) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 186.0 in stage 8.0 (TID 193) in 9 ms on localhost (187/199) -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:36 INFO Executor:59 - Finished task 187.0 in stage 8.0 (TID 194). 1736 bytes result sent to driver -2016-06-13 13:39:36 INFO TaskSetManager:59 - Starting task 188.0 in stage 8.0 (TID 195, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:36 INFO Executor:59 - Running task 188.0 in stage 8.0 (TID 195) -2016-06-13 13:39:36 INFO TaskSetManager:59 - Finished task 187.0 in stage 8.0 (TID 194) in 8 ms on localhost (188/199) -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:37 INFO Executor:59 - Finished task 188.0 in stage 8.0 (TID 195). 1437 bytes result sent to driver -2016-06-13 13:39:37 INFO TaskSetManager:59 - Starting task 189.0 in stage 8.0 (TID 196, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:37 INFO Executor:59 - Running task 189.0 in stage 8.0 (TID 196) -2016-06-13 13:39:37 INFO TaskSetManager:59 - Finished task 188.0 in stage 8.0 (TID 195) in 8 ms on localhost (189/199) -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:37 INFO Executor:59 - Finished task 189.0 in stage 8.0 (TID 196). 1437 bytes result sent to driver -2016-06-13 13:39:37 INFO TaskSetManager:59 - Starting task 190.0 in stage 8.0 (TID 197, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:37 INFO TaskSetManager:59 - Finished task 189.0 in stage 8.0 (TID 196) in 8 ms on localhost (190/199) -2016-06-13 13:39:37 INFO Executor:59 - Running task 190.0 in stage 8.0 (TID 197) -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:37 INFO Executor:59 - Finished task 190.0 in stage 8.0 (TID 197). 1437 bytes result sent to driver -2016-06-13 13:39:37 INFO TaskSetManager:59 - Starting task 191.0 in stage 8.0 (TID 198, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:37 INFO TaskSetManager:59 - Finished task 190.0 in stage 8.0 (TID 197) in 9 ms on localhost (191/199) -2016-06-13 13:39:37 INFO Executor:59 - Running task 191.0 in stage 8.0 (TID 198) -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:37 INFO Executor:59 - Finished task 191.0 in stage 8.0 (TID 198). 1437 bytes result sent to driver -2016-06-13 13:39:37 INFO TaskSetManager:59 - Starting task 192.0 in stage 8.0 (TID 199, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:37 INFO Executor:59 - Running task 192.0 in stage 8.0 (TID 199) -2016-06-13 13:39:37 INFO TaskSetManager:59 - Finished task 191.0 in stage 8.0 (TID 198) in 11 ms on localhost (192/199) -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:37 INFO Executor:59 - Finished task 192.0 in stage 8.0 (TID 199). 1437 bytes result sent to driver -2016-06-13 13:39:37 INFO TaskSetManager:59 - Starting task 193.0 in stage 8.0 (TID 200, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:37 INFO TaskSetManager:59 - Finished task 192.0 in stage 8.0 (TID 199) in 8 ms on localhost (193/199) -2016-06-13 13:39:37 INFO Executor:59 - Running task 193.0 in stage 8.0 (TID 200) -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:37 INFO Executor:59 - Finished task 193.0 in stage 8.0 (TID 200). 1437 bytes result sent to driver -2016-06-13 13:39:37 INFO TaskSetManager:59 - Starting task 194.0 in stage 8.0 (TID 201, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:37 INFO Executor:59 - Running task 194.0 in stage 8.0 (TID 201) -2016-06-13 13:39:37 INFO TaskSetManager:59 - Finished task 193.0 in stage 8.0 (TID 200) in 8 ms on localhost (194/199) -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:37 INFO Executor:59 - Finished task 194.0 in stage 8.0 (TID 201). 1437 bytes result sent to driver -2016-06-13 13:39:37 INFO TaskSetManager:59 - Starting task 195.0 in stage 8.0 (TID 202, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:37 INFO Executor:59 - Running task 195.0 in stage 8.0 (TID 202) -2016-06-13 13:39:37 INFO TaskSetManager:59 - Finished task 194.0 in stage 8.0 (TID 201) in 8 ms on localhost (195/199) -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:37 INFO Executor:59 - Finished task 195.0 in stage 8.0 (TID 202). 1437 bytes result sent to driver -2016-06-13 13:39:37 INFO TaskSetManager:59 - Starting task 196.0 in stage 8.0 (TID 203, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:37 INFO TaskSetManager:59 - Finished task 195.0 in stage 8.0 (TID 202) in 8 ms on localhost (196/199) -2016-06-13 13:39:37 INFO Executor:59 - Running task 196.0 in stage 8.0 (TID 203) -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 13:39:37 INFO Executor:59 - Finished task 196.0 in stage 8.0 (TID 203). 1437 bytes result sent to driver -2016-06-13 13:39:37 INFO TaskSetManager:59 - Starting task 197.0 in stage 8.0 (TID 204, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:37 INFO TaskSetManager:59 - Finished task 196.0 in stage 8.0 (TID 203) in 8 ms on localhost (197/199) -2016-06-13 13:39:37 INFO Executor:59 - Running task 197.0 in stage 8.0 (TID 204) -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:37 INFO Executor:59 - Finished task 197.0 in stage 8.0 (TID 204). 1437 bytes result sent to driver -2016-06-13 13:39:37 INFO TaskSetManager:59 - Starting task 198.0 in stage 8.0 (TID 205, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 13:39:37 INFO Executor:59 - Running task 198.0 in stage 8.0 (TID 205) -2016-06-13 13:39:37 INFO TaskSetManager:59 - Finished task 197.0 in stage 8.0 (TID 204) in 14 ms on localhost (198/199) -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 13:39:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 13:39:37 INFO Executor:59 - Finished task 198.0 in stage 8.0 (TID 205). 1736 bytes result sent to driver -2016-06-13 13:39:37 INFO TaskSetManager:59 - Finished task 198.0 in stage 8.0 (TID 205) in 9 ms on localhost (199/199) -2016-06-13 13:39:37 INFO DAGScheduler:59 - ResultStage 8 (main at null:-2) finished in 2.420 s -2016-06-13 13:39:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 13:39:37 INFO DAGScheduler:59 - Job 6 finished: main at null:-2, took 2.538063 s -2016-06-13 13:39:37 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 13:39:37 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 13:39:37 INFO SparkUI:59 - Stopped Spark web UI at http://219.219.220.162:4040 -2016-06-13 13:39:37 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 13:39:37 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 13:39:40 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 13:39:40 INFO BlockManager:59 - BlockManager stopped -2016-06-13 13:39:40 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 13:39:40 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 13:39:40 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 13:39:40 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 13:39:40 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-e9b3bc5c-60ec-4b92-ac17-03f7480e7e3c -2016-06-13 14:02:49 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 14:02:50 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 14:02:50 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 14:02:50 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 14:02:50 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 14:02:51 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 14:02:51 INFO Remoting:74 - Starting remoting -2016-06-13 14:02:51 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@219.219.220.162:55571] -2016-06-13 14:02:51 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55571. -2016-06-13 14:02:51 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 14:02:51 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 14:02:51 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-eb0bda55-86e5-43e6-868a-0e63b6a0978b -2016-06-13 14:02:51 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 14:02:51 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-1de259ae-f8e5-414c-8eec-7cc9b122a7b3\httpd-5e4d31e0-443f-49be-986f-f278bcfabb2e -2016-06-13 14:02:51 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 14:02:51 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 14:02:51 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55572 -2016-06-13 14:02:51 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55572. -2016-06-13 14:02:51 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 14:02:52 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 14:02:52 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 14:02:52 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 14:02:52 INFO SparkUI:59 - Started SparkUI at http://219.219.220.162:4040 -2016-06-13 14:02:52 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 14:02:52 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 14:02:52 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55591. -2016-06-13 14:02:52 INFO NettyBlockTransferService:59 - Server created on 55591 -2016-06-13 14:02:52 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 14:02:52 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55591 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55591) -2016-06-13 14:02:52 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 14:02:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 14:02:52 INFO SparkUI:59 - Stopped Spark web UI at http://219.219.220.162:4040 -2016-06-13 14:02:52 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 14:02:52 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 14:02:52 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 14:02:52 INFO BlockManager:59 - BlockManager stopped -2016-06-13 14:02:52 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 14:02:52 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 14:02:52 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 14:02:52 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 14:02:52 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-1de259ae-f8e5-414c-8eec-7cc9b122a7b3 -2016-06-13 14:02:52 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 14:02:52 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 14:03:03 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 14:03:04 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 14:03:04 INFO Remoting:74 - Starting remoting -2016-06-13 14:03:04 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@219.219.220.162:55611] -2016-06-13 14:03:04 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 14:03:04 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55612 -2016-06-13 14:03:05 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 14:03:05 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 14:03:08 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: 2002:dbdb:dca2:0:0:0:dbdb:dca2, but we couldn't find any external IP address! -2016-06-13 14:03:08 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 14:03:08 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 14:03:08 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 14:03:08 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 14:03:08 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 14:03:08 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 14:03:30 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 14:03:30 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 14:03:41 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201606131403_0079_m_000000_79' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/LinearRegression/20160613140336856/result/_temporary/0/task_201606131403_0079_m_000000 -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 14:03:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 14:03:43 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 14:03:43 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 14:27:22 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 14:27:28 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 14:27:29 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 14:27:29 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 14:27:29 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 14:27:31 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 14:27:32 INFO Remoting:74 - Starting remoting -2016-06-13 14:27:32 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-8] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 14:27:32 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 14:27:32 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 14:27:32 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 14:27:32 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 14:27:42 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.util.MLlibTestSparkContext$class.beforeAll(MLlibTestSparkContext.scala:21) - at org.apache.spark.mllib.learning.basic.testFunSuite.beforeAll(testFunSuite.scala:8) - at org.scalatest.BeforeAndAfterAll$class.beforeAll(BeforeAndAfterAll.scala:187) - at org.apache.spark.mllib.learning.basic.testFunSuite.beforeAll(testFunSuite.scala:8) - at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:253) - at org.apache.spark.mllib.learning.basic.testFunSuite.run(testFunSuite.scala:8) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 14:27:42 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 14:27:55 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 14:27:56 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 14:27:56 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 14:27:56 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 14:27:56 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 14:27:57 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 14:27:57 INFO Remoting:74 - Starting remoting -2016-06-13 14:27:57 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-5] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 14:27:57 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 14:27:57 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 14:27:57 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 14:27:57 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 14:28:07 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.util.MLlibTestSparkContext$class.beforeAll(MLlibTestSparkContext.scala:21) - at org.apache.spark.mllib.learning.basic.testFunSuite.beforeAll(testFunSuite.scala:8) - at org.scalatest.BeforeAndAfterAll$class.beforeAll(BeforeAndAfterAll.scala:187) - at org.apache.spark.mllib.learning.basic.testFunSuite.beforeAll(testFunSuite.scala:8) - at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:253) - at org.apache.spark.mllib.learning.basic.testFunSuite.run(testFunSuite.scala:8) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 14:28:07 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 14:28:56 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 14:28:57 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 14:28:57 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 14:28:57 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 14:28:57 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 14:28:58 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 14:28:58 INFO Remoting:74 - Starting remoting -2016-06-13 14:28:58 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-5] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 14:28:58 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 14:28:58 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 14:28:58 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 14:28:58 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 14:29:08 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.util.MLlibTestSparkContext$class.beforeAll(MLlibTestSparkContext.scala:21) - at org.apache.spark.mllib.learning.basic.testFunSuite.beforeAll(testFunSuite.scala:8) - at org.scalatest.BeforeAndAfterAll$class.beforeAll(BeforeAndAfterAll.scala:187) - at org.apache.spark.mllib.learning.basic.testFunSuite.beforeAll(testFunSuite.scala:8) - at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:253) - at org.apache.spark.mllib.learning.basic.testFunSuite.run(testFunSuite.scala:8) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 14:29:08 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 14:29:41 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 14:29:42 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 14:29:42 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 14:29:42 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 14:29:42 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 14:29:43 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 14:29:43 INFO Remoting:74 - Starting remoting -2016-06-13 14:29:43 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-8] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 14:29:43 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 14:29:43 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 14:29:43 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 14:29:43 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 14:29:53 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.util.MLlibTestSparkContext$class.beforeAll(MLlibTestSparkContext.scala:21) - at org.apache.spark.mllib.learning.basic.testFunSuite.beforeAll(testFunSuite.scala:13) - at org.scalatest.BeforeAndAfterAll$class.beforeAll(BeforeAndAfterAll.scala:187) - at org.apache.spark.mllib.learning.basic.testFunSuite.beforeAll(testFunSuite.scala:13) - at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:253) - at org.apache.spark.mllib.learning.basic.testFunSuite.run(testFunSuite.scala:13) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 14:29:53 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 14:30:54 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 14:30:55 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 14:30:55 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 14:30:55 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 14:30:55 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 14:30:56 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 14:30:56 INFO Remoting:74 - Starting remoting -2016-06-13 14:30:57 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-6] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 14:30:57 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 14:30:57 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 14:30:57 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 14:30:57 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 14:31:06 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.util.MLlibTestSparkContext$class.beforeAll(MLlibTestSparkContext.scala:21) - at org.apache.spark.mllib.learning.basic.testFunSuite.beforeAll(testFunSuite.scala:13) - at org.scalatest.BeforeAndAfterAll$class.beforeAll(BeforeAndAfterAll.scala:187) - at org.apache.spark.mllib.learning.basic.testFunSuite.beforeAll(testFunSuite.scala:13) - at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:253) - at org.apache.spark.mllib.learning.basic.testFunSuite.run(testFunSuite.scala:13) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 14:31:07 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 14:32:08 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 14:32:09 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 14:32:09 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 14:32:09 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 14:32:09 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 14:32:10 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 14:32:10 INFO Remoting:74 - Starting remoting -2016-06-13 14:32:10 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-5] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 14:32:10 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 14:32:10 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 14:32:10 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 14:32:10 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 14:32:20 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.util.MLlibTestSparkContext$class.beforeAll(MLlibTestSparkContext.scala:21) - at org.apache.spark.mllib.learning.basic.testFunSuite.beforeAll(testFunSuite.scala:8) - at org.scalatest.BeforeAndAfterAll$class.beforeAll(BeforeAndAfterAll.scala:187) - at org.apache.spark.mllib.learning.basic.testFunSuite.beforeAll(testFunSuite.scala:8) - at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:253) - at org.apache.spark.mllib.learning.basic.testFunSuite.run(testFunSuite.scala:8) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 14:32:20 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 14:34:18 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 14:34:19 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 14:34:19 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 14:34:19 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 14:34:19 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 14:34:20 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 14:34:20 INFO Remoting:74 - Starting remoting -2016-06-13 14:34:20 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-5] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 14:34:20 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 14:34:20 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 14:34:20 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 14:34:20 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 14:34:30 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.util.MLlibTestSparkContext$class.beforeAll(MLlibTestSparkContext.scala:21) - at org.apache.spark.mllib.learning.basic.testFunSuite.beforeAll(testFunSuite.scala:8) - at org.scalatest.BeforeAndAfterAll$class.beforeAll(BeforeAndAfterAll.scala:187) - at org.apache.spark.mllib.learning.basic.testFunSuite.beforeAll(testFunSuite.scala:8) - at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:253) - at org.apache.spark.mllib.learning.basic.testFunSuite.run(testFunSuite.scala:8) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 14:34:30 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 14:35:47 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 14:35:48 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 14:35:48 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 14:35:48 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 14:35:48 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 14:35:49 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 14:35:49 INFO Remoting:74 - Starting remoting -2016-06-13 14:35:49 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-5] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 14:35:49 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 14:35:49 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 14:35:49 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 14:35:49 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 14:35:59 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.util.MLlibTestSparkContext$class.beforeAll(MLlibTestSparkContext.scala:21) - at org.apache.spark.mllib.learning.basic.testFunSuite.beforeAll(testFunSuite.scala:8) - at org.scalatest.BeforeAndAfterAll$class.beforeAll(BeforeAndAfterAll.scala:187) - at org.apache.spark.mllib.learning.basic.testFunSuite.beforeAll(testFunSuite.scala:8) - at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:253) - at org.apache.spark.mllib.learning.basic.testFunSuite.run(testFunSuite.scala:8) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 14:35:59 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 14:37:11 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 14:37:12 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 14:37:12 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 14:37:12 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 14:37:12 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 14:37:13 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 14:37:13 INFO Remoting:74 - Starting remoting -2016-06-13 14:37:13 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-5] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 14:37:13 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 14:37:13 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 14:37:13 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 14:37:13 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 14:37:23 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.util.MLlibTestSparkContext$class.beforeAll(MLlibTestSparkContext.scala:21) - at org.apache.spark.mllib.learning.basic.testFunSuite.beforeAll(testFunSuite.scala:9) - at org.scalatest.BeforeAndAfterAll$class.beforeAll(BeforeAndAfterAll.scala:187) - at org.apache.spark.mllib.learning.basic.testFunSuite.beforeAll(testFunSuite.scala:9) - at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:253) - at org.apache.spark.mllib.learning.basic.testFunSuite.run(testFunSuite.scala:9) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 14:37:23 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 14:38:33 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 14:38:34 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 14:38:34 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 14:38:34 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 14:38:34 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 14:38:35 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 14:38:35 INFO Remoting:74 - Starting remoting -2016-06-13 14:38:35 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-6] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 14:38:35 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 14:38:35 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 14:38:35 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 14:38:35 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 14:38:45 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.util.MLlibTestSparkContext$class.beforeAll(MLlibTestSparkContext.scala:21) - at org.apache.spark.mllib.learning.basic.testFunSuite.beforeAll(testFunSuite.scala:9) - at org.scalatest.BeforeAndAfterAll$class.beforeAll(BeforeAndAfterAll.scala:187) - at org.apache.spark.mllib.learning.basic.testFunSuite.beforeAll(testFunSuite.scala:9) - at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:253) - at org.apache.spark.mllib.learning.basic.testFunSuite.run(testFunSuite.scala:9) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 14:38:45 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 14:58:24 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 14:58:25 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 14:58:25 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 14:58:25 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 14:58:25 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 14:58:26 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 14:58:26 INFO Remoting:74 - Starting remoting -2016-06-13 14:58:26 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-6] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 14:58:26 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 14:58:26 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 14:58:26 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 14:58:26 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 14:58:36 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.util.SparkLearningFunSuite.(SparkLearningFunSuite.scala:14) - at org.apache.spark.mllib.learning.basic.testFunSuite.(testFunSuite.scala:9) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at java.lang.Class.newInstance(Unknown Source) - at org.scalatest.tools.Runner$.genSuiteConfig(Runner.scala:2644) - at org.scalatest.tools.Runner$$anonfun$37.apply(Runner.scala:2461) - at org.scalatest.tools.Runner$$anonfun$37.apply(Runner.scala:2460) - at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) - at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) - at scala.collection.immutable.List.foreach(List.scala:318) - at scala.collection.TraversableLike$class.map(TraversableLike.scala:244) - at scala.collection.AbstractTraversable.map(Traversable.scala:105) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2460) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 14:58:36 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 14:59:06 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 14:59:08 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 14:59:08 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 14:59:08 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 14:59:08 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 14:59:09 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 14:59:09 INFO Remoting:74 - Starting remoting -2016-06-13 14:59:09 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-6] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 14:59:09 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 14:59:09 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 14:59:09 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 14:59:09 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 14:59:19 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.util.SparkLearningFunSuite.(SparkLearningFunSuite.scala:14) - at org.apache.spark.mllib.learning.basic.testFunSuite.(testFunSuite.scala:8) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at java.lang.Class.newInstance(Unknown Source) - at org.scalatest.tools.Runner$.genSuiteConfig(Runner.scala:2644) - at org.scalatest.tools.Runner$$anonfun$37.apply(Runner.scala:2461) - at org.scalatest.tools.Runner$$anonfun$37.apply(Runner.scala:2460) - at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) - at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) - at scala.collection.immutable.List.foreach(List.scala:318) - at scala.collection.TraversableLike$class.map(TraversableLike.scala:244) - at scala.collection.AbstractTraversable.map(Traversable.scala:105) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2460) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 14:59:19 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 14:59:36 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 14:59:37 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 14:59:37 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 14:59:37 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 14:59:37 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 14:59:38 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 14:59:38 INFO Remoting:74 - Starting remoting -2016-06-13 14:59:38 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-6] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 14:59:38 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 14:59:38 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 14:59:38 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 14:59:38 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 14:59:48 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.util.SparkLearningFunSuite.(SparkLearningFunSuite.scala:14) - at org.apache.spark.mllib.learning.basic.testFunSuite.(testFunSuite.scala:8) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at java.lang.Class.newInstance(Unknown Source) - at org.scalatest.tools.Runner$.genSuiteConfig(Runner.scala:2644) - at org.scalatest.tools.Runner$$anonfun$37.apply(Runner.scala:2461) - at org.scalatest.tools.Runner$$anonfun$37.apply(Runner.scala:2460) - at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) - at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) - at scala.collection.immutable.List.foreach(List.scala:318) - at scala.collection.TraversableLike$class.map(TraversableLike.scala:244) - at scala.collection.AbstractTraversable.map(Traversable.scala:105) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2460) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 14:59:48 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:00:28 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:00:29 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:00:29 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:00:29 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:00:29 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:00:30 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:00:30 INFO Remoting:74 - Starting remoting -2016-06-13 15:00:30 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-5] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 15:00:30 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:00:30 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:00:30 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 15:00:30 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 15:00:40 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.util.SparkLearningFunSuite.(SparkLearningFunSuite.scala:12) - at org.apache.spark.mllib.learning.basic.testFunSuite.(testFunSuite.scala:8) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at java.lang.Class.newInstance(Unknown Source) - at org.scalatest.tools.Runner$.genSuiteConfig(Runner.scala:2644) - at org.scalatest.tools.Runner$$anonfun$37.apply(Runner.scala:2461) - at org.scalatest.tools.Runner$$anonfun$37.apply(Runner.scala:2460) - at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) - at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) - at scala.collection.immutable.List.foreach(List.scala:318) - at scala.collection.TraversableLike$class.map(TraversableLike.scala:244) - at scala.collection.AbstractTraversable.map(Traversable.scala:105) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2460) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 15:00:40 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:01:14 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:01:15 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:01:15 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:01:15 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:01:15 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:01:16 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:01:16 INFO Remoting:74 - Starting remoting -2016-06-13 15:01:16 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-7] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 15:01:16 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:01:16 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:01:16 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 15:01:16 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 15:01:26 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.util.SparkLearningFunSuite.(SparkLearningFunSuite.scala:15) - at org.apache.spark.mllib.learning.basic.testFunSuite.(testFunSuite.scala:8) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at java.lang.Class.newInstance(Unknown Source) - at org.scalatest.tools.Runner$.genSuiteConfig(Runner.scala:2644) - at org.scalatest.tools.Runner$$anonfun$37.apply(Runner.scala:2461) - at org.scalatest.tools.Runner$$anonfun$37.apply(Runner.scala:2460) - at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) - at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) - at scala.collection.immutable.List.foreach(List.scala:318) - at scala.collection.TraversableLike$class.map(TraversableLike.scala:244) - at scala.collection.AbstractTraversable.map(Traversable.scala:105) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2460) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 15:01:26 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:02:15 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:02:16 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:02:16 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:02:16 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:02:16 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:02:17 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:02:17 INFO Remoting:74 - Starting remoting -2016-06-13 15:02:17 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-6] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 15:02:27 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.util.SparkLearningFunSuite.beforeAll(SparkLearningFunSuite.scala:22) - at org.scalatest.BeforeAndAfterAll$class.beforeAll(BeforeAndAfterAll.scala:187) - at org.apache.spark.util.SparkLearningFunSuite.beforeAll(SparkLearningFunSuite.scala:10) - at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:253) - at org.apache.spark.util.SparkLearningFunSuite.run(SparkLearningFunSuite.scala:10) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 15:02:27 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:03:39 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:03:41 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:03:41 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:03:41 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:03:41 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:03:42 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:03:42 INFO Remoting:74 - Starting remoting -2016-06-13 15:03:42 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-6] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 15:03:42 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:03:42 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:03:42 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 15:03:42 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 15:03:52 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.util.SparkLearningFunSuite.beforeAll(SparkLearningFunSuite.scala:22) - at org.scalatest.BeforeAndAfterAll$class.beforeAll(BeforeAndAfterAll.scala:187) - at org.apache.spark.util.SparkLearningFunSuite.beforeAll(SparkLearningFunSuite.scala:10) - at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:253) - at org.apache.spark.util.SparkLearningFunSuite.run(SparkLearningFunSuite.scala:10) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 15:03:52 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:05:33 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:05:34 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:05:34 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:05:34 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:05:34 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:05:35 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:05:35 INFO Remoting:74 - Starting remoting -2016-06-13 15:05:35 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-5] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 15:05:35 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:05:35 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:05:35 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 15:05:35 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 15:05:52 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:05:52 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:05:52 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:05:52 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:05:52 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:05:53 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:05:53 INFO Remoting:74 - Starting remoting -2016-06-13 15:05:53 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-5] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 15:05:53 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:05:53 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:05:53 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 15:05:53 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 15:06:03 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.mllib.learning.basic.testFunSuite$$anonfun$1.apply$mcV$sp(testFunSuite.scala:15) - at org.apache.spark.mllib.learning.basic.testFunSuite$$anonfun$1.apply(testFunSuite.scala:11) - at org.apache.spark.mllib.learning.basic.testFunSuite$$anonfun$1.apply(testFunSuite.scala:11) - at org.scalatest.Transformer$$anonfun$apply$1.apply$mcV$sp(Transformer.scala:22) - at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85) - at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104) - at org.scalatest.Transformer.apply(Transformer.scala:22) - at org.scalatest.Transformer.apply(Transformer.scala:20) - at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:166) - at org.scalatest.Suite$class.withFixture(Suite.scala:1122) - at org.scalatest.FunSuite.withFixture(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$class.invokeWithFixture$1(FunSuiteLike.scala:163) - at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175) - at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175) - at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306) - at org.scalatest.FunSuiteLike$class.runTest(FunSuiteLike.scala:175) - at org.scalatest.FunSuite.runTest(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208) - at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208) - at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:413) - at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401) - at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:396) - at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:483) - at org.scalatest.FunSuiteLike$class.runTests(FunSuiteLike.scala:208) - at org.scalatest.FunSuite.runTests(FunSuite.scala:1555) - at org.scalatest.Suite$class.run(Suite.scala:1424) - at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212) - at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212) - at org.scalatest.SuperEngine.runImpl(Engine.scala:545) - at org.scalatest.FunSuiteLike$class.run(FunSuiteLike.scala:212) - at org.scalatest.FunSuite.run(FunSuite.scala:1555) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 15:06:03 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:06:55 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:06:55 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:06:56 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:06:56 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:06:56 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:06:56 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:06:56 INFO Remoting:74 - Starting remoting -2016-06-13 15:06:56 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-5] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 15:06:56 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:06:56 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:06:56 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 15:06:56 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 15:07:06 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.mllib.learning.basic.testFunSuite$$anonfun$1.apply$mcV$sp(testFunSuite.scala:15) - at org.apache.spark.mllib.learning.basic.testFunSuite$$anonfun$1.apply(testFunSuite.scala:11) - at org.apache.spark.mllib.learning.basic.testFunSuite$$anonfun$1.apply(testFunSuite.scala:11) - at org.scalatest.Transformer$$anonfun$apply$1.apply$mcV$sp(Transformer.scala:22) - at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85) - at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104) - at org.scalatest.Transformer.apply(Transformer.scala:22) - at org.scalatest.Transformer.apply(Transformer.scala:20) - at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:166) - at org.scalatest.Suite$class.withFixture(Suite.scala:1122) - at org.scalatest.FunSuite.withFixture(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$class.invokeWithFixture$1(FunSuiteLike.scala:163) - at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175) - at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175) - at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306) - at org.scalatest.FunSuiteLike$class.runTest(FunSuiteLike.scala:175) - at org.scalatest.FunSuite.runTest(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208) - at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208) - at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:413) - at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401) - at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:396) - at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:483) - at org.scalatest.FunSuiteLike$class.runTests(FunSuiteLike.scala:208) - at org.scalatest.FunSuite.runTests(FunSuite.scala:1555) - at org.scalatest.Suite$class.run(Suite.scala:1424) - at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212) - at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212) - at org.scalatest.SuperEngine.runImpl(Engine.scala:545) - at org.scalatest.FunSuiteLike$class.run(FunSuiteLike.scala:212) - at org.scalatest.FunSuite.run(FunSuite.scala:1555) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 15:07:06 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:07:43 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:07:44 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:07:45 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:07:45 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:07:45 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:07:46 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:07:46 INFO Remoting:74 - Starting remoting -2016-06-13 15:07:46 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-5] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 15:07:46 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:07:46 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:07:46 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 15:07:46 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 15:07:56 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.mllib.learning.basic.testFunSuite$$anonfun$1.apply$mcV$sp(testFunSuite.scala:14) - at org.apache.spark.mllib.learning.basic.testFunSuite$$anonfun$1.apply(testFunSuite.scala:10) - at org.apache.spark.mllib.learning.basic.testFunSuite$$anonfun$1.apply(testFunSuite.scala:10) - at org.scalatest.Transformer$$anonfun$apply$1.apply$mcV$sp(Transformer.scala:22) - at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85) - at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104) - at org.scalatest.Transformer.apply(Transformer.scala:22) - at org.scalatest.Transformer.apply(Transformer.scala:20) - at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:166) - at org.scalatest.Suite$class.withFixture(Suite.scala:1122) - at org.scalatest.FunSuite.withFixture(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$class.invokeWithFixture$1(FunSuiteLike.scala:163) - at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175) - at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175) - at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306) - at org.scalatest.FunSuiteLike$class.runTest(FunSuiteLike.scala:175) - at org.scalatest.FunSuite.runTest(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208) - at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208) - at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:413) - at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401) - at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:396) - at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:483) - at org.scalatest.FunSuiteLike$class.runTests(FunSuiteLike.scala:208) - at org.scalatest.FunSuite.runTests(FunSuite.scala:1555) - at org.scalatest.Suite$class.run(Suite.scala:1424) - at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212) - at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212) - at org.scalatest.SuperEngine.runImpl(Engine.scala:545) - at org.scalatest.FunSuiteLike$class.run(FunSuiteLike.scala:212) - at org.scalatest.FunSuite.run(FunSuite.scala:1555) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 15:07:56 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:08:40 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:08:41 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:08:41 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:08:41 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:08:41 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:08:42 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:08:42 INFO Remoting:74 - Starting remoting -2016-06-13 15:08:42 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-7] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 15:08:42 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:08:42 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:08:42 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 15:08:42 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 15:08:52 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.mllib.learning.basic.testFunSuite$$anonfun$1.apply$mcV$sp(testFunSuite.scala:14) - at org.apache.spark.mllib.learning.basic.testFunSuite$$anonfun$1.apply(testFunSuite.scala:10) - at org.apache.spark.mllib.learning.basic.testFunSuite$$anonfun$1.apply(testFunSuite.scala:10) - at org.scalatest.Transformer$$anonfun$apply$1.apply$mcV$sp(Transformer.scala:22) - at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85) - at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104) - at org.scalatest.Transformer.apply(Transformer.scala:22) - at org.scalatest.Transformer.apply(Transformer.scala:20) - at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:166) - at org.scalatest.Suite$class.withFixture(Suite.scala:1122) - at org.scalatest.FunSuite.withFixture(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$class.invokeWithFixture$1(FunSuiteLike.scala:163) - at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175) - at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175) - at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306) - at org.scalatest.FunSuiteLike$class.runTest(FunSuiteLike.scala:175) - at org.scalatest.FunSuite.runTest(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208) - at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208) - at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:413) - at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401) - at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:396) - at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:483) - at org.scalatest.FunSuiteLike$class.runTests(FunSuiteLike.scala:208) - at org.scalatest.FunSuite.runTests(FunSuite.scala:1555) - at org.scalatest.Suite$class.run(Suite.scala:1424) - at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212) - at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212) - at org.scalatest.SuperEngine.runImpl(Engine.scala:545) - at org.scalatest.FunSuiteLike$class.run(FunSuiteLike.scala:212) - at org.scalatest.FunSuite.run(FunSuite.scala:1555) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 15:08:52 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:08:59 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:08:59 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:08:59 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:08:59 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:08:59 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:09:00 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:09:00 INFO Remoting:74 - Starting remoting -2016-06-13 15:09:00 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-6] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 15:09:00 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:09:00 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:09:00 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 15:09:00 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 15:09:10 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.mllib.learning.basic.testFunSuite$$anonfun$1.apply$mcV$sp(testFunSuite.scala:14) - at org.apache.spark.mllib.learning.basic.testFunSuite$$anonfun$1.apply(testFunSuite.scala:10) - at org.apache.spark.mllib.learning.basic.testFunSuite$$anonfun$1.apply(testFunSuite.scala:10) - at org.scalatest.Transformer$$anonfun$apply$1.apply$mcV$sp(Transformer.scala:22) - at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85) - at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104) - at org.scalatest.Transformer.apply(Transformer.scala:22) - at org.scalatest.Transformer.apply(Transformer.scala:20) - at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:166) - at org.scalatest.Suite$class.withFixture(Suite.scala:1122) - at org.scalatest.FunSuite.withFixture(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$class.invokeWithFixture$1(FunSuiteLike.scala:163) - at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175) - at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175) - at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306) - at org.scalatest.FunSuiteLike$class.runTest(FunSuiteLike.scala:175) - at org.scalatest.FunSuite.runTest(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208) - at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208) - at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:413) - at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401) - at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:396) - at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:483) - at org.scalatest.FunSuiteLike$class.runTests(FunSuiteLike.scala:208) - at org.scalatest.FunSuite.runTests(FunSuite.scala:1555) - at org.scalatest.Suite$class.run(Suite.scala:1424) - at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212) - at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212) - at org.scalatest.SuperEngine.runImpl(Engine.scala:545) - at org.scalatest.FunSuiteLike$class.run(FunSuiteLike.scala:212) - at org.scalatest.FunSuite.run(FunSuite.scala:1555) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 15:09:10 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:09:51 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:09:52 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:09:52 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:09:52 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:09:52 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:09:53 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:09:53 INFO Remoting:74 - Starting remoting -2016-06-13 15:09:53 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-6] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 15:09:53 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:09:53 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:09:53 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 15:09:53 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 15:10:03 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.sql.learning.SparkSQLExamplesFromJson$.main(SparkSQLExamplesFromJson.scala:17) - at org.apache.spark.sql.learning.SparkSQLExamplesFromJson.main(SparkSQLExamplesFromJson.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 15:10:03 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:13:35 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:13:36 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:13:36 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:13:36 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:13:36 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:13:37 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:13:37 INFO Remoting:74 - Starting remoting -2016-06-13 15:13:37 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-6] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 15:13:37 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:13:37 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:13:37 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 15:13:37 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 15:13:47 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.sql.learning.SparkSQLExamplesFromJson$.main(SparkSQLExamplesFromJson.scala:17) - at org.apache.spark.sql.learning.SparkSQLExamplesFromJson.main(SparkSQLExamplesFromJson.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 15:13:47 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:15:23 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:15:23 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:15:24 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:15:24 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:15:24 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:15:24 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:15:24 INFO Remoting:74 - Starting remoting -2016-06-13 15:15:24 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-5] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 15:15:24 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:15:24 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:15:24 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 15:15:24 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 15:15:34 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.sql.learning.SparkSQLExamplesFromJson$.main(SparkSQLExamplesFromJson.scala:17) - at org.apache.spark.sql.learning.SparkSQLExamplesFromJson.main(SparkSQLExamplesFromJson.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 15:15:34 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:16:01 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:16:02 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:16:02 INFO Remoting:74 - Starting remoting -2016-06-13 15:16:02 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-7] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 15:16:02 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:16:02 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:16:02 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 15:16:02 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 15:16:12 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.mllib.learning.regression.LinearRegression1Learning$.main(LinearRegression1Learning.scala:21) - at org.apache.spark.mllib.learning.regression.LinearRegression1Learning.main(LinearRegression1Learning.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 15:16:53 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:16:54 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:16:54 INFO Remoting:74 - Starting remoting -2016-06-13 15:16:54 ERROR ActorSystemImpl:66 - Uncaught fatal error from thread [sparkDriver-akka.remote.default-remote-dispatcher-6] shutting down ActorSystem [sparkDriver] -java.lang.VerifyError: (class: org/jboss/netty/channel/socket/nio/NioWorkerPool, method: createWorker signature: (Ljava/util/concurrent/Executor;)Lorg/jboss/netty/channel/socket/nio/AbstractNioWorker;) Wrong return type in function - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:283) - at akka.remote.transport.netty.NettyTransport.(NettyTransport.scala:240) - at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) - at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source) - at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source) - at java.lang.reflect.Constructor.newInstance(Unknown Source) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:78) - at scala.util.Try$.apply(Try.scala:161) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:73) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:84) - at scala.util.Success.flatMap(Try.scala:200) - at akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:84) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:711) - at akka.remote.EndpointManager$$anonfun$9.apply(Remoting.scala:703) - at scala.collection.TraversableLike$WithFilter$$anonfun$map$2.apply(TraversableLike.scala:722) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at scala.collection.AbstractIterator.foreach(Iterator.scala:1157) - at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) - at scala.collection.AbstractIterable.foreach(Iterable.scala:54) - at scala.collection.TraversableLike$WithFilter.map(TraversableLike.scala:721) - at akka.remote.EndpointManager.akka$remote$EndpointManager$$listens(Remoting.scala:703) - at akka.remote.EndpointManager$$anonfun$receive$2.applyOrElse(Remoting.scala:491) - at akka.actor.Actor$class.aroundReceive(Actor.scala:467) - at akka.remote.EndpointManager.aroundReceive(Remoting.scala:394) - at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) - at akka.actor.ActorCell.invoke(ActorCell.scala:487) - at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) - at akka.dispatch.Mailbox.run(Mailbox.scala:220) - at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) - at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) - at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) - at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) - at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) -2016-06-13 15:16:54 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:16:54 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:16:54 ERROR Remoting:65 - Remoting system has been terminated abrubtly. Attempting to shut down transports -2016-06-13 15:16:54 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 15:17:04 ERROR SparkContext:96 - Error initializing SparkContext. -java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] - at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) - at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) - at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) - at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) - at scala.concurrent.Await$.result(package.scala:107) - at akka.remote.Remoting.start(Remoting.scala:179) - at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) - at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:620) - at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl._start(ActorSystem.scala:617) - at akka.actor.ActorSystemImpl.start(ActorSystem.scala:634) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:142) - at akka.actor.ActorSystem$.apply(ActorSystem.scala:119) - at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) - at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) - at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) - at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) - at org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) - at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) - at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) - at org.apache.spark.SparkContext.(SparkContext.scala:450) - at org.apache.spark.mllib.learning.regression.LinearRegression1Learning$.main(LinearRegression1Learning.scala:21) - at org.apache.spark.mllib.learning.regression.LinearRegression1Learning.main(LinearRegression1Learning.scala) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 15:18:50 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:18:51 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:18:51 INFO Remoting:74 - Starting remoting -2016-06-13 15:18:52 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:57551] -2016-06-13 15:18:53 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:18:53 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:57552 -2016-06-13 15:18:54 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:18:54 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:19:07 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 15:19:07 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 15:19:07 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 15:19:07 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 15:19:07 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 15:19:07 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 15:19:07 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 15:19:11 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 15:19:11 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 15:19:16 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201606131519_0079_m_000000_79' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/LinearRegression/20160613151915553/result/_temporary/0/task_201606131519_0079_m_000000 -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:19:16 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:19:17 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:19:17 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:19:23 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:19:23 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:19:23 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:19:23 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:19:23 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:19:24 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:19:24 INFO Remoting:74 - Starting remoting -2016-06-13 15:19:25 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:57592] -2016-06-13 15:19:25 INFO Utils:59 - Successfully started service 'sparkDriver' on port 57592. -2016-06-13 15:19:25 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 15:19:25 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 15:19:25 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-b0ac282f-9bbb-4623-9d3e-9ecbf4bc4603 -2016-06-13 15:19:25 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 15:19:25 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-b43c5227-6c52-411f-87f4-1dd04d3ae58c\httpd-d06a0093-7643-4b34-9b6c-3811345dfc00 -2016-06-13 15:19:25 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 15:19:25 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:19:25 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:57593 -2016-06-13 15:19:25 INFO Utils:59 - Successfully started service 'HTTP file server' on port 57593. -2016-06-13 15:19:25 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 15:19:25 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:19:25 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:19:25 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 15:19:25 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 15:19:25 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 15:19:25 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 15:19:26 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 57600. -2016-06-13 15:19:26 INFO NettyBlockTransferService:59 - Server created on 57600 -2016-06-13 15:19:26 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 15:19:26 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:57600 with 730.6 MB RAM, BlockManagerId(driver, localhost, 57600) -2016-06-13 15:19:26 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 15:19:28 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 15:19:28 INFO JSONRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/examples/src/main/resources/people.json on driver -2016-06-13 15:19:28 INFO MemoryStore:59 - ensureFreeSpace(231288) called with curMem=0, maxMem=766075207 -2016-06-13 15:19:28 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 225.9 KB, free 730.4 MB) -2016-06-13 15:19:29 INFO MemoryStore:59 - ensureFreeSpace(19919) called with curMem=231288, maxMem=766075207 -2016-06-13 15:19:29 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 19.5 KB, free 730.3 MB) -2016-06-13 15:19:29 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:57600 (size: 19.5 KB, free: 730.6 MB) -2016-06-13 15:19:29 INFO SparkContext:59 - Created broadcast 0 from main at null:-2 -2016-06-13 15:19:29 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 15:19:29 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-13 15:19:29 INFO DAGScheduler:59 - Got job 0 (main at null:-2) with 1 output partitions -2016-06-13 15:19:29 INFO DAGScheduler:59 - Final stage: ResultStage 0(main at null:-2) -2016-06-13 15:19:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:19:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:19:29 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at main at null:-2), which has no missing parents -2016-06-13 15:19:29 INFO MemoryStore:59 - ensureFreeSpace(4056) called with curMem=251207, maxMem=766075207 -2016-06-13 15:19:29 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 15:19:29 INFO MemoryStore:59 - ensureFreeSpace(2295) called with curMem=255263, maxMem=766075207 -2016-06-13 15:19:29 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 15:19:29 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:57600 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 15:19:29 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:19:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at main at null:-2) -2016-06-13 15:19:29 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 15:19:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 15:19:29 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 15:19:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/examples/src/main/resources/people.json:0+73 -2016-06-13 15:19:29 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 15:19:29 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 15:19:29 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 15:19:29 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 15:19:29 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 15:19:30 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2845 bytes result sent to driver -2016-06-13 15:19:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 1478 ms on localhost (1/1) -2016-06-13 15:19:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 15:19:30 INFO DAGScheduler:59 - ResultStage 0 (main at null:-2) finished in 1.491 s -2016-06-13 15:19:30 INFO DAGScheduler:59 - Job 0 finished: main at null:-2, took 1.619983 s -2016-06-13 15:19:32 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=257558, maxMem=766075207 -2016-06-13 15:19:32 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 86.4 KB, free 730.3 MB) -2016-06-13 15:19:32 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=346070, maxMem=766075207 -2016-06-13 15:19:32 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.2 MB) -2016-06-13 15:19:32 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:57600 (size: 19.3 KB, free: 730.5 MB) -2016-06-13 15:19:32 INFO SparkContext:59 - Created broadcast 2 from main at null:-2 -2016-06-13 15:19:32 INFO MemoryStore:59 - ensureFreeSpace(231288) called with curMem=365858, maxMem=766075207 -2016-06-13 15:19:32 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 225.9 KB, free 730.0 MB) -2016-06-13 15:19:32 INFO MemoryStore:59 - ensureFreeSpace(19919) called with curMem=597146, maxMem=766075207 -2016-06-13 15:19:32 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 19.5 KB, free 730.0 MB) -2016-06-13 15:19:32 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:57600 (size: 19.5 KB, free: 730.5 MB) -2016-06-13 15:19:32 INFO SparkContext:59 - Created broadcast 3 from main at null:-2 -2016-06-13 15:19:32 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 15:19:32 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-13 15:19:32 INFO DAGScheduler:59 - Got job 1 (main at null:-2) with 1 output partitions -2016-06-13 15:19:32 INFO DAGScheduler:59 - Final stage: ResultStage 1(main at null:-2) -2016-06-13 15:19:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:19:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:19:32 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at main at null:-2), which has no missing parents -2016-06-13 15:19:32 INFO MemoryStore:59 - ensureFreeSpace(3960) called with curMem=617065, maxMem=766075207 -2016-06-13 15:19:32 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.9 KB, free 730.0 MB) -2016-06-13 15:19:32 INFO MemoryStore:59 - ensureFreeSpace(2299) called with curMem=621025, maxMem=766075207 -2016-06-13 15:19:32 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.0 MB) -2016-06-13 15:19:32 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:57600 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 15:19:32 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:19:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at main at null:-2) -2016-06-13 15:19:32 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 15:19:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 15:19:32 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 15:19:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/examples/src/main/resources/people.json:0+73 -2016-06-13 15:19:32 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2508 bytes result sent to driver -2016-06-13 15:19:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 37 ms on localhost (1/1) -2016-06-13 15:19:32 INFO DAGScheduler:59 - ResultStage 1 (main at null:-2) finished in 0.037 s -2016-06-13 15:19:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 15:19:32 INFO DAGScheduler:59 - Job 1 finished: main at null:-2, took 0.051879 s -2016-06-13 15:19:32 INFO MemoryStore:59 - ensureFreeSpace(230728) called with curMem=623324, maxMem=766075207 -2016-06-13 15:19:32 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 225.3 KB, free 729.8 MB) -2016-06-13 15:19:32 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=854052, maxMem=766075207 -2016-06-13 15:19:32 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.8 MB) -2016-06-13 15:19:32 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:57600 (size: 19.3 KB, free: 730.5 MB) -2016-06-13 15:19:32 INFO SparkContext:59 - Created broadcast 5 from main at null:-2 -2016-06-13 15:19:32 INFO MemoryStore:59 - ensureFreeSpace(231288) called with curMem=873840, maxMem=766075207 -2016-06-13 15:19:32 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 225.9 KB, free 729.5 MB) -2016-06-13 15:19:32 INFO MemoryStore:59 - ensureFreeSpace(19919) called with curMem=1105128, maxMem=766075207 -2016-06-13 15:19:32 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 19.5 KB, free 729.5 MB) -2016-06-13 15:19:32 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:57600 (size: 19.5 KB, free: 730.5 MB) -2016-06-13 15:19:32 INFO SparkContext:59 - Created broadcast 6 from main at null:-2 -2016-06-13 15:19:32 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 15:19:32 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-13 15:19:32 INFO DAGScheduler:59 - Got job 2 (main at null:-2) with 1 output partitions -2016-06-13 15:19:32 INFO DAGScheduler:59 - Final stage: ResultStage 2(main at null:-2) -2016-06-13 15:19:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:19:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:19:32 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[11] at main at null:-2), which has no missing parents -2016-06-13 15:19:32 INFO MemoryStore:59 - ensureFreeSpace(3888) called with curMem=1125047, maxMem=766075207 -2016-06-13 15:19:32 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.8 KB, free 729.5 MB) -2016-06-13 15:19:32 INFO MemoryStore:59 - ensureFreeSpace(2254) called with curMem=1128935, maxMem=766075207 -2016-06-13 15:19:32 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 729.5 MB) -2016-06-13 15:19:32 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:57600 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 15:19:32 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:19:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[11] at main at null:-2) -2016-06-13 15:19:32 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-06-13 15:19:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 15:19:32 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-06-13 15:19:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/examples/src/main/resources/people.json:0+73 -2016-06-13 15:19:32 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 2415 bytes result sent to driver -2016-06-13 15:19:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 15 ms on localhost (1/1) -2016-06-13 15:19:32 INFO DAGScheduler:59 - ResultStage 2 (main at null:-2) finished in 0.015 s -2016-06-13 15:19:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 15:19:32 INFO DAGScheduler:59 - Job 2 finished: main at null:-2, took 0.029010 s -2016-06-13 15:19:33 INFO MemoryStore:59 - ensureFreeSpace(230728) called with curMem=1131189, maxMem=766075207 -2016-06-13 15:19:33 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 225.3 KB, free 729.3 MB) -2016-06-13 15:19:33 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1361917, maxMem=766075207 -2016-06-13 15:19:33 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.3 MB) -2016-06-13 15:19:33 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:57600 (size: 19.3 KB, free: 730.5 MB) -2016-06-13 15:19:33 INFO SparkContext:59 - Created broadcast 8 from main at null:-2 -2016-06-13 15:19:33 INFO MemoryStore:59 - ensureFreeSpace(231288) called with curMem=1381705, maxMem=766075207 -2016-06-13 15:19:33 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 225.9 KB, free 729.0 MB) -2016-06-13 15:19:33 INFO MemoryStore:59 - ensureFreeSpace(19919) called with curMem=1612993, maxMem=766075207 -2016-06-13 15:19:33 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 19.5 KB, free 729.0 MB) -2016-06-13 15:19:33 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:57600 (size: 19.5 KB, free: 730.4 MB) -2016-06-13 15:19:33 INFO SparkContext:59 - Created broadcast 9 from main at null:-2 -2016-06-13 15:19:33 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 15:19:33 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-13 15:19:33 INFO DAGScheduler:59 - Got job 3 (main at null:-2) with 1 output partitions -2016-06-13 15:19:33 INFO DAGScheduler:59 - Final stage: ResultStage 3(main at null:-2) -2016-06-13 15:19:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:19:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:19:33 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[16] at main at null:-2), which has no missing parents -2016-06-13 15:19:33 INFO MemoryStore:59 - ensureFreeSpace(6608) called with curMem=1632912, maxMem=766075207 -2016-06-13 15:19:33 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 6.5 KB, free 729.0 MB) -2016-06-13 15:19:33 INFO MemoryStore:59 - ensureFreeSpace(3682) called with curMem=1639520, maxMem=766075207 -2016-06-13 15:19:33 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.6 KB, free 729.0 MB) -2016-06-13 15:19:33 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:57600 (size: 3.6 KB, free: 730.4 MB) -2016-06-13 15:19:33 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:19:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 3 (MapPartitionsRDD[16] at main at null:-2) -2016-06-13 15:19:33 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-06-13 15:19:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 3, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 15:19:33 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 3) -2016-06-13 15:19:33 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/examples/src/main/resources/people.json:0+73 -2016-06-13 15:19:33 INFO GenerateMutableProjection:59 - Code generated in 383.151115 ms -2016-06-13 15:19:33 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:57600 in memory (size: 19.3 KB, free: 730.5 MB) -2016-06-13 15:19:33 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 3). 2653 bytes result sent to driver -2016-06-13 15:19:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 3) in 499 ms on localhost (1/1) -2016-06-13 15:19:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 15:19:33 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:57600 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 15:19:33 INFO DAGScheduler:59 - ResultStage 3 (main at null:-2) finished in 0.501 s -2016-06-13 15:19:33 INFO DAGScheduler:59 - Job 3 finished: main at null:-2, took 0.521974 s -2016-06-13 15:19:33 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 15:19:33 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:57600 in memory (size: 19.5 KB, free: 730.5 MB) -2016-06-13 15:19:33 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:57600 in memory (size: 19.3 KB, free: 730.5 MB) -2016-06-13 15:19:33 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:57600 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 15:19:33 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 15:19:33 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:57600 in memory (size: 19.5 KB, free: 730.5 MB) -2016-06-13 15:19:33 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:57600 in memory (size: 19.3 KB, free: 730.5 MB) -2016-06-13 15:19:33 INFO MemoryStore:59 - ensureFreeSpace(230728) called with curMem=519055, maxMem=766075207 -2016-06-13 15:19:33 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 225.3 KB, free 729.9 MB) -2016-06-13 15:19:33 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:57600 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 15:19:33 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 15:19:33 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:57600 in memory (size: 19.5 KB, free: 730.6 MB) -2016-06-13 15:19:33 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=492225, maxMem=766075207 -2016-06-13 15:19:33 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-06-13 15:19:33 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:57600 (size: 19.3 KB, free: 730.5 MB) -2016-06-13 15:19:33 INFO SparkContext:59 - Created broadcast 11 from main at null:-2 -2016-06-13 15:19:33 INFO MemoryStore:59 - ensureFreeSpace(231288) called with curMem=512013, maxMem=766075207 -2016-06-13 15:19:33 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 225.9 KB, free 729.9 MB) -2016-06-13 15:19:33 INFO MemoryStore:59 - ensureFreeSpace(19919) called with curMem=743301, maxMem=766075207 -2016-06-13 15:19:33 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 19.5 KB, free 729.9 MB) -2016-06-13 15:19:33 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:57600 (size: 19.5 KB, free: 730.5 MB) -2016-06-13 15:19:33 INFO SparkContext:59 - Created broadcast 12 from main at null:-2 -2016-06-13 15:19:34 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 15:19:34 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-13 15:19:34 INFO DAGScheduler:59 - Got job 4 (main at null:-2) with 1 output partitions -2016-06-13 15:19:34 INFO DAGScheduler:59 - Final stage: ResultStage 4(main at null:-2) -2016-06-13 15:19:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:19:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:19:34 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[21] at main at null:-2), which has no missing parents -2016-06-13 15:19:34 INFO MemoryStore:59 - ensureFreeSpace(6408) called with curMem=763220, maxMem=766075207 -2016-06-13 15:19:34 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 6.3 KB, free 729.9 MB) -2016-06-13 15:19:34 INFO MemoryStore:59 - ensureFreeSpace(3558) called with curMem=769628, maxMem=766075207 -2016-06-13 15:19:34 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.8 MB) -2016-06-13 15:19:34 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:57600 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 15:19:34 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:19:34 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (MapPartitionsRDD[21] at main at null:-2) -2016-06-13 15:19:34 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-06-13 15:19:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 4, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 15:19:34 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 4) -2016-06-13 15:19:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/examples/src/main/resources/people.json:0+73 -2016-06-13 15:19:34 INFO GeneratePredicate:59 - Code generated in 8.116644 ms -2016-06-13 15:19:34 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 4). 2636 bytes result sent to driver -2016-06-13 15:19:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 4) in 41 ms on localhost (1/1) -2016-06-13 15:19:34 INFO DAGScheduler:59 - ResultStage 4 (main at null:-2) finished in 0.041 s -2016-06-13 15:19:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 15:19:34 INFO DAGScheduler:59 - Job 4 finished: main at null:-2, took 0.059670 s -2016-06-13 15:19:34 INFO MemoryStore:59 - ensureFreeSpace(230728) called with curMem=773186, maxMem=766075207 -2016-06-13 15:19:34 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 225.3 KB, free 729.6 MB) -2016-06-13 15:19:34 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1003914, maxMem=766075207 -2016-06-13 15:19:34 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.6 MB) -2016-06-13 15:19:34 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:57600 (size: 19.3 KB, free: 730.5 MB) -2016-06-13 15:19:34 INFO SparkContext:59 - Created broadcast 14 from main at null:-2 -2016-06-13 15:19:34 INFO MemoryStore:59 - ensureFreeSpace(231288) called with curMem=1023702, maxMem=766075207 -2016-06-13 15:19:34 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 225.9 KB, free 729.4 MB) -2016-06-13 15:19:34 INFO MemoryStore:59 - ensureFreeSpace(19919) called with curMem=1254990, maxMem=766075207 -2016-06-13 15:19:34 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 19.5 KB, free 729.4 MB) -2016-06-13 15:19:34 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:57600 (size: 19.5 KB, free: 730.5 MB) -2016-06-13 15:19:34 INFO SparkContext:59 - Created broadcast 15 from main at null:-2 -2016-06-13 15:19:34 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-13 15:19:34 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 15:19:34 INFO DAGScheduler:59 - Registering RDD 26 (main at null:-2) -2016-06-13 15:19:34 INFO DAGScheduler:59 - Got job 5 (main at null:-2) with 1 output partitions -2016-06-13 15:19:34 INFO DAGScheduler:59 - Final stage: ResultStage 6(main at null:-2) -2016-06-13 15:19:34 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 5) -2016-06-13 15:19:34 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 5) -2016-06-13 15:19:34 INFO DAGScheduler:59 - Submitting ShuffleMapStage 5 (MapPartitionsRDD[26] at main at null:-2), which has no missing parents -2016-06-13 15:19:34 INFO MemoryStore:59 - ensureFreeSpace(10424) called with curMem=1274909, maxMem=766075207 -2016-06-13 15:19:34 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 10.2 KB, free 729.4 MB) -2016-06-13 15:19:34 INFO MemoryStore:59 - ensureFreeSpace(5241) called with curMem=1285333, maxMem=766075207 -2016-06-13 15:19:34 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 5.1 KB, free 729.4 MB) -2016-06-13 15:19:34 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:57600 (size: 5.1 KB, free: 730.5 MB) -2016-06-13 15:19:34 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:19:34 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 5 (MapPartitionsRDD[26] at main at null:-2) -2016-06-13 15:19:34 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 1 tasks -2016-06-13 15:19:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 5, localhost, PROCESS_LOCAL, 2179 bytes) -2016-06-13 15:19:34 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 5) -2016-06-13 15:19:34 INFO GenerateMutableProjection:59 - Code generated in 9.293758 ms -2016-06-13 15:19:34 INFO GenerateUnsafeProjection:59 - Code generated in 11.976846 ms -2016-06-13 15:19:34 INFO GenerateMutableProjection:59 - Code generated in 10.672562 ms -2016-06-13 15:19:34 INFO GenerateUnsafeRowJoiner:59 - Code generated in 11.125916 ms -2016-06-13 15:19:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/examples/src/main/resources/people.json:0+73 -2016-06-13 15:19:34 INFO GenerateMutableProjection:59 - Code generated in 7.983226 ms -2016-06-13 15:19:35 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:57600 in memory (size: 19.3 KB, free: 730.5 MB) -2016-06-13 15:19:35 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:57600 in memory (size: 3.5 KB, free: 730.5 MB) -2016-06-13 15:19:35 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 15:19:35 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 15:19:35 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 15:19:35 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:57600 in memory (size: 19.5 KB, free: 730.5 MB) -2016-06-13 15:19:35 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:57600 in memory (size: 19.3 KB, free: 730.5 MB) -2016-06-13 15:19:35 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:57600 in memory (size: 3.6 KB, free: 730.5 MB) -2016-06-13 15:19:35 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 15:19:35 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 15:19:35 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:57600 in memory (size: 19.5 KB, free: 730.6 MB) -2016-06-13 15:19:36 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 5). 2669 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 5) in 1942 ms on localhost (1/1) -2016-06-13 15:19:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 15:19:36 INFO DAGScheduler:59 - ShuffleMapStage 5 (main at null:-2) finished in 1.943 s -2016-06-13 15:19:36 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 15:19:36 INFO DAGScheduler:59 - running: Set() -2016-06-13 15:19:36 INFO DAGScheduler:59 - waiting: Set(ResultStage 6) -2016-06-13 15:19:36 INFO DAGScheduler:59 - failed: Set() -2016-06-13 15:19:36 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List() -2016-06-13 15:19:36 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[30] at main at null:-2), which is now runnable -2016-06-13 15:19:36 INFO MemoryStore:59 - ensureFreeSpace(11224) called with curMem=266872, maxMem=766075207 -2016-06-13 15:19:36 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 11.0 KB, free 730.3 MB) -2016-06-13 15:19:36 INFO MemoryStore:59 - ensureFreeSpace(5645) called with curMem=278096, maxMem=766075207 -2016-06-13 15:19:36 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 5.5 KB, free 730.3 MB) -2016-06-13 15:19:36 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:57600 (size: 5.5 KB, free: 730.6 MB) -2016-06-13 15:19:36 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:19:36 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 6 (MapPartitionsRDD[30] at main at null:-2) -2016-06-13 15:19:36 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 1 tasks -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 6, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 6) -2016-06-13 15:19:36 INFO GenerateMutableProjection:59 - Code generated in 7.068486 ms -2016-06-13 15:19:36 INFO GenerateUnsafeProjection:59 - Code generated in 8.108166 ms -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 12 ms -2016-06-13 15:19:36 INFO GenerateSafeProjection:59 - Code generated in 6.726239 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 6). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 6) in 149 ms on localhost (1/1) -2016-06-13 15:19:36 INFO DAGScheduler:59 - ResultStage 6 (main at null:-2) finished in 0.151 s -2016-06-13 15:19:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 15:19:36 INFO DAGScheduler:59 - Job 5 finished: main at null:-2, took 2.154696 s -2016-06-13 15:19:36 INFO SparkContext:59 - Starting job: main at null:-2 -2016-06-13 15:19:36 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 150 bytes -2016-06-13 15:19:36 INFO DAGScheduler:59 - Got job 6 (main at null:-2) with 199 output partitions -2016-06-13 15:19:36 INFO DAGScheduler:59 - Final stage: ResultStage 8(main at null:-2) -2016-06-13 15:19:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 7) -2016-06-13 15:19:36 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:19:36 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[30] at main at null:-2), which has no missing parents -2016-06-13 15:19:36 INFO MemoryStore:59 - ensureFreeSpace(11224) called with curMem=283741, maxMem=766075207 -2016-06-13 15:19:36 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 11.0 KB, free 730.3 MB) -2016-06-13 15:19:36 INFO MemoryStore:59 - ensureFreeSpace(5645) called with curMem=294965, maxMem=766075207 -2016-06-13 15:19:36 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 5.5 KB, free 730.3 MB) -2016-06-13 15:19:36 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:57600 (size: 5.5 KB, free: 730.6 MB) -2016-06-13 15:19:36 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:19:36 INFO DAGScheduler:59 - Submitting 199 missing tasks from ResultStage 8 (MapPartitionsRDD[30] at main at null:-2) -2016-06-13 15:19:36 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 199 tasks -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 7, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 7) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 7). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 8, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 7) in 17 ms on localhost (1/199) -2016-06-13 15:19:36 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 8) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 8). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 2.0 in stage 8.0 (TID 9, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 2.0 in stage 8.0 (TID 9) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 8) in 18 ms on localhost (2/199) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 2.0 in stage 8.0 (TID 9). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 3.0 in stage 8.0 (TID 10, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 2.0 in stage 8.0 (TID 9) in 21 ms on localhost (3/199) -2016-06-13 15:19:36 INFO Executor:59 - Running task 3.0 in stage 8.0 (TID 10) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 3.0 in stage 8.0 (TID 10). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 4.0 in stage 8.0 (TID 11, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 4.0 in stage 8.0 (TID 11) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 3.0 in stage 8.0 (TID 10) in 17 ms on localhost (4/199) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 4.0 in stage 8.0 (TID 11). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 5.0 in stage 8.0 (TID 12, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 5.0 in stage 8.0 (TID 12) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 4.0 in stage 8.0 (TID 11) in 17 ms on localhost (5/199) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 5.0 in stage 8.0 (TID 12). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 6.0 in stage 8.0 (TID 13, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 5.0 in stage 8.0 (TID 12) in 19 ms on localhost (6/199) -2016-06-13 15:19:36 INFO Executor:59 - Running task 6.0 in stage 8.0 (TID 13) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 6.0 in stage 8.0 (TID 13). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 7.0 in stage 8.0 (TID 14, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 7.0 in stage 8.0 (TID 14) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 6.0 in stage 8.0 (TID 13) in 29 ms on localhost (7/199) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 7.0 in stage 8.0 (TID 14). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 8.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 8.0 in stage 8.0 (TID 15) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 7.0 in stage 8.0 (TID 14) in 19 ms on localhost (8/199) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 8.0 in stage 8.0 (TID 15). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 9.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 9.0 in stage 8.0 (TID 16) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 8.0 in stage 8.0 (TID 15) in 21 ms on localhost (9/199) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 9.0 in stage 8.0 (TID 16). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 10.0 in stage 8.0 (TID 17, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 10.0 in stage 8.0 (TID 17) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 9.0 in stage 8.0 (TID 16) in 19 ms on localhost (10/199) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 10.0 in stage 8.0 (TID 17). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 11.0 in stage 8.0 (TID 18, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 11.0 in stage 8.0 (TID 18) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 10.0 in stage 8.0 (TID 17) in 15 ms on localhost (11/199) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 11.0 in stage 8.0 (TID 18). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 12.0 in stage 8.0 (TID 19, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 12.0 in stage 8.0 (TID 19) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 11.0 in stage 8.0 (TID 18) in 17 ms on localhost (12/199) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 12.0 in stage 8.0 (TID 19). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 13.0 in stage 8.0 (TID 20, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 13.0 in stage 8.0 (TID 20) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 12.0 in stage 8.0 (TID 19) in 16 ms on localhost (13/199) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 13.0 in stage 8.0 (TID 20). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 14.0 in stage 8.0 (TID 21, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 14.0 in stage 8.0 (TID 21) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 13.0 in stage 8.0 (TID 20) in 16 ms on localhost (14/199) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 14.0 in stage 8.0 (TID 21). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 15.0 in stage 8.0 (TID 22, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 15.0 in stage 8.0 (TID 22) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 14.0 in stage 8.0 (TID 21) in 18 ms on localhost (15/199) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 15.0 in stage 8.0 (TID 22). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 16.0 in stage 8.0 (TID 23, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 16.0 in stage 8.0 (TID 23) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 15.0 in stage 8.0 (TID 22) in 19 ms on localhost (16/199) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 16.0 in stage 8.0 (TID 23). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 17.0 in stage 8.0 (TID 24, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 17.0 in stage 8.0 (TID 24) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 16.0 in stage 8.0 (TID 23) in 18 ms on localhost (17/199) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 17.0 in stage 8.0 (TID 24). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 18.0 in stage 8.0 (TID 25, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 18.0 in stage 8.0 (TID 25) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 17.0 in stage 8.0 (TID 24) in 21 ms on localhost (18/199) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 18.0 in stage 8.0 (TID 25). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 19.0 in stage 8.0 (TID 26, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 19.0 in stage 8.0 (TID 26) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 18.0 in stage 8.0 (TID 25) in 21 ms on localhost (19/199) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 19.0 in stage 8.0 (TID 26). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 20.0 in stage 8.0 (TID 27, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 20.0 in stage 8.0 (TID 27) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 19.0 in stage 8.0 (TID 26) in 16 ms on localhost (20/199) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 20.0 in stage 8.0 (TID 27). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 21.0 in stage 8.0 (TID 28, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 21.0 in stage 8.0 (TID 28) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 20.0 in stage 8.0 (TID 27) in 14 ms on localhost (21/199) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:36 INFO Executor:59 - Finished task 21.0 in stage 8.0 (TID 28). 1437 bytes result sent to driver -2016-06-13 15:19:36 INFO TaskSetManager:59 - Starting task 22.0 in stage 8.0 (TID 29, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:36 INFO Executor:59 - Running task 22.0 in stage 8.0 (TID 29) -2016-06-13 15:19:36 INFO TaskSetManager:59 - Finished task 21.0 in stage 8.0 (TID 28) in 14 ms on localhost (22/199) -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 22.0 in stage 8.0 (TID 29). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 23.0 in stage 8.0 (TID 30, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 23.0 in stage 8.0 (TID 30) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 22.0 in stage 8.0 (TID 29) in 15 ms on localhost (23/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 23.0 in stage 8.0 (TID 30). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 24.0 in stage 8.0 (TID 31, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 24.0 in stage 8.0 (TID 31) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 23.0 in stage 8.0 (TID 30) in 19 ms on localhost (24/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 24.0 in stage 8.0 (TID 31). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 25.0 in stage 8.0 (TID 32, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 25.0 in stage 8.0 (TID 32) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 24.0 in stage 8.0 (TID 31) in 21 ms on localhost (25/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 25.0 in stage 8.0 (TID 32). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 26.0 in stage 8.0 (TID 33, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 26.0 in stage 8.0 (TID 33) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 25.0 in stage 8.0 (TID 32) in 21 ms on localhost (26/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 26.0 in stage 8.0 (TID 33). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 27.0 in stage 8.0 (TID 34, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 27.0 in stage 8.0 (TID 34) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 26.0 in stage 8.0 (TID 33) in 19 ms on localhost (27/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 27.0 in stage 8.0 (TID 34). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 28.0 in stage 8.0 (TID 35, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 27.0 in stage 8.0 (TID 34) in 16 ms on localhost (28/199) -2016-06-13 15:19:37 INFO Executor:59 - Running task 28.0 in stage 8.0 (TID 35) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 28.0 in stage 8.0 (TID 35). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 29.0 in stage 8.0 (TID 36, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 29.0 in stage 8.0 (TID 36) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 28.0 in stage 8.0 (TID 35) in 17 ms on localhost (29/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 29.0 in stage 8.0 (TID 36). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 30.0 in stage 8.0 (TID 37, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 30.0 in stage 8.0 (TID 37) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 29.0 in stage 8.0 (TID 36) in 20 ms on localhost (30/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 30.0 in stage 8.0 (TID 37). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 31.0 in stage 8.0 (TID 38, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 31.0 in stage 8.0 (TID 38) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 30.0 in stage 8.0 (TID 37) in 19 ms on localhost (31/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 31.0 in stage 8.0 (TID 38). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 32.0 in stage 8.0 (TID 39, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 32.0 in stage 8.0 (TID 39) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 31.0 in stage 8.0 (TID 38) in 20 ms on localhost (32/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 32.0 in stage 8.0 (TID 39). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 33.0 in stage 8.0 (TID 40, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 33.0 in stage 8.0 (TID 40) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 32.0 in stage 8.0 (TID 39) in 16 ms on localhost (33/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 33.0 in stage 8.0 (TID 40). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 34.0 in stage 8.0 (TID 41, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 34.0 in stage 8.0 (TID 41) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 33.0 in stage 8.0 (TID 40) in 15 ms on localhost (34/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 34.0 in stage 8.0 (TID 41). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 35.0 in stage 8.0 (TID 42, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 35.0 in stage 8.0 (TID 42) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 34.0 in stage 8.0 (TID 41) in 15 ms on localhost (35/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 35.0 in stage 8.0 (TID 42). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 36.0 in stage 8.0 (TID 43, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 36.0 in stage 8.0 (TID 43) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 35.0 in stage 8.0 (TID 42) in 17 ms on localhost (36/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 36.0 in stage 8.0 (TID 43). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 37.0 in stage 8.0 (TID 44, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 37.0 in stage 8.0 (TID 44) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 36.0 in stage 8.0 (TID 43) in 18 ms on localhost (37/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 37.0 in stage 8.0 (TID 44). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 38.0 in stage 8.0 (TID 45, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 38.0 in stage 8.0 (TID 45) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 37.0 in stage 8.0 (TID 44) in 20 ms on localhost (38/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 38.0 in stage 8.0 (TID 45). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 39.0 in stage 8.0 (TID 46, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 39.0 in stage 8.0 (TID 46) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 38.0 in stage 8.0 (TID 45) in 19 ms on localhost (39/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 39.0 in stage 8.0 (TID 46). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 40.0 in stage 8.0 (TID 47, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 40.0 in stage 8.0 (TID 47) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 39.0 in stage 8.0 (TID 46) in 17 ms on localhost (40/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 40.0 in stage 8.0 (TID 47). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 41.0 in stage 8.0 (TID 48, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 41.0 in stage 8.0 (TID 48) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 40.0 in stage 8.0 (TID 47) in 11 ms on localhost (41/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 41.0 in stage 8.0 (TID 48). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 42.0 in stage 8.0 (TID 49, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 42.0 in stage 8.0 (TID 49) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 41.0 in stage 8.0 (TID 48) in 13 ms on localhost (42/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 42.0 in stage 8.0 (TID 49). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 43.0 in stage 8.0 (TID 50, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 43.0 in stage 8.0 (TID 50) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 42.0 in stage 8.0 (TID 49) in 11 ms on localhost (43/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 43.0 in stage 8.0 (TID 50). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 44.0 in stage 8.0 (TID 51, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 44.0 in stage 8.0 (TID 51) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 43.0 in stage 8.0 (TID 50) in 11 ms on localhost (44/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 44.0 in stage 8.0 (TID 51). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 45.0 in stage 8.0 (TID 52, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 45.0 in stage 8.0 (TID 52) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 44.0 in stage 8.0 (TID 51) in 11 ms on localhost (45/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 45.0 in stage 8.0 (TID 52). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 46.0 in stage 8.0 (TID 53, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 46.0 in stage 8.0 (TID 53) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 45.0 in stage 8.0 (TID 52) in 11 ms on localhost (46/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 46.0 in stage 8.0 (TID 53). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 47.0 in stage 8.0 (TID 54, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 47.0 in stage 8.0 (TID 54) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 46.0 in stage 8.0 (TID 53) in 12 ms on localhost (47/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 47.0 in stage 8.0 (TID 54). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 48.0 in stage 8.0 (TID 55, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 48.0 in stage 8.0 (TID 55) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 47.0 in stage 8.0 (TID 54) in 12 ms on localhost (48/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 48.0 in stage 8.0 (TID 55). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 49.0 in stage 8.0 (TID 56, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 49.0 in stage 8.0 (TID 56) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 48.0 in stage 8.0 (TID 55) in 11 ms on localhost (49/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 49.0 in stage 8.0 (TID 56). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 50.0 in stage 8.0 (TID 57, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 50.0 in stage 8.0 (TID 57) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 49.0 in stage 8.0 (TID 56) in 10 ms on localhost (50/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 50.0 in stage 8.0 (TID 57). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 51.0 in stage 8.0 (TID 58, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 51.0 in stage 8.0 (TID 58) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 50.0 in stage 8.0 (TID 57) in 10 ms on localhost (51/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 51.0 in stage 8.0 (TID 58). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 52.0 in stage 8.0 (TID 59, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 52.0 in stage 8.0 (TID 59) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 51.0 in stage 8.0 (TID 58) in 12 ms on localhost (52/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 52.0 in stage 8.0 (TID 59). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 53.0 in stage 8.0 (TID 60, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 53.0 in stage 8.0 (TID 60) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 52.0 in stage 8.0 (TID 59) in 10 ms on localhost (53/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 53.0 in stage 8.0 (TID 60). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 54.0 in stage 8.0 (TID 61, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 54.0 in stage 8.0 (TID 61) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 53.0 in stage 8.0 (TID 60) in 13 ms on localhost (54/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 54.0 in stage 8.0 (TID 61). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 55.0 in stage 8.0 (TID 62, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 55.0 in stage 8.0 (TID 62) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 54.0 in stage 8.0 (TID 61) in 11 ms on localhost (55/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 55.0 in stage 8.0 (TID 62). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 56.0 in stage 8.0 (TID 63, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 56.0 in stage 8.0 (TID 63) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 55.0 in stage 8.0 (TID 62) in 10 ms on localhost (56/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 56.0 in stage 8.0 (TID 63). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 57.0 in stage 8.0 (TID 64, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 57.0 in stage 8.0 (TID 64) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 56.0 in stage 8.0 (TID 63) in 10 ms on localhost (57/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 57.0 in stage 8.0 (TID 64). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 58.0 in stage 8.0 (TID 65, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 58.0 in stage 8.0 (TID 65) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 57.0 in stage 8.0 (TID 64) in 10 ms on localhost (58/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 58.0 in stage 8.0 (TID 65). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 59.0 in stage 8.0 (TID 66, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 58.0 in stage 8.0 (TID 65) in 11 ms on localhost (59/199) -2016-06-13 15:19:37 INFO Executor:59 - Running task 59.0 in stage 8.0 (TID 66) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 59.0 in stage 8.0 (TID 66). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 60.0 in stage 8.0 (TID 67, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 60.0 in stage 8.0 (TID 67) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 59.0 in stage 8.0 (TID 66) in 11 ms on localhost (60/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 60.0 in stage 8.0 (TID 67). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 61.0 in stage 8.0 (TID 68, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 61.0 in stage 8.0 (TID 68) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 60.0 in stage 8.0 (TID 67) in 10 ms on localhost (61/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 61.0 in stage 8.0 (TID 68). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 62.0 in stage 8.0 (TID 69, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 62.0 in stage 8.0 (TID 69) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 61.0 in stage 8.0 (TID 68) in 9 ms on localhost (62/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 62.0 in stage 8.0 (TID 69). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 63.0 in stage 8.0 (TID 70, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 63.0 in stage 8.0 (TID 70) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 62.0 in stage 8.0 (TID 69) in 11 ms on localhost (63/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 63.0 in stage 8.0 (TID 70). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 64.0 in stage 8.0 (TID 71, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 63.0 in stage 8.0 (TID 70) in 11 ms on localhost (64/199) -2016-06-13 15:19:37 INFO Executor:59 - Running task 64.0 in stage 8.0 (TID 71) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 64.0 in stage 8.0 (TID 71). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 65.0 in stage 8.0 (TID 72, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 65.0 in stage 8.0 (TID 72) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 64.0 in stage 8.0 (TID 71) in 10 ms on localhost (65/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 65.0 in stage 8.0 (TID 72). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 66.0 in stage 8.0 (TID 73, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 66.0 in stage 8.0 (TID 73) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 65.0 in stage 8.0 (TID 72) in 10 ms on localhost (66/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 66.0 in stage 8.0 (TID 73). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 67.0 in stage 8.0 (TID 74, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 66.0 in stage 8.0 (TID 73) in 11 ms on localhost (67/199) -2016-06-13 15:19:37 INFO Executor:59 - Running task 67.0 in stage 8.0 (TID 74) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 67.0 in stage 8.0 (TID 74). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 68.0 in stage 8.0 (TID 75, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 68.0 in stage 8.0 (TID 75) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 67.0 in stage 8.0 (TID 74) in 13 ms on localhost (68/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 68.0 in stage 8.0 (TID 75). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 69.0 in stage 8.0 (TID 76, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 68.0 in stage 8.0 (TID 75) in 14 ms on localhost (69/199) -2016-06-13 15:19:37 INFO Executor:59 - Running task 69.0 in stage 8.0 (TID 76) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 69.0 in stage 8.0 (TID 76). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 70.0 in stage 8.0 (TID 77, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 70.0 in stage 8.0 (TID 77) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 69.0 in stage 8.0 (TID 76) in 14 ms on localhost (70/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 70.0 in stage 8.0 (TID 77). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 71.0 in stage 8.0 (TID 78, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 71.0 in stage 8.0 (TID 78) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 70.0 in stage 8.0 (TID 77) in 14 ms on localhost (71/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 71.0 in stage 8.0 (TID 78). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 72.0 in stage 8.0 (TID 79, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 72.0 in stage 8.0 (TID 79) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 71.0 in stage 8.0 (TID 78) in 11 ms on localhost (72/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 72.0 in stage 8.0 (TID 79). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 73.0 in stage 8.0 (TID 80, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 73.0 in stage 8.0 (TID 80) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 72.0 in stage 8.0 (TID 79) in 9 ms on localhost (73/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 73.0 in stage 8.0 (TID 80). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 74.0 in stage 8.0 (TID 81, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 74.0 in stage 8.0 (TID 81) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 73.0 in stage 8.0 (TID 80) in 10 ms on localhost (74/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 74.0 in stage 8.0 (TID 81). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 75.0 in stage 8.0 (TID 82, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 75.0 in stage 8.0 (TID 82) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 74.0 in stage 8.0 (TID 81) in 11 ms on localhost (75/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 75.0 in stage 8.0 (TID 82). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 76.0 in stage 8.0 (TID 83, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 76.0 in stage 8.0 (TID 83) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 75.0 in stage 8.0 (TID 82) in 10 ms on localhost (76/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 76.0 in stage 8.0 (TID 83). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 77.0 in stage 8.0 (TID 84, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 77.0 in stage 8.0 (TID 84) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 76.0 in stage 8.0 (TID 83) in 10 ms on localhost (77/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 77.0 in stage 8.0 (TID 84). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 78.0 in stage 8.0 (TID 85, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 78.0 in stage 8.0 (TID 85) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 77.0 in stage 8.0 (TID 84) in 12 ms on localhost (78/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 78.0 in stage 8.0 (TID 85). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 79.0 in stage 8.0 (TID 86, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 79.0 in stage 8.0 (TID 86) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 78.0 in stage 8.0 (TID 85) in 9 ms on localhost (79/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 79.0 in stage 8.0 (TID 86). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 80.0 in stage 8.0 (TID 87, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 80.0 in stage 8.0 (TID 87) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 79.0 in stage 8.0 (TID 86) in 10 ms on localhost (80/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 80.0 in stage 8.0 (TID 87). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 81.0 in stage 8.0 (TID 88, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 81.0 in stage 8.0 (TID 88) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 80.0 in stage 8.0 (TID 87) in 10 ms on localhost (81/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 81.0 in stage 8.0 (TID 88). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 82.0 in stage 8.0 (TID 89, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 82.0 in stage 8.0 (TID 89) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 81.0 in stage 8.0 (TID 88) in 11 ms on localhost (82/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 82.0 in stage 8.0 (TID 89). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 83.0 in stage 8.0 (TID 90, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 83.0 in stage 8.0 (TID 90) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 82.0 in stage 8.0 (TID 89) in 11 ms on localhost (83/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 83.0 in stage 8.0 (TID 90). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 84.0 in stage 8.0 (TID 91, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 84.0 in stage 8.0 (TID 91) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 83.0 in stage 8.0 (TID 90) in 10 ms on localhost (84/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 84.0 in stage 8.0 (TID 91). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 85.0 in stage 8.0 (TID 92, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 85.0 in stage 8.0 (TID 92) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 84.0 in stage 8.0 (TID 91) in 11 ms on localhost (85/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 85.0 in stage 8.0 (TID 92). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 86.0 in stage 8.0 (TID 93, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 86.0 in stage 8.0 (TID 93) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 85.0 in stage 8.0 (TID 92) in 10 ms on localhost (86/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 86.0 in stage 8.0 (TID 93). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 87.0 in stage 8.0 (TID 94, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 87.0 in stage 8.0 (TID 94) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 86.0 in stage 8.0 (TID 93) in 14 ms on localhost (87/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 87.0 in stage 8.0 (TID 94). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 88.0 in stage 8.0 (TID 95, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 88.0 in stage 8.0 (TID 95) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 87.0 in stage 8.0 (TID 94) in 12 ms on localhost (88/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 88.0 in stage 8.0 (TID 95). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 89.0 in stage 8.0 (TID 96, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 89.0 in stage 8.0 (TID 96) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 88.0 in stage 8.0 (TID 95) in 10 ms on localhost (89/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 89.0 in stage 8.0 (TID 96). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 90.0 in stage 8.0 (TID 97, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 90.0 in stage 8.0 (TID 97) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 89.0 in stage 8.0 (TID 96) in 8 ms on localhost (90/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 90.0 in stage 8.0 (TID 97). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 91.0 in stage 8.0 (TID 98, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 91.0 in stage 8.0 (TID 98) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 90.0 in stage 8.0 (TID 97) in 8 ms on localhost (91/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 91.0 in stage 8.0 (TID 98). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 92.0 in stage 8.0 (TID 99, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 92.0 in stage 8.0 (TID 99) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 91.0 in stage 8.0 (TID 98) in 8 ms on localhost (92/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 92.0 in stage 8.0 (TID 99). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 93.0 in stage 8.0 (TID 100, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 93.0 in stage 8.0 (TID 100) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 92.0 in stage 8.0 (TID 99) in 9 ms on localhost (93/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 93.0 in stage 8.0 (TID 100). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 94.0 in stage 8.0 (TID 101, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 94.0 in stage 8.0 (TID 101) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 93.0 in stage 8.0 (TID 100) in 9 ms on localhost (94/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 94.0 in stage 8.0 (TID 101). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 95.0 in stage 8.0 (TID 102, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 95.0 in stage 8.0 (TID 102) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 94.0 in stage 8.0 (TID 101) in 8 ms on localhost (95/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 95.0 in stage 8.0 (TID 102). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 96.0 in stage 8.0 (TID 103, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 96.0 in stage 8.0 (TID 103) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 95.0 in stage 8.0 (TID 102) in 9 ms on localhost (96/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 96.0 in stage 8.0 (TID 103). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 97.0 in stage 8.0 (TID 104, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 97.0 in stage 8.0 (TID 104) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 96.0 in stage 8.0 (TID 103) in 9 ms on localhost (97/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 97.0 in stage 8.0 (TID 104). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 98.0 in stage 8.0 (TID 105, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 98.0 in stage 8.0 (TID 105) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 97.0 in stage 8.0 (TID 104) in 9 ms on localhost (98/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 98.0 in stage 8.0 (TID 105). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 99.0 in stage 8.0 (TID 106, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 99.0 in stage 8.0 (TID 106) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 98.0 in stage 8.0 (TID 105) in 9 ms on localhost (99/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 99.0 in stage 8.0 (TID 106). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 100.0 in stage 8.0 (TID 107, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 100.0 in stage 8.0 (TID 107) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 99.0 in stage 8.0 (TID 106) in 10 ms on localhost (100/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 100.0 in stage 8.0 (TID 107). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 101.0 in stage 8.0 (TID 108, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 101.0 in stage 8.0 (TID 108) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 100.0 in stage 8.0 (TID 107) in 9 ms on localhost (101/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 101.0 in stage 8.0 (TID 108). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 102.0 in stage 8.0 (TID 109, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 102.0 in stage 8.0 (TID 109) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 101.0 in stage 8.0 (TID 108) in 10 ms on localhost (102/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 102.0 in stage 8.0 (TID 109). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 103.0 in stage 8.0 (TID 110, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 103.0 in stage 8.0 (TID 110) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 102.0 in stage 8.0 (TID 109) in 12 ms on localhost (103/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 103.0 in stage 8.0 (TID 110). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 104.0 in stage 8.0 (TID 111, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 104.0 in stage 8.0 (TID 111) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 103.0 in stage 8.0 (TID 110) in 12 ms on localhost (104/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 104.0 in stage 8.0 (TID 111). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 105.0 in stage 8.0 (TID 112, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 105.0 in stage 8.0 (TID 112) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 104.0 in stage 8.0 (TID 111) in 8 ms on localhost (105/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 105.0 in stage 8.0 (TID 112). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 106.0 in stage 8.0 (TID 113, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 106.0 in stage 8.0 (TID 113) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 105.0 in stage 8.0 (TID 112) in 8 ms on localhost (106/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 106.0 in stage 8.0 (TID 113). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 107.0 in stage 8.0 (TID 114, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 107.0 in stage 8.0 (TID 114) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 106.0 in stage 8.0 (TID 113) in 12 ms on localhost (107/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 107.0 in stage 8.0 (TID 114). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 108.0 in stage 8.0 (TID 115, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 108.0 in stage 8.0 (TID 115) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 107.0 in stage 8.0 (TID 114) in 10 ms on localhost (108/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 108.0 in stage 8.0 (TID 115). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 109.0 in stage 8.0 (TID 116, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 109.0 in stage 8.0 (TID 116) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 108.0 in stage 8.0 (TID 115) in 12 ms on localhost (109/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 109.0 in stage 8.0 (TID 116). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 110.0 in stage 8.0 (TID 117, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 110.0 in stage 8.0 (TID 117) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 109.0 in stage 8.0 (TID 116) in 9 ms on localhost (110/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 110.0 in stage 8.0 (TID 117). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 111.0 in stage 8.0 (TID 118, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 111.0 in stage 8.0 (TID 118) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 110.0 in stage 8.0 (TID 117) in 9 ms on localhost (111/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 111.0 in stage 8.0 (TID 118). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 112.0 in stage 8.0 (TID 119, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 112.0 in stage 8.0 (TID 119) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 111.0 in stage 8.0 (TID 118) in 9 ms on localhost (112/199) -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:37 INFO Executor:59 - Finished task 112.0 in stage 8.0 (TID 119). 1437 bytes result sent to driver -2016-06-13 15:19:37 INFO TaskSetManager:59 - Starting task 113.0 in stage 8.0 (TID 120, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:37 INFO Executor:59 - Running task 113.0 in stage 8.0 (TID 120) -2016-06-13 15:19:37 INFO TaskSetManager:59 - Finished task 112.0 in stage 8.0 (TID 119) in 11 ms on localhost (113/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 113.0 in stage 8.0 (TID 120). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 114.0 in stage 8.0 (TID 121, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 114.0 in stage 8.0 (TID 121) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 113.0 in stage 8.0 (TID 120) in 12 ms on localhost (114/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 114.0 in stage 8.0 (TID 121). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 115.0 in stage 8.0 (TID 122, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 115.0 in stage 8.0 (TID 122) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 114.0 in stage 8.0 (TID 121) in 9 ms on localhost (115/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 115.0 in stage 8.0 (TID 122). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 116.0 in stage 8.0 (TID 123, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 116.0 in stage 8.0 (TID 123) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 115.0 in stage 8.0 (TID 122) in 8 ms on localhost (116/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 116.0 in stage 8.0 (TID 123). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 117.0 in stage 8.0 (TID 124, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 117.0 in stage 8.0 (TID 124) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 116.0 in stage 8.0 (TID 123) in 8 ms on localhost (117/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 117.0 in stage 8.0 (TID 124). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 118.0 in stage 8.0 (TID 125, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 118.0 in stage 8.0 (TID 125) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 117.0 in stage 8.0 (TID 124) in 9 ms on localhost (118/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 118.0 in stage 8.0 (TID 125). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 119.0 in stage 8.0 (TID 126, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 119.0 in stage 8.0 (TID 126) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 118.0 in stage 8.0 (TID 125) in 9 ms on localhost (119/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 119.0 in stage 8.0 (TID 126). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 120.0 in stage 8.0 (TID 127, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 119.0 in stage 8.0 (TID 126) in 8 ms on localhost (120/199) -2016-06-13 15:19:38 INFO Executor:59 - Running task 120.0 in stage 8.0 (TID 127) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 120.0 in stage 8.0 (TID 127). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 121.0 in stage 8.0 (TID 128, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 120.0 in stage 8.0 (TID 127) in 8 ms on localhost (121/199) -2016-06-13 15:19:38 INFO Executor:59 - Running task 121.0 in stage 8.0 (TID 128) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 121.0 in stage 8.0 (TID 128). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 122.0 in stage 8.0 (TID 129, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 122.0 in stage 8.0 (TID 129) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 121.0 in stage 8.0 (TID 128) in 8 ms on localhost (122/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 122.0 in stage 8.0 (TID 129). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 123.0 in stage 8.0 (TID 130, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 122.0 in stage 8.0 (TID 129) in 8 ms on localhost (123/199) -2016-06-13 15:19:38 INFO Executor:59 - Running task 123.0 in stage 8.0 (TID 130) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 123.0 in stage 8.0 (TID 130). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 124.0 in stage 8.0 (TID 131, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 123.0 in stage 8.0 (TID 130) in 8 ms on localhost (124/199) -2016-06-13 15:19:38 INFO Executor:59 - Running task 124.0 in stage 8.0 (TID 131) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 124.0 in stage 8.0 (TID 131). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 125.0 in stage 8.0 (TID 132, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 124.0 in stage 8.0 (TID 131) in 9 ms on localhost (125/199) -2016-06-13 15:19:38 INFO Executor:59 - Running task 125.0 in stage 8.0 (TID 132) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 125.0 in stage 8.0 (TID 132). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 126.0 in stage 8.0 (TID 133, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 125.0 in stage 8.0 (TID 132) in 8 ms on localhost (126/199) -2016-06-13 15:19:38 INFO Executor:59 - Running task 126.0 in stage 8.0 (TID 133) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 126.0 in stage 8.0 (TID 133). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 127.0 in stage 8.0 (TID 134, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 127.0 in stage 8.0 (TID 134) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 126.0 in stage 8.0 (TID 133) in 8 ms on localhost (127/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 127.0 in stage 8.0 (TID 134). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 128.0 in stage 8.0 (TID 135, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 128.0 in stage 8.0 (TID 135) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 127.0 in stage 8.0 (TID 134) in 8 ms on localhost (128/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 128.0 in stage 8.0 (TID 135). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 129.0 in stage 8.0 (TID 136, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 129.0 in stage 8.0 (TID 136) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 128.0 in stage 8.0 (TID 135) in 8 ms on localhost (129/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 129.0 in stage 8.0 (TID 136). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 130.0 in stage 8.0 (TID 137, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 130.0 in stage 8.0 (TID 137) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 129.0 in stage 8.0 (TID 136) in 11 ms on localhost (130/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 130.0 in stage 8.0 (TID 137). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 131.0 in stage 8.0 (TID 138, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 131.0 in stage 8.0 (TID 138) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 130.0 in stage 8.0 (TID 137) in 8 ms on localhost (131/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 131.0 in stage 8.0 (TID 138). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 132.0 in stage 8.0 (TID 139, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 132.0 in stage 8.0 (TID 139) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 131.0 in stage 8.0 (TID 138) in 7 ms on localhost (132/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 132.0 in stage 8.0 (TID 139). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 133.0 in stage 8.0 (TID 140, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 133.0 in stage 8.0 (TID 140) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 132.0 in stage 8.0 (TID 139) in 8 ms on localhost (133/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 133.0 in stage 8.0 (TID 140). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 134.0 in stage 8.0 (TID 141, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 134.0 in stage 8.0 (TID 141) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 133.0 in stage 8.0 (TID 140) in 8 ms on localhost (134/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 134.0 in stage 8.0 (TID 141). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 135.0 in stage 8.0 (TID 142, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 135.0 in stage 8.0 (TID 142) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 134.0 in stage 8.0 (TID 141) in 8 ms on localhost (135/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 135.0 in stage 8.0 (TID 142). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 136.0 in stage 8.0 (TID 143, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 135.0 in stage 8.0 (TID 142) in 8 ms on localhost (136/199) -2016-06-13 15:19:38 INFO Executor:59 - Running task 136.0 in stage 8.0 (TID 143) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 136.0 in stage 8.0 (TID 143). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 137.0 in stage 8.0 (TID 144, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 136.0 in stage 8.0 (TID 143) in 8 ms on localhost (137/199) -2016-06-13 15:19:38 INFO Executor:59 - Running task 137.0 in stage 8.0 (TID 144) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 137.0 in stage 8.0 (TID 144). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 138.0 in stage 8.0 (TID 145, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 138.0 in stage 8.0 (TID 145) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 137.0 in stage 8.0 (TID 144) in 9 ms on localhost (138/199) -2016-06-13 15:19:38 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:57600 in memory (size: 5.5 KB, free: 730.6 MB) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 138.0 in stage 8.0 (TID 145). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 139.0 in stage 8.0 (TID 146, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 139.0 in stage 8.0 (TID 146) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 138.0 in stage 8.0 (TID 145) in 43 ms on localhost (139/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 139.0 in stage 8.0 (TID 146). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 140.0 in stage 8.0 (TID 147, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 140.0 in stage 8.0 (TID 147) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 139.0 in stage 8.0 (TID 146) in 10 ms on localhost (140/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 140.0 in stage 8.0 (TID 147). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 141.0 in stage 8.0 (TID 148, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 141.0 in stage 8.0 (TID 148) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 140.0 in stage 8.0 (TID 147) in 8 ms on localhost (141/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 141.0 in stage 8.0 (TID 148). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 142.0 in stage 8.0 (TID 149, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 142.0 in stage 8.0 (TID 149) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 141.0 in stage 8.0 (TID 148) in 8 ms on localhost (142/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 142.0 in stage 8.0 (TID 149). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 143.0 in stage 8.0 (TID 150, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 143.0 in stage 8.0 (TID 150) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 142.0 in stage 8.0 (TID 149) in 8 ms on localhost (143/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 143.0 in stage 8.0 (TID 150). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 144.0 in stage 8.0 (TID 151, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 144.0 in stage 8.0 (TID 151) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 143.0 in stage 8.0 (TID 150) in 8 ms on localhost (144/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 144.0 in stage 8.0 (TID 151). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 145.0 in stage 8.0 (TID 152, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 145.0 in stage 8.0 (TID 152) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 144.0 in stage 8.0 (TID 151) in 8 ms on localhost (145/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 145.0 in stage 8.0 (TID 152). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 146.0 in stage 8.0 (TID 153, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 146.0 in stage 8.0 (TID 153) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 145.0 in stage 8.0 (TID 152) in 7 ms on localhost (146/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 146.0 in stage 8.0 (TID 153). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 147.0 in stage 8.0 (TID 154, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 147.0 in stage 8.0 (TID 154) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 146.0 in stage 8.0 (TID 153) in 10 ms on localhost (147/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 147.0 in stage 8.0 (TID 154). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 148.0 in stage 8.0 (TID 155, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 148.0 in stage 8.0 (TID 155) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 147.0 in stage 8.0 (TID 154) in 9 ms on localhost (148/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 148.0 in stage 8.0 (TID 155). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 149.0 in stage 8.0 (TID 156, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 149.0 in stage 8.0 (TID 156) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 148.0 in stage 8.0 (TID 155) in 8 ms on localhost (149/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 149.0 in stage 8.0 (TID 156). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 150.0 in stage 8.0 (TID 157, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 150.0 in stage 8.0 (TID 157) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 149.0 in stage 8.0 (TID 156) in 9 ms on localhost (150/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 150.0 in stage 8.0 (TID 157). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 151.0 in stage 8.0 (TID 158, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 151.0 in stage 8.0 (TID 158) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 150.0 in stage 8.0 (TID 157) in 8 ms on localhost (151/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 151.0 in stage 8.0 (TID 158). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 152.0 in stage 8.0 (TID 159, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 152.0 in stage 8.0 (TID 159) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 151.0 in stage 8.0 (TID 158) in 8 ms on localhost (152/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 152.0 in stage 8.0 (TID 159). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 153.0 in stage 8.0 (TID 160, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 153.0 in stage 8.0 (TID 160) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 152.0 in stage 8.0 (TID 159) in 8 ms on localhost (153/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 153.0 in stage 8.0 (TID 160). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 154.0 in stage 8.0 (TID 161, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 154.0 in stage 8.0 (TID 161) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 153.0 in stage 8.0 (TID 160) in 7 ms on localhost (154/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 154.0 in stage 8.0 (TID 161). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 155.0 in stage 8.0 (TID 162, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 155.0 in stage 8.0 (TID 162) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 154.0 in stage 8.0 (TID 161) in 8 ms on localhost (155/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 155.0 in stage 8.0 (TID 162). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 156.0 in stage 8.0 (TID 163, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 156.0 in stage 8.0 (TID 163) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 155.0 in stage 8.0 (TID 162) in 8 ms on localhost (156/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 156.0 in stage 8.0 (TID 163). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 157.0 in stage 8.0 (TID 164, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 157.0 in stage 8.0 (TID 164) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 156.0 in stage 8.0 (TID 163) in 8 ms on localhost (157/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 157.0 in stage 8.0 (TID 164). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 158.0 in stage 8.0 (TID 165, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 158.0 in stage 8.0 (TID 165) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 157.0 in stage 8.0 (TID 164) in 8 ms on localhost (158/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 158.0 in stage 8.0 (TID 165). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 159.0 in stage 8.0 (TID 166, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 159.0 in stage 8.0 (TID 166) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 158.0 in stage 8.0 (TID 165) in 8 ms on localhost (159/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 159.0 in stage 8.0 (TID 166). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 160.0 in stage 8.0 (TID 167, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 160.0 in stage 8.0 (TID 167) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 159.0 in stage 8.0 (TID 166) in 7 ms on localhost (160/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 160.0 in stage 8.0 (TID 167). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 161.0 in stage 8.0 (TID 168, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 161.0 in stage 8.0 (TID 168) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 160.0 in stage 8.0 (TID 167) in 8 ms on localhost (161/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 161.0 in stage 8.0 (TID 168). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 162.0 in stage 8.0 (TID 169, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 162.0 in stage 8.0 (TID 169) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 161.0 in stage 8.0 (TID 168) in 8 ms on localhost (162/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 162.0 in stage 8.0 (TID 169). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 163.0 in stage 8.0 (TID 170, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 163.0 in stage 8.0 (TID 170) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 162.0 in stage 8.0 (TID 169) in 8 ms on localhost (163/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 163.0 in stage 8.0 (TID 170). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 164.0 in stage 8.0 (TID 171, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 164.0 in stage 8.0 (TID 171) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 163.0 in stage 8.0 (TID 170) in 8 ms on localhost (164/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 164.0 in stage 8.0 (TID 171). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 165.0 in stage 8.0 (TID 172, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 165.0 in stage 8.0 (TID 172) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 164.0 in stage 8.0 (TID 171) in 8 ms on localhost (165/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 165.0 in stage 8.0 (TID 172). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 166.0 in stage 8.0 (TID 173, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 166.0 in stage 8.0 (TID 173) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 165.0 in stage 8.0 (TID 172) in 8 ms on localhost (166/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 166.0 in stage 8.0 (TID 173). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 167.0 in stage 8.0 (TID 174, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 167.0 in stage 8.0 (TID 174) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 166.0 in stage 8.0 (TID 173) in 8 ms on localhost (167/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 167.0 in stage 8.0 (TID 174). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 168.0 in stage 8.0 (TID 175, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 168.0 in stage 8.0 (TID 175) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 167.0 in stage 8.0 (TID 174) in 7 ms on localhost (168/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 168.0 in stage 8.0 (TID 175). 1723 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 169.0 in stage 8.0 (TID 176, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 169.0 in stage 8.0 (TID 176) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 168.0 in stage 8.0 (TID 175) in 8 ms on localhost (169/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 169.0 in stage 8.0 (TID 176). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 170.0 in stage 8.0 (TID 177, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 170.0 in stage 8.0 (TID 177) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 169.0 in stage 8.0 (TID 176) in 8 ms on localhost (170/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 170.0 in stage 8.0 (TID 177). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 171.0 in stage 8.0 (TID 178, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 171.0 in stage 8.0 (TID 178) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 170.0 in stage 8.0 (TID 177) in 8 ms on localhost (171/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 171.0 in stage 8.0 (TID 178). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 172.0 in stage 8.0 (TID 179, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 171.0 in stage 8.0 (TID 178) in 8 ms on localhost (172/199) -2016-06-13 15:19:38 INFO Executor:59 - Running task 172.0 in stage 8.0 (TID 179) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 172.0 in stage 8.0 (TID 179). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 173.0 in stage 8.0 (TID 180, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 173.0 in stage 8.0 (TID 180) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 172.0 in stage 8.0 (TID 179) in 8 ms on localhost (173/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 173.0 in stage 8.0 (TID 180). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 174.0 in stage 8.0 (TID 181, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 174.0 in stage 8.0 (TID 181) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 173.0 in stage 8.0 (TID 180) in 8 ms on localhost (174/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 174.0 in stage 8.0 (TID 181). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 175.0 in stage 8.0 (TID 182, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 174.0 in stage 8.0 (TID 181) in 11 ms on localhost (175/199) -2016-06-13 15:19:38 INFO Executor:59 - Running task 175.0 in stage 8.0 (TID 182) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 175.0 in stage 8.0 (TID 182). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 176.0 in stage 8.0 (TID 183, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 176.0 in stage 8.0 (TID 183) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 175.0 in stage 8.0 (TID 182) in 9 ms on localhost (176/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 176.0 in stage 8.0 (TID 183). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 177.0 in stage 8.0 (TID 184, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 177.0 in stage 8.0 (TID 184) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 176.0 in stage 8.0 (TID 183) in 11 ms on localhost (177/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 177.0 in stage 8.0 (TID 184). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 178.0 in stage 8.0 (TID 185, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 178.0 in stage 8.0 (TID 185) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 177.0 in stage 8.0 (TID 184) in 10 ms on localhost (178/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 178.0 in stage 8.0 (TID 185). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 179.0 in stage 8.0 (TID 186, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 178.0 in stage 8.0 (TID 185) in 11 ms on localhost (179/199) -2016-06-13 15:19:38 INFO Executor:59 - Running task 179.0 in stage 8.0 (TID 186) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 179.0 in stage 8.0 (TID 186). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 180.0 in stage 8.0 (TID 187, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 180.0 in stage 8.0 (TID 187) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 179.0 in stage 8.0 (TID 186) in 9 ms on localhost (180/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 180.0 in stage 8.0 (TID 187). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 181.0 in stage 8.0 (TID 188, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 181.0 in stage 8.0 (TID 188) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 180.0 in stage 8.0 (TID 187) in 8 ms on localhost (181/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 181.0 in stage 8.0 (TID 188). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 182.0 in stage 8.0 (TID 189, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 182.0 in stage 8.0 (TID 189) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 181.0 in stage 8.0 (TID 188) in 7 ms on localhost (182/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 182.0 in stage 8.0 (TID 189). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 183.0 in stage 8.0 (TID 190, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 183.0 in stage 8.0 (TID 190) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 182.0 in stage 8.0 (TID 189) in 7 ms on localhost (183/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 183.0 in stage 8.0 (TID 190). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 184.0 in stage 8.0 (TID 191, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 184.0 in stage 8.0 (TID 191) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 183.0 in stage 8.0 (TID 190) in 7 ms on localhost (184/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 184.0 in stage 8.0 (TID 191). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 185.0 in stage 8.0 (TID 192, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 185.0 in stage 8.0 (TID 192) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 184.0 in stage 8.0 (TID 191) in 8 ms on localhost (185/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 185.0 in stage 8.0 (TID 192). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 186.0 in stage 8.0 (TID 193, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 186.0 in stage 8.0 (TID 193) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 185.0 in stage 8.0 (TID 192) in 8 ms on localhost (186/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 186.0 in stage 8.0 (TID 193). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 187.0 in stage 8.0 (TID 194, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 187.0 in stage 8.0 (TID 194) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 186.0 in stage 8.0 (TID 193) in 8 ms on localhost (187/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 187.0 in stage 8.0 (TID 194). 1736 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 188.0 in stage 8.0 (TID 195, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 188.0 in stage 8.0 (TID 195) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 187.0 in stage 8.0 (TID 194) in 9 ms on localhost (188/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 188.0 in stage 8.0 (TID 195). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 189.0 in stage 8.0 (TID 196, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 189.0 in stage 8.0 (TID 196) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 188.0 in stage 8.0 (TID 195) in 7 ms on localhost (189/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 189.0 in stage 8.0 (TID 196). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 190.0 in stage 8.0 (TID 197, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 189.0 in stage 8.0 (TID 196) in 8 ms on localhost (190/199) -2016-06-13 15:19:38 INFO Executor:59 - Running task 190.0 in stage 8.0 (TID 197) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 190.0 in stage 8.0 (TID 197). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 191.0 in stage 8.0 (TID 198, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 191.0 in stage 8.0 (TID 198) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 190.0 in stage 8.0 (TID 197) in 9 ms on localhost (191/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 191.0 in stage 8.0 (TID 198). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 192.0 in stage 8.0 (TID 199, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 192.0 in stage 8.0 (TID 199) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 191.0 in stage 8.0 (TID 198) in 8 ms on localhost (192/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 192.0 in stage 8.0 (TID 199). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 193.0 in stage 8.0 (TID 200, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 193.0 in stage 8.0 (TID 200) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 192.0 in stage 8.0 (TID 199) in 8 ms on localhost (193/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 193.0 in stage 8.0 (TID 200). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 194.0 in stage 8.0 (TID 201, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 194.0 in stage 8.0 (TID 201) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 193.0 in stage 8.0 (TID 200) in 7 ms on localhost (194/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 194.0 in stage 8.0 (TID 201). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 195.0 in stage 8.0 (TID 202, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 195.0 in stage 8.0 (TID 202) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 194.0 in stage 8.0 (TID 201) in 7 ms on localhost (195/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 195.0 in stage 8.0 (TID 202). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 196.0 in stage 8.0 (TID 203, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 196.0 in stage 8.0 (TID 203) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 195.0 in stage 8.0 (TID 202) in 7 ms on localhost (196/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 196.0 in stage 8.0 (TID 203). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 197.0 in stage 8.0 (TID 204, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 196.0 in stage 8.0 (TID 203) in 8 ms on localhost (197/199) -2016-06-13 15:19:38 INFO Executor:59 - Running task 197.0 in stage 8.0 (TID 204) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 197.0 in stage 8.0 (TID 204). 1437 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Starting task 198.0 in stage 8.0 (TID 205, localhost, PROCESS_LOCAL, 1914 bytes) -2016-06-13 15:19:38 INFO Executor:59 - Running task 198.0 in stage 8.0 (TID 205) -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 197.0 in stage 8.0 (TID 204) in 8 ms on localhost (198/199) -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 15:19:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 15:19:38 INFO Executor:59 - Finished task 198.0 in stage 8.0 (TID 205). 1736 bytes result sent to driver -2016-06-13 15:19:38 INFO TaskSetManager:59 - Finished task 198.0 in stage 8.0 (TID 205) in 7 ms on localhost (199/199) -2016-06-13 15:19:38 INFO DAGScheduler:59 - ResultStage 8 (main at null:-2) finished in 2.074 s -2016-06-13 15:19:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 15:19:38 INFO DAGScheduler:59 - Job 6 finished: main at null:-2, took 2.118376 s -2016-06-13 15:19:38 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:19:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:19:38 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 15:19:38 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 15:19:38 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 15:19:39 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 15:19:39 INFO BlockManager:59 - BlockManager stopped -2016-06-13 15:19:39 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 15:19:39 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 15:19:39 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:19:39 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 15:19:39 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-b43c5227-6c52-411f-87f4-1dd04d3ae58c -2016-06-13 15:19:39 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:19:39 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:29:25 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:29:25 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:29:26 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:29:26 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:29:26 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:29:26 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:29:26 INFO Remoting:74 - Starting remoting -2016-06-13 15:29:26 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:57648] -2016-06-13 15:29:26 INFO Utils:59 - Successfully started service 'sparkDriver' on port 57648. -2016-06-13 15:29:26 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 15:29:26 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 15:29:27 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-fbd93e21-b758-4400-a5b8-2362d6c587f6 -2016-06-13 15:29:27 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 15:29:27 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-4c39f055-fa0d-4aad-add1-c71748a3d76d\httpd-dc49a112-c520-4a00-b434-9b3beeb6f14b -2016-06-13 15:29:27 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 15:29:27 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:29:27 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:57649 -2016-06-13 15:29:27 INFO Utils:59 - Successfully started service 'HTTP file server' on port 57649. -2016-06-13 15:29:27 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 15:29:27 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:29:27 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:29:27 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 15:29:27 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 15:29:27 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 15:29:27 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 15:29:27 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 57660. -2016-06-13 15:29:27 INFO NettyBlockTransferService:59 - Server created on 57660 -2016-06-13 15:29:27 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 15:29:27 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:57660 with 730.6 MB RAM, BlockManagerId(driver, localhost, 57660) -2016-06-13 15:29:27 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 15:29:28 INFO SparkContext:59 - Starting job: count at testFunSuite.scala:16 -2016-06-13 15:29:28 INFO DAGScheduler:59 - Got job 0 (count at testFunSuite.scala:16) with 2 output partitions -2016-06-13 15:29:28 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at testFunSuite.scala:16) -2016-06-13 15:29:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:29:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:29:28 INFO DAGScheduler:59 - Submitting ResultStage 0 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:15), which has no missing parents -2016-06-13 15:29:28 INFO MemoryStore:59 - ensureFreeSpace(1096) called with curMem=0, maxMem=766075207 -2016-06-13 15:29:28 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1096.0 B, free 730.6 MB) -2016-06-13 15:29:28 INFO MemoryStore:59 - ensureFreeSpace(804) called with curMem=1096, maxMem=766075207 -2016-06-13 15:29:28 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 804.0 B, free 730.6 MB) -2016-06-13 15:29:28 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:57660 (size: 804.0 B, free: 730.6 MB) -2016-06-13 15:29:28 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:29:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:15) -2016-06-13 15:29:28 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 15:29:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:29:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2033 bytes) -2016-06-13 15:29:28 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 15:29:28 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 15:29:28 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 953 bytes result sent to driver -2016-06-13 15:29:28 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 953 bytes result sent to driver -2016-06-13 15:29:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 65 ms on localhost (1/2) -2016-06-13 15:29:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 43 ms on localhost (2/2) -2016-06-13 15:29:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 15:29:28 INFO DAGScheduler:59 - ResultStage 0 (count at testFunSuite.scala:16) finished in 0.078 s -2016-06-13 15:29:28 INFO DAGScheduler:59 - Job 0 finished: count at testFunSuite.scala:16, took 0.319186 s -2016-06-13 15:29:28 INFO SparkContext:59 - Starting job: foreach at testFunSuite.scala:17 -2016-06-13 15:29:28 INFO DAGScheduler:59 - Got job 1 (foreach at testFunSuite.scala:17) with 2 output partitions -2016-06-13 15:29:28 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at testFunSuite.scala:17) -2016-06-13 15:29:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:29:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:29:28 INFO DAGScheduler:59 - Submitting ResultStage 1 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:15), which has no missing parents -2016-06-13 15:29:28 INFO MemoryStore:59 - ensureFreeSpace(1232) called with curMem=1900, maxMem=766075207 -2016-06-13 15:29:28 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1232.0 B, free 730.6 MB) -2016-06-13 15:29:28 INFO MemoryStore:59 - ensureFreeSpace(887) called with curMem=3132, maxMem=766075207 -2016-06-13 15:29:28 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 887.0 B, free 730.6 MB) -2016-06-13 15:29:28 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:57660 (size: 887.0 B, free: 730.6 MB) -2016-06-13 15:29:28 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:29:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:15) -2016-06-13 15:29:28 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 15:29:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:29:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2033 bytes) -2016-06-13 15:29:28 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 15:29:28 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 15:29:28 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 915 bytes result sent to driver -2016-06-13 15:29:28 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 915 bytes result sent to driver -2016-06-13 15:29:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 12 ms on localhost (1/2) -2016-06-13 15:29:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 15 ms on localhost (2/2) -2016-06-13 15:29:28 INFO DAGScheduler:59 - ResultStage 1 (foreach at testFunSuite.scala:17) finished in 0.015 s -2016-06-13 15:29:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 15:29:28 INFO DAGScheduler:59 - Job 1 finished: foreach at testFunSuite.scala:17, took 0.026418 s -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:29:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:29:28 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 15:29:28 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 15:29:28 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 15:29:28 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 15:29:28 INFO BlockManager:59 - BlockManager stopped -2016-06-13 15:29:28 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 15:29:28 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 15:29:28 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:29:29 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:29:29 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:29:29 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 15:29:29 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-4c39f055-fa0d-4aad-add1-c71748a3d76d -2016-06-13 15:30:53 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:30:54 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:30:54 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:30:54 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:30:54 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:30:55 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:30:55 INFO Remoting:74 - Starting remoting -2016-06-13 15:30:55 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:57679] -2016-06-13 15:30:55 INFO Utils:59 - Successfully started service 'sparkDriver' on port 57679. -2016-06-13 15:30:55 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 15:30:55 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 15:30:55 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-fe8fd57e-6d7d-4e2a-bea4-7c01680d3772 -2016-06-13 15:30:55 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 15:30:55 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-ca0347c7-f975-4428-ab55-90358a54be57\httpd-eb18f0ec-ecfc-44ae-9a38-ba492a398b34 -2016-06-13 15:30:55 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 15:30:55 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:30:55 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:57680 -2016-06-13 15:30:55 INFO Utils:59 - Successfully started service 'HTTP file server' on port 57680. -2016-06-13 15:30:55 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 15:30:55 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:30:55 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:30:55 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 15:30:55 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 15:30:55 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 15:30:55 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 15:30:56 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 57691. -2016-06-13 15:30:56 INFO NettyBlockTransferService:59 - Server created on 57691 -2016-06-13 15:30:56 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 15:30:56 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:57691 with 730.6 MB RAM, BlockManagerId(driver, localhost, 57691) -2016-06-13 15:30:56 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 15:30:56 INFO SparkContext:59 - Starting job: count at testFunSuite.scala:17 -2016-06-13 15:30:56 INFO DAGScheduler:59 - Got job 0 (count at testFunSuite.scala:17) with 2 output partitions -2016-06-13 15:30:56 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at testFunSuite.scala:17) -2016-06-13 15:30:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:30:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:30:56 INFO DAGScheduler:59 - Submitting ResultStage 0 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:16), which has no missing parents -2016-06-13 15:30:56 INFO MemoryStore:59 - ensureFreeSpace(1096) called with curMem=0, maxMem=766075207 -2016-06-13 15:30:56 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1096.0 B, free 730.6 MB) -2016-06-13 15:30:56 INFO MemoryStore:59 - ensureFreeSpace(804) called with curMem=1096, maxMem=766075207 -2016-06-13 15:30:56 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 804.0 B, free 730.6 MB) -2016-06-13 15:30:56 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:57691 (size: 804.0 B, free: 730.6 MB) -2016-06-13 15:30:56 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:30:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:16) -2016-06-13 15:30:56 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 15:30:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:30:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2033 bytes) -2016-06-13 15:30:56 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 15:30:56 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 15:30:56 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 953 bytes result sent to driver -2016-06-13 15:30:56 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 953 bytes result sent to driver -2016-06-13 15:30:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 63 ms on localhost (1/2) -2016-06-13 15:30:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 39 ms on localhost (2/2) -2016-06-13 15:30:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 15:30:56 INFO DAGScheduler:59 - ResultStage 0 (count at testFunSuite.scala:17) finished in 0.076 s -2016-06-13 15:30:56 INFO DAGScheduler:59 - Job 0 finished: count at testFunSuite.scala:17, took 0.251935 s -2016-06-13 15:30:57 INFO SparkContext:59 - Starting job: foreach at testFunSuite.scala:18 -2016-06-13 15:30:57 INFO DAGScheduler:59 - Got job 1 (foreach at testFunSuite.scala:18) with 2 output partitions -2016-06-13 15:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at testFunSuite.scala:18) -2016-06-13 15:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:30:57 INFO DAGScheduler:59 - Submitting ResultStage 1 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:16), which has no missing parents -2016-06-13 15:30:57 INFO MemoryStore:59 - ensureFreeSpace(1232) called with curMem=1900, maxMem=766075207 -2016-06-13 15:30:57 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1232.0 B, free 730.6 MB) -2016-06-13 15:30:57 INFO MemoryStore:59 - ensureFreeSpace(887) called with curMem=3132, maxMem=766075207 -2016-06-13 15:30:57 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 887.0 B, free 730.6 MB) -2016-06-13 15:30:57 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:57691 (size: 887.0 B, free: 730.6 MB) -2016-06-13 15:30:57 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:16) -2016-06-13 15:30:57 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 15:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2033 bytes) -2016-06-13 15:30:57 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 15:30:57 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 15:30:57 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 915 bytes result sent to driver -2016-06-13 15:30:57 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 915 bytes result sent to driver -2016-06-13 15:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 10 ms on localhost (1/2) -2016-06-13 15:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 13 ms on localhost (2/2) -2016-06-13 15:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 15:30:57 INFO DAGScheduler:59 - ResultStage 1 (foreach at testFunSuite.scala:18) finished in 0.013 s -2016-06-13 15:30:57 INFO DAGScheduler:59 - Job 1 finished: foreach at testFunSuite.scala:18, took 0.026386 s -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:30:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:30:57 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 15:30:57 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 15:30:57 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 15:30:57 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 15:30:57 INFO BlockManager:59 - BlockManager stopped -2016-06-13 15:30:57 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 15:30:57 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 15:30:57 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:30:57 INFO SparkContext:59 - SparkContext already stopped. -2016-06-13 15:30:57 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:30:57 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:30:57 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 15:30:57 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-ca0347c7-f975-4428-ab55-90358a54be57 -2016-06-13 15:31:19 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:31:19 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:31:20 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:31:20 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:31:20 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:31:20 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:31:20 INFO Remoting:74 - Starting remoting -2016-06-13 15:31:21 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:57708] -2016-06-13 15:31:21 INFO Utils:59 - Successfully started service 'sparkDriver' on port 57708. -2016-06-13 15:31:21 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 15:31:21 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 15:31:21 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-94b11ca8-a3fb-48a8-9fe6-d16ef74a0738 -2016-06-13 15:31:21 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 15:31:21 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-c2d15d9d-33d3-4c41-a46d-1ee60d1083c7\httpd-e2779b87-2a8d-471f-a6e6-ba9ff1e68449 -2016-06-13 15:31:21 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 15:31:21 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:31:21 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:57709 -2016-06-13 15:31:21 INFO Utils:59 - Successfully started service 'HTTP file server' on port 57709. -2016-06-13 15:31:21 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 15:31:21 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:31:21 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:31:21 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 15:31:21 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 15:31:21 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 15:31:21 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 15:31:21 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 57720. -2016-06-13 15:31:21 INFO NettyBlockTransferService:59 - Server created on 57720 -2016-06-13 15:31:21 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 15:31:21 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:57720 with 730.6 MB RAM, BlockManagerId(driver, localhost, 57720) -2016-06-13 15:31:21 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 15:31:22 INFO SparkContext:59 - Starting job: count at testFunSuite.scala:17 -2016-06-13 15:31:22 INFO DAGScheduler:59 - Got job 0 (count at testFunSuite.scala:17) with 2 output partitions -2016-06-13 15:31:22 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at testFunSuite.scala:17) -2016-06-13 15:31:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:31:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:31:22 INFO DAGScheduler:59 - Submitting ResultStage 0 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:16), which has no missing parents -2016-06-13 15:31:22 INFO MemoryStore:59 - ensureFreeSpace(1096) called with curMem=0, maxMem=766075207 -2016-06-13 15:31:22 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1096.0 B, free 730.6 MB) -2016-06-13 15:31:22 INFO MemoryStore:59 - ensureFreeSpace(804) called with curMem=1096, maxMem=766075207 -2016-06-13 15:31:22 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 804.0 B, free 730.6 MB) -2016-06-13 15:31:22 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:57720 (size: 804.0 B, free: 730.6 MB) -2016-06-13 15:31:22 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:31:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:16) -2016-06-13 15:31:22 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 15:31:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:31:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2033 bytes) -2016-06-13 15:31:22 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 15:31:22 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 15:31:22 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 953 bytes result sent to driver -2016-06-13 15:31:22 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 953 bytes result sent to driver -2016-06-13 15:31:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 38 ms on localhost (1/2) -2016-06-13 15:31:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 64 ms on localhost (2/2) -2016-06-13 15:31:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 15:31:22 INFO DAGScheduler:59 - ResultStage 0 (count at testFunSuite.scala:17) finished in 0.074 s -2016-06-13 15:31:22 INFO DAGScheduler:59 - Job 0 finished: count at testFunSuite.scala:17, took 0.268981 s -2016-06-13 15:31:22 INFO SparkContext:59 - Starting job: foreach at testFunSuite.scala:18 -2016-06-13 15:31:22 INFO DAGScheduler:59 - Got job 1 (foreach at testFunSuite.scala:18) with 2 output partitions -2016-06-13 15:31:22 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at testFunSuite.scala:18) -2016-06-13 15:31:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:31:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:31:22 INFO DAGScheduler:59 - Submitting ResultStage 1 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:16), which has no missing parents -2016-06-13 15:31:22 INFO MemoryStore:59 - ensureFreeSpace(1232) called with curMem=1900, maxMem=766075207 -2016-06-13 15:31:22 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1232.0 B, free 730.6 MB) -2016-06-13 15:31:22 INFO MemoryStore:59 - ensureFreeSpace(887) called with curMem=3132, maxMem=766075207 -2016-06-13 15:31:22 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 887.0 B, free 730.6 MB) -2016-06-13 15:31:22 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:57720 (size: 887.0 B, free: 730.6 MB) -2016-06-13 15:31:22 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:31:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:16) -2016-06-13 15:31:22 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 15:31:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:31:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2033 bytes) -2016-06-13 15:31:22 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 15:31:22 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 15:31:22 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 915 bytes result sent to driver -2016-06-13 15:31:22 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 915 bytes result sent to driver -2016-06-13 15:31:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 12 ms on localhost (1/2) -2016-06-13 15:31:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 10 ms on localhost (2/2) -2016-06-13 15:31:22 INFO DAGScheduler:59 - ResultStage 1 (foreach at testFunSuite.scala:18) finished in 0.012 s -2016-06-13 15:31:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 15:31:22 INFO DAGScheduler:59 - Job 1 finished: foreach at testFunSuite.scala:18, took 0.024025 s -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:31:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:31:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:31:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:31:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:31:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:31:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:31:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:31:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:31:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:31:23 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 15:31:23 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 15:31:23 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 15:31:23 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 15:31:23 INFO BlockManager:59 - BlockManager stopped -2016-06-13 15:31:23 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 15:31:23 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 15:31:23 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:31:23 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:31:23 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:31:23 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 15:31:23 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-c2d15d9d-33d3-4c41-a46d-1ee60d1083c7 -2016-06-13 15:31:42 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:31:43 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:31:43 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:31:43 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:31:43 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:31:44 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:31:44 INFO Remoting:74 - Starting remoting -2016-06-13 15:31:44 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:57737] -2016-06-13 15:31:44 INFO Utils:59 - Successfully started service 'sparkDriver' on port 57737. -2016-06-13 15:31:44 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 15:31:44 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 15:31:44 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-bf923729-6976-4050-9c7b-d3527c2acff2 -2016-06-13 15:31:44 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 15:31:44 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-27f91b43-9868-45c0-9df6-14666606f1df\httpd-7baa67c8-2b25-4003-9cee-6ae415316c03 -2016-06-13 15:31:44 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 15:31:44 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:31:44 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:57738 -2016-06-13 15:31:44 INFO Utils:59 - Successfully started service 'HTTP file server' on port 57738. -2016-06-13 15:31:44 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 15:31:44 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:31:44 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:31:44 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 15:31:45 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 15:31:45 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 15:31:45 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 15:31:45 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 57757. -2016-06-13 15:31:45 INFO NettyBlockTransferService:59 - Server created on 57757 -2016-06-13 15:31:45 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 15:31:45 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:57757 with 730.6 MB RAM, BlockManagerId(driver, localhost, 57757) -2016-06-13 15:31:45 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 15:31:46 INFO SparkContext:59 - Starting job: count at testFunSuite.scala:17 -2016-06-13 15:31:46 INFO DAGScheduler:59 - Got job 0 (count at testFunSuite.scala:17) with 4 output partitions -2016-06-13 15:31:46 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at testFunSuite.scala:17) -2016-06-13 15:31:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:31:46 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:31:46 INFO DAGScheduler:59 - Submitting ResultStage 0 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:16), which has no missing parents -2016-06-13 15:31:46 INFO MemoryStore:59 - ensureFreeSpace(1096) called with curMem=0, maxMem=766075207 -2016-06-13 15:31:46 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1096.0 B, free 730.6 MB) -2016-06-13 15:31:46 INFO MemoryStore:59 - ensureFreeSpace(804) called with curMem=1096, maxMem=766075207 -2016-06-13 15:31:46 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 804.0 B, free 730.6 MB) -2016-06-13 15:31:46 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:57757 (size: 804.0 B, free: 730.6 MB) -2016-06-13 15:31:46 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:31:46 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 0 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:16) -2016-06-13 15:31:46 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 4 tasks -2016-06-13 15:31:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2025 bytes) -2016-06-13 15:31:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:31:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:31:46 INFO TaskSetManager:59 - Starting task 3.0 in stage 0.0 (TID 3, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:31:46 INFO Executor:59 - Running task 3.0 in stage 0.0 (TID 3) -2016-06-13 15:31:46 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 15:31:46 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-06-13 15:31:46 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 15:31:46 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 953 bytes result sent to driver -2016-06-13 15:31:46 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 953 bytes result sent to driver -2016-06-13 15:31:46 INFO Executor:59 - Finished task 3.0 in stage 0.0 (TID 3). 953 bytes result sent to driver -2016-06-13 15:31:46 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 953 bytes result sent to driver -2016-06-13 15:31:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 46 ms on localhost (1/4) -2016-06-13 15:31:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 0.0 (TID 3) in 46 ms on localhost (2/4) -2016-06-13 15:31:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 48 ms on localhost (3/4) -2016-06-13 15:31:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 81 ms on localhost (4/4) -2016-06-13 15:31:46 INFO DAGScheduler:59 - ResultStage 0 (count at testFunSuite.scala:17) finished in 0.090 s -2016-06-13 15:31:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 15:31:46 INFO DAGScheduler:59 - Job 0 finished: count at testFunSuite.scala:17, took 0.292342 s -2016-06-13 15:31:46 INFO SparkContext:59 - Starting job: foreach at testFunSuite.scala:18 -2016-06-13 15:31:46 INFO DAGScheduler:59 - Got job 1 (foreach at testFunSuite.scala:18) with 4 output partitions -2016-06-13 15:31:46 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at testFunSuite.scala:18) -2016-06-13 15:31:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:31:46 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:31:46 INFO DAGScheduler:59 - Submitting ResultStage 1 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:16), which has no missing parents -2016-06-13 15:31:46 INFO MemoryStore:59 - ensureFreeSpace(1232) called with curMem=1900, maxMem=766075207 -2016-06-13 15:31:46 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1232.0 B, free 730.6 MB) -2016-06-13 15:31:46 INFO MemoryStore:59 - ensureFreeSpace(887) called with curMem=3132, maxMem=766075207 -2016-06-13 15:31:46 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 887.0 B, free 730.6 MB) -2016-06-13 15:31:46 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:57757 (size: 887.0 B, free: 730.6 MB) -2016-06-13 15:31:46 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:31:46 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:16) -2016-06-13 15:31:46 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 4 tasks -2016-06-13 15:31:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2025 bytes) -2016-06-13 15:31:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 5, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:31:46 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 6, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:31:46 INFO TaskSetManager:59 - Starting task 3.0 in stage 1.0 (TID 7, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:31:46 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 5) -2016-06-13 15:31:46 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 6) -2016-06-13 15:31:46 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 4) -2016-06-13 15:31:46 INFO Executor:59 - Running task 3.0 in stage 1.0 (TID 7) -2016-06-13 15:31:46 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 6). 915 bytes result sent to driver -2016-06-13 15:31:46 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 4). 915 bytes result sent to driver -2016-06-13 15:31:46 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 6) in 13 ms on localhost (1/4) -2016-06-13 15:31:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 4) in 18 ms on localhost (2/4) -2016-06-13 15:31:46 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 5). 915 bytes result sent to driver -2016-06-13 15:31:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 5) in 19 ms on localhost (3/4) -2016-06-13 15:31:46 INFO Executor:59 - Finished task 3.0 in stage 1.0 (TID 7). 915 bytes result sent to driver -2016-06-13 15:31:46 INFO TaskSetManager:59 - Finished task 3.0 in stage 1.0 (TID 7) in 20 ms on localhost (4/4) -2016-06-13 15:31:46 INFO DAGScheduler:59 - ResultStage 1 (foreach at testFunSuite.scala:18) finished in 0.024 s -2016-06-13 15:31:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 15:31:46 INFO DAGScheduler:59 - Job 1 finished: foreach at testFunSuite.scala:18, took 0.039441 s -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:31:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:31:46 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 15:31:46 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 15:31:46 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 15:31:46 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 15:31:46 INFO BlockManager:59 - BlockManager stopped -2016-06-13 15:31:46 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 15:31:46 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 15:31:46 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:31:46 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:31:46 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:31:46 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 15:31:46 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-27f91b43-9868-45c0-9df6-14666606f1df -2016-06-13 15:32:15 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:32:16 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:32:16 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:32:16 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:32:16 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:32:16 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:32:17 INFO Remoting:74 - Starting remoting -2016-06-13 15:32:17 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:57774] -2016-06-13 15:32:17 INFO Utils:59 - Successfully started service 'sparkDriver' on port 57774. -2016-06-13 15:32:17 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 15:32:17 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 15:32:17 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-1ec069f4-5920-4d6d-81c9-97298a435707 -2016-06-13 15:32:17 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 15:32:17 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-db4e4b40-384d-4cad-9812-3e96b3fea340\httpd-28374c79-2902-4ea6-8bad-471c9a0dcd56 -2016-06-13 15:32:17 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 15:32:17 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:32:17 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:57775 -2016-06-13 15:32:17 INFO Utils:59 - Successfully started service 'HTTP file server' on port 57775. -2016-06-13 15:32:17 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 15:32:17 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:32:17 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:32:17 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 15:32:17 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 15:32:17 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 15:32:17 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 15:32:18 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 57794. -2016-06-13 15:32:18 INFO NettyBlockTransferService:59 - Server created on 57794 -2016-06-13 15:32:18 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 15:32:18 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:57794 with 730.6 MB RAM, BlockManagerId(driver, localhost, 57794) -2016-06-13 15:32:18 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 15:32:18 INFO SparkContext:59 - Starting job: count at testFunSuite.scala:17 -2016-06-13 15:32:18 INFO DAGScheduler:59 - Got job 0 (count at testFunSuite.scala:17) with 4 output partitions -2016-06-13 15:32:18 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at testFunSuite.scala:17) -2016-06-13 15:32:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:32:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:32:18 INFO DAGScheduler:59 - Submitting ResultStage 0 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:16), which has no missing parents -2016-06-13 15:32:18 INFO MemoryStore:59 - ensureFreeSpace(1096) called with curMem=0, maxMem=766075207 -2016-06-13 15:32:18 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1096.0 B, free 730.6 MB) -2016-06-13 15:32:18 INFO MemoryStore:59 - ensureFreeSpace(804) called with curMem=1096, maxMem=766075207 -2016-06-13 15:32:18 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 804.0 B, free 730.6 MB) -2016-06-13 15:32:18 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:57794 (size: 804.0 B, free: 730.6 MB) -2016-06-13 15:32:18 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:32:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 0 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:16) -2016-06-13 15:32:18 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 4 tasks -2016-06-13 15:32:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2025 bytes) -2016-06-13 15:32:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:32:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:32:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 0.0 (TID 3, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:32:18 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-06-13 15:32:18 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 15:32:18 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 15:32:18 INFO Executor:59 - Running task 3.0 in stage 0.0 (TID 3) -2016-06-13 15:32:18 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 953 bytes result sent to driver -2016-06-13 15:32:18 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 953 bytes result sent to driver -2016-06-13 15:32:18 INFO Executor:59 - Finished task 3.0 in stage 0.0 (TID 3). 953 bytes result sent to driver -2016-06-13 15:32:18 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 953 bytes result sent to driver -2016-06-13 15:32:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 47 ms on localhost (1/4) -2016-06-13 15:32:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 0.0 (TID 3) in 47 ms on localhost (2/4) -2016-06-13 15:32:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 50 ms on localhost (3/4) -2016-06-13 15:32:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 78 ms on localhost (4/4) -2016-06-13 15:32:18 INFO DAGScheduler:59 - ResultStage 0 (count at testFunSuite.scala:17) finished in 0.087 s -2016-06-13 15:32:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 15:32:18 INFO DAGScheduler:59 - Job 0 finished: count at testFunSuite.scala:17, took 0.279749 s -2016-06-13 15:32:18 INFO SparkContext:59 - Starting job: foreach at testFunSuite.scala:18 -2016-06-13 15:32:18 INFO DAGScheduler:59 - Got job 1 (foreach at testFunSuite.scala:18) with 4 output partitions -2016-06-13 15:32:18 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at testFunSuite.scala:18) -2016-06-13 15:32:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:32:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:32:18 INFO DAGScheduler:59 - Submitting ResultStage 1 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:16), which has no missing parents -2016-06-13 15:32:18 INFO MemoryStore:59 - ensureFreeSpace(1232) called with curMem=1900, maxMem=766075207 -2016-06-13 15:32:18 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1232.0 B, free 730.6 MB) -2016-06-13 15:32:18 INFO MemoryStore:59 - ensureFreeSpace(887) called with curMem=3132, maxMem=766075207 -2016-06-13 15:32:18 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 887.0 B, free 730.6 MB) -2016-06-13 15:32:18 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:57794 (size: 887.0 B, free: 730.6 MB) -2016-06-13 15:32:18 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:32:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:16) -2016-06-13 15:32:18 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 4 tasks -2016-06-13 15:32:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2025 bytes) -2016-06-13 15:32:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 5, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:32:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 6, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:32:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 1.0 (TID 7, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:32:18 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 6) -2016-06-13 15:32:18 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 4) -2016-06-13 15:32:18 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 5) -2016-06-13 15:32:19 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 6). 915 bytes result sent to driver -2016-06-13 15:32:19 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 5). 915 bytes result sent to driver -2016-06-13 15:32:19 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 4). 915 bytes result sent to driver -2016-06-13 15:32:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 6) in 13 ms on localhost (1/4) -2016-06-13 15:32:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 5) in 15 ms on localhost (2/4) -2016-06-13 15:32:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 4) in 17 ms on localhost (3/4) -2016-06-13 15:32:19 INFO Executor:59 - Running task 3.0 in stage 1.0 (TID 7) -2016-06-13 15:32:19 INFO Executor:59 - Finished task 3.0 in stage 1.0 (TID 7). 915 bytes result sent to driver -2016-06-13 15:32:19 INFO DAGScheduler:59 - ResultStage 1 (foreach at testFunSuite.scala:18) finished in 0.024 s -2016-06-13 15:32:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 1.0 (TID 7) in 20 ms on localhost (4/4) -2016-06-13 15:32:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 15:32:19 INFO DAGScheduler:59 - Job 1 finished: foreach at testFunSuite.scala:18, took 0.038152 s -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:32:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:32:19 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 15:32:19 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 15:32:19 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 15:32:19 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 15:32:19 INFO BlockManager:59 - BlockManager stopped -2016-06-13 15:32:19 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 15:32:19 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 15:32:19 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:32:19 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:32:19 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:32:19 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 15:32:19 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-db4e4b40-384d-4cad-9812-3e96b3fea340 -2016-06-13 15:36:47 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:36:48 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:36:48 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:36:48 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:36:48 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:36:49 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:36:49 INFO Remoting:74 - Starting remoting -2016-06-13 15:36:49 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:57857] -2016-06-13 15:36:49 INFO Utils:59 - Successfully started service 'sparkDriver' on port 57857. -2016-06-13 15:36:49 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 15:36:49 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 15:36:49 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-7b0b1b7e-0e9a-4dc5-9c9d-abc0bcd1d55b -2016-06-13 15:36:49 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 15:36:49 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-8647ef56-d50f-4524-8a18-30a208fda992\httpd-b8462134-6f9b-4441-afd1-a3f0eb3a3f8f -2016-06-13 15:36:49 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 15:36:49 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:36:49 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:57858 -2016-06-13 15:36:49 INFO Utils:59 - Successfully started service 'HTTP file server' on port 57858. -2016-06-13 15:36:49 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 15:36:50 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:36:50 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:36:50 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 15:36:50 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 15:36:50 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 15:36:50 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 15:36:50 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 57877. -2016-06-13 15:36:50 INFO NettyBlockTransferService:59 - Server created on 57877 -2016-06-13 15:36:50 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 15:36:50 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:57877 with 730.6 MB RAM, BlockManagerId(driver, localhost, 57877) -2016-06-13 15:36:50 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 15:36:51 INFO SparkContext:59 - Starting job: count at testFunSuite.scala:14 -2016-06-13 15:36:51 INFO DAGScheduler:59 - Got job 0 (count at testFunSuite.scala:14) with 4 output partitions -2016-06-13 15:36:51 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at testFunSuite.scala:14) -2016-06-13 15:36:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:36:51 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:36:51 INFO DAGScheduler:59 - Submitting ResultStage 0 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:12), which has no missing parents -2016-06-13 15:36:51 INFO MemoryStore:59 - ensureFreeSpace(1096) called with curMem=0, maxMem=766075207 -2016-06-13 15:36:51 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1096.0 B, free 730.6 MB) -2016-06-13 15:36:51 INFO MemoryStore:59 - ensureFreeSpace(804) called with curMem=1096, maxMem=766075207 -2016-06-13 15:36:51 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 804.0 B, free 730.6 MB) -2016-06-13 15:36:51 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:57877 (size: 804.0 B, free: 730.6 MB) -2016-06-13 15:36:51 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:36:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 0 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:12) -2016-06-13 15:36:51 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 4 tasks -2016-06-13 15:36:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2025 bytes) -2016-06-13 15:36:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:36:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:36:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 0.0 (TID 3, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:36:51 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 15:36:51 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 15:36:51 INFO Executor:59 - Running task 3.0 in stage 0.0 (TID 3) -2016-06-13 15:36:51 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-06-13 15:36:51 INFO Executor:59 - Finished task 3.0 in stage 0.0 (TID 3). 953 bytes result sent to driver -2016-06-13 15:36:51 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 953 bytes result sent to driver -2016-06-13 15:36:51 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 953 bytes result sent to driver -2016-06-13 15:36:51 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 953 bytes result sent to driver -2016-06-13 15:36:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 73 ms on localhost (1/4) -2016-06-13 15:36:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 52 ms on localhost (2/4) -2016-06-13 15:36:51 INFO TaskSetManager:59 - Finished task 3.0 in stage 0.0 (TID 3) in 51 ms on localhost (3/4) -2016-06-13 15:36:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 53 ms on localhost (4/4) -2016-06-13 15:36:51 INFO DAGScheduler:59 - ResultStage 0 (count at testFunSuite.scala:14) finished in 0.088 s -2016-06-13 15:36:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 15:36:51 INFO DAGScheduler:59 - Job 0 finished: count at testFunSuite.scala:14, took 0.345358 s -2016-06-13 15:36:51 INFO SparkContext:59 - Starting job: foreach at testFunSuite.scala:15 -2016-06-13 15:36:51 INFO DAGScheduler:59 - Got job 1 (foreach at testFunSuite.scala:15) with 4 output partitions -2016-06-13 15:36:51 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at testFunSuite.scala:15) -2016-06-13 15:36:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:36:51 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:36:51 INFO DAGScheduler:59 - Submitting ResultStage 1 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:12), which has no missing parents -2016-06-13 15:36:51 INFO MemoryStore:59 - ensureFreeSpace(1216) called with curMem=1900, maxMem=766075207 -2016-06-13 15:36:51 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1216.0 B, free 730.6 MB) -2016-06-13 15:36:51 INFO MemoryStore:59 - ensureFreeSpace(871) called with curMem=3116, maxMem=766075207 -2016-06-13 15:36:51 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 871.0 B, free 730.6 MB) -2016-06-13 15:36:51 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:57877 (size: 871.0 B, free: 730.6 MB) -2016-06-13 15:36:51 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:36:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1 (ParallelCollectionRDD[0] at parallelize at testFunSuite.scala:12) -2016-06-13 15:36:51 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 4 tasks -2016-06-13 15:36:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2025 bytes) -2016-06-13 15:36:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 5, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:36:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 6, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:36:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 1.0 (TID 7, localhost, PROCESS_LOCAL, 2029 bytes) -2016-06-13 15:36:51 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 4) -2016-06-13 15:36:51 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 5) -2016-06-13 15:36:51 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 4). 915 bytes result sent to driver -2016-06-13 15:36:51 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 6) -2016-06-13 15:36:51 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 5). 915 bytes result sent to driver -2016-06-13 15:36:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 4) in 18 ms on localhost (1/4) -2016-06-13 15:36:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 5) in 18 ms on localhost (2/4) -2016-06-13 15:36:51 INFO Executor:59 - Running task 3.0 in stage 1.0 (TID 7) -2016-06-13 15:36:51 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 6). 915 bytes result sent to driver -2016-06-13 15:36:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 6) in 20 ms on localhost (3/4) -2016-06-13 15:36:51 INFO Executor:59 - Finished task 3.0 in stage 1.0 (TID 7). 915 bytes result sent to driver -2016-06-13 15:36:51 INFO TaskSetManager:59 - Finished task 3.0 in stage 1.0 (TID 7) in 24 ms on localhost (4/4) -2016-06-13 15:36:51 INFO DAGScheduler:59 - ResultStage 1 (foreach at testFunSuite.scala:15) finished in 0.029 s -2016-06-13 15:36:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 15:36:51 INFO DAGScheduler:59 - Job 1 finished: foreach at testFunSuite.scala:15, took 0.044574 s -2016-06-13 15:36:51 INFO SparkContext:59 - Starting job: foreach at testFunSuite.scala:17 -2016-06-13 15:36:51 INFO DAGScheduler:59 - Got job 2 (foreach at testFunSuite.scala:17) with 8 output partitions -2016-06-13 15:36:51 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at testFunSuite.scala:17) -2016-06-13 15:36:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:36:51 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:36:51 INFO DAGScheduler:59 - Submitting ResultStage 2 (UnionRDD[2] at union at testFunSuite.scala:17), which has no missing parents -2016-06-13 15:36:51 INFO MemoryStore:59 - ensureFreeSpace(1960) called with curMem=3987, maxMem=766075207 -2016-06-13 15:36:51 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 1960.0 B, free 730.6 MB) -2016-06-13 15:36:51 INFO MemoryStore:59 - ensureFreeSpace(1328) called with curMem=5947, maxMem=766075207 -2016-06-13 15:36:51 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1328.0 B, free 730.6 MB) -2016-06-13 15:36:51 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:57877 (size: 1328.0 B, free: 730.6 MB) -2016-06-13 15:36:51 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:36:51 INFO DAGScheduler:59 - Submitting 8 missing tasks from ResultStage 2 (UnionRDD[2] at union at testFunSuite.scala:17) -2016-06-13 15:36:51 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 8 tasks -2016-06-13 15:36:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 8, localhost, PROCESS_LOCAL, 2134 bytes) -2016-06-13 15:36:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 9, localhost, PROCESS_LOCAL, 2138 bytes) -2016-06-13 15:36:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 10, localhost, PROCESS_LOCAL, 2138 bytes) -2016-06-13 15:36:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 2.0 (TID 11, localhost, PROCESS_LOCAL, 2138 bytes) -2016-06-13 15:36:51 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 9) -2016-06-13 15:36:51 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 10) -2016-06-13 15:36:51 INFO Executor:59 - Running task 3.0 in stage 2.0 (TID 11) -2016-06-13 15:36:51 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 8) -2016-06-13 15:36:51 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 9). 915 bytes result sent to driver -2016-06-13 15:36:51 INFO TaskSetManager:59 - Starting task 4.0 in stage 2.0 (TID 12, localhost, PROCESS_LOCAL, 2134 bytes) -2016-06-13 15:36:51 INFO Executor:59 - Finished task 3.0 in stage 2.0 (TID 11). 915 bytes result sent to driver -2016-06-13 15:36:51 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 10). 915 bytes result sent to driver -2016-06-13 15:36:51 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 8). 915 bytes result sent to driver -2016-06-13 15:36:51 INFO TaskSetManager:59 - Starting task 5.0 in stage 2.0 (TID 13, localhost, PROCESS_LOCAL, 2138 bytes) -2016-06-13 15:36:51 INFO Executor:59 - Running task 4.0 in stage 2.0 (TID 12) -2016-06-13 15:36:51 INFO TaskSetManager:59 - Starting task 6.0 in stage 2.0 (TID 14, localhost, PROCESS_LOCAL, 2138 bytes) -2016-06-13 15:36:51 INFO TaskSetManager:59 - Starting task 7.0 in stage 2.0 (TID 15, localhost, PROCESS_LOCAL, 2138 bytes) -2016-06-13 15:36:51 INFO TaskSetManager:59 - Finished task 3.0 in stage 2.0 (TID 11) in 20 ms on localhost (1/8) -2016-06-13 15:36:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 10) in 22 ms on localhost (2/8) -2016-06-13 15:36:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 8) in 27 ms on localhost (3/8) -2016-06-13 15:36:51 INFO Executor:59 - Running task 5.0 in stage 2.0 (TID 13) -2016-06-13 15:36:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 9) in 26 ms on localhost (4/8) -2016-06-13 15:36:51 INFO Executor:59 - Running task 6.0 in stage 2.0 (TID 14) -2016-06-13 15:36:51 INFO Executor:59 - Finished task 4.0 in stage 2.0 (TID 12). 915 bytes result sent to driver -2016-06-13 15:36:51 INFO Executor:59 - Running task 7.0 in stage 2.0 (TID 15) -2016-06-13 15:36:51 INFO Executor:59 - Finished task 6.0 in stage 2.0 (TID 14). 915 bytes result sent to driver -2016-06-13 15:36:51 INFO TaskSetManager:59 - Finished task 4.0 in stage 2.0 (TID 12) in 22 ms on localhost (5/8) -2016-06-13 15:36:51 INFO Executor:59 - Finished task 5.0 in stage 2.0 (TID 13). 915 bytes result sent to driver -2016-06-13 15:36:51 INFO TaskSetManager:59 - Finished task 6.0 in stage 2.0 (TID 14) in 16 ms on localhost (6/8) -2016-06-13 15:36:51 INFO TaskSetManager:59 - Finished task 5.0 in stage 2.0 (TID 13) in 19 ms on localhost (7/8) -2016-06-13 15:36:51 INFO Executor:59 - Finished task 7.0 in stage 2.0 (TID 15). 915 bytes result sent to driver -2016-06-13 15:36:51 INFO TaskSetManager:59 - Finished task 7.0 in stage 2.0 (TID 15) in 19 ms on localhost (8/8) -2016-06-13 15:36:51 INFO DAGScheduler:59 - ResultStage 2 (foreach at testFunSuite.scala:17) finished in 0.041 s -2016-06-13 15:36:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 15:36:51 INFO DAGScheduler:59 - Job 2 finished: foreach at testFunSuite.scala:17, took 0.071871 s -2016-06-13 15:36:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 15:36:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 15:36:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 15:36:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 15:36:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 15:36:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:36:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:36:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:36:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:36:52 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 15:36:52 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 15:36:52 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 15:36:52 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 15:36:52 INFO BlockManager:59 - BlockManager stopped -2016-06-13 15:36:52 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 15:36:52 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 15:36:52 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:36:52 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:36:52 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:36:52 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 15:36:52 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-8647ef56-d50f-4524-8a18-30a208fda992 -2016-06-13 15:42:13 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:42:13 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:42:14 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:42:14 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:42:14 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:42:15 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:42:15 INFO Remoting:74 - Starting remoting -2016-06-13 15:42:15 INFO Utils:59 - Successfully started service 'sparkDriver' on port 57925. -2016-06-13 15:42:15 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:57925] -2016-06-13 15:42:15 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 15:42:15 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 15:42:15 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-05a4ef24-8dee-4554-8b91-5686e0339f93 -2016-06-13 15:42:15 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 15:42:15 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-9e46792f-0676-4755-aa64-865a08ea2507\httpd-a4590343-0945-4e62-994d-4ba38ee9be0c -2016-06-13 15:42:15 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 15:42:16 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:42:16 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:57926 -2016-06-13 15:42:16 INFO Utils:59 - Successfully started service 'HTTP file server' on port 57926. -2016-06-13 15:42:16 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 15:42:16 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:42:16 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:42:16 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 15:42:16 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 15:42:16 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 15:42:16 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 15:42:16 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 57945. -2016-06-13 15:42:16 INFO NettyBlockTransferService:59 - Server created on 57945 -2016-06-13 15:42:16 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 15:42:16 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:57945 with 730.6 MB RAM, BlockManagerId(driver, localhost, 57945) -2016-06-13 15:42:16 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 15:42:17 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 15:42:17 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 15:42:18 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 15:42:18 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 15:42:18 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:57945 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 15:42:18 INFO SparkContext:59 - Created broadcast 0 from textFile at RowMatrixSuite.scala:16 -2016-06-13 15:42:19 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 15:42:19 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 15:42:19 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-06-13 15:42:19 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 2 output partitions -2016-06-13 15:42:19 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-06-13 15:42:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:42:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:42:19 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:42:19 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=171580, maxMem=766075207 -2016-06-13 15:42:19 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 15:42:19 INFO MemoryStore:59 - ensureFreeSpace(1943) called with curMem=174956, maxMem=766075207 -2016-06-13 15:42:19 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1943.0 B, free 730.4 MB) -2016-06-13 15:42:19 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:57945 (size: 1943.0 B, free: 730.6 MB) -2016-06-13 15:42:19 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:42:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:42:19 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 15:42:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2182 bytes) -2016-06-13 15:42:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2182 bytes) -2016-06-13 15:42:19 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 15:42:19 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 15:42:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+6 -2016-06-13 15:42:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:6+6 -2016-06-13 15:42:19 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 15:42:19 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 15:42:19 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 15:42:19 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 15:42:19 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 15:42:19 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 15:42:19 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 15:42:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 98 ms on localhost (1/2) -2016-06-13 15:42:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 139 ms on localhost (2/2) -2016-06-13 15:42:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 15:42:19 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.152 s -2016-06-13 15:42:19 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.304943 s -2016-06-13 15:42:19 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 15:42:19 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 15:42:19 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-06-13 15:42:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:42:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:42:19 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:42:19 INFO MemoryStore:59 - ensureFreeSpace(3544) called with curMem=176899, maxMem=766075207 -2016-06-13 15:42:19 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 15:42:19 INFO MemoryStore:59 - ensureFreeSpace(2001) called with curMem=180443, maxMem=766075207 -2016-06-13 15:42:19 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2001.0 B, free 730.4 MB) -2016-06-13 15:42:19 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:57945 (size: 2001.0 B, free: 730.6 MB) -2016-06-13 15:42:19 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:42:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:42:19 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 15:42:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2182 bytes) -2016-06-13 15:42:19 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 15:42:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+6 -2016-06-13 15:42:19 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2183 bytes result sent to driver -2016-06-13 15:42:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 31 ms on localhost (1/1) -2016-06-13 15:42:19 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.033 s -2016-06-13 15:42:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 15:42:19 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.073134 s -2016-06-13 15:42:20 INFO SparkContext:59 - Starting job: treeReduce at RowMatrix.scala:537 -2016-06-13 15:42:20 INFO DAGScheduler:59 - Got job 2 (treeReduce at RowMatrix.scala:537) with 2 output partitions -2016-06-13 15:42:20 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeReduce at RowMatrix.scala:537) -2016-06-13 15:42:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:42:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:42:20 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537), which has no missing parents -2016-06-13 15:42:20 INFO MemoryStore:59 - ensureFreeSpace(5072) called with curMem=182444, maxMem=766075207 -2016-06-13 15:42:20 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-06-13 15:42:20 INFO MemoryStore:59 - ensureFreeSpace(2502) called with curMem=187516, maxMem=766075207 -2016-06-13 15:42:20 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 15:42:20 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:57945 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 15:42:20 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:42:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537) -2016-06-13 15:42:20 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 15:42:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2182 bytes) -2016-06-13 15:42:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2182 bytes) -2016-06-13 15:42:20 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-06-13 15:42:20 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 15:42:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:6+6 -2016-06-13 15:42:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+6 -2016-06-13 15:42:22 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 15:42:22 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 15:42:23 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2261 bytes result sent to driver -2016-06-13 15:42:23 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2261 bytes result sent to driver -2016-06-13 15:42:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 3049 ms on localhost (1/2) -2016-06-13 15:42:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 3051 ms on localhost (2/2) -2016-06-13 15:42:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 15:42:23 INFO DAGScheduler:59 - ResultStage 2 (treeReduce at RowMatrix.scala:537) finished in 3.052 s -2016-06-13 15:42:23 INFO DAGScheduler:59 - Job 2 finished: treeReduce at RowMatrix.scala:537, took 3.180352 s -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:42:23 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:42:23 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 15:42:23 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 15:42:23 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 15:42:23 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 15:42:23 INFO BlockManager:59 - BlockManager stopped -2016-06-13 15:42:23 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 15:42:23 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 15:42:23 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:42:23 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:42:23 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:42:23 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 15:42:23 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-9e46792f-0676-4755-aa64-865a08ea2507 -2016-06-13 15:43:56 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:43:57 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:43:57 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:43:57 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:43:57 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:43:58 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:43:58 INFO Remoting:74 - Starting remoting -2016-06-13 15:43:58 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:57972] -2016-06-13 15:43:58 INFO Utils:59 - Successfully started service 'sparkDriver' on port 57972. -2016-06-13 15:43:58 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 15:43:58 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 15:43:58 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-6a39197a-b89e-40d6-9ced-351ad6c2b3f8 -2016-06-13 15:43:58 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 15:43:58 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-3e31e020-8ef6-495d-9b13-db1e3cad36ce\httpd-f242900e-f0d1-472c-bd7e-5162e5c358ac -2016-06-13 15:43:58 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 15:43:59 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:43:59 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:57973 -2016-06-13 15:43:59 INFO Utils:59 - Successfully started service 'HTTP file server' on port 57973. -2016-06-13 15:43:59 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 15:43:59 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:43:59 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:43:59 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 15:43:59 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 15:43:59 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 15:43:59 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 15:43:59 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 57992. -2016-06-13 15:43:59 INFO NettyBlockTransferService:59 - Server created on 57992 -2016-06-13 15:43:59 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 15:43:59 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:57992 with 730.6 MB RAM, BlockManagerId(driver, localhost, 57992) -2016-06-13 15:43:59 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 15:44:00 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 15:44:00 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 15:44:00 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 15:44:00 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 15:44:00 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:57992 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 15:44:00 INFO SparkContext:59 - Created broadcast 0 from textFile at RowMatrixSuite.scala:16 -2016-06-13 15:44:01 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 15:44:01 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 15:44:01 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-06-13 15:44:01 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 2 output partitions -2016-06-13 15:44:01 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-06-13 15:44:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:44:01 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:44:01 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:44:01 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=171580, maxMem=766075207 -2016-06-13 15:44:01 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 15:44:01 INFO MemoryStore:59 - ensureFreeSpace(1943) called with curMem=174956, maxMem=766075207 -2016-06-13 15:44:01 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1943.0 B, free 730.4 MB) -2016-06-13 15:44:01 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:57992 (size: 1943.0 B, free: 730.6 MB) -2016-06-13 15:44:01 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:44:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:44:01 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 15:44:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2182 bytes) -2016-06-13 15:44:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2182 bytes) -2016-06-13 15:44:01 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 15:44:01 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 15:44:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:6+6 -2016-06-13 15:44:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+6 -2016-06-13 15:44:01 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 15:44:01 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 15:44:01 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 15:44:01 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 15:44:01 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 15:44:01 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 15:44:01 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 15:44:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 111 ms on localhost (1/2) -2016-06-13 15:44:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 134 ms on localhost (2/2) -2016-06-13 15:44:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 15:44:01 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.151 s -2016-06-13 15:44:01 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.301877 s -2016-06-13 15:44:01 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 15:44:01 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 15:44:01 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-06-13 15:44:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:44:01 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:44:01 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:44:01 INFO MemoryStore:59 - ensureFreeSpace(3544) called with curMem=176899, maxMem=766075207 -2016-06-13 15:44:01 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 15:44:01 INFO MemoryStore:59 - ensureFreeSpace(2001) called with curMem=180443, maxMem=766075207 -2016-06-13 15:44:01 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2001.0 B, free 730.4 MB) -2016-06-13 15:44:01 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:57992 (size: 2001.0 B, free: 730.6 MB) -2016-06-13 15:44:01 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:44:01 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:44:01 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 15:44:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2182 bytes) -2016-06-13 15:44:01 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 15:44:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+6 -2016-06-13 15:44:02 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2183 bytes result sent to driver -2016-06-13 15:44:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 22 ms on localhost (1/1) -2016-06-13 15:44:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 15:44:02 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.024 s -2016-06-13 15:44:02 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.043433 s -2016-06-13 15:44:02 INFO SparkContext:59 - Starting job: treeReduce at RowMatrix.scala:537 -2016-06-13 15:44:02 INFO DAGScheduler:59 - Got job 2 (treeReduce at RowMatrix.scala:537) with 2 output partitions -2016-06-13 15:44:02 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeReduce at RowMatrix.scala:537) -2016-06-13 15:44:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:44:02 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:44:02 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537), which has no missing parents -2016-06-13 15:44:02 INFO MemoryStore:59 - ensureFreeSpace(5072) called with curMem=182444, maxMem=766075207 -2016-06-13 15:44:02 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-06-13 15:44:02 INFO MemoryStore:59 - ensureFreeSpace(2502) called with curMem=187516, maxMem=766075207 -2016-06-13 15:44:02 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 15:44:02 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:57992 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 15:44:02 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:44:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537) -2016-06-13 15:44:02 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 15:44:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2182 bytes) -2016-06-13 15:44:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2182 bytes) -2016-06-13 15:44:02 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-06-13 15:44:02 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 15:44:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:6+6 -2016-06-13 15:44:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+6 -2016-06-13 15:44:02 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 15:44:02 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 15:44:02 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2261 bytes result sent to driver -2016-06-13 15:44:02 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2261 bytes result sent to driver -2016-06-13 15:44:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 562 ms on localhost (1/2) -2016-06-13 15:44:02 INFO DAGScheduler:59 - ResultStage 2 (treeReduce at RowMatrix.scala:537) finished in 0.566 s -2016-06-13 15:44:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 566 ms on localhost (2/2) -2016-06-13 15:44:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 15:44:02 INFO DAGScheduler:59 - Job 2 finished: treeReduce at RowMatrix.scala:537, took 0.585073 s -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:44:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:44:02 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 15:44:02 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 15:44:02 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 15:44:02 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 15:44:02 INFO BlockManager:59 - BlockManager stopped -2016-06-13 15:44:02 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 15:44:02 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 15:44:02 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:44:02 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:44:02 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:44:02 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 15:44:02 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-3e31e020-8ef6-495d-9b13-db1e3cad36ce -2016-06-13 15:45:25 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:45:25 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:45:25 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:45:25 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:45:25 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:45:26 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:45:26 INFO Remoting:74 - Starting remoting -2016-06-13 15:45:26 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:58027] -2016-06-13 15:45:26 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58027. -2016-06-13 15:45:26 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 15:45:26 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 15:45:26 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-6dcf55de-5566-4c23-b236-f20dc405e2e7 -2016-06-13 15:45:26 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 15:45:27 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-6e2c8f55-9fe9-4bd1-a736-57192e8fed01\httpd-cf5b50e6-039d-42ba-a57f-76fa609a35e3 -2016-06-13 15:45:27 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 15:45:27 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:45:27 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58028 -2016-06-13 15:45:27 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58028. -2016-06-13 15:45:27 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 15:45:27 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:45:27 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:45:27 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 15:45:27 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 15:45:27 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 15:45:27 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 15:45:27 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58047. -2016-06-13 15:45:27 INFO NettyBlockTransferService:59 - Server created on 58047 -2016-06-13 15:45:27 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 15:45:27 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:58047 with 730.6 MB RAM, BlockManagerId(driver, localhost, 58047) -2016-06-13 15:45:27 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 15:45:28 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 15:45:28 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 15:45:28 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 15:45:28 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 15:45:28 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:58047 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 15:45:28 INFO SparkContext:59 - Created broadcast 0 from textFile at RowMatrixSuite.scala:16 -2016-06-13 15:45:29 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 15:45:29 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 15:45:29 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-06-13 15:45:29 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 3 output partitions -2016-06-13 15:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-06-13 15:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:45:29 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:45:29 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=171580, maxMem=766075207 -2016-06-13 15:45:29 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 15:45:29 INFO MemoryStore:59 - ensureFreeSpace(1945) called with curMem=174956, maxMem=766075207 -2016-06-13 15:45:29 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1945.0 B, free 730.4 MB) -2016-06-13 15:45:29 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:58047 (size: 1945.0 B, free: 730.6 MB) -2016-06-13 15:45:29 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:45:29 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:45:29 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-06-13 15:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:45:29 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-06-13 15:45:29 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 15:45:29 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 15:45:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:9+9 -2016-06-13 15:45:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+9 -2016-06-13 15:45:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:18+1 -2016-06-13 15:45:29 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 15:45:29 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 15:45:29 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 15:45:29 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 15:45:29 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 15:45:29 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 2082 bytes result sent to driver -2016-06-13 15:45:29 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 15:45:29 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 15:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 111 ms on localhost (1/3) -2016-06-13 15:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 131 ms on localhost (2/3) -2016-06-13 15:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 115 ms on localhost (3/3) -2016-06-13 15:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 15:45:29 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.144 s -2016-06-13 15:45:29 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.268528 s -2016-06-13 15:45:29 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 15:45:29 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 15:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-06-13 15:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:45:29 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:45:29 INFO MemoryStore:59 - ensureFreeSpace(3544) called with curMem=176901, maxMem=766075207 -2016-06-13 15:45:29 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 15:45:29 INFO MemoryStore:59 - ensureFreeSpace(2003) called with curMem=180445, maxMem=766075207 -2016-06-13 15:45:29 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2003.0 B, free 730.4 MB) -2016-06-13 15:45:29 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:58047 (size: 2003.0 B, free: 730.6 MB) -2016-06-13 15:45:29 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:45:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:45:29 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 15:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:45:29 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 3) -2016-06-13 15:45:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+9 -2016-06-13 15:45:29 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 3). 2183 bytes result sent to driver -2016-06-13 15:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 3) in 21 ms on localhost (1/1) -2016-06-13 15:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 15:45:29 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.022 s -2016-06-13 15:45:29 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.043229 s -2016-06-13 15:45:30 INFO SparkContext:59 - Starting job: treeReduce at RowMatrix.scala:537 -2016-06-13 15:45:30 INFO DAGScheduler:59 - Got job 2 (treeReduce at RowMatrix.scala:537) with 3 output partitions -2016-06-13 15:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeReduce at RowMatrix.scala:537) -2016-06-13 15:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:45:30 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537), which has no missing parents -2016-06-13 15:45:30 INFO MemoryStore:59 - ensureFreeSpace(5080) called with curMem=182448, maxMem=766075207 -2016-06-13 15:45:30 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-06-13 15:45:30 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=187528, maxMem=766075207 -2016-06-13 15:45:30 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 15:45:30 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:58047 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 15:45:30 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:45:30 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537) -2016-06-13 15:45:30 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-06-13 15:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:45:30 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 15:45:30 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 15:45:30 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-06-13 15:45:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+9 -2016-06-13 15:45:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:18+1 -2016-06-13 15:45:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:9+9 -2016-06-13 15:45:30 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 15:45:30 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 15:45:30 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:45:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:45:30 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:58047 in memory (size: 2003.0 B, free: 730.6 MB) -2016-06-13 15:45:30 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 15:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:58047 in memory (size: 1945.0 B, free: 730.6 MB) -2016-06-13 15:45:30 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 15:45:30 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 15:45:30 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 15:45:30 INFO DAGScheduler:59 - Job 2 failed: treeReduce at RowMatrix.scala:537, took 0.520451 s -2016-06-13 15:45:30 INFO DAGScheduler:59 - ResultStage 2 (treeReduce at RowMatrix.scala:537) failed in 0.505 s -2016-06-13 15:45:30 INFO SparkContext:59 - SparkContext already stopped. -2016-06-13 15:45:30 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 15:45:30 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 15:45:30 INFO BlockManager:59 - BlockManager stopped -2016-06-13 15:45:30 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 15:45:30 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 15:45:30 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:45:30 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 15:45:30 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-6e2c8f55-9fe9-4bd1-a736-57192e8fed01 -2016-06-13 15:46:02 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:46:02 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:46:02 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:46:02 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:46:02 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:46:03 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:46:03 INFO Remoting:74 - Starting remoting -2016-06-13 15:46:03 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:58067] -2016-06-13 15:46:03 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58067. -2016-06-13 15:46:03 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 15:46:03 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 15:46:03 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-63dd8f07-227d-4fec-bef8-d1785dcc0f45 -2016-06-13 15:46:03 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 15:46:03 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-ef98c6ce-bed9-4c5d-a578-3eb502021633\httpd-bf8fc7fa-eeac-469a-81a9-b5c8a7aeb32d -2016-06-13 15:46:03 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 15:46:04 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:46:04 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58068 -2016-06-13 15:46:04 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58068. -2016-06-13 15:46:04 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 15:46:04 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:46:04 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:46:04 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 15:46:04 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 15:46:04 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 15:46:04 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 15:46:04 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58087. -2016-06-13 15:46:04 INFO NettyBlockTransferService:59 - Server created on 58087 -2016-06-13 15:46:04 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 15:46:04 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:58087 with 730.6 MB RAM, BlockManagerId(driver, localhost, 58087) -2016-06-13 15:46:04 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 15:46:05 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 15:46:05 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 15:46:05 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 15:46:05 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 15:46:05 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:58087 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 15:46:05 INFO SparkContext:59 - Created broadcast 0 from textFile at RowMatrixSuite.scala:16 -2016-06-13 15:46:06 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 15:46:06 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 15:46:06 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-06-13 15:46:06 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 3 output partitions -2016-06-13 15:46:06 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-06-13 15:46:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:46:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:46:06 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:46:06 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=171580, maxMem=766075207 -2016-06-13 15:46:06 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 15:46:06 INFO MemoryStore:59 - ensureFreeSpace(1945) called with curMem=174956, maxMem=766075207 -2016-06-13 15:46:06 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1945.0 B, free 730.4 MB) -2016-06-13 15:46:06 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:58087 (size: 1945.0 B, free: 730.6 MB) -2016-06-13 15:46:06 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:46:06 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:46:06 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-06-13 15:46:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:46:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:46:06 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:46:06 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 15:46:06 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 15:46:06 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-06-13 15:46:06 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:9+9 -2016-06-13 15:46:06 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:18+1 -2016-06-13 15:46:06 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+9 -2016-06-13 15:46:06 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 15:46:06 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 15:46:06 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 15:46:06 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 15:46:06 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 15:46:06 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 15:46:06 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 15:46:06 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 2082 bytes result sent to driver -2016-06-13 15:46:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 135 ms on localhost (1/3) -2016-06-13 15:46:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 116 ms on localhost (2/3) -2016-06-13 15:46:06 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 115 ms on localhost (3/3) -2016-06-13 15:46:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 15:46:06 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.155 s -2016-06-13 15:46:06 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.271855 s -2016-06-13 15:46:06 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 15:46:06 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 15:46:06 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-06-13 15:46:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:46:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:46:06 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:46:06 INFO MemoryStore:59 - ensureFreeSpace(3544) called with curMem=176901, maxMem=766075207 -2016-06-13 15:46:06 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 15:46:06 INFO MemoryStore:59 - ensureFreeSpace(2003) called with curMem=180445, maxMem=766075207 -2016-06-13 15:46:06 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2003.0 B, free 730.4 MB) -2016-06-13 15:46:06 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:58087 (size: 2003.0 B, free: 730.6 MB) -2016-06-13 15:46:06 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:46:06 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:46:06 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 15:46:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:46:06 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 3) -2016-06-13 15:46:06 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+9 -2016-06-13 15:46:06 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 3). 2183 bytes result sent to driver -2016-06-13 15:46:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 3) in 21 ms on localhost (1/1) -2016-06-13 15:46:06 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.021 s -2016-06-13 15:46:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 15:46:06 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.039840 s -2016-06-13 15:46:07 INFO SparkContext:59 - Starting job: treeReduce at RowMatrix.scala:537 -2016-06-13 15:46:07 INFO DAGScheduler:59 - Got job 2 (treeReduce at RowMatrix.scala:537) with 3 output partitions -2016-06-13 15:46:07 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeReduce at RowMatrix.scala:537) -2016-06-13 15:46:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:46:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:46:07 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537), which has no missing parents -2016-06-13 15:46:07 INFO MemoryStore:59 - ensureFreeSpace(5080) called with curMem=182448, maxMem=766075207 -2016-06-13 15:46:07 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-06-13 15:46:07 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=187528, maxMem=766075207 -2016-06-13 15:46:07 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 15:46:07 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:58087 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 15:46:07 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:46:07 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537) -2016-06-13 15:46:07 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-06-13 15:46:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:46:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:46:07 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:46:07 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 15:46:07 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 15:46:07 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-06-13 15:46:07 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:9+9 -2016-06-13 15:46:07 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:18+1 -2016-06-13 15:46:07 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+9 -2016-06-13 15:46:07 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 15:46:07 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 15:46:07 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:46:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:46:07 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 15:46:07 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 15:46:07 INFO DAGScheduler:59 - Job 2 failed: treeReduce at RowMatrix.scala:537, took 0.415237 s -2016-06-13 15:46:07 INFO DAGScheduler:59 - ResultStage 2 (treeReduce at RowMatrix.scala:537) failed in 0.401 s -2016-06-13 15:46:07 INFO SparkContext:59 - SparkContext already stopped. -2016-06-13 15:46:07 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 15:46:07 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 15:46:07 INFO BlockManager:59 - BlockManager stopped -2016-06-13 15:46:07 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 15:46:07 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:46:07 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 15:46:07 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-ef98c6ce-bed9-4c5d-a578-3eb502021633 -2016-06-13 15:46:07 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 15:47:25 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:47:26 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:47:26 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:47:26 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:47:26 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:47:27 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:47:27 INFO Remoting:74 - Starting remoting -2016-06-13 15:47:27 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:58224] -2016-06-13 15:47:27 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58224. -2016-06-13 15:47:27 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 15:47:27 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 15:47:27 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-10e253bf-c4c4-41aa-b7dd-8abbdeba14fd -2016-06-13 15:47:27 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 15:47:27 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-c5256219-c1f3-47cc-8acd-84a24278fd99\httpd-aedd2815-bf23-459e-a611-28eb9f1134c4 -2016-06-13 15:47:27 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 15:47:27 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:47:28 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58225 -2016-06-13 15:47:28 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58225. -2016-06-13 15:47:28 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 15:47:28 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:47:28 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:47:28 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 15:47:28 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 15:47:28 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 15:47:28 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 15:47:28 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58244. -2016-06-13 15:47:28 INFO NettyBlockTransferService:59 - Server created on 58244 -2016-06-13 15:47:28 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 15:47:28 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:58244 with 730.6 MB RAM, BlockManagerId(driver, localhost, 58244) -2016-06-13 15:47:28 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 15:47:29 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 15:47:29 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 15:47:29 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 15:47:29 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 15:47:29 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:58244 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 15:47:29 INFO SparkContext:59 - Created broadcast 0 from textFile at RowMatrixSuite.scala:16 -2016-06-13 15:47:30 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 15:47:30 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 15:47:30 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-06-13 15:47:30 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 2 output partitions -2016-06-13 15:47:30 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-06-13 15:47:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:47:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:47:30 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:47:30 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=171580, maxMem=766075207 -2016-06-13 15:47:30 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 15:47:30 INFO MemoryStore:59 - ensureFreeSpace(1945) called with curMem=174956, maxMem=766075207 -2016-06-13 15:47:30 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1945.0 B, free 730.4 MB) -2016-06-13 15:47:30 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:58244 (size: 1945.0 B, free: 730.6 MB) -2016-06-13 15:47:30 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:47:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:47:30 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 15:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:47:30 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 15:47:30 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 15:47:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+11 -2016-06-13 15:47:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:11+11 -2016-06-13 15:47:30 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 15:47:30 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 15:47:30 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 15:47:30 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 15:47:30 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 15:47:30 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 15:47:30 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 15:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 98 ms on localhost (1/2) -2016-06-13 15:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 81 ms on localhost (2/2) -2016-06-13 15:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 15:47:30 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.112 s -2016-06-13 15:47:30 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.259890 s -2016-06-13 15:47:30 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 15:47:30 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 15:47:30 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-06-13 15:47:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:47:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:47:30 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:47:30 INFO MemoryStore:59 - ensureFreeSpace(3544) called with curMem=176901, maxMem=766075207 -2016-06-13 15:47:30 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 15:47:30 INFO MemoryStore:59 - ensureFreeSpace(2003) called with curMem=180445, maxMem=766075207 -2016-06-13 15:47:30 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2003.0 B, free 730.4 MB) -2016-06-13 15:47:30 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:58244 (size: 2003.0 B, free: 730.6 MB) -2016-06-13 15:47:30 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:47:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:47:30 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 15:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:47:30 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 15:47:30 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+11 -2016-06-13 15:47:30 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2183 bytes result sent to driver -2016-06-13 15:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 18 ms on localhost (1/1) -2016-06-13 15:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 15:47:30 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.018 s -2016-06-13 15:47:30 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.037656 s -2016-06-13 15:47:31 INFO SparkContext:59 - Starting job: treeReduce at RowMatrix.scala:537 -2016-06-13 15:47:31 INFO DAGScheduler:59 - Got job 2 (treeReduce at RowMatrix.scala:537) with 2 output partitions -2016-06-13 15:47:31 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeReduce at RowMatrix.scala:537) -2016-06-13 15:47:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:47:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:47:31 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537), which has no missing parents -2016-06-13 15:47:31 INFO MemoryStore:59 - ensureFreeSpace(5080) called with curMem=182448, maxMem=766075207 -2016-06-13 15:47:31 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-06-13 15:47:31 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=187528, maxMem=766075207 -2016-06-13 15:47:31 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 15:47:31 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:58244 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 15:47:31 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:47:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537) -2016-06-13 15:47:31 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 15:47:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:47:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:47:31 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-06-13 15:47:31 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 15:47:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+11 -2016-06-13 15:47:31 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:11+11 -2016-06-13 15:47:31 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 15:47:31 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 15:47:31 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2285 bytes result sent to driver -2016-06-13 15:47:31 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2261 bytes result sent to driver -2016-06-13 15:47:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 631 ms on localhost (1/2) -2016-06-13 15:47:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 632 ms on localhost (2/2) -2016-06-13 15:47:31 INFO DAGScheduler:59 - ResultStage 2 (treeReduce at RowMatrix.scala:537) finished in 0.633 s -2016-06-13 15:47:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 15:47:31 INFO DAGScheduler:59 - Job 2 finished: treeReduce at RowMatrix.scala:537, took 0.652269 s -2016-06-13 15:47:31 WARN RowMatrix:71 - R is not invertible and return Q as null -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:47:31 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:47:31 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 15:47:31 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 15:47:31 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 15:47:31 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 15:47:31 INFO BlockManager:59 - BlockManager stopped -2016-06-13 15:47:31 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 15:47:31 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:47:31 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 15:47:31 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:47:31 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:47:31 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 15:47:31 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-c5256219-c1f3-47cc-8acd-84a24278fd99 -2016-06-13 15:48:32 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:48:32 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:48:32 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:48:32 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:48:32 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:48:33 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:48:33 INFO Remoting:74 - Starting remoting -2016-06-13 15:48:33 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58275. -2016-06-13 15:48:33 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:58275] -2016-06-13 15:48:34 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 15:48:34 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 15:48:34 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-5c05da7b-b4e7-4afc-891f-485ef48cdee3 -2016-06-13 15:48:34 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 15:48:34 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-a40025d5-48b2-4d31-ad1a-3f9e8f5ece0a\httpd-fd7be4b3-426f-49fa-9d9e-d5064cf20ca5 -2016-06-13 15:48:34 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 15:48:34 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:48:34 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58276 -2016-06-13 15:48:34 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58276. -2016-06-13 15:48:34 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 15:48:34 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:48:34 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:48:34 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 15:48:34 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 15:48:34 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 15:48:34 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 15:48:34 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58295. -2016-06-13 15:48:34 INFO NettyBlockTransferService:59 - Server created on 58295 -2016-06-13 15:48:34 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 15:48:34 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:58295 with 730.6 MB RAM, BlockManagerId(driver, localhost, 58295) -2016-06-13 15:48:34 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 15:48:35 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 15:48:35 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 15:48:35 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 15:48:35 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 15:48:35 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:58295 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 15:48:35 INFO SparkContext:59 - Created broadcast 0 from textFile at RowMatrixSuite.scala:16 -2016-06-13 15:48:36 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 15:48:36 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 15:48:36 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-06-13 15:48:36 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 2 output partitions -2016-06-13 15:48:36 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-06-13 15:48:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:48:36 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:48:36 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:48:37 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=171580, maxMem=766075207 -2016-06-13 15:48:37 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 15:48:37 INFO MemoryStore:59 - ensureFreeSpace(1945) called with curMem=174956, maxMem=766075207 -2016-06-13 15:48:37 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1945.0 B, free 730.4 MB) -2016-06-13 15:48:37 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:58295 (size: 1945.0 B, free: 730.6 MB) -2016-06-13 15:48:37 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:48:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:48:37 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 15:48:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:48:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:48:37 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 15:48:37 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 15:48:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:11+11 -2016-06-13 15:48:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+11 -2016-06-13 15:48:37 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 15:48:37 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 15:48:37 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 15:48:37 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 15:48:37 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 15:48:37 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 15:48:37 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 15:48:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 88 ms on localhost (1/2) -2016-06-13 15:48:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 110 ms on localhost (2/2) -2016-06-13 15:48:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 15:48:37 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.132 s -2016-06-13 15:48:37 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.279669 s -2016-06-13 15:48:37 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 15:48:37 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 15:48:37 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-06-13 15:48:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:48:37 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:48:37 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:48:37 INFO MemoryStore:59 - ensureFreeSpace(3544) called with curMem=176901, maxMem=766075207 -2016-06-13 15:48:37 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 15:48:37 INFO MemoryStore:59 - ensureFreeSpace(2003) called with curMem=180445, maxMem=766075207 -2016-06-13 15:48:37 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2003.0 B, free 730.4 MB) -2016-06-13 15:48:37 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:58295 (size: 2003.0 B, free: 730.6 MB) -2016-06-13 15:48:37 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:48:37 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:48:37 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 15:48:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:48:37 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 15:48:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+11 -2016-06-13 15:48:37 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2183 bytes result sent to driver -2016-06-13 15:48:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 25 ms on localhost (1/1) -2016-06-13 15:48:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 15:48:37 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.027 s -2016-06-13 15:48:37 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.048819 s -2016-06-13 15:48:37 INFO SparkContext:59 - Starting job: treeReduce at RowMatrix.scala:537 -2016-06-13 15:48:37 INFO DAGScheduler:59 - Got job 2 (treeReduce at RowMatrix.scala:537) with 2 output partitions -2016-06-13 15:48:37 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeReduce at RowMatrix.scala:537) -2016-06-13 15:48:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:48:37 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:48:37 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537), which has no missing parents -2016-06-13 15:48:37 INFO MemoryStore:59 - ensureFreeSpace(5080) called with curMem=182448, maxMem=766075207 -2016-06-13 15:48:37 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-06-13 15:48:37 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=187528, maxMem=766075207 -2016-06-13 15:48:37 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 15:48:37 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:58295 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 15:48:37 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:48:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537) -2016-06-13 15:48:37 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 15:48:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:48:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:48:37 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 15:48:37 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-06-13 15:48:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+11 -2016-06-13 15:48:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:11+11 -2016-06-13 15:48:37 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:58295 in memory (size: 2003.0 B, free: 730.6 MB) -2016-06-13 15:48:37 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:58295 in memory (size: 1945.0 B, free: 730.6 MB) -2016-06-13 15:48:38 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 15:48:38 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 15:48:38 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2261 bytes result sent to driver -2016-06-13 15:48:38 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2285 bytes result sent to driver -2016-06-13 15:48:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 728 ms on localhost (1/2) -2016-06-13 15:48:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 728 ms on localhost (2/2) -2016-06-13 15:48:38 INFO DAGScheduler:59 - ResultStage 2 (treeReduce at RowMatrix.scala:537) finished in 0.730 s -2016-06-13 15:48:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 15:48:38 INFO DAGScheduler:59 - Job 2 finished: treeReduce at RowMatrix.scala:537, took 0.761579 s -2016-06-13 15:48:38 WARN RowMatrix:71 - R is not invertible and return Q as null -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:48:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:48:38 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 15:48:38 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 15:48:38 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 15:48:38 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 15:48:38 INFO BlockManager:59 - BlockManager stopped -2016-06-13 15:48:38 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 15:48:38 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 15:48:38 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:48:38 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:48:38 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:48:38 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 15:48:38 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-a40025d5-48b2-4d31-ad1a-3f9e8f5ece0a -2016-06-13 15:52:21 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:52:21 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:52:21 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:52:21 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:52:21 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:52:22 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:52:22 INFO Remoting:74 - Starting remoting -2016-06-13 15:52:22 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:58719] -2016-06-13 15:52:22 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58719. -2016-06-13 15:52:22 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 15:52:22 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 15:52:23 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-ad0de282-f903-4222-bdb8-64cd86b9fadb -2016-06-13 15:52:23 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 15:52:23 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-e3686cc0-c675-44dd-9fad-c6e3ab584fd2\httpd-95340a99-5b1d-47d0-b829-6be197c634c6 -2016-06-13 15:52:23 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 15:52:23 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:52:23 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58720 -2016-06-13 15:52:23 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58720. -2016-06-13 15:52:23 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 15:52:23 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:52:23 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:52:23 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 15:52:23 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 15:52:23 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 15:52:23 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 15:52:23 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58739. -2016-06-13 15:52:23 INFO NettyBlockTransferService:59 - Server created on 58739 -2016-06-13 15:52:23 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 15:52:23 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:58739 with 730.6 MB RAM, BlockManagerId(driver, localhost, 58739) -2016-06-13 15:52:23 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 15:52:24 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 15:52:24 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 15:52:24 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 15:52:24 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 15:52:24 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:58739 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 15:52:24 INFO SparkContext:59 - Created broadcast 0 from textFile at RowMatrixSuite.scala:16 -2016-06-13 15:52:25 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 15:52:26 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 15:52:26 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-06-13 15:52:26 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 2 output partitions -2016-06-13 15:52:26 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-06-13 15:52:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:52:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:52:26 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:52:26 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=171580, maxMem=766075207 -2016-06-13 15:52:26 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 15:52:26 INFO MemoryStore:59 - ensureFreeSpace(1945) called with curMem=174956, maxMem=766075207 -2016-06-13 15:52:26 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1945.0 B, free 730.4 MB) -2016-06-13 15:52:26 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:58739 (size: 1945.0 B, free: 730.6 MB) -2016-06-13 15:52:26 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:52:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:52:26 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 15:52:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:52:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:52:26 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 15:52:26 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 15:52:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:11+11 -2016-06-13 15:52:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+11 -2016-06-13 15:52:26 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 15:52:26 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 15:52:26 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 15:52:26 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 15:52:26 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 15:52:26 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 15:52:26 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 15:52:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 140 ms on localhost (1/2) -2016-06-13 15:52:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 87 ms on localhost (2/2) -2016-06-13 15:52:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 15:52:26 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.160 s -2016-06-13 15:52:26 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.310323 s -2016-06-13 15:52:26 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 15:52:26 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 15:52:26 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-06-13 15:52:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:52:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:52:26 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:52:26 INFO MemoryStore:59 - ensureFreeSpace(3544) called with curMem=176901, maxMem=766075207 -2016-06-13 15:52:26 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 15:52:26 INFO MemoryStore:59 - ensureFreeSpace(2003) called with curMem=180445, maxMem=766075207 -2016-06-13 15:52:26 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2003.0 B, free 730.4 MB) -2016-06-13 15:52:26 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:58739 (size: 2003.0 B, free: 730.6 MB) -2016-06-13 15:52:26 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:52:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:52:26 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 15:52:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:52:26 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 15:52:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+11 -2016-06-13 15:52:26 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2183 bytes result sent to driver -2016-06-13 15:52:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 21 ms on localhost (1/1) -2016-06-13 15:52:26 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.023 s -2016-06-13 15:52:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 15:52:26 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.043971 s -2016-06-13 15:52:26 INFO SparkContext:59 - Starting job: treeReduce at RowMatrix.scala:537 -2016-06-13 15:52:26 INFO DAGScheduler:59 - Got job 2 (treeReduce at RowMatrix.scala:537) with 2 output partitions -2016-06-13 15:52:26 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeReduce at RowMatrix.scala:537) -2016-06-13 15:52:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:52:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:52:26 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537), which has no missing parents -2016-06-13 15:52:26 INFO MemoryStore:59 - ensureFreeSpace(5080) called with curMem=182448, maxMem=766075207 -2016-06-13 15:52:26 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-06-13 15:52:26 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=187528, maxMem=766075207 -2016-06-13 15:52:26 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 15:52:26 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:58739 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 15:52:26 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:52:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537) -2016-06-13 15:52:26 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 15:52:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:52:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:52:26 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-06-13 15:52:26 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 15:52:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:11+11 -2016-06-13 15:52:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+11 -2016-06-13 15:52:26 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:58739 in memory (size: 2003.0 B, free: 730.6 MB) -2016-06-13 15:52:26 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 15:52:27 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:58739 in memory (size: 1945.0 B, free: 730.6 MB) -2016-06-13 15:52:27 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 15:52:27 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 15:52:27 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 15:52:27 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2261 bytes result sent to driver -2016-06-13 15:52:27 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2285 bytes result sent to driver -2016-06-13 15:52:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 710 ms on localhost (1/2) -2016-06-13 15:52:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 711 ms on localhost (2/2) -2016-06-13 15:52:27 INFO DAGScheduler:59 - ResultStage 2 (treeReduce at RowMatrix.scala:537) finished in 0.712 s -2016-06-13 15:52:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 15:52:27 INFO DAGScheduler:59 - Job 2 finished: treeReduce at RowMatrix.scala:537, took 0.729457 s -2016-06-13 15:52:27 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=179164, maxMem=766075207 -2016-06-13 15:52:27 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 112.0 B, free 730.4 MB) -2016-06-13 15:52:27 INFO MemoryStore:59 - ensureFreeSpace(106) called with curMem=179276, maxMem=766075207 -2016-06-13 15:52:27 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 106.0 B, free 730.4 MB) -2016-06-13 15:52:27 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:58739 (size: 106.0 B, free: 730.6 MB) -2016-06-13 15:52:27 INFO SparkContext:59 - Created broadcast 4 from broadcast at RowMatrix.scala:425 -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:52:27 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:52:27 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 15:52:27 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 15:52:27 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 15:52:27 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 15:52:27 INFO BlockManager:59 - BlockManager stopped -2016-06-13 15:52:27 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 15:52:27 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:52:27 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 15:52:27 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:52:27 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:52:27 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 15:52:27 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-e3686cc0-c675-44dd-9fad-c6e3ab584fd2 -2016-06-13 15:53:33 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:53:34 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:53:34 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:53:34 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:53:34 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:53:35 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:53:35 INFO Remoting:74 - Starting remoting -2016-06-13 15:53:35 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:58769] -2016-06-13 15:53:35 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58769. -2016-06-13 15:53:35 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 15:53:35 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 15:53:35 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-a3d7cedd-e970-4a6c-81bc-f66eabfd4b49 -2016-06-13 15:53:35 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 15:53:35 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-b8e699e1-ae8b-4384-a3f7-857e97556223\httpd-ba9d75ec-5134-4d51-b381-d01f2ac01d44 -2016-06-13 15:53:35 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 15:53:36 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:53:36 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58770 -2016-06-13 15:53:36 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58770. -2016-06-13 15:53:36 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 15:53:36 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:53:36 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:53:36 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 15:53:36 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 15:53:36 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 15:53:36 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 15:53:36 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58789. -2016-06-13 15:53:36 INFO NettyBlockTransferService:59 - Server created on 58789 -2016-06-13 15:53:36 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 15:53:36 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:58789 with 730.6 MB RAM, BlockManagerId(driver, localhost, 58789) -2016-06-13 15:53:36 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 15:53:37 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 15:53:37 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 15:53:37 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 15:53:37 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 15:53:37 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:58789 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 15:53:37 INFO SparkContext:59 - Created broadcast 0 from textFile at RowMatrixSuite.scala:16 -2016-06-13 15:53:38 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 15:53:38 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 15:53:38 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-06-13 15:53:38 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 2 output partitions -2016-06-13 15:53:38 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-06-13 15:53:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:53:38 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:53:38 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:53:38 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=171580, maxMem=766075207 -2016-06-13 15:53:38 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 15:53:38 INFO MemoryStore:59 - ensureFreeSpace(1945) called with curMem=174956, maxMem=766075207 -2016-06-13 15:53:38 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1945.0 B, free 730.4 MB) -2016-06-13 15:53:38 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:58789 (size: 1945.0 B, free: 730.6 MB) -2016-06-13 15:53:38 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:53:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:53:38 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 15:53:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:53:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:53:38 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 15:53:38 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 15:53:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+11 -2016-06-13 15:53:38 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:11+11 -2016-06-13 15:53:38 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 15:53:38 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 15:53:38 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 15:53:38 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 15:53:38 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 15:53:38 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 15:53:38 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 15:53:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 81 ms on localhost (1/2) -2016-06-13 15:53:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 101 ms on localhost (2/2) -2016-06-13 15:53:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 15:53:38 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.114 s -2016-06-13 15:53:38 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.240512 s -2016-06-13 15:53:39 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 15:53:39 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 15:53:39 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-06-13 15:53:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:53:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:53:39 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:53:39 INFO MemoryStore:59 - ensureFreeSpace(3544) called with curMem=176901, maxMem=766075207 -2016-06-13 15:53:39 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 15:53:39 INFO MemoryStore:59 - ensureFreeSpace(2003) called with curMem=180445, maxMem=766075207 -2016-06-13 15:53:39 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2003.0 B, free 730.4 MB) -2016-06-13 15:53:39 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:58789 (size: 2003.0 B, free: 730.6 MB) -2016-06-13 15:53:39 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:53:39 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:53:39 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 15:53:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:53:39 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 15:53:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+11 -2016-06-13 15:53:39 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2183 bytes result sent to driver -2016-06-13 15:53:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 24 ms on localhost (1/1) -2016-06-13 15:53:39 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.025 s -2016-06-13 15:53:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 15:53:39 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.045122 s -2016-06-13 15:53:39 INFO SparkContext:59 - Starting job: treeReduce at RowMatrix.scala:537 -2016-06-13 15:53:39 INFO DAGScheduler:59 - Got job 2 (treeReduce at RowMatrix.scala:537) with 2 output partitions -2016-06-13 15:53:39 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeReduce at RowMatrix.scala:537) -2016-06-13 15:53:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:53:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:53:39 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537), which has no missing parents -2016-06-13 15:53:39 INFO MemoryStore:59 - ensureFreeSpace(5080) called with curMem=182448, maxMem=766075207 -2016-06-13 15:53:39 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-06-13 15:53:39 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=187528, maxMem=766075207 -2016-06-13 15:53:39 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 15:53:39 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:58789 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 15:53:39 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:53:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537) -2016-06-13 15:53:39 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 15:53:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:53:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:53:39 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 15:53:39 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-06-13 15:53:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+11 -2016-06-13 15:53:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:11+11 -2016-06-13 15:53:39 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 15:53:39 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 15:53:39 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2285 bytes result sent to driver -2016-06-13 15:53:39 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2261 bytes result sent to driver -2016-06-13 15:53:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 661 ms on localhost (1/2) -2016-06-13 15:53:39 INFO DAGScheduler:59 - ResultStage 2 (treeReduce at RowMatrix.scala:537) finished in 0.663 s -2016-06-13 15:53:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 660 ms on localhost (2/2) -2016-06-13 15:53:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 15:53:39 INFO DAGScheduler:59 - Job 2 finished: treeReduce at RowMatrix.scala:537, took 0.680505 s -2016-06-13 15:53:39 WARN RowMatrix:71 - R is not invertible and return Q as null -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:53:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:53:39 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 15:53:39 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 15:53:40 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 15:53:40 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 15:53:40 INFO BlockManager:59 - BlockManager stopped -2016-06-13 15:53:40 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 15:53:40 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 15:53:40 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:53:40 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:53:40 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:53:40 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 15:53:40 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-b8e699e1-ae8b-4384-a3f7-857e97556223 -2016-06-13 15:54:14 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:54:14 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:54:15 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:54:15 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:54:15 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:54:15 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:54:15 INFO Remoting:74 - Starting remoting -2016-06-13 15:54:16 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:58809] -2016-06-13 15:54:16 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58809. -2016-06-13 15:54:16 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 15:54:16 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 15:54:16 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-1a15a446-5c01-4718-be1c-147b12133df1 -2016-06-13 15:54:16 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 15:54:16 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-91f6ebe7-ca60-4657-bce5-044150d26050\httpd-d18d8296-baf4-43dc-bc6a-309b6ee88c67 -2016-06-13 15:54:16 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 15:54:16 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:54:16 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58810 -2016-06-13 15:54:16 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58810. -2016-06-13 15:54:16 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 15:54:16 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:54:16 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:54:16 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 15:54:16 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 15:54:16 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 15:54:16 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 15:54:16 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58829. -2016-06-13 15:54:16 INFO NettyBlockTransferService:59 - Server created on 58829 -2016-06-13 15:54:16 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 15:54:16 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:58829 with 730.6 MB RAM, BlockManagerId(driver, localhost, 58829) -2016-06-13 15:54:16 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 15:54:17 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 15:54:17 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 15:54:17 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 15:54:17 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 15:54:17 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:58829 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 15:54:17 INFO SparkContext:59 - Created broadcast 0 from textFile at RowMatrixSuite.scala:16 -2016-06-13 15:54:18 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 15:54:18 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 15:54:18 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-06-13 15:54:18 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 2 output partitions -2016-06-13 15:54:18 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-06-13 15:54:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:54:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:54:18 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:54:18 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=171580, maxMem=766075207 -2016-06-13 15:54:18 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 15:54:18 INFO MemoryStore:59 - ensureFreeSpace(1945) called with curMem=174956, maxMem=766075207 -2016-06-13 15:54:18 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1945.0 B, free 730.4 MB) -2016-06-13 15:54:18 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:58829 (size: 1945.0 B, free: 730.6 MB) -2016-06-13 15:54:18 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:54:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:54:18 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 15:54:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:54:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:54:19 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 15:54:19 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 15:54:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:11+11 -2016-06-13 15:54:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+11 -2016-06-13 15:54:19 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 15:54:19 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 15:54:19 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 15:54:19 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 15:54:19 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 15:54:19 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 15:54:19 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 15:54:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 107 ms on localhost (1/2) -2016-06-13 15:54:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 127 ms on localhost (2/2) -2016-06-13 15:54:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 15:54:19 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.143 s -2016-06-13 15:54:19 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.242512 s -2016-06-13 15:54:19 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 15:54:19 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 15:54:19 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-06-13 15:54:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:54:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:54:19 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:54:19 INFO MemoryStore:59 - ensureFreeSpace(3544) called with curMem=176901, maxMem=766075207 -2016-06-13 15:54:19 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 15:54:19 INFO MemoryStore:59 - ensureFreeSpace(2003) called with curMem=180445, maxMem=766075207 -2016-06-13 15:54:19 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2003.0 B, free 730.4 MB) -2016-06-13 15:54:19 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:58829 (size: 2003.0 B, free: 730.6 MB) -2016-06-13 15:54:19 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:54:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:54:19 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 15:54:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:54:19 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 15:54:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+11 -2016-06-13 15:54:19 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2183 bytes result sent to driver -2016-06-13 15:54:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 22 ms on localhost (1/1) -2016-06-13 15:54:19 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.023 s -2016-06-13 15:54:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 15:54:19 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.043304 s -2016-06-13 15:54:19 INFO SparkContext:59 - Starting job: treeReduce at RowMatrix.scala:537 -2016-06-13 15:54:19 INFO DAGScheduler:59 - Got job 2 (treeReduce at RowMatrix.scala:537) with 2 output partitions -2016-06-13 15:54:19 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeReduce at RowMatrix.scala:537) -2016-06-13 15:54:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:54:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:54:19 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537), which has no missing parents -2016-06-13 15:54:19 INFO MemoryStore:59 - ensureFreeSpace(5080) called with curMem=182448, maxMem=766075207 -2016-06-13 15:54:19 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-06-13 15:54:19 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=187528, maxMem=766075207 -2016-06-13 15:54:19 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 15:54:19 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:58829 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 15:54:19 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:54:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537) -2016-06-13 15:54:19 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 15:54:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:54:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:54:19 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 15:54:19 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-06-13 15:54:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:11+11 -2016-06-13 15:54:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+11 -2016-06-13 15:54:19 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 15:54:19 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 15:54:19 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2285 bytes result sent to driver -2016-06-13 15:54:19 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2261 bytes result sent to driver -2016-06-13 15:54:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 586 ms on localhost (1/2) -2016-06-13 15:54:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 588 ms on localhost (2/2) -2016-06-13 15:54:19 INFO DAGScheduler:59 - ResultStage 2 (treeReduce at RowMatrix.scala:537) finished in 0.589 s -2016-06-13 15:54:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 15:54:19 INFO DAGScheduler:59 - Job 2 finished: treeReduce at RowMatrix.scala:537, took 0.606048 s -2016-06-13 15:54:19 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=190032, maxMem=766075207 -2016-06-13 15:54:19 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 112.0 B, free 730.4 MB) -2016-06-13 15:54:19 INFO MemoryStore:59 - ensureFreeSpace(106) called with curMem=190144, maxMem=766075207 -2016-06-13 15:54:19 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 106.0 B, free 730.4 MB) -2016-06-13 15:54:19 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:58829 (size: 106.0 B, free: 730.6 MB) -2016-06-13 15:54:19 INFO SparkContext:59 - Created broadcast 4 from broadcast at RowMatrix.scala:425 -2016-06-13 15:54:19 INFO SparkContext:59 - Starting job: foreach at RowMatrixSuite.scala:33 -2016-06-13 15:54:19 INFO DAGScheduler:59 - Got job 3 (foreach at RowMatrixSuite.scala:33) with 2 output partitions -2016-06-13 15:54:19 INFO DAGScheduler:59 - Final stage: ResultStage 3(foreach at RowMatrixSuite.scala:33) -2016-06-13 15:54:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:54:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:54:19 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at mapPartitions at RowMatrix.scala:426), which has no missing parents -2016-06-13 15:54:19 INFO MemoryStore:59 - ensureFreeSpace(3848) called with curMem=190250, maxMem=766075207 -2016-06-13 15:54:19 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 15:54:19 INFO MemoryStore:59 - ensureFreeSpace(2119) called with curMem=194098, maxMem=766075207 -2016-06-13 15:54:19 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 15:54:19 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:58829 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 15:54:19 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:54:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at mapPartitions at RowMatrix.scala:426) -2016-06-13 15:54:19 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 15:54:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:54:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:54:19 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-06-13 15:54:19 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-06-13 15:54:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+11 -2016-06-13 15:54:19 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:11+11 -2016-06-13 15:54:19 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 15:54:19 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 15:54:19 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2044 bytes result sent to driver -2016-06-13 15:54:19 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2044 bytes result sent to driver -2016-06-13 15:54:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 36 ms on localhost (1/2) -2016-06-13 15:54:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 35 ms on localhost (2/2) -2016-06-13 15:54:19 INFO DAGScheduler:59 - ResultStage 3 (foreach at RowMatrixSuite.scala:33) finished in 0.037 s -2016-06-13 15:54:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 15:54:19 INFO DAGScheduler:59 - Job 3 finished: foreach at RowMatrixSuite.scala:33, took 0.050711 s -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:54:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:54:20 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 15:54:20 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 15:54:20 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 15:54:20 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 15:54:20 INFO BlockManager:59 - BlockManager stopped -2016-06-13 15:54:20 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 15:54:20 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 15:54:20 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:54:20 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:54:20 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:54:20 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 15:54:20 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-91f6ebe7-ca60-4657-bce5-044150d26050 -2016-06-13 15:55:00 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:55:01 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:55:01 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:55:01 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:55:01 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:55:02 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:55:02 INFO Remoting:74 - Starting remoting -2016-06-13 15:55:02 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:58850] -2016-06-13 15:55:02 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58850. -2016-06-13 15:55:02 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 15:55:02 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 15:55:02 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-0cbb2b12-794e-42be-aa41-f86edb7d959b -2016-06-13 15:55:02 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 15:55:02 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-1e4b4518-df9c-4065-b3b7-ba1c7dc11def\httpd-878d8b1d-79d5-4028-8706-2f2091529b09 -2016-06-13 15:55:02 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 15:55:02 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:55:02 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58851 -2016-06-13 15:55:02 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58851. -2016-06-13 15:55:02 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 15:55:03 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:55:03 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:55:03 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 15:55:03 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 15:55:03 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 15:55:03 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 15:55:03 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58870. -2016-06-13 15:55:03 INFO NettyBlockTransferService:59 - Server created on 58870 -2016-06-13 15:55:03 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 15:55:03 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:58870 with 730.6 MB RAM, BlockManagerId(driver, localhost, 58870) -2016-06-13 15:55:03 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 15:55:04 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 15:55:04 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 15:55:04 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 15:55:04 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 15:55:04 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:58870 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 15:55:04 INFO SparkContext:59 - Created broadcast 0 from textFile at RowMatrixSuite.scala:16 -2016-06-13 15:55:05 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 15:55:05 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 15:55:05 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-06-13 15:55:05 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 3 output partitions -2016-06-13 15:55:05 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-06-13 15:55:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:55:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:55:05 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:55:05 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=171580, maxMem=766075207 -2016-06-13 15:55:05 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 15:55:05 INFO MemoryStore:59 - ensureFreeSpace(1945) called with curMem=174956, maxMem=766075207 -2016-06-13 15:55:05 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1945.0 B, free 730.4 MB) -2016-06-13 15:55:05 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:58870 (size: 1945.0 B, free: 730.6 MB) -2016-06-13 15:55:05 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:55:05 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:55:05 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-06-13 15:55:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:05 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:05 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 15:55:05 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-06-13 15:55:05 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 15:55:05 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:9+9 -2016-06-13 15:55:05 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:18+1 -2016-06-13 15:55:05 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+9 -2016-06-13 15:55:05 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 15:55:05 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 15:55:05 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 15:55:05 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 15:55:05 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 15:55:05 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 2082 bytes result sent to driver -2016-06-13 15:55:05 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 15:55:05 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 15:55:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 116 ms on localhost (1/3) -2016-06-13 15:55:05 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 115 ms on localhost (2/3) -2016-06-13 15:55:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 140 ms on localhost (3/3) -2016-06-13 15:55:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 15:55:05 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.159 s -2016-06-13 15:55:05 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.284595 s -2016-06-13 15:55:05 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 15:55:05 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 15:55:05 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-06-13 15:55:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:55:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:55:05 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:55:05 INFO MemoryStore:59 - ensureFreeSpace(3544) called with curMem=176901, maxMem=766075207 -2016-06-13 15:55:05 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 15:55:05 INFO MemoryStore:59 - ensureFreeSpace(2003) called with curMem=180445, maxMem=766075207 -2016-06-13 15:55:05 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2003.0 B, free 730.4 MB) -2016-06-13 15:55:05 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:58870 (size: 2003.0 B, free: 730.6 MB) -2016-06-13 15:55:05 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:55:05 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:55:05 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 15:55:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:05 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 3) -2016-06-13 15:55:05 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+9 -2016-06-13 15:55:05 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 3). 2183 bytes result sent to driver -2016-06-13 15:55:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 3) in 24 ms on localhost (1/1) -2016-06-13 15:55:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 15:55:06 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.027 s -2016-06-13 15:55:06 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.049339 s -2016-06-13 15:55:06 INFO SparkContext:59 - Starting job: treeReduce at RowMatrix.scala:537 -2016-06-13 15:55:06 INFO DAGScheduler:59 - Got job 2 (treeReduce at RowMatrix.scala:537) with 3 output partitions -2016-06-13 15:55:06 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeReduce at RowMatrix.scala:537) -2016-06-13 15:55:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:55:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:55:06 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537), which has no missing parents -2016-06-13 15:55:06 INFO MemoryStore:59 - ensureFreeSpace(5080) called with curMem=182448, maxMem=766075207 -2016-06-13 15:55:06 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-06-13 15:55:06 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=187528, maxMem=766075207 -2016-06-13 15:55:06 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 15:55:06 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:58870 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 15:55:06 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:55:06 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537) -2016-06-13 15:55:06 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-06-13 15:55:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:06 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:06 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 15:55:06 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-06-13 15:55:06 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 15:55:06 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+9 -2016-06-13 15:55:06 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:18+1 -2016-06-13 15:55:06 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:9+9 -2016-06-13 15:55:06 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 15:55:06 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 15:55:06 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:55:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:55:06 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 15:55:06 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 15:55:06 INFO DAGScheduler:59 - Job 2 failed: treeReduce at RowMatrix.scala:537, took 0.469306 s -2016-06-13 15:55:06 INFO DAGScheduler:59 - ResultStage 2 (treeReduce at RowMatrix.scala:537) failed in 0.453 s -2016-06-13 15:55:06 INFO SparkContext:59 - SparkContext already stopped. -2016-06-13 15:55:06 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 15:55:06 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 15:55:06 INFO BlockManager:59 - BlockManager stopped -2016-06-13 15:55:06 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 15:55:06 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:55:06 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 15:55:06 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-1e4b4518-df9c-4065-b3b7-ba1c7dc11def -2016-06-13 15:55:06 ERROR Utils:96 - Uncaught exception in thread Executor task launch worker-0 -java.lang.NullPointerException - at org.apache.spark.scheduler.Task$$anonfun$run$1.apply$mcV$sp(Task.scala:94) - at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1185) - at org.apache.spark.scheduler.Task.run(Task.scala:92) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-06-13 15:55:06 ERROR Utils:96 - Uncaught exception in thread Executor task launch worker-0 -java.lang.NullPointerException - at org.apache.spark.scheduler.Task$$anonfun$run$2.apply$mcV$sp(Task.scala:98) - at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1185) - at org.apache.spark.scheduler.Task.run(Task.scala:96) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-06-13 15:55:06 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2261 bytes result sent to driver -2016-06-13 15:55:06 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 15:55:06 ERROR Utils:96 - Uncaught exception in thread Executor task launch worker-1 -java.lang.NullPointerException - at org.apache.spark.scheduler.Task$$anonfun$run$1.apply$mcV$sp(Task.scala:94) - at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1185) - at org.apache.spark.scheduler.Task.run(Task.scala:92) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-06-13 15:55:06 ERROR Utils:96 - Uncaught exception in thread Executor task launch worker-1 -java.lang.NullPointerException - at org.apache.spark.scheduler.Task$$anonfun$run$2.apply$mcV$sp(Task.scala:98) - at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1185) - at org.apache.spark.scheduler.Task.run(Task.scala:96) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-06-13 15:55:06 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2285 bytes result sent to driver -2016-06-13 15:55:06 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:55:06 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:55:12 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:55:13 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:55:13 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:55:13 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:55:13 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:55:14 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:55:14 INFO Remoting:74 - Starting remoting -2016-06-13 15:55:14 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:58890] -2016-06-13 15:55:14 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58890. -2016-06-13 15:55:14 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 15:55:14 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 15:55:14 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-3467bb88-34ce-4c88-bb93-b59f881f3b6f -2016-06-13 15:55:14 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 15:55:14 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-bfcd61dc-1682-4b0b-8723-0b71c94ffc50\httpd-3dadbf9f-eeb7-4ca0-b28a-151154424bc4 -2016-06-13 15:55:14 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 15:55:14 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:55:14 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58891 -2016-06-13 15:55:14 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58891. -2016-06-13 15:55:14 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 15:55:14 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:55:14 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:55:14 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 15:55:14 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 15:55:15 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 15:55:15 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 15:55:15 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58910. -2016-06-13 15:55:15 INFO NettyBlockTransferService:59 - Server created on 58910 -2016-06-13 15:55:15 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 15:55:15 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:58910 with 730.6 MB RAM, BlockManagerId(driver, localhost, 58910) -2016-06-13 15:55:15 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 15:55:16 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 15:55:16 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 15:55:16 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 15:55:16 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 15:55:16 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:58910 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 15:55:16 INFO SparkContext:59 - Created broadcast 0 from textFile at RowMatrixSuite.scala:16 -2016-06-13 15:55:17 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 15:55:17 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 15:55:17 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-06-13 15:55:17 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 3 output partitions -2016-06-13 15:55:17 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-06-13 15:55:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:55:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:55:17 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:55:17 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=171580, maxMem=766075207 -2016-06-13 15:55:17 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 15:55:17 INFO MemoryStore:59 - ensureFreeSpace(1945) called with curMem=174956, maxMem=766075207 -2016-06-13 15:55:17 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1945.0 B, free 730.4 MB) -2016-06-13 15:55:17 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:58910 (size: 1945.0 B, free: 730.6 MB) -2016-06-13 15:55:17 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:55:17 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:55:17 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-06-13 15:55:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:17 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 15:55:17 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-06-13 15:55:17 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 15:55:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:18+1 -2016-06-13 15:55:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:9+9 -2016-06-13 15:55:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+9 -2016-06-13 15:55:17 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 15:55:17 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 15:55:17 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 15:55:17 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 15:55:17 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 15:55:17 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 15:55:17 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 2082 bytes result sent to driver -2016-06-13 15:55:17 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 15:55:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 87 ms on localhost (1/3) -2016-06-13 15:55:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 111 ms on localhost (2/3) -2016-06-13 15:55:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 87 ms on localhost (3/3) -2016-06-13 15:55:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 15:55:17 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.127 s -2016-06-13 15:55:17 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.255486 s -2016-06-13 15:55:17 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 15:55:17 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 15:55:17 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-06-13 15:55:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:55:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:55:17 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:55:17 INFO MemoryStore:59 - ensureFreeSpace(3544) called with curMem=176901, maxMem=766075207 -2016-06-13 15:55:17 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 15:55:17 INFO MemoryStore:59 - ensureFreeSpace(2003) called with curMem=180445, maxMem=766075207 -2016-06-13 15:55:17 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2003.0 B, free 730.4 MB) -2016-06-13 15:55:17 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:58910 (size: 2003.0 B, free: 730.6 MB) -2016-06-13 15:55:17 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:55:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:55:17 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 15:55:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:17 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 3) -2016-06-13 15:55:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+9 -2016-06-13 15:55:17 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 3). 2183 bytes result sent to driver -2016-06-13 15:55:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 3) in 24 ms on localhost (1/1) -2016-06-13 15:55:17 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.024 s -2016-06-13 15:55:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 15:55:17 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.046625 s -2016-06-13 15:55:17 INFO SparkContext:59 - Starting job: treeReduce at RowMatrix.scala:537 -2016-06-13 15:55:17 INFO DAGScheduler:59 - Got job 2 (treeReduce at RowMatrix.scala:537) with 3 output partitions -2016-06-13 15:55:17 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeReduce at RowMatrix.scala:537) -2016-06-13 15:55:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:55:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:55:17 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537), which has no missing parents -2016-06-13 15:55:17 INFO MemoryStore:59 - ensureFreeSpace(5080) called with curMem=182448, maxMem=766075207 -2016-06-13 15:55:17 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-06-13 15:55:17 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=187528, maxMem=766075207 -2016-06-13 15:55:17 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 15:55:17 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:58910 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 15:55:17 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:55:17 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537) -2016-06-13 15:55:17 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-06-13 15:55:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:17 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 15:55:17 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 15:55:17 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-06-13 15:55:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+9 -2016-06-13 15:55:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:9+9 -2016-06-13 15:55:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:18+1 -2016-06-13 15:55:18 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 15:55:18 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 15:55:18 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:55:18 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:58910 in memory (size: 2003.0 B, free: 730.6 MB) -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:55:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:55:18 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:58910 in memory (size: 1945.0 B, free: 730.6 MB) -2016-06-13 15:55:18 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 15:55:18 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 15:55:18 INFO DAGScheduler:59 - Job 2 failed: treeReduce at RowMatrix.scala:537, took 0.503513 s -2016-06-13 15:55:18 INFO DAGScheduler:59 - ResultStage 2 (treeReduce at RowMatrix.scala:537) failed in 0.488 s -2016-06-13 15:55:18 INFO SparkContext:59 - SparkContext already stopped. -2016-06-13 15:55:18 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 15:55:18 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 15:55:18 INFO BlockManager:59 - BlockManager stopped -2016-06-13 15:55:18 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 15:55:18 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 15:55:18 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:55:18 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 15:55:18 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-bfcd61dc-1682-4b0b-8723-0b71c94ffc50 -2016-06-13 15:55:40 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:55:41 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:55:41 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:55:41 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:55:41 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:55:42 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:55:42 INFO Remoting:74 - Starting remoting -2016-06-13 15:55:42 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:58930] -2016-06-13 15:55:42 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58930. -2016-06-13 15:55:42 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 15:55:42 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 15:55:42 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-a541a633-be60-48c0-a08e-d182c11f5250 -2016-06-13 15:55:42 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 15:55:42 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-bb9b7e6e-f8c7-4e88-b570-77d77cfa0a77\httpd-dfc5849e-fa26-45d2-bf55-603add6d7ef5 -2016-06-13 15:55:42 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 15:55:42 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:55:42 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58931 -2016-06-13 15:55:42 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58931. -2016-06-13 15:55:42 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 15:55:42 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:55:42 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:55:42 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 15:55:42 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 15:55:42 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 15:55:42 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 15:55:43 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58950. -2016-06-13 15:55:43 INFO NettyBlockTransferService:59 - Server created on 58950 -2016-06-13 15:55:43 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 15:55:43 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:58950 with 730.6 MB RAM, BlockManagerId(driver, localhost, 58950) -2016-06-13 15:55:43 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 15:55:44 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 15:55:44 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 15:55:44 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 15:55:44 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 15:55:44 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:58950 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 15:55:44 INFO SparkContext:59 - Created broadcast 0 from textFile at RowMatrixSuite.scala:16 -2016-06-13 15:55:45 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 15:55:45 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 15:55:45 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-06-13 15:55:45 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 3 output partitions -2016-06-13 15:55:45 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-06-13 15:55:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:55:45 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:55:45 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:55:45 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=171580, maxMem=766075207 -2016-06-13 15:55:45 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 15:55:45 INFO MemoryStore:59 - ensureFreeSpace(1945) called with curMem=174956, maxMem=766075207 -2016-06-13 15:55:45 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1945.0 B, free 730.4 MB) -2016-06-13 15:55:45 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:58950 (size: 1945.0 B, free: 730.6 MB) -2016-06-13 15:55:45 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:55:45 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:55:45 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-06-13 15:55:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:45 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:45 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 15:55:45 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-06-13 15:55:45 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 15:55:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+9 -2016-06-13 15:55:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:18+1 -2016-06-13 15:55:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:9+9 -2016-06-13 15:55:45 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 15:55:45 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 15:55:45 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 15:55:45 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 15:55:45 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 15:55:45 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 15:55:45 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 15:55:45 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 2082 bytes result sent to driver -2016-06-13 15:55:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 186 ms on localhost (1/3) -2016-06-13 15:55:45 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 139 ms on localhost (2/3) -2016-06-13 15:55:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 142 ms on localhost (3/3) -2016-06-13 15:55:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 15:55:45 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.202 s -2016-06-13 15:55:45 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.338231 s -2016-06-13 15:55:45 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 15:55:45 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 15:55:45 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-06-13 15:55:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:55:45 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:55:45 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:55:45 INFO MemoryStore:59 - ensureFreeSpace(3544) called with curMem=176901, maxMem=766075207 -2016-06-13 15:55:45 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 15:55:45 INFO MemoryStore:59 - ensureFreeSpace(2003) called with curMem=180445, maxMem=766075207 -2016-06-13 15:55:45 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2003.0 B, free 730.4 MB) -2016-06-13 15:55:45 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:58950 (size: 2003.0 B, free: 730.6 MB) -2016-06-13 15:55:45 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:55:45 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:55:45 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 15:55:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:45 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 3) -2016-06-13 15:55:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+9 -2016-06-13 15:55:45 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 3). 2183 bytes result sent to driver -2016-06-13 15:55:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 3) in 22 ms on localhost (1/1) -2016-06-13 15:55:45 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.023 s -2016-06-13 15:55:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 15:55:45 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.044362 s -2016-06-13 15:55:45 INFO SparkContext:59 - Starting job: treeReduce at RowMatrix.scala:537 -2016-06-13 15:55:45 INFO DAGScheduler:59 - Got job 2 (treeReduce at RowMatrix.scala:537) with 3 output partitions -2016-06-13 15:55:45 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeReduce at RowMatrix.scala:537) -2016-06-13 15:55:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:55:45 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:55:45 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537), which has no missing parents -2016-06-13 15:55:45 INFO MemoryStore:59 - ensureFreeSpace(5080) called with curMem=182448, maxMem=766075207 -2016-06-13 15:55:45 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-06-13 15:55:45 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=187528, maxMem=766075207 -2016-06-13 15:55:45 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 15:55:45 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:58950 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 15:55:45 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:55:45 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537) -2016-06-13 15:55:45 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-06-13 15:55:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:45 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:45 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 15:55:45 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-06-13 15:55:45 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 15:55:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:9+9 -2016-06-13 15:55:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+9 -2016-06-13 15:55:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:18+1 -2016-06-13 15:55:46 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 15:55:46 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 15:55:46 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:55:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:55:46 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 15:55:46 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 15:55:46 INFO DAGScheduler:59 - Job 2 failed: treeReduce at RowMatrix.scala:537, took 0.453551 s -2016-06-13 15:55:46 INFO DAGScheduler:59 - ResultStage 2 (treeReduce at RowMatrix.scala:537) failed in 0.439 s -2016-06-13 15:55:46 INFO SparkContext:59 - SparkContext already stopped. -2016-06-13 15:55:46 WARN QueuedThreadPool:145 - 2 threads could not be stopped -2016-06-13 15:55:46 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 15:55:46 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 15:55:46 INFO BlockManager:59 - BlockManager stopped -2016-06-13 15:55:46 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 15:55:46 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 15:55:46 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:55:46 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 15:55:46 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-bb9b7e6e-f8c7-4e88-b570-77d77cfa0a77 -2016-06-13 15:55:51 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 15:55:52 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 15:55:52 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 15:55:52 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 15:55:52 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 15:55:53 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 15:55:53 INFO Remoting:74 - Starting remoting -2016-06-13 15:55:53 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:58969] -2016-06-13 15:55:53 INFO Utils:59 - Successfully started service 'sparkDriver' on port 58969. -2016-06-13 15:55:53 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 15:55:53 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 15:55:53 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-25e2fb6c-a632-48ec-970e-574a294e9e17 -2016-06-13 15:55:53 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 15:55:53 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-49fa9851-ef98-4d5e-a6d0-9e069eeaf6a3\httpd-89fcc7a2-3a1f-44fe-9706-5baa261969cd -2016-06-13 15:55:53 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 15:55:53 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:55:53 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:58970 -2016-06-13 15:55:53 INFO Utils:59 - Successfully started service 'HTTP file server' on port 58970. -2016-06-13 15:55:53 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 15:55:53 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 15:55:53 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 15:55:53 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 15:55:53 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 15:55:54 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 15:55:54 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 15:55:54 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58989. -2016-06-13 15:55:54 INFO NettyBlockTransferService:59 - Server created on 58989 -2016-06-13 15:55:54 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 15:55:54 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:58989 with 730.6 MB RAM, BlockManagerId(driver, localhost, 58989) -2016-06-13 15:55:54 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 15:55:55 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 15:55:55 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 15:55:55 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 15:55:55 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 15:55:55 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:58989 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 15:55:55 INFO SparkContext:59 - Created broadcast 0 from textFile at RowMatrixSuite.scala:16 -2016-06-13 15:55:56 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 15:55:56 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 15:55:56 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-06-13 15:55:56 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 2 output partitions -2016-06-13 15:55:56 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-06-13 15:55:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:55:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:55:56 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:55:56 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=171580, maxMem=766075207 -2016-06-13 15:55:56 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 15:55:56 INFO MemoryStore:59 - ensureFreeSpace(1945) called with curMem=174956, maxMem=766075207 -2016-06-13 15:55:56 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1945.0 B, free 730.4 MB) -2016-06-13 15:55:56 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:58989 (size: 1945.0 B, free: 730.6 MB) -2016-06-13 15:55:56 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:55:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:55:56 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 15:55:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:56 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 15:55:56 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 15:55:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 15:55:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 15:55:56 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 15:55:56 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 15:55:56 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 15:55:56 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 15:55:56 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 15:55:56 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 15:55:56 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 15:55:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 129 ms on localhost (1/2) -2016-06-13 15:55:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 114 ms on localhost (2/2) -2016-06-13 15:55:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 15:55:56 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.145 s -2016-06-13 15:55:56 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.281436 s -2016-06-13 15:55:56 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 15:55:56 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 15:55:56 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-06-13 15:55:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:55:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:55:56 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19), which has no missing parents -2016-06-13 15:55:56 INFO MemoryStore:59 - ensureFreeSpace(3544) called with curMem=176901, maxMem=766075207 -2016-06-13 15:55:56 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 15:55:56 INFO MemoryStore:59 - ensureFreeSpace(2003) called with curMem=180445, maxMem=766075207 -2016-06-13 15:55:56 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2003.0 B, free 730.4 MB) -2016-06-13 15:55:56 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:58989 (size: 2003.0 B, free: 730.6 MB) -2016-06-13 15:55:56 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:55:56 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:19) -2016-06-13 15:55:56 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 15:55:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:56 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 15:55:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 15:55:56 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2183 bytes result sent to driver -2016-06-13 15:55:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 37 ms on localhost (1/1) -2016-06-13 15:55:56 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.039 s -2016-06-13 15:55:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 15:55:56 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.061704 s -2016-06-13 15:55:56 INFO SparkContext:59 - Starting job: treeReduce at RowMatrix.scala:537 -2016-06-13 15:55:56 INFO DAGScheduler:59 - Got job 2 (treeReduce at RowMatrix.scala:537) with 2 output partitions -2016-06-13 15:55:56 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeReduce at RowMatrix.scala:537) -2016-06-13 15:55:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:55:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:55:56 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537), which has no missing parents -2016-06-13 15:55:56 INFO MemoryStore:59 - ensureFreeSpace(5080) called with curMem=182448, maxMem=766075207 -2016-06-13 15:55:56 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-06-13 15:55:56 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=187528, maxMem=766075207 -2016-06-13 15:55:56 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 15:55:56 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:58989 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 15:55:56 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:55:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537) -2016-06-13 15:55:56 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 15:55:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:56 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 15:55:56 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-06-13 15:55:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 15:55:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 15:55:57 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 15:55:57 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 15:55:57 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2285 bytes result sent to driver -2016-06-13 15:55:57 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2285 bytes result sent to driver -2016-06-13 15:55:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 621 ms on localhost (1/2) -2016-06-13 15:55:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 624 ms on localhost (2/2) -2016-06-13 15:55:57 INFO DAGScheduler:59 - ResultStage 2 (treeReduce at RowMatrix.scala:537) finished in 0.624 s -2016-06-13 15:55:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 15:55:57 INFO DAGScheduler:59 - Job 2 finished: treeReduce at RowMatrix.scala:537, took 0.644460 s -2016-06-13 15:55:57 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=190032, maxMem=766075207 -2016-06-13 15:55:57 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 112.0 B, free 730.4 MB) -2016-06-13 15:55:57 INFO MemoryStore:59 - ensureFreeSpace(98) called with curMem=190144, maxMem=766075207 -2016-06-13 15:55:57 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 98.0 B, free 730.4 MB) -2016-06-13 15:55:57 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:58989 (size: 98.0 B, free: 730.6 MB) -2016-06-13 15:55:57 INFO SparkContext:59 - Created broadcast 4 from broadcast at RowMatrix.scala:425 -2016-06-13 15:55:57 INFO SparkContext:59 - Starting job: foreach at RowMatrixSuite.scala:33 -2016-06-13 15:55:57 INFO DAGScheduler:59 - Got job 3 (foreach at RowMatrixSuite.scala:33) with 2 output partitions -2016-06-13 15:55:57 INFO DAGScheduler:59 - Final stage: ResultStage 3(foreach at RowMatrixSuite.scala:33) -2016-06-13 15:55:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 15:55:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 15:55:57 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at mapPartitions at RowMatrix.scala:426), which has no missing parents -2016-06-13 15:55:57 INFO MemoryStore:59 - ensureFreeSpace(3848) called with curMem=190242, maxMem=766075207 -2016-06-13 15:55:57 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 15:55:57 INFO MemoryStore:59 - ensureFreeSpace(2119) called with curMem=194090, maxMem=766075207 -2016-06-13 15:55:57 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 15:55:57 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:58989 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 15:55:57 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 15:55:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at mapPartitions at RowMatrix.scala:426) -2016-06-13 15:55:57 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 15:55:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 15:55:57 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-06-13 15:55:57 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-06-13 15:55:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 15:55:57 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 15:55:57 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 15:55:57 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 15:55:57 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2044 bytes result sent to driver -2016-06-13 15:55:57 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2044 bytes result sent to driver -2016-06-13 15:55:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 36 ms on localhost (1/2) -2016-06-13 15:55:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 36 ms on localhost (2/2) -2016-06-13 15:55:57 INFO DAGScheduler:59 - ResultStage 3 (foreach at RowMatrixSuite.scala:33) finished in 0.037 s -2016-06-13 15:55:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 15:55:57 INFO DAGScheduler:59 - Job 3 finished: foreach at RowMatrixSuite.scala:33, took 0.052305 s -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 15:55:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 15:55:57 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 15:55:57 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 15:55:57 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 15:55:57 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 15:55:57 INFO BlockManager:59 - BlockManager stopped -2016-06-13 15:55:57 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 15:55:57 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 15:55:57 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 15:55:57 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 15:55:57 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 15:55:57 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 15:55:57 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-49fa9851-ef98-4d5e-a6d0-9e069eeaf6a3 -2016-06-13 16:07:59 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 16:07:59 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 16:08:00 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 16:08:00 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 16:08:00 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 16:08:01 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 16:08:01 INFO Remoting:74 - Starting remoting -2016-06-13 16:08:01 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:59899] -2016-06-13 16:08:01 INFO Utils:59 - Successfully started service 'sparkDriver' on port 59899. -2016-06-13 16:08:01 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 16:08:01 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 16:08:01 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-f72e5d74-0f41-4e19-b966-2e4abc2b0499 -2016-06-13 16:08:01 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 16:08:01 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-6579a20a-2889-4ec8-af4d-214dad18d8e4\httpd-9da63133-c671-4cc2-a931-25b3273368cb -2016-06-13 16:08:01 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 16:08:01 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 16:08:01 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:59900 -2016-06-13 16:08:01 INFO Utils:59 - Successfully started service 'HTTP file server' on port 59900. -2016-06-13 16:08:01 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 16:08:01 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 16:08:01 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 16:08:01 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 16:08:01 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 16:08:02 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 16:08:02 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 16:08:02 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 59919. -2016-06-13 16:08:02 INFO NettyBlockTransferService:59 - Server created on 59919 -2016-06-13 16:08:02 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 16:08:02 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:59919 with 730.6 MB RAM, BlockManagerId(driver, localhost, 59919) -2016-06-13 16:08:02 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 16:08:03 INFO SparkContext:59 - Starting job: reduce at BlockMatrix.scala:189 -2016-06-13 16:08:03 INFO DAGScheduler:59 - Got job 0 (reduce at BlockMatrix.scala:189) with 3 output partitions -2016-06-13 16:08:03 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at BlockMatrix.scala:189) -2016-06-13 16:08:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:03 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:03 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at BlockMatrix.scala:186), which has no missing parents -2016-06-13 16:08:03 INFO MemoryStore:59 - ensureFreeSpace(2552) called with curMem=0, maxMem=766075207 -2016-06-13 16:08:03 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 2.5 KB, free 730.6 MB) -2016-06-13 16:08:03 INFO MemoryStore:59 - ensureFreeSpace(1464) called with curMem=2552, maxMem=766075207 -2016-06-13 16:08:03 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1464.0 B, free 730.6 MB) -2016-06-13 16:08:03 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:59919 (size: 1464.0 B, free: 730.6 MB) -2016-06-13 16:08:03 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:03 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at BlockMatrix.scala:186) -2016-06-13 16:08:03 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-06-13 16:08:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:03 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:03 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-06-13 16:08:03 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 16:08:03 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 16:08:03 INFO CacheManager:59 - Partition rdd_1_2 not found, computing it -2016-06-13 16:08:03 INFO CacheManager:59 - Partition rdd_1_0 not found, computing it -2016-06-13 16:08:03 INFO CacheManager:59 - Partition rdd_1_1 not found, computing it -2016-06-13 16:08:03 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=4016, maxMem=766075207 -2016-06-13 16:08:03 INFO MemoryStore:59 - Block rdd_1_0 stored as values in memory (estimated size 112.0 B, free 730.6 MB) -2016-06-13 16:08:03 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=4128, maxMem=766075207 -2016-06-13 16:08:03 INFO MemoryStore:59 - Block rdd_1_1 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:03 INFO BlockManagerInfo:59 - Added rdd_1_0 in memory on localhost:59919 (size: 112.0 B, free: 730.6 MB) -2016-06-13 16:08:03 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=4328, maxMem=766075207 -2016-06-13 16:08:03 INFO MemoryStore:59 - Block rdd_1_2 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:03 INFO BlockManagerInfo:59 - Added rdd_1_1 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:03 INFO BlockManagerInfo:59 - Added rdd_1_2 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:03 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1694 bytes result sent to driver -2016-06-13 16:08:03 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 1694 bytes result sent to driver -2016-06-13 16:08:03 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1694 bytes result sent to driver -2016-06-13 16:08:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 119 ms on localhost (1/3) -2016-06-13 16:08:03 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 123 ms on localhost (2/3) -2016-06-13 16:08:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 153 ms on localhost (3/3) -2016-06-13 16:08:03 INFO DAGScheduler:59 - ResultStage 0 (reduce at BlockMatrix.scala:189) finished in 0.163 s -2016-06-13 16:08:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 16:08:03 INFO DAGScheduler:59 - Job 0 finished: reduce at BlockMatrix.scala:189, took 0.360766 s -2016-06-13 16:08:03 INFO SparkContext:59 - Starting job: collect at CoordinateMatrix.scala:166 -2016-06-13 16:08:03 INFO DAGScheduler:59 - Got job 1 (collect at CoordinateMatrix.scala:166) with 3 output partitions -2016-06-13 16:08:03 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at CoordinateMatrix.scala:166) -2016-06-13 16:08:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:03 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:03 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at flatMap at BlockMatrix.scala:255), which has no missing parents -2016-06-13 16:08:03 INFO MemoryStore:59 - ensureFreeSpace(2664) called with curMem=4528, maxMem=766075207 -2016-06-13 16:08:03 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 16:08:03 INFO MemoryStore:59 - ensureFreeSpace(1522) called with curMem=7192, maxMem=766075207 -2016-06-13 16:08:03 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1522.0 B, free 730.6 MB) -2016-06-13 16:08:03 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:59919 (size: 1522.0 B, free: 730.6 MB) -2016-06-13 16:08:03 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:03 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at flatMap at BlockMatrix.scala:255) -2016-06-13 16:08:03 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 3 tasks -2016-06-13 16:08:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:03 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 5, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:03 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 3) -2016-06-13 16:08:03 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 5) -2016-06-13 16:08:03 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 4) -2016-06-13 16:08:03 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 3). 1092 bytes result sent to driver -2016-06-13 16:08:03 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 5). 1185 bytes result sent to driver -2016-06-13 16:08:03 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 4). 1155 bytes result sent to driver -2016-06-13 16:08:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 3) in 17 ms on localhost (1/3) -2016-06-13 16:08:03 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 5) in 16 ms on localhost (2/3) -2016-06-13 16:08:03 INFO DAGScheduler:59 - ResultStage 1 (collect at CoordinateMatrix.scala:166) finished in 0.020 s -2016-06-13 16:08:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 4) in 18 ms on localhost (3/3) -2016-06-13 16:08:03 INFO DAGScheduler:59 - Job 1 finished: collect at CoordinateMatrix.scala:166, took 0.036981 s -2016-06-13 16:08:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 16:08:04 INFO SparkContext:59 - Starting job: collect at BlockMatrix.scala:289 -2016-06-13 16:08:04 INFO DAGScheduler:59 - Got job 2 (collect at BlockMatrix.scala:289) with 3 output partitions -2016-06-13 16:08:04 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at BlockMatrix.scala:289) -2016-06-13 16:08:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:04 INFO DAGScheduler:59 - Submitting ResultStage 2 (ParallelCollectionRDD[0] at parallelize at BlockMatrixSuite.scala:46), which has no missing parents -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(1208) called with curMem=8714, maxMem=766075207 -2016-06-13 16:08:04 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 1208.0 B, free 730.6 MB) -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(774) called with curMem=9922, maxMem=766075207 -2016-06-13 16:08:04 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 774.0 B, free 730.6 MB) -2016-06-13 16:08:04 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:59919 (size: 774.0 B, free: 730.6 MB) -2016-06-13 16:08:04 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:04 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 2 (ParallelCollectionRDD[0] at parallelize at BlockMatrixSuite.scala:46) -2016-06-13 16:08:04 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 8, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:04 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 6) -2016-06-13 16:08:04 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 8) -2016-06-13 16:08:04 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 7) -2016-06-13 16:08:04 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 6). 1225 bytes result sent to driver -2016-06-13 16:08:04 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 7). 1304 bytes result sent to driver -2016-06-13 16:08:04 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 8). 1288 bytes result sent to driver -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 6) in 14 ms on localhost (1/3) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 7) in 14 ms on localhost (2/3) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 8) in 13 ms on localhost (3/3) -2016-06-13 16:08:04 INFO DAGScheduler:59 - ResultStage 2 (collect at BlockMatrix.scala:289) finished in 0.016 s -2016-06-13 16:08:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 16:08:04 INFO DAGScheduler:59 - Job 2 finished: collect at BlockMatrix.scala:289, took 0.027912 s -2016-06-13 16:08:04 INFO SparkContext:59 - Starting job: collect at IndexedRowMatrix.scala:193 -2016-06-13 16:08:04 INFO DAGScheduler:59 - Registering RDD 5 (map at CoordinateMatrix.scala:91) -2016-06-13 16:08:04 INFO DAGScheduler:59 - Got job 3 (collect at IndexedRowMatrix.scala:193) with 3 output partitions -2016-06-13 16:08:04 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at IndexedRowMatrix.scala:193) -2016-06-13 16:08:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3) -2016-06-13 16:08:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 3) -2016-06-13 16:08:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (MapPartitionsRDD[5] at map at CoordinateMatrix.scala:91), which has no missing parents -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(4352) called with curMem=10696, maxMem=766075207 -2016-06-13 16:08:04 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.3 KB, free 730.6 MB) -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(2276) called with curMem=15048, maxMem=766075207 -2016-06-13 16:08:04 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.6 MB) -2016-06-13 16:08:04 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:59919 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 16:08:04 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:04 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[5] at map at CoordinateMatrix.scala:91) -2016-06-13 16:08:04 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 3 tasks -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2446 bytes) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 11, localhost, PROCESS_LOCAL, 2430 bytes) -2016-06-13 16:08:04 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 9) -2016-06-13 16:08:04 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 10) -2016-06-13 16:08:04 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 11) -2016-06-13 16:08:04 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 10). 1160 bytes result sent to driver -2016-06-13 16:08:04 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 9). 1160 bytes result sent to driver -2016-06-13 16:08:04 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 11). 1160 bytes result sent to driver -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 9) in 85 ms on localhost (1/3) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 10) in 84 ms on localhost (2/3) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 11) in 83 ms on localhost (3/3) -2016-06-13 16:08:04 INFO DAGScheduler:59 - ShuffleMapStage 3 (map at CoordinateMatrix.scala:91) finished in 0.088 s -2016-06-13 16:08:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 16:08:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:08:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 16:08:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 4) -2016-06-13 16:08:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:08:04 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List() -2016-06-13 16:08:04 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[7] at map at CoordinateMatrix.scala:93), which is now runnable -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(5152) called with curMem=17324, maxMem=766075207 -2016-06-13 16:08:04 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.0 KB, free 730.6 MB) -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(2595) called with curMem=22476, maxMem=766075207 -2016-06-13 16:08:04 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.6 MB) -2016-06-13 16:08:04 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:59919 (size: 2.5 KB, free: 730.6 MB) -2016-06-13 16:08:04 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:04 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 4 (MapPartitionsRDD[7] at map at CoordinateMatrix.scala:93) -2016-06-13 16:08:04 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 3 tasks -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 12, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 14, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:04 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 12) -2016-06-13 16:08:04 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 13) -2016-06-13 16:08:04 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 14) -2016-06-13 16:08:04 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:08:04 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:08:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:04 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:08:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 8 ms -2016-06-13 16:08:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-06-13 16:08:04 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 14). 1520 bytes result sent to driver -2016-06-13 16:08:04 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 12). 1576 bytes result sent to driver -2016-06-13 16:08:04 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 13). 1576 bytes result sent to driver -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 13) in 128 ms on localhost (1/3) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 14) in 129 ms on localhost (2/3) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 12) in 133 ms on localhost (3/3) -2016-06-13 16:08:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 16:08:04 INFO DAGScheduler:59 - ResultStage 4 (collect at IndexedRowMatrix.scala:193) finished in 0.134 s -2016-06-13 16:08:04 INFO DAGScheduler:59 - Job 3 finished: collect at IndexedRowMatrix.scala:193, took 0.269920 s -2016-06-13 16:08:04 INFO SparkContext:59 - Starting job: collect at BlockMatrix.scala:289 -2016-06-13 16:08:04 INFO DAGScheduler:59 - Got job 4 (collect at BlockMatrix.scala:289) with 3 output partitions -2016-06-13 16:08:04 INFO DAGScheduler:59 - Final stage: ResultStage 5(collect at BlockMatrix.scala:289) -2016-06-13 16:08:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:04 INFO DAGScheduler:59 - Submitting ResultStage 5 (ParallelCollectionRDD[0] at parallelize at BlockMatrixSuite.scala:46), which has no missing parents -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(1208) called with curMem=25071, maxMem=766075207 -2016-06-13 16:08:04 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 1208.0 B, free 730.6 MB) -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(774) called with curMem=26279, maxMem=766075207 -2016-06-13 16:08:04 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 774.0 B, free 730.6 MB) -2016-06-13 16:08:04 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:59919 (size: 774.0 B, free: 730.6 MB) -2016-06-13 16:08:04 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:04 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 5 (ParallelCollectionRDD[0] at parallelize at BlockMatrixSuite.scala:46) -2016-06-13 16:08:04 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 3 tasks -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 15, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 16, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 17, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:04 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 16) -2016-06-13 16:08:04 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 15) -2016-06-13 16:08:04 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 15). 1225 bytes result sent to driver -2016-06-13 16:08:04 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 16). 1304 bytes result sent to driver -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 15) in 15 ms on localhost (1/3) -2016-06-13 16:08:04 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 17) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 16) in 17 ms on localhost (2/3) -2016-06-13 16:08:04 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 17). 1288 bytes result sent to driver -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 17) in 21 ms on localhost (3/3) -2016-06-13 16:08:04 INFO DAGScheduler:59 - ResultStage 5 (collect at BlockMatrix.scala:289) finished in 0.027 s -2016-06-13 16:08:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 16:08:04 INFO DAGScheduler:59 - Job 4 finished: collect at BlockMatrix.scala:289, took 0.043801 s -2016-06-13 16:08:04 INFO SparkContext:59 - Starting job: collect at BlockMatrix.scala:289 -2016-06-13 16:08:04 INFO DAGScheduler:59 - Got job 5 (collect at BlockMatrix.scala:289) with 3 output partitions -2016-06-13 16:08:04 INFO DAGScheduler:59 - Final stage: ResultStage 6(collect at BlockMatrix.scala:289) -2016-06-13 16:08:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:04 INFO DAGScheduler:59 - Submitting ResultStage 6 (ParallelCollectionRDD[0] at parallelize at BlockMatrixSuite.scala:46), which has no missing parents -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(1208) called with curMem=27053, maxMem=766075207 -2016-06-13 16:08:04 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 1208.0 B, free 730.6 MB) -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(774) called with curMem=28261, maxMem=766075207 -2016-06-13 16:08:04 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 774.0 B, free 730.6 MB) -2016-06-13 16:08:04 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:59919 (size: 774.0 B, free: 730.6 MB) -2016-06-13 16:08:04 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:04 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 6 (ParallelCollectionRDD[0] at parallelize at BlockMatrixSuite.scala:46) -2016-06-13 16:08:04 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 3 tasks -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 18, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 19, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 20, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:04 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 19) -2016-06-13 16:08:04 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 19). 1304 bytes result sent to driver -2016-06-13 16:08:04 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 18) -2016-06-13 16:08:04 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 18). 1225 bytes result sent to driver -2016-06-13 16:08:04 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 20) -2016-06-13 16:08:04 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 20). 1288 bytes result sent to driver -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 19) in 25 ms on localhost (1/3) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 18) in 28 ms on localhost (2/3) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 20) in 27 ms on localhost (3/3) -2016-06-13 16:08:04 INFO DAGScheduler:59 - ResultStage 6 (collect at BlockMatrix.scala:289) finished in 0.032 s -2016-06-13 16:08:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 16:08:04 INFO DAGScheduler:59 - Job 5 finished: collect at BlockMatrix.scala:289, took 0.048185 s -2016-06-13 16:08:04 INFO SparkContext:59 - Starting job: collect at BlockMatrix.scala:289 -2016-06-13 16:08:04 INFO DAGScheduler:59 - Got job 6 (collect at BlockMatrix.scala:289) with 3 output partitions -2016-06-13 16:08:04 INFO DAGScheduler:59 - Final stage: ResultStage 7(collect at BlockMatrix.scala:289) -2016-06-13 16:08:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:04 INFO DAGScheduler:59 - Submitting ResultStage 7 (ParallelCollectionRDD[0] at parallelize at BlockMatrixSuite.scala:46), which has no missing parents -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(1208) called with curMem=29035, maxMem=766075207 -2016-06-13 16:08:04 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 1208.0 B, free 730.6 MB) -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(774) called with curMem=30243, maxMem=766075207 -2016-06-13 16:08:04 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 774.0 B, free 730.6 MB) -2016-06-13 16:08:04 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:59919 (size: 774.0 B, free: 730.6 MB) -2016-06-13 16:08:04 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:04 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 7 (ParallelCollectionRDD[0] at parallelize at BlockMatrixSuite.scala:46) -2016-06-13 16:08:04 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 3 tasks -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 21, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 22, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 2.0 in stage 7.0 (TID 23, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:04 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 22) -2016-06-13 16:08:04 INFO Executor:59 - Running task 2.0 in stage 7.0 (TID 23) -2016-06-13 16:08:04 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 21) -2016-06-13 16:08:04 INFO Executor:59 - Finished task 2.0 in stage 7.0 (TID 23). 1288 bytes result sent to driver -2016-06-13 16:08:04 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 21). 1225 bytes result sent to driver -2016-06-13 16:08:04 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 22). 1304 bytes result sent to driver -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 21) in 12 ms on localhost (1/3) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 2.0 in stage 7.0 (TID 23) in 10 ms on localhost (2/3) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 22) in 12 ms on localhost (3/3) -2016-06-13 16:08:04 INFO DAGScheduler:59 - ResultStage 7 (collect at BlockMatrix.scala:289) finished in 0.013 s -2016-06-13 16:08:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 16:08:04 INFO DAGScheduler:59 - Job 6 finished: collect at BlockMatrix.scala:289, took 0.030997 s -2016-06-13 16:08:04 INFO SparkContext:59 - Starting job: reduce at BlockMatrix.scala:189 -2016-06-13 16:08:04 INFO DAGScheduler:59 - Got job 7 (reduce at BlockMatrix.scala:189) with 3 output partitions -2016-06-13 16:08:04 INFO DAGScheduler:59 - Final stage: ResultStage 8(reduce at BlockMatrix.scala:189) -2016-06-13 16:08:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:04 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[10] at map at BlockMatrix.scala:186), which has no missing parents -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(2552) called with curMem=31017, maxMem=766075207 -2016-06-13 16:08:04 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 2.5 KB, free 730.6 MB) -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(1458) called with curMem=33569, maxMem=766075207 -2016-06-13 16:08:04 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 1458.0 B, free 730.6 MB) -2016-06-13 16:08:04 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:59919 (size: 1458.0 B, free: 730.6 MB) -2016-06-13 16:08:04 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:04 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 8 (MapPartitionsRDD[10] at map at BlockMatrix.scala:186) -2016-06-13 16:08:04 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 3 tasks -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 24, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 25, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 2.0 in stage 8.0 (TID 26, localhost, PROCESS_LOCAL, 2425 bytes) -2016-06-13 16:08:04 INFO Executor:59 - Running task 2.0 in stage 8.0 (TID 26) -2016-06-13 16:08:04 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 25) -2016-06-13 16:08:04 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 24) -2016-06-13 16:08:04 INFO CacheManager:59 - Partition rdd_9_1 not found, computing it -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=35027, maxMem=766075207 -2016-06-13 16:08:04 INFO MemoryStore:59 - Block rdd_9_1 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:04 INFO BlockManagerInfo:59 - Added rdd_9_1 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:04 INFO CacheManager:59 - Partition rdd_9_0 not found, computing it -2016-06-13 16:08:04 INFO CacheManager:59 - Partition rdd_9_2 not found, computing it -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=35227, maxMem=766075207 -2016-06-13 16:08:04 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 25). 1694 bytes result sent to driver -2016-06-13 16:08:04 INFO MemoryStore:59 - Block rdd_9_0 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:04 INFO BlockManagerInfo:59 - Added rdd_9_0 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=35427, maxMem=766075207 -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 25) in 20 ms on localhost (1/3) -2016-06-13 16:08:04 INFO MemoryStore:59 - Block rdd_9_2 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:04 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 24). 1694 bytes result sent to driver -2016-06-13 16:08:04 INFO BlockManagerInfo:59 - Added rdd_9_2 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:04 INFO Executor:59 - Finished task 2.0 in stage 8.0 (TID 26). 1694 bytes result sent to driver -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 24) in 26 ms on localhost (2/3) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 2.0 in stage 8.0 (TID 26) in 24 ms on localhost (3/3) -2016-06-13 16:08:04 INFO DAGScheduler:59 - ResultStage 8 (reduce at BlockMatrix.scala:189) finished in 0.028 s -2016-06-13 16:08:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 16:08:04 INFO DAGScheduler:59 - Job 7 finished: reduce at BlockMatrix.scala:189, took 0.052684 s -2016-06-13 16:08:04 INFO SparkContext:59 - Starting job: collect at BlockMatrix.scala:289 -2016-06-13 16:08:04 INFO DAGScheduler:59 - Registering RDD 0 (parallelize at BlockMatrixSuite.scala:46) -2016-06-13 16:08:04 INFO DAGScheduler:59 - Registering RDD 8 (parallelize at BlockMatrixSuite.scala:159) -2016-06-13 16:08:04 INFO DAGScheduler:59 - Got job 8 (collect at BlockMatrix.scala:289) with 4 output partitions -2016-06-13 16:08:04 INFO DAGScheduler:59 - Final stage: ResultStage 11(collect at BlockMatrix.scala:289) -2016-06-13 16:08:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9, ShuffleMapStage 10) -2016-06-13 16:08:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 9, ShuffleMapStage 10) -2016-06-13 16:08:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 9 (ParallelCollectionRDD[0] at parallelize at BlockMatrixSuite.scala:46), which has no missing parents -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(1616) called with curMem=35627, maxMem=766075207 -2016-06-13 16:08:04 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 1616.0 B, free 730.6 MB) -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(1077) called with curMem=37243, maxMem=766075207 -2016-06-13 16:08:04 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1077.0 B, free 730.5 MB) -2016-06-13 16:08:04 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:59919 (size: 1077.0 B, free: 730.6 MB) -2016-06-13 16:08:04 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:04 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 9 (ParallelCollectionRDD[0] at parallelize at BlockMatrixSuite.scala:46) -2016-06-13 16:08:04 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 3 tasks -2016-06-13 16:08:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 10 (ParallelCollectionRDD[8] at parallelize at BlockMatrixSuite.scala:159), which has no missing parents -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 27, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(1616) called with curMem=38320, maxMem=766075207 -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 28, localhost, PROCESS_LOCAL, 2446 bytes) -2016-06-13 16:08:04 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 1616.0 B, free 730.5 MB) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 2.0 in stage 9.0 (TID 29, localhost, PROCESS_LOCAL, 2430 bytes) -2016-06-13 16:08:04 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 27) -2016-06-13 16:08:04 INFO Executor:59 - Running task 2.0 in stage 9.0 (TID 29) -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(1080) called with curMem=39936, maxMem=766075207 -2016-06-13 16:08:04 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 1080.0 B, free 730.5 MB) -2016-06-13 16:08:04 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 28) -2016-06-13 16:08:04 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:59919 (size: 1080.0 B, free: 730.6 MB) -2016-06-13 16:08:04 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:04 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 10 (ParallelCollectionRDD[8] at parallelize at BlockMatrixSuite.scala:159) -2016-06-13 16:08:04 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 3 tasks -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 30, localhost, PROCESS_LOCAL, 2446 bytes) -2016-06-13 16:08:04 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 30) -2016-06-13 16:08:04 INFO Executor:59 - Finished task 2.0 in stage 9.0 (TID 29). 1161 bytes result sent to driver -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 31, localhost, PROCESS_LOCAL, 2446 bytes) -2016-06-13 16:08:04 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 31) -2016-06-13 16:08:04 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 27). 1161 bytes result sent to driver -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 2.0 in stage 9.0 (TID 29) in 73 ms on localhost (1/3) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Starting task 2.0 in stage 10.0 (TID 32, localhost, PROCESS_LOCAL, 2414 bytes) -2016-06-13 16:08:04 INFO Executor:59 - Running task 2.0 in stage 10.0 (TID 32) -2016-06-13 16:08:04 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 28). 1161 bytes result sent to driver -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 27) in 107 ms on localhost (2/3) -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 28) in 107 ms on localhost (3/3) -2016-06-13 16:08:04 INFO DAGScheduler:59 - ShuffleMapStage 9 (parallelize at BlockMatrixSuite.scala:46) finished in 0.109 s -2016-06-13 16:08:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 16:08:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:08:04 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 10) -2016-06-13 16:08:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 11) -2016-06-13 16:08:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:08:04 INFO DAGScheduler:59 - Missing parents for ResultStage 11: List(ShuffleMapStage 10) -2016-06-13 16:08:04 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 30). 1161 bytes result sent to driver -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 30) in 98 ms on localhost (1/3) -2016-06-13 16:08:04 INFO Executor:59 - Finished task 2.0 in stage 10.0 (TID 32). 1161 bytes result sent to driver -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 2.0 in stage 10.0 (TID 32) in 52 ms on localhost (2/3) -2016-06-13 16:08:04 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 31). 1161 bytes result sent to driver -2016-06-13 16:08:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 31) in 78 ms on localhost (3/3) -2016-06-13 16:08:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 16:08:04 INFO DAGScheduler:59 - ShuffleMapStage 10 (parallelize at BlockMatrixSuite.scala:159) finished in 0.124 s -2016-06-13 16:08:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:08:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 16:08:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 11) -2016-06-13 16:08:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:08:04 INFO DAGScheduler:59 - Missing parents for ResultStage 11: List() -2016-06-13 16:08:04 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[13] at map at BlockMatrix.scala:335), which is now runnable -2016-06-13 16:08:04 INFO MemoryStore:59 - ensureFreeSpace(2872) called with curMem=41016, maxMem=766075207 -2016-06-13 16:08:04 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 2.8 KB, free 730.5 MB) -2016-06-13 16:08:05 INFO MemoryStore:59 - ensureFreeSpace(1660) called with curMem=43888, maxMem=766075207 -2016-06-13 16:08:05 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 1660.0 B, free 730.5 MB) -2016-06-13 16:08:05 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:59919 (size: 1660.0 B, free: 730.6 MB) -2016-06-13 16:08:05 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:05 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 11 (MapPartitionsRDD[13] at map at BlockMatrix.scala:335) -2016-06-13 16:08:05 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 4 tasks -2016-06-13 16:08:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 33, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 16:08:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 34, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 16:08:05 INFO TaskSetManager:59 - Starting task 2.0 in stage 11.0 (TID 35, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 16:08:05 INFO TaskSetManager:59 - Starting task 3.0 in stage 11.0 (TID 36, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 16:08:05 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 33) -2016-06-13 16:08:05 INFO Executor:59 - Running task 3.0 in stage 11.0 (TID 36) -2016-06-13 16:08:05 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 34) -2016-06-13 16:08:05 INFO Executor:59 - Running task 2.0 in stage 11.0 (TID 35) -2016-06-13 16:08:05 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:59919 in memory (size: 1077.0 B, free: 730.6 MB) -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:05 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 34). 1459 bytes result sent to driver -2016-06-13 16:08:05 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:59919 in memory (size: 1458.0 B, free: 730.6 MB) -2016-06-13 16:08:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 34) in 45 ms on localhost (1/4) -2016-06-13 16:08:05 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 16:08:05 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:59919 in memory (size: 774.0 B, free: 730.6 MB) -2016-06-13 16:08:05 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 16:08:05 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:59919 in memory (size: 774.0 B, free: 730.6 MB) -2016-06-13 16:08:05 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 16:08:05 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:59919 in memory (size: 774.0 B, free: 730.6 MB) -2016-06-13 16:08:05 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 16:08:05 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:59919 in memory (size: 2.5 KB, free: 730.6 MB) -2016-06-13 16:08:05 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 16:08:05 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:59919 in memory (size: 2.2 KB, free: 730.6 MB) -2016-06-13 16:08:05 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 16:08:05 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 33). 1554 bytes result sent to driver -2016-06-13 16:08:05 INFO Executor:59 - Finished task 3.0 in stage 11.0 (TID 36). 1459 bytes result sent to driver -2016-06-13 16:08:05 INFO Executor:59 - Finished task 2.0 in stage 11.0 (TID 35). 1554 bytes result sent to driver -2016-06-13 16:08:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 33) in 83 ms on localhost (2/4) -2016-06-13 16:08:05 INFO TaskSetManager:59 - Finished task 3.0 in stage 11.0 (TID 36) in 79 ms on localhost (3/4) -2016-06-13 16:08:05 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-06-13 16:08:05 INFO TaskSetManager:59 - Finished task 2.0 in stage 11.0 (TID 35) in 83 ms on localhost (4/4) -2016-06-13 16:08:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 16:08:05 INFO DAGScheduler:59 - ResultStage 11 (collect at BlockMatrix.scala:289) finished in 0.089 s -2016-06-13 16:08:05 INFO DAGScheduler:59 - Job 8 finished: collect at BlockMatrix.scala:289, took 0.306621 s -2016-06-13 16:08:05 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:59919 in memory (size: 774.0 B, free: 730.6 MB) -2016-06-13 16:08:05 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 16:08:05 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:59919 in memory (size: 1522.0 B, free: 730.6 MB) -2016-06-13 16:08:05 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 16:08:05 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:59919 in memory (size: 1464.0 B, free: 730.6 MB) -2016-06-13 16:08:05 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 16:08:05 INFO SparkContext:59 - Starting job: collect at BlockMatrix.scala:289 -2016-06-13 16:08:05 INFO DAGScheduler:59 - Registering RDD 15 (makeRDD at BlockMatrixSuite.scala:188) -2016-06-13 16:08:05 INFO DAGScheduler:59 - Registering RDD 15 (makeRDD at BlockMatrixSuite.scala:188) -2016-06-13 16:08:05 INFO DAGScheduler:59 - Got job 9 (collect at BlockMatrix.scala:289) with 4 output partitions -2016-06-13 16:08:05 INFO DAGScheduler:59 - Final stage: ResultStage 14(collect at BlockMatrix.scala:289) -2016-06-13 16:08:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12, ShuffleMapStage 13) -2016-06-13 16:08:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12, ShuffleMapStage 13) -2016-06-13 16:08:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (ParallelCollectionRDD[15] at makeRDD at BlockMatrixSuite.scala:188), which has no missing parents -2016-06-13 16:08:05 INFO MemoryStore:59 - ensureFreeSpace(1616) called with curMem=8340, maxMem=766075207 -2016-06-13 16:08:05 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 1616.0 B, free 730.6 MB) -2016-06-13 16:08:05 INFO MemoryStore:59 - ensureFreeSpace(1077) called with curMem=9956, maxMem=766075207 -2016-06-13 16:08:05 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 1077.0 B, free 730.6 MB) -2016-06-13 16:08:05 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:59919 (size: 1077.0 B, free: 730.6 MB) -2016-06-13 16:08:05 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:05 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 12 (ParallelCollectionRDD[15] at makeRDD at BlockMatrixSuite.scala:188) -2016-06-13 16:08:05 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 4 tasks -2016-06-13 16:08:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 13 (ParallelCollectionRDD[15] at makeRDD at BlockMatrixSuite.scala:188), which has no missing parents -2016-06-13 16:08:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 37, localhost, PROCESS_LOCAL, 2470 bytes) -2016-06-13 16:08:05 INFO MemoryStore:59 - ensureFreeSpace(1616) called with curMem=11033, maxMem=766075207 -2016-06-13 16:08:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 38, localhost, PROCESS_LOCAL, 2470 bytes) -2016-06-13 16:08:05 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 1616.0 B, free 730.6 MB) -2016-06-13 16:08:05 INFO TaskSetManager:59 - Starting task 2.0 in stage 12.0 (TID 39, localhost, PROCESS_LOCAL, 2470 bytes) -2016-06-13 16:08:05 INFO MemoryStore:59 - ensureFreeSpace(1075) called with curMem=12649, maxMem=766075207 -2016-06-13 16:08:05 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 1075.0 B, free 730.6 MB) -2016-06-13 16:08:05 INFO TaskSetManager:59 - Starting task 3.0 in stage 12.0 (TID 40, localhost, PROCESS_LOCAL, 2470 bytes) -2016-06-13 16:08:05 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 37) -2016-06-13 16:08:05 INFO Executor:59 - Running task 3.0 in stage 12.0 (TID 40) -2016-06-13 16:08:05 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:59919 (size: 1075.0 B, free: 730.6 MB) -2016-06-13 16:08:05 INFO Executor:59 - Running task 2.0 in stage 12.0 (TID 39) -2016-06-13 16:08:05 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:05 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 13 (ParallelCollectionRDD[15] at makeRDD at BlockMatrixSuite.scala:188) -2016-06-13 16:08:05 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 4 tasks -2016-06-13 16:08:05 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 38) -2016-06-13 16:08:05 INFO Executor:59 - Finished task 2.0 in stage 12.0 (TID 39). 1161 bytes result sent to driver -2016-06-13 16:08:05 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 37). 1161 bytes result sent to driver -2016-06-13 16:08:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 41, localhost, PROCESS_LOCAL, 2470 bytes) -2016-06-13 16:08:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 42, localhost, PROCESS_LOCAL, 2470 bytes) -2016-06-13 16:08:05 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 41) -2016-06-13 16:08:05 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 42) -2016-06-13 16:08:05 INFO TaskSetManager:59 - Finished task 2.0 in stage 12.0 (TID 39) in 96 ms on localhost (1/4) -2016-06-13 16:08:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 37) in 102 ms on localhost (2/4) -2016-06-13 16:08:05 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 38). 1161 bytes result sent to driver -2016-06-13 16:08:05 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 42). 1161 bytes result sent to driver -2016-06-13 16:08:05 INFO TaskSetManager:59 - Starting task 2.0 in stage 13.0 (TID 43, localhost, PROCESS_LOCAL, 2470 bytes) -2016-06-13 16:08:05 INFO Executor:59 - Running task 2.0 in stage 13.0 (TID 43) -2016-06-13 16:08:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 38) in 134 ms on localhost (3/4) -2016-06-13 16:08:05 INFO TaskSetManager:59 - Starting task 3.0 in stage 13.0 (TID 44, localhost, PROCESS_LOCAL, 2470 bytes) -2016-06-13 16:08:05 INFO Executor:59 - Running task 3.0 in stage 13.0 (TID 44) -2016-06-13 16:08:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 42) in 43 ms on localhost (1/4) -2016-06-13 16:08:05 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 41). 1161 bytes result sent to driver -2016-06-13 16:08:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 41) in 52 ms on localhost (2/4) -2016-06-13 16:08:05 INFO Executor:59 - Finished task 3.0 in stage 12.0 (TID 40). 1161 bytes result sent to driver -2016-06-13 16:08:05 INFO TaskSetManager:59 - Finished task 3.0 in stage 12.0 (TID 40) in 146 ms on localhost (4/4) -2016-06-13 16:08:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 16:08:05 INFO DAGScheduler:59 - ShuffleMapStage 12 (makeRDD at BlockMatrixSuite.scala:188) finished in 0.151 s -2016-06-13 16:08:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:08:05 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 13) -2016-06-13 16:08:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 14) -2016-06-13 16:08:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:08:05 INFO DAGScheduler:59 - Missing parents for ResultStage 14: List(ShuffleMapStage 13) -2016-06-13 16:08:05 INFO Executor:59 - Finished task 2.0 in stage 13.0 (TID 43). 1161 bytes result sent to driver -2016-06-13 16:08:05 INFO Executor:59 - Finished task 3.0 in stage 13.0 (TID 44). 1161 bytes result sent to driver -2016-06-13 16:08:05 INFO TaskSetManager:59 - Finished task 2.0 in stage 13.0 (TID 43) in 58 ms on localhost (3/4) -2016-06-13 16:08:05 INFO TaskSetManager:59 - Finished task 3.0 in stage 13.0 (TID 44) in 56 ms on localhost (4/4) -2016-06-13 16:08:05 INFO DAGScheduler:59 - ShuffleMapStage 13 (makeRDD at BlockMatrixSuite.scala:188) finished in 0.179 s -2016-06-13 16:08:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 16:08:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:08:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 16:08:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 14) -2016-06-13 16:08:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:08:05 INFO DAGScheduler:59 - Missing parents for ResultStage 14: List() -2016-06-13 16:08:05 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[19] at map at BlockMatrix.scala:335), which is now runnable -2016-06-13 16:08:05 INFO MemoryStore:59 - ensureFreeSpace(2872) called with curMem=13724, maxMem=766075207 -2016-06-13 16:08:05 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 2.8 KB, free 730.6 MB) -2016-06-13 16:08:05 INFO MemoryStore:59 - ensureFreeSpace(1662) called with curMem=16596, maxMem=766075207 -2016-06-13 16:08:05 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 1662.0 B, free 730.6 MB) -2016-06-13 16:08:05 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:59919 (size: 1662.0 B, free: 730.6 MB) -2016-06-13 16:08:05 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:05 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 14 (MapPartitionsRDD[19] at map at BlockMatrix.scala:335) -2016-06-13 16:08:05 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 4 tasks -2016-06-13 16:08:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 45, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 16:08:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 46, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 16:08:05 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 47, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 16:08:05 INFO TaskSetManager:59 - Starting task 3.0 in stage 14.0 (TID 48, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 16:08:05 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 45) -2016-06-13 16:08:05 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 47) -2016-06-13 16:08:05 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 46) -2016-06-13 16:08:05 INFO Executor:59 - Running task 3.0 in stage 14.0 (TID 48) -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 16:08:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:06 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 47). 1578 bytes result sent to driver -2016-06-13 16:08:06 INFO Executor:59 - Finished task 3.0 in stage 14.0 (TID 48). 1578 bytes result sent to driver -2016-06-13 16:08:06 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 46). 1578 bytes result sent to driver -2016-06-13 16:08:06 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 47) in 1125 ms on localhost (1/4) -2016-06-13 16:08:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 46) in 1127 ms on localhost (2/4) -2016-06-13 16:08:06 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 45). 1578 bytes result sent to driver -2016-06-13 16:08:06 INFO TaskSetManager:59 - Finished task 3.0 in stage 14.0 (TID 48) in 1126 ms on localhost (3/4) -2016-06-13 16:08:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 45) in 1131 ms on localhost (4/4) -2016-06-13 16:08:06 INFO DAGScheduler:59 - ResultStage 14 (collect at BlockMatrix.scala:289) finished in 1.131 s -2016-06-13 16:08:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 16:08:06 INFO DAGScheduler:59 - Job 9 finished: collect at BlockMatrix.scala:289, took 1.356140 s -2016-06-13 16:08:06 INFO SparkContext:59 - Starting job: collect at BlockMatrix.scala:289 -2016-06-13 16:08:06 INFO DAGScheduler:59 - Registering RDD 15 (makeRDD at BlockMatrixSuite.scala:188) -2016-06-13 16:08:06 INFO DAGScheduler:59 - Registering RDD 16 (makeRDD at BlockMatrixSuite.scala:189) -2016-06-13 16:08:06 INFO DAGScheduler:59 - Got job 10 (collect at BlockMatrix.scala:289) with 4 output partitions -2016-06-13 16:08:06 INFO DAGScheduler:59 - Final stage: ResultStage 17(collect at BlockMatrix.scala:289) -2016-06-13 16:08:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15, ShuffleMapStage 16) -2016-06-13 16:08:06 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 15, ShuffleMapStage 16) -2016-06-13 16:08:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 15 (ParallelCollectionRDD[15] at makeRDD at BlockMatrixSuite.scala:188), which has no missing parents -2016-06-13 16:08:06 INFO MemoryStore:59 - ensureFreeSpace(1616) called with curMem=18258, maxMem=766075207 -2016-06-13 16:08:06 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1616.0 B, free 730.6 MB) -2016-06-13 16:08:06 INFO MemoryStore:59 - ensureFreeSpace(1077) called with curMem=19874, maxMem=766075207 -2016-06-13 16:08:06 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1077.0 B, free 730.6 MB) -2016-06-13 16:08:06 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:59919 (size: 1077.0 B, free: 730.6 MB) -2016-06-13 16:08:06 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:06 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 15 (ParallelCollectionRDD[15] at makeRDD at BlockMatrixSuite.scala:188) -2016-06-13 16:08:06 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 4 tasks -2016-06-13 16:08:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 16 (ParallelCollectionRDD[16] at makeRDD at BlockMatrixSuite.scala:189), which has no missing parents -2016-06-13 16:08:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 49, localhost, PROCESS_LOCAL, 2470 bytes) -2016-06-13 16:08:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 50, localhost, PROCESS_LOCAL, 2470 bytes) -2016-06-13 16:08:06 INFO MemoryStore:59 - ensureFreeSpace(1616) called with curMem=20951, maxMem=766075207 -2016-06-13 16:08:06 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 1616.0 B, free 730.6 MB) -2016-06-13 16:08:06 INFO TaskSetManager:59 - Starting task 2.0 in stage 15.0 (TID 51, localhost, PROCESS_LOCAL, 2470 bytes) -2016-06-13 16:08:06 INFO TaskSetManager:59 - Starting task 3.0 in stage 15.0 (TID 52, localhost, PROCESS_LOCAL, 2470 bytes) -2016-06-13 16:08:06 INFO MemoryStore:59 - ensureFreeSpace(1077) called with curMem=22567, maxMem=766075207 -2016-06-13 16:08:06 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 50) -2016-06-13 16:08:06 INFO Executor:59 - Running task 3.0 in stage 15.0 (TID 52) -2016-06-13 16:08:06 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1077.0 B, free 730.6 MB) -2016-06-13 16:08:06 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:59919 (size: 1077.0 B, free: 730.6 MB) -2016-06-13 16:08:06 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:06 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 49) -2016-06-13 16:08:06 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 16 (ParallelCollectionRDD[16] at makeRDD at BlockMatrixSuite.scala:189) -2016-06-13 16:08:06 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 4 tasks -2016-06-13 16:08:06 INFO Executor:59 - Running task 2.0 in stage 15.0 (TID 51) -2016-06-13 16:08:06 INFO Executor:59 - Finished task 3.0 in stage 15.0 (TID 52). 1161 bytes result sent to driver -2016-06-13 16:08:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 53, localhost, PROCESS_LOCAL, 2463 bytes) -2016-06-13 16:08:06 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 53) -2016-06-13 16:08:06 INFO TaskSetManager:59 - Finished task 3.0 in stage 15.0 (TID 52) in 50 ms on localhost (1/4) -2016-06-13 16:08:06 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 50). 1161 bytes result sent to driver -2016-06-13 16:08:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 54, localhost, PROCESS_LOCAL, 2463 bytes) -2016-06-13 16:08:06 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 54) -2016-06-13 16:08:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 50) in 63 ms on localhost (2/4) -2016-06-13 16:08:06 INFO Executor:59 - Finished task 2.0 in stage 15.0 (TID 51). 1161 bytes result sent to driver -2016-06-13 16:08:06 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 49). 1161 bytes result sent to driver -2016-06-13 16:08:06 INFO TaskSetManager:59 - Starting task 2.0 in stage 16.0 (TID 55, localhost, PROCESS_LOCAL, 2463 bytes) -2016-06-13 16:08:06 INFO TaskSetManager:59 - Starting task 3.0 in stage 16.0 (TID 56, localhost, PROCESS_LOCAL, 2463 bytes) -2016-06-13 16:08:06 INFO Executor:59 - Running task 2.0 in stage 16.0 (TID 55) -2016-06-13 16:08:06 INFO TaskSetManager:59 - Finished task 2.0 in stage 15.0 (TID 51) in 85 ms on localhost (3/4) -2016-06-13 16:08:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 49) in 87 ms on localhost (4/4) -2016-06-13 16:08:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 16:08:06 INFO Executor:59 - Running task 3.0 in stage 16.0 (TID 56) -2016-06-13 16:08:06 INFO DAGScheduler:59 - ShuffleMapStage 15 (makeRDD at BlockMatrixSuite.scala:188) finished in 0.097 s -2016-06-13 16:08:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:08:06 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 16) -2016-06-13 16:08:06 INFO DAGScheduler:59 - waiting: Set(ResultStage 17) -2016-06-13 16:08:06 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:08:06 INFO DAGScheduler:59 - Missing parents for ResultStage 17: List(ShuffleMapStage 16) -2016-06-13 16:08:06 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 53). 1161 bytes result sent to driver -2016-06-13 16:08:06 INFO Executor:59 - Finished task 3.0 in stage 16.0 (TID 56). 1161 bytes result sent to driver -2016-06-13 16:08:06 INFO TaskSetManager:59 - Finished task 3.0 in stage 16.0 (TID 56) in 39 ms on localhost (1/4) -2016-06-13 16:08:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 53) in 76 ms on localhost (2/4) -2016-06-13 16:08:06 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 54). 1161 bytes result sent to driver -2016-06-13 16:08:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 54) in 89 ms on localhost (3/4) -2016-06-13 16:08:06 INFO Executor:59 - Finished task 2.0 in stage 16.0 (TID 55). 1161 bytes result sent to driver -2016-06-13 16:08:06 INFO TaskSetManager:59 - Finished task 2.0 in stage 16.0 (TID 55) in 70 ms on localhost (4/4) -2016-06-13 16:08:06 INFO DAGScheduler:59 - ShuffleMapStage 16 (makeRDD at BlockMatrixSuite.scala:189) finished in 0.136 s -2016-06-13 16:08:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:08:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 16:08:06 INFO DAGScheduler:59 - running: Set() -2016-06-13 16:08:06 INFO DAGScheduler:59 - waiting: Set(ResultStage 17) -2016-06-13 16:08:06 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:08:06 INFO DAGScheduler:59 - Missing parents for ResultStage 17: List() -2016-06-13 16:08:06 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[22] at map at BlockMatrix.scala:335), which is now runnable -2016-06-13 16:08:06 INFO MemoryStore:59 - ensureFreeSpace(2872) called with curMem=23644, maxMem=766075207 -2016-06-13 16:08:06 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 2.8 KB, free 730.6 MB) -2016-06-13 16:08:06 INFO MemoryStore:59 - ensureFreeSpace(1666) called with curMem=26516, maxMem=766075207 -2016-06-13 16:08:06 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1666.0 B, free 730.6 MB) -2016-06-13 16:08:06 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:59919 (size: 1666.0 B, free: 730.6 MB) -2016-06-13 16:08:06 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:06 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (MapPartitionsRDD[22] at map at BlockMatrix.scala:335) -2016-06-13 16:08:06 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-06-13 16:08:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 57, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 16:08:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 58, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 16:08:06 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 59, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 16:08:06 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 60, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 16:08:06 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 57) -2016-06-13 16:08:06 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 59) -2016-06-13 16:08:06 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 58) -2016-06-13 16:08:06 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 60) -2016-06-13 16:08:06 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 16:08:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:06 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 16:08:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:06 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 16:08:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 3 ms -2016-06-13 16:08:06 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 16:08:06 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 16:08:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:06 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 16:08:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:06 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 16:08:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:06 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 16:08:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:06 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 60). 1578 bytes result sent to driver -2016-06-13 16:08:06 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 60) in 29 ms on localhost (1/4) -2016-06-13 16:08:06 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 57). 1578 bytes result sent to driver -2016-06-13 16:08:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 57) in 36 ms on localhost (2/4) -2016-06-13 16:08:06 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 59). 1578 bytes result sent to driver -2016-06-13 16:08:06 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 58). 1578 bytes result sent to driver -2016-06-13 16:08:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 58) in 40 ms on localhost (3/4) -2016-06-13 16:08:06 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 59) in 38 ms on localhost (4/4) -2016-06-13 16:08:06 INFO DAGScheduler:59 - ResultStage 17 (collect at BlockMatrix.scala:289) finished in 0.042 s -2016-06-13 16:08:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 16:08:06 INFO DAGScheduler:59 - Job 10 finished: collect at BlockMatrix.scala:289, took 0.215348 s -2016-06-13 16:08:06 INFO SparkContext:59 - Starting job: reduce at BlockMatrix.scala:189 -2016-06-13 16:08:06 INFO DAGScheduler:59 - Got job 11 (reduce at BlockMatrix.scala:189) with 2 output partitions -2016-06-13 16:08:06 INFO DAGScheduler:59 - Final stage: ResultStage 18(reduce at BlockMatrix.scala:189) -2016-06-13 16:08:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:06 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[25] at map at BlockMatrix.scala:186), which has no missing parents -2016-06-13 16:08:06 INFO MemoryStore:59 - ensureFreeSpace(2552) called with curMem=28182, maxMem=766075207 -2016-06-13 16:08:06 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 2.5 KB, free 730.6 MB) -2016-06-13 16:08:06 INFO MemoryStore:59 - ensureFreeSpace(1458) called with curMem=30734, maxMem=766075207 -2016-06-13 16:08:06 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1458.0 B, free 730.6 MB) -2016-06-13 16:08:06 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:59919 (size: 1458.0 B, free: 730.6 MB) -2016-06-13 16:08:06 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[25] at map at BlockMatrix.scala:186) -2016-06-13 16:08:06 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-06-13 16:08:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 61, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 62, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:06 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 61) -2016-06-13 16:08:06 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 62) -2016-06-13 16:08:06 INFO CacheManager:59 - Partition rdd_24_0 not found, computing it -2016-06-13 16:08:06 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=32192, maxMem=766075207 -2016-06-13 16:08:06 INFO MemoryStore:59 - Block rdd_24_0 stored as values in memory (estimated size 112.0 B, free 730.6 MB) -2016-06-13 16:08:06 INFO CacheManager:59 - Partition rdd_24_1 not found, computing it -2016-06-13 16:08:06 INFO BlockManagerInfo:59 - Added rdd_24_0 in memory on localhost:59919 (size: 112.0 B, free: 730.6 MB) -2016-06-13 16:08:06 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=32304, maxMem=766075207 -2016-06-13 16:08:06 INFO MemoryStore:59 - Block rdd_24_1 stored as values in memory (estimated size 112.0 B, free 730.6 MB) -2016-06-13 16:08:06 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 61). 1694 bytes result sent to driver -2016-06-13 16:08:06 INFO BlockManagerInfo:59 - Added rdd_24_1 in memory on localhost:59919 (size: 112.0 B, free: 730.6 MB) -2016-06-13 16:08:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 61) in 19 ms on localhost (1/2) -2016-06-13 16:08:06 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 62). 1694 bytes result sent to driver -2016-06-13 16:08:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 62) in 21 ms on localhost (2/2) -2016-06-13 16:08:06 INFO DAGScheduler:59 - ResultStage 18 (reduce at BlockMatrix.scala:189) finished in 0.022 s -2016-06-13 16:08:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 16:08:06 INFO DAGScheduler:59 - Job 11 finished: reduce at BlockMatrix.scala:189, took 0.032998 s -2016-06-13 16:08:06 INFO SparkContext:59 - Starting job: collect at BlockMatrix.scala:289 -2016-06-13 16:08:06 INFO DAGScheduler:59 - Registering RDD 27 (flatMap at BlockMatrix.scala:377) -2016-06-13 16:08:06 INFO DAGScheduler:59 - Registering RDD 26 (flatMap at BlockMatrix.scala:373) -2016-06-13 16:08:06 INFO DAGScheduler:59 - Registering RDD 30 (flatMap at BlockMatrix.scala:381) -2016-06-13 16:08:06 INFO DAGScheduler:59 - Got job 12 (collect at BlockMatrix.scala:289) with 4 output partitions -2016-06-13 16:08:06 INFO DAGScheduler:59 - Final stage: ResultStage 22(collect at BlockMatrix.scala:289) -2016-06-13 16:08:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 21) -2016-06-13 16:08:06 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 21) -2016-06-13 16:08:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 19 (MapPartitionsRDD[27] at flatMap at BlockMatrix.scala:377), which has no missing parents -2016-06-13 16:08:06 INFO MemoryStore:59 - ensureFreeSpace(3032) called with curMem=32416, maxMem=766075207 -2016-06-13 16:08:06 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.0 KB, free 730.6 MB) -2016-06-13 16:08:06 INFO MemoryStore:59 - ensureFreeSpace(1769) called with curMem=35448, maxMem=766075207 -2016-06-13 16:08:06 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1769.0 B, free 730.6 MB) -2016-06-13 16:08:06 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:59919 (size: 1769.0 B, free: 730.6 MB) -2016-06-13 16:08:06 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 19 (MapPartitionsRDD[27] at flatMap at BlockMatrix.scala:377) -2016-06-13 16:08:06 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-06-13 16:08:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 20 (MapPartitionsRDD[26] at flatMap at BlockMatrix.scala:373), which has no missing parents -2016-06-13 16:08:06 INFO MemoryStore:59 - ensureFreeSpace(3032) called with curMem=37217, maxMem=766075207 -2016-06-13 16:08:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 63, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 16:08:06 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 3.0 KB, free 730.5 MB) -2016-06-13 16:08:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 64, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 16:08:06 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 63) -2016-06-13 16:08:06 INFO MemoryStore:59 - ensureFreeSpace(1776) called with curMem=40249, maxMem=766075207 -2016-06-13 16:08:06 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 1776.0 B, free 730.5 MB) -2016-06-13 16:08:06 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:59919 (size: 1776.0 B, free: 730.6 MB) -2016-06-13 16:08:06 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 64) -2016-06-13 16:08:06 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:06 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 20 (MapPartitionsRDD[26] at flatMap at BlockMatrix.scala:373) -2016-06-13 16:08:06 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 3 tasks -2016-06-13 16:08:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 65, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 16:08:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 66, localhost, PROCESS_LOCAL, 2446 bytes) -2016-06-13 16:08:06 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 65) -2016-06-13 16:08:06 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 66) -2016-06-13 16:08:06 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 66). 1161 bytes result sent to driver -2016-06-13 16:08:06 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 67, localhost, PROCESS_LOCAL, 2430 bytes) -2016-06-13 16:08:07 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 67) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 66) in 58 ms on localhost (1/3) -2016-06-13 16:08:07 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 67). 1161 bytes result sent to driver -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 67) in 62 ms on localhost (2/3) -2016-06-13 16:08:07 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 63). 1161 bytes result sent to driver -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 63) in 121 ms on localhost (1/2) -2016-06-13 16:08:07 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 65). 1161 bytes result sent to driver -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 65) in 115 ms on localhost (3/3) -2016-06-13 16:08:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 16:08:07 INFO DAGScheduler:59 - ShuffleMapStage 20 (flatMap at BlockMatrix.scala:373) finished in 0.118 s -2016-06-13 16:08:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:08:07 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 19) -2016-06-13 16:08:07 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 21, ResultStage 22) -2016-06-13 16:08:07 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:08:07 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 21: List(ShuffleMapStage 19) -2016-06-13 16:08:07 INFO DAGScheduler:59 - Missing parents for ResultStage 22: List(ShuffleMapStage 21) -2016-06-13 16:08:07 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 64). 1161 bytes result sent to driver -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 64) in 137 ms on localhost (2/2) -2016-06-13 16:08:07 INFO DAGScheduler:59 - ShuffleMapStage 19 (flatMap at BlockMatrix.scala:377) finished in 0.143 s -2016-06-13 16:08:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 16:08:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:08:07 INFO DAGScheduler:59 - running: Set() -2016-06-13 16:08:07 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 21, ResultStage 22) -2016-06-13 16:08:07 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:08:07 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 21: List() -2016-06-13 16:08:07 INFO DAGScheduler:59 - Missing parents for ResultStage 22: List(ShuffleMapStage 21) -2016-06-13 16:08:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 21 (MapPartitionsRDD[30] at flatMap at BlockMatrix.scala:381), which is now runnable -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(2952) called with curMem=42025, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 2.9 KB, free 730.5 MB) -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(1717) called with curMem=44977, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1717.0 B, free 730.5 MB) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:59919 (size: 1717.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:07 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 21 (MapPartitionsRDD[30] at flatMap at BlockMatrix.scala:381) -2016-06-13 16:08:07 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 68, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 69, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 70, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 71, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:08:07 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 71) -2016-06-13 16:08:07 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 70) -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:07 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 68) -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:07 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 69) -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:07 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 16:08:07 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 16:08:07 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 69). 1377 bytes result sent to driver -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 69) in 53 ms on localhost (1/4) -2016-06-13 16:08:07 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 68). 1377 bytes result sent to driver -2016-06-13 16:08:07 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 71). 1377 bytes result sent to driver -2016-06-13 16:08:07 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 70). 1377 bytes result sent to driver -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 68) in 75 ms on localhost (2/4) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 71) in 73 ms on localhost (3/4) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 70) in 75 ms on localhost (4/4) -2016-06-13 16:08:07 INFO DAGScheduler:59 - ShuffleMapStage 21 (flatMap at BlockMatrix.scala:381) finished in 0.076 s -2016-06-13 16:08:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 16:08:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:08:07 INFO DAGScheduler:59 - running: Set() -2016-06-13 16:08:07 INFO DAGScheduler:59 - waiting: Set(ResultStage 22) -2016-06-13 16:08:07 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:08:07 INFO DAGScheduler:59 - Missing parents for ResultStage 22: List() -2016-06-13 16:08:07 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[32] at mapValues at BlockMatrix.scala:397), which is now runnable -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=46694, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 2.8 KB, free 730.5 MB) -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(1676) called with curMem=49590, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 1676.0 B, free 730.5 MB) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:59919 (size: 1676.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:07 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[32] at mapValues at BlockMatrix.scala:397) -2016-06-13 16:08:07 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 72, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 73, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 74, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 75, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:07 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 74) -2016-06-13 16:08:07 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 73) -2016-06-13 16:08:07 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 72) -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 4 blocks -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:07 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 75) -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 4 blocks -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 4 blocks -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 4 blocks -2016-06-13 16:08:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:07 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 75). 1459 bytes result sent to driver -2016-06-13 16:08:07 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 73). 1161 bytes result sent to driver -2016-06-13 16:08:07 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 72). 1554 bytes result sent to driver -2016-06-13 16:08:07 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 74). 1554 bytes result sent to driver -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 75) in 19 ms on localhost (1/4) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 73) in 22 ms on localhost (2/4) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 72) in 24 ms on localhost (3/4) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 74) in 24 ms on localhost (4/4) -2016-06-13 16:08:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 16:08:07 INFO DAGScheduler:59 - ResultStage 22 (collect at BlockMatrix.scala:289) finished in 0.026 s -2016-06-13 16:08:07 INFO DAGScheduler:59 - Job 12 finished: collect at BlockMatrix.scala:289, took 0.278649 s -2016-06-13 16:08:07 INFO SparkContext:59 - Starting job: reduce at BlockMatrix.scala:189 -2016-06-13 16:08:07 INFO DAGScheduler:59 - Got job 13 (reduce at BlockMatrix.scala:189) with 3 output partitions -2016-06-13 16:08:07 INFO DAGScheduler:59 - Final stage: ResultStage 23(reduce at BlockMatrix.scala:189) -2016-06-13 16:08:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:07 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[35] at map at BlockMatrix.scala:186), which has no missing parents -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(2552) called with curMem=49590, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:59919 in memory (size: 1676.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(1461) called with curMem=49246, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 1461.0 B, free 730.5 MB) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:59919 (size: 1461.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:07 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 23 (MapPartitionsRDD[35] at map at BlockMatrix.scala:186) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:59919 in memory (size: 1717.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 3 tasks -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 76, localhost, PROCESS_LOCAL, 2083 bytes) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 77, localhost, PROCESS_LOCAL, 2083 bytes) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 78, localhost, PROCESS_LOCAL, 2474 bytes) -2016-06-13 16:08:07 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 77) -2016-06-13 16:08:07 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 76) -2016-06-13 16:08:07 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 78) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:59919 in memory (size: 1776.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_34_0 not found, computing it -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=41230, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_34_0 stored as values in memory (estimated size 16.0 B, free 730.5 MB) -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_34_2 not found, computing it -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_34_1 not found, computing it -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=36445, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_34_1 stored as values in memory (estimated size 16.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_34_0 in memory on localhost:59919 (size: 16.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=36461, maxMem=766075207 -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_34_1 in memory on localhost:59919 (size: 16.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_34_2 stored as values in memory (estimated size 112.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 76). 1544 bytes result sent to driver -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:59919 in memory (size: 1769.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_34_2 in memory on localhost:59919 (size: 112.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 77). 1544 bytes result sent to driver -2016-06-13 16:08:07 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 78). 1694 bytes result sent to driver -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 76) in 20 ms on localhost (1/3) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 77) in 19 ms on localhost (2/3) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 78) in 18 ms on localhost (3/3) -2016-06-13 16:08:07 INFO DAGScheduler:59 - ResultStage 23 (reduce at BlockMatrix.scala:189) finished in 0.022 s -2016-06-13 16:08:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 16:08:07 INFO DAGScheduler:59 - Job 13 finished: reduce at BlockMatrix.scala:189, took 0.050619 s -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:59919 in memory (size: 1458.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:59919 in memory (size: 1666.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:59919 in memory (size: 1077.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:59919 in memory (size: 1077.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned shuffle 6 -2016-06-13 16:08:07 INFO SparkContext:59 - Starting job: reduce at BlockMatrix.scala:189 -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned shuffle 5 -2016-06-13 16:08:07 INFO DAGScheduler:59 - Got job 14 (reduce at BlockMatrix.scala:189) with 10 output partitions -2016-06-13 16:08:07 INFO DAGScheduler:59 - Final stage: ResultStage 24(reduce at BlockMatrix.scala:189) -2016-06-13 16:08:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:07 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[38] at map at BlockMatrix.scala:186), which has no missing parents -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:59919 in memory (size: 1662.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(2552) called with curMem=20977, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 2.5 KB, free 730.6 MB) -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(1461) called with curMem=23529, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 1461.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:59919 (size: 1461.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:07 INFO DAGScheduler:59 - Submitting 10 missing tasks from ResultStage 24 (MapPartitionsRDD[38] at map at BlockMatrix.scala:186) -2016-06-13 16:08:07 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 10 tasks -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 79, localhost, PROCESS_LOCAL, 2777 bytes) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 80, localhost, PROCESS_LOCAL, 2777 bytes) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:59919 in memory (size: 1075.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 81, localhost, PROCESS_LOCAL, 2777 bytes) -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 3.0 in stage 24.0 (TID 82, localhost, PROCESS_LOCAL, 2777 bytes) -2016-06-13 16:08:07 INFO Executor:59 - Running task 3.0 in stage 24.0 (TID 82) -2016-06-13 16:08:07 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 79) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:59919 in memory (size: 1077.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 81) -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_37_3 not found, computing it -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_37_0 not found, computing it -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=16734, maxMem=766075207 -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_37_2 not found, computing it -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned shuffle 4 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_37_3 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=16934, maxMem=766075207 -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_37_3 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_37_2 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_37_2 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO Executor:59 - Finished task 3.0 in stage 24.0 (TID 82). 1694 bytes result sent to driver -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=17134, maxMem=766075207 -2016-06-13 16:08:07 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 81). 1694 bytes result sent to driver -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_37_0 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 4.0 in stage 24.0 (TID 83, localhost, PROCESS_LOCAL, 2777 bytes) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 5.0 in stage 24.0 (TID 84, localhost, PROCESS_LOCAL, 2777 bytes) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_37_0 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 3.0 in stage 24.0 (TID 82) in 24 ms on localhost (1/10) -2016-06-13 16:08:07 INFO Executor:59 - Running task 4.0 in stage 24.0 (TID 83) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 81) in 27 ms on localhost (2/10) -2016-06-13 16:08:07 INFO Executor:59 - Running task 5.0 in stage 24.0 (TID 84) -2016-06-13 16:08:07 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 79). 1694 bytes result sent to driver -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 6.0 in stage 24.0 (TID 85, localhost, PROCESS_LOCAL, 2777 bytes) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:59919 in memory (size: 1660.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_37_4 not found, computing it -2016-06-13 16:08:07 INFO Executor:59 - Running task 6.0 in stage 24.0 (TID 85) -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_37_5 not found, computing it -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=12802, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_37_5 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=13002, maxMem=766075207 -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_37_5 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_37_4 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_37_6 not found, computing it -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=13202, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_37_6 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_37_6 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO Executor:59 - Finished task 6.0 in stage 24.0 (TID 85). 1694 bytes result sent to driver -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 7.0 in stage 24.0 (TID 86, localhost, PROCESS_LOCAL, 2777 bytes) -2016-06-13 16:08:07 INFO Executor:59 - Running task 7.0 in stage 24.0 (TID 86) -2016-06-13 16:08:07 INFO Executor:59 - Finished task 5.0 in stage 24.0 (TID 84). 1694 bytes result sent to driver -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_37_4 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 8.0 in stage 24.0 (TID 87, localhost, PROCESS_LOCAL, 2777 bytes) -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_37_7 not found, computing it -2016-06-13 16:08:07 INFO Executor:59 - Running task 8.0 in stage 24.0 (TID 87) -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=13402, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_37_7 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_37_7 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO Executor:59 - Finished task 4.0 in stage 24.0 (TID 83). 1694 bytes result sent to driver -2016-06-13 16:08:07 INFO Executor:59 - Finished task 7.0 in stage 24.0 (TID 86). 1694 bytes result sent to driver -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 9.0 in stage 24.0 (TID 88, localhost, PROCESS_LOCAL, 2777 bytes) -2016-06-13 16:08:07 INFO Executor:59 - Running task 9.0 in stage 24.0 (TID 88) -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_37_8 not found, computing it -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=13602, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_37_8 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_37_8 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_37_9 not found, computing it -2016-06-13 16:08:07 INFO Executor:59 - Finished task 8.0 in stage 24.0 (TID 87). 1694 bytes result sent to driver -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=13802, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_37_9 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_37_9 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO Executor:59 - Finished task 9.0 in stage 24.0 (TID 88). 1694 bytes result sent to driver -2016-06-13 16:08:07 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 80) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 6.0 in stage 24.0 (TID 85) in 67 ms on localhost (3/10) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 5.0 in stage 24.0 (TID 84) in 85 ms on localhost (4/10) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 4.0 in stage 24.0 (TID 83) in 87 ms on localhost (5/10) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 8.0 in stage 24.0 (TID 87) in 42 ms on localhost (6/10) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 9.0 in stage 24.0 (TID 88) in 37 ms on localhost (7/10) -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_37_1 not found, computing it -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=14002, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_37_1 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 79) in 120 ms on localhost (8/10) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 7.0 in stage 24.0 (TID 86) in 54 ms on localhost (9/10) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_37_1 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 80). 1694 bytes result sent to driver -2016-06-13 16:08:07 INFO DAGScheduler:59 - ResultStage 24 (reduce at BlockMatrix.scala:189) finished in 0.128 s -2016-06-13 16:08:07 INFO DAGScheduler:59 - Job 14 finished: reduce at BlockMatrix.scala:189, took 0.146595 s -2016-06-13 16:08:07 INFO SparkContext:59 - Starting job: reduce at BlockMatrix.scala:189 -2016-06-13 16:08:07 INFO DAGScheduler:59 - Got job 15 (reduce at BlockMatrix.scala:189) with 8 output partitions -2016-06-13 16:08:07 INFO DAGScheduler:59 - Final stage: ResultStage 25(reduce at BlockMatrix.scala:189) -2016-06-13 16:08:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 80) in 125 ms on localhost (10/10) -2016-06-13 16:08:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-06-13 16:08:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:07 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[41] at map at BlockMatrix.scala:186), which has no missing parents -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(2552) called with curMem=14202, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 2.5 KB, free 730.6 MB) -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(1461) called with curMem=16754, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 1461.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:59919 (size: 1461.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:07 INFO DAGScheduler:59 - Submitting 8 missing tasks from ResultStage 25 (MapPartitionsRDD[41] at map at BlockMatrix.scala:186) -2016-06-13 16:08:07 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 8 tasks -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 89, localhost, PROCESS_LOCAL, 2649 bytes) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 90, localhost, PROCESS_LOCAL, 2649 bytes) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 2.0 in stage 25.0 (TID 91, localhost, PROCESS_LOCAL, 2649 bytes) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 3.0 in stage 25.0 (TID 92, localhost, PROCESS_LOCAL, 2649 bytes) -2016-06-13 16:08:07 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 89) -2016-06-13 16:08:07 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 90) -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_40_1 not found, computing it -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=18215, maxMem=766075207 -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_40_0 not found, computing it -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_40_1 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=18415, maxMem=766075207 -2016-06-13 16:08:07 INFO Executor:59 - Running task 3.0 in stage 25.0 (TID 92) -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_40_0 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_40_1 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_40_0 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_40_3 not found, computing it -2016-06-13 16:08:07 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 90). 1694 bytes result sent to driver -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 4.0 in stage 25.0 (TID 93, localhost, PROCESS_LOCAL, 2649 bytes) -2016-06-13 16:08:07 INFO Executor:59 - Running task 4.0 in stage 25.0 (TID 93) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:59919 in memory (size: 1080.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_40_4 not found, computing it -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=18615, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_40_3 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 16:08:07 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 89). 1694 bytes result sent to driver -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=16119, maxMem=766075207 -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_40_3 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 5.0 in stage 25.0 (TID 94, localhost, PROCESS_LOCAL, 2649 bytes) -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-06-13 16:08:07 INFO Executor:59 - Running task 5.0 in stage 25.0 (TID 94) -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_40_5 not found, computing it -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_40_4 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_40_4 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=16319, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_40_5 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO Executor:59 - Finished task 4.0 in stage 25.0 (TID 93). 1694 bytes result sent to driver -2016-06-13 16:08:07 INFO Executor:59 - Finished task 3.0 in stage 25.0 (TID 92). 1694 bytes result sent to driver -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 6.0 in stage 25.0 (TID 95, localhost, PROCESS_LOCAL, 2649 bytes) -2016-06-13 16:08:07 INFO Executor:59 - Running task 6.0 in stage 25.0 (TID 95) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 7.0 in stage 25.0 (TID 96, localhost, PROCESS_LOCAL, 2649 bytes) -2016-06-13 16:08:07 INFO Executor:59 - Running task 7.0 in stage 25.0 (TID 96) -2016-06-13 16:08:07 INFO Executor:59 - Running task 2.0 in stage 25.0 (TID 91) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 90) in 66 ms on localhost (1/8) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 89) in 68 ms on localhost (2/8) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 4.0 in stage 25.0 (TID 93) in 50 ms on localhost (3/8) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 3.0 in stage 25.0 (TID 92) in 67 ms on localhost (4/8) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_40_5 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO BlockManager:59 - Removing RDD 9 -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_40_6 not found, computing it -2016-06-13 16:08:07 INFO Executor:59 - Finished task 5.0 in stage 25.0 (TID 94). 1694 bytes result sent to driver -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_40_2 not found, computing it -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=16519, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_40_6 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=16719, maxMem=766075207 -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 5.0 in stage 25.0 (TID 94) in 85 ms on localhost (5/8) -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_40_2 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_40_6 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_40_2 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO Executor:59 - Finished task 6.0 in stage 25.0 (TID 95). 1694 bytes result sent to driver -2016-06-13 16:08:07 INFO Executor:59 - Finished task 2.0 in stage 25.0 (TID 91). 1694 bytes result sent to driver -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 6.0 in stage 25.0 (TID 95) in 58 ms on localhost (6/8) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 2.0 in stage 25.0 (TID 91) in 115 ms on localhost (7/8) -2016-06-13 16:08:07 INFO CacheManager:59 - Partition rdd_40_7 not found, computing it -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=16319, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block rdd_40_7 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added rdd_40_7 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO Executor:59 - Finished task 7.0 in stage 25.0 (TID 96). 1694 bytes result sent to driver -2016-06-13 16:08:07 INFO TaskSetManager:59 - Finished task 7.0 in stage 25.0 (TID 96) in 87 ms on localhost (8/8) -2016-06-13 16:08:07 INFO DAGScheduler:59 - ResultStage 25 (reduce at BlockMatrix.scala:189) finished in 0.147 s -2016-06-13 16:08:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-06-13 16:08:07 INFO DAGScheduler:59 - Job 15 finished: reduce at BlockMatrix.scala:189, took 0.214678 s -2016-06-13 16:08:07 INFO SparkContext:59 - Starting job: collect at BlockMatrix.scala:289 -2016-06-13 16:08:07 INFO DAGScheduler:59 - Registering RDD 43 (flatMap at BlockMatrix.scala:377) -2016-06-13 16:08:07 INFO DAGScheduler:59 - Registering RDD 42 (flatMap at BlockMatrix.scala:373) -2016-06-13 16:08:07 INFO DAGScheduler:59 - Registering RDD 46 (flatMap at BlockMatrix.scala:381) -2016-06-13 16:08:07 INFO DAGScheduler:59 - Got job 16 (collect at BlockMatrix.scala:289) with 12 output partitions -2016-06-13 16:08:07 INFO DAGScheduler:59 - Final stage: ResultStage 29(collect at BlockMatrix.scala:289) -2016-06-13 16:08:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 28) -2016-06-13 16:08:07 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 28) -2016-06-13 16:08:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (MapPartitionsRDD[43] at flatMap at BlockMatrix.scala:377), which has no missing parents -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(3032) called with curMem=16519, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 3.0 KB, free 730.6 MB) -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(1778) called with curMem=19551, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 1778.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO ContextCleaner:59 - Cleaned RDD 9 -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:59919 (size: 1778.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:07 INFO DAGScheduler:59 - Submitting 8 missing tasks from ShuffleMapStage 26 (MapPartitionsRDD[43] at flatMap at BlockMatrix.scala:377) -2016-06-13 16:08:07 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 8 tasks -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 97, localhost, PROCESS_LOCAL, 2638 bytes) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 98, localhost, PROCESS_LOCAL, 2638 bytes) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 99, localhost, PROCESS_LOCAL, 2638 bytes) -2016-06-13 16:08:07 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 100, localhost, PROCESS_LOCAL, 2638 bytes) -2016-06-13 16:08:07 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 97) -2016-06-13 16:08:07 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 100) -2016-06-13 16:08:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 27 (MapPartitionsRDD[42] at flatMap at BlockMatrix.scala:373), which has no missing parents -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(3032) called with curMem=21329, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 3.0 KB, free 730.6 MB) -2016-06-13 16:08:07 INFO MemoryStore:59 - ensureFreeSpace(1779) called with curMem=24361, maxMem=766075207 -2016-06-13 16:08:07 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 1779.0 B, free 730.6 MB) -2016-06-13 16:08:07 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:59919 (size: 1779.0 B, free: 730.6 MB) -2016-06-13 16:08:07 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 98) -2016-06-13 16:08:07 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:07 INFO DAGScheduler:59 - Submitting 10 missing tasks from ShuffleMapStage 27 (MapPartitionsRDD[42] at flatMap at BlockMatrix.scala:373) -2016-06-13 16:08:07 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 10 tasks -2016-06-13 16:08:07 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 99) -2016-06-13 16:08:08 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 100). 1169 bytes result sent to driver -2016-06-13 16:08:08 INFO TaskSetManager:59 - Starting task 4.0 in stage 26.0 (TID 101, localhost, PROCESS_LOCAL, 2638 bytes) -2016-06-13 16:08:08 INFO Executor:59 - Running task 4.0 in stage 26.0 (TID 101) -2016-06-13 16:08:08 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 100) in 1067 ms on localhost (1/8) -2016-06-13 16:08:08 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 97). 1169 bytes result sent to driver -2016-06-13 16:08:08 INFO TaskSetManager:59 - Starting task 5.0 in stage 26.0 (TID 102, localhost, PROCESS_LOCAL, 2638 bytes) -2016-06-13 16:08:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 97) in 1233 ms on localhost (2/8) -2016-06-13 16:08:08 INFO Executor:59 - Running task 5.0 in stage 26.0 (TID 102) -2016-06-13 16:08:08 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 99). 1169 bytes result sent to driver -2016-06-13 16:08:08 INFO TaskSetManager:59 - Starting task 6.0 in stage 26.0 (TID 103, localhost, PROCESS_LOCAL, 2638 bytes) -2016-06-13 16:08:08 INFO Executor:59 - Running task 6.0 in stage 26.0 (TID 103) -2016-06-13 16:08:08 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 99) in 1209 ms on localhost (3/8) -2016-06-13 16:08:09 INFO Executor:59 - Finished task 5.0 in stage 26.0 (TID 102). 1169 bytes result sent to driver -2016-06-13 16:08:09 INFO TaskSetManager:59 - Starting task 7.0 in stage 26.0 (TID 104, localhost, PROCESS_LOCAL, 2638 bytes) -2016-06-13 16:08:09 INFO Executor:59 - Running task 7.0 in stage 26.0 (TID 104) -2016-06-13 16:08:09 INFO TaskSetManager:59 - Finished task 5.0 in stage 26.0 (TID 102) in 377 ms on localhost (4/8) -2016-06-13 16:08:09 INFO Executor:59 - Finished task 7.0 in stage 26.0 (TID 104). 1169 bytes result sent to driver -2016-06-13 16:08:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 105, localhost, PROCESS_LOCAL, 2766 bytes) -2016-06-13 16:08:09 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 105) -2016-06-13 16:08:09 INFO TaskSetManager:59 - Finished task 7.0 in stage 26.0 (TID 104) in 92 ms on localhost (5/8) -2016-06-13 16:08:09 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 98). 1169 bytes result sent to driver -2016-06-13 16:08:09 INFO Executor:59 - Finished task 6.0 in stage 26.0 (TID 103). 1169 bytes result sent to driver -2016-06-13 16:08:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 106, localhost, PROCESS_LOCAL, 2766 bytes) -2016-06-13 16:08:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 27.0 (TID 107, localhost, PROCESS_LOCAL, 2766 bytes) -2016-06-13 16:08:09 INFO Executor:59 - Running task 2.0 in stage 27.0 (TID 107) -2016-06-13 16:08:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 98) in 2149 ms on localhost (6/8) -2016-06-13 16:08:09 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 106) -2016-06-13 16:08:09 INFO TaskSetManager:59 - Finished task 6.0 in stage 26.0 (TID 103) in 915 ms on localhost (7/8) -2016-06-13 16:08:09 INFO Executor:59 - Finished task 4.0 in stage 26.0 (TID 101). 1169 bytes result sent to driver -2016-06-13 16:08:09 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 105). 1169 bytes result sent to driver -2016-06-13 16:08:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 27.0 (TID 108, localhost, PROCESS_LOCAL, 2766 bytes) -2016-06-13 16:08:09 INFO TaskSetManager:59 - Starting task 4.0 in stage 27.0 (TID 109, localhost, PROCESS_LOCAL, 2766 bytes) -2016-06-13 16:08:09 INFO TaskSetManager:59 - Finished task 4.0 in stage 26.0 (TID 101) in 1116 ms on localhost (8/8) -2016-06-13 16:08:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-06-13 16:08:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 105) in 522 ms on localhost (1/10) -2016-06-13 16:08:09 INFO DAGScheduler:59 - ShuffleMapStage 26 (flatMap at BlockMatrix.scala:377) finished in 2.176 s -2016-06-13 16:08:09 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:08:09 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 27) -2016-06-13 16:08:09 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 28, ResultStage 29) -2016-06-13 16:08:09 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:08:09 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 28: List(ShuffleMapStage 27) -2016-06-13 16:08:09 INFO DAGScheduler:59 - Missing parents for ResultStage 29: List(ShuffleMapStage 28) -2016-06-13 16:08:09 INFO Executor:59 - Running task 3.0 in stage 27.0 (TID 108) -2016-06-13 16:08:09 INFO Executor:59 - Finished task 2.0 in stage 27.0 (TID 107). 1169 bytes result sent to driver -2016-06-13 16:08:10 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 106). 1169 bytes result sent to driver -2016-06-13 16:08:10 INFO Executor:59 - Running task 4.0 in stage 27.0 (TID 109) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 5.0 in stage 27.0 (TID 110, localhost, PROCESS_LOCAL, 2766 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Running task 5.0 in stage 27.0 (TID 110) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 6.0 in stage 27.0 (TID 111, localhost, PROCESS_LOCAL, 2766 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Running task 6.0 in stage 27.0 (TID 111) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 106) in 115 ms on localhost (2/10) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 27.0 (TID 107) in 118 ms on localhost (3/10) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 3.0 in stage 27.0 (TID 108). 1169 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 7.0 in stage 27.0 (TID 112, localhost, PROCESS_LOCAL, 2766 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Running task 7.0 in stage 27.0 (TID 112) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 27.0 (TID 108) in 311 ms on localhost (4/10) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 6.0 in stage 27.0 (TID 111). 1169 bytes result sent to driver -2016-06-13 16:08:10 INFO Executor:59 - Finished task 5.0 in stage 27.0 (TID 110). 1169 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 8.0 in stage 27.0 (TID 113, localhost, PROCESS_LOCAL, 2766 bytes) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 9.0 in stage 27.0 (TID 114, localhost, PROCESS_LOCAL, 2766 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Running task 9.0 in stage 27.0 (TID 114) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 6.0 in stage 27.0 (TID 111) in 257 ms on localhost (5/10) -2016-06-13 16:08:10 INFO Executor:59 - Running task 8.0 in stage 27.0 (TID 113) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 5.0 in stage 27.0 (TID 110) in 259 ms on localhost (6/10) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 4.0 in stage 27.0 (TID 109). 1169 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 4.0 in stage 27.0 (TID 109) in 336 ms on localhost (7/10) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 9.0 in stage 27.0 (TID 114). 1169 bytes result sent to driver -2016-06-13 16:08:10 INFO Executor:59 - Finished task 8.0 in stage 27.0 (TID 113). 1169 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 9.0 in stage 27.0 (TID 114) in 208 ms on localhost (8/10) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 8.0 in stage 27.0 (TID 113) in 210 ms on localhost (9/10) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 7.0 in stage 27.0 (TID 112). 1169 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 7.0 in stage 27.0 (TID 112) in 247 ms on localhost (10/10) -2016-06-13 16:08:10 INFO DAGScheduler:59 - ShuffleMapStage 27 (flatMap at BlockMatrix.scala:373) finished in 2.677 s -2016-06-13 16:08:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:08:10 INFO DAGScheduler:59 - running: Set() -2016-06-13 16:08:10 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 28, ResultStage 29) -2016-06-13 16:08:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-06-13 16:08:10 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:08:10 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 28: List() -2016-06-13 16:08:10 INFO DAGScheduler:59 - Missing parents for ResultStage 29: List(ShuffleMapStage 28) -2016-06-13 16:08:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 28 (MapPartitionsRDD[46] at flatMap at BlockMatrix.scala:381), which is now runnable -2016-06-13 16:08:10 INFO MemoryStore:59 - ensureFreeSpace(2952) called with curMem=26140, maxMem=766075207 -2016-06-13 16:08:10 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-06-13 16:08:10 INFO MemoryStore:59 - ensureFreeSpace(1715) called with curMem=29092, maxMem=766075207 -2016-06-13 16:08:10 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1715.0 B, free 730.6 MB) -2016-06-13 16:08:10 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:59919 (size: 1715.0 B, free: 730.6 MB) -2016-06-13 16:08:10 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:10 INFO DAGScheduler:59 - Submitting 12 missing tasks from ShuffleMapStage 28 (MapPartitionsRDD[46] at flatMap at BlockMatrix.scala:381) -2016-06-13 16:08:10 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 12 tasks -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 115, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 116, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 28.0 (TID 117, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 28.0 (TID 118, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 116) -2016-06-13 16:08:10 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 115) -2016-06-13 16:08:10 INFO Executor:59 - Running task 3.0 in stage 28.0 (TID 118) -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 10 non-empty blocks out of 10 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 10 non-empty blocks out of 10 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 10 non-empty blocks out of 10 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 8 non-empty blocks out of 8 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:10 INFO Executor:59 - Running task 2.0 in stage 28.0 (TID 117) -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 10 non-empty blocks out of 10 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 8 non-empty blocks out of 8 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 8 non-empty blocks out of 8 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 8 non-empty blocks out of 8 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 115). 1385 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 4.0 in stage 28.0 (TID 119, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 115) in 70 ms on localhost (1/12) -2016-06-13 16:08:10 INFO Executor:59 - Running task 4.0 in stage 28.0 (TID 119) -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 10 non-empty blocks out of 10 blocks -2016-06-13 16:08:10 INFO Executor:59 - Finished task 2.0 in stage 28.0 (TID 117). 1385 bytes result sent to driver -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 5.0 in stage 28.0 (TID 120, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Running task 5.0 in stage 28.0 (TID 120) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 28.0 (TID 117) in 77 ms on localhost (2/12) -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 10 non-empty blocks out of 10 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO Executor:59 - Finished task 3.0 in stage 28.0 (TID 118). 1385 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 6.0 in stage 28.0 (TID 121, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 28.0 (TID 118) in 89 ms on localhost (3/12) -2016-06-13 16:08:10 INFO Executor:59 - Running task 6.0 in stage 28.0 (TID 121) -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 10 non-empty blocks out of 10 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 8 non-empty blocks out of 8 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 8 non-empty blocks out of 8 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 8 non-empty blocks out of 8 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-06-13 16:08:10 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 116). 1385 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 7.0 in stage 28.0 (TID 122, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 116) in 151 ms on localhost (4/12) -2016-06-13 16:08:10 INFO Executor:59 - Running task 7.0 in stage 28.0 (TID 122) -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 10 non-empty blocks out of 10 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:10 INFO Executor:59 - Finished task 5.0 in stage 28.0 (TID 120). 1385 bytes result sent to driver -2016-06-13 16:08:10 INFO Executor:59 - Finished task 6.0 in stage 28.0 (TID 121). 1385 bytes result sent to driver -2016-06-13 16:08:10 INFO Executor:59 - Finished task 4.0 in stage 28.0 (TID 119). 1385 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 8.0 in stage 28.0 (TID 123, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Running task 8.0 in stage 28.0 (TID 123) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 9.0 in stage 28.0 (TID 124, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 10.0 in stage 28.0 (TID 125, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Running task 9.0 in stage 28.0 (TID 124) -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 10 non-empty blocks out of 10 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 10 non-empty blocks out of 10 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 8 non-empty blocks out of 8 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 8 non-empty blocks out of 8 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 5.0 in stage 28.0 (TID 120) in 127 ms on localhost (5/12) -2016-06-13 16:08:10 INFO Executor:59 - Running task 10.0 in stage 28.0 (TID 125) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 6.0 in stage 28.0 (TID 121) in 115 ms on localhost (6/12) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 4.0 in stage 28.0 (TID 119) in 137 ms on localhost (7/12) -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 10 non-empty blocks out of 10 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 8 non-empty blocks out of 8 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO Executor:59 - Finished task 7.0 in stage 28.0 (TID 122). 1385 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 11.0 in stage 28.0 (TID 126, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Running task 11.0 in stage 28.0 (TID 126) -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 10 non-empty blocks out of 10 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO Executor:59 - Finished task 8.0 in stage 28.0 (TID 123). 1385 bytes result sent to driver -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 8 non-empty blocks out of 8 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO Executor:59 - Finished task 9.0 in stage 28.0 (TID 124). 1385 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 7.0 in stage 28.0 (TID 122) in 92 ms on localhost (8/12) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 8.0 in stage 28.0 (TID 123) in 67 ms on localhost (9/12) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 9.0 in stage 28.0 (TID 124) in 68 ms on localhost (10/12) -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 8 non-empty blocks out of 8 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:10 INFO Executor:59 - Finished task 11.0 in stage 28.0 (TID 126). 1385 bytes result sent to driver -2016-06-13 16:08:10 INFO Executor:59 - Finished task 10.0 in stage 28.0 (TID 125). 1385 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 11.0 in stage 28.0 (TID 126) in 52 ms on localhost (11/12) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 10.0 in stage 28.0 (TID 125) in 101 ms on localhost (12/12) -2016-06-13 16:08:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-06-13 16:08:10 INFO DAGScheduler:59 - ShuffleMapStage 28 (flatMap at BlockMatrix.scala:381) finished in 0.282 s -2016-06-13 16:08:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:08:10 INFO DAGScheduler:59 - running: Set() -2016-06-13 16:08:10 INFO DAGScheduler:59 - waiting: Set(ResultStage 29) -2016-06-13 16:08:10 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:08:10 INFO DAGScheduler:59 - Missing parents for ResultStage 29: List() -2016-06-13 16:08:10 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[48] at mapValues at BlockMatrix.scala:397), which is now runnable -2016-06-13 16:08:10 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=30807, maxMem=766075207 -2016-06-13 16:08:10 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 2.8 KB, free 730.6 MB) -2016-06-13 16:08:10 INFO MemoryStore:59 - ensureFreeSpace(1687) called with curMem=33703, maxMem=766075207 -2016-06-13 16:08:10 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1687.0 B, free 730.6 MB) -2016-06-13 16:08:10 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:59919 (size: 1687.0 B, free: 730.6 MB) -2016-06-13 16:08:10 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:10 INFO DAGScheduler:59 - Submitting 12 missing tasks from ResultStage 29 (MapPartitionsRDD[48] at mapValues at BlockMatrix.scala:397) -2016-06-13 16:08:10 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 12 tasks -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 127, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 128, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 29.0 (TID 129, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 29.0 (TID 130, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Running task 3.0 in stage 29.0 (TID 130) -2016-06-13 16:08:10 INFO Executor:59 - Running task 2.0 in stage 29.0 (TID 129) -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 12 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 12 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO Executor:59 - Finished task 2.0 in stage 29.0 (TID 129). 1635 bytes result sent to driver -2016-06-13 16:08:10 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 128) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 4.0 in stage 29.0 (TID 131, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 3.0 in stage 29.0 (TID 130). 1874 bytes result sent to driver -2016-06-13 16:08:10 INFO Executor:59 - Running task 4.0 in stage 29.0 (TID 131) -2016-06-13 16:08:10 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 127) -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 12 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 12 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:10 INFO Executor:59 - Finished task 4.0 in stage 29.0 (TID 131). 1874 bytes result sent to driver -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 12 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 8 ms -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 5.0 in stage 29.0 (TID 132, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Running task 5.0 in stage 29.0 (TID 132) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 127). 1874 bytes result sent to driver -2016-06-13 16:08:10 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 128). 1874 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 6.0 in stage 29.0 (TID 133, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 7.0 in stage 29.0 (TID 134, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Running task 6.0 in stage 29.0 (TID 133) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 8.0 in stage 29.0 (TID 135, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Running task 8.0 in stage 29.0 (TID 135) -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 12 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 12 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:10 INFO Executor:59 - Finished task 5.0 in stage 29.0 (TID 132). 1635 bytes result sent to driver -2016-06-13 16:08:10 INFO Executor:59 - Finished task 8.0 in stage 29.0 (TID 135). 1635 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 9.0 in stage 29.0 (TID 136, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Running task 9.0 in stage 29.0 (TID 136) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 10.0 in stage 29.0 (TID 137, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Running task 10.0 in stage 29.0 (TID 137) -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 12 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 12 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO Executor:59 - Finished task 9.0 in stage 29.0 (TID 136). 1874 bytes result sent to driver -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 12 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO Executor:59 - Finished task 10.0 in stage 29.0 (TID 137). 1874 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 11.0 in stage 29.0 (TID 138, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 6.0 in stage 29.0 (TID 133). 1874 bytes result sent to driver -2016-06-13 16:08:10 INFO Executor:59 - Running task 11.0 in stage 29.0 (TID 138) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 29.0 (TID 130) in 48 ms on localhost (1/12) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 4.0 in stage 29.0 (TID 131) in 38 ms on localhost (2/12) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 127) in 52 ms on localhost (3/12) -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 12 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 128) in 53 ms on localhost (4/12) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 5.0 in stage 29.0 (TID 132) in 35 ms on localhost (5/12) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 8.0 in stage 29.0 (TID 135) in 29 ms on localhost (6/12) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 11.0 in stage 29.0 (TID 138). 1635 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 9.0 in stage 29.0 (TID 136) in 23 ms on localhost (7/12) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 10.0 in stage 29.0 (TID 137) in 21 ms on localhost (8/12) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 11.0 in stage 29.0 (TID 138) in 12 ms on localhost (9/12) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 29.0 (TID 129) in 58 ms on localhost (10/12) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 6.0 in stage 29.0 (TID 133) in 36 ms on localhost (11/12) -2016-06-13 16:08:10 INFO Executor:59 - Running task 7.0 in stage 29.0 (TID 134) -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 12 blocks -2016-06-13 16:08:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:10 INFO Executor:59 - Finished task 7.0 in stage 29.0 (TID 134). 1874 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 7.0 in stage 29.0 (TID 134) in 44 ms on localhost (12/12) -2016-06-13 16:08:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 16:08:10 INFO DAGScheduler:59 - ResultStage 29 (collect at BlockMatrix.scala:289) finished in 0.069 s -2016-06-13 16:08:10 INFO DAGScheduler:59 - Job 16 finished: collect at BlockMatrix.scala:289, took 3.114577 s -2016-06-13 16:08:10 INFO SparkContext:59 - Starting job: collect at BlockMatrix.scala:289 -2016-06-13 16:08:10 INFO DAGScheduler:59 - Got job 17 (collect at BlockMatrix.scala:289) with 10 output partitions -2016-06-13 16:08:10 INFO DAGScheduler:59 - Final stage: ResultStage 30(collect at BlockMatrix.scala:289) -2016-06-13 16:08:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:10 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:10 INFO DAGScheduler:59 - Submitting ResultStage 30 (ParallelCollectionRDD[36] at parallelize at BlockMatrixSuite.scala:227), which has no missing parents -2016-06-13 16:08:10 INFO MemoryStore:59 - ensureFreeSpace(1208) called with curMem=35390, maxMem=766075207 -2016-06-13 16:08:10 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 1208.0 B, free 730.6 MB) -2016-06-13 16:08:10 INFO MemoryStore:59 - ensureFreeSpace(776) called with curMem=36598, maxMem=766075207 -2016-06-13 16:08:10 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 776.0 B, free 730.6 MB) -2016-06-13 16:08:10 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:59919 (size: 776.0 B, free: 730.6 MB) -2016-06-13 16:08:10 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:10 INFO DAGScheduler:59 - Submitting 10 missing tasks from ResultStage 30 (ParallelCollectionRDD[36] at parallelize at BlockMatrixSuite.scala:227) -2016-06-13 16:08:10 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 10 tasks -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 139, localhost, PROCESS_LOCAL, 2777 bytes) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 140, localhost, PROCESS_LOCAL, 2777 bytes) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 141, localhost, PROCESS_LOCAL, 2777 bytes) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 142, localhost, PROCESS_LOCAL, 2777 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 142) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 142). 1624 bytes result sent to driver -2016-06-13 16:08:10 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 140) -2016-06-13 16:08:10 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 141) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 4.0 in stage 30.0 (TID 143, localhost, PROCESS_LOCAL, 2777 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 140). 1624 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 142) in 9 ms on localhost (1/10) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 141). 1624 bytes result sent to driver -2016-06-13 16:08:10 INFO Executor:59 - Running task 4.0 in stage 30.0 (TID 143) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 5.0 in stage 30.0 (TID 144, localhost, PROCESS_LOCAL, 2777 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 4.0 in stage 30.0 (TID 143). 1624 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 6.0 in stage 30.0 (TID 145, localhost, PROCESS_LOCAL, 2777 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Running task 6.0 in stage 30.0 (TID 145) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 7.0 in stage 30.0 (TID 146, localhost, PROCESS_LOCAL, 2777 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 6.0 in stage 30.0 (TID 145). 1624 bytes result sent to driver -2016-06-13 16:08:10 INFO Executor:59 - Running task 7.0 in stage 30.0 (TID 146) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 4.0 in stage 30.0 (TID 143) in 11 ms on localhost (2/10) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 140) in 21 ms on localhost (3/10) -2016-06-13 16:08:10 INFO Executor:59 - Running task 5.0 in stage 30.0 (TID 144) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 7.0 in stage 30.0 (TID 146). 1624 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 141) in 23 ms on localhost (4/10) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 5.0 in stage 30.0 (TID 144). 1624 bytes result sent to driver -2016-06-13 16:08:10 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 139) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 139). 1624 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 8.0 in stage 30.0 (TID 147, localhost, PROCESS_LOCAL, 2777 bytes) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 9.0 in stage 30.0 (TID 148, localhost, PROCESS_LOCAL, 2777 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Running task 8.0 in stage 30.0 (TID 147) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 6.0 in stage 30.0 (TID 145) in 20 ms on localhost (5/10) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 7.0 in stage 30.0 (TID 146) in 18 ms on localhost (6/10) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 8.0 in stage 30.0 (TID 147). 1624 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 5.0 in stage 30.0 (TID 144) in 24 ms on localhost (7/10) -2016-06-13 16:08:10 INFO Executor:59 - Running task 9.0 in stage 30.0 (TID 148) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 139) in 41 ms on localhost (8/10) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 9.0 in stage 30.0 (TID 148). 1624 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 9.0 in stage 30.0 (TID 148) in 11 ms on localhost (9/10) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 8.0 in stage 30.0 (TID 147) in 13 ms on localhost (10/10) -2016-06-13 16:08:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 16:08:10 INFO DAGScheduler:59 - ResultStage 30 (collect at BlockMatrix.scala:289) finished in 0.046 s -2016-06-13 16:08:10 INFO DAGScheduler:59 - Job 17 finished: collect at BlockMatrix.scala:289, took 0.059595 s -2016-06-13 16:08:10 INFO SparkContext:59 - Starting job: collect at BlockMatrix.scala:289 -2016-06-13 16:08:10 INFO DAGScheduler:59 - Got job 18 (collect at BlockMatrix.scala:289) with 8 output partitions -2016-06-13 16:08:10 INFO DAGScheduler:59 - Final stage: ResultStage 31(collect at BlockMatrix.scala:289) -2016-06-13 16:08:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:10 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:10 INFO DAGScheduler:59 - Submitting ResultStage 31 (ParallelCollectionRDD[39] at parallelize at BlockMatrixSuite.scala:228), which has no missing parents -2016-06-13 16:08:10 INFO MemoryStore:59 - ensureFreeSpace(1208) called with curMem=37374, maxMem=766075207 -2016-06-13 16:08:10 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 1208.0 B, free 730.5 MB) -2016-06-13 16:08:10 INFO MemoryStore:59 - ensureFreeSpace(776) called with curMem=38582, maxMem=766075207 -2016-06-13 16:08:10 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 776.0 B, free 730.5 MB) -2016-06-13 16:08:10 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:59919 (size: 776.0 B, free: 730.6 MB) -2016-06-13 16:08:10 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:10 INFO DAGScheduler:59 - Submitting 8 missing tasks from ResultStage 31 (ParallelCollectionRDD[39] at parallelize at BlockMatrixSuite.scala:228) -2016-06-13 16:08:10 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 8 tasks -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 149, localhost, PROCESS_LOCAL, 2649 bytes) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 150, localhost, PROCESS_LOCAL, 2649 bytes) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 2.0 in stage 31.0 (TID 151, localhost, PROCESS_LOCAL, 2649 bytes) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 3.0 in stage 31.0 (TID 152, localhost, PROCESS_LOCAL, 2649 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 149) -2016-06-13 16:08:10 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 150) -2016-06-13 16:08:10 INFO Executor:59 - Running task 2.0 in stage 31.0 (TID 151) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 150). 1496 bytes result sent to driver -2016-06-13 16:08:10 INFO Executor:59 - Finished task 2.0 in stage 31.0 (TID 151). 1496 bytes result sent to driver -2016-06-13 16:08:10 INFO Executor:59 - Running task 3.0 in stage 31.0 (TID 152) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 149). 1496 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 4.0 in stage 31.0 (TID 153, localhost, PROCESS_LOCAL, 2649 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 3.0 in stage 31.0 (TID 152). 1496 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 5.0 in stage 31.0 (TID 154, localhost, PROCESS_LOCAL, 2649 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Running task 4.0 in stage 31.0 (TID 153) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 6.0 in stage 31.0 (TID 155, localhost, PROCESS_LOCAL, 2649 bytes) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Starting task 7.0 in stage 31.0 (TID 156, localhost, PROCESS_LOCAL, 2649 bytes) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 4.0 in stage 31.0 (TID 153). 1496 bytes result sent to driver -2016-06-13 16:08:10 INFO Executor:59 - Running task 6.0 in stage 31.0 (TID 155) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 6.0 in stage 31.0 (TID 155). 1496 bytes result sent to driver -2016-06-13 16:08:10 INFO Executor:59 - Running task 5.0 in stage 31.0 (TID 154) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 5.0 in stage 31.0 (TID 154). 1496 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 2.0 in stage 31.0 (TID 151) in 20 ms on localhost (1/8) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 150) in 21 ms on localhost (2/8) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 3.0 in stage 31.0 (TID 152) in 20 ms on localhost (3/8) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 149) in 24 ms on localhost (4/8) -2016-06-13 16:08:10 INFO Executor:59 - Running task 7.0 in stage 31.0 (TID 156) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 5.0 in stage 31.0 (TID 154) in 14 ms on localhost (5/8) -2016-06-13 16:08:10 INFO Executor:59 - Finished task 7.0 in stage 31.0 (TID 156). 1496 bytes result sent to driver -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 4.0 in stage 31.0 (TID 153) in 20 ms on localhost (6/8) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 6.0 in stage 31.0 (TID 155) in 18 ms on localhost (7/8) -2016-06-13 16:08:10 INFO TaskSetManager:59 - Finished task 7.0 in stage 31.0 (TID 156) in 18 ms on localhost (8/8) -2016-06-13 16:08:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 16:08:10 INFO DAGScheduler:59 - ResultStage 31 (collect at BlockMatrix.scala:289) finished in 0.031 s -2016-06-13 16:08:10 INFO DAGScheduler:59 - Job 18 finished: collect at BlockMatrix.scala:289, took 0.043663 s -2016-06-13 16:08:11 INFO SparkContext:59 - Starting job: reduce at BlockMatrix.scala:189 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Got job 19 (reduce at BlockMatrix.scala:189) with 3 output partitions -2016-06-13 16:08:11 INFO DAGScheduler:59 - Final stage: ResultStage 32(reduce at BlockMatrix.scala:189) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[49] at map at BlockMatrix.scala:186), which has no missing parents -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(2552) called with curMem=39358, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(1479) called with curMem=41910, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 1479.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:59919 (size: 1479.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 32 (MapPartitionsRDD[49] at map at BlockMatrix.scala:186) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 3 tasks -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 157, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 158, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 32.0 (TID 159, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:11 INFO Executor:59 - Running task 2.0 in stage 32.0 (TID 159) -2016-06-13 16:08:11 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 158) -2016-06-13 16:08:11 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 157) -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 16:08:11 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 158). 2209 bytes result sent to driver -2016-06-13 16:08:11 INFO Executor:59 - Finished task 2.0 in stage 32.0 (TID 159). 2209 bytes result sent to driver -2016-06-13 16:08:11 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 157). 2209 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 158) in 21 ms on localhost (1/3) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 32.0 (TID 159) in 21 ms on localhost (2/3) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 157) in 24 ms on localhost (3/3) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 INFO DAGScheduler:59 - ResultStage 32 (reduce at BlockMatrix.scala:189) finished in 0.025 s -2016-06-13 16:08:11 INFO DAGScheduler:59 - Job 19 finished: reduce at BlockMatrix.scala:189, took 0.079346 s -2016-06-13 16:08:11 INFO SparkContext:59 - Starting job: countByKey at BlockMatrix.scala:210 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Registering RDD 50 (countByKey at BlockMatrix.scala:210) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Got job 20 (countByKey at BlockMatrix.scala:210) with 3 output partitions -2016-06-13 16:08:11 INFO DAGScheduler:59 - Final stage: ResultStage 34(countByKey at BlockMatrix.scala:210) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 33) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 33) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting ShuffleMapStage 33 (MapPartitionsRDD[50] at countByKey at BlockMatrix.scala:210), which has no missing parents -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(2784) called with curMem=43389, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 2.7 KB, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(1653) called with curMem=46173, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 1653.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:59919 (size: 1653.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 33 (MapPartitionsRDD[50] at countByKey at BlockMatrix.scala:210) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 3 tasks -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 160, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 161, localhost, PROCESS_LOCAL, 2446 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 162, localhost, PROCESS_LOCAL, 2430 bytes) -2016-06-13 16:08:11 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 161) -2016-06-13 16:08:11 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 162) -2016-06-13 16:08:11 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 160) -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 16:08:11 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 162). 2255 bytes result sent to driver -2016-06-13 16:08:11 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 161). 2255 bytes result sent to driver -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 162) in 17 ms on localhost (1/3) -2016-06-13 16:08:11 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 160). 2255 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 161) in 20 ms on localhost (2/3) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 160) in 24 ms on localhost (3/3) -2016-06-13 16:08:11 INFO DAGScheduler:59 - ShuffleMapStage 33 (countByKey at BlockMatrix.scala:210) finished in 0.024 s -2016-06-13 16:08:11 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:08:11 INFO DAGScheduler:59 - running: Set() -2016-06-13 16:08:11 INFO DAGScheduler:59 - waiting: Set(ResultStage 34) -2016-06-13 16:08:11 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 INFO DAGScheduler:59 - Missing parents for ResultStage 34: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting ResultStage 34 (ShuffledRDD[51] at countByKey at BlockMatrix.scala:210), which is now runnable -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=47826, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(1371) called with curMem=50162, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 1371.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:59919 (size: 1371.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 34 (ShuffledRDD[51] at countByKey at BlockMatrix.scala:210) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 3 tasks -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 163, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 164, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 165, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:11 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 165) -2016-06-13 16:08:11 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 164) -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:11 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 165). 1161 bytes result sent to driver -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:11 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 164). 1161 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 165) in 9 ms on localhost (1/3) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 164) in 10 ms on localhost (2/3) -2016-06-13 16:08:11 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 163) -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:11 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 163). 1508 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 163) in 28 ms on localhost (3/3) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 INFO DAGScheduler:59 - ResultStage 34 (countByKey at BlockMatrix.scala:210) finished in 0.029 s -2016-06-13 16:08:11 INFO DAGScheduler:59 - Job 20 finished: countByKey at BlockMatrix.scala:210, took 0.078478 s -2016-06-13 16:08:11 INFO SparkContext:59 - Starting job: foreach at BlockMatrix.scala:222 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Got job 21 (foreach at BlockMatrix.scala:222) with 3 output partitions -2016-06-13 16:08:11 INFO DAGScheduler:59 - Final stage: ResultStage 35(foreach at BlockMatrix.scala:222) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[1] at mapValues at BlockMatrix.scala:182), which has no missing parents -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=51533, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(1574) called with curMem=54077, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 1574.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:59919 (size: 1574.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 35 (MapPartitionsRDD[1] at mapValues at BlockMatrix.scala:182) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 3 tasks -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 166, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 167, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 35.0 (TID 168, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:11 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 167) -2016-06-13 16:08:11 INFO Executor:59 - Running task 2.0 in stage 35.0 (TID 168) -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 16:08:11 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 167). 2044 bytes result sent to driver -2016-06-13 16:08:11 INFO Executor:59 - Finished task 2.0 in stage 35.0 (TID 168). 2044 bytes result sent to driver -2016-06-13 16:08:11 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 166) -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 16:08:11 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 166). 2044 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 167) in 10 ms on localhost (1/3) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 35.0 (TID 168) in 10 ms on localhost (2/3) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 166) in 12 ms on localhost (3/3) -2016-06-13 16:08:11 INFO DAGScheduler:59 - ResultStage 35 (foreach at BlockMatrix.scala:222) finished in 0.013 s -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 INFO DAGScheduler:59 - Job 21 finished: foreach at BlockMatrix.scala:222, took 0.024109 s -2016-06-13 16:08:11 INFO SparkContext:59 - Starting job: reduce at BlockMatrix.scala:189 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Got job 22 (reduce at BlockMatrix.scala:189) with 3 output partitions -2016-06-13 16:08:11 INFO DAGScheduler:59 - Final stage: ResultStage 36(reduce at BlockMatrix.scala:189) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[54] at map at BlockMatrix.scala:186), which has no missing parents -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(2552) called with curMem=55651, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(1458) called with curMem=58203, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 1458.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:59919 (size: 1458.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 36 (MapPartitionsRDD[54] at map at BlockMatrix.scala:186) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 3 tasks -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 169, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 170, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 36.0 (TID 171, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:11 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 170) -2016-06-13 16:08:11 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 169) -2016-06-13 16:08:11 INFO Executor:59 - Running task 2.0 in stage 36.0 (TID 171) -2016-06-13 16:08:11 INFO CacheManager:59 - Partition rdd_53_2 not found, computing it -2016-06-13 16:08:11 INFO CacheManager:59 - Partition rdd_53_1 not found, computing it -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=59661, maxMem=766075207 -2016-06-13 16:08:11 INFO CacheManager:59 - Partition rdd_53_0 not found, computing it -2016-06-13 16:08:11 INFO MemoryStore:59 - Block rdd_53_2 stored as values in memory (estimated size 200.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=59861, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block rdd_53_0 stored as values in memory (estimated size 112.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added rdd_53_2 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=59973, maxMem=766075207 -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added rdd_53_0 in memory on localhost:59919 (size: 112.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - Block rdd_53_1 stored as values in memory (estimated size 200.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO Executor:59 - Finished task 2.0 in stage 36.0 (TID 171). 1694 bytes result sent to driver -2016-06-13 16:08:11 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 169). 1694 bytes result sent to driver -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added rdd_53_1 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 36.0 (TID 171) in 12 ms on localhost (1/3) -2016-06-13 16:08:11 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 170). 1694 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 169) in 14 ms on localhost (2/3) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 170) in 14 ms on localhost (3/3) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 INFO DAGScheduler:59 - ResultStage 36 (reduce at BlockMatrix.scala:189) finished in 0.016 s -2016-06-13 16:08:11 INFO DAGScheduler:59 - Job 22 finished: reduce at BlockMatrix.scala:189, took 0.028346 s -2016-06-13 16:08:11 INFO SparkContext:59 - Starting job: reduce at BlockMatrix.scala:189 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Got job 23 (reduce at BlockMatrix.scala:189) with 3 output partitions -2016-06-13 16:08:11 INFO DAGScheduler:59 - Final stage: ResultStage 37(reduce at BlockMatrix.scala:189) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[56] at map at BlockMatrix.scala:186), which has no missing parents -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(2552) called with curMem=60173, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(1459) called with curMem=62725, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 1459.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:59919 (size: 1459.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 37 (MapPartitionsRDD[56] at map at BlockMatrix.scala:186) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 3 tasks -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 172, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 173, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 174, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:11 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 172) -2016-06-13 16:08:11 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 173) -2016-06-13 16:08:11 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 174) -2016-06-13 16:08:11 INFO CacheManager:59 - Partition rdd_55_0 not found, computing it -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=64184, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block rdd_55_0 stored as values in memory (estimated size 112.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added rdd_55_0 in memory on localhost:59919 (size: 112.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO CacheManager:59 - Partition rdd_55_2 not found, computing it -2016-06-13 16:08:11 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 172). 1694 bytes result sent to driver -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=64296, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block rdd_55_2 stored as values in memory (estimated size 200.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 172) in 10 ms on localhost (1/3) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added rdd_55_2 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 174). 1694 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 174) in 12 ms on localhost (2/3) -2016-06-13 16:08:11 INFO CacheManager:59 - Partition rdd_55_1 not found, computing it -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=64496, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block rdd_55_1 stored as values in memory (estimated size 200.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added rdd_55_1 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 173). 1694 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 173) in 17 ms on localhost (3/3) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 INFO DAGScheduler:59 - ResultStage 37 (reduce at BlockMatrix.scala:189) finished in 0.019 s -2016-06-13 16:08:11 INFO DAGScheduler:59 - Job 23 finished: reduce at BlockMatrix.scala:189, took 0.033706 s -2016-06-13 16:08:11 INFO SparkContext:59 - Starting job: reduce at BlockMatrix.scala:189 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Got job 24 (reduce at BlockMatrix.scala:189) with 3 output partitions -2016-06-13 16:08:11 INFO DAGScheduler:59 - Final stage: ResultStage 38(reduce at BlockMatrix.scala:189) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[57] at map at BlockMatrix.scala:186), which has no missing parents -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(2552) called with curMem=64696, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(1475) called with curMem=67248, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 1475.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:59919 (size: 1475.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 38 (MapPartitionsRDD[57] at map at BlockMatrix.scala:186) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 3 tasks -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 175, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 176, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 177, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:11 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 175) -2016-06-13 16:08:11 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 176) -2016-06-13 16:08:11 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 177) -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_53_2 locally -2016-06-13 16:08:11 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 177). 2209 bytes result sent to driver -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_53_1 locally -2016-06-13 16:08:11 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 176). 2209 bytes result sent to driver -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_53_0 locally -2016-06-13 16:08:11 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 175). 2209 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 177) in 8 ms on localhost (1/3) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 176) in 9 ms on localhost (2/3) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 175) in 10 ms on localhost (3/3) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 INFO DAGScheduler:59 - ResultStage 38 (reduce at BlockMatrix.scala:189) finished in 0.011 s -2016-06-13 16:08:11 INFO DAGScheduler:59 - Job 24 finished: reduce at BlockMatrix.scala:189, took 0.027394 s -2016-06-13 16:08:11 INFO SparkContext:59 - Starting job: countByKey at BlockMatrix.scala:210 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Registering RDD 58 (countByKey at BlockMatrix.scala:210) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Got job 25 (countByKey at BlockMatrix.scala:210) with 3 output partitions -2016-06-13 16:08:11 INFO DAGScheduler:59 - Final stage: ResultStage 40(countByKey at BlockMatrix.scala:210) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 39) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 39) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting ShuffleMapStage 39 (MapPartitionsRDD[58] at countByKey at BlockMatrix.scala:210), which has no missing parents -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(2784) called with curMem=68723, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 2.7 KB, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(1647) called with curMem=71507, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 1647.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:59919 (size: 1647.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 39 (MapPartitionsRDD[58] at countByKey at BlockMatrix.scala:210) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 3 tasks -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 178, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 179, localhost, PROCESS_LOCAL, 2446 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 39.0 (TID 180, localhost, PROCESS_LOCAL, 2430 bytes) -2016-06-13 16:08:11 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 178) -2016-06-13 16:08:11 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 179) -2016-06-13 16:08:11 INFO Executor:59 - Running task 2.0 in stage 39.0 (TID 180) -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_53_0 locally -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_53_2 locally -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_53_1 locally -2016-06-13 16:08:11 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 178). 2255 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 178) in 13 ms on localhost (1/3) -2016-06-13 16:08:11 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 179). 2255 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 179) in 14 ms on localhost (2/3) -2016-06-13 16:08:11 INFO Executor:59 - Finished task 2.0 in stage 39.0 (TID 180). 2255 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 39.0 (TID 180) in 35 ms on localhost (3/3) -2016-06-13 16:08:11 INFO DAGScheduler:59 - ShuffleMapStage 39 (countByKey at BlockMatrix.scala:210) finished in 0.037 s -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:08:11 INFO DAGScheduler:59 - running: Set() -2016-06-13 16:08:11 INFO DAGScheduler:59 - waiting: Set(ResultStage 40) -2016-06-13 16:08:11 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Missing parents for ResultStage 40: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting ResultStage 40 (ShuffledRDD[59] at countByKey at BlockMatrix.scala:210), which is now runnable -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=73154, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(1372) called with curMem=75490, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 1372.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:59919 (size: 1372.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 40 (ShuffledRDD[59] at countByKey at BlockMatrix.scala:210) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 3 tasks -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 181, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 182, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 40.0 (TID 183, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:11 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 181) -2016-06-13 16:08:11 INFO Executor:59 - Running task 2.0 in stage 40.0 (TID 183) -2016-06-13 16:08:11 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 182) -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:11 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 182). 1161 bytes result sent to driver -2016-06-13 16:08:11 INFO Executor:59 - Finished task 2.0 in stage 40.0 (TID 183). 1161 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 182) in 7 ms on localhost (1/3) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 40.0 (TID 183) in 7 ms on localhost (2/3) -2016-06-13 16:08:11 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 181). 1508 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 181) in 11 ms on localhost (3/3) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 INFO DAGScheduler:59 - ResultStage 40 (countByKey at BlockMatrix.scala:210) finished in 0.012 s -2016-06-13 16:08:11 INFO DAGScheduler:59 - Job 25 finished: countByKey at BlockMatrix.scala:210, took 0.073151 s -2016-06-13 16:08:11 INFO SparkContext:59 - Starting job: foreach at BlockMatrix.scala:222 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Got job 26 (foreach at BlockMatrix.scala:222) with 3 output partitions -2016-06-13 16:08:11 INFO DAGScheduler:59 - Final stage: ResultStage 41(foreach at BlockMatrix.scala:222) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[53] at mapValues at BlockMatrix.scala:182), which has no missing parents -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=76862, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(1561) called with curMem=79406, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 1561.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:59919 (size: 1561.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 41 (MapPartitionsRDD[53] at mapValues at BlockMatrix.scala:182) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 3 tasks -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 184, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 185, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 41.0 (TID 186, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:11 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 184) -2016-06-13 16:08:11 INFO Executor:59 - Running task 2.0 in stage 41.0 (TID 186) -2016-06-13 16:08:11 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 185) -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_53_2 locally -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_53_0 locally -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_53_1 locally -2016-06-13 16:08:11 ERROR Executor:96 - Exception in task 1.0 in stage 41.0 (TID 185) -org.apache.spark.SparkException: The MatrixBlock at (0, 1) has dimensions different than rowsPerBlock: 3, and colsPerBlock: 2. Blocks on the right and bottom edges can have smaller dimensions. You may use the repartition method to fix this issue. - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:225) - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:222) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-06-13 16:08:11 ERROR Executor:96 - Exception in task 0.0 in stage 41.0 (TID 184) -org.apache.spark.SparkException: The MatrixBlock at (0, 0) has dimensions different than rowsPerBlock: 3, and colsPerBlock: 2. Blocks on the right and bottom edges can have smaller dimensions. You may use the repartition method to fix this issue. - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:225) - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:222) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-06-13 16:08:11 ERROR Executor:96 - Exception in task 2.0 in stage 41.0 (TID 186) -org.apache.spark.SparkException: The MatrixBlock at (1, 1) has dimensions different than rowsPerBlock: 3, and colsPerBlock: 2. Blocks on the right and bottom edges can have smaller dimensions. You may use the repartition method to fix this issue. - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:225) - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:222) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-06-13 16:08:11 WARN TaskSetManager:71 - Lost task 1.0 in stage 41.0 (TID 185, localhost): org.apache.spark.SparkException: The MatrixBlock at (0, 1) has dimensions different than rowsPerBlock: 3, and colsPerBlock: 2. Blocks on the right and bottom edges can have smaller dimensions. You may use the repartition method to fix this issue. - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:225) - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:222) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) - -2016-06-13 16:08:11 ERROR TaskSetManager:75 - Task 1 in stage 41.0 failed 1 times; aborting job -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 WARN TaskSetManager:71 - Lost task 0.0 in stage 41.0 (TID 184, localhost): org.apache.spark.SparkException: The MatrixBlock at (0, 0) has dimensions different than rowsPerBlock: 3, and colsPerBlock: 2. Blocks on the right and bottom edges can have smaller dimensions. You may use the repartition method to fix this issue. - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:225) - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:222) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) - -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 WARN TaskSetManager:71 - Lost task 2.0 in stage 41.0 (TID 186, localhost): org.apache.spark.SparkException: The MatrixBlock at (1, 1) has dimensions different than rowsPerBlock: 3, and colsPerBlock: 2. Blocks on the right and bottom edges can have smaller dimensions. You may use the repartition method to fix this issue. - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:225) - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:222) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) - -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Cancelling stage 41 -2016-06-13 16:08:11 INFO DAGScheduler:59 - ResultStage 41 (foreach at BlockMatrix.scala:222) failed in 0.034 s -2016-06-13 16:08:11 INFO DAGScheduler:59 - Job 26 failed: foreach at BlockMatrix.scala:222, took 0.046795 s -2016-06-13 16:08:11 INFO SparkContext:59 - Starting job: reduce at BlockMatrix.scala:189 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Got job 27 (reduce at BlockMatrix.scala:189) with 3 output partitions -2016-06-13 16:08:11 INFO DAGScheduler:59 - Final stage: ResultStage 42(reduce at BlockMatrix.scala:189) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[60] at map at BlockMatrix.scala:186), which has no missing parents -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(2552) called with curMem=80967, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(1478) called with curMem=83519, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1478.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:59919 (size: 1478.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 42 (MapPartitionsRDD[60] at map at BlockMatrix.scala:186) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 3 tasks -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 187, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 188, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 42.0 (TID 189, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:11 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 187) -2016-06-13 16:08:11 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 188) -2016-06-13 16:08:11 INFO Executor:59 - Running task 2.0 in stage 42.0 (TID 189) -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_55_1 locally -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_55_2 locally -2016-06-13 16:08:11 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 188). 2209 bytes result sent to driver -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_55_0 locally -2016-06-13 16:08:11 INFO Executor:59 - Finished task 2.0 in stage 42.0 (TID 189). 2209 bytes result sent to driver -2016-06-13 16:08:11 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 187). 2209 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 188) in 6 ms on localhost (1/3) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 42.0 (TID 189) in 5 ms on localhost (2/3) -2016-06-13 16:08:11 INFO DAGScheduler:59 - ResultStage 42 (reduce at BlockMatrix.scala:189) finished in 0.008 s -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 187) in 8 ms on localhost (3/3) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 INFO DAGScheduler:59 - Job 27 finished: reduce at BlockMatrix.scala:189, took 0.019101 s -2016-06-13 16:08:11 INFO SparkContext:59 - Starting job: countByKey at BlockMatrix.scala:210 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Registering RDD 61 (countByKey at BlockMatrix.scala:210) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Got job 28 (countByKey at BlockMatrix.scala:210) with 3 output partitions -2016-06-13 16:08:11 INFO DAGScheduler:59 - Final stage: ResultStage 44(countByKey at BlockMatrix.scala:210) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 43) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 43) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting ShuffleMapStage 43 (MapPartitionsRDD[61] at countByKey at BlockMatrix.scala:210), which has no missing parents -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(2784) called with curMem=84997, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 2.7 KB, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(1647) called with curMem=87781, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 1647.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:59919 (size: 1647.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 43 (MapPartitionsRDD[61] at countByKey at BlockMatrix.scala:210) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 3 tasks -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 190, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 191, localhost, PROCESS_LOCAL, 2446 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 43.0 (TID 192, localhost, PROCESS_LOCAL, 2430 bytes) -2016-06-13 16:08:11 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 190) -2016-06-13 16:08:11 INFO Executor:59 - Running task 2.0 in stage 43.0 (TID 192) -2016-06-13 16:08:11 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 191) -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_55_0 locally -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_55_1 locally -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_55_2 locally -2016-06-13 16:08:11 INFO Executor:59 - Finished task 2.0 in stage 43.0 (TID 192). 2255 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 43.0 (TID 192) in 9 ms on localhost (1/3) -2016-06-13 16:08:11 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 190). 2255 bytes result sent to driver -2016-06-13 16:08:11 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 191). 2255 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 190) in 21 ms on localhost (2/3) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 191) in 21 ms on localhost (3/3) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 INFO DAGScheduler:59 - ShuffleMapStage 43 (countByKey at BlockMatrix.scala:210) finished in 0.022 s -2016-06-13 16:08:11 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:08:11 INFO DAGScheduler:59 - running: Set() -2016-06-13 16:08:11 INFO DAGScheduler:59 - waiting: Set(ResultStage 44) -2016-06-13 16:08:11 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Missing parents for ResultStage 44: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting ResultStage 44 (ShuffledRDD[62] at countByKey at BlockMatrix.scala:210), which is now runnable -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=89428, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(1372) called with curMem=91764, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 1372.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:59919 (size: 1372.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 44 (ShuffledRDD[62] at countByKey at BlockMatrix.scala:210) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 3 tasks -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 193, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 194, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 44.0 (TID 195, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:11 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 193) -2016-06-13 16:08:11 INFO Executor:59 - Running task 2.0 in stage 44.0 (TID 195) -2016-06-13 16:08:11 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 194) -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:08:11 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 194). 1161 bytes result sent to driver -2016-06-13 16:08:11 INFO Executor:59 - Finished task 2.0 in stage 44.0 (TID 195). 1161 bytes result sent to driver -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 44.0 (TID 195) in 5 ms on localhost (1/3) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 194) in 6 ms on localhost (2/3) -2016-06-13 16:08:11 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 193). 1508 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 193) in 10 ms on localhost (3/3) -2016-06-13 16:08:11 INFO DAGScheduler:59 - ResultStage 44 (countByKey at BlockMatrix.scala:210) finished in 0.011 s -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 INFO DAGScheduler:59 - Job 28 finished: countByKey at BlockMatrix.scala:210, took 0.058816 s -2016-06-13 16:08:11 INFO SparkContext:59 - Starting job: foreach at BlockMatrix.scala:222 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Got job 29 (foreach at BlockMatrix.scala:222) with 3 output partitions -2016-06-13 16:08:11 INFO DAGScheduler:59 - Final stage: ResultStage 45(foreach at BlockMatrix.scala:222) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[55] at mapValues at BlockMatrix.scala:182), which has no missing parents -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=93136, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(1568) called with curMem=95680, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 1568.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:59919 (size: 1568.0 B, free: 730.5 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 45 (MapPartitionsRDD[55] at mapValues at BlockMatrix.scala:182) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 3 tasks -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 196, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 197, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 45.0 (TID 198, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:11 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 196) -2016-06-13 16:08:11 INFO Executor:59 - Running task 2.0 in stage 45.0 (TID 198) -2016-06-13 16:08:11 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 197) -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_55_0 locally -2016-06-13 16:08:11 ERROR Executor:96 - Exception in task 0.0 in stage 45.0 (TID 196) -org.apache.spark.SparkException: The MatrixBlock at (0, 0) has dimensions different than rowsPerBlock: 2, and colsPerBlock: 3. Blocks on the right and bottom edges can have smaller dimensions. You may use the repartition method to fix this issue. - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:230) - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:222) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_55_2 locally -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_55_1 locally -2016-06-13 16:08:11 ERROR Executor:96 - Exception in task 1.0 in stage 45.0 (TID 197) -org.apache.spark.SparkException: The MatrixBlock at (1, 0) has dimensions different than rowsPerBlock: 2, and colsPerBlock: 3. Blocks on the right and bottom edges can have smaller dimensions. You may use the repartition method to fix this issue. - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:230) - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:222) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-06-13 16:08:11 INFO Executor:59 - Finished task 2.0 in stage 45.0 (TID 198). 2044 bytes result sent to driver -2016-06-13 16:08:11 WARN TaskSetManager:71 - Lost task 0.0 in stage 45.0 (TID 196, localhost): org.apache.spark.SparkException: The MatrixBlock at (0, 0) has dimensions different than rowsPerBlock: 2, and colsPerBlock: 3. Blocks on the right and bottom edges can have smaller dimensions. You may use the repartition method to fix this issue. - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:230) - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:222) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) - -2016-06-13 16:08:11 ERROR TaskSetManager:75 - Task 0 in stage 45.0 failed 1 times; aborting job -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 45.0 (TID 198) in 11 ms on localhost (1/3) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 WARN TaskSetManager:71 - Lost task 1.0 in stage 45.0 (TID 197, localhost): org.apache.spark.SparkException: The MatrixBlock at (1, 0) has dimensions different than rowsPerBlock: 2, and colsPerBlock: 3. Blocks on the right and bottom edges can have smaller dimensions. You may use the repartition method to fix this issue. - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:230) - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:222) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) - -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Cancelling stage 45 -2016-06-13 16:08:11 INFO DAGScheduler:59 - ResultStage 45 (foreach at BlockMatrix.scala:222) failed in 0.014 s -2016-06-13 16:08:11 INFO DAGScheduler:59 - Job 29 failed: foreach at BlockMatrix.scala:222, took 0.027151 s -2016-06-13 16:08:11 INFO SparkContext:59 - Starting job: reduce at BlockMatrix.scala:189 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Got job 30 (reduce at BlockMatrix.scala:189) with 3 output partitions -2016-06-13 16:08:11 INFO DAGScheduler:59 - Final stage: ResultStage 46(reduce at BlockMatrix.scala:189) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[64] at map at BlockMatrix.scala:186), which has no missing parents -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(2552) called with curMem=97248, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(1473) called with curMem=99800, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 1473.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:59919 (size: 1473.0 B, free: 730.5 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 46 (MapPartitionsRDD[64] at map at BlockMatrix.scala:186) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 3 tasks -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 199, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 200, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 46.0 (TID 201, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:11 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 200) -2016-06-13 16:08:11 INFO Executor:59 - Running task 2.0 in stage 46.0 (TID 201) -2016-06-13 16:08:11 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 199) -2016-06-13 16:08:11 INFO CacheManager:59 - Partition rdd_63_1 not found, computing it -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=101273, maxMem=766075207 -2016-06-13 16:08:11 INFO CacheManager:59 - Partition rdd_63_0 not found, computing it -2016-06-13 16:08:11 INFO MemoryStore:59 - Block rdd_63_1 stored as values in memory (estimated size 200.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=101473, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block rdd_63_0 stored as values in memory (estimated size 112.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added rdd_63_1 in memory on localhost:59919 (size: 200.0 B, free: 730.5 MB) -2016-06-13 16:08:11 INFO CacheManager:59 - Partition rdd_63_2 not found, computing it -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=101585, maxMem=766075207 -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added rdd_63_0 in memory on localhost:59919 (size: 112.0 B, free: 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - Block rdd_63_2 stored as values in memory (estimated size 200.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 200). 1694 bytes result sent to driver -2016-06-13 16:08:11 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 199). 1694 bytes result sent to driver -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added rdd_63_2 in memory on localhost:59919 (size: 200.0 B, free: 730.5 MB) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 199) in 15 ms on localhost (1/3) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 200) in 14 ms on localhost (2/3) -2016-06-13 16:08:11 INFO Executor:59 - Finished task 2.0 in stage 46.0 (TID 201). 1694 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 46.0 (TID 201) in 15 ms on localhost (3/3) -2016-06-13 16:08:11 INFO DAGScheduler:59 - ResultStage 46 (reduce at BlockMatrix.scala:189) finished in 0.017 s -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 INFO DAGScheduler:59 - Job 30 finished: reduce at BlockMatrix.scala:189, took 0.028770 s -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:59919 in memory (size: 1715.0 B, free: 730.5 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Starting job: reduce at BlockMatrix.scala:189 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Got job 31 (reduce at BlockMatrix.scala:189) with 3 output partitions -2016-06-13 16:08:11 INFO DAGScheduler:59 - Final stage: ResultStage 47(reduce at BlockMatrix.scala:189) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:59919 in memory (size: 1473.0 B, free: 730.5 MB) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[66] at map at BlockMatrix.scala:186), which has no missing parents -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(2552) called with curMem=88981, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:59919 in memory (size: 1568.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(1478) called with curMem=91533, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 1478.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:59919 (size: 1478.0 B, free: 730.5 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:59919 in memory (size: 1372.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 47 (MapPartitionsRDD[66] at map at BlockMatrix.scala:186) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 3 tasks -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 202, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 203, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 47.0 (TID 204, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:11 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 202) -2016-06-13 16:08:11 INFO Executor:59 - Running task 2.0 in stage 47.0 (TID 204) -2016-06-13 16:08:11 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 203) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:59919 in memory (size: 1647.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-06-13 16:08:11 INFO CacheManager:59 - Partition rdd_65_0 not found, computing it -2016-06-13 16:08:11 INFO CacheManager:59 - Partition rdd_65_2 not found, computing it -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=84872, maxMem=766075207 -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned shuffle 15 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block rdd_65_0 stored as values in memory (estimated size 112.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO CacheManager:59 - Partition rdd_65_1 not found, computing it -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=84984, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block rdd_65_2 stored as values in memory (estimated size 200.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=85184, maxMem=766075207 -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added rdd_65_0 in memory on localhost:59919 (size: 112.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - Block rdd_65_1 stored as values in memory (estimated size 200.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added rdd_65_2 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 202). 1694 bytes result sent to driver -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added rdd_65_1 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO Executor:59 - Finished task 2.0 in stage 47.0 (TID 204). 1694 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 202) in 9 ms on localhost (1/3) -2016-06-13 16:08:11 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 203). 1694 bytes result sent to driver -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:59919 in memory (size: 1478.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 47.0 (TID 204) in 9 ms on localhost (2/3) -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 203) in 11 ms on localhost (3/3) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 INFO DAGScheduler:59 - ResultStage 47 (reduce at BlockMatrix.scala:189) finished in 0.012 s -2016-06-13 16:08:11 INFO DAGScheduler:59 - Job 31 finished: reduce at BlockMatrix.scala:189, took 0.022354 s -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:59919 in memory (size: 1561.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:59919 in memory (size: 1372.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:59919 in memory (size: 1647.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned shuffle 14 -2016-06-13 16:08:11 INFO SparkContext:59 - Starting job: reduce at BlockMatrix.scala:189 -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:59919 in memory (size: 1475.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Got job 32 (reduce at BlockMatrix.scala:189) with 3 output partitions -2016-06-13 16:08:11 INFO DAGScheduler:59 - Final stage: ResultStage 48(reduce at BlockMatrix.scala:189) -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[69] at map at BlockMatrix.scala:186), which has no missing parents -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:59919 in memory (size: 1459.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(2552) called with curMem=61072, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(1458) called with curMem=62166, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 1458.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:59919 in memory (size: 1458.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:59919 (size: 1458.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-06-13 16:08:11 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 48 (MapPartitionsRDD[69] at map at BlockMatrix.scala:186) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 3 tasks -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 205, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 206, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 48.0 (TID 207, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:59919 in memory (size: 1574.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 206) -2016-06-13 16:08:11 INFO Executor:59 - Running task 2.0 in stage 48.0 (TID 207) -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-06-13 16:08:11 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 205) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:59919 in memory (size: 1371.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO CacheManager:59 - Partition rdd_68_1 not found, computing it -2016-06-13 16:08:11 INFO CacheManager:59 - Partition rdd_68_0 not found, computing it -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=53247, maxMem=766075207 -2016-06-13 16:08:11 INFO CacheManager:59 - Partition rdd_68_2 not found, computing it -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block rdd_68_1 stored as values in memory (estimated size 200.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=53447, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block rdd_68_0 stored as values in memory (estimated size 112.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=51906, maxMem=766075207 -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added rdd_68_1 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - Block rdd_68_2 stored as values in memory (estimated size 200.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added rdd_68_0 in memory on localhost:59919 (size: 112.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 206). 1694 bytes result sent to driver -2016-06-13 16:08:11 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 205). 1694 bytes result sent to driver -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added rdd_68_2 in memory on localhost:59919 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 206) in 8 ms on localhost (1/3) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:59919 in memory (size: 1653.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO Executor:59 - Finished task 2.0 in stage 48.0 (TID 207). 1694 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 205) in 10 ms on localhost (2/3) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 48.0 (TID 207) in 10 ms on localhost (3/3) -2016-06-13 16:08:11 INFO DAGScheduler:59 - ResultStage 48 (reduce at BlockMatrix.scala:189) finished in 0.011 s -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Job 32 finished: reduce at BlockMatrix.scala:189, took 0.023249 s -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned shuffle 13 -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:59919 in memory (size: 1479.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:59919 in memory (size: 776.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-06-13 16:08:11 INFO SparkContext:59 - Starting job: reduce at BlockMatrix.scala:189 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Got job 33 (reduce at BlockMatrix.scala:189) with 3 output partitions -2016-06-13 16:08:11 INFO DAGScheduler:59 - Final stage: ResultStage 49(reduce at BlockMatrix.scala:189) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:59919 in memory (size: 776.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[70] at map at BlockMatrix.scala:186), which has no missing parents -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(2552) called with curMem=39636, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:59919 in memory (size: 1687.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(1475) called with curMem=39292, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 1475.0 B, free 730.5 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:59919 (size: 1475.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:59919 in memory (size: 1779.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 49 (MapPartitionsRDD[70] at map at BlockMatrix.scala:186) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 3 tasks -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 208, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 209, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 49.0 (TID 210, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:11 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 208) -2016-06-13 16:08:11 INFO Executor:59 - Running task 2.0 in stage 49.0 (TID 210) -2016-06-13 16:08:11 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 209) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:59919 in memory (size: 1778.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_68_0 locally -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_68_1 locally -2016-06-13 16:08:11 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 208). 2209 bytes result sent to driver -2016-06-13 16:08:11 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 209). 2209 bytes result sent to driver -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_68_2 locally -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned shuffle 10 -2016-06-13 16:08:11 INFO Executor:59 - Finished task 2.0 in stage 49.0 (TID 210). 2209 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 208) in 6 ms on localhost (1/3) -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned shuffle 12 -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 49.0 (TID 210) in 6 ms on localhost (2/3) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 209) in 7 ms on localhost (3/3) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 INFO DAGScheduler:59 - ResultStage 49 (reduce at BlockMatrix.scala:189) finished in 0.007 s -2016-06-13 16:08:11 INFO DAGScheduler:59 - Job 33 finished: reduce at BlockMatrix.scala:189, took 0.023780 s -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned shuffle 11 -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:59919 in memory (size: 1461.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Starting job: countByKey at BlockMatrix.scala:210 -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Registering RDD 71 (countByKey at BlockMatrix.scala:210) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Got job 34 (countByKey at BlockMatrix.scala:210) with 3 output partitions -2016-06-13 16:08:11 INFO DAGScheduler:59 - Final stage: ResultStage 51(countByKey at BlockMatrix.scala:210) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 50) -2016-06-13 16:08:11 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 50) -2016-06-13 16:08:11 INFO BlockManager:59 - Removing RDD 40 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting ShuffleMapStage 50 (MapPartitionsRDD[71] at countByKey at BlockMatrix.scala:210), which has no missing parents -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(2784) called with curMem=25933, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 2.7 KB, free 730.6 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(1647) called with curMem=28317, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 1647.0 B, free 730.6 MB) -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned RDD 40 -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:59919 (size: 1647.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 50 (MapPartitionsRDD[71] at countByKey at BlockMatrix.scala:210) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 3 tasks -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 211, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 212, localhost, PROCESS_LOCAL, 2446 bytes) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:59919 in memory (size: 1461.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 50.0 (TID 213, localhost, PROCESS_LOCAL, 2430 bytes) -2016-06-13 16:08:11 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 211) -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_68_0 locally -2016-06-13 16:08:11 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 212) -2016-06-13 16:08:11 INFO Executor:59 - Running task 2.0 in stage 50.0 (TID 213) -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_68_1 locally -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-06-13 16:08:11 INFO BlockManager:59 - Found block rdd_68_2 locally -2016-06-13 16:08:11 INFO BlockManager:59 - Removing RDD 37 -2016-06-13 16:08:11 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 211). 2255 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 211) in 23 ms on localhost (1/3) -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned RDD 37 -2016-06-13 16:08:11 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 212). 2255 bytes result sent to driver -2016-06-13 16:08:11 INFO Executor:59 - Finished task 2.0 in stage 50.0 (TID 213). 2255 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 212) in 28 ms on localhost (2/3) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 50.0 (TID 213) in 29 ms on localhost (3/3) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 INFO DAGScheduler:59 - ShuffleMapStage 50 (countByKey at BlockMatrix.scala:210) finished in 0.035 s -2016-06-13 16:08:11 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:08:11 INFO DAGScheduler:59 - running: Set() -2016-06-13 16:08:11 INFO DAGScheduler:59 - waiting: Set(ResultStage 51) -2016-06-13 16:08:11 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Missing parents for ResultStage 51: List() -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting ResultStage 51 (ShuffledRDD[72] at countByKey at BlockMatrix.scala:210), which is now runnable -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(2336) called with curMem=23951, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 2.3 KB, free 730.6 MB) -2016-06-13 16:08:11 INFO MemoryStore:59 - ensureFreeSpace(1372) called with curMem=24826, maxMem=766075207 -2016-06-13 16:08:11 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 1372.0 B, free 730.6 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:59919 in memory (size: 1461.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:59919 (size: 1372.0 B, free: 730.6 MB) -2016-06-13 16:08:11 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:11 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 51 (ShuffledRDD[72] at countByKey at BlockMatrix.scala:210) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 3 tasks -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 214, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:11 INFO BlockManager:59 - Removing RDD 34 -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 215, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Starting task 2.0 in stage 51.0 (TID 216, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:08:11 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 214) -2016-06-13 16:08:11 INFO Executor:59 - Running task 2.0 in stage 51.0 (TID 216) -2016-06-13 16:08:11 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 215) -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned RDD 34 -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned shuffle 7 -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:08:11 INFO Executor:59 - Finished task 2.0 in stage 51.0 (TID 216). 1161 bytes result sent to driver -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned shuffle 9 -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 16:08:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:08:11 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 215). 1161 bytes result sent to driver -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned shuffle 8 -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 2.0 in stage 51.0 (TID 216) in 10 ms on localhost (1/3) -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 215) in 11 ms on localhost (2/3) -2016-06-13 16:08:11 INFO BlockManager:59 - Removing RDD 24 -2016-06-13 16:08:11 INFO ContextCleaner:59 - Cleaned RDD 24 -2016-06-13 16:08:11 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 214). 1445 bytes result sent to driver -2016-06-13 16:08:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 214) in 20 ms on localhost (3/3) -2016-06-13 16:08:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-06-13 16:08:11 INFO DAGScheduler:59 - ResultStage 51 (countByKey at BlockMatrix.scala:210) finished in 0.020 s -2016-06-13 16:08:11 INFO DAGScheduler:59 - Job 34 finished: countByKey at BlockMatrix.scala:210, took 0.089223 s -2016-06-13 16:08:12 INFO SparkContext:59 - Starting job: collect at BlockMatrix.scala:289 -2016-06-13 16:08:12 INFO DAGScheduler:59 - Got job 35 (collect at BlockMatrix.scala:289) with 3 output partitions -2016-06-13 16:08:12 INFO DAGScheduler:59 - Final stage: ResultStage 52(collect at BlockMatrix.scala:289) -2016-06-13 16:08:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:12 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:12 INFO DAGScheduler:59 - Submitting ResultStage 52 (MapPartitionsRDD[73] at map at BlockMatrix.scala:308), which has no missing parents -2016-06-13 16:08:12 INFO MemoryStore:59 - ensureFreeSpace(1896) called with curMem=23278, maxMem=766075207 -2016-06-13 16:08:12 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 1896.0 B, free 730.6 MB) -2016-06-13 16:08:12 INFO MemoryStore:59 - ensureFreeSpace(1188) called with curMem=25174, maxMem=766075207 -2016-06-13 16:08:12 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 1188.0 B, free 730.6 MB) -2016-06-13 16:08:12 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:59919 (size: 1188.0 B, free: 730.6 MB) -2016-06-13 16:08:12 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:12 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 52 (MapPartitionsRDD[73] at map at BlockMatrix.scala:308) -2016-06-13 16:08:12 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 3 tasks -2016-06-13 16:08:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 217, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 218, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 52.0 (TID 219, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:12 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 217) -2016-06-13 16:08:12 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 218) -2016-06-13 16:08:12 INFO Executor:59 - Running task 2.0 in stage 52.0 (TID 219) -2016-06-13 16:08:12 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 217). 1225 bytes result sent to driver -2016-06-13 16:08:12 INFO Executor:59 - Finished task 2.0 in stage 52.0 (TID 219). 1288 bytes result sent to driver -2016-06-13 16:08:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 217) in 5 ms on localhost (1/3) -2016-06-13 16:08:12 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 218). 1304 bytes result sent to driver -2016-06-13 16:08:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 52.0 (TID 219) in 10 ms on localhost (2/3) -2016-06-13 16:08:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 218) in 10 ms on localhost (3/3) -2016-06-13 16:08:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-06-13 16:08:12 INFO DAGScheduler:59 - ResultStage 52 (collect at BlockMatrix.scala:289) finished in 0.013 s -2016-06-13 16:08:12 INFO DAGScheduler:59 - Job 35 finished: collect at BlockMatrix.scala:289, took 0.020802 s -2016-06-13 16:08:12 INFO SparkContext:59 - Starting job: collect at BlockMatrix.scala:289 -2016-06-13 16:08:12 INFO DAGScheduler:59 - Got job 36 (collect at BlockMatrix.scala:289) with 3 output partitions -2016-06-13 16:08:12 INFO DAGScheduler:59 - Final stage: ResultStage 53(collect at BlockMatrix.scala:289) -2016-06-13 16:08:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:12 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:12 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[74] at map at BlockMatrix.scala:308), which has no missing parents -2016-06-13 16:08:12 INFO MemoryStore:59 - ensureFreeSpace(1896) called with curMem=26362, maxMem=766075207 -2016-06-13 16:08:12 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 1896.0 B, free 730.6 MB) -2016-06-13 16:08:12 INFO MemoryStore:59 - ensureFreeSpace(1188) called with curMem=28258, maxMem=766075207 -2016-06-13 16:08:12 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 1188.0 B, free 730.6 MB) -2016-06-13 16:08:12 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:59919 (size: 1188.0 B, free: 730.6 MB) -2016-06-13 16:08:12 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:12 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 53 (MapPartitionsRDD[74] at map at BlockMatrix.scala:308) -2016-06-13 16:08:12 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 3 tasks -2016-06-13 16:08:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 220, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 221, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 53.0 (TID 222, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:12 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 221) -2016-06-13 16:08:12 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 220) -2016-06-13 16:08:12 INFO Executor:59 - Running task 2.0 in stage 53.0 (TID 222) -2016-06-13 16:08:12 INFO CacheManager:59 - Partition rdd_74_0 not found, computing it -2016-06-13 16:08:12 INFO CacheManager:59 - Partition rdd_0_0 not found, computing it -2016-06-13 16:08:12 INFO MemoryStore:59 - ensureFreeSpace(160) called with curMem=29446, maxMem=766075207 -2016-06-13 16:08:12 INFO MemoryStore:59 - Block rdd_0_0 stored as values in memory (estimated size 160.0 B, free 730.6 MB) -2016-06-13 16:08:12 INFO CacheManager:59 - Partition rdd_74_1 not found, computing it -2016-06-13 16:08:12 INFO CacheManager:59 - Partition rdd_74_2 not found, computing it -2016-06-13 16:08:12 INFO BlockManagerInfo:59 - Added rdd_0_0 in memory on localhost:59919 (size: 160.0 B, free: 730.6 MB) -2016-06-13 16:08:12 INFO CacheManager:59 - Partition rdd_0_1 not found, computing it -2016-06-13 16:08:12 INFO MemoryStore:59 - ensureFreeSpace(296) called with curMem=29606, maxMem=766075207 -2016-06-13 16:08:12 INFO MemoryStore:59 - Block rdd_0_1 stored as values in memory (estimated size 296.0 B, free 730.6 MB) -2016-06-13 16:08:12 INFO MemoryStore:59 - ensureFreeSpace(160) called with curMem=29902, maxMem=766075207 -2016-06-13 16:08:12 INFO CacheManager:59 - Partition rdd_0_2 not found, computing it -2016-06-13 16:08:12 INFO MemoryStore:59 - Block rdd_74_0 stored as values in memory (estimated size 160.0 B, free 730.6 MB) -2016-06-13 16:08:12 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=30062, maxMem=766075207 -2016-06-13 16:08:12 INFO BlockManagerInfo:59 - Added rdd_0_1 in memory on localhost:59919 (size: 296.0 B, free: 730.6 MB) -2016-06-13 16:08:12 INFO MemoryStore:59 - Block rdd_0_2 stored as values in memory (estimated size 280.0 B, free 730.6 MB) -2016-06-13 16:08:12 INFO BlockManagerInfo:59 - Added rdd_74_0 in memory on localhost:59919 (size: 160.0 B, free: 730.6 MB) -2016-06-13 16:08:12 INFO MemoryStore:59 - ensureFreeSpace(296) called with curMem=30342, maxMem=766075207 -2016-06-13 16:08:12 INFO MemoryStore:59 - Block rdd_74_1 stored as values in memory (estimated size 296.0 B, free 730.6 MB) -2016-06-13 16:08:12 INFO BlockManagerInfo:59 - Added rdd_0_2 in memory on localhost:59919 (size: 280.0 B, free: 730.6 MB) -2016-06-13 16:08:12 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=30638, maxMem=766075207 -2016-06-13 16:08:12 INFO MemoryStore:59 - Block rdd_74_2 stored as values in memory (estimated size 280.0 B, free 730.6 MB) -2016-06-13 16:08:12 INFO BlockManagerInfo:59 - Added rdd_74_1 in memory on localhost:59919 (size: 296.0 B, free: 730.6 MB) -2016-06-13 16:08:12 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 220). 1894 bytes result sent to driver -2016-06-13 16:08:12 INFO BlockManagerInfo:59 - Added rdd_74_2 in memory on localhost:59919 (size: 280.0 B, free: 730.6 MB) -2016-06-13 16:08:12 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 221). 1973 bytes result sent to driver -2016-06-13 16:08:12 INFO Executor:59 - Finished task 2.0 in stage 53.0 (TID 222). 1957 bytes result sent to driver -2016-06-13 16:08:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 220) in 15 ms on localhost (1/3) -2016-06-13 16:08:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 53.0 (TID 222) in 13 ms on localhost (2/3) -2016-06-13 16:08:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 221) in 14 ms on localhost (3/3) -2016-06-13 16:08:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-06-13 16:08:12 INFO DAGScheduler:59 - ResultStage 53 (collect at BlockMatrix.scala:289) finished in 0.015 s -2016-06-13 16:08:12 INFO DAGScheduler:59 - Job 36 finished: collect at BlockMatrix.scala:289, took 0.026676 s -2016-06-13 16:08:12 INFO SparkContext:59 - Starting job: collect at BlockMatrix.scala:289 -2016-06-13 16:08:12 INFO DAGScheduler:59 - Got job 37 (collect at BlockMatrix.scala:289) with 3 output partitions -2016-06-13 16:08:12 INFO DAGScheduler:59 - Final stage: ResultStage 54(collect at BlockMatrix.scala:289) -2016-06-13 16:08:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:12 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:12 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[73] at map at BlockMatrix.scala:308), which has no missing parents -2016-06-13 16:08:12 INFO MemoryStore:59 - ensureFreeSpace(1904) called with curMem=30918, maxMem=766075207 -2016-06-13 16:08:12 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 1904.0 B, free 730.6 MB) -2016-06-13 16:08:12 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=32822, maxMem=766075207 -2016-06-13 16:08:12 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 1193.0 B, free 730.6 MB) -2016-06-13 16:08:12 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:59919 (size: 1193.0 B, free: 730.6 MB) -2016-06-13 16:08:12 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:12 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 54 (MapPartitionsRDD[73] at map at BlockMatrix.scala:308) -2016-06-13 16:08:12 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 3 tasks -2016-06-13 16:08:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 223, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 224, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 54.0 (TID 225, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:12 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 223) -2016-06-13 16:08:12 INFO Executor:59 - Running task 2.0 in stage 54.0 (TID 225) -2016-06-13 16:08:12 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 224) -2016-06-13 16:08:12 INFO BlockManager:59 - Found block rdd_0_0 locally -2016-06-13 16:08:12 INFO BlockManager:59 - Found block rdd_0_1 locally -2016-06-13 16:08:12 INFO BlockManager:59 - Found block rdd_0_2 locally -2016-06-13 16:08:12 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 223). 2354 bytes result sent to driver -2016-06-13 16:08:12 INFO Executor:59 - Finished task 2.0 in stage 54.0 (TID 225). 2417 bytes result sent to driver -2016-06-13 16:08:12 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 224). 2433 bytes result sent to driver -2016-06-13 16:08:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 223) in 4 ms on localhost (1/3) -2016-06-13 16:08:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 224) in 5 ms on localhost (2/3) -2016-06-13 16:08:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 54.0 (TID 225) in 4 ms on localhost (3/3) -2016-06-13 16:08:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-06-13 16:08:12 INFO DAGScheduler:59 - ResultStage 54 (collect at BlockMatrix.scala:289) finished in 0.005 s -2016-06-13 16:08:12 INFO DAGScheduler:59 - Job 37 finished: collect at BlockMatrix.scala:289, took 0.013385 s -2016-06-13 16:08:12 INFO SparkContext:59 - Starting job: collect at BlockMatrix.scala:289 -2016-06-13 16:08:12 INFO DAGScheduler:59 - Got job 38 (collect at BlockMatrix.scala:289) with 3 output partitions -2016-06-13 16:08:12 INFO DAGScheduler:59 - Final stage: ResultStage 55(collect at BlockMatrix.scala:289) -2016-06-13 16:08:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:12 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:12 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[75] at map at BlockMatrix.scala:308), which has no missing parents -2016-06-13 16:08:12 INFO MemoryStore:59 - ensureFreeSpace(1976) called with curMem=34015, maxMem=766075207 -2016-06-13 16:08:12 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 1976.0 B, free 730.6 MB) -2016-06-13 16:08:12 INFO MemoryStore:59 - ensureFreeSpace(1243) called with curMem=35991, maxMem=766075207 -2016-06-13 16:08:12 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 1243.0 B, free 730.6 MB) -2016-06-13 16:08:12 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:59919 (size: 1243.0 B, free: 730.6 MB) -2016-06-13 16:08:12 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:12 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 55 (MapPartitionsRDD[75] at map at BlockMatrix.scala:308) -2016-06-13 16:08:12 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 3 tasks -2016-06-13 16:08:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 226, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 227, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 55.0 (TID 228, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:12 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 226) -2016-06-13 16:08:12 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 227) -2016-06-13 16:08:12 INFO Executor:59 - Running task 2.0 in stage 55.0 (TID 228) -2016-06-13 16:08:12 INFO BlockManager:59 - Found block rdd_74_0 locally -2016-06-13 16:08:12 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 226). 2354 bytes result sent to driver -2016-06-13 16:08:12 INFO BlockManager:59 - Found block rdd_74_1 locally -2016-06-13 16:08:12 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 227). 2433 bytes result sent to driver -2016-06-13 16:08:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 226) in 7 ms on localhost (1/3) -2016-06-13 16:08:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 227) in 8 ms on localhost (2/3) -2016-06-13 16:08:12 INFO BlockManager:59 - Found block rdd_74_2 locally -2016-06-13 16:08:12 INFO Executor:59 - Finished task 2.0 in stage 55.0 (TID 228). 2417 bytes result sent to driver -2016-06-13 16:08:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 55.0 (TID 228) in 9 ms on localhost (3/3) -2016-06-13 16:08:12 INFO DAGScheduler:59 - ResultStage 55 (collect at BlockMatrix.scala:289) finished in 0.011 s -2016-06-13 16:08:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-06-13 16:08:12 INFO DAGScheduler:59 - Job 38 finished: collect at BlockMatrix.scala:289, took 0.020352 s -2016-06-13 16:08:12 INFO SparkContext:59 - Starting job: collect at BlockMatrix.scala:289 -2016-06-13 16:08:12 INFO DAGScheduler:59 - Got job 39 (collect at BlockMatrix.scala:289) with 3 output partitions -2016-06-13 16:08:12 INFO DAGScheduler:59 - Final stage: ResultStage 56(collect at BlockMatrix.scala:289) -2016-06-13 16:08:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:08:12 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:08:12 INFO DAGScheduler:59 - Submitting ResultStage 56 (ParallelCollectionRDD[0] at parallelize at BlockMatrixSuite.scala:46), which has no missing parents -2016-06-13 16:08:12 INFO MemoryStore:59 - ensureFreeSpace(1208) called with curMem=37234, maxMem=766075207 -2016-06-13 16:08:12 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 1208.0 B, free 730.5 MB) -2016-06-13 16:08:12 INFO MemoryStore:59 - ensureFreeSpace(774) called with curMem=38442, maxMem=766075207 -2016-06-13 16:08:12 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 774.0 B, free 730.5 MB) -2016-06-13 16:08:12 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:59919 (size: 774.0 B, free: 730.6 MB) -2016-06-13 16:08:12 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:08:12 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 56 (ParallelCollectionRDD[0] at parallelize at BlockMatrixSuite.scala:46) -2016-06-13 16:08:12 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 3 tasks -2016-06-13 16:08:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 229, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:08:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 230, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:08:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 56.0 (TID 231, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:08:12 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 229) -2016-06-13 16:08:12 INFO Executor:59 - Running task 2.0 in stage 56.0 (TID 231) -2016-06-13 16:08:12 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 230) -2016-06-13 16:08:12 INFO BlockManager:59 - Found block rdd_0_0 locally -2016-06-13 16:08:12 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 229). 2354 bytes result sent to driver -2016-06-13 16:08:12 INFO BlockManager:59 - Found block rdd_0_1 locally -2016-06-13 16:08:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 229) in 5 ms on localhost (1/3) -2016-06-13 16:08:12 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 230). 2433 bytes result sent to driver -2016-06-13 16:08:12 INFO BlockManager:59 - Found block rdd_0_2 locally -2016-06-13 16:08:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 230) in 5 ms on localhost (2/3) -2016-06-13 16:08:12 INFO Executor:59 - Finished task 2.0 in stage 56.0 (TID 231). 2417 bytes result sent to driver -2016-06-13 16:08:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 56.0 (TID 231) in 6 ms on localhost (3/3) -2016-06-13 16:08:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-06-13 16:08:12 INFO DAGScheduler:59 - ResultStage 56 (collect at BlockMatrix.scala:289) finished in 0.008 s -2016-06-13 16:08:12 INFO DAGScheduler:59 - Job 39 finished: collect at BlockMatrix.scala:289, took 0.017505 s -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 16:08:12 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 16:08:12 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 16:08:12 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 16:08:12 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 16:08:12 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 16:08:12 INFO BlockManager:59 - BlockManager stopped -2016-06-13 16:08:12 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 16:08:12 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 16:08:12 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 16:08:12 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 16:08:12 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 16:08:12 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-6579a20a-2889-4ec8-af4d-214dad18d8e4 -2016-06-13 16:08:12 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 16:30:32 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 16:30:34 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 16:30:35 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 16:30:35 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 16:30:35 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 16:30:38 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 16:30:38 INFO Remoting:74 - Starting remoting -2016-06-13 16:30:38 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:61055] -2016-06-13 16:30:38 INFO Utils:59 - Successfully started service 'sparkDriver' on port 61055. -2016-06-13 16:30:39 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 16:30:39 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 16:30:39 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-cdf39ffe-991f-419e-b1a4-50b28db566b9 -2016-06-13 16:30:39 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 16:30:39 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-937223ae-4106-4597-adb3-fb21df02149e\httpd-f500cb3a-20e2-46e4-94cd-515c14575b45 -2016-06-13 16:30:39 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 16:30:39 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 16:30:39 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:61056 -2016-06-13 16:30:39 INFO Utils:59 - Successfully started service 'HTTP file server' on port 61056. -2016-06-13 16:30:39 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 16:30:40 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 16:30:40 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 16:30:40 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 16:30:40 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 16:30:41 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 16:30:41 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 16:30:41 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 61075. -2016-06-13 16:30:41 INFO NettyBlockTransferService:59 - Server created on 61075 -2016-06-13 16:30:41 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 16:30:41 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:61075 with 730.6 MB RAM, BlockManagerId(driver, localhost, 61075) -2016-06-13 16:30:42 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 16:30:46 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 16:30:46 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 16:30:46 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 16:30:46 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 16:30:46 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:61075 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 16:30:46 INFO SparkContext:59 - Created broadcast 0 from textFile at RowMatrixSuite.scala:25 -2016-06-13 16:30:48 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 16:30:48 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 16:30:48 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-06-13 16:30:49 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 2 output partitions -2016-06-13 16:30:49 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-06-13 16:30:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:30:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:30:49 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:28), which has no missing parents -2016-06-13 16:30:49 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=171580, maxMem=766075207 -2016-06-13 16:30:49 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 16:30:49 INFO MemoryStore:59 - ensureFreeSpace(1945) called with curMem=174956, maxMem=766075207 -2016-06-13 16:30:49 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1945.0 B, free 730.4 MB) -2016-06-13 16:30:49 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:61075 (size: 1945.0 B, free: 730.6 MB) -2016-06-13 16:30:49 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:30:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:28) -2016-06-13 16:30:49 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 16:30:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 16:30:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 16:30:49 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 16:30:49 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 16:30:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 16:30:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 16:30:49 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 16:30:49 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 16:30:49 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 16:30:49 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 16:30:49 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 16:30:49 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 16:30:49 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 16:30:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 464 ms on localhost (1/2) -2016-06-13 16:30:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 417 ms on localhost (2/2) -2016-06-13 16:30:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 16:30:49 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.488 s -2016-06-13 16:30:49 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.957961 s -2016-06-13 16:30:49 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 16:30:49 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 16:30:49 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-06-13 16:30:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:30:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:30:50 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:28), which has no missing parents -2016-06-13 16:30:50 INFO MemoryStore:59 - ensureFreeSpace(3544) called with curMem=176901, maxMem=766075207 -2016-06-13 16:30:50 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 16:30:50 INFO MemoryStore:59 - ensureFreeSpace(2003) called with curMem=180445, maxMem=766075207 -2016-06-13 16:30:50 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2003.0 B, free 730.4 MB) -2016-06-13 16:30:50 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:61075 (size: 2003.0 B, free: 730.6 MB) -2016-06-13 16:30:50 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:30:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at RowMatrixSuite.scala:28) -2016-06-13 16:30:50 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 16:30:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 16:30:50 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 16:30:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 16:30:50 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2183 bytes result sent to driver -2016-06-13 16:30:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 23 ms on localhost (1/1) -2016-06-13 16:30:50 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.024 s -2016-06-13 16:30:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 16:30:50 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.057426 s -2016-06-13 16:30:50 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:61075 in memory (size: 1945.0 B, free: 730.6 MB) -2016-06-13 16:30:50 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 16:30:50 INFO SparkContext:59 - Starting job: treeReduce at RowMatrix.scala:537 -2016-06-13 16:30:50 INFO DAGScheduler:59 - Got job 2 (treeReduce at RowMatrix.scala:537) with 2 output partitions -2016-06-13 16:30:50 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeReduce at RowMatrix.scala:537) -2016-06-13 16:30:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:30:50 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:30:50 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537), which has no missing parents -2016-06-13 16:30:50 INFO MemoryStore:59 - ensureFreeSpace(5080) called with curMem=177127, maxMem=766075207 -2016-06-13 16:30:50 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-06-13 16:30:50 INFO MemoryStore:59 - ensureFreeSpace(2504) called with curMem=182207, maxMem=766075207 -2016-06-13 16:30:50 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 16:30:50 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:61075 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 16:30:50 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:30:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at treeReduce at RowMatrix.scala:537) -2016-06-13 16:30:50 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 16:30:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 16:30:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 16:30:50 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 16:30:50 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-06-13 16:30:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 16:30:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 16:30:51 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:61075 in memory (size: 2003.0 B, free: 730.6 MB) -2016-06-13 16:30:51 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 16:30:55 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 16:30:55 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 16:30:55 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2285 bytes result sent to driver -2016-06-13 16:30:55 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2285 bytes result sent to driver -2016-06-13 16:30:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 5088 ms on localhost (1/2) -2016-06-13 16:30:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 5089 ms on localhost (2/2) -2016-06-13 16:30:55 INFO DAGScheduler:59 - ResultStage 2 (treeReduce at RowMatrix.scala:537) finished in 5.091 s -2016-06-13 16:30:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 16:30:55 INFO DAGScheduler:59 - Job 2 finished: treeReduce at RowMatrix.scala:537, took 5.140134 s -2016-06-13 16:30:55 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=179164, maxMem=766075207 -2016-06-13 16:30:55 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 112.0 B, free 730.4 MB) -2016-06-13 16:30:55 INFO MemoryStore:59 - ensureFreeSpace(104) called with curMem=179276, maxMem=766075207 -2016-06-13 16:30:55 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 104.0 B, free 730.4 MB) -2016-06-13 16:30:55 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:61075 (size: 104.0 B, free: 730.6 MB) -2016-06-13 16:30:55 INFO SparkContext:59 - Created broadcast 4 from broadcast at RowMatrix.scala:425 -2016-06-13 16:30:55 INFO SparkContext:59 - Starting job: foreach at RowMatrixSuite.scala:42 -2016-06-13 16:30:55 INFO DAGScheduler:59 - Got job 3 (foreach at RowMatrixSuite.scala:42) with 2 output partitions -2016-06-13 16:30:55 INFO DAGScheduler:59 - Final stage: ResultStage 3(foreach at RowMatrixSuite.scala:42) -2016-06-13 16:30:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:30:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:30:55 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at mapPartitions at RowMatrix.scala:426), which has no missing parents -2016-06-13 16:30:55 INFO MemoryStore:59 - ensureFreeSpace(3848) called with curMem=179380, maxMem=766075207 -2016-06-13 16:30:55 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 16:30:55 INFO MemoryStore:59 - ensureFreeSpace(2119) called with curMem=183228, maxMem=766075207 -2016-06-13 16:30:55 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 16:30:55 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:61075 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 16:30:55 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:30:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at mapPartitions at RowMatrix.scala:426) -2016-06-13 16:30:55 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 16:30:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 16:30:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 16:30:56 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-06-13 16:30:56 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-06-13 16:30:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 16:30:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 16:30:56 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 16:30:56 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 16:30:56 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2044 bytes result sent to driver -2016-06-13 16:30:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 106 ms on localhost (1/2) -2016-06-13 16:30:56 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2044 bytes result sent to driver -2016-06-13 16:30:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 128 ms on localhost (2/2) -2016-06-13 16:30:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 16:30:56 INFO DAGScheduler:59 - ResultStage 3 (foreach at RowMatrixSuite.scala:42) finished in 0.130 s -2016-06-13 16:30:56 INFO DAGScheduler:59 - Job 3 finished: foreach at RowMatrixSuite.scala:42, took 0.149584 s -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 16:30:56 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:61075 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 16:30:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 16:30:56 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 16:30:56 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:61075 in memory (size: 104.0 B, free: 730.6 MB) -2016-06-13 16:30:56 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 16:30:56 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 16:30:56 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 16:30:56 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 16:30:56 INFO BlockManager:59 - BlockManager stopped -2016-06-13 16:30:56 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 16:30:56 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 16:30:56 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 16:30:56 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 16:30:56 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 16:30:56 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-937223ae-4106-4597-adb3-fb21df02149e -2016-06-13 16:30:56 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 16:31:56 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 16:31:57 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 16:31:57 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 16:31:57 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 16:31:57 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 16:31:58 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 16:31:58 INFO Remoting:74 - Starting remoting -2016-06-13 16:31:58 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:61098] -2016-06-13 16:31:58 INFO Utils:59 - Successfully started service 'sparkDriver' on port 61098. -2016-06-13 16:31:58 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 16:31:58 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 16:31:58 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-dcfeaf81-5176-4e76-93dd-954fcc2a3d4b -2016-06-13 16:31:58 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 16:31:58 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-d00e90e5-56dd-4896-ae71-63d8aaaf024d\httpd-3f193252-17f5-40b4-9a62-cc0b43e58b8c -2016-06-13 16:31:58 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 16:31:59 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 16:31:59 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:61099 -2016-06-13 16:31:59 INFO Utils:59 - Successfully started service 'HTTP file server' on port 61099. -2016-06-13 16:31:59 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 16:31:59 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 16:31:59 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 16:31:59 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 16:31:59 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 16:31:59 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 16:31:59 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 16:31:59 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 61106. -2016-06-13 16:31:59 INFO NettyBlockTransferService:59 - Server created on 61106 -2016-06-13 16:31:59 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 16:31:59 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:61106 with 730.6 MB RAM, BlockManagerId(driver, localhost, 61106) -2016-06-13 16:31:59 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 16:32:00 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 16:32:00 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 16:32:00 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 16:32:00 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 16:32:00 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:61106 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 16:32:00 INFO SparkContext:59 - Created broadcast 0 from textFile at MatrixRowLearning.scala:22 -2016-06-13 16:32:01 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 16:32:01 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 16:32:01 INFO SparkContext:59 - Starting job: count at RowMatrix.scala:79 -2016-06-13 16:32:01 INFO DAGScheduler:59 - Got job 0 (count at RowMatrix.scala:79) with 1 output partitions -2016-06-13 16:32:01 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at RowMatrix.scala:79) -2016-06-13 16:32:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:32:01 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:32:01 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25), which has no missing parents -2016-06-13 16:32:01 INFO MemoryStore:59 - ensureFreeSpace(3360) called with curMem=171580, maxMem=766075207 -2016-06-13 16:32:01 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 16:32:01 INFO MemoryStore:59 - ensureFreeSpace(1935) called with curMem=174940, maxMem=766075207 -2016-06-13 16:32:01 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1935.0 B, free 730.4 MB) -2016-06-13 16:32:01 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:61106 (size: 1935.0 B, free: 730.6 MB) -2016-06-13 16:32:01 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:32:01 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25) -2016-06-13 16:32:01 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 16:32:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2182 bytes) -2016-06-13 16:32:01 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 16:32:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-06-13 16:32:01 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 16:32:01 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 16:32:01 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 16:32:01 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 16:32:01 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 16:32:01 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 16:32:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 87 ms on localhost (1/1) -2016-06-13 16:32:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 16:32:01 INFO DAGScheduler:59 - ResultStage 0 (count at RowMatrix.scala:79) finished in 0.102 s -2016-06-13 16:32:01 INFO DAGScheduler:59 - Job 0 finished: count at RowMatrix.scala:79, took 0.195702 s -2016-06-13 16:32:01 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 16:32:01 INFO DAGScheduler:59 - Got job 1 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 16:32:01 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at RowMatrix.scala:65) -2016-06-13 16:32:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:32:01 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:32:01 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25), which has no missing parents -2016-06-13 16:32:01 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=176875, maxMem=766075207 -2016-06-13 16:32:01 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-06-13 16:32:01 INFO MemoryStore:59 - ensureFreeSpace(1993) called with curMem=180403, maxMem=766075207 -2016-06-13 16:32:01 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1993.0 B, free 730.4 MB) -2016-06-13 16:32:01 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:61106 (size: 1993.0 B, free: 730.6 MB) -2016-06-13 16:32:01 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:32:01 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25) -2016-06-13 16:32:01 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 16:32:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2182 bytes) -2016-06-13 16:32:01 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 16:32:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-06-13 16:32:01 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2183 bytes result sent to driver -2016-06-13 16:32:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 19 ms on localhost (1/1) -2016-06-13 16:32:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 16:32:01 INFO DAGScheduler:59 - ResultStage 1 (first at RowMatrix.scala:65) finished in 0.019 s -2016-06-13 16:32:01 INFO DAGScheduler:59 - Job 1 finished: first at RowMatrix.scala:65, took 0.035987 s -2016-06-13 16:32:02 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:34 -2016-06-13 16:32:02 INFO DAGScheduler:59 - Got job 2 (foreach at MatrixRowLearning.scala:34) with 1 output partitions -2016-06-13 16:32:02 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at MatrixRowLearning.scala:34) -2016-06-13 16:32:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:32:02 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:32:02 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25), which has no missing parents -2016-06-13 16:32:02 INFO MemoryStore:59 - ensureFreeSpace(3480) called with curMem=182396, maxMem=766075207 -2016-06-13 16:32:02 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-06-13 16:32:02 INFO MemoryStore:59 - ensureFreeSpace(1973) called with curMem=185876, maxMem=766075207 -2016-06-13 16:32:02 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1973.0 B, free 730.4 MB) -2016-06-13 16:32:02 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:61106 (size: 1973.0 B, free: 730.6 MB) -2016-06-13 16:32:02 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:32:02 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[3] at map at MatrixRowLearning.scala:25) -2016-06-13 16:32:02 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-06-13 16:32:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2182 bytes) -2016-06-13 16:32:02 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-06-13 16:32:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-06-13 16:32:02 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 2044 bytes result sent to driver -2016-06-13 16:32:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 18 ms on localhost (1/1) -2016-06-13 16:32:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 16:32:02 INFO DAGScheduler:59 - ResultStage 2 (foreach at MatrixRowLearning.scala:34) finished in 0.019 s -2016-06-13 16:32:02 INFO DAGScheduler:59 - Job 2 finished: foreach at MatrixRowLearning.scala:34, took 0.051253 s -2016-06-13 16:32:02 INFO MemoryStore:59 - ensureFreeSpace(88528) called with curMem=187849, maxMem=766075207 -2016-06-13 16:32:02 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 86.5 KB, free 730.3 MB) -2016-06-13 16:32:02 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=276377, maxMem=766075207 -2016-06-13 16:32:02 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.3 MB) -2016-06-13 16:32:02 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:61106 (size: 19.3 KB, free: 730.5 MB) -2016-06-13 16:32:02 INFO SparkContext:59 - Created broadcast 4 from textFile at MatrixRowLearning.scala:37 -2016-06-13 16:32:02 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 16:32:02 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:44 -2016-06-13 16:32:02 INFO DAGScheduler:59 - Got job 3 (foreach at MatrixRowLearning.scala:44) with 1 output partitions -2016-06-13 16:32:02 INFO DAGScheduler:59 - Final stage: ResultStage 3(foreach at MatrixRowLearning.scala:44) -2016-06-13 16:32:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:32:02 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:32:02 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at map at MatrixRowLearning.scala:41), which has no missing parents -2016-06-13 16:32:02 INFO MemoryStore:59 - ensureFreeSpace(3752) called with curMem=296165, maxMem=766075207 -2016-06-13 16:32:02 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.7 KB, free 730.3 MB) -2016-06-13 16:32:02 INFO MemoryStore:59 - ensureFreeSpace(2063) called with curMem=299917, maxMem=766075207 -2016-06-13 16:32:02 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 16:32:02 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:61106 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 16:32:02 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:32:02 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at map at MatrixRowLearning.scala:41) -2016-06-13 16:32:02 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-06-13 16:32:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 3, localhost, PROCESS_LOCAL, 2182 bytes) -2016-06-13 16:32:02 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 3) -2016-06-13 16:32:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-06-13 16:32:02 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 3). 2044 bytes result sent to driver -2016-06-13 16:32:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 3) in 16 ms on localhost (1/1) -2016-06-13 16:32:02 INFO DAGScheduler:59 - ResultStage 3 (foreach at MatrixRowLearning.scala:44) finished in 0.017 s -2016-06-13 16:32:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 16:32:02 INFO DAGScheduler:59 - Job 3 finished: foreach at MatrixRowLearning.scala:44, took 0.065248 s -2016-06-13 16:32:02 INFO MemoryStore:59 - ensureFreeSpace(230744) called with curMem=301980, maxMem=766075207 -2016-06-13 16:32:02 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-06-13 16:32:02 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=532724, maxMem=766075207 -2016-06-13 16:32:02 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-06-13 16:32:02 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:61106 (size: 19.3 KB, free: 730.5 MB) -2016-06-13 16:32:02 INFO SparkContext:59 - Created broadcast 6 from textFile at MatrixRowLearning.scala:48 -2016-06-13 16:32:02 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 16:32:02 INFO SparkContext:59 - Starting job: foreach at MatrixRowLearning.scala:54 -2016-06-13 16:32:02 INFO DAGScheduler:59 - Got job 4 (foreach at MatrixRowLearning.scala:54) with 1 output partitions -2016-06-13 16:32:02 INFO DAGScheduler:59 - Final stage: ResultStage 4(foreach at MatrixRowLearning.scala:54) -2016-06-13 16:32:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:32:02 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:32:02 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[13] at map at MatrixRowLearning.scala:52), which has no missing parents -2016-06-13 16:32:02 INFO MemoryStore:59 - ensureFreeSpace(3720) called with curMem=552512, maxMem=766075207 -2016-06-13 16:32:02 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.6 KB, free 730.1 MB) -2016-06-13 16:32:02 INFO MemoryStore:59 - ensureFreeSpace(2055) called with curMem=556232, maxMem=766075207 -2016-06-13 16:32:02 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.1 MB) -2016-06-13 16:32:02 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:61106 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 16:32:02 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:32:02 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (MapPartitionsRDD[13] at map at MatrixRowLearning.scala:52) -2016-06-13 16:32:02 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-06-13 16:32:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 4, localhost, PROCESS_LOCAL, 2182 bytes) -2016-06-13 16:32:02 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 4) -2016-06-13 16:32:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-06-13 16:32:02 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 4). 2044 bytes result sent to driver -2016-06-13 16:32:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 4) in 16 ms on localhost (1/1) -2016-06-13 16:32:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 16:32:02 INFO DAGScheduler:59 - ResultStage 4 (foreach at MatrixRowLearning.scala:54) finished in 0.017 s -2016-06-13 16:32:02 INFO DAGScheduler:59 - Job 4 finished: foreach at MatrixRowLearning.scala:54, took 0.037850 s -2016-06-13 16:32:02 INFO SparkContext:59 - Starting job: reduce at CoordinateMatrix.scala:153 -2016-06-13 16:32:02 INFO DAGScheduler:59 - Got job 5 (reduce at CoordinateMatrix.scala:153) with 1 output partitions -2016-06-13 16:32:02 INFO DAGScheduler:59 - Final stage: ResultStage 5(reduce at CoordinateMatrix.scala:153) -2016-06-13 16:32:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:32:02 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:32:02 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[14] at map at CoordinateMatrix.scala:153), which has no missing parents -2016-06-13 16:32:02 INFO MemoryStore:59 - ensureFreeSpace(3872) called with curMem=558287, maxMem=766075207 -2016-06-13 16:32:02 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 3.8 KB, free 730.1 MB) -2016-06-13 16:32:02 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=562159, maxMem=766075207 -2016-06-13 16:32:02 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.0 MB) -2016-06-13 16:32:02 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:61106 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 16:32:02 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:32:02 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 5 (MapPartitionsRDD[14] at map at CoordinateMatrix.scala:153) -2016-06-13 16:32:02 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 1 tasks -2016-06-13 16:32:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 5, localhost, PROCESS_LOCAL, 2182 bytes) -2016-06-13 16:32:02 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 5) -2016-06-13 16:32:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-06-13 16:32:02 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 5). 2209 bytes result sent to driver -2016-06-13 16:32:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 5) in 20 ms on localhost (1/1) -2016-06-13 16:32:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 16:32:02 INFO DAGScheduler:59 - ResultStage 5 (reduce at CoordinateMatrix.scala:153) finished in 0.021 s -2016-06-13 16:32:02 INFO DAGScheduler:59 - Job 5 finished: reduce at CoordinateMatrix.scala:153, took 0.042450 s -2016-06-13 16:32:02 INFO SparkContext:59 - Starting job: countApproxDistinct at MatrixRowLearning.scala:58 -2016-06-13 16:32:02 INFO DAGScheduler:59 - Got job 6 (countApproxDistinct at MatrixRowLearning.scala:58) with 1 output partitions -2016-06-13 16:32:02 INFO DAGScheduler:59 - Final stage: ResultStage 6(countApproxDistinct at MatrixRowLearning.scala:58) -2016-06-13 16:32:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:32:02 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:32:02 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[13] at map at MatrixRowLearning.scala:52), which has no missing parents -2016-06-13 16:32:02 INFO MemoryStore:59 - ensureFreeSpace(4880) called with curMem=564243, maxMem=766075207 -2016-06-13 16:32:02 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 4.8 KB, free 730.0 MB) -2016-06-13 16:32:02 INFO MemoryStore:59 - ensureFreeSpace(2486) called with curMem=569123, maxMem=766075207 -2016-06-13 16:32:02 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.0 MB) -2016-06-13 16:32:02 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:61106 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 16:32:02 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:32:02 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 6 (MapPartitionsRDD[13] at map at MatrixRowLearning.scala:52) -2016-06-13 16:32:02 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 1 tasks -2016-06-13 16:32:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 6, localhost, PROCESS_LOCAL, 2182 bytes) -2016-06-13 16:32:02 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 6) -2016-06-13 16:32:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow.txt:0+12 -2016-06-13 16:32:02 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 6). 2463 bytes result sent to driver -2016-06-13 16:32:02 INFO DAGScheduler:59 - ResultStage 6 (countApproxDistinct at MatrixRowLearning.scala:58) finished in 0.019 s -2016-06-13 16:32:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 6) in 18 ms on localhost (1/1) -2016-06-13 16:32:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 16:32:02 INFO DAGScheduler:59 - Job 6 finished: countApproxDistinct at MatrixRowLearning.scala:58, took 0.036117 s -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 16:32:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 16:32:02 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 16:32:02 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 16:32:02 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 16:32:03 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 16:32:03 INFO BlockManager:59 - BlockManager stopped -2016-06-13 16:32:03 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 16:32:03 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 16:32:03 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 16:32:03 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 16:32:03 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-d00e90e5-56dd-4896-ae71-63d8aaaf024d -2016-06-13 16:38:47 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 16:38:47 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 16:38:48 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 16:38:48 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 16:38:48 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 16:38:48 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 16:38:48 INFO Remoting:74 - Starting remoting -2016-06-13 16:38:49 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:61158] -2016-06-13 16:38:49 INFO Utils:59 - Successfully started service 'sparkDriver' on port 61158. -2016-06-13 16:38:49 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 16:38:49 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 16:38:49 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-e5fb8810-f6d7-497a-a25d-af1536866eca -2016-06-13 16:38:49 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 16:38:49 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-1e4456bf-0e9d-42f9-8e22-1178f6717b0f\httpd-b2d741e0-0ae4-4ba4-af8a-549e0faba47f -2016-06-13 16:38:49 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 16:38:49 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 16:38:49 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:61159 -2016-06-13 16:38:49 INFO Utils:59 - Successfully started service 'HTTP file server' on port 61159. -2016-06-13 16:38:49 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 16:38:49 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 16:38:49 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 16:38:49 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 16:38:49 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 16:38:49 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 16:38:49 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 16:38:49 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 61178. -2016-06-13 16:38:49 INFO NettyBlockTransferService:59 - Server created on 61178 -2016-06-13 16:38:49 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 16:38:49 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:61178 with 730.6 MB RAM, BlockManagerId(driver, localhost, 61178) -2016-06-13 16:38:49 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 16:38:50 INFO SparkContext:59 - Starting job: reduce at BlockMatrix.scala:189 -2016-06-13 16:38:50 INFO DAGScheduler:59 - Got job 0 (reduce at BlockMatrix.scala:189) with 3 output partitions -2016-06-13 16:38:50 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at BlockMatrix.scala:189) -2016-06-13 16:38:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:38:50 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:38:50 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at BlockMatrix.scala:186), which has no missing parents -2016-06-13 16:38:51 INFO MemoryStore:59 - ensureFreeSpace(2552) called with curMem=0, maxMem=766075207 -2016-06-13 16:38:51 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 2.5 KB, free 730.6 MB) -2016-06-13 16:38:51 INFO MemoryStore:59 - ensureFreeSpace(1464) called with curMem=2552, maxMem=766075207 -2016-06-13 16:38:51 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1464.0 B, free 730.6 MB) -2016-06-13 16:38:51 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:61178 (size: 1464.0 B, free: 730.6 MB) -2016-06-13 16:38:51 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:38:51 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at BlockMatrix.scala:186) -2016-06-13 16:38:51 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-06-13 16:38:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:38:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:38:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:38:51 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 16:38:51 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-06-13 16:38:51 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 16:38:51 INFO CacheManager:59 - Partition rdd_1_0 not found, computing it -2016-06-13 16:38:51 INFO CacheManager:59 - Partition rdd_1_2 not found, computing it -2016-06-13 16:38:51 INFO CacheManager:59 - Partition rdd_1_1 not found, computing it -2016-06-13 16:38:51 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=4016, maxMem=766075207 -2016-06-13 16:38:51 INFO MemoryStore:59 - Block rdd_1_0 stored as values in memory (estimated size 112.0 B, free 730.6 MB) -2016-06-13 16:38:51 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=4128, maxMem=766075207 -2016-06-13 16:38:51 INFO MemoryStore:59 - Block rdd_1_1 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:38:51 INFO BlockManagerInfo:59 - Added rdd_1_0 in memory on localhost:61178 (size: 112.0 B, free: 730.6 MB) -2016-06-13 16:38:51 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=4328, maxMem=766075207 -2016-06-13 16:38:51 INFO MemoryStore:59 - Block rdd_1_2 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:38:51 INFO BlockManagerInfo:59 - Added rdd_1_1 in memory on localhost:61178 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:38:51 INFO BlockManagerInfo:59 - Added rdd_1_2 in memory on localhost:61178 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:38:51 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1694 bytes result sent to driver -2016-06-13 16:38:51 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1694 bytes result sent to driver -2016-06-13 16:38:51 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 1694 bytes result sent to driver -2016-06-13 16:38:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 93 ms on localhost (1/3) -2016-06-13 16:38:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 72 ms on localhost (2/3) -2016-06-13 16:38:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 74 ms on localhost (3/3) -2016-06-13 16:38:51 INFO DAGScheduler:59 - ResultStage 0 (reduce at BlockMatrix.scala:189) finished in 0.107 s -2016-06-13 16:38:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 16:38:51 INFO DAGScheduler:59 - Job 0 finished: reduce at BlockMatrix.scala:189, took 0.442005 s -2016-06-13 16:38:51 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:61178 in memory (size: 1464.0 B, free: 730.6 MB) -2016-06-13 16:38:51 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 16:38:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 16:38:51 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 16:38:51 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 16:38:51 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 16:38:51 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 16:38:51 INFO BlockManager:59 - BlockManager stopped -2016-06-13 16:38:51 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 16:38:51 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 16:38:51 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 16:38:51 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 16:38:51 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 16:38:51 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 16:38:51 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-1e4456bf-0e9d-42f9-8e22-1178f6717b0f -2016-06-13 16:40:29 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 16:40:30 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 16:40:30 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 16:40:30 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 16:40:30 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 16:40:31 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 16:40:31 INFO Remoting:74 - Starting remoting -2016-06-13 16:40:31 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:61196] -2016-06-13 16:40:31 INFO Utils:59 - Successfully started service 'sparkDriver' on port 61196. -2016-06-13 16:40:31 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 16:40:31 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 16:40:31 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-39986dbe-1291-477e-8594-a2165035c0f2 -2016-06-13 16:40:31 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 16:40:31 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-391667d1-0d63-4851-bc91-c228cdc088d7\httpd-a6a3e5ae-909f-4c22-bf12-71cd94f8b7fa -2016-06-13 16:40:31 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 16:40:31 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 16:40:31 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:61197 -2016-06-13 16:40:31 INFO Utils:59 - Successfully started service 'HTTP file server' on port 61197. -2016-06-13 16:40:31 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 16:40:31 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 16:40:31 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 16:40:31 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 16:40:31 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 16:40:32 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 16:40:32 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 16:40:32 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 61216. -2016-06-13 16:40:32 INFO NettyBlockTransferService:59 - Server created on 61216 -2016-06-13 16:40:32 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 16:40:32 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:61216 with 730.6 MB RAM, BlockManagerId(driver, localhost, 61216) -2016-06-13 16:40:32 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 16:40:32 INFO SparkContext:59 - Starting job: reduce at BlockMatrix.scala:189 -2016-06-13 16:40:33 INFO DAGScheduler:59 - Got job 0 (reduce at BlockMatrix.scala:189) with 3 output partitions -2016-06-13 16:40:33 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at BlockMatrix.scala:189) -2016-06-13 16:40:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:40:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:40:33 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at BlockMatrix.scala:186), which has no missing parents -2016-06-13 16:40:33 INFO MemoryStore:59 - ensureFreeSpace(2552) called with curMem=0, maxMem=766075207 -2016-06-13 16:40:33 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 2.5 KB, free 730.6 MB) -2016-06-13 16:40:33 INFO MemoryStore:59 - ensureFreeSpace(1464) called with curMem=2552, maxMem=766075207 -2016-06-13 16:40:33 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1464.0 B, free 730.6 MB) -2016-06-13 16:40:33 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:61216 (size: 1464.0 B, free: 730.6 MB) -2016-06-13 16:40:33 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:40:33 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at BlockMatrix.scala:186) -2016-06-13 16:40:33 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-06-13 16:40:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:40:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:40:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:40:33 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-06-13 16:40:33 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 16:40:33 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 16:40:33 INFO CacheManager:59 - Partition rdd_1_0 not found, computing it -2016-06-13 16:40:33 INFO CacheManager:59 - Partition rdd_1_1 not found, computing it -2016-06-13 16:40:33 INFO CacheManager:59 - Partition rdd_1_2 not found, computing it -2016-06-13 16:40:33 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=4016, maxMem=766075207 -2016-06-13 16:40:33 INFO MemoryStore:59 - Block rdd_1_0 stored as values in memory (estimated size 112.0 B, free 730.6 MB) -2016-06-13 16:40:33 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=4128, maxMem=766075207 -2016-06-13 16:40:33 INFO BlockManagerInfo:59 - Added rdd_1_0 in memory on localhost:61216 (size: 112.0 B, free: 730.6 MB) -2016-06-13 16:40:33 INFO MemoryStore:59 - Block rdd_1_1 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:40:33 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=4328, maxMem=766075207 -2016-06-13 16:40:33 INFO MemoryStore:59 - Block rdd_1_2 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:40:33 INFO BlockManagerInfo:59 - Added rdd_1_1 in memory on localhost:61216 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:40:33 INFO BlockManagerInfo:59 - Added rdd_1_2 in memory on localhost:61216 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:40:33 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1694 bytes result sent to driver -2016-06-13 16:40:33 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1694 bytes result sent to driver -2016-06-13 16:40:33 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 1694 bytes result sent to driver -2016-06-13 16:40:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 69 ms on localhost (1/3) -2016-06-13 16:40:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 74 ms on localhost (2/3) -2016-06-13 16:40:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 102 ms on localhost (3/3) -2016-06-13 16:40:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 16:40:33 INFO DAGScheduler:59 - ResultStage 0 (reduce at BlockMatrix.scala:189) finished in 0.113 s -2016-06-13 16:40:33 INFO DAGScheduler:59 - Job 0 finished: reduce at BlockMatrix.scala:189, took 0.366456 s -2016-06-13 16:40:33 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:61216 in memory (size: 1464.0 B, free: 730.6 MB) -2016-06-13 16:40:33 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 16:40:33 INFO SparkContext:59 - Starting job: foreach at BlockMatrixSuite.scala:58 -2016-06-13 16:40:33 INFO DAGScheduler:59 - Got job 1 (foreach at BlockMatrixSuite.scala:58) with 3 output partitions -2016-06-13 16:40:33 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at BlockMatrixSuite.scala:58) -2016-06-13 16:40:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:40:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:40:33 INFO DAGScheduler:59 - Submitting ResultStage 1 (ParallelCollectionRDD[0] at parallelize at BlockMatrixSuite.scala:46), which has no missing parents -2016-06-13 16:40:33 INFO MemoryStore:59 - ensureFreeSpace(1176) called with curMem=512, maxMem=766075207 -2016-06-13 16:40:33 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1176.0 B, free 730.6 MB) -2016-06-13 16:40:33 INFO MemoryStore:59 - ensureFreeSpace(827) called with curMem=1688, maxMem=766075207 -2016-06-13 16:40:33 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 827.0 B, free 730.6 MB) -2016-06-13 16:40:33 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:61216 (size: 827.0 B, free: 730.6 MB) -2016-06-13 16:40:33 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:40:33 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 1 (ParallelCollectionRDD[0] at parallelize at BlockMatrixSuite.scala:46) -2016-06-13 16:40:33 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 3 tasks -2016-06-13 16:40:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:40:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:40:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 5, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:40:33 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 4) -2016-06-13 16:40:33 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 5) -2016-06-13 16:40:33 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 3) -2016-06-13 16:40:33 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 5). 915 bytes result sent to driver -2016-06-13 16:40:33 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 3). 915 bytes result sent to driver -2016-06-13 16:40:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 3) in 267 ms on localhost (1/3) -2016-06-13 16:40:33 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 4). 915 bytes result sent to driver -2016-06-13 16:40:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 4) in 278 ms on localhost (2/3) -2016-06-13 16:40:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 5) in 280 ms on localhost (3/3) -2016-06-13 16:40:33 INFO DAGScheduler:59 - ResultStage 1 (foreach at BlockMatrixSuite.scala:58) finished in 0.283 s -2016-06-13 16:40:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 16:40:33 INFO DAGScheduler:59 - Job 1 finished: foreach at BlockMatrixSuite.scala:58, took 0.295457 s -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 16:40:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 16:40:33 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 16:40:33 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 16:40:33 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 16:40:33 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 16:40:33 INFO BlockManager:59 - BlockManager stopped -2016-06-13 16:40:33 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 16:40:33 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 16:40:33 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 16:40:33 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 16:40:33 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 16:40:33 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-391667d1-0d63-4851-bc91-c228cdc088d7 -2016-06-13 16:40:33 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 16:43:00 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 16:43:01 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 16:43:01 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 16:43:01 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 16:43:01 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 16:43:02 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 16:43:02 INFO Remoting:74 - Starting remoting -2016-06-13 16:43:02 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:61241] -2016-06-13 16:43:02 INFO Utils:59 - Successfully started service 'sparkDriver' on port 61241. -2016-06-13 16:43:02 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 16:43:02 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 16:43:02 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-9fea0c30-84b5-4f71-ab13-2b5e48611594 -2016-06-13 16:43:02 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 16:43:02 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-28ad8b09-d087-4f26-b9fa-9564d612ea71\httpd-ed396e01-cf0e-47e1-b9a6-c7a16baee69c -2016-06-13 16:43:02 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 16:43:02 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 16:43:02 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:61242 -2016-06-13 16:43:02 INFO Utils:59 - Successfully started service 'HTTP file server' on port 61242. -2016-06-13 16:43:02 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 16:43:03 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 16:43:03 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 16:43:03 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 16:43:03 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 16:43:03 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 16:43:03 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 16:43:03 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 61261. -2016-06-13 16:43:03 INFO NettyBlockTransferService:59 - Server created on 61261 -2016-06-13 16:43:03 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 16:43:03 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:61261 with 730.6 MB RAM, BlockManagerId(driver, localhost, 61261) -2016-06-13 16:43:03 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 16:43:04 INFO SparkContext:59 - Starting job: reduce at BlockMatrix.scala:189 -2016-06-13 16:43:04 INFO DAGScheduler:59 - Got job 0 (reduce at BlockMatrix.scala:189) with 3 output partitions -2016-06-13 16:43:04 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at BlockMatrix.scala:189) -2016-06-13 16:43:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:43:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:43:04 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at BlockMatrix.scala:186), which has no missing parents -2016-06-13 16:43:04 INFO MemoryStore:59 - ensureFreeSpace(2552) called with curMem=0, maxMem=766075207 -2016-06-13 16:43:04 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 2.5 KB, free 730.6 MB) -2016-06-13 16:43:04 INFO MemoryStore:59 - ensureFreeSpace(1464) called with curMem=2552, maxMem=766075207 -2016-06-13 16:43:04 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1464.0 B, free 730.6 MB) -2016-06-13 16:43:04 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:61261 (size: 1464.0 B, free: 730.6 MB) -2016-06-13 16:43:04 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:43:04 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at BlockMatrix.scala:186) -2016-06-13 16:43:04 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-06-13 16:43:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:43:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:43:04 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:43:04 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 16:43:04 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 16:43:04 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-06-13 16:43:04 INFO CacheManager:59 - Partition rdd_1_1 not found, computing it -2016-06-13 16:43:04 INFO CacheManager:59 - Partition rdd_1_0 not found, computing it -2016-06-13 16:43:04 INFO CacheManager:59 - Partition rdd_1_2 not found, computing it -2016-06-13 16:43:04 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=4016, maxMem=766075207 -2016-06-13 16:43:04 INFO MemoryStore:59 - Block rdd_1_0 stored as values in memory (estimated size 112.0 B, free 730.6 MB) -2016-06-13 16:43:04 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=4128, maxMem=766075207 -2016-06-13 16:43:04 INFO MemoryStore:59 - Block rdd_1_2 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:43:04 INFO BlockManagerInfo:59 - Added rdd_1_0 in memory on localhost:61261 (size: 112.0 B, free: 730.6 MB) -2016-06-13 16:43:04 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=4328, maxMem=766075207 -2016-06-13 16:43:04 INFO MemoryStore:59 - Block rdd_1_1 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:43:04 INFO BlockManagerInfo:59 - Added rdd_1_2 in memory on localhost:61261 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:43:04 INFO BlockManagerInfo:59 - Added rdd_1_1 in memory on localhost:61261 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:43:04 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 1694 bytes result sent to driver -2016-06-13 16:43:04 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1694 bytes result sent to driver -2016-06-13 16:43:04 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1694 bytes result sent to driver -2016-06-13 16:43:04 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 105 ms on localhost (1/3) -2016-06-13 16:43:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 112 ms on localhost (2/3) -2016-06-13 16:43:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 145 ms on localhost (3/3) -2016-06-13 16:43:04 INFO DAGScheduler:59 - ResultStage 0 (reduce at BlockMatrix.scala:189) finished in 0.155 s -2016-06-13 16:43:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 16:43:04 INFO DAGScheduler:59 - Job 0 finished: reduce at BlockMatrix.scala:189, took 0.379363 s -2016-06-13 16:43:04 INFO SparkContext:59 - Starting job: foreach at BlockMatrixSuite.scala:58 -2016-06-13 16:43:04 INFO DAGScheduler:59 - Got job 1 (foreach at BlockMatrixSuite.scala:58) with 3 output partitions -2016-06-13 16:43:04 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at BlockMatrixSuite.scala:58) -2016-06-13 16:43:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:43:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:43:04 INFO DAGScheduler:59 - Submitting ResultStage 1 (ParallelCollectionRDD[0] at parallelize at BlockMatrixSuite.scala:46), which has no missing parents -2016-06-13 16:43:04 INFO MemoryStore:59 - ensureFreeSpace(1176) called with curMem=4528, maxMem=766075207 -2016-06-13 16:43:04 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1176.0 B, free 730.6 MB) -2016-06-13 16:43:04 INFO MemoryStore:59 - ensureFreeSpace(827) called with curMem=5704, maxMem=766075207 -2016-06-13 16:43:04 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 827.0 B, free 730.6 MB) -2016-06-13 16:43:04 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:61261 (size: 827.0 B, free: 730.6 MB) -2016-06-13 16:43:04 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:43:04 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 1 (ParallelCollectionRDD[0] at parallelize at BlockMatrixSuite.scala:46) -2016-06-13 16:43:04 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 3 tasks -2016-06-13 16:43:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:43:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:43:04 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 5, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:43:04 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 3) -2016-06-13 16:43:04 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 5) -2016-06-13 16:43:04 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 4) -2016-06-13 16:43:05 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 3). 915 bytes result sent to driver -2016-06-13 16:43:05 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 4). 915 bytes result sent to driver -2016-06-13 16:43:05 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 5). 915 bytes result sent to driver -2016-06-13 16:43:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 4) in 385 ms on localhost (1/3) -2016-06-13 16:43:05 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 5) in 385 ms on localhost (2/3) -2016-06-13 16:43:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 3) in 389 ms on localhost (3/3) -2016-06-13 16:43:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 16:43:05 INFO DAGScheduler:59 - ResultStage 1 (foreach at BlockMatrixSuite.scala:58) finished in 0.390 s -2016-06-13 16:43:05 INFO DAGScheduler:59 - Job 1 finished: foreach at BlockMatrixSuite.scala:58, took 0.415677 s -2016-06-13 16:43:05 INFO SparkContext:59 - Starting job: foreach at BlockMatrixSuite.scala:65 -2016-06-13 16:43:05 INFO DAGScheduler:59 - Registering RDD 5 (flatMap at BlockMatrix.scala:377) -2016-06-13 16:43:05 INFO DAGScheduler:59 - Registering RDD 4 (flatMap at BlockMatrix.scala:373) -2016-06-13 16:43:05 INFO DAGScheduler:59 - Registering RDD 8 (flatMap at BlockMatrix.scala:381) -2016-06-13 16:43:05 INFO DAGScheduler:59 - Got job 2 (foreach at BlockMatrixSuite.scala:65) with 4 output partitions -2016-06-13 16:43:05 INFO DAGScheduler:59 - Final stage: ResultStage 5(foreach at BlockMatrixSuite.scala:65) -2016-06-13 16:43:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-06-13 16:43:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-06-13 16:43:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[5] at flatMap at BlockMatrix.scala:377), which has no missing parents -2016-06-13 16:43:05 INFO MemoryStore:59 - ensureFreeSpace(2944) called with curMem=6531, maxMem=766075207 -2016-06-13 16:43:05 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-06-13 16:43:05 INFO MemoryStore:59 - ensureFreeSpace(1704) called with curMem=9475, maxMem=766075207 -2016-06-13 16:43:05 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1704.0 B, free 730.6 MB) -2016-06-13 16:43:05 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:61261 (size: 1704.0 B, free: 730.6 MB) -2016-06-13 16:43:05 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:43:05 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[5] at flatMap at BlockMatrix.scala:377) -2016-06-13 16:43:05 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-06-13 16:43:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (MapPartitionsRDD[4] at flatMap at BlockMatrix.scala:373), which has no missing parents -2016-06-13 16:43:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 16:43:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2446 bytes) -2016-06-13 16:43:05 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=11179, maxMem=766075207 -2016-06-13 16:43:05 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.1 KB, free 730.6 MB) -2016-06-13 16:43:05 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 8, localhost, PROCESS_LOCAL, 2430 bytes) -2016-06-13 16:43:05 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 6) -2016-06-13 16:43:05 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 7) -2016-06-13 16:43:05 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 8) -2016-06-13 16:43:05 INFO MemoryStore:59 - ensureFreeSpace(1820) called with curMem=14395, maxMem=766075207 -2016-06-13 16:43:05 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1820.0 B, free 730.6 MB) -2016-06-13 16:43:05 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:61261 (size: 1820.0 B, free: 730.6 MB) -2016-06-13 16:43:05 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:43:05 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[4] at flatMap at BlockMatrix.scala:373) -2016-06-13 16:43:05 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 3 tasks -2016-06-13 16:43:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 16:43:05 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 9) -2016-06-13 16:43:05 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 6). 1161 bytes result sent to driver -2016-06-13 16:43:05 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 7). 1161 bytes result sent to driver -2016-06-13 16:43:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2446 bytes) -2016-06-13 16:43:05 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 11, localhost, PROCESS_LOCAL, 2430 bytes) -2016-06-13 16:43:05 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 11) -2016-06-13 16:43:05 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 9). 1161 bytes result sent to driver -2016-06-13 16:43:05 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 8). 1161 bytes result sent to driver -2016-06-13 16:43:05 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 10) -2016-06-13 16:43:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 7) in 217 ms on localhost (1/3) -2016-06-13 16:43:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 9) in 204 ms on localhost (1/3) -2016-06-13 16:43:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 6) in 223 ms on localhost (2/3) -2016-06-13 16:43:05 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 8) in 220 ms on localhost (3/3) -2016-06-13 16:43:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 16:43:05 INFO DAGScheduler:59 - ShuffleMapStage 2 (flatMap at BlockMatrix.scala:377) finished in 0.226 s -2016-06-13 16:43:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:43:05 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 3) -2016-06-13 16:43:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 5, ShuffleMapStage 4) -2016-06-13 16:43:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:43:05 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List(ShuffleMapStage 4) -2016-06-13 16:43:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 4: List(ShuffleMapStage 3) -2016-06-13 16:43:05 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 11). 1161 bytes result sent to driver -2016-06-13 16:43:05 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 11) in 56 ms on localhost (2/3) -2016-06-13 16:43:05 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 10). 1161 bytes result sent to driver -2016-06-13 16:43:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 10) in 61 ms on localhost (3/3) -2016-06-13 16:43:05 INFO DAGScheduler:59 - ShuffleMapStage 3 (flatMap at BlockMatrix.scala:373) finished in 0.250 s -2016-06-13 16:43:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 16:43:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:43:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 16:43:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 5, ShuffleMapStage 4) -2016-06-13 16:43:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:43:05 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List(ShuffleMapStage 4) -2016-06-13 16:43:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 4: List() -2016-06-13 16:43:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[8] at flatMap at BlockMatrix.scala:381), which is now runnable -2016-06-13 16:43:05 INFO MemoryStore:59 - ensureFreeSpace(2952) called with curMem=16215, maxMem=766075207 -2016-06-13 16:43:05 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-06-13 16:43:05 INFO MemoryStore:59 - ensureFreeSpace(1712) called with curMem=19167, maxMem=766075207 -2016-06-13 16:43:05 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 1712.0 B, free 730.6 MB) -2016-06-13 16:43:05 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:61261 (size: 1712.0 B, free: 730.6 MB) -2016-06-13 16:43:05 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:43:05 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[8] at flatMap at BlockMatrix.scala:381) -2016-06-13 16:43:05 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 4 tasks -2016-06-13 16:43:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 12, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:43:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:43:05 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 14, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:43:05 INFO TaskSetManager:59 - Starting task 3.0 in stage 4.0 (TID 15, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:43:05 INFO Executor:59 - Running task 3.0 in stage 4.0 (TID 15) -2016-06-13 16:43:05 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 13) -2016-06-13 16:43:05 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 14) -2016-06-13 16:43:05 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 12) -2016-06-13 16:43:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:43:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:43:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:43:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:43:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 14 ms -2016-06-13 16:43:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 12 ms -2016-06-13 16:43:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 9 ms -2016-06-13 16:43:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-06-13 16:43:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:43:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:43:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:43:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:43:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:43:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:43:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:43:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:43:05 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 16:43:05 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 16:43:06 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 12). 1377 bytes result sent to driver -2016-06-13 16:43:06 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 14). 1377 bytes result sent to driver -2016-06-13 16:43:06 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 13). 1377 bytes result sent to driver -2016-06-13 16:43:06 INFO Executor:59 - Finished task 3.0 in stage 4.0 (TID 15). 1377 bytes result sent to driver -2016-06-13 16:43:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 12) in 283 ms on localhost (1/4) -2016-06-13 16:43:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 13) in 281 ms on localhost (2/4) -2016-06-13 16:43:06 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 14) in 279 ms on localhost (3/4) -2016-06-13 16:43:06 INFO TaskSetManager:59 - Finished task 3.0 in stage 4.0 (TID 15) in 279 ms on localhost (4/4) -2016-06-13 16:43:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 16:43:06 INFO DAGScheduler:59 - ShuffleMapStage 4 (flatMap at BlockMatrix.scala:381) finished in 0.286 s -2016-06-13 16:43:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:43:06 INFO DAGScheduler:59 - running: Set() -2016-06-13 16:43:06 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-06-13 16:43:06 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:43:06 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-06-13 16:43:06 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[10] at mapValues at BlockMatrix.scala:397), which is now runnable -2016-06-13 16:43:06 INFO MemoryStore:59 - ensureFreeSpace(2888) called with curMem=20879, maxMem=766075207 -2016-06-13 16:43:06 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 2.8 KB, free 730.6 MB) -2016-06-13 16:43:06 INFO MemoryStore:59 - ensureFreeSpace(1718) called with curMem=23767, maxMem=766075207 -2016-06-13 16:43:06 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 1718.0 B, free 730.6 MB) -2016-06-13 16:43:06 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:61261 (size: 1718.0 B, free: 730.6 MB) -2016-06-13 16:43:06 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:43:06 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 5 (MapPartitionsRDD[10] at mapValues at BlockMatrix.scala:397) -2016-06-13 16:43:06 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 4 tasks -2016-06-13 16:43:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:43:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:43:06 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 18, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:43:06 INFO TaskSetManager:59 - Starting task 3.0 in stage 5.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:43:06 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 16) -2016-06-13 16:43:06 INFO Executor:59 - Running task 3.0 in stage 5.0 (TID 19) -2016-06-13 16:43:06 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 17) -2016-06-13 16:43:06 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 18) -2016-06-13 16:43:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 4 blocks -2016-06-13 16:43:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:43:06 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 16). 1165 bytes result sent to driver -2016-06-13 16:43:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 4 blocks -2016-06-13 16:43:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:43:06 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 17). 1165 bytes result sent to driver -2016-06-13 16:43:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 4 blocks -2016-06-13 16:43:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:43:06 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 4 blocks -2016-06-13 16:43:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:43:06 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 18). 1165 bytes result sent to driver -2016-06-13 16:43:06 INFO Executor:59 - Finished task 3.0 in stage 5.0 (TID 19). 1165 bytes result sent to driver -2016-06-13 16:43:06 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 18) in 33 ms on localhost (1/4) -2016-06-13 16:43:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 16) in 38 ms on localhost (2/4) -2016-06-13 16:43:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 17) in 39 ms on localhost (3/4) -2016-06-13 16:43:06 INFO TaskSetManager:59 - Finished task 3.0 in stage 5.0 (TID 19) in 37 ms on localhost (4/4) -2016-06-13 16:43:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 16:43:06 INFO DAGScheduler:59 - ResultStage 5 (foreach at BlockMatrixSuite.scala:65) finished in 0.042 s -2016-06-13 16:43:06 INFO DAGScheduler:59 - Job 2 finished: foreach at BlockMatrixSuite.scala:65, took 0.695525 s -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 16:43:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 16:43:06 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 16:43:06 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 16:43:06 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 16:43:06 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 16:43:06 INFO BlockManager:59 - BlockManager stopped -2016-06-13 16:43:06 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 16:43:06 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 16:43:06 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 16:43:06 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 16:43:06 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 16:43:06 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 16:43:06 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-28ad8b09-d087-4f26-b9fa-9564d612ea71 -2016-06-13 16:46:35 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 16:46:36 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 16:46:36 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 16:46:36 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 16:46:36 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 16:46:37 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 16:46:37 INFO Remoting:74 - Starting remoting -2016-06-13 16:46:37 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:61290] -2016-06-13 16:46:37 INFO Utils:59 - Successfully started service 'sparkDriver' on port 61290. -2016-06-13 16:46:37 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 16:46:37 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 16:46:37 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-88903d15-f0db-40f2-823d-28d5be282327 -2016-06-13 16:46:37 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 16:46:37 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-8435e46a-6768-4fdb-8e01-0b93ff9288af\httpd-58f31b91-368e-4fcc-b7e2-fe737acf1999 -2016-06-13 16:46:37 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 16:46:37 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 16:46:37 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:61291 -2016-06-13 16:46:37 INFO Utils:59 - Successfully started service 'HTTP file server' on port 61291. -2016-06-13 16:46:37 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 16:46:37 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 16:46:37 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 16:46:37 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 16:46:37 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 16:46:37 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 16:46:37 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 16:46:38 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 61310. -2016-06-13 16:46:38 INFO NettyBlockTransferService:59 - Server created on 61310 -2016-06-13 16:46:38 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 16:46:38 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:61310 with 730.6 MB RAM, BlockManagerId(driver, localhost, 61310) -2016-06-13 16:46:38 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 16:46:38 INFO SparkContext:59 - Starting job: reduce at BlockMatrix.scala:189 -2016-06-13 16:46:38 INFO DAGScheduler:59 - Got job 0 (reduce at BlockMatrix.scala:189) with 3 output partitions -2016-06-13 16:46:38 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at BlockMatrix.scala:189) -2016-06-13 16:46:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:46:38 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:46:38 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at BlockMatrix.scala:186), which has no missing parents -2016-06-13 16:46:39 INFO MemoryStore:59 - ensureFreeSpace(2552) called with curMem=0, maxMem=766075207 -2016-06-13 16:46:39 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 2.5 KB, free 730.6 MB) -2016-06-13 16:46:39 INFO MemoryStore:59 - ensureFreeSpace(1464) called with curMem=2552, maxMem=766075207 -2016-06-13 16:46:39 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1464.0 B, free 730.6 MB) -2016-06-13 16:46:39 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:61310 (size: 1464.0 B, free: 730.6 MB) -2016-06-13 16:46:39 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:46:39 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at BlockMatrix.scala:186) -2016-06-13 16:46:39 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-06-13 16:46:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:46:39 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 16:46:39 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 16:46:39 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-06-13 16:46:39 INFO CacheManager:59 - Partition rdd_1_2 not found, computing it -2016-06-13 16:46:39 INFO CacheManager:59 - Partition rdd_1_0 not found, computing it -2016-06-13 16:46:39 INFO CacheManager:59 - Partition rdd_1_1 not found, computing it -2016-06-13 16:46:39 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=4016, maxMem=766075207 -2016-06-13 16:46:39 INFO MemoryStore:59 - Block rdd_1_2 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:46:39 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=4216, maxMem=766075207 -2016-06-13 16:46:39 INFO MemoryStore:59 - Block rdd_1_0 stored as values in memory (estimated size 112.0 B, free 730.6 MB) -2016-06-13 16:46:39 INFO BlockManagerInfo:59 - Added rdd_1_2 in memory on localhost:61310 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:46:39 INFO MemoryStore:59 - ensureFreeSpace(200) called with curMem=4328, maxMem=766075207 -2016-06-13 16:46:39 INFO BlockManagerInfo:59 - Added rdd_1_0 in memory on localhost:61310 (size: 112.0 B, free: 730.6 MB) -2016-06-13 16:46:39 INFO MemoryStore:59 - Block rdd_1_1 stored as values in memory (estimated size 200.0 B, free 730.6 MB) -2016-06-13 16:46:39 INFO BlockManagerInfo:59 - Added rdd_1_1 in memory on localhost:61310 (size: 200.0 B, free: 730.6 MB) -2016-06-13 16:46:39 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1694 bytes result sent to driver -2016-06-13 16:46:39 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1694 bytes result sent to driver -2016-06-13 16:46:39 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 1694 bytes result sent to driver -2016-06-13 16:46:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 145 ms on localhost (1/3) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 131 ms on localhost (2/3) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 134 ms on localhost (3/3) -2016-06-13 16:46:39 INFO DAGScheduler:59 - ResultStage 0 (reduce at BlockMatrix.scala:189) finished in 0.168 s -2016-06-13 16:46:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 16:46:39 INFO DAGScheduler:59 - Job 0 finished: reduce at BlockMatrix.scala:189, took 0.369238 s -2016-06-13 16:46:39 INFO SparkContext:59 - Starting job: foreach at BlockMatrixSuite.scala:58 -2016-06-13 16:46:39 INFO DAGScheduler:59 - Got job 1 (foreach at BlockMatrixSuite.scala:58) with 3 output partitions -2016-06-13 16:46:39 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at BlockMatrixSuite.scala:58) -2016-06-13 16:46:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:46:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:46:39 INFO DAGScheduler:59 - Submitting ResultStage 1 (ParallelCollectionRDD[0] at parallelize at BlockMatrixSuite.scala:46), which has no missing parents -2016-06-13 16:46:39 INFO MemoryStore:59 - ensureFreeSpace(1176) called with curMem=4528, maxMem=766075207 -2016-06-13 16:46:39 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1176.0 B, free 730.6 MB) -2016-06-13 16:46:39 INFO MemoryStore:59 - ensureFreeSpace(827) called with curMem=5704, maxMem=766075207 -2016-06-13 16:46:39 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 827.0 B, free 730.6 MB) -2016-06-13 16:46:39 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:61310 (size: 827.0 B, free: 730.6 MB) -2016-06-13 16:46:39 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:46:39 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 1 (ParallelCollectionRDD[0] at parallelize at BlockMatrixSuite.scala:46) -2016-06-13 16:46:39 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 3 tasks -2016-06-13 16:46:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 5, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:46:39 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 4) -2016-06-13 16:46:39 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 5) -2016-06-13 16:46:39 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 3) -2016-06-13 16:46:39 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 5). 915 bytes result sent to driver -2016-06-13 16:46:39 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 4). 915 bytes result sent to driver -2016-06-13 16:46:39 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 3). 915 bytes result sent to driver -2016-06-13 16:46:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 4) in 256 ms on localhost (1/3) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 5) in 255 ms on localhost (2/3) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 3) in 258 ms on localhost (3/3) -2016-06-13 16:46:39 INFO DAGScheduler:59 - ResultStage 1 (foreach at BlockMatrixSuite.scala:58) finished in 0.259 s -2016-06-13 16:46:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 16:46:39 INFO DAGScheduler:59 - Job 1 finished: foreach at BlockMatrixSuite.scala:58, took 0.271256 s -2016-06-13 16:46:39 INFO SparkContext:59 - Starting job: foreach at BlockMatrixSuite.scala:65 -2016-06-13 16:46:39 INFO DAGScheduler:59 - Got job 2 (foreach at BlockMatrixSuite.scala:65) with 3 output partitions -2016-06-13 16:46:39 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at BlockMatrixSuite.scala:65) -2016-06-13 16:46:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 16:46:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 16:46:39 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[3] at map at BlockMatrix.scala:308), which has no missing parents -2016-06-13 16:46:39 INFO MemoryStore:59 - ensureFreeSpace(1888) called with curMem=6531, maxMem=766075207 -2016-06-13 16:46:39 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 1888.0 B, free 730.6 MB) -2016-06-13 16:46:39 INFO MemoryStore:59 - ensureFreeSpace(1218) called with curMem=8419, maxMem=766075207 -2016-06-13 16:46:39 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1218.0 B, free 730.6 MB) -2016-06-13 16:46:39 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:61310 (size: 1218.0 B, free: 730.6 MB) -2016-06-13 16:46:39 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:46:39 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 2 (MapPartitionsRDD[3] at map at BlockMatrix.scala:308) -2016-06-13 16:46:39 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-06-13 16:46:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2457 bytes) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 8, localhost, PROCESS_LOCAL, 2441 bytes) -2016-06-13 16:46:39 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 6) -2016-06-13 16:46:39 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 8) -2016-06-13 16:46:39 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 7) -2016-06-13 16:46:39 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 6). 915 bytes result sent to driver -2016-06-13 16:46:39 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 7). 915 bytes result sent to driver -2016-06-13 16:46:39 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 8). 915 bytes result sent to driver -2016-06-13 16:46:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 7) in 18 ms on localhost (1/3) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 6) in 20 ms on localhost (2/3) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 8) in 19 ms on localhost (3/3) -2016-06-13 16:46:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 16:46:39 INFO DAGScheduler:59 - ResultStage 2 (foreach at BlockMatrixSuite.scala:65) finished in 0.023 s -2016-06-13 16:46:39 INFO DAGScheduler:59 - Job 2 finished: foreach at BlockMatrixSuite.scala:65, took 0.036910 s -2016-06-13 16:46:39 INFO SparkContext:59 - Starting job: foreach at BlockMatrixSuite.scala:72 -2016-06-13 16:46:39 INFO DAGScheduler:59 - Registering RDD 6 (flatMap at BlockMatrix.scala:377) -2016-06-13 16:46:39 INFO DAGScheduler:59 - Registering RDD 5 (flatMap at BlockMatrix.scala:373) -2016-06-13 16:46:39 INFO DAGScheduler:59 - Registering RDD 9 (flatMap at BlockMatrix.scala:381) -2016-06-13 16:46:39 INFO DAGScheduler:59 - Got job 3 (foreach at BlockMatrixSuite.scala:72) with 4 output partitions -2016-06-13 16:46:39 INFO DAGScheduler:59 - Final stage: ResultStage 6(foreach at BlockMatrixSuite.scala:72) -2016-06-13 16:46:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 5) -2016-06-13 16:46:39 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 5) -2016-06-13 16:46:39 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (MapPartitionsRDD[6] at flatMap at BlockMatrix.scala:377), which has no missing parents -2016-06-13 16:46:39 INFO MemoryStore:59 - ensureFreeSpace(2944) called with curMem=9637, maxMem=766075207 -2016-06-13 16:46:39 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-06-13 16:46:39 INFO MemoryStore:59 - ensureFreeSpace(1705) called with curMem=12581, maxMem=766075207 -2016-06-13 16:46:39 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1705.0 B, free 730.6 MB) -2016-06-13 16:46:39 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:61310 (size: 1705.0 B, free: 730.6 MB) -2016-06-13 16:46:39 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:46:39 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[6] at flatMap at BlockMatrix.scala:377) -2016-06-13 16:46:39 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 3 tasks -2016-06-13 16:46:39 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[5] at flatMap at BlockMatrix.scala:373), which has no missing parents -2016-06-13 16:46:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2446 bytes) -2016-06-13 16:46:39 INFO MemoryStore:59 - ensureFreeSpace(3216) called with curMem=14286, maxMem=766075207 -2016-06-13 16:46:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 11, localhost, PROCESS_LOCAL, 2430 bytes) -2016-06-13 16:46:39 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.1 KB, free 730.6 MB) -2016-06-13 16:46:39 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 11) -2016-06-13 16:46:39 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 10) -2016-06-13 16:46:39 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 9) -2016-06-13 16:46:39 INFO MemoryStore:59 - ensureFreeSpace(1821) called with curMem=17502, maxMem=766075207 -2016-06-13 16:46:39 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 1821.0 B, free 730.6 MB) -2016-06-13 16:46:39 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:61310 (size: 1821.0 B, free: 730.6 MB) -2016-06-13 16:46:39 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:46:39 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[5] at flatMap at BlockMatrix.scala:373) -2016-06-13 16:46:39 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 3 tasks -2016-06-13 16:46:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 12, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 16:46:39 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 12) -2016-06-13 16:46:39 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 10). 1161 bytes result sent to driver -2016-06-13 16:46:39 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 12). 1161 bytes result sent to driver -2016-06-13 16:46:39 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 11). 1161 bytes result sent to driver -2016-06-13 16:46:39 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 9). 1161 bytes result sent to driver -2016-06-13 16:46:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 2446 bytes) -2016-06-13 16:46:39 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 13) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 14, localhost, PROCESS_LOCAL, 2430 bytes) -2016-06-13 16:46:39 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 14) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 12) in 111 ms on localhost (1/3) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 9) in 134 ms on localhost (1/3) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 10) in 137 ms on localhost (2/3) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 11) in 138 ms on localhost (3/3) -2016-06-13 16:46:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 16:46:39 INFO DAGScheduler:59 - ShuffleMapStage 3 (flatMap at BlockMatrix.scala:377) finished in 0.143 s -2016-06-13 16:46:39 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:46:39 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 4) -2016-06-13 16:46:39 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 5, ResultStage 6) -2016-06-13 16:46:39 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:46:39 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 5: List(ShuffleMapStage 4) -2016-06-13 16:46:39 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List(ShuffleMapStage 5) -2016-06-13 16:46:39 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 13). 1161 bytes result sent to driver -2016-06-13 16:46:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 13) in 58 ms on localhost (2/3) -2016-06-13 16:46:39 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 14). 1161 bytes result sent to driver -2016-06-13 16:46:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 14) in 58 ms on localhost (3/3) -2016-06-13 16:46:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 16:46:39 INFO DAGScheduler:59 - ShuffleMapStage 4 (flatMap at BlockMatrix.scala:373) finished in 0.167 s -2016-06-13 16:46:39 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:46:39 INFO DAGScheduler:59 - running: Set() -2016-06-13 16:46:39 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 5, ResultStage 6) -2016-06-13 16:46:39 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:46:39 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 5: List() -2016-06-13 16:46:39 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List(ShuffleMapStage 5) -2016-06-13 16:46:39 INFO DAGScheduler:59 - Submitting ShuffleMapStage 5 (MapPartitionsRDD[9] at flatMap at BlockMatrix.scala:381), which is now runnable -2016-06-13 16:46:39 INFO MemoryStore:59 - ensureFreeSpace(2952) called with curMem=19323, maxMem=766075207 -2016-06-13 16:46:39 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-06-13 16:46:39 INFO MemoryStore:59 - ensureFreeSpace(1712) called with curMem=22275, maxMem=766075207 -2016-06-13 16:46:39 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 1712.0 B, free 730.6 MB) -2016-06-13 16:46:39 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:61310 (size: 1712.0 B, free: 730.6 MB) -2016-06-13 16:46:39 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:46:39 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 5 (MapPartitionsRDD[9] at flatMap at BlockMatrix.scala:381) -2016-06-13 16:46:39 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 4 tasks -2016-06-13 16:46:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 15, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 16, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 17, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:46:39 INFO TaskSetManager:59 - Starting task 3.0 in stage 5.0 (TID 18, localhost, PROCESS_LOCAL, 1963 bytes) -2016-06-13 16:46:39 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 16) -2016-06-13 16:46:39 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 17) -2016-06-13 16:46:39 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 15) -2016-06-13 16:46:39 INFO Executor:59 - Running task 3.0 in stage 5.0 (TID 18) -2016-06-13 16:46:39 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:46:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 9 ms -2016-06-13 16:46:39 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:46:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 22 ms -2016-06-13 16:46:39 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:46:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 24 ms -2016-06-13 16:46:39 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:46:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:46:40 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:46:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:46:39 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:46:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 13 ms -2016-06-13 16:46:40 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:46:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:46:40 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 16:46:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:46:40 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 16:46:40 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 16:46:40 INFO Executor:59 - Finished task 3.0 in stage 5.0 (TID 18). 1377 bytes result sent to driver -2016-06-13 16:46:40 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 15). 1377 bytes result sent to driver -2016-06-13 16:46:40 INFO TaskSetManager:59 - Finished task 3.0 in stage 5.0 (TID 18) in 163 ms on localhost (1/4) -2016-06-13 16:46:40 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 16). 1377 bytes result sent to driver -2016-06-13 16:46:40 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 17). 1377 bytes result sent to driver -2016-06-13 16:46:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 15) in 172 ms on localhost (2/4) -2016-06-13 16:46:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 16) in 172 ms on localhost (3/4) -2016-06-13 16:46:40 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 17) in 171 ms on localhost (4/4) -2016-06-13 16:46:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 16:46:40 INFO DAGScheduler:59 - ShuffleMapStage 5 (flatMap at BlockMatrix.scala:381) finished in 0.176 s -2016-06-13 16:46:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 16:46:40 INFO DAGScheduler:59 - running: Set() -2016-06-13 16:46:40 INFO DAGScheduler:59 - waiting: Set(ResultStage 6) -2016-06-13 16:46:40 INFO DAGScheduler:59 - failed: Set() -2016-06-13 16:46:40 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List() -2016-06-13 16:46:40 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[11] at mapValues at BlockMatrix.scala:397), which is now runnable -2016-06-13 16:46:40 INFO MemoryStore:59 - ensureFreeSpace(2888) called with curMem=23987, maxMem=766075207 -2016-06-13 16:46:40 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 2.8 KB, free 730.6 MB) -2016-06-13 16:46:40 INFO MemoryStore:59 - ensureFreeSpace(1718) called with curMem=26875, maxMem=766075207 -2016-06-13 16:46:40 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1718.0 B, free 730.6 MB) -2016-06-13 16:46:40 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:61310 (size: 1718.0 B, free: 730.6 MB) -2016-06-13 16:46:40 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 16:46:40 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 6 (MapPartitionsRDD[11] at mapValues at BlockMatrix.scala:397) -2016-06-13 16:46:40 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 4 tasks -2016-06-13 16:46:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 19, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:46:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:46:40 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 21, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:46:40 INFO TaskSetManager:59 - Starting task 3.0 in stage 6.0 (TID 22, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 16:46:40 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 21) -2016-06-13 16:46:40 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 20) -2016-06-13 16:46:40 INFO Executor:59 - Running task 3.0 in stage 6.0 (TID 22) -2016-06-13 16:46:40 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 4 blocks -2016-06-13 16:46:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:46:40 INFO Executor:59 - Finished task 3.0 in stage 6.0 (TID 22). 1165 bytes result sent to driver -2016-06-13 16:46:40 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 19) -2016-06-13 16:46:40 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 4 blocks -2016-06-13 16:46:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:46:40 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 4 blocks -2016-06-13 16:46:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 16:46:40 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 19). 1165 bytes result sent to driver -2016-06-13 16:46:40 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 4 blocks -2016-06-13 16:46:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 16:46:40 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 21). 1165 bytes result sent to driver -2016-06-13 16:46:40 INFO TaskSetManager:59 - Finished task 3.0 in stage 6.0 (TID 22) in 41 ms on localhost (1/4) -2016-06-13 16:46:40 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 20). 1165 bytes result sent to driver -2016-06-13 16:46:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 19) in 48 ms on localhost (2/4) -2016-06-13 16:46:40 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 21) in 47 ms on localhost (3/4) -2016-06-13 16:46:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 20) in 51 ms on localhost (4/4) -2016-06-13 16:46:40 INFO DAGScheduler:59 - ResultStage 6 (foreach at BlockMatrixSuite.scala:72) finished in 0.054 s -2016-06-13 16:46:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 16:46:40 INFO DAGScheduler:59 - Job 3 finished: foreach at BlockMatrixSuite.scala:72, took 0.502555 s -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 16:46:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 16:46:40 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 16:46:40 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 16:46:40 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 16:46:40 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 16:46:40 INFO BlockManager:59 - BlockManager stopped -2016-06-13 16:46:40 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 16:46:40 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 16:46:40 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 16:46:40 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 16:46:40 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 16:46:40 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 16:46:40 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-8435e46a-6768-4fdb-8e01-0b93ff9288af -2016-06-13 17:02:35 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 17:02:36 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 17:02:36 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 17:02:36 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 17:02:36 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 17:02:37 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 17:02:37 INFO Remoting:74 - Starting remoting -2016-06-13 17:02:37 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:61496] -2016-06-13 17:02:37 INFO Utils:59 - Successfully started service 'sparkDriver' on port 61496. -2016-06-13 17:02:37 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 17:02:37 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 17:02:37 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-729ebd3b-a5e2-42cf-8cb7-3d3ad19fb978 -2016-06-13 17:02:37 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 17:02:37 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-d257938b-6ffc-4ac8-849e-fbd25bf36892\httpd-fc76747a-37fe-4626-ae4c-1150e51fb0e0 -2016-06-13 17:02:37 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 17:02:37 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:02:37 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:61497 -2016-06-13 17:02:37 INFO Utils:59 - Successfully started service 'HTTP file server' on port 61497. -2016-06-13 17:02:37 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 17:02:38 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:02:38 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 17:02:38 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 17:02:38 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 17:02:38 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 17:02:38 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 17:02:38 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 61508. -2016-06-13 17:02:38 INFO NettyBlockTransferService:59 - Server created on 61508 -2016-06-13 17:02:38 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 17:02:38 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:61508 with 730.6 MB RAM, BlockManagerId(driver, localhost, 61508) -2016-06-13 17:02:38 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 17:02:39 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 17:02:39 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 17:02:39 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 17:02:39 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 17:02:39 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:61508 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 17:02:39 INFO SparkContext:59 - Created broadcast 0 from textFile at KMeansTest2.scala:20 -2016-06-13 17:02:40 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 17:02:40 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 17:02:40 INFO SparkContext:59 - Starting job: count at KMeansTest2.scala:23 -2016-06-13 17:02:40 INFO DAGScheduler:59 - Got job 0 (count at KMeansTest2.scala:23) with 2 output partitions -2016-06-13 17:02:40 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at KMeansTest2.scala:23) -2016-06-13 17:02:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:02:40 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:02:40 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[1] at textFile at KMeansTest2.scala:20), which has no missing parents -2016-06-13 17:02:40 INFO MemoryStore:59 - ensureFreeSpace(3000) called with curMem=171580, maxMem=766075207 -2016-06-13 17:02:40 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-06-13 17:02:40 INFO MemoryStore:59 - ensureFreeSpace(1778) called with curMem=174580, maxMem=766075207 -2016-06-13 17:02:40 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1778.0 B, free 730.4 MB) -2016-06-13 17:02:40 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:61508 (size: 1778.0 B, free: 730.6 MB) -2016-06-13 17:02:40 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:02:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at textFile at KMeansTest2.scala:20) -2016-06-13 17:02:40 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 17:02:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2178 bytes) -2016-06-13 17:02:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2178 bytes) -2016-06-13 17:02:40 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 17:02:40 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 17:02:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-06-13 17:02:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-06-13 17:02:40 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 17:02:40 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 17:02:40 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 17:02:40 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 17:02:40 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 17:02:40 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 17:02:40 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 17:02:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 102 ms on localhost (1/2) -2016-06-13 17:02:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 79 ms on localhost (2/2) -2016-06-13 17:02:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 17:02:40 INFO DAGScheduler:59 - ResultStage 0 (count at KMeansTest2.scala:23) finished in 0.121 s -2016-06-13 17:02:40 INFO DAGScheduler:59 - Job 0 finished: count at KMeansTest2.scala:23, took 0.251075 s -2016-06-13 17:02:40 INFO SparkContext:59 - Starting job: foreach at KMeansTest2.scala:24 -2016-06-13 17:02:40 INFO DAGScheduler:59 - Got job 1 (foreach at KMeansTest2.scala:24) with 2 output partitions -2016-06-13 17:02:40 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at KMeansTest2.scala:24) -2016-06-13 17:02:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:02:40 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:02:40 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[1] at textFile at KMeansTest2.scala:20), which has no missing parents -2016-06-13 17:02:40 INFO MemoryStore:59 - ensureFreeSpace(3120) called with curMem=176358, maxMem=766075207 -2016-06-13 17:02:40 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 17:02:40 INFO MemoryStore:59 - ensureFreeSpace(1837) called with curMem=179478, maxMem=766075207 -2016-06-13 17:02:40 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1837.0 B, free 730.4 MB) -2016-06-13 17:02:40 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:61508 (size: 1837.0 B, free: 730.6 MB) -2016-06-13 17:02:40 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:02:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[1] at textFile at KMeansTest2.scala:20) -2016-06-13 17:02:40 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 17:02:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2178 bytes) -2016-06-13 17:02:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2178 bytes) -2016-06-13 17:02:40 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 17:02:40 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 17:02:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-06-13 17:02:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-06-13 17:02:40 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2044 bytes result sent to driver -2016-06-13 17:02:40 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 2044 bytes result sent to driver -2016-06-13 17:02:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 18 ms on localhost (1/2) -2016-06-13 17:02:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 18 ms on localhost (2/2) -2016-06-13 17:02:40 INFO DAGScheduler:59 - ResultStage 1 (foreach at KMeansTest2.scala:24) finished in 0.020 s -2016-06-13 17:02:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 17:02:40 INFO DAGScheduler:59 - Job 1 finished: foreach at KMeansTest2.scala:24, took 0.035014 s -2016-06-13 17:02:40 INFO SparkContext:59 - Starting job: foreach at KMeansTest2.scala:27 -2016-06-13 17:02:40 INFO DAGScheduler:59 - Got job 2 (foreach at KMeansTest2.scala:27) with 2 output partitions -2016-06-13 17:02:40 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at KMeansTest2.scala:27) -2016-06-13 17:02:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:02:40 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:02:40 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[2] at map at KMeansTest2.scala:26), which has no missing parents -2016-06-13 17:02:40 INFO MemoryStore:59 - ensureFreeSpace(3304) called with curMem=181315, maxMem=766075207 -2016-06-13 17:02:40 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 17:02:40 INFO MemoryStore:59 - ensureFreeSpace(1926) called with curMem=184619, maxMem=766075207 -2016-06-13 17:02:40 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1926.0 B, free 730.4 MB) -2016-06-13 17:02:40 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:61508 (size: 1926.0 B, free: 730.6 MB) -2016-06-13 17:02:40 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:02:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[2] at map at KMeansTest2.scala:26) -2016-06-13 17:02:40 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 17:02:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2178 bytes) -2016-06-13 17:02:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2178 bytes) -2016-06-13 17:02:40 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 17:02:40 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 17:02:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-06-13 17:02:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-06-13 17:02:40 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2044 bytes result sent to driver -2016-06-13 17:02:40 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2044 bytes result sent to driver -2016-06-13 17:02:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 20 ms on localhost (1/2) -2016-06-13 17:02:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 22 ms on localhost (2/2) -2016-06-13 17:02:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 17:02:40 INFO DAGScheduler:59 - ResultStage 2 (foreach at KMeansTest2.scala:27) finished in 0.022 s -2016-06-13 17:02:40 INFO DAGScheduler:59 - Job 2 finished: foreach at KMeansTest2.scala:27, took 0.070509 s -2016-06-13 17:02:40 WARN KMeans:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 17:02:40 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 17:02:40 INFO DAGScheduler:59 - Got job 3 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 17:02:40 INFO DAGScheduler:59 - Final stage: ResultStage 3(takeSample at KMeans.scala:376) -2016-06-13 17:02:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:02:40 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:02:40 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[5] at map at KMeans.scala:210), which has no missing parents -2016-06-13 17:02:40 INFO MemoryStore:59 - ensureFreeSpace(4072) called with curMem=186545, maxMem=766075207 -2016-06-13 17:02:40 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:02:40 INFO MemoryStore:59 - ensureFreeSpace(2290) called with curMem=190617, maxMem=766075207 -2016-06-13 17:02:40 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 17:02:40 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:61508 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 17:02:40 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:02:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[5] at map at KMeans.scala:210) -2016-06-13 17:02:40 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 17:02:40 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:61508 in memory (size: 1926.0 B, free: 730.6 MB) -2016-06-13 17:02:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2352 bytes) -2016-06-13 17:02:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2352 bytes) -2016-06-13 17:02:40 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-06-13 17:02:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-06-13 17:02:40 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-06-13 17:02:40 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 17:02:40 INFO CacheManager:59 - Partition rdd_3_0 not found, computing it -2016-06-13 17:02:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-06-13 17:02:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-06-13 17:02:40 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:61508 in memory (size: 1837.0 B, free: 730.6 MB) -2016-06-13 17:02:40 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 17:02:40 INFO CacheManager:59 - Partition rdd_3_1 not found, computing it -2016-06-13 17:02:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-06-13 17:02:40 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:61508 in memory (size: 1778.0 B, free: 730.6 MB) -2016-06-13 17:02:40 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 17:02:40 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=177942, maxMem=766075207 -2016-06-13 17:02:40 INFO MemoryStore:59 - Block rdd_3_1 stored as values in memory (estimated size 72.0 B, free 730.4 MB) -2016-06-13 17:02:40 INFO MemoryStore:59 - ensureFreeSpace(128) called with curMem=178014, maxMem=766075207 -2016-06-13 17:02:40 INFO MemoryStore:59 - Block rdd_3_0 stored as values in memory (estimated size 128.0 B, free 730.4 MB) -2016-06-13 17:02:40 INFO BlockManagerInfo:59 - Added rdd_3_1 in memory on localhost:61508 (size: 72.0 B, free: 730.6 MB) -2016-06-13 17:02:40 INFO BlockManagerInfo:59 - Added rdd_3_0 in memory on localhost:61508 (size: 128.0 B, free: 730.6 MB) -2016-06-13 17:02:40 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 2662 bytes result sent to driver -2016-06-13 17:02:40 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 2662 bytes result sent to driver -2016-06-13 17:02:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 78 ms on localhost (1/2) -2016-06-13 17:02:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 64 ms on localhost (2/2) -2016-06-13 17:02:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 17:02:40 INFO DAGScheduler:59 - ResultStage 3 (takeSample at KMeans.scala:376) finished in 0.081 s -2016-06-13 17:02:40 INFO DAGScheduler:59 - Job 3 finished: takeSample at KMeans.scala:376, took 0.149794 s -2016-06-13 17:02:41 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 17:02:41 INFO DAGScheduler:59 - Got job 4 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 17:02:41 INFO DAGScheduler:59 - Final stage: ResultStage 4(takeSample at KMeans.scala:376) -2016-06-13 17:02:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:02:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:02:41 INFO DAGScheduler:59 - Submitting ResultStage 4 (PartitionwiseSampledRDD[7] at takeSample at KMeans.scala:376), which has no missing parents -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(21464) called with curMem=178142, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 21.0 KB, free 730.4 MB) -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(8961) called with curMem=199606, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 8.8 KB, free 730.4 MB) -2016-06-13 17:02:41 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:61508 (size: 8.8 KB, free: 730.6 MB) -2016-06-13 17:02:41 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:02:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (PartitionwiseSampledRDD[7] at takeSample at KMeans.scala:376) -2016-06-13 17:02:41 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 17:02:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2461 bytes) -2016-06-13 17:02:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2461 bytes) -2016-06-13 17:02:41 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-06-13 17:02:41 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-06-13 17:02:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-06-13 17:02:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:02:41 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2442 bytes result sent to driver -2016-06-13 17:02:41 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2630 bytes result sent to driver -2016-06-13 17:02:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 34 ms on localhost (1/2) -2016-06-13 17:02:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 38 ms on localhost (2/2) -2016-06-13 17:02:41 INFO DAGScheduler:59 - ResultStage 4 (takeSample at KMeans.scala:376) finished in 0.038 s -2016-06-13 17:02:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 17:02:41 INFO DAGScheduler:59 - Job 4 finished: takeSample at KMeans.scala:376, took 0.087174 s -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(1720) called with curMem=208567, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 1720.0 B, free 730.4 MB) -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(563) called with curMem=210287, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 563.0 B, free 730.4 MB) -2016-06-13 17:02:41 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:61508 (size: 563.0 B, free: 730.6 MB) -2016-06-13 17:02:41 INFO SparkContext:59 - Created broadcast 6 from broadcast at KMeans.scala:394 -2016-06-13 17:02:41 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 17:02:41 INFO DAGScheduler:59 - Got job 5 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 17:02:41 INFO DAGScheduler:59 - Final stage: ResultStage 5(aggregate at KMeans.scala:402) -2016-06-13 17:02:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:02:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:02:41 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[9] at map at KMeans.scala:396), which has no missing parents -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(5664) called with curMem=210850, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.4 MB) -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(2875) called with curMem=216514, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-06-13 17:02:41 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:61508 (size: 2.8 KB, free: 730.6 MB) -2016-06-13 17:02:41 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:02:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[9] at map at KMeans.scala:396) -2016-06-13 17:02:41 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 17:02:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2384 bytes) -2016-06-13 17:02:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 2384 bytes) -2016-06-13 17:02:41 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 10) -2016-06-13 17:02:41 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 11) -2016-06-13 17:02:41 INFO CacheManager:59 - Partition rdd_9_0 not found, computing it -2016-06-13 17:02:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-06-13 17:02:41 INFO CacheManager:59 - Partition rdd_9_1 not found, computing it -2016-06-13 17:02:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:02:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:02:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:02:41 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 17:02:41 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(216) called with curMem=219389, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block rdd_9_1 stored as values in memory (estimated size 216.0 B, free 730.4 MB) -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(416) called with curMem=219605, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block rdd_9_0 stored as values in memory (estimated size 416.0 B, free 730.4 MB) -2016-06-13 17:02:41 INFO BlockManagerInfo:59 - Added rdd_9_1 in memory on localhost:61508 (size: 216.0 B, free: 730.6 MB) -2016-06-13 17:02:41 INFO BlockManagerInfo:59 - Added rdd_9_0 in memory on localhost:61508 (size: 416.0 B, free: 730.6 MB) -2016-06-13 17:02:41 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 11). 2687 bytes result sent to driver -2016-06-13 17:02:41 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 10). 2687 bytes result sent to driver -2016-06-13 17:02:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 11) in 86 ms on localhost (1/2) -2016-06-13 17:02:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 10) in 88 ms on localhost (2/2) -2016-06-13 17:02:41 INFO DAGScheduler:59 - ResultStage 5 (aggregate at KMeans.scala:402) finished in 0.089 s -2016-06-13 17:02:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 17:02:41 INFO DAGScheduler:59 - Job 5 finished: aggregate at KMeans.scala:402, took 0.112527 s -2016-06-13 17:02:41 INFO MapPartitionsRDD:59 - Removing RDD 6 from persistence list -2016-06-13 17:02:41 INFO BlockManager:59 - Removing RDD 6 -2016-06-13 17:02:41 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 17:02:41 INFO DAGScheduler:59 - Got job 6 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 17:02:41 INFO DAGScheduler:59 - Final stage: ResultStage 6(collect at KMeans.scala:431) -2016-06-13 17:02:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:02:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:02:41 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[11] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(5824) called with curMem=220021, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.7 KB, free 730.4 MB) -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(3006) called with curMem=225845, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-06-13 17:02:41 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:61508 (size: 2.9 KB, free: 730.6 MB) -2016-06-13 17:02:41 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:02:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[11] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 17:02:41 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-06-13 17:02:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2416 bytes) -2016-06-13 17:02:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2416 bytes) -2016-06-13 17:02:41 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 13) -2016-06-13 17:02:41 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 12) -2016-06-13 17:02:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-06-13 17:02:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:02:41 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 13). 2963 bytes result sent to driver -2016-06-13 17:02:41 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 12). 3227 bytes result sent to driver -2016-06-13 17:02:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 13) in 31 ms on localhost (1/2) -2016-06-13 17:02:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 12) in 33 ms on localhost (2/2) -2016-06-13 17:02:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 17:02:41 INFO DAGScheduler:59 - ResultStage 6 (collect at KMeans.scala:431) finished in 0.034 s -2016-06-13 17:02:41 INFO DAGScheduler:59 - Job 6 finished: collect at KMeans.scala:431, took 0.055654 s -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(2560) called with curMem=228851, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(638) called with curMem=231411, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 638.0 B, free 730.4 MB) -2016-06-13 17:02:41 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:61508 (size: 638.0 B, free: 730.6 MB) -2016-06-13 17:02:41 INFO SparkContext:59 - Created broadcast 9 from broadcast at KMeans.scala:394 -2016-06-13 17:02:41 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 17:02:41 INFO DAGScheduler:59 - Got job 7 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 17:02:41 INFO DAGScheduler:59 - Final stage: ResultStage 7(aggregate at KMeans.scala:402) -2016-06-13 17:02:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:02:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:02:41 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[13] at map at KMeans.scala:396), which has no missing parents -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(5896) called with curMem=232049, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.8 KB, free 730.4 MB) -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(2957) called with curMem=237945, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-06-13 17:02:41 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:61508 (size: 2.9 KB, free: 730.6 MB) -2016-06-13 17:02:41 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:02:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[13] at map at KMeans.scala:396) -2016-06-13 17:02:41 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-06-13 17:02:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2416 bytes) -2016-06-13 17:02:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 15, localhost, PROCESS_LOCAL, 2416 bytes) -2016-06-13 17:02:41 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 14) -2016-06-13 17:02:41 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 15) -2016-06-13 17:02:41 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-06-13 17:02:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-06-13 17:02:41 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-06-13 17:02:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(216) called with curMem=240902, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 216.0 B, free 730.4 MB) -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(416) called with curMem=241118, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 416.0 B, free 730.4 MB) -2016-06-13 17:02:41 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:61508 (size: 216.0 B, free: 730.6 MB) -2016-06-13 17:02:41 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:61508 (size: 416.0 B, free: 730.6 MB) -2016-06-13 17:02:41 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 15). 2687 bytes result sent to driver -2016-06-13 17:02:41 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 14). 2687 bytes result sent to driver -2016-06-13 17:02:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 15) in 26 ms on localhost (1/2) -2016-06-13 17:02:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 14) in 29 ms on localhost (2/2) -2016-06-13 17:02:41 INFO DAGScheduler:59 - ResultStage 7 (aggregate at KMeans.scala:402) finished in 0.030 s -2016-06-13 17:02:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 17:02:41 INFO DAGScheduler:59 - Job 7 finished: aggregate at KMeans.scala:402, took 0.050287 s -2016-06-13 17:02:41 INFO MapPartitionsRDD:59 - Removing RDD 9 from persistence list -2016-06-13 17:02:41 INFO BlockManager:59 - Removing RDD 9 -2016-06-13 17:02:41 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 17:02:41 INFO DAGScheduler:59 - Got job 8 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 17:02:41 INFO DAGScheduler:59 - Final stage: ResultStage 8(collect at KMeans.scala:431) -2016-06-13 17:02:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:02:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:02:41 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[15] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(6056) called with curMem=240902, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.9 KB, free 730.4 MB) -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(3080) called with curMem=246958, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.3 MB) -2016-06-13 17:02:41 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:61508 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 17:02:41 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:02:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[15] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 17:02:41 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 17:02:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2448 bytes) -2016-06-13 17:02:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 17, localhost, PROCESS_LOCAL, 2448 bytes) -2016-06-13 17:02:41 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 17) -2016-06-13 17:02:41 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 16) -2016-06-13 17:02:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-06-13 17:02:41 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 16). 3243 bytes result sent to driver -2016-06-13 17:02:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-06-13 17:02:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 16) in 19 ms on localhost (1/2) -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-06-13 17:02:41 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 17). 2877 bytes result sent to driver -2016-06-13 17:02:41 INFO DAGScheduler:59 - ResultStage 8 (collect at KMeans.scala:431) finished in 0.025 s -2016-06-13 17:02:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 17) in 24 ms on localhost (2/2) -2016-06-13 17:02:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 17:02:41 INFO DAGScheduler:59 - Job 8 finished: collect at KMeans.scala:431, took 0.041543 s -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(2160) called with curMem=250038, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(625) called with curMem=252198, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 625.0 B, free 730.3 MB) -2016-06-13 17:02:41 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:61508 (size: 625.0 B, free: 730.5 MB) -2016-06-13 17:02:41 INFO SparkContext:59 - Created broadcast 12 from broadcast at KMeans.scala:394 -2016-06-13 17:02:41 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 17:02:41 INFO DAGScheduler:59 - Got job 9 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 17:02:41 INFO DAGScheduler:59 - Final stage: ResultStage 9(aggregate at KMeans.scala:402) -2016-06-13 17:02:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:02:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:02:41 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[17] at map at KMeans.scala:396), which has no missing parents -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(6128) called with curMem=252823, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 6.0 KB, free 730.3 MB) -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(3015) called with curMem=258951, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-06-13 17:02:41 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:61508 (size: 2.9 KB, free: 730.5 MB) -2016-06-13 17:02:41 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:02:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[17] at map at KMeans.scala:396) -2016-06-13 17:02:41 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-06-13 17:02:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2448 bytes) -2016-06-13 17:02:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 19, localhost, PROCESS_LOCAL, 2448 bytes) -2016-06-13 17:02:41 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 18) -2016-06-13 17:02:41 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 19) -2016-06-13 17:02:41 INFO CacheManager:59 - Partition rdd_17_0 not found, computing it -2016-06-13 17:02:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-06-13 17:02:41 INFO CacheManager:59 - Partition rdd_17_1 not found, computing it -2016-06-13 17:02:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(416) called with curMem=261966, maxMem=766075207 -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-06-13 17:02:41 INFO MemoryStore:59 - Block rdd_17_0 stored as values in memory (estimated size 416.0 B, free 730.3 MB) -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(216) called with curMem=262382, maxMem=766075207 -2016-06-13 17:02:41 INFO BlockManagerInfo:59 - Added rdd_17_0 in memory on localhost:61508 (size: 416.0 B, free: 730.5 MB) -2016-06-13 17:02:41 INFO MemoryStore:59 - Block rdd_17_1 stored as values in memory (estimated size 216.0 B, free 730.3 MB) -2016-06-13 17:02:41 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 18). 2687 bytes result sent to driver -2016-06-13 17:02:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 18) in 22 ms on localhost (1/2) -2016-06-13 17:02:41 INFO BlockManagerInfo:59 - Added rdd_17_1 in memory on localhost:61508 (size: 216.0 B, free: 730.5 MB) -2016-06-13 17:02:41 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 19). 2687 bytes result sent to driver -2016-06-13 17:02:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 19) in 34 ms on localhost (2/2) -2016-06-13 17:02:41 INFO DAGScheduler:59 - ResultStage 9 (aggregate at KMeans.scala:402) finished in 0.036 s -2016-06-13 17:02:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 17:02:41 INFO DAGScheduler:59 - Job 9 finished: aggregate at KMeans.scala:402, took 0.052838 s -2016-06-13 17:02:41 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-06-13 17:02:41 INFO BlockManager:59 - Removing RDD 13 -2016-06-13 17:02:41 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 17:02:41 INFO DAGScheduler:59 - Got job 10 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 17:02:41 INFO DAGScheduler:59 - Final stage: ResultStage 10(collect at KMeans.scala:431) -2016-06-13 17:02:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:02:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:02:41 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[19] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(6288) called with curMem=261966, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 6.1 KB, free 730.3 MB) -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(3104) called with curMem=268254, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.3 MB) -2016-06-13 17:02:41 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:61508 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 17:02:41 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:02:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[19] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 17:02:41 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-06-13 17:02:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2480 bytes) -2016-06-13 17:02:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 21, localhost, PROCESS_LOCAL, 2480 bytes) -2016-06-13 17:02:41 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 21) -2016-06-13 17:02:41 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 20) -2016-06-13 17:02:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-06-13 17:02:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_17_0 locally -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_17_1 locally -2016-06-13 17:02:41 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 20). 2040 bytes result sent to driver -2016-06-13 17:02:41 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 21). 2040 bytes result sent to driver -2016-06-13 17:02:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 20) in 15 ms on localhost (1/2) -2016-06-13 17:02:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 21) in 15 ms on localhost (2/2) -2016-06-13 17:02:41 INFO DAGScheduler:59 - ResultStage 10 (collect at KMeans.scala:431) finished in 0.016 s -2016-06-13 17:02:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 17:02:41 INFO DAGScheduler:59 - Job 10 finished: collect at KMeans.scala:431, took 0.032907 s -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(1120) called with curMem=271358, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1120.0 B, free 730.3 MB) -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(247) called with curMem=272478, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 247.0 B, free 730.3 MB) -2016-06-13 17:02:41 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:61508 (size: 247.0 B, free: 730.5 MB) -2016-06-13 17:02:41 INFO SparkContext:59 - Created broadcast 15 from broadcast at KMeans.scala:394 -2016-06-13 17:02:41 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 17:02:41 INFO DAGScheduler:59 - Got job 11 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 17:02:41 INFO DAGScheduler:59 - Final stage: ResultStage 11(aggregate at KMeans.scala:402) -2016-06-13 17:02:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:02:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:02:41 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[21] at map at KMeans.scala:396), which has no missing parents -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(6360) called with curMem=272725, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 6.2 KB, free 730.3 MB) -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(3075) called with curMem=279085, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.3 MB) -2016-06-13 17:02:41 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:61508 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 17:02:41 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:02:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[21] at map at KMeans.scala:396) -2016-06-13 17:02:41 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 17:02:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2480 bytes) -2016-06-13 17:02:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 23, localhost, PROCESS_LOCAL, 2480 bytes) -2016-06-13 17:02:41 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 23) -2016-06-13 17:02:41 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 22) -2016-06-13 17:02:41 INFO CacheManager:59 - Partition rdd_21_1 not found, computing it -2016-06-13 17:02:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-06-13 17:02:41 INFO CacheManager:59 - Partition rdd_21_0 not found, computing it -2016-06-13 17:02:41 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_17_1 locally -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:02:41 INFO BlockManager:59 - Found block rdd_17_0 locally -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(216) called with curMem=282160, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block rdd_21_1 stored as values in memory (estimated size 216.0 B, free 730.3 MB) -2016-06-13 17:02:41 INFO BlockManagerInfo:59 - Added rdd_21_1 in memory on localhost:61508 (size: 216.0 B, free: 730.5 MB) -2016-06-13 17:02:41 INFO MemoryStore:59 - ensureFreeSpace(416) called with curMem=282376, maxMem=766075207 -2016-06-13 17:02:41 INFO MemoryStore:59 - Block rdd_21_0 stored as values in memory (estimated size 416.0 B, free 730.3 MB) -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Added rdd_21_0 in memory on localhost:61508 (size: 416.0 B, free: 730.5 MB) -2016-06-13 17:02:42 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 23). 2687 bytes result sent to driver -2016-06-13 17:02:42 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 22). 2687 bytes result sent to driver -2016-06-13 17:02:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 23) in 23 ms on localhost (1/2) -2016-06-13 17:02:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 22) in 26 ms on localhost (2/2) -2016-06-13 17:02:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 17:02:42 INFO DAGScheduler:59 - ResultStage 11 (aggregate at KMeans.scala:402) finished in 0.026 s -2016-06-13 17:02:42 INFO DAGScheduler:59 - Job 11 finished: aggregate at KMeans.scala:402, took 0.042939 s -2016-06-13 17:02:42 INFO MapPartitionsRDD:59 - Removing RDD 17 from persistence list -2016-06-13 17:02:42 INFO BlockManager:59 - Removing RDD 17 -2016-06-13 17:02:42 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 17:02:42 INFO DAGScheduler:59 - Got job 12 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 17:02:42 INFO DAGScheduler:59 - Final stage: ResultStage 12(collect at KMeans.scala:431) -2016-06-13 17:02:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:02:42 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:02:42 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[23] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(6520) called with curMem=282160, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 6.4 KB, free 730.3 MB) -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(3158) called with curMem=288680, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.3 MB) -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:61508 (size: 3.1 KB, free: 730.5 MB) -2016-06-13 17:02:42 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:02:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[23] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 17:02:42 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:61508 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 17:02:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2512 bytes) -2016-06-13 17:02:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 25, localhost, PROCESS_LOCAL, 2512 bytes) -2016-06-13 17:02:42 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 24) -2016-06-13 17:02:42 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 25) -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:61508 in memory (size: 2.9 KB, free: 730.5 MB) -2016-06-13 17:02:42 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 17:02:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:61508 in memory (size: 2.9 KB, free: 730.5 MB) -2016-06-13 17:02:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-06-13 17:02:42 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 17:02:42 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:02:42 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:61508 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 17:02:42 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:02:42 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 25). 2040 bytes result sent to driver -2016-06-13 17:02:42 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 17:02:42 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 17:02:42 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 24). 2040 bytes result sent to driver -2016-06-13 17:02:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 25) in 22 ms on localhost (1/2) -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:61508 in memory (size: 8.8 KB, free: 730.6 MB) -2016-06-13 17:02:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 24) in 26 ms on localhost (2/2) -2016-06-13 17:02:42 INFO DAGScheduler:59 - ResultStage 12 (collect at KMeans.scala:431) finished in 0.026 s -2016-06-13 17:02:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 17:02:42 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 17:02:42 INFO DAGScheduler:59 - Job 12 finished: collect at KMeans.scala:431, took 0.061689 s -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(1120) called with curMem=219693, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 1120.0 B, free 730.4 MB) -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:61508 in memory (size: 2.2 KB, free: 730.6 MB) -2016-06-13 17:02:42 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(247) called with curMem=217738, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 247.0 B, free 730.4 MB) -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:61508 in memory (size: 3.0 KB, free: 730.6 MB) -2016-06-13 17:02:42 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:61508 (size: 247.0 B, free: 730.6 MB) -2016-06-13 17:02:42 INFO SparkContext:59 - Created broadcast 18 from broadcast at KMeans.scala:394 -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:61508 in memory (size: 3.0 KB, free: 730.6 MB) -2016-06-13 17:02:42 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:61508 in memory (size: 2.9 KB, free: 730.6 MB) -2016-06-13 17:02:42 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 17:02:42 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 17:02:42 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 17:02:42 INFO DAGScheduler:59 - Got job 13 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 17:02:42 INFO DAGScheduler:59 - Final stage: ResultStage 13(aggregate at KMeans.scala:402) -2016-06-13 17:02:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:02:42 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:02:42 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[25] at map at KMeans.scala:396), which has no missing parents -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(6600) called with curMem=193090, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 6.4 KB, free 730.4 MB) -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(3127) called with curMem=199690, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.4 MB) -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:61508 (size: 3.1 KB, free: 730.6 MB) -2016-06-13 17:02:42 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:02:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[25] at map at KMeans.scala:396) -2016-06-13 17:02:42 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 17:02:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2512 bytes) -2016-06-13 17:02:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 27, localhost, PROCESS_LOCAL, 2512 bytes) -2016-06-13 17:02:42 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 27) -2016-06-13 17:02:42 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 26) -2016-06-13 17:02:42 INFO CacheManager:59 - Partition rdd_25_0 not found, computing it -2016-06-13 17:02:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-06-13 17:02:42 INFO CacheManager:59 - Partition rdd_25_1 not found, computing it -2016-06-13 17:02:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-06-13 17:02:42 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:02:42 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 17:02:42 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:02:42 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(416) called with curMem=202817, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block rdd_25_0 stored as values in memory (estimated size 416.0 B, free 730.4 MB) -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(216) called with curMem=203233, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block rdd_25_1 stored as values in memory (estimated size 216.0 B, free 730.4 MB) -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Added rdd_25_0 in memory on localhost:61508 (size: 416.0 B, free: 730.6 MB) -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Added rdd_25_1 in memory on localhost:61508 (size: 216.0 B, free: 730.6 MB) -2016-06-13 17:02:42 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 26). 2687 bytes result sent to driver -2016-06-13 17:02:42 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 27). 2687 bytes result sent to driver -2016-06-13 17:02:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 26) in 31 ms on localhost (1/2) -2016-06-13 17:02:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 27) in 31 ms on localhost (2/2) -2016-06-13 17:02:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 17:02:42 INFO DAGScheduler:59 - ResultStage 13 (aggregate at KMeans.scala:402) finished in 0.033 s -2016-06-13 17:02:42 INFO DAGScheduler:59 - Job 13 finished: aggregate at KMeans.scala:402, took 0.055793 s -2016-06-13 17:02:42 INFO MapPartitionsRDD:59 - Removing RDD 21 from persistence list -2016-06-13 17:02:42 INFO BlockManager:59 - Removing RDD 21 -2016-06-13 17:02:42 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 17:02:42 INFO DAGScheduler:59 - Got job 14 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 17:02:42 INFO DAGScheduler:59 - Final stage: ResultStage 14(collect at KMeans.scala:431) -2016-06-13 17:02:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:02:42 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:02:42 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[27] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(6760) called with curMem=202817, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 6.6 KB, free 730.4 MB) -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(3214) called with curMem=209577, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.4 MB) -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:61508 (size: 3.1 KB, free: 730.6 MB) -2016-06-13 17:02:42 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:02:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[27] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 17:02:42 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-06-13 17:02:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2544 bytes) -2016-06-13 17:02:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 29, localhost, PROCESS_LOCAL, 2544 bytes) -2016-06-13 17:02:42 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 28) -2016-06-13 17:02:42 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 29) -2016-06-13 17:02:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-06-13 17:02:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-06-13 17:02:42 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:02:42 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 17:02:42 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 29). 2040 bytes result sent to driver -2016-06-13 17:02:42 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:02:42 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 17:02:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 29) in 22 ms on localhost (1/2) -2016-06-13 17:02:42 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 28). 2040 bytes result sent to driver -2016-06-13 17:02:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 28) in 26 ms on localhost (2/2) -2016-06-13 17:02:42 INFO DAGScheduler:59 - ResultStage 14 (collect at KMeans.scala:431) finished in 0.026 s -2016-06-13 17:02:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 17:02:42 INFO DAGScheduler:59 - Job 14 finished: collect at KMeans.scala:431, took 0.046456 s -2016-06-13 17:02:42 INFO MapPartitionsRDD:59 - Removing RDD 25 from persistence list -2016-06-13 17:02:42 INFO BlockManager:59 - Removing RDD 25 -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(4200) called with curMem=212159, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 4.1 KB, free 730.4 MB) -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(821) called with curMem=216359, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 821.0 B, free 730.4 MB) -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:61508 (size: 821.0 B, free: 730.6 MB) -2016-06-13 17:02:42 INFO SparkContext:59 - Created broadcast 21 from broadcast at KMeans.scala:445 -2016-06-13 17:02:42 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:450 -2016-06-13 17:02:42 INFO DAGScheduler:59 - Registering RDD 28 (flatMap at KMeans.scala:446) -2016-06-13 17:02:42 INFO DAGScheduler:59 - Got job 15 (collectAsMap at KMeans.scala:450) with 2 output partitions -2016-06-13 17:02:42 INFO DAGScheduler:59 - Final stage: ResultStage 16(collectAsMap at KMeans.scala:450) -2016-06-13 17:02:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15) -2016-06-13 17:02:42 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 15) -2016-06-13 17:02:42 INFO DAGScheduler:59 - Submitting ShuffleMapStage 15 (MapPartitionsRDD[28] at flatMap at KMeans.scala:446), which has no missing parents -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(5656) called with curMem=217180, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 5.5 KB, free 730.4 MB) -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(3027) called with curMem=222836, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:61508 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 17:02:42 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:02:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 15 (MapPartitionsRDD[28] at flatMap at KMeans.scala:446) -2016-06-13 17:02:42 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-06-13 17:02:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2341 bytes) -2016-06-13 17:02:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 31, localhost, PROCESS_LOCAL, 2341 bytes) -2016-06-13 17:02:42 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 31) -2016-06-13 17:02:42 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 30) -2016-06-13 17:02:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-06-13 17:02:42 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:02:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-06-13 17:02:42 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:02:42 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 30). 2254 bytes result sent to driver -2016-06-13 17:02:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 30) in 94 ms on localhost (1/2) -2016-06-13 17:02:42 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 31). 2254 bytes result sent to driver -2016-06-13 17:02:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 31) in 97 ms on localhost (2/2) -2016-06-13 17:02:42 INFO DAGScheduler:59 - ShuffleMapStage 15 (flatMap at KMeans.scala:446) finished in 0.101 s -2016-06-13 17:02:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 17:02:42 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 17:02:42 INFO DAGScheduler:59 - running: Set() -2016-06-13 17:02:42 INFO DAGScheduler:59 - waiting: Set(ResultStage 16) -2016-06-13 17:02:42 INFO DAGScheduler:59 - failed: Set() -2016-06-13 17:02:42 INFO DAGScheduler:59 - Missing parents for ResultStage 16: List() -2016-06-13 17:02:42 INFO DAGScheduler:59 - Submitting ResultStage 16 (ShuffledRDD[29] at reduceByKey at KMeans.scala:450), which is now runnable -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=225863, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(1388) called with curMem=228175, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 1388.0 B, free 730.4 MB) -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:61508 (size: 1388.0 B, free: 730.6 MB) -2016-06-13 17:02:42 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:02:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (ShuffledRDD[29] at reduceByKey at KMeans.scala:450) -2016-06-13 17:02:42 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-06-13 17:02:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 17:02:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 33, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 17:02:42 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 32) -2016-06-13 17:02:42 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 33) -2016-06-13 17:02:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 17:02:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 17:02:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 3 ms -2016-06-13 17:02:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 3 ms -2016-06-13 17:02:42 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 33). 2505 bytes result sent to driver -2016-06-13 17:02:42 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 32). 2358 bytes result sent to driver -2016-06-13 17:02:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 33) in 37 ms on localhost (1/2) -2016-06-13 17:02:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 32) in 39 ms on localhost (2/2) -2016-06-13 17:02:42 INFO DAGScheduler:59 - ResultStage 16 (collectAsMap at KMeans.scala:450) finished in 0.040 s -2016-06-13 17:02:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 17:02:42 INFO DAGScheduler:59 - Job 15 finished: collectAsMap at KMeans.scala:450, took 0.172519 s -2016-06-13 17:02:42 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 17:02:42 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 17:02:42 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 17:02:42 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 17:02:42 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 17:02:42 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 17:02:42 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 17:02:42 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 17:02:42 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 17:02:42 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 17:02:42 INFO KMeans:59 - Initialization with k-means|| took 1.929 seconds. -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(1920) called with curMem=229563, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 1920.0 B, free 730.4 MB) -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(403) called with curMem=231483, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 403.0 B, free 730.4 MB) -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:61508 (size: 403.0 B, free: 730.6 MB) -2016-06-13 17:02:42 INFO SparkContext:59 - Created broadcast 24 from broadcast at KMeans.scala:276 -2016-06-13 17:02:42 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:302 -2016-06-13 17:02:42 INFO DAGScheduler:59 - Registering RDD 30 (mapPartitions at KMeans.scala:279) -2016-06-13 17:02:42 INFO DAGScheduler:59 - Got job 16 (collectAsMap at KMeans.scala:302) with 2 output partitions -2016-06-13 17:02:42 INFO DAGScheduler:59 - Final stage: ResultStage 18(collectAsMap at KMeans.scala:302) -2016-06-13 17:02:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 17) -2016-06-13 17:02:42 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 17) -2016-06-13 17:02:42 INFO DAGScheduler:59 - Submitting ShuffleMapStage 17 (MapPartitionsRDD[30] at mapPartitions at KMeans.scala:279), which has no missing parents -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(6640) called with curMem=231886, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 6.5 KB, free 730.4 MB) -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(3455) called with curMem=238526, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.4 MB) -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:61508 (size: 3.4 KB, free: 730.6 MB) -2016-06-13 17:02:42 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:02:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 17 (MapPartitionsRDD[30] at mapPartitions at KMeans.scala:279) -2016-06-13 17:02:42 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-06-13 17:02:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2341 bytes) -2016-06-13 17:02:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 35, localhost, PROCESS_LOCAL, 2341 bytes) -2016-06-13 17:02:42 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 35) -2016-06-13 17:02:42 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 34) -2016-06-13 17:02:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-06-13 17:02:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-06-13 17:02:42 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:02:42 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:02:42 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 34). 2533 bytes result sent to driver -2016-06-13 17:02:42 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 35). 2533 bytes result sent to driver -2016-06-13 17:02:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 34) in 36 ms on localhost (1/2) -2016-06-13 17:02:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 35) in 36 ms on localhost (2/2) -2016-06-13 17:02:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 17:02:42 INFO DAGScheduler:59 - ShuffleMapStage 17 (mapPartitions at KMeans.scala:279) finished in 0.038 s -2016-06-13 17:02:42 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 17:02:42 INFO DAGScheduler:59 - running: Set() -2016-06-13 17:02:42 INFO DAGScheduler:59 - waiting: Set(ResultStage 18) -2016-06-13 17:02:42 INFO DAGScheduler:59 - failed: Set() -2016-06-13 17:02:42 INFO DAGScheduler:59 - Missing parents for ResultStage 18: List() -2016-06-13 17:02:42 INFO DAGScheduler:59 - Submitting ResultStage 18 (ShuffledRDD[31] at reduceByKey at KMeans.scala:302), which is now runnable -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(2808) called with curMem=241981, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(1591) called with curMem=244789, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 1591.0 B, free 730.4 MB) -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:61508 (size: 1591.0 B, free: 730.6 MB) -2016-06-13 17:02:42 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:02:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (ShuffledRDD[31] at reduceByKey at KMeans.scala:302) -2016-06-13 17:02:42 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-06-13 17:02:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 17:02:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 37, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 17:02:42 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 36) -2016-06-13 17:02:42 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 37) -2016-06-13 17:02:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 17:02:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 17:02:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 17:02:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 17:02:42 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 37). 2018 bytes result sent to driver -2016-06-13 17:02:42 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 36). 2312 bytes result sent to driver -2016-06-13 17:02:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 37) in 22 ms on localhost (1/2) -2016-06-13 17:02:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 36) in 24 ms on localhost (2/2) -2016-06-13 17:02:42 INFO DAGScheduler:59 - ResultStage 18 (collectAsMap at KMeans.scala:302) finished in 0.024 s -2016-06-13 17:02:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 17:02:42 INFO DAGScheduler:59 - Job 16 finished: collectAsMap at KMeans.scala:302, took 0.086403 s -2016-06-13 17:02:42 INFO KMeans:59 - Run 0 finished in 1 iterations -2016-06-13 17:02:42 INFO KMeans:59 - Run 1 finished in 1 iterations -2016-06-13 17:02:42 INFO KMeans:59 - Run 2 finished in 1 iterations -2016-06-13 17:02:42 INFO KMeans:59 - Run 3 finished in 1 iterations -2016-06-13 17:02:42 INFO KMeans:59 - Run 4 finished in 1 iterations -2016-06-13 17:02:42 INFO KMeans:59 - Run 5 finished in 1 iterations -2016-06-13 17:02:42 INFO KMeans:59 - Run 6 finished in 1 iterations -2016-06-13 17:02:42 INFO KMeans:59 - Run 7 finished in 1 iterations -2016-06-13 17:02:42 INFO KMeans:59 - Run 8 finished in 1 iterations -2016-06-13 17:02:42 INFO KMeans:59 - Run 9 finished in 1 iterations -2016-06-13 17:02:42 INFO KMeans:59 - Iterations took 0.153 seconds. -2016-06-13 17:02:42 INFO KMeans:59 - KMeans converged in 1 iterations. -2016-06-13 17:02:42 INFO KMeans:59 - The cost for the best run is 0.11999999999994547. -2016-06-13 17:02:42 INFO MapPartitionsRDD:59 - Removing RDD 3 from persistence list -2016-06-13 17:02:42 INFO BlockManager:59 - Removing RDD 3 -2016-06-13 17:02:42 WARN KMeans:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(232) called with curMem=246180, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 232.0 B, free 730.4 MB) -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(395) called with curMem=246412, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 395.0 B, free 730.4 MB) -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:61508 (size: 395.0 B, free: 730.6 MB) -2016-06-13 17:02:42 INFO SparkContext:59 - Created broadcast 27 from broadcast at KMeansModel.scala:87 -2016-06-13 17:02:42 INFO SparkContext:59 - Starting job: sum at KMeansModel.scala:88 -2016-06-13 17:02:42 INFO DAGScheduler:59 - Got job 17 (sum at KMeansModel.scala:88) with 2 output partitions -2016-06-13 17:02:42 INFO DAGScheduler:59 - Final stage: ResultStage 19(sum at KMeansModel.scala:88) -2016-06-13 17:02:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:02:42 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:02:42 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[32] at map at KMeansModel.scala:88), which has no missing parents -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(3968) called with curMem=246807, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 3.9 KB, free 730.3 MB) -2016-06-13 17:02:42 INFO MemoryStore:59 - ensureFreeSpace(2327) called with curMem=250775, maxMem=766075207 -2016-06-13 17:02:42 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.3 MB) -2016-06-13 17:02:42 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:61508 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 17:02:42 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:02:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[32] at map at KMeansModel.scala:88) -2016-06-13 17:02:42 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-06-13 17:02:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2178 bytes) -2016-06-13 17:02:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 39, localhost, PROCESS_LOCAL, 2178 bytes) -2016-06-13 17:02:42 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 39) -2016-06-13 17:02:42 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 38) -2016-06-13 17:02:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:0+36 -2016-06-13 17:02:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/kmeans_data.txt:36+36 -2016-06-13 17:02:42 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 39). 2084 bytes result sent to driver -2016-06-13 17:02:42 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 38). 2084 bytes result sent to driver -2016-06-13 17:02:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 39) in 15 ms on localhost (1/2) -2016-06-13 17:02:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 38) in 17 ms on localhost (2/2) -2016-06-13 17:02:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 17:02:42 INFO DAGScheduler:59 - ResultStage 19 (sum at KMeansModel.scala:88) finished in 0.017 s -2016-06-13 17:02:42 INFO DAGScheduler:59 - Job 17 finished: sum at KMeansModel.scala:88, took 0.030733 s -2016-06-13 17:02:42 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 17:02:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 17:02:43 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 17:02:43 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 17:02:43 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 17:02:43 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 17:02:43 INFO BlockManager:59 - BlockManager stopped -2016-06-13 17:02:43 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 17:02:43 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 17:02:43 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 17:02:43 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 17:02:43 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-d257938b-6ffc-4ac8-849e-fbd25bf36892 -2016-06-13 17:02:43 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 17:02:43 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 17:12:47 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 17:12:47 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 17:12:47 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 17:12:47 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 17:12:47 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 17:12:48 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 17:12:48 INFO Remoting:74 - Starting remoting -2016-06-13 17:12:48 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:61565] -2016-06-13 17:12:48 INFO Utils:59 - Successfully started service 'sparkDriver' on port 61565. -2016-06-13 17:12:48 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 17:12:48 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 17:12:48 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-bb1842e2-fc10-400b-97f0-d76ef8a925bd -2016-06-13 17:12:48 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 17:12:49 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-530e8ae5-fa7d-4d4b-9b6a-995d848421c3\httpd-a8f03c05-c434-45d3-9555-a655602590cd -2016-06-13 17:12:49 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 17:12:49 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:12:49 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:61566 -2016-06-13 17:12:49 INFO Utils:59 - Successfully started service 'HTTP file server' on port 61566. -2016-06-13 17:12:49 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 17:12:49 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:12:49 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 17:12:49 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 17:12:49 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 17:12:49 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 17:12:49 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 17:12:49 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 61585. -2016-06-13 17:12:49 INFO NettyBlockTransferService:59 - Server created on 61585 -2016-06-13 17:12:49 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 17:12:49 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:61585 with 730.6 MB RAM, BlockManagerId(driver, localhost, 61585) -2016-06-13 17:12:49 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 17:12:50 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 17:12:50 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 17:12:50 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 17:12:50 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 17:12:50 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:61585 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 17:12:50 INFO SparkContext:59 - Created broadcast 0 from textFile at GaussianMixtureSuite.scala:23 -2016-06-13 17:12:51 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 17:12:51 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 17:12:52 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:12:52 INFO DAGScheduler:59 - Got job 0 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:12:52 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GaussianMixture.scala:171) -2016-06-13 17:12:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:52 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:52 INFO MemoryStore:59 - ensureFreeSpace(3584) called with curMem=171580, maxMem=766075207 -2016-06-13 17:12:52 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 17:12:52 INFO MemoryStore:59 - ensureFreeSpace(2049) called with curMem=175164, maxMem=766075207 -2016-06-13 17:12:52 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 17:12:52 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:61585 (size: 2.0 KB, free: 730.6 MB) -2016-06-13 17:12:52 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:52 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:52 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 17:12:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:52 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 17:12:52 INFO CacheManager:59 - Partition rdd_3_0 not found, computing it -2016-06-13 17:12:52 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-06-13 17:12:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/gmm_data.txt:0+31986 -2016-06-13 17:12:52 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 17:12:52 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 17:12:52 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 17:12:52 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 17:12:52 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 17:12:52 INFO MemoryStore:59 - ensureFreeSpace(52024) called with curMem=177213, maxMem=766075207 -2016-06-13 17:12:52 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 50.8 KB, free 730.4 MB) -2016-06-13 17:12:52 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:61585 (size: 50.8 KB, free: 730.5 MB) -2016-06-13 17:12:52 INFO MemoryStore:59 - ensureFreeSpace(76024) called with curMem=229237, maxMem=766075207 -2016-06-13 17:12:52 INFO MemoryStore:59 - Block rdd_3_0 stored as values in memory (estimated size 74.2 KB, free 730.3 MB) -2016-06-13 17:12:52 INFO BlockManagerInfo:59 - Added rdd_3_0 in memory on localhost:61585 (size: 74.2 KB, free: 730.4 MB) -2016-06-13 17:12:53 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2846 bytes result sent to driver -2016-06-13 17:12:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 827 ms on localhost (1/1) -2016-06-13 17:12:53 INFO DAGScheduler:59 - ResultStage 0 (first at GaussianMixture.scala:171) finished in 0.841 s -2016-06-13 17:12:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 17:12:53 INFO DAGScheduler:59 - Job 0 finished: first at GaussianMixture.scala:171, took 0.983023 s -2016-06-13 17:12:53 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:12:53 INFO DAGScheduler:59 - Got job 1 (takeSample at GaussianMixture.scala:184) with 2 output partitions -2016-06-13 17:12:53 INFO DAGScheduler:59 - Final stage: ResultStage 1(takeSample at GaussianMixture.scala:184) -2016-06-13 17:12:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:53 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:53 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:53 INFO MemoryStore:59 - ensureFreeSpace(3424) called with curMem=305261, maxMem=766075207 -2016-06-13 17:12:53 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.3 KB, free 730.3 MB) -2016-06-13 17:12:53 INFO MemoryStore:59 - ensureFreeSpace(1984) called with curMem=308685, maxMem=766075207 -2016-06-13 17:12:53 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1984.0 B, free 730.3 MB) -2016-06-13 17:12:53 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:61585 (size: 1984.0 B, free: 730.4 MB) -2016-06-13 17:12:53 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:53 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 17:12:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:53 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 17:12:53 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 17:12:53 INFO CacheManager:59 - Partition rdd_3_1 not found, computing it -2016-06-13 17:12:53 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-06-13 17:12:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/gmm_data.txt:31986+31987 -2016-06-13 17:12:53 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:53 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 17:12:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 41 ms on localhost (1/2) -2016-06-13 17:12:53 INFO MemoryStore:59 - ensureFreeSpace(47216) called with curMem=310669, maxMem=766075207 -2016-06-13 17:12:53 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 46.1 KB, free 730.2 MB) -2016-06-13 17:12:53 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:61585 (size: 46.1 KB, free: 730.4 MB) -2016-06-13 17:12:53 INFO MemoryStore:59 - ensureFreeSpace(68816) called with curMem=357885, maxMem=766075207 -2016-06-13 17:12:53 INFO MemoryStore:59 - Block rdd_3_1 stored as values in memory (estimated size 67.2 KB, free 730.2 MB) -2016-06-13 17:12:53 INFO BlockManagerInfo:59 - Added rdd_3_1 in memory on localhost:61585 (size: 67.2 KB, free: 730.3 MB) -2016-06-13 17:12:53 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2717 bytes result sent to driver -2016-06-13 17:12:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 101 ms on localhost (2/2) -2016-06-13 17:12:53 INFO DAGScheduler:59 - ResultStage 1 (takeSample at GaussianMixture.scala:184) finished in 0.113 s -2016-06-13 17:12:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 17:12:53 INFO DAGScheduler:59 - Job 1 finished: takeSample at GaussianMixture.scala:184, took 0.135945 s -2016-06-13 17:12:53 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:12:53 INFO DAGScheduler:59 - Got job 2 (takeSample at GaussianMixture.scala:184) with 2 output partitions -2016-06-13 17:12:53 INFO DAGScheduler:59 - Final stage: ResultStage 2(takeSample at GaussianMixture.scala:184) -2016-06-13 17:12:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:53 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:53 INFO DAGScheduler:59 - Submitting ResultStage 2 (PartitionwiseSampledRDD[4] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:12:53 INFO MemoryStore:59 - ensureFreeSpace(20816) called with curMem=426701, maxMem=766075207 -2016-06-13 17:12:53 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 20.3 KB, free 730.2 MB) -2016-06-13 17:12:53 INFO MemoryStore:59 - ensureFreeSpace(8648) called with curMem=447517, maxMem=766075207 -2016-06-13 17:12:53 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 8.4 KB, free 730.2 MB) -2016-06-13 17:12:53 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:61585 (size: 8.4 KB, free: 730.3 MB) -2016-06-13 17:12:53 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (PartitionwiseSampledRDD[4] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:12:53 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 17:12:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2299 bytes) -2016-06-13 17:12:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2299 bytes) -2016-06-13 17:12:53 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 17:12:53 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-06-13 17:12:53 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:53 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:53 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2698 bytes result sent to driver -2016-06-13 17:12:53 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 3006 bytes result sent to driver -2016-06-13 17:12:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 51 ms on localhost (1/2) -2016-06-13 17:12:53 INFO DAGScheduler:59 - ResultStage 2 (takeSample at GaussianMixture.scala:184) finished in 0.052 s -2016-06-13 17:12:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 49 ms on localhost (2/2) -2016-06-13 17:12:53 INFO DAGScheduler:59 - Job 2 finished: takeSample at GaussianMixture.scala:184, took 0.068132 s -2016-06-13 17:12:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 17:12:53 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 17:12:53 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 17:12:53 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 17:12:53 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=456165, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 840.0 B, free 730.2 MB) -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(1001) called with curMem=457005, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 1001.0 B, free 730.1 MB) -2016-06-13 17:12:54 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:61585 (size: 1001.0 B, free: 730.3 MB) -2016-06-13 17:12:54 INFO SparkContext:59 - Created broadcast 4 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:54 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:54 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:61585 in memory (size: 8.4 KB, free: 730.3 MB) -2016-06-13 17:12:54 INFO DAGScheduler:59 - Got job 3 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:54 INFO DAGScheduler:59 - Final stage: ResultStage 3(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:54 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:54 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=425118, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.7 KB, free 730.2 MB) -2016-06-13 17:12:54 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:61585 in memory (size: 1984.0 B, free: 730.3 MB) -2016-06-13 17:12:54 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(3075) called with curMem=429014, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-06-13 17:12:54 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:54 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:54 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:61585 in memory (size: 2.0 KB, free: 730.3 MB) -2016-06-13 17:12:54 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 17:12:54 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 17:12:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:54 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-06-13 17:12:54 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-06-13 17:12:54 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:54 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:54 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2826 bytes result sent to driver -2016-06-13 17:12:54 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2826 bytes result sent to driver -2016-06-13 17:12:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 208 ms on localhost (1/2) -2016-06-13 17:12:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 210 ms on localhost (2/2) -2016-06-13 17:12:54 INFO DAGScheduler:59 - ResultStage 3 (aggregate at GaussianMixture.scala:201) finished in 0.210 s -2016-06-13 17:12:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 17:12:54 INFO DAGScheduler:59 - Job 3 finished: aggregate at GaussianMixture.scala:201, took 0.226812 s -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=426456, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 840.0 B, free 730.2 MB) -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(1034) called with curMem=427296, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1034.0 B, free 730.2 MB) -2016-06-13 17:12:54 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:61585 (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:12:54 INFO SparkContext:59 - Created broadcast 6 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:54 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:54 INFO DAGScheduler:59 - Got job 4 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:54 INFO DAGScheduler:59 - Final stage: ResultStage 4(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:54 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=428330, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.7 KB, free 730.2 MB) -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=434210, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-06-13 17:12:54 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:54 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:54 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 17:12:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:54 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-06-13 17:12:54 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-06-13 17:12:54 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:54 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:54 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2826 bytes result sent to driver -2016-06-13 17:12:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 117 ms on localhost (1/2) -2016-06-13 17:12:54 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2826 bytes result sent to driver -2016-06-13 17:12:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 119 ms on localhost (2/2) -2016-06-13 17:12:54 INFO DAGScheduler:59 - ResultStage 4 (aggregate at GaussianMixture.scala:201) finished in 0.120 s -2016-06-13 17:12:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 17:12:54 INFO DAGScheduler:59 - Job 4 finished: aggregate at GaussianMixture.scala:201, took 0.134304 s -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=437323, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 840.0 B, free 730.2 MB) -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=438163, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 1032.0 B, free 730.2 MB) -2016-06-13 17:12:54 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:61585 (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:12:54 INFO SparkContext:59 - Created broadcast 8 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:54 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:54 INFO DAGScheduler:59 - Got job 5 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:54 INFO DAGScheduler:59 - Final stage: ResultStage 5(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:54 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=439195, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.7 KB, free 730.2 MB) -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(3115) called with curMem=445075, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-06-13 17:12:54 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:54 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:54 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 17:12:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:54 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-06-13 17:12:54 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-06-13 17:12:54 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:54 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:54 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2826 bytes result sent to driver -2016-06-13 17:12:54 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2826 bytes result sent to driver -2016-06-13 17:12:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 61 ms on localhost (1/2) -2016-06-13 17:12:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 64 ms on localhost (2/2) -2016-06-13 17:12:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 17:12:54 INFO DAGScheduler:59 - ResultStage 5 (aggregate at GaussianMixture.scala:201) finished in 0.064 s -2016-06-13 17:12:54 INFO DAGScheduler:59 - Job 5 finished: aggregate at GaussianMixture.scala:201, took 0.078265 s -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=448190, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 840.0 B, free 730.2 MB) -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=449030, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 1032.0 B, free 730.2 MB) -2016-06-13 17:12:54 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:61585 (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:12:54 INFO SparkContext:59 - Created broadcast 10 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:54 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:54 INFO DAGScheduler:59 - Got job 6 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:54 INFO DAGScheduler:59 - Final stage: ResultStage 6(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:54 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=450062, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.7 KB, free 730.2 MB) -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(3114) called with curMem=455942, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:54 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:54 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:54 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-06-13 17:12:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:54 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-06-13 17:12:54 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-06-13 17:12:54 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:54 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:54 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2826 bytes result sent to driver -2016-06-13 17:12:54 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2826 bytes result sent to driver -2016-06-13 17:12:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 28 ms on localhost (1/2) -2016-06-13 17:12:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 33 ms on localhost (2/2) -2016-06-13 17:12:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 17:12:54 INFO DAGScheduler:59 - ResultStage 6 (aggregate at GaussianMixture.scala:201) finished in 0.033 s -2016-06-13 17:12:54 INFO DAGScheduler:59 - Job 6 finished: aggregate at GaussianMixture.scala:201, took 0.051131 s -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=459056, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(1034) called with curMem=459896, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 1034.0 B, free 730.1 MB) -2016-06-13 17:12:54 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:61585 (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:12:54 INFO SparkContext:59 - Created broadcast 12 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:54 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:54 INFO DAGScheduler:59 - Got job 7 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:54 INFO DAGScheduler:59 - Final stage: ResultStage 7(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:54 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=460930, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=466810, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:54 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:54 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:54 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-06-13 17:12:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:54 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-06-13 17:12:54 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-06-13 17:12:54 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:54 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:54 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2826 bytes result sent to driver -2016-06-13 17:12:54 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2826 bytes result sent to driver -2016-06-13 17:12:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 17 ms on localhost (1/2) -2016-06-13 17:12:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 19 ms on localhost (2/2) -2016-06-13 17:12:54 INFO DAGScheduler:59 - ResultStage 7 (aggregate at GaussianMixture.scala:201) finished in 0.020 s -2016-06-13 17:12:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 17:12:54 INFO DAGScheduler:59 - Job 7 finished: aggregate at GaussianMixture.scala:201, took 0.032635 s -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=469923, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=470763, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 1032.0 B, free 730.1 MB) -2016-06-13 17:12:54 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:61585 (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:12:54 INFO SparkContext:59 - Created broadcast 14 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:54 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:54 INFO DAGScheduler:59 - Got job 8 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:54 INFO DAGScheduler:59 - Final stage: ResultStage 8(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:54 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=471795, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=477675, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:54 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:54 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:54 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 17:12:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:54 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-06-13 17:12:54 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-06-13 17:12:54 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:54 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:54 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2826 bytes result sent to driver -2016-06-13 17:12:54 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2826 bytes result sent to driver -2016-06-13 17:12:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 23 ms on localhost (1/2) -2016-06-13 17:12:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 23 ms on localhost (2/2) -2016-06-13 17:12:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 17:12:54 INFO DAGScheduler:59 - ResultStage 8 (aggregate at GaussianMixture.scala:201) finished in 0.024 s -2016-06-13 17:12:54 INFO DAGScheduler:59 - Job 8 finished: aggregate at GaussianMixture.scala:201, took 0.036834 s -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=480788, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(1033) called with curMem=481628, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1033.0 B, free 730.1 MB) -2016-06-13 17:12:54 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:61585 (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:12:54 INFO SparkContext:59 - Created broadcast 16 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:54 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:54 INFO DAGScheduler:59 - Got job 9 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:54 INFO DAGScheduler:59 - Final stage: ResultStage 9(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:54 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=482661, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=488541, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:54 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:54 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:54 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-06-13 17:12:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:54 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-06-13 17:12:54 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-06-13 17:12:54 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:54 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:54 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2826 bytes result sent to driver -2016-06-13 17:12:54 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2826 bytes result sent to driver -2016-06-13 17:12:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 39 ms on localhost (1/2) -2016-06-13 17:12:54 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 40 ms on localhost (2/2) -2016-06-13 17:12:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 17:12:54 INFO DAGScheduler:59 - ResultStage 9 (aggregate at GaussianMixture.scala:201) finished in 0.044 s -2016-06-13 17:12:54 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 17:12:54 INFO DAGScheduler:59 - Job 9 finished: aggregate at GaussianMixture.scala:201, took 0.055312 s -2016-06-13 17:12:54 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=481629, maxMem=766075207 -2016-06-13 17:12:54 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:54 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:61585 in memory (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(1034) called with curMem=481629, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1034.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:61585 (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 18 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:55 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:61585 in memory (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Got job 10 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:55 INFO DAGScheduler:59 - Final stage: ResultStage 10(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:55 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:61585 in memory (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=460930, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(3117) called with curMem=460930, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:61585 in memory (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-06-13 17:12:55 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:55 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 26 ms on localhost (1/2) -2016-06-13 17:12:55 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 27 ms on localhost (2/2) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 17:12:55 INFO DAGScheduler:59 - ResultStage 10 (aggregate at GaussianMixture.scala:201) finished in 0.029 s -2016-06-13 17:12:55 INFO DAGScheduler:59 - Job 10 finished: aggregate at GaussianMixture.scala:201, took 0.048317 s -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=449033, maxMem=766075207 -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:61585 in memory (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 840.0 B, free 730.2 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(1034) called with curMem=445958, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 1034.0 B, free 730.2 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:61585 (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 20 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:61585 in memory (size: 1001.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Got job 11 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:55 INFO DAGScheduler:59 - Final stage: ResultStage 11(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=439271, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.7 KB, free 730.2 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=445151, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-06-13 17:12:55 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:55 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 16 ms on localhost (1/2) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 17 ms on localhost (2/2) -2016-06-13 17:12:55 INFO DAGScheduler:59 - ResultStage 11 (aggregate at GaussianMixture.scala:201) finished in 0.018 s -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 17:12:55 INFO DAGScheduler:59 - Job 11 finished: aggregate at GaussianMixture.scala:201, took 0.033441 s -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=448264, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 840.0 B, free 730.2 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(1034) called with curMem=449104, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 1034.0 B, free 730.2 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:61585 (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 22 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Got job 12 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:55 INFO DAGScheduler:59 - Final stage: ResultStage 12(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=450138, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 5.7 KB, free 730.2 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(3116) called with curMem=456018, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-06-13 17:12:55 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:55 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 13 ms on localhost (1/2) -2016-06-13 17:12:55 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 15 ms on localhost (2/2) -2016-06-13 17:12:55 INFO DAGScheduler:59 - ResultStage 12 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 17:12:55 INFO DAGScheduler:59 - Job 12 finished: aggregate at GaussianMixture.scala:201, took 0.030127 s -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=459134, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(1033) called with curMem=459974, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 1033.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:61585 (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 24 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Got job 13 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:55 INFO DAGScheduler:59 - Final stage: ResultStage 13(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=461007, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=466887, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-06-13 17:12:55 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:55 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 15 ms on localhost (1/2) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 14 ms on localhost (2/2) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 17:12:55 INFO DAGScheduler:59 - ResultStage 13 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 17:12:55 INFO DAGScheduler:59 - Job 13 finished: aggregate at GaussianMixture.scala:201, took 0.030591 s -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=470000, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(1031) called with curMem=470840, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 1031.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:61585 (size: 1031.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 26 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Got job 14 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:55 INFO DAGScheduler:59 - Final stage: ResultStage 14(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=471871, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=477751, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-06-13 17:12:55 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:55 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 15 ms on localhost (1/2) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 15 ms on localhost (2/2) -2016-06-13 17:12:55 INFO DAGScheduler:59 - ResultStage 14 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 17:12:55 INFO DAGScheduler:59 - Job 14 finished: aggregate at GaussianMixture.scala:201, took 0.030353 s -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=480864, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=481704, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1032.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:61585 (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 28 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Got job 15 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:55 INFO DAGScheduler:59 - Final stage: ResultStage 15(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=482736, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=488616, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-06-13 17:12:55 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:55 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 17 ms on localhost (1/2) -2016-06-13 17:12:55 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 22 ms on localhost (2/2) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 17:12:55 INFO DAGScheduler:59 - ResultStage 15 (aggregate at GaussianMixture.scala:201) finished in 0.023 s -2016-06-13 17:12:55 INFO DAGScheduler:59 - Job 15 finished: aggregate at GaussianMixture.scala:201, took 0.036504 s -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=491728, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=492568, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1032.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:61585 (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 30 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Got job 16 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:55 INFO DAGScheduler:59 - Final stage: ResultStage 16(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=493600, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=499480, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-06-13 17:12:55 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:55 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 16 ms on localhost (1/2) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 15 ms on localhost (2/2) -2016-06-13 17:12:55 INFO DAGScheduler:59 - ResultStage 16 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 17:12:55 INFO DAGScheduler:59 - Job 16 finished: aggregate at GaussianMixture.scala:201, took 0.031372 s -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=502593, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(1034) called with curMem=503433, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 1034.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:61585 (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 32 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Got job 17 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:55 INFO DAGScheduler:59 - Final stage: ResultStage 17(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=504467, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(3114) called with curMem=510347, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-06-13 17:12:55 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:55 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 13 ms on localhost (1/2) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 15 ms on localhost (2/2) -2016-06-13 17:12:55 INFO DAGScheduler:59 - ResultStage 17 (aggregate at GaussianMixture.scala:201) finished in 0.015 s -2016-06-13 17:12:55 INFO DAGScheduler:59 - Job 17 finished: aggregate at GaussianMixture.scala:201, took 0.029088 s -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=513461, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(1036) called with curMem=514301, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 1036.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:61585 (size: 1036.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 34 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Got job 18 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:55 INFO DAGScheduler:59 - Final stage: ResultStage 18(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=515337, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(3114) called with curMem=521217, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-06-13 17:12:55 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:55 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 16 ms on localhost (1/2) -2016-06-13 17:12:55 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 19 ms on localhost (2/2) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 17:12:55 INFO DAGScheduler:59 - ResultStage 18 (aggregate at GaussianMixture.scala:201) finished in 0.020 s -2016-06-13 17:12:55 INFO DAGScheduler:59 - Job 18 finished: aggregate at GaussianMixture.scala:201, took 0.032978 s -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=524331, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(1033) called with curMem=525171, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 1033.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:61585 (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 36 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Got job 19 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:55 INFO DAGScheduler:59 - Final stage: ResultStage 19(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=526204, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=532084, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-06-13 17:12:55 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:55 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 16 ms on localhost (1/2) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 22 ms on localhost (2/2) -2016-06-13 17:12:55 INFO DAGScheduler:59 - ResultStage 19 (aggregate at GaussianMixture.scala:201) finished in 0.022 s -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 17:12:55 INFO DAGScheduler:59 - Job 19 finished: aggregate at GaussianMixture.scala:201, took 0.039341 s -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=535196, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(1036) called with curMem=536036, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 1036.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:61585 (size: 1036.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 38 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Got job 20 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:55 INFO DAGScheduler:59 - Final stage: ResultStage 20(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=537072, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=542952, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-06-13 17:12:55 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:55 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 19 ms on localhost (1/2) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 21 ms on localhost (2/2) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 17:12:55 INFO DAGScheduler:59 - ResultStage 20 (aggregate at GaussianMixture.scala:201) finished in 0.022 s -2016-06-13 17:12:55 INFO DAGScheduler:59 - Job 20 finished: aggregate at GaussianMixture.scala:201, took 0.039965 s -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=546065, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(1034) called with curMem=546905, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 1034.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:61585 (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 40 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Got job 21 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:55 INFO DAGScheduler:59 - Final stage: ResultStage 21(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=547939, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=553819, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-06-13 17:12:55 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:55 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 15 ms on localhost (1/2) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 17 ms on localhost (2/2) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 17:12:55 INFO DAGScheduler:59 - ResultStage 21 (aggregate at GaussianMixture.scala:201) finished in 0.018 s -2016-06-13 17:12:55 INFO DAGScheduler:59 - Job 21 finished: aggregate at GaussianMixture.scala:201, took 0.035466 s -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=556932, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(1034) called with curMem=557772, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1034.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:61585 (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 42 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Got job 22 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:55 INFO DAGScheduler:59 - Final stage: ResultStage 22(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=558806, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 5.7 KB, free 730.0 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=564686, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.0 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-06-13 17:12:55 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:55 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 16 ms on localhost (1/2) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 19 ms on localhost (2/2) -2016-06-13 17:12:55 INFO DAGScheduler:59 - ResultStage 22 (aggregate at GaussianMixture.scala:201) finished in 0.019 s -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 17:12:55 INFO DAGScheduler:59 - Job 22 finished: aggregate at GaussianMixture.scala:201, took 0.032112 s -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=567799, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 840.0 B, free 730.0 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(1036) called with curMem=568639, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 1036.0 B, free 730.0 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:61585 (size: 1036.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 44 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Got job 23 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:55 INFO DAGScheduler:59 - Final stage: ResultStage 23(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=569675, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 5.7 KB, free 730.0 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(3116) called with curMem=575555, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.0 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-06-13 17:12:55 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:55 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 29 ms on localhost (1/2) -2016-06-13 17:12:55 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 36 ms on localhost (2/2) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 17:12:55 INFO DAGScheduler:59 - ResultStage 23 (aggregate at GaussianMixture.scala:201) finished in 0.038 s -2016-06-13 17:12:55 INFO DAGScheduler:59 - Job 23 finished: aggregate at GaussianMixture.scala:201, took 0.050611 s -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=567804, maxMem=766075207 -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:61585 in memory (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 840.0 B, free 730.0 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(1043) called with curMem=568644, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 1043.0 B, free 730.0 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:61585 (size: 1043.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 46 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:61585 in memory (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Got job 24 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Final stage: ResultStage 24(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:55 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=549827, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(3115) called with curMem=554671, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:61585 in memory (size: 1036.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-06-13 17:12:55 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:55 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 16 ms on localhost (1/2) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 18 ms on localhost (2/2) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-06-13 17:12:55 INFO DAGScheduler:59 - ResultStage 24 (aggregate at GaussianMixture.scala:201) finished in 0.018 s -2016-06-13 17:12:55 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Job 24 finished: aggregate at GaussianMixture.scala:201, took 0.032367 s -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=547954, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(1035) called with curMem=547761, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 1035.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:61585 in memory (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:61585 (size: 1035.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 48 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 17:12:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:61585 in memory (size: 1036.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Got job 25 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:55 INFO DAGScheduler:59 - Final stage: ResultStage 25(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=533972, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=539852, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:61585 in memory (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:55 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 17 ms on localhost (1/2) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 18 ms on localhost (2/2) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-06-13 17:12:55 INFO DAGScheduler:59 - ResultStage 25 (aggregate at GaussianMixture.scala:201) finished in 0.020 s -2016-06-13 17:12:55 INFO DAGScheduler:59 - Job 25 finished: aggregate at GaussianMixture.scala:201, took 0.036765 s -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=526218, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(1033) called with curMem=527058, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 1033.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:61585 in memory (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:61585 (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 50 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 17:12:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:61585 in memory (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Got job 26 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:55 INFO DAGScheduler:59 - Final stage: ResultStage 26(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=506362, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(3117) called with curMem=512242, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:61585 in memory (size: 1031.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:55 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:55 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 20 ms on localhost (1/2) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 20 ms on localhost (2/2) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-06-13 17:12:55 INFO DAGScheduler:59 - ResultStage 26 (aggregate at GaussianMixture.scala:201) finished in 0.022 s -2016-06-13 17:12:55 INFO DAGScheduler:59 - Job 26 finished: aggregate at GaussianMixture.scala:201, took 0.037244 s -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=503462, maxMem=766075207 -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:61585 in memory (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(1031) called with curMem=503462, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 1031.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:61585 (size: 1031.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 52 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:61585 in memory (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Got job 27 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:55 INFO DAGScheduler:59 - Final stage: ResultStage 27(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:55 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=484630, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=488636, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:61585 in memory (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:55 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:55 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 2826 bytes result sent to driver -2016-06-13 17:12:55 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 16 ms on localhost (1/2) -2016-06-13 17:12:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 18 ms on localhost (2/2) -2016-06-13 17:12:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-06-13 17:12:55 INFO DAGScheduler:59 - ResultStage 27 (aggregate at GaussianMixture.scala:201) finished in 0.020 s -2016-06-13 17:12:55 INFO DAGScheduler:59 - Job 27 finished: aggregate at GaussianMixture.scala:201, took 0.033973 s -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:61585 in memory (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=480878, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(1033) called with curMem=478605, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 1033.0 B, free 730.1 MB) -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:55 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:61585 (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Created broadcast 54 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:55 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:61585 in memory (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:12:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:55 INFO DAGScheduler:59 - Got job 28 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:55 INFO DAGScheduler:59 - Final stage: ResultStage 28(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:55 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=471885, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:55 INFO MemoryStore:59 - ensureFreeSpace(3114) called with curMem=477765, maxMem=766075207 -2016-06-13 17:12:55 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-06-13 17:12:56 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:56 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 13 ms on localhost (1/2) -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 17 ms on localhost (2/2) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-06-13 17:12:56 INFO DAGScheduler:59 - ResultStage 28 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 17:12:56 INFO DAGScheduler:59 - Job 28 finished: aggregate at GaussianMixture.scala:201, took 0.029696 s -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=480879, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=481719, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 1032.0 B, free 730.1 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:61585 (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 56 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Got job 29 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:56 INFO DAGScheduler:59 - Final stage: ResultStage 29(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=482751, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(3117) called with curMem=488631, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-06-13 17:12:56 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:56 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 15 ms on localhost (1/2) -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 21 ms on localhost (2/2) -2016-06-13 17:12:56 INFO DAGScheduler:59 - ResultStage 29 (aggregate at GaussianMixture.scala:201) finished in 0.021 s -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 17:12:56 INFO DAGScheduler:59 - Job 29 finished: aggregate at GaussianMixture.scala:201, took 0.034666 s -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=491748, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=492588, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 1032.0 B, free 730.1 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:61585 (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 58 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Got job 30 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:56 INFO DAGScheduler:59 - Final stage: ResultStage 30(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=493620, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(3118) called with curMem=499500, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:56 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 17 ms on localhost (1/2) -2016-06-13 17:12:56 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:56 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 33 ms on localhost (2/2) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 17:12:56 INFO DAGScheduler:59 - ResultStage 30 (aggregate at GaussianMixture.scala:201) finished in 0.035 s -2016-06-13 17:12:56 INFO DAGScheduler:59 - Job 30 finished: aggregate at GaussianMixture.scala:201, took 0.052643 s -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=502618, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(1036) called with curMem=503458, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 1036.0 B, free 730.1 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:61585 (size: 1036.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 60 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Got job 31 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:56 INFO DAGScheduler:59 - Final stage: ResultStage 31(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=504494, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(3116) called with curMem=510374, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 61 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:56 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 13 ms on localhost (1/2) -2016-06-13 17:12:56 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:56 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 28 ms on localhost (2/2) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 17:12:56 INFO DAGScheduler:59 - ResultStage 31 (aggregate at GaussianMixture.scala:201) finished in 0.029 s -2016-06-13 17:12:56 INFO DAGScheduler:59 - Job 31 finished: aggregate at GaussianMixture.scala:201, took 0.055109 s -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=513490, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(1038) called with curMem=514330, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 1038.0 B, free 730.1 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:61585 (size: 1038.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 62 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Got job 32 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:56 INFO DAGScheduler:59 - Final stage: ResultStage 32(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=515368, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=521248, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:56 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 16 ms on localhost (1/2) -2016-06-13 17:12:56 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:56 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 36 ms on localhost (2/2) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 17:12:56 INFO DAGScheduler:59 - ResultStage 32 (aggregate at GaussianMixture.scala:201) finished in 0.037 s -2016-06-13 17:12:56 INFO DAGScheduler:59 - Job 32 finished: aggregate at GaussianMixture.scala:201, took 0.052689 s -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=524361, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(1033) called with curMem=525201, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 1033.0 B, free 730.1 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:61585 (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 64 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Got job 33 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:56 INFO DAGScheduler:59 - Final stage: ResultStage 33(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=526234, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=532114, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:56 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 12 ms on localhost (1/2) -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:56 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 22 ms on localhost (2/2) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 17:12:56 INFO DAGScheduler:59 - ResultStage 33 (aggregate at GaussianMixture.scala:201) finished in 0.024 s -2016-06-13 17:12:56 INFO DAGScheduler:59 - Job 33 finished: aggregate at GaussianMixture.scala:201, took 0.052512 s -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=535227, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(1031) called with curMem=536067, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 1031.0 B, free 730.1 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:61585 (size: 1031.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 66 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Got job 34 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:56 INFO DAGScheduler:59 - Final stage: ResultStage 34(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=537098, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=542978, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 67 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-06-13 17:12:56 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:56 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 13 ms on localhost (1/2) -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 14 ms on localhost (2/2) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-06-13 17:12:56 INFO DAGScheduler:59 - ResultStage 34 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 17:12:56 INFO DAGScheduler:59 - Job 34 finished: aggregate at GaussianMixture.scala:201, took 0.037859 s -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=546090, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=546930, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 1032.0 B, free 730.1 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:61585 (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 68 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Got job 35 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:56 INFO DAGScheduler:59 - Final stage: ResultStage 35(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=547962, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=553842, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 69 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-06-13 17:12:56 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:56 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 12 ms on localhost (1/2) -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 13 ms on localhost (2/2) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-06-13 17:12:56 INFO DAGScheduler:59 - ResultStage 35 (aggregate at GaussianMixture.scala:201) finished in 0.014 s -2016-06-13 17:12:56 INFO DAGScheduler:59 - Job 35 finished: aggregate at GaussianMixture.scala:201, took 0.026416 s -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=556954, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(1036) called with curMem=557794, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 1036.0 B, free 730.1 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:61585 (size: 1036.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 70 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Got job 36 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:56 INFO DAGScheduler:59 - Final stage: ResultStage 36(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=558830, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 5.7 KB, free 730.0 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=564710, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.0 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 71 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-06-13 17:12:56 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:56 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 12 ms on localhost (1/2) -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 11 ms on localhost (2/2) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-06-13 17:12:56 INFO DAGScheduler:59 - ResultStage 36 (aggregate at GaussianMixture.scala:201) finished in 0.012 s -2016-06-13 17:12:56 INFO DAGScheduler:59 - Job 36 finished: aggregate at GaussianMixture.scala:201, took 0.023635 s -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=567822, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 840.0 B, free 730.0 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(1036) called with curMem=568662, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 1036.0 B, free 730.0 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:61585 (size: 1036.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 72 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Got job 37 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:56 INFO DAGScheduler:59 - Final stage: ResultStage 37(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=569698, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 5.7 KB, free 730.0 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=575578, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.0 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 73 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-06-13 17:12:56 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:56 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 11 ms on localhost (1/2) -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 11 ms on localhost (2/2) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-06-13 17:12:56 INFO DAGScheduler:59 - ResultStage 37 (aggregate at GaussianMixture.scala:201) finished in 0.012 s -2016-06-13 17:12:56 INFO DAGScheduler:59 - Job 37 finished: aggregate at GaussianMixture.scala:201, took 0.024245 s -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=578691, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 840.0 B, free 730.0 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(1038) called with curMem=579531, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 1038.0 B, free 730.0 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:61585 in memory (size: 1043.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:61585 (size: 1038.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 74 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:12:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO DAGScheduler:59 - Got job 38 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:12:56 INFO DAGScheduler:59 - Final stage: ResultStage 38(aggregate at GaussianMixture.scala:201) -2016-06-13 17:12:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:56 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=567817, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 5.7 KB, free 730.0 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:61585 in memory (size: 1036.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(3109) called with curMem=573697, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.0 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 75 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-06-13 17:12:56 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:12:56 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:61585 in memory (size: 1036.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:12:56 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 11 ms on localhost (1/2) -2016-06-13 17:12:56 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 2826 bytes result sent to driver -2016-06-13 17:12:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 15 ms on localhost (2/2) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-06-13 17:12:56 INFO DAGScheduler:59 - ResultStage 38 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 17:12:56 INFO DAGScheduler:59 - Job 38 finished: aggregate at GaussianMixture.scala:201, took 0.030457 s -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:61585 in memory (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:61585 in memory (size: 1031.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:61585 in memory (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:61585 in memory (size: 1038.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:61585 in memory (size: 1036.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:61585 in memory (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:61585 in memory (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:61585 in memory (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:61585 in memory (size: 1031.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:61585 in memory (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:61585 in memory (size: 1035.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:61585 in memory (size: 1036.0 B, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Starting job: saveAsTextFile at GaussianMixtureModel.scala:158 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Got job 39 (saveAsTextFile at GaussianMixtureModel.scala:158) with 1 output partitions -2016-06-13 17:12:56 INFO DAGScheduler:59 - Final stage: ResultStage 39(saveAsTextFile at GaussianMixtureModel.scala:158) -2016-06-13 17:12:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[6] at saveAsTextFile at GaussianMixtureModel.scala:158), which has no missing parents -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(126968) called with curMem=426527, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 124.0 KB, free 730.1 MB) -2016-06-13 17:12:56 INFO MemoryStore:59 - ensureFreeSpace(42558) called with curMem=553495, maxMem=766075207 -2016-06-13 17:12:56 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 41.6 KB, free 730.0 MB) -2016-06-13 17:12:56 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:61585 (size: 41.6 KB, free: 730.3 MB) -2016-06-13 17:12:56 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:12:56 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 39 (MapPartitionsRDD[6] at saveAsTextFile at GaussianMixtureModel.scala:158) -2016-06-13 17:12:56 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 1 tasks -2016-06-13 17:12:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2178 bytes) -2016-06-13 17:12:56 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-06-13 17:12:57 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201606131712_0039_m_000000_77' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/mllibFromSpark/myGMMModel20160613171256600/metadata/_temporary/0/task_201606131712_0039_m_000000 -2016-06-13 17:12:57 INFO SparkHadoopMapRedUtil:59 - attempt_201606131712_0039_m_000000_77: Committed -2016-06-13 17:12:57 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 1864 bytes result sent to driver -2016-06-13 17:12:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 612 ms on localhost (1/1) -2016-06-13 17:12:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-06-13 17:12:57 INFO DAGScheduler:59 - ResultStage 39 (saveAsTextFile at GaussianMixtureModel.scala:158) finished in 0.615 s -2016-06-13 17:12:57 INFO DAGScheduler:59 - Job 39 finished: saveAsTextFile at GaussianMixtureModel.scala:158, took 0.680467 s -2016-06-13 17:13:00 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 17:13:00 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 17:13:01 INFO SparkContext:59 - Starting job: parquet at GaussianMixtureModel.scala:164 -2016-06-13 17:13:01 INFO DAGScheduler:59 - Got job 40 (parquet at GaussianMixtureModel.scala:164) with 1 output partitions -2016-06-13 17:13:01 INFO DAGScheduler:59 - Final stage: ResultStage 40(parquet at GaussianMixtureModel.scala:164) -2016-06-13 17:13:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:13:01 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:13:01 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[8] at rddToDataFrameHolder at GaussianMixtureModel.scala:164), which has no missing parents -2016-06-13 17:13:01 INFO MemoryStore:59 - ensureFreeSpace(67888) called with curMem=596053, maxMem=766075207 -2016-06-13 17:13:01 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 66.3 KB, free 730.0 MB) -2016-06-13 17:13:01 INFO MemoryStore:59 - ensureFreeSpace(23636) called with curMem=663941, maxMem=766075207 -2016-06-13 17:13:01 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 23.1 KB, free 729.9 MB) -2016-06-13 17:13:01 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:61585 (size: 23.1 KB, free: 730.3 MB) -2016-06-13 17:13:01 INFO SparkContext:59 - Created broadcast 77 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:13:01 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 40 (MapPartitionsRDD[8] at rddToDataFrameHolder at GaussianMixtureModel.scala:164) -2016-06-13 17:13:01 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 1 tasks -2016-06-13 17:13:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 78, localhost, PROCESS_LOCAL, 2716 bytes) -2016-06-13 17:13:01 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 78) -2016-06-13 17:13:01 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 17:13:01 INFO CodecConfig:151 - Compression: GZIP -2016-06-13 17:13:01 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-06-13 17:13:01 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-06-13 17:13:01 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-06-13 17:13:01 INFO ParquetOutputFormat:151 - Dictionary is on -2016-06-13 17:13:01 INFO ParquetOutputFormat:151 - Validation is off -2016-06-13 17:13:01 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-06-13 17:13:01 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-06-13 17:13:01 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,832 -2016-06-13 17:13:01 INFO ColumnChunkPageWriteStore:151 - written 84B for [weight] DOUBLE: 2 values, 22B raw, 43B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-06-13 17:13:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [mu, type] INT32: 2 values, 8B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN_DICTIONARY, RLE], dic { 1 entries, 4B raw, 1B comp} -2016-06-13 17:13:01 INFO ColumnChunkPageWriteStore:151 - written 48B for [mu, size] INT32: 2 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-06-13 17:13:01 INFO ColumnChunkPageWriteStore:151 - written 51B for [mu, indices, array] INT32: 2 values, 13B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-06-13 17:13:01 INFO ColumnChunkPageWriteStore:151 - written 106B for [mu, values, array] DOUBLE: 4 values, 45B raw, 64B comp, 1 pages, encodings: [PLAIN, RLE] -2016-06-13 17:13:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [sigma, type] INT32: 2 values, 8B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN_DICTIONARY, RLE], dic { 1 entries, 4B raw, 1B comp} -2016-06-13 17:13:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [sigma, numRows] INT32: 2 values, 8B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN_DICTIONARY, RLE], dic { 1 entries, 4B raw, 1B comp} -2016-06-13 17:13:01 INFO ColumnChunkPageWriteStore:151 - written 61B for [sigma, numCols] INT32: 2 values, 8B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN_DICTIONARY, RLE], dic { 1 entries, 4B raw, 1B comp} -2016-06-13 17:13:01 INFO ColumnChunkPageWriteStore:151 - written 51B for [sigma, colPtrs, array] INT32: 2 values, 13B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-06-13 17:13:01 INFO ColumnChunkPageWriteStore:151 - written 51B for [sigma, rowIndices, array] INT32: 2 values, 13B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-06-13 17:13:01 INFO ColumnChunkPageWriteStore:151 - written 77B for [sigma, values, array] DOUBLE: 8 values, 17B raw, 36B comp, 1 pages, encodings: [PLAIN_DICTIONARY, RLE], dic { 6 entries, 48B raw, 6B comp} -2016-06-13 17:13:01 INFO ColumnChunkPageWriteStore:151 - written 54B for [sigma, isTransposed] BOOLEAN: 2 values, 7B raw, 27B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-06-13 17:13:02 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201606131713_0040_m_000000_0' to file:/D:/all/idea/SparkLearning/file/data/mllib/output/mllibFromSpark/myGMMModel20160613171256600/data/_temporary/0/task_201606131713_0040_m_000000 -2016-06-13 17:13:02 INFO SparkHadoopMapRedUtil:59 - attempt_201606131713_0040_m_000000_0: Committed -2016-06-13 17:13:02 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 78). 915 bytes result sent to driver -2016-06-13 17:13:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 78) in 1218 ms on localhost (1/1) -2016-06-13 17:13:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-06-13 17:13:02 INFO DAGScheduler:59 - ResultStage 40 (parquet at GaussianMixtureModel.scala:164) finished in 1.223 s -2016-06-13 17:13:02 INFO DAGScheduler:59 - Job 40 finished: parquet at GaussianMixtureModel.scala:164, took 1.264420 s -2016-06-13 17:13:02 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-06-13 17:13:02 INFO DefaultWriterContainer:59 - Job job_201606131713_0000 committed. -2016-06-13 17:13:02 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/mllibFromSpark/myGMMModel20160613171256600/data on driver -2016-06-13 17:13:02 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/mllibFromSpark/myGMMModel20160613171256600/data on driver -2016-06-13 17:13:02 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=687577, maxMem=766075207 -2016-06-13 17:13:02 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 225.3 KB, free 729.7 MB) -2016-06-13 17:13:02 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=918281, maxMem=766075207 -2016-06-13 17:13:02 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.7 MB) -2016-06-13 17:13:02 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:61585 (size: 19.3 KB, free: 730.3 MB) -2016-06-13 17:13:02 INFO SparkContext:59 - Created broadcast 78 from textFile at modelSaveLoad.scala:129 -2016-06-13 17:13:03 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 17:13:03 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-06-13 17:13:03 INFO DAGScheduler:59 - Got job 41 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-06-13 17:13:03 INFO DAGScheduler:59 - Final stage: ResultStage 41(first at modelSaveLoad.scala:129) -2016-06-13 17:13:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:13:03 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:13:03 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[11] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-06-13 17:13:03 INFO MemoryStore:59 - ensureFreeSpace(3200) called with curMem=938069, maxMem=766075207 -2016-06-13 17:13:03 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 3.1 KB, free 729.7 MB) -2016-06-13 17:13:03 INFO MemoryStore:59 - ensureFreeSpace(1877) called with curMem=941269, maxMem=766075207 -2016-06-13 17:13:03 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 1877.0 B, free 729.7 MB) -2016-06-13 17:13:03 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:61585 (size: 1877.0 B, free: 730.3 MB) -2016-06-13 17:13:03 INFO SparkContext:59 - Created broadcast 79 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:13:03 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 41 (MapPartitionsRDD[11] at textFile at modelSaveLoad.scala:129) -2016-06-13 17:13:03 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 1 tasks -2016-06-13 17:13:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 79, localhost, PROCESS_LOCAL, 2226 bytes) -2016-06-13 17:13:03 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 79) -2016-06-13 17:13:03 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/output/mllibFromSpark/myGMMModel20160613171256600/metadata/part-00000:0+44 -2016-06-13 17:13:03 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 79). 2135 bytes result sent to driver -2016-06-13 17:13:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 79) in 10 ms on localhost (1/1) -2016-06-13 17:13:03 INFO DAGScheduler:59 - ResultStage 41 (first at modelSaveLoad.scala:129) finished in 0.011 s -2016-06-13 17:13:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-06-13 17:13:03 INFO DAGScheduler:59 - Job 41 finished: first at modelSaveLoad.scala:129, took 0.018697 s -2016-06-13 17:13:03 INFO ParquetRelation:59 - Listing file:/D:/all/idea/SparkLearning/file/data/mllib/output/mllibFromSpark/myGMMModel20160613171256600/data on driver -2016-06-13 17:13:03 INFO SparkContext:59 - Starting job: parquet at GaussianMixtureModel.scala:170 -2016-06-13 17:13:03 INFO DAGScheduler:59 - Got job 42 (parquet at GaussianMixtureModel.scala:170) with 4 output partitions -2016-06-13 17:13:03 INFO DAGScheduler:59 - Final stage: ResultStage 42(parquet at GaussianMixtureModel.scala:170) -2016-06-13 17:13:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:13:03 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:13:03 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[13] at parquet at GaussianMixtureModel.scala:170), which has no missing parents -2016-06-13 17:13:03 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=943146, maxMem=766075207 -2016-06-13 17:13:03 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 61.5 KB, free 729.6 MB) -2016-06-13 17:13:03 INFO MemoryStore:59 - ensureFreeSpace(21071) called with curMem=1006146, maxMem=766075207 -2016-06-13 17:13:03 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 20.6 KB, free 729.6 MB) -2016-06-13 17:13:03 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:61585 (size: 20.6 KB, free: 730.2 MB) -2016-06-13 17:13:03 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:13:03 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 42 (MapPartitionsRDD[13] at parquet at GaussianMixtureModel.scala:170) -2016-06-13 17:13:03 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 4 tasks -2016-06-13 17:13:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 80, localhost, PROCESS_LOCAL, 2083 bytes) -2016-06-13 17:13:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 81, localhost, PROCESS_LOCAL, 2083 bytes) -2016-06-13 17:13:03 INFO TaskSetManager:59 - Starting task 2.0 in stage 42.0 (TID 82, localhost, PROCESS_LOCAL, 2083 bytes) -2016-06-13 17:13:03 INFO TaskSetManager:59 - Starting task 3.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2268 bytes) -2016-06-13 17:13:03 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 81) -2016-06-13 17:13:03 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 80) -2016-06-13 17:13:03 INFO Executor:59 - Running task 2.0 in stage 42.0 (TID 82) -2016-06-13 17:13:03 INFO Executor:59 - Running task 3.0 in stage 42.0 (TID 83) -2016-06-13 17:13:03 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-06-13 17:13:03 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 81). 936 bytes result sent to driver -2016-06-13 17:13:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 81) in 68 ms on localhost (1/4) -2016-06-13 17:13:03 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-06-13 17:13:03 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-06-13 17:13:03 INFO Executor:59 - Finished task 2.0 in stage 42.0 (TID 82). 936 bytes result sent to driver -2016-06-13 17:13:03 INFO TaskSetManager:59 - Finished task 2.0 in stage 42.0 (TID 82) in 87 ms on localhost (2/4) -2016-06-13 17:13:03 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-06-13 17:13:03 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 80). 936 bytes result sent to driver -2016-06-13 17:13:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 80) in 100 ms on localhost (3/4) -2016-06-13 17:13:03 INFO Executor:59 - Finished task 3.0 in stage 42.0 (TID 83). 1845 bytes result sent to driver -2016-06-13 17:13:03 INFO TaskSetManager:59 - Finished task 3.0 in stage 42.0 (TID 83) in 116 ms on localhost (4/4) -2016-06-13 17:13:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-06-13 17:13:03 INFO DAGScheduler:59 - ResultStage 42 (parquet at GaussianMixtureModel.scala:170) finished in 0.119 s -2016-06-13 17:13:03 INFO DAGScheduler:59 - Job 42 finished: parquet at GaussianMixtureModel.scala:170, took 0.154936 s -2016-06-13 17:13:03 INFO BlockManager:59 - Removing RDD 3 -2016-06-13 17:13:04 INFO ContextCleaner:59 - Cleaned RDD 3 -2016-06-13 17:13:04 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:61585 in memory (size: 20.6 KB, free: 730.4 MB) -2016-06-13 17:13:04 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-06-13 17:13:04 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:61585 in memory (size: 1877.0 B, free: 730.4 MB) -2016-06-13 17:13:04 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-06-13 17:13:04 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=542737, maxMem=766075207 -2016-06-13 17:13:04 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:61585 in memory (size: 19.3 KB, free: 730.4 MB) -2016-06-13 17:13:04 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 86.4 KB, free 730.0 MB) -2016-06-13 17:13:04 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:61585 in memory (size: 23.1 KB, free: 730.4 MB) -2016-06-13 17:13:04 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-06-13 17:13:04 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:61585 in memory (size: 41.6 KB, free: 730.5 MB) -2016-06-13 17:13:04 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-06-13 17:13:04 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:61585 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 17:13:04 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-06-13 17:13:04 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:61585 in memory (size: 1038.0 B, free: 730.5 MB) -2016-06-13 17:13:04 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=359332, maxMem=766075207 -2016-06-13 17:13:04 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.2 MB) -2016-06-13 17:13:04 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:61585 (size: 19.3 KB, free: 730.5 MB) -2016-06-13 17:13:04 INFO SparkContext:59 - Created broadcast 81 from collect at GaussianMixtureModel.scala:171 -2016-06-13 17:13:04 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-06-13 17:13:04 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/D:/all/idea/SparkLearning/file/data/mllib/output/mllibFromSpark/myGMMModel20160613171256600/data/part-r-00000-a4ef1669-d4a2-4ea6-93dd-968ab542c73b.gz.parquet -2016-06-13 17:13:04 INFO SparkContext:59 - Starting job: collect at GaussianMixtureModel.scala:171 -2016-06-13 17:13:04 INFO DAGScheduler:59 - Got job 43 (collect at GaussianMixtureModel.scala:171) with 1 output partitions -2016-06-13 17:13:04 INFO DAGScheduler:59 - Final stage: ResultStage 43(collect at GaussianMixtureModel.scala:171) -2016-06-13 17:13:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:13:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:13:04 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[15] at collect at GaussianMixtureModel.scala:171), which has no missing parents -2016-06-13 17:13:04 INFO MemoryStore:59 - ensureFreeSpace(5304) called with curMem=379120, maxMem=766075207 -2016-06-13 17:13:04 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 5.2 KB, free 730.2 MB) -2016-06-13 17:13:04 INFO MemoryStore:59 - ensureFreeSpace(3054) called with curMem=384424, maxMem=766075207 -2016-06-13 17:13:04 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-06-13 17:13:04 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:61585 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 17:13:04 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:13:04 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 43 (MapPartitionsRDD[15] at collect at GaussianMixtureModel.scala:171) -2016-06-13 17:13:04 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 1 tasks -2016-06-13 17:13:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 84, localhost, PROCESS_LOCAL, 2293 bytes) -2016-06-13 17:13:04 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 84) -2016-06-13 17:13:05 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/D:/all/idea/SparkLearning/file/data/mllib/output/mllibFromSpark/myGMMModel20160613171256600/data/part-r-00000-a4ef1669-d4a2-4ea6-93dd-968ab542c73b.gz.parquet start: 0 end: 3495 length: 3495 hosts: []} -2016-06-13 17:13:05 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-06-13 17:21:16 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 17:21:16 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 17:21:17 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 17:21:17 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 17:21:17 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 17:21:17 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 17:21:17 INFO Remoting:74 - Starting remoting -2016-06-13 17:21:17 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:61627] -2016-06-13 17:21:17 INFO Utils:59 - Successfully started service 'sparkDriver' on port 61627. -2016-06-13 17:21:18 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 17:21:18 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 17:21:18 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-fbfcf71b-5be9-497a-a09e-51262b099ce0 -2016-06-13 17:21:18 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 17:21:18 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-46889cdf-4c9d-4ea3-8836-61798b6087bb\httpd-8e493772-16f2-4d89-8ac8-443086366c40 -2016-06-13 17:21:18 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 17:21:18 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:21:18 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:61628 -2016-06-13 17:21:18 INFO Utils:59 - Successfully started service 'HTTP file server' on port 61628. -2016-06-13 17:21:18 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 17:21:18 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:21:18 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 17:21:18 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 17:21:18 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 17:21:18 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 17:21:18 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 17:21:18 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 61647. -2016-06-13 17:21:18 INFO NettyBlockTransferService:59 - Server created on 61647 -2016-06-13 17:21:18 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 17:21:18 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:61647 with 730.6 MB RAM, BlockManagerId(driver, localhost, 61647) -2016-06-13 17:21:18 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 17:21:19 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 17:21:19 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 17:21:19 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 17:21:19 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 17:21:19 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:61647 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 17:21:19 INFO SparkContext:59 - Created broadcast 0 from textFile at GaussianMixtureSuite.scala:23 -2016-06-13 17:21:20 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 17:21:20 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 17:21:20 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:21:21 INFO DAGScheduler:59 - Got job 0 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:21:21 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GaussianMixture.scala:171) -2016-06-13 17:21:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:21 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:21 INFO MemoryStore:59 - ensureFreeSpace(3584) called with curMem=171580, maxMem=766075207 -2016-06-13 17:21:21 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 17:21:21 INFO MemoryStore:59 - ensureFreeSpace(2049) called with curMem=175164, maxMem=766075207 -2016-06-13 17:21:21 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 17:21:21 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:61647 (size: 2.0 KB, free: 730.6 MB) -2016-06-13 17:21:21 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:21 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 17:21:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:21 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 17:21:21 INFO CacheManager:59 - Partition rdd_3_0 not found, computing it -2016-06-13 17:21:21 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-06-13 17:21:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/gmm_data.txt:0+31986 -2016-06-13 17:21:21 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 17:21:21 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 17:21:21 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 17:21:21 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 17:21:21 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 17:21:21 INFO MemoryStore:59 - ensureFreeSpace(52024) called with curMem=177213, maxMem=766075207 -2016-06-13 17:21:21 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 50.8 KB, free 730.4 MB) -2016-06-13 17:21:21 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:61647 (size: 50.8 KB, free: 730.5 MB) -2016-06-13 17:21:21 INFO MemoryStore:59 - ensureFreeSpace(76024) called with curMem=229237, maxMem=766075207 -2016-06-13 17:21:21 INFO MemoryStore:59 - Block rdd_3_0 stored as values in memory (estimated size 74.2 KB, free 730.3 MB) -2016-06-13 17:21:21 INFO BlockManagerInfo:59 - Added rdd_3_0 in memory on localhost:61647 (size: 74.2 KB, free: 730.4 MB) -2016-06-13 17:21:21 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2846 bytes result sent to driver -2016-06-13 17:21:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 622 ms on localhost (1/1) -2016-06-13 17:21:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 17:21:21 INFO DAGScheduler:59 - ResultStage 0 (first at GaussianMixture.scala:171) finished in 0.636 s -2016-06-13 17:21:21 INFO DAGScheduler:59 - Job 0 finished: first at GaussianMixture.scala:171, took 0.712600 s -2016-06-13 17:21:21 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:21:21 INFO DAGScheduler:59 - Got job 1 (takeSample at GaussianMixture.scala:184) with 2 output partitions -2016-06-13 17:21:21 INFO DAGScheduler:59 - Final stage: ResultStage 1(takeSample at GaussianMixture.scala:184) -2016-06-13 17:21:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:21 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:21 INFO MemoryStore:59 - ensureFreeSpace(3424) called with curMem=305261, maxMem=766075207 -2016-06-13 17:21:21 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.3 KB, free 730.3 MB) -2016-06-13 17:21:21 INFO MemoryStore:59 - ensureFreeSpace(1984) called with curMem=308685, maxMem=766075207 -2016-06-13 17:21:21 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1984.0 B, free 730.3 MB) -2016-06-13 17:21:21 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:61647 (size: 1984.0 B, free: 730.4 MB) -2016-06-13 17:21:21 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:21 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 17:21:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:21 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 17:21:21 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 17:21:21 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:21 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 17:21:21 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:61647 in memory (size: 2.0 KB, free: 730.4 MB) -2016-06-13 17:21:21 INFO CacheManager:59 - Partition rdd_3_1 not found, computing it -2016-06-13 17:21:21 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-06-13 17:21:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/gmm_data.txt:31986+31987 -2016-06-13 17:21:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 25 ms on localhost (1/2) -2016-06-13 17:21:21 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 17:21:21 INFO MemoryStore:59 - ensureFreeSpace(47216) called with curMem=305036, maxMem=766075207 -2016-06-13 17:21:21 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 46.1 KB, free 730.3 MB) -2016-06-13 17:21:21 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:61647 (size: 46.1 KB, free: 730.4 MB) -2016-06-13 17:21:21 INFO MemoryStore:59 - ensureFreeSpace(68816) called with curMem=352252, maxMem=766075207 -2016-06-13 17:21:21 INFO MemoryStore:59 - Block rdd_3_1 stored as values in memory (estimated size 67.2 KB, free 730.2 MB) -2016-06-13 17:21:21 INFO BlockManagerInfo:59 - Added rdd_3_1 in memory on localhost:61647 (size: 67.2 KB, free: 730.3 MB) -2016-06-13 17:21:21 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2717 bytes result sent to driver -2016-06-13 17:21:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 107 ms on localhost (2/2) -2016-06-13 17:21:21 INFO DAGScheduler:59 - ResultStage 1 (takeSample at GaussianMixture.scala:184) finished in 0.115 s -2016-06-13 17:21:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 17:21:21 INFO DAGScheduler:59 - Job 1 finished: takeSample at GaussianMixture.scala:184, took 0.157344 s -2016-06-13 17:21:21 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:21:21 INFO DAGScheduler:59 - Got job 2 (takeSample at GaussianMixture.scala:184) with 2 output partitions -2016-06-13 17:21:21 INFO DAGScheduler:59 - Final stage: ResultStage 2(takeSample at GaussianMixture.scala:184) -2016-06-13 17:21:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:21 INFO DAGScheduler:59 - Submitting ResultStage 2 (PartitionwiseSampledRDD[4] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:21:21 INFO MemoryStore:59 - ensureFreeSpace(20816) called with curMem=421068, maxMem=766075207 -2016-06-13 17:21:21 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 20.3 KB, free 730.2 MB) -2016-06-13 17:21:21 INFO MemoryStore:59 - ensureFreeSpace(8644) called with curMem=441884, maxMem=766075207 -2016-06-13 17:21:21 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 8.4 KB, free 730.2 MB) -2016-06-13 17:21:21 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:61647 (size: 8.4 KB, free: 730.3 MB) -2016-06-13 17:21:21 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (PartitionwiseSampledRDD[4] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:21:21 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 17:21:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2299 bytes) -2016-06-13 17:21:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2299 bytes) -2016-06-13 17:21:21 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 17:21:21 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-06-13 17:21:21 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:21 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:21 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2874 bytes result sent to driver -2016-06-13 17:21:21 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 3319 bytes result sent to driver -2016-06-13 17:21:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 23 ms on localhost (1/2) -2016-06-13 17:21:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 25 ms on localhost (2/2) -2016-06-13 17:21:21 INFO DAGScheduler:59 - ResultStage 2 (takeSample at GaussianMixture.scala:184) finished in 0.026 s -2016-06-13 17:21:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 17:21:21 INFO DAGScheduler:59 - Job 2 finished: takeSample at GaussianMixture.scala:184, took 0.040468 s -2016-06-13 17:21:21 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 17:21:21 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 17:21:22 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 17:21:22 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 17:21:22 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=450528, maxMem=766075207 -2016-06-13 17:21:22 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 840.0 B, free 730.2 MB) -2016-06-13 17:21:22 INFO MemoryStore:59 - ensureFreeSpace(1001) called with curMem=451368, maxMem=766075207 -2016-06-13 17:21:22 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 1001.0 B, free 730.2 MB) -2016-06-13 17:21:22 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:61647 (size: 1001.0 B, free: 730.3 MB) -2016-06-13 17:21:22 INFO SparkContext:59 - Created broadcast 4 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:22 INFO DAGScheduler:59 - Got job 3 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:22 INFO DAGScheduler:59 - Final stage: ResultStage 3(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:22 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:22 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=452369, maxMem=766075207 -2016-06-13 17:21:22 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:22 INFO MemoryStore:59 - ensureFreeSpace(3068) called with curMem=458249, maxMem=766075207 -2016-06-13 17:21:22 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:22 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:22 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:22 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 17:21:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:22 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-06-13 17:21:22 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-06-13 17:21:22 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:22 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:22 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2826 bytes result sent to driver -2016-06-13 17:21:22 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2826 bytes result sent to driver -2016-06-13 17:21:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 206 ms on localhost (1/2) -2016-06-13 17:21:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 206 ms on localhost (2/2) -2016-06-13 17:21:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 17:21:22 INFO DAGScheduler:59 - ResultStage 3 (aggregate at GaussianMixture.scala:201) finished in 0.208 s -2016-06-13 17:21:22 INFO DAGScheduler:59 - Job 3 finished: aggregate at GaussianMixture.scala:201, took 0.221402 s -2016-06-13 17:21:22 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=461317, maxMem=766075207 -2016-06-13 17:21:22 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:22 INFO MemoryStore:59 - ensureFreeSpace(1031) called with curMem=462157, maxMem=766075207 -2016-06-13 17:21:22 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1031.0 B, free 730.1 MB) -2016-06-13 17:21:22 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:61647 (size: 1031.0 B, free: 730.3 MB) -2016-06-13 17:21:22 INFO SparkContext:59 - Created broadcast 6 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:22 INFO DAGScheduler:59 - Got job 4 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:22 INFO DAGScheduler:59 - Final stage: ResultStage 4(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:22 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:22 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=463188, maxMem=766075207 -2016-06-13 17:21:22 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:22 INFO MemoryStore:59 - ensureFreeSpace(3115) called with curMem=469068, maxMem=766075207 -2016-06-13 17:21:22 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:22 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:22 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:22 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 17:21:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:22 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-06-13 17:21:22 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-06-13 17:21:22 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:22 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:22 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2826 bytes result sent to driver -2016-06-13 17:21:22 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2826 bytes result sent to driver -2016-06-13 17:21:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 115 ms on localhost (1/2) -2016-06-13 17:21:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 119 ms on localhost (2/2) -2016-06-13 17:21:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 17:21:22 INFO DAGScheduler:59 - ResultStage 4 (aggregate at GaussianMixture.scala:201) finished in 0.119 s -2016-06-13 17:21:22 INFO DAGScheduler:59 - Job 4 finished: aggregate at GaussianMixture.scala:201, took 0.133859 s -2016-06-13 17:21:22 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=472183, maxMem=766075207 -2016-06-13 17:21:22 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:22 INFO MemoryStore:59 - ensureFreeSpace(1031) called with curMem=473023, maxMem=766075207 -2016-06-13 17:21:22 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 1031.0 B, free 730.1 MB) -2016-06-13 17:21:22 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:61647 (size: 1031.0 B, free: 730.3 MB) -2016-06-13 17:21:22 INFO SparkContext:59 - Created broadcast 8 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:22 INFO DAGScheduler:59 - Got job 5 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:22 INFO DAGScheduler:59 - Final stage: ResultStage 5(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:22 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:22 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=474054, maxMem=766075207 -2016-06-13 17:21:22 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:22 INFO MemoryStore:59 - ensureFreeSpace(3114) called with curMem=479934, maxMem=766075207 -2016-06-13 17:21:22 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:22 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:22 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:22 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 17:21:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:22 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-06-13 17:21:22 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-06-13 17:21:22 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:22 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:22 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2826 bytes result sent to driver -2016-06-13 17:21:22 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2826 bytes result sent to driver -2016-06-13 17:21:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 62 ms on localhost (1/2) -2016-06-13 17:21:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 65 ms on localhost (2/2) -2016-06-13 17:21:22 INFO DAGScheduler:59 - ResultStage 5 (aggregate at GaussianMixture.scala:201) finished in 0.066 s -2016-06-13 17:21:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 17:21:22 INFO DAGScheduler:59 - Job 5 finished: aggregate at GaussianMixture.scala:201, took 0.080639 s -2016-06-13 17:21:22 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=483048, maxMem=766075207 -2016-06-13 17:21:22 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:22 INFO MemoryStore:59 - ensureFreeSpace(1036) called with curMem=483888, maxMem=766075207 -2016-06-13 17:21:22 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 1036.0 B, free 730.1 MB) -2016-06-13 17:21:22 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:61647 (size: 1036.0 B, free: 730.3 MB) -2016-06-13 17:21:22 INFO SparkContext:59 - Created broadcast 10 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:22 INFO DAGScheduler:59 - Got job 6 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:22 INFO DAGScheduler:59 - Final stage: ResultStage 6(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:22 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:22 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=484924, maxMem=766075207 -2016-06-13 17:21:22 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:22 INFO MemoryStore:59 - ensureFreeSpace(3114) called with curMem=490804, maxMem=766075207 -2016-06-13 17:21:22 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:22 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:22 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:22 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-06-13 17:21:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:22 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-06-13 17:21:22 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-06-13 17:21:22 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:22 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:22 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2826 bytes result sent to driver -2016-06-13 17:21:22 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2826 bytes result sent to driver -2016-06-13 17:21:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 40 ms on localhost (1/2) -2016-06-13 17:21:22 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:22 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 17:21:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 44 ms on localhost (2/2) -2016-06-13 17:21:22 INFO DAGScheduler:59 - ResultStage 6 (aggregate at GaussianMixture.scala:201) finished in 0.044 s -2016-06-13 17:21:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 17:21:22 INFO DAGScheduler:59 - Job 6 finished: aggregate at GaussianMixture.scala:201, took 0.064782 s -2016-06-13 17:21:22 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=483053, maxMem=766075207 -2016-06-13 17:21:22 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:61647 in memory (size: 1031.0 B, free: 730.3 MB) -2016-06-13 17:21:22 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:22 INFO MemoryStore:59 - ensureFreeSpace(1036) called with curMem=474898, maxMem=766075207 -2016-06-13 17:21:22 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:22 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 1036.0 B, free 730.1 MB) -2016-06-13 17:21:22 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:61647 (size: 1036.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:61647 in memory (size: 1031.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 12 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:61647 in memory (size: 1001.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Got job 7 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:23 INFO DAGScheduler:59 - Final stage: ResultStage 7(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:61647 in memory (size: 8.4 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:23 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:61647 in memory (size: 1984.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=428406, maxMem=766075207 -2016-06-13 17:21:23 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.7 KB, free 730.2 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=434286, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-06-13 17:21:23 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:23 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 17 ms on localhost (1/2) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 19 ms on localhost (2/2) -2016-06-13 17:21:23 INFO DAGScheduler:59 - ResultStage 7 (aggregate at GaussianMixture.scala:201) finished in 0.020 s -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 17:21:23 INFO DAGScheduler:59 - Job 7 finished: aggregate at GaussianMixture.scala:201, took 0.032246 s -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=437398, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 840.0 B, free 730.2 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=438238, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 1032.0 B, free 730.2 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:61647 (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 14 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Got job 8 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:23 INFO DAGScheduler:59 - Final stage: ResultStage 8(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=439270, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 5.7 KB, free 730.2 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=445150, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-06-13 17:21:23 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:23 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 22 ms on localhost (1/2) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 21 ms on localhost (2/2) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 17:21:23 INFO DAGScheduler:59 - ResultStage 8 (aggregate at GaussianMixture.scala:201) finished in 0.024 s -2016-06-13 17:21:23 INFO DAGScheduler:59 - Job 8 finished: aggregate at GaussianMixture.scala:201, took 0.037636 s -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=448262, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 840.0 B, free 730.2 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(1036) called with curMem=449102, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1036.0 B, free 730.2 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:61647 (size: 1036.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 16 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Got job 9 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:23 INFO DAGScheduler:59 - Final stage: ResultStage 9(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=450138, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.7 KB, free 730.2 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(3111) called with curMem=456018, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-06-13 17:21:23 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:23 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 16 ms on localhost (1/2) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 18 ms on localhost (2/2) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 17:21:23 INFO DAGScheduler:59 - ResultStage 9 (aggregate at GaussianMixture.scala:201) finished in 0.019 s -2016-06-13 17:21:23 INFO DAGScheduler:59 - Job 9 finished: aggregate at GaussianMixture.scala:201, took 0.030858 s -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=459129, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(1033) called with curMem=459969, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1033.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:61647 (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 18 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Got job 10 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:23 INFO DAGScheduler:59 - Final stage: ResultStage 10(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=461002, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=466882, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-06-13 17:21:23 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:23 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 17 ms on localhost (1/2) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 20 ms on localhost (2/2) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 17:21:23 INFO DAGScheduler:59 - ResultStage 10 (aggregate at GaussianMixture.scala:201) finished in 0.021 s -2016-06-13 17:21:23 INFO DAGScheduler:59 - Job 10 finished: aggregate at GaussianMixture.scala:201, took 0.040614 s -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=469994, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(1035) called with curMem=470834, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 1035.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:61647 (size: 1035.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 20 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Got job 11 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:23 INFO DAGScheduler:59 - Final stage: ResultStage 11(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=471869, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(3111) called with curMem=477749, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-06-13 17:21:23 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:23 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 19 ms on localhost (1/2) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 19 ms on localhost (2/2) -2016-06-13 17:21:23 INFO DAGScheduler:59 - ResultStage 11 (aggregate at GaussianMixture.scala:201) finished in 0.021 s -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 17:21:23 INFO DAGScheduler:59 - Job 11 finished: aggregate at GaussianMixture.scala:201, took 0.034080 s -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=480860, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(1037) called with curMem=481700, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 1037.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:61647 (size: 1037.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 22 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Got job 12 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:23 INFO DAGScheduler:59 - Final stage: ResultStage 12(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=482737, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(3111) called with curMem=488617, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-06-13 17:21:23 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:23 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 13 ms on localhost (1/2) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 15 ms on localhost (2/2) -2016-06-13 17:21:23 INFO DAGScheduler:59 - ResultStage 12 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 17:21:23 INFO DAGScheduler:59 - Job 12 finished: aggregate at GaussianMixture.scala:201, took 0.031069 s -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=491728, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(1041) called with curMem=492568, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 1041.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:61647 (size: 1041.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 24 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Got job 13 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:23 INFO DAGScheduler:59 - Final stage: ResultStage 13(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=493609, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(3116) called with curMem=499489, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-06-13 17:21:23 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:23 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 31 ms on localhost (1/2) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 30 ms on localhost (2/2) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 17:21:23 INFO DAGScheduler:59 - ResultStage 13 (aggregate at GaussianMixture.scala:201) finished in 0.032 s -2016-06-13 17:21:23 INFO DAGScheduler:59 - Job 13 finished: aggregate at GaussianMixture.scala:201, took 0.046705 s -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:61647 in memory (size: 1037.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=491737, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(1038) called with curMem=489466, maxMem=766075207 -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 1038.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:61647 (size: 1038.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 26 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:61647 in memory (size: 1035.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 17:21:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Got job 14 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:23 INFO DAGScheduler:59 - Final stage: ResultStage 14(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:61647 in memory (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=468773, maxMem=766075207 -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(3118) called with curMem=467737, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:61647 in memory (size: 1036.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-06-13 17:21:23 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:61647 in memory (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:23 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 15 ms on localhost (1/2) -2016-06-13 17:21:23 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 19 ms on localhost (2/2) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 17:21:23 INFO DAGScheduler:59 - ResultStage 14 (aggregate at GaussianMixture.scala:201) finished in 0.020 s -2016-06-13 17:21:23 INFO DAGScheduler:59 - Job 14 finished: aggregate at GaussianMixture.scala:201, took 0.038852 s -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:61647 in memory (size: 1036.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=448283, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 840.0 B, free 730.2 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(1038) called with curMem=440129, maxMem=766075207 -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1038.0 B, free 730.2 MB) -2016-06-13 17:21:23 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:61647 (size: 1038.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 28 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:61647 in memory (size: 1036.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Got job 15 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:23 INFO DAGScheduler:59 - Final stage: ResultStage 15(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=439291, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.7 KB, free 730.2 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=445171, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-06-13 17:21:23 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:23 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 14 ms on localhost (1/2) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 18 ms on localhost (2/2) -2016-06-13 17:21:23 INFO DAGScheduler:59 - ResultStage 15 (aggregate at GaussianMixture.scala:201) finished in 0.018 s -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 17:21:23 INFO DAGScheduler:59 - Job 15 finished: aggregate at GaussianMixture.scala:201, took 0.031790 s -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=448283, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 840.0 B, free 730.2 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(1033) called with curMem=449123, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1033.0 B, free 730.2 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:61647 (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 30 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Got job 16 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:23 INFO DAGScheduler:59 - Final stage: ResultStage 16(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=450156, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 5.7 KB, free 730.2 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=456036, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-06-13 17:21:23 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:23 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 14 ms on localhost (1/2) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 16 ms on localhost (2/2) -2016-06-13 17:21:23 INFO DAGScheduler:59 - ResultStage 16 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 17:21:23 INFO DAGScheduler:59 - Job 16 finished: aggregate at GaussianMixture.scala:201, took 0.030594 s -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=459149, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(1038) called with curMem=459989, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 1038.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:61647 (size: 1038.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 32 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Got job 17 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:23 INFO DAGScheduler:59 - Final stage: ResultStage 17(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=461027, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=466907, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-06-13 17:21:23 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:23 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 11 ms on localhost (1/2) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 14 ms on localhost (2/2) -2016-06-13 17:21:23 INFO DAGScheduler:59 - ResultStage 17 (aggregate at GaussianMixture.scala:201) finished in 0.015 s -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 17:21:23 INFO DAGScheduler:59 - Job 17 finished: aggregate at GaussianMixture.scala:201, took 0.029406 s -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=470020, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(1033) called with curMem=470860, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 1033.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:61647 (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 34 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Got job 18 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:23 INFO DAGScheduler:59 - Final stage: ResultStage 18(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=471893, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=477773, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-06-13 17:21:23 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:23 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 16 ms on localhost (1/2) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 17 ms on localhost (2/2) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 17:21:23 INFO DAGScheduler:59 - ResultStage 18 (aggregate at GaussianMixture.scala:201) finished in 0.019 s -2016-06-13 17:21:23 INFO DAGScheduler:59 - Job 18 finished: aggregate at GaussianMixture.scala:201, took 0.030561 s -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=480885, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(1035) called with curMem=481725, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 1035.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:61647 (size: 1035.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 36 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Got job 19 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:23 INFO DAGScheduler:59 - Final stage: ResultStage 19(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=482760, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=488640, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-06-13 17:21:23 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:23 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 15 ms on localhost (1/2) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 19 ms on localhost (2/2) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 17:21:23 INFO DAGScheduler:59 - ResultStage 19 (aggregate at GaussianMixture.scala:201) finished in 0.020 s -2016-06-13 17:21:23 INFO DAGScheduler:59 - Job 19 finished: aggregate at GaussianMixture.scala:201, took 0.034490 s -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=491752, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(1038) called with curMem=492592, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 1038.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:61647 (size: 1038.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 38 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Got job 20 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:23 INFO DAGScheduler:59 - Final stage: ResultStage 20(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=493630, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=499510, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-06-13 17:21:23 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:23 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 15 ms on localhost (1/2) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 15 ms on localhost (2/2) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 17:21:23 INFO DAGScheduler:59 - ResultStage 20 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 17:21:23 INFO DAGScheduler:59 - Job 20 finished: aggregate at GaussianMixture.scala:201, took 0.030378 s -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=502622, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(1033) called with curMem=503462, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 1033.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:61647 (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 40 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Got job 21 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:23 INFO DAGScheduler:59 - Final stage: ResultStage 21(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=504495, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=510375, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-06-13 17:21:23 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:23 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 18 ms on localhost (1/2) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 18 ms on localhost (2/2) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 17:21:23 INFO DAGScheduler:59 - ResultStage 21 (aggregate at GaussianMixture.scala:201) finished in 0.020 s -2016-06-13 17:21:23 INFO DAGScheduler:59 - Job 21 finished: aggregate at GaussianMixture.scala:201, took 0.039450 s -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=513487, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(1033) called with curMem=514327, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1033.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:61647 (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 42 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Got job 22 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:23 INFO DAGScheduler:59 - Final stage: ResultStage 22(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=515360, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=521240, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-06-13 17:21:23 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:23 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 13 ms on localhost (1/2) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 15 ms on localhost (2/2) -2016-06-13 17:21:23 INFO DAGScheduler:59 - ResultStage 22 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 17:21:23 INFO DAGScheduler:59 - Job 22 finished: aggregate at GaussianMixture.scala:201, took 0.027219 s -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=524352, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(1034) called with curMem=525192, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 1034.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:61647 (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 44 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Got job 23 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:23 INFO DAGScheduler:59 - Final stage: ResultStage 23(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=526226, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=532106, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-06-13 17:21:23 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:23 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 15 ms on localhost (1/2) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 18 ms on localhost (2/2) -2016-06-13 17:21:23 INFO DAGScheduler:59 - ResultStage 23 (aggregate at GaussianMixture.scala:201) finished in 0.018 s -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 17:21:23 INFO DAGScheduler:59 - Job 23 finished: aggregate at GaussianMixture.scala:201, took 0.030650 s -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=535218, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(1035) called with curMem=536058, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 1035.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:61647 (size: 1035.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 46 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Got job 24 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:23 INFO DAGScheduler:59 - Final stage: ResultStage 24(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=537093, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=542973, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-06-13 17:21:23 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:23 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 15 ms on localhost (1/2) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 15 ms on localhost (2/2) -2016-06-13 17:21:23 INFO DAGScheduler:59 - ResultStage 24 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-06-13 17:21:23 INFO DAGScheduler:59 - Job 24 finished: aggregate at GaussianMixture.scala:201, took 0.031651 s -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=546085, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=546925, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 1032.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:61647 (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 48 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:61647 in memory (size: 1038.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Got job 25 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:23 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 17:21:23 INFO DAGScheduler:59 - Final stage: ResultStage 25(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=536052, maxMem=766075207 -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:61647 in memory (size: 1035.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=541092, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-06-13 17:21:23 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:23 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-06-13 17:21:23 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:23 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:23 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:61647 in memory (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 2826 bytes result sent to driver -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 17 ms on localhost (1/2) -2016-06-13 17:21:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 20 ms on localhost (2/2) -2016-06-13 17:21:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-06-13 17:21:23 INFO DAGScheduler:59 - ResultStage 25 (aggregate at GaussianMixture.scala:201) finished in 0.021 s -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:23 INFO DAGScheduler:59 - Job 25 finished: aggregate at GaussianMixture.scala:201, took 0.036980 s -2016-06-13 17:21:23 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=524346, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:23 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:61647 in memory (size: 1038.0 B, free: 730.3 MB) -2016-06-13 17:21:23 INFO MemoryStore:59 - ensureFreeSpace(1033) called with curMem=523308, maxMem=766075207 -2016-06-13 17:21:23 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 1033.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:61647 (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 50 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:24 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:61647 in memory (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO DAGScheduler:59 - Got job 26 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:24 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Final stage: ResultStage 26(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=502605, maxMem=766075207 -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:61647 in memory (size: 1038.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=502605, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-06-13 17:21:24 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:61647 in memory (size: 1038.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:24 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 15 ms on localhost (1/2) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 18 ms on localhost (2/2) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-06-13 17:21:24 INFO DAGScheduler:59 - ResultStage 26 (aggregate at GaussianMixture.scala:201) finished in 0.020 s -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO DAGScheduler:59 - Job 26 finished: aggregate at GaussianMixture.scala:201, took 0.034796 s -2016-06-13 17:21:24 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=490694, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:61647 in memory (size: 1035.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=490694, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 1032.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:61647 (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 52 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:61647 in memory (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Got job 27 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:24 INFO DAGScheduler:59 - Final stage: ResultStage 27(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:61647 in memory (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=469995, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=472763, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:61647 in memory (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-06-13 17:21:24 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:24 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 20 ms on localhost (1/2) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 20 ms on localhost (2/2) -2016-06-13 17:21:24 INFO DAGScheduler:59 - ResultStage 27 (aggregate at GaussianMixture.scala:201) finished in 0.022 s -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-06-13 17:21:24 INFO DAGScheduler:59 - Job 27 finished: aggregate at GaussianMixture.scala:201, took 0.039086 s -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=459130, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(1033) called with curMem=459970, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 1033.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:61647 (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 54 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Got job 28 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:24 INFO DAGScheduler:59 - Final stage: ResultStage 28(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=461003, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=466883, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-06-13 17:21:24 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:24 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 14 ms on localhost (1/2) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 14 ms on localhost (2/2) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-06-13 17:21:24 INFO DAGScheduler:59 - ResultStage 28 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 17:21:24 INFO DAGScheduler:59 - Job 28 finished: aggregate at GaussianMixture.scala:201, took 0.027591 s -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=469996, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(1037) called with curMem=470836, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 1037.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:61647 (size: 1037.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 56 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Got job 29 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:24 INFO DAGScheduler:59 - Final stage: ResultStage 29(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=471873, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(3114) called with curMem=477753, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-06-13 17:21:24 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:24 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 14 ms on localhost (1/2) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 16 ms on localhost (2/2) -2016-06-13 17:21:24 INFO DAGScheduler:59 - ResultStage 29 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 17:21:24 INFO DAGScheduler:59 - Job 29 finished: aggregate at GaussianMixture.scala:201, took 0.027111 s -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=480867, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=481707, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 1032.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:61647 (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 58 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Got job 30 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:24 INFO DAGScheduler:59 - Final stage: ResultStage 30(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=482739, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=488619, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-06-13 17:21:24 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:24 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 13 ms on localhost (1/2) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 15 ms on localhost (2/2) -2016-06-13 17:21:24 INFO DAGScheduler:59 - ResultStage 30 (aggregate at GaussianMixture.scala:201) finished in 0.015 s -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 17:21:24 INFO DAGScheduler:59 - Job 30 finished: aggregate at GaussianMixture.scala:201, took 0.029580 s -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=491732, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(1038) called with curMem=492572, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 1038.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:61647 (size: 1038.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 60 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Got job 31 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:24 INFO DAGScheduler:59 - Final stage: ResultStage 31(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=493610, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=499490, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 61 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-06-13 17:21:24 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:24 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 14 ms on localhost (1/2) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 16 ms on localhost (2/2) -2016-06-13 17:21:24 INFO DAGScheduler:59 - ResultStage 31 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 17:21:24 INFO DAGScheduler:59 - Job 31 finished: aggregate at GaussianMixture.scala:201, took 0.030248 s -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=502603, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(1034) called with curMem=503443, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 1034.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:61647 (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 62 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Got job 32 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:24 INFO DAGScheduler:59 - Final stage: ResultStage 32(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=504477, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=510357, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-06-13 17:21:24 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:24 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 12 ms on localhost (1/2) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 13 ms on localhost (2/2) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 17:21:24 INFO DAGScheduler:59 - ResultStage 32 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 17:21:24 INFO DAGScheduler:59 - Job 32 finished: aggregate at GaussianMixture.scala:201, took 0.028529 s -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=513470, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(1036) called with curMem=514310, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 1036.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:61647 (size: 1036.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 64 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Got job 33 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:24 INFO DAGScheduler:59 - Final stage: ResultStage 33(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=515346, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(3113) called with curMem=521226, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-06-13 17:21:24 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:24 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 54 ms on localhost (1/2) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 56 ms on localhost (2/2) -2016-06-13 17:21:24 INFO DAGScheduler:59 - ResultStage 33 (aggregate at GaussianMixture.scala:201) finished in 0.057 s -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 17:21:24 INFO DAGScheduler:59 - Job 33 finished: aggregate at GaussianMixture.scala:201, took 0.071058 s -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=524339, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(1034) called with curMem=525179, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 1034.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:61647 (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 66 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Got job 34 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:24 INFO DAGScheduler:59 - Final stage: ResultStage 34(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=526213, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(3111) called with curMem=532093, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 67 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-06-13 17:21:24 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:24 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 12 ms on localhost (1/2) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 12 ms on localhost (2/2) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-06-13 17:21:24 INFO DAGScheduler:59 - ResultStage 34 (aggregate at GaussianMixture.scala:201) finished in 0.013 s -2016-06-13 17:21:24 INFO DAGScheduler:59 - Job 34 finished: aggregate at GaussianMixture.scala:201, took 0.027274 s -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=535204, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=536044, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 1032.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:61647 (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 68 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Got job 35 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:24 INFO DAGScheduler:59 - Final stage: ResultStage 35(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=537076, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(3114) called with curMem=542956, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 69 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-06-13 17:21:24 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:24 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 11 ms on localhost (1/2) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 12 ms on localhost (2/2) -2016-06-13 17:21:24 INFO DAGScheduler:59 - ResultStage 35 (aggregate at GaussianMixture.scala:201) finished in 0.013 s -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-06-13 17:21:24 INFO DAGScheduler:59 - Job 35 finished: aggregate at GaussianMixture.scala:201, took 0.023222 s -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=546070, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(1037) called with curMem=546910, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 1037.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:61647 (size: 1037.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 70 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:24 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:61647 in memory (size: 1038.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Got job 36 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:24 INFO DAGScheduler:59 - Final stage: ResultStage 36(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=533962, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=533962, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 71 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:61647 in memory (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-06-13 17:21:24 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:24 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 13 ms on localhost (1/2) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 16 ms on localhost (2/2) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-06-13 17:21:24 INFO DAGScheduler:59 - ResultStage 36 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:61647 in memory (size: 1037.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO DAGScheduler:59 - Job 36 finished: aggregate at GaussianMixture.scala:201, took 0.032440 s -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=521218, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(1031) called with curMem=516178, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 1031.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:61647 (size: 1031.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 72 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:61647 in memory (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO DAGScheduler:59 - Got job 37 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:24 INFO DAGScheduler:59 - Final stage: ResultStage 37(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:24 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=504472, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:61647 in memory (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=510352, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 73 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-06-13 17:21:24 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:24 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:61647 in memory (size: 1033.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 14 ms on localhost (1/2) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 14 ms on localhost (2/2) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-06-13 17:21:24 INFO DAGScheduler:59 - ResultStage 37 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 17:21:24 INFO DAGScheduler:59 - Job 37 finished: aggregate at GaussianMixture.scala:201, took 0.031987 s -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=493607, maxMem=766075207 -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(1034) called with curMem=494447, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 1034.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:61647 in memory (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:61647 (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 74 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:61647 in memory (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Got job 38 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:24 INFO DAGScheduler:59 - Final stage: ResultStage 38(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=481712, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:61647 in memory (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(3112) called with curMem=486752, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 75 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-06-13 17:21:24 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-06-13 17:21:24 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:24 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:61647 in memory (size: 1036.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 13 ms on localhost (1/2) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 13 ms on localhost (2/2) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-06-13 17:21:24 INFO DAGScheduler:59 - ResultStage 38 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 17:21:24 INFO DAGScheduler:59 - Job 38 finished: aggregate at GaussianMixture.scala:201, took 0.031467 s -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(840) called with curMem=470002, maxMem=766075207 -2016-06-13 17:21:24 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 840.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=469808, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 1032.0 B, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:61647 in memory (size: 1034.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:61647 (size: 1032.0 B, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 76 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:61647 in memory (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Got job 39 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:21:24 INFO DAGScheduler:59 - Final stage: ResultStage 39(aggregate at GaussianMixture.scala:201) -2016-06-13 17:21:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=461007, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO MemoryStore:59 - ensureFreeSpace(3110) called with curMem=466887, maxMem=766075207 -2016-06-13 17:21:24 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.1 MB) -2016-06-13 17:21:24 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:61647 (size: 3.0 KB, free: 730.3 MB) -2016-06-13 17:21:24 INFO SparkContext:59 - Created broadcast 77 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:21:24 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-06-13 17:21:24 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:21:24 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:21:24 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 2826 bytes result sent to driver -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 10 ms on localhost (1/2) -2016-06-13 17:21:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 11 ms on localhost (2/2) -2016-06-13 17:21:24 INFO DAGScheduler:59 - ResultStage 39 (aggregate at GaussianMixture.scala:201) finished in 0.012 s -2016-06-13 17:21:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-06-13 17:21:24 INFO DAGScheduler:59 - Job 39 finished: aggregate at GaussianMixture.scala:201, took 0.026416 s -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 17:21:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 17:21:24 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 17:21:24 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 17:21:24 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 17:21:24 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 17:21:24 INFO BlockManager:59 - BlockManager stopped -2016-06-13 17:21:24 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 17:21:24 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 17:21:25 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 17:21:25 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 17:21:25 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 17:21:25 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 17:21:25 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-46889cdf-4c9d-4ea3-8836-61798b6087bb -2016-06-13 17:30:48 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 17:30:48 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 17:30:49 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 17:30:49 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 17:30:49 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 17:30:49 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 17:30:49 INFO Remoting:74 - Starting remoting -2016-06-13 17:30:50 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:61962] -2016-06-13 17:30:50 INFO Utils:59 - Successfully started service 'sparkDriver' on port 61962. -2016-06-13 17:30:50 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 17:30:50 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 17:30:50 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-674b0f9a-5ea8-4538-97ea-ace21d50dc16 -2016-06-13 17:30:50 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 17:30:50 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-639d48a8-037d-4552-864f-09f31b3972f9\httpd-5fb763d6-de24-4361-bb81-d1c0fc29ebba -2016-06-13 17:30:50 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 17:30:50 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:30:50 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:61963 -2016-06-13 17:30:50 INFO Utils:59 - Successfully started service 'HTTP file server' on port 61963. -2016-06-13 17:30:50 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 17:30:50 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:30:50 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 17:30:50 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 17:30:50 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 17:30:50 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 17:30:50 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 17:30:51 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 61982. -2016-06-13 17:30:51 INFO NettyBlockTransferService:59 - Server created on 61982 -2016-06-13 17:30:51 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 17:30:51 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:61982 with 730.6 MB RAM, BlockManagerId(driver, localhost, 61982) -2016-06-13 17:30:51 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 17:30:52 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 17:30:52 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 17:30:52 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 17:30:52 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 17:30:52 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:61982 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 17:30:52 INFO SparkContext:59 - Created broadcast 0 from textFile at GaussianMixtureSuite.scala:23 -2016-06-13 17:30:53 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 17:30:53 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 17:30:53 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:30:53 INFO DAGScheduler:59 - Got job 0 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:30:53 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GaussianMixture.scala:171) -2016-06-13 17:30:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:53 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:53 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:53 INFO MemoryStore:59 - ensureFreeSpace(3584) called with curMem=171580, maxMem=766075207 -2016-06-13 17:30:53 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 17:30:53 INFO MemoryStore:59 - ensureFreeSpace(2049) called with curMem=175164, maxMem=766075207 -2016-06-13 17:30:53 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 17:30:53 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:61982 (size: 2.0 KB, free: 730.6 MB) -2016-06-13 17:30:53 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:53 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:53 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 17:30:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:53 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 17:30:53 INFO CacheManager:59 - Partition rdd_3_0 not found, computing it -2016-06-13 17:30:53 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-06-13 17:30:53 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/gmm_data.txt:0+31986 -2016-06-13 17:30:53 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 17:30:53 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 17:30:53 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 17:30:53 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 17:30:53 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 17:30:53 INFO MemoryStore:59 - ensureFreeSpace(52024) called with curMem=177213, maxMem=766075207 -2016-06-13 17:30:53 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 50.8 KB, free 730.4 MB) -2016-06-13 17:30:53 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:61982 (size: 50.8 KB, free: 730.5 MB) -2016-06-13 17:30:54 INFO MemoryStore:59 - ensureFreeSpace(76024) called with curMem=229237, maxMem=766075207 -2016-06-13 17:30:54 INFO MemoryStore:59 - Block rdd_3_0 stored as values in memory (estimated size 74.2 KB, free 730.3 MB) -2016-06-13 17:30:54 INFO BlockManagerInfo:59 - Added rdd_3_0 in memory on localhost:61982 (size: 74.2 KB, free: 730.4 MB) -2016-06-13 17:30:54 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2846 bytes result sent to driver -2016-06-13 17:30:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 672 ms on localhost (1/1) -2016-06-13 17:30:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 17:30:54 INFO DAGScheduler:59 - ResultStage 0 (first at GaussianMixture.scala:171) finished in 0.687 s -2016-06-13 17:30:54 INFO DAGScheduler:59 - Job 0 finished: first at GaussianMixture.scala:171, took 0.757736 s -2016-06-13 17:30:54 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:30:54 INFO DAGScheduler:59 - Got job 1 (takeSample at GaussianMixture.scala:184) with 2 output partitions -2016-06-13 17:30:54 INFO DAGScheduler:59 - Final stage: ResultStage 1(takeSample at GaussianMixture.scala:184) -2016-06-13 17:30:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:54 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:54 INFO MemoryStore:59 - ensureFreeSpace(3424) called with curMem=305261, maxMem=766075207 -2016-06-13 17:30:54 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.3 KB, free 730.3 MB) -2016-06-13 17:30:54 INFO MemoryStore:59 - ensureFreeSpace(1984) called with curMem=308685, maxMem=766075207 -2016-06-13 17:30:54 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1984.0 B, free 730.3 MB) -2016-06-13 17:30:54 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:61982 (size: 1984.0 B, free: 730.4 MB) -2016-06-13 17:30:54 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:54 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 17:30:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:54 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 17:30:54 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 17:30:54 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:54 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 17:30:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 18 ms on localhost (1/2) -2016-06-13 17:30:54 INFO CacheManager:59 - Partition rdd_3_1 not found, computing it -2016-06-13 17:30:54 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-06-13 17:30:54 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/gmm_data.txt:31986+31987 -2016-06-13 17:30:54 INFO MemoryStore:59 - ensureFreeSpace(47216) called with curMem=310669, maxMem=766075207 -2016-06-13 17:30:54 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 46.1 KB, free 730.2 MB) -2016-06-13 17:30:54 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:61982 (size: 46.1 KB, free: 730.4 MB) -2016-06-13 17:30:54 INFO MemoryStore:59 - ensureFreeSpace(68816) called with curMem=357885, maxMem=766075207 -2016-06-13 17:30:54 INFO MemoryStore:59 - Block rdd_3_1 stored as values in memory (estimated size 67.2 KB, free 730.2 MB) -2016-06-13 17:30:54 INFO BlockManagerInfo:59 - Added rdd_3_1 in memory on localhost:61982 (size: 67.2 KB, free: 730.3 MB) -2016-06-13 17:30:54 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2717 bytes result sent to driver -2016-06-13 17:30:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 113 ms on localhost (2/2) -2016-06-13 17:30:54 INFO DAGScheduler:59 - ResultStage 1 (takeSample at GaussianMixture.scala:184) finished in 0.116 s -2016-06-13 17:30:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 17:30:54 INFO DAGScheduler:59 - Job 1 finished: takeSample at GaussianMixture.scala:184, took 0.136859 s -2016-06-13 17:30:54 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:30:54 INFO DAGScheduler:59 - Got job 2 (takeSample at GaussianMixture.scala:184) with 2 output partitions -2016-06-13 17:30:54 INFO DAGScheduler:59 - Final stage: ResultStage 2(takeSample at GaussianMixture.scala:184) -2016-06-13 17:30:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:54 INFO DAGScheduler:59 - Submitting ResultStage 2 (PartitionwiseSampledRDD[4] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:30:54 INFO MemoryStore:59 - ensureFreeSpace(20816) called with curMem=426701, maxMem=766075207 -2016-06-13 17:30:54 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 20.3 KB, free 730.2 MB) -2016-06-13 17:30:54 INFO MemoryStore:59 - ensureFreeSpace(8646) called with curMem=447517, maxMem=766075207 -2016-06-13 17:30:54 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 8.4 KB, free 730.2 MB) -2016-06-13 17:30:54 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:61982 (size: 8.4 KB, free: 730.3 MB) -2016-06-13 17:30:54 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (PartitionwiseSampledRDD[4] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:30:54 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 17:30:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2299 bytes) -2016-06-13 17:30:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2299 bytes) -2016-06-13 17:30:54 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-06-13 17:30:54 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 17:30:54 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:54 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:54 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 3407 bytes result sent to driver -2016-06-13 17:30:54 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 3451 bytes result sent to driver -2016-06-13 17:30:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 26 ms on localhost (1/2) -2016-06-13 17:30:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 25 ms on localhost (2/2) -2016-06-13 17:30:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 17:30:54 INFO DAGScheduler:59 - ResultStage 2 (takeSample at GaussianMixture.scala:184) finished in 0.028 s -2016-06-13 17:30:54 INFO DAGScheduler:59 - Job 2 finished: takeSample at GaussianMixture.scala:184, took 0.045224 s -2016-06-13 17:30:54 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 17:30:54 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 17:30:54 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 17:30:54 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 17:30:54 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=456163, maxMem=766075207 -2016-06-13 17:30:54 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(1095) called with curMem=457387, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 1095.0 B, free 730.1 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:61982 (size: 1095.0 B, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Created broadcast 4 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:61982 in memory (size: 8.4 KB, free: 730.3 MB) -2016-06-13 17:30:55 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:61982 in memory (size: 1984.0 B, free: 730.3 MB) -2016-06-13 17:30:55 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 17:30:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:55 INFO DAGScheduler:59 - Got job 3 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:61982 in memory (size: 2.0 KB, free: 730.3 MB) -2016-06-13 17:30:55 INFO DAGScheduler:59 - Final stage: ResultStage 3(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:55 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 17:30:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:55 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=417979, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 6.1 KB, free 730.2 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(3180) called with curMem=424219, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:61982 (size: 3.1 KB, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:55 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 17:30:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:55 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-06-13 17:30:55 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-06-13 17:30:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:55 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2944 bytes result sent to driver -2016-06-13 17:30:55 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2944 bytes result sent to driver -2016-06-13 17:30:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 234 ms on localhost (1/2) -2016-06-13 17:30:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 237 ms on localhost (2/2) -2016-06-13 17:30:55 INFO DAGScheduler:59 - ResultStage 3 (aggregate at GaussianMixture.scala:201) finished in 0.238 s -2016-06-13 17:30:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 17:30:55 INFO DAGScheduler:59 - Job 3 finished: aggregate at GaussianMixture.scala:201, took 0.253329 s -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=427399, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(1165) called with curMem=428623, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1165.0 B, free 730.2 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:61982 (size: 1165.0 B, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Created broadcast 6 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:55 INFO DAGScheduler:59 - Got job 4 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:55 INFO DAGScheduler:59 - Final stage: ResultStage 4(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:55 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=429788, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 6.1 KB, free 730.2 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=436028, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.2 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:55 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 17:30:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:55 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-06-13 17:30:55 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-06-13 17:30:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:55 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2944 bytes result sent to driver -2016-06-13 17:30:55 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2944 bytes result sent to driver -2016-06-13 17:30:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 126 ms on localhost (1/2) -2016-06-13 17:30:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 129 ms on localhost (2/2) -2016-06-13 17:30:55 INFO DAGScheduler:59 - ResultStage 4 (aggregate at GaussianMixture.scala:201) finished in 0.131 s -2016-06-13 17:30:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 17:30:55 INFO DAGScheduler:59 - Job 4 finished: aggregate at GaussianMixture.scala:201, took 0.147125 s -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=439281, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(1170) called with curMem=440505, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 1170.0 B, free 730.2 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:61982 (size: 1170.0 B, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Created broadcast 8 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:55 INFO DAGScheduler:59 - Got job 5 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:55 INFO DAGScheduler:59 - Final stage: ResultStage 5(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:55 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=441675, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 6.1 KB, free 730.2 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(3248) called with curMem=447915, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.2 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:55 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 17:30:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:55 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-06-13 17:30:55 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-06-13 17:30:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:55 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2944 bytes result sent to driver -2016-06-13 17:30:55 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2944 bytes result sent to driver -2016-06-13 17:30:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 34 ms on localhost (1/2) -2016-06-13 17:30:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 33 ms on localhost (2/2) -2016-06-13 17:30:55 INFO DAGScheduler:59 - ResultStage 5 (aggregate at GaussianMixture.scala:201) finished in 0.035 s -2016-06-13 17:30:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 17:30:55 INFO DAGScheduler:59 - Job 5 finished: aggregate at GaussianMixture.scala:201, took 0.056366 s -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=451163, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(1163) called with curMem=452387, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 1163.0 B, free 730.2 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:61982 (size: 1163.0 B, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Created broadcast 10 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:55 INFO DAGScheduler:59 - Got job 6 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:55 INFO DAGScheduler:59 - Final stage: ResultStage 6(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:55 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=453550, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=459790, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:55 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-06-13 17:30:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:55 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-06-13 17:30:55 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-06-13 17:30:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:55 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2944 bytes result sent to driver -2016-06-13 17:30:55 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2944 bytes result sent to driver -2016-06-13 17:30:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 21 ms on localhost (1/2) -2016-06-13 17:30:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 20 ms on localhost (2/2) -2016-06-13 17:30:55 INFO DAGScheduler:59 - ResultStage 6 (aggregate at GaussianMixture.scala:201) finished in 0.022 s -2016-06-13 17:30:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 17:30:55 INFO DAGScheduler:59 - Job 6 finished: aggregate at GaussianMixture.scala:201, took 0.035727 s -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=463043, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(1166) called with curMem=464267, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 1166.0 B, free 730.1 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:61982 (size: 1166.0 B, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Created broadcast 12 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:55 INFO DAGScheduler:59 - Got job 7 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:55 INFO DAGScheduler:59 - Final stage: ResultStage 7(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:55 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=465433, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(3255) called with curMem=471673, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:55 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-06-13 17:30:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:55 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-06-13 17:30:55 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-06-13 17:30:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:55 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2944 bytes result sent to driver -2016-06-13 17:30:55 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2944 bytes result sent to driver -2016-06-13 17:30:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 18 ms on localhost (1/2) -2016-06-13 17:30:55 INFO DAGScheduler:59 - ResultStage 7 (aggregate at GaussianMixture.scala:201) finished in 0.019 s -2016-06-13 17:30:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 19 ms on localhost (2/2) -2016-06-13 17:30:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 17:30:55 INFO DAGScheduler:59 - Job 7 finished: aggregate at GaussianMixture.scala:201, took 0.033244 s -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=474928, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=476152, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 1168.0 B, free 730.1 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:61982 (size: 1168.0 B, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Created broadcast 14 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:55 INFO DAGScheduler:59 - Got job 8 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:55 INFO DAGScheduler:59 - Final stage: ResultStage 8(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:55 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=477320, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=483560, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:55 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 17:30:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:55 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-06-13 17:30:55 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-06-13 17:30:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:55 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2944 bytes result sent to driver -2016-06-13 17:30:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:55 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2944 bytes result sent to driver -2016-06-13 17:30:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 29 ms on localhost (1/2) -2016-06-13 17:30:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 29 ms on localhost (2/2) -2016-06-13 17:30:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 17:30:55 INFO DAGScheduler:59 - ResultStage 8 (aggregate at GaussianMixture.scala:201) finished in 0.036 s -2016-06-13 17:30:55 INFO DAGScheduler:59 - Job 8 finished: aggregate at GaussianMixture.scala:201, took 0.056010 s -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=486813, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(1163) called with curMem=488037, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1163.0 B, free 730.1 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:61982 (size: 1163.0 B, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Created broadcast 16 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:55 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:61982 in memory (size: 1168.0 B, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:55 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 17:30:55 INFO DAGScheduler:59 - Got job 9 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:55 INFO DAGScheduler:59 - Final stage: ResultStage 9(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:55 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:61982 in memory (size: 1166.0 B, free: 730.3 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=465430, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=462177, maxMem=766075207 -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:55 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:55 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:61982 in memory (size: 1163.0 B, free: 730.3 MB) -2016-06-13 17:30:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:55 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:55 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-06-13 17:30:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:55 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2944 bytes result sent to driver -2016-06-13 17:30:55 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2944 bytes result sent to driver -2016-06-13 17:30:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 20 ms on localhost (1/2) -2016-06-13 17:30:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 22 ms on localhost (2/2) -2016-06-13 17:30:55 INFO DAGScheduler:59 - ResultStage 9 (aggregate at GaussianMixture.scala:201) finished in 0.023 s -2016-06-13 17:30:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 17:30:55 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 17:30:55 INFO DAGScheduler:59 - Job 9 finished: aggregate at GaussianMixture.scala:201, took 0.039917 s -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=451161, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:61982 in memory (size: 1170.0 B, free: 730.3 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(1169) called with curMem=452385, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1169.0 B, free 730.2 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:61982 (size: 1169.0 B, free: 730.3 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Created broadcast 18 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:55 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:61982 in memory (size: 1165.0 B, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:61982 in memory (size: 3.1 KB, free: 730.3 MB) -2016-06-13 17:30:55 INFO DAGScheduler:59 - Got job 10 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:55 INFO DAGScheduler:59 - Final stage: ResultStage 10(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:55 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 17:30:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:55 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=429933, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 6.1 KB, free 730.2 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:61982 in memory (size: 1095.0 B, free: 730.3 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(3254) called with curMem=436173, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.2 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:55 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-06-13 17:30:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:55 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-06-13 17:30:55 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-06-13 17:30:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:55 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2944 bytes result sent to driver -2016-06-13 17:30:55 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2944 bytes result sent to driver -2016-06-13 17:30:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 20 ms on localhost (1/2) -2016-06-13 17:30:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 21 ms on localhost (2/2) -2016-06-13 17:30:55 INFO DAGScheduler:59 - ResultStage 10 (aggregate at GaussianMixture.scala:201) finished in 0.023 s -2016-06-13 17:30:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 17:30:55 INFO DAGScheduler:59 - Job 10 finished: aggregate at GaussianMixture.scala:201, took 0.043228 s -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=439427, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(1170) called with curMem=440651, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 1170.0 B, free 730.2 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:61982 (size: 1170.0 B, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Created broadcast 20 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:55 INFO DAGScheduler:59 - Got job 11 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:55 INFO DAGScheduler:59 - Final stage: ResultStage 11(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:55 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=441821, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 6.1 KB, free 730.2 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(3263) called with curMem=448061, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.2 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:55 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:55 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 17:30:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:55 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-06-13 17:30:55 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-06-13 17:30:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:55 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2944 bytes result sent to driver -2016-06-13 17:30:55 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2944 bytes result sent to driver -2016-06-13 17:30:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 19 ms on localhost (1/2) -2016-06-13 17:30:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 20 ms on localhost (2/2) -2016-06-13 17:30:55 INFO DAGScheduler:59 - ResultStage 11 (aggregate at GaussianMixture.scala:201) finished in 0.021 s -2016-06-13 17:30:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 17:30:55 INFO DAGScheduler:59 - Job 11 finished: aggregate at GaussianMixture.scala:201, took 0.035010 s -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=451324, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-06-13 17:30:55 INFO MemoryStore:59 - ensureFreeSpace(1171) called with curMem=452548, maxMem=766075207 -2016-06-13 17:30:55 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 1171.0 B, free 730.2 MB) -2016-06-13 17:30:55 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:61982 (size: 1171.0 B, free: 730.3 MB) -2016-06-13 17:30:55 INFO SparkContext:59 - Created broadcast 22 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Got job 12 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:56 INFO DAGScheduler:59 - Final stage: ResultStage 12(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=453719, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(3251) called with curMem=459959, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-06-13 17:30:56 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:56 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 14 ms on localhost (1/2) -2016-06-13 17:30:56 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 16 ms on localhost (2/2) -2016-06-13 17:30:56 INFO DAGScheduler:59 - ResultStage 12 (aggregate at GaussianMixture.scala:201) finished in 0.018 s -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 17:30:56 INFO DAGScheduler:59 - Job 12 finished: aggregate at GaussianMixture.scala:201, took 0.032390 s -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=463210, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1164) called with curMem=464434, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 1164.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:61982 (size: 1164.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 24 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Got job 13 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:56 INFO DAGScheduler:59 - Final stage: ResultStage 13(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=465598, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=471838, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-06-13 17:30:56 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:56 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 16 ms on localhost (1/2) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 17 ms on localhost (2/2) -2016-06-13 17:30:56 INFO DAGScheduler:59 - ResultStage 13 (aggregate at GaussianMixture.scala:201) finished in 0.018 s -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 17:30:56 INFO DAGScheduler:59 - Job 13 finished: aggregate at GaussianMixture.scala:201, took 0.031843 s -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=475091, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=476315, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 1168.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:61982 (size: 1168.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 26 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Got job 14 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:56 INFO DAGScheduler:59 - Final stage: ResultStage 14(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=477483, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=483723, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-06-13 17:30:56 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:56 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 15 ms on localhost (1/2) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 17 ms on localhost (2/2) -2016-06-13 17:30:56 INFO DAGScheduler:59 - ResultStage 14 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 17:30:56 INFO DAGScheduler:59 - Job 14 finished: aggregate at GaussianMixture.scala:201, took 0.033281 s -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=486976, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1163) called with curMem=488200, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1163.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:61982 (size: 1163.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 28 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Got job 15 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:56 INFO DAGScheduler:59 - Final stage: ResultStage 15(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=489363, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=495603, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-06-13 17:30:56 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:56 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 16 ms on localhost (1/2) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 17 ms on localhost (2/2) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 17:30:56 INFO DAGScheduler:59 - ResultStage 15 (aggregate at GaussianMixture.scala:201) finished in 0.018 s -2016-06-13 17:30:56 INFO DAGScheduler:59 - Job 15 finished: aggregate at GaussianMixture.scala:201, took 0.033076 s -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=498856, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1160) called with curMem=500080, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1160.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:61982 (size: 1160.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 30 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Got job 16 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:56 INFO DAGScheduler:59 - Final stage: ResultStage 16(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=501240, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(3252) called with curMem=507480, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-06-13 17:30:56 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:56 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 19 ms on localhost (1/2) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 19 ms on localhost (2/2) -2016-06-13 17:30:56 INFO DAGScheduler:59 - ResultStage 16 (aggregate at GaussianMixture.scala:201) finished in 0.020 s -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 17:30:56 INFO DAGScheduler:59 - Job 16 finished: aggregate at GaussianMixture.scala:201, took 0.036721 s -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=510732, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1176) called with curMem=511956, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 1176.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:61982 (size: 1176.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 32 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Got job 17 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:56 INFO DAGScheduler:59 - Final stage: ResultStage 17(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=513132, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(3252) called with curMem=519372, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-06-13 17:30:56 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:56 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 20 ms on localhost (1/2) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 20 ms on localhost (2/2) -2016-06-13 17:30:56 INFO DAGScheduler:59 - ResultStage 17 (aggregate at GaussianMixture.scala:201) finished in 0.021 s -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 17:30:56 INFO DAGScheduler:59 - Job 17 finished: aggregate at GaussianMixture.scala:201, took 0.037328 s -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=522624, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1165) called with curMem=523848, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 1165.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:61982 (size: 1165.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 34 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Got job 18 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:56 INFO DAGScheduler:59 - Final stage: ResultStage 18(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=525013, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=531253, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-06-13 17:30:56 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:56 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 23 ms on localhost (1/2) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 23 ms on localhost (2/2) -2016-06-13 17:30:56 INFO DAGScheduler:59 - ResultStage 18 (aggregate at GaussianMixture.scala:201) finished in 0.025 s -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 17:30:56 INFO DAGScheduler:59 - Job 18 finished: aggregate at GaussianMixture.scala:201, took 0.043724 s -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=534506, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=535730, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 1168.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:61982 (size: 1168.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 36 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Got job 19 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:56 INFO DAGScheduler:59 - Final stage: ResultStage 19(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=536898, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(3256) called with curMem=543138, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-06-13 17:30:56 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:56 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 21 ms on localhost (1/2) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 23 ms on localhost (2/2) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 17:30:56 INFO DAGScheduler:59 - ResultStage 19 (aggregate at GaussianMixture.scala:201) finished in 0.024 s -2016-06-13 17:30:56 INFO DAGScheduler:59 - Job 19 finished: aggregate at GaussianMixture.scala:201, took 0.040927 s -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=546394, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1167) called with curMem=547618, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 1167.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:61982 (size: 1167.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 38 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Got job 20 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:56 INFO DAGScheduler:59 - Final stage: ResultStage 20(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=548785, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(3254) called with curMem=555025, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-06-13 17:30:56 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:56 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 21 ms on localhost (1/2) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 23 ms on localhost (2/2) -2016-06-13 17:30:56 INFO DAGScheduler:59 - ResultStage 20 (aggregate at GaussianMixture.scala:201) finished in 0.024 s -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 17:30:56 INFO DAGScheduler:59 - Job 20 finished: aggregate at GaussianMixture.scala:201, took 0.040046 s -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=558279, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1163) called with curMem=559503, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 1163.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:61982 (size: 1163.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 40 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Got job 21 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:56 INFO DAGScheduler:59 - Final stage: ResultStage 21(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=560666, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(3251) called with curMem=566906, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-06-13 17:30:56 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:56 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 37 ms on localhost (1/2) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 38 ms on localhost (2/2) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 17:30:56 INFO DAGScheduler:59 - ResultStage 21 (aggregate at GaussianMixture.scala:201) finished in 0.040 s -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Job 21 finished: aggregate at GaussianMixture.scala:201, took 0.064505 s -2016-06-13 17:30:56 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=559496, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:61982 in memory (size: 1167.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1166) called with curMem=559496, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1166.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:61982 (size: 1166.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 42 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 17:30:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Got job 22 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:56 INFO DAGScheduler:59 - Final stage: ResultStage 22(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:61982 in memory (size: 1168.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=548774, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=545521, maxMem=766075207 -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:61982 in memory (size: 1165.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-06-13 17:30:56 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:56 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 17 ms on localhost (1/2) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 17 ms on localhost (2/2) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 17:30:56 INFO DAGScheduler:59 - ResultStage 22 (aggregate at GaussianMixture.scala:201) finished in 0.020 s -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Job 22 finished: aggregate at GaussianMixture.scala:201, took 0.035777 s -2016-06-13 17:30:56 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=536893, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1162) called with curMem=535717, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 1162.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:61982 in memory (size: 1176.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:61982 (size: 1162.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 44 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:56 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Got job 23 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:56 INFO DAGScheduler:59 - Final stage: ResultStage 23(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:61982 in memory (size: 1160.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=526227, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(3256) called with curMem=531243, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-06-13 17:30:56 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-06-13 17:30:56 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:56 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 15 ms on localhost (1/2) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 18 ms on localhost (2/2) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 17:30:56 INFO DAGScheduler:59 - ResultStage 23 (aggregate at GaussianMixture.scala:201) finished in 0.019 s -2016-06-13 17:30:56 INFO DAGScheduler:59 - Job 23 finished: aggregate at GaussianMixture.scala:201, took 0.033286 s -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:61982 in memory (size: 1163.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=522619, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1164) called with curMem=514350, maxMem=766075207 -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 1164.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:61982 (size: 1164.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 46 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:61982 in memory (size: 1168.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Got job 24 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:56 INFO DAGScheduler:59 - Final stage: ResultStage 24(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:56 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=502465, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:61982 in memory (size: 1164.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(3255) called with curMem=507481, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-06-13 17:30:56 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:61982 in memory (size: 1171.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:56 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 15 ms on localhost (1/2) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 20 ms on localhost (2/2) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-06-13 17:30:56 INFO DAGScheduler:59 - ResultStage 24 (aggregate at GaussianMixture.scala:201) finished in 0.022 s -2016-06-13 17:30:56 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Job 24 finished: aggregate at GaussianMixture.scala:201, took 0.041317 s -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=486953, maxMem=766075207 -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:61982 in memory (size: 1170.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1167) called with curMem=488177, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 1167.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:61982 (size: 1167.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 48 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:56 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:61982 in memory (size: 1169.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Got job 25 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:56 INFO DAGScheduler:59 - Final stage: ResultStage 25(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=474204, maxMem=766075207 -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(3256) called with curMem=474204, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:61982 in memory (size: 1163.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-06-13 17:30:56 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:56 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 16 ms on localhost (1/2) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 16 ms on localhost (2/2) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-06-13 17:30:56 INFO DAGScheduler:59 - ResultStage 25 (aggregate at GaussianMixture.scala:201) finished in 0.019 s -2016-06-13 17:30:56 INFO DAGScheduler:59 - Job 25 finished: aggregate at GaussianMixture.scala:201, took 0.035012 s -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=475073, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=476297, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 1168.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:61982 (size: 1168.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 50 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Got job 26 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:56 INFO DAGScheduler:59 - Final stage: ResultStage 26(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=477465, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=483705, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-06-13 17:30:56 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:56 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 17 ms on localhost (1/2) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 20 ms on localhost (2/2) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-06-13 17:30:56 INFO DAGScheduler:59 - ResultStage 26 (aggregate at GaussianMixture.scala:201) finished in 0.020 s -2016-06-13 17:30:56 INFO DAGScheduler:59 - Job 26 finished: aggregate at GaussianMixture.scala:201, took 0.035806 s -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=486958, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1171) called with curMem=488182, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 1171.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:61982 (size: 1171.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 52 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Got job 27 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:56 INFO DAGScheduler:59 - Final stage: ResultStage 27(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=489353, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(3258) called with curMem=495593, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-06-13 17:30:56 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:56 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 20 ms on localhost (1/2) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 20 ms on localhost (2/2) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-06-13 17:30:56 INFO DAGScheduler:59 - ResultStage 27 (aggregate at GaussianMixture.scala:201) finished in 0.022 s -2016-06-13 17:30:56 INFO DAGScheduler:59 - Job 27 finished: aggregate at GaussianMixture.scala:201, took 0.035597 s -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=498851, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1164) called with curMem=500075, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 1164.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:61982 (size: 1164.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 54 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Got job 28 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:56 INFO DAGScheduler:59 - Final stage: ResultStage 28(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=501239, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(3251) called with curMem=507479, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:56 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-06-13 17:30:56 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:56 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:56 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 2944 bytes result sent to driver -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 14 ms on localhost (1/2) -2016-06-13 17:30:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 14 ms on localhost (2/2) -2016-06-13 17:30:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-06-13 17:30:56 INFO DAGScheduler:59 - ResultStage 28 (aggregate at GaussianMixture.scala:201) finished in 0.015 s -2016-06-13 17:30:56 INFO DAGScheduler:59 - Job 28 finished: aggregate at GaussianMixture.scala:201, took 0.030032 s -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=510730, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO MemoryStore:59 - ensureFreeSpace(1164) called with curMem=511954, maxMem=766075207 -2016-06-13 17:30:56 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 1164.0 B, free 730.1 MB) -2016-06-13 17:30:56 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:61982 (size: 1164.0 B, free: 730.3 MB) -2016-06-13 17:30:56 INFO SparkContext:59 - Created broadcast 56 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:56 INFO DAGScheduler:59 - Got job 29 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:56 INFO DAGScheduler:59 - Final stage: ResultStage 29(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:56 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=513118, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3251) called with curMem=519358, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 14 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 16 ms on localhost (2/2) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 29 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 29 finished: aggregate at GaussianMixture.scala:201, took 0.029919 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=522609, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1166) called with curMem=523833, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 1166.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:61982 (size: 1166.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 58 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 30 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 30(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=524999, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3251) called with curMem=531239, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 15 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 17 ms on localhost (2/2) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 30 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 30 finished: aggregate at GaussianMixture.scala:201, took 0.030878 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=534490, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1166) called with curMem=535714, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 1166.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:61982 (size: 1166.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 60 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 31 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 31(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=536880, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=543120, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 61 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 8 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 8 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 31 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 31 finished: aggregate at GaussianMixture.scala:201, took 0.020703 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=546373, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1167) called with curMem=547597, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 1167.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:61982 (size: 1167.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 62 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 32 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 32(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=548764, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=555004, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 8 ms on localhost (1/2) -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 11 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 32 (aggregate at GaussianMixture.scala:201) finished in 0.011 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 32 finished: aggregate at GaussianMixture.scala:201, took 0.020091 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=558257, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1167) called with curMem=559481, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 1167.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:61982 (size: 1167.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 64 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 33 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 33(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=560648, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=566888, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 8 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 8 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 33 (aggregate at GaussianMixture.scala:201) finished in 0.009 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 33 finished: aggregate at GaussianMixture.scala:201, took 0.016130 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=570141, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1163) called with curMem=571365, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 1163.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:61982 (size: 1163.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 66 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 34 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 34(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=572528, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=578768, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 67 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 17 ms on localhost (1/2) -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 19 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 34 (aggregate at GaussianMixture.scala:201) finished in 0.019 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 34 finished: aggregate at GaussianMixture.scala:201, took 0.027384 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=570142, maxMem=766075207 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:61982 in memory (size: 1164.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1170) called with curMem=568108, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 1170.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:61982 (size: 1170.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 68 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:61982 in memory (size: 1171.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 35 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 35(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:61982 in memory (size: 1168.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=549982, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3252) called with curMem=548758, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 69 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:61982 in memory (size: 1167.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:61982 in memory (size: 1164.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 13 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 14 ms on localhost (2/2) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 35 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 35 finished: aggregate at GaussianMixture.scala:201, took 0.027776 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=534480, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1163) called with curMem=526208, maxMem=766075207 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 1163.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:61982 (size: 1163.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:61982 in memory (size: 1162.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 70 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 36 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 36(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:61982 in memory (size: 1166.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=509851, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=509851, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 71 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:61982 in memory (size: 1163.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:61982 in memory (size: 1167.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 10 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 12 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 36 (aggregate at GaussianMixture.scala:201) finished in 0.012 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 36 finished: aggregate at GaussianMixture.scala:201, took 0.022719 s -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:61982 in memory (size: 1166.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=496443, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1166) called with curMem=494416, maxMem=766075207 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 1166.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:61982 (size: 1166.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 72 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:61982 in memory (size: 1166.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 37 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 37(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:61982 in memory (size: 1164.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=475073, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3254) called with curMem=478060, maxMem=766075207 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 73 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:61982 in memory (size: 1167.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 9 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 11 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 37 (aggregate at GaussianMixture.scala:201) finished in 0.011 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 37 finished: aggregate at GaussianMixture.scala:201, took 0.021100 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=463190, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1166) called with curMem=464414, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 1166.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:61982 (size: 1166.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 74 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 38 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 38(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=465580, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3252) called with curMem=471820, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 75 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 7 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 7 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 38 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 38 finished: aggregate at GaussianMixture.scala:201, took 0.016139 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=475072, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1166) called with curMem=476296, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 1166.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:61982 (size: 1166.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 76 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 39 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 39(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=477462, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3260) called with curMem=483702, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 77 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 9 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 10 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 39 (aggregate at GaussianMixture.scala:201) finished in 0.010 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 39 finished: aggregate at GaussianMixture.scala:201, took 0.018676 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=486962, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1164) called with curMem=488186, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 1164.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:61982 (size: 1164.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 78 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 40 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 40(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=489350, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=495590, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 79 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 7 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 9 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 40 (aggregate at GaussianMixture.scala:201) finished in 0.009 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 40 finished: aggregate at GaussianMixture.scala:201, took 0.018272 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=498843, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1165) called with curMem=500067, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 1165.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:61982 (size: 1165.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 80 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 41 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 41(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=501232, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=507472, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 81 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 8 ms on localhost (1/2) -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 11 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 41 (aggregate at GaussianMixture.scala:201) finished in 0.011 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 41 finished: aggregate at GaussianMixture.scala:201, took 0.021925 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=510725, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1171) called with curMem=511949, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 1171.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:61982 (size: 1171.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 82 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 42 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 42(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=513120, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=519360, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 83 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 10 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 10 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 42 (aggregate at GaussianMixture.scala:201) finished in 0.011 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 42 finished: aggregate at GaussianMixture.scala:201, took 0.021089 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=522613, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1167) called with curMem=523837, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 1167.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:61982 (size: 1167.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 84 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 43 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 43(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=525004, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3259) called with curMem=531244, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 85 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 7 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 8 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 43 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 43 finished: aggregate at GaussianMixture.scala:201, took 0.016118 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=534503, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1165) called with curMem=535727, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 1165.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:61982 (size: 1165.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 86 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 44 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 44(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=536892, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=543132, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 87 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 9 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 8 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 44 (aggregate at GaussianMixture.scala:201) finished in 0.009 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 44 finished: aggregate at GaussianMixture.scala:201, took 0.018427 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=546385, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1163) called with curMem=547609, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 1163.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:61982 (size: 1163.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 88 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 45 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 45(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=548772, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3252) called with curMem=555012, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 89 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 10 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 10 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 45 (aggregate at GaussianMixture.scala:201) finished in 0.011 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 45 finished: aggregate at GaussianMixture.scala:201, took 0.019374 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=558264, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1171) called with curMem=559488, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 1171.0 B, free 730.1 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:61982 (size: 1171.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 90 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 46 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 46(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=560659, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3252) called with curMem=566899, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 91 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 12 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 13 ms on localhost (2/2) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 46 (aggregate at GaussianMixture.scala:201) finished in 0.015 s -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 46 finished: aggregate at GaussianMixture.scala:201, took 0.026250 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=570151, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1166) called with curMem=571375, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 1166.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:61982 (size: 1166.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 92 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 47 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 47(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=572541, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=578781, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 93 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 47 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 93, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 94, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 93) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 94) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 94). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 93). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 94) in 8 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 93) in 9 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 47 (aggregate at GaussianMixture.scala:201) finished in 0.009 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 47 finished: aggregate at GaussianMixture.scala:201, took 0.018015 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=582034, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1165) called with curMem=583258, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 1165.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:61982 (size: 1165.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 94 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 48 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 48(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=584423, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3255) called with curMem=590663, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 95 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 48 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 95, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 96, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 95) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 96) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 95). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 96). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 95) in 9 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 96) in 10 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 48 (aggregate at GaussianMixture.scala:201) finished in 0.011 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 48 finished: aggregate at GaussianMixture.scala:201, took 0.019384 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=593918, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1177) called with curMem=595142, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 1177.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:61982 (size: 1177.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 96 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 49 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 49(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=596319, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3254) called with curMem=602559, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 97 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 49 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 97, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 98, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 97) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 98) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 97). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 98). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 97) in 9 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 98) in 9 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 49 (aggregate at GaussianMixture.scala:201) finished in 0.010 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 49 finished: aggregate at GaussianMixture.scala:201, took 0.018322 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=605813, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1171) called with curMem=607037, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 1171.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:61982 (size: 1171.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 98 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 50 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 50(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=608208, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=614448, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 99 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 50 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 9 ms on localhost (1/2) -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 11 ms on localhost (2/2) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 50 (aggregate at GaussianMixture.scala:201) finished in 0.011 s -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 50 finished: aggregate at GaussianMixture.scala:201, took 0.021549 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=617701, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1164) called with curMem=618925, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 1164.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:61982 (size: 1164.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 100 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 51 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 51(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=620089, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=626329, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 101 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 51 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 8 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 8 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 51 (aggregate at GaussianMixture.scala:201) finished in 0.009 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 51 finished: aggregate at GaussianMixture.scala:201, took 0.019164 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=629582, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1165) called with curMem=630806, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 1165.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:61982 (size: 1165.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 102 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 52 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 52(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 52 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=631971, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3254) called with curMem=638211, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 103 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 52 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 103, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 104, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 104) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 103) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 103). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 104). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 103) in 11 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 104) in 10 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 52 (aggregate at GaussianMixture.scala:201) finished in 0.011 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 52 finished: aggregate at GaussianMixture.scala:201, took 0.020155 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=641465, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1171) called with curMem=642689, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 1171.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:61982 (size: 1171.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 104 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 53 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 53(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=643860, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3255) called with curMem=650100, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 105 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 105, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 106, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 106) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 105) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 106). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 105). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 106) in 11 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 105) in 13 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 53 (aggregate at GaussianMixture.scala:201) finished in 0.013 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 53 finished: aggregate at GaussianMixture.scala:201, took 0.022729 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=653355, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1163) called with curMem=654579, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 1163.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:61982 (size: 1163.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 106 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 54 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 54(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=655742, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3249) called with curMem=661982, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 107 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 54 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 107) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 108) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 108). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 107). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 108) in 14 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 107) in 15 ms on localhost (2/2) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 54 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 54 finished: aggregate at GaussianMixture.scala:201, took 0.028049 s -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=665231, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1164) called with curMem=666455, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 1164.0 B, free 729.9 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:61982 (size: 1164.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 108 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 55 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 55(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=667619, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 6.1 KB, free 729.9 MB) -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3254) called with curMem=673859, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 3.2 KB, free 729.9 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.2 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:61982 in memory (size: 1163.0 B, free: 730.2 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 109 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 55 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 109, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 110, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 109) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 110) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:61982 in memory (size: 1170.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 109). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 110). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 110) in 9 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 109) in 11 ms on localhost (2/2) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_107_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 55 (aggregate at GaussianMixture.scala:201) finished in 0.012 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 55 finished: aggregate at GaussianMixture.scala:201, took 0.035709 s -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=652127, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_106_piece0 on localhost:61982 in memory (size: 1163.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1163) called with curMem=652188, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 1163.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:61982 (size: 1163.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 110 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_105_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_104_piece0 on localhost:61982 in memory (size: 1171.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 56 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 56(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_103_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=631967, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=638207, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 111 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_102_piece0 on localhost:61982 in memory (size: 1165.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 56 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 2 tasks -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 111, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 112, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 112) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 111) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_101_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_100_piece0 on localhost:61982 in memory (size: 1164.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 111). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 112). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 111) in 9 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 112) in 9 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 56 (aggregate at GaussianMixture.scala:201) finished in 0.009 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_99_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 56 finished: aggregate at GaussianMixture.scala:201, took 0.020364 s -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=617697, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_98_piece0 on localhost:61982 in memory (size: 1171.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1164) called with curMem=617750, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 1164.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:61982 (size: 1164.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 112 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_97_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_96_piece0 on localhost:61982 in memory (size: 1177.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Got job 57 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:57 INFO DAGScheduler:59 - Final stage: ResultStage 57(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_95_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=595135, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_94_piece0 on localhost:61982 in memory (size: 1165.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(3259) called with curMem=601375, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO SparkContext:59 - Created broadcast 113 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:57 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 57 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 2 tasks -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_93_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 113, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 114, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:57 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 114) -2016-06-13 17:30:57 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 113) -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_92_piece0 on localhost:61982 in memory (size: 1166.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:57 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_91_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 114). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 113). 2944 bytes result sent to driver -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 114) in 8 ms on localhost (1/2) -2016-06-13 17:30:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 113) in 8 ms on localhost (2/2) -2016-06-13 17:30:57 INFO DAGScheduler:59 - ResultStage 57 (aggregate at GaussianMixture.scala:201) finished in 0.010 s -2016-06-13 17:30:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-06-13 17:30:57 INFO DAGScheduler:59 - Job 57 finished: aggregate at GaussianMixture.scala:201, took 0.018311 s -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_90_piece0 on localhost:61982 in memory (size: 1171.0 B, free: 730.3 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=579640, maxMem=766075207 -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - ensureFreeSpace(1163) called with curMem=577612, maxMem=766075207 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Removed broadcast_89_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:57 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 1163.0 B, free 730.0 MB) -2016-06-13 17:30:57 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-06-13 17:30:57 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:61982 (size: 1163.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 114 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_88_piece0 on localhost:61982 in memory (size: 1163.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_87_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:58 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Got job 58 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:58 INFO DAGScheduler:59 - Final stage: ResultStage 58(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting ResultStage 58 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:61982 in memory (size: 1165.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=559490, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_115 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(3257) called with curMem=558266, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_115_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_115_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 115 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 58 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 2 tasks -2016-06-13 17:30:58 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 115, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 116, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 115) -2016-06-13 17:30:58 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 116) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:61982 in memory (size: 1167.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 115). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 116). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 116) in 11 ms on localhost (1/2) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 115) in 13 ms on localhost (2/2) -2016-06-13 17:30:58 INFO DAGScheduler:59 - ResultStage 58 (aggregate at GaussianMixture.scala:201) finished in 0.013 s -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-06-13 17:30:58 INFO DAGScheduler:59 - Job 58 finished: aggregate at GaussianMixture.scala:201, took 0.022583 s -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=543985, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_116 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:61982 in memory (size: 1171.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1172) called with curMem=535716, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_116_piece0 stored as bytes in memory (estimated size 1172.0 B, free 730.1 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_116_piece0 in memory on localhost:61982 (size: 1172.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 116 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:61982 in memory (size: 1165.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Got job 59 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:58 INFO DAGScheduler:59 - Final stage: ResultStage 59(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:58 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting ResultStage 59 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=522618, maxMem=766075207 -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:61982 in memory (size: 1164.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_117 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(3259) called with curMem=528858, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_117_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_117_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 117 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 59 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 2 tasks -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 117, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 118, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 117) -2016-06-13 17:30:58 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 118) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:61982 in memory (size: 1166.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:58 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 117). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 118). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 117) in 9 ms on localhost (1/2) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 118) in 9 ms on localhost (2/2) -2016-06-13 17:30:58 INFO DAGScheduler:59 - ResultStage 59 (aggregate at GaussianMixture.scala:201) finished in 0.010 s -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-06-13 17:30:58 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Job 59 finished: aggregate at GaussianMixture.scala:201, took 0.019851 s -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=509569, maxMem=766075207 -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:61982 in memory (size: 1166.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_118 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1171) called with curMem=509569, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_118_piece0 stored as bytes in memory (estimated size 1171.0 B, free 730.1 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_118_piece0 in memory on localhost:61982 (size: 1171.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 118 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:58 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:61982 in memory (size: 1166.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Got job 60 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:58 INFO DAGScheduler:59 - Final stage: ResultStage 60(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting ResultStage 60 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=489363, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_119 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(3251) called with curMem=495603, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_119_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_119_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 119 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 60 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Adding task set 60.0 with 2 tasks -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 60.0 (TID 119, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 60.0 (TID 120, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO Executor:59 - Running task 0.0 in stage 60.0 (TID 119) -2016-06-13 17:30:58 INFO Executor:59 - Running task 1.0 in stage 60.0 (TID 120) -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:58 INFO Executor:59 - Finished task 1.0 in stage 60.0 (TID 120). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 60.0 (TID 120) in 7 ms on localhost (1/2) -2016-06-13 17:30:58 INFO Executor:59 - Finished task 0.0 in stage 60.0 (TID 119). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 60.0 (TID 119) in 8 ms on localhost (2/2) -2016-06-13 17:30:58 INFO DAGScheduler:59 - ResultStage 60 (aggregate at GaussianMixture.scala:201) finished in 0.010 s -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 60.0, whose tasks have all completed, from pool -2016-06-13 17:30:58 INFO DAGScheduler:59 - Job 60 finished: aggregate at GaussianMixture.scala:201, took 0.017317 s -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=498854, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_120 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1181) called with curMem=500078, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_120_piece0 stored as bytes in memory (estimated size 1181.0 B, free 730.1 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_120_piece0 in memory on localhost:61982 (size: 1181.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 120 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:58 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Got job 61 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:58 INFO DAGScheduler:59 - Final stage: ResultStage 61(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting ResultStage 61 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=501259, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_121 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=507499, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_121_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_121_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 121 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 61 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 2 tasks -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 121, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 122, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 122) -2016-06-13 17:30:58 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 121) -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:58 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 121). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 122). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 121) in 8 ms on localhost (1/2) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 122) in 7 ms on localhost (2/2) -2016-06-13 17:30:58 INFO DAGScheduler:59 - ResultStage 61 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-06-13 17:30:58 INFO DAGScheduler:59 - Job 61 finished: aggregate at GaussianMixture.scala:201, took 0.018295 s -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=510752, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_122 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1165) called with curMem=511976, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_122_piece0 stored as bytes in memory (estimated size 1165.0 B, free 730.1 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_122_piece0 in memory on localhost:61982 (size: 1165.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 122 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:58 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Got job 62 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:58 INFO DAGScheduler:59 - Final stage: ResultStage 62(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=513141, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_123 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=519381, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_123_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_123_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 123 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 62 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 2 tasks -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 123, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 62.0 (TID 124, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 123) -2016-06-13 17:30:58 INFO Executor:59 - Running task 1.0 in stage 62.0 (TID 124) -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:58 INFO Executor:59 - Finished task 1.0 in stage 62.0 (TID 124). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 123). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 62.0 (TID 124) in 6 ms on localhost (1/2) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 123) in 7 ms on localhost (2/2) -2016-06-13 17:30:58 INFO DAGScheduler:59 - ResultStage 62 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-06-13 17:30:58 INFO DAGScheduler:59 - Job 62 finished: aggregate at GaussianMixture.scala:201, took 0.014898 s -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=522634, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_124 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1166) called with curMem=523858, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_124_piece0 stored as bytes in memory (estimated size 1166.0 B, free 730.1 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_124_piece0 in memory on localhost:61982 (size: 1166.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 124 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:58 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Got job 63 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:58 INFO DAGScheduler:59 - Final stage: ResultStage 63(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting ResultStage 63 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=525024, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_125 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(3251) called with curMem=531264, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_125_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_125_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 125 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 63 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Adding task set 63.0 with 2 tasks -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 63.0 (TID 125, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 63.0 (TID 126, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO Executor:59 - Running task 0.0 in stage 63.0 (TID 125) -2016-06-13 17:30:58 INFO Executor:59 - Running task 1.0 in stage 63.0 (TID 126) -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:58 INFO Executor:59 - Finished task 1.0 in stage 63.0 (TID 126). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO Executor:59 - Finished task 0.0 in stage 63.0 (TID 125). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 63.0 (TID 126) in 9 ms on localhost (1/2) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 63.0 (TID 125) in 10 ms on localhost (2/2) -2016-06-13 17:30:58 INFO DAGScheduler:59 - ResultStage 63 (aggregate at GaussianMixture.scala:201) finished in 0.010 s -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 63.0, whose tasks have all completed, from pool -2016-06-13 17:30:58 INFO DAGScheduler:59 - Job 63 finished: aggregate at GaussianMixture.scala:201, took 0.020635 s -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=534515, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_126 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1165) called with curMem=535739, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_126_piece0 stored as bytes in memory (estimated size 1165.0 B, free 730.1 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_126_piece0 in memory on localhost:61982 (size: 1165.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 126 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:58 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Got job 64 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:58 INFO DAGScheduler:59 - Final stage: ResultStage 64(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting ResultStage 64 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=536904, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_127 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=543144, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_127_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_127_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 127 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 64 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Adding task set 64.0 with 2 tasks -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 64.0 (TID 127, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 64.0 (TID 128, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO Executor:59 - Running task 0.0 in stage 64.0 (TID 127) -2016-06-13 17:30:58 INFO Executor:59 - Running task 1.0 in stage 64.0 (TID 128) -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:58 INFO Executor:59 - Finished task 0.0 in stage 64.0 (TID 127). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO Executor:59 - Finished task 1.0 in stage 64.0 (TID 128). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 64.0 (TID 127) in 9 ms on localhost (1/2) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 64.0 (TID 128) in 9 ms on localhost (2/2) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 64.0, whose tasks have all completed, from pool -2016-06-13 17:30:58 INFO DAGScheduler:59 - ResultStage 64 (aggregate at GaussianMixture.scala:201) finished in 0.011 s -2016-06-13 17:30:58 INFO DAGScheduler:59 - Job 64 finished: aggregate at GaussianMixture.scala:201, took 0.020692 s -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=546397, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_128 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1175) called with curMem=547621, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_128_piece0 stored as bytes in memory (estimated size 1175.0 B, free 730.1 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_128_piece0 in memory on localhost:61982 (size: 1175.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 128 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:58 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Got job 65 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:58 INFO DAGScheduler:59 - Final stage: ResultStage 65(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=548796, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_129 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(3263) called with curMem=555036, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_129_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_129_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 129 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 65 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 2 tasks -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 129, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 130, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 129) -2016-06-13 17:30:58 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 130) -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:58 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 129). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 130). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 129) in 7 ms on localhost (1/2) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 130) in 6 ms on localhost (2/2) -2016-06-13 17:30:58 INFO DAGScheduler:59 - ResultStage 65 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-06-13 17:30:58 INFO DAGScheduler:59 - Job 65 finished: aggregate at GaussianMixture.scala:201, took 0.017706 s -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=558299, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_130 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1163) called with curMem=559523, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_130_piece0 stored as bytes in memory (estimated size 1163.0 B, free 730.1 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_130_piece0 in memory on localhost:61982 (size: 1163.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 130 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:58 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Got job 66 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:58 INFO DAGScheduler:59 - Final stage: ResultStage 66(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting ResultStage 66 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=560686, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_131 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=566926, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_131_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_131_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 131 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 66 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Adding task set 66.0 with 2 tasks -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 66.0 (TID 131, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 66.0 (TID 132, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO Executor:59 - Running task 0.0 in stage 66.0 (TID 131) -2016-06-13 17:30:58 INFO Executor:59 - Running task 1.0 in stage 66.0 (TID 132) -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:58 INFO Executor:59 - Finished task 1.0 in stage 66.0 (TID 132). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO Executor:59 - Finished task 0.0 in stage 66.0 (TID 131). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 66.0 (TID 132) in 11 ms on localhost (1/2) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 66.0 (TID 131) in 12 ms on localhost (2/2) -2016-06-13 17:30:58 INFO DAGScheduler:59 - ResultStage 66 (aggregate at GaussianMixture.scala:201) finished in 0.013 s -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 66.0, whose tasks have all completed, from pool -2016-06-13 17:30:58 INFO DAGScheduler:59 - Job 66 finished: aggregate at GaussianMixture.scala:201, took 0.026813 s -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=570179, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_132 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1167) called with curMem=571403, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_132_piece0 stored as bytes in memory (estimated size 1167.0 B, free 730.0 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_132_piece0 in memory on localhost:61982 (size: 1167.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 132 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:58 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Got job 67 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:58 INFO DAGScheduler:59 - Final stage: ResultStage 67(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting ResultStage 67 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=572570, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_133 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=578810, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_133_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_133_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 133 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 67 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Adding task set 67.0 with 2 tasks -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 67.0 (TID 133, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 67.0 (TID 134, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO Executor:59 - Running task 0.0 in stage 67.0 (TID 133) -2016-06-13 17:30:58 INFO Executor:59 - Running task 1.0 in stage 67.0 (TID 134) -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:58 INFO Executor:59 - Finished task 0.0 in stage 67.0 (TID 133). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO Executor:59 - Finished task 1.0 in stage 67.0 (TID 134). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 67.0 (TID 133) in 15 ms on localhost (1/2) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 67.0 (TID 134) in 15 ms on localhost (2/2) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 67.0, whose tasks have all completed, from pool -2016-06-13 17:30:58 INFO DAGScheduler:59 - ResultStage 67 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 17:30:58 INFO DAGScheduler:59 - Job 67 finished: aggregate at GaussianMixture.scala:201, took 0.028018 s -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=582063, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_134 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1167) called with curMem=583287, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_134_piece0 stored as bytes in memory (estimated size 1167.0 B, free 730.0 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_134_piece0 in memory on localhost:61982 (size: 1167.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 134 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:58 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Got job 68 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:58 INFO DAGScheduler:59 - Final stage: ResultStage 68(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting ResultStage 68 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=584454, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_135 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=590694, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_135_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_135_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 135 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 68 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Adding task set 68.0 with 2 tasks -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 68.0 (TID 135, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 68.0 (TID 136, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO Executor:59 - Running task 0.0 in stage 68.0 (TID 135) -2016-06-13 17:30:58 INFO Executor:59 - Running task 1.0 in stage 68.0 (TID 136) -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:58 INFO Executor:59 - Finished task 0.0 in stage 68.0 (TID 135). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO Executor:59 - Finished task 1.0 in stage 68.0 (TID 136). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 68.0 (TID 135) in 10 ms on localhost (1/2) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 68.0 (TID 136) in 11 ms on localhost (2/2) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 68.0, whose tasks have all completed, from pool -2016-06-13 17:30:58 INFO DAGScheduler:59 - ResultStage 68 (aggregate at GaussianMixture.scala:201) finished in 0.012 s -2016-06-13 17:30:58 INFO DAGScheduler:59 - Job 68 finished: aggregate at GaussianMixture.scala:201, took 0.024750 s -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=593947, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_136 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=595171, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_136_piece0 stored as bytes in memory (estimated size 1168.0 B, free 730.0 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_136_piece0 in memory on localhost:61982 (size: 1168.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 136 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:58 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Got job 69 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:58 INFO DAGScheduler:59 - Final stage: ResultStage 69(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting ResultStage 69 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=596339, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_137 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(3252) called with curMem=602579, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_137_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_137_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 137 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 69 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Adding task set 69.0 with 2 tasks -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 69.0 (TID 137, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 69.0 (TID 138, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO Executor:59 - Running task 0.0 in stage 69.0 (TID 137) -2016-06-13 17:30:58 INFO Executor:59 - Running task 1.0 in stage 69.0 (TID 138) -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:58 INFO Executor:59 - Finished task 0.0 in stage 69.0 (TID 137). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO Executor:59 - Finished task 1.0 in stage 69.0 (TID 138). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 69.0 (TID 137) in 9 ms on localhost (1/2) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 69.0 (TID 138) in 10 ms on localhost (2/2) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 69.0, whose tasks have all completed, from pool -2016-06-13 17:30:58 INFO DAGScheduler:59 - ResultStage 69 (aggregate at GaussianMixture.scala:201) finished in 0.011 s -2016-06-13 17:30:58 INFO DAGScheduler:59 - Job 69 finished: aggregate at GaussianMixture.scala:201, took 0.022033 s -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=605831, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_138 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1169) called with curMem=607055, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_138_piece0 stored as bytes in memory (estimated size 1169.0 B, free 730.0 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_138_piece0 in memory on localhost:61982 (size: 1169.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 138 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:58 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Got job 70 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:58 INFO DAGScheduler:59 - Final stage: ResultStage 70(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting ResultStage 70 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=608224, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_139 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=614464, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_139_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_139_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 139 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 70 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Adding task set 70.0 with 2 tasks -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 70.0 (TID 139, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 70.0 (TID 140, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO Executor:59 - Running task 1.0 in stage 70.0 (TID 140) -2016-06-13 17:30:58 INFO Executor:59 - Running task 0.0 in stage 70.0 (TID 139) -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:58 INFO Executor:59 - Finished task 1.0 in stage 70.0 (TID 140). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO Executor:59 - Finished task 0.0 in stage 70.0 (TID 139). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 70.0 (TID 140) in 9 ms on localhost (1/2) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 70.0 (TID 139) in 12 ms on localhost (2/2) -2016-06-13 17:30:58 INFO DAGScheduler:59 - ResultStage 70 (aggregate at GaussianMixture.scala:201) finished in 0.013 s -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 70.0, whose tasks have all completed, from pool -2016-06-13 17:30:58 INFO DAGScheduler:59 - Job 70 finished: aggregate at GaussianMixture.scala:201, took 0.026858 s -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=617717, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_140 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1164) called with curMem=618941, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_140_piece0 stored as bytes in memory (estimated size 1164.0 B, free 730.0 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_140_piece0 in memory on localhost:61982 (size: 1164.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 140 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:58 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Got job 71 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:58 INFO DAGScheduler:59 - Final stage: ResultStage 71(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting ResultStage 71 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=620105, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_141 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(3262) called with curMem=626345, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_141_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_141_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 141 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 71 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Adding task set 71.0 with 2 tasks -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 71.0 (TID 141, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 71.0 (TID 142, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO Executor:59 - Running task 0.0 in stage 71.0 (TID 141) -2016-06-13 17:30:58 INFO Executor:59 - Running task 1.0 in stage 71.0 (TID 142) -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:58 INFO Executor:59 - Finished task 1.0 in stage 71.0 (TID 142). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO Executor:59 - Finished task 0.0 in stage 71.0 (TID 141). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 71.0 (TID 142) in 9 ms on localhost (1/2) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 71.0 (TID 141) in 10 ms on localhost (2/2) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 71.0, whose tasks have all completed, from pool -2016-06-13 17:30:58 INFO DAGScheduler:59 - ResultStage 71 (aggregate at GaussianMixture.scala:201) finished in 0.010 s -2016-06-13 17:30:58 INFO DAGScheduler:59 - Job 71 finished: aggregate at GaussianMixture.scala:201, took 0.020270 s -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=629607, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_142 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1165) called with curMem=630831, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_142_piece0 stored as bytes in memory (estimated size 1165.0 B, free 730.0 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_142_piece0 in memory on localhost:61982 (size: 1165.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 142 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:58 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Got job 72 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:58 INFO DAGScheduler:59 - Final stage: ResultStage 72(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting ResultStage 72 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=631996, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_143 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=638236, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_143_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_143_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 143 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 72 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Adding task set 72.0 with 2 tasks -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 72.0 (TID 143, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 72.0 (TID 144, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO Executor:59 - Running task 0.0 in stage 72.0 (TID 143) -2016-06-13 17:30:58 INFO Executor:59 - Running task 1.0 in stage 72.0 (TID 144) -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:58 INFO Executor:59 - Finished task 0.0 in stage 72.0 (TID 143). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO Executor:59 - Finished task 1.0 in stage 72.0 (TID 144). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 72.0 (TID 143) in 7 ms on localhost (1/2) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 72.0 (TID 144) in 8 ms on localhost (2/2) -2016-06-13 17:30:58 INFO DAGScheduler:59 - ResultStage 72 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 72.0, whose tasks have all completed, from pool -2016-06-13 17:30:58 INFO DAGScheduler:59 - Job 72 finished: aggregate at GaussianMixture.scala:201, took 0.014749 s -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=641489, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_144 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1166) called with curMem=642713, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_144_piece0 stored as bytes in memory (estimated size 1166.0 B, free 730.0 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_144_piece0 in memory on localhost:61982 (size: 1166.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 144 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:58 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Got job 73 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:58 INFO DAGScheduler:59 - Final stage: ResultStage 73(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting ResultStage 73 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=643879, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_145 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=650119, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_145_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_145_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 145 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 73 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Adding task set 73.0 with 2 tasks -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 73.0 (TID 145, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 73.0 (TID 146, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO Executor:59 - Running task 0.0 in stage 73.0 (TID 145) -2016-06-13 17:30:58 INFO Executor:59 - Running task 1.0 in stage 73.0 (TID 146) -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:58 INFO Executor:59 - Finished task 1.0 in stage 73.0 (TID 146). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO Executor:59 - Finished task 0.0 in stage 73.0 (TID 145). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 73.0 (TID 146) in 6 ms on localhost (1/2) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 73.0 (TID 145) in 8 ms on localhost (2/2) -2016-06-13 17:30:58 INFO DAGScheduler:59 - ResultStage 73 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 73.0, whose tasks have all completed, from pool -2016-06-13 17:30:58 INFO DAGScheduler:59 - Job 73 finished: aggregate at GaussianMixture.scala:201, took 0.014315 s -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=653372, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_146 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1167) called with curMem=654596, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_146_piece0 stored as bytes in memory (estimated size 1167.0 B, free 730.0 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_146_piece0 in memory on localhost:61982 (size: 1167.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 146 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:58 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Got job 74 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:58 INFO DAGScheduler:59 - Final stage: ResultStage 74(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting ResultStage 74 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=655763, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_147 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=662003, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_147_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_147_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 147 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 74 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Adding task set 74.0 with 2 tasks -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 74.0 (TID 147, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 74.0 (TID 148, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO Executor:59 - Running task 0.0 in stage 74.0 (TID 147) -2016-06-13 17:30:58 INFO Executor:59 - Running task 1.0 in stage 74.0 (TID 148) -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:58 INFO Executor:59 - Finished task 1.0 in stage 74.0 (TID 148). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO Executor:59 - Finished task 0.0 in stage 74.0 (TID 147). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 74.0 (TID 148) in 6 ms on localhost (1/2) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 74.0 (TID 147) in 7 ms on localhost (2/2) -2016-06-13 17:30:58 INFO DAGScheduler:59 - ResultStage 74 (aggregate at GaussianMixture.scala:201) finished in 0.007 s -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 74.0, whose tasks have all completed, from pool -2016-06-13 17:30:58 INFO DAGScheduler:59 - Job 74 finished: aggregate at GaussianMixture.scala:201, took 0.014310 s -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=665256, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_148 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=666480, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_148_piece0 stored as bytes in memory (estimated size 1168.0 B, free 729.9 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_148_piece0 in memory on localhost:61982 (size: 1168.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 148 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:30:58 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Got job 75 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:30:58 INFO DAGScheduler:59 - Final stage: ResultStage 75(aggregate at GaussianMixture.scala:201) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting ResultStage 75 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(6240) called with curMem=667648, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_149 stored as values in memory (estimated size 6.1 KB, free 729.9 MB) -2016-06-13 17:30:58 INFO MemoryStore:59 - ensureFreeSpace(3252) called with curMem=673888, maxMem=766075207 -2016-06-13 17:30:58 INFO MemoryStore:59 - Block broadcast_149_piece0 stored as bytes in memory (estimated size 3.2 KB, free 729.9 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Added broadcast_149_piece0 in memory on localhost:61982 (size: 3.2 KB, free: 730.2 MB) -2016-06-13 17:30:58 INFO SparkContext:59 - Created broadcast 149 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:30:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 75 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Adding task set 75.0 with 2 tasks -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 75.0 (TID 149, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 75.0 (TID 150, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 17:30:58 INFO Executor:59 - Running task 1.0 in stage 75.0 (TID 150) -2016-06-13 17:30:58 INFO Executor:59 - Running task 0.0 in stage 75.0 (TID 149) -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:30:58 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:30:58 INFO Executor:59 - Finished task 1.0 in stage 75.0 (TID 150). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO Executor:59 - Finished task 0.0 in stage 75.0 (TID 149). 2944 bytes result sent to driver -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 75.0 (TID 150) in 10 ms on localhost (1/2) -2016-06-13 17:30:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 75.0 (TID 149) in 11 ms on localhost (2/2) -2016-06-13 17:30:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 75.0, whose tasks have all completed, from pool -2016-06-13 17:30:58 INFO DAGScheduler:59 - ResultStage 75 (aggregate at GaussianMixture.scala:201) finished in 0.012 s -2016-06-13 17:30:58 INFO DAGScheduler:59 - Job 75 finished: aggregate at GaussianMixture.scala:201, took 0.020024 s -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_111_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_110_piece0 on localhost:61982 in memory (size: 1163.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_109_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_149_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 17:30:58 INFO ContextCleaner:59 - Cleaned accumulator 76 -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_148_piece0 on localhost:61982 in memory (size: 1168.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_147_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO ContextCleaner:59 - Cleaned accumulator 75 -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_146_piece0 on localhost:61982 in memory (size: 1167.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_145_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO ContextCleaner:59 - Cleaned accumulator 74 -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_144_piece0 on localhost:61982 in memory (size: 1166.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_143_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO ContextCleaner:59 - Cleaned accumulator 73 -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_142_piece0 on localhost:61982 in memory (size: 1165.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_141_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO ContextCleaner:59 - Cleaned accumulator 72 -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_140_piece0 on localhost:61982 in memory (size: 1164.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_139_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO ContextCleaner:59 - Cleaned accumulator 71 -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_138_piece0 on localhost:61982 in memory (size: 1169.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_137_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO ContextCleaner:59 - Cleaned accumulator 70 -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_136_piece0 on localhost:61982 in memory (size: 1168.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_135_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO ContextCleaner:59 - Cleaned accumulator 69 -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_134_piece0 on localhost:61982 in memory (size: 1167.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_133_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_132_piece0 on localhost:61982 in memory (size: 1167.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_131_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_130_piece0 on localhost:61982 in memory (size: 1163.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_129_piece0 on localhost:61982 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 17:30:58 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-06-13 17:30:58 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 17:30:58 INFO BlockManagerInfo:59 - Removed broadcast_128_piece0 on localhost:61982 in memory (size: 1175.0 B, free: 730.3 MB) -2016-06-13 17:30:58 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 17:30:58 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 17:30:58 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 17:30:58 INFO BlockManager:59 - BlockManager stopped -2016-06-13 17:30:58 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 17:30:58 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 17:30:58 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 17:30:58 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 17:30:58 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 17:30:58 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 17:30:58 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-639d48a8-037d-4552-864f-09f31b3972f9 -2016-06-13 17:35:16 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 17:35:17 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 17:35:17 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 17:35:17 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 17:35:17 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 17:35:18 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 17:35:18 INFO Remoting:74 - Starting remoting -2016-06-13 17:35:18 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:62011] -2016-06-13 17:35:18 INFO Utils:59 - Successfully started service 'sparkDriver' on port 62011. -2016-06-13 17:35:18 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 17:35:18 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 17:35:18 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-67f2692e-2366-48b5-8d68-75bd120fb423 -2016-06-13 17:35:18 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 17:35:18 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-7600a1b0-f405-402e-b1c0-2f0ef956237f\httpd-1c359a74-d5d7-4a23-972d-d7ac545ac407 -2016-06-13 17:35:18 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 17:35:18 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:35:18 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:62012 -2016-06-13 17:35:18 INFO Utils:59 - Successfully started service 'HTTP file server' on port 62012. -2016-06-13 17:35:18 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 17:35:19 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:35:19 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 17:35:19 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 17:35:19 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 17:35:19 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 17:35:19 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 17:35:19 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 62031. -2016-06-13 17:35:19 INFO NettyBlockTransferService:59 - Server created on 62031 -2016-06-13 17:35:19 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 17:35:19 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:62031 with 730.6 MB RAM, BlockManagerId(driver, localhost, 62031) -2016-06-13 17:35:19 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 17:35:20 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:20 INFO DAGScheduler:59 - Got job 0 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:35:20 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GaussianMixture.scala:171) -2016-06-13 17:35:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:20 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:20 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=0, maxMem=766075207 -2016-06-13 17:35:20 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:35:20 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1992, maxMem=766075207 -2016-06-13 17:35:20 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.6 MB) -2016-06-13 17:35:20 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:62031 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:35:20 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:35:20 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 17:35:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:20 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 17:35:20 INFO CacheManager:59 - Partition rdd_1_0 not found, computing it -2016-06-13 17:35:20 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=3242, maxMem=766075207 -2016-06-13 17:35:20 INFO MemoryStore:59 - Block rdd_1_0 stored as values in memory (estimated size 16.0 B, free 730.6 MB) -2016-06-13 17:35:20 INFO BlockManagerInfo:59 - Added rdd_1_0 in memory on localhost:62031 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:35:20 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1533 bytes result sent to driver -2016-06-13 17:35:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 71 ms on localhost (1/1) -2016-06-13 17:35:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 17:35:20 INFO DAGScheduler:59 - ResultStage 0 (first at GaussianMixture.scala:171) finished in 0.084 s -2016-06-13 17:35:20 INFO DAGScheduler:59 - Job 0 finished: first at GaussianMixture.scala:171, took 0.254269 s -2016-06-13 17:35:20 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:20 INFO DAGScheduler:59 - Got job 1 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:35:20 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GaussianMixture.scala:171) -2016-06-13 17:35:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:20 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:20 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=3258, maxMem=766075207 -2016-06-13 17:35:20 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:35:20 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=5250, maxMem=766075207 -2016-06-13 17:35:20 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.6 MB) -2016-06-13 17:35:20 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:62031 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:35:20 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:20 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:35:20 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 3 tasks -2016-06-13 17:35:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:20 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 17:35:20 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 17:35:20 INFO CacheManager:59 - Partition rdd_1_1 not found, computing it -2016-06-13 17:35:20 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 3) -2016-06-13 17:35:20 INFO CacheManager:59 - Partition rdd_1_3 not found, computing it -2016-06-13 17:35:20 INFO CacheManager:59 - Partition rdd_1_2 not found, computing it -2016-06-13 17:35:20 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=6500, maxMem=766075207 -2016-06-13 17:35:20 INFO MemoryStore:59 - Block rdd_1_3 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:35:20 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=6596, maxMem=766075207 -2016-06-13 17:35:20 INFO BlockManagerInfo:59 - Added rdd_1_3 in memory on localhost:62031 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:35:20 INFO MemoryStore:59 - Block rdd_1_2 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:35:20 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 3). 1696 bytes result sent to driver -2016-06-13 17:35:20 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=6692, maxMem=766075207 -2016-06-13 17:35:20 INFO MemoryStore:59 - Block rdd_1_1 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:35:20 INFO BlockManagerInfo:59 - Added rdd_1_2 in memory on localhost:62031 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:35:20 INFO BlockManagerInfo:59 - Added rdd_1_1 in memory on localhost:62031 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:35:20 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 1696 bytes result sent to driver -2016-06-13 17:35:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 3) in 329 ms on localhost (1/3) -2016-06-13 17:35:20 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 1696 bytes result sent to driver -2016-06-13 17:35:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 335 ms on localhost (2/3) -2016-06-13 17:35:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 339 ms on localhost (3/3) -2016-06-13 17:35:20 INFO DAGScheduler:59 - ResultStage 1 (first at GaussianMixture.scala:171) finished in 0.340 s -2016-06-13 17:35:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 17:35:20 INFO DAGScheduler:59 - Job 1 finished: first at GaussianMixture.scala:171, took 0.362142 s -2016-06-13 17:35:20 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:20 INFO DAGScheduler:59 - Got job 2 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:20 INFO DAGScheduler:59 - Final stage: ResultStage 2(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:20 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:20 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=6788, maxMem=766075207 -2016-06-13 17:35:20 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 17:35:20 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=8620, maxMem=766075207 -2016-06-13 17:35:20 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.6 MB) -2016-06-13 17:35:20 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:62031 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:35:20 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 2 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:35:20 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 4 tasks -2016-06-13 17:35:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:20 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 17:35:20 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 17:35:20 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-06-13 17:35:20 INFO Executor:59 - Running task 3.0 in stage 2.0 (TID 7) -2016-06-13 17:35:20 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:35:20 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:35:20 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:35:20 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:35:20 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2082 bytes result sent to driver -2016-06-13 17:35:20 INFO Executor:59 - Finished task 3.0 in stage 2.0 (TID 7). 2082 bytes result sent to driver -2016-06-13 17:35:20 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-06-13 17:35:20 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 6). 2082 bytes result sent to driver -2016-06-13 17:35:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 6) in 23 ms on localhost (1/4) -2016-06-13 17:35:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 26 ms on localhost (2/4) -2016-06-13 17:35:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 2.0 (TID 7) in 24 ms on localhost (3/4) -2016-06-13 17:35:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 29 ms on localhost (4/4) -2016-06-13 17:35:20 INFO DAGScheduler:59 - ResultStage 2 (takeSample at GaussianMixture.scala:184) finished in 0.030 s -2016-06-13 17:35:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 17:35:20 INFO DAGScheduler:59 - Job 2 finished: takeSample at GaussianMixture.scala:184, took 0.046869 s -2016-06-13 17:35:20 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:20 INFO DAGScheduler:59 - Got job 3 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:20 INFO DAGScheduler:59 - Final stage: ResultStage 3(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:20 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:35:20 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=9815, maxMem=766075207 -2016-06-13 17:35:20 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 18.8 KB, free 730.6 MB) -2016-06-13 17:35:20 INFO MemoryStore:59 - ensureFreeSpace(7883) called with curMem=29039, maxMem=766075207 -2016-06-13 17:35:20 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.6 MB) -2016-06-13 17:35:20 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:62031 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:35:20 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:20 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 4 tasks -2016-06-13 17:35:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:35:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:35:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:35:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 3.0 (TID 11, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:35:20 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 8) -2016-06-13 17:35:20 INFO Executor:59 - Running task 3.0 in stage 3.0 (TID 11) -2016-06-13 17:35:20 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 10) -2016-06-13 17:35:20 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 9) -2016-06-13 17:35:20 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:35:20 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:35:20 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:35:20 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 8). 2048 bytes result sent to driver -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:35:21 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 10). 2279 bytes result sent to driver -2016-06-13 17:35:20 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 9). 2269 bytes result sent to driver -2016-06-13 17:35:21 INFO Executor:59 - Finished task 3.0 in stage 3.0 (TID 11). 2269 bytes result sent to driver -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 8) in 29 ms on localhost (1/4) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 10) in 27 ms on localhost (2/4) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 9) in 30 ms on localhost (3/4) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 3.0 (TID 11) in 28 ms on localhost (4/4) -2016-06-13 17:35:21 INFO DAGScheduler:59 - ResultStage 3 (takeSample at GaussianMixture.scala:184) finished in 0.034 s -2016-06-13 17:35:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 17:35:21 INFO DAGScheduler:59 - Job 3 finished: takeSample at GaussianMixture.scala:184, took 0.053046 s -2016-06-13 17:35:21 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 17:35:21 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 17:35:21 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:62031 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:35:21 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:62031 in memory (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:35:21 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:62031 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:35:21 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:62031 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:35:21 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 17:35:21 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=304, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 464.0 B, free 730.6 MB) -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(854) called with curMem=768, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 854.0 B, free 730.6 MB) -2016-06-13 17:35:21 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:62031 (size: 854.0 B, free: 730.6 MB) -2016-06-13 17:35:21 INFO SparkContext:59 - Created broadcast 4 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:21 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:21 INFO DAGScheduler:59 - Got job 4 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:21 INFO DAGScheduler:59 - Final stage: ResultStage 4(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:21 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=1622, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.8 KB, free 730.6 MB) -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(2148) called with curMem=5542, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.6 MB) -2016-06-13 17:35:21 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:21 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:35:21 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 4 tasks -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 12, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 14, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 4.0 (TID 15, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:21 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 12) -2016-06-13 17:35:21 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 14) -2016-06-13 17:35:21 INFO Executor:59 - Running task 3.0 in stage 4.0 (TID 15) -2016-06-13 17:35:21 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 13) -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:35:21 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 12). 2708 bytes result sent to driver -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 12) in 28 ms on localhost (1/4) -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:35:21 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 13). 2708 bytes result sent to driver -2016-06-13 17:35:21 INFO Executor:59 - Finished task 3.0 in stage 4.0 (TID 15). 2708 bytes result sent to driver -2016-06-13 17:35:21 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 14). 2708 bytes result sent to driver -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 13) in 34 ms on localhost (2/4) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 4.0 (TID 15) in 35 ms on localhost (3/4) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 14) in 41 ms on localhost (4/4) -2016-06-13 17:35:21 INFO DAGScheduler:59 - ResultStage 4 (aggregate at GaussianMixture.scala:201) finished in 0.043 s -2016-06-13 17:35:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 17:35:21 INFO DAGScheduler:59 - Job 4 finished: aggregate at GaussianMixture.scala:201, took 0.056957 s -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=7690, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 464.0 B, free 730.6 MB) -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=8154, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.6 MB) -2016-06-13 17:35:21 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:62031 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:35:21 INFO SparkContext:59 - Created broadcast 6 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:21 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:21 INFO DAGScheduler:59 - Got job 5 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:21 INFO DAGScheduler:59 - Final stage: ResultStage 5(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:21 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=9007, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.8 KB, free 730.6 MB) -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(2150) called with curMem=12927, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.6 MB) -2016-06-13 17:35:21 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:21 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:35:21 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 4 tasks -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 16, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 17, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 18, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 5.0 (TID 19, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:21 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 16) -2016-06-13 17:35:21 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 18) -2016-06-13 17:35:21 INFO Executor:59 - Running task 3.0 in stage 5.0 (TID 19) -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:35:21 INFO Executor:59 - Finished task 3.0 in stage 5.0 (TID 19). 2708 bytes result sent to driver -2016-06-13 17:35:21 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 17) -2016-06-13 17:35:21 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 18). 2708 bytes result sent to driver -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 5.0 (TID 19) in 19 ms on localhost (1/4) -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:35:21 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 16). 2708 bytes result sent to driver -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 18) in 21 ms on localhost (2/4) -2016-06-13 17:35:21 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 17). 2708 bytes result sent to driver -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 16) in 26 ms on localhost (3/4) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 17) in 26 ms on localhost (4/4) -2016-06-13 17:35:21 INFO DAGScheduler:59 - ResultStage 5 (aggregate at GaussianMixture.scala:201) finished in 0.029 s -2016-06-13 17:35:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 17:35:21 INFO DAGScheduler:59 - Job 5 finished: aggregate at GaussianMixture.scala:201, took 0.044498 s -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=15077, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 464.0 B, free 730.6 MB) -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=15541, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.6 MB) -2016-06-13 17:35:21 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:62031 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:35:21 INFO SparkContext:59 - Created broadcast 8 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:21 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:21 INFO DAGScheduler:59 - Got job 6 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:21 INFO DAGScheduler:59 - Final stage: ResultStage 6(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:21 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=16394, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 3.8 KB, free 730.6 MB) -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(2150) called with curMem=20314, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.6 MB) -2016-06-13 17:35:21 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:21 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:35:21 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 4 tasks -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 20, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 21, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 22, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 6.0 (TID 23, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:21 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 22) -2016-06-13 17:35:21 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 21) -2016-06-13 17:35:21 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 20) -2016-06-13 17:35:21 INFO Executor:59 - Running task 3.0 in stage 6.0 (TID 23) -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:35:21 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 20). 2708 bytes result sent to driver -2016-06-13 17:35:21 INFO Executor:59 - Finished task 3.0 in stage 6.0 (TID 23). 2708 bytes result sent to driver -2016-06-13 17:35:21 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 22). 2708 bytes result sent to driver -2016-06-13 17:35:21 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 21). 2708 bytes result sent to driver -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 20) in 14 ms on localhost (1/4) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 6.0 (TID 23) in 12 ms on localhost (2/4) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 21) in 15 ms on localhost (3/4) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 22) in 15 ms on localhost (4/4) -2016-06-13 17:35:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 17:35:21 INFO DAGScheduler:59 - ResultStage 6 (aggregate at GaussianMixture.scala:201) finished in 0.018 s -2016-06-13 17:35:21 INFO DAGScheduler:59 - Job 6 finished: aggregate at GaussianMixture.scala:201, took 0.031753 s -2016-06-13 17:35:21 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:21 INFO DAGScheduler:59 - Got job 7 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:35:21 INFO DAGScheduler:59 - Final stage: ResultStage 7(first at GaussianMixture.scala:171) -2016-06-13 17:35:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:21 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=22464, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=24456, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.6 MB) -2016-06-13 17:35:21 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:62031 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:35:21 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:35:21 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 24, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:21 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 24) -2016-06-13 17:35:21 INFO CacheManager:59 - Partition rdd_3_0 not found, computing it -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=25708, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block rdd_3_0 stored as values in memory (estimated size 16.0 B, free 730.6 MB) -2016-06-13 17:35:21 INFO BlockManagerInfo:59 - Added rdd_3_0 in memory on localhost:62031 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:35:21 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 24). 1533 bytes result sent to driver -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 24) in 12 ms on localhost (1/1) -2016-06-13 17:35:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 17:35:21 INFO DAGScheduler:59 - ResultStage 7 (first at GaussianMixture.scala:171) finished in 0.012 s -2016-06-13 17:35:21 INFO DAGScheduler:59 - Job 7 finished: first at GaussianMixture.scala:171, took 0.030455 s -2016-06-13 17:35:21 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:21 INFO DAGScheduler:59 - Got job 8 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:35:21 INFO DAGScheduler:59 - Final stage: ResultStage 8(first at GaussianMixture.scala:171) -2016-06-13 17:35:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:21 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=25724, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=27716, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.6 MB) -2016-06-13 17:35:21 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:62031 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:35:21 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:21 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 8 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:35:21 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 3 tasks -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 25, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 26, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 8.0 (TID 27, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:21 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 26) -2016-06-13 17:35:21 INFO Executor:59 - Running task 2.0 in stage 8.0 (TID 27) -2016-06-13 17:35:21 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 25) -2016-06-13 17:35:21 INFO CacheManager:59 - Partition rdd_3_1 not found, computing it -2016-06-13 17:35:21 INFO CacheManager:59 - Partition rdd_3_2 not found, computing it -2016-06-13 17:35:21 INFO CacheManager:59 - Partition rdd_3_3 not found, computing it -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=28968, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block rdd_3_1 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=29064, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block rdd_3_2 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:35:21 INFO BlockManagerInfo:59 - Added rdd_3_1 in memory on localhost:62031 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=29160, maxMem=766075207 -2016-06-13 17:35:21 INFO BlockManagerInfo:59 - Added rdd_3_2 in memory on localhost:62031 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:35:21 INFO MemoryStore:59 - Block rdd_3_3 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:35:21 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 26). 1696 bytes result sent to driver -2016-06-13 17:35:21 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 25). 1696 bytes result sent to driver -2016-06-13 17:35:21 INFO BlockManagerInfo:59 - Added rdd_3_3 in memory on localhost:62031 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 26) in 15 ms on localhost (1/3) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 25) in 18 ms on localhost (2/3) -2016-06-13 17:35:21 INFO Executor:59 - Finished task 2.0 in stage 8.0 (TID 27). 1696 bytes result sent to driver -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 8.0 (TID 27) in 17 ms on localhost (3/3) -2016-06-13 17:35:21 INFO DAGScheduler:59 - ResultStage 8 (first at GaussianMixture.scala:171) finished in 0.020 s -2016-06-13 17:35:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 17:35:21 INFO DAGScheduler:59 - Job 8 finished: first at GaussianMixture.scala:171, took 0.035557 s -2016-06-13 17:35:21 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:21 INFO DAGScheduler:59 - Got job 9 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:21 INFO DAGScheduler:59 - Final stage: ResultStage 9(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:21 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=29256, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=31088, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.6 MB) -2016-06-13 17:35:21 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:62031 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:35:21 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 9 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:35:21 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 4 tasks -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 28, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 29, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 9.0 (TID 30, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 9.0 (TID 31, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:21 INFO Executor:59 - Running task 2.0 in stage 9.0 (TID 30) -2016-06-13 17:35:21 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 29) -2016-06-13 17:35:21 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 28) -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:35:21 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 29). 2082 bytes result sent to driver -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:35:21 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 28). 2082 bytes result sent to driver -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:35:21 INFO Executor:59 - Finished task 2.0 in stage 9.0 (TID 30). 2082 bytes result sent to driver -2016-06-13 17:35:21 INFO Executor:59 - Running task 3.0 in stage 9.0 (TID 31) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 29) in 17 ms on localhost (1/4) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 28) in 21 ms on localhost (2/4) -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:35:21 INFO Executor:59 - Finished task 3.0 in stage 9.0 (TID 31). 2082 bytes result sent to driver -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 9.0 (TID 31) in 22 ms on localhost (3/4) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 9.0 (TID 30) in 23 ms on localhost (4/4) -2016-06-13 17:35:21 INFO DAGScheduler:59 - ResultStage 9 (takeSample at GaussianMixture.scala:184) finished in 0.027 s -2016-06-13 17:35:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 17:35:21 INFO DAGScheduler:59 - Job 9 finished: takeSample at GaussianMixture.scala:184, took 0.043825 s -2016-06-13 17:35:21 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:21 INFO DAGScheduler:59 - Got job 10 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:21 INFO DAGScheduler:59 - Final stage: ResultStage 10(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:21 INFO DAGScheduler:59 - Submitting ResultStage 10 (PartitionwiseSampledRDD[4] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=32283, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(7881) called with curMem=51507, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:35:21 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:62031 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:35:21 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 10 (PartitionwiseSampledRDD[4] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:21 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 4 tasks -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 32, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 33, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 10.0 (TID 34, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 10.0 (TID 35, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:35:21 INFO Executor:59 - Running task 2.0 in stage 10.0 (TID 34) -2016-06-13 17:35:21 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 33) -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:35:21 INFO Executor:59 - Finished task 2.0 in stage 10.0 (TID 34). 2244 bytes result sent to driver -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:35:21 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 33). 2274 bytes result sent to driver -2016-06-13 17:35:21 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 32) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 10.0 (TID 34) in 18 ms on localhost (1/4) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 33) in 24 ms on localhost (2/4) -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:35:21 INFO Executor:59 - Running task 3.0 in stage 10.0 (TID 35) -2016-06-13 17:35:21 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 32). 2048 bytes result sent to driver -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:35:21 INFO Executor:59 - Finished task 3.0 in stage 10.0 (TID 35). 2274 bytes result sent to driver -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 32) in 38 ms on localhost (3/4) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 10.0 (TID 35) in 33 ms on localhost (4/4) -2016-06-13 17:35:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 17:35:21 INFO DAGScheduler:59 - ResultStage 10 (takeSample at GaussianMixture.scala:184) finished in 0.041 s -2016-06-13 17:35:21 INFO DAGScheduler:59 - Job 10 finished: takeSample at GaussianMixture.scala:184, took 0.065151 s -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=59388, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(850) called with curMem=59852, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 850.0 B, free 730.5 MB) -2016-06-13 17:35:21 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:62031 (size: 850.0 B, free: 730.6 MB) -2016-06-13 17:35:21 INFO SparkContext:59 - Created broadcast 14 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:21 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:21 INFO DAGScheduler:59 - Got job 11 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:21 INFO DAGScheduler:59 - Final stage: ResultStage 11(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:21 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=60702, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:35:21 INFO MemoryStore:59 - ensureFreeSpace(2147) called with curMem=64622, maxMem=766075207 -2016-06-13 17:35:21 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:35:21 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:21 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 11 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:35:21 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 4 tasks -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 36, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 37, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 11.0 (TID 38, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 11.0 (TID 39, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:21 INFO Executor:59 - Running task 2.0 in stage 11.0 (TID 38) -2016-06-13 17:35:21 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 37) -2016-06-13 17:35:21 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 36) -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:35:21 INFO Executor:59 - Finished task 2.0 in stage 11.0 (TID 38). 2708 bytes result sent to driver -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:35:21 INFO Executor:59 - Running task 3.0 in stage 11.0 (TID 39) -2016-06-13 17:35:21 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 36). 2708 bytes result sent to driver -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:35:21 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:35:21 INFO Executor:59 - Finished task 3.0 in stage 11.0 (TID 39). 2708 bytes result sent to driver -2016-06-13 17:35:21 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 37). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 11.0 (TID 38) in 24 ms on localhost (1/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 36) in 29 ms on localhost (2/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 11.0 (TID 39) in 27 ms on localhost (3/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 37) in 29 ms on localhost (4/4) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 17:35:22 INFO DAGScheduler:59 - ResultStage 11 (aggregate at GaussianMixture.scala:201) finished in 0.031 s -2016-06-13 17:35:22 INFO DAGScheduler:59 - Job 11 finished: aggregate at GaussianMixture.scala:201, took 0.049615 s -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=66769, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=67233, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:62031 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 16 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Got job 12 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:22 INFO DAGScheduler:59 - Final stage: ResultStage 12(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=68086, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=72006, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 12 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 4 tasks -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 40, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 41, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 12.0 (TID 42, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 12.0 (TID 43, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO Executor:59 - Running task 3.0 in stage 12.0 (TID 43) -2016-06-13 17:35:22 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 17:35:22 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 41) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 3.0 in stage 12.0 (TID 43). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 41). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 40) -2016-06-13 17:35:22 INFO Executor:59 - Running task 2.0 in stage 12.0 (TID 42) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 2.0 in stage 12.0 (TID 42). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 40). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:62031 in memory (size: 850.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 41) in 27 ms on localhost (1/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 40) in 46 ms on localhost (2/4) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:62031 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 12.0 (TID 43) in 46 ms on localhost (3/4) -2016-06-13 17:35:22 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 12.0 (TID 42) in 49 ms on localhost (4/4) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 17:35:22 INFO DAGScheduler:59 - ResultStage 12 (aggregate at GaussianMixture.scala:201) finished in 0.053 s -2016-06-13 17:35:22 INFO DAGScheduler:59 - Job 12 finished: aggregate at GaussianMixture.scala:201, took 0.105528 s -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:62031 in memory (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=36645, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 464.0 B, free 730.6 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=35117, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.6 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:62031 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:62031 in memory (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 18 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:22 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:62031 in memory (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 17:35:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Got job 13 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:22 INFO DAGScheduler:59 - Final stage: ResultStage 13(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:22 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=24087, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.8 KB, free 730.6 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:62031 in memory (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=28007, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.6 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 13 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 4 tasks -2016-06-13 17:35:22 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 44, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:62031 in memory (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 45, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 13.0 (TID 46, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 13.0 (TID 47, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO Executor:59 - Running task 2.0 in stage 13.0 (TID 46) -2016-06-13 17:35:22 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 45) -2016-06-13 17:35:22 INFO Executor:59 - Running task 3.0 in stage 13.0 (TID 47) -2016-06-13 17:35:22 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 44) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 2.0 in stage 13.0 (TID 46). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO Executor:59 - Finished task 3.0 in stage 13.0 (TID 47). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 44). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 13.0 (TID 46) in 17 ms on localhost (1/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 13.0 (TID 47) in 17 ms on localhost (2/4) -2016-06-13 17:35:22 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 17:35:22 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 45). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 44) in 25 ms on localhost (3/4) -2016-06-13 17:35:22 INFO DAGScheduler:59 - ResultStage 13 (aggregate at GaussianMixture.scala:201) finished in 0.026 s -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 45) in 25 ms on localhost (4/4) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 17:35:22 INFO DAGScheduler:59 - Job 13 finished: aggregate at GaussianMixture.scala:201, took 0.058238 s -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:62031 in memory (size: 854.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:22 INFO BlockManager:59 - Removing RDD 1 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Got job 14 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:35:22 INFO DAGScheduler:59 - Final stage: ResultStage 14(first at GaussianMixture.scala:171) -2016-06-13 17:35:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=15194, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(1251) called with curMem=17074, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 1251.0 B, free 730.6 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:62031 (size: 1251.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO ContextCleaner:59 - Cleaned RDD 1 -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 14 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 1 tasks -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 48, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:22 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 48) -2016-06-13 17:35:22 INFO CacheManager:59 - Partition rdd_5_0 not found, computing it -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=18325, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block rdd_5_0 stored as values in memory (estimated size 16.0 B, free 730.6 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added rdd_5_0 in memory on localhost:62031 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 48). 1533 bytes result sent to driver -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 48) in 12 ms on localhost (1/1) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 17:35:22 INFO DAGScheduler:59 - ResultStage 14 (first at GaussianMixture.scala:171) finished in 0.012 s -2016-06-13 17:35:22 INFO DAGScheduler:59 - Job 14 finished: first at GaussianMixture.scala:171, took 0.031006 s -2016-06-13 17:35:22 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Got job 15 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:35:22 INFO DAGScheduler:59 - Final stage: ResultStage 15(first at GaussianMixture.scala:171) -2016-06-13 17:35:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=18341, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(1251) called with curMem=20333, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1251.0 B, free 730.6 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:62031 (size: 1251.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 15 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 3 tasks -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 49, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 50, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 15.0 (TID 51, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 49) -2016-06-13 17:35:22 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 50) -2016-06-13 17:35:22 INFO Executor:59 - Running task 2.0 in stage 15.0 (TID 51) -2016-06-13 17:35:22 INFO CacheManager:59 - Partition rdd_5_3 not found, computing it -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=21584, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block rdd_5_3 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added rdd_5_3 in memory on localhost:62031 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO Executor:59 - Finished task 2.0 in stage 15.0 (TID 51). 1696 bytes result sent to driver -2016-06-13 17:35:22 INFO CacheManager:59 - Partition rdd_5_1 not found, computing it -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 15.0 (TID 51) in 13 ms on localhost (1/3) -2016-06-13 17:35:22 INFO CacheManager:59 - Partition rdd_5_2 not found, computing it -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=21680, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block rdd_5_1 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=21776, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block rdd_5_2 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added rdd_5_1 in memory on localhost:62031 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added rdd_5_2 in memory on localhost:62031 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 49). 1696 bytes result sent to driver -2016-06-13 17:35:22 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 50). 1696 bytes result sent to driver -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 49) in 28 ms on localhost (2/3) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 50) in 27 ms on localhost (3/3) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 17:35:22 INFO DAGScheduler:59 - ResultStage 15 (first at GaussianMixture.scala:171) finished in 0.029 s -2016-06-13 17:35:22 INFO DAGScheduler:59 - Job 15 finished: first at GaussianMixture.scala:171, took 0.043804 s -2016-06-13 17:35:22 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Got job 16 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:22 INFO DAGScheduler:59 - Final stage: ResultStage 16(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=21872, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(1194) called with curMem=23704, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 1194.0 B, free 730.6 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:62031 (size: 1194.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 16 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 4 tasks -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 52, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 53, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 16.0 (TID 54, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 16.0 (TID 55, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO Executor:59 - Running task 2.0 in stage 16.0 (TID 54) -2016-06-13 17:35:22 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 53) -2016-06-13 17:35:22 INFO Executor:59 - Running task 3.0 in stage 16.0 (TID 55) -2016-06-13 17:35:22 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 52) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 2.0 in stage 16.0 (TID 54). 2082 bytes result sent to driver -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 3.0 in stage 16.0 (TID 55). 2082 bytes result sent to driver -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 16.0 (TID 55) in 13 ms on localhost (1/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 16.0 (TID 54) in 15 ms on localhost (2/4) -2016-06-13 17:35:22 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 53). 2082 bytes result sent to driver -2016-06-13 17:35:22 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 52). 2082 bytes result sent to driver -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 53) in 21 ms on localhost (3/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 52) in 24 ms on localhost (4/4) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 17:35:22 INFO DAGScheduler:59 - ResultStage 16 (takeSample at GaussianMixture.scala:184) finished in 0.026 s -2016-06-13 17:35:22 INFO DAGScheduler:59 - Job 16 finished: takeSample at GaussianMixture.scala:184, took 0.039667 s -2016-06-13 17:35:22 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Got job 17 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:22 INFO DAGScheduler:59 - Final stage: ResultStage 17(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting ResultStage 17 (PartitionwiseSampledRDD[6] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=24898, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(7883) called with curMem=44122, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:62031 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (PartitionwiseSampledRDD[6] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 56, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 57, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 58, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 59, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:35:22 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 59) -2016-06-13 17:35:22 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 58) -2016-06-13 17:35:22 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 56) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 59). 2226 bytes result sent to driver -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 56). 2048 bytes result sent to driver -2016-06-13 17:35:22 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 57) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 59) in 15 ms on localhost (1/4) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 56) in 20 ms on localhost (2/4) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 57). 2269 bytes result sent to driver -2016-06-13 17:35:22 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 58). 2259 bytes result sent to driver -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 58) in 23 ms on localhost (3/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 57) in 25 ms on localhost (4/4) -2016-06-13 17:35:22 INFO DAGScheduler:59 - ResultStage 17 (takeSample at GaussianMixture.scala:184) finished in 0.026 s -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 17:35:22 INFO DAGScheduler:59 - Job 17 finished: takeSample at GaussianMixture.scala:184, took 0.043695 s -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=52005, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(850) called with curMem=52469, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 850.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:62031 (size: 850.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 24 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Got job 18 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:22 INFO DAGScheduler:59 - Final stage: ResultStage 18(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=53319, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=57239, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 60, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 61, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 62, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 63, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 63) -2016-06-13 17:35:22 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 62) -2016-06-13 17:35:22 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 61) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 63). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 60) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 60). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 61). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 63) in 19 ms on localhost (1/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 60) in 26 ms on localhost (2/4) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 62). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 61) in 34 ms on localhost (3/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 62) in 33 ms on localhost (4/4) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 17:35:22 INFO DAGScheduler:59 - ResultStage 18 (aggregate at GaussianMixture.scala:201) finished in 0.037 s -2016-06-13 17:35:22 INFO DAGScheduler:59 - Job 18 finished: aggregate at GaussianMixture.scala:201, took 0.056009 s -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=59384, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=59848, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:62031 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 26 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Got job 19 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:22 INFO DAGScheduler:59 - Final stage: ResultStage 19(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=60701, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(2151) called with curMem=64621, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 19 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 4 tasks -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 64, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 65, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 19.0 (TID 66, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 19.0 (TID 67, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO Executor:59 - Running task 2.0 in stage 19.0 (TID 66) -2016-06-13 17:35:22 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 65) -2016-06-13 17:35:22 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 64) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 2.0 in stage 19.0 (TID 66). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO Executor:59 - Running task 3.0 in stage 19.0 (TID 67) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 64). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 3.0 in stage 19.0 (TID 67). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 65). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 64) in 25 ms on localhost (1/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 19.0 (TID 66) in 24 ms on localhost (2/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 19.0 (TID 67) in 25 ms on localhost (3/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 65) in 29 ms on localhost (4/4) -2016-06-13 17:35:22 INFO DAGScheduler:59 - ResultStage 19 (aggregate at GaussianMixture.scala:201) finished in 0.031 s -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 17:35:22 INFO DAGScheduler:59 - Job 19 finished: aggregate at GaussianMixture.scala:201, took 0.050488 s -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=66772, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=67236, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:62031 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 28 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Got job 20 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:22 INFO DAGScheduler:59 - Final stage: ResultStage 20(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=68089, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(2151) called with curMem=72009, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 68, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 69, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 70, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 71, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 69) -2016-06-13 17:35:22 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 71) -2016-06-13 17:35:22 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 68) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 69). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 68). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:35:22 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 70) -2016-06-13 17:35:22 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 71). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 69) in 17 ms on localhost (1/4) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 70). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 68) in 20 ms on localhost (2/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 71) in 20 ms on localhost (3/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 70) in 24 ms on localhost (4/4) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 17:35:22 INFO DAGScheduler:59 - ResultStage 20 (aggregate at GaussianMixture.scala:201) finished in 0.028 s -2016-06-13 17:35:22 INFO DAGScheduler:59 - Job 20 finished: aggregate at GaussianMixture.scala:201, took 0.040634 s -2016-06-13 17:35:22 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Got job 21 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:35:22 INFO DAGScheduler:59 - Final stage: ResultStage 21(first at GaussianMixture.scala:171) -2016-06-13 17:35:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=74160, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=76152, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:62031 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 21 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 1 tasks -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 72, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:22 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 72) -2016-06-13 17:35:22 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=77404, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 16.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:62031 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 72). 1533 bytes result sent to driver -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 72) in 11 ms on localhost (1/1) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 17:35:22 INFO DAGScheduler:59 - ResultStage 21 (first at GaussianMixture.scala:171) finished in 0.012 s -2016-06-13 17:35:22 INFO DAGScheduler:59 - Job 21 finished: first at GaussianMixture.scala:171, took 0.027336 s -2016-06-13 17:35:22 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Got job 22 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:35:22 INFO DAGScheduler:59 - Final stage: ResultStage 22(first at GaussianMixture.scala:171) -2016-06-13 17:35:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=77420, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=79412, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:62031 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 22 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 3 tasks -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 73, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 74, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 75, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 74) -2016-06-13 17:35:22 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 75) -2016-06-13 17:35:22 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 73) -2016-06-13 17:35:22 INFO CacheManager:59 - Partition rdd_7_2 not found, computing it -2016-06-13 17:35:22 INFO CacheManager:59 - Partition rdd_7_3 not found, computing it -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=80664, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block rdd_7_2 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=80760, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block rdd_7_3 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added rdd_7_2 in memory on localhost:62031 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added rdd_7_3 in memory on localhost:62031 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=80856, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 75). 1696 bytes result sent to driver -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:62031 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 74). 1696 bytes result sent to driver -2016-06-13 17:35:22 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 73). 1696 bytes result sent to driver -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 75) in 19 ms on localhost (1/3) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 73) in 22 ms on localhost (2/3) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 74) in 22 ms on localhost (3/3) -2016-06-13 17:35:22 INFO DAGScheduler:59 - ResultStage 22 (first at GaussianMixture.scala:171) finished in 0.023 s -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 17:35:22 INFO DAGScheduler:59 - Job 22 finished: first at GaussianMixture.scala:171, took 0.041767 s -2016-06-13 17:35:22 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Got job 23 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:22 INFO DAGScheduler:59 - Final stage: ResultStage 23(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=80952, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 1832.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=82784, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:62031 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 23 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 76, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 77, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 78, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 79, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 79) -2016-06-13 17:35:22 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 78) -2016-06-13 17:35:22 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 76) -2016-06-13 17:35:22 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 77) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 76). 2082 bytes result sent to driver -2016-06-13 17:35:22 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 77). 2082 bytes result sent to driver -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 79). 2082 bytes result sent to driver -2016-06-13 17:35:22 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 78). 2082 bytes result sent to driver -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 76) in 18 ms on localhost (1/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 77) in 18 ms on localhost (2/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 78) in 18 ms on localhost (3/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 79) in 18 ms on localhost (4/4) -2016-06-13 17:35:22 INFO DAGScheduler:59 - ResultStage 23 (takeSample at GaussianMixture.scala:184) finished in 0.023 s -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 17:35:22 INFO DAGScheduler:59 - Job 23 finished: takeSample at GaussianMixture.scala:184, took 0.039224 s -2016-06-13 17:35:22 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Got job 24 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:22 INFO DAGScheduler:59 - Final stage: ResultStage 24(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting ResultStage 24 (PartitionwiseSampledRDD[8] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=83979, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(7881) called with curMem=103203, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:62031 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 24 (PartitionwiseSampledRDD[8] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 4 tasks -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 80, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 81, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 82, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 24.0 (TID 83, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:35:22 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 81) -2016-06-13 17:35:22 INFO Executor:59 - Running task 3.0 in stage 24.0 (TID 83) -2016-06-13 17:35:22 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 80) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 81). 2269 bytes result sent to driver -2016-06-13 17:35:22 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 82) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 80). 2048 bytes result sent to driver -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 82). 2221 bytes result sent to driver -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 81) in 16 ms on localhost (1/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 80) in 18 ms on localhost (2/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 82) in 17 ms on localhost (3/4) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 3.0 in stage 24.0 (TID 83). 2269 bytes result sent to driver -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 24.0 (TID 83) in 21 ms on localhost (4/4) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-06-13 17:35:22 INFO DAGScheduler:59 - ResultStage 24 (takeSample at GaussianMixture.scala:184) finished in 0.024 s -2016-06-13 17:35:22 INFO DAGScheduler:59 - Job 24 finished: takeSample at GaussianMixture.scala:184, took 0.038221 s -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=111084, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(851) called with curMem=111548, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 851.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:62031 (size: 851.0 B, free: 730.5 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 34 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Got job 25 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:22 INFO DAGScheduler:59 - Final stage: ResultStage 25(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=112399, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=116319, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 25 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 4 tasks -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 84, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 85, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 25.0 (TID 86, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 25.0 (TID 87, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 85) -2016-06-13 17:35:22 INFO Executor:59 - Running task 3.0 in stage 25.0 (TID 87) -2016-06-13 17:35:22 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 84) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 3.0 in stage 25.0 (TID 87). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 84). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 85). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO Executor:59 - Running task 2.0 in stage 25.0 (TID 86) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 25.0 (TID 87) in 13 ms on localhost (1/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 84) in 18 ms on localhost (2/4) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 85) in 19 ms on localhost (3/4) -2016-06-13 17:35:22 INFO Executor:59 - Finished task 2.0 in stage 25.0 (TID 86). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 25.0 (TID 86) in 22 ms on localhost (4/4) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-06-13 17:35:22 INFO DAGScheduler:59 - ResultStage 25 (aggregate at GaussianMixture.scala:201) finished in 0.024 s -2016-06-13 17:35:22 INFO DAGScheduler:59 - Job 25 finished: aggregate at GaussianMixture.scala:201, took 0.038514 s -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=118464, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=118928, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:62031 (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 36 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Got job 26 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:22 INFO DAGScheduler:59 - Final stage: ResultStage 26(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=119781, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=123701, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 26 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 88, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 89, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 90, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 91, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 91) -2016-06-13 17:35:22 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 90) -2016-06-13 17:35:22 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 88) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 90). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:35:22 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 89) -2016-06-13 17:35:22 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 88). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 91). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 89). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 90) in 17 ms on localhost (1/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 88) in 22 ms on localhost (2/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 89) in 21 ms on localhost (3/4) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 91) in 19 ms on localhost (4/4) -2016-06-13 17:35:22 INFO DAGScheduler:59 - ResultStage 26 (aggregate at GaussianMixture.scala:201) finished in 0.024 s -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-06-13 17:35:22 INFO DAGScheduler:59 - Job 26 finished: aggregate at GaussianMixture.scala:201, took 0.039548 s -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=125853, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=126317, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:62031 (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 38 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Got job 27 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:22 INFO DAGScheduler:59 - Final stage: ResultStage 27(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=127170, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:35:22 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=131090, maxMem=766075207 -2016-06-13 17:35:22 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:35:22 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:22 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 27 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:35:22 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 4 tasks -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 92, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 93, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 27.0 (TID 94, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 27.0 (TID 95, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:22 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 92) -2016-06-13 17:35:22 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 93) -2016-06-13 17:35:22 INFO Executor:59 - Running task 3.0 in stage 27.0 (TID 95) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 92). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 93). 2708 bytes result sent to driver -2016-06-13 17:35:22 INFO Executor:59 - Running task 2.0 in stage 27.0 (TID 94) -2016-06-13 17:35:22 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:35:22 INFO Executor:59 - Finished task 2.0 in stage 27.0 (TID 94). 2708 bytes result sent to driver -2016-06-13 17:35:23 INFO Executor:59 - Finished task 3.0 in stage 27.0 (TID 95). 2708 bytes result sent to driver -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 92) in 35 ms on localhost (1/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 93) in 35 ms on localhost (2/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 27.0 (TID 94) in 36 ms on localhost (3/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 27.0 (TID 95) in 46 ms on localhost (4/4) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-06-13 17:35:23 INFO DAGScheduler:59 - ResultStage 27 (aggregate at GaussianMixture.scala:201) finished in 0.052 s -2016-06-13 17:35:23 INFO DAGScheduler:59 - Job 27 finished: aggregate at GaussianMixture.scala:201, took 0.078198 s -2016-06-13 17:35:23 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Got job 28 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:35:23 INFO DAGScheduler:59 - Final stage: ResultStage 28(first at GaussianMixture.scala:171) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=133242, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=135234, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.5 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:62031 (size: 1252.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 28 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 1 tasks -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 96, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:23 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 96) -2016-06-13 17:35:23 INFO CacheManager:59 - Partition rdd_9_0 not found, computing it -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=136486, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block rdd_9_0 stored as values in memory (estimated size 16.0 B, free 730.5 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added rdd_9_0 in memory on localhost:62031 (size: 16.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 96). 1533 bytes result sent to driver -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 96) in 28 ms on localhost (1/1) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-06-13 17:35:23 INFO DAGScheduler:59 - ResultStage 28 (first at GaussianMixture.scala:171) finished in 0.029 s -2016-06-13 17:35:23 INFO DAGScheduler:59 - Job 28 finished: first at GaussianMixture.scala:171, took 0.077110 s -2016-06-13 17:35:23 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Got job 29 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:35:23 INFO DAGScheduler:59 - Final stage: ResultStage 29(first at GaussianMixture.scala:171) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=136502, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=138494, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.5 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:62031 (size: 1252.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 29 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 3 tasks -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 97, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 98, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 29.0 (TID 99, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:23 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 97) -2016-06-13 17:35:23 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 98) -2016-06-13 17:35:23 INFO Executor:59 - Running task 2.0 in stage 29.0 (TID 99) -2016-06-13 17:35:23 INFO CacheManager:59 - Partition rdd_9_1 not found, computing it -2016-06-13 17:35:23 INFO CacheManager:59 - Partition rdd_9_3 not found, computing it -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=139746, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block rdd_9_1 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:35:23 INFO CacheManager:59 - Partition rdd_9_2 not found, computing it -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=139842, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block rdd_9_3 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added rdd_9_1 in memory on localhost:62031 (size: 96.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=139938, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block rdd_9_2 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added rdd_9_3 in memory on localhost:62031 (size: 96.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added rdd_9_2 in memory on localhost:62031 (size: 96.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO Executor:59 - Finished task 2.0 in stage 29.0 (TID 99). 1696 bytes result sent to driver -2016-06-13 17:35:23 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 98). 1696 bytes result sent to driver -2016-06-13 17:35:23 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 97). 1696 bytes result sent to driver -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 29.0 (TID 99) in 21 ms on localhost (1/3) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 98) in 23 ms on localhost (2/3) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 97) in 24 ms on localhost (3/3) -2016-06-13 17:35:23 INFO DAGScheduler:59 - ResultStage 29 (first at GaussianMixture.scala:171) finished in 0.031 s -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 17:35:23 INFO DAGScheduler:59 - Job 29 finished: first at GaussianMixture.scala:171, took 0.055974 s -2016-06-13 17:35:23 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Got job 30 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:23 INFO DAGScheduler:59 - Final stage: ResultStage 30(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=140034, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 1832.0 B, free 730.5 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=141866, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.4 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:62031 (size: 1195.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 100, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 101, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 102, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 103, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:23 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 103) -2016-06-13 17:35:23 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 100) -2016-06-13 17:35:23 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 102) -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:35:23 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 101) -2016-06-13 17:35:23 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 100). 2082 bytes result sent to driver -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:35:23 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 102). 2082 bytes result sent to driver -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:35:23 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 103). 2082 bytes result sent to driver -2016-06-13 17:35:23 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 101). 2082 bytes result sent to driver -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 100) in 11 ms on localhost (1/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 102) in 9 ms on localhost (2/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 103) in 10 ms on localhost (3/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 101) in 12 ms on localhost (4/4) -2016-06-13 17:35:23 INFO DAGScheduler:59 - ResultStage 30 (takeSample at GaussianMixture.scala:184) finished in 0.013 s -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 17:35:23 INFO DAGScheduler:59 - Job 30 finished: takeSample at GaussianMixture.scala:184, took 0.026166 s -2016-06-13 17:35:23 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Got job 31 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:23 INFO DAGScheduler:59 - Final stage: ResultStage 31(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting ResultStage 31 (PartitionwiseSampledRDD[10] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=143061, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 18.8 KB, free 730.4 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(7884) called with curMem=162285, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.4 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:62031 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 31 (PartitionwiseSampledRDD[10] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 4 tasks -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 104, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 105, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 31.0 (TID 106, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 31.0 (TID 107, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:35:23 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 104) -2016-06-13 17:35:23 INFO Executor:59 - Running task 3.0 in stage 31.0 (TID 107) -2016-06-13 17:35:23 INFO Executor:59 - Running task 2.0 in stage 31.0 (TID 106) -2016-06-13 17:35:23 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 105) -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:35:23 INFO Executor:59 - Finished task 2.0 in stage 31.0 (TID 106). 2249 bytes result sent to driver -2016-06-13 17:35:23 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 104). 2048 bytes result sent to driver -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:35:23 INFO Executor:59 - Finished task 3.0 in stage 31.0 (TID 107). 2274 bytes result sent to driver -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 31.0 (TID 106) in 8 ms on localhost (1/4) -2016-06-13 17:35:23 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 105). 2269 bytes result sent to driver -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 104) in 10 ms on localhost (2/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 31.0 (TID 107) in 10 ms on localhost (3/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 105) in 12 ms on localhost (4/4) -2016-06-13 17:35:23 INFO DAGScheduler:59 - ResultStage 31 (takeSample at GaussianMixture.scala:184) finished in 0.013 s -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 17:35:23 INFO DAGScheduler:59 - Job 31 finished: takeSample at GaussianMixture.scala:184, took 0.024090 s -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=170169, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 464.0 B, free 730.4 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(849) called with curMem=170633, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 849.0 B, free 730.4 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:62031 (size: 849.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 44 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Got job 32 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:23 INFO DAGScheduler:59 - Final stage: ResultStage 32(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=171482, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=175402, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 32 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 4 tasks -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 108, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 109, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 32.0 (TID 110, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 32.0 (TID 111, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:23 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 108) -2016-06-13 17:35:23 INFO Executor:59 - Running task 2.0 in stage 32.0 (TID 110) -2016-06-13 17:35:23 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 109) -2016-06-13 17:35:23 INFO Executor:59 - Running task 3.0 in stage 32.0 (TID 111) -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:35:23 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 108). 2708 bytes result sent to driver -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:35:23 INFO Executor:59 - Finished task 3.0 in stage 32.0 (TID 111). 2708 bytes result sent to driver -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:35:23 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 109). 2708 bytes result sent to driver -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 108) in 8 ms on localhost (1/4) -2016-06-13 17:35:23 INFO Executor:59 - Finished task 2.0 in stage 32.0 (TID 110). 2708 bytes result sent to driver -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 32.0 (TID 111) in 6 ms on localhost (2/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 109) in 9 ms on localhost (3/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 32.0 (TID 110) in 8 ms on localhost (4/4) -2016-06-13 17:35:23 INFO DAGScheduler:59 - ResultStage 32 (aggregate at GaussianMixture.scala:201) finished in 0.010 s -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 17:35:23 INFO DAGScheduler:59 - Job 32 finished: aggregate at GaussianMixture.scala:201, took 0.022913 s -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=177547, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 464.0 B, free 730.4 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=178011, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.4 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:62031 (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 46 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Got job 33 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:23 INFO DAGScheduler:59 - Final stage: ResultStage 33(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=178864, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=182784, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 112, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 113, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 114, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 115, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:23 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 113) -2016-06-13 17:35:23 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 115) -2016-06-13 17:35:23 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 114) -2016-06-13 17:35:23 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 112) -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:35:23 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 112). 2708 bytes result sent to driver -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:35:23 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 113). 2708 bytes result sent to driver -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:35:23 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 114). 2708 bytes result sent to driver -2016-06-13 17:35:23 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 115). 2708 bytes result sent to driver -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 112) in 7 ms on localhost (1/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 114) in 7 ms on localhost (2/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 113) in 7 ms on localhost (3/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 115) in 6 ms on localhost (4/4) -2016-06-13 17:35:23 INFO DAGScheduler:59 - ResultStage 33 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 17:35:23 INFO DAGScheduler:59 - Job 33 finished: aggregate at GaussianMixture.scala:201, took 0.020214 s -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=184936, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 464.0 B, free 730.4 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=185400, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.4 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:62031 (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 48 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Got job 34 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:23 INFO DAGScheduler:59 - Final stage: ResultStage 34(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=186253, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=190173, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 116, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 117, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 118, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 119, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:35:23 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 118) -2016-06-13 17:35:23 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 119) -2016-06-13 17:35:23 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 116) -2016-06-13 17:35:23 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 117) -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:35:23 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 116). 2708 bytes result sent to driver -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:35:23 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 118). 2708 bytes result sent to driver -2016-06-13 17:35:23 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 119). 2708 bytes result sent to driver -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:35:23 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 117). 2708 bytes result sent to driver -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 116) in 11 ms on localhost (1/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 119) in 10 ms on localhost (2/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 118) in 10 ms on localhost (3/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 117) in 12 ms on localhost (4/4) -2016-06-13 17:35:23 INFO DAGScheduler:59 - ResultStage 34 (aggregate at GaussianMixture.scala:201) finished in 0.013 s -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-06-13 17:35:23 INFO DAGScheduler:59 - Job 34 finished: aggregate at GaussianMixture.scala:201, took 0.024717 s -2016-06-13 17:35:23 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Got job 35 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:35:23 INFO DAGScheduler:59 - Final stage: ResultStage 35(first at GaussianMixture.scala:171) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=192325, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 1992.0 B, free 730.4 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=194317, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.4 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:62031 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 35 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 1 tasks -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 120, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:35:23 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 120) -2016-06-13 17:35:23 INFO CacheManager:59 - Partition rdd_12_0 not found, computing it -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(224) called with curMem=195567, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block rdd_12_0 stored as values in memory (estimated size 224.0 B, free 730.4 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added rdd_12_0 in memory on localhost:62031 (size: 224.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 120). 1688 bytes result sent to driver -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 120) in 25 ms on localhost (1/1) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-06-13 17:35:23 INFO DAGScheduler:59 - ResultStage 35 (first at GaussianMixture.scala:171) finished in 0.026 s -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:62031 in memory (size: 850.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Job 35 finished: first at GaussianMixture.scala:171, took 0.039574 s -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=194477, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 712.0 B, free 730.4 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=187306, maxMem=766075207 -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:62031 in memory (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 880.0 B, free 730.4 MB) -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:62031 (size: 880.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 51 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:62031 in memory (size: 1194.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 17:35:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Got job 36 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:62031 in memory (size: 1251.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Final stage: ResultStage 36(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=159450, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:62031 in memory (size: 1251.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(2181) called with curMem=163546, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:35:23 INFO BlockManager:59 - Removing RDD 5 -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned RDD 5 -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 36 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 4 tasks -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 121, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 122, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 36.0 (TID 123, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 36.0 (TID 124, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:23 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 121) -2016-06-13 17:35:23 INFO Executor:59 - Running task 3.0 in stage 36.0 (TID 124) -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_12_0 locally -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:62031 in memory (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 121). 2762 bytes result sent to driver -2016-06-13 17:35:23 INFO CacheManager:59 - Partition rdd_12_3 not found, computing it -2016-06-13 17:35:23 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 122) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 121) in 13 ms on localhost (1/4) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=158498, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block rdd_12_3 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-06-13 17:35:23 INFO Executor:59 - Running task 2.0 in stage 36.0 (TID 123) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added rdd_12_3 in memory on localhost:62031 (size: 288.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO Executor:59 - Finished task 3.0 in stage 36.0 (TID 124). 2247 bytes result sent to driver -2016-06-13 17:35:23 INFO CacheManager:59 - Partition rdd_12_1 not found, computing it -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 36.0 (TID 124) in 19 ms on localhost (2/4) -2016-06-13 17:35:23 INFO CacheManager:59 - Partition rdd_12_2 not found, computing it -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=158322, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block rdd_12_1 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added rdd_12_1 in memory on localhost:62031 (size: 288.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=156458, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block rdd_12_2 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:23 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 122). 2247 bytes result sent to driver -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added rdd_12_2 in memory on localhost:62031 (size: 288.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 122) in 27 ms on localhost (3/4) -2016-06-13 17:35:23 INFO Executor:59 - Finished task 2.0 in stage 36.0 (TID 123). 2247 bytes result sent to driver -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 36.0 (TID 123) in 30 ms on localhost (4/4) -2016-06-13 17:35:23 INFO DAGScheduler:59 - ResultStage 36 (aggregate at GaussianMixture.scala:201) finished in 0.033 s -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-06-13 17:35:23 INFO DAGScheduler:59 - Job 36 finished: aggregate at GaussianMixture.scala:201, took 0.053156 s -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:62031 in memory (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=151509, maxMem=766075207 -2016-06-13 17:35:23 INFO BlockManager:59 - Removing RDD 3 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 712.0 B, free 730.4 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(934) called with curMem=152013, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 934.0 B, free 730.4 MB) -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned RDD 3 -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:62031 (size: 934.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 53 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Got job 37 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:23 INFO DAGScheduler:59 - Final stage: ResultStage 37(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:62031 in memory (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=145462, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(2241) called with curMem=143486, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:62031 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 37 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 4 tasks -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 125, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:62031 in memory (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 126, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 127, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 37.0 (TID 128, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:23 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 127) -2016-06-13 17:35:23 INFO Executor:59 - Running task 3.0 in stage 37.0 (TID 128) -2016-06-13 17:35:23 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 126) -2016-06-13 17:35:23 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 125) -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_12_2 locally -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_12_3 locally -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:23 INFO Executor:59 - Finished task 3.0 in stage 37.0 (TID 128). 2762 bytes result sent to driver -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_12_0 locally -2016-06-13 17:35:23 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 127). 2762 bytes result sent to driver -2016-06-13 17:35:23 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 125). 2762 bytes result sent to driver -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_12_1 locally -2016-06-13 17:35:23 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 126). 2762 bytes result sent to driver -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 37.0 (TID 128) in 11 ms on localhost (1/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 127) in 12 ms on localhost (2/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 125) in 15 ms on localhost (3/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 126) in 15 ms on localhost (4/4) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:62031 in memory (size: 849.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO DAGScheduler:59 - ResultStage 37 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 17:35:23 INFO DAGScheduler:59 - Job 37 finished: aggregate at GaussianMixture.scala:201, took 0.029521 s -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=137496, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 712.0 B, free 730.5 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=137744, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.5 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:62031 (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 55 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:62031 in memory (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:62031 in memory (size: 1195.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-06-13 17:35:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Got job 38 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:23 INFO DAGScheduler:59 - Final stage: ResultStage 38(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:62031 in memory (size: 1252.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=105297, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:62031 in memory (size: 1252.0 B, free: 730.5 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=108141, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:62031 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:23 INFO BlockManager:59 - Removing RDD 9 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned RDD 9 -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 129, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 130, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 131, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 132, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:23 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 129) -2016-06-13 17:35:23 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 131) -2016-06-13 17:35:23 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 132) -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_12_2 locally -2016-06-13 17:35:23 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 130) -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_12_3 locally -2016-06-13 17:35:23 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 131). 2762 bytes result sent to driver -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 131) in 11 ms on localhost (1/4) -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_12_1 locally -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_12_0 locally -2016-06-13 17:35:23 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 132). 2762 bytes result sent to driver -2016-06-13 17:35:23 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 130). 2762 bytes result sent to driver -2016-06-13 17:35:23 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 129). 2762 bytes result sent to driver -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 132) in 15 ms on localhost (2/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 129) in 19 ms on localhost (3/4) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 130) in 19 ms on localhost (4/4) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-06-13 17:35:23 INFO DAGScheduler:59 - ResultStage 38 (aggregate at GaussianMixture.scala:201) finished in 0.022 s -2016-06-13 17:35:23 INFO DAGScheduler:59 - Job 38 finished: aggregate at GaussianMixture.scala:201, took 0.042047 s -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:62031 in memory (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Starting job: first at GaussianMixtureFromSparkSuite.scala:83 -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Got job 39 (first at GaussianMixtureFromSparkSuite.scala:83) with 1 output partitions -2016-06-13 17:35:23 INFO DAGScheduler:59 - Final stage: ResultStage 39(first at GaussianMixtureFromSparkSuite.scala:83) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting ResultStage 39 (ParallelCollectionRDD[13] at parallelize at GaussianMixtureFromSparkSuite.scala:80), which has no missing parents -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(1272) called with curMem=94622, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 1272.0 B, free 730.5 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(837) called with curMem=95041, maxMem=766075207 -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:62031 in memory (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 837.0 B, free 730.5 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:62031 (size: 837.0 B, free: 730.6 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 39 (ParallelCollectionRDD[13] at parallelize at GaussianMixtureFromSparkSuite.scala:80) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 1 tasks -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 133, localhost, PROCESS_LOCAL, 7178 bytes) -2016-06-13 17:35:23 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 133) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-06-13 17:35:23 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 133). 1433 bytes result sent to driver -2016-06-13 17:35:23 INFO DAGScheduler:59 - ResultStage 39 (first at GaussianMixtureFromSparkSuite.scala:83) finished in 0.005 s -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 133) in 5 ms on localhost (1/1) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-06-13 17:35:23 INFO DAGScheduler:59 - Job 39 finished: first at GaussianMixtureFromSparkSuite.scala:83, took 0.018717 s -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:62031 in memory (size: 851.0 B, free: 730.6 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:62031 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-06-13 17:35:23 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:62031 in memory (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Got job 40 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:35:23 INFO DAGScheduler:59 - Final stage: ResultStage 40(first at GaussianMixture.scala:171) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=54658, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:62031 in memory (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=56650, maxMem=766075207 -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.5 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:62031 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 40 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:62031 in memory (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 1 tasks -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 134, localhost, PROCESS_LOCAL, 7178 bytes) -2016-06-13 17:35:23 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 134) -2016-06-13 17:35:23 INFO BlockManager:59 - Removing RDD 7 -2016-06-13 17:35:23 INFO CacheManager:59 - Partition rdd_14_0 not found, computing it -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned RDD 7 -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=54352, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block rdd_14_0 stored as values in memory (estimated size 5.4 KB, free 730.5 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added rdd_14_0 in memory on localhost:62031 (size: 5.4 KB, free: 730.6 MB) -2016-06-13 17:35:23 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 134). 2083 bytes result sent to driver -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:62031 in memory (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:62031 in memory (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 134) in 57 ms on localhost (1/1) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-06-13 17:35:23 INFO DAGScheduler:59 - ResultStage 40 (first at GaussianMixture.scala:171) finished in 0.059 s -2016-06-13 17:35:23 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Job 40 finished: first at GaussianMixture.scala:171, took 0.075140 s -2016-06-13 17:35:23 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Got job 41 (takeSample at GaussianMixture.scala:184) with 2 output partitions -2016-06-13 17:35:23 INFO DAGScheduler:59 - Final stage: ResultStage 41(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=39047, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 1832.0 B, free 730.5 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=40879, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 1193.0 B, free 730.5 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:62031 (size: 1193.0 B, free: 730.6 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 135, localhost, PROCESS_LOCAL, 7178 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 136, localhost, PROCESS_LOCAL, 7594 bytes) -2016-06-13 17:35:23 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 135) -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-06-13 17:35:23 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 135). 2082 bytes result sent to driver -2016-06-13 17:35:23 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 136) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 135) in 7 ms on localhost (1/2) -2016-06-13 17:35:23 INFO CacheManager:59 - Partition rdd_14_1 not found, computing it -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(6000) called with curMem=42072, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block rdd_14_1 stored as values in memory (estimated size 5.9 KB, free 730.5 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added rdd_14_1 in memory on localhost:62031 (size: 5.9 KB, free: 730.6 MB) -2016-06-13 17:35:23 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 136). 1567 bytes result sent to driver -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 136) in 15 ms on localhost (2/2) -2016-06-13 17:35:23 INFO DAGScheduler:59 - ResultStage 41 (takeSample at GaussianMixture.scala:184) finished in 0.016 s -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-06-13 17:35:23 INFO DAGScheduler:59 - Job 41 finished: takeSample at GaussianMixture.scala:184, took 0.028384 s -2016-06-13 17:35:23 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Got job 42 (takeSample at GaussianMixture.scala:184) with 2 output partitions -2016-06-13 17:35:23 INFO DAGScheduler:59 - Final stage: ResultStage 42(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting ResultStage 42 (PartitionwiseSampledRDD[15] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=48072, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(7876) called with curMem=67296, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:62031 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (PartitionwiseSampledRDD[15] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 137, localhost, PROCESS_LOCAL, 7287 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 138, localhost, PROCESS_LOCAL, 7703 bytes) -2016-06-13 17:35:23 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 137) -2016-06-13 17:35:23 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 138) -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_14_1 locally -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-06-13 17:35:23 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 138). 6585 bytes result sent to driver -2016-06-13 17:35:23 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 137). 6993 bytes result sent to driver -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 138) in 5 ms on localhost (1/2) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 137) in 6 ms on localhost (2/2) -2016-06-13 17:35:23 INFO DAGScheduler:59 - ResultStage 42 (takeSample at GaussianMixture.scala:184) finished in 0.006 s -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-06-13 17:35:23 INFO DAGScheduler:59 - Job 42 finished: takeSample at GaussianMixture.scala:184, took 0.016175 s -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(205504) called with curMem=75172, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 200.7 KB, free 730.3 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(12132) called with curMem=280676, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 11.8 KB, free 730.3 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:62031 (size: 11.8 KB, free: 730.5 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 61 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Got job 43 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:35:23 INFO DAGScheduler:59 - Final stage: ResultStage 43(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(310656) called with curMem=292808, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 303.4 KB, free 730.0 MB) -2016-06-13 17:35:23 INFO MemoryStore:59 - ensureFreeSpace(18345) called with curMem=603464, maxMem=766075207 -2016-06-13 17:35:23 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 17.9 KB, free 730.0 MB) -2016-06-13 17:35:23 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:62031 (size: 17.9 KB, free: 730.5 MB) -2016-06-13 17:35:23 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 139, localhost, PROCESS_LOCAL, 7178 bytes) -2016-06-13 17:35:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 140, localhost, PROCESS_LOCAL, 7594 bytes) -2016-06-13 17:35:23 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 139) -2016-06-13 17:35:23 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 140) -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-06-13 17:35:23 INFO BlockManager:59 - Found block rdd_14_1 locally -2016-06-13 17:35:23 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 139). 105440 bytes result sent to driver -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 139) in 80 ms on localhost (1/2) -2016-06-13 17:35:23 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 140). 105440 bytes result sent to driver -2016-06-13 17:35:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 140) in 85 ms on localhost (2/2) -2016-06-13 17:35:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-06-13 17:35:23 INFO DAGScheduler:59 - ResultStage 43 (aggregate at GaussianMixture.scala:201) finished in 0.086 s -2016-06-13 17:35:23 INFO DAGScheduler:59 - Job 43 finished: aggregate at GaussianMixture.scala:201, took 0.106778 s -2016-06-13 17:35:24 INFO SparkContext:59 - Starting job: collect at GaussianMixture.scala:213 -2016-06-13 17:35:24 INFO DAGScheduler:59 - Got job 44 (collect at GaussianMixture.scala:213) with 5 output partitions -2016-06-13 17:35:24 INFO DAGScheduler:59 - Final stage: ResultStage 44(collect at GaussianMixture.scala:213) -2016-06-13 17:35:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:24 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[17] at map at GaussianMixture.scala:211), which has no missing parents -2016-06-13 17:35:24 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=621809, maxMem=766075207 -2016-06-13 17:35:24 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 2.3 KB, free 730.0 MB) -2016-06-13 17:35:24 INFO MemoryStore:59 - ensureFreeSpace(1389) called with curMem=624121, maxMem=766075207 -2016-06-13 17:35:24 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 1389.0 B, free 730.0 MB) -2016-06-13 17:35:24 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:62031 (size: 1389.0 B, free: 730.5 MB) -2016-06-13 17:35:24 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:24 INFO DAGScheduler:59 - Submitting 5 missing tasks from ResultStage 44 (MapPartitionsRDD[17] at map at GaussianMixture.scala:211) -2016-06-13 17:35:24 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 5 tasks -2016-06-13 17:35:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 141, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:35:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 142, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:35:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 44.0 (TID 143, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:35:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 44.0 (TID 144, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:35:24 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 141) -2016-06-13 17:35:24 INFO Executor:59 - Running task 2.0 in stage 44.0 (TID 143) -2016-06-13 17:35:24 INFO Executor:59 - Running task 3.0 in stage 44.0 (TID 144) -2016-06-13 17:35:24 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 142) -2016-06-13 17:35:24 INFO Executor:59 - Finished task 3.0 in stage 44.0 (TID 144). 42916 bytes result sent to driver -2016-06-13 17:35:24 INFO TaskSetManager:59 - Starting task 4.0 in stage 44.0 (TID 145, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:35:24 INFO Executor:59 - Running task 4.0 in stage 44.0 (TID 145) -2016-06-13 17:35:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 44.0 (TID 144) in 235 ms on localhost (1/5) -2016-06-13 17:35:24 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 142). 42916 bytes result sent to driver -2016-06-13 17:35:24 INFO Executor:59 - Finished task 2.0 in stage 44.0 (TID 143). 42916 bytes result sent to driver -2016-06-13 17:35:24 INFO Executor:59 - Finished task 4.0 in stage 44.0 (TID 145). 42916 bytes result sent to driver -2016-06-13 17:35:24 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 141). 42916 bytes result sent to driver -2016-06-13 17:35:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 142) in 331 ms on localhost (2/5) -2016-06-13 17:35:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 141) in 335 ms on localhost (3/5) -2016-06-13 17:35:24 INFO TaskSetManager:59 - Finished task 4.0 in stage 44.0 (TID 145) in 106 ms on localhost (4/5) -2016-06-13 17:35:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 44.0 (TID 143) in 334 ms on localhost (5/5) -2016-06-13 17:35:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-06-13 17:35:24 INFO DAGScheduler:59 - ResultStage 44 (collect at GaussianMixture.scala:213) finished in 0.338 s -2016-06-13 17:35:24 INFO DAGScheduler:59 - Job 44 finished: collect at GaussianMixture.scala:213, took 0.346872 s -2016-06-13 17:35:24 INFO MemoryStore:59 - ensureFreeSpace(205544) called with curMem=625510, maxMem=766075207 -2016-06-13 17:35:24 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 200.7 KB, free 729.8 MB) -2016-06-13 17:35:24 INFO MemoryStore:59 - ensureFreeSpace(23825) called with curMem=831054, maxMem=766075207 -2016-06-13 17:35:24 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 23.3 KB, free 729.8 MB) -2016-06-13 17:35:24 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:62031 (size: 23.3 KB, free: 730.5 MB) -2016-06-13 17:35:24 INFO SparkContext:59 - Created broadcast 64 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:24 INFO DAGScheduler:59 - Got job 45 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:35:24 INFO DAGScheduler:59 - Final stage: ResultStage 45(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:24 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:24 INFO MemoryStore:59 - ensureFreeSpace(310656) called with curMem=854879, maxMem=766075207 -2016-06-13 17:35:24 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 303.4 KB, free 729.5 MB) -2016-06-13 17:35:24 INFO MemoryStore:59 - ensureFreeSpace(30131) called with curMem=1165535, maxMem=766075207 -2016-06-13 17:35:24 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 29.4 KB, free 729.4 MB) -2016-06-13 17:35:24 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:62031 (size: 29.4 KB, free: 730.5 MB) -2016-06-13 17:35:24 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168) -2016-06-13 17:35:24 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-06-13 17:35:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 146, localhost, PROCESS_LOCAL, 7178 bytes) -2016-06-13 17:35:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 147, localhost, PROCESS_LOCAL, 7594 bytes) -2016-06-13 17:35:24 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 147) -2016-06-13 17:35:24 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 146) -2016-06-13 17:35:24 INFO BlockManager:59 - Found block rdd_14_1 locally -2016-06-13 17:35:24 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-06-13 17:35:24 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 147). 105440 bytes result sent to driver -2016-06-13 17:35:24 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 146). 105440 bytes result sent to driver -2016-06-13 17:35:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 147) in 25 ms on localhost (1/2) -2016-06-13 17:35:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 146) in 28 ms on localhost (2/2) -2016-06-13 17:35:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-06-13 17:35:24 INFO DAGScheduler:59 - ResultStage 45 (aggregate at GaussianMixture.scala:201) finished in 0.029 s -2016-06-13 17:35:24 INFO DAGScheduler:59 - Job 45 finished: aggregate at GaussianMixture.scala:201, took 0.045415 s -2016-06-13 17:35:24 INFO SparkContext:59 - Starting job: collect at GaussianMixture.scala:213 -2016-06-13 17:35:24 INFO DAGScheduler:59 - Got job 46 (collect at GaussianMixture.scala:213) with 5 output partitions -2016-06-13 17:35:24 INFO DAGScheduler:59 - Final stage: ResultStage 46(collect at GaussianMixture.scala:213) -2016-06-13 17:35:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:24 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[19] at map at GaussianMixture.scala:211), which has no missing parents -2016-06-13 17:35:24 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=1195666, maxMem=766075207 -2016-06-13 17:35:24 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 2.3 KB, free 729.4 MB) -2016-06-13 17:35:24 INFO MemoryStore:59 - ensureFreeSpace(1389) called with curMem=1197978, maxMem=766075207 -2016-06-13 17:35:24 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 1389.0 B, free 729.4 MB) -2016-06-13 17:35:24 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:62031 (size: 1389.0 B, free: 730.5 MB) -2016-06-13 17:35:24 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:24 INFO DAGScheduler:59 - Submitting 5 missing tasks from ResultStage 46 (MapPartitionsRDD[19] at map at GaussianMixture.scala:211) -2016-06-13 17:35:24 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 5 tasks -2016-06-13 17:35:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 148, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:35:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 149, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:35:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 46.0 (TID 150, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:35:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 46.0 (TID 151, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:35:24 INFO Executor:59 - Running task 2.0 in stage 46.0 (TID 150) -2016-06-13 17:35:24 INFO Executor:59 - Running task 3.0 in stage 46.0 (TID 151) -2016-06-13 17:35:24 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 148) -2016-06-13 17:35:24 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 149) -2016-06-13 17:35:24 INFO Executor:59 - Finished task 2.0 in stage 46.0 (TID 150). 42916 bytes result sent to driver -2016-06-13 17:35:24 INFO Executor:59 - Finished task 3.0 in stage 46.0 (TID 151). 42916 bytes result sent to driver -2016-06-13 17:35:24 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 148). 42916 bytes result sent to driver -2016-06-13 17:35:24 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 149). 42916 bytes result sent to driver -2016-06-13 17:35:24 INFO TaskSetManager:59 - Starting task 4.0 in stage 46.0 (TID 152, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:35:24 INFO Executor:59 - Running task 4.0 in stage 46.0 (TID 152) -2016-06-13 17:35:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 46.0 (TID 150) in 29 ms on localhost (1/5) -2016-06-13 17:35:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 46.0 (TID 151) in 29 ms on localhost (2/5) -2016-06-13 17:35:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 148) in 33 ms on localhost (3/5) -2016-06-13 17:35:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 149) in 33 ms on localhost (4/5) -2016-06-13 17:35:24 INFO Executor:59 - Finished task 4.0 in stage 46.0 (TID 152). 42916 bytes result sent to driver -2016-06-13 17:35:24 INFO TaskSetManager:59 - Finished task 4.0 in stage 46.0 (TID 152) in 13 ms on localhost (5/5) -2016-06-13 17:35:24 INFO DAGScheduler:59 - ResultStage 46 (collect at GaussianMixture.scala:213) finished in 0.041 s -2016-06-13 17:35:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-06-13 17:35:24 INFO DAGScheduler:59 - Job 46 finished: collect at GaussianMixture.scala:213, took 0.051458 s -2016-06-13 17:35:24 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:24 INFO DAGScheduler:59 - Got job 47 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:35:24 INFO DAGScheduler:59 - Final stage: ResultStage 47(first at GaussianMixture.scala:171) -2016-06-13 17:35:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:24 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:24 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=1199367, maxMem=766075207 -2016-06-13 17:35:24 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 1992.0 B, free 729.4 MB) -2016-06-13 17:35:24 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1201359, maxMem=766075207 -2016-06-13 17:35:24 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 1250.0 B, free 729.4 MB) -2016-06-13 17:35:24 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:62031 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:35:24 INFO SparkContext:59 - Created broadcast 67 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 47 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168) -2016-06-13 17:35:24 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 1 tasks -2016-06-13 17:35:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 153, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:24 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 153) -2016-06-13 17:35:24 INFO CacheManager:59 - Partition rdd_21_0 not found, computing it -2016-06-13 17:35:24 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=1202609, maxMem=766075207 -2016-06-13 17:35:24 INFO MemoryStore:59 - Block rdd_21_0 stored as values in memory (estimated size 16.0 B, free 729.4 MB) -2016-06-13 17:35:24 INFO BlockManagerInfo:59 - Added rdd_21_0 in memory on localhost:62031 (size: 16.0 B, free: 730.5 MB) -2016-06-13 17:35:24 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 153). 1533 bytes result sent to driver -2016-06-13 17:35:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 153) in 16 ms on localhost (1/1) -2016-06-13 17:35:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-06-13 17:35:24 INFO DAGScheduler:59 - ResultStage 47 (first at GaussianMixture.scala:171) finished in 0.017 s -2016-06-13 17:35:24 INFO DAGScheduler:59 - Job 47 finished: first at GaussianMixture.scala:171, took 0.029427 s -2016-06-13 17:35:24 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:24 INFO DAGScheduler:59 - Got job 48 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:35:24 INFO DAGScheduler:59 - Final stage: ResultStage 48(first at GaussianMixture.scala:171) -2016-06-13 17:35:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:24 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:24 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=1202625, maxMem=766075207 -2016-06-13 17:35:24 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 1992.0 B, free 729.4 MB) -2016-06-13 17:35:24 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1204617, maxMem=766075207 -2016-06-13 17:35:24 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 1250.0 B, free 729.4 MB) -2016-06-13 17:35:24 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:62031 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:35:24 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:24 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 48 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168) -2016-06-13 17:35:24 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 3 tasks -2016-06-13 17:35:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 154, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 155, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 48.0 (TID 156, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:24 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 154) -2016-06-13 17:35:24 INFO Executor:59 - Running task 2.0 in stage 48.0 (TID 156) -2016-06-13 17:35:24 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 155) -2016-06-13 17:35:24 INFO CacheManager:59 - Partition rdd_21_1 not found, computing it -2016-06-13 17:35:24 INFO CacheManager:59 - Partition rdd_21_2 not found, computing it -2016-06-13 17:35:24 INFO CacheManager:59 - Partition rdd_21_3 not found, computing it -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=1205867, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block rdd_21_3 stored as values in memory (estimated size 184.0 B, free 729.4 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=1206051, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block rdd_21_2 stored as values in memory (estimated size 192.0 B, free 729.4 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added rdd_21_3 in memory on localhost:62031 (size: 184.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=1206243, maxMem=766075207 -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added rdd_21_2 in memory on localhost:62031 (size: 192.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - Block rdd_21_1 stored as values in memory (estimated size 192.0 B, free 729.4 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added rdd_21_1 in memory on localhost:62031 (size: 192.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 155). 2144 bytes result sent to driver -2016-06-13 17:35:25 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 154). 2144 bytes result sent to driver -2016-06-13 17:35:25 INFO Executor:59 - Finished task 2.0 in stage 48.0 (TID 156). 2132 bytes result sent to driver -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 155) in 742 ms on localhost (1/3) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 154) in 744 ms on localhost (2/3) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 48.0 (TID 156) in 742 ms on localhost (3/3) -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-06-13 17:35:25 INFO DAGScheduler:59 - ResultStage 48 (first at GaussianMixture.scala:171) finished in 0.745 s -2016-06-13 17:35:25 INFO DAGScheduler:59 - Job 48 finished: first at GaussianMixture.scala:171, took 0.754674 s -2016-06-13 17:35:25 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Got job 49 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:25 INFO DAGScheduler:59 - Final stage: ResultStage 49(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=1206435, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 1832.0 B, free 729.4 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=1208267, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 1193.0 B, free 729.4 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:62031 (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO SparkContext:59 - Created broadcast 69 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 49 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168) -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 4 tasks -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 157, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 158, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 49.0 (TID 159, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 49.0 (TID 160, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:25 INFO Executor:59 - Running task 3.0 in stage 49.0 (TID 160) -2016-06-13 17:35:25 INFO Executor:59 - Running task 2.0 in stage 49.0 (TID 159) -2016-06-13 17:35:25 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 158) -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 158). 2082 bytes result sent to driver -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_21_3 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 3.0 in stage 49.0 (TID 160). 2082 bytes result sent to driver -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_21_2 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 2.0 in stage 49.0 (TID 159). 2082 bytes result sent to driver -2016-06-13 17:35:25 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 157) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 158) in 9 ms on localhost (1/4) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 49.0 (TID 160) in 8 ms on localhost (2/4) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 49.0 (TID 159) in 9 ms on localhost (3/4) -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 157). 2082 bytes result sent to driver -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 157) in 13 ms on localhost (4/4) -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-06-13 17:35:25 INFO DAGScheduler:59 - ResultStage 49 (takeSample at GaussianMixture.scala:184) finished in 0.013 s -2016-06-13 17:35:25 INFO DAGScheduler:59 - Job 49 finished: takeSample at GaussianMixture.scala:184, took 0.022531 s -2016-06-13 17:35:25 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Got job 50 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:25 INFO DAGScheduler:59 - Final stage: ResultStage 50(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting ResultStage 50 (PartitionwiseSampledRDD[22] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=1209460, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 18.8 KB, free 729.4 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(7881) called with curMem=1228684, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 7.7 KB, free 729.4 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:62031 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:35:25 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 50 (PartitionwiseSampledRDD[22] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 4 tasks -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 161, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 162, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 50.0 (TID 163, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 50.0 (TID 164, localhost, PROCESS_LOCAL, 2373 bytes) -2016-06-13 17:35:25 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 161) -2016-06-13 17:35:25 INFO Executor:59 - Running task 2.0 in stage 50.0 (TID 163) -2016-06-13 17:35:25 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 162) -2016-06-13 17:35:25 INFO Executor:59 - Running task 3.0 in stage 50.0 (TID 164) -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 161). 2048 bytes result sent to driver -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_21_3 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 3.0 in stage 50.0 (TID 164). 2692 bytes result sent to driver -2016-06-13 17:35:25 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 162). 2724 bytes result sent to driver -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_21_2 locally -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 161) in 9 ms on localhost (1/4) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 50.0 (TID 164) in 9 ms on localhost (2/4) -2016-06-13 17:35:25 INFO Executor:59 - Finished task 2.0 in stage 50.0 (TID 163). 2704 bytes result sent to driver -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 162) in 10 ms on localhost (3/4) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 50.0 (TID 163) in 10 ms on localhost (4/4) -2016-06-13 17:35:25 INFO DAGScheduler:59 - ResultStage 50 (takeSample at GaussianMixture.scala:184) finished in 0.012 s -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-06-13 17:35:25 INFO DAGScheduler:59 - Job 50 finished: takeSample at GaussianMixture.scala:184, took 0.023185 s -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=1236565, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 560.0 B, free 729.4 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(890) called with curMem=1237125, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 890.0 B, free 729.4 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:62031 (size: 890.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO SparkContext:59 - Created broadcast 71 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:25 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Got job 51 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:25 INFO DAGScheduler:59 - Final stage: ResultStage 51(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=1238015, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(2188) called with curMem=1242079, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.4 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:25 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 51 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168) -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 4 tasks -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 165, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 166, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 51.0 (TID 167, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 51.0 (TID 168, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:25 INFO Executor:59 - Running task 2.0 in stage 51.0 (TID 167) -2016-06-13 17:35:25 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 166) -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_21_2 locally -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 17:35:25 INFO Executor:59 - Running task 3.0 in stage 51.0 (TID 168) -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_21_3 locally -2016-06-13 17:35:25 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 165) -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 165). 2756 bytes result sent to driver -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 165) in 36 ms on localhost (1/4) -2016-06-13 17:35:25 INFO Executor:59 - Finished task 2.0 in stage 51.0 (TID 167). 2756 bytes result sent to driver -2016-06-13 17:35:25 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 166). 2756 bytes result sent to driver -2016-06-13 17:35:25 INFO Executor:59 - Finished task 3.0 in stage 51.0 (TID 168). 2756 bytes result sent to driver -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 51.0 (TID 167) in 104 ms on localhost (2/4) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 166) in 106 ms on localhost (3/4) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 51.0 (TID 168) in 106 ms on localhost (4/4) -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-06-13 17:35:25 INFO DAGScheduler:59 - ResultStage 51 (aggregate at GaussianMixture.scala:201) finished in 0.108 s -2016-06-13 17:35:25 INFO DAGScheduler:59 - Job 51 finished: aggregate at GaussianMixture.scala:201, took 0.118741 s -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=1244267, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 560.0 B, free 729.4 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=1244827, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 900.0 B, free 729.4 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:62031 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO SparkContext:59 - Created broadcast 73 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:25 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Got job 52 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:25 INFO DAGScheduler:59 - Final stage: ResultStage 52(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting ResultStage 52 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=1245727, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=1249791, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.4 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:25 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 52 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168) -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 4 tasks -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 169, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 170, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 52.0 (TID 171, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 52.0 (TID 172, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:25 INFO Executor:59 - Running task 2.0 in stage 52.0 (TID 171) -2016-06-13 17:35:25 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 170) -2016-06-13 17:35:25 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 169) -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_21_2 locally -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 17:35:25 INFO Executor:59 - Running task 3.0 in stage 52.0 (TID 172) -2016-06-13 17:35:25 INFO Executor:59 - Finished task 2.0 in stage 52.0 (TID 171). 2756 bytes result sent to driver -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_21_3 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 169). 2756 bytes result sent to driver -2016-06-13 17:35:25 INFO Executor:59 - Finished task 3.0 in stage 52.0 (TID 172). 2756 bytes result sent to driver -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 52.0 (TID 171) in 11 ms on localhost (1/4) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 52.0 (TID 172) in 11 ms on localhost (2/4) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 169) in 15 ms on localhost (3/4) -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 170). 2756 bytes result sent to driver -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 170) in 18 ms on localhost (4/4) -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-06-13 17:35:25 INFO DAGScheduler:59 - ResultStage 52 (aggregate at GaussianMixture.scala:201) finished in 0.020 s -2016-06-13 17:35:25 INFO DAGScheduler:59 - Job 52 finished: aggregate at GaussianMixture.scala:201, took 0.034441 s -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=1251985, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 560.0 B, free 729.4 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=1252545, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 900.0 B, free 729.4 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:62031 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO SparkContext:59 - Created broadcast 75 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:25 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Got job 53 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:25 INFO DAGScheduler:59 - Final stage: ResultStage 53(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=1253445, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=1257509, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.4 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.4 MB) -2016-06-13 17:35:25 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 53 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168) -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 4 tasks -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 173, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:62031 in memory (size: 23.3 KB, free: 730.5 MB) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 174, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 53.0 (TID 175, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 53.0 (TID 176, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:25 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 174) -2016-06-13 17:35:25 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 173) -2016-06-13 17:35:25 INFO Executor:59 - Running task 3.0 in stage 53.0 (TID 176) -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 174). 2756 bytes result sent to driver -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_21_3 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 3.0 in stage 53.0 (TID 176). 2756 bytes result sent to driver -2016-06-13 17:35:25 INFO Executor:59 - Running task 2.0 in stage 53.0 (TID 175) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 174) in 13 ms on localhost (1/4) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 53.0 (TID 176) in 13 ms on localhost (2/4) -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_21_2 locally -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 2.0 in stage 53.0 (TID 175). 2756 bytes result sent to driver -2016-06-13 17:35:25 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 173). 2756 bytes result sent to driver -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 53.0 (TID 175) in 22 ms on localhost (3/4) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 173) in 26 ms on localhost (4/4) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:62031 in memory (size: 1389.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO DAGScheduler:59 - ResultStage 53 (aggregate at GaussianMixture.scala:201) finished in 0.029 s -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-06-13 17:35:25 INFO DAGScheduler:59 - Job 53 finished: aggregate at GaussianMixture.scala:201, took 0.061889 s -2016-06-13 17:35:25 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:62031 in memory (size: 17.9 KB, free: 730.5 MB) -2016-06-13 17:35:25 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-06-13 17:35:25 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Got job 54 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:35:25 INFO DAGScheduler:59 - Final stage: ResultStage 54(first at GaussianMixture.scala:171) -2016-06-13 17:35:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=685500, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 1992.0 B, free 729.9 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=687492, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 1250.0 B, free 729.9 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:62031 in memory (size: 11.8 KB, free: 730.5 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:62031 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO SparkContext:59 - Created broadcast 77 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 54 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168) -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 1 tasks -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 177, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:25 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 177) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:62031 in memory (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:35:25 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:62031 in memory (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO CacheManager:59 - Partition rdd_23_0 not found, computing it -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=454945, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block rdd_23_0 stored as values in memory (estimated size 16.0 B, free 730.2 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added rdd_23_0 in memory on localhost:62031 (size: 16.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 177). 1533 bytes result sent to driver -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 177) in 15 ms on localhost (1/1) -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-06-13 17:35:25 INFO DAGScheduler:59 - ResultStage 54 (first at GaussianMixture.scala:171) finished in 0.015 s -2016-06-13 17:35:25 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Job 54 finished: first at GaussianMixture.scala:171, took 0.033257 s -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:62031 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:25 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Got job 55 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:35:25 INFO DAGScheduler:59 - Final stage: ResultStage 55(first at GaussianMixture.scala:171) -2016-06-13 17:35:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:25 INFO BlockManager:59 - Removing RDD 14 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=438351, maxMem=766075207 -2016-06-13 17:35:25 INFO ContextCleaner:59 - Cleaned RDD 14 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 1992.0 B, free 730.2 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=438234, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.2 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:62031 in memory (size: 837.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:62031 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 55 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168) -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 3 tasks -2016-06-13 17:35:25 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 178, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 179, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 55.0 (TID 180, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:25 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 178) -2016-06-13 17:35:25 INFO Executor:59 - Running task 2.0 in stage 55.0 (TID 180) -2016-06-13 17:35:25 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 179) -2016-06-13 17:35:25 INFO CacheManager:59 - Partition rdd_23_3 not found, computing it -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=435388, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block rdd_23_3 stored as values in memory (estimated size 184.0 B, free 730.2 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added rdd_23_3 in memory on localhost:62031 (size: 184.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO Executor:59 - Finished task 2.0 in stage 55.0 (TID 180). 2132 bytes result sent to driver -2016-06-13 17:35:25 INFO CacheManager:59 - Partition rdd_23_1 not found, computing it -2016-06-13 17:35:25 INFO CacheManager:59 - Partition rdd_23_2 not found, computing it -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=433334, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block rdd_23_1 stored as values in memory (estimated size 192.0 B, free 730.2 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:62031 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added rdd_23_1 in memory on localhost:62031 (size: 192.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=433526, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block rdd_23_2 stored as values in memory (estimated size 192.0 B, free 730.2 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added rdd_23_2 in memory on localhost:62031 (size: 192.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 178). 2144 bytes result sent to driver -2016-06-13 17:35:25 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 179). 2144 bytes result sent to driver -2016-06-13 17:35:25 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 55.0 (TID 180) in 24 ms on localhost (1/3) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 178) in 27 ms on localhost (2/3) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:62031 in memory (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO DAGScheduler:59 - ResultStage 55 (first at GaussianMixture.scala:171) finished in 0.029 s -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 179) in 27 ms on localhost (3/3) -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-06-13 17:35:25 INFO DAGScheduler:59 - Job 55 finished: first at GaussianMixture.scala:171, took 0.048097 s -2016-06-13 17:35:25 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Got job 56 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:25 INFO DAGScheduler:59 - Final stage: ResultStage 56(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:62031 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=425737, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 1832.0 B, free 730.2 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=427569, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 1193.0 B, free 730.2 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:62031 (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO SparkContext:59 - Created broadcast 79 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 56 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168) -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 4 tasks -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 181, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 182, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 56.0 (TID 183, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 56.0 (TID 184, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:25 INFO Executor:59 - Running task 3.0 in stage 56.0 (TID 184) -2016-06-13 17:35:25 INFO Executor:59 - Running task 2.0 in stage 56.0 (TID 183) -2016-06-13 17:35:25 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-06-13 17:35:25 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 182) -2016-06-13 17:35:25 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 181) -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_23_1 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 182). 2082 bytes result sent to driver -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_23_0 locally -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_23_3 locally -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_23_2 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 2.0 in stage 56.0 (TID 183). 2082 bytes result sent to driver -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 182) in 11 ms on localhost (1/4) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:62031 in memory (size: 934.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO Executor:59 - Finished task 3.0 in stage 56.0 (TID 184). 2082 bytes result sent to driver -2016-06-13 17:35:25 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 181). 2082 bytes result sent to driver -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 56.0 (TID 184) in 16 ms on localhost (2/4) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 181) in 20 ms on localhost (3/4) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 56.0 (TID 183) in 19 ms on localhost (4/4) -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-06-13 17:35:25 INFO DAGScheduler:59 - ResultStage 56 (takeSample at GaussianMixture.scala:184) finished in 0.021 s -2016-06-13 17:35:25 INFO DAGScheduler:59 - Job 56 finished: takeSample at GaussianMixture.scala:184, took 0.034828 s -2016-06-13 17:35:25 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:25 INFO DAGScheduler:59 - Got job 57 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:25 INFO DAGScheduler:59 - Final stage: ResultStage 57(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:25 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting ResultStage 57 (PartitionwiseSampledRDD[24] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=420127, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 18.8 KB, free 730.2 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:62031 in memory (size: 880.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(7878) called with curMem=438471, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.2 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:62031 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:35:25 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 57 (PartitionwiseSampledRDD[24] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 4 tasks -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 185, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 186, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 57.0 (TID 187, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 57.0 (TID 188, localhost, PROCESS_LOCAL, 2373 bytes) -2016-06-13 17:35:25 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 185) -2016-06-13 17:35:25 INFO Executor:59 - Running task 2.0 in stage 57.0 (TID 187) -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_23_0 locally -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_23_2 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 185). 2048 bytes result sent to driver -2016-06-13 17:35:25 INFO Executor:59 - Running task 3.0 in stage 57.0 (TID 188) -2016-06-13 17:35:25 INFO Executor:59 - Finished task 2.0 in stage 57.0 (TID 187). 2714 bytes result sent to driver -2016-06-13 17:35:25 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 186) -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_23_3 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 3.0 in stage 57.0 (TID 188). 2682 bytes result sent to driver -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_23_1 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 186). 2724 bytes result sent to driver -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 57.0 (TID 187) in 13 ms on localhost (1/4) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 57.0 (TID 188) in 13 ms on localhost (2/4) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 186) in 16 ms on localhost (3/4) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 185) in 18 ms on localhost (4/4) -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-06-13 17:35:25 INFO DAGScheduler:59 - ResultStage 57 (takeSample at GaussianMixture.scala:184) finished in 0.019 s -2016-06-13 17:35:25 INFO DAGScheduler:59 - Job 57 finished: takeSample at GaussianMixture.scala:184, took 0.039337 s -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=446349, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 560.0 B, free 730.2 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(891) called with curMem=445659, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 891.0 B, free 730.2 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:62031 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:62031 (size: 891.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-06-13 17:35:25 INFO SparkContext:59 - Created broadcast 81 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:25 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 17:35:25 INFO ContextCleaner:59 - Cleaned RDD 12 -2016-06-13 17:35:25 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Got job 58 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:25 INFO DAGScheduler:59 - Final stage: ResultStage 58(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting ResultStage 58 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=441276, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 4.0 KB, free 730.2 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=441276, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.2 MB) -2016-06-13 17:35:25 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:25 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 58 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168) -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 4 tasks -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 189, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 190, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 58.0 (TID 191, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 58.0 (TID 192, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:25 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 190) -2016-06-13 17:35:25 INFO Executor:59 - Running task 2.0 in stage 58.0 (TID 191) -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_23_1 locally -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_23_2 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 2.0 in stage 58.0 (TID 191). 2756 bytes result sent to driver -2016-06-13 17:35:25 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 189) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 58.0 (TID 191) in 7 ms on localhost (1/4) -2016-06-13 17:35:25 INFO Executor:59 - Running task 3.0 in stage 58.0 (TID 192) -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_23_0 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 189). 2756 bytes result sent to driver -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_23_3 locally -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 189) in 13 ms on localhost (2/4) -2016-06-13 17:35:25 INFO Executor:59 - Finished task 3.0 in stage 58.0 (TID 192). 2756 bytes result sent to driver -2016-06-13 17:35:25 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 190). 2756 bytes result sent to driver -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 58.0 (TID 192) in 18 ms on localhost (3/4) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 190) in 22 ms on localhost (4/4) -2016-06-13 17:35:25 INFO DAGScheduler:59 - ResultStage 58 (aggregate at GaussianMixture.scala:201) finished in 0.024 s -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:62031 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO DAGScheduler:59 - Job 58 finished: aggregate at GaussianMixture.scala:201, took 0.039566 s -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=437946, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 560.0 B, free 730.2 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=436318, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.2 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:62031 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-06-13 17:35:25 INFO SparkContext:59 - Created broadcast 83 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:62031 in memory (size: 890.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Got job 59 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:25 INFO DAGScheduler:59 - Final stage: ResultStage 59(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting ResultStage 59 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=427887, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 4.0 KB, free 730.2 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=431951, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.2 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:62031 in memory (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:25 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 59 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168) -2016-06-13 17:35:25 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 4 tasks -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 193, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 194, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 59.0 (TID 195, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:62031 in memory (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 59.0 (TID 196, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:25 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-06-13 17:35:25 INFO Executor:59 - Running task 3.0 in stage 59.0 (TID 196) -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_23_3 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 3.0 in stage 59.0 (TID 196). 2756 bytes result sent to driver -2016-06-13 17:35:25 INFO Executor:59 - Running task 2.0 in stage 59.0 (TID 195) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 59.0 (TID 196) in 7 ms on localhost (1/4) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:62031 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_23_2 locally -2016-06-13 17:35:25 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-06-13 17:35:25 INFO Executor:59 - Finished task 2.0 in stage 59.0 (TID 195). 2756 bytes result sent to driver -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 59.0 (TID 195) in 11 ms on localhost (2/4) -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:62031 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-06-13 17:35:25 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 194) -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_23_1 locally -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:62031 in memory (size: 1389.0 B, free: 730.5 MB) -2016-06-13 17:35:25 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 194). 2756 bytes result sent to driver -2016-06-13 17:35:25 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 193) -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 194) in 22 ms on localhost (3/4) -2016-06-13 17:35:25 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-06-13 17:35:25 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:62031 in memory (size: 29.4 KB, free: 730.6 MB) -2016-06-13 17:35:25 INFO BlockManager:59 - Found block rdd_23_0 locally -2016-06-13 17:35:25 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 193). 2756 bytes result sent to driver -2016-06-13 17:35:25 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-06-13 17:35:25 INFO DAGScheduler:59 - ResultStage 59 (aggregate at GaussianMixture.scala:201) finished in 0.040 s -2016-06-13 17:35:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 193) in 39 ms on localhost (4/4) -2016-06-13 17:35:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-06-13 17:35:25 INFO DAGScheduler:59 - Job 59 finished: aggregate at GaussianMixture.scala:201, took 0.056805 s -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=60924, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:35:25 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=61484, maxMem=766075207 -2016-06-13 17:35:25 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:62031 (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 85 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 60 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 60(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 60 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=62384, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=66448, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 60 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 60.0 with 4 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 60.0 (TID 197, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 60.0 (TID 198, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 60.0 (TID 199, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 60.0 (TID 200, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 60.0 (TID 199) -2016-06-13 17:35:26 INFO Executor:59 - Running task 3.0 in stage 60.0 (TID 200) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_23_3 locally -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_23_2 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 3.0 in stage 60.0 (TID 200). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 60.0 (TID 197) -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 60.0 (TID 198) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_23_0 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 60.0 (TID 197). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 60.0 (TID 199). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 60.0 (TID 200) in 10 ms on localhost (1/4) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_23_1 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 60.0 (TID 198). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 60.0 (TID 197) in 43 ms on localhost (2/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 60.0 (TID 198) in 43 ms on localhost (3/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 60.0 (TID 199) in 43 ms on localhost (4/4) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 60.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 60 (aggregate at GaussianMixture.scala:201) finished in 0.046 s -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 60 finished: aggregate at GaussianMixture.scala:201, took 0.057567 s -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 61 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 61(first at GaussianMixture.scala:171) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 61 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=68642, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=70634, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:62031 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 87 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 61 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 1 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 201, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 201) -2016-06-13 17:35:26 INFO CacheManager:59 - Partition rdd_25_0 not found, computing it -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=71884, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block rdd_25_0 stored as values in memory (estimated size 16.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added rdd_25_0 in memory on localhost:62031 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 201). 1533 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 201) in 9 ms on localhost (1/1) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 61 (first at GaussianMixture.scala:171) finished in 0.010 s -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 61 finished: first at GaussianMixture.scala:171, took 0.025263 s -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 62 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 62(first at GaussianMixture.scala:171) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=71900, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=73892, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:62031 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 62 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 3 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 202, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 62.0 (TID 203, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 62.0 (TID 204, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 202) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 62.0 (TID 204) -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 62.0 (TID 203) -2016-06-13 17:35:26 INFO CacheManager:59 - Partition rdd_25_2 not found, computing it -2016-06-13 17:35:26 INFO CacheManager:59 - Partition rdd_25_1 not found, computing it -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=75142, maxMem=766075207 -2016-06-13 17:35:26 INFO CacheManager:59 - Partition rdd_25_3 not found, computing it -2016-06-13 17:35:26 INFO MemoryStore:59 - Block rdd_25_2 stored as values in memory (estimated size 192.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=75334, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block rdd_25_3 stored as values in memory (estimated size 184.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added rdd_25_2 in memory on localhost:62031 (size: 192.0 B, free: 730.6 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=75518, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block rdd_25_1 stored as values in memory (estimated size 192.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added rdd_25_3 in memory on localhost:62031 (size: 184.0 B, free: 730.6 MB) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 62.0 (TID 203). 2144 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added rdd_25_1 in memory on localhost:62031 (size: 192.0 B, free: 730.6 MB) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 62.0 (TID 204). 2132 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 202). 2144 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 62.0 (TID 204) in 13 ms on localhost (1/3) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 62.0 (TID 203) in 15 ms on localhost (2/3) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 202) in 17 ms on localhost (3/3) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 62 (first at GaussianMixture.scala:171) finished in 0.018 s -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 62 finished: first at GaussianMixture.scala:171, took 0.031285 s -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 63 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 63(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 63 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=75710, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 1832.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=77542, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 1193.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:62031 (size: 1193.0 B, free: 730.6 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 89 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 63 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 63.0 with 4 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 63.0 (TID 205, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 63.0 (TID 206, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 63.0 (TID 207, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 63.0 (TID 208, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 3.0 in stage 63.0 (TID 208) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 63.0 (TID 205) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 63.0 (TID 207) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 63.0 (TID 205). 2082 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_25_2 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 63.0 (TID 207). 2082 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 63.0 (TID 206) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_25_3 locally -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 63.0 (TID 206). 2082 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 63.0 (TID 205) in 10 ms on localhost (1/4) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 3.0 in stage 63.0 (TID 208). 2082 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 63.0 (TID 207) in 8 ms on localhost (2/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 63.0 (TID 208) in 11 ms on localhost (3/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 63.0 (TID 206) in 13 ms on localhost (4/4) -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 63 (takeSample at GaussianMixture.scala:184) finished in 0.015 s -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 63.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 63 finished: takeSample at GaussianMixture.scala:184, took 0.027745 s -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 64 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 64(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 64 (PartitionwiseSampledRDD[26] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=78735, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(7883) called with curMem=97959, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:62031 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 64 (PartitionwiseSampledRDD[26] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 64.0 with 4 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 64.0 (TID 209, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 64.0 (TID 210, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 64.0 (TID 211, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 64.0 (TID 212, localhost, PROCESS_LOCAL, 2373 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 64.0 (TID 209) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 64.0 (TID 211) -2016-06-13 17:35:26 INFO Executor:59 - Running task 3.0 in stage 64.0 (TID 212) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 64.0 (TID 209). 2048 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_25_3 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 3.0 in stage 64.0 (TID 212). 2702 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_25_2 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 64.0 (TID 211). 2714 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 64.0 (TID 210) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 64.0 (TID 209) in 10 ms on localhost (1/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 64.0 (TID 212) in 10 ms on localhost (2/4) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 64.0 (TID 210). 2684 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 64.0 (TID 211) in 13 ms on localhost (3/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 64.0 (TID 210) in 14 ms on localhost (4/4) -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 64 (takeSample at GaussianMixture.scala:184) finished in 0.016 s -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 64.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 64 finished: takeSample at GaussianMixture.scala:184, took 0.049306 s -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=105842, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(899) called with curMem=106402, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 899.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:62031 (size: 899.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 91 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 65 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 65(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=107301, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(2204) called with curMem=111365, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:62031 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 65 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 4 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 213, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 214, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 65.0 (TID 215, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 65.0 (TID 216, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 214) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 213) -2016-06-13 17:35:26 INFO Executor:59 - Running task 3.0 in stage 65.0 (TID 216) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_25_3 locally -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 65.0 (TID 215) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 3.0 in stage 65.0 (TID 216). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 213). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 213) in 9 ms on localhost (1/4) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 214). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_25_2 locally -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 65.0 (TID 216) in 7 ms on localhost (2/4) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 65.0 (TID 215). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 214) in 11 ms on localhost (3/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 65.0 (TID 215) in 11 ms on localhost (4/4) -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 65 (aggregate at GaussianMixture.scala:201) finished in 0.013 s -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 65 finished: aggregate at GaussianMixture.scala:201, took 0.023389 s -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=113569, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=114129, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:62031 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 93 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 66 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 66(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 66 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=115029, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=119093, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 66 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 66.0 with 4 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 66.0 (TID 217, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 66.0 (TID 218, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 66.0 (TID 219, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 66.0 (TID 220, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 66.0 (TID 218) -2016-06-13 17:35:26 INFO Executor:59 - Running task 3.0 in stage 66.0 (TID 220) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 66.0 (TID 217) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 66.0 (TID 219) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 66.0 (TID 217). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_25_3 locally -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 3.0 in stage 66.0 (TID 220). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 66.0 (TID 218). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_25_2 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 66.0 (TID 219). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 66.0 (TID 217) in 9 ms on localhost (1/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 66.0 (TID 220) in 7 ms on localhost (2/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 66.0 (TID 218) in 9 ms on localhost (3/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 66.0 (TID 219) in 10 ms on localhost (4/4) -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 66 (aggregate at GaussianMixture.scala:201) finished in 0.011 s -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 66.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 66 finished: aggregate at GaussianMixture.scala:201, took 0.020032 s -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=121287, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=121847, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:62031 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 95 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 67 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 67(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 67 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=122747, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=126811, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 67 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 67.0 with 4 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 67.0 (TID 221, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 67.0 (TID 222, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 67.0 (TID 223, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 67.0 (TID 224, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 67.0 (TID 221) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 67.0 (TID 223) -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 67.0 (TID 222) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 67.0 (TID 221). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Running task 3.0 in stage 67.0 (TID 224) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_25_2 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 67.0 (TID 223). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 67.0 (TID 222). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_25_3 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 3.0 in stage 67.0 (TID 224). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 67.0 (TID 221) in 17 ms on localhost (1/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 67.0 (TID 223) in 16 ms on localhost (2/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 67.0 (TID 222) in 18 ms on localhost (3/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 67.0 (TID 224) in 17 ms on localhost (4/4) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 67.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 67 (aggregate at GaussianMixture.scala:201) finished in 0.021 s -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 67 finished: aggregate at GaussianMixture.scala:201, took 0.032669 s -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 68 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 68(first at GaussianMixture.scala:171) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 68 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=129005, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=130997, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:62031 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 97 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 68 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 68.0 with 1 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 68.0 (TID 225, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 68.0 (TID 225) -2016-06-13 17:35:26 INFO CacheManager:59 - Partition rdd_27_0 not found, computing it -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=132247, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block rdd_27_0 stored as values in memory (estimated size 16.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added rdd_27_0 in memory on localhost:62031 (size: 16.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 68.0 (TID 225). 1533 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 68.0 (TID 225) in 8 ms on localhost (1/1) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 68.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 68 (first at GaussianMixture.scala:171) finished in 0.010 s -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 68 finished: first at GaussianMixture.scala:171, took 0.021266 s -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 69 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 69(first at GaussianMixture.scala:171) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 69 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=132263, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=134255, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:62031 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 69 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 69.0 with 3 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 69.0 (TID 226, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 69.0 (TID 227, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 69.0 (TID 228, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 69.0 (TID 227) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 69.0 (TID 228) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 69.0 (TID 226) -2016-06-13 17:35:26 INFO CacheManager:59 - Partition rdd_27_1 not found, computing it -2016-06-13 17:35:26 INFO CacheManager:59 - Partition rdd_27_2 not found, computing it -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=135505, maxMem=766075207 -2016-06-13 17:35:26 INFO CacheManager:59 - Partition rdd_27_3 not found, computing it -2016-06-13 17:35:26 INFO MemoryStore:59 - Block rdd_27_1 stored as values in memory (estimated size 192.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=135697, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block rdd_27_2 stored as values in memory (estimated size 192.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added rdd_27_1 in memory on localhost:62031 (size: 192.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=135889, maxMem=766075207 -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added rdd_27_2 in memory on localhost:62031 (size: 192.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - Block rdd_27_3 stored as values in memory (estimated size 184.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 69.0 (TID 227). 2144 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added rdd_27_3 in memory on localhost:62031 (size: 184.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 69.0 (TID 226). 2144 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 69.0 (TID 227) in 10 ms on localhost (1/3) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 69.0 (TID 228). 2132 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 69.0 (TID 228) in 12 ms on localhost (2/3) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 69.0 (TID 226) in 13 ms on localhost (3/3) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 69.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 69 (first at GaussianMixture.scala:171) finished in 0.014 s -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 69 finished: first at GaussianMixture.scala:171, took 0.024880 s -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 70 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 70(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 70 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=136073, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 1832.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=137905, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 1193.0 B, free 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:62031 (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 99 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 70 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 70.0 with 4 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 70.0 (TID 229, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 70.0 (TID 230, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 70.0 (TID 231, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 70.0 (TID 232, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 70.0 (TID 231) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 70.0 (TID 229) -2016-06-13 17:35:26 INFO Executor:59 - Running task 3.0 in stage 70.0 (TID 232) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_27_2 locally -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_27_3 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 3.0 in stage 70.0 (TID 232). 2082 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 70.0 (TID 230) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 70.0 (TID 229). 2082 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 70.0 (TID 230). 2082 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 70.0 (TID 231). 2082 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 70.0 (TID 232) in 7 ms on localhost (1/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 70.0 (TID 229) in 13 ms on localhost (2/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 70.0 (TID 230) in 14 ms on localhost (3/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 70.0 (TID 231) in 14 ms on localhost (4/4) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 70.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 70 (takeSample at GaussianMixture.scala:184) finished in 0.018 s -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 70 finished: takeSample at GaussianMixture.scala:184, took 0.029699 s -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 71 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 71(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 71 (PartitionwiseSampledRDD[28] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=139098, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 18.8 KB, free 730.4 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(7885) called with curMem=158322, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.4 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:62031 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 71 (PartitionwiseSampledRDD[28] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 71.0 with 4 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 71.0 (TID 233, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 71.0 (TID 234, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 71.0 (TID 235, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 71.0 (TID 236, localhost, PROCESS_LOCAL, 2373 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 71.0 (TID 233) -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 71.0 (TID 234) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 71.0 (TID 235) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_27_2 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 71.0 (TID 233). 2048 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 71.0 (TID 235). 2714 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 71.0 (TID 233) in 6 ms on localhost (1/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 71.0 (TID 235) in 6 ms on localhost (2/4) -2016-06-13 17:35:26 INFO Executor:59 - Running task 3.0 in stage 71.0 (TID 236) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_27_3 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 3.0 in stage 71.0 (TID 236). 2722 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 71.0 (TID 236) in 13 ms on localhost (3/4) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 71.0 (TID 234). 2719 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 71.0 (TID 234) in 15 ms on localhost (4/4) -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 71 (takeSample at GaussianMixture.scala:184) finished in 0.016 s -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 71.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 71 finished: takeSample at GaussianMixture.scala:184, took 0.026357 s -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=166207, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 560.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(895) called with curMem=166767, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 895.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:62031 (size: 895.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 101 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 72 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 72(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 72 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=167662, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(2197) called with curMem=171726, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 72 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 72.0 with 4 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 72.0 (TID 237, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 72.0 (TID 238, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 72.0 (TID 239, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 72.0 (TID 240, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 3.0 in stage 72.0 (TID 240) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 72.0 (TID 237) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 72.0 (TID 239) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 72.0 (TID 237). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_27_2 locally -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_27_3 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 3.0 in stage 72.0 (TID 240). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 72.0 (TID 239). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 72.0 (TID 237) in 9 ms on localhost (1/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 72.0 (TID 240) in 8 ms on localhost (2/4) -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 72.0 (TID 238) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 72.0 (TID 239) in 9 ms on localhost (3/4) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 72.0 (TID 238). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 72.0 (TID 238) in 14 ms on localhost (4/4) -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 72 (aggregate at GaussianMixture.scala:201) finished in 0.015 s -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 72.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 72 finished: aggregate at GaussianMixture.scala:201, took 0.030258 s -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=173923, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 560.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=174483, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:62031 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 103 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 73 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 73(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 73 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=175383, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=179447, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 104 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 73 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 73.0 with 4 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 73.0 (TID 241, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 73.0 (TID 242, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 73.0 (TID 243, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 73.0 (TID 244, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 73.0 (TID 243) -2016-06-13 17:35:26 INFO Executor:59 - Running task 3.0 in stage 73.0 (TID 244) -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 73.0 (TID 242) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_27_2 locally -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 73.0 (TID 243). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 73.0 (TID 241) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 73.0 (TID 242). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 73.0 (TID 243) in 6 ms on localhost (1/4) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 73.0 (TID 241). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_27_3 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 3.0 in stage 73.0 (TID 244). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 73.0 (TID 242) in 8 ms on localhost (2/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 73.0 (TID 241) in 11 ms on localhost (3/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 73.0 (TID 244) in 10 ms on localhost (4/4) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 73.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 73 (aggregate at GaussianMixture.scala:201) finished in 0.012 s -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 73 finished: aggregate at GaussianMixture.scala:201, took 0.024698 s -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=181641, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 560.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=182201, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:62031 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 105 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 74 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 74(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 74 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=183101, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=187165, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 106 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 74 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 74.0 with 4 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 74.0 (TID 245, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 74.0 (TID 246, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 74.0 (TID 247, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 74.0 (TID 248, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 74.0 (TID 247) -2016-06-13 17:35:26 INFO Executor:59 - Running task 3.0 in stage 74.0 (TID 248) -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 74.0 (TID 246) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_27_2 locally -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 74.0 (TID 246). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 74.0 (TID 245) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_27_3 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 3.0 in stage 74.0 (TID 248). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 74.0 (TID 247). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 74.0 (TID 245). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 74.0 (TID 246) in 9 ms on localhost (1/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 74.0 (TID 248) in 9 ms on localhost (2/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 74.0 (TID 245) in 12 ms on localhost (3/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 74.0 (TID 247) in 13 ms on localhost (4/4) -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 74 (aggregate at GaussianMixture.scala:201) finished in 0.014 s -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 74.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 74 finished: aggregate at GaussianMixture.scala:201, took 0.026399 s -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 75 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 75(first at GaussianMixture.scala:171) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 75 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=189359, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 1992.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=191351, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:62031 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 107 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 75 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 75.0 with 1 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 75.0 (TID 249, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 75.0 (TID 249) -2016-06-13 17:35:26 INFO CacheManager:59 - Partition rdd_29_0 not found, computing it -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=192601, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block rdd_29_0 stored as values in memory (estimated size 16.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added rdd_29_0 in memory on localhost:62031 (size: 16.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 75.0 (TID 249). 1533 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 75.0 (TID 249) in 9 ms on localhost (1/1) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 75.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 75 (first at GaussianMixture.scala:171) finished in 0.010 s -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 75 finished: first at GaussianMixture.scala:171, took 0.021136 s -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 76 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 76(first at GaussianMixture.scala:171) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 76 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=192617, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 1992.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=194609, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:62031 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 108 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 76 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 3 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 250, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 251, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 76.0 (TID 252, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 76.0 (TID 252) -2016-06-13 17:35:26 INFO CacheManager:59 - Partition rdd_29_3 not found, computing it -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 250) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=195859, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block rdd_29_3 stored as values in memory (estimated size 184.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added rdd_29_3 in memory on localhost:62031 (size: 184.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 76.0 (TID 252). 2132 bytes result sent to driver -2016-06-13 17:35:26 INFO CacheManager:59 - Partition rdd_29_1 not found, computing it -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 251) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=196043, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block rdd_29_1 stored as values in memory (estimated size 192.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added rdd_29_1 in memory on localhost:62031 (size: 192.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 76.0 (TID 252) in 10 ms on localhost (1/3) -2016-06-13 17:35:26 INFO CacheManager:59 - Partition rdd_29_2 not found, computing it -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 250). 2144 bytes result sent to driver -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=196235, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block rdd_29_2 stored as values in memory (estimated size 192.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 250) in 15 ms on localhost (2/3) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added rdd_29_2 in memory on localhost:62031 (size: 192.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 251). 2144 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 251) in 17 ms on localhost (3/3) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 76 (first at GaussianMixture.scala:171) finished in 0.018 s -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 76 finished: first at GaussianMixture.scala:171, took 0.030435 s -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 77 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 77(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=196427, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 1832.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=198259, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 1193.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:62031 (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 109 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 77 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 4 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 253, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 254, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 77.0 (TID 255, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 77.0 (TID 256, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 254) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 253) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 77.0 (TID 255) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_29_1 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 254). 2082 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Running task 3.0 in stage 77.0 (TID 256) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_29_2 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 77.0 (TID 255). 2082 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_29_3 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 3.0 in stage 77.0 (TID 256). 2082 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 254) in 7 ms on localhost (1/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 77.0 (TID 255) in 7 ms on localhost (2/4) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_29_0 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 253). 2082 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 77.0 (TID 256) in 8 ms on localhost (3/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 253) in 11 ms on localhost (4/4) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 77 (takeSample at GaussianMixture.scala:184) finished in 0.012 s -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 77 finished: takeSample at GaussianMixture.scala:184, took 0.022457 s -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 78 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 78(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 78 (PartitionwiseSampledRDD[30] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=199452, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 18.8 KB, free 730.4 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(7883) called with curMem=218676, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.4 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:62031 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 110 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 78 (PartitionwiseSampledRDD[30] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 4 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 257, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 258, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 78.0 (TID 259, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 78.0 (TID 260, localhost, PROCESS_LOCAL, 2373 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 258) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 257) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 78.0 (TID 259) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_29_1 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 258). 2684 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Running task 3.0 in stage 78.0 (TID 260) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_29_2 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 78.0 (TID 259). 2699 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_29_3 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 3.0 in stage 78.0 (TID 260). 2687 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_29_0 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 257). 2048 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 258) in 11 ms on localhost (1/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 78.0 (TID 260) in 10 ms on localhost (2/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 78.0 (TID 259) in 11 ms on localhost (3/4) -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 78 (takeSample at GaussianMixture.scala:184) finished in 0.014 s -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 78 finished: takeSample at GaussianMixture.scala:184, took 0.027931 s -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 257) in 12 ms on localhost (4/4) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=226559, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 560.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(882) called with curMem=227119, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 882.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:62031 (size: 882.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 111 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 79 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 79(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 79 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=228001, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(2182) called with curMem=232065, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 112 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 79 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 79.0 with 4 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 79.0 (TID 261, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 79.0 (TID 262, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 79.0 (TID 263, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 79.0 (TID 264, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 79.0 (TID 262) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 79.0 (TID 261) -2016-06-13 17:35:26 INFO Executor:59 - Running task 3.0 in stage 79.0 (TID 264) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 79.0 (TID 263) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_29_1 locally -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_29_2 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 79.0 (TID 262). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_29_3 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 79.0 (TID 263). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_29_0 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 3.0 in stage 79.0 (TID 264). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 79.0 (TID 261). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 79.0 (TID 262) in 5 ms on localhost (1/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 79.0 (TID 263) in 5 ms on localhost (2/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 79.0 (TID 264) in 5 ms on localhost (3/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 79.0 (TID 261) in 6 ms on localhost (4/4) -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 79 (aggregate at GaussianMixture.scala:201) finished in 0.006 s -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 79.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 79 finished: aggregate at GaussianMixture.scala:201, took 0.018040 s -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=234247, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 560.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=234807, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:62031 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 113 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 80 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 80(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 80 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=235707, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(2192) called with curMem=239771, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 114 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 80 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 80.0 with 4 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 80.0 (TID 265, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 80.0 (TID 266, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 80.0 (TID 267, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 80.0 (TID 268, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 80.0 (TID 265) -2016-06-13 17:35:26 INFO Executor:59 - Running task 3.0 in stage 80.0 (TID 268) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 80.0 (TID 267) -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 80.0 (TID 266) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_29_0 locally -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_29_2 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 80.0 (TID 265). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 80.0 (TID 267). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_29_1 locally -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_29_3 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 80.0 (TID 266). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Finished task 3.0 in stage 80.0 (TID 268). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 80.0 (TID 265) in 5 ms on localhost (1/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 80.0 (TID 267) in 5 ms on localhost (2/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 80.0 (TID 266) in 6 ms on localhost (3/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 80.0 (TID 268) in 5 ms on localhost (4/4) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 80.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 80 (aggregate at GaussianMixture.scala:201) finished in 0.007 s -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 80 finished: aggregate at GaussianMixture.scala:201, took 0.015665 s -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=241963, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_115 stored as values in memory (estimated size 560.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=242523, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_115_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.4 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_115_piece0 in memory on localhost:62031 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 115 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 81 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 81(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 81 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=243423, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_116 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(2192) called with curMem=247487, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_116_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_116_piece0 in memory on localhost:62031 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 116 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 81 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 81.0 with 4 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 81.0 (TID 269, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 81.0 (TID 270, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 81.0 (TID 271, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 81.0 (TID 272, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 81.0 (TID 271) -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 81.0 (TID 270) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 81.0 (TID 269) -2016-06-13 17:35:26 INFO Executor:59 - Running task 3.0 in stage 81.0 (TID 272) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_29_1 locally -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_29_0 locally -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_29_2 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 81.0 (TID 270). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 81.0 (TID 271). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 81.0 (TID 269). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_29_3 locally -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 81.0 (TID 270) in 6 ms on localhost (1/4) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 3.0 in stage 81.0 (TID 272). 2756 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 81.0 (TID 271) in 6 ms on localhost (2/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 81.0 (TID 269) in 7 ms on localhost (3/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 81.0 (TID 272) in 7 ms on localhost (4/4) -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 81 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 81.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 81 finished: aggregate at GaussianMixture.scala:201, took 0.018613 s -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 82 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 82(first at GaussianMixture.scala:171) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 82 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(2184) called with curMem=249679, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_117 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(1345) called with curMem=251863, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_117_piece0 stored as bytes in memory (estimated size 1345.0 B, free 730.3 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_117_piece0 in memory on localhost:62031 (size: 1345.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 117 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 82 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 82.0 with 1 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 82.0 (TID 273, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 82.0 (TID 273) -2016-06-13 17:35:26 INFO CacheManager:59 - Partition rdd_33_0 not found, computing it -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(480) called with curMem=253208, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block rdd_33_0 stored as values in memory (estimated size 480.0 B, free 730.3 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added rdd_33_0 in memory on localhost:62031 (size: 480.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 82.0 (TID 273). 2132 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 82.0 (TID 273) in 5 ms on localhost (1/1) -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 82 (first at GaussianMixture.scala:171) finished in 0.005 s -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 82.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 82 finished: first at GaussianMixture.scala:171, took 0.014431 s -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=253688, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_118 stored as values in memory (estimated size 712.0 B, free 730.3 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=254400, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_118_piece0 stored as bytes in memory (estimated size 880.0 B, free 730.3 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_118_piece0 in memory on localhost:62031 (size: 880.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 118 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 83 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 83(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 83 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(4288) called with curMem=255280, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_119 stored as values in memory (estimated size 4.2 KB, free 730.3 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(2279) called with curMem=259568, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_119_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_119_piece0 in memory on localhost:62031 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 119 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 83 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 83.0 with 4 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 83.0 (TID 274, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 83.0 (TID 275, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 83.0 (TID 276, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 83.0 (TID 277, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 83.0 (TID 276) -2016-06-13 17:35:26 INFO Executor:59 - Running task 3.0 in stage 83.0 (TID 277) -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 83.0 (TID 275) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 83.0 (TID 274) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_33_0 locally -2016-06-13 17:35:26 INFO CacheManager:59 - Partition rdd_33_2 not found, computing it -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 83.0 (TID 274). 2762 bytes result sent to driver -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(624) called with curMem=261847, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block rdd_33_2 stored as values in memory (estimated size 624.0 B, free 730.3 MB) -2016-06-13 17:35:26 INFO CacheManager:59 - Partition rdd_33_3 not found, computing it -2016-06-13 17:35:26 INFO CacheManager:59 - Partition rdd_33_1 not found, computing it -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added rdd_33_2 in memory on localhost:62031 (size: 624.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(624) called with curMem=262471, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block rdd_33_3 stored as values in memory (estimated size 624.0 B, free 730.3 MB) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 83.0 (TID 274) in 8 ms on localhost (1/4) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(624) called with curMem=263095, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block rdd_33_1 stored as values in memory (estimated size 624.0 B, free 730.3 MB) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 83.0 (TID 276). 2247 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added rdd_33_3 in memory on localhost:62031 (size: 624.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added rdd_33_1 in memory on localhost:62031 (size: 624.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 3.0 in stage 83.0 (TID 277). 2247 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 83.0 (TID 276) in 11 ms on localhost (2/4) -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 83.0 (TID 275). 2247 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 83.0 (TID 277) in 12 ms on localhost (3/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 83.0 (TID 275) in 13 ms on localhost (4/4) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 83.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 83 (aggregate at GaussianMixture.scala:201) finished in 0.015 s -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 83 finished: aggregate at GaussianMixture.scala:201, took 0.024709 s -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=263719, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_120 stored as values in memory (estimated size 712.0 B, free 730.3 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(934) called with curMem=264431, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_120_piece0 stored as bytes in memory (estimated size 934.0 B, free 730.3 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_120_piece0 in memory on localhost:62031 (size: 934.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 120 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 84 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 84(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 84 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(4288) called with curMem=265365, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_121 stored as values in memory (estimated size 4.2 KB, free 730.3 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(2343) called with curMem=269653, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_121_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.3 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_121_piece0 in memory on localhost:62031 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 121 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 84 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 84.0 with 4 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 84.0 (TID 278, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 84.0 (TID 279, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 84.0 (TID 280, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 84.0 (TID 281, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 84.0 (TID 278) -2016-06-13 17:35:26 INFO Executor:59 - Running task 3.0 in stage 84.0 (TID 281) -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 84.0 (TID 279) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 84.0 (TID 280) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_33_3 locally -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_33_1 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 3.0 in stage 84.0 (TID 281). 2762 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_33_0 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 84.0 (TID 278). 2762 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 84.0 (TID 279). 2762 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_33_2 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 84.0 (TID 280). 2762 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 84.0 (TID 281) in 5 ms on localhost (1/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 84.0 (TID 279) in 7 ms on localhost (2/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 84.0 (TID 278) in 9 ms on localhost (3/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 84.0 (TID 280) in 8 ms on localhost (4/4) -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 84 (aggregate at GaussianMixture.scala:201) finished in 0.010 s -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 84.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 84 finished: aggregate at GaussianMixture.scala:201, took 0.020834 s -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=271996, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_122 stored as values in memory (estimated size 712.0 B, free 730.3 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=272708, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_122_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.3 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_122_piece0 in memory on localhost:62031 (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 122 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:26 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Got job 85 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:26 INFO DAGScheduler:59 - Final stage: ResultStage 85(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting ResultStage 85 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(4288) called with curMem=273640, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_123 stored as values in memory (estimated size 4.2 KB, free 730.3 MB) -2016-06-13 17:35:26 INFO MemoryStore:59 - ensureFreeSpace(2340) called with curMem=277928, maxMem=766075207 -2016-06-13 17:35:26 INFO MemoryStore:59 - Block broadcast_123_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.3 MB) -2016-06-13 17:35:26 INFO BlockManagerInfo:59 - Added broadcast_123_piece0 in memory on localhost:62031 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 17:35:26 INFO SparkContext:59 - Created broadcast 123 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 85 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Adding task set 85.0 with 4 tasks -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 85.0 (TID 282, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 85.0 (TID 283, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 85.0 (TID 284, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 85.0 (TID 285, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:26 INFO Executor:59 - Running task 1.0 in stage 85.0 (TID 283) -2016-06-13 17:35:26 INFO Executor:59 - Running task 2.0 in stage 85.0 (TID 284) -2016-06-13 17:35:26 INFO Executor:59 - Running task 0.0 in stage 85.0 (TID 282) -2016-06-13 17:35:26 INFO Executor:59 - Running task 3.0 in stage 85.0 (TID 285) -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_33_1 locally -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_33_0 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 1.0 in stage 85.0 (TID 283). 2762 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_33_2 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 0.0 in stage 85.0 (TID 282). 2762 bytes result sent to driver -2016-06-13 17:35:26 INFO BlockManager:59 - Found block rdd_33_3 locally -2016-06-13 17:35:26 INFO Executor:59 - Finished task 3.0 in stage 85.0 (TID 285). 2762 bytes result sent to driver -2016-06-13 17:35:26 INFO Executor:59 - Finished task 2.0 in stage 85.0 (TID 284). 2762 bytes result sent to driver -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 85.0 (TID 283) in 6 ms on localhost (1/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 85.0 (TID 285) in 7 ms on localhost (2/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 85.0 (TID 282) in 9 ms on localhost (3/4) -2016-06-13 17:35:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 85.0 (TID 284) in 8 ms on localhost (4/4) -2016-06-13 17:35:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 85.0, whose tasks have all completed, from pool -2016-06-13 17:35:26 INFO DAGScheduler:59 - ResultStage 85 (aggregate at GaussianMixture.scala:201) finished in 0.009 s -2016-06-13 17:35:26 INFO DAGScheduler:59 - Job 85 finished: aggregate at GaussianMixture.scala:201, took 0.019407 s -2016-06-13 17:35:27 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:35:27 INFO DAGScheduler:59 - Got job 86 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:35:27 INFO DAGScheduler:59 - Final stage: ResultStage 86(first at GaussianMixture.scala:171) -2016-06-13 17:35:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:27 INFO DAGScheduler:59 - Submitting ResultStage 86 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=280268, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_124 stored as values in memory (estimated size 1992.0 B, free 730.3 MB) -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=282260, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_124_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.3 MB) -2016-06-13 17:35:27 INFO BlockManagerInfo:59 - Added broadcast_124_piece0 in memory on localhost:62031 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:35:27 INFO SparkContext:59 - Created broadcast 124 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 86 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:35:27 INFO TaskSchedulerImpl:59 - Adding task set 86.0 with 1 tasks -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 86.0 (TID 286, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:35:27 INFO Executor:59 - Running task 0.0 in stage 86.0 (TID 286) -2016-06-13 17:35:27 INFO CacheManager:59 - Partition rdd_35_0 not found, computing it -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(224) called with curMem=283510, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block rdd_35_0 stored as values in memory (estimated size 224.0 B, free 730.3 MB) -2016-06-13 17:35:27 INFO BlockManagerInfo:59 - Added rdd_35_0 in memory on localhost:62031 (size: 224.0 B, free: 730.5 MB) -2016-06-13 17:35:27 INFO Executor:59 - Finished task 0.0 in stage 86.0 (TID 286). 1688 bytes result sent to driver -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 86.0 (TID 286) in 5 ms on localhost (1/1) -2016-06-13 17:35:27 INFO DAGScheduler:59 - ResultStage 86 (first at GaussianMixture.scala:171) finished in 0.005 s -2016-06-13 17:35:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 86.0, whose tasks have all completed, from pool -2016-06-13 17:35:27 INFO DAGScheduler:59 - Job 86 finished: first at GaussianMixture.scala:171, took 0.014376 s -2016-06-13 17:35:27 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:27 INFO DAGScheduler:59 - Got job 87 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:27 INFO DAGScheduler:59 - Final stage: ResultStage 87(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:27 INFO DAGScheduler:59 - Submitting ResultStage 87 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=283734, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_125 stored as values in memory (estimated size 1832.0 B, free 730.3 MB) -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=285566, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_125_piece0 stored as bytes in memory (estimated size 1193.0 B, free 730.3 MB) -2016-06-13 17:35:27 INFO BlockManagerInfo:59 - Added broadcast_125_piece0 in memory on localhost:62031 (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:35:27 INFO SparkContext:59 - Created broadcast 125 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 87 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:35:27 INFO TaskSchedulerImpl:59 - Adding task set 87.0 with 4 tasks -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 87.0 (TID 287, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 87.0 (TID 288, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 87.0 (TID 289, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 87.0 (TID 290, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO Executor:59 - Running task 1.0 in stage 87.0 (TID 288) -2016-06-13 17:35:27 INFO Executor:59 - Running task 3.0 in stage 87.0 (TID 290) -2016-06-13 17:35:27 INFO Executor:59 - Running task 0.0 in stage 87.0 (TID 287) -2016-06-13 17:35:27 INFO Executor:59 - Running task 2.0 in stage 87.0 (TID 289) -2016-06-13 17:35:27 INFO CacheManager:59 - Partition rdd_35_1 not found, computing it -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:35:27 INFO Executor:59 - Finished task 0.0 in stage 87.0 (TID 287). 2082 bytes result sent to driver -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=286759, maxMem=766075207 -2016-06-13 17:35:27 INFO CacheManager:59 - Partition rdd_35_2 not found, computing it -2016-06-13 17:35:27 INFO MemoryStore:59 - Block rdd_35_1 stored as values in memory (estimated size 288.0 B, free 730.3 MB) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 87.0 (TID 287) in 5 ms on localhost (1/4) -2016-06-13 17:35:27 INFO CacheManager:59 - Partition rdd_35_3 not found, computing it -2016-06-13 17:35:27 INFO BlockManagerInfo:59 - Added rdd_35_1 in memory on localhost:62031 (size: 288.0 B, free: 730.5 MB) -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=287047, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block rdd_35_2 stored as values in memory (estimated size 288.0 B, free 730.3 MB) -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=287335, maxMem=766075207 -2016-06-13 17:35:27 INFO Executor:59 - Finished task 1.0 in stage 87.0 (TID 288). 1567 bytes result sent to driver -2016-06-13 17:35:27 INFO MemoryStore:59 - Block rdd_35_3 stored as values in memory (estimated size 288.0 B, free 730.3 MB) -2016-06-13 17:35:27 INFO BlockManagerInfo:59 - Added rdd_35_2 in memory on localhost:62031 (size: 288.0 B, free: 730.5 MB) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 87.0 (TID 288) in 7 ms on localhost (2/4) -2016-06-13 17:35:27 INFO BlockManagerInfo:59 - Added rdd_35_3 in memory on localhost:62031 (size: 288.0 B, free: 730.5 MB) -2016-06-13 17:35:27 INFO Executor:59 - Finished task 2.0 in stage 87.0 (TID 289). 1567 bytes result sent to driver -2016-06-13 17:35:27 INFO Executor:59 - Finished task 3.0 in stage 87.0 (TID 290). 1567 bytes result sent to driver -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 87.0 (TID 289) in 8 ms on localhost (3/4) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 87.0 (TID 290) in 7 ms on localhost (4/4) -2016-06-13 17:35:27 INFO DAGScheduler:59 - ResultStage 87 (takeSample at GaussianMixture.scala:184) finished in 0.009 s -2016-06-13 17:35:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 87.0, whose tasks have all completed, from pool -2016-06-13 17:35:27 INFO DAGScheduler:59 - Job 87 finished: takeSample at GaussianMixture.scala:184, took 0.017959 s -2016-06-13 17:35:27 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:35:27 INFO DAGScheduler:59 - Got job 88 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:35:27 INFO DAGScheduler:59 - Final stage: ResultStage 88(takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:27 INFO DAGScheduler:59 - Submitting ResultStage 88 (PartitionwiseSampledRDD[36] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=287623, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_126 stored as values in memory (estimated size 18.8 KB, free 730.3 MB) -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(7892) called with curMem=306847, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_126_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.3 MB) -2016-06-13 17:35:27 INFO BlockManagerInfo:59 - Added broadcast_126_piece0 in memory on localhost:62031 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:35:27 INFO SparkContext:59 - Created broadcast 126 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 88 (PartitionwiseSampledRDD[36] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:35:27 INFO TaskSchedulerImpl:59 - Adding task set 88.0 with 4 tasks -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 88.0 (TID 291, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 88.0 (TID 292, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 88.0 (TID 293, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 88.0 (TID 294, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 17:35:27 INFO Executor:59 - Running task 1.0 in stage 88.0 (TID 292) -2016-06-13 17:35:27 INFO Executor:59 - Running task 3.0 in stage 88.0 (TID 294) -2016-06-13 17:35:27 INFO Executor:59 - Running task 2.0 in stage 88.0 (TID 293) -2016-06-13 17:35:27 INFO Executor:59 - Running task 0.0 in stage 88.0 (TID 291) -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:35:27 INFO Executor:59 - Finished task 3.0 in stage 88.0 (TID 294). 2339 bytes result sent to driver -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:35:27 INFO Executor:59 - Finished task 1.0 in stage 88.0 (TID 292). 2344 bytes result sent to driver -2016-06-13 17:35:27 INFO Executor:59 - Finished task 0.0 in stage 88.0 (TID 291). 2247 bytes result sent to driver -2016-06-13 17:35:27 INFO Executor:59 - Finished task 2.0 in stage 88.0 (TID 293). 2303 bytes result sent to driver -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 88.0 (TID 292) in 6 ms on localhost (1/4) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 88.0 (TID 291) in 6 ms on localhost (2/4) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 88.0 (TID 294) in 5 ms on localhost (3/4) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 88.0 (TID 293) in 6 ms on localhost (4/4) -2016-06-13 17:35:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 88.0, whose tasks have all completed, from pool -2016-06-13 17:35:27 INFO DAGScheduler:59 - ResultStage 88 (takeSample at GaussianMixture.scala:184) finished in 0.007 s -2016-06-13 17:35:27 INFO DAGScheduler:59 - Job 88 finished: takeSample at GaussianMixture.scala:184, took 0.016415 s -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=314739, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_127 stored as values in memory (estimated size 712.0 B, free 730.3 MB) -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(937) called with curMem=315451, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_127_piece0 stored as bytes in memory (estimated size 937.0 B, free 730.3 MB) -2016-06-13 17:35:27 INFO BlockManagerInfo:59 - Added broadcast_127_piece0 in memory on localhost:62031 (size: 937.0 B, free: 730.5 MB) -2016-06-13 17:35:27 INFO SparkContext:59 - Created broadcast 127 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:27 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:27 INFO DAGScheduler:59 - Got job 89 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:27 INFO DAGScheduler:59 - Final stage: ResultStage 89(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:27 INFO DAGScheduler:59 - Submitting ResultStage 89 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=316388, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_128 stored as values in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=320484, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_128_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 17:35:27 INFO BlockManagerInfo:59 - Added broadcast_128_piece0 in memory on localhost:62031 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:35:27 INFO SparkContext:59 - Created broadcast 128 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 89 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:35:27 INFO TaskSchedulerImpl:59 - Adding task set 89.0 with 4 tasks -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 89.0 (TID 295, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 89.0 (TID 296, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 89.0 (TID 297, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 89.0 (TID 298, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO Executor:59 - Running task 0.0 in stage 89.0 (TID 295) -2016-06-13 17:35:27 INFO Executor:59 - Running task 2.0 in stage 89.0 (TID 297) -2016-06-13 17:35:27 INFO Executor:59 - Running task 1.0 in stage 89.0 (TID 296) -2016-06-13 17:35:27 INFO Executor:59 - Running task 3.0 in stage 89.0 (TID 298) -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:35:27 INFO Executor:59 - Finished task 0.0 in stage 89.0 (TID 295). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:35:27 INFO Executor:59 - Finished task 1.0 in stage 89.0 (TID 296). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:35:27 INFO Executor:59 - Finished task 2.0 in stage 89.0 (TID 297). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO Executor:59 - Finished task 3.0 in stage 89.0 (TID 298). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 89.0 (TID 295) in 7 ms on localhost (1/4) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 89.0 (TID 297) in 6 ms on localhost (2/4) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 89.0 (TID 296) in 7 ms on localhost (3/4) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 89.0 (TID 298) in 7 ms on localhost (4/4) -2016-06-13 17:35:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 89.0, whose tasks have all completed, from pool -2016-06-13 17:35:27 INFO DAGScheduler:59 - ResultStage 89 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 17:35:27 INFO DAGScheduler:59 - Job 89 finished: aggregate at GaussianMixture.scala:201, took 0.018760 s -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=322722, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_129 stored as values in memory (estimated size 712.0 B, free 730.3 MB) -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(934) called with curMem=323434, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_129_piece0 stored as bytes in memory (estimated size 934.0 B, free 730.3 MB) -2016-06-13 17:35:27 INFO BlockManagerInfo:59 - Added broadcast_129_piece0 in memory on localhost:62031 (size: 934.0 B, free: 730.5 MB) -2016-06-13 17:35:27 INFO SparkContext:59 - Created broadcast 129 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:27 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:27 INFO DAGScheduler:59 - Got job 90 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:27 INFO DAGScheduler:59 - Final stage: ResultStage 90(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:27 INFO DAGScheduler:59 - Submitting ResultStage 90 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=324368, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_130 stored as values in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(2239) called with curMem=328464, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_130_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 17:35:27 INFO BlockManagerInfo:59 - Added broadcast_130_piece0 in memory on localhost:62031 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:35:27 INFO SparkContext:59 - Created broadcast 130 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 90 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:35:27 INFO TaskSchedulerImpl:59 - Adding task set 90.0 with 4 tasks -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 90.0 (TID 299, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 90.0 (TID 300, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 90.0 (TID 301, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 90.0 (TID 302, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO Executor:59 - Running task 0.0 in stage 90.0 (TID 299) -2016-06-13 17:35:27 INFO Executor:59 - Running task 1.0 in stage 90.0 (TID 300) -2016-06-13 17:35:27 INFO Executor:59 - Running task 2.0 in stage 90.0 (TID 301) -2016-06-13 17:35:27 INFO Executor:59 - Running task 3.0 in stage 90.0 (TID 302) -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:35:27 INFO Executor:59 - Finished task 2.0 in stage 90.0 (TID 301). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO Executor:59 - Finished task 1.0 in stage 90.0 (TID 300). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO Executor:59 - Finished task 3.0 in stage 90.0 (TID 302). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO Executor:59 - Finished task 0.0 in stage 90.0 (TID 299). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 90.0 (TID 301) in 6 ms on localhost (1/4) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 90.0 (TID 299) in 8 ms on localhost (2/4) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 90.0 (TID 302) in 6 ms on localhost (3/4) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 90.0 (TID 300) in 7 ms on localhost (4/4) -2016-06-13 17:35:27 INFO DAGScheduler:59 - ResultStage 90 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 17:35:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 90.0, whose tasks have all completed, from pool -2016-06-13 17:35:27 INFO DAGScheduler:59 - Job 90 finished: aggregate at GaussianMixture.scala:201, took 0.019721 s -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=330703, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_131 stored as values in memory (estimated size 712.0 B, free 730.3 MB) -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=331415, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_131_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.3 MB) -2016-06-13 17:35:27 INFO BlockManagerInfo:59 - Added broadcast_131_piece0 in memory on localhost:62031 (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:35:27 INFO SparkContext:59 - Created broadcast 131 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:27 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:27 INFO DAGScheduler:59 - Got job 91 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:27 INFO DAGScheduler:59 - Final stage: ResultStage 91(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:27 INFO DAGScheduler:59 - Submitting ResultStage 91 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=332347, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_132 stored as values in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(2239) called with curMem=336443, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_132_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 17:35:27 INFO BlockManagerInfo:59 - Added broadcast_132_piece0 in memory on localhost:62031 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:35:27 INFO SparkContext:59 - Created broadcast 132 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 91 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:35:27 INFO TaskSchedulerImpl:59 - Adding task set 91.0 with 4 tasks -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 91.0 (TID 303, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 91.0 (TID 304, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 91.0 (TID 305, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 91.0 (TID 306, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO Executor:59 - Running task 0.0 in stage 91.0 (TID 303) -2016-06-13 17:35:27 INFO Executor:59 - Running task 3.0 in stage 91.0 (TID 306) -2016-06-13 17:35:27 INFO Executor:59 - Running task 2.0 in stage 91.0 (TID 305) -2016-06-13 17:35:27 INFO Executor:59 - Running task 1.0 in stage 91.0 (TID 304) -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:35:27 INFO Executor:59 - Finished task 3.0 in stage 91.0 (TID 306). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:35:27 INFO Executor:59 - Finished task 0.0 in stage 91.0 (TID 303). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:35:27 INFO Executor:59 - Finished task 2.0 in stage 91.0 (TID 305). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 91.0 (TID 306) in 5 ms on localhost (1/4) -2016-06-13 17:35:27 INFO Executor:59 - Finished task 1.0 in stage 91.0 (TID 304). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 91.0 (TID 303) in 7 ms on localhost (2/4) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 91.0 (TID 305) in 6 ms on localhost (3/4) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 91.0 (TID 304) in 7 ms on localhost (4/4) -2016-06-13 17:35:27 INFO DAGScheduler:59 - ResultStage 91 (aggregate at GaussianMixture.scala:201) finished in 0.007 s -2016-06-13 17:35:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 91.0, whose tasks have all completed, from pool -2016-06-13 17:35:27 INFO DAGScheduler:59 - Job 91 finished: aggregate at GaussianMixture.scala:201, took 0.016815 s -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=338682, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_133 stored as values in memory (estimated size 712.0 B, free 730.3 MB) -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(933) called with curMem=339394, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_133_piece0 stored as bytes in memory (estimated size 933.0 B, free 730.3 MB) -2016-06-13 17:35:27 INFO BlockManagerInfo:59 - Added broadcast_133_piece0 in memory on localhost:62031 (size: 933.0 B, free: 730.5 MB) -2016-06-13 17:35:27 INFO SparkContext:59 - Created broadcast 133 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:27 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:27 INFO DAGScheduler:59 - Got job 92 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:27 INFO DAGScheduler:59 - Final stage: ResultStage 92(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:27 INFO DAGScheduler:59 - Submitting ResultStage 92 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=340327, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_134 stored as values in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(2239) called with curMem=344423, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_134_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 17:35:27 INFO BlockManagerInfo:59 - Added broadcast_134_piece0 in memory on localhost:62031 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:35:27 INFO SparkContext:59 - Created broadcast 134 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 92 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:35:27 INFO TaskSchedulerImpl:59 - Adding task set 92.0 with 4 tasks -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 92.0 (TID 307, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 92.0 (TID 308, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 92.0 (TID 309, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 92.0 (TID 310, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO Executor:59 - Running task 1.0 in stage 92.0 (TID 308) -2016-06-13 17:35:27 INFO Executor:59 - Running task 3.0 in stage 92.0 (TID 310) -2016-06-13 17:35:27 INFO Executor:59 - Running task 2.0 in stage 92.0 (TID 309) -2016-06-13 17:35:27 INFO Executor:59 - Running task 0.0 in stage 92.0 (TID 307) -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:35:27 INFO Executor:59 - Finished task 3.0 in stage 92.0 (TID 310). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:35:27 INFO Executor:59 - Finished task 2.0 in stage 92.0 (TID 309). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:35:27 INFO Executor:59 - Finished task 1.0 in stage 92.0 (TID 308). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO Executor:59 - Finished task 0.0 in stage 92.0 (TID 307). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 92.0 (TID 310) in 5 ms on localhost (1/4) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 92.0 (TID 308) in 6 ms on localhost (2/4) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 92.0 (TID 309) in 7 ms on localhost (3/4) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 92.0 (TID 307) in 8 ms on localhost (4/4) -2016-06-13 17:35:27 INFO DAGScheduler:59 - ResultStage 92 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 17:35:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 92.0, whose tasks have all completed, from pool -2016-06-13 17:35:27 INFO DAGScheduler:59 - Job 92 finished: aggregate at GaussianMixture.scala:201, took 0.016721 s -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=346662, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_135 stored as values in memory (estimated size 712.0 B, free 730.3 MB) -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=347374, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_135_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.3 MB) -2016-06-13 17:35:27 INFO BlockManagerInfo:59 - Added broadcast_135_piece0 in memory on localhost:62031 (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:35:27 INFO SparkContext:59 - Created broadcast 135 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:27 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:27 INFO DAGScheduler:59 - Got job 93 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:27 INFO DAGScheduler:59 - Final stage: ResultStage 93(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:27 INFO DAGScheduler:59 - Submitting ResultStage 93 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=348306, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_136 stored as values in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(2241) called with curMem=352402, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_136_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 17:35:27 INFO BlockManagerInfo:59 - Added broadcast_136_piece0 in memory on localhost:62031 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:35:27 INFO SparkContext:59 - Created broadcast 136 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 93 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:35:27 INFO TaskSchedulerImpl:59 - Adding task set 93.0 with 4 tasks -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 93.0 (TID 311, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 93.0 (TID 312, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 93.0 (TID 313, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 93.0 (TID 314, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO Executor:59 - Running task 0.0 in stage 93.0 (TID 311) -2016-06-13 17:35:27 INFO Executor:59 - Running task 1.0 in stage 93.0 (TID 312) -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:35:27 INFO Executor:59 - Finished task 0.0 in stage 93.0 (TID 311). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO Executor:59 - Running task 2.0 in stage 93.0 (TID 313) -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:35:27 INFO Executor:59 - Finished task 1.0 in stage 93.0 (TID 312). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO Executor:59 - Running task 3.0 in stage 93.0 (TID 314) -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 93.0 (TID 311) in 9 ms on localhost (1/4) -2016-06-13 17:35:27 INFO Executor:59 - Finished task 3.0 in stage 93.0 (TID 314). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 93.0 (TID 312) in 9 ms on localhost (2/4) -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:35:27 INFO Executor:59 - Finished task 2.0 in stage 93.0 (TID 313). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 93.0 (TID 314) in 9 ms on localhost (3/4) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 93.0 (TID 313) in 11 ms on localhost (4/4) -2016-06-13 17:35:27 INFO DAGScheduler:59 - ResultStage 93 (aggregate at GaussianMixture.scala:201) finished in 0.014 s -2016-06-13 17:35:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 93.0, whose tasks have all completed, from pool -2016-06-13 17:35:27 INFO DAGScheduler:59 - Job 93 finished: aggregate at GaussianMixture.scala:201, took 0.023788 s -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=354643, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_137 stored as values in memory (estimated size 712.0 B, free 730.2 MB) -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=355355, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_137_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.2 MB) -2016-06-13 17:35:27 INFO BlockManagerInfo:59 - Added broadcast_137_piece0 in memory on localhost:62031 (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:35:27 INFO SparkContext:59 - Created broadcast 137 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:35:27 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:35:27 INFO DAGScheduler:59 - Got job 94 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:35:27 INFO DAGScheduler:59 - Final stage: ResultStage 94(aggregate at GaussianMixture.scala:201) -2016-06-13 17:35:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:27 INFO DAGScheduler:59 - Submitting ResultStage 94 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=356287, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_138 stored as values in memory (estimated size 4.0 KB, free 730.2 MB) -2016-06-13 17:35:27 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=360383, maxMem=766075207 -2016-06-13 17:35:27 INFO MemoryStore:59 - Block broadcast_138_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 17:35:27 INFO BlockManagerInfo:59 - Added broadcast_138_piece0 in memory on localhost:62031 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:35:27 INFO SparkContext:59 - Created broadcast 138 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 94 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:35:27 INFO TaskSchedulerImpl:59 - Adding task set 94.0 with 4 tasks -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 94.0 (TID 315, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 94.0 (TID 316, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 94.0 (TID 317, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 94.0 (TID 318, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:35:27 INFO Executor:59 - Running task 2.0 in stage 94.0 (TID 317) -2016-06-13 17:35:27 INFO Executor:59 - Running task 1.0 in stage 94.0 (TID 316) -2016-06-13 17:35:27 INFO Executor:59 - Running task 3.0 in stage 94.0 (TID 318) -2016-06-13 17:35:27 INFO Executor:59 - Running task 0.0 in stage 94.0 (TID 315) -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:35:27 INFO Executor:59 - Finished task 3.0 in stage 94.0 (TID 318). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:35:27 INFO Executor:59 - Finished task 1.0 in stage 94.0 (TID 316). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO Executor:59 - Finished task 0.0 in stage 94.0 (TID 315). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO Executor:59 - Finished task 2.0 in stage 94.0 (TID 317). 2762 bytes result sent to driver -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 94.0 (TID 318) in 5 ms on localhost (1/4) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 94.0 (TID 316) in 6 ms on localhost (2/4) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 94.0 (TID 315) in 7 ms on localhost (3/4) -2016-06-13 17:35:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 94.0 (TID 317) in 6 ms on localhost (4/4) -2016-06-13 17:35:27 INFO DAGScheduler:59 - ResultStage 94 (aggregate at GaussianMixture.scala:201) finished in 0.007 s -2016-06-13 17:35:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 94.0, whose tasks have all completed, from pool -2016-06-13 17:35:27 INFO DAGScheduler:59 - Job 94 finished: aggregate at GaussianMixture.scala:201, took 0.017075 s -2016-06-13 17:35:28 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 17:35:28 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 17:35:28 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 17:35:28 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 17:35:28 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 17:35:28 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 17:35:28 INFO SparkContext:59 - Starting job: saveAsTextFile at GaussianMixtureModel.scala:158 -2016-06-13 17:35:28 INFO DAGScheduler:59 - Got job 95 (saveAsTextFile at GaussianMixtureModel.scala:158) with 1 output partitions -2016-06-13 17:35:28 INFO DAGScheduler:59 - Final stage: ResultStage 95(saveAsTextFile at GaussianMixtureModel.scala:158) -2016-06-13 17:35:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:28 INFO DAGScheduler:59 - Submitting ResultStage 95 (MapPartitionsRDD[38] at saveAsTextFile at GaussianMixtureModel.scala:158), which has no missing parents -2016-06-13 17:35:28 INFO MemoryStore:59 - ensureFreeSpace(126936) called with curMem=362621, maxMem=766075207 -2016-06-13 17:35:28 INFO MemoryStore:59 - Block broadcast_139 stored as values in memory (estimated size 124.0 KB, free 730.1 MB) -2016-06-13 17:35:28 INFO MemoryStore:59 - ensureFreeSpace(42532) called with curMem=489557, maxMem=766075207 -2016-06-13 17:35:28 INFO MemoryStore:59 - Block broadcast_139_piece0 stored as bytes in memory (estimated size 41.5 KB, free 730.1 MB) -2016-06-13 17:35:28 INFO BlockManagerInfo:59 - Added broadcast_139_piece0 in memory on localhost:62031 (size: 41.5 KB, free: 730.4 MB) -2016-06-13 17:35:28 INFO SparkContext:59 - Created broadcast 139 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 95 (MapPartitionsRDD[38] at saveAsTextFile at GaussianMixtureModel.scala:158) -2016-06-13 17:35:28 INFO TaskSchedulerImpl:59 - Adding task set 95.0 with 1 tasks -2016-06-13 17:35:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 95.0 (TID 319, localhost, PROCESS_LOCAL, 2178 bytes) -2016-06-13 17:35:28 INFO Executor:59 - Running task 0.0 in stage 95.0 (TID 319) -2016-06-13 17:35:29 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201606131735_0095_m_000000_319' to file:/C:/Users/xubo/AppData/Local/Temp/spark-00cd8c2a-fbd2-46d2-91c1-6b44b1715abf/metadata/_temporary/0/task_201606131735_0095_m_000000 -2016-06-13 17:35:29 INFO SparkHadoopMapRedUtil:59 - attempt_201606131735_0095_m_000000_319: Committed -2016-06-13 17:35:29 INFO Executor:59 - Finished task 0.0 in stage 95.0 (TID 319). 1864 bytes result sent to driver -2016-06-13 17:35:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 95.0 (TID 319) in 353 ms on localhost (1/1) -2016-06-13 17:35:29 INFO DAGScheduler:59 - ResultStage 95 (saveAsTextFile at GaussianMixtureModel.scala:158) finished in 0.353 s -2016-06-13 17:35:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 95.0, whose tasks have all completed, from pool -2016-06-13 17:35:29 INFO DAGScheduler:59 - Job 95 finished: saveAsTextFile at GaussianMixtureModel.scala:158, took 0.439270 s -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_139_piece0 on localhost:62031 in memory (size: 41.5 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 96 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_138_piece0 on localhost:62031 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 95 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_137_piece0 on localhost:62031 in memory (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_136_piece0 on localhost:62031 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 94 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_135_piece0 on localhost:62031 in memory (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_134_piece0 on localhost:62031 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 93 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_133_piece0 on localhost:62031 in memory (size: 933.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_132_piece0 on localhost:62031 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 92 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_131_piece0 on localhost:62031 in memory (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_130_piece0 on localhost:62031 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 91 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_129_piece0 on localhost:62031 in memory (size: 934.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_128_piece0 on localhost:62031 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 90 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_127_piece0 on localhost:62031 in memory (size: 937.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_126_piece0 on localhost:62031 in memory (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 89 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_125_piece0 on localhost:62031 in memory (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 88 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_124_piece0 on localhost:62031 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 87 -2016-06-13 17:35:29 INFO BlockManager:59 - Removing RDD 35 -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned RDD 35 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_123_piece0 on localhost:62031 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 86 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_122_piece0 on localhost:62031 in memory (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_121_piece0 on localhost:62031 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 85 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_120_piece0 on localhost:62031 in memory (size: 934.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_119_piece0 on localhost:62031 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 84 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_118_piece0 on localhost:62031 in memory (size: 880.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_117_piece0 on localhost:62031 in memory (size: 1345.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 83 -2016-06-13 17:35:29 INFO BlockManager:59 - Removing RDD 33 -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned RDD 33 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_116_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 82 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_115_piece0 on localhost:62031 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_114_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 81 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_113_piece0 on localhost:62031 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_112_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 80 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_111_piece0 on localhost:62031 in memory (size: 882.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_110_piece0 on localhost:62031 in memory (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 79 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_109_piece0 on localhost:62031 in memory (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 78 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_108_piece0 on localhost:62031 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 77 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_107_piece0 on localhost:62031 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 76 -2016-06-13 17:35:29 INFO BlockManager:59 - Removing RDD 29 -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned RDD 29 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_106_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 75 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_105_piece0 on localhost:62031 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_104_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 74 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_103_piece0 on localhost:62031 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_102_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 73 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_101_piece0 on localhost:62031 in memory (size: 895.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_100_piece0 on localhost:62031 in memory (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 72 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_99_piece0 on localhost:62031 in memory (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 71 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_98_piece0 on localhost:62031 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 70 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_97_piece0 on localhost:62031 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 69 -2016-06-13 17:35:29 INFO BlockManager:59 - Removing RDD 27 -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned RDD 27 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_96_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_95_piece0 on localhost:62031 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_94_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_93_piece0 on localhost:62031 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_92_piece0 on localhost:62031 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_91_piece0 on localhost:62031 in memory (size: 899.0 B, free: 730.6 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_90_piece0 on localhost:62031 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_89_piece0 on localhost:62031 in memory (size: 1193.0 B, free: 730.6 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_88_piece0 on localhost:62031 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_87_piece0 on localhost:62031 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-06-13 17:35:29 INFO BlockManager:59 - Removing RDD 25 -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned RDD 25 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:62031 in memory (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:62031 in memory (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:62031 in memory (size: 891.0 B, free: 730.6 MB) -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:62031 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:62031 in memory (size: 1193.0 B, free: 730.6 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:62031 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:62031 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-06-13 17:35:29 INFO BlockManager:59 - Removing RDD 23 -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned RDD 23 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:62031 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-06-13 17:35:29 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:62031 in memory (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:35:29 INFO BlockManager:59 - Removing RDD 21 -2016-06-13 17:35:29 INFO ContextCleaner:59 - Cleaned RDD 21 -2016-06-13 17:35:30 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 17:35:30 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 17:35:30 INFO SparkContext:59 - Starting job: parquet at GaussianMixtureModel.scala:164 -2016-06-13 17:35:30 INFO DAGScheduler:59 - Got job 96 (parquet at GaussianMixtureModel.scala:164) with 1 output partitions -2016-06-13 17:35:30 INFO DAGScheduler:59 - Final stage: ResultStage 96(parquet at GaussianMixtureModel.scala:164) -2016-06-13 17:35:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:35:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:35:30 INFO DAGScheduler:59 - Submitting ResultStage 96 (MapPartitionsRDD[40] at rddToDataFrameHolder at GaussianMixtureModel.scala:164), which has no missing parents -2016-06-13 17:35:30 INFO MemoryStore:59 - ensureFreeSpace(67856) called with curMem=0, maxMem=766075207 -2016-06-13 17:35:30 INFO MemoryStore:59 - Block broadcast_140 stored as values in memory (estimated size 66.3 KB, free 730.5 MB) -2016-06-13 17:35:30 INFO MemoryStore:59 - ensureFreeSpace(23696) called with curMem=67856, maxMem=766075207 -2016-06-13 17:35:30 INFO MemoryStore:59 - Block broadcast_140_piece0 stored as bytes in memory (estimated size 23.1 KB, free 730.5 MB) -2016-06-13 17:35:30 INFO BlockManagerInfo:59 - Added broadcast_140_piece0 in memory on localhost:62031 (size: 23.1 KB, free: 730.6 MB) -2016-06-13 17:35:30 INFO SparkContext:59 - Created broadcast 140 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:35:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 96 (MapPartitionsRDD[40] at rddToDataFrameHolder at GaussianMixtureModel.scala:164) -2016-06-13 17:35:30 INFO TaskSchedulerImpl:59 - Adding task set 96.0 with 1 tasks -2016-06-13 17:35:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 96.0 (TID 320, localhost, PROCESS_LOCAL, 2652 bytes) -2016-06-13 17:35:30 INFO Executor:59 - Running task 0.0 in stage 96.0 (TID 320) -2016-06-13 17:35:30 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 17:35:30 INFO CodecConfig:151 - Compression: GZIP -2016-06-13 17:35:30 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-06-13 17:35:30 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-06-13 17:35:30 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-06-13 17:35:30 INFO ParquetOutputFormat:151 - Dictionary is on -2016-06-13 17:35:30 INFO ParquetOutputFormat:151 - Validation is off -2016-06-13 17:35:30 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-06-13 17:35:30 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-06-13 17:35:30 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 65,688 -2016-06-13 17:35:31 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 17:35:31 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 17:35:31 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 17:35:31 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 17:35:31 INFO ColumnChunkPageWriteStore:151 - written 79B for [weight] DOUBLE: 2 values, 22B raw, 38B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-06-13 17:35:31 INFO ColumnChunkPageWriteStore:151 - written 61B for [mu, type] INT32: 2 values, 8B raw, 28B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN_DICTIONARY], dic { 1 entries, 4B raw, 1B comp} -2016-06-13 17:35:31 INFO ColumnChunkPageWriteStore:151 - written 48B for [mu, size] INT32: 2 values, 7B raw, 27B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-06-13 17:35:31 INFO ColumnChunkPageWriteStore:151 - written 51B for [mu, indices, array] INT32: 2 values, 13B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-06-13 17:35:31 INFO ColumnChunkPageWriteStore:151 - written 87B for [mu, values, array] DOUBLE: 2 values, 29B raw, 46B comp, 1 pages, encodings: [PLAIN, RLE] -2016-06-13 17:35:31 INFO ColumnChunkPageWriteStore:151 - written 61B for [sigma, type] INT32: 2 values, 8B raw, 28B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN_DICTIONARY], dic { 1 entries, 4B raw, 1B comp} -2016-06-13 17:35:31 INFO ColumnChunkPageWriteStore:151 - written 61B for [sigma, numRows] INT32: 2 values, 8B raw, 28B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN_DICTIONARY], dic { 1 entries, 4B raw, 1B comp} -2016-06-13 17:35:31 INFO ColumnChunkPageWriteStore:151 - written 61B for [sigma, numCols] INT32: 2 values, 8B raw, 28B comp, 1 pages, encodings: [RLE, BIT_PACKED, PLAIN_DICTIONARY], dic { 1 entries, 4B raw, 1B comp} -2016-06-13 17:35:31 INFO ColumnChunkPageWriteStore:151 - written 51B for [sigma, colPtrs, array] INT32: 2 values, 13B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-06-13 17:35:31 INFO ColumnChunkPageWriteStore:151 - written 51B for [sigma, rowIndices, array] INT32: 2 values, 13B raw, 30B comp, 1 pages, encodings: [PLAIN, RLE] -2016-06-13 17:35:31 INFO ColumnChunkPageWriteStore:151 - written 88B for [sigma, values, array] DOUBLE: 2 values, 29B raw, 47B comp, 1 pages, encodings: [PLAIN, RLE] -2016-06-13 17:35:31 INFO ColumnChunkPageWriteStore:151 - written 54B for [sigma, isTransposed] BOOLEAN: 2 values, 7B raw, 27B comp, 1 pages, encodings: [PLAIN, RLE, BIT_PACKED] -2016-06-13 17:43:44 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 17:43:45 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 17:43:45 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 17:43:45 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 17:43:45 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 17:43:46 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 17:43:47 INFO Remoting:74 - Starting remoting -2016-06-13 17:43:47 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:62196] -2016-06-13 17:43:47 INFO Utils:59 - Successfully started service 'sparkDriver' on port 62196. -2016-06-13 17:43:47 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 17:43:47 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 17:43:47 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-9b1d645d-ec60-4a92-904f-c5ae107524d8 -2016-06-13 17:43:47 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 17:43:47 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-9fcf877e-4556-4638-b76e-2c1c0c4d9df0\httpd-594a575e-23f7-44ee-b585-840e9a700081 -2016-06-13 17:43:47 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 17:43:48 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:43:48 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:62197 -2016-06-13 17:43:48 INFO Utils:59 - Successfully started service 'HTTP file server' on port 62197. -2016-06-13 17:43:48 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 17:43:48 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:43:48 WARN AbstractLifeCycle:204 - FAILED SelectChannelConnector@0.0.0.0:4040: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.util.SparkLearningFunSuite.beforeAll(SparkLearningFunSuite.scala:23) - at org.scalatest.BeforeAndAfterAll$class.beforeAll(BeforeAndAfterAll.scala:187) - at org.apache.spark.util.SparkLearningFunSuite.beforeAll(SparkLearningFunSuite.scala:10) - at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:253) - at org.apache.spark.util.SparkLearningFunSuite.run(SparkLearningFunSuite.scala:10) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 17:43:48 WARN AbstractLifeCycle:204 - FAILED org.spark-project.jetty.server.Server@62eb3516: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.util.SparkLearningFunSuite.beforeAll(SparkLearningFunSuite.scala:23) - at org.scalatest.BeforeAndAfterAll$class.beforeAll(BeforeAndAfterAll.scala:187) - at org.apache.spark.util.SparkLearningFunSuite.beforeAll(SparkLearningFunSuite.scala:10) - at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:253) - at org.apache.spark.util.SparkLearningFunSuite.run(SparkLearningFunSuite.scala:10) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 17:43:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 17:43:48 WARN QueuedThreadPool:145 - 4 threads could not be stopped -2016-06-13 17:43:48 WARN Utils:71 - Service 'SparkUI' could not bind on port 4040. Attempting port 4041. -2016-06-13 17:43:48 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:43:48 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4041 -2016-06-13 17:43:48 INFO Utils:59 - Successfully started service 'SparkUI' on port 4041. -2016-06-13 17:43:48 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4041 -2016-06-13 17:43:49 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 17:43:49 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 17:43:49 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 62216. -2016-06-13 17:43:49 INFO NettyBlockTransferService:59 - Server created on 62216 -2016-06-13 17:43:49 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 17:43:49 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:62216 with 730.6 MB RAM, BlockManagerId(driver, localhost, 62216) -2016-06-13 17:43:49 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 17:43:51 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:43:51 INFO DAGScheduler:59 - Got job 0 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:43:51 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GaussianMixture.scala:171) -2016-06-13 17:43:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:51 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:51 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:51 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=0, maxMem=766075207 -2016-06-13 17:43:51 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:43:51 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1992, maxMem=766075207 -2016-06-13 17:43:51 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.6 MB) -2016-06-13 17:43:51 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:62216 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:43:51 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:51 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:43:51 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 17:43:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:51 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 17:43:51 INFO CacheManager:59 - Partition rdd_1_0 not found, computing it -2016-06-13 17:43:51 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=3242, maxMem=766075207 -2016-06-13 17:43:51 INFO MemoryStore:59 - Block rdd_1_0 stored as values in memory (estimated size 16.0 B, free 730.6 MB) -2016-06-13 17:43:51 INFO BlockManagerInfo:59 - Added rdd_1_0 in memory on localhost:62216 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:43:51 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1533 bytes result sent to driver -2016-06-13 17:43:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 130 ms on localhost (1/1) -2016-06-13 17:43:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 17:43:51 INFO DAGScheduler:59 - ResultStage 0 (first at GaussianMixture.scala:171) finished in 0.146 s -2016-06-13 17:43:51 INFO DAGScheduler:59 - Job 0 finished: first at GaussianMixture.scala:171, took 0.485423 s -2016-06-13 17:43:51 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:43:51 INFO DAGScheduler:59 - Got job 1 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:43:51 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GaussianMixture.scala:171) -2016-06-13 17:43:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:51 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:51 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:51 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=3258, maxMem=766075207 -2016-06-13 17:43:51 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:43:51 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=5250, maxMem=766075207 -2016-06-13 17:43:51 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.6 MB) -2016-06-13 17:43:51 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:62216 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:43:51 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:51 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:43:51 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 3 tasks -2016-06-13 17:43:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:51 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 17:43:51 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 3) -2016-06-13 17:43:51 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 17:43:51 INFO CacheManager:59 - Partition rdd_1_1 not found, computing it -2016-06-13 17:43:51 INFO CacheManager:59 - Partition rdd_1_2 not found, computing it -2016-06-13 17:43:51 INFO CacheManager:59 - Partition rdd_1_3 not found, computing it -2016-06-13 17:43:52 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=6500, maxMem=766075207 -2016-06-13 17:43:52 INFO MemoryStore:59 - Block rdd_1_1 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:43:52 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=6596, maxMem=766075207 -2016-06-13 17:43:52 INFO MemoryStore:59 - Block rdd_1_2 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:43:52 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=6692, maxMem=766075207 -2016-06-13 17:43:52 INFO MemoryStore:59 - Block rdd_1_3 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:43:52 INFO BlockManagerInfo:59 - Added rdd_1_1 in memory on localhost:62216 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:43:52 INFO BlockManagerInfo:59 - Added rdd_1_2 in memory on localhost:62216 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:43:52 INFO BlockManagerInfo:59 - Added rdd_1_3 in memory on localhost:62216 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:43:52 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 1696 bytes result sent to driver -2016-06-13 17:43:52 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 3). 1696 bytes result sent to driver -2016-06-13 17:43:52 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 1696 bytes result sent to driver -2016-06-13 17:43:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 697 ms on localhost (1/3) -2016-06-13 17:43:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 701 ms on localhost (2/3) -2016-06-13 17:43:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 3) in 714 ms on localhost (3/3) -2016-06-13 17:43:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 17:43:52 INFO DAGScheduler:59 - ResultStage 1 (first at GaussianMixture.scala:171) finished in 0.727 s -2016-06-13 17:43:52 INFO DAGScheduler:59 - Job 1 finished: first at GaussianMixture.scala:171, took 0.779722 s -2016-06-13 17:43:52 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:43:52 INFO DAGScheduler:59 - Got job 2 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:43:52 INFO DAGScheduler:59 - Final stage: ResultStage 2(takeSample at GaussianMixture.scala:184) -2016-06-13 17:43:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:52 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:52 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=6788, maxMem=766075207 -2016-06-13 17:43:52 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 17:43:52 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=8620, maxMem=766075207 -2016-06-13 17:43:52 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.6 MB) -2016-06-13 17:43:52 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:62216 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:43:52 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:52 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 2 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:43:52 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 4 tasks -2016-06-13 17:43:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:52 INFO TaskSetManager:59 - Starting task 3.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:52 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 17:43:52 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 17:43:52 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-06-13 17:43:52 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:43:52 INFO Executor:59 - Running task 3.0 in stage 2.0 (TID 7) -2016-06-13 17:43:52 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:43:52 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:43:52 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:43:52 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-06-13 17:43:52 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 6). 2082 bytes result sent to driver -2016-06-13 17:43:52 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2082 bytes result sent to driver -2016-06-13 17:43:52 INFO Executor:59 - Finished task 3.0 in stage 2.0 (TID 7). 2082 bytes result sent to driver -2016-06-13 17:43:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 6) in 38 ms on localhost (1/4) -2016-06-13 17:43:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 40 ms on localhost (2/4) -2016-06-13 17:43:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 46 ms on localhost (3/4) -2016-06-13 17:43:52 INFO TaskSetManager:59 - Finished task 3.0 in stage 2.0 (TID 7) in 39 ms on localhost (4/4) -2016-06-13 17:43:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 17:43:52 INFO DAGScheduler:59 - ResultStage 2 (takeSample at GaussianMixture.scala:184) finished in 0.048 s -2016-06-13 17:43:52 INFO DAGScheduler:59 - Job 2 finished: takeSample at GaussianMixture.scala:184, took 0.075350 s -2016-06-13 17:43:52 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:43:52 INFO DAGScheduler:59 - Got job 3 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:43:52 INFO DAGScheduler:59 - Final stage: ResultStage 3(takeSample at GaussianMixture.scala:184) -2016-06-13 17:43:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:52 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:43:52 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=9815, maxMem=766075207 -2016-06-13 17:43:52 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 18.8 KB, free 730.6 MB) -2016-06-13 17:43:52 INFO MemoryStore:59 - ensureFreeSpace(7884) called with curMem=29039, maxMem=766075207 -2016-06-13 17:43:52 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.6 MB) -2016-06-13 17:43:52 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:62216 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:43:52 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:52 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:43:52 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 4 tasks -2016-06-13 17:43:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:43:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:43:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:43:52 INFO TaskSetManager:59 - Starting task 3.0 in stage 3.0 (TID 11, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:43:52 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 8) -2016-06-13 17:43:52 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 9) -2016-06-13 17:43:52 INFO Executor:59 - Running task 3.0 in stage 3.0 (TID 11) -2016-06-13 17:43:52 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 10) -2016-06-13 17:43:52 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:43:52 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:43:52 INFO Executor:59 - Finished task 3.0 in stage 3.0 (TID 11). 2269 bytes result sent to driver -2016-06-13 17:43:52 INFO TaskSetManager:59 - Finished task 3.0 in stage 3.0 (TID 11) in 34 ms on localhost (1/4) -2016-06-13 17:43:52 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:43:52 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:43:52 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 8). 2048 bytes result sent to driver -2016-06-13 17:43:52 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 10). 2279 bytes result sent to driver -2016-06-13 17:43:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 8) in 51 ms on localhost (2/4) -2016-06-13 17:43:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 10) in 49 ms on localhost (3/4) -2016-06-13 17:43:52 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 9). 2269 bytes result sent to driver -2016-06-13 17:43:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 9) in 65 ms on localhost (4/4) -2016-06-13 17:43:52 INFO DAGScheduler:59 - ResultStage 3 (takeSample at GaussianMixture.scala:184) finished in 0.069 s -2016-06-13 17:43:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 17:43:52 INFO DAGScheduler:59 - Job 3 finished: takeSample at GaussianMixture.scala:184, took 0.094727 s -2016-06-13 17:43:52 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 17:43:52 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 17:43:52 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:62216 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:43:52 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:62216 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:43:52 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:62216 in memory (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:43:52 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:62216 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:43:53 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 17:43:53 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 17:43:53 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=304, maxMem=766075207 -2016-06-13 17:43:53 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 464.0 B, free 730.6 MB) -2016-06-13 17:43:53 INFO MemoryStore:59 - ensureFreeSpace(854) called with curMem=768, maxMem=766075207 -2016-06-13 17:43:53 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 854.0 B, free 730.6 MB) -2016-06-13 17:43:53 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:62216 (size: 854.0 B, free: 730.6 MB) -2016-06-13 17:43:53 INFO SparkContext:59 - Created broadcast 4 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:43:53 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:43:53 INFO DAGScheduler:59 - Got job 4 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:43:53 INFO DAGScheduler:59 - Final stage: ResultStage 4(aggregate at GaussianMixture.scala:201) -2016-06-13 17:43:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:53 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:53 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:53 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=1622, maxMem=766075207 -2016-06-13 17:43:53 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.8 KB, free 730.6 MB) -2016-06-13 17:43:53 INFO MemoryStore:59 - ensureFreeSpace(2148) called with curMem=5542, maxMem=766075207 -2016-06-13 17:43:53 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.6 MB) -2016-06-13 17:43:53 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:62216 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:43:53 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:53 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:43:53 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 4 tasks -2016-06-13 17:43:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 12, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:53 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 14, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:53 INFO TaskSetManager:59 - Starting task 3.0 in stage 4.0 (TID 15, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:53 INFO Executor:59 - Running task 3.0 in stage 4.0 (TID 15) -2016-06-13 17:43:53 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 12) -2016-06-13 17:43:53 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 14) -2016-06-13 17:43:53 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:43:53 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:43:53 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 13) -2016-06-13 17:43:53 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:43:53 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 12). 2708 bytes result sent to driver -2016-06-13 17:43:53 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:43:53 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 13). 2708 bytes result sent to driver -2016-06-13 17:43:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 12) in 38 ms on localhost (1/4) -2016-06-13 17:43:53 INFO Executor:59 - Finished task 3.0 in stage 4.0 (TID 15). 2708 bytes result sent to driver -2016-06-13 17:43:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 13) in 39 ms on localhost (2/4) -2016-06-13 17:43:53 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 14). 2708 bytes result sent to driver -2016-06-13 17:43:53 INFO TaskSetManager:59 - Finished task 3.0 in stage 4.0 (TID 15) in 39 ms on localhost (3/4) -2016-06-13 17:43:53 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 14) in 45 ms on localhost (4/4) -2016-06-13 17:43:53 INFO DAGScheduler:59 - ResultStage 4 (aggregate at GaussianMixture.scala:201) finished in 0.049 s -2016-06-13 17:43:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 17:43:53 INFO DAGScheduler:59 - Job 4 finished: aggregate at GaussianMixture.scala:201, took 0.077203 s -2016-06-13 17:43:53 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=7690, maxMem=766075207 -2016-06-13 17:43:53 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 464.0 B, free 730.6 MB) -2016-06-13 17:43:53 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=8154, maxMem=766075207 -2016-06-13 17:43:53 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.6 MB) -2016-06-13 17:43:53 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:62216 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:43:54 INFO SparkContext:59 - Created broadcast 6 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:43:54 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:43:54 INFO DAGScheduler:59 - Got job 5 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:43:54 INFO DAGScheduler:59 - Final stage: ResultStage 5(aggregate at GaussianMixture.scala:201) -2016-06-13 17:43:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:54 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:54 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=9007, maxMem=766075207 -2016-06-13 17:43:54 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.8 KB, free 730.6 MB) -2016-06-13 17:43:54 INFO MemoryStore:59 - ensureFreeSpace(2150) called with curMem=12927, maxMem=766075207 -2016-06-13 17:43:54 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.6 MB) -2016-06-13 17:43:54 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:62216 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:43:54 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:54 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:43:54 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 4 tasks -2016-06-13 17:43:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 16, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 17, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 18, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:54 INFO TaskSetManager:59 - Starting task 3.0 in stage 5.0 (TID 19, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:54 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 17) -2016-06-13 17:43:54 INFO Executor:59 - Running task 3.0 in stage 5.0 (TID 19) -2016-06-13 17:43:54 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 18) -2016-06-13 17:43:54 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 16) -2016-06-13 17:43:54 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:43:54 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:43:54 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 18). 2708 bytes result sent to driver -2016-06-13 17:43:54 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 17). 2708 bytes result sent to driver -2016-06-13 17:43:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 18) in 16 ms on localhost (1/4) -2016-06-13 17:43:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 17) in 18 ms on localhost (2/4) -2016-06-13 17:43:54 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:43:54 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 16). 2708 bytes result sent to driver -2016-06-13 17:43:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 16) in 25 ms on localhost (3/4) -2016-06-13 17:43:54 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:43:54 INFO Executor:59 - Finished task 3.0 in stage 5.0 (TID 19). 2708 bytes result sent to driver -2016-06-13 17:43:54 INFO TaskSetManager:59 - Finished task 3.0 in stage 5.0 (TID 19) in 29 ms on localhost (4/4) -2016-06-13 17:43:54 INFO DAGScheduler:59 - ResultStage 5 (aggregate at GaussianMixture.scala:201) finished in 0.034 s -2016-06-13 17:43:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 17:43:54 INFO DAGScheduler:59 - Job 5 finished: aggregate at GaussianMixture.scala:201, took 0.054344 s -2016-06-13 17:43:54 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=15077, maxMem=766075207 -2016-06-13 17:43:54 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 464.0 B, free 730.6 MB) -2016-06-13 17:43:54 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=15541, maxMem=766075207 -2016-06-13 17:43:54 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.6 MB) -2016-06-13 17:43:54 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:62216 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:43:54 INFO SparkContext:59 - Created broadcast 8 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:43:54 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:43:54 INFO DAGScheduler:59 - Got job 6 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:43:54 INFO DAGScheduler:59 - Final stage: ResultStage 6(aggregate at GaussianMixture.scala:201) -2016-06-13 17:43:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:54 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:54 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=16394, maxMem=766075207 -2016-06-13 17:43:54 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 3.8 KB, free 730.6 MB) -2016-06-13 17:43:54 INFO MemoryStore:59 - ensureFreeSpace(2150) called with curMem=20314, maxMem=766075207 -2016-06-13 17:43:54 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.6 MB) -2016-06-13 17:43:54 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:62216 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:43:54 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:54 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:43:54 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 4 tasks -2016-06-13 17:43:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 20, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 21, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 22, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:54 INFO TaskSetManager:59 - Starting task 3.0 in stage 6.0 (TID 23, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:54 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 20) -2016-06-13 17:43:54 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 21) -2016-06-13 17:43:54 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 22) -2016-06-13 17:43:54 INFO Executor:59 - Running task 3.0 in stage 6.0 (TID 23) -2016-06-13 17:43:54 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:43:54 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 22). 2708 bytes result sent to driver -2016-06-13 17:43:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 22) in 17 ms on localhost (1/4) -2016-06-13 17:43:54 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:43:54 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:43:54 INFO Executor:59 - Finished task 3.0 in stage 6.0 (TID 23). 2708 bytes result sent to driver -2016-06-13 17:43:54 INFO TaskSetManager:59 - Finished task 3.0 in stage 6.0 (TID 23) in 29 ms on localhost (2/4) -2016-06-13 17:43:54 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:43:54 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 20). 2708 bytes result sent to driver -2016-06-13 17:43:54 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 21). 2708 bytes result sent to driver -2016-06-13 17:43:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 21) in 49 ms on localhost (3/4) -2016-06-13 17:43:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 20) in 52 ms on localhost (4/4) -2016-06-13 17:43:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 17:43:54 INFO DAGScheduler:59 - ResultStage 6 (aggregate at GaussianMixture.scala:201) finished in 0.054 s -2016-06-13 17:43:54 INFO DAGScheduler:59 - Job 6 finished: aggregate at GaussianMixture.scala:201, took 0.076745 s -2016-06-13 17:43:54 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:43:54 INFO DAGScheduler:59 - Got job 7 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:43:54 INFO DAGScheduler:59 - Final stage: ResultStage 7(first at GaussianMixture.scala:171) -2016-06-13 17:43:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:54 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:54 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=22464, maxMem=766075207 -2016-06-13 17:43:54 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:43:54 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=24456, maxMem=766075207 -2016-06-13 17:43:54 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.6 MB) -2016-06-13 17:43:54 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:62216 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:43:54 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:54 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:43:54 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-06-13 17:43:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 24, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:54 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 24) -2016-06-13 17:43:54 INFO CacheManager:59 - Partition rdd_3_0 not found, computing it -2016-06-13 17:43:54 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=25708, maxMem=766075207 -2016-06-13 17:43:54 INFO MemoryStore:59 - Block rdd_3_0 stored as values in memory (estimated size 16.0 B, free 730.6 MB) -2016-06-13 17:43:54 INFO BlockManagerInfo:59 - Added rdd_3_0 in memory on localhost:62216 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:43:54 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 24). 1533 bytes result sent to driver -2016-06-13 17:43:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 24) in 20 ms on localhost (1/1) -2016-06-13 17:43:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 17:43:54 INFO DAGScheduler:59 - ResultStage 7 (first at GaussianMixture.scala:171) finished in 0.021 s -2016-06-13 17:43:54 INFO DAGScheduler:59 - Job 7 finished: first at GaussianMixture.scala:171, took 0.042640 s -2016-06-13 17:43:54 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:43:54 INFO DAGScheduler:59 - Got job 8 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:43:54 INFO DAGScheduler:59 - Final stage: ResultStage 8(first at GaussianMixture.scala:171) -2016-06-13 17:43:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:54 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:54 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=25724, maxMem=766075207 -2016-06-13 17:43:54 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:43:54 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=27716, maxMem=766075207 -2016-06-13 17:43:54 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.6 MB) -2016-06-13 17:43:54 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:62216 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:43:54 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:54 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 8 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:43:54 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 3 tasks -2016-06-13 17:43:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 25, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 26, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 8.0 (TID 27, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:54 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 25) -2016-06-13 17:43:54 INFO Executor:59 - Running task 2.0 in stage 8.0 (TID 27) -2016-06-13 17:43:54 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 26) -2016-06-13 17:43:54 INFO CacheManager:59 - Partition rdd_3_1 not found, computing it -2016-06-13 17:43:54 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=28968, maxMem=766075207 -2016-06-13 17:43:54 INFO MemoryStore:59 - Block rdd_3_1 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:43:54 INFO CacheManager:59 - Partition rdd_3_3 not found, computing it -2016-06-13 17:43:54 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=29064, maxMem=766075207 -2016-06-13 17:43:54 INFO MemoryStore:59 - Block rdd_3_3 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:43:54 INFO BlockManagerInfo:59 - Added rdd_3_3 in memory on localhost:62216 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:43:54 INFO Executor:59 - Finished task 2.0 in stage 8.0 (TID 27). 1696 bytes result sent to driver -2016-06-13 17:43:54 INFO CacheManager:59 - Partition rdd_3_2 not found, computing it -2016-06-13 17:43:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 8.0 (TID 27) in 22 ms on localhost (1/3) -2016-06-13 17:43:54 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=29160, maxMem=766075207 -2016-06-13 17:43:54 INFO MemoryStore:59 - Block rdd_3_2 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:43:54 INFO BlockManagerInfo:59 - Added rdd_3_1 in memory on localhost:62216 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:43:54 INFO BlockManagerInfo:59 - Added rdd_3_2 in memory on localhost:62216 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:43:54 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 26). 1696 bytes result sent to driver -2016-06-13 17:43:54 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 25). 1696 bytes result sent to driver -2016-06-13 17:43:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 26) in 50 ms on localhost (2/3) -2016-06-13 17:43:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 25) in 54 ms on localhost (3/3) -2016-06-13 17:43:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 17:43:54 INFO DAGScheduler:59 - ResultStage 8 (first at GaussianMixture.scala:171) finished in 0.124 s -2016-06-13 17:43:54 INFO DAGScheduler:59 - Job 8 finished: first at GaussianMixture.scala:171, took 0.149304 s -2016-06-13 17:43:54 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:43:54 INFO DAGScheduler:59 - Got job 9 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:43:54 INFO DAGScheduler:59 - Final stage: ResultStage 9(takeSample at GaussianMixture.scala:184) -2016-06-13 17:43:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:54 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:54 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=29256, maxMem=766075207 -2016-06-13 17:43:54 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 17:43:54 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=31088, maxMem=766075207 -2016-06-13 17:43:54 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.6 MB) -2016-06-13 17:43:54 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:62216 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:43:54 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:54 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 9 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:43:54 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 4 tasks -2016-06-13 17:43:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 28, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 29, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 9.0 (TID 30, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:54 INFO TaskSetManager:59 - Starting task 3.0 in stage 9.0 (TID 31, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:54 INFO Executor:59 - Running task 2.0 in stage 9.0 (TID 30) -2016-06-13 17:43:54 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:43:54 INFO Executor:59 - Finished task 2.0 in stage 9.0 (TID 30). 2082 bytes result sent to driver -2016-06-13 17:43:54 INFO Executor:59 - Running task 3.0 in stage 9.0 (TID 31) -2016-06-13 17:43:54 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:43:54 INFO Executor:59 - Finished task 3.0 in stage 9.0 (TID 31). 2082 bytes result sent to driver -2016-06-13 17:43:54 INFO TaskSetManager:59 - Finished task 3.0 in stage 9.0 (TID 31) in 65 ms on localhost (1/4) -2016-06-13 17:43:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 9.0 (TID 30) in 79 ms on localhost (2/4) -2016-06-13 17:43:54 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 28) -2016-06-13 17:43:54 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 29) -2016-06-13 17:43:54 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:43:54 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 29). 2082 bytes result sent to driver -2016-06-13 17:43:54 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:43:54 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 28). 2082 bytes result sent to driver -2016-06-13 17:43:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 29) in 201 ms on localhost (3/4) -2016-06-13 17:43:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 28) in 221 ms on localhost (4/4) -2016-06-13 17:43:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 17:43:54 INFO DAGScheduler:59 - ResultStage 9 (takeSample at GaussianMixture.scala:184) finished in 0.314 s -2016-06-13 17:43:55 INFO DAGScheduler:59 - Job 9 finished: takeSample at GaussianMixture.scala:184, took 0.605476 s -2016-06-13 17:43:55 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:43:55 INFO DAGScheduler:59 - Got job 10 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:43:55 INFO DAGScheduler:59 - Final stage: ResultStage 10(takeSample at GaussianMixture.scala:184) -2016-06-13 17:43:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:55 INFO DAGScheduler:59 - Submitting ResultStage 10 (PartitionwiseSampledRDD[4] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=32283, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(7878) called with curMem=51507, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:62216 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:43:55 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 10 (PartitionwiseSampledRDD[4] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:43:55 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 4 tasks -2016-06-13 17:43:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 32, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 33, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 10.0 (TID 34, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 10.0 (TID 35, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:43:55 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 32) -2016-06-13 17:43:55 INFO Executor:59 - Running task 3.0 in stage 10.0 (TID 35) -2016-06-13 17:43:55 INFO Executor:59 - Running task 2.0 in stage 10.0 (TID 34) -2016-06-13 17:43:55 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 33) -2016-06-13 17:43:55 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:43:55 INFO Executor:59 - Finished task 3.0 in stage 10.0 (TID 35). 2274 bytes result sent to driver -2016-06-13 17:43:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:43:55 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 33). 2274 bytes result sent to driver -2016-06-13 17:43:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:43:55 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 32). 2048 bytes result sent to driver -2016-06-13 17:43:55 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:43:55 INFO Executor:59 - Finished task 2.0 in stage 10.0 (TID 34). 2244 bytes result sent to driver -2016-06-13 17:43:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 10.0 (TID 35) in 64 ms on localhost (1/4) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 33) in 72 ms on localhost (2/4) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 32) in 75 ms on localhost (3/4) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 10.0 (TID 34) in 73 ms on localhost (4/4) -2016-06-13 17:43:55 INFO DAGScheduler:59 - ResultStage 10 (takeSample at GaussianMixture.scala:184) finished in 0.107 s -2016-06-13 17:43:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 17:43:55 INFO DAGScheduler:59 - Job 10 finished: takeSample at GaussianMixture.scala:184, took 0.195737 s -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=59385, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(850) called with curMem=59849, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 850.0 B, free 730.5 MB) -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:62216 (size: 850.0 B, free: 730.6 MB) -2016-06-13 17:43:55 INFO SparkContext:59 - Created broadcast 14 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:43:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:43:55 INFO DAGScheduler:59 - Got job 11 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:43:55 INFO DAGScheduler:59 - Final stage: ResultStage 11(aggregate at GaussianMixture.scala:201) -2016-06-13 17:43:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:55 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=60699, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(2147) called with curMem=64619, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:62216 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:43:55 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 11 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:43:55 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 4 tasks -2016-06-13 17:43:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 36, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 37, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 11.0 (TID 38, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 11.0 (TID 39, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:55 INFO Executor:59 - Running task 3.0 in stage 11.0 (TID 39) -2016-06-13 17:43:55 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 37) -2016-06-13 17:43:55 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 36) -2016-06-13 17:43:55 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:43:55 INFO Executor:59 - Finished task 3.0 in stage 11.0 (TID 39). 2708 bytes result sent to driver -2016-06-13 17:43:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:43:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:43:55 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 36). 2708 bytes result sent to driver -2016-06-13 17:43:55 INFO Executor:59 - Running task 2.0 in stage 11.0 (TID 38) -2016-06-13 17:43:55 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:43:55 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 37). 2708 bytes result sent to driver -2016-06-13 17:43:55 INFO Executor:59 - Finished task 2.0 in stage 11.0 (TID 38). 2708 bytes result sent to driver -2016-06-13 17:43:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 11.0 (TID 39) in 28 ms on localhost (1/4) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 36) in 34 ms on localhost (2/4) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 37) in 35 ms on localhost (3/4) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 11.0 (TID 38) in 37 ms on localhost (4/4) -2016-06-13 17:43:55 INFO DAGScheduler:59 - ResultStage 11 (aggregate at GaussianMixture.scala:201) finished in 0.041 s -2016-06-13 17:43:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 17:43:55 INFO DAGScheduler:59 - Job 11 finished: aggregate at GaussianMixture.scala:201, took 0.076671 s -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=66766, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=67230, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:62216 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:43:55 INFO SparkContext:59 - Created broadcast 16 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:43:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:43:55 INFO DAGScheduler:59 - Got job 12 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:43:55 INFO DAGScheduler:59 - Final stage: ResultStage 12(aggregate at GaussianMixture.scala:201) -2016-06-13 17:43:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:55 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=68083, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=72003, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:62216 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:43:55 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 12 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:43:55 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 4 tasks -2016-06-13 17:43:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 40, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 41, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 12.0 (TID 42, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 12.0 (TID 43, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:55 INFO Executor:59 - Running task 3.0 in stage 12.0 (TID 43) -2016-06-13 17:43:55 INFO Executor:59 - Running task 2.0 in stage 12.0 (TID 42) -2016-06-13 17:43:55 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:43:55 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:43:55 INFO Executor:59 - Finished task 3.0 in stage 12.0 (TID 43). 2708 bytes result sent to driver -2016-06-13 17:43:55 INFO Executor:59 - Finished task 2.0 in stage 12.0 (TID 42). 2708 bytes result sent to driver -2016-06-13 17:43:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 12.0 (TID 43) in 11 ms on localhost (1/4) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 12.0 (TID 42) in 15 ms on localhost (2/4) -2016-06-13 17:43:55 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 40) -2016-06-13 17:43:55 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 41) -2016-06-13 17:43:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:43:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:43:55 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 40). 2708 bytes result sent to driver -2016-06-13 17:43:55 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 41). 2708 bytes result sent to driver -2016-06-13 17:43:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 40) in 32 ms on localhost (3/4) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 41) in 35 ms on localhost (4/4) -2016-06-13 17:43:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 17:43:55 INFO DAGScheduler:59 - ResultStage 12 (aggregate at GaussianMixture.scala:201) finished in 0.037 s -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:62216 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:43:55 INFO DAGScheduler:59 - Job 12 finished: aggregate at GaussianMixture.scala:201, took 0.267930 s -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=68088, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=68552, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:62216 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:43:55 INFO SparkContext:59 - Created broadcast 18 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:43:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:43:55 INFO DAGScheduler:59 - Got job 13 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:43:55 INFO DAGScheduler:59 - Final stage: ResultStage 13(aggregate at GaussianMixture.scala:201) -2016-06-13 17:43:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:55 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 17:43:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:55 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:62216 in memory (size: 850.0 B, free: 730.6 MB) -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=68091, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=72011, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:62216 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:43:55 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 13 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:43:55 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 4 tasks -2016-06-13 17:43:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 44, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:62216 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 45, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 13.0 (TID 46, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 13.0 (TID 47, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:55 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 44) -2016-06-13 17:43:55 INFO Executor:59 - Running task 2.0 in stage 13.0 (TID 46) -2016-06-13 17:43:55 INFO Executor:59 - Running task 3.0 in stage 13.0 (TID 47) -2016-06-13 17:43:55 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:43:55 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:43:55 INFO Executor:59 - Finished task 3.0 in stage 13.0 (TID 47). 2708 bytes result sent to driver -2016-06-13 17:43:55 INFO Executor:59 - Finished task 2.0 in stage 13.0 (TID 46). 2708 bytes result sent to driver -2016-06-13 17:43:55 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 17:43:55 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:43:55 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 44). 2708 bytes result sent to driver -2016-06-13 17:43:55 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 45) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 13.0 (TID 47) in 15 ms on localhost (1/4) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 13.0 (TID 46) in 18 ms on localhost (2/4) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 44) in 24 ms on localhost (3/4) -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:62216 in memory (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:43:55 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:43:55 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 45). 2708 bytes result sent to driver -2016-06-13 17:43:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 45) in 38 ms on localhost (4/4) -2016-06-13 17:43:55 INFO DAGScheduler:59 - ResultStage 13 (aggregate at GaussianMixture.scala:201) finished in 0.041 s -2016-06-13 17:43:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 17:43:55 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:62216 in memory (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:43:55 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 17:43:55 INFO DAGScheduler:59 - Job 13 finished: aggregate at GaussianMixture.scala:201, took 0.082925 s -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:62216 in memory (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:43:55 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 17:43:55 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:43:55 INFO DAGScheduler:59 - Got job 14 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:43:55 INFO DAGScheduler:59 - Final stage: ResultStage 14(first at GaussianMixture.scala:171) -2016-06-13 17:43:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:62216 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:43:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:55 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=31476, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(1251) called with curMem=33468, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 1251.0 B, free 730.6 MB) -2016-06-13 17:43:55 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:62216 (size: 1251.0 B, free: 730.6 MB) -2016-06-13 17:43:55 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:55 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 14 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:43:55 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 1 tasks -2016-06-13 17:43:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 48, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:62216 in memory (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:43:55 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 48) -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:62216 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:43:55 INFO CacheManager:59 - Partition rdd_5_0 not found, computing it -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=27332, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block rdd_5_0 stored as values in memory (estimated size 16.0 B, free 730.6 MB) -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Added rdd_5_0 in memory on localhost:62216 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:43:55 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 48). 1533 bytes result sent to driver -2016-06-13 17:43:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 48) in 46 ms on localhost (1/1) -2016-06-13 17:43:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 17:43:55 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 17:43:55 INFO DAGScheduler:59 - ResultStage 14 (first at GaussianMixture.scala:171) finished in 0.055 s -2016-06-13 17:43:55 INFO DAGScheduler:59 - Job 14 finished: first at GaussianMixture.scala:171, took 0.075620 s -2016-06-13 17:43:55 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:43:55 INFO DAGScheduler:59 - Got job 15 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:43:55 INFO DAGScheduler:59 - Final stage: ResultStage 15(first at GaussianMixture.scala:171) -2016-06-13 17:43:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:62216 in memory (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:43:55 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=26031, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(1251) called with curMem=28023, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1251.0 B, free 730.6 MB) -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:62216 (size: 1251.0 B, free: 730.6 MB) -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:62216 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:43:55 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:55 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 15 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:43:55 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 3 tasks -2016-06-13 17:43:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 49, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 50, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 15.0 (TID 51, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:55 INFO Executor:59 - Running task 2.0 in stage 15.0 (TID 51) -2016-06-13 17:43:55 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 50) -2016-06-13 17:43:55 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:62216 in memory (size: 854.0 B, free: 730.6 MB) -2016-06-13 17:43:55 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 49) -2016-06-13 17:43:55 INFO CacheManager:59 - Partition rdd_5_2 not found, computing it -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=21888, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block rdd_5_2 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:43:55 INFO CacheManager:59 - Partition rdd_5_1 not found, computing it -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=21984, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block rdd_5_1 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Added rdd_5_2 in memory on localhost:62216 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Added rdd_5_1 in memory on localhost:62216 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:43:55 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 50). 1696 bytes result sent to driver -2016-06-13 17:43:55 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 49). 1696 bytes result sent to driver -2016-06-13 17:43:55 INFO CacheManager:59 - Partition rdd_5_3 not found, computing it -2016-06-13 17:43:55 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=22080, maxMem=766075207 -2016-06-13 17:43:55 INFO MemoryStore:59 - Block rdd_5_3 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:43:55 INFO BlockManagerInfo:59 - Added rdd_5_3 in memory on localhost:62216 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:43:55 INFO Executor:59 - Finished task 2.0 in stage 15.0 (TID 51). 1696 bytes result sent to driver -2016-06-13 17:43:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 50) in 36 ms on localhost (1/3) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 49) in 39 ms on localhost (2/3) -2016-06-13 17:43:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 15.0 (TID 51) in 39 ms on localhost (3/3) -2016-06-13 17:43:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 17:43:55 INFO DAGScheduler:59 - ResultStage 15 (first at GaussianMixture.scala:171) finished in 0.044 s -2016-06-13 17:43:55 INFO DAGScheduler:59 - Job 15 finished: first at GaussianMixture.scala:171, took 0.066983 s -2016-06-13 17:43:56 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Got job 16 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:43:56 INFO DAGScheduler:59 - Final stage: ResultStage 16(takeSample at GaussianMixture.scala:184) -2016-06-13 17:43:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=22176, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(1194) called with curMem=24008, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 1194.0 B, free 730.6 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:62216 (size: 1194.0 B, free: 730.6 MB) -2016-06-13 17:43:56 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 16 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 4 tasks -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 52, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 53, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 16.0 (TID 54, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 3.0 in stage 16.0 (TID 55, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 53) -2016-06-13 17:43:56 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 52) -2016-06-13 17:43:56 INFO Executor:59 - Running task 3.0 in stage 16.0 (TID 55) -2016-06-13 17:43:56 INFO Executor:59 - Running task 2.0 in stage 16.0 (TID 54) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 3.0 in stage 16.0 (TID 55). 2082 bytes result sent to driver -2016-06-13 17:43:56 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 52). 2082 bytes result sent to driver -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 2.0 in stage 16.0 (TID 54). 2082 bytes result sent to driver -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 53). 2082 bytes result sent to driver -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 16.0 (TID 55) in 28 ms on localhost (1/4) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 52) in 34 ms on localhost (2/4) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 16.0 (TID 54) in 33 ms on localhost (3/4) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 53) in 38 ms on localhost (4/4) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 17:43:56 INFO DAGScheduler:59 - ResultStage 16 (takeSample at GaussianMixture.scala:184) finished in 0.040 s -2016-06-13 17:43:56 INFO DAGScheduler:59 - Job 16 finished: takeSample at GaussianMixture.scala:184, took 0.064922 s -2016-06-13 17:43:56 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Got job 17 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:43:56 INFO DAGScheduler:59 - Final stage: ResultStage 17(takeSample at GaussianMixture.scala:184) -2016-06-13 17:43:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting ResultStage 17 (PartitionwiseSampledRDD[6] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=25202, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(7884) called with curMem=44426, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:62216 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:43:56 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (PartitionwiseSampledRDD[6] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 56, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 57, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 58, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 59, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:43:56 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 58) -2016-06-13 17:43:56 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 59) -2016-06-13 17:43:56 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 56) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 58). 2259 bytes result sent to driver -2016-06-13 17:43:56 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 59). 2226 bytes result sent to driver -2016-06-13 17:43:56 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 57) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 56). 2048 bytes result sent to driver -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 59) in 14 ms on localhost (1/4) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 56) in 19 ms on localhost (2/4) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 58) in 17 ms on localhost (3/4) -2016-06-13 17:43:56 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 57). 2269 bytes result sent to driver -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 57) in 26 ms on localhost (4/4) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 17:43:56 INFO DAGScheduler:59 - ResultStage 17 (takeSample at GaussianMixture.scala:184) finished in 0.030 s -2016-06-13 17:43:56 INFO DAGScheduler:59 - Job 17 finished: takeSample at GaussianMixture.scala:184, took 0.064811 s -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=52310, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(850) called with curMem=52774, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 850.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:62216 (size: 850.0 B, free: 730.6 MB) -2016-06-13 17:43:56 INFO SparkContext:59 - Created broadcast 24 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:43:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Got job 18 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:43:56 INFO DAGScheduler:59 - Final stage: ResultStage 18(aggregate at GaussianMixture.scala:201) -2016-06-13 17:43:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=53624, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=57544, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:62216 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:43:56 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 60, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 61, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 62, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 63, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 63) -2016-06-13 17:43:56 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 62) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 62). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 60) -2016-06-13 17:43:56 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 63). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 62) in 12 ms on localhost (1/4) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 63) in 12 ms on localhost (2/4) -2016-06-13 17:43:56 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 61) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 60). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 61). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 60) in 28 ms on localhost (3/4) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 61) in 30 ms on localhost (4/4) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 17:43:56 INFO DAGScheduler:59 - ResultStage 18 (aggregate at GaussianMixture.scala:201) finished in 0.033 s -2016-06-13 17:43:56 INFO DAGScheduler:59 - Job 18 finished: aggregate at GaussianMixture.scala:201, took 0.054185 s -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=59689, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=60153, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:62216 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:43:56 INFO SparkContext:59 - Created broadcast 26 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:43:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Got job 19 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:43:56 INFO DAGScheduler:59 - Final stage: ResultStage 19(aggregate at GaussianMixture.scala:201) -2016-06-13 17:43:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=61006, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(2151) called with curMem=64926, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:62216 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:43:56 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 19 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 4 tasks -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 64, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 65, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 19.0 (TID 66, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 3.0 in stage 19.0 (TID 67, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO Executor:59 - Running task 3.0 in stage 19.0 (TID 67) -2016-06-13 17:43:56 INFO Executor:59 - Running task 2.0 in stage 19.0 (TID 66) -2016-06-13 17:43:56 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 64) -2016-06-13 17:43:56 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 65) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 2.0 in stage 19.0 (TID 66). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 19.0 (TID 66) in 10 ms on localhost (1/4) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 3.0 in stage 19.0 (TID 67). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 19.0 (TID 67) in 15 ms on localhost (2/4) -2016-06-13 17:43:56 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 64). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 65). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 65) in 30 ms on localhost (3/4) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 64) in 33 ms on localhost (4/4) -2016-06-13 17:43:56 INFO DAGScheduler:59 - ResultStage 19 (aggregate at GaussianMixture.scala:201) finished in 0.034 s -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 17:43:56 INFO DAGScheduler:59 - Job 19 finished: aggregate at GaussianMixture.scala:201, took 0.051016 s -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=67077, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=67541, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:62216 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:43:56 INFO SparkContext:59 - Created broadcast 28 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:43:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Got job 20 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:43:56 INFO DAGScheduler:59 - Final stage: ResultStage 20(aggregate at GaussianMixture.scala:201) -2016-06-13 17:43:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=68394, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(2151) called with curMem=72314, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:62216 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:43:56 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 68, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 69, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 70, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 71, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 70) -2016-06-13 17:43:56 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 68) -2016-06-13 17:43:56 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 71) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 70). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 70) in 10 ms on localhost (1/4) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 71). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 71) in 15 ms on localhost (2/4) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 68). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 69) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 69). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 69) in 30 ms on localhost (3/4) -2016-06-13 17:43:56 INFO DAGScheduler:59 - ResultStage 20 (aggregate at GaussianMixture.scala:201) finished in 0.048 s -2016-06-13 17:43:56 INFO DAGScheduler:59 - Job 20 finished: aggregate at GaussianMixture.scala:201, took 0.086600 s -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 68) in 47 ms on localhost (4/4) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 17:43:56 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Got job 21 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:43:56 INFO DAGScheduler:59 - Final stage: ResultStage 21(first at GaussianMixture.scala:171) -2016-06-13 17:43:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=74465, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=76457, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:62216 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:43:56 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 21 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 1 tasks -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 72, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:56 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 72) -2016-06-13 17:43:56 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=77709, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 16.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:62216 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:43:56 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 72). 1533 bytes result sent to driver -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 72) in 11 ms on localhost (1/1) -2016-06-13 17:43:56 INFO DAGScheduler:59 - ResultStage 21 (first at GaussianMixture.scala:171) finished in 0.011 s -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 17:43:56 INFO DAGScheduler:59 - Job 21 finished: first at GaussianMixture.scala:171, took 0.027124 s -2016-06-13 17:43:56 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Got job 22 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:43:56 INFO DAGScheduler:59 - Final stage: ResultStage 22(first at GaussianMixture.scala:171) -2016-06-13 17:43:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=77725, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=79717, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:62216 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:43:56 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 22 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 3 tasks -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 73, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 74, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 75, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 73) -2016-06-13 17:43:56 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 74) -2016-06-13 17:43:56 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 75) -2016-06-13 17:43:56 INFO CacheManager:59 - Partition rdd_7_3 not found, computing it -2016-06-13 17:43:56 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=80969, maxMem=766075207 -2016-06-13 17:43:56 INFO CacheManager:59 - Partition rdd_7_2 not found, computing it -2016-06-13 17:43:56 INFO MemoryStore:59 - Block rdd_7_3 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added rdd_7_3 in memory on localhost:62216 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=81065, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block rdd_7_2 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 75). 1696 bytes result sent to driver -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=81161, maxMem=766075207 -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added rdd_7_2 in memory on localhost:62216 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:43:56 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:62216 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:43:56 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 74). 1696 bytes result sent to driver -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 75) in 14 ms on localhost (1/3) -2016-06-13 17:43:56 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 73). 1696 bytes result sent to driver -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 74) in 17 ms on localhost (2/3) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 73) in 19 ms on localhost (3/3) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 17:43:56 INFO DAGScheduler:59 - ResultStage 22 (first at GaussianMixture.scala:171) finished in 0.022 s -2016-06-13 17:43:56 INFO DAGScheduler:59 - Job 22 finished: first at GaussianMixture.scala:171, took 0.044593 s -2016-06-13 17:43:56 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Got job 23 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:43:56 INFO DAGScheduler:59 - Final stage: ResultStage 23(takeSample at GaussianMixture.scala:184) -2016-06-13 17:43:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=81257, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 1832.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=83089, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:62216 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:43:56 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 23 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 76, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 77, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 78, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 79, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 78) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 78). 2082 bytes result sent to driver -2016-06-13 17:43:56 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 79) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 78) in 8 ms on localhost (1/4) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:43:56 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 76) -2016-06-13 17:43:56 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 79). 2082 bytes result sent to driver -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 79) in 13 ms on localhost (2/4) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 76). 2082 bytes result sent to driver -2016-06-13 17:43:56 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 77) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 77). 2082 bytes result sent to driver -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 76) in 34 ms on localhost (3/4) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 77) in 34 ms on localhost (4/4) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 17:43:56 INFO DAGScheduler:59 - ResultStage 23 (takeSample at GaussianMixture.scala:184) finished in 0.037 s -2016-06-13 17:43:56 INFO DAGScheduler:59 - Job 23 finished: takeSample at GaussianMixture.scala:184, took 0.053328 s -2016-06-13 17:43:56 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Got job 24 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:43:56 INFO DAGScheduler:59 - Final stage: ResultStage 24(takeSample at GaussianMixture.scala:184) -2016-06-13 17:43:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting ResultStage 24 (PartitionwiseSampledRDD[8] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=84284, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(7885) called with curMem=103508, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:62216 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:43:56 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 24 (PartitionwiseSampledRDD[8] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 4 tasks -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 80, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 81, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 82, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 3.0 in stage 24.0 (TID 83, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:43:56 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 80) -2016-06-13 17:43:56 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 81) -2016-06-13 17:43:56 INFO Executor:59 - Running task 3.0 in stage 24.0 (TID 83) -2016-06-13 17:43:56 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 82) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 82). 2221 bytes result sent to driver -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 82) in 11 ms on localhost (1/4) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 80). 2048 bytes result sent to driver -2016-06-13 17:43:56 INFO Executor:59 - Finished task 3.0 in stage 24.0 (TID 83). 2269 bytes result sent to driver -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 81). 2269 bytes result sent to driver -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 24.0 (TID 83) in 18 ms on localhost (2/4) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 81) in 21 ms on localhost (3/4) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 80) in 23 ms on localhost (4/4) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-06-13 17:43:56 INFO DAGScheduler:59 - ResultStage 24 (takeSample at GaussianMixture.scala:184) finished in 0.025 s -2016-06-13 17:43:56 INFO DAGScheduler:59 - Job 24 finished: takeSample at GaussianMixture.scala:184, took 0.084780 s -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=111393, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(851) called with curMem=111857, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 851.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:62216 (size: 851.0 B, free: 730.5 MB) -2016-06-13 17:43:56 INFO SparkContext:59 - Created broadcast 34 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:43:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Got job 25 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:43:56 INFO DAGScheduler:59 - Final stage: ResultStage 25(aggregate at GaussianMixture.scala:201) -2016-06-13 17:43:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=112708, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=116628, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:62216 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:43:56 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 25 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 4 tasks -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 84, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 85, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 25.0 (TID 86, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 3.0 in stage 25.0 (TID 87, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 85) -2016-06-13 17:43:56 INFO Executor:59 - Running task 3.0 in stage 25.0 (TID 87) -2016-06-13 17:43:56 INFO Executor:59 - Running task 2.0 in stage 25.0 (TID 86) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 3.0 in stage 25.0 (TID 87). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 84) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 2.0 in stage 25.0 (TID 86). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 84). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 25.0 (TID 87) in 25 ms on localhost (1/4) -2016-06-13 17:43:56 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 85). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 84) in 31 ms on localhost (2/4) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 25.0 (TID 86) in 30 ms on localhost (3/4) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 85) in 31 ms on localhost (4/4) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-06-13 17:43:56 INFO DAGScheduler:59 - ResultStage 25 (aggregate at GaussianMixture.scala:201) finished in 0.047 s -2016-06-13 17:43:56 INFO DAGScheduler:59 - Job 25 finished: aggregate at GaussianMixture.scala:201, took 0.062805 s -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=118773, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=119237, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:62216 (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:43:56 INFO SparkContext:59 - Created broadcast 36 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:43:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Got job 26 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:43:56 INFO DAGScheduler:59 - Final stage: ResultStage 26(aggregate at GaussianMixture.scala:201) -2016-06-13 17:43:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=120090, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=124010, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:62216 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:43:56 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 26 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 88, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 89, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 90, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 91, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 91) -2016-06-13 17:43:56 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 89) -2016-06-13 17:43:56 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 88) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 89). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 91). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 88). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 90) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 90). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 89) in 26 ms on localhost (1/4) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 91) in 26 ms on localhost (2/4) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 88) in 32 ms on localhost (3/4) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 90) in 31 ms on localhost (4/4) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-06-13 17:43:56 INFO DAGScheduler:59 - ResultStage 26 (aggregate at GaussianMixture.scala:201) finished in 0.036 s -2016-06-13 17:43:56 INFO DAGScheduler:59 - Job 26 finished: aggregate at GaussianMixture.scala:201, took 0.131797 s -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=126162, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=126626, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:62216 (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:43:56 INFO SparkContext:59 - Created broadcast 38 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:43:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Got job 27 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:43:56 INFO DAGScheduler:59 - Final stage: ResultStage 27(aggregate at GaussianMixture.scala:201) -2016-06-13 17:43:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=127479, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:43:56 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=131399, maxMem=766075207 -2016-06-13 17:43:56 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:43:56 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:62216 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:43:56 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:56 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 27 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 4 tasks -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 92, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 93, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 27.0 (TID 94, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Starting task 3.0 in stage 27.0 (TID 95, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:56 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 93) -2016-06-13 17:43:56 INFO Executor:59 - Running task 2.0 in stage 27.0 (TID 94) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 2.0 in stage 27.0 (TID 94). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 93). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 92) -2016-06-13 17:43:56 INFO Executor:59 - Running task 3.0 in stage 27.0 (TID 95) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 3.0 in stage 27.0 (TID 95). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 27.0 (TID 94) in 20 ms on localhost (1/4) -2016-06-13 17:43:56 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:43:56 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 92). 2708 bytes result sent to driver -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 93) in 23 ms on localhost (2/4) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 27.0 (TID 95) in 22 ms on localhost (3/4) -2016-06-13 17:43:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 92) in 26 ms on localhost (4/4) -2016-06-13 17:43:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-06-13 17:43:56 INFO DAGScheduler:59 - ResultStage 27 (aggregate at GaussianMixture.scala:201) finished in 0.027 s -2016-06-13 17:43:56 INFO DAGScheduler:59 - Job 27 finished: aggregate at GaussianMixture.scala:201, took 0.046407 s -2016-06-13 17:43:56 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:43:57 INFO DAGScheduler:59 - Got job 28 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:43:57 INFO DAGScheduler:59 - Final stage: ResultStage 28(first at GaussianMixture.scala:171) -2016-06-13 17:43:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:57 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=133551, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=135543, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.5 MB) -2016-06-13 17:43:57 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:62216 (size: 1252.0 B, free: 730.5 MB) -2016-06-13 17:43:57 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:57 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 28 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:43:57 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 1 tasks -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 96, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:57 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 96) -2016-06-13 17:43:57 INFO CacheManager:59 - Partition rdd_9_0 not found, computing it -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=136795, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block rdd_9_0 stored as values in memory (estimated size 16.0 B, free 730.5 MB) -2016-06-13 17:43:57 INFO BlockManagerInfo:59 - Added rdd_9_0 in memory on localhost:62216 (size: 16.0 B, free: 730.5 MB) -2016-06-13 17:43:57 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 96). 1533 bytes result sent to driver -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 96) in 35 ms on localhost (1/1) -2016-06-13 17:43:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-06-13 17:43:57 INFO DAGScheduler:59 - ResultStage 28 (first at GaussianMixture.scala:171) finished in 0.037 s -2016-06-13 17:43:57 INFO DAGScheduler:59 - Job 28 finished: first at GaussianMixture.scala:171, took 0.078705 s -2016-06-13 17:43:57 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:43:57 INFO DAGScheduler:59 - Got job 29 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:43:57 INFO DAGScheduler:59 - Final stage: ResultStage 29(first at GaussianMixture.scala:171) -2016-06-13 17:43:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:57 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=136811, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=138803, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.5 MB) -2016-06-13 17:43:57 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:62216 (size: 1252.0 B, free: 730.5 MB) -2016-06-13 17:43:57 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:57 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 29 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:43:57 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 3 tasks -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 97, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 98, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 2.0 in stage 29.0 (TID 99, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:57 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 97) -2016-06-13 17:43:57 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 98) -2016-06-13 17:43:57 INFO Executor:59 - Running task 2.0 in stage 29.0 (TID 99) -2016-06-13 17:43:57 INFO CacheManager:59 - Partition rdd_9_3 not found, computing it -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=140055, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block rdd_9_3 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:43:57 INFO CacheManager:59 - Partition rdd_9_2 not found, computing it -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=140151, maxMem=766075207 -2016-06-13 17:43:57 INFO BlockManagerInfo:59 - Added rdd_9_3 in memory on localhost:62216 (size: 96.0 B, free: 730.5 MB) -2016-06-13 17:43:57 INFO MemoryStore:59 - Block rdd_9_2 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:43:57 INFO CacheManager:59 - Partition rdd_9_1 not found, computing it -2016-06-13 17:43:57 INFO BlockManagerInfo:59 - Added rdd_9_2 in memory on localhost:62216 (size: 96.0 B, free: 730.5 MB) -2016-06-13 17:43:57 INFO Executor:59 - Finished task 2.0 in stage 29.0 (TID 99). 1696 bytes result sent to driver -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=140247, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block rdd_9_1 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 2.0 in stage 29.0 (TID 99) in 13 ms on localhost (1/3) -2016-06-13 17:43:57 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 98). 1696 bytes result sent to driver -2016-06-13 17:43:57 INFO BlockManagerInfo:59 - Added rdd_9_1 in memory on localhost:62216 (size: 96.0 B, free: 730.5 MB) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 98) in 16 ms on localhost (2/3) -2016-06-13 17:43:57 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 97). 1696 bytes result sent to driver -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 97) in 19 ms on localhost (3/3) -2016-06-13 17:43:57 INFO DAGScheduler:59 - ResultStage 29 (first at GaussianMixture.scala:171) finished in 0.019 s -2016-06-13 17:43:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 17:43:57 INFO DAGScheduler:59 - Job 29 finished: first at GaussianMixture.scala:171, took 0.042223 s -2016-06-13 17:43:57 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:43:57 INFO DAGScheduler:59 - Got job 30 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:43:57 INFO DAGScheduler:59 - Final stage: ResultStage 30(takeSample at GaussianMixture.scala:184) -2016-06-13 17:43:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:57 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=140343, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 1832.0 B, free 730.5 MB) -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=142175, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.4 MB) -2016-06-13 17:43:57 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:62216 (size: 1195.0 B, free: 730.5 MB) -2016-06-13 17:43:57 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:57 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:43:57 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 100, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 101, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 102, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 103, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:57 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 100) -2016-06-13 17:43:57 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 102) -2016-06-13 17:43:57 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 103) -2016-06-13 17:43:57 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 101) -2016-06-13 17:43:57 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:43:57 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 102). 2082 bytes result sent to driver -2016-06-13 17:43:57 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:43:57 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 101). 2082 bytes result sent to driver -2016-06-13 17:43:57 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:43:57 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 103). 2082 bytes result sent to driver -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 102) in 9 ms on localhost (1/4) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 101) in 10 ms on localhost (2/4) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 103) in 9 ms on localhost (3/4) -2016-06-13 17:43:57 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:43:57 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 100). 2082 bytes result sent to driver -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 100) in 16 ms on localhost (4/4) -2016-06-13 17:43:57 INFO DAGScheduler:59 - ResultStage 30 (takeSample at GaussianMixture.scala:184) finished in 0.017 s -2016-06-13 17:43:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 17:43:57 INFO DAGScheduler:59 - Job 30 finished: takeSample at GaussianMixture.scala:184, took 0.032676 s -2016-06-13 17:43:57 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:43:57 INFO DAGScheduler:59 - Got job 31 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:43:57 INFO DAGScheduler:59 - Final stage: ResultStage 31(takeSample at GaussianMixture.scala:184) -2016-06-13 17:43:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:57 INFO DAGScheduler:59 - Submitting ResultStage 31 (PartitionwiseSampledRDD[10] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=143370, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 18.8 KB, free 730.4 MB) -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(7881) called with curMem=162594, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.4 MB) -2016-06-13 17:43:57 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:62216 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:43:57 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:57 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 31 (PartitionwiseSampledRDD[10] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:43:57 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 4 tasks -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 104, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 105, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 2.0 in stage 31.0 (TID 106, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 3.0 in stage 31.0 (TID 107, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:43:57 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 105) -2016-06-13 17:43:57 INFO Executor:59 - Running task 3.0 in stage 31.0 (TID 107) -2016-06-13 17:43:57 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:43:57 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:43:57 INFO Executor:59 - Finished task 3.0 in stage 31.0 (TID 107). 2274 bytes result sent to driver -2016-06-13 17:43:57 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 105). 2269 bytes result sent to driver -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 3.0 in stage 31.0 (TID 107) in 7 ms on localhost (1/4) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 105) in 9 ms on localhost (2/4) -2016-06-13 17:43:57 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 104) -2016-06-13 17:43:57 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:43:57 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 104). 2048 bytes result sent to driver -2016-06-13 17:43:57 INFO Executor:59 - Running task 2.0 in stage 31.0 (TID 106) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 104) in 32 ms on localhost (3/4) -2016-06-13 17:43:57 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:43:57 INFO Executor:59 - Finished task 2.0 in stage 31.0 (TID 106). 2249 bytes result sent to driver -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 2.0 in stage 31.0 (TID 106) in 36 ms on localhost (4/4) -2016-06-13 17:43:57 INFO DAGScheduler:59 - ResultStage 31 (takeSample at GaussianMixture.scala:184) finished in 0.038 s -2016-06-13 17:43:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 17:43:57 INFO DAGScheduler:59 - Job 31 finished: takeSample at GaussianMixture.scala:184, took 0.079852 s -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=170475, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 464.0 B, free 730.4 MB) -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(849) called with curMem=170939, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 849.0 B, free 730.4 MB) -2016-06-13 17:43:57 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:62216 (size: 849.0 B, free: 730.5 MB) -2016-06-13 17:43:57 INFO SparkContext:59 - Created broadcast 44 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:43:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:43:57 INFO DAGScheduler:59 - Got job 32 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:43:57 INFO DAGScheduler:59 - Final stage: ResultStage 32(aggregate at GaussianMixture.scala:201) -2016-06-13 17:43:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:57 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=171788, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=175708, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:43:57 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:62216 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:43:57 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:57 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 32 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:43:57 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 4 tasks -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 108, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 109, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 2.0 in stage 32.0 (TID 110, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 3.0 in stage 32.0 (TID 111, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:57 INFO Executor:59 - Running task 3.0 in stage 32.0 (TID 111) -2016-06-13 17:43:57 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 108) -2016-06-13 17:43:57 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 109) -2016-06-13 17:43:57 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:43:57 INFO Executor:59 - Finished task 3.0 in stage 32.0 (TID 111). 2708 bytes result sent to driver -2016-06-13 17:43:57 INFO Executor:59 - Running task 2.0 in stage 32.0 (TID 110) -2016-06-13 17:43:57 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:43:57 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 109). 2708 bytes result sent to driver -2016-06-13 17:43:57 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:43:57 INFO Executor:59 - Finished task 2.0 in stage 32.0 (TID 110). 2708 bytes result sent to driver -2016-06-13 17:43:57 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:43:57 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 108). 2708 bytes result sent to driver -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 3.0 in stage 32.0 (TID 111) in 15 ms on localhost (1/4) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 109) in 17 ms on localhost (2/4) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 2.0 in stage 32.0 (TID 110) in 19 ms on localhost (3/4) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 108) in 22 ms on localhost (4/4) -2016-06-13 17:43:57 INFO DAGScheduler:59 - ResultStage 32 (aggregate at GaussianMixture.scala:201) finished in 0.023 s -2016-06-13 17:43:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 17:43:57 INFO DAGScheduler:59 - Job 32 finished: aggregate at GaussianMixture.scala:201, took 0.037809 s -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=177853, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 464.0 B, free 730.4 MB) -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=178317, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.4 MB) -2016-06-13 17:43:57 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:62216 (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:43:57 INFO SparkContext:59 - Created broadcast 46 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:43:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:43:57 INFO DAGScheduler:59 - Got job 33 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:43:57 INFO DAGScheduler:59 - Final stage: ResultStage 33(aggregate at GaussianMixture.scala:201) -2016-06-13 17:43:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:57 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=179170, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=183090, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:43:57 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:62216 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:43:57 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:57 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:43:57 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 112, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 113, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 114, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 115, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:57 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 115) -2016-06-13 17:43:57 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:43:57 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 115). 2708 bytes result sent to driver -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 115) in 9 ms on localhost (1/4) -2016-06-13 17:43:57 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 113) -2016-06-13 17:43:57 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:43:57 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 113). 2708 bytes result sent to driver -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 113) in 107 ms on localhost (2/4) -2016-06-13 17:43:57 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 112) -2016-06-13 17:43:57 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:43:57 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 112). 2708 bytes result sent to driver -2016-06-13 17:43:57 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 114) -2016-06-13 17:43:57 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:43:57 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 114). 2708 bytes result sent to driver -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 112) in 159 ms on localhost (3/4) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 114) in 157 ms on localhost (4/4) -2016-06-13 17:43:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 17:43:57 INFO DAGScheduler:59 - ResultStage 33 (aggregate at GaussianMixture.scala:201) finished in 0.161 s -2016-06-13 17:43:57 INFO DAGScheduler:59 - Job 33 finished: aggregate at GaussianMixture.scala:201, took 0.184206 s -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=185242, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 464.0 B, free 730.4 MB) -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=185706, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.4 MB) -2016-06-13 17:43:57 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:62216 (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:43:57 INFO SparkContext:59 - Created broadcast 48 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:43:57 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:43:57 INFO DAGScheduler:59 - Got job 34 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:43:57 INFO DAGScheduler:59 - Final stage: ResultStage 34(aggregate at GaussianMixture.scala:201) -2016-06-13 17:43:57 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:43:57 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:43:57 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=186559, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 17:43:57 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=190479, maxMem=766075207 -2016-06-13 17:43:57 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:43:57 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:62216 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:43:57 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:43:57 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:43:57 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 116, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 117, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 118, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 119, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:43:57 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 116) -2016-06-13 17:43:57 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 119) -2016-06-13 17:43:57 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 118) -2016-06-13 17:43:57 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:43:57 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:43:57 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 116). 2708 bytes result sent to driver -2016-06-13 17:43:57 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 117) -2016-06-13 17:43:57 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 118). 2708 bytes result sent to driver -2016-06-13 17:43:57 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:43:57 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:43:57 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 119). 2708 bytes result sent to driver -2016-06-13 17:43:57 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 117). 2708 bytes result sent to driver -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 116) in 18 ms on localhost (1/4) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 118) in 17 ms on localhost (2/4) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 119) in 19 ms on localhost (3/4) -2016-06-13 17:43:57 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 117) in 22 ms on localhost (4/4) -2016-06-13 17:43:57 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-06-13 17:43:57 INFO DAGScheduler:59 - ResultStage 34 (aggregate at GaussianMixture.scala:201) finished in 0.063 s -2016-06-13 17:43:57 INFO DAGScheduler:59 - Job 34 finished: aggregate at GaussianMixture.scala:201, took 0.080324 s -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 17:43:57 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 17:43:57 WARN QueuedThreadPool:145 - 6 threads could not be stopped -2016-06-13 17:43:57 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4041 -2016-06-13 17:43:58 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 17:43:58 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 17:43:58 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 17:43:58 INFO BlockManager:59 - BlockManager stopped -2016-06-13 17:43:58 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 17:43:58 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 17:43:58 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 17:43:58 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 17:43:58 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 17:43:58 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 17:43:58 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-9fcf877e-4556-4638-b76e-2c1c0c4d9df0 -2016-06-13 17:44:46 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 17:44:47 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 17:44:47 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 17:44:47 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 17:44:47 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 17:44:48 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 17:44:48 INFO Remoting:74 - Starting remoting -2016-06-13 17:44:48 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:62234] -2016-06-13 17:44:48 INFO Utils:59 - Successfully started service 'sparkDriver' on port 62234. -2016-06-13 17:44:48 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 17:44:48 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 17:44:48 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-78b9ba2f-93c7-4510-8585-83dfc832a8ff -2016-06-13 17:44:48 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 17:44:48 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-524746cd-5e39-43a3-81dc-cdfa2ec46f32\httpd-dbe1e003-de16-4c6b-9f80-f8b2e88144b4 -2016-06-13 17:44:48 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 17:44:48 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:44:48 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:62235 -2016-06-13 17:44:48 INFO Utils:59 - Successfully started service 'HTTP file server' on port 62235. -2016-06-13 17:44:48 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 17:44:48 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:44:48 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 17:44:48 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 17:44:48 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 17:44:48 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 17:44:48 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 17:44:49 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 62254. -2016-06-13 17:44:49 INFO NettyBlockTransferService:59 - Server created on 62254 -2016-06-13 17:44:49 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 17:44:49 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:62254 with 730.6 MB RAM, BlockManagerId(driver, localhost, 62254) -2016-06-13 17:44:49 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 17:44:50 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:50 INFO DAGScheduler:59 - Got job 0 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:44:50 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GaussianMixture.scala:171) -2016-06-13 17:44:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:50 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:50 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:50 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=0, maxMem=766075207 -2016-06-13 17:44:50 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:44:50 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1992, maxMem=766075207 -2016-06-13 17:44:50 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.6 MB) -2016-06-13 17:44:50 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:62254 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:44:50 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:44:50 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 17:44:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:50 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 17:44:50 INFO CacheManager:59 - Partition rdd_1_0 not found, computing it -2016-06-13 17:44:50 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=3242, maxMem=766075207 -2016-06-13 17:44:50 INFO MemoryStore:59 - Block rdd_1_0 stored as values in memory (estimated size 16.0 B, free 730.6 MB) -2016-06-13 17:44:50 INFO BlockManagerInfo:59 - Added rdd_1_0 in memory on localhost:62254 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:44:50 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1533 bytes result sent to driver -2016-06-13 17:44:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 85 ms on localhost (1/1) -2016-06-13 17:44:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 17:44:50 INFO DAGScheduler:59 - ResultStage 0 (first at GaussianMixture.scala:171) finished in 0.099 s -2016-06-13 17:44:50 INFO DAGScheduler:59 - Job 0 finished: first at GaussianMixture.scala:171, took 0.353323 s -2016-06-13 17:44:50 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:50 INFO DAGScheduler:59 - Got job 1 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:44:50 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GaussianMixture.scala:171) -2016-06-13 17:44:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:50 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:50 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:50 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=3258, maxMem=766075207 -2016-06-13 17:44:50 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:44:50 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=5250, maxMem=766075207 -2016-06-13 17:44:50 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.6 MB) -2016-06-13 17:44:50 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:62254 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:44:50 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:62254 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:44:50 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:50 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:44:50 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 3 tasks -2016-06-13 17:44:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:50 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:50 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 17:44:50 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 17:44:50 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 17:44:50 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 3) -2016-06-13 17:44:50 INFO CacheManager:59 - Partition rdd_1_1 not found, computing it -2016-06-13 17:44:50 INFO CacheManager:59 - Partition rdd_1_2 not found, computing it -2016-06-13 17:44:50 INFO CacheManager:59 - Partition rdd_1_3 not found, computing it -2016-06-13 17:44:50 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=3258, maxMem=766075207 -2016-06-13 17:44:50 INFO MemoryStore:59 - Block rdd_1_2 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:44:50 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=3354, maxMem=766075207 -2016-06-13 17:44:50 INFO MemoryStore:59 - Block rdd_1_3 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:44:50 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=3450, maxMem=766075207 -2016-06-13 17:44:50 INFO MemoryStore:59 - Block rdd_1_1 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:44:50 INFO BlockManagerInfo:59 - Added rdd_1_2 in memory on localhost:62254 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:44:50 INFO BlockManagerInfo:59 - Added rdd_1_3 in memory on localhost:62254 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:44:50 INFO BlockManagerInfo:59 - Added rdd_1_1 in memory on localhost:62254 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:44:50 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 1696 bytes result sent to driver -2016-06-13 17:44:50 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 3). 1696 bytes result sent to driver -2016-06-13 17:44:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 336 ms on localhost (1/3) -2016-06-13 17:44:50 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 3) in 339 ms on localhost (2/3) -2016-06-13 17:44:50 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 1696 bytes result sent to driver -2016-06-13 17:44:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 352 ms on localhost (3/3) -2016-06-13 17:44:50 INFO DAGScheduler:59 - ResultStage 1 (first at GaussianMixture.scala:171) finished in 0.355 s -2016-06-13 17:44:50 INFO DAGScheduler:59 - Job 1 finished: first at GaussianMixture.scala:171, took 0.410648 s -2016-06-13 17:44:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 17:44:50 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:50 INFO DAGScheduler:59 - Got job 2 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:50 INFO DAGScheduler:59 - Final stage: ResultStage 2(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:50 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:50 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:50 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=3546, maxMem=766075207 -2016-06-13 17:44:50 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 17:44:50 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=5378, maxMem=766075207 -2016-06-13 17:44:50 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.6 MB) -2016-06-13 17:44:50 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:62254 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:44:50 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:50 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 2 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:44:50 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 4 tasks -2016-06-13 17:44:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:50 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:50 INFO TaskSetManager:59 - Starting task 3.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:50 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 17:44:50 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-06-13 17:44:50 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 17:44:50 INFO Executor:59 - Running task 3.0 in stage 2.0 (TID 7) -2016-06-13 17:44:50 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:44:50 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:44:50 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:44:50 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:44:50 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 6). 2082 bytes result sent to driver -2016-06-13 17:44:50 INFO Executor:59 - Finished task 3.0 in stage 2.0 (TID 7). 2082 bytes result sent to driver -2016-06-13 17:44:50 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-06-13 17:44:50 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2082 bytes result sent to driver -2016-06-13 17:44:50 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 6) in 26 ms on localhost (1/4) -2016-06-13 17:44:50 INFO TaskSetManager:59 - Finished task 3.0 in stage 2.0 (TID 7) in 25 ms on localhost (2/4) -2016-06-13 17:44:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 28 ms on localhost (3/4) -2016-06-13 17:44:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 31 ms on localhost (4/4) -2016-06-13 17:44:50 INFO DAGScheduler:59 - ResultStage 2 (takeSample at GaussianMixture.scala:184) finished in 0.031 s -2016-06-13 17:44:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 17:44:50 INFO DAGScheduler:59 - Job 2 finished: takeSample at GaussianMixture.scala:184, took 0.046714 s -2016-06-13 17:44:50 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:50 INFO DAGScheduler:59 - Got job 3 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:50 INFO DAGScheduler:59 - Final stage: ResultStage 3(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:50 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:50 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:44:50 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=6573, maxMem=766075207 -2016-06-13 17:44:50 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 18.8 KB, free 730.6 MB) -2016-06-13 17:44:50 INFO MemoryStore:59 - ensureFreeSpace(7884) called with curMem=25797, maxMem=766075207 -2016-06-13 17:44:50 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.6 MB) -2016-06-13 17:44:50 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:62254 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:44:50 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:50 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:50 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 4 tasks -2016-06-13 17:44:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:44:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:44:50 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:44:50 INFO TaskSetManager:59 - Starting task 3.0 in stage 3.0 (TID 11, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:44:50 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 8) -2016-06-13 17:44:50 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 10) -2016-06-13 17:44:50 INFO Executor:59 - Running task 3.0 in stage 3.0 (TID 11) -2016-06-13 17:44:50 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 9) -2016-06-13 17:44:50 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:44:50 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:44:50 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 8). 2048 bytes result sent to driver -2016-06-13 17:44:50 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:44:50 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 10). 2279 bytes result sent to driver -2016-06-13 17:44:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 8) in 25 ms on localhost (1/4) -2016-06-13 17:44:50 INFO Executor:59 - Finished task 3.0 in stage 3.0 (TID 11). 2269 bytes result sent to driver -2016-06-13 17:44:50 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:44:50 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 9). 2269 bytes result sent to driver -2016-06-13 17:44:50 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 10) in 26 ms on localhost (2/4) -2016-06-13 17:44:50 INFO TaskSetManager:59 - Finished task 3.0 in stage 3.0 (TID 11) in 27 ms on localhost (3/4) -2016-06-13 17:44:50 INFO DAGScheduler:59 - ResultStage 3 (takeSample at GaussianMixture.scala:184) finished in 0.033 s -2016-06-13 17:44:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 9) in 31 ms on localhost (4/4) -2016-06-13 17:44:50 INFO DAGScheduler:59 - Job 3 finished: takeSample at GaussianMixture.scala:184, took 0.059842 s -2016-06-13 17:44:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 17:44:51 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 17:44:51 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 17:44:51 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 17:44:51 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=33681, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 464.0 B, free 730.6 MB) -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(854) called with curMem=34145, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 854.0 B, free 730.6 MB) -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:62254 (size: 854.0 B, free: 730.6 MB) -2016-06-13 17:44:51 INFO SparkContext:59 - Created broadcast 4 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:51 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:51 INFO DAGScheduler:59 - Got job 4 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:51 INFO DAGScheduler:59 - Final stage: ResultStage 4(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:51 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:51 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=34999, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(2148) called with curMem=38919, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:51 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:44:51 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 4 tasks -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 12, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 14, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 4.0 (TID 15, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:51 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 12) -2016-06-13 17:44:51 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 13) -2016-06-13 17:44:51 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 14) -2016-06-13 17:44:51 INFO Executor:59 - Running task 3.0 in stage 4.0 (TID 15) -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:44:51 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 12). 2708 bytes result sent to driver -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 12) in 22 ms on localhost (1/4) -2016-06-13 17:44:51 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 13). 2708 bytes result sent to driver -2016-06-13 17:44:51 INFO Executor:59 - Finished task 3.0 in stage 4.0 (TID 15). 2708 bytes result sent to driver -2016-06-13 17:44:51 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 14). 2708 bytes result sent to driver -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 13) in 32 ms on localhost (2/4) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 3.0 in stage 4.0 (TID 15) in 31 ms on localhost (3/4) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 14) in 33 ms on localhost (4/4) -2016-06-13 17:44:51 INFO DAGScheduler:59 - ResultStage 4 (aggregate at GaussianMixture.scala:201) finished in 0.035 s -2016-06-13 17:44:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 17:44:51 INFO DAGScheduler:59 - Job 4 finished: aggregate at GaussianMixture.scala:201, took 0.052380 s -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=41067, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=41531, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:62254 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:44:51 INFO SparkContext:59 - Created broadcast 6 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:51 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:51 INFO DAGScheduler:59 - Got job 5 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:51 INFO DAGScheduler:59 - Final stage: ResultStage 5(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:51 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:51 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=42384, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(2150) called with curMem=46304, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:51 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:44:51 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 4 tasks -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 16, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 17, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 18, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 5.0 (TID 19, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:51 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 18) -2016-06-13 17:44:51 INFO Executor:59 - Running task 3.0 in stage 5.0 (TID 19) -2016-06-13 17:44:51 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 17) -2016-06-13 17:44:51 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 16) -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:44:51 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 18). 2708 bytes result sent to driver -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:44:51 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 16). 2708 bytes result sent to driver -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 18) in 16 ms on localhost (1/4) -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 16) in 20 ms on localhost (2/4) -2016-06-13 17:44:51 INFO Executor:59 - Finished task 3.0 in stage 5.0 (TID 19). 2708 bytes result sent to driver -2016-06-13 17:44:51 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 17). 2708 bytes result sent to driver -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 3.0 in stage 5.0 (TID 19) in 20 ms on localhost (3/4) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 17) in 23 ms on localhost (4/4) -2016-06-13 17:44:51 INFO DAGScheduler:59 - ResultStage 5 (aggregate at GaussianMixture.scala:201) finished in 0.025 s -2016-06-13 17:44:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 17:44:51 INFO DAGScheduler:59 - Job 5 finished: aggregate at GaussianMixture.scala:201, took 0.041099 s -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=48454, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=48918, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:62254 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:44:51 INFO SparkContext:59 - Created broadcast 8 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:51 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:51 INFO DAGScheduler:59 - Got job 6 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:51 INFO DAGScheduler:59 - Final stage: ResultStage 6(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:51 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:51 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=49771, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(2150) called with curMem=53691, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:51 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:44:51 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 4 tasks -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 20, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:51 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 21, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 22, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 6.0 (TID 23, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:62254 in memory (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:44:51 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 21) -2016-06-13 17:44:51 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 22) -2016-06-13 17:44:51 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 20) -2016-06-13 17:44:51 INFO Executor:59 - Running task 3.0 in stage 6.0 (TID 23) -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:51 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 21). 2708 bytes result sent to driver -2016-06-13 17:44:51 INFO Executor:59 - Finished task 3.0 in stage 6.0 (TID 23). 2708 bytes result sent to driver -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:44:51 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 20). 2708 bytes result sent to driver -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 21) in 14 ms on localhost (1/4) -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 3.0 in stage 6.0 (TID 23) in 13 ms on localhost (2/4) -2016-06-13 17:44:51 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 22). 2708 bytes result sent to driver -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 20) in 21 ms on localhost (3/4) -2016-06-13 17:44:51 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 22) in 22 ms on localhost (4/4) -2016-06-13 17:44:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 17:44:51 INFO DAGScheduler:59 - ResultStage 6 (aggregate at GaussianMixture.scala:201) finished in 0.026 s -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:62254 in memory (size: 854.0 B, free: 730.6 MB) -2016-06-13 17:44:51 INFO DAGScheduler:59 - Job 6 finished: aggregate at GaussianMixture.scala:201, took 0.068632 s -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:62254 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:44:51 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:62254 in memory (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:44:51 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:62254 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:44:51 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 17:44:51 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:51 INFO DAGScheduler:59 - Got job 7 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:44:51 INFO DAGScheduler:59 - Final stage: ResultStage 7(first at GaussianMixture.scala:171) -2016-06-13 17:44:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:51 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:51 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=7691, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=9683, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.6 MB) -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:62254 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:44:51 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:51 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:44:51 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 24, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:51 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 24) -2016-06-13 17:44:51 INFO CacheManager:59 - Partition rdd_3_0 not found, computing it -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=10935, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block rdd_3_0 stored as values in memory (estimated size 16.0 B, free 730.6 MB) -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Added rdd_3_0 in memory on localhost:62254 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:44:51 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 24). 1533 bytes result sent to driver -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 24) in 10 ms on localhost (1/1) -2016-06-13 17:44:51 INFO DAGScheduler:59 - ResultStage 7 (first at GaussianMixture.scala:171) finished in 0.011 s -2016-06-13 17:44:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 17:44:51 INFO DAGScheduler:59 - Job 7 finished: first at GaussianMixture.scala:171, took 0.025478 s -2016-06-13 17:44:51 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:51 INFO DAGScheduler:59 - Got job 8 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:44:51 INFO DAGScheduler:59 - Final stage: ResultStage 8(first at GaussianMixture.scala:171) -2016-06-13 17:44:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:51 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:51 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=10951, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=12943, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.6 MB) -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:62254 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:44:51 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:51 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 8 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:44:51 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 3 tasks -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 25, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 26, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 8.0 (TID 27, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:51 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 25) -2016-06-13 17:44:51 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 26) -2016-06-13 17:44:51 INFO Executor:59 - Running task 2.0 in stage 8.0 (TID 27) -2016-06-13 17:44:51 INFO CacheManager:59 - Partition rdd_3_2 not found, computing it -2016-06-13 17:44:51 INFO CacheManager:59 - Partition rdd_3_1 not found, computing it -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=14195, maxMem=766075207 -2016-06-13 17:44:51 INFO CacheManager:59 - Partition rdd_3_3 not found, computing it -2016-06-13 17:44:51 INFO MemoryStore:59 - Block rdd_3_2 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=14291, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block rdd_3_1 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Added rdd_3_2 in memory on localhost:62254 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=14387, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block rdd_3_3 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Added rdd_3_1 in memory on localhost:62254 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:44:51 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 26). 1696 bytes result sent to driver -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Added rdd_3_3 in memory on localhost:62254 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:44:51 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 25). 1696 bytes result sent to driver -2016-06-13 17:44:51 INFO Executor:59 - Finished task 2.0 in stage 8.0 (TID 27). 1696 bytes result sent to driver -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 26) in 15 ms on localhost (1/3) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 25) in 17 ms on localhost (2/3) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 8.0 (TID 27) in 17 ms on localhost (3/3) -2016-06-13 17:44:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 17:44:51 INFO DAGScheduler:59 - ResultStage 8 (first at GaussianMixture.scala:171) finished in 0.020 s -2016-06-13 17:44:51 INFO DAGScheduler:59 - Job 8 finished: first at GaussianMixture.scala:171, took 0.037683 s -2016-06-13 17:44:51 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:51 INFO DAGScheduler:59 - Got job 9 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:51 INFO DAGScheduler:59 - Final stage: ResultStage 9(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:51 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:51 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=14483, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=16315, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.6 MB) -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:62254 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:44:51 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 9 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:44:51 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 4 tasks -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 28, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 29, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 9.0 (TID 30, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 9.0 (TID 31, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:51 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 28) -2016-06-13 17:44:51 INFO Executor:59 - Running task 3.0 in stage 9.0 (TID 31) -2016-06-13 17:44:51 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 29) -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:44:51 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 28). 2082 bytes result sent to driver -2016-06-13 17:44:51 INFO Executor:59 - Running task 2.0 in stage 9.0 (TID 30) -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:44:51 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 29). 2082 bytes result sent to driver -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 28) in 18 ms on localhost (1/4) -2016-06-13 17:44:51 INFO Executor:59 - Finished task 3.0 in stage 9.0 (TID 31). 2082 bytes result sent to driver -2016-06-13 17:44:51 INFO Executor:59 - Finished task 2.0 in stage 9.0 (TID 30). 2082 bytes result sent to driver -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 29) in 20 ms on localhost (2/4) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 3.0 in stage 9.0 (TID 31) in 18 ms on localhost (3/4) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 9.0 (TID 30) in 21 ms on localhost (4/4) -2016-06-13 17:44:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 17:44:51 INFO DAGScheduler:59 - ResultStage 9 (takeSample at GaussianMixture.scala:184) finished in 0.024 s -2016-06-13 17:44:51 INFO DAGScheduler:59 - Job 9 finished: takeSample at GaussianMixture.scala:184, took 0.045424 s -2016-06-13 17:44:51 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:51 INFO DAGScheduler:59 - Got job 10 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:51 INFO DAGScheduler:59 - Final stage: ResultStage 10(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:51 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:51 INFO DAGScheduler:59 - Submitting ResultStage 10 (PartitionwiseSampledRDD[4] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=17510, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 18.8 KB, free 730.6 MB) -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(7877) called with curMem=36734, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:62254 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:44:51 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 10 (PartitionwiseSampledRDD[4] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:51 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 4 tasks -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 32, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 33, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 10.0 (TID 34, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 10.0 (TID 35, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:44:51 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 32) -2016-06-13 17:44:51 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 33) -2016-06-13 17:44:51 INFO Executor:59 - Running task 3.0 in stage 10.0 (TID 35) -2016-06-13 17:44:51 INFO Executor:59 - Running task 2.0 in stage 10.0 (TID 34) -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:44:51 INFO Executor:59 - Finished task 2.0 in stage 10.0 (TID 34). 2244 bytes result sent to driver -2016-06-13 17:44:51 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 32). 2048 bytes result sent to driver -2016-06-13 17:44:51 INFO Executor:59 - Finished task 3.0 in stage 10.0 (TID 35). 2274 bytes result sent to driver -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 10.0 (TID 34) in 17 ms on localhost (1/4) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 32) in 24 ms on localhost (2/4) -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:44:51 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 33). 2274 bytes result sent to driver -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 3.0 in stage 10.0 (TID 35) in 21 ms on localhost (3/4) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 33) in 30 ms on localhost (4/4) -2016-06-13 17:44:51 INFO DAGScheduler:59 - ResultStage 10 (takeSample at GaussianMixture.scala:184) finished in 0.031 s -2016-06-13 17:44:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 17:44:51 INFO DAGScheduler:59 - Job 10 finished: takeSample at GaussianMixture.scala:184, took 0.051300 s -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=44611, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(850) called with curMem=45075, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 850.0 B, free 730.5 MB) -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:62254 (size: 850.0 B, free: 730.6 MB) -2016-06-13 17:44:51 INFO SparkContext:59 - Created broadcast 14 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:51 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:51 INFO DAGScheduler:59 - Got job 11 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:51 INFO DAGScheduler:59 - Final stage: ResultStage 11(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:51 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:51 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=45925, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(2147) called with curMem=49845, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:51 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 11 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:44:51 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 4 tasks -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 36, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 37, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 2.0 in stage 11.0 (TID 38, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 3.0 in stage 11.0 (TID 39, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:51 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 37) -2016-06-13 17:44:51 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 36) -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:44:51 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 37). 2708 bytes result sent to driver -2016-06-13 17:44:51 INFO Executor:59 - Running task 2.0 in stage 11.0 (TID 38) -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:44:51 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 36). 2708 bytes result sent to driver -2016-06-13 17:44:51 INFO Executor:59 - Running task 3.0 in stage 11.0 (TID 39) -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:44:51 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:44:51 INFO Executor:59 - Finished task 2.0 in stage 11.0 (TID 38). 2708 bytes result sent to driver -2016-06-13 17:44:51 INFO Executor:59 - Finished task 3.0 in stage 11.0 (TID 39). 2708 bytes result sent to driver -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 37) in 25 ms on localhost (1/4) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 36) in 28 ms on localhost (2/4) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 2.0 in stage 11.0 (TID 38) in 27 ms on localhost (3/4) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Finished task 3.0 in stage 11.0 (TID 39) in 27 ms on localhost (4/4) -2016-06-13 17:44:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 17:44:51 INFO DAGScheduler:59 - ResultStage 11 (aggregate at GaussianMixture.scala:201) finished in 0.033 s -2016-06-13 17:44:51 INFO DAGScheduler:59 - Job 11 finished: aggregate at GaussianMixture.scala:201, took 0.046851 s -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=51992, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=52456, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:62254 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:44:51 INFO SparkContext:59 - Created broadcast 16 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:51 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:51 INFO DAGScheduler:59 - Got job 12 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:51 INFO DAGScheduler:59 - Final stage: ResultStage 12(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:51 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:51 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=53309, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:44:51 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=57229, maxMem=766075207 -2016-06-13 17:44:51 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:44:51 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:51 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:51 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 12 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:44:51 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 4 tasks -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 40, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 41, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 12.0 (TID 42, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 3.0 in stage 12.0 (TID 43, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 40) -2016-06-13 17:44:52 INFO Executor:59 - Running task 3.0 in stage 12.0 (TID 43) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 40). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 41) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 3.0 in stage 12.0 (TID 43). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 40) in 17 ms on localhost (1/4) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 3.0 in stage 12.0 (TID 43) in 15 ms on localhost (2/4) -2016-06-13 17:44:52 INFO Executor:59 - Running task 2.0 in stage 12.0 (TID 42) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 41). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 41) in 22 ms on localhost (3/4) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 2.0 in stage 12.0 (TID 42). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 12.0 (TID 42) in 27 ms on localhost (4/4) -2016-06-13 17:44:52 INFO DAGScheduler:59 - ResultStage 12 (aggregate at GaussianMixture.scala:201) finished in 0.030 s -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 17:44:52 INFO DAGScheduler:59 - Job 12 finished: aggregate at GaussianMixture.scala:201, took 0.043771 s -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=59381, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=59845, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:62254 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 18 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:52 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Got job 13 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:52 INFO DAGScheduler:59 - Final stage: ResultStage 13(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=60698, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=64618, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 13 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 4 tasks -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 44, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 45, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 13.0 (TID 46, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 3.0 in stage 13.0 (TID 47, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 44) -2016-06-13 17:44:52 INFO Executor:59 - Running task 3.0 in stage 13.0 (TID 47) -2016-06-13 17:44:52 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 45) -2016-06-13 17:44:52 INFO Executor:59 - Running task 2.0 in stage 13.0 (TID 46) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 44). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 45). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 44) in 14 ms on localhost (1/4) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 45) in 14 ms on localhost (2/4) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 3.0 in stage 13.0 (TID 47). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 3.0 in stage 13.0 (TID 47) in 15 ms on localhost (3/4) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 2.0 in stage 13.0 (TID 46). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 13.0 (TID 46) in 19 ms on localhost (4/4) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 17:44:52 INFO DAGScheduler:59 - ResultStage 13 (aggregate at GaussianMixture.scala:201) finished in 0.023 s -2016-06-13 17:44:52 INFO DAGScheduler:59 - Job 13 finished: aggregate at GaussianMixture.scala:201, took 0.039032 s -2016-06-13 17:44:52 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Got job 14 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:44:52 INFO DAGScheduler:59 - Final stage: ResultStage 14(first at GaussianMixture.scala:171) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=66770, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(1251) called with curMem=68762, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 1251.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:62254 (size: 1251.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 14 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 1 tasks -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 48, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:52 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 48) -2016-06-13 17:44:52 INFO CacheManager:59 - Partition rdd_5_0 not found, computing it -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=70013, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block rdd_5_0 stored as values in memory (estimated size 16.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added rdd_5_0 in memory on localhost:62254 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 48). 1533 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 48) in 11 ms on localhost (1/1) -2016-06-13 17:44:52 INFO DAGScheduler:59 - ResultStage 14 (first at GaussianMixture.scala:171) finished in 0.013 s -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 17:44:52 INFO DAGScheduler:59 - Job 14 finished: first at GaussianMixture.scala:171, took 0.029674 s -2016-06-13 17:44:52 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Got job 15 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:44:52 INFO DAGScheduler:59 - Final stage: ResultStage 15(first at GaussianMixture.scala:171) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=70029, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(1251) called with curMem=72021, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1251.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:62254 (size: 1251.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 15 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 3 tasks -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 49, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 50, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 15.0 (TID 51, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 50) -2016-06-13 17:44:52 INFO Executor:59 - Running task 2.0 in stage 15.0 (TID 51) -2016-06-13 17:44:52 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 49) -2016-06-13 17:44:52 INFO CacheManager:59 - Partition rdd_5_3 not found, computing it -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=73272, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block rdd_5_3 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO CacheManager:59 - Partition rdd_5_1 not found, computing it -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added rdd_5_3 in memory on localhost:62254 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=73368, maxMem=766075207 -2016-06-13 17:44:52 INFO CacheManager:59 - Partition rdd_5_2 not found, computing it -2016-06-13 17:44:52 INFO MemoryStore:59 - Block rdd_5_1 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 2.0 in stage 15.0 (TID 51). 1696 bytes result sent to driver -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added rdd_5_1 in memory on localhost:62254 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 49). 1696 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 15.0 (TID 51) in 15 ms on localhost (1/3) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 49) in 18 ms on localhost (2/3) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=73464, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block rdd_5_2 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added rdd_5_2 in memory on localhost:62254 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 50). 1696 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 50) in 23 ms on localhost (3/3) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 17:44:52 INFO DAGScheduler:59 - ResultStage 15 (first at GaussianMixture.scala:171) finished in 0.025 s -2016-06-13 17:44:52 INFO DAGScheduler:59 - Job 15 finished: first at GaussianMixture.scala:171, took 0.039604 s -2016-06-13 17:44:52 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Got job 16 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:52 INFO DAGScheduler:59 - Final stage: ResultStage 16(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=73560, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 1832.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(1194) called with curMem=75392, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 1194.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:62254 (size: 1194.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 16 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 4 tasks -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 52, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 53, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 16.0 (TID 54, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 3.0 in stage 16.0 (TID 55, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 53) -2016-06-13 17:44:52 INFO Executor:59 - Running task 3.0 in stage 16.0 (TID 55) -2016-06-13 17:44:52 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 52) -2016-06-13 17:44:52 INFO Executor:59 - Running task 2.0 in stage 16.0 (TID 54) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 3.0 in stage 16.0 (TID 55). 2082 bytes result sent to driver -2016-06-13 17:44:52 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 53). 2082 bytes result sent to driver -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 2.0 in stage 16.0 (TID 54). 2082 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 3.0 in stage 16.0 (TID 55) in 8 ms on localhost (1/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 53) in 12 ms on localhost (2/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 16.0 (TID 54) in 14 ms on localhost (3/4) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 52). 2082 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 52) in 20 ms on localhost (4/4) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 17:44:52 INFO DAGScheduler:59 - ResultStage 16 (takeSample at GaussianMixture.scala:184) finished in 0.023 s -2016-06-13 17:44:52 INFO DAGScheduler:59 - Job 16 finished: takeSample at GaussianMixture.scala:184, took 0.038864 s -2016-06-13 17:44:52 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Got job 17 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:52 INFO DAGScheduler:59 - Final stage: ResultStage 17(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting ResultStage 17 (PartitionwiseSampledRDD[6] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=76586, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(7891) called with curMem=95810, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:62254 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (PartitionwiseSampledRDD[6] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 56, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 57, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 58, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 59, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:44:52 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 56) -2016-06-13 17:44:52 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 57) -2016-06-13 17:44:52 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 58) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 56). 2048 bytes result sent to driver -2016-06-13 17:44:52 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 59) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 58). 2259 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 56) in 15 ms on localhost (1/4) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 58) in 16 ms on localhost (2/4) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 59). 2226 bytes result sent to driver -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 59) in 19 ms on localhost (3/4) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 57). 2269 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 57) in 26 ms on localhost (4/4) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 17:44:52 INFO DAGScheduler:59 - ResultStage 17 (takeSample at GaussianMixture.scala:184) finished in 0.027 s -2016-06-13 17:44:52 INFO DAGScheduler:59 - Job 17 finished: takeSample at GaussianMixture.scala:184, took 0.044267 s -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=103701, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(850) called with curMem=104165, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 850.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:62254 (size: 850.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 24 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:52 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Got job 18 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:52 INFO DAGScheduler:59 - Final stage: ResultStage 18(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=105015, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:62254 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=81820, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:44:52 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:62254 in memory (size: 1194.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-06-13 17:44:52 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 60, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 61, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:62254 in memory (size: 1251.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 62, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 63, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 60) -2016-06-13 17:44:52 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 63) -2016-06-13 17:44:52 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 61) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 60). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 61). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 62) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 63). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 62). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 60) in 25 ms on localhost (1/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 61) in 24 ms on localhost (2/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 63) in 22 ms on localhost (3/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 62) in 26 ms on localhost (4/4) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 17:44:52 INFO DAGScheduler:59 - ResultStage 18 (aggregate at GaussianMixture.scala:201) finished in 0.033 s -2016-06-13 17:44:52 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Job 18 finished: aggregate at GaussianMixture.scala:201, took 0.077034 s -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=74453, maxMem=766075207 -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:62254 in memory (size: 1251.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=70997, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:52 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:62254 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 26 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:62254 in memory (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Got job 19 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:52 INFO DAGScheduler:59 - Final stage: ResultStage 19(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:52 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:62254 in memory (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=61456, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(2151) called with curMem=62765, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:44:52 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 19 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 4 tasks -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:62254 in memory (size: 850.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 64, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 65, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 19.0 (TID 66, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 3.0 in stage 19.0 (TID 67, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:62254 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:44:52 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 65) -2016-06-13 17:44:52 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 64) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 65). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 64). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO Executor:59 - Running task 3.0 in stage 19.0 (TID 67) -2016-06-13 17:44:52 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:62254 in memory (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 17:44:52 INFO Executor:59 - Finished task 3.0 in stage 19.0 (TID 67). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO Executor:59 - Running task 2.0 in stage 19.0 (TID 66) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:62254 in memory (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:44:52 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 17:44:52 INFO Executor:59 - Finished task 2.0 in stage 19.0 (TID 66). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 64) in 31 ms on localhost (1/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 3.0 in stage 19.0 (TID 67) in 30 ms on localhost (2/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 19.0 (TID 66) in 33 ms on localhost (3/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 65) in 36 ms on localhost (4/4) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:62254 in memory (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO DAGScheduler:59 - ResultStage 19 (aggregate at GaussianMixture.scala:201) finished in 0.044 s -2016-06-13 17:44:52 INFO DAGScheduler:59 - Job 19 finished: aggregate at GaussianMixture.scala:201, took 0.063823 s -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=23066, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 464.0 B, free 730.6 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=23530, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.6 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:62254 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 28 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:52 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 17:44:52 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Got job 20 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:52 INFO DAGScheduler:59 - Final stage: ResultStage 20(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:52 INFO BlockManager:59 - Removing RDD 3 -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=24383, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 3.8 KB, free 730.6 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(2151) called with curMem=27999, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.6 MB) -2016-06-13 17:44:52 INFO ContextCleaner:59 - Cleaned RDD 3 -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 68, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 69, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 70, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 71, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:52 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 69) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:44:52 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 17:44:52 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 69). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 68) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 69) in 12 ms on localhost (1/4) -2016-06-13 17:44:52 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 70) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:62254 in memory (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 70). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 68). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 71) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 70) in 20 ms on localhost (2/4) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 68) in 27 ms on localhost (3/4) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 71). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO BlockManager:59 - Removing RDD 1 -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 71) in 28 ms on localhost (4/4) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 17:44:52 INFO DAGScheduler:59 - ResultStage 20 (aggregate at GaussianMixture.scala:201) finished in 0.032 s -2016-06-13 17:44:52 INFO DAGScheduler:59 - Job 20 finished: aggregate at GaussianMixture.scala:201, took 0.061246 s -2016-06-13 17:44:52 INFO ContextCleaner:59 - Cleaned RDD 1 -2016-06-13 17:44:52 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Got job 21 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:44:52 INFO DAGScheduler:59 - Final stage: ResultStage 21(first at GaussianMixture.scala:171) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=22459, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=24451, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.6 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:62254 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 21 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 1 tasks -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 72, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:52 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 72) -2016-06-13 17:44:52 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=25703, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 16.0 B, free 730.6 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:62254 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 72). 1533 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 72) in 8 ms on localhost (1/1) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 17:44:52 INFO DAGScheduler:59 - ResultStage 21 (first at GaussianMixture.scala:171) finished in 0.008 s -2016-06-13 17:44:52 INFO DAGScheduler:59 - Job 21 finished: first at GaussianMixture.scala:171, took 0.019584 s -2016-06-13 17:44:52 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Got job 22 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:44:52 INFO DAGScheduler:59 - Final stage: ResultStage 22(first at GaussianMixture.scala:171) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=25719, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=27711, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.6 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:62254 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 22 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 3 tasks -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 73, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 74, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 75, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 75) -2016-06-13 17:44:52 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 73) -2016-06-13 17:44:52 INFO CacheManager:59 - Partition rdd_7_3 not found, computing it -2016-06-13 17:44:52 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 74) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=28963, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block rdd_7_3 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added rdd_7_3 in memory on localhost:62254 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=29059, maxMem=766075207 -2016-06-13 17:44:52 INFO CacheManager:59 - Partition rdd_7_2 not found, computing it -2016-06-13 17:44:52 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=29155, maxMem=766075207 -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:62254 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - Block rdd_7_2 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 73). 1696 bytes result sent to driver -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added rdd_7_2 in memory on localhost:62254 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 75). 1696 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 73) in 17 ms on localhost (1/3) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 74). 1696 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 75) in 16 ms on localhost (2/3) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 74) in 18 ms on localhost (3/3) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 17:44:52 INFO DAGScheduler:59 - ResultStage 22 (first at GaussianMixture.scala:171) finished in 0.020 s -2016-06-13 17:44:52 INFO DAGScheduler:59 - Job 22 finished: first at GaussianMixture.scala:171, took 0.037295 s -2016-06-13 17:44:52 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Got job 23 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:52 INFO DAGScheduler:59 - Final stage: ResultStage 23(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=29251, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=31083, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.6 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:62254 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 23 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 76, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 77, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 78, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 79, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 77) -2016-06-13 17:44:52 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 78) -2016-06-13 17:44:52 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 79) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 78). 2082 bytes result sent to driver -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 79). 2082 bytes result sent to driver -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 77). 2082 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 78) in 10 ms on localhost (1/4) -2016-06-13 17:44:52 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 76) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 77) in 13 ms on localhost (2/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 79) in 11 ms on localhost (3/4) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 76). 2082 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 76) in 18 ms on localhost (4/4) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 17:44:52 INFO DAGScheduler:59 - ResultStage 23 (takeSample at GaussianMixture.scala:184) finished in 0.019 s -2016-06-13 17:44:52 INFO DAGScheduler:59 - Job 23 finished: takeSample at GaussianMixture.scala:184, took 0.033658 s -2016-06-13 17:44:52 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Got job 24 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:52 INFO DAGScheduler:59 - Final stage: ResultStage 24(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting ResultStage 24 (PartitionwiseSampledRDD[8] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=32278, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(7882) called with curMem=51502, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:62254 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 24 (PartitionwiseSampledRDD[8] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 4 tasks -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 80, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 81, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 82, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 3.0 in stage 24.0 (TID 83, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:44:52 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 80) -2016-06-13 17:44:52 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 82) -2016-06-13 17:44:52 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 81) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 80). 2048 bytes result sent to driver -2016-06-13 17:44:52 INFO Executor:59 - Running task 3.0 in stage 24.0 (TID 83) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 81). 2269 bytes result sent to driver -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 80) in 13 ms on localhost (1/4) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 3.0 in stage 24.0 (TID 83). 2269 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 81) in 13 ms on localhost (2/4) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 82). 2221 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 3.0 in stage 24.0 (TID 83) in 15 ms on localhost (3/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 82) in 17 ms on localhost (4/4) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-06-13 17:44:52 INFO DAGScheduler:59 - ResultStage 24 (takeSample at GaussianMixture.scala:184) finished in 0.021 s -2016-06-13 17:44:52 INFO DAGScheduler:59 - Job 24 finished: takeSample at GaussianMixture.scala:184, took 0.037617 s -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=59384, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(851) called with curMem=59848, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 851.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:62254 (size: 851.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 34 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:52 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Got job 25 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:52 INFO DAGScheduler:59 - Final stage: ResultStage 25(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=60699, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=64619, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 25 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 4 tasks -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 84, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 85, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 25.0 (TID 86, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 3.0 in stage 25.0 (TID 87, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 84) -2016-06-13 17:44:52 INFO Executor:59 - Running task 3.0 in stage 25.0 (TID 87) -2016-06-13 17:44:52 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 85) -2016-06-13 17:44:52 INFO Executor:59 - Running task 2.0 in stage 25.0 (TID 86) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 84). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 3.0 in stage 25.0 (TID 87). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 85). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 2.0 in stage 25.0 (TID 86). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 84) in 14 ms on localhost (1/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 3.0 in stage 25.0 (TID 87) in 15 ms on localhost (2/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 85) in 19 ms on localhost (3/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 25.0 (TID 86) in 21 ms on localhost (4/4) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-06-13 17:44:52 INFO DAGScheduler:59 - ResultStage 25 (aggregate at GaussianMixture.scala:201) finished in 0.023 s -2016-06-13 17:44:52 INFO DAGScheduler:59 - Job 25 finished: aggregate at GaussianMixture.scala:201, took 0.038590 s -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=66764, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=67228, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:62254 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 36 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:52 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Got job 26 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:52 INFO DAGScheduler:59 - Final stage: ResultStage 26(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=68081, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=72001, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 26 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 88, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 89, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 90, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 91, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 89) -2016-06-13 17:44:52 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 91) -2016-06-13 17:44:52 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 90) -2016-06-13 17:44:52 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 88) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 89). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 91). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 89) in 10 ms on localhost (1/4) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 90). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 88). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 91) in 16 ms on localhost (2/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 90) in 18 ms on localhost (3/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 88) in 20 ms on localhost (4/4) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-06-13 17:44:52 INFO DAGScheduler:59 - ResultStage 26 (aggregate at GaussianMixture.scala:201) finished in 0.020 s -2016-06-13 17:44:52 INFO DAGScheduler:59 - Job 26 finished: aggregate at GaussianMixture.scala:201, took 0.033901 s -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=74153, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=74617, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:62254 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 38 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:52 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Got job 27 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:52 INFO DAGScheduler:59 - Final stage: ResultStage 27(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=75470, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=79390, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 27 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 4 tasks -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 92, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 93, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 27.0 (TID 94, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 3.0 in stage 27.0 (TID 95, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 93) -2016-06-13 17:44:52 INFO Executor:59 - Running task 2.0 in stage 27.0 (TID 94) -2016-06-13 17:44:52 INFO Executor:59 - Running task 3.0 in stage 27.0 (TID 95) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 2.0 in stage 27.0 (TID 94). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 92) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 3.0 in stage 27.0 (TID 95). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 92). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 93). 2708 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 27.0 (TID 94) in 14 ms on localhost (1/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 92) in 17 ms on localhost (2/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 3.0 in stage 27.0 (TID 95) in 14 ms on localhost (3/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 93) in 17 ms on localhost (4/4) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-06-13 17:44:52 INFO DAGScheduler:59 - ResultStage 27 (aggregate at GaussianMixture.scala:201) finished in 0.019 s -2016-06-13 17:44:52 INFO DAGScheduler:59 - Job 27 finished: aggregate at GaussianMixture.scala:201, took 0.032724 s -2016-06-13 17:44:52 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Got job 28 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:44:52 INFO DAGScheduler:59 - Final stage: ResultStage 28(first at GaussianMixture.scala:171) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=81542, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=83534, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:62254 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 28 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 1 tasks -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 96, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:52 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 96) -2016-06-13 17:44:52 INFO CacheManager:59 - Partition rdd_9_0 not found, computing it -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=84786, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block rdd_9_0 stored as values in memory (estimated size 16.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added rdd_9_0 in memory on localhost:62254 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 96). 1533 bytes result sent to driver -2016-06-13 17:44:52 INFO DAGScheduler:59 - ResultStage 28 (first at GaussianMixture.scala:171) finished in 0.009 s -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 96) in 9 ms on localhost (1/1) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Job 28 finished: first at GaussianMixture.scala:171, took 0.019840 s -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-06-13 17:44:52 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Got job 29 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:44:52 INFO DAGScheduler:59 - Final stage: ResultStage 29(first at GaussianMixture.scala:171) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=84802, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=86794, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:62254 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 29 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 3 tasks -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 97, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 98, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 29.0 (TID 99, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 98) -2016-06-13 17:44:52 INFO Executor:59 - Running task 2.0 in stage 29.0 (TID 99) -2016-06-13 17:44:52 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 97) -2016-06-13 17:44:52 INFO CacheManager:59 - Partition rdd_9_3 not found, computing it -2016-06-13 17:44:52 INFO CacheManager:59 - Partition rdd_9_2 not found, computing it -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=88046, maxMem=766075207 -2016-06-13 17:44:52 INFO CacheManager:59 - Partition rdd_9_1 not found, computing it -2016-06-13 17:44:52 INFO MemoryStore:59 - Block rdd_9_3 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=88142, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block rdd_9_2 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added rdd_9_3 in memory on localhost:62254 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=88238, maxMem=766075207 -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added rdd_9_2 in memory on localhost:62254 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - Block rdd_9_1 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 2.0 in stage 29.0 (TID 99). 1696 bytes result sent to driver -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added rdd_9_1 in memory on localhost:62254 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 29.0 (TID 99) in 10 ms on localhost (1/3) -2016-06-13 17:44:52 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 98). 1696 bytes result sent to driver -2016-06-13 17:44:52 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 97). 1696 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 98) in 13 ms on localhost (2/3) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 97) in 14 ms on localhost (3/3) -2016-06-13 17:44:52 INFO DAGScheduler:59 - ResultStage 29 (first at GaussianMixture.scala:171) finished in 0.014 s -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 17:44:52 INFO DAGScheduler:59 - Job 29 finished: first at GaussianMixture.scala:171, took 0.024786 s -2016-06-13 17:44:52 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Got job 30 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:52 INFO DAGScheduler:59 - Final stage: ResultStage 30(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=88334, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 1832.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=90166, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:62254 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 100, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 101, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 102, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 103, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:52 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 100) -2016-06-13 17:44:52 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 103) -2016-06-13 17:44:52 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 102) -2016-06-13 17:44:52 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 101) -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 100). 2082 bytes result sent to driver -2016-06-13 17:44:52 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 101). 2082 bytes result sent to driver -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:44:52 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:44:52 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 102). 2082 bytes result sent to driver -2016-06-13 17:44:52 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 103). 2082 bytes result sent to driver -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 100) in 7 ms on localhost (1/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 101) in 7 ms on localhost (2/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 103) in 6 ms on localhost (3/4) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 102) in 8 ms on localhost (4/4) -2016-06-13 17:44:52 INFO DAGScheduler:59 - ResultStage 30 (takeSample at GaussianMixture.scala:184) finished in 0.009 s -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 17:44:52 INFO DAGScheduler:59 - Job 30 finished: takeSample at GaussianMixture.scala:184, took 0.021494 s -2016-06-13 17:44:52 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Got job 31 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:52 INFO DAGScheduler:59 - Final stage: ResultStage 31(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting ResultStage 31 (PartitionwiseSampledRDD[10] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=91361, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:44:52 INFO MemoryStore:59 - ensureFreeSpace(7882) called with curMem=110585, maxMem=766075207 -2016-06-13 17:44:52 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:44:52 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:62254 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:44:52 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:52 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 31 (PartitionwiseSampledRDD[10] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:52 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 4 tasks -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 104, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 105, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 31.0 (TID 106, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:44:52 INFO TaskSetManager:59 - Starting task 3.0 in stage 31.0 (TID 107, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:44:52 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 104) -2016-06-13 17:44:52 INFO Executor:59 - Running task 3.0 in stage 31.0 (TID 107) -2016-06-13 17:44:52 INFO Executor:59 - Running task 2.0 in stage 31.0 (TID 106) -2016-06-13 17:44:52 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 105) -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:44:53 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 104). 2048 bytes result sent to driver -2016-06-13 17:44:53 INFO Executor:59 - Finished task 3.0 in stage 31.0 (TID 107). 2274 bytes result sent to driver -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:44:53 INFO Executor:59 - Finished task 2.0 in stage 31.0 (TID 106). 2249 bytes result sent to driver -2016-06-13 17:44:53 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 105). 2269 bytes result sent to driver -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 104) in 9 ms on localhost (1/4) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 2.0 in stage 31.0 (TID 106) in 9 ms on localhost (2/4) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 105) in 9 ms on localhost (3/4) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 3.0 in stage 31.0 (TID 107) in 9 ms on localhost (4/4) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 17:44:53 INFO DAGScheduler:59 - ResultStage 31 (takeSample at GaussianMixture.scala:184) finished in 0.011 s -2016-06-13 17:44:53 INFO DAGScheduler:59 - Job 31 finished: takeSample at GaussianMixture.scala:184, took 0.023396 s -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=118467, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(849) called with curMem=118931, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 849.0 B, free 730.5 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:62254 (size: 849.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Created broadcast 44 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:53 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Got job 32 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:53 INFO DAGScheduler:59 - Final stage: ResultStage 32(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=119780, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=123700, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 32 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 4 tasks -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 108, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 109, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 2.0 in stage 32.0 (TID 110, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 3.0 in stage 32.0 (TID 111, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:53 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 108) -2016-06-13 17:44:53 INFO Executor:59 - Running task 3.0 in stage 32.0 (TID 111) -2016-06-13 17:44:53 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 109) -2016-06-13 17:44:53 INFO Executor:59 - Running task 2.0 in stage 32.0 (TID 110) -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:44:53 INFO Executor:59 - Finished task 3.0 in stage 32.0 (TID 111). 2708 bytes result sent to driver -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:44:53 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 109). 2708 bytes result sent to driver -2016-06-13 17:44:53 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 108). 2708 bytes result sent to driver -2016-06-13 17:44:53 INFO Executor:59 - Finished task 2.0 in stage 32.0 (TID 110). 2708 bytes result sent to driver -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 3.0 in stage 32.0 (TID 111) in 8 ms on localhost (1/4) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 108) in 11 ms on localhost (2/4) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 2.0 in stage 32.0 (TID 110) in 10 ms on localhost (3/4) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 109) in 12 ms on localhost (4/4) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 17:44:53 INFO DAGScheduler:59 - ResultStage 32 (aggregate at GaussianMixture.scala:201) finished in 0.014 s -2016-06-13 17:44:53 INFO DAGScheduler:59 - Job 32 finished: aggregate at GaussianMixture.scala:201, took 0.024030 s -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=125845, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=126309, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:62254 (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Created broadcast 46 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:53 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Got job 33 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:53 INFO DAGScheduler:59 - Final stage: ResultStage 33(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=127162, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=131082, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 112, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 113, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 114, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 115, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:53 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 114) -2016-06-13 17:44:53 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 115) -2016-06-13 17:44:53 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 112) -2016-06-13 17:44:53 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 113) -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:44:53 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 112). 2708 bytes result sent to driver -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:44:53 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 115). 2708 bytes result sent to driver -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:44:53 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 113). 2708 bytes result sent to driver -2016-06-13 17:44:53 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 114). 2708 bytes result sent to driver -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 112) in 7 ms on localhost (1/4) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 114) in 6 ms on localhost (2/4) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 115) in 7 ms on localhost (3/4) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 113) in 8 ms on localhost (4/4) -2016-06-13 17:44:53 INFO DAGScheduler:59 - ResultStage 33 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 17:44:53 INFO DAGScheduler:59 - Job 33 finished: aggregate at GaussianMixture.scala:201, took 0.018184 s -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=133234, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=133698, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:62254 (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Created broadcast 48 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:53 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Got job 34 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:53 INFO DAGScheduler:59 - Final stage: ResultStage 34(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=134551, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=138471, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 116, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 117, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 118, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 119, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:44:53 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 117) -2016-06-13 17:44:53 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 119) -2016-06-13 17:44:53 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 116) -2016-06-13 17:44:53 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 118) -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:44:53 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 116). 2708 bytes result sent to driver -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:44:53 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 117). 2708 bytes result sent to driver -2016-06-13 17:44:53 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 119). 2708 bytes result sent to driver -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:44:53 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 118). 2708 bytes result sent to driver -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 116) in 10 ms on localhost (1/4) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 117) in 10 ms on localhost (2/4) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 119) in 9 ms on localhost (3/4) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 118) in 11 ms on localhost (4/4) -2016-06-13 17:44:53 INFO DAGScheduler:59 - ResultStage 34 (aggregate at GaussianMixture.scala:201) finished in 0.013 s -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-06-13 17:44:53 INFO DAGScheduler:59 - Job 34 finished: aggregate at GaussianMixture.scala:201, took 0.026096 s -2016-06-13 17:44:53 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Got job 35 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:44:53 INFO DAGScheduler:59 - Final stage: ResultStage 35(first at GaussianMixture.scala:171) -2016-06-13 17:44:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=140623, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=142615, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.4 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:62254 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 35 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 1 tasks -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 120, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:44:53 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 120) -2016-06-13 17:44:53 INFO CacheManager:59 - Partition rdd_12_0 not found, computing it -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(224) called with curMem=143865, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block rdd_12_0 stored as values in memory (estimated size 224.0 B, free 730.4 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added rdd_12_0 in memory on localhost:62254 (size: 224.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 120). 1688 bytes result sent to driver -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 120) in 7 ms on localhost (1/1) -2016-06-13 17:44:53 INFO DAGScheduler:59 - ResultStage 35 (first at GaussianMixture.scala:171) finished in 0.007 s -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-06-13 17:44:53 INFO DAGScheduler:59 - Job 35 finished: first at GaussianMixture.scala:171, took 0.017915 s -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=144089, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 712.0 B, free 730.4 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=144801, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 880.0 B, free 730.4 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:62254 (size: 880.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Created broadcast 51 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:53 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Got job 36 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:53 INFO DAGScheduler:59 - Final stage: ResultStage 36(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=145681, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(2181) called with curMem=149777, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 36 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 4 tasks -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 121, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 122, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 2.0 in stage 36.0 (TID 123, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 3.0 in stage 36.0 (TID 124, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:53 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 121) -2016-06-13 17:44:53 INFO Executor:59 - Running task 2.0 in stage 36.0 (TID 123) -2016-06-13 17:44:53 INFO Executor:59 - Running task 3.0 in stage 36.0 (TID 124) -2016-06-13 17:44:53 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 122) -2016-06-13 17:44:53 INFO CacheManager:59 - Partition rdd_12_2 not found, computing it -2016-06-13 17:44:53 INFO CacheManager:59 - Partition rdd_12_3 not found, computing it -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_12_0 locally -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=151958, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block rdd_12_2 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-06-13 17:44:53 INFO CacheManager:59 - Partition rdd_12_1 not found, computing it -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=152246, maxMem=766075207 -2016-06-13 17:44:53 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 121). 2762 bytes result sent to driver -2016-06-13 17:44:53 INFO MemoryStore:59 - Block rdd_12_3 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added rdd_12_2 in memory on localhost:62254 (size: 288.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 121) in 14 ms on localhost (1/4) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=152534, maxMem=766075207 -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added rdd_12_3 in memory on localhost:62254 (size: 288.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - Block rdd_12_1 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added rdd_12_1 in memory on localhost:62254 (size: 288.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO Executor:59 - Finished task 2.0 in stage 36.0 (TID 123). 2247 bytes result sent to driver -2016-06-13 17:44:53 INFO Executor:59 - Finished task 3.0 in stage 36.0 (TID 124). 2247 bytes result sent to driver -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 2.0 in stage 36.0 (TID 123) in 15 ms on localhost (2/4) -2016-06-13 17:44:53 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 122). 2247 bytes result sent to driver -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 3.0 in stage 36.0 (TID 124) in 15 ms on localhost (3/4) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 122) in 17 ms on localhost (4/4) -2016-06-13 17:44:53 INFO DAGScheduler:59 - ResultStage 36 (aggregate at GaussianMixture.scala:201) finished in 0.019 s -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-06-13 17:44:53 INFO DAGScheduler:59 - Job 36 finished: aggregate at GaussianMixture.scala:201, took 0.031523 s -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=152822, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 712.0 B, free 730.4 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(934) called with curMem=153534, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 934.0 B, free 730.4 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:62254 (size: 934.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Created broadcast 53 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:53 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Got job 37 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:53 INFO DAGScheduler:59 - Final stage: ResultStage 37(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=154468, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(2241) called with curMem=158564, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:62254 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 37 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 4 tasks -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 125, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 126, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 127, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 3.0 in stage 37.0 (TID 128, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:53 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 125) -2016-06-13 17:44:53 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 127) -2016-06-13 17:44:53 INFO Executor:59 - Running task 3.0 in stage 37.0 (TID 128) -2016-06-13 17:44:53 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 126) -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_12_2 locally -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_12_1 locally -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_12_3 locally -2016-06-13 17:44:53 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 127). 2762 bytes result sent to driver -2016-06-13 17:44:53 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 126). 2762 bytes result sent to driver -2016-06-13 17:44:53 INFO Executor:59 - Finished task 3.0 in stage 37.0 (TID 128). 2762 bytes result sent to driver -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_12_0 locally -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 127) in 7 ms on localhost (1/4) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 126) in 8 ms on localhost (2/4) -2016-06-13 17:44:53 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 125). 2762 bytes result sent to driver -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 3.0 in stage 37.0 (TID 128) in 7 ms on localhost (3/4) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 125) in 9 ms on localhost (4/4) -2016-06-13 17:44:53 INFO DAGScheduler:59 - ResultStage 37 (aggregate at GaussianMixture.scala:201) finished in 0.010 s -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-06-13 17:44:53 INFO DAGScheduler:59 - Job 37 finished: aggregate at GaussianMixture.scala:201, took 0.020012 s -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=160805, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 712.0 B, free 730.4 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=161517, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.4 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:62254 (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Created broadcast 55 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:53 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Got job 38 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:53 INFO DAGScheduler:59 - Final stage: ResultStage 38(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=162449, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=166545, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:62254 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 129, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 130, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 131, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 132, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:53 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 129) -2016-06-13 17:44:53 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 132) -2016-06-13 17:44:53 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 131) -2016-06-13 17:44:53 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 130) -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_12_3 locally -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_12_2 locally -2016-06-13 17:44:53 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 132). 2762 bytes result sent to driver -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_12_1 locally -2016-06-13 17:44:53 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 131). 2762 bytes result sent to driver -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_12_0 locally -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 131) in 9 ms on localhost (1/4) -2016-06-13 17:44:53 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 130). 2762 bytes result sent to driver -2016-06-13 17:44:53 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 129). 2762 bytes result sent to driver -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 132) in 9 ms on localhost (2/4) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 129) in 12 ms on localhost (3/4) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 130) in 12 ms on localhost (4/4) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-06-13 17:44:53 INFO DAGScheduler:59 - ResultStage 38 (aggregate at GaussianMixture.scala:201) finished in 0.013 s -2016-06-13 17:44:53 INFO DAGScheduler:59 - Job 38 finished: aggregate at GaussianMixture.scala:201, took 0.023151 s -2016-06-13 17:44:53 INFO SparkContext:59 - Starting job: first at GaussianMixtureFromSparkSuite.scala:90 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Got job 39 (first at GaussianMixtureFromSparkSuite.scala:90) with 1 output partitions -2016-06-13 17:44:53 INFO DAGScheduler:59 - Final stage: ResultStage 39(first at GaussianMixtureFromSparkSuite.scala:90) -2016-06-13 17:44:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting ResultStage 39 (ParallelCollectionRDD[13] at parallelize at GaussianMixtureFromSparkSuite.scala:87), which has no missing parents -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(1272) called with curMem=168783, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 1272.0 B, free 730.4 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(837) called with curMem=170055, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 837.0 B, free 730.4 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:62254 (size: 837.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 39 (ParallelCollectionRDD[13] at parallelize at GaussianMixtureFromSparkSuite.scala:87) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 1 tasks -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:62254 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 133, localhost, PROCESS_LOCAL, 7178 bytes) -2016-06-13 17:44:53 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 133) -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-06-13 17:44:53 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 133). 1433 bytes result sent to driver -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 133) in 13 ms on localhost (1/1) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-06-13 17:44:53 INFO DAGScheduler:59 - ResultStage 39 (first at GaussianMixtureFromSparkSuite.scala:90) finished in 0.015 s -2016-06-13 17:44:53 INFO DAGScheduler:59 - Job 39 finished: first at GaussianMixtureFromSparkSuite.scala:90, took 0.039264 s -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:62254 in memory (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Got job 40 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:44:53 INFO DAGScheduler:59 - Final stage: ResultStage 40(first at GaussianMixture.scala:171) -2016-06-13 17:44:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:62254 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=155643, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 1992.0 B, free 730.4 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:62254 in memory (size: 934.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=156923, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.4 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:62254 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 40 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 1 tasks -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 134, localhost, PROCESS_LOCAL, 7178 bytes) -2016-06-13 17:44:53 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 134) -2016-06-13 17:44:53 INFO CacheManager:59 - Partition rdd_14_0 not found, computing it -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:62254 in memory (size: 880.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:62254 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=149054, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block rdd_14_0 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added rdd_14_0 in memory on localhost:62254 (size: 5.4 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 134). 2083 bytes result sent to driver -2016-06-13 17:44:53 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 134) in 12 ms on localhost (1/1) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-06-13 17:44:53 INFO DAGScheduler:59 - ResultStage 40 (first at GaussianMixture.scala:171) finished in 0.013 s -2016-06-13 17:44:53 INFO DAGScheduler:59 - Job 40 finished: first at GaussianMixture.scala:171, took 0.024412 s -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned RDD 12 -2016-06-13 17:44:53 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Got job 41 (takeSample at GaussianMixture.scala:184) with 2 output partitions -2016-06-13 17:44:53 INFO DAGScheduler:59 - Final stage: ResultStage 41(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=145438, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 1832.0 B, free 730.4 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=146417, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 1193.0 B, free 730.4 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:62254 (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:62254 in memory (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 135, localhost, PROCESS_LOCAL, 7178 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 136, localhost, PROCESS_LOCAL, 7594 bytes) -2016-06-13 17:44:53 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 135) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 136) -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-06-13 17:44:53 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 135). 2082 bytes result sent to driver -2016-06-13 17:44:53 INFO CacheManager:59 - Partition rdd_14_1 not found, computing it -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 135) in 12 ms on localhost (1/2) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(6000) called with curMem=141074, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block rdd_14_1 stored as values in memory (estimated size 5.9 KB, free 730.4 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added rdd_14_1 in memory on localhost:62254 (size: 5.9 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:62254 in memory (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 136). 1567 bytes result sent to driver -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 136) in 22 ms on localhost (2/2) -2016-06-13 17:44:53 INFO DAGScheduler:59 - ResultStage 41 (takeSample at GaussianMixture.scala:184) finished in 0.023 s -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-06-13 17:44:53 INFO DAGScheduler:59 - Job 41 finished: takeSample at GaussianMixture.scala:184, took 0.038562 s -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:62254 in memory (size: 849.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Got job 42 (takeSample at GaussianMixture.scala:184) with 2 output partitions -2016-06-13 17:44:53 INFO DAGScheduler:59 - Final stage: ResultStage 42(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting ResultStage 42 (PartitionwiseSampledRDD[15] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=111273, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:62254 in memory (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(7877) called with curMem=130497, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:62254 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (PartitionwiseSampledRDD[15] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:62254 in memory (size: 1195.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 137, localhost, PROCESS_LOCAL, 7287 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 138, localhost, PROCESS_LOCAL, 7703 bytes) -2016-06-13 17:44:53 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 137) -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-06-13 17:44:53 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 137). 7406 bytes result sent to driver -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:62254 in memory (size: 1252.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 137) in 19 ms on localhost (1/2) -2016-06-13 17:44:53 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 138) -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_14_1 locally -2016-06-13 17:44:53 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 138). 7844 bytes result sent to driver -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 138) in 29 ms on localhost (2/2) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-06-13 17:44:53 INFO DAGScheduler:59 - ResultStage 42 (takeSample at GaussianMixture.scala:184) finished in 0.030 s -2016-06-13 17:44:53 INFO DAGScheduler:59 - Job 42 finished: takeSample at GaussianMixture.scala:184, took 0.049891 s -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:62254 in memory (size: 1252.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-06-13 17:44:53 INFO BlockManager:59 - Removing RDD 9 -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned RDD 9 -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:62254 in memory (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:62254 in memory (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:62254 in memory (size: 851.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:62254 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:62254 in memory (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:62254 in memory (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:62254 in memory (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 17:44:53 INFO BlockManager:59 - Removing RDD 7 -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(205504) called with curMem=69472, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 200.7 KB, free 730.3 MB) -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned RDD 7 -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(12070) called with curMem=268905, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 11.8 KB, free 730.3 MB) -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:62254 (size: 11.8 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Created broadcast 61 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:62254 in memory (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 17:44:53 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:62254 in memory (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO DAGScheduler:59 - Got job 43 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:44:53 INFO DAGScheduler:59 - Final stage: ResultStage 43(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(310656) called with curMem=266205, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 303.4 KB, free 730.0 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:62254 in memory (size: 850.0 B, free: 730.6 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(18284) called with curMem=576011, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 17.9 KB, free 730.0 MB) -2016-06-13 17:44:53 INFO BlockManager:59 - Removing RDD 5 -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:62254 (size: 17.9 KB, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-06-13 17:44:53 INFO ContextCleaner:59 - Cleaned RDD 5 -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 139, localhost, PROCESS_LOCAL, 7178 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 140, localhost, PROCESS_LOCAL, 7594 bytes) -2016-06-13 17:44:53 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 139) -2016-06-13 17:44:53 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 140) -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_14_1 locally -2016-06-13 17:44:53 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-06-13 17:44:53 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 139). 105440 bytes result sent to driver -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 139) in 79 ms on localhost (1/2) -2016-06-13 17:44:53 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 140). 105440 bytes result sent to driver -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 140) in 85 ms on localhost (2/2) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-06-13 17:44:53 INFO DAGScheduler:59 - ResultStage 43 (aggregate at GaussianMixture.scala:201) finished in 0.087 s -2016-06-13 17:44:53 INFO DAGScheduler:59 - Job 43 finished: aggregate at GaussianMixture.scala:201, took 0.102593 s -2016-06-13 17:44:53 INFO SparkContext:59 - Starting job: collect at GaussianMixture.scala:213 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Got job 44 (collect at GaussianMixture.scala:213) with 5 output partitions -2016-06-13 17:44:53 INFO DAGScheduler:59 - Final stage: ResultStage 44(collect at GaussianMixture.scala:213) -2016-06-13 17:44:53 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[17] at map at GaussianMixture.scala:211), which has no missing parents -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=593527, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 2.3 KB, free 730.0 MB) -2016-06-13 17:44:53 INFO MemoryStore:59 - ensureFreeSpace(1389) called with curMem=595839, maxMem=766075207 -2016-06-13 17:44:53 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 1389.0 B, free 730.0 MB) -2016-06-13 17:44:53 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:62254 (size: 1389.0 B, free: 730.5 MB) -2016-06-13 17:44:53 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:53 INFO DAGScheduler:59 - Submitting 5 missing tasks from ResultStage 44 (MapPartitionsRDD[17] at map at GaussianMixture.scala:211) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 5 tasks -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 141, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 142, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 2.0 in stage 44.0 (TID 143, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 3.0 in stage 44.0 (TID 144, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:44:53 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 141) -2016-06-13 17:44:53 INFO Executor:59 - Running task 3.0 in stage 44.0 (TID 144) -2016-06-13 17:44:53 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 142) -2016-06-13 17:44:53 INFO Executor:59 - Running task 2.0 in stage 44.0 (TID 143) -2016-06-13 17:44:53 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 141). 42916 bytes result sent to driver -2016-06-13 17:44:53 INFO TaskSetManager:59 - Starting task 4.0 in stage 44.0 (TID 145, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 141) in 226 ms on localhost (1/5) -2016-06-13 17:44:53 INFO Executor:59 - Running task 4.0 in stage 44.0 (TID 145) -2016-06-13 17:44:53 INFO Executor:59 - Finished task 2.0 in stage 44.0 (TID 143). 42916 bytes result sent to driver -2016-06-13 17:44:53 INFO Executor:59 - Finished task 3.0 in stage 44.0 (TID 144). 42916 bytes result sent to driver -2016-06-13 17:44:53 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 142). 42916 bytes result sent to driver -2016-06-13 17:44:53 INFO Executor:59 - Finished task 4.0 in stage 44.0 (TID 145). 42916 bytes result sent to driver -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 2.0 in stage 44.0 (TID 143) in 292 ms on localhost (2/5) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 4.0 in stage 44.0 (TID 145) in 70 ms on localhost (3/5) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 3.0 in stage 44.0 (TID 144) in 292 ms on localhost (4/5) -2016-06-13 17:44:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 142) in 295 ms on localhost (5/5) -2016-06-13 17:44:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-06-13 17:44:53 INFO DAGScheduler:59 - ResultStage 44 (collect at GaussianMixture.scala:213) finished in 0.298 s -2016-06-13 17:44:53 INFO DAGScheduler:59 - Job 44 finished: collect at GaussianMixture.scala:213, took 0.309460 s -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(205544) called with curMem=597228, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 200.7 KB, free 729.8 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(23825) called with curMem=802772, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 23.3 KB, free 729.8 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:62254 (size: 23.3 KB, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 64 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:54 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Got job 45 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:44:54 INFO DAGScheduler:59 - Final stage: ResultStage 45(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(310656) called with curMem=826597, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 303.4 KB, free 729.5 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(30131) called with curMem=1137253, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 29.4 KB, free 729.5 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:62254 (size: 29.4 KB, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 146, localhost, PROCESS_LOCAL, 7178 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 147, localhost, PROCESS_LOCAL, 7594 bytes) -2016-06-13 17:44:54 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 146) -2016-06-13 17:44:54 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 147) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_14_1 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 146). 105440 bytes result sent to driver -2016-06-13 17:44:54 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 147). 105440 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 146) in 16 ms on localhost (1/2) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 147) in 18 ms on localhost (2/2) -2016-06-13 17:44:54 INFO DAGScheduler:59 - ResultStage 45 (aggregate at GaussianMixture.scala:201) finished in 0.019 s -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-06-13 17:44:54 INFO DAGScheduler:59 - Job 45 finished: aggregate at GaussianMixture.scala:201, took 0.031902 s -2016-06-13 17:44:54 INFO SparkContext:59 - Starting job: collect at GaussianMixture.scala:213 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Got job 46 (collect at GaussianMixture.scala:213) with 5 output partitions -2016-06-13 17:44:54 INFO DAGScheduler:59 - Final stage: ResultStage 46(collect at GaussianMixture.scala:213) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[19] at map at GaussianMixture.scala:211), which has no missing parents -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=1167384, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 2.3 KB, free 729.5 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(1389) called with curMem=1169696, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 1389.0 B, free 729.5 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:62254 (size: 1389.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting 5 missing tasks from ResultStage 46 (MapPartitionsRDD[19] at map at GaussianMixture.scala:211) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 5 tasks -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 148, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 149, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 46.0 (TID 150, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 3.0 in stage 46.0 (TID 151, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:44:54 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 149) -2016-06-13 17:44:54 INFO Executor:59 - Running task 2.0 in stage 46.0 (TID 150) -2016-06-13 17:44:54 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 148) -2016-06-13 17:44:54 INFO Executor:59 - Running task 3.0 in stage 46.0 (TID 151) -2016-06-13 17:44:54 INFO Executor:59 - Finished task 3.0 in stage 46.0 (TID 151). 42916 bytes result sent to driver -2016-06-13 17:44:54 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 148). 42916 bytes result sent to driver -2016-06-13 17:44:54 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 149). 42916 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 4.0 in stage 46.0 (TID 152, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:44:54 INFO Executor:59 - Running task 4.0 in stage 46.0 (TID 152) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 3.0 in stage 46.0 (TID 151) in 20 ms on localhost (1/5) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 148) in 25 ms on localhost (2/5) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 149) in 25 ms on localhost (3/5) -2016-06-13 17:44:54 INFO Executor:59 - Finished task 2.0 in stage 46.0 (TID 150). 42916 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 46.0 (TID 150) in 26 ms on localhost (4/5) -2016-06-13 17:44:54 INFO Executor:59 - Finished task 4.0 in stage 46.0 (TID 152). 42916 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 4.0 in stage 46.0 (TID 152) in 14 ms on localhost (5/5) -2016-06-13 17:44:54 INFO DAGScheduler:59 - ResultStage 46 (collect at GaussianMixture.scala:213) finished in 0.031 s -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-06-13 17:44:54 INFO DAGScheduler:59 - Job 46 finished: collect at GaussianMixture.scala:213, took 0.040814 s -2016-06-13 17:44:54 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Got job 47 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:44:54 INFO DAGScheduler:59 - Final stage: ResultStage 47(first at GaussianMixture.scala:171) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=1171085, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 1992.0 B, free 729.5 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1173077, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 1250.0 B, free 729.5 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:62254 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 67 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 47 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 1 tasks -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 153, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:54 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 153) -2016-06-13 17:44:54 INFO CacheManager:59 - Partition rdd_21_0 not found, computing it -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=1174327, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block rdd_21_0 stored as values in memory (estimated size 16.0 B, free 729.5 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added rdd_21_0 in memory on localhost:62254 (size: 16.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 153). 1533 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 153) in 6 ms on localhost (1/1) -2016-06-13 17:44:54 INFO DAGScheduler:59 - ResultStage 47 (first at GaussianMixture.scala:171) finished in 0.007 s -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-06-13 17:44:54 INFO DAGScheduler:59 - Job 47 finished: first at GaussianMixture.scala:171, took 0.015337 s -2016-06-13 17:44:54 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Got job 48 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:44:54 INFO DAGScheduler:59 - Final stage: ResultStage 48(first at GaussianMixture.scala:171) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=1174343, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 1992.0 B, free 729.5 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1176335, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 1250.0 B, free 729.5 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:62254 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 48 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 3 tasks -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 154, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 155, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 48.0 (TID 156, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:54 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 154) -2016-06-13 17:44:54 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 155) -2016-06-13 17:44:54 INFO Executor:59 - Running task 2.0 in stage 48.0 (TID 156) -2016-06-13 17:44:54 INFO CacheManager:59 - Partition rdd_21_1 not found, computing it -2016-06-13 17:44:54 INFO CacheManager:59 - Partition rdd_21_3 not found, computing it -2016-06-13 17:44:54 INFO CacheManager:59 - Partition rdd_21_2 not found, computing it -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=1177585, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block rdd_21_1 stored as values in memory (estimated size 192.0 B, free 729.5 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=1177777, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block rdd_21_3 stored as values in memory (estimated size 184.0 B, free 729.5 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=1177961, maxMem=766075207 -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added rdd_21_1 in memory on localhost:62254 (size: 192.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - Block rdd_21_2 stored as values in memory (estimated size 192.0 B, free 729.5 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added rdd_21_3 in memory on localhost:62254 (size: 184.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added rdd_21_2 in memory on localhost:62254 (size: 192.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 155). 2144 bytes result sent to driver -2016-06-13 17:44:54 INFO Executor:59 - Finished task 2.0 in stage 48.0 (TID 156). 2132 bytes result sent to driver -2016-06-13 17:44:54 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 154). 2144 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 48.0 (TID 156) in 134 ms on localhost (1/3) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 155) in 136 ms on localhost (2/3) -2016-06-13 17:44:54 INFO DAGScheduler:59 - ResultStage 48 (first at GaussianMixture.scala:171) finished in 0.138 s -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 154) in 137 ms on localhost (3/3) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-06-13 17:44:54 INFO DAGScheduler:59 - Job 48 finished: first at GaussianMixture.scala:171, took 0.145221 s -2016-06-13 17:44:54 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Got job 49 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:54 INFO DAGScheduler:59 - Final stage: ResultStage 49(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=1178153, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 1832.0 B, free 729.5 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=1179985, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 1193.0 B, free 729.5 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:62254 (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 69 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 49 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 4 tasks -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 157, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 158, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 49.0 (TID 159, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 3.0 in stage 49.0 (TID 160, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:54 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 157) -2016-06-13 17:44:54 INFO Executor:59 - Running task 2.0 in stage 49.0 (TID 159) -2016-06-13 17:44:54 INFO Executor:59 - Running task 3.0 in stage 49.0 (TID 160) -2016-06-13 17:44:54 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 158) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 157). 2082 bytes result sent to driver -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_21_3 locally -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_21_2 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 3.0 in stage 49.0 (TID 160). 2082 bytes result sent to driver -2016-06-13 17:44:54 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 158). 2082 bytes result sent to driver -2016-06-13 17:44:54 INFO Executor:59 - Finished task 2.0 in stage 49.0 (TID 159). 2082 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 157) in 6 ms on localhost (1/4) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 3.0 in stage 49.0 (TID 160) in 5 ms on localhost (2/4) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 158) in 6 ms on localhost (3/4) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 49.0 (TID 159) in 8 ms on localhost (4/4) -2016-06-13 17:44:54 INFO DAGScheduler:59 - ResultStage 49 (takeSample at GaussianMixture.scala:184) finished in 0.009 s -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-06-13 17:44:54 INFO DAGScheduler:59 - Job 49 finished: takeSample at GaussianMixture.scala:184, took 0.017140 s -2016-06-13 17:44:54 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Got job 50 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:54 INFO DAGScheduler:59 - Final stage: ResultStage 50(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting ResultStage 50 (PartitionwiseSampledRDD[22] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=1181178, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 18.8 KB, free 729.4 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(7882) called with curMem=1200402, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 7.7 KB, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:62254 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 50 (PartitionwiseSampledRDD[22] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 4 tasks -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 161, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 162, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 50.0 (TID 163, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 3.0 in stage 50.0 (TID 164, localhost, PROCESS_LOCAL, 2373 bytes) -2016-06-13 17:44:54 INFO Executor:59 - Running task 2.0 in stage 50.0 (TID 163) -2016-06-13 17:44:54 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 162) -2016-06-13 17:44:54 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 161) -2016-06-13 17:44:54 INFO Executor:59 - Running task 3.0 in stage 50.0 (TID 164) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_21_2 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 2.0 in stage 50.0 (TID 163). 2704 bytes result sent to driver -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 161). 2048 bytes result sent to driver -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_21_3 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 3.0 in stage 50.0 (TID 164). 2692 bytes result sent to driver -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 162). 2724 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 50.0 (TID 163) in 6 ms on localhost (1/4) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 161) in 8 ms on localhost (2/4) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 3.0 in stage 50.0 (TID 164) in 6 ms on localhost (3/4) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 162) in 8 ms on localhost (4/4) -2016-06-13 17:44:54 INFO DAGScheduler:59 - ResultStage 50 (takeSample at GaussianMixture.scala:184) finished in 0.009 s -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-06-13 17:44:54 INFO DAGScheduler:59 - Job 50 finished: takeSample at GaussianMixture.scala:184, took 0.017428 s -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=1208284, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 560.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(890) called with curMem=1208844, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 890.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:62254 (size: 890.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 71 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:54 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Got job 51 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:54 INFO DAGScheduler:59 - Final stage: ResultStage 51(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=1209734, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(2188) called with curMem=1213798, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 51 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 4 tasks -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 165, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 166, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 51.0 (TID 167, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 3.0 in stage 51.0 (TID 168, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:54 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 166) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 17:44:54 INFO Executor:59 - Running task 3.0 in stage 51.0 (TID 168) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_21_3 locally -2016-06-13 17:44:54 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 165) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 165). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 165) in 28 ms on localhost (1/4) -2016-06-13 17:44:54 INFO Executor:59 - Running task 2.0 in stage 51.0 (TID 167) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_21_2 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 166). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO Executor:59 - Finished task 3.0 in stage 51.0 (TID 168). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 166) in 37 ms on localhost (2/4) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 3.0 in stage 51.0 (TID 168) in 36 ms on localhost (3/4) -2016-06-13 17:44:54 INFO Executor:59 - Finished task 2.0 in stage 51.0 (TID 167). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 51.0 (TID 167) in 38 ms on localhost (4/4) -2016-06-13 17:44:54 INFO DAGScheduler:59 - ResultStage 51 (aggregate at GaussianMixture.scala:201) finished in 0.040 s -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-06-13 17:44:54 INFO DAGScheduler:59 - Job 51 finished: aggregate at GaussianMixture.scala:201, took 0.050022 s -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=1215986, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 560.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=1216546, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 900.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:62254 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 73 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:54 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Got job 52 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:54 INFO DAGScheduler:59 - Final stage: ResultStage 52(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting ResultStage 52 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=1217446, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=1221510, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 52 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 4 tasks -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 169, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 170, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 52.0 (TID 171, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 3.0 in stage 52.0 (TID 172, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:54 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 170) -2016-06-13 17:44:54 INFO Executor:59 - Running task 3.0 in stage 52.0 (TID 172) -2016-06-13 17:44:54 INFO Executor:59 - Running task 2.0 in stage 52.0 (TID 171) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_21_3 locally -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_21_2 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 3.0 in stage 52.0 (TID 172). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO Executor:59 - Finished task 2.0 in stage 52.0 (TID 171). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 3.0 in stage 52.0 (TID 172) in 7 ms on localhost (1/4) -2016-06-13 17:44:54 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 169) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 52.0 (TID 171) in 9 ms on localhost (2/4) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 170). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 169). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 170) in 12 ms on localhost (3/4) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 169) in 14 ms on localhost (4/4) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-06-13 17:44:54 INFO DAGScheduler:59 - ResultStage 52 (aggregate at GaussianMixture.scala:201) finished in 0.014 s -2016-06-13 17:44:54 INFO DAGScheduler:59 - Job 52 finished: aggregate at GaussianMixture.scala:201, took 0.026486 s -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=1223704, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 560.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=1224264, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 900.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:62254 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 75 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:54 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Got job 53 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:54 INFO DAGScheduler:59 - Final stage: ResultStage 53(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=1225164, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=1229228, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 53 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 4 tasks -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 173, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 174, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 53.0 (TID 175, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 3.0 in stage 53.0 (TID 176, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:54 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 173) -2016-06-13 17:44:54 INFO Executor:59 - Running task 3.0 in stage 53.0 (TID 176) -2016-06-13 17:44:54 INFO Executor:59 - Running task 2.0 in stage 53.0 (TID 175) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_21_3 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 3.0 in stage 53.0 (TID 176). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_21_2 locally -2016-06-13 17:44:54 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 174) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 173). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 174). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO Executor:59 - Finished task 2.0 in stage 53.0 (TID 175). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 3.0 in stage 53.0 (TID 176) in 10 ms on localhost (1/4) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 173) in 13 ms on localhost (2/4) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 174) in 12 ms on localhost (3/4) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 53.0 (TID 175) in 12 ms on localhost (4/4) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-06-13 17:44:54 INFO DAGScheduler:59 - ResultStage 53 (aggregate at GaussianMixture.scala:201) finished in 0.014 s -2016-06-13 17:44:54 INFO DAGScheduler:59 - Job 53 finished: aggregate at GaussianMixture.scala:201, took 0.024487 s -2016-06-13 17:44:54 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Got job 54 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:44:54 INFO DAGScheduler:59 - Final stage: ResultStage 54(first at GaussianMixture.scala:171) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=1231422, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 1992.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1233414, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 1250.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:62254 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 77 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 54 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 1 tasks -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 177, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:54 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 177) -2016-06-13 17:44:54 INFO CacheManager:59 - Partition rdd_23_0 not found, computing it -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=1234664, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block rdd_23_0 stored as values in memory (estimated size 16.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added rdd_23_0 in memory on localhost:62254 (size: 16.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 177). 1533 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 177) in 8 ms on localhost (1/1) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-06-13 17:44:54 INFO DAGScheduler:59 - ResultStage 54 (first at GaussianMixture.scala:171) finished in 0.009 s -2016-06-13 17:44:54 INFO DAGScheduler:59 - Job 54 finished: first at GaussianMixture.scala:171, took 0.021985 s -2016-06-13 17:44:54 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Got job 55 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:44:54 INFO DAGScheduler:59 - Final stage: ResultStage 55(first at GaussianMixture.scala:171) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=1234680, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 1992.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1236672, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 1250.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:62254 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 55 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 3 tasks -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 178, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 179, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 55.0 (TID 180, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:54 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 178) -2016-06-13 17:44:54 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 179) -2016-06-13 17:44:54 INFO Executor:59 - Running task 2.0 in stage 55.0 (TID 180) -2016-06-13 17:44:54 INFO CacheManager:59 - Partition rdd_23_1 not found, computing it -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=1237922, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block rdd_23_1 stored as values in memory (estimated size 192.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added rdd_23_1 in memory on localhost:62254 (size: 192.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO CacheManager:59 - Partition rdd_23_2 not found, computing it -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=1238114, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block rdd_23_2 stored as values in memory (estimated size 192.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added rdd_23_2 in memory on localhost:62254 (size: 192.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 178). 2144 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 178) in 21 ms on localhost (1/3) -2016-06-13 17:44:54 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 179). 2144 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 179) in 23 ms on localhost (2/3) -2016-06-13 17:44:54 INFO CacheManager:59 - Partition rdd_23_3 not found, computing it -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=1238306, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block rdd_23_3 stored as values in memory (estimated size 184.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added rdd_23_3 in memory on localhost:62254 (size: 184.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO Executor:59 - Finished task 2.0 in stage 55.0 (TID 180). 2132 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 55.0 (TID 180) in 28 ms on localhost (3/3) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-06-13 17:44:54 INFO DAGScheduler:59 - ResultStage 55 (first at GaussianMixture.scala:171) finished in 0.034 s -2016-06-13 17:44:54 INFO DAGScheduler:59 - Job 55 finished: first at GaussianMixture.scala:171, took 0.047025 s -2016-06-13 17:44:54 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Got job 56 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:54 INFO DAGScheduler:59 - Final stage: ResultStage 56(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=1238490, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 1832.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=1240322, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 1193.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:62254 (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 79 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 56 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 4 tasks -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 181, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 182, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 56.0 (TID 183, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 3.0 in stage 56.0 (TID 184, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:54 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 181) -2016-06-13 17:44:54 INFO Executor:59 - Running task 2.0 in stage 56.0 (TID 183) -2016-06-13 17:44:54 INFO Executor:59 - Running task 3.0 in stage 56.0 (TID 184) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_23_2 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 2.0 in stage 56.0 (TID 183). 2082 bytes result sent to driver -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_23_3 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 3.0 in stage 56.0 (TID 184). 2082 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 56.0 (TID 183) in 5 ms on localhost (1/4) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_23_0 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 181). 2082 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 3.0 in stage 56.0 (TID 184) in 7 ms on localhost (2/4) -2016-06-13 17:44:54 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 182) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_23_1 locally -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 181) in 11 ms on localhost (3/4) -2016-06-13 17:44:54 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 182). 2082 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 182) in 12 ms on localhost (4/4) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-06-13 17:44:54 INFO DAGScheduler:59 - ResultStage 56 (takeSample at GaussianMixture.scala:184) finished in 0.013 s -2016-06-13 17:44:54 INFO DAGScheduler:59 - Job 56 finished: takeSample at GaussianMixture.scala:184, took 0.027422 s -2016-06-13 17:44:54 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Got job 57 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:54 INFO DAGScheduler:59 - Final stage: ResultStage 57(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting ResultStage 57 (PartitionwiseSampledRDD[24] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=1241515, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 18.8 KB, free 729.4 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(7876) called with curMem=1260739, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 7.7 KB, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:62254 (size: 7.7 KB, free: 730.4 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 57 (PartitionwiseSampledRDD[24] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 4 tasks -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 185, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 186, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 57.0 (TID 187, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 3.0 in stage 57.0 (TID 188, localhost, PROCESS_LOCAL, 2373 bytes) -2016-06-13 17:44:54 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 185) -2016-06-13 17:44:54 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 186) -2016-06-13 17:44:54 INFO Executor:59 - Running task 2.0 in stage 57.0 (TID 187) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_23_0 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 185). 2048 bytes result sent to driver -2016-06-13 17:44:54 INFO Executor:59 - Running task 3.0 in stage 57.0 (TID 188) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_23_2 locally -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_23_3 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 2.0 in stage 57.0 (TID 187). 2714 bytes result sent to driver -2016-06-13 17:44:54 INFO Executor:59 - Finished task 3.0 in stage 57.0 (TID 188). 2682 bytes result sent to driver -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_23_1 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 186). 2724 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 185) in 18 ms on localhost (1/4) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 3.0 in stage 57.0 (TID 188) in 16 ms on localhost (2/4) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 57.0 (TID 187) in 18 ms on localhost (3/4) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 186) in 19 ms on localhost (4/4) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-06-13 17:44:54 INFO DAGScheduler:59 - ResultStage 57 (takeSample at GaussianMixture.scala:184) finished in 0.021 s -2016-06-13 17:44:54 INFO DAGScheduler:59 - Job 57 finished: takeSample at GaussianMixture.scala:184, took 0.047879 s -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=1268615, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 560.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(891) called with curMem=1269175, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 891.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:62254 (size: 891.0 B, free: 730.4 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 81 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:62254 in memory (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Got job 58 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:54 INFO DAGScheduler:59 - Final stage: ResultStage 58(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting ResultStage 58 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:62254 in memory (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=1239941, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=1244005, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 58 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 4 tasks -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 189, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:62254 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 190, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 58.0 (TID 191, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 3.0 in stage 58.0 (TID 192, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:54 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 189) -2016-06-13 17:44:54 INFO Executor:59 - Running task 2.0 in stage 58.0 (TID 191) -2016-06-13 17:44:54 INFO Executor:59 - Running task 3.0 in stage 58.0 (TID 192) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_23_0 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 189). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_23_3 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 3.0 in stage 58.0 (TID 192). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 190) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:62254 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 3.0 in stage 58.0 (TID 192) in 8 ms on localhost (1/4) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 189) in 13 ms on localhost (2/4) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_23_2 locally -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_23_1 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 2.0 in stage 58.0 (TID 191). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 190). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 58.0 (TID 191) in 12 ms on localhost (3/4) -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-06-13 17:44:54 INFO DAGScheduler:59 - ResultStage 58 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 190) in 14 ms on localhost (4/4) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-06-13 17:44:54 INFO DAGScheduler:59 - Job 58 finished: aggregate at GaussianMixture.scala:201, took 0.033680 s -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=1237521, maxMem=766075207 -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 560.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=1234017, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 900.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:62254 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:62254 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 83 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-06-13 17:44:54 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Got job 59 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:54 INFO DAGScheduler:59 - Final stage: ResultStage 59(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:62254 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting ResultStage 59 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=1221675, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=1223551, maxMem=766075207 -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.4 MB) -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 59 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 4 tasks -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:62254 in memory (size: 890.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 193, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 194, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 59.0 (TID 195, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 3.0 in stage 59.0 (TID 196, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:54 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 193) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_23_0 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 193). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO Executor:59 - Running task 3.0 in stage 59.0 (TID 196) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:62254 in memory (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:44:54 INFO Executor:59 - Running task 2.0 in stage 59.0 (TID 195) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_23_3 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 3.0 in stage 59.0 (TID 196). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 194) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_23_2 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 2.0 in stage 59.0 (TID 195). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 193) in 14 ms on localhost (1/4) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_23_1 locally -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 59.0 (TID 195) in 15 ms on localhost (2/4) -2016-06-13 17:44:54 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 194). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 3.0 in stage 59.0 (TID 196) in 16 ms on localhost (3/4) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 194) in 19 ms on localhost (4/4) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-06-13 17:44:54 INFO DAGScheduler:59 - ResultStage 59 (aggregate at GaussianMixture.scala:201) finished in 0.021 s -2016-06-13 17:44:54 INFO DAGScheduler:59 - Job 59 finished: aggregate at GaussianMixture.scala:201, took 0.033968 s -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=1195357, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 560.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:62254 in memory (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=1194724, maxMem=766075207 -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 900.0 B, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:62254 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 85 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:62254 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-06-13 17:44:54 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:62254 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Got job 60 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:54 INFO DAGScheduler:59 - Final stage: ResultStage 60(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-06-13 17:44:54 INFO BlockManager:59 - Removing RDD 21 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting ResultStage 60 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned RDD 21 -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=1188556, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:62254 in memory (size: 1389.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=1191231, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.5 MB) -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 60 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Adding task set 60.0 with 4 tasks -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 60.0 (TID 197, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 60.0 (TID 198, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 60.0 (TID 199, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 3.0 in stage 60.0 (TID 200, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:54 INFO Executor:59 - Running task 0.0 in stage 60.0 (TID 197) -2016-06-13 17:44:54 INFO Executor:59 - Running task 2.0 in stage 60.0 (TID 199) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_23_0 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 0.0 in stage 60.0 (TID 197). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO Executor:59 - Running task 1.0 in stage 60.0 (TID 198) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_23_1 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 1.0 in stage 60.0 (TID 198). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_23_2 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 2.0 in stage 60.0 (TID 199). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO Executor:59 - Running task 3.0 in stage 60.0 (TID 200) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 60.0 (TID 197) in 15 ms on localhost (1/4) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_23_3 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 3.0 in stage 60.0 (TID 200). 2756 bytes result sent to driver -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:62254 in memory (size: 29.4 KB, free: 730.5 MB) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 60.0 (TID 199) in 21 ms on localhost (2/4) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 3.0 in stage 60.0 (TID 200) in 21 ms on localhost (3/4) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 60.0 (TID 198) in 23 ms on localhost (4/4) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 60.0, whose tasks have all completed, from pool -2016-06-13 17:44:54 INFO DAGScheduler:59 - ResultStage 60 (aggregate at GaussianMixture.scala:201) finished in 0.027 s -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Job 60 finished: aggregate at GaussianMixture.scala:201, took 0.048357 s -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:62254 in memory (size: 23.3 KB, free: 730.5 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:62254 in memory (size: 1389.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-06-13 17:44:54 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Got job 61 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:44:54 INFO DAGScheduler:59 - Final stage: ResultStage 61(first at GaussianMixture.scala:171) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting ResultStage 61 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:62254 in memory (size: 17.9 KB, free: 730.5 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=598972, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 1992.0 B, free 730.0 MB) -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=290308, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.3 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:62254 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 87 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 61 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 1 tasks -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:62254 in memory (size: 11.8 KB, free: 730.6 MB) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 201, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:54 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 201) -2016-06-13 17:44:54 INFO CacheManager:59 - Partition rdd_25_0 not found, computing it -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=46883, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block rdd_25_0 stored as values in memory (estimated size 16.0 B, free 730.5 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:62254 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added rdd_25_0 in memory on localhost:62254 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-06-13 17:44:54 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 201). 1533 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 201) in 9 ms on localhost (1/1) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-06-13 17:44:54 INFO DAGScheduler:59 - ResultStage 61 (first at GaussianMixture.scala:171) finished in 0.010 s -2016-06-13 17:44:54 INFO DAGScheduler:59 - Job 61 finished: first at GaussianMixture.scala:171, took 0.021049 s -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:62254 in memory (size: 1193.0 B, free: 730.6 MB) -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-06-13 17:44:54 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Got job 62 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:44:54 INFO DAGScheduler:59 - Final stage: ResultStage 62(first at GaussianMixture.scala:171) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:62254 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:54 INFO BlockManager:59 - Removing RDD 14 -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=34632, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned RDD 14 -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=31088, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.6 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:62254 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 62 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 3 tasks -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:62254 in memory (size: 837.0 B, free: 730.6 MB) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 202, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 62.0 (TID 203, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 62.0 (TID 204, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:54 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 202) -2016-06-13 17:44:54 INFO Executor:59 - Running task 2.0 in stage 62.0 (TID 204) -2016-06-13 17:44:54 INFO Executor:59 - Running task 1.0 in stage 62.0 (TID 203) -2016-06-13 17:44:54 INFO CacheManager:59 - Partition rdd_25_2 not found, computing it -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=30229, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block rdd_25_2 stored as values in memory (estimated size 192.0 B, free 730.6 MB) -2016-06-13 17:44:54 INFO CacheManager:59 - Partition rdd_25_3 not found, computing it -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added rdd_25_2 in memory on localhost:62254 (size: 192.0 B, free: 730.6 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=30421, maxMem=766075207 -2016-06-13 17:44:54 INFO CacheManager:59 - Partition rdd_25_1 not found, computing it -2016-06-13 17:44:54 INFO MemoryStore:59 - Block rdd_25_3 stored as values in memory (estimated size 184.0 B, free 730.6 MB) -2016-06-13 17:44:54 INFO Executor:59 - Finished task 1.0 in stage 62.0 (TID 203). 2144 bytes result sent to driver -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added rdd_25_3 in memory on localhost:62254 (size: 184.0 B, free: 730.6 MB) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 62.0 (TID 203) in 10 ms on localhost (1/3) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=30605, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block rdd_25_1 stored as values in memory (estimated size 192.0 B, free 730.6 MB) -2016-06-13 17:44:54 INFO Executor:59 - Finished task 2.0 in stage 62.0 (TID 204). 2132 bytes result sent to driver -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added rdd_25_1 in memory on localhost:62254 (size: 192.0 B, free: 730.6 MB) -2016-06-13 17:44:54 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 202). 2144 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 62.0 (TID 204) in 14 ms on localhost (2/3) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 202) in 16 ms on localhost (3/3) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-06-13 17:44:54 INFO DAGScheduler:59 - ResultStage 62 (first at GaussianMixture.scala:171) finished in 0.017 s -2016-06-13 17:44:54 INFO DAGScheduler:59 - Job 62 finished: first at GaussianMixture.scala:171, took 0.029228 s -2016-06-13 17:44:54 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Got job 63 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:54 INFO DAGScheduler:59 - Final stage: ResultStage 63(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:54 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting ResultStage 63 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=30797, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 17:44:54 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=32629, maxMem=766075207 -2016-06-13 17:44:54 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 1193.0 B, free 730.6 MB) -2016-06-13 17:44:54 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:62254 (size: 1193.0 B, free: 730.6 MB) -2016-06-13 17:44:54 INFO SparkContext:59 - Created broadcast 89 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:54 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 63 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168) -2016-06-13 17:44:54 INFO TaskSchedulerImpl:59 - Adding task set 63.0 with 4 tasks -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 0.0 in stage 63.0 (TID 205, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 1.0 in stage 63.0 (TID 206, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 2.0 in stage 63.0 (TID 207, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Starting task 3.0 in stage 63.0 (TID 208, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:54 INFO Executor:59 - Running task 2.0 in stage 63.0 (TID 207) -2016-06-13 17:44:54 INFO Executor:59 - Running task 1.0 in stage 63.0 (TID 206) -2016-06-13 17:44:54 INFO Executor:59 - Running task 3.0 in stage 63.0 (TID 208) -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 1.0 in stage 63.0 (TID 206). 2082 bytes result sent to driver -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_25_3 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 3.0 in stage 63.0 (TID 208). 2082 bytes result sent to driver -2016-06-13 17:44:54 INFO BlockManager:59 - Found block rdd_25_2 locally -2016-06-13 17:44:54 INFO Executor:59 - Finished task 2.0 in stage 63.0 (TID 207). 2082 bytes result sent to driver -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 3.0 in stage 63.0 (TID 208) in 7 ms on localhost (1/4) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 2.0 in stage 63.0 (TID 207) in 9 ms on localhost (2/4) -2016-06-13 17:44:54 INFO TaskSetManager:59 - Finished task 1.0 in stage 63.0 (TID 206) in 10 ms on localhost (3/4) -2016-06-13 17:44:54 INFO Executor:59 - Running task 0.0 in stage 63.0 (TID 205) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 63.0 (TID 205). 2082 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 63.0 (TID 205) in 17 ms on localhost (4/4) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 63.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 63 (takeSample at GaussianMixture.scala:184) finished in 0.018 s -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 63 finished: takeSample at GaussianMixture.scala:184, took 0.032280 s -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 64 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 64(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 64 (PartitionwiseSampledRDD[26] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=33822, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(7888) called with curMem=53046, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:62254 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 64 (PartitionwiseSampledRDD[26] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 64.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 64.0 (TID 209, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 64.0 (TID 210, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 64.0 (TID 211, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 64.0 (TID 212, localhost, PROCESS_LOCAL, 2373 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 64.0 (TID 210) -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 64.0 (TID 212) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 64.0 (TID 211) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_25_3 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 64.0 (TID 212). 2702 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 64.0 (TID 210). 2684 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 64.0 (TID 209) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 64.0 (TID 212) in 7 ms on localhost (1/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 64.0 (TID 210) in 11 ms on localhost (2/4) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_25_2 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 64.0 (TID 211). 2714 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 64.0 (TID 209). 2048 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 64.0 (TID 211) in 13 ms on localhost (3/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 64.0 (TID 209) in 16 ms on localhost (4/4) -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 64 (takeSample at GaussianMixture.scala:184) finished in 0.016 s -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 64 finished: takeSample at GaussianMixture.scala:184, took 0.031293 s -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 64.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=60934, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(899) called with curMem=61494, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 899.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:62254 (size: 899.0 B, free: 730.6 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 91 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 65 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 65(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=62393, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(2204) called with curMem=66457, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:62254 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 65 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 213, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 214, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 65.0 (TID 215, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 65.0 (TID 216, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 213) -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 65.0 (TID 216) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 65.0 (TID 215) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 214) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 213). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_25_3 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_25_2 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 65.0 (TID 216). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 65.0 (TID 215). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 213) in 13 ms on localhost (1/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 65.0 (TID 216) in 11 ms on localhost (2/4) -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 214). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 65.0 (TID 215) in 13 ms on localhost (3/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 214) in 17 ms on localhost (4/4) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 65 (aggregate at GaussianMixture.scala:201) finished in 0.018 s -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 65 finished: aggregate at GaussianMixture.scala:201, took 0.033153 s -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=68661, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=69221, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:62254 (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 93 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 66 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 66(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 66 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=70121, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=74185, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 66 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 66.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 66.0 (TID 217, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 66.0 (TID 218, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 66.0 (TID 219, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 66.0 (TID 220, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 66.0 (TID 217) -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 66.0 (TID 220) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 66.0 (TID 219) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 66.0 (TID 218) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 66.0 (TID 217). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_25_2 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 66.0 (TID 219). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 66.0 (TID 218). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_25_3 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 66.0 (TID 220). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 66.0 (TID 218) in 8 ms on localhost (1/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 66.0 (TID 219) in 7 ms on localhost (2/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 66.0 (TID 217) in 9 ms on localhost (3/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 66.0 (TID 220) in 8 ms on localhost (4/4) -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 66 (aggregate at GaussianMixture.scala:201) finished in 0.010 s -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 66.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 66 finished: aggregate at GaussianMixture.scala:201, took 0.022875 s -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=76379, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=76939, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:62254 (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 95 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 67 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 67(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 67 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=77839, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=81903, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 67 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 67.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 67.0 (TID 221, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 67.0 (TID 222, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 67.0 (TID 223, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 67.0 (TID 224, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 67.0 (TID 221) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 67.0 (TID 223) -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 67.0 (TID 224) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_25_2 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 67.0 (TID 223). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 67.0 (TID 222) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 67.0 (TID 223) in 7 ms on localhost (1/4) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_25_3 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 67.0 (TID 221). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 67.0 (TID 222). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 67.0 (TID 221) in 14 ms on localhost (2/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 67.0 (TID 222) in 14 ms on localhost (3/4) -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 67.0 (TID 224). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 67.0 (TID 224) in 15 ms on localhost (4/4) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 67.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 67 (aggregate at GaussianMixture.scala:201) finished in 0.018 s -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 67 finished: aggregate at GaussianMixture.scala:201, took 0.032647 s -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 68 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 68(first at GaussianMixture.scala:171) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 68 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=84097, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=86089, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:62254 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 97 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 68 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 68.0 with 1 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 68.0 (TID 225, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 68.0 (TID 225) -2016-06-13 17:44:55 INFO CacheManager:59 - Partition rdd_27_0 not found, computing it -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=87339, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block rdd_27_0 stored as values in memory (estimated size 16.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added rdd_27_0 in memory on localhost:62254 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 68.0 (TID 225). 1533 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 68.0 (TID 225) in 7 ms on localhost (1/1) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 68.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 68 (first at GaussianMixture.scala:171) finished in 0.007 s -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 68 finished: first at GaussianMixture.scala:171, took 0.019457 s -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 69 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 69(first at GaussianMixture.scala:171) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 69 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=87355, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=89347, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:62254 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 69 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 69.0 with 3 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 69.0 (TID 226, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 69.0 (TID 227, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 69.0 (TID 228, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 69.0 (TID 226) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 69.0 (TID 227) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 69.0 (TID 228) -2016-06-13 17:44:55 INFO CacheManager:59 - Partition rdd_27_1 not found, computing it -2016-06-13 17:44:55 INFO CacheManager:59 - Partition rdd_27_3 not found, computing it -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=90597, maxMem=766075207 -2016-06-13 17:44:55 INFO CacheManager:59 - Partition rdd_27_2 not found, computing it -2016-06-13 17:44:55 INFO MemoryStore:59 - Block rdd_27_1 stored as values in memory (estimated size 192.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=90789, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block rdd_27_3 stored as values in memory (estimated size 184.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added rdd_27_1 in memory on localhost:62254 (size: 192.0 B, free: 730.6 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=90973, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block rdd_27_2 stored as values in memory (estimated size 192.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added rdd_27_3 in memory on localhost:62254 (size: 184.0 B, free: 730.6 MB) -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 69.0 (TID 226). 2144 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 69.0 (TID 228). 2132 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added rdd_27_2 in memory on localhost:62254 (size: 192.0 B, free: 730.6 MB) -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 69.0 (TID 227). 2144 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 69.0 (TID 226) in 11 ms on localhost (1/3) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 69.0 (TID 228) in 10 ms on localhost (2/3) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 69.0 (TID 227) in 12 ms on localhost (3/3) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 69.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 69 (first at GaussianMixture.scala:171) finished in 0.013 s -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 69 finished: first at GaussianMixture.scala:171, took 0.024084 s -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 70 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 70(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 70 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=91165, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 1832.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=92997, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 1193.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:62254 (size: 1193.0 B, free: 730.6 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 99 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 70 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 70.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 70.0 (TID 229, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 70.0 (TID 230, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 70.0 (TID 231, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 70.0 (TID 232, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 70.0 (TID 230) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 70.0 (TID 229) -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 70.0 (TID 232) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 70.0 (TID 229). 2082 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 70.0 (TID 231) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 70.0 (TID 230). 2082 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_27_3 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_27_2 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 70.0 (TID 232). 2082 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 70.0 (TID 231). 2082 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 70.0 (TID 230) in 9 ms on localhost (1/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 70.0 (TID 232) in 9 ms on localhost (2/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 70.0 (TID 229) in 11 ms on localhost (3/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 70.0 (TID 231) in 11 ms on localhost (4/4) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 70.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 70 (takeSample at GaussianMixture.scala:184) finished in 0.014 s -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 70 finished: takeSample at GaussianMixture.scala:184, took 0.022674 s -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 71 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 71(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 71 (PartitionwiseSampledRDD[28] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=94190, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(7884) called with curMem=113414, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:62254 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 71 (PartitionwiseSampledRDD[28] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 71.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 71.0 (TID 233, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 71.0 (TID 234, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 71.0 (TID 235, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 71.0 (TID 236, localhost, PROCESS_LOCAL, 2373 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 71.0 (TID 233) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 71.0 (TID 235) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 71.0 (TID 234) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_27_2 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 71.0 (TID 235). 2714 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 71.0 (TID 236) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_27_3 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 71.0 (TID 234). 2719 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 71.0 (TID 236). 2722 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 71.0 (TID 233). 2048 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 71.0 (TID 235) in 9 ms on localhost (1/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 71.0 (TID 234) in 10 ms on localhost (2/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 71.0 (TID 236) in 9 ms on localhost (3/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 71.0 (TID 233) in 13 ms on localhost (4/4) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 71.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 71 (takeSample at GaussianMixture.scala:184) finished in 0.013 s -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 71 finished: takeSample at GaussianMixture.scala:184, took 0.049802 s -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=121298, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(895) called with curMem=121858, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 895.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:62254 (size: 895.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 101 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 72 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 72(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 72 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=122753, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(2197) called with curMem=126817, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 72 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 72.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 72.0 (TID 237, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 72.0 (TID 238, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 72.0 (TID 239, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 72.0 (TID 240, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 72.0 (TID 239) -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 72.0 (TID 240) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 72.0 (TID 238) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_27_3 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 72.0 (TID 240). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 72.0 (TID 237) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_27_2 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 72.0 (TID 238). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 72.0 (TID 239). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 72.0 (TID 237). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 72.0 (TID 238) in 17 ms on localhost (1/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 72.0 (TID 240) in 17 ms on localhost (2/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 72.0 (TID 239) in 18 ms on localhost (3/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 72.0 (TID 237) in 20 ms on localhost (4/4) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 72.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 72 (aggregate at GaussianMixture.scala:201) finished in 0.023 s -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 72 finished: aggregate at GaussianMixture.scala:201, took 0.036997 s -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=129014, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=129574, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:62254 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 103 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 73 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 73(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 73 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=130474, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=134538, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 104 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 73 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 73.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 73.0 (TID 241, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 73.0 (TID 242, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 73.0 (TID 243, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 73.0 (TID 244, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 73.0 (TID 242) -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 73.0 (TID 244) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 73.0 (TID 243) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_27_3 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_27_2 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 73.0 (TID 244). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 73.0 (TID 243). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 73.0 (TID 241) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 73.0 (TID 242). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 73.0 (TID 241). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 73.0 (TID 244) in 8 ms on localhost (1/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 73.0 (TID 242) in 11 ms on localhost (2/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 73.0 (TID 243) in 10 ms on localhost (3/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 73.0 (TID 241) in 12 ms on localhost (4/4) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 73.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 73 (aggregate at GaussianMixture.scala:201) finished in 0.013 s -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 73 finished: aggregate at GaussianMixture.scala:201, took 0.025215 s -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=136732, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=137292, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.5 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:62254 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 105 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 74 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 74(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 74 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=138192, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=142256, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 106 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 74 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 74.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 74.0 (TID 245, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 74.0 (TID 246, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 74.0 (TID 247, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 74.0 (TID 248, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 74.0 (TID 246) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 74.0 (TID 247) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 74.0 (TID 245) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 74.0 (TID 246). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 74.0 (TID 245). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 74.0 (TID 248) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_27_2 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_27_3 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 74.0 (TID 247). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 74.0 (TID 248). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 74.0 (TID 246) in 11 ms on localhost (1/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 74.0 (TID 245) in 12 ms on localhost (2/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 74.0 (TID 248) in 12 ms on localhost (3/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 74.0 (TID 247) in 13 ms on localhost (4/4) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 74.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 74 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 74 finished: aggregate at GaussianMixture.scala:201, took 0.026971 s -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 75 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 75(first at GaussianMixture.scala:171) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 75 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=144450, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 1992.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=146442, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:62254 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 107 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 75 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 75.0 with 1 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 75.0 (TID 249, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 75.0 (TID 249) -2016-06-13 17:44:55 INFO CacheManager:59 - Partition rdd_29_0 not found, computing it -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=147692, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block rdd_29_0 stored as values in memory (estimated size 16.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added rdd_29_0 in memory on localhost:62254 (size: 16.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 75.0 (TID 249). 1533 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 75.0 (TID 249) in 10 ms on localhost (1/1) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 75.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 75 (first at GaussianMixture.scala:171) finished in 0.011 s -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 75 finished: first at GaussianMixture.scala:171, took 0.023178 s -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 76 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 76(first at GaussianMixture.scala:171) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 76 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=147708, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 1992.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=149700, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:62254 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 108 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 76 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 3 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 250, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 251, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 76.0 (TID 252, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 250) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 76.0 (TID 252) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 251) -2016-06-13 17:44:55 INFO CacheManager:59 - Partition rdd_29_3 not found, computing it -2016-06-13 17:44:55 INFO CacheManager:59 - Partition rdd_29_1 not found, computing it -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=150950, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block rdd_29_3 stored as values in memory (estimated size 184.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO CacheManager:59 - Partition rdd_29_2 not found, computing it -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added rdd_29_3 in memory on localhost:62254 (size: 184.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=151134, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block rdd_29_1 stored as values in memory (estimated size 192.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 76.0 (TID 252). 2132 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added rdd_29_1 in memory on localhost:62254 (size: 192.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=151326, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block rdd_29_2 stored as values in memory (estimated size 192.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 76.0 (TID 252) in 17 ms on localhost (1/3) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added rdd_29_2 in memory on localhost:62254 (size: 192.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 250). 2144 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 251). 2144 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 250) in 21 ms on localhost (2/3) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 251) in 20 ms on localhost (3/3) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 76 (first at GaussianMixture.scala:171) finished in 0.022 s -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 76 finished: first at GaussianMixture.scala:171, took 0.034876 s -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 77 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 77(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=151518, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 1832.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=153350, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 1193.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:62254 (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 109 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 77 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 253, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 254, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 77.0 (TID 255, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 77.0 (TID 256, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 254) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 77.0 (TID 255) -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 77.0 (TID 256) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_29_2 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 77.0 (TID 255). 2082 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 253) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_29_0 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 253). 2082 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_29_3 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_29_1 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 254). 2082 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 77.0 (TID 256). 2082 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 77.0 (TID 255) in 10 ms on localhost (1/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 253) in 12 ms on localhost (2/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 254) in 12 ms on localhost (3/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 77.0 (TID 256) in 10 ms on localhost (4/4) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 77 (takeSample at GaussianMixture.scala:184) finished in 0.013 s -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 77 finished: takeSample at GaussianMixture.scala:184, took 0.026803 s -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 78 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 78(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 78 (PartitionwiseSampledRDD[30] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=154543, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 18.8 KB, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(7883) called with curMem=173767, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:62254 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 110 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 78 (PartitionwiseSampledRDD[30] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 257, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 258, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 78.0 (TID 259, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 78.0 (TID 260, localhost, PROCESS_LOCAL, 2373 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 257) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 78.0 (TID 259) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 258) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_29_2 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 78.0 (TID 259). 2699 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 78.0 (TID 260) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_29_1 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 258). 2684 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_29_3 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 78.0 (TID 260). 2687 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_29_0 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 257). 2048 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 78.0 (TID 259) in 11 ms on localhost (1/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 258) in 12 ms on localhost (2/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 78.0 (TID 260) in 12 ms on localhost (3/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 257) in 15 ms on localhost (4/4) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 78 (takeSample at GaussianMixture.scala:184) finished in 0.015 s -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 78 finished: takeSample at GaussianMixture.scala:184, took 0.031462 s -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=181650, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 560.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(882) called with curMem=182210, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 882.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:62254 (size: 882.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 111 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 79 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 79(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 79 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=183092, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(2182) called with curMem=187156, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 112 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 79 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 79.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 79.0 (TID 261, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 79.0 (TID 262, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 79.0 (TID 263, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 79.0 (TID 264, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 79.0 (TID 262) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 79.0 (TID 263) -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 79.0 (TID 264) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 79.0 (TID 261) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_29_1 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 79.0 (TID 262). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_29_3 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_29_0 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 79.0 (TID 264). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 79.0 (TID 261). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_29_2 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 79.0 (TID 263). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 79.0 (TID 262) in 7 ms on localhost (1/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 79.0 (TID 264) in 6 ms on localhost (2/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 79.0 (TID 261) in 8 ms on localhost (3/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 79.0 (TID 263) in 8 ms on localhost (4/4) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 79.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 79 (aggregate at GaussianMixture.scala:201) finished in 0.009 s -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 79 finished: aggregate at GaussianMixture.scala:201, took 0.023144 s -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=189338, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 560.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=189898, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:62254 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 113 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 80 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 80(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 80 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=190798, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(2192) called with curMem=194862, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 114 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 80 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 80.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 80.0 (TID 265, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 80.0 (TID 266, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 80.0 (TID 267, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 80.0 (TID 268, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 80.0 (TID 265) -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 80.0 (TID 268) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 80.0 (TID 267) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 80.0 (TID 266) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_29_0 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_29_1 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 80.0 (TID 265). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_29_2 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 80.0 (TID 266). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 80.0 (TID 267). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 80.0 (TID 265) in 6 ms on localhost (1/4) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_29_3 locally -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 80.0 (TID 266) in 5 ms on localhost (2/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 80.0 (TID 267) in 5 ms on localhost (3/4) -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 80.0 (TID 268). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 80.0 (TID 268) in 6 ms on localhost (4/4) -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 80 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 80.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 80 finished: aggregate at GaussianMixture.scala:201, took 0.016481 s -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=197054, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_115 stored as values in memory (estimated size 560.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=197614, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_115_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_115_piece0 in memory on localhost:62254 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 115 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 81 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 81(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 81 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=198514, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_116 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(2192) called with curMem=202578, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_116_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_116_piece0 in memory on localhost:62254 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 116 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 81 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 81.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 81.0 (TID 269, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 81.0 (TID 270, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 81.0 (TID 271, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 81.0 (TID 272, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 81.0 (TID 269) -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 81.0 (TID 272) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 81.0 (TID 271) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 81.0 (TID 270) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_29_0 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_29_1 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 81.0 (TID 269). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 81.0 (TID 270). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_29_2 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_29_3 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 81.0 (TID 271). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 81.0 (TID 269) in 6 ms on localhost (1/4) -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 81.0 (TID 272). 2756 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 81.0 (TID 270) in 6 ms on localhost (2/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 81.0 (TID 271) in 6 ms on localhost (3/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 81.0 (TID 272) in 6 ms on localhost (4/4) -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 81 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 81.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 81 finished: aggregate at GaussianMixture.scala:201, took 0.018377 s -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 82 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 82(first at GaussianMixture.scala:171) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 82 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(2184) called with curMem=204770, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_117 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(1348) called with curMem=206954, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_117_piece0 stored as bytes in memory (estimated size 1348.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_117_piece0 in memory on localhost:62254 (size: 1348.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 117 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 82 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 82.0 with 1 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 82.0 (TID 273, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 82.0 (TID 273) -2016-06-13 17:44:55 INFO CacheManager:59 - Partition rdd_33_0 not found, computing it -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(480) called with curMem=208302, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block rdd_33_0 stored as values in memory (estimated size 480.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added rdd_33_0 in memory on localhost:62254 (size: 480.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 82.0 (TID 273). 2132 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 82.0 (TID 273) in 6 ms on localhost (1/1) -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 82 (first at GaussianMixture.scala:171) finished in 0.006 s -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 82.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 82 finished: first at GaussianMixture.scala:171, took 0.013732 s -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=208782, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_118 stored as values in memory (estimated size 712.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=209494, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_118_piece0 stored as bytes in memory (estimated size 880.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_118_piece0 in memory on localhost:62254 (size: 880.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 118 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 83 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 83(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 83 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(4288) called with curMem=210374, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_119 stored as values in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(2279) called with curMem=214662, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_119_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_119_piece0 in memory on localhost:62254 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 119 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 83 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 83.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 83.0 (TID 274, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 83.0 (TID 275, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 83.0 (TID 276, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 83.0 (TID 277, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 83.0 (TID 275) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 83.0 (TID 274) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 83.0 (TID 276) -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 83.0 (TID 277) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_33_0 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 83.0 (TID 274). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO CacheManager:59 - Partition rdd_33_3 not found, computing it -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 83.0 (TID 274) in 7 ms on localhost (1/4) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(624) called with curMem=216941, maxMem=766075207 -2016-06-13 17:44:55 INFO CacheManager:59 - Partition rdd_33_2 not found, computing it -2016-06-13 17:44:55 INFO MemoryStore:59 - Block rdd_33_3 stored as values in memory (estimated size 624.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO CacheManager:59 - Partition rdd_33_1 not found, computing it -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(624) called with curMem=217565, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block rdd_33_2 stored as values in memory (estimated size 624.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(624) called with curMem=218189, maxMem=766075207 -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added rdd_33_3 in memory on localhost:62254 (size: 624.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - Block rdd_33_1 stored as values in memory (estimated size 624.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added rdd_33_2 in memory on localhost:62254 (size: 624.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added rdd_33_1 in memory on localhost:62254 (size: 624.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 83.0 (TID 277). 2247 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 83.0 (TID 276). 2247 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 83.0 (TID 275). 2247 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 83.0 (TID 277) in 16 ms on localhost (2/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 83.0 (TID 276) in 17 ms on localhost (3/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 83.0 (TID 275) in 18 ms on localhost (4/4) -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 83 (aggregate at GaussianMixture.scala:201) finished in 0.019 s -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 83.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 83 finished: aggregate at GaussianMixture.scala:201, took 0.027559 s -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=218813, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_120 stored as values in memory (estimated size 712.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(934) called with curMem=219525, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_120_piece0 stored as bytes in memory (estimated size 934.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_120_piece0 in memory on localhost:62254 (size: 934.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 120 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 84 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 84(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 84 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(4288) called with curMem=220459, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_121 stored as values in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(2343) called with curMem=224747, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_121_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_121_piece0 in memory on localhost:62254 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 121 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 84 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 84.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 84.0 (TID 278, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 84.0 (TID 279, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 84.0 (TID 280, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 84.0 (TID 281, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 84.0 (TID 278) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 84.0 (TID 280) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 84.0 (TID 279) -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 84.0 (TID 281) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_33_0 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_33_3 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_33_2 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_33_1 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 84.0 (TID 281). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 84.0 (TID 280). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 84.0 (TID 278). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 84.0 (TID 279). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 84.0 (TID 281) in 6 ms on localhost (1/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 84.0 (TID 280) in 7 ms on localhost (2/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 84.0 (TID 278) in 8 ms on localhost (3/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 84.0 (TID 279) in 8 ms on localhost (4/4) -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 84 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 84.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 84 finished: aggregate at GaussianMixture.scala:201, took 0.018018 s -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=227090, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_122 stored as values in memory (estimated size 712.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=227802, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_122_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_122_piece0 in memory on localhost:62254 (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 122 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 85 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 85(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 85 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(4288) called with curMem=228734, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_123 stored as values in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(2340) called with curMem=233022, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_123_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_123_piece0 in memory on localhost:62254 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 123 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 85 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 85.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 85.0 (TID 282, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 85.0 (TID 283, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 85.0 (TID 284, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 85.0 (TID 285, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 85.0 (TID 283) -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 85.0 (TID 285) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 85.0 (TID 284) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 85.0 (TID 282) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_33_1 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_33_3 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_33_0 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 85.0 (TID 283). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 85.0 (TID 285). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 85.0 (TID 282). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 85.0 (TID 283) in 6 ms on localhost (1/4) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_33_2 locally -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 85.0 (TID 285) in 5 ms on localhost (2/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 85.0 (TID 282) in 6 ms on localhost (3/4) -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 85.0 (TID 284). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 85.0 (TID 284) in 8 ms on localhost (4/4) -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 85 (aggregate at GaussianMixture.scala:201) finished in 0.009 s -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 85.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 85 finished: aggregate at GaussianMixture.scala:201, took 0.018200 s -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 86 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 86(first at GaussianMixture.scala:171) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 86 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=235362, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_124 stored as values in memory (estimated size 1992.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=237354, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_124_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_124_piece0 in memory on localhost:62254 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 124 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 86 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 86.0 with 1 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 86.0 (TID 286, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 86.0 (TID 286) -2016-06-13 17:44:55 INFO CacheManager:59 - Partition rdd_35_0 not found, computing it -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(224) called with curMem=238604, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block rdd_35_0 stored as values in memory (estimated size 224.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added rdd_35_0 in memory on localhost:62254 (size: 224.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 86.0 (TID 286). 1688 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 86.0 (TID 286) in 5 ms on localhost (1/1) -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 86 (first at GaussianMixture.scala:171) finished in 0.005 s -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 86.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 86 finished: first at GaussianMixture.scala:171, took 0.013639 s -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 87 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 87(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 87 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=238828, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_125 stored as values in memory (estimated size 1832.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=240660, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_125_piece0 stored as bytes in memory (estimated size 1193.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_125_piece0 in memory on localhost:62254 (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 125 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 87 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 87.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 87.0 (TID 287, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 87.0 (TID 288, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 87.0 (TID 289, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 87.0 (TID 290, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 87.0 (TID 288) -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 87.0 (TID 290) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 87.0 (TID 289) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 87.0 (TID 287) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 87.0 (TID 287). 2082 bytes result sent to driver -2016-06-13 17:44:55 INFO CacheManager:59 - Partition rdd_35_1 not found, computing it -2016-06-13 17:44:55 INFO CacheManager:59 - Partition rdd_35_2 not found, computing it -2016-06-13 17:44:55 INFO CacheManager:59 - Partition rdd_35_3 not found, computing it -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=241853, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block rdd_35_1 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=242141, maxMem=766075207 -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 87.0 (TID 287) in 6 ms on localhost (1/4) -2016-06-13 17:44:55 INFO MemoryStore:59 - Block rdd_35_2 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added rdd_35_1 in memory on localhost:62254 (size: 288.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=242429, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block rdd_35_3 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added rdd_35_2 in memory on localhost:62254 (size: 288.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 87.0 (TID 288). 1567 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 87.0 (TID 289). 1567 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added rdd_35_3 in memory on localhost:62254 (size: 288.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 87.0 (TID 288) in 9 ms on localhost (2/4) -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 87.0 (TID 290). 1567 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 87.0 (TID 289) in 9 ms on localhost (3/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 87.0 (TID 290) in 9 ms on localhost (4/4) -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 87 (takeSample at GaussianMixture.scala:184) finished in 0.011 s -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 87.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 87 finished: takeSample at GaussianMixture.scala:184, took 0.018842 s -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 88 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 88(takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 88 (PartitionwiseSampledRDD[36] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=242717, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_126 stored as values in memory (estimated size 18.8 KB, free 730.3 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(7882) called with curMem=261941, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_126_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.3 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_126_piece0 in memory on localhost:62254 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 126 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 88 (PartitionwiseSampledRDD[36] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 88.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 88.0 (TID 291, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 88.0 (TID 292, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 88.0 (TID 293, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 88.0 (TID 294, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 88.0 (TID 293) -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 88.0 (TID 294) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 88.0 (TID 292) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 88.0 (TID 291) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 88.0 (TID 294). 2339 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 88.0 (TID 291). 2247 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 88.0 (TID 292). 2344 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 88.0 (TID 293). 2303 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 88.0 (TID 294) in 4 ms on localhost (1/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 88.0 (TID 292) in 6 ms on localhost (2/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 88.0 (TID 291) in 6 ms on localhost (3/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 88.0 (TID 293) in 5 ms on localhost (4/4) -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 88 (takeSample at GaussianMixture.scala:184) finished in 0.006 s -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 88.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 88 finished: takeSample at GaussianMixture.scala:184, took 0.015348 s -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=269823, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_127 stored as values in memory (estimated size 712.0 B, free 730.3 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(937) called with curMem=270535, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_127_piece0 stored as bytes in memory (estimated size 937.0 B, free 730.3 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_127_piece0 in memory on localhost:62254 (size: 937.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 127 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 89 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 89(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 89 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=271472, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_128 stored as values in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=275568, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_128_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_128_piece0 in memory on localhost:62254 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 128 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 89 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 89.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 89.0 (TID 295, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 89.0 (TID 296, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 89.0 (TID 297, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 89.0 (TID 298, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 89.0 (TID 298) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 89.0 (TID 297) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 89.0 (TID 296) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 89.0 (TID 295) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 89.0 (TID 298). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 89.0 (TID 297). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 89.0 (TID 295). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 89.0 (TID 296). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 89.0 (TID 298) in 5 ms on localhost (1/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 89.0 (TID 297) in 5 ms on localhost (2/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 89.0 (TID 295) in 7 ms on localhost (3/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 89.0 (TID 296) in 7 ms on localhost (4/4) -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 89 (aggregate at GaussianMixture.scala:201) finished in 0.007 s -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 89.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 89 finished: aggregate at GaussianMixture.scala:201, took 0.015137 s -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=277806, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_129 stored as values in memory (estimated size 712.0 B, free 730.3 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(934) called with curMem=278518, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_129_piece0 stored as bytes in memory (estimated size 934.0 B, free 730.3 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_129_piece0 in memory on localhost:62254 (size: 934.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 129 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 90 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 90(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 90 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=279452, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_130 stored as values in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(2239) called with curMem=283548, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_130_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_130_piece0 in memory on localhost:62254 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 130 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 90 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 90.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 90.0 (TID 299, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 90.0 (TID 300, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 90.0 (TID 301, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 90.0 (TID 302, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 90.0 (TID 299) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 90.0 (TID 301) -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 90.0 (TID 302) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 90.0 (TID 300) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 90.0 (TID 301). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 90.0 (TID 302). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 90.0 (TID 299). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 90.0 (TID 300). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 90.0 (TID 301) in 6 ms on localhost (1/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 90.0 (TID 299) in 8 ms on localhost (2/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 90.0 (TID 302) in 7 ms on localhost (3/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 90.0 (TID 300) in 8 ms on localhost (4/4) -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 90 (aggregate at GaussianMixture.scala:201) finished in 0.009 s -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 90.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 90 finished: aggregate at GaussianMixture.scala:201, took 0.017548 s -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=285787, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_131 stored as values in memory (estimated size 712.0 B, free 730.3 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=286499, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_131_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.3 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_131_piece0 in memory on localhost:62254 (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 131 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:55 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Got job 91 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:55 INFO DAGScheduler:59 - Final stage: ResultStage 91(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:55 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting ResultStage 91 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=287431, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_132 stored as values in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(2239) called with curMem=291527, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_132_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_132_piece0 in memory on localhost:62254 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:44:55 INFO SparkContext:59 - Created broadcast 132 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:55 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 91 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Adding task set 91.0 with 4 tasks -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 0.0 in stage 91.0 (TID 303, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 1.0 in stage 91.0 (TID 304, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 2.0 in stage 91.0 (TID 305, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Starting task 3.0 in stage 91.0 (TID 306, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:55 INFO Executor:59 - Running task 0.0 in stage 91.0 (TID 303) -2016-06-13 17:44:55 INFO Executor:59 - Running task 2.0 in stage 91.0 (TID 305) -2016-06-13 17:44:55 INFO Executor:59 - Running task 1.0 in stage 91.0 (TID 304) -2016-06-13 17:44:55 INFO Executor:59 - Running task 3.0 in stage 91.0 (TID 306) -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:44:55 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:44:55 INFO Executor:59 - Finished task 1.0 in stage 91.0 (TID 304). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 2.0 in stage 91.0 (TID 305). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 0.0 in stage 91.0 (TID 303). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO Executor:59 - Finished task 3.0 in stage 91.0 (TID 306). 2762 bytes result sent to driver -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 2.0 in stage 91.0 (TID 305) in 6 ms on localhost (1/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 1.0 in stage 91.0 (TID 304) in 7 ms on localhost (2/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 3.0 in stage 91.0 (TID 306) in 6 ms on localhost (3/4) -2016-06-13 17:44:55 INFO TaskSetManager:59 - Finished task 0.0 in stage 91.0 (TID 303) in 8 ms on localhost (4/4) -2016-06-13 17:44:55 INFO DAGScheduler:59 - ResultStage 91 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 17:44:55 INFO TaskSchedulerImpl:59 - Removed TaskSet 91.0, whose tasks have all completed, from pool -2016-06-13 17:44:55 INFO DAGScheduler:59 - Job 91 finished: aggregate at GaussianMixture.scala:201, took 0.016649 s -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=293766, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_133 stored as values in memory (estimated size 712.0 B, free 730.3 MB) -2016-06-13 17:44:55 INFO MemoryStore:59 - ensureFreeSpace(936) called with curMem=294478, maxMem=766075207 -2016-06-13 17:44:55 INFO MemoryStore:59 - Block broadcast_133_piece0 stored as bytes in memory (estimated size 936.0 B, free 730.3 MB) -2016-06-13 17:44:55 INFO BlockManagerInfo:59 - Added broadcast_133_piece0 in memory on localhost:62254 (size: 936.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO SparkContext:59 - Created broadcast 133 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:56 INFO DAGScheduler:59 - Got job 92 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:56 INFO DAGScheduler:59 - Final stage: ResultStage 92(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:56 INFO DAGScheduler:59 - Submitting ResultStage 92 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:56 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=295414, maxMem=766075207 -2016-06-13 17:44:56 INFO MemoryStore:59 - Block broadcast_134 stored as values in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 17:44:56 INFO MemoryStore:59 - ensureFreeSpace(2237) called with curMem=299510, maxMem=766075207 -2016-06-13 17:44:56 INFO MemoryStore:59 - Block broadcast_134_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Added broadcast_134_piece0 in memory on localhost:62254 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO SparkContext:59 - Created broadcast 134 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:56 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 92 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:44:56 INFO TaskSchedulerImpl:59 - Adding task set 92.0 with 4 tasks -2016-06-13 17:44:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 92.0 (TID 307, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_89_piece0 on localhost:62254 in memory (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 92.0 (TID 308, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 92.0 (TID 309, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:56 INFO TaskSetManager:59 - Starting task 3.0 in stage 92.0 (TID 310, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:56 INFO Executor:59 - Running task 1.0 in stage 92.0 (TID 308) -2016-06-13 17:44:56 INFO Executor:59 - Running task 2.0 in stage 92.0 (TID 309) -2016-06-13 17:44:56 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:44:56 INFO Executor:59 - Running task 0.0 in stage 92.0 (TID 307) -2016-06-13 17:44:56 INFO Executor:59 - Finished task 1.0 in stage 92.0 (TID 308). 2762 bytes result sent to driver -2016-06-13 17:44:56 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:44:56 INFO Executor:59 - Finished task 2.0 in stage 92.0 (TID 309). 2762 bytes result sent to driver -2016-06-13 17:44:56 INFO Executor:59 - Running task 3.0 in stage 92.0 (TID 310) -2016-06-13 17:44:56 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:44:56 INFO Executor:59 - Finished task 3.0 in stage 92.0 (TID 310). 2762 bytes result sent to driver -2016-06-13 17:44:56 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:44:56 INFO Executor:59 - Finished task 0.0 in stage 92.0 (TID 307). 2762 bytes result sent to driver -2016-06-13 17:44:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 92.0 (TID 308) in 13 ms on localhost (1/4) -2016-06-13 17:44:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 92.0 (TID 307) in 14 ms on localhost (2/4) -2016-06-13 17:44:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 92.0 (TID 309) in 14 ms on localhost (3/4) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-06-13 17:44:56 INFO DAGScheduler:59 - ResultStage 92 (aggregate at GaussianMixture.scala:201) finished in 0.015 s -2016-06-13 17:44:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 92.0 (TID 310) in 13 ms on localhost (4/4) -2016-06-13 17:44:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 92.0, whose tasks have all completed, from pool -2016-06-13 17:44:56 INFO DAGScheduler:59 - Job 92 finished: aggregate at GaussianMixture.scala:201, took 0.029038 s -2016-06-13 17:44:56 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=297472, maxMem=766075207 -2016-06-13 17:44:56 INFO MemoryStore:59 - Block broadcast_135 stored as values in memory (estimated size 712.0 B, free 730.3 MB) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_88_piece0 on localhost:62254 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=296192, maxMem=766075207 -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-06-13 17:44:56 INFO MemoryStore:59 - Block broadcast_135_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.3 MB) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Added broadcast_135_piece0 in memory on localhost:62254 (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO SparkContext:59 - Created broadcast 135 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_87_piece0 on localhost:62254 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-06-13 17:44:56 INFO BlockManager:59 - Removing RDD 25 -2016-06-13 17:44:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:56 INFO DAGScheduler:59 - Got job 93 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:56 INFO DAGScheduler:59 - Final stage: ResultStage 93(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned RDD 25 -2016-06-13 17:44:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:56 INFO DAGScheduler:59 - Submitting ResultStage 93 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:56 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=291104, maxMem=766075207 -2016-06-13 17:44:56 INFO MemoryStore:59 - Block broadcast_136 stored as values in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-06-13 17:44:56 INFO MemoryStore:59 - ensureFreeSpace(2241) called with curMem=291136, maxMem=766075207 -2016-06-13 17:44:56 INFO MemoryStore:59 - Block broadcast_136_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Added broadcast_136_piece0 in memory on localhost:62254 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO SparkContext:59 - Created broadcast 136 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:62254 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 93 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:44:56 INFO TaskSchedulerImpl:59 - Adding task set 93.0 with 4 tasks -2016-06-13 17:44:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 93.0 (TID 311, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:44:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 93.0 (TID 312, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 93.0 (TID 313, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:56 INFO TaskSetManager:59 - Starting task 3.0 in stage 93.0 (TID 314, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:56 INFO Executor:59 - Running task 0.0 in stage 93.0 (TID 311) -2016-06-13 17:44:56 INFO Executor:59 - Running task 3.0 in stage 93.0 (TID 314) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO Executor:59 - Running task 1.0 in stage 93.0 (TID 312) -2016-06-13 17:44:56 INFO Executor:59 - Running task 2.0 in stage 93.0 (TID 313) -2016-06-13 17:44:56 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:44:56 INFO Executor:59 - Finished task 3.0 in stage 93.0 (TID 314). 2762 bytes result sent to driver -2016-06-13 17:44:56 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-06-13 17:44:56 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:44:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 93.0 (TID 314) in 6 ms on localhost (1/4) -2016-06-13 17:44:56 INFO Executor:59 - Finished task 2.0 in stage 93.0 (TID 313). 2762 bytes result sent to driver -2016-06-13 17:44:56 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:44:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 93.0 (TID 313) in 8 ms on localhost (2/4) -2016-06-13 17:44:56 INFO Executor:59 - Finished task 1.0 in stage 93.0 (TID 312). 2762 bytes result sent to driver -2016-06-13 17:44:56 INFO Executor:59 - Finished task 0.0 in stage 93.0 (TID 311). 2762 bytes result sent to driver -2016-06-13 17:44:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 93.0 (TID 312) in 11 ms on localhost (3/4) -2016-06-13 17:44:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 93.0 (TID 311) in 12 ms on localhost (4/4) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:62254 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO DAGScheduler:59 - ResultStage 93 (aggregate at GaussianMixture.scala:201) finished in 0.012 s -2016-06-13 17:44:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 93.0, whose tasks have all completed, from pool -2016-06-13 17:44:56 INFO DAGScheduler:59 - Job 93 finished: aggregate at GaussianMixture.scala:201, took 0.026679 s -2016-06-13 17:44:56 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=280135, maxMem=766075207 -2016-06-13 17:44:56 INFO MemoryStore:59 - Block broadcast_137 stored as values in memory (estimated size 712.0 B, free 730.3 MB) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO MemoryStore:59 - ensureFreeSpace(936) called with curMem=278653, maxMem=766075207 -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-06-13 17:44:56 INFO MemoryStore:59 - Block broadcast_137_piece0 stored as bytes in memory (estimated size 936.0 B, free 730.3 MB) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Added broadcast_137_piece0 in memory on localhost:62254 (size: 936.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO SparkContext:59 - Created broadcast 137 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:62254 in memory (size: 891.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_132_piece0 on localhost:62254 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 92 -2016-06-13 17:44:56 INFO DAGScheduler:59 - Got job 94 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:44:56 INFO DAGScheduler:59 - Final stage: ResultStage 94(aggregate at GaussianMixture.scala:201) -2016-06-13 17:44:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:56 INFO DAGScheduler:59 - Submitting ResultStage 94 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_131_piece0 on localhost:62254 in memory (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=270159, maxMem=766075207 -2016-06-13 17:44:56 INFO MemoryStore:59 - Block broadcast_138 stored as values in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 17:44:56 INFO MemoryStore:59 - ensureFreeSpace(2242) called with curMem=270159, maxMem=766075207 -2016-06-13 17:44:56 INFO MemoryStore:59 - Block broadcast_138_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Added broadcast_138_piece0 in memory on localhost:62254 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO SparkContext:59 - Created broadcast 138 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:56 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 94 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_130_piece0 on localhost:62254 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO TaskSchedulerImpl:59 - Adding task set 94.0 with 4 tasks -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 91 -2016-06-13 17:44:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 94.0 (TID 315, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:44:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 94.0 (TID 316, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 94.0 (TID 317, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:56 INFO TaskSetManager:59 - Starting task 3.0 in stage 94.0 (TID 318, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:44:56 INFO Executor:59 - Running task 0.0 in stage 94.0 (TID 315) -2016-06-13 17:44:56 INFO Executor:59 - Running task 3.0 in stage 94.0 (TID 318) -2016-06-13 17:44:56 INFO Executor:59 - Running task 2.0 in stage 94.0 (TID 317) -2016-06-13 17:44:56 INFO Executor:59 - Running task 1.0 in stage 94.0 (TID 316) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_129_piece0 on localhost:62254 in memory (size: 934.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:44:56 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:44:56 INFO Executor:59 - Finished task 0.0 in stage 94.0 (TID 315). 2762 bytes result sent to driver -2016-06-13 17:44:56 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:44:56 INFO Executor:59 - Finished task 1.0 in stage 94.0 (TID 316). 2762 bytes result sent to driver -2016-06-13 17:44:56 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:44:56 INFO Executor:59 - Finished task 2.0 in stage 94.0 (TID 317). 2762 bytes result sent to driver -2016-06-13 17:44:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 94.0 (TID 316) in 7 ms on localhost (1/4) -2016-06-13 17:44:56 INFO Executor:59 - Finished task 3.0 in stage 94.0 (TID 318). 2762 bytes result sent to driver -2016-06-13 17:44:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 94.0 (TID 315) in 7 ms on localhost (2/4) -2016-06-13 17:44:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 94.0 (TID 317) in 7 ms on localhost (3/4) -2016-06-13 17:44:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 94.0 (TID 318) in 7 ms on localhost (4/4) -2016-06-13 17:44:56 INFO DAGScheduler:59 - ResultStage 94 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 17:44:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 94.0, whose tasks have all completed, from pool -2016-06-13 17:44:56 INFO DAGScheduler:59 - Job 94 finished: aggregate at GaussianMixture.scala:201, took 0.017032 s -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_128_piece0 on localhost:62254 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 90 -2016-06-13 17:44:56 INFO MemoryStore:59 - ensureFreeSpace(656) called with curMem=262182, maxMem=766075207 -2016-06-13 17:44:56 INFO MemoryStore:59 - Block broadcast_139 stored as values in memory (estimated size 656.0 B, free 730.3 MB) -2016-06-13 17:44:56 INFO MemoryStore:59 - ensureFreeSpace(833) called with curMem=261189, maxMem=766075207 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_127_piece0 on localhost:62254 in memory (size: 937.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO MemoryStore:59 - Block broadcast_139_piece0 stored as bytes in memory (estimated size 833.0 B, free 730.3 MB) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Added broadcast_139_piece0 in memory on localhost:62254 (size: 833.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO SparkContext:59 - Created broadcast 139 from broadcast at GaussianMixtureModel.scala:100 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_126_piece0 on localhost:62254 in memory (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO MemoryStore:59 - ensureFreeSpace(56) called with curMem=254140, maxMem=766075207 -2016-06-13 17:44:56 INFO MemoryStore:59 - Block broadcast_140 stored as values in memory (estimated size 56.0 B, free 730.4 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 89 -2016-06-13 17:44:56 INFO MemoryStore:59 - ensureFreeSpace(65) called with curMem=234972, maxMem=766075207 -2016-06-13 17:44:56 INFO MemoryStore:59 - Block broadcast_140_piece0 stored as bytes in memory (estimated size 65.0 B, free 730.4 MB) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Added broadcast_140_piece0 in memory on localhost:62254 (size: 65.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_125_piece0 on localhost:62254 in memory (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO SparkContext:59 - Created broadcast 140 from broadcast at GaussianMixtureModel.scala:101 -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 88 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_124_piece0 on localhost:62254 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 87 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_123_piece0 on localhost:62254 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 86 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_122_piece0 on localhost:62254 in memory (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_121_piece0 on localhost:62254 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 85 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_120_piece0 on localhost:62254 in memory (size: 934.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_119_piece0 on localhost:62254 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 84 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_118_piece0 on localhost:62254 in memory (size: 880.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_117_piece0 on localhost:62254 in memory (size: 1348.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 83 -2016-06-13 17:44:56 INFO BlockManager:59 - Removing RDD 33 -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned RDD 33 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_116_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 82 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_115_piece0 on localhost:62254 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO SparkContext:59 - Starting job: collect at GaussianMixtureFromSparkSuite.scala:177 -2016-06-13 17:44:56 INFO DAGScheduler:59 - Got job 95 (collect at GaussianMixtureFromSparkSuite.scala:177) with 4 output partitions -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_114_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO DAGScheduler:59 - Final stage: ResultStage 95(collect at GaussianMixtureFromSparkSuite.scala:177) -2016-06-13 17:44:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:44:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 81 -2016-06-13 17:44:56 INFO DAGScheduler:59 - Submitting ResultStage 95 (ZippedPartitionsRDD2[39] at zip at GaussianMixtureFromSparkSuite.scala:177), which has no missing parents -2016-06-13 17:44:56 INFO MemoryStore:59 - ensureFreeSpace(4712) called with curMem=183306, maxMem=766075207 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_113_piece0 on localhost:62254 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO MemoryStore:59 - Block broadcast_141 stored as values in memory (estimated size 4.6 KB, free 730.4 MB) -2016-06-13 17:44:56 INFO MemoryStore:59 - ensureFreeSpace(2639) called with curMem=187458, maxMem=766075207 -2016-06-13 17:44:56 INFO MemoryStore:59 - Block broadcast_141_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Added broadcast_141_piece0 in memory on localhost:62254 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO SparkContext:59 - Created broadcast 141 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:44:56 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 95 (ZippedPartitionsRDD2[39] at zip at GaussianMixtureFromSparkSuite.scala:177) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_112_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO TaskSchedulerImpl:59 - Adding task set 95.0 with 4 tasks -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 80 -2016-06-13 17:44:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 95.0 (TID 319, localhost, PROCESS_LOCAL, 2432 bytes) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_111_piece0 on localhost:62254 in memory (size: 882.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 95.0 (TID 320, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 17:44:56 INFO TaskSetManager:59 - Starting task 2.0 in stage 95.0 (TID 321, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 17:44:56 INFO TaskSetManager:59 - Starting task 3.0 in stage 95.0 (TID 322, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 17:44:56 INFO Executor:59 - Running task 0.0 in stage 95.0 (TID 319) -2016-06-13 17:44:56 INFO Executor:59 - Running task 3.0 in stage 95.0 (TID 322) -2016-06-13 17:44:56 INFO Executor:59 - Running task 2.0 in stage 95.0 (TID 321) -2016-06-13 17:44:56 INFO Executor:59 - Running task 1.0 in stage 95.0 (TID 320) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_110_piece0 on localhost:62254 in memory (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 79 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_109_piece0 on localhost:62254 in memory (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 78 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_108_piece0 on localhost:62254 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 77 -2016-06-13 17:44:56 INFO Executor:59 - Finished task 0.0 in stage 95.0 (TID 319). 1229 bytes result sent to driver -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_107_piece0 on localhost:62254 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO Executor:59 - Finished task 1.0 in stage 95.0 (TID 320). 1269 bytes result sent to driver -2016-06-13 17:44:56 INFO Executor:59 - Finished task 2.0 in stage 95.0 (TID 321). 1264 bytes result sent to driver -2016-06-13 17:44:56 INFO Executor:59 - Finished task 3.0 in stage 95.0 (TID 322). 1264 bytes result sent to driver -2016-06-13 17:44:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 95.0 (TID 319) in 25 ms on localhost (1/4) -2016-06-13 17:44:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 95.0 (TID 320) in 24 ms on localhost (2/4) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 76 -2016-06-13 17:44:56 INFO TaskSetManager:59 - Finished task 2.0 in stage 95.0 (TID 321) in 24 ms on localhost (3/4) -2016-06-13 17:44:56 INFO TaskSetManager:59 - Finished task 3.0 in stage 95.0 (TID 322) in 24 ms on localhost (4/4) -2016-06-13 17:44:56 INFO DAGScheduler:59 - ResultStage 95 (collect at GaussianMixtureFromSparkSuite.scala:177) finished in 0.028 s -2016-06-13 17:44:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 95.0, whose tasks have all completed, from pool -2016-06-13 17:44:56 INFO DAGScheduler:59 - Job 95 finished: collect at GaussianMixtureFromSparkSuite.scala:177, took 0.037391 s -2016-06-13 17:44:56 INFO BlockManager:59 - Removing RDD 29 -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned RDD 29 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_106_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 75 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_105_piece0 on localhost:62254 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_104_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 74 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_103_piece0 on localhost:62254 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_102_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 73 -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 17:44:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_101_piece0 on localhost:62254 in memory (size: 895.0 B, free: 730.5 MB) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_100_piece0 on localhost:62254 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 72 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_99_piece0 on localhost:62254 in memory (size: 1193.0 B, free: 730.6 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 71 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_98_piece0 on localhost:62254 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 70 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_97_piece0 on localhost:62254 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 69 -2016-06-13 17:44:56 INFO BlockManager:59 - Removing RDD 27 -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned RDD 27 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_96_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_95_piece0 on localhost:62254 in memory (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_94_piece0 on localhost:62254 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_93_piece0 on localhost:62254 in memory (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_92_piece0 on localhost:62254 in memory (size: 2.2 KB, free: 730.6 MB) -2016-06-13 17:44:56 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_91_piece0 on localhost:62254 in memory (size: 899.0 B, free: 730.6 MB) -2016-06-13 17:44:56 INFO BlockManagerInfo:59 - Removed broadcast_90_piece0 on localhost:62254 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:44:56 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 17:44:56 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 17:44:56 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 17:44:56 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 17:44:56 INFO BlockManager:59 - BlockManager stopped -2016-06-13 17:44:56 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 17:44:56 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 17:44:56 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 17:44:56 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 17:44:56 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 17:44:56 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 17:44:56 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-524746cd-5e39-43a3-81dc-cdfa2ec46f32 -2016-06-13 17:48:41 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 17:48:41 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 17:48:41 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 17:48:41 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 17:48:41 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 17:48:42 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 17:48:42 INFO Remoting:74 - Starting remoting -2016-06-13 17:48:42 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:62318] -2016-06-13 17:48:42 INFO Utils:59 - Successfully started service 'sparkDriver' on port 62318. -2016-06-13 17:48:42 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 17:48:42 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 17:48:42 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-737bae4e-a7bf-47e4-9020-de9c0c59061e -2016-06-13 17:48:42 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 17:48:43 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-551e35e9-679b-43ac-83e5-3539b103e92f\httpd-819be78d-500b-48ff-aff9-95cf55c5eb60 -2016-06-13 17:48:43 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 17:48:43 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:48:43 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:62319 -2016-06-13 17:48:43 INFO Utils:59 - Successfully started service 'HTTP file server' on port 62319. -2016-06-13 17:48:43 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 17:48:43 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:48:43 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 17:48:43 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 17:48:43 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 17:48:43 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 17:48:43 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 17:48:43 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 62338. -2016-06-13 17:48:43 INFO NettyBlockTransferService:59 - Server created on 62338 -2016-06-13 17:48:43 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 17:48:43 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:62338 with 730.6 MB RAM, BlockManagerId(driver, localhost, 62338) -2016-06-13 17:48:43 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 17:48:45 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 17:48:45 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 17:48:45 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 17:48:45 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 17:48:45 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:48:45 INFO DAGScheduler:59 - Got job 0 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:48:45 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GaussianMixture.scala:171) -2016-06-13 17:48:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:48:45 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:48:45 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:48:45 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=0, maxMem=766075207 -2016-06-13 17:48:45 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:48:45 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1992, maxMem=766075207 -2016-06-13 17:48:45 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.6 MB) -2016-06-13 17:48:45 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:62338 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:48:45 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:48:45 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:48:45 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 17:48:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:48:45 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 17:48:45 INFO CacheManager:59 - Partition rdd_1_0 not found, computing it -2016-06-13 17:48:45 INFO MemoryStore:59 - ensureFreeSpace(224) called with curMem=3242, maxMem=766075207 -2016-06-13 17:48:45 INFO MemoryStore:59 - Block rdd_1_0 stored as values in memory (estimated size 224.0 B, free 730.6 MB) -2016-06-13 17:48:45 INFO BlockManagerInfo:59 - Added rdd_1_0 in memory on localhost:62338 (size: 224.0 B, free: 730.6 MB) -2016-06-13 17:48:45 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1688 bytes result sent to driver -2016-06-13 17:48:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 88 ms on localhost (1/1) -2016-06-13 17:48:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 17:48:45 INFO DAGScheduler:59 - ResultStage 0 (first at GaussianMixture.scala:171) finished in 0.102 s -2016-06-13 17:48:45 INFO DAGScheduler:59 - Job 0 finished: first at GaussianMixture.scala:171, took 0.303845 s -2016-06-13 17:48:45 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=3466, maxMem=766075207 -2016-06-13 17:48:45 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 712.0 B, free 730.6 MB) -2016-06-13 17:48:45 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=4178, maxMem=766075207 -2016-06-13 17:48:45 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 880.0 B, free 730.6 MB) -2016-06-13 17:48:45 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:62338 (size: 880.0 B, free: 730.6 MB) -2016-06-13 17:48:45 INFO SparkContext:59 - Created broadcast 1 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:48:45 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:48:45 INFO DAGScheduler:59 - Got job 1 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:48:45 INFO DAGScheduler:59 - Final stage: ResultStage 1(aggregate at GaussianMixture.scala:201) -2016-06-13 17:48:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:48:45 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:48:45 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:48:45 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=5058, maxMem=766075207 -2016-06-13 17:48:45 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.0 KB, free 730.6 MB) -2016-06-13 17:48:45 INFO MemoryStore:59 - ensureFreeSpace(2180) called with curMem=9154, maxMem=766075207 -2016-06-13 17:48:45 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.6 MB) -2016-06-13 17:48:45 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:62338 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:48:45 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:48:45 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:48:45 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 4 tasks -2016-06-13 17:48:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:48:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:48:45 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:48:45 INFO TaskSetManager:59 - Starting task 3.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:48:45 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 17:48:45 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 17:48:45 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 3) -2016-06-13 17:48:45 INFO Executor:59 - Running task 3.0 in stage 1.0 (TID 4) -2016-06-13 17:48:45 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:48:45 INFO CacheManager:59 - Partition rdd_1_1 not found, computing it -2016-06-13 17:48:45 INFO CacheManager:59 - Partition rdd_1_3 not found, computing it -2016-06-13 17:48:45 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=11334, maxMem=766075207 -2016-06-13 17:48:45 INFO MemoryStore:59 - Block rdd_1_1 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 17:48:45 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=11622, maxMem=766075207 -2016-06-13 17:48:45 INFO MemoryStore:59 - Block rdd_1_3 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 17:48:45 INFO BlockManagerInfo:59 - Added rdd_1_1 in memory on localhost:62338 (size: 288.0 B, free: 730.6 MB) -2016-06-13 17:48:45 INFO BlockManagerInfo:59 - Added rdd_1_3 in memory on localhost:62338 (size: 288.0 B, free: 730.6 MB) -2016-06-13 17:48:45 INFO CacheManager:59 - Partition rdd_1_2 not found, computing it -2016-06-13 17:48:45 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=11910, maxMem=766075207 -2016-06-13 17:48:45 INFO MemoryStore:59 - Block rdd_1_2 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 17:48:45 INFO BlockManagerInfo:59 - Added rdd_1_2 in memory on localhost:62338 (size: 288.0 B, free: 730.6 MB) -2016-06-13 17:48:45 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 3). 2247 bytes result sent to driver -2016-06-13 17:48:45 INFO Executor:59 - Finished task 3.0 in stage 1.0 (TID 4). 2247 bytes result sent to driver -2016-06-13 17:48:45 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2247 bytes result sent to driver -2016-06-13 17:48:45 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 3) in 76 ms on localhost (1/4) -2016-06-13 17:48:45 INFO TaskSetManager:59 - Finished task 3.0 in stage 1.0 (TID 4) in 74 ms on localhost (2/4) -2016-06-13 17:48:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 82 ms on localhost (3/4) -2016-06-13 17:48:45 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2762 bytes result sent to driver -2016-06-13 17:48:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 94 ms on localhost (4/4) -2016-06-13 17:48:45 INFO DAGScheduler:59 - ResultStage 1 (aggregate at GaussianMixture.scala:201) finished in 0.094 s -2016-06-13 17:48:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 17:48:45 INFO DAGScheduler:59 - Job 1 finished: aggregate at GaussianMixture.scala:201, took 0.119803 s -2016-06-13 17:48:45 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=12198, maxMem=766075207 -2016-06-13 17:48:45 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 712.0 B, free 730.6 MB) -2016-06-13 17:48:45 INFO MemoryStore:59 - ensureFreeSpace(934) called with curMem=12910, maxMem=766075207 -2016-06-13 17:48:45 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 934.0 B, free 730.6 MB) -2016-06-13 17:48:45 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:62338 (size: 934.0 B, free: 730.6 MB) -2016-06-13 17:48:45 INFO SparkContext:59 - Created broadcast 3 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:48:45 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:48:45 INFO DAGScheduler:59 - Got job 2 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:48:45 INFO DAGScheduler:59 - Final stage: ResultStage 2(aggregate at GaussianMixture.scala:201) -2016-06-13 17:48:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:48:45 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:48:45 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:48:45 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=13844, maxMem=766075207 -2016-06-13 17:48:45 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.0 KB, free 730.6 MB) -2016-06-13 17:48:45 INFO MemoryStore:59 - ensureFreeSpace(2240) called with curMem=17940, maxMem=766075207 -2016-06-13 17:48:45 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.6 MB) -2016-06-13 17:48:45 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:62338 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 17:48:45 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:48:45 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 2 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:48:45 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 4 tasks -2016-06-13 17:48:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:48:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:48:45 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:48:45 INFO TaskSetManager:59 - Starting task 3.0 in stage 2.0 (TID 8, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:48:45 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 6) -2016-06-13 17:48:45 INFO Executor:59 - Running task 3.0 in stage 2.0 (TID 8) -2016-06-13 17:48:45 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 7) -2016-06-13 17:48:45 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:48:45 INFO Executor:59 - Finished task 3.0 in stage 2.0 (TID 8). 2762 bytes result sent to driver -2016-06-13 17:48:45 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 5) -2016-06-13 17:48:45 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:48:45 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 7). 2762 bytes result sent to driver -2016-06-13 17:48:45 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:48:45 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 6). 2762 bytes result sent to driver -2016-06-13 17:48:45 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:48:45 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 5). 2762 bytes result sent to driver -2016-06-13 17:48:45 INFO TaskSetManager:59 - Finished task 3.0 in stage 2.0 (TID 8) in 24 ms on localhost (1/4) -2016-06-13 17:48:45 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 7) in 27 ms on localhost (2/4) -2016-06-13 17:48:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 6) in 31 ms on localhost (3/4) -2016-06-13 17:48:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 5) in 33 ms on localhost (4/4) -2016-06-13 17:48:45 INFO DAGScheduler:59 - ResultStage 2 (aggregate at GaussianMixture.scala:201) finished in 0.033 s -2016-06-13 17:48:45 INFO DAGScheduler:59 - Job 2 finished: aggregate at GaussianMixture.scala:201, took 0.049018 s -2016-06-13 17:48:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 17:48:45 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=20180, maxMem=766075207 -2016-06-13 17:48:45 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 712.0 B, free 730.6 MB) -2016-06-13 17:48:45 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=20892, maxMem=766075207 -2016-06-13 17:48:45 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.6 MB) -2016-06-13 17:48:45 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:62338 (size: 932.0 B, free: 730.6 MB) -2016-06-13 17:48:45 INFO SparkContext:59 - Created broadcast 5 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:48:45 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:48:45 INFO DAGScheduler:59 - Got job 3 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:48:45 INFO DAGScheduler:59 - Final stage: ResultStage 3(aggregate at GaussianMixture.scala:201) -2016-06-13 17:48:45 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:48:45 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:48:45 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:48:45 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=21824, maxMem=766075207 -2016-06-13 17:48:45 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 4.0 KB, free 730.6 MB) -2016-06-13 17:48:45 INFO MemoryStore:59 - ensureFreeSpace(2237) called with curMem=25920, maxMem=766075207 -2016-06-13 17:48:45 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.6 MB) -2016-06-13 17:48:45 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:62338 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 17:48:45 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:48:45 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 3 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:48:45 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 4 tasks -2016-06-13 17:48:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:48:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:48:45 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 11, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:48:45 INFO TaskSetManager:59 - Starting task 3.0 in stage 3.0 (TID 12, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:48:45 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 9) -2016-06-13 17:48:45 INFO Executor:59 - Running task 3.0 in stage 3.0 (TID 12) -2016-06-13 17:48:45 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 11) -2016-06-13 17:48:45 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:48:45 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:48:45 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 9). 2762 bytes result sent to driver -2016-06-13 17:48:45 INFO Executor:59 - Finished task 3.0 in stage 3.0 (TID 12). 2762 bytes result sent to driver -2016-06-13 17:48:45 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 10) -2016-06-13 17:48:45 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:48:45 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 11). 2762 bytes result sent to driver -2016-06-13 17:48:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 9) in 22 ms on localhost (1/4) -2016-06-13 17:48:45 INFO TaskSetManager:59 - Finished task 3.0 in stage 3.0 (TID 12) in 18 ms on localhost (2/4) -2016-06-13 17:48:45 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:48:45 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 10). 2762 bytes result sent to driver -2016-06-13 17:48:45 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 11) in 26 ms on localhost (3/4) -2016-06-13 17:48:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 10) in 31 ms on localhost (4/4) -2016-06-13 17:48:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 17:48:45 INFO DAGScheduler:59 - ResultStage 3 (aggregate at GaussianMixture.scala:201) finished in 0.034 s -2016-06-13 17:48:45 INFO DAGScheduler:59 - Job 3 finished: aggregate at GaussianMixture.scala:201, took 0.065565 s -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 17:48:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 17:48:45 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 17:48:45 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 17:48:45 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 17:48:46 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 17:48:46 INFO BlockManager:59 - BlockManager stopped -2016-06-13 17:48:46 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 17:48:46 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 17:48:46 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 17:48:46 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 17:48:46 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 17:48:46 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-551e35e9-679b-43ac-83e5-3539b103e92f -2016-06-13 17:48:46 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 17:50:15 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 17:50:16 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 17:50:16 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 17:50:16 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 17:50:16 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 17:50:17 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 17:50:17 INFO Remoting:74 - Starting remoting -2016-06-13 17:50:18 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:62356] -2016-06-13 17:50:18 INFO Utils:59 - Successfully started service 'sparkDriver' on port 62356. -2016-06-13 17:50:18 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 17:50:18 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 17:50:18 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-76829b7e-3e26-4bc2-8198-158c1aaa7f8a -2016-06-13 17:50:18 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 17:50:18 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-dee942b7-c8a2-4baf-881b-5d5cb2997d3e\httpd-6a7a5d16-b130-4d6b-a358-60519001cae0 -2016-06-13 17:50:18 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 17:50:18 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:50:18 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:62357 -2016-06-13 17:50:18 INFO Utils:59 - Successfully started service 'HTTP file server' on port 62357. -2016-06-13 17:50:18 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 17:50:18 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:50:18 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 17:50:18 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 17:50:18 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 17:50:18 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 17:50:18 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 17:50:18 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 62376. -2016-06-13 17:50:18 INFO NettyBlockTransferService:59 - Server created on 62376 -2016-06-13 17:50:18 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 17:50:18 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:62376 with 730.6 MB RAM, BlockManagerId(driver, localhost, 62376) -2016-06-13 17:50:18 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 17:50:19 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:50:19 INFO DAGScheduler:59 - Got job 0 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:50:19 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GaussianMixture.scala:171) -2016-06-13 17:50:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:19 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:19 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=0, maxMem=766075207 -2016-06-13 17:50:19 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:50:19 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1992, maxMem=766075207 -2016-06-13 17:50:19 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.6 MB) -2016-06-13 17:50:19 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:62376 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:50:19 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:50:19 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 17:50:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:19 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 17:50:19 INFO CacheManager:59 - Partition rdd_1_0 not found, computing it -2016-06-13 17:50:19 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=3242, maxMem=766075207 -2016-06-13 17:50:19 INFO MemoryStore:59 - Block rdd_1_0 stored as values in memory (estimated size 16.0 B, free 730.6 MB) -2016-06-13 17:50:19 INFO BlockManagerInfo:59 - Added rdd_1_0 in memory on localhost:62376 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:50:19 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1533 bytes result sent to driver -2016-06-13 17:50:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 68 ms on localhost (1/1) -2016-06-13 17:50:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 17:50:19 INFO DAGScheduler:59 - ResultStage 0 (first at GaussianMixture.scala:171) finished in 0.082 s -2016-06-13 17:50:19 INFO DAGScheduler:59 - Job 0 finished: first at GaussianMixture.scala:171, took 0.280883 s -2016-06-13 17:50:19 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:50:19 INFO DAGScheduler:59 - Got job 1 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:50:19 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GaussianMixture.scala:171) -2016-06-13 17:50:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:19 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:19 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=3258, maxMem=766075207 -2016-06-13 17:50:19 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:50:19 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=5250, maxMem=766075207 -2016-06-13 17:50:19 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.6 MB) -2016-06-13 17:50:19 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:62376 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:50:19 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:19 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:50:19 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 3 tasks -2016-06-13 17:50:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:19 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 17:50:19 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 17:50:19 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 3) -2016-06-13 17:50:20 INFO CacheManager:59 - Partition rdd_1_3 not found, computing it -2016-06-13 17:50:20 INFO CacheManager:59 - Partition rdd_1_2 not found, computing it -2016-06-13 17:50:20 INFO CacheManager:59 - Partition rdd_1_1 not found, computing it -2016-06-13 17:50:20 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:62376 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:50:20 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=3258, maxMem=766075207 -2016-06-13 17:50:20 INFO MemoryStore:59 - Block rdd_1_1 stored as values in memory (estimated size 192.0 B, free 730.6 MB) -2016-06-13 17:50:20 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=3450, maxMem=766075207 -2016-06-13 17:50:20 INFO MemoryStore:59 - Block rdd_1_2 stored as values in memory (estimated size 192.0 B, free 730.6 MB) -2016-06-13 17:50:20 INFO BlockManagerInfo:59 - Added rdd_1_1 in memory on localhost:62376 (size: 192.0 B, free: 730.6 MB) -2016-06-13 17:50:20 INFO BlockManagerInfo:59 - Added rdd_1_2 in memory on localhost:62376 (size: 192.0 B, free: 730.6 MB) -2016-06-13 17:50:20 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=3642, maxMem=766075207 -2016-06-13 17:50:20 INFO MemoryStore:59 - Block rdd_1_3 stored as values in memory (estimated size 184.0 B, free 730.6 MB) -2016-06-13 17:50:20 INFO BlockManagerInfo:59 - Added rdd_1_3 in memory on localhost:62376 (size: 184.0 B, free: 730.6 MB) -2016-06-13 17:50:20 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 17:50:20 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 3). 2132 bytes result sent to driver -2016-06-13 17:50:20 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2144 bytes result sent to driver -2016-06-13 17:50:20 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2144 bytes result sent to driver -2016-06-13 17:50:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 894 ms on localhost (1/3) -2016-06-13 17:50:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 3) in 891 ms on localhost (2/3) -2016-06-13 17:50:20 INFO DAGScheduler:59 - ResultStage 1 (first at GaussianMixture.scala:171) finished in 0.896 s -2016-06-13 17:50:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 894 ms on localhost (3/3) -2016-06-13 17:50:20 INFO DAGScheduler:59 - Job 1 finished: first at GaussianMixture.scala:171, took 0.931491 s -2016-06-13 17:50:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 17:50:20 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:50:20 INFO DAGScheduler:59 - Got job 2 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:50:20 INFO DAGScheduler:59 - Final stage: ResultStage 2(takeSample at GaussianMixture.scala:184) -2016-06-13 17:50:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:20 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:20 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=3826, maxMem=766075207 -2016-06-13 17:50:20 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 17:50:20 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=5658, maxMem=766075207 -2016-06-13 17:50:20 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.6 MB) -2016-06-13 17:50:20 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:62376 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:50:20 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 2 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:50:20 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 4 tasks -2016-06-13 17:50:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:20 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 17:50:20 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-06-13 17:50:20 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 17:50:20 INFO Executor:59 - Running task 3.0 in stage 2.0 (TID 7) -2016-06-13 17:50:20 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:50:20 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:50:20 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:50:20 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:50:20 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2082 bytes result sent to driver -2016-06-13 17:50:20 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 6). 2082 bytes result sent to driver -2016-06-13 17:50:20 INFO Executor:59 - Finished task 3.0 in stage 2.0 (TID 7). 2082 bytes result sent to driver -2016-06-13 17:50:20 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-06-13 17:50:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 26 ms on localhost (1/4) -2016-06-13 17:50:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 6) in 25 ms on localhost (2/4) -2016-06-13 17:50:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 2.0 (TID 7) in 25 ms on localhost (3/4) -2016-06-13 17:50:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 29 ms on localhost (4/4) -2016-06-13 17:50:20 INFO DAGScheduler:59 - ResultStage 2 (takeSample at GaussianMixture.scala:184) finished in 0.029 s -2016-06-13 17:50:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 17:50:20 INFO DAGScheduler:59 - Job 2 finished: takeSample at GaussianMixture.scala:184, took 0.045532 s -2016-06-13 17:50:20 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:50:20 INFO DAGScheduler:59 - Got job 3 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:50:20 INFO DAGScheduler:59 - Final stage: ResultStage 3(takeSample at GaussianMixture.scala:184) -2016-06-13 17:50:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:20 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=6853, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 18.8 KB, free 730.6 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(7882) called with curMem=26077, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.6 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:62376 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:50:21 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 4 tasks -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 3.0 (TID 11, localhost, PROCESS_LOCAL, 2373 bytes) -2016-06-13 17:50:21 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 8) -2016-06-13 17:50:21 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 10) -2016-06-13 17:50:21 INFO Executor:59 - Running task 3.0 in stage 3.0 (TID 11) -2016-06-13 17:50:21 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 9) -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:50:21 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 8). 2048 bytes result sent to driver -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:50:21 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 10). 2704 bytes result sent to driver -2016-06-13 17:50:21 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 9). 2724 bytes result sent to driver -2016-06-13 17:50:21 INFO Executor:59 - Finished task 3.0 in stage 3.0 (TID 11). 2692 bytes result sent to driver -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 8) in 26 ms on localhost (1/4) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 10) in 24 ms on localhost (2/4) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 9) in 27 ms on localhost (3/4) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 3.0 (TID 11) in 25 ms on localhost (4/4) -2016-06-13 17:50:21 INFO DAGScheduler:59 - ResultStage 3 (takeSample at GaussianMixture.scala:184) finished in 0.030 s -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 17:50:21 INFO DAGScheduler:59 - Job 3 finished: takeSample at GaussianMixture.scala:184, took 0.050401 s -2016-06-13 17:50:21 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 17:50:21 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 17:50:21 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 17:50:21 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=33959, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 560.0 B, free 730.6 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(890) called with curMem=34519, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 890.0 B, free 730.6 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:62376 (size: 890.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO SparkContext:59 - Created broadcast 4 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:50:21 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Got job 4 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:50:21 INFO DAGScheduler:59 - Final stage: ResultStage 4(aggregate at GaussianMixture.scala:201) -2016-06-13 17:50:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=35409, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(2187) called with curMem=39473, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:62376 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:50:21 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 4 tasks -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 12, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 14, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 4.0 (TID 15, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:21 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 14) -2016-06-13 17:50:21 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 12) -2016-06-13 17:50:21 INFO Executor:59 - Running task 3.0 in stage 4.0 (TID 15) -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:50:21 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 12). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 13) -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 12) in 32 ms on localhost (1/4) -2016-06-13 17:50:21 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 14). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO Executor:59 - Finished task 3.0 in stage 4.0 (TID 15). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 13). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 14) in 42 ms on localhost (2/4) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 4.0 (TID 15) in 43 ms on localhost (3/4) -2016-06-13 17:50:21 INFO DAGScheduler:59 - ResultStage 4 (aggregate at GaussianMixture.scala:201) finished in 0.048 s -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 13) in 46 ms on localhost (4/4) -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 17:50:21 INFO DAGScheduler:59 - Job 4 finished: aggregate at GaussianMixture.scala:201, took 0.061803 s -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=41660, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=42220, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.5 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:62376 (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO SparkContext:59 - Created broadcast 6 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:50:21 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Got job 5 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:50:21 INFO DAGScheduler:59 - Final stage: ResultStage 5(aggregate at GaussianMixture.scala:201) -2016-06-13 17:50:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=43120, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(2193) called with curMem=47184, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:62376 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:50:21 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 4 tasks -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 16, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 17, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 18, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 5.0 (TID 19, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:21 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 17) -2016-06-13 17:50:21 INFO Executor:59 - Running task 3.0 in stage 5.0 (TID 19) -2016-06-13 17:50:21 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 18) -2016-06-13 17:50:21 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 16) -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:50:21 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 17). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:50:21 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 16). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 18). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO Executor:59 - Finished task 3.0 in stage 5.0 (TID 19). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 17) in 19 ms on localhost (1/4) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 16) in 21 ms on localhost (2/4) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 18) in 20 ms on localhost (3/4) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 5.0 (TID 19) in 22 ms on localhost (4/4) -2016-06-13 17:50:21 INFO DAGScheduler:59 - ResultStage 5 (aggregate at GaussianMixture.scala:201) finished in 0.025 s -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 17:50:21 INFO DAGScheduler:59 - Job 5 finished: aggregate at GaussianMixture.scala:201, took 0.040719 s -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=49377, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=49937, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.5 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:62376 (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO SparkContext:59 - Created broadcast 8 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:50:21 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Got job 6 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:50:21 INFO DAGScheduler:59 - Final stage: ResultStage 6(aggregate at GaussianMixture.scala:201) -2016-06-13 17:50:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=50837, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(2193) called with curMem=54901, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:62376 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:50:21 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 4 tasks -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 20, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 21, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 22, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 6.0 (TID 23, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:21 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 20) -2016-06-13 17:50:21 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 22) -2016-06-13 17:50:21 INFO Executor:59 - Running task 3.0 in stage 6.0 (TID 23) -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:50:21 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 20). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 21) -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:50:21 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 22). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:50:21 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 21). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:50:21 INFO Executor:59 - Finished task 3.0 in stage 6.0 (TID 23). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 22) in 14 ms on localhost (1/4) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 21) in 18 ms on localhost (2/4) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 20) in 20 ms on localhost (3/4) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 6.0 (TID 23) in 19 ms on localhost (4/4) -2016-06-13 17:50:21 INFO DAGScheduler:59 - ResultStage 6 (aggregate at GaussianMixture.scala:201) finished in 0.022 s -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 17:50:21 INFO DAGScheduler:59 - Job 6 finished: aggregate at GaussianMixture.scala:201, took 0.037507 s -2016-06-13 17:50:21 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Got job 7 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:50:21 INFO DAGScheduler:59 - Final stage: ResultStage 7(first at GaussianMixture.scala:171) -2016-06-13 17:50:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=57094, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=59086, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.5 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:62376 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 24, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:21 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 24) -2016-06-13 17:50:21 INFO CacheManager:59 - Partition rdd_3_0 not found, computing it -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=60338, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block rdd_3_0 stored as values in memory (estimated size 16.0 B, free 730.5 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added rdd_3_0 in memory on localhost:62376 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 24). 1533 bytes result sent to driver -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 24) in 11 ms on localhost (1/1) -2016-06-13 17:50:21 INFO DAGScheduler:59 - ResultStage 7 (first at GaussianMixture.scala:171) finished in 0.011 s -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 17:50:21 INFO DAGScheduler:59 - Job 7 finished: first at GaussianMixture.scala:171, took 0.050034 s -2016-06-13 17:50:21 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Got job 8 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:50:21 INFO DAGScheduler:59 - Final stage: ResultStage 8(first at GaussianMixture.scala:171) -2016-06-13 17:50:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=60354, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=62346, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.5 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:62376 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 8 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 3 tasks -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 25, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 26, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 8.0 (TID 27, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:21 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 25) -2016-06-13 17:50:21 INFO Executor:59 - Running task 2.0 in stage 8.0 (TID 27) -2016-06-13 17:50:21 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 26) -2016-06-13 17:50:21 INFO CacheManager:59 - Partition rdd_3_3 not found, computing it -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=63598, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block rdd_3_3 stored as values in memory (estimated size 184.0 B, free 730.5 MB) -2016-06-13 17:50:21 INFO CacheManager:59 - Partition rdd_3_1 not found, computing it -2016-06-13 17:50:21 INFO CacheManager:59 - Partition rdd_3_2 not found, computing it -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added rdd_3_3 in memory on localhost:62376 (size: 184.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=63782, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block rdd_3_1 stored as values in memory (estimated size 192.0 B, free 730.5 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=63974, maxMem=766075207 -2016-06-13 17:50:21 INFO Executor:59 - Finished task 2.0 in stage 8.0 (TID 27). 2132 bytes result sent to driver -2016-06-13 17:50:21 INFO MemoryStore:59 - Block rdd_3_2 stored as values in memory (estimated size 192.0 B, free 730.5 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added rdd_3_1 in memory on localhost:62376 (size: 192.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added rdd_3_2 in memory on localhost:62376 (size: 192.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 25). 2144 bytes result sent to driver -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 8.0 (TID 27) in 15 ms on localhost (1/3) -2016-06-13 17:50:21 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 26). 2144 bytes result sent to driver -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 25) in 20 ms on localhost (2/3) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 26) in 19 ms on localhost (3/3) -2016-06-13 17:50:21 INFO DAGScheduler:59 - ResultStage 8 (first at GaussianMixture.scala:171) finished in 0.021 s -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 17:50:21 INFO DAGScheduler:59 - Job 8 finished: first at GaussianMixture.scala:171, took 0.037418 s -2016-06-13 17:50:21 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Got job 9 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:50:21 INFO DAGScheduler:59 - Final stage: ResultStage 9(takeSample at GaussianMixture.scala:184) -2016-06-13 17:50:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=64166, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 1832.0 B, free 730.5 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=65998, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.5 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:62376 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 9 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 4 tasks -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 28, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 29, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 9.0 (TID 30, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 9.0 (TID 31, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:21 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 28) -2016-06-13 17:50:21 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 29) -2016-06-13 17:50:21 INFO Executor:59 - Running task 3.0 in stage 9.0 (TID 31) -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:50:21 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 28). 2082 bytes result sent to driver -2016-06-13 17:50:21 INFO Executor:59 - Running task 2.0 in stage 9.0 (TID 30) -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:50:21 INFO Executor:59 - Finished task 3.0 in stage 9.0 (TID 31). 2082 bytes result sent to driver -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:50:21 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 29). 2082 bytes result sent to driver -2016-06-13 17:50:21 INFO Executor:59 - Finished task 2.0 in stage 9.0 (TID 30). 2082 bytes result sent to driver -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 28) in 19 ms on localhost (1/4) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 9.0 (TID 31) in 16 ms on localhost (2/4) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 29) in 20 ms on localhost (3/4) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 9.0 (TID 30) in 18 ms on localhost (4/4) -2016-06-13 17:50:21 INFO DAGScheduler:59 - ResultStage 9 (takeSample at GaussianMixture.scala:184) finished in 0.023 s -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 17:50:21 INFO DAGScheduler:59 - Job 9 finished: takeSample at GaussianMixture.scala:184, took 0.041582 s -2016-06-13 17:50:21 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Got job 10 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:50:21 INFO DAGScheduler:59 - Final stage: ResultStage 10(takeSample at GaussianMixture.scala:184) -2016-06-13 17:50:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting ResultStage 10 (PartitionwiseSampledRDD[4] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=67193, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(7880) called with curMem=86417, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:62376 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:50:21 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 10 (PartitionwiseSampledRDD[4] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 4 tasks -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 32, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 33, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 10.0 (TID 34, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 10.0 (TID 35, localhost, PROCESS_LOCAL, 2373 bytes) -2016-06-13 17:50:21 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 33) -2016-06-13 17:50:21 INFO Executor:59 - Running task 3.0 in stage 10.0 (TID 35) -2016-06-13 17:50:21 INFO Executor:59 - Running task 2.0 in stage 10.0 (TID 34) -2016-06-13 17:50:21 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 32) -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:50:21 INFO Executor:59 - Finished task 3.0 in stage 10.0 (TID 35). 2682 bytes result sent to driver -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:50:21 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 33). 2724 bytes result sent to driver -2016-06-13 17:50:21 INFO Executor:59 - Finished task 2.0 in stage 10.0 (TID 34). 2714 bytes result sent to driver -2016-06-13 17:50:21 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 32). 2048 bytes result sent to driver -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 10.0 (TID 35) in 17 ms on localhost (1/4) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 33) in 23 ms on localhost (2/4) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 10.0 (TID 34) in 24 ms on localhost (3/4) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 32) in 29 ms on localhost (4/4) -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 17:50:21 INFO DAGScheduler:59 - ResultStage 10 (takeSample at GaussianMixture.scala:184) finished in 0.031 s -2016-06-13 17:50:21 INFO DAGScheduler:59 - Job 10 finished: takeSample at GaussianMixture.scala:184, took 0.047293 s -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=94297, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(891) called with curMem=94857, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 891.0 B, free 730.5 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:62376 (size: 891.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO SparkContext:59 - Created broadcast 14 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:50:21 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Got job 11 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:50:21 INFO DAGScheduler:59 - Final stage: ResultStage 11(aggregate at GaussianMixture.scala:201) -2016-06-13 17:50:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=95748, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(2193) called with curMem=99812, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:62376 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:50:21 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 11 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 4 tasks -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:62376 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 36, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:21 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 37, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 11.0 (TID 38, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 11.0 (TID 39, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:21 INFO Executor:59 - Running task 3.0 in stage 11.0 (TID 39) -2016-06-13 17:50:21 INFO Executor:59 - Running task 2.0 in stage 11.0 (TID 38) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:62376 in memory (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:50:21 INFO Executor:59 - Finished task 3.0 in stage 11.0 (TID 39). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:50:21 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 36) -2016-06-13 17:50:21 INFO Executor:59 - Finished task 2.0 in stage 11.0 (TID 38). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 37) -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:50:21 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 36). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:50:21 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 37). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 11.0 (TID 38) in 25 ms on localhost (1/4) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 36) in 30 ms on localhost (2/4) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 37) in 33 ms on localhost (3/4) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:62376 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 11.0 (TID 39) in 31 ms on localhost (4/4) -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 17:50:21 INFO DAGScheduler:59 - ResultStage 11 (aggregate at GaussianMixture.scala:201) finished in 0.036 s -2016-06-13 17:50:21 INFO DAGScheduler:59 - Job 11 finished: aggregate at GaussianMixture.scala:201, took 0.063955 s -2016-06-13 17:50:21 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=65352, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:62376 in memory (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=64717, maxMem=766075207 -2016-06-13 17:50:21 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.5 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:62376 (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO SparkContext:59 - Created broadcast 16 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:62376 in memory (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:62376 in memory (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 17:50:21 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 17:50:21 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Got job 12 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:50:21 INFO DAGScheduler:59 - Final stage: ResultStage 12(aggregate at GaussianMixture.scala:201) -2016-06-13 17:50:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:62376 in memory (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:21 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=56102, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(2193) called with curMem=52284, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:62376 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:62376 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:50:21 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 12 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 4 tasks -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:62376 in memory (size: 890.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 40, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:21 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 41, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 12.0 (TID 42, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 12.0 (TID 43, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:21 INFO Executor:59 - Running task 3.0 in stage 12.0 (TID 43) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:62376 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:50:21 INFO Executor:59 - Running task 2.0 in stage 12.0 (TID 42) -2016-06-13 17:50:21 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 40) -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:50:21 INFO Executor:59 - Finished task 2.0 in stage 12.0 (TID 42). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 41) -2016-06-13 17:50:21 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 40). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 12.0 (TID 42) in 16 ms on localhost (1/4) -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 40) in 21 ms on localhost (2/4) -2016-06-13 17:50:21 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 41). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 41) in 25 ms on localhost (3/4) -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:62376 in memory (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO Executor:59 - Finished task 3.0 in stage 12.0 (TID 43). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 12.0 (TID 43) in 31 ms on localhost (4/4) -2016-06-13 17:50:21 INFO DAGScheduler:59 - ResultStage 12 (aggregate at GaussianMixture.scala:201) finished in 0.035 s -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:62376 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 17:50:21 INFO DAGScheduler:59 - Job 12 finished: aggregate at GaussianMixture.scala:201, took 0.052859 s -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=19835, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 560.0 B, free 730.6 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=20395, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.6 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:62376 (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO SparkContext:59 - Created broadcast 18 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:50:21 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Got job 13 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:50:21 INFO DAGScheduler:59 - Final stage: ResultStage 13(aggregate at GaussianMixture.scala:201) -2016-06-13 17:50:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=21295, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 4.0 KB, free 730.6 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(2193) called with curMem=25359, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.6 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:62376 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:50:21 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 13 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 4 tasks -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 44, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 45, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 13.0 (TID 46, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 13.0 (TID 47, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:21 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 45) -2016-06-13 17:50:21 INFO Executor:59 - Running task 3.0 in stage 13.0 (TID 47) -2016-06-13 17:50:21 INFO Executor:59 - Running task 2.0 in stage 13.0 (TID 46) -2016-06-13 17:50:21 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 44) -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:50:21 INFO Executor:59 - Finished task 2.0 in stage 13.0 (TID 46). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 44). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO Executor:59 - Finished task 3.0 in stage 13.0 (TID 47). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 13.0 (TID 47) in 14 ms on localhost (1/4) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 44) in 18 ms on localhost (2/4) -2016-06-13 17:50:21 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 45). 2756 bytes result sent to driver -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 13.0 (TID 46) in 18 ms on localhost (3/4) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 45) in 22 ms on localhost (4/4) -2016-06-13 17:50:21 INFO DAGScheduler:59 - ResultStage 13 (aggregate at GaussianMixture.scala:201) finished in 0.023 s -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 17:50:21 INFO DAGScheduler:59 - Job 13 finished: aggregate at GaussianMixture.scala:201, took 0.037470 s -2016-06-13 17:50:21 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Got job 14 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:50:21 INFO DAGScheduler:59 - Final stage: ResultStage 14(first at GaussianMixture.scala:171) -2016-06-13 17:50:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=27552, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(1251) called with curMem=29544, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 1251.0 B, free 730.6 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:62376 (size: 1251.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 14 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 1 tasks -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 48, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:21 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 48) -2016-06-13 17:50:21 INFO CacheManager:59 - Partition rdd_5_0 not found, computing it -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=30795, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block rdd_5_0 stored as values in memory (estimated size 16.0 B, free 730.6 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added rdd_5_0 in memory on localhost:62376 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 48). 1533 bytes result sent to driver -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 48) in 9 ms on localhost (1/1) -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 17:50:21 INFO DAGScheduler:59 - ResultStage 14 (first at GaussianMixture.scala:171) finished in 0.009 s -2016-06-13 17:50:21 INFO DAGScheduler:59 - Job 14 finished: first at GaussianMixture.scala:171, took 0.029728 s -2016-06-13 17:50:21 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Got job 15 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:50:21 INFO DAGScheduler:59 - Final stage: ResultStage 15(first at GaussianMixture.scala:171) -2016-06-13 17:50:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=30811, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(1251) called with curMem=32803, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1251.0 B, free 730.6 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:62376 (size: 1251.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 15 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 3 tasks -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 49, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 50, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 15.0 (TID 51, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:21 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 49) -2016-06-13 17:50:21 INFO Executor:59 - Running task 2.0 in stage 15.0 (TID 51) -2016-06-13 17:50:21 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 50) -2016-06-13 17:50:21 INFO CacheManager:59 - Partition rdd_5_1 not found, computing it -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=34054, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block rdd_5_1 stored as values in memory (estimated size 192.0 B, free 730.6 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added rdd_5_1 in memory on localhost:62376 (size: 192.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 49). 2144 bytes result sent to driver -2016-06-13 17:50:21 INFO CacheManager:59 - Partition rdd_5_3 not found, computing it -2016-06-13 17:50:21 INFO CacheManager:59 - Partition rdd_5_2 not found, computing it -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=34246, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block rdd_5_3 stored as values in memory (estimated size 184.0 B, free 730.6 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added rdd_5_3 in memory on localhost:62376 (size: 184.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 49) in 17 ms on localhost (1/3) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=34430, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block rdd_5_2 stored as values in memory (estimated size 192.0 B, free 730.6 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added rdd_5_2 in memory on localhost:62376 (size: 192.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO Executor:59 - Finished task 2.0 in stage 15.0 (TID 51). 2132 bytes result sent to driver -2016-06-13 17:50:21 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 50). 2144 bytes result sent to driver -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 15.0 (TID 51) in 21 ms on localhost (2/3) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 50) in 23 ms on localhost (3/3) -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 17:50:21 INFO DAGScheduler:59 - ResultStage 15 (first at GaussianMixture.scala:171) finished in 0.025 s -2016-06-13 17:50:21 INFO DAGScheduler:59 - Job 15 finished: first at GaussianMixture.scala:171, took 0.042957 s -2016-06-13 17:50:21 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Got job 16 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:50:21 INFO DAGScheduler:59 - Final stage: ResultStage 16(takeSample at GaussianMixture.scala:184) -2016-06-13 17:50:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=34622, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 17:50:21 INFO MemoryStore:59 - ensureFreeSpace(1194) called with curMem=36454, maxMem=766075207 -2016-06-13 17:50:21 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 1194.0 B, free 730.6 MB) -2016-06-13 17:50:21 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:62376 (size: 1194.0 B, free: 730.6 MB) -2016-06-13 17:50:21 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 16 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:50:21 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 4 tasks -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 52, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 53, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 16.0 (TID 54, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 16.0 (TID 55, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:21 INFO Executor:59 - Running task 2.0 in stage 16.0 (TID 54) -2016-06-13 17:50:21 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 52) -2016-06-13 17:50:21 INFO Executor:59 - Running task 3.0 in stage 16.0 (TID 55) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 52). 2082 bytes result sent to driver -2016-06-13 17:50:21 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 53) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 3.0 in stage 16.0 (TID 55). 2082 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 53). 2082 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 2.0 in stage 16.0 (TID 54). 2082 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 16.0 (TID 55) in 14 ms on localhost (1/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 52) in 19 ms on localhost (2/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 53) in 19 ms on localhost (3/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 16.0 (TID 54) in 19 ms on localhost (4/4) -2016-06-13 17:50:22 INFO DAGScheduler:59 - ResultStage 16 (takeSample at GaussianMixture.scala:184) finished in 0.021 s -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 17:50:22 INFO DAGScheduler:59 - Job 16 finished: takeSample at GaussianMixture.scala:184, took 0.032697 s -2016-06-13 17:50:22 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Got job 17 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:50:22 INFO DAGScheduler:59 - Final stage: ResultStage 17(takeSample at GaussianMixture.scala:184) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting ResultStage 17 (PartitionwiseSampledRDD[6] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=37648, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(7883) called with curMem=56872, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:62376 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (PartitionwiseSampledRDD[6] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 56, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 57, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 58, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 59, localhost, PROCESS_LOCAL, 2373 bytes) -2016-06-13 17:50:22 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 59) -2016-06-13 17:50:22 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 58) -2016-06-13 17:50:22 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 56) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 59). 2702 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 56). 2048 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 59) in 13 ms on localhost (1/4) -2016-06-13 17:50:22 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 58). 2714 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 56) in 20 ms on localhost (2/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 58) in 20 ms on localhost (3/4) -2016-06-13 17:50:22 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 57) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 57). 2684 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 57) in 36 ms on localhost (4/4) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 17:50:22 INFO DAGScheduler:59 - ResultStage 17 (takeSample at GaussianMixture.scala:184) finished in 0.037 s -2016-06-13 17:50:22 INFO DAGScheduler:59 - Job 17 finished: takeSample at GaussianMixture.scala:184, took 0.056263 s -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=64755, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(899) called with curMem=65315, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 899.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:62376 (size: 899.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 24 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:50:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Got job 18 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:50:22 INFO DAGScheduler:59 - Final stage: ResultStage 18(aggregate at GaussianMixture.scala:201) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=66214, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(2204) called with curMem=70278, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:62376 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 60, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 61, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 62, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 63, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:22 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 60) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 60). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 61) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 61). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 62) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 62). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 63) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 63). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 60) in 47 ms on localhost (1/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 61) in 47 ms on localhost (2/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 62) in 48 ms on localhost (3/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 63) in 49 ms on localhost (4/4) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 17:50:22 INFO DAGScheduler:59 - ResultStage 18 (aggregate at GaussianMixture.scala:201) finished in 0.058 s -2016-06-13 17:50:22 INFO DAGScheduler:59 - Job 18 finished: aggregate at GaussianMixture.scala:201, took 0.081502 s -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=72482, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=73042, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:62376 (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 26 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:50:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Got job 19 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:50:22 INFO DAGScheduler:59 - Final stage: ResultStage 19(aggregate at GaussianMixture.scala:201) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=73942, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=78006, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:62376 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 19 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 4 tasks -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 64, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 65, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 19.0 (TID 66, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 19.0 (TID 67, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:22 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 65) -2016-06-13 17:50:22 INFO Executor:59 - Running task 2.0 in stage 19.0 (TID 66) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 65). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO Executor:59 - Running task 3.0 in stage 19.0 (TID 67) -2016-06-13 17:50:22 INFO Executor:59 - Finished task 2.0 in stage 19.0 (TID 66). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 3.0 in stage 19.0 (TID 67). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 65) in 16 ms on localhost (1/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 19.0 (TID 66) in 19 ms on localhost (2/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 19.0 (TID 67) in 19 ms on localhost (3/4) -2016-06-13 17:50:22 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 64) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 64). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 64) in 31 ms on localhost (4/4) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 17:50:22 INFO DAGScheduler:59 - ResultStage 19 (aggregate at GaussianMixture.scala:201) finished in 0.032 s -2016-06-13 17:50:22 INFO DAGScheduler:59 - Job 19 finished: aggregate at GaussianMixture.scala:201, took 0.052258 s -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=80200, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=80760, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:62376 (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 28 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:50:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Got job 20 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:50:22 INFO DAGScheduler:59 - Final stage: ResultStage 20(aggregate at GaussianMixture.scala:201) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=81660, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=85724, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:62376 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 68, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 69, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 70, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 71, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:22 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 70) -2016-06-13 17:50:22 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 69) -2016-06-13 17:50:22 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 68) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 70). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 71) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 69). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 71). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 68). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 69) in 16 ms on localhost (1/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 70) in 16 ms on localhost (2/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 71) in 16 ms on localhost (3/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 68) in 20 ms on localhost (4/4) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 17:50:22 INFO DAGScheduler:59 - ResultStage 20 (aggregate at GaussianMixture.scala:201) finished in 0.020 s -2016-06-13 17:50:22 INFO DAGScheduler:59 - Job 20 finished: aggregate at GaussianMixture.scala:201, took 0.032209 s -2016-06-13 17:50:22 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Got job 21 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:50:22 INFO DAGScheduler:59 - Final stage: ResultStage 21(first at GaussianMixture.scala:171) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=87918, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=89910, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:62376 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 21 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 1 tasks -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 72, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:22 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 72) -2016-06-13 17:50:22 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=91162, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 16.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:62376 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 72). 1533 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 72) in 9 ms on localhost (1/1) -2016-06-13 17:50:22 INFO DAGScheduler:59 - ResultStage 21 (first at GaussianMixture.scala:171) finished in 0.010 s -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 17:50:22 INFO DAGScheduler:59 - Job 21 finished: first at GaussianMixture.scala:171, took 0.023416 s -2016-06-13 17:50:22 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Got job 22 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:50:22 INFO DAGScheduler:59 - Final stage: ResultStage 22(first at GaussianMixture.scala:171) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=91178, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=93170, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:62376 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 22 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 3 tasks -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 73, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 74, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 75, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:22 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 73) -2016-06-13 17:50:22 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 75) -2016-06-13 17:50:22 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 74) -2016-06-13 17:50:22 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=94422, maxMem=766075207 -2016-06-13 17:50:22 INFO CacheManager:59 - Partition rdd_7_2 not found, computing it -2016-06-13 17:50:22 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 192.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=94614, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block rdd_7_2 stored as values in memory (estimated size 192.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:62376 (size: 192.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 73). 2144 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added rdd_7_2 in memory on localhost:62376 (size: 192.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO CacheManager:59 - Partition rdd_7_3 not found, computing it -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=94806, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block rdd_7_3 stored as values in memory (estimated size 184.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 73) in 15 ms on localhost (1/3) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added rdd_7_3 in memory on localhost:62376 (size: 184.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 74). 2144 bytes result sent to driver -2016-06-13 17:50:22 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 75). 2132 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 74) in 19 ms on localhost (2/3) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 75) in 20 ms on localhost (3/3) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 17:50:22 INFO DAGScheduler:59 - ResultStage 22 (first at GaussianMixture.scala:171) finished in 0.023 s -2016-06-13 17:50:22 INFO DAGScheduler:59 - Job 22 finished: first at GaussianMixture.scala:171, took 0.038558 s -2016-06-13 17:50:22 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Got job 23 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:50:22 INFO DAGScheduler:59 - Final stage: ResultStage 23(takeSample at GaussianMixture.scala:184) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=94990, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 1832.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=96822, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:62376 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 23 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 76, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 77, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 78, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 79, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:22 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 78) -2016-06-13 17:50:22 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 76) -2016-06-13 17:50:22 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 79) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 76). 2082 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 79). 2082 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 76) in 10 ms on localhost (1/4) -2016-06-13 17:50:22 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 77) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 78). 2082 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 77). 2082 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 79) in 15 ms on localhost (2/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 78) in 17 ms on localhost (3/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 77) in 19 ms on localhost (4/4) -2016-06-13 17:50:22 INFO DAGScheduler:59 - ResultStage 23 (takeSample at GaussianMixture.scala:184) finished in 0.020 s -2016-06-13 17:50:22 INFO DAGScheduler:59 - Job 23 finished: takeSample at GaussianMixture.scala:184, took 0.032804 s -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 17:50:22 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Got job 24 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:50:22 INFO DAGScheduler:59 - Final stage: ResultStage 24(takeSample at GaussianMixture.scala:184) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting ResultStage 24 (PartitionwiseSampledRDD[8] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=98017, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(7882) called with curMem=117241, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:62376 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 24 (PartitionwiseSampledRDD[8] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 4 tasks -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 80, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 81, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 82, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 24.0 (TID 83, localhost, PROCESS_LOCAL, 2373 bytes) -2016-06-13 17:50:22 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 80) -2016-06-13 17:50:22 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 81) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 80). 2048 bytes result sent to driver -2016-06-13 17:50:22 INFO Executor:59 - Running task 3.0 in stage 24.0 (TID 83) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 81). 2719 bytes result sent to driver -2016-06-13 17:50:22 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 82) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 3.0 in stage 24.0 (TID 83). 2722 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 80) in 16 ms on localhost (1/4) -2016-06-13 17:50:22 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 82). 2714 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 82) in 17 ms on localhost (2/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 24.0 (TID 83) in 18 ms on localhost (3/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 81) in 19 ms on localhost (4/4) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-06-13 17:50:22 INFO DAGScheduler:59 - ResultStage 24 (takeSample at GaussianMixture.scala:184) finished in 0.021 s -2016-06-13 17:50:22 INFO DAGScheduler:59 - Job 24 finished: takeSample at GaussianMixture.scala:184, took 0.036775 s -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=125123, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(895) called with curMem=125683, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 895.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:62376 (size: 895.0 B, free: 730.5 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 34 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:50:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Got job 25 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:50:22 INFO DAGScheduler:59 - Final stage: ResultStage 25(aggregate at GaussianMixture.scala:201) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=126578, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(2198) called with curMem=130642, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:62376 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 25 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 4 tasks -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 84, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 85, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 25.0 (TID 86, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 25.0 (TID 87, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:22 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 85) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 85). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 84) -2016-06-13 17:50:22 INFO Executor:59 - Running task 3.0 in stage 25.0 (TID 87) -2016-06-13 17:50:22 INFO Executor:59 - Running task 2.0 in stage 25.0 (TID 86) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 84). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 3.0 in stage 25.0 (TID 87). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO Executor:59 - Finished task 2.0 in stage 25.0 (TID 86). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 84) in 22 ms on localhost (1/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 25.0 (TID 87) in 20 ms on localhost (2/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 85) in 23 ms on localhost (3/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 25.0 (TID 86) in 22 ms on localhost (4/4) -2016-06-13 17:50:22 INFO DAGScheduler:59 - ResultStage 25 (aggregate at GaussianMixture.scala:201) finished in 0.025 s -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-06-13 17:50:22 INFO DAGScheduler:59 - Job 25 finished: aggregate at GaussianMixture.scala:201, took 0.039056 s -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=132840, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=133400, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:62376 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 36 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:50:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Got job 26 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:50:22 INFO DAGScheduler:59 - Final stage: ResultStage 26(aggregate at GaussianMixture.scala:201) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=134300, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(2195) called with curMem=138364, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:62376 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 26 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 88, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 89, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 90, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 91, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:22 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 91) -2016-06-13 17:50:22 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 90) -2016-06-13 17:50:22 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 89) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 90). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 88) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 91). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 88). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 89). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 91) in 15 ms on localhost (1/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 88) in 18 ms on localhost (2/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 90) in 17 ms on localhost (3/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 89) in 18 ms on localhost (4/4) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-06-13 17:50:22 INFO DAGScheduler:59 - ResultStage 26 (aggregate at GaussianMixture.scala:201) finished in 0.021 s -2016-06-13 17:50:22 INFO DAGScheduler:59 - Job 26 finished: aggregate at GaussianMixture.scala:201, took 0.034797 s -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=140559, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=141119, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:62376 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:62376 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 38 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:50:22 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:62376 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Got job 27 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:50:22 INFO DAGScheduler:59 - Final stage: ResultStage 27(aggregate at GaussianMixture.scala:201) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:62376 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=128038, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(2195) called with curMem=132102, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:62376 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:62376 in memory (size: 895.0 B, free: 730.5 MB) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 27 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 4 tasks -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 92, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 93, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 27.0 (TID 94, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 27.0 (TID 95, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:22 INFO Executor:59 - Running task 2.0 in stage 27.0 (TID 94) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:62376 in memory (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:50:22 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 93) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 2.0 in stage 27.0 (TID 94). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 93). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 92) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 27.0 (TID 94) in 9 ms on localhost (1/4) -2016-06-13 17:50:22 INFO Executor:59 - Running task 3.0 in stage 27.0 (TID 95) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 92). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 93) in 14 ms on localhost (2/4) -2016-06-13 17:50:22 INFO Executor:59 - Finished task 3.0 in stage 27.0 (TID 95). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 92) in 18 ms on localhost (3/4) -2016-06-13 17:50:22 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 27.0 (TID 95) in 17 ms on localhost (4/4) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-06-13 17:50:22 INFO DAGScheduler:59 - ResultStage 27 (aggregate at GaussianMixture.scala:201) finished in 0.020 s -2016-06-13 17:50:22 INFO DAGScheduler:59 - Job 27 finished: aggregate at GaussianMixture.scala:201, took 0.035424 s -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:62376 in memory (size: 1195.0 B, free: 730.5 MB) -2016-06-13 17:50:22 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:62376 in memory (size: 1252.0 B, free: 730.5 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Got job 28 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:50:22 INFO DAGScheduler:59 - Final stage: ResultStage 28(first at GaussianMixture.scala:171) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:22 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=99465, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=101457, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:62376 (size: 1252.0 B, free: 730.5 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 28 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:62376 in memory (size: 1252.0 B, free: 730.5 MB) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 1 tasks -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 96, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:22 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 17:50:22 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 96) -2016-06-13 17:50:22 INFO CacheManager:59 - Partition rdd_9_0 not found, computing it -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:62376 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=97271, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block rdd_9_0 stored as values in memory (estimated size 16.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added rdd_9_0 in memory on localhost:62376 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 96). 1533 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 96) in 9 ms on localhost (1/1) -2016-06-13 17:50:22 INFO DAGScheduler:59 - ResultStage 28 (first at GaussianMixture.scala:171) finished in 0.009 s -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-06-13 17:50:22 INFO DAGScheduler:59 - Job 28 finished: first at GaussianMixture.scala:171, took 0.024074 s -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:62376 in memory (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:62376 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:50:22 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 17:50:22 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Got job 29 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:62376 in memory (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Final stage: ResultStage 29(first at GaussianMixture.scala:171) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=81841, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:62376 in memory (size: 2.2 KB, free: 730.6 MB) -2016-06-13 17:50:22 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=79769, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:62376 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:62376 in memory (size: 899.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 29 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 3 tasks -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 97, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 98, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 29.0 (TID 99, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:22 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 97) -2016-06-13 17:50:22 INFO Executor:59 - Running task 2.0 in stage 29.0 (TID 99) -2016-06-13 17:50:22 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 98) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:62376 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:50:22 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 17:50:22 INFO CacheManager:59 - Partition rdd_9_1 not found, computing it -2016-06-13 17:50:22 INFO CacheManager:59 - Partition rdd_9_2 not found, computing it -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=51261, maxMem=766075207 -2016-06-13 17:50:22 INFO CacheManager:59 - Partition rdd_9_3 not found, computing it -2016-06-13 17:50:22 INFO MemoryStore:59 - Block rdd_9_1 stored as values in memory (estimated size 192.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:62376 in memory (size: 1194.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=51453, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block rdd_9_2 stored as values in memory (estimated size 192.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added rdd_9_1 in memory on localhost:62376 (size: 192.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=49813, maxMem=766075207 -2016-06-13 17:50:22 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block rdd_9_3 stored as values in memory (estimated size 184.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added rdd_9_2 in memory on localhost:62376 (size: 192.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 97). 2144 bytes result sent to driver -2016-06-13 17:50:22 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 98). 2144 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added rdd_9_3 in memory on localhost:62376 (size: 184.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 97) in 10 ms on localhost (1/3) -2016-06-13 17:50:22 INFO Executor:59 - Finished task 2.0 in stage 29.0 (TID 99). 2132 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 98) in 11 ms on localhost (2/3) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:62376 in memory (size: 1251.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 29.0 (TID 99) in 11 ms on localhost (3/3) -2016-06-13 17:50:22 INFO DAGScheduler:59 - ResultStage 29 (first at GaussianMixture.scala:171) finished in 0.013 s -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 17:50:22 INFO DAGScheduler:59 - Job 29 finished: first at GaussianMixture.scala:171, took 0.025724 s -2016-06-13 17:50:22 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 17:50:22 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:62376 in memory (size: 1251.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Got job 30 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:50:22 INFO DAGScheduler:59 - Final stage: ResultStage 30(takeSample at GaussianMixture.scala:184) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=43511, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 1832.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManager:59 - Removing RDD 5 -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=45343, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:62376 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 100, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:22 INFO ContextCleaner:59 - Cleaned RDD 5 -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 101, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 102, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 103, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:22 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 103) -2016-06-13 17:50:22 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 101) -2016-06-13 17:50:22 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 102) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 103). 2082 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:62376 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:50:22 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 100) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 103) in 7 ms on localhost (1/4) -2016-06-13 17:50:22 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 102). 2082 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 100). 2082 bytes result sent to driver -2016-06-13 17:50:22 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 17:50:22 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 101). 2082 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 102) in 11 ms on localhost (2/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 100) in 13 ms on localhost (3/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 101) in 13 ms on localhost (4/4) -2016-06-13 17:50:22 INFO DAGScheduler:59 - ResultStage 30 (takeSample at GaussianMixture.scala:184) finished in 0.014 s -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 17:50:22 INFO DAGScheduler:59 - Job 30 finished: takeSample at GaussianMixture.scala:184, took 0.026478 s -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:62376 in memory (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:62376 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:50:22 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 17:50:22 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:62376 in memory (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Got job 31 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:50:22 INFO DAGScheduler:59 - Final stage: ResultStage 31(takeSample at GaussianMixture.scala:184) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting ResultStage 31 (PartitionwiseSampledRDD[10] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:62376 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:50:22 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=24263, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(7890) called with curMem=42596, maxMem=766075207 -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:62376 in memory (size: 891.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:62376 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 31 (PartitionwiseSampledRDD[10] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 4 tasks -2016-06-13 17:50:22 INFO BlockManager:59 - Removing RDD 3 -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 104, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 105, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 31.0 (TID 106, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 31.0 (TID 107, localhost, PROCESS_LOCAL, 2373 bytes) -2016-06-13 17:50:22 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 104) -2016-06-13 17:50:22 INFO Executor:59 - Running task 2.0 in stage 31.0 (TID 106) -2016-06-13 17:50:22 INFO Executor:59 - Running task 3.0 in stage 31.0 (TID 107) -2016-06-13 17:50:22 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 105) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 105). 2684 bytes result sent to driver -2016-06-13 17:50:22 INFO ContextCleaner:59 - Cleaned RDD 3 -2016-06-13 17:50:22 INFO Executor:59 - Finished task 2.0 in stage 31.0 (TID 106). 2699 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 105) in 9 ms on localhost (1/4) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 104). 2048 bytes result sent to driver -2016-06-13 17:50:22 INFO Executor:59 - Finished task 3.0 in stage 31.0 (TID 107). 2687 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 31.0 (TID 106) in 12 ms on localhost (2/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 104) in 14 ms on localhost (3/4) -2016-06-13 17:50:22 INFO DAGScheduler:59 - ResultStage 31 (takeSample at GaussianMixture.scala:184) finished in 0.015 s -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 31.0 (TID 107) in 13 ms on localhost (4/4) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 17:50:22 INFO DAGScheduler:59 - Job 31 finished: takeSample at GaussianMixture.scala:184, took 0.025785 s -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=49342, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(882) called with curMem=49902, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 882.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:62376 (size: 882.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 44 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:50:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Got job 32 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:50:22 INFO DAGScheduler:59 - Final stage: ResultStage 32(aggregate at GaussianMixture.scala:201) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=50784, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(2185) called with curMem=54848, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:62376 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 32 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 4 tasks -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 108, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 109, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 32.0 (TID 110, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 32.0 (TID 111, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:22 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 108) -2016-06-13 17:50:22 INFO Executor:59 - Running task 2.0 in stage 32.0 (TID 110) -2016-06-13 17:50:22 INFO Executor:59 - Running task 3.0 in stage 32.0 (TID 111) -2016-06-13 17:50:22 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 109) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 108). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 3.0 in stage 32.0 (TID 111). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 109). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO Executor:59 - Finished task 2.0 in stage 32.0 (TID 110). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 108) in 8 ms on localhost (1/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 32.0 (TID 111) in 7 ms on localhost (2/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 109) in 9 ms on localhost (3/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 32.0 (TID 110) in 10 ms on localhost (4/4) -2016-06-13 17:50:22 INFO DAGScheduler:59 - ResultStage 32 (aggregate at GaussianMixture.scala:201) finished in 0.011 s -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 17:50:22 INFO DAGScheduler:59 - Job 32 finished: aggregate at GaussianMixture.scala:201, took 0.022798 s -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=57033, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=57593, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:62376 (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 46 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:50:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Got job 33 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:50:22 INFO DAGScheduler:59 - Final stage: ResultStage 33(aggregate at GaussianMixture.scala:201) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=58493, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(2195) called with curMem=62557, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:62376 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 112, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 113, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 114, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 115, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:22 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 113) -2016-06-13 17:50:22 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 114) -2016-06-13 17:50:22 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 115) -2016-06-13 17:50:22 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 112) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 113). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 112). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 115). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 113) in 13 ms on localhost (1/4) -2016-06-13 17:50:22 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 114). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 115) in 13 ms on localhost (2/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 112) in 15 ms on localhost (3/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 114) in 15 ms on localhost (4/4) -2016-06-13 17:50:22 INFO DAGScheduler:59 - ResultStage 33 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 17:50:22 INFO DAGScheduler:59 - Job 33 finished: aggregate at GaussianMixture.scala:201, took 0.027195 s -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=64752, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=65312, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:62376 (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 48 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:50:22 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Got job 34 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:50:22 INFO DAGScheduler:59 - Final stage: ResultStage 34(aggregate at GaussianMixture.scala:201) -2016-06-13 17:50:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=66212, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO MemoryStore:59 - ensureFreeSpace(2195) called with curMem=70276, maxMem=766075207 -2016-06-13 17:50:22 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:50:22 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:62376 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:50:22 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:50:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 116, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 117, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 118, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 119, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:50:22 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 116) -2016-06-13 17:50:22 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 119) -2016-06-13 17:50:22 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 117) -2016-06-13 17:50:22 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 118) -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 117). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 118). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 116). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:50:22 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 119). 2756 bytes result sent to driver -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 117) in 10 ms on localhost (1/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 118) in 11 ms on localhost (2/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 116) in 13 ms on localhost (3/4) -2016-06-13 17:50:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 119) in 12 ms on localhost (4/4) -2016-06-13 17:50:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-06-13 17:50:22 INFO DAGScheduler:59 - ResultStage 34 (aggregate at GaussianMixture.scala:201) finished in 0.014 s -2016-06-13 17:50:22 INFO DAGScheduler:59 - Job 34 finished: aggregate at GaussianMixture.scala:201, took 0.025551 s -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 17:50:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 17:50:23 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 17:50:23 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 17:50:23 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 17:50:23 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 17:50:23 INFO BlockManager:59 - BlockManager stopped -2016-06-13 17:50:23 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 17:50:23 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 17:50:23 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 17:50:23 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 17:50:23 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 17:50:23 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 17:50:23 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-dee942b7-c8a2-4baf-881b-5d5cb2997d3e -2016-06-13 17:54:19 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 17:54:19 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 17:54:19 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 17:54:19 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 17:54:19 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 17:54:20 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 17:54:20 INFO Remoting:74 - Starting remoting -2016-06-13 17:54:20 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:62398] -2016-06-13 17:54:20 INFO Utils:59 - Successfully started service 'sparkDriver' on port 62398. -2016-06-13 17:54:20 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 17:54:20 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 17:54:20 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-81735d25-ebe6-4440-a598-0f660ca7f135 -2016-06-13 17:54:20 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 17:54:20 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-17bd487a-5160-49ec-b4c3-75a180c56fea\httpd-9b3aa8a9-5313-48d5-b866-4a075edf83e8 -2016-06-13 17:54:20 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 17:54:20 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:54:20 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:62399 -2016-06-13 17:54:20 INFO Utils:59 - Successfully started service 'HTTP file server' on port 62399. -2016-06-13 17:54:21 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 17:54:21 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 17:54:21 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 17:54:21 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 17:54:21 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 17:54:21 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 17:54:21 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 17:54:21 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 62418. -2016-06-13 17:54:21 INFO NettyBlockTransferService:59 - Server created on 62418 -2016-06-13 17:54:21 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 17:54:21 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:62418 with 730.6 MB RAM, BlockManagerId(driver, localhost, 62418) -2016-06-13 17:54:21 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 17:54:22 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:22 INFO DAGScheduler:59 - Got job 0 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:54:22 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GaussianMixture.scala:171) -2016-06-13 17:54:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:22 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:22 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=0, maxMem=766075207 -2016-06-13 17:54:22 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:54:22 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1992, maxMem=766075207 -2016-06-13 17:54:22 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.6 MB) -2016-06-13 17:54:22 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:62418 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:54:22 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:54:22 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 17:54:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:22 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 17:54:22 INFO CacheManager:59 - Partition rdd_1_0 not found, computing it -2016-06-13 17:54:22 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=3242, maxMem=766075207 -2016-06-13 17:54:22 INFO MemoryStore:59 - Block rdd_1_0 stored as values in memory (estimated size 16.0 B, free 730.6 MB) -2016-06-13 17:54:22 INFO BlockManagerInfo:59 - Added rdd_1_0 in memory on localhost:62418 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:54:22 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1533 bytes result sent to driver -2016-06-13 17:54:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 69 ms on localhost (1/1) -2016-06-13 17:54:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 17:54:22 INFO DAGScheduler:59 - ResultStage 0 (first at GaussianMixture.scala:171) finished in 0.082 s -2016-06-13 17:54:22 INFO DAGScheduler:59 - Job 0 finished: first at GaussianMixture.scala:171, took 0.291408 s -2016-06-13 17:54:22 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:22 INFO DAGScheduler:59 - Got job 1 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:54:22 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at GaussianMixture.scala:171) -2016-06-13 17:54:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:22 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:22 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=3258, maxMem=766075207 -2016-06-13 17:54:22 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:54:22 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=5250, maxMem=766075207 -2016-06-13 17:54:22 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.6 MB) -2016-06-13 17:54:22 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:62418 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:54:22 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:22 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:54:22 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 3 tasks -2016-06-13 17:54:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:22 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 17:54:22 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 3) -2016-06-13 17:54:22 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 17:54:22 INFO CacheManager:59 - Partition rdd_1_3 not found, computing it -2016-06-13 17:54:22 INFO CacheManager:59 - Partition rdd_1_2 not found, computing it -2016-06-13 17:54:22 INFO CacheManager:59 - Partition rdd_1_1 not found, computing it -2016-06-13 17:54:22 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:62418 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:54:22 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 17:54:22 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=3258, maxMem=766075207 -2016-06-13 17:54:22 INFO MemoryStore:59 - Block rdd_1_3 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:54:22 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=3354, maxMem=766075207 -2016-06-13 17:54:22 INFO MemoryStore:59 - Block rdd_1_1 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:54:22 INFO BlockManagerInfo:59 - Added rdd_1_3 in memory on localhost:62418 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:54:22 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=3450, maxMem=766075207 -2016-06-13 17:54:22 INFO MemoryStore:59 - Block rdd_1_2 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:54:22 INFO BlockManagerInfo:59 - Added rdd_1_1 in memory on localhost:62418 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:54:22 INFO BlockManagerInfo:59 - Added rdd_1_2 in memory on localhost:62418 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:54:22 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 3). 1696 bytes result sent to driver -2016-06-13 17:54:22 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 1696 bytes result sent to driver -2016-06-13 17:54:22 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 1696 bytes result sent to driver -2016-06-13 17:54:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 325 ms on localhost (1/3) -2016-06-13 17:54:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 3) in 324 ms on localhost (2/3) -2016-06-13 17:54:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 335 ms on localhost (3/3) -2016-06-13 17:54:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 17:54:22 INFO DAGScheduler:59 - ResultStage 1 (first at GaussianMixture.scala:171) finished in 0.338 s -2016-06-13 17:54:22 INFO DAGScheduler:59 - Job 1 finished: first at GaussianMixture.scala:171, took 0.393005 s -2016-06-13 17:54:22 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:22 INFO DAGScheduler:59 - Got job 2 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:22 INFO DAGScheduler:59 - Final stage: ResultStage 2(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:22 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:22 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=3546, maxMem=766075207 -2016-06-13 17:54:22 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 17:54:22 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=5378, maxMem=766075207 -2016-06-13 17:54:22 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.6 MB) -2016-06-13 17:54:22 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:62418 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:54:22 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 2 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:54:22 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 4 tasks -2016-06-13 17:54:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:22 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 17:54:22 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 17:54:22 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-06-13 17:54:22 INFO Executor:59 - Running task 3.0 in stage 2.0 (TID 7) -2016-06-13 17:54:22 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:54:22 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:54:23 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2082 bytes result sent to driver -2016-06-13 17:54:23 INFO Executor:59 - Finished task 3.0 in stage 2.0 (TID 7). 2082 bytes result sent to driver -2016-06-13 17:54:23 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 6). 2082 bytes result sent to driver -2016-06-13 17:54:23 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2082 bytes result sent to driver -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 2.0 (TID 7) in 25 ms on localhost (1/4) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 6) in 27 ms on localhost (2/4) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 30 ms on localhost (3/4) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 29 ms on localhost (4/4) -2016-06-13 17:54:23 INFO DAGScheduler:59 - ResultStage 2 (takeSample at GaussianMixture.scala:184) finished in 0.031 s -2016-06-13 17:54:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 17:54:23 INFO DAGScheduler:59 - Job 2 finished: takeSample at GaussianMixture.scala:184, took 0.046103 s -2016-06-13 17:54:23 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:23 INFO DAGScheduler:59 - Got job 3 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:23 INFO DAGScheduler:59 - Final stage: ResultStage 3(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:23 INFO DAGScheduler:59 - Submitting ResultStage 3 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=6573, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 18.8 KB, free 730.6 MB) -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(7883) called with curMem=25797, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.6 MB) -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:62418 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:54:23 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 3 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:23 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 4 tasks -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 3.0 (TID 11, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:54:23 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 8) -2016-06-13 17:54:23 INFO Executor:59 - Running task 3.0 in stage 3.0 (TID 11) -2016-06-13 17:54:23 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 10) -2016-06-13 17:54:23 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 9) -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:54:23 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 8). 2048 bytes result sent to driver -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:54:23 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 9). 2269 bytes result sent to driver -2016-06-13 17:54:23 INFO Executor:59 - Finished task 3.0 in stage 3.0 (TID 11). 2269 bytes result sent to driver -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 8) in 28 ms on localhost (1/4) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 9) in 28 ms on localhost (2/4) -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:54:23 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 10). 2279 bytes result sent to driver -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 3.0 (TID 11) in 28 ms on localhost (3/4) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 10) in 31 ms on localhost (4/4) -2016-06-13 17:54:23 INFO DAGScheduler:59 - ResultStage 3 (takeSample at GaussianMixture.scala:184) finished in 0.037 s -2016-06-13 17:54:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 17:54:23 INFO DAGScheduler:59 - Job 3 finished: takeSample at GaussianMixture.scala:184, took 0.058093 s -2016-06-13 17:54:23 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 17:54:23 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 17:54:23 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 17:54:23 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=33680, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 464.0 B, free 730.6 MB) -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(854) called with curMem=34144, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 854.0 B, free 730.6 MB) -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:62418 (size: 854.0 B, free: 730.6 MB) -2016-06-13 17:54:23 INFO SparkContext:59 - Created broadcast 4 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:23 INFO DAGScheduler:59 - Got job 4 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:23 INFO DAGScheduler:59 - Final stage: ResultStage 4(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:23 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=34998, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(2148) called with curMem=38918, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:23 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:54:23 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 4 tasks -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 12, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 14, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 4.0 (TID 15, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:23 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 12) -2016-06-13 17:54:23 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 13) -2016-06-13 17:54:23 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 14) -2016-06-13 17:54:23 INFO Executor:59 - Running task 3.0 in stage 4.0 (TID 15) -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:54:23 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 12). 2708 bytes result sent to driver -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 12) in 21 ms on localhost (1/4) -2016-06-13 17:54:23 INFO Executor:59 - Finished task 3.0 in stage 4.0 (TID 15). 2708 bytes result sent to driver -2016-06-13 17:54:23 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 14). 2708 bytes result sent to driver -2016-06-13 17:54:23 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 13). 2708 bytes result sent to driver -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 4.0 (TID 15) in 27 ms on localhost (2/4) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 14) in 28 ms on localhost (3/4) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 13) in 30 ms on localhost (4/4) -2016-06-13 17:54:23 INFO DAGScheduler:59 - ResultStage 4 (aggregate at GaussianMixture.scala:201) finished in 0.032 s -2016-06-13 17:54:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 17:54:23 INFO DAGScheduler:59 - Job 4 finished: aggregate at GaussianMixture.scala:201, took 0.046670 s -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=41066, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=41530, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:62418 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:54:23 INFO SparkContext:59 - Created broadcast 6 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:23 INFO DAGScheduler:59 - Got job 5 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:23 INFO DAGScheduler:59 - Final stage: ResultStage 5(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:23 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=42383, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(2150) called with curMem=46303, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:23 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:54:23 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 4 tasks -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 16, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 17, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 18, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 5.0 (TID 19, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:23 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 18) -2016-06-13 17:54:23 INFO Executor:59 - Running task 3.0 in stage 5.0 (TID 19) -2016-06-13 17:54:23 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 16) -2016-06-13 17:54:23 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 17) -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:54:23 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 17). 2708 bytes result sent to driver -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:54:23 INFO Executor:59 - Finished task 3.0 in stage 5.0 (TID 19). 2708 bytes result sent to driver -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 17) in 18 ms on localhost (1/4) -2016-06-13 17:54:23 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 18). 2708 bytes result sent to driver -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 5.0 (TID 19) in 17 ms on localhost (2/4) -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:54:23 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 16). 2708 bytes result sent to driver -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 18) in 21 ms on localhost (3/4) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 16) in 26 ms on localhost (4/4) -2016-06-13 17:54:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 17:54:23 INFO DAGScheduler:59 - ResultStage 5 (aggregate at GaussianMixture.scala:201) finished in 0.026 s -2016-06-13 17:54:23 INFO DAGScheduler:59 - Job 5 finished: aggregate at GaussianMixture.scala:201, took 0.043136 s -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=48453, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=48917, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:62418 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:54:23 INFO SparkContext:59 - Created broadcast 8 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:23 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:23 INFO DAGScheduler:59 - Got job 6 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:23 INFO DAGScheduler:59 - Final stage: ResultStage 6(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:23 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=49770, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(2150) called with curMem=53690, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:23 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 17:54:23 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 4 tasks -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 20, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 21, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 22, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 6.0 (TID 23, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:23 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 21) -2016-06-13 17:54:23 INFO Executor:59 - Running task 3.0 in stage 6.0 (TID 23) -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 17:54:23 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 20) -2016-06-13 17:54:23 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 21). 2708 bytes result sent to driver -2016-06-13 17:54:23 INFO Executor:59 - Finished task 3.0 in stage 6.0 (TID 23). 2708 bytes result sent to driver -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 21) in 16 ms on localhost (1/4) -2016-06-13 17:54:23 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 22) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 6.0 (TID 23) in 14 ms on localhost (2/4) -2016-06-13 17:54:23 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 17:54:23 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 20). 2708 bytes result sent to driver -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:62418 in memory (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 17:54:23 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 22). 2708 bytes result sent to driver -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:23 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 20) in 30 ms on localhost (3/4) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 22) in 27 ms on localhost (4/4) -2016-06-13 17:54:23 INFO DAGScheduler:59 - ResultStage 6 (aggregate at GaussianMixture.scala:201) finished in 0.032 s -2016-06-13 17:54:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 17:54:23 INFO DAGScheduler:59 - Job 6 finished: aggregate at GaussianMixture.scala:201, took 0.072964 s -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:62418 in memory (size: 854.0 B, free: 730.6 MB) -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:62418 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:54:23 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:62418 in memory (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:54:23 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:62418 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:54:23 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 17:54:23 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:23 INFO DAGScheduler:59 - Got job 7 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:54:23 INFO DAGScheduler:59 - Final stage: ResultStage 7(first at GaussianMixture.scala:171) -2016-06-13 17:54:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:23 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=7691, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=9683, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.6 MB) -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:62418 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:54:23 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 7 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:54:23 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 1 tasks -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 24, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:23 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 24) -2016-06-13 17:54:23 INFO CacheManager:59 - Partition rdd_3_0 not found, computing it -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=10935, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block rdd_3_0 stored as values in memory (estimated size 16.0 B, free 730.6 MB) -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Added rdd_3_0 in memory on localhost:62418 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:54:23 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 24). 1533 bytes result sent to driver -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 24) in 10 ms on localhost (1/1) -2016-06-13 17:54:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 17:54:23 INFO DAGScheduler:59 - ResultStage 7 (first at GaussianMixture.scala:171) finished in 0.012 s -2016-06-13 17:54:23 INFO DAGScheduler:59 - Job 7 finished: first at GaussianMixture.scala:171, took 0.026753 s -2016-06-13 17:54:23 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:23 INFO DAGScheduler:59 - Got job 8 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:54:23 INFO DAGScheduler:59 - Final stage: ResultStage 8(first at GaussianMixture.scala:171) -2016-06-13 17:54:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:23 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=10951, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=12943, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.6 MB) -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:62418 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:54:23 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:23 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 8 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:54:23 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 3 tasks -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 25, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 26, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 8.0 (TID 27, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:23 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 25) -2016-06-13 17:54:23 INFO Executor:59 - Running task 2.0 in stage 8.0 (TID 27) -2016-06-13 17:54:23 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 26) -2016-06-13 17:54:23 INFO CacheManager:59 - Partition rdd_3_1 not found, computing it -2016-06-13 17:54:23 INFO CacheManager:59 - Partition rdd_3_3 not found, computing it -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=14195, maxMem=766075207 -2016-06-13 17:54:23 INFO CacheManager:59 - Partition rdd_3_2 not found, computing it -2016-06-13 17:54:23 INFO MemoryStore:59 - Block rdd_3_1 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=14291, maxMem=766075207 -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Added rdd_3_1 in memory on localhost:62418 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:54:23 INFO MemoryStore:59 - Block rdd_3_3 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=14387, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block rdd_3_2 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Added rdd_3_3 in memory on localhost:62418 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:54:23 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 25). 1696 bytes result sent to driver -2016-06-13 17:54:23 INFO Executor:59 - Finished task 2.0 in stage 8.0 (TID 27). 1696 bytes result sent to driver -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Added rdd_3_2 in memory on localhost:62418 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 25) in 16 ms on localhost (1/3) -2016-06-13 17:54:23 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 26). 1696 bytes result sent to driver -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 8.0 (TID 27) in 15 ms on localhost (2/3) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 26) in 18 ms on localhost (3/3) -2016-06-13 17:54:23 INFO DAGScheduler:59 - ResultStage 8 (first at GaussianMixture.scala:171) finished in 0.019 s -2016-06-13 17:54:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 17:54:23 INFO DAGScheduler:59 - Job 8 finished: first at GaussianMixture.scala:171, took 0.036603 s -2016-06-13 17:54:23 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:23 INFO DAGScheduler:59 - Got job 9 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:23 INFO DAGScheduler:59 - Final stage: ResultStage 9(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:23 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=14483, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=16315, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.6 MB) -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:62418 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:54:23 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 9 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:54:23 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 4 tasks -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 28, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 29, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 9.0 (TID 30, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 9.0 (TID 31, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:23 INFO Executor:59 - Running task 3.0 in stage 9.0 (TID 31) -2016-06-13 17:54:23 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 28) -2016-06-13 17:54:23 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 29) -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:54:23 INFO Executor:59 - Running task 2.0 in stage 9.0 (TID 30) -2016-06-13 17:54:23 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 28). 2082 bytes result sent to driver -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:54:23 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 29). 2082 bytes result sent to driver -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 28) in 15 ms on localhost (1/4) -2016-06-13 17:54:23 INFO Executor:59 - Finished task 3.0 in stage 9.0 (TID 31). 2082 bytes result sent to driver -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 29) in 16 ms on localhost (2/4) -2016-06-13 17:54:23 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 9.0 (TID 31) in 15 ms on localhost (3/4) -2016-06-13 17:54:23 INFO Executor:59 - Finished task 2.0 in stage 9.0 (TID 30). 2082 bytes result sent to driver -2016-06-13 17:54:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 9.0 (TID 30) in 20 ms on localhost (4/4) -2016-06-13 17:54:23 INFO DAGScheduler:59 - ResultStage 9 (takeSample at GaussianMixture.scala:184) finished in 0.024 s -2016-06-13 17:54:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 17:54:23 INFO DAGScheduler:59 - Job 9 finished: takeSample at GaussianMixture.scala:184, took 0.044968 s -2016-06-13 17:54:23 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:23 INFO DAGScheduler:59 - Got job 10 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:23 INFO DAGScheduler:59 - Final stage: ResultStage 10(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:23 INFO DAGScheduler:59 - Submitting ResultStage 10 (PartitionwiseSampledRDD[4] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=17510, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 18.8 KB, free 730.6 MB) -2016-06-13 17:54:23 INFO MemoryStore:59 - ensureFreeSpace(7879) called with curMem=36734, maxMem=766075207 -2016-06-13 17:54:23 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:54:23 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:62418 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:54:23 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 10 (PartitionwiseSampledRDD[4] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 4 tasks -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 32, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 33, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 10.0 (TID 34, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 10.0 (TID 35, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:54:24 INFO Executor:59 - Running task 2.0 in stage 10.0 (TID 34) -2016-06-13 17:54:24 INFO Executor:59 - Running task 3.0 in stage 10.0 (TID 35) -2016-06-13 17:54:24 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 32) -2016-06-13 17:54:24 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 33) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 2.0 in stage 10.0 (TID 34). 2244 bytes result sent to driver -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 33). 2274 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 10.0 (TID 34) in 15 ms on localhost (1/4) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 32). 2048 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 33) in 18 ms on localhost (2/4) -2016-06-13 17:54:24 INFO Executor:59 - Finished task 3.0 in stage 10.0 (TID 35). 2274 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 32) in 22 ms on localhost (3/4) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 10.0 (TID 35) in 19 ms on localhost (4/4) -2016-06-13 17:54:24 INFO DAGScheduler:59 - ResultStage 10 (takeSample at GaussianMixture.scala:184) finished in 0.023 s -2016-06-13 17:54:24 INFO DAGScheduler:59 - Job 10 finished: takeSample at GaussianMixture.scala:184, took 0.042598 s -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=44613, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(850) called with curMem=45077, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 850.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:62418 (size: 850.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 14 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Got job 11 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:24 INFO DAGScheduler:59 - Final stage: ResultStage 11(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=45927, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(2147) called with curMem=49847, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 11 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 4 tasks -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 36, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 37, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 11.0 (TID 38, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 11.0 (TID 39, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 36) -2016-06-13 17:54:24 INFO Executor:59 - Running task 2.0 in stage 11.0 (TID 38) -2016-06-13 17:54:24 INFO Executor:59 - Running task 3.0 in stage 11.0 (TID 39) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 2.0 in stage 11.0 (TID 38). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 3.0 in stage 11.0 (TID 39). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 11.0 (TID 38) in 14 ms on localhost (1/4) -2016-06-13 17:54:24 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 37) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 11.0 (TID 39) in 14 ms on localhost (2/4) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 36). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 37). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 36) in 29 ms on localhost (3/4) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 37) in 29 ms on localhost (4/4) -2016-06-13 17:54:24 INFO DAGScheduler:59 - ResultStage 11 (aggregate at GaussianMixture.scala:201) finished in 0.031 s -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 17:54:24 INFO DAGScheduler:59 - Job 11 finished: aggregate at GaussianMixture.scala:201, took 0.048135 s -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=51994, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=52458, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:62418 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 16 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Got job 12 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:24 INFO DAGScheduler:59 - Final stage: ResultStage 12(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=53311, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=57231, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 12 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 4 tasks -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 40, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 41, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 12.0 (TID 42, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 12.0 (TID 43, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO Executor:59 - Running task 3.0 in stage 12.0 (TID 43) -2016-06-13 17:54:24 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 41) -2016-06-13 17:54:24 INFO Executor:59 - Running task 2.0 in stage 12.0 (TID 42) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 41). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 2.0 in stage 12.0 (TID 42). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 41) in 20 ms on localhost (1/4) -2016-06-13 17:54:24 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 40) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 12.0 (TID 42) in 19 ms on localhost (2/4) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 3.0 in stage 12.0 (TID 43). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 12.0 (TID 43) in 23 ms on localhost (3/4) -2016-06-13 17:54:24 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 40). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO DAGScheduler:59 - ResultStage 12 (aggregate at GaussianMixture.scala:201) finished in 0.031 s -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 40) in 31 ms on localhost (4/4) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 17:54:24 INFO DAGScheduler:59 - Job 12 finished: aggregate at GaussianMixture.scala:201, took 0.045420 s -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=59383, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=59847, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:62418 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 18 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Got job 13 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:24 INFO DAGScheduler:59 - Final stage: ResultStage 13(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=60700, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=64620, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 13 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 4 tasks -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 44, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 45, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 13.0 (TID 46, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 13.0 (TID 47, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 45) -2016-06-13 17:54:24 INFO Executor:59 - Running task 2.0 in stage 13.0 (TID 46) -2016-06-13 17:54:24 INFO Executor:59 - Running task 3.0 in stage 13.0 (TID 47) -2016-06-13 17:54:24 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 44) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_3_2 locally -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_3_3 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 2.0 in stage 13.0 (TID 46). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO Executor:59 - Finished task 3.0 in stage 13.0 (TID 47). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 13.0 (TID 46) in 12 ms on localhost (1/4) -2016-06-13 17:54:24 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 45). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 44). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 13.0 (TID 47) in 14 ms on localhost (2/4) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 45) in 17 ms on localhost (3/4) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 44) in 21 ms on localhost (4/4) -2016-06-13 17:54:24 INFO DAGScheduler:59 - ResultStage 13 (aggregate at GaussianMixture.scala:201) finished in 0.021 s -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 17:54:24 INFO DAGScheduler:59 - Job 13 finished: aggregate at GaussianMixture.scala:201, took 0.038129 s -2016-06-13 17:54:24 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Got job 14 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:54:24 INFO DAGScheduler:59 - Final stage: ResultStage 14(first at GaussianMixture.scala:171) -2016-06-13 17:54:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=66772, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(1251) called with curMem=68764, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 1251.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:62418 (size: 1251.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 14 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 1 tasks -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 48, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:24 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 48) -2016-06-13 17:54:24 INFO CacheManager:59 - Partition rdd_5_0 not found, computing it -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=70015, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block rdd_5_0 stored as values in memory (estimated size 16.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added rdd_5_0 in memory on localhost:62418 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 48). 1533 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 48) in 8 ms on localhost (1/1) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 17:54:24 INFO DAGScheduler:59 - ResultStage 14 (first at GaussianMixture.scala:171) finished in 0.009 s -2016-06-13 17:54:24 INFO DAGScheduler:59 - Job 14 finished: first at GaussianMixture.scala:171, took 0.024428 s -2016-06-13 17:54:24 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Got job 15 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:54:24 INFO DAGScheduler:59 - Final stage: ResultStage 15(first at GaussianMixture.scala:171) -2016-06-13 17:54:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=70031, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(1251) called with curMem=72023, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1251.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:62418 (size: 1251.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 15 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 3 tasks -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 49, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 50, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 15.0 (TID 51, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 49) -2016-06-13 17:54:24 INFO Executor:59 - Running task 2.0 in stage 15.0 (TID 51) -2016-06-13 17:54:24 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 50) -2016-06-13 17:54:24 INFO CacheManager:59 - Partition rdd_5_2 not found, computing it -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=73274, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block rdd_5_2 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added rdd_5_2 in memory on localhost:62418 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 50). 1696 bytes result sent to driver -2016-06-13 17:54:24 INFO CacheManager:59 - Partition rdd_5_3 not found, computing it -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 50) in 13 ms on localhost (1/3) -2016-06-13 17:54:24 INFO CacheManager:59 - Partition rdd_5_1 not found, computing it -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=73370, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block rdd_5_3 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=73466, maxMem=766075207 -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added rdd_5_3 in memory on localhost:62418 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - Block rdd_5_1 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO Executor:59 - Finished task 2.0 in stage 15.0 (TID 51). 1696 bytes result sent to driver -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added rdd_5_1 in memory on localhost:62418 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 49). 1696 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 15.0 (TID 51) in 19 ms on localhost (2/3) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 49) in 21 ms on localhost (3/3) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 17:54:24 INFO DAGScheduler:59 - ResultStage 15 (first at GaussianMixture.scala:171) finished in 0.022 s -2016-06-13 17:54:24 INFO DAGScheduler:59 - Job 15 finished: first at GaussianMixture.scala:171, took 0.038849 s -2016-06-13 17:54:24 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Got job 16 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:24 INFO DAGScheduler:59 - Final stage: ResultStage 16(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=73562, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 1832.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(1194) called with curMem=75394, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 1194.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:62418 (size: 1194.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 16 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 4 tasks -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 52, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 53, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 16.0 (TID 54, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 16.0 (TID 55, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO Executor:59 - Running task 2.0 in stage 16.0 (TID 54) -2016-06-13 17:54:24 INFO Executor:59 - Running task 3.0 in stage 16.0 (TID 55) -2016-06-13 17:54:24 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 52) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 2.0 in stage 16.0 (TID 54). 2082 bytes result sent to driver -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 52). 2082 bytes result sent to driver -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 16.0 (TID 54) in 12 ms on localhost (1/4) -2016-06-13 17:54:24 INFO Executor:59 - Finished task 3.0 in stage 16.0 (TID 55). 2082 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 52) in 16 ms on localhost (2/4) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 16.0 (TID 55) in 14 ms on localhost (3/4) -2016-06-13 17:54:24 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 53) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 53). 2082 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 53) in 25 ms on localhost (4/4) -2016-06-13 17:54:24 INFO DAGScheduler:59 - ResultStage 16 (takeSample at GaussianMixture.scala:184) finished in 0.026 s -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 17:54:24 INFO DAGScheduler:59 - Job 16 finished: takeSample at GaussianMixture.scala:184, took 0.042291 s -2016-06-13 17:54:24 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Got job 17 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:24 INFO DAGScheduler:59 - Final stage: ResultStage 17(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting ResultStage 17 (PartitionwiseSampledRDD[6] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=76588, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(7881) called with curMem=95812, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:62418 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (PartitionwiseSampledRDD[6] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 56, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 57, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 58, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 59, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:54:24 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 57) -2016-06-13 17:54:24 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 59) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 57). 2269 bytes result sent to driver -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 59). 2226 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 57) in 13 ms on localhost (1/4) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 59) in 11 ms on localhost (2/4) -2016-06-13 17:54:24 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 56) -2016-06-13 17:54:24 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 58) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 56). 2048 bytes result sent to driver -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 58). 2259 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 56) in 24 ms on localhost (3/4) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 58) in 23 ms on localhost (4/4) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 17:54:24 INFO DAGScheduler:59 - ResultStage 17 (takeSample at GaussianMixture.scala:184) finished in 0.016 s -2016-06-13 17:54:24 INFO DAGScheduler:59 - Job 17 finished: takeSample at GaussianMixture.scala:184, took 0.045034 s -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=103693, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(850) called with curMem=104157, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 850.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:62418 (size: 850.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 24 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Got job 18 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:24 INFO DAGScheduler:59 - Final stage: ResultStage 18(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=105007, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=108927, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:62418 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 60, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 61, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 62, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 63, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 63) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 63). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 62) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 62). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 63) in 19 ms on localhost (1/4) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 62) in 24 ms on localhost (2/4) -2016-06-13 17:54:24 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:62418 in memory (size: 1194.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 60) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 60). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 60) in 46 ms on localhost (3/4) -2016-06-13 17:54:24 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 61) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 61). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 61) in 57 ms on localhost (4/4) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 17:54:24 INFO DAGScheduler:59 - ResultStage 18 (aggregate at GaussianMixture.scala:201) finished in 0.060 s -2016-06-13 17:54:24 INFO DAGScheduler:59 - Job 18 finished: aggregate at GaussianMixture.scala:201, took 0.115318 s -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=80941, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=81405, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:62418 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 26 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Got job 19 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:24 INFO DAGScheduler:59 - Final stage: ResultStage 19(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=82258, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(2151) called with curMem=86178, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 19 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 4 tasks -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 64, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 65, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 19.0 (TID 66, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 19.0 (TID 67, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO Executor:59 - Running task 3.0 in stage 19.0 (TID 67) -2016-06-13 17:54:24 INFO Executor:59 - Running task 2.0 in stage 19.0 (TID 66) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 3.0 in stage 19.0 (TID 67). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO Executor:59 - Finished task 2.0 in stage 19.0 (TID 66). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 19.0 (TID 67) in 11 ms on localhost (1/4) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 19.0 (TID 66) in 13 ms on localhost (2/4) -2016-06-13 17:54:24 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 64) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 64). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 64) in 25 ms on localhost (3/4) -2016-06-13 17:54:24 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 65) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 65). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 65) in 34 ms on localhost (4/4) -2016-06-13 17:54:24 INFO DAGScheduler:59 - ResultStage 19 (aggregate at GaussianMixture.scala:201) finished in 0.036 s -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 17:54:24 INFO DAGScheduler:59 - Job 19 finished: aggregate at GaussianMixture.scala:201, took 0.055198 s -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=88329, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=88793, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:62418 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 28 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:24 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 17:54:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Got job 20 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:24 INFO DAGScheduler:59 - Final stage: ResultStage 20(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:62418 in memory (size: 1251.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=86403, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(2151) called with curMem=90323, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[5] at map at GaussianMixture.scala:168) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 68, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 69, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 70, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 71, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 70) -2016-06-13 17:54:24 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 71) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_5_2 locally -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_5_3 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 70). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 71). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 70) in 10 ms on localhost (1/4) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:62418 in memory (size: 1251.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 71) in 10 ms on localhost (2/4) -2016-06-13 17:54:24 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 68) -2016-06-13 17:54:24 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 69) -2016-06-13 17:54:24 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_5_0 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 68). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 68) in 21 ms on localhost (3/4) -2016-06-13 17:54:24 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_5_1 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 69). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:62418 in memory (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 69) in 26 ms on localhost (4/4) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 17:54:24 INFO DAGScheduler:59 - ResultStage 20 (aggregate at GaussianMixture.scala:201) finished in 0.029 s -2016-06-13 17:54:24 INFO DAGScheduler:59 - Job 20 finished: aggregate at GaussianMixture.scala:201, took 0.044770 s -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:24 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:62418 in memory (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Got job 21 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:54:24 INFO DAGScheduler:59 - Final stage: ResultStage 21(first at GaussianMixture.scala:171) -2016-06-13 17:54:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:24 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=68386, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:62418 in memory (size: 850.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=69528, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:62418 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 21 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 1 tasks -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 72, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:62418 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:54:24 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 72) -2016-06-13 17:54:24 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 17:54:24 INFO CacheManager:59 - Partition rdd_7_0 not found, computing it -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=40186, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block rdd_7_0 stored as values in memory (estimated size 16.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:62418 in memory (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added rdd_7_0 in memory on localhost:62418 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 72). 1533 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 72) in 13 ms on localhost (1/1) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:62418 in memory (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO DAGScheduler:59 - ResultStage 21 (first at GaussianMixture.scala:171) finished in 0.014 s -2016-06-13 17:54:24 INFO DAGScheduler:59 - Job 21 finished: first at GaussianMixture.scala:171, took 0.032515 s -2016-06-13 17:54:24 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:62418 in memory (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 17:54:24 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Got job 22 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:54:24 INFO DAGScheduler:59 - Final stage: ResultStage 22(first at GaussianMixture.scala:171) -2016-06-13 17:54:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:24 INFO BlockManager:59 - Removing RDD 3 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=33698, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=35402, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.6 MB) -2016-06-13 17:54:24 INFO ContextCleaner:59 - Cleaned RDD 3 -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:62418 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 22 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 3 tasks -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:24 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 73, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 74, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 75, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 73) -2016-06-13 17:54:24 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 74) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:62418 in memory (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 75) -2016-06-13 17:54:24 INFO CacheManager:59 - Partition rdd_7_2 not found, computing it -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=29267, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block rdd_7_2 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:54:24 INFO BlockManager:59 - Removing RDD 1 -2016-06-13 17:54:24 INFO CacheManager:59 - Partition rdd_7_1 not found, computing it -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=29267, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block rdd_7_1 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:54:24 INFO CacheManager:59 - Partition rdd_7_3 not found, computing it -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added rdd_7_2 in memory on localhost:62418 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=29155, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block rdd_7_3 stored as values in memory (estimated size 96.0 B, free 730.6 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added rdd_7_1 in memory on localhost:62418 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 74). 1696 bytes result sent to driver -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added rdd_7_3 in memory on localhost:62418 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 73). 1696 bytes result sent to driver -2016-06-13 17:54:24 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 75). 1696 bytes result sent to driver -2016-06-13 17:54:24 INFO ContextCleaner:59 - Cleaned RDD 1 -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 74) in 23 ms on localhost (1/3) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 73) in 25 ms on localhost (2/3) -2016-06-13 17:54:24 INFO DAGScheduler:59 - ResultStage 22 (first at GaussianMixture.scala:171) finished in 0.026 s -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 75) in 24 ms on localhost (3/3) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 17:54:24 INFO DAGScheduler:59 - Job 22 finished: first at GaussianMixture.scala:171, took 0.079279 s -2016-06-13 17:54:24 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Got job 23 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:24 INFO DAGScheduler:59 - Final stage: ResultStage 23(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=29251, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=31083, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.6 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:62418 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 23 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 76, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 77, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 78, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 79, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 77) -2016-06-13 17:54:24 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 78) -2016-06-13 17:54:24 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 76) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 77). 2082 bytes result sent to driver -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 76). 2082 bytes result sent to driver -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 78). 2082 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 77) in 9 ms on localhost (1/4) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 76) in 11 ms on localhost (2/4) -2016-06-13 17:54:24 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 79) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 78) in 12 ms on localhost (3/4) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 79). 2082 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 79) in 15 ms on localhost (4/4) -2016-06-13 17:54:24 INFO DAGScheduler:59 - ResultStage 23 (takeSample at GaussianMixture.scala:184) finished in 0.018 s -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 17:54:24 INFO DAGScheduler:59 - Job 23 finished: takeSample at GaussianMixture.scala:184, took 0.030795 s -2016-06-13 17:54:24 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Got job 24 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:24 INFO DAGScheduler:59 - Final stage: ResultStage 24(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting ResultStage 24 (PartitionwiseSampledRDD[8] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=32278, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(7887) called with curMem=51502, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:62418 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 24 (PartitionwiseSampledRDD[8] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 4 tasks -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 80, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 81, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 24.0 (TID 82, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 24.0 (TID 83, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:54:24 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 81) -2016-06-13 17:54:24 INFO Executor:59 - Running task 2.0 in stage 24.0 (TID 82) -2016-06-13 17:54:24 INFO Executor:59 - Running task 3.0 in stage 24.0 (TID 83) -2016-06-13 17:54:24 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 80) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 81). 2269 bytes result sent to driver -2016-06-13 17:54:24 INFO Executor:59 - Finished task 2.0 in stage 24.0 (TID 82). 2221 bytes result sent to driver -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 24.0 (TID 82) in 8 ms on localhost (1/4) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 81) in 10 ms on localhost (2/4) -2016-06-13 17:54:24 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 80). 2048 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 80) in 15 ms on localhost (3/4) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 3.0 in stage 24.0 (TID 83). 2269 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 24.0 (TID 83) in 17 ms on localhost (4/4) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-06-13 17:54:24 INFO DAGScheduler:59 - ResultStage 24 (takeSample at GaussianMixture.scala:184) finished in 0.020 s -2016-06-13 17:54:24 INFO DAGScheduler:59 - Job 24 finished: takeSample at GaussianMixture.scala:184, took 0.037586 s -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=59389, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(851) called with curMem=59853, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 851.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:62418 (size: 851.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 34 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Got job 25 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:24 INFO DAGScheduler:59 - Final stage: ResultStage 25(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=60704, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=64624, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 25 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 4 tasks -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 84, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 85, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 25.0 (TID 86, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 25.0 (TID 87, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 85) -2016-06-13 17:54:24 INFO Executor:59 - Running task 3.0 in stage 25.0 (TID 87) -2016-06-13 17:54:24 INFO Executor:59 - Running task 2.0 in stage 25.0 (TID 86) -2016-06-13 17:54:24 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 84) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 3.0 in stage 25.0 (TID 87). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 84). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 85). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 25.0 (TID 87) in 8 ms on localhost (1/4) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 84) in 10 ms on localhost (2/4) -2016-06-13 17:54:24 INFO Executor:59 - Finished task 2.0 in stage 25.0 (TID 86). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 85) in 11 ms on localhost (3/4) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 25.0 (TID 86) in 12 ms on localhost (4/4) -2016-06-13 17:54:24 INFO DAGScheduler:59 - ResultStage 25 (aggregate at GaussianMixture.scala:201) finished in 0.014 s -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-06-13 17:54:24 INFO DAGScheduler:59 - Job 25 finished: aggregate at GaussianMixture.scala:201, took 0.023966 s -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=66769, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=67233, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:62418 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 36 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:24 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Got job 26 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:24 INFO DAGScheduler:59 - Final stage: ResultStage 26(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=68086, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:54:24 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=72006, maxMem=766075207 -2016-06-13 17:54:24 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:54:24 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:24 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 26 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:54:24 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 4 tasks -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 88, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 89, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 90, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 26.0 (TID 91, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:24 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 89) -2016-06-13 17:54:24 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 88) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 88). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 90) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 89). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO Executor:59 - Running task 3.0 in stage 26.0 (TID 91) -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 90). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:54:24 INFO Executor:59 - Finished task 3.0 in stage 26.0 (TID 91). 2708 bytes result sent to driver -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 88) in 15 ms on localhost (1/4) -2016-06-13 17:54:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 89) in 15 ms on localhost (2/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 26.0 (TID 91) in 15 ms on localhost (3/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 90) in 17 ms on localhost (4/4) -2016-06-13 17:54:25 INFO DAGScheduler:59 - ResultStage 26 (aggregate at GaussianMixture.scala:201) finished in 0.018 s -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-06-13 17:54:25 INFO DAGScheduler:59 - Job 26 finished: aggregate at GaussianMixture.scala:201, took 0.030250 s -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=74158, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=74622, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:62418 (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 38 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:25 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Got job 27 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:25 INFO DAGScheduler:59 - Final stage: ResultStage 27(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=75475, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=79395, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 27 (MapPartitionsRDD[7] at map at GaussianMixture.scala:168) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 4 tasks -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 92, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 93, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 27.0 (TID 94, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 27.0 (TID 95, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:25 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 92) -2016-06-13 17:54:25 INFO Executor:59 - Running task 2.0 in stage 27.0 (TID 94) -2016-06-13 17:54:25 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 93) -2016-06-13 17:54:25 INFO Executor:59 - Running task 3.0 in stage 27.0 (TID 95) -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_7_0 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 92). 2708 bytes result sent to driver -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_7_3 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 3.0 in stage 27.0 (TID 95). 2708 bytes result sent to driver -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_7_1 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 93). 2708 bytes result sent to driver -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 92) in 12 ms on localhost (1/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 27.0 (TID 95) in 11 ms on localhost (2/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 93) in 13 ms on localhost (3/4) -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_7_2 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 2.0 in stage 27.0 (TID 94). 2708 bytes result sent to driver -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 27.0 (TID 94) in 18 ms on localhost (4/4) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-06-13 17:54:25 INFO DAGScheduler:59 - ResultStage 27 (aggregate at GaussianMixture.scala:201) finished in 0.021 s -2016-06-13 17:54:25 INFO DAGScheduler:59 - Job 27 finished: aggregate at GaussianMixture.scala:201, took 0.035531 s -2016-06-13 17:54:25 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Got job 28 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:54:25 INFO DAGScheduler:59 - Final stage: ResultStage 28(first at GaussianMixture.scala:171) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=81547, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=83539, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.5 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:62418 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 28 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 1 tasks -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 96, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:25 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 96) -2016-06-13 17:54:25 INFO CacheManager:59 - Partition rdd_9_0 not found, computing it -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=84791, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block rdd_9_0 stored as values in memory (estimated size 16.0 B, free 730.5 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added rdd_9_0 in memory on localhost:62418 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:54:25 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 96). 1533 bytes result sent to driver -2016-06-13 17:54:25 INFO DAGScheduler:59 - ResultStage 28 (first at GaussianMixture.scala:171) finished in 0.008 s -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 96) in 8 ms on localhost (1/1) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Job 28 finished: first at GaussianMixture.scala:171, took 0.015705 s -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-06-13 17:54:25 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Got job 29 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:54:25 INFO DAGScheduler:59 - Final stage: ResultStage 29(first at GaussianMixture.scala:171) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=84807, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(1252) called with curMem=86799, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 1252.0 B, free 730.5 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:62418 (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 29 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 3 tasks -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 97, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 98, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 29.0 (TID 99, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:25 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 98) -2016-06-13 17:54:25 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 97) -2016-06-13 17:54:25 INFO Executor:59 - Running task 2.0 in stage 29.0 (TID 99) -2016-06-13 17:54:25 INFO CacheManager:59 - Partition rdd_9_2 not found, computing it -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=88051, maxMem=766075207 -2016-06-13 17:54:25 INFO CacheManager:59 - Partition rdd_9_1 not found, computing it -2016-06-13 17:54:25 INFO MemoryStore:59 - Block rdd_9_2 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=88147, maxMem=766075207 -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added rdd_9_2 in memory on localhost:62418 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - Block rdd_9_1 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:54:25 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 98). 1696 bytes result sent to driver -2016-06-13 17:54:25 INFO CacheManager:59 - Partition rdd_9_3 not found, computing it -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added rdd_9_1 in memory on localhost:62418 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(96) called with curMem=88243, maxMem=766075207 -2016-06-13 17:54:25 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 97). 1696 bytes result sent to driver -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 98) in 11 ms on localhost (1/3) -2016-06-13 17:54:25 INFO MemoryStore:59 - Block rdd_9_3 stored as values in memory (estimated size 96.0 B, free 730.5 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added rdd_9_3 in memory on localhost:62418 (size: 96.0 B, free: 730.6 MB) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 97) in 14 ms on localhost (2/3) -2016-06-13 17:54:25 INFO Executor:59 - Finished task 2.0 in stage 29.0 (TID 99). 1696 bytes result sent to driver -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 29.0 (TID 99) in 13 ms on localhost (3/3) -2016-06-13 17:54:25 INFO DAGScheduler:59 - ResultStage 29 (first at GaussianMixture.scala:171) finished in 0.015 s -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 17:54:25 INFO DAGScheduler:59 - Job 29 finished: first at GaussianMixture.scala:171, took 0.024844 s -2016-06-13 17:54:25 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Got job 30 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:25 INFO DAGScheduler:59 - Final stage: ResultStage 30(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=88339, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 1832.0 B, free 730.5 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=90171, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.5 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:62418 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 30 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 4 tasks -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 100, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 101, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 102, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 30.0 (TID 103, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:25 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 100) -2016-06-13 17:54:25 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 101) -2016-06-13 17:54:25 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 102) -2016-06-13 17:54:25 INFO Executor:59 - Running task 3.0 in stage 30.0 (TID 103) -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 100). 2082 bytes result sent to driver -2016-06-13 17:54:25 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 101). 2082 bytes result sent to driver -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 102). 2082 bytes result sent to driver -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 3.0 in stage 30.0 (TID 103). 2082 bytes result sent to driver -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 100) in 7 ms on localhost (1/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 102) in 9 ms on localhost (2/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 101) in 10 ms on localhost (3/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 30.0 (TID 103) in 10 ms on localhost (4/4) -2016-06-13 17:54:25 INFO DAGScheduler:59 - ResultStage 30 (takeSample at GaussianMixture.scala:184) finished in 0.011 s -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 17:54:25 INFO DAGScheduler:59 - Job 30 finished: takeSample at GaussianMixture.scala:184, took 0.020380 s -2016-06-13 17:54:25 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Got job 31 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:25 INFO DAGScheduler:59 - Final stage: ResultStage 31(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting ResultStage 31 (PartitionwiseSampledRDD[10] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=91366, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(7881) called with curMem=110590, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:62418 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 31 (PartitionwiseSampledRDD[10] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 4 tasks -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 104, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 105, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 31.0 (TID 106, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 31.0 (TID 107, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 17:54:25 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 104) -2016-06-13 17:54:25 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 105) -2016-06-13 17:54:25 INFO Executor:59 - Running task 2.0 in stage 31.0 (TID 106) -2016-06-13 17:54:25 INFO Executor:59 - Running task 3.0 in stage 31.0 (TID 107) -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 104). 2048 bytes result sent to driver -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 3.0 in stage 31.0 (TID 107). 2274 bytes result sent to driver -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 104) in 9 ms on localhost (1/4) -2016-06-13 17:54:25 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 105). 2269 bytes result sent to driver -2016-06-13 17:54:25 INFO Executor:59 - Finished task 2.0 in stage 31.0 (TID 106). 2249 bytes result sent to driver -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 31.0 (TID 107) in 8 ms on localhost (2/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 105) in 10 ms on localhost (3/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 31.0 (TID 106) in 9 ms on localhost (4/4) -2016-06-13 17:54:25 INFO DAGScheduler:59 - ResultStage 31 (takeSample at GaussianMixture.scala:184) finished in 0.011 s -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 17:54:25 INFO DAGScheduler:59 - Job 31 finished: takeSample at GaussianMixture.scala:184, took 0.022141 s -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=118471, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(849) called with curMem=118935, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 849.0 B, free 730.5 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:62418 (size: 849.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 44 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:25 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Got job 32 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:25 INFO DAGScheduler:59 - Final stage: ResultStage 32(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=119784, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(2145) called with curMem=123704, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 32 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 4 tasks -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 108, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 109, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 32.0 (TID 110, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 32.0 (TID 111, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:25 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 108) -2016-06-13 17:54:25 INFO Executor:59 - Running task 3.0 in stage 32.0 (TID 111) -2016-06-13 17:54:25 INFO Executor:59 - Running task 2.0 in stage 32.0 (TID 110) -2016-06-13 17:54:25 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 109) -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 108). 2708 bytes result sent to driver -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 109). 2708 bytes result sent to driver -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 3.0 in stage 32.0 (TID 111). 2708 bytes result sent to driver -2016-06-13 17:54:25 INFO Executor:59 - Finished task 2.0 in stage 32.0 (TID 110). 2708 bytes result sent to driver -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 108) in 7 ms on localhost (1/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 109) in 7 ms on localhost (2/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 32.0 (TID 111) in 6 ms on localhost (3/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 32.0 (TID 110) in 8 ms on localhost (4/4) -2016-06-13 17:54:25 INFO DAGScheduler:59 - ResultStage 32 (aggregate at GaussianMixture.scala:201) finished in 0.009 s -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 17:54:25 INFO DAGScheduler:59 - Job 32 finished: aggregate at GaussianMixture.scala:201, took 0.017669 s -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=125849, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=126313, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:62418 (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 46 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:25 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Got job 33 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:25 INFO DAGScheduler:59 - Final stage: ResultStage 33(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=127166, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=131086, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 33 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 4 tasks -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 112, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 113, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 33.0 (TID 114, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 33.0 (TID 115, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:25 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 113) -2016-06-13 17:54:25 INFO Executor:59 - Running task 3.0 in stage 33.0 (TID 115) -2016-06-13 17:54:25 INFO Executor:59 - Running task 2.0 in stage 33.0 (TID 114) -2016-06-13 17:54:25 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 112) -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 113). 2708 bytes result sent to driver -2016-06-13 17:54:25 INFO Executor:59 - Finished task 2.0 in stage 33.0 (TID 114). 2708 bytes result sent to driver -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 112). 2708 bytes result sent to driver -2016-06-13 17:54:25 INFO Executor:59 - Finished task 3.0 in stage 33.0 (TID 115). 2708 bytes result sent to driver -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 113) in 6 ms on localhost (1/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 33.0 (TID 115) in 7 ms on localhost (2/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 33.0 (TID 114) in 7 ms on localhost (3/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 112) in 9 ms on localhost (4/4) -2016-06-13 17:54:25 INFO DAGScheduler:59 - ResultStage 33 (aggregate at GaussianMixture.scala:201) finished in 0.009 s -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 17:54:25 INFO DAGScheduler:59 - Job 33 finished: aggregate at GaussianMixture.scala:201, took 0.018668 s -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(464) called with curMem=133238, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 464.0 B, free 730.5 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(853) called with curMem=133702, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 853.0 B, free 730.5 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:62418 (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 48 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:25 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Got job 34 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:25 INFO DAGScheduler:59 - Final stage: ResultStage 34(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=134555, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=138475, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 34 (MapPartitionsRDD[9] at map at GaussianMixture.scala:168) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 4 tasks -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 116, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 117, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 118, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 34.0 (TID 119, localhost, PROCESS_LOCAL, 2218 bytes) -2016-06-13 17:54:25 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 116) -2016-06-13 17:54:25 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 118) -2016-06-13 17:54:25 INFO Executor:59 - Running task 3.0 in stage 34.0 (TID 119) -2016-06-13 17:54:25 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 117) -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_9_2 locally -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_9_3 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 118). 2708 bytes result sent to driver -2016-06-13 17:54:25 INFO Executor:59 - Finished task 3.0 in stage 34.0 (TID 119). 2708 bytes result sent to driver -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_9_0 locally -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_9_1 locally -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 118) in 7 ms on localhost (1/4) -2016-06-13 17:54:25 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 116). 2708 bytes result sent to driver -2016-06-13 17:54:25 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 117). 2708 bytes result sent to driver -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 116) in 11 ms on localhost (2/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 34.0 (TID 119) in 9 ms on localhost (3/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 117) in 11 ms on localhost (4/4) -2016-06-13 17:54:25 INFO DAGScheduler:59 - ResultStage 34 (aggregate at GaussianMixture.scala:201) finished in 0.012 s -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-06-13 17:54:25 INFO DAGScheduler:59 - Job 34 finished: aggregate at GaussianMixture.scala:201, took 0.022195 s -2016-06-13 17:54:25 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Got job 35 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:54:25 INFO DAGScheduler:59 - Final stage: ResultStage 35(first at GaussianMixture.scala:171) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=140627, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=142619, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.4 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:62418 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 35 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 1 tasks -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 120, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:54:25 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 120) -2016-06-13 17:54:25 INFO CacheManager:59 - Partition rdd_12_0 not found, computing it -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(224) called with curMem=143869, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block rdd_12_0 stored as values in memory (estimated size 224.0 B, free 730.4 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added rdd_12_0 in memory on localhost:62418 (size: 224.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 120). 1688 bytes result sent to driver -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 120) in 7 ms on localhost (1/1) -2016-06-13 17:54:25 INFO DAGScheduler:59 - ResultStage 35 (first at GaussianMixture.scala:171) finished in 0.007 s -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-06-13 17:54:25 INFO DAGScheduler:59 - Job 35 finished: first at GaussianMixture.scala:171, took 0.017187 s -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=144093, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 712.0 B, free 730.4 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=144805, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 880.0 B, free 730.4 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:62418 (size: 880.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 51 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:25 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Got job 36 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:25 INFO DAGScheduler:59 - Final stage: ResultStage 36(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=145685, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(2181) called with curMem=149781, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 36 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 4 tasks -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 121, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 122, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 36.0 (TID 123, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 36.0 (TID 124, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:25 INFO Executor:59 - Running task 2.0 in stage 36.0 (TID 123) -2016-06-13 17:54:25 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 122) -2016-06-13 17:54:25 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 121) -2016-06-13 17:54:25 INFO Executor:59 - Running task 3.0 in stage 36.0 (TID 124) -2016-06-13 17:54:25 INFO CacheManager:59 - Partition rdd_12_2 not found, computing it -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=151962, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block rdd_12_2 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-06-13 17:54:25 INFO CacheManager:59 - Partition rdd_12_3 not found, computing it -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added rdd_12_2 in memory on localhost:62418 (size: 288.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=152250, maxMem=766075207 -2016-06-13 17:54:25 INFO CacheManager:59 - Partition rdd_12_1 not found, computing it -2016-06-13 17:54:25 INFO MemoryStore:59 - Block rdd_12_3 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_12_0 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 2.0 in stage 36.0 (TID 123). 2247 bytes result sent to driver -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added rdd_12_3 in memory on localhost:62418 (size: 288.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=152538, maxMem=766075207 -2016-06-13 17:54:25 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 121). 2762 bytes result sent to driver -2016-06-13 17:54:25 INFO MemoryStore:59 - Block rdd_12_1 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 36.0 (TID 123) in 12 ms on localhost (1/4) -2016-06-13 17:54:25 INFO Executor:59 - Finished task 3.0 in stage 36.0 (TID 124). 2247 bytes result sent to driver -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added rdd_12_1 in memory on localhost:62418 (size: 288.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 121) in 16 ms on localhost (2/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 36.0 (TID 124) in 14 ms on localhost (3/4) -2016-06-13 17:54:25 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 122). 2247 bytes result sent to driver -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 122) in 16 ms on localhost (4/4) -2016-06-13 17:54:25 INFO DAGScheduler:59 - ResultStage 36 (aggregate at GaussianMixture.scala:201) finished in 0.019 s -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-06-13 17:54:25 INFO DAGScheduler:59 - Job 36 finished: aggregate at GaussianMixture.scala:201, took 0.031852 s -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=152826, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 712.0 B, free 730.4 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(934) called with curMem=153538, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 934.0 B, free 730.4 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:62418 (size: 934.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 53 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:25 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Got job 37 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:25 INFO DAGScheduler:59 - Final stage: ResultStage 37(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=154472, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(2241) called with curMem=158568, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:62418 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 37 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 4 tasks -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 125, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 126, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 37.0 (TID 127, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 37.0 (TID 128, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:25 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 126) -2016-06-13 17:54:25 INFO Executor:59 - Running task 2.0 in stage 37.0 (TID 127) -2016-06-13 17:54:25 INFO Executor:59 - Running task 3.0 in stage 37.0 (TID 128) -2016-06-13 17:54:25 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 125) -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_12_3 locally -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_12_0 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 3.0 in stage 37.0 (TID 128). 2762 bytes result sent to driver -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_12_2 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 125). 2762 bytes result sent to driver -2016-06-13 17:54:25 INFO Executor:59 - Finished task 2.0 in stage 37.0 (TID 127). 2762 bytes result sent to driver -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_12_1 locally -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 37.0 (TID 128) in 7 ms on localhost (1/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 125) in 10 ms on localhost (2/4) -2016-06-13 17:54:25 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 126). 2762 bytes result sent to driver -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 37.0 (TID 127) in 11 ms on localhost (3/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 126) in 12 ms on localhost (4/4) -2016-06-13 17:54:25 INFO DAGScheduler:59 - ResultStage 37 (aggregate at GaussianMixture.scala:201) finished in 0.012 s -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-06-13 17:54:25 INFO DAGScheduler:59 - Job 37 finished: aggregate at GaussianMixture.scala:201, took 0.023973 s -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=160809, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 712.0 B, free 730.4 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=161521, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.4 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:62418 (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 55 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:25 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Got job 38 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:25 INFO DAGScheduler:59 - Final stage: ResultStage 38(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=162453, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=166549, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:62418 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 38 (MapPartitionsRDD[12] at map at GaussianMixture.scala:168) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 4 tasks -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 129, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 130, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 131, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 38.0 (TID 132, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:25 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 130) -2016-06-13 17:54:25 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 131) -2016-06-13 17:54:25 INFO Executor:59 - Running task 3.0 in stage 38.0 (TID 132) -2016-06-13 17:54:25 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 129) -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_12_2 locally -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_12_1 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 131). 2762 bytes result sent to driver -2016-06-13 17:54:25 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 130). 2762 bytes result sent to driver -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_12_0 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 129). 2762 bytes result sent to driver -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_12_3 locally -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 131) in 14 ms on localhost (1/4) -2016-06-13 17:54:25 INFO Executor:59 - Finished task 3.0 in stage 38.0 (TID 132). 2762 bytes result sent to driver -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 130) in 15 ms on localhost (2/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 129) in 18 ms on localhost (3/4) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 38.0 (TID 132) in 16 ms on localhost (4/4) -2016-06-13 17:54:25 INFO DAGScheduler:59 - ResultStage 38 (aggregate at GaussianMixture.scala:201) finished in 0.019 s -2016-06-13 17:54:25 INFO DAGScheduler:59 - Job 38 finished: aggregate at GaussianMixture.scala:201, took 0.027923 s -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-06-13 17:54:25 INFO SparkContext:59 - Starting job: first at GaussianMixtureFromSparkSuite.scala:97 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Got job 39 (first at GaussianMixtureFromSparkSuite.scala:97) with 1 output partitions -2016-06-13 17:54:25 INFO DAGScheduler:59 - Final stage: ResultStage 39(first at GaussianMixtureFromSparkSuite.scala:97) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting ResultStage 39 (ParallelCollectionRDD[13] at parallelize at GaussianMixtureFromSparkSuite.scala:94), which has no missing parents -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(1272) called with curMem=168787, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 1272.0 B, free 730.4 MB) -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(837) called with curMem=165963, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 837.0 B, free 730.4 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:62418 (size: 837.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:62418 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 39 (ParallelCollectionRDD[13] at parallelize at GaussianMixtureFromSparkSuite.scala:94) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 1 tasks -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 133, localhost, PROCESS_LOCAL, 7178 bytes) -2016-06-13 17:54:25 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 133) -2016-06-13 17:54:25 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 133). 1433 bytes result sent to driver -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:62418 in memory (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 133) in 11 ms on localhost (1/1) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-06-13 17:54:25 INFO DAGScheduler:59 - ResultStage 39 (first at GaussianMixtureFromSparkSuite.scala:97) finished in 0.014 s -2016-06-13 17:54:25 INFO DAGScheduler:59 - Job 39 finished: first at GaussianMixtureFromSparkSuite.scala:97, took 0.045222 s -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:62418 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-06-13 17:54:25 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Got job 40 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:54:25 INFO DAGScheduler:59 - Final stage: ResultStage 40(first at GaussianMixture.scala:171) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:62418 in memory (size: 934.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=150839, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 1992.0 B, free 730.4 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=150650, maxMem=766075207 -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.4 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:62418 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 40 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 1 tasks -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 134, localhost, PROCESS_LOCAL, 7178 bytes) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:62418 in memory (size: 880.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 134) -2016-06-13 17:54:25 INFO CacheManager:59 - Partition rdd_14_0 not found, computing it -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:62418 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(5536) called with curMem=147066, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block rdd_14_0 stored as values in memory (estimated size 5.4 KB, free 730.4 MB) -2016-06-13 17:54:25 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added rdd_14_0 in memory on localhost:62418 (size: 5.4 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 134). 2083 bytes result sent to driver -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 134) in 19 ms on localhost (1/1) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-06-13 17:54:25 INFO DAGScheduler:59 - ResultStage 40 (first at GaussianMixture.scala:171) finished in 0.021 s -2016-06-13 17:54:25 INFO DAGScheduler:59 - Job 40 finished: first at GaussianMixture.scala:171, took 0.033359 s -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned RDD 12 -2016-06-13 17:54:25 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Got job 41 (takeSample at GaussianMixture.scala:184) with 2 output partitions -2016-06-13 17:54:25 INFO DAGScheduler:59 - Final stage: ResultStage 41(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=145442, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 1832.0 B, free 730.4 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=146421, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 1193.0 B, free 730.4 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:62418 in memory (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:62418 (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 135, localhost, PROCESS_LOCAL, 7178 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 136, localhost, PROCESS_LOCAL, 7594 bytes) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 135) -2016-06-13 17:54:25 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 136) -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 135). 2082 bytes result sent to driver -2016-06-13 17:54:25 INFO CacheManager:59 - Partition rdd_14_1 not found, computing it -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 135) in 10 ms on localhost (1/2) -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(6000) called with curMem=140225, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block rdd_14_1 stored as values in memory (estimated size 5.9 KB, free 730.4 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:62418 in memory (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added rdd_14_1 in memory on localhost:62418 (size: 5.9 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 136). 1567 bytes result sent to driver -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 136) in 18 ms on localhost (2/2) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-06-13 17:54:25 INFO DAGScheduler:59 - ResultStage 41 (takeSample at GaussianMixture.scala:184) finished in 0.022 s -2016-06-13 17:54:25 INFO DAGScheduler:59 - Job 41 finished: takeSample at GaussianMixture.scala:184, took 0.044644 s -2016-06-13 17:54:25 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Got job 42 (takeSample at GaussianMixture.scala:184) with 2 output partitions -2016-06-13 17:54:25 INFO DAGScheduler:59 - Final stage: ResultStage 42(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting ResultStage 42 (PartitionwiseSampledRDD[15] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=138847, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 18.8 KB, free 730.4 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:62418 in memory (size: 849.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(7879) called with curMem=157607, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.4 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:62418 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (PartitionwiseSampledRDD[15] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 137, localhost, PROCESS_LOCAL, 7287 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 138, localhost, PROCESS_LOCAL, 7703 bytes) -2016-06-13 17:54:25 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 137) -2016-06-13 17:54:25 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 138) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:62418 in memory (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_14_1 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 137). 6072 bytes result sent to driver -2016-06-13 17:54:25 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 138). 6973 bytes result sent to driver -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 137) in 6 ms on localhost (1/2) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 138) in 7 ms on localhost (2/2) -2016-06-13 17:54:25 INFO DAGScheduler:59 - ResultStage 42 (takeSample at GaussianMixture.scala:184) finished in 0.007 s -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:62418 in memory (size: 1195.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Job 42 finished: takeSample at GaussianMixture.scala:184, took 0.020801 s -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:62418 in memory (size: 1252.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:62418 in memory (size: 1252.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-06-13 17:54:25 INFO BlockManager:59 - Removing RDD 9 -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned RDD 9 -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:62418 in memory (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:62418 in memory (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:62418 in memory (size: 851.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:62418 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:62418 in memory (size: 1195.0 B, free: 730.6 MB) -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:62418 in memory (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:62418 in memory (size: 1252.0 B, free: 730.6 MB) -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 17:54:25 INFO BlockManager:59 - Removing RDD 7 -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned RDD 7 -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(205504) called with curMem=63403, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 200.7 KB, free 730.3 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(12077) called with curMem=267590, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 11.8 KB, free 730.3 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:62418 in memory (size: 853.0 B, free: 730.6 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:62418 (size: 11.8 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 61 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:62418 in memory (size: 853.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Got job 43 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:54:25 INFO DAGScheduler:59 - Final stage: ResultStage 43(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:62418 in memory (size: 850.0 B, free: 730.6 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(310656) called with curMem=265364, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 303.4 KB, free 730.0 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(18290) called with curMem=575556, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 17.9 KB, free 730.0 MB) -2016-06-13 17:54:25 INFO BlockManager:59 - Removing RDD 5 -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:62418 (size: 17.9 KB, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-06-13 17:54:25 INFO ContextCleaner:59 - Cleaned RDD 5 -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 139, localhost, PROCESS_LOCAL, 7178 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 140, localhost, PROCESS_LOCAL, 7594 bytes) -2016-06-13 17:54:25 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 139) -2016-06-13 17:54:25 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 140) -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-06-13 17:54:25 INFO BlockManager:59 - Found block rdd_14_1 locally -2016-06-13 17:54:25 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 139). 105440 bytes result sent to driver -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 139) in 69 ms on localhost (1/2) -2016-06-13 17:54:25 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 140). 105440 bytes result sent to driver -2016-06-13 17:54:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 140) in 80 ms on localhost (2/2) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-06-13 17:54:25 INFO DAGScheduler:59 - ResultStage 43 (aggregate at GaussianMixture.scala:201) finished in 0.081 s -2016-06-13 17:54:25 INFO DAGScheduler:59 - Job 43 finished: aggregate at GaussianMixture.scala:201, took 0.096623 s -2016-06-13 17:54:25 INFO SparkContext:59 - Starting job: collect at GaussianMixture.scala:213 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Got job 44 (collect at GaussianMixture.scala:213) with 5 output partitions -2016-06-13 17:54:25 INFO DAGScheduler:59 - Final stage: ResultStage 44(collect at GaussianMixture.scala:213) -2016-06-13 17:54:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[17] at map at GaussianMixture.scala:211), which has no missing parents -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=593542, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 2.3 KB, free 730.0 MB) -2016-06-13 17:54:25 INFO MemoryStore:59 - ensureFreeSpace(1389) called with curMem=595854, maxMem=766075207 -2016-06-13 17:54:25 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 1389.0 B, free 730.0 MB) -2016-06-13 17:54:25 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:62418 (size: 1389.0 B, free: 730.5 MB) -2016-06-13 17:54:25 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:25 INFO DAGScheduler:59 - Submitting 5 missing tasks from ResultStage 44 (MapPartitionsRDD[17] at map at GaussianMixture.scala:211) -2016-06-13 17:54:25 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 5 tasks -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 141, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 142, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 44.0 (TID 143, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:54:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 44.0 (TID 144, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:54:25 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 141) -2016-06-13 17:54:25 INFO Executor:59 - Running task 3.0 in stage 44.0 (TID 144) -2016-06-13 17:54:25 INFO Executor:59 - Running task 2.0 in stage 44.0 (TID 143) -2016-06-13 17:54:25 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 142) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 142). 42916 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 4.0 in stage 44.0 (TID 145, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:54:26 INFO Executor:59 - Running task 4.0 in stage 44.0 (TID 145) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 142) in 194 ms on localhost (1/5) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 141). 42916 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 141) in 222 ms on localhost (2/5) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 4.0 in stage 44.0 (TID 145). 42916 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 4.0 in stage 44.0 (TID 145) in 74 ms on localhost (3/5) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 3.0 in stage 44.0 (TID 144). 42916 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 44.0 (TID 144) in 303 ms on localhost (4/5) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 2.0 in stage 44.0 (TID 143). 42916 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 44.0 (TID 143) in 308 ms on localhost (5/5) -2016-06-13 17:54:26 INFO DAGScheduler:59 - ResultStage 44 (collect at GaussianMixture.scala:213) finished in 0.310 s -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-06-13 17:54:26 INFO DAGScheduler:59 - Job 44 finished: collect at GaussianMixture.scala:213, took 0.320777 s -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(205544) called with curMem=597243, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 200.7 KB, free 729.8 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(23825) called with curMem=802787, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 23.3 KB, free 729.8 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:62418 (size: 23.3 KB, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 64 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:26 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Got job 45 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 17:54:26 INFO DAGScheduler:59 - Final stage: ResultStage 45(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(310656) called with curMem=826612, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 303.4 KB, free 729.5 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(30131) called with curMem=1137268, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 29.4 KB, free 729.5 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:62418 (size: 29.4 KB, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[14] at map at GaussianMixture.scala:168) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 146, localhost, PROCESS_LOCAL, 7178 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 147, localhost, PROCESS_LOCAL, 7594 bytes) -2016-06-13 17:54:26 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 146) -2016-06-13 17:54:26 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 147) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_14_0 locally -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_14_1 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 146). 105440 bytes result sent to driver -2016-06-13 17:54:26 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 147). 105440 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 146) in 17 ms on localhost (1/2) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 147) in 18 ms on localhost (2/2) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-06-13 17:54:26 INFO DAGScheduler:59 - ResultStage 45 (aggregate at GaussianMixture.scala:201) finished in 0.019 s -2016-06-13 17:54:26 INFO DAGScheduler:59 - Job 45 finished: aggregate at GaussianMixture.scala:201, took 0.032860 s -2016-06-13 17:54:26 INFO SparkContext:59 - Starting job: collect at GaussianMixture.scala:213 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Got job 46 (collect at GaussianMixture.scala:213) with 5 output partitions -2016-06-13 17:54:26 INFO DAGScheduler:59 - Final stage: ResultStage 46(collect at GaussianMixture.scala:213) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[19] at map at GaussianMixture.scala:211), which has no missing parents -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=1167399, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 2.3 KB, free 729.5 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(1389) called with curMem=1169711, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 1389.0 B, free 729.5 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:62418 (size: 1389.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting 5 missing tasks from ResultStage 46 (MapPartitionsRDD[19] at map at GaussianMixture.scala:211) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 5 tasks -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 148, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 149, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 46.0 (TID 150, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 46.0 (TID 151, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:54:26 INFO Executor:59 - Running task 2.0 in stage 46.0 (TID 150) -2016-06-13 17:54:26 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 149) -2016-06-13 17:54:26 INFO Executor:59 - Running task 3.0 in stage 46.0 (TID 151) -2016-06-13 17:54:26 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 148) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 3.0 in stage 46.0 (TID 151). 42916 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 4.0 in stage 46.0 (TID 152, localhost, PROCESS_LOCAL, 22925 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 46.0 (TID 151) in 16 ms on localhost (1/5) -2016-06-13 17:54:26 INFO Executor:59 - Running task 4.0 in stage 46.0 (TID 152) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 149). 42916 bytes result sent to driver -2016-06-13 17:54:26 INFO Executor:59 - Finished task 2.0 in stage 46.0 (TID 150). 42916 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 149) in 24 ms on localhost (2/5) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 46.0 (TID 150) in 24 ms on localhost (3/5) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 148). 42916 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 148) in 28 ms on localhost (4/5) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 4.0 in stage 46.0 (TID 152). 42916 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 4.0 in stage 46.0 (TID 152) in 14 ms on localhost (5/5) -2016-06-13 17:54:26 INFO DAGScheduler:59 - ResultStage 46 (collect at GaussianMixture.scala:213) finished in 0.032 s -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-06-13 17:54:26 INFO DAGScheduler:59 - Job 46 finished: collect at GaussianMixture.scala:213, took 0.040702 s -2016-06-13 17:54:26 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Got job 47 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:54:26 INFO DAGScheduler:59 - Final stage: ResultStage 47(first at GaussianMixture.scala:171) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=1171100, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 1992.0 B, free 729.5 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1173092, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 1250.0 B, free 729.5 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:62418 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 67 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 47 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 1 tasks -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 153, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:26 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 153) -2016-06-13 17:54:26 INFO CacheManager:59 - Partition rdd_21_0 not found, computing it -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=1174342, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block rdd_21_0 stored as values in memory (estimated size 16.0 B, free 729.5 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added rdd_21_0 in memory on localhost:62418 (size: 16.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 153). 1533 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 153) in 5 ms on localhost (1/1) -2016-06-13 17:54:26 INFO DAGScheduler:59 - ResultStage 47 (first at GaussianMixture.scala:171) finished in 0.005 s -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-06-13 17:54:26 INFO DAGScheduler:59 - Job 47 finished: first at GaussianMixture.scala:171, took 0.012073 s -2016-06-13 17:54:26 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Got job 48 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:54:26 INFO DAGScheduler:59 - Final stage: ResultStage 48(first at GaussianMixture.scala:171) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=1174358, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 1992.0 B, free 729.5 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1176350, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 1250.0 B, free 729.5 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:62418 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 48 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 3 tasks -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 154, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 155, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 48.0 (TID 156, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:26 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 155) -2016-06-13 17:54:26 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 154) -2016-06-13 17:54:26 INFO Executor:59 - Running task 2.0 in stage 48.0 (TID 156) -2016-06-13 17:54:26 INFO CacheManager:59 - Partition rdd_21_1 not found, computing it -2016-06-13 17:54:26 INFO CacheManager:59 - Partition rdd_21_3 not found, computing it -2016-06-13 17:54:26 INFO CacheManager:59 - Partition rdd_21_2 not found, computing it -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=1177600, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block rdd_21_3 stored as values in memory (estimated size 184.0 B, free 729.5 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=1177784, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block rdd_21_2 stored as values in memory (estimated size 192.0 B, free 729.5 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=1177976, maxMem=766075207 -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added rdd_21_3 in memory on localhost:62418 (size: 184.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - Block rdd_21_1 stored as values in memory (estimated size 192.0 B, free 729.5 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added rdd_21_2 in memory on localhost:62418 (size: 192.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added rdd_21_1 in memory on localhost:62418 (size: 192.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 154). 2144 bytes result sent to driver -2016-06-13 17:54:26 INFO Executor:59 - Finished task 2.0 in stage 48.0 (TID 156). 2132 bytes result sent to driver -2016-06-13 17:54:26 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 155). 2144 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 154) in 138 ms on localhost (1/3) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 48.0 (TID 156) in 136 ms on localhost (2/3) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 155) in 137 ms on localhost (3/3) -2016-06-13 17:54:26 INFO DAGScheduler:59 - ResultStage 48 (first at GaussianMixture.scala:171) finished in 0.139 s -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-06-13 17:54:26 INFO DAGScheduler:59 - Job 48 finished: first at GaussianMixture.scala:171, took 0.146391 s -2016-06-13 17:54:26 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Got job 49 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:26 INFO DAGScheduler:59 - Final stage: ResultStage 49(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=1178168, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 1832.0 B, free 729.5 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=1180000, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 1193.0 B, free 729.5 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:62418 (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 69 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 49 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 4 tasks -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 157, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 158, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 49.0 (TID 159, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 49.0 (TID 160, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:26 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 157) -2016-06-13 17:54:26 INFO Executor:59 - Running task 2.0 in stage 49.0 (TID 159) -2016-06-13 17:54:26 INFO Executor:59 - Running task 3.0 in stage 49.0 (TID 160) -2016-06-13 17:54:26 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 158) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 157). 2082 bytes result sent to driver -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_21_2 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 2.0 in stage 49.0 (TID 159). 2082 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 157) in 5 ms on localhost (1/4) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 158). 2082 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 49.0 (TID 159) in 5 ms on localhost (2/4) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_21_3 locally -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 158) in 6 ms on localhost (3/4) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 3.0 in stage 49.0 (TID 160). 2082 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 49.0 (TID 160) in 7 ms on localhost (4/4) -2016-06-13 17:54:26 INFO DAGScheduler:59 - ResultStage 49 (takeSample at GaussianMixture.scala:184) finished in 0.009 s -2016-06-13 17:54:26 INFO DAGScheduler:59 - Job 49 finished: takeSample at GaussianMixture.scala:184, took 0.016246 s -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-06-13 17:54:26 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Got job 50 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:26 INFO DAGScheduler:59 - Final stage: ResultStage 50(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting ResultStage 50 (PartitionwiseSampledRDD[22] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=1181193, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 18.8 KB, free 729.4 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(7882) called with curMem=1200417, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 7.7 KB, free 729.4 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:62418 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 50 (PartitionwiseSampledRDD[22] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 4 tasks -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 161, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 162, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 50.0 (TID 163, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 50.0 (TID 164, localhost, PROCESS_LOCAL, 2373 bytes) -2016-06-13 17:54:26 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 161) -2016-06-13 17:54:26 INFO Executor:59 - Running task 2.0 in stage 50.0 (TID 163) -2016-06-13 17:54:26 INFO Executor:59 - Running task 3.0 in stage 50.0 (TID 164) -2016-06-13 17:54:26 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 162) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_21_3 locally -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_21_2 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 3.0 in stage 50.0 (TID 164). 2692 bytes result sent to driver -2016-06-13 17:54:26 INFO Executor:59 - Finished task 2.0 in stage 50.0 (TID 163). 2704 bytes result sent to driver -2016-06-13 17:54:26 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 161). 2048 bytes result sent to driver -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 50.0 (TID 164) in 6 ms on localhost (1/4) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 162). 2724 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 50.0 (TID 163) in 7 ms on localhost (2/4) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 161) in 9 ms on localhost (3/4) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 162) in 10 ms on localhost (4/4) -2016-06-13 17:54:26 INFO DAGScheduler:59 - ResultStage 50 (takeSample at GaussianMixture.scala:184) finished in 0.011 s -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-06-13 17:54:26 INFO DAGScheduler:59 - Job 50 finished: takeSample at GaussianMixture.scala:184, took 0.019182 s -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=1208299, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 560.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(890) called with curMem=1208859, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 890.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:62418 (size: 890.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 71 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:26 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Got job 51 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:26 INFO DAGScheduler:59 - Final stage: ResultStage 51(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=1209749, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(2188) called with curMem=1213813, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.4 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 51 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 4 tasks -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 165, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 166, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 51.0 (TID 167, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 51.0 (TID 168, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:26 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 165) -2016-06-13 17:54:26 INFO Executor:59 - Running task 3.0 in stage 51.0 (TID 168) -2016-06-13 17:54:26 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 166) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 165). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO Executor:59 - Running task 2.0 in stage 51.0 (TID 167) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 165) in 9 ms on localhost (1/4) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_21_2 locally -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_21_3 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 2.0 in stage 51.0 (TID 167). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 166). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO Executor:59 - Finished task 3.0 in stage 51.0 (TID 168). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 51.0 (TID 167) in 33 ms on localhost (2/4) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 166) in 35 ms on localhost (3/4) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 51.0 (TID 168) in 33 ms on localhost (4/4) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-06-13 17:54:26 INFO DAGScheduler:59 - ResultStage 51 (aggregate at GaussianMixture.scala:201) finished in 0.036 s -2016-06-13 17:54:26 INFO DAGScheduler:59 - Job 51 finished: aggregate at GaussianMixture.scala:201, took 0.048381 s -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=1216001, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 560.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=1216561, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 900.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:62418 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 73 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:26 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Got job 52 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:26 INFO DAGScheduler:59 - Final stage: ResultStage 52(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting ResultStage 52 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=1217461, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=1221525, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.4 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 52 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 4 tasks -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 169, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 170, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 52.0 (TID 171, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 52.0 (TID 172, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:26 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 169) -2016-06-13 17:54:26 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 170) -2016-06-13 17:54:26 INFO Executor:59 - Running task 3.0 in stage 52.0 (TID 172) -2016-06-13 17:54:26 INFO Executor:59 - Running task 2.0 in stage 52.0 (TID 171) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_21_3 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 169). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 3.0 in stage 52.0 (TID 172). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 170). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_21_2 locally -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 169) in 7 ms on localhost (1/4) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 2.0 in stage 52.0 (TID 171). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 170) in 8 ms on localhost (2/4) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 52.0 (TID 172) in 7 ms on localhost (3/4) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 52.0 (TID 171) in 9 ms on localhost (4/4) -2016-06-13 17:54:26 INFO DAGScheduler:59 - ResultStage 52 (aggregate at GaussianMixture.scala:201) finished in 0.010 s -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-06-13 17:54:26 INFO DAGScheduler:59 - Job 52 finished: aggregate at GaussianMixture.scala:201, took 0.022106 s -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=1223719, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 560.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=1224279, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 900.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:62418 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 75 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:26 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Got job 53 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:26 INFO DAGScheduler:59 - Final stage: ResultStage 53(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=1225179, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=1229243, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.4 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 53 (MapPartitionsRDD[21] at map at GaussianMixture.scala:168) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 4 tasks -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 173, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 174, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 53.0 (TID 175, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 53.0 (TID 176, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:26 INFO Executor:59 - Running task 2.0 in stage 53.0 (TID 175) -2016-06-13 17:54:26 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 173) -2016-06-13 17:54:26 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 174) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 173). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 174). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 173) in 9 ms on localhost (1/4) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_21_2 locally -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 174) in 9 ms on localhost (2/4) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 2.0 in stage 53.0 (TID 175). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO Executor:59 - Running task 3.0 in stage 53.0 (TID 176) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 53.0 (TID 175) in 10 ms on localhost (3/4) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_21_3 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 3.0 in stage 53.0 (TID 176). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 53.0 (TID 176) in 13 ms on localhost (4/4) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-06-13 17:54:26 INFO DAGScheduler:59 - ResultStage 53 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 17:54:26 INFO DAGScheduler:59 - Job 53 finished: aggregate at GaussianMixture.scala:201, took 0.030816 s -2016-06-13 17:54:26 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Got job 54 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:54:26 INFO DAGScheduler:59 - Final stage: ResultStage 54(first at GaussianMixture.scala:171) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=1231437, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 1992.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1233429, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 1250.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:62418 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 77 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 54 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 1 tasks -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 177, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:26 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 177) -2016-06-13 17:54:26 INFO CacheManager:59 - Partition rdd_23_0 not found, computing it -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=1234679, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block rdd_23_0 stored as values in memory (estimated size 16.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added rdd_23_0 in memory on localhost:62418 (size: 16.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 177). 1533 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 177) in 8 ms on localhost (1/1) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-06-13 17:54:26 INFO DAGScheduler:59 - ResultStage 54 (first at GaussianMixture.scala:171) finished in 0.008 s -2016-06-13 17:54:26 INFO DAGScheduler:59 - Job 54 finished: first at GaussianMixture.scala:171, took 0.019539 s -2016-06-13 17:54:26 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Got job 55 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:54:26 INFO DAGScheduler:59 - Final stage: ResultStage 55(first at GaussianMixture.scala:171) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=1234695, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 1992.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1236687, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 1250.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:62418 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 55 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 3 tasks -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 178, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 179, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 55.0 (TID 180, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:26 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 178) -2016-06-13 17:54:26 INFO Executor:59 - Running task 2.0 in stage 55.0 (TID 180) -2016-06-13 17:54:26 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 179) -2016-06-13 17:54:26 INFO CacheManager:59 - Partition rdd_23_2 not found, computing it -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=1237937, maxMem=766075207 -2016-06-13 17:54:26 INFO CacheManager:59 - Partition rdd_23_3 not found, computing it -2016-06-13 17:54:26 INFO CacheManager:59 - Partition rdd_23_1 not found, computing it -2016-06-13 17:54:26 INFO MemoryStore:59 - Block rdd_23_2 stored as values in memory (estimated size 192.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added rdd_23_2 in memory on localhost:62418 (size: 192.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=1238129, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block rdd_23_1 stored as values in memory (estimated size 192.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 179). 2144 bytes result sent to driver -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added rdd_23_1 in memory on localhost:62418 (size: 192.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=1238321, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block rdd_23_3 stored as values in memory (estimated size 184.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 179) in 10 ms on localhost (1/3) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added rdd_23_3 in memory on localhost:62418 (size: 184.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 178). 2144 bytes result sent to driver -2016-06-13 17:54:26 INFO Executor:59 - Finished task 2.0 in stage 55.0 (TID 180). 2132 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 178) in 13 ms on localhost (2/3) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 55.0 (TID 180) in 12 ms on localhost (3/3) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-06-13 17:54:26 INFO DAGScheduler:59 - ResultStage 55 (first at GaussianMixture.scala:171) finished in 0.015 s -2016-06-13 17:54:26 INFO DAGScheduler:59 - Job 55 finished: first at GaussianMixture.scala:171, took 0.025581 s -2016-06-13 17:54:26 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Got job 56 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:26 INFO DAGScheduler:59 - Final stage: ResultStage 56(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=1238505, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 1832.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=1240337, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 1193.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:62418 (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 79 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 56 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 4 tasks -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 181, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 182, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 56.0 (TID 183, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 56.0 (TID 184, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:26 INFO Executor:59 - Running task 3.0 in stage 56.0 (TID 184) -2016-06-13 17:54:26 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 182) -2016-06-13 17:54:26 INFO Executor:59 - Running task 2.0 in stage 56.0 (TID 183) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_23_1 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 182). 2082 bytes result sent to driver -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_23_2 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 2.0 in stage 56.0 (TID 183). 2082 bytes result sent to driver -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_23_3 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 3.0 in stage 56.0 (TID 184). 2082 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 182) in 7 ms on localhost (1/4) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 56.0 (TID 184) in 7 ms on localhost (2/4) -2016-06-13 17:54:26 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 181) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 56.0 (TID 183) in 8 ms on localhost (3/4) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_23_0 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 181). 2082 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 181) in 13 ms on localhost (4/4) -2016-06-13 17:54:26 INFO DAGScheduler:59 - ResultStage 56 (takeSample at GaussianMixture.scala:184) finished in 0.014 s -2016-06-13 17:54:26 INFO DAGScheduler:59 - Job 56 finished: takeSample at GaussianMixture.scala:184, took 0.022868 s -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-06-13 17:54:26 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Got job 57 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:26 INFO DAGScheduler:59 - Final stage: ResultStage 57(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting ResultStage 57 (PartitionwiseSampledRDD[24] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=1241530, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 18.8 KB, free 729.4 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(7875) called with curMem=1260754, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 7.7 KB, free 729.4 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:62418 (size: 7.7 KB, free: 730.4 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 57 (PartitionwiseSampledRDD[24] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 4 tasks -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 185, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 186, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 57.0 (TID 187, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 57.0 (TID 188, localhost, PROCESS_LOCAL, 2373 bytes) -2016-06-13 17:54:26 INFO Executor:59 - Running task 2.0 in stage 57.0 (TID 187) -2016-06-13 17:54:26 INFO Executor:59 - Running task 3.0 in stage 57.0 (TID 188) -2016-06-13 17:54:26 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 185) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_23_3 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 3.0 in stage 57.0 (TID 188). 2682 bytes result sent to driver -2016-06-13 17:54:26 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 186) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_23_2 locally -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_23_1 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 186). 2724 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 57.0 (TID 188) in 9 ms on localhost (1/4) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 186) in 11 ms on localhost (2/4) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_23_0 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 2.0 in stage 57.0 (TID 187). 2714 bytes result sent to driver -2016-06-13 17:54:26 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 185). 2048 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 57.0 (TID 187) in 13 ms on localhost (3/4) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 185) in 16 ms on localhost (4/4) -2016-06-13 17:54:26 INFO DAGScheduler:59 - ResultStage 57 (takeSample at GaussianMixture.scala:184) finished in 0.016 s -2016-06-13 17:54:26 INFO DAGScheduler:59 - Job 57 finished: takeSample at GaussianMixture.scala:184, took 0.030886 s -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=1268629, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 560.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(891) called with curMem=1269189, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 891.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:62418 (size: 891.0 B, free: 730.4 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 81 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:62418 in memory (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:54:26 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-06-13 17:54:26 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Got job 58 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:26 INFO DAGScheduler:59 - Final stage: ResultStage 58(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:62418 in memory (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting ResultStage 58 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:26 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=1239956, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=1240778, maxMem=766075207 -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:62418 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.4 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 58 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 4 tasks -2016-06-13 17:54:26 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 189, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 190, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 58.0 (TID 191, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 58.0 (TID 192, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:26 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 189) -2016-06-13 17:54:26 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 190) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_23_0 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 189). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO Executor:59 - Running task 3.0 in stage 58.0 (TID 192) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:62418 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_23_1 locally -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_23_3 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 190). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO Executor:59 - Finished task 3.0 in stage 58.0 (TID 192). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO Executor:59 - Running task 2.0 in stage 58.0 (TID 191) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 189) in 11 ms on localhost (1/4) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_23_2 locally -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 190) in 12 ms on localhost (2/4) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 2.0 in stage 58.0 (TID 191). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 58.0 (TID 192) in 11 ms on localhost (3/4) -2016-06-13 17:54:26 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 58.0 (TID 191) in 14 ms on localhost (4/4) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-06-13 17:54:26 INFO DAGScheduler:59 - ResultStage 58 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 17:54:26 INFO DAGScheduler:59 - Job 58 finished: aggregate at GaussianMixture.scala:201, took 0.031419 s -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=1237536, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 560.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=1234032, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 900.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:62418 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 83 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:62418 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Got job 59 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:26 INFO DAGScheduler:59 - Final stage: ResultStage 59(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting ResultStage 59 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:26 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=1226654, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:62418 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=1229818, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.4 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 59 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 4 tasks -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 193, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 194, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 59.0 (TID 195, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 59.0 (TID 196, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:26 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 193) -2016-06-13 17:54:26 INFO Executor:59 - Running task 2.0 in stage 59.0 (TID 195) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_23_0 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 193). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 193) in 8 ms on localhost (1/4) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_23_2 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 2.0 in stage 59.0 (TID 195). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 59.0 (TID 195) in 11 ms on localhost (2/4) -2016-06-13 17:54:26 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 194) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_23_1 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 194). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO Executor:59 - Running task 3.0 in stage 59.0 (TID 196) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 194) in 18 ms on localhost (3/4) -2016-06-13 17:54:26 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_23_3 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 3.0 in stage 59.0 (TID 196). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:62418 in memory (size: 890.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 59.0 (TID 196) in 20 ms on localhost (4/4) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-06-13 17:54:26 INFO DAGScheduler:59 - ResultStage 59 (aggregate at GaussianMixture.scala:201) finished in 0.024 s -2016-06-13 17:54:26 INFO DAGScheduler:59 - Job 59 finished: aggregate at GaussianMixture.scala:201, took 0.039070 s -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=1224310, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 560.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:62418 in memory (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=1197764, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 900.0 B, free 729.4 MB) -2016-06-13 17:54:26 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:62418 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 85 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:62418 in memory (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:62418 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:26 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Got job 60 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:26 INFO DAGScheduler:59 - Final stage: ResultStage 60(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting ResultStage 60 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:62418 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=1189155, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 4.0 KB, free 729.4 MB) -2016-06-13 17:54:26 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=1193219, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 2.1 KB, free 729.4 MB) -2016-06-13 17:54:26 INFO BlockManager:59 - Removing RDD 21 -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 60 (MapPartitionsRDD[23] at map at GaussianMixture.scala:168) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Adding task set 60.0 with 4 tasks -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 60.0 (TID 197, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:26 INFO ContextCleaner:59 - Cleaned RDD 21 -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 60.0 (TID 198, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 60.0 (TID 199, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 60.0 (TID 200, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:26 INFO Executor:59 - Running task 2.0 in stage 60.0 (TID 199) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:62418 in memory (size: 1389.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO Executor:59 - Running task 3.0 in stage 60.0 (TID 200) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_23_2 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 2.0 in stage 60.0 (TID 199). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_23_3 locally -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 60.0 (TID 199) in 7 ms on localhost (1/4) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 3.0 in stage 60.0 (TID 200). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO Executor:59 - Running task 0.0 in stage 60.0 (TID 197) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 60.0 (TID 200) in 7 ms on localhost (2/4) -2016-06-13 17:54:26 INFO Executor:59 - Running task 1.0 in stage 60.0 (TID 198) -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_23_0 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 0.0 in stage 60.0 (TID 197). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO BlockManager:59 - Found block rdd_23_1 locally -2016-06-13 17:54:26 INFO Executor:59 - Finished task 1.0 in stage 60.0 (TID 198). 2756 bytes result sent to driver -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 60.0 (TID 197) in 15 ms on localhost (3/4) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 60.0 (TID 198) in 15 ms on localhost (4/4) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 60.0, whose tasks have all completed, from pool -2016-06-13 17:54:26 INFO DAGScheduler:59 - ResultStage 60 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 17:54:26 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Job 60 finished: aggregate at GaussianMixture.scala:201, took 0.028729 s -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:62418 in memory (size: 29.4 KB, free: 730.5 MB) -2016-06-13 17:54:26 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:62418 in memory (size: 23.3 KB, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Got job 61 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:54:26 INFO DAGScheduler:59 - Final stage: ResultStage 61(first at GaussianMixture.scala:171) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:62418 in memory (size: 1389.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:26 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting ResultStage 61 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=598981, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 1992.0 B, free 730.0 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:62418 in memory (size: 17.9 KB, free: 730.5 MB) -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=290317, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.3 MB) -2016-06-13 17:54:26 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:62418 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:54:26 INFO SparkContext:59 - Created broadcast 87 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 61 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:62418 in memory (size: 11.8 KB, free: 730.6 MB) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 1 tasks -2016-06-13 17:54:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 201, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:26 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 201) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:62418 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:54:26 INFO CacheManager:59 - Partition rdd_25_0 not found, computing it -2016-06-13 17:54:26 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-06-13 17:54:26 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=46883, maxMem=766075207 -2016-06-13 17:54:26 INFO MemoryStore:59 - Block rdd_25_0 stored as values in memory (estimated size 16.0 B, free 730.5 MB) -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Added rdd_25_0 in memory on localhost:62418 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:54:26 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 201). 1533 bytes result sent to driver -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:62418 in memory (size: 1193.0 B, free: 730.6 MB) -2016-06-13 17:54:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 201) in 11 ms on localhost (1/1) -2016-06-13 17:54:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-06-13 17:54:26 INFO DAGScheduler:59 - ResultStage 61 (first at GaussianMixture.scala:171) finished in 0.011 s -2016-06-13 17:54:26 INFO DAGScheduler:59 - Job 61 finished: first at GaussianMixture.scala:171, took 0.024627 s -2016-06-13 17:54:26 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-06-13 17:54:26 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:62418 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:54:26 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-06-13 17:54:26 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:26 INFO BlockManager:59 - Removing RDD 14 -2016-06-13 17:54:26 INFO DAGScheduler:59 - Got job 62 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:54:26 INFO DAGScheduler:59 - Final stage: ResultStage 62(first at GaussianMixture.scala:171) -2016-06-13 17:54:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:26 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:26 INFO ContextCleaner:59 - Cleaned RDD 14 -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=29096, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=28979, maxMem=766075207 -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:62418 in memory (size: 837.0 B, free: 730.6 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.6 MB) -2016-06-13 17:54:27 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:62418 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 62 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 3 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 202, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 62.0 (TID 203, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 62.0 (TID 204, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 62.0 (TID 204) -2016-06-13 17:54:27 INFO CacheManager:59 - Partition rdd_25_3 not found, computing it -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=30229, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block rdd_25_3 stored as values in memory (estimated size 184.0 B, free 730.6 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added rdd_25_3 in memory on localhost:62418 (size: 184.0 B, free: 730.6 MB) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 62.0 (TID 204). 2132 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 202) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 62.0 (TID 204) in 9 ms on localhost (1/3) -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 62.0 (TID 203) -2016-06-13 17:54:27 INFO CacheManager:59 - Partition rdd_25_1 not found, computing it -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=30413, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block rdd_25_1 stored as values in memory (estimated size 192.0 B, free 730.6 MB) -2016-06-13 17:54:27 INFO CacheManager:59 - Partition rdd_25_2 not found, computing it -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added rdd_25_1 in memory on localhost:62418 (size: 192.0 B, free: 730.6 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=30605, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block rdd_25_2 stored as values in memory (estimated size 192.0 B, free 730.6 MB) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 202). 2144 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added rdd_25_2 in memory on localhost:62418 (size: 192.0 B, free: 730.6 MB) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 62.0 (TID 203). 2144 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 202) in 19 ms on localhost (2/3) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 62.0 (TID 203) in 18 ms on localhost (3/3) -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 62 (first at GaussianMixture.scala:171) finished in 0.020 s -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 62 finished: first at GaussianMixture.scala:171, took 0.032781 s -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 63 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 63(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 63 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=30797, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=32629, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 1193.0 B, free 730.6 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:62418 (size: 1193.0 B, free: 730.6 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 89 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 63 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 63.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 63.0 (TID 205, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 63.0 (TID 206, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 63.0 (TID 207, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 63.0 (TID 208, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 63.0 (TID 208) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 63.0 (TID 205) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 63.0 (TID 207) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_25_2 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 63.0 (TID 207). 2082 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_25_3 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 63.0 (TID 208). 2082 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 63.0 (TID 206) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 63.0 (TID 205). 2082 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 63.0 (TID 207) in 6 ms on localhost (1/4) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 63.0 (TID 208) in 7 ms on localhost (2/4) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 63.0 (TID 206). 2082 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 63.0 (TID 205) in 10 ms on localhost (3/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 63.0 (TID 206) in 11 ms on localhost (4/4) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 63.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 63 (takeSample at GaussianMixture.scala:184) finished in 0.012 s -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 63 finished: takeSample at GaussianMixture.scala:184, took 0.021054 s -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 64 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 64(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 64 (PartitionwiseSampledRDD[26] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=33822, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(7885) called with curMem=53046, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:62418 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 64 (PartitionwiseSampledRDD[26] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 64.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 64.0 (TID 209, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 64.0 (TID 210, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 64.0 (TID 211, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 64.0 (TID 212, localhost, PROCESS_LOCAL, 2373 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 64.0 (TID 211) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 64.0 (TID 209) -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 64.0 (TID 212) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 64.0 (TID 209). 2048 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_25_3 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 64.0 (TID 212). 2702 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 64.0 (TID 209) in 9 ms on localhost (1/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 64.0 (TID 212) in 8 ms on localhost (2/4) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_25_2 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 64.0 (TID 211). 2714 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 64.0 (TID 210) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 64.0 (TID 211) in 11 ms on localhost (3/4) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 64.0 (TID 210). 2684 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 64.0 (TID 210) in 15 ms on localhost (4/4) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 64.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 64 (takeSample at GaussianMixture.scala:184) finished in 0.016 s -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 64 finished: takeSample at GaussianMixture.scala:184, took 0.027421 s -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=60931, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(899) called with curMem=61491, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 899.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:62418 (size: 899.0 B, free: 730.6 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 91 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 65 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 65(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=62390, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(2204) called with curMem=66454, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:62418 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 65 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 213, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 214, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 65.0 (TID 215, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 65.0 (TID 216, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 214) -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 65.0 (TID 216) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 65.0 (TID 215) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_25_2 locally -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 213) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 65.0 (TID 215). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 65.0 (TID 215) in 7 ms on localhost (1/4) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 214). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 214) in 10 ms on localhost (2/4) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_25_3 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 213). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 65.0 (TID 216). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 213) in 14 ms on localhost (3/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 65.0 (TID 216) in 13 ms on localhost (4/4) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 65 (aggregate at GaussianMixture.scala:201) finished in 0.015 s -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 65 finished: aggregate at GaussianMixture.scala:201, took 0.026768 s -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=68658, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=69218, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:62418 (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 93 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 66 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 66(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 66 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=70118, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=74182, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 66 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 66.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 66.0 (TID 217, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 66.0 (TID 218, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 66.0 (TID 219, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 66.0 (TID 220, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 66.0 (TID 220) -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 66.0 (TID 218) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 66.0 (TID 219) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 66.0 (TID 218). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_25_2 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 66.0 (TID 219). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 66.0 (TID 218) in 7 ms on localhost (1/4) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_25_3 locally -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 66.0 (TID 219) in 9 ms on localhost (2/4) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 66.0 (TID 220). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 66.0 (TID 220) in 10 ms on localhost (3/4) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 66.0 (TID 217) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 66.0 (TID 217). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 66.0 (TID 217) in 17 ms on localhost (4/4) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 66.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 66 (aggregate at GaussianMixture.scala:201) finished in 0.018 s -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 66 finished: aggregate at GaussianMixture.scala:201, took 0.033666 s -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=76376, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=76936, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:62418 (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 95 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 67 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 67(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 67 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=77836, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=81900, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 67 (MapPartitionsRDD[25] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 67.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 67.0 (TID 221, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 67.0 (TID 222, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 67.0 (TID 223, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 67.0 (TID 224, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 67.0 (TID 223) -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 67.0 (TID 224) -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 67.0 (TID 222) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_25_3 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 67.0 (TID 224). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_25_2 locally -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 67.0 (TID 224) in 6 ms on localhost (1/4) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 67.0 (TID 223). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 67.0 (TID 221) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 67.0 (TID 223) in 9 ms on localhost (2/4) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 67.0 (TID 222). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 67.0 (TID 221). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 67.0 (TID 222) in 13 ms on localhost (3/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 67.0 (TID 221) in 15 ms on localhost (4/4) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 67.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 67 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 67 finished: aggregate at GaussianMixture.scala:201, took 0.029240 s -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 68 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 68(first at GaussianMixture.scala:171) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 68 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=84094, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=86086, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:62418 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 97 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 68 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 68.0 with 1 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 68.0 (TID 225, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 68.0 (TID 225) -2016-06-13 17:54:27 INFO CacheManager:59 - Partition rdd_27_0 not found, computing it -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=87336, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block rdd_27_0 stored as values in memory (estimated size 16.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added rdd_27_0 in memory on localhost:62418 (size: 16.0 B, free: 730.6 MB) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 68.0 (TID 225). 1533 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 68.0 (TID 225) in 7 ms on localhost (1/1) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 68.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 68 (first at GaussianMixture.scala:171) finished in 0.009 s -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 68 finished: first at GaussianMixture.scala:171, took 0.021081 s -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 69 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 69(first at GaussianMixture.scala:171) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 69 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=87352, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 1992.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=89344, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:62418 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 69 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 69.0 with 3 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 69.0 (TID 226, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 69.0 (TID 227, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 69.0 (TID 228, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 69.0 (TID 228) -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 69.0 (TID 227) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 69.0 (TID 226) -2016-06-13 17:54:27 INFO CacheManager:59 - Partition rdd_27_3 not found, computing it -2016-06-13 17:54:27 INFO CacheManager:59 - Partition rdd_27_1 not found, computing it -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=90594, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block rdd_27_3 stored as values in memory (estimated size 184.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=90778, maxMem=766075207 -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added rdd_27_3 in memory on localhost:62418 (size: 184.0 B, free: 730.6 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - Block rdd_27_1 stored as values in memory (estimated size 192.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO CacheManager:59 - Partition rdd_27_2 not found, computing it -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added rdd_27_1 in memory on localhost:62418 (size: 192.0 B, free: 730.6 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=90970, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block rdd_27_2 stored as values in memory (estimated size 192.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 69.0 (TID 226). 2144 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added rdd_27_2 in memory on localhost:62418 (size: 192.0 B, free: 730.6 MB) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 69.0 (TID 228). 2132 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 69.0 (TID 226) in 12 ms on localhost (1/3) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 69.0 (TID 227). 2144 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 69.0 (TID 228) in 13 ms on localhost (2/3) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 69.0 (TID 227) in 15 ms on localhost (3/3) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 69.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 69 (first at GaussianMixture.scala:171) finished in 0.016 s -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 69 finished: first at GaussianMixture.scala:171, took 0.027612 s -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 70 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 70(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 70 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=91162, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 1832.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=92994, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 1193.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:62418 (size: 1193.0 B, free: 730.6 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 99 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 70 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 70.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 70.0 (TID 229, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 70.0 (TID 230, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 70.0 (TID 231, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 70.0 (TID 232, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 70.0 (TID 231) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 70.0 (TID 229) -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 70.0 (TID 232) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 70.0 (TID 229). 2082 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 70.0 (TID 230) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_27_2 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 70.0 (TID 231). 2082 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_27_3 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 70.0 (TID 232). 2082 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 70.0 (TID 230). 2082 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 70.0 (TID 229) in 13 ms on localhost (1/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 70.0 (TID 231) in 12 ms on localhost (2/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 70.0 (TID 230) in 13 ms on localhost (3/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 70.0 (TID 232) in 13 ms on localhost (4/4) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 70.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 70 (takeSample at GaussianMixture.scala:184) finished in 0.015 s -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 70 finished: takeSample at GaussianMixture.scala:184, took 0.027708 s -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 71 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 71(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 71 (PartitionwiseSampledRDD[28] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=94187, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 18.8 KB, free 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(7882) called with curMem=113411, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:62418 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 71 (PartitionwiseSampledRDD[28] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 71.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 71.0 (TID 233, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 71.0 (TID 234, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 71.0 (TID 235, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 71.0 (TID 236, localhost, PROCESS_LOCAL, 2373 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 71.0 (TID 236) -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 71.0 (TID 234) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 71.0 (TID 233) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_27_3 locally -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 71.0 (TID 236). 2722 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 71.0 (TID 233). 2048 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 71.0 (TID 236) in 4 ms on localhost (1/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 71.0 (TID 233) in 7 ms on localhost (2/4) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 71.0 (TID 235) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 71.0 (TID 234). 2719 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_27_2 locally -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 71.0 (TID 234) in 10 ms on localhost (3/4) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 71.0 (TID 235). 2714 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 71.0 (TID 235) in 12 ms on localhost (4/4) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 71.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 71 (takeSample at GaussianMixture.scala:184) finished in 0.013 s -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 71 finished: takeSample at GaussianMixture.scala:184, took 0.022992 s -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=121293, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(895) called with curMem=121853, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 895.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:62418 (size: 895.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 101 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 72 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 72(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 72 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=122748, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(2197) called with curMem=126812, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 72 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 72.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 72.0 (TID 237, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 72.0 (TID 238, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 72.0 (TID 239, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 72.0 (TID 240, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 72.0 (TID 239) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 72.0 (TID 237) -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 72.0 (TID 240) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 72.0 (TID 237). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_27_3 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 72.0 (TID 240). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 72.0 (TID 237) in 7 ms on localhost (1/4) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_27_2 locally -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 72.0 (TID 240) in 6 ms on localhost (2/4) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 72.0 (TID 239). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 72.0 (TID 238) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 72.0 (TID 239) in 9 ms on localhost (3/4) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 72.0 (TID 238). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 72.0 (TID 238) in 13 ms on localhost (4/4) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 72.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 72 (aggregate at GaussianMixture.scala:201) finished in 0.014 s -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 72 finished: aggregate at GaussianMixture.scala:201, took 0.028702 s -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=129009, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=129569, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:62418 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 103 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 73 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 73(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 73 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=130469, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=134533, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 104 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 73 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 73.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 73.0 (TID 241, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 73.0 (TID 242, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 73.0 (TID 243, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 73.0 (TID 244, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 73.0 (TID 243) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_27_2 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 73.0 (TID 243). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 73.0 (TID 242) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 73.0 (TID 242). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 73.0 (TID 243) in 10 ms on localhost (1/4) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 73.0 (TID 241) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 73.0 (TID 242) in 12 ms on localhost (2/4) -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 73.0 (TID 244) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 73.0 (TID 241). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_27_3 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 73.0 (TID 244). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 73.0 (TID 241) in 16 ms on localhost (3/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 73.0 (TID 244) in 15 ms on localhost (4/4) -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 73 (aggregate at GaussianMixture.scala:201) finished in 0.018 s -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 73 finished: aggregate at GaussianMixture.scala:201, took 0.027260 s -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 73.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=136727, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 560.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=137287, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.5 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:62418 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 105 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 74 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 74(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 74 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=138187, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(2194) called with curMem=142251, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 106 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 74 (MapPartitionsRDD[27] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 74.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 74.0 (TID 245, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 74.0 (TID 246, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 74.0 (TID 247, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 74.0 (TID 248, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 74.0 (TID 245) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 74.0 (TID 247) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 74.0 (TID 245). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 74.0 (TID 246) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_27_2 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 74.0 (TID 247). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 74.0 (TID 248) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 74.0 (TID 246). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_27_3 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 74.0 (TID 248). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 74.0 (TID 245) in 13 ms on localhost (1/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 74.0 (TID 246) in 14 ms on localhost (2/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 74.0 (TID 247) in 14 ms on localhost (3/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 74.0 (TID 248) in 14 ms on localhost (4/4) -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 74 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 74.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 74 finished: aggregate at GaussianMixture.scala:201, took 0.024672 s -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 75 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 75(first at GaussianMixture.scala:171) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 75 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=144445, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 1992.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=146437, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:62418 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 107 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 75 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 75.0 with 1 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 75.0 (TID 249, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 75.0 (TID 249) -2016-06-13 17:54:27 INFO CacheManager:59 - Partition rdd_29_0 not found, computing it -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(16) called with curMem=147687, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block rdd_29_0 stored as values in memory (estimated size 16.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added rdd_29_0 in memory on localhost:62418 (size: 16.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 75.0 (TID 249). 1533 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 75.0 (TID 249) in 7 ms on localhost (1/1) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 75.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 75 (first at GaussianMixture.scala:171) finished in 0.007 s -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 75 finished: first at GaussianMixture.scala:171, took 0.033872 s -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 76 (first at GaussianMixture.scala:171) with 3 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 76(first at GaussianMixture.scala:171) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 76 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=147703, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 1992.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=149695, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:62418 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 108 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 76 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 3 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 250, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 251, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 76.0 (TID 252, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 250) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 76.0 (TID 252) -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 251) -2016-06-13 17:54:27 INFO CacheManager:59 - Partition rdd_29_3 not found, computing it -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=150945, maxMem=766075207 -2016-06-13 17:54:27 INFO CacheManager:59 - Partition rdd_29_2 not found, computing it -2016-06-13 17:54:27 INFO CacheManager:59 - Partition rdd_29_1 not found, computing it -2016-06-13 17:54:27 INFO MemoryStore:59 - Block rdd_29_3 stored as values in memory (estimated size 184.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=151129, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block rdd_29_1 stored as values in memory (estimated size 192.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added rdd_29_3 in memory on localhost:62418 (size: 184.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added rdd_29_1 in memory on localhost:62418 (size: 192.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 250). 2144 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 250) in 10 ms on localhost (1/3) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(192) called with curMem=151321, maxMem=766075207 -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 76.0 (TID 252). 2132 bytes result sent to driver -2016-06-13 17:54:27 INFO MemoryStore:59 - Block rdd_29_2 stored as values in memory (estimated size 192.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added rdd_29_2 in memory on localhost:62418 (size: 192.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 76.0 (TID 252) in 11 ms on localhost (2/3) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 251). 2144 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 251) in 13 ms on localhost (3/3) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 76 (first at GaussianMixture.scala:171) finished in 0.015 s -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 76 finished: first at GaussianMixture.scala:171, took 0.025383 s -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 77 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 77(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=151513, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 1832.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=153345, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 1193.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:62418 (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 109 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 77 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 253, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 254, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 77.0 (TID 255, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 77.0 (TID 256, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 77.0 (TID 256) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 77.0 (TID 255) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_29_3 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 77.0 (TID 256). 2082 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_29_2 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 77.0 (TID 255). 2082 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 77.0 (TID 256) in 6 ms on localhost (1/4) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 253) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 77.0 (TID 255) in 7 ms on localhost (2/4) -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 254) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_29_0 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 253). 2082 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_29_1 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 254). 2082 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 253) in 12 ms on localhost (3/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 254) in 12 ms on localhost (4/4) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 77 (takeSample at GaussianMixture.scala:184) finished in 0.014 s -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 77 finished: takeSample at GaussianMixture.scala:184, took 0.024609 s -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 78 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 78(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 78 (PartitionwiseSampledRDD[30] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=154538, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 18.8 KB, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(7882) called with curMem=173762, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:62418 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 110 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 78 (PartitionwiseSampledRDD[30] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 257, localhost, PROCESS_LOCAL, 2216 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 258, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 78.0 (TID 259, localhost, PROCESS_LOCAL, 2385 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 78.0 (TID 260, localhost, PROCESS_LOCAL, 2373 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 78.0 (TID 259) -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 258) -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 78.0 (TID 260) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_29_1 locally -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_29_3 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 258). 2684 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 78.0 (TID 260). 2687 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_29_2 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 78.0 (TID 259). 2699 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 257) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 258) in 7 ms on localhost (1/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 78.0 (TID 259) in 8 ms on localhost (2/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 78.0 (TID 260) in 9 ms on localhost (3/4) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_29_0 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 257). 2048 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 257) in 12 ms on localhost (4/4) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 78 (takeSample at GaussianMixture.scala:184) finished in 0.013 s -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 78 finished: takeSample at GaussianMixture.scala:184, took 0.029088 s -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=181644, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 560.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(882) called with curMem=182204, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 882.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:62418 (size: 882.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 111 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 79 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 79(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 79 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=183086, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(2182) called with curMem=187150, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 112 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 79 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 79.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 79.0 (TID 261, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 79.0 (TID 262, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 79.0 (TID 263, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 79.0 (TID 264, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 79.0 (TID 263) -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 79.0 (TID 262) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 79.0 (TID 261) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_29_2 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 79.0 (TID 263). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_29_0 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 79.0 (TID 261). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_29_1 locally -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 79.0 (TID 263) in 7 ms on localhost (1/4) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 79.0 (TID 262). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 79.0 (TID 261) in 9 ms on localhost (2/4) -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 79.0 (TID 264) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 79.0 (TID 262) in 10 ms on localhost (3/4) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_29_3 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 79.0 (TID 264). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 79.0 (TID 264) in 13 ms on localhost (4/4) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 79.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 79 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 79 finished: aggregate at GaussianMixture.scala:201, took 0.027914 s -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=189332, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 560.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=189892, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:62418 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 113 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 80 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 80(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 80 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=190792, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(2192) called with curMem=194856, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 114 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 80 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 80.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 80.0 (TID 265, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 80.0 (TID 266, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 80.0 (TID 267, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 80.0 (TID 268, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 80.0 (TID 266) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 80.0 (TID 265) -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 80.0 (TID 268) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_29_0 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 80.0 (TID 265). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_29_3 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 80.0 (TID 268). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_29_1 locally -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 80.0 (TID 265) in 8 ms on localhost (1/4) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 80.0 (TID 266). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 80.0 (TID 266) in 10 ms on localhost (2/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 80.0 (TID 268) in 8 ms on localhost (3/4) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 80.0 (TID 267) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_29_2 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 80.0 (TID 267). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 80.0 (TID 267) in 14 ms on localhost (4/4) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 80.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 80 (aggregate at GaussianMixture.scala:201) finished in 0.015 s -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 80 finished: aggregate at GaussianMixture.scala:201, took 0.030383 s -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(560) called with curMem=197048, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_115 stored as values in memory (estimated size 560.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(900) called with curMem=197608, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_115_piece0 stored as bytes in memory (estimated size 900.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_115_piece0 in memory on localhost:62418 (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 115 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 81 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 81(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 81 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(4064) called with curMem=198508, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_116 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(2192) called with curMem=202572, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_116_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_116_piece0 in memory on localhost:62418 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 116 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 81 (MapPartitionsRDD[29] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 81.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 81.0 (TID 269, localhost, PROCESS_LOCAL, 2107 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 81.0 (TID 270, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 81.0 (TID 271, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 81.0 (TID 272, localhost, PROCESS_LOCAL, 2264 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 81.0 (TID 270) -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 81.0 (TID 272) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 81.0 (TID 269) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 81.0 (TID 271) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_29_0 locally -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_29_2 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 81.0 (TID 269). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 81.0 (TID 271). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_29_3 locally -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_29_1 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 81.0 (TID 272). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 81.0 (TID 270). 2756 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 81.0 (TID 269) in 5 ms on localhost (1/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 81.0 (TID 271) in 6 ms on localhost (2/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 81.0 (TID 272) in 5 ms on localhost (3/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 81.0 (TID 270) in 6 ms on localhost (4/4) -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 81 (aggregate at GaussianMixture.scala:201) finished in 0.007 s -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 81.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 81 finished: aggregate at GaussianMixture.scala:201, took 0.020907 s -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 82 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 82(first at GaussianMixture.scala:171) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 82 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(2184) called with curMem=204764, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_117 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1348) called with curMem=206948, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_117_piece0 stored as bytes in memory (estimated size 1348.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_117_piece0 in memory on localhost:62418 (size: 1348.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 117 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 82 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 82.0 with 1 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 82.0 (TID 273, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 82.0 (TID 273) -2016-06-13 17:54:27 INFO CacheManager:59 - Partition rdd_33_0 not found, computing it -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(480) called with curMem=208296, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block rdd_33_0 stored as values in memory (estimated size 480.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added rdd_33_0 in memory on localhost:62418 (size: 480.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 82.0 (TID 273). 2132 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 82.0 (TID 273) in 5 ms on localhost (1/1) -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 82 (first at GaussianMixture.scala:171) finished in 0.005 s -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 82.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 82 finished: first at GaussianMixture.scala:171, took 0.012813 s -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=208776, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_118 stored as values in memory (estimated size 712.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=209488, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_118_piece0 stored as bytes in memory (estimated size 880.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_118_piece0 in memory on localhost:62418 (size: 880.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 118 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 83 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 83(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 83 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(4288) called with curMem=210368, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_119 stored as values in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(2279) called with curMem=214656, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_119_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_119_piece0 in memory on localhost:62418 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 119 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 83 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 83.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 83.0 (TID 274, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 83.0 (TID 275, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 83.0 (TID 276, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 83.0 (TID 277, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 83.0 (TID 274) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 83.0 (TID 276) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_33_0 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 83.0 (TID 274). 2762 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 83.0 (TID 277) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 83.0 (TID 274) in 12 ms on localhost (1/4) -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 83.0 (TID 275) -2016-06-13 17:54:27 INFO CacheManager:59 - Partition rdd_33_2 not found, computing it -2016-06-13 17:54:27 INFO CacheManager:59 - Partition rdd_33_3 not found, computing it -2016-06-13 17:54:27 INFO CacheManager:59 - Partition rdd_33_1 not found, computing it -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(624) called with curMem=216935, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block rdd_33_2 stored as values in memory (estimated size 624.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(624) called with curMem=217559, maxMem=766075207 -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added rdd_33_2 in memory on localhost:62418 (size: 624.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - Block rdd_33_3 stored as values in memory (estimated size 624.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(624) called with curMem=218183, maxMem=766075207 -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added rdd_33_3 in memory on localhost:62418 (size: 624.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - Block rdd_33_1 stored as values in memory (estimated size 624.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added rdd_33_1 in memory on localhost:62418 (size: 624.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 83.0 (TID 277). 2247 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 83.0 (TID 276). 2247 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 83.0 (TID 277) in 19 ms on localhost (2/4) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 83.0 (TID 275). 2247 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 83.0 (TID 276) in 20 ms on localhost (3/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 83.0 (TID 275) in 21 ms on localhost (4/4) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 83.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 83 (aggregate at GaussianMixture.scala:201) finished in 0.022 s -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 83 finished: aggregate at GaussianMixture.scala:201, took 0.030377 s -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=218807, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_120 stored as values in memory (estimated size 712.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(934) called with curMem=219519, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_120_piece0 stored as bytes in memory (estimated size 934.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_120_piece0 in memory on localhost:62418 (size: 934.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 120 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 84 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 84(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 84 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(4288) called with curMem=220453, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_121 stored as values in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(2343) called with curMem=224741, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_121_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_121_piece0 in memory on localhost:62418 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 121 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 84 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 84.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 84.0 (TID 278, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 84.0 (TID 279, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 84.0 (TID 280, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 84.0 (TID 281, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 84.0 (TID 278) -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 84.0 (TID 281) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 84.0 (TID 280) -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 84.0 (TID 279) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_33_0 locally -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_33_3 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 84.0 (TID 281). 2762 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_33_2 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 84.0 (TID 278). 2762 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 84.0 (TID 280). 2762 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_33_1 locally -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 84.0 (TID 281) in 6 ms on localhost (1/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 84.0 (TID 278) in 8 ms on localhost (2/4) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 84.0 (TID 279). 2762 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 84.0 (TID 280) in 8 ms on localhost (3/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 84.0 (TID 279) in 9 ms on localhost (4/4) -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 84 (aggregate at GaussianMixture.scala:201) finished in 0.010 s -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 84.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 84 finished: aggregate at GaussianMixture.scala:201, took 0.018486 s -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=227084, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_122 stored as values in memory (estimated size 712.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=227796, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_122_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_122_piece0 in memory on localhost:62418 (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 122 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 85 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 85(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 85 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(4288) called with curMem=228728, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_123 stored as values in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(2340) called with curMem=233016, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_123_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_123_piece0 in memory on localhost:62418 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 123 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 85 (MapPartitionsRDD[33] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 85.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 85.0 (TID 282, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 85.0 (TID 283, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 85.0 (TID 284, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 85.0 (TID 285, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 85.0 (TID 285) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 85.0 (TID 284) -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 85.0 (TID 283) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 85.0 (TID 282) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_33_2 locally -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_33_1 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 85.0 (TID 284). 2762 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_33_0 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 85.0 (TID 283). 2762 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_33_3 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 85.0 (TID 282). 2762 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 85.0 (TID 285). 2762 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 85.0 (TID 284) in 6 ms on localhost (1/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 85.0 (TID 282) in 6 ms on localhost (2/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 85.0 (TID 285) in 6 ms on localhost (3/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 85.0 (TID 283) in 7 ms on localhost (4/4) -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 85 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 85.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 85 finished: aggregate at GaussianMixture.scala:201, took 0.017187 s -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 86 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 86(first at GaussianMixture.scala:171) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 86 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=235356, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_124 stored as values in memory (estimated size 1992.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=237348, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_124_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_124_piece0 in memory on localhost:62418 (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 124 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 86 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 86.0 with 1 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 86.0 (TID 286, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 86.0 (TID 286) -2016-06-13 17:54:27 INFO CacheManager:59 - Partition rdd_35_0 not found, computing it -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(224) called with curMem=238598, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block rdd_35_0 stored as values in memory (estimated size 224.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added rdd_35_0 in memory on localhost:62418 (size: 224.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 86.0 (TID 286). 1688 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 86.0 (TID 286) in 4 ms on localhost (1/1) -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 86 (first at GaussianMixture.scala:171) finished in 0.004 s -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 86.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 86 finished: first at GaussianMixture.scala:171, took 0.013170 s -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 87 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 87(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 87 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=238822, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_125 stored as values in memory (estimated size 1832.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(1193) called with curMem=240654, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_125_piece0 stored as bytes in memory (estimated size 1193.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_125_piece0 in memory on localhost:62418 (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 125 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 87 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 87.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 87.0 (TID 287, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 87.0 (TID 288, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 87.0 (TID 289, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 87.0 (TID 290, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 87.0 (TID 287) -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 87.0 (TID 290) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 87.0 (TID 289) -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 87.0 (TID 288) -2016-06-13 17:54:27 INFO CacheManager:59 - Partition rdd_35_3 not found, computing it -2016-06-13 17:54:27 INFO CacheManager:59 - Partition rdd_35_1 not found, computing it -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=241847, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block rdd_35_3 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=242135, maxMem=766075207 -2016-06-13 17:54:27 INFO CacheManager:59 - Partition rdd_35_2 not found, computing it -2016-06-13 17:54:27 INFO MemoryStore:59 - Block rdd_35_1 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added rdd_35_3 in memory on localhost:62418 (size: 288.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 87.0 (TID 287). 2082 bytes result sent to driver -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=242423, maxMem=766075207 -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added rdd_35_1 in memory on localhost:62418 (size: 288.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - Block rdd_35_2 stored as values in memory (estimated size 288.0 B, free 730.4 MB) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 87.0 (TID 287) in 7 ms on localhost (1/4) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 87.0 (TID 290). 1567 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added rdd_35_2 in memory on localhost:62418 (size: 288.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 87.0 (TID 288). 1567 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 87.0 (TID 289). 1567 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 87.0 (TID 290) in 7 ms on localhost (2/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 87.0 (TID 288) in 8 ms on localhost (3/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 87.0 (TID 289) in 8 ms on localhost (4/4) -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 87 (takeSample at GaussianMixture.scala:184) finished in 0.009 s -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 87.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 87 finished: takeSample at GaussianMixture.scala:184, took 0.018716 s -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 88 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 88(takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 88 (PartitionwiseSampledRDD[36] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=242711, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_126 stored as values in memory (estimated size 18.8 KB, free 730.3 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(7882) called with curMem=261935, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_126_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.3 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_126_piece0 in memory on localhost:62418 (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 126 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 88 (PartitionwiseSampledRDD[36] at takeSample at GaussianMixture.scala:184) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 88.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 88.0 (TID 291, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 88.0 (TID 292, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 88.0 (TID 293, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 88.0 (TID 294, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 88.0 (TID 291) -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 88.0 (TID 294) -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 88.0 (TID 292) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 88.0 (TID 293) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 88.0 (TID 292). 2344 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 88.0 (TID 294). 2339 bytes result sent to driver -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 88.0 (TID 291). 2247 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 88.0 (TID 293). 2303 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 88.0 (TID 292) in 6 ms on localhost (1/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 88.0 (TID 294) in 5 ms on localhost (2/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 88.0 (TID 291) in 6 ms on localhost (3/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 88.0 (TID 293) in 6 ms on localhost (4/4) -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 88 (takeSample at GaussianMixture.scala:184) finished in 0.007 s -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 88.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 88 finished: takeSample at GaussianMixture.scala:184, took 0.014714 s -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=269817, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_127 stored as values in memory (estimated size 712.0 B, free 730.3 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(937) called with curMem=270529, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_127_piece0 stored as bytes in memory (estimated size 937.0 B, free 730.3 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_127_piece0 in memory on localhost:62418 (size: 937.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 127 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 89 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 89(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 89 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=271466, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_128 stored as values in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=275562, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_128_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_128_piece0 in memory on localhost:62418 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 128 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 89 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 89.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 89.0 (TID 295, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 89.0 (TID 296, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 89.0 (TID 297, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 89.0 (TID 298, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:27 INFO Executor:59 - Running task 1.0 in stage 89.0 (TID 296) -2016-06-13 17:54:27 INFO Executor:59 - Running task 2.0 in stage 89.0 (TID 297) -2016-06-13 17:54:27 INFO Executor:59 - Running task 3.0 in stage 89.0 (TID 298) -2016-06-13 17:54:27 INFO Executor:59 - Running task 0.0 in stage 89.0 (TID 295) -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:54:27 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:54:27 INFO Executor:59 - Finished task 0.0 in stage 89.0 (TID 295). 2762 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Finished task 3.0 in stage 89.0 (TID 298). 2762 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Finished task 1.0 in stage 89.0 (TID 296). 2762 bytes result sent to driver -2016-06-13 17:54:27 INFO Executor:59 - Finished task 2.0 in stage 89.0 (TID 297). 2762 bytes result sent to driver -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 89.0 (TID 295) in 8 ms on localhost (1/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 89.0 (TID 298) in 6 ms on localhost (2/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 89.0 (TID 296) in 8 ms on localhost (3/4) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 89.0 (TID 297) in 7 ms on localhost (4/4) -2016-06-13 17:54:27 INFO DAGScheduler:59 - ResultStage 89 (aggregate at GaussianMixture.scala:201) finished in 0.009 s -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 89.0, whose tasks have all completed, from pool -2016-06-13 17:54:27 INFO DAGScheduler:59 - Job 89 finished: aggregate at GaussianMixture.scala:201, took 0.017190 s -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=277800, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_129 stored as values in memory (estimated size 712.0 B, free 730.3 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(934) called with curMem=278512, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_129_piece0 stored as bytes in memory (estimated size 934.0 B, free 730.3 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_129_piece0 in memory on localhost:62418 (size: 934.0 B, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 129 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:27 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Got job 90 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:27 INFO DAGScheduler:59 - Final stage: ResultStage 90(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting ResultStage 90 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=279446, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_130 stored as values in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 17:54:27 INFO MemoryStore:59 - ensureFreeSpace(2239) called with curMem=283542, maxMem=766075207 -2016-06-13 17:54:27 INFO MemoryStore:59 - Block broadcast_130_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 17:54:27 INFO BlockManagerInfo:59 - Added broadcast_130_piece0 in memory on localhost:62418 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:54:27 INFO SparkContext:59 - Created broadcast 130 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 90 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:54:27 INFO TaskSchedulerImpl:59 - Adding task set 90.0 with 4 tasks -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 90.0 (TID 299, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:54:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 90.0 (TID 300, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 90.0 (TID 301, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 90.0 (TID 302, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:28 INFO Executor:59 - Running task 0.0 in stage 90.0 (TID 299) -2016-06-13 17:54:28 INFO Executor:59 - Running task 3.0 in stage 90.0 (TID 302) -2016-06-13 17:54:28 INFO Executor:59 - Running task 1.0 in stage 90.0 (TID 300) -2016-06-13 17:54:28 INFO Executor:59 - Running task 2.0 in stage 90.0 (TID 301) -2016-06-13 17:54:28 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:54:28 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:54:28 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:54:28 INFO Executor:59 - Finished task 1.0 in stage 90.0 (TID 300). 2762 bytes result sent to driver -2016-06-13 17:54:28 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:54:28 INFO Executor:59 - Finished task 2.0 in stage 90.0 (TID 301). 2762 bytes result sent to driver -2016-06-13 17:54:28 INFO Executor:59 - Finished task 0.0 in stage 90.0 (TID 299). 2762 bytes result sent to driver -2016-06-13 17:54:28 INFO Executor:59 - Finished task 3.0 in stage 90.0 (TID 302). 2762 bytes result sent to driver -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 90.0 (TID 301) in 4 ms on localhost (1/4) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 90.0 (TID 300) in 6 ms on localhost (2/4) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 90.0 (TID 299) in 7 ms on localhost (3/4) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 90.0 (TID 302) in 5 ms on localhost (4/4) -2016-06-13 17:54:28 INFO DAGScheduler:59 - ResultStage 90 (aggregate at GaussianMixture.scala:201) finished in 0.007 s -2016-06-13 17:54:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 90.0, whose tasks have all completed, from pool -2016-06-13 17:54:28 INFO DAGScheduler:59 - Job 90 finished: aggregate at GaussianMixture.scala:201, took 0.015902 s -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=285781, maxMem=766075207 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_131 stored as values in memory (estimated size 712.0 B, free 730.3 MB) -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=286493, maxMem=766075207 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_131_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.3 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Added broadcast_131_piece0 in memory on localhost:62418 (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO SparkContext:59 - Created broadcast 131 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:28 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:28 INFO DAGScheduler:59 - Got job 91 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:28 INFO DAGScheduler:59 - Final stage: ResultStage 91(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:28 INFO DAGScheduler:59 - Submitting ResultStage 91 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=287425, maxMem=766075207 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_132 stored as values in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(2239) called with curMem=291521, maxMem=766075207 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_132_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Added broadcast_132_piece0 in memory on localhost:62418 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO SparkContext:59 - Created broadcast 132 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 91 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:54:28 INFO TaskSchedulerImpl:59 - Adding task set 91.0 with 4 tasks -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 91.0 (TID 303, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 91.0 (TID 304, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 91.0 (TID 305, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 91.0 (TID 306, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:28 INFO Executor:59 - Running task 0.0 in stage 91.0 (TID 303) -2016-06-13 17:54:28 INFO Executor:59 - Running task 2.0 in stage 91.0 (TID 305) -2016-06-13 17:54:28 INFO Executor:59 - Running task 3.0 in stage 91.0 (TID 306) -2016-06-13 17:54:28 INFO Executor:59 - Running task 1.0 in stage 91.0 (TID 304) -2016-06-13 17:54:28 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:54:28 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:54:28 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:54:28 INFO Executor:59 - Finished task 0.0 in stage 91.0 (TID 303). 2762 bytes result sent to driver -2016-06-13 17:54:28 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:54:28 INFO Executor:59 - Finished task 1.0 in stage 91.0 (TID 304). 2762 bytes result sent to driver -2016-06-13 17:54:28 INFO Executor:59 - Finished task 3.0 in stage 91.0 (TID 306). 2762 bytes result sent to driver -2016-06-13 17:54:28 INFO Executor:59 - Finished task 2.0 in stage 91.0 (TID 305). 2762 bytes result sent to driver -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 91.0 (TID 303) in 5 ms on localhost (1/4) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 91.0 (TID 304) in 6 ms on localhost (2/4) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 91.0 (TID 306) in 5 ms on localhost (3/4) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 91.0 (TID 305) in 6 ms on localhost (4/4) -2016-06-13 17:54:28 INFO DAGScheduler:59 - ResultStage 91 (aggregate at GaussianMixture.scala:201) finished in 0.007 s -2016-06-13 17:54:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 91.0, whose tasks have all completed, from pool -2016-06-13 17:54:28 INFO DAGScheduler:59 - Job 91 finished: aggregate at GaussianMixture.scala:201, took 0.015071 s -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=293760, maxMem=766075207 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_133 stored as values in memory (estimated size 712.0 B, free 730.3 MB) -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(934) called with curMem=294472, maxMem=766075207 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_133_piece0 stored as bytes in memory (estimated size 934.0 B, free 730.3 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Added broadcast_133_piece0 in memory on localhost:62418 (size: 934.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO SparkContext:59 - Created broadcast 133 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:28 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:28 INFO DAGScheduler:59 - Got job 92 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:28 INFO DAGScheduler:59 - Final stage: ResultStage 92(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:28 INFO DAGScheduler:59 - Submitting ResultStage 92 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=295406, maxMem=766075207 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_134 stored as values in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 72 -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(2241) called with curMem=299502, maxMem=766075207 -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_134_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 17:54:28 INFO BlockManager:59 - Removing RDD 25 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Added broadcast_134_piece0 in memory on localhost:62418 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO SparkContext:59 - Created broadcast 134 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 92 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:54:28 INFO TaskSchedulerImpl:59 - Adding task set 92.0 with 4 tasks -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 92.0 (TID 307, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 92.0 (TID 308, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned RDD 25 -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 92.0 (TID 309, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 92.0 (TID 310, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:28 INFO Executor:59 - Running task 0.0 in stage 92.0 (TID 307) -2016-06-13 17:54:28 INFO Executor:59 - Running task 3.0 in stage 92.0 (TID 310) -2016-06-13 17:54:28 INFO Executor:59 - Running task 2.0 in stage 92.0 (TID 309) -2016-06-13 17:54:28 INFO Executor:59 - Running task 1.0 in stage 92.0 (TID 308) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:54:28 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:54:28 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:54:28 INFO Executor:59 - Finished task 1.0 in stage 92.0 (TID 308). 2762 bytes result sent to driver -2016-06-13 17:54:28 INFO Executor:59 - Finished task 2.0 in stage 92.0 (TID 309). 2762 bytes result sent to driver -2016-06-13 17:54:28 INFO Executor:59 - Finished task 3.0 in stage 92.0 (TID 310). 2762 bytes result sent to driver -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-06-13 17:54:28 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 92.0 (TID 309) in 6 ms on localhost (1/4) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 92.0 (TID 308) in 7 ms on localhost (2/4) -2016-06-13 17:54:28 INFO Executor:59 - Finished task 0.0 in stage 92.0 (TID 307). 2762 bytes result sent to driver -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 92.0 (TID 310) in 8 ms on localhost (3/4) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 92.0 (TID 307) in 10 ms on localhost (4/4) -2016-06-13 17:54:28 INFO DAGScheduler:59 - ResultStage 92 (aggregate at GaussianMixture.scala:201) finished in 0.010 s -2016-06-13 17:54:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 92.0, whose tasks have all completed, from pool -2016-06-13 17:54:28 INFO DAGScheduler:59 - Job 92 finished: aggregate at GaussianMixture.scala:201, took 0.022126 s -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:62418 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=289377, maxMem=766075207 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_135 stored as values in memory (estimated size 712.0 B, free 730.3 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=287895, maxMem=766075207 -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_135_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.3 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Added broadcast_135_piece0 in memory on localhost:62418 (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO SparkContext:59 - Created broadcast 135 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:62418 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO DAGScheduler:59 - Got job 93 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:28 INFO DAGScheduler:59 - Final stage: ResultStage 93(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-06-13 17:54:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:28 INFO DAGScheduler:59 - Submitting ResultStage 93 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:62418 in memory (size: 891.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=279658, maxMem=766075207 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_136 stored as values in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(2241) called with curMem=283754, maxMem=766075207 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_136_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Added broadcast_136_piece0 in memory on localhost:62418 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO SparkContext:59 - Created broadcast 136 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_132_piece0 on localhost:62418 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 93 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:54:28 INFO TaskSchedulerImpl:59 - Adding task set 93.0 with 4 tasks -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 92 -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 93.0 (TID 311, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 93.0 (TID 312, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 93.0 (TID 313, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 93.0 (TID 314, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_131_piece0 on localhost:62418 in memory (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO Executor:59 - Running task 2.0 in stage 93.0 (TID 313) -2016-06-13 17:54:28 INFO Executor:59 - Running task 1.0 in stage 93.0 (TID 312) -2016-06-13 17:54:28 INFO Executor:59 - Running task 3.0 in stage 93.0 (TID 314) -2016-06-13 17:54:28 INFO Executor:59 - Running task 0.0 in stage 93.0 (TID 311) -2016-06-13 17:54:28 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:54:28 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:54:28 INFO Executor:59 - Finished task 2.0 in stage 93.0 (TID 313). 2762 bytes result sent to driver -2016-06-13 17:54:28 INFO Executor:59 - Finished task 0.0 in stage 93.0 (TID 311). 2762 bytes result sent to driver -2016-06-13 17:54:28 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_130_piece0 on localhost:62418 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 93.0 (TID 313) in 6 ms on localhost (1/4) -2016-06-13 17:54:28 INFO Executor:59 - Finished task 3.0 in stage 93.0 (TID 314). 2762 bytes result sent to driver -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 93.0 (TID 311) in 8 ms on localhost (2/4) -2016-06-13 17:54:28 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 93.0 (TID 314) in 9 ms on localhost (3/4) -2016-06-13 17:54:28 INFO Executor:59 - Finished task 1.0 in stage 93.0 (TID 312). 2762 bytes result sent to driver -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 91 -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 93.0 (TID 312) in 12 ms on localhost (4/4) -2016-06-13 17:54:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 93.0, whose tasks have all completed, from pool -2016-06-13 17:54:28 INFO DAGScheduler:59 - ResultStage 93 (aggregate at GaussianMixture.scala:201) finished in 0.012 s -2016-06-13 17:54:28 INFO DAGScheduler:59 - Job 93 finished: aggregate at GaussianMixture.scala:201, took 0.024990 s -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_129_piece0 on localhost:62418 in memory (size: 934.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=270747, maxMem=766075207 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_137 stored as values in memory (estimated size 712.0 B, free 730.3 MB) -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(936) called with curMem=268509, maxMem=766075207 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_137_piece0 stored as bytes in memory (estimated size 936.0 B, free 730.3 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_128_piece0 on localhost:62418 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Added broadcast_137_piece0 in memory on localhost:62418 (size: 936.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 90 -2016-06-13 17:54:28 INFO SparkContext:59 - Created broadcast 137 from broadcast at GaussianMixture.scala:198 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_127_piece0 on localhost:62418 in memory (size: 937.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 17:54:28 INFO DAGScheduler:59 - Got job 94 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 17:54:28 INFO DAGScheduler:59 - Final stage: ResultStage 94(aggregate at GaussianMixture.scala:201) -2016-06-13 17:54:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_126_piece0 on localhost:62418 in memory (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 89 -2016-06-13 17:54:28 INFO DAGScheduler:59 - Submitting ResultStage 94 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=236594, maxMem=766075207 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_138 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(2242) called with curMem=237665, maxMem=766075207 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_138_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_125_piece0 on localhost:62418 in memory (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 88 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Added broadcast_138_piece0 in memory on localhost:62418 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO SparkContext:59 - Created broadcast 138 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 94 (MapPartitionsRDD[35] at map at GaussianMixture.scala:168) -2016-06-13 17:54:28 INFO TaskSchedulerImpl:59 - Adding task set 94.0 with 4 tasks -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 94.0 (TID 315, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 94.0 (TID 316, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_124_piece0 on localhost:62418 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 94.0 (TID 317, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 87 -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 94.0 (TID 318, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 17:54:28 INFO Executor:59 - Running task 0.0 in stage 94.0 (TID 315) -2016-06-13 17:54:28 INFO Executor:59 - Running task 2.0 in stage 94.0 (TID 317) -2016-06-13 17:54:28 INFO Executor:59 - Running task 3.0 in stage 94.0 (TID 318) -2016-06-13 17:54:28 INFO Executor:59 - Running task 1.0 in stage 94.0 (TID 316) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_123_piece0 on localhost:62418 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 17:54:28 INFO Executor:59 - Finished task 0.0 in stage 94.0 (TID 315). 2762 bytes result sent to driver -2016-06-13 17:54:28 INFO BlockManager:59 - Found block rdd_35_3 locally -2016-06-13 17:54:28 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 17:54:28 INFO BlockManager:59 - Found block rdd_35_2 locally -2016-06-13 17:54:28 INFO Executor:59 - Finished task 3.0 in stage 94.0 (TID 318). 2762 bytes result sent to driver -2016-06-13 17:54:28 INFO Executor:59 - Finished task 2.0 in stage 94.0 (TID 317). 2762 bytes result sent to driver -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 86 -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 94.0 (TID 315) in 9 ms on localhost (1/4) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 94.0 (TID 317) in 9 ms on localhost (2/4) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 94.0 (TID 318) in 9 ms on localhost (3/4) -2016-06-13 17:54:28 INFO Executor:59 - Finished task 1.0 in stage 94.0 (TID 316). 2762 bytes result sent to driver -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_122_piece0 on localhost:62418 in memory (size: 932.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 94.0 (TID 316) in 14 ms on localhost (4/4) -2016-06-13 17:54:28 INFO DAGScheduler:59 - ResultStage 94 (aggregate at GaussianMixture.scala:201) finished in 0.015 s -2016-06-13 17:54:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 94.0, whose tasks have all completed, from pool -2016-06-13 17:54:28 INFO DAGScheduler:59 - Job 94 finished: aggregate at GaussianMixture.scala:201, took 0.025654 s -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(656) called with curMem=221762, maxMem=766075207 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_121_piece0 on localhost:62418 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_139 stored as values in memory (estimated size 656.0 B, free 730.4 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 85 -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(833) called with curMem=221706, maxMem=766075207 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_139_piece0 stored as bytes in memory (estimated size 833.0 B, free 730.4 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_120_piece0 on localhost:62418 in memory (size: 934.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Added broadcast_139_piece0 in memory on localhost:62418 (size: 833.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO SparkContext:59 - Created broadcast 139 from broadcast at GaussianMixtureModel.scala:100 -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(56) called with curMem=219326, maxMem=766075207 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_140 stored as values in memory (estimated size 56.0 B, free 730.4 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_119_piece0 on localhost:62418 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 84 -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(65) called with curMem=215094, maxMem=766075207 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_140_piece0 stored as bytes in memory (estimated size 65.0 B, free 730.4 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Added broadcast_140_piece0 in memory on localhost:62418 (size: 65.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO SparkContext:59 - Created broadcast 140 from broadcast at GaussianMixtureModel.scala:101 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_118_piece0 on localhost:62418 in memory (size: 880.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_117_piece0 on localhost:62418 in memory (size: 1348.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 83 -2016-06-13 17:54:28 INFO BlockManager:59 - Removing RDD 33 -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned RDD 33 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_116_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 82 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_115_piece0 on localhost:62418 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_114_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 81 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_113_piece0 on localhost:62418 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_112_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 80 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_111_piece0 on localhost:62418 in memory (size: 882.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO SparkContext:59 - Starting job: collect at GaussianMixtureFromSparkSuite.scala:201 -2016-06-13 17:54:28 INFO DAGScheduler:59 - Got job 95 (collect at GaussianMixtureFromSparkSuite.scala:201) with 4 output partitions -2016-06-13 17:54:28 INFO DAGScheduler:59 - Final stage: ResultStage 95(collect at GaussianMixtureFromSparkSuite.scala:201) -2016-06-13 17:54:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 17:54:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_110_piece0 on localhost:62418 in memory (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO DAGScheduler:59 - Submitting ResultStage 95 (ZippedPartitionsRDD2[39] at zip at GaussianMixtureFromSparkSuite.scala:201), which has no missing parents -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 79 -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(4712) called with curMem=155625, maxMem=766075207 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_141 stored as values in memory (estimated size 4.6 KB, free 730.4 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_109_piece0 on localhost:62418 in memory (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO MemoryStore:59 - ensureFreeSpace(2639) called with curMem=159144, maxMem=766075207 -2016-06-13 17:54:28 INFO MemoryStore:59 - Block broadcast_141_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 78 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Added broadcast_141_piece0 in memory on localhost:62418 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO SparkContext:59 - Created broadcast 141 from broadcast at DAGScheduler.scala:861 -2016-06-13 17:54:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 95 (ZippedPartitionsRDD2[39] at zip at GaussianMixtureFromSparkSuite.scala:201) -2016-06-13 17:54:28 INFO TaskSchedulerImpl:59 - Adding task set 95.0 with 4 tasks -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_108_piece0 on localhost:62418 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 77 -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 95.0 (TID 319, localhost, PROCESS_LOCAL, 2432 bytes) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 95.0 (TID 320, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 95.0 (TID 321, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_107_piece0 on localhost:62418 in memory (size: 1250.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 95.0 (TID 322, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 76 -2016-06-13 17:54:28 INFO Executor:59 - Running task 2.0 in stage 95.0 (TID 321) -2016-06-13 17:54:28 INFO Executor:59 - Running task 1.0 in stage 95.0 (TID 320) -2016-06-13 17:54:28 INFO Executor:59 - Running task 0.0 in stage 95.0 (TID 319) -2016-06-13 17:54:28 INFO Executor:59 - Running task 3.0 in stage 95.0 (TID 322) -2016-06-13 17:54:28 INFO BlockManager:59 - Removing RDD 29 -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned RDD 29 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_106_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO Executor:59 - Finished task 2.0 in stage 95.0 (TID 321). 1264 bytes result sent to driver -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 95.0 (TID 321) in 13 ms on localhost (1/4) -2016-06-13 17:54:28 INFO Executor:59 - Finished task 1.0 in stage 95.0 (TID 320). 1269 bytes result sent to driver -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 95.0 (TID 320) in 15 ms on localhost (2/4) -2016-06-13 17:54:28 INFO Executor:59 - Finished task 0.0 in stage 95.0 (TID 319). 1229 bytes result sent to driver -2016-06-13 17:54:28 INFO Executor:59 - Finished task 3.0 in stage 95.0 (TID 322). 1264 bytes result sent to driver -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 95.0 (TID 319) in 19 ms on localhost (3/4) -2016-06-13 17:54:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 95.0 (TID 322) in 16 ms on localhost (4/4) -2016-06-13 17:54:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 95.0, whose tasks have all completed, from pool -2016-06-13 17:54:28 INFO DAGScheduler:59 - ResultStage 95 (collect at GaussianMixtureFromSparkSuite.scala:201) finished in 0.020 s -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 75 -2016-06-13 17:54:28 INFO DAGScheduler:59 - Job 95 finished: collect at GaussianMixtureFromSparkSuite.scala:201, took 0.028680 s -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_105_piece0 on localhost:62418 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_104_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 74 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_103_piece0 on localhost:62418 in memory (size: 900.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_102_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 73 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_101_piece0 on localhost:62418 in memory (size: 895.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_100_piece0 on localhost:62418 in memory (size: 7.7 KB, free: 730.5 MB) -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_99_piece0 on localhost:62418 in memory (size: 1193.0 B, free: 730.5 MB) -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 71 -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 17:54:28 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_98_piece0 on localhost:62418 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 70 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_97_piece0 on localhost:62418 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 69 -2016-06-13 17:54:28 INFO BlockManager:59 - Removing RDD 27 -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned RDD 27 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_96_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_95_piece0 on localhost:62418 in memory (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_94_piece0 on localhost:62418 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_93_piece0 on localhost:62418 in memory (size: 900.0 B, free: 730.6 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_92_piece0 on localhost:62418 in memory (size: 2.2 KB, free: 730.6 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_91_piece0 on localhost:62418 in memory (size: 899.0 B, free: 730.6 MB) -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_90_piece0 on localhost:62418 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_89_piece0 on localhost:62418 in memory (size: 1193.0 B, free: 730.6 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_88_piece0 on localhost:62418 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:54:28 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-06-13 17:54:28 INFO BlockManagerInfo:59 - Removed broadcast_87_piece0 on localhost:62418 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 17:54:28 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 17:54:28 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 17:54:28 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 17:54:28 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 17:54:28 INFO BlockManager:59 - BlockManager stopped -2016-06-13 17:54:28 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 17:54:28 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 17:54:28 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 17:54:28 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 17:54:28 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 17:54:28 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 17:54:28 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-17bd487a-5160-49ec-b4c3-75a180c56fea -2016-06-13 18:46:13 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 18:46:14 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 18:46:14 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 18:46:14 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 18:46:14 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 18:46:15 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 18:46:15 INFO Remoting:74 - Starting remoting -2016-06-13 18:46:15 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:65018] -2016-06-13 18:46:15 INFO Utils:59 - Successfully started service 'sparkDriver' on port 65018. -2016-06-13 18:46:15 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 18:46:15 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 18:46:15 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-e4a5b4d2-d1a9-4c32-9933-5d93d6811e1a -2016-06-13 18:46:15 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 18:46:15 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-fd110d63-a09a-467f-8c71-78936f438787\httpd-414880b0-efa8-42b7-a550-32eabd0dc24f -2016-06-13 18:46:15 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 18:46:15 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 18:46:15 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:65019 -2016-06-13 18:46:15 INFO Utils:59 - Successfully started service 'HTTP file server' on port 65019. -2016-06-13 18:46:15 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 18:46:15 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 18:46:15 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 18:46:15 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 18:46:15 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 18:46:15 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 18:46:15 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 18:46:16 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 65038. -2016-06-13 18:46:16 INFO NettyBlockTransferService:59 - Server created on 65038 -2016-06-13 18:46:16 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 18:46:16 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:65038 with 730.6 MB RAM, BlockManagerId(driver, localhost, 65038) -2016-06-13 18:46:16 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 18:46:16 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 18:46:16 INFO DAGScheduler:59 - Got job 0 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 18:46:16 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GaussianMixture.scala:171) -2016-06-13 18:46:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:46:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:46:16 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:46:17 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=0, maxMem=766075207 -2016-06-13 18:46:17 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 18:46:17 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1992, maxMem=766075207 -2016-06-13 18:46:17 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.6 MB) -2016-06-13 18:46:17 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:65038 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 18:46:17 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:46:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:46:17 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 18:46:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:46:17 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 18:46:17 INFO CacheManager:59 - Partition rdd_1_0 not found, computing it -2016-06-13 18:46:17 INFO MemoryStore:59 - ensureFreeSpace(224) called with curMem=3242, maxMem=766075207 -2016-06-13 18:46:17 INFO MemoryStore:59 - Block rdd_1_0 stored as values in memory (estimated size 224.0 B, free 730.6 MB) -2016-06-13 18:46:17 INFO BlockManagerInfo:59 - Added rdd_1_0 in memory on localhost:65038 (size: 224.0 B, free: 730.6 MB) -2016-06-13 18:46:17 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1688 bytes result sent to driver -2016-06-13 18:46:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 332 ms on localhost (1/1) -2016-06-13 18:46:17 INFO DAGScheduler:59 - ResultStage 0 (first at GaussianMixture.scala:171) finished in 0.343 s -2016-06-13 18:46:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 18:46:17 INFO DAGScheduler:59 - Job 0 finished: first at GaussianMixture.scala:171, took 0.540907 s -2016-06-13 18:46:17 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 18:46:17 INFO DAGScheduler:59 - Got job 1 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 18:46:17 INFO DAGScheduler:59 - Final stage: ResultStage 1(takeSample at GaussianMixture.scala:184) -2016-06-13 18:46:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:46:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:46:17 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:46:17 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=3466, maxMem=766075207 -2016-06-13 18:46:17 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 18:46:17 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=5298, maxMem=766075207 -2016-06-13 18:46:17 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.6 MB) -2016-06-13 18:46:17 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:65038 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 18:46:17 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:46:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:46:17 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 4 tasks -2016-06-13 18:46:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:46:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:17 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 18:46:17 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 18:46:17 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 3) -2016-06-13 18:46:17 INFO Executor:59 - Running task 3.0 in stage 1.0 (TID 4) -2016-06-13 18:46:17 INFO CacheManager:59 - Partition rdd_1_1 not found, computing it -2016-06-13 18:46:17 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=6493, maxMem=766075207 -2016-06-13 18:46:17 INFO CacheManager:59 - Partition rdd_1_3 not found, computing it -2016-06-13 18:46:17 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:46:17 INFO MemoryStore:59 - Block rdd_1_1 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 18:46:17 INFO BlockManagerInfo:59 - Added rdd_1_1 in memory on localhost:65038 (size: 288.0 B, free: 730.6 MB) -2016-06-13 18:46:17 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 1567 bytes result sent to driver -2016-06-13 18:46:17 INFO CacheManager:59 - Partition rdd_1_2 not found, computing it -2016-06-13 18:46:17 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=6781, maxMem=766075207 -2016-06-13 18:46:17 INFO MemoryStore:59 - Block rdd_1_3 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 18:46:17 INFO BlockManagerInfo:59 - Added rdd_1_3 in memory on localhost:65038 (size: 288.0 B, free: 730.6 MB) -2016-06-13 18:46:17 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=7069, maxMem=766075207 -2016-06-13 18:46:17 INFO MemoryStore:59 - Block rdd_1_2 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 18:46:17 INFO Executor:59 - Finished task 3.0 in stage 1.0 (TID 4). 1567 bytes result sent to driver -2016-06-13 18:46:17 INFO BlockManagerInfo:59 - Added rdd_1_2 in memory on localhost:65038 (size: 288.0 B, free: 730.6 MB) -2016-06-13 18:46:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 39 ms on localhost (1/4) -2016-06-13 18:46:17 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 18:46:17 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 3). 1567 bytes result sent to driver -2016-06-13 18:46:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 1.0 (TID 4) in 43 ms on localhost (2/4) -2016-06-13 18:46:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 3) in 45 ms on localhost (3/4) -2016-06-13 18:46:17 INFO DAGScheduler:59 - ResultStage 1 (takeSample at GaussianMixture.scala:184) finished in 0.055 s -2016-06-13 18:46:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 55 ms on localhost (4/4) -2016-06-13 18:46:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 18:46:17 INFO DAGScheduler:59 - Job 1 finished: takeSample at GaussianMixture.scala:184, took 0.096744 s -2016-06-13 18:46:17 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 18:46:17 INFO DAGScheduler:59 - Got job 2 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 18:46:17 INFO DAGScheduler:59 - Final stage: ResultStage 2(takeSample at GaussianMixture.scala:184) -2016-06-13 18:46:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:46:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:46:17 INFO DAGScheduler:59 - Submitting ResultStage 2 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 18:46:17 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=7357, maxMem=766075207 -2016-06-13 18:46:17 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 18.8 KB, free 730.6 MB) -2016-06-13 18:46:17 INFO MemoryStore:59 - ensureFreeSpace(7882) called with curMem=26581, maxMem=766075207 -2016-06-13 18:46:17 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.6 MB) -2016-06-13 18:46:17 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:65038 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 18:46:17 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:46:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 2 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184) -2016-06-13 18:46:17 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 4 tasks -2016-06-13 18:46:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 18:46:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 18:46:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 18:46:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 2.0 (TID 8, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 18:46:17 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 5) -2016-06-13 18:46:17 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 6) -2016-06-13 18:46:17 INFO Executor:59 - Running task 3.0 in stage 2.0 (TID 8) -2016-06-13 18:46:17 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 7) -2016-06-13 18:46:17 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:46:17 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:46:17 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:46:17 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:46:17 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 6). 2344 bytes result sent to driver -2016-06-13 18:46:17 INFO Executor:59 - Finished task 3.0 in stage 2.0 (TID 8). 2339 bytes result sent to driver -2016-06-13 18:46:17 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 5). 2247 bytes result sent to driver -2016-06-13 18:46:17 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 7). 2303 bytes result sent to driver -2016-06-13 18:46:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 6) in 27 ms on localhost (1/4) -2016-06-13 18:46:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 2.0 (TID 8) in 25 ms on localhost (2/4) -2016-06-13 18:46:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 5) in 31 ms on localhost (3/4) -2016-06-13 18:46:17 INFO DAGScheduler:59 - ResultStage 2 (takeSample at GaussianMixture.scala:184) finished in 0.031 s -2016-06-13 18:46:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 7) in 28 ms on localhost (4/4) -2016-06-13 18:46:17 INFO DAGScheduler:59 - Job 2 finished: takeSample at GaussianMixture.scala:184, took 0.048435 s -2016-06-13 18:46:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 18:46:17 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 18:46:17 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 18:46:17 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:65038 in memory (size: 1195.0 B, free: 730.6 MB) -2016-06-13 18:46:17 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:65038 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 18:46:17 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:65038 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 18:46:18 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 18:46:18 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=1088, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 712.0 B, free 730.6 MB) -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(937) called with curMem=1800, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 937.0 B, free 730.6 MB) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:65038 (size: 937.0 B, free: 730.6 MB) -2016-06-13 18:46:18 INFO SparkContext:59 - Created broadcast 3 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:46:18 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:46:18 INFO DAGScheduler:59 - Got job 3 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:46:18 INFO DAGScheduler:59 - Final stage: ResultStage 3(aggregate at GaussianMixture.scala:201) -2016-06-13 18:46:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:46:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:46:18 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=2737, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.0 KB, free 730.6 MB) -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(2237) called with curMem=6833, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.6 MB) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:65038 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:18 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:46:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 3 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:46:18 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 4 tasks -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 11, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 3.0 (TID 12, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:18 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 10) -2016-06-13 18:46:18 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 11) -2016-06-13 18:46:18 INFO Executor:59 - Running task 3.0 in stage 3.0 (TID 12) -2016-06-13 18:46:18 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 9) -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:46:18 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 9). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 11). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO Executor:59 - Finished task 3.0 in stage 3.0 (TID 12). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 10). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 9) in 33 ms on localhost (1/4) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 11) in 32 ms on localhost (2/4) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 3.0 (TID 12) in 32 ms on localhost (3/4) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 10) in 35 ms on localhost (4/4) -2016-06-13 18:46:18 INFO DAGScheduler:59 - ResultStage 3 (aggregate at GaussianMixture.scala:201) finished in 0.037 s -2016-06-13 18:46:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 18:46:18 INFO DAGScheduler:59 - Job 3 finished: aggregate at GaussianMixture.scala:201, took 0.053670 s -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=9070, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 712.0 B, free 730.6 MB) -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(934) called with curMem=9782, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 934.0 B, free 730.6 MB) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:65038 (size: 934.0 B, free: 730.6 MB) -2016-06-13 18:46:18 INFO SparkContext:59 - Created broadcast 5 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:46:18 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:46:18 INFO DAGScheduler:59 - Got job 4 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:46:18 INFO DAGScheduler:59 - Final stage: ResultStage 4(aggregate at GaussianMixture.scala:201) -2016-06-13 18:46:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:46:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:46:18 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=10716, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 4.0 KB, free 730.6 MB) -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=14812, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.6 MB) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:65038 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:18 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:46:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:46:18 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 4 tasks -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 14, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 15, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 4.0 (TID 16, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:18 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 13) -2016-06-13 18:46:18 INFO Executor:59 - Running task 3.0 in stage 4.0 (TID 16) -2016-06-13 18:46:18 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 15) -2016-06-13 18:46:18 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 14) -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:46:18 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 13). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO Executor:59 - Finished task 3.0 in stage 4.0 (TID 16). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 13) in 22 ms on localhost (1/4) -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 4.0 (TID 16) in 20 ms on localhost (2/4) -2016-06-13 18:46:18 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 15). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:46:18 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 14). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 15) in 29 ms on localhost (3/4) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 14) in 34 ms on localhost (4/4) -2016-06-13 18:46:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 18:46:18 INFO DAGScheduler:59 - ResultStage 4 (aggregate at GaussianMixture.scala:201) finished in 0.037 s -2016-06-13 18:46:18 INFO DAGScheduler:59 - Job 4 finished: aggregate at GaussianMixture.scala:201, took 0.051477 s -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=17050, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 712.0 B, free 730.6 MB) -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=17762, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.6 MB) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:65038 (size: 932.0 B, free: 730.6 MB) -2016-06-13 18:46:18 INFO SparkContext:59 - Created broadcast 7 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:46:18 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:46:18 INFO DAGScheduler:59 - Got job 5 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:46:18 INFO DAGScheduler:59 - Final stage: ResultStage 5(aggregate at GaussianMixture.scala:201) -2016-06-13 18:46:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:46:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:46:18 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=18694, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.0 KB, free 730.6 MB) -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(2237) called with curMem=22790, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.6 MB) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:65038 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:18 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:46:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:46:18 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 4 tasks -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 17, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 18, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 19, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 5.0 (TID 20, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:18 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 17) -2016-06-13 18:46:18 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 19) -2016-06-13 18:46:18 INFO Executor:59 - Running task 3.0 in stage 5.0 (TID 20) -2016-06-13 18:46:18 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 18) -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:46:18 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 18). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 19). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO Executor:59 - Finished task 3.0 in stage 5.0 (TID 20). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 18) in 18 ms on localhost (1/4) -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 19) in 18 ms on localhost (2/4) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 5.0 (TID 20) in 17 ms on localhost (3/4) -2016-06-13 18:46:18 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 17). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 17) in 24 ms on localhost (4/4) -2016-06-13 18:46:18 INFO DAGScheduler:59 - ResultStage 5 (aggregate at GaussianMixture.scala:201) finished in 0.024 s -2016-06-13 18:46:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 18:46:18 INFO DAGScheduler:59 - Job 5 finished: aggregate at GaussianMixture.scala:201, took 0.042299 s -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=25027, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 712.0 B, free 730.6 MB) -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=25739, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.6 MB) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:65038 (size: 932.0 B, free: 730.6 MB) -2016-06-13 18:46:18 INFO SparkContext:59 - Created broadcast 9 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:46:18 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:46:18 INFO DAGScheduler:59 - Got job 6 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:46:18 INFO DAGScheduler:59 - Final stage: ResultStage 6(aggregate at GaussianMixture.scala:201) -2016-06-13 18:46:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:46:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:46:18 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=26671, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.0 KB, free 730.6 MB) -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=30767, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.6 MB) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:65038 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:18 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:46:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:46:18 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 4 tasks -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 21, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 22, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 23, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 6.0 (TID 24, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:18 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 22) -2016-06-13 18:46:18 INFO Executor:59 - Running task 3.0 in stage 6.0 (TID 24) -2016-06-13 18:46:18 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 23) -2016-06-13 18:46:18 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 21) -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:46:18 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 22). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO Executor:59 - Finished task 3.0 in stage 6.0 (TID 24). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 21). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 23). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 22) in 19 ms on localhost (1/4) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 21) in 21 ms on localhost (2/4) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 6.0 (TID 24) in 18 ms on localhost (3/4) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 23) in 20 ms on localhost (4/4) -2016-06-13 18:46:18 INFO DAGScheduler:59 - ResultStage 6 (aggregate at GaussianMixture.scala:201) finished in 0.022 s -2016-06-13 18:46:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 18:46:18 INFO DAGScheduler:59 - Job 6 finished: aggregate at GaussianMixture.scala:201, took 0.037309 s -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=33005, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 712.0 B, free 730.6 MB) -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=33717, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.6 MB) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:65038 (size: 932.0 B, free: 730.6 MB) -2016-06-13 18:46:18 INFO SparkContext:59 - Created broadcast 11 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:46:18 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:46:18 INFO DAGScheduler:59 - Got job 7 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:46:18 INFO DAGScheduler:59 - Final stage: ResultStage 7(aggregate at GaussianMixture.scala:201) -2016-06-13 18:46:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:46:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:46:18 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=34649, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(2240) called with curMem=38745, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:65038 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:18 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:46:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 7 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:46:18 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 4 tasks -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 25, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 26, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 7.0 (TID 27, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 7.0 (TID 28, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:18 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 25) -2016-06-13 18:46:18 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 26) -2016-06-13 18:46:18 INFO Executor:59 - Running task 3.0 in stage 7.0 (TID 28) -2016-06-13 18:46:18 INFO Executor:59 - Running task 2.0 in stage 7.0 (TID 27) -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:46:18 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 26). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO Executor:59 - Finished task 2.0 in stage 7.0 (TID 27). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO Executor:59 - Finished task 3.0 in stage 7.0 (TID 28). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 25). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 26) in 14 ms on localhost (1/4) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 7.0 (TID 28) in 13 ms on localhost (2/4) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 25) in 17 ms on localhost (3/4) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 7.0 (TID 27) in 15 ms on localhost (4/4) -2016-06-13 18:46:18 INFO DAGScheduler:59 - ResultStage 7 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 18:46:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 18:46:18 INFO DAGScheduler:59 - Job 7 finished: aggregate at GaussianMixture.scala:201, took 0.029796 s -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=40985, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 712.0 B, free 730.5 MB) -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=41697, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.5 MB) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:65038 (size: 932.0 B, free: 730.6 MB) -2016-06-13 18:46:18 INFO SparkContext:59 - Created broadcast 13 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:46:18 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:46:18 INFO DAGScheduler:59 - Got job 8 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:46:18 INFO DAGScheduler:59 - Final stage: ResultStage 8(aggregate at GaussianMixture.scala:201) -2016-06-13 18:46:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:46:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:46:18 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=42629, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(2237) called with curMem=46725, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:65038 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:18 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:46:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 8 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:46:18 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 4 tasks -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 29, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 30, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 8.0 (TID 31, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 8.0 (TID 32, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:18 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 30) -2016-06-13 18:46:18 INFO Executor:59 - Running task 2.0 in stage 8.0 (TID 31) -2016-06-13 18:46:18 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 29) -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:46:18 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 30). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:46:18 INFO Executor:59 - Finished task 2.0 in stage 8.0 (TID 31). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 30) in 13 ms on localhost (1/4) -2016-06-13 18:46:18 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 29). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO Executor:59 - Running task 3.0 in stage 8.0 (TID 32) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 8.0 (TID 31) in 14 ms on localhost (2/4) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 29) in 17 ms on localhost (3/4) -2016-06-13 18:46:18 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:46:18 INFO Executor:59 - Finished task 3.0 in stage 8.0 (TID 32). 2762 bytes result sent to driver -2016-06-13 18:46:18 INFO DAGScheduler:59 - ResultStage 8 (aggregate at GaussianMixture.scala:201) finished in 0.025 s -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 8.0 (TID 32) in 21 ms on localhost (4/4) -2016-06-13 18:46:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 18:46:18 INFO DAGScheduler:59 - Job 8 finished: aggregate at GaussianMixture.scala:201, took 0.037644 s -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(656) called with curMem=48962, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 656.0 B, free 730.5 MB) -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(833) called with curMem=49618, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 833.0 B, free 730.5 MB) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:65038 (size: 833.0 B, free: 730.6 MB) -2016-06-13 18:46:18 INFO SparkContext:59 - Created broadcast 15 from broadcast at GaussianMixtureModel.scala:100 -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(56) called with curMem=50451, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 56.0 B, free 730.5 MB) -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(65) called with curMem=50507, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 65.0 B, free 730.5 MB) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:65038 (size: 65.0 B, free: 730.6 MB) -2016-06-13 18:46:18 INFO SparkContext:59 - Created broadcast 16 from broadcast at GaussianMixtureModel.scala:101 -2016-06-13 18:46:18 INFO SparkContext:59 - Starting job: collect at GaussianMixtureFromSparkSuite.scala:201 -2016-06-13 18:46:18 INFO DAGScheduler:59 - Got job 9 (collect at GaussianMixtureFromSparkSuite.scala:201) with 4 output partitions -2016-06-13 18:46:18 INFO DAGScheduler:59 - Final stage: ResultStage 9(collect at GaussianMixtureFromSparkSuite.scala:201) -2016-06-13 18:46:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:46:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:46:18 INFO DAGScheduler:59 - Submitting ResultStage 9 (ZippedPartitionsRDD2[5] at zip at GaussianMixtureFromSparkSuite.scala:201), which has no missing parents -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(4712) called with curMem=50572, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 18:46:18 INFO MemoryStore:59 - ensureFreeSpace(2631) called with curMem=55284, maxMem=766075207 -2016-06-13 18:46:18 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:65038 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 18:46:18 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:46:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 9 (ZippedPartitionsRDD2[5] at zip at GaussianMixtureFromSparkSuite.scala:201) -2016-06-13 18:46:18 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 4 tasks -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 33, localhost, PROCESS_LOCAL, 2432 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 34, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 9.0 (TID 35, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 9.0 (TID 36, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 18:46:18 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 33) -2016-06-13 18:46:18 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 34) -2016-06-13 18:46:18 INFO Executor:59 - Running task 3.0 in stage 9.0 (TID 36) -2016-06-13 18:46:18 INFO Executor:59 - Running task 2.0 in stage 9.0 (TID 35) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:65038 in memory (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:18 INFO Executor:59 - Finished task 3.0 in stage 9.0 (TID 36). 1264 bytes result sent to driver -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 9.0 (TID 36) in 40 ms on localhost (1/4) -2016-06-13 18:46:18 INFO Executor:59 - Finished task 2.0 in stage 9.0 (TID 35). 1264 bytes result sent to driver -2016-06-13 18:46:18 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 18:46:18 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 33). 1229 bytes result sent to driver -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 33) in 61 ms on localhost (2/4) -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 9.0 (TID 35) in 56 ms on localhost (3/4) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:65038 in memory (size: 932.0 B, free: 730.6 MB) -2016-06-13 18:46:18 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 34). 1269 bytes result sent to driver -2016-06-13 18:46:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 34) in 64 ms on localhost (4/4) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:65038 in memory (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 18:46:18 INFO DAGScheduler:59 - ResultStage 9 (collect at GaussianMixtureFromSparkSuite.scala:201) finished in 0.070 s -2016-06-13 18:46:18 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 18:46:18 INFO DAGScheduler:59 - Job 9 finished: collect at GaussianMixtureFromSparkSuite.scala:201, took 0.085797 s -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:65038 in memory (size: 932.0 B, free: 730.6 MB) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:65038 in memory (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:18 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:65038 in memory (size: 932.0 B, free: 730.6 MB) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:65038 in memory (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:18 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:65038 in memory (size: 932.0 B, free: 730.6 MB) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:65038 in memory (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:18 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:65038 in memory (size: 934.0 B, free: 730.6 MB) -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:65038 in memory (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:18 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 18:46:18 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:65038 in memory (size: 937.0 B, free: 730.6 MB) -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 18:46:18 INFO BlockManager:59 - Removing RDD 1 -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 18:46:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 18:46:18 INFO ContextCleaner:59 - Cleaned RDD 1 -2016-06-13 18:46:18 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 18:46:18 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 18:46:18 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 18:46:18 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 18:46:18 INFO BlockManager:59 - BlockManager stopped -2016-06-13 18:46:18 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 18:46:18 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 18:46:19 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 18:46:19 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 18:46:19 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 18:46:19 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 18:46:19 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-fd110d63-a09a-467f-8c71-78936f438787 -2016-06-13 18:46:34 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 18:46:35 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 18:46:35 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 18:46:35 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 18:46:35 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 18:46:36 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 18:46:36 INFO Remoting:74 - Starting remoting -2016-06-13 18:46:36 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:65055] -2016-06-13 18:46:36 INFO Utils:59 - Successfully started service 'sparkDriver' on port 65055. -2016-06-13 18:46:36 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 18:46:36 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 18:46:36 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-923a539c-8cd4-4ed3-9c4b-1d77d4fa88de -2016-06-13 18:46:36 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 18:46:36 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-772e6e8f-1e3a-465c-ab98-feeb18386da2\httpd-dd737ad3-329f-4f63-9245-59767b8fe9a1 -2016-06-13 18:46:36 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 18:46:36 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 18:46:36 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:65056 -2016-06-13 18:46:36 INFO Utils:59 - Successfully started service 'HTTP file server' on port 65056. -2016-06-13 18:46:36 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 18:46:36 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 18:46:36 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 18:46:36 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 18:46:36 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 18:46:36 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 18:46:36 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 18:46:37 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 65075. -2016-06-13 18:46:37 INFO NettyBlockTransferService:59 - Server created on 65075 -2016-06-13 18:46:37 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 18:46:37 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:65075 with 730.6 MB RAM, BlockManagerId(driver, localhost, 65075) -2016-06-13 18:46:37 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 18:46:37 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 18:46:37 INFO DAGScheduler:59 - Got job 0 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 18:46:37 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GaussianMixture.scala:171) -2016-06-13 18:46:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:46:37 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:46:37 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:46:38 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=0, maxMem=766075207 -2016-06-13 18:46:38 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 18:46:38 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1992, maxMem=766075207 -2016-06-13 18:46:38 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.6 MB) -2016-06-13 18:46:38 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:65075 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 18:46:38 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:46:38 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:46:38 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 18:46:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:46:38 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 18:46:38 INFO CacheManager:59 - Partition rdd_1_0 not found, computing it -2016-06-13 18:46:38 INFO MemoryStore:59 - ensureFreeSpace(224) called with curMem=3242, maxMem=766075207 -2016-06-13 18:46:38 INFO MemoryStore:59 - Block rdd_1_0 stored as values in memory (estimated size 224.0 B, free 730.6 MB) -2016-06-13 18:46:38 INFO BlockManagerInfo:59 - Added rdd_1_0 in memory on localhost:65075 (size: 224.0 B, free: 730.6 MB) -2016-06-13 18:46:38 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1688 bytes result sent to driver -2016-06-13 18:46:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 426 ms on localhost (1/1) -2016-06-13 18:46:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 18:46:38 INFO DAGScheduler:59 - ResultStage 0 (first at GaussianMixture.scala:171) finished in 0.439 s -2016-06-13 18:46:38 INFO DAGScheduler:59 - Job 0 finished: first at GaussianMixture.scala:171, took 0.605238 s -2016-06-13 18:46:38 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 18:46:38 INFO DAGScheduler:59 - Got job 1 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 18:46:38 INFO DAGScheduler:59 - Final stage: ResultStage 1(takeSample at GaussianMixture.scala:184) -2016-06-13 18:46:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:46:38 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:46:38 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:46:38 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=3466, maxMem=766075207 -2016-06-13 18:46:38 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 18:46:38 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=5298, maxMem=766075207 -2016-06-13 18:46:38 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.6 MB) -2016-06-13 18:46:38 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:65075 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 18:46:38 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:46:38 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:46:38 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 4 tasks -2016-06-13 18:46:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:46:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:38 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:38 INFO TaskSetManager:59 - Starting task 3.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:38 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 18:46:38 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 18:46:38 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 3) -2016-06-13 18:46:38 INFO Executor:59 - Running task 3.0 in stage 1.0 (TID 4) -2016-06-13 18:46:38 INFO CacheManager:59 - Partition rdd_1_1 not found, computing it -2016-06-13 18:46:38 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:46:38 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=6493, maxMem=766075207 -2016-06-13 18:46:38 INFO MemoryStore:59 - Block rdd_1_1 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 18:46:38 INFO BlockManagerInfo:59 - Added rdd_1_1 in memory on localhost:65075 (size: 288.0 B, free: 730.6 MB) -2016-06-13 18:46:38 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 1567 bytes result sent to driver -2016-06-13 18:46:38 INFO CacheManager:59 - Partition rdd_1_2 not found, computing it -2016-06-13 18:46:38 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=6781, maxMem=766075207 -2016-06-13 18:46:38 INFO MemoryStore:59 - Block rdd_1_2 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 18:46:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 31 ms on localhost (1/4) -2016-06-13 18:46:38 INFO BlockManagerInfo:59 - Added rdd_1_2 in memory on localhost:65075 (size: 288.0 B, free: 730.6 MB) -2016-06-13 18:46:38 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 3). 1567 bytes result sent to driver -2016-06-13 18:46:38 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 3) in 37 ms on localhost (2/4) -2016-06-13 18:46:38 INFO CacheManager:59 - Partition rdd_1_3 not found, computing it -2016-06-13 18:46:38 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=7069, maxMem=766075207 -2016-06-13 18:46:38 INFO MemoryStore:59 - Block rdd_1_3 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 18:46:38 INFO BlockManagerInfo:59 - Added rdd_1_3 in memory on localhost:65075 (size: 288.0 B, free: 730.6 MB) -2016-06-13 18:46:38 INFO Executor:59 - Finished task 3.0 in stage 1.0 (TID 4). 1567 bytes result sent to driver -2016-06-13 18:46:38 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 18:46:38 INFO TaskSetManager:59 - Finished task 3.0 in stage 1.0 (TID 4) in 45 ms on localhost (3/4) -2016-06-13 18:46:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 57 ms on localhost (4/4) -2016-06-13 18:46:38 INFO DAGScheduler:59 - ResultStage 1 (takeSample at GaussianMixture.scala:184) finished in 0.057 s -2016-06-13 18:46:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 18:46:38 INFO DAGScheduler:59 - Job 1 finished: takeSample at GaussianMixture.scala:184, took 0.080301 s -2016-06-13 18:46:38 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 18:46:38 INFO DAGScheduler:59 - Got job 2 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 18:46:38 INFO DAGScheduler:59 - Final stage: ResultStage 2(takeSample at GaussianMixture.scala:184) -2016-06-13 18:46:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:46:38 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:46:38 INFO DAGScheduler:59 - Submitting ResultStage 2 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 18:46:38 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=7357, maxMem=766075207 -2016-06-13 18:46:38 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 18.8 KB, free 730.6 MB) -2016-06-13 18:46:38 INFO MemoryStore:59 - ensureFreeSpace(7882) called with curMem=26581, maxMem=766075207 -2016-06-13 18:46:38 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.6 MB) -2016-06-13 18:46:38 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:65075 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 18:46:38 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:46:38 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 2 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184) -2016-06-13 18:46:38 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 4 tasks -2016-06-13 18:46:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 18:46:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 18:46:38 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 18:46:38 INFO TaskSetManager:59 - Starting task 3.0 in stage 2.0 (TID 8, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 18:46:38 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 5) -2016-06-13 18:46:38 INFO Executor:59 - Running task 3.0 in stage 2.0 (TID 8) -2016-06-13 18:46:38 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 6) -2016-06-13 18:46:38 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 7) -2016-06-13 18:46:38 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:46:38 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 7). 2303 bytes result sent to driver -2016-06-13 18:46:38 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:46:38 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 5). 2247 bytes result sent to driver -2016-06-13 18:46:38 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:46:38 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:46:38 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 6). 2344 bytes result sent to driver -2016-06-13 18:46:38 INFO Executor:59 - Finished task 3.0 in stage 2.0 (TID 8). 2339 bytes result sent to driver -2016-06-13 18:46:38 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 7) in 29 ms on localhost (1/4) -2016-06-13 18:46:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 5) in 33 ms on localhost (2/4) -2016-06-13 18:46:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 6) in 33 ms on localhost (3/4) -2016-06-13 18:46:38 INFO TaskSetManager:59 - Finished task 3.0 in stage 2.0 (TID 8) in 31 ms on localhost (4/4) -2016-06-13 18:46:38 INFO DAGScheduler:59 - ResultStage 2 (takeSample at GaussianMixture.scala:184) finished in 0.036 s -2016-06-13 18:46:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 18:46:38 INFO DAGScheduler:59 - Job 2 finished: takeSample at GaussianMixture.scala:184, took 0.052420 s -2016-06-13 18:46:38 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 18:46:38 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 18:46:39 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 18:46:39 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=34463, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 712.0 B, free 730.6 MB) -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(937) called with curMem=35175, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 937.0 B, free 730.6 MB) -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:65075 (size: 937.0 B, free: 730.6 MB) -2016-06-13 18:46:39 INFO SparkContext:59 - Created broadcast 3 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:46:39 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:46:39 INFO DAGScheduler:59 - Got job 3 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:46:39 INFO DAGScheduler:59 - Final stage: ResultStage 3(aggregate at GaussianMixture.scala:201) -2016-06-13 18:46:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:46:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:46:39 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=36112, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(2237) called with curMem=40208, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:65075 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:39 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:46:39 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 3 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:46:39 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 4 tasks -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 11, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 3.0 in stage 3.0 (TID 12, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:39 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 9) -2016-06-13 18:46:39 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 10) -2016-06-13 18:46:39 INFO Executor:59 - Running task 3.0 in stage 3.0 (TID 12) -2016-06-13 18:46:39 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 11) -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:46:39 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 11). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 9). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO Executor:59 - Finished task 3.0 in stage 3.0 (TID 12). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 10). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 11) in 32 ms on localhost (1/4) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 10) in 35 ms on localhost (2/4) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 9) in 37 ms on localhost (3/4) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 3.0 in stage 3.0 (TID 12) in 35 ms on localhost (4/4) -2016-06-13 18:46:39 INFO DAGScheduler:59 - ResultStage 3 (aggregate at GaussianMixture.scala:201) finished in 0.038 s -2016-06-13 18:46:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 18:46:39 INFO DAGScheduler:59 - Job 3 finished: aggregate at GaussianMixture.scala:201, took 0.055336 s -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=42445, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 712.0 B, free 730.5 MB) -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(934) called with curMem=43157, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 934.0 B, free 730.5 MB) -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:65075 (size: 934.0 B, free: 730.6 MB) -2016-06-13 18:46:39 INFO SparkContext:59 - Created broadcast 5 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:46:39 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:46:39 INFO DAGScheduler:59 - Got job 4 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:46:39 INFO DAGScheduler:59 - Final stage: ResultStage 4(aggregate at GaussianMixture.scala:201) -2016-06-13 18:46:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:46:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:46:39 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=44091, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=48187, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:65075 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:39 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:46:39 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:46:39 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 4 tasks -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 14, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 15, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 3.0 in stage 4.0 (TID 16, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:39 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 15) -2016-06-13 18:46:39 INFO Executor:59 - Running task 3.0 in stage 4.0 (TID 16) -2016-06-13 18:46:39 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 13) -2016-06-13 18:46:39 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 14) -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:46:39 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 15). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:46:39 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 14). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 15) in 17 ms on localhost (1/4) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 14) in 22 ms on localhost (2/4) -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:46:39 INFO Executor:59 - Finished task 3.0 in stage 4.0 (TID 16). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 13). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 3.0 in stage 4.0 (TID 16) in 31 ms on localhost (3/4) -2016-06-13 18:46:39 INFO DAGScheduler:59 - ResultStage 4 (aggregate at GaussianMixture.scala:201) finished in 0.036 s -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 13) in 35 ms on localhost (4/4) -2016-06-13 18:46:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 18:46:39 INFO DAGScheduler:59 - Job 4 finished: aggregate at GaussianMixture.scala:201, took 0.051824 s -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=50425, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 712.0 B, free 730.5 MB) -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=51137, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.5 MB) -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:65075 (size: 932.0 B, free: 730.6 MB) -2016-06-13 18:46:39 INFO SparkContext:59 - Created broadcast 7 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:46:39 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:46:39 INFO DAGScheduler:59 - Got job 5 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:46:39 INFO DAGScheduler:59 - Final stage: ResultStage 5(aggregate at GaussianMixture.scala:201) -2016-06-13 18:46:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:46:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:46:39 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=52069, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=56165, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:65075 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:39 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:46:39 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:46:39 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 4 tasks -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 17, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 18, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 19, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 3.0 in stage 5.0 (TID 20, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:39 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 17) -2016-06-13 18:46:39 INFO Executor:59 - Running task 3.0 in stage 5.0 (TID 20) -2016-06-13 18:46:39 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 19) -2016-06-13 18:46:39 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 18) -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:46:39 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 18). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 17). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:46:39 INFO Executor:59 - Finished task 3.0 in stage 5.0 (TID 20). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 19). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 18) in 18 ms on localhost (1/4) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 17) in 20 ms on localhost (2/4) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 3.0 in stage 5.0 (TID 20) in 17 ms on localhost (3/4) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 19) in 18 ms on localhost (4/4) -2016-06-13 18:46:39 INFO DAGScheduler:59 - ResultStage 5 (aggregate at GaussianMixture.scala:201) finished in 0.022 s -2016-06-13 18:46:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 18:46:39 INFO DAGScheduler:59 - Job 5 finished: aggregate at GaussianMixture.scala:201, took 0.036134 s -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=58403, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 712.0 B, free 730.5 MB) -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(934) called with curMem=59115, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 934.0 B, free 730.5 MB) -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:65075 (size: 934.0 B, free: 730.6 MB) -2016-06-13 18:46:39 INFO SparkContext:59 - Created broadcast 9 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:46:39 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:46:39 INFO DAGScheduler:59 - Got job 6 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:46:39 INFO DAGScheduler:59 - Final stage: ResultStage 6(aggregate at GaussianMixture.scala:201) -2016-06-13 18:46:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:65075 in memory (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:46:39 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:46:39 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=53715, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(2240) called with curMem=56167, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:65075 in memory (size: 932.0 B, free: 730.6 MB) -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:65075 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:39 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:46:39 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:46:39 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 4 tasks -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:65075 in memory (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 21, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:46:39 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 22, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 23, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:65075 in memory (size: 934.0 B, free: 730.6 MB) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 3.0 in stage 6.0 (TID 24, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:39 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 22) -2016-06-13 18:46:39 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 23) -2016-06-13 18:46:39 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 21) -2016-06-13 18:46:39 INFO Executor:59 - Running task 3.0 in stage 6.0 (TID 24) -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:65075 in memory (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:39 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 22). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 21). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO Executor:59 - Finished task 3.0 in stage 6.0 (TID 24). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 23). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 21) in 19 ms on localhost (1/4) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 22) in 19 ms on localhost (2/4) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 23) in 19 ms on localhost (3/4) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 3.0 in stage 6.0 (TID 24) in 18 ms on localhost (4/4) -2016-06-13 18:46:39 INFO DAGScheduler:59 - ResultStage 6 (aggregate at GaussianMixture.scala:201) finished in 0.022 s -2016-06-13 18:46:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 18:46:39 INFO DAGScheduler:59 - Job 6 finished: aggregate at GaussianMixture.scala:201, took 0.040139 s -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:65075 in memory (size: 937.0 B, free: 730.6 MB) -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=42445, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 712.0 B, free 730.6 MB) -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:65075 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 18:46:39 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=14856, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.6 MB) -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:65075 in memory (size: 1195.0 B, free: 730.6 MB) -2016-06-13 18:46:39 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:65075 (size: 932.0 B, free: 730.6 MB) -2016-06-13 18:46:39 INFO SparkContext:59 - Created broadcast 11 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:65075 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 18:46:39 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 18:46:39 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:46:39 INFO DAGScheduler:59 - Got job 7 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:46:39 INFO DAGScheduler:59 - Final stage: ResultStage 7(aggregate at GaussianMixture.scala:201) -2016-06-13 18:46:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:46:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:46:39 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=10714, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.0 KB, free 730.6 MB) -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(2240) called with curMem=14810, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.6 MB) -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:65075 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:39 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:46:39 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 7 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:46:39 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 4 tasks -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 25, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 26, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 7.0 (TID 27, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 3.0 in stage 7.0 (TID 28, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:39 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 25) -2016-06-13 18:46:39 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 26) -2016-06-13 18:46:39 INFO Executor:59 - Running task 2.0 in stage 7.0 (TID 27) -2016-06-13 18:46:39 INFO Executor:59 - Running task 3.0 in stage 7.0 (TID 28) -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:46:39 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 25). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO Executor:59 - Finished task 2.0 in stage 7.0 (TID 27). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 26). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 25) in 16 ms on localhost (1/4) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 7.0 (TID 27) in 15 ms on localhost (2/4) -2016-06-13 18:46:39 INFO Executor:59 - Finished task 3.0 in stage 7.0 (TID 28). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 26) in 19 ms on localhost (3/4) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 3.0 in stage 7.0 (TID 28) in 17 ms on localhost (4/4) -2016-06-13 18:46:39 INFO DAGScheduler:59 - ResultStage 7 (aggregate at GaussianMixture.scala:201) finished in 0.020 s -2016-06-13 18:46:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 18:46:39 INFO DAGScheduler:59 - Job 7 finished: aggregate at GaussianMixture.scala:201, took 0.033204 s -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=17050, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 712.0 B, free 730.6 MB) -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(936) called with curMem=17762, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 936.0 B, free 730.6 MB) -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:65075 (size: 936.0 B, free: 730.6 MB) -2016-06-13 18:46:39 INFO SparkContext:59 - Created broadcast 13 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:46:39 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:46:39 INFO DAGScheduler:59 - Got job 8 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:46:39 INFO DAGScheduler:59 - Final stage: ResultStage 8(aggregate at GaussianMixture.scala:201) -2016-06-13 18:46:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:46:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:46:39 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=18698, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 4.0 KB, free 730.6 MB) -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(2241) called with curMem=22794, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.6 MB) -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:65075 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:46:39 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:46:39 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 8 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:46:39 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 4 tasks -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 29, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 30, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 8.0 (TID 31, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 3.0 in stage 8.0 (TID 32, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:46:39 INFO Executor:59 - Running task 2.0 in stage 8.0 (TID 31) -2016-06-13 18:46:39 INFO Executor:59 - Running task 3.0 in stage 8.0 (TID 32) -2016-06-13 18:46:39 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 29) -2016-06-13 18:46:39 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 30) -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:46:39 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:46:39 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 30). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO Executor:59 - Finished task 3.0 in stage 8.0 (TID 32). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 29). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO Executor:59 - Finished task 2.0 in stage 8.0 (TID 31). 2762 bytes result sent to driver -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 30) in 14 ms on localhost (1/4) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 3.0 in stage 8.0 (TID 32) in 13 ms on localhost (2/4) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 29) in 16 ms on localhost (3/4) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 8.0 (TID 31) in 15 ms on localhost (4/4) -2016-06-13 18:46:39 INFO DAGScheduler:59 - ResultStage 8 (aggregate at GaussianMixture.scala:201) finished in 0.018 s -2016-06-13 18:46:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 18:46:39 INFO DAGScheduler:59 - Job 8 finished: aggregate at GaussianMixture.scala:201, took 0.029268 s -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(656) called with curMem=25035, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 656.0 B, free 730.6 MB) -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(833) called with curMem=25691, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 833.0 B, free 730.6 MB) -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:65075 (size: 833.0 B, free: 730.6 MB) -2016-06-13 18:46:39 INFO SparkContext:59 - Created broadcast 15 from broadcast at GaussianMixtureModel.scala:100 -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(56) called with curMem=26524, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 56.0 B, free 730.6 MB) -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(65) called with curMem=26580, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 65.0 B, free 730.6 MB) -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:65075 (size: 65.0 B, free: 730.6 MB) -2016-06-13 18:46:39 INFO SparkContext:59 - Created broadcast 16 from broadcast at GaussianMixtureModel.scala:101 -2016-06-13 18:46:39 INFO SparkContext:59 - Starting job: collect at GaussianMixtureFromSparkSuite.scala:201 -2016-06-13 18:46:39 INFO DAGScheduler:59 - Got job 9 (collect at GaussianMixtureFromSparkSuite.scala:201) with 4 output partitions -2016-06-13 18:46:39 INFO DAGScheduler:59 - Final stage: ResultStage 9(collect at GaussianMixtureFromSparkSuite.scala:201) -2016-06-13 18:46:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:46:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:46:39 INFO DAGScheduler:59 - Submitting ResultStage 9 (ZippedPartitionsRDD2[5] at zip at GaussianMixtureFromSparkSuite.scala:201), which has no missing parents -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(4712) called with curMem=26645, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 18:46:39 INFO MemoryStore:59 - ensureFreeSpace(2631) called with curMem=31357, maxMem=766075207 -2016-06-13 18:46:39 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 18:46:39 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:65075 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 18:46:39 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:46:39 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 9 (ZippedPartitionsRDD2[5] at zip at GaussianMixtureFromSparkSuite.scala:201) -2016-06-13 18:46:39 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 4 tasks -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 33, localhost, PROCESS_LOCAL, 2432 bytes) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 34, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 9.0 (TID 35, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Starting task 3.0 in stage 9.0 (TID 36, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 18:46:39 INFO Executor:59 - Running task 2.0 in stage 9.0 (TID 35) -2016-06-13 18:46:39 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 34) -2016-06-13 18:46:39 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 33) -2016-06-13 18:46:39 INFO Executor:59 - Running task 3.0 in stage 9.0 (TID 36) -2016-06-13 18:46:39 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 33). 1229 bytes result sent to driver -2016-06-13 18:46:39 INFO Executor:59 - Finished task 3.0 in stage 9.0 (TID 36). 1264 bytes result sent to driver -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 3.0 in stage 9.0 (TID 36) in 21 ms on localhost (1/4) -2016-06-13 18:46:39 INFO Executor:59 - Finished task 2.0 in stage 9.0 (TID 35). 1264 bytes result sent to driver -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 33) in 27 ms on localhost (2/4) -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 9.0 (TID 35) in 25 ms on localhost (3/4) -2016-06-13 18:46:39 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 34). 1269 bytes result sent to driver -2016-06-13 18:46:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 34) in 30 ms on localhost (4/4) -2016-06-13 18:46:39 INFO DAGScheduler:59 - ResultStage 9 (collect at GaussianMixtureFromSparkSuite.scala:201) finished in 0.033 s -2016-06-13 18:46:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 18:46:39 INFO DAGScheduler:59 - Job 9 finished: collect at GaussianMixtureFromSparkSuite.scala:201, took 0.045349 s -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 18:46:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 18:46:39 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 18:46:39 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 18:46:39 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 18:46:39 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 18:46:39 INFO BlockManager:59 - BlockManager stopped -2016-06-13 18:46:39 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 18:46:39 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 18:46:39 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 18:46:40 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 18:46:40 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 18:46:40 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 18:46:40 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-772e6e8f-1e3a-465c-ab98-feeb18386da2 -2016-06-13 18:47:24 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 18:47:25 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 18:47:25 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 18:47:25 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 18:47:25 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 18:47:26 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 18:47:26 INFO Remoting:74 - Starting remoting -2016-06-13 18:47:26 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:65092] -2016-06-13 18:47:26 INFO Utils:59 - Successfully started service 'sparkDriver' on port 65092. -2016-06-13 18:47:26 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 18:47:26 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 18:47:26 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-f1aa414e-d19a-4bfd-bbb4-39c4c8bcf1af -2016-06-13 18:47:26 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 18:47:26 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-270aad89-5a50-4590-8bdd-997912f2dedc\httpd-447d2238-7ca5-4d32-b232-80f0e7c5bf27 -2016-06-13 18:47:26 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 18:47:26 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 18:47:26 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:65094 -2016-06-13 18:47:26 INFO Utils:59 - Successfully started service 'HTTP file server' on port 65094. -2016-06-13 18:47:26 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 18:47:26 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 18:47:26 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 18:47:26 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 18:47:26 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 18:47:26 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 18:47:26 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 18:47:27 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 65113. -2016-06-13 18:47:27 INFO NettyBlockTransferService:59 - Server created on 65113 -2016-06-13 18:47:27 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 18:47:27 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:65113 with 730.6 MB RAM, BlockManagerId(driver, localhost, 65113) -2016-06-13 18:47:27 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 18:47:28 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 18:47:28 INFO DAGScheduler:59 - Got job 0 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 18:47:28 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GaussianMixture.scala:171) -2016-06-13 18:47:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:47:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:47:28 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:47:28 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=0, maxMem=766075207 -2016-06-13 18:47:28 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 18:47:28 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1992, maxMem=766075207 -2016-06-13 18:47:28 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.6 MB) -2016-06-13 18:47:28 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:65113 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 18:47:28 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:47:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:47:28 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 18:47:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:47:28 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 18:47:28 INFO CacheManager:59 - Partition rdd_1_0 not found, computing it -2016-06-13 18:47:28 INFO MemoryStore:59 - ensureFreeSpace(224) called with curMem=3242, maxMem=766075207 -2016-06-13 18:47:28 INFO MemoryStore:59 - Block rdd_1_0 stored as values in memory (estimated size 224.0 B, free 730.6 MB) -2016-06-13 18:47:28 INFO BlockManagerInfo:59 - Added rdd_1_0 in memory on localhost:65113 (size: 224.0 B, free: 730.6 MB) -2016-06-13 18:47:28 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1688 bytes result sent to driver -2016-06-13 18:47:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 372 ms on localhost (1/1) -2016-06-13 18:47:28 INFO DAGScheduler:59 - ResultStage 0 (first at GaussianMixture.scala:171) finished in 0.384 s -2016-06-13 18:47:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 18:47:28 INFO DAGScheduler:59 - Job 0 finished: first at GaussianMixture.scala:171, took 0.610290 s -2016-06-13 18:47:28 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 18:47:28 INFO DAGScheduler:59 - Got job 1 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 18:47:28 INFO DAGScheduler:59 - Final stage: ResultStage 1(takeSample at GaussianMixture.scala:184) -2016-06-13 18:47:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:47:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:47:28 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:47:28 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=3466, maxMem=766075207 -2016-06-13 18:47:28 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 18:47:28 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=5298, maxMem=766075207 -2016-06-13 18:47:28 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.6 MB) -2016-06-13 18:47:28 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:65113 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 18:47:28 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:47:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:47:28 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 4 tasks -2016-06-13 18:47:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:47:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:28 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 18:47:28 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 18:47:28 INFO Executor:59 - Running task 3.0 in stage 1.0 (TID 4) -2016-06-13 18:47:28 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 3) -2016-06-13 18:47:28 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:47:28 INFO CacheManager:59 - Partition rdd_1_1 not found, computing it -2016-06-13 18:47:28 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=6493, maxMem=766075207 -2016-06-13 18:47:28 INFO MemoryStore:59 - Block rdd_1_1 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 18:47:28 INFO CacheManager:59 - Partition rdd_1_3 not found, computing it -2016-06-13 18:47:28 INFO BlockManagerInfo:59 - Added rdd_1_1 in memory on localhost:65113 (size: 288.0 B, free: 730.6 MB) -2016-06-13 18:47:28 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 1567 bytes result sent to driver -2016-06-13 18:47:28 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=6781, maxMem=766075207 -2016-06-13 18:47:28 INFO MemoryStore:59 - Block rdd_1_3 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 18:47:28 INFO CacheManager:59 - Partition rdd_1_2 not found, computing it -2016-06-13 18:47:28 INFO BlockManagerInfo:59 - Added rdd_1_3 in memory on localhost:65113 (size: 288.0 B, free: 730.6 MB) -2016-06-13 18:47:28 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=7069, maxMem=766075207 -2016-06-13 18:47:28 INFO MemoryStore:59 - Block rdd_1_2 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 18:47:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 50 ms on localhost (1/4) -2016-06-13 18:47:28 INFO Executor:59 - Finished task 3.0 in stage 1.0 (TID 4). 1567 bytes result sent to driver -2016-06-13 18:47:28 INFO BlockManagerInfo:59 - Added rdd_1_2 in memory on localhost:65113 (size: 288.0 B, free: 730.6 MB) -2016-06-13 18:47:28 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 18:47:28 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 3). 1567 bytes result sent to driver -2016-06-13 18:47:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 1.0 (TID 4) in 53 ms on localhost (2/4) -2016-06-13 18:47:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 3) in 57 ms on localhost (3/4) -2016-06-13 18:47:28 INFO DAGScheduler:59 - ResultStage 1 (takeSample at GaussianMixture.scala:184) finished in 0.070 s -2016-06-13 18:47:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 69 ms on localhost (4/4) -2016-06-13 18:47:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 18:47:28 INFO DAGScheduler:59 - Job 1 finished: takeSample at GaussianMixture.scala:184, took 0.102392 s -2016-06-13 18:47:28 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 18:47:28 INFO DAGScheduler:59 - Got job 2 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 18:47:28 INFO DAGScheduler:59 - Final stage: ResultStage 2(takeSample at GaussianMixture.scala:184) -2016-06-13 18:47:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:47:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:47:28 INFO DAGScheduler:59 - Submitting ResultStage 2 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 18:47:28 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=7357, maxMem=766075207 -2016-06-13 18:47:28 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 18.8 KB, free 730.6 MB) -2016-06-13 18:47:28 INFO MemoryStore:59 - ensureFreeSpace(7885) called with curMem=26581, maxMem=766075207 -2016-06-13 18:47:28 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.6 MB) -2016-06-13 18:47:28 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:65113 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 18:47:28 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:47:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 2 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184) -2016-06-13 18:47:28 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 4 tasks -2016-06-13 18:47:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 18:47:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 18:47:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 18:47:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 2.0 (TID 8, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 18:47:28 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 5) -2016-06-13 18:47:28 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 7) -2016-06-13 18:47:28 INFO Executor:59 - Running task 3.0 in stage 2.0 (TID 8) -2016-06-13 18:47:28 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 6) -2016-06-13 18:47:28 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:47:28 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:47:28 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:47:28 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:47:28 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 5). 2303 bytes result sent to driver -2016-06-13 18:47:28 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 7). 2344 bytes result sent to driver -2016-06-13 18:47:28 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 6). 2354 bytes result sent to driver -2016-06-13 18:47:28 INFO Executor:59 - Finished task 3.0 in stage 2.0 (TID 8). 2339 bytes result sent to driver -2016-06-13 18:47:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 5) in 30 ms on localhost (1/4) -2016-06-13 18:47:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 7) in 28 ms on localhost (2/4) -2016-06-13 18:47:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 6) in 29 ms on localhost (3/4) -2016-06-13 18:47:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 2.0 (TID 8) in 28 ms on localhost (4/4) -2016-06-13 18:47:28 INFO DAGScheduler:59 - ResultStage 2 (takeSample at GaussianMixture.scala:184) finished in 0.032 s -2016-06-13 18:47:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 18:47:28 INFO DAGScheduler:59 - Job 2 finished: takeSample at GaussianMixture.scala:184, took 0.048936 s -2016-06-13 18:47:28 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 18:47:28 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:65113 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 18:47:29 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:65113 in memory (size: 1195.0 B, free: 730.6 MB) -2016-06-13 18:47:29 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:65113 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 18:47:29 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 18:47:29 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 18:47:29 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=1088, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 1032.0 B, free 730.6 MB) -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(989) called with curMem=2120, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 989.0 B, free 730.6 MB) -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:65113 (size: 989.0 B, free: 730.6 MB) -2016-06-13 18:47:29 INFO SparkContext:59 - Created broadcast 3 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:47:29 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:47:29 INFO DAGScheduler:59 - Got job 3 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:47:29 INFO DAGScheduler:59 - Final stage: ResultStage 3(aggregate at GaussianMixture.scala:201) -2016-06-13 18:47:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:47:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:47:29 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(4360) called with curMem=3109, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.3 KB, free 730.6 MB) -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(2289) called with curMem=7469, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.6 MB) -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:65113 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:47:29 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:47:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 3 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:47:29 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 4 tasks -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 11, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 3.0 (TID 12, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 9) -2016-06-13 18:47:29 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 11) -2016-06-13 18:47:29 INFO Executor:59 - Running task 3.0 in stage 3.0 (TID 12) -2016-06-13 18:47:29 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 10) -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:47:29 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 11). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO Executor:59 - Finished task 3.0 in stage 3.0 (TID 12). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 10). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 9). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 11) in 35 ms on localhost (1/4) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 3.0 (TID 12) in 37 ms on localhost (2/4) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 9) in 42 ms on localhost (3/4) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 10) in 43 ms on localhost (4/4) -2016-06-13 18:47:29 INFO DAGScheduler:59 - ResultStage 3 (aggregate at GaussianMixture.scala:201) finished in 0.044 s -2016-06-13 18:47:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 18:47:29 INFO DAGScheduler:59 - Job 3 finished: aggregate at GaussianMixture.scala:201, took 0.060818 s -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=9758, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 1032.0 B, free 730.6 MB) -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(1006) called with curMem=10790, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 1006.0 B, free 730.6 MB) -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:65113 (size: 1006.0 B, free: 730.6 MB) -2016-06-13 18:47:29 INFO SparkContext:59 - Created broadcast 5 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:47:29 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:47:29 INFO DAGScheduler:59 - Got job 4 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:47:29 INFO DAGScheduler:59 - Final stage: ResultStage 4(aggregate at GaussianMixture.scala:201) -2016-06-13 18:47:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:47:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:47:29 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(4360) called with curMem=11796, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 4.3 KB, free 730.6 MB) -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(2308) called with curMem=16156, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.6 MB) -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:65113 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:47:29 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:47:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:47:29 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 4 tasks -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 14, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 15, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 4.0 (TID 16, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 13) -2016-06-13 18:47:29 INFO Executor:59 - Running task 3.0 in stage 4.0 (TID 16) -2016-06-13 18:47:29 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 15) -2016-06-13 18:47:29 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 14) -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:47:29 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 14). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:47:29 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 15). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 13). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 14) in 30 ms on localhost (1/4) -2016-06-13 18:47:29 INFO Executor:59 - Finished task 3.0 in stage 4.0 (TID 16). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 15) in 30 ms on localhost (2/4) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 13) in 35 ms on localhost (3/4) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 4.0 (TID 16) in 32 ms on localhost (4/4) -2016-06-13 18:47:29 INFO DAGScheduler:59 - ResultStage 4 (aggregate at GaussianMixture.scala:201) finished in 0.036 s -2016-06-13 18:47:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 18:47:29 INFO DAGScheduler:59 - Job 4 finished: aggregate at GaussianMixture.scala:201, took 0.054042 s -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=18464, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 1032.0 B, free 730.6 MB) -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(1009) called with curMem=19496, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 1009.0 B, free 730.6 MB) -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:65113 (size: 1009.0 B, free: 730.6 MB) -2016-06-13 18:47:29 INFO SparkContext:59 - Created broadcast 7 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:47:29 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:47:29 INFO DAGScheduler:59 - Got job 5 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:47:29 INFO DAGScheduler:59 - Final stage: ResultStage 5(aggregate at GaussianMixture.scala:201) -2016-06-13 18:47:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:47:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:47:29 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(4360) called with curMem=20505, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.3 KB, free 730.6 MB) -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(2310) called with curMem=24865, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.6 MB) -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:65113 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:47:29 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:47:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:47:29 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 4 tasks -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 17, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 18, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 19, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 5.0 (TID 20, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 17) -2016-06-13 18:47:29 INFO Executor:59 - Running task 3.0 in stage 5.0 (TID 20) -2016-06-13 18:47:29 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 19) -2016-06-13 18:47:29 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 18) -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:47:29 INFO Executor:59 - Finished task 3.0 in stage 5.0 (TID 20). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 17). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:47:29 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 19). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 5.0 (TID 20) in 18 ms on localhost (1/4) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 17) in 24 ms on localhost (2/4) -2016-06-13 18:47:29 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 18). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 19) in 26 ms on localhost (3/4) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 18) in 28 ms on localhost (4/4) -2016-06-13 18:47:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 18:47:29 INFO DAGScheduler:59 - ResultStage 5 (aggregate at GaussianMixture.scala:201) finished in 0.030 s -2016-06-13 18:47:29 INFO DAGScheduler:59 - Job 5 finished: aggregate at GaussianMixture.scala:201, took 0.047086 s -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=27175, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 1032.0 B, free 730.6 MB) -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(1010) called with curMem=28207, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1010.0 B, free 730.6 MB) -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:65113 (size: 1010.0 B, free: 730.6 MB) -2016-06-13 18:47:29 INFO SparkContext:59 - Created broadcast 9 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:47:29 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:47:29 INFO DAGScheduler:59 - Got job 6 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:47:29 INFO DAGScheduler:59 - Final stage: ResultStage 6(aggregate at GaussianMixture.scala:201) -2016-06-13 18:47:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:47:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:47:29 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(4360) called with curMem=29217, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.3 KB, free 730.6 MB) -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(2307) called with curMem=33577, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.6 MB) -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:65113 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:47:29 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:47:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:47:29 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 4 tasks -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 21, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 22, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 23, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 6.0 (TID 24, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 21) -2016-06-13 18:47:29 INFO Executor:59 - Running task 3.0 in stage 6.0 (TID 24) -2016-06-13 18:47:29 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 23) -2016-06-13 18:47:29 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 22) -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:47:29 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 21). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO Executor:59 - Finished task 3.0 in stage 6.0 (TID 24). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:47:29 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 22). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 21) in 18 ms on localhost (1/4) -2016-06-13 18:47:29 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 23). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 6.0 (TID 24) in 17 ms on localhost (2/4) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 22) in 21 ms on localhost (3/4) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 23) in 22 ms on localhost (4/4) -2016-06-13 18:47:29 INFO DAGScheduler:59 - ResultStage 6 (aggregate at GaussianMixture.scala:201) finished in 0.024 s -2016-06-13 18:47:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 18:47:29 INFO DAGScheduler:59 - Job 6 finished: aggregate at GaussianMixture.scala:201, took 0.040860 s -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=35884, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 1032.0 B, free 730.6 MB) -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(1009) called with curMem=36916, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 1009.0 B, free 730.6 MB) -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:65113 (size: 1009.0 B, free: 730.6 MB) -2016-06-13 18:47:29 INFO SparkContext:59 - Created broadcast 11 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:47:29 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:47:29 INFO DAGScheduler:59 - Got job 7 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:47:29 INFO DAGScheduler:59 - Final stage: ResultStage 7(aggregate at GaussianMixture.scala:201) -2016-06-13 18:47:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:47:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:47:29 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(4360) called with curMem=37925, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.3 KB, free 730.5 MB) -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(2310) called with curMem=42285, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:65113 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:47:29 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:47:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 7 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:47:29 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 4 tasks -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 25, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 26, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 7.0 (TID 27, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 7.0 (TID 28, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO Executor:59 - Running task 2.0 in stage 7.0 (TID 27) -2016-06-13 18:47:29 INFO Executor:59 - Running task 3.0 in stage 7.0 (TID 28) -2016-06-13 18:47:29 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 26) -2016-06-13 18:47:29 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 25) -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:47:29 INFO Executor:59 - Finished task 2.0 in stage 7.0 (TID 27). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 25). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO Executor:59 - Finished task 3.0 in stage 7.0 (TID 28). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 7.0 (TID 27) in 12 ms on localhost (1/4) -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 25) in 17 ms on localhost (2/4) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 7.0 (TID 28) in 16 ms on localhost (3/4) -2016-06-13 18:47:29 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 26). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 26) in 19 ms on localhost (4/4) -2016-06-13 18:47:29 INFO DAGScheduler:59 - ResultStage 7 (aggregate at GaussianMixture.scala:201) finished in 0.021 s -2016-06-13 18:47:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 18:47:29 INFO DAGScheduler:59 - Job 7 finished: aggregate at GaussianMixture.scala:201, took 0.035191 s -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=44595, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 1032.0 B, free 730.5 MB) -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(1008) called with curMem=45627, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 1008.0 B, free 730.5 MB) -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:65113 (size: 1008.0 B, free: 730.6 MB) -2016-06-13 18:47:29 INFO SparkContext:59 - Created broadcast 13 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:47:29 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:47:29 INFO DAGScheduler:59 - Got job 8 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:47:29 INFO DAGScheduler:59 - Final stage: ResultStage 8(aggregate at GaussianMixture.scala:201) -2016-06-13 18:47:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:47:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:47:29 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(4360) called with curMem=46635, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 4.3 KB, free 730.5 MB) -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(2310) called with curMem=50995, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:65113 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:47:29 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:47:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 8 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:47:29 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 4 tasks -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 29, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 30, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 8.0 (TID 31, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 8.0 (TID 32, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 29) -2016-06-13 18:47:29 INFO Executor:59 - Running task 3.0 in stage 8.0 (TID 32) -2016-06-13 18:47:29 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 30) -2016-06-13 18:47:29 INFO Executor:59 - Running task 2.0 in stage 8.0 (TID 31) -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:47:29 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 29). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO Executor:59 - Finished task 3.0 in stage 8.0 (TID 32). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 30). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO Executor:59 - Finished task 2.0 in stage 8.0 (TID 31). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 29) in 16 ms on localhost (1/4) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 8.0 (TID 32) in 14 ms on localhost (2/4) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 30) in 17 ms on localhost (3/4) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 8.0 (TID 31) in 16 ms on localhost (4/4) -2016-06-13 18:47:29 INFO DAGScheduler:59 - ResultStage 8 (aggregate at GaussianMixture.scala:201) finished in 0.019 s -2016-06-13 18:47:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 18:47:29 INFO DAGScheduler:59 - Job 8 finished: aggregate at GaussianMixture.scala:201, took 0.032269 s -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=53305, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1032.0 B, free 730.5 MB) -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(1012) called with curMem=54337, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1012.0 B, free 730.5 MB) -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:65113 (size: 1012.0 B, free: 730.6 MB) -2016-06-13 18:47:29 INFO SparkContext:59 - Created broadcast 15 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:47:29 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:47:29 INFO DAGScheduler:59 - Got job 9 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:47:29 INFO DAGScheduler:59 - Final stage: ResultStage 9(aggregate at GaussianMixture.scala:201) -2016-06-13 18:47:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:47:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:47:29 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(4360) called with curMem=55349, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 4.3 KB, free 730.5 MB) -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(2311) called with curMem=59709, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:65113 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:47:29 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:47:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 9 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:47:29 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 4 tasks -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 33, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 34, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 9.0 (TID 35, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 9.0 (TID 36, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO Executor:59 - Running task 3.0 in stage 9.0 (TID 36) -2016-06-13 18:47:29 INFO Executor:59 - Running task 2.0 in stage 9.0 (TID 35) -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:47:29 INFO Executor:59 - Finished task 2.0 in stage 9.0 (TID 35). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 33) -2016-06-13 18:47:29 INFO Executor:59 - Finished task 3.0 in stage 9.0 (TID 36). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:47:29 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 33). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 34) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 9.0 (TID 35) in 27 ms on localhost (1/4) -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 9.0 (TID 36) in 28 ms on localhost (2/4) -2016-06-13 18:47:29 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 34). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 33) in 37 ms on localhost (3/4) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 34) in 39 ms on localhost (4/4) -2016-06-13 18:47:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 18:47:29 INFO DAGScheduler:59 - ResultStage 9 (aggregate at GaussianMixture.scala:201) finished in 0.042 s -2016-06-13 18:47:29 INFO DAGScheduler:59 - Job 9 finished: aggregate at GaussianMixture.scala:201, took 0.061662 s -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=62020, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 1032.0 B, free 730.5 MB) -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(1009) called with curMem=63052, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1009.0 B, free 730.5 MB) -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:65113 (size: 1009.0 B, free: 730.6 MB) -2016-06-13 18:47:29 INFO SparkContext:59 - Created broadcast 17 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:47:29 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:47:29 INFO DAGScheduler:59 - Got job 10 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:47:29 INFO DAGScheduler:59 - Final stage: ResultStage 10(aggregate at GaussianMixture.scala:201) -2016-06-13 18:47:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:47:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:47:29 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(4360) called with curMem=64061, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 4.3 KB, free 730.5 MB) -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(2310) called with curMem=68421, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:65113 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:47:29 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:47:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 10 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:47:29 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 4 tasks -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:65113 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 37, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 38, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 10.0 (TID 39, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 10.0 (TID 40, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:29 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 38) -2016-06-13 18:47:29 INFO Executor:59 - Running task 2.0 in stage 10.0 (TID 39) -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:47:29 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 38). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO Executor:59 - Running task 3.0 in stage 10.0 (TID 40) -2016-06-13 18:47:29 INFO Executor:59 - Finished task 2.0 in stage 10.0 (TID 39). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:65113 in memory (size: 1012.0 B, free: 730.6 MB) -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:47:29 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 37) -2016-06-13 18:47:29 INFO Executor:59 - Finished task 3.0 in stage 10.0 (TID 40). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:47:29 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 37). 2848 bytes result sent to driver -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 38) in 28 ms on localhost (1/4) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 10.0 (TID 40) in 32 ms on localhost (2/4) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 10.0 (TID 39) in 34 ms on localhost (3/4) -2016-06-13 18:47:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 37) in 39 ms on localhost (4/4) -2016-06-13 18:47:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:65113 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:47:29 INFO DAGScheduler:59 - ResultStage 10 (aggregate at GaussianMixture.scala:201) finished in 0.042 s -2016-06-13 18:47:29 INFO DAGScheduler:59 - Job 10 finished: aggregate at GaussianMixture.scala:201, took 0.076391 s -2016-06-13 18:47:29 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=54338, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 1032.0 B, free 730.5 MB) -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:65113 in memory (size: 1008.0 B, free: 730.6 MB) -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(1009) called with curMem=54338, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1009.0 B, free 730.5 MB) -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:65113 (size: 1009.0 B, free: 730.6 MB) -2016-06-13 18:47:29 INFO SparkContext:59 - Created broadcast 19 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:65113 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:47:29 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:65113 in memory (size: 1009.0 B, free: 730.6 MB) -2016-06-13 18:47:29 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:47:29 INFO DAGScheduler:59 - Got job 11 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:47:29 INFO DAGScheduler:59 - Final stage: ResultStage 11(aggregate at GaussianMixture.scala:201) -2016-06-13 18:47:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:47:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:65113 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:47:29 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:47:29 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(4360) called with curMem=37927, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 4.3 KB, free 730.5 MB) -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:65113 in memory (size: 1010.0 B, free: 730.6 MB) -2016-06-13 18:47:29 INFO MemoryStore:59 - ensureFreeSpace(2310) called with curMem=35617, maxMem=766075207 -2016-06-13 18:47:29 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.6 MB) -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:65113 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:47:29 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 18:47:29 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:65113 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:47:29 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:47:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 11 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:47:29 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 4 tasks -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 41, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:47:30 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:65113 in memory (size: 1009.0 B, free: 730.6 MB) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 42, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 11.0 (TID 43, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 11.0 (TID 44, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:30 INFO Executor:59 - Running task 2.0 in stage 11.0 (TID 43) -2016-06-13 18:47:30 INFO Executor:59 - Running task 3.0 in stage 11.0 (TID 44) -2016-06-13 18:47:30 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 42) -2016-06-13 18:47:30 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:47:30 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:47:30 INFO Executor:59 - Finished task 2.0 in stage 11.0 (TID 43). 2848 bytes result sent to driver -2016-06-13 18:47:30 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:65113 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:47:30 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 42). 2848 bytes result sent to driver -2016-06-13 18:47:30 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 41) -2016-06-13 18:47:30 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:47:30 INFO Executor:59 - Finished task 3.0 in stage 11.0 (TID 44). 2848 bytes result sent to driver -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 42) in 26 ms on localhost (1/4) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 11.0 (TID 44) in 24 ms on localhost (2/4) -2016-06-13 18:47:30 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 11.0 (TID 43) in 28 ms on localhost (3/4) -2016-06-13 18:47:30 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 41). 2848 bytes result sent to driver -2016-06-13 18:47:30 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 41) in 35 ms on localhost (4/4) -2016-06-13 18:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 18:47:30 INFO DAGScheduler:59 - ResultStage 11 (aggregate at GaussianMixture.scala:201) finished in 0.037 s -2016-06-13 18:47:30 INFO DAGScheduler:59 - Job 11 finished: aggregate at GaussianMixture.scala:201, took 0.059935 s -2016-06-13 18:47:30 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:65113 in memory (size: 1006.0 B, free: 730.6 MB) -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=27180, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 1032.0 B, free 730.6 MB) -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(1011) called with curMem=21563, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1011.0 B, free 730.6 MB) -2016-06-13 18:47:30 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:65113 in memory (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:47:30 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 18:47:30 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:65113 (size: 1011.0 B, free: 730.6 MB) -2016-06-13 18:47:30 INFO SparkContext:59 - Created broadcast 21 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:47:30 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:65113 in memory (size: 989.0 B, free: 730.6 MB) -2016-06-13 18:47:30 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:47:30 INFO DAGScheduler:59 - Got job 12 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:47:30 INFO DAGScheduler:59 - Final stage: ResultStage 12(aggregate at GaussianMixture.scala:201) -2016-06-13 18:47:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:47:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:47:30 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(4360) called with curMem=20553, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.3 KB, free 730.6 MB) -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(2309) called with curMem=24913, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.6 MB) -2016-06-13 18:47:30 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:65113 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:47:30 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:47:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 12 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:47:30 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 4 tasks -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 45, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 46, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 12.0 (TID 47, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 12.0 (TID 48, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:30 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 45) -2016-06-13 18:47:30 INFO Executor:59 - Running task 2.0 in stage 12.0 (TID 47) -2016-06-13 18:47:30 INFO Executor:59 - Running task 3.0 in stage 12.0 (TID 48) -2016-06-13 18:47:30 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:47:30 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 45). 2848 bytes result sent to driver -2016-06-13 18:47:30 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 46) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 45) in 16 ms on localhost (1/4) -2016-06-13 18:47:30 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:47:30 INFO Executor:59 - Finished task 3.0 in stage 12.0 (TID 48). 2848 bytes result sent to driver -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 12.0 (TID 48) in 17 ms on localhost (2/4) -2016-06-13 18:47:30 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:47:30 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:47:30 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 46). 2848 bytes result sent to driver -2016-06-13 18:47:30 INFO Executor:59 - Finished task 2.0 in stage 12.0 (TID 47). 2848 bytes result sent to driver -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 46) in 25 ms on localhost (3/4) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 12.0 (TID 47) in 26 ms on localhost (4/4) -2016-06-13 18:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 18:47:30 INFO DAGScheduler:59 - ResultStage 12 (aggregate at GaussianMixture.scala:201) finished in 0.029 s -2016-06-13 18:47:30 INFO DAGScheduler:59 - Job 12 finished: aggregate at GaussianMixture.scala:201, took 0.044620 s -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=27222, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 1032.0 B, free 730.6 MB) -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(1009) called with curMem=28254, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 1009.0 B, free 730.6 MB) -2016-06-13 18:47:30 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:65113 (size: 1009.0 B, free: 730.6 MB) -2016-06-13 18:47:30 INFO SparkContext:59 - Created broadcast 23 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:47:30 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:47:30 INFO DAGScheduler:59 - Got job 13 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:47:30 INFO DAGScheduler:59 - Final stage: ResultStage 13(aggregate at GaussianMixture.scala:201) -2016-06-13 18:47:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:47:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:47:30 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(4360) called with curMem=29263, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 4.3 KB, free 730.6 MB) -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(2310) called with curMem=33623, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.6 MB) -2016-06-13 18:47:30 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:65113 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:47:30 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:47:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 13 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:47:30 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 4 tasks -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 49, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 50, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 13.0 (TID 51, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 13.0 (TID 52, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:30 INFO Executor:59 - Running task 3.0 in stage 13.0 (TID 52) -2016-06-13 18:47:30 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 49) -2016-06-13 18:47:30 INFO Executor:59 - Running task 2.0 in stage 13.0 (TID 51) -2016-06-13 18:47:30 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 50) -2016-06-13 18:47:30 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:47:30 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:47:30 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 50). 2848 bytes result sent to driver -2016-06-13 18:47:30 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 49). 2848 bytes result sent to driver -2016-06-13 18:47:30 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:47:30 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:47:30 INFO Executor:59 - Finished task 2.0 in stage 13.0 (TID 51). 2848 bytes result sent to driver -2016-06-13 18:47:30 INFO Executor:59 - Finished task 3.0 in stage 13.0 (TID 52). 2848 bytes result sent to driver -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 50) in 21 ms on localhost (1/4) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 49) in 23 ms on localhost (2/4) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 13.0 (TID 51) in 22 ms on localhost (3/4) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 13.0 (TID 52) in 22 ms on localhost (4/4) -2016-06-13 18:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 18:47:30 INFO DAGScheduler:59 - ResultStage 13 (aggregate at GaussianMixture.scala:201) finished in 0.029 s -2016-06-13 18:47:30 INFO DAGScheduler:59 - Job 13 finished: aggregate at GaussianMixture.scala:201, took 0.046610 s -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=35933, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 1032.0 B, free 730.6 MB) -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(1016) called with curMem=36965, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 1016.0 B, free 730.6 MB) -2016-06-13 18:47:30 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:65113 (size: 1016.0 B, free: 730.6 MB) -2016-06-13 18:47:30 INFO SparkContext:59 - Created broadcast 25 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:47:30 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:47:30 INFO DAGScheduler:59 - Got job 14 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:47:30 INFO DAGScheduler:59 - Final stage: ResultStage 14(aggregate at GaussianMixture.scala:201) -2016-06-13 18:47:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:47:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:47:30 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(4360) called with curMem=37981, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 4.3 KB, free 730.5 MB) -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(2309) called with curMem=42341, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:47:30 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:65113 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:47:30 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:47:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 14 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:47:30 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 4 tasks -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 53, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 54, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 55, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 14.0 (TID 56, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:30 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 53) -2016-06-13 18:47:30 INFO Executor:59 - Running task 3.0 in stage 14.0 (TID 56) -2016-06-13 18:47:30 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 54) -2016-06-13 18:47:30 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:47:30 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 53). 2848 bytes result sent to driver -2016-06-13 18:47:30 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:47:30 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 54). 2848 bytes result sent to driver -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 53) in 13 ms on localhost (1/4) -2016-06-13 18:47:30 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 55) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 54) in 15 ms on localhost (2/4) -2016-06-13 18:47:30 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:47:30 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:47:30 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 55). 2848 bytes result sent to driver -2016-06-13 18:47:30 INFO Executor:59 - Finished task 3.0 in stage 14.0 (TID 56). 2848 bytes result sent to driver -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 55) in 22 ms on localhost (3/4) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 14.0 (TID 56) in 22 ms on localhost (4/4) -2016-06-13 18:47:30 INFO DAGScheduler:59 - ResultStage 14 (aggregate at GaussianMixture.scala:201) finished in 0.025 s -2016-06-13 18:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 18:47:30 INFO DAGScheduler:59 - Job 14 finished: aggregate at GaussianMixture.scala:201, took 0.040112 s -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=44650, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1032.0 B, free 730.5 MB) -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(1005) called with curMem=45682, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 1005.0 B, free 730.5 MB) -2016-06-13 18:47:30 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:65113 (size: 1005.0 B, free: 730.6 MB) -2016-06-13 18:47:30 INFO SparkContext:59 - Created broadcast 27 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:47:30 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:47:30 INFO DAGScheduler:59 - Got job 15 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:47:30 INFO DAGScheduler:59 - Final stage: ResultStage 15(aggregate at GaussianMixture.scala:201) -2016-06-13 18:47:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:47:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:47:30 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(4360) called with curMem=46687, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 4.3 KB, free 730.5 MB) -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(2308) called with curMem=51047, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:47:30 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:65113 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:47:30 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:47:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 15 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:47:30 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 4 tasks -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 57, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 58, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 15.0 (TID 59, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 15.0 (TID 60, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:47:30 INFO Executor:59 - Running task 3.0 in stage 15.0 (TID 60) -2016-06-13 18:47:30 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 58) -2016-06-13 18:47:30 INFO Executor:59 - Running task 2.0 in stage 15.0 (TID 59) -2016-06-13 18:47:30 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:47:30 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:47:30 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 57) -2016-06-13 18:47:30 INFO Executor:59 - Finished task 2.0 in stage 15.0 (TID 59). 2848 bytes result sent to driver -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 15.0 (TID 59) in 12 ms on localhost (1/4) -2016-06-13 18:47:30 INFO Executor:59 - Finished task 3.0 in stage 15.0 (TID 60). 2848 bytes result sent to driver -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 15.0 (TID 60) in 15 ms on localhost (2/4) -2016-06-13 18:47:30 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:47:30 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:47:30 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 57). 2848 bytes result sent to driver -2016-06-13 18:47:30 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 58). 2848 bytes result sent to driver -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 57) in 23 ms on localhost (3/4) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 58) in 23 ms on localhost (4/4) -2016-06-13 18:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 18:47:30 INFO DAGScheduler:59 - ResultStage 15 (aggregate at GaussianMixture.scala:201) finished in 0.025 s -2016-06-13 18:47:30 INFO DAGScheduler:59 - Job 15 finished: aggregate at GaussianMixture.scala:201, took 0.041733 s -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(968) called with curMem=53355, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 968.0 B, free 730.5 MB) -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(901) called with curMem=54323, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 901.0 B, free 730.5 MB) -2016-06-13 18:47:30 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:65113 (size: 901.0 B, free: 730.6 MB) -2016-06-13 18:47:30 INFO SparkContext:59 - Created broadcast 29 from broadcast at GaussianMixtureModel.scala:100 -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(64) called with curMem=55224, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 64.0 B, free 730.5 MB) -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(73) called with curMem=55288, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 73.0 B, free 730.5 MB) -2016-06-13 18:47:30 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:65113 (size: 73.0 B, free: 730.6 MB) -2016-06-13 18:47:30 INFO SparkContext:59 - Created broadcast 30 from broadcast at GaussianMixtureModel.scala:101 -2016-06-13 18:47:30 INFO SparkContext:59 - Starting job: collect at GaussianMixtureFromSparkSuite.scala:201 -2016-06-13 18:47:30 INFO DAGScheduler:59 - Got job 16 (collect at GaussianMixtureFromSparkSuite.scala:201) with 4 output partitions -2016-06-13 18:47:30 INFO DAGScheduler:59 - Final stage: ResultStage 16(collect at GaussianMixtureFromSparkSuite.scala:201) -2016-06-13 18:47:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:47:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:47:30 INFO DAGScheduler:59 - Submitting ResultStage 16 (ZippedPartitionsRDD2[5] at zip at GaussianMixtureFromSparkSuite.scala:201), which has no missing parents -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(4896) called with curMem=55361, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 4.8 KB, free 730.5 MB) -2016-06-13 18:47:30 INFO MemoryStore:59 - ensureFreeSpace(2705) called with curMem=60257, maxMem=766075207 -2016-06-13 18:47:30 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 18:47:30 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:65113 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 18:47:30 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:47:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 16 (ZippedPartitionsRDD2[5] at zip at GaussianMixtureFromSparkSuite.scala:201) -2016-06-13 18:47:30 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 4 tasks -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 61, localhost, PROCESS_LOCAL, 2432 bytes) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 62, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 16.0 (TID 63, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 18:47:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 16.0 (TID 64, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 18:47:30 INFO Executor:59 - Running task 3.0 in stage 16.0 (TID 64) -2016-06-13 18:47:30 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 62) -2016-06-13 18:47:30 INFO Executor:59 - Running task 2.0 in stage 16.0 (TID 63) -2016-06-13 18:47:30 INFO Executor:59 - Finished task 3.0 in stage 16.0 (TID 64). 1264 bytes result sent to driver -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 16.0 (TID 64) in 19 ms on localhost (1/4) -2016-06-13 18:47:30 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 61) -2016-06-13 18:47:30 INFO Executor:59 - Finished task 2.0 in stage 16.0 (TID 63). 1264 bytes result sent to driver -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 16.0 (TID 63) in 25 ms on localhost (2/4) -2016-06-13 18:47:30 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 62). 1269 bytes result sent to driver -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 62) in 29 ms on localhost (3/4) -2016-06-13 18:47:30 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 61). 1229 bytes result sent to driver -2016-06-13 18:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 61) in 40 ms on localhost (4/4) -2016-06-13 18:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 18:47:30 INFO DAGScheduler:59 - ResultStage 16 (collect at GaussianMixtureFromSparkSuite.scala:201) finished in 0.040 s -2016-06-13 18:47:30 INFO DAGScheduler:59 - Job 16 finished: collect at GaussianMixtureFromSparkSuite.scala:201, took 0.058481 s -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 18:47:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 18:47:30 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 18:47:30 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 18:47:30 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 18:47:30 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 18:47:30 INFO BlockManager:59 - BlockManager stopped -2016-06-13 18:47:30 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 18:47:30 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 18:47:30 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 18:47:30 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 18:47:30 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 18:47:30 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 18:47:30 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-270aad89-5a50-4590-8bdd-997912f2dedc -2016-06-13 18:48:43 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 18:48:44 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 18:48:44 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 18:48:44 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 18:48:44 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 18:48:45 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 18:48:45 INFO Remoting:74 - Starting remoting -2016-06-13 18:48:45 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:65137] -2016-06-13 18:48:45 INFO Utils:59 - Successfully started service 'sparkDriver' on port 65137. -2016-06-13 18:48:45 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 18:48:45 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 18:48:45 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-91c59352-1505-4a13-9ac2-a6cc6ad55147 -2016-06-13 18:48:45 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 18:48:45 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-f0bbc7d6-39c9-4b0a-8ad2-3299aa18dc98\httpd-6e1774b8-fd58-4a89-be2f-46a9490f2693 -2016-06-13 18:48:45 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 18:48:45 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 18:48:45 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:65138 -2016-06-13 18:48:45 INFO Utils:59 - Successfully started service 'HTTP file server' on port 65138. -2016-06-13 18:48:45 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 18:48:45 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 18:48:45 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 18:48:45 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 18:48:45 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 18:48:45 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 18:48:45 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 18:48:46 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 65157. -2016-06-13 18:48:46 INFO NettyBlockTransferService:59 - Server created on 65157 -2016-06-13 18:48:46 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 18:48:46 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:65157 with 730.6 MB RAM, BlockManagerId(driver, localhost, 65157) -2016-06-13 18:48:46 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 18:48:46 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 18:48:47 INFO DAGScheduler:59 - Got job 0 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 18:48:47 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GaussianMixture.scala:171) -2016-06-13 18:48:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:47 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:47 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:47 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=0, maxMem=766075207 -2016-06-13 18:48:47 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 18:48:47 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1992, maxMem=766075207 -2016-06-13 18:48:47 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.6 MB) -2016-06-13 18:48:47 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:65157 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 18:48:47 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:47 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:47 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 18:48:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:47 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 18:48:47 INFO CacheManager:59 - Partition rdd_1_0 not found, computing it -2016-06-13 18:48:47 INFO MemoryStore:59 - ensureFreeSpace(224) called with curMem=3242, maxMem=766075207 -2016-06-13 18:48:47 INFO MemoryStore:59 - Block rdd_1_0 stored as values in memory (estimated size 224.0 B, free 730.6 MB) -2016-06-13 18:48:47 INFO BlockManagerInfo:59 - Added rdd_1_0 in memory on localhost:65157 (size: 224.0 B, free: 730.6 MB) -2016-06-13 18:48:47 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1688 bytes result sent to driver -2016-06-13 18:48:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 390 ms on localhost (1/1) -2016-06-13 18:48:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 18:48:47 INFO DAGScheduler:59 - ResultStage 0 (first at GaussianMixture.scala:171) finished in 0.404 s -2016-06-13 18:48:47 INFO DAGScheduler:59 - Job 0 finished: first at GaussianMixture.scala:171, took 0.608008 s -2016-06-13 18:48:47 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 18:48:47 INFO DAGScheduler:59 - Got job 1 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 18:48:47 INFO DAGScheduler:59 - Final stage: ResultStage 1(takeSample at GaussianMixture.scala:184) -2016-06-13 18:48:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:47 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:47 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:47 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=3466, maxMem=766075207 -2016-06-13 18:48:47 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 18:48:47 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=5298, maxMem=766075207 -2016-06-13 18:48:47 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.6 MB) -2016-06-13 18:48:47 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:65157 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 18:48:47 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:47 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 4 tasks -2016-06-13 18:48:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:47 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 18:48:47 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 3) -2016-06-13 18:48:47 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:47 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 18:48:47 INFO Executor:59 - Running task 3.0 in stage 1.0 (TID 4) -2016-06-13 18:48:47 INFO CacheManager:59 - Partition rdd_1_1 not found, computing it -2016-06-13 18:48:47 INFO CacheManager:59 - Partition rdd_1_3 not found, computing it -2016-06-13 18:48:47 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 18:48:47 INFO CacheManager:59 - Partition rdd_1_2 not found, computing it -2016-06-13 18:48:47 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=6493, maxMem=766075207 -2016-06-13 18:48:47 INFO MemoryStore:59 - Block rdd_1_1 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 18:48:47 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=6781, maxMem=766075207 -2016-06-13 18:48:47 INFO MemoryStore:59 - Block rdd_1_3 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 18:48:47 INFO BlockManagerInfo:59 - Added rdd_1_1 in memory on localhost:65157 (size: 288.0 B, free: 730.6 MB) -2016-06-13 18:48:47 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=7069, maxMem=766075207 -2016-06-13 18:48:47 INFO MemoryStore:59 - Block rdd_1_2 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 18:48:47 INFO BlockManagerInfo:59 - Added rdd_1_3 in memory on localhost:65157 (size: 288.0 B, free: 730.6 MB) -2016-06-13 18:48:47 INFO BlockManagerInfo:59 - Added rdd_1_2 in memory on localhost:65157 (size: 288.0 B, free: 730.6 MB) -2016-06-13 18:48:47 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 1567 bytes result sent to driver -2016-06-13 18:48:47 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 3). 1567 bytes result sent to driver -2016-06-13 18:48:47 INFO Executor:59 - Finished task 3.0 in stage 1.0 (TID 4). 1567 bytes result sent to driver -2016-06-13 18:48:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 50 ms on localhost (1/4) -2016-06-13 18:48:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 53 ms on localhost (2/4) -2016-06-13 18:48:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 1.0 (TID 4) in 51 ms on localhost (3/4) -2016-06-13 18:48:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 3) in 54 ms on localhost (4/4) -2016-06-13 18:48:47 INFO DAGScheduler:59 - ResultStage 1 (takeSample at GaussianMixture.scala:184) finished in 0.060 s -2016-06-13 18:48:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 18:48:47 INFO DAGScheduler:59 - Job 1 finished: takeSample at GaussianMixture.scala:184, took 0.085023 s -2016-06-13 18:48:47 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 18:48:47 INFO DAGScheduler:59 - Got job 2 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 18:48:47 INFO DAGScheduler:59 - Final stage: ResultStage 2(takeSample at GaussianMixture.scala:184) -2016-06-13 18:48:47 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:47 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:47 INFO DAGScheduler:59 - Submitting ResultStage 2 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 18:48:47 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=7357, maxMem=766075207 -2016-06-13 18:48:47 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 18.8 KB, free 730.6 MB) -2016-06-13 18:48:47 INFO MemoryStore:59 - ensureFreeSpace(7899) called with curMem=26581, maxMem=766075207 -2016-06-13 18:48:47 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.6 MB) -2016-06-13 18:48:47 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:65157 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 18:48:47 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:47 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 2 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184) -2016-06-13 18:48:47 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 4 tasks -2016-06-13 18:48:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 18:48:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 18:48:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 18:48:47 INFO TaskSetManager:59 - Starting task 3.0 in stage 2.0 (TID 8, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 18:48:47 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 5) -2016-06-13 18:48:47 INFO Executor:59 - Running task 3.0 in stage 2.0 (TID 8) -2016-06-13 18:48:47 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 6) -2016-06-13 18:48:47 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 7) -2016-06-13 18:48:47 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:47 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:47 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:47 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:47 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 7). 2308 bytes result sent to driver -2016-06-13 18:48:47 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 5). 2298 bytes result sent to driver -2016-06-13 18:48:47 INFO Executor:59 - Finished task 3.0 in stage 2.0 (TID 8). 2339 bytes result sent to driver -2016-06-13 18:48:47 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 6). 2349 bytes result sent to driver -2016-06-13 18:48:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 7) in 26 ms on localhost (1/4) -2016-06-13 18:48:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 5) in 30 ms on localhost (2/4) -2016-06-13 18:48:47 INFO TaskSetManager:59 - Finished task 3.0 in stage 2.0 (TID 8) in 27 ms on localhost (3/4) -2016-06-13 18:48:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 6) in 29 ms on localhost (4/4) -2016-06-13 18:48:47 INFO DAGScheduler:59 - ResultStage 2 (takeSample at GaussianMixture.scala:184) finished in 0.032 s -2016-06-13 18:48:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 18:48:47 INFO DAGScheduler:59 - Job 2 finished: takeSample at GaussianMixture.scala:184, took 0.047629 s -2016-06-13 18:48:47 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 18:48:47 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:65157 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 18:48:48 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:65157 in memory (size: 1195.0 B, free: 730.6 MB) -2016-06-13 18:48:48 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:65157 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 18:48:48 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 18:48:48 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 18:48:48 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=1088, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 1344.0 B, free 730.6 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(1060) called with curMem=2432, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1060.0 B, free 730.6 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:65157 (size: 1060.0 B, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Created broadcast 3 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:48:48 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Got job 3 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:48:48 INFO DAGScheduler:59 - Final stage: ResultStage 3(aggregate at GaussianMixture.scala:201) -2016-06-13 18:48:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:48 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:48 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=3492, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.6 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(2354) called with curMem=8124, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.6 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:65157 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 3 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:48 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 4 tasks -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 11, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 3.0 (TID 12, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 9) -2016-06-13 18:48:48 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 10) -2016-06-13 18:48:48 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 11) -2016-06-13 18:48:48 INFO Executor:59 - Running task 3.0 in stage 3.0 (TID 12) -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:48 INFO Executor:59 - Finished task 3.0 in stage 3.0 (TID 12). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 10). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 11). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 9). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 3.0 (TID 12) in 34 ms on localhost (1/4) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 10) in 36 ms on localhost (2/4) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 11) in 38 ms on localhost (3/4) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 9) in 42 ms on localhost (4/4) -2016-06-13 18:48:48 INFO DAGScheduler:59 - ResultStage 3 (aggregate at GaussianMixture.scala:201) finished in 0.042 s -2016-06-13 18:48:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 18:48:48 INFO DAGScheduler:59 - Job 3 finished: aggregate at GaussianMixture.scala:201, took 0.057257 s -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=10478, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 1344.0 B, free 730.6 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(1082) called with curMem=11822, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 1082.0 B, free 730.6 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:65157 (size: 1082.0 B, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Created broadcast 5 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:48:48 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Got job 4 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:48:48 INFO DAGScheduler:59 - Final stage: ResultStage 4(aggregate at GaussianMixture.scala:201) -2016-06-13 18:48:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:48 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:48 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=12904, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 4.5 KB, free 730.6 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(2380) called with curMem=17536, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.6 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:65157 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:48 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 4 tasks -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 14, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 15, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 4.0 (TID 16, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 14) -2016-06-13 18:48:48 INFO Executor:59 - Running task 3.0 in stage 4.0 (TID 16) -2016-06-13 18:48:48 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 13) -2016-06-13 18:48:48 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 15) -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:48 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 13). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO Executor:59 - Finished task 3.0 in stage 4.0 (TID 16). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:48 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 15). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 13) in 34 ms on localhost (1/4) -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 4.0 (TID 16) in 33 ms on localhost (2/4) -2016-06-13 18:48:48 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 14). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 15) in 38 ms on localhost (3/4) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 14) in 41 ms on localhost (4/4) -2016-06-13 18:48:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 18:48:48 INFO DAGScheduler:59 - ResultStage 4 (aggregate at GaussianMixture.scala:201) finished in 0.043 s -2016-06-13 18:48:48 INFO DAGScheduler:59 - Job 4 finished: aggregate at GaussianMixture.scala:201, took 0.060562 s -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=19916, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 1344.0 B, free 730.6 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(1082) called with curMem=21260, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 1082.0 B, free 730.6 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:65157 (size: 1082.0 B, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Created broadcast 7 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:48:48 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Got job 5 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:48:48 INFO DAGScheduler:59 - Final stage: ResultStage 5(aggregate at GaussianMixture.scala:201) -2016-06-13 18:48:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:48 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:48 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=22342, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.5 KB, free 730.6 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(2378) called with curMem=26974, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.6 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:65157 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:48 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 4 tasks -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 17, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 18, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 19, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 5.0 (TID 20, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 19) -2016-06-13 18:48:48 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 17) -2016-06-13 18:48:48 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 18) -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:48 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 19). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO Executor:59 - Running task 3.0 in stage 5.0 (TID 20) -2016-06-13 18:48:48 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 17). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:48 INFO Executor:59 - Finished task 3.0 in stage 5.0 (TID 20). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 19) in 20 ms on localhost (1/4) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 17) in 24 ms on localhost (2/4) -2016-06-13 18:48:48 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 18). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 5.0 (TID 20) in 22 ms on localhost (3/4) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 18) in 26 ms on localhost (4/4) -2016-06-13 18:48:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 18:48:48 INFO DAGScheduler:59 - ResultStage 5 (aggregate at GaussianMixture.scala:201) finished in 0.027 s -2016-06-13 18:48:48 INFO DAGScheduler:59 - Job 5 finished: aggregate at GaussianMixture.scala:201, took 0.041842 s -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=29352, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 1344.0 B, free 730.6 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(1086) called with curMem=30696, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1086.0 B, free 730.6 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:65157 (size: 1086.0 B, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Created broadcast 9 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:48:48 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Got job 6 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:48:48 INFO DAGScheduler:59 - Final stage: ResultStage 6(aggregate at GaussianMixture.scala:201) -2016-06-13 18:48:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:48 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:48 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=31782, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.5 KB, free 730.6 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(2383) called with curMem=36414, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:65157 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:48 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 4 tasks -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 21, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 22, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 23, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 6.0 (TID 24, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO Executor:59 - Running task 3.0 in stage 6.0 (TID 24) -2016-06-13 18:48:48 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 21) -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:48 INFO Executor:59 - Finished task 3.0 in stage 6.0 (TID 24). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 22) -2016-06-13 18:48:48 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 23) -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 6.0 (TID 24) in 14 ms on localhost (1/4) -2016-06-13 18:48:48 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 21). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 22). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 23). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 21) in 22 ms on localhost (2/4) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 22) in 21 ms on localhost (3/4) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 23) in 21 ms on localhost (4/4) -2016-06-13 18:48:48 INFO DAGScheduler:59 - ResultStage 6 (aggregate at GaussianMixture.scala:201) finished in 0.023 s -2016-06-13 18:48:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 18:48:48 INFO DAGScheduler:59 - Job 6 finished: aggregate at GaussianMixture.scala:201, took 0.039345 s -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=38797, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(1083) called with curMem=40141, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 1083.0 B, free 730.5 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:65157 (size: 1083.0 B, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Created broadcast 11 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:48:48 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Got job 7 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:48:48 INFO DAGScheduler:59 - Final stage: ResultStage 7(aggregate at GaussianMixture.scala:201) -2016-06-13 18:48:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:48 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:48 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=41224, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(2379) called with curMem=45856, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:65157 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 7 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:48 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 4 tasks -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 25, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 26, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 7.0 (TID 27, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 7.0 (TID 28, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO Executor:59 - Running task 2.0 in stage 7.0 (TID 27) -2016-06-13 18:48:48 INFO Executor:59 - Running task 3.0 in stage 7.0 (TID 28) -2016-06-13 18:48:48 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 26) -2016-06-13 18:48:48 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 25) -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:48 INFO Executor:59 - Finished task 3.0 in stage 7.0 (TID 28). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 25). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 26). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO Executor:59 - Finished task 2.0 in stage 7.0 (TID 27). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 7.0 (TID 28) in 13 ms on localhost (1/4) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 25) in 17 ms on localhost (2/4) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 26) in 16 ms on localhost (3/4) -2016-06-13 18:48:48 INFO DAGScheduler:59 - ResultStage 7 (aggregate at GaussianMixture.scala:201) finished in 0.018 s -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 7.0 (TID 27) in 16 ms on localhost (4/4) -2016-06-13 18:48:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 18:48:48 INFO DAGScheduler:59 - Job 7 finished: aggregate at GaussianMixture.scala:201, took 0.031220 s -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=48235, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(1082) called with curMem=49579, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 1082.0 B, free 730.5 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:65157 (size: 1082.0 B, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Created broadcast 13 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:48:48 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Got job 8 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:48:48 INFO DAGScheduler:59 - Final stage: ResultStage 8(aggregate at GaussianMixture.scala:201) -2016-06-13 18:48:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:48 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:48 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=50661, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(2379) called with curMem=55293, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:65157 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 8 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:48 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 4 tasks -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 29, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 30, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 8.0 (TID 31, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 8.0 (TID 32, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 30) -2016-06-13 18:48:48 INFO Executor:59 - Running task 3.0 in stage 8.0 (TID 32) -2016-06-13 18:48:48 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 29) -2016-06-13 18:48:48 INFO Executor:59 - Running task 2.0 in stage 8.0 (TID 31) -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:48 INFO Executor:59 - Finished task 3.0 in stage 8.0 (TID 32). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 30). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO Executor:59 - Finished task 2.0 in stage 8.0 (TID 31). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 29). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 8.0 (TID 32) in 14 ms on localhost (1/4) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 30) in 17 ms on localhost (2/4) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 29) in 18 ms on localhost (3/4) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 8.0 (TID 31) in 17 ms on localhost (4/4) -2016-06-13 18:48:48 INFO DAGScheduler:59 - ResultStage 8 (aggregate at GaussianMixture.scala:201) finished in 0.019 s -2016-06-13 18:48:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 18:48:48 INFO DAGScheduler:59 - Job 8 finished: aggregate at GaussianMixture.scala:201, took 0.031898 s -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=57672, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(1080) called with curMem=59016, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1080.0 B, free 730.5 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:65157 (size: 1080.0 B, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Created broadcast 15 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:48:48 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Got job 9 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:48:48 INFO DAGScheduler:59 - Final stage: ResultStage 9(aggregate at GaussianMixture.scala:201) -2016-06-13 18:48:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:48 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:48 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=60096, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(2377) called with curMem=64728, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:65157 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 9 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:48 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 4 tasks -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 33, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 34, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 9.0 (TID 35, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 9.0 (TID 36, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 33) -2016-06-13 18:48:48 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 34) -2016-06-13 18:48:48 INFO Executor:59 - Running task 3.0 in stage 9.0 (TID 36) -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:48 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 33). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:48 INFO Executor:59 - Finished task 3.0 in stage 9.0 (TID 36). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 33) in 19 ms on localhost (1/4) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 9.0 (TID 36) in 18 ms on localhost (2/4) -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:48 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 34). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO Executor:59 - Running task 2.0 in stage 9.0 (TID 35) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 34) in 30 ms on localhost (3/4) -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:48 INFO Executor:59 - Finished task 2.0 in stage 9.0 (TID 35). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO DAGScheduler:59 - ResultStage 9 (aggregate at GaussianMixture.scala:201) finished in 0.041 s -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 9.0 (TID 35) in 37 ms on localhost (4/4) -2016-06-13 18:48:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 18:48:48 INFO DAGScheduler:59 - Job 9 finished: aggregate at GaussianMixture.scala:201, took 0.061106 s -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=67105, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(1084) called with curMem=68449, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1084.0 B, free 730.5 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:65157 (size: 1084.0 B, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Created broadcast 17 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:48:48 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Got job 10 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:48:48 INFO DAGScheduler:59 - Final stage: ResultStage 10(aggregate at GaussianMixture.scala:201) -2016-06-13 18:48:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:48 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:48 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=69533, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(2380) called with curMem=74165, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:65157 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 10 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:48 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 4 tasks -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:65157 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 37, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:48 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 38, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 10.0 (TID 39, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 10.0 (TID 40, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 38) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:65157 in memory (size: 1080.0 B, free: 730.6 MB) -2016-06-13 18:48:48 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 37) -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:48 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 38). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 37). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO Executor:59 - Running task 3.0 in stage 10.0 (TID 40) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 38) in 17 ms on localhost (1/4) -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 37) in 21 ms on localhost (2/4) -2016-06-13 18:48:48 INFO Executor:59 - Running task 2.0 in stage 10.0 (TID 39) -2016-06-13 18:48:48 INFO Executor:59 - Finished task 3.0 in stage 10.0 (TID 40). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 10.0 (TID 40) in 28 ms on localhost (3/4) -2016-06-13 18:48:48 INFO Executor:59 - Finished task 2.0 in stage 10.0 (TID 39). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:65157 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 10.0 (TID 39) in 35 ms on localhost (4/4) -2016-06-13 18:48:48 INFO DAGScheduler:59 - ResultStage 10 (aggregate at GaussianMixture.scala:201) finished in 0.039 s -2016-06-13 18:48:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 18:48:48 INFO DAGScheduler:59 - Job 10 finished: aggregate at GaussianMixture.scala:201, took 0.087749 s -2016-06-13 18:48:48 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=59019, maxMem=766075207 -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:65157 in memory (size: 1082.0 B, free: 730.6 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(1082) called with curMem=59019, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1082.0 B, free 730.5 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:65157 (size: 1082.0 B, free: 730.6 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:65157 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Created broadcast 19 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:48:48 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:65157 in memory (size: 1083.0 B, free: 730.6 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:65157 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Got job 11 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:48:48 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Final stage: ResultStage 11(aggregate at GaussianMixture.scala:201) -2016-06-13 18:48:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:48 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:48 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:65157 in memory (size: 1086.0 B, free: 730.6 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=41218, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(2376) called with curMem=45850, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:65157 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 11 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:65157 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:48 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 4 tasks -2016-06-13 18:48:48 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 41, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 42, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 11.0 (TID 43, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:65157 in memory (size: 1082.0 B, free: 730.6 MB) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 11.0 (TID 44, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 41) -2016-06-13 18:48:48 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 42) -2016-06-13 18:48:48 INFO Executor:59 - Running task 3.0 in stage 11.0 (TID 44) -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:48 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 41). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 41) in 17 ms on localhost (1/4) -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:48 INFO Executor:59 - Finished task 3.0 in stage 11.0 (TID 44). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 42). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 11.0 (TID 44) in 19 ms on localhost (2/4) -2016-06-13 18:48:48 INFO Executor:59 - Running task 2.0 in stage 11.0 (TID 43) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 42) in 25 ms on localhost (3/4) -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:65157 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:48 INFO Executor:59 - Finished task 2.0 in stage 11.0 (TID 43). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 18:48:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 11.0 (TID 43) in 33 ms on localhost (4/4) -2016-06-13 18:48:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 18:48:48 INFO DAGScheduler:59 - ResultStage 11 (aggregate at GaussianMixture.scala:201) finished in 0.037 s -2016-06-13 18:48:48 INFO DAGScheduler:59 - Job 11 finished: aggregate at GaussianMixture.scala:201, took 0.059329 s -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:65157 in memory (size: 1082.0 B, free: 730.6 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=29352, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 1344.0 B, free 730.6 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(1080) called with curMem=23710, maxMem=766075207 -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:65157 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1080.0 B, free 730.6 MB) -2016-06-13 18:48:48 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:65157 (size: 1080.0 B, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Created broadcast 21 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:65157 in memory (size: 1060.0 B, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Got job 12 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:48:48 INFO DAGScheduler:59 - Final stage: ResultStage 12(aggregate at GaussianMixture.scala:201) -2016-06-13 18:48:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:48 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:48 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=22386, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.5 KB, free 730.6 MB) -2016-06-13 18:48:48 INFO MemoryStore:59 - ensureFreeSpace(2377) called with curMem=27018, maxMem=766075207 -2016-06-13 18:48:48 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.6 MB) -2016-06-13 18:48:48 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:65157 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:48 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 12 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:48 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 4 tasks -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 45, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 46, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 12.0 (TID 47, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 12.0 (TID 48, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:48 INFO Executor:59 - Running task 3.0 in stage 12.0 (TID 48) -2016-06-13 18:48:48 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 46) -2016-06-13 18:48:48 INFO Executor:59 - Running task 2.0 in stage 12.0 (TID 47) -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:48 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:48 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 46). 2934 bytes result sent to driver -2016-06-13 18:48:48 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 45) -2016-06-13 18:48:48 INFO Executor:59 - Finished task 3.0 in stage 12.0 (TID 48). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:49 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 45). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Finished task 2.0 in stage 12.0 (TID 47). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 3.0 in stage 12.0 (TID 48) in 23 ms on localhost (1/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 46) in 28 ms on localhost (2/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 45) in 30 ms on localhost (3/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 12.0 (TID 47) in 27 ms on localhost (4/4) -2016-06-13 18:48:49 INFO DAGScheduler:59 - ResultStage 12 (aggregate at GaussianMixture.scala:201) finished in 0.030 s -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 18:48:49 INFO DAGScheduler:59 - Job 12 finished: aggregate at GaussianMixture.scala:201, took 0.047041 s -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=29395, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 1344.0 B, free 730.6 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1081) called with curMem=30739, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 1081.0 B, free 730.6 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:65157 (size: 1081.0 B, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 23 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:48:49 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Got job 13 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:48:49 INFO DAGScheduler:59 - Final stage: ResultStage 13(aggregate at GaussianMixture.scala:201) -2016-06-13 18:48:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=31820, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 4.5 KB, free 730.6 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(2377) called with curMem=36452, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:65157 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 13 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 4 tasks -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 49, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 50, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 13.0 (TID 51, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 3.0 in stage 13.0 (TID 52, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO Executor:59 - Running task 2.0 in stage 13.0 (TID 51) -2016-06-13 18:48:49 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 49) -2016-06-13 18:48:49 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 50) -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:49 INFO Executor:59 - Finished task 2.0 in stage 13.0 (TID 51). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Running task 3.0 in stage 13.0 (TID 52) -2016-06-13 18:48:49 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 50). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:49 INFO Executor:59 - Finished task 3.0 in stage 13.0 (TID 52). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 49). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 13.0 (TID 51) in 21 ms on localhost (1/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 50) in 24 ms on localhost (2/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 3.0 in stage 13.0 (TID 52) in 23 ms on localhost (3/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 49) in 27 ms on localhost (4/4) -2016-06-13 18:48:49 INFO DAGScheduler:59 - ResultStage 13 (aggregate at GaussianMixture.scala:201) finished in 0.028 s -2016-06-13 18:48:49 INFO DAGScheduler:59 - Job 13 finished: aggregate at GaussianMixture.scala:201, took 0.046759 s -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=38829, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1082) called with curMem=40173, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 1082.0 B, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:65157 (size: 1082.0 B, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 25 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:48:49 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Got job 14 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:48:49 INFO DAGScheduler:59 - Final stage: ResultStage 14(aggregate at GaussianMixture.scala:201) -2016-06-13 18:48:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=41255, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(2378) called with curMem=45887, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:65157 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 14 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 4 tasks -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 53, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 54, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 55, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 3.0 in stage 14.0 (TID 56, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 53) -2016-06-13 18:48:49 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 55) -2016-06-13 18:48:49 INFO Executor:59 - Running task 3.0 in stage 14.0 (TID 56) -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:49 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 55). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Finished task 3.0 in stage 14.0 (TID 56). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 54) -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:49 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 53). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 54). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 55) in 21 ms on localhost (1/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 53) in 25 ms on localhost (2/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 3.0 in stage 14.0 (TID 56) in 24 ms on localhost (3/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 54) in 27 ms on localhost (4/4) -2016-06-13 18:48:49 INFO DAGScheduler:59 - ResultStage 14 (aggregate at GaussianMixture.scala:201) finished in 0.028 s -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 18:48:49 INFO DAGScheduler:59 - Job 14 finished: aggregate at GaussianMixture.scala:201, took 0.042049 s -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=48265, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1082) called with curMem=49609, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 1082.0 B, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:65157 (size: 1082.0 B, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 27 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:48:49 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Got job 15 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:48:49 INFO DAGScheduler:59 - Final stage: ResultStage 15(aggregate at GaussianMixture.scala:201) -2016-06-13 18:48:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=50691, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(2378) called with curMem=55323, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:65157 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 15 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 4 tasks -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 57, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 58, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 15.0 (TID 59, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 3.0 in stage 15.0 (TID 60, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO Executor:59 - Running task 2.0 in stage 15.0 (TID 59) -2016-06-13 18:48:49 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 58) -2016-06-13 18:48:49 INFO Executor:59 - Running task 3.0 in stage 15.0 (TID 60) -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:49 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 57) -2016-06-13 18:48:49 INFO Executor:59 - Finished task 3.0 in stage 15.0 (TID 60). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:49 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 57). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Finished task 2.0 in stage 15.0 (TID 59). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 58). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 3.0 in stage 15.0 (TID 60) in 19 ms on localhost (1/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 57) in 23 ms on localhost (2/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 15.0 (TID 59) in 22 ms on localhost (3/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 58) in 24 ms on localhost (4/4) -2016-06-13 18:48:49 INFO DAGScheduler:59 - ResultStage 15 (aggregate at GaussianMixture.scala:201) finished in 0.027 s -2016-06-13 18:48:49 INFO DAGScheduler:59 - Job 15 finished: aggregate at GaussianMixture.scala:201, took 0.041132 s -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=57701, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1084) called with curMem=59045, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1084.0 B, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:65157 (size: 1084.0 B, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 29 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:48:49 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Got job 16 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:48:49 INFO DAGScheduler:59 - Final stage: ResultStage 16(aggregate at GaussianMixture.scala:201) -2016-06-13 18:48:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=60129, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(2378) called with curMem=64761, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:65157 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 16 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 4 tasks -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 61, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 62, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 16.0 (TID 63, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 3.0 in stage 16.0 (TID 64, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 62) -2016-06-13 18:48:49 INFO Executor:59 - Running task 2.0 in stage 16.0 (TID 63) -2016-06-13 18:48:49 INFO Executor:59 - Running task 3.0 in stage 16.0 (TID 64) -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:49 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 62). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 61) -2016-06-13 18:48:49 INFO Executor:59 - Finished task 3.0 in stage 16.0 (TID 64). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:49 INFO Executor:59 - Finished task 2.0 in stage 16.0 (TID 63). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:49 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 61). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 62) in 26 ms on localhost (1/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 3.0 in stage 16.0 (TID 64) in 24 ms on localhost (2/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 16.0 (TID 63) in 28 ms on localhost (3/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 61) in 31 ms on localhost (4/4) -2016-06-13 18:48:49 INFO DAGScheduler:59 - ResultStage 16 (aggregate at GaussianMixture.scala:201) finished in 0.031 s -2016-06-13 18:48:49 INFO DAGScheduler:59 - Job 16 finished: aggregate at GaussianMixture.scala:201, took 0.047832 s -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=67139, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1090) called with curMem=68483, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1090.0 B, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:65157 (size: 1090.0 B, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 31 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:48:49 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Got job 17 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:48:49 INFO DAGScheduler:59 - Final stage: ResultStage 17(aggregate at GaussianMixture.scala:201) -2016-06-13 18:48:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=69573, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(2376) called with curMem=74205, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:65157 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 65, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 66, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 67, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 68, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 67) -2016-06-13 18:48:49 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 68) -2016-06-13 18:48:49 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 66) -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:49 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 67). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 65) -2016-06-13 18:48:49 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 66). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:49 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 65). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 68). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 67) in 23 ms on localhost (1/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 66) in 25 ms on localhost (2/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 65) in 27 ms on localhost (3/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 68) in 25 ms on localhost (4/4) -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 18:48:49 INFO DAGScheduler:59 - ResultStage 17 (aggregate at GaussianMixture.scala:201) finished in 0.029 s -2016-06-13 18:48:49 INFO DAGScheduler:59 - Job 17 finished: aggregate at GaussianMixture.scala:201, took 0.043467 s -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=76581, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1082) called with curMem=77925, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 1082.0 B, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:65157 (size: 1082.0 B, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 33 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:48:49 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Got job 18 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:48:49 INFO DAGScheduler:59 - Final stage: ResultStage 18(aggregate at GaussianMixture.scala:201) -2016-06-13 18:48:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=79007, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(2378) called with curMem=83639, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:65157 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:65157 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:49 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 69, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 70, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 71, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 72, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 70) -2016-06-13 18:48:49 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 69) -2016-06-13 18:48:49 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 71) -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:49 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 70). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:65157 in memory (size: 1090.0 B, free: 730.6 MB) -2016-06-13 18:48:49 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 71). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 70) in 16 ms on localhost (1/4) -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 71) in 17 ms on localhost (2/4) -2016-06-13 18:48:49 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 69). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 72) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 69) in 25 ms on localhost (3/4) -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:49 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 72). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:65157 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 72) in 29 ms on localhost (4/4) -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 18:48:49 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 18:48:49 INFO DAGScheduler:59 - ResultStage 18 (aggregate at GaussianMixture.scala:201) finished in 0.033 s -2016-06-13 18:48:49 INFO DAGScheduler:59 - Job 18 finished: aggregate at GaussianMixture.scala:201, took 0.077107 s -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:65157 in memory (size: 1084.0 B, free: 730.6 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=67137, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1083) called with curMem=68481, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 1083.0 B, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:65157 (size: 1083.0 B, free: 730.6 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:65157 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 35 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:48:49 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:65157 in memory (size: 1082.0 B, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Got job 19 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:48:49 INFO DAGScheduler:59 - Final stage: ResultStage 19(aggregate at GaussianMixture.scala:201) -2016-06-13 18:48:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:65157 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:49 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=52036, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:65157 in memory (size: 1082.0 B, free: 730.6 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(2378) called with curMem=55324, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:65157 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:65157 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 19 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 4 tasks -2016-06-13 18:48:49 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 73, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 74, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 19.0 (TID 75, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 3.0 in stage 19.0 (TID 76, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO Executor:59 - Running task 2.0 in stage 19.0 (TID 75) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:65157 in memory (size: 1081.0 B, free: 730.6 MB) -2016-06-13 18:48:49 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 73) -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:49 INFO Executor:59 - Finished task 2.0 in stage 19.0 (TID 75). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 73). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Running task 3.0 in stage 19.0 (TID 76) -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:49 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 74) -2016-06-13 18:48:49 INFO Executor:59 - Finished task 3.0 in stage 19.0 (TID 76). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 19.0 (TID 75) in 18 ms on localhost (1/4) -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 73) in 22 ms on localhost (2/4) -2016-06-13 18:48:49 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 74). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 3.0 in stage 19.0 (TID 76) in 24 ms on localhost (3/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 74) in 28 ms on localhost (4/4) -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 18:48:49 INFO DAGScheduler:59 - ResultStage 19 (aggregate at GaussianMixture.scala:201) finished in 0.029 s -2016-06-13 18:48:49 INFO DAGScheduler:59 - Job 19 finished: aggregate at GaussianMixture.scala:201, took 0.046583 s -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:65157 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=41259, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1084) called with curMem=42603, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 1084.0 B, free 730.5 MB) -2016-06-13 18:48:49 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:65157 (size: 1084.0 B, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 37 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:48:49 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:65157 in memory (size: 1080.0 B, free: 730.6 MB) -2016-06-13 18:48:49 INFO DAGScheduler:59 - Got job 20 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:48:49 INFO DAGScheduler:59 - Final stage: ResultStage 20(aggregate at GaussianMixture.scala:201) -2016-06-13 18:48:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=34255, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:65157 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(2380) called with curMem=38887, maxMem=766075207 -2016-06-13 18:48:49 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:65157 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:65157 in memory (size: 1082.0 B, free: 730.6 MB) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 77, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 78, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 79, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 80, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 79) -2016-06-13 18:48:49 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 77) -2016-06-13 18:48:49 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 78) -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:49 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 77). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:49 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 80) -2016-06-13 18:48:49 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 78). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:49 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 80). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 79). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 77) in 22 ms on localhost (1/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 78) in 24 ms on localhost (2/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 80) in 21 ms on localhost (3/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 79) in 24 ms on localhost (4/4) -2016-06-13 18:48:49 INFO DAGScheduler:59 - ResultStage 20 (aggregate at GaussianMixture.scala:201) finished in 0.027 s -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 18:48:49 INFO DAGScheduler:59 - Job 20 finished: aggregate at GaussianMixture.scala:201, took 0.042688 s -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:65157 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=31829, maxMem=766075207 -2016-06-13 18:48:49 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 1344.0 B, free 730.6 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1083) called with curMem=32089, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 1083.0 B, free 730.6 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:65157 in memory (size: 1084.0 B, free: 730.6 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:65157 (size: 1083.0 B, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 39 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:48:49 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Got job 21 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:48:49 INFO DAGScheduler:59 - Final stage: ResultStage 21(aggregate at GaussianMixture.scala:201) -2016-06-13 18:48:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=31828, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 4.5 KB, free 730.6 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(2379) called with curMem=36460, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:65157 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 81, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 82, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 83, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 84, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 84) -2016-06-13 18:48:49 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 83) -2016-06-13 18:48:49 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 81) -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:49 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 82) -2016-06-13 18:48:49 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 81). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:49 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 82). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 84). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 83). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 81) in 23 ms on localhost (1/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 83) in 24 ms on localhost (2/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 84) in 23 ms on localhost (3/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 82) in 26 ms on localhost (4/4) -2016-06-13 18:48:49 INFO DAGScheduler:59 - ResultStage 21 (aggregate at GaussianMixture.scala:201) finished in 0.027 s -2016-06-13 18:48:49 INFO DAGScheduler:59 - Job 21 finished: aggregate at GaussianMixture.scala:201, took 0.042008 s -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=38839, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1084) called with curMem=40183, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 1084.0 B, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:65157 (size: 1084.0 B, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 41 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:48:49 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Got job 22 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:48:49 INFO DAGScheduler:59 - Final stage: ResultStage 22(aggregate at GaussianMixture.scala:201) -2016-06-13 18:48:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=41267, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(2381) called with curMem=45899, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:65157 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 85, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 86, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 87, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 88, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:48:49 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 88) -2016-06-13 18:48:49 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 86) -2016-06-13 18:48:49 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 87) -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:48:49 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 88). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 87). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 85) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 88) in 10 ms on localhost (1/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 87) in 13 ms on localhost (2/4) -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:48:49 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 85). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 85) in 20 ms on localhost (3/4) -2016-06-13 18:48:49 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 86). 2934 bytes result sent to driver -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 86) in 22 ms on localhost (4/4) -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 18:48:49 INFO DAGScheduler:59 - ResultStage 22 (aggregate at GaussianMixture.scala:201) finished in 0.024 s -2016-06-13 18:48:49 INFO DAGScheduler:59 - Job 22 finished: aggregate at GaussianMixture.scala:201, took 0.038713 s -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(1272) called with curMem=48280, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 1272.0 B, free 730.5 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(968) called with curMem=49552, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 968.0 B, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:65157 (size: 968.0 B, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 43 from broadcast at GaussianMixtureModel.scala:100 -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=50520, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 72.0 B, free 730.5 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=50592, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:65157 (size: 81.0 B, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 44 from broadcast at GaussianMixtureModel.scala:101 -2016-06-13 18:48:49 INFO SparkContext:59 - Starting job: collect at GaussianMixtureFromSparkSuite.scala:201 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Got job 23 (collect at GaussianMixtureFromSparkSuite.scala:201) with 4 output partitions -2016-06-13 18:48:49 INFO DAGScheduler:59 - Final stage: ResultStage 23(collect at GaussianMixtureFromSparkSuite.scala:201) -2016-06-13 18:48:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting ResultStage 23 (ZippedPartitionsRDD2[5] at zip at GaussianMixtureFromSparkSuite.scala:201), which has no missing parents -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(5072) called with curMem=50673, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 5.0 KB, free 730.5 MB) -2016-06-13 18:48:49 INFO MemoryStore:59 - ensureFreeSpace(2781) called with curMem=55745, maxMem=766075207 -2016-06-13 18:48:49 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.5 MB) -2016-06-13 18:48:49 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:65157 (size: 2.7 KB, free: 730.6 MB) -2016-06-13 18:48:49 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 23 (ZippedPartitionsRDD2[5] at zip at GaussianMixtureFromSparkSuite.scala:201) -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 89, localhost, PROCESS_LOCAL, 2432 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 90, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 91, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 92, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 18:48:49 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 92) -2016-06-13 18:48:49 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 89) -2016-06-13 18:48:49 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 91) -2016-06-13 18:48:49 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 90) -2016-06-13 18:48:49 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 92). 1264 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 90). 1274 bytes result sent to driver -2016-06-13 18:48:49 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 91). 1264 bytes result sent to driver -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 92) in 22 ms on localhost (1/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 90) in 25 ms on localhost (2/4) -2016-06-13 18:48:49 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 89). 1234 bytes result sent to driver -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 91) in 27 ms on localhost (3/4) -2016-06-13 18:48:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 89) in 32 ms on localhost (4/4) -2016-06-13 18:48:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 18:48:49 INFO DAGScheduler:59 - ResultStage 23 (collect at GaussianMixtureFromSparkSuite.scala:201) finished in 0.033 s -2016-06-13 18:48:49 INFO DAGScheduler:59 - Job 23 finished: collect at GaussianMixtureFromSparkSuite.scala:201, took 0.047191 s -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 18:48:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 18:48:49 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 18:48:49 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 18:48:50 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 18:48:50 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 18:48:50 INFO BlockManager:59 - BlockManager stopped -2016-06-13 18:48:50 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 18:48:50 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 18:48:50 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 18:48:50 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 18:48:50 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 18:48:50 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 18:48:50 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-f0bbc7d6-39c9-4b0a-8ad2-3299aa18dc98 -2016-06-13 18:50:04 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 18:50:05 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 18:50:05 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 18:50:05 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 18:50:05 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 18:50:06 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 18:50:06 INFO Remoting:74 - Starting remoting -2016-06-13 18:50:06 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:65175] -2016-06-13 18:50:06 INFO Utils:59 - Successfully started service 'sparkDriver' on port 65175. -2016-06-13 18:50:06 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 18:50:06 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 18:50:06 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-c0c00437-507a-4197-8658-88842b22840f -2016-06-13 18:50:06 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 18:50:06 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-f086406a-a2eb-434d-9520-6599d4a262f4\httpd-3cc02545-2ece-4e42-af4d-e2f50821b5f3 -2016-06-13 18:50:06 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 18:50:06 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 18:50:06 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:65176 -2016-06-13 18:50:06 INFO Utils:59 - Successfully started service 'HTTP file server' on port 65176. -2016-06-13 18:50:06 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 18:50:06 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 18:50:06 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 18:50:06 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 18:50:06 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 18:50:06 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 18:50:06 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 18:50:07 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 65195. -2016-06-13 18:50:07 INFO NettyBlockTransferService:59 - Server created on 65195 -2016-06-13 18:50:07 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 18:50:07 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:65195 with 730.6 MB RAM, BlockManagerId(driver, localhost, 65195) -2016-06-13 18:50:07 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 18:50:07 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 18:50:07 INFO DAGScheduler:59 - Got job 0 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 18:50:07 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GaussianMixture.scala:171) -2016-06-13 18:50:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:50:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:50:07 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:50:08 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=0, maxMem=766075207 -2016-06-13 18:50:08 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 18:50:08 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1992, maxMem=766075207 -2016-06-13 18:50:08 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.6 MB) -2016-06-13 18:50:08 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:65195 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 18:50:08 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:50:08 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:50:08 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 18:50:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:50:08 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 18:50:08 INFO CacheManager:59 - Partition rdd_1_0 not found, computing it -2016-06-13 18:50:08 INFO MemoryStore:59 - ensureFreeSpace(224) called with curMem=3242, maxMem=766075207 -2016-06-13 18:50:08 INFO MemoryStore:59 - Block rdd_1_0 stored as values in memory (estimated size 224.0 B, free 730.6 MB) -2016-06-13 18:50:08 INFO BlockManagerInfo:59 - Added rdd_1_0 in memory on localhost:65195 (size: 224.0 B, free: 730.6 MB) -2016-06-13 18:50:08 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1688 bytes result sent to driver -2016-06-13 18:50:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 376 ms on localhost (1/1) -2016-06-13 18:50:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 18:50:08 INFO DAGScheduler:59 - ResultStage 0 (first at GaussianMixture.scala:171) finished in 0.388 s -2016-06-13 18:50:08 INFO DAGScheduler:59 - Job 0 finished: first at GaussianMixture.scala:171, took 0.573597 s -2016-06-13 18:50:08 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 18:50:08 INFO DAGScheduler:59 - Got job 1 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 18:50:08 INFO DAGScheduler:59 - Final stage: ResultStage 1(takeSample at GaussianMixture.scala:184) -2016-06-13 18:50:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:50:08 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:50:08 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:50:08 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=3466, maxMem=766075207 -2016-06-13 18:50:08 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 18:50:08 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=5298, maxMem=766075207 -2016-06-13 18:50:08 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.6 MB) -2016-06-13 18:50:08 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:65195 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 18:50:08 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:50:08 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:50:08 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 4 tasks -2016-06-13 18:50:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:50:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:08 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:08 INFO TaskSetManager:59 - Starting task 3.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:08 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 18:50:08 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 3) -2016-06-13 18:50:08 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 18:50:08 INFO CacheManager:59 - Partition rdd_1_2 not found, computing it -2016-06-13 18:50:08 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=6493, maxMem=766075207 -2016-06-13 18:50:08 INFO MemoryStore:59 - Block rdd_1_2 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 18:50:08 INFO Executor:59 - Running task 3.0 in stage 1.0 (TID 4) -2016-06-13 18:50:08 INFO BlockManagerInfo:59 - Added rdd_1_2 in memory on localhost:65195 (size: 288.0 B, free: 730.6 MB) -2016-06-13 18:50:08 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 3). 1567 bytes result sent to driver -2016-06-13 18:50:08 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 3) in 32 ms on localhost (1/4) -2016-06-13 18:50:08 INFO CacheManager:59 - Partition rdd_1_1 not found, computing it -2016-06-13 18:50:08 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=6781, maxMem=766075207 -2016-06-13 18:50:08 INFO MemoryStore:59 - Block rdd_1_1 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 18:50:08 INFO CacheManager:59 - Partition rdd_1_3 not found, computing it -2016-06-13 18:50:08 INFO BlockManagerInfo:59 - Added rdd_1_1 in memory on localhost:65195 (size: 288.0 B, free: 730.6 MB) -2016-06-13 18:50:08 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 1567 bytes result sent to driver -2016-06-13 18:50:08 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:50:08 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=7069, maxMem=766075207 -2016-06-13 18:50:08 INFO MemoryStore:59 - Block rdd_1_3 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 18:50:08 INFO BlockManagerInfo:59 - Added rdd_1_3 in memory on localhost:65195 (size: 288.0 B, free: 730.6 MB) -2016-06-13 18:50:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 47 ms on localhost (2/4) -2016-06-13 18:50:08 INFO Executor:59 - Finished task 3.0 in stage 1.0 (TID 4). 1567 bytes result sent to driver -2016-06-13 18:50:08 INFO TaskSetManager:59 - Finished task 3.0 in stage 1.0 (TID 4) in 49 ms on localhost (3/4) -2016-06-13 18:50:08 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 18:50:08 INFO DAGScheduler:59 - ResultStage 1 (takeSample at GaussianMixture.scala:184) finished in 0.069 s -2016-06-13 18:50:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 67 ms on localhost (4/4) -2016-06-13 18:50:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 18:50:08 INFO DAGScheduler:59 - Job 1 finished: takeSample at GaussianMixture.scala:184, took 0.092370 s -2016-06-13 18:50:08 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 18:50:08 INFO DAGScheduler:59 - Got job 2 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 18:50:08 INFO DAGScheduler:59 - Final stage: ResultStage 2(takeSample at GaussianMixture.scala:184) -2016-06-13 18:50:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:50:08 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:50:08 INFO DAGScheduler:59 - Submitting ResultStage 2 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 18:50:08 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=7357, maxMem=766075207 -2016-06-13 18:50:08 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 18.8 KB, free 730.6 MB) -2016-06-13 18:50:08 INFO MemoryStore:59 - ensureFreeSpace(7883) called with curMem=26581, maxMem=766075207 -2016-06-13 18:50:08 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.6 MB) -2016-06-13 18:50:08 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:65195 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 18:50:08 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:50:08 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 2 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184) -2016-06-13 18:50:08 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 4 tasks -2016-06-13 18:50:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 18:50:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 18:50:08 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 18:50:08 INFO TaskSetManager:59 - Starting task 3.0 in stage 2.0 (TID 8, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 18:50:08 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 5) -2016-06-13 18:50:08 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 7) -2016-06-13 18:50:08 INFO Executor:59 - Running task 3.0 in stage 2.0 (TID 8) -2016-06-13 18:50:08 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 6) -2016-06-13 18:50:08 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:50:08 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:50:08 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:50:08 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 7). 2303 bytes result sent to driver -2016-06-13 18:50:08 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:50:08 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 6). 2344 bytes result sent to driver -2016-06-13 18:50:08 INFO Executor:59 - Finished task 3.0 in stage 2.0 (TID 8). 2339 bytes result sent to driver -2016-06-13 18:50:08 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 5). 2247 bytes result sent to driver -2016-06-13 18:50:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 6) in 30 ms on localhost (1/4) -2016-06-13 18:50:08 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 7) in 29 ms on localhost (2/4) -2016-06-13 18:50:08 INFO TaskSetManager:59 - Finished task 3.0 in stage 2.0 (TID 8) in 30 ms on localhost (3/4) -2016-06-13 18:50:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 5) in 35 ms on localhost (4/4) -2016-06-13 18:50:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 18:50:08 INFO DAGScheduler:59 - ResultStage 2 (takeSample at GaussianMixture.scala:184) finished in 0.035 s -2016-06-13 18:50:08 INFO DAGScheduler:59 - Job 2 finished: takeSample at GaussianMixture.scala:184, took 0.052985 s -2016-06-13 18:50:08 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 18:50:08 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 18:50:08 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 18:50:08 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=34464, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 712.0 B, free 730.6 MB) -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(937) called with curMem=35176, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 937.0 B, free 730.6 MB) -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:65195 (size: 937.0 B, free: 730.6 MB) -2016-06-13 18:50:09 INFO SparkContext:59 - Created broadcast 3 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:50:09 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:50:09 INFO DAGScheduler:59 - Got job 3 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:50:09 INFO DAGScheduler:59 - Final stage: ResultStage 3(aggregate at GaussianMixture.scala:201) -2016-06-13 18:50:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:50:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:50:09 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=36113, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(2237) called with curMem=40209, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:65195 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:50:09 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:50:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 3 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:50:09 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 4 tasks -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 11, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 3.0 (TID 12, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:09 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 10) -2016-06-13 18:50:09 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 9) -2016-06-13 18:50:09 INFO Executor:59 - Running task 3.0 in stage 3.0 (TID 12) -2016-06-13 18:50:09 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 11) -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:50:09 INFO Executor:59 - Finished task 3.0 in stage 3.0 (TID 12). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 9). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:50:09 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 11). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 10). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 3.0 (TID 12) in 28 ms on localhost (1/4) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 9) in 35 ms on localhost (2/4) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 10) in 35 ms on localhost (3/4) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 11) in 35 ms on localhost (4/4) -2016-06-13 18:50:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 18:50:09 INFO DAGScheduler:59 - ResultStage 3 (aggregate at GaussianMixture.scala:201) finished in 0.038 s -2016-06-13 18:50:09 INFO DAGScheduler:59 - Job 3 finished: aggregate at GaussianMixture.scala:201, took 0.054539 s -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=42446, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 712.0 B, free 730.5 MB) -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(934) called with curMem=43158, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 934.0 B, free 730.5 MB) -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:65195 (size: 934.0 B, free: 730.6 MB) -2016-06-13 18:50:09 INFO SparkContext:59 - Created broadcast 5 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:50:09 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:50:09 INFO DAGScheduler:59 - Got job 4 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:50:09 INFO DAGScheduler:59 - Final stage: ResultStage 4(aggregate at GaussianMixture.scala:201) -2016-06-13 18:50:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:50:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:50:09 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=44092, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=48188, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:65195 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:50:09 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:50:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:50:09 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 4 tasks -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 14, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 15, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 4.0 (TID 16, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:09 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 13) -2016-06-13 18:50:09 INFO Executor:59 - Running task 3.0 in stage 4.0 (TID 16) -2016-06-13 18:50:09 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 15) -2016-06-13 18:50:09 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 14) -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:50:09 INFO Executor:59 - Finished task 3.0 in stage 4.0 (TID 16). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:50:09 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 13). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 15). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 4.0 (TID 16) in 21 ms on localhost (1/4) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 13) in 27 ms on localhost (2/4) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 15) in 27 ms on localhost (3/4) -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:50:09 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 14). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 14) in 36 ms on localhost (4/4) -2016-06-13 18:50:09 INFO DAGScheduler:59 - ResultStage 4 (aggregate at GaussianMixture.scala:201) finished in 0.038 s -2016-06-13 18:50:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 18:50:09 INFO DAGScheduler:59 - Job 4 finished: aggregate at GaussianMixture.scala:201, took 0.055862 s -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=50426, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 712.0 B, free 730.5 MB) -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=51138, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.5 MB) -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:65195 (size: 932.0 B, free: 730.6 MB) -2016-06-13 18:50:09 INFO SparkContext:59 - Created broadcast 7 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:50:09 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:50:09 INFO DAGScheduler:59 - Got job 5 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:50:09 INFO DAGScheduler:59 - Final stage: ResultStage 5(aggregate at GaussianMixture.scala:201) -2016-06-13 18:50:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:50:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:50:09 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=52070, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(2238) called with curMem=56166, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:65195 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:50:09 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:50:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:50:09 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 4 tasks -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 17, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 18, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 19, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 5.0 (TID 20, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:09 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 17) -2016-06-13 18:50:09 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 18) -2016-06-13 18:50:09 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 19) -2016-06-13 18:50:09 INFO Executor:59 - Running task 3.0 in stage 5.0 (TID 20) -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:50:09 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 19). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO Executor:59 - Finished task 3.0 in stage 5.0 (TID 20). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:50:09 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 18). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 17). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 19) in 18 ms on localhost (1/4) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 5.0 (TID 20) in 18 ms on localhost (2/4) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 17) in 23 ms on localhost (3/4) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 18) in 23 ms on localhost (4/4) -2016-06-13 18:50:09 INFO DAGScheduler:59 - ResultStage 5 (aggregate at GaussianMixture.scala:201) finished in 0.024 s -2016-06-13 18:50:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 18:50:09 INFO DAGScheduler:59 - Job 5 finished: aggregate at GaussianMixture.scala:201, took 0.039215 s -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=58404, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 712.0 B, free 730.5 MB) -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(934) called with curMem=59116, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 934.0 B, free 730.5 MB) -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:65195 (size: 934.0 B, free: 730.6 MB) -2016-06-13 18:50:09 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 18:50:09 INFO SparkContext:59 - Created broadcast 9 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:50:09 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:50:09 INFO DAGScheduler:59 - Got job 6 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:50:09 INFO DAGScheduler:59 - Final stage: ResultStage 6(aggregate at GaussianMixture.scala:201) -2016-06-13 18:50:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:50:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:50:09 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:65195 in memory (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:50:09 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=53716, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(2240) called with curMem=56168, maxMem=766075207 -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:65195 in memory (size: 932.0 B, free: 730.6 MB) -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:65195 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:50:09 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:65195 in memory (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:50:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:50:09 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 4 tasks -2016-06-13 18:50:09 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 21, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 22, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:65195 in memory (size: 934.0 B, free: 730.6 MB) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 23, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 6.0 (TID 24, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:09 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 21) -2016-06-13 18:50:09 INFO Executor:59 - Running task 3.0 in stage 6.0 (TID 24) -2016-06-13 18:50:09 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 23) -2016-06-13 18:50:09 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 22) -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:65195 in memory (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:50:09 INFO Executor:59 - Finished task 3.0 in stage 6.0 (TID 24). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:50:09 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 22). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:50:09 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 23). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:65195 in memory (size: 937.0 B, free: 730.6 MB) -2016-06-13 18:50:09 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 21). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 6.0 (TID 24) in 17 ms on localhost (1/4) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 22) in 21 ms on localhost (2/4) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 23) in 21 ms on localhost (3/4) -2016-06-13 18:50:09 INFO DAGScheduler:59 - ResultStage 6 (aggregate at GaussianMixture.scala:201) finished in 0.025 s -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 21) in 25 ms on localhost (4/4) -2016-06-13 18:50:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 18:50:09 INFO DAGScheduler:59 - Job 6 finished: aggregate at GaussianMixture.scala:201, took 0.041547 s -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:65195 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 18:50:09 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=14144, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 712.0 B, free 730.6 MB) -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:65195 in memory (size: 1195.0 B, free: 730.6 MB) -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(932) called with curMem=11032, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 932.0 B, free 730.6 MB) -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:65195 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 18:50:09 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:65195 (size: 932.0 B, free: 730.6 MB) -2016-06-13 18:50:09 INFO SparkContext:59 - Created broadcast 11 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:50:09 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:50:09 INFO DAGScheduler:59 - Got job 7 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:50:09 INFO DAGScheduler:59 - Final stage: ResultStage 7(aggregate at GaussianMixture.scala:201) -2016-06-13 18:50:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:50:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:50:09 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=10714, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.0 KB, free 730.6 MB) -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(2240) called with curMem=14810, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.6 MB) -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:65195 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:50:09 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:50:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 7 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:50:09 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 4 tasks -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 25, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 26, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 7.0 (TID 27, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 7.0 (TID 28, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:09 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 26) -2016-06-13 18:50:09 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 25) -2016-06-13 18:50:09 INFO Executor:59 - Running task 2.0 in stage 7.0 (TID 27) -2016-06-13 18:50:09 INFO Executor:59 - Running task 3.0 in stage 7.0 (TID 28) -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:50:09 INFO Executor:59 - Finished task 3.0 in stage 7.0 (TID 28). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO Executor:59 - Finished task 2.0 in stage 7.0 (TID 27). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 25). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 26). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 7.0 (TID 28) in 15 ms on localhost (1/4) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 7.0 (TID 27) in 17 ms on localhost (2/4) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 25) in 19 ms on localhost (3/4) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 26) in 18 ms on localhost (4/4) -2016-06-13 18:50:09 INFO DAGScheduler:59 - ResultStage 7 (aggregate at GaussianMixture.scala:201) finished in 0.021 s -2016-06-13 18:50:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 18:50:09 INFO DAGScheduler:59 - Job 7 finished: aggregate at GaussianMixture.scala:201, took 0.033305 s -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(712) called with curMem=17050, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 712.0 B, free 730.6 MB) -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(936) called with curMem=17762, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 936.0 B, free 730.6 MB) -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:65195 (size: 936.0 B, free: 730.6 MB) -2016-06-13 18:50:09 INFO SparkContext:59 - Created broadcast 13 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:50:09 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:50:09 INFO DAGScheduler:59 - Got job 8 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:50:09 INFO DAGScheduler:59 - Final stage: ResultStage 8(aggregate at GaussianMixture.scala:201) -2016-06-13 18:50:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:50:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:50:09 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=18698, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 4.0 KB, free 730.6 MB) -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(2241) called with curMem=22794, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.6 MB) -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:65195 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 18:50:09 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:50:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 8 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:50:09 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 4 tasks -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 29, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 30, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 8.0 (TID 31, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 8.0 (TID 32, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:50:09 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 29) -2016-06-13 18:50:09 INFO Executor:59 - Running task 2.0 in stage 8.0 (TID 31) -2016-06-13 18:50:09 INFO Executor:59 - Running task 3.0 in stage 8.0 (TID 32) -2016-06-13 18:50:09 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 30) -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:50:09 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 29). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 29) in 12 ms on localhost (1/4) -2016-06-13 18:50:09 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 30). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO Executor:59 - Finished task 3.0 in stage 8.0 (TID 32). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 30) in 15 ms on localhost (2/4) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 8.0 (TID 32) in 14 ms on localhost (3/4) -2016-06-13 18:50:09 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:50:09 INFO Executor:59 - Finished task 2.0 in stage 8.0 (TID 31). 2762 bytes result sent to driver -2016-06-13 18:50:09 INFO DAGScheduler:59 - ResultStage 8 (aggregate at GaussianMixture.scala:201) finished in 0.021 s -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 8.0 (TID 31) in 19 ms on localhost (4/4) -2016-06-13 18:50:09 INFO DAGScheduler:59 - Job 8 finished: aggregate at GaussianMixture.scala:201, took 0.033716 s -2016-06-13 18:50:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(656) called with curMem=25035, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 656.0 B, free 730.6 MB) -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(833) called with curMem=25691, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 833.0 B, free 730.6 MB) -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:65195 (size: 833.0 B, free: 730.6 MB) -2016-06-13 18:50:09 INFO SparkContext:59 - Created broadcast 15 from broadcast at GaussianMixtureModel.scala:100 -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(56) called with curMem=26524, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 56.0 B, free 730.6 MB) -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(65) called with curMem=26580, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 65.0 B, free 730.6 MB) -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:65195 (size: 65.0 B, free: 730.6 MB) -2016-06-13 18:50:09 INFO SparkContext:59 - Created broadcast 16 from broadcast at GaussianMixtureModel.scala:101 -2016-06-13 18:50:09 INFO SparkContext:59 - Starting job: collect at GaussianMixtureFromSparkSuite.scala:201 -2016-06-13 18:50:09 INFO DAGScheduler:59 - Got job 9 (collect at GaussianMixtureFromSparkSuite.scala:201) with 4 output partitions -2016-06-13 18:50:09 INFO DAGScheduler:59 - Final stage: ResultStage 9(collect at GaussianMixtureFromSparkSuite.scala:201) -2016-06-13 18:50:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:50:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:50:09 INFO DAGScheduler:59 - Submitting ResultStage 9 (ZippedPartitionsRDD2[5] at zip at GaussianMixtureFromSparkSuite.scala:201), which has no missing parents -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(4712) called with curMem=26645, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 18:50:09 INFO MemoryStore:59 - ensureFreeSpace(2631) called with curMem=31357, maxMem=766075207 -2016-06-13 18:50:09 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 18:50:09 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:65195 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 18:50:09 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:50:09 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 9 (ZippedPartitionsRDD2[5] at zip at GaussianMixtureFromSparkSuite.scala:201) -2016-06-13 18:50:09 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 4 tasks -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 33, localhost, PROCESS_LOCAL, 2432 bytes) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 34, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 2.0 in stage 9.0 (TID 35, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Starting task 3.0 in stage 9.0 (TID 36, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 18:50:09 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 34) -2016-06-13 18:50:09 INFO Executor:59 - Running task 2.0 in stage 9.0 (TID 35) -2016-06-13 18:50:09 INFO Executor:59 - Running task 3.0 in stage 9.0 (TID 36) -2016-06-13 18:50:09 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 34). 1269 bytes result sent to driver -2016-06-13 18:50:09 INFO Executor:59 - Finished task 3.0 in stage 9.0 (TID 36). 1264 bytes result sent to driver -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 34) in 22 ms on localhost (1/4) -2016-06-13 18:50:09 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 33) -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 3.0 in stage 9.0 (TID 36) in 21 ms on localhost (2/4) -2016-06-13 18:50:09 INFO Executor:59 - Finished task 2.0 in stage 9.0 (TID 35). 1264 bytes result sent to driver -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 2.0 in stage 9.0 (TID 35) in 27 ms on localhost (3/4) -2016-06-13 18:50:09 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 33). 1229 bytes result sent to driver -2016-06-13 18:50:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 33) in 38 ms on localhost (4/4) -2016-06-13 18:50:09 INFO DAGScheduler:59 - ResultStage 9 (collect at GaussianMixtureFromSparkSuite.scala:201) finished in 0.038 s -2016-06-13 18:50:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 18:50:09 INFO DAGScheduler:59 - Job 9 finished: collect at GaussianMixtureFromSparkSuite.scala:201, took 0.055920 s -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 18:50:09 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 18:50:09 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 18:50:09 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 18:50:09 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 18:50:09 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 18:50:09 INFO BlockManager:59 - BlockManager stopped -2016-06-13 18:50:09 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 18:50:09 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 18:50:09 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 18:50:09 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 18:50:09 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 18:50:09 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 18:50:09 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-f086406a-a2eb-434d-9520-6599d4a262f4 -2016-06-13 18:51:13 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 18:51:13 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 18:51:13 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 18:51:13 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 18:51:13 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 18:51:14 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 18:51:14 INFO Remoting:74 - Starting remoting -2016-06-13 18:51:14 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:65214] -2016-06-13 18:51:14 INFO Utils:59 - Successfully started service 'sparkDriver' on port 65214. -2016-06-13 18:51:14 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 18:51:14 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 18:51:14 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-9b5bb439-a011-4a66-b487-1e15a3d51d2b -2016-06-13 18:51:14 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 18:51:15 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-a4f89345-b9c7-4ae5-ac38-91284e31bb6b\httpd-e9236881-ee8e-426d-914a-a2ab321fbda1 -2016-06-13 18:51:15 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 18:51:15 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 18:51:15 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:65215 -2016-06-13 18:51:15 INFO Utils:59 - Successfully started service 'HTTP file server' on port 65215. -2016-06-13 18:51:15 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 18:51:15 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 18:51:15 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 18:51:15 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 18:51:15 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 18:51:15 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 18:51:15 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 18:51:15 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 65234. -2016-06-13 18:51:15 INFO NettyBlockTransferService:59 - Server created on 65234 -2016-06-13 18:51:15 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 18:51:15 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:65234 with 730.6 MB RAM, BlockManagerId(driver, localhost, 65234) -2016-06-13 18:51:15 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 18:51:16 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 18:51:16 INFO DAGScheduler:59 - Got job 0 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 18:51:16 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GaussianMixture.scala:171) -2016-06-13 18:51:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:16 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:16 INFO MemoryStore:59 - ensureFreeSpace(1992) called with curMem=0, maxMem=766075207 -2016-06-13 18:51:16 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1992.0 B, free 730.6 MB) -2016-06-13 18:51:16 INFO MemoryStore:59 - ensureFreeSpace(1250) called with curMem=1992, maxMem=766075207 -2016-06-13 18:51:16 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1250.0 B, free 730.6 MB) -2016-06-13 18:51:16 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:65234 (size: 1250.0 B, free: 730.6 MB) -2016-06-13 18:51:16 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:16 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:16 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 18:51:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:16 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 18:51:16 INFO CacheManager:59 - Partition rdd_1_0 not found, computing it -2016-06-13 18:51:17 INFO MemoryStore:59 - ensureFreeSpace(224) called with curMem=3242, maxMem=766075207 -2016-06-13 18:51:17 INFO MemoryStore:59 - Block rdd_1_0 stored as values in memory (estimated size 224.0 B, free 730.6 MB) -2016-06-13 18:51:17 INFO BlockManagerInfo:59 - Added rdd_1_0 in memory on localhost:65234 (size: 224.0 B, free: 730.6 MB) -2016-06-13 18:51:17 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1688 bytes result sent to driver -2016-06-13 18:51:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 370 ms on localhost (1/1) -2016-06-13 18:51:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 18:51:17 INFO DAGScheduler:59 - ResultStage 0 (first at GaussianMixture.scala:171) finished in 0.381 s -2016-06-13 18:51:17 INFO DAGScheduler:59 - Job 0 finished: first at GaussianMixture.scala:171, took 0.610626 s -2016-06-13 18:51:17 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 18:51:17 INFO DAGScheduler:59 - Got job 1 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 18:51:17 INFO DAGScheduler:59 - Final stage: ResultStage 1(takeSample at GaussianMixture.scala:184) -2016-06-13 18:51:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:17 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:17 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=3466, maxMem=766075207 -2016-06-13 18:51:17 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 18:51:17 INFO MemoryStore:59 - ensureFreeSpace(1195) called with curMem=5298, maxMem=766075207 -2016-06-13 18:51:17 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1195.0 B, free 730.6 MB) -2016-06-13 18:51:17 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:65234 (size: 1195.0 B, free: 730.6 MB) -2016-06-13 18:51:17 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:17 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 4 tasks -2016-06-13 18:51:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 1.0 (TID 4, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:17 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 18:51:17 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 18:51:17 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:17 INFO CacheManager:59 - Partition rdd_1_1 not found, computing it -2016-06-13 18:51:17 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=6493, maxMem=766075207 -2016-06-13 18:51:17 INFO MemoryStore:59 - Block rdd_1_1 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 18:51:17 INFO BlockManagerInfo:59 - Added rdd_1_1 in memory on localhost:65234 (size: 288.0 B, free: 730.6 MB) -2016-06-13 18:51:17 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 1567 bytes result sent to driver -2016-06-13 18:51:17 INFO Executor:59 - Running task 3.0 in stage 1.0 (TID 4) -2016-06-13 18:51:17 INFO Executor:59 - Running task 2.0 in stage 1.0 (TID 3) -2016-06-13 18:51:17 INFO CacheManager:59 - Partition rdd_1_3 not found, computing it -2016-06-13 18:51:17 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=6781, maxMem=766075207 -2016-06-13 18:51:17 INFO MemoryStore:59 - Block rdd_1_3 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 18:51:17 INFO BlockManagerInfo:59 - Added rdd_1_3 in memory on localhost:65234 (size: 288.0 B, free: 730.6 MB) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 67 ms on localhost (1/4) -2016-06-13 18:51:17 INFO Executor:59 - Finished task 3.0 in stage 1.0 (TID 4). 1567 bytes result sent to driver -2016-06-13 18:51:17 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 18:51:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 1.0 (TID 4) in 68 ms on localhost (2/4) -2016-06-13 18:51:17 INFO CacheManager:59 - Partition rdd_1_2 not found, computing it -2016-06-13 18:51:17 INFO MemoryStore:59 - ensureFreeSpace(288) called with curMem=7069, maxMem=766075207 -2016-06-13 18:51:17 INFO MemoryStore:59 - Block rdd_1_2 stored as values in memory (estimated size 288.0 B, free 730.6 MB) -2016-06-13 18:51:17 INFO BlockManagerInfo:59 - Added rdd_1_2 in memory on localhost:65234 (size: 288.0 B, free: 730.6 MB) -2016-06-13 18:51:17 INFO Executor:59 - Finished task 2.0 in stage 1.0 (TID 3). 1567 bytes result sent to driver -2016-06-13 18:51:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 90 ms on localhost (3/4) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 1.0 (TID 3) in 85 ms on localhost (4/4) -2016-06-13 18:51:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 18:51:17 INFO DAGScheduler:59 - ResultStage 1 (takeSample at GaussianMixture.scala:184) finished in 0.093 s -2016-06-13 18:51:17 INFO DAGScheduler:59 - Job 1 finished: takeSample at GaussianMixture.scala:184, took 0.115902 s -2016-06-13 18:51:17 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 18:51:17 INFO DAGScheduler:59 - Got job 2 (takeSample at GaussianMixture.scala:184) with 4 output partitions -2016-06-13 18:51:17 INFO DAGScheduler:59 - Final stage: ResultStage 2(takeSample at GaussianMixture.scala:184) -2016-06-13 18:51:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:17 INFO DAGScheduler:59 - Submitting ResultStage 2 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 18:51:17 INFO MemoryStore:59 - ensureFreeSpace(19224) called with curMem=7357, maxMem=766075207 -2016-06-13 18:51:17 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 18.8 KB, free 730.6 MB) -2016-06-13 18:51:17 INFO MemoryStore:59 - ensureFreeSpace(7886) called with curMem=26581, maxMem=766075207 -2016-06-13 18:51:17 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 7.7 KB, free 730.6 MB) -2016-06-13 18:51:17 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:65234 (size: 7.7 KB, free: 730.6 MB) -2016-06-13 18:51:17 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 2 (PartitionwiseSampledRDD[2] at takeSample at GaussianMixture.scala:184) -2016-06-13 18:51:17 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 4 tasks -2016-06-13 18:51:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 2.0 (TID 8, localhost, PROCESS_LOCAL, 2391 bytes) -2016-06-13 18:51:17 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 6) -2016-06-13 18:51:17 INFO Executor:59 - Running task 3.0 in stage 2.0 (TID 8) -2016-06-13 18:51:17 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 5) -2016-06-13 18:51:17 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 7) -2016-06-13 18:51:17 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:17 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:17 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:17 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:17 INFO Executor:59 - Finished task 3.0 in stage 2.0 (TID 8). 2339 bytes result sent to driver -2016-06-13 18:51:17 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 5). 2298 bytes result sent to driver -2016-06-13 18:51:17 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 7). 2308 bytes result sent to driver -2016-06-13 18:51:17 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 6). 2349 bytes result sent to driver -2016-06-13 18:51:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 2.0 (TID 8) in 25 ms on localhost (1/4) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 5) in 32 ms on localhost (2/4) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 7) in 29 ms on localhost (3/4) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 6) in 31 ms on localhost (4/4) -2016-06-13 18:51:17 INFO DAGScheduler:59 - ResultStage 2 (takeSample at GaussianMixture.scala:184) finished in 0.033 s -2016-06-13 18:51:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 18:51:17 INFO DAGScheduler:59 - Job 2 finished: takeSample at GaussianMixture.scala:184, took 0.049052 s -2016-06-13 18:51:17 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 18:51:17 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 18:51:17 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 18:51:17 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 18:51:17 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=34467, maxMem=766075207 -2016-06-13 18:51:17 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 1344.0 B, free 730.6 MB) -2016-06-13 18:51:17 INFO MemoryStore:59 - ensureFreeSpace(1060) called with curMem=35811, maxMem=766075207 -2016-06-13 18:51:17 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1060.0 B, free 730.6 MB) -2016-06-13 18:51:17 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:65234 (size: 1060.0 B, free: 730.6 MB) -2016-06-13 18:51:17 INFO SparkContext:59 - Created broadcast 3 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:51:17 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:51:17 INFO DAGScheduler:59 - Got job 3 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:51:17 INFO DAGScheduler:59 - Final stage: ResultStage 3(aggregate at GaussianMixture.scala:201) -2016-06-13 18:51:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:17 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:17 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=36871, maxMem=766075207 -2016-06-13 18:51:17 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:51:17 INFO MemoryStore:59 - ensureFreeSpace(2354) called with curMem=41503, maxMem=766075207 -2016-06-13 18:51:17 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:51:17 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:65234 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:17 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 3 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:17 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 4 tasks -2016-06-13 18:51:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 10, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 3.0 (TID 11, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 3.0 (TID 12, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:17 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 9) -2016-06-13 18:51:17 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 10) -2016-06-13 18:51:17 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:17 INFO Executor:59 - Running task 2.0 in stage 3.0 (TID 11) -2016-06-13 18:51:17 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:17 INFO Executor:59 - Running task 3.0 in stage 3.0 (TID 12) -2016-06-13 18:51:17 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 9). 2934 bytes result sent to driver -2016-06-13 18:51:17 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:17 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:17 INFO Executor:59 - Finished task 3.0 in stage 3.0 (TID 12). 2934 bytes result sent to driver -2016-06-13 18:51:17 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 10). 2934 bytes result sent to driver -2016-06-13 18:51:17 INFO Executor:59 - Finished task 2.0 in stage 3.0 (TID 11). 2934 bytes result sent to driver -2016-06-13 18:51:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 9) in 41 ms on localhost (1/4) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 10) in 42 ms on localhost (2/4) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 3.0 (TID 11) in 44 ms on localhost (3/4) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 3.0 (TID 12) in 46 ms on localhost (4/4) -2016-06-13 18:51:17 INFO DAGScheduler:59 - ResultStage 3 (aggregate at GaussianMixture.scala:201) finished in 0.052 s -2016-06-13 18:51:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 18:51:17 INFO DAGScheduler:59 - Job 3 finished: aggregate at GaussianMixture.scala:201, took 0.081394 s -2016-06-13 18:51:17 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=43857, maxMem=766075207 -2016-06-13 18:51:17 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:51:17 INFO MemoryStore:59 - ensureFreeSpace(1083) called with curMem=45201, maxMem=766075207 -2016-06-13 18:51:17 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 1083.0 B, free 730.5 MB) -2016-06-13 18:51:17 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:65234 (size: 1083.0 B, free: 730.6 MB) -2016-06-13 18:51:17 INFO SparkContext:59 - Created broadcast 5 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:51:17 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:51:17 INFO DAGScheduler:59 - Got job 4 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:51:17 INFO DAGScheduler:59 - Final stage: ResultStage 4(aggregate at GaussianMixture.scala:201) -2016-06-13 18:51:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:17 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:17 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=46284, maxMem=766075207 -2016-06-13 18:51:17 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:51:17 INFO MemoryStore:59 - ensureFreeSpace(2381) called with curMem=50916, maxMem=766075207 -2016-06-13 18:51:17 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:51:17 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:65234 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:17 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 4 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:17 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 4 tasks -2016-06-13 18:51:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 13, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 14, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 4.0 (TID 15, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 4.0 (TID 16, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:17 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 13) -2016-06-13 18:51:17 INFO Executor:59 - Running task 3.0 in stage 4.0 (TID 16) -2016-06-13 18:51:17 INFO Executor:59 - Running task 2.0 in stage 4.0 (TID 15) -2016-06-13 18:51:17 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 14) -2016-06-13 18:51:17 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:17 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:17 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 13). 2934 bytes result sent to driver -2016-06-13 18:51:17 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:17 INFO Executor:59 - Finished task 3.0 in stage 4.0 (TID 16). 2934 bytes result sent to driver -2016-06-13 18:51:17 INFO Executor:59 - Finished task 2.0 in stage 4.0 (TID 15). 2934 bytes result sent to driver -2016-06-13 18:51:17 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:17 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 14). 2934 bytes result sent to driver -2016-06-13 18:51:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 13) in 34 ms on localhost (1/4) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 4.0 (TID 16) in 29 ms on localhost (2/4) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 4.0 (TID 15) in 35 ms on localhost (3/4) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 14) in 37 ms on localhost (4/4) -2016-06-13 18:51:17 INFO DAGScheduler:59 - ResultStage 4 (aggregate at GaussianMixture.scala:201) finished in 0.038 s -2016-06-13 18:51:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 18:51:17 INFO DAGScheduler:59 - Job 4 finished: aggregate at GaussianMixture.scala:201, took 0.053553 s -2016-06-13 18:51:17 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=53297, maxMem=766075207 -2016-06-13 18:51:17 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:51:17 INFO MemoryStore:59 - ensureFreeSpace(1082) called with curMem=54641, maxMem=766075207 -2016-06-13 18:51:17 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 1082.0 B, free 730.5 MB) -2016-06-13 18:51:17 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:65234 (size: 1082.0 B, free: 730.6 MB) -2016-06-13 18:51:17 INFO SparkContext:59 - Created broadcast 7 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:51:17 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:51:17 INFO DAGScheduler:59 - Got job 5 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:51:17 INFO DAGScheduler:59 - Final stage: ResultStage 5(aggregate at GaussianMixture.scala:201) -2016-06-13 18:51:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:17 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:17 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=55723, maxMem=766075207 -2016-06-13 18:51:17 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:51:17 INFO MemoryStore:59 - ensureFreeSpace(2378) called with curMem=60355, maxMem=766075207 -2016-06-13 18:51:17 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:51:17 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:65234 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:17 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 5 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:17 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 4 tasks -2016-06-13 18:51:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 17, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 18, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 5.0 (TID 19, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 5.0 (TID 20, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:17 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 18) -2016-06-13 18:51:17 INFO Executor:59 - Running task 3.0 in stage 5.0 (TID 20) -2016-06-13 18:51:17 INFO Executor:59 - Running task 2.0 in stage 5.0 (TID 19) -2016-06-13 18:51:17 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 17) -2016-06-13 18:51:17 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:17 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:17 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 17). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 18). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO Executor:59 - Finished task 2.0 in stage 5.0 (TID 19). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO Executor:59 - Finished task 3.0 in stage 5.0 (TID 20). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 17) in 23 ms on localhost (1/4) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 18) in 23 ms on localhost (2/4) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 5.0 (TID 20) in 22 ms on localhost (3/4) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 5.0 (TID 19) in 24 ms on localhost (4/4) -2016-06-13 18:51:18 INFO DAGScheduler:59 - ResultStage 5 (aggregate at GaussianMixture.scala:201) finished in 0.027 s -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 18:51:18 INFO DAGScheduler:59 - Job 5 finished: aggregate at GaussianMixture.scala:201, took 0.041690 s -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=62733, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1087) called with curMem=64077, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 1087.0 B, free 730.5 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:65234 (size: 1087.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 9 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:51:18 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Got job 6 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:51:18 INFO DAGScheduler:59 - Final stage: ResultStage 6(aggregate at GaussianMixture.scala:201) -2016-06-13 18:51:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:65234 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=58154, maxMem=766075207 -2016-06-13 18:51:18 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(2381) called with curMem=60360, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:65234 in memory (size: 1082.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:65234 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:65234 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 6 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 4 tasks -2016-06-13 18:51:18 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 21, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 22, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:65234 in memory (size: 1083.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 23, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 6.0 (TID 24, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 23) -2016-06-13 18:51:18 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 21) -2016-06-13 18:51:18 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 22) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:65234 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO Executor:59 - Running task 3.0 in stage 6.0 (TID 24) -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 23). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 22). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 21). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 23) in 20 ms on localhost (1/4) -2016-06-13 18:51:18 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 18:51:18 INFO Executor:59 - Finished task 3.0 in stage 6.0 (TID 24). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 22) in 25 ms on localhost (2/4) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 21) in 27 ms on localhost (3/4) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:65234 in memory (size: 1060.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO DAGScheduler:59 - ResultStage 6 (aggregate at GaussianMixture.scala:201) finished in 0.029 s -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 6.0 (TID 24) in 24 ms on localhost (4/4) -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 18:51:18 INFO DAGScheduler:59 - Job 6 finished: aggregate at GaussianMixture.scala:201, took 0.046398 s -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=16801, maxMem=766075207 -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:65234 in memory (size: 7.7 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 1344.0 B, free 730.6 MB) -2016-06-13 18:51:18 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1078) called with curMem=16950, maxMem=766075207 -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:65234 in memory (size: 1195.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 1078.0 B, free 730.6 MB) -2016-06-13 18:51:18 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:65234 (size: 1078.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:65234 in memory (size: 1250.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 11 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:51:18 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 18:51:18 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Got job 7 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:51:18 INFO DAGScheduler:59 - Final stage: ResultStage 7(aggregate at GaussianMixture.scala:201) -2016-06-13 18:51:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=12954, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.5 KB, free 730.6 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(2379) called with curMem=17586, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.6 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:65234 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 7 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 4 tasks -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 25, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 26, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 7.0 (TID 27, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 7.0 (TID 28, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 26) -2016-06-13 18:51:18 INFO Executor:59 - Running task 2.0 in stage 7.0 (TID 27) -2016-06-13 18:51:18 INFO Executor:59 - Running task 3.0 in stage 7.0 (TID 28) -2016-06-13 18:51:18 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 25) -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 3.0 in stage 7.0 (TID 28). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 26). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO Executor:59 - Finished task 2.0 in stage 7.0 (TID 27). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 25). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 26) in 16 ms on localhost (1/4) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 7.0 (TID 28) in 15 ms on localhost (2/4) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 7.0 (TID 27) in 16 ms on localhost (3/4) -2016-06-13 18:51:18 INFO DAGScheduler:59 - ResultStage 7 (aggregate at GaussianMixture.scala:201) finished in 0.020 s -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 25) in 19 ms on localhost (4/4) -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 18:51:18 INFO DAGScheduler:59 - Job 7 finished: aggregate at GaussianMixture.scala:201, took 0.032543 s -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=19965, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 1344.0 B, free 730.6 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1082) called with curMem=21309, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 1082.0 B, free 730.6 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:65234 (size: 1082.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 13 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:51:18 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Got job 8 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:51:18 INFO DAGScheduler:59 - Final stage: ResultStage 8(aggregate at GaussianMixture.scala:201) -2016-06-13 18:51:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=22391, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 4.5 KB, free 730.6 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(2378) called with curMem=27023, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.6 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:65234 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 8 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 4 tasks -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 29, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 30, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 8.0 (TID 31, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 8.0 (TID 32, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 29) -2016-06-13 18:51:18 INFO Executor:59 - Running task 2.0 in stage 8.0 (TID 31) -2016-06-13 18:51:18 INFO Executor:59 - Running task 3.0 in stage 8.0 (TID 32) -2016-06-13 18:51:18 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 30) -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 2.0 in stage 8.0 (TID 31). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 29). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 30). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO Executor:59 - Finished task 3.0 in stage 8.0 (TID 32). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 8.0 (TID 31) in 14 ms on localhost (1/4) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 29) in 16 ms on localhost (2/4) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 30) in 17 ms on localhost (3/4) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 8.0 (TID 32) in 15 ms on localhost (4/4) -2016-06-13 18:51:18 INFO DAGScheduler:59 - ResultStage 8 (aggregate at GaussianMixture.scala:201) finished in 0.018 s -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 18:51:18 INFO DAGScheduler:59 - Job 8 finished: aggregate at GaussianMixture.scala:201, took 0.030614 s -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=29401, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 1344.0 B, free 730.6 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1081) called with curMem=30745, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1081.0 B, free 730.6 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:65234 (size: 1081.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 15 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:51:18 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Got job 9 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:51:18 INFO DAGScheduler:59 - Final stage: ResultStage 9(aggregate at GaussianMixture.scala:201) -2016-06-13 18:51:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=31826, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 4.5 KB, free 730.6 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(2378) called with curMem=36458, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:65234 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 9 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 4 tasks -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 33, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 34, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 9.0 (TID 35, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 9.0 (TID 36, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 33) -2016-06-13 18:51:18 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 34) -2016-06-13 18:51:18 INFO Executor:59 - Running task 2.0 in stage 9.0 (TID 35) -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 33). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO Executor:59 - Finished task 2.0 in stage 9.0 (TID 35). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 33) in 23 ms on localhost (1/4) -2016-06-13 18:51:18 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 34). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 9.0 (TID 35) in 24 ms on localhost (2/4) -2016-06-13 18:51:18 INFO Executor:59 - Running task 3.0 in stage 9.0 (TID 36) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 34) in 28 ms on localhost (3/4) -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 3.0 in stage 9.0 (TID 36). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 9.0 (TID 36) in 36 ms on localhost (4/4) -2016-06-13 18:51:18 INFO DAGScheduler:59 - ResultStage 9 (aggregate at GaussianMixture.scala:201) finished in 0.041 s -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 18:51:18 INFO DAGScheduler:59 - Job 9 finished: aggregate at GaussianMixture.scala:201, took 0.060345 s -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=38836, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1083) called with curMem=40180, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 1083.0 B, free 730.5 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:65234 (size: 1083.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 17 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:51:18 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Got job 10 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:51:18 INFO DAGScheduler:59 - Final stage: ResultStage 10(aggregate at GaussianMixture.scala:201) -2016-06-13 18:51:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=41263, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(2379) called with curMem=45895, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:65234 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 10 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 4 tasks -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 37, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 38, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 10.0 (TID 39, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 10.0 (TID 40, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO Executor:59 - Running task 3.0 in stage 10.0 (TID 40) -2016-06-13 18:51:18 INFO Executor:59 - Running task 2.0 in stage 10.0 (TID 39) -2016-06-13 18:51:18 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 38) -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 3.0 in stage 10.0 (TID 40). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 38). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 10.0 (TID 40) in 19 ms on localhost (1/4) -2016-06-13 18:51:18 INFO Executor:59 - Finished task 2.0 in stage 10.0 (TID 39). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 37) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 38) in 29 ms on localhost (2/4) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 10.0 (TID 39) in 30 ms on localhost (3/4) -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 37). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 37) in 43 ms on localhost (4/4) -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 18:51:18 INFO DAGScheduler:59 - ResultStage 10 (aggregate at GaussianMixture.scala:201) finished in 0.043 s -2016-06-13 18:51:18 INFO DAGScheduler:59 - Job 10 finished: aggregate at GaussianMixture.scala:201, took 0.063512 s -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=48274, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1081) called with curMem=49618, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1081.0 B, free 730.5 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:65234 (size: 1081.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 19 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:51:18 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Got job 11 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:51:18 INFO DAGScheduler:59 - Final stage: ResultStage 11(aggregate at GaussianMixture.scala:201) -2016-06-13 18:51:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=50699, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(2378) called with curMem=55331, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:65234 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 11 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 4 tasks -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 41, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 42, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 11.0 (TID 43, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 11.0 (TID 44, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 41) -2016-06-13 18:51:18 INFO Executor:59 - Running task 3.0 in stage 11.0 (TID 44) -2016-06-13 18:51:18 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 42) -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 3.0 in stage 11.0 (TID 44). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO Executor:59 - Running task 2.0 in stage 11.0 (TID 43) -2016-06-13 18:51:18 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 41). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 2.0 in stage 11.0 (TID 43). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 42). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 11.0 (TID 44) in 23 ms on localhost (1/4) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 41) in 29 ms on localhost (2/4) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 11.0 (TID 43) in 27 ms on localhost (3/4) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 42) in 30 ms on localhost (4/4) -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 18:51:18 INFO DAGScheduler:59 - ResultStage 11 (aggregate at GaussianMixture.scala:201) finished in 0.033 s -2016-06-13 18:51:18 INFO DAGScheduler:59 - Job 11 finished: aggregate at GaussianMixture.scala:201, took 0.051819 s -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=57709, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1089) called with curMem=59053, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 1089.0 B, free 730.5 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:65234 (size: 1089.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 21 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:51:18 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Got job 12 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:51:18 INFO DAGScheduler:59 - Final stage: ResultStage 12(aggregate at GaussianMixture.scala:201) -2016-06-13 18:51:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=60142, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(2377) called with curMem=64774, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:65234 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 12 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 4 tasks -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 45, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 46, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 12.0 (TID 47, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 12.0 (TID 48, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO Executor:59 - Running task 3.0 in stage 12.0 (TID 48) -2016-06-13 18:51:18 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 46) -2016-06-13 18:51:18 INFO Executor:59 - Running task 2.0 in stage 12.0 (TID 47) -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 3.0 in stage 12.0 (TID 48). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 45) -2016-06-13 18:51:18 INFO Executor:59 - Finished task 2.0 in stage 12.0 (TID 47). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 12.0 (TID 48) in 16 ms on localhost (1/4) -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 12.0 (TID 47) in 20 ms on localhost (2/4) -2016-06-13 18:51:18 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 45). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 45) in 28 ms on localhost (3/4) -2016-06-13 18:51:18 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 46). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 46) in 33 ms on localhost (4/4) -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 18:51:18 INFO DAGScheduler:59 - ResultStage 12 (aggregate at GaussianMixture.scala:201) finished in 0.036 s -2016-06-13 18:51:18 INFO DAGScheduler:59 - Job 12 finished: aggregate at GaussianMixture.scala:201, took 0.056198 s -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=67151, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1081) called with curMem=68495, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 1081.0 B, free 730.5 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:65234 (size: 1081.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 23 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:51:18 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Got job 13 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:51:18 INFO DAGScheduler:59 - Final stage: ResultStage 13(aggregate at GaussianMixture.scala:201) -2016-06-13 18:51:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=69576, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(2377) called with curMem=74208, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:65234 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 13 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 4 tasks -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 49, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 50, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 13.0 (TID 51, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 13.0 (TID 52, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO Executor:59 - Running task 3.0 in stage 13.0 (TID 52) -2016-06-13 18:51:18 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 49) -2016-06-13 18:51:18 INFO Executor:59 - Running task 2.0 in stage 13.0 (TID 51) -2016-06-13 18:51:18 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 50) -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 3.0 in stage 13.0 (TID 52). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 50). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 13.0 (TID 52) in 13 ms on localhost (1/4) -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 2.0 in stage 13.0 (TID 51). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 49). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 50) in 27 ms on localhost (2/4) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 13.0 (TID 51) in 27 ms on localhost (3/4) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 49) in 31 ms on localhost (4/4) -2016-06-13 18:51:18 INFO DAGScheduler:59 - ResultStage 13 (aggregate at GaussianMixture.scala:201) finished in 0.031 s -2016-06-13 18:51:18 INFO DAGScheduler:59 - Job 13 finished: aggregate at GaussianMixture.scala:201, took 0.049935 s -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=76585, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1083) called with curMem=77929, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 1083.0 B, free 730.5 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:65234 (size: 1083.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 25 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:51:18 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Got job 14 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:51:18 INFO DAGScheduler:59 - Final stage: ResultStage 14(aggregate at GaussianMixture.scala:201) -2016-06-13 18:51:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=79012, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(2378) called with curMem=83644, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:65234 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 14 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 4 tasks -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 53, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 54, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:65234 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 55, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 14.0 (TID 56, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 53) -2016-06-13 18:51:18 INFO Executor:59 - Running task 3.0 in stage 14.0 (TID 56) -2016-06-13 18:51:18 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 55) -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 53). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 3.0 in stage 14.0 (TID 56). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 55). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 53) in 21 ms on localhost (1/4) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 14.0 (TID 56) in 19 ms on localhost (2/4) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 55) in 22 ms on localhost (3/4) -2016-06-13 18:51:18 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 18:51:18 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 54) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:65234 in memory (size: 1081.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 54). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:65234 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 54) in 37 ms on localhost (4/4) -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 18:51:18 INFO DAGScheduler:59 - ResultStage 14 (aggregate at GaussianMixture.scala:201) finished in 0.039 s -2016-06-13 18:51:18 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Job 14 finished: aggregate at GaussianMixture.scala:201, took 0.075095 s -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=68490, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:65234 in memory (size: 1089.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1082) called with curMem=68490, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 1082.0 B, free 730.5 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:65234 (size: 1082.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 27 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:51:18 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Got job 15 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:51:18 INFO DAGScheduler:59 - Final stage: ResultStage 15(aggregate at GaussianMixture.scala:201) -2016-06-13 18:51:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=69572, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(2377) called with curMem=74204, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:65234 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:65234 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 15 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 4 tasks -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 57, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 58, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 15.0 (TID 59, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 15.0 (TID 60, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO Executor:59 - Running task 2.0 in stage 15.0 (TID 59) -2016-06-13 18:51:18 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 57) -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:18 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 58) -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 58). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO Executor:59 - Running task 3.0 in stage 15.0 (TID 60) -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 3.0 in stage 15.0 (TID 60). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:65234 in memory (size: 1081.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 15.0 (TID 60) in 46 ms on localhost (1/4) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 58) in 51 ms on localhost (2/4) -2016-06-13 18:51:18 INFO Executor:59 - Finished task 2.0 in stage 15.0 (TID 59). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 15.0 (TID 59) in 61 ms on localhost (3/4) -2016-06-13 18:51:18 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 57). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 57) in 70 ms on localhost (4/4) -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 18:51:18 INFO DAGScheduler:59 - ResultStage 15 (aggregate at GaussianMixture.scala:201) finished in 0.071 s -2016-06-13 18:51:18 INFO DAGScheduler:59 - Job 15 finished: aggregate at GaussianMixture.scala:201, took 0.094174 s -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:65234 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=64767, maxMem=766075207 -2016-06-13 18:51:18 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(1084) called with curMem=61479, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 1084.0 B, free 730.5 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:65234 (size: 1084.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 29 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:65234 in memory (size: 1083.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:65234 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:51:18 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Got job 16 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:51:18 INFO DAGScheduler:59 - Final stage: ResultStage 16(aggregate at GaussianMixture.scala:201) -2016-06-13 18:51:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:65234 in memory (size: 1081.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=50701, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:51:18 INFO MemoryStore:59 - ensureFreeSpace(2377) called with curMem=52955, maxMem=766075207 -2016-06-13 18:51:18 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:65234 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:65234 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 18:51:18 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 16 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 4 tasks -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:65234 in memory (size: 1082.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 61, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 62, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 16.0 (TID 63, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 16.0 (TID 64, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:18 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 62) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:65234 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO Executor:59 - Running task 2.0 in stage 16.0 (TID 63) -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 62). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:18 INFO Executor:59 - Running task 3.0 in stage 16.0 (TID 64) -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 2.0 in stage 16.0 (TID 63). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 62) in 30 ms on localhost (1/4) -2016-06-13 18:51:18 INFO Executor:59 - Finished task 3.0 in stage 16.0 (TID 64). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 16.0 (TID 63) in 33 ms on localhost (2/4) -2016-06-13 18:51:18 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 16.0 (TID 64) in 34 ms on localhost (3/4) -2016-06-13 18:51:18 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 61) -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:65234 in memory (size: 1078.0 B, free: 730.6 MB) -2016-06-13 18:51:18 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:18 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 61). 2934 bytes result sent to driver -2016-06-13 18:51:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 61) in 56 ms on localhost (4/4) -2016-06-13 18:51:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 18:51:18 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:65234 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:18 INFO DAGScheduler:59 - ResultStage 16 (aggregate at GaussianMixture.scala:201) finished in 0.059 s -2016-06-13 18:51:19 INFO DAGScheduler:59 - Job 16 finished: aggregate at GaussianMixture.scala:201, took 0.096374 s -2016-06-13 18:51:19 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=31828, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 1344.0 B, free 730.6 MB) -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(1089) called with curMem=30741, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1089.0 B, free 730.6 MB) -2016-06-13 18:51:19 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:65234 in memory (size: 1087.0 B, free: 730.6 MB) -2016-06-13 18:51:19 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:65234 (size: 1089.0 B, free: 730.6 MB) -2016-06-13 18:51:19 INFO SparkContext:59 - Created broadcast 31 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:51:19 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:51:19 INFO DAGScheduler:59 - Got job 17 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:51:19 INFO DAGScheduler:59 - Final stage: ResultStage 17(aggregate at GaussianMixture.scala:201) -2016-06-13 18:51:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:19 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=31830, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 4.5 KB, free 730.6 MB) -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(2378) called with curMem=36462, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:51:19 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:65234 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:19 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 17 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:19 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 4 tasks -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 65, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 66, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 17.0 (TID 67, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 17.0 (TID 68, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:19 INFO Executor:59 - Running task 3.0 in stage 17.0 (TID 68) -2016-06-13 18:51:19 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 65) -2016-06-13 18:51:19 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 66) -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:19 INFO Executor:59 - Finished task 3.0 in stage 17.0 (TID 68). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 66). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:19 INFO Executor:59 - Running task 2.0 in stage 17.0 (TID 67) -2016-06-13 18:51:19 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 65). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:19 INFO Executor:59 - Finished task 2.0 in stage 17.0 (TID 67). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 17.0 (TID 68) in 22 ms on localhost (1/4) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 66) in 25 ms on localhost (2/4) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 17.0 (TID 67) in 26 ms on localhost (3/4) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 65) in 29 ms on localhost (4/4) -2016-06-13 18:51:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 18:51:19 INFO DAGScheduler:59 - ResultStage 17 (aggregate at GaussianMixture.scala:201) finished in 0.031 s -2016-06-13 18:51:19 INFO DAGScheduler:59 - Job 17 finished: aggregate at GaussianMixture.scala:201, took 0.050056 s -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=38840, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(1082) called with curMem=40184, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 1082.0 B, free 730.5 MB) -2016-06-13 18:51:19 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:65234 (size: 1082.0 B, free: 730.6 MB) -2016-06-13 18:51:19 INFO SparkContext:59 - Created broadcast 33 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:51:19 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:51:19 INFO DAGScheduler:59 - Got job 18 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:51:19 INFO DAGScheduler:59 - Final stage: ResultStage 18(aggregate at GaussianMixture.scala:201) -2016-06-13 18:51:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:19 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=41266, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(2379) called with curMem=45898, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:51:19 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:65234 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:19 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 18 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:19 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 4 tasks -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 69, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 70, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 71, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 18.0 (TID 72, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:19 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 70) -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:19 INFO Executor:59 - Running task 3.0 in stage 18.0 (TID 72) -2016-06-13 18:51:19 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 70). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 69) -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:19 INFO Executor:59 - Finished task 3.0 in stage 18.0 (TID 72). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:19 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 71) -2016-06-13 18:51:19 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 69). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 70) in 23 ms on localhost (1/4) -2016-06-13 18:51:19 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 71). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 18.0 (TID 72) in 27 ms on localhost (2/4) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 69) in 32 ms on localhost (3/4) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 71) in 35 ms on localhost (4/4) -2016-06-13 18:51:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 18:51:19 INFO DAGScheduler:59 - ResultStage 18 (aggregate at GaussianMixture.scala:201) finished in 0.037 s -2016-06-13 18:51:19 INFO DAGScheduler:59 - Job 18 finished: aggregate at GaussianMixture.scala:201, took 0.060092 s -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=48277, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(1090) called with curMem=49621, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 1090.0 B, free 730.5 MB) -2016-06-13 18:51:19 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:65234 (size: 1090.0 B, free: 730.6 MB) -2016-06-13 18:51:19 INFO SparkContext:59 - Created broadcast 35 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:51:19 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:51:19 INFO DAGScheduler:59 - Got job 19 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:51:19 INFO DAGScheduler:59 - Final stage: ResultStage 19(aggregate at GaussianMixture.scala:201) -2016-06-13 18:51:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:19 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=50711, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(2379) called with curMem=55343, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:51:19 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:65234 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:19 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 19 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:19 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 4 tasks -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 73, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 74, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 19.0 (TID 75, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 19.0 (TID 76, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:19 INFO Executor:59 - Running task 3.0 in stage 19.0 (TID 76) -2016-06-13 18:51:19 INFO Executor:59 - Running task 2.0 in stage 19.0 (TID 75) -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:19 INFO Executor:59 - Finished task 3.0 in stage 19.0 (TID 76). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:19 INFO Executor:59 - Finished task 2.0 in stage 19.0 (TID 75). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 19.0 (TID 76) in 14 ms on localhost (1/4) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 19.0 (TID 75) in 16 ms on localhost (2/4) -2016-06-13 18:51:19 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 73) -2016-06-13 18:51:19 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 74) -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:19 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 73). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 74). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 73) in 34 ms on localhost (3/4) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 74) in 35 ms on localhost (4/4) -2016-06-13 18:51:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 18:51:19 INFO DAGScheduler:59 - ResultStage 19 (aggregate at GaussianMixture.scala:201) finished in 0.037 s -2016-06-13 18:51:19 INFO DAGScheduler:59 - Job 19 finished: aggregate at GaussianMixture.scala:201, took 0.058847 s -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=57722, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(1079) called with curMem=59066, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 1079.0 B, free 730.5 MB) -2016-06-13 18:51:19 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:65234 (size: 1079.0 B, free: 730.6 MB) -2016-06-13 18:51:19 INFO SparkContext:59 - Created broadcast 37 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:51:19 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:51:19 INFO DAGScheduler:59 - Got job 20 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:51:19 INFO DAGScheduler:59 - Final stage: ResultStage 20(aggregate at GaussianMixture.scala:201) -2016-06-13 18:51:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:19 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=60145, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(2376) called with curMem=64777, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:51:19 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:65234 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:19 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 20 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:19 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 4 tasks -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 77, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 78, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 20.0 (TID 79, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 20.0 (TID 80, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:19 INFO Executor:59 - Running task 3.0 in stage 20.0 (TID 80) -2016-06-13 18:51:19 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 78) -2016-06-13 18:51:19 INFO Executor:59 - Running task 2.0 in stage 20.0 (TID 79) -2016-06-13 18:51:19 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 77) -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:19 INFO Executor:59 - Finished task 3.0 in stage 20.0 (TID 80). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 77). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 20.0 (TID 80) in 14 ms on localhost (1/4) -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:19 INFO Executor:59 - Finished task 2.0 in stage 20.0 (TID 79). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 78). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 77) in 20 ms on localhost (2/4) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 20.0 (TID 79) in 21 ms on localhost (3/4) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 78) in 22 ms on localhost (4/4) -2016-06-13 18:51:19 INFO DAGScheduler:59 - ResultStage 20 (aggregate at GaussianMixture.scala:201) finished in 0.024 s -2016-06-13 18:51:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 18:51:19 INFO DAGScheduler:59 - Job 20 finished: aggregate at GaussianMixture.scala:201, took 0.038897 s -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=67153, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(1083) called with curMem=68497, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 1083.0 B, free 730.5 MB) -2016-06-13 18:51:19 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:65234 (size: 1083.0 B, free: 730.6 MB) -2016-06-13 18:51:19 INFO SparkContext:59 - Created broadcast 39 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:51:19 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:51:19 INFO DAGScheduler:59 - Got job 21 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:51:19 INFO DAGScheduler:59 - Final stage: ResultStage 21(aggregate at GaussianMixture.scala:201) -2016-06-13 18:51:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:19 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=69580, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(2379) called with curMem=74212, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:51:19 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:65234 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:19 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 21 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:19 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 4 tasks -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 81, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 82, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 21.0 (TID 83, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 21.0 (TID 84, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:19 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 82) -2016-06-13 18:51:19 INFO Executor:59 - Running task 2.0 in stage 21.0 (TID 83) -2016-06-13 18:51:19 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 81) -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:19 INFO Executor:59 - Finished task 2.0 in stage 21.0 (TID 83). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO Executor:59 - Running task 3.0 in stage 21.0 (TID 84) -2016-06-13 18:51:19 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 81). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:19 INFO Executor:59 - Finished task 3.0 in stage 21.0 (TID 84). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 82). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 81) in 19 ms on localhost (1/4) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 21.0 (TID 83) in 19 ms on localhost (2/4) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 82) in 20 ms on localhost (3/4) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 21.0 (TID 84) in 19 ms on localhost (4/4) -2016-06-13 18:51:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 18:51:19 INFO DAGScheduler:59 - ResultStage 21 (aggregate at GaussianMixture.scala:201) finished in 0.021 s -2016-06-13 18:51:19 INFO DAGScheduler:59 - Job 21 finished: aggregate at GaussianMixture.scala:201, took 0.034340 s -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(1344) called with curMem=76591, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 1344.0 B, free 730.5 MB) -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(1084) called with curMem=77935, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 1084.0 B, free 730.5 MB) -2016-06-13 18:51:19 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:65234 (size: 1084.0 B, free: 730.6 MB) -2016-06-13 18:51:19 INFO SparkContext:59 - Created broadcast 41 from broadcast at GaussianMixture.scala:198 -2016-06-13 18:51:19 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 18:51:19 INFO DAGScheduler:59 - Got job 22 (aggregate at GaussianMixture.scala:201) with 4 output partitions -2016-06-13 18:51:19 INFO DAGScheduler:59 - Final stage: ResultStage 22(aggregate at GaussianMixture.scala:201) -2016-06-13 18:51:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:19 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(4632) called with curMem=79019, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(2381) called with curMem=83651, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 18:51:19 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:65234 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 18:51:19 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 22 (MapPartitionsRDD[1] at map at GaussianMixture.scala:168) -2016-06-13 18:51:19 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 4 tasks -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 85, localhost, PROCESS_LOCAL, 2258 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 86, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 87, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 22.0 (TID 88, localhost, PROCESS_LOCAL, 2282 bytes) -2016-06-13 18:51:19 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 86) -2016-06-13 18:51:19 INFO Executor:59 - Running task 3.0 in stage 22.0 (TID 88) -2016-06-13 18:51:19 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 85) -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_1 locally -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_0 locally -2016-06-13 18:51:19 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 86). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 85). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 86) in 12 ms on localhost (1/4) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 85) in 14 ms on localhost (2/4) -2016-06-13 18:51:19 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 87) -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_3 locally -2016-06-13 18:51:19 INFO BlockManager:59 - Found block rdd_1_2 locally -2016-06-13 18:51:19 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 87). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO Executor:59 - Finished task 3.0 in stage 22.0 (TID 88). 2934 bytes result sent to driver -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 87) in 21 ms on localhost (3/4) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 22.0 (TID 88) in 22 ms on localhost (4/4) -2016-06-13 18:51:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 18:51:19 INFO DAGScheduler:59 - ResultStage 22 (aggregate at GaussianMixture.scala:201) finished in 0.025 s -2016-06-13 18:51:19 INFO DAGScheduler:59 - Job 22 finished: aggregate at GaussianMixture.scala:201, took 0.039199 s -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(1272) called with curMem=86032, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 1272.0 B, free 730.5 MB) -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(969) called with curMem=87304, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 969.0 B, free 730.5 MB) -2016-06-13 18:51:19 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:65234 (size: 969.0 B, free: 730.6 MB) -2016-06-13 18:51:19 INFO SparkContext:59 - Created broadcast 43 from broadcast at GaussianMixtureModel.scala:100 -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(72) called with curMem=88273, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 72.0 B, free 730.5 MB) -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(81) called with curMem=88345, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 81.0 B, free 730.5 MB) -2016-06-13 18:51:19 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:65234 (size: 81.0 B, free: 730.6 MB) -2016-06-13 18:51:19 INFO SparkContext:59 - Created broadcast 44 from broadcast at GaussianMixtureModel.scala:101 -2016-06-13 18:51:19 INFO SparkContext:59 - Starting job: collect at GaussianMixtureFromSparkSuite.scala:201 -2016-06-13 18:51:19 INFO DAGScheduler:59 - Got job 23 (collect at GaussianMixtureFromSparkSuite.scala:201) with 4 output partitions -2016-06-13 18:51:19 INFO DAGScheduler:59 - Final stage: ResultStage 23(collect at GaussianMixtureFromSparkSuite.scala:201) -2016-06-13 18:51:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 18:51:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 18:51:19 INFO DAGScheduler:59 - Submitting ResultStage 23 (ZippedPartitionsRDD2[5] at zip at GaussianMixtureFromSparkSuite.scala:201), which has no missing parents -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(5072) called with curMem=88426, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 5.0 KB, free 730.5 MB) -2016-06-13 18:51:19 INFO MemoryStore:59 - ensureFreeSpace(2783) called with curMem=93498, maxMem=766075207 -2016-06-13 18:51:19 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.5 MB) -2016-06-13 18:51:19 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:65234 (size: 2.7 KB, free: 730.6 MB) -2016-06-13 18:51:19 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 18:51:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 23 (ZippedPartitionsRDD2[5] at zip at GaussianMixtureFromSparkSuite.scala:201) -2016-06-13 18:51:19 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 4 tasks -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 89, localhost, PROCESS_LOCAL, 2432 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 90, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 23.0 (TID 91, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 23.0 (TID 92, localhost, PROCESS_LOCAL, 2456 bytes) -2016-06-13 18:51:19 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 89) -2016-06-13 18:51:19 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 90) -2016-06-13 18:51:19 INFO Executor:59 - Running task 2.0 in stage 23.0 (TID 91) -2016-06-13 18:51:19 INFO Executor:59 - Running task 3.0 in stage 23.0 (TID 92) -2016-06-13 18:51:19 INFO Executor:59 - Finished task 2.0 in stage 23.0 (TID 91). 1264 bytes result sent to driver -2016-06-13 18:51:19 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 89). 1234 bytes result sent to driver -2016-06-13 18:51:19 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 90). 1274 bytes result sent to driver -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 23.0 (TID 91) in 18 ms on localhost (1/4) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 89) in 22 ms on localhost (2/4) -2016-06-13 18:51:19 INFO Executor:59 - Finished task 3.0 in stage 23.0 (TID 92). 1264 bytes result sent to driver -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 90) in 21 ms on localhost (3/4) -2016-06-13 18:51:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 23.0 (TID 92) in 21 ms on localhost (4/4) -2016-06-13 18:51:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 18:51:19 INFO DAGScheduler:59 - ResultStage 23 (collect at GaussianMixtureFromSparkSuite.scala:201) finished in 0.025 s -2016-06-13 18:51:19 INFO DAGScheduler:59 - Job 23 finished: collect at GaussianMixtureFromSparkSuite.scala:201, took 0.038567 s -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 18:51:19 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 18:51:19 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 18:51:19 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 18:51:19 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 18:51:19 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 18:51:19 INFO BlockManager:59 - BlockManager stopped -2016-06-13 18:51:19 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 18:51:19 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 18:51:19 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 18:51:19 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 18:51:19 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-a4f89345-b9c7-4ae5-ac38-91284e31bb6b -2016-06-13 18:51:19 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 18:51:19 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 19:10:23 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 19:10:23 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 19:10:23 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 19:10:23 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 19:10:23 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 19:10:24 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 19:10:24 INFO Remoting:74 - Starting remoting -2016-06-13 19:10:24 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:65372] -2016-06-13 19:10:24 INFO Utils:59 - Successfully started service 'sparkDriver' on port 65372. -2016-06-13 19:10:24 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 19:10:24 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 19:10:24 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-de7f7294-b396-4394-87bb-f8df8a2e6c69 -2016-06-13 19:10:24 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 19:10:24 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-135bf678-7f09-4b71-a629-ecc687625ef6\httpd-e8a7fe35-673f-42bd-8610-7a22d131ae45 -2016-06-13 19:10:24 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 19:10:24 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:10:24 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:65373 -2016-06-13 19:10:24 INFO Utils:59 - Successfully started service 'HTTP file server' on port 65373. -2016-06-13 19:10:25 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 19:10:25 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:10:25 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 19:10:25 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 19:10:25 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 19:10:25 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 19:10:25 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 19:10:25 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 65392. -2016-06-13 19:10:25 INFO NettyBlockTransferService:59 - Server created on 65392 -2016-06-13 19:10:25 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 19:10:25 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:65392 with 730.6 MB RAM, BlockManagerId(driver, localhost, 65392) -2016-06-13 19:10:25 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 19:10:26 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 19:10:26 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 19:10:26 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 19:10:26 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 19:10:26 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:65392 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 19:10:26 INFO SparkContext:59 - Created broadcast 0 from textFile at GaussianMixtureFromWebSuite.scala:23 -2016-06-13 19:10:27 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 19:10:27 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 19:10:27 INFO SparkContext:59 - Starting job: first at GaussianMixture.scala:171 -2016-06-13 19:10:27 INFO DAGScheduler:59 - Got job 0 (first at GaussianMixture.scala:171) with 1 output partitions -2016-06-13 19:10:27 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at GaussianMixture.scala:171) -2016-06-13 19:10:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:27 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:27 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=171580, maxMem=766075207 -2016-06-13 19:10:27 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:10:27 INFO MemoryStore:59 - ensureFreeSpace(2050) called with curMem=175172, maxMem=766075207 -2016-06-13 19:10:27 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:10:27 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:65392 (size: 2.0 KB, free: 730.6 MB) -2016-06-13 19:10:27 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:27 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 19:10:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:27 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 19:10:27 INFO CacheManager:59 - Partition rdd_3_0 not found, computing it -2016-06-13 19:10:27 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-06-13 19:10:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/gmm_data.txt:0+31986 -2016-06-13 19:10:27 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 19:10:27 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 19:10:27 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 19:10:27 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 19:10:27 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 19:10:28 INFO MemoryStore:59 - ensureFreeSpace(52024) called with curMem=177222, maxMem=766075207 -2016-06-13 19:10:28 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 50.8 KB, free 730.4 MB) -2016-06-13 19:10:28 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:65392 (size: 50.8 KB, free: 730.5 MB) -2016-06-13 19:10:28 INFO MemoryStore:59 - ensureFreeSpace(76024) called with curMem=229246, maxMem=766075207 -2016-06-13 19:10:28 INFO MemoryStore:59 - Block rdd_3_0 stored as values in memory (estimated size 74.2 KB, free 730.3 MB) -2016-06-13 19:10:28 INFO BlockManagerInfo:59 - Added rdd_3_0 in memory on localhost:65392 (size: 74.2 KB, free: 730.4 MB) -2016-06-13 19:10:28 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2846 bytes result sent to driver -2016-06-13 19:10:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 663 ms on localhost (1/1) -2016-06-13 19:10:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 19:10:28 INFO DAGScheduler:59 - ResultStage 0 (first at GaussianMixture.scala:171) finished in 0.677 s -2016-06-13 19:10:28 INFO DAGScheduler:59 - Job 0 finished: first at GaussianMixture.scala:171, took 0.760493 s -2016-06-13 19:10:28 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 19:10:28 INFO DAGScheduler:59 - Got job 1 (takeSample at GaussianMixture.scala:184) with 2 output partitions -2016-06-13 19:10:28 INFO DAGScheduler:59 - Final stage: ResultStage 1(takeSample at GaussianMixture.scala:184) -2016-06-13 19:10:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:28 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:28 INFO MemoryStore:59 - ensureFreeSpace(3424) called with curMem=305270, maxMem=766075207 -2016-06-13 19:10:28 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.3 KB, free 730.3 MB) -2016-06-13 19:10:28 INFO MemoryStore:59 - ensureFreeSpace(1985) called with curMem=308694, maxMem=766075207 -2016-06-13 19:10:28 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1985.0 B, free 730.3 MB) -2016-06-13 19:10:28 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:65392 (size: 1985.0 B, free: 730.4 MB) -2016-06-13 19:10:28 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:28 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 19:10:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:28 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 19:10:28 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 19:10:28 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:28 INFO CacheManager:59 - Partition rdd_3_1 not found, computing it -2016-06-13 19:10:28 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-06-13 19:10:28 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/gmm_data.txt:31986+31987 -2016-06-13 19:10:28 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2082 bytes result sent to driver -2016-06-13 19:10:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 20 ms on localhost (1/2) -2016-06-13 19:10:28 INFO MemoryStore:59 - ensureFreeSpace(47216) called with curMem=310679, maxMem=766075207 -2016-06-13 19:10:28 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 46.1 KB, free 730.2 MB) -2016-06-13 19:10:28 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:65392 (size: 46.1 KB, free: 730.4 MB) -2016-06-13 19:10:28 INFO MemoryStore:59 - ensureFreeSpace(68816) called with curMem=357895, maxMem=766075207 -2016-06-13 19:10:28 INFO MemoryStore:59 - Block rdd_3_1 stored as values in memory (estimated size 67.2 KB, free 730.2 MB) -2016-06-13 19:10:28 INFO BlockManagerInfo:59 - Added rdd_3_1 in memory on localhost:65392 (size: 67.2 KB, free: 730.3 MB) -2016-06-13 19:10:28 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2717 bytes result sent to driver -2016-06-13 19:10:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 107 ms on localhost (2/2) -2016-06-13 19:10:28 INFO DAGScheduler:59 - ResultStage 1 (takeSample at GaussianMixture.scala:184) finished in 0.110 s -2016-06-13 19:10:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 19:10:28 INFO DAGScheduler:59 - Job 1 finished: takeSample at GaussianMixture.scala:184, took 0.128520 s -2016-06-13 19:10:28 INFO SparkContext:59 - Starting job: takeSample at GaussianMixture.scala:184 -2016-06-13 19:10:28 INFO DAGScheduler:59 - Got job 2 (takeSample at GaussianMixture.scala:184) with 2 output partitions -2016-06-13 19:10:28 INFO DAGScheduler:59 - Final stage: ResultStage 2(takeSample at GaussianMixture.scala:184) -2016-06-13 19:10:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:28 INFO DAGScheduler:59 - Submitting ResultStage 2 (PartitionwiseSampledRDD[4] at takeSample at GaussianMixture.scala:184), which has no missing parents -2016-06-13 19:10:28 INFO MemoryStore:59 - ensureFreeSpace(20816) called with curMem=426711, maxMem=766075207 -2016-06-13 19:10:28 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 20.3 KB, free 730.2 MB) -2016-06-13 19:10:28 INFO MemoryStore:59 - ensureFreeSpace(8647) called with curMem=447527, maxMem=766075207 -2016-06-13 19:10:29 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 8.4 KB, free 730.2 MB) -2016-06-13 19:10:29 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:65392 (size: 8.4 KB, free: 730.3 MB) -2016-06-13 19:10:29 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (PartitionwiseSampledRDD[4] at takeSample at GaussianMixture.scala:184) -2016-06-13 19:10:29 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 19:10:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2299 bytes) -2016-06-13 19:10:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2299 bytes) -2016-06-13 19:10:29 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-06-13 19:10:29 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 19:10:29 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:29 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:29 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2786 bytes result sent to driver -2016-06-13 19:10:29 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 3275 bytes result sent to driver -2016-06-13 19:10:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 35 ms on localhost (1/2) -2016-06-13 19:10:29 INFO DAGScheduler:59 - ResultStage 2 (takeSample at GaussianMixture.scala:184) finished in 0.041 s -2016-06-13 19:10:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 39 ms on localhost (2/2) -2016-06-13 19:10:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 19:10:29 INFO DAGScheduler:59 - Job 2 finished: takeSample at GaussianMixture.scala:184, took 0.346999 s -2016-06-13 19:10:29 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 19:10:29 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 19:10:29 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 19:10:29 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 19:10:29 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=456174, maxMem=766075207 -2016-06-13 19:10:29 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-06-13 19:10:29 INFO MemoryStore:59 - ensureFreeSpace(1093) called with curMem=457398, maxMem=766075207 -2016-06-13 19:10:29 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 1093.0 B, free 730.1 MB) -2016-06-13 19:10:29 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:65392 (size: 1093.0 B, free: 730.3 MB) -2016-06-13 19:10:29 INFO SparkContext:59 - Created broadcast 4 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:29 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:65392 in memory (size: 8.4 KB, free: 730.3 MB) -2016-06-13 19:10:29 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 19:10:29 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:29 INFO DAGScheduler:59 - Got job 3 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:29 INFO DAGScheduler:59 - Final stage: ResultStage 3(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:29 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:29 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:65392 in memory (size: 1985.0 B, free: 730.3 MB) -2016-06-13 19:10:29 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 19:10:29 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=421569, maxMem=766075207 -2016-06-13 19:10:29 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:65392 in memory (size: 2.0 KB, free: 730.3 MB) -2016-06-13 19:10:29 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 6.1 KB, free 730.2 MB) -2016-06-13 19:10:29 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 19:10:29 INFO MemoryStore:59 - ensureFreeSpace(3178) called with curMem=424225, maxMem=766075207 -2016-06-13 19:10:29 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.2 MB) -2016-06-13 19:10:29 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:65392 (size: 3.1 KB, free: 730.3 MB) -2016-06-13 19:10:29 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:29 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 19:10:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:29 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-06-13 19:10:29 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-06-13 19:10:29 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:29 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:29 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2944 bytes result sent to driver -2016-06-13 19:10:29 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2944 bytes result sent to driver -2016-06-13 19:10:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 247 ms on localhost (1/2) -2016-06-13 19:10:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 249 ms on localhost (2/2) -2016-06-13 19:10:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 19:10:29 INFO DAGScheduler:59 - ResultStage 3 (aggregate at GaussianMixture.scala:201) finished in 0.249 s -2016-06-13 19:10:29 INFO DAGScheduler:59 - Job 3 finished: aggregate at GaussianMixture.scala:201, took 0.266046 s -2016-06-13 19:10:29 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=427403, maxMem=766075207 -2016-06-13 19:10:29 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-06-13 19:10:29 INFO MemoryStore:59 - ensureFreeSpace(1175) called with curMem=428627, maxMem=766075207 -2016-06-13 19:10:29 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1175.0 B, free 730.2 MB) -2016-06-13 19:10:29 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:65392 (size: 1175.0 B, free: 730.3 MB) -2016-06-13 19:10:29 INFO SparkContext:59 - Created broadcast 6 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:29 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:29 INFO DAGScheduler:59 - Got job 4 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:29 INFO DAGScheduler:59 - Final stage: ResultStage 4(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:29 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:29 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=429802, maxMem=766075207 -2016-06-13 19:10:29 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 6.1 KB, free 730.2 MB) -2016-06-13 19:10:29 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=436050, maxMem=766075207 -2016-06-13 19:10:29 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.2 MB) -2016-06-13 19:10:29 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:29 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:29 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 19:10:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:29 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-06-13 19:10:29 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-06-13 19:10:29 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:29 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:30 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 128 ms on localhost (1/2) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 128 ms on localhost (2/2) -2016-06-13 19:10:30 INFO DAGScheduler:59 - ResultStage 4 (aggregate at GaussianMixture.scala:201) finished in 0.129 s -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 19:10:30 INFO DAGScheduler:59 - Job 4 finished: aggregate at GaussianMixture.scala:201, took 0.144611 s -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=439303, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1166) called with curMem=440527, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 1166.0 B, free 730.2 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:65392 (size: 1166.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 8 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:30 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Got job 5 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 5(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=441693, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 6.1 KB, free 730.2 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(3254) called with curMem=447941, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.2 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-06-13 19:10:30 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:30 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 32 ms on localhost (1/2) -2016-06-13 19:10:30 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO DAGScheduler:59 - ResultStage 5 (aggregate at GaussianMixture.scala:201) finished in 0.041 s -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 41 ms on localhost (2/2) -2016-06-13 19:10:30 INFO DAGScheduler:59 - Job 5 finished: aggregate at GaussianMixture.scala:201, took 0.061265 s -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=451195, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1173) called with curMem=452419, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 1173.0 B, free 730.2 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:65392 (size: 1173.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 10 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:30 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Got job 6 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 6(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=453592, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(3257) called with curMem=459840, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 12, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 11) -2016-06-13 19:10:30 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 12) -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:30 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 11). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 12). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 11) in 22 ms on localhost (1/2) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 12) in 22 ms on localhost (2/2) -2016-06-13 19:10:30 INFO DAGScheduler:59 - ResultStage 6 (aggregate at GaussianMixture.scala:201) finished in 0.025 s -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 19:10:30 INFO DAGScheduler:59 - Job 6 finished: aggregate at GaussianMixture.scala:201, took 0.039104 s -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=463097, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1166) called with curMem=464321, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 1166.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:65392 (size: 1166.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 12 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:30 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Got job 7 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 7(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=465487, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(3258) called with curMem=471735, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 14, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 14) -2016-06-13 19:10:30 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 13) -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:30 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 14). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 13). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 14) in 22 ms on localhost (1/2) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 13) in 24 ms on localhost (2/2) -2016-06-13 19:10:30 INFO DAGScheduler:59 - ResultStage 7 (aggregate at GaussianMixture.scala:201) finished in 0.025 s -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 19:10:30 INFO DAGScheduler:59 - Job 7 finished: aggregate at GaussianMixture.scala:201, took 0.038247 s -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=474993, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1170) called with curMem=476217, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 1170.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:65392 (size: 1170.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 14 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:30 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Got job 8 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 8(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=477387, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(3261) called with curMem=483635, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 16, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 15) -2016-06-13 19:10:30 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 16) -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:30 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 16). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 15). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 16) in 23 ms on localhost (1/2) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 15) in 28 ms on localhost (2/2) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 19:10:30 INFO DAGScheduler:59 - ResultStage 8 (aggregate at GaussianMixture.scala:201) finished in 0.030 s -2016-06-13 19:10:30 INFO DAGScheduler:59 - Job 8 finished: aggregate at GaussianMixture.scala:201, took 0.050271 s -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=486896, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1174) called with curMem=488120, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 1174.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:65392 (size: 1174.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 16 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:65392 in memory (size: 1170.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:30 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Got job 9 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 9(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:65392 in memory (size: 1166.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=459247, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(3258) called with curMem=462238, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 17, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 18, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:65392 in memory (size: 1173.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 17) -2016-06-13 19:10:30 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 18) -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:30 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 17). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 18). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 17) in 25 ms on localhost (1/2) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 18) in 25 ms on localhost (2/2) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 19:10:30 INFO DAGScheduler:59 - ResultStage 9 (aggregate at GaussianMixture.scala:201) finished in 0.030 s -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO DAGScheduler:59 - Job 9 finished: aggregate at GaussianMixture.scala:201, took 0.046154 s -2016-06-13 19:10:30 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=453597, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:65392 in memory (size: 1166.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1165) called with curMem=452431, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 1165.0 B, free 730.2 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:65392 (size: 1165.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 18 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:65392 in memory (size: 1175.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:65392 in memory (size: 3.1 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:30 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Got job 10 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 10(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting ResultStage 10 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:65392 in memory (size: 1093.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=429953, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 6.1 KB, free 730.2 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(3265) called with curMem=436201, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.2 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 10 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 19, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 20, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 19) -2016-06-13 19:10:30 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 20) -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:30 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 19). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 20). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 19) in 19 ms on localhost (1/2) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 20) in 19 ms on localhost (2/2) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 19:10:30 INFO DAGScheduler:59 - ResultStage 10 (aggregate at GaussianMixture.scala:201) finished in 0.022 s -2016-06-13 19:10:30 INFO DAGScheduler:59 - Job 10 finished: aggregate at GaussianMixture.scala:201, took 0.040882 s -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=439466, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1161) called with curMem=440690, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 1161.0 B, free 730.2 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:65392 (size: 1161.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 20 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:30 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Got job 11 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 11(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=441851, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 6.1 KB, free 730.2 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=448099, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.2 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 21, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 22, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 21) -2016-06-13 19:10:30 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 22) -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:30 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 21). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 21) in 14 ms on localhost (1/2) -2016-06-13 19:10:30 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 22). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 22) in 15 ms on localhost (2/2) -2016-06-13 19:10:30 INFO DAGScheduler:59 - ResultStage 11 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 19:10:30 INFO DAGScheduler:59 - Job 11 finished: aggregate at GaussianMixture.scala:201, took 0.032917 s -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=451352, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 1224.0 B, free 730.2 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1163) called with curMem=452576, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 1163.0 B, free 730.2 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:65392 (size: 1163.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 22 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:30 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Got job 12 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 12(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=453739, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(3254) called with curMem=459987, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 23, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 24, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 23) -2016-06-13 19:10:30 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 24) -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:30 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 24). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 23). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 24) in 15 ms on localhost (1/2) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 23) in 17 ms on localhost (2/2) -2016-06-13 19:10:30 INFO DAGScheduler:59 - ResultStage 12 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 19:10:30 INFO DAGScheduler:59 - Job 12 finished: aggregate at GaussianMixture.scala:201, took 0.031853 s -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=463241, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1164) called with curMem=464465, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 1164.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:65392 (size: 1164.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 24 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:30 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Got job 13 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 13(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=465629, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(3254) called with curMem=471877, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 25, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 26, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 25) -2016-06-13 19:10:30 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 26) -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:30 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 26). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 25). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 26) in 16 ms on localhost (1/2) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 25) in 19 ms on localhost (2/2) -2016-06-13 19:10:30 INFO DAGScheduler:59 - ResultStage 13 (aggregate at GaussianMixture.scala:201) finished in 0.019 s -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 19:10:30 INFO DAGScheduler:59 - Job 13 finished: aggregate at GaussianMixture.scala:201, took 0.032191 s -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=475131, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1165) called with curMem=476355, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 1165.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:65392 (size: 1165.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 26 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:30 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Got job 14 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 14(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting ResultStage 14 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=477520, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(3252) called with curMem=483768, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 14 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 2 tasks -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 27, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 28, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 27) -2016-06-13 19:10:30 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 28) -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:30 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 28). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 27). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 28) in 13 ms on localhost (1/2) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 27) in 17 ms on localhost (2/2) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 19:10:30 INFO DAGScheduler:59 - ResultStage 14 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 19:10:30 INFO DAGScheduler:59 - Job 14 finished: aggregate at GaussianMixture.scala:201, took 0.032735 s -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=487020, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1167) called with curMem=488244, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 1167.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:65392 (size: 1167.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 28 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:30 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Got job 15 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 15(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=489411, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=495659, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 29, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 30, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 29) -2016-06-13 19:10:30 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 30) -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:30 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 30). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 29). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 30) in 14 ms on localhost (1/2) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 29) in 16 ms on localhost (2/2) -2016-06-13 19:10:30 INFO DAGScheduler:59 - ResultStage 15 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 19:10:30 INFO DAGScheduler:59 - Job 15 finished: aggregate at GaussianMixture.scala:201, took 0.031784 s -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=498912, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1163) called with curMem=500136, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 1163.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:65392 (size: 1163.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 30 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:30 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Got job 16 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 16(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting ResultStage 16 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=501299, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(3252) called with curMem=507547, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 16 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 31, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 32, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 31) -2016-06-13 19:10:30 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 32) -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:30 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 32). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 31). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 32) in 18 ms on localhost (1/2) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 31) in 19 ms on localhost (2/2) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 19:10:30 INFO DAGScheduler:59 - ResultStage 16 (aggregate at GaussianMixture.scala:201) finished in 0.020 s -2016-06-13 19:10:30 INFO DAGScheduler:59 - Job 16 finished: aggregate at GaussianMixture.scala:201, took 0.033637 s -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=510799, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1165) called with curMem=512023, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 1165.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:65392 (size: 1165.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 32 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:30 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Got job 17 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 17(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting ResultStage 17 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=513188, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=519436, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 33, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 34, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 33) -2016-06-13 19:10:30 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 34) -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:30 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 33). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 34). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 33) in 22 ms on localhost (1/2) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 34) in 22 ms on localhost (2/2) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 19:10:30 INFO DAGScheduler:59 - ResultStage 17 (aggregate at GaussianMixture.scala:201) finished in 0.023 s -2016-06-13 19:10:30 INFO DAGScheduler:59 - Job 17 finished: aggregate at GaussianMixture.scala:201, took 0.039572 s -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=522689, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1169) called with curMem=523913, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 1169.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:65392 (size: 1169.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 34 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:30 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Got job 18 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 18(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting ResultStage 18 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=525082, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=531330, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 18 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 2 tasks -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 35, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 36, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 35) -2016-06-13 19:10:30 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 36) -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:30 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 35). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 36). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 35) in 24 ms on localhost (1/2) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 36) in 26 ms on localhost (2/2) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 19:10:30 INFO DAGScheduler:59 - ResultStage 18 (aggregate at GaussianMixture.scala:201) finished in 0.027 s -2016-06-13 19:10:30 INFO DAGScheduler:59 - Job 18 finished: aggregate at GaussianMixture.scala:201, took 0.047347 s -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=534583, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1164) called with curMem=535807, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 1164.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:65392 (size: 1164.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 36 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:30 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Got job 19 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 19(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=536971, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(3254) called with curMem=543219, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 37, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 38, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:30 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 38) -2016-06-13 19:10:30 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 37) -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:30 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:30 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 37). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 37) in 22 ms on localhost (1/2) -2016-06-13 19:10:30 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 38). 2944 bytes result sent to driver -2016-06-13 19:10:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 38) in 27 ms on localhost (2/2) -2016-06-13 19:10:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 19:10:30 INFO DAGScheduler:59 - ResultStage 19 (aggregate at GaussianMixture.scala:201) finished in 0.029 s -2016-06-13 19:10:30 INFO DAGScheduler:59 - Job 19 finished: aggregate at GaussianMixture.scala:201, took 0.049255 s -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=546473, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO MemoryStore:59 - ensureFreeSpace(1166) called with curMem=547697, maxMem=766075207 -2016-06-13 19:10:30 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 1166.0 B, free 730.1 MB) -2016-06-13 19:10:30 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:65392 (size: 1166.0 B, free: 730.3 MB) -2016-06-13 19:10:30 INFO SparkContext:59 - Created broadcast 38 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:30 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:30 INFO DAGScheduler:59 - Got job 20 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:30 INFO DAGScheduler:59 - Final stage: ResultStage 20(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 20 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=548863, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=555111, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 20 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 39, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 40, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 39) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 40) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 39). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 40). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 39) in 24 ms on localhost (1/2) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 40) in 24 ms on localhost (2/2) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 20 (aggregate at GaussianMixture.scala:201) finished in 0.026 s -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 20 finished: aggregate at GaussianMixture.scala:201, took 0.044801 s -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=558364, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1175) called with curMem=559588, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 1175.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:65392 (size: 1175.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 40 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 21 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 21(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=560763, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=567011, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 41, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 42, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 41) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 42) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 42). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 42) in 56 ms on localhost (1/2) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 41). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 41) in 66 ms on localhost (2/2) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 21 (aggregate at GaussianMixture.scala:201) finished in 0.066 s -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 21 finished: aggregate at GaussianMixture.scala:201, took 0.081440 s -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:65392 in memory (size: 1166.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=558373, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1171) called with curMem=559597, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1171.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:65392 (size: 1171.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 42 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:65392 in memory (size: 1164.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 22 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 22(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=539377, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3255) called with curMem=545625, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 43, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:65392 in memory (size: 1169.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 44, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 44) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 44). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:65392 in memory (size: 1165.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 44) in 14 ms on localhost (1/2) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 43) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:65392 in memory (size: 1163.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 43). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 43) in 29 ms on localhost (2/2) -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 22 (aggregate at GaussianMixture.scala:201) finished in 0.031 s -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 22 finished: aggregate at GaussianMixture.scala:201, took 0.044772 s -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=513209, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1169) called with curMem=512042, maxMem=766075207 -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:65392 in memory (size: 1167.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 1169.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:65392 (size: 1169.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 44 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:65392 in memory (size: 1165.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 23 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 23(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=491820, maxMem=766075207 -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3254) called with curMem=496904, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:65392 in memory (size: 1164.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 45, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 46, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 46) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 46). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:65392 in memory (size: 1163.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 46) in 14 ms on localhost (1/2) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 45) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 45). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:65392 in memory (size: 1161.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 45) in 29 ms on localhost (2/2) -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 23 (aggregate at GaussianMixture.scala:201) finished in 0.030 s -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 23 finished: aggregate at GaussianMixture.scala:201, took 0.044791 s -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=475159, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1176) called with curMem=466870, maxMem=766075207 -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 1176.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:65392 (size: 1176.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 46 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:65392 in memory (size: 1165.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 24 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 24(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 24 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=454977, maxMem=766075207 -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:65392 in memory (size: 1174.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3258) called with curMem=460001, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 24 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 47, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 48, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 48) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 47) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 47). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 48). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 47) in 16 ms on localhost (1/2) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 48) in 16 ms on localhost (2/2) -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 24 (aggregate at GaussianMixture.scala:201) finished in 0.018 s -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 24 finished: aggregate at GaussianMixture.scala:201, took 0.031422 s -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=463259, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1163) called with curMem=464483, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 1163.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:65392 (size: 1163.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 48 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 25 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 25(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 25 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=465646, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3254) called with curMem=471894, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 49, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 50, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 49) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 50) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 49). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 50). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 49) in 17 ms on localhost (1/2) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 50) in 18 ms on localhost (2/2) -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 25 (aggregate at GaussianMixture.scala:201) finished in 0.019 s -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 25 finished: aggregate at GaussianMixture.scala:201, took 0.034161 s -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=475148, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1164) called with curMem=476372, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 1164.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:65392 (size: 1164.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 50 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 26 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 26(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 26 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=477536, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3260) called with curMem=483784, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 26 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 51, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 52, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 51) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 52) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 51). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 52). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 51) in 14 ms on localhost (1/2) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 52) in 15 ms on localhost (2/2) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 26 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 26 finished: aggregate at GaussianMixture.scala:201, took 0.028720 s -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=487044, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1167) called with curMem=488268, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 1167.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:65392 (size: 1167.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 52 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 27 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 27(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=489435, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3257) called with curMem=495683, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 53, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 54, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 53) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 54) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 54). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 53). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 54) in 14 ms on localhost (1/2) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 53) in 16 ms on localhost (2/2) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 27 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 27 finished: aggregate at GaussianMixture.scala:201, took 0.032677 s -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=498940, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1167) called with curMem=500164, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 1167.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:65392 (size: 1167.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 54 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 28 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 28(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 28 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=501331, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3256) called with curMem=507579, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 28 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 55, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 56, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 55) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 56) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 56). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 55). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 56) in 14 ms on localhost (1/2) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 55) in 16 ms on localhost (2/2) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 28 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 28 finished: aggregate at GaussianMixture.scala:201, took 0.030827 s -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=510835, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1164) called with curMem=512059, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 1164.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:65392 (size: 1164.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 56 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 29 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 29(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 29 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=513223, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=519471, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 57, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 58, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 57) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 58) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 58). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 58) in 13 ms on localhost (1/2) -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 57). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 57) in 16 ms on localhost (2/2) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 29 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 29 finished: aggregate at GaussianMixture.scala:201, took 0.030423 s -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=522724, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1167) called with curMem=523948, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 1167.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:65392 (size: 1167.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 58 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 30 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 30(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 30 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=525115, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3255) called with curMem=531363, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 30 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 59, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 60, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 59) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 60) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 60). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 59). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 60) in 12 ms on localhost (1/2) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 59) in 14 ms on localhost (2/2) -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 30 (aggregate at GaussianMixture.scala:201) finished in 0.014 s -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 30 finished: aggregate at GaussianMixture.scala:201, took 0.024615 s -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=534618, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1163) called with curMem=535842, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 1163.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:65392 (size: 1163.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 60 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 31 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 31(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=537005, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=543253, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 61 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 61, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 62, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 61) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 62) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 61). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 62). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 61) in 7 ms on localhost (1/2) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 62) in 8 ms on localhost (2/2) -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 31 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 31 finished: aggregate at GaussianMixture.scala:201, took 0.017641 s -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=546506, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1169) called with curMem=547730, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 1169.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:65392 (size: 1169.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 62 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 32 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 32(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=548899, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3261) called with curMem=555147, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 63, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 64, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 63) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 64) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 63). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 63) in 9 ms on localhost (1/2) -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 64). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 64) in 13 ms on localhost (2/2) -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 32 (aggregate at GaussianMixture.scala:201) finished in 0.014 s -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 32 finished: aggregate at GaussianMixture.scala:201, took 0.022924 s -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=558408, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1161) called with curMem=559632, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 1161.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:65392 (size: 1161.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 64 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 33 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 33(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=560793, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3249) called with curMem=567041, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 65, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 66, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 66) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 65) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 65). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 66). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 66) in 8 ms on localhost (1/2) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 65) in 8 ms on localhost (2/2) -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 33 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 33 finished: aggregate at GaussianMixture.scala:201, took 0.016568 s -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=570290, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1167) called with curMem=571514, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 1167.0 B, free 730.0 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:65392 (size: 1167.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 66 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 34 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 34(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 34 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=572681, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3255) called with curMem=578929, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 67 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 34 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 67, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 68, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 67) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 68) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 67). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 68). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 67) in 16 ms on localhost (1/2) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 68) in 17 ms on localhost (2/2) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 34 (aggregate at GaussianMixture.scala:201) finished in 0.017 s -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 34 finished: aggregate at GaussianMixture.scala:201, took 0.025829 s -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:65392 in memory (size: 1171.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=576540, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1167) called with curMem=571516, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 1167.0 B, free 730.0 MB) -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:65392 (size: 1167.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 68 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:65392 in memory (size: 1161.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 35 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 35(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:65392 in memory (size: 1169.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=555143, maxMem=766075207 -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3254) called with curMem=555143, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 69 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:65392 in memory (size: 1163.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 69, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 70, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 70) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 69) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:65392 in memory (size: 1167.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 69). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 70). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 70) in 13 ms on localhost (1/2) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 69) in 15 ms on localhost (2/2) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 35 (aggregate at GaussianMixture.scala:201) finished in 0.016 s -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 35 finished: aggregate at GaussianMixture.scala:201, took 0.028474 s -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:65392 in memory (size: 1164.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=528971, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1162) called with curMem=523947, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 1162.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:65392 (size: 1162.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:65392 in memory (size: 1167.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 70 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 36 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 36(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:65392 in memory (size: 1167.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 36 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=501314, maxMem=766075207 -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3252) called with curMem=506398, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 71 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 36 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:65392 in memory (size: 1164.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 71, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 72, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 71) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 72) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:65392 in memory (size: 1163.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 71). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 72). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 72) in 11 ms on localhost (1/2) -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 36 (aggregate at GaussianMixture.scala:201) finished in 0.013 s -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 71) in 13 ms on localhost (2/2) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 36 finished: aggregate at GaussianMixture.scala:201, took 0.023377 s -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=485855, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:65392 in memory (size: 1176.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1165) called with curMem=485855, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 1165.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:65392 (size: 1165.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 72 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:65392 in memory (size: 1169.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 37 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 37(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 37 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=462369, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=462369, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 73 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:65392 in memory (size: 1175.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 73, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 74, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 73) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 74) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 74). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 73). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 74) in 9 ms on localhost (1/2) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 73) in 10 ms on localhost (2/2) -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 37 (aggregate at GaussianMixture.scala:201) finished in 0.010 s -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 37 finished: aggregate at GaussianMixture.scala:201, took 0.020473 s -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=463223, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1165) called with curMem=464447, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 1165.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:65392 (size: 1165.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 74 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 38 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 38(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 38 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=465612, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3248) called with curMem=471860, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 75 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 38 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 75, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 76, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 75) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 76) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 76). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 75). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 76) in 7 ms on localhost (1/2) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 75) in 7 ms on localhost (2/2) -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 38 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 38 finished: aggregate at GaussianMixture.scala:201, took 0.016271 s -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=475108, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1167) called with curMem=476332, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 1167.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:65392 (size: 1167.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 76 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 39 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 39(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=477499, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=483747, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 77 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 77, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 78, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 77) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 78) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 78). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 77). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 78) in 9 ms on localhost (1/2) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 77) in 10 ms on localhost (2/2) -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 39 (aggregate at GaussianMixture.scala:201) finished in 0.010 s -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 39 finished: aggregate at GaussianMixture.scala:201, took 0.018999 s -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=487000, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1163) called with curMem=488224, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 1163.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:65392 (size: 1163.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 78 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 40 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 40(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 40 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=489387, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=495635, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 79 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 40 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 40.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 40.0 (TID 79, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 40.0 (TID 80, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 40.0 (TID 79) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 40.0 (TID 80) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO Executor:59 - Finished task 0.0 in stage 40.0 (TID 79). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO Executor:59 - Finished task 1.0 in stage 40.0 (TID 80). 2944 bytes result sent to driver -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 40.0 (TID 79) in 10 ms on localhost (1/2) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 40.0 (TID 80) in 10 ms on localhost (2/2) -2016-06-13 19:10:31 INFO DAGScheduler:59 - ResultStage 40 (aggregate at GaussianMixture.scala:201) finished in 0.011 s -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 40.0, whose tasks have all completed, from pool -2016-06-13 19:10:31 INFO DAGScheduler:59 - Job 40 finished: aggregate at GaussianMixture.scala:201, took 0.020352 s -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=498888, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(1163) called with curMem=500112, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 1163.0 B, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:65392 (size: 1163.0 B, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 80 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:31 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Got job 41 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:31 INFO DAGScheduler:59 - Final stage: ResultStage 41(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=501275, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=507523, maxMem=766075207 -2016-06-13 19:10:31 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:31 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:31 INFO SparkContext:59 - Created broadcast 81 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:31 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 81, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 82, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:31 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 81) -2016-06-13 19:10:31 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 82) -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:31 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 81). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 82). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 81) in 11 ms on localhost (1/2) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 82) in 11 ms on localhost (2/2) -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 41 (aggregate at GaussianMixture.scala:201) finished in 0.012 s -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 41 finished: aggregate at GaussianMixture.scala:201, took 0.024304 s -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=510776, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1163) called with curMem=512000, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 1163.0 B, free 730.1 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:65392 (size: 1163.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 82 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 42 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 42(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 42 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=513163, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=519411, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 83 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 42 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 42.0 with 2 tasks -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 42.0 (TID 83, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 42.0 (TID 84, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 42.0 (TID 83) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 42.0 (TID 84) -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 42.0 (TID 84). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 42.0 (TID 83). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 42.0 (TID 84) in 8 ms on localhost (1/2) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 42.0 (TID 83) in 10 ms on localhost (2/2) -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 42 (aggregate at GaussianMixture.scala:201) finished in 0.011 s -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 42.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 42 finished: aggregate at GaussianMixture.scala:201, took 0.021835 s -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=522664, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1163) called with curMem=523888, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 1163.0 B, free 730.1 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:65392 (size: 1163.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 84 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 43 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 43(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 43 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=525051, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3254) called with curMem=531299, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 85 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 43 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 85, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 86, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 85) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 86) -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 85). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 86). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 85) in 7 ms on localhost (1/2) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 86) in 7 ms on localhost (2/2) -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 43 (aggregate at GaussianMixture.scala:201) finished in 0.007 s -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 43 finished: aggregate at GaussianMixture.scala:201, took 0.015855 s -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=534553, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1164) called with curMem=535777, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 1164.0 B, free 730.1 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:65392 (size: 1164.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 86 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 44 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 44(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 44 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=536941, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=543189, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 87 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 44 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 87, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 88, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 87) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 88) -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 87). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 88). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 87) in 7 ms on localhost (1/2) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 88) in 7 ms on localhost (2/2) -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 44 (aggregate at GaussianMixture.scala:201) finished in 0.008 s -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 44 finished: aggregate at GaussianMixture.scala:201, took 0.015146 s -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=546442, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=547666, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 1168.0 B, free 730.1 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:65392 (size: 1168.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 88 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 45 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 45(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=548834, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3250) called with curMem=555082, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 89 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 89, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 90, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 89) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 90) -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 90). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 89). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 90) in 10 ms on localhost (1/2) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 89) in 11 ms on localhost (2/2) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 45 (aggregate at GaussianMixture.scala:201) finished in 0.013 s -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 45 finished: aggregate at GaussianMixture.scala:201, took 0.021562 s -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=558332, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1162) called with curMem=559556, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 1162.0 B, free 730.1 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:65392 (size: 1162.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 90 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 46 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 46(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=560718, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3252) called with curMem=566966, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 91 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 91, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 92, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 92) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 91) -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 91). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 92). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 91) in 12 ms on localhost (1/2) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 92) in 12 ms on localhost (2/2) -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 46 (aggregate at GaussianMixture.scala:201) finished in 0.013 s -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 46 finished: aggregate at GaussianMixture.scala:201, took 0.023158 s -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=570218, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1165) called with curMem=571442, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 1165.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:65392 (size: 1165.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 92 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 47 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 47(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 47 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=572607, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3258) called with curMem=578855, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 93 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 47 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 47.0 with 2 tasks -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 47.0 (TID 93, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 47.0 (TID 94, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 47.0 (TID 94) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 47.0 (TID 93) -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 47.0 (TID 94). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 47.0 (TID 93). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 47.0 (TID 94) in 8 ms on localhost (1/2) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 47.0 (TID 93) in 9 ms on localhost (2/2) -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 47 (aggregate at GaussianMixture.scala:201) finished in 0.009 s -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 47.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 47 finished: aggregate at GaussianMixture.scala:201, took 0.017974 s -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=582113, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1162) called with curMem=583337, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 1162.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:65392 (size: 1162.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 94 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 48 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 48(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 48 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=584499, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=590747, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 95 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 48 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 48.0 with 2 tasks -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 48.0 (TID 95, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 48.0 (TID 96, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 48.0 (TID 96) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 48.0 (TID 95) -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 48.0 (TID 95). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 48.0 (TID 96). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 48.0 (TID 95) in 12 ms on localhost (1/2) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 48.0 (TID 96) in 11 ms on localhost (2/2) -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 48 (aggregate at GaussianMixture.scala:201) finished in 0.012 s -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 48.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 48 finished: aggregate at GaussianMixture.scala:201, took 0.021731 s -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=594000, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1165) called with curMem=595224, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 1165.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:65392 (size: 1165.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 96 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 49 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 49(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 49 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=596389, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3252) called with curMem=602637, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 97 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 49 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 49.0 with 2 tasks -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 49.0 (TID 97, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 49.0 (TID 98, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 49.0 (TID 98) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 49.0 (TID 97) -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 49.0 (TID 97). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 49.0 (TID 98). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 49.0 (TID 97) in 14 ms on localhost (1/2) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 49.0 (TID 98) in 14 ms on localhost (2/2) -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 49 (aggregate at GaussianMixture.scala:201) finished in 0.015 s -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 49.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 49 finished: aggregate at GaussianMixture.scala:201, took 0.026303 s -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=605889, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=607113, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 1168.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:65392 (size: 1168.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 98 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 50 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 50(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 50 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=608281, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3258) called with curMem=614529, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 99 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 50 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 50.0 with 2 tasks -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 50.0 (TID 99, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 50.0 (TID 100, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 50.0 (TID 99) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 50.0 (TID 100) -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 50.0 (TID 99). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 50.0 (TID 100). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 50.0 (TID 99) in 8 ms on localhost (1/2) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 50.0 (TID 100) in 9 ms on localhost (2/2) -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 50 (aggregate at GaussianMixture.scala:201) finished in 0.010 s -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 50.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 50 finished: aggregate at GaussianMixture.scala:201, took 0.019328 s -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=617787, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1167) called with curMem=619011, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 1167.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:65392 (size: 1167.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 100 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 51 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 51(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 51 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=620178, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3254) called with curMem=626426, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 101 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 51 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 101, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 102, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 101) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 102) -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 102). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 101). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 102) in 8 ms on localhost (1/2) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 101) in 8 ms on localhost (2/2) -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 51 (aggregate at GaussianMixture.scala:201) finished in 0.009 s -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 51 finished: aggregate at GaussianMixture.scala:201, took 0.019754 s -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=629680, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1172) called with curMem=630904, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 1172.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:65392 (size: 1172.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 102 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 52 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 52(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 52 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=632076, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3261) called with curMem=638324, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 103 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 52 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 103, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 104, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 103) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 104) -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 103). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 104). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 103) in 9 ms on localhost (1/2) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 104) in 9 ms on localhost (2/2) -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 52 (aggregate at GaussianMixture.scala:201) finished in 0.010 s -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 52 finished: aggregate at GaussianMixture.scala:201, took 0.021321 s -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=641585, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1163) called with curMem=642809, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 1163.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:65392 (size: 1163.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 104 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 53 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 53(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=643972, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3250) called with curMem=650220, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 105 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 105, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 106, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 106) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 105) -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 106). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 105). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 106) in 11 ms on localhost (1/2) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 105) in 11 ms on localhost (2/2) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 53 (aggregate at GaussianMixture.scala:201) finished in 0.011 s -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 53 finished: aggregate at GaussianMixture.scala:201, took 0.022561 s -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=653470, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1167) called with curMem=654694, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 1167.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:65392 (size: 1167.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 106 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 54 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 54(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 54 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=655861, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3254) called with curMem=662109, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 107 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 54 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 54.0 with 2 tasks -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 54.0 (TID 107, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 54.0 (TID 108, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 54.0 (TID 107) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 54.0 (TID 108) -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 54.0 (TID 107). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 54.0 (TID 108). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 54.0 (TID 107) in 9 ms on localhost (1/2) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 54.0 (TID 108) in 9 ms on localhost (2/2) -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 54 (aggregate at GaussianMixture.scala:201) finished in 0.010 s -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 54.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 54 finished: aggregate at GaussianMixture.scala:201, took 0.020966 s -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=665363, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1169) called with curMem=666587, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 1169.0 B, free 729.9 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:65392 (size: 1169.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 108 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 55 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 55(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 55 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=667756, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 6.1 KB, free 729.9 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3254) called with curMem=674004, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 3.2 KB, free 729.9 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.2 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 109 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 55 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 55.0 with 2 tasks -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 55.0 (TID 109, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 55.0 (TID 110, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 55.0 (TID 109) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 55.0 (TID 110) -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 55.0 (TID 109). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 55.0 (TID 110). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 55.0 (TID 109) in 15 ms on localhost (1/2) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 55.0 (TID 110) in 16 ms on localhost (2/2) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 55.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 55 (aggregate at GaussianMixture.scala:201) finished in 0.021 s -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 55 finished: aggregate at GaussianMixture.scala:201, took 0.034534 s -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=667758, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 1224.0 B, free 729.9 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1170) called with curMem=668982, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 1170.0 B, free 729.9 MB) -2016-06-13 19:10:32 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:65392 (size: 1170.0 B, free: 730.2 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 110 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:65392 in memory (size: 1162.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 56 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 56(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 56 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=655873, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:65392 in memory (size: 1167.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3255) called with curMem=662121, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 111 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_107_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 56 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 56.0 with 2 tasks -2016-06-13 19:10:32 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 56.0 (TID 111, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 56.0 (TID 112, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 56.0 (TID 112) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 56.0 (TID 111) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_106_piece0 on localhost:65392 in memory (size: 1167.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_105_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 56.0 (TID 112). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 56.0 (TID 112) in 10 ms on localhost (1/2) -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 56.0 (TID 111). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_104_piece0 on localhost:65392 in memory (size: 1163.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 56.0 (TID 111) in 13 ms on localhost (2/2) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 56.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 56 (aggregate at GaussianMixture.scala:201) finished in 0.013 s -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 56 finished: aggregate at GaussianMixture.scala:201, took 0.022742 s -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=632089, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_103_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1164) called with curMem=632141, maxMem=766075207 -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_102_piece0 on localhost:65392 in memory (size: 1172.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 1164.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:65392 (size: 1164.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 112 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_101_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_100_piece0 on localhost:65392 in memory (size: 1167.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 57 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 57(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 57 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_99_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=610682, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3255) called with curMem=615762, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_98_piece0 on localhost:65392 in memory (size: 1168.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 113 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 57 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 57.0 with 2 tasks -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 57.0 (TID 113, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 57.0 (TID 114, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_97_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 57.0 (TID 113) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 57.0 (TID 114) -2016-06-13 19:10:32 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_96_piece0 on localhost:65392 in memory (size: 1165.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_95_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 57.0 (TID 114). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 57.0 (TID 113). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 57.0 (TID 114) in 9 ms on localhost (1/2) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 57.0 (TID 113) in 10 ms on localhost (2/2) -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 57 (aggregate at GaussianMixture.scala:201) finished in 0.011 s -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 57.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 57 finished: aggregate at GaussianMixture.scala:201, took 0.019736 s -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_94_piece0 on localhost:65392 in memory (size: 1162.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=594017, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 1224.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=591983, maxMem=766075207 -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_93_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 1168.0 B, free 730.0 MB) -2016-06-13 19:10:32 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:65392 (size: 1168.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 114 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_92_piece0 on localhost:65392 in memory (size: 1165.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_91_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 58 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 58(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 58 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=572628, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_115 stored as values in memory (estimated size 6.1 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_90_piece0 on localhost:65392 in memory (size: 1162.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3253) called with curMem=578876, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_115_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.0 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_115_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 115 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_89_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 58 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 2 tasks -2016-06-13 19:10:32 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 115, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 116, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 116) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_88_piece0 on localhost:65392 in memory (size: 1168.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 115) -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 116). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 115). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 116) in 11 ms on localhost (1/2) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 115) in 13 ms on localhost (2/2) -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 58 (aggregate at GaussianMixture.scala:201) finished in 0.013 s -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_87_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 58 finished: aggregate at GaussianMixture.scala:201, took 0.021038 s -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=559574, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_116 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:65392 in memory (size: 1164.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1169) called with curMem=559574, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_116_piece0 stored as bytes in memory (estimated size 1169.0 B, free 730.1 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_116_piece0 in memory on localhost:65392 (size: 1169.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 116 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:65392 in memory (size: 1163.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 59 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 59(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 59 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=538129, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_117 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:65392 in memory (size: 1163.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3251) called with curMem=543214, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_117_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_117_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 117 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 59 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 2 tasks -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 117, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 118, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 118) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 117) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:65392 in memory (size: 1163.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 117). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 118). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 117) in 9 ms on localhost (1/2) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 118) in 8 ms on localhost (2/2) -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 59 (aggregate at GaussianMixture.scala:201) finished in 0.009 s -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 59 finished: aggregate at GaussianMixture.scala:201, took 0.018978 s -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:65392 in memory (size: 1163.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=522689, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_118 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=514412, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_118_piece0 stored as bytes in memory (estimated size 1168.0 B, free 730.1 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_118_piece0 in memory on localhost:65392 (size: 1168.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 118 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:65392 in memory (size: 1167.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 60 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 60(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 60 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:65392 in memory (size: 1165.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=502528, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_119 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3257) called with curMem=501304, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_119_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:65392 in memory (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_119_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 119 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 60 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 60.0 with 2 tasks -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 60.0 (TID 119, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 60.0 (TID 120, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 60.0 (TID 119) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:65392 in memory (size: 1165.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 60.0 (TID 120) -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 60.0 (TID 119). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 60.0 (TID 119) in 9 ms on localhost (1/2) -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 60.0 (TID 120). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 60.0 (TID 120) in 10 ms on localhost (2/2) -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 60 (aggregate at GaussianMixture.scala:201) finished in 0.011 s -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 60.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 60 finished: aggregate at GaussianMixture.scala:201, took 0.020456 s -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=498919, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_120 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1166) called with curMem=500143, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_120_piece0 stored as bytes in memory (estimated size 1166.0 B, free 730.1 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_120_piece0 in memory on localhost:65392 (size: 1166.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 120 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 61 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 61(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 61 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=501309, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_121 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3260) called with curMem=507557, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_121_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_121_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 121 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 61 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 2 tasks -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 121, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 122, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 121) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 122) -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 122). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 121). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 122) in 6 ms on localhost (1/2) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 121) in 7 ms on localhost (2/2) -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 61 (aggregate at GaussianMixture.scala:201) finished in 0.007 s -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 61 finished: aggregate at GaussianMixture.scala:201, took 0.015252 s -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=510817, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_122 stored as values in memory (estimated size 1224.0 B, free 730.1 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(1169) called with curMem=512041, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_122_piece0 stored as bytes in memory (estimated size 1169.0 B, free 730.1 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_122_piece0 in memory on localhost:65392 (size: 1169.0 B, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 122 from broadcast at GaussianMixture.scala:198 -2016-06-13 19:10:32 INFO SparkContext:59 - Starting job: aggregate at GaussianMixture.scala:201 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Got job 62 (aggregate at GaussianMixture.scala:201) with 2 output partitions -2016-06-13 19:10:32 INFO DAGScheduler:59 - Final stage: ResultStage 62(aggregate at GaussianMixture.scala:201) -2016-06-13 19:10:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting ResultStage 62 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168), which has no missing parents -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(6248) called with curMem=513210, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_123 stored as values in memory (estimated size 6.1 KB, free 730.1 MB) -2016-06-13 19:10:32 INFO MemoryStore:59 - ensureFreeSpace(3251) called with curMem=519458, maxMem=766075207 -2016-06-13 19:10:32 INFO MemoryStore:59 - Block broadcast_123_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:10:32 INFO BlockManagerInfo:59 - Added broadcast_123_piece0 in memory on localhost:65392 (size: 3.2 KB, free: 730.3 MB) -2016-06-13 19:10:32 INFO SparkContext:59 - Created broadcast 123 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 62 (MapPartitionsRDD[3] at map at GaussianMixture.scala:168) -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Adding task set 62.0 with 2 tasks -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 62.0 (TID 123, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 62.0 (TID 124, localhost, PROCESS_LOCAL, 2190 bytes) -2016-06-13 19:10:32 INFO Executor:59 - Running task 1.0 in stage 62.0 (TID 124) -2016-06-13 19:10:32 INFO Executor:59 - Running task 0.0 in stage 62.0 (TID 123) -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 19:10:32 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 19:10:32 INFO Executor:59 - Finished task 1.0 in stage 62.0 (TID 124). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO Executor:59 - Finished task 0.0 in stage 62.0 (TID 123). 2944 bytes result sent to driver -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 62.0 (TID 124) in 6 ms on localhost (1/2) -2016-06-13 19:10:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 62.0 (TID 123) in 6 ms on localhost (2/2) -2016-06-13 19:10:32 INFO DAGScheduler:59 - ResultStage 62 (aggregate at GaussianMixture.scala:201) finished in 0.006 s -2016-06-13 19:10:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 62.0, whose tasks have all completed, from pool -2016-06-13 19:10:32 INFO DAGScheduler:59 - Job 62 finished: aggregate at GaussianMixture.scala:201, took 0.014458 s -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 19:10:32 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 19:10:32 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 19:10:32 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 19:10:32 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 19:10:33 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 19:10:33 INFO BlockManager:59 - BlockManager stopped -2016-06-13 19:10:33 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 19:10:33 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 19:10:33 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 19:10:33 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 19:10:33 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 19:10:33 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 19:10:33 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-135bf678-7f09-4b71-a629-ecc687625ef6 -2016-06-13 19:10:58 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 19:10:59 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 19:10:59 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 19:10:59 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 19:10:59 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 19:11:00 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 19:11:00 INFO Remoting:74 - Starting remoting -2016-06-13 19:11:00 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:65412] -2016-06-13 19:11:00 INFO Utils:59 - Successfully started service 'sparkDriver' on port 65412. -2016-06-13 19:11:00 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 19:11:00 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 19:11:00 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-d1cc5b2c-b235-4a63-8b6a-e86b355e2076 -2016-06-13 19:11:00 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 19:11:00 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-f759b974-a642-4607-ad6b-808498eb7738\httpd-185fd719-87ed-439c-9629-5e2fa7d162b4 -2016-06-13 19:11:00 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 19:11:00 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:11:00 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:65413 -2016-06-13 19:11:00 INFO Utils:59 - Successfully started service 'HTTP file server' on port 65413. -2016-06-13 19:11:00 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 19:11:01 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:11:01 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 19:11:01 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 19:11:01 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 19:11:01 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 19:11:01 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 19:11:01 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 65433. -2016-06-13 19:11:01 INFO NettyBlockTransferService:59 - Server created on 65433 -2016-06-13 19:11:01 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 19:11:01 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:65433 with 730.6 MB RAM, BlockManagerId(driver, localhost, 65433) -2016-06-13 19:11:01 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 19:11:02 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:325 -2016-06-13 19:11:02 INFO DAGScheduler:59 - Registering RDD 11 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:02 INFO DAGScheduler:59 - Registering RDD 5 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:11:02 INFO DAGScheduler:59 - Got job 0 (sum at PowerIterationClustering.scala:325) with 2 output partitions -2016-06-13 19:11:02 INFO DAGScheduler:59 - Final stage: ResultStage 2(sum at PowerIterationClustering.scala:325) -2016-06-13 19:11:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0, ShuffleMapStage 1) -2016-06-13 19:11:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0, ShuffleMapStage 1) -2016-06-13 19:11:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[11] at mapPartitions at GraphImpl.scala:235), which has no missing parents -2016-06-13 19:11:02 INFO MemoryStore:59 - ensureFreeSpace(3432) called with curMem=0, maxMem=766075207 -2016-06-13 19:11:02 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 3.4 KB, free 730.6 MB) -2016-06-13 19:11:02 INFO MemoryStore:59 - ensureFreeSpace(1979) called with curMem=3432, maxMem=766075207 -2016-06-13 19:11:02 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1979.0 B, free 730.6 MB) -2016-06-13 19:11:02 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:65433 (size: 1979.0 B, free: 730.6 MB) -2016-06-13 19:11:02 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[11] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:02 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 19:11:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[5] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:11:02 INFO MemoryStore:59 - ensureFreeSpace(3016) called with curMem=5411, maxMem=766075207 -2016-06-13 19:11:02 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-06-13 19:11:02 INFO MemoryStore:59 - ensureFreeSpace(1776) called with curMem=8427, maxMem=766075207 -2016-06-13 19:11:02 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1776.0 B, free 730.6 MB) -2016-06-13 19:11:02 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:65433 (size: 1776.0 B, free: 730.6 MB) -2016-06-13 19:11:02 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[5] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:11:02 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 19:11:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2488 bytes) -2016-06-13 19:11:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2563 bytes) -2016-06-13 19:11:02 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 19:11:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2488 bytes) -2016-06-13 19:11:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2563 bytes) -2016-06-13 19:11:02 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 19:11:02 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 19:11:02 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 19:11:02 INFO CacheManager:59 - Another thread is loading rdd_2_1, waiting for it to finish... -2016-06-13 19:11:02 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-06-13 19:11:02 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-06-13 19:11:02 INFO CacheManager:59 - Another thread is loading rdd_2_0, waiting for it to finish... -2016-06-13 19:11:02 INFO MemoryStore:59 - ensureFreeSpace(2952) called with curMem=10203, maxMem=766075207 -2016-06-13 19:11:02 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-06-13 19:11:02 INFO MemoryStore:59 - ensureFreeSpace(2856) called with curMem=13155, maxMem=766075207 -2016-06-13 19:11:02 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 2.8 KB, free 730.6 MB) -2016-06-13 19:11:02 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:65433 (size: 2.9 KB, free: 730.6 MB) -2016-06-13 19:11:02 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:65433 (size: 2.8 KB, free: 730.6 MB) -2016-06-13 19:11:02 INFO CacheManager:59 - Finished waiting for rdd_2_1 -2016-06-13 19:11:02 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-06-13 19:11:02 INFO CacheManager:59 - Finished waiting for rdd_2_0 -2016-06-13 19:11:02 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-06-13 19:11:03 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1159 bytes result sent to driver -2016-06-13 19:11:03 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1159 bytes result sent to driver -2016-06-13 19:11:03 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1739 bytes result sent to driver -2016-06-13 19:11:03 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1739 bytes result sent to driver -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 273 ms on localhost (1/2) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 271 ms on localhost (2/2) -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 313 ms on localhost (1/2) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 286 ms on localhost (2/2) -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 19:11:03 INFO DAGScheduler:59 - ShuffleMapStage 1 (mapPartitions at VertexRDD.scala:358) finished in 0.283 s -2016-06-13 19:11:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:03 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 0) -2016-06-13 19:11:03 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-06-13 19:11:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:03 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List(ShuffleMapStage 0) -2016-06-13 19:11:03 INFO DAGScheduler:59 - ShuffleMapStage 0 (mapPartitions at GraphImpl.scala:235) finished in 0.357 s -2016-06-13 19:11:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:03 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-06-13 19:11:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:03 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List() -2016-06-13 19:11:03 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[23] at map at PowerIterationClustering.scala:325), which is now runnable -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(5136) called with curMem=16011, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 5.0 KB, free 730.6 MB) -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(2654) called with curMem=21147, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:65433 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:11:03 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[23] at map at PowerIterationClustering.scala:325) -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 19:11:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:11:03 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 19:11:03 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 19:11:03 INFO CacheManager:59 - Partition rdd_21_1 not found, computing it -2016-06-13 19:11:03 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-06-13 19:11:03 INFO CacheManager:59 - Partition rdd_21_0 not found, computing it -2016-06-13 19:11:03 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-06-13 19:11:03 INFO CacheManager:59 - Partition rdd_8_1 not found, computing it -2016-06-13 19:11:03 INFO CacheManager:59 - Partition rdd_8_0 not found, computing it -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 3 ms -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(1792) called with curMem=23801, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block rdd_8_1 stored as values in memory (estimated size 1792.0 B, free 730.6 MB) -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(1776) called with curMem=25593, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block rdd_8_0 stored as values in memory (estimated size 1776.0 B, free 730.6 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added rdd_8_1 in memory on localhost:65433 (size: 1792.0 B, free: 730.6 MB) -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added rdd_8_0 in memory on localhost:65433 (size: 1776.0 B, free: 730.6 MB) -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(1824) called with curMem=27369, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 1824.0 B, free 730.6 MB) -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(1840) called with curMem=29193, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 1840.0 B, free 730.6 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:65433 (size: 1824.0 B, free: 730.6 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:65433 (size: 1840.0 B, free: 730.6 MB) -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(368) called with curMem=31033, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block rdd_21_1 stored as values in memory (estimated size 368.0 B, free 730.6 MB) -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(368) called with curMem=31401, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block rdd_21_0 stored as values in memory (estimated size 368.0 B, free 730.6 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added rdd_21_1 in memory on localhost:65433 (size: 368.0 B, free: 730.6 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added rdd_21_0 in memory on localhost:65433 (size: 368.0 B, free: 730.6 MB) -2016-06-13 19:11:03 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1895 bytes result sent to driver -2016-06-13 19:11:03 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1895 bytes result sent to driver -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 144 ms on localhost (1/2) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 140 ms on localhost (2/2) -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 19:11:03 INFO DAGScheduler:59 - ResultStage 2 (sum at PowerIterationClustering.scala:325) finished in 0.147 s -2016-06-13 19:11:03 INFO DAGScheduler:59 - Job 0 finished: sum at PowerIterationClustering.scala:325, took 0.784849 s -2016-06-13 19:11:03 INFO SparkContext:59 - Starting job: reduce at VertexRDDImpl.scala:90 -2016-06-13 19:11:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:11:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:11:03 INFO DAGScheduler:59 - Got job 1 (reduce at VertexRDDImpl.scala:90) with 2 output partitions -2016-06-13 19:11:03 INFO DAGScheduler:59 - Final stage: ResultStage 5(reduce at VertexRDDImpl.scala:90) -2016-06-13 19:11:03 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3, ShuffleMapStage 4) -2016-06-13 19:11:03 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:03 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[27] at map at VertexRDDImpl.scala:90), which has no missing parents -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(5288) called with curMem=31769, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.2 KB, free 730.6 MB) -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(2670) called with curMem=37057, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:65433 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:11:03 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[27] at map at VertexRDDImpl.scala:90) -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 19:11:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 6, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 7, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:11:03 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 7) -2016-06-13 19:11:03 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 6) -2016-06-13 19:11:03 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:11:03 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:11:03 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 6). 2161 bytes result sent to driver -2016-06-13 19:11:03 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 7). 2161 bytes result sent to driver -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 6) in 35 ms on localhost (1/2) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 7) in 33 ms on localhost (2/2) -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 19:11:03 INFO DAGScheduler:59 - ResultStage 5 (reduce at VertexRDDImpl.scala:90) finished in 0.037 s -2016-06-13 19:11:03 INFO DAGScheduler:59 - Job 1 finished: reduce at VertexRDDImpl.scala:90, took 0.071725 s -2016-06-13 19:11:03 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:11:03 INFO DAGScheduler:59 - Registering RDD 15 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:03 INFO DAGScheduler:59 - Registering RDD 28 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:03 INFO DAGScheduler:59 - Registering RDD 32 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:03 INFO DAGScheduler:59 - Got job 2 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:11:03 INFO DAGScheduler:59 - Final stage: ResultStage 13(sum at PowerIterationClustering.scala:369) -2016-06-13 19:11:03 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12, ShuffleMapStage 9, ShuffleMapStage 10) -2016-06-13 19:11:03 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-06-13 19:11:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 8 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[15] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=39727, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.1 KB, free 730.5 MB) -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(2367) called with curMem=43975, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:65433 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 19:11:03 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 8 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[15] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 19:11:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 11 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[28] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:11:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 8, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 9, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:11:03 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 8) -2016-06-13 19:11:03 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 9) -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(4992) called with curMem=46342, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 4.9 KB, free 730.5 MB) -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(2604) called with curMem=51334, maxMem=766075207 -2016-06-13 19:11:03 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-06-13 19:11:03 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 19:11:03 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:65433 (size: 2.5 KB, free: 730.6 MB) -2016-06-13 19:11:03 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 11 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[28] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 19:11:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 10, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 11, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:11:03 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 10) -2016-06-13 19:11:03 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 11) -2016-06-13 19:11:03 INFO CacheManager:59 - Partition rdd_25_0 not found, computing it -2016-06-13 19:11:03 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:11:03 INFO CacheManager:59 - Partition rdd_25_1 not found, computing it -2016-06-13 19:11:03 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=53938, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block rdd_25_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=55370, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block rdd_25_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added rdd_25_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.6 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added rdd_25_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.6 MB) -2016-06-13 19:11:03 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 9). 2254 bytes result sent to driver -2016-06-13 19:11:03 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 8). 2254 bytes result sent to driver -2016-06-13 19:11:03 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 11). 2834 bytes result sent to driver -2016-06-13 19:11:03 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 10). 2834 bytes result sent to driver -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 9) in 75 ms on localhost (1/2) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 8) in 79 ms on localhost (2/2) -2016-06-13 19:11:03 INFO DAGScheduler:59 - ShuffleMapStage 8 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.080 s -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 19:11:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:03 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 11) -2016-06-13 19:11:03 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 12, ResultStage 13) -2016-06-13 19:11:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 10) in 39 ms on localhost (1/2) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 11) in 39 ms on localhost (2/2) -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 19:11:03 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 12: List(ShuffleMapStage 11) -2016-06-13 19:11:03 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List(ShuffleMapStage 12) -2016-06-13 19:11:03 INFO DAGScheduler:59 - ShuffleMapStage 11 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.049 s -2016-06-13 19:11:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:03 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 12, ResultStage 13) -2016-06-13 19:11:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:03 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 12: List() -2016-06-13 19:11:03 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List(ShuffleMapStage 12) -2016-06-13 19:11:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[32] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=56802, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 4.7 KB, free 730.5 MB) -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(2539) called with curMem=61650, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:65433 (size: 2.5 KB, free: 730.6 MB) -2016-06-13 19:11:03 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[32] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 19:11:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 12, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 13, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:11:03 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 13) -2016-06-13 19:11:03 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 12) -2016-06-13 19:11:03 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-06-13 19:11:03 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:03 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 12). 2470 bytes result sent to driver -2016-06-13 19:11:03 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 13). 2470 bytes result sent to driver -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 12) in 42 ms on localhost (1/2) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 13) in 42 ms on localhost (2/2) -2016-06-13 19:11:03 INFO DAGScheduler:59 - ShuffleMapStage 12 (mapPartitions at GraphImpl.scala:235) finished in 0.044 s -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 19:11:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:03 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-06-13 19:11:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:03 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-06-13 19:11:03 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[37] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(6024) called with curMem=64189, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.9 KB, free 730.5 MB) -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(2867) called with curMem=70213, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.5 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:65433 (size: 2.8 KB, free: 730.6 MB) -2016-06-13 19:11:03 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[37] at map at PowerIterationClustering.scala:369) -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 19:11:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 14, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 15, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:11:03 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 14) -2016-06-13 19:11:03 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 15) -2016-06-13 19:11:03 INFO CacheManager:59 - Partition rdd_34_1 not found, computing it -2016-06-13 19:11:03 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 19:11:03 INFO CacheManager:59 - Partition rdd_34_0 not found, computing it -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:03 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=73080, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block rdd_34_1 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=74560, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block rdd_34_0 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added rdd_34_1 in memory on localhost:65433 (size: 1480.0 B, free: 730.6 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added rdd_34_0 in memory on localhost:65433 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:11:03 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 14). 2880 bytes result sent to driver -2016-06-13 19:11:03 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 15). 2880 bytes result sent to driver -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 14) in 20 ms on localhost (1/2) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 15) in 20 ms on localhost (2/2) -2016-06-13 19:11:03 INFO DAGScheduler:59 - ResultStage 13 (sum at PowerIterationClustering.scala:369) finished in 0.022 s -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 19:11:03 INFO DAGScheduler:59 - Job 2 finished: sum at PowerIterationClustering.scala:369, took 0.223132 s -2016-06-13 19:11:03 INFO PowerIterationClustering:59 - Iteration 0: norm(v) = 36.22334869431644. -2016-06-13 19:11:03 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:11:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:11:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:11:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:11:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:11:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:11:03 INFO DAGScheduler:59 - Got job 3 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:11:03 INFO DAGScheduler:59 - Final stage: ResultStage 21(sum at PowerIterationClustering.scala:375) -2016-06-13 19:11:03 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 20, ShuffleMapStage 17, ShuffleMapStage 18) -2016-06-13 19:11:03 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:03 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[48] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(6760) called with curMem=76040, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 6.6 KB, free 730.5 MB) -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(3106) called with curMem=82800, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.5 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:65433 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:11:03 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[48] at values at PowerIterationClustering.scala:375) -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 19:11:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 16, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 17, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:11:03 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 16) -2016-06-13 19:11:03 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 17) -2016-06-13 19:11:03 INFO CacheManager:59 - Partition rdd_40_0 not found, computing it -2016-06-13 19:11:03 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 19:11:03 INFO BlockManager:59 - Found block rdd_34_0 locally -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=85906, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block rdd_40_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added rdd_40_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:03 INFO CacheManager:59 - Partition rdd_40_1 not found, computing it -2016-06-13 19:11:03 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 19:11:03 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 16). 2664 bytes result sent to driver -2016-06-13 19:11:03 INFO BlockManager:59 - Found block rdd_34_1 locally -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=87338, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block rdd_40_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 16) in 19 ms on localhost (1/2) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added rdd_40_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:03 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 17). 2664 bytes result sent to driver -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 17) in 22 ms on localhost (2/2) -2016-06-13 19:11:03 INFO DAGScheduler:59 - ResultStage 21 (sum at PowerIterationClustering.scala:375) finished in 0.023 s -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 19:11:03 INFO DAGScheduler:59 - Job 3 finished: sum at PowerIterationClustering.scala:375, took 0.049736 s -2016-06-13 19:11:03 INFO PowerIterationClustering:59 - Iteration 0: delta = 1.2794887333174252. -2016-06-13 19:11:03 INFO PowerIterationClustering:59 - Iteration 0: diff(delta) = 1.7976931348623157E308. -2016-06-13 19:11:03 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:11:03 INFO DAGScheduler:59 - Registering RDD 51 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:03 INFO DAGScheduler:59 - Registering RDD 55 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:03 INFO DAGScheduler:59 - Got job 4 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:11:03 INFO DAGScheduler:59 - Final stage: ResultStage 31(sum at PowerIterationClustering.scala:369) -2016-06-13 19:11:03 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30, ShuffleMapStage 28, ShuffleMapStage 25, ShuffleMapStage 26) -2016-06-13 19:11:03 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 30) -2016-06-13 19:11:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 29 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[51] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(5984) called with curMem=88770, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.8 KB, free 730.5 MB) -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(2861) called with curMem=94754, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.5 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:65433 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:11:03 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 29 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[51] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-06-13 19:11:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 18, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 19, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:11:03 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 18) -2016-06-13 19:11:03 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 19) -2016-06-13 19:11:03 INFO CacheManager:59 - Partition rdd_49_1 not found, computing it -2016-06-13 19:11:03 INFO BlockManager:59 - Found block rdd_34_1 locally -2016-06-13 19:11:03 INFO CacheManager:59 - Partition rdd_49_0 not found, computing it -2016-06-13 19:11:03 INFO BlockManager:59 - Found block rdd_34_0 locally -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=97615, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block rdd_49_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=99047, maxMem=766075207 -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added rdd_49_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:03 INFO MemoryStore:59 - Block rdd_49_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added rdd_49_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:03 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 18). 2834 bytes result sent to driver -2016-06-13 19:11:03 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 19). 2834 bytes result sent to driver -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 18) in 17 ms on localhost (1/2) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 19) in 18 ms on localhost (2/2) -2016-06-13 19:11:03 INFO DAGScheduler:59 - ShuffleMapStage 29 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.019 s -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 19:11:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:03 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 30, ResultStage 31) -2016-06-13 19:11:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:03 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List() -2016-06-13 19:11:03 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-06-13 19:11:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 30 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[55] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=100479, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.7 KB, free 730.5 MB) -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=105327, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:65433 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:11:03 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 30 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[55] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-06-13 19:11:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 20, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 21, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:11:03 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 20) -2016-06-13 19:11:03 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 21) -2016-06-13 19:11:03 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-06-13 19:11:03 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:03 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 20). 2470 bytes result sent to driver -2016-06-13 19:11:03 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 21). 2470 bytes result sent to driver -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 20) in 32 ms on localhost (1/2) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 21) in 32 ms on localhost (2/2) -2016-06-13 19:11:03 INFO DAGScheduler:59 - ShuffleMapStage 30 (mapPartitions at GraphImpl.scala:235) finished in 0.033 s -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 19:11:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:03 INFO DAGScheduler:59 - waiting: Set(ResultStage 31) -2016-06-13 19:11:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:03 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List() -2016-06-13 19:11:03 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[60] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(6920) called with curMem=107869, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 6.8 KB, free 730.5 MB) -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(3114) called with curMem=114789, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.5 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:65433 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:11:03 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[60] at map at PowerIterationClustering.scala:369) -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 19:11:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 22, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 23, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:11:03 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 22) -2016-06-13 19:11:03 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 23) -2016-06-13 19:11:03 INFO CacheManager:59 - Partition rdd_57_0 not found, computing it -2016-06-13 19:11:03 INFO BlockManager:59 - Found block rdd_49_0 locally -2016-06-13 19:11:03 INFO CacheManager:59 - Partition rdd_57_1 not found, computing it -2016-06-13 19:11:03 INFO BlockManager:59 - Found block rdd_49_1 locally -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=117903, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block rdd_57_1 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:11:03 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=119383, maxMem=766075207 -2016-06-13 19:11:03 INFO MemoryStore:59 - Block rdd_57_0 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added rdd_57_1 in memory on localhost:65433 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:11:03 INFO BlockManagerInfo:59 - Added rdd_57_0 in memory on localhost:65433 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:11:03 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 23). 2880 bytes result sent to driver -2016-06-13 19:11:03 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 22). 2880 bytes result sent to driver -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 23) in 24 ms on localhost (1/2) -2016-06-13 19:11:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 22) in 26 ms on localhost (2/2) -2016-06-13 19:11:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 19:11:03 INFO DAGScheduler:59 - ResultStage 31 (sum at PowerIterationClustering.scala:369) finished in 0.026 s -2016-06-13 19:11:03 INFO DAGScheduler:59 - Job 4 finished: sum at PowerIterationClustering.scala:369, took 0.139209 s -2016-06-13 19:11:03 INFO PowerIterationClustering:59 - Iteration 1: norm(v) = 36.22334869431644. -2016-06-13 19:11:04 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:11:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:11:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:11:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:11:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:11:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:11:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 151 bytes -2016-06-13 19:11:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:11:04 INFO DAGScheduler:59 - Got job 5 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:11:04 INFO DAGScheduler:59 - Final stage: ResultStage 41(sum at PowerIterationClustering.scala:375) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 38, ShuffleMapStage 35, ShuffleMapStage 36, ShuffleMapStage 40) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[71] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(7352) called with curMem=120863, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 7.2 KB, free 730.5 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(3294) called with curMem=128215, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:65433 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[71] at values at PowerIterationClustering.scala:375) -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 24, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 25, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:11:04 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 25) -2016-06-13 19:11:04 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 24) -2016-06-13 19:11:04 INFO CacheManager:59 - Partition rdd_63_1 not found, computing it -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_49_1 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_57_1 locally -2016-06-13 19:11:04 INFO CacheManager:59 - Partition rdd_63_0 not found, computing it -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_49_0 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_57_0 locally -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=131509, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block rdd_63_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=132941, maxMem=766075207 -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added rdd_63_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - Block rdd_63_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added rdd_63_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:04 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 25). 2664 bytes result sent to driver -2016-06-13 19:11:04 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 24). 2664 bytes result sent to driver -2016-06-13 19:11:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 25) in 19 ms on localhost (1/2) -2016-06-13 19:11:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 24) in 22 ms on localhost (2/2) -2016-06-13 19:11:04 INFO DAGScheduler:59 - ResultStage 41 (sum at PowerIterationClustering.scala:375) finished in 0.023 s -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-06-13 19:11:04 INFO DAGScheduler:59 - Job 5 finished: sum at PowerIterationClustering.scala:375, took 0.053454 s -2016-06-13 19:11:04 INFO PowerIterationClustering:59 - Iteration 1: delta = 0.0. -2016-06-13 19:11:04 INFO PowerIterationClustering:59 - Iteration 1: diff(delta) = 1.2794887333174252. -2016-06-13 19:11:04 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:11:04 INFO DAGScheduler:59 - Registering RDD 74 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Registering RDD 78 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Got job 6 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:11:04 INFO DAGScheduler:59 - Final stage: ResultStage 53(sum at PowerIterationClustering.scala:369) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 48, ShuffleMapStage 45, ShuffleMapStage 52, ShuffleMapStage 46, ShuffleMapStage 50) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 52) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 51 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[74] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(6568) called with curMem=134373, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 6.4 KB, free 730.5 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(3022) called with curMem=140941, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:65433 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 51 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[74] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 26, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 27, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:11:04 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 27) -2016-06-13 19:11:04 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 26) -2016-06-13 19:11:04 INFO CacheManager:59 - Partition rdd_72_1 not found, computing it -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_57_1 locally -2016-06-13 19:11:04 INFO CacheManager:59 - Partition rdd_72_0 not found, computing it -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_57_0 locally -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=143963, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block rdd_72_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added rdd_72_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=145395, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block rdd_72_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added rdd_72_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:04 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 27). 2834 bytes result sent to driver -2016-06-13 19:11:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 27) in 18 ms on localhost (1/2) -2016-06-13 19:11:04 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 26). 2834 bytes result sent to driver -2016-06-13 19:11:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 26) in 23 ms on localhost (2/2) -2016-06-13 19:11:04 INFO DAGScheduler:59 - ShuffleMapStage 51 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.023 s -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-06-13 19:11:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:04 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 52, ResultStage 53) -2016-06-13 19:11:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:04 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 52: List() -2016-06-13 19:11:04 INFO DAGScheduler:59 - Missing parents for ResultStage 53: List(ShuffleMapStage 52) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 52 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[78] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=146827, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=151675, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:65433 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 52 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[78] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 28, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 29, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:11:04 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 29) -2016-06-13 19:11:04 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 28) -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-06-13 19:11:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:04 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:04 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:04 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 28). 2470 bytes result sent to driver -2016-06-13 19:11:04 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 29). 2470 bytes result sent to driver -2016-06-13 19:11:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 28) in 39 ms on localhost (1/2) -2016-06-13 19:11:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 29) in 37 ms on localhost (2/2) -2016-06-13 19:11:04 INFO DAGScheduler:59 - ShuffleMapStage 52 (mapPartitions at GraphImpl.scala:235) finished in 0.040 s -2016-06-13 19:11:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 53) -2016-06-13 19:11:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-06-13 19:11:04 INFO DAGScheduler:59 - Missing parents for ResultStage 53: List() -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[83] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(7504) called with curMem=154217, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 7.3 KB, free 730.4 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(3266) called with curMem=161721, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:65433 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[83] at map at PowerIterationClustering.scala:369) -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 30, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 31, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:11:04 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 30) -2016-06-13 19:11:04 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 31) -2016-06-13 19:11:04 INFO CacheManager:59 - Partition rdd_80_0 not found, computing it -2016-06-13 19:11:04 INFO CacheManager:59 - Partition rdd_80_1 not found, computing it -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_72_0 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_72_1 locally -2016-06-13 19:11:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=164987, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block rdd_80_0 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=166467, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block rdd_80_1 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added rdd_80_0 in memory on localhost:65433 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added rdd_80_1 in memory on localhost:65433 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:11:04 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 30). 2880 bytes result sent to driver -2016-06-13 19:11:04 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 31). 2880 bytes result sent to driver -2016-06-13 19:11:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 30) in 19 ms on localhost (1/2) -2016-06-13 19:11:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 31) in 19 ms on localhost (2/2) -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-06-13 19:11:04 INFO DAGScheduler:59 - ResultStage 53 (sum at PowerIterationClustering.scala:369) finished in 0.021 s -2016-06-13 19:11:04 INFO DAGScheduler:59 - Job 6 finished: sum at PowerIterationClustering.scala:369, took 0.159571 s -2016-06-13 19:11:04 INFO PowerIterationClustering:59 - Iteration 2: norm(v) = 36.22334869431644. -2016-06-13 19:11:04 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:11:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:11:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:11:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:11:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:11:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:11:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 151 bytes -2016-06-13 19:11:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:11:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 151 bytes -2016-06-13 19:11:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:11:04 INFO DAGScheduler:59 - Got job 7 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:11:04 INFO DAGScheduler:59 - Final stage: ResultStage 65(sum at PowerIterationClustering.scala:375) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 60, ShuffleMapStage 64, ShuffleMapStage 57, ShuffleMapStage 58, ShuffleMapStage 62) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[94] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(7936) called with curMem=167947, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 7.8 KB, free 730.4 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(3392) called with curMem=175883, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:65433 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 65 (MapPartitionsRDD[94] at values at PowerIterationClustering.scala:375) -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 2 tasks -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 32, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 33, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:11:04 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 32) -2016-06-13 19:11:04 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 33) -2016-06-13 19:11:04 INFO CacheManager:59 - Partition rdd_86_0 not found, computing it -2016-06-13 19:11:04 INFO CacheManager:59 - Partition rdd_86_1 not found, computing it -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_72_0 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_72_1 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_80_0 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_80_1 locally -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=179275, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block rdd_86_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added rdd_86_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=180707, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block rdd_86_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added rdd_86_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:04 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 33). 2664 bytes result sent to driver -2016-06-13 19:11:04 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 32). 2664 bytes result sent to driver -2016-06-13 19:11:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 33) in 19 ms on localhost (1/2) -2016-06-13 19:11:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 32) in 21 ms on localhost (2/2) -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-06-13 19:11:04 INFO DAGScheduler:59 - ResultStage 65 (sum at PowerIterationClustering.scala:375) finished in 0.021 s -2016-06-13 19:11:04 INFO DAGScheduler:59 - Job 7 finished: sum at PowerIterationClustering.scala:375, took 0.055340 s -2016-06-13 19:11:04 INFO PowerIterationClustering:59 - Iteration 2: delta = 0.0. -2016-06-13 19:11:04 INFO PowerIterationClustering:59 - Iteration 2: diff(delta) = 0.0. -2016-06-13 19:11:04 WARN KMeans:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:11:04 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:11:04 INFO DAGScheduler:59 - Got job 8 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:11:04 INFO DAGScheduler:59 - Final stage: ResultStage 77(takeSample at KMeans.scala:376) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 70, ShuffleMapStage 74, ShuffleMapStage 72, ShuffleMapStage 69, ShuffleMapStage 76) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[102] at map at KMeans.scala:210), which has no missing parents -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(8168) called with curMem=182139, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 8.0 KB, free 730.4 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(3463) called with curMem=190307, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:65433 (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 77 (MapPartitionsRDD[102] at map at KMeans.scala:210) -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 2 tasks -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 34, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 35, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:11:04 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 35) -2016-06-13 19:11:04 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 34) -2016-06-13 19:11:04 INFO CacheManager:59 - Partition rdd_97_1 not found, computing it -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_80_1 locally -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=193770, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block rdd_97_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added rdd_97_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:04 INFO CacheManager:59 - Partition rdd_97_0 not found, computing it -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_80_0 locally -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=195202, maxMem=766075207 -2016-06-13 19:11:04 INFO CacheManager:59 - Partition rdd_100_1 not found, computing it -2016-06-13 19:11:04 INFO MemoryStore:59 - Block rdd_97_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added rdd_97_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=196634, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block rdd_100_1 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added rdd_100_1 in memory on localhost:65433 (size: 240.0 B, free: 730.5 MB) -2016-06-13 19:11:04 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 35). 2717 bytes result sent to driver -2016-06-13 19:11:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 35) in 95 ms on localhost (1/2) -2016-06-13 19:11:04 INFO CacheManager:59 - Partition rdd_100_0 not found, computing it -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:65433 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=188029, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block rdd_100_0 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added rdd_100_0 in memory on localhost:65433 (size: 240.0 B, free: 730.5 MB) -2016-06-13 19:11:04 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 34). 2717 bytes result sent to driver -2016-06-13 19:11:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 34) in 109 ms on localhost (2/2) -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-06-13 19:11:04 INFO DAGScheduler:59 - ResultStage 77 (takeSample at KMeans.scala:376) finished in 0.110 s -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:65433 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Job 8 finished: takeSample at KMeans.scala:376, took 0.143458 s -2016-06-13 19:11:04 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 19:11:04 INFO BlockManager:59 - Removing RDD 40 -2016-06-13 19:11:04 INFO ContextCleaner:59 - Cleaned RDD 40 -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:65433 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:65433 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:65433 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 19:11:04 INFO BlockManager:59 - Removing RDD 86 -2016-06-13 19:11:04 INFO ContextCleaner:59 - Cleaned RDD 86 -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:65433 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:11:04 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:65433 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Got job 9 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:11:04 INFO DAGScheduler:59 - Final stage: ResultStage 89(takeSample at KMeans.scala:376) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 84, ShuffleMapStage 81, ShuffleMapStage 88, ShuffleMapStage 82, ShuffleMapStage 86) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting ResultStage 89 (PartitionwiseSampledRDD[104] at takeSample at KMeans.scala:376), which has no missing parents -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(25560) called with curMem=126909, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 25.0 KB, free 730.4 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(10152) called with curMem=152469, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 9.9 KB, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:65433 (size: 9.9 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 89 (PartitionwiseSampledRDD[104] at takeSample at KMeans.scala:376) -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Adding task set 89.0 with 2 tasks -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 89.0 (TID 36, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 89.0 (TID 37, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:11:04 INFO Executor:59 - Running task 1.0 in stage 89.0 (TID 37) -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:11:04 INFO Executor:59 - Finished task 1.0 in stage 89.0 (TID 37). 2575 bytes result sent to driver -2016-06-13 19:11:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 89.0 (TID 37) in 12 ms on localhost (1/2) -2016-06-13 19:11:04 INFO Executor:59 - Running task 0.0 in stage 89.0 (TID 36) -2016-06-13 19:11:04 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:11:04 INFO Executor:59 - Finished task 0.0 in stage 89.0 (TID 36). 2633 bytes result sent to driver -2016-06-13 19:11:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 89.0 (TID 36) in 36 ms on localhost (2/2) -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 89.0, whose tasks have all completed, from pool -2016-06-13 19:11:04 INFO DAGScheduler:59 - ResultStage 89 (takeSample at KMeans.scala:376) finished in 0.037 s -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:65433 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Job 9 finished: takeSample at KMeans.scala:376, took 0.061500 s -2016-06-13 19:11:04 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=145679, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 880.0 B, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:65433 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(487) called with curMem=143265, maxMem=766075207 -2016-06-13 19:11:04 INFO BlockManager:59 - Removing RDD 63 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 487.0 B, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:65433 (size: 487.0 B, free: 730.5 MB) -2016-06-13 19:11:04 INFO ContextCleaner:59 - Cleaned RDD 63 -2016-06-13 19:11:04 INFO SparkContext:59 - Created broadcast 19 from broadcast at KMeans.scala:394 -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:65433 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:65433 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 19:11:04 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 19:11:04 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:11:04 INFO DAGScheduler:59 - Got job 10 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:11:04 INFO DAGScheduler:59 - Final stage: ResultStage 101(aggregate at KMeans.scala:402) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 96, ShuffleMapStage 100, ShuffleMapStage 93, ShuffleMapStage 94, ShuffleMapStage 98) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting ResultStage 101 (MapPartitionsRDD[106] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(10192) called with curMem=123464, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 10.0 KB, free 730.5 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(4263) called with curMem=133656, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.5 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:65433 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 101 (MapPartitionsRDD[106] at map at KMeans.scala:396) -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Adding task set 101.0 with 2 tasks -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 101.0 (TID 38, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 101.0 (TID 39, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:11:04 INFO Executor:59 - Running task 0.0 in stage 101.0 (TID 38) -2016-06-13 19:11:04 INFO Executor:59 - Running task 1.0 in stage 101.0 (TID 39) -2016-06-13 19:11:04 INFO CacheManager:59 - Partition rdd_106_1 not found, computing it -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:11:04 INFO CacheManager:59 - Partition rdd_106_0 not found, computing it -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:11:04 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 19:11:04 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=137919, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block rdd_106_0 stored as values in memory (estimated size 496.0 B, free 730.5 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=138415, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block rdd_106_1 stored as values in memory (estimated size 496.0 B, free 730.5 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added rdd_106_0 in memory on localhost:65433 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added rdd_106_1 in memory on localhost:65433 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:11:04 INFO Executor:59 - Finished task 0.0 in stage 101.0 (TID 38). 2647 bytes result sent to driver -2016-06-13 19:11:04 INFO Executor:59 - Finished task 1.0 in stage 101.0 (TID 39). 2647 bytes result sent to driver -2016-06-13 19:11:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 101.0 (TID 38) in 33 ms on localhost (1/2) -2016-06-13 19:11:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 101.0 (TID 39) in 33 ms on localhost (2/2) -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 101.0, whose tasks have all completed, from pool -2016-06-13 19:11:04 INFO DAGScheduler:59 - ResultStage 101 (aggregate at KMeans.scala:402) finished in 0.036 s -2016-06-13 19:11:04 INFO DAGScheduler:59 - Job 10 finished: aggregate at KMeans.scala:402, took 0.062372 s -2016-06-13 19:11:04 INFO MapPartitionsRDD:59 - Removing RDD 103 from persistence list -2016-06-13 19:11:04 INFO BlockManager:59 - Removing RDD 103 -2016-06-13 19:11:04 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:11:04 INFO DAGScheduler:59 - Got job 11 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:11:04 INFO DAGScheduler:59 - Final stage: ResultStage 113(collect at KMeans.scala:431) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 108, ShuffleMapStage 105, ShuffleMapStage 112, ShuffleMapStage 106, ShuffleMapStage 110) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting ResultStage 113 (MapPartitionsRDD[108] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(10240) called with curMem=138911, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 10.0 KB, free 730.4 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(4343) called with curMem=149151, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:65433 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 113 (MapPartitionsRDD[108] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Adding task set 113.0 with 2 tasks -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 113.0 (TID 40, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 113.0 (TID 41, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:11:04 INFO Executor:59 - Running task 0.0 in stage 113.0 (TID 40) -2016-06-13 19:11:04 INFO Executor:59 - Running task 1.0 in stage 113.0 (TID 41) -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_106_1 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_106_0 locally -2016-06-13 19:11:04 INFO Executor:59 - Finished task 1.0 in stage 113.0 (TID 41). 3471 bytes result sent to driver -2016-06-13 19:11:04 INFO Executor:59 - Finished task 0.0 in stage 113.0 (TID 40). 3118 bytes result sent to driver -2016-06-13 19:11:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 113.0 (TID 41) in 19 ms on localhost (1/2) -2016-06-13 19:11:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 113.0 (TID 40) in 21 ms on localhost (2/2) -2016-06-13 19:11:04 INFO DAGScheduler:59 - ResultStage 113 (collect at KMeans.scala:431) finished in 0.022 s -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 113.0, whose tasks have all completed, from pool -2016-06-13 19:11:04 INFO DAGScheduler:59 - Job 11 finished: collect at KMeans.scala:431, took 0.050493 s -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(1648) called with curMem=153494, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 1648.0 B, free 730.4 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(600) called with curMem=155142, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 600.0 B, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:65433 (size: 600.0 B, free: 730.5 MB) -2016-06-13 19:11:04 INFO SparkContext:59 - Created broadcast 22 from broadcast at KMeans.scala:394 -2016-06-13 19:11:04 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:11:04 INFO DAGScheduler:59 - Got job 12 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:11:04 INFO DAGScheduler:59 - Final stage: ResultStage 125(aggregate at KMeans.scala:402) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 117, ShuffleMapStage 118, ShuffleMapStage 122, ShuffleMapStage 120, ShuffleMapStage 124) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting ResultStage 125 (MapPartitionsRDD[110] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(10424) called with curMem=155742, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 10.2 KB, free 730.4 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(4357) called with curMem=166166, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:65433 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 125 (MapPartitionsRDD[110] at map at KMeans.scala:396) -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Adding task set 125.0 with 2 tasks -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 125.0 (TID 42, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 125.0 (TID 43, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:11:04 INFO Executor:59 - Running task 0.0 in stage 125.0 (TID 42) -2016-06-13 19:11:04 INFO Executor:59 - Running task 1.0 in stage 125.0 (TID 43) -2016-06-13 19:11:04 INFO CacheManager:59 - Partition rdd_110_0 not found, computing it -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:11:04 INFO CacheManager:59 - Partition rdd_110_1 not found, computing it -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_106_0 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_106_1 locally -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=170523, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block rdd_110_0 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=171019, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block rdd_110_1 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added rdd_110_0 in memory on localhost:65433 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:11:04 INFO Executor:59 - Finished task 0.0 in stage 125.0 (TID 42). 2647 bytes result sent to driver -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added rdd_110_1 in memory on localhost:65433 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:11:04 INFO Executor:59 - Finished task 1.0 in stage 125.0 (TID 43). 2647 bytes result sent to driver -2016-06-13 19:11:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 125.0 (TID 42) in 21 ms on localhost (1/2) -2016-06-13 19:11:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 125.0 (TID 43) in 22 ms on localhost (2/2) -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 125.0, whose tasks have all completed, from pool -2016-06-13 19:11:04 INFO DAGScheduler:59 - ResultStage 125 (aggregate at KMeans.scala:402) finished in 0.023 s -2016-06-13 19:11:04 INFO DAGScheduler:59 - Job 12 finished: aggregate at KMeans.scala:402, took 0.058053 s -2016-06-13 19:11:04 INFO MapPartitionsRDD:59 - Removing RDD 106 from persistence list -2016-06-13 19:11:04 INFO BlockManager:59 - Removing RDD 106 -2016-06-13 19:11:04 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:11:04 INFO DAGScheduler:59 - Got job 13 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:11:04 INFO DAGScheduler:59 - Final stage: ResultStage 137(collect at KMeans.scala:431) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 132, ShuffleMapStage 129, ShuffleMapStage 136, ShuffleMapStage 130, ShuffleMapStage 134) -2016-06-13 19:11:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting ResultStage 137 (MapPartitionsRDD[112] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(10480) called with curMem=170523, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 10.2 KB, free 730.4 MB) -2016-06-13 19:11:04 INFO MemoryStore:59 - ensureFreeSpace(4430) called with curMem=181003, maxMem=766075207 -2016-06-13 19:11:04 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 19:11:04 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:65433 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:11:04 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 137 (MapPartitionsRDD[112] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:11:04 INFO TaskSchedulerImpl:59 - Adding task set 137.0 with 2 tasks -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 137.0 (TID 44, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:11:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 137.0 (TID 45, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:11:04 INFO Executor:59 - Running task 1.0 in stage 137.0 (TID 45) -2016-06-13 19:11:04 INFO Executor:59 - Running task 0.0 in stage 137.0 (TID 44) -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_110_1 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:11:04 INFO BlockManager:59 - Found block rdd_110_0 locally -2016-06-13 19:11:04 INFO Executor:59 - Finished task 1.0 in stage 137.0 (TID 45). 3240 bytes result sent to driver -2016-06-13 19:11:04 INFO Executor:59 - Finished task 0.0 in stage 137.0 (TID 44). 2990 bytes result sent to driver -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 137.0 (TID 45) in 11 ms on localhost (1/2) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 137.0 (TID 44) in 13 ms on localhost (2/2) -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 137.0, whose tasks have all completed, from pool -2016-06-13 19:11:05 INFO DAGScheduler:59 - ResultStage 137 (collect at KMeans.scala:431) finished in 0.014 s -2016-06-13 19:11:05 INFO DAGScheduler:59 - Job 13 finished: collect at KMeans.scala:431, took 0.046577 s -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(1456) called with curMem=185433, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 1456.0 B, free 730.4 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(582) called with curMem=186889, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 582.0 B, free 730.4 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:65433 (size: 582.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO SparkContext:59 - Created broadcast 25 from broadcast at KMeans.scala:394 -2016-06-13 19:11:05 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Got job 14 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:11:05 INFO DAGScheduler:59 - Final stage: ResultStage 149(aggregate at KMeans.scala:402) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 144, ShuffleMapStage 141, ShuffleMapStage 148, ShuffleMapStage 142, ShuffleMapStage 146) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting ResultStage 149 (MapPartitionsRDD[114] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(10656) called with curMem=187471, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 10.4 KB, free 730.4 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(4409) called with curMem=198127, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:65433 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:11:05 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 149 (MapPartitionsRDD[114] at map at KMeans.scala:396) -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Adding task set 149.0 with 2 tasks -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 149.0 (TID 46, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 149.0 (TID 47, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:11:05 INFO Executor:59 - Running task 0.0 in stage 149.0 (TID 46) -2016-06-13 19:11:05 INFO Executor:59 - Running task 1.0 in stage 149.0 (TID 47) -2016-06-13 19:11:05 INFO CacheManager:59 - Partition rdd_114_1 not found, computing it -2016-06-13 19:11:05 INFO CacheManager:59 - Partition rdd_114_0 not found, computing it -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_110_1 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_110_0 locally -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=202536, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block rdd_114_1 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=203032, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block rdd_114_0 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added rdd_114_1 in memory on localhost:65433 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO Executor:59 - Finished task 1.0 in stage 149.0 (TID 47). 2647 bytes result sent to driver -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added rdd_114_0 in memory on localhost:65433 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO Executor:59 - Finished task 0.0 in stage 149.0 (TID 46). 2647 bytes result sent to driver -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 149.0 (TID 47) in 18 ms on localhost (1/2) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 149.0 (TID 46) in 19 ms on localhost (2/2) -2016-06-13 19:11:05 INFO DAGScheduler:59 - ResultStage 149 (aggregate at KMeans.scala:402) finished in 0.019 s -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 149.0, whose tasks have all completed, from pool -2016-06-13 19:11:05 INFO DAGScheduler:59 - Job 14 finished: aggregate at KMeans.scala:402, took 0.049724 s -2016-06-13 19:11:05 INFO MapPartitionsRDD:59 - Removing RDD 110 from persistence list -2016-06-13 19:11:05 INFO BlockManager:59 - Removing RDD 110 -2016-06-13 19:11:05 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Got job 15 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:11:05 INFO DAGScheduler:59 - Final stage: ResultStage 161(collect at KMeans.scala:431) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 153, ShuffleMapStage 154, ShuffleMapStage 158, ShuffleMapStage 156, ShuffleMapStage 160) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting ResultStage 161 (MapPartitionsRDD[116] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(10712) called with curMem=202536, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 10.5 KB, free 730.4 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(4468) called with curMem=213248, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:65433 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:11:05 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 161 (MapPartitionsRDD[116] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Adding task set 161.0 with 2 tasks -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 161.0 (TID 48, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 161.0 (TID 49, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:11:05 INFO Executor:59 - Running task 0.0 in stage 161.0 (TID 48) -2016-06-13 19:11:05 INFO Executor:59 - Running task 1.0 in stage 161.0 (TID 49) -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_114_0 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_114_1 locally -2016-06-13 19:11:05 INFO Executor:59 - Finished task 1.0 in stage 161.0 (TID 49). 2040 bytes result sent to driver -2016-06-13 19:11:05 INFO Executor:59 - Finished task 0.0 in stage 161.0 (TID 48). 2722 bytes result sent to driver -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 161.0 (TID 49) in 7 ms on localhost (1/2) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 161.0 (TID 48) in 9 ms on localhost (2/2) -2016-06-13 19:11:05 INFO DAGScheduler:59 - ResultStage 161 (collect at KMeans.scala:431) finished in 0.009 s -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 161.0, whose tasks have all completed, from pool -2016-06-13 19:11:05 INFO DAGScheduler:59 - Job 15 finished: collect at KMeans.scala:431, took 0.031460 s -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(648) called with curMem=217716, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 648.0 B, free 730.4 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(429) called with curMem=218364, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 429.0 B, free 730.4 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:65433 (size: 429.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO SparkContext:59 - Created broadcast 28 from broadcast at KMeans.scala:394 -2016-06-13 19:11:05 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Got job 16 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:11:05 INFO DAGScheduler:59 - Final stage: ResultStage 173(aggregate at KMeans.scala:402) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 168, ShuffleMapStage 165, ShuffleMapStage 172, ShuffleMapStage 166, ShuffleMapStage 170) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting ResultStage 173 (MapPartitionsRDD[118] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(10888) called with curMem=218793, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 10.6 KB, free 730.4 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(4468) called with curMem=229681, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:65433 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:11:05 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 173 (MapPartitionsRDD[118] at map at KMeans.scala:396) -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Adding task set 173.0 with 2 tasks -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 173.0 (TID 50, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 173.0 (TID 51, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:11:05 INFO Executor:59 - Running task 1.0 in stage 173.0 (TID 51) -2016-06-13 19:11:05 INFO Executor:59 - Running task 0.0 in stage 173.0 (TID 50) -2016-06-13 19:11:05 INFO CacheManager:59 - Partition rdd_118_1 not found, computing it -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:11:05 INFO CacheManager:59 - Partition rdd_118_0 not found, computing it -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_114_1 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_114_0 locally -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=234149, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block rdd_118_1 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=234645, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block rdd_118_0 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added rdd_118_1 in memory on localhost:65433 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO Executor:59 - Finished task 1.0 in stage 173.0 (TID 51). 2647 bytes result sent to driver -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added rdd_118_0 in memory on localhost:65433 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO Executor:59 - Finished task 0.0 in stage 173.0 (TID 50). 2647 bytes result sent to driver -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 173.0 (TID 51) in 16 ms on localhost (1/2) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 173.0 (TID 50) in 18 ms on localhost (2/2) -2016-06-13 19:11:05 INFO DAGScheduler:59 - ResultStage 173 (aggregate at KMeans.scala:402) finished in 0.017 s -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 173.0, whose tasks have all completed, from pool -2016-06-13 19:11:05 INFO DAGScheduler:59 - Job 16 finished: aggregate at KMeans.scala:402, took 0.043742 s -2016-06-13 19:11:05 INFO MapPartitionsRDD:59 - Removing RDD 114 from persistence list -2016-06-13 19:11:05 INFO BlockManager:59 - Removing RDD 114 -2016-06-13 19:11:05 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Got job 17 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:11:05 INFO DAGScheduler:59 - Final stage: ResultStage 185(collect at KMeans.scala:431) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 180, ShuffleMapStage 184, ShuffleMapStage 177, ShuffleMapStage 178, ShuffleMapStage 182) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting ResultStage 185 (MapPartitionsRDD[120] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(10944) called with curMem=234149, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 10.7 KB, free 730.4 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(4514) called with curMem=245093, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:65433 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:11:05 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 185 (MapPartitionsRDD[120] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Adding task set 185.0 with 2 tasks -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 185.0 (TID 52, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 185.0 (TID 53, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:11:05 INFO Executor:59 - Running task 1.0 in stage 185.0 (TID 53) -2016-06-13 19:11:05 INFO Executor:59 - Running task 0.0 in stage 185.0 (TID 52) -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_118_1 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_118_0 locally -2016-06-13 19:11:05 INFO Executor:59 - Finished task 1.0 in stage 185.0 (TID 53). 2040 bytes result sent to driver -2016-06-13 19:11:05 INFO Executor:59 - Finished task 0.0 in stage 185.0 (TID 52). 2040 bytes result sent to driver -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 185.0 (TID 53) in 8 ms on localhost (1/2) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 185.0 (TID 52) in 10 ms on localhost (2/2) -2016-06-13 19:11:05 INFO DAGScheduler:59 - ResultStage 185 (collect at KMeans.scala:431) finished in 0.010 s -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 185.0, whose tasks have all completed, from pool -2016-06-13 19:11:05 INFO DAGScheduler:59 - Job 17 finished: collect at KMeans.scala:431, took 0.033041 s -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(584) called with curMem=249607, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 584.0 B, free 730.3 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(236) called with curMem=250191, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 236.0 B, free 730.3 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:65433 (size: 236.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO SparkContext:59 - Created broadcast 31 from broadcast at KMeans.scala:394 -2016-06-13 19:11:05 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Got job 18 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:11:05 INFO DAGScheduler:59 - Final stage: ResultStage 197(aggregate at KMeans.scala:402) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 190, ShuffleMapStage 194, ShuffleMapStage 192, ShuffleMapStage 189, ShuffleMapStage 196) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting ResultStage 197 (MapPartitionsRDD[122] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(11128) called with curMem=250427, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 10.9 KB, free 730.3 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(4524) called with curMem=261555, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:65433 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:11:05 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 197 (MapPartitionsRDD[122] at map at KMeans.scala:396) -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Adding task set 197.0 with 2 tasks -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 197.0 (TID 54, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 197.0 (TID 55, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:11:05 INFO Executor:59 - Running task 0.0 in stage 197.0 (TID 54) -2016-06-13 19:11:05 INFO Executor:59 - Running task 1.0 in stage 197.0 (TID 55) -2016-06-13 19:11:05 INFO CacheManager:59 - Partition rdd_122_1 not found, computing it -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:11:05 INFO CacheManager:59 - Partition rdd_122_0 not found, computing it -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_118_1 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_118_0 locally -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=266079, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block rdd_122_1 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=266575, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block rdd_122_0 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added rdd_122_1 in memory on localhost:65433 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO Executor:59 - Finished task 1.0 in stage 197.0 (TID 55). 2647 bytes result sent to driver -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added rdd_122_0 in memory on localhost:65433 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO Executor:59 - Finished task 0.0 in stage 197.0 (TID 54). 2647 bytes result sent to driver -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 197.0 (TID 55) in 10 ms on localhost (1/2) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 197.0 (TID 54) in 13 ms on localhost (2/2) -2016-06-13 19:11:05 INFO DAGScheduler:59 - ResultStage 197 (aggregate at KMeans.scala:402) finished in 0.013 s -2016-06-13 19:11:05 INFO DAGScheduler:59 - Job 18 finished: aggregate at KMeans.scala:402, took 0.037259 s -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 197.0, whose tasks have all completed, from pool -2016-06-13 19:11:05 INFO MapPartitionsRDD:59 - Removing RDD 118 from persistence list -2016-06-13 19:11:05 INFO BlockManager:59 - Removing RDD 118 -2016-06-13 19:11:05 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Got job 19 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:11:05 INFO DAGScheduler:59 - Final stage: ResultStage 209(collect at KMeans.scala:431) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 204, ShuffleMapStage 201, ShuffleMapStage 208, ShuffleMapStage 202, ShuffleMapStage 206) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting ResultStage 209 (MapPartitionsRDD[124] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(11176) called with curMem=266079, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 10.9 KB, free 730.3 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(4565) called with curMem=277255, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.3 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:65433 (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:11:05 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 209 (MapPartitionsRDD[124] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Adding task set 209.0 with 2 tasks -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 209.0 (TID 56, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 209.0 (TID 57, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:11:05 INFO Executor:59 - Running task 0.0 in stage 209.0 (TID 56) -2016-06-13 19:11:05 INFO Executor:59 - Running task 1.0 in stage 209.0 (TID 57) -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_122_1 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_122_0 locally -2016-06-13 19:11:05 INFO Executor:59 - Finished task 1.0 in stage 209.0 (TID 57). 2040 bytes result sent to driver -2016-06-13 19:11:05 INFO Executor:59 - Finished task 0.0 in stage 209.0 (TID 56). 2040 bytes result sent to driver -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 209.0 (TID 57) in 5 ms on localhost (1/2) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 209.0 (TID 56) in 7 ms on localhost (2/2) -2016-06-13 19:11:05 INFO DAGScheduler:59 - ResultStage 209 (collect at KMeans.scala:431) finished in 0.007 s -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 209.0, whose tasks have all completed, from pool -2016-06-13 19:11:05 INFO DAGScheduler:59 - Job 19 finished: collect at KMeans.scala:431, took 0.024985 s -2016-06-13 19:11:05 INFO MapPartitionsRDD:59 - Removing RDD 122 from persistence list -2016-06-13 19:11:05 INFO BlockManager:59 - Removing RDD 122 -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(2880) called with curMem=280828, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 2.8 KB, free 730.3 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(708) called with curMem=283708, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 708.0 B, free 730.3 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:65433 (size: 708.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO SparkContext:59 - Created broadcast 34 from broadcast at KMeans.scala:445 -2016-06-13 19:11:05 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:450 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Registering RDD 125 (flatMap at KMeans.scala:446) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Got job 20 (collectAsMap at KMeans.scala:450) with 2 output partitions -2016-06-13 19:11:05 INFO DAGScheduler:59 - Final stage: ResultStage 222(collectAsMap at KMeans.scala:450) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 221) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 221) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 221 (MapPartitionsRDD[125] at flatMap at KMeans.scala:446), which has no missing parents -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(9672) called with curMem=284416, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 9.4 KB, free 730.3 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(4106) called with curMem=294088, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:65433 (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:11:05 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 221 (MapPartitionsRDD[125] at flatMap at KMeans.scala:446) -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Adding task set 221.0 with 2 tasks -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 221.0 (TID 58, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 221.0 (TID 59, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:11:05 INFO Executor:59 - Running task 1.0 in stage 221.0 (TID 59) -2016-06-13 19:11:05 INFO Executor:59 - Running task 0.0 in stage 221.0 (TID 58) -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:11:05 INFO Executor:59 - Finished task 0.0 in stage 221.0 (TID 58). 2254 bytes result sent to driver -2016-06-13 19:11:05 INFO Executor:59 - Finished task 1.0 in stage 221.0 (TID 59). 2254 bytes result sent to driver -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 221.0 (TID 58) in 60 ms on localhost (1/2) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 221.0 (TID 59) in 59 ms on localhost (2/2) -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 221.0, whose tasks have all completed, from pool -2016-06-13 19:11:05 INFO DAGScheduler:59 - ShuffleMapStage 221 (flatMap at KMeans.scala:446) finished in 0.062 s -2016-06-13 19:11:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 222) -2016-06-13 19:11:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:05 INFO DAGScheduler:59 - Missing parents for ResultStage 222: List() -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting ResultStage 222 (ShuffledRDD[126] at reduceByKey at KMeans.scala:450), which is now runnable -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=298194, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 2.3 KB, free 730.3 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(1382) called with curMem=300506, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 1382.0 B, free 730.3 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:65433 (size: 1382.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 222 (ShuffledRDD[126] at reduceByKey at KMeans.scala:450) -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Adding task set 222.0 with 2 tasks -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 222.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 222.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:11:05 INFO Executor:59 - Running task 0.0 in stage 222.0 (TID 60) -2016-06-13 19:11:05 INFO Executor:59 - Running task 1.0 in stage 222.0 (TID 61) -2016-06-13 19:11:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:05 INFO Executor:59 - Finished task 1.0 in stage 222.0 (TID 61). 1744 bytes result sent to driver -2016-06-13 19:11:05 INFO Executor:59 - Finished task 0.0 in stage 222.0 (TID 60). 1852 bytes result sent to driver -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 222.0 (TID 61) in 13 ms on localhost (1/2) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 222.0 (TID 60) in 14 ms on localhost (2/2) -2016-06-13 19:11:05 INFO DAGScheduler:59 - ResultStage 222 (collectAsMap at KMeans.scala:450) finished in 0.015 s -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 222.0, whose tasks have all completed, from pool -2016-06-13 19:11:05 INFO DAGScheduler:59 - Job 20 finished: collectAsMap at KMeans.scala:450, took 0.111194 s -2016-06-13 19:11:05 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:11:05 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:11:05 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:11:05 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:11:05 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:11:05 INFO KMeans:59 - Initialization with k-means|| took 1.262 seconds. -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(824) called with curMem=301888, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 824.0 B, free 730.3 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(339) called with curMem=302712, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 339.0 B, free 730.3 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:65433 (size: 339.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO SparkContext:59 - Created broadcast 37 from broadcast at KMeans.scala:276 -2016-06-13 19:11:05 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:302 -2016-06-13 19:11:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:11:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:11:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:11:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:11:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:11:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 151 bytes -2016-06-13 19:11:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:11:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 151 bytes -2016-06-13 19:11:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:11:05 INFO DAGScheduler:59 - Registering RDD 127 (mapPartitions at KMeans.scala:279) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Got job 21 (collectAsMap at KMeans.scala:302) with 2 output partitions -2016-06-13 19:11:05 INFO DAGScheduler:59 - Final stage: ResultStage 235(collectAsMap at KMeans.scala:302) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 234) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 234) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 234 (MapPartitionsRDD[127] at mapPartitions at KMeans.scala:279), which has no missing parents -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(10296) called with curMem=303051, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 10.1 KB, free 730.3 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(4440) called with curMem=313347, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:65433 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:11:05 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 234 (MapPartitionsRDD[127] at mapPartitions at KMeans.scala:279) -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Adding task set 234.0 with 2 tasks -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 234.0 (TID 62, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 234.0 (TID 63, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:11:05 INFO Executor:59 - Running task 0.0 in stage 234.0 (TID 62) -2016-06-13 19:11:05 INFO Executor:59 - Running task 1.0 in stage 234.0 (TID 63) -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:11:05 INFO Executor:59 - Finished task 0.0 in stage 234.0 (TID 62). 2413 bytes result sent to driver -2016-06-13 19:11:05 INFO Executor:59 - Finished task 1.0 in stage 234.0 (TID 63). 2413 bytes result sent to driver -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 234.0 (TID 62) in 17 ms on localhost (1/2) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 234.0 (TID 63) in 19 ms on localhost (2/2) -2016-06-13 19:11:05 INFO DAGScheduler:59 - ShuffleMapStage 234 (mapPartitions at KMeans.scala:279) finished in 0.020 s -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 234.0, whose tasks have all completed, from pool -2016-06-13 19:11:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 235) -2016-06-13 19:11:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:05 INFO DAGScheduler:59 - Missing parents for ResultStage 235: List() -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting ResultStage 235 (ShuffledRDD[128] at reduceByKey at KMeans.scala:302), which is now runnable -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(2808) called with curMem=317787, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 2.7 KB, free 730.3 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(1589) called with curMem=320595, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 1589.0 B, free 730.3 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:65433 (size: 1589.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 235 (ShuffledRDD[128] at reduceByKey at KMeans.scala:302) -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Adding task set 235.0 with 2 tasks -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 235.0 (TID 64, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 235.0 (TID 65, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:11:05 INFO Executor:59 - Running task 0.0 in stage 235.0 (TID 64) -2016-06-13 19:11:05 INFO Executor:59 - Running task 1.0 in stage 235.0 (TID 65) -2016-06-13 19:11:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:05 INFO Executor:59 - Finished task 1.0 in stage 235.0 (TID 65). 1671 bytes result sent to driver -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 235.0 (TID 65) in 8 ms on localhost (1/2) -2016-06-13 19:11:05 INFO Executor:59 - Finished task 0.0 in stage 235.0 (TID 64). 1785 bytes result sent to driver -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 235.0 (TID 64) in 11 ms on localhost (2/2) -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 235.0, whose tasks have all completed, from pool -2016-06-13 19:11:05 INFO DAGScheduler:59 - ResultStage 235 (collectAsMap at KMeans.scala:302) finished in 0.011 s -2016-06-13 19:11:05 INFO DAGScheduler:59 - Job 21 finished: collectAsMap at KMeans.scala:302, took 0.057150 s -2016-06-13 19:11:05 INFO KMeans:59 - Run 0 finished in 1 iterations -2016-06-13 19:11:05 INFO KMeans:59 - Run 1 finished in 1 iterations -2016-06-13 19:11:05 INFO KMeans:59 - Run 2 finished in 1 iterations -2016-06-13 19:11:05 INFO KMeans:59 - Run 3 finished in 1 iterations -2016-06-13 19:11:05 INFO KMeans:59 - Run 4 finished in 1 iterations -2016-06-13 19:11:05 INFO KMeans:59 - Iterations took 0.085 seconds. -2016-06-13 19:11:05 INFO KMeans:59 - KMeans converged in 1 iterations. -2016-06-13 19:11:05 INFO KMeans:59 - The cost for the best run is 6.650150218788335E-6. -2016-06-13 19:11:05 INFO MapPartitionsRDD:59 - Removing RDD 100 from persistence list -2016-06-13 19:11:05 INFO BlockManager:59 - Removing RDD 100 -2016-06-13 19:11:05 WARN KMeans:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:11:05 INFO SparkContext:59 - Starting job: collect at PowerIterationClusteringSuite.scala:56 -2016-06-13 19:11:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:11:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:11:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:11:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:11:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:11:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 151 bytes -2016-06-13 19:11:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:11:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 151 bytes -2016-06-13 19:11:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:11:05 INFO DAGScheduler:59 - Got job 22 (collect at PowerIterationClusteringSuite.scala:56) with 2 output partitions -2016-06-13 19:11:05 INFO DAGScheduler:59 - Final stage: ResultStage 247(collect at PowerIterationClusteringSuite.scala:56) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 242, ShuffleMapStage 239, ShuffleMapStage 246, ShuffleMapStage 240, ShuffleMapStage 244) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting ResultStage 247 (MapPartitionsRDD[131] at mapPartitions at PowerIterationClustering.scala:232), which has no missing parents -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(8472) called with curMem=321704, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 8.3 KB, free 730.3 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=330176, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:65433 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:11:05 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 247 (MapPartitionsRDD[131] at mapPartitions at PowerIterationClustering.scala:232) -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Adding task set 247.0 with 2 tasks -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 247.0 (TID 66, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 247.0 (TID 67, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:11:05 INFO Executor:59 - Running task 0.0 in stage 247.0 (TID 66) -2016-06-13 19:11:05 INFO Executor:59 - Running task 1.0 in stage 247.0 (TID 67) -2016-06-13 19:11:05 INFO CacheManager:59 - Partition rdd_129_1 not found, computing it -2016-06-13 19:11:05 INFO CacheManager:59 - Partition rdd_129_0 not found, computing it -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=333768, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block rdd_129_0 stored as values in memory (estimated size 1168.0 B, free 730.3 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=334936, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block rdd_129_1 stored as values in memory (estimated size 1168.0 B, free 730.3 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added rdd_129_0 in memory on localhost:65433 (size: 1168.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added rdd_129_1 in memory on localhost:65433 (size: 1168.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO Executor:59 - Finished task 0.0 in stage 247.0 (TID 66). 2919 bytes result sent to driver -2016-06-13 19:11:05 INFO Executor:59 - Finished task 1.0 in stage 247.0 (TID 67). 2919 bytes result sent to driver -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 247.0 (TID 66) in 9 ms on localhost (1/2) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 247.0 (TID 67) in 9 ms on localhost (2/2) -2016-06-13 19:11:05 INFO DAGScheduler:59 - ResultStage 247 (collect at PowerIterationClusteringSuite.scala:56) finished in 0.010 s -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 247.0, whose tasks have all completed, from pool -2016-06-13 19:11:05 INFO DAGScheduler:59 - Job 22 finished: collect at PowerIterationClusteringSuite.scala:56, took 0.028589 s -2016-06-13 19:11:05 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:340 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Registering RDD 143 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Registering RDD 137 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Got job 23 (sum at PowerIterationClustering.scala:340) with 2 output partitions -2016-06-13 19:11:05 INFO DAGScheduler:59 - Final stage: ResultStage 250(sum at PowerIterationClustering.scala:340) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 248, ShuffleMapStage 249) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 248, ShuffleMapStage 249) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 248 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[143] at mapPartitions at GraphImpl.scala:235), which has no missing parents -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(3432) called with curMem=336104, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(1974) called with curMem=339536, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 1974.0 B, free 730.3 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:65433 (size: 1974.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 248 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[143] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Adding task set 248.0 with 2 tasks -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 249 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[137] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 248.0 (TID 68, localhost, PROCESS_LOCAL, 2488 bytes) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 248.0 (TID 69, localhost, PROCESS_LOCAL, 2563 bytes) -2016-06-13 19:11:05 INFO Executor:59 - Running task 0.0 in stage 248.0 (TID 68) -2016-06-13 19:11:05 INFO Executor:59 - Running task 1.0 in stage 248.0 (TID 69) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(3016) called with curMem=341510, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 2.9 KB, free 730.3 MB) -2016-06-13 19:11:05 INFO CacheManager:59 - Partition rdd_134_0 not found, computing it -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(1779) called with curMem=344526, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1779.0 B, free 730.3 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(2856) called with curMem=346305, maxMem=766075207 -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:65433 (size: 1779.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - Block rdd_134_0 stored as values in memory (estimated size 2.8 KB, free 730.3 MB) -2016-06-13 19:11:05 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:05 INFO CacheManager:59 - Partition rdd_134_1 not found, computing it -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 249 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[137] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Adding task set 249.0 with 2 tasks -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(2952) called with curMem=349161, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block rdd_134_1 stored as values in memory (estimated size 2.9 KB, free 730.3 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added rdd_134_0 in memory on localhost:65433 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added rdd_134_1 in memory on localhost:65433 (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 249.0 (TID 70, localhost, PROCESS_LOCAL, 2488 bytes) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 249.0 (TID 71, localhost, PROCESS_LOCAL, 2563 bytes) -2016-06-13 19:11:05 INFO Executor:59 - Running task 0.0 in stage 249.0 (TID 70) -2016-06-13 19:11:05 INFO Executor:59 - Running task 1.0 in stage 249.0 (TID 71) -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_134_0 locally -2016-06-13 19:11:05 INFO BlockManager:59 - Found block rdd_134_1 locally -2016-06-13 19:11:05 INFO Executor:59 - Finished task 1.0 in stage 248.0 (TID 69). 1739 bytes result sent to driver -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 248.0 (TID 69) in 45 ms on localhost (1/2) -2016-06-13 19:11:05 INFO Executor:59 - Finished task 1.0 in stage 249.0 (TID 71). 2254 bytes result sent to driver -2016-06-13 19:11:05 INFO Executor:59 - Finished task 0.0 in stage 249.0 (TID 70). 2254 bytes result sent to driver -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 249.0 (TID 71) in 36 ms on localhost (1/2) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 249.0 (TID 70) in 38 ms on localhost (2/2) -2016-06-13 19:11:05 INFO DAGScheduler:59 - ShuffleMapStage 249 (mapPartitions at VertexRDD.scala:358) finished in 0.041 s -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 249.0, whose tasks have all completed, from pool -2016-06-13 19:11:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:05 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 248) -2016-06-13 19:11:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 250) -2016-06-13 19:11:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:05 INFO DAGScheduler:59 - Missing parents for ResultStage 250: List(ShuffleMapStage 248) -2016-06-13 19:11:05 INFO Executor:59 - Finished task 0.0 in stage 248.0 (TID 68). 1739 bytes result sent to driver -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 248.0 (TID 68) in 59 ms on localhost (2/2) -2016-06-13 19:11:05 INFO DAGScheduler:59 - ShuffleMapStage 248 (mapPartitions at GraphImpl.scala:235) finished in 0.060 s -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 248.0, whose tasks have all completed, from pool -2016-06-13 19:11:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 250) -2016-06-13 19:11:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:05 INFO DAGScheduler:59 - Missing parents for ResultStage 250: List() -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting ResultStage 250 (MapPartitionsRDD[153] at values at PowerIterationClustering.scala:340), which is now runnable -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(4728) called with curMem=352113, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 4.6 KB, free 730.2 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(2556) called with curMem=356841, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:65433 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:11:05 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 250 (MapPartitionsRDD[153] at values at PowerIterationClustering.scala:340) -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Adding task set 250.0 with 2 tasks -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 250.0 (TID 72, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 250.0 (TID 73, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:11:05 INFO Executor:59 - Running task 1.0 in stage 250.0 (TID 73) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:65433 in memory (size: 1779.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO Executor:59 - Running task 0.0 in stage 250.0 (TID 72) -2016-06-13 19:11:05 INFO CacheManager:59 - Partition rdd_145_1 not found, computing it -2016-06-13 19:11:05 INFO CacheManager:59 - Partition rdd_140_1 not found, computing it -2016-06-13 19:11:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:05 INFO CacheManager:59 - Partition rdd_145_0 not found, computing it -2016-06-13 19:11:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:05 INFO CacheManager:59 - Partition rdd_140_0 not found, computing it -2016-06-13 19:11:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:65433 in memory (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(1792) called with curMem=351010, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block rdd_140_1 stored as values in memory (estimated size 1792.0 B, free 730.2 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(1776) called with curMem=344330, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block rdd_140_0 stored as values in memory (estimated size 1776.0 B, free 730.3 MB) -2016-06-13 19:11:05 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added rdd_140_1 in memory on localhost:65433 (size: 1792.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added rdd_140_0 in memory on localhost:65433 (size: 1776.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:65433 in memory (size: 1589.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(1840) called with curMem=344517, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block rdd_145_1 stored as values in memory (estimated size 1840.0 B, free 730.3 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(1824) called with curMem=343549, maxMem=766075207 -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added rdd_145_1 in memory on localhost:65433 (size: 1840.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO MemoryStore:59 - Block rdd_145_0 stored as values in memory (estimated size 1824.0 B, free 730.3 MB) -2016-06-13 19:11:05 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added rdd_145_0 in memory on localhost:65433 (size: 1824.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO Executor:59 - Finished task 0.0 in stage 250.0 (TID 72). 1840 bytes result sent to driver -2016-06-13 19:11:05 INFO Executor:59 - Finished task 1.0 in stage 250.0 (TID 73). 1840 bytes result sent to driver -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 250.0 (TID 72) in 24 ms on localhost (1/2) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 250.0 (TID 73) in 24 ms on localhost (2/2) -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 250.0, whose tasks have all completed, from pool -2016-06-13 19:11:05 INFO DAGScheduler:59 - ResultStage 250 (sum at PowerIterationClustering.scala:340) finished in 0.027 s -2016-06-13 19:11:05 INFO DAGScheduler:59 - Job 23 finished: sum at PowerIterationClustering.scala:340, took 0.127105 s -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:65433 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:11:05 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-06-13 19:11:05 INFO ContextCleaner:59 - Cleaned shuffle 10 -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:65433 in memory (size: 339.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-06-13 19:11:05 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-06-13 19:11:05 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-06-13 19:11:05 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-06-13 19:11:05 INFO SparkContext:59 - Starting job: reduce at VertexRDDImpl.scala:90 -2016-06-13 19:11:05 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-06-13 19:11:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:11:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:11:05 INFO DAGScheduler:59 - Got job 24 (reduce at VertexRDDImpl.scala:90) with 2 output partitions -2016-06-13 19:11:05 INFO DAGScheduler:59 - Final stage: ResultStage 253(reduce at VertexRDDImpl.scala:90) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 251, ShuffleMapStage 252) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:65433 in memory (size: 1382.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting ResultStage 253 (MapPartitionsRDD[158] at map at VertexRDDImpl.scala:90), which has no missing parents -2016-06-13 19:11:05 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(5200) called with curMem=312002, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 5.1 KB, free 730.3 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:65433 in memory (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:11:05 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-06-13 19:11:05 INFO ContextCleaner:59 - Cleaned shuffle 9 -2016-06-13 19:11:05 INFO MemoryStore:59 - ensureFreeSpace(2659) called with curMem=317202, maxMem=766075207 -2016-06-13 19:11:05 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.3 MB) -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:65433 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:11:05 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:05 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:65433 in memory (size: 708.0 B, free: 730.5 MB) -2016-06-13 19:11:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 253 (MapPartitionsRDD[158] at map at VertexRDDImpl.scala:90) -2016-06-13 19:11:05 INFO TaskSchedulerImpl:59 - Adding task set 253.0 with 2 tasks -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 253.0 (TID 74, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:11:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 253.0 (TID 75, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:11:05 INFO Executor:59 - Running task 0.0 in stage 253.0 (TID 74) -2016-06-13 19:11:05 INFO Executor:59 - Running task 1.0 in stage 253.0 (TID 75) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:65433 in memory (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_145_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_145_0 locally -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 253.0 (TID 75). 2161 bytes result sent to driver -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 253.0 (TID 74). 2161 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 253.0 (TID 75) in 7 ms on localhost (1/2) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 253.0 (TID 74) in 8 ms on localhost (2/2) -2016-06-13 19:11:06 INFO DAGScheduler:59 - ResultStage 253 (reduce at VertexRDDImpl.scala:90) finished in 0.008 s -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 253.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - Job 24 finished: reduce at VertexRDDImpl.scala:90, took 0.021913 s -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:65433 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-06-13 19:11:06 INFO BlockManager:59 - Removing RDD 122 -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned RDD 122 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:65433 in memory (size: 236.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:65433 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:65433 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-06-13 19:11:06 INFO BlockManager:59 - Removing RDD 118 -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned RDD 118 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:65433 in memory (size: 429.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:65433 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Registering RDD 147 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Registering RDD 159 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:65433 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Registering RDD 163 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Got job 25 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:11:06 INFO DAGScheduler:59 - Final stage: ResultStage 261(sum at PowerIterationClustering.scala:369) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 260, ShuffleMapStage 257, ShuffleMapStage 258) -2016-06-13 19:11:06 INFO BlockManager:59 - Removing RDD 114 -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned RDD 114 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 260) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 256 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[147] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:65433 in memory (size: 582.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=219886, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 4.1 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(2378) called with curMem=219704, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:65433 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:65433 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 256 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[147] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 256.0 with 2 tasks -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 259 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[159] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 256.0 (TID 76, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:65433 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 256.0 (TID 77, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 256.0 (TID 77) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 256.0 (TID 76) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(4904) called with curMem=196821, maxMem=766075207 -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_145_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Removing RDD 110 -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_145_1 locally -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(2572) called with curMem=201725, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned RDD 110 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:65433 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 259 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[159] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 259.0 with 2 tasks -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:65433 in memory (size: 600.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 259.0 (TID 78, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 259.0 (TID 79, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 259.0 (TID 79) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 259.0 (TID 78) -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_156_1 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_145_1 locally -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_156_0 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_145_0 locally -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=202049, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_156_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=203481, maxMem=766075207 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_156_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_156_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_156_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 259.0 (TID 79). 2834 bytes result sent to driver -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 259.0 (TID 78). 2834 bytes result sent to driver -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:65433 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 259.0 (TID 79) in 14 ms on localhost (1/2) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 259.0 (TID 78) in 15 ms on localhost (2/2) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 259.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - ShuffleMapStage 259 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.016 s -2016-06-13 19:11:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:06 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 256) -2016-06-13 19:11:06 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 260, ResultStage 261) -2016-06-13 19:11:06 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 256.0 (TID 76). 2254 bytes result sent to driver -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 256.0 (TID 77). 2254 bytes result sent to driver -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 260: List(ShuffleMapStage 256) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 256.0 (TID 76) in 29 ms on localhost (1/2) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 256.0 (TID 77) in 29 ms on localhost (2/2) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 256.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:65433 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 19:11:06 INFO BlockManager:59 - Removing RDD 106 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents for ResultStage 261: List(ShuffleMapStage 260) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned RDD 106 -2016-06-13 19:11:06 INFO DAGScheduler:59 - ShuffleMapStage 256 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.034 s -2016-06-13 19:11:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:06 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:06 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 260, ResultStage 261) -2016-06-13 19:11:06 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 260: List() -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:65433 in memory (size: 487.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents for ResultStage 261: List(ShuffleMapStage 260) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 260 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[163] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:65433 in memory (size: 9.9 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=138796, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(2538) called with curMem=140181, maxMem=766075207 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:65433 in memory (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:65433 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 260 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[163] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 260.0 with 2 tasks -2016-06-13 19:11:06 INFO BlockManager:59 - Removing RDD 100 -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 260.0 (TID 80, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned RDD 100 -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 260.0 (TID 81, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 260.0 (TID 81) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 260.0 (TID 80) -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_134_0 locally -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_134_1 locally -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 260.0 (TID 80). 2470 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 260.0 (TID 80) in 36 ms on localhost (1/2) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 260.0 (TID 81). 2470 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 260.0 (TID 81) in 56 ms on localhost (2/2) -2016-06-13 19:11:06 INFO DAGScheduler:59 - ShuffleMapStage 260 (mapPartitions at GraphImpl.scala:235) finished in 0.058 s -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 260.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:06 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:06 INFO DAGScheduler:59 - waiting: Set(ResultStage 261) -2016-06-13 19:11:06 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents for ResultStage 261: List() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ResultStage 261 (MapPartitionsRDD[168] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(5944) called with curMem=134551, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 5.8 KB, free 730.5 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=140495, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:65433 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 261 (MapPartitionsRDD[168] at map at PowerIterationClustering.scala:369) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 261.0 with 2 tasks -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 261.0 (TID 82, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 261.0 (TID 83, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 261.0 (TID 83) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 261.0 (TID 82) -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_165_1 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_156_1 locally -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_165_0 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_156_0 locally -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=143387, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_165_0 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=144867, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_165_1 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_165_0 in memory on localhost:65433 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_165_1 in memory on localhost:65433 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 261.0 (TID 82). 2880 bytes result sent to driver -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 261.0 (TID 83). 2880 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 261.0 (TID 82) in 10 ms on localhost (1/2) -2016-06-13 19:11:06 INFO DAGScheduler:59 - ResultStage 261 (sum at PowerIterationClustering.scala:369) finished in 0.011 s -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 261.0 (TID 83) in 10 ms on localhost (2/2) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 261.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - Job 25 finished: sum at PowerIterationClustering.scala:369, took 0.136381 s -2016-06-13 19:11:06 INFO PowerIterationClustering:59 - Iteration 0: norm(v) = 36.22334869431644. -2016-06-13 19:11:06 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 151 bytes -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:11:06 INFO DAGScheduler:59 - Got job 26 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:11:06 INFO DAGScheduler:59 - Final stage: ResultStage 269(sum at PowerIterationClustering.scala:375) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 266, ShuffleMapStage 268, ShuffleMapStage 265) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ResultStage 269 (MapPartitionsRDD[179] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(6480) called with curMem=146347, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 6.3 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(3063) called with curMem=152827, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:65433 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 269 (MapPartitionsRDD[179] at values at PowerIterationClustering.scala:375) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 269.0 with 2 tasks -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 269.0 (TID 84, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 269.0 (TID 85, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 269.0 (TID 85) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 269.0 (TID 84) -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_171_1 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_156_1 locally -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_171_0 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_165_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_156_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_165_0 locally -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=155890, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_171_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=157322, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_171_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_171_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_171_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 269.0 (TID 85). 2664 bytes result sent to driver -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 269.0 (TID 84). 2664 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 269.0 (TID 85) in 7 ms on localhost (1/2) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 269.0 (TID 84) in 8 ms on localhost (2/2) -2016-06-13 19:11:06 INFO DAGScheduler:59 - ResultStage 269 (sum at PowerIterationClustering.scala:375) finished in 0.009 s -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 269.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - Job 26 finished: sum at PowerIterationClustering.scala:375, took 0.021989 s -2016-06-13 19:11:06 INFO PowerIterationClustering:59 - Iteration 0: delta = 0.010579646370791257. -2016-06-13 19:11:06 INFO PowerIterationClustering:59 - Iteration 0: diff(delta) = 1.7976931348623157E308. -2016-06-13 19:11:06 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Registering RDD 182 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Registering RDD 186 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Got job 27 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:11:06 INFO DAGScheduler:59 - Final stage: ResultStage 279(sum at PowerIterationClustering.scala:369) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 273, ShuffleMapStage 274, ShuffleMapStage 278, ShuffleMapStage 276) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 278) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 277 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[182] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(5704) called with curMem=158754, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.6 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(2815) called with curMem=164458, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:65433 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 277 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[182] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 277.0 with 2 tasks -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 277.0 (TID 86, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 277.0 (TID 87, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 277.0 (TID 86) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 277.0 (TID 87) -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_180_1 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_165_1 locally -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_180_0 not found, computing it -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=167273, maxMem=766075207 -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_165_0 locally -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_180_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=168705, maxMem=766075207 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_180_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_180_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_180_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 277.0 (TID 87). 2834 bytes result sent to driver -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 277.0 (TID 86). 2834 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 277.0 (TID 87) in 10 ms on localhost (1/2) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 277.0 (TID 86) in 12 ms on localhost (2/2) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 277.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - ShuffleMapStage 277 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.012 s -2016-06-13 19:11:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:06 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:06 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 278, ResultStage 279) -2016-06-13 19:11:06 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 278: List() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents for ResultStage 279: List(ShuffleMapStage 278) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 278 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[186] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=170137, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=174985, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:65433 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 278 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[186] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 278.0 with 2 tasks -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 278.0 (TID 88, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 278.0 (TID 89, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 278.0 (TID 88) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 278.0 (TID 89) -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_134_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_134_1 locally -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 278.0 (TID 88). 2470 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 278.0 (TID 88) in 22 ms on localhost (1/2) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 278.0 (TID 89). 2470 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 278.0 (TID 89) in 25 ms on localhost (2/2) -2016-06-13 19:11:06 INFO DAGScheduler:59 - ShuffleMapStage 278 (mapPartitions at GraphImpl.scala:235) finished in 0.025 s -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 278.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:06 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:06 INFO DAGScheduler:59 - waiting: Set(ResultStage 279) -2016-06-13 19:11:06 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents for ResultStage 279: List() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ResultStage 279 (MapPartitionsRDD[191] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(6640) called with curMem=177527, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 6.5 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(3103) called with curMem=184167, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:65433 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 279 (MapPartitionsRDD[191] at map at PowerIterationClustering.scala:369) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 279.0 with 2 tasks -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 279.0 (TID 90, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 279.0 (TID 91, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 279.0 (TID 91) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 279.0 (TID 90) -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_188_1 not found, computing it -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_188_0 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_180_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_180_0 locally -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=187270, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_188_0 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=188750, maxMem=766075207 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_188_0 in memory on localhost:65433 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_188_1 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 279.0 (TID 90). 2880 bytes result sent to driver -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_188_1 in memory on localhost:65433 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 279.0 (TID 91). 2880 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 279.0 (TID 90) in 11 ms on localhost (1/2) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 279.0 (TID 91) in 11 ms on localhost (2/2) -2016-06-13 19:11:06 INFO DAGScheduler:59 - ResultStage 279 (sum at PowerIterationClustering.scala:369) finished in 0.012 s -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 279.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - Job 27 finished: sum at PowerIterationClustering.scala:369, took 0.082611 s -2016-06-13 19:11:06 INFO PowerIterationClustering:59 - Iteration 1: norm(v) = 36.22334869431644. -2016-06-13 19:11:06 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 151 bytes -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 151 bytes -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:11:06 INFO DAGScheduler:59 - Got job 28 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:11:06 INFO DAGScheduler:59 - Final stage: ResultStage 289(sum at PowerIterationClustering.scala:375) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 284, ShuffleMapStage 288, ShuffleMapStage 286, ShuffleMapStage 283) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ResultStage 289 (MapPartitionsRDD[202] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(7072) called with curMem=190230, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 6.9 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(3228) called with curMem=197302, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:65433 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 289 (MapPartitionsRDD[202] at values at PowerIterationClustering.scala:375) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 289.0 with 2 tasks -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 289.0 (TID 92, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 289.0 (TID 93, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 289.0 (TID 93) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 289.0 (TID 92) -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_194_1 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_180_1 locally -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_194_0 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_188_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_180_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_188_0 locally -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=200530, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_194_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=201962, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_194_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_194_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_194_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 289.0 (TID 93). 2664 bytes result sent to driver -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 289.0 (TID 92). 2664 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 289.0 (TID 93) in 9 ms on localhost (1/2) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 289.0 (TID 92) in 9 ms on localhost (2/2) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 289.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - ResultStage 289 (sum at PowerIterationClustering.scala:375) finished in 0.010 s -2016-06-13 19:11:06 INFO DAGScheduler:59 - Job 28 finished: sum at PowerIterationClustering.scala:375, took 0.026023 s -2016-06-13 19:11:06 INFO PowerIterationClustering:59 - Iteration 1: delta = 0.0. -2016-06-13 19:11:06 INFO PowerIterationClustering:59 - Iteration 1: diff(delta) = 0.010579646370791257. -2016-06-13 19:11:06 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Registering RDD 205 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Registering RDD 209 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Got job 29 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:11:06 INFO DAGScheduler:59 - Final stage: ResultStage 301(sum at PowerIterationClustering.scala:369) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 300, ShuffleMapStage 296, ShuffleMapStage 293, ShuffleMapStage 294, ShuffleMapStage 298) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 300) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 299 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[205] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(6288) called with curMem=203394, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 6.1 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=209682, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:65433 (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 299 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[205] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 299.0 with 2 tasks -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 299.0 (TID 94, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 299.0 (TID 95, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 299.0 (TID 95) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 299.0 (TID 94) -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_203_1 not found, computing it -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_203_0 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_188_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_188_0 locally -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=212645, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_203_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=214077, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_203_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_203_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_203_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 299.0 (TID 94). 2834 bytes result sent to driver -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 299.0 (TID 95). 2834 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 299.0 (TID 94) in 8 ms on localhost (1/2) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 299.0 (TID 95) in 9 ms on localhost (2/2) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 299.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - ShuffleMapStage 299 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.009 s -2016-06-13 19:11:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:06 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:06 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 300, ResultStage 301) -2016-06-13 19:11:06 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 300: List() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents for ResultStage 301: List(ShuffleMapStage 300) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 300 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[209] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=215509, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=220357, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:65433 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 300 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[209] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 300.0 with 2 tasks -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 300.0 (TID 96, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 300.0 (TID 97, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 300.0 (TID 97) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 300.0 (TID 96) -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_134_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_134_1 locally -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 300.0 (TID 97). 2470 bytes result sent to driver -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 300.0 (TID 96). 2470 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 300.0 (TID 97) in 19 ms on localhost (1/2) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 300.0 (TID 96) in 21 ms on localhost (2/2) -2016-06-13 19:11:06 INFO DAGScheduler:59 - ShuffleMapStage 300 (mapPartitions at GraphImpl.scala:235) finished in 0.021 s -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 300.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:06 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:06 INFO DAGScheduler:59 - waiting: Set(ResultStage 301) -2016-06-13 19:11:06 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents for ResultStage 301: List() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ResultStage 301 (MapPartitionsRDD[214] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(7224) called with curMem=222899, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 7.1 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(3233) called with curMem=230123, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:65433 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 301 (MapPartitionsRDD[214] at map at PowerIterationClustering.scala:369) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 301.0 with 2 tasks -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 301.0 (TID 98, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 301.0 (TID 99, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 301.0 (TID 98) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 301.0 (TID 99) -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_211_0 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_203_0 locally -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_211_1 not found, computing it -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_203_1 locally -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=233356, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_211_1 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=234836, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_211_0 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_211_1 in memory on localhost:65433 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_211_0 in memory on localhost:65433 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 301.0 (TID 99). 2880 bytes result sent to driver -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 301.0 (TID 98). 2880 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 301.0 (TID 99) in 12 ms on localhost (1/2) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 301.0 (TID 98) in 13 ms on localhost (2/2) -2016-06-13 19:11:06 INFO DAGScheduler:59 - ResultStage 301 (sum at PowerIterationClustering.scala:369) finished in 0.013 s -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 301.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - Job 29 finished: sum at PowerIterationClustering.scala:369, took 0.074772 s -2016-06-13 19:11:06 INFO PowerIterationClustering:59 - Iteration 2: norm(v) = 36.22334869431644. -2016-06-13 19:11:06 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 151 bytes -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 151 bytes -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 151 bytes -2016-06-13 19:11:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:11:06 INFO DAGScheduler:59 - Got job 30 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:11:06 INFO DAGScheduler:59 - Final stage: ResultStage 313(sum at PowerIterationClustering.scala:375) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 305, ShuffleMapStage 306, ShuffleMapStage 310, ShuffleMapStage 308, ShuffleMapStage 312) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ResultStage 313 (MapPartitionsRDD[225] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=236316, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 7.5 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(3360) called with curMem=243972, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:65433 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 313 (MapPartitionsRDD[225] at values at PowerIterationClustering.scala:375) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 313.0 with 2 tasks -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 313.0 (TID 100, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 313.0 (TID 101, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 313.0 (TID 100) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 313.0 (TID 101) -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_217_1 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_203_1 locally -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_217_0 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_211_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_203_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_211_0 locally -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=247332, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_217_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=248764, maxMem=766075207 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_217_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_217_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 313.0 (TID 101). 2664 bytes result sent to driver -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_217_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 313.0 (TID 100). 2664 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 313.0 (TID 101) in 7 ms on localhost (1/2) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 313.0 (TID 100) in 8 ms on localhost (2/2) -2016-06-13 19:11:06 INFO DAGScheduler:59 - ResultStage 313 (sum at PowerIterationClustering.scala:375) finished in 0.008 s -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 313.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - Job 30 finished: sum at PowerIterationClustering.scala:375, took 0.025303 s -2016-06-13 19:11:06 INFO PowerIterationClustering:59 - Iteration 2: delta = 0.0. -2016-06-13 19:11:06 INFO PowerIterationClustering:59 - Iteration 2: diff(delta) = 0.0. -2016-06-13 19:11:06 WARN KMeans:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:11:06 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Got job 31 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:11:06 INFO DAGScheduler:59 - Final stage: ResultStage 325(takeSample at KMeans.scala:376) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 320, ShuffleMapStage 317, ShuffleMapStage 324, ShuffleMapStage 318, ShuffleMapStage 322) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ResultStage 325 (MapPartitionsRDD[233] at map at KMeans.scala:210), which has no missing parents -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(7888) called with curMem=250196, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 7.7 KB, free 730.3 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(3424) called with curMem=258084, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.3 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:65433 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 325 (MapPartitionsRDD[233] at map at KMeans.scala:210) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 325.0 with 2 tasks -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 325.0 (TID 102, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 325.0 (TID 103, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 325.0 (TID 102) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 325.0 (TID 103) -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_228_0 not found, computing it -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_228_1 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_211_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_211_1 locally -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=261508, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_228_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=262940, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_228_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_228_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_228_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_231_0 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_231_1 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=264372, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_231_0 stored as values in memory (estimated size 240.0 B, free 730.3 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=264612, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_231_1 stored as values in memory (estimated size 240.0 B, free 730.3 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_231_0 in memory on localhost:65433 (size: 240.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_231_1 in memory on localhost:65433 (size: 240.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 325.0 (TID 102). 2717 bytes result sent to driver -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 325.0 (TID 103). 2717 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 325.0 (TID 102) in 14 ms on localhost (1/2) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 325.0 (TID 103) in 14 ms on localhost (2/2) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 325.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - ResultStage 325 (takeSample at KMeans.scala:376) finished in 0.015 s -2016-06-13 19:11:06 INFO DAGScheduler:59 - Job 31 finished: takeSample at KMeans.scala:376, took 0.029200 s -2016-06-13 19:11:06 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Got job 32 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:11:06 INFO DAGScheduler:59 - Final stage: ResultStage 337(takeSample at KMeans.scala:376) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 332, ShuffleMapStage 336, ShuffleMapStage 329, ShuffleMapStage 330, ShuffleMapStage 334) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ResultStage 337 (PartitionwiseSampledRDD[235] at takeSample at KMeans.scala:376), which has no missing parents -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(25280) called with curMem=264852, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 24.7 KB, free 730.3 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(10100) called with curMem=290132, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 9.9 KB, free 730.3 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:65433 (size: 9.9 KB, free: 730.4 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 337 (PartitionwiseSampledRDD[235] at takeSample at KMeans.scala:376) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 337.0 with 2 tasks -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 337.0 (TID 104, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 337.0 (TID 105, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 337.0 (TID 105) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 337.0 (TID 104) -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 337.0 (TID 105). 2575 bytes result sent to driver -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 337.0 (TID 105) in 8 ms on localhost (1/2) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 337.0 (TID 104). 2633 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 337.0 (TID 104) in 11 ms on localhost (2/2) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 337.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - ResultStage 337 (takeSample at KMeans.scala:376) finished in 0.012 s -2016-06-13 19:11:06 INFO DAGScheduler:59 - Job 32 finished: takeSample at KMeans.scala:376, took 0.036520 s -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=300232, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 880.0 B, free 730.3 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(487) called with curMem=301112, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 487.0 B, free 730.3 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:65433 (size: 487.0 B, free: 730.4 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 60 from broadcast at KMeans.scala:394 -2016-06-13 19:11:06 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Got job 33 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:11:06 INFO DAGScheduler:59 - Final stage: ResultStage 349(aggregate at KMeans.scala:402) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 342, ShuffleMapStage 344, ShuffleMapStage 341, ShuffleMapStage 348, ShuffleMapStage 346) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ResultStage 349 (MapPartitionsRDD[237] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(9912) called with curMem=301599, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 9.7 KB, free 730.3 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(4245) called with curMem=311511, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 4.1 KB, free 730.3 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:65433 (size: 4.1 KB, free: 730.4 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 61 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 349 (MapPartitionsRDD[237] at map at KMeans.scala:396) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 349.0 with 2 tasks -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 349.0 (TID 106, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 349.0 (TID 107, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 349.0 (TID 107) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 349.0 (TID 106) -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_237_0 not found, computing it -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_237_1 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=315756, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_237_0 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=316252, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_237_1 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_237_0 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_237_1 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 349.0 (TID 106). 2647 bytes result sent to driver -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 349.0 (TID 107). 2647 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 349.0 (TID 106) in 11 ms on localhost (1/2) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 349.0 (TID 107) in 11 ms on localhost (2/2) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 349.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - ResultStage 349 (aggregate at KMeans.scala:402) finished in 0.012 s -2016-06-13 19:11:06 INFO DAGScheduler:59 - Job 33 finished: aggregate at KMeans.scala:402, took 0.037260 s -2016-06-13 19:11:06 INFO MapPartitionsRDD:59 - Removing RDD 234 from persistence list -2016-06-13 19:11:06 INFO BlockManager:59 - Removing RDD 234 -2016-06-13 19:11:06 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Got job 34 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:11:06 INFO DAGScheduler:59 - Final stage: ResultStage 361(collect at KMeans.scala:431) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 356, ShuffleMapStage 353, ShuffleMapStage 354, ShuffleMapStage 358, ShuffleMapStage 360) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ResultStage 361 (MapPartitionsRDD[239] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(10072) called with curMem=316748, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 9.8 KB, free 730.3 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(4365) called with curMem=326820, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:65433 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 361 (MapPartitionsRDD[239] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 361.0 with 2 tasks -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 361.0 (TID 108, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 361.0 (TID 109, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 361.0 (TID 109) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 361.0 (TID 108) -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_237_1 locally -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 361.0 (TID 109). 3471 bytes result sent to driver -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_237_0 locally -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 361.0 (TID 108). 3118 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 361.0 (TID 109) in 7 ms on localhost (1/2) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 361.0 (TID 108) in 9 ms on localhost (2/2) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 361.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - ResultStage 361 (collect at KMeans.scala:431) finished in 0.009 s -2016-06-13 19:11:06 INFO DAGScheduler:59 - Job 34 finished: collect at KMeans.scala:431, took 0.025241 s -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1648) called with curMem=331185, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 1648.0 B, free 730.3 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(600) called with curMem=332833, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 600.0 B, free 730.3 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:65433 (size: 600.0 B, free: 730.4 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 63 from broadcast at KMeans.scala:394 -2016-06-13 19:11:06 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Got job 35 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:11:06 INFO DAGScheduler:59 - Final stage: ResultStage 373(aggregate at KMeans.scala:402) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 368, ShuffleMapStage 372, ShuffleMapStage 365, ShuffleMapStage 366, ShuffleMapStage 370) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ResultStage 373 (MapPartitionsRDD[241] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(10144) called with curMem=333433, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 9.9 KB, free 730.3 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(4338) called with curMem=343577, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.3 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:65433 (size: 4.2 KB, free: 730.4 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 373 (MapPartitionsRDD[241] at map at KMeans.scala:396) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 373.0 with 2 tasks -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 373.0 (TID 110, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 373.0 (TID 111, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 373.0 (TID 110) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 373.0 (TID 111) -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_241_0 not found, computing it -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_241_1 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_237_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_237_1 locally -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=347915, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_241_0 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=348411, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_241_1 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_241_0 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_241_1 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 373.0 (TID 110). 2647 bytes result sent to driver -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 373.0 (TID 111). 2647 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 373.0 (TID 110) in 9 ms on localhost (1/2) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 373.0 (TID 111) in 9 ms on localhost (2/2) -2016-06-13 19:11:06 INFO DAGScheduler:59 - ResultStage 373 (aggregate at KMeans.scala:402) finished in 0.010 s -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 373.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - Job 35 finished: aggregate at KMeans.scala:402, took 0.029784 s -2016-06-13 19:11:06 INFO MapPartitionsRDD:59 - Removing RDD 237 from persistence list -2016-06-13 19:11:06 INFO BlockManager:59 - Removing RDD 237 -2016-06-13 19:11:06 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Got job 36 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:11:06 INFO DAGScheduler:59 - Final stage: ResultStage 385(collect at KMeans.scala:431) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 380, ShuffleMapStage 377, ShuffleMapStage 384, ShuffleMapStage 378, ShuffleMapStage 382) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ResultStage 385 (MapPartitionsRDD[243] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(10304) called with curMem=347915, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 10.1 KB, free 730.2 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(4458) called with curMem=358219, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.2 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:65433 (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 385 (MapPartitionsRDD[243] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 385.0 with 2 tasks -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 385.0 (TID 112, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 385.0 (TID 113, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 385.0 (TID 113) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 385.0 (TID 112) -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_241_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_241_0 locally -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 385.0 (TID 112). 2990 bytes result sent to driver -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 385.0 (TID 113). 3240 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 385.0 (TID 112) in 5 ms on localhost (1/2) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 385.0 (TID 113) in 6 ms on localhost (2/2) -2016-06-13 19:11:06 INFO DAGScheduler:59 - ResultStage 385 (collect at KMeans.scala:431) finished in 0.006 s -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 385.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - Job 36 finished: collect at KMeans.scala:431, took 0.019123 s -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(1456) called with curMem=362677, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 1456.0 B, free 730.2 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(582) called with curMem=364133, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 582.0 B, free 730.2 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:65433 (size: 582.0 B, free: 730.4 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 66 from broadcast at KMeans.scala:394 -2016-06-13 19:11:06 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Got job 37 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:11:06 INFO DAGScheduler:59 - Final stage: ResultStage 397(aggregate at KMeans.scala:402) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 389, ShuffleMapStage 390, ShuffleMapStage 394, ShuffleMapStage 392, ShuffleMapStage 396) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ResultStage 397 (MapPartitionsRDD[245] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(10376) called with curMem=364715, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 10.1 KB, free 730.2 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(4388) called with curMem=375091, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.2 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:65433 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 67 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 397 (MapPartitionsRDD[245] at map at KMeans.scala:396) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 397.0 with 2 tasks -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 397.0 (TID 114, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 397.0 (TID 115, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 397.0 (TID 114) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 397.0 (TID 115) -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_245_0 not found, computing it -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_245_1 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_241_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_241_1 locally -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=379479, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_245_0 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=379975, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_245_1 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_245_0 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_245_1 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 397.0 (TID 114). 2647 bytes result sent to driver -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 397.0 (TID 115). 2647 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 397.0 (TID 114) in 9 ms on localhost (1/2) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 397.0 (TID 115) in 8 ms on localhost (2/2) -2016-06-13 19:11:06 INFO DAGScheduler:59 - ResultStage 397 (aggregate at KMeans.scala:402) finished in 0.009 s -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 397.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - Job 37 finished: aggregate at KMeans.scala:402, took 0.021978 s -2016-06-13 19:11:06 INFO MapPartitionsRDD:59 - Removing RDD 241 from persistence list -2016-06-13 19:11:06 INFO BlockManager:59 - Removing RDD 241 -2016-06-13 19:11:06 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Got job 38 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:11:06 INFO DAGScheduler:59 - Final stage: ResultStage 409(collect at KMeans.scala:431) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 404, ShuffleMapStage 401, ShuffleMapStage 408, ShuffleMapStage 406, ShuffleMapStage 402) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ResultStage 409 (MapPartitionsRDD[247] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(10536) called with curMem=379479, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 10.3 KB, free 730.2 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(4485) called with curMem=390015, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.2 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:65433 (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 409 (MapPartitionsRDD[247] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 409.0 with 2 tasks -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 409.0 (TID 116, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 409.0 (TID 117, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 409.0 (TID 116) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 409.0 (TID 117) -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_245_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 409.0 (TID 117). 2040 bytes result sent to driver -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_245_0 locally -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 409.0 (TID 117) in 4 ms on localhost (1/2) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 409.0 (TID 116). 2722 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 409.0 (TID 116) in 6 ms on localhost (2/2) -2016-06-13 19:11:06 INFO DAGScheduler:59 - ResultStage 409 (collect at KMeans.scala:431) finished in 0.006 s -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 409.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - Job 38 finished: collect at KMeans.scala:431, took 0.018202 s -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(648) called with curMem=394500, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 648.0 B, free 730.2 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(429) called with curMem=395148, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 429.0 B, free 730.2 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:65433 (size: 429.0 B, free: 730.4 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 69 from broadcast at KMeans.scala:394 -2016-06-13 19:11:06 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Got job 39 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:11:06 INFO DAGScheduler:59 - Final stage: ResultStage 421(aggregate at KMeans.scala:402) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 420, ShuffleMapStage 416, ShuffleMapStage 413, ShuffleMapStage 414, ShuffleMapStage 418) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ResultStage 421 (MapPartitionsRDD[249] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(10608) called with curMem=395577, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 10.4 KB, free 730.2 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(4443) called with curMem=406185, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.2 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:65433 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:65433 in memory (size: 1974.0 B, free: 730.4 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 421 (MapPartitionsRDD[249] at map at KMeans.scala:396) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 421.0 with 2 tasks -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 421.0 (TID 118, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 421.0 (TID 119, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 421.0 (TID 118) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 421.0 (TID 119) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_249_0 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_245_0 locally -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:65433 in memory (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=390201, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_249_0 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_249_1 not found, computing it -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_249_0 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_245_1 locally -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 421.0 (TID 118). 2647 bytes result sent to driver -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=380321, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_249_1 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_249_1 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 421.0 (TID 118) in 13 ms on localhost (1/2) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 421.0 (TID 119). 2647 bytes result sent to driver -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:65433 in memory (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-06-13 19:11:06 INFO DAGScheduler:59 - ResultStage 421 (aggregate at KMeans.scala:402) finished in 0.016 s -2016-06-13 19:11:06 INFO DAGScheduler:59 - Job 39 finished: aggregate at KMeans.scala:402, took 0.052296 s -2016-06-13 19:11:06 INFO MapPartitionsRDD:59 - Removing RDD 245 from persistence list -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 421.0 (TID 119) in 14 ms on localhost (2/2) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 421.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO BlockManager:59 - Removing RDD 245 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:65433 in memory (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:65433 in memory (size: 4.2 KB, free: 730.4 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:65433 in memory (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:65433 in memory (size: 4.1 KB, free: 730.4 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-06-13 19:11:06 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:65433 in memory (size: 9.9 KB, free: 730.4 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Got job 40 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:11:06 INFO DAGScheduler:59 - Final stage: ResultStage 433(collect at KMeans.scala:431) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 425, ShuffleMapStage 426, ShuffleMapStage 430, ShuffleMapStage 428, ShuffleMapStage 432) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:65433 in memory (size: 3.3 KB, free: 730.4 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ResultStage 433 (MapPartitionsRDD[251] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:65433 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(10768) called with curMem=259891, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 10.5 KB, free 730.3 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(4525) called with curMem=270659, maxMem=766075207 -2016-06-13 19:11:06 INFO BlockManager:59 - Removing RDD 217 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:65433 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 71 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned RDD 217 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 433 (MapPartitionsRDD[251] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 433.0 with 2 tasks -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 433.0 (TID 120, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 433.0 (TID 121, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 433.0 (TID 120) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 433.0 (TID 121) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:65433 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_249_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_249_0 locally -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 433.0 (TID 121). 2040 bytes result sent to driver -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 433.0 (TID 120). 2040 bytes result sent to driver -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:65433 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 433.0 (TID 121) in 5 ms on localhost (1/2) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 433.0 (TID 120) in 6 ms on localhost (2/2) -2016-06-13 19:11:06 INFO DAGScheduler:59 - ResultStage 433 (collect at KMeans.scala:431) finished in 0.007 s -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 433.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Job 40 finished: collect at KMeans.scala:431, took 0.021114 s -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(584) called with curMem=245222, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 584.0 B, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:65433 in memory (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(236) called with curMem=245806, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 236.0 B, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:65433 (size: 236.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 72 from broadcast at KMeans.scala:394 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:65433 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-06-13 19:11:06 INFO BlockManager:59 - Removing RDD 194 -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned RDD 194 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:65433 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:65433 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:65433 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Got job 41 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:11:06 INFO DAGScheduler:59 - Final stage: ResultStage 445(aggregate at KMeans.scala:402) -2016-06-13 19:11:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 440, ShuffleMapStage 437, ShuffleMapStage 444, ShuffleMapStage 438, ShuffleMapStage 442) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:65433 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-06-13 19:11:06 INFO BlockManager:59 - Removing RDD 171 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting ResultStage 445 (MapPartitionsRDD[253] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned RDD 171 -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(10848) called with curMem=191927, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 10.6 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:65433 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(4496) called with curMem=196831, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:65433 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO SparkContext:59 - Created broadcast 73 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 445 (MapPartitionsRDD[253] at map at KMeans.scala:396) -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Adding task set 445.0 with 2 tasks -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:65433 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 445.0 (TID 122, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 445.0 (TID 123, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:11:06 INFO Executor:59 - Running task 1.0 in stage 445.0 (TID 123) -2016-06-13 19:11:06 INFO Executor:59 - Running task 0.0 in stage 445.0 (TID 122) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:65433 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_253_1 not found, computing it -2016-06-13 19:11:06 INFO CacheManager:59 - Partition rdd_253_0 not found, computing it -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_249_1 locally -2016-06-13 19:11:06 INFO BlockManager:59 - Found block rdd_249_0 locally -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=179839, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_253_1 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:11:06 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=180335, maxMem=766075207 -2016-06-13 19:11:06 INFO MemoryStore:59 - Block rdd_253_0 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:65433 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_253_1 in memory on localhost:65433 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Added rdd_253_0 in memory on localhost:65433 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:11:06 INFO Executor:59 - Finished task 1.0 in stage 445.0 (TID 123). 2647 bytes result sent to driver -2016-06-13 19:11:06 INFO Executor:59 - Finished task 0.0 in stage 445.0 (TID 122). 2647 bytes result sent to driver -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 445.0 (TID 123) in 9 ms on localhost (1/2) -2016-06-13 19:11:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 445.0 (TID 122) in 10 ms on localhost (2/2) -2016-06-13 19:11:06 INFO DAGScheduler:59 - ResultStage 445 (aggregate at KMeans.scala:402) finished in 0.010 s -2016-06-13 19:11:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 445.0, whose tasks have all completed, from pool -2016-06-13 19:11:06 INFO DAGScheduler:59 - Job 41 finished: aggregate at KMeans.scala:402, took 0.027478 s -2016-06-13 19:11:06 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:65433 in memory (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:11:06 INFO MapPartitionsRDD:59 - Removing RDD 249 from persistence list -2016-06-13 19:11:06 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-06-13 19:11:06 INFO BlockManager:59 - Removing RDD 249 -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:65433 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Got job 42 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:11:07 INFO DAGScheduler:59 - Final stage: ResultStage 457(collect at KMeans.scala:431) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 452, ShuffleMapStage 456, ShuffleMapStage 449, ShuffleMapStage 450, ShuffleMapStage 454) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ResultStage 457 (MapPartitionsRDD[255] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=164696, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 10.8 KB, free 730.4 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(4575) called with curMem=175704, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.4 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:65433 (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 457 (MapPartitionsRDD[255] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 457.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 457.0 (TID 124, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 457.0 (TID 125, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 457.0 (TID 124) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 457.0 (TID 125) -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_253_0 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_253_1 locally -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 457.0 (TID 124). 2040 bytes result sent to driver -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 457.0 (TID 125). 2040 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 457.0 (TID 124) in 7 ms on localhost (1/2) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 457.0 (TID 125) in 7 ms on localhost (2/2) -2016-06-13 19:11:07 INFO DAGScheduler:59 - ResultStage 457 (collect at KMeans.scala:431) finished in 0.009 s -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 457.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - Job 42 finished: collect at KMeans.scala:431, took 0.025393 s -2016-06-13 19:11:07 INFO MapPartitionsRDD:59 - Removing RDD 253 from persistence list -2016-06-13 19:11:07 INFO BlockManager:59 - Removing RDD 253 -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(2880) called with curMem=180279, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 2.8 KB, free 730.4 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(708) called with curMem=182167, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 708.0 B, free 730.4 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:65433 (size: 708.0 B, free: 730.5 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 75 from broadcast at KMeans.scala:445 -2016-06-13 19:11:07 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:450 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Registering RDD 256 (flatMap at KMeans.scala:446) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Got job 43 (collectAsMap at KMeans.scala:450) with 2 output partitions -2016-06-13 19:11:07 INFO DAGScheduler:59 - Final stage: ResultStage 470(collectAsMap at KMeans.scala:450) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 469) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 469) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 469 (MapPartitionsRDD[256] at flatMap at KMeans.scala:446), which has no missing parents -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(9392) called with curMem=182875, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 9.2 KB, free 730.4 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(4070) called with curMem=192267, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:65433 (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 469 (MapPartitionsRDD[256] at flatMap at KMeans.scala:446) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 469.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 469.0 (TID 126, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 469.0 (TID 127, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 469.0 (TID 127) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 469.0 (TID 126) -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 469.0 (TID 126). 2254 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 469.0 (TID 126) in 15 ms on localhost (1/2) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 469.0 (TID 127). 2254 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 469.0 (TID 127) in 17 ms on localhost (2/2) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 469.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - ShuffleMapStage 469 (flatMap at KMeans.scala:446) finished in 0.018 s -2016-06-13 19:11:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:07 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:07 INFO DAGScheduler:59 - waiting: Set(ResultStage 470) -2016-06-13 19:11:07 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents for ResultStage 470: List() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ResultStage 470 (ShuffledRDD[257] at reduceByKey at KMeans.scala:450), which is now runnable -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=196337, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1382) called with curMem=198649, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 1382.0 B, free 730.4 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:65433 (size: 1382.0 B, free: 730.5 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 77 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 470 (ShuffledRDD[257] at reduceByKey at KMeans.scala:450) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 470.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 470.0 (TID 128, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 470.0 (TID 129, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 470.0 (TID 128) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 470.0 (TID 129) -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 470.0 (TID 128). 1852 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 470.0 (TID 128) in 7 ms on localhost (1/2) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 470.0 (TID 129). 1744 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 470.0 (TID 129) in 7 ms on localhost (2/2) -2016-06-13 19:11:07 INFO DAGScheduler:59 - ResultStage 470 (collectAsMap at KMeans.scala:450) finished in 0.009 s -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 470.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - Job 43 finished: collectAsMap at KMeans.scala:450, took 0.045610 s -2016-06-13 19:11:07 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:11:07 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:11:07 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:11:07 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:11:07 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:11:07 INFO KMeans:59 - Initialization with k-means|| took 0.585 seconds. -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(824) called with curMem=200031, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 824.0 B, free 730.4 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(339) called with curMem=200855, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 339.0 B, free 730.4 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:65433 (size: 339.0 B, free: 730.5 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 78 from broadcast at KMeans.scala:276 -2016-06-13 19:11:07 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:302 -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 151 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 151 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 151 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:11:07 INFO DAGScheduler:59 - Registering RDD 258 (mapPartitions at KMeans.scala:279) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Got job 44 (collectAsMap at KMeans.scala:302) with 2 output partitions -2016-06-13 19:11:07 INFO DAGScheduler:59 - Final stage: ResultStage 483(collectAsMap at KMeans.scala:302) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 482) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 482) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 482 (MapPartitionsRDD[258] at mapPartitions at KMeans.scala:279), which has no missing parents -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(10016) called with curMem=201194, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 9.8 KB, free 730.4 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(4410) called with curMem=211210, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:65433 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 79 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 482 (MapPartitionsRDD[258] at mapPartitions at KMeans.scala:279) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 482.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 482.0 (TID 130, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 482.0 (TID 131, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 482.0 (TID 131) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 482.0 (TID 130) -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 482.0 (TID 130). 2413 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 482.0 (TID 130) in 11 ms on localhost (1/2) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 482.0 (TID 131). 2413 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 482.0 (TID 131) in 91 ms on localhost (2/2) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 482.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - ShuffleMapStage 482 (mapPartitions at KMeans.scala:279) finished in 0.092 s -2016-06-13 19:11:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:07 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:07 INFO DAGScheduler:59 - waiting: Set(ResultStage 483) -2016-06-13 19:11:07 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents for ResultStage 483: List() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ResultStage 483 (ShuffledRDD[259] at reduceByKey at KMeans.scala:302), which is now runnable -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(2808) called with curMem=215620, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1589) called with curMem=218428, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 1589.0 B, free 730.4 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:65433 (size: 1589.0 B, free: 730.5 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 483 (ShuffledRDD[259] at reduceByKey at KMeans.scala:302) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 483.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 483.0 (TID 132, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 483.0 (TID 133, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 483.0 (TID 133) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 483.0 (TID 132) -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 483.0 (TID 133). 1671 bytes result sent to driver -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 483.0 (TID 132). 1785 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 483.0 (TID 133) in 8 ms on localhost (1/2) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 483.0 (TID 132) in 10 ms on localhost (2/2) -2016-06-13 19:11:07 INFO DAGScheduler:59 - ResultStage 483 (collectAsMap at KMeans.scala:302) finished in 0.010 s -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 483.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - Job 44 finished: collectAsMap at KMeans.scala:302, took 0.129079 s -2016-06-13 19:11:07 INFO KMeans:59 - Run 0 finished in 1 iterations -2016-06-13 19:11:07 INFO KMeans:59 - Run 1 finished in 1 iterations -2016-06-13 19:11:07 INFO KMeans:59 - Run 2 finished in 1 iterations -2016-06-13 19:11:07 INFO KMeans:59 - Run 3 finished in 1 iterations -2016-06-13 19:11:07 INFO KMeans:59 - Run 4 finished in 1 iterations -2016-06-13 19:11:07 INFO KMeans:59 - Iterations took 0.153 seconds. -2016-06-13 19:11:07 INFO KMeans:59 - KMeans converged in 1 iterations. -2016-06-13 19:11:07 INFO KMeans:59 - The cost for the best run is 6.650150218788335E-6. -2016-06-13 19:11:07 INFO MapPartitionsRDD:59 - Removing RDD 231 from persistence list -2016-06-13 19:11:07 INFO BlockManager:59 - Removing RDD 231 -2016-06-13 19:11:07 WARN KMeans:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:11:07 INFO SparkContext:59 - Starting job: collect at PowerIterationClusteringSuite.scala:66 -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 151 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 151 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 151 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:11:07 INFO DAGScheduler:59 - Got job 45 (collect at PowerIterationClusteringSuite.scala:66) with 2 output partitions -2016-06-13 19:11:07 INFO DAGScheduler:59 - Final stage: ResultStage 495(collect at PowerIterationClusteringSuite.scala:66) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 494, ShuffleMapStage 488, ShuffleMapStage 492, ShuffleMapStage 490, ShuffleMapStage 487) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ResultStage 495 (MapPartitionsRDD[262] at mapPartitions at PowerIterationClustering.scala:232), which has no missing parents -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(8192) called with curMem=219537, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 8.0 KB, free 730.4 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(3573) called with curMem=227729, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:65433 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 81 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 495 (MapPartitionsRDD[262] at mapPartitions at PowerIterationClustering.scala:232) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 495.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 495.0 (TID 134, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 495.0 (TID 135, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 495.0 (TID 135) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 495.0 (TID 134) -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_260_1 not found, computing it -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_260_0 not found, computing it -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=231302, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_260_1 stored as values in memory (estimated size 1168.0 B, free 730.4 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=232470, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_260_0 stored as values in memory (estimated size 1168.0 B, free 730.4 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_260_1 in memory on localhost:65433 (size: 1168.0 B, free: 730.5 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_260_0 in memory on localhost:65433 (size: 1168.0 B, free: 730.5 MB) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 495.0 (TID 135). 2919 bytes result sent to driver -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 495.0 (TID 134). 2919 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 495.0 (TID 135) in 7 ms on localhost (1/2) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 495.0 (TID 134) in 7 ms on localhost (2/2) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 495.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - ResultStage 495 (collect at PowerIterationClusteringSuite.scala:66) finished in 0.008 s -2016-06-13 19:11:07 INFO DAGScheduler:59 - Job 45 finished: collect at PowerIterationClusteringSuite.scala:66, took 0.022645 s -2016-06-13 19:11:07 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:325 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Registering RDD 273 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Registering RDD 267 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Got job 46 (sum at PowerIterationClustering.scala:325) with 2 output partitions -2016-06-13 19:11:07 INFO DAGScheduler:59 - Final stage: ResultStage 498(sum at PowerIterationClustering.scala:325) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 496, ShuffleMapStage 497) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 496, ShuffleMapStage 497) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 496 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[273] at mapPartitions at GraphImpl.scala:235), which has no missing parents -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(3136) called with curMem=233638, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 3.1 KB, free 730.4 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1868) called with curMem=236774, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 1868.0 B, free 730.4 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:65433 (size: 1868.0 B, free: 730.5 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 496 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[273] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 496.0 with 2 tasks -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 497 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[267] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 496.0 (TID 136, localhost, PROCESS_LOCAL, 2808 bytes) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(2720) called with curMem=238642, maxMem=766075207 -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 496.0 (TID 137, localhost, PROCESS_LOCAL, 2808 bytes) -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 496.0 (TID 137) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1642) called with curMem=241362, maxMem=766075207 -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_264_1 not found, computing it -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 1642.0 B, free 730.4 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(2944) called with curMem=243004, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_264_1 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 496.0 (TID 136) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:65433 (size: 1642.0 B, free: 730.5 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_264_1 in memory on localhost:65433 (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 83 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 497 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[267] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 497.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 497.0 (TID 138, localhost, PROCESS_LOCAL, 2808 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 497.0 (TID 139, localhost, PROCESS_LOCAL, 2808 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 497.0 (TID 138) -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_264_0 not found, computing it -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=245948, maxMem=766075207 -2016-06-13 19:11:07 INFO CacheManager:59 - Another thread is loading rdd_264_0, waiting for it to finish... -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 497.0 (TID 139) -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_264_0 stored as values in memory (estimated size 2.8 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_264_0 in memory on localhost:65433 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_264_1 locally -2016-06-13 19:11:07 INFO CacheManager:59 - Finished waiting for rdd_264_0 -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_264_0 locally -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 497.0 (TID 138). 1159 bytes result sent to driver -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 497.0 (TID 139). 2254 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 497.0 (TID 138) in 23 ms on localhost (1/2) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 497.0 (TID 139) in 24 ms on localhost (2/2) -2016-06-13 19:11:07 INFO DAGScheduler:59 - ShuffleMapStage 497 (mapPartitions at VertexRDD.scala:358) finished in 0.026 s -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 497.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:07 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 496) -2016-06-13 19:11:07 INFO DAGScheduler:59 - waiting: Set(ResultStage 498) -2016-06-13 19:11:07 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents for ResultStage 498: List(ShuffleMapStage 496) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 496.0 (TID 137). 1739 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 496.0 (TID 137) in 44 ms on localhost (1/2) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 496.0 (TID 136). 1739 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 496.0 (TID 136) in 80 ms on localhost (2/2) -2016-06-13 19:11:07 INFO DAGScheduler:59 - ShuffleMapStage 496 (mapPartitions at GraphImpl.scala:235) finished in 0.080 s -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 496.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:07 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:07 INFO DAGScheduler:59 - waiting: Set(ResultStage 498) -2016-06-13 19:11:07 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents for ResultStage 498: List() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ResultStage 498 (MapPartitionsRDD[285] at map at PowerIterationClustering.scala:325), which is now runnable -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(5136) called with curMem=248844, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 5.0 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(2652) called with curMem=253980, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:65433 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 498 (MapPartitionsRDD[285] at map at PowerIterationClustering.scala:325) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 498.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 498.0 (TID 140, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 498.0 (TID 141, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 498.0 (TID 140) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 498.0 (TID 141) -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_283_0 not found, computing it -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_283_1 not found, computing it -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_275_0 not found, computing it -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_275_1 not found, computing it -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_270_0 not found, computing it -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_270_1 not found, computing it -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1784) called with curMem=256632, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_270_0 stored as values in memory (estimated size 1784.0 B, free 730.3 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1792) called with curMem=258416, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_270_1 stored as values in memory (estimated size 1792.0 B, free 730.3 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_270_0 in memory on localhost:65433 (size: 1784.0 B, free: 730.5 MB) -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_270_1 in memory on localhost:65433 (size: 1792.0 B, free: 730.5 MB) -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=260208, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_275_0 stored as values in memory (estimated size 1832.0 B, free 730.3 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1840) called with curMem=262040, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_275_1 stored as values in memory (estimated size 1840.0 B, free 730.3 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_275_0 in memory on localhost:65433 (size: 1832.0 B, free: 730.5 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_275_1 in memory on localhost:65433 (size: 1840.0 B, free: 730.5 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(368) called with curMem=263880, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_283_0 stored as values in memory (estimated size 368.0 B, free 730.3 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(368) called with curMem=264248, maxMem=766075207 -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_283_0 in memory on localhost:65433 (size: 368.0 B, free: 730.5 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_283_1 stored as values in memory (estimated size 368.0 B, free 730.3 MB) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 498.0 (TID 140). 1895 bytes result sent to driver -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_283_1 in memory on localhost:65433 (size: 368.0 B, free: 730.5 MB) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 498.0 (TID 141). 1895 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 498.0 (TID 140) in 15 ms on localhost (1/2) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 498.0 (TID 141) in 15 ms on localhost (2/2) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 498.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - ResultStage 498 (sum at PowerIterationClustering.scala:325) finished in 0.016 s -2016-06-13 19:11:07 INFO DAGScheduler:59 - Job 46 finished: sum at PowerIterationClustering.scala:325, took 0.112248 s -2016-06-13 19:11:07 INFO SparkContext:59 - Starting job: reduce at VertexRDDImpl.scala:90 -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:11:07 INFO DAGScheduler:59 - Got job 47 (reduce at VertexRDDImpl.scala:90) with 2 output partitions -2016-06-13 19:11:07 INFO DAGScheduler:59 - Final stage: ResultStage 501(reduce at VertexRDDImpl.scala:90) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 499, ShuffleMapStage 500) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ResultStage 501 (MapPartitionsRDD[289] at map at VertexRDDImpl.scala:90), which has no missing parents -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(5288) called with curMem=264616, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 5.2 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(2680) called with curMem=269904, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:65433 (size: 2.6 KB, free: 730.4 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 85 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 501 (MapPartitionsRDD[289] at map at VertexRDDImpl.scala:90) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 501.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 501.0 (TID 142, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 501.0 (TID 143, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 501.0 (TID 142) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 501.0 (TID 143) -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_283_0 locally -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 501.0 (TID 142). 2161 bytes result sent to driver -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_283_1 locally -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 501.0 (TID 143). 2161 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 501.0 (TID 142) in 4 ms on localhost (1/2) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 501.0 (TID 143) in 4 ms on localhost (2/2) -2016-06-13 19:11:07 INFO DAGScheduler:59 - ResultStage 501 (reduce at VertexRDDImpl.scala:90) finished in 0.005 s -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 501.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - Job 47 finished: reduce at VertexRDDImpl.scala:90, took 0.011981 s -2016-06-13 19:11:07 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Registering RDD 277 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Registering RDD 290 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Registering RDD 294 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Got job 48 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:11:07 INFO DAGScheduler:59 - Final stage: ResultStage 509(sum at PowerIterationClustering.scala:369) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 506, ShuffleMapStage 508, ShuffleMapStage 505) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 508) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 504 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[277] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=272584, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 4.1 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(2374) called with curMem=276832, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:65433 (size: 2.3 KB, free: 730.4 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 504 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[277] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 504.0 with 2 tasks -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 507 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[290] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 504.0 (TID 144, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 504.0 (TID 145, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 504.0 (TID 144) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 504.0 (TID 145) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(4992) called with curMem=279206, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 4.9 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_275_1 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_275_0 locally -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(2606) called with curMem=284198, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:65433 (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 87 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 507 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[290] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 507.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 507.0 (TID 146, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 507.0 (TID 147, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 507.0 (TID 147) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 507.0 (TID 146) -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_287_1 not found, computing it -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_283_1 locally -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_287_0 not found, computing it -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_283_0 locally -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=286804, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_287_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=288236, maxMem=766075207 -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_287_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_287_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_287_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 507.0 (TID 147). 2834 bytes result sent to driver -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 507.0 (TID 146). 2834 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 507.0 (TID 146) in 15 ms on localhost (1/2) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 507.0 (TID 147) in 14 ms on localhost (2/2) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 507.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - ShuffleMapStage 507 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.017 s -2016-06-13 19:11:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:07 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 504) -2016-06-13 19:11:07 INFO DAGScheduler:59 - waiting: Set(ResultStage 509, ShuffleMapStage 508) -2016-06-13 19:11:07 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents for ResultStage 509: List(ShuffleMapStage 508) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 508: List(ShuffleMapStage 504) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 504.0 (TID 145). 2254 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 504.0 (TID 145) in 46 ms on localhost (1/2) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 504.0 (TID 144). 2254 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 504.0 (TID 144) in 52 ms on localhost (2/2) -2016-06-13 19:11:07 INFO DAGScheduler:59 - ShuffleMapStage 504 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.052 s -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 504.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:07 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:07 INFO DAGScheduler:59 - waiting: Set(ResultStage 509, ShuffleMapStage 508) -2016-06-13 19:11:07 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents for ResultStage 509: List(ShuffleMapStage 508) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 508: List() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 508 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[294] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(4552) called with curMem=289668, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(2431) called with curMem=294220, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:65433 (size: 2.4 KB, free: 730.4 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 508 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[294] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 508.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 508.0 (TID 148, localhost, PROCESS_LOCAL, 3098 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 508.0 (TID 149, localhost, PROCESS_LOCAL, 3098 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 508.0 (TID 148) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 508.0 (TID 149) -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_264_0 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_264_1 locally -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 508.0 (TID 149). 2470 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 508.0 (TID 149) in 19 ms on localhost (1/2) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 508.0 (TID 148). 2470 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 508.0 (TID 148) in 47 ms on localhost (2/2) -2016-06-13 19:11:07 INFO DAGScheduler:59 - ShuffleMapStage 508 (mapPartitions at GraphImpl.scala:235) finished in 0.047 s -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 508.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:07 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:07 INFO DAGScheduler:59 - waiting: Set(ResultStage 509) -2016-06-13 19:11:07 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents for ResultStage 509: List() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ResultStage 509 (MapPartitionsRDD[299] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(6024) called with curMem=296651, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 5.9 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(2877) called with curMem=302675, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:65433 (size: 2.8 KB, free: 730.4 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 89 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 509 (MapPartitionsRDD[299] at map at PowerIterationClustering.scala:369) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 509.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 509.0 (TID 150, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 509.0 (TID 151, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 509.0 (TID 150) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 509.0 (TID 151) -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_296_1 not found, computing it -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_287_1 locally -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_296_0 not found, computing it -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_287_0 locally -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=305552, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_296_1 stored as values in memory (estimated size 1480.0 B, free 730.3 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=307032, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_296_0 stored as values in memory (estimated size 1480.0 B, free 730.3 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_296_1 in memory on localhost:65433 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_296_0 in memory on localhost:65433 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 509.0 (TID 151). 2880 bytes result sent to driver -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 509.0 (TID 150). 2880 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 509.0 (TID 151) in 12 ms on localhost (1/2) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 509.0 (TID 150) in 14 ms on localhost (2/2) -2016-06-13 19:11:07 INFO DAGScheduler:59 - ResultStage 509 (sum at PowerIterationClustering.scala:369) finished in 0.018 s -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 509.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - Job 48 finished: sum at PowerIterationClustering.scala:369, took 0.155581 s -2016-06-13 19:11:07 INFO PowerIterationClustering:59 - Iteration 0: norm(v) = 34.22334869431644. -2016-06-13 19:11:07 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 151 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:11:07 INFO DAGScheduler:59 - Got job 49 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:11:07 INFO DAGScheduler:59 - Final stage: ResultStage 517(sum at PowerIterationClustering.scala:375) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 513, ShuffleMapStage 514, ShuffleMapStage 516) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ResultStage 517 (MapPartitionsRDD[310] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(6760) called with curMem=308512, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 6.6 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(3125) called with curMem=315272, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:65433 (size: 3.1 KB, free: 730.4 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 517 (MapPartitionsRDD[310] at values at PowerIterationClustering.scala:375) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 517.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 517.0 (TID 152, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 517.0 (TID 153, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 517.0 (TID 153) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 517.0 (TID 152) -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_302_1 not found, computing it -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_287_1 locally -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_302_0 not found, computing it -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_296_1 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_287_0 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_296_0 locally -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=318397, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_302_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=319829, maxMem=766075207 -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_302_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_302_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 517.0 (TID 153). 2664 bytes result sent to driver -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_302_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 517.0 (TID 152). 2664 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 517.0 (TID 153) in 6 ms on localhost (1/2) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 517.0 (TID 152) in 7 ms on localhost (2/2) -2016-06-13 19:11:07 INFO DAGScheduler:59 - ResultStage 517 (sum at PowerIterationClustering.scala:375) finished in 0.007 s -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 517.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - Job 49 finished: sum at PowerIterationClustering.scala:375, took 0.018933 s -2016-06-13 19:11:07 INFO PowerIterationClustering:59 - Iteration 0: delta = 1.3216876740804122. -2016-06-13 19:11:07 INFO PowerIterationClustering:59 - Iteration 0: diff(delta) = 1.7976931348623157E308. -2016-06-13 19:11:07 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Registering RDD 313 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Registering RDD 317 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Got job 50 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:11:07 INFO DAGScheduler:59 - Final stage: ResultStage 527(sum at PowerIterationClustering.scala:369) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 524, ShuffleMapStage 521, ShuffleMapStage 526, ShuffleMapStage 522) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 526) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 525 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[313] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(5984) called with curMem=321261, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 5.8 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(2872) called with curMem=327245, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:65433 (size: 2.8 KB, free: 730.4 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 91 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 525 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[313] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 525.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 525.0 (TID 154, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 525.0 (TID 155, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 525.0 (TID 155) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 525.0 (TID 154) -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_311_1 not found, computing it -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_296_1 locally -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_311_0 not found, computing it -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_296_0 locally -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=330117, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_311_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=331549, maxMem=766075207 -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_311_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_311_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_311_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 525.0 (TID 155). 2834 bytes result sent to driver -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 525.0 (TID 154). 2834 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 525.0 (TID 155) in 8 ms on localhost (1/2) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 525.0 (TID 154) in 10 ms on localhost (2/2) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 525.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - ShuffleMapStage 525 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.010 s -2016-06-13 19:11:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:07 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:07 INFO DAGScheduler:59 - waiting: Set(ResultStage 527, ShuffleMapStage 526) -2016-06-13 19:11:07 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents for ResultStage 527: List(ShuffleMapStage 526) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 526: List() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 526 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[317] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(4552) called with curMem=332981, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(2433) called with curMem=337533, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:65433 (size: 2.4 KB, free: 730.4 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 526 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[317] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 526.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 526.0 (TID 156, localhost, PROCESS_LOCAL, 3098 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 526.0 (TID 157, localhost, PROCESS_LOCAL, 3098 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 526.0 (TID 157) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 526.0 (TID 156) -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_264_0 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_264_1 locally -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 526.0 (TID 157). 2470 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 526.0 (TID 157) in 49 ms on localhost (1/2) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 526.0 (TID 156). 2470 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 526.0 (TID 156) in 54 ms on localhost (2/2) -2016-06-13 19:11:07 INFO DAGScheduler:59 - ShuffleMapStage 526 (mapPartitions at GraphImpl.scala:235) finished in 0.054 s -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 526.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:07 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:07 INFO DAGScheduler:59 - waiting: Set(ResultStage 527) -2016-06-13 19:11:07 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents for ResultStage 527: List() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ResultStage 527 (MapPartitionsRDD[322] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(6920) called with curMem=339966, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 6.8 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(3130) called with curMem=346886, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.3 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:65433 (size: 3.1 KB, free: 730.4 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 93 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 527 (MapPartitionsRDD[322] at map at PowerIterationClustering.scala:369) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 527.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 527.0 (TID 158, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 527.0 (TID 159, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 527.0 (TID 159) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 527.0 (TID 158) -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_319_0 not found, computing it -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_319_1 not found, computing it -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_311_0 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_311_1 locally -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=350016, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_319_1 stored as values in memory (estimated size 1480.0 B, free 730.3 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=351496, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_319_0 stored as values in memory (estimated size 1480.0 B, free 730.2 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_319_1 in memory on localhost:65433 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_319_0 in memory on localhost:65433 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 527.0 (TID 159). 2880 bytes result sent to driver -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 527.0 (TID 158). 2880 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 527.0 (TID 159) in 8 ms on localhost (1/2) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 527.0 (TID 158) in 9 ms on localhost (2/2) -2016-06-13 19:11:07 INFO DAGScheduler:59 - ResultStage 527 (sum at PowerIterationClustering.scala:369) finished in 0.009 s -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 527.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - Job 50 finished: sum at PowerIterationClustering.scala:369, took 0.099880 s -2016-06-13 19:11:07 INFO PowerIterationClustering:59 - Iteration 1: norm(v) = 34.22334869431644. -2016-06-13 19:11:07 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 151 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 151 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:11:07 INFO DAGScheduler:59 - Got job 51 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:11:07 INFO DAGScheduler:59 - Final stage: ResultStage 537(sum at PowerIterationClustering.scala:375) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 531, ShuffleMapStage 532, ShuffleMapStage 536, ShuffleMapStage 534) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ResultStage 537 (MapPartitionsRDD[333] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(7352) called with curMem=352976, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 7.2 KB, free 730.2 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(3304) called with curMem=360328, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.2 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:65433 (size: 3.2 KB, free: 730.4 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 537 (MapPartitionsRDD[333] at values at PowerIterationClustering.scala:375) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 537.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 537.0 (TID 160, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 537.0 (TID 161, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 537.0 (TID 161) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 537.0 (TID 160) -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_325_0 not found, computing it -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_325_1 not found, computing it -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_311_0 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_311_1 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_319_0 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_319_1 locally -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=363632, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_325_0 stored as values in memory (estimated size 1432.0 B, free 730.2 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=365064, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_325_1 stored as values in memory (estimated size 1432.0 B, free 730.2 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_325_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_325_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 537.0 (TID 160). 2664 bytes result sent to driver -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 537.0 (TID 161). 2664 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 537.0 (TID 160) in 6 ms on localhost (1/2) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 537.0 (TID 161) in 6 ms on localhost (2/2) -2016-06-13 19:11:07 INFO DAGScheduler:59 - ResultStage 537 (sum at PowerIterationClustering.scala:375) finished in 0.007 s -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 537.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - Job 51 finished: sum at PowerIterationClustering.scala:375, took 0.019542 s -2016-06-13 19:11:07 INFO PowerIterationClustering:59 - Iteration 1: delta = 0.0. -2016-06-13 19:11:07 INFO PowerIterationClustering:59 - Iteration 1: diff(delta) = 1.3216876740804122. -2016-06-13 19:11:07 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Registering RDD 336 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Registering RDD 340 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Got job 52 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:11:07 INFO DAGScheduler:59 - Final stage: ResultStage 549(sum at PowerIterationClustering.scala:369) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 542, ShuffleMapStage 546, ShuffleMapStage 544, ShuffleMapStage 548, ShuffleMapStage 541) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 548) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 547 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[336] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(6568) called with curMem=366496, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 6.4 KB, free 730.2 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(3034) called with curMem=373064, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:65433 (size: 3.0 KB, free: 730.4 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 95 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 547 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[336] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 547.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 547.0 (TID 162, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 547.0 (TID 163, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 547.0 (TID 163) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 547.0 (TID 162) -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_334_0 not found, computing it -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_334_1 not found, computing it -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_319_0 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_319_1 locally -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=376098, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_334_0 stored as values in memory (estimated size 1432.0 B, free 730.2 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=377530, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_334_1 stored as values in memory (estimated size 1432.0 B, free 730.2 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_334_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_334_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 547.0 (TID 162). 2834 bytes result sent to driver -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 547.0 (TID 163). 2834 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 547.0 (TID 162) in 7 ms on localhost (1/2) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 547.0 (TID 163) in 8 ms on localhost (2/2) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 547.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - ShuffleMapStage 547 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.008 s -2016-06-13 19:11:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:07 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:07 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 548, ResultStage 549) -2016-06-13 19:11:07 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 548: List() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents for ResultStage 549: List(ShuffleMapStage 548) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 548 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[340] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(4552) called with curMem=378962, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 4.4 KB, free 730.2 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(2432) called with curMem=383514, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:65433 (size: 2.4 KB, free: 730.4 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 548 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[340] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 548.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 548.0 (TID 164, localhost, PROCESS_LOCAL, 3098 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 548.0 (TID 165, localhost, PROCESS_LOCAL, 3098 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 548.0 (TID 164) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 548.0 (TID 165) -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_264_1 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_264_0 locally -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 548.0 (TID 164). 2470 bytes result sent to driver -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 548.0 (TID 165). 2470 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 548.0 (TID 164) in 31 ms on localhost (1/2) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 548.0 (TID 165) in 32 ms on localhost (2/2) -2016-06-13 19:11:07 INFO DAGScheduler:59 - ShuffleMapStage 548 (mapPartitions at GraphImpl.scala:235) finished in 0.033 s -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 548.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:07 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:07 INFO DAGScheduler:59 - waiting: Set(ResultStage 549) -2016-06-13 19:11:07 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents for ResultStage 549: List() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ResultStage 549 (MapPartitionsRDD[345] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(7504) called with curMem=385946, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 7.3 KB, free 730.2 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(3289) called with curMem=393450, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.2 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:65433 (size: 3.2 KB, free: 730.4 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 97 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 549 (MapPartitionsRDD[345] at map at PowerIterationClustering.scala:369) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 549.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 549.0 (TID 166, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 549.0 (TID 167, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 549.0 (TID 166) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 549.0 (TID 167) -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_342_1 not found, computing it -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_342_0 not found, computing it -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_334_1 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_334_0 locally -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=396739, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_342_1 stored as values in memory (estimated size 1480.0 B, free 730.2 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=398219, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_342_0 stored as values in memory (estimated size 1480.0 B, free 730.2 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_342_1 in memory on localhost:65433 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_342_0 in memory on localhost:65433 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 549.0 (TID 167). 2880 bytes result sent to driver -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 549.0 (TID 166). 2880 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 549.0 (TID 167) in 9 ms on localhost (1/2) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 549.0 (TID 166) in 9 ms on localhost (2/2) -2016-06-13 19:11:07 INFO DAGScheduler:59 - ResultStage 549 (sum at PowerIterationClustering.scala:369) finished in 0.009 s -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 549.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - Job 52 finished: sum at PowerIterationClustering.scala:369, took 0.080402 s -2016-06-13 19:11:07 INFO PowerIterationClustering:59 - Iteration 2: norm(v) = 34.22334869431644. -2016-06-13 19:11:07 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 151 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 151 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 151 bytes -2016-06-13 19:11:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:11:07 INFO DAGScheduler:59 - Got job 53 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:11:07 INFO DAGScheduler:59 - Final stage: ResultStage 561(sum at PowerIterationClustering.scala:375) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 556, ShuffleMapStage 553, ShuffleMapStage 560, ShuffleMapStage 558, ShuffleMapStage 554) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ResultStage 561 (MapPartitionsRDD[356] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(7936) called with curMem=399699, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 7.8 KB, free 730.2 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(3435) called with curMem=407635, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:65433 (size: 3.4 KB, free: 730.4 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 561 (MapPartitionsRDD[356] at values at PowerIterationClustering.scala:375) -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Adding task set 561.0 with 2 tasks -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 561.0 (TID 168, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 561.0 (TID 169, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:11:07 INFO Executor:59 - Running task 0.0 in stage 561.0 (TID 168) -2016-06-13 19:11:07 INFO Executor:59 - Running task 1.0 in stage 561.0 (TID 169) -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_348_1 not found, computing it -2016-06-13 19:11:07 INFO CacheManager:59 - Partition rdd_348_0 not found, computing it -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_334_1 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_334_0 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_342_1 locally -2016-06-13 19:11:07 INFO BlockManager:59 - Found block rdd_342_0 locally -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=411070, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_348_1 stored as values in memory (estimated size 1432.0 B, free 730.2 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=412502, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block rdd_348_0 stored as values in memory (estimated size 1432.0 B, free 730.2 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_348_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added rdd_348_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:07 INFO Executor:59 - Finished task 1.0 in stage 561.0 (TID 169). 2664 bytes result sent to driver -2016-06-13 19:11:07 INFO Executor:59 - Finished task 0.0 in stage 561.0 (TID 168). 2664 bytes result sent to driver -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 561.0 (TID 169) in 5 ms on localhost (1/2) -2016-06-13 19:11:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 561.0 (TID 168) in 6 ms on localhost (2/2) -2016-06-13 19:11:07 INFO DAGScheduler:59 - ResultStage 561 (sum at PowerIterationClustering.scala:375) finished in 0.006 s -2016-06-13 19:11:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 561.0, whose tasks have all completed, from pool -2016-06-13 19:11:07 INFO DAGScheduler:59 - Job 53 finished: sum at PowerIterationClustering.scala:375, took 0.022595 s -2016-06-13 19:11:07 INFO PowerIterationClustering:59 - Iteration 2: delta = 0.0. -2016-06-13 19:11:07 INFO PowerIterationClustering:59 - Iteration 2: diff(delta) = 0.0. -2016-06-13 19:11:07 WARN KMeans:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:11:07 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Got job 54 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:11:07 INFO DAGScheduler:59 - Final stage: ResultStage 573(takeSample at KMeans.scala:376) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 572, ShuffleMapStage 568, ShuffleMapStage 565, ShuffleMapStage 566, ShuffleMapStage 570) -2016-06-13 19:11:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting ResultStage 573 (MapPartitionsRDD[364] at map at KMeans.scala:210), which has no missing parents -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(8168) called with curMem=413934, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 8.0 KB, free 730.2 MB) -2016-06-13 19:11:07 INFO MemoryStore:59 - ensureFreeSpace(3470) called with curMem=422102, maxMem=766075207 -2016-06-13 19:11:07 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-06-13 19:11:07 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:65433 (size: 3.4 KB, free: 730.4 MB) -2016-06-13 19:11:07 INFO SparkContext:59 - Created broadcast 99 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 573 (MapPartitionsRDD[364] at map at KMeans.scala:210) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 573.0 with 2 tasks -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 573.0 (TID 170, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 573.0 (TID 171, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 573.0 (TID 170) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 573.0 (TID 171) -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_359_0 not found, computing it -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_359_1 not found, computing it -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_342_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_342_1 locally -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=425572, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_359_0 stored as values in memory (estimated size 1432.0 B, free 730.2 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=427004, maxMem=766075207 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_359_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_359_1 stored as values in memory (estimated size 1432.0 B, free 730.2 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_359_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_362_0 not found, computing it -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_362_1 not found, computing it -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=428436, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_362_1 stored as values in memory (estimated size 240.0 B, free 730.2 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=428676, maxMem=766075207 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_362_1 in memory on localhost:65433 (size: 240.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_362_0 stored as values in memory (estimated size 240.0 B, free 730.2 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_362_0 in memory on localhost:65433 (size: 240.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 573.0 (TID 171). 2717 bytes result sent to driver -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 573.0 (TID 170). 2717 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 573.0 (TID 171) in 10 ms on localhost (1/2) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 573.0 (TID 170) in 12 ms on localhost (2/2) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 573.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - ResultStage 573 (takeSample at KMeans.scala:376) finished in 0.012 s -2016-06-13 19:11:08 INFO DAGScheduler:59 - Job 54 finished: takeSample at KMeans.scala:376, took 0.028176 s -2016-06-13 19:11:08 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Got job 55 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:11:08 INFO DAGScheduler:59 - Final stage: ResultStage 585(takeSample at KMeans.scala:376) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 578, ShuffleMapStage 582, ShuffleMapStage 580, ShuffleMapStage 584, ShuffleMapStage 577) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ResultStage 585 (PartitionwiseSampledRDD[366] at takeSample at KMeans.scala:376), which has no missing parents -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(25560) called with curMem=428916, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 25.0 KB, free 730.2 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(10148) called with curMem=454476, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 9.9 KB, free 730.1 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:65433 (size: 9.9 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 585 (PartitionwiseSampledRDD[366] at takeSample at KMeans.scala:376) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 585.0 with 2 tasks -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 585.0 (TID 172, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 585.0 (TID 173, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 585.0 (TID 172) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 585.0 (TID 173) -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 585.0 (TID 172). 2633 bytes result sent to driver -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 585.0 (TID 173). 2575 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 585.0 (TID 172) in 5 ms on localhost (1/2) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 585.0 (TID 173) in 5 ms on localhost (2/2) -2016-06-13 19:11:08 INFO DAGScheduler:59 - ResultStage 585 (takeSample at KMeans.scala:376) finished in 0.005 s -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 585.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - Job 55 finished: takeSample at KMeans.scala:376, took 0.016324 s -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=464624, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 880.0 B, free 730.1 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(493) called with curMem=465504, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 493.0 B, free 730.1 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:65433 (size: 493.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 101 from broadcast at KMeans.scala:394 -2016-06-13 19:11:08 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Got job 56 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:11:08 INFO DAGScheduler:59 - Final stage: ResultStage 597(aggregate at KMeans.scala:402) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 592, ShuffleMapStage 589, ShuffleMapStage 596, ShuffleMapStage 590, ShuffleMapStage 594) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ResultStage 597 (MapPartitionsRDD[368] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(10192) called with curMem=465997, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 10.0 KB, free 730.1 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(4270) called with curMem=476189, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.1 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:65433 (size: 4.2 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 597 (MapPartitionsRDD[368] at map at KMeans.scala:396) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 597.0 with 2 tasks -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 597.0 (TID 174, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 597.0 (TID 175, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 597.0 (TID 174) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 597.0 (TID 175) -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_368_0 not found, computing it -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_368_1 not found, computing it -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=480459, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_368_0 stored as values in memory (estimated size 496.0 B, free 730.1 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=480955, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_368_1 stored as values in memory (estimated size 496.0 B, free 730.1 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_368_0 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_368_1 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 597.0 (TID 174). 2647 bytes result sent to driver -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 597.0 (TID 175). 2647 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 597.0 (TID 174) in 8 ms on localhost (1/2) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 597.0 (TID 175) in 7 ms on localhost (2/2) -2016-06-13 19:11:08 INFO DAGScheduler:59 - ResultStage 597 (aggregate at KMeans.scala:402) finished in 0.008 s -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 597.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - Job 56 finished: aggregate at KMeans.scala:402, took 0.022538 s -2016-06-13 19:11:08 INFO MapPartitionsRDD:59 - Removing RDD 365 from persistence list -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 365 -2016-06-13 19:11:08 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Got job 57 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:11:08 INFO DAGScheduler:59 - Final stage: ResultStage 609(collect at KMeans.scala:431) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 604, ShuffleMapStage 608, ShuffleMapStage 601, ShuffleMapStage 602, ShuffleMapStage 606) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ResultStage 609 (MapPartitionsRDD[370] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(10240) called with curMem=481451, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 10.0 KB, free 730.1 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(4347) called with curMem=491691, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.1 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:65433 (size: 4.2 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 103 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 609 (MapPartitionsRDD[370] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 609.0 with 2 tasks -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 609.0 (TID 176, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 609.0 (TID 177, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 609.0 (TID 176) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 609.0 (TID 177) -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_368_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_368_0 locally -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 609.0 (TID 176). 3244 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 609.0 (TID 176) in 5 ms on localhost (1/2) -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 609.0 (TID 177). 3471 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 609.0 (TID 177) in 6 ms on localhost (2/2) -2016-06-13 19:11:08 INFO DAGScheduler:59 - ResultStage 609 (collect at KMeans.scala:431) finished in 0.006 s -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 609.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - Job 57 finished: collect at KMeans.scala:431, took 0.019231 s -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(1792) called with curMem=496038, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 1792.0 B, free 730.1 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(636) called with curMem=497830, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 636.0 B, free 730.1 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:65433 (size: 636.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 104 from broadcast at KMeans.scala:394 -2016-06-13 19:11:08 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Got job 58 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:11:08 INFO DAGScheduler:59 - Final stage: ResultStage 621(aggregate at KMeans.scala:402) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 614, ShuffleMapStage 616, ShuffleMapStage 613, ShuffleMapStage 620, ShuffleMapStage 618) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ResultStage 621 (MapPartitionsRDD[372] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(10424) called with curMem=498466, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 10.2 KB, free 730.1 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(4357) called with curMem=508890, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.1 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:65433 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 105 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 621 (MapPartitionsRDD[372] at map at KMeans.scala:396) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 621.0 with 2 tasks -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 621.0 (TID 178, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 621.0 (TID 179, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 621.0 (TID 178) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 621.0 (TID 179) -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_372_0 not found, computing it -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_372_1 not found, computing it -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_368_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_368_1 locally -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=513247, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_372_0 stored as values in memory (estimated size 496.0 B, free 730.1 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=513743, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_372_1 stored as values in memory (estimated size 496.0 B, free 730.1 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_372_0 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_372_1 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 621.0 (TID 178). 2647 bytes result sent to driver -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 621.0 (TID 179). 2647 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 621.0 (TID 178) in 8 ms on localhost (1/2) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 621.0 (TID 179) in 8 ms on localhost (2/2) -2016-06-13 19:11:08 INFO DAGScheduler:59 - ResultStage 621 (aggregate at KMeans.scala:402) finished in 0.008 s -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 621.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - Job 58 finished: aggregate at KMeans.scala:402, took 0.020513 s -2016-06-13 19:11:08 INFO MapPartitionsRDD:59 - Removing RDD 368 from persistence list -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 368 -2016-06-13 19:11:08 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Got job 59 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:11:08 INFO DAGScheduler:59 - Final stage: ResultStage 633(collect at KMeans.scala:431) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 628, ShuffleMapStage 625, ShuffleMapStage 626, ShuffleMapStage 630, ShuffleMapStage 632) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ResultStage 633 (MapPartitionsRDD[374] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(10480) called with curMem=513247, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 10.2 KB, free 730.1 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(4442) called with curMem=523727, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.1 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:65433 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 106 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 633 (MapPartitionsRDD[374] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 633.0 with 2 tasks -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 633.0 (TID 180, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 633.0 (TID 181, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 633.0 (TID 180) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 633.0 (TID 181) -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_372_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_372_0 locally -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 633.0 (TID 181). 3114 bytes result sent to driver -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 633.0 (TID 180). 2990 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 633.0 (TID 181) in 7 ms on localhost (1/2) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 633.0 (TID 180) in 7 ms on localhost (2/2) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 633.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - ResultStage 633 (collect at KMeans.scala:431) finished in 0.007 s -2016-06-13 19:11:08 INFO DAGScheduler:59 - Job 59 finished: collect at KMeans.scala:431, took 0.022397 s -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(1312) called with curMem=528169, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 1312.0 B, free 730.1 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(579) called with curMem=529481, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 579.0 B, free 730.1 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:65433 (size: 579.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 107 from broadcast at KMeans.scala:394 -2016-06-13 19:11:08 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Got job 60 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:11:08 INFO DAGScheduler:59 - Final stage: ResultStage 645(aggregate at KMeans.scala:402) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 640, ShuffleMapStage 644, ShuffleMapStage 637, ShuffleMapStage 638, ShuffleMapStage 642) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ResultStage 645 (MapPartitionsRDD[376] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(10656) called with curMem=530060, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 10.4 KB, free 730.1 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(4410) called with curMem=540716, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.1 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:65433 (size: 4.3 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 108 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 645 (MapPartitionsRDD[376] at map at KMeans.scala:396) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 645.0 with 2 tasks -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 645.0 (TID 182, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 645.0 (TID 183, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 645.0 (TID 182) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 645.0 (TID 183) -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_376_0 not found, computing it -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_376_1 not found, computing it -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_372_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_372_1 locally -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=545126, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_376_0 stored as values in memory (estimated size 496.0 B, free 730.1 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=545622, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_376_1 stored as values in memory (estimated size 496.0 B, free 730.1 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_376_0 in memory on localhost:65433 (size: 496.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_376_1 in memory on localhost:65433 (size: 496.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 645.0 (TID 182). 2647 bytes result sent to driver -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 645.0 (TID 183). 2647 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 645.0 (TID 182) in 11 ms on localhost (1/2) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 645.0 (TID 183) in 13 ms on localhost (2/2) -2016-06-13 19:11:08 INFO DAGScheduler:59 - ResultStage 645 (aggregate at KMeans.scala:402) finished in 0.014 s -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 645.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - Job 60 finished: aggregate at KMeans.scala:402, took 0.029938 s -2016-06-13 19:11:08 INFO MapPartitionsRDD:59 - Removing RDD 372 from persistence list -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 372 -2016-06-13 19:11:08 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Got job 61 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:11:08 INFO DAGScheduler:59 - Final stage: ResultStage 657(collect at KMeans.scala:431) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 652, ShuffleMapStage 649, ShuffleMapStage 656, ShuffleMapStage 650, ShuffleMapStage 654) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ResultStage 657 (MapPartitionsRDD[378] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(10712) called with curMem=545126, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 10.5 KB, free 730.1 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(4479) called with curMem=555838, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.1 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:65433 (size: 4.4 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 109 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 657 (MapPartitionsRDD[378] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 657.0 with 2 tasks -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 657.0 (TID 184, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 657.0 (TID 185, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 657.0 (TID 185) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 657.0 (TID 184) -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_376_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_376_1 locally -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 657.0 (TID 185). 2722 bytes result sent to driver -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 657.0 (TID 184). 2040 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 657.0 (TID 185) in 6 ms on localhost (1/2) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 657.0 (TID 184) in 7 ms on localhost (2/2) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 657.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - ResultStage 657 (collect at KMeans.scala:431) finished in 0.007 s -2016-06-13 19:11:08 INFO DAGScheduler:59 - Job 61 finished: collect at KMeans.scala:431, took 0.021118 s -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(648) called with curMem=560317, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 648.0 B, free 730.1 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(429) called with curMem=560965, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 429.0 B, free 730.1 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:65433 (size: 429.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 110 from broadcast at KMeans.scala:394 -2016-06-13 19:11:08 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Got job 62 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:11:08 INFO DAGScheduler:59 - Final stage: ResultStage 669(aggregate at KMeans.scala:402) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 661, ShuffleMapStage 662, ShuffleMapStage 666, ShuffleMapStage 664, ShuffleMapStage 668) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ResultStage 669 (MapPartitionsRDD[380] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(10888) called with curMem=561394, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 10.6 KB, free 730.0 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(4476) called with curMem=572282, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.0 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:65433 (size: 4.4 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 111 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 669 (MapPartitionsRDD[380] at map at KMeans.scala:396) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 669.0 with 2 tasks -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 669.0 (TID 186, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 669.0 (TID 187, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 669.0 (TID 186) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 669.0 (TID 187) -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_380_1 not found, computing it -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_380_0 not found, computing it -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_376_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_376_0 locally -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=576758, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_380_1 stored as values in memory (estimated size 496.0 B, free 730.0 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=577254, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_380_0 stored as values in memory (estimated size 496.0 B, free 730.0 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_380_1 in memory on localhost:65433 (size: 496.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_380_0 in memory on localhost:65433 (size: 496.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 669.0 (TID 187). 2647 bytes result sent to driver -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 669.0 (TID 186). 2647 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 669.0 (TID 187) in 7 ms on localhost (1/2) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 669.0 (TID 186) in 8 ms on localhost (2/2) -2016-06-13 19:11:08 INFO DAGScheduler:59 - ResultStage 669 (aggregate at KMeans.scala:402) finished in 0.008 s -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 669.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - Job 62 finished: aggregate at KMeans.scala:402, took 0.020755 s -2016-06-13 19:11:08 INFO MapPartitionsRDD:59 - Removing RDD 376 from persistence list -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 376 -2016-06-13 19:11:08 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Got job 63 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:11:08 INFO DAGScheduler:59 - Final stage: ResultStage 681(collect at KMeans.scala:431) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 676, ShuffleMapStage 673, ShuffleMapStage 680, ShuffleMapStage 678, ShuffleMapStage 674) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ResultStage 681 (MapPartitionsRDD[382] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(10944) called with curMem=576758, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 10.7 KB, free 730.0 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(4537) called with curMem=587702, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.0 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:65433 (size: 4.4 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 112 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 681 (MapPartitionsRDD[382] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 681.0 with 2 tasks -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 681.0 (TID 188, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 681.0 (TID 189, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 681.0 (TID 189) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 681.0 (TID 188) -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_380_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_380_1 locally -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 681.0 (TID 188). 2722 bytes result sent to driver -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 681.0 (TID 189). 2040 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 681.0 (TID 188) in 4 ms on localhost (1/2) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 681.0 (TID 189) in 5 ms on localhost (2/2) -2016-06-13 19:11:08 INFO DAGScheduler:59 - ResultStage 681 (collect at KMeans.scala:431) finished in 0.005 s -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 681.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - Job 63 finished: collect at KMeans.scala:431, took 0.020930 s -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(648) called with curMem=592239, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 648.0 B, free 730.0 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(429) called with curMem=592887, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 429.0 B, free 730.0 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:65433 (size: 429.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 113 from broadcast at KMeans.scala:394 -2016-06-13 19:11:08 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Got job 64 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:11:08 INFO DAGScheduler:59 - Final stage: ResultStage 693(aggregate at KMeans.scala:402) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 692, ShuffleMapStage 688, ShuffleMapStage 685, ShuffleMapStage 686, ShuffleMapStage 690) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ResultStage 693 (MapPartitionsRDD[384] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(11128) called with curMem=593316, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 10.9 KB, free 730.0 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(4537) called with curMem=604444, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.0 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:65433 (size: 4.4 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 114 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 693 (MapPartitionsRDD[384] at map at KMeans.scala:396) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 693.0 with 2 tasks -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 693.0 (TID 190, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 693.0 (TID 191, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 693.0 (TID 190) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 693.0 (TID 191) -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_384_0 not found, computing it -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_384_1 not found, computing it -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_380_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_380_1 locally -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=608981, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_384_0 stored as values in memory (estimated size 496.0 B, free 730.0 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=609477, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_384_1 stored as values in memory (estimated size 496.0 B, free 730.0 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_384_0 in memory on localhost:65433 (size: 496.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_384_1 in memory on localhost:65433 (size: 496.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 693.0 (TID 190). 2647 bytes result sent to driver -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 693.0 (TID 191). 2647 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 693.0 (TID 190) in 8 ms on localhost (1/2) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 693.0 (TID 191) in 8 ms on localhost (2/2) -2016-06-13 19:11:08 INFO DAGScheduler:59 - ResultStage 693 (aggregate at KMeans.scala:402) finished in 0.009 s -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 693.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - Job 64 finished: aggregate at KMeans.scala:402, took 0.021794 s -2016-06-13 19:11:08 INFO MapPartitionsRDD:59 - Removing RDD 380 from persistence list -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 380 -2016-06-13 19:11:08 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Got job 65 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:11:08 INFO DAGScheduler:59 - Final stage: ResultStage 705(collect at KMeans.scala:431) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 698, ShuffleMapStage 702, ShuffleMapStage 700, ShuffleMapStage 697, ShuffleMapStage 704) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ResultStage 705 (MapPartitionsRDD[386] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(11176) called with curMem=608981, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_115 stored as values in memory (estimated size 10.9 KB, free 730.0 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(4569) called with curMem=620157, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_115_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.0 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_115_piece0 in memory on localhost:65433 (size: 4.5 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 115 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 705 (MapPartitionsRDD[386] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 705.0 with 2 tasks -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 705.0 (TID 192, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 705.0 (TID 193, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 705.0 (TID 193) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 705.0 (TID 192) -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_384_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_384_1 locally -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 705.0 (TID 192). 2040 bytes result sent to driver -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 705.0 (TID 193). 2040 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 705.0 (TID 192) in 4 ms on localhost (1/2) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 705.0 (TID 193) in 4 ms on localhost (2/2) -2016-06-13 19:11:08 INFO DAGScheduler:59 - ResultStage 705 (collect at KMeans.scala:431) finished in 0.004 s -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 705.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - Job 65 finished: collect at KMeans.scala:431, took 0.017790 s -2016-06-13 19:11:08 INFO MapPartitionsRDD:59 - Removing RDD 384 from persistence list -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 384 -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(2944) called with curMem=624726, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_116 stored as values in memory (estimated size 2.9 KB, free 730.0 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(787) called with curMem=626678, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_116_piece0 stored as bytes in memory (estimated size 787.0 B, free 730.0 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_116_piece0 in memory on localhost:65433 (size: 787.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 116 from broadcast at KMeans.scala:445 -2016-06-13 19:11:08 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:450 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Registering RDD 387 (flatMap at KMeans.scala:446) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Got job 66 (collectAsMap at KMeans.scala:450) with 2 output partitions -2016-06-13 19:11:08 INFO DAGScheduler:59 - Final stage: ResultStage 718(collectAsMap at KMeans.scala:450) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 717) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 717) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ShuffleMapStage 717 (MapPartitionsRDD[387] at flatMap at KMeans.scala:446), which has no missing parents -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(9672) called with curMem=627465, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_117 stored as values in memory (estimated size 9.4 KB, free 730.0 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(4104) called with curMem=637137, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_117_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.0 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_117_piece0 in memory on localhost:65433 (size: 4.0 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 117 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 717 (MapPartitionsRDD[387] at flatMap at KMeans.scala:446) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 717.0 with 2 tasks -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 717.0 (TID 194, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 717.0 (TID 195, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 717.0 (TID 194) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 717.0 (TID 195) -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 717.0 (TID 195). 2254 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 717.0 (TID 195) in 14 ms on localhost (1/2) -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 717.0 (TID 194). 2254 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 717.0 (TID 194) in 25 ms on localhost (2/2) -2016-06-13 19:11:08 INFO DAGScheduler:59 - ShuffleMapStage 717 (flatMap at KMeans.scala:446) finished in 0.025 s -2016-06-13 19:11:08 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:08 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:08 INFO DAGScheduler:59 - waiting: Set(ResultStage 718) -2016-06-13 19:11:08 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 717.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents for ResultStage 718: List() -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ResultStage 718 (ShuffledRDD[388] at reduceByKey at KMeans.scala:450), which is now runnable -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=641241, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_118 stored as values in memory (estimated size 2.3 KB, free 730.0 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(1383) called with curMem=643553, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_118_piece0 stored as bytes in memory (estimated size 1383.0 B, free 730.0 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_118_piece0 in memory on localhost:65433 (size: 1383.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 118 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 718 (ShuffledRDD[388] at reduceByKey at KMeans.scala:450) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 718.0 with 2 tasks -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 718.0 (TID 196, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 718.0 (TID 197, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 718.0 (TID 197) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 718.0 (TID 196) -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 718.0 (TID 197). 2032 bytes result sent to driver -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 718.0 (TID 196). 1924 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 718.0 (TID 197) in 7 ms on localhost (1/2) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 718.0 (TID 196) in 7 ms on localhost (2/2) -2016-06-13 19:11:08 INFO DAGScheduler:59 - ResultStage 718 (collectAsMap at KMeans.scala:450) finished in 0.008 s -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 718.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - Job 66 finished: collectAsMap at KMeans.scala:450, took 0.052841 s -2016-06-13 19:11:08 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:11:08 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:11:08 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:11:08 INFO LocalKMeans:59 - Local KMeans++ converged in 3 iterations. -2016-06-13 19:11:08 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:11:08 INFO KMeans:59 - Initialization with k-means|| took 0.497 seconds. -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(824) called with curMem=644936, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_119 stored as values in memory (estimated size 824.0 B, free 730.0 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(363) called with curMem=645760, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_119_piece0 stored as bytes in memory (estimated size 363.0 B, free 730.0 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_119_piece0 in memory on localhost:65433 (size: 363.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 119 from broadcast at KMeans.scala:276 -2016-06-13 19:11:08 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:302 -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 151 bytes -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 151 bytes -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 151 bytes -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:11:08 INFO DAGScheduler:59 - Registering RDD 389 (mapPartitions at KMeans.scala:279) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Got job 67 (collectAsMap at KMeans.scala:302) with 2 output partitions -2016-06-13 19:11:08 INFO DAGScheduler:59 - Final stage: ResultStage 731(collectAsMap at KMeans.scala:302) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 730) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 730) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ShuffleMapStage 730 (MapPartitionsRDD[389] at mapPartitions at KMeans.scala:279), which has no missing parents -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(10296) called with curMem=646123, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_120 stored as values in memory (estimated size 10.1 KB, free 730.0 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(4440) called with curMem=656419, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_120_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.0 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_120_piece0 in memory on localhost:65433 (size: 4.3 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 120 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 730 (MapPartitionsRDD[389] at mapPartitions at KMeans.scala:279) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 730.0 with 2 tasks -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 730.0 (TID 198, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 730.0 (TID 199, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 730.0 (TID 199) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 730.0 (TID 198) -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 730.0 (TID 198). 2413 bytes result sent to driver -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 730.0 (TID 199). 2413 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 730.0 (TID 198) in 34 ms on localhost (1/2) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 730.0 (TID 199) in 33 ms on localhost (2/2) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 730.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - ShuffleMapStage 730 (mapPartitions at KMeans.scala:279) finished in 0.034 s -2016-06-13 19:11:08 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:08 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:08 INFO DAGScheduler:59 - waiting: Set(ResultStage 731) -2016-06-13 19:11:08 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents for ResultStage 731: List() -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ResultStage 731 (ShuffledRDD[390] at reduceByKey at KMeans.scala:302), which is now runnable -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(2808) called with curMem=660859, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_121 stored as values in memory (estimated size 2.7 KB, free 730.0 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(1593) called with curMem=663667, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_121_piece0 stored as bytes in memory (estimated size 1593.0 B, free 730.0 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_121_piece0 in memory on localhost:65433 (size: 1593.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 121 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 731 (ShuffledRDD[390] at reduceByKey at KMeans.scala:302) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 731.0 with 2 tasks -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 731.0 (TID 200, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 731.0 (TID 201, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 731.0 (TID 201) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 731.0 (TID 200) -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 731.0 (TID 201). 1689 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 731.0 (TID 201) in 6 ms on localhost (1/2) -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 731.0 (TID 200). 1803 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 731.0 (TID 200) in 9 ms on localhost (2/2) -2016-06-13 19:11:08 INFO DAGScheduler:59 - ResultStage 731 (collectAsMap at KMeans.scala:302) finished in 0.009 s -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 731.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - Job 67 finished: collectAsMap at KMeans.scala:302, took 0.062994 s -2016-06-13 19:11:08 INFO KMeans:59 - Run 0 finished in 1 iterations -2016-06-13 19:11:08 INFO KMeans:59 - Run 1 finished in 1 iterations -2016-06-13 19:11:08 INFO KMeans:59 - Run 2 finished in 1 iterations -2016-06-13 19:11:08 INFO KMeans:59 - Run 3 finished in 1 iterations -2016-06-13 19:11:08 INFO KMeans:59 - Run 4 finished in 1 iterations -2016-06-13 19:11:08 INFO KMeans:59 - Iterations took 0.080 seconds. -2016-06-13 19:11:08 INFO KMeans:59 - KMeans converged in 1 iterations. -2016-06-13 19:11:08 INFO KMeans:59 - The cost for the best run is 0.0013884340729950003. -2016-06-13 19:11:08 INFO MapPartitionsRDD:59 - Removing RDD 362 from persistence list -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 362 -2016-06-13 19:11:08 WARN KMeans:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:11:08 INFO SparkContext:59 - Starting job: collect at PowerIterationClusteringSuite.scala:104 -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 151 bytes -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 151 bytes -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 151 bytes -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:11:08 INFO DAGScheduler:59 - Got job 68 (collect at PowerIterationClusteringSuite.scala:104) with 2 output partitions -2016-06-13 19:11:08 INFO DAGScheduler:59 - Final stage: ResultStage 743(collect at PowerIterationClusteringSuite.scala:104) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 742, ShuffleMapStage 735, ShuffleMapStage 736, ShuffleMapStage 740, ShuffleMapStage 738) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ResultStage 743 (MapPartitionsRDD[393] at mapPartitions at PowerIterationClustering.scala:232), which has no missing parents -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(8472) called with curMem=664780, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_122 stored as values in memory (estimated size 8.3 KB, free 729.9 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(3625) called with curMem=673252, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_122_piece0 stored as bytes in memory (estimated size 3.5 KB, free 729.9 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_122_piece0 in memory on localhost:65433 (size: 3.5 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 122 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 743 (MapPartitionsRDD[393] at mapPartitions at PowerIterationClustering.scala:232) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 743.0 with 2 tasks -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 743.0 (TID 202, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 743.0 (TID 203, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 743.0 (TID 202) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 743.0 (TID 203) -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_391_0 not found, computing it -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_391_1 not found, computing it -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=676877, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_391_0 stored as values in memory (estimated size 1168.0 B, free 729.9 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=678045, maxMem=766075207 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_391_0 in memory on localhost:65433 (size: 1168.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_391_1 stored as values in memory (estimated size 1168.0 B, free 729.9 MB) -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 743.0 (TID 202). 2919 bytes result sent to driver -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_391_1 in memory on localhost:65433 (size: 1168.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 743.0 (TID 202) in 6 ms on localhost (1/2) -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 743.0 (TID 203). 2919 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 743.0 (TID 203) in 7 ms on localhost (2/2) -2016-06-13 19:11:08 INFO DAGScheduler:59 - ResultStage 743 (collect at PowerIterationClusteringSuite.scala:104) finished in 0.007 s -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 743.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - Job 68 finished: collect at PowerIterationClusteringSuite.scala:104, took 0.023277 s -2016-06-13 19:11:08 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:340 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Registering RDD 405 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Registering RDD 399 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Got job 69 (sum at PowerIterationClustering.scala:340) with 2 output partitions -2016-06-13 19:11:08 INFO DAGScheduler:59 - Final stage: ResultStage 746(sum at PowerIterationClustering.scala:340) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 745, ShuffleMapStage 744) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 745, ShuffleMapStage 744) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ShuffleMapStage 744 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[405] at mapPartitions at GraphImpl.scala:235), which has no missing parents -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(3432) called with curMem=679213, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_123 stored as values in memory (estimated size 3.4 KB, free 729.9 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(1974) called with curMem=682645, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_123_piece0 stored as bytes in memory (estimated size 1974.0 B, free 729.9 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_123_piece0 in memory on localhost:65433 (size: 1974.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 123 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 744 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[405] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 744.0 with 2 tasks -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ShuffleMapStage 745 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[399] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 744.0 (TID 204, localhost, PROCESS_LOCAL, 2488 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 744.0 (TID 205, localhost, PROCESS_LOCAL, 2563 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 744.0 (TID 204) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 744.0 (TID 205) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(3016) called with curMem=684619, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_124 stored as values in memory (estimated size 2.9 KB, free 729.9 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(1779) called with curMem=687635, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_124_piece0 stored as bytes in memory (estimated size 1779.0 B, free 729.9 MB) -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_396_1 not found, computing it -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(2952) called with curMem=689414, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_396_1 stored as values in memory (estimated size 2.9 KB, free 729.9 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_124_piece0 in memory on localhost:65433 (size: 1779.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_396_1 in memory on localhost:65433 (size: 2.9 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 124 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 745 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[399] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 745.0 with 2 tasks -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_122_piece0 on localhost:65433 in memory (size: 3.5 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 745.0 (TID 206, localhost, PROCESS_LOCAL, 2488 bytes) -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_396_0 not found, computing it -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 745.0 (TID 207, localhost, PROCESS_LOCAL, 2563 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 745.0 (TID 207) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(2856) called with curMem=680269, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_396_0 stored as values in memory (estimated size 2.8 KB, free 729.9 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 117 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_396_0 in memory on localhost:65433 (size: 2.8 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_396_1 locally -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 745.0 (TID 206) -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_396_0 locally -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 744.0 (TID 205). 1739 bytes result sent to driver -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_121_piece0 on localhost:65433 in memory (size: 1593.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 744.0 (TID 205) in 43 ms on localhost (1/2) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 116 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_120_piece0 on localhost:65433 in memory (size: 4.3 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 115 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned shuffle 32 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_119_piece0 on localhost:65433 in memory (size: 363.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 745.0 (TID 206). 2254 bytes result sent to driver -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 114 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 113 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 112 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 111 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 110 -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 744.0 (TID 204). 1739 bytes result sent to driver -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_118_piece0 on localhost:65433 in memory (size: 1383.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 745.0 (TID 206) in 39 ms on localhost (1/2) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 744.0 (TID 204) in 70 ms on localhost (2/2) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 744.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - ShuffleMapStage 744 (mapPartitions at GraphImpl.scala:235) finished in 0.070 s -2016-06-13 19:11:08 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:08 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 745) -2016-06-13 19:11:08 INFO DAGScheduler:59 - waiting: Set(ResultStage 746) -2016-06-13 19:11:08 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 109 -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 745.0 (TID 207). 2254 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 745.0 (TID 207) in 40 ms on localhost (2/2) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 745.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents for ResultStage 746: List(ShuffleMapStage 745) -2016-06-13 19:11:08 INFO DAGScheduler:59 - ShuffleMapStage 745 (mapPartitions at VertexRDD.scala:358) finished in 0.044 s -2016-06-13 19:11:08 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:08 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:08 INFO DAGScheduler:59 - waiting: Set(ResultStage 746) -2016-06-13 19:11:08 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_117_piece0 on localhost:65433 in memory (size: 4.0 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 108 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents for ResultStage 746: List() -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned shuffle 31 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ResultStage 746 (MapPartitionsRDD[415] at values at PowerIterationClustering.scala:340), which is now runnable -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(4728) called with curMem=644543, maxMem=766075207 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_116_piece0 on localhost:65433 in memory (size: 787.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_125 stored as values in memory (estimated size 4.6 KB, free 730.0 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(2556) called with curMem=646327, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_125_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.0 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_125_piece0 in memory on localhost:65433 (size: 2.5 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 125 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_115_piece0 on localhost:65433 in memory (size: 4.5 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 746 (MapPartitionsRDD[415] at values at PowerIterationClustering.scala:340) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 746.0 with 2 tasks -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 107 -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 746.0 (TID 208, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 746.0 (TID 209, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 746.0 (TID 209) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_114_piece0 on localhost:65433 in memory (size: 4.4 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_407_1 not found, computing it -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_402_1 not found, computing it -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 106 -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 746.0 (TID 208) -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(1792) called with curMem=617473, maxMem=766075207 -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 384 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_402_1 stored as values in memory (estimated size 1792.0 B, free 730.0 MB) -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_407_0 not found, computing it -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_402_1 in memory on localhost:65433 (size: 1792.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_402_0 not found, computing it -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(1840) called with curMem=619265, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_407_1 stored as values in memory (estimated size 1840.0 B, free 730.0 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_407_1 in memory on localhost:65433 (size: 1840.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 746.0 (TID 209). 1840 bytes result sent to driver -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(1776) called with curMem=621105, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_402_0 stored as values in memory (estimated size 1776.0 B, free 730.0 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 384 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_402_0 in memory on localhost:65433 (size: 1776.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 746.0 (TID 209) in 19 ms on localhost (1/2) -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_113_piece0 on localhost:65433 in memory (size: 429.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(1824) called with curMem=617267, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_407_0 stored as values in memory (estimated size 1824.0 B, free 730.0 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_112_piece0 on localhost:65433 in memory (size: 4.4 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_407_0 in memory on localhost:65433 (size: 1824.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 105 -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 746.0 (TID 208). 1840 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 746.0 (TID 208) in 27 ms on localhost (2/2) -2016-06-13 19:11:08 INFO DAGScheduler:59 - ResultStage 746 (sum at PowerIterationClustering.scala:340) finished in 0.027 s -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 746.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - Job 69 finished: sum at PowerIterationClustering.scala:340, took 0.122109 s -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_111_piece0 on localhost:65433 in memory (size: 4.4 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 104 -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 380 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 380 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_110_piece0 on localhost:65433 in memory (size: 429.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Starting job: reduce at VertexRDDImpl.scala:90 -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_109_piece0 on localhost:65433 in memory (size: 4.4 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 103 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Got job 70 (reduce at VertexRDDImpl.scala:90) with 2 output partitions -2016-06-13 19:11:08 INFO DAGScheduler:59 - Final stage: ResultStage 749(reduce at VertexRDDImpl.scala:90) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 748, ShuffleMapStage 747) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ResultStage 749 (MapPartitionsRDD[420] at map at VertexRDDImpl.scala:90), which has no missing parents -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_108_piece0 on localhost:65433 in memory (size: 4.3 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(5200) called with curMem=561449, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_126 stored as values in memory (estimated size 5.1 KB, free 730.0 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 102 -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(2659) called with curMem=566649, maxMem=766075207 -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 376 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_126_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.0 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 376 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_126_piece0 in memory on localhost:65433 (size: 2.6 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 126 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 749 (MapPartitionsRDD[420] at map at VertexRDDImpl.scala:90) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 749.0 with 2 tasks -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 749.0 (TID 210, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 749.0 (TID 211, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 749.0 (TID 211) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 749.0 (TID 210) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_107_piece0 on localhost:65433 in memory (size: 579.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_407_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_407_0 locally -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 749.0 (TID 211). 2161 bytes result sent to driver -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 749.0 (TID 210). 2161 bytes result sent to driver -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_106_piece0 on localhost:65433 in memory (size: 4.3 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 749.0 (TID 211) in 4 ms on localhost (1/2) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 101 -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 749.0 (TID 210) in 6 ms on localhost (2/2) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 749.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - ResultStage 749 (reduce at VertexRDDImpl.scala:90) finished in 0.006 s -2016-06-13 19:11:08 INFO DAGScheduler:59 - Job 70 finished: reduce at VertexRDDImpl.scala:90, took 0.015418 s -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_105_piece0 on localhost:65433 in memory (size: 4.3 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 100 -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 372 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 372 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_104_piece0 on localhost:65433 in memory (size: 636.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_103_piece0 on localhost:65433 in memory (size: 4.2 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 99 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_102_piece0 on localhost:65433 in memory (size: 4.2 KB, free: 730.3 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 98 -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 368 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 368 -2016-06-13 19:11:08 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Registering RDD 409 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Registering RDD 421 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_101_piece0 on localhost:65433 in memory (size: 493.0 B, free: 730.3 MB) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Registering RDD 425 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Got job 71 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:11:08 INFO DAGScheduler:59 - Final stage: ResultStage 757(sum at PowerIterationClustering.scala:369) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 753, ShuffleMapStage 754, ShuffleMapStage 756) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 756) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_100_piece0 on localhost:65433 in memory (size: 9.9 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 97 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ShuffleMapStage 752 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[409] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=465686, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_127 stored as values in memory (estimated size 4.1 KB, free 730.1 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_99_piece0 on localhost:65433 in memory (size: 3.4 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 96 -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(2379) called with curMem=461766, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_127_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.1 MB) -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 362 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_127_piece0 in memory on localhost:65433 (size: 2.3 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 127 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 362 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 752 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[409] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 752.0 with 2 tasks -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ShuffleMapStage 755 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[421] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_98_piece0 on localhost:65433 in memory (size: 3.4 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(4904) called with curMem=460710, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_128 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 752.0 (TID 212, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 752.0 (TID 213, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 95 -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 752.0 (TID 213) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(2574) called with curMem=457678, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_128_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 752.0 (TID 212) -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 348 -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_407_1 locally -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_128_piece0 in memory on localhost:65433 (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 128 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_407_0 locally -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 755 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[421] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 755.0 with 2 tasks -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 348 -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 755.0 (TID 214, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 755.0 (TID 215, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 755.0 (TID 215) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 755.0 (TID 214) -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_418_1 not found, computing it -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_407_1 locally -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=457388, maxMem=766075207 -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_418_0 not found, computing it -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_418_1 stored as values in memory (estimated size 1432.0 B, free 730.1 MB) -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_407_0 locally -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=458820, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_418_0 stored as values in memory (estimated size 1432.0 B, free 730.1 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_418_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_418_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 755.0 (TID 215). 2834 bytes result sent to driver -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 755.0 (TID 214). 2834 bytes result sent to driver -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_97_piece0 on localhost:65433 in memory (size: 3.2 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 755.0 (TID 215) in 12 ms on localhost (1/2) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 755.0 (TID 214) in 13 ms on localhost (2/2) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 755.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - ShuffleMapStage 755 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.015 s -2016-06-13 19:11:08 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:08 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 752) -2016-06-13 19:11:08 INFO DAGScheduler:59 - waiting: Set(ResultStage 757, ShuffleMapStage 756) -2016-06-13 19:11:08 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 752.0 (TID 213). 2254 bytes result sent to driver -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 94 -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 752.0 (TID 213) in 23 ms on localhost (1/2) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents for ResultStage 757: List(ShuffleMapStage 756) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_96_piece0 on localhost:65433 in memory (size: 2.4 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 756: List(ShuffleMapStage 752) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 93 -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 752.0 (TID 212). 2254 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 752.0 (TID 212) in 31 ms on localhost (2/2) -2016-06-13 19:11:08 INFO DAGScheduler:59 - ShuffleMapStage 752 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.033 s -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 752.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:08 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:08 INFO DAGScheduler:59 - waiting: Set(ResultStage 757, ShuffleMapStage 756) -2016-06-13 19:11:08 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_95_piece0 on localhost:65433 in memory (size: 3.0 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 92 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents for ResultStage 757: List(ShuffleMapStage 756) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 756: List() -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ShuffleMapStage 756 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[425] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_94_piece0 on localhost:65433 in memory (size: 3.2 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 91 -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=422217, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_129 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 325 -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=425633, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_129_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 325 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_129_piece0 in memory on localhost:65433 (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 129 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 756 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[425] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 756.0 with 2 tasks -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_93_piece0 on localhost:65433 in memory (size: 3.1 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 756.0 (TID 216, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 90 -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 756.0 (TID 217, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 756.0 (TID 217) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 756.0 (TID 216) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_92_piece0 on localhost:65433 in memory (size: 2.4 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_396_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_396_0 locally -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 89 -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_91_piece0 on localhost:65433 in memory (size: 2.8 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 88 -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_90_piece0 on localhost:65433 in memory (size: 3.1 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 87 -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 302 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 302 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_89_piece0 on localhost:65433 in memory (size: 2.8 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 86 -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 756.0 (TID 217). 2470 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 756.0 (TID 217) in 24 ms on localhost (1/2) -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 756.0 (TID 216). 2470 bytes result sent to driver -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:65433 in memory (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 756.0 (TID 216) in 26 ms on localhost (2/2) -2016-06-13 19:11:08 INFO DAGScheduler:59 - ShuffleMapStage 756 (mapPartitions at GraphImpl.scala:235) finished in 0.026 s -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 756.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 76 -2016-06-13 19:11:08 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:08 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:08 INFO DAGScheduler:59 - waiting: Set(ResultStage 757) -2016-06-13 19:11:08 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned shuffle 21 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents for ResultStage 757: List() -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ResultStage 757 (MapPartitionsRDD[430] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:65433 in memory (size: 339.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 75 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 74 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 73 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 72 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 71 -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(5944) called with curMem=363613, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_130 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=365863, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_130_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:65433 in memory (size: 1382.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 70 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_130_piece0 in memory on localhost:65433 (size: 2.8 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 130 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 757 (MapPartitionsRDD[430] at map at PowerIterationClustering.scala:369) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 757.0 with 2 tasks -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 757.0 (TID 218, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:65433 in memory (size: 4.0 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 757.0 (TID 219, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 757.0 (TID 218) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 757.0 (TID 219) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 69 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned shuffle 20 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:65433 in memory (size: 708.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_427_0 not found, computing it -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_418_0 locally -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_427_1 not found, computing it -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_418_1 locally -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=347130, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_427_0 stored as values in memory (estimated size 1480.0 B, free 730.3 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:65433 in memory (size: 4.5 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=348610, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_427_1 stored as values in memory (estimated size 1480.0 B, free 730.3 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_427_0 in memory on localhost:65433 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 757.0 (TID 218). 2880 bytes result sent to driver -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_427_1 in memory on localhost:65433 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 757.0 (TID 219). 2880 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 757.0 (TID 218) in 14 ms on localhost (1/2) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 757.0 (TID 219) in 15 ms on localhost (2/2) -2016-06-13 19:11:08 INFO DAGScheduler:59 - ResultStage 757 (sum at PowerIterationClustering.scala:369) finished in 0.016 s -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 757.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - Job 71 finished: sum at PowerIterationClustering.scala:369, took 0.108539 s -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:65433 in memory (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-06-13 19:11:08 INFO PowerIterationClustering:59 - Iteration 0: norm(v) = 36.22334869431644. -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 253 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 253 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:65433 in memory (size: 236.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:65433 in memory (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:65433 in memory (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 249 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 249 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:65433 in memory (size: 429.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 245 -2016-06-13 19:11:08 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 245 -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:65433 in memory (size: 582.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 151 bytes -2016-06-13 19:11:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:11:08 INFO DAGScheduler:59 - Got job 72 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:11:08 INFO DAGScheduler:59 - Final stage: ResultStage 765(sum at PowerIterationClustering.scala:375) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 764, ShuffleMapStage 761, ShuffleMapStage 762) -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 241 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 241 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting ResultStage 765 (MapPartitionsRDD[441] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:65433 in memory (size: 600.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(6480) called with curMem=287211, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_131 stored as values in memory (estimated size 6.3 KB, free 730.3 MB) -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 237 -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(3052) called with curMem=293691, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block broadcast_131_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.3 MB) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 237 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added broadcast_131_piece0 in memory on localhost:65433 (size: 3.0 KB, free: 730.4 MB) -2016-06-13 19:11:08 INFO SparkContext:59 - Created broadcast 131 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 765 (MapPartitionsRDD[441] at values at PowerIterationClustering.scala:375) -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Adding task set 765.0 with 2 tasks -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 765.0 (TID 220, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:65433 in memory (size: 487.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 765.0 (TID 221, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:11:08 INFO Executor:59 - Running task 1.0 in stage 765.0 (TID 221) -2016-06-13 19:11:08 INFO Executor:59 - Running task 0.0 in stage 765.0 (TID 220) -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 231 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 231 -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_433_1 not found, computing it -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_418_1 locally -2016-06-13 19:11:08 INFO CacheManager:59 - Partition rdd_433_0 not found, computing it -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_427_1 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_418_0 locally -2016-06-13 19:11:08 INFO BlockManager:59 - Found block rdd_427_0 locally -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=295376, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_433_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:11:08 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=296808, maxMem=766075207 -2016-06-13 19:11:08 INFO MemoryStore:59 - Block rdd_433_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 228 -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_433_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO BlockManagerInfo:59 - Added rdd_433_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:08 INFO Executor:59 - Finished task 1.0 in stage 765.0 (TID 221). 2664 bytes result sent to driver -2016-06-13 19:11:08 INFO Executor:59 - Finished task 0.0 in stage 765.0 (TID 220). 2664 bytes result sent to driver -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 765.0 (TID 221) in 6 ms on localhost (1/2) -2016-06-13 19:11:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 765.0 (TID 220) in 7 ms on localhost (2/2) -2016-06-13 19:11:08 INFO DAGScheduler:59 - ResultStage 765 (sum at PowerIterationClustering.scala:375) finished in 0.008 s -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 228 -2016-06-13 19:11:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 765.0, whose tasks have all completed, from pool -2016-06-13 19:11:08 INFO DAGScheduler:59 - Job 72 finished: sum at PowerIterationClustering.scala:375, took 0.027431 s -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned shuffle 19 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned shuffle 18 -2016-06-13 19:11:08 INFO PowerIterationClustering:59 - Iteration 0: delta = 0.010579646370791257. -2016-06-13 19:11:08 INFO PowerIterationClustering:59 - Iteration 0: diff(delta) = 1.7976931348623157E308. -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 211 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 211 -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 203 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 203 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned shuffle 17 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned shuffle 16 -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 188 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 188 -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 180 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 180 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned shuffle 15 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned shuffle 14 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned shuffle 13 -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 165 -2016-06-13 19:11:08 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 165 -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 156 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Registering RDD 444 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Registering RDD 448 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:08 INFO DAGScheduler:59 - Got job 73 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:11:08 INFO DAGScheduler:59 - Final stage: ResultStage 775(sum at PowerIterationClustering.scala:369) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned RDD 156 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 772, ShuffleMapStage 769, ShuffleMapStage 770, ShuffleMapStage 774) -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned shuffle 12 -2016-06-13 19:11:08 INFO ContextCleaner:59 - Cleaned shuffle 11 -2016-06-13 19:11:08 INFO BlockManager:59 - Removing RDD 145 -2016-06-13 19:11:08 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 774) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ShuffleMapStage 773 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[444] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(5704) called with curMem=274240, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_132 stored as values in memory (estimated size 5.6 KB, free 730.3 MB) -2016-06-13 19:11:09 INFO ContextCleaner:59 - Cleaned RDD 145 -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(2813) called with curMem=279944, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_132_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.3 MB) -2016-06-13 19:11:09 INFO BlockManager:59 - Removing RDD 140 -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_132_piece0 in memory on localhost:65433 (size: 2.7 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 132 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 773 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[444] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 773.0 with 2 tasks -2016-06-13 19:11:09 INFO ContextCleaner:59 - Cleaned RDD 140 -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 773.0 (TID 222, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 773.0 (TID 223, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 773.0 (TID 223) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 773.0 (TID 222) -2016-06-13 19:11:09 INFO BlockManager:59 - Removing RDD 134 -2016-06-13 19:11:09 INFO ContextCleaner:59 - Cleaned RDD 134 -2016-06-13 19:11:09 INFO BlockManager:59 - Removing RDD 129 -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_442_0 not found, computing it -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_427_0 locally -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_442_1 not found, computing it -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_427_1 locally -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=272213, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_442_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=272477, maxMem=766075207 -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_442_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO ContextCleaner:59 - Cleaned RDD 129 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_442_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_442_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 773.0 (TID 222). 2834 bytes result sent to driver -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Removed broadcast_88_piece0 on localhost:65433 in memory (size: 2.4 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 773.0 (TID 223). 2834 bytes result sent to driver -2016-06-13 19:11:09 INFO ContextCleaner:59 - Cleaned accumulator 85 -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 773.0 (TID 223) in 10 ms on localhost (1/2) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 773.0 (TID 222) in 12 ms on localhost (2/2) -2016-06-13 19:11:09 INFO DAGScheduler:59 - ShuffleMapStage 773 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.012 s -2016-06-13 19:11:09 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:09 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:09 INFO DAGScheduler:59 - waiting: Set(ResultStage 775, ShuffleMapStage 774) -2016-06-13 19:11:09 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 773.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents for ResultStage 775: List(ShuffleMapStage 774) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Removed broadcast_87_piece0 on localhost:65433 in memory (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 774: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ShuffleMapStage 774 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[448] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:11:09 INFO ContextCleaner:59 - Cleaned accumulator 84 -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=256954, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_133 stored as values in memory (estimated size 4.7 KB, free 730.3 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:65433 in memory (size: 2.3 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO ContextCleaner:59 - Cleaned accumulator 83 -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=257554, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_133_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_133_piece0 in memory on localhost:65433 (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 133 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:65433 in memory (size: 2.6 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 774 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[448] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 774.0 with 2 tasks -2016-06-13 19:11:09 INFO ContextCleaner:59 - Cleaned accumulator 82 -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 774.0 (TID 224, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 774.0 (TID 225, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 774.0 (TID 224) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 774.0 (TID 225) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:65433 in memory (size: 2.6 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO ContextCleaner:59 - Cleaned accumulator 81 -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_396_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_396_1 locally -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:65433 in memory (size: 1642.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO ContextCleaner:59 - Cleaned accumulator 80 -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:65433 in memory (size: 1868.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO ContextCleaner:59 - Cleaned accumulator 79 -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:65433 in memory (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:11:09 INFO ContextCleaner:59 - Cleaned accumulator 78 -2016-06-13 19:11:09 INFO BlockManager:59 - Removing RDD 260 -2016-06-13 19:11:09 INFO ContextCleaner:59 - Cleaned RDD 260 -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 774.0 (TID 225). 2470 bytes result sent to driver -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:65433 in memory (size: 1589.0 B, free: 730.5 MB) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 774.0 (TID 224). 2470 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 774.0 (TID 224) in 25 ms on localhost (1/2) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 774.0 (TID 225) in 26 ms on localhost (2/2) -2016-06-13 19:11:09 INFO DAGScheduler:59 - ShuffleMapStage 774 (mapPartitions at GraphImpl.scala:235) finished in 0.026 s -2016-06-13 19:11:09 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:09 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 774.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - waiting: Set(ResultStage 775) -2016-06-13 19:11:09 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:09 INFO ContextCleaner:59 - Cleaned accumulator 77 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents for ResultStage 775: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ResultStage 775 (MapPartitionsRDD[453] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(6640) called with curMem=216476, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_134 stored as values in memory (estimated size 6.5 KB, free 730.4 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(3099) called with curMem=223116, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_134_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_134_piece0 in memory on localhost:65433 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 134 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 775 (MapPartitionsRDD[453] at map at PowerIterationClustering.scala:369) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 775.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 775.0 (TID 226, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 775.0 (TID 227, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 775.0 (TID 226) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 775.0 (TID 227) -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_450_1 not found, computing it -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_442_1 locally -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_450_0 not found, computing it -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_442_0 locally -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=226215, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_450_1 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_450_1 in memory on localhost:65433 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=227695, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_450_0 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 775.0 (TID 227). 2880 bytes result sent to driver -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_450_0 in memory on localhost:65433 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 775.0 (TID 226). 2880 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 775.0 (TID 227) in 9 ms on localhost (1/2) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 775.0 (TID 226) in 11 ms on localhost (2/2) -2016-06-13 19:11:09 INFO DAGScheduler:59 - ResultStage 775 (sum at PowerIterationClustering.scala:369) finished in 0.011 s -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 775.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - Job 73 finished: sum at PowerIterationClustering.scala:369, took 0.086859 s -2016-06-13 19:11:09 INFO PowerIterationClustering:59 - Iteration 1: norm(v) = 36.22334869431644. -2016-06-13 19:11:09 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 151 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 151 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:11:09 INFO DAGScheduler:59 - Got job 74 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:11:09 INFO DAGScheduler:59 - Final stage: ResultStage 785(sum at PowerIterationClustering.scala:375) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 782, ShuffleMapStage 779, ShuffleMapStage 780, ShuffleMapStage 784) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ResultStage 785 (MapPartitionsRDD[464] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(7072) called with curMem=229175, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_135 stored as values in memory (estimated size 6.9 KB, free 730.4 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(3227) called with curMem=236247, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_135_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_135_piece0 in memory on localhost:65433 (size: 3.2 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 135 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 785 (MapPartitionsRDD[464] at values at PowerIterationClustering.scala:375) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 785.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 785.0 (TID 228, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 785.0 (TID 229, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 785.0 (TID 228) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 785.0 (TID 229) -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_456_1 not found, computing it -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_442_1 locally -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_456_0 not found, computing it -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_450_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_442_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_450_0 locally -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=239474, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_456_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=240906, maxMem=766075207 -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_456_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_456_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 785.0 (TID 229). 2664 bytes result sent to driver -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_456_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 785.0 (TID 229) in 7 ms on localhost (1/2) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 785.0 (TID 228). 2664 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 785.0 (TID 228) in 10 ms on localhost (2/2) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 785.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - ResultStage 785 (sum at PowerIterationClustering.scala:375) finished in 0.010 s -2016-06-13 19:11:09 INFO DAGScheduler:59 - Job 74 finished: sum at PowerIterationClustering.scala:375, took 0.021253 s -2016-06-13 19:11:09 INFO PowerIterationClustering:59 - Iteration 1: delta = 0.0. -2016-06-13 19:11:09 INFO PowerIterationClustering:59 - Iteration 1: diff(delta) = 0.010579646370791257. -2016-06-13 19:11:09 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Registering RDD 467 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Registering RDD 471 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Got job 75 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:11:09 INFO DAGScheduler:59 - Final stage: ResultStage 797(sum at PowerIterationClustering.scala:369) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 789, ShuffleMapStage 796, ShuffleMapStage 790, ShuffleMapStage 794, ShuffleMapStage 792) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 796) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ShuffleMapStage 795 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[467] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(6288) called with curMem=242338, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_136 stored as values in memory (estimated size 6.1 KB, free 730.3 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(2958) called with curMem=248626, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_136_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_136_piece0 in memory on localhost:65433 (size: 2.9 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 136 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 795 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[467] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 795.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 795.0 (TID 230, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 795.0 (TID 231, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 795.0 (TID 231) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 795.0 (TID 230) -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_465_1 not found, computing it -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_450_1 locally -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_465_0 not found, computing it -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=251584, maxMem=766075207 -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_450_0 locally -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_465_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=253016, maxMem=766075207 -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_465_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_465_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_465_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 795.0 (TID 231). 2834 bytes result sent to driver -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 795.0 (TID 230). 2834 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 795.0 (TID 231) in 12 ms on localhost (1/2) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 795.0 (TID 230) in 14 ms on localhost (2/2) -2016-06-13 19:11:09 INFO DAGScheduler:59 - ShuffleMapStage 795 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.015 s -2016-06-13 19:11:09 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:09 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:09 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 796, ResultStage 797) -2016-06-13 19:11:09 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 795.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 796: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents for ResultStage 797: List(ShuffleMapStage 796) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ShuffleMapStage 796 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[471] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=254448, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_137 stored as values in memory (estimated size 4.7 KB, free 730.3 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=259296, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_137_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_137_piece0 in memory on localhost:65433 (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 137 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 796 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[471] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 796.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 796.0 (TID 232, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 796.0 (TID 233, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 796.0 (TID 233) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 796.0 (TID 232) -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_396_1 locally -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_396_0 locally -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 796.0 (TID 232). 2470 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 796.0 (TID 232) in 32 ms on localhost (1/2) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 796.0 (TID 233). 2470 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 796.0 (TID 233) in 41 ms on localhost (2/2) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 796.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - ShuffleMapStage 796 (mapPartitions at GraphImpl.scala:235) finished in 0.042 s -2016-06-13 19:11:09 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:09 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:09 INFO DAGScheduler:59 - waiting: Set(ResultStage 797) -2016-06-13 19:11:09 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents for ResultStage 797: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ResultStage 797 (MapPartitionsRDD[476] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(7224) called with curMem=261838, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_138 stored as values in memory (estimated size 7.1 KB, free 730.3 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(3221) called with curMem=269062, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_138_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.3 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_138_piece0 in memory on localhost:65433 (size: 3.1 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 138 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 797 (MapPartitionsRDD[476] at map at PowerIterationClustering.scala:369) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 797.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 797.0 (TID 234, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 797.0 (TID 235, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 797.0 (TID 234) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 797.0 (TID 235) -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_473_1 not found, computing it -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_473_0 not found, computing it -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_465_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_465_0 locally -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=272283, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_473_1 stored as values in memory (estimated size 1480.0 B, free 730.3 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=273763, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_473_0 stored as values in memory (estimated size 1480.0 B, free 730.3 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_473_1 in memory on localhost:65433 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_473_0 in memory on localhost:65433 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 797.0 (TID 235). 2880 bytes result sent to driver -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 797.0 (TID 234). 2880 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 797.0 (TID 235) in 10 ms on localhost (1/2) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 797.0 (TID 234) in 11 ms on localhost (2/2) -2016-06-13 19:11:09 INFO DAGScheduler:59 - ResultStage 797 (sum at PowerIterationClustering.scala:369) finished in 0.011 s -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 797.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - Job 75 finished: sum at PowerIterationClustering.scala:369, took 0.107658 s -2016-06-13 19:11:09 INFO PowerIterationClustering:59 - Iteration 2: norm(v) = 36.22334869431644. -2016-06-13 19:11:09 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 151 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 151 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 41 is 151 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 40 is 155 bytes -2016-06-13 19:11:09 INFO DAGScheduler:59 - Got job 76 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:11:09 INFO DAGScheduler:59 - Final stage: ResultStage 809(sum at PowerIterationClustering.scala:375) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 804, ShuffleMapStage 801, ShuffleMapStage 808, ShuffleMapStage 802, ShuffleMapStage 806) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ResultStage 809 (MapPartitionsRDD[487] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=275243, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_139 stored as values in memory (estimated size 7.5 KB, free 730.3 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(3351) called with curMem=282899, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_139_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.3 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_139_piece0 in memory on localhost:65433 (size: 3.3 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 139 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 809 (MapPartitionsRDD[487] at values at PowerIterationClustering.scala:375) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 809.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 809.0 (TID 236, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 809.0 (TID 237, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 809.0 (TID 237) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 809.0 (TID 236) -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_479_1 not found, computing it -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_465_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_473_1 locally -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_479_0 not found, computing it -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_465_0 locally -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=286250, maxMem=766075207 -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_473_0 locally -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_479_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=287682, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_479_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_479_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_479_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 809.0 (TID 237). 2664 bytes result sent to driver -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 809.0 (TID 236). 2664 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 809.0 (TID 237) in 7 ms on localhost (1/2) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 809.0 (TID 236) in 7 ms on localhost (2/2) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 809.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - ResultStage 809 (sum at PowerIterationClustering.scala:375) finished in 0.007 s -2016-06-13 19:11:09 INFO DAGScheduler:59 - Job 76 finished: sum at PowerIterationClustering.scala:375, took 0.020956 s -2016-06-13 19:11:09 INFO PowerIterationClustering:59 - Iteration 2: delta = 0.0. -2016-06-13 19:11:09 INFO PowerIterationClustering:59 - Iteration 2: diff(delta) = 0.0. -2016-06-13 19:11:09 WARN KMeans:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:11:09 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Got job 77 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:11:09 INFO DAGScheduler:59 - Final stage: ResultStage 821(takeSample at KMeans.scala:376) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 814, ShuffleMapStage 818, ShuffleMapStage 816, ShuffleMapStage 820, ShuffleMapStage 813) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ResultStage 821 (MapPartitionsRDD[495] at map at KMeans.scala:210), which has no missing parents -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(7888) called with curMem=289114, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_140 stored as values in memory (estimated size 7.7 KB, free 730.3 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(3422) called with curMem=297002, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_140_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.3 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_140_piece0 in memory on localhost:65433 (size: 3.3 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 140 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 821 (MapPartitionsRDD[495] at map at KMeans.scala:210) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 821.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 821.0 (TID 238, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 821.0 (TID 239, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 821.0 (TID 238) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 821.0 (TID 239) -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_490_0 not found, computing it -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_490_1 not found, computing it -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_473_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_473_1 locally -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=300424, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_490_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=301856, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_490_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_490_0 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_490_1 in memory on localhost:65433 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_493_0 not found, computing it -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_493_1 not found, computing it -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=303288, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_493_0 stored as values in memory (estimated size 240.0 B, free 730.3 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=303528, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_493_1 stored as values in memory (estimated size 240.0 B, free 730.3 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_493_0 in memory on localhost:65433 (size: 240.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_493_1 in memory on localhost:65433 (size: 240.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 821.0 (TID 238). 2717 bytes result sent to driver -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 821.0 (TID 239). 2717 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 821.0 (TID 238) in 11 ms on localhost (1/2) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 821.0 (TID 239) in 11 ms on localhost (2/2) -2016-06-13 19:11:09 INFO DAGScheduler:59 - ResultStage 821 (takeSample at KMeans.scala:376) finished in 0.012 s -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 821.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - Job 77 finished: takeSample at KMeans.scala:376, took 0.023354 s -2016-06-13 19:11:09 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Got job 78 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:11:09 INFO DAGScheduler:59 - Final stage: ResultStage 833(takeSample at KMeans.scala:376) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 825, ShuffleMapStage 832, ShuffleMapStage 826, ShuffleMapStage 830, ShuffleMapStage 828) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ResultStage 833 (PartitionwiseSampledRDD[497] at takeSample at KMeans.scala:376), which has no missing parents -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(25280) called with curMem=303768, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_141 stored as values in memory (estimated size 24.7 KB, free 730.3 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(10133) called with curMem=329048, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_141_piece0 stored as bytes in memory (estimated size 9.9 KB, free 730.3 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_141_piece0 in memory on localhost:65433 (size: 9.9 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 141 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 833 (PartitionwiseSampledRDD[497] at takeSample at KMeans.scala:376) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 833.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 833.0 (TID 240, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 833.0 (TID 241, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 833.0 (TID 240) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 833.0 (TID 241) -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 833.0 (TID 241). 2575 bytes result sent to driver -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 833.0 (TID 240). 2633 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 833.0 (TID 241) in 6 ms on localhost (1/2) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 833.0 (TID 240) in 6 ms on localhost (2/2) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 833.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - ResultStage 833 (takeSample at KMeans.scala:376) finished in 0.007 s -2016-06-13 19:11:09 INFO DAGScheduler:59 - Job 78 finished: takeSample at KMeans.scala:376, took 0.020115 s -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=339181, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_142 stored as values in memory (estimated size 880.0 B, free 730.3 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(487) called with curMem=340061, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_142_piece0 stored as bytes in memory (estimated size 487.0 B, free 730.3 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_142_piece0 in memory on localhost:65433 (size: 487.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 142 from broadcast at KMeans.scala:394 -2016-06-13 19:11:09 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Got job 79 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:11:09 INFO DAGScheduler:59 - Final stage: ResultStage 845(aggregate at KMeans.scala:402) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 840, ShuffleMapStage 844, ShuffleMapStage 837, ShuffleMapStage 838, ShuffleMapStage 842) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ResultStage 845 (MapPartitionsRDD[499] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(9912) called with curMem=340548, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_143 stored as values in memory (estimated size 9.7 KB, free 730.3 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(4237) called with curMem=350460, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_143_piece0 stored as bytes in memory (estimated size 4.1 KB, free 730.2 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_143_piece0 in memory on localhost:65433 (size: 4.1 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 143 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 845 (MapPartitionsRDD[499] at map at KMeans.scala:396) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 845.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 845.0 (TID 242, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 845.0 (TID 243, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 845.0 (TID 242) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 845.0 (TID 243) -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_499_0 not found, computing it -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_499_1 not found, computing it -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=354697, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_499_0 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=355193, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_499_1 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_499_0 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_499_1 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 845.0 (TID 243). 2647 bytes result sent to driver -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 845.0 (TID 242). 2647 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 845.0 (TID 243) in 12 ms on localhost (1/2) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 845.0 (TID 242) in 14 ms on localhost (2/2) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 845.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - ResultStage 845 (aggregate at KMeans.scala:402) finished in 0.015 s -2016-06-13 19:11:09 INFO DAGScheduler:59 - Job 79 finished: aggregate at KMeans.scala:402, took 0.028622 s -2016-06-13 19:11:09 INFO MapPartitionsRDD:59 - Removing RDD 496 from persistence list -2016-06-13 19:11:09 INFO BlockManager:59 - Removing RDD 496 -2016-06-13 19:11:09 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Got job 80 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:11:09 INFO DAGScheduler:59 - Final stage: ResultStage 857(collect at KMeans.scala:431) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 850, ShuffleMapStage 854, ShuffleMapStage 852, ShuffleMapStage 849, ShuffleMapStage 856) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ResultStage 857 (MapPartitionsRDD[501] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(10072) called with curMem=355689, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_144 stored as values in memory (estimated size 9.8 KB, free 730.2 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(4342) called with curMem=365761, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_144_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.2 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_144_piece0 in memory on localhost:65433 (size: 4.2 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 144 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 857 (MapPartitionsRDD[501] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 857.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 857.0 (TID 244, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 857.0 (TID 245, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 857.0 (TID 245) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 857.0 (TID 244) -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_499_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_499_0 locally -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 857.0 (TID 245). 3471 bytes result sent to driver -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 857.0 (TID 244). 3118 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 857.0 (TID 245) in 4 ms on localhost (1/2) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 857.0 (TID 244) in 5 ms on localhost (2/2) -2016-06-13 19:11:09 INFO DAGScheduler:59 - ResultStage 857 (collect at KMeans.scala:431) finished in 0.005 s -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 857.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - Job 80 finished: collect at KMeans.scala:431, took 0.018559 s -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(1648) called with curMem=370103, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_145 stored as values in memory (estimated size 1648.0 B, free 730.2 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(600) called with curMem=371751, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_145_piece0 stored as bytes in memory (estimated size 600.0 B, free 730.2 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_145_piece0 in memory on localhost:65433 (size: 600.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 145 from broadcast at KMeans.scala:394 -2016-06-13 19:11:09 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Got job 81 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:11:09 INFO DAGScheduler:59 - Final stage: ResultStage 869(aggregate at KMeans.scala:402) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 861, ShuffleMapStage 868, ShuffleMapStage 862, ShuffleMapStage 866, ShuffleMapStage 864) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ResultStage 869 (MapPartitionsRDD[503] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(10144) called with curMem=372351, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_146 stored as values in memory (estimated size 9.9 KB, free 730.2 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(4326) called with curMem=382495, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_146_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.2 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_146_piece0 in memory on localhost:65433 (size: 4.2 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 146 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 869 (MapPartitionsRDD[503] at map at KMeans.scala:396) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 869.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 869.0 (TID 246, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 869.0 (TID 247, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 869.0 (TID 246) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 869.0 (TID 247) -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_503_1 not found, computing it -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_503_0 not found, computing it -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_499_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_499_0 locally -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=386821, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_503_1 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=387317, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_503_0 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_503_1 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_503_0 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 869.0 (TID 247). 2647 bytes result sent to driver -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 869.0 (TID 246). 2647 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 869.0 (TID 247) in 9 ms on localhost (1/2) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 869.0 (TID 246) in 10 ms on localhost (2/2) -2016-06-13 19:11:09 INFO DAGScheduler:59 - ResultStage 869 (aggregate at KMeans.scala:402) finished in 0.010 s -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 869.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - Job 81 finished: aggregate at KMeans.scala:402, took 0.023697 s -2016-06-13 19:11:09 INFO MapPartitionsRDD:59 - Removing RDD 499 from persistence list -2016-06-13 19:11:09 INFO BlockManager:59 - Removing RDD 499 -2016-06-13 19:11:09 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Got job 82 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:11:09 INFO DAGScheduler:59 - Final stage: ResultStage 881(collect at KMeans.scala:431) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 876, ShuffleMapStage 880, ShuffleMapStage 873, ShuffleMapStage 874, ShuffleMapStage 878) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ResultStage 881 (MapPartitionsRDD[505] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(10304) called with curMem=386821, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_147 stored as values in memory (estimated size 10.1 KB, free 730.2 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(4431) called with curMem=397125, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_147_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.2 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_147_piece0 in memory on localhost:65433 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 147 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 881 (MapPartitionsRDD[505] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 881.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 881.0 (TID 248, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 881.0 (TID 249, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 881.0 (TID 248) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 881.0 (TID 249) -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_503_0 locally -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 881.0 (TID 248). 2990 bytes result sent to driver -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_503_1 locally -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 881.0 (TID 248) in 6 ms on localhost (1/2) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 881.0 (TID 249). 3240 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 881.0 (TID 249) in 7 ms on localhost (2/2) -2016-06-13 19:11:09 INFO DAGScheduler:59 - ResultStage 881 (collect at KMeans.scala:431) finished in 0.008 s -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 881.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - Job 82 finished: collect at KMeans.scala:431, took 0.021267 s -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(1456) called with curMem=401556, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_148 stored as values in memory (estimated size 1456.0 B, free 730.2 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(582) called with curMem=403012, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_148_piece0 stored as bytes in memory (estimated size 582.0 B, free 730.2 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_148_piece0 in memory on localhost:65433 (size: 582.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 148 from broadcast at KMeans.scala:394 -2016-06-13 19:11:09 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Got job 83 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:11:09 INFO DAGScheduler:59 - Final stage: ResultStage 893(aggregate at KMeans.scala:402) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 888, ShuffleMapStage 885, ShuffleMapStage 892, ShuffleMapStage 886, ShuffleMapStage 890) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ResultStage 893 (MapPartitionsRDD[507] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(10376) called with curMem=403594, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_149 stored as values in memory (estimated size 10.1 KB, free 730.2 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(4373) called with curMem=413970, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_149_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.2 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_149_piece0 in memory on localhost:65433 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 149 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 893 (MapPartitionsRDD[507] at map at KMeans.scala:396) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 893.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 893.0 (TID 250, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 893.0 (TID 251, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 893.0 (TID 250) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 893.0 (TID 251) -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_507_0 not found, computing it -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_507_1 not found, computing it -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_503_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_503_1 locally -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=418343, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_507_0 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=418839, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_507_1 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_507_0 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 893.0 (TID 250). 2647 bytes result sent to driver -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_507_1 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 893.0 (TID 250) in 10 ms on localhost (1/2) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 893.0 (TID 251). 2647 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 893.0 (TID 251) in 10 ms on localhost (2/2) -2016-06-13 19:11:09 INFO DAGScheduler:59 - ResultStage 893 (aggregate at KMeans.scala:402) finished in 0.011 s -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 893.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - Job 83 finished: aggregate at KMeans.scala:402, took 0.027449 s -2016-06-13 19:11:09 INFO MapPartitionsRDD:59 - Removing RDD 503 from persistence list -2016-06-13 19:11:09 INFO BlockManager:59 - Removing RDD 503 -2016-06-13 19:11:09 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Got job 84 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:11:09 INFO DAGScheduler:59 - Final stage: ResultStage 905(collect at KMeans.scala:431) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 897, ShuffleMapStage 898, ShuffleMapStage 902, ShuffleMapStage 900, ShuffleMapStage 904) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ResultStage 905 (MapPartitionsRDD[509] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(10536) called with curMem=418343, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_150 stored as values in memory (estimated size 10.3 KB, free 730.2 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(4460) called with curMem=428879, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_150_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.2 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_150_piece0 in memory on localhost:65433 (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 150 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 905 (MapPartitionsRDD[509] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 905.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 905.0 (TID 252, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 905.0 (TID 253, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 905.0 (TID 253) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 905.0 (TID 252) -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_507_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_507_1 locally -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 905.0 (TID 252). 2722 bytes result sent to driver -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 905.0 (TID 253). 2040 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 905.0 (TID 252) in 5 ms on localhost (1/2) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 905.0 (TID 253) in 4 ms on localhost (2/2) -2016-06-13 19:11:09 INFO DAGScheduler:59 - ResultStage 905 (collect at KMeans.scala:431) finished in 0.006 s -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 905.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - Job 84 finished: collect at KMeans.scala:431, took 0.018924 s -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(648) called with curMem=433339, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_151 stored as values in memory (estimated size 648.0 B, free 730.2 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(429) called with curMem=433987, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_151_piece0 stored as bytes in memory (estimated size 429.0 B, free 730.2 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_151_piece0 in memory on localhost:65433 (size: 429.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 151 from broadcast at KMeans.scala:394 -2016-06-13 19:11:09 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Got job 85 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:11:09 INFO DAGScheduler:59 - Final stage: ResultStage 917(aggregate at KMeans.scala:402) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 912, ShuffleMapStage 909, ShuffleMapStage 916, ShuffleMapStage 910, ShuffleMapStage 914) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ResultStage 917 (MapPartitionsRDD[511] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(10608) called with curMem=434416, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_152 stored as values in memory (estimated size 10.4 KB, free 730.2 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(4431) called with curMem=445024, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_152_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.2 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_152_piece0 in memory on localhost:65433 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 152 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 917 (MapPartitionsRDD[511] at map at KMeans.scala:396) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 917.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 917.0 (TID 254, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 917.0 (TID 255, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 917.0 (TID 254) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 917.0 (TID 255) -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_511_1 not found, computing it -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_511_0 not found, computing it -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_507_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_507_1 locally -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=449455, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_511_0 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=449951, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_511_1 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_511_0 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_511_1 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 917.0 (TID 254). 2647 bytes result sent to driver -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 917.0 (TID 255). 2647 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 917.0 (TID 254) in 10 ms on localhost (1/2) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 917.0 (TID 255) in 10 ms on localhost (2/2) -2016-06-13 19:11:09 INFO DAGScheduler:59 - ResultStage 917 (aggregate at KMeans.scala:402) finished in 0.010 s -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 917.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - Job 85 finished: aggregate at KMeans.scala:402, took 0.025183 s -2016-06-13 19:11:09 INFO MapPartitionsRDD:59 - Removing RDD 507 from persistence list -2016-06-13 19:11:09 INFO BlockManager:59 - Removing RDD 507 -2016-06-13 19:11:09 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Got job 86 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:11:09 INFO DAGScheduler:59 - Final stage: ResultStage 929(collect at KMeans.scala:431) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 924, ShuffleMapStage 921, ShuffleMapStage 928, ShuffleMapStage 922, ShuffleMapStage 926) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ResultStage 929 (MapPartitionsRDD[513] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(10768) called with curMem=449455, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_153 stored as values in memory (estimated size 10.5 KB, free 730.1 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(4505) called with curMem=460223, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_153_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.1 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_153_piece0 in memory on localhost:65433 (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 153 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 929 (MapPartitionsRDD[513] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 929.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 929.0 (TID 256, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 929.0 (TID 257, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 929.0 (TID 256) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 929.0 (TID 257) -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_511_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_511_0 locally -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 929.0 (TID 257). 2040 bytes result sent to driver -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 929.0 (TID 256). 2040 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 929.0 (TID 257) in 4 ms on localhost (1/2) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 929.0 (TID 256) in 6 ms on localhost (2/2) -2016-06-13 19:11:09 INFO DAGScheduler:59 - ResultStage 929 (collect at KMeans.scala:431) finished in 0.006 s -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 929.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - Job 86 finished: collect at KMeans.scala:431, took 0.016967 s -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(584) called with curMem=464728, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_154 stored as values in memory (estimated size 584.0 B, free 730.1 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(236) called with curMem=465312, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_154_piece0 stored as bytes in memory (estimated size 236.0 B, free 730.1 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_154_piece0 in memory on localhost:65433 (size: 236.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 154 from broadcast at KMeans.scala:394 -2016-06-13 19:11:09 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Got job 87 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:11:09 INFO DAGScheduler:59 - Final stage: ResultStage 941(aggregate at KMeans.scala:402) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 933, ShuffleMapStage 934, ShuffleMapStage 938, ShuffleMapStage 936, ShuffleMapStage 940) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ResultStage 941 (MapPartitionsRDD[515] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(10848) called with curMem=465548, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_155 stored as values in memory (estimated size 10.6 KB, free 730.1 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(4483) called with curMem=476396, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_155_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.1 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_155_piece0 in memory on localhost:65433 (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 155 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 941 (MapPartitionsRDD[515] at map at KMeans.scala:396) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 941.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 941.0 (TID 258, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 941.0 (TID 259, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 941.0 (TID 259) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 941.0 (TID 258) -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_515_1 not found, computing it -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_515_0 not found, computing it -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_511_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_511_0 locally -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=480879, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_515_1 stored as values in memory (estimated size 496.0 B, free 730.1 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=481375, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_515_0 stored as values in memory (estimated size 496.0 B, free 730.1 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_515_1 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_515_0 in memory on localhost:65433 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 941.0 (TID 259). 2647 bytes result sent to driver -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 941.0 (TID 258). 2647 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 941.0 (TID 259) in 13 ms on localhost (1/2) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 941.0 (TID 258) in 14 ms on localhost (2/2) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 941.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - ResultStage 941 (aggregate at KMeans.scala:402) finished in 0.015 s -2016-06-13 19:11:09 INFO DAGScheduler:59 - Job 87 finished: aggregate at KMeans.scala:402, took 0.028288 s -2016-06-13 19:11:09 INFO MapPartitionsRDD:59 - Removing RDD 511 from persistence list -2016-06-13 19:11:09 INFO BlockManager:59 - Removing RDD 511 -2016-06-13 19:11:09 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Got job 88 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:11:09 INFO DAGScheduler:59 - Final stage: ResultStage 953(collect at KMeans.scala:431) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 948, ShuffleMapStage 945, ShuffleMapStage 952, ShuffleMapStage 946, ShuffleMapStage 950) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ResultStage 953 (MapPartitionsRDD[517] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=480879, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_156 stored as values in memory (estimated size 10.8 KB, free 730.1 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(4564) called with curMem=491887, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_156_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.1 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_156_piece0 in memory on localhost:65433 (size: 4.5 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 156 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 953 (MapPartitionsRDD[517] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 953.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 953.0 (TID 260, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 953.0 (TID 261, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 953.0 (TID 261) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 953.0 (TID 260) -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_515_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 953.0 (TID 261). 2040 bytes result sent to driver -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_515_0 locally -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 953.0 (TID 261) in 4 ms on localhost (1/2) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 953.0 (TID 260). 2040 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 953.0 (TID 260) in 5 ms on localhost (2/2) -2016-06-13 19:11:09 INFO DAGScheduler:59 - ResultStage 953 (collect at KMeans.scala:431) finished in 0.006 s -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 953.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - Job 88 finished: collect at KMeans.scala:431, took 0.017709 s -2016-06-13 19:11:09 INFO MapPartitionsRDD:59 - Removing RDD 515 from persistence list -2016-06-13 19:11:09 INFO BlockManager:59 - Removing RDD 515 -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(2880) called with curMem=495955, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_157 stored as values in memory (estimated size 2.8 KB, free 730.1 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(708) called with curMem=498339, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_157_piece0 stored as bytes in memory (estimated size 708.0 B, free 730.1 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_157_piece0 in memory on localhost:65433 (size: 708.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 157 from broadcast at KMeans.scala:445 -2016-06-13 19:11:09 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:450 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Registering RDD 518 (flatMap at KMeans.scala:446) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Got job 89 (collectAsMap at KMeans.scala:450) with 2 output partitions -2016-06-13 19:11:09 INFO DAGScheduler:59 - Final stage: ResultStage 966(collectAsMap at KMeans.scala:450) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 965) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 965) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ShuffleMapStage 965 (MapPartitionsRDD[518] at flatMap at KMeans.scala:446), which has no missing parents -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(9392) called with curMem=499047, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_158 stored as values in memory (estimated size 9.2 KB, free 730.1 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(4066) called with curMem=508439, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_158_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.1 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_158_piece0 in memory on localhost:65433 (size: 4.0 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 158 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 965 (MapPartitionsRDD[518] at flatMap at KMeans.scala:446) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 965.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 965.0 (TID 262, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 965.0 (TID 263, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 965.0 (TID 263) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 965.0 (TID 262) -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 965.0 (TID 263). 2254 bytes result sent to driver -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 965.0 (TID 262). 2254 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 965.0 (TID 262) in 16 ms on localhost (1/2) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 965.0 (TID 263) in 17 ms on localhost (2/2) -2016-06-13 19:11:09 INFO DAGScheduler:59 - ShuffleMapStage 965 (flatMap at KMeans.scala:446) finished in 0.017 s -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 965.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:09 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:09 INFO DAGScheduler:59 - waiting: Set(ResultStage 966) -2016-06-13 19:11:09 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents for ResultStage 966: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ResultStage 966 (ShuffledRDD[519] at reduceByKey at KMeans.scala:450), which is now runnable -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=512505, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_159 stored as values in memory (estimated size 2.3 KB, free 730.1 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(1383) called with curMem=514817, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_159_piece0 stored as bytes in memory (estimated size 1383.0 B, free 730.1 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_159_piece0 in memory on localhost:65433 (size: 1383.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 159 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 966 (ShuffledRDD[519] at reduceByKey at KMeans.scala:450) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 966.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 966.0 (TID 264, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 966.0 (TID 265, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 966.0 (TID 264) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 966.0 (TID 265) -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 966.0 (TID 265). 1744 bytes result sent to driver -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 966.0 (TID 264). 1852 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 966.0 (TID 265) in 10 ms on localhost (1/2) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 966.0 (TID 264) in 10 ms on localhost (2/2) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 966.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - ResultStage 966 (collectAsMap at KMeans.scala:450) finished in 0.011 s -2016-06-13 19:11:09 INFO DAGScheduler:59 - Job 89 finished: collectAsMap at KMeans.scala:450, took 0.048592 s -2016-06-13 19:11:09 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:11:09 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:11:09 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:11:09 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:11:09 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:11:09 INFO KMeans:59 - Initialization with k-means|| took 0.549 seconds. -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(824) called with curMem=516200, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_160 stored as values in memory (estimated size 824.0 B, free 730.1 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(339) called with curMem=517024, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_160_piece0 stored as bytes in memory (estimated size 339.0 B, free 730.1 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_160_piece0 in memory on localhost:65433 (size: 339.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 160 from broadcast at KMeans.scala:276 -2016-06-13 19:11:09 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:302 -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 151 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 151 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 41 is 151 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 40 is 155 bytes -2016-06-13 19:11:09 INFO DAGScheduler:59 - Registering RDD 520 (mapPartitions at KMeans.scala:279) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Got job 90 (collectAsMap at KMeans.scala:302) with 2 output partitions -2016-06-13 19:11:09 INFO DAGScheduler:59 - Final stage: ResultStage 979(collectAsMap at KMeans.scala:302) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 978) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 978) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ShuffleMapStage 978 (MapPartitionsRDD[520] at mapPartitions at KMeans.scala:279), which has no missing parents -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(10016) called with curMem=517363, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_161 stored as values in memory (estimated size 9.8 KB, free 730.1 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(4398) called with curMem=527379, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_161_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.1 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_161_piece0 in memory on localhost:65433 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 161 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 978 (MapPartitionsRDD[520] at mapPartitions at KMeans.scala:279) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 978.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 978.0 (TID 266, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 978.0 (TID 267, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 978.0 (TID 267) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 978.0 (TID 266) -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 978.0 (TID 266). 2413 bytes result sent to driver -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 978.0 (TID 267). 2413 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 978.0 (TID 266) in 16 ms on localhost (1/2) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 978.0 (TID 267) in 16 ms on localhost (2/2) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 978.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - ShuffleMapStage 978 (mapPartitions at KMeans.scala:279) finished in 0.017 s -2016-06-13 19:11:09 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:09 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:09 INFO DAGScheduler:59 - waiting: Set(ResultStage 979) -2016-06-13 19:11:09 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents for ResultStage 979: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ResultStage 979 (ShuffledRDD[521] at reduceByKey at KMeans.scala:302), which is now runnable -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(2808) called with curMem=531777, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_162 stored as values in memory (estimated size 2.7 KB, free 730.1 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(1591) called with curMem=534585, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_162_piece0 stored as bytes in memory (estimated size 1591.0 B, free 730.1 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_162_piece0 in memory on localhost:65433 (size: 1591.0 B, free: 730.4 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 162 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 979 (ShuffledRDD[521] at reduceByKey at KMeans.scala:302) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 979.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 979.0 (TID 268, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 979.0 (TID 269, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 979.0 (TID 268) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 979.0 (TID 269) -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 979.0 (TID 269). 1671 bytes result sent to driver -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 979.0 (TID 268). 1785 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 979.0 (TID 269) in 8 ms on localhost (1/2) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 979.0 (TID 268) in 9 ms on localhost (2/2) -2016-06-13 19:11:09 INFO DAGScheduler:59 - ResultStage 979 (collectAsMap at KMeans.scala:302) finished in 0.010 s -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 979.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - Job 90 finished: collectAsMap at KMeans.scala:302, took 0.056942 s -2016-06-13 19:11:09 INFO KMeans:59 - Run 0 finished in 1 iterations -2016-06-13 19:11:09 INFO KMeans:59 - Run 1 finished in 1 iterations -2016-06-13 19:11:09 INFO KMeans:59 - Run 2 finished in 1 iterations -2016-06-13 19:11:09 INFO KMeans:59 - Run 3 finished in 1 iterations -2016-06-13 19:11:09 INFO KMeans:59 - Run 4 finished in 1 iterations -2016-06-13 19:11:09 INFO KMeans:59 - Iterations took 0.084 seconds. -2016-06-13 19:11:09 INFO KMeans:59 - KMeans converged in 1 iterations. -2016-06-13 19:11:09 INFO KMeans:59 - The cost for the best run is 6.650150218788335E-6. -2016-06-13 19:11:09 INFO MapPartitionsRDD:59 - Removing RDD 493 from persistence list -2016-06-13 19:11:09 INFO BlockManager:59 - Removing RDD 493 -2016-06-13 19:11:09 WARN KMeans:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:11:09 INFO SparkContext:59 - Starting job: collect at PowerIterationClusteringSuite.scala:114 -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 151 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 151 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 41 is 151 bytes -2016-06-13 19:11:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 40 is 155 bytes -2016-06-13 19:11:09 INFO DAGScheduler:59 - Got job 91 (collect at PowerIterationClusteringSuite.scala:114) with 2 output partitions -2016-06-13 19:11:09 INFO DAGScheduler:59 - Final stage: ResultStage 991(collect at PowerIterationClusteringSuite.scala:114) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 984, ShuffleMapStage 988, ShuffleMapStage 986, ShuffleMapStage 983, ShuffleMapStage 990) -2016-06-13 19:11:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting ResultStage 991 (MapPartitionsRDD[524] at mapPartitions at PowerIterationClustering.scala:232), which has no missing parents -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(8192) called with curMem=535696, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_163 stored as values in memory (estimated size 8.0 KB, free 730.1 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(3569) called with curMem=543888, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block broadcast_163_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.1 MB) -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added broadcast_163_piece0 in memory on localhost:65433 (size: 3.5 KB, free: 730.3 MB) -2016-06-13 19:11:09 INFO SparkContext:59 - Created broadcast 163 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 991 (MapPartitionsRDD[524] at mapPartitions at PowerIterationClustering.scala:232) -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Adding task set 991.0 with 2 tasks -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 991.0 (TID 270, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 991.0 (TID 271, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:11:09 INFO Executor:59 - Running task 1.0 in stage 991.0 (TID 271) -2016-06-13 19:11:09 INFO Executor:59 - Running task 0.0 in stage 991.0 (TID 270) -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_522_0 not found, computing it -2016-06-13 19:11:09 INFO CacheManager:59 - Partition rdd_522_1 not found, computing it -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:11:09 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=547457, maxMem=766075207 -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_522_1 stored as values in memory (estimated size 1168.0 B, free 730.1 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=548625, maxMem=766075207 -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_522_1 in memory on localhost:65433 (size: 1168.0 B, free: 730.3 MB) -2016-06-13 19:11:09 INFO MemoryStore:59 - Block rdd_522_0 stored as values in memory (estimated size 1168.0 B, free 730.1 MB) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 1.0 in stage 991.0 (TID 271). 2919 bytes result sent to driver -2016-06-13 19:11:09 INFO BlockManagerInfo:59 - Added rdd_522_0 in memory on localhost:65433 (size: 1168.0 B, free: 730.3 MB) -2016-06-13 19:11:09 INFO Executor:59 - Finished task 0.0 in stage 991.0 (TID 270). 2919 bytes result sent to driver -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 991.0 (TID 271) in 8 ms on localhost (1/2) -2016-06-13 19:11:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 991.0 (TID 270) in 9 ms on localhost (2/2) -2016-06-13 19:11:09 INFO DAGScheduler:59 - ResultStage 991 (collect at PowerIterationClusteringSuite.scala:114) finished in 0.009 s -2016-06-13 19:11:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 991.0, whose tasks have all completed, from pool -2016-06-13 19:11:09 INFO DAGScheduler:59 - Job 91 finished: collect at PowerIterationClusteringSuite.scala:114, took 0.031225 s -2016-06-13 19:11:10 INFO SparkContext:59 - Starting job: collect at EdgeRDDImpl.scala:51 -2016-06-13 19:11:10 INFO DAGScheduler:59 - Registering RDD 530 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Registering RDD 536 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Registering RDD 540 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Got job 92 (collect at EdgeRDDImpl.scala:51) with 2 output partitions -2016-06-13 19:11:10 INFO DAGScheduler:59 - Final stage: ResultStage 995(collect at EdgeRDDImpl.scala:51) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 994) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 994) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 992 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[530] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(3016) called with curMem=549793, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_164 stored as values in memory (estimated size 2.9 KB, free 730.1 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(1777) called with curMem=552809, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_164_piece0 stored as bytes in memory (estimated size 1777.0 B, free 730.1 MB) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added broadcast_164_piece0 in memory on localhost:65433 (size: 1777.0 B, free: 730.3 MB) -2016-06-13 19:11:10 INFO SparkContext:59 - Created broadcast 164 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 992 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[530] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Adding task set 992.0 with 2 tasks -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 993 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[536] at mapPartitions at GraphImpl.scala:235), which has no missing parents -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 992.0 (TID 272, localhost, PROCESS_LOCAL, 2242 bytes) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 992.0 (TID 273, localhost, PROCESS_LOCAL, 2281 bytes) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(3432) called with curMem=554586, maxMem=766075207 -2016-06-13 19:11:10 INFO Executor:59 - Running task 0.0 in stage 992.0 (TID 272) -2016-06-13 19:11:10 INFO Executor:59 - Running task 1.0 in stage 992.0 (TID 273) -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_165 stored as values in memory (estimated size 3.4 KB, free 730.1 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(1972) called with curMem=558018, maxMem=766075207 -2016-06-13 19:11:10 INFO CacheManager:59 - Partition rdd_527_0 not found, computing it -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_165_piece0 stored as bytes in memory (estimated size 1972.0 B, free 730.1 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(2568) called with curMem=559990, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block rdd_527_0 stored as values in memory (estimated size 2.5 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added broadcast_165_piece0 in memory on localhost:65433 (size: 1972.0 B, free: 730.3 MB) -2016-06-13 19:11:10 INFO SparkContext:59 - Created broadcast 165 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added rdd_527_0 in memory on localhost:65433 (size: 2.5 KB, free: 730.3 MB) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 993 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[536] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Adding task set 993.0 with 2 tasks -2016-06-13 19:11:10 INFO CacheManager:59 - Partition rdd_527_1 not found, computing it -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 993.0 (TID 274, localhost, PROCESS_LOCAL, 2242 bytes) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 993.0 (TID 275, localhost, PROCESS_LOCAL, 2281 bytes) -2016-06-13 19:11:10 INFO Executor:59 - Running task 1.0 in stage 993.0 (TID 275) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(2616) called with curMem=562558, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block rdd_527_1 stored as values in memory (estimated size 2.6 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO Executor:59 - Running task 0.0 in stage 993.0 (TID 274) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added rdd_527_1 in memory on localhost:65433 (size: 2.6 KB, free: 730.3 MB) -2016-06-13 19:11:10 INFO BlockManager:59 - Found block rdd_527_0 locally -2016-06-13 19:11:10 INFO BlockManager:59 - Found block rdd_527_1 locally -2016-06-13 19:11:10 INFO Executor:59 - Finished task 0.0 in stage 992.0 (TID 272). 1739 bytes result sent to driver -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 992.0 (TID 272) in 33 ms on localhost (1/2) -2016-06-13 19:11:10 INFO Executor:59 - Finished task 1.0 in stage 992.0 (TID 273). 1739 bytes result sent to driver -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 992.0 (TID 273) in 44 ms on localhost (2/2) -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 992.0, whose tasks have all completed, from pool -2016-06-13 19:11:10 INFO DAGScheduler:59 - ShuffleMapStage 992 (mapPartitions at VertexRDD.scala:358) finished in 0.044 s -2016-06-13 19:11:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:10 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 993) -2016-06-13 19:11:10 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 994, ResultStage 995) -2016-06-13 19:11:10 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:10 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 994: List(ShuffleMapStage 993) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Missing parents for ResultStage 995: List(ShuffleMapStage 994) -2016-06-13 19:11:10 INFO Executor:59 - Finished task 0.0 in stage 993.0 (TID 274). 2254 bytes result sent to driver -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 993.0 (TID 274) in 43 ms on localhost (1/2) -2016-06-13 19:11:10 INFO Executor:59 - Finished task 1.0 in stage 993.0 (TID 275). 2254 bytes result sent to driver -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 993.0 (TID 275) in 44 ms on localhost (2/2) -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 993.0, whose tasks have all completed, from pool -2016-06-13 19:11:10 INFO DAGScheduler:59 - ShuffleMapStage 993 (mapPartitions at GraphImpl.scala:235) finished in 0.046 s -2016-06-13 19:11:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:10 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:10 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 994, ResultStage 995) -2016-06-13 19:11:10 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:10 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 994: List() -2016-06-13 19:11:10 INFO DAGScheduler:59 - Missing parents for ResultStage 995: List(ShuffleMapStage 994) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 994 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[540] at mapPartitions at VertexRDDImpl.scala:247), which is now runnable -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=565174, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_166 stored as values in memory (estimated size 4.1 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(2376) called with curMem=569422, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_166_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added broadcast_166_piece0 in memory on localhost:65433 (size: 2.3 KB, free: 730.3 MB) -2016-06-13 19:11:10 INFO SparkContext:59 - Created broadcast 166 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 994 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[540] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Adding task set 994.0 with 2 tasks -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 994.0 (TID 276, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 994.0 (TID 277, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:11:10 INFO Executor:59 - Running task 0.0 in stage 994.0 (TID 276) -2016-06-13 19:11:10 INFO Executor:59 - Running task 1.0 in stage 994.0 (TID 277) -2016-06-13 19:11:10 INFO CacheManager:59 - Partition rdd_538_0 not found, computing it -2016-06-13 19:11:10 INFO CacheManager:59 - Partition rdd_533_0 not found, computing it -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:10 INFO CacheManager:59 - Partition rdd_538_1 not found, computing it -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:10 INFO CacheManager:59 - Partition rdd_533_1 not found, computing it -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(1744) called with curMem=571798, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block rdd_533_0 stored as values in memory (estimated size 1744.0 B, free 730.0 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(1736) called with curMem=573542, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block rdd_533_1 stored as values in memory (estimated size 1736.0 B, free 730.0 MB) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added rdd_533_0 in memory on localhost:65433 (size: 1744.0 B, free: 730.3 MB) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added rdd_533_1 in memory on localhost:65433 (size: 1736.0 B, free: 730.3 MB) -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(1784) called with curMem=575278, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block rdd_538_1 stored as values in memory (estimated size 1784.0 B, free 730.0 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(1792) called with curMem=577062, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block rdd_538_0 stored as values in memory (estimated size 1792.0 B, free 730.0 MB) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added rdd_538_1 in memory on localhost:65433 (size: 1784.0 B, free: 730.3 MB) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added rdd_538_0 in memory on localhost:65433 (size: 1792.0 B, free: 730.3 MB) -2016-06-13 19:11:10 INFO Executor:59 - Finished task 0.0 in stage 994.0 (TID 276). 2010 bytes result sent to driver -2016-06-13 19:11:10 INFO Executor:59 - Finished task 1.0 in stage 994.0 (TID 277). 2010 bytes result sent to driver -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 994.0 (TID 276) in 34 ms on localhost (1/2) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 994.0 (TID 277) in 34 ms on localhost (2/2) -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 994.0, whose tasks have all completed, from pool -2016-06-13 19:11:10 INFO DAGScheduler:59 - ShuffleMapStage 994 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.035 s -2016-06-13 19:11:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:10 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:10 INFO DAGScheduler:59 - waiting: Set(ResultStage 995) -2016-06-13 19:11:10 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:10 INFO DAGScheduler:59 - Missing parents for ResultStage 995: List() -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting ResultStage 995 (MapPartitionsRDD[546] at map at EdgeRDDImpl.scala:51), which is now runnable -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(4872) called with curMem=578854, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_167 stored as values in memory (estimated size 4.8 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(2549) called with curMem=583726, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_167_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added broadcast_167_piece0 in memory on localhost:65433 (size: 2.5 KB, free: 730.3 MB) -2016-06-13 19:11:10 INFO SparkContext:59 - Created broadcast 167 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 995 (MapPartitionsRDD[546] at map at EdgeRDDImpl.scala:51) -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Adding task set 995.0 with 2 tasks -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 995.0 (TID 278, localhost, PROCESS_LOCAL, 2502 bytes) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 995.0 (TID 279, localhost, PROCESS_LOCAL, 2541 bytes) -2016-06-13 19:11:10 INFO Executor:59 - Running task 0.0 in stage 995.0 (TID 278) -2016-06-13 19:11:10 INFO Executor:59 - Running task 1.0 in stage 995.0 (TID 279) -2016-06-13 19:11:10 INFO BlockManager:59 - Found block rdd_527_1 locally -2016-06-13 19:11:10 INFO BlockManager:59 - Found block rdd_527_0 locally -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:10 INFO Executor:59 - Finished task 0.0 in stage 995.0 (TID 278). 2568 bytes result sent to driver -2016-06-13 19:11:10 INFO Executor:59 - Finished task 1.0 in stage 995.0 (TID 279). 2640 bytes result sent to driver -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 995.0 (TID 278) in 10 ms on localhost (1/2) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 995.0 (TID 279) in 9 ms on localhost (2/2) -2016-06-13 19:11:10 INFO DAGScheduler:59 - ResultStage 995 (collect at EdgeRDDImpl.scala:51) finished in 0.010 s -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 995.0, whose tasks have all completed, from pool -2016-06-13 19:11:10 INFO DAGScheduler:59 - Job 92 finished: collect at EdgeRDDImpl.scala:51, took 0.133420 s -2016-06-13 19:11:10 INFO SparkContext:59 - Starting job: reduce at VertexRDDImpl.scala:90 -2016-06-13 19:11:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 46 is 154 bytes -2016-06-13 19:11:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 45 is 154 bytes -2016-06-13 19:11:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 44 is 152 bytes -2016-06-13 19:11:10 INFO DAGScheduler:59 - Registering RDD 552 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Registering RDD 547 (parallelize at PowerIterationClusteringSuite.scala:155) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Got job 93 (reduce at VertexRDDImpl.scala:90) with 2 output partitions -2016-06-13 19:11:10 INFO DAGScheduler:59 - Final stage: ResultStage 1001(reduce at VertexRDDImpl.scala:90) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 999, ShuffleMapStage 1000) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 999, ShuffleMapStage 1000) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 999 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[552] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(4616) called with curMem=586275, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_168 stored as values in memory (estimated size 4.5 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(2474) called with curMem=590891, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_168_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added broadcast_168_piece0 in memory on localhost:65433 (size: 2.4 KB, free: 730.3 MB) -2016-06-13 19:11:10 INFO SparkContext:59 - Created broadcast 168 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 999 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[552] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Adding task set 999.0 with 2 tasks -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1000 (ParallelCollectionRDD[547] at parallelize at PowerIterationClusteringSuite.scala:155), which has no missing parents -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(1520) called with curMem=593365, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_169 stored as values in memory (estimated size 1520.0 B, free 730.0 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(984) called with curMem=594885, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_169_piece0 stored as bytes in memory (estimated size 984.0 B, free 730.0 MB) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 999.0 (TID 280, localhost, PROCESS_LOCAL, 2491 bytes) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 999.0 (TID 281, localhost, PROCESS_LOCAL, 2530 bytes) -2016-06-13 19:11:10 INFO Executor:59 - Running task 0.0 in stage 999.0 (TID 280) -2016-06-13 19:11:10 INFO Executor:59 - Running task 1.0 in stage 999.0 (TID 281) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added broadcast_169_piece0 in memory on localhost:65433 (size: 984.0 B, free: 730.3 MB) -2016-06-13 19:11:10 INFO BlockManager:59 - Found block rdd_527_1 locally -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:10 INFO BlockManager:59 - Found block rdd_527_0 locally -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:10 INFO SparkContext:59 - Created broadcast 169 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1000 (ParallelCollectionRDD[547] at parallelize at PowerIterationClusteringSuite.scala:155) -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Adding task set 1000.0 with 2 tasks -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 1000.0 (TID 282, localhost, PROCESS_LOCAL, 2222 bytes) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 1000.0 (TID 283, localhost, PROCESS_LOCAL, 2222 bytes) -2016-06-13 19:11:10 INFO Executor:59 - Running task 1.0 in stage 1000.0 (TID 283) -2016-06-13 19:11:10 INFO Executor:59 - Running task 0.0 in stage 1000.0 (TID 282) -2016-06-13 19:11:10 INFO Executor:59 - Finished task 1.0 in stage 999.0 (TID 281). 2470 bytes result sent to driver -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 999.0 (TID 281) in 29 ms on localhost (1/2) -2016-06-13 19:11:10 INFO Executor:59 - Finished task 0.0 in stage 999.0 (TID 280). 2470 bytes result sent to driver -2016-06-13 19:11:10 INFO Executor:59 - Finished task 1.0 in stage 1000.0 (TID 283). 1159 bytes result sent to driver -2016-06-13 19:11:10 INFO Executor:59 - Finished task 0.0 in stage 1000.0 (TID 282). 1159 bytes result sent to driver -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 999.0 (TID 280) in 46 ms on localhost (2/2) -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 999.0, whose tasks have all completed, from pool -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 1000.0 (TID 282) in 34 ms on localhost (1/2) -2016-06-13 19:11:10 INFO DAGScheduler:59 - ShuffleMapStage 999 (mapPartitions at VertexRDD.scala:358) finished in 0.050 s -2016-06-13 19:11:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:10 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 1000) -2016-06-13 19:11:10 INFO DAGScheduler:59 - waiting: Set(ResultStage 1001) -2016-06-13 19:11:10 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 1000.0 (TID 283) in 32 ms on localhost (2/2) -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 1000.0, whose tasks have all completed, from pool -2016-06-13 19:11:10 INFO DAGScheduler:59 - Missing parents for ResultStage 1001: List(ShuffleMapStage 1000) -2016-06-13 19:11:10 INFO DAGScheduler:59 - ShuffleMapStage 1000 (parallelize at PowerIterationClusteringSuite.scala:155) finished in 0.036 s -2016-06-13 19:11:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:10 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:10 INFO DAGScheduler:59 - waiting: Set(ResultStage 1001) -2016-06-13 19:11:10 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:10 INFO DAGScheduler:59 - Missing parents for ResultStage 1001: List() -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting ResultStage 1001 (MapPartitionsRDD[560] at map at VertexRDDImpl.scala:90), which is now runnable -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(4016) called with curMem=595869, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_170 stored as values in memory (estimated size 3.9 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(2203) called with curMem=599885, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_170_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added broadcast_170_piece0 in memory on localhost:65433 (size: 2.2 KB, free: 730.3 MB) -2016-06-13 19:11:10 INFO SparkContext:59 - Created broadcast 170 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1001 (MapPartitionsRDD[560] at map at VertexRDDImpl.scala:90) -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Adding task set 1001.0 with 2 tasks -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 1001.0 (TID 284, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 1001.0 (TID 285, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:11:10 INFO Executor:59 - Running task 1.0 in stage 1001.0 (TID 285) -2016-06-13 19:11:10 INFO Executor:59 - Running task 0.0 in stage 1001.0 (TID 284) -2016-06-13 19:11:10 INFO CacheManager:59 - Partition rdd_555_0 not found, computing it -2016-06-13 19:11:10 INFO CacheManager:59 - Partition rdd_555_1 not found, computing it -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(1736) called with curMem=602088, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block rdd_555_1 stored as values in memory (estimated size 1736.0 B, free 730.0 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(1744) called with curMem=603824, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block rdd_555_0 stored as values in memory (estimated size 1744.0 B, free 730.0 MB) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added rdd_555_1 in memory on localhost:65433 (size: 1736.0 B, free: 730.3 MB) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added rdd_555_0 in memory on localhost:65433 (size: 1744.0 B, free: 730.3 MB) -2016-06-13 19:11:10 INFO Executor:59 - Finished task 1.0 in stage 1001.0 (TID 285). 1862 bytes result sent to driver -2016-06-13 19:11:10 INFO Executor:59 - Finished task 0.0 in stage 1001.0 (TID 284). 1862 bytes result sent to driver -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 1001.0 (TID 285) in 13 ms on localhost (1/2) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 1001.0 (TID 284) in 14 ms on localhost (2/2) -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 1001.0, whose tasks have all completed, from pool -2016-06-13 19:11:10 INFO DAGScheduler:59 - ResultStage 1001 (reduce at VertexRDDImpl.scala:90) finished in 0.016 s -2016-06-13 19:11:10 INFO DAGScheduler:59 - Job 93 finished: reduce at VertexRDDImpl.scala:90, took 0.098961 s -2016-06-13 19:11:10 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:11:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 48 is 152 bytes -2016-06-13 19:11:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 46 is 154 bytes -2016-06-13 19:11:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 45 is 154 bytes -2016-06-13 19:11:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 44 is 152 bytes -2016-06-13 19:11:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 47 is 154 bytes -2016-06-13 19:11:10 INFO DAGScheduler:59 - Registering RDD 561 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Registering RDD 565 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Got job 94 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:11:10 INFO DAGScheduler:59 - Final stage: ResultStage 1009(sum at PowerIterationClustering.scala:369) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1002, ShuffleMapStage 1008, ShuffleMapStage 1006) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1008) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1007 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[561] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(3720) called with curMem=605568, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_171 stored as values in memory (estimated size 3.6 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(2111) called with curMem=609288, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_171_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added broadcast_171_piece0 in memory on localhost:65433 (size: 2.1 KB, free: 730.3 MB) -2016-06-13 19:11:10 INFO SparkContext:59 - Created broadcast 171 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1007 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[561] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Adding task set 1007.0 with 2 tasks -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 1007.0 (TID 286, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 1007.0 (TID 287, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:11:10 INFO Executor:59 - Running task 1.0 in stage 1007.0 (TID 287) -2016-06-13 19:11:10 INFO Executor:59 - Running task 0.0 in stage 1007.0 (TID 286) -2016-06-13 19:11:10 INFO BlockManager:59 - Found block rdd_555_0 locally -2016-06-13 19:11:10 INFO BlockManager:59 - Found block rdd_555_1 locally -2016-06-13 19:11:10 INFO Executor:59 - Finished task 0.0 in stage 1007.0 (TID 286). 2254 bytes result sent to driver -2016-06-13 19:11:10 INFO Executor:59 - Finished task 1.0 in stage 1007.0 (TID 287). 2254 bytes result sent to driver -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 1007.0 (TID 286) in 21 ms on localhost (1/2) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 1007.0 (TID 287) in 20 ms on localhost (2/2) -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 1007.0, whose tasks have all completed, from pool -2016-06-13 19:11:10 INFO DAGScheduler:59 - ShuffleMapStage 1007 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.022 s -2016-06-13 19:11:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:10 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:10 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 1008, ResultStage 1009) -2016-06-13 19:11:10 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:10 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 1008: List() -2016-06-13 19:11:10 INFO DAGScheduler:59 - Missing parents for ResultStage 1009: List(ShuffleMapStage 1008) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1008 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[565] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(5352) called with curMem=611399, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_172 stored as values in memory (estimated size 5.2 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(2725) called with curMem=616751, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_172_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added broadcast_172_piece0 in memory on localhost:65433 (size: 2.7 KB, free: 730.3 MB) -2016-06-13 19:11:10 INFO SparkContext:59 - Created broadcast 172 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1008 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[565] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Adding task set 1008.0 with 2 tasks -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 1008.0 (TID 288, localhost, PROCESS_LOCAL, 2532 bytes) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 1008.0 (TID 289, localhost, PROCESS_LOCAL, 2571 bytes) -2016-06-13 19:11:10 INFO Executor:59 - Running task 0.0 in stage 1008.0 (TID 288) -2016-06-13 19:11:10 INFO Executor:59 - Running task 1.0 in stage 1008.0 (TID 289) -2016-06-13 19:11:10 INFO CacheManager:59 - Partition rdd_558_1 not found, computing it -2016-06-13 19:11:10 INFO CacheManager:59 - Partition rdd_558_0 not found, computing it -2016-06-13 19:11:10 INFO BlockManager:59 - Found block rdd_527_1 locally -2016-06-13 19:11:10 INFO BlockManager:59 - Found block rdd_527_0 locally -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(2616) called with curMem=619476, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block rdd_558_1 stored as values in memory (estimated size 2.6 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(2568) called with curMem=622092, maxMem=766075207 -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added rdd_558_1 in memory on localhost:65433 (size: 2.6 KB, free: 730.3 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - Block rdd_558_0 stored as values in memory (estimated size 2.5 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added rdd_558_0 in memory on localhost:65433 (size: 2.5 KB, free: 730.3 MB) -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:10 INFO Executor:59 - Finished task 0.0 in stage 1008.0 (TID 288). 3050 bytes result sent to driver -2016-06-13 19:11:10 INFO Executor:59 - Finished task 1.0 in stage 1008.0 (TID 289). 3050 bytes result sent to driver -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 1008.0 (TID 288) in 27 ms on localhost (1/2) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 1008.0 (TID 289) in 27 ms on localhost (2/2) -2016-06-13 19:11:10 INFO DAGScheduler:59 - ShuffleMapStage 1008 (mapPartitions at GraphImpl.scala:235) finished in 0.028 s -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 1008.0, whose tasks have all completed, from pool -2016-06-13 19:11:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:10 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:10 INFO DAGScheduler:59 - waiting: Set(ResultStage 1009) -2016-06-13 19:11:10 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:10 INFO DAGScheduler:59 - Missing parents for ResultStage 1009: List() -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting ResultStage 1009 (MapPartitionsRDD[570] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(5144) called with curMem=624660, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_173 stored as values in memory (estimated size 5.0 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(2659) called with curMem=629804, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_173_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added broadcast_173_piece0 in memory on localhost:65433 (size: 2.6 KB, free: 730.3 MB) -2016-06-13 19:11:10 INFO SparkContext:59 - Created broadcast 173 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1009 (MapPartitionsRDD[570] at map at PowerIterationClustering.scala:369) -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Adding task set 1009.0 with 2 tasks -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 1009.0 (TID 290, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 1009.0 (TID 291, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:11:10 INFO Executor:59 - Running task 1.0 in stage 1009.0 (TID 291) -2016-06-13 19:11:10 INFO Executor:59 - Running task 0.0 in stage 1009.0 (TID 290) -2016-06-13 19:11:10 INFO CacheManager:59 - Partition rdd_567_0 not found, computing it -2016-06-13 19:11:10 INFO CacheManager:59 - Partition rdd_567_1 not found, computing it -2016-06-13 19:11:10 INFO BlockManager:59 - Found block rdd_555_0 locally -2016-06-13 19:11:10 INFO BlockManager:59 - Found block rdd_555_1 locally -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(1784) called with curMem=632463, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block rdd_567_1 stored as values in memory (estimated size 1784.0 B, free 730.0 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(1792) called with curMem=634247, maxMem=766075207 -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added rdd_567_1 in memory on localhost:65433 (size: 1784.0 B, free: 730.3 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - Block rdd_567_0 stored as values in memory (estimated size 1792.0 B, free 730.0 MB) -2016-06-13 19:11:10 INFO Executor:59 - Finished task 1.0 in stage 1009.0 (TID 291). 2880 bytes result sent to driver -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added rdd_567_0 in memory on localhost:65433 (size: 1792.0 B, free: 730.3 MB) -2016-06-13 19:11:10 INFO Executor:59 - Finished task 0.0 in stage 1009.0 (TID 290). 2880 bytes result sent to driver -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 1009.0 (TID 291) in 8 ms on localhost (1/2) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 1009.0 (TID 290) in 10 ms on localhost (2/2) -2016-06-13 19:11:10 INFO DAGScheduler:59 - ResultStage 1009 (sum at PowerIterationClustering.scala:369) finished in 0.010 s -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 1009.0, whose tasks have all completed, from pool -2016-06-13 19:11:10 INFO DAGScheduler:59 - Job 94 finished: sum at PowerIterationClustering.scala:369, took 0.092090 s -2016-06-13 19:11:10 INFO PowerIterationClustering:59 - Iteration 0: norm(v) = 0.9333333333333333. -2016-06-13 19:11:10 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:11:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 48 is 152 bytes -2016-06-13 19:11:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 46 is 154 bytes -2016-06-13 19:11:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 45 is 154 bytes -2016-06-13 19:11:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 44 is 152 bytes -2016-06-13 19:11:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 47 is 154 bytes -2016-06-13 19:11:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 50 is 152 bytes -2016-06-13 19:11:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 49 is 154 bytes -2016-06-13 19:11:10 INFO DAGScheduler:59 - Got job 95 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:11:10 INFO DAGScheduler:59 - Final stage: ResultStage 1017(sum at PowerIterationClustering.scala:375) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1014, ShuffleMapStage 1016, ShuffleMapStage 1010) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting ResultStage 1017 (MapPartitionsRDD[581] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=636039, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_174 stored as values in memory (estimated size 5.7 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(2910) called with curMem=641919, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_174_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added broadcast_174_piece0 in memory on localhost:65433 (size: 2.8 KB, free: 730.3 MB) -2016-06-13 19:11:10 INFO SparkContext:59 - Created broadcast 174 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1017 (MapPartitionsRDD[581] at values at PowerIterationClustering.scala:375) -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Adding task set 1017.0 with 2 tasks -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 1017.0 (TID 292, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 1017.0 (TID 293, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:11:10 INFO Executor:59 - Running task 1.0 in stage 1017.0 (TID 293) -2016-06-13 19:11:10 INFO Executor:59 - Running task 0.0 in stage 1017.0 (TID 292) -2016-06-13 19:11:10 INFO CacheManager:59 - Partition rdd_573_1 not found, computing it -2016-06-13 19:11:10 INFO BlockManager:59 - Found block rdd_555_1 locally -2016-06-13 19:11:10 INFO BlockManager:59 - Found block rdd_567_1 locally -2016-06-13 19:11:10 INFO CacheManager:59 - Partition rdd_573_0 not found, computing it -2016-06-13 19:11:10 INFO BlockManager:59 - Found block rdd_555_0 locally -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(1736) called with curMem=644829, maxMem=766075207 -2016-06-13 19:11:10 INFO BlockManager:59 - Found block rdd_567_0 locally -2016-06-13 19:11:10 INFO MemoryStore:59 - Block rdd_573_1 stored as values in memory (estimated size 1736.0 B, free 730.0 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(1744) called with curMem=646565, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block rdd_573_0 stored as values in memory (estimated size 1744.0 B, free 730.0 MB) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added rdd_573_1 in memory on localhost:65433 (size: 1736.0 B, free: 730.3 MB) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added rdd_573_0 in memory on localhost:65433 (size: 1744.0 B, free: 730.3 MB) -2016-06-13 19:11:10 INFO Executor:59 - Finished task 1.0 in stage 1017.0 (TID 293). 2664 bytes result sent to driver -2016-06-13 19:11:10 INFO Executor:59 - Finished task 0.0 in stage 1017.0 (TID 292). 2664 bytes result sent to driver -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 1017.0 (TID 293) in 11 ms on localhost (1/2) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 1017.0 (TID 292) in 12 ms on localhost (2/2) -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 1017.0, whose tasks have all completed, from pool -2016-06-13 19:11:10 INFO DAGScheduler:59 - ResultStage 1017 (sum at PowerIterationClustering.scala:375) finished in 0.012 s -2016-06-13 19:11:10 INFO DAGScheduler:59 - Job 95 finished: sum at PowerIterationClustering.scala:375, took 0.025738 s -2016-06-13 19:11:10 INFO PowerIterationClustering:59 - Iteration 0: delta = 0.47142857142857136. -2016-06-13 19:11:10 INFO PowerIterationClustering:59 - Iteration 0: diff(delta) = 1.7976931348623157E308. -2016-06-13 19:11:10 INFO SparkContext:59 - Starting job: collect at PowerIterationClusteringSuite.scala:157 -2016-06-13 19:11:10 INFO DAGScheduler:59 - Got job 96 (collect at PowerIterationClusteringSuite.scala:157) with 2 output partitions -2016-06-13 19:11:10 INFO DAGScheduler:59 - Final stage: ResultStage 1025(collect at PowerIterationClusteringSuite.scala:157) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1018, ShuffleMapStage 1022, ShuffleMapStage 1024) -2016-06-13 19:11:10 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting ResultStage 1025 (VertexRDDImpl[583] at RDD at VertexRDD.scala:57), which has no missing parents -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=648309, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_175 stored as values in memory (estimated size 5.1 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO MemoryStore:59 - ensureFreeSpace(2626) called with curMem=653541, maxMem=766075207 -2016-06-13 19:11:10 INFO MemoryStore:59 - Block broadcast_175_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.0 MB) -2016-06-13 19:11:10 INFO BlockManagerInfo:59 - Added broadcast_175_piece0 in memory on localhost:65433 (size: 2.6 KB, free: 730.3 MB) -2016-06-13 19:11:10 INFO SparkContext:59 - Created broadcast 175 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1025 (VertexRDDImpl[583] at RDD at VertexRDD.scala:57) -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Adding task set 1025.0 with 2 tasks -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 1025.0 (TID 294, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 1025.0 (TID 295, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:11:10 INFO Executor:59 - Running task 0.0 in stage 1025.0 (TID 294) -2016-06-13 19:11:10 INFO Executor:59 - Running task 1.0 in stage 1025.0 (TID 295) -2016-06-13 19:11:10 INFO BlockManager:59 - Found block rdd_567_1 locally -2016-06-13 19:11:10 INFO BlockManager:59 - Found block rdd_567_0 locally -2016-06-13 19:11:10 INFO Executor:59 - Finished task 1.0 in stage 1025.0 (TID 295). 2273 bytes result sent to driver -2016-06-13 19:11:10 INFO Executor:59 - Finished task 0.0 in stage 1025.0 (TID 294). 2273 bytes result sent to driver -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 1025.0 (TID 295) in 3 ms on localhost (1/2) -2016-06-13 19:11:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 1025.0 (TID 294) in 3 ms on localhost (2/2) -2016-06-13 19:11:10 INFO DAGScheduler:59 - ResultStage 1025 (collect at PowerIterationClusteringSuite.scala:157) finished in 0.004 s -2016-06-13 19:11:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 1025.0, whose tasks have all completed, from pool -2016-06-13 19:11:10 INFO DAGScheduler:59 - Job 96 finished: collect at PowerIterationClusteringSuite.scala:157, took 0.012117 s -2016-06-13 19:11:11 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 19:11:11 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 19:11:11 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 19:11:11 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 19:11:11 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 19:11:11 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 19:11:12 INFO SparkContext:59 - Starting job: saveAsTextFile at PowerIterationClustering.scala:81 -2016-06-13 19:11:12 INFO DAGScheduler:59 - Got job 97 (saveAsTextFile at PowerIterationClustering.scala:81) with 1 output partitions -2016-06-13 19:11:12 INFO DAGScheduler:59 - Final stage: ResultStage 1026(saveAsTextFile at PowerIterationClustering.scala:81) -2016-06-13 19:11:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:11:12 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:12 INFO DAGScheduler:59 - Submitting ResultStage 1026 (MapPartitionsRDD[586] at saveAsTextFile at PowerIterationClustering.scala:81), which has no missing parents -2016-06-13 19:11:12 INFO MemoryStore:59 - ensureFreeSpace(126936) called with curMem=656167, maxMem=766075207 -2016-06-13 19:11:12 INFO MemoryStore:59 - Block broadcast_176 stored as values in memory (estimated size 124.0 KB, free 729.8 MB) -2016-06-13 19:11:12 INFO MemoryStore:59 - ensureFreeSpace(42531) called with curMem=783103, maxMem=766075207 -2016-06-13 19:11:12 INFO MemoryStore:59 - Block broadcast_176_piece0 stored as bytes in memory (estimated size 41.5 KB, free 729.8 MB) -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Added broadcast_176_piece0 in memory on localhost:65433 (size: 41.5 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO SparkContext:59 - Created broadcast 176 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:12 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1026 (MapPartitionsRDD[586] at saveAsTextFile at PowerIterationClustering.scala:81) -2016-06-13 19:11:12 INFO TaskSchedulerImpl:59 - Adding task set 1026.0 with 1 tasks -2016-06-13 19:11:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 1026.0 (TID 296, localhost, PROCESS_LOCAL, 2187 bytes) -2016-06-13 19:11:12 INFO Executor:59 - Running task 0.0 in stage 1026.0 (TID 296) -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_175_piece0 on localhost:65433 in memory (size: 2.6 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 168 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_174_piece0 on localhost:65433 in memory (size: 2.8 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 167 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 573 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 573 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_173_piece0 on localhost:65433 in memory (size: 2.6 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 166 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_172_piece0 on localhost:65433 in memory (size: 2.7 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 165 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_171_piece0 on localhost:65433 in memory (size: 2.1 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 164 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 50 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 49 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 567 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 567 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_170_piece0 on localhost:65433 in memory (size: 2.2 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 163 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_169_piece0 on localhost:65433 in memory (size: 984.0 B, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 162 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 48 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 47 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 558 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 558 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 555 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 555 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_167_piece0 on localhost:65433 in memory (size: 2.5 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 160 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_166_piece0 on localhost:65433 in memory (size: 2.3 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 159 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_165_piece0 on localhost:65433 in memory (size: 1972.0 B, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 158 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_164_piece0 on localhost:65433 in memory (size: 1777.0 B, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 157 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_163_piece0 on localhost:65433 in memory (size: 3.5 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 156 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 522 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 522 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_162_piece0 on localhost:65433 in memory (size: 1591.0 B, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 155 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_161_piece0 on localhost:65433 in memory (size: 4.3 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 154 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 43 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_160_piece0 on localhost:65433 in memory (size: 339.0 B, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 153 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 152 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 151 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 150 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 149 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_159_piece0 on localhost:65433 in memory (size: 1383.0 B, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 148 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_158_piece0 on localhost:65433 in memory (size: 4.0 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 147 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 42 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_157_piece0 on localhost:65433 in memory (size: 708.0 B, free: 730.3 MB) -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_156_piece0 on localhost:65433 in memory (size: 4.5 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 146 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_155_piece0 on localhost:65433 in memory (size: 4.4 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 145 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 515 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 515 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_154_piece0 on localhost:65433 in memory (size: 236.0 B, free: 730.3 MB) -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_153_piece0 on localhost:65433 in memory (size: 4.4 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 144 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_152_piece0 on localhost:65433 in memory (size: 4.3 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 143 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 511 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 511 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_151_piece0 on localhost:65433 in memory (size: 429.0 B, free: 730.3 MB) -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_150_piece0 on localhost:65433 in memory (size: 4.4 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 142 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_149_piece0 on localhost:65433 in memory (size: 4.3 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 141 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 507 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 507 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_148_piece0 on localhost:65433 in memory (size: 582.0 B, free: 730.3 MB) -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_147_piece0 on localhost:65433 in memory (size: 4.3 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 140 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_146_piece0 on localhost:65433 in memory (size: 4.2 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 139 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 503 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 503 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_145_piece0 on localhost:65433 in memory (size: 600.0 B, free: 730.3 MB) -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_144_piece0 on localhost:65433 in memory (size: 4.2 KB, free: 730.3 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 138 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_143_piece0 on localhost:65433 in memory (size: 4.1 KB, free: 730.4 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 137 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 499 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 499 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_142_piece0 on localhost:65433 in memory (size: 487.0 B, free: 730.4 MB) -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_141_piece0 on localhost:65433 in memory (size: 9.9 KB, free: 730.4 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 136 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_140_piece0 on localhost:65433 in memory (size: 3.3 KB, free: 730.4 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 135 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 493 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 493 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 490 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 490 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_139_piece0 on localhost:65433 in memory (size: 3.3 KB, free: 730.4 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 134 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 479 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 479 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_138_piece0 on localhost:65433 in memory (size: 3.1 KB, free: 730.4 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 133 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_137_piece0 on localhost:65433 in memory (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 132 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_136_piece0 on localhost:65433 in memory (size: 2.9 KB, free: 730.4 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 131 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 41 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 40 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 473 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 473 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 465 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 465 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_135_piece0 on localhost:65433 in memory (size: 3.2 KB, free: 730.4 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 130 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 456 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 456 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_134_piece0 on localhost:65433 in memory (size: 3.0 KB, free: 730.4 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 129 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_133_piece0 on localhost:65433 in memory (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 128 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_132_piece0 on localhost:65433 in memory (size: 2.7 KB, free: 730.4 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 127 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 39 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 38 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 450 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 450 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 442 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 442 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_131_piece0 on localhost:65433 in memory (size: 3.0 KB, free: 730.4 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 126 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 433 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 433 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_130_piece0 on localhost:65433 in memory (size: 2.8 KB, free: 730.4 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 125 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_129_piece0 on localhost:65433 in memory (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 124 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_128_piece0 on localhost:65433 in memory (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 123 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_127_piece0 on localhost:65433 in memory (size: 2.3 KB, free: 730.4 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 122 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 37 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 36 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 35 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 427 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 427 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 418 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 418 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_126_piece0 on localhost:65433 in memory (size: 2.6 KB, free: 730.4 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 121 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_125_piece0 on localhost:65433 in memory (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 120 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_124_piece0 on localhost:65433 in memory (size: 1779.0 B, free: 730.4 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 119 -2016-06-13 19:11:12 INFO BlockManagerInfo:59 - Removed broadcast_123_piece0 on localhost:65433 in memory (size: 1974.0 B, free: 730.4 MB) -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned accumulator 118 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 34 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 33 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 407 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 407 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 402 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 402 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 396 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 396 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 391 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 391 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 359 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 359 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 30 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 29 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 342 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 342 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 334 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 334 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 28 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 27 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 319 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 319 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 311 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 311 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 26 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 25 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 24 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 296 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 296 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 287 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 287 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 23 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned shuffle 22 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 283 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 283 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 275 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 275 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 270 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 270 -2016-06-13 19:11:12 INFO BlockManager:59 - Removing RDD 264 -2016-06-13 19:11:12 INFO ContextCleaner:59 - Cleaned RDD 264 -2016-06-13 19:11:13 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201606131911_1026_m_000000_296' to file:/C:/Users/xubo/AppData/Local/Temp/spark-8acf08aa-9f84-43e5-9ee2-3ff9e64dd113/metadata/_temporary/0/task_201606131911_1026_m_000000 -2016-06-13 19:11:13 INFO SparkHadoopMapRedUtil:59 - attempt_201606131911_1026_m_000000_296: Committed -2016-06-13 19:11:13 INFO Executor:59 - Finished task 0.0 in stage 1026.0 (TID 296). 1864 bytes result sent to driver -2016-06-13 19:11:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 1026.0 (TID 296) in 913 ms on localhost (1/1) -2016-06-13 19:11:13 INFO DAGScheduler:59 - ResultStage 1026 (saveAsTextFile at PowerIterationClustering.scala:81) finished in 0.913 s -2016-06-13 19:11:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 1026.0, whose tasks have all completed, from pool -2016-06-13 19:11:13 INFO DAGScheduler:59 - Job 97 finished: saveAsTextFile at PowerIterationClustering.scala:81, took 0.978886 s -2016-06-13 19:11:15 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 19:11:15 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 19:11:15 INFO SparkContext:59 - Starting job: parquet at PowerIterationClustering.scala:84 -2016-06-13 19:11:15 INFO DAGScheduler:59 - Got job 98 (parquet at PowerIterationClustering.scala:84) with 4 output partitions -2016-06-13 19:11:15 INFO DAGScheduler:59 - Final stage: ResultStage 1027(parquet at PowerIterationClustering.scala:84) -2016-06-13 19:11:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:11:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:15 INFO DAGScheduler:59 - Submitting ResultStage 1027 (MapPartitionsRDD[587] at rddToDataFrameHolder at PowerIterationClustering.scala:83), which has no missing parents -2016-06-13 19:11:15 INFO MemoryStore:59 - ensureFreeSpace(66296) called with curMem=263071, maxMem=766075207 -2016-06-13 19:11:15 INFO MemoryStore:59 - Block broadcast_177 stored as values in memory (estimated size 64.7 KB, free 730.3 MB) -2016-06-13 19:11:15 INFO MemoryStore:59 - ensureFreeSpace(23076) called with curMem=329367, maxMem=766075207 -2016-06-13 19:11:15 INFO MemoryStore:59 - Block broadcast_177_piece0 stored as bytes in memory (estimated size 22.5 KB, free 730.3 MB) -2016-06-13 19:11:15 INFO BlockManagerInfo:59 - Added broadcast_177_piece0 in memory on localhost:65433 (size: 22.5 KB, free: 730.5 MB) -2016-06-13 19:11:15 INFO SparkContext:59 - Created broadcast 177 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1027 (MapPartitionsRDD[587] at rddToDataFrameHolder at PowerIterationClustering.scala:83) -2016-06-13 19:11:15 INFO TaskSchedulerImpl:59 - Adding task set 1027.0 with 4 tasks -2016-06-13 19:11:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 1027.0 (TID 297, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:11:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 1027.0 (TID 298, localhost, PROCESS_LOCAL, 2289 bytes) -2016-06-13 19:11:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 1027.0 (TID 299, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:11:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 1027.0 (TID 300, localhost, PROCESS_LOCAL, 2289 bytes) -2016-06-13 19:11:15 INFO Executor:59 - Running task 1.0 in stage 1027.0 (TID 298) -2016-06-13 19:11:15 INFO Executor:59 - Running task 0.0 in stage 1027.0 (TID 297) -2016-06-13 19:11:15 INFO Executor:59 - Running task 3.0 in stage 1027.0 (TID 300) -2016-06-13 19:11:15 INFO Executor:59 - Running task 2.0 in stage 1027.0 (TID 299) -2016-06-13 19:11:15 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 19:11:15 INFO CodecConfig:151 - Compression: GZIP -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Dictionary is on -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Validation is off -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-06-13 19:11:15 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 19:11:15 INFO CodecConfig:151 - Compression: GZIP -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Dictionary is on -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Validation is off -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-06-13 19:11:15 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 19:11:15 INFO CodecConfig:151 - Compression: GZIP -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Dictionary is on -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Validation is off -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-06-13 19:11:15 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 19:11:15 INFO CodecConfig:151 - Compression: GZIP -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Dictionary is on -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Validation is off -2016-06-13 19:11:15 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-06-13 19:11:16 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-06-13 19:11:16 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-06-13 19:11:16 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-06-13 19:11:16 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-06-13 19:11:16 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 36 -2016-06-13 19:11:16 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 40 -2016-06-13 19:11:16 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 56 -2016-06-13 19:11:16 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 56 -2016-06-13 19:11:16 INFO ColumnChunkPageWriteStore:151 - written 73B for [id] INT64: 2 values, 22B raw, 32B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-06-13 19:11:16 INFO ColumnChunkPageWriteStore:151 - written 75B for [id] INT64: 3 values, 30B raw, 34B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-06-13 19:11:16 INFO ColumnChunkPageWriteStore:151 - written 72B for [id] INT64: 2 values, 22B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-06-13 19:11:16 INFO ColumnChunkPageWriteStore:151 - written 75B for [id] INT64: 3 values, 30B raw, 34B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-06-13 19:11:16 INFO ColumnChunkPageWriteStore:151 - written 62B for [cluster] INT32: 2 values, 14B raw, 29B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN] -2016-06-13 19:11:16 INFO ColumnChunkPageWriteStore:151 - written 62B for [cluster] INT32: 3 values, 9B raw, 29B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN_DICTIONARY], dic { 2 entries, 8B raw, 2B comp} -2016-06-13 19:11:16 INFO ColumnChunkPageWriteStore:151 - written 61B for [cluster] INT32: 2 values, 8B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN_DICTIONARY], dic { 1 entries, 4B raw, 1B comp} -2016-06-13 19:11:16 INFO ColumnChunkPageWriteStore:151 - written 62B for [cluster] INT32: 3 values, 9B raw, 29B comp, 1 pages, encodings: [BIT_PACKED, RLE, PLAIN_DICTIONARY], dic { 2 entries, 8B raw, 2B comp} -2016-06-13 19:11:16 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201606131911_1027_m_000002_0' to file:/C:/Users/xubo/AppData/Local/Temp/spark-8acf08aa-9f84-43e5-9ee2-3ff9e64dd113/data/_temporary/0/task_201606131911_1027_m_000002 -2016-06-13 19:11:16 INFO SparkHadoopMapRedUtil:59 - attempt_201606131911_1027_m_000002_0: Committed -2016-06-13 19:11:16 INFO Executor:59 - Finished task 2.0 in stage 1027.0 (TID 299). 915 bytes result sent to driver -2016-06-13 19:11:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 1027.0 (TID 299) in 1196 ms on localhost (1/4) -2016-06-13 19:11:16 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201606131911_1027_m_000001_0' to file:/C:/Users/xubo/AppData/Local/Temp/spark-8acf08aa-9f84-43e5-9ee2-3ff9e64dd113/data/_temporary/0/task_201606131911_1027_m_000001 -2016-06-13 19:11:16 INFO SparkHadoopMapRedUtil:59 - attempt_201606131911_1027_m_000001_0: Committed -2016-06-13 19:11:16 INFO Executor:59 - Finished task 1.0 in stage 1027.0 (TID 298). 915 bytes result sent to driver -2016-06-13 19:11:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 1027.0 (TID 298) in 1221 ms on localhost (2/4) -2016-06-13 19:11:16 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201606131911_1027_m_000003_0' to file:/C:/Users/xubo/AppData/Local/Temp/spark-8acf08aa-9f84-43e5-9ee2-3ff9e64dd113/data/_temporary/0/task_201606131911_1027_m_000003 -2016-06-13 19:11:16 INFO SparkHadoopMapRedUtil:59 - attempt_201606131911_1027_m_000003_0: Committed -2016-06-13 19:11:16 INFO Executor:59 - Finished task 3.0 in stage 1027.0 (TID 300). 915 bytes result sent to driver -2016-06-13 19:11:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 1027.0 (TID 300) in 1222 ms on localhost (3/4) -2016-06-13 19:11:16 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201606131911_1027_m_000000_0' to file:/C:/Users/xubo/AppData/Local/Temp/spark-8acf08aa-9f84-43e5-9ee2-3ff9e64dd113/data/_temporary/0/task_201606131911_1027_m_000000 -2016-06-13 19:11:16 INFO SparkHadoopMapRedUtil:59 - attempt_201606131911_1027_m_000000_0: Committed -2016-06-13 19:11:16 INFO Executor:59 - Finished task 0.0 in stage 1027.0 (TID 297). 915 bytes result sent to driver -2016-06-13 19:11:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 1027.0 (TID 297) in 1224 ms on localhost (4/4) -2016-06-13 19:11:16 INFO DAGScheduler:59 - ResultStage 1027 (parquet at PowerIterationClustering.scala:84) finished in 1.224 s -2016-06-13 19:11:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 1027.0, whose tasks have all completed, from pool -2016-06-13 19:11:16 INFO DAGScheduler:59 - Job 98 finished: parquet at PowerIterationClustering.scala:84, took 1.256286 s -2016-06-13 19:11:17 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-06-13 19:11:17 INFO DefaultWriterContainer:59 - Job job_201606131911_0000 committed. -2016-06-13 19:11:17 INFO ParquetRelation:59 - Listing file:/C:/Users/xubo/AppData/Local/Temp/spark-8acf08aa-9f84-43e5-9ee2-3ff9e64dd113/data on driver -2016-06-13 19:11:17 INFO ParquetRelation:59 - Listing file:/C:/Users/xubo/AppData/Local/Temp/spark-8acf08aa-9f84-43e5-9ee2-3ff9e64dd113/data on driver -2016-06-13 19:11:17 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=352443, maxMem=766075207 -2016-06-13 19:11:17 INFO MemoryStore:59 - Block broadcast_178 stored as values in memory (estimated size 225.3 KB, free 730.0 MB) -2016-06-13 19:11:17 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=583147, maxMem=766075207 -2016-06-13 19:11:17 INFO MemoryStore:59 - Block broadcast_178_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.0 MB) -2016-06-13 19:11:17 INFO BlockManagerInfo:59 - Added broadcast_178_piece0 in memory on localhost:65433 (size: 19.3 KB, free: 730.4 MB) -2016-06-13 19:11:17 INFO SparkContext:59 - Created broadcast 178 from textFile at modelSaveLoad.scala:129 -2016-06-13 19:11:17 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 19:11:17 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-06-13 19:11:17 INFO DAGScheduler:59 - Got job 99 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-06-13 19:11:17 INFO DAGScheduler:59 - Final stage: ResultStage 1028(first at modelSaveLoad.scala:129) -2016-06-13 19:11:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:11:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:17 INFO DAGScheduler:59 - Submitting ResultStage 1028 (MapPartitionsRDD[590] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-06-13 19:11:17 INFO MemoryStore:59 - ensureFreeSpace(3224) called with curMem=602935, maxMem=766075207 -2016-06-13 19:11:17 INFO MemoryStore:59 - Block broadcast_179 stored as values in memory (estimated size 3.1 KB, free 730.0 MB) -2016-06-13 19:11:17 INFO MemoryStore:59 - ensureFreeSpace(1902) called with curMem=606159, maxMem=766075207 -2016-06-13 19:11:17 INFO MemoryStore:59 - Block broadcast_179_piece0 stored as bytes in memory (estimated size 1902.0 B, free 730.0 MB) -2016-06-13 19:11:17 INFO BlockManagerInfo:59 - Added broadcast_179_piece0 in memory on localhost:65433 (size: 1902.0 B, free: 730.4 MB) -2016-06-13 19:11:17 INFO SparkContext:59 - Created broadcast 179 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1028 (MapPartitionsRDD[590] at textFile at modelSaveLoad.scala:129) -2016-06-13 19:11:17 INFO TaskSchedulerImpl:59 - Adding task set 1028.0 with 1 tasks -2016-06-13 19:11:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 1028.0 (TID 301, localhost, PROCESS_LOCAL, 2210 bytes) -2016-06-13 19:11:17 INFO Executor:59 - Running task 0.0 in stage 1028.0 (TID 301) -2016-06-13 19:11:17 INFO HadoopRDD:59 - Input split: file:/C:/Users/xubo/AppData/Local/Temp/spark-8acf08aa-9f84-43e5-9ee2-3ff9e64dd113/metadata/part-00000:0+49 -2016-06-13 19:11:17 INFO Executor:59 - Finished task 0.0 in stage 1028.0 (TID 301). 2144 bytes result sent to driver -2016-06-13 19:11:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 1028.0 (TID 301) in 17 ms on localhost (1/1) -2016-06-13 19:11:17 INFO DAGScheduler:59 - ResultStage 1028 (first at modelSaveLoad.scala:129) finished in 0.017 s -2016-06-13 19:11:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 1028.0, whose tasks have all completed, from pool -2016-06-13 19:11:17 INFO DAGScheduler:59 - Job 99 finished: first at modelSaveLoad.scala:129, took 0.024578 s -2016-06-13 19:11:17 INFO ParquetRelation:59 - Listing file:/C:/Users/xubo/AppData/Local/Temp/spark-8acf08aa-9f84-43e5-9ee2-3ff9e64dd113/data on driver -2016-06-13 19:11:17 INFO SparkContext:59 - Starting job: parquet at PowerIterationClustering.scala:97 -2016-06-13 19:11:17 INFO DAGScheduler:59 - Got job 100 (parquet at PowerIterationClustering.scala:97) with 4 output partitions -2016-06-13 19:11:17 INFO DAGScheduler:59 - Final stage: ResultStage 1029(parquet at PowerIterationClustering.scala:97) -2016-06-13 19:11:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:11:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:11:17 INFO DAGScheduler:59 - Submitting ResultStage 1029 (MapPartitionsRDD[592] at parquet at PowerIterationClustering.scala:97), which has no missing parents -2016-06-13 19:11:17 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=608061, maxMem=766075207 -2016-06-13 19:11:17 INFO MemoryStore:59 - Block broadcast_180 stored as values in memory (estimated size 61.5 KB, free 729.9 MB) -2016-06-13 19:11:17 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=671061, maxMem=766075207 -2016-06-13 19:11:17 INFO MemoryStore:59 - Block broadcast_180_piece0 stored as bytes in memory (estimated size 20.6 KB, free 729.9 MB) -2016-06-13 19:11:17 INFO BlockManagerInfo:59 - Added broadcast_180_piece0 in memory on localhost:65433 (size: 20.6 KB, free: 730.4 MB) -2016-06-13 19:11:17 INFO SparkContext:59 - Created broadcast 180 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1029 (MapPartitionsRDD[592] at parquet at PowerIterationClustering.scala:97) -2016-06-13 19:11:17 INFO TaskSchedulerImpl:59 - Adding task set 1029.0 with 4 tasks -2016-06-13 19:11:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 1029.0 (TID 302, localhost, PROCESS_LOCAL, 2083 bytes) -2016-06-13 19:11:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 1029.0 (TID 303, localhost, PROCESS_LOCAL, 2083 bytes) -2016-06-13 19:11:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 1029.0 (TID 304, localhost, PROCESS_LOCAL, 2083 bytes) -2016-06-13 19:11:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 1029.0 (TID 305, localhost, PROCESS_LOCAL, 2252 bytes) -2016-06-13 19:11:17 INFO Executor:59 - Running task 1.0 in stage 1029.0 (TID 303) -2016-06-13 19:11:17 INFO Executor:59 - Running task 3.0 in stage 1029.0 (TID 305) -2016-06-13 19:11:17 INFO Executor:59 - Running task 2.0 in stage 1029.0 (TID 304) -2016-06-13 19:11:17 INFO Executor:59 - Running task 0.0 in stage 1029.0 (TID 302) -2016-06-13 19:11:18 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-06-13 19:11:18 INFO Executor:59 - Finished task 2.0 in stage 1029.0 (TID 304). 936 bytes result sent to driver -2016-06-13 19:11:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 1029.0 (TID 304) in 66 ms on localhost (1/4) -2016-06-13 19:11:18 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-06-13 19:11:18 INFO Executor:59 - Finished task 0.0 in stage 1029.0 (TID 302). 936 bytes result sent to driver -2016-06-13 19:11:18 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-06-13 19:11:18 INFO Executor:59 - Finished task 1.0 in stage 1029.0 (TID 303). 936 bytes result sent to driver -2016-06-13 19:11:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 1029.0 (TID 302) in 72 ms on localhost (2/4) -2016-06-13 19:11:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 1029.0 (TID 303) in 72 ms on localhost (3/4) -2016-06-13 19:11:18 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-06-13 19:11:18 INFO Executor:59 - Finished task 3.0 in stage 1029.0 (TID 305). 1701 bytes result sent to driver -2016-06-13 19:11:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 1029.0 (TID 305) in 115 ms on localhost (4/4) -2016-06-13 19:11:18 INFO DAGScheduler:59 - ResultStage 1029 (parquet at PowerIterationClustering.scala:97) finished in 0.116 s -2016-06-13 19:11:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 1029.0, whose tasks have all completed, from pool -2016-06-13 19:11:18 INFO DAGScheduler:59 - Job 100 finished: parquet at PowerIterationClustering.scala:97, took 0.157458 s -2016-06-13 19:11:18 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=692134, maxMem=766075207 -2016-06-13 19:11:18 INFO MemoryStore:59 - Block broadcast_181 stored as values in memory (estimated size 86.4 KB, free 729.8 MB) -2016-06-13 19:11:18 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=780646, maxMem=766075207 -2016-06-13 19:11:18 INFO MemoryStore:59 - Block broadcast_181_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.8 MB) -2016-06-13 19:11:18 INFO BlockManagerInfo:59 - Added broadcast_181_piece0 in memory on localhost:65433 (size: 19.3 KB, free: 730.4 MB) -2016-06-13 19:11:18 INFO SparkContext:59 - Created broadcast 181 from map at PowerIterationClustering.scala:100 -2016-06-13 19:11:18 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-06-13 19:11:18 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/C:/Users/xubo/AppData/Local/Temp/spark-8acf08aa-9f84-43e5-9ee2-3ff9e64dd113/data/part-r-00002-93f032cb-1f17-48f7-b37a-6777dbb644c8.gz.parquet, file:/C:/Users/xubo/AppData/Local/Temp/spark-8acf08aa-9f84-43e5-9ee2-3ff9e64dd113/data/part-r-00000-93f032cb-1f17-48f7-b37a-6777dbb644c8.gz.parquet, file:/C:/Users/xubo/AppData/Local/Temp/spark-8acf08aa-9f84-43e5-9ee2-3ff9e64dd113/data/part-r-00001-93f032cb-1f17-48f7-b37a-6777dbb644c8.gz.parquet, file:/C:/Users/xubo/AppData/Local/Temp/spark-8acf08aa-9f84-43e5-9ee2-3ff9e64dd113/data/part-r-00003-93f032cb-1f17-48f7-b37a-6777dbb644c8.gz.parquet -2016-06-13 19:11:18 INFO SparkContext:59 - Starting job: count at PowerIterationClusteringSuite.scala:193 -2016-06-13 19:11:18 INFO DAGScheduler:59 - Registering RDD 596 (map at PowerIterationClusteringSuite.scala:190) -2016-06-13 19:11:18 INFO DAGScheduler:59 - Registering RDD 597 (map at PowerIterationClusteringSuite.scala:191) -2016-06-13 19:11:18 INFO DAGScheduler:59 - Got job 101 (count at PowerIterationClusteringSuite.scala:193) with 4 output partitions -2016-06-13 19:11:18 INFO DAGScheduler:59 - Final stage: ResultStage 1032(count at PowerIterationClusteringSuite.scala:193) -2016-06-13 19:11:18 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1030, ShuffleMapStage 1031) -2016-06-13 19:11:18 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1030, ShuffleMapStage 1031) -2016-06-13 19:11:18 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1030 (MapPartitionsRDD[596] at map at PowerIterationClusteringSuite.scala:190), which has no missing parents -2016-06-13 19:11:18 INFO MemoryStore:59 - ensureFreeSpace(2352) called with curMem=800434, maxMem=766075207 -2016-06-13 19:11:18 INFO MemoryStore:59 - Block broadcast_182 stored as values in memory (estimated size 2.3 KB, free 729.8 MB) -2016-06-13 19:11:18 INFO MemoryStore:59 - ensureFreeSpace(1425) called with curMem=802786, maxMem=766075207 -2016-06-13 19:11:18 INFO MemoryStore:59 - Block broadcast_182_piece0 stored as bytes in memory (estimated size 1425.0 B, free 729.8 MB) -2016-06-13 19:11:18 INFO BlockManagerInfo:59 - Added broadcast_182_piece0 in memory on localhost:65433 (size: 1425.0 B, free: 730.4 MB) -2016-06-13 19:11:18 INFO SparkContext:59 - Created broadcast 182 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 1030 (MapPartitionsRDD[596] at map at PowerIterationClusteringSuite.scala:190) -2016-06-13 19:11:18 INFO TaskSchedulerImpl:59 - Adding task set 1030.0 with 4 tasks -2016-06-13 19:11:18 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1031 (MapPartitionsRDD[597] at map at PowerIterationClusteringSuite.scala:191), which has no missing parents -2016-06-13 19:11:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 1030.0 (TID 306, localhost, PROCESS_LOCAL, 2260 bytes) -2016-06-13 19:11:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 1030.0 (TID 307, localhost, PROCESS_LOCAL, 2278 bytes) -2016-06-13 19:11:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 1030.0 (TID 308, localhost, PROCESS_LOCAL, 2260 bytes) -2016-06-13 19:11:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 1030.0 (TID 309, localhost, PROCESS_LOCAL, 2278 bytes) -2016-06-13 19:11:18 INFO Executor:59 - Running task 0.0 in stage 1030.0 (TID 306) -2016-06-13 19:11:18 INFO Executor:59 - Running task 1.0 in stage 1030.0 (TID 307) -2016-06-13 19:11:18 INFO Executor:59 - Running task 3.0 in stage 1030.0 (TID 309) -2016-06-13 19:11:18 INFO Executor:59 - Running task 2.0 in stage 1030.0 (TID 308) -2016-06-13 19:11:18 INFO MemoryStore:59 - ensureFreeSpace(5128) called with curMem=804211, maxMem=766075207 -2016-06-13 19:11:18 INFO MemoryStore:59 - Block broadcast_183 stored as values in memory (estimated size 5.0 KB, free 729.8 MB) -2016-06-13 19:11:18 INFO MemoryStore:59 - ensureFreeSpace(2887) called with curMem=809339, maxMem=766075207 -2016-06-13 19:11:18 INFO MemoryStore:59 - Block broadcast_183_piece0 stored as bytes in memory (estimated size 2.8 KB, free 729.8 MB) -2016-06-13 19:11:18 INFO BlockManagerInfo:59 - Added broadcast_183_piece0 in memory on localhost:65433 (size: 2.8 KB, free: 730.4 MB) -2016-06-13 19:11:18 INFO SparkContext:59 - Created broadcast 183 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 1031 (MapPartitionsRDD[597] at map at PowerIterationClusteringSuite.scala:191) -2016-06-13 19:11:18 INFO TaskSchedulerImpl:59 - Adding task set 1031.0 with 4 tasks -2016-06-13 19:11:18 INFO Executor:59 - Finished task 0.0 in stage 1030.0 (TID 306). 1161 bytes result sent to driver -2016-06-13 19:11:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 1031.0 (TID 310, localhost, PROCESS_LOCAL, 2269 bytes) -2016-06-13 19:11:18 INFO Executor:59 - Running task 0.0 in stage 1031.0 (TID 310) -2016-06-13 19:11:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 1030.0 (TID 306) in 41 ms on localhost (1/4) -2016-06-13 19:11:18 INFO Executor:59 - Finished task 1.0 in stage 1030.0 (TID 307). 1161 bytes result sent to driver -2016-06-13 19:11:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 1031.0 (TID 311, localhost, PROCESS_LOCAL, 2270 bytes) -2016-06-13 19:11:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 1030.0 (TID 307) in 44 ms on localhost (2/4) -2016-06-13 19:11:18 INFO Executor:59 - Running task 1.0 in stage 1031.0 (TID 311) -2016-06-13 19:11:18 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/C:/Users/xubo/AppData/Local/Temp/spark-8acf08aa-9f84-43e5-9ee2-3ff9e64dd113/data/part-r-00002-93f032cb-1f17-48f7-b37a-6777dbb644c8.gz.parquet start: 0 end: 562 length: 562 hosts: []} -2016-06-13 19:11:18 INFO Executor:59 - Finished task 2.0 in stage 1030.0 (TID 308). 1161 bytes result sent to driver -2016-06-13 19:11:18 INFO Executor:59 - Finished task 3.0 in stage 1030.0 (TID 309). 1161 bytes result sent to driver -2016-06-13 19:11:18 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/C:/Users/xubo/AppData/Local/Temp/spark-8acf08aa-9f84-43e5-9ee2-3ff9e64dd113/data/part-r-00000-93f032cb-1f17-48f7-b37a-6777dbb644c8.gz.parquet start: 0 end: 524 length: 524 hosts: []} -2016-06-13 19:11:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 1031.0 (TID 312, localhost, PROCESS_LOCAL, 2269 bytes) -2016-06-13 19:11:18 INFO Executor:59 - Running task 2.0 in stage 1031.0 (TID 312) -2016-06-13 19:11:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 1031.0 (TID 313, localhost, PROCESS_LOCAL, 2269 bytes) -2016-06-13 19:11:18 INFO Executor:59 - Running task 3.0 in stage 1031.0 (TID 313) -2016-06-13 19:11:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 1030.0 (TID 309) in 59 ms on localhost (3/4) -2016-06-13 19:11:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 1030.0 (TID 308) in 60 ms on localhost (4/4) -2016-06-13 19:11:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 1030.0, whose tasks have all completed, from pool -2016-06-13 19:11:18 INFO DAGScheduler:59 - ShuffleMapStage 1030 (map at PowerIterationClusteringSuite.scala:190) finished in 0.061 s -2016-06-13 19:11:18 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:18 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 1031) -2016-06-13 19:11:18 INFO DAGScheduler:59 - waiting: Set(ResultStage 1032) -2016-06-13 19:11:18 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:18 INFO DAGScheduler:59 - Missing parents for ResultStage 1032: List(ShuffleMapStage 1031) -2016-06-13 19:11:18 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/C:/Users/xubo/AppData/Local/Temp/spark-8acf08aa-9f84-43e5-9ee2-3ff9e64dd113/data/part-r-00001-93f032cb-1f17-48f7-b37a-6777dbb644c8.gz.parquet start: 0 end: 568 length: 568 hosts: []} -2016-06-13 19:11:18 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/C:/Users/xubo/AppData/Local/Temp/spark-8acf08aa-9f84-43e5-9ee2-3ff9e64dd113/data/part-r-00003-93f032cb-1f17-48f7-b37a-6777dbb644c8.gz.parquet start: 0 end: 565 length: 565 hosts: []} -2016-06-13 19:11:18 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-06-13 19:11:18 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-06-13 19:11:18 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-06-13 19:11:18 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-06-13 19:11:18 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int64 id; - optional int32 cluster; -} - -Catalyst form: -StructType(StructField(id,LongType,true), StructField(cluster,IntegerType,true)) - -2016-06-13 19:11:18 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int64 id; - optional int32 cluster; -} - -Catalyst form: -StructType(StructField(id,LongType,true), StructField(cluster,IntegerType,true)) - -2016-06-13 19:11:18 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int64 id; - optional int32 cluster; -} - -Catalyst form: -StructType(StructField(id,LongType,true), StructField(cluster,IntegerType,true)) - -2016-06-13 19:11:18 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int64 id; - optional int32 cluster; -} - -Catalyst form: -StructType(StructField(id,LongType,true), StructField(cluster,IntegerType,true)) - -2016-06-13 19:11:18 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 3 records. -2016-06-13 19:11:18 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 3 records. -2016-06-13 19:11:18 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 2 records. -2016-06-13 19:11:18 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 2 records. -2016-06-13 19:11:18 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-06-13 19:11:18 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-06-13 19:11:18 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-06-13 19:11:18 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-06-13 19:11:18 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-06-13 19:11:18 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-06-13 19:11:18 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-06-13 19:11:18 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-06-13 19:11:18 INFO InternalParquetRecordReader:151 - block read in memory in 19 ms. row count = 2 -2016-06-13 19:11:18 INFO InternalParquetRecordReader:151 - block read in memory in 19 ms. row count = 3 -2016-06-13 19:11:18 INFO InternalParquetRecordReader:151 - block read in memory in 19 ms. row count = 2 -2016-06-13 19:11:18 INFO InternalParquetRecordReader:151 - block read in memory in 19 ms. row count = 3 -2016-06-13 19:11:18 INFO Executor:59 - Finished task 0.0 in stage 1031.0 (TID 310). 2256 bytes result sent to driver -2016-06-13 19:11:18 INFO Executor:59 - Finished task 2.0 in stage 1031.0 (TID 312). 2256 bytes result sent to driver -2016-06-13 19:11:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 1031.0 (TID 310) in 224 ms on localhost (1/4) -2016-06-13 19:11:18 INFO Executor:59 - Finished task 3.0 in stage 1031.0 (TID 313). 2256 bytes result sent to driver -2016-06-13 19:11:18 INFO Executor:59 - Finished task 1.0 in stage 1031.0 (TID 311). 2256 bytes result sent to driver -2016-06-13 19:11:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 1031.0 (TID 312) in 206 ms on localhost (2/4) -2016-06-13 19:11:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 1031.0 (TID 313) in 206 ms on localhost (3/4) -2016-06-13 19:11:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 1031.0 (TID 311) in 223 ms on localhost (4/4) -2016-06-13 19:11:18 INFO DAGScheduler:59 - ShuffleMapStage 1031 (map at PowerIterationClusteringSuite.scala:191) finished in 0.250 s -2016-06-13 19:11:18 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:11:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 1031.0, whose tasks have all completed, from pool -2016-06-13 19:11:18 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:11:18 INFO DAGScheduler:59 - waiting: Set(ResultStage 1032) -2016-06-13 19:11:18 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:11:18 INFO DAGScheduler:59 - Missing parents for ResultStage 1032: List() -2016-06-13 19:11:18 INFO DAGScheduler:59 - Submitting ResultStage 1032 (MapPartitionsRDD[601] at filter at PowerIterationClusteringSuite.scala:192), which is now runnable -2016-06-13 19:11:18 INFO MemoryStore:59 - ensureFreeSpace(2936) called with curMem=812226, maxMem=766075207 -2016-06-13 19:11:18 INFO MemoryStore:59 - Block broadcast_184 stored as values in memory (estimated size 2.9 KB, free 729.8 MB) -2016-06-13 19:11:18 INFO MemoryStore:59 - ensureFreeSpace(1627) called with curMem=815162, maxMem=766075207 -2016-06-13 19:11:18 INFO MemoryStore:59 - Block broadcast_184_piece0 stored as bytes in memory (estimated size 1627.0 B, free 729.8 MB) -2016-06-13 19:11:18 INFO BlockManagerInfo:59 - Added broadcast_184_piece0 in memory on localhost:65433 (size: 1627.0 B, free: 730.4 MB) -2016-06-13 19:11:18 INFO SparkContext:59 - Created broadcast 184 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:11:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1032 (MapPartitionsRDD[601] at filter at PowerIterationClusteringSuite.scala:192) -2016-06-13 19:11:18 INFO TaskSchedulerImpl:59 - Adding task set 1032.0 with 4 tasks -2016-06-13 19:11:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 1032.0 (TID 314, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 19:11:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 1032.0 (TID 315, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 19:11:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 1032.0 (TID 316, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 19:11:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 1032.0 (TID 317, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 19:11:18 INFO Executor:59 - Running task 1.0 in stage 1032.0 (TID 315) -2016-06-13 19:11:18 INFO Executor:59 - Running task 3.0 in stage 1032.0 (TID 317) -2016-06-13 19:11:18 INFO Executor:59 - Running task 2.0 in stage 1032.0 (TID 316) -2016-06-13 19:11:18 INFO Executor:59 - Running task 0.0 in stage 1032.0 (TID 314) -2016-06-13 19:11:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 19:11:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 19:11:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 19:11:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 19:11:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 19:11:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 19:11:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 19:11:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:11:18 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 19:11:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:11:18 INFO Executor:59 - Finished task 2.0 in stage 1032.0 (TID 316). 1203 bytes result sent to driver -2016-06-13 19:11:18 INFO Executor:59 - Finished task 1.0 in stage 1032.0 (TID 315). 1203 bytes result sent to driver -2016-06-13 19:11:18 INFO Executor:59 - Finished task 3.0 in stage 1032.0 (TID 317). 1203 bytes result sent to driver -2016-06-13 19:11:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 1032.0 (TID 316) in 44 ms on localhost (1/4) -2016-06-13 19:11:18 INFO Executor:59 - Finished task 0.0 in stage 1032.0 (TID 314). 1203 bytes result sent to driver -2016-06-13 19:11:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 1032.0 (TID 315) in 46 ms on localhost (2/4) -2016-06-13 19:11:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 1032.0 (TID 317) in 45 ms on localhost (3/4) -2016-06-13 19:11:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 1032.0 (TID 314) in 48 ms on localhost (4/4) -2016-06-13 19:11:18 INFO DAGScheduler:59 - ResultStage 1032 (count at PowerIterationClusteringSuite.scala:193) finished in 0.048 s -2016-06-13 19:11:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 1032.0, whose tasks have all completed, from pool -2016-06-13 19:11:18 INFO DAGScheduler:59 - Job 101 finished: count at PowerIterationClusteringSuite.scala:193, took 0.329732 s -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL2/execution/json,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL2/execution,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL2/json,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL2,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 19:11:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 19:11:19 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 19:11:19 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 19:11:19 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 19:11:19 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 19:11:19 INFO BlockManager:59 - BlockManager stopped -2016-06-13 19:11:19 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 19:11:19 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 19:11:19 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 19:11:19 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 19:11:19 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-f759b974-a642-4607-ad6b-808498eb7738 -2016-06-13 19:11:19 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 19:11:19 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 19:12:41 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 19:12:42 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 19:12:42 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 19:12:42 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 19:12:42 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 19:12:43 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 19:12:43 INFO Remoting:74 - Starting remoting -2016-06-13 19:12:43 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:65471] -2016-06-13 19:12:43 INFO Utils:59 - Successfully started service 'sparkDriver' on port 65471. -2016-06-13 19:12:43 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 19:12:43 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 19:12:43 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-5b5add3d-90f4-4306-ac5b-fc902729b780 -2016-06-13 19:12:43 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 19:12:43 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-b7a57d14-165c-4541-b0c2-ebf9ecc08c87\httpd-4fb46712-4fdd-4aca-a52d-1e2945117d93 -2016-06-13 19:12:43 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 19:12:44 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:12:44 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:65472 -2016-06-13 19:12:44 INFO Utils:59 - Successfully started service 'HTTP file server' on port 65472. -2016-06-13 19:12:44 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 19:12:44 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:12:44 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 19:12:44 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 19:12:44 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 19:12:44 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 19:12:44 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 19:12:44 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 65491. -2016-06-13 19:12:44 INFO NettyBlockTransferService:59 - Server created on 65491 -2016-06-13 19:12:44 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 19:12:44 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:65491 with 730.6 MB RAM, BlockManagerId(driver, localhost, 65491) -2016-06-13 19:12:44 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 19:12:45 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 19:12:45 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 19:12:45 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 19:12:45 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 19:12:45 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:65491 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 19:12:45 INFO SparkContext:59 - Created broadcast 0 from textFile at PICFromWebSuite.scala:20 -2016-06-13 19:12:46 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 19:12:46 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 19:12:47 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:325 -2016-06-13 19:12:47 INFO DAGScheduler:59 - Registering RDD 13 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:12:47 INFO DAGScheduler:59 - Registering RDD 7 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:12:47 INFO DAGScheduler:59 - Got job 0 (sum at PowerIterationClustering.scala:325) with 2 output partitions -2016-06-13 19:12:47 INFO DAGScheduler:59 - Final stage: ResultStage 2(sum at PowerIterationClustering.scala:325) -2016-06-13 19:12:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0, ShuffleMapStage 1) -2016-06-13 19:12:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0, ShuffleMapStage 1) -2016-06-13 19:12:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[13] at mapPartitions at GraphImpl.scala:235), which has no missing parents -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(5096) called with curMem=171580, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(2769) called with curMem=176676, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:12:47 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:65491 (size: 2.7 KB, free: 730.6 MB) -2016-06-13 19:12:47 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[13] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:12:47 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 19:12:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[7] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(4672) called with curMem=179445, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.6 KB, free 730.4 MB) -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(2581) called with curMem=184117, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 19:12:47 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:65491 (size: 2.5 KB, free: 730.6 MB) -2016-06-13 19:12:47 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[7] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:12:47 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 19:12:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2179 bytes) -2016-06-13 19:12:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2179 bytes) -2016-06-13 19:12:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2179 bytes) -2016-06-13 19:12:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2179 bytes) -2016-06-13 19:12:47 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 19:12:47 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 19:12:47 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 19:12:47 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 19:12:47 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-06-13 19:12:47 INFO CacheManager:59 - Another thread is loading rdd_4_1, waiting for it to finish... -2016-06-13 19:12:47 INFO CacheManager:59 - Another thread is loading rdd_4_0, waiting for it to finish... -2016-06-13 19:12:47 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-06-13 19:12:47 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/pic_data.txt:82+82 -2016-06-13 19:12:47 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/pic_data.txt:0+82 -2016-06-13 19:12:47 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 19:12:47 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 19:12:47 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 19:12:47 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 19:12:47 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(2952) called with curMem=186698, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(2856) called with curMem=189650, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 2.8 KB, free 730.4 MB) -2016-06-13 19:12:47 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:65491 (size: 2.9 KB, free: 730.6 MB) -2016-06-13 19:12:47 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:65491 (size: 2.8 KB, free: 730.6 MB) -2016-06-13 19:12:47 INFO CacheManager:59 - Finished waiting for rdd_4_0 -2016-06-13 19:12:47 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:12:47 INFO CacheManager:59 - Finished waiting for rdd_4_1 -2016-06-13 19:12:47 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 19:12:47 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1159 bytes result sent to driver -2016-06-13 19:12:47 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1159 bytes result sent to driver -2016-06-13 19:12:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 187 ms on localhost (1/2) -2016-06-13 19:12:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 225 ms on localhost (1/2) -2016-06-13 19:12:47 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 2834 bytes result sent to driver -2016-06-13 19:12:47 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2834 bytes result sent to driver -2016-06-13 19:12:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 199 ms on localhost (2/2) -2016-06-13 19:12:47 INFO DAGScheduler:59 - ShuffleMapStage 1 (mapPartitions at VertexRDD.scala:358) finished in 0.204 s -2016-06-13 19:12:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 19:12:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:12:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 209 ms on localhost (2/2) -2016-06-13 19:12:47 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 0) -2016-06-13 19:12:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 19:12:47 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-06-13 19:12:47 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:12:47 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List(ShuffleMapStage 0) -2016-06-13 19:12:47 INFO DAGScheduler:59 - ShuffleMapStage 0 (mapPartitions at GraphImpl.scala:235) finished in 0.260 s -2016-06-13 19:12:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:12:47 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:12:47 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-06-13 19:12:47 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:12:47 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List() -2016-06-13 19:12:47 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[25] at map at PowerIterationClustering.scala:325), which is now runnable -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(5136) called with curMem=192506, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(2651) called with curMem=197642, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-06-13 19:12:47 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:65491 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:12:47 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[25] at map at PowerIterationClustering.scala:325) -2016-06-13 19:12:47 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 19:12:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:12:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:12:47 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 19:12:47 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 19:12:47 INFO CacheManager:59 - Partition rdd_23_0 not found, computing it -2016-06-13 19:12:47 INFO CacheManager:59 - Partition rdd_15_0 not found, computing it -2016-06-13 19:12:47 INFO CacheManager:59 - Partition rdd_23_1 not found, computing it -2016-06-13 19:12:47 INFO CacheManager:59 - Partition rdd_15_1 not found, computing it -2016-06-13 19:12:47 INFO CacheManager:59 - Partition rdd_10_0 not found, computing it -2016-06-13 19:12:47 INFO CacheManager:59 - Partition rdd_10_1 not found, computing it -2016-06-13 19:12:47 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:12:47 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:12:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-06-13 19:12:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(1792) called with curMem=200293, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block rdd_10_1 stored as values in memory (estimated size 1792.0 B, free 730.4 MB) -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(1776) called with curMem=202085, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block rdd_10_0 stored as values in memory (estimated size 1776.0 B, free 730.4 MB) -2016-06-13 19:12:47 INFO BlockManagerInfo:59 - Added rdd_10_1 in memory on localhost:65491 (size: 1792.0 B, free: 730.6 MB) -2016-06-13 19:12:47 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:12:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:12:47 INFO BlockManagerInfo:59 - Added rdd_10_0 in memory on localhost:65491 (size: 1776.0 B, free: 730.6 MB) -2016-06-13 19:12:47 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:12:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(1840) called with curMem=203861, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block rdd_15_1 stored as values in memory (estimated size 1840.0 B, free 730.4 MB) -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(1824) called with curMem=205701, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block rdd_15_0 stored as values in memory (estimated size 1824.0 B, free 730.4 MB) -2016-06-13 19:12:47 INFO BlockManagerInfo:59 - Added rdd_15_1 in memory on localhost:65491 (size: 1840.0 B, free: 730.6 MB) -2016-06-13 19:12:47 INFO BlockManagerInfo:59 - Added rdd_15_0 in memory on localhost:65491 (size: 1824.0 B, free: 730.6 MB) -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(368) called with curMem=207525, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block rdd_23_0 stored as values in memory (estimated size 368.0 B, free 730.4 MB) -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(368) called with curMem=207893, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block rdd_23_1 stored as values in memory (estimated size 368.0 B, free 730.4 MB) -2016-06-13 19:12:47 INFO BlockManagerInfo:59 - Added rdd_23_0 in memory on localhost:65491 (size: 368.0 B, free: 730.6 MB) -2016-06-13 19:12:47 INFO BlockManagerInfo:59 - Added rdd_23_1 in memory on localhost:65491 (size: 368.0 B, free: 730.6 MB) -2016-06-13 19:12:47 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1895 bytes result sent to driver -2016-06-13 19:12:47 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1895 bytes result sent to driver -2016-06-13 19:12:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 89 ms on localhost (1/2) -2016-06-13 19:12:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 86 ms on localhost (2/2) -2016-06-13 19:12:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 19:12:47 INFO DAGScheduler:59 - ResultStage 2 (sum at PowerIterationClustering.scala:325) finished in 0.092 s -2016-06-13 19:12:47 INFO DAGScheduler:59 - Job 0 finished: sum at PowerIterationClustering.scala:325, took 0.495848 s -2016-06-13 19:12:47 INFO SparkContext:59 - Starting job: reduce at VertexRDDImpl.scala:90 -2016-06-13 19:12:47 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:12:47 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 152 bytes -2016-06-13 19:12:47 INFO DAGScheduler:59 - Got job 1 (reduce at VertexRDDImpl.scala:90) with 2 output partitions -2016-06-13 19:12:47 INFO DAGScheduler:59 - Final stage: ResultStage 5(reduce at VertexRDDImpl.scala:90) -2016-06-13 19:12:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3, ShuffleMapStage 4) -2016-06-13 19:12:47 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:12:47 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[29] at map at VertexRDDImpl.scala:90), which has no missing parents -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(5288) called with curMem=208261, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 5.2 KB, free 730.4 MB) -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(2668) called with curMem=213549, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-06-13 19:12:47 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:65491 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:12:47 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[29] at map at VertexRDDImpl.scala:90) -2016-06-13 19:12:47 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 19:12:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 6, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:12:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 7, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:12:47 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 6) -2016-06-13 19:12:47 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 7) -2016-06-13 19:12:47 INFO BlockManager:59 - Found block rdd_23_1 locally -2016-06-13 19:12:47 INFO BlockManager:59 - Found block rdd_23_0 locally -2016-06-13 19:12:47 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 7). 2161 bytes result sent to driver -2016-06-13 19:12:47 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 6). 2161 bytes result sent to driver -2016-06-13 19:12:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 7) in 17 ms on localhost (1/2) -2016-06-13 19:12:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 6) in 20 ms on localhost (2/2) -2016-06-13 19:12:47 INFO DAGScheduler:59 - ResultStage 5 (reduce at VertexRDDImpl.scala:90) finished in 0.021 s -2016-06-13 19:12:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 19:12:47 INFO DAGScheduler:59 - Job 1 finished: reduce at VertexRDDImpl.scala:90, took 0.053413 s -2016-06-13 19:12:47 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:12:47 INFO DAGScheduler:59 - Registering RDD 17 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:12:47 INFO DAGScheduler:59 - Registering RDD 30 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:12:47 INFO DAGScheduler:59 - Registering RDD 34 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:12:47 INFO DAGScheduler:59 - Got job 2 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:12:47 INFO DAGScheduler:59 - Final stage: ResultStage 13(sum at PowerIterationClustering.scala:369) -2016-06-13 19:12:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12, ShuffleMapStage 9, ShuffleMapStage 10) -2016-06-13 19:12:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-06-13 19:12:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 8 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[17] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=216217, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 4.1 KB, free 730.4 MB) -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(2364) called with curMem=220465, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 19:12:47 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:65491 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:12:47 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 8 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[17] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:12:47 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 19:12:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 11 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[30] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:12:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 8, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:12:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 9, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:12:47 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 9) -2016-06-13 19:12:47 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 8) -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(4992) called with curMem=222829, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 4.9 KB, free 730.4 MB) -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(2602) called with curMem=227821, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 19:12:47 INFO BlockManager:59 - Found block rdd_15_0 locally -2016-06-13 19:12:47 INFO BlockManager:59 - Found block rdd_15_1 locally -2016-06-13 19:12:47 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:65491 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:12:47 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 11 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[30] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:12:47 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 19:12:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 10, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:12:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 11, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:12:47 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 11) -2016-06-13 19:12:47 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 10) -2016-06-13 19:12:47 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 8). 2254 bytes result sent to driver -2016-06-13 19:12:47 INFO CacheManager:59 - Partition rdd_27_1 not found, computing it -2016-06-13 19:12:47 INFO BlockManager:59 - Found block rdd_23_1 locally -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=230423, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block rdd_27_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:12:47 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 9). 2254 bytes result sent to driver -2016-06-13 19:12:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 8) in 58 ms on localhost (1/2) -2016-06-13 19:12:47 INFO CacheManager:59 - Partition rdd_27_0 not found, computing it -2016-06-13 19:12:47 INFO BlockManager:59 - Found block rdd_23_0 locally -2016-06-13 19:12:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 9) in 58 ms on localhost (2/2) -2016-06-13 19:12:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=231855, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block rdd_27_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:12:47 INFO DAGScheduler:59 - ShuffleMapStage 8 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.062 s -2016-06-13 19:12:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:12:47 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 11) -2016-06-13 19:12:47 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 12, ResultStage 13) -2016-06-13 19:12:47 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:12:47 INFO BlockManagerInfo:59 - Added rdd_27_1 in memory on localhost:65491 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:12:47 INFO BlockManagerInfo:59 - Added rdd_27_0 in memory on localhost:65491 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:12:47 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 12: List(ShuffleMapStage 11) -2016-06-13 19:12:47 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 11). 2834 bytes result sent to driver -2016-06-13 19:12:47 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List(ShuffleMapStage 12) -2016-06-13 19:12:47 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 10). 2834 bytes result sent to driver -2016-06-13 19:12:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 11) in 48 ms on localhost (1/2) -2016-06-13 19:12:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 10) in 51 ms on localhost (2/2) -2016-06-13 19:12:47 INFO DAGScheduler:59 - ShuffleMapStage 11 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.055 s -2016-06-13 19:12:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 19:12:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:12:47 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:12:47 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 12, ResultStage 13) -2016-06-13 19:12:47 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:12:47 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 12: List() -2016-06-13 19:12:47 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List(ShuffleMapStage 12) -2016-06-13 19:12:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[34] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(6472) called with curMem=233287, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 6.3 KB, free 730.4 MB) -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(3319) called with curMem=239759, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:12:47 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:65491 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:12:47 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[34] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:12:47 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 19:12:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 12, localhost, PROCESS_LOCAL, 2469 bytes) -2016-06-13 19:12:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 13, localhost, PROCESS_LOCAL, 2469 bytes) -2016-06-13 19:12:47 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 12) -2016-06-13 19:12:47 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 13) -2016-06-13 19:12:47 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:12:47 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 19:12:47 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:12:47 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:12:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:12:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:12:47 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:12:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:12:47 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:12:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:12:47 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 13). 2470 bytes result sent to driver -2016-06-13 19:12:47 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 12). 2470 bytes result sent to driver -2016-06-13 19:12:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 13) in 48 ms on localhost (1/2) -2016-06-13 19:12:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 12) in 54 ms on localhost (2/2) -2016-06-13 19:12:47 INFO DAGScheduler:59 - ShuffleMapStage 12 (mapPartitions at GraphImpl.scala:235) finished in 0.054 s -2016-06-13 19:12:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:12:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 19:12:47 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:12:47 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-06-13 19:12:47 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:12:47 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-06-13 19:12:47 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[39] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(6024) called with curMem=243078, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 5.9 KB, free 730.3 MB) -2016-06-13 19:12:47 INFO MemoryStore:59 - ensureFreeSpace(2866) called with curMem=249102, maxMem=766075207 -2016-06-13 19:12:47 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-06-13 19:12:47 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:65491 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:12:47 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[39] at map at PowerIterationClustering.scala:369) -2016-06-13 19:12:47 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 19:12:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 14, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:12:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 15, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:12:47 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 15) -2016-06-13 19:12:47 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 14) -2016-06-13 19:12:48 INFO CacheManager:59 - Partition rdd_36_1 not found, computing it -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-06-13 19:12:48 INFO CacheManager:59 - Partition rdd_36_0 not found, computing it -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=251968, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block rdd_36_1 stored as values in memory (estimated size 1480.0 B, free 730.3 MB) -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=253448, maxMem=766075207 -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added rdd_36_1 in memory on localhost:65491 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:12:48 INFO MemoryStore:59 - Block rdd_36_0 stored as values in memory (estimated size 1480.0 B, free 730.3 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added rdd_36_0 in memory on localhost:65491 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:12:48 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 15). 2880 bytes result sent to driver -2016-06-13 19:12:48 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 14). 2880 bytes result sent to driver -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 15) in 26 ms on localhost (1/2) -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 14) in 29 ms on localhost (2/2) -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 19:12:48 INFO DAGScheduler:59 - ResultStage 13 (sum at PowerIterationClustering.scala:369) finished in 0.029 s -2016-06-13 19:12:48 INFO DAGScheduler:59 - Job 2 finished: sum at PowerIterationClustering.scala:369, took 0.238899 s -2016-06-13 19:12:48 INFO PowerIterationClustering:59 - Iteration 0: norm(v) = 36.22334869431644. -2016-06-13 19:12:48 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 152 bytes -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 152 bytes -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:12:48 INFO DAGScheduler:59 - Got job 3 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:12:48 INFO DAGScheduler:59 - Final stage: ResultStage 21(sum at PowerIterationClustering.scala:375) -2016-06-13 19:12:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 20, ShuffleMapStage 17, ShuffleMapStage 18) -2016-06-13 19:12:48 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:12:48 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[50] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(6760) called with curMem=254928, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 6.6 KB, free 730.3 MB) -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(3105) called with curMem=261688, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.3 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:65491 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:12:48 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[50] at values at PowerIterationClustering.scala:375) -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 19:12:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 16, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:12:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 17, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:12:48 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 17) -2016-06-13 19:12:48 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 16) -2016-06-13 19:12:48 INFO CacheManager:59 - Partition rdd_42_0 not found, computing it -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_27_0 locally -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_36_0 locally -2016-06-13 19:12:48 INFO CacheManager:59 - Partition rdd_42_1 not found, computing it -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=264793, maxMem=766075207 -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_27_1 locally -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_36_1 locally -2016-06-13 19:12:48 INFO MemoryStore:59 - Block rdd_42_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=266225, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block rdd_42_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added rdd_42_0 in memory on localhost:65491 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added rdd_42_1 in memory on localhost:65491 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:12:48 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 16). 2664 bytes result sent to driver -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 16) in 24 ms on localhost (1/2) -2016-06-13 19:12:48 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 17). 2664 bytes result sent to driver -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 17) in 28 ms on localhost (2/2) -2016-06-13 19:12:48 INFO DAGScheduler:59 - ResultStage 21 (sum at PowerIterationClustering.scala:375) finished in 0.030 s -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 19:12:48 INFO DAGScheduler:59 - Job 3 finished: sum at PowerIterationClustering.scala:375, took 0.067511 s -2016-06-13 19:12:48 INFO PowerIterationClustering:59 - Iteration 0: delta = 1.2794887333174252. -2016-06-13 19:12:48 INFO PowerIterationClustering:59 - Iteration 0: diff(delta) = 1.7976931348623157E308. -2016-06-13 19:12:48 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:12:48 INFO DAGScheduler:59 - Registering RDD 53 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:12:48 INFO DAGScheduler:59 - Registering RDD 57 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:12:48 INFO DAGScheduler:59 - Got job 4 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:12:48 INFO DAGScheduler:59 - Final stage: ResultStage 31(sum at PowerIterationClustering.scala:369) -2016-06-13 19:12:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30, ShuffleMapStage 28, ShuffleMapStage 25, ShuffleMapStage 26) -2016-06-13 19:12:48 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 30) -2016-06-13 19:12:48 INFO DAGScheduler:59 - Submitting ShuffleMapStage 29 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[53] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(5984) called with curMem=267657, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 5.8 KB, free 730.3 MB) -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(2863) called with curMem=273641, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:65491 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:12:48 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 29 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[53] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-06-13 19:12:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 18, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:12:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 19, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:12:48 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 19) -2016-06-13 19:12:48 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 18) -2016-06-13 19:12:48 INFO CacheManager:59 - Partition rdd_51_1 not found, computing it -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_36_1 locally -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=276504, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block rdd_51_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added rdd_51_1 in memory on localhost:65491 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:12:48 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 19). 2834 bytes result sent to driver -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:65491 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:12:48 INFO CacheManager:59 - Partition rdd_51_0 not found, computing it -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 19) in 36 ms on localhost (1/2) -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_36_0 locally -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=268071, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block rdd_51_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added rdd_51_0 in memory on localhost:65491 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:12:48 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 19:12:48 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 18). 2834 bytes result sent to driver -2016-06-13 19:12:48 INFO BlockManager:59 - Removing RDD 42 -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 18) in 52 ms on localhost (2/2) -2016-06-13 19:12:48 INFO DAGScheduler:59 - ShuffleMapStage 29 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.055 s -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 19:12:48 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:12:48 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:12:48 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 30, ResultStage 31) -2016-06-13 19:12:48 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:12:48 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List() -2016-06-13 19:12:48 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-06-13 19:12:48 INFO DAGScheduler:59 - Submitting ShuffleMapStage 30 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[57] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:12:48 INFO ContextCleaner:59 - Cleaned RDD 42 -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:65491 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(6472) called with curMem=257749, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 6.3 KB, free 730.3 MB) -2016-06-13 19:12:48 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(3317) called with curMem=257749, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.3 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:65491 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:12:48 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:65491 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:12:48 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 30 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[57] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:65491 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:12:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 20, localhost, PROCESS_LOCAL, 2469 bytes) -2016-06-13 19:12:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 21, localhost, PROCESS_LOCAL, 2469 bytes) -2016-06-13 19:12:48 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 21) -2016-06-13 19:12:48 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 20) -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:12:48 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:65491 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:12:48 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:65491 in memory (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:12:48 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 21). 2470 bytes result sent to driver -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 21) in 40 ms on localhost (1/2) -2016-06-13 19:12:48 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 20). 2470 bytes result sent to driver -2016-06-13 19:12:48 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:65491 in memory (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:12:48 INFO DAGScheduler:59 - ShuffleMapStage 30 (mapPartitions at GraphImpl.scala:235) finished in 0.049 s -2016-06-13 19:12:48 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:12:48 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:12:48 INFO DAGScheduler:59 - waiting: Set(ResultStage 31) -2016-06-13 19:12:48 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 20) in 48 ms on localhost (2/2) -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 19:12:48 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 19:12:48 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List() -2016-06-13 19:12:48 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[62] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:65491 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(6920) called with curMem=220545, maxMem=766075207 -2016-06-13 19:12:48 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 6.8 KB, free 730.4 MB) -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(3109) called with curMem=224696, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:65491 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:12:48 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:65491 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:12:48 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[62] at map at PowerIterationClustering.scala:369) -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 19:12:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 22, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:12:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 23, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:12:48 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 22) -2016-06-13 19:12:48 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 23) -2016-06-13 19:12:48 INFO CacheManager:59 - Partition rdd_59_1 not found, computing it -2016-06-13 19:12:48 INFO CacheManager:59 - Partition rdd_59_0 not found, computing it -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_51_1 locally -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_51_0 locally -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=222709, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block rdd_59_0 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=224189, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block rdd_59_1 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added rdd_59_0 in memory on localhost:65491 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added rdd_59_1 in memory on localhost:65491 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:12:48 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 22). 2880 bytes result sent to driver -2016-06-13 19:12:48 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 23). 2880 bytes result sent to driver -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 22) in 20 ms on localhost (1/2) -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 23) in 20 ms on localhost (2/2) -2016-06-13 19:12:48 INFO DAGScheduler:59 - ResultStage 31 (sum at PowerIterationClustering.scala:369) finished in 0.021 s -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 19:12:48 INFO DAGScheduler:59 - Job 4 finished: sum at PowerIterationClustering.scala:369, took 0.256429 s -2016-06-13 19:12:48 INFO PowerIterationClustering:59 - Iteration 1: norm(v) = 36.22334869431644. -2016-06-13 19:12:48 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 152 bytes -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 152 bytes -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 151 bytes -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:12:48 INFO DAGScheduler:59 - Got job 5 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:12:48 INFO DAGScheduler:59 - Final stage: ResultStage 41(sum at PowerIterationClustering.scala:375) -2016-06-13 19:12:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 38, ShuffleMapStage 35, ShuffleMapStage 36, ShuffleMapStage 40) -2016-06-13 19:12:48 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:12:48 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[73] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(7352) called with curMem=225669, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 7.2 KB, free 730.4 MB) -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(3281) called with curMem=233021, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:65491 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:12:48 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[73] at values at PowerIterationClustering.scala:375) -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-06-13 19:12:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 24, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:12:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 25, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:12:48 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 25) -2016-06-13 19:12:48 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 24) -2016-06-13 19:12:48 INFO CacheManager:59 - Partition rdd_65_1 not found, computing it -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_51_1 locally -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_59_1 locally -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=236302, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block rdd_65_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:12:48 INFO CacheManager:59 - Partition rdd_65_0 not found, computing it -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_51_0 locally -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_59_0 locally -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=237734, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block rdd_65_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added rdd_65_1 in memory on localhost:65491 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added rdd_65_0 in memory on localhost:65491 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:12:48 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 25). 2664 bytes result sent to driver -2016-06-13 19:12:48 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 24). 2664 bytes result sent to driver -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 25) in 26 ms on localhost (1/2) -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 24) in 31 ms on localhost (2/2) -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-06-13 19:12:48 INFO DAGScheduler:59 - ResultStage 41 (sum at PowerIterationClustering.scala:375) finished in 0.036 s -2016-06-13 19:12:48 INFO DAGScheduler:59 - Job 5 finished: sum at PowerIterationClustering.scala:375, took 0.066123 s -2016-06-13 19:12:48 INFO PowerIterationClustering:59 - Iteration 1: delta = 0.0. -2016-06-13 19:12:48 INFO PowerIterationClustering:59 - Iteration 1: diff(delta) = 1.2794887333174252. -2016-06-13 19:12:48 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:12:48 INFO DAGScheduler:59 - Registering RDD 76 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:12:48 INFO DAGScheduler:59 - Registering RDD 80 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:12:48 INFO DAGScheduler:59 - Got job 6 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:12:48 INFO DAGScheduler:59 - Final stage: ResultStage 53(sum at PowerIterationClustering.scala:369) -2016-06-13 19:12:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 48, ShuffleMapStage 45, ShuffleMapStage 52, ShuffleMapStage 46, ShuffleMapStage 50) -2016-06-13 19:12:48 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 52) -2016-06-13 19:12:48 INFO DAGScheduler:59 - Submitting ShuffleMapStage 51 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[76] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(6568) called with curMem=239166, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 6.4 KB, free 730.4 MB) -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(3009) called with curMem=245734, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:65491 (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:12:48 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 51 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[76] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-06-13 19:12:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 26, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:12:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 27, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:12:48 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 27) -2016-06-13 19:12:48 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 26) -2016-06-13 19:12:48 INFO CacheManager:59 - Partition rdd_74_0 not found, computing it -2016-06-13 19:12:48 INFO CacheManager:59 - Partition rdd_74_1 not found, computing it -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_59_0 locally -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_59_1 locally -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=248743, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block rdd_74_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=250175, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block rdd_74_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added rdd_74_1 in memory on localhost:65491 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added rdd_74_0 in memory on localhost:65491 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:12:48 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 27). 2834 bytes result sent to driver -2016-06-13 19:12:48 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 26). 2834 bytes result sent to driver -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 27) in 17 ms on localhost (1/2) -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 26) in 20 ms on localhost (2/2) -2016-06-13 19:12:48 INFO DAGScheduler:59 - ShuffleMapStage 51 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.020 s -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-06-13 19:12:48 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:12:48 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:12:48 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 52, ResultStage 53) -2016-06-13 19:12:48 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:12:48 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 52: List() -2016-06-13 19:12:48 INFO DAGScheduler:59 - Missing parents for ResultStage 53: List(ShuffleMapStage 52) -2016-06-13 19:12:48 INFO DAGScheduler:59 - Submitting ShuffleMapStage 52 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[80] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(6472) called with curMem=251607, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 6.3 KB, free 730.3 MB) -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(3319) called with curMem=258079, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.3 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:65491 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:12:48 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 52 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[80] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-06-13 19:12:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 28, localhost, PROCESS_LOCAL, 2469 bytes) -2016-06-13 19:12:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 29, localhost, PROCESS_LOCAL, 2469 bytes) -2016-06-13 19:12:48 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 28) -2016-06-13 19:12:48 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 29) -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:12:48 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 28). 2470 bytes result sent to driver -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 28) in 33 ms on localhost (1/2) -2016-06-13 19:12:48 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 29). 2470 bytes result sent to driver -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 29) in 35 ms on localhost (2/2) -2016-06-13 19:12:48 INFO DAGScheduler:59 - ShuffleMapStage 52 (mapPartitions at GraphImpl.scala:235) finished in 0.036 s -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-06-13 19:12:48 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:12:48 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:12:48 INFO DAGScheduler:59 - waiting: Set(ResultStage 53) -2016-06-13 19:12:48 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:12:48 INFO DAGScheduler:59 - Missing parents for ResultStage 53: List() -2016-06-13 19:12:48 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[85] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(7504) called with curMem=261398, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 7.3 KB, free 730.3 MB) -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(3259) called with curMem=268902, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.3 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:65491 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:12:48 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[85] at map at PowerIterationClustering.scala:369) -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-06-13 19:12:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 30, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:12:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 31, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:12:48 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 31) -2016-06-13 19:12:48 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 30) -2016-06-13 19:12:48 INFO CacheManager:59 - Partition rdd_82_1 not found, computing it -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_74_1 locally -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:12:48 INFO CacheManager:59 - Partition rdd_82_0 not found, computing it -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_74_0 locally -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:12:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=272161, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block rdd_82_1 stored as values in memory (estimated size 1480.0 B, free 730.3 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added rdd_82_1 in memory on localhost:65491 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:12:48 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 31). 2880 bytes result sent to driver -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=273641, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block rdd_82_0 stored as values in memory (estimated size 1480.0 B, free 730.3 MB) -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 31) in 24 ms on localhost (1/2) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added rdd_82_0 in memory on localhost:65491 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:12:48 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 30). 2880 bytes result sent to driver -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 30) in 30 ms on localhost (2/2) -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-06-13 19:12:48 INFO DAGScheduler:59 - ResultStage 53 (sum at PowerIterationClustering.scala:369) finished in 0.031 s -2016-06-13 19:12:48 INFO DAGScheduler:59 - Job 6 finished: sum at PowerIterationClustering.scala:369, took 0.159487 s -2016-06-13 19:12:48 INFO PowerIterationClustering:59 - Iteration 2: norm(v) = 36.22334869431644. -2016-06-13 19:12:48 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 152 bytes -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 152 bytes -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 151 bytes -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 151 bytes -2016-06-13 19:12:48 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:12:48 INFO DAGScheduler:59 - Got job 7 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:12:48 INFO DAGScheduler:59 - Final stage: ResultStage 65(sum at PowerIterationClustering.scala:375) -2016-06-13 19:12:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 60, ShuffleMapStage 64, ShuffleMapStage 57, ShuffleMapStage 58, ShuffleMapStage 62) -2016-06-13 19:12:48 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:12:48 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[96] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(7936) called with curMem=275121, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 7.8 KB, free 730.3 MB) -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(3411) called with curMem=283057, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.3 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:65491 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:12:48 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 65 (MapPartitionsRDD[96] at values at PowerIterationClustering.scala:375) -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 2 tasks -2016-06-13 19:12:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 32, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:12:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 33, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:12:48 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 32) -2016-06-13 19:12:48 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 33) -2016-06-13 19:12:48 INFO CacheManager:59 - Partition rdd_88_1 not found, computing it -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_74_1 locally -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_82_1 locally -2016-06-13 19:12:48 INFO CacheManager:59 - Partition rdd_88_0 not found, computing it -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_74_0 locally -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_82_0 locally -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=286468, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block rdd_88_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=287900, maxMem=766075207 -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added rdd_88_1 in memory on localhost:65491 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:12:48 INFO MemoryStore:59 - Block rdd_88_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added rdd_88_0 in memory on localhost:65491 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:12:48 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 33). 2664 bytes result sent to driver -2016-06-13 19:12:48 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 32). 2664 bytes result sent to driver -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 33) in 18 ms on localhost (1/2) -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 32) in 23 ms on localhost (2/2) -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-06-13 19:12:48 INFO DAGScheduler:59 - ResultStage 65 (sum at PowerIterationClustering.scala:375) finished in 0.023 s -2016-06-13 19:12:48 INFO DAGScheduler:59 - Job 7 finished: sum at PowerIterationClustering.scala:375, took 0.053722 s -2016-06-13 19:12:48 INFO PowerIterationClustering:59 - Iteration 2: delta = 0.0. -2016-06-13 19:12:48 INFO PowerIterationClustering:59 - Iteration 2: diff(delta) = 0.0. -2016-06-13 19:12:48 WARN KMeans:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:12:48 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:12:48 INFO DAGScheduler:59 - Got job 8 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:12:48 INFO DAGScheduler:59 - Final stage: ResultStage 77(takeSample at KMeans.scala:376) -2016-06-13 19:12:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 70, ShuffleMapStage 74, ShuffleMapStage 72, ShuffleMapStage 69, ShuffleMapStage 76) -2016-06-13 19:12:48 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:12:48 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[104] at map at KMeans.scala:210), which has no missing parents -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(8168) called with curMem=289332, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 8.0 KB, free 730.3 MB) -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(3451) called with curMem=297500, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.3 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:65491 (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:12:48 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 77 (MapPartitionsRDD[104] at map at KMeans.scala:210) -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 2 tasks -2016-06-13 19:12:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 34, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:12:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 35, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:12:48 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 35) -2016-06-13 19:12:48 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 34) -2016-06-13 19:12:48 INFO CacheManager:59 - Partition rdd_99_1 not found, computing it -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_82_1 locally -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=300951, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block rdd_99_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:12:48 INFO CacheManager:59 - Partition rdd_99_0 not found, computing it -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_82_0 locally -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added rdd_99_1 in memory on localhost:65491 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:12:48 INFO CacheManager:59 - Partition rdd_102_1 not found, computing it -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_99_1 locally -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=302383, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block rdd_99_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added rdd_99_0 in memory on localhost:65491 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:12:48 INFO CacheManager:59 - Partition rdd_102_0 not found, computing it -2016-06-13 19:12:48 INFO BlockManager:59 - Found block rdd_99_0 locally -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=303815, maxMem=766075207 -2016-06-13 19:12:48 INFO MemoryStore:59 - Block rdd_102_1 stored as values in memory (estimated size 240.0 B, free 730.3 MB) -2016-06-13 19:12:48 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=304055, maxMem=766075207 -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added rdd_102_1 in memory on localhost:65491 (size: 240.0 B, free: 730.5 MB) -2016-06-13 19:12:48 INFO MemoryStore:59 - Block rdd_102_0 stored as values in memory (estimated size 240.0 B, free 730.3 MB) -2016-06-13 19:12:48 INFO BlockManagerInfo:59 - Added rdd_102_0 in memory on localhost:65491 (size: 240.0 B, free: 730.5 MB) -2016-06-13 19:12:48 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 34). 2717 bytes result sent to driver -2016-06-13 19:12:48 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 35). 2717 bytes result sent to driver -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 34) in 32 ms on localhost (1/2) -2016-06-13 19:12:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 35) in 31 ms on localhost (2/2) -2016-06-13 19:12:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-06-13 19:12:48 INFO DAGScheduler:59 - ResultStage 77 (takeSample at KMeans.scala:376) finished in 0.034 s -2016-06-13 19:12:48 INFO DAGScheduler:59 - Job 8 finished: takeSample at KMeans.scala:376, took 0.071504 s -2016-06-13 19:12:49 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Got job 9 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:12:49 INFO DAGScheduler:59 - Final stage: ResultStage 89(takeSample at KMeans.scala:376) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 84, ShuffleMapStage 81, ShuffleMapStage 88, ShuffleMapStage 82, ShuffleMapStage 86) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting ResultStage 89 (PartitionwiseSampledRDD[106] at takeSample at KMeans.scala:376), which has no missing parents -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(25560) called with curMem=304295, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 25.0 KB, free 730.3 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(10162) called with curMem=329855, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 9.9 KB, free 730.3 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:65491 (size: 9.9 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 89 (PartitionwiseSampledRDD[106] at takeSample at KMeans.scala:376) -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Adding task set 89.0 with 2 tasks -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 89.0 (TID 36, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 89.0 (TID 37, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:12:49 INFO Executor:59 - Running task 0.0 in stage 89.0 (TID 36) -2016-06-13 19:12:49 INFO Executor:59 - Running task 1.0 in stage 89.0 (TID 37) -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_0 locally -2016-06-13 19:12:49 INFO Executor:59 - Finished task 0.0 in stage 89.0 (TID 36). 2633 bytes result sent to driver -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_1 locally -2016-06-13 19:12:49 INFO Executor:59 - Finished task 1.0 in stage 89.0 (TID 37). 2575 bytes result sent to driver -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 89.0 (TID 36) in 22 ms on localhost (1/2) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 89.0 (TID 37) in 21 ms on localhost (2/2) -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 89.0, whose tasks have all completed, from pool -2016-06-13 19:12:49 INFO DAGScheduler:59 - ResultStage 89 (takeSample at KMeans.scala:376) finished in 0.025 s -2016-06-13 19:12:49 INFO DAGScheduler:59 - Job 9 finished: takeSample at KMeans.scala:376, took 0.058596 s -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=340017, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 880.0 B, free 730.3 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(487) called with curMem=340897, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 487.0 B, free 730.3 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:65491 (size: 487.0 B, free: 730.5 MB) -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 20 from broadcast at KMeans.scala:394 -2016-06-13 19:12:49 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Got job 10 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:12:49 INFO DAGScheduler:59 - Final stage: ResultStage 101(aggregate at KMeans.scala:402) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 96, ShuffleMapStage 100, ShuffleMapStage 93, ShuffleMapStage 94, ShuffleMapStage 98) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting ResultStage 101 (MapPartitionsRDD[108] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(10192) called with curMem=341384, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 10.0 KB, free 730.3 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(4258) called with curMem=351576, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.2 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:65491 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 101 (MapPartitionsRDD[108] at map at KMeans.scala:396) -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Adding task set 101.0 with 2 tasks -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 101.0 (TID 38, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 101.0 (TID 39, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:12:49 INFO Executor:59 - Running task 0.0 in stage 101.0 (TID 38) -2016-06-13 19:12:49 INFO Executor:59 - Running task 1.0 in stage 101.0 (TID 39) -2016-06-13 19:12:49 INFO CacheManager:59 - Partition rdd_108_1 not found, computing it -2016-06-13 19:12:49 INFO CacheManager:59 - Partition rdd_108_0 not found, computing it -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_0 locally -2016-06-13 19:12:49 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 19:12:49 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=355834, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block rdd_108_1 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=356330, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block rdd_108_0 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added rdd_108_1 in memory on localhost:65491 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added rdd_108_0 in memory on localhost:65491 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:12:49 INFO Executor:59 - Finished task 1.0 in stage 101.0 (TID 39). 2647 bytes result sent to driver -2016-06-13 19:12:49 INFO Executor:59 - Finished task 0.0 in stage 101.0 (TID 38). 2647 bytes result sent to driver -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 101.0 (TID 39) in 31 ms on localhost (1/2) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 101.0 (TID 38) in 34 ms on localhost (2/2) -2016-06-13 19:12:49 INFO DAGScheduler:59 - ResultStage 101 (aggregate at KMeans.scala:402) finished in 0.034 s -2016-06-13 19:12:49 INFO DAGScheduler:59 - Job 10 finished: aggregate at KMeans.scala:402, took 0.059954 s -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 101.0, whose tasks have all completed, from pool -2016-06-13 19:12:49 INFO MapPartitionsRDD:59 - Removing RDD 105 from persistence list -2016-06-13 19:12:49 INFO BlockManager:59 - Removing RDD 105 -2016-06-13 19:12:49 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:65491 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Got job 11 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:12:49 INFO DAGScheduler:59 - Final stage: ResultStage 113(collect at KMeans.scala:431) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 108, ShuffleMapStage 105, ShuffleMapStage 112, ShuffleMapStage 106, ShuffleMapStage 110) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:65491 in memory (size: 9.9 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting ResultStage 113 (MapPartitionsRDD[110] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(10240) called with curMem=303203, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 10.0 KB, free 730.3 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(4341) called with curMem=313443, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.3 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:65491 in memory (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:65491 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 113 (MapPartitionsRDD[110] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Adding task set 113.0 with 2 tasks -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 113.0 (TID 40, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 113.0 (TID 41, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:12:49 INFO Executor:59 - Running task 1.0 in stage 113.0 (TID 41) -2016-06-13 19:12:49 INFO Executor:59 - Running task 0.0 in stage 113.0 (TID 40) -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_108_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_108_0 locally -2016-06-13 19:12:49 INFO Executor:59 - Finished task 1.0 in stage 113.0 (TID 41). 3471 bytes result sent to driver -2016-06-13 19:12:49 INFO Executor:59 - Finished task 0.0 in stage 113.0 (TID 40). 3118 bytes result sent to driver -2016-06-13 19:12:49 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 113.0 (TID 40) in 21 ms on localhost (1/2) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 113.0 (TID 41) in 21 ms on localhost (2/2) -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 113.0, whose tasks have all completed, from pool -2016-06-13 19:12:49 INFO DAGScheduler:59 - ResultStage 113 (collect at KMeans.scala:431) finished in 0.023 s -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:65491 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Job 11 finished: collect at KMeans.scala:431, took 0.055894 s -2016-06-13 19:12:49 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(1648) called with curMem=298269, maxMem=766075207 -2016-06-13 19:12:49 INFO BlockManager:59 - Removing RDD 88 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 1648.0 B, free 730.3 MB) -2016-06-13 19:12:49 INFO ContextCleaner:59 - Cleaned RDD 88 -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(600) called with curMem=297053, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 600.0 B, free 730.3 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:65491 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:65491 (size: 600.0 B, free: 730.5 MB) -2016-06-13 19:12:49 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 23 from broadcast at KMeans.scala:394 -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:65491 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:65491 in memory (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:65491 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 19:12:49 INFO BlockManager:59 - Removing RDD 65 -2016-06-13 19:12:49 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:12:49 INFO ContextCleaner:59 - Cleaned RDD 65 -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:65491 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:65491 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Got job 12 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:12:49 INFO DAGScheduler:59 - Final stage: ResultStage 125(aggregate at KMeans.scala:402) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 117, ShuffleMapStage 118, ShuffleMapStage 122, ShuffleMapStage 120, ShuffleMapStage 124) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:65491 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting ResultStage 125 (MapPartitionsRDD[112] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(10424) called with curMem=225360, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 10.2 KB, free 730.4 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(4351) called with curMem=235784, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:65491 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 125 (MapPartitionsRDD[112] at map at KMeans.scala:396) -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Adding task set 125.0 with 2 tasks -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 125.0 (TID 42, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 125.0 (TID 43, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:12:49 INFO Executor:59 - Running task 0.0 in stage 125.0 (TID 42) -2016-06-13 19:12:49 INFO Executor:59 - Running task 1.0 in stage 125.0 (TID 43) -2016-06-13 19:12:49 INFO CacheManager:59 - Partition rdd_112_1 not found, computing it -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_1 locally -2016-06-13 19:12:49 INFO CacheManager:59 - Partition rdd_112_0 not found, computing it -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_108_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_108_0 locally -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=240135, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block rdd_112_1 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=240631, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block rdd_112_0 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added rdd_112_1 in memory on localhost:65491 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:12:49 INFO Executor:59 - Finished task 1.0 in stage 125.0 (TID 43). 2647 bytes result sent to driver -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added rdd_112_0 in memory on localhost:65491 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:12:49 INFO Executor:59 - Finished task 0.0 in stage 125.0 (TID 42). 2647 bytes result sent to driver -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 125.0 (TID 43) in 13 ms on localhost (1/2) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 125.0 (TID 42) in 14 ms on localhost (2/2) -2016-06-13 19:12:49 INFO DAGScheduler:59 - ResultStage 125 (aggregate at KMeans.scala:402) finished in 0.015 s -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 125.0, whose tasks have all completed, from pool -2016-06-13 19:12:49 INFO DAGScheduler:59 - Job 12 finished: aggregate at KMeans.scala:402, took 0.040029 s -2016-06-13 19:12:49 INFO MapPartitionsRDD:59 - Removing RDD 108 from persistence list -2016-06-13 19:12:49 INFO BlockManager:59 - Removing RDD 108 -2016-06-13 19:12:49 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Got job 13 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:12:49 INFO DAGScheduler:59 - Final stage: ResultStage 137(collect at KMeans.scala:431) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 132, ShuffleMapStage 129, ShuffleMapStage 136, ShuffleMapStage 130, ShuffleMapStage 134) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting ResultStage 137 (MapPartitionsRDD[114] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(10480) called with curMem=240135, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 10.2 KB, free 730.3 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(4428) called with curMem=250615, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:65491 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 137 (MapPartitionsRDD[114] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Adding task set 137.0 with 2 tasks -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 137.0 (TID 44, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 137.0 (TID 45, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:12:49 INFO Executor:59 - Running task 1.0 in stage 137.0 (TID 45) -2016-06-13 19:12:49 INFO Executor:59 - Running task 0.0 in stage 137.0 (TID 44) -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_112_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_112_0 locally -2016-06-13 19:12:49 INFO Executor:59 - Finished task 1.0 in stage 137.0 (TID 45). 3240 bytes result sent to driver -2016-06-13 19:12:49 INFO Executor:59 - Finished task 0.0 in stage 137.0 (TID 44). 2990 bytes result sent to driver -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 137.0 (TID 45) in 7 ms on localhost (1/2) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 137.0 (TID 44) in 8 ms on localhost (2/2) -2016-06-13 19:12:49 INFO DAGScheduler:59 - ResultStage 137 (collect at KMeans.scala:431) finished in 0.008 s -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 137.0, whose tasks have all completed, from pool -2016-06-13 19:12:49 INFO DAGScheduler:59 - Job 13 finished: collect at KMeans.scala:431, took 0.036830 s -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(1456) called with curMem=255043, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 1456.0 B, free 730.3 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(582) called with curMem=256499, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 582.0 B, free 730.3 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:65491 (size: 582.0 B, free: 730.5 MB) -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 26 from broadcast at KMeans.scala:394 -2016-06-13 19:12:49 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Got job 14 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:12:49 INFO DAGScheduler:59 - Final stage: ResultStage 149(aggregate at KMeans.scala:402) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 144, ShuffleMapStage 141, ShuffleMapStage 148, ShuffleMapStage 142, ShuffleMapStage 146) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting ResultStage 149 (MapPartitionsRDD[116] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(10656) called with curMem=257081, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 10.4 KB, free 730.3 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(4411) called with curMem=267737, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:65491 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 149 (MapPartitionsRDD[116] at map at KMeans.scala:396) -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Adding task set 149.0 with 2 tasks -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 149.0 (TID 46, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 149.0 (TID 47, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:12:49 INFO Executor:59 - Running task 0.0 in stage 149.0 (TID 46) -2016-06-13 19:12:49 INFO Executor:59 - Running task 1.0 in stage 149.0 (TID 47) -2016-06-13 19:12:49 INFO CacheManager:59 - Partition rdd_116_1 not found, computing it -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_112_1 locally -2016-06-13 19:12:49 INFO CacheManager:59 - Partition rdd_116_0 not found, computing it -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_112_0 locally -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=272148, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block rdd_116_1 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=272644, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block rdd_116_0 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added rdd_116_1 in memory on localhost:65491 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:12:49 INFO Executor:59 - Finished task 1.0 in stage 149.0 (TID 47). 2647 bytes result sent to driver -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added rdd_116_0 in memory on localhost:65491 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:12:49 INFO Executor:59 - Finished task 0.0 in stage 149.0 (TID 46). 2647 bytes result sent to driver -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 149.0 (TID 47) in 14 ms on localhost (1/2) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 149.0 (TID 46) in 16 ms on localhost (2/2) -2016-06-13 19:12:49 INFO DAGScheduler:59 - ResultStage 149 (aggregate at KMeans.scala:402) finished in 0.016 s -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 149.0, whose tasks have all completed, from pool -2016-06-13 19:12:49 INFO DAGScheduler:59 - Job 14 finished: aggregate at KMeans.scala:402, took 0.038498 s -2016-06-13 19:12:49 INFO MapPartitionsRDD:59 - Removing RDD 112 from persistence list -2016-06-13 19:12:49 INFO BlockManager:59 - Removing RDD 112 -2016-06-13 19:12:49 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Got job 15 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:12:49 INFO DAGScheduler:59 - Final stage: ResultStage 161(collect at KMeans.scala:431) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 153, ShuffleMapStage 154, ShuffleMapStage 158, ShuffleMapStage 156, ShuffleMapStage 160) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting ResultStage 161 (MapPartitionsRDD[118] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(10712) called with curMem=272148, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 10.5 KB, free 730.3 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(4449) called with curMem=282860, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:65491 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 161 (MapPartitionsRDD[118] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Adding task set 161.0 with 2 tasks -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 161.0 (TID 48, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 161.0 (TID 49, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:12:49 INFO Executor:59 - Running task 1.0 in stage 161.0 (TID 49) -2016-06-13 19:12:49 INFO Executor:59 - Running task 0.0 in stage 161.0 (TID 48) -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_116_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_116_0 locally -2016-06-13 19:12:49 INFO Executor:59 - Finished task 0.0 in stage 161.0 (TID 48). 2722 bytes result sent to driver -2016-06-13 19:12:49 INFO Executor:59 - Finished task 1.0 in stage 161.0 (TID 49). 2040 bytes result sent to driver -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 161.0 (TID 48) in 8 ms on localhost (1/2) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 161.0 (TID 49) in 8 ms on localhost (2/2) -2016-06-13 19:12:49 INFO DAGScheduler:59 - ResultStage 161 (collect at KMeans.scala:431) finished in 0.009 s -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 161.0, whose tasks have all completed, from pool -2016-06-13 19:12:49 INFO DAGScheduler:59 - Job 15 finished: collect at KMeans.scala:431, took 0.030005 s -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(648) called with curMem=287309, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 648.0 B, free 730.3 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(429) called with curMem=287957, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 429.0 B, free 730.3 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:65491 (size: 429.0 B, free: 730.5 MB) -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 29 from broadcast at KMeans.scala:394 -2016-06-13 19:12:49 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Got job 16 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:12:49 INFO DAGScheduler:59 - Final stage: ResultStage 173(aggregate at KMeans.scala:402) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 168, ShuffleMapStage 165, ShuffleMapStage 172, ShuffleMapStage 166, ShuffleMapStage 170) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting ResultStage 173 (MapPartitionsRDD[120] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(10888) called with curMem=288386, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 10.6 KB, free 730.3 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(4444) called with curMem=299274, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:65491 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 173 (MapPartitionsRDD[120] at map at KMeans.scala:396) -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Adding task set 173.0 with 2 tasks -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 173.0 (TID 50, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 173.0 (TID 51, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:12:49 INFO Executor:59 - Running task 0.0 in stage 173.0 (TID 50) -2016-06-13 19:12:49 INFO Executor:59 - Running task 1.0 in stage 173.0 (TID 51) -2016-06-13 19:12:49 INFO CacheManager:59 - Partition rdd_120_0 not found, computing it -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_116_0 locally -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=303718, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block rdd_120_0 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added rdd_120_0 in memory on localhost:65491 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:12:49 INFO Executor:59 - Finished task 0.0 in stage 173.0 (TID 50). 2647 bytes result sent to driver -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 173.0 (TID 50) in 14 ms on localhost (1/2) -2016-06-13 19:12:49 INFO CacheManager:59 - Partition rdd_120_1 not found, computing it -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_116_1 locally -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=304214, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block rdd_120_1 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added rdd_120_1 in memory on localhost:65491 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:12:49 INFO Executor:59 - Finished task 1.0 in stage 173.0 (TID 51). 2647 bytes result sent to driver -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 173.0 (TID 51) in 40 ms on localhost (2/2) -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 173.0, whose tasks have all completed, from pool -2016-06-13 19:12:49 INFO DAGScheduler:59 - ResultStage 173 (aggregate at KMeans.scala:402) finished in 0.042 s -2016-06-13 19:12:49 INFO DAGScheduler:59 - Job 16 finished: aggregate at KMeans.scala:402, took 0.060013 s -2016-06-13 19:12:49 INFO MapPartitionsRDD:59 - Removing RDD 116 from persistence list -2016-06-13 19:12:49 INFO BlockManager:59 - Removing RDD 116 -2016-06-13 19:12:49 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Got job 17 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:12:49 INFO DAGScheduler:59 - Final stage: ResultStage 185(collect at KMeans.scala:431) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 180, ShuffleMapStage 184, ShuffleMapStage 177, ShuffleMapStage 178, ShuffleMapStage 182) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting ResultStage 185 (MapPartitionsRDD[122] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(10944) called with curMem=303718, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 10.7 KB, free 730.3 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(4496) called with curMem=314662, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:65491 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 185 (MapPartitionsRDD[122] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Adding task set 185.0 with 2 tasks -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 185.0 (TID 52, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 185.0 (TID 53, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:12:49 INFO Executor:59 - Running task 0.0 in stage 185.0 (TID 52) -2016-06-13 19:12:49 INFO Executor:59 - Running task 1.0 in stage 185.0 (TID 53) -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_120_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_120_0 locally -2016-06-13 19:12:49 INFO Executor:59 - Finished task 1.0 in stage 185.0 (TID 53). 2040 bytes result sent to driver -2016-06-13 19:12:49 INFO Executor:59 - Finished task 0.0 in stage 185.0 (TID 52). 2040 bytes result sent to driver -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 185.0 (TID 53) in 6 ms on localhost (1/2) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 185.0 (TID 52) in 7 ms on localhost (2/2) -2016-06-13 19:12:49 INFO DAGScheduler:59 - ResultStage 185 (collect at KMeans.scala:431) finished in 0.009 s -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 185.0, whose tasks have all completed, from pool -2016-06-13 19:12:49 INFO DAGScheduler:59 - Job 17 finished: collect at KMeans.scala:431, took 0.031888 s -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(584) called with curMem=319158, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 584.0 B, free 730.3 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(236) called with curMem=319742, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 236.0 B, free 730.3 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:65491 (size: 236.0 B, free: 730.5 MB) -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 32 from broadcast at KMeans.scala:394 -2016-06-13 19:12:49 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Got job 18 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:12:49 INFO DAGScheduler:59 - Final stage: ResultStage 197(aggregate at KMeans.scala:402) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 190, ShuffleMapStage 194, ShuffleMapStage 192, ShuffleMapStage 189, ShuffleMapStage 196) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting ResultStage 197 (MapPartitionsRDD[124] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(11128) called with curMem=319978, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 10.9 KB, free 730.3 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(4507) called with curMem=331106, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:65491 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 197 (MapPartitionsRDD[124] at map at KMeans.scala:396) -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Adding task set 197.0 with 2 tasks -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 197.0 (TID 54, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 197.0 (TID 55, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:12:49 INFO Executor:59 - Running task 0.0 in stage 197.0 (TID 54) -2016-06-13 19:12:49 INFO Executor:59 - Running task 1.0 in stage 197.0 (TID 55) -2016-06-13 19:12:49 INFO CacheManager:59 - Partition rdd_124_0 not found, computing it -2016-06-13 19:12:49 INFO CacheManager:59 - Partition rdd_124_1 not found, computing it -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_120_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_120_1 locally -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=335613, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block rdd_124_0 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=336109, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block rdd_124_1 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added rdd_124_0 in memory on localhost:65491 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:12:49 INFO Executor:59 - Finished task 0.0 in stage 197.0 (TID 54). 2647 bytes result sent to driver -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added rdd_124_1 in memory on localhost:65491 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 197.0 (TID 54) in 10 ms on localhost (1/2) -2016-06-13 19:12:49 INFO Executor:59 - Finished task 1.0 in stage 197.0 (TID 55). 2647 bytes result sent to driver -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 197.0 (TID 55) in 11 ms on localhost (2/2) -2016-06-13 19:12:49 INFO DAGScheduler:59 - ResultStage 197 (aggregate at KMeans.scala:402) finished in 0.012 s -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 197.0, whose tasks have all completed, from pool -2016-06-13 19:12:49 INFO DAGScheduler:59 - Job 18 finished: aggregate at KMeans.scala:402, took 0.032265 s -2016-06-13 19:12:49 INFO MapPartitionsRDD:59 - Removing RDD 120 from persistence list -2016-06-13 19:12:49 INFO BlockManager:59 - Removing RDD 120 -2016-06-13 19:12:49 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Got job 19 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:12:49 INFO DAGScheduler:59 - Final stage: ResultStage 209(collect at KMeans.scala:431) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 204, ShuffleMapStage 201, ShuffleMapStage 208, ShuffleMapStage 202, ShuffleMapStage 206) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting ResultStage 209 (MapPartitionsRDD[126] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(11176) called with curMem=335613, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 10.9 KB, free 730.3 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(4538) called with curMem=346789, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:65491 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 209 (MapPartitionsRDD[126] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Adding task set 209.0 with 2 tasks -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 209.0 (TID 56, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 209.0 (TID 57, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:12:49 INFO Executor:59 - Running task 1.0 in stage 209.0 (TID 57) -2016-06-13 19:12:49 INFO Executor:59 - Running task 0.0 in stage 209.0 (TID 56) -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_124_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_1 locally -2016-06-13 19:12:49 INFO Executor:59 - Finished task 0.0 in stage 209.0 (TID 56). 2040 bytes result sent to driver -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_124_1 locally -2016-06-13 19:12:49 INFO Executor:59 - Finished task 1.0 in stage 209.0 (TID 57). 2040 bytes result sent to driver -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 209.0 (TID 56) in 8 ms on localhost (1/2) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 209.0 (TID 57) in 8 ms on localhost (2/2) -2016-06-13 19:12:49 INFO DAGScheduler:59 - ResultStage 209 (collect at KMeans.scala:431) finished in 0.010 s -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 209.0, whose tasks have all completed, from pool -2016-06-13 19:12:49 INFO DAGScheduler:59 - Job 19 finished: collect at KMeans.scala:431, took 0.027013 s -2016-06-13 19:12:49 INFO MapPartitionsRDD:59 - Removing RDD 124 from persistence list -2016-06-13 19:12:49 INFO BlockManager:59 - Removing RDD 124 -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(2880) called with curMem=350335, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 2.8 KB, free 730.2 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(708) called with curMem=353215, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 708.0 B, free 730.2 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:65491 (size: 708.0 B, free: 730.5 MB) -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 35 from broadcast at KMeans.scala:445 -2016-06-13 19:12:49 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:450 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Registering RDD 127 (flatMap at KMeans.scala:446) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Got job 20 (collectAsMap at KMeans.scala:450) with 2 output partitions -2016-06-13 19:12:49 INFO DAGScheduler:59 - Final stage: ResultStage 222(collectAsMap at KMeans.scala:450) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 221) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 221) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting ShuffleMapStage 221 (MapPartitionsRDD[127] at flatMap at KMeans.scala:446), which has no missing parents -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(9672) called with curMem=353923, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 9.4 KB, free 730.2 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(4099) called with curMem=363595, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.2 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:65491 (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 221 (MapPartitionsRDD[127] at flatMap at KMeans.scala:446) -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Adding task set 221.0 with 2 tasks -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 221.0 (TID 58, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 221.0 (TID 59, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:12:49 INFO Executor:59 - Running task 0.0 in stage 221.0 (TID 58) -2016-06-13 19:12:49 INFO Executor:59 - Running task 1.0 in stage 221.0 (TID 59) -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_0 locally -2016-06-13 19:12:49 INFO Executor:59 - Finished task 0.0 in stage 221.0 (TID 58). 2254 bytes result sent to driver -2016-06-13 19:12:49 INFO Executor:59 - Finished task 1.0 in stage 221.0 (TID 59). 2254 bytes result sent to driver -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 221.0 (TID 58) in 35 ms on localhost (1/2) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 221.0 (TID 59) in 35 ms on localhost (2/2) -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 221.0, whose tasks have all completed, from pool -2016-06-13 19:12:49 INFO DAGScheduler:59 - ShuffleMapStage 221 (flatMap at KMeans.scala:446) finished in 0.036 s -2016-06-13 19:12:49 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:12:49 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:12:49 INFO DAGScheduler:59 - waiting: Set(ResultStage 222) -2016-06-13 19:12:49 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:12:49 INFO DAGScheduler:59 - Missing parents for ResultStage 222: List() -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting ResultStage 222 (ShuffledRDD[128] at reduceByKey at KMeans.scala:450), which is now runnable -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=367694, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 2.3 KB, free 730.2 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(1382) called with curMem=370006, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 1382.0 B, free 730.2 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:65491 (size: 1382.0 B, free: 730.5 MB) -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 222 (ShuffledRDD[128] at reduceByKey at KMeans.scala:450) -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Adding task set 222.0 with 2 tasks -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 222.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 222.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:12:49 INFO Executor:59 - Running task 0.0 in stage 222.0 (TID 60) -2016-06-13 19:12:49 INFO Executor:59 - Running task 1.0 in stage 222.0 (TID 61) -2016-06-13 19:12:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:12:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:12:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:12:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:12:49 INFO Executor:59 - Finished task 1.0 in stage 222.0 (TID 61). 1744 bytes result sent to driver -2016-06-13 19:12:49 INFO Executor:59 - Finished task 0.0 in stage 222.0 (TID 60). 1852 bytes result sent to driver -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 222.0 (TID 61) in 16 ms on localhost (1/2) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 222.0 (TID 60) in 18 ms on localhost (2/2) -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 222.0, whose tasks have all completed, from pool -2016-06-13 19:12:49 INFO DAGScheduler:59 - ResultStage 222 (collectAsMap at KMeans.scala:450) finished in 0.019 s -2016-06-13 19:12:49 INFO DAGScheduler:59 - Job 20 finished: collectAsMap at KMeans.scala:450, took 0.077623 s -2016-06-13 19:12:49 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:12:49 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:12:49 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:12:49 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:12:49 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:12:49 INFO KMeans:59 - Initialization with k-means|| took 1.059 seconds. -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(824) called with curMem=371388, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 824.0 B, free 730.2 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(339) called with curMem=372212, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 339.0 B, free 730.2 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:65491 (size: 339.0 B, free: 730.5 MB) -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 38 from broadcast at KMeans.scala:276 -2016-06-13 19:12:49 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:302 -2016-06-13 19:12:49 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:12:49 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 152 bytes -2016-06-13 19:12:49 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 152 bytes -2016-06-13 19:12:49 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:12:49 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:12:49 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 151 bytes -2016-06-13 19:12:49 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:12:49 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 151 bytes -2016-06-13 19:12:49 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:12:49 INFO DAGScheduler:59 - Registering RDD 129 (mapPartitions at KMeans.scala:279) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Got job 21 (collectAsMap at KMeans.scala:302) with 2 output partitions -2016-06-13 19:12:49 INFO DAGScheduler:59 - Final stage: ResultStage 235(collectAsMap at KMeans.scala:302) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 234) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 234) -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting ShuffleMapStage 234 (MapPartitionsRDD[129] at mapPartitions at KMeans.scala:279), which has no missing parents -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(10296) called with curMem=372551, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 10.1 KB, free 730.2 MB) -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(4432) called with curMem=382847, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.2 MB) -2016-06-13 19:12:49 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:65491 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:12:49 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 234 (MapPartitionsRDD[129] at mapPartitions at KMeans.scala:279) -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Adding task set 234.0 with 2 tasks -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 234.0 (TID 62, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 234.0 (TID 63, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:12:49 INFO Executor:59 - Running task 0.0 in stage 234.0 (TID 62) -2016-06-13 19:12:49 INFO Executor:59 - Running task 1.0 in stage 234.0 (TID 63) -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_1 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_99_0 locally -2016-06-13 19:12:49 INFO BlockManager:59 - Found block rdd_102_0 locally -2016-06-13 19:12:49 INFO Executor:59 - Finished task 1.0 in stage 234.0 (TID 63). 2413 bytes result sent to driver -2016-06-13 19:12:49 INFO Executor:59 - Finished task 0.0 in stage 234.0 (TID 62). 2413 bytes result sent to driver -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 234.0 (TID 63) in 17 ms on localhost (1/2) -2016-06-13 19:12:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 234.0 (TID 62) in 17 ms on localhost (2/2) -2016-06-13 19:12:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 234.0, whose tasks have all completed, from pool -2016-06-13 19:12:49 INFO DAGScheduler:59 - ShuffleMapStage 234 (mapPartitions at KMeans.scala:279) finished in 0.018 s -2016-06-13 19:12:49 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:12:49 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:12:49 INFO DAGScheduler:59 - waiting: Set(ResultStage 235) -2016-06-13 19:12:49 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:12:49 INFO DAGScheduler:59 - Missing parents for ResultStage 235: List() -2016-06-13 19:12:49 INFO DAGScheduler:59 - Submitting ResultStage 235 (ShuffledRDD[130] at reduceByKey at KMeans.scala:302), which is now runnable -2016-06-13 19:12:49 INFO MemoryStore:59 - ensureFreeSpace(2808) called with curMem=387279, maxMem=766075207 -2016-06-13 19:12:49 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 2.7 KB, free 730.2 MB) -2016-06-13 19:12:50 INFO MemoryStore:59 - ensureFreeSpace(1589) called with curMem=390087, maxMem=766075207 -2016-06-13 19:12:50 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 1589.0 B, free 730.2 MB) -2016-06-13 19:12:50 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:65491 (size: 1589.0 B, free: 730.5 MB) -2016-06-13 19:12:50 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 235 (ShuffledRDD[130] at reduceByKey at KMeans.scala:302) -2016-06-13 19:12:50 INFO TaskSchedulerImpl:59 - Adding task set 235.0 with 2 tasks -2016-06-13 19:12:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 235.0 (TID 64, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:12:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 235.0 (TID 65, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:12:50 INFO Executor:59 - Running task 0.0 in stage 235.0 (TID 64) -2016-06-13 19:12:50 INFO Executor:59 - Running task 1.0 in stage 235.0 (TID 65) -2016-06-13 19:12:50 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:12:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:12:50 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:12:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:12:50 INFO Executor:59 - Finished task 0.0 in stage 235.0 (TID 64). 1785 bytes result sent to driver -2016-06-13 19:12:50 INFO Executor:59 - Finished task 1.0 in stage 235.0 (TID 65). 1671 bytes result sent to driver -2016-06-13 19:12:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 235.0 (TID 64) in 8 ms on localhost (1/2) -2016-06-13 19:12:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 235.0 (TID 65) in 9 ms on localhost (2/2) -2016-06-13 19:12:50 INFO DAGScheduler:59 - ResultStage 235 (collectAsMap at KMeans.scala:302) finished in 0.009 s -2016-06-13 19:12:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 235.0, whose tasks have all completed, from pool -2016-06-13 19:12:50 INFO DAGScheduler:59 - Job 21 finished: collectAsMap at KMeans.scala:302, took 0.052929 s -2016-06-13 19:12:50 INFO KMeans:59 - Run 0 finished in 1 iterations -2016-06-13 19:12:50 INFO KMeans:59 - Run 1 finished in 1 iterations -2016-06-13 19:12:50 INFO KMeans:59 - Run 2 finished in 1 iterations -2016-06-13 19:12:50 INFO KMeans:59 - Run 3 finished in 1 iterations -2016-06-13 19:12:50 INFO KMeans:59 - Run 4 finished in 1 iterations -2016-06-13 19:12:50 INFO KMeans:59 - Iterations took 0.080 seconds. -2016-06-13 19:12:50 INFO KMeans:59 - KMeans converged in 1 iterations. -2016-06-13 19:12:50 INFO KMeans:59 - The cost for the best run is 6.650150218788335E-6. -2016-06-13 19:12:50 INFO MapPartitionsRDD:59 - Removing RDD 102 from persistence list -2016-06-13 19:12:50 INFO BlockManager:59 - Removing RDD 102 -2016-06-13 19:12:50 WARN KMeans:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:12:50 INFO SparkContext:59 - Starting job: foreach at PICFromWebSuite.scala:32 -2016-06-13 19:12:50 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:12:50 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 152 bytes -2016-06-13 19:12:50 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 152 bytes -2016-06-13 19:12:50 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:12:50 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:12:50 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 151 bytes -2016-06-13 19:12:50 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:12:50 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 151 bytes -2016-06-13 19:12:50 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:12:50 INFO DAGScheduler:59 - Got job 22 (foreach at PICFromWebSuite.scala:32) with 2 output partitions -2016-06-13 19:12:50 INFO DAGScheduler:59 - Final stage: ResultStage 247(foreach at PICFromWebSuite.scala:32) -2016-06-13 19:12:50 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 242, ShuffleMapStage 239, ShuffleMapStage 246, ShuffleMapStage 240, ShuffleMapStage 244) -2016-06-13 19:12:50 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:12:50 INFO DAGScheduler:59 - Submitting ResultStage 247 (MapPartitionsRDD[133] at mapPartitions at PowerIterationClustering.scala:232), which has no missing parents -2016-06-13 19:12:50 INFO MemoryStore:59 - ensureFreeSpace(8488) called with curMem=391196, maxMem=766075207 -2016-06-13 19:12:50 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 8.3 KB, free 730.2 MB) -2016-06-13 19:12:50 INFO MemoryStore:59 - ensureFreeSpace(3631) called with curMem=399684, maxMem=766075207 -2016-06-13 19:12:50 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.2 MB) -2016-06-13 19:12:50 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:65491 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:12:50 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:12:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 247 (MapPartitionsRDD[133] at mapPartitions at PowerIterationClustering.scala:232) -2016-06-13 19:12:50 INFO TaskSchedulerImpl:59 - Adding task set 247.0 with 2 tasks -2016-06-13 19:12:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 247.0 (TID 66, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:12:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 247.0 (TID 67, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:12:50 INFO Executor:59 - Running task 0.0 in stage 247.0 (TID 66) -2016-06-13 19:12:50 INFO Executor:59 - Running task 1.0 in stage 247.0 (TID 67) -2016-06-13 19:12:50 INFO CacheManager:59 - Partition rdd_131_0 not found, computing it -2016-06-13 19:12:50 INFO CacheManager:59 - Partition rdd_131_1 not found, computing it -2016-06-13 19:12:50 INFO BlockManager:59 - Found block rdd_99_0 locally -2016-06-13 19:12:50 INFO BlockManager:59 - Found block rdd_99_1 locally -2016-06-13 19:12:50 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=403315, maxMem=766075207 -2016-06-13 19:12:50 INFO MemoryStore:59 - Block rdd_131_0 stored as values in memory (estimated size 1168.0 B, free 730.2 MB) -2016-06-13 19:12:50 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=404483, maxMem=766075207 -2016-06-13 19:12:50 INFO MemoryStore:59 - Block rdd_131_1 stored as values in memory (estimated size 1168.0 B, free 730.2 MB) -2016-06-13 19:12:50 INFO BlockManagerInfo:59 - Added rdd_131_0 in memory on localhost:65491 (size: 1168.0 B, free: 730.5 MB) -2016-06-13 19:12:50 INFO BlockManagerInfo:59 - Added rdd_131_1 in memory on localhost:65491 (size: 1168.0 B, free: 730.5 MB) -2016-06-13 19:12:50 INFO Executor:59 - Finished task 1.0 in stage 247.0 (TID 67). 2624 bytes result sent to driver -2016-06-13 19:12:50 INFO Executor:59 - Finished task 0.0 in stage 247.0 (TID 66). 2624 bytes result sent to driver -2016-06-13 19:12:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 247.0 (TID 67) in 9 ms on localhost (1/2) -2016-06-13 19:12:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 247.0 (TID 66) in 11 ms on localhost (2/2) -2016-06-13 19:12:50 INFO DAGScheduler:59 - ResultStage 247 (foreach at PICFromWebSuite.scala:32) finished in 0.011 s -2016-06-13 19:12:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 247.0, whose tasks have all completed, from pool -2016-06-13 19:12:50 INFO DAGScheduler:59 - Job 22 finished: foreach at PICFromWebSuite.scala:32, took 0.041833 s -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 19:12:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 19:12:50 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 19:12:50 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 19:12:50 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 19:12:50 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 19:12:50 INFO BlockManager:59 - BlockManager stopped -2016-06-13 19:12:50 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 19:12:50 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 19:12:50 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 19:12:50 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 19:12:50 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 19:12:50 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 19:12:50 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-b7a57d14-165c-4541-b0c2-ebf9ecc08c87 -2016-06-13 19:15:33 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 19:15:34 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 19:15:34 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 19:15:34 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 19:15:34 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 19:15:35 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 19:15:35 INFO Remoting:74 - Starting remoting -2016-06-13 19:15:35 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:65515] -2016-06-13 19:15:35 INFO Utils:59 - Successfully started service 'sparkDriver' on port 65515. -2016-06-13 19:15:35 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 19:15:35 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 19:15:35 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-e220cc12-656f-4834-9e40-76272c560d2f -2016-06-13 19:15:35 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 19:15:35 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-41a10bd2-5df1-40b2-b7d2-1ad475c899ef\httpd-ace9412d-1b4a-4a87-9012-cb6eb624c9ac -2016-06-13 19:15:35 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 19:15:35 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:15:35 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:65516 -2016-06-13 19:15:35 INFO Utils:59 - Successfully started service 'HTTP file server' on port 65516. -2016-06-13 19:15:35 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 19:15:35 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:15:35 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 19:15:35 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 19:15:35 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 19:15:36 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 19:15:36 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 19:15:36 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 49158. -2016-06-13 19:15:36 INFO NettyBlockTransferService:59 - Server created on 49158 -2016-06-13 19:15:36 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 19:15:36 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:49158 with 730.6 MB RAM, BlockManagerId(driver, localhost, 49158) -2016-06-13 19:15:36 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 19:15:37 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:325 -2016-06-13 19:15:37 INFO DAGScheduler:59 - Registering RDD 11 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:37 INFO DAGScheduler:59 - Registering RDD 5 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:15:37 INFO DAGScheduler:59 - Got job 0 (sum at PowerIterationClustering.scala:325) with 2 output partitions -2016-06-13 19:15:37 INFO DAGScheduler:59 - Final stage: ResultStage 2(sum at PowerIterationClustering.scala:325) -2016-06-13 19:15:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0, ShuffleMapStage 1) -2016-06-13 19:15:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0, ShuffleMapStage 1) -2016-06-13 19:15:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[11] at mapPartitions at GraphImpl.scala:235), which has no missing parents -2016-06-13 19:15:37 INFO MemoryStore:59 - ensureFreeSpace(3432) called with curMem=0, maxMem=766075207 -2016-06-13 19:15:37 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 3.4 KB, free 730.6 MB) -2016-06-13 19:15:37 INFO MemoryStore:59 - ensureFreeSpace(1979) called with curMem=3432, maxMem=766075207 -2016-06-13 19:15:37 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1979.0 B, free 730.6 MB) -2016-06-13 19:15:37 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:49158 (size: 1979.0 B, free: 730.6 MB) -2016-06-13 19:15:37 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[11] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:37 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 19:15:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[5] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:15:37 INFO MemoryStore:59 - ensureFreeSpace(3016) called with curMem=5411, maxMem=766075207 -2016-06-13 19:15:37 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-06-13 19:15:37 INFO MemoryStore:59 - ensureFreeSpace(1776) called with curMem=8427, maxMem=766075207 -2016-06-13 19:15:37 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1776.0 B, free 730.6 MB) -2016-06-13 19:15:37 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:49158 (size: 1776.0 B, free: 730.6 MB) -2016-06-13 19:15:37 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[5] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:15:37 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 19:15:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2488 bytes) -2016-06-13 19:15:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2563 bytes) -2016-06-13 19:15:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2488 bytes) -2016-06-13 19:15:37 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 19:15:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2563 bytes) -2016-06-13 19:15:37 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 19:15:37 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 19:15:37 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 19:15:37 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-06-13 19:15:37 INFO CacheManager:59 - Another thread is loading rdd_2_0, waiting for it to finish... -2016-06-13 19:15:37 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-06-13 19:15:37 INFO CacheManager:59 - Another thread is loading rdd_2_1, waiting for it to finish... -2016-06-13 19:15:37 INFO MemoryStore:59 - ensureFreeSpace(2952) called with curMem=10203, maxMem=766075207 -2016-06-13 19:15:37 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-06-13 19:15:37 INFO MemoryStore:59 - ensureFreeSpace(2856) called with curMem=13155, maxMem=766075207 -2016-06-13 19:15:37 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 2.8 KB, free 730.6 MB) -2016-06-13 19:15:37 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:49158 (size: 2.9 KB, free: 730.6 MB) -2016-06-13 19:15:37 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:49158 (size: 2.8 KB, free: 730.6 MB) -2016-06-13 19:15:37 INFO CacheManager:59 - Finished waiting for rdd_2_1 -2016-06-13 19:15:37 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-06-13 19:15:37 INFO CacheManager:59 - Finished waiting for rdd_2_0 -2016-06-13 19:15:37 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-06-13 19:15:37 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1159 bytes result sent to driver -2016-06-13 19:15:37 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1159 bytes result sent to driver -2016-06-13 19:15:37 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1739 bytes result sent to driver -2016-06-13 19:15:37 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1739 bytes result sent to driver -2016-06-13 19:15:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 160 ms on localhost (1/2) -2016-06-13 19:15:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 169 ms on localhost (1/2) -2016-06-13 19:15:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 161 ms on localhost (2/2) -2016-06-13 19:15:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 19:15:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 198 ms on localhost (2/2) -2016-06-13 19:15:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 19:15:37 INFO DAGScheduler:59 - ShuffleMapStage 1 (mapPartitions at VertexRDD.scala:358) finished in 0.170 s -2016-06-13 19:15:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:37 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 0) -2016-06-13 19:15:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-06-13 19:15:37 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:37 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List(ShuffleMapStage 0) -2016-06-13 19:15:37 INFO DAGScheduler:59 - ShuffleMapStage 0 (mapPartitions at GraphImpl.scala:235) finished in 0.227 s -2016-06-13 19:15:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:37 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-06-13 19:15:37 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:37 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List() -2016-06-13 19:15:37 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[23] at map at PowerIterationClustering.scala:325), which is now runnable -2016-06-13 19:15:37 INFO MemoryStore:59 - ensureFreeSpace(5136) called with curMem=16011, maxMem=766075207 -2016-06-13 19:15:37 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 5.0 KB, free 730.6 MB) -2016-06-13 19:15:37 INFO MemoryStore:59 - ensureFreeSpace(2654) called with curMem=21147, maxMem=766075207 -2016-06-13 19:15:37 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 19:15:37 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:49158 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:15:37 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[23] at map at PowerIterationClustering.scala:325) -2016-06-13 19:15:37 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 19:15:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:15:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:15:37 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 19:15:37 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 19:15:37 INFO CacheManager:59 - Partition rdd_21_1 not found, computing it -2016-06-13 19:15:37 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-06-13 19:15:37 INFO CacheManager:59 - Partition rdd_21_0 not found, computing it -2016-06-13 19:15:37 INFO CacheManager:59 - Partition rdd_8_1 not found, computing it -2016-06-13 19:15:37 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-06-13 19:15:37 INFO CacheManager:59 - Partition rdd_8_0 not found, computing it -2016-06-13 19:15:37 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:37 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-06-13 19:15:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-06-13 19:15:37 INFO MemoryStore:59 - ensureFreeSpace(1792) called with curMem=23801, maxMem=766075207 -2016-06-13 19:15:37 INFO MemoryStore:59 - Block rdd_8_1 stored as values in memory (estimated size 1792.0 B, free 730.6 MB) -2016-06-13 19:15:37 INFO MemoryStore:59 - ensureFreeSpace(1776) called with curMem=25593, maxMem=766075207 -2016-06-13 19:15:37 INFO BlockManagerInfo:59 - Added rdd_8_1 in memory on localhost:49158 (size: 1792.0 B, free: 730.6 MB) -2016-06-13 19:15:37 INFO MemoryStore:59 - Block rdd_8_0 stored as values in memory (estimated size 1776.0 B, free 730.6 MB) -2016-06-13 19:15:37 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:37 INFO BlockManagerInfo:59 - Added rdd_8_0 in memory on localhost:49158 (size: 1776.0 B, free: 730.6 MB) -2016-06-13 19:15:37 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:37 INFO MemoryStore:59 - ensureFreeSpace(1824) called with curMem=27369, maxMem=766075207 -2016-06-13 19:15:37 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 1824.0 B, free 730.6 MB) -2016-06-13 19:15:37 INFO MemoryStore:59 - ensureFreeSpace(1840) called with curMem=29193, maxMem=766075207 -2016-06-13 19:15:37 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:49158 (size: 1824.0 B, free: 730.6 MB) -2016-06-13 19:15:37 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 1840.0 B, free 730.6 MB) -2016-06-13 19:15:37 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:49158 (size: 1840.0 B, free: 730.6 MB) -2016-06-13 19:15:37 INFO MemoryStore:59 - ensureFreeSpace(368) called with curMem=31033, maxMem=766075207 -2016-06-13 19:15:37 INFO MemoryStore:59 - Block rdd_21_1 stored as values in memory (estimated size 368.0 B, free 730.6 MB) -2016-06-13 19:15:37 INFO MemoryStore:59 - ensureFreeSpace(368) called with curMem=31401, maxMem=766075207 -2016-06-13 19:15:37 INFO MemoryStore:59 - Block rdd_21_0 stored as values in memory (estimated size 368.0 B, free 730.6 MB) -2016-06-13 19:15:37 INFO BlockManagerInfo:59 - Added rdd_21_1 in memory on localhost:49158 (size: 368.0 B, free: 730.6 MB) -2016-06-13 19:15:37 INFO BlockManagerInfo:59 - Added rdd_21_0 in memory on localhost:49158 (size: 368.0 B, free: 730.6 MB) -2016-06-13 19:15:37 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1895 bytes result sent to driver -2016-06-13 19:15:37 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1895 bytes result sent to driver -2016-06-13 19:15:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 103 ms on localhost (1/2) -2016-06-13 19:15:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 98 ms on localhost (2/2) -2016-06-13 19:15:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 19:15:37 INFO DAGScheduler:59 - ResultStage 2 (sum at PowerIterationClustering.scala:325) finished in 0.105 s -2016-06-13 19:15:37 INFO DAGScheduler:59 - Job 0 finished: sum at PowerIterationClustering.scala:325, took 0.639166 s -2016-06-13 19:15:37 INFO SparkContext:59 - Starting job: reduce at VertexRDDImpl.scala:90 -2016-06-13 19:15:37 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:15:37 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:15:37 INFO DAGScheduler:59 - Got job 1 (reduce at VertexRDDImpl.scala:90) with 2 output partitions -2016-06-13 19:15:37 INFO DAGScheduler:59 - Final stage: ResultStage 5(reduce at VertexRDDImpl.scala:90) -2016-06-13 19:15:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3, ShuffleMapStage 4) -2016-06-13 19:15:37 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:37 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[27] at map at VertexRDDImpl.scala:90), which has no missing parents -2016-06-13 19:15:37 INFO MemoryStore:59 - ensureFreeSpace(5288) called with curMem=31769, maxMem=766075207 -2016-06-13 19:15:37 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.2 KB, free 730.6 MB) -2016-06-13 19:15:37 INFO MemoryStore:59 - ensureFreeSpace(2670) called with curMem=37057, maxMem=766075207 -2016-06-13 19:15:37 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 19:15:37 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:49158 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:15:37 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[27] at map at VertexRDDImpl.scala:90) -2016-06-13 19:15:37 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 19:15:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 6, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:15:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 7, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:15:37 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 7) -2016-06-13 19:15:37 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 6) -2016-06-13 19:15:37 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:15:37 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:15:38 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 6). 2161 bytes result sent to driver -2016-06-13 19:15:38 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 7). 2161 bytes result sent to driver -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 6) in 34 ms on localhost (1/2) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 7) in 32 ms on localhost (2/2) -2016-06-13 19:15:38 INFO DAGScheduler:59 - ResultStage 5 (reduce at VertexRDDImpl.scala:90) finished in 0.035 s -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 19:15:38 INFO DAGScheduler:59 - Job 1 finished: reduce at VertexRDDImpl.scala:90, took 0.075315 s -2016-06-13 19:15:38 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:15:38 INFO DAGScheduler:59 - Registering RDD 15 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Registering RDD 28 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Registering RDD 32 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Got job 2 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:15:38 INFO DAGScheduler:59 - Final stage: ResultStage 13(sum at PowerIterationClustering.scala:369) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12, ShuffleMapStage 9, ShuffleMapStage 10) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 8 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[15] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=39727, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.1 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(2367) called with curMem=43975, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:49158 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 19:15:38 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 8 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[15] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 11 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[28] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 8, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 9, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:15:38 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 8) -2016-06-13 19:15:38 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 9) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(4992) called with curMem=46342, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 4.9 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(2604) called with curMem=51334, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:49158 (size: 2.5 KB, free: 730.6 MB) -2016-06-13 19:15:38 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 11 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[28] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 10, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 11, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:15:38 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 10) -2016-06-13 19:15:38 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 8). 2254 bytes result sent to driver -2016-06-13 19:15:38 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 9). 2254 bytes result sent to driver -2016-06-13 19:15:38 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 11) -2016-06-13 19:15:38 INFO CacheManager:59 - Partition rdd_25_0 not found, computing it -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 9) in 78 ms on localhost (1/2) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=51268, maxMem=766075207 -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:49158 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - Block rdd_25_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added rdd_25_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.6 MB) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 8) in 85 ms on localhost (2/2) -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 19:15:38 INFO DAGScheduler:59 - ShuffleMapStage 8 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.089 s -2016-06-13 19:15:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:38 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 11) -2016-06-13 19:15:38 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 12, ResultStage 13) -2016-06-13 19:15:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:38 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 12: List(ShuffleMapStage 11) -2016-06-13 19:15:38 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 10). 2834 bytes result sent to driver -2016-06-13 19:15:38 INFO CacheManager:59 - Partition rdd_25_1 not found, computing it -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 10) in 55 ms on localhost (1/2) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=47412, maxMem=766075207 -2016-06-13 19:15:38 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List(ShuffleMapStage 12) -2016-06-13 19:15:38 INFO MemoryStore:59 - Block rdd_25_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added rdd_25_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.6 MB) -2016-06-13 19:15:38 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 19:15:38 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 11). 2834 bytes result sent to driver -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:49158 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:15:38 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 11) in 68 ms on localhost (2/2) -2016-06-13 19:15:38 INFO DAGScheduler:59 - ShuffleMapStage 11 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.070 s -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 19:15:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:38 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 12, ResultStage 13) -2016-06-13 19:15:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:49158 in memory (size: 1776.0 B, free: 730.6 MB) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 12: List() -2016-06-13 19:15:38 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 19:15:38 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List(ShuffleMapStage 12) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[32] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:49158 in memory (size: 1979.0 B, free: 730.6 MB) -2016-06-13 19:15:38 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=30851, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 4.7 KB, free 730.6 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(2539) called with curMem=35699, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:49158 (size: 2.5 KB, free: 730.6 MB) -2016-06-13 19:15:38 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[32] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 12, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 13, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:15:38 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 13) -2016-06-13 19:15:38 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 12) -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:38 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 13). 2470 bytes result sent to driver -2016-06-13 19:15:38 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 12). 2470 bytes result sent to driver -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 13) in 49 ms on localhost (1/2) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 12) in 54 ms on localhost (2/2) -2016-06-13 19:15:38 INFO DAGScheduler:59 - ShuffleMapStage 12 (mapPartitions at GraphImpl.scala:235) finished in 0.054 s -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 19:15:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-06-13 19:15:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:38 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[37] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(6024) called with curMem=38238, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.9 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(2867) called with curMem=44262, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:49158 (size: 2.8 KB, free: 730.6 MB) -2016-06-13 19:15:38 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[37] at map at PowerIterationClustering.scala:369) -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 14, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 15, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:15:38 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 14) -2016-06-13 19:15:38 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 15) -2016-06-13 19:15:38 INFO CacheManager:59 - Partition rdd_34_1 not found, computing it -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 19:15:38 INFO CacheManager:59 - Partition rdd_34_0 not found, computing it -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=47129, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block rdd_34_1 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=48609, maxMem=766075207 -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added rdd_34_1 in memory on localhost:49158 (size: 1480.0 B, free: 730.6 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - Block rdd_34_0 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added rdd_34_0 in memory on localhost:49158 (size: 1480.0 B, free: 730.6 MB) -2016-06-13 19:15:38 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 15). 2880 bytes result sent to driver -2016-06-13 19:15:38 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 14). 2880 bytes result sent to driver -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 15) in 25 ms on localhost (1/2) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 14) in 28 ms on localhost (2/2) -2016-06-13 19:15:38 INFO DAGScheduler:59 - ResultStage 13 (sum at PowerIterationClustering.scala:369) finished in 0.029 s -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 19:15:38 INFO DAGScheduler:59 - Job 2 finished: sum at PowerIterationClustering.scala:369, took 0.265372 s -2016-06-13 19:15:38 INFO PowerIterationClustering:59 - Iteration 0: norm(v) = 36.22334869431644. -2016-06-13 19:15:38 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:15:38 INFO DAGScheduler:59 - Got job 3 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:15:38 INFO DAGScheduler:59 - Final stage: ResultStage 21(sum at PowerIterationClustering.scala:375) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 20, ShuffleMapStage 17, ShuffleMapStage 18) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[48] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(6760) called with curMem=50089, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 6.6 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(3106) called with curMem=56849, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:49158 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 19:15:38 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[48] at values at PowerIterationClustering.scala:375) -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 16, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 17, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:15:38 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 16) -2016-06-13 19:15:38 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 17) -2016-06-13 19:15:38 INFO CacheManager:59 - Partition rdd_40_0 not found, computing it -2016-06-13 19:15:38 INFO CacheManager:59 - Partition rdd_40_1 not found, computing it -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_34_0 locally -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_34_1 locally -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=59955, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block rdd_40_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=61387, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block rdd_40_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added rdd_40_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.6 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added rdd_40_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.6 MB) -2016-06-13 19:15:38 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 17). 2664 bytes result sent to driver -2016-06-13 19:15:38 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 16). 2664 bytes result sent to driver -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 17) in 17 ms on localhost (1/2) -2016-06-13 19:15:38 INFO DAGScheduler:59 - ResultStage 21 (sum at PowerIterationClustering.scala:375) finished in 0.020 s -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 16) in 19 ms on localhost (2/2) -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 19:15:38 INFO DAGScheduler:59 - Job 3 finished: sum at PowerIterationClustering.scala:375, took 0.045565 s -2016-06-13 19:15:38 INFO PowerIterationClustering:59 - Iteration 0: delta = 1.2794887333174252. -2016-06-13 19:15:38 INFO PowerIterationClustering:59 - Iteration 0: diff(delta) = 1.7976931348623157E308. -2016-06-13 19:15:38 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:15:38 INFO DAGScheduler:59 - Registering RDD 51 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Registering RDD 55 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Got job 4 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:15:38 INFO DAGScheduler:59 - Final stage: ResultStage 31(sum at PowerIterationClustering.scala:369) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30, ShuffleMapStage 28, ShuffleMapStage 25, ShuffleMapStage 26) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 30) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 29 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[51] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(5984) called with curMem=62819, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.8 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(2861) called with curMem=68803, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:49158 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:15:38 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 29 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[51] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 18, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 19, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:15:38 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 19) -2016-06-13 19:15:38 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 18) -2016-06-13 19:15:38 INFO CacheManager:59 - Partition rdd_49_0 not found, computing it -2016-06-13 19:15:38 INFO CacheManager:59 - Partition rdd_49_1 not found, computing it -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_34_0 locally -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_34_1 locally -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=71664, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block rdd_49_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=73096, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block rdd_49_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added rdd_49_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added rdd_49_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:38 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 18). 2834 bytes result sent to driver -2016-06-13 19:15:38 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 19). 2834 bytes result sent to driver -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 18) in 18 ms on localhost (1/2) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 19) in 20 ms on localhost (2/2) -2016-06-13 19:15:38 INFO DAGScheduler:59 - ShuffleMapStage 29 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.021 s -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 19:15:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:38 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 30, ResultStage 31) -2016-06-13 19:15:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:38 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List() -2016-06-13 19:15:38 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 30 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[55] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=74528, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.7 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=79376, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:49158 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:15:38 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 30 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[55] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 20, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 21, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:15:38 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 21) -2016-06-13 19:15:38 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 20) -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:38 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 21). 2470 bytes result sent to driver -2016-06-13 19:15:38 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 20). 2470 bytes result sent to driver -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 21) in 30 ms on localhost (1/2) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 20) in 32 ms on localhost (2/2) -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 19:15:38 INFO DAGScheduler:59 - ShuffleMapStage 30 (mapPartitions at GraphImpl.scala:235) finished in 0.032 s -2016-06-13 19:15:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 31) -2016-06-13 19:15:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:38 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List() -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[60] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(6920) called with curMem=81918, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 6.8 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(3114) called with curMem=88838, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:49158 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:15:38 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[60] at map at PowerIterationClustering.scala:369) -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 22, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 23, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:15:38 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 23) -2016-06-13 19:15:38 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 22) -2016-06-13 19:15:38 INFO CacheManager:59 - Partition rdd_57_1 not found, computing it -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_49_1 locally -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:38 INFO CacheManager:59 - Partition rdd_57_0 not found, computing it -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_49_0 locally -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=91952, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block rdd_57_1 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added rdd_57_1 in memory on localhost:49158 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=93432, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block rdd_57_0 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added rdd_57_0 in memory on localhost:49158 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:15:38 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 23). 2880 bytes result sent to driver -2016-06-13 19:15:38 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 22). 2880 bytes result sent to driver -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 23) in 22 ms on localhost (1/2) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 22) in 24 ms on localhost (2/2) -2016-06-13 19:15:38 INFO DAGScheduler:59 - ResultStage 31 (sum at PowerIterationClustering.scala:369) finished in 0.024 s -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 19:15:38 INFO DAGScheduler:59 - Job 4 finished: sum at PowerIterationClustering.scala:369, took 0.134573 s -2016-06-13 19:15:38 INFO PowerIterationClustering:59 - Iteration 1: norm(v) = 36.22334869431644. -2016-06-13 19:15:38 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 151 bytes -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:15:38 INFO DAGScheduler:59 - Got job 5 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:15:38 INFO DAGScheduler:59 - Final stage: ResultStage 41(sum at PowerIterationClustering.scala:375) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 38, ShuffleMapStage 35, ShuffleMapStage 36, ShuffleMapStage 40) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[71] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(7352) called with curMem=94912, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 7.2 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(3294) called with curMem=102264, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:49158 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:15:38 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[71] at values at PowerIterationClustering.scala:375) -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 24, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 25, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:15:38 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 24) -2016-06-13 19:15:38 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 25) -2016-06-13 19:15:38 INFO CacheManager:59 - Partition rdd_63_1 not found, computing it -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_49_1 locally -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_57_1 locally -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=105558, maxMem=766075207 -2016-06-13 19:15:38 INFO CacheManager:59 - Partition rdd_63_0 not found, computing it -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_49_0 locally -2016-06-13 19:15:38 INFO MemoryStore:59 - Block rdd_63_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_57_0 locally -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added rdd_63_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=106990, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block rdd_63_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:15:38 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 25). 2664 bytes result sent to driver -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added rdd_63_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:38 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 24). 2664 bytes result sent to driver -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 25) in 18 ms on localhost (1/2) -2016-06-13 19:15:38 INFO DAGScheduler:59 - ResultStage 41 (sum at PowerIterationClustering.scala:375) finished in 0.020 s -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 24) in 20 ms on localhost (2/2) -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-06-13 19:15:38 INFO DAGScheduler:59 - Job 5 finished: sum at PowerIterationClustering.scala:375, took 0.051010 s -2016-06-13 19:15:38 INFO PowerIterationClustering:59 - Iteration 1: delta = 0.0. -2016-06-13 19:15:38 INFO PowerIterationClustering:59 - Iteration 1: diff(delta) = 1.2794887333174252. -2016-06-13 19:15:38 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:15:38 INFO DAGScheduler:59 - Registering RDD 74 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Registering RDD 78 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Got job 6 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:15:38 INFO DAGScheduler:59 - Final stage: ResultStage 53(sum at PowerIterationClustering.scala:369) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 48, ShuffleMapStage 45, ShuffleMapStage 52, ShuffleMapStage 46, ShuffleMapStage 50) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 52) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 51 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[74] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(6568) called with curMem=108422, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 6.4 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(3022) called with curMem=114990, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:49158 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:15:38 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 51 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[74] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 26, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 27, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:15:38 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 26) -2016-06-13 19:15:38 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 27) -2016-06-13 19:15:38 INFO CacheManager:59 - Partition rdd_72_0 not found, computing it -2016-06-13 19:15:38 INFO CacheManager:59 - Partition rdd_72_1 not found, computing it -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_57_0 locally -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_57_1 locally -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=118012, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block rdd_72_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=119444, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block rdd_72_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added rdd_72_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added rdd_72_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:38 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 26). 2834 bytes result sent to driver -2016-06-13 19:15:38 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 27). 2834 bytes result sent to driver -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 26) in 19 ms on localhost (1/2) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 27) in 19 ms on localhost (2/2) -2016-06-13 19:15:38 INFO DAGScheduler:59 - ShuffleMapStage 51 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.020 s -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-06-13 19:15:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:38 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 52, ResultStage 53) -2016-06-13 19:15:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:38 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 52: List() -2016-06-13 19:15:38 INFO DAGScheduler:59 - Missing parents for ResultStage 53: List(ShuffleMapStage 52) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 52 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[78] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=120876, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 4.7 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=125724, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:49158 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:15:38 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 52 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[78] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 28, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 29, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:15:38 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 28) -2016-06-13 19:15:38 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 29) -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:38 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 29). 2470 bytes result sent to driver -2016-06-13 19:15:38 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 28). 2470 bytes result sent to driver -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 29) in 38 ms on localhost (1/2) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 28) in 41 ms on localhost (2/2) -2016-06-13 19:15:38 INFO DAGScheduler:59 - ShuffleMapStage 52 (mapPartitions at GraphImpl.scala:235) finished in 0.042 s -2016-06-13 19:15:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 53) -2016-06-13 19:15:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-06-13 19:15:38 INFO DAGScheduler:59 - Missing parents for ResultStage 53: List() -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[83] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(7504) called with curMem=128266, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 7.3 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(3266) called with curMem=135770, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:49158 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:15:38 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[83] at map at PowerIterationClustering.scala:369) -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 30, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 31, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:15:38 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 31) -2016-06-13 19:15:38 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 30) -2016-06-13 19:15:38 INFO CacheManager:59 - Partition rdd_80_0 not found, computing it -2016-06-13 19:15:38 INFO CacheManager:59 - Partition rdd_80_1 not found, computing it -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_72_0 locally -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_72_1 locally -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=139036, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block rdd_80_1 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=140516, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block rdd_80_0 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added rdd_80_1 in memory on localhost:49158 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added rdd_80_0 in memory on localhost:49158 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:15:38 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 31). 2880 bytes result sent to driver -2016-06-13 19:15:38 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 30). 2880 bytes result sent to driver -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 31) in 19 ms on localhost (1/2) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 30) in 21 ms on localhost (2/2) -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-06-13 19:15:38 INFO DAGScheduler:59 - ResultStage 53 (sum at PowerIterationClustering.scala:369) finished in 0.022 s -2016-06-13 19:15:38 INFO DAGScheduler:59 - Job 6 finished: sum at PowerIterationClustering.scala:369, took 0.152200 s -2016-06-13 19:15:38 INFO PowerIterationClustering:59 - Iteration 2: norm(v) = 36.22334869431644. -2016-06-13 19:15:38 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 151 bytes -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 151 bytes -2016-06-13 19:15:38 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:15:38 INFO DAGScheduler:59 - Got job 7 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:15:38 INFO DAGScheduler:59 - Final stage: ResultStage 65(sum at PowerIterationClustering.scala:375) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 60, ShuffleMapStage 64, ShuffleMapStage 57, ShuffleMapStage 58, ShuffleMapStage 62) -2016-06-13 19:15:38 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[94] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(7936) called with curMem=141996, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 7.8 KB, free 730.4 MB) -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(3392) called with curMem=149932, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:49158 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:15:38 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 65 (MapPartitionsRDD[94] at values at PowerIterationClustering.scala:375) -2016-06-13 19:15:38 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 2 tasks -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 32, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:15:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 33, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:15:38 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 33) -2016-06-13 19:15:38 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 32) -2016-06-13 19:15:38 INFO CacheManager:59 - Partition rdd_86_1 not found, computing it -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_72_1 locally -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_80_1 locally -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=153324, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block rdd_86_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:15:38 INFO BlockManagerInfo:59 - Added rdd_86_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:38 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 33). 2664 bytes result sent to driver -2016-06-13 19:15:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 33) in 23 ms on localhost (1/2) -2016-06-13 19:15:38 INFO CacheManager:59 - Partition rdd_86_0 not found, computing it -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_72_0 locally -2016-06-13 19:15:38 INFO BlockManager:59 - Found block rdd_80_0 locally -2016-06-13 19:15:38 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=154756, maxMem=766075207 -2016-06-13 19:15:38 INFO MemoryStore:59 - Block rdd_86_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added rdd_86_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:39 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 32). 2664 bytes result sent to driver -2016-06-13 19:15:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 32) in 77 ms on localhost (2/2) -2016-06-13 19:15:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-06-13 19:15:39 INFO DAGScheduler:59 - ResultStage 65 (sum at PowerIterationClustering.scala:375) finished in 0.079 s -2016-06-13 19:15:39 INFO DAGScheduler:59 - Job 7 finished: sum at PowerIterationClustering.scala:375, took 0.120792 s -2016-06-13 19:15:39 INFO PowerIterationClustering:59 - Iteration 2: delta = 0.0. -2016-06-13 19:15:39 INFO PowerIterationClustering:59 - Iteration 2: diff(delta) = 0.0. -2016-06-13 19:15:39 WARN KMeans:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:15:39 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:15:39 INFO DAGScheduler:59 - Got job 8 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:15:39 INFO DAGScheduler:59 - Final stage: ResultStage 77(takeSample at KMeans.scala:376) -2016-06-13 19:15:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 70, ShuffleMapStage 74, ShuffleMapStage 72, ShuffleMapStage 69, ShuffleMapStage 76) -2016-06-13 19:15:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:39 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[102] at map at KMeans.scala:210), which has no missing parents -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(8168) called with curMem=156188, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 8.0 KB, free 730.4 MB) -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(3463) called with curMem=164356, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.4 MB) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:49158 (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 77 (MapPartitionsRDD[102] at map at KMeans.scala:210) -2016-06-13 19:15:39 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 2 tasks -2016-06-13 19:15:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 34, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:49158 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 35, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:15:39 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 34) -2016-06-13 19:15:39 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 35) -2016-06-13 19:15:39 INFO CacheManager:59 - Partition rdd_97_1 not found, computing it -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_80_1 locally -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=156491, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block rdd_97_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:15:39 INFO CacheManager:59 - Partition rdd_97_0 not found, computing it -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_80_0 locally -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=157923, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block rdd_97_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added rdd_97_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:39 INFO BlockManager:59 - Removing RDD 86 -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added rdd_97_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:39 INFO CacheManager:59 - Partition rdd_100_1 not found, computing it -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=156491, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block rdd_100_1 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added rdd_100_1 in memory on localhost:49158 (size: 240.0 B, free: 730.5 MB) -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned RDD 86 -2016-06-13 19:15:39 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 35). 2717 bytes result sent to driver -2016-06-13 19:15:39 INFO CacheManager:59 - Partition rdd_100_0 not found, computing it -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:49158 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 35) in 67 ms on localhost (1/2) -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=145961, maxMem=766075207 -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block rdd_100_0 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added rdd_100_0 in memory on localhost:49158 (size: 240.0 B, free: 730.5 MB) -2016-06-13 19:15:39 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 34). 2717 bytes result sent to driver -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:49158 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 19:15:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 34) in 76 ms on localhost (2/2) -2016-06-13 19:15:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-06-13 19:15:39 INFO DAGScheduler:59 - ResultStage 77 (takeSample at KMeans.scala:376) finished in 0.077 s -2016-06-13 19:15:39 INFO DAGScheduler:59 - Job 8 finished: takeSample at KMeans.scala:376, took 0.160836 s -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:49158 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:49158 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 19:15:39 INFO BlockManager:59 - Removing RDD 63 -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned RDD 63 -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:49158 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:49158 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:49158 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:49158 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 19:15:39 INFO BlockManager:59 - Removing RDD 40 -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned RDD 40 -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:49158 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:49158 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:49158 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:49158 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 19:15:39 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:15:39 INFO DAGScheduler:59 - Got job 9 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:15:39 INFO DAGScheduler:59 - Final stage: ResultStage 89(takeSample at KMeans.scala:376) -2016-06-13 19:15:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 84, ShuffleMapStage 81, ShuffleMapStage 88, ShuffleMapStage 82, ShuffleMapStage 86) -2016-06-13 19:15:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:39 INFO DAGScheduler:59 - Submitting ResultStage 89 (PartitionwiseSampledRDD[104] at takeSample at KMeans.scala:376), which has no missing parents -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(25560) called with curMem=46223, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 25.0 KB, free 730.5 MB) -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(10154) called with curMem=71783, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 9.9 KB, free 730.5 MB) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:49158 (size: 9.9 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 89 (PartitionwiseSampledRDD[104] at takeSample at KMeans.scala:376) -2016-06-13 19:15:39 INFO TaskSchedulerImpl:59 - Adding task set 89.0 with 2 tasks -2016-06-13 19:15:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 89.0 (TID 36, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:15:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 89.0 (TID 37, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:15:39 INFO Executor:59 - Running task 0.0 in stage 89.0 (TID 36) -2016-06-13 19:15:39 INFO Executor:59 - Running task 1.0 in stage 89.0 (TID 37) -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:15:39 INFO Executor:59 - Finished task 1.0 in stage 89.0 (TID 37). 2575 bytes result sent to driver -2016-06-13 19:15:39 INFO Executor:59 - Finished task 0.0 in stage 89.0 (TID 36). 2633 bytes result sent to driver -2016-06-13 19:15:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 89.0 (TID 37) in 14 ms on localhost (1/2) -2016-06-13 19:15:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 89.0 (TID 36) in 18 ms on localhost (2/2) -2016-06-13 19:15:39 INFO DAGScheduler:59 - ResultStage 89 (takeSample at KMeans.scala:376) finished in 0.019 s -2016-06-13 19:15:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 89.0, whose tasks have all completed, from pool -2016-06-13 19:15:39 INFO DAGScheduler:59 - Job 9 finished: takeSample at KMeans.scala:376, took 0.057960 s -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=81937, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 880.0 B, free 730.5 MB) -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(487) called with curMem=82817, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 487.0 B, free 730.5 MB) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:49158 (size: 487.0 B, free: 730.5 MB) -2016-06-13 19:15:39 INFO SparkContext:59 - Created broadcast 19 from broadcast at KMeans.scala:394 -2016-06-13 19:15:39 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:15:39 INFO DAGScheduler:59 - Got job 10 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:15:39 INFO DAGScheduler:59 - Final stage: ResultStage 101(aggregate at KMeans.scala:402) -2016-06-13 19:15:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 96, ShuffleMapStage 100, ShuffleMapStage 93, ShuffleMapStage 94, ShuffleMapStage 98) -2016-06-13 19:15:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:39 INFO DAGScheduler:59 - Submitting ResultStage 101 (MapPartitionsRDD[106] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(10192) called with curMem=83304, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 10.0 KB, free 730.5 MB) -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(4263) called with curMem=93496, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.5 MB) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:49158 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 101 (MapPartitionsRDD[106] at map at KMeans.scala:396) -2016-06-13 19:15:39 INFO TaskSchedulerImpl:59 - Adding task set 101.0 with 2 tasks -2016-06-13 19:15:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 101.0 (TID 38, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:15:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 101.0 (TID 39, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:15:39 INFO Executor:59 - Running task 0.0 in stage 101.0 (TID 38) -2016-06-13 19:15:39 INFO Executor:59 - Running task 1.0 in stage 101.0 (TID 39) -2016-06-13 19:15:39 INFO CacheManager:59 - Partition rdd_106_0 not found, computing it -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:15:39 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 19:15:39 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=97759, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block rdd_106_0 stored as values in memory (estimated size 496.0 B, free 730.5 MB) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added rdd_106_0 in memory on localhost:49158 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:15:39 INFO Executor:59 - Finished task 0.0 in stage 101.0 (TID 38). 2647 bytes result sent to driver -2016-06-13 19:15:39 INFO CacheManager:59 - Partition rdd_106_1 not found, computing it -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=98255, maxMem=766075207 -2016-06-13 19:15:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 101.0 (TID 38) in 39 ms on localhost (1/2) -2016-06-13 19:15:39 INFO MemoryStore:59 - Block rdd_106_1 stored as values in memory (estimated size 496.0 B, free 730.5 MB) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added rdd_106_1 in memory on localhost:49158 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:15:39 INFO Executor:59 - Finished task 1.0 in stage 101.0 (TID 39). 2647 bytes result sent to driver -2016-06-13 19:15:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 101.0 (TID 39) in 44 ms on localhost (2/2) -2016-06-13 19:15:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 101.0, whose tasks have all completed, from pool -2016-06-13 19:15:39 INFO DAGScheduler:59 - ResultStage 101 (aggregate at KMeans.scala:402) finished in 0.045 s -2016-06-13 19:15:39 INFO DAGScheduler:59 - Job 10 finished: aggregate at KMeans.scala:402, took 0.081158 s -2016-06-13 19:15:39 INFO MapPartitionsRDD:59 - Removing RDD 103 from persistence list -2016-06-13 19:15:39 INFO BlockManager:59 - Removing RDD 103 -2016-06-13 19:15:39 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:15:39 INFO DAGScheduler:59 - Got job 11 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:15:39 INFO DAGScheduler:59 - Final stage: ResultStage 113(collect at KMeans.scala:431) -2016-06-13 19:15:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 108, ShuffleMapStage 105, ShuffleMapStage 112, ShuffleMapStage 106, ShuffleMapStage 110) -2016-06-13 19:15:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:39 INFO DAGScheduler:59 - Submitting ResultStage 113 (MapPartitionsRDD[108] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(10240) called with curMem=98751, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 10.0 KB, free 730.5 MB) -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(4343) called with curMem=108991, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.5 MB) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:49158 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 113 (MapPartitionsRDD[108] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:15:39 INFO TaskSchedulerImpl:59 - Adding task set 113.0 with 2 tasks -2016-06-13 19:15:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 113.0 (TID 40, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:15:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 113.0 (TID 41, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:15:39 INFO Executor:59 - Running task 0.0 in stage 113.0 (TID 40) -2016-06-13 19:15:39 INFO Executor:59 - Running task 1.0 in stage 113.0 (TID 41) -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_106_0 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_106_1 locally -2016-06-13 19:15:39 INFO Executor:59 - Finished task 0.0 in stage 113.0 (TID 40). 3118 bytes result sent to driver -2016-06-13 19:15:39 INFO Executor:59 - Finished task 1.0 in stage 113.0 (TID 41). 3471 bytes result sent to driver -2016-06-13 19:15:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 113.0 (TID 40) in 24 ms on localhost (1/2) -2016-06-13 19:15:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 113.0 (TID 41) in 24 ms on localhost (2/2) -2016-06-13 19:15:39 INFO DAGScheduler:59 - ResultStage 113 (collect at KMeans.scala:431) finished in 0.025 s -2016-06-13 19:15:39 INFO DAGScheduler:59 - Job 11 finished: collect at KMeans.scala:431, took 0.052234 s -2016-06-13 19:15:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 113.0, whose tasks have all completed, from pool -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(1648) called with curMem=113334, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 1648.0 B, free 730.5 MB) -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(600) called with curMem=114982, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 600.0 B, free 730.5 MB) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:49158 (size: 600.0 B, free: 730.5 MB) -2016-06-13 19:15:39 INFO SparkContext:59 - Created broadcast 22 from broadcast at KMeans.scala:394 -2016-06-13 19:15:39 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:15:39 INFO DAGScheduler:59 - Got job 12 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:15:39 INFO DAGScheduler:59 - Final stage: ResultStage 125(aggregate at KMeans.scala:402) -2016-06-13 19:15:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 117, ShuffleMapStage 118, ShuffleMapStage 122, ShuffleMapStage 120, ShuffleMapStage 124) -2016-06-13 19:15:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:39 INFO DAGScheduler:59 - Submitting ResultStage 125 (MapPartitionsRDD[110] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(10424) called with curMem=115582, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 10.2 KB, free 730.5 MB) -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(4357) called with curMem=126006, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.5 MB) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:49158 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 125 (MapPartitionsRDD[110] at map at KMeans.scala:396) -2016-06-13 19:15:39 INFO TaskSchedulerImpl:59 - Adding task set 125.0 with 2 tasks -2016-06-13 19:15:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 125.0 (TID 42, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:15:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 125.0 (TID 43, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:15:39 INFO Executor:59 - Running task 1.0 in stage 125.0 (TID 43) -2016-06-13 19:15:39 INFO Executor:59 - Running task 0.0 in stage 125.0 (TID 42) -2016-06-13 19:15:39 INFO CacheManager:59 - Partition rdd_110_0 not found, computing it -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_106_0 locally -2016-06-13 19:15:39 INFO CacheManager:59 - Partition rdd_110_1 not found, computing it -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_106_1 locally -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=130363, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block rdd_110_0 stored as values in memory (estimated size 496.0 B, free 730.5 MB) -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=130859, maxMem=766075207 -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added rdd_110_0 in memory on localhost:49158 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:15:39 INFO MemoryStore:59 - Block rdd_110_1 stored as values in memory (estimated size 496.0 B, free 730.5 MB) -2016-06-13 19:15:39 INFO Executor:59 - Finished task 0.0 in stage 125.0 (TID 42). 2647 bytes result sent to driver -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added rdd_110_1 in memory on localhost:49158 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:15:39 INFO Executor:59 - Finished task 1.0 in stage 125.0 (TID 43). 2647 bytes result sent to driver -2016-06-13 19:15:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 125.0 (TID 42) in 28 ms on localhost (1/2) -2016-06-13 19:15:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 125.0 (TID 43) in 27 ms on localhost (2/2) -2016-06-13 19:15:39 INFO DAGScheduler:59 - ResultStage 125 (aggregate at KMeans.scala:402) finished in 0.029 s -2016-06-13 19:15:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 125.0, whose tasks have all completed, from pool -2016-06-13 19:15:39 INFO DAGScheduler:59 - Job 12 finished: aggregate at KMeans.scala:402, took 0.063189 s -2016-06-13 19:15:39 INFO MapPartitionsRDD:59 - Removing RDD 106 from persistence list -2016-06-13 19:15:39 INFO BlockManager:59 - Removing RDD 106 -2016-06-13 19:15:39 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:15:39 INFO DAGScheduler:59 - Got job 13 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:15:39 INFO DAGScheduler:59 - Final stage: ResultStage 137(collect at KMeans.scala:431) -2016-06-13 19:15:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 132, ShuffleMapStage 129, ShuffleMapStage 136, ShuffleMapStage 130, ShuffleMapStage 134) -2016-06-13 19:15:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:39 INFO DAGScheduler:59 - Submitting ResultStage 137 (MapPartitionsRDD[112] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(10480) called with curMem=130363, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 10.2 KB, free 730.5 MB) -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(4430) called with curMem=140843, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:49158 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 137 (MapPartitionsRDD[112] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:15:39 INFO TaskSchedulerImpl:59 - Adding task set 137.0 with 2 tasks -2016-06-13 19:15:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 137.0 (TID 44, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:15:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 137.0 (TID 45, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:15:39 INFO Executor:59 - Running task 0.0 in stage 137.0 (TID 44) -2016-06-13 19:15:39 INFO Executor:59 - Running task 1.0 in stage 137.0 (TID 45) -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_110_1 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_110_0 locally -2016-06-13 19:15:39 INFO Executor:59 - Finished task 1.0 in stage 137.0 (TID 45). 3240 bytes result sent to driver -2016-06-13 19:15:39 INFO Executor:59 - Finished task 0.0 in stage 137.0 (TID 44). 2990 bytes result sent to driver -2016-06-13 19:15:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 137.0 (TID 45) in 11 ms on localhost (1/2) -2016-06-13 19:15:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 137.0 (TID 44) in 13 ms on localhost (2/2) -2016-06-13 19:15:39 INFO DAGScheduler:59 - ResultStage 137 (collect at KMeans.scala:431) finished in 0.014 s -2016-06-13 19:15:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 137.0, whose tasks have all completed, from pool -2016-06-13 19:15:39 INFO DAGScheduler:59 - Job 13 finished: collect at KMeans.scala:431, took 0.048800 s -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(1456) called with curMem=145273, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 1456.0 B, free 730.4 MB) -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(582) called with curMem=146729, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 582.0 B, free 730.4 MB) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:49158 (size: 582.0 B, free: 730.5 MB) -2016-06-13 19:15:39 INFO SparkContext:59 - Created broadcast 25 from broadcast at KMeans.scala:394 -2016-06-13 19:15:39 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:15:39 INFO DAGScheduler:59 - Got job 14 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:15:39 INFO DAGScheduler:59 - Final stage: ResultStage 149(aggregate at KMeans.scala:402) -2016-06-13 19:15:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 144, ShuffleMapStage 141, ShuffleMapStage 148, ShuffleMapStage 142, ShuffleMapStage 146) -2016-06-13 19:15:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:39 INFO DAGScheduler:59 - Submitting ResultStage 149 (MapPartitionsRDD[114] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(10656) called with curMem=147311, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 10.4 KB, free 730.4 MB) -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(4409) called with curMem=157967, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:49158 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 149 (MapPartitionsRDD[114] at map at KMeans.scala:396) -2016-06-13 19:15:39 INFO TaskSchedulerImpl:59 - Adding task set 149.0 with 2 tasks -2016-06-13 19:15:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 149.0 (TID 46, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:15:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 149.0 (TID 47, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:15:39 INFO Executor:59 - Running task 1.0 in stage 149.0 (TID 47) -2016-06-13 19:15:39 INFO Executor:59 - Running task 0.0 in stage 149.0 (TID 46) -2016-06-13 19:15:39 INFO CacheManager:59 - Partition rdd_114_1 not found, computing it -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_110_1 locally -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=162376, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block rdd_114_1 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:15:39 INFO CacheManager:59 - Partition rdd_114_0 not found, computing it -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added rdd_114_1 in memory on localhost:49158 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_110_0 locally -2016-06-13 19:15:39 INFO Executor:59 - Finished task 1.0 in stage 149.0 (TID 47). 2647 bytes result sent to driver -2016-06-13 19:15:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 149.0 (TID 47) in 20 ms on localhost (1/2) -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=162872, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block rdd_114_0 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added rdd_114_0 in memory on localhost:49158 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:15:39 INFO Executor:59 - Finished task 0.0 in stage 149.0 (TID 46). 2647 bytes result sent to driver -2016-06-13 19:15:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 149.0 (TID 46) in 26 ms on localhost (2/2) -2016-06-13 19:15:39 INFO DAGScheduler:59 - ResultStage 149 (aggregate at KMeans.scala:402) finished in 0.026 s -2016-06-13 19:15:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 149.0, whose tasks have all completed, from pool -2016-06-13 19:15:39 INFO DAGScheduler:59 - Job 14 finished: aggregate at KMeans.scala:402, took 0.058712 s -2016-06-13 19:15:39 INFO MapPartitionsRDD:59 - Removing RDD 110 from persistence list -2016-06-13 19:15:39 INFO BlockManager:59 - Removing RDD 110 -2016-06-13 19:15:39 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:15:39 INFO DAGScheduler:59 - Got job 15 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:15:39 INFO DAGScheduler:59 - Final stage: ResultStage 161(collect at KMeans.scala:431) -2016-06-13 19:15:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 153, ShuffleMapStage 154, ShuffleMapStage 158, ShuffleMapStage 156, ShuffleMapStage 160) -2016-06-13 19:15:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:39 INFO DAGScheduler:59 - Submitting ResultStage 161 (MapPartitionsRDD[116] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(10712) called with curMem=162376, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 10.5 KB, free 730.4 MB) -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(4468) called with curMem=173088, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:49158 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 161 (MapPartitionsRDD[116] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:15:39 INFO TaskSchedulerImpl:59 - Adding task set 161.0 with 2 tasks -2016-06-13 19:15:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 161.0 (TID 48, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:15:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 161.0 (TID 49, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:15:39 INFO Executor:59 - Running task 1.0 in stage 161.0 (TID 49) -2016-06-13 19:15:39 INFO Executor:59 - Running task 0.0 in stage 161.0 (TID 48) -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_114_1 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_114_0 locally -2016-06-13 19:15:39 INFO Executor:59 - Finished task 0.0 in stage 161.0 (TID 48). 2722 bytes result sent to driver -2016-06-13 19:15:39 INFO Executor:59 - Finished task 1.0 in stage 161.0 (TID 49). 2040 bytes result sent to driver -2016-06-13 19:15:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 161.0 (TID 48) in 8 ms on localhost (1/2) -2016-06-13 19:15:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 161.0 (TID 49) in 8 ms on localhost (2/2) -2016-06-13 19:15:39 INFO DAGScheduler:59 - ResultStage 161 (collect at KMeans.scala:431) finished in 0.010 s -2016-06-13 19:15:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 161.0, whose tasks have all completed, from pool -2016-06-13 19:15:39 INFO DAGScheduler:59 - Job 15 finished: collect at KMeans.scala:431, took 0.031451 s -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(648) called with curMem=177556, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 648.0 B, free 730.4 MB) -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(429) called with curMem=178204, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 429.0 B, free 730.4 MB) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:49158 (size: 429.0 B, free: 730.5 MB) -2016-06-13 19:15:39 INFO SparkContext:59 - Created broadcast 28 from broadcast at KMeans.scala:394 -2016-06-13 19:15:39 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:15:39 INFO DAGScheduler:59 - Got job 16 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:15:39 INFO DAGScheduler:59 - Final stage: ResultStage 173(aggregate at KMeans.scala:402) -2016-06-13 19:15:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 168, ShuffleMapStage 165, ShuffleMapStage 172, ShuffleMapStage 166, ShuffleMapStage 170) -2016-06-13 19:15:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:39 INFO DAGScheduler:59 - Submitting ResultStage 173 (MapPartitionsRDD[118] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(10888) called with curMem=178633, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 10.6 KB, free 730.4 MB) -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(4468) called with curMem=189521, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:49158 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 173 (MapPartitionsRDD[118] at map at KMeans.scala:396) -2016-06-13 19:15:39 INFO TaskSchedulerImpl:59 - Adding task set 173.0 with 2 tasks -2016-06-13 19:15:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 173.0 (TID 50, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:15:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 173.0 (TID 51, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:15:39 INFO Executor:59 - Running task 0.0 in stage 173.0 (TID 50) -2016-06-13 19:15:39 INFO Executor:59 - Running task 1.0 in stage 173.0 (TID 51) -2016-06-13 19:15:39 INFO CacheManager:59 - Partition rdd_118_1 not found, computing it -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_114_1 locally -2016-06-13 19:15:39 INFO CacheManager:59 - Partition rdd_118_0 not found, computing it -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=193989, maxMem=766075207 -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:15:39 INFO MemoryStore:59 - Block rdd_118_1 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:15:39 INFO BlockManager:59 - Found block rdd_114_0 locally -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added rdd_118_1 in memory on localhost:49158 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:15:39 INFO Executor:59 - Finished task 1.0 in stage 173.0 (TID 51). 2647 bytes result sent to driver -2016-06-13 19:15:39 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=194485, maxMem=766075207 -2016-06-13 19:15:39 INFO MemoryStore:59 - Block rdd_118_0 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:15:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 173.0 (TID 51) in 17 ms on localhost (1/2) -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Added rdd_118_0 in memory on localhost:49158 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:15:39 INFO Executor:59 - Finished task 0.0 in stage 173.0 (TID 50). 2647 bytes result sent to driver -2016-06-13 19:15:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 173.0 (TID 50) in 32 ms on localhost (2/2) -2016-06-13 19:15:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 173.0, whose tasks have all completed, from pool -2016-06-13 19:15:39 INFO DAGScheduler:59 - ResultStage 173 (aggregate at KMeans.scala:402) finished in 0.033 s -2016-06-13 19:15:39 INFO DAGScheduler:59 - Job 16 finished: aggregate at KMeans.scala:402, took 0.062944 s -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:49158 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO MapPartitionsRDD:59 - Removing RDD 114 from persistence list -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-06-13 19:15:39 INFO BlockManager:59 - Removing RDD 114 -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:49158 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:49158 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:49158 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 19:15:39 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:15:39 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:49158 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:15:39 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:49158 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Got job 17 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Final stage: ResultStage 185(collect at KMeans.scala:431) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 180, ShuffleMapStage 184, ShuffleMapStage 177, ShuffleMapStage 178, ShuffleMapStage 182) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:49158 in memory (size: 9.9 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting ResultStage 185 (MapPartitionsRDD[120] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(10944) called with curMem=69301, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 10.7 KB, free 730.5 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(4514) called with curMem=80245, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.5 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:49158 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 185 (MapPartitionsRDD[120] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Adding task set 185.0 with 2 tasks -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:49158 in memory (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 185.0 (TID 52, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 185.0 (TID 53, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 19:15:40 INFO Executor:59 - Running task 1.0 in stage 185.0 (TID 53) -2016-06-13 19:15:40 INFO Executor:59 - Running task 0.0 in stage 185.0 (TID 52) -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_118_1 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_118_0 locally -2016-06-13 19:15:40 INFO Executor:59 - Finished task 0.0 in stage 185.0 (TID 52). 2040 bytes result sent to driver -2016-06-13 19:15:40 INFO Executor:59 - Finished task 1.0 in stage 185.0 (TID 53). 2040 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 185.0 (TID 52) in 10 ms on localhost (1/2) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 185.0 (TID 53) in 10 ms on localhost (2/2) -2016-06-13 19:15:40 INFO DAGScheduler:59 - ResultStage 185 (collect at KMeans.scala:431) finished in 0.011 s -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 185.0, whose tasks have all completed, from pool -2016-06-13 19:15:40 INFO DAGScheduler:59 - Job 17 finished: collect at KMeans.scala:431, took 0.043482 s -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(584) called with curMem=73128, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 584.0 B, free 730.5 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(236) called with curMem=73712, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 236.0 B, free 730.5 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:49158 (size: 236.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 31 from broadcast at KMeans.scala:394 -2016-06-13 19:15:40 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Got job 18 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:15:40 INFO DAGScheduler:59 - Final stage: ResultStage 197(aggregate at KMeans.scala:402) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 190, ShuffleMapStage 194, ShuffleMapStage 192, ShuffleMapStage 189, ShuffleMapStage 196) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting ResultStage 197 (MapPartitionsRDD[122] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(11128) called with curMem=73948, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 10.9 KB, free 730.5 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(4524) called with curMem=85076, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.5 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:49158 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 197 (MapPartitionsRDD[122] at map at KMeans.scala:396) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Adding task set 197.0 with 2 tasks -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 197.0 (TID 54, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 197.0 (TID 55, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:15:40 INFO Executor:59 - Running task 0.0 in stage 197.0 (TID 54) -2016-06-13 19:15:40 INFO Executor:59 - Running task 1.0 in stage 197.0 (TID 55) -2016-06-13 19:15:40 INFO CacheManager:59 - Partition rdd_122_0 not found, computing it -2016-06-13 19:15:40 INFO CacheManager:59 - Partition rdd_122_1 not found, computing it -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_118_0 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_118_1 locally -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=89600, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block rdd_122_0 stored as values in memory (estimated size 496.0 B, free 730.5 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=90096, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block rdd_122_1 stored as values in memory (estimated size 496.0 B, free 730.5 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added rdd_122_0 in memory on localhost:49158 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO Executor:59 - Finished task 0.0 in stage 197.0 (TID 54). 2647 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 197.0 (TID 54) in 11 ms on localhost (1/2) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added rdd_122_1 in memory on localhost:49158 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO Executor:59 - Finished task 1.0 in stage 197.0 (TID 55). 2647 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 197.0 (TID 55) in 12 ms on localhost (2/2) -2016-06-13 19:15:40 INFO DAGScheduler:59 - ResultStage 197 (aggregate at KMeans.scala:402) finished in 0.013 s -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 197.0, whose tasks have all completed, from pool -2016-06-13 19:15:40 INFO DAGScheduler:59 - Job 18 finished: aggregate at KMeans.scala:402, took 0.033523 s -2016-06-13 19:15:40 INFO MapPartitionsRDD:59 - Removing RDD 118 from persistence list -2016-06-13 19:15:40 INFO BlockManager:59 - Removing RDD 118 -2016-06-13 19:15:40 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Got job 19 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:15:40 INFO DAGScheduler:59 - Final stage: ResultStage 209(collect at KMeans.scala:431) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 204, ShuffleMapStage 201, ShuffleMapStage 208, ShuffleMapStage 202, ShuffleMapStage 206) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting ResultStage 209 (MapPartitionsRDD[124] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(11176) called with curMem=89600, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 10.9 KB, free 730.5 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(4565) called with curMem=100776, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.5 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:49158 (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 209 (MapPartitionsRDD[124] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Adding task set 209.0 with 2 tasks -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 209.0 (TID 56, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 209.0 (TID 57, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:15:40 INFO Executor:59 - Running task 1.0 in stage 209.0 (TID 57) -2016-06-13 19:15:40 INFO Executor:59 - Running task 0.0 in stage 209.0 (TID 56) -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_122_1 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_122_0 locally -2016-06-13 19:15:40 INFO Executor:59 - Finished task 1.0 in stage 209.0 (TID 57). 2040 bytes result sent to driver -2016-06-13 19:15:40 INFO Executor:59 - Finished task 0.0 in stage 209.0 (TID 56). 2040 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 209.0 (TID 57) in 6 ms on localhost (1/2) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 209.0 (TID 56) in 6 ms on localhost (2/2) -2016-06-13 19:15:40 INFO DAGScheduler:59 - ResultStage 209 (collect at KMeans.scala:431) finished in 0.006 s -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 209.0, whose tasks have all completed, from pool -2016-06-13 19:15:40 INFO DAGScheduler:59 - Job 19 finished: collect at KMeans.scala:431, took 0.026521 s -2016-06-13 19:15:40 INFO MapPartitionsRDD:59 - Removing RDD 122 from persistence list -2016-06-13 19:15:40 INFO BlockManager:59 - Removing RDD 122 -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(2880) called with curMem=104349, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 2.8 KB, free 730.5 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(708) called with curMem=107229, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 708.0 B, free 730.5 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:49158 (size: 708.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 34 from broadcast at KMeans.scala:445 -2016-06-13 19:15:40 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:450 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Registering RDD 125 (flatMap at KMeans.scala:446) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Got job 20 (collectAsMap at KMeans.scala:450) with 2 output partitions -2016-06-13 19:15:40 INFO DAGScheduler:59 - Final stage: ResultStage 222(collectAsMap at KMeans.scala:450) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 221) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 221) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 221 (MapPartitionsRDD[125] at flatMap at KMeans.scala:446), which has no missing parents -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(9672) called with curMem=107937, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 9.4 KB, free 730.5 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(4106) called with curMem=117609, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.5 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:49158 (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 221 (MapPartitionsRDD[125] at flatMap at KMeans.scala:446) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Adding task set 221.0 with 2 tasks -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 221.0 (TID 58, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 221.0 (TID 59, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:15:40 INFO Executor:59 - Running task 0.0 in stage 221.0 (TID 58) -2016-06-13 19:15:40 INFO Executor:59 - Running task 1.0 in stage 221.0 (TID 59) -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:15:40 INFO Executor:59 - Finished task 1.0 in stage 221.0 (TID 59). 2254 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 221.0 (TID 59) in 34 ms on localhost (1/2) -2016-06-13 19:15:40 INFO Executor:59 - Finished task 0.0 in stage 221.0 (TID 58). 2254 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 221.0 (TID 58) in 36 ms on localhost (2/2) -2016-06-13 19:15:40 INFO DAGScheduler:59 - ShuffleMapStage 221 (flatMap at KMeans.scala:446) finished in 0.037 s -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 221.0, whose tasks have all completed, from pool -2016-06-13 19:15:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:40 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:40 INFO DAGScheduler:59 - waiting: Set(ResultStage 222) -2016-06-13 19:15:40 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents for ResultStage 222: List() -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting ResultStage 222 (ShuffledRDD[126] at reduceByKey at KMeans.scala:450), which is now runnable -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=121715, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(1382) called with curMem=124027, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 1382.0 B, free 730.5 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:49158 (size: 1382.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 222 (ShuffledRDD[126] at reduceByKey at KMeans.scala:450) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Adding task set 222.0 with 2 tasks -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 222.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 222.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:15:40 INFO Executor:59 - Running task 1.0 in stage 222.0 (TID 61) -2016-06-13 19:15:40 INFO Executor:59 - Running task 0.0 in stage 222.0 (TID 60) -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:40 INFO Executor:59 - Finished task 1.0 in stage 222.0 (TID 61). 1744 bytes result sent to driver -2016-06-13 19:15:40 INFO Executor:59 - Finished task 0.0 in stage 222.0 (TID 60). 1852 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 222.0 (TID 61) in 13 ms on localhost (1/2) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 222.0 (TID 60) in 14 ms on localhost (2/2) -2016-06-13 19:15:40 INFO DAGScheduler:59 - ResultStage 222 (collectAsMap at KMeans.scala:450) finished in 0.014 s -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 222.0, whose tasks have all completed, from pool -2016-06-13 19:15:40 INFO DAGScheduler:59 - Job 20 finished: collectAsMap at KMeans.scala:450, took 0.076070 s -2016-06-13 19:15:40 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:15:40 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:15:40 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:15:40 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:15:40 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:15:40 INFO KMeans:59 - Initialization with k-means|| took 1.258 seconds. -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(824) called with curMem=125409, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 824.0 B, free 730.5 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(339) called with curMem=126233, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 339.0 B, free 730.5 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:49158 (size: 339.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 37 from broadcast at KMeans.scala:276 -2016-06-13 19:15:40 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:302 -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 151 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 151 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:15:40 INFO DAGScheduler:59 - Registering RDD 127 (mapPartitions at KMeans.scala:279) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Got job 21 (collectAsMap at KMeans.scala:302) with 2 output partitions -2016-06-13 19:15:40 INFO DAGScheduler:59 - Final stage: ResultStage 235(collectAsMap at KMeans.scala:302) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 234) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 234) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 234 (MapPartitionsRDD[127] at mapPartitions at KMeans.scala:279), which has no missing parents -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(10296) called with curMem=126572, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 10.1 KB, free 730.5 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(4440) called with curMem=136868, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.5 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:49158 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 234 (MapPartitionsRDD[127] at mapPartitions at KMeans.scala:279) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Adding task set 234.0 with 2 tasks -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 234.0 (TID 62, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 234.0 (TID 63, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:15:40 INFO Executor:59 - Running task 0.0 in stage 234.0 (TID 62) -2016-06-13 19:15:40 INFO Executor:59 - Running task 1.0 in stage 234.0 (TID 63) -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:15:40 INFO Executor:59 - Finished task 0.0 in stage 234.0 (TID 62). 2413 bytes result sent to driver -2016-06-13 19:15:40 INFO Executor:59 - Finished task 1.0 in stage 234.0 (TID 63). 2413 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 234.0 (TID 63) in 23 ms on localhost (1/2) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 234.0 (TID 62) in 25 ms on localhost (2/2) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 234.0, whose tasks have all completed, from pool -2016-06-13 19:15:40 INFO DAGScheduler:59 - ShuffleMapStage 234 (mapPartitions at KMeans.scala:279) finished in 0.025 s -2016-06-13 19:15:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:40 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:40 INFO DAGScheduler:59 - waiting: Set(ResultStage 235) -2016-06-13 19:15:40 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents for ResultStage 235: List() -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting ResultStage 235 (ShuffledRDD[128] at reduceByKey at KMeans.scala:302), which is now runnable -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(2808) called with curMem=141308, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(1589) called with curMem=144116, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 1589.0 B, free 730.4 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:49158 (size: 1589.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 235 (ShuffledRDD[128] at reduceByKey at KMeans.scala:302) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Adding task set 235.0 with 2 tasks -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 235.0 (TID 64, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 235.0 (TID 65, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:15:40 INFO Executor:59 - Running task 1.0 in stage 235.0 (TID 65) -2016-06-13 19:15:40 INFO Executor:59 - Running task 0.0 in stage 235.0 (TID 64) -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:40 INFO Executor:59 - Finished task 1.0 in stage 235.0 (TID 65). 1671 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 235.0 (TID 65) in 11 ms on localhost (1/2) -2016-06-13 19:15:40 INFO Executor:59 - Finished task 0.0 in stage 235.0 (TID 64). 1785 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 235.0 (TID 64) in 15 ms on localhost (2/2) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 235.0, whose tasks have all completed, from pool -2016-06-13 19:15:40 INFO DAGScheduler:59 - ResultStage 235 (collectAsMap at KMeans.scala:302) finished in 0.016 s -2016-06-13 19:15:40 INFO DAGScheduler:59 - Job 21 finished: collectAsMap at KMeans.scala:302, took 0.072736 s -2016-06-13 19:15:40 INFO KMeans:59 - Run 0 finished in 1 iterations -2016-06-13 19:15:40 INFO KMeans:59 - Run 1 finished in 1 iterations -2016-06-13 19:15:40 INFO KMeans:59 - Run 2 finished in 1 iterations -2016-06-13 19:15:40 INFO KMeans:59 - Run 3 finished in 1 iterations -2016-06-13 19:15:40 INFO KMeans:59 - Run 4 finished in 1 iterations -2016-06-13 19:15:40 INFO KMeans:59 - Iterations took 0.113 seconds. -2016-06-13 19:15:40 INFO KMeans:59 - KMeans converged in 1 iterations. -2016-06-13 19:15:40 INFO KMeans:59 - The cost for the best run is 6.650150218788335E-6. -2016-06-13 19:15:40 INFO MapPartitionsRDD:59 - Removing RDD 100 from persistence list -2016-06-13 19:15:40 INFO BlockManager:59 - Removing RDD 100 -2016-06-13 19:15:40 WARN KMeans:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:15:40 INFO SparkContext:59 - Starting job: collect at PowerIterationClusteringSuite.scala:56 -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 151 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 151 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:15:40 INFO DAGScheduler:59 - Got job 22 (collect at PowerIterationClusteringSuite.scala:56) with 2 output partitions -2016-06-13 19:15:40 INFO DAGScheduler:59 - Final stage: ResultStage 247(collect at PowerIterationClusteringSuite.scala:56) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 242, ShuffleMapStage 239, ShuffleMapStage 246, ShuffleMapStage 240, ShuffleMapStage 244) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting ResultStage 247 (MapPartitionsRDD[131] at mapPartitions at PowerIterationClustering.scala:232), which has no missing parents -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(8472) called with curMem=145225, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 8.3 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=153697, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:49158 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 247 (MapPartitionsRDD[131] at mapPartitions at PowerIterationClustering.scala:232) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Adding task set 247.0 with 2 tasks -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 247.0 (TID 66, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 247.0 (TID 67, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:15:40 INFO Executor:59 - Running task 1.0 in stage 247.0 (TID 67) -2016-06-13 19:15:40 INFO Executor:59 - Running task 0.0 in stage 247.0 (TID 66) -2016-06-13 19:15:40 INFO CacheManager:59 - Partition rdd_129_1 not found, computing it -2016-06-13 19:15:40 INFO CacheManager:59 - Partition rdd_129_0 not found, computing it -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=157289, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block rdd_129_1 stored as values in memory (estimated size 1168.0 B, free 730.4 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=158457, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block rdd_129_0 stored as values in memory (estimated size 1168.0 B, free 730.4 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added rdd_129_1 in memory on localhost:49158 (size: 1168.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added rdd_129_0 in memory on localhost:49158 (size: 1168.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO Executor:59 - Finished task 1.0 in stage 247.0 (TID 67). 2919 bytes result sent to driver -2016-06-13 19:15:40 INFO Executor:59 - Finished task 0.0 in stage 247.0 (TID 66). 2919 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 247.0 (TID 67) in 12 ms on localhost (1/2) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 247.0 (TID 66) in 14 ms on localhost (2/2) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 247.0, whose tasks have all completed, from pool -2016-06-13 19:15:40 INFO DAGScheduler:59 - ResultStage 247 (collect at PowerIterationClusteringSuite.scala:56) finished in 0.014 s -2016-06-13 19:15:40 INFO DAGScheduler:59 - Job 22 finished: collect at PowerIterationClusteringSuite.scala:56, took 0.034183 s -2016-06-13 19:15:40 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:340 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Registering RDD 143 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Registering RDD 137 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Got job 23 (sum at PowerIterationClustering.scala:340) with 2 output partitions -2016-06-13 19:15:40 INFO DAGScheduler:59 - Final stage: ResultStage 250(sum at PowerIterationClustering.scala:340) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 248, ShuffleMapStage 249) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 248, ShuffleMapStage 249) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 248 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[143] at mapPartitions at GraphImpl.scala:235), which has no missing parents -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(3432) called with curMem=159625, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(1974) called with curMem=163057, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 1974.0 B, free 730.4 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:49158 (size: 1974.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 248 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[143] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Adding task set 248.0 with 2 tasks -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 249 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[137] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 248.0 (TID 68, localhost, PROCESS_LOCAL, 2488 bytes) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 248.0 (TID 69, localhost, PROCESS_LOCAL, 2563 bytes) -2016-06-13 19:15:40 INFO Executor:59 - Running task 1.0 in stage 248.0 (TID 69) -2016-06-13 19:15:40 INFO Executor:59 - Running task 0.0 in stage 248.0 (TID 68) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(3016) called with curMem=165031, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(1779) called with curMem=168047, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1779.0 B, free 730.4 MB) -2016-06-13 19:15:40 INFO CacheManager:59 - Partition rdd_134_0 not found, computing it -2016-06-13 19:15:40 INFO CacheManager:59 - Partition rdd_134_1 not found, computing it -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(2952) called with curMem=169826, maxMem=766075207 -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:49158 (size: 1779.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - Block rdd_134_1 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(2856) called with curMem=172778, maxMem=766075207 -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block rdd_134_0 stored as values in memory (estimated size 2.8 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 249 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[137] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added rdd_134_1 in memory on localhost:49158 (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Adding task set 249.0 with 2 tasks -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added rdd_134_0 in memory on localhost:49158 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 249.0 (TID 70, localhost, PROCESS_LOCAL, 2488 bytes) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 249.0 (TID 71, localhost, PROCESS_LOCAL, 2563 bytes) -2016-06-13 19:15:40 INFO Executor:59 - Running task 1.0 in stage 249.0 (TID 71) -2016-06-13 19:15:40 INFO Executor:59 - Running task 0.0 in stage 249.0 (TID 70) -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_134_0 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_134_1 locally -2016-06-13 19:15:40 INFO Executor:59 - Finished task 0.0 in stage 248.0 (TID 68). 1739 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 248.0 (TID 68) in 28 ms on localhost (1/2) -2016-06-13 19:15:40 INFO Executor:59 - Finished task 0.0 in stage 249.0 (TID 70). 2254 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 249.0 (TID 70) in 22 ms on localhost (1/2) -2016-06-13 19:15:40 INFO Executor:59 - Finished task 1.0 in stage 249.0 (TID 71). 2254 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 249.0 (TID 71) in 29 ms on localhost (2/2) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 249.0, whose tasks have all completed, from pool -2016-06-13 19:15:40 INFO DAGScheduler:59 - ShuffleMapStage 249 (mapPartitions at VertexRDD.scala:358) finished in 0.031 s -2016-06-13 19:15:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:40 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 248) -2016-06-13 19:15:40 INFO DAGScheduler:59 - waiting: Set(ResultStage 250) -2016-06-13 19:15:40 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:40 INFO Executor:59 - Finished task 1.0 in stage 248.0 (TID 69). 1739 bytes result sent to driver -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents for ResultStage 250: List(ShuffleMapStage 248) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 248.0 (TID 69) in 43 ms on localhost (2/2) -2016-06-13 19:15:40 INFO DAGScheduler:59 - ShuffleMapStage 248 (mapPartitions at GraphImpl.scala:235) finished in 0.044 s -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 248.0, whose tasks have all completed, from pool -2016-06-13 19:15:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:40 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:40 INFO DAGScheduler:59 - waiting: Set(ResultStage 250) -2016-06-13 19:15:40 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents for ResultStage 250: List() -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting ResultStage 250 (MapPartitionsRDD[153] at values at PowerIterationClustering.scala:340), which is now runnable -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(4728) called with curMem=175634, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 4.6 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(2556) called with curMem=180362, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:49158 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 250 (MapPartitionsRDD[153] at values at PowerIterationClustering.scala:340) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Adding task set 250.0 with 2 tasks -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 250.0 (TID 72, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 250.0 (TID 73, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:15:40 INFO Executor:59 - Running task 0.0 in stage 250.0 (TID 72) -2016-06-13 19:15:40 INFO Executor:59 - Running task 1.0 in stage 250.0 (TID 73) -2016-06-13 19:15:40 INFO CacheManager:59 - Partition rdd_145_0 not found, computing it -2016-06-13 19:15:40 INFO CacheManager:59 - Partition rdd_145_1 not found, computing it -2016-06-13 19:15:40 INFO CacheManager:59 - Partition rdd_140_0 not found, computing it -2016-06-13 19:15:40 INFO CacheManager:59 - Partition rdd_140_1 not found, computing it -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(1792) called with curMem=182918, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block rdd_140_1 stored as values in memory (estimated size 1792.0 B, free 730.4 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(1776) called with curMem=184710, maxMem=766075207 -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added rdd_140_1 in memory on localhost:49158 (size: 1792.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - Block rdd_140_0 stored as values in memory (estimated size 1776.0 B, free 730.4 MB) -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added rdd_140_0 in memory on localhost:49158 (size: 1776.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(1840) called with curMem=186486, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block rdd_145_1 stored as values in memory (estimated size 1840.0 B, free 730.4 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(1824) called with curMem=188326, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block rdd_145_0 stored as values in memory (estimated size 1824.0 B, free 730.4 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added rdd_145_1 in memory on localhost:49158 (size: 1840.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added rdd_145_0 in memory on localhost:49158 (size: 1824.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO Executor:59 - Finished task 1.0 in stage 250.0 (TID 73). 1840 bytes result sent to driver -2016-06-13 19:15:40 INFO Executor:59 - Finished task 0.0 in stage 250.0 (TID 72). 1840 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 250.0 (TID 73) in 16 ms on localhost (1/2) -2016-06-13 19:15:40 INFO DAGScheduler:59 - ResultStage 250 (sum at PowerIterationClustering.scala:340) finished in 0.016 s -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 250.0 (TID 72) in 16 ms on localhost (2/2) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 250.0, whose tasks have all completed, from pool -2016-06-13 19:15:40 INFO DAGScheduler:59 - Job 23 finished: sum at PowerIterationClustering.scala:340, took 0.080889 s -2016-06-13 19:15:40 INFO SparkContext:59 - Starting job: reduce at VertexRDDImpl.scala:90 -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:15:40 INFO DAGScheduler:59 - Got job 24 (reduce at VertexRDDImpl.scala:90) with 2 output partitions -2016-06-13 19:15:40 INFO DAGScheduler:59 - Final stage: ResultStage 253(reduce at VertexRDDImpl.scala:90) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 251, ShuffleMapStage 252) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting ResultStage 253 (MapPartitionsRDD[158] at map at VertexRDDImpl.scala:90), which has no missing parents -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(5200) called with curMem=190150, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(2659) called with curMem=195350, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:49158 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 253 (MapPartitionsRDD[158] at map at VertexRDDImpl.scala:90) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Adding task set 253.0 with 2 tasks -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 253.0 (TID 74, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 253.0 (TID 75, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:15:40 INFO Executor:59 - Running task 1.0 in stage 253.0 (TID 75) -2016-06-13 19:15:40 INFO Executor:59 - Running task 0.0 in stage 253.0 (TID 74) -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_145_0 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_145_1 locally -2016-06-13 19:15:40 INFO Executor:59 - Finished task 0.0 in stage 253.0 (TID 74). 2161 bytes result sent to driver -2016-06-13 19:15:40 INFO Executor:59 - Finished task 1.0 in stage 253.0 (TID 75). 2161 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 253.0 (TID 74) in 6 ms on localhost (1/2) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 253.0 (TID 75) in 6 ms on localhost (2/2) -2016-06-13 19:15:40 INFO DAGScheduler:59 - ResultStage 253 (reduce at VertexRDDImpl.scala:90) finished in 0.006 s -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 253.0, whose tasks have all completed, from pool -2016-06-13 19:15:40 INFO DAGScheduler:59 - Job 24 finished: reduce at VertexRDDImpl.scala:90, took 0.018409 s -2016-06-13 19:15:40 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Registering RDD 147 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Registering RDD 159 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Registering RDD 163 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Got job 25 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:15:40 INFO DAGScheduler:59 - Final stage: ResultStage 261(sum at PowerIterationClustering.scala:369) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 260, ShuffleMapStage 257, ShuffleMapStage 258) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 260) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 256 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[147] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=198009, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 4.1 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(2378) called with curMem=202257, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:49158 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 256 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[147] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Adding task set 256.0 with 2 tasks -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 259 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[159] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 256.0 (TID 76, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 256.0 (TID 77, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:15:40 INFO Executor:59 - Running task 0.0 in stage 256.0 (TID 76) -2016-06-13 19:15:40 INFO Executor:59 - Running task 1.0 in stage 256.0 (TID 77) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(4904) called with curMem=204635, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_145_0 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_145_1 locally -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(2572) called with curMem=209539, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:49158 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 259 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[159] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Adding task set 259.0 with 2 tasks -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 259.0 (TID 78, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 259.0 (TID 79, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:15:40 INFO Executor:59 - Running task 1.0 in stage 259.0 (TID 79) -2016-06-13 19:15:40 INFO Executor:59 - Running task 0.0 in stage 259.0 (TID 78) -2016-06-13 19:15:40 INFO Executor:59 - Finished task 0.0 in stage 256.0 (TID 76). 2254 bytes result sent to driver -2016-06-13 19:15:40 INFO Executor:59 - Finished task 1.0 in stage 256.0 (TID 77). 2254 bytes result sent to driver -2016-06-13 19:15:40 INFO CacheManager:59 - Partition rdd_156_1 not found, computing it -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_145_1 locally -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=212111, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block rdd_156_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 256.0 (TID 76) in 21 ms on localhost (1/2) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added rdd_156_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO CacheManager:59 - Partition rdd_156_0 not found, computing it -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_145_0 locally -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 256.0 (TID 77) in 22 ms on localhost (2/2) -2016-06-13 19:15:40 INFO DAGScheduler:59 - ShuffleMapStage 256 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.023 s -2016-06-13 19:15:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:40 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 259) -2016-06-13 19:15:40 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 260, ResultStage 261) -2016-06-13 19:15:40 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 256.0, whose tasks have all completed, from pool -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=213543, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block rdd_156_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:15:40 INFO Executor:59 - Finished task 1.0 in stage 259.0 (TID 79). 2834 bytes result sent to driver -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 260: List(ShuffleMapStage 259) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added rdd_156_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 259.0 (TID 79) in 14 ms on localhost (1/2) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents for ResultStage 261: List(ShuffleMapStage 260) -2016-06-13 19:15:40 INFO Executor:59 - Finished task 0.0 in stage 259.0 (TID 78). 2834 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 259.0 (TID 78) in 18 ms on localhost (2/2) -2016-06-13 19:15:40 INFO DAGScheduler:59 - ShuffleMapStage 259 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.020 s -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 259.0, whose tasks have all completed, from pool -2016-06-13 19:15:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:40 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:40 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 260, ResultStage 261) -2016-06-13 19:15:40 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 260: List() -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents for ResultStage 261: List(ShuffleMapStage 260) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 260 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[163] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=214975, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(2538) called with curMem=219823, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:49158 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 260 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[163] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Adding task set 260.0 with 2 tasks -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 260.0 (TID 80, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 260.0 (TID 81, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:15:40 INFO Executor:59 - Running task 1.0 in stage 260.0 (TID 81) -2016-06-13 19:15:40 INFO Executor:59 - Running task 0.0 in stage 260.0 (TID 80) -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_134_0 locally -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_134_1 locally -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:40 INFO Executor:59 - Finished task 0.0 in stage 260.0 (TID 80). 2470 bytes result sent to driver -2016-06-13 19:15:40 INFO Executor:59 - Finished task 1.0 in stage 260.0 (TID 81). 2470 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 260.0 (TID 80) in 22 ms on localhost (1/2) -2016-06-13 19:15:40 INFO DAGScheduler:59 - ShuffleMapStage 260 (mapPartitions at GraphImpl.scala:235) finished in 0.024 s -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 260.0 (TID 81) in 22 ms on localhost (2/2) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 260.0, whose tasks have all completed, from pool -2016-06-13 19:15:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:40 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:40 INFO DAGScheduler:59 - waiting: Set(ResultStage 261) -2016-06-13 19:15:40 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents for ResultStage 261: List() -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting ResultStage 261 (MapPartitionsRDD[168] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(5944) called with curMem=222361, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 5.8 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=228305, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:49158 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 261 (MapPartitionsRDD[168] at map at PowerIterationClustering.scala:369) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Adding task set 261.0 with 2 tasks -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 261.0 (TID 82, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 261.0 (TID 83, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:15:40 INFO Executor:59 - Running task 0.0 in stage 261.0 (TID 82) -2016-06-13 19:15:40 INFO Executor:59 - Running task 1.0 in stage 261.0 (TID 83) -2016-06-13 19:15:40 INFO CacheManager:59 - Partition rdd_165_1 not found, computing it -2016-06-13 19:15:40 INFO CacheManager:59 - Partition rdd_165_0 not found, computing it -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_156_1 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_156_0 locally -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=231197, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block rdd_165_0 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added rdd_165_0 in memory on localhost:49158 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=232677, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block rdd_165_1 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:15:40 INFO Executor:59 - Finished task 0.0 in stage 261.0 (TID 82). 2880 bytes result sent to driver -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added rdd_165_1 in memory on localhost:49158 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO Executor:59 - Finished task 1.0 in stage 261.0 (TID 83). 2880 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 261.0 (TID 82) in 14 ms on localhost (1/2) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 261.0 (TID 83) in 14 ms on localhost (2/2) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 261.0, whose tasks have all completed, from pool -2016-06-13 19:15:40 INFO DAGScheduler:59 - ResultStage 261 (sum at PowerIterationClustering.scala:369) finished in 0.016 s -2016-06-13 19:15:40 INFO DAGScheduler:59 - Job 25 finished: sum at PowerIterationClustering.scala:369, took 0.100943 s -2016-06-13 19:15:40 INFO PowerIterationClustering:59 - Iteration 0: norm(v) = 36.22334869431644. -2016-06-13 19:15:40 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 151 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:15:40 INFO DAGScheduler:59 - Got job 26 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:15:40 INFO DAGScheduler:59 - Final stage: ResultStage 269(sum at PowerIterationClustering.scala:375) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 266, ShuffleMapStage 268, ShuffleMapStage 265) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting ResultStage 269 (MapPartitionsRDD[179] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(6480) called with curMem=234157, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 6.3 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(3063) called with curMem=240637, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:49158 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 269 (MapPartitionsRDD[179] at values at PowerIterationClustering.scala:375) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Adding task set 269.0 with 2 tasks -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 269.0 (TID 84, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 269.0 (TID 85, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:15:40 INFO Executor:59 - Running task 1.0 in stage 269.0 (TID 85) -2016-06-13 19:15:40 INFO Executor:59 - Running task 0.0 in stage 269.0 (TID 84) -2016-06-13 19:15:40 INFO CacheManager:59 - Partition rdd_171_1 not found, computing it -2016-06-13 19:15:40 INFO CacheManager:59 - Partition rdd_171_0 not found, computing it -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_156_1 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_156_0 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_165_1 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_165_0 locally -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=243700, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block rdd_171_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=245132, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block rdd_171_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added rdd_171_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added rdd_171_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO Executor:59 - Finished task 1.0 in stage 269.0 (TID 85). 2664 bytes result sent to driver -2016-06-13 19:15:40 INFO Executor:59 - Finished task 0.0 in stage 269.0 (TID 84). 2664 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 269.0 (TID 85) in 8 ms on localhost (1/2) -2016-06-13 19:15:40 INFO DAGScheduler:59 - ResultStage 269 (sum at PowerIterationClustering.scala:375) finished in 0.010 s -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 269.0 (TID 84) in 9 ms on localhost (2/2) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 269.0, whose tasks have all completed, from pool -2016-06-13 19:15:40 INFO DAGScheduler:59 - Job 26 finished: sum at PowerIterationClustering.scala:375, took 0.027461 s -2016-06-13 19:15:40 INFO PowerIterationClustering:59 - Iteration 0: delta = 0.010579646370791257. -2016-06-13 19:15:40 INFO PowerIterationClustering:59 - Iteration 0: diff(delta) = 1.7976931348623157E308. -2016-06-13 19:15:40 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Registering RDD 182 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Registering RDD 186 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Got job 27 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:15:40 INFO DAGScheduler:59 - Final stage: ResultStage 279(sum at PowerIterationClustering.scala:369) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 273, ShuffleMapStage 274, ShuffleMapStage 278, ShuffleMapStage 276) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 278) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 277 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[182] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(5704) called with curMem=246564, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 5.6 KB, free 730.3 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(2815) called with curMem=252268, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.3 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:49158 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 277 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[182] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Adding task set 277.0 with 2 tasks -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 277.0 (TID 86, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 277.0 (TID 87, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:15:40 INFO Executor:59 - Running task 0.0 in stage 277.0 (TID 86) -2016-06-13 19:15:40 INFO Executor:59 - Running task 1.0 in stage 277.0 (TID 87) -2016-06-13 19:15:40 INFO CacheManager:59 - Partition rdd_180_0 not found, computing it -2016-06-13 19:15:40 INFO CacheManager:59 - Partition rdd_180_1 not found, computing it -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_165_0 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_165_1 locally -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=255083, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block rdd_180_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=256515, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block rdd_180_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added rdd_180_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added rdd_180_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO Executor:59 - Finished task 1.0 in stage 277.0 (TID 87). 2834 bytes result sent to driver -2016-06-13 19:15:40 INFO Executor:59 - Finished task 0.0 in stage 277.0 (TID 86). 2834 bytes result sent to driver -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 277.0 (TID 87) in 13 ms on localhost (1/2) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 277.0 (TID 86) in 13 ms on localhost (2/2) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 277.0, whose tasks have all completed, from pool -2016-06-13 19:15:40 INFO DAGScheduler:59 - ShuffleMapStage 277 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.014 s -2016-06-13 19:15:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:40 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:40 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 278, ResultStage 279) -2016-06-13 19:15:40 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 278: List() -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents for ResultStage 279: List(ShuffleMapStage 278) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 278 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[186] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=257947, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 4.7 KB, free 730.3 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=262795, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:15:40 INFO BlockManager:59 - Removing RDD 122 -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:49158 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 278 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[186] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Adding task set 278.0 with 2 tasks -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned RDD 122 -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 278.0 (TID 88, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 278.0 (TID 89, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:15:40 INFO Executor:59 - Running task 0.0 in stage 278.0 (TID 88) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:49158 in memory (size: 236.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_134_0 locally -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:40 INFO Executor:59 - Running task 1.0 in stage 278.0 (TID 89) -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_134_1 locally -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:40 INFO Executor:59 - Finished task 1.0 in stage 278.0 (TID 89). 2470 bytes result sent to driver -2016-06-13 19:15:40 INFO Executor:59 - Finished task 0.0 in stage 278.0 (TID 88). 2470 bytes result sent to driver -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:49158 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 278.0 (TID 89) in 37 ms on localhost (1/2) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 278.0 (TID 88) in 39 ms on localhost (2/2) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 278.0, whose tasks have all completed, from pool -2016-06-13 19:15:40 INFO DAGScheduler:59 - ShuffleMapStage 278 (mapPartitions at GraphImpl.scala:235) finished in 0.040 s -2016-06-13 19:15:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:40 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:40 INFO DAGScheduler:59 - waiting: Set(ResultStage 279) -2016-06-13 19:15:40 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Missing parents for ResultStage 279: List() -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting ResultStage 279 (MapPartitionsRDD[191] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:49158 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(6640) called with curMem=246487, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 6.5 KB, free 730.3 MB) -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(3103) called with curMem=248223, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.3 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:49158 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 279 (MapPartitionsRDD[191] at map at PowerIterationClustering.scala:369) -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Adding task set 279.0 with 2 tasks -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:49158 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 279.0 (TID 90, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-06-13 19:15:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 279.0 (TID 91, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:15:40 INFO Executor:59 - Running task 0.0 in stage 279.0 (TID 90) -2016-06-13 19:15:40 INFO Executor:59 - Running task 1.0 in stage 279.0 (TID 91) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:49158 in memory (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-06-13 19:15:40 INFO CacheManager:59 - Partition rdd_188_0 not found, computing it -2016-06-13 19:15:40 INFO CacheManager:59 - Partition rdd_188_1 not found, computing it -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_180_0 locally -2016-06-13 19:15:40 INFO BlockManager:59 - Found block rdd_180_1 locally -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:49158 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=229557, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block rdd_188_0 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:15:40 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=231037, maxMem=766075207 -2016-06-13 19:15:40 INFO MemoryStore:59 - Block rdd_188_1 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added rdd_188_0 in memory on localhost:49158 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Added rdd_188_1 in memory on localhost:49158 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO Executor:59 - Finished task 0.0 in stage 279.0 (TID 90). 2880 bytes result sent to driver -2016-06-13 19:15:40 INFO Executor:59 - Finished task 1.0 in stage 279.0 (TID 91). 2880 bytes result sent to driver -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:49158 in memory (size: 1779.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 279.0 (TID 90) in 13 ms on localhost (1/2) -2016-06-13 19:15:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 279.0 (TID 91) in 12 ms on localhost (2/2) -2016-06-13 19:15:40 INFO DAGScheduler:59 - ResultStage 279 (sum at PowerIterationClustering.scala:369) finished in 0.013 s -2016-06-13 19:15:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 279.0, whose tasks have all completed, from pool -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-06-13 19:15:40 INFO DAGScheduler:59 - Job 27 finished: sum at PowerIterationClustering.scala:369, took 0.116642 s -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:49158 in memory (size: 1974.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO PowerIterationClustering:59 - Iteration 1: norm(v) = 36.22334869431644. -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:49158 in memory (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:49158 in memory (size: 1589.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:49158 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned shuffle 10 -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:49158 in memory (size: 339.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:49158 in memory (size: 1382.0 B, free: 730.5 MB) -2016-06-13 19:15:40 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:49158 in memory (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:49158 in memory (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 151 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:15:40 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 151 bytes -2016-06-13 19:15:40 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:15:40 INFO DAGScheduler:59 - Got job 28 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:15:40 INFO DAGScheduler:59 - Final stage: ResultStage 289(sum at PowerIterationClustering.scala:375) -2016-06-13 19:15:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 284, ShuffleMapStage 288, ShuffleMapStage 286, ShuffleMapStage 283) -2016-06-13 19:15:40 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:49158 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ResultStage 289 (MapPartitionsRDD[202] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:49158 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(7072) called with curMem=131548, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 6.9 KB, free 730.5 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(3232) called with curMem=138620, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-06-13 19:15:41 INFO BlockManager:59 - Removing RDD 171 -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:49158 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 289 (MapPartitionsRDD[202] at values at PowerIterationClustering.scala:375) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 289.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 289.0 (TID 92, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned RDD 171 -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 289.0 (TID 93, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 289.0 (TID 93) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 289.0 (TID 92) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:49158 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_194_1 not found, computing it -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_180_1 locally -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_194_0 not found, computing it -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_188_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_180_0 locally -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_188_0 locally -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=130152, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_194_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=131584, maxMem=766075207 -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_194_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_194_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 289.0 (TID 93). 2664 bytes result sent to driver -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_194_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:49158 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 289.0 (TID 93) in 9 ms on localhost (1/2) -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 289.0 (TID 92). 2664 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 289.0 (TID 92) in 12 ms on localhost (2/2) -2016-06-13 19:15:41 INFO DAGScheduler:59 - ResultStage 289 (sum at PowerIterationClustering.scala:375) finished in 0.013 s -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 289.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - Job 28 finished: sum at PowerIterationClustering.scala:375, took 0.035946 s -2016-06-13 19:15:41 INFO PowerIterationClustering:59 - Iteration 1: delta = 0.0. -2016-06-13 19:15:41 INFO PowerIterationClustering:59 - Iteration 1: diff(delta) = 0.010579646370791257. -2016-06-13 19:15:41 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Registering RDD 205 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Registering RDD 209 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Got job 29 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 301(sum at PowerIterationClustering.scala:369) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 300, ShuffleMapStage 296, ShuffleMapStage 293, ShuffleMapStage 294, ShuffleMapStage 298) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 300) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ShuffleMapStage 299 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[205] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(6288) called with curMem=125630, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 6.1 KB, free 730.5 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=131918, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.5 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:49158 (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 299 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[205] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 299.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 299.0 (TID 94, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 299.0 (TID 95, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 299.0 (TID 94) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 299.0 (TID 95) -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_203_1 not found, computing it -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_188_1 locally -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=134881, maxMem=766075207 -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_203_0 not found, computing it -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_188_0 locally -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_203_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=136313, maxMem=766075207 -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_203_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_203_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_203_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 299.0 (TID 95). 2834 bytes result sent to driver -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 299.0 (TID 94). 2834 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 299.0 (TID 95) in 9 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 299.0 (TID 94) in 10 ms on localhost (2/2) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 299.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - ShuffleMapStage 299 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.010 s -2016-06-13 19:15:41 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:41 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:41 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 300, ResultStage 301) -2016-06-13 19:15:41 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 300: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents for ResultStage 301: List(ShuffleMapStage 300) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ShuffleMapStage 300 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[209] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=137745, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 4.7 KB, free 730.5 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=142593, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:49158 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 300 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[209] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 300.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 300.0 (TID 96, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 300.0 (TID 97, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 300.0 (TID 97) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 300.0 (TID 96) -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_134_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_134_1 locally -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 300.0 (TID 97). 2470 bytes result sent to driver -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 300.0 (TID 96). 2470 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 300.0 (TID 97) in 21 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 300.0 (TID 96) in 21 ms on localhost (2/2) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 300.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - ShuffleMapStage 300 (mapPartitions at GraphImpl.scala:235) finished in 0.023 s -2016-06-13 19:15:41 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:41 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:41 INFO DAGScheduler:59 - waiting: Set(ResultStage 301) -2016-06-13 19:15:41 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents for ResultStage 301: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ResultStage 301 (MapPartitionsRDD[214] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(7224) called with curMem=145135, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 7.1 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(3233) called with curMem=152359, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:49158 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 301 (MapPartitionsRDD[214] at map at PowerIterationClustering.scala:369) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 301.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 301.0 (TID 98, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 301.0 (TID 99, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 301.0 (TID 98) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 301.0 (TID 99) -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_211_0 not found, computing it -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_211_1 not found, computing it -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_203_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_203_1 locally -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=155592, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_211_0 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=157072, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_211_1 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_211_0 in memory on localhost:49158 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_211_1 in memory on localhost:49158 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 301.0 (TID 98). 2880 bytes result sent to driver -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 301.0 (TID 99). 2880 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 301.0 (TID 98) in 9 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 301.0 (TID 99) in 10 ms on localhost (2/2) -2016-06-13 19:15:41 INFO DAGScheduler:59 - ResultStage 301 (sum at PowerIterationClustering.scala:369) finished in 0.010 s -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 301.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - Job 29 finished: sum at PowerIterationClustering.scala:369, took 0.074284 s -2016-06-13 19:15:41 INFO PowerIterationClustering:59 - Iteration 2: norm(v) = 36.22334869431644. -2016-06-13 19:15:41 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 151 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 151 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 151 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:15:41 INFO DAGScheduler:59 - Got job 30 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 313(sum at PowerIterationClustering.scala:375) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 305, ShuffleMapStage 306, ShuffleMapStage 310, ShuffleMapStage 308, ShuffleMapStage 312) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ResultStage 313 (MapPartitionsRDD[225] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=158552, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 7.5 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(3360) called with curMem=166208, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:49158 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 313 (MapPartitionsRDD[225] at values at PowerIterationClustering.scala:375) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 313.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 313.0 (TID 100, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 313.0 (TID 101, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 313.0 (TID 100) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 313.0 (TID 101) -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_217_0 not found, computing it -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_217_1 not found, computing it -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_203_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_203_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_211_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_211_1 locally -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=169568, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_217_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=171000, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_217_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_217_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_217_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 313.0 (TID 100). 2664 bytes result sent to driver -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 313.0 (TID 101). 2664 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 313.0 (TID 100) in 8 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 313.0 (TID 101) in 8 ms on localhost (2/2) -2016-06-13 19:15:41 INFO DAGScheduler:59 - ResultStage 313 (sum at PowerIterationClustering.scala:375) finished in 0.008 s -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 313.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - Job 30 finished: sum at PowerIterationClustering.scala:375, took 0.026009 s -2016-06-13 19:15:41 INFO PowerIterationClustering:59 - Iteration 2: delta = 0.0. -2016-06-13 19:15:41 INFO PowerIterationClustering:59 - Iteration 2: diff(delta) = 0.0. -2016-06-13 19:15:41 WARN KMeans:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:15:41 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Got job 31 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 325(takeSample at KMeans.scala:376) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 320, ShuffleMapStage 317, ShuffleMapStage 324, ShuffleMapStage 318, ShuffleMapStage 322) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ResultStage 325 (MapPartitionsRDD[233] at map at KMeans.scala:210), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(7888) called with curMem=172432, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 7.7 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(3424) called with curMem=180320, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:49158 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 325 (MapPartitionsRDD[233] at map at KMeans.scala:210) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 325.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 325.0 (TID 102, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 325.0 (TID 103, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 325.0 (TID 103) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 325.0 (TID 102) -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_228_1 not found, computing it -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_211_1 locally -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_228_0 not found, computing it -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_211_0 locally -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=183744, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_228_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=185176, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_228_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_228_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_228_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_231_1 not found, computing it -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_231_0 not found, computing it -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=186608, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_231_1 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=186848, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_231_0 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_231_1 in memory on localhost:49158 (size: 240.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_231_0 in memory on localhost:49158 (size: 240.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 325.0 (TID 103). 2717 bytes result sent to driver -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 325.0 (TID 102). 2717 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 325.0 (TID 103) in 13 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 325.0 (TID 102) in 14 ms on localhost (2/2) -2016-06-13 19:15:41 INFO DAGScheduler:59 - ResultStage 325 (takeSample at KMeans.scala:376) finished in 0.014 s -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 325.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - Job 31 finished: takeSample at KMeans.scala:376, took 0.030128 s -2016-06-13 19:15:41 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Got job 32 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 337(takeSample at KMeans.scala:376) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 332, ShuffleMapStage 336, ShuffleMapStage 329, ShuffleMapStage 330, ShuffleMapStage 334) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ResultStage 337 (PartitionwiseSampledRDD[235] at takeSample at KMeans.scala:376), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(25280) called with curMem=187088, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 24.7 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(10106) called with curMem=212368, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 9.9 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:49158 (size: 9.9 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 337 (PartitionwiseSampledRDD[235] at takeSample at KMeans.scala:376) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 337.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 337.0 (TID 104, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 337.0 (TID 105, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 337.0 (TID 104) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 337.0 (TID 105) -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 337.0 (TID 105). 2575 bytes result sent to driver -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 337.0 (TID 104). 2633 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 337.0 (TID 105) in 8 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 337.0 (TID 104) in 9 ms on localhost (2/2) -2016-06-13 19:15:41 INFO DAGScheduler:59 - ResultStage 337 (takeSample at KMeans.scala:376) finished in 0.010 s -2016-06-13 19:15:41 INFO DAGScheduler:59 - Job 32 finished: takeSample at KMeans.scala:376, took 0.028725 s -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 337.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=222474, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 880.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(487) called with curMem=223354, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 487.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:49158 (size: 487.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 60 from broadcast at KMeans.scala:394 -2016-06-13 19:15:41 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Got job 33 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 349(aggregate at KMeans.scala:402) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 342, ShuffleMapStage 344, ShuffleMapStage 341, ShuffleMapStage 348, ShuffleMapStage 346) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ResultStage 349 (MapPartitionsRDD[237] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(9912) called with curMem=223841, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 9.7 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(4245) called with curMem=233753, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 4.1 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:49158 (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 61 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 349 (MapPartitionsRDD[237] at map at KMeans.scala:396) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 349.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 349.0 (TID 106, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 349.0 (TID 107, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 349.0 (TID 106) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 349.0 (TID 107) -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_237_0 not found, computing it -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_237_1 not found, computing it -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=237998, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_237_0 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=238494, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_237_1 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_237_0 in memory on localhost:49158 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_237_1 in memory on localhost:49158 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 349.0 (TID 106). 2647 bytes result sent to driver -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 349.0 (TID 107). 2647 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 349.0 (TID 106) in 12 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 349.0 (TID 107) in 11 ms on localhost (2/2) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 349.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - ResultStage 349 (aggregate at KMeans.scala:402) finished in 0.013 s -2016-06-13 19:15:41 INFO DAGScheduler:59 - Job 33 finished: aggregate at KMeans.scala:402, took 0.035542 s -2016-06-13 19:15:41 INFO MapPartitionsRDD:59 - Removing RDD 234 from persistence list -2016-06-13 19:15:41 INFO BlockManager:59 - Removing RDD 234 -2016-06-13 19:15:41 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Got job 34 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 361(collect at KMeans.scala:431) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 356, ShuffleMapStage 353, ShuffleMapStage 354, ShuffleMapStage 358, ShuffleMapStage 360) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ResultStage 361 (MapPartitionsRDD[239] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(10072) called with curMem=238990, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 9.8 KB, free 730.3 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(4365) called with curMem=249062, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:49158 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 361 (MapPartitionsRDD[239] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 361.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 361.0 (TID 108, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 361.0 (TID 109, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 361.0 (TID 108) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 361.0 (TID 109) -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_237_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_237_1 locally -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 361.0 (TID 109). 3471 bytes result sent to driver -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 361.0 (TID 108). 3118 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 361.0 (TID 109) in 5 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 361.0 (TID 108) in 7 ms on localhost (2/2) -2016-06-13 19:15:41 INFO DAGScheduler:59 - ResultStage 361 (collect at KMeans.scala:431) finished in 0.007 s -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 361.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - Job 34 finished: collect at KMeans.scala:431, took 0.021931 s -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1648) called with curMem=253427, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 1648.0 B, free 730.3 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(600) called with curMem=255075, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 600.0 B, free 730.3 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:49158 (size: 600.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 63 from broadcast at KMeans.scala:394 -2016-06-13 19:15:41 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Got job 35 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 373(aggregate at KMeans.scala:402) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 368, ShuffleMapStage 372, ShuffleMapStage 365, ShuffleMapStage 366, ShuffleMapStage 370) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ResultStage 373 (MapPartitionsRDD[241] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(10144) called with curMem=255675, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 9.9 KB, free 730.3 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(4338) called with curMem=265819, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.3 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:49158 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 373 (MapPartitionsRDD[241] at map at KMeans.scala:396) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 373.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 373.0 (TID 110, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 373.0 (TID 111, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 373.0 (TID 110) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 373.0 (TID 111) -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_241_0 not found, computing it -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_241_1 not found, computing it -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_237_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_237_1 locally -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=270157, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_241_0 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=270653, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_241_1 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_241_0 in memory on localhost:49158 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 373.0 (TID 110). 2647 bytes result sent to driver -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_241_1 in memory on localhost:49158 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 373.0 (TID 111). 2647 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 373.0 (TID 110) in 9 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 373.0 (TID 111) in 9 ms on localhost (2/2) -2016-06-13 19:15:41 INFO DAGScheduler:59 - ResultStage 373 (aggregate at KMeans.scala:402) finished in 0.010 s -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 373.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - Job 35 finished: aggregate at KMeans.scala:402, took 0.027293 s -2016-06-13 19:15:41 INFO MapPartitionsRDD:59 - Removing RDD 237 from persistence list -2016-06-13 19:15:41 INFO BlockManager:59 - Removing RDD 237 -2016-06-13 19:15:41 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Got job 36 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 385(collect at KMeans.scala:431) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 380, ShuffleMapStage 377, ShuffleMapStage 384, ShuffleMapStage 378, ShuffleMapStage 382) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ResultStage 385 (MapPartitionsRDD[243] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(10304) called with curMem=270157, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 10.1 KB, free 730.3 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(4458) called with curMem=280461, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:49158 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 385 (MapPartitionsRDD[243] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 385.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 385.0 (TID 112, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 385.0 (TID 113, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 385.0 (TID 112) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 385.0 (TID 113) -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_241_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_241_1 locally -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 385.0 (TID 112). 2990 bytes result sent to driver -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 385.0 (TID 113). 3240 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 385.0 (TID 112) in 4 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 385.0 (TID 113) in 5 ms on localhost (2/2) -2016-06-13 19:15:41 INFO DAGScheduler:59 - ResultStage 385 (collect at KMeans.scala:431) finished in 0.005 s -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 385.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - Job 36 finished: collect at KMeans.scala:431, took 0.019170 s -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1456) called with curMem=284919, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 1456.0 B, free 730.3 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(582) called with curMem=286375, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 582.0 B, free 730.3 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:49158 (size: 582.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 66 from broadcast at KMeans.scala:394 -2016-06-13 19:15:41 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Got job 37 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 397(aggregate at KMeans.scala:402) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 389, ShuffleMapStage 390, ShuffleMapStage 394, ShuffleMapStage 392, ShuffleMapStage 396) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ResultStage 397 (MapPartitionsRDD[245] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(10376) called with curMem=286957, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 10.1 KB, free 730.3 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(4388) called with curMem=297333, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:49158 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 67 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 397 (MapPartitionsRDD[245] at map at KMeans.scala:396) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 397.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 397.0 (TID 114, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 397.0 (TID 115, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 397.0 (TID 114) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 397.0 (TID 115) -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_245_0 not found, computing it -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_245_1 not found, computing it -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_241_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_241_1 locally -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=301721, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_245_0 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=302217, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_245_1 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_245_0 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 397.0 (TID 114). 2647 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 397.0 (TID 114) in 8 ms on localhost (1/2) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_245_1 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 397.0 (TID 115). 2647 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 397.0 (TID 115) in 9 ms on localhost (2/2) -2016-06-13 19:15:41 INFO DAGScheduler:59 - ResultStage 397 (aggregate at KMeans.scala:402) finished in 0.010 s -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 397.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - Job 37 finished: aggregate at KMeans.scala:402, took 0.022474 s -2016-06-13 19:15:41 INFO MapPartitionsRDD:59 - Removing RDD 241 from persistence list -2016-06-13 19:15:41 INFO BlockManager:59 - Removing RDD 241 -2016-06-13 19:15:41 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Got job 38 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 409(collect at KMeans.scala:431) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 404, ShuffleMapStage 401, ShuffleMapStage 408, ShuffleMapStage 406, ShuffleMapStage 402) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ResultStage 409 (MapPartitionsRDD[247] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(10536) called with curMem=301721, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 10.3 KB, free 730.3 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(4485) called with curMem=312257, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:49158 (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 409 (MapPartitionsRDD[247] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 409.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 409.0 (TID 116, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 409.0 (TID 117, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 409.0 (TID 117) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 409.0 (TID 116) -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_245_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_245_0 locally -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 409.0 (TID 117). 2040 bytes result sent to driver -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 409.0 (TID 116). 2722 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 409.0 (TID 117) in 4 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 409.0 (TID 116) in 5 ms on localhost (2/2) -2016-06-13 19:15:41 INFO DAGScheduler:59 - ResultStage 409 (collect at KMeans.scala:431) finished in 0.006 s -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 409.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - Job 38 finished: collect at KMeans.scala:431, took 0.017714 s -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(648) called with curMem=316742, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 648.0 B, free 730.3 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(429) called with curMem=317390, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 429.0 B, free 730.3 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:49158 (size: 429.0 B, free: 730.4 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 69 from broadcast at KMeans.scala:394 -2016-06-13 19:15:41 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Got job 39 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 421(aggregate at KMeans.scala:402) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 420, ShuffleMapStage 416, ShuffleMapStage 413, ShuffleMapStage 414, ShuffleMapStage 418) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ResultStage 421 (MapPartitionsRDD[249] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(10608) called with curMem=317819, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 10.4 KB, free 730.3 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(4443) called with curMem=328427, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:49158 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 421 (MapPartitionsRDD[249] at map at KMeans.scala:396) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 421.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 421.0 (TID 118, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 421.0 (TID 119, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 421.0 (TID 118) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 421.0 (TID 119) -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_249_0 not found, computing it -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_249_1 not found, computing it -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_245_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_245_1 locally -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=332870, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_249_0 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=333366, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_249_1 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_249_0 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_249_1 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 421.0 (TID 118). 2647 bytes result sent to driver -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 421.0 (TID 119). 2647 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 421.0 (TID 118) in 8 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 421.0 (TID 119) in 8 ms on localhost (2/2) -2016-06-13 19:15:41 INFO DAGScheduler:59 - ResultStage 421 (aggregate at KMeans.scala:402) finished in 0.008 s -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 421.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - Job 39 finished: aggregate at KMeans.scala:402, took 0.025269 s -2016-06-13 19:15:41 INFO MapPartitionsRDD:59 - Removing RDD 245 from persistence list -2016-06-13 19:15:41 INFO BlockManager:59 - Removing RDD 245 -2016-06-13 19:15:41 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Got job 40 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 433(collect at KMeans.scala:431) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 425, ShuffleMapStage 426, ShuffleMapStage 430, ShuffleMapStage 428, ShuffleMapStage 432) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ResultStage 433 (MapPartitionsRDD[251] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(10768) called with curMem=332870, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 10.5 KB, free 730.3 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(4525) called with curMem=343638, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:49158 (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 71 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 433 (MapPartitionsRDD[251] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 433.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 433.0 (TID 120, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 433.0 (TID 121, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 433.0 (TID 121) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 433.0 (TID 120) -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_249_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_249_0 locally -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 433.0 (TID 121). 2040 bytes result sent to driver -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 433.0 (TID 120). 2040 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 433.0 (TID 121) in 4 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 433.0 (TID 120) in 5 ms on localhost (2/2) -2016-06-13 19:15:41 INFO DAGScheduler:59 - ResultStage 433 (collect at KMeans.scala:431) finished in 0.005 s -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 433.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - Job 40 finished: collect at KMeans.scala:431, took 0.018786 s -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(584) called with curMem=348163, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 584.0 B, free 730.3 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(236) called with curMem=348747, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 236.0 B, free 730.3 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:49158 (size: 236.0 B, free: 730.4 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 72 from broadcast at KMeans.scala:394 -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:49158 in memory (size: 3.2 KB, free: 730.4 MB) -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:49158 in memory (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:49158 in memory (size: 2.9 KB, free: 730.4 MB) -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:49158 in memory (size: 3.2 KB, free: 730.4 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-06-13 19:15:41 INFO BlockManager:59 - Removing RDD 194 -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned RDD 194 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Got job 41 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 445(aggregate at KMeans.scala:402) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 440, ShuffleMapStage 437, ShuffleMapStage 444, ShuffleMapStage 438, ShuffleMapStage 442) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:49158 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ResultStage 445 (MapPartitionsRDD[253] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(10848) called with curMem=291584, maxMem=766075207 -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:49158 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 10.6 KB, free 730.3 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(4496) called with curMem=297989, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:49158 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:49158 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 73 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 445 (MapPartitionsRDD[253] at map at KMeans.scala:396) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 445.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 445.0 (TID 122, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 445.0 (TID 123, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 445.0 (TID 123) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 445.0 (TID 122) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:49158 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_253_0 not found, computing it -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_253_1 not found, computing it -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_249_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_249_1 locally -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=262092, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_253_0 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:49158 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=262588, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_253_1 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_253_0 in memory on localhost:49158 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 445.0 (TID 122). 2647 bytes result sent to driver -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_253_1 in memory on localhost:49158 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 445.0 (TID 123). 2647 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 445.0 (TID 122) in 9 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 445.0 (TID 123) in 9 ms on localhost (2/2) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:49158 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO DAGScheduler:59 - ResultStage 445 (aggregate at KMeans.scala:402) finished in 0.010 s -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 445.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - Job 41 finished: aggregate at KMeans.scala:402, took 0.028256 s -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-06-13 19:15:41 INFO MapPartitionsRDD:59 - Removing RDD 249 from persistence list -2016-06-13 19:15:41 INFO BlockManager:59 - Removing RDD 249 -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:49158 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:49158 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:49158 in memory (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:49158 in memory (size: 9.9 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-06-13 19:15:41 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:49158 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Got job 42 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 457(collect at KMeans.scala:431) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 452, ShuffleMapStage 456, ShuffleMapStage 449, ShuffleMapStage 450, ShuffleMapStage 454) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:49158 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-06-13 19:15:41 INFO BlockManager:59 - Removing RDD 217 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ResultStage 457 (MapPartitionsRDD[255] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=143676, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 10.8 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(4575) called with curMem=154684, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned RDD 217 -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:49158 (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 457 (MapPartitionsRDD[255] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 457.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 457.0 (TID 124, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:49158 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 457.0 (TID 125, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 457.0 (TID 125) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 457.0 (TID 124) -2016-06-13 19:15:41 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_253_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_253_0 locally -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 457.0 (TID 125). 2040 bytes result sent to driver -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 457.0 (TID 124). 2040 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 457.0 (TID 125) in 7 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 457.0 (TID 124) in 7 ms on localhost (2/2) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 457.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - ResultStage 457 (collect at KMeans.scala:431) finished in 0.009 s -2016-06-13 19:15:41 INFO DAGScheduler:59 - Job 42 finished: collect at KMeans.scala:431, took 0.030291 s -2016-06-13 19:15:41 INFO MapPartitionsRDD:59 - Removing RDD 253 from persistence list -2016-06-13 19:15:41 INFO BlockManager:59 - Removing RDD 253 -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(2880) called with curMem=143966, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 2.8 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(708) called with curMem=145854, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 708.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:49158 (size: 708.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 75 from broadcast at KMeans.scala:445 -2016-06-13 19:15:41 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:450 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Registering RDD 256 (flatMap at KMeans.scala:446) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Got job 43 (collectAsMap at KMeans.scala:450) with 2 output partitions -2016-06-13 19:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 470(collectAsMap at KMeans.scala:450) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 469) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 469) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ShuffleMapStage 469 (MapPartitionsRDD[256] at flatMap at KMeans.scala:446), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(9392) called with curMem=146562, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 9.2 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(4070) called with curMem=155954, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:49158 (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 76 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 469 (MapPartitionsRDD[256] at flatMap at KMeans.scala:446) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 469.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 469.0 (TID 126, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 469.0 (TID 127, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 469.0 (TID 127) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 469.0 (TID 126) -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 469.0 (TID 126). 2254 bytes result sent to driver -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 469.0 (TID 127). 2254 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 469.0 (TID 126) in 13 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 469.0 (TID 127) in 12 ms on localhost (2/2) -2016-06-13 19:15:41 INFO DAGScheduler:59 - ShuffleMapStage 469 (flatMap at KMeans.scala:446) finished in 0.013 s -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 469.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:41 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:41 INFO DAGScheduler:59 - waiting: Set(ResultStage 470) -2016-06-13 19:15:41 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents for ResultStage 470: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ResultStage 470 (ShuffledRDD[257] at reduceByKey at KMeans.scala:450), which is now runnable -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=160024, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1382) called with curMem=162336, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 1382.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:49158 (size: 1382.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 77 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 470 (ShuffledRDD[257] at reduceByKey at KMeans.scala:450) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 470.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 470.0 (TID 128, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 470.0 (TID 129, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 470.0 (TID 128) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 470.0 (TID 129) -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 470.0 (TID 129). 1744 bytes result sent to driver -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 470.0 (TID 128). 1852 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 470.0 (TID 129) in 6 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 470.0 (TID 128) in 6 ms on localhost (2/2) -2016-06-13 19:15:41 INFO DAGScheduler:59 - ResultStage 470 (collectAsMap at KMeans.scala:450) finished in 0.006 s -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 470.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - Job 43 finished: collectAsMap at KMeans.scala:450, took 0.040824 s -2016-06-13 19:15:41 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:15:41 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:15:41 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:15:41 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:15:41 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:15:41 INFO KMeans:59 - Initialization with k-means|| took 0.552 seconds. -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(824) called with curMem=163718, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 824.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(339) called with curMem=164542, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 339.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:49158 (size: 339.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 78 from broadcast at KMeans.scala:276 -2016-06-13 19:15:41 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:302 -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 151 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 151 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 151 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:15:41 INFO DAGScheduler:59 - Registering RDD 258 (mapPartitions at KMeans.scala:279) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Got job 44 (collectAsMap at KMeans.scala:302) with 2 output partitions -2016-06-13 19:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 483(collectAsMap at KMeans.scala:302) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 482) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 482) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ShuffleMapStage 482 (MapPartitionsRDD[258] at mapPartitions at KMeans.scala:279), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(10016) called with curMem=164881, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 9.8 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(4410) called with curMem=174897, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:49158 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 79 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 482 (MapPartitionsRDD[258] at mapPartitions at KMeans.scala:279) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 482.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 482.0 (TID 130, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 482.0 (TID 131, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 482.0 (TID 130) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 482.0 (TID 131) -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 482.0 (TID 130). 2413 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 482.0 (TID 130) in 12 ms on localhost (1/2) -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 482.0 (TID 131). 2413 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 482.0 (TID 131) in 13 ms on localhost (2/2) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 482.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - ShuffleMapStage 482 (mapPartitions at KMeans.scala:279) finished in 0.014 s -2016-06-13 19:15:41 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:41 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:41 INFO DAGScheduler:59 - waiting: Set(ResultStage 483) -2016-06-13 19:15:41 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents for ResultStage 483: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ResultStage 483 (ShuffledRDD[259] at reduceByKey at KMeans.scala:302), which is now runnable -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(2808) called with curMem=179307, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1589) called with curMem=182115, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 1589.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:49158 (size: 1589.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 483 (ShuffledRDD[259] at reduceByKey at KMeans.scala:302) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 483.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 483.0 (TID 132, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 483.0 (TID 133, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 483.0 (TID 132) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 483.0 (TID 133) -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 483.0 (TID 132). 1785 bytes result sent to driver -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 483.0 (TID 133). 1671 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 483.0 (TID 132) in 6 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 483.0 (TID 133) in 6 ms on localhost (2/2) -2016-06-13 19:15:41 INFO DAGScheduler:59 - ResultStage 483 (collectAsMap at KMeans.scala:302) finished in 0.007 s -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 483.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - Job 44 finished: collectAsMap at KMeans.scala:302, took 0.039459 s -2016-06-13 19:15:41 INFO KMeans:59 - Run 0 finished in 1 iterations -2016-06-13 19:15:41 INFO KMeans:59 - Run 1 finished in 1 iterations -2016-06-13 19:15:41 INFO KMeans:59 - Run 2 finished in 1 iterations -2016-06-13 19:15:41 INFO KMeans:59 - Run 3 finished in 1 iterations -2016-06-13 19:15:41 INFO KMeans:59 - Run 4 finished in 1 iterations -2016-06-13 19:15:41 INFO KMeans:59 - Iterations took 0.057 seconds. -2016-06-13 19:15:41 INFO KMeans:59 - KMeans converged in 1 iterations. -2016-06-13 19:15:41 INFO KMeans:59 - The cost for the best run is 6.650150218788335E-6. -2016-06-13 19:15:41 INFO MapPartitionsRDD:59 - Removing RDD 231 from persistence list -2016-06-13 19:15:41 INFO BlockManager:59 - Removing RDD 231 -2016-06-13 19:15:41 WARN KMeans:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:15:41 INFO SparkContext:59 - Starting job: collect at PowerIterationClusteringSuite.scala:66 -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 151 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 151 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 151 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:15:41 INFO DAGScheduler:59 - Got job 45 (collect at PowerIterationClusteringSuite.scala:66) with 2 output partitions -2016-06-13 19:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 495(collect at PowerIterationClusteringSuite.scala:66) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 494, ShuffleMapStage 488, ShuffleMapStage 492, ShuffleMapStage 490, ShuffleMapStage 487) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ResultStage 495 (MapPartitionsRDD[262] at mapPartitions at PowerIterationClustering.scala:232), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(8192) called with curMem=183224, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 8.0 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(3573) called with curMem=191416, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:49158 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 81 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 495 (MapPartitionsRDD[262] at mapPartitions at PowerIterationClustering.scala:232) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 495.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 495.0 (TID 134, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 495.0 (TID 135, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 495.0 (TID 134) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 495.0 (TID 135) -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_260_0 not found, computing it -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_260_1 not found, computing it -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=194989, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_260_0 stored as values in memory (estimated size 1168.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=196157, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_260_1 stored as values in memory (estimated size 1168.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_260_0 in memory on localhost:49158 (size: 1168.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_260_1 in memory on localhost:49158 (size: 1168.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 495.0 (TID 134). 2919 bytes result sent to driver -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 495.0 (TID 135). 2919 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 495.0 (TID 134) in 7 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 495.0 (TID 135) in 6 ms on localhost (2/2) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 495.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - ResultStage 495 (collect at PowerIterationClusteringSuite.scala:66) finished in 0.007 s -2016-06-13 19:15:41 INFO DAGScheduler:59 - Job 45 finished: collect at PowerIterationClusteringSuite.scala:66, took 0.023177 s -2016-06-13 19:15:41 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:325 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Registering RDD 273 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Registering RDD 267 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Got job 46 (sum at PowerIterationClustering.scala:325) with 2 output partitions -2016-06-13 19:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 498(sum at PowerIterationClustering.scala:325) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 496, ShuffleMapStage 497) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 496, ShuffleMapStage 497) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ShuffleMapStage 496 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[273] at mapPartitions at GraphImpl.scala:235), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(3136) called with curMem=197325, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 3.1 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1868) called with curMem=200461, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 1868.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:49158 (size: 1868.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 496 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[273] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 496.0 with 2 tasks -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ShuffleMapStage 497 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[267] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(2720) called with curMem=202329, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 496.0 (TID 136, localhost, PROCESS_LOCAL, 2808 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 496.0 (TID 137, localhost, PROCESS_LOCAL, 2808 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 496.0 (TID 137) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 496.0 (TID 136) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1642) called with curMem=205049, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 1642.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:49158 (size: 1642.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 83 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_264_1 not found, computing it -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 497 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[267] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 497.0 with 2 tasks -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_264_0 not found, computing it -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(2944) called with curMem=206691, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_264_1 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(2896) called with curMem=209635, maxMem=766075207 -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 497.0 (TID 138, localhost, PROCESS_LOCAL, 2808 bytes) -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_264_0 stored as values in memory (estimated size 2.8 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_264_1 in memory on localhost:49158 (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 497.0 (TID 139, localhost, PROCESS_LOCAL, 2808 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 497.0 (TID 139) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 497.0 (TID 138) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_264_0 in memory on localhost:49158 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_264_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_264_0 locally -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 497.0 (TID 139). 2254 bytes result sent to driver -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 497.0 (TID 138). 2254 bytes result sent to driver -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 496.0 (TID 136). 1739 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 497.0 (TID 139) in 24 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 497.0 (TID 138) in 24 ms on localhost (2/2) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 497.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 496.0 (TID 136) in 31 ms on localhost (1/2) -2016-06-13 19:15:41 INFO DAGScheduler:59 - ShuffleMapStage 497 (mapPartitions at VertexRDD.scala:358) finished in 0.027 s -2016-06-13 19:15:41 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:41 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 496) -2016-06-13 19:15:41 INFO DAGScheduler:59 - waiting: Set(ResultStage 498) -2016-06-13 19:15:41 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 496.0 (TID 137). 1739 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 496.0 (TID 137) in 32 ms on localhost (2/2) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 496.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents for ResultStage 498: List(ShuffleMapStage 496) -2016-06-13 19:15:41 INFO DAGScheduler:59 - ShuffleMapStage 496 (mapPartitions at GraphImpl.scala:235) finished in 0.040 s -2016-06-13 19:15:41 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:41 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:41 INFO DAGScheduler:59 - waiting: Set(ResultStage 498) -2016-06-13 19:15:41 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents for ResultStage 498: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ResultStage 498 (MapPartitionsRDD[285] at map at PowerIterationClustering.scala:325), which is now runnable -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(5136) called with curMem=212531, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 5.0 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(2652) called with curMem=217667, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:49158 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 498 (MapPartitionsRDD[285] at map at PowerIterationClustering.scala:325) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 498.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 498.0 (TID 140, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 498.0 (TID 141, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 498.0 (TID 140) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 498.0 (TID 141) -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_283_1 not found, computing it -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_283_0 not found, computing it -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_275_1 not found, computing it -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_275_0 not found, computing it -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_270_1 not found, computing it -2016-06-13 19:15:41 INFO CacheManager:59 - Partition rdd_270_0 not found, computing it -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1792) called with curMem=220319, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_270_1 stored as values in memory (estimated size 1792.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1784) called with curMem=222111, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_270_0 stored as values in memory (estimated size 1784.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_270_1 in memory on localhost:49158 (size: 1792.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_270_0 in memory on localhost:49158 (size: 1784.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1840) called with curMem=223895, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_275_1 stored as values in memory (estimated size 1840.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=225735, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_275_0 stored as values in memory (estimated size 1832.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_275_1 in memory on localhost:49158 (size: 1840.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_275_0 in memory on localhost:49158 (size: 1832.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(368) called with curMem=227567, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_283_1 stored as values in memory (estimated size 368.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(368) called with curMem=227935, maxMem=766075207 -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_283_1 in memory on localhost:49158 (size: 368.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - Block rdd_283_0 stored as values in memory (estimated size 368.0 B, free 730.4 MB) -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 498.0 (TID 141). 1895 bytes result sent to driver -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added rdd_283_0 in memory on localhost:49158 (size: 368.0 B, free: 730.5 MB) -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 498.0 (TID 140). 1895 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 498.0 (TID 141) in 17 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 498.0 (TID 140) in 18 ms on localhost (2/2) -2016-06-13 19:15:41 INFO DAGScheduler:59 - ResultStage 498 (sum at PowerIterationClustering.scala:325) finished in 0.018 s -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 498.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - Job 46 finished: sum at PowerIterationClustering.scala:325, took 0.080116 s -2016-06-13 19:15:41 INFO SparkContext:59 - Starting job: reduce at VertexRDDImpl.scala:90 -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:15:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:15:41 INFO DAGScheduler:59 - Got job 47 (reduce at VertexRDDImpl.scala:90) with 2 output partitions -2016-06-13 19:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 501(reduce at VertexRDDImpl.scala:90) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 499, ShuffleMapStage 500) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ResultStage 501 (MapPartitionsRDD[289] at map at VertexRDDImpl.scala:90), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(5288) called with curMem=228303, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 5.2 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(2680) called with curMem=233591, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:49158 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 85 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 501 (MapPartitionsRDD[289] at map at VertexRDDImpl.scala:90) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 501.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 501.0 (TID 142, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 501.0 (TID 143, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 501.0 (TID 143) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 501.0 (TID 142) -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_283_1 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_283_0 locally -2016-06-13 19:15:41 INFO Executor:59 - Finished task 1.0 in stage 501.0 (TID 143). 2161 bytes result sent to driver -2016-06-13 19:15:41 INFO Executor:59 - Finished task 0.0 in stage 501.0 (TID 142). 2161 bytes result sent to driver -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 501.0 (TID 143) in 4 ms on localhost (1/2) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 501.0 (TID 142) in 4 ms on localhost (2/2) -2016-06-13 19:15:41 INFO DAGScheduler:59 - ResultStage 501 (reduce at VertexRDDImpl.scala:90) finished in 0.004 s -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 501.0, whose tasks have all completed, from pool -2016-06-13 19:15:41 INFO DAGScheduler:59 - Job 47 finished: reduce at VertexRDDImpl.scala:90, took 0.012465 s -2016-06-13 19:15:41 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Registering RDD 277 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Registering RDD 290 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Registering RDD 294 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Got job 48 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:15:41 INFO DAGScheduler:59 - Final stage: ResultStage 509(sum at PowerIterationClustering.scala:369) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 506, ShuffleMapStage 508, ShuffleMapStage 505) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 508) -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ShuffleMapStage 504 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[277] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=236271, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 4.1 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(2374) called with curMem=240519, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:49158 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 86 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 504 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[277] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 504.0 with 2 tasks -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting ShuffleMapStage 507 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[290] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 504.0 (TID 144, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 504.0 (TID 145, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 504.0 (TID 145) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 504.0 (TID 144) -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(4992) called with curMem=242893, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 4.9 KB, free 730.3 MB) -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_275_0 locally -2016-06-13 19:15:41 INFO BlockManager:59 - Found block rdd_275_1 locally -2016-06-13 19:15:41 INFO MemoryStore:59 - ensureFreeSpace(2606) called with curMem=247885, maxMem=766075207 -2016-06-13 19:15:41 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:15:41 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:49158 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:15:41 INFO SparkContext:59 - Created broadcast 87 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 507 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[290] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:41 INFO TaskSchedulerImpl:59 - Adding task set 507.0 with 2 tasks -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 507.0 (TID 146, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:15:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 507.0 (TID 147, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:15:41 INFO Executor:59 - Running task 1.0 in stage 507.0 (TID 147) -2016-06-13 19:15:41 INFO Executor:59 - Running task 0.0 in stage 507.0 (TID 146) -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_287_1 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_283_1 locally -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=250491, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_287_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_287_0 not found, computing it -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 504.0 (TID 144). 2254 bytes result sent to driver -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_283_0 locally -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=251923, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_287_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_287_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_287_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 504.0 (TID 144) in 20 ms on localhost (1/2) -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 504.0 (TID 145). 2254 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 504.0 (TID 145) in 21 ms on localhost (2/2) -2016-06-13 19:15:42 INFO DAGScheduler:59 - ShuffleMapStage 504 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.021 s -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 504.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:42 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 507) -2016-06-13 19:15:42 INFO DAGScheduler:59 - waiting: Set(ResultStage 509, ShuffleMapStage 508) -2016-06-13 19:15:42 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 507.0 (TID 147). 2834 bytes result sent to driver -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 507.0 (TID 146). 2834 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 507.0 (TID 147) in 15 ms on localhost (1/2) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 507.0 (TID 146) in 15 ms on localhost (2/2) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents for ResultStage 509: List(ShuffleMapStage 508) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 507.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 508: List(ShuffleMapStage 507) -2016-06-13 19:15:42 INFO DAGScheduler:59 - ShuffleMapStage 507 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.019 s -2016-06-13 19:15:42 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:42 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:42 INFO DAGScheduler:59 - waiting: Set(ResultStage 509, ShuffleMapStage 508) -2016-06-13 19:15:42 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents for ResultStage 509: List(ShuffleMapStage 508) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 508: List() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting ShuffleMapStage 508 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[294] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(4552) called with curMem=253355, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(2431) called with curMem=257907, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:49158 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 508 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[294] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Adding task set 508.0 with 2 tasks -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 508.0 (TID 148, localhost, PROCESS_LOCAL, 3098 bytes) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 508.0 (TID 149, localhost, PROCESS_LOCAL, 3098 bytes) -2016-06-13 19:15:42 INFO Executor:59 - Running task 0.0 in stage 508.0 (TID 148) -2016-06-13 19:15:42 INFO Executor:59 - Running task 1.0 in stage 508.0 (TID 149) -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_264_1 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_264_0 locally -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 508.0 (TID 148). 2470 bytes result sent to driver -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 508.0 (TID 149). 2470 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 508.0 (TID 148) in 19 ms on localhost (1/2) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 508.0 (TID 149) in 20 ms on localhost (2/2) -2016-06-13 19:15:42 INFO DAGScheduler:59 - ShuffleMapStage 508 (mapPartitions at GraphImpl.scala:235) finished in 0.021 s -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 508.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:42 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:42 INFO DAGScheduler:59 - waiting: Set(ResultStage 509) -2016-06-13 19:15:42 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents for ResultStage 509: List() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting ResultStage 509 (MapPartitionsRDD[299] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(6024) called with curMem=260338, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 5.9 KB, free 730.3 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(2877) called with curMem=266362, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:49158 (size: 2.8 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 89 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 509 (MapPartitionsRDD[299] at map at PowerIterationClustering.scala:369) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Adding task set 509.0 with 2 tasks -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 509.0 (TID 150, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 509.0 (TID 151, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:15:42 INFO Executor:59 - Running task 0.0 in stage 509.0 (TID 150) -2016-06-13 19:15:42 INFO Executor:59 - Running task 1.0 in stage 509.0 (TID 151) -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_296_1 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_287_1 locally -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_296_0 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_287_0 locally -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=269239, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_296_1 stored as values in memory (estimated size 1480.0 B, free 730.3 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=270719, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_296_0 stored as values in memory (estimated size 1480.0 B, free 730.3 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_296_1 in memory on localhost:49158 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_296_0 in memory on localhost:49158 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 509.0 (TID 151). 2880 bytes result sent to driver -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 509.0 (TID 150). 2880 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 509.0 (TID 151) in 8 ms on localhost (1/2) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 509.0 (TID 150) in 9 ms on localhost (2/2) -2016-06-13 19:15:42 INFO DAGScheduler:59 - ResultStage 509 (sum at PowerIterationClustering.scala:369) finished in 0.009 s -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 509.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO DAGScheduler:59 - Job 48 finished: sum at PowerIterationClustering.scala:369, took 0.083997 s -2016-06-13 19:15:42 INFO PowerIterationClustering:59 - Iteration 0: norm(v) = 34.22334869431644. -2016-06-13 19:15:42 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 151 bytes -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:15:42 INFO DAGScheduler:59 - Got job 49 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:15:42 INFO DAGScheduler:59 - Final stage: ResultStage 517(sum at PowerIterationClustering.scala:375) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 513, ShuffleMapStage 514, ShuffleMapStage 516) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting ResultStage 517 (MapPartitionsRDD[310] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(6760) called with curMem=272199, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 6.6 KB, free 730.3 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(3125) called with curMem=278959, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.3 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:49158 (size: 3.1 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 90 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 517 (MapPartitionsRDD[310] at values at PowerIterationClustering.scala:375) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Adding task set 517.0 with 2 tasks -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 517.0 (TID 152, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 517.0 (TID 153, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:15:42 INFO Executor:59 - Running task 0.0 in stage 517.0 (TID 152) -2016-06-13 19:15:42 INFO Executor:59 - Running task 1.0 in stage 517.0 (TID 153) -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_302_0 not found, computing it -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_302_1 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_287_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_287_1 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_296_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_296_1 locally -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=282084, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_302_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=283516, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_302_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_302_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_302_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 517.0 (TID 152). 2664 bytes result sent to driver -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 517.0 (TID 153). 2664 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 517.0 (TID 152) in 6 ms on localhost (1/2) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 517.0 (TID 153) in 6 ms on localhost (2/2) -2016-06-13 19:15:42 INFO DAGScheduler:59 - ResultStage 517 (sum at PowerIterationClustering.scala:375) finished in 0.007 s -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 517.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO DAGScheduler:59 - Job 49 finished: sum at PowerIterationClustering.scala:375, took 0.017943 s -2016-06-13 19:15:42 INFO PowerIterationClustering:59 - Iteration 0: delta = 1.3216876740804122. -2016-06-13 19:15:42 INFO PowerIterationClustering:59 - Iteration 0: diff(delta) = 1.7976931348623157E308. -2016-06-13 19:15:42 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Registering RDD 313 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Registering RDD 317 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Got job 50 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:15:42 INFO DAGScheduler:59 - Final stage: ResultStage 527(sum at PowerIterationClustering.scala:369) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 524, ShuffleMapStage 521, ShuffleMapStage 526, ShuffleMapStage 522) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 526) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting ShuffleMapStage 525 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[313] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(5984) called with curMem=284948, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 5.8 KB, free 730.3 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(2872) called with curMem=290932, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:49158 (size: 2.8 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 91 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 525 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[313] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Adding task set 525.0 with 2 tasks -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 525.0 (TID 154, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 525.0 (TID 155, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:15:42 INFO Executor:59 - Running task 0.0 in stage 525.0 (TID 154) -2016-06-13 19:15:42 INFO Executor:59 - Running task 1.0 in stage 525.0 (TID 155) -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_311_0 not found, computing it -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_311_1 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_296_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_296_1 locally -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=293804, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_311_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=295236, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_311_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_311_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_311_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 525.0 (TID 154). 2834 bytes result sent to driver -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 525.0 (TID 155). 2834 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 525.0 (TID 155) in 7 ms on localhost (1/2) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 525.0 (TID 154) in 8 ms on localhost (2/2) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 525.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO DAGScheduler:59 - ShuffleMapStage 525 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.008 s -2016-06-13 19:15:42 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:42 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:42 INFO DAGScheduler:59 - waiting: Set(ResultStage 527, ShuffleMapStage 526) -2016-06-13 19:15:42 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents for ResultStage 527: List(ShuffleMapStage 526) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 526: List() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting ShuffleMapStage 526 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[317] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(4552) called with curMem=296668, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(2433) called with curMem=301220, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:49158 (size: 2.4 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 526 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[317] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Adding task set 526.0 with 2 tasks -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 526.0 (TID 156, localhost, PROCESS_LOCAL, 3098 bytes) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 526.0 (TID 157, localhost, PROCESS_LOCAL, 3098 bytes) -2016-06-13 19:15:42 INFO Executor:59 - Running task 1.0 in stage 526.0 (TID 157) -2016-06-13 19:15:42 INFO Executor:59 - Running task 0.0 in stage 526.0 (TID 156) -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_264_0 locally -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_264_1 locally -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 526.0 (TID 156). 2470 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 526.0 (TID 156) in 21 ms on localhost (1/2) -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 526.0 (TID 157). 2470 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 526.0 (TID 157) in 98 ms on localhost (2/2) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 526.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO DAGScheduler:59 - ShuffleMapStage 526 (mapPartitions at GraphImpl.scala:235) finished in 0.100 s -2016-06-13 19:15:42 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:42 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:42 INFO DAGScheduler:59 - waiting: Set(ResultStage 527) -2016-06-13 19:15:42 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents for ResultStage 527: List() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting ResultStage 527 (MapPartitionsRDD[322] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(6920) called with curMem=303653, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 6.8 KB, free 730.3 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(3130) called with curMem=310573, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.3 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:49158 (size: 3.1 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 93 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 527 (MapPartitionsRDD[322] at map at PowerIterationClustering.scala:369) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Adding task set 527.0 with 2 tasks -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 527.0 (TID 158, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 527.0 (TID 159, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:15:42 INFO Executor:59 - Running task 0.0 in stage 527.0 (TID 158) -2016-06-13 19:15:42 INFO Executor:59 - Running task 1.0 in stage 527.0 (TID 159) -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_319_0 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_311_0 locally -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=313703, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_319_0 stored as values in memory (estimated size 1480.0 B, free 730.3 MB) -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_319_1 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_311_1 locally -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=315183, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_319_1 stored as values in memory (estimated size 1480.0 B, free 730.3 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_319_0 in memory on localhost:49158 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_319_1 in memory on localhost:49158 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 527.0 (TID 158). 2880 bytes result sent to driver -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 527.0 (TID 159). 2880 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 527.0 (TID 158) in 17 ms on localhost (1/2) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 527.0 (TID 159) in 16 ms on localhost (2/2) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 527.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO DAGScheduler:59 - ResultStage 527 (sum at PowerIterationClustering.scala:369) finished in 0.018 s -2016-06-13 19:15:42 INFO DAGScheduler:59 - Job 50 finished: sum at PowerIterationClustering.scala:369, took 0.174250 s -2016-06-13 19:15:42 INFO PowerIterationClustering:59 - Iteration 1: norm(v) = 34.22334869431644. -2016-06-13 19:15:42 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 151 bytes -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 151 bytes -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:15:42 INFO DAGScheduler:59 - Got job 51 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:15:42 INFO DAGScheduler:59 - Final stage: ResultStage 537(sum at PowerIterationClustering.scala:375) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 531, ShuffleMapStage 532, ShuffleMapStage 536, ShuffleMapStage 534) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting ResultStage 537 (MapPartitionsRDD[333] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(7352) called with curMem=316663, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 7.2 KB, free 730.3 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(3304) called with curMem=324015, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.3 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:49158 (size: 3.2 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 537 (MapPartitionsRDD[333] at values at PowerIterationClustering.scala:375) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Adding task set 537.0 with 2 tasks -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 537.0 (TID 160, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 537.0 (TID 161, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:15:42 INFO Executor:59 - Running task 0.0 in stage 537.0 (TID 160) -2016-06-13 19:15:42 INFO Executor:59 - Running task 1.0 in stage 537.0 (TID 161) -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_325_1 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_311_1 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_319_1 locally -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=327319, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_325_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_325_0 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_311_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_319_0 locally -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=328751, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_325_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_325_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_325_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 537.0 (TID 161). 2664 bytes result sent to driver -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 537.0 (TID 160). 2664 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 537.0 (TID 161) in 11 ms on localhost (1/2) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 537.0 (TID 160) in 12 ms on localhost (2/2) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 537.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO DAGScheduler:59 - ResultStage 537 (sum at PowerIterationClustering.scala:375) finished in 0.013 s -2016-06-13 19:15:42 INFO DAGScheduler:59 - Job 51 finished: sum at PowerIterationClustering.scala:375, took 0.030446 s -2016-06-13 19:15:42 INFO PowerIterationClustering:59 - Iteration 1: delta = 0.0. -2016-06-13 19:15:42 INFO PowerIterationClustering:59 - Iteration 1: diff(delta) = 1.3216876740804122. -2016-06-13 19:15:42 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Registering RDD 336 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Registering RDD 340 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Got job 52 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:15:42 INFO DAGScheduler:59 - Final stage: ResultStage 549(sum at PowerIterationClustering.scala:369) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 542, ShuffleMapStage 546, ShuffleMapStage 544, ShuffleMapStage 548, ShuffleMapStage 541) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 548) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting ShuffleMapStage 547 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[336] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(6568) called with curMem=330183, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 6.4 KB, free 730.3 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(3034) called with curMem=336751, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.3 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:49158 (size: 3.0 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 95 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 547 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[336] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Adding task set 547.0 with 2 tasks -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 547.0 (TID 162, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 547.0 (TID 163, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:15:42 INFO Executor:59 - Running task 1.0 in stage 547.0 (TID 163) -2016-06-13 19:15:42 INFO Executor:59 - Running task 0.0 in stage 547.0 (TID 162) -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_334_0 not found, computing it -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_334_1 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_319_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_319_1 locally -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=339785, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_334_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=341217, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_334_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_334_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_334_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 547.0 (TID 162). 2834 bytes result sent to driver -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 547.0 (TID 163). 2834 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 547.0 (TID 162) in 9 ms on localhost (1/2) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 547.0 (TID 163) in 8 ms on localhost (2/2) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 547.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO DAGScheduler:59 - ShuffleMapStage 547 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.010 s -2016-06-13 19:15:42 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:42 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:42 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 548, ResultStage 549) -2016-06-13 19:15:42 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 548: List() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents for ResultStage 549: List(ShuffleMapStage 548) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting ShuffleMapStage 548 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[340] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(4552) called with curMem=342649, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(2432) called with curMem=347201, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:49158 (size: 2.4 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 548 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[340] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Adding task set 548.0 with 2 tasks -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 548.0 (TID 164, localhost, PROCESS_LOCAL, 3098 bytes) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 548.0 (TID 165, localhost, PROCESS_LOCAL, 3098 bytes) -2016-06-13 19:15:42 INFO Executor:59 - Running task 0.0 in stage 548.0 (TID 164) -2016-06-13 19:15:42 INFO Executor:59 - Running task 1.0 in stage 548.0 (TID 165) -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_264_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_264_1 locally -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 548.0 (TID 165). 2470 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 548.0 (TID 165) in 41 ms on localhost (1/2) -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 548.0 (TID 164). 2470 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 548.0 (TID 164) in 127 ms on localhost (2/2) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 548.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO DAGScheduler:59 - ShuffleMapStage 548 (mapPartitions at GraphImpl.scala:235) finished in 0.128 s -2016-06-13 19:15:42 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:42 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:42 INFO DAGScheduler:59 - waiting: Set(ResultStage 549) -2016-06-13 19:15:42 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents for ResultStage 549: List() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting ResultStage 549 (MapPartitionsRDD[345] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(7504) called with curMem=349633, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 7.3 KB, free 730.2 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(3289) called with curMem=357137, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.2 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:49158 (size: 3.2 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 97 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 549 (MapPartitionsRDD[345] at map at PowerIterationClustering.scala:369) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Adding task set 549.0 with 2 tasks -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 549.0 (TID 166, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 549.0 (TID 167, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:15:42 INFO Executor:59 - Running task 0.0 in stage 549.0 (TID 166) -2016-06-13 19:15:42 INFO Executor:59 - Running task 1.0 in stage 549.0 (TID 167) -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_342_0 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_334_0 locally -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_342_1 not found, computing it -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_334_1 locally -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=360426, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_342_0 stored as values in memory (estimated size 1480.0 B, free 730.2 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=361906, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_342_1 stored as values in memory (estimated size 1480.0 B, free 730.2 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_342_0 in memory on localhost:49158 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_342_1 in memory on localhost:49158 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 549.0 (TID 166). 2880 bytes result sent to driver -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 549.0 (TID 167). 2880 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 549.0 (TID 166) in 11 ms on localhost (1/2) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 549.0 (TID 167) in 12 ms on localhost (2/2) -2016-06-13 19:15:42 INFO DAGScheduler:59 - ResultStage 549 (sum at PowerIterationClustering.scala:369) finished in 0.012 s -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 549.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO DAGScheduler:59 - Job 52 finished: sum at PowerIterationClustering.scala:369, took 0.195466 s -2016-06-13 19:15:42 INFO PowerIterationClustering:59 - Iteration 2: norm(v) = 34.22334869431644. -2016-06-13 19:15:42 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 151 bytes -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 151 bytes -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 151 bytes -2016-06-13 19:15:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:15:42 INFO DAGScheduler:59 - Got job 53 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:15:42 INFO DAGScheduler:59 - Final stage: ResultStage 561(sum at PowerIterationClustering.scala:375) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 556, ShuffleMapStage 553, ShuffleMapStage 560, ShuffleMapStage 558, ShuffleMapStage 554) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting ResultStage 561 (MapPartitionsRDD[356] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(7936) called with curMem=363386, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 7.8 KB, free 730.2 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(3435) called with curMem=371322, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:49158 (size: 3.4 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 561 (MapPartitionsRDD[356] at values at PowerIterationClustering.scala:375) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Adding task set 561.0 with 2 tasks -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 561.0 (TID 168, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 561.0 (TID 169, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:15:42 INFO Executor:59 - Running task 1.0 in stage 561.0 (TID 169) -2016-06-13 19:15:42 INFO Executor:59 - Running task 0.0 in stage 561.0 (TID 168) -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_348_1 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_334_1 locally -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_348_0 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_342_1 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_334_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_342_0 locally -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=374757, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_348_1 stored as values in memory (estimated size 1432.0 B, free 730.2 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=376189, maxMem=766075207 -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_348_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_348_0 stored as values in memory (estimated size 1432.0 B, free 730.2 MB) -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 561.0 (TID 169). 2664 bytes result sent to driver -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_348_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 561.0 (TID 169) in 5 ms on localhost (1/2) -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 561.0 (TID 168). 2664 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 561.0 (TID 168) in 6 ms on localhost (2/2) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 561.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO DAGScheduler:59 - ResultStage 561 (sum at PowerIterationClustering.scala:375) finished in 0.006 s -2016-06-13 19:15:42 INFO DAGScheduler:59 - Job 53 finished: sum at PowerIterationClustering.scala:375, took 0.023647 s -2016-06-13 19:15:42 INFO PowerIterationClustering:59 - Iteration 2: delta = 0.0. -2016-06-13 19:15:42 INFO PowerIterationClustering:59 - Iteration 2: diff(delta) = 0.0. -2016-06-13 19:15:42 WARN KMeans:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:15:42 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Got job 54 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:15:42 INFO DAGScheduler:59 - Final stage: ResultStage 573(takeSample at KMeans.scala:376) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 572, ShuffleMapStage 568, ShuffleMapStage 565, ShuffleMapStage 566, ShuffleMapStage 570) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting ResultStage 573 (MapPartitionsRDD[364] at map at KMeans.scala:210), which has no missing parents -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(8168) called with curMem=377621, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 8.0 KB, free 730.2 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(3470) called with curMem=385789, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:49158 (size: 3.4 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 99 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 573 (MapPartitionsRDD[364] at map at KMeans.scala:210) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Adding task set 573.0 with 2 tasks -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 573.0 (TID 170, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 573.0 (TID 171, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:15:42 INFO Executor:59 - Running task 1.0 in stage 573.0 (TID 171) -2016-06-13 19:15:42 INFO Executor:59 - Running task 0.0 in stage 573.0 (TID 170) -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_359_1 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_342_1 locally -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_359_0 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_342_0 locally -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=389259, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_359_1 stored as values in memory (estimated size 1432.0 B, free 730.2 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=390691, maxMem=766075207 -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_359_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_359_0 stored as values in memory (estimated size 1432.0 B, free 730.2 MB) -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_362_1 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_359_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=392123, maxMem=766075207 -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_362_0 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_362_1 stored as values in memory (estimated size 240.0 B, free 730.2 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=392363, maxMem=766075207 -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_362_1 in memory on localhost:49158 (size: 240.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_362_0 stored as values in memory (estimated size 240.0 B, free 730.2 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_362_0 in memory on localhost:49158 (size: 240.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 573.0 (TID 171). 2717 bytes result sent to driver -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 573.0 (TID 170). 2717 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 573.0 (TID 171) in 12 ms on localhost (1/2) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 573.0 (TID 170) in 13 ms on localhost (2/2) -2016-06-13 19:15:42 INFO DAGScheduler:59 - ResultStage 573 (takeSample at KMeans.scala:376) finished in 0.013 s -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 573.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO DAGScheduler:59 - Job 54 finished: takeSample at KMeans.scala:376, took 0.028245 s -2016-06-13 19:15:42 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Got job 55 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:15:42 INFO DAGScheduler:59 - Final stage: ResultStage 585(takeSample at KMeans.scala:376) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 578, ShuffleMapStage 582, ShuffleMapStage 580, ShuffleMapStage 584, ShuffleMapStage 577) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting ResultStage 585 (PartitionwiseSampledRDD[366] at takeSample at KMeans.scala:376), which has no missing parents -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(25560) called with curMem=392603, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 25.0 KB, free 730.2 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(10147) called with curMem=418163, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 9.9 KB, free 730.2 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:49158 (size: 9.9 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 100 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 585 (PartitionwiseSampledRDD[366] at takeSample at KMeans.scala:376) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Adding task set 585.0 with 2 tasks -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 585.0 (TID 172, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 585.0 (TID 173, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:15:42 INFO Executor:59 - Running task 0.0 in stage 585.0 (TID 172) -2016-06-13 19:15:42 INFO Executor:59 - Running task 1.0 in stage 585.0 (TID 173) -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 585.0 (TID 173). 2575 bytes result sent to driver -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 585.0 (TID 172). 2633 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 585.0 (TID 172) in 6 ms on localhost (1/2) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 585.0 (TID 173) in 6 ms on localhost (2/2) -2016-06-13 19:15:42 INFO DAGScheduler:59 - ResultStage 585 (takeSample at KMeans.scala:376) finished in 0.007 s -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 585.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO DAGScheduler:59 - Job 55 finished: takeSample at KMeans.scala:376, took 0.020353 s -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=428310, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 880.0 B, free 730.2 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(493) called with curMem=429190, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 493.0 B, free 730.2 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:49158 (size: 493.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 101 from broadcast at KMeans.scala:394 -2016-06-13 19:15:42 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Got job 56 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:15:42 INFO DAGScheduler:59 - Final stage: ResultStage 597(aggregate at KMeans.scala:402) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 592, ShuffleMapStage 589, ShuffleMapStage 596, ShuffleMapStage 590, ShuffleMapStage 594) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting ResultStage 597 (MapPartitionsRDD[368] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(10192) called with curMem=429683, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 10.0 KB, free 730.2 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(4270) called with curMem=439875, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.2 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:49158 (size: 4.2 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 597 (MapPartitionsRDD[368] at map at KMeans.scala:396) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Adding task set 597.0 with 2 tasks -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 597.0 (TID 174, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 597.0 (TID 175, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:15:42 INFO Executor:59 - Running task 0.0 in stage 597.0 (TID 174) -2016-06-13 19:15:42 INFO Executor:59 - Running task 1.0 in stage 597.0 (TID 175) -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_368_0 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_368_1 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=444145, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_368_0 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=444641, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_368_1 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_368_0 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_368_1 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 597.0 (TID 174). 2647 bytes result sent to driver -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 597.0 (TID 175). 2647 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 597.0 (TID 174) in 8 ms on localhost (1/2) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 597.0 (TID 175) in 9 ms on localhost (2/2) -2016-06-13 19:15:42 INFO DAGScheduler:59 - ResultStage 597 (aggregate at KMeans.scala:402) finished in 0.010 s -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 597.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO DAGScheduler:59 - Job 56 finished: aggregate at KMeans.scala:402, took 0.024973 s -2016-06-13 19:15:42 INFO MapPartitionsRDD:59 - Removing RDD 365 from persistence list -2016-06-13 19:15:42 INFO BlockManager:59 - Removing RDD 365 -2016-06-13 19:15:42 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Got job 57 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:15:42 INFO DAGScheduler:59 - Final stage: ResultStage 609(collect at KMeans.scala:431) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 604, ShuffleMapStage 608, ShuffleMapStage 601, ShuffleMapStage 602, ShuffleMapStage 606) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting ResultStage 609 (MapPartitionsRDD[370] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(10240) called with curMem=445137, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 10.0 KB, free 730.2 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(4347) called with curMem=455377, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.1 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:49158 (size: 4.2 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 103 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 609 (MapPartitionsRDD[370] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Adding task set 609.0 with 2 tasks -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 609.0 (TID 176, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 609.0 (TID 177, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:15:42 INFO Executor:59 - Running task 0.0 in stage 609.0 (TID 176) -2016-06-13 19:15:42 INFO Executor:59 - Running task 1.0 in stage 609.0 (TID 177) -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_368_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_368_1 locally -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 609.0 (TID 176). 3244 bytes result sent to driver -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 609.0 (TID 177). 3471 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 609.0 (TID 176) in 6 ms on localhost (1/2) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 609.0 (TID 177) in 5 ms on localhost (2/2) -2016-06-13 19:15:42 INFO DAGScheduler:59 - ResultStage 609 (collect at KMeans.scala:431) finished in 0.006 s -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 609.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO DAGScheduler:59 - Job 57 finished: collect at KMeans.scala:431, took 0.021483 s -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1792) called with curMem=459724, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 1792.0 B, free 730.1 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(636) called with curMem=461516, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 636.0 B, free 730.1 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:49158 (size: 636.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 104 from broadcast at KMeans.scala:394 -2016-06-13 19:15:42 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Got job 58 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:15:42 INFO DAGScheduler:59 - Final stage: ResultStage 621(aggregate at KMeans.scala:402) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 614, ShuffleMapStage 616, ShuffleMapStage 613, ShuffleMapStage 620, ShuffleMapStage 618) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting ResultStage 621 (MapPartitionsRDD[372] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(10424) called with curMem=462152, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 10.2 KB, free 730.1 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(4357) called with curMem=472576, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.1 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:49158 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 105 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 621 (MapPartitionsRDD[372] at map at KMeans.scala:396) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Adding task set 621.0 with 2 tasks -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 621.0 (TID 178, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 621.0 (TID 179, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:15:42 INFO Executor:59 - Running task 0.0 in stage 621.0 (TID 178) -2016-06-13 19:15:42 INFO Executor:59 - Running task 1.0 in stage 621.0 (TID 179) -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_372_1 not found, computing it -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_372_0 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_368_1 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_368_0 locally -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=476933, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_372_1 stored as values in memory (estimated size 496.0 B, free 730.1 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=477429, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_372_0 stored as values in memory (estimated size 496.0 B, free 730.1 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_372_1 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 621.0 (TID 179). 2647 bytes result sent to driver -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_372_0 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 621.0 (TID 178). 2647 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 621.0 (TID 179) in 9 ms on localhost (1/2) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 621.0 (TID 178) in 10 ms on localhost (2/2) -2016-06-13 19:15:42 INFO DAGScheduler:59 - ResultStage 621 (aggregate at KMeans.scala:402) finished in 0.010 s -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 621.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO DAGScheduler:59 - Job 58 finished: aggregate at KMeans.scala:402, took 0.024150 s -2016-06-13 19:15:42 INFO MapPartitionsRDD:59 - Removing RDD 368 from persistence list -2016-06-13 19:15:42 INFO BlockManager:59 - Removing RDD 368 -2016-06-13 19:15:42 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 95 -2016-06-13 19:15:42 INFO BlockManager:59 - Removing RDD 348 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Got job 59 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:15:42 INFO DAGScheduler:59 - Final stage: ResultStage 633(collect at KMeans.scala:431) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 628, ShuffleMapStage 625, ShuffleMapStage 626, ShuffleMapStage 630, ShuffleMapStage 632) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned RDD 348 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting ResultStage 633 (MapPartitionsRDD[374] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(10480) called with curMem=474069, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 10.2 KB, free 730.1 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(4442) called with curMem=473756, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.1 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Removed broadcast_97_piece0 on localhost:49158 in memory (size: 3.2 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 94 -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:49158 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 106 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 633 (MapPartitionsRDD[374] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Adding task set 633.0 with 2 tasks -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 633.0 (TID 180, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Removed broadcast_96_piece0 on localhost:49158 in memory (size: 2.4 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 633.0 (TID 181, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:15:42 INFO Executor:59 - Running task 0.0 in stage 633.0 (TID 180) -2016-06-13 19:15:42 INFO Executor:59 - Running task 1.0 in stage 633.0 (TID 181) -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 93 -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_372_1 locally -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 633.0 (TID 181). 3114 bytes result sent to driver -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_372_0 locally -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 633.0 (TID 180). 2990 bytes result sent to driver -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Removed broadcast_95_piece0 on localhost:49158 in memory (size: 3.0 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 633.0 (TID 181) in 8 ms on localhost (1/2) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 633.0 (TID 180) in 10 ms on localhost (2/2) -2016-06-13 19:15:42 INFO DAGScheduler:59 - ResultStage 633 (collect at KMeans.scala:431) finished in 0.010 s -2016-06-13 19:15:42 INFO DAGScheduler:59 - Job 59 finished: collect at KMeans.scala:431, took 0.033959 s -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 633.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 92 -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(1312) called with curMem=461612, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 1312.0 B, free 730.1 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(579) called with curMem=459620, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 579.0 B, free 730.1 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Removed broadcast_94_piece0 on localhost:49158 in memory (size: 3.2 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:49158 (size: 579.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 107 from broadcast at KMeans.scala:394 -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 91 -2016-06-13 19:15:42 INFO BlockManager:59 - Removing RDD 325 -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned RDD 325 -2016-06-13 19:15:42 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Got job 60 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:15:42 INFO DAGScheduler:59 - Final stage: ResultStage 645(aggregate at KMeans.scala:402) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 640, ShuffleMapStage 644, ShuffleMapStage 637, ShuffleMapStage 638, ShuffleMapStage 642) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Removed broadcast_93_piece0 on localhost:49158 in memory (size: 3.1 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 90 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting ResultStage 645 (MapPartitionsRDD[376] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(10656) called with curMem=439933, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 10.4 KB, free 730.2 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(4410) called with curMem=450589, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.2 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:49158 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 108 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 645 (MapPartitionsRDD[376] at map at KMeans.scala:396) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Adding task set 645.0 with 2 tasks -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Removed broadcast_92_piece0 on localhost:49158 in memory (size: 2.4 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 645.0 (TID 182, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 89 -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 645.0 (TID 183, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:15:42 INFO Executor:59 - Running task 0.0 in stage 645.0 (TID 182) -2016-06-13 19:15:42 INFO Executor:59 - Running task 1.0 in stage 645.0 (TID 183) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Removed broadcast_91_piece0 on localhost:49158 in memory (size: 2.8 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_376_0 not found, computing it -2016-06-13 19:15:42 INFO CacheManager:59 - Partition rdd_376_1 not found, computing it -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 88 -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_372_1 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_372_0 locally -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=429273, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_376_1 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Removed broadcast_90_piece0 on localhost:49158 in memory (size: 3.1 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=429769, maxMem=766075207 -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 87 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block rdd_376_0 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_376_1 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 645.0 (TID 183). 2647 bytes result sent to driver -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added rdd_376_0 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 645.0 (TID 183) in 11 ms on localhost (1/2) -2016-06-13 19:15:42 INFO BlockManager:59 - Removing RDD 302 -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 645.0 (TID 182). 2647 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 645.0 (TID 182) in 14 ms on localhost (2/2) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 645.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO DAGScheduler:59 - ResultStage 645 (aggregate at KMeans.scala:402) finished in 0.015 s -2016-06-13 19:15:42 INFO DAGScheduler:59 - Job 60 finished: aggregate at KMeans.scala:402, took 0.040423 s -2016-06-13 19:15:42 INFO MapPartitionsRDD:59 - Removing RDD 372 from persistence list -2016-06-13 19:15:42 INFO BlockManager:59 - Removing RDD 372 -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned RDD 302 -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Removed broadcast_89_piece0 on localhost:49158 in memory (size: 2.8 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 86 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Got job 61 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:15:42 INFO DAGScheduler:59 - Final stage: ResultStage 657(collect at KMeans.scala:431) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 652, ShuffleMapStage 649, ShuffleMapStage 656, ShuffleMapStage 650, ShuffleMapStage 654) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Removed broadcast_88_piece0 on localhost:49158 in memory (size: 2.4 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 85 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting ResultStage 657 (MapPartitionsRDD[378] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Removed broadcast_87_piece0 on localhost:49158 in memory (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(10712) called with curMem=402927, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 10.5 KB, free 730.2 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(4479) called with curMem=413639, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.2 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:49158 (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 109 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 657 (MapPartitionsRDD[378] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Adding task set 657.0 with 2 tasks -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 84 -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 657.0 (TID 184, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 657.0 (TID 185, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:15:42 INFO Executor:59 - Running task 0.0 in stage 657.0 (TID 184) -2016-06-13 19:15:42 INFO Executor:59 - Running task 1.0 in stage 657.0 (TID 185) -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:49158 in memory (size: 2.3 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_376_0 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:15:42 INFO Executor:59 - Finished task 0.0 in stage 657.0 (TID 184). 2040 bytes result sent to driver -2016-06-13 19:15:42 INFO BlockManager:59 - Found block rdd_376_1 locally -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 83 -2016-06-13 19:15:42 INFO Executor:59 - Finished task 1.0 in stage 657.0 (TID 185). 2722 bytes result sent to driver -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 657.0 (TID 184) in 8 ms on localhost (1/2) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 657.0 (TID 185) in 8 ms on localhost (2/2) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 657.0, whose tasks have all completed, from pool -2016-06-13 19:15:42 INFO DAGScheduler:59 - ResultStage 657 (collect at KMeans.scala:431) finished in 0.009 s -2016-06-13 19:15:42 INFO DAGScheduler:59 - Job 61 finished: collect at KMeans.scala:431, took 0.031872 s -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:49158 in memory (size: 2.6 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(648) called with curMem=403528, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 648.0 B, free 730.2 MB) -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 82 -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(429) called with curMem=404176, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 429.0 B, free 730.2 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:49158 in memory (size: 2.6 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 81 -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:49158 (size: 429.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 110 from broadcast at KMeans.scala:394 -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:49158 in memory (size: 1642.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 80 -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:49158 in memory (size: 1868.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 79 -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:49158 in memory (size: 3.5 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 78 -2016-06-13 19:15:42 INFO BlockManager:59 - Removing RDD 260 -2016-06-13 19:15:42 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned RDD 260 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Got job 62 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:49158 in memory (size: 1589.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Final stage: ResultStage 669(aggregate at KMeans.scala:402) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 661, ShuffleMapStage 662, ShuffleMapStage 666, ShuffleMapStage 664, ShuffleMapStage 668) -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 77 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:49158 in memory (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting ResultStage 669 (MapPartitionsRDD[380] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 76 -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned shuffle 21 -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(10888) called with curMem=354527, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 10.6 KB, free 730.2 MB) -2016-06-13 19:15:42 INFO MemoryStore:59 - ensureFreeSpace(4476) called with curMem=364252, maxMem=766075207 -2016-06-13 19:15:42 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.2 MB) -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:49158 in memory (size: 339.0 B, free: 730.4 MB) -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 75 -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 74 -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 73 -2016-06-13 19:15:42 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:49158 (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 72 -2016-06-13 19:15:42 INFO ContextCleaner:59 - Cleaned accumulator 71 -2016-06-13 19:15:42 INFO SparkContext:59 - Created broadcast 111 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 669 (MapPartitionsRDD[380] at map at KMeans.scala:396) -2016-06-13 19:15:42 INFO TaskSchedulerImpl:59 - Adding task set 669.0 with 2 tasks -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 669.0 (TID 186, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:15:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 669.0 (TID 187, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 669.0 (TID 186) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 669.0 (TID 187) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:49158 in memory (size: 1382.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 70 -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:49158 in memory (size: 4.0 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_380_0 not found, computing it -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_380_1 not found, computing it -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 69 -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_376_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_376_1 locally -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned shuffle 20 -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=350864, maxMem=766075207 -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:49158 in memory (size: 708.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_380_1 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=348480, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_380_0 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_380_1 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 669.0 (TID 187). 2647 bytes result sent to driver -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_380_0 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 669.0 (TID 186). 2647 bytes result sent to driver -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:49158 in memory (size: 4.5 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 669.0 (TID 187) in 14 ms on localhost (1/2) -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 669.0 (TID 186) in 16 ms on localhost (2/2) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 669.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - ResultStage 669 (aggregate at KMeans.scala:402) finished in 0.016 s -2016-06-13 19:15:43 INFO DAGScheduler:59 - Job 62 finished: aggregate at KMeans.scala:402, took 0.032926 s -2016-06-13 19:15:43 INFO MapPartitionsRDD:59 - Removing RDD 376 from persistence list -2016-06-13 19:15:43 INFO BlockManager:59 - Removing RDD 376 -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:49158 in memory (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-06-13 19:15:43 INFO BlockManager:59 - Removing RDD 253 -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned RDD 253 -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:49158 in memory (size: 236.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_105_piece0 on localhost:49158 in memory (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 100 -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_103_piece0 on localhost:49158 in memory (size: 4.2 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 99 -2016-06-13 19:15:43 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_102_piece0 on localhost:49158 in memory (size: 4.2 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 98 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Got job 63 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:15:43 INFO DAGScheduler:59 - Final stage: ResultStage 681(collect at KMeans.scala:431) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 676, ShuffleMapStage 673, ShuffleMapStage 680, ShuffleMapStage 678, ShuffleMapStage 674) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_100_piece0 on localhost:49158 in memory (size: 9.9 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 97 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ResultStage 681 (MapPartitionsRDD[382] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_99_piece0 on localhost:49158 in memory (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 96 -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(10944) called with curMem=221627, maxMem=766075207 -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_98_piece0 on localhost:49158 in memory (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 10.7 KB, free 730.4 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(4537) called with curMem=224635, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:49158 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 112 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 681 (MapPartitionsRDD[382] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 681.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 681.0 (TID 188, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 681.0 (TID 189, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 681.0 (TID 188) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 681.0 (TID 189) -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_380_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_380_1 locally -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 681.0 (TID 188). 2722 bytes result sent to driver -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 681.0 (TID 189). 2040 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 681.0 (TID 188) in 4 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 681.0 (TID 189) in 5 ms on localhost (2/2) -2016-06-13 19:15:43 INFO DAGScheduler:59 - ResultStage 681 (collect at KMeans.scala:431) finished in 0.005 s -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 681.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - Job 63 finished: collect at KMeans.scala:431, took 0.019950 s -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(648) called with curMem=229172, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 648.0 B, free 730.4 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(429) called with curMem=229820, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 429.0 B, free 730.4 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:49158 (size: 429.0 B, free: 730.5 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 113 from broadcast at KMeans.scala:394 -2016-06-13 19:15:43 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Got job 64 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:15:43 INFO DAGScheduler:59 - Final stage: ResultStage 693(aggregate at KMeans.scala:402) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 692, ShuffleMapStage 688, ShuffleMapStage 685, ShuffleMapStage 686, ShuffleMapStage 690) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ResultStage 693 (MapPartitionsRDD[384] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(11128) called with curMem=230249, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 10.9 KB, free 730.4 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(4537) called with curMem=241377, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:49158 (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 114 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 693 (MapPartitionsRDD[384] at map at KMeans.scala:396) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 693.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 693.0 (TID 190, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 693.0 (TID 191, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 693.0 (TID 191) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 693.0 (TID 190) -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_384_0 not found, computing it -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_384_1 not found, computing it -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_380_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_380_1 locally -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=245914, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_384_0 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=246410, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_384_1 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_384_0 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_384_1 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 693.0 (TID 190). 2647 bytes result sent to driver -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 693.0 (TID 191). 2647 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 693.0 (TID 190) in 8 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 693.0 (TID 191) in 8 ms on localhost (2/2) -2016-06-13 19:15:43 INFO DAGScheduler:59 - ResultStage 693 (aggregate at KMeans.scala:402) finished in 0.008 s -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 693.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - Job 64 finished: aggregate at KMeans.scala:402, took 0.020690 s -2016-06-13 19:15:43 INFO MapPartitionsRDD:59 - Removing RDD 380 from persistence list -2016-06-13 19:15:43 INFO BlockManager:59 - Removing RDD 380 -2016-06-13 19:15:43 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Got job 65 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:15:43 INFO DAGScheduler:59 - Final stage: ResultStage 705(collect at KMeans.scala:431) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 698, ShuffleMapStage 702, ShuffleMapStage 700, ShuffleMapStage 697, ShuffleMapStage 704) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ResultStage 705 (MapPartitionsRDD[386] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(11176) called with curMem=245914, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_115 stored as values in memory (estimated size 10.9 KB, free 730.3 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(4569) called with curMem=257090, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_115_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.3 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_115_piece0 in memory on localhost:49158 (size: 4.5 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 115 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 705 (MapPartitionsRDD[386] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 705.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 705.0 (TID 192, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 705.0 (TID 193, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 705.0 (TID 192) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 705.0 (TID 193) -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_384_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_384_0 locally -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 705.0 (TID 193). 2040 bytes result sent to driver -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 705.0 (TID 192). 2040 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 705.0 (TID 193) in 5 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 705.0 (TID 192) in 5 ms on localhost (2/2) -2016-06-13 19:15:43 INFO DAGScheduler:59 - ResultStage 705 (collect at KMeans.scala:431) finished in 0.007 s -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 705.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - Job 65 finished: collect at KMeans.scala:431, took 0.017961 s -2016-06-13 19:15:43 INFO MapPartitionsRDD:59 - Removing RDD 384 from persistence list -2016-06-13 19:15:43 INFO BlockManager:59 - Removing RDD 384 -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(2944) called with curMem=261659, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_116 stored as values in memory (estimated size 2.9 KB, free 730.3 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(787) called with curMem=263611, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_116_piece0 stored as bytes in memory (estimated size 787.0 B, free 730.3 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_116_piece0 in memory on localhost:49158 (size: 787.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 116 from broadcast at KMeans.scala:445 -2016-06-13 19:15:43 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:450 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Registering RDD 387 (flatMap at KMeans.scala:446) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Got job 66 (collectAsMap at KMeans.scala:450) with 2 output partitions -2016-06-13 19:15:43 INFO DAGScheduler:59 - Final stage: ResultStage 718(collectAsMap at KMeans.scala:450) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 717) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 717) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 717 (MapPartitionsRDD[387] at flatMap at KMeans.scala:446), which has no missing parents -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(9672) called with curMem=264398, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_117 stored as values in memory (estimated size 9.4 KB, free 730.3 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(4104) called with curMem=274070, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_117_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_117_piece0 in memory on localhost:49158 (size: 4.0 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 117 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 717 (MapPartitionsRDD[387] at flatMap at KMeans.scala:446) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 717.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 717.0 (TID 194, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 717.0 (TID 195, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 717.0 (TID 194) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 717.0 (TID 195) -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 717.0 (TID 195). 2254 bytes result sent to driver -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 717.0 (TID 194). 2254 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 717.0 (TID 195) in 13 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 717.0 (TID 194) in 14 ms on localhost (2/2) -2016-06-13 19:15:43 INFO DAGScheduler:59 - ShuffleMapStage 717 (flatMap at KMeans.scala:446) finished in 0.014 s -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 717.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:43 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:43 INFO DAGScheduler:59 - waiting: Set(ResultStage 718) -2016-06-13 19:15:43 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents for ResultStage 718: List() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ResultStage 718 (ShuffledRDD[388] at reduceByKey at KMeans.scala:450), which is now runnable -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=278174, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_118 stored as values in memory (estimated size 2.3 KB, free 730.3 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1383) called with curMem=280486, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_118_piece0 stored as bytes in memory (estimated size 1383.0 B, free 730.3 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_118_piece0 in memory on localhost:49158 (size: 1383.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 118 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 718 (ShuffledRDD[388] at reduceByKey at KMeans.scala:450) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 718.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 718.0 (TID 196, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 718.0 (TID 197, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 718.0 (TID 196) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 718.0 (TID 197) -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 718.0 (TID 197). 2032 bytes result sent to driver -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 718.0 (TID 196). 1924 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 718.0 (TID 197) in 5 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 718.0 (TID 196) in 7 ms on localhost (2/2) -2016-06-13 19:15:43 INFO DAGScheduler:59 - ResultStage 718 (collectAsMap at KMeans.scala:450) finished in 0.007 s -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 718.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - Job 66 finished: collectAsMap at KMeans.scala:450, took 0.040844 s -2016-06-13 19:15:43 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:15:43 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:15:43 INFO LocalKMeans:59 - Local KMeans++ converged in 3 iterations. -2016-06-13 19:15:43 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:15:43 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:15:43 INFO KMeans:59 - Initialization with k-means|| took 0.553 seconds. -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(824) called with curMem=281869, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_119 stored as values in memory (estimated size 824.0 B, free 730.3 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(363) called with curMem=282693, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_119_piece0 stored as bytes in memory (estimated size 363.0 B, free 730.3 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_119_piece0 in memory on localhost:49158 (size: 363.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 119 from broadcast at KMeans.scala:276 -2016-06-13 19:15:43 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:302 -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 151 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 151 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 151 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:15:43 INFO DAGScheduler:59 - Registering RDD 389 (mapPartitions at KMeans.scala:279) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Got job 67 (collectAsMap at KMeans.scala:302) with 2 output partitions -2016-06-13 19:15:43 INFO DAGScheduler:59 - Final stage: ResultStage 731(collectAsMap at KMeans.scala:302) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 730) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 730) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 730 (MapPartitionsRDD[389] at mapPartitions at KMeans.scala:279), which has no missing parents -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(10296) called with curMem=283056, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_120 stored as values in memory (estimated size 10.1 KB, free 730.3 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(4440) called with curMem=293352, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_120_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_120_piece0 in memory on localhost:49158 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 120 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 730 (MapPartitionsRDD[389] at mapPartitions at KMeans.scala:279) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 730.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 730.0 (TID 198, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 730.0 (TID 199, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 730.0 (TID 198) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 730.0 (TID 199) -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_362_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_362_1 locally -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 730.0 (TID 198). 2413 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 730.0 (TID 198) in 11 ms on localhost (1/2) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 730.0 (TID 199). 2413 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 730.0 (TID 199) in 13 ms on localhost (2/2) -2016-06-13 19:15:43 INFO DAGScheduler:59 - ShuffleMapStage 730 (mapPartitions at KMeans.scala:279) finished in 0.015 s -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 730.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:43 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:43 INFO DAGScheduler:59 - waiting: Set(ResultStage 731) -2016-06-13 19:15:43 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents for ResultStage 731: List() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ResultStage 731 (ShuffledRDD[390] at reduceByKey at KMeans.scala:302), which is now runnable -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(2808) called with curMem=297792, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_121 stored as values in memory (estimated size 2.7 KB, free 730.3 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1593) called with curMem=300600, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_121_piece0 stored as bytes in memory (estimated size 1593.0 B, free 730.3 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_121_piece0 in memory on localhost:49158 (size: 1593.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 121 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 731 (ShuffledRDD[390] at reduceByKey at KMeans.scala:302) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 731.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 731.0 (TID 200, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 731.0 (TID 201, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 731.0 (TID 200) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 731.0 (TID 201) -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 731.0 (TID 201). 1689 bytes result sent to driver -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 731.0 (TID 200). 1803 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 731.0 (TID 201) in 8 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 731.0 (TID 200) in 9 ms on localhost (2/2) -2016-06-13 19:15:43 INFO DAGScheduler:59 - ResultStage 731 (collectAsMap at KMeans.scala:302) finished in 0.009 s -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 731.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - Job 67 finished: collectAsMap at KMeans.scala:302, took 0.043430 s -2016-06-13 19:15:43 INFO KMeans:59 - Run 0 finished in 1 iterations -2016-06-13 19:15:43 INFO KMeans:59 - Run 1 finished in 1 iterations -2016-06-13 19:15:43 INFO KMeans:59 - Run 2 finished in 1 iterations -2016-06-13 19:15:43 INFO KMeans:59 - Run 3 finished in 1 iterations -2016-06-13 19:15:43 INFO KMeans:59 - Run 4 finished in 1 iterations -2016-06-13 19:15:43 INFO KMeans:59 - Iterations took 0.059 seconds. -2016-06-13 19:15:43 INFO KMeans:59 - KMeans converged in 1 iterations. -2016-06-13 19:15:43 INFO KMeans:59 - The cost for the best run is 0.0013884340729950003. -2016-06-13 19:15:43 INFO MapPartitionsRDD:59 - Removing RDD 362 from persistence list -2016-06-13 19:15:43 INFO BlockManager:59 - Removing RDD 362 -2016-06-13 19:15:43 WARN KMeans:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:15:43 INFO SparkContext:59 - Starting job: collect at PowerIterationClusteringSuite.scala:104 -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 151 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 151 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 151 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:15:43 INFO DAGScheduler:59 - Got job 68 (collect at PowerIterationClusteringSuite.scala:104) with 2 output partitions -2016-06-13 19:15:43 INFO DAGScheduler:59 - Final stage: ResultStage 743(collect at PowerIterationClusteringSuite.scala:104) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 742, ShuffleMapStage 735, ShuffleMapStage 736, ShuffleMapStage 740, ShuffleMapStage 738) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ResultStage 743 (MapPartitionsRDD[393] at mapPartitions at PowerIterationClustering.scala:232), which has no missing parents -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(8472) called with curMem=301713, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_122 stored as values in memory (estimated size 8.3 KB, free 730.3 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(3625) called with curMem=310185, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_122_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_122_piece0 in memory on localhost:49158 (size: 3.5 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 122 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 743 (MapPartitionsRDD[393] at mapPartitions at PowerIterationClustering.scala:232) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 743.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 743.0 (TID 202, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 743.0 (TID 203, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 743.0 (TID 202) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 743.0 (TID 203) -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_391_0 not found, computing it -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_391_1 not found, computing it -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_359_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_359_1 locally -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=313810, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_391_0 stored as values in memory (estimated size 1168.0 B, free 730.3 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=314978, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_391_1 stored as values in memory (estimated size 1168.0 B, free 730.3 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_391_0 in memory on localhost:49158 (size: 1168.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_391_1 in memory on localhost:49158 (size: 1168.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 743.0 (TID 202). 2919 bytes result sent to driver -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 743.0 (TID 203). 2919 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 743.0 (TID 202) in 6 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 743.0 (TID 203) in 6 ms on localhost (2/2) -2016-06-13 19:15:43 INFO DAGScheduler:59 - ResultStage 743 (collect at PowerIterationClusteringSuite.scala:104) finished in 0.006 s -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 743.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - Job 68 finished: collect at PowerIterationClusteringSuite.scala:104, took 0.018149 s -2016-06-13 19:15:43 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:340 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Registering RDD 405 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Registering RDD 399 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Got job 69 (sum at PowerIterationClustering.scala:340) with 2 output partitions -2016-06-13 19:15:43 INFO DAGScheduler:59 - Final stage: ResultStage 746(sum at PowerIterationClustering.scala:340) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 745, ShuffleMapStage 744) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 745, ShuffleMapStage 744) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 744 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[405] at mapPartitions at GraphImpl.scala:235), which has no missing parents -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(3432) called with curMem=316146, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_123 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1974) called with curMem=319578, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_123_piece0 stored as bytes in memory (estimated size 1974.0 B, free 730.3 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_123_piece0 in memory on localhost:49158 (size: 1974.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 123 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 744 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[405] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 744.0 with 2 tasks -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 745 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[399] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 744.0 (TID 204, localhost, PROCESS_LOCAL, 2488 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 744.0 (TID 205, localhost, PROCESS_LOCAL, 2563 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 744.0 (TID 204) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 744.0 (TID 205) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(3016) called with curMem=321552, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_124 stored as values in memory (estimated size 2.9 KB, free 730.3 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1779) called with curMem=324568, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_124_piece0 stored as bytes in memory (estimated size 1779.0 B, free 730.3 MB) -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_396_0 not found, computing it -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_396_1 not found, computing it -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_124_piece0 in memory on localhost:49158 (size: 1779.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 124 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(2952) called with curMem=326347, maxMem=766075207 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 745 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[399] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_396_1 stored as values in memory (estimated size 2.9 KB, free 730.3 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(2856) called with curMem=329299, maxMem=766075207 -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 745.0 with 2 tasks -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_396_0 stored as values in memory (estimated size 2.8 KB, free 730.3 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_396_1 in memory on localhost:49158 (size: 2.9 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_396_0 in memory on localhost:49158 (size: 2.8 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 745.0 (TID 206, localhost, PROCESS_LOCAL, 2488 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 745.0 (TID 207, localhost, PROCESS_LOCAL, 2563 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 745.0 (TID 207) -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_396_1 locally -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 745.0 (TID 206) -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_396_0 locally -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 744.0 (TID 204). 1739 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 744.0 (TID 204) in 24 ms on localhost (1/2) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 744.0 (TID 205). 1739 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 744.0 (TID 205) in 28 ms on localhost (2/2) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 744.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - ShuffleMapStage 744 (mapPartitions at GraphImpl.scala:235) finished in 0.028 s -2016-06-13 19:15:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:43 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 745) -2016-06-13 19:15:43 INFO DAGScheduler:59 - waiting: Set(ResultStage 746) -2016-06-13 19:15:43 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents for ResultStage 746: List(ShuffleMapStage 745) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 745.0 (TID 207). 2254 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 745.0 (TID 207) in 33 ms on localhost (1/2) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 745.0 (TID 206). 2254 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 745.0 (TID 206) in 38 ms on localhost (2/2) -2016-06-13 19:15:43 INFO DAGScheduler:59 - ShuffleMapStage 745 (mapPartitions at VertexRDD.scala:358) finished in 0.039 s -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 745.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:43 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:43 INFO DAGScheduler:59 - waiting: Set(ResultStage 746) -2016-06-13 19:15:43 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents for ResultStage 746: List() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ResultStage 746 (MapPartitionsRDD[415] at values at PowerIterationClustering.scala:340), which is now runnable -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(4728) called with curMem=332155, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_125 stored as values in memory (estimated size 4.6 KB, free 730.3 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(2556) called with curMem=336883, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_125_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_125_piece0 in memory on localhost:49158 (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 125 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 746 (MapPartitionsRDD[415] at values at PowerIterationClustering.scala:340) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 746.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 746.0 (TID 208, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 746.0 (TID 209, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 746.0 (TID 208) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 746.0 (TID 209) -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_407_0 not found, computing it -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_407_1 not found, computing it -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_402_0 not found, computing it -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_402_1 not found, computing it -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1776) called with curMem=339439, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_402_0 stored as values in memory (estimated size 1776.0 B, free 730.3 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1792) called with curMem=341215, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_402_1 stored as values in memory (estimated size 1792.0 B, free 730.3 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_402_0 in memory on localhost:49158 (size: 1776.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_402_1 in memory on localhost:49158 (size: 1792.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1824) called with curMem=343007, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_407_0 stored as values in memory (estimated size 1824.0 B, free 730.3 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1840) called with curMem=344831, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_407_1 stored as values in memory (estimated size 1840.0 B, free 730.3 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_407_0 in memory on localhost:49158 (size: 1824.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_407_1 in memory on localhost:49158 (size: 1840.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 746.0 (TID 208). 1840 bytes result sent to driver -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 746.0 (TID 209). 1840 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 746.0 (TID 208) in 13 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 746.0 (TID 209) in 12 ms on localhost (2/2) -2016-06-13 19:15:43 INFO DAGScheduler:59 - ResultStage 746 (sum at PowerIterationClustering.scala:340) finished in 0.013 s -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 746.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - Job 69 finished: sum at PowerIterationClustering.scala:340, took 0.071857 s -2016-06-13 19:15:43 INFO SparkContext:59 - Starting job: reduce at VertexRDDImpl.scala:90 -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:15:43 INFO DAGScheduler:59 - Got job 70 (reduce at VertexRDDImpl.scala:90) with 2 output partitions -2016-06-13 19:15:43 INFO DAGScheduler:59 - Final stage: ResultStage 749(reduce at VertexRDDImpl.scala:90) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 748, ShuffleMapStage 747) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ResultStage 749 (MapPartitionsRDD[420] at map at VertexRDDImpl.scala:90), which has no missing parents -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(5200) called with curMem=346671, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_126 stored as values in memory (estimated size 5.1 KB, free 730.3 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(2659) called with curMem=351871, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_126_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_126_piece0 in memory on localhost:49158 (size: 2.6 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 126 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 749 (MapPartitionsRDD[420] at map at VertexRDDImpl.scala:90) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 749.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 749.0 (TID 210, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 749.0 (TID 211, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 749.0 (TID 210) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 749.0 (TID 211) -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_407_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_407_0 locally -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 749.0 (TID 210). 2161 bytes result sent to driver -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 749.0 (TID 211). 2161 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 749.0 (TID 210) in 3 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 749.0 (TID 211) in 4 ms on localhost (2/2) -2016-06-13 19:15:43 INFO DAGScheduler:59 - ResultStage 749 (reduce at VertexRDDImpl.scala:90) finished in 0.005 s -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 749.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - Job 70 finished: reduce at VertexRDDImpl.scala:90, took 0.011890 s -2016-06-13 19:15:43 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Registering RDD 409 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Registering RDD 421 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Registering RDD 425 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Got job 71 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:15:43 INFO DAGScheduler:59 - Final stage: ResultStage 757(sum at PowerIterationClustering.scala:369) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 753, ShuffleMapStage 754, ShuffleMapStage 756) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 756) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 752 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[409] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=354530, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_127 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(2379) called with curMem=358778, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_127_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_127_piece0 in memory on localhost:49158 (size: 2.3 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 127 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 752 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[409] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 752.0 with 2 tasks -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 755 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[421] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 752.0 (TID 212, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 752.0 (TID 213, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 752.0 (TID 212) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 752.0 (TID 213) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(4904) called with curMem=361157, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_128 stored as values in memory (estimated size 4.8 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_407_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_407_0 locally -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(2574) called with curMem=366061, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_128_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_128_piece0 in memory on localhost:49158 (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 128 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 755 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[421] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 755.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 755.0 (TID 214, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 755.0 (TID 215, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 755.0 (TID 215) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 755.0 (TID 214) -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_418_1 not found, computing it -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_407_1 locally -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_418_0 not found, computing it -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_407_0 locally -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=368635, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_418_1 stored as values in memory (estimated size 1432.0 B, free 730.2 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=370067, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_418_0 stored as values in memory (estimated size 1432.0 B, free 730.2 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_418_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 752.0 (TID 213). 2254 bytes result sent to driver -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_418_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 752.0 (TID 212). 2254 bytes result sent to driver -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 755.0 (TID 215). 2834 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 752.0 (TID 213) in 19 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 752.0 (TID 212) in 20 ms on localhost (2/2) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 752.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - ShuffleMapStage 752 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.021 s -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 755.0 (TID 215) in 13 ms on localhost (1/2) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 755.0 (TID 214). 2834 bytes result sent to driver -2016-06-13 19:15:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:43 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 755) -2016-06-13 19:15:43 INFO DAGScheduler:59 - waiting: Set(ResultStage 757, ShuffleMapStage 756) -2016-06-13 19:15:43 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 755.0 (TID 214) in 15 ms on localhost (2/2) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 755.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents for ResultStage 757: List(ShuffleMapStage 756) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 756: List(ShuffleMapStage 755) -2016-06-13 19:15:43 INFO DAGScheduler:59 - ShuffleMapStage 755 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.017 s -2016-06-13 19:15:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:43 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:43 INFO DAGScheduler:59 - waiting: Set(ResultStage 757, ShuffleMapStage 756) -2016-06-13 19:15:43 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents for ResultStage 757: List(ShuffleMapStage 756) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 756: List() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 756 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[425] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=371499, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_129 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=376347, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_129_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_129_piece0 in memory on localhost:49158 (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 129 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 756 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[425] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 756.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 756.0 (TID 216, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 756.0 (TID 217, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 756.0 (TID 216) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 756.0 (TID 217) -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_396_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_396_1 locally -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 756.0 (TID 217). 2470 bytes result sent to driver -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 756.0 (TID 216). 2470 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 756.0 (TID 217) in 19 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 756.0 (TID 216) in 19 ms on localhost (2/2) -2016-06-13 19:15:43 INFO DAGScheduler:59 - ShuffleMapStage 756 (mapPartitions at GraphImpl.scala:235) finished in 0.020 s -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 756.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:43 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:43 INFO DAGScheduler:59 - waiting: Set(ResultStage 757) -2016-06-13 19:15:43 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents for ResultStage 757: List() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ResultStage 757 (MapPartitionsRDD[430] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(5944) called with curMem=378889, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_130 stored as values in memory (estimated size 5.8 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=384833, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_130_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_130_piece0 in memory on localhost:49158 (size: 2.8 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 130 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 757 (MapPartitionsRDD[430] at map at PowerIterationClustering.scala:369) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 757.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 757.0 (TID 218, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 757.0 (TID 219, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 757.0 (TID 218) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 757.0 (TID 219) -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_427_0 not found, computing it -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_418_0 locally -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_427_1 not found, computing it -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_418_1 locally -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=387725, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_427_1 stored as values in memory (estimated size 1480.0 B, free 730.2 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=389205, maxMem=766075207 -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_427_1 in memory on localhost:49158 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_427_0 stored as values in memory (estimated size 1480.0 B, free 730.2 MB) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 757.0 (TID 219). 2880 bytes result sent to driver -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_427_0 in memory on localhost:49158 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 757.0 (TID 218). 2880 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 757.0 (TID 219) in 11 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 757.0 (TID 218) in 13 ms on localhost (2/2) -2016-06-13 19:15:43 INFO DAGScheduler:59 - ResultStage 757 (sum at PowerIterationClustering.scala:369) finished in 0.014 s -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 757.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - Job 71 finished: sum at PowerIterationClustering.scala:369, took 0.086840 s -2016-06-13 19:15:43 INFO PowerIterationClustering:59 - Iteration 0: norm(v) = 36.22334869431644. -2016-06-13 19:15:43 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 151 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:15:43 INFO DAGScheduler:59 - Got job 72 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:15:43 INFO DAGScheduler:59 - Final stage: ResultStage 765(sum at PowerIterationClustering.scala:375) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 764, ShuffleMapStage 761, ShuffleMapStage 762) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ResultStage 765 (MapPartitionsRDD[441] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(6480) called with curMem=390685, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_131 stored as values in memory (estimated size 6.3 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(3052) called with curMem=397165, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_131_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_131_piece0 in memory on localhost:49158 (size: 3.0 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 131 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 765 (MapPartitionsRDD[441] at values at PowerIterationClustering.scala:375) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 765.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 765.0 (TID 220, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 765.0 (TID 221, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 765.0 (TID 221) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 765.0 (TID 220) -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_433_0 not found, computing it -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_433_1 not found, computing it -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_418_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_418_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_427_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_427_1 locally -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=400217, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_433_1 stored as values in memory (estimated size 1432.0 B, free 730.2 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=401649, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_433_0 stored as values in memory (estimated size 1432.0 B, free 730.2 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_433_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_433_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 765.0 (TID 221). 2664 bytes result sent to driver -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 765.0 (TID 220). 2664 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 765.0 (TID 221) in 7 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 765.0 (TID 220) in 8 ms on localhost (2/2) -2016-06-13 19:15:43 INFO DAGScheduler:59 - ResultStage 765 (sum at PowerIterationClustering.scala:375) finished in 0.008 s -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 765.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - Job 72 finished: sum at PowerIterationClustering.scala:375, took 0.023839 s -2016-06-13 19:15:43 INFO PowerIterationClustering:59 - Iteration 0: delta = 0.010579646370791257. -2016-06-13 19:15:43 INFO PowerIterationClustering:59 - Iteration 0: diff(delta) = 1.7976931348623157E308. -2016-06-13 19:15:43 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Registering RDD 444 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Registering RDD 448 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Got job 73 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:15:43 INFO DAGScheduler:59 - Final stage: ResultStage 775(sum at PowerIterationClustering.scala:369) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 772, ShuffleMapStage 769, ShuffleMapStage 770, ShuffleMapStage 774) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 774) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 773 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[444] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(5704) called with curMem=403081, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_132 stored as values in memory (estimated size 5.6 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(2813) called with curMem=408785, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_132_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_132_piece0 in memory on localhost:49158 (size: 2.7 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 132 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 773 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[444] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 773.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 773.0 (TID 222, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 773.0 (TID 223, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 773.0 (TID 222) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 773.0 (TID 223) -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_442_1 not found, computing it -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_442_0 not found, computing it -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_427_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_427_0 locally -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=411598, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_442_1 stored as values in memory (estimated size 1432.0 B, free 730.2 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=413030, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_442_0 stored as values in memory (estimated size 1432.0 B, free 730.2 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_442_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_442_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 773.0 (TID 223). 2834 bytes result sent to driver -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 773.0 (TID 222). 2834 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 773.0 (TID 223) in 6 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 773.0 (TID 222) in 7 ms on localhost (2/2) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 773.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - ShuffleMapStage 773 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.007 s -2016-06-13 19:15:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:43 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:43 INFO DAGScheduler:59 - waiting: Set(ResultStage 775, ShuffleMapStage 774) -2016-06-13 19:15:43 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents for ResultStage 775: List(ShuffleMapStage 774) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 774: List() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 774 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[448] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=414462, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_133 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=419310, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_133_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_133_piece0 in memory on localhost:49158 (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 133 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 774 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[448] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 774.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 774.0 (TID 224, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 774.0 (TID 225, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 774.0 (TID 224) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 774.0 (TID 225) -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_396_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_396_1 locally -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 774.0 (TID 225). 2470 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 774.0 (TID 225) in 34 ms on localhost (1/2) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 774.0 (TID 224). 2470 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 774.0 (TID 224) in 38 ms on localhost (2/2) -2016-06-13 19:15:43 INFO DAGScheduler:59 - ShuffleMapStage 774 (mapPartitions at GraphImpl.scala:235) finished in 0.038 s -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 774.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:43 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:43 INFO DAGScheduler:59 - waiting: Set(ResultStage 775) -2016-06-13 19:15:43 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents for ResultStage 775: List() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ResultStage 775 (MapPartitionsRDD[453] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(6640) called with curMem=421852, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_134 stored as values in memory (estimated size 6.5 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(3104) called with curMem=428492, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_134_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_134_piece0 in memory on localhost:49158 (size: 3.0 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 134 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 775 (MapPartitionsRDD[453] at map at PowerIterationClustering.scala:369) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 775.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 775.0 (TID 226, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 775.0 (TID 227, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 775.0 (TID 227) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 775.0 (TID 226) -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_450_1 not found, computing it -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_450_0 not found, computing it -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_442_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_442_0 locally -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=431596, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_450_0 stored as values in memory (estimated size 1480.0 B, free 730.2 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=433076, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_450_1 stored as values in memory (estimated size 1480.0 B, free 730.2 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_450_0 in memory on localhost:49158 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_450_1 in memory on localhost:49158 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 775.0 (TID 226). 2880 bytes result sent to driver -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 775.0 (TID 227). 2880 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 775.0 (TID 226) in 8 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 775.0 (TID 227) in 8 ms on localhost (2/2) -2016-06-13 19:15:43 INFO DAGScheduler:59 - ResultStage 775 (sum at PowerIterationClustering.scala:369) finished in 0.008 s -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 775.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - Job 73 finished: sum at PowerIterationClustering.scala:369, took 0.079866 s -2016-06-13 19:15:43 INFO PowerIterationClustering:59 - Iteration 1: norm(v) = 36.22334869431644. -2016-06-13 19:15:43 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 151 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 151 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:15:43 INFO DAGScheduler:59 - Got job 74 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:15:43 INFO DAGScheduler:59 - Final stage: ResultStage 785(sum at PowerIterationClustering.scala:375) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 782, ShuffleMapStage 779, ShuffleMapStage 780, ShuffleMapStage 784) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ResultStage 785 (MapPartitionsRDD[464] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(7072) called with curMem=434556, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_135 stored as values in memory (estimated size 6.9 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(3227) called with curMem=441628, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_135_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_135_piece0 in memory on localhost:49158 (size: 3.2 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 135 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 785 (MapPartitionsRDD[464] at values at PowerIterationClustering.scala:375) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 785.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 785.0 (TID 228, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 785.0 (TID 229, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 785.0 (TID 228) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 785.0 (TID 229) -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_456_0 not found, computing it -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_442_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_450_0 locally -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=444855, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_456_0 stored as values in memory (estimated size 1432.0 B, free 730.2 MB) -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_456_1 not found, computing it -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_442_1 locally -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_456_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_450_1 locally -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=446287, maxMem=766075207 -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 785.0 (TID 228). 2664 bytes result sent to driver -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_456_1 stored as values in memory (estimated size 1432.0 B, free 730.2 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_456_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 785.0 (TID 228) in 10 ms on localhost (1/2) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 785.0 (TID 229). 2664 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 785.0 (TID 229) in 10 ms on localhost (2/2) -2016-06-13 19:15:43 INFO DAGScheduler:59 - ResultStage 785 (sum at PowerIterationClustering.scala:375) finished in 0.011 s -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 785.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - Job 74 finished: sum at PowerIterationClustering.scala:375, took 0.028310 s -2016-06-13 19:15:43 INFO PowerIterationClustering:59 - Iteration 1: delta = 0.0. -2016-06-13 19:15:43 INFO PowerIterationClustering:59 - Iteration 1: diff(delta) = 0.010579646370791257. -2016-06-13 19:15:43 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Registering RDD 467 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Registering RDD 471 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Got job 75 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:15:43 INFO DAGScheduler:59 - Final stage: ResultStage 797(sum at PowerIterationClustering.scala:369) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 789, ShuffleMapStage 796, ShuffleMapStage 790, ShuffleMapStage 794, ShuffleMapStage 792) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 796) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 795 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[467] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(6288) called with curMem=447719, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_136 stored as values in memory (estimated size 6.1 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(2965) called with curMem=454007, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_136_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_136_piece0 in memory on localhost:49158 (size: 2.9 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 136 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 795 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[467] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 795.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 795.0 (TID 230, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 795.0 (TID 231, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 795.0 (TID 231) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 795.0 (TID 230) -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_465_1 not found, computing it -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_465_0 not found, computing it -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_450_0 locally -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=456972, maxMem=766075207 -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_450_1 locally -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_465_0 stored as values in memory (estimated size 1432.0 B, free 730.1 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=458404, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_465_1 stored as values in memory (estimated size 1432.0 B, free 730.1 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_465_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_465_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 795.0 (TID 230). 2834 bytes result sent to driver -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 795.0 (TID 231). 2834 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 795.0 (TID 230) in 13 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 795.0 (TID 231) in 13 ms on localhost (2/2) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 795.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - ShuffleMapStage 795 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.014 s -2016-06-13 19:15:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:43 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:43 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 796, ResultStage 797) -2016-06-13 19:15:43 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 796: List() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents for ResultStage 797: List(ShuffleMapStage 796) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 796 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[471] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=459836, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_137 stored as values in memory (estimated size 4.7 KB, free 730.1 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=464684, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_137_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_137_piece0 in memory on localhost:49158 (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 137 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 796 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[471] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 796.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 796.0 (TID 232, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 796.0 (TID 233, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 796.0 (TID 233) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 796.0 (TID 232) -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_396_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_396_0 locally -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 796.0 (TID 232). 2470 bytes result sent to driver -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 796.0 (TID 233). 2470 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 796.0 (TID 232) in 35 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 796.0 (TID 233) in 35 ms on localhost (2/2) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 796.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - ShuffleMapStage 796 (mapPartitions at GraphImpl.scala:235) finished in 0.036 s -2016-06-13 19:15:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:43 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:43 INFO DAGScheduler:59 - waiting: Set(ResultStage 797) -2016-06-13 19:15:43 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents for ResultStage 797: List() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ResultStage 797 (MapPartitionsRDD[476] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(7224) called with curMem=467226, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_138 stored as values in memory (estimated size 7.1 KB, free 730.1 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(3226) called with curMem=474450, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_138_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.1 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_138_piece0 in memory on localhost:49158 (size: 3.2 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 138 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 797 (MapPartitionsRDD[476] at map at PowerIterationClustering.scala:369) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 797.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 797.0 (TID 234, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 797.0 (TID 235, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 797.0 (TID 235) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 797.0 (TID 234) -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_473_0 not found, computing it -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_473_1 not found, computing it -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_465_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_465_1 locally -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=477676, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_473_0 stored as values in memory (estimated size 1480.0 B, free 730.1 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=479156, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_473_1 stored as values in memory (estimated size 1480.0 B, free 730.1 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_473_0 in memory on localhost:49158 (size: 1480.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_473_1 in memory on localhost:49158 (size: 1480.0 B, free: 730.3 MB) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 797.0 (TID 234). 2880 bytes result sent to driver -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 797.0 (TID 235). 2880 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 797.0 (TID 234) in 10 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 797.0 (TID 235) in 11 ms on localhost (2/2) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 797.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - ResultStage 797 (sum at PowerIterationClustering.scala:369) finished in 0.011 s -2016-06-13 19:15:43 INFO DAGScheduler:59 - Job 75 finished: sum at PowerIterationClustering.scala:369, took 0.103658 s -2016-06-13 19:15:43 INFO PowerIterationClustering:59 - Iteration 2: norm(v) = 36.22334869431644. -2016-06-13 19:15:43 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 151 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 151 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 41 is 151 bytes -2016-06-13 19:15:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 40 is 155 bytes -2016-06-13 19:15:43 INFO DAGScheduler:59 - Got job 76 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:15:43 INFO DAGScheduler:59 - Final stage: ResultStage 809(sum at PowerIterationClustering.scala:375) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 804, ShuffleMapStage 801, ShuffleMapStage 808, ShuffleMapStage 802, ShuffleMapStage 806) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ResultStage 809 (MapPartitionsRDD[487] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=480636, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_139 stored as values in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(3358) called with curMem=488292, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_139_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.1 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_139_piece0 in memory on localhost:49158 (size: 3.3 KB, free: 730.3 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 139 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 809 (MapPartitionsRDD[487] at values at PowerIterationClustering.scala:375) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 809.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 809.0 (TID 236, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 809.0 (TID 237, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 809.0 (TID 236) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 809.0 (TID 237) -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_479_1 not found, computing it -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_479_0 not found, computing it -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_465_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_465_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_473_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_473_0 locally -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=491650, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_479_1 stored as values in memory (estimated size 1432.0 B, free 730.1 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=493082, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_479_0 stored as values in memory (estimated size 1432.0 B, free 730.1 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_479_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.3 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_479_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.3 MB) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 809.0 (TID 237). 2664 bytes result sent to driver -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 809.0 (TID 236). 2664 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 809.0 (TID 237) in 6 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 809.0 (TID 236) in 8 ms on localhost (2/2) -2016-06-13 19:15:43 INFO DAGScheduler:59 - ResultStage 809 (sum at PowerIterationClustering.scala:375) finished in 0.008 s -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 809.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - Job 76 finished: sum at PowerIterationClustering.scala:375, took 0.023265 s -2016-06-13 19:15:43 INFO PowerIterationClustering:59 - Iteration 2: delta = 0.0. -2016-06-13 19:15:43 INFO PowerIterationClustering:59 - Iteration 2: diff(delta) = 0.0. -2016-06-13 19:15:43 WARN KMeans:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:15:43 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Got job 77 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:15:43 INFO DAGScheduler:59 - Final stage: ResultStage 821(takeSample at KMeans.scala:376) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 814, ShuffleMapStage 818, ShuffleMapStage 816, ShuffleMapStage 820, ShuffleMapStage 813) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ResultStage 821 (MapPartitionsRDD[495] at map at KMeans.scala:210), which has no missing parents -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(7888) called with curMem=494514, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_140 stored as values in memory (estimated size 7.7 KB, free 730.1 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(3429) called with curMem=502402, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_140_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.1 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_140_piece0 in memory on localhost:49158 (size: 3.3 KB, free: 730.3 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 140 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 821 (MapPartitionsRDD[495] at map at KMeans.scala:210) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 821.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 821.0 (TID 238, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 821.0 (TID 239, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 821.0 (TID 239) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 821.0 (TID 238) -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_490_1 not found, computing it -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_473_1 locally -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_490_0 not found, computing it -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=505831, maxMem=766075207 -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_473_0 locally -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_490_1 stored as values in memory (estimated size 1432.0 B, free 730.1 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=507263, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_490_0 stored as values in memory (estimated size 1432.0 B, free 730.1 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_490_1 in memory on localhost:49158 (size: 1432.0 B, free: 730.3 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_490_0 in memory on localhost:49158 (size: 1432.0 B, free: 730.3 MB) -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_493_1 not found, computing it -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_493_0 not found, computing it -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=508695, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_493_0 stored as values in memory (estimated size 240.0 B, free 730.1 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=508935, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_493_1 stored as values in memory (estimated size 240.0 B, free 730.1 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_493_0 in memory on localhost:49158 (size: 240.0 B, free: 730.3 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_493_1 in memory on localhost:49158 (size: 240.0 B, free: 730.3 MB) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 821.0 (TID 238). 2717 bytes result sent to driver -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 821.0 (TID 239). 2717 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 821.0 (TID 238) in 10 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 821.0 (TID 239) in 11 ms on localhost (2/2) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 821.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - ResultStage 821 (takeSample at KMeans.scala:376) finished in 0.012 s -2016-06-13 19:15:43 INFO DAGScheduler:59 - Job 77 finished: takeSample at KMeans.scala:376, took 0.023989 s -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_118_piece0 on localhost:49158 in memory (size: 1383.0 B, free: 730.3 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 109 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Got job 78 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:15:43 INFO DAGScheduler:59 - Final stage: ResultStage 833(takeSample at KMeans.scala:376) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 825, ShuffleMapStage 832, ShuffleMapStage 826, ShuffleMapStage 830, ShuffleMapStage 828) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_117_piece0 on localhost:49158 in memory (size: 4.0 KB, free: 730.3 MB) -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 108 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned shuffle 31 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ResultStage 833 (PartitionwiseSampledRDD[497] at takeSample at KMeans.scala:376), which has no missing parents -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(25280) called with curMem=491704, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_141 stored as values in memory (estimated size 24.7 KB, free 730.1 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(10115) called with curMem=516197, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_141_piece0 stored as bytes in memory (estimated size 9.9 KB, free 730.1 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_116_piece0 on localhost:49158 in memory (size: 787.0 B, free: 730.3 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_141_piece0 in memory on localhost:49158 (size: 9.9 KB, free: 730.3 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 141 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 833 (PartitionwiseSampledRDD[497] at takeSample at KMeans.scala:376) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 833.0 with 2 tasks -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 833.0 (TID 240, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 833.0 (TID 241, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 833.0 (TID 240) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_115_piece0 on localhost:49158 in memory (size: 4.5 KB, free: 730.3 MB) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 833.0 (TID 241) -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 107 -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 833.0 (TID 240). 2633 bytes result sent to driver -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 833.0 (TID 241). 2575 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 833.0 (TID 240) in 10 ms on localhost (1/2) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 833.0 (TID 241) in 10 ms on localhost (2/2) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_114_piece0 on localhost:49158 in memory (size: 4.4 KB, free: 730.3 MB) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 833.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO DAGScheduler:59 - ResultStage 833 (takeSample at KMeans.scala:376) finished in 0.013 s -2016-06-13 19:15:43 INFO DAGScheduler:59 - Job 78 finished: takeSample at KMeans.scala:376, took 0.030719 s -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 106 -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=491958, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_142 stored as values in memory (estimated size 880.0 B, free 730.1 MB) -2016-06-13 19:15:43 INFO BlockManager:59 - Removing RDD 384 -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(487) called with curMem=492838, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_142_piece0 stored as bytes in memory (estimated size 487.0 B, free 730.1 MB) -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned RDD 384 -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_142_piece0 in memory on localhost:49158 (size: 487.0 B, free: 730.3 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 142 from broadcast at KMeans.scala:394 -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_113_piece0 on localhost:49158 in memory (size: 429.0 B, free: 730.3 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_112_piece0 on localhost:49158 in memory (size: 4.4 KB, free: 730.3 MB) -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 105 -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_111_piece0 on localhost:49158 in memory (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 104 -2016-06-13 19:15:43 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:15:43 INFO BlockManager:59 - Removing RDD 380 -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned RDD 380 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Got job 79 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:15:43 INFO DAGScheduler:59 - Final stage: ResultStage 845(aggregate at KMeans.scala:402) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 840, ShuffleMapStage 844, ShuffleMapStage 837, ShuffleMapStage 838, ShuffleMapStage 842) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_110_piece0 on localhost:49158 in memory (size: 429.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting ResultStage 845 (MapPartitionsRDD[499] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(9912) called with curMem=445135, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_143 stored as values in memory (estimated size 9.7 KB, free 730.2 MB) -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(4237) called with curMem=455047, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block broadcast_143_piece0 stored as bytes in memory (estimated size 4.1 KB, free 730.1 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_109_piece0 on localhost:49158 in memory (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added broadcast_143_piece0 in memory on localhost:49158 (size: 4.1 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO SparkContext:59 - Created broadcast 143 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 845 (MapPartitionsRDD[499] at map at KMeans.scala:396) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Adding task set 845.0 with 2 tasks -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 103 -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 845.0 (TID 242, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:15:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 845.0 (TID 243, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:15:43 INFO Executor:59 - Running task 1.0 in stage 845.0 (TID 243) -2016-06-13 19:15:43 INFO Executor:59 - Running task 0.0 in stage 845.0 (TID 242) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_108_piece0 on localhost:49158 in memory (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_499_1 not found, computing it -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=444218, maxMem=766075207 -2016-06-13 19:15:43 INFO CacheManager:59 - Partition rdd_499_0 not found, computing it -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_499_1 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 102 -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:15:43 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:15:43 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=444714, maxMem=766075207 -2016-06-13 19:15:43 INFO MemoryStore:59 - Block rdd_499_0 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_499_1 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO BlockManager:59 - Removing RDD 376 -2016-06-13 19:15:43 INFO Executor:59 - Finished task 1.0 in stage 845.0 (TID 243). 2647 bytes result sent to driver -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Added rdd_499_0 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO Executor:59 - Finished task 0.0 in stage 845.0 (TID 242). 2647 bytes result sent to driver -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 845.0 (TID 243) in 16 ms on localhost (1/2) -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned RDD 376 -2016-06-13 19:15:43 INFO DAGScheduler:59 - ResultStage 845 (aggregate at KMeans.scala:402) finished in 0.018 s -2016-06-13 19:15:43 INFO DAGScheduler:59 - Job 79 finished: aggregate at KMeans.scala:402, took 0.041454 s -2016-06-13 19:15:43 INFO MapPartitionsRDD:59 - Removing RDD 496 from persistence list -2016-06-13 19:15:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 845.0 (TID 242) in 17 ms on localhost (2/2) -2016-06-13 19:15:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 845.0, whose tasks have all completed, from pool -2016-06-13 19:15:43 INFO BlockManager:59 - Removing RDD 496 -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_107_piece0 on localhost:49158 in memory (size: 579.0 B, free: 730.4 MB) -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_106_piece0 on localhost:49158 in memory (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 101 -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 129 -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_133_piece0 on localhost:49158 in memory (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 128 -2016-06-13 19:15:43 INFO BlockManagerInfo:59 - Removed broadcast_132_piece0 on localhost:49158 in memory (size: 2.7 KB, free: 730.4 MB) -2016-06-13 19:15:43 INFO ContextCleaner:59 - Cleaned accumulator 127 -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Removed broadcast_131_piece0 on localhost:49158 in memory (size: 3.0 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 126 -2016-06-13 19:15:44 INFO BlockManager:59 - Removing RDD 433 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Got job 80 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:15:44 INFO DAGScheduler:59 - Final stage: ResultStage 857(collect at KMeans.scala:431) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 850, ShuffleMapStage 854, ShuffleMapStage 852, ShuffleMapStage 849, ShuffleMapStage 856) -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned RDD 433 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ResultStage 857 (MapPartitionsRDD[501] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Removed broadcast_130_piece0 on localhost:49158 in memory (size: 2.8 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 125 -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(10072) called with curMem=391258, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_144 stored as values in memory (estimated size 9.8 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(4342) called with curMem=398788, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_144_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Removed broadcast_129_piece0 on localhost:49158 in memory (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_144_piece0 in memory on localhost:49158 (size: 4.2 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 144 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 857 (MapPartitionsRDD[501] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 857.0 with 2 tasks -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 124 -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 857.0 (TID 244, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 857.0 (TID 245, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 857.0 (TID 244) -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 857.0 (TID 245) -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_499_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_499_1 locally -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 857.0 (TID 244). 3118 bytes result sent to driver -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 857.0 (TID 245). 3471 bytes result sent to driver -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Removed broadcast_128_piece0 on localhost:49158 in memory (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 857.0 (TID 244) in 7 ms on localhost (1/2) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 857.0 (TID 245) in 7 ms on localhost (2/2) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 857.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO DAGScheduler:59 - ResultStage 857 (collect at KMeans.scala:431) finished in 0.009 s -2016-06-13 19:15:44 INFO DAGScheduler:59 - Job 80 finished: collect at KMeans.scala:431, took 0.027813 s -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(1648) called with curMem=390804, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_145 stored as values in memory (estimated size 1648.0 B, free 730.2 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(600) called with curMem=392452, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_145_piece0 stored as bytes in memory (estimated size 600.0 B, free 730.2 MB) -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 123 -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_145_piece0 in memory on localhost:49158 (size: 600.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 145 from broadcast at KMeans.scala:394 -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Removed broadcast_127_piece0 on localhost:49158 in memory (size: 2.3 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 122 -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Removed broadcast_126_piece0 on localhost:49158 in memory (size: 2.6 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 121 -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Removed broadcast_125_piece0 on localhost:49158 in memory (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 120 -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Removed broadcast_124_piece0 on localhost:49158 in memory (size: 1779.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 119 -2016-06-13 19:15:44 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Removed broadcast_123_piece0 on localhost:49158 in memory (size: 1974.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Got job 81 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:15:44 INFO DAGScheduler:59 - Final stage: ResultStage 869(aggregate at KMeans.scala:402) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 861, ShuffleMapStage 868, ShuffleMapStage 862, ShuffleMapStage 866, ShuffleMapStage 864) -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 118 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Removed broadcast_122_piece0 on localhost:49158 in memory (size: 3.5 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ResultStage 869 (MapPartitionsRDD[503] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 117 -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(10144) called with curMem=348984, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_146 stored as values in memory (estimated size 9.9 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(4326) called with curMem=359128, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_146_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_146_piece0 in memory on localhost:49158 (size: 4.2 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 146 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Removed broadcast_121_piece0 on localhost:49158 in memory (size: 1593.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 869 (MapPartitionsRDD[503] at map at KMeans.scala:396) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 869.0 with 2 tasks -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 116 -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 869.0 (TID 246, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 869.0 (TID 247, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 869.0 (TID 247) -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 869.0 (TID 246) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Removed broadcast_120_piece0 on localhost:49158 in memory (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 115 -2016-06-13 19:15:44 INFO CacheManager:59 - Partition rdd_503_1 not found, computing it -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_499_1 locally -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned shuffle 32 -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Removed broadcast_119_piece0 on localhost:49158 in memory (size: 363.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=343954, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block rdd_503_1 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added rdd_503_1 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO CacheManager:59 - Partition rdd_503_0 not found, computing it -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_499_0 locally -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 869.0 (TID 247). 2647 bytes result sent to driver -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 114 -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 113 -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 112 -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 111 -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 110 -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=343626, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block rdd_503_0 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added rdd_503_0 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 869.0 (TID 247) in 17 ms on localhost (1/2) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Removed broadcast_140_piece0 on localhost:49158 in memory (size: 3.3 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 869.0 (TID 246). 2647 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 869.0 (TID 246) in 18 ms on localhost (2/2) -2016-06-13 19:15:44 INFO DAGScheduler:59 - ResultStage 869 (aggregate at KMeans.scala:402) finished in 0.018 s -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 869.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 135 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Job 81 finished: aggregate at KMeans.scala:402, took 0.034354 s -2016-06-13 19:15:44 INFO MapPartitionsRDD:59 - Removing RDD 499 from persistence list -2016-06-13 19:15:44 INFO BlockManager:59 - Removing RDD 499 -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Removed broadcast_139_piece0 on localhost:49158 in memory (size: 3.3 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 134 -2016-06-13 19:15:44 INFO BlockManager:59 - Removing RDD 479 -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned RDD 479 -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Removed broadcast_138_piece0 on localhost:49158 in memory (size: 3.2 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 133 -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Removed broadcast_137_piece0 on localhost:49158 in memory (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 132 -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Removed broadcast_136_piece0 on localhost:49158 in memory (size: 2.9 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 131 -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Removed broadcast_135_piece0 on localhost:49158 in memory (size: 3.2 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned accumulator 130 -2016-06-13 19:15:44 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:15:44 INFO BlockManager:59 - Removing RDD 456 -2016-06-13 19:15:44 INFO ContextCleaner:59 - Cleaned RDD 456 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Got job 82 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:15:44 INFO DAGScheduler:59 - Final stage: ResultStage 881(collect at KMeans.scala:431) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 876, ShuffleMapStage 880, ShuffleMapStage 873, ShuffleMapStage 874, ShuffleMapStage 878) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Removed broadcast_134_piece0 on localhost:49158 in memory (size: 3.0 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ResultStage 881 (MapPartitionsRDD[505] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(10304) called with curMem=267935, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_147 stored as values in memory (estimated size 10.1 KB, free 730.3 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(4431) called with curMem=278239, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_147_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_147_piece0 in memory on localhost:49158 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 147 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 881 (MapPartitionsRDD[505] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 881.0 with 2 tasks -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 881.0 (TID 248, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 881.0 (TID 249, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 881.0 (TID 249) -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 881.0 (TID 248) -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_503_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_503_0 locally -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 881.0 (TID 249). 3240 bytes result sent to driver -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 881.0 (TID 248). 2990 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 881.0 (TID 249) in 6 ms on localhost (1/2) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 881.0 (TID 248) in 7 ms on localhost (2/2) -2016-06-13 19:15:44 INFO DAGScheduler:59 - ResultStage 881 (collect at KMeans.scala:431) finished in 0.007 s -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 881.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO DAGScheduler:59 - Job 82 finished: collect at KMeans.scala:431, took 0.037007 s -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(1456) called with curMem=282670, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_148 stored as values in memory (estimated size 1456.0 B, free 730.3 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(582) called with curMem=284126, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_148_piece0 stored as bytes in memory (estimated size 582.0 B, free 730.3 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_148_piece0 in memory on localhost:49158 (size: 582.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 148 from broadcast at KMeans.scala:394 -2016-06-13 19:15:44 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Got job 83 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:15:44 INFO DAGScheduler:59 - Final stage: ResultStage 893(aggregate at KMeans.scala:402) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 888, ShuffleMapStage 885, ShuffleMapStage 892, ShuffleMapStage 886, ShuffleMapStage 890) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ResultStage 893 (MapPartitionsRDD[507] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(10376) called with curMem=284708, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_149 stored as values in memory (estimated size 10.1 KB, free 730.3 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(4373) called with curMem=295084, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_149_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_149_piece0 in memory on localhost:49158 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 149 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 893 (MapPartitionsRDD[507] at map at KMeans.scala:396) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 893.0 with 2 tasks -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 893.0 (TID 250, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 893.0 (TID 251, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 893.0 (TID 250) -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 893.0 (TID 251) -2016-06-13 19:15:44 INFO CacheManager:59 - Partition rdd_507_0 not found, computing it -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:15:44 INFO CacheManager:59 - Partition rdd_507_1 not found, computing it -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_503_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_503_1 locally -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=299457, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block rdd_507_0 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=299953, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block rdd_507_1 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added rdd_507_0 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added rdd_507_1 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 893.0 (TID 250). 2647 bytes result sent to driver -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 893.0 (TID 251). 2647 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 893.0 (TID 250) in 10 ms on localhost (1/2) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 893.0 (TID 251) in 9 ms on localhost (2/2) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 893.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO DAGScheduler:59 - ResultStage 893 (aggregate at KMeans.scala:402) finished in 0.011 s -2016-06-13 19:15:44 INFO DAGScheduler:59 - Job 83 finished: aggregate at KMeans.scala:402, took 0.025398 s -2016-06-13 19:15:44 INFO MapPartitionsRDD:59 - Removing RDD 503 from persistence list -2016-06-13 19:15:44 INFO BlockManager:59 - Removing RDD 503 -2016-06-13 19:15:44 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Got job 84 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:15:44 INFO DAGScheduler:59 - Final stage: ResultStage 905(collect at KMeans.scala:431) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 897, ShuffleMapStage 898, ShuffleMapStage 902, ShuffleMapStage 900, ShuffleMapStage 904) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ResultStage 905 (MapPartitionsRDD[509] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(10536) called with curMem=299457, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_150 stored as values in memory (estimated size 10.3 KB, free 730.3 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(4460) called with curMem=309993, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_150_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_150_piece0 in memory on localhost:49158 (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 150 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 905 (MapPartitionsRDD[509] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 905.0 with 2 tasks -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 905.0 (TID 252, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 905.0 (TID 253, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 905.0 (TID 253) -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 905.0 (TID 252) -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_507_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_507_0 locally -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 905.0 (TID 252). 2722 bytes result sent to driver -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 905.0 (TID 253). 2040 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 905.0 (TID 252) in 6 ms on localhost (1/2) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 905.0 (TID 253) in 6 ms on localhost (2/2) -2016-06-13 19:15:44 INFO DAGScheduler:59 - ResultStage 905 (collect at KMeans.scala:431) finished in 0.006 s -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 905.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO DAGScheduler:59 - Job 84 finished: collect at KMeans.scala:431, took 0.017241 s -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(648) called with curMem=314453, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_151 stored as values in memory (estimated size 648.0 B, free 730.3 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(429) called with curMem=315101, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_151_piece0 stored as bytes in memory (estimated size 429.0 B, free 730.3 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_151_piece0 in memory on localhost:49158 (size: 429.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 151 from broadcast at KMeans.scala:394 -2016-06-13 19:15:44 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Got job 85 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:15:44 INFO DAGScheduler:59 - Final stage: ResultStage 917(aggregate at KMeans.scala:402) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 912, ShuffleMapStage 909, ShuffleMapStage 916, ShuffleMapStage 910, ShuffleMapStage 914) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ResultStage 917 (MapPartitionsRDD[511] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(10608) called with curMem=315530, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_152 stored as values in memory (estimated size 10.4 KB, free 730.3 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(4431) called with curMem=326138, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_152_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_152_piece0 in memory on localhost:49158 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 152 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 917 (MapPartitionsRDD[511] at map at KMeans.scala:396) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 917.0 with 2 tasks -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 917.0 (TID 254, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 917.0 (TID 255, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 917.0 (TID 254) -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 917.0 (TID 255) -2016-06-13 19:15:44 INFO CacheManager:59 - Partition rdd_511_1 not found, computing it -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_507_1 locally -2016-06-13 19:15:44 INFO CacheManager:59 - Partition rdd_511_0 not found, computing it -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_507_0 locally -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=330569, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block rdd_511_1 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=331065, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block rdd_511_0 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added rdd_511_1 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added rdd_511_0 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 917.0 (TID 255). 2647 bytes result sent to driver -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 917.0 (TID 254). 2647 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 917.0 (TID 255) in 7 ms on localhost (1/2) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 917.0 (TID 254) in 8 ms on localhost (2/2) -2016-06-13 19:15:44 INFO DAGScheduler:59 - ResultStage 917 (aggregate at KMeans.scala:402) finished in 0.008 s -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 917.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO DAGScheduler:59 - Job 85 finished: aggregate at KMeans.scala:402, took 0.021283 s -2016-06-13 19:15:44 INFO MapPartitionsRDD:59 - Removing RDD 507 from persistence list -2016-06-13 19:15:44 INFO BlockManager:59 - Removing RDD 507 -2016-06-13 19:15:44 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Got job 86 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:15:44 INFO DAGScheduler:59 - Final stage: ResultStage 929(collect at KMeans.scala:431) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 924, ShuffleMapStage 921, ShuffleMapStage 928, ShuffleMapStage 922, ShuffleMapStage 926) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ResultStage 929 (MapPartitionsRDD[513] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(10768) called with curMem=330569, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_153 stored as values in memory (estimated size 10.5 KB, free 730.3 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(4505) called with curMem=341337, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_153_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_153_piece0 in memory on localhost:49158 (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 153 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 929 (MapPartitionsRDD[513] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 929.0 with 2 tasks -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 929.0 (TID 256, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 929.0 (TID 257, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 929.0 (TID 257) -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 929.0 (TID 256) -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_511_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_511_1 locally -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 929.0 (TID 257). 2040 bytes result sent to driver -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 929.0 (TID 256). 2040 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 929.0 (TID 257) in 5 ms on localhost (1/2) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 929.0 (TID 256) in 5 ms on localhost (2/2) -2016-06-13 19:15:44 INFO DAGScheduler:59 - ResultStage 929 (collect at KMeans.scala:431) finished in 0.005 s -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 929.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO DAGScheduler:59 - Job 86 finished: collect at KMeans.scala:431, took 0.017372 s -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(584) called with curMem=345842, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_154 stored as values in memory (estimated size 584.0 B, free 730.3 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(236) called with curMem=346426, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_154_piece0 stored as bytes in memory (estimated size 236.0 B, free 730.3 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_154_piece0 in memory on localhost:49158 (size: 236.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 154 from broadcast at KMeans.scala:394 -2016-06-13 19:15:44 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Got job 87 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:15:44 INFO DAGScheduler:59 - Final stage: ResultStage 941(aggregate at KMeans.scala:402) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 933, ShuffleMapStage 934, ShuffleMapStage 938, ShuffleMapStage 936, ShuffleMapStage 940) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ResultStage 941 (MapPartitionsRDD[515] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(10848) called with curMem=346662, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_155 stored as values in memory (estimated size 10.6 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(4483) called with curMem=357510, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_155_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_155_piece0 in memory on localhost:49158 (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 155 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 941 (MapPartitionsRDD[515] at map at KMeans.scala:396) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 941.0 with 2 tasks -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 941.0 (TID 258, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 941.0 (TID 259, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 941.0 (TID 258) -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 941.0 (TID 259) -2016-06-13 19:15:44 INFO CacheManager:59 - Partition rdd_515_1 not found, computing it -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:15:44 INFO CacheManager:59 - Partition rdd_515_0 not found, computing it -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_511_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_511_0 locally -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=361993, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block rdd_515_1 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=362489, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block rdd_515_0 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added rdd_515_1 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added rdd_515_0 in memory on localhost:49158 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 941.0 (TID 259). 2647 bytes result sent to driver -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 941.0 (TID 258). 2647 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 941.0 (TID 259) in 10 ms on localhost (1/2) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 941.0 (TID 258) in 10 ms on localhost (2/2) -2016-06-13 19:15:44 INFO DAGScheduler:59 - ResultStage 941 (aggregate at KMeans.scala:402) finished in 0.010 s -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 941.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO DAGScheduler:59 - Job 87 finished: aggregate at KMeans.scala:402, took 0.022636 s -2016-06-13 19:15:44 INFO MapPartitionsRDD:59 - Removing RDD 511 from persistence list -2016-06-13 19:15:44 INFO BlockManager:59 - Removing RDD 511 -2016-06-13 19:15:44 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Got job 88 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:15:44 INFO DAGScheduler:59 - Final stage: ResultStage 953(collect at KMeans.scala:431) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 948, ShuffleMapStage 945, ShuffleMapStage 952, ShuffleMapStage 946, ShuffleMapStage 950) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ResultStage 953 (MapPartitionsRDD[517] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=361993, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_156 stored as values in memory (estimated size 10.8 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(4564) called with curMem=373001, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_156_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_156_piece0 in memory on localhost:49158 (size: 4.5 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 156 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 953 (MapPartitionsRDD[517] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 953.0 with 2 tasks -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 953.0 (TID 260, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 953.0 (TID 261, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 953.0 (TID 261) -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 953.0 (TID 260) -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_515_1 locally -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 953.0 (TID 261). 2040 bytes result sent to driver -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_515_0 locally -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 953.0 (TID 260). 2040 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 953.0 (TID 261) in 6 ms on localhost (1/2) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 953.0 (TID 260) in 8 ms on localhost (2/2) -2016-06-13 19:15:44 INFO DAGScheduler:59 - ResultStage 953 (collect at KMeans.scala:431) finished in 0.008 s -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 953.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO DAGScheduler:59 - Job 88 finished: collect at KMeans.scala:431, took 0.020467 s -2016-06-13 19:15:44 INFO MapPartitionsRDD:59 - Removing RDD 515 from persistence list -2016-06-13 19:15:44 INFO BlockManager:59 - Removing RDD 515 -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(2880) called with curMem=377565, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_157 stored as values in memory (estimated size 2.8 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(708) called with curMem=379453, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_157_piece0 stored as bytes in memory (estimated size 708.0 B, free 730.2 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_157_piece0 in memory on localhost:49158 (size: 708.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 157 from broadcast at KMeans.scala:445 -2016-06-13 19:15:44 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:450 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Registering RDD 518 (flatMap at KMeans.scala:446) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Got job 89 (collectAsMap at KMeans.scala:450) with 2 output partitions -2016-06-13 19:15:44 INFO DAGScheduler:59 - Final stage: ResultStage 966(collectAsMap at KMeans.scala:450) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 965) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 965) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 965 (MapPartitionsRDD[518] at flatMap at KMeans.scala:446), which has no missing parents -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(9392) called with curMem=380161, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_158 stored as values in memory (estimated size 9.2 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(4066) called with curMem=389553, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_158_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_158_piece0 in memory on localhost:49158 (size: 4.0 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 158 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 965 (MapPartitionsRDD[518] at flatMap at KMeans.scala:446) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 965.0 with 2 tasks -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 965.0 (TID 262, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 965.0 (TID 263, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 965.0 (TID 263) -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 965.0 (TID 262) -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 965.0 (TID 262). 2254 bytes result sent to driver -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 965.0 (TID 263). 2254 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 965.0 (TID 262) in 18 ms on localhost (1/2) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 965.0 (TID 263) in 17 ms on localhost (2/2) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 965.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO DAGScheduler:59 - ShuffleMapStage 965 (flatMap at KMeans.scala:446) finished in 0.019 s -2016-06-13 19:15:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:44 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:44 INFO DAGScheduler:59 - waiting: Set(ResultStage 966) -2016-06-13 19:15:44 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents for ResultStage 966: List() -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ResultStage 966 (ShuffledRDD[519] at reduceByKey at KMeans.scala:450), which is now runnable -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=393619, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_159 stored as values in memory (estimated size 2.3 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(1383) called with curMem=395931, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_159_piece0 stored as bytes in memory (estimated size 1383.0 B, free 730.2 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_159_piece0 in memory on localhost:49158 (size: 1383.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 159 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 966 (ShuffledRDD[519] at reduceByKey at KMeans.scala:450) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 966.0 with 2 tasks -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 966.0 (TID 264, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 966.0 (TID 265, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 966.0 (TID 265) -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 966.0 (TID 264) -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 966.0 (TID 264). 1852 bytes result sent to driver -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 966.0 (TID 265). 1744 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 966.0 (TID 264) in 9 ms on localhost (1/2) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 966.0 (TID 265) in 9 ms on localhost (2/2) -2016-06-13 19:15:44 INFO DAGScheduler:59 - ResultStage 966 (collectAsMap at KMeans.scala:450) finished in 0.010 s -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 966.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO DAGScheduler:59 - Job 89 finished: collectAsMap at KMeans.scala:450, took 0.048035 s -2016-06-13 19:15:44 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:15:44 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:15:44 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:15:44 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:15:44 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:15:44 INFO KMeans:59 - Initialization with k-means|| took 0.612 seconds. -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(824) called with curMem=397314, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_160 stored as values in memory (estimated size 824.0 B, free 730.2 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(339) called with curMem=398138, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_160_piece0 stored as bytes in memory (estimated size 339.0 B, free 730.2 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_160_piece0 in memory on localhost:49158 (size: 339.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 160 from broadcast at KMeans.scala:276 -2016-06-13 19:15:44 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:302 -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 151 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 151 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 41 is 151 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 40 is 155 bytes -2016-06-13 19:15:44 INFO DAGScheduler:59 - Registering RDD 520 (mapPartitions at KMeans.scala:279) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Got job 90 (collectAsMap at KMeans.scala:302) with 2 output partitions -2016-06-13 19:15:44 INFO DAGScheduler:59 - Final stage: ResultStage 979(collectAsMap at KMeans.scala:302) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 978) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 978) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 978 (MapPartitionsRDD[520] at mapPartitions at KMeans.scala:279), which has no missing parents -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(10016) called with curMem=398477, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_161 stored as values in memory (estimated size 9.8 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(4398) called with curMem=408493, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_161_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_161_piece0 in memory on localhost:49158 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 161 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 978 (MapPartitionsRDD[520] at mapPartitions at KMeans.scala:279) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 978.0 with 2 tasks -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 978.0 (TID 266, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 978.0 (TID 267, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 978.0 (TID 266) -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 978.0 (TID 267) -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_493_0 locally -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 978.0 (TID 266). 2413 bytes result sent to driver -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 978.0 (TID 267). 2413 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 978.0 (TID 266) in 22 ms on localhost (1/2) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 978.0 (TID 267) in 21 ms on localhost (2/2) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 978.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO DAGScheduler:59 - ShuffleMapStage 978 (mapPartitions at KMeans.scala:279) finished in 0.023 s -2016-06-13 19:15:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:44 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:44 INFO DAGScheduler:59 - waiting: Set(ResultStage 979) -2016-06-13 19:15:44 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents for ResultStage 979: List() -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ResultStage 979 (ShuffledRDD[521] at reduceByKey at KMeans.scala:302), which is now runnable -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(2808) called with curMem=412891, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_162 stored as values in memory (estimated size 2.7 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(1591) called with curMem=415699, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_162_piece0 stored as bytes in memory (estimated size 1591.0 B, free 730.2 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_162_piece0 in memory on localhost:49158 (size: 1591.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 162 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 979 (ShuffledRDD[521] at reduceByKey at KMeans.scala:302) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 979.0 with 2 tasks -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 979.0 (TID 268, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 979.0 (TID 269, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 979.0 (TID 268) -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 979.0 (TID 269) -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 979.0 (TID 268). 1785 bytes result sent to driver -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 979.0 (TID 269). 1671 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 979.0 (TID 268) in 9 ms on localhost (1/2) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 979.0 (TID 269) in 10 ms on localhost (2/2) -2016-06-13 19:15:44 INFO DAGScheduler:59 - ResultStage 979 (collectAsMap at KMeans.scala:302) finished in 0.011 s -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 979.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO DAGScheduler:59 - Job 90 finished: collectAsMap at KMeans.scala:302, took 0.060699 s -2016-06-13 19:15:44 INFO KMeans:59 - Run 0 finished in 1 iterations -2016-06-13 19:15:44 INFO KMeans:59 - Run 1 finished in 1 iterations -2016-06-13 19:15:44 INFO KMeans:59 - Run 2 finished in 1 iterations -2016-06-13 19:15:44 INFO KMeans:59 - Run 3 finished in 1 iterations -2016-06-13 19:15:44 INFO KMeans:59 - Run 4 finished in 1 iterations -2016-06-13 19:15:44 INFO KMeans:59 - Iterations took 0.086 seconds. -2016-06-13 19:15:44 INFO KMeans:59 - KMeans converged in 1 iterations. -2016-06-13 19:15:44 INFO KMeans:59 - The cost for the best run is 6.650150218788335E-6. -2016-06-13 19:15:44 INFO MapPartitionsRDD:59 - Removing RDD 493 from persistence list -2016-06-13 19:15:44 INFO BlockManager:59 - Removing RDD 493 -2016-06-13 19:15:44 WARN KMeans:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:15:44 INFO SparkContext:59 - Starting job: collect at PowerIterationClusteringSuite.scala:114 -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 151 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 151 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 41 is 151 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 40 is 155 bytes -2016-06-13 19:15:44 INFO DAGScheduler:59 - Got job 91 (collect at PowerIterationClusteringSuite.scala:114) with 2 output partitions -2016-06-13 19:15:44 INFO DAGScheduler:59 - Final stage: ResultStage 991(collect at PowerIterationClusteringSuite.scala:114) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 984, ShuffleMapStage 988, ShuffleMapStage 986, ShuffleMapStage 983, ShuffleMapStage 990) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ResultStage 991 (MapPartitionsRDD[524] at mapPartitions at PowerIterationClustering.scala:232), which has no missing parents -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(8192) called with curMem=416810, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_163 stored as values in memory (estimated size 8.0 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(3576) called with curMem=425002, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_163_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_163_piece0 in memory on localhost:49158 (size: 3.5 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 163 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 991 (MapPartitionsRDD[524] at mapPartitions at PowerIterationClustering.scala:232) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 991.0 with 2 tasks -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 991.0 (TID 270, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 991.0 (TID 271, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 991.0 (TID 271) -2016-06-13 19:15:44 INFO CacheManager:59 - Partition rdd_522_1 not found, computing it -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_1 locally -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=428578, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block rdd_522_1 stored as values in memory (estimated size 1168.0 B, free 730.2 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added rdd_522_1 in memory on localhost:49158 (size: 1168.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 991.0 (TID 271). 2919 bytes result sent to driver -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 991.0 (TID 270) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 991.0 (TID 271) in 12 ms on localhost (1/2) -2016-06-13 19:15:44 INFO CacheManager:59 - Partition rdd_522_0 not found, computing it -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_490_0 locally -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=429746, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block rdd_522_0 stored as values in memory (estimated size 1168.0 B, free 730.2 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added rdd_522_0 in memory on localhost:49158 (size: 1168.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 991.0 (TID 270). 2919 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 991.0 (TID 270) in 21 ms on localhost (2/2) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 991.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO DAGScheduler:59 - ResultStage 991 (collect at PowerIterationClusteringSuite.scala:114) finished in 0.023 s -2016-06-13 19:15:44 INFO DAGScheduler:59 - Job 91 finished: collect at PowerIterationClusteringSuite.scala:114, took 0.049776 s -2016-06-13 19:15:44 INFO SparkContext:59 - Starting job: collect at EdgeRDDImpl.scala:51 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Registering RDD 530 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Registering RDD 536 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Registering RDD 540 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Got job 92 (collect at EdgeRDDImpl.scala:51) with 2 output partitions -2016-06-13 19:15:44 INFO DAGScheduler:59 - Final stage: ResultStage 995(collect at EdgeRDDImpl.scala:51) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 994) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 994) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 992 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[530] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(3016) called with curMem=430914, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_164 stored as values in memory (estimated size 2.9 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(1777) called with curMem=433930, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_164_piece0 stored as bytes in memory (estimated size 1777.0 B, free 730.2 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_164_piece0 in memory on localhost:49158 (size: 1777.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 164 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 992 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[530] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 992.0 with 2 tasks -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 993 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[536] at mapPartitions at GraphImpl.scala:235), which has no missing parents -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 992.0 (TID 272, localhost, PROCESS_LOCAL, 2242 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 992.0 (TID 273, localhost, PROCESS_LOCAL, 2281 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 992.0 (TID 273) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(3432) called with curMem=435707, maxMem=766075207 -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 992.0 (TID 272) -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_165 stored as values in memory (estimated size 3.4 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(1972) called with curMem=439139, maxMem=766075207 -2016-06-13 19:15:44 INFO CacheManager:59 - Partition rdd_527_1 not found, computing it -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_165_piece0 stored as bytes in memory (estimated size 1972.0 B, free 730.2 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(2616) called with curMem=441111, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block rdd_527_1 stored as values in memory (estimated size 2.6 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO CacheManager:59 - Partition rdd_527_0 not found, computing it -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_165_piece0 in memory on localhost:49158 (size: 1972.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(2568) called with curMem=443727, maxMem=766075207 -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added rdd_527_1 in memory on localhost:49158 (size: 2.6 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - Block rdd_527_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added rdd_527_0 in memory on localhost:49158 (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 165 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 993 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[536] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 993.0 with 2 tasks -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 993.0 (TID 274, localhost, PROCESS_LOCAL, 2242 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 993.0 (TID 275, localhost, PROCESS_LOCAL, 2281 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 993.0 (TID 274) -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_527_0 locally -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 993.0 (TID 275) -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_527_1 locally -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 992.0 (TID 272). 1739 bytes result sent to driver -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 993.0 (TID 274). 2254 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 992.0 (TID 272) in 61 ms on localhost (1/2) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 993.0 (TID 274) in 49 ms on localhost (1/2) -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 993.0 (TID 275). 2254 bytes result sent to driver -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 992.0 (TID 273). 1739 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 993.0 (TID 275) in 61 ms on localhost (2/2) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 993.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO DAGScheduler:59 - ShuffleMapStage 993 (mapPartitions at GraphImpl.scala:235) finished in 0.064 s -2016-06-13 19:15:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 992.0 (TID 273) in 74 ms on localhost (2/2) -2016-06-13 19:15:44 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 992) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 992.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 994, ResultStage 995) -2016-06-13 19:15:44 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 994: List(ShuffleMapStage 992) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents for ResultStage 995: List(ShuffleMapStage 994) -2016-06-13 19:15:44 INFO DAGScheduler:59 - ShuffleMapStage 992 (mapPartitions at VertexRDD.scala:358) finished in 0.077 s -2016-06-13 19:15:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:44 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:44 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 994, ResultStage 995) -2016-06-13 19:15:44 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 994: List() -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents for ResultStage 995: List(ShuffleMapStage 994) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 994 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[540] at mapPartitions at VertexRDDImpl.scala:247), which is now runnable -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=446295, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_166 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(2376) called with curMem=450543, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_166_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.2 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_166_piece0 in memory on localhost:49158 (size: 2.3 KB, free: 730.4 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 166 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 994 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[540] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 994.0 with 2 tasks -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 994.0 (TID 276, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 994.0 (TID 277, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 994.0 (TID 276) -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 994.0 (TID 277) -2016-06-13 19:15:44 INFO CacheManager:59 - Partition rdd_538_1 not found, computing it -2016-06-13 19:15:44 INFO CacheManager:59 - Partition rdd_538_0 not found, computing it -2016-06-13 19:15:44 INFO CacheManager:59 - Partition rdd_533_1 not found, computing it -2016-06-13 19:15:44 INFO CacheManager:59 - Partition rdd_533_0 not found, computing it -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(1736) called with curMem=452919, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block rdd_533_1 stored as values in memory (estimated size 1736.0 B, free 730.2 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(1744) called with curMem=454655, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block rdd_533_0 stored as values in memory (estimated size 1744.0 B, free 730.2 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added rdd_533_1 in memory on localhost:49158 (size: 1736.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added rdd_533_0 in memory on localhost:49158 (size: 1744.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(1792) called with curMem=456399, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block rdd_538_0 stored as values in memory (estimated size 1792.0 B, free 730.1 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(1784) called with curMem=458191, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block rdd_538_1 stored as values in memory (estimated size 1784.0 B, free 730.1 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added rdd_538_0 in memory on localhost:49158 (size: 1792.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added rdd_538_1 in memory on localhost:49158 (size: 1784.0 B, free: 730.4 MB) -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 994.0 (TID 276). 2010 bytes result sent to driver -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 994.0 (TID 277). 2010 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 994.0 (TID 276) in 30 ms on localhost (1/2) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 994.0 (TID 277) in 30 ms on localhost (2/2) -2016-06-13 19:15:44 INFO DAGScheduler:59 - ShuffleMapStage 994 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.030 s -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 994.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:44 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:44 INFO DAGScheduler:59 - waiting: Set(ResultStage 995) -2016-06-13 19:15:44 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents for ResultStage 995: List() -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ResultStage 995 (MapPartitionsRDD[546] at map at EdgeRDDImpl.scala:51), which is now runnable -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(4872) called with curMem=459975, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_167 stored as values in memory (estimated size 4.8 KB, free 730.1 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(2549) called with curMem=464847, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_167_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.1 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_167_piece0 in memory on localhost:49158 (size: 2.5 KB, free: 730.3 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 167 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 995 (MapPartitionsRDD[546] at map at EdgeRDDImpl.scala:51) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 995.0 with 2 tasks -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 995.0 (TID 278, localhost, PROCESS_LOCAL, 2502 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 995.0 (TID 279, localhost, PROCESS_LOCAL, 2541 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 995.0 (TID 278) -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 995.0 (TID 279) -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_527_0 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_527_1 locally -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 995.0 (TID 279). 2640 bytes result sent to driver -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 995.0 (TID 278). 2568 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 995.0 (TID 279) in 9 ms on localhost (1/2) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 995.0 (TID 278) in 9 ms on localhost (2/2) -2016-06-13 19:15:44 INFO DAGScheduler:59 - ResultStage 995 (collect at EdgeRDDImpl.scala:51) finished in 0.009 s -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 995.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO DAGScheduler:59 - Job 92 finished: collect at EdgeRDDImpl.scala:51, took 0.144742 s -2016-06-13 19:15:44 INFO SparkContext:59 - Starting job: reduce at VertexRDDImpl.scala:90 -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 46 is 154 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 45 is 154 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 44 is 152 bytes -2016-06-13 19:15:44 INFO DAGScheduler:59 - Registering RDD 552 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Registering RDD 547 (parallelize at PowerIterationClusteringSuite.scala:155) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Got job 93 (reduce at VertexRDDImpl.scala:90) with 2 output partitions -2016-06-13 19:15:44 INFO DAGScheduler:59 - Final stage: ResultStage 1001(reduce at VertexRDDImpl.scala:90) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 999, ShuffleMapStage 1000) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 999, ShuffleMapStage 1000) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 999 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[552] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(4616) called with curMem=467396, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_168 stored as values in memory (estimated size 4.5 KB, free 730.1 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(2474) called with curMem=472012, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_168_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.1 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_168_piece0 in memory on localhost:49158 (size: 2.4 KB, free: 730.3 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 168 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 999 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[552] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 999.0 with 2 tasks -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1000 (ParallelCollectionRDD[547] at parallelize at PowerIterationClusteringSuite.scala:155), which has no missing parents -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 999.0 (TID 280, localhost, PROCESS_LOCAL, 2491 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 999.0 (TID 281, localhost, PROCESS_LOCAL, 2530 bytes) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(1520) called with curMem=474486, maxMem=766075207 -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 999.0 (TID 281) -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 999.0 (TID 280) -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_169 stored as values in memory (estimated size 1520.0 B, free 730.1 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(984) called with curMem=476006, maxMem=766075207 -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_527_0 locally -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_169_piece0 stored as bytes in memory (estimated size 984.0 B, free 730.1 MB) -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_169_piece0 in memory on localhost:49158 (size: 984.0 B, free: 730.3 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 169 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1000 (ParallelCollectionRDD[547] at parallelize at PowerIterationClusteringSuite.scala:155) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 1000.0 with 2 tasks -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_527_1 locally -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 1000.0 (TID 282, localhost, PROCESS_LOCAL, 2222 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 1000.0 (TID 283, localhost, PROCESS_LOCAL, 2222 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 1000.0 (TID 283) -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 999.0 (TID 280). 2470 bytes result sent to driver -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 999.0 (TID 281). 2470 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 999.0 (TID 280) in 29 ms on localhost (1/2) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 999.0 (TID 281) in 29 ms on localhost (2/2) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 999.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO DAGScheduler:59 - ShuffleMapStage 999 (mapPartitions at VertexRDD.scala:358) finished in 0.031 s -2016-06-13 19:15:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:44 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 1000) -2016-06-13 19:15:44 INFO DAGScheduler:59 - waiting: Set(ResultStage 1001) -2016-06-13 19:15:44 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents for ResultStage 1001: List(ShuffleMapStage 1000) -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 1000.0 (TID 282) -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 1000.0 (TID 283). 1159 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 1000.0 (TID 283) in 24 ms on localhost (1/2) -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 1000.0 (TID 282). 1159 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 1000.0 (TID 282) in 39 ms on localhost (2/2) -2016-06-13 19:15:44 INFO DAGScheduler:59 - ShuffleMapStage 1000 (parallelize at PowerIterationClusteringSuite.scala:155) finished in 0.042 s -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 1000.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:44 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:44 INFO DAGScheduler:59 - waiting: Set(ResultStage 1001) -2016-06-13 19:15:44 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents for ResultStage 1001: List() -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ResultStage 1001 (MapPartitionsRDD[560] at map at VertexRDDImpl.scala:90), which is now runnable -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(4016) called with curMem=476990, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_170 stored as values in memory (estimated size 3.9 KB, free 730.1 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(2203) called with curMem=481006, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_170_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_170_piece0 in memory on localhost:49158 (size: 2.2 KB, free: 730.3 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 170 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1001 (MapPartitionsRDD[560] at map at VertexRDDImpl.scala:90) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 1001.0 with 2 tasks -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 1001.0 (TID 284, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 1001.0 (TID 285, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 1001.0 (TID 284) -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 1001.0 (TID 285) -2016-06-13 19:15:44 INFO CacheManager:59 - Partition rdd_555_0 not found, computing it -2016-06-13 19:15:44 INFO CacheManager:59 - Partition rdd_555_1 not found, computing it -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(1736) called with curMem=483209, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block rdd_555_1 stored as values in memory (estimated size 1736.0 B, free 730.1 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(1744) called with curMem=484945, maxMem=766075207 -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added rdd_555_1 in memory on localhost:49158 (size: 1736.0 B, free: 730.3 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - Block rdd_555_0 stored as values in memory (estimated size 1744.0 B, free 730.1 MB) -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 1001.0 (TID 285). 1862 bytes result sent to driver -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added rdd_555_0 in memory on localhost:49158 (size: 1744.0 B, free: 730.3 MB) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 1001.0 (TID 285) in 11 ms on localhost (1/2) -2016-06-13 19:15:44 INFO Executor:59 - Finished task 0.0 in stage 1001.0 (TID 284). 1862 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 1001.0 (TID 284) in 13 ms on localhost (2/2) -2016-06-13 19:15:44 INFO DAGScheduler:59 - ResultStage 1001 (reduce at VertexRDDImpl.scala:90) finished in 0.013 s -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 1001.0, whose tasks have all completed, from pool -2016-06-13 19:15:44 INFO DAGScheduler:59 - Job 93 finished: reduce at VertexRDDImpl.scala:90, took 0.086079 s -2016-06-13 19:15:44 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 48 is 152 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 46 is 154 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 45 is 154 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 44 is 152 bytes -2016-06-13 19:15:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 47 is 154 bytes -2016-06-13 19:15:44 INFO DAGScheduler:59 - Registering RDD 561 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Registering RDD 565 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Got job 94 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:15:44 INFO DAGScheduler:59 - Final stage: ResultStage 1009(sum at PowerIterationClustering.scala:369) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1002, ShuffleMapStage 1008, ShuffleMapStage 1006) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1008) -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1007 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[561] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(3720) called with curMem=486689, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_171 stored as values in memory (estimated size 3.6 KB, free 730.1 MB) -2016-06-13 19:15:44 INFO MemoryStore:59 - ensureFreeSpace(2111) called with curMem=490409, maxMem=766075207 -2016-06-13 19:15:44 INFO MemoryStore:59 - Block broadcast_171_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.1 MB) -2016-06-13 19:15:44 INFO BlockManagerInfo:59 - Added broadcast_171_piece0 in memory on localhost:49158 (size: 2.1 KB, free: 730.3 MB) -2016-06-13 19:15:44 INFO SparkContext:59 - Created broadcast 171 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1007 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[561] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:15:44 INFO TaskSchedulerImpl:59 - Adding task set 1007.0 with 2 tasks -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 1007.0 (TID 286, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:15:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 1007.0 (TID 287, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:15:44 INFO Executor:59 - Running task 0.0 in stage 1007.0 (TID 286) -2016-06-13 19:15:44 INFO Executor:59 - Running task 1.0 in stage 1007.0 (TID 287) -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_555_1 locally -2016-06-13 19:15:44 INFO BlockManager:59 - Found block rdd_555_0 locally -2016-06-13 19:15:44 INFO Executor:59 - Finished task 1.0 in stage 1007.0 (TID 287). 2254 bytes result sent to driver -2016-06-13 19:15:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 1007.0 (TID 287) in 18 ms on localhost (1/2) -2016-06-13 19:15:45 INFO Executor:59 - Finished task 0.0 in stage 1007.0 (TID 286). 2254 bytes result sent to driver -2016-06-13 19:15:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 1007.0 (TID 286) in 50 ms on localhost (2/2) -2016-06-13 19:15:45 INFO DAGScheduler:59 - ShuffleMapStage 1007 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.050 s -2016-06-13 19:15:45 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:45 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:45 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 1008, ResultStage 1009) -2016-06-13 19:15:45 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 1007.0, whose tasks have all completed, from pool -2016-06-13 19:15:45 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 1008: List() -2016-06-13 19:15:45 INFO DAGScheduler:59 - Missing parents for ResultStage 1009: List(ShuffleMapStage 1008) -2016-06-13 19:15:45 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1008 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[565] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:15:45 INFO MemoryStore:59 - ensureFreeSpace(5352) called with curMem=492520, maxMem=766075207 -2016-06-13 19:15:45 INFO MemoryStore:59 - Block broadcast_172 stored as values in memory (estimated size 5.2 KB, free 730.1 MB) -2016-06-13 19:15:45 INFO MemoryStore:59 - ensureFreeSpace(2725) called with curMem=497872, maxMem=766075207 -2016-06-13 19:15:45 INFO MemoryStore:59 - Block broadcast_172_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.1 MB) -2016-06-13 19:15:45 INFO BlockManagerInfo:59 - Added broadcast_172_piece0 in memory on localhost:49158 (size: 2.7 KB, free: 730.3 MB) -2016-06-13 19:15:45 INFO SparkContext:59 - Created broadcast 172 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1008 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[565] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:15:45 INFO TaskSchedulerImpl:59 - Adding task set 1008.0 with 2 tasks -2016-06-13 19:15:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 1008.0 (TID 288, localhost, PROCESS_LOCAL, 2532 bytes) -2016-06-13 19:15:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 1008.0 (TID 289, localhost, PROCESS_LOCAL, 2571 bytes) -2016-06-13 19:15:45 INFO Executor:59 - Running task 0.0 in stage 1008.0 (TID 288) -2016-06-13 19:15:45 INFO Executor:59 - Running task 1.0 in stage 1008.0 (TID 289) -2016-06-13 19:15:45 INFO CacheManager:59 - Partition rdd_558_1 not found, computing it -2016-06-13 19:15:45 INFO BlockManager:59 - Found block rdd_527_1 locally -2016-06-13 19:15:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:45 INFO CacheManager:59 - Partition rdd_558_0 not found, computing it -2016-06-13 19:15:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:45 INFO BlockManager:59 - Found block rdd_527_0 locally -2016-06-13 19:15:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:45 INFO MemoryStore:59 - ensureFreeSpace(2616) called with curMem=500597, maxMem=766075207 -2016-06-13 19:15:45 INFO MemoryStore:59 - Block rdd_558_1 stored as values in memory (estimated size 2.6 KB, free 730.1 MB) -2016-06-13 19:15:45 INFO BlockManagerInfo:59 - Added rdd_558_1 in memory on localhost:49158 (size: 2.6 KB, free: 730.3 MB) -2016-06-13 19:15:45 INFO MemoryStore:59 - ensureFreeSpace(2568) called with curMem=503213, maxMem=766075207 -2016-06-13 19:15:45 INFO MemoryStore:59 - Block rdd_558_0 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-06-13 19:15:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:45 INFO BlockManagerInfo:59 - Added rdd_558_0 in memory on localhost:49158 (size: 2.5 KB, free: 730.3 MB) -2016-06-13 19:15:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:45 INFO Executor:59 - Finished task 0.0 in stage 1008.0 (TID 288). 3050 bytes result sent to driver -2016-06-13 19:15:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 1008.0 (TID 288) in 39 ms on localhost (1/2) -2016-06-13 19:15:45 INFO Executor:59 - Finished task 1.0 in stage 1008.0 (TID 289). 3050 bytes result sent to driver -2016-06-13 19:15:45 INFO DAGScheduler:59 - ShuffleMapStage 1008 (mapPartitions at GraphImpl.scala:235) finished in 0.044 s -2016-06-13 19:15:45 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:45 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:45 INFO DAGScheduler:59 - waiting: Set(ResultStage 1009) -2016-06-13 19:15:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 1008.0 (TID 289) in 44 ms on localhost (2/2) -2016-06-13 19:15:45 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 1008.0, whose tasks have all completed, from pool -2016-06-13 19:15:45 INFO DAGScheduler:59 - Missing parents for ResultStage 1009: List() -2016-06-13 19:15:45 INFO DAGScheduler:59 - Submitting ResultStage 1009 (MapPartitionsRDD[570] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:15:45 INFO MemoryStore:59 - ensureFreeSpace(5144) called with curMem=505781, maxMem=766075207 -2016-06-13 19:15:45 INFO MemoryStore:59 - Block broadcast_173 stored as values in memory (estimated size 5.0 KB, free 730.1 MB) -2016-06-13 19:15:45 INFO MemoryStore:59 - ensureFreeSpace(2659) called with curMem=510925, maxMem=766075207 -2016-06-13 19:15:45 INFO MemoryStore:59 - Block broadcast_173_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.1 MB) -2016-06-13 19:15:45 INFO BlockManagerInfo:59 - Added broadcast_173_piece0 in memory on localhost:49158 (size: 2.6 KB, free: 730.3 MB) -2016-06-13 19:15:45 INFO SparkContext:59 - Created broadcast 173 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1009 (MapPartitionsRDD[570] at map at PowerIterationClustering.scala:369) -2016-06-13 19:15:45 INFO TaskSchedulerImpl:59 - Adding task set 1009.0 with 2 tasks -2016-06-13 19:15:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 1009.0 (TID 290, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:15:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 1009.0 (TID 291, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:15:45 INFO Executor:59 - Running task 0.0 in stage 1009.0 (TID 290) -2016-06-13 19:15:45 INFO Executor:59 - Running task 1.0 in stage 1009.0 (TID 291) -2016-06-13 19:15:45 INFO CacheManager:59 - Partition rdd_567_0 not found, computing it -2016-06-13 19:15:45 INFO CacheManager:59 - Partition rdd_567_1 not found, computing it -2016-06-13 19:15:45 INFO BlockManager:59 - Found block rdd_555_0 locally -2016-06-13 19:15:45 INFO BlockManager:59 - Found block rdd_555_1 locally -2016-06-13 19:15:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:15:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:45 INFO MemoryStore:59 - ensureFreeSpace(1792) called with curMem=513584, maxMem=766075207 -2016-06-13 19:15:45 INFO MemoryStore:59 - Block rdd_567_0 stored as values in memory (estimated size 1792.0 B, free 730.1 MB) -2016-06-13 19:15:45 INFO MemoryStore:59 - ensureFreeSpace(1784) called with curMem=515376, maxMem=766075207 -2016-06-13 19:15:45 INFO MemoryStore:59 - Block rdd_567_1 stored as values in memory (estimated size 1784.0 B, free 730.1 MB) -2016-06-13 19:15:45 INFO BlockManagerInfo:59 - Added rdd_567_0 in memory on localhost:49158 (size: 1792.0 B, free: 730.3 MB) -2016-06-13 19:15:45 INFO BlockManagerInfo:59 - Added rdd_567_1 in memory on localhost:49158 (size: 1784.0 B, free: 730.3 MB) -2016-06-13 19:15:45 INFO Executor:59 - Finished task 0.0 in stage 1009.0 (TID 290). 2880 bytes result sent to driver -2016-06-13 19:15:45 INFO Executor:59 - Finished task 1.0 in stage 1009.0 (TID 291). 2880 bytes result sent to driver -2016-06-13 19:15:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 1009.0 (TID 290) in 9 ms on localhost (1/2) -2016-06-13 19:15:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 1009.0 (TID 291) in 8 ms on localhost (2/2) -2016-06-13 19:15:45 INFO DAGScheduler:59 - ResultStage 1009 (sum at PowerIterationClustering.scala:369) finished in 0.009 s -2016-06-13 19:15:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 1009.0, whose tasks have all completed, from pool -2016-06-13 19:15:45 INFO DAGScheduler:59 - Job 94 finished: sum at PowerIterationClustering.scala:369, took 0.133867 s -2016-06-13 19:15:45 INFO PowerIterationClustering:59 - Iteration 0: norm(v) = 0.9333333333333333. -2016-06-13 19:15:45 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:15:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 48 is 152 bytes -2016-06-13 19:15:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 46 is 154 bytes -2016-06-13 19:15:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 45 is 154 bytes -2016-06-13 19:15:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 44 is 152 bytes -2016-06-13 19:15:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 47 is 154 bytes -2016-06-13 19:15:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 50 is 152 bytes -2016-06-13 19:15:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 49 is 154 bytes -2016-06-13 19:15:45 INFO DAGScheduler:59 - Got job 95 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:15:45 INFO DAGScheduler:59 - Final stage: ResultStage 1017(sum at PowerIterationClustering.scala:375) -2016-06-13 19:15:45 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1014, ShuffleMapStage 1016, ShuffleMapStage 1010) -2016-06-13 19:15:45 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:45 INFO DAGScheduler:59 - Submitting ResultStage 1017 (MapPartitionsRDD[581] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:15:45 INFO MemoryStore:59 - ensureFreeSpace(5880) called with curMem=517160, maxMem=766075207 -2016-06-13 19:15:45 INFO MemoryStore:59 - Block broadcast_174 stored as values in memory (estimated size 5.7 KB, free 730.1 MB) -2016-06-13 19:15:45 INFO MemoryStore:59 - ensureFreeSpace(2910) called with curMem=523040, maxMem=766075207 -2016-06-13 19:15:45 INFO MemoryStore:59 - Block broadcast_174_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.1 MB) -2016-06-13 19:15:45 INFO BlockManagerInfo:59 - Added broadcast_174_piece0 in memory on localhost:49158 (size: 2.8 KB, free: 730.3 MB) -2016-06-13 19:15:45 INFO SparkContext:59 - Created broadcast 174 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1017 (MapPartitionsRDD[581] at values at PowerIterationClustering.scala:375) -2016-06-13 19:15:45 INFO TaskSchedulerImpl:59 - Adding task set 1017.0 with 2 tasks -2016-06-13 19:15:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 1017.0 (TID 292, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:15:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 1017.0 (TID 293, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:15:45 INFO Executor:59 - Running task 1.0 in stage 1017.0 (TID 293) -2016-06-13 19:15:45 INFO Executor:59 - Running task 0.0 in stage 1017.0 (TID 292) -2016-06-13 19:15:45 INFO CacheManager:59 - Partition rdd_573_0 not found, computing it -2016-06-13 19:15:45 INFO BlockManager:59 - Found block rdd_555_0 locally -2016-06-13 19:15:45 INFO CacheManager:59 - Partition rdd_573_1 not found, computing it -2016-06-13 19:15:45 INFO BlockManager:59 - Found block rdd_567_0 locally -2016-06-13 19:15:45 INFO BlockManager:59 - Found block rdd_555_1 locally -2016-06-13 19:15:45 INFO BlockManager:59 - Found block rdd_567_1 locally -2016-06-13 19:15:45 INFO MemoryStore:59 - ensureFreeSpace(1744) called with curMem=525950, maxMem=766075207 -2016-06-13 19:15:45 INFO MemoryStore:59 - Block rdd_573_0 stored as values in memory (estimated size 1744.0 B, free 730.1 MB) -2016-06-13 19:15:45 INFO MemoryStore:59 - ensureFreeSpace(1736) called with curMem=527694, maxMem=766075207 -2016-06-13 19:15:45 INFO BlockManagerInfo:59 - Added rdd_573_0 in memory on localhost:49158 (size: 1744.0 B, free: 730.3 MB) -2016-06-13 19:15:45 INFO MemoryStore:59 - Block rdd_573_1 stored as values in memory (estimated size 1736.0 B, free 730.1 MB) -2016-06-13 19:15:45 INFO BlockManagerInfo:59 - Added rdd_573_1 in memory on localhost:49158 (size: 1736.0 B, free: 730.3 MB) -2016-06-13 19:15:45 INFO Executor:59 - Finished task 0.0 in stage 1017.0 (TID 292). 2664 bytes result sent to driver -2016-06-13 19:15:45 INFO Executor:59 - Finished task 1.0 in stage 1017.0 (TID 293). 2664 bytes result sent to driver -2016-06-13 19:15:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 1017.0 (TID 292) in 5 ms on localhost (1/2) -2016-06-13 19:15:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 1017.0 (TID 293) in 5 ms on localhost (2/2) -2016-06-13 19:15:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 1017.0, whose tasks have all completed, from pool -2016-06-13 19:15:45 INFO DAGScheduler:59 - ResultStage 1017 (sum at PowerIterationClustering.scala:375) finished in 0.006 s -2016-06-13 19:15:45 INFO DAGScheduler:59 - Job 95 finished: sum at PowerIterationClustering.scala:375, took 0.016273 s -2016-06-13 19:15:45 INFO PowerIterationClustering:59 - Iteration 0: delta = 0.47142857142857136. -2016-06-13 19:15:45 INFO PowerIterationClustering:59 - Iteration 0: diff(delta) = 1.7976931348623157E308. -2016-06-13 19:15:45 INFO SparkContext:59 - Starting job: collect at PowerIterationClusteringSuite.scala:157 -2016-06-13 19:15:45 INFO DAGScheduler:59 - Got job 96 (collect at PowerIterationClusteringSuite.scala:157) with 2 output partitions -2016-06-13 19:15:45 INFO DAGScheduler:59 - Final stage: ResultStage 1025(collect at PowerIterationClusteringSuite.scala:157) -2016-06-13 19:15:45 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1018, ShuffleMapStage 1022, ShuffleMapStage 1024) -2016-06-13 19:15:45 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:45 INFO DAGScheduler:59 - Submitting ResultStage 1025 (VertexRDDImpl[583] at RDD at VertexRDD.scala:57), which has no missing parents -2016-06-13 19:15:45 INFO MemoryStore:59 - ensureFreeSpace(5232) called with curMem=529430, maxMem=766075207 -2016-06-13 19:15:45 INFO MemoryStore:59 - Block broadcast_175 stored as values in memory (estimated size 5.1 KB, free 730.1 MB) -2016-06-13 19:15:45 INFO MemoryStore:59 - ensureFreeSpace(2626) called with curMem=534662, maxMem=766075207 -2016-06-13 19:15:45 INFO MemoryStore:59 - Block broadcast_175_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.1 MB) -2016-06-13 19:15:45 INFO BlockManagerInfo:59 - Added broadcast_175_piece0 in memory on localhost:49158 (size: 2.6 KB, free: 730.3 MB) -2016-06-13 19:15:45 INFO SparkContext:59 - Created broadcast 175 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1025 (VertexRDDImpl[583] at RDD at VertexRDD.scala:57) -2016-06-13 19:15:45 INFO TaskSchedulerImpl:59 - Adding task set 1025.0 with 2 tasks -2016-06-13 19:15:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 1025.0 (TID 294, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:15:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 1025.0 (TID 295, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:15:45 INFO Executor:59 - Running task 0.0 in stage 1025.0 (TID 294) -2016-06-13 19:15:45 INFO Executor:59 - Running task 1.0 in stage 1025.0 (TID 295) -2016-06-13 19:15:45 INFO BlockManager:59 - Found block rdd_567_0 locally -2016-06-13 19:15:45 INFO BlockManager:59 - Found block rdd_567_1 locally -2016-06-13 19:15:45 INFO Executor:59 - Finished task 0.0 in stage 1025.0 (TID 294). 2273 bytes result sent to driver -2016-06-13 19:15:45 INFO Executor:59 - Finished task 1.0 in stage 1025.0 (TID 295). 2273 bytes result sent to driver -2016-06-13 19:15:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 1025.0 (TID 294) in 3 ms on localhost (1/2) -2016-06-13 19:15:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 1025.0 (TID 295) in 3 ms on localhost (2/2) -2016-06-13 19:15:45 INFO DAGScheduler:59 - ResultStage 1025 (collect at PowerIterationClusteringSuite.scala:157) finished in 0.004 s -2016-06-13 19:15:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 1025.0, whose tasks have all completed, from pool -2016-06-13 19:15:45 INFO DAGScheduler:59 - Job 96 finished: collect at PowerIterationClusteringSuite.scala:157, took 0.011164 s -2016-06-13 19:15:46 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 19:15:46 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 19:15:46 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 19:15:46 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 19:15:46 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 19:15:46 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 19:15:46 INFO SparkContext:59 - Starting job: saveAsTextFile at PowerIterationClustering.scala:81 -2016-06-13 19:15:46 INFO DAGScheduler:59 - Got job 97 (saveAsTextFile at PowerIterationClustering.scala:81) with 1 output partitions -2016-06-13 19:15:46 INFO DAGScheduler:59 - Final stage: ResultStage 1026(saveAsTextFile at PowerIterationClustering.scala:81) -2016-06-13 19:15:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:15:46 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:46 INFO DAGScheduler:59 - Submitting ResultStage 1026 (MapPartitionsRDD[586] at saveAsTextFile at PowerIterationClustering.scala:81), which has no missing parents -2016-06-13 19:15:46 INFO MemoryStore:59 - ensureFreeSpace(126936) called with curMem=537288, maxMem=766075207 -2016-06-13 19:15:46 INFO MemoryStore:59 - Block broadcast_176 stored as values in memory (estimated size 124.0 KB, free 730.0 MB) -2016-06-13 19:15:46 INFO MemoryStore:59 - ensureFreeSpace(42522) called with curMem=664224, maxMem=766075207 -2016-06-13 19:15:46 INFO MemoryStore:59 - Block broadcast_176_piece0 stored as bytes in memory (estimated size 41.5 KB, free 729.9 MB) -2016-06-13 19:15:46 INFO BlockManagerInfo:59 - Added broadcast_176_piece0 in memory on localhost:49158 (size: 41.5 KB, free: 730.3 MB) -2016-06-13 19:15:46 INFO SparkContext:59 - Created broadcast 176 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:46 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1026 (MapPartitionsRDD[586] at saveAsTextFile at PowerIterationClustering.scala:81) -2016-06-13 19:15:46 INFO TaskSchedulerImpl:59 - Adding task set 1026.0 with 1 tasks -2016-06-13 19:15:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 1026.0 (TID 296, localhost, PROCESS_LOCAL, 2187 bytes) -2016-06-13 19:15:46 INFO Executor:59 - Running task 0.0 in stage 1026.0 (TID 296) -2016-06-13 19:15:46 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201606131915_1026_m_000000_296' to file:/C:/Users/xubo/AppData/Local/Temp/spark-67d62659-adad-48b4-a8b8-d96fe9c220a2/metadata/_temporary/0/task_201606131915_1026_m_000000 -2016-06-13 19:15:46 INFO SparkHadoopMapRedUtil:59 - attempt_201606131915_1026_m_000000_296: Committed -2016-06-13 19:15:46 INFO Executor:59 - Finished task 0.0 in stage 1026.0 (TID 296). 1864 bytes result sent to driver -2016-06-13 19:15:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 1026.0 (TID 296) in 172 ms on localhost (1/1) -2016-06-13 19:15:46 INFO DAGScheduler:59 - ResultStage 1026 (saveAsTextFile at PowerIterationClustering.scala:81) finished in 0.172 s -2016-06-13 19:15:46 INFO DAGScheduler:59 - Job 97 finished: saveAsTextFile at PowerIterationClustering.scala:81, took 0.257453 s -2016-06-13 19:15:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 1026.0, whose tasks have all completed, from pool -2016-06-13 19:15:48 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 19:15:48 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 19:15:48 INFO SparkContext:59 - Starting job: parquet at PowerIterationClustering.scala:84 -2016-06-13 19:15:48 INFO DAGScheduler:59 - Got job 98 (parquet at PowerIterationClustering.scala:84) with 4 output partitions -2016-06-13 19:15:48 INFO DAGScheduler:59 - Final stage: ResultStage 1027(parquet at PowerIterationClustering.scala:84) -2016-06-13 19:15:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:15:48 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:48 INFO DAGScheduler:59 - Submitting ResultStage 1027 (MapPartitionsRDD[587] at rddToDataFrameHolder at PowerIterationClustering.scala:83), which has no missing parents -2016-06-13 19:15:48 INFO MemoryStore:59 - ensureFreeSpace(66296) called with curMem=706746, maxMem=766075207 -2016-06-13 19:15:48 INFO MemoryStore:59 - Block broadcast_177 stored as values in memory (estimated size 64.7 KB, free 729.8 MB) -2016-06-13 19:15:48 INFO MemoryStore:59 - ensureFreeSpace(23071) called with curMem=773042, maxMem=766075207 -2016-06-13 19:15:48 INFO MemoryStore:59 - Block broadcast_177_piece0 stored as bytes in memory (estimated size 22.5 KB, free 729.8 MB) -2016-06-13 19:15:48 INFO BlockManagerInfo:59 - Added broadcast_177_piece0 in memory on localhost:49158 (size: 22.5 KB, free: 730.3 MB) -2016-06-13 19:15:48 INFO SparkContext:59 - Created broadcast 177 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:48 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1027 (MapPartitionsRDD[587] at rddToDataFrameHolder at PowerIterationClustering.scala:83) -2016-06-13 19:15:48 INFO TaskSchedulerImpl:59 - Adding task set 1027.0 with 4 tasks -2016-06-13 19:15:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 1027.0 (TID 297, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:15:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 1027.0 (TID 298, localhost, PROCESS_LOCAL, 2289 bytes) -2016-06-13 19:15:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 1027.0 (TID 299, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:15:48 INFO TaskSetManager:59 - Starting task 3.0 in stage 1027.0 (TID 300, localhost, PROCESS_LOCAL, 2289 bytes) -2016-06-13 19:15:48 INFO Executor:59 - Running task 1.0 in stage 1027.0 (TID 298) -2016-06-13 19:15:48 INFO Executor:59 - Running task 3.0 in stage 1027.0 (TID 300) -2016-06-13 19:15:48 INFO Executor:59 - Running task 2.0 in stage 1027.0 (TID 299) -2016-06-13 19:15:48 INFO Executor:59 - Running task 0.0 in stage 1027.0 (TID 297) -2016-06-13 19:15:48 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 19:15:48 INFO CodecConfig:151 - Compression: GZIP -2016-06-13 19:15:48 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 19:15:48 INFO CodecConfig:151 - Compression: GZIP -2016-06-13 19:15:48 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 19:15:48 INFO CodecConfig:151 - Compression: GZIP -2016-06-13 19:15:48 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 19:15:48 INFO CodecConfig:151 - Compression: GZIP -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Dictionary is on -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Validation is off -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Dictionary is on -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Validation is off -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Dictionary is on -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Validation is off -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Dictionary is on -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Validation is off -2016-06-13 19:15:48 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-06-13 19:15:48 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-06-13 19:15:48 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-06-13 19:15:48 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-06-13 19:15:48 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-06-13 19:15:48 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 60 -2016-06-13 19:15:48 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 40 -2016-06-13 19:15:48 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 60 -2016-06-13 19:15:48 INFO InternalParquetRecordWriter:151 - Flushing mem columnStore to file. allocated memory: 36 -2016-06-13 19:15:48 INFO ColumnChunkPageWriteStore:151 - written 72B for [id] INT64: 2 values, 22B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-06-13 19:15:48 INFO ColumnChunkPageWriteStore:151 - written 75B for [id] INT64: 3 values, 30B raw, 34B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-06-13 19:15:48 INFO ColumnChunkPageWriteStore:151 - written 64B for [cluster] INT32: 2 values, 14B raw, 31B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-06-13 19:15:48 INFO ColumnChunkPageWriteStore:151 - written 73B for [id] INT64: 2 values, 22B raw, 32B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-06-13 19:15:48 INFO ColumnChunkPageWriteStore:151 - written 61B for [cluster] INT32: 2 values, 8B raw, 28B comp, 1 pages, encodings: [BIT_PACKED, PLAIN_DICTIONARY, RLE], dic { 1 entries, 4B raw, 1B comp} -2016-06-13 19:15:48 INFO ColumnChunkPageWriteStore:151 - written 75B for [id] INT64: 3 values, 30B raw, 34B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-06-13 19:15:48 INFO ColumnChunkPageWriteStore:151 - written 65B for [cluster] INT32: 3 values, 18B raw, 32B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-06-13 19:15:48 INFO ColumnChunkPageWriteStore:151 - written 66B for [cluster] INT32: 3 values, 18B raw, 33B comp, 1 pages, encodings: [BIT_PACKED, PLAIN, RLE] -2016-06-13 19:15:48 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201606131915_1027_m_000000_0' to file:/C:/Users/xubo/AppData/Local/Temp/spark-67d62659-adad-48b4-a8b8-d96fe9c220a2/data/_temporary/0/task_201606131915_1027_m_000000 -2016-06-13 19:15:48 INFO SparkHadoopMapRedUtil:59 - attempt_201606131915_1027_m_000000_0: Committed -2016-06-13 19:15:48 INFO Executor:59 - Finished task 0.0 in stage 1027.0 (TID 297). 915 bytes result sent to driver -2016-06-13 19:15:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 1027.0 (TID 297) in 622 ms on localhost (1/4) -2016-06-13 19:15:48 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201606131915_1027_m_000003_0' to file:/C:/Users/xubo/AppData/Local/Temp/spark-67d62659-adad-48b4-a8b8-d96fe9c220a2/data/_temporary/0/task_201606131915_1027_m_000003 -2016-06-13 19:15:48 INFO SparkHadoopMapRedUtil:59 - attempt_201606131915_1027_m_000003_0: Committed -2016-06-13 19:15:48 INFO Executor:59 - Finished task 3.0 in stage 1027.0 (TID 300). 915 bytes result sent to driver -2016-06-13 19:15:48 INFO TaskSetManager:59 - Finished task 3.0 in stage 1027.0 (TID 300) in 646 ms on localhost (2/4) -2016-06-13 19:15:48 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201606131915_1027_m_000002_0' to file:/C:/Users/xubo/AppData/Local/Temp/spark-67d62659-adad-48b4-a8b8-d96fe9c220a2/data/_temporary/0/task_201606131915_1027_m_000002 -2016-06-13 19:15:48 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201606131915_1027_m_000001_0' to file:/C:/Users/xubo/AppData/Local/Temp/spark-67d62659-adad-48b4-a8b8-d96fe9c220a2/data/_temporary/0/task_201606131915_1027_m_000001 -2016-06-13 19:15:48 INFO SparkHadoopMapRedUtil:59 - attempt_201606131915_1027_m_000002_0: Committed -2016-06-13 19:15:48 INFO SparkHadoopMapRedUtil:59 - attempt_201606131915_1027_m_000001_0: Committed -2016-06-13 19:15:48 INFO Executor:59 - Finished task 1.0 in stage 1027.0 (TID 298). 915 bytes result sent to driver -2016-06-13 19:15:48 INFO Executor:59 - Finished task 2.0 in stage 1027.0 (TID 299). 915 bytes result sent to driver -2016-06-13 19:15:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 1027.0 (TID 298) in 649 ms on localhost (3/4) -2016-06-13 19:15:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 1027.0 (TID 299) in 650 ms on localhost (4/4) -2016-06-13 19:15:48 INFO DAGScheduler:59 - ResultStage 1027 (parquet at PowerIterationClustering.scala:84) finished in 0.650 s -2016-06-13 19:15:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 1027.0, whose tasks have all completed, from pool -2016-06-13 19:15:48 INFO DAGScheduler:59 - Job 98 finished: parquet at PowerIterationClustering.scala:84, took 0.688486 s -2016-06-13 19:15:49 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-06-13 19:15:49 INFO DefaultWriterContainer:59 - Job job_201606131915_0000 committed. -2016-06-13 19:15:49 INFO ParquetRelation:59 - Listing file:/C:/Users/xubo/AppData/Local/Temp/spark-67d62659-adad-48b4-a8b8-d96fe9c220a2/data on driver -2016-06-13 19:15:49 INFO ParquetRelation:59 - Listing file:/C:/Users/xubo/AppData/Local/Temp/spark-67d62659-adad-48b4-a8b8-d96fe9c220a2/data on driver -2016-06-13 19:15:49 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=796113, maxMem=766075207 -2016-06-13 19:15:49 INFO MemoryStore:59 - Block broadcast_178 stored as values in memory (estimated size 225.3 KB, free 729.6 MB) -2016-06-13 19:15:49 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1026817, maxMem=766075207 -2016-06-13 19:15:49 INFO MemoryStore:59 - Block broadcast_178_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.6 MB) -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Added broadcast_178_piece0 in memory on localhost:49158 (size: 19.3 KB, free: 730.2 MB) -2016-06-13 19:15:49 INFO SparkContext:59 - Created broadcast 178 from textFile at modelSaveLoad.scala:129 -2016-06-13 19:15:49 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 19:15:49 INFO SparkContext:59 - Starting job: first at modelSaveLoad.scala:129 -2016-06-13 19:15:49 INFO DAGScheduler:59 - Got job 99 (first at modelSaveLoad.scala:129) with 1 output partitions -2016-06-13 19:15:49 INFO DAGScheduler:59 - Final stage: ResultStage 1028(first at modelSaveLoad.scala:129) -2016-06-13 19:15:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:15:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:49 INFO DAGScheduler:59 - Submitting ResultStage 1028 (MapPartitionsRDD[590] at textFile at modelSaveLoad.scala:129), which has no missing parents -2016-06-13 19:15:49 INFO MemoryStore:59 - ensureFreeSpace(3224) called with curMem=1046605, maxMem=766075207 -2016-06-13 19:15:49 INFO MemoryStore:59 - Block broadcast_179 stored as values in memory (estimated size 3.1 KB, free 729.6 MB) -2016-06-13 19:15:49 INFO MemoryStore:59 - ensureFreeSpace(1905) called with curMem=1049829, maxMem=766075207 -2016-06-13 19:15:49 INFO MemoryStore:59 - Block broadcast_179_piece0 stored as bytes in memory (estimated size 1905.0 B, free 729.6 MB) -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Added broadcast_179_piece0 in memory on localhost:49158 (size: 1905.0 B, free: 730.2 MB) -2016-06-13 19:15:49 INFO SparkContext:59 - Created broadcast 179 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:49 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1028 (MapPartitionsRDD[590] at textFile at modelSaveLoad.scala:129) -2016-06-13 19:15:49 INFO TaskSchedulerImpl:59 - Adding task set 1028.0 with 1 tasks -2016-06-13 19:15:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 1028.0 (TID 301, localhost, PROCESS_LOCAL, 2210 bytes) -2016-06-13 19:15:49 INFO Executor:59 - Running task 0.0 in stage 1028.0 (TID 301) -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_152_piece0 on localhost:49158 in memory (size: 4.3 KB, free: 730.2 MB) -2016-06-13 19:15:49 INFO HadoopRDD:59 - Input split: file:/C:/Users/xubo/AppData/Local/Temp/spark-67d62659-adad-48b4-a8b8-d96fe9c220a2/metadata/part-00000:0+49 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_177_piece0 on localhost:49158 in memory (size: 22.5 KB, free: 730.3 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 170 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_176_piece0 on localhost:49158 in memory (size: 41.5 KB, free: 730.3 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 169 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_175_piece0 on localhost:49158 in memory (size: 2.6 KB, free: 730.3 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 168 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_174_piece0 on localhost:49158 in memory (size: 2.8 KB, free: 730.3 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 167 -2016-06-13 19:15:49 INFO Executor:59 - Finished task 0.0 in stage 1028.0 (TID 301). 2144 bytes result sent to driver -2016-06-13 19:15:49 INFO BlockManager:59 - Removing RDD 573 -2016-06-13 19:15:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 1028.0 (TID 301) in 23 ms on localhost (1/1) -2016-06-13 19:15:49 INFO DAGScheduler:59 - ResultStage 1028 (first at modelSaveLoad.scala:129) finished in 0.024 s -2016-06-13 19:15:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 1028.0, whose tasks have all completed, from pool -2016-06-13 19:15:49 INFO DAGScheduler:59 - Job 99 finished: first at modelSaveLoad.scala:129, took 0.063438 s -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned RDD 573 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_173_piece0 on localhost:49158 in memory (size: 2.6 KB, free: 730.3 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 166 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_172_piece0 on localhost:49158 in memory (size: 2.7 KB, free: 730.3 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 165 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_171_piece0 on localhost:49158 in memory (size: 2.1 KB, free: 730.3 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 164 -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned shuffle 50 -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned shuffle 49 -2016-06-13 19:15:49 INFO BlockManager:59 - Removing RDD 567 -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned RDD 567 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_170_piece0 on localhost:49158 in memory (size: 2.2 KB, free: 730.3 MB) -2016-06-13 19:15:49 INFO ParquetRelation:59 - Listing file:/C:/Users/xubo/AppData/Local/Temp/spark-67d62659-adad-48b4-a8b8-d96fe9c220a2/data on driver -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 163 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_169_piece0 on localhost:49158 in memory (size: 984.0 B, free: 730.3 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 162 -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned shuffle 48 -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned shuffle 47 -2016-06-13 19:15:49 INFO BlockManager:59 - Removing RDD 558 -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned RDD 558 -2016-06-13 19:15:49 INFO BlockManager:59 - Removing RDD 555 -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned RDD 555 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_167_piece0 on localhost:49158 in memory (size: 2.5 KB, free: 730.3 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 160 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_166_piece0 on localhost:49158 in memory (size: 2.3 KB, free: 730.3 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 159 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_165_piece0 on localhost:49158 in memory (size: 1972.0 B, free: 730.3 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 158 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_164_piece0 on localhost:49158 in memory (size: 1777.0 B, free: 730.3 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 157 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_163_piece0 on localhost:49158 in memory (size: 3.5 KB, free: 730.3 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 156 -2016-06-13 19:15:49 INFO BlockManager:59 - Removing RDD 522 -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned RDD 522 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_162_piece0 on localhost:49158 in memory (size: 1591.0 B, free: 730.3 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 155 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_161_piece0 on localhost:49158 in memory (size: 4.3 KB, free: 730.3 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 154 -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned shuffle 43 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_160_piece0 on localhost:49158 in memory (size: 339.0 B, free: 730.3 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 153 -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 152 -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 151 -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 150 -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 149 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_159_piece0 on localhost:49158 in memory (size: 1383.0 B, free: 730.4 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 148 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_158_piece0 on localhost:49158 in memory (size: 4.0 KB, free: 730.4 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 147 -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned shuffle 42 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_157_piece0 on localhost:49158 in memory (size: 708.0 B, free: 730.4 MB) -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_156_piece0 on localhost:49158 in memory (size: 4.5 KB, free: 730.4 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 146 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_155_piece0 on localhost:49158 in memory (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 145 -2016-06-13 19:15:49 INFO BlockManager:59 - Removing RDD 515 -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned RDD 515 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_154_piece0 on localhost:49158 in memory (size: 236.0 B, free: 730.4 MB) -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_153_piece0 on localhost:49158 in memory (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 144 -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 143 -2016-06-13 19:15:49 INFO BlockManager:59 - Removing RDD 511 -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned RDD 511 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_151_piece0 on localhost:49158 in memory (size: 429.0 B, free: 730.4 MB) -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_150_piece0 on localhost:49158 in memory (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 142 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_149_piece0 on localhost:49158 in memory (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 141 -2016-06-13 19:15:49 INFO BlockManager:59 - Removing RDD 507 -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned RDD 507 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_148_piece0 on localhost:49158 in memory (size: 582.0 B, free: 730.4 MB) -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_147_piece0 on localhost:49158 in memory (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 140 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_146_piece0 on localhost:49158 in memory (size: 4.2 KB, free: 730.4 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 139 -2016-06-13 19:15:49 INFO BlockManager:59 - Removing RDD 503 -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned RDD 503 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_145_piece0 on localhost:49158 in memory (size: 600.0 B, free: 730.4 MB) -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_144_piece0 on localhost:49158 in memory (size: 4.2 KB, free: 730.4 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 138 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_143_piece0 on localhost:49158 in memory (size: 4.1 KB, free: 730.4 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 137 -2016-06-13 19:15:49 INFO BlockManager:59 - Removing RDD 499 -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned RDD 499 -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_142_piece0 on localhost:49158 in memory (size: 487.0 B, free: 730.4 MB) -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Removed broadcast_141_piece0 on localhost:49158 in memory (size: 9.9 KB, free: 730.4 MB) -2016-06-13 19:15:49 INFO ContextCleaner:59 - Cleaned accumulator 136 -2016-06-13 19:15:49 INFO SparkContext:59 - Starting job: parquet at PowerIterationClustering.scala:97 -2016-06-13 19:15:49 INFO DAGScheduler:59 - Got job 100 (parquet at PowerIterationClustering.scala:97) with 4 output partitions -2016-06-13 19:15:49 INFO DAGScheduler:59 - Final stage: ResultStage 1029(parquet at PowerIterationClustering.scala:97) -2016-06-13 19:15:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:15:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:15:49 INFO DAGScheduler:59 - Submitting ResultStage 1029 (MapPartitionsRDD[592] at parquet at PowerIterationClustering.scala:97), which has no missing parents -2016-06-13 19:15:49 INFO MemoryStore:59 - ensureFreeSpace(63000) called with curMem=459371, maxMem=766075207 -2016-06-13 19:15:49 INFO MemoryStore:59 - Block broadcast_180 stored as values in memory (estimated size 61.5 KB, free 730.1 MB) -2016-06-13 19:15:49 INFO MemoryStore:59 - ensureFreeSpace(21073) called with curMem=522371, maxMem=766075207 -2016-06-13 19:15:49 INFO MemoryStore:59 - Block broadcast_180_piece0 stored as bytes in memory (estimated size 20.6 KB, free 730.1 MB) -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Added broadcast_180_piece0 in memory on localhost:49158 (size: 20.6 KB, free: 730.4 MB) -2016-06-13 19:15:49 INFO SparkContext:59 - Created broadcast 180 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:49 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1029 (MapPartitionsRDD[592] at parquet at PowerIterationClustering.scala:97) -2016-06-13 19:15:49 INFO TaskSchedulerImpl:59 - Adding task set 1029.0 with 4 tasks -2016-06-13 19:15:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 1029.0 (TID 302, localhost, PROCESS_LOCAL, 2083 bytes) -2016-06-13 19:15:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 1029.0 (TID 303, localhost, PROCESS_LOCAL, 2083 bytes) -2016-06-13 19:15:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 1029.0 (TID 304, localhost, PROCESS_LOCAL, 2083 bytes) -2016-06-13 19:15:49 INFO TaskSetManager:59 - Starting task 3.0 in stage 1029.0 (TID 305, localhost, PROCESS_LOCAL, 2252 bytes) -2016-06-13 19:15:49 INFO Executor:59 - Running task 2.0 in stage 1029.0 (TID 304) -2016-06-13 19:15:49 INFO Executor:59 - Running task 3.0 in stage 1029.0 (TID 305) -2016-06-13 19:15:49 INFO Executor:59 - Running task 1.0 in stage 1029.0 (TID 303) -2016-06-13 19:15:49 INFO Executor:59 - Running task 0.0 in stage 1029.0 (TID 302) -2016-06-13 19:15:49 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-06-13 19:15:49 INFO Executor:59 - Finished task 0.0 in stage 1029.0 (TID 302). 936 bytes result sent to driver -2016-06-13 19:15:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 1029.0 (TID 302) in 40 ms on localhost (1/4) -2016-06-13 19:15:49 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-06-13 19:15:49 INFO Executor:59 - Finished task 1.0 in stage 1029.0 (TID 303). 936 bytes result sent to driver -2016-06-13 19:15:49 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-06-13 19:15:49 INFO Executor:59 - Finished task 2.0 in stage 1029.0 (TID 304). 936 bytes result sent to driver -2016-06-13 19:15:49 INFO ParquetFileReader:151 - Initiating action with parallelism: 5 -2016-06-13 19:15:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 1029.0 (TID 303) in 53 ms on localhost (2/4) -2016-06-13 19:15:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 1029.0 (TID 304) in 53 ms on localhost (3/4) -2016-06-13 19:15:49 INFO Executor:59 - Finished task 3.0 in stage 1029.0 (TID 305). 1701 bytes result sent to driver -2016-06-13 19:15:49 INFO TaskSetManager:59 - Finished task 3.0 in stage 1029.0 (TID 305) in 73 ms on localhost (4/4) -2016-06-13 19:15:49 INFO DAGScheduler:59 - ResultStage 1029 (parquet at PowerIterationClustering.scala:97) finished in 0.074 s -2016-06-13 19:15:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 1029.0, whose tasks have all completed, from pool -2016-06-13 19:15:49 INFO DAGScheduler:59 - Job 100 finished: parquet at PowerIterationClustering.scala:97, took 0.096429 s -2016-06-13 19:15:49 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=543444, maxMem=766075207 -2016-06-13 19:15:49 INFO MemoryStore:59 - Block broadcast_181 stored as values in memory (estimated size 86.4 KB, free 730.0 MB) -2016-06-13 19:15:49 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=631956, maxMem=766075207 -2016-06-13 19:15:49 INFO MemoryStore:59 - Block broadcast_181_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.0 MB) -2016-06-13 19:15:49 INFO BlockManagerInfo:59 - Added broadcast_181_piece0 in memory on localhost:49158 (size: 19.3 KB, free: 730.4 MB) -2016-06-13 19:15:49 INFO SparkContext:59 - Created broadcast 181 from map at PowerIterationClustering.scala:100 -2016-06-13 19:15:50 INFO deprecation:1049 - mapred.min.split.size is deprecated. Instead, use mapreduce.input.fileinputformat.split.minsize -2016-06-13 19:15:50 INFO ParquetRelation:59 - Reading Parquet file(s) from file:/C:/Users/xubo/AppData/Local/Temp/spark-67d62659-adad-48b4-a8b8-d96fe9c220a2/data/part-r-00000-f478b09f-bd50-4d42-b405-b5751cfd1ed1.gz.parquet, file:/C:/Users/xubo/AppData/Local/Temp/spark-67d62659-adad-48b4-a8b8-d96fe9c220a2/data/part-r-00003-f478b09f-bd50-4d42-b405-b5751cfd1ed1.gz.parquet, file:/C:/Users/xubo/AppData/Local/Temp/spark-67d62659-adad-48b4-a8b8-d96fe9c220a2/data/part-r-00001-f478b09f-bd50-4d42-b405-b5751cfd1ed1.gz.parquet, file:/C:/Users/xubo/AppData/Local/Temp/spark-67d62659-adad-48b4-a8b8-d96fe9c220a2/data/part-r-00002-f478b09f-bd50-4d42-b405-b5751cfd1ed1.gz.parquet -2016-06-13 19:15:50 INFO SparkContext:59 - Starting job: count at PowerIterationClusteringSuite.scala:193 -2016-06-13 19:15:50 INFO DAGScheduler:59 - Registering RDD 596 (map at PowerIterationClusteringSuite.scala:190) -2016-06-13 19:15:50 INFO DAGScheduler:59 - Registering RDD 597 (map at PowerIterationClusteringSuite.scala:191) -2016-06-13 19:15:50 INFO DAGScheduler:59 - Got job 101 (count at PowerIterationClusteringSuite.scala:193) with 4 output partitions -2016-06-13 19:15:50 INFO DAGScheduler:59 - Final stage: ResultStage 1032(count at PowerIterationClusteringSuite.scala:193) -2016-06-13 19:15:50 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1030, ShuffleMapStage 1031) -2016-06-13 19:15:50 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1030, ShuffleMapStage 1031) -2016-06-13 19:15:50 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1030 (MapPartitionsRDD[596] at map at PowerIterationClusteringSuite.scala:190), which has no missing parents -2016-06-13 19:15:50 INFO MemoryStore:59 - ensureFreeSpace(2352) called with curMem=651744, maxMem=766075207 -2016-06-13 19:15:50 INFO MemoryStore:59 - Block broadcast_182 stored as values in memory (estimated size 2.3 KB, free 730.0 MB) -2016-06-13 19:15:50 INFO MemoryStore:59 - ensureFreeSpace(1425) called with curMem=654096, maxMem=766075207 -2016-06-13 19:15:50 INFO MemoryStore:59 - Block broadcast_182_piece0 stored as bytes in memory (estimated size 1425.0 B, free 730.0 MB) -2016-06-13 19:15:50 INFO BlockManagerInfo:59 - Added broadcast_182_piece0 in memory on localhost:49158 (size: 1425.0 B, free: 730.4 MB) -2016-06-13 19:15:50 INFO SparkContext:59 - Created broadcast 182 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:50 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 1030 (MapPartitionsRDD[596] at map at PowerIterationClusteringSuite.scala:190) -2016-06-13 19:15:50 INFO TaskSchedulerImpl:59 - Adding task set 1030.0 with 4 tasks -2016-06-13 19:15:50 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1031 (MapPartitionsRDD[597] at map at PowerIterationClusteringSuite.scala:191), which has no missing parents -2016-06-13 19:15:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 1030.0 (TID 306, localhost, PROCESS_LOCAL, 2260 bytes) -2016-06-13 19:15:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 1030.0 (TID 307, localhost, PROCESS_LOCAL, 2278 bytes) -2016-06-13 19:15:50 INFO TaskSetManager:59 - Starting task 2.0 in stage 1030.0 (TID 308, localhost, PROCESS_LOCAL, 2260 bytes) -2016-06-13 19:15:50 INFO TaskSetManager:59 - Starting task 3.0 in stage 1030.0 (TID 309, localhost, PROCESS_LOCAL, 2278 bytes) -2016-06-13 19:15:50 INFO Executor:59 - Running task 2.0 in stage 1030.0 (TID 308) -2016-06-13 19:15:50 INFO Executor:59 - Running task 0.0 in stage 1030.0 (TID 306) -2016-06-13 19:15:50 INFO Executor:59 - Running task 1.0 in stage 1030.0 (TID 307) -2016-06-13 19:15:50 INFO Executor:59 - Running task 3.0 in stage 1030.0 (TID 309) -2016-06-13 19:15:50 INFO MemoryStore:59 - ensureFreeSpace(5128) called with curMem=655521, maxMem=766075207 -2016-06-13 19:15:50 INFO MemoryStore:59 - Block broadcast_183 stored as values in memory (estimated size 5.0 KB, free 730.0 MB) -2016-06-13 19:15:50 INFO MemoryStore:59 - ensureFreeSpace(2887) called with curMem=660649, maxMem=766075207 -2016-06-13 19:15:50 INFO MemoryStore:59 - Block broadcast_183_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.0 MB) -2016-06-13 19:15:50 INFO BlockManagerInfo:59 - Added broadcast_183_piece0 in memory on localhost:49158 (size: 2.8 KB, free: 730.4 MB) -2016-06-13 19:15:50 INFO SparkContext:59 - Created broadcast 183 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:50 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 1031 (MapPartitionsRDD[597] at map at PowerIterationClusteringSuite.scala:191) -2016-06-13 19:15:50 INFO TaskSchedulerImpl:59 - Adding task set 1031.0 with 4 tasks -2016-06-13 19:15:50 INFO Executor:59 - Finished task 2.0 in stage 1030.0 (TID 308). 1161 bytes result sent to driver -2016-06-13 19:15:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 1031.0 (TID 310, localhost, PROCESS_LOCAL, 2270 bytes) -2016-06-13 19:15:50 INFO TaskSetManager:59 - Finished task 2.0 in stage 1030.0 (TID 308) in 48 ms on localhost (1/4) -2016-06-13 19:15:50 INFO Executor:59 - Running task 0.0 in stage 1031.0 (TID 310) -2016-06-13 19:15:50 INFO Executor:59 - Finished task 0.0 in stage 1030.0 (TID 306). 1161 bytes result sent to driver -2016-06-13 19:15:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 1031.0 (TID 311, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:15:50 INFO Executor:59 - Running task 1.0 in stage 1031.0 (TID 311) -2016-06-13 19:15:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 1030.0 (TID 306) in 51 ms on localhost (2/4) -2016-06-13 19:15:50 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/C:/Users/xubo/AppData/Local/Temp/spark-67d62659-adad-48b4-a8b8-d96fe9c220a2/data/part-r-00003-f478b09f-bd50-4d42-b405-b5751cfd1ed1.gz.parquet start: 0 end: 533 length: 533 hosts: []} -2016-06-13 19:15:50 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/C:/Users/xubo/AppData/Local/Temp/spark-67d62659-adad-48b4-a8b8-d96fe9c220a2/data/part-r-00000-f478b09f-bd50-4d42-b405-b5751cfd1ed1.gz.parquet start: 0 end: 527 length: 527 hosts: []} -2016-06-13 19:15:50 INFO Executor:59 - Finished task 1.0 in stage 1030.0 (TID 307). 1161 bytes result sent to driver -2016-06-13 19:15:50 INFO TaskSetManager:59 - Starting task 2.0 in stage 1031.0 (TID 312, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:15:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 1030.0 (TID 307) in 66 ms on localhost (3/4) -2016-06-13 19:15:50 INFO Executor:59 - Running task 2.0 in stage 1031.0 (TID 312) -2016-06-13 19:15:50 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-06-13 19:15:50 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-06-13 19:15:50 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/C:/Users/xubo/AppData/Local/Temp/spark-67d62659-adad-48b4-a8b8-d96fe9c220a2/data/part-r-00001-f478b09f-bd50-4d42-b405-b5751cfd1ed1.gz.parquet start: 0 end: 532 length: 532 hosts: []} -2016-06-13 19:15:50 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-06-13 19:15:50 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int64 id; - optional int32 cluster; -} - -Catalyst form: -StructType(StructField(id,LongType,true), StructField(cluster,IntegerType,true)) - -2016-06-13 19:15:50 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int64 id; - optional int32 cluster; -} - -Catalyst form: -StructType(StructField(id,LongType,true), StructField(cluster,IntegerType,true)) - -2016-06-13 19:15:50 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int64 id; - optional int32 cluster; -} - -Catalyst form: -StructType(StructField(id,LongType,true), StructField(cluster,IntegerType,true)) - -2016-06-13 19:15:50 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 3 records. -2016-06-13 19:15:50 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 2 records. -2016-06-13 19:15:50 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 3 records. -2016-06-13 19:15:50 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-06-13 19:15:50 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-06-13 19:15:50 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-06-13 19:15:50 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-06-13 19:15:50 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-06-13 19:15:50 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-06-13 19:15:50 INFO InternalParquetRecordReader:151 - block read in memory in 8 ms. row count = 3 -2016-06-13 19:15:50 INFO InternalParquetRecordReader:151 - block read in memory in 8 ms. row count = 3 -2016-06-13 19:15:50 INFO InternalParquetRecordReader:151 - block read in memory in 8 ms. row count = 2 -2016-06-13 19:15:50 INFO Executor:59 - Finished task 3.0 in stage 1030.0 (TID 309). 1161 bytes result sent to driver -2016-06-13 19:15:50 INFO TaskSetManager:59 - Starting task 3.0 in stage 1031.0 (TID 313, localhost, PROCESS_LOCAL, 2268 bytes) -2016-06-13 19:15:50 INFO Executor:59 - Running task 3.0 in stage 1031.0 (TID 313) -2016-06-13 19:15:50 INFO TaskSetManager:59 - Finished task 3.0 in stage 1030.0 (TID 309) in 120 ms on localhost (4/4) -2016-06-13 19:15:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 1030.0, whose tasks have all completed, from pool -2016-06-13 19:15:50 INFO DAGScheduler:59 - ShuffleMapStage 1030 (map at PowerIterationClusteringSuite.scala:190) finished in 0.123 s -2016-06-13 19:15:50 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:50 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 1031) -2016-06-13 19:15:50 INFO DAGScheduler:59 - waiting: Set(ResultStage 1032) -2016-06-13 19:15:50 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:50 INFO DAGScheduler:59 - Missing parents for ResultStage 1032: List(ShuffleMapStage 1031) -2016-06-13 19:15:50 INFO ParquetRelation$$anonfun$buildScan$1$$anon$1:59 - Input split: ParquetInputSplit{part: file:/C:/Users/xubo/AppData/Local/Temp/spark-67d62659-adad-48b4-a8b8-d96fe9c220a2/data/part-r-00002-f478b09f-bd50-4d42-b405-b5751cfd1ed1.gz.parquet start: 0 end: 562 length: 562 hosts: []} -2016-06-13 19:15:50 WARN ParquetRecordReader:193 - Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl -2016-06-13 19:15:50 INFO CatalystReadSupport:59 - Going to read the following fields from the Parquet file: - -Parquet form: -message root { - optional int64 id; - optional int32 cluster; -} - -Catalyst form: -StructType(StructField(id,LongType,true), StructField(cluster,IntegerType,true)) - -2016-06-13 19:15:50 INFO InternalParquetRecordReader:151 - RecordReader initialized will read a total of 2 records. -2016-06-13 19:15:50 INFO InternalParquetRecordReader:151 - at row 0. reading next block -2016-06-13 19:15:50 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-06-13 19:15:50 INFO InternalParquetRecordReader:151 - block read in memory in 5 ms. row count = 2 -2016-06-13 19:15:50 INFO Executor:59 - Finished task 1.0 in stage 1031.0 (TID 311). 2256 bytes result sent to driver -2016-06-13 19:15:50 INFO Executor:59 - Finished task 2.0 in stage 1031.0 (TID 312). 2256 bytes result sent to driver -2016-06-13 19:15:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 1031.0 (TID 311) in 126 ms on localhost (1/4) -2016-06-13 19:15:50 INFO TaskSetManager:59 - Finished task 2.0 in stage 1031.0 (TID 312) in 111 ms on localhost (2/4) -2016-06-13 19:15:50 INFO Executor:59 - Finished task 0.0 in stage 1031.0 (TID 310). 2256 bytes result sent to driver -2016-06-13 19:15:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 1031.0 (TID 310) in 145 ms on localhost (3/4) -2016-06-13 19:15:50 INFO Executor:59 - Finished task 3.0 in stage 1031.0 (TID 313). 2256 bytes result sent to driver -2016-06-13 19:15:50 INFO TaskSetManager:59 - Finished task 3.0 in stage 1031.0 (TID 313) in 89 ms on localhost (4/4) -2016-06-13 19:15:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 1031.0, whose tasks have all completed, from pool -2016-06-13 19:15:50 INFO DAGScheduler:59 - ShuffleMapStage 1031 (map at PowerIterationClusteringSuite.scala:191) finished in 0.191 s -2016-06-13 19:15:50 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:15:50 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:15:50 INFO DAGScheduler:59 - waiting: Set(ResultStage 1032) -2016-06-13 19:15:50 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:15:50 INFO DAGScheduler:59 - Missing parents for ResultStage 1032: List() -2016-06-13 19:15:50 INFO DAGScheduler:59 - Submitting ResultStage 1032 (MapPartitionsRDD[601] at filter at PowerIterationClusteringSuite.scala:192), which is now runnable -2016-06-13 19:15:50 INFO MemoryStore:59 - ensureFreeSpace(2936) called with curMem=663536, maxMem=766075207 -2016-06-13 19:15:50 INFO MemoryStore:59 - Block broadcast_184 stored as values in memory (estimated size 2.9 KB, free 730.0 MB) -2016-06-13 19:15:50 INFO MemoryStore:59 - ensureFreeSpace(1627) called with curMem=666472, maxMem=766075207 -2016-06-13 19:15:50 INFO MemoryStore:59 - Block broadcast_184_piece0 stored as bytes in memory (estimated size 1627.0 B, free 729.9 MB) -2016-06-13 19:15:50 INFO BlockManagerInfo:59 - Added broadcast_184_piece0 in memory on localhost:49158 (size: 1627.0 B, free: 730.4 MB) -2016-06-13 19:15:50 INFO SparkContext:59 - Created broadcast 184 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:15:50 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1032 (MapPartitionsRDD[601] at filter at PowerIterationClusteringSuite.scala:192) -2016-06-13 19:15:50 INFO TaskSchedulerImpl:59 - Adding task set 1032.0 with 4 tasks -2016-06-13 19:15:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 1032.0 (TID 314, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 19:15:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 1032.0 (TID 315, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 19:15:50 INFO TaskSetManager:59 - Starting task 2.0 in stage 1032.0 (TID 316, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 19:15:50 INFO TaskSetManager:59 - Starting task 3.0 in stage 1032.0 (TID 317, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 19:15:50 INFO Executor:59 - Running task 2.0 in stage 1032.0 (TID 316) -2016-06-13 19:15:50 INFO Executor:59 - Running task 3.0 in stage 1032.0 (TID 317) -2016-06-13 19:15:50 INFO Executor:59 - Running task 0.0 in stage 1032.0 (TID 314) -2016-06-13 19:15:50 INFO Executor:59 - Running task 1.0 in stage 1032.0 (TID 315) -2016-06-13 19:15:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 19:15:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 19:15:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 19:15:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 19:15:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 19:15:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 19:15:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 19:15:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:15:50 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 19:15:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:15:50 INFO Executor:59 - Finished task 3.0 in stage 1032.0 (TID 317). 1203 bytes result sent to driver -2016-06-13 19:15:50 INFO Executor:59 - Finished task 1.0 in stage 1032.0 (TID 315). 1203 bytes result sent to driver -2016-06-13 19:15:50 INFO Executor:59 - Finished task 0.0 in stage 1032.0 (TID 314). 1203 bytes result sent to driver -2016-06-13 19:15:50 INFO Executor:59 - Finished task 2.0 in stage 1032.0 (TID 316). 1203 bytes result sent to driver -2016-06-13 19:15:50 INFO TaskSetManager:59 - Finished task 3.0 in stage 1032.0 (TID 317) in 26 ms on localhost (1/4) -2016-06-13 19:15:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 1032.0 (TID 314) in 28 ms on localhost (2/4) -2016-06-13 19:15:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 1032.0 (TID 315) in 27 ms on localhost (3/4) -2016-06-13 19:15:50 INFO TaskSetManager:59 - Finished task 2.0 in stage 1032.0 (TID 316) in 27 ms on localhost (4/4) -2016-06-13 19:15:50 INFO DAGScheduler:59 - ResultStage 1032 (count at PowerIterationClusteringSuite.scala:193) finished in 0.029 s -2016-06-13 19:15:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 1032.0, whose tasks have all completed, from pool -2016-06-13 19:15:50 INFO DAGScheduler:59 - Job 101 finished: count at PowerIterationClusteringSuite.scala:193, took 0.257058 s -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL2/execution/json,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL2/execution,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL2/json,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL2,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution/json,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/execution,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1/json,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL1,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 19:15:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 19:15:50 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 19:15:50 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 19:15:50 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 19:15:51 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 19:15:51 INFO BlockManager:59 - BlockManager stopped -2016-06-13 19:15:51 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 19:15:51 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 19:15:51 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 19:15:51 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 19:15:51 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 19:15:51 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 19:15:51 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-41a10bd2-5df1-40b2-b7d2-1ad475c899ef -2016-06-13 19:26:19 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 19:26:20 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 19:26:20 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 19:26:20 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 19:26:20 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 19:26:21 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 19:26:21 INFO Remoting:74 - Starting remoting -2016-06-13 19:26:21 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:49616] -2016-06-13 19:26:21 INFO Utils:59 - Successfully started service 'sparkDriver' on port 49616. -2016-06-13 19:26:21 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 19:26:21 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 19:26:21 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-d0f99c91-1dc4-41b3-ac31-25d4d70251ae -2016-06-13 19:26:21 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 19:26:21 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-b2a4fabc-31f1-49ac-a943-75cb46e80b6f\httpd-ae3a6ef4-6b08-4215-9da1-60e81d9f669a -2016-06-13 19:26:21 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 19:26:21 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:26:22 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:49617 -2016-06-13 19:26:22 INFO Utils:59 - Successfully started service 'HTTP file server' on port 49617. -2016-06-13 19:26:22 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 19:26:22 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:26:22 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 19:26:22 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 19:26:22 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 19:26:22 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 19:26:22 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 19:26:22 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 49636. -2016-06-13 19:26:22 INFO NettyBlockTransferService:59 - Server created on 49636 -2016-06-13 19:26:22 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 19:26:22 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:49636 with 730.6 MB RAM, BlockManagerId(driver, localhost, 49636) -2016-06-13 19:26:22 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 19:26:23 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:325 -2016-06-13 19:26:23 INFO DAGScheduler:59 - Registering RDD 11 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:26:23 INFO DAGScheduler:59 - Registering RDD 5 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:26:23 INFO DAGScheduler:59 - Got job 0 (sum at PowerIterationClustering.scala:325) with 2 output partitions -2016-06-13 19:26:23 INFO DAGScheduler:59 - Final stage: ResultStage 2(sum at PowerIterationClustering.scala:325) -2016-06-13 19:26:23 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0, ShuffleMapStage 1) -2016-06-13 19:26:23 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0, ShuffleMapStage 1) -2016-06-13 19:26:23 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[11] at mapPartitions at GraphImpl.scala:235), which has no missing parents -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(3432) called with curMem=0, maxMem=766075207 -2016-06-13 19:26:24 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 3.4 KB, free 730.6 MB) -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(1979) called with curMem=3432, maxMem=766075207 -2016-06-13 19:26:24 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1979.0 B, free 730.6 MB) -2016-06-13 19:26:24 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:49636 (size: 1979.0 B, free: 730.6 MB) -2016-06-13 19:26:24 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[11] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:26:24 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 19:26:24 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[5] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(3016) called with curMem=5411, maxMem=766075207 -2016-06-13 19:26:24 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(1776) called with curMem=8427, maxMem=766075207 -2016-06-13 19:26:24 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1776.0 B, free 730.6 MB) -2016-06-13 19:26:24 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:49636 (size: 1776.0 B, free: 730.6 MB) -2016-06-13 19:26:24 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[5] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:26:24 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 19:26:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2488 bytes) -2016-06-13 19:26:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2563 bytes) -2016-06-13 19:26:24 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 19:26:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2488 bytes) -2016-06-13 19:26:24 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 19:26:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2563 bytes) -2016-06-13 19:26:24 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 19:26:24 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 19:26:24 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-06-13 19:26:24 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-06-13 19:26:24 INFO CacheManager:59 - Another thread is loading rdd_2_0, waiting for it to finish... -2016-06-13 19:26:24 INFO CacheManager:59 - Another thread is loading rdd_2_1, waiting for it to finish... -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(2952) called with curMem=10203, maxMem=766075207 -2016-06-13 19:26:24 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(2856) called with curMem=13155, maxMem=766075207 -2016-06-13 19:26:24 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 2.8 KB, free 730.6 MB) -2016-06-13 19:26:24 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:49636 (size: 2.9 KB, free: 730.6 MB) -2016-06-13 19:26:24 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:49636 (size: 2.8 KB, free: 730.6 MB) -2016-06-13 19:26:24 INFO CacheManager:59 - Finished waiting for rdd_2_1 -2016-06-13 19:26:24 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-06-13 19:26:24 INFO CacheManager:59 - Finished waiting for rdd_2_0 -2016-06-13 19:26:24 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-06-13 19:26:24 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1159 bytes result sent to driver -2016-06-13 19:26:24 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1739 bytes result sent to driver -2016-06-13 19:26:24 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1739 bytes result sent to driver -2016-06-13 19:26:24 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1159 bytes result sent to driver -2016-06-13 19:26:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 210 ms on localhost (1/2) -2016-06-13 19:26:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 259 ms on localhost (1/2) -2016-06-13 19:26:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 226 ms on localhost (2/2) -2016-06-13 19:26:24 INFO DAGScheduler:59 - ShuffleMapStage 0 (mapPartitions at GraphImpl.scala:235) finished in 0.282 s -2016-06-13 19:26:24 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:24 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 1) -2016-06-13 19:26:24 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-06-13 19:26:24 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 19:26:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 232 ms on localhost (2/2) -2016-06-13 19:26:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 19:26:24 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List(ShuffleMapStage 1) -2016-06-13 19:26:24 INFO DAGScheduler:59 - ShuffleMapStage 1 (mapPartitions at VertexRDD.scala:358) finished in 0.254 s -2016-06-13 19:26:24 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:24 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:26:24 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-06-13 19:26:24 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:24 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List() -2016-06-13 19:26:24 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[23] at map at PowerIterationClustering.scala:325), which is now runnable -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(5136) called with curMem=16011, maxMem=766075207 -2016-06-13 19:26:24 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 5.0 KB, free 730.6 MB) -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(2654) called with curMem=21147, maxMem=766075207 -2016-06-13 19:26:24 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 19:26:24 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:49636 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:26:24 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[23] at map at PowerIterationClustering.scala:325) -2016-06-13 19:26:24 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 19:26:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:26:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:26:24 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 19:26:24 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 19:26:24 INFO CacheManager:59 - Partition rdd_21_1 not found, computing it -2016-06-13 19:26:24 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-06-13 19:26:24 INFO CacheManager:59 - Partition rdd_8_1 not found, computing it -2016-06-13 19:26:24 INFO CacheManager:59 - Partition rdd_21_0 not found, computing it -2016-06-13 19:26:24 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-06-13 19:26:24 INFO CacheManager:59 - Partition rdd_8_0 not found, computing it -2016-06-13 19:26:24 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:24 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 11 ms -2016-06-13 19:26:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(1792) called with curMem=23801, maxMem=766075207 -2016-06-13 19:26:24 INFO MemoryStore:59 - Block rdd_8_1 stored as values in memory (estimated size 1792.0 B, free 730.6 MB) -2016-06-13 19:26:24 INFO BlockManagerInfo:59 - Added rdd_8_1 in memory on localhost:49636 (size: 1792.0 B, free: 730.6 MB) -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(1776) called with curMem=25593, maxMem=766075207 -2016-06-13 19:26:24 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:24 INFO MemoryStore:59 - Block rdd_8_0 stored as values in memory (estimated size 1776.0 B, free 730.6 MB) -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(1840) called with curMem=27369, maxMem=766075207 -2016-06-13 19:26:24 INFO BlockManagerInfo:59 - Added rdd_8_0 in memory on localhost:49636 (size: 1776.0 B, free: 730.6 MB) -2016-06-13 19:26:24 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 1840.0 B, free 730.6 MB) -2016-06-13 19:26:24 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:26:24 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:49636 (size: 1840.0 B, free: 730.6 MB) -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(1824) called with curMem=29209, maxMem=766075207 -2016-06-13 19:26:24 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 1824.0 B, free 730.6 MB) -2016-06-13 19:26:24 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:49636 (size: 1824.0 B, free: 730.6 MB) -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(368) called with curMem=31033, maxMem=766075207 -2016-06-13 19:26:24 INFO MemoryStore:59 - Block rdd_21_1 stored as values in memory (estimated size 368.0 B, free 730.6 MB) -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(368) called with curMem=31401, maxMem=766075207 -2016-06-13 19:26:24 INFO MemoryStore:59 - Block rdd_21_0 stored as values in memory (estimated size 368.0 B, free 730.6 MB) -2016-06-13 19:26:24 INFO BlockManagerInfo:59 - Added rdd_21_1 in memory on localhost:49636 (size: 368.0 B, free: 730.6 MB) -2016-06-13 19:26:24 INFO BlockManagerInfo:59 - Added rdd_21_0 in memory on localhost:49636 (size: 368.0 B, free: 730.6 MB) -2016-06-13 19:26:24 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1895 bytes result sent to driver -2016-06-13 19:26:24 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1895 bytes result sent to driver -2016-06-13 19:26:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 143 ms on localhost (1/2) -2016-06-13 19:26:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 155 ms on localhost (2/2) -2016-06-13 19:26:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 19:26:24 INFO DAGScheduler:59 - ResultStage 2 (sum at PowerIterationClustering.scala:325) finished in 0.157 s -2016-06-13 19:26:24 INFO DAGScheduler:59 - Job 0 finished: sum at PowerIterationClustering.scala:325, took 0.808264 s -2016-06-13 19:26:24 INFO SparkContext:59 - Starting job: reduce at VertexRDDImpl.scala:90 -2016-06-13 19:26:24 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:26:24 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:26:24 INFO DAGScheduler:59 - Got job 1 (reduce at VertexRDDImpl.scala:90) with 2 output partitions -2016-06-13 19:26:24 INFO DAGScheduler:59 - Final stage: ResultStage 5(reduce at VertexRDDImpl.scala:90) -2016-06-13 19:26:24 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3, ShuffleMapStage 4) -2016-06-13 19:26:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:24 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[27] at map at VertexRDDImpl.scala:90), which has no missing parents -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(5288) called with curMem=31769, maxMem=766075207 -2016-06-13 19:26:24 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.2 KB, free 730.6 MB) -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(2670) called with curMem=37057, maxMem=766075207 -2016-06-13 19:26:24 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 19:26:24 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:49636 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:26:24 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[27] at map at VertexRDDImpl.scala:90) -2016-06-13 19:26:24 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 19:26:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 6, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:26:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 7, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:26:24 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 6) -2016-06-13 19:26:24 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 7) -2016-06-13 19:26:24 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:26:24 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:26:24 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 7). 2161 bytes result sent to driver -2016-06-13 19:26:24 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 6). 2161 bytes result sent to driver -2016-06-13 19:26:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 7) in 37 ms on localhost (1/2) -2016-06-13 19:26:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 6) in 40 ms on localhost (2/2) -2016-06-13 19:26:24 INFO DAGScheduler:59 - ResultStage 5 (reduce at VertexRDDImpl.scala:90) finished in 0.041 s -2016-06-13 19:26:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 19:26:24 INFO DAGScheduler:59 - Job 1 finished: reduce at VertexRDDImpl.scala:90, took 0.077468 s -2016-06-13 19:26:24 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:26:24 INFO DAGScheduler:59 - Registering RDD 15 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:26:24 INFO DAGScheduler:59 - Registering RDD 28 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:26:24 INFO DAGScheduler:59 - Registering RDD 32 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:26:24 INFO DAGScheduler:59 - Got job 2 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:26:24 INFO DAGScheduler:59 - Final stage: ResultStage 13(sum at PowerIterationClustering.scala:369) -2016-06-13 19:26:24 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12, ShuffleMapStage 9, ShuffleMapStage 10) -2016-06-13 19:26:24 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-06-13 19:26:24 INFO DAGScheduler:59 - Submitting ShuffleMapStage 8 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[15] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=39727, maxMem=766075207 -2016-06-13 19:26:24 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.1 KB, free 730.5 MB) -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(2367) called with curMem=43975, maxMem=766075207 -2016-06-13 19:26:24 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 19:26:24 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:49636 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 19:26:24 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 8 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[15] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:26:24 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 19:26:24 INFO DAGScheduler:59 - Submitting ShuffleMapStage 11 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[28] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:26:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 8, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:26:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 9, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:26:24 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 8) -2016-06-13 19:26:24 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 9) -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(4992) called with curMem=46342, maxMem=766075207 -2016-06-13 19:26:24 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 4.9 KB, free 730.5 MB) -2016-06-13 19:26:24 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-06-13 19:26:24 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(2604) called with curMem=51334, maxMem=766075207 -2016-06-13 19:26:24 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 19:26:24 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:49636 (size: 2.5 KB, free: 730.6 MB) -2016-06-13 19:26:24 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:49636 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:26:24 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 9). 2254 bytes result sent to driver -2016-06-13 19:26:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 9) in 100 ms on localhost (1/2) -2016-06-13 19:26:24 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 8). 2254 bytes result sent to driver -2016-06-13 19:26:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 8) in 108 ms on localhost (2/2) -2016-06-13 19:26:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 19:26:24 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:24 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:49636 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:26:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 11 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[28] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:26:24 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 19:26:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 10, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:26:24 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:49636 in memory (size: 1776.0 B, free: 730.6 MB) -2016-06-13 19:26:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 11, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:26:24 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 11) -2016-06-13 19:26:24 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 10) -2016-06-13 19:26:24 INFO DAGScheduler:59 - ShuffleMapStage 8 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.123 s -2016-06-13 19:26:24 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:24 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 11) -2016-06-13 19:26:24 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 12, ResultStage 13) -2016-06-13 19:26:24 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:24 INFO CacheManager:59 - Partition rdd_25_1 not found, computing it -2016-06-13 19:26:24 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=33398, maxMem=766075207 -2016-06-13 19:26:24 INFO CacheManager:59 - Partition rdd_25_0 not found, computing it -2016-06-13 19:26:24 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:26:24 INFO MemoryStore:59 - Block rdd_25_1 stored as values in memory (estimated size 1432.0 B, free 730.6 MB) -2016-06-13 19:26:24 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 12: List(ShuffleMapStage 11) -2016-06-13 19:26:24 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=34830, maxMem=766075207 -2016-06-13 19:26:24 INFO BlockManagerInfo:59 - Added rdd_25_1 in memory on localhost:49636 (size: 1432.0 B, free: 730.6 MB) -2016-06-13 19:26:24 INFO MemoryStore:59 - Block rdd_25_0 stored as values in memory (estimated size 1432.0 B, free 730.6 MB) -2016-06-13 19:26:24 INFO BlockManagerInfo:59 - Added rdd_25_0 in memory on localhost:49636 (size: 1432.0 B, free: 730.6 MB) -2016-06-13 19:26:24 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List(ShuffleMapStage 12) -2016-06-13 19:26:24 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 11). 2834 bytes result sent to driver -2016-06-13 19:26:24 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 10). 2834 bytes result sent to driver -2016-06-13 19:26:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 11) in 34 ms on localhost (1/2) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:49636 in memory (size: 1979.0 B, free: 730.6 MB) -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 10) in 45 ms on localhost (2/2) -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 19:26:25 INFO DAGScheduler:59 - ShuffleMapStage 11 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.048 s -2016-06-13 19:26:25 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:25 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:26:25 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 12, ResultStage 13) -2016-06-13 19:26:25 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:25 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 12: List() -2016-06-13 19:26:25 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List(ShuffleMapStage 12) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[32] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=30851, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 4.7 KB, free 730.6 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(2539) called with curMem=35699, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:49636 (size: 2.5 KB, free: 730.6 MB) -2016-06-13 19:26:25 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[32] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 12, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 13, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:26:25 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 13) -2016-06-13 19:26:25 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 12) -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:26:25 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 13). 2470 bytes result sent to driver -2016-06-13 19:26:25 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 12). 2470 bytes result sent to driver -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 13) in 54 ms on localhost (1/2) -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 12) in 70 ms on localhost (2/2) -2016-06-13 19:26:25 INFO DAGScheduler:59 - ShuffleMapStage 12 (mapPartitions at GraphImpl.scala:235) finished in 0.075 s -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 19:26:25 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:25 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:26:25 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-06-13 19:26:25 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:25 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[37] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(6024) called with curMem=38238, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.9 KB, free 730.5 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(2867) called with curMem=44262, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.5 MB) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:49636 (size: 2.8 KB, free: 730.6 MB) -2016-06-13 19:26:25 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[37] at map at PowerIterationClustering.scala:369) -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 14, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 15, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:26:25 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 15) -2016-06-13 19:26:25 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 14) -2016-06-13 19:26:25 INFO CacheManager:59 - Partition rdd_34_0 not found, computing it -2016-06-13 19:26:25 INFO CacheManager:59 - Partition rdd_34_1 not found, computing it -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=47129, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block rdd_34_1 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=48609, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block rdd_34_0 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added rdd_34_1 in memory on localhost:49636 (size: 1480.0 B, free: 730.6 MB) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added rdd_34_0 in memory on localhost:49636 (size: 1480.0 B, free: 730.6 MB) -2016-06-13 19:26:25 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 15). 2880 bytes result sent to driver -2016-06-13 19:26:25 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 14). 2880 bytes result sent to driver -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 15) in 23 ms on localhost (1/2) -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 14) in 26 ms on localhost (2/2) -2016-06-13 19:26:25 INFO DAGScheduler:59 - ResultStage 13 (sum at PowerIterationClustering.scala:369) finished in 0.026 s -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 19:26:25 INFO DAGScheduler:59 - Job 2 finished: sum at PowerIterationClustering.scala:369, took 0.345193 s -2016-06-13 19:26:25 INFO PowerIterationClustering:59 - Iteration 0: norm(v) = 36.22334869431644. -2016-06-13 19:26:25 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:26:25 INFO DAGScheduler:59 - Got job 3 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:26:25 INFO DAGScheduler:59 - Final stage: ResultStage 21(sum at PowerIterationClustering.scala:375) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 20, ShuffleMapStage 17, ShuffleMapStage 18) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[48] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(6760) called with curMem=50089, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 6.6 KB, free 730.5 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(3106) called with curMem=56849, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.5 MB) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:49636 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 19:26:25 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[48] at values at PowerIterationClustering.scala:375) -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 16, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 17, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:26:25 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 17) -2016-06-13 19:26:25 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 16) -2016-06-13 19:26:25 INFO CacheManager:59 - Partition rdd_40_0 not found, computing it -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_34_0 locally -2016-06-13 19:26:25 INFO CacheManager:59 - Partition rdd_40_1 not found, computing it -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_34_1 locally -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=59955, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block rdd_40_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=61387, maxMem=766075207 -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added rdd_40_0 in memory on localhost:49636 (size: 1432.0 B, free: 730.6 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - Block rdd_40_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added rdd_40_1 in memory on localhost:49636 (size: 1432.0 B, free: 730.6 MB) -2016-06-13 19:26:25 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 16). 2664 bytes result sent to driver -2016-06-13 19:26:25 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 17). 2664 bytes result sent to driver -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 16) in 22 ms on localhost (1/2) -2016-06-13 19:26:25 INFO DAGScheduler:59 - ResultStage 21 (sum at PowerIterationClustering.scala:375) finished in 0.023 s -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 17) in 22 ms on localhost (2/2) -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 19:26:25 INFO DAGScheduler:59 - Job 3 finished: sum at PowerIterationClustering.scala:375, took 0.053411 s -2016-06-13 19:26:25 INFO PowerIterationClustering:59 - Iteration 0: delta = 1.2794887333174252. -2016-06-13 19:26:25 INFO PowerIterationClustering:59 - Iteration 0: diff(delta) = 1.7976931348623157E308. -2016-06-13 19:26:25 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:26:25 INFO DAGScheduler:59 - Registering RDD 51 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Registering RDD 55 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Got job 4 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:26:25 INFO DAGScheduler:59 - Final stage: ResultStage 31(sum at PowerIterationClustering.scala:369) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30, ShuffleMapStage 28, ShuffleMapStage 25, ShuffleMapStage 26) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 30) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting ShuffleMapStage 29 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[51] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(5984) called with curMem=62819, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.8 KB, free 730.5 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(2861) called with curMem=68803, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.5 MB) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:49636 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:26:25 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 29 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[51] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 18, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 19, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:26:25 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 18) -2016-06-13 19:26:25 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 19) -2016-06-13 19:26:25 INFO CacheManager:59 - Partition rdd_49_0 not found, computing it -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_34_0 locally -2016-06-13 19:26:25 INFO CacheManager:59 - Partition rdd_49_1 not found, computing it -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_34_1 locally -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=71664, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block rdd_49_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=73096, maxMem=766075207 -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added rdd_49_0 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - Block rdd_49_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added rdd_49_1 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:25 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 18). 2834 bytes result sent to driver -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 18) in 19 ms on localhost (1/2) -2016-06-13 19:26:25 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 19). 2834 bytes result sent to driver -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 19) in 21 ms on localhost (2/2) -2016-06-13 19:26:25 INFO DAGScheduler:59 - ShuffleMapStage 29 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.022 s -2016-06-13 19:26:25 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 19:26:25 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:26:25 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 30, ResultStage 31) -2016-06-13 19:26:25 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:25 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List() -2016-06-13 19:26:25 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting ShuffleMapStage 30 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[55] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=74528, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.7 KB, free 730.5 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=79376, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:49636 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:26:25 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 30 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[55] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 20, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 21, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:26:25 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 20) -2016-06-13 19:26:25 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 21) -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:25 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 20). 2470 bytes result sent to driver -2016-06-13 19:26:25 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 21). 2470 bytes result sent to driver -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 20) in 41 ms on localhost (1/2) -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 21) in 41 ms on localhost (2/2) -2016-06-13 19:26:25 INFO DAGScheduler:59 - ShuffleMapStage 30 (mapPartitions at GraphImpl.scala:235) finished in 0.043 s -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 19:26:25 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:25 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:26:25 INFO DAGScheduler:59 - waiting: Set(ResultStage 31) -2016-06-13 19:26:25 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:25 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List() -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[60] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(6920) called with curMem=81918, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 6.8 KB, free 730.5 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(3114) called with curMem=88838, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.5 MB) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:49636 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:26:25 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[60] at map at PowerIterationClustering.scala:369) -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 22, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 23, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:26:25 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 23) -2016-06-13 19:26:25 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 22) -2016-06-13 19:26:25 INFO CacheManager:59 - Partition rdd_57_1 not found, computing it -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_49_1 locally -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:25 INFO CacheManager:59 - Partition rdd_57_0 not found, computing it -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_49_0 locally -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=91952, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block rdd_57_1 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=93432, maxMem=766075207 -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added rdd_57_1 in memory on localhost:49636 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - Block rdd_57_0 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added rdd_57_0 in memory on localhost:49636 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:26:25 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 23). 2880 bytes result sent to driver -2016-06-13 19:26:25 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 22). 2880 bytes result sent to driver -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 23) in 31 ms on localhost (1/2) -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 22) in 35 ms on localhost (2/2) -2016-06-13 19:26:25 INFO DAGScheduler:59 - ResultStage 31 (sum at PowerIterationClustering.scala:369) finished in 0.035 s -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 19:26:25 INFO DAGScheduler:59 - Job 4 finished: sum at PowerIterationClustering.scala:369, took 0.177154 s -2016-06-13 19:26:25 INFO PowerIterationClustering:59 - Iteration 1: norm(v) = 36.22334869431644. -2016-06-13 19:26:25 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 151 bytes -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:26:25 INFO DAGScheduler:59 - Got job 5 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:26:25 INFO DAGScheduler:59 - Final stage: ResultStage 41(sum at PowerIterationClustering.scala:375) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 38, ShuffleMapStage 35, ShuffleMapStage 36, ShuffleMapStage 40) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[71] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(7352) called with curMem=94912, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 7.2 KB, free 730.5 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(3294) called with curMem=102264, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:49636 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:26:25 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[71] at values at PowerIterationClustering.scala:375) -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 24, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 25, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:26:25 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 25) -2016-06-13 19:26:25 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 24) -2016-06-13 19:26:25 INFO CacheManager:59 - Partition rdd_63_0 not found, computing it -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_49_0 locally -2016-06-13 19:26:25 INFO CacheManager:59 - Partition rdd_63_1 not found, computing it -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_57_0 locally -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_49_1 locally -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_57_1 locally -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=105558, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block rdd_63_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=106990, maxMem=766075207 -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added rdd_63_0 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - Block rdd_63_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:26:25 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 24). 2664 bytes result sent to driver -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added rdd_63_1 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:25 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 25). 2664 bytes result sent to driver -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 24) in 26 ms on localhost (1/2) -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 25) in 27 ms on localhost (2/2) -2016-06-13 19:26:25 INFO DAGScheduler:59 - ResultStage 41 (sum at PowerIterationClustering.scala:375) finished in 0.028 s -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-06-13 19:26:25 INFO DAGScheduler:59 - Job 5 finished: sum at PowerIterationClustering.scala:375, took 0.064942 s -2016-06-13 19:26:25 INFO PowerIterationClustering:59 - Iteration 1: delta = 0.0. -2016-06-13 19:26:25 INFO PowerIterationClustering:59 - Iteration 1: diff(delta) = 1.2794887333174252. -2016-06-13 19:26:25 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:26:25 INFO DAGScheduler:59 - Registering RDD 74 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Registering RDD 78 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Got job 6 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:26:25 INFO DAGScheduler:59 - Final stage: ResultStage 53(sum at PowerIterationClustering.scala:369) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 48, ShuffleMapStage 45, ShuffleMapStage 52, ShuffleMapStage 46, ShuffleMapStage 50) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 52) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting ShuffleMapStage 51 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[74] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(6568) called with curMem=108422, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 6.4 KB, free 730.5 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(3022) called with curMem=114990, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.5 MB) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:49636 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:26:25 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 51 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[74] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 26, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 27, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:26:25 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 26) -2016-06-13 19:26:25 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 27) -2016-06-13 19:26:25 INFO CacheManager:59 - Partition rdd_72_0 not found, computing it -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_57_0 locally -2016-06-13 19:26:25 INFO CacheManager:59 - Partition rdd_72_1 not found, computing it -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_57_1 locally -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=118012, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block rdd_72_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=119444, maxMem=766075207 -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added rdd_72_0 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - Block rdd_72_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added rdd_72_1 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:25 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 26). 2834 bytes result sent to driver -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 26) in 22 ms on localhost (1/2) -2016-06-13 19:26:25 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 27). 2834 bytes result sent to driver -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 27) in 26 ms on localhost (2/2) -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-06-13 19:26:25 INFO DAGScheduler:59 - ShuffleMapStage 51 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.029 s -2016-06-13 19:26:25 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:25 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:26:25 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 52, ResultStage 53) -2016-06-13 19:26:25 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:25 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 52: List() -2016-06-13 19:26:25 INFO DAGScheduler:59 - Missing parents for ResultStage 53: List(ShuffleMapStage 52) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting ShuffleMapStage 52 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[78] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=120876, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 4.7 KB, free 730.5 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=125724, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:49636 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:26:25 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 52 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[78] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 28, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 29, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:26:25 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 28) -2016-06-13 19:26:25 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 29) -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:25 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 28). 2470 bytes result sent to driver -2016-06-13 19:26:25 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 29). 2470 bytes result sent to driver -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 28) in 46 ms on localhost (1/2) -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 29) in 45 ms on localhost (2/2) -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-06-13 19:26:25 INFO DAGScheduler:59 - ShuffleMapStage 52 (mapPartitions at GraphImpl.scala:235) finished in 0.049 s -2016-06-13 19:26:25 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:25 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:26:25 INFO DAGScheduler:59 - waiting: Set(ResultStage 53) -2016-06-13 19:26:25 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:25 INFO DAGScheduler:59 - Missing parents for ResultStage 53: List() -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[83] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(7504) called with curMem=128266, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 7.3 KB, free 730.5 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(3266) called with curMem=135770, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:49636 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:26:25 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[83] at map at PowerIterationClustering.scala:369) -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 30, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 31, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:26:25 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 31) -2016-06-13 19:26:25 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 30) -2016-06-13 19:26:25 INFO CacheManager:59 - Partition rdd_80_1 not found, computing it -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_72_1 locally -2016-06-13 19:26:25 INFO CacheManager:59 - Partition rdd_80_0 not found, computing it -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_72_0 locally -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=139036, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block rdd_80_0 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=140516, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block rdd_80_1 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added rdd_80_0 in memory on localhost:49636 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added rdd_80_1 in memory on localhost:49636 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:26:25 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 30). 2880 bytes result sent to driver -2016-06-13 19:26:25 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 31). 2880 bytes result sent to driver -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 30) in 26 ms on localhost (1/2) -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 31) in 25 ms on localhost (2/2) -2016-06-13 19:26:25 INFO DAGScheduler:59 - ResultStage 53 (sum at PowerIterationClustering.scala:369) finished in 0.027 s -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-06-13 19:26:25 INFO DAGScheduler:59 - Job 6 finished: sum at PowerIterationClustering.scala:369, took 0.182819 s -2016-06-13 19:26:25 INFO PowerIterationClustering:59 - Iteration 2: norm(v) = 36.22334869431644. -2016-06-13 19:26:25 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 151 bytes -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 151 bytes -2016-06-13 19:26:25 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:26:25 INFO DAGScheduler:59 - Got job 7 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:26:25 INFO DAGScheduler:59 - Final stage: ResultStage 65(sum at PowerIterationClustering.scala:375) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 60, ShuffleMapStage 64, ShuffleMapStage 57, ShuffleMapStage 58, ShuffleMapStage 62) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[94] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(7936) called with curMem=141996, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 7.8 KB, free 730.4 MB) -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(3392) called with curMem=149932, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:49636 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:26:25 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 65 (MapPartitionsRDD[94] at values at PowerIterationClustering.scala:375) -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 2 tasks -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 32, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:26:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 33, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:26:25 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 32) -2016-06-13 19:26:25 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 33) -2016-06-13 19:26:25 INFO CacheManager:59 - Partition rdd_86_1 not found, computing it -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_72_1 locally -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_80_1 locally -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=153324, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block rdd_86_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added rdd_86_1 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:25 INFO CacheManager:59 - Partition rdd_86_0 not found, computing it -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_72_0 locally -2016-06-13 19:26:25 INFO BlockManager:59 - Found block rdd_80_0 locally -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=154756, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block rdd_86_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:26:25 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 33). 2664 bytes result sent to driver -2016-06-13 19:26:25 INFO BlockManagerInfo:59 - Added rdd_86_0 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:25 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 32). 2664 bytes result sent to driver -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 33) in 24 ms on localhost (1/2) -2016-06-13 19:26:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 32) in 26 ms on localhost (2/2) -2016-06-13 19:26:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-06-13 19:26:25 INFO DAGScheduler:59 - ResultStage 65 (sum at PowerIterationClustering.scala:375) finished in 0.027 s -2016-06-13 19:26:25 INFO DAGScheduler:59 - Job 7 finished: sum at PowerIterationClustering.scala:375, took 0.068301 s -2016-06-13 19:26:25 INFO PowerIterationClustering:59 - Iteration 2: delta = 0.0. -2016-06-13 19:26:25 INFO PowerIterationClustering:59 - Iteration 2: diff(delta) = 0.0. -2016-06-13 19:26:25 WARN KMeans:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:26:25 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:26:25 INFO DAGScheduler:59 - Got job 8 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:26:25 INFO DAGScheduler:59 - Final stage: ResultStage 77(takeSample at KMeans.scala:376) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 70, ShuffleMapStage 74, ShuffleMapStage 72, ShuffleMapStage 69, ShuffleMapStage 76) -2016-06-13 19:26:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:25 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[102] at map at KMeans.scala:210), which has no missing parents -2016-06-13 19:26:25 INFO MemoryStore:59 - ensureFreeSpace(8168) called with curMem=156188, maxMem=766075207 -2016-06-13 19:26:25 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 8.0 KB, free 730.4 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(3463) called with curMem=164356, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.4 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:49636 (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:49636 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 77 (MapPartitionsRDD[102] at map at KMeans.scala:210) -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 2 tasks -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 34, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:26:26 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 35, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:26:26 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 35) -2016-06-13 19:26:26 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 34) -2016-06-13 19:26:26 INFO BlockManager:59 - Removing RDD 86 -2016-06-13 19:26:26 INFO CacheManager:59 - Partition rdd_97_0 not found, computing it -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_80_0 locally -2016-06-13 19:26:26 INFO CacheManager:59 - Partition rdd_97_1 not found, computing it -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_80_1 locally -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=153627, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block rdd_97_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=155059, maxMem=766075207 -2016-06-13 19:26:26 INFO ContextCleaner:59 - Cleaned RDD 86 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block rdd_97_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added rdd_97_1 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added rdd_97_0 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:26 INFO CacheManager:59 - Partition rdd_100_1 not found, computing it -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:26:26 INFO CacheManager:59 - Partition rdd_100_0 not found, computing it -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:49636 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=145721, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block rdd_100_0 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added rdd_100_0 in memory on localhost:49636 (size: 240.0 B, free: 730.5 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:49636 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=143419, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block rdd_100_1 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added rdd_100_1 in memory on localhost:49636 (size: 240.0 B, free: 730.5 MB) -2016-06-13 19:26:26 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 34). 2717 bytes result sent to driver -2016-06-13 19:26:26 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 19:26:26 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 35). 2717 bytes result sent to driver -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 34) in 40 ms on localhost (1/2) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 35) in 41 ms on localhost (2/2) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:49636 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-06-13 19:26:26 INFO DAGScheduler:59 - ResultStage 77 (takeSample at KMeans.scala:376) finished in 0.045 s -2016-06-13 19:26:26 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Job 8 finished: takeSample at KMeans.scala:376, took 0.092473 s -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:49636 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 19:26:26 INFO BlockManager:59 - Removing RDD 63 -2016-06-13 19:26:26 INFO ContextCleaner:59 - Cleaned RDD 63 -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:49636 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:49636 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:49636 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:49636 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 19:26:26 INFO BlockManager:59 - Removing RDD 40 -2016-06-13 19:26:26 INFO ContextCleaner:59 - Cleaned RDD 40 -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:49636 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:49636 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:49636 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Got job 9 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:26:26 INFO DAGScheduler:59 - Final stage: ResultStage 89(takeSample at KMeans.scala:376) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 84, ShuffleMapStage 81, ShuffleMapStage 88, ShuffleMapStage 82, ShuffleMapStage 86) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting ResultStage 89 (PartitionwiseSampledRDD[104] at takeSample at KMeans.scala:376), which has no missing parents -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(25560) called with curMem=53819, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 25.0 KB, free 730.5 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(10147) called with curMem=79379, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 9.9 KB, free 730.5 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:49636 (size: 9.9 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 89 (PartitionwiseSampledRDD[104] at takeSample at KMeans.scala:376) -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Adding task set 89.0 with 2 tasks -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 89.0 (TID 36, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 89.0 (TID 37, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:26:26 INFO Executor:59 - Running task 1.0 in stage 89.0 (TID 37) -2016-06-13 19:26:26 INFO Executor:59 - Running task 0.0 in stage 89.0 (TID 36) -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:26:26 INFO Executor:59 - Finished task 0.0 in stage 89.0 (TID 36). 2633 bytes result sent to driver -2016-06-13 19:26:26 INFO Executor:59 - Finished task 1.0 in stage 89.0 (TID 37). 2575 bytes result sent to driver -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 89.0 (TID 36) in 23 ms on localhost (1/2) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 89.0 (TID 37) in 21 ms on localhost (2/2) -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 89.0, whose tasks have all completed, from pool -2016-06-13 19:26:26 INFO DAGScheduler:59 - ResultStage 89 (takeSample at KMeans.scala:376) finished in 0.024 s -2016-06-13 19:26:26 INFO DAGScheduler:59 - Job 9 finished: takeSample at KMeans.scala:376, took 0.067618 s -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=89526, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 880.0 B, free 730.5 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(487) called with curMem=90406, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 487.0 B, free 730.5 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:49636 (size: 487.0 B, free: 730.5 MB) -2016-06-13 19:26:26 INFO SparkContext:59 - Created broadcast 19 from broadcast at KMeans.scala:394 -2016-06-13 19:26:26 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Got job 10 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:26:26 INFO DAGScheduler:59 - Final stage: ResultStage 101(aggregate at KMeans.scala:402) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 96, ShuffleMapStage 100, ShuffleMapStage 93, ShuffleMapStage 94, ShuffleMapStage 98) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting ResultStage 101 (MapPartitionsRDD[106] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(10192) called with curMem=90893, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 10.0 KB, free 730.5 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(4263) called with curMem=101085, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.5 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:49636 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 101 (MapPartitionsRDD[106] at map at KMeans.scala:396) -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Adding task set 101.0 with 2 tasks -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 101.0 (TID 38, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 101.0 (TID 39, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:26:26 INFO Executor:59 - Running task 0.0 in stage 101.0 (TID 38) -2016-06-13 19:26:26 INFO Executor:59 - Running task 1.0 in stage 101.0 (TID 39) -2016-06-13 19:26:26 INFO CacheManager:59 - Partition rdd_106_0 not found, computing it -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:26:26 INFO CacheManager:59 - Partition rdd_106_1 not found, computing it -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:26:26 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 19:26:26 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=105348, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block rdd_106_0 stored as values in memory (estimated size 496.0 B, free 730.5 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=105844, maxMem=766075207 -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added rdd_106_0 in memory on localhost:49636 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - Block rdd_106_1 stored as values in memory (estimated size 496.0 B, free 730.5 MB) -2016-06-13 19:26:26 INFO Executor:59 - Finished task 0.0 in stage 101.0 (TID 38). 2647 bytes result sent to driver -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added rdd_106_1 in memory on localhost:49636 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:26:26 INFO Executor:59 - Finished task 1.0 in stage 101.0 (TID 39). 2647 bytes result sent to driver -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 101.0 (TID 39) in 40 ms on localhost (1/2) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 101.0 (TID 38) in 44 ms on localhost (2/2) -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 101.0, whose tasks have all completed, from pool -2016-06-13 19:26:26 INFO DAGScheduler:59 - ResultStage 101 (aggregate at KMeans.scala:402) finished in 0.045 s -2016-06-13 19:26:26 INFO DAGScheduler:59 - Job 10 finished: aggregate at KMeans.scala:402, took 0.080906 s -2016-06-13 19:26:26 INFO MapPartitionsRDD:59 - Removing RDD 103 from persistence list -2016-06-13 19:26:26 INFO BlockManager:59 - Removing RDD 103 -2016-06-13 19:26:26 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Got job 11 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:26:26 INFO DAGScheduler:59 - Final stage: ResultStage 113(collect at KMeans.scala:431) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 108, ShuffleMapStage 105, ShuffleMapStage 112, ShuffleMapStage 106, ShuffleMapStage 110) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting ResultStage 113 (MapPartitionsRDD[108] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(10240) called with curMem=106340, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 10.0 KB, free 730.5 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(4343) called with curMem=116580, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.5 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:49636 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 113 (MapPartitionsRDD[108] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Adding task set 113.0 with 2 tasks -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 113.0 (TID 40, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 113.0 (TID 41, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:26:26 INFO Executor:59 - Running task 1.0 in stage 113.0 (TID 41) -2016-06-13 19:26:26 INFO Executor:59 - Running task 0.0 in stage 113.0 (TID 40) -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_106_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_106_1 locally -2016-06-13 19:26:26 INFO Executor:59 - Finished task 1.0 in stage 113.0 (TID 41). 3471 bytes result sent to driver -2016-06-13 19:26:26 INFO Executor:59 - Finished task 0.0 in stage 113.0 (TID 40). 3118 bytes result sent to driver -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 113.0 (TID 41) in 21 ms on localhost (1/2) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 113.0 (TID 40) in 24 ms on localhost (2/2) -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 113.0, whose tasks have all completed, from pool -2016-06-13 19:26:26 INFO DAGScheduler:59 - ResultStage 113 (collect at KMeans.scala:431) finished in 0.028 s -2016-06-13 19:26:26 INFO DAGScheduler:59 - Job 11 finished: collect at KMeans.scala:431, took 0.052329 s -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(1648) called with curMem=120923, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 1648.0 B, free 730.5 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(600) called with curMem=122571, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 600.0 B, free 730.5 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:49636 (size: 600.0 B, free: 730.5 MB) -2016-06-13 19:26:26 INFO SparkContext:59 - Created broadcast 22 from broadcast at KMeans.scala:394 -2016-06-13 19:26:26 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Got job 12 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:26:26 INFO DAGScheduler:59 - Final stage: ResultStage 125(aggregate at KMeans.scala:402) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 117, ShuffleMapStage 118, ShuffleMapStage 122, ShuffleMapStage 120, ShuffleMapStage 124) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting ResultStage 125 (MapPartitionsRDD[110] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(10424) called with curMem=123171, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 10.2 KB, free 730.5 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(4357) called with curMem=133595, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.5 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:49636 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 125 (MapPartitionsRDD[110] at map at KMeans.scala:396) -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Adding task set 125.0 with 2 tasks -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 125.0 (TID 42, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 125.0 (TID 43, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:26:26 INFO Executor:59 - Running task 0.0 in stage 125.0 (TID 42) -2016-06-13 19:26:26 INFO Executor:59 - Running task 1.0 in stage 125.0 (TID 43) -2016-06-13 19:26:26 INFO CacheManager:59 - Partition rdd_110_1 not found, computing it -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:26:26 INFO CacheManager:59 - Partition rdd_110_0 not found, computing it -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_106_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_106_0 locally -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=137952, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block rdd_110_1 stored as values in memory (estimated size 496.0 B, free 730.5 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=138448, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block rdd_110_0 stored as values in memory (estimated size 496.0 B, free 730.5 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added rdd_110_1 in memory on localhost:49636 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added rdd_110_0 in memory on localhost:49636 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:26:26 INFO Executor:59 - Finished task 1.0 in stage 125.0 (TID 43). 2647 bytes result sent to driver -2016-06-13 19:26:26 INFO Executor:59 - Finished task 0.0 in stage 125.0 (TID 42). 2647 bytes result sent to driver -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 125.0 (TID 43) in 30 ms on localhost (1/2) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 125.0 (TID 42) in 33 ms on localhost (2/2) -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 125.0, whose tasks have all completed, from pool -2016-06-13 19:26:26 INFO DAGScheduler:59 - ResultStage 125 (aggregate at KMeans.scala:402) finished in 0.035 s -2016-06-13 19:26:26 INFO DAGScheduler:59 - Job 12 finished: aggregate at KMeans.scala:402, took 0.080094 s -2016-06-13 19:26:26 INFO MapPartitionsRDD:59 - Removing RDD 106 from persistence list -2016-06-13 19:26:26 INFO BlockManager:59 - Removing RDD 106 -2016-06-13 19:26:26 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Got job 13 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:26:26 INFO DAGScheduler:59 - Final stage: ResultStage 137(collect at KMeans.scala:431) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 132, ShuffleMapStage 129, ShuffleMapStage 136, ShuffleMapStage 130, ShuffleMapStage 134) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting ResultStage 137 (MapPartitionsRDD[112] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(10480) called with curMem=137952, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 10.2 KB, free 730.4 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(4430) called with curMem=148432, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:49636 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 137 (MapPartitionsRDD[112] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Adding task set 137.0 with 2 tasks -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 137.0 (TID 44, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 137.0 (TID 45, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:26:26 INFO Executor:59 - Running task 1.0 in stage 137.0 (TID 45) -2016-06-13 19:26:26 INFO Executor:59 - Running task 0.0 in stage 137.0 (TID 44) -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_110_1 locally -2016-06-13 19:26:26 INFO Executor:59 - Finished task 1.0 in stage 137.0 (TID 45). 3240 bytes result sent to driver -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_110_0 locally -2016-06-13 19:26:26 INFO Executor:59 - Finished task 0.0 in stage 137.0 (TID 44). 2990 bytes result sent to driver -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 137.0 (TID 45) in 17 ms on localhost (1/2) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 137.0 (TID 44) in 20 ms on localhost (2/2) -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 137.0, whose tasks have all completed, from pool -2016-06-13 19:26:26 INFO DAGScheduler:59 - ResultStage 137 (collect at KMeans.scala:431) finished in 0.024 s -2016-06-13 19:26:26 INFO DAGScheduler:59 - Job 13 finished: collect at KMeans.scala:431, took 0.067545 s -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(1456) called with curMem=152862, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 1456.0 B, free 730.4 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(582) called with curMem=154318, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 582.0 B, free 730.4 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:49636 (size: 582.0 B, free: 730.5 MB) -2016-06-13 19:26:26 INFO SparkContext:59 - Created broadcast 25 from broadcast at KMeans.scala:394 -2016-06-13 19:26:26 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Got job 14 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:26:26 INFO DAGScheduler:59 - Final stage: ResultStage 149(aggregate at KMeans.scala:402) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 144, ShuffleMapStage 141, ShuffleMapStage 148, ShuffleMapStage 142, ShuffleMapStage 146) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting ResultStage 149 (MapPartitionsRDD[114] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(10656) called with curMem=154900, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 10.4 KB, free 730.4 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(4409) called with curMem=165556, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:49636 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 149 (MapPartitionsRDD[114] at map at KMeans.scala:396) -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Adding task set 149.0 with 2 tasks -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 149.0 (TID 46, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 149.0 (TID 47, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:26:26 INFO Executor:59 - Running task 1.0 in stage 149.0 (TID 47) -2016-06-13 19:26:26 INFO Executor:59 - Running task 0.0 in stage 149.0 (TID 46) -2016-06-13 19:26:26 INFO CacheManager:59 - Partition rdd_114_0 not found, computing it -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:26:26 INFO CacheManager:59 - Partition rdd_114_1 not found, computing it -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_110_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_110_1 locally -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=169965, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block rdd_114_0 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=170461, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block rdd_114_1 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added rdd_114_0 in memory on localhost:49636 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added rdd_114_1 in memory on localhost:49636 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:26:26 INFO Executor:59 - Finished task 0.0 in stage 149.0 (TID 46). 2647 bytes result sent to driver -2016-06-13 19:26:26 INFO Executor:59 - Finished task 1.0 in stage 149.0 (TID 47). 2647 bytes result sent to driver -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 149.0 (TID 46) in 16 ms on localhost (1/2) -2016-06-13 19:26:26 INFO DAGScheduler:59 - ResultStage 149 (aggregate at KMeans.scala:402) finished in 0.016 s -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 149.0 (TID 47) in 15 ms on localhost (2/2) -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 149.0, whose tasks have all completed, from pool -2016-06-13 19:26:26 INFO DAGScheduler:59 - Job 14 finished: aggregate at KMeans.scala:402, took 0.043845 s -2016-06-13 19:26:26 INFO MapPartitionsRDD:59 - Removing RDD 110 from persistence list -2016-06-13 19:26:26 INFO BlockManager:59 - Removing RDD 110 -2016-06-13 19:26:26 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Got job 15 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:26:26 INFO DAGScheduler:59 - Final stage: ResultStage 161(collect at KMeans.scala:431) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 153, ShuffleMapStage 154, ShuffleMapStage 158, ShuffleMapStage 156, ShuffleMapStage 160) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting ResultStage 161 (MapPartitionsRDD[116] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(10712) called with curMem=169965, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 10.5 KB, free 730.4 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(4468) called with curMem=180677, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:49636 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 161 (MapPartitionsRDD[116] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Adding task set 161.0 with 2 tasks -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 161.0 (TID 48, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 161.0 (TID 49, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:26:26 INFO Executor:59 - Running task 1.0 in stage 161.0 (TID 49) -2016-06-13 19:26:26 INFO Executor:59 - Running task 0.0 in stage 161.0 (TID 48) -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_114_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_114_1 locally -2016-06-13 19:26:26 INFO Executor:59 - Finished task 1.0 in stage 161.0 (TID 49). 2040 bytes result sent to driver -2016-06-13 19:26:26 INFO Executor:59 - Finished task 0.0 in stage 161.0 (TID 48). 2722 bytes result sent to driver -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 161.0 (TID 49) in 9 ms on localhost (1/2) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 161.0 (TID 48) in 11 ms on localhost (2/2) -2016-06-13 19:26:26 INFO DAGScheduler:59 - ResultStage 161 (collect at KMeans.scala:431) finished in 0.011 s -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 161.0, whose tasks have all completed, from pool -2016-06-13 19:26:26 INFO DAGScheduler:59 - Job 15 finished: collect at KMeans.scala:431, took 0.032165 s -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(648) called with curMem=185145, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 648.0 B, free 730.4 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(429) called with curMem=185793, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 429.0 B, free 730.4 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:49636 (size: 429.0 B, free: 730.5 MB) -2016-06-13 19:26:26 INFO SparkContext:59 - Created broadcast 28 from broadcast at KMeans.scala:394 -2016-06-13 19:26:26 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Got job 16 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:26:26 INFO DAGScheduler:59 - Final stage: ResultStage 173(aggregate at KMeans.scala:402) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 168, ShuffleMapStage 165, ShuffleMapStage 172, ShuffleMapStage 166, ShuffleMapStage 170) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting ResultStage 173 (MapPartitionsRDD[118] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(10888) called with curMem=186222, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 10.6 KB, free 730.4 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(4468) called with curMem=197110, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:49636 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 173 (MapPartitionsRDD[118] at map at KMeans.scala:396) -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Adding task set 173.0 with 2 tasks -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 173.0 (TID 50, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 173.0 (TID 51, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:26:26 INFO Executor:59 - Running task 0.0 in stage 173.0 (TID 50) -2016-06-13 19:26:26 INFO Executor:59 - Running task 1.0 in stage 173.0 (TID 51) -2016-06-13 19:26:26 INFO CacheManager:59 - Partition rdd_118_0 not found, computing it -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_114_0 locally -2016-06-13 19:26:26 INFO CacheManager:59 - Partition rdd_118_1 not found, computing it -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=201578, maxMem=766075207 -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_114_1 locally -2016-06-13 19:26:26 INFO MemoryStore:59 - Block rdd_118_0 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added rdd_118_0 in memory on localhost:49636 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=202074, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block rdd_118_1 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:26:26 INFO Executor:59 - Finished task 0.0 in stage 173.0 (TID 50). 2647 bytes result sent to driver -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added rdd_118_1 in memory on localhost:49636 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 173.0 (TID 50) in 18 ms on localhost (1/2) -2016-06-13 19:26:26 INFO Executor:59 - Finished task 1.0 in stage 173.0 (TID 51). 2647 bytes result sent to driver -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 173.0 (TID 51) in 19 ms on localhost (2/2) -2016-06-13 19:26:26 INFO DAGScheduler:59 - ResultStage 173 (aggregate at KMeans.scala:402) finished in 0.021 s -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 173.0, whose tasks have all completed, from pool -2016-06-13 19:26:26 INFO DAGScheduler:59 - Job 16 finished: aggregate at KMeans.scala:402, took 0.046575 s -2016-06-13 19:26:26 INFO MapPartitionsRDD:59 - Removing RDD 114 from persistence list -2016-06-13 19:26:26 INFO BlockManager:59 - Removing RDD 114 -2016-06-13 19:26:26 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Got job 17 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:26:26 INFO DAGScheduler:59 - Final stage: ResultStage 185(collect at KMeans.scala:431) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 180, ShuffleMapStage 184, ShuffleMapStage 177, ShuffleMapStage 178, ShuffleMapStage 182) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting ResultStage 185 (MapPartitionsRDD[120] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(10944) called with curMem=201578, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 10.7 KB, free 730.4 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(4514) called with curMem=212522, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:49636 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 185 (MapPartitionsRDD[120] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Adding task set 185.0 with 2 tasks -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 185.0 (TID 52, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 185.0 (TID 53, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:26:26 INFO Executor:59 - Running task 0.0 in stage 185.0 (TID 52) -2016-06-13 19:26:26 INFO Executor:59 - Running task 1.0 in stage 185.0 (TID 53) -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_118_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_118_1 locally -2016-06-13 19:26:26 INFO Executor:59 - Finished task 0.0 in stage 185.0 (TID 52). 2040 bytes result sent to driver -2016-06-13 19:26:26 INFO Executor:59 - Finished task 1.0 in stage 185.0 (TID 53). 2040 bytes result sent to driver -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 185.0 (TID 52) in 10 ms on localhost (1/2) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 185.0 (TID 53) in 10 ms on localhost (2/2) -2016-06-13 19:26:26 INFO DAGScheduler:59 - ResultStage 185 (collect at KMeans.scala:431) finished in 0.011 s -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 185.0, whose tasks have all completed, from pool -2016-06-13 19:26:26 INFO DAGScheduler:59 - Job 17 finished: collect at KMeans.scala:431, took 0.033923 s -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(584) called with curMem=217036, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 584.0 B, free 730.4 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(236) called with curMem=217620, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 236.0 B, free 730.4 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:49636 (size: 236.0 B, free: 730.5 MB) -2016-06-13 19:26:26 INFO SparkContext:59 - Created broadcast 31 from broadcast at KMeans.scala:394 -2016-06-13 19:26:26 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Got job 18 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:26:26 INFO DAGScheduler:59 - Final stage: ResultStage 197(aggregate at KMeans.scala:402) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 190, ShuffleMapStage 194, ShuffleMapStage 192, ShuffleMapStage 189, ShuffleMapStage 196) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting ResultStage 197 (MapPartitionsRDD[122] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(11128) called with curMem=217856, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 10.9 KB, free 730.4 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(4524) called with curMem=228984, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:49636 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 197 (MapPartitionsRDD[122] at map at KMeans.scala:396) -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Adding task set 197.0 with 2 tasks -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 197.0 (TID 54, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 197.0 (TID 55, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:26:26 INFO Executor:59 - Running task 1.0 in stage 197.0 (TID 55) -2016-06-13 19:26:26 INFO Executor:59 - Running task 0.0 in stage 197.0 (TID 54) -2016-06-13 19:26:26 INFO CacheManager:59 - Partition rdd_122_1 not found, computing it -2016-06-13 19:26:26 INFO CacheManager:59 - Partition rdd_122_0 not found, computing it -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_118_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_118_0 locally -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=233508, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block rdd_122_1 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=234004, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block rdd_122_0 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added rdd_122_1 in memory on localhost:49636 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added rdd_122_0 in memory on localhost:49636 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:26:26 INFO Executor:59 - Finished task 0.0 in stage 197.0 (TID 54). 2647 bytes result sent to driver -2016-06-13 19:26:26 INFO Executor:59 - Finished task 1.0 in stage 197.0 (TID 55). 2647 bytes result sent to driver -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 197.0 (TID 54) in 17 ms on localhost (1/2) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 197.0 (TID 55) in 17 ms on localhost (2/2) -2016-06-13 19:26:26 INFO DAGScheduler:59 - ResultStage 197 (aggregate at KMeans.scala:402) finished in 0.019 s -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 197.0, whose tasks have all completed, from pool -2016-06-13 19:26:26 INFO DAGScheduler:59 - Job 18 finished: aggregate at KMeans.scala:402, took 0.042559 s -2016-06-13 19:26:26 INFO MapPartitionsRDD:59 - Removing RDD 118 from persistence list -2016-06-13 19:26:26 INFO BlockManager:59 - Removing RDD 118 -2016-06-13 19:26:26 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Got job 19 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:26:26 INFO DAGScheduler:59 - Final stage: ResultStage 209(collect at KMeans.scala:431) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 204, ShuffleMapStage 201, ShuffleMapStage 208, ShuffleMapStage 202, ShuffleMapStage 206) -2016-06-13 19:26:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting ResultStage 209 (MapPartitionsRDD[124] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(11176) called with curMem=233508, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 10.9 KB, free 730.4 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(4565) called with curMem=244684, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.3 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:49636 (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:26:26 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 209 (MapPartitionsRDD[124] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Adding task set 209.0 with 2 tasks -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 209.0 (TID 56, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 209.0 (TID 57, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:26:26 INFO Executor:59 - Running task 1.0 in stage 209.0 (TID 57) -2016-06-13 19:26:26 INFO Executor:59 - Running task 0.0 in stage 209.0 (TID 56) -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_122_0 locally -2016-06-13 19:26:26 INFO BlockManager:59 - Found block rdd_122_1 locally -2016-06-13 19:26:26 INFO Executor:59 - Finished task 0.0 in stage 209.0 (TID 56). 2040 bytes result sent to driver -2016-06-13 19:26:26 INFO Executor:59 - Finished task 1.0 in stage 209.0 (TID 57). 2040 bytes result sent to driver -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 209.0 (TID 56) in 10 ms on localhost (1/2) -2016-06-13 19:26:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 209.0 (TID 57) in 11 ms on localhost (2/2) -2016-06-13 19:26:26 INFO DAGScheduler:59 - ResultStage 209 (collect at KMeans.scala:431) finished in 0.012 s -2016-06-13 19:26:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 209.0, whose tasks have all completed, from pool -2016-06-13 19:26:26 INFO DAGScheduler:59 - Job 19 finished: collect at KMeans.scala:431, took 0.038018 s -2016-06-13 19:26:26 INFO MapPartitionsRDD:59 - Removing RDD 122 from persistence list -2016-06-13 19:26:26 INFO BlockManager:59 - Removing RDD 122 -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(2880) called with curMem=248257, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 2.8 KB, free 730.3 MB) -2016-06-13 19:26:26 INFO MemoryStore:59 - ensureFreeSpace(708) called with curMem=251137, maxMem=766075207 -2016-06-13 19:26:26 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 708.0 B, free 730.3 MB) -2016-06-13 19:26:26 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:49636 (size: 708.0 B, free: 730.5 MB) -2016-06-13 19:26:26 INFO SparkContext:59 - Created broadcast 34 from broadcast at KMeans.scala:445 -2016-06-13 19:26:27 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:450 -2016-06-13 19:26:27 INFO DAGScheduler:59 - Registering RDD 125 (flatMap at KMeans.scala:446) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Got job 20 (collectAsMap at KMeans.scala:450) with 2 output partitions -2016-06-13 19:26:27 INFO DAGScheduler:59 - Final stage: ResultStage 222(collectAsMap at KMeans.scala:450) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 221) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 221) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 221 (MapPartitionsRDD[125] at flatMap at KMeans.scala:446), which has no missing parents -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(9672) called with curMem=251845, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 9.4 KB, free 730.3 MB) -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(4106) called with curMem=261517, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:49636 (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 221 (MapPartitionsRDD[125] at flatMap at KMeans.scala:446) -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Adding task set 221.0 with 2 tasks -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 221.0 (TID 58, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 221.0 (TID 59, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:26:27 INFO Executor:59 - Running task 0.0 in stage 221.0 (TID 58) -2016-06-13 19:26:27 INFO Executor:59 - Running task 1.0 in stage 221.0 (TID 59) -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:26:27 INFO Executor:59 - Finished task 0.0 in stage 221.0 (TID 58). 2254 bytes result sent to driver -2016-06-13 19:26:27 INFO Executor:59 - Finished task 1.0 in stage 221.0 (TID 59). 2254 bytes result sent to driver -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 221.0 (TID 58) in 47 ms on localhost (1/2) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 221.0 (TID 59) in 49 ms on localhost (2/2) -2016-06-13 19:26:27 INFO DAGScheduler:59 - ShuffleMapStage 221 (flatMap at KMeans.scala:446) finished in 0.050 s -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 221.0, whose tasks have all completed, from pool -2016-06-13 19:26:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:27 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:26:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 222) -2016-06-13 19:26:27 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:27 INFO DAGScheduler:59 - Missing parents for ResultStage 222: List() -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting ResultStage 222 (ShuffledRDD[126] at reduceByKey at KMeans.scala:450), which is now runnable -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=265623, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 2.3 KB, free 730.3 MB) -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(1382) called with curMem=267935, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 1382.0 B, free 730.3 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:49636 (size: 1382.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 222 (ShuffledRDD[126] at reduceByKey at KMeans.scala:450) -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Adding task set 222.0 with 2 tasks -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 222.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 222.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:26:27 INFO Executor:59 - Running task 1.0 in stage 222.0 (TID 61) -2016-06-13 19:26:27 INFO Executor:59 - Running task 0.0 in stage 222.0 (TID 60) -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:27 INFO Executor:59 - Finished task 1.0 in stage 222.0 (TID 61). 1744 bytes result sent to driver -2016-06-13 19:26:27 INFO Executor:59 - Finished task 0.0 in stage 222.0 (TID 60). 1852 bytes result sent to driver -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 222.0 (TID 61) in 20 ms on localhost (1/2) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 222.0 (TID 60) in 23 ms on localhost (2/2) -2016-06-13 19:26:27 INFO DAGScheduler:59 - ResultStage 222 (collectAsMap at KMeans.scala:450) finished in 0.023 s -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 222.0, whose tasks have all completed, from pool -2016-06-13 19:26:27 INFO DAGScheduler:59 - Job 20 finished: collectAsMap at KMeans.scala:450, took 0.106964 s -2016-06-13 19:26:27 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:26:27 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:26:27 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:26:27 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:26:27 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:26:27 INFO KMeans:59 - Initialization with k-means|| took 1.294 seconds. -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(824) called with curMem=269317, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 824.0 B, free 730.3 MB) -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(339) called with curMem=270141, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 339.0 B, free 730.3 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:49636 (size: 339.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO SparkContext:59 - Created broadcast 37 from broadcast at KMeans.scala:276 -2016-06-13 19:26:27 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:302 -2016-06-13 19:26:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:26:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:26:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:26:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:26:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:26:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 151 bytes -2016-06-13 19:26:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:26:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 151 bytes -2016-06-13 19:26:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:26:27 INFO DAGScheduler:59 - Registering RDD 127 (mapPartitions at KMeans.scala:279) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Got job 21 (collectAsMap at KMeans.scala:302) with 2 output partitions -2016-06-13 19:26:27 INFO DAGScheduler:59 - Final stage: ResultStage 235(collectAsMap at KMeans.scala:302) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 234) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 234) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 234 (MapPartitionsRDD[127] at mapPartitions at KMeans.scala:279), which has no missing parents -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(10296) called with curMem=270480, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 10.1 KB, free 730.3 MB) -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(4440) called with curMem=280776, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:49636 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 234 (MapPartitionsRDD[127] at mapPartitions at KMeans.scala:279) -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Adding task set 234.0 with 2 tasks -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 234.0 (TID 62, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 234.0 (TID 63, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:26:27 INFO Executor:59 - Running task 0.0 in stage 234.0 (TID 62) -2016-06-13 19:26:27 INFO Executor:59 - Running task 1.0 in stage 234.0 (TID 63) -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:26:27 INFO Executor:59 - Finished task 1.0 in stage 234.0 (TID 63). 2413 bytes result sent to driver -2016-06-13 19:26:27 INFO Executor:59 - Finished task 0.0 in stage 234.0 (TID 62). 2413 bytes result sent to driver -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 234.0 (TID 63) in 20 ms on localhost (1/2) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 234.0 (TID 62) in 23 ms on localhost (2/2) -2016-06-13 19:26:27 INFO DAGScheduler:59 - ShuffleMapStage 234 (mapPartitions at KMeans.scala:279) finished in 0.023 s -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 234.0, whose tasks have all completed, from pool -2016-06-13 19:26:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:27 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:26:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 235) -2016-06-13 19:26:27 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:27 INFO DAGScheduler:59 - Missing parents for ResultStage 235: List() -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting ResultStage 235 (ShuffledRDD[128] at reduceByKey at KMeans.scala:302), which is now runnable -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(2808) called with curMem=285216, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 2.7 KB, free 730.3 MB) -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(1589) called with curMem=288024, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 1589.0 B, free 730.3 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:49636 (size: 1589.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 235 (ShuffledRDD[128] at reduceByKey at KMeans.scala:302) -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Adding task set 235.0 with 2 tasks -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 235.0 (TID 64, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 235.0 (TID 65, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:26:27 INFO Executor:59 - Running task 1.0 in stage 235.0 (TID 65) -2016-06-13 19:26:27 INFO Executor:59 - Running task 0.0 in stage 235.0 (TID 64) -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:27 INFO Executor:59 - Finished task 1.0 in stage 235.0 (TID 65). 1671 bytes result sent to driver -2016-06-13 19:26:27 INFO Executor:59 - Finished task 0.0 in stage 235.0 (TID 64). 1785 bytes result sent to driver -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 235.0 (TID 65) in 10 ms on localhost (1/2) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 235.0 (TID 64) in 11 ms on localhost (2/2) -2016-06-13 19:26:27 INFO DAGScheduler:59 - ResultStage 235 (collectAsMap at KMeans.scala:302) finished in 0.013 s -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 235.0, whose tasks have all completed, from pool -2016-06-13 19:26:27 INFO DAGScheduler:59 - Job 21 finished: collectAsMap at KMeans.scala:302, took 0.072291 s -2016-06-13 19:26:27 INFO KMeans:59 - Run 0 finished in 1 iterations -2016-06-13 19:26:27 INFO KMeans:59 - Run 1 finished in 1 iterations -2016-06-13 19:26:27 INFO KMeans:59 - Run 2 finished in 1 iterations -2016-06-13 19:26:27 INFO KMeans:59 - Run 3 finished in 1 iterations -2016-06-13 19:26:27 INFO KMeans:59 - Run 4 finished in 1 iterations -2016-06-13 19:26:27 INFO KMeans:59 - Iterations took 0.105 seconds. -2016-06-13 19:26:27 INFO KMeans:59 - KMeans converged in 1 iterations. -2016-06-13 19:26:27 INFO KMeans:59 - The cost for the best run is 6.650150218788335E-6. -2016-06-13 19:26:27 INFO MapPartitionsRDD:59 - Removing RDD 100 from persistence list -2016-06-13 19:26:27 INFO BlockManager:59 - Removing RDD 100 -2016-06-13 19:26:27 WARN KMeans:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:26:27 INFO SparkContext:59 - Starting job: collect at PowerIterationClusteringSuite.scala:56 -2016-06-13 19:26:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:26:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:26:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:26:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:26:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:26:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 151 bytes -2016-06-13 19:26:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:26:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 151 bytes -2016-06-13 19:26:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:26:27 INFO DAGScheduler:59 - Got job 22 (collect at PowerIterationClusteringSuite.scala:56) with 2 output partitions -2016-06-13 19:26:27 INFO DAGScheduler:59 - Final stage: ResultStage 247(collect at PowerIterationClusteringSuite.scala:56) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 242, ShuffleMapStage 239, ShuffleMapStage 246, ShuffleMapStage 240, ShuffleMapStage 244) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting ResultStage 247 (MapPartitionsRDD[131] at mapPartitions at PowerIterationClustering.scala:232), which has no missing parents -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(8472) called with curMem=289133, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 8.3 KB, free 730.3 MB) -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=297605, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:49636 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 247 (MapPartitionsRDD[131] at mapPartitions at PowerIterationClustering.scala:232) -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Adding task set 247.0 with 2 tasks -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 247.0 (TID 66, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 247.0 (TID 67, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:26:27 INFO Executor:59 - Running task 0.0 in stage 247.0 (TID 66) -2016-06-13 19:26:27 INFO Executor:59 - Running task 1.0 in stage 247.0 (TID 67) -2016-06-13 19:26:27 INFO CacheManager:59 - Partition rdd_129_0 not found, computing it -2016-06-13 19:26:27 INFO CacheManager:59 - Partition rdd_129_1 not found, computing it -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=301197, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block rdd_129_0 stored as values in memory (estimated size 1168.0 B, free 730.3 MB) -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=302365, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block rdd_129_1 stored as values in memory (estimated size 1168.0 B, free 730.3 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added rdd_129_0 in memory on localhost:49636 (size: 1168.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added rdd_129_1 in memory on localhost:49636 (size: 1168.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO Executor:59 - Finished task 0.0 in stage 247.0 (TID 66). 2919 bytes result sent to driver -2016-06-13 19:26:27 INFO Executor:59 - Finished task 1.0 in stage 247.0 (TID 67). 2919 bytes result sent to driver -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 247.0 (TID 66) in 11 ms on localhost (1/2) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 247.0 (TID 67) in 10 ms on localhost (2/2) -2016-06-13 19:26:27 INFO DAGScheduler:59 - ResultStage 247 (collect at PowerIterationClusteringSuite.scala:56) finished in 0.012 s -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 247.0, whose tasks have all completed, from pool -2016-06-13 19:26:27 INFO DAGScheduler:59 - Job 22 finished: collect at PowerIterationClusteringSuite.scala:56, took 0.031247 s -2016-06-13 19:26:27 INFO SparkContext:59 - Starting job: foreach at PowerIterationClusteringSuite.scala:62 -2016-06-13 19:26:27 INFO DAGScheduler:59 - Got job 23 (foreach at PowerIterationClusteringSuite.scala:62) with 2 output partitions -2016-06-13 19:26:27 INFO DAGScheduler:59 - Final stage: ResultStage 259(foreach at PowerIterationClusteringSuite.scala:62) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 251, ShuffleMapStage 252, ShuffleMapStage 256, ShuffleMapStage 258, ShuffleMapStage 254) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting ResultStage 259 (MapPartitionsRDD[131] at mapPartitions at PowerIterationClustering.scala:232), which has no missing parents -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(8504) called with curMem=303533, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 8.3 KB, free 730.3 MB) -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(3621) called with curMem=312037, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:49636 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 259 (MapPartitionsRDD[131] at mapPartitions at PowerIterationClustering.scala:232) -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Adding task set 259.0 with 2 tasks -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 259.0 (TID 68, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 259.0 (TID 69, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:26:27 INFO Executor:59 - Running task 0.0 in stage 259.0 (TID 68) -2016-06-13 19:26:27 INFO Executor:59 - Running task 1.0 in stage 259.0 (TID 69) -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_129_1 locally -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_129_0 locally -2016-06-13 19:26:27 INFO Executor:59 - Finished task 0.0 in stage 259.0 (TID 68). 2044 bytes result sent to driver -2016-06-13 19:26:27 INFO Executor:59 - Finished task 1.0 in stage 259.0 (TID 69). 2044 bytes result sent to driver -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 259.0 (TID 68) in 9 ms on localhost (1/2) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 259.0 (TID 69) in 8 ms on localhost (2/2) -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 259.0, whose tasks have all completed, from pool -2016-06-13 19:26:27 INFO DAGScheduler:59 - ResultStage 259 (foreach at PowerIterationClusteringSuite.scala:62) finished in 0.009 s -2016-06-13 19:26:27 INFO DAGScheduler:59 - Job 23 finished: foreach at PowerIterationClusteringSuite.scala:62, took 0.034104 s -2016-06-13 19:26:27 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:340 -2016-06-13 19:26:27 INFO DAGScheduler:59 - Registering RDD 143 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Registering RDD 137 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Got job 24 (sum at PowerIterationClustering.scala:340) with 2 output partitions -2016-06-13 19:26:27 INFO DAGScheduler:59 - Final stage: ResultStage 262(sum at PowerIterationClustering.scala:340) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 260, ShuffleMapStage 261) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 260, ShuffleMapStage 261) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 260 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[143] at mapPartitions at GraphImpl.scala:235), which has no missing parents -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(3432) called with curMem=315658, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 3.4 KB, free 730.3 MB) -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(1974) called with curMem=319090, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1974.0 B, free 730.3 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:49636 (size: 1974.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 260 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[143] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Adding task set 260.0 with 2 tasks -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 261 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[137] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(3016) called with curMem=317443, maxMem=766075207 -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:49636 in memory (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 2.9 KB, free 730.3 MB) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 260.0 (TID 70, localhost, PROCESS_LOCAL, 2488 bytes) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 260.0 (TID 71, localhost, PROCESS_LOCAL, 2563 bytes) -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(1779) called with curMem=320459, maxMem=766075207 -2016-06-13 19:26:27 INFO Executor:59 - Running task 1.0 in stage 260.0 (TID 71) -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 1779.0 B, free 730.3 MB) -2016-06-13 19:26:27 INFO Executor:59 - Running task 0.0 in stage 260.0 (TID 70) -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:49636 (size: 1779.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 261 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[137] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Adding task set 261.0 with 2 tasks -2016-06-13 19:26:27 INFO CacheManager:59 - Partition rdd_134_1 not found, computing it -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 261.0 (TID 72, localhost, PROCESS_LOCAL, 2488 bytes) -2016-06-13 19:26:27 INFO CacheManager:59 - Partition rdd_134_0 not found, computing it -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 261.0 (TID 73, localhost, PROCESS_LOCAL, 2563 bytes) -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(2952) called with curMem=313734, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block rdd_134_1 stored as values in memory (estimated size 2.9 KB, free 730.3 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added rdd_134_1 in memory on localhost:49636 (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO Executor:59 - Running task 0.0 in stage 261.0 (TID 72) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:49636 in memory (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(2856) called with curMem=304622, maxMem=766075207 -2016-06-13 19:26:27 INFO CacheManager:59 - Another thread is loading rdd_134_0, waiting for it to finish... -2016-06-13 19:26:27 INFO Executor:59 - Running task 1.0 in stage 261.0 (TID 73) -2016-06-13 19:26:27 INFO MemoryStore:59 - Block rdd_134_0 stored as values in memory (estimated size 2.8 KB, free 730.3 MB) -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added rdd_134_0 in memory on localhost:49636 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:49636 in memory (size: 1589.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_134_1 locally -2016-06-13 19:26:27 INFO CacheManager:59 - Finished waiting for rdd_134_0 -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_134_0 locally -2016-06-13 19:26:27 INFO Executor:59 - Finished task 0.0 in stage 261.0 (TID 72). 1159 bytes result sent to driver -2016-06-13 19:26:27 INFO Executor:59 - Finished task 0.0 in stage 260.0 (TID 70). 1739 bytes result sent to driver -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 261.0 (TID 72) in 41 ms on localhost (1/2) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 260.0 (TID 70) in 48 ms on localhost (1/2) -2016-06-13 19:26:27 INFO Executor:59 - Finished task 1.0 in stage 261.0 (TID 73). 2254 bytes result sent to driver -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 261.0 (TID 73) in 45 ms on localhost (2/2) -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 261.0, whose tasks have all completed, from pool -2016-06-13 19:26:27 INFO DAGScheduler:59 - ShuffleMapStage 261 (mapPartitions at VertexRDD.scala:358) finished in 0.047 s -2016-06-13 19:26:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:27 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 260) -2016-06-13 19:26:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 262) -2016-06-13 19:26:27 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:49636 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Missing parents for ResultStage 262: List(ShuffleMapStage 260) -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned shuffle 10 -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:49636 in memory (size: 339.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:49636 in memory (size: 1382.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:49636 in memory (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned shuffle 9 -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:49636 in memory (size: 708.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:49636 in memory (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:49636 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-06-13 19:26:27 INFO BlockManager:59 - Removing RDD 122 -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned RDD 122 -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:49636 in memory (size: 236.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:49636 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:49636 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-06-13 19:26:27 INFO BlockManager:59 - Removing RDD 118 -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned RDD 118 -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:49636 in memory (size: 429.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:49636 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:49636 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 19:26:27 INFO BlockManager:59 - Removing RDD 114 -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned RDD 114 -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:49636 in memory (size: 582.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:49636 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:49636 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 19:26:27 INFO BlockManager:59 - Removing RDD 110 -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned RDD 110 -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:49636 in memory (size: 600.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:49636 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:49636 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 19:26:27 INFO BlockManager:59 - Removing RDD 106 -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned RDD 106 -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:49636 in memory (size: 487.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:49636 in memory (size: 9.9 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:49636 in memory (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 19:26:27 INFO BlockManager:59 - Removing RDD 100 -2016-06-13 19:26:27 INFO ContextCleaner:59 - Cleaned RDD 100 -2016-06-13 19:26:27 INFO Executor:59 - Finished task 1.0 in stage 260.0 (TID 71). 1739 bytes result sent to driver -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 260.0 (TID 71) in 184 ms on localhost (2/2) -2016-06-13 19:26:27 INFO DAGScheduler:59 - ShuffleMapStage 260 (mapPartitions at GraphImpl.scala:235) finished in 0.188 s -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 260.0, whose tasks have all completed, from pool -2016-06-13 19:26:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:27 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:26:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 262) -2016-06-13 19:26:27 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:27 INFO DAGScheduler:59 - Missing parents for ResultStage 262: List() -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting ResultStage 262 (MapPartitionsRDD[153] at values at PowerIterationClustering.scala:340), which is now runnable -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(4728) called with curMem=60053, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(2556) called with curMem=64781, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:49636 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 262 (MapPartitionsRDD[153] at values at PowerIterationClustering.scala:340) -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Adding task set 262.0 with 2 tasks -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 262.0 (TID 74, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 262.0 (TID 75, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:26:27 INFO Executor:59 - Running task 1.0 in stage 262.0 (TID 75) -2016-06-13 19:26:27 INFO Executor:59 - Running task 0.0 in stage 262.0 (TID 74) -2016-06-13 19:26:27 INFO CacheManager:59 - Partition rdd_145_0 not found, computing it -2016-06-13 19:26:27 INFO CacheManager:59 - Partition rdd_145_1 not found, computing it -2016-06-13 19:26:27 INFO CacheManager:59 - Partition rdd_140_0 not found, computing it -2016-06-13 19:26:27 INFO CacheManager:59 - Partition rdd_140_1 not found, computing it -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(1792) called with curMem=67337, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block rdd_140_1 stored as values in memory (estimated size 1792.0 B, free 730.5 MB) -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(1776) called with curMem=69129, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block rdd_140_0 stored as values in memory (estimated size 1776.0 B, free 730.5 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added rdd_140_1 in memory on localhost:49636 (size: 1792.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added rdd_140_0 in memory on localhost:49636 (size: 1776.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(1824) called with curMem=70905, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block rdd_145_0 stored as values in memory (estimated size 1824.0 B, free 730.5 MB) -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(1840) called with curMem=72729, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block rdd_145_1 stored as values in memory (estimated size 1840.0 B, free 730.5 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added rdd_145_0 in memory on localhost:49636 (size: 1824.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added rdd_145_1 in memory on localhost:49636 (size: 1840.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO Executor:59 - Finished task 0.0 in stage 262.0 (TID 74). 1840 bytes result sent to driver -2016-06-13 19:26:27 INFO Executor:59 - Finished task 1.0 in stage 262.0 (TID 75). 1840 bytes result sent to driver -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 262.0 (TID 74) in 15 ms on localhost (1/2) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 262.0 (TID 75) in 16 ms on localhost (2/2) -2016-06-13 19:26:27 INFO DAGScheduler:59 - ResultStage 262 (sum at PowerIterationClustering.scala:340) finished in 0.017 s -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 262.0, whose tasks have all completed, from pool -2016-06-13 19:26:27 INFO DAGScheduler:59 - Job 24 finished: sum at PowerIterationClustering.scala:340, took 0.253972 s -2016-06-13 19:26:27 INFO SparkContext:59 - Starting job: reduce at VertexRDDImpl.scala:90 -2016-06-13 19:26:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:26:27 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:26:27 INFO DAGScheduler:59 - Got job 25 (reduce at VertexRDDImpl.scala:90) with 2 output partitions -2016-06-13 19:26:27 INFO DAGScheduler:59 - Final stage: ResultStage 265(reduce at VertexRDDImpl.scala:90) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 263, ShuffleMapStage 264) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting ResultStage 265 (MapPartitionsRDD[158] at map at VertexRDDImpl.scala:90), which has no missing parents -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(5200) called with curMem=74569, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(2659) called with curMem=79769, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:49636 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 265 (MapPartitionsRDD[158] at map at VertexRDDImpl.scala:90) -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Adding task set 265.0 with 2 tasks -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 265.0 (TID 76, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 265.0 (TID 77, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:26:27 INFO Executor:59 - Running task 1.0 in stage 265.0 (TID 77) -2016-06-13 19:26:27 INFO Executor:59 - Running task 0.0 in stage 265.0 (TID 76) -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_145_1 locally -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_145_0 locally -2016-06-13 19:26:27 INFO Executor:59 - Finished task 0.0 in stage 265.0 (TID 76). 2161 bytes result sent to driver -2016-06-13 19:26:27 INFO Executor:59 - Finished task 1.0 in stage 265.0 (TID 77). 2161 bytes result sent to driver -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 265.0 (TID 76) in 4 ms on localhost (1/2) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 265.0 (TID 77) in 4 ms on localhost (2/2) -2016-06-13 19:26:27 INFO DAGScheduler:59 - ResultStage 265 (reduce at VertexRDDImpl.scala:90) finished in 0.005 s -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 265.0, whose tasks have all completed, from pool -2016-06-13 19:26:27 INFO DAGScheduler:59 - Job 25 finished: reduce at VertexRDDImpl.scala:90, took 0.015377 s -2016-06-13 19:26:27 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:26:27 INFO DAGScheduler:59 - Registering RDD 147 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Registering RDD 159 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Registering RDD 163 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Got job 26 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:26:27 INFO DAGScheduler:59 - Final stage: ResultStage 273(sum at PowerIterationClustering.scala:369) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 269, ShuffleMapStage 270, ShuffleMapStage 272) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 272) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 268 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[147] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=82428, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 4.1 KB, free 730.5 MB) -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(2378) called with curMem=86676, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:49636 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 268 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[147] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Adding task set 268.0 with 2 tasks -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 271 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[159] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(4904) called with curMem=89054, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 4.8 KB, free 730.5 MB) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 268.0 (TID 78, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(2572) called with curMem=93958, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 268.0 (TID 79, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:49636 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO Executor:59 - Running task 0.0 in stage 268.0 (TID 78) -2016-06-13 19:26:27 INFO Executor:59 - Running task 1.0 in stage 268.0 (TID 79) -2016-06-13 19:26:27 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 271 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[159] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Adding task set 271.0 with 2 tasks -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_145_1 locally -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 271.0 (TID 80, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_145_0 locally -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 271.0 (TID 81, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:26:27 INFO Executor:59 - Running task 0.0 in stage 271.0 (TID 80) -2016-06-13 19:26:27 INFO Executor:59 - Running task 1.0 in stage 271.0 (TID 81) -2016-06-13 19:26:27 INFO CacheManager:59 - Partition rdd_156_1 not found, computing it -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_145_1 locally -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=96530, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block rdd_156_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:26:27 INFO CacheManager:59 - Partition rdd_156_0 not found, computing it -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_145_0 locally -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=97962, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block rdd_156_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added rdd_156_1 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added rdd_156_0 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:27 INFO Executor:59 - Finished task 0.0 in stage 271.0 (TID 80). 2834 bytes result sent to driver -2016-06-13 19:26:27 INFO Executor:59 - Finished task 1.0 in stage 271.0 (TID 81). 2834 bytes result sent to driver -2016-06-13 19:26:27 INFO Executor:59 - Finished task 0.0 in stage 268.0 (TID 78). 2254 bytes result sent to driver -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 271.0 (TID 80) in 23 ms on localhost (1/2) -2016-06-13 19:26:27 INFO Executor:59 - Finished task 1.0 in stage 268.0 (TID 79). 2254 bytes result sent to driver -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 271.0 (TID 81) in 24 ms on localhost (2/2) -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 271.0, whose tasks have all completed, from pool -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 268.0 (TID 78) in 34 ms on localhost (1/2) -2016-06-13 19:26:27 INFO DAGScheduler:59 - ShuffleMapStage 271 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.029 s -2016-06-13 19:26:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:27 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 268) -2016-06-13 19:26:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 273, ShuffleMapStage 272) -2016-06-13 19:26:27 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 268.0 (TID 79) in 36 ms on localhost (2/2) -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 268.0, whose tasks have all completed, from pool -2016-06-13 19:26:27 INFO DAGScheduler:59 - Missing parents for ResultStage 273: List(ShuffleMapStage 272) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 272: List(ShuffleMapStage 268) -2016-06-13 19:26:27 INFO DAGScheduler:59 - ShuffleMapStage 268 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.050 s -2016-06-13 19:26:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:27 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:26:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 273, ShuffleMapStage 272) -2016-06-13 19:26:27 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:27 INFO DAGScheduler:59 - Missing parents for ResultStage 273: List(ShuffleMapStage 272) -2016-06-13 19:26:27 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 272: List() -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 272 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[163] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=99394, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 4.7 KB, free 730.5 MB) -2016-06-13 19:26:27 INFO MemoryStore:59 - ensureFreeSpace(2538) called with curMem=104242, maxMem=766075207 -2016-06-13 19:26:27 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 19:26:27 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:49636 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:26:27 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 272 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[163] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:26:27 INFO TaskSchedulerImpl:59 - Adding task set 272.0 with 2 tasks -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 272.0 (TID 82, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:26:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 272.0 (TID 83, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:26:27 INFO Executor:59 - Running task 0.0 in stage 272.0 (TID 82) -2016-06-13 19:26:27 INFO Executor:59 - Running task 1.0 in stage 272.0 (TID 83) -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_134_1 locally -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:27 INFO BlockManager:59 - Found block rdd_134_0 locally -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:26:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:26:28 INFO Executor:59 - Finished task 1.0 in stage 272.0 (TID 83). 2470 bytes result sent to driver -2016-06-13 19:26:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 272.0 (TID 83) in 381 ms on localhost (1/2) -2016-06-13 19:26:28 INFO Executor:59 - Finished task 0.0 in stage 272.0 (TID 82). 2470 bytes result sent to driver -2016-06-13 19:26:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 272.0 (TID 82) in 386 ms on localhost (2/2) -2016-06-13 19:26:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 272.0, whose tasks have all completed, from pool -2016-06-13 19:26:28 INFO DAGScheduler:59 - ShuffleMapStage 272 (mapPartitions at GraphImpl.scala:235) finished in 0.386 s -2016-06-13 19:26:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:28 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:26:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 273) -2016-06-13 19:26:28 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:28 INFO DAGScheduler:59 - Missing parents for ResultStage 273: List() -2016-06-13 19:26:28 INFO DAGScheduler:59 - Submitting ResultStage 273 (MapPartitionsRDD[168] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(5944) called with curMem=106780, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 5.8 KB, free 730.5 MB) -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=112724, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.5 MB) -2016-06-13 19:26:28 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:49636 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:26:28 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 273 (MapPartitionsRDD[168] at map at PowerIterationClustering.scala:369) -2016-06-13 19:26:28 INFO TaskSchedulerImpl:59 - Adding task set 273.0 with 2 tasks -2016-06-13 19:26:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 273.0 (TID 84, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:26:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 273.0 (TID 85, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:26:28 INFO Executor:59 - Running task 0.0 in stage 273.0 (TID 84) -2016-06-13 19:26:28 INFO Executor:59 - Running task 1.0 in stage 273.0 (TID 85) -2016-06-13 19:26:28 INFO CacheManager:59 - Partition rdd_165_0 not found, computing it -2016-06-13 19:26:28 INFO BlockManager:59 - Found block rdd_156_0 locally -2016-06-13 19:26:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:26:28 INFO CacheManager:59 - Partition rdd_165_1 not found, computing it -2016-06-13 19:26:28 INFO BlockManager:59 - Found block rdd_156_1 locally -2016-06-13 19:26:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=115616, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block rdd_165_0 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:26:28 INFO BlockManagerInfo:59 - Added rdd_165_0 in memory on localhost:49636 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:26:28 INFO Executor:59 - Finished task 0.0 in stage 273.0 (TID 84). 2880 bytes result sent to driver -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=117096, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block rdd_165_1 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:26:28 INFO BlockManagerInfo:59 - Added rdd_165_1 in memory on localhost:49636 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:26:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 273.0 (TID 84) in 15 ms on localhost (1/2) -2016-06-13 19:26:28 INFO Executor:59 - Finished task 1.0 in stage 273.0 (TID 85). 2880 bytes result sent to driver -2016-06-13 19:26:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 273.0 (TID 85) in 16 ms on localhost (2/2) -2016-06-13 19:26:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 273.0, whose tasks have all completed, from pool -2016-06-13 19:26:28 INFO DAGScheduler:59 - ResultStage 273 (sum at PowerIterationClustering.scala:369) finished in 0.018 s -2016-06-13 19:26:28 INFO DAGScheduler:59 - Job 26 finished: sum at PowerIterationClustering.scala:369, took 0.506163 s -2016-06-13 19:26:28 INFO PowerIterationClustering:59 - Iteration 0: norm(v) = 36.22334869431644. -2016-06-13 19:26:28 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:26:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:26:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:26:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:26:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 151 bytes -2016-06-13 19:26:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:26:28 INFO DAGScheduler:59 - Got job 27 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:26:28 INFO DAGScheduler:59 - Final stage: ResultStage 281(sum at PowerIterationClustering.scala:375) -2016-06-13 19:26:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 277, ShuffleMapStage 278, ShuffleMapStage 280) -2016-06-13 19:26:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:28 INFO DAGScheduler:59 - Submitting ResultStage 281 (MapPartitionsRDD[179] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(6480) called with curMem=118576, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 6.3 KB, free 730.5 MB) -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(3063) called with curMem=125056, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.5 MB) -2016-06-13 19:26:28 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:49636 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:26:28 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 281 (MapPartitionsRDD[179] at values at PowerIterationClustering.scala:375) -2016-06-13 19:26:28 INFO TaskSchedulerImpl:59 - Adding task set 281.0 with 2 tasks -2016-06-13 19:26:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 281.0 (TID 86, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:26:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 281.0 (TID 87, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:26:28 INFO Executor:59 - Running task 0.0 in stage 281.0 (TID 86) -2016-06-13 19:26:28 INFO CacheManager:59 - Partition rdd_171_0 not found, computing it -2016-06-13 19:26:28 INFO BlockManager:59 - Found block rdd_156_0 locally -2016-06-13 19:26:28 INFO BlockManager:59 - Found block rdd_165_0 locally -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=128119, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block rdd_171_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:26:28 INFO Executor:59 - Running task 1.0 in stage 281.0 (TID 87) -2016-06-13 19:26:28 INFO BlockManagerInfo:59 - Added rdd_171_0 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:28 INFO Executor:59 - Finished task 0.0 in stage 281.0 (TID 86). 2664 bytes result sent to driver -2016-06-13 19:26:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 281.0 (TID 86) in 11 ms on localhost (1/2) -2016-06-13 19:26:28 INFO CacheManager:59 - Partition rdd_171_1 not found, computing it -2016-06-13 19:26:28 INFO BlockManager:59 - Found block rdd_156_1 locally -2016-06-13 19:26:28 INFO BlockManager:59 - Found block rdd_165_1 locally -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=129551, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block rdd_171_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:26:28 INFO BlockManagerInfo:59 - Added rdd_171_1 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:28 INFO Executor:59 - Finished task 1.0 in stage 281.0 (TID 87). 2664 bytes result sent to driver -2016-06-13 19:26:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 281.0 (TID 87) in 20 ms on localhost (2/2) -2016-06-13 19:26:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 281.0, whose tasks have all completed, from pool -2016-06-13 19:26:28 INFO DAGScheduler:59 - ResultStage 281 (sum at PowerIterationClustering.scala:375) finished in 0.023 s -2016-06-13 19:26:28 INFO DAGScheduler:59 - Job 27 finished: sum at PowerIterationClustering.scala:375, took 0.050788 s -2016-06-13 19:26:28 INFO PowerIterationClustering:59 - Iteration 0: delta = 0.010579646370791257. -2016-06-13 19:26:28 INFO PowerIterationClustering:59 - Iteration 0: diff(delta) = 1.7976931348623157E308. -2016-06-13 19:26:28 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:26:28 INFO DAGScheduler:59 - Registering RDD 182 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:26:28 INFO DAGScheduler:59 - Registering RDD 186 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:26:28 INFO DAGScheduler:59 - Got job 28 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:26:28 INFO DAGScheduler:59 - Final stage: ResultStage 291(sum at PowerIterationClustering.scala:369) -2016-06-13 19:26:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 288, ShuffleMapStage 285, ShuffleMapStage 286, ShuffleMapStage 290) -2016-06-13 19:26:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 290) -2016-06-13 19:26:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 289 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[182] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(5704) called with curMem=130983, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.6 KB, free 730.5 MB) -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(2815) called with curMem=136687, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.5 MB) -2016-06-13 19:26:28 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:49636 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:26:28 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 289 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[182] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:26:28 INFO TaskSchedulerImpl:59 - Adding task set 289.0 with 2 tasks -2016-06-13 19:26:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 289.0 (TID 88, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:26:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 289.0 (TID 89, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:26:28 INFO Executor:59 - Running task 0.0 in stage 289.0 (TID 88) -2016-06-13 19:26:28 INFO Executor:59 - Running task 1.0 in stage 289.0 (TID 89) -2016-06-13 19:26:28 INFO CacheManager:59 - Partition rdd_180_1 not found, computing it -2016-06-13 19:26:28 INFO BlockManager:59 - Found block rdd_165_1 locally -2016-06-13 19:26:28 INFO CacheManager:59 - Partition rdd_180_0 not found, computing it -2016-06-13 19:26:28 INFO BlockManager:59 - Found block rdd_165_0 locally -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=139502, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block rdd_180_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=140934, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block rdd_180_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:26:28 INFO BlockManagerInfo:59 - Added rdd_180_1 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:28 INFO BlockManagerInfo:59 - Added rdd_180_0 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:28 INFO Executor:59 - Finished task 0.0 in stage 289.0 (TID 88). 2834 bytes result sent to driver -2016-06-13 19:26:28 INFO Executor:59 - Finished task 1.0 in stage 289.0 (TID 89). 2834 bytes result sent to driver -2016-06-13 19:26:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 289.0 (TID 88) in 12 ms on localhost (1/2) -2016-06-13 19:26:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 289.0 (TID 89) in 13 ms on localhost (2/2) -2016-06-13 19:26:28 INFO DAGScheduler:59 - ShuffleMapStage 289 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.014 s -2016-06-13 19:26:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 289.0, whose tasks have all completed, from pool -2016-06-13 19:26:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:28 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:26:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 291, ShuffleMapStage 290) -2016-06-13 19:26:28 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:28 INFO DAGScheduler:59 - Missing parents for ResultStage 291: List(ShuffleMapStage 290) -2016-06-13 19:26:28 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 290: List() -2016-06-13 19:26:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 290 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[186] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=142366, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=147214, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 19:26:28 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:49636 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:26:28 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 290 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[186] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:26:28 INFO TaskSchedulerImpl:59 - Adding task set 290.0 with 2 tasks -2016-06-13 19:26:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 290.0 (TID 90, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:26:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 290.0 (TID 91, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:26:28 INFO Executor:59 - Running task 0.0 in stage 290.0 (TID 90) -2016-06-13 19:26:28 INFO Executor:59 - Running task 1.0 in stage 290.0 (TID 91) -2016-06-13 19:26:28 INFO BlockManager:59 - Found block rdd_134_1 locally -2016-06-13 19:26:28 INFO BlockManager:59 - Found block rdd_134_0 locally -2016-06-13 19:26:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:26:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:28 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:26:28 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:26:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:26:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:26:28 INFO Executor:59 - Finished task 0.0 in stage 290.0 (TID 90). 2470 bytes result sent to driver -2016-06-13 19:26:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 290.0 (TID 90) in 27 ms on localhost (1/2) -2016-06-13 19:26:28 INFO Executor:59 - Finished task 1.0 in stage 290.0 (TID 91). 2470 bytes result sent to driver -2016-06-13 19:26:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 290.0 (TID 91) in 442 ms on localhost (2/2) -2016-06-13 19:26:28 INFO DAGScheduler:59 - ShuffleMapStage 290 (mapPartitions at GraphImpl.scala:235) finished in 0.443 s -2016-06-13 19:26:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 290.0, whose tasks have all completed, from pool -2016-06-13 19:26:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:28 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:26:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 291) -2016-06-13 19:26:28 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:28 INFO DAGScheduler:59 - Missing parents for ResultStage 291: List() -2016-06-13 19:26:28 INFO DAGScheduler:59 - Submitting ResultStage 291 (MapPartitionsRDD[191] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(6640) called with curMem=149756, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 6.5 KB, free 730.4 MB) -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(3103) called with curMem=156396, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 19:26:28 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:49636 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:26:28 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 291 (MapPartitionsRDD[191] at map at PowerIterationClustering.scala:369) -2016-06-13 19:26:28 INFO TaskSchedulerImpl:59 - Adding task set 291.0 with 2 tasks -2016-06-13 19:26:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 291.0 (TID 92, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:26:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 291.0 (TID 93, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:26:28 INFO Executor:59 - Running task 1.0 in stage 291.0 (TID 93) -2016-06-13 19:26:28 INFO Executor:59 - Running task 0.0 in stage 291.0 (TID 92) -2016-06-13 19:26:28 INFO CacheManager:59 - Partition rdd_188_1 not found, computing it -2016-06-13 19:26:28 INFO BlockManager:59 - Found block rdd_180_1 locally -2016-06-13 19:26:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:28 INFO CacheManager:59 - Partition rdd_188_0 not found, computing it -2016-06-13 19:26:28 INFO BlockManager:59 - Found block rdd_180_0 locally -2016-06-13 19:26:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=159499, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block rdd_188_1 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=160979, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block rdd_188_0 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:26:28 INFO BlockManagerInfo:59 - Added rdd_188_1 in memory on localhost:49636 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:26:28 INFO Executor:59 - Finished task 1.0 in stage 291.0 (TID 93). 2880 bytes result sent to driver -2016-06-13 19:26:28 INFO BlockManagerInfo:59 - Added rdd_188_0 in memory on localhost:49636 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:26:28 INFO Executor:59 - Finished task 0.0 in stage 291.0 (TID 92). 2880 bytes result sent to driver -2016-06-13 19:26:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 291.0 (TID 93) in 12 ms on localhost (1/2) -2016-06-13 19:26:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 291.0 (TID 92) in 13 ms on localhost (2/2) -2016-06-13 19:26:28 INFO DAGScheduler:59 - ResultStage 291 (sum at PowerIterationClustering.scala:369) finished in 0.013 s -2016-06-13 19:26:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 291.0, whose tasks have all completed, from pool -2016-06-13 19:26:28 INFO DAGScheduler:59 - Job 28 finished: sum at PowerIterationClustering.scala:369, took 0.508109 s -2016-06-13 19:26:28 INFO PowerIterationClustering:59 - Iteration 1: norm(v) = 36.22334869431644. -2016-06-13 19:26:28 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:26:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:26:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:26:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:26:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 151 bytes -2016-06-13 19:26:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:26:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 151 bytes -2016-06-13 19:26:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:26:28 INFO DAGScheduler:59 - Got job 29 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:26:28 INFO DAGScheduler:59 - Final stage: ResultStage 301(sum at PowerIterationClustering.scala:375) -2016-06-13 19:26:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 300, ShuffleMapStage 296, ShuffleMapStage 298, ShuffleMapStage 295) -2016-06-13 19:26:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:28 INFO DAGScheduler:59 - Submitting ResultStage 301 (MapPartitionsRDD[202] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(7072) called with curMem=162459, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 6.9 KB, free 730.4 MB) -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(3232) called with curMem=169531, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:26:28 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:49636 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:26:28 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 301 (MapPartitionsRDD[202] at values at PowerIterationClustering.scala:375) -2016-06-13 19:26:28 INFO TaskSchedulerImpl:59 - Adding task set 301.0 with 2 tasks -2016-06-13 19:26:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 301.0 (TID 94, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:26:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 301.0 (TID 95, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:26:28 INFO Executor:59 - Running task 1.0 in stage 301.0 (TID 95) -2016-06-13 19:26:28 INFO Executor:59 - Running task 0.0 in stage 301.0 (TID 94) -2016-06-13 19:26:28 INFO CacheManager:59 - Partition rdd_194_1 not found, computing it -2016-06-13 19:26:28 INFO BlockManager:59 - Found block rdd_180_1 locally -2016-06-13 19:26:28 INFO CacheManager:59 - Partition rdd_194_0 not found, computing it -2016-06-13 19:26:28 INFO BlockManager:59 - Found block rdd_188_1 locally -2016-06-13 19:26:28 INFO BlockManager:59 - Found block rdd_180_0 locally -2016-06-13 19:26:28 INFO BlockManager:59 - Found block rdd_188_0 locally -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=172763, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block rdd_194_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:26:28 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=174195, maxMem=766075207 -2016-06-13 19:26:28 INFO MemoryStore:59 - Block rdd_194_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:26:28 INFO BlockManagerInfo:59 - Added rdd_194_1 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:28 INFO BlockManagerInfo:59 - Added rdd_194_0 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:28 INFO Executor:59 - Finished task 1.0 in stage 301.0 (TID 95). 2664 bytes result sent to driver -2016-06-13 19:26:28 INFO Executor:59 - Finished task 0.0 in stage 301.0 (TID 94). 2664 bytes result sent to driver -2016-06-13 19:26:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 301.0 (TID 95) in 9 ms on localhost (1/2) -2016-06-13 19:26:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 301.0 (TID 94) in 10 ms on localhost (2/2) -2016-06-13 19:26:28 INFO DAGScheduler:59 - ResultStage 301 (sum at PowerIterationClustering.scala:375) finished in 0.011 s -2016-06-13 19:26:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 301.0, whose tasks have all completed, from pool -2016-06-13 19:26:28 INFO DAGScheduler:59 - Job 29 finished: sum at PowerIterationClustering.scala:375, took 0.026313 s -2016-06-13 19:26:28 INFO PowerIterationClustering:59 - Iteration 1: delta = 0.0. -2016-06-13 19:26:28 INFO PowerIterationClustering:59 - Iteration 1: diff(delta) = 0.010579646370791257. -2016-06-13 19:26:28 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:26:28 INFO DAGScheduler:59 - Registering RDD 205 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:26:28 INFO DAGScheduler:59 - Registering RDD 209 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:26:28 INFO DAGScheduler:59 - Got job 30 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:26:28 INFO DAGScheduler:59 - Final stage: ResultStage 313(sum at PowerIterationClustering.scala:369) -2016-06-13 19:26:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 305, ShuffleMapStage 306, ShuffleMapStage 310, ShuffleMapStage 308, ShuffleMapStage 312) -2016-06-13 19:26:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 312) -2016-06-13 19:26:29 INFO DAGScheduler:59 - Submitting ShuffleMapStage 311 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[205] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:26:29 INFO MemoryStore:59 - ensureFreeSpace(6288) called with curMem=175627, maxMem=766075207 -2016-06-13 19:26:29 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 6.1 KB, free 730.4 MB) -2016-06-13 19:26:29 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=181915, maxMem=766075207 -2016-06-13 19:26:29 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.4 MB) -2016-06-13 19:26:29 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:49636 (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:26:29 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 311 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[205] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:26:29 INFO TaskSchedulerImpl:59 - Adding task set 311.0 with 2 tasks -2016-06-13 19:26:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 311.0 (TID 96, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:26:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 311.0 (TID 97, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:26:29 INFO Executor:59 - Running task 0.0 in stage 311.0 (TID 96) -2016-06-13 19:26:29 INFO Executor:59 - Running task 1.0 in stage 311.0 (TID 97) -2016-06-13 19:26:29 INFO CacheManager:59 - Partition rdd_203_0 not found, computing it -2016-06-13 19:26:29 INFO BlockManager:59 - Found block rdd_188_0 locally -2016-06-13 19:26:29 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=184878, maxMem=766075207 -2016-06-13 19:26:29 INFO CacheManager:59 - Partition rdd_203_1 not found, computing it -2016-06-13 19:26:29 INFO MemoryStore:59 - Block rdd_203_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:26:29 INFO BlockManager:59 - Found block rdd_188_1 locally -2016-06-13 19:26:29 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=186310, maxMem=766075207 -2016-06-13 19:26:29 INFO MemoryStore:59 - Block rdd_203_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:26:29 INFO BlockManagerInfo:59 - Added rdd_203_0 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:29 INFO BlockManagerInfo:59 - Added rdd_203_1 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:29 INFO Executor:59 - Finished task 0.0 in stage 311.0 (TID 96). 2834 bytes result sent to driver -2016-06-13 19:26:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 311.0 (TID 96) in 15 ms on localhost (1/2) -2016-06-13 19:26:29 INFO Executor:59 - Finished task 1.0 in stage 311.0 (TID 97). 2834 bytes result sent to driver -2016-06-13 19:26:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 311.0 (TID 97) in 199 ms on localhost (2/2) -2016-06-13 19:26:29 INFO DAGScheduler:59 - ShuffleMapStage 311 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.200 s -2016-06-13 19:26:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 311.0, whose tasks have all completed, from pool -2016-06-13 19:26:29 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:29 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:26:29 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 312, ResultStage 313) -2016-06-13 19:26:29 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:29 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 312: List() -2016-06-13 19:26:29 INFO DAGScheduler:59 - Missing parents for ResultStage 313: List(ShuffleMapStage 312) -2016-06-13 19:26:29 INFO DAGScheduler:59 - Submitting ShuffleMapStage 312 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[209] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:26:29 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=187742, maxMem=766075207 -2016-06-13 19:26:29 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-06-13 19:26:29 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=192590, maxMem=766075207 -2016-06-13 19:26:29 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 19:26:29 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:49636 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:26:29 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 312 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[209] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:26:29 INFO TaskSchedulerImpl:59 - Adding task set 312.0 with 2 tasks -2016-06-13 19:26:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 312.0 (TID 98, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:26:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 312.0 (TID 99, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:26:29 INFO Executor:59 - Running task 1.0 in stage 312.0 (TID 99) -2016-06-13 19:26:29 INFO Executor:59 - Running task 0.0 in stage 312.0 (TID 98) -2016-06-13 19:26:29 INFO BlockManager:59 - Found block rdd_134_1 locally -2016-06-13 19:26:29 INFO BlockManager:59 - Found block rdd_134_0 locally -2016-06-13 19:26:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:29 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:26:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:26:29 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:26:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:29 INFO Executor:59 - Finished task 1.0 in stage 312.0 (TID 99). 2470 bytes result sent to driver -2016-06-13 19:26:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 312.0 (TID 99) in 649 ms on localhost (1/2) -2016-06-13 19:26:29 INFO Executor:59 - Finished task 0.0 in stage 312.0 (TID 98). 2470 bytes result sent to driver -2016-06-13 19:26:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 312.0 (TID 98) in 655 ms on localhost (2/2) -2016-06-13 19:26:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 312.0, whose tasks have all completed, from pool -2016-06-13 19:26:29 INFO DAGScheduler:59 - ShuffleMapStage 312 (mapPartitions at GraphImpl.scala:235) finished in 0.656 s -2016-06-13 19:26:29 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:29 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:26:29 INFO DAGScheduler:59 - waiting: Set(ResultStage 313) -2016-06-13 19:26:29 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:29 INFO DAGScheduler:59 - Missing parents for ResultStage 313: List() -2016-06-13 19:26:29 INFO DAGScheduler:59 - Submitting ResultStage 313 (MapPartitionsRDD[214] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:26:29 INFO MemoryStore:59 - ensureFreeSpace(7224) called with curMem=195132, maxMem=766075207 -2016-06-13 19:26:29 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 7.1 KB, free 730.4 MB) -2016-06-13 19:26:29 INFO MemoryStore:59 - ensureFreeSpace(3233) called with curMem=202356, maxMem=766075207 -2016-06-13 19:26:29 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:26:29 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:49636 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:26:29 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 313 (MapPartitionsRDD[214] at map at PowerIterationClustering.scala:369) -2016-06-13 19:26:29 INFO TaskSchedulerImpl:59 - Adding task set 313.0 with 2 tasks -2016-06-13 19:26:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 313.0 (TID 100, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:26:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 313.0 (TID 101, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:26:29 INFO Executor:59 - Running task 0.0 in stage 313.0 (TID 100) -2016-06-13 19:26:29 INFO Executor:59 - Running task 1.0 in stage 313.0 (TID 101) -2016-06-13 19:26:29 INFO CacheManager:59 - Partition rdd_211_0 not found, computing it -2016-06-13 19:26:29 INFO BlockManager:59 - Found block rdd_203_0 locally -2016-06-13 19:26:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:29 INFO CacheManager:59 - Partition rdd_211_1 not found, computing it -2016-06-13 19:26:29 INFO BlockManager:59 - Found block rdd_203_1 locally -2016-06-13 19:26:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:29 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=205589, maxMem=766075207 -2016-06-13 19:26:29 INFO MemoryStore:59 - Block rdd_211_0 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:26:29 INFO BlockManagerInfo:59 - Added rdd_211_0 in memory on localhost:49636 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:26:29 INFO Executor:59 - Finished task 0.0 in stage 313.0 (TID 100). 2880 bytes result sent to driver -2016-06-13 19:26:29 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=207069, maxMem=766075207 -2016-06-13 19:26:29 INFO MemoryStore:59 - Block rdd_211_1 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:26:29 INFO BlockManagerInfo:59 - Added rdd_211_1 in memory on localhost:49636 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:26:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 313.0 (TID 100) in 14 ms on localhost (1/2) -2016-06-13 19:26:29 INFO Executor:59 - Finished task 1.0 in stage 313.0 (TID 101). 2880 bytes result sent to driver -2016-06-13 19:26:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 313.0 (TID 101) in 14 ms on localhost (2/2) -2016-06-13 19:26:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 313.0, whose tasks have all completed, from pool -2016-06-13 19:26:29 INFO DAGScheduler:59 - ResultStage 313 (sum at PowerIterationClustering.scala:369) finished in 0.016 s -2016-06-13 19:26:29 INFO DAGScheduler:59 - Job 30 finished: sum at PowerIterationClustering.scala:369, took 0.913055 s -2016-06-13 19:26:29 INFO PowerIterationClustering:59 - Iteration 2: norm(v) = 36.22334869431644. -2016-06-13 19:26:29 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:26:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:26:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:26:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:26:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 151 bytes -2016-06-13 19:26:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:26:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 151 bytes -2016-06-13 19:26:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:26:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 151 bytes -2016-06-13 19:26:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:26:29 INFO DAGScheduler:59 - Got job 31 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:26:29 INFO DAGScheduler:59 - Final stage: ResultStage 325(sum at PowerIterationClustering.scala:375) -2016-06-13 19:26:29 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 320, ShuffleMapStage 317, ShuffleMapStage 324, ShuffleMapStage 318, ShuffleMapStage 322) -2016-06-13 19:26:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:29 INFO DAGScheduler:59 - Submitting ResultStage 325 (MapPartitionsRDD[225] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:26:29 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=208549, maxMem=766075207 -2016-06-13 19:26:29 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 7.5 KB, free 730.4 MB) -2016-06-13 19:26:29 INFO MemoryStore:59 - ensureFreeSpace(3360) called with curMem=216205, maxMem=766075207 -2016-06-13 19:26:29 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 19:26:29 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:49636 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:26:29 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 325 (MapPartitionsRDD[225] at values at PowerIterationClustering.scala:375) -2016-06-13 19:26:29 INFO TaskSchedulerImpl:59 - Adding task set 325.0 with 2 tasks -2016-06-13 19:26:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 325.0 (TID 102, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:26:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 325.0 (TID 103, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:26:29 INFO Executor:59 - Running task 0.0 in stage 325.0 (TID 102) -2016-06-13 19:26:29 INFO Executor:59 - Running task 1.0 in stage 325.0 (TID 103) -2016-06-13 19:26:29 INFO CacheManager:59 - Partition rdd_217_1 not found, computing it -2016-06-13 19:26:29 INFO BlockManager:59 - Found block rdd_203_1 locally -2016-06-13 19:26:29 INFO BlockManager:59 - Found block rdd_211_1 locally -2016-06-13 19:26:29 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=219565, maxMem=766075207 -2016-06-13 19:26:29 INFO MemoryStore:59 - Block rdd_217_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:26:29 INFO BlockManagerInfo:59 - Added rdd_217_1 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:29 INFO Executor:59 - Finished task 1.0 in stage 325.0 (TID 103). 2664 bytes result sent to driver -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 325.0 (TID 103) in 43 ms on localhost (1/2) -2016-06-13 19:26:30 INFO CacheManager:59 - Partition rdd_217_0 not found, computing it -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_203_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_211_0 locally -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=220997, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block rdd_217_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added rdd_217_0 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:30 INFO Executor:59 - Finished task 0.0 in stage 325.0 (TID 102). 2664 bytes result sent to driver -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 325.0 (TID 102) in 61 ms on localhost (2/2) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 325.0, whose tasks have all completed, from pool -2016-06-13 19:26:30 INFO DAGScheduler:59 - ResultStage 325 (sum at PowerIterationClustering.scala:375) finished in 0.061 s -2016-06-13 19:26:30 INFO DAGScheduler:59 - Job 31 finished: sum at PowerIterationClustering.scala:375, took 0.089047 s -2016-06-13 19:26:30 INFO PowerIterationClustering:59 - Iteration 2: delta = 0.0. -2016-06-13 19:26:30 INFO PowerIterationClustering:59 - Iteration 2: diff(delta) = 0.0. -2016-06-13 19:26:30 WARN KMeans:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:26:30 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Got job 32 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:26:30 INFO DAGScheduler:59 - Final stage: ResultStage 337(takeSample at KMeans.scala:376) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 332, ShuffleMapStage 336, ShuffleMapStage 329, ShuffleMapStage 330, ShuffleMapStage 334) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting ResultStage 337 (MapPartitionsRDD[233] at map at KMeans.scala:210), which has no missing parents -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(7888) called with curMem=222429, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 7.7 KB, free 730.4 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(3424) called with curMem=230317, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:49636 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 337 (MapPartitionsRDD[233] at map at KMeans.scala:210) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Adding task set 337.0 with 2 tasks -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 337.0 (TID 104, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 337.0 (TID 105, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:26:30 INFO Executor:59 - Running task 0.0 in stage 337.0 (TID 104) -2016-06-13 19:26:30 INFO Executor:59 - Running task 1.0 in stage 337.0 (TID 105) -2016-06-13 19:26:30 INFO CacheManager:59 - Partition rdd_228_0 not found, computing it -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_211_0 locally -2016-06-13 19:26:30 INFO CacheManager:59 - Partition rdd_228_1 not found, computing it -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_211_1 locally -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=233741, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block rdd_228_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=235173, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block rdd_228_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added rdd_228_0 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added rdd_228_1 in memory on localhost:49636 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:26:30 INFO CacheManager:59 - Partition rdd_231_0 not found, computing it -2016-06-13 19:26:30 INFO CacheManager:59 - Partition rdd_231_1 not found, computing it -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=236605, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block rdd_231_0 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=236845, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block rdd_231_1 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added rdd_231_0 in memory on localhost:49636 (size: 240.0 B, free: 730.5 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added rdd_231_1 in memory on localhost:49636 (size: 240.0 B, free: 730.5 MB) -2016-06-13 19:26:30 INFO Executor:59 - Finished task 0.0 in stage 337.0 (TID 104). 2717 bytes result sent to driver -2016-06-13 19:26:30 INFO Executor:59 - Finished task 1.0 in stage 337.0 (TID 105). 2717 bytes result sent to driver -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 337.0 (TID 104) in 15 ms on localhost (1/2) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 337.0 (TID 105) in 15 ms on localhost (2/2) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 337.0, whose tasks have all completed, from pool -2016-06-13 19:26:30 INFO DAGScheduler:59 - ResultStage 337 (takeSample at KMeans.scala:376) finished in 0.016 s -2016-06-13 19:26:30 INFO DAGScheduler:59 - Job 32 finished: takeSample at KMeans.scala:376, took 0.033880 s -2016-06-13 19:26:30 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Got job 33 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:26:30 INFO DAGScheduler:59 - Final stage: ResultStage 349(takeSample at KMeans.scala:376) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 342, ShuffleMapStage 344, ShuffleMapStage 341, ShuffleMapStage 348, ShuffleMapStage 346) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting ResultStage 349 (PartitionwiseSampledRDD[235] at takeSample at KMeans.scala:376), which has no missing parents -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(25280) called with curMem=237085, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 24.7 KB, free 730.3 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(10107) called with curMem=262365, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 9.9 KB, free 730.3 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:49636 (size: 9.9 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 349 (PartitionwiseSampledRDD[235] at takeSample at KMeans.scala:376) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Adding task set 349.0 with 2 tasks -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 349.0 (TID 106, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 349.0 (TID 107, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:26:30 INFO Executor:59 - Running task 1.0 in stage 349.0 (TID 107) -2016-06-13 19:26:30 INFO Executor:59 - Running task 0.0 in stage 349.0 (TID 106) -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:26:30 INFO Executor:59 - Finished task 1.0 in stage 349.0 (TID 107). 2575 bytes result sent to driver -2016-06-13 19:26:30 INFO Executor:59 - Finished task 0.0 in stage 349.0 (TID 106). 2633 bytes result sent to driver -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 349.0 (TID 107) in 6 ms on localhost (1/2) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 349.0 (TID 106) in 7 ms on localhost (2/2) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 349.0, whose tasks have all completed, from pool -2016-06-13 19:26:30 INFO DAGScheduler:59 - ResultStage 349 (takeSample at KMeans.scala:376) finished in 0.008 s -2016-06-13 19:26:30 INFO DAGScheduler:59 - Job 33 finished: takeSample at KMeans.scala:376, took 0.025258 s -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=272472, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 880.0 B, free 730.3 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(487) called with curMem=273352, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 487.0 B, free 730.3 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:49636 (size: 487.0 B, free: 730.5 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 61 from broadcast at KMeans.scala:394 -2016-06-13 19:26:30 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Got job 34 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:26:30 INFO DAGScheduler:59 - Final stage: ResultStage 361(aggregate at KMeans.scala:402) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 356, ShuffleMapStage 353, ShuffleMapStage 354, ShuffleMapStage 358, ShuffleMapStage 360) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting ResultStage 361 (MapPartitionsRDD[237] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(9912) called with curMem=273839, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 9.7 KB, free 730.3 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(4245) called with curMem=283751, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 4.1 KB, free 730.3 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:49636 (size: 4.1 KB, free: 730.4 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 361 (MapPartitionsRDD[237] at map at KMeans.scala:396) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Adding task set 361.0 with 2 tasks -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 361.0 (TID 108, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 361.0 (TID 109, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:26:30 INFO Executor:59 - Running task 0.0 in stage 361.0 (TID 108) -2016-06-13 19:26:30 INFO Executor:59 - Running task 1.0 in stage 361.0 (TID 109) -2016-06-13 19:26:30 INFO CacheManager:59 - Partition rdd_237_0 not found, computing it -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:26:30 INFO CacheManager:59 - Partition rdd_237_1 not found, computing it -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=287996, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block rdd_237_0 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=288492, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block rdd_237_1 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added rdd_237_0 in memory on localhost:49636 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:26:30 INFO Executor:59 - Finished task 0.0 in stage 361.0 (TID 108). 2647 bytes result sent to driver -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added rdd_237_1 in memory on localhost:49636 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:26:30 INFO Executor:59 - Finished task 1.0 in stage 361.0 (TID 109). 2647 bytes result sent to driver -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 361.0 (TID 108) in 12 ms on localhost (1/2) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 361.0 (TID 109) in 16 ms on localhost (2/2) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 361.0, whose tasks have all completed, from pool -2016-06-13 19:26:30 INFO DAGScheduler:59 - ResultStage 361 (aggregate at KMeans.scala:402) finished in 0.018 s -2016-06-13 19:26:30 INFO DAGScheduler:59 - Job 34 finished: aggregate at KMeans.scala:402, took 0.037720 s -2016-06-13 19:26:30 INFO MapPartitionsRDD:59 - Removing RDD 234 from persistence list -2016-06-13 19:26:30 INFO BlockManager:59 - Removing RDD 234 -2016-06-13 19:26:30 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Got job 35 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:26:30 INFO DAGScheduler:59 - Final stage: ResultStage 373(collect at KMeans.scala:431) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 368, ShuffleMapStage 372, ShuffleMapStage 365, ShuffleMapStage 366, ShuffleMapStage 370) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting ResultStage 373 (MapPartitionsRDD[239] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(10072) called with curMem=288988, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 9.8 KB, free 730.3 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(4365) called with curMem=299060, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:49636 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 373 (MapPartitionsRDD[239] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Adding task set 373.0 with 2 tasks -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 373.0 (TID 110, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 373.0 (TID 111, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:26:30 INFO Executor:59 - Running task 0.0 in stage 373.0 (TID 110) -2016-06-13 19:26:30 INFO Executor:59 - Running task 1.0 in stage 373.0 (TID 111) -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_237_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_237_0 locally -2016-06-13 19:26:30 INFO Executor:59 - Finished task 1.0 in stage 373.0 (TID 111). 3471 bytes result sent to driver -2016-06-13 19:26:30 INFO Executor:59 - Finished task 0.0 in stage 373.0 (TID 110). 3118 bytes result sent to driver -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 373.0 (TID 111) in 5 ms on localhost (1/2) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 373.0 (TID 110) in 6 ms on localhost (2/2) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 373.0, whose tasks have all completed, from pool -2016-06-13 19:26:30 INFO DAGScheduler:59 - ResultStage 373 (collect at KMeans.scala:431) finished in 0.007 s -2016-06-13 19:26:30 INFO DAGScheduler:59 - Job 35 finished: collect at KMeans.scala:431, took 0.026132 s -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(1648) called with curMem=303425, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 1648.0 B, free 730.3 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(600) called with curMem=305073, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 600.0 B, free 730.3 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:49636 (size: 600.0 B, free: 730.4 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 64 from broadcast at KMeans.scala:394 -2016-06-13 19:26:30 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Got job 36 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:26:30 INFO DAGScheduler:59 - Final stage: ResultStage 385(aggregate at KMeans.scala:402) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 380, ShuffleMapStage 377, ShuffleMapStage 384, ShuffleMapStage 378, ShuffleMapStage 382) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting ResultStage 385 (MapPartitionsRDD[241] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(10144) called with curMem=305673, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 9.9 KB, free 730.3 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(4338) called with curMem=315817, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.3 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:49636 (size: 4.2 KB, free: 730.4 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 385 (MapPartitionsRDD[241] at map at KMeans.scala:396) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Adding task set 385.0 with 2 tasks -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 385.0 (TID 112, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 385.0 (TID 113, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:26:30 INFO Executor:59 - Running task 0.0 in stage 385.0 (TID 112) -2016-06-13 19:26:30 INFO Executor:59 - Running task 1.0 in stage 385.0 (TID 113) -2016-06-13 19:26:30 INFO CacheManager:59 - Partition rdd_241_1 not found, computing it -2016-06-13 19:26:30 INFO CacheManager:59 - Partition rdd_241_0 not found, computing it -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_237_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_237_0 locally -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=320155, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block rdd_241_1 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=320651, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block rdd_241_0 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added rdd_241_1 in memory on localhost:49636 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added rdd_241_0 in memory on localhost:49636 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:26:30 INFO Executor:59 - Finished task 1.0 in stage 385.0 (TID 113). 2647 bytes result sent to driver -2016-06-13 19:26:30 INFO Executor:59 - Finished task 0.0 in stage 385.0 (TID 112). 2647 bytes result sent to driver -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 385.0 (TID 113) in 7 ms on localhost (1/2) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 385.0 (TID 112) in 9 ms on localhost (2/2) -2016-06-13 19:26:30 INFO DAGScheduler:59 - ResultStage 385 (aggregate at KMeans.scala:402) finished in 0.009 s -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 385.0, whose tasks have all completed, from pool -2016-06-13 19:26:30 INFO DAGScheduler:59 - Job 36 finished: aggregate at KMeans.scala:402, took 0.027626 s -2016-06-13 19:26:30 INFO MapPartitionsRDD:59 - Removing RDD 237 from persistence list -2016-06-13 19:26:30 INFO BlockManager:59 - Removing RDD 237 -2016-06-13 19:26:30 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Got job 37 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:26:30 INFO DAGScheduler:59 - Final stage: ResultStage 397(collect at KMeans.scala:431) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 389, ShuffleMapStage 390, ShuffleMapStage 394, ShuffleMapStage 392, ShuffleMapStage 396) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting ResultStage 397 (MapPartitionsRDD[243] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(10304) called with curMem=320155, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 10.1 KB, free 730.3 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(4454) called with curMem=330459, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:49636 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 397 (MapPartitionsRDD[243] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Adding task set 397.0 with 2 tasks -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 397.0 (TID 114, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 397.0 (TID 115, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:26:30 INFO Executor:59 - Running task 1.0 in stage 397.0 (TID 115) -2016-06-13 19:26:30 INFO Executor:59 - Running task 0.0 in stage 397.0 (TID 114) -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_241_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_241_0 locally -2016-06-13 19:26:30 INFO Executor:59 - Finished task 0.0 in stage 397.0 (TID 114). 2990 bytes result sent to driver -2016-06-13 19:26:30 INFO Executor:59 - Finished task 1.0 in stage 397.0 (TID 115). 3240 bytes result sent to driver -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 397.0 (TID 114) in 6 ms on localhost (1/2) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 397.0 (TID 115) in 6 ms on localhost (2/2) -2016-06-13 19:26:30 INFO DAGScheduler:59 - ResultStage 397 (collect at KMeans.scala:431) finished in 0.007 s -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 397.0, whose tasks have all completed, from pool -2016-06-13 19:26:30 INFO DAGScheduler:59 - Job 37 finished: collect at KMeans.scala:431, took 0.020320 s -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(1456) called with curMem=334913, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 1456.0 B, free 730.3 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(582) called with curMem=336369, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 582.0 B, free 730.3 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:49636 (size: 582.0 B, free: 730.4 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 67 from broadcast at KMeans.scala:394 -2016-06-13 19:26:30 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Got job 38 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:26:30 INFO DAGScheduler:59 - Final stage: ResultStage 409(aggregate at KMeans.scala:402) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 404, ShuffleMapStage 401, ShuffleMapStage 408, ShuffleMapStage 406, ShuffleMapStage 402) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting ResultStage 409 (MapPartitionsRDD[245] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(10376) called with curMem=336951, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 10.1 KB, free 730.3 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(4387) called with curMem=347327, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:49636 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 409 (MapPartitionsRDD[245] at map at KMeans.scala:396) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Adding task set 409.0 with 2 tasks -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 409.0 (TID 116, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 409.0 (TID 117, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:26:30 INFO Executor:59 - Running task 0.0 in stage 409.0 (TID 116) -2016-06-13 19:26:30 INFO Executor:59 - Running task 1.0 in stage 409.0 (TID 117) -2016-06-13 19:26:30 INFO CacheManager:59 - Partition rdd_245_0 not found, computing it -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:26:30 INFO CacheManager:59 - Partition rdd_245_1 not found, computing it -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_241_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_241_1 locally -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=351714, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block rdd_245_0 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=352210, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block rdd_245_1 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added rdd_245_0 in memory on localhost:49636 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:26:30 INFO Executor:59 - Finished task 0.0 in stage 409.0 (TID 116). 2647 bytes result sent to driver -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 409.0 (TID 116) in 9 ms on localhost (1/2) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added rdd_245_1 in memory on localhost:49636 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:26:30 INFO Executor:59 - Finished task 1.0 in stage 409.0 (TID 117). 2647 bytes result sent to driver -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 409.0 (TID 117) in 12 ms on localhost (2/2) -2016-06-13 19:26:30 INFO DAGScheduler:59 - ResultStage 409 (aggregate at KMeans.scala:402) finished in 0.012 s -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 409.0, whose tasks have all completed, from pool -2016-06-13 19:26:30 INFO DAGScheduler:59 - Job 38 finished: aggregate at KMeans.scala:402, took 0.027814 s -2016-06-13 19:26:30 INFO MapPartitionsRDD:59 - Removing RDD 241 from persistence list -2016-06-13 19:26:30 INFO BlockManager:59 - Removing RDD 241 -2016-06-13 19:26:30 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Got job 39 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:26:30 INFO DAGScheduler:59 - Final stage: ResultStage 421(collect at KMeans.scala:431) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 420, ShuffleMapStage 416, ShuffleMapStage 413, ShuffleMapStage 414, ShuffleMapStage 418) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting ResultStage 421 (MapPartitionsRDD[247] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(10536) called with curMem=351714, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 10.3 KB, free 730.2 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(4484) called with curMem=362250, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.2 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:49636 (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 69 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 421 (MapPartitionsRDD[247] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Adding task set 421.0 with 2 tasks -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 421.0 (TID 118, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 421.0 (TID 119, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:26:30 INFO Executor:59 - Running task 1.0 in stage 421.0 (TID 119) -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_245_1 locally -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:49636 in memory (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:26:30 INFO Executor:59 - Finished task 1.0 in stage 421.0 (TID 119). 2040 bytes result sent to driver -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 421.0 (TID 119) in 7 ms on localhost (1/2) -2016-06-13 19:26:30 INFO Executor:59 - Running task 0.0 in stage 421.0 (TID 118) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_245_0 locally -2016-06-13 19:26:30 INFO Executor:59 - Finished task 0.0 in stage 421.0 (TID 118). 2722 bytes result sent to driver -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:49636 in memory (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 421.0 (TID 118) in 16 ms on localhost (2/2) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 421.0, whose tasks have all completed, from pool -2016-06-13 19:26:30 INFO DAGScheduler:59 - ResultStage 421 (collect at KMeans.scala:431) finished in 0.018 s -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Job 39 finished: collect at KMeans.scala:431, took 0.053140 s -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(648) called with curMem=337213, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 648.0 B, free 730.3 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(429) called with curMem=333523, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 429.0 B, free 730.3 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:49636 in memory (size: 4.2 KB, free: 730.4 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:49636 (size: 429.0 B, free: 730.4 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 70 from broadcast at KMeans.scala:394 -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:49636 in memory (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:49636 in memory (size: 4.1 KB, free: 730.4 MB) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-06-13 19:26:30 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:49636 in memory (size: 9.9 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:49636 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Got job 40 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:26:30 INFO DAGScheduler:59 - Final stage: ResultStage 433(aggregate at KMeans.scala:402) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 425, ShuffleMapStage 426, ShuffleMapStage 430, ShuffleMapStage 428, ShuffleMapStage 432) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting ResultStage 433 (MapPartitionsRDD[249] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:49636 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(10608) called with curMem=237499, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 10.4 KB, free 730.3 MB) -2016-06-13 19:26:30 INFO BlockManager:59 - Removing RDD 217 -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(4446) called with curMem=245243, maxMem=766075207 -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned RDD 217 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:49636 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 71 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 433 (MapPartitionsRDD[249] at map at KMeans.scala:396) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Adding task set 433.0 with 2 tasks -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:49636 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 433.0 (TID 120, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 433.0 (TID 121, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:26:30 INFO Executor:59 - Running task 1.0 in stage 433.0 (TID 121) -2016-06-13 19:26:30 INFO Executor:59 - Running task 0.0 in stage 433.0 (TID 120) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:49636 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-06-13 19:26:30 INFO CacheManager:59 - Partition rdd_249_1 not found, computing it -2016-06-13 19:26:30 INFO CacheManager:59 - Partition rdd_249_0 not found, computing it -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_245_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_245_0 locally -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:49636 in memory (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=228879, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block rdd_249_1 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=223087, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block rdd_249_0 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added rdd_249_1 in memory on localhost:49636 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:26:30 INFO Executor:59 - Finished task 1.0 in stage 433.0 (TID 121). 2647 bytes result sent to driver -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added rdd_249_0 in memory on localhost:49636 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:26:30 INFO Executor:59 - Finished task 0.0 in stage 433.0 (TID 120). 2647 bytes result sent to driver -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 433.0 (TID 121) in 11 ms on localhost (1/2) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 433.0 (TID 120) in 13 ms on localhost (2/2) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 433.0, whose tasks have all completed, from pool -2016-06-13 19:26:30 INFO DAGScheduler:59 - ResultStage 433 (aggregate at KMeans.scala:402) finished in 0.013 s -2016-06-13 19:26:30 INFO DAGScheduler:59 - Job 40 finished: aggregate at KMeans.scala:402, took 0.033351 s -2016-06-13 19:26:30 INFO MapPartitionsRDD:59 - Removing RDD 245 from persistence list -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:49636 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-06-13 19:26:30 INFO BlockManager:59 - Removing RDD 245 -2016-06-13 19:26:30 INFO BlockManager:59 - Removing RDD 194 -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned RDD 194 -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:49636 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:49636 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-06-13 19:26:30 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:49636 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:49636 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Got job 41 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:26:30 INFO DAGScheduler:59 - Final stage: ResultStage 445(collect at KMeans.scala:431) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 440, ShuffleMapStage 437, ShuffleMapStage 444, ShuffleMapStage 438, ShuffleMapStage 442) -2016-06-13 19:26:30 INFO BlockManager:59 - Removing RDD 171 -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned RDD 171 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting ResultStage 445 (MapPartitionsRDD[251] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(10768) called with curMem=162528, maxMem=766075207 -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:49636 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 10.5 KB, free 730.4 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(4526) called with curMem=173296, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:49636 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 445 (MapPartitionsRDD[251] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Adding task set 445.0 with 2 tasks -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 445.0 (TID 122, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 445.0 (TID 123, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:26:30 INFO Executor:59 - Running task 0.0 in stage 445.0 (TID 122) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:49636 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO Executor:59 - Running task 1.0 in stage 445.0 (TID 123) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_249_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_249_1 locally -2016-06-13 19:26:30 INFO Executor:59 - Finished task 0.0 in stage 445.0 (TID 122). 2040 bytes result sent to driver -2016-06-13 19:26:30 INFO Executor:59 - Finished task 1.0 in stage 445.0 (TID 123). 2040 bytes result sent to driver -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:49636 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 445.0 (TID 122) in 5 ms on localhost (1/2) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 445.0 (TID 123) in 6 ms on localhost (2/2) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 445.0, whose tasks have all completed, from pool -2016-06-13 19:26:30 INFO DAGScheduler:59 - ResultStage 445 (collect at KMeans.scala:431) finished in 0.007 s -2016-06-13 19:26:30 INFO DAGScheduler:59 - Job 41 finished: collect at KMeans.scala:431, took 0.029831 s -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(584) called with curMem=160582, maxMem=766075207 -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:49636 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 584.0 B, free 730.4 MB) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(236) called with curMem=156918, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 236.0 B, free 730.4 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:49636 (size: 236.0 B, free: 730.5 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 73 from broadcast at KMeans.scala:394 -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:49636 in memory (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:49636 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:49636 in memory (size: 1779.0 B, free: 730.5 MB) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-06-13 19:26:30 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:49636 in memory (size: 1974.0 B, free: 730.5 MB) -2016-06-13 19:26:30 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Got job 42 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:26:30 INFO DAGScheduler:59 - Final stage: ResultStage 457(aggregate at KMeans.scala:402) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 452, ShuffleMapStage 456, ShuffleMapStage 449, ShuffleMapStage 450, ShuffleMapStage 454) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting ResultStage 457 (MapPartitionsRDD[253] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(10848) called with curMem=131810, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 10.6 KB, free 730.5 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(4495) called with curMem=142658, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:49636 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 457 (MapPartitionsRDD[253] at map at KMeans.scala:396) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Adding task set 457.0 with 2 tasks -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 457.0 (TID 124, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 457.0 (TID 125, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:26:30 INFO Executor:59 - Running task 0.0 in stage 457.0 (TID 124) -2016-06-13 19:26:30 INFO Executor:59 - Running task 1.0 in stage 457.0 (TID 125) -2016-06-13 19:26:30 INFO CacheManager:59 - Partition rdd_253_0 not found, computing it -2016-06-13 19:26:30 INFO CacheManager:59 - Partition rdd_253_1 not found, computing it -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_249_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_249_1 locally -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=147153, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block rdd_253_0 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added rdd_253_0 in memory on localhost:49636 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:26:30 INFO Executor:59 - Finished task 0.0 in stage 457.0 (TID 124). 2647 bytes result sent to driver -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=147649, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block rdd_253_1 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added rdd_253_1 in memory on localhost:49636 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:26:30 INFO Executor:59 - Finished task 1.0 in stage 457.0 (TID 125). 2647 bytes result sent to driver -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 457.0 (TID 124) in 17 ms on localhost (1/2) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 457.0 (TID 125) in 17 ms on localhost (2/2) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 457.0, whose tasks have all completed, from pool -2016-06-13 19:26:30 INFO DAGScheduler:59 - ResultStage 457 (aggregate at KMeans.scala:402) finished in 0.019 s -2016-06-13 19:26:30 INFO DAGScheduler:59 - Job 42 finished: aggregate at KMeans.scala:402, took 0.040293 s -2016-06-13 19:26:30 INFO MapPartitionsRDD:59 - Removing RDD 249 from persistence list -2016-06-13 19:26:30 INFO BlockManager:59 - Removing RDD 249 -2016-06-13 19:26:30 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Got job 43 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:26:30 INFO DAGScheduler:59 - Final stage: ResultStage 469(collect at KMeans.scala:431) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 466, ShuffleMapStage 462, ShuffleMapStage 464, ShuffleMapStage 461, ShuffleMapStage 468) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting ResultStage 469 (MapPartitionsRDD[255] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=147153, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 10.8 KB, free 730.4 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(4571) called with curMem=158161, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.4 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:49636 (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 75 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 469 (MapPartitionsRDD[255] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Adding task set 469.0 with 2 tasks -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 469.0 (TID 126, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 469.0 (TID 127, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:26:30 INFO Executor:59 - Running task 1.0 in stage 469.0 (TID 127) -2016-06-13 19:26:30 INFO Executor:59 - Running task 0.0 in stage 469.0 (TID 126) -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_253_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_253_0 locally -2016-06-13 19:26:30 INFO Executor:59 - Finished task 1.0 in stage 469.0 (TID 127). 2040 bytes result sent to driver -2016-06-13 19:26:30 INFO Executor:59 - Finished task 0.0 in stage 469.0 (TID 126). 2040 bytes result sent to driver -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 469.0 (TID 127) in 6 ms on localhost (1/2) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 469.0 (TID 126) in 8 ms on localhost (2/2) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 469.0, whose tasks have all completed, from pool -2016-06-13 19:26:30 INFO DAGScheduler:59 - ResultStage 469 (collect at KMeans.scala:431) finished in 0.008 s -2016-06-13 19:26:30 INFO DAGScheduler:59 - Job 43 finished: collect at KMeans.scala:431, took 0.024249 s -2016-06-13 19:26:30 INFO MapPartitionsRDD:59 - Removing RDD 253 from persistence list -2016-06-13 19:26:30 INFO BlockManager:59 - Removing RDD 253 -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(2880) called with curMem=162732, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 2.8 KB, free 730.4 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(708) called with curMem=164620, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 708.0 B, free 730.4 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:49636 (size: 708.0 B, free: 730.5 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 76 from broadcast at KMeans.scala:445 -2016-06-13 19:26:30 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:450 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Registering RDD 256 (flatMap at KMeans.scala:446) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Got job 44 (collectAsMap at KMeans.scala:450) with 2 output partitions -2016-06-13 19:26:30 INFO DAGScheduler:59 - Final stage: ResultStage 482(collectAsMap at KMeans.scala:450) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 481) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 481) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting ShuffleMapStage 481 (MapPartitionsRDD[256] at flatMap at KMeans.scala:446), which has no missing parents -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(9392) called with curMem=165328, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 9.2 KB, free 730.4 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(4071) called with curMem=174720, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:49636 (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 77 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 481 (MapPartitionsRDD[256] at flatMap at KMeans.scala:446) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Adding task set 481.0 with 2 tasks -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 481.0 (TID 128, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 481.0 (TID 129, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:26:30 INFO Executor:59 - Running task 0.0 in stage 481.0 (TID 128) -2016-06-13 19:26:30 INFO Executor:59 - Running task 1.0 in stage 481.0 (TID 129) -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:26:30 INFO Executor:59 - Finished task 1.0 in stage 481.0 (TID 129). 2254 bytes result sent to driver -2016-06-13 19:26:30 INFO Executor:59 - Finished task 0.0 in stage 481.0 (TID 128). 2254 bytes result sent to driver -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 481.0 (TID 129) in 15 ms on localhost (1/2) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 481.0 (TID 128) in 16 ms on localhost (2/2) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 481.0, whose tasks have all completed, from pool -2016-06-13 19:26:30 INFO DAGScheduler:59 - ShuffleMapStage 481 (flatMap at KMeans.scala:446) finished in 0.016 s -2016-06-13 19:26:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:30 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:26:30 INFO DAGScheduler:59 - waiting: Set(ResultStage 482) -2016-06-13 19:26:30 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:30 INFO DAGScheduler:59 - Missing parents for ResultStage 482: List() -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting ResultStage 482 (ShuffledRDD[257] at reduceByKey at KMeans.scala:450), which is now runnable -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=178791, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(1382) called with curMem=181103, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 1382.0 B, free 730.4 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:49636 (size: 1382.0 B, free: 730.5 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 482 (ShuffledRDD[257] at reduceByKey at KMeans.scala:450) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Adding task set 482.0 with 2 tasks -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 482.0 (TID 130, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 482.0 (TID 131, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:26:30 INFO Executor:59 - Running task 1.0 in stage 482.0 (TID 131) -2016-06-13 19:26:30 INFO Executor:59 - Running task 0.0 in stage 482.0 (TID 130) -2016-06-13 19:26:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:26:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:26:30 INFO Executor:59 - Finished task 1.0 in stage 482.0 (TID 131). 1744 bytes result sent to driver -2016-06-13 19:26:30 INFO Executor:59 - Finished task 0.0 in stage 482.0 (TID 130). 1852 bytes result sent to driver -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 482.0 (TID 131) in 7 ms on localhost (1/2) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 482.0 (TID 130) in 8 ms on localhost (2/2) -2016-06-13 19:26:30 INFO DAGScheduler:59 - ResultStage 482 (collectAsMap at KMeans.scala:450) finished in 0.009 s -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 482.0, whose tasks have all completed, from pool -2016-06-13 19:26:30 INFO DAGScheduler:59 - Job 44 finished: collectAsMap at KMeans.scala:450, took 0.045076 s -2016-06-13 19:26:30 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:26:30 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:26:30 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:26:30 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:26:30 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:26:30 INFO KMeans:59 - Initialization with k-means|| took 0.654 seconds. -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(824) called with curMem=182485, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 824.0 B, free 730.4 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(339) called with curMem=183309, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 339.0 B, free 730.4 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:49636 (size: 339.0 B, free: 730.5 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 79 from broadcast at KMeans.scala:276 -2016-06-13 19:26:30 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:302 -2016-06-13 19:26:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:26:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:26:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:26:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 151 bytes -2016-06-13 19:26:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:26:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 151 bytes -2016-06-13 19:26:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:26:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 151 bytes -2016-06-13 19:26:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:26:30 INFO DAGScheduler:59 - Registering RDD 258 (mapPartitions at KMeans.scala:279) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Got job 45 (collectAsMap at KMeans.scala:302) with 2 output partitions -2016-06-13 19:26:30 INFO DAGScheduler:59 - Final stage: ResultStage 495(collectAsMap at KMeans.scala:302) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 494) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 494) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting ShuffleMapStage 494 (MapPartitionsRDD[258] at mapPartitions at KMeans.scala:279), which has no missing parents -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(10016) called with curMem=183648, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 9.8 KB, free 730.4 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(4409) called with curMem=193664, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:49636 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 494 (MapPartitionsRDD[258] at mapPartitions at KMeans.scala:279) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Adding task set 494.0 with 2 tasks -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 494.0 (TID 132, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 494.0 (TID 133, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:26:30 INFO Executor:59 - Running task 1.0 in stage 494.0 (TID 133) -2016-06-13 19:26:30 INFO Executor:59 - Running task 0.0 in stage 494.0 (TID 132) -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:26:30 INFO Executor:59 - Finished task 1.0 in stage 494.0 (TID 133). 2413 bytes result sent to driver -2016-06-13 19:26:30 INFO Executor:59 - Finished task 0.0 in stage 494.0 (TID 132). 2413 bytes result sent to driver -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 494.0 (TID 133) in 13 ms on localhost (1/2) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 494.0 (TID 132) in 15 ms on localhost (2/2) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 494.0, whose tasks have all completed, from pool -2016-06-13 19:26:30 INFO DAGScheduler:59 - ShuffleMapStage 494 (mapPartitions at KMeans.scala:279) finished in 0.015 s -2016-06-13 19:26:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:26:30 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:26:30 INFO DAGScheduler:59 - waiting: Set(ResultStage 495) -2016-06-13 19:26:30 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:26:30 INFO DAGScheduler:59 - Missing parents for ResultStage 495: List() -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting ResultStage 495 (ShuffledRDD[259] at reduceByKey at KMeans.scala:302), which is now runnable -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(2808) called with curMem=198073, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(1589) called with curMem=200881, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 1589.0 B, free 730.4 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:49636 (size: 1589.0 B, free: 730.5 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 81 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 495 (ShuffledRDD[259] at reduceByKey at KMeans.scala:302) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Adding task set 495.0 with 2 tasks -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 495.0 (TID 134, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 495.0 (TID 135, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:26:30 INFO Executor:59 - Running task 1.0 in stage 495.0 (TID 135) -2016-06-13 19:26:30 INFO Executor:59 - Running task 0.0 in stage 495.0 (TID 134) -2016-06-13 19:26:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:26:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:26:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:26:30 INFO Executor:59 - Finished task 1.0 in stage 495.0 (TID 135). 1671 bytes result sent to driver -2016-06-13 19:26:30 INFO Executor:59 - Finished task 0.0 in stage 495.0 (TID 134). 1785 bytes result sent to driver -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 495.0 (TID 135) in 6 ms on localhost (1/2) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 495.0 (TID 134) in 7 ms on localhost (2/2) -2016-06-13 19:26:30 INFO DAGScheduler:59 - ResultStage 495 (collectAsMap at KMeans.scala:302) finished in 0.007 s -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 495.0, whose tasks have all completed, from pool -2016-06-13 19:26:30 INFO DAGScheduler:59 - Job 45 finished: collectAsMap at KMeans.scala:302, took 0.047626 s -2016-06-13 19:26:30 INFO KMeans:59 - Run 0 finished in 1 iterations -2016-06-13 19:26:30 INFO KMeans:59 - Run 1 finished in 1 iterations -2016-06-13 19:26:30 INFO KMeans:59 - Run 2 finished in 1 iterations -2016-06-13 19:26:30 INFO KMeans:59 - Run 3 finished in 1 iterations -2016-06-13 19:26:30 INFO KMeans:59 - Run 4 finished in 1 iterations -2016-06-13 19:26:30 INFO KMeans:59 - Iterations took 0.066 seconds. -2016-06-13 19:26:30 INFO KMeans:59 - KMeans converged in 1 iterations. -2016-06-13 19:26:30 INFO KMeans:59 - The cost for the best run is 6.650150218788335E-6. -2016-06-13 19:26:30 INFO MapPartitionsRDD:59 - Removing RDD 231 from persistence list -2016-06-13 19:26:30 INFO BlockManager:59 - Removing RDD 231 -2016-06-13 19:26:30 WARN KMeans:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:26:30 INFO SparkContext:59 - Starting job: collect at PowerIterationClusteringSuite.scala:71 -2016-06-13 19:26:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:26:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:26:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:26:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 151 bytes -2016-06-13 19:26:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:26:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 151 bytes -2016-06-13 19:26:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:26:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 151 bytes -2016-06-13 19:26:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:26:30 INFO DAGScheduler:59 - Got job 46 (collect at PowerIterationClusteringSuite.scala:71) with 2 output partitions -2016-06-13 19:26:30 INFO DAGScheduler:59 - Final stage: ResultStage 507(collect at PowerIterationClusteringSuite.scala:71) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 502, ShuffleMapStage 506, ShuffleMapStage 499, ShuffleMapStage 500, ShuffleMapStage 504) -2016-06-13 19:26:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting ResultStage 507 (MapPartitionsRDD[262] at mapPartitions at PowerIterationClustering.scala:232), which has no missing parents -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(8192) called with curMem=201990, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 8.0 KB, free 730.4 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(3573) called with curMem=210182, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:49636 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:26:30 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 507 (MapPartitionsRDD[262] at mapPartitions at PowerIterationClustering.scala:232) -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Adding task set 507.0 with 2 tasks -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 507.0 (TID 136, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 507.0 (TID 137, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:26:30 INFO Executor:59 - Running task 0.0 in stage 507.0 (TID 136) -2016-06-13 19:26:30 INFO Executor:59 - Running task 1.0 in stage 507.0 (TID 137) -2016-06-13 19:26:30 INFO CacheManager:59 - Partition rdd_260_0 not found, computing it -2016-06-13 19:26:30 INFO CacheManager:59 - Partition rdd_260_1 not found, computing it -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:26:30 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=213755, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block rdd_260_1 stored as values in memory (estimated size 1168.0 B, free 730.4 MB) -2016-06-13 19:26:30 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=214923, maxMem=766075207 -2016-06-13 19:26:30 INFO MemoryStore:59 - Block rdd_260_0 stored as values in memory (estimated size 1168.0 B, free 730.4 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added rdd_260_1 in memory on localhost:49636 (size: 1168.0 B, free: 730.5 MB) -2016-06-13 19:26:30 INFO BlockManagerInfo:59 - Added rdd_260_0 in memory on localhost:49636 (size: 1168.0 B, free: 730.5 MB) -2016-06-13 19:26:30 INFO Executor:59 - Finished task 1.0 in stage 507.0 (TID 137). 2919 bytes result sent to driver -2016-06-13 19:26:30 INFO Executor:59 - Finished task 0.0 in stage 507.0 (TID 136). 2919 bytes result sent to driver -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 507.0 (TID 137) in 6 ms on localhost (1/2) -2016-06-13 19:26:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 507.0 (TID 136) in 6 ms on localhost (2/2) -2016-06-13 19:26:30 INFO DAGScheduler:59 - ResultStage 507 (collect at PowerIterationClusteringSuite.scala:71) finished in 0.006 s -2016-06-13 19:26:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 507.0, whose tasks have all completed, from pool -2016-06-13 19:26:30 INFO DAGScheduler:59 - Job 46 finished: collect at PowerIterationClusteringSuite.scala:71, took 0.025561 s -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 19:26:30 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 19:26:30 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 19:26:30 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 19:26:30 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 19:26:31 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 19:26:31 INFO BlockManager:59 - BlockManager stopped -2016-06-13 19:26:31 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 19:26:31 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 19:26:31 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 19:26:31 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 19:26:31 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 19:26:31 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 19:26:31 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-b2a4fabc-31f1-49ac-a943-75cb46e80b6f -2016-06-13 19:27:58 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 19:27:58 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 19:27:59 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 19:27:59 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 19:27:59 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 19:28:00 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 19:28:00 INFO Remoting:74 - Starting remoting -2016-06-13 19:28:00 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:49664] -2016-06-13 19:28:00 INFO Utils:59 - Successfully started service 'sparkDriver' on port 49664. -2016-06-13 19:28:00 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 19:28:00 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 19:28:00 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-eb01680a-be57-4f0d-b5fb-39b03cc1a47a -2016-06-13 19:28:00 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 19:28:00 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-81d0df0c-a0cf-43be-b5ce-aeef302e25c2\httpd-5cf8c6a7-8d00-4e4c-9eac-0bd0890a3787 -2016-06-13 19:28:00 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 19:28:00 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:28:00 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:49665 -2016-06-13 19:28:00 INFO Utils:59 - Successfully started service 'HTTP file server' on port 49665. -2016-06-13 19:28:00 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 19:28:00 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:28:00 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 19:28:00 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 19:28:00 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 19:28:01 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 19:28:01 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 19:28:01 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 49684. -2016-06-13 19:28:01 INFO NettyBlockTransferService:59 - Server created on 49684 -2016-06-13 19:28:01 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 19:28:01 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:49684 with 730.6 MB RAM, BlockManagerId(driver, localhost, 49684) -2016-06-13 19:28:01 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 19:28:02 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:325 -2016-06-13 19:28:02 INFO DAGScheduler:59 - Registering RDD 11 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:28:02 INFO DAGScheduler:59 - Registering RDD 5 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:28:02 INFO DAGScheduler:59 - Got job 0 (sum at PowerIterationClustering.scala:325) with 2 output partitions -2016-06-13 19:28:02 INFO DAGScheduler:59 - Final stage: ResultStage 2(sum at PowerIterationClustering.scala:325) -2016-06-13 19:28:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0, ShuffleMapStage 1) -2016-06-13 19:28:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0, ShuffleMapStage 1) -2016-06-13 19:28:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[11] at mapPartitions at GraphImpl.scala:235), which has no missing parents -2016-06-13 19:28:02 INFO MemoryStore:59 - ensureFreeSpace(3432) called with curMem=0, maxMem=766075207 -2016-06-13 19:28:02 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 3.4 KB, free 730.6 MB) -2016-06-13 19:28:02 INFO MemoryStore:59 - ensureFreeSpace(1979) called with curMem=3432, maxMem=766075207 -2016-06-13 19:28:02 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1979.0 B, free 730.6 MB) -2016-06-13 19:28:02 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:49684 (size: 1979.0 B, free: 730.6 MB) -2016-06-13 19:28:02 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[11] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:28:02 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 19:28:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[5] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:28:02 INFO MemoryStore:59 - ensureFreeSpace(3016) called with curMem=5411, maxMem=766075207 -2016-06-13 19:28:02 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-06-13 19:28:02 INFO MemoryStore:59 - ensureFreeSpace(1776) called with curMem=8427, maxMem=766075207 -2016-06-13 19:28:02 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1776.0 B, free 730.6 MB) -2016-06-13 19:28:02 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:49684 (size: 1776.0 B, free: 730.6 MB) -2016-06-13 19:28:02 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[5] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:28:02 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 19:28:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2488 bytes) -2016-06-13 19:28:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2563 bytes) -2016-06-13 19:28:02 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 19:28:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2488 bytes) -2016-06-13 19:28:02 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 19:28:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2563 bytes) -2016-06-13 19:28:02 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 19:28:02 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 19:28:02 INFO CacheManager:59 - Partition rdd_2_0 not found, computing it -2016-06-13 19:28:02 INFO CacheManager:59 - Partition rdd_2_1 not found, computing it -2016-06-13 19:28:02 INFO CacheManager:59 - Another thread is loading rdd_2_1, waiting for it to finish... -2016-06-13 19:28:02 INFO CacheManager:59 - Another thread is loading rdd_2_0, waiting for it to finish... -2016-06-13 19:28:02 INFO MemoryStore:59 - ensureFreeSpace(2856) called with curMem=10203, maxMem=766075207 -2016-06-13 19:28:02 INFO MemoryStore:59 - Block rdd_2_0 stored as values in memory (estimated size 2.8 KB, free 730.6 MB) -2016-06-13 19:28:02 INFO MemoryStore:59 - ensureFreeSpace(2952) called with curMem=13059, maxMem=766075207 -2016-06-13 19:28:02 INFO MemoryStore:59 - Block rdd_2_1 stored as values in memory (estimated size 2.9 KB, free 730.6 MB) -2016-06-13 19:28:02 INFO BlockManagerInfo:59 - Added rdd_2_0 in memory on localhost:49684 (size: 2.8 KB, free: 730.6 MB) -2016-06-13 19:28:02 INFO BlockManagerInfo:59 - Added rdd_2_1 in memory on localhost:49684 (size: 2.9 KB, free: 730.6 MB) -2016-06-13 19:28:02 INFO CacheManager:59 - Finished waiting for rdd_2_1 -2016-06-13 19:28:02 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-06-13 19:28:02 INFO CacheManager:59 - Finished waiting for rdd_2_0 -2016-06-13 19:28:02 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-06-13 19:28:02 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1159 bytes result sent to driver -2016-06-13 19:28:02 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1739 bytes result sent to driver -2016-06-13 19:28:03 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1159 bytes result sent to driver -2016-06-13 19:28:03 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1739 bytes result sent to driver -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 209 ms on localhost (1/2) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 209 ms on localhost (1/2) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 258 ms on localhost (2/2) -2016-06-13 19:28:03 INFO DAGScheduler:59 - ShuffleMapStage 0 (mapPartitions at GraphImpl.scala:235) finished in 0.273 s -2016-06-13 19:28:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:03 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 1) -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 19:28:03 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-06-13 19:28:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 220 ms on localhost (2/2) -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 19:28:03 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List(ShuffleMapStage 1) -2016-06-13 19:28:03 INFO DAGScheduler:59 - ShuffleMapStage 1 (mapPartitions at VertexRDD.scala:358) finished in 0.244 s -2016-06-13 19:28:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:28:03 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-06-13 19:28:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:03 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List() -2016-06-13 19:28:03 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[23] at map at PowerIterationClustering.scala:325), which is now runnable -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(5136) called with curMem=16011, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 5.0 KB, free 730.6 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(2654) called with curMem=21147, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:49684 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:28:03 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[23] at map at PowerIterationClustering.scala:325) -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 19:28:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:28:03 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 19:28:03 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 19:28:03 INFO CacheManager:59 - Partition rdd_21_0 not found, computing it -2016-06-13 19:28:03 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-06-13 19:28:03 INFO CacheManager:59 - Partition rdd_8_0 not found, computing it -2016-06-13 19:28:03 INFO CacheManager:59 - Partition rdd_21_1 not found, computing it -2016-06-13 19:28:03 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-06-13 19:28:03 INFO CacheManager:59 - Partition rdd_8_1 not found, computing it -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 11 ms -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(1776) called with curMem=23801, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block rdd_8_0 stored as values in memory (estimated size 1776.0 B, free 730.6 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(1792) called with curMem=25577, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block rdd_8_1 stored as values in memory (estimated size 1792.0 B, free 730.6 MB) -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added rdd_8_0 in memory on localhost:49684 (size: 1776.0 B, free: 730.6 MB) -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added rdd_8_1 in memory on localhost:49684 (size: 1792.0 B, free: 730.6 MB) -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(1824) called with curMem=27369, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 1824.0 B, free 730.6 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(1840) called with curMem=29193, maxMem=766075207 -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:49684 (size: 1824.0 B, free: 730.6 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 1840.0 B, free 730.6 MB) -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:49684 (size: 1840.0 B, free: 730.6 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(368) called with curMem=31033, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block rdd_21_0 stored as values in memory (estimated size 368.0 B, free 730.6 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(368) called with curMem=31401, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block rdd_21_1 stored as values in memory (estimated size 368.0 B, free 730.6 MB) -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added rdd_21_0 in memory on localhost:49684 (size: 368.0 B, free: 730.6 MB) -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added rdd_21_1 in memory on localhost:49684 (size: 368.0 B, free: 730.6 MB) -2016-06-13 19:28:03 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1895 bytes result sent to driver -2016-06-13 19:28:03 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1895 bytes result sent to driver -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 119 ms on localhost (1/2) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 129 ms on localhost (2/2) -2016-06-13 19:28:03 INFO DAGScheduler:59 - ResultStage 2 (sum at PowerIterationClustering.scala:325) finished in 0.131 s -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 19:28:03 INFO DAGScheduler:59 - Job 0 finished: sum at PowerIterationClustering.scala:325, took 0.807949 s -2016-06-13 19:28:03 INFO SparkContext:59 - Starting job: reduce at VertexRDDImpl.scala:90 -2016-06-13 19:28:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:28:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:28:03 INFO DAGScheduler:59 - Got job 1 (reduce at VertexRDDImpl.scala:90) with 2 output partitions -2016-06-13 19:28:03 INFO DAGScheduler:59 - Final stage: ResultStage 5(reduce at VertexRDDImpl.scala:90) -2016-06-13 19:28:03 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3, ShuffleMapStage 4) -2016-06-13 19:28:03 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:03 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[27] at map at VertexRDDImpl.scala:90), which has no missing parents -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(5288) called with curMem=31769, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.2 KB, free 730.6 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(2670) called with curMem=37057, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:49684 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:28:03 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[27] at map at VertexRDDImpl.scala:90) -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 19:28:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 6, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 7, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:28:03 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 7) -2016-06-13 19:28:03 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 6) -2016-06-13 19:28:03 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:28:03 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:28:03 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 7). 2161 bytes result sent to driver -2016-06-13 19:28:03 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 6). 2161 bytes result sent to driver -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 7) in 37 ms on localhost (1/2) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 6) in 40 ms on localhost (2/2) -2016-06-13 19:28:03 INFO DAGScheduler:59 - ResultStage 5 (reduce at VertexRDDImpl.scala:90) finished in 0.041 s -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 19:28:03 INFO DAGScheduler:59 - Job 1 finished: reduce at VertexRDDImpl.scala:90, took 0.077825 s -2016-06-13 19:28:03 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:28:03 INFO DAGScheduler:59 - Registering RDD 15 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:28:03 INFO DAGScheduler:59 - Registering RDD 28 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:28:03 INFO DAGScheduler:59 - Registering RDD 32 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:28:03 INFO DAGScheduler:59 - Got job 2 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:28:03 INFO DAGScheduler:59 - Final stage: ResultStage 13(sum at PowerIterationClustering.scala:369) -2016-06-13 19:28:03 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12, ShuffleMapStage 9, ShuffleMapStage 10) -2016-06-13 19:28:03 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-06-13 19:28:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 8 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[15] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=39727, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.1 KB, free 730.5 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(2367) called with curMem=43975, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:49684 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 19:28:03 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 8 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[15] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 19:28:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 11 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[28] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:28:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 8, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 9, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:28:03 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 8) -2016-06-13 19:28:03 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 9) -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(4992) called with curMem=46342, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 4.9 KB, free 730.5 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(2604) called with curMem=51334, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 19:28:03 INFO BlockManager:59 - Found block rdd_13_1 locally -2016-06-13 19:28:03 INFO BlockManager:59 - Found block rdd_13_0 locally -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:49684 (size: 2.5 KB, free: 730.6 MB) -2016-06-13 19:28:03 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 11 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[28] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 19:28:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 10, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 11, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:28:03 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 11) -2016-06-13 19:28:03 INFO CacheManager:59 - Partition rdd_25_1 not found, computing it -2016-06-13 19:28:03 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=53938, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block rdd_25_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added rdd_25_1 in memory on localhost:49684 (size: 1432.0 B, free: 730.6 MB) -2016-06-13 19:28:03 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 11). 2834 bytes result sent to driver -2016-06-13 19:28:03 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 9). 2254 bytes result sent to driver -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 11) in 29 ms on localhost (1/2) -2016-06-13 19:28:03 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 8). 2254 bytes result sent to driver -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 9) in 56 ms on localhost (1/2) -2016-06-13 19:28:03 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 10) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 8) in 64 ms on localhost (2/2) -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 19:28:03 INFO DAGScheduler:59 - ShuffleMapStage 8 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.067 s -2016-06-13 19:28:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:03 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 11) -2016-06-13 19:28:03 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 12, ResultStage 13) -2016-06-13 19:28:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:03 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 12: List(ShuffleMapStage 11) -2016-06-13 19:28:03 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List(ShuffleMapStage 12) -2016-06-13 19:28:03 INFO CacheManager:59 - Partition rdd_25_0 not found, computing it -2016-06-13 19:28:03 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=55370, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block rdd_25_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added rdd_25_0 in memory on localhost:49684 (size: 1432.0 B, free: 730.6 MB) -2016-06-13 19:28:03 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 10). 2834 bytes result sent to driver -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 10) in 70 ms on localhost (2/2) -2016-06-13 19:28:03 INFO DAGScheduler:59 - ShuffleMapStage 11 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.073 s -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 19:28:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:28:03 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 12, ResultStage 13) -2016-06-13 19:28:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:03 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 12: List() -2016-06-13 19:28:03 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List(ShuffleMapStage 12) -2016-06-13 19:28:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[32] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=56802, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 4.7 KB, free 730.5 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(2539) called with curMem=61650, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:49684 (size: 2.5 KB, free: 730.6 MB) -2016-06-13 19:28:03 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[32] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 19:28:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 12, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 13, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:28:03 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 13) -2016-06-13 19:28:03 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 12) -2016-06-13 19:28:03 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:03 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:03 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 13). 2470 bytes result sent to driver -2016-06-13 19:28:03 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 12). 2470 bytes result sent to driver -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 13) in 47 ms on localhost (1/2) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 12) in 51 ms on localhost (2/2) -2016-06-13 19:28:03 INFO DAGScheduler:59 - ShuffleMapStage 12 (mapPartitions at GraphImpl.scala:235) finished in 0.052 s -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 19:28:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:28:03 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-06-13 19:28:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:03 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-06-13 19:28:03 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[37] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(6024) called with curMem=64189, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.9 KB, free 730.5 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(2867) called with curMem=70213, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.5 MB) -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:49684 (size: 2.8 KB, free: 730.6 MB) -2016-06-13 19:28:03 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[37] at map at PowerIterationClustering.scala:369) -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 19:28:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 14, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 15, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:28:03 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 15) -2016-06-13 19:28:03 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 14) -2016-06-13 19:28:03 INFO CacheManager:59 - Partition rdd_34_1 not found, computing it -2016-06-13 19:28:03 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-06-13 19:28:03 INFO CacheManager:59 - Partition rdd_34_0 not found, computing it -2016-06-13 19:28:03 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=73080, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block rdd_34_1 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=74560, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block rdd_34_0 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added rdd_34_1 in memory on localhost:49684 (size: 1480.0 B, free: 730.6 MB) -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added rdd_34_0 in memory on localhost:49684 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:28:03 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 15). 2880 bytes result sent to driver -2016-06-13 19:28:03 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 14). 2880 bytes result sent to driver -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 15) in 33 ms on localhost (1/2) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 14) in 37 ms on localhost (2/2) -2016-06-13 19:28:03 INFO DAGScheduler:59 - ResultStage 13 (sum at PowerIterationClustering.scala:369) finished in 0.038 s -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 19:28:03 INFO DAGScheduler:59 - Job 2 finished: sum at PowerIterationClustering.scala:369, took 0.268656 s -2016-06-13 19:28:03 INFO PowerIterationClustering:59 - Iteration 0: norm(v) = 36.22334869431644. -2016-06-13 19:28:03 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:28:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:28:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:28:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:28:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:28:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:28:03 INFO DAGScheduler:59 - Got job 3 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:28:03 INFO DAGScheduler:59 - Final stage: ResultStage 21(sum at PowerIterationClustering.scala:375) -2016-06-13 19:28:03 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 20, ShuffleMapStage 17, ShuffleMapStage 18) -2016-06-13 19:28:03 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:03 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[48] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(6760) called with curMem=76040, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 6.6 KB, free 730.5 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(3106) called with curMem=82800, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.5 MB) -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:49684 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:28:03 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[48] at values at PowerIterationClustering.scala:375) -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 19:28:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 16, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 17, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:28:03 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 16) -2016-06-13 19:28:03 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 17) -2016-06-13 19:28:03 INFO CacheManager:59 - Partition rdd_40_0 not found, computing it -2016-06-13 19:28:03 INFO BlockManager:59 - Found block rdd_25_0 locally -2016-06-13 19:28:03 INFO BlockManager:59 - Found block rdd_34_0 locally -2016-06-13 19:28:03 INFO CacheManager:59 - Partition rdd_40_1 not found, computing it -2016-06-13 19:28:03 INFO BlockManager:59 - Found block rdd_25_1 locally -2016-06-13 19:28:03 INFO BlockManager:59 - Found block rdd_34_1 locally -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=85906, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block rdd_40_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=87338, maxMem=766075207 -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added rdd_40_0 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - Block rdd_40_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added rdd_40_1 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:03 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 16). 2664 bytes result sent to driver -2016-06-13 19:28:03 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 17). 2664 bytes result sent to driver -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 16) in 21 ms on localhost (1/2) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 17) in 22 ms on localhost (2/2) -2016-06-13 19:28:03 INFO DAGScheduler:59 - ResultStage 21 (sum at PowerIterationClustering.scala:375) finished in 0.023 s -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 19:28:03 INFO DAGScheduler:59 - Job 3 finished: sum at PowerIterationClustering.scala:375, took 0.053438 s -2016-06-13 19:28:03 INFO PowerIterationClustering:59 - Iteration 0: delta = 1.2794887333174252. -2016-06-13 19:28:03 INFO PowerIterationClustering:59 - Iteration 0: diff(delta) = 1.7976931348623157E308. -2016-06-13 19:28:03 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:28:03 INFO DAGScheduler:59 - Registering RDD 51 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:28:03 INFO DAGScheduler:59 - Registering RDD 55 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:28:03 INFO DAGScheduler:59 - Got job 4 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:28:03 INFO DAGScheduler:59 - Final stage: ResultStage 31(sum at PowerIterationClustering.scala:369) -2016-06-13 19:28:03 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30, ShuffleMapStage 28, ShuffleMapStage 25, ShuffleMapStage 26) -2016-06-13 19:28:03 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 30) -2016-06-13 19:28:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 29 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[51] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(5984) called with curMem=88770, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.8 KB, free 730.5 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(2861) called with curMem=94754, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.5 MB) -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:49684 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:28:03 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 29 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[51] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-06-13 19:28:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 18, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 19, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:28:03 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 19) -2016-06-13 19:28:03 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 18) -2016-06-13 19:28:03 INFO CacheManager:59 - Partition rdd_49_1 not found, computing it -2016-06-13 19:28:03 INFO BlockManager:59 - Found block rdd_34_1 locally -2016-06-13 19:28:03 INFO CacheManager:59 - Partition rdd_49_0 not found, computing it -2016-06-13 19:28:03 INFO BlockManager:59 - Found block rdd_34_0 locally -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=97615, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block rdd_49_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=99047, maxMem=766075207 -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added rdd_49_1 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - Block rdd_49_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added rdd_49_0 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:03 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 19). 2834 bytes result sent to driver -2016-06-13 19:28:03 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 18). 2834 bytes result sent to driver -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 19) in 19 ms on localhost (1/2) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 18) in 21 ms on localhost (2/2) -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 19:28:03 INFO DAGScheduler:59 - ShuffleMapStage 29 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.023 s -2016-06-13 19:28:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:28:03 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 30, ResultStage 31) -2016-06-13 19:28:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:03 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 30: List() -2016-06-13 19:28:03 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List(ShuffleMapStage 30) -2016-06-13 19:28:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 30 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[55] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=100479, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 4.7 KB, free 730.5 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=105327, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:49684 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:28:03 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 30 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[55] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-06-13 19:28:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 20, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 21, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:28:03 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 21) -2016-06-13 19:28:03 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 20) -2016-06-13 19:28:03 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-06-13 19:28:03 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:03 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 20). 2470 bytes result sent to driver -2016-06-13 19:28:03 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 21). 2470 bytes result sent to driver -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 20) in 41 ms on localhost (1/2) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 21) in 40 ms on localhost (2/2) -2016-06-13 19:28:03 INFO DAGScheduler:59 - ShuffleMapStage 30 (mapPartitions at GraphImpl.scala:235) finished in 0.042 s -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 19:28:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:28:03 INFO DAGScheduler:59 - waiting: Set(ResultStage 31) -2016-06-13 19:28:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:03 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List() -2016-06-13 19:28:03 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[60] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(6920) called with curMem=107869, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 6.8 KB, free 730.5 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(3114) called with curMem=114789, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.5 MB) -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:49684 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:28:03 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[60] at map at PowerIterationClustering.scala:369) -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 19:28:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 22, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 23, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:28:03 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 23) -2016-06-13 19:28:03 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 22) -2016-06-13 19:28:03 INFO CacheManager:59 - Partition rdd_57_0 not found, computing it -2016-06-13 19:28:03 INFO CacheManager:59 - Partition rdd_57_1 not found, computing it -2016-06-13 19:28:03 INFO BlockManager:59 - Found block rdd_49_0 locally -2016-06-13 19:28:03 INFO BlockManager:59 - Found block rdd_49_1 locally -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:28:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=117903, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block rdd_57_1 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:28:03 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=119383, maxMem=766075207 -2016-06-13 19:28:03 INFO MemoryStore:59 - Block rdd_57_0 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added rdd_57_1 in memory on localhost:49684 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:28:03 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 23). 2880 bytes result sent to driver -2016-06-13 19:28:03 INFO BlockManagerInfo:59 - Added rdd_57_0 in memory on localhost:49684 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:28:03 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 22). 2880 bytes result sent to driver -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 23) in 28 ms on localhost (1/2) -2016-06-13 19:28:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 22) in 32 ms on localhost (2/2) -2016-06-13 19:28:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 19:28:03 INFO DAGScheduler:59 - ResultStage 31 (sum at PowerIterationClustering.scala:369) finished in 0.033 s -2016-06-13 19:28:03 INFO DAGScheduler:59 - Job 4 finished: sum at PowerIterationClustering.scala:369, took 0.177195 s -2016-06-13 19:28:03 INFO PowerIterationClustering:59 - Iteration 1: norm(v) = 36.22334869431644. -2016-06-13 19:28:04 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:28:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:28:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:49684 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:28:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:28:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:28:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:28:04 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 19:28:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 151 bytes -2016-06-13 19:28:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:49684 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:28:04 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 19:28:04 INFO DAGScheduler:59 - Got job 5 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:28:04 INFO DAGScheduler:59 - Final stage: ResultStage 41(sum at PowerIterationClustering.scala:375) -2016-06-13 19:28:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 38, ShuffleMapStage 35, ShuffleMapStage 36, ShuffleMapStage 40) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:49684 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:28:04 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 19:28:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:04 INFO DAGScheduler:59 - Submitting ResultStage 41 (MapPartitionsRDD[71] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:49684 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:28:04 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(7352) called with curMem=84728, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 7.2 KB, free 730.5 MB) -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(3294) called with curMem=92080, maxMem=766075207 -2016-06-13 19:28:04 INFO BlockManager:59 - Removing RDD 40 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 19:28:04 INFO ContextCleaner:59 - Cleaned RDD 40 -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:49684 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:28:04 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 41 (MapPartitionsRDD[71] at values at PowerIterationClustering.scala:375) -2016-06-13 19:28:04 INFO TaskSchedulerImpl:59 - Adding task set 41.0 with 2 tasks -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:49684 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:28:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 41.0 (TID 24, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:28:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 41.0 (TID 25, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:28:04 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 19:28:04 INFO Executor:59 - Running task 1.0 in stage 41.0 (TID 25) -2016-06-13 19:28:04 INFO Executor:59 - Running task 0.0 in stage 41.0 (TID 24) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:49684 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:28:04 INFO CacheManager:59 - Partition rdd_63_1 not found, computing it -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_49_1 locally -2016-06-13 19:28:04 INFO CacheManager:59 - Partition rdd_63_0 not found, computing it -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_49_0 locally -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_57_0 locally -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=76232, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block rdd_63_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added rdd_63_0 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:04 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_57_1 locally -2016-06-13 19:28:04 INFO Executor:59 - Finished task 0.0 in stage 41.0 (TID 24). 2664 bytes result sent to driver -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=77664, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block rdd_63_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added rdd_63_1 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:04 INFO Executor:59 - Finished task 1.0 in stage 41.0 (TID 25). 2664 bytes result sent to driver -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:49684 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:28:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 41.0 (TID 24) in 38 ms on localhost (1/2) -2016-06-13 19:28:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 41.0 (TID 25) in 41 ms on localhost (2/2) -2016-06-13 19:28:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 41.0, whose tasks have all completed, from pool -2016-06-13 19:28:04 INFO DAGScheduler:59 - ResultStage 41 (sum at PowerIterationClustering.scala:375) finished in 0.047 s -2016-06-13 19:28:04 INFO DAGScheduler:59 - Job 5 finished: sum at PowerIterationClustering.scala:375, took 0.111236 s -2016-06-13 19:28:04 INFO PowerIterationClustering:59 - Iteration 1: delta = 0.0. -2016-06-13 19:28:04 INFO PowerIterationClustering:59 - Iteration 1: diff(delta) = 1.2794887333174252. -2016-06-13 19:28:04 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:49684 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:28:04 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:49684 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:28:04 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:49684 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:28:04 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:49684 in memory (size: 1776.0 B, free: 730.6 MB) -2016-06-13 19:28:04 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:49684 in memory (size: 1979.0 B, free: 730.6 MB) -2016-06-13 19:28:04 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 19:28:04 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:28:04 INFO DAGScheduler:59 - Registering RDD 74 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:28:04 INFO DAGScheduler:59 - Registering RDD 78 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:28:04 INFO DAGScheduler:59 - Got job 6 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:28:04 INFO DAGScheduler:59 - Final stage: ResultStage 53(sum at PowerIterationClustering.scala:369) -2016-06-13 19:28:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 48, ShuffleMapStage 45, ShuffleMapStage 52, ShuffleMapStage 46, ShuffleMapStage 50) -2016-06-13 19:28:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 52) -2016-06-13 19:28:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 51 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[74] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(6568) called with curMem=38934, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 6.4 KB, free 730.5 MB) -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(3022) called with curMem=45502, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.5 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:49684 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 19:28:04 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 51 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[74] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:28:04 INFO TaskSchedulerImpl:59 - Adding task set 51.0 with 2 tasks -2016-06-13 19:28:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 51.0 (TID 26, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:28:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 51.0 (TID 27, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:28:04 INFO Executor:59 - Running task 0.0 in stage 51.0 (TID 26) -2016-06-13 19:28:04 INFO Executor:59 - Running task 1.0 in stage 51.0 (TID 27) -2016-06-13 19:28:04 INFO CacheManager:59 - Partition rdd_72_1 not found, computing it -2016-06-13 19:28:04 INFO CacheManager:59 - Partition rdd_72_0 not found, computing it -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_57_1 locally -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_57_0 locally -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=48524, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block rdd_72_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=49956, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block rdd_72_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added rdd_72_1 in memory on localhost:49684 (size: 1432.0 B, free: 730.6 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added rdd_72_0 in memory on localhost:49684 (size: 1432.0 B, free: 730.6 MB) -2016-06-13 19:28:04 INFO Executor:59 - Finished task 1.0 in stage 51.0 (TID 27). 2834 bytes result sent to driver -2016-06-13 19:28:04 INFO Executor:59 - Finished task 0.0 in stage 51.0 (TID 26). 2834 bytes result sent to driver -2016-06-13 19:28:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 51.0 (TID 26) in 33 ms on localhost (1/2) -2016-06-13 19:28:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 51.0 (TID 27) in 32 ms on localhost (2/2) -2016-06-13 19:28:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 51.0, whose tasks have all completed, from pool -2016-06-13 19:28:04 INFO DAGScheduler:59 - ShuffleMapStage 51 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.035 s -2016-06-13 19:28:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:28:04 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 52, ResultStage 53) -2016-06-13 19:28:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:04 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 52: List() -2016-06-13 19:28:04 INFO DAGScheduler:59 - Missing parents for ResultStage 53: List(ShuffleMapStage 52) -2016-06-13 19:28:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 52 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[78] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=51388, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 4.7 KB, free 730.5 MB) -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=56236, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:49684 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:28:04 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 52 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[78] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:28:04 INFO TaskSchedulerImpl:59 - Adding task set 52.0 with 2 tasks -2016-06-13 19:28:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 52.0 (TID 28, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:28:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 52.0 (TID 29, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:28:04 INFO Executor:59 - Running task 1.0 in stage 52.0 (TID 29) -2016-06-13 19:28:04 INFO Executor:59 - Running task 0.0 in stage 52.0 (TID 28) -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_2_0 locally -2016-06-13 19:28:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_2_1 locally -2016-06-13 19:28:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:04 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:28:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:28:04 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:28:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:04 INFO Executor:59 - Finished task 0.0 in stage 52.0 (TID 28). 2470 bytes result sent to driver -2016-06-13 19:28:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 52.0 (TID 28) in 42 ms on localhost (1/2) -2016-06-13 19:28:04 INFO Executor:59 - Finished task 1.0 in stage 52.0 (TID 29). 2470 bytes result sent to driver -2016-06-13 19:28:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 52.0 (TID 29) in 46 ms on localhost (2/2) -2016-06-13 19:28:04 INFO DAGScheduler:59 - ShuffleMapStage 52 (mapPartitions at GraphImpl.scala:235) finished in 0.049 s -2016-06-13 19:28:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 52.0, whose tasks have all completed, from pool -2016-06-13 19:28:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:28:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 53) -2016-06-13 19:28:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:04 INFO DAGScheduler:59 - Missing parents for ResultStage 53: List() -2016-06-13 19:28:04 INFO DAGScheduler:59 - Submitting ResultStage 53 (MapPartitionsRDD[83] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(7504) called with curMem=58778, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 7.3 KB, free 730.5 MB) -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(3266) called with curMem=66282, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:49684 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:28:04 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 53 (MapPartitionsRDD[83] at map at PowerIterationClustering.scala:369) -2016-06-13 19:28:04 INFO TaskSchedulerImpl:59 - Adding task set 53.0 with 2 tasks -2016-06-13 19:28:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 53.0 (TID 30, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:28:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 53.0 (TID 31, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:28:04 INFO Executor:59 - Running task 0.0 in stage 53.0 (TID 30) -2016-06-13 19:28:04 INFO Executor:59 - Running task 1.0 in stage 53.0 (TID 31) -2016-06-13 19:28:04 INFO CacheManager:59 - Partition rdd_80_0 not found, computing it -2016-06-13 19:28:04 INFO CacheManager:59 - Partition rdd_80_1 not found, computing it -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_72_0 locally -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_72_1 locally -2016-06-13 19:28:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=69548, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block rdd_80_1 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=71028, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block rdd_80_0 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added rdd_80_1 in memory on localhost:49684 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added rdd_80_0 in memory on localhost:49684 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:28:04 INFO Executor:59 - Finished task 1.0 in stage 53.0 (TID 31). 2880 bytes result sent to driver -2016-06-13 19:28:04 INFO Executor:59 - Finished task 0.0 in stage 53.0 (TID 30). 2880 bytes result sent to driver -2016-06-13 19:28:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 53.0 (TID 31) in 22 ms on localhost (1/2) -2016-06-13 19:28:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 53.0 (TID 30) in 26 ms on localhost (2/2) -2016-06-13 19:28:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 53.0, whose tasks have all completed, from pool -2016-06-13 19:28:04 INFO DAGScheduler:59 - ResultStage 53 (sum at PowerIterationClustering.scala:369) finished in 0.026 s -2016-06-13 19:28:04 INFO DAGScheduler:59 - Job 6 finished: sum at PowerIterationClustering.scala:369, took 0.199404 s -2016-06-13 19:28:04 INFO PowerIterationClustering:59 - Iteration 2: norm(v) = 36.22334869431644. -2016-06-13 19:28:04 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:28:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:28:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:28:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:28:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:28:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:28:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 151 bytes -2016-06-13 19:28:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:28:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 151 bytes -2016-06-13 19:28:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:28:04 INFO DAGScheduler:59 - Got job 7 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:28:04 INFO DAGScheduler:59 - Final stage: ResultStage 65(sum at PowerIterationClustering.scala:375) -2016-06-13 19:28:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 60, ShuffleMapStage 64, ShuffleMapStage 57, ShuffleMapStage 58, ShuffleMapStage 62) -2016-06-13 19:28:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:04 INFO DAGScheduler:59 - Submitting ResultStage 65 (MapPartitionsRDD[94] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(7936) called with curMem=72508, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 7.8 KB, free 730.5 MB) -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(3392) called with curMem=80444, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.5 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:49684 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:28:04 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 65 (MapPartitionsRDD[94] at values at PowerIterationClustering.scala:375) -2016-06-13 19:28:04 INFO TaskSchedulerImpl:59 - Adding task set 65.0 with 2 tasks -2016-06-13 19:28:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 65.0 (TID 32, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:28:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 65.0 (TID 33, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:28:04 INFO Executor:59 - Running task 0.0 in stage 65.0 (TID 32) -2016-06-13 19:28:04 INFO Executor:59 - Running task 1.0 in stage 65.0 (TID 33) -2016-06-13 19:28:04 INFO CacheManager:59 - Partition rdd_86_0 not found, computing it -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_72_0 locally -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_80_0 locally -2016-06-13 19:28:04 INFO CacheManager:59 - Partition rdd_86_1 not found, computing it -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_72_1 locally -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_80_1 locally -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=83836, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block rdd_86_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=85268, maxMem=766075207 -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added rdd_86_0 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:04 INFO MemoryStore:59 - Block rdd_86_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:28:04 INFO Executor:59 - Finished task 0.0 in stage 65.0 (TID 32). 2664 bytes result sent to driver -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added rdd_86_1 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:04 INFO Executor:59 - Finished task 1.0 in stage 65.0 (TID 33). 2664 bytes result sent to driver -2016-06-13 19:28:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 65.0 (TID 33) in 21 ms on localhost (1/2) -2016-06-13 19:28:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 65.0 (TID 32) in 25 ms on localhost (2/2) -2016-06-13 19:28:04 INFO DAGScheduler:59 - ResultStage 65 (sum at PowerIterationClustering.scala:375) finished in 0.026 s -2016-06-13 19:28:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 65.0, whose tasks have all completed, from pool -2016-06-13 19:28:04 INFO DAGScheduler:59 - Job 7 finished: sum at PowerIterationClustering.scala:375, took 0.066104 s -2016-06-13 19:28:04 INFO PowerIterationClustering:59 - Iteration 2: delta = 0.0. -2016-06-13 19:28:04 INFO PowerIterationClustering:59 - Iteration 2: diff(delta) = 0.0. -2016-06-13 19:28:04 WARN KMeans:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:28:04 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:28:04 INFO DAGScheduler:59 - Got job 8 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:28:04 INFO DAGScheduler:59 - Final stage: ResultStage 77(takeSample at KMeans.scala:376) -2016-06-13 19:28:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 70, ShuffleMapStage 74, ShuffleMapStage 72, ShuffleMapStage 69, ShuffleMapStage 76) -2016-06-13 19:28:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:04 INFO DAGScheduler:59 - Submitting ResultStage 77 (MapPartitionsRDD[102] at map at KMeans.scala:210), which has no missing parents -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(8168) called with curMem=86700, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 8.0 KB, free 730.5 MB) -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(3463) called with curMem=94868, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.5 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:49684 (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:28:04 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 77 (MapPartitionsRDD[102] at map at KMeans.scala:210) -2016-06-13 19:28:04 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 2 tasks -2016-06-13 19:28:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 34, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:28:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 35, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:28:04 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 34) -2016-06-13 19:28:04 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 35) -2016-06-13 19:28:04 INFO CacheManager:59 - Partition rdd_97_0 not found, computing it -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_80_0 locally -2016-06-13 19:28:04 INFO CacheManager:59 - Partition rdd_97_1 not found, computing it -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_80_1 locally -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=98331, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block rdd_97_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=99763, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block rdd_97_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added rdd_97_1 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added rdd_97_0 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:04 INFO CacheManager:59 - Partition rdd_100_1 not found, computing it -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:28:04 INFO CacheManager:59 - Partition rdd_100_0 not found, computing it -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=101195, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block rdd_100_0 stored as values in memory (estimated size 240.0 B, free 730.5 MB) -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=101435, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block rdd_100_1 stored as values in memory (estimated size 240.0 B, free 730.5 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added rdd_100_0 in memory on localhost:49684 (size: 240.0 B, free: 730.5 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added rdd_100_1 in memory on localhost:49684 (size: 240.0 B, free: 730.5 MB) -2016-06-13 19:28:04 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 34). 2717 bytes result sent to driver -2016-06-13 19:28:04 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 35). 2717 bytes result sent to driver -2016-06-13 19:28:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 34) in 37 ms on localhost (1/2) -2016-06-13 19:28:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 35) in 38 ms on localhost (2/2) -2016-06-13 19:28:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-06-13 19:28:04 INFO DAGScheduler:59 - ResultStage 77 (takeSample at KMeans.scala:376) finished in 0.041 s -2016-06-13 19:28:04 INFO DAGScheduler:59 - Job 8 finished: takeSample at KMeans.scala:376, took 0.076396 s -2016-06-13 19:28:04 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:28:04 INFO DAGScheduler:59 - Got job 9 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:28:04 INFO DAGScheduler:59 - Final stage: ResultStage 89(takeSample at KMeans.scala:376) -2016-06-13 19:28:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 84, ShuffleMapStage 81, ShuffleMapStage 88, ShuffleMapStage 82, ShuffleMapStage 86) -2016-06-13 19:28:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:04 INFO DAGScheduler:59 - Submitting ResultStage 89 (PartitionwiseSampledRDD[104] at takeSample at KMeans.scala:376), which has no missing parents -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(25560) called with curMem=101675, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 25.0 KB, free 730.5 MB) -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(10149) called with curMem=127235, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 9.9 KB, free 730.5 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:49684 (size: 9.9 KB, free: 730.5 MB) -2016-06-13 19:28:04 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 89 (PartitionwiseSampledRDD[104] at takeSample at KMeans.scala:376) -2016-06-13 19:28:04 INFO TaskSchedulerImpl:59 - Adding task set 89.0 with 2 tasks -2016-06-13 19:28:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 89.0 (TID 36, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:28:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 89.0 (TID 37, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:28:04 INFO Executor:59 - Running task 1.0 in stage 89.0 (TID 37) -2016-06-13 19:28:04 INFO Executor:59 - Running task 0.0 in stage 89.0 (TID 36) -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:28:04 INFO Executor:59 - Finished task 1.0 in stage 89.0 (TID 37). 2575 bytes result sent to driver -2016-06-13 19:28:04 INFO Executor:59 - Finished task 0.0 in stage 89.0 (TID 36). 2633 bytes result sent to driver -2016-06-13 19:28:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 89.0 (TID 37) in 13 ms on localhost (1/2) -2016-06-13 19:28:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 89.0 (TID 36) in 16 ms on localhost (2/2) -2016-06-13 19:28:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 89.0, whose tasks have all completed, from pool -2016-06-13 19:28:04 INFO DAGScheduler:59 - ResultStage 89 (takeSample at KMeans.scala:376) finished in 0.017 s -2016-06-13 19:28:04 INFO DAGScheduler:59 - Job 9 finished: takeSample at KMeans.scala:376, took 0.058300 s -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=137384, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 880.0 B, free 730.5 MB) -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(487) called with curMem=138264, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 487.0 B, free 730.5 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:49684 (size: 487.0 B, free: 730.5 MB) -2016-06-13 19:28:04 INFO SparkContext:59 - Created broadcast 19 from broadcast at KMeans.scala:394 -2016-06-13 19:28:04 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:28:04 INFO DAGScheduler:59 - Got job 10 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:28:04 INFO DAGScheduler:59 - Final stage: ResultStage 101(aggregate at KMeans.scala:402) -2016-06-13 19:28:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 96, ShuffleMapStage 100, ShuffleMapStage 93, ShuffleMapStage 94, ShuffleMapStage 98) -2016-06-13 19:28:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:04 INFO DAGScheduler:59 - Submitting ResultStage 101 (MapPartitionsRDD[106] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(10192) called with curMem=138751, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 10.0 KB, free 730.4 MB) -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(4263) called with curMem=148943, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:49684 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:28:04 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 101 (MapPartitionsRDD[106] at map at KMeans.scala:396) -2016-06-13 19:28:04 INFO TaskSchedulerImpl:59 - Adding task set 101.0 with 2 tasks -2016-06-13 19:28:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 101.0 (TID 38, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:28:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 101.0 (TID 39, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:28:04 INFO Executor:59 - Running task 1.0 in stage 101.0 (TID 39) -2016-06-13 19:28:04 INFO Executor:59 - Running task 0.0 in stage 101.0 (TID 38) -2016-06-13 19:28:04 INFO CacheManager:59 - Partition rdd_106_1 not found, computing it -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:28:04 INFO CacheManager:59 - Partition rdd_106_0 not found, computing it -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:28:04 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 19:28:04 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=153206, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block rdd_106_1 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=153702, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block rdd_106_0 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added rdd_106_1 in memory on localhost:49684 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added rdd_106_0 in memory on localhost:49684 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:28:04 INFO Executor:59 - Finished task 1.0 in stage 101.0 (TID 39). 2647 bytes result sent to driver -2016-06-13 19:28:04 INFO Executor:59 - Finished task 0.0 in stage 101.0 (TID 38). 2647 bytes result sent to driver -2016-06-13 19:28:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 101.0 (TID 39) in 47 ms on localhost (1/2) -2016-06-13 19:28:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 101.0 (TID 38) in 50 ms on localhost (2/2) -2016-06-13 19:28:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 101.0, whose tasks have all completed, from pool -2016-06-13 19:28:04 INFO DAGScheduler:59 - ResultStage 101 (aggregate at KMeans.scala:402) finished in 0.051 s -2016-06-13 19:28:04 INFO DAGScheduler:59 - Job 10 finished: aggregate at KMeans.scala:402, took 0.088747 s -2016-06-13 19:28:04 INFO MapPartitionsRDD:59 - Removing RDD 103 from persistence list -2016-06-13 19:28:04 INFO BlockManager:59 - Removing RDD 103 -2016-06-13 19:28:04 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:28:04 INFO DAGScheduler:59 - Got job 11 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:28:04 INFO DAGScheduler:59 - Final stage: ResultStage 113(collect at KMeans.scala:431) -2016-06-13 19:28:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 108, ShuffleMapStage 105, ShuffleMapStage 112, ShuffleMapStage 106, ShuffleMapStage 110) -2016-06-13 19:28:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:04 INFO DAGScheduler:59 - Submitting ResultStage 113 (MapPartitionsRDD[108] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(10240) called with curMem=154198, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 10.0 KB, free 730.4 MB) -2016-06-13 19:28:04 INFO MemoryStore:59 - ensureFreeSpace(4343) called with curMem=164438, maxMem=766075207 -2016-06-13 19:28:04 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 19:28:04 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:49684 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:28:04 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 113 (MapPartitionsRDD[108] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:28:04 INFO TaskSchedulerImpl:59 - Adding task set 113.0 with 2 tasks -2016-06-13 19:28:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 113.0 (TID 40, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:28:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 113.0 (TID 41, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:28:04 INFO Executor:59 - Running task 1.0 in stage 113.0 (TID 41) -2016-06-13 19:28:04 INFO Executor:59 - Running task 0.0 in stage 113.0 (TID 40) -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_106_1 locally -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:28:04 INFO BlockManager:59 - Found block rdd_106_0 locally -2016-06-13 19:28:04 INFO Executor:59 - Finished task 1.0 in stage 113.0 (TID 41). 3471 bytes result sent to driver -2016-06-13 19:28:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 113.0 (TID 41) in 21 ms on localhost (1/2) -2016-06-13 19:28:04 INFO Executor:59 - Finished task 0.0 in stage 113.0 (TID 40). 3118 bytes result sent to driver -2016-06-13 19:28:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 113.0 (TID 40) in 26 ms on localhost (2/2) -2016-06-13 19:28:04 INFO DAGScheduler:59 - ResultStage 113 (collect at KMeans.scala:431) finished in 0.026 s -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 113.0, whose tasks have all completed, from pool -2016-06-13 19:28:05 INFO DAGScheduler:59 - Job 11 finished: collect at KMeans.scala:431, took 0.058496 s -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(1648) called with curMem=168781, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 1648.0 B, free 730.4 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(600) called with curMem=170429, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 600.0 B, free 730.4 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:49684 (size: 600.0 B, free: 730.5 MB) -2016-06-13 19:28:05 INFO SparkContext:59 - Created broadcast 22 from broadcast at KMeans.scala:394 -2016-06-13 19:28:05 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Got job 12 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:28:05 INFO DAGScheduler:59 - Final stage: ResultStage 125(aggregate at KMeans.scala:402) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 117, ShuffleMapStage 118, ShuffleMapStage 122, ShuffleMapStage 120, ShuffleMapStage 124) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting ResultStage 125 (MapPartitionsRDD[110] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(10424) called with curMem=171029, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 10.2 KB, free 730.4 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(4357) called with curMem=181453, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:49684 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:28:05 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 125 (MapPartitionsRDD[110] at map at KMeans.scala:396) -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Adding task set 125.0 with 2 tasks -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 125.0 (TID 42, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 125.0 (TID 43, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:28:05 INFO Executor:59 - Running task 1.0 in stage 125.0 (TID 43) -2016-06-13 19:28:05 INFO CacheManager:59 - Partition rdd_110_1 not found, computing it -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_106_1 locally -2016-06-13 19:28:05 INFO Executor:59 - Running task 0.0 in stage 125.0 (TID 42) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:49684 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:28:05 INFO CacheManager:59 - Partition rdd_110_0 not found, computing it -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_106_0 locally -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=171227, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block rdd_110_1 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added rdd_110_1 in memory on localhost:49684 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=171723, maxMem=766075207 -2016-06-13 19:28:05 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block rdd_110_0 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:28:05 INFO Executor:59 - Finished task 1.0 in stage 125.0 (TID 43). 2647 bytes result sent to driver -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added rdd_110_0 in memory on localhost:49684 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 125.0 (TID 43) in 42 ms on localhost (1/2) -2016-06-13 19:28:05 INFO Executor:59 - Finished task 0.0 in stage 125.0 (TID 42). 2647 bytes result sent to driver -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:49684 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 125.0 (TID 42) in 50 ms on localhost (2/2) -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 125.0, whose tasks have all completed, from pool -2016-06-13 19:28:05 INFO DAGScheduler:59 - ResultStage 125 (aggregate at KMeans.scala:402) finished in 0.052 s -2016-06-13 19:28:05 INFO DAGScheduler:59 - Job 12 finished: aggregate at KMeans.scala:402, took 0.111468 s -2016-06-13 19:28:05 INFO MapPartitionsRDD:59 - Removing RDD 106 from persistence list -2016-06-13 19:28:05 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Got job 13 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:28:05 INFO DAGScheduler:59 - Final stage: ResultStage 137(collect at KMeans.scala:431) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 132, ShuffleMapStage 129, ShuffleMapStage 136, ShuffleMapStage 130, ShuffleMapStage 134) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting ResultStage 137 (MapPartitionsRDD[112] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:28:05 INFO BlockManager:59 - Removing RDD 106 -2016-06-13 19:28:05 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(10480) called with curMem=156772, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 10.2 KB, free 730.4 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(4430) called with curMem=167252, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:49684 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:28:05 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 137 (MapPartitionsRDD[112] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Adding task set 137.0 with 2 tasks -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:49684 in memory (size: 9.9 KB, free: 730.5 MB) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 137.0 (TID 44, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 137.0 (TID 45, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:28:05 INFO Executor:59 - Running task 0.0 in stage 137.0 (TID 44) -2016-06-13 19:28:05 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 19:28:05 INFO Executor:59 - Running task 1.0 in stage 137.0 (TID 45) -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_110_0 locally -2016-06-13 19:28:05 INFO Executor:59 - Finished task 0.0 in stage 137.0 (TID 44). 2990 bytes result sent to driver -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 137.0 (TID 44) in 15 ms on localhost (1/2) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:49684 in memory (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_110_1 locally -2016-06-13 19:28:05 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 19:28:05 INFO Executor:59 - Finished task 1.0 in stage 137.0 (TID 45). 3240 bytes result sent to driver -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 137.0 (TID 45) in 19 ms on localhost (2/2) -2016-06-13 19:28:05 INFO DAGScheduler:59 - ResultStage 137 (collect at KMeans.scala:431) finished in 0.021 s -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 137.0, whose tasks have all completed, from pool -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:49684 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Job 13 finished: collect at KMeans.scala:431, took 0.067514 s -2016-06-13 19:28:05 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(1456) called with curMem=113014, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 1456.0 B, free 730.5 MB) -2016-06-13 19:28:05 INFO BlockManager:59 - Removing RDD 86 -2016-06-13 19:28:05 INFO ContextCleaner:59 - Cleaned RDD 86 -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(582) called with curMem=111606, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 582.0 B, free 730.5 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:49684 (size: 582.0 B, free: 730.5 MB) -2016-06-13 19:28:05 INFO SparkContext:59 - Created broadcast 25 from broadcast at KMeans.scala:394 -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:49684 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:28:05 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:49684 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:28:05 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:49684 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:28:05 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 19:28:05 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:49684 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:28:05 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 19:28:05 INFO BlockManager:59 - Removing RDD 63 -2016-06-13 19:28:05 INFO ContextCleaner:59 - Cleaned RDD 63 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Got job 14 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:28:05 INFO DAGScheduler:59 - Final stage: ResultStage 149(aggregate at KMeans.scala:402) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 144, ShuffleMapStage 141, ShuffleMapStage 148, ShuffleMapStage 142, ShuffleMapStage 146) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting ResultStage 149 (MapPartitionsRDD[114] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(10656) called with curMem=70928, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 10.4 KB, free 730.5 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(4409) called with curMem=81584, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.5 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:49684 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:28:05 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 149 (MapPartitionsRDD[114] at map at KMeans.scala:396) -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Adding task set 149.0 with 2 tasks -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 149.0 (TID 46, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 149.0 (TID 47, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:28:05 INFO Executor:59 - Running task 0.0 in stage 149.0 (TID 46) -2016-06-13 19:28:05 INFO Executor:59 - Running task 1.0 in stage 149.0 (TID 47) -2016-06-13 19:28:05 INFO CacheManager:59 - Partition rdd_114_1 not found, computing it -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:28:05 INFO CacheManager:59 - Partition rdd_114_0 not found, computing it -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_110_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_110_0 locally -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=85993, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block rdd_114_1 stored as values in memory (estimated size 496.0 B, free 730.5 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=86489, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block rdd_114_0 stored as values in memory (estimated size 496.0 B, free 730.5 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added rdd_114_1 in memory on localhost:49684 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:28:05 INFO Executor:59 - Finished task 1.0 in stage 149.0 (TID 47). 2647 bytes result sent to driver -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added rdd_114_0 in memory on localhost:49684 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:28:05 INFO Executor:59 - Finished task 0.0 in stage 149.0 (TID 46). 2647 bytes result sent to driver -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 149.0 (TID 47) in 14 ms on localhost (1/2) -2016-06-13 19:28:05 INFO DAGScheduler:59 - ResultStage 149 (aggregate at KMeans.scala:402) finished in 0.016 s -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 149.0 (TID 46) in 15 ms on localhost (2/2) -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 149.0, whose tasks have all completed, from pool -2016-06-13 19:28:05 INFO DAGScheduler:59 - Job 14 finished: aggregate at KMeans.scala:402, took 0.047690 s -2016-06-13 19:28:05 INFO MapPartitionsRDD:59 - Removing RDD 110 from persistence list -2016-06-13 19:28:05 INFO BlockManager:59 - Removing RDD 110 -2016-06-13 19:28:05 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Got job 15 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:28:05 INFO DAGScheduler:59 - Final stage: ResultStage 161(collect at KMeans.scala:431) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 153, ShuffleMapStage 154, ShuffleMapStage 158, ShuffleMapStage 156, ShuffleMapStage 160) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting ResultStage 161 (MapPartitionsRDD[116] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(10712) called with curMem=85993, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 10.5 KB, free 730.5 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(4468) called with curMem=96705, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.5 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:49684 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:28:05 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 161 (MapPartitionsRDD[116] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Adding task set 161.0 with 2 tasks -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 161.0 (TID 48, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 161.0 (TID 49, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:28:05 INFO Executor:59 - Running task 1.0 in stage 161.0 (TID 49) -2016-06-13 19:28:05 INFO Executor:59 - Running task 0.0 in stage 161.0 (TID 48) -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_114_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_114_0 locally -2016-06-13 19:28:05 INFO Executor:59 - Finished task 1.0 in stage 161.0 (TID 49). 2040 bytes result sent to driver -2016-06-13 19:28:05 INFO Executor:59 - Finished task 0.0 in stage 161.0 (TID 48). 2722 bytes result sent to driver -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 161.0 (TID 49) in 8 ms on localhost (1/2) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 161.0 (TID 48) in 10 ms on localhost (2/2) -2016-06-13 19:28:05 INFO DAGScheduler:59 - ResultStage 161 (collect at KMeans.scala:431) finished in 0.011 s -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 161.0, whose tasks have all completed, from pool -2016-06-13 19:28:05 INFO DAGScheduler:59 - Job 15 finished: collect at KMeans.scala:431, took 0.031949 s -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(648) called with curMem=101173, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 648.0 B, free 730.5 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(429) called with curMem=101821, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 429.0 B, free 730.5 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:49684 (size: 429.0 B, free: 730.5 MB) -2016-06-13 19:28:05 INFO SparkContext:59 - Created broadcast 28 from broadcast at KMeans.scala:394 -2016-06-13 19:28:05 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Got job 16 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:28:05 INFO DAGScheduler:59 - Final stage: ResultStage 173(aggregate at KMeans.scala:402) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 168, ShuffleMapStage 165, ShuffleMapStage 172, ShuffleMapStage 166, ShuffleMapStage 170) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting ResultStage 173 (MapPartitionsRDD[118] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(10888) called with curMem=102250, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 10.6 KB, free 730.5 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(4468) called with curMem=113138, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.5 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:49684 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:28:05 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 173 (MapPartitionsRDD[118] at map at KMeans.scala:396) -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Adding task set 173.0 with 2 tasks -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 173.0 (TID 50, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 173.0 (TID 51, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:28:05 INFO Executor:59 - Running task 1.0 in stage 173.0 (TID 51) -2016-06-13 19:28:05 INFO Executor:59 - Running task 0.0 in stage 173.0 (TID 50) -2016-06-13 19:28:05 INFO CacheManager:59 - Partition rdd_118_1 not found, computing it -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_114_1 locally -2016-06-13 19:28:05 INFO CacheManager:59 - Partition rdd_118_0 not found, computing it -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_114_0 locally -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=117606, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block rdd_118_1 stored as values in memory (estimated size 496.0 B, free 730.5 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=118102, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block rdd_118_0 stored as values in memory (estimated size 496.0 B, free 730.5 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added rdd_118_1 in memory on localhost:49684 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:28:05 INFO Executor:59 - Finished task 1.0 in stage 173.0 (TID 51). 2647 bytes result sent to driver -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added rdd_118_0 in memory on localhost:49684 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:28:05 INFO Executor:59 - Finished task 0.0 in stage 173.0 (TID 50). 2647 bytes result sent to driver -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 173.0 (TID 51) in 19 ms on localhost (1/2) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 173.0 (TID 50) in 21 ms on localhost (2/2) -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 173.0, whose tasks have all completed, from pool -2016-06-13 19:28:05 INFO DAGScheduler:59 - ResultStage 173 (aggregate at KMeans.scala:402) finished in 0.022 s -2016-06-13 19:28:05 INFO DAGScheduler:59 - Job 16 finished: aggregate at KMeans.scala:402, took 0.051040 s -2016-06-13 19:28:05 INFO MapPartitionsRDD:59 - Removing RDD 114 from persistence list -2016-06-13 19:28:05 INFO BlockManager:59 - Removing RDD 114 -2016-06-13 19:28:05 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Got job 17 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:28:05 INFO DAGScheduler:59 - Final stage: ResultStage 185(collect at KMeans.scala:431) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 180, ShuffleMapStage 184, ShuffleMapStage 177, ShuffleMapStage 178, ShuffleMapStage 182) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting ResultStage 185 (MapPartitionsRDD[120] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(10944) called with curMem=117606, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 10.7 KB, free 730.5 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(4514) called with curMem=128550, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.5 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:49684 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:28:05 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 185 (MapPartitionsRDD[120] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Adding task set 185.0 with 2 tasks -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 185.0 (TID 52, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 185.0 (TID 53, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:28:05 INFO Executor:59 - Running task 1.0 in stage 185.0 (TID 53) -2016-06-13 19:28:05 INFO Executor:59 - Running task 0.0 in stage 185.0 (TID 52) -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_118_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_118_0 locally -2016-06-13 19:28:05 INFO Executor:59 - Finished task 1.0 in stage 185.0 (TID 53). 2040 bytes result sent to driver -2016-06-13 19:28:05 INFO Executor:59 - Finished task 0.0 in stage 185.0 (TID 52). 2040 bytes result sent to driver -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 185.0 (TID 53) in 13 ms on localhost (1/2) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 185.0 (TID 52) in 14 ms on localhost (2/2) -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 185.0, whose tasks have all completed, from pool -2016-06-13 19:28:05 INFO DAGScheduler:59 - ResultStage 185 (collect at KMeans.scala:431) finished in 0.015 s -2016-06-13 19:28:05 INFO DAGScheduler:59 - Job 17 finished: collect at KMeans.scala:431, took 0.043497 s -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(584) called with curMem=133064, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 584.0 B, free 730.5 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(236) called with curMem=133648, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 236.0 B, free 730.5 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:49684 (size: 236.0 B, free: 730.5 MB) -2016-06-13 19:28:05 INFO SparkContext:59 - Created broadcast 31 from broadcast at KMeans.scala:394 -2016-06-13 19:28:05 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Got job 18 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:28:05 INFO DAGScheduler:59 - Final stage: ResultStage 197(aggregate at KMeans.scala:402) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 190, ShuffleMapStage 194, ShuffleMapStage 192, ShuffleMapStage 189, ShuffleMapStage 196) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting ResultStage 197 (MapPartitionsRDD[122] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(11128) called with curMem=133884, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 10.9 KB, free 730.4 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(4524) called with curMem=145012, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:49684 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:28:05 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 197 (MapPartitionsRDD[122] at map at KMeans.scala:396) -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Adding task set 197.0 with 2 tasks -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 197.0 (TID 54, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 197.0 (TID 55, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:28:05 INFO Executor:59 - Running task 0.0 in stage 197.0 (TID 54) -2016-06-13 19:28:05 INFO Executor:59 - Running task 1.0 in stage 197.0 (TID 55) -2016-06-13 19:28:05 INFO CacheManager:59 - Partition rdd_122_0 not found, computing it -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:28:05 INFO CacheManager:59 - Partition rdd_122_1 not found, computing it -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_118_0 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_118_1 locally -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=149536, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block rdd_122_0 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=150032, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block rdd_122_1 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added rdd_122_0 in memory on localhost:49684 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:28:05 INFO Executor:59 - Finished task 0.0 in stage 197.0 (TID 54). 2647 bytes result sent to driver -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added rdd_122_1 in memory on localhost:49684 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:28:05 INFO Executor:59 - Finished task 1.0 in stage 197.0 (TID 55). 2647 bytes result sent to driver -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 197.0 (TID 54) in 12 ms on localhost (1/2) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 197.0 (TID 55) in 13 ms on localhost (2/2) -2016-06-13 19:28:05 INFO DAGScheduler:59 - ResultStage 197 (aggregate at KMeans.scala:402) finished in 0.014 s -2016-06-13 19:28:05 INFO DAGScheduler:59 - Job 18 finished: aggregate at KMeans.scala:402, took 0.038616 s -2016-06-13 19:28:05 INFO MapPartitionsRDD:59 - Removing RDD 118 from persistence list -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 197.0, whose tasks have all completed, from pool -2016-06-13 19:28:05 INFO BlockManager:59 - Removing RDD 118 -2016-06-13 19:28:05 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Got job 19 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:28:05 INFO DAGScheduler:59 - Final stage: ResultStage 209(collect at KMeans.scala:431) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 204, ShuffleMapStage 201, ShuffleMapStage 208, ShuffleMapStage 202, ShuffleMapStage 206) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting ResultStage 209 (MapPartitionsRDD[124] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(11176) called with curMem=149536, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 10.9 KB, free 730.4 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(4565) called with curMem=160712, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.4 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:49684 (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:28:05 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 209 (MapPartitionsRDD[124] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Adding task set 209.0 with 2 tasks -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 209.0 (TID 56, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 209.0 (TID 57, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:28:05 INFO Executor:59 - Running task 1.0 in stage 209.0 (TID 57) -2016-06-13 19:28:05 INFO Executor:59 - Running task 0.0 in stage 209.0 (TID 56) -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_122_0 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_122_1 locally -2016-06-13 19:28:05 INFO Executor:59 - Finished task 1.0 in stage 209.0 (TID 57). 2040 bytes result sent to driver -2016-06-13 19:28:05 INFO Executor:59 - Finished task 0.0 in stage 209.0 (TID 56). 2040 bytes result sent to driver -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 209.0 (TID 57) in 8 ms on localhost (1/2) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 209.0 (TID 56) in 9 ms on localhost (2/2) -2016-06-13 19:28:05 INFO DAGScheduler:59 - ResultStage 209 (collect at KMeans.scala:431) finished in 0.009 s -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 209.0, whose tasks have all completed, from pool -2016-06-13 19:28:05 INFO DAGScheduler:59 - Job 19 finished: collect at KMeans.scala:431, took 0.035031 s -2016-06-13 19:28:05 INFO MapPartitionsRDD:59 - Removing RDD 122 from persistence list -2016-06-13 19:28:05 INFO BlockManager:59 - Removing RDD 122 -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(2880) called with curMem=164285, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 2.8 KB, free 730.4 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(708) called with curMem=167165, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 708.0 B, free 730.4 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:49684 (size: 708.0 B, free: 730.5 MB) -2016-06-13 19:28:05 INFO SparkContext:59 - Created broadcast 34 from broadcast at KMeans.scala:445 -2016-06-13 19:28:05 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:450 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Registering RDD 125 (flatMap at KMeans.scala:446) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Got job 20 (collectAsMap at KMeans.scala:450) with 2 output partitions -2016-06-13 19:28:05 INFO DAGScheduler:59 - Final stage: ResultStage 222(collectAsMap at KMeans.scala:450) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 221) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 221) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 221 (MapPartitionsRDD[125] at flatMap at KMeans.scala:446), which has no missing parents -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(9672) called with curMem=167873, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 9.4 KB, free 730.4 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(4106) called with curMem=177545, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:49684 (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:28:05 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 221 (MapPartitionsRDD[125] at flatMap at KMeans.scala:446) -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Adding task set 221.0 with 2 tasks -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 221.0 (TID 58, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 221.0 (TID 59, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:28:05 INFO Executor:59 - Running task 0.0 in stage 221.0 (TID 58) -2016-06-13 19:28:05 INFO Executor:59 - Running task 1.0 in stage 221.0 (TID 59) -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:28:05 INFO Executor:59 - Finished task 0.0 in stage 221.0 (TID 58). 2254 bytes result sent to driver -2016-06-13 19:28:05 INFO Executor:59 - Finished task 1.0 in stage 221.0 (TID 59). 2254 bytes result sent to driver -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 221.0 (TID 58) in 42 ms on localhost (1/2) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 221.0 (TID 59) in 42 ms on localhost (2/2) -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 221.0, whose tasks have all completed, from pool -2016-06-13 19:28:05 INFO DAGScheduler:59 - ShuffleMapStage 221 (flatMap at KMeans.scala:446) finished in 0.044 s -2016-06-13 19:28:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:28:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 222) -2016-06-13 19:28:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:05 INFO DAGScheduler:59 - Missing parents for ResultStage 222: List() -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting ResultStage 222 (ShuffledRDD[126] at reduceByKey at KMeans.scala:450), which is now runnable -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=181651, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(1382) called with curMem=183963, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 1382.0 B, free 730.4 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:49684 (size: 1382.0 B, free: 730.5 MB) -2016-06-13 19:28:05 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 222 (ShuffledRDD[126] at reduceByKey at KMeans.scala:450) -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Adding task set 222.0 with 2 tasks -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 222.0 (TID 60, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 222.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:28:05 INFO Executor:59 - Running task 0.0 in stage 222.0 (TID 60) -2016-06-13 19:28:05 INFO Executor:59 - Running task 1.0 in stage 222.0 (TID 61) -2016-06-13 19:28:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:28:05 INFO Executor:59 - Finished task 0.0 in stage 222.0 (TID 60). 1852 bytes result sent to driver -2016-06-13 19:28:05 INFO Executor:59 - Finished task 1.0 in stage 222.0 (TID 61). 1744 bytes result sent to driver -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 222.0 (TID 60) in 16 ms on localhost (1/2) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 222.0 (TID 61) in 15 ms on localhost (2/2) -2016-06-13 19:28:05 INFO DAGScheduler:59 - ResultStage 222 (collectAsMap at KMeans.scala:450) finished in 0.016 s -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 222.0, whose tasks have all completed, from pool -2016-06-13 19:28:05 INFO DAGScheduler:59 - Job 20 finished: collectAsMap at KMeans.scala:450, took 0.087429 s -2016-06-13 19:28:05 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:28:05 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:28:05 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:28:05 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:28:05 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:28:05 INFO KMeans:59 - Initialization with k-means|| took 1.268 seconds. -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(824) called with curMem=185345, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 824.0 B, free 730.4 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(339) called with curMem=186169, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 339.0 B, free 730.4 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:49684 (size: 339.0 B, free: 730.5 MB) -2016-06-13 19:28:05 INFO SparkContext:59 - Created broadcast 37 from broadcast at KMeans.scala:276 -2016-06-13 19:28:05 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:302 -2016-06-13 19:28:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:28:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:28:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:28:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:28:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:28:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 151 bytes -2016-06-13 19:28:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:28:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 151 bytes -2016-06-13 19:28:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:28:05 INFO DAGScheduler:59 - Registering RDD 127 (mapPartitions at KMeans.scala:279) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Got job 21 (collectAsMap at KMeans.scala:302) with 2 output partitions -2016-06-13 19:28:05 INFO DAGScheduler:59 - Final stage: ResultStage 235(collectAsMap at KMeans.scala:302) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 234) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 234) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 234 (MapPartitionsRDD[127] at mapPartitions at KMeans.scala:279), which has no missing parents -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(10296) called with curMem=186508, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 10.1 KB, free 730.4 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(4440) called with curMem=196804, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:49684 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:28:05 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 234 (MapPartitionsRDD[127] at mapPartitions at KMeans.scala:279) -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Adding task set 234.0 with 2 tasks -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 234.0 (TID 62, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 234.0 (TID 63, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:28:05 INFO Executor:59 - Running task 0.0 in stage 234.0 (TID 62) -2016-06-13 19:28:05 INFO Executor:59 - Running task 1.0 in stage 234.0 (TID 63) -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_100_1 locally -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_100_0 locally -2016-06-13 19:28:05 INFO Executor:59 - Finished task 1.0 in stage 234.0 (TID 63). 2413 bytes result sent to driver -2016-06-13 19:28:05 INFO Executor:59 - Finished task 0.0 in stage 234.0 (TID 62). 2413 bytes result sent to driver -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 234.0 (TID 63) in 19 ms on localhost (1/2) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 234.0 (TID 62) in 20 ms on localhost (2/2) -2016-06-13 19:28:05 INFO DAGScheduler:59 - ShuffleMapStage 234 (mapPartitions at KMeans.scala:279) finished in 0.020 s -2016-06-13 19:28:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:28:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 235) -2016-06-13 19:28:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 234.0, whose tasks have all completed, from pool -2016-06-13 19:28:05 INFO DAGScheduler:59 - Missing parents for ResultStage 235: List() -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting ResultStage 235 (ShuffledRDD[128] at reduceByKey at KMeans.scala:302), which is now runnable -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(2808) called with curMem=201244, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(1589) called with curMem=204052, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 1589.0 B, free 730.4 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:49684 (size: 1589.0 B, free: 730.5 MB) -2016-06-13 19:28:05 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 235 (ShuffledRDD[128] at reduceByKey at KMeans.scala:302) -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Adding task set 235.0 with 2 tasks -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 235.0 (TID 64, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 235.0 (TID 65, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:28:05 INFO Executor:59 - Running task 0.0 in stage 235.0 (TID 64) -2016-06-13 19:28:05 INFO Executor:59 - Running task 1.0 in stage 235.0 (TID 65) -2016-06-13 19:28:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:05 INFO Executor:59 - Finished task 0.0 in stage 235.0 (TID 64). 1785 bytes result sent to driver -2016-06-13 19:28:05 INFO Executor:59 - Finished task 1.0 in stage 235.0 (TID 65). 1671 bytes result sent to driver -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 235.0 (TID 64) in 12 ms on localhost (1/2) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 235.0 (TID 65) in 13 ms on localhost (2/2) -2016-06-13 19:28:05 INFO DAGScheduler:59 - ResultStage 235 (collectAsMap at KMeans.scala:302) finished in 0.013 s -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 235.0, whose tasks have all completed, from pool -2016-06-13 19:28:05 INFO DAGScheduler:59 - Job 21 finished: collectAsMap at KMeans.scala:302, took 0.066415 s -2016-06-13 19:28:05 INFO KMeans:59 - Run 0 finished in 1 iterations -2016-06-13 19:28:05 INFO KMeans:59 - Run 1 finished in 1 iterations -2016-06-13 19:28:05 INFO KMeans:59 - Run 2 finished in 1 iterations -2016-06-13 19:28:05 INFO KMeans:59 - Run 3 finished in 1 iterations -2016-06-13 19:28:05 INFO KMeans:59 - Run 4 finished in 1 iterations -2016-06-13 19:28:05 INFO KMeans:59 - Iterations took 0.097 seconds. -2016-06-13 19:28:05 INFO KMeans:59 - KMeans converged in 1 iterations. -2016-06-13 19:28:05 INFO KMeans:59 - The cost for the best run is 6.650150218788335E-6. -2016-06-13 19:28:05 INFO MapPartitionsRDD:59 - Removing RDD 100 from persistence list -2016-06-13 19:28:05 INFO BlockManager:59 - Removing RDD 100 -2016-06-13 19:28:05 WARN KMeans:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:28:05 INFO SparkContext:59 - Starting job: collect at PowerIterationClusteringSuite.scala:56 -2016-06-13 19:28:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:28:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:28:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:28:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 151 bytes -2016-06-13 19:28:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:28:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 151 bytes -2016-06-13 19:28:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:28:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 151 bytes -2016-06-13 19:28:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:28:05 INFO DAGScheduler:59 - Got job 22 (collect at PowerIterationClusteringSuite.scala:56) with 2 output partitions -2016-06-13 19:28:05 INFO DAGScheduler:59 - Final stage: ResultStage 247(collect at PowerIterationClusteringSuite.scala:56) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 242, ShuffleMapStage 239, ShuffleMapStage 246, ShuffleMapStage 240, ShuffleMapStage 244) -2016-06-13 19:28:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting ResultStage 247 (MapPartitionsRDD[131] at mapPartitions at PowerIterationClustering.scala:232), which has no missing parents -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(8472) called with curMem=205161, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 8.3 KB, free 730.4 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=213633, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:49684 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:28:05 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 247 (MapPartitionsRDD[131] at mapPartitions at PowerIterationClustering.scala:232) -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Adding task set 247.0 with 2 tasks -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 247.0 (TID 66, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 247.0 (TID 67, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:28:05 INFO Executor:59 - Running task 0.0 in stage 247.0 (TID 66) -2016-06-13 19:28:05 INFO Executor:59 - Running task 1.0 in stage 247.0 (TID 67) -2016-06-13 19:28:05 INFO CacheManager:59 - Partition rdd_129_1 not found, computing it -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_1 locally -2016-06-13 19:28:05 INFO CacheManager:59 - Partition rdd_129_0 not found, computing it -2016-06-13 19:28:05 INFO BlockManager:59 - Found block rdd_97_0 locally -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=217225, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block rdd_129_1 stored as values in memory (estimated size 1168.0 B, free 730.4 MB) -2016-06-13 19:28:05 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=218393, maxMem=766075207 -2016-06-13 19:28:05 INFO MemoryStore:59 - Block rdd_129_0 stored as values in memory (estimated size 1168.0 B, free 730.4 MB) -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added rdd_129_1 in memory on localhost:49684 (size: 1168.0 B, free: 730.5 MB) -2016-06-13 19:28:05 INFO Executor:59 - Finished task 1.0 in stage 247.0 (TID 67). 2919 bytes result sent to driver -2016-06-13 19:28:05 INFO BlockManagerInfo:59 - Added rdd_129_0 in memory on localhost:49684 (size: 1168.0 B, free: 730.5 MB) -2016-06-13 19:28:05 INFO Executor:59 - Finished task 0.0 in stage 247.0 (TID 66). 2919 bytes result sent to driver -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 247.0 (TID 67) in 10 ms on localhost (1/2) -2016-06-13 19:28:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 247.0 (TID 66) in 12 ms on localhost (2/2) -2016-06-13 19:28:05 INFO DAGScheduler:59 - ResultStage 247 (collect at PowerIterationClusteringSuite.scala:56) finished in 0.012 s -2016-06-13 19:28:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 247.0, whose tasks have all completed, from pool -2016-06-13 19:28:05 INFO DAGScheduler:59 - Job 22 finished: collect at PowerIterationClusteringSuite.scala:56, took 0.034357 s -2016-06-13 19:28:06 INFO SparkContext:59 - Starting job: foreach at PowerIterationClusteringSuite.scala:62 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Got job 23 (foreach at PowerIterationClusteringSuite.scala:62) with 2 output partitions -2016-06-13 19:28:06 INFO DAGScheduler:59 - Final stage: ResultStage 259(foreach at PowerIterationClusteringSuite.scala:62) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 251, ShuffleMapStage 252, ShuffleMapStage 256, ShuffleMapStage 258, ShuffleMapStage 254) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting ResultStage 259 (MapPartitionsRDD[131] at mapPartitions at PowerIterationClustering.scala:232), which has no missing parents -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(8504) called with curMem=219561, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 8.3 KB, free 730.4 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(3621) called with curMem=228065, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:49684 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 259 (MapPartitionsRDD[131] at mapPartitions at PowerIterationClustering.scala:232) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Adding task set 259.0 with 2 tasks -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 259.0 (TID 68, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 259.0 (TID 69, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:28:06 INFO Executor:59 - Running task 0.0 in stage 259.0 (TID 68) -2016-06-13 19:28:06 INFO Executor:59 - Running task 1.0 in stage 259.0 (TID 69) -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_129_0 locally -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_129_1 locally -2016-06-13 19:28:06 INFO Executor:59 - Finished task 1.0 in stage 259.0 (TID 69). 2044 bytes result sent to driver -2016-06-13 19:28:06 INFO Executor:59 - Finished task 0.0 in stage 259.0 (TID 68). 2044 bytes result sent to driver -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 259.0 (TID 69) in 10 ms on localhost (1/2) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 259.0 (TID 68) in 10 ms on localhost (2/2) -2016-06-13 19:28:06 INFO DAGScheduler:59 - ResultStage 259 (foreach at PowerIterationClusteringSuite.scala:62) finished in 0.011 s -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 259.0, whose tasks have all completed, from pool -2016-06-13 19:28:06 INFO DAGScheduler:59 - Job 23 finished: foreach at PowerIterationClusteringSuite.scala:62, took 0.035533 s -2016-06-13 19:28:06 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:340 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Registering RDD 143 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Registering RDD 137 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Got job 24 (sum at PowerIterationClustering.scala:340) with 2 output partitions -2016-06-13 19:28:06 INFO DAGScheduler:59 - Final stage: ResultStage 262(sum at PowerIterationClustering.scala:340) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 260, ShuffleMapStage 261) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 260, ShuffleMapStage 261) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 260 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[143] at mapPartitions at GraphImpl.scala:235), which has no missing parents -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(3432) called with curMem=231686, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1974) called with curMem=235118, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 1974.0 B, free 730.4 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:49684 (size: 1974.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 260 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[143] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Adding task set 260.0 with 2 tasks -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 261 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[137] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 260.0 (TID 70, localhost, PROCESS_LOCAL, 2488 bytes) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 260.0 (TID 71, localhost, PROCESS_LOCAL, 2563 bytes) -2016-06-13 19:28:06 INFO Executor:59 - Running task 1.0 in stage 260.0 (TID 71) -2016-06-13 19:28:06 INFO Executor:59 - Running task 0.0 in stage 260.0 (TID 70) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(3016) called with curMem=237092, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1779) called with curMem=240108, maxMem=766075207 -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_134_0 not found, computing it -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 1779.0 B, free 730.4 MB) -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_134_1 not found, computing it -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(2856) called with curMem=241887, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_134_0 stored as values in memory (estimated size 2.8 KB, free 730.4 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(2952) called with curMem=244743, maxMem=766075207 -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:49684 (size: 1779.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_134_1 stored as values in memory (estimated size 2.9 KB, free 730.4 MB) -2016-06-13 19:28:06 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_134_0 in memory on localhost:49684 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 261 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[137] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Adding task set 261.0 with 2 tasks -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_134_1 in memory on localhost:49684 (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 261.0 (TID 72, localhost, PROCESS_LOCAL, 2488 bytes) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 261.0 (TID 73, localhost, PROCESS_LOCAL, 2563 bytes) -2016-06-13 19:28:06 INFO Executor:59 - Running task 0.0 in stage 261.0 (TID 72) -2016-06-13 19:28:06 INFO Executor:59 - Running task 1.0 in stage 261.0 (TID 73) -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_134_0 locally -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_134_1 locally -2016-06-13 19:28:06 INFO Executor:59 - Finished task 0.0 in stage 260.0 (TID 70). 1739 bytes result sent to driver -2016-06-13 19:28:06 INFO Executor:59 - Finished task 1.0 in stage 261.0 (TID 73). 2254 bytes result sent to driver -2016-06-13 19:28:06 INFO Executor:59 - Finished task 0.0 in stage 261.0 (TID 72). 2254 bytes result sent to driver -2016-06-13 19:28:06 INFO Executor:59 - Finished task 1.0 in stage 260.0 (TID 71). 1739 bytes result sent to driver -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 260.0 (TID 70) in 37 ms on localhost (1/2) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 260.0 (TID 71) in 37 ms on localhost (2/2) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 260.0, whose tasks have all completed, from pool -2016-06-13 19:28:06 INFO DAGScheduler:59 - ShuffleMapStage 260 (mapPartitions at GraphImpl.scala:235) finished in 0.039 s -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 261.0 (TID 73) in 31 ms on localhost (1/2) -2016-06-13 19:28:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:06 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 261) -2016-06-13 19:28:06 INFO DAGScheduler:59 - waiting: Set(ResultStage 262) -2016-06-13 19:28:06 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 261.0 (TID 72) in 32 ms on localhost (2/2) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 261.0, whose tasks have all completed, from pool -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents for ResultStage 262: List(ShuffleMapStage 261) -2016-06-13 19:28:06 INFO DAGScheduler:59 - ShuffleMapStage 261 (mapPartitions at VertexRDD.scala:358) finished in 0.037 s -2016-06-13 19:28:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:06 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:28:06 INFO DAGScheduler:59 - waiting: Set(ResultStage 262) -2016-06-13 19:28:06 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents for ResultStage 262: List() -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting ResultStage 262 (MapPartitionsRDD[153] at values at PowerIterationClustering.scala:340), which is now runnable -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(4728) called with curMem=247695, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 4.6 KB, free 730.3 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(2556) called with curMem=252423, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:49684 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 262 (MapPartitionsRDD[153] at values at PowerIterationClustering.scala:340) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Adding task set 262.0 with 2 tasks -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 262.0 (TID 74, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 262.0 (TID 75, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:28:06 INFO Executor:59 - Running task 1.0 in stage 262.0 (TID 75) -2016-06-13 19:28:06 INFO Executor:59 - Running task 0.0 in stage 262.0 (TID 74) -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_145_1 not found, computing it -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_140_1 not found, computing it -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1792) called with curMem=254979, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_140_1 stored as values in memory (estimated size 1792.0 B, free 730.3 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_140_1 in memory on localhost:49684 (size: 1792.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1840) called with curMem=256771, maxMem=766075207 -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_145_0 not found, computing it -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_145_1 stored as values in memory (estimated size 1840.0 B, free 730.3 MB) -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_140_0 not found, computing it -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_145_1 in memory on localhost:49684 (size: 1840.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:06 INFO Executor:59 - Finished task 1.0 in stage 262.0 (TID 75). 1840 bytes result sent to driver -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 262.0 (TID 75) in 32 ms on localhost (1/2) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1776) called with curMem=258611, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_140_0 stored as values in memory (estimated size 1776.0 B, free 730.3 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_140_0 in memory on localhost:49684 (size: 1776.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1824) called with curMem=260387, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_145_0 stored as values in memory (estimated size 1824.0 B, free 730.3 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_145_0 in memory on localhost:49684 (size: 1824.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO Executor:59 - Finished task 0.0 in stage 262.0 (TID 74). 1840 bytes result sent to driver -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 262.0 (TID 74) in 79 ms on localhost (2/2) -2016-06-13 19:28:06 INFO DAGScheduler:59 - ResultStage 262 (sum at PowerIterationClustering.scala:340) finished in 0.079 s -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 262.0, whose tasks have all completed, from pool -2016-06-13 19:28:06 INFO DAGScheduler:59 - Job 24 finished: sum at PowerIterationClustering.scala:340, took 0.149758 s -2016-06-13 19:28:06 INFO SparkContext:59 - Starting job: reduce at VertexRDDImpl.scala:90 -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:28:06 INFO DAGScheduler:59 - Got job 25 (reduce at VertexRDDImpl.scala:90) with 2 output partitions -2016-06-13 19:28:06 INFO DAGScheduler:59 - Final stage: ResultStage 265(reduce at VertexRDDImpl.scala:90) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 263, ShuffleMapStage 264) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting ResultStage 265 (MapPartitionsRDD[158] at map at VertexRDDImpl.scala:90), which has no missing parents -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(5200) called with curMem=262211, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 5.1 KB, free 730.3 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(2659) called with curMem=267411, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.3 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:49684 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 265 (MapPartitionsRDD[158] at map at VertexRDDImpl.scala:90) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Adding task set 265.0 with 2 tasks -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 265.0 (TID 76, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 265.0 (TID 77, localhost, PROCESS_LOCAL, 2084 bytes) -2016-06-13 19:28:06 INFO Executor:59 - Running task 0.0 in stage 265.0 (TID 76) -2016-06-13 19:28:06 INFO Executor:59 - Running task 1.0 in stage 265.0 (TID 77) -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_145_0 locally -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_145_1 locally -2016-06-13 19:28:06 INFO Executor:59 - Finished task 0.0 in stage 265.0 (TID 76). 2161 bytes result sent to driver -2016-06-13 19:28:06 INFO Executor:59 - Finished task 1.0 in stage 265.0 (TID 77). 2161 bytes result sent to driver -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 265.0 (TID 76) in 7 ms on localhost (1/2) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 265.0 (TID 77) in 8 ms on localhost (2/2) -2016-06-13 19:28:06 INFO DAGScheduler:59 - ResultStage 265 (reduce at VertexRDDImpl.scala:90) finished in 0.008 s -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 265.0, whose tasks have all completed, from pool -2016-06-13 19:28:06 INFO DAGScheduler:59 - Job 25 finished: reduce at VertexRDDImpl.scala:90, took 0.019467 s -2016-06-13 19:28:06 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Registering RDD 147 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Registering RDD 159 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Registering RDD 163 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Got job 26 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:28:06 INFO DAGScheduler:59 - Final stage: ResultStage 273(sum at PowerIterationClustering.scala:369) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 269, ShuffleMapStage 270, ShuffleMapStage 272) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 272) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 268 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[147] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(4248) called with curMem=270070, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 4.1 KB, free 730.3 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(2378) called with curMem=274318, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.3 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:49684 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 268 (ReplicatedVertexView.upgrade(true, false) - shippedVerts true false (broadcast) MapPartitionsRDD[147] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Adding task set 268.0 with 2 tasks -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 271 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[159] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 268.0 (TID 78, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 268.0 (TID 79, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:28:06 INFO Executor:59 - Running task 1.0 in stage 268.0 (TID 79) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(4904) called with curMem=276696, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 4.8 KB, free 730.3 MB) -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_145_1 locally -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(2572) called with curMem=281600, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:28:06 INFO Executor:59 - Running task 0.0 in stage 268.0 (TID 78) -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_145_0 locally -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:49684 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 271 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[159] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Adding task set 271.0 with 2 tasks -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 271.0 (TID 80, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 271.0 (TID 81, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:28:06 INFO Executor:59 - Running task 1.0 in stage 271.0 (TID 81) -2016-06-13 19:28:06 INFO Executor:59 - Running task 0.0 in stage 271.0 (TID 80) -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_156_1 not found, computing it -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_145_1 locally -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_156_0 not found, computing it -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_145_0 locally -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=284172, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_156_1 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=285604, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_156_0 stored as values in memory (estimated size 1432.0 B, free 730.3 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_156_1 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_156_0 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO Executor:59 - Finished task 0.0 in stage 271.0 (TID 80). 2834 bytes result sent to driver -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 271.0 (TID 80) in 21 ms on localhost (1/2) -2016-06-13 19:28:06 INFO Executor:59 - Finished task 1.0 in stage 271.0 (TID 81). 2834 bytes result sent to driver -2016-06-13 19:28:06 INFO Executor:59 - Finished task 0.0 in stage 268.0 (TID 78). 2254 bytes result sent to driver -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 271.0 (TID 81) in 31 ms on localhost (2/2) -2016-06-13 19:28:06 INFO DAGScheduler:59 - ShuffleMapStage 271 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.032 s -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 271.0, whose tasks have all completed, from pool -2016-06-13 19:28:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:06 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 268) -2016-06-13 19:28:06 INFO DAGScheduler:59 - waiting: Set(ResultStage 273, ShuffleMapStage 272) -2016-06-13 19:28:06 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 268.0 (TID 78) in 49 ms on localhost (1/2) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents for ResultStage 273: List(ShuffleMapStage 272) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 272: List(ShuffleMapStage 268) -2016-06-13 19:28:06 INFO Executor:59 - Finished task 1.0 in stage 268.0 (TID 79). 2254 bytes result sent to driver -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 268.0 (TID 79) in 53 ms on localhost (2/2) -2016-06-13 19:28:06 INFO DAGScheduler:59 - ShuffleMapStage 268 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.055 s -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 268.0, whose tasks have all completed, from pool -2016-06-13 19:28:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:06 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:28:06 INFO DAGScheduler:59 - waiting: Set(ResultStage 273, ShuffleMapStage 272) -2016-06-13 19:28:06 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents for ResultStage 273: List(ShuffleMapStage 272) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 272: List() -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 272 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[163] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=287036, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 4.7 KB, free 730.3 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(2538) called with curMem=291884, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:49684 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 272 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[163] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Adding task set 272.0 with 2 tasks -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 272.0 (TID 82, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 272.0 (TID 83, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:28:06 INFO Executor:59 - Running task 1.0 in stage 272.0 (TID 83) -2016-06-13 19:28:06 INFO Executor:59 - Running task 0.0 in stage 272.0 (TID 82) -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_134_1 locally -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_134_0 locally -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:28:06 INFO Executor:59 - Finished task 0.0 in stage 272.0 (TID 82). 2470 bytes result sent to driver -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 272.0 (TID 82) in 32 ms on localhost (1/2) -2016-06-13 19:28:06 INFO Executor:59 - Finished task 1.0 in stage 272.0 (TID 83). 2470 bytes result sent to driver -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 272.0 (TID 83) in 35 ms on localhost (2/2) -2016-06-13 19:28:06 INFO DAGScheduler:59 - ShuffleMapStage 272 (mapPartitions at GraphImpl.scala:235) finished in 0.035 s -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 272.0, whose tasks have all completed, from pool -2016-06-13 19:28:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:06 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:28:06 INFO DAGScheduler:59 - waiting: Set(ResultStage 273) -2016-06-13 19:28:06 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents for ResultStage 273: List() -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting ResultStage 273 (MapPartitionsRDD[168] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(5944) called with curMem=294422, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 5.8 KB, free 730.3 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(2892) called with curMem=300366, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:49684 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 273 (MapPartitionsRDD[168] at map at PowerIterationClustering.scala:369) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Adding task set 273.0 with 2 tasks -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 273.0 (TID 84, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 273.0 (TID 85, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:28:06 INFO Executor:59 - Running task 0.0 in stage 273.0 (TID 84) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:49684 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO Executor:59 - Running task 1.0 in stage 273.0 (TID 85) -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_165_0 not found, computing it -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_156_0 locally -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:49684 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=289156, maxMem=766075207 -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_165_1 not found, computing it -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_165_0 stored as values in memory (estimated size 1480.0 B, free 730.3 MB) -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_156_1 locally -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_165_0 in memory on localhost:49684 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO Executor:59 - Finished task 0.0 in stage 273.0 (TID 84). 2880 bytes result sent to driver -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 273.0 (TID 84) in 14 ms on localhost (1/2) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=290636, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_165_1 stored as values in memory (estimated size 1480.0 B, free 730.3 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_165_1 in memory on localhost:49684 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO Executor:59 - Finished task 1.0 in stage 273.0 (TID 85). 2880 bytes result sent to driver -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:49684 in memory (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 273.0 (TID 85) in 23 ms on localhost (2/2) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 273.0, whose tasks have all completed, from pool -2016-06-13 19:28:06 INFO DAGScheduler:59 - ResultStage 273 (sum at PowerIterationClustering.scala:369) finished in 0.025 s -2016-06-13 19:28:06 INFO DAGScheduler:59 - Job 26 finished: sum at PowerIterationClustering.scala:369, took 0.192571 s -2016-06-13 19:28:06 INFO PowerIterationClustering:59 - Iteration 0: norm(v) = 36.22334869431644. -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:49684 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:49684 in memory (size: 1779.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:49684 in memory (size: 1974.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-06-13 19:28:06 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:49684 in memory (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 151 bytes -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:49684 in memory (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:28:06 INFO DAGScheduler:59 - Got job 27 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:28:06 INFO DAGScheduler:59 - Final stage: ResultStage 281(sum at PowerIterationClustering.scala:375) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 277, ShuffleMapStage 278, ShuffleMapStage 280) -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting ResultStage 281 (MapPartitionsRDD[179] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:49684 in memory (size: 1589.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(6480) called with curMem=238186, maxMem=766075207 -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 6.3 KB, free 730.4 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(3063) called with curMem=240226, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:49684 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:49684 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 281 (MapPartitionsRDD[179] at values at PowerIterationClustering.scala:375) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Adding task set 281.0 with 2 tasks -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 281.0 (TID 86, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 281.0 (TID 87, localhost, PROCESS_LOCAL, 2157 bytes) -2016-06-13 19:28:06 INFO Executor:59 - Running task 0.0 in stage 281.0 (TID 86) -2016-06-13 19:28:06 INFO Executor:59 - Running task 1.0 in stage 281.0 (TID 87) -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned shuffle 10 -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_171_1 not found, computing it -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_156_1 locally -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_165_1 locally -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=232654, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_171_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:49684 in memory (size: 339.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_171_1 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO Executor:59 - Finished task 1.0 in stage 281.0 (TID 87). 2664 bytes result sent to driver -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:49684 in memory (size: 1382.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_171_0 not found, computing it -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_156_0 locally -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_165_0 locally -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=229568, maxMem=766075207 -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 281.0 (TID 87) in 20 ms on localhost (1/2) -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_171_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_171_0 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO Executor:59 - Finished task 0.0 in stage 281.0 (TID 86). 2664 bytes result sent to driver -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:49684 in memory (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 281.0 (TID 86) in 25 ms on localhost (2/2) -2016-06-13 19:28:06 INFO DAGScheduler:59 - ResultStage 281 (sum at PowerIterationClustering.scala:375) finished in 0.025 s -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned shuffle 9 -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 281.0, whose tasks have all completed, from pool -2016-06-13 19:28:06 INFO DAGScheduler:59 - Job 27 finished: sum at PowerIterationClustering.scala:375, took 0.048141 s -2016-06-13 19:28:06 INFO PowerIterationClustering:59 - Iteration 0: delta = 0.010579646370791257. -2016-06-13 19:28:06 INFO PowerIterationClustering:59 - Iteration 0: diff(delta) = 1.7976931348623157E308. -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:49684 in memory (size: 708.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:49684 in memory (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:49684 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-06-13 19:28:06 INFO BlockManager:59 - Removing RDD 122 -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned RDD 122 -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:49684 in memory (size: 236.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:49684 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Registering RDD 182 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Registering RDD 186 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:49684 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Got job 28 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:28:06 INFO DAGScheduler:59 - Final stage: ResultStage 291(sum at PowerIterationClustering.scala:369) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 288, ShuffleMapStage 285, ShuffleMapStage 286, ShuffleMapStage 290) -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-06-13 19:28:06 INFO BlockManager:59 - Removing RDD 118 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 290) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 289 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[182] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned RDD 118 -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(5704) called with curMem=150607, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 5.6 KB, free 730.4 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(2815) called with curMem=155234, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:49684 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:49684 in memory (size: 429.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 289 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[182] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Adding task set 289.0 with 2 tasks -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 289.0 (TID 88, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 289.0 (TID 89, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:28:06 INFO Executor:59 - Running task 1.0 in stage 289.0 (TID 89) -2016-06-13 19:28:06 INFO Executor:59 - Running task 0.0 in stage 289.0 (TID 88) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:49684 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_180_1 not found, computing it -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_165_1 locally -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=142869, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_180_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_180_1 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_180_0 not found, computing it -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_165_0 locally -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=144301, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_180_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:28:06 INFO Executor:59 - Finished task 1.0 in stage 289.0 (TID 89). 2834 bytes result sent to driver -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_180_0 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 289.0 (TID 89) in 15 ms on localhost (1/2) -2016-06-13 19:28:06 INFO Executor:59 - Finished task 0.0 in stage 289.0 (TID 88). 2834 bytes result sent to driver -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-06-13 19:28:06 INFO DAGScheduler:59 - ShuffleMapStage 289 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.018 s -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 289.0 (TID 88) in 17 ms on localhost (2/2) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 289.0, whose tasks have all completed, from pool -2016-06-13 19:28:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:06 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:28:06 INFO DAGScheduler:59 - waiting: Set(ResultStage 291, ShuffleMapStage 290) -2016-06-13 19:28:06 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents for ResultStage 291: List(ShuffleMapStage 290) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:49684 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 290: List() -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 290 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[186] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=130668, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 4.7 KB, free 730.5 MB) -2016-06-13 19:28:06 INFO BlockManager:59 - Removing RDD 114 -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned RDD 114 -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=135516, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:49684 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:49684 in memory (size: 582.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 290 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[186] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Adding task set 290.0 with 2 tasks -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 290.0 (TID 90, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 290.0 (TID 91, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:28:06 INFO Executor:59 - Running task 1.0 in stage 290.0 (TID 91) -2016-06-13 19:28:06 INFO Executor:59 - Running task 0.0 in stage 290.0 (TID 90) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:49684 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_134_1 locally -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_134_0 locally -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:49684 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 19:28:06 INFO BlockManager:59 - Removing RDD 110 -2016-06-13 19:28:06 INFO Executor:59 - Finished task 0.0 in stage 290.0 (TID 90). 2470 bytes result sent to driver -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned RDD 110 -2016-06-13 19:28:06 INFO Executor:59 - Finished task 1.0 in stage 290.0 (TID 91). 2470 bytes result sent to driver -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 290.0 (TID 90) in 38 ms on localhost (1/2) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:49684 in memory (size: 600.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 290.0 (TID 91) in 44 ms on localhost (2/2) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 290.0, whose tasks have all completed, from pool -2016-06-13 19:28:06 INFO DAGScheduler:59 - ShuffleMapStage 290 (mapPartitions at GraphImpl.scala:235) finished in 0.046 s -2016-06-13 19:28:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:06 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:28:06 INFO DAGScheduler:59 - waiting: Set(ResultStage 291) -2016-06-13 19:28:06 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:06 INFO BlockManager:59 - Removing RDD 106 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents for ResultStage 291: List() -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting ResultStage 291 (MapPartitionsRDD[191] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned RDD 106 -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(6640) called with curMem=102714, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 6.5 KB, free 730.5 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:49684 in memory (size: 487.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(3103) called with curMem=109354, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.5 MB) -2016-06-13 19:28:06 INFO BlockManager:59 - Removing RDD 100 -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:49684 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 291 (MapPartitionsRDD[191] at map at PowerIterationClustering.scala:369) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Adding task set 291.0 with 2 tasks -2016-06-13 19:28:06 INFO ContextCleaner:59 - Cleaned RDD 100 -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 291.0 (TID 92, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 291.0 (TID 93, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:28:06 INFO Executor:59 - Running task 1.0 in stage 291.0 (TID 93) -2016-06-13 19:28:06 INFO Executor:59 - Running task 0.0 in stage 291.0 (TID 92) -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_188_0 not found, computing it -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_180_0 locally -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_188_1 not found, computing it -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_180_1 locally -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=112457, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_188_0 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_188_0 in memory on localhost:49684 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO Executor:59 - Finished task 0.0 in stage 291.0 (TID 92). 2880 bytes result sent to driver -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=113937, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_188_1 stored as values in memory (estimated size 1480.0 B, free 730.5 MB) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 291.0 (TID 92) in 13 ms on localhost (1/2) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_188_1 in memory on localhost:49684 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO Executor:59 - Finished task 1.0 in stage 291.0 (TID 93). 2880 bytes result sent to driver -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 291.0 (TID 93) in 16 ms on localhost (2/2) -2016-06-13 19:28:06 INFO DAGScheduler:59 - ResultStage 291 (sum at PowerIterationClustering.scala:369) finished in 0.017 s -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 291.0, whose tasks have all completed, from pool -2016-06-13 19:28:06 INFO DAGScheduler:59 - Job 28 finished: sum at PowerIterationClustering.scala:369, took 0.134711 s -2016-06-13 19:28:06 INFO PowerIterationClustering:59 - Iteration 1: norm(v) = 36.22334869431644. -2016-06-13 19:28:06 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 151 bytes -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 151 bytes -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:28:06 INFO DAGScheduler:59 - Got job 29 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:28:06 INFO DAGScheduler:59 - Final stage: ResultStage 301(sum at PowerIterationClustering.scala:375) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 300, ShuffleMapStage 296, ShuffleMapStage 298, ShuffleMapStage 295) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting ResultStage 301 (MapPartitionsRDD[202] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(7072) called with curMem=115417, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 6.9 KB, free 730.5 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(3232) called with curMem=122489, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:49684 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 301 (MapPartitionsRDD[202] at values at PowerIterationClustering.scala:375) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Adding task set 301.0 with 2 tasks -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 301.0 (TID 94, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 301.0 (TID 95, localhost, PROCESS_LOCAL, 2198 bytes) -2016-06-13 19:28:06 INFO Executor:59 - Running task 1.0 in stage 301.0 (TID 95) -2016-06-13 19:28:06 INFO Executor:59 - Running task 0.0 in stage 301.0 (TID 94) -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_194_1 not found, computing it -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_180_1 locally -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_188_1 locally -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=125721, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_194_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_194_0 not found, computing it -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_180_0 locally -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_194_1 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_188_0 locally -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=127153, maxMem=766075207 -2016-06-13 19:28:06 INFO Executor:59 - Finished task 1.0 in stage 301.0 (TID 95). 2664 bytes result sent to driver -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_194_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_194_0 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 301.0 (TID 95) in 12 ms on localhost (1/2) -2016-06-13 19:28:06 INFO Executor:59 - Finished task 0.0 in stage 301.0 (TID 94). 2664 bytes result sent to driver -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 301.0 (TID 94) in 19 ms on localhost (2/2) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 301.0, whose tasks have all completed, from pool -2016-06-13 19:28:06 INFO DAGScheduler:59 - ResultStage 301 (sum at PowerIterationClustering.scala:375) finished in 0.020 s -2016-06-13 19:28:06 INFO DAGScheduler:59 - Job 29 finished: sum at PowerIterationClustering.scala:375, took 0.039971 s -2016-06-13 19:28:06 INFO PowerIterationClustering:59 - Iteration 1: delta = 0.0. -2016-06-13 19:28:06 INFO PowerIterationClustering:59 - Iteration 1: diff(delta) = 0.010579646370791257. -2016-06-13 19:28:06 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:369 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Registering RDD 205 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Registering RDD 209 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Got job 30 (sum at PowerIterationClustering.scala:369) with 2 output partitions -2016-06-13 19:28:06 INFO DAGScheduler:59 - Final stage: ResultStage 313(sum at PowerIterationClustering.scala:369) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 305, ShuffleMapStage 306, ShuffleMapStage 310, ShuffleMapStage 308, ShuffleMapStage 312) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 312) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 311 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[205] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(6288) called with curMem=128585, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 6.1 KB, free 730.5 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(2963) called with curMem=134873, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.5 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:49684 (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 311 (ReplicatedVertexView.upgrade(false, true) - shippedVerts false true (broadcast) MapPartitionsRDD[205] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Adding task set 311.0 with 2 tasks -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 311.0 (TID 96, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 311.0 (TID 97, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:28:06 INFO Executor:59 - Running task 1.0 in stage 311.0 (TID 97) -2016-06-13 19:28:06 INFO Executor:59 - Running task 0.0 in stage 311.0 (TID 96) -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_203_1 not found, computing it -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_188_1 locally -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=137836, maxMem=766075207 -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_203_0 not found, computing it -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_188_0 locally -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_203_1 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=139268, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_203_0 stored as values in memory (estimated size 1432.0 B, free 730.5 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_203_1 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_203_0 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO Executor:59 - Finished task 1.0 in stage 311.0 (TID 97). 2834 bytes result sent to driver -2016-06-13 19:28:06 INFO Executor:59 - Finished task 0.0 in stage 311.0 (TID 96). 2834 bytes result sent to driver -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 311.0 (TID 97) in 13 ms on localhost (1/2) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 311.0 (TID 96) in 15 ms on localhost (2/2) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 311.0, whose tasks have all completed, from pool -2016-06-13 19:28:06 INFO DAGScheduler:59 - ShuffleMapStage 311 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.016 s -2016-06-13 19:28:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:06 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:28:06 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 312, ResultStage 313) -2016-06-13 19:28:06 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 312: List() -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents for ResultStage 313: List(ShuffleMapStage 312) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting ShuffleMapStage 312 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[209] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(4848) called with curMem=140700, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(2542) called with curMem=145548, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:49684 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 312 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[209] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Adding task set 312.0 with 2 tasks -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 312.0 (TID 98, localhost, PROCESS_LOCAL, 2778 bytes) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 312.0 (TID 99, localhost, PROCESS_LOCAL, 2853 bytes) -2016-06-13 19:28:06 INFO Executor:59 - Running task 1.0 in stage 312.0 (TID 99) -2016-06-13 19:28:06 INFO Executor:59 - Running task 0.0 in stage 312.0 (TID 98) -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_134_1 locally -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_134_0 locally -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:06 INFO Executor:59 - Finished task 1.0 in stage 312.0 (TID 99). 2470 bytes result sent to driver -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 312.0 (TID 99) in 28 ms on localhost (1/2) -2016-06-13 19:28:06 INFO Executor:59 - Finished task 0.0 in stage 312.0 (TID 98). 2470 bytes result sent to driver -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 312.0 (TID 98) in 35 ms on localhost (2/2) -2016-06-13 19:28:06 INFO DAGScheduler:59 - ShuffleMapStage 312 (mapPartitions at GraphImpl.scala:235) finished in 0.035 s -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 312.0, whose tasks have all completed, from pool -2016-06-13 19:28:06 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:06 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:28:06 INFO DAGScheduler:59 - waiting: Set(ResultStage 313) -2016-06-13 19:28:06 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents for ResultStage 313: List() -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting ResultStage 313 (MapPartitionsRDD[214] at map at PowerIterationClustering.scala:369), which is now runnable -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(7224) called with curMem=148090, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 7.1 KB, free 730.4 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(3233) called with curMem=155314, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:49684 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 313 (MapPartitionsRDD[214] at map at PowerIterationClustering.scala:369) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Adding task set 313.0 with 2 tasks -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 313.0 (TID 100, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 313.0 (TID 101, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:28:06 INFO Executor:59 - Running task 0.0 in stage 313.0 (TID 100) -2016-06-13 19:28:06 INFO Executor:59 - Running task 1.0 in stage 313.0 (TID 101) -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_211_1 not found, computing it -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_203_1 locally -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:06 INFO CacheManager:59 - Partition rdd_211_0 not found, computing it -2016-06-13 19:28:06 INFO BlockManager:59 - Found block rdd_203_0 locally -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:06 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=158547, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_211_1 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_211_1 in memory on localhost:49684 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(1480) called with curMem=160027, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block rdd_211_0 stored as values in memory (estimated size 1480.0 B, free 730.4 MB) -2016-06-13 19:28:06 INFO Executor:59 - Finished task 1.0 in stage 313.0 (TID 101). 2880 bytes result sent to driver -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added rdd_211_0 in memory on localhost:49684 (size: 1480.0 B, free: 730.5 MB) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 313.0 (TID 101) in 14 ms on localhost (1/2) -2016-06-13 19:28:06 INFO Executor:59 - Finished task 0.0 in stage 313.0 (TID 100). 2880 bytes result sent to driver -2016-06-13 19:28:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 313.0 (TID 100) in 17 ms on localhost (2/2) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 313.0, whose tasks have all completed, from pool -2016-06-13 19:28:06 INFO DAGScheduler:59 - ResultStage 313 (sum at PowerIterationClustering.scala:369) finished in 0.018 s -2016-06-13 19:28:06 INFO DAGScheduler:59 - Job 30 finished: sum at PowerIterationClustering.scala:369, took 0.118779 s -2016-06-13 19:28:06 INFO PowerIterationClustering:59 - Iteration 2: norm(v) = 36.22334869431644. -2016-06-13 19:28:06 INFO SparkContext:59 - Starting job: sum at PowerIterationClustering.scala:375 -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 151 bytes -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 151 bytes -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 151 bytes -2016-06-13 19:28:06 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:28:06 INFO DAGScheduler:59 - Got job 31 (sum at PowerIterationClustering.scala:375) with 2 output partitions -2016-06-13 19:28:06 INFO DAGScheduler:59 - Final stage: ResultStage 325(sum at PowerIterationClustering.scala:375) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 320, ShuffleMapStage 317, ShuffleMapStage 324, ShuffleMapStage 318, ShuffleMapStage 322) -2016-06-13 19:28:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting ResultStage 325 (MapPartitionsRDD[225] at values at PowerIterationClustering.scala:375), which has no missing parents -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=161507, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 7.5 KB, free 730.4 MB) -2016-06-13 19:28:06 INFO MemoryStore:59 - ensureFreeSpace(3360) called with curMem=169163, maxMem=766075207 -2016-06-13 19:28:06 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 19:28:06 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:49684 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:28:06 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 325 (MapPartitionsRDD[225] at values at PowerIterationClustering.scala:375) -2016-06-13 19:28:06 INFO TaskSchedulerImpl:59 - Adding task set 325.0 with 2 tasks -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 325.0 (TID 102, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:28:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 325.0 (TID 103, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:28:06 INFO Executor:59 - Running task 1.0 in stage 325.0 (TID 103) -2016-06-13 19:28:06 INFO Executor:59 - Running task 0.0 in stage 325.0 (TID 102) -2016-06-13 19:28:07 INFO CacheManager:59 - Partition rdd_217_1 not found, computing it -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_203_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_211_1 locally -2016-06-13 19:28:07 INFO CacheManager:59 - Partition rdd_217_0 not found, computing it -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_203_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_211_0 locally -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=172523, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block rdd_217_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=173955, maxMem=766075207 -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added rdd_217_1 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - Block rdd_217_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added rdd_217_0 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:07 INFO Executor:59 - Finished task 1.0 in stage 325.0 (TID 103). 2664 bytes result sent to driver -2016-06-13 19:28:07 INFO Executor:59 - Finished task 0.0 in stage 325.0 (TID 102). 2664 bytes result sent to driver -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 325.0 (TID 103) in 10 ms on localhost (1/2) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 325.0 (TID 102) in 12 ms on localhost (2/2) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 325.0, whose tasks have all completed, from pool -2016-06-13 19:28:07 INFO DAGScheduler:59 - ResultStage 325 (sum at PowerIterationClustering.scala:375) finished in 0.012 s -2016-06-13 19:28:07 INFO DAGScheduler:59 - Job 31 finished: sum at PowerIterationClustering.scala:375, took 0.040464 s -2016-06-13 19:28:07 INFO PowerIterationClustering:59 - Iteration 2: delta = 0.0. -2016-06-13 19:28:07 INFO PowerIterationClustering:59 - Iteration 2: diff(delta) = 0.0. -2016-06-13 19:28:07 WARN KMeans:71 - The input data is not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:28:07 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Got job 32 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:28:07 INFO DAGScheduler:59 - Final stage: ResultStage 337(takeSample at KMeans.scala:376) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 332, ShuffleMapStage 336, ShuffleMapStage 329, ShuffleMapStage 330, ShuffleMapStage 334) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting ResultStage 337 (MapPartitionsRDD[233] at map at KMeans.scala:210), which has no missing parents -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(7888) called with curMem=175387, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 7.7 KB, free 730.4 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(3424) called with curMem=183275, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:49684 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 337 (MapPartitionsRDD[233] at map at KMeans.scala:210) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Adding task set 337.0 with 2 tasks -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 337.0 (TID 104, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 337.0 (TID 105, localhost, PROCESS_LOCAL, 2239 bytes) -2016-06-13 19:28:07 INFO Executor:59 - Running task 0.0 in stage 337.0 (TID 104) -2016-06-13 19:28:07 INFO Executor:59 - Running task 1.0 in stage 337.0 (TID 105) -2016-06-13 19:28:07 INFO CacheManager:59 - Partition rdd_228_1 not found, computing it -2016-06-13 19:28:07 INFO CacheManager:59 - Partition rdd_228_0 not found, computing it -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_211_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_211_0 locally -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=186699, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block rdd_228_1 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(1432) called with curMem=188131, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block rdd_228_0 stored as values in memory (estimated size 1432.0 B, free 730.4 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added rdd_228_1 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added rdd_228_0 in memory on localhost:49684 (size: 1432.0 B, free: 730.5 MB) -2016-06-13 19:28:07 INFO CacheManager:59 - Partition rdd_231_1 not found, computing it -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:28:07 INFO CacheManager:59 - Partition rdd_231_0 not found, computing it -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=189563, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block rdd_231_1 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=189803, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block rdd_231_0 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added rdd_231_1 in memory on localhost:49684 (size: 240.0 B, free: 730.5 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added rdd_231_0 in memory on localhost:49684 (size: 240.0 B, free: 730.5 MB) -2016-06-13 19:28:07 INFO Executor:59 - Finished task 1.0 in stage 337.0 (TID 105). 2717 bytes result sent to driver -2016-06-13 19:28:07 INFO Executor:59 - Finished task 0.0 in stage 337.0 (TID 104). 2717 bytes result sent to driver -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 337.0 (TID 105) in 11 ms on localhost (1/2) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 337.0 (TID 104) in 12 ms on localhost (2/2) -2016-06-13 19:28:07 INFO DAGScheduler:59 - ResultStage 337 (takeSample at KMeans.scala:376) finished in 0.012 s -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 337.0, whose tasks have all completed, from pool -2016-06-13 19:28:07 INFO DAGScheduler:59 - Job 32 finished: takeSample at KMeans.scala:376, took 0.034706 s -2016-06-13 19:28:07 INFO SparkContext:59 - Starting job: takeSample at KMeans.scala:376 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Got job 33 (takeSample at KMeans.scala:376) with 2 output partitions -2016-06-13 19:28:07 INFO DAGScheduler:59 - Final stage: ResultStage 349(takeSample at KMeans.scala:376) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 342, ShuffleMapStage 344, ShuffleMapStage 341, ShuffleMapStage 348, ShuffleMapStage 346) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting ResultStage 349 (PartitionwiseSampledRDD[235] at takeSample at KMeans.scala:376), which has no missing parents -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(25280) called with curMem=190043, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 24.7 KB, free 730.4 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(10101) called with curMem=215323, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 9.9 KB, free 730.4 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:49684 (size: 9.9 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 349 (PartitionwiseSampledRDD[235] at takeSample at KMeans.scala:376) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Adding task set 349.0 with 2 tasks -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 349.0 (TID 106, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 349.0 (TID 107, localhost, PROCESS_LOCAL, 2348 bytes) -2016-06-13 19:28:07 INFO Executor:59 - Running task 0.0 in stage 349.0 (TID 106) -2016-06-13 19:28:07 INFO Executor:59 - Running task 1.0 in stage 349.0 (TID 107) -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:28:07 INFO Executor:59 - Finished task 0.0 in stage 349.0 (TID 106). 2633 bytes result sent to driver -2016-06-13 19:28:07 INFO Executor:59 - Finished task 1.0 in stage 349.0 (TID 107). 2575 bytes result sent to driver -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 349.0 (TID 106) in 7 ms on localhost (1/2) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 349.0 (TID 107) in 7 ms on localhost (2/2) -2016-06-13 19:28:07 INFO DAGScheduler:59 - ResultStage 349 (takeSample at KMeans.scala:376) finished in 0.008 s -2016-06-13 19:28:07 INFO DAGScheduler:59 - Job 33 finished: takeSample at KMeans.scala:376, took 0.027256 s -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 349.0, whose tasks have all completed, from pool -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=225424, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 880.0 B, free 730.4 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(487) called with curMem=226304, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 487.0 B, free 730.4 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:49684 (size: 487.0 B, free: 730.5 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 61 from broadcast at KMeans.scala:394 -2016-06-13 19:28:07 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Got job 34 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:28:07 INFO DAGScheduler:59 - Final stage: ResultStage 361(aggregate at KMeans.scala:402) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 356, ShuffleMapStage 353, ShuffleMapStage 354, ShuffleMapStage 358, ShuffleMapStage 360) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting ResultStage 361 (MapPartitionsRDD[237] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(9912) called with curMem=226791, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 9.7 KB, free 730.4 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(4245) called with curMem=236703, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 4.1 KB, free 730.4 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:49684 (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 361 (MapPartitionsRDD[237] at map at KMeans.scala:396) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Adding task set 361.0 with 2 tasks -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 361.0 (TID 108, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 361.0 (TID 109, localhost, PROCESS_LOCAL, 2271 bytes) -2016-06-13 19:28:07 INFO Executor:59 - Running task 1.0 in stage 361.0 (TID 109) -2016-06-13 19:28:07 INFO Executor:59 - Running task 0.0 in stage 361.0 (TID 108) -2016-06-13 19:28:07 INFO CacheManager:59 - Partition rdd_237_1 not found, computing it -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:28:07 INFO CacheManager:59 - Partition rdd_237_0 not found, computing it -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=240948, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block rdd_237_1 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=241444, maxMem=766075207 -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added rdd_237_1 in memory on localhost:49684 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - Block rdd_237_0 stored as values in memory (estimated size 496.0 B, free 730.4 MB) -2016-06-13 19:28:07 INFO Executor:59 - Finished task 1.0 in stage 361.0 (TID 109). 2647 bytes result sent to driver -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added rdd_237_0 in memory on localhost:49684 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 361.0 (TID 109) in 8 ms on localhost (1/2) -2016-06-13 19:28:07 INFO Executor:59 - Finished task 0.0 in stage 361.0 (TID 108). 2647 bytes result sent to driver -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 361.0 (TID 108) in 12 ms on localhost (2/2) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 361.0, whose tasks have all completed, from pool -2016-06-13 19:28:07 INFO DAGScheduler:59 - ResultStage 361 (aggregate at KMeans.scala:402) finished in 0.012 s -2016-06-13 19:28:07 INFO DAGScheduler:59 - Job 34 finished: aggregate at KMeans.scala:402, took 0.032180 s -2016-06-13 19:28:07 INFO MapPartitionsRDD:59 - Removing RDD 234 from persistence list -2016-06-13 19:28:07 INFO BlockManager:59 - Removing RDD 234 -2016-06-13 19:28:07 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Got job 35 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:28:07 INFO DAGScheduler:59 - Final stage: ResultStage 373(collect at KMeans.scala:431) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 368, ShuffleMapStage 372, ShuffleMapStage 365, ShuffleMapStage 366, ShuffleMapStage 370) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting ResultStage 373 (MapPartitionsRDD[239] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(10072) called with curMem=241940, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 9.8 KB, free 730.3 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(4365) called with curMem=252012, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:49684 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 373 (MapPartitionsRDD[239] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Adding task set 373.0 with 2 tasks -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 373.0 (TID 110, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 373.0 (TID 111, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:28:07 INFO Executor:59 - Running task 1.0 in stage 373.0 (TID 111) -2016-06-13 19:28:07 INFO Executor:59 - Running task 0.0 in stage 373.0 (TID 110) -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_237_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_237_1 locally -2016-06-13 19:28:07 INFO Executor:59 - Finished task 0.0 in stage 373.0 (TID 110). 3118 bytes result sent to driver -2016-06-13 19:28:07 INFO Executor:59 - Finished task 1.0 in stage 373.0 (TID 111). 3471 bytes result sent to driver -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 373.0 (TID 110) in 6 ms on localhost (1/2) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 373.0 (TID 111) in 5 ms on localhost (2/2) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 373.0, whose tasks have all completed, from pool -2016-06-13 19:28:07 INFO DAGScheduler:59 - ResultStage 373 (collect at KMeans.scala:431) finished in 0.007 s -2016-06-13 19:28:07 INFO DAGScheduler:59 - Job 35 finished: collect at KMeans.scala:431, took 0.027317 s -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(1648) called with curMem=256377, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 1648.0 B, free 730.3 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(600) called with curMem=258025, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 600.0 B, free 730.3 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:49684 (size: 600.0 B, free: 730.5 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 64 from broadcast at KMeans.scala:394 -2016-06-13 19:28:07 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Got job 36 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:28:07 INFO DAGScheduler:59 - Final stage: ResultStage 385(aggregate at KMeans.scala:402) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 380, ShuffleMapStage 377, ShuffleMapStage 384, ShuffleMapStage 378, ShuffleMapStage 382) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting ResultStage 385 (MapPartitionsRDD[241] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(10144) called with curMem=258625, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 9.9 KB, free 730.3 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(4338) called with curMem=268769, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.3 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:49684 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 385 (MapPartitionsRDD[241] at map at KMeans.scala:396) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Adding task set 385.0 with 2 tasks -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 385.0 (TID 112, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 385.0 (TID 113, localhost, PROCESS_LOCAL, 2303 bytes) -2016-06-13 19:28:07 INFO Executor:59 - Running task 1.0 in stage 385.0 (TID 113) -2016-06-13 19:28:07 INFO Executor:59 - Running task 0.0 in stage 385.0 (TID 112) -2016-06-13 19:28:07 INFO CacheManager:59 - Partition rdd_241_1 not found, computing it -2016-06-13 19:28:07 INFO CacheManager:59 - Partition rdd_241_0 not found, computing it -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_237_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_237_0 locally -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=273107, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block rdd_241_1 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=273603, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block rdd_241_0 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added rdd_241_1 in memory on localhost:49684 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added rdd_241_0 in memory on localhost:49684 (size: 496.0 B, free: 730.5 MB) -2016-06-13 19:28:07 INFO Executor:59 - Finished task 1.0 in stage 385.0 (TID 113). 2647 bytes result sent to driver -2016-06-13 19:28:07 INFO Executor:59 - Finished task 0.0 in stage 385.0 (TID 112). 2647 bytes result sent to driver -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 385.0 (TID 113) in 9 ms on localhost (1/2) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 385.0 (TID 112) in 11 ms on localhost (2/2) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 385.0, whose tasks have all completed, from pool -2016-06-13 19:28:07 INFO DAGScheduler:59 - ResultStage 385 (aggregate at KMeans.scala:402) finished in 0.012 s -2016-06-13 19:28:07 INFO DAGScheduler:59 - Job 36 finished: aggregate at KMeans.scala:402, took 0.029050 s -2016-06-13 19:28:07 INFO MapPartitionsRDD:59 - Removing RDD 237 from persistence list -2016-06-13 19:28:07 INFO BlockManager:59 - Removing RDD 237 -2016-06-13 19:28:07 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Got job 37 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:28:07 INFO DAGScheduler:59 - Final stage: ResultStage 397(collect at KMeans.scala:431) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 389, ShuffleMapStage 390, ShuffleMapStage 394, ShuffleMapStage 392, ShuffleMapStage 396) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting ResultStage 397 (MapPartitionsRDD[243] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(10304) called with curMem=273107, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 10.1 KB, free 730.3 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(4454) called with curMem=283411, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:49684 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 397 (MapPartitionsRDD[243] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Adding task set 397.0 with 2 tasks -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 397.0 (TID 114, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 397.0 (TID 115, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:28:07 INFO Executor:59 - Running task 0.0 in stage 397.0 (TID 114) -2016-06-13 19:28:07 INFO Executor:59 - Running task 1.0 in stage 397.0 (TID 115) -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_241_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_241_1 locally -2016-06-13 19:28:07 INFO Executor:59 - Finished task 0.0 in stage 397.0 (TID 114). 2990 bytes result sent to driver -2016-06-13 19:28:07 INFO Executor:59 - Finished task 1.0 in stage 397.0 (TID 115). 3240 bytes result sent to driver -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 397.0 (TID 114) in 5 ms on localhost (1/2) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 397.0 (TID 115) in 4 ms on localhost (2/2) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 397.0, whose tasks have all completed, from pool -2016-06-13 19:28:07 INFO DAGScheduler:59 - ResultStage 397 (collect at KMeans.scala:431) finished in 0.006 s -2016-06-13 19:28:07 INFO DAGScheduler:59 - Job 37 finished: collect at KMeans.scala:431, took 0.023071 s -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(1456) called with curMem=287865, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 1456.0 B, free 730.3 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(582) called with curMem=289321, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 582.0 B, free 730.3 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:49684 (size: 582.0 B, free: 730.4 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 67 from broadcast at KMeans.scala:394 -2016-06-13 19:28:07 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Got job 38 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:28:07 INFO DAGScheduler:59 - Final stage: ResultStage 409(aggregate at KMeans.scala:402) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 404, ShuffleMapStage 401, ShuffleMapStage 408, ShuffleMapStage 406, ShuffleMapStage 402) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting ResultStage 409 (MapPartitionsRDD[245] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(10376) called with curMem=289903, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 10.1 KB, free 730.3 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(4387) called with curMem=300279, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:49684 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 409 (MapPartitionsRDD[245] at map at KMeans.scala:396) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Adding task set 409.0 with 2 tasks -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 409.0 (TID 116, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 409.0 (TID 117, localhost, PROCESS_LOCAL, 2335 bytes) -2016-06-13 19:28:07 INFO Executor:59 - Running task 1.0 in stage 409.0 (TID 117) -2016-06-13 19:28:07 INFO Executor:59 - Running task 0.0 in stage 409.0 (TID 116) -2016-06-13 19:28:07 INFO CacheManager:59 - Partition rdd_245_1 not found, computing it -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_241_1 locally -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=304666, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block rdd_245_1 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:28:07 INFO CacheManager:59 - Partition rdd_245_0 not found, computing it -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_241_0 locally -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added rdd_245_1 in memory on localhost:49684 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=305162, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block rdd_245_0 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:28:07 INFO Executor:59 - Finished task 1.0 in stage 409.0 (TID 117). 2647 bytes result sent to driver -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added rdd_245_0 in memory on localhost:49684 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 409.0 (TID 117) in 13 ms on localhost (1/2) -2016-06-13 19:28:07 INFO Executor:59 - Finished task 0.0 in stage 409.0 (TID 116). 2647 bytes result sent to driver -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 409.0 (TID 116) in 17 ms on localhost (2/2) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 409.0, whose tasks have all completed, from pool -2016-06-13 19:28:07 INFO DAGScheduler:59 - ResultStage 409 (aggregate at KMeans.scala:402) finished in 0.018 s -2016-06-13 19:28:07 INFO DAGScheduler:59 - Job 38 finished: aggregate at KMeans.scala:402, took 0.034234 s -2016-06-13 19:28:07 INFO MapPartitionsRDD:59 - Removing RDD 241 from persistence list -2016-06-13 19:28:07 INFO BlockManager:59 - Removing RDD 241 -2016-06-13 19:28:07 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Got job 39 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:28:07 INFO DAGScheduler:59 - Final stage: ResultStage 421(collect at KMeans.scala:431) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 420, ShuffleMapStage 416, ShuffleMapStage 413, ShuffleMapStage 414, ShuffleMapStage 418) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting ResultStage 421 (MapPartitionsRDD[247] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(10536) called with curMem=304666, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 10.3 KB, free 730.3 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(4484) called with curMem=315202, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:49684 (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 69 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 421 (MapPartitionsRDD[247] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Adding task set 421.0 with 2 tasks -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 421.0 (TID 118, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 421.0 (TID 119, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:28:07 INFO Executor:59 - Running task 0.0 in stage 421.0 (TID 118) -2016-06-13 19:28:07 INFO Executor:59 - Running task 1.0 in stage 421.0 (TID 119) -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_245_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_245_0 locally -2016-06-13 19:28:07 INFO Executor:59 - Finished task 1.0 in stage 421.0 (TID 119). 2040 bytes result sent to driver -2016-06-13 19:28:07 INFO Executor:59 - Finished task 0.0 in stage 421.0 (TID 118). 2722 bytes result sent to driver -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 421.0 (TID 119) in 5 ms on localhost (1/2) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 421.0 (TID 118) in 7 ms on localhost (2/2) -2016-06-13 19:28:07 INFO DAGScheduler:59 - ResultStage 421 (collect at KMeans.scala:431) finished in 0.007 s -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 421.0, whose tasks have all completed, from pool -2016-06-13 19:28:07 INFO DAGScheduler:59 - Job 39 finished: collect at KMeans.scala:431, took 0.022825 s -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(648) called with curMem=319686, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 648.0 B, free 730.3 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(429) called with curMem=320334, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 429.0 B, free 730.3 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:49684 (size: 429.0 B, free: 730.4 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 70 from broadcast at KMeans.scala:394 -2016-06-13 19:28:07 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Got job 40 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:28:07 INFO DAGScheduler:59 - Final stage: ResultStage 433(aggregate at KMeans.scala:402) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 425, ShuffleMapStage 426, ShuffleMapStage 430, ShuffleMapStage 428, ShuffleMapStage 432) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting ResultStage 433 (MapPartitionsRDD[249] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(10608) called with curMem=320763, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 10.4 KB, free 730.3 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(4446) called with curMem=331371, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:49684 (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 71 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 433 (MapPartitionsRDD[249] at map at KMeans.scala:396) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Adding task set 433.0 with 2 tasks -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 433.0 (TID 120, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 433.0 (TID 121, localhost, PROCESS_LOCAL, 2367 bytes) -2016-06-13 19:28:07 INFO Executor:59 - Running task 0.0 in stage 433.0 (TID 120) -2016-06-13 19:28:07 INFO Executor:59 - Running task 1.0 in stage 433.0 (TID 121) -2016-06-13 19:28:07 INFO CacheManager:59 - Partition rdd_249_1 not found, computing it -2016-06-13 19:28:07 INFO CacheManager:59 - Partition rdd_249_0 not found, computing it -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_245_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_245_0 locally -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=335817, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block rdd_249_1 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=336313, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block rdd_249_0 stored as values in memory (estimated size 496.0 B, free 730.3 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added rdd_249_1 in memory on localhost:49684 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added rdd_249_0 in memory on localhost:49684 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:28:07 INFO Executor:59 - Finished task 1.0 in stage 433.0 (TID 121). 2647 bytes result sent to driver -2016-06-13 19:28:07 INFO Executor:59 - Finished task 0.0 in stage 433.0 (TID 120). 2647 bytes result sent to driver -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 433.0 (TID 121) in 8 ms on localhost (1/2) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 433.0 (TID 120) in 8 ms on localhost (2/2) -2016-06-13 19:28:07 INFO DAGScheduler:59 - ResultStage 433 (aggregate at KMeans.scala:402) finished in 0.009 s -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 433.0, whose tasks have all completed, from pool -2016-06-13 19:28:07 INFO DAGScheduler:59 - Job 40 finished: aggregate at KMeans.scala:402, took 0.027826 s -2016-06-13 19:28:07 INFO MapPartitionsRDD:59 - Removing RDD 245 from persistence list -2016-06-13 19:28:07 INFO BlockManager:59 - Removing RDD 245 -2016-06-13 19:28:07 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Got job 41 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:28:07 INFO DAGScheduler:59 - Final stage: ResultStage 445(collect at KMeans.scala:431) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 440, ShuffleMapStage 437, ShuffleMapStage 444, ShuffleMapStage 438, ShuffleMapStage 442) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting ResultStage 445 (MapPartitionsRDD[251] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(10768) called with curMem=335817, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 10.5 KB, free 730.3 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(4526) called with curMem=346585, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:49684 (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 72 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 445 (MapPartitionsRDD[251] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Adding task set 445.0 with 2 tasks -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 445.0 (TID 122, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 445.0 (TID 123, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:28:07 INFO Executor:59 - Running task 0.0 in stage 445.0 (TID 122) -2016-06-13 19:28:07 INFO Executor:59 - Running task 1.0 in stage 445.0 (TID 123) -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_249_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_249_1 locally -2016-06-13 19:28:07 INFO Executor:59 - Finished task 0.0 in stage 445.0 (TID 122). 2040 bytes result sent to driver -2016-06-13 19:28:07 INFO Executor:59 - Finished task 1.0 in stage 445.0 (TID 123). 2040 bytes result sent to driver -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 445.0 (TID 122) in 5 ms on localhost (1/2) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 445.0 (TID 123) in 6 ms on localhost (2/2) -2016-06-13 19:28:07 INFO DAGScheduler:59 - ResultStage 445 (collect at KMeans.scala:431) finished in 0.007 s -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 445.0, whose tasks have all completed, from pool -2016-06-13 19:28:07 INFO DAGScheduler:59 - Job 41 finished: collect at KMeans.scala:431, took 0.024196 s -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(584) called with curMem=351111, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 584.0 B, free 730.3 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(236) called with curMem=351695, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 236.0 B, free 730.3 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:49684 (size: 236.0 B, free: 730.4 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 73 from broadcast at KMeans.scala:394 -2016-06-13 19:28:07 INFO SparkContext:59 - Starting job: aggregate at KMeans.scala:402 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Got job 42 (aggregate at KMeans.scala:402) with 2 output partitions -2016-06-13 19:28:07 INFO DAGScheduler:59 - Final stage: ResultStage 457(aggregate at KMeans.scala:402) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 452, ShuffleMapStage 456, ShuffleMapStage 449, ShuffleMapStage 450, ShuffleMapStage 454) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting ResultStage 457 (MapPartitionsRDD[253] at map at KMeans.scala:396), which has no missing parents -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(10848) called with curMem=351931, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 10.6 KB, free 730.2 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(4495) called with curMem=362779, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.2 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:49684 (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 457 (MapPartitionsRDD[253] at map at KMeans.scala:396) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Adding task set 457.0 with 2 tasks -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 457.0 (TID 124, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 457.0 (TID 125, localhost, PROCESS_LOCAL, 2399 bytes) -2016-06-13 19:28:07 INFO Executor:59 - Running task 0.0 in stage 457.0 (TID 124) -2016-06-13 19:28:07 INFO Executor:59 - Running task 1.0 in stage 457.0 (TID 125) -2016-06-13 19:28:07 INFO CacheManager:59 - Partition rdd_253_1 not found, computing it -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_249_1 locally -2016-06-13 19:28:07 INFO CacheManager:59 - Partition rdd_253_0 not found, computing it -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=367274, maxMem=766075207 -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_249_0 locally -2016-06-13 19:28:07 INFO MemoryStore:59 - Block rdd_253_1 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added rdd_253_1 in memory on localhost:49684 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(496) called with curMem=367770, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block rdd_253_0 stored as values in memory (estimated size 496.0 B, free 730.2 MB) -2016-06-13 19:28:07 INFO Executor:59 - Finished task 1.0 in stage 457.0 (TID 125). 2647 bytes result sent to driver -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added rdd_253_0 in memory on localhost:49684 (size: 496.0 B, free: 730.4 MB) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 457.0 (TID 125) in 12 ms on localhost (1/2) -2016-06-13 19:28:07 INFO Executor:59 - Finished task 0.0 in stage 457.0 (TID 124). 2647 bytes result sent to driver -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 457.0 (TID 124) in 13 ms on localhost (2/2) -2016-06-13 19:28:07 INFO DAGScheduler:59 - ResultStage 457 (aggregate at KMeans.scala:402) finished in 0.013 s -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 457.0, whose tasks have all completed, from pool -2016-06-13 19:28:07 INFO DAGScheduler:59 - Job 42 finished: aggregate at KMeans.scala:402, took 0.027834 s -2016-06-13 19:28:07 INFO MapPartitionsRDD:59 - Removing RDD 249 from persistence list -2016-06-13 19:28:07 INFO BlockManager:59 - Removing RDD 249 -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:49684 in memory (size: 2.5 KB, free: 730.4 MB) -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:49684 in memory (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Starting job: collect at KMeans.scala:431 -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:49684 in memory (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Got job 43 (collect at KMeans.scala:431) with 2 output partitions -2016-06-13 19:28:07 INFO DAGScheduler:59 - Final stage: ResultStage 469(collect at KMeans.scala:431) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 466, ShuffleMapStage 462, ShuffleMapStage 464, ShuffleMapStage 461, ShuffleMapStage 468) -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:49684 in memory (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting ResultStage 469 (MapPartitionsRDD[255] at mapPartitionsWithIndex at KMeans.scala:423), which has no missing parents -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=324805, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 10.8 KB, free 730.3 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(4571) called with curMem=335813, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.3 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:49684 (size: 4.5 KB, free: 730.4 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 75 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 469 (MapPartitionsRDD[255] at mapPartitionsWithIndex at KMeans.scala:423) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Adding task set 469.0 with 2 tasks -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 469.0 (TID 126, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 469.0 (TID 127, localhost, PROCESS_LOCAL, 2431 bytes) -2016-06-13 19:28:07 INFO Executor:59 - Running task 0.0 in stage 469.0 (TID 126) -2016-06-13 19:28:07 INFO Executor:59 - Running task 1.0 in stage 469.0 (TID 127) -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_253_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:28:07 INFO Executor:59 - Finished task 1.0 in stage 469.0 (TID 127). 2040 bytes result sent to driver -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:49684 in memory (size: 4.4 KB, free: 730.4 MB) -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_253_0 locally -2016-06-13 19:28:07 INFO Executor:59 - Finished task 0.0 in stage 469.0 (TID 126). 2040 bytes result sent to driver -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 469.0 (TID 127) in 11 ms on localhost (1/2) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 469.0 (TID 126) in 12 ms on localhost (2/2) -2016-06-13 19:28:07 INFO DAGScheduler:59 - ResultStage 469 (collect at KMeans.scala:431) finished in 0.014 s -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 469.0, whose tasks have all completed, from pool -2016-06-13 19:28:07 INFO DAGScheduler:59 - Job 43 finished: collect at KMeans.scala:431, took 0.042242 s -2016-06-13 19:28:07 INFO MapPartitionsRDD:59 - Removing RDD 253 from persistence list -2016-06-13 19:28:07 INFO BlockManager:59 - Removing RDD 253 -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(2880) called with curMem=314756, maxMem=766075207 -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 2.8 KB, free 730.3 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(708) called with curMem=306268, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 708.0 B, free 730.3 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:49684 (size: 708.0 B, free: 730.4 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 76 from broadcast at KMeans.scala:445 -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:49684 in memory (size: 4.3 KB, free: 730.4 MB) -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:49684 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:49684 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-06-13 19:28:07 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:450 -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:49684 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Registering RDD 256 (flatMap at KMeans.scala:446) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Got job 44 (collectAsMap at KMeans.scala:450) with 2 output partitions -2016-06-13 19:28:07 INFO DAGScheduler:59 - Final stage: ResultStage 482(collectAsMap at KMeans.scala:450) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 481) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 481) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:49684 in memory (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 481 (MapPartitionsRDD[256] at flatMap at KMeans.scala:446), which has no missing parents -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(9392) called with curMem=244755, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 9.2 KB, free 730.3 MB) -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(4071) called with curMem=254147, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:49684 (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 77 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 481 (MapPartitionsRDD[256] at flatMap at KMeans.scala:446) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Adding task set 481.0 with 2 tasks -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:49684 in memory (size: 9.9 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 481.0 (TID 128, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 481.0 (TID 129, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:28:07 INFO Executor:59 - Running task 1.0 in stage 481.0 (TID 129) -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:28:07 INFO Executor:59 - Running task 0.0 in stage 481.0 (TID 128) -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:49684 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO Executor:59 - Finished task 1.0 in stage 481.0 (TID 129). 2254 bytes result sent to driver -2016-06-13 19:28:07 INFO Executor:59 - Finished task 0.0 in stage 481.0 (TID 128). 2254 bytes result sent to driver -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 481.0 (TID 129) in 21 ms on localhost (1/2) -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 481.0 (TID 128) in 24 ms on localhost (2/2) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 481.0, whose tasks have all completed, from pool -2016-06-13 19:28:07 INFO DAGScheduler:59 - ShuffleMapStage 481 (flatMap at KMeans.scala:446) finished in 0.025 s -2016-06-13 19:28:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:07 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:28:07 INFO DAGScheduler:59 - waiting: Set(ResultStage 482) -2016-06-13 19:28:07 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:07 INFO DAGScheduler:59 - Missing parents for ResultStage 482: List() -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting ResultStage 482 (ShuffledRDD[257] at reduceByKey at KMeans.scala:450), which is now runnable -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(2312) called with curMem=200509, maxMem=766075207 -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:49684 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(1382) called with curMem=202821, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 1382.0 B, free 730.4 MB) -2016-06-13 19:28:07 INFO BlockManager:59 - Removing RDD 217 -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:49684 (size: 1382.0 B, free: 730.5 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned RDD 217 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 482 (ShuffledRDD[257] at reduceByKey at KMeans.scala:450) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Adding task set 482.0 with 2 tasks -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 482.0 (TID 130, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 482.0 (TID 131, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:28:07 INFO Executor:59 - Running task 0.0 in stage 482.0 (TID 130) -2016-06-13 19:28:07 INFO Executor:59 - Running task 1.0 in stage 482.0 (TID 131) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:49684 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-06-13 19:28:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:49684 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO Executor:59 - Finished task 0.0 in stage 482.0 (TID 130). 1852 bytes result sent to driver -2016-06-13 19:28:07 INFO Executor:59 - Finished task 1.0 in stage 482.0 (TID 131). 1744 bytes result sent to driver -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 482.0 (TID 130) in 9 ms on localhost (1/2) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 482.0 (TID 131) in 8 ms on localhost (2/2) -2016-06-13 19:28:07 INFO DAGScheduler:59 - ResultStage 482 (collectAsMap at KMeans.scala:450) finished in 0.009 s -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 482.0, whose tasks have all completed, from pool -2016-06-13 19:28:07 INFO DAGScheduler:59 - Job 44 finished: collectAsMap at KMeans.scala:450, took 0.067844 s -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:49684 in memory (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:28:07 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:28:07 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-06-13 19:28:07 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:28:07 INFO LocalKMeans:59 - Local KMeans++ converged in 2 iterations. -2016-06-13 19:28:07 INFO KMeans:59 - Initialization with k-means|| took 0.655 seconds. -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(824) called with curMem=163937, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 824.0 B, free 730.4 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:49684 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(339) called with curMem=164761, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 339.0 B, free 730.4 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:49684 (size: 339.0 B, free: 730.5 MB) -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 79 from broadcast at KMeans.scala:276 -2016-06-13 19:28:07 INFO BlockManager:59 - Removing RDD 194 -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned RDD 194 -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:49684 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:49684 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-06-13 19:28:07 INFO SparkContext:59 - Starting job: collectAsMap at KMeans.scala:302 -2016-06-13 19:28:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:49684 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-06-13 19:28:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:28:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 151 bytes -2016-06-13 19:28:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:49684 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 151 bytes -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-06-13 19:28:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:28:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 151 bytes -2016-06-13 19:28:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:28:07 INFO BlockManager:59 - Removing RDD 171 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Registering RDD 258 (mapPartitions at KMeans.scala:279) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Got job 45 (collectAsMap at KMeans.scala:302) with 2 output partitions -2016-06-13 19:28:07 INFO DAGScheduler:59 - Final stage: ResultStage 495(collectAsMap at KMeans.scala:302) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 494) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 494) -2016-06-13 19:28:07 INFO ContextCleaner:59 - Cleaned RDD 171 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting ShuffleMapStage 494 (MapPartitionsRDD[258] at mapPartitions at KMeans.scala:279), which has no missing parents -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:49684 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(10016) called with curMem=115341, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 9.8 KB, free 730.5 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(4409) called with curMem=125357, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.5 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:49684 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 80 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 494 (MapPartitionsRDD[258] at mapPartitions at KMeans.scala:279) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Adding task set 494.0 with 2 tasks -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 494.0 (TID 132, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 494.0 (TID 133, localhost, PROCESS_LOCAL, 2228 bytes) -2016-06-13 19:28:07 INFO Executor:59 - Running task 0.0 in stage 494.0 (TID 132) -2016-06-13 19:28:07 INFO Executor:59 - Running task 1.0 in stage 494.0 (TID 133) -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_0 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_231_1 locally -2016-06-13 19:28:07 INFO Executor:59 - Finished task 1.0 in stage 494.0 (TID 133). 2413 bytes result sent to driver -2016-06-13 19:28:07 INFO Executor:59 - Finished task 0.0 in stage 494.0 (TID 132). 2413 bytes result sent to driver -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 494.0 (TID 133) in 14 ms on localhost (1/2) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 494.0 (TID 132) in 14 ms on localhost (2/2) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 494.0, whose tasks have all completed, from pool -2016-06-13 19:28:07 INFO DAGScheduler:59 - ShuffleMapStage 494 (mapPartitions at KMeans.scala:279) finished in 0.015 s -2016-06-13 19:28:07 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:28:07 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:28:07 INFO DAGScheduler:59 - waiting: Set(ResultStage 495) -2016-06-13 19:28:07 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:28:07 INFO DAGScheduler:59 - Missing parents for ResultStage 495: List() -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting ResultStage 495 (ShuffledRDD[259] at reduceByKey at KMeans.scala:302), which is now runnable -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(2808) called with curMem=129766, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 2.7 KB, free 730.5 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(1589) called with curMem=132574, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 1589.0 B, free 730.5 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:49684 (size: 1589.0 B, free: 730.5 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 81 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 495 (ShuffledRDD[259] at reduceByKey at KMeans.scala:302) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Adding task set 495.0 with 2 tasks -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 495.0 (TID 134, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 495.0 (TID 135, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:28:07 INFO Executor:59 - Running task 0.0 in stage 495.0 (TID 134) -2016-06-13 19:28:07 INFO Executor:59 - Running task 1.0 in stage 495.0 (TID 135) -2016-06-13 19:28:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:28:07 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:28:07 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:28:07 INFO Executor:59 - Finished task 0.0 in stage 495.0 (TID 134). 1785 bytes result sent to driver -2016-06-13 19:28:07 INFO Executor:59 - Finished task 1.0 in stage 495.0 (TID 135). 1671 bytes result sent to driver -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 495.0 (TID 134) in 7 ms on localhost (1/2) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 495.0 (TID 135) in 7 ms on localhost (2/2) -2016-06-13 19:28:07 INFO DAGScheduler:59 - ResultStage 495 (collectAsMap at KMeans.scala:302) finished in 0.008 s -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 495.0, whose tasks have all completed, from pool -2016-06-13 19:28:07 INFO DAGScheduler:59 - Job 45 finished: collectAsMap at KMeans.scala:302, took 0.047887 s -2016-06-13 19:28:07 INFO KMeans:59 - Run 0 finished in 1 iterations -2016-06-13 19:28:07 INFO KMeans:59 - Run 1 finished in 1 iterations -2016-06-13 19:28:07 INFO KMeans:59 - Run 2 finished in 1 iterations -2016-06-13 19:28:07 INFO KMeans:59 - Run 3 finished in 1 iterations -2016-06-13 19:28:07 INFO KMeans:59 - Run 4 finished in 1 iterations -2016-06-13 19:28:07 INFO KMeans:59 - Iterations took 0.074 seconds. -2016-06-13 19:28:07 INFO KMeans:59 - KMeans converged in 1 iterations. -2016-06-13 19:28:07 INFO KMeans:59 - The cost for the best run is 6.650150218788335E-6. -2016-06-13 19:28:07 INFO MapPartitionsRDD:59 - Removing RDD 231 from persistence list -2016-06-13 19:28:07 INFO BlockManager:59 - Removing RDD 231 -2016-06-13 19:28:07 WARN KMeans:71 - The input data was not directly cached, which may hurt performance if its parent RDDs are also uncached. -2016-06-13 19:28:07 INFO SparkContext:59 - Starting job: collect at PowerIterationClusteringSuite.scala:71 -2016-06-13 19:28:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:28:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:28:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:28:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 151 bytes -2016-06-13 19:28:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:28:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 151 bytes -2016-06-13 19:28:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:28:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 151 bytes -2016-06-13 19:28:07 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:28:07 INFO DAGScheduler:59 - Got job 46 (collect at PowerIterationClusteringSuite.scala:71) with 2 output partitions -2016-06-13 19:28:07 INFO DAGScheduler:59 - Final stage: ResultStage 507(collect at PowerIterationClusteringSuite.scala:71) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 502, ShuffleMapStage 506, ShuffleMapStage 499, ShuffleMapStage 500, ShuffleMapStage 504) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting ResultStage 507 (MapPartitionsRDD[262] at mapPartitions at PowerIterationClustering.scala:232), which has no missing parents -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(8192) called with curMem=133683, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 8.0 KB, free 730.5 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(3573) called with curMem=141875, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:49684 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 507 (MapPartitionsRDD[262] at mapPartitions at PowerIterationClustering.scala:232) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Adding task set 507.0 with 2 tasks -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 507.0 (TID 136, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 507.0 (TID 137, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:28:07 INFO Executor:59 - Running task 1.0 in stage 507.0 (TID 137) -2016-06-13 19:28:07 INFO Executor:59 - Running task 0.0 in stage 507.0 (TID 136) -2016-06-13 19:28:07 INFO CacheManager:59 - Partition rdd_260_1 not found, computing it -2016-06-13 19:28:07 INFO CacheManager:59 - Partition rdd_260_0 not found, computing it -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_228_0 locally -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=145448, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block rdd_260_0 stored as values in memory (estimated size 1168.0 B, free 730.4 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(1168) called with curMem=146616, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block rdd_260_1 stored as values in memory (estimated size 1168.0 B, free 730.4 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added rdd_260_0 in memory on localhost:49684 (size: 1168.0 B, free: 730.5 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added rdd_260_1 in memory on localhost:49684 (size: 1168.0 B, free: 730.5 MB) -2016-06-13 19:28:07 INFO Executor:59 - Finished task 0.0 in stage 507.0 (TID 136). 2919 bytes result sent to driver -2016-06-13 19:28:07 INFO Executor:59 - Finished task 1.0 in stage 507.0 (TID 137). 2919 bytes result sent to driver -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 507.0 (TID 136) in 8 ms on localhost (1/2) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 507.0 (TID 137) in 7 ms on localhost (2/2) -2016-06-13 19:28:07 INFO DAGScheduler:59 - ResultStage 507 (collect at PowerIterationClusteringSuite.scala:71) finished in 0.010 s -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 507.0, whose tasks have all completed, from pool -2016-06-13 19:28:07 INFO DAGScheduler:59 - Job 46 finished: collect at PowerIterationClusteringSuite.scala:71, took 0.026403 s -2016-06-13 19:28:07 INFO SparkContext:59 - Starting job: foreach at PowerIterationClusteringSuite.scala:75 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Got job 47 (foreach at PowerIterationClusteringSuite.scala:75) with 2 output partitions -2016-06-13 19:28:07 INFO DAGScheduler:59 - Final stage: ResultStage 519(foreach at PowerIterationClusteringSuite.scala:75) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 514, ShuffleMapStage 511, ShuffleMapStage 518, ShuffleMapStage 512, ShuffleMapStage 516) -2016-06-13 19:28:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting ResultStage 519 (MapPartitionsRDD[262] at mapPartitions at PowerIterationClustering.scala:232), which has no missing parents -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(8224) called with curMem=147784, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 8.0 KB, free 730.4 MB) -2016-06-13 19:28:07 INFO MemoryStore:59 - ensureFreeSpace(3588) called with curMem=156008, maxMem=766075207 -2016-06-13 19:28:07 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:28:07 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:49684 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:28:07 INFO SparkContext:59 - Created broadcast 83 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 519 (MapPartitionsRDD[262] at mapPartitions at PowerIterationClustering.scala:232) -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Adding task set 519.0 with 2 tasks -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 0.0 in stage 519.0 (TID 138, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Starting task 1.0 in stage 519.0 (TID 139, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:28:07 INFO Executor:59 - Running task 1.0 in stage 519.0 (TID 139) -2016-06-13 19:28:07 INFO Executor:59 - Running task 0.0 in stage 519.0 (TID 138) -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_260_1 locally -2016-06-13 19:28:07 INFO BlockManager:59 - Found block rdd_260_0 locally -2016-06-13 19:28:07 INFO Executor:59 - Finished task 1.0 in stage 519.0 (TID 139). 2044 bytes result sent to driver -2016-06-13 19:28:07 INFO Executor:59 - Finished task 0.0 in stage 519.0 (TID 138). 2044 bytes result sent to driver -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 1.0 in stage 519.0 (TID 139) in 4 ms on localhost (1/2) -2016-06-13 19:28:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 519.0 (TID 138) in 5 ms on localhost (2/2) -2016-06-13 19:28:07 INFO DAGScheduler:59 - ResultStage 519 (foreach at PowerIterationClusteringSuite.scala:75) finished in 0.005 s -2016-06-13 19:28:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 519.0, whose tasks have all completed, from pool -2016-06-13 19:28:07 INFO DAGScheduler:59 - Job 47 finished: foreach at PowerIterationClusteringSuite.scala:75, took 0.019313 s -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 19:28:07 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 19:28:07 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 19:28:07 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 19:28:07 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 19:28:08 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 19:28:08 INFO BlockManager:59 - BlockManager stopped -2016-06-13 19:28:08 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 19:28:08 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 19:28:08 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 19:28:08 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 19:28:08 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 19:28:08 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 19:28:08 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-81d0df0c-a0cf-43be-b5ce-aeef302e25c2 -2016-06-13 19:38:34 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 19:38:35 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 19:38:35 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 19:38:35 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 19:38:35 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 19:38:36 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 19:38:36 INFO Remoting:74 - Starting remoting -2016-06-13 19:38:36 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:49827] -2016-06-13 19:38:36 INFO Utils:59 - Successfully started service 'sparkDriver' on port 49827. -2016-06-13 19:38:36 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 19:38:36 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 19:38:36 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-179e7163-dffc-4186-bc1a-eee0f6ae78a1 -2016-06-13 19:38:36 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 19:38:36 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-7b4dee71-2128-43dd-8d5f-4ebe4fb25ea1\httpd-b38877a5-bb65-4991-ac1a-9048e798bd1c -2016-06-13 19:38:36 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 19:38:36 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:38:36 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:49828 -2016-06-13 19:38:36 INFO Utils:59 - Successfully started service 'HTTP file server' on port 49828. -2016-06-13 19:38:36 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 19:38:36 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:38:36 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 19:38:36 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 19:38:36 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 19:38:37 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 19:38:37 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 19:38:37 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 49848. -2016-06-13 19:38:37 INFO NettyBlockTransferService:59 - Server created on 49848 -2016-06-13 19:38:37 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 19:38:37 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:49848 with 730.6 MB RAM, BlockManagerId(driver, localhost, 49848) -2016-06-13 19:38:37 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 19:38:38 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 19:38:38 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 19:38:38 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 19:38:38 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 19:38:38 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:49848 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 19:38:38 INFO SparkContext:59 - Created broadcast 0 from textFile at LDAFromWebSuite.scala:22 -2016-06-13 19:38:39 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 19:38:39 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 19:38:39 INFO SparkContext:59 - Starting job: zipWithIndex at LDAFromWebSuite.scala:25 -2016-06-13 19:38:39 INFO DAGScheduler:59 - Got job 0 (zipWithIndex at LDAFromWebSuite.scala:25) with 1 output partitions -2016-06-13 19:38:39 INFO DAGScheduler:59 - Final stage: ResultStage 0(zipWithIndex at LDAFromWebSuite.scala:25) -2016-06-13 19:38:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:38:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:38:39 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at LDAFromWebSuite.scala:23), which has no missing parents -2016-06-13 19:38:39 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=171580, maxMem=766075207 -2016-06-13 19:38:39 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:38:39 INFO MemoryStore:59 - ensureFreeSpace(1939) called with curMem=174820, maxMem=766075207 -2016-06-13 19:38:39 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1939.0 B, free 730.4 MB) -2016-06-13 19:38:39 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:49848 (size: 1939.0 B, free: 730.6 MB) -2016-06-13 19:38:39 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:39 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at LDAFromWebSuite.scala:23) -2016-06-13 19:38:39 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 19:38:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2197 bytes) -2016-06-13 19:38:39 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 19:38:39 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:0+132 -2016-06-13 19:38:39 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 19:38:39 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 19:38:39 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 19:38:39 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 19:38:39 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 19:38:40 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 19:38:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 111 ms on localhost (1/1) -2016-06-13 19:38:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 19:38:40 INFO DAGScheduler:59 - ResultStage 0 (zipWithIndex at LDAFromWebSuite.scala:25) finished in 0.129 s -2016-06-13 19:38:40 INFO DAGScheduler:59 - Job 0 finished: zipWithIndex at LDAFromWebSuite.scala:25, took 0.295084 s -2016-06-13 19:38:40 INFO SparkContext:59 - Starting job: take at LDAOptimizer.scala:142 -2016-06-13 19:38:40 INFO DAGScheduler:59 - Got job 1 (take at LDAOptimizer.scala:142) with 1 output partitions -2016-06-13 19:38:40 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at LDAOptimizer.scala:142) -2016-06-13 19:38:40 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:38:40 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:38:40 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at map at LDAFromWebSuite.scala:25), which has no missing parents -2016-06-13 19:38:40 INFO MemoryStore:59 - ensureFreeSpace(3696) called with curMem=176759, maxMem=766075207 -2016-06-13 19:38:40 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-06-13 19:38:40 INFO MemoryStore:59 - ensureFreeSpace(2071) called with curMem=180455, maxMem=766075207 -2016-06-13 19:38:40 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:38:40 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:49848 (size: 2.0 KB, free: 730.6 MB) -2016-06-13 19:38:40 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:40 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at map at LDAFromWebSuite.scala:25) -2016-06-13 19:38:40 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 19:38:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2307 bytes) -2016-06-13 19:38:40 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 19:38:40 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-06-13 19:38:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:0+132 -2016-06-13 19:38:40 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=182526, maxMem=766075207 -2016-06-13 19:38:40 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 1224.0 B, free 730.4 MB) -2016-06-13 19:38:40 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:49848 (size: 1224.0 B, free: 730.6 MB) -2016-06-13 19:38:40 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2949 bytes result sent to driver -2016-06-13 19:38:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 51 ms on localhost (1/1) -2016-06-13 19:38:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 19:38:40 INFO DAGScheduler:59 - ResultStage 1 (take at LDAOptimizer.scala:142) finished in 0.056 s -2016-06-13 19:38:40 INFO DAGScheduler:59 - Job 1 finished: take at LDAOptimizer.scala:142, took 0.085003 s -2016-06-13 19:38:40 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:40 INFO DAGScheduler:59 - Registering RDD 19 (map at GraphImpl.scala:104) -2016-06-13 19:38:40 INFO DAGScheduler:59 - Registering RDD 23 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:38:40 INFO DAGScheduler:59 - Registering RDD 11 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:38:40 INFO DAGScheduler:59 - Registering RDD 6 (mapPartitionsWithIndex at LDAOptimizer.scala:128) -2016-06-13 19:38:40 INFO DAGScheduler:59 - Got job 2 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:40 INFO DAGScheduler:59 - Final stage: ResultStage 6(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 5, ShuffleMapStage 3, ShuffleMapStage 4) -2016-06-13 19:38:40 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 5, ShuffleMapStage 3, ShuffleMapStage 4) -2016-06-13 19:38:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[19] at map at GraphImpl.scala:104), which has no missing parents -2016-06-13 19:38:40 INFO MemoryStore:59 - ensureFreeSpace(5752) called with curMem=183750, maxMem=766075207 -2016-06-13 19:38:40 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.6 KB, free 730.4 MB) -2016-06-13 19:38:40 INFO MemoryStore:59 - ensureFreeSpace(3053) called with curMem=189502, maxMem=766075207 -2016-06-13 19:38:40 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 19:38:40 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:49848 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 19:38:40 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[19] at map at GraphImpl.scala:104) -2016-06-13 19:38:40 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 19:38:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[11] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:38:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:38:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:38:40 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-06-13 19:38:40 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=192555, maxMem=766075207 -2016-06-13 19:38:40 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 3) -2016-06-13 19:38:40 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.4 MB) -2016-06-13 19:38:40 INFO MemoryStore:59 - ensureFreeSpace(2762) called with curMem=197619, maxMem=766075207 -2016-06-13 19:38:40 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:38:40 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:49848 (size: 2.7 KB, free: 730.6 MB) -2016-06-13 19:38:40 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[11] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:38:40 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 19:38:40 INFO CacheManager:59 - Partition rdd_17_1 not found, computing it -2016-06-13 19:38:40 INFO CacheManager:59 - Partition rdd_17_0 not found, computing it -2016-06-13 19:38:40 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:38:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 5 (MapPartitionsRDD[6] at mapPartitionsWithIndex at LDAOptimizer.scala:128), which has no missing parents -2016-06-13 19:38:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 4, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:38:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 5, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:38:40 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 4) -2016-06-13 19:38:40 INFO MemoryStore:59 - ensureFreeSpace(4880) called with curMem=200381, maxMem=766075207 -2016-06-13 19:38:40 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-06-13 19:38:40 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:38:40 INFO MemoryStore:59 - ensureFreeSpace(2642) called with curMem=205261, maxMem=766075207 -2016-06-13 19:38:40 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-06-13 19:38:40 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 5) -2016-06-13 19:38:40 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-06-13 19:38:40 INFO MemoryStore:59 - ensureFreeSpace(3520) called with curMem=207903, maxMem=766075207 -2016-06-13 19:38:40 INFO MemoryStore:59 - Block rdd_17_0 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-06-13 19:38:40 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:49848 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:38:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:132+132 -2016-06-13 19:38:40 INFO BlockManagerInfo:59 - Added rdd_17_0 in memory on localhost:49848 (size: 3.4 KB, free: 730.6 MB) -2016-06-13 19:38:40 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 5 (MapPartitionsRDD[6] at mapPartitionsWithIndex at LDAOptimizer.scala:128) -2016-06-13 19:38:40 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 19:38:40 INFO CacheManager:59 - Another thread is loading rdd_4_1, waiting for it to finish... -2016-06-13 19:38:40 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=211423, maxMem=766075207 -2016-06-13 19:38:40 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 880.0 B, free 730.4 MB) -2016-06-13 19:38:40 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:49848 (size: 880.0 B, free: 730.6 MB) -2016-06-13 19:38:40 INFO MemoryStore:59 - ensureFreeSpace(3296) called with curMem=212303, maxMem=766075207 -2016-06-13 19:38:41 INFO CacheManager:59 - Finished waiting for rdd_4_1 -2016-06-13 19:38:41 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 19:38:41 INFO MemoryStore:59 - Block rdd_17_1 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Added rdd_17_1 in memory on localhost:49848 (size: 3.2 KB, free: 730.6 MB) -2016-06-13 19:38:41 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 4). 2254 bytes result sent to driver -2016-06-13 19:38:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 6, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 4) in 136 ms on localhost (1/2) -2016-06-13 19:38:41 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 3). 2889 bytes result sent to driver -2016-06-13 19:38:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 7, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:38:41 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 7) -2016-06-13 19:38:41 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 6) -2016-06-13 19:38:41 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 2834 bytes result sent to driver -2016-06-13 19:38:41 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 5). 1159 bytes result sent to driver -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 5) in 165 ms on localhost (2/2) -2016-06-13 19:38:41 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at VertexRDD.scala:358) finished in 0.168 s -2016-06-13 19:38:41 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:41 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 5, ShuffleMapStage 2) -2016-06-13 19:38:41 INFO DAGScheduler:59 - waiting: Set(ResultStage 6, ShuffleMapStage 3) -2016-06-13 19:38:41 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:41 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 19:38:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 206 ms on localhost (1/2) -2016-06-13 19:38:41 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:38:41 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List(ShuffleMapStage 5, ShuffleMapStage 3) -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 3) in 207 ms on localhost (2/2) -2016-06-13 19:38:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 19:38:41 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List(ShuffleMapStage 2) -2016-06-13 19:38:41 INFO DAGScheduler:59 - ShuffleMapStage 2 (map at GraphImpl.scala:104) finished in 0.221 s -2016-06-13 19:38:41 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:41 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 5) -2016-06-13 19:38:41 INFO DAGScheduler:59 - waiting: Set(ResultStage 6, ShuffleMapStage 3) -2016-06-13 19:38:41 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:41 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List(ShuffleMapStage 5, ShuffleMapStage 3) -2016-06-13 19:38:41 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List() -2016-06-13 19:38:41 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[23] at mapPartitions at VertexRDD.scala:358), which is now runnable -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(2784) called with curMem=215599, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(1698) called with curMem=218383, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1698.0 B, free 730.4 MB) -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:49848 (size: 1698.0 B, free: 730.6 MB) -2016-06-13 19:38:41 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 3 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[23] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:38:41 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 19:38:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 1890 bytes) -2016-06-13 19:38:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 1890 bytes) -2016-06-13 19:38:41 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 9) -2016-06-13 19:38:41 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 8) -2016-06-13 19:38:41 INFO CacheManager:59 - Partition rdd_21_1 not found, computing it -2016-06-13 19:38:41 INFO CacheManager:59 - Partition rdd_21_0 not found, computing it -2016-06-13 19:38:41 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 19:38:41 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 10 ms -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 10 ms -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(3416) called with curMem=220081, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block rdd_21_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Added rdd_21_1 in memory on localhost:49848 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(3408) called with curMem=223497, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block rdd_21_0 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Added rdd_21_0 in memory on localhost:49848 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 6). 2254 bytes result sent to driver -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 6) in 226 ms on localhost (1/2) -2016-06-13 19:38:41 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 9). 1955 bytes result sent to driver -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 9) in 137 ms on localhost (1/2) -2016-06-13 19:38:41 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 8). 1955 bytes result sent to driver -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 8) in 168 ms on localhost (2/2) -2016-06-13 19:38:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 19:38:41 INFO DAGScheduler:59 - ShuffleMapStage 3 (mapPartitions at VertexRDD.scala:358) finished in 0.169 s -2016-06-13 19:38:41 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 7). 2254 bytes result sent to driver -2016-06-13 19:38:41 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:41 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 5) -2016-06-13 19:38:41 INFO DAGScheduler:59 - waiting: Set(ResultStage 6) -2016-06-13 19:38:41 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 7) in 246 ms on localhost (2/2) -2016-06-13 19:38:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 19:38:41 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List(ShuffleMapStage 5) -2016-06-13 19:38:41 INFO DAGScheduler:59 - ShuffleMapStage 5 (mapPartitionsWithIndex at LDAOptimizer.scala:128) finished in 0.335 s -2016-06-13 19:38:41 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:41 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:41 INFO DAGScheduler:59 - waiting: Set(ResultStage 6) -2016-06-13 19:38:41 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:41 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List() -2016-06-13 19:38:41 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[30] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(5584) called with curMem=226905, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(2802) called with curMem=232489, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:49848 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[30] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:41 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-06-13 19:38:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 10, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:38:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:38:41 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 11) -2016-06-13 19:38:41 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 10) -2016-06-13 19:38:41 INFO CacheManager:59 - Partition rdd_26_1 not found, computing it -2016-06-13 19:38:41 INFO CacheManager:59 - Partition rdd_14_1 not found, computing it -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:41 INFO CacheManager:59 - Partition rdd_26_0 not found, computing it -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:41 INFO CacheManager:59 - Partition rdd_14_0 not found, computing it -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(2400) called with curMem=235291, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block rdd_14_1 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Added rdd_14_1 in memory on localhost:49848 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(2496) called with curMem=237691, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block rdd_14_0 stored as values in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Added rdd_14_0 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(2400) called with curMem=240187, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block rdd_26_1 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Added rdd_26_1 in memory on localhost:49848 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(2496) called with curMem=242587, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block rdd_26_0 stored as values in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Added rdd_26_0 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 11). 1931 bytes result sent to driver -2016-06-13 19:38:41 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 10). 1931 bytes result sent to driver -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 11) in 88 ms on localhost (1/2) -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 10) in 93 ms on localhost (2/2) -2016-06-13 19:38:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 19:38:41 INFO DAGScheduler:59 - ResultStage 6 (fold at LDAOptimizer.scala:205) finished in 0.094 s -2016-06-13 19:38:41 INFO DAGScheduler:59 - Job 2 finished: fold at LDAOptimizer.scala:205, took 0.609835 s -2016-06-13 19:38:41 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:41 INFO DAGScheduler:59 - Registering RDD 31 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:41 INFO DAGScheduler:59 - Registering RDD 35 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:41 INFO DAGScheduler:59 - Got job 3 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:41 INFO DAGScheduler:59 - Final stage: ResultStage 13(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12, ShuffleMapStage 10, ShuffleMapStage 7, ShuffleMapStage 8) -2016-06-13 19:38:41 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-06-13 19:38:41 INFO DAGScheduler:59 - Submitting ShuffleMapStage 11 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[31] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(4360) called with curMem=236697, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:49848 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(2388) called with curMem=241057, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:49848 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 11 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[31] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:41 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 19:38:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 12, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:38:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 13, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:38:41 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 13) -2016-06-13 19:38:41 INFO BlockManager:59 - Found block rdd_26_1 locally -2016-06-13 19:38:41 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 12) -2016-06-13 19:38:41 INFO BlockManager:59 - Found block rdd_26_0 locally -2016-06-13 19:38:41 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 12). 2254 bytes result sent to driver -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 12) in 47 ms on localhost (1/2) -2016-06-13 19:38:41 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 13). 2254 bytes result sent to driver -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 13) in 52 ms on localhost (2/2) -2016-06-13 19:38:41 INFO DAGScheduler:59 - ShuffleMapStage 11 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.055 s -2016-06-13 19:38:41 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:41 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:41 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 12, ResultStage 13) -2016-06-13 19:38:41 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 19:38:41 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 12: List() -2016-06-13 19:38:41 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 19:38:41 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List(ShuffleMapStage 12) -2016-06-13 19:38:41 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[35] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=243445, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=245643, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:49848 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[35] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:41 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:49848 in memory (size: 1698.0 B, free: 730.5 MB) -2016-06-13 19:38:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 14, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 15, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:41 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 19:38:41 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 14) -2016-06-13 19:38:41 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 15) -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:49848 in memory (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:41 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:41 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:49848 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:49848 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 19:38:41 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 14). 2470 bytes result sent to driver -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 14) in 57 ms on localhost (1/2) -2016-06-13 19:38:41 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 15). 2470 bytes result sent to driver -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 15) in 63 ms on localhost (2/2) -2016-06-13 19:38:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 19:38:41 INFO DAGScheduler:59 - ShuffleMapStage 12 (mapPartitions at GraphImpl.scala:235) finished in 0.066 s -2016-06-13 19:38:41 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:41 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:41 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-06-13 19:38:41 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:49848 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-06-13 19:38:41 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[43] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:41 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(6432) called with curMem=215225, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 6.3 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(3040) called with curMem=219718, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:49848 in memory (size: 1939.0 B, free: 730.5 MB) -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:49848 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[43] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:41 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 19:38:41 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 19:38:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 16, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:38:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:38:41 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 17) -2016-06-13 19:38:41 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 16) -2016-06-13 19:38:41 INFO CacheManager:59 - Partition rdd_39_1 not found, computing it -2016-06-13 19:38:41 INFO BlockManager:59 - Found block rdd_26_1 locally -2016-06-13 19:38:41 INFO CacheManager:59 - Partition rdd_39_0 not found, computing it -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:41 INFO BlockManager:59 - Found block rdd_26_0 locally -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=219518, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block rdd_39_1 stored as values in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=221966, maxMem=766075207 -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Added rdd_39_1 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO MemoryStore:59 - Block rdd_39_0 stored as values in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Added rdd_39_0 in memory on localhost:49848 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 17). 2971 bytes result sent to driver -2016-06-13 19:38:41 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 16). 2971 bytes result sent to driver -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 17) in 27 ms on localhost (1/2) -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 16) in 30 ms on localhost (2/2) -2016-06-13 19:38:41 INFO DAGScheduler:59 - ResultStage 13 (fold at LDAOptimizer.scala:205) finished in 0.030 s -2016-06-13 19:38:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 19:38:41 INFO DAGScheduler:59 - Job 3 finished: fold at LDAOptimizer.scala:205, took 0.279555 s -2016-06-13 19:38:41 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:38:41 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:38:41 INFO DAGScheduler:59 - Registering RDD 44 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:41 INFO DAGScheduler:59 - Registering RDD 48 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:41 INFO DAGScheduler:59 - Got job 4 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:41 INFO DAGScheduler:59 - Final stage: ResultStage 22(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:41 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15, ShuffleMapStage 19, ShuffleMapStage 17, ShuffleMapStage 21, ShuffleMapStage 14) -2016-06-13 19:38:41 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 21) -2016-06-13 19:38:41 INFO DAGScheduler:59 - Submitting ShuffleMapStage 20 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[44] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=224510, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.3 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(2727) called with curMem=229958, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:49848 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 20 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[44] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:41 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-06-13 19:38:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 18, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:38:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 19, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:38:41 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 19) -2016-06-13 19:38:41 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 18) -2016-06-13 19:38:41 INFO BlockManager:59 - Found block rdd_39_0 locally -2016-06-13 19:38:41 INFO BlockManager:59 - Found block rdd_39_1 locally -2016-06-13 19:38:41 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 18). 2254 bytes result sent to driver -2016-06-13 19:38:41 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 19). 2254 bytes result sent to driver -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 19) in 33 ms on localhost (1/2) -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 18) in 34 ms on localhost (2/2) -2016-06-13 19:38:41 INFO DAGScheduler:59 - ShuffleMapStage 20 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.036 s -2016-06-13 19:38:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 19:38:41 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:41 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:41 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 21, ResultStage 22) -2016-06-13 19:38:41 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:41 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 21: List() -2016-06-13 19:38:41 INFO DAGScheduler:59 - Missing parents for ResultStage 22: List(ShuffleMapStage 21) -2016-06-13 19:38:41 INFO DAGScheduler:59 - Submitting ShuffleMapStage 21 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[48] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=232685, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=236581, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:49848 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 21 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[48] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:41 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 19:38:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 20, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 21, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:41 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 20) -2016-06-13 19:38:41 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 21) -2016-06-13 19:38:41 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:38:41 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:41 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 20). 2470 bytes result sent to driver -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 20) in 42 ms on localhost (1/2) -2016-06-13 19:38:41 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 21). 2470 bytes result sent to driver -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 21) in 49 ms on localhost (2/2) -2016-06-13 19:38:41 INFO DAGScheduler:59 - ShuffleMapStage 21 (mapPartitions at GraphImpl.scala:235) finished in 0.050 s -2016-06-13 19:38:41 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:41 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:41 INFO DAGScheduler:59 - waiting: Set(ResultStage 22) -2016-06-13 19:38:41 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 19:38:41 INFO DAGScheduler:59 - Missing parents for ResultStage 22: List() -2016-06-13 19:38:41 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[56] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(6864) called with curMem=238867, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 6.7 KB, free 730.4 MB) -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(3157) called with curMem=245731, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.3 MB) -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:49848 (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[56] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:41 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-06-13 19:38:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 22, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:38:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 23, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:38:41 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 23) -2016-06-13 19:38:41 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 22) -2016-06-13 19:38:41 INFO CacheManager:59 - Partition rdd_52_1 not found, computing it -2016-06-13 19:38:41 INFO CacheManager:59 - Partition rdd_52_0 not found, computing it -2016-06-13 19:38:41 INFO BlockManager:59 - Found block rdd_39_1 locally -2016-06-13 19:38:41 INFO BlockManager:59 - Found block rdd_39_0 locally -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:41 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=248888, maxMem=766075207 -2016-06-13 19:38:41 INFO MemoryStore:59 - Block rdd_52_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:38:41 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=251336, maxMem=766075207 -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Added rdd_52_1 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO MemoryStore:59 - Block rdd_52_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:38:41 INFO BlockManagerInfo:59 - Added rdd_52_0 in memory on localhost:49848 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:38:41 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 23). 2971 bytes result sent to driver -2016-06-13 19:38:41 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 22). 2971 bytes result sent to driver -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 23) in 24 ms on localhost (1/2) -2016-06-13 19:38:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 22) in 26 ms on localhost (2/2) -2016-06-13 19:38:41 INFO DAGScheduler:59 - ResultStage 22 (fold at LDAOptimizer.scala:205) finished in 0.026 s -2016-06-13 19:38:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 19:38:41 INFO DAGScheduler:59 - Job 4 finished: fold at LDAOptimizer.scala:205, took 0.177171 s -2016-06-13 19:38:42 INFO ZippedPartitionsRDD2:59 - Removing RDD 26 from persistence list -2016-06-13 19:38:42 INFO ZippedPartitionsRDD2:59 - Removing RDD 33 from persistence list -2016-06-13 19:38:42 INFO BlockManager:59 - Removing RDD 26 -2016-06-13 19:38:42 INFO BlockManager:59 - Removing RDD 33 -2016-06-13 19:38:42 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:38:42 INFO DAGScheduler:59 - Registering RDD 57 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:42 INFO DAGScheduler:59 - Registering RDD 61 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:42 INFO DAGScheduler:59 - Got job 5 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:42 INFO DAGScheduler:59 - Final stage: ResultStage 33(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30, ShuffleMapStage 32, ShuffleMapStage 24, ShuffleMapStage 28, ShuffleMapStage 26, ShuffleMapStage 23) -2016-06-13 19:38:42 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 32) -2016-06-13 19:38:42 INFO DAGScheduler:59 - Submitting ShuffleMapStage 31 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[57] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:38:42 INFO MemoryStore:59 - ensureFreeSpace(5872) called with curMem=248984, maxMem=766075207 -2016-06-13 19:38:42 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 5.7 KB, free 730.3 MB) -2016-06-13 19:38:42 INFO MemoryStore:59 - ensureFreeSpace(2848) called with curMem=254856, maxMem=766075207 -2016-06-13 19:38:42 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-06-13 19:38:42 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:49848 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:38:42 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 31 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[57] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:42 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 19:38:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 24, localhost, PROCESS_LOCAL, 2196 bytes) -2016-06-13 19:38:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 25, localhost, PROCESS_LOCAL, 2196 bytes) -2016-06-13 19:38:42 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 25) -2016-06-13 19:38:42 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 24) -2016-06-13 19:38:42 INFO BlockManager:59 - Found block rdd_52_0 locally -2016-06-13 19:38:42 INFO BlockManager:59 - Found block rdd_52_1 locally -2016-06-13 19:38:42 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 24). 2254 bytes result sent to driver -2016-06-13 19:38:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 24) in 44 ms on localhost (1/2) -2016-06-13 19:38:42 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 25). 2254 bytes result sent to driver -2016-06-13 19:38:42 INFO DAGScheduler:59 - ShuffleMapStage 31 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.067 s -2016-06-13 19:38:42 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:42 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:42 INFO DAGScheduler:59 - waiting: Set(ResultStage 33, ShuffleMapStage 32) -2016-06-13 19:38:42 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 25) in 64 ms on localhost (2/2) -2016-06-13 19:38:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 19:38:42 INFO DAGScheduler:59 - Missing parents for ResultStage 33: List(ShuffleMapStage 32) -2016-06-13 19:38:42 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 32: List() -2016-06-13 19:38:42 INFO DAGScheduler:59 - Submitting ShuffleMapStage 32 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[61] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:38:42 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=257704, maxMem=766075207 -2016-06-13 19:38:42 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:38:42 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=261600, maxMem=766075207 -2016-06-13 19:38:42 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:38:42 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:49848 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:42 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 32 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[61] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:42 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-06-13 19:38:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 26, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 27, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:42 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 26) -2016-06-13 19:38:42 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 27) -2016-06-13 19:38:42 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:38:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:42 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:38:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:42 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 26). 2470 bytes result sent to driver -2016-06-13 19:38:42 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 27). 2470 bytes result sent to driver -2016-06-13 19:38:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 27) in 45 ms on localhost (1/2) -2016-06-13 19:38:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 26) in 47 ms on localhost (2/2) -2016-06-13 19:38:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 19:38:42 INFO DAGScheduler:59 - ShuffleMapStage 32 (mapPartitions at GraphImpl.scala:235) finished in 0.048 s -2016-06-13 19:38:42 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:42 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:42 INFO DAGScheduler:59 - waiting: Set(ResultStage 33) -2016-06-13 19:38:42 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:42 INFO DAGScheduler:59 - Missing parents for ResultStage 33: List() -2016-06-13 19:38:42 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[69] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:42 INFO MemoryStore:59 - ensureFreeSpace(7288) called with curMem=263886, maxMem=766075207 -2016-06-13 19:38:42 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 7.1 KB, free 730.3 MB) -2016-06-13 19:38:42 INFO MemoryStore:59 - ensureFreeSpace(3298) called with curMem=271174, maxMem=766075207 -2016-06-13 19:38:42 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.3 MB) -2016-06-13 19:38:42 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:49848 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:38:42 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[69] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:42 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-06-13 19:38:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 28, localhost, PROCESS_LOCAL, 2248 bytes) -2016-06-13 19:38:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 29, localhost, PROCESS_LOCAL, 2248 bytes) -2016-06-13 19:38:42 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 28) -2016-06-13 19:38:42 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 29) -2016-06-13 19:38:42 INFO CacheManager:59 - Partition rdd_65_1 not found, computing it -2016-06-13 19:38:42 INFO BlockManager:59 - Found block rdd_52_1 locally -2016-06-13 19:38:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:42 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=274472, maxMem=766075207 -2016-06-13 19:38:42 INFO MemoryStore:59 - Block rdd_65_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:38:42 INFO BlockManagerInfo:59 - Added rdd_65_1 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:42 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 29). 2971 bytes result sent to driver -2016-06-13 19:38:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 29) in 32 ms on localhost (1/2) -2016-06-13 19:38:42 INFO CacheManager:59 - Partition rdd_65_0 not found, computing it -2016-06-13 19:38:42 INFO BlockManager:59 - Found block rdd_52_0 locally -2016-06-13 19:38:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:42 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=276920, maxMem=766075207 -2016-06-13 19:38:42 INFO MemoryStore:59 - Block rdd_65_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:38:42 INFO BlockManagerInfo:59 - Added rdd_65_0 in memory on localhost:49848 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:38:42 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 28). 2971 bytes result sent to driver -2016-06-13 19:38:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 28) in 50 ms on localhost (2/2) -2016-06-13 19:38:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 19:38:42 INFO DAGScheduler:59 - ResultStage 33 (fold at LDAOptimizer.scala:205) finished in 0.050 s -2016-06-13 19:38:42 INFO DAGScheduler:59 - Job 5 finished: fold at LDAOptimizer.scala:205, took 0.252885 s -2016-06-13 19:38:42 INFO MapPartitionsRDD:59 - Removing RDD 39 from persistence list -2016-06-13 19:38:42 INFO ZippedPartitionsRDD2:59 - Removing RDD 46 from persistence list -2016-06-13 19:38:42 INFO BlockManager:59 - Removing RDD 46 -2016-06-13 19:38:42 INFO BlockManager:59 - Removing RDD 39 -2016-06-13 19:38:42 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:38:42 INFO DAGScheduler:59 - Registering RDD 70 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:42 INFO DAGScheduler:59 - Registering RDD 74 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:42 INFO DAGScheduler:59 - Got job 6 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:42 INFO DAGScheduler:59 - Final stage: ResultStage 46(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 45, ShuffleMapStage 37, ShuffleMapStage 34, ShuffleMapStage 35, ShuffleMapStage 39, ShuffleMapStage 43, ShuffleMapStage 41) -2016-06-13 19:38:42 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 45) -2016-06-13 19:38:42 INFO DAGScheduler:59 - Submitting ShuffleMapStage 44 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[70] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:38:42 INFO MemoryStore:59 - ensureFreeSpace(6304) called with curMem=274472, maxMem=766075207 -2016-06-13 19:38:42 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 6.2 KB, free 730.3 MB) -2016-06-13 19:38:42 INFO MemoryStore:59 - ensureFreeSpace(2991) called with curMem=280776, maxMem=766075207 -2016-06-13 19:38:42 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-06-13 19:38:42 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:49848 (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:38:42 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 44 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[70] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:42 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-06-13 19:38:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 30, localhost, PROCESS_LOCAL, 2237 bytes) -2016-06-13 19:38:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 31, localhost, PROCESS_LOCAL, 2237 bytes) -2016-06-13 19:38:42 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 31) -2016-06-13 19:38:42 INFO BlockManager:59 - Found block rdd_65_1 locally -2016-06-13 19:38:42 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 31). 2254 bytes result sent to driver -2016-06-13 19:38:42 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 30) -2016-06-13 19:38:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 31) in 31 ms on localhost (1/2) -2016-06-13 19:38:42 INFO BlockManager:59 - Found block rdd_65_0 locally -2016-06-13 19:38:42 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 30). 2254 bytes result sent to driver -2016-06-13 19:38:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 30) in 62 ms on localhost (2/2) -2016-06-13 19:38:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-06-13 19:38:42 INFO DAGScheduler:59 - ShuffleMapStage 44 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.064 s -2016-06-13 19:38:42 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:42 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:42 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 45, ResultStage 46) -2016-06-13 19:38:42 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:42 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 45: List() -2016-06-13 19:38:42 INFO DAGScheduler:59 - Missing parents for ResultStage 46: List(ShuffleMapStage 45) -2016-06-13 19:38:42 INFO DAGScheduler:59 - Submitting ShuffleMapStage 45 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[74] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:38:42 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=283767, maxMem=766075207 -2016-06-13 19:38:42 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:38:42 INFO MemoryStore:59 - ensureFreeSpace(2284) called with curMem=287663, maxMem=766075207 -2016-06-13 19:38:42 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:38:42 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:49848 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:42 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 45 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[74] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:42 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-06-13 19:38:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 32, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 33, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:42 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 33) -2016-06-13 19:38:42 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 32) -2016-06-13 19:38:42 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:38:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:42 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:38:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:42 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 33). 2470 bytes result sent to driver -2016-06-13 19:38:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 33) in 51 ms on localhost (1/2) -2016-06-13 19:38:42 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 32). 2470 bytes result sent to driver -2016-06-13 19:38:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 32) in 82 ms on localhost (2/2) -2016-06-13 19:38:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-06-13 19:38:42 INFO DAGScheduler:59 - ShuffleMapStage 45 (mapPartitions at GraphImpl.scala:235) finished in 0.084 s -2016-06-13 19:38:42 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:42 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:42 INFO DAGScheduler:59 - waiting: Set(ResultStage 46) -2016-06-13 19:38:42 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:42 INFO DAGScheduler:59 - Missing parents for ResultStage 46: List() -2016-06-13 19:38:42 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[82] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:42 INFO MemoryStore:59 - ensureFreeSpace(7720) called with curMem=289947, maxMem=766075207 -2016-06-13 19:38:42 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 7.5 KB, free 730.3 MB) -2016-06-13 19:38:42 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=297667, maxMem=766075207 -2016-06-13 19:38:42 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.3 MB) -2016-06-13 19:38:42 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:49848 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:38:42 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[82] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:42 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-06-13 19:38:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 34, localhost, PROCESS_LOCAL, 2289 bytes) -2016-06-13 19:38:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 35, localhost, PROCESS_LOCAL, 2289 bytes) -2016-06-13 19:38:42 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 35) -2016-06-13 19:38:42 INFO CacheManager:59 - Partition rdd_78_1 not found, computing it -2016-06-13 19:38:42 INFO BlockManager:59 - Found block rdd_65_1 locally -2016-06-13 19:38:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:42 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=301067, maxMem=766075207 -2016-06-13 19:38:42 INFO MemoryStore:59 - Block rdd_78_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:38:42 INFO BlockManagerInfo:59 - Added rdd_78_1 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:42 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 35). 2971 bytes result sent to driver -2016-06-13 19:38:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 35) in 28 ms on localhost (1/2) -2016-06-13 19:38:42 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 34) -2016-06-13 19:38:42 INFO CacheManager:59 - Partition rdd_78_0 not found, computing it -2016-06-13 19:38:42 INFO BlockManager:59 - Found block rdd_65_0 locally -2016-06-13 19:38:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:42 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=303515, maxMem=766075207 -2016-06-13 19:38:42 INFO MemoryStore:59 - Block rdd_78_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:38:42 INFO BlockManagerInfo:59 - Added rdd_78_0 in memory on localhost:49848 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:38:42 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 34). 2971 bytes result sent to driver -2016-06-13 19:38:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 34) in 76 ms on localhost (2/2) -2016-06-13 19:38:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-06-13 19:38:42 INFO DAGScheduler:59 - ResultStage 46 (fold at LDAOptimizer.scala:205) finished in 0.078 s -2016-06-13 19:38:42 INFO DAGScheduler:59 - Job 6 finished: fold at LDAOptimizer.scala:205, took 0.323917 s -2016-06-13 19:38:42 INFO MapPartitionsRDD:59 - Removing RDD 52 from persistence list -2016-06-13 19:38:42 INFO BlockManager:59 - Removing RDD 52 -2016-06-13 19:38:42 INFO ZippedPartitionsRDD2:59 - Removing RDD 59 from persistence list -2016-06-13 19:38:42 INFO BlockManager:59 - Removing RDD 59 -2016-06-13 19:38:42 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:38:42 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:38:42 INFO DAGScheduler:59 - Registering RDD 83 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:42 INFO DAGScheduler:59 - Registering RDD 87 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:42 INFO DAGScheduler:59 - Got job 7 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:42 INFO DAGScheduler:59 - Final stage: ResultStage 61(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 48, ShuffleMapStage 60, ShuffleMapStage 52, ShuffleMapStage 56, ShuffleMapStage 50, ShuffleMapStage 54, ShuffleMapStage 58, ShuffleMapStage 47) -2016-06-13 19:38:42 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 60) -2016-06-13 19:38:42 INFO DAGScheduler:59 - Submitting ShuffleMapStage 59 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[83] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:38:42 INFO MemoryStore:59 - ensureFreeSpace(6728) called with curMem=301067, maxMem=766075207 -2016-06-13 19:38:42 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 6.6 KB, free 730.3 MB) -2016-06-13 19:38:42 INFO MemoryStore:59 - ensureFreeSpace(3078) called with curMem=307795, maxMem=766075207 -2016-06-13 19:38:42 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.3 MB) -2016-06-13 19:38:42 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:49848 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:38:42 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 59 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[83] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:42 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 2 tasks -2016-06-13 19:38:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 36, localhost, PROCESS_LOCAL, 2278 bytes) -2016-06-13 19:38:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 37, localhost, PROCESS_LOCAL, 2278 bytes) -2016-06-13 19:38:42 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 37) -2016-06-13 19:38:42 INFO BlockManager:59 - Found block rdd_78_1 locally -2016-06-13 19:38:42 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 36) -2016-06-13 19:38:42 INFO BlockManager:59 - Found block rdd_78_0 locally -2016-06-13 19:38:42 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 37). 2254 bytes result sent to driver -2016-06-13 19:38:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 37) in 37 ms on localhost (1/2) -2016-06-13 19:38:42 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 36). 2254 bytes result sent to driver -2016-06-13 19:38:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 36) in 51 ms on localhost (2/2) -2016-06-13 19:38:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-06-13 19:38:42 INFO DAGScheduler:59 - ShuffleMapStage 59 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.051 s -2016-06-13 19:38:42 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:42 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:42 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 60, ResultStage 61) -2016-06-13 19:38:42 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:42 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 60: List() -2016-06-13 19:38:42 INFO DAGScheduler:59 - Missing parents for ResultStage 61: List(ShuffleMapStage 60) -2016-06-13 19:38:42 INFO DAGScheduler:59 - Submitting ShuffleMapStage 60 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[87] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:38:42 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=310873, maxMem=766075207 -2016-06-13 19:38:42 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:38:42 INFO MemoryStore:59 - ensureFreeSpace(2280) called with curMem=314769, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:49848 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 60 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[87] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Adding task set 60.0 with 2 tasks -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 60.0 (TID 38, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:49848 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 60.0 (TID 39, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:43 INFO Executor:59 - Running task 1.0 in stage 60.0 (TID 39) -2016-06-13 19:38:43 INFO Executor:59 - Running task 0.0 in stage 60.0 (TID 38) -2016-06-13 19:38:43 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:43 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:43 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:49848 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:49848 in memory (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:49848 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 19:38:43 INFO Executor:59 - Finished task 1.0 in stage 60.0 (TID 39). 2470 bytes result sent to driver -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:49848 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 60.0 (TID 39) in 66 ms on localhost (1/2) -2016-06-13 19:38:43 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:49848 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:49848 in memory (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:49848 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO Executor:59 - Finished task 0.0 in stage 60.0 (TID 38). 2470 bytes result sent to driver -2016-06-13 19:38:43 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 19:38:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 60.0 (TID 38) in 100 ms on localhost (2/2) -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 60.0, whose tasks have all completed, from pool -2016-06-13 19:38:43 INFO DAGScheduler:59 - ShuffleMapStage 60 (mapPartitions at GraphImpl.scala:235) finished in 0.106 s -2016-06-13 19:38:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:43 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:43 INFO DAGScheduler:59 - waiting: Set(ResultStage 61) -2016-06-13 19:38:43 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:49848 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 19:38:43 INFO DAGScheduler:59 - Missing parents for ResultStage 61: List() -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting ResultStage 61 (MapPartitionsRDD[95] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:49848 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(8144) called with curMem=231116, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 8.0 KB, free 730.4 MB) -2016-06-13 19:38:43 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=239260, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.4 MB) -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:49848 (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:49848 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 61 (MapPartitionsRDD[95] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 2 tasks -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 40, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 41, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:38:43 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 19:38:43 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 40) -2016-06-13 19:38:43 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 41) -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:49848 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO CacheManager:59 - Partition rdd_91_0 not found, computing it -2016-06-13 19:38:43 INFO BlockManager:59 - Found block rdd_78_0 locally -2016-06-13 19:38:43 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 19:38:43 INFO CacheManager:59 - Partition rdd_91_1 not found, computing it -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:43 INFO BlockManager:59 - Found block rdd_78_1 locally -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=229821, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block rdd_91_0 stored as values in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=232365, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block rdd_91_1 stored as values in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Added rdd_91_0 in memory on localhost:49848 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Added rdd_91_1 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 40). 2971 bytes result sent to driver -2016-06-13 19:38:43 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 41). 2971 bytes result sent to driver -2016-06-13 19:38:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 40) in 27 ms on localhost (1/2) -2016-06-13 19:38:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 41) in 29 ms on localhost (2/2) -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-06-13 19:38:43 INFO DAGScheduler:59 - ResultStage 61 (fold at LDAOptimizer.scala:205) finished in 0.032 s -2016-06-13 19:38:43 INFO DAGScheduler:59 - Job 7 finished: fold at LDAOptimizer.scala:205, took 0.429482 s -2016-06-13 19:38:43 INFO MapPartitionsRDD:59 - Removing RDD 65 from persistence list -2016-06-13 19:38:43 INFO ZippedPartitionsRDD2:59 - Removing RDD 72 from persistence list -2016-06-13 19:38:43 INFO BlockManager:59 - Removing RDD 65 -2016-06-13 19:38:43 INFO BlockManager:59 - Removing RDD 72 -2016-06-13 19:38:43 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:38:43 INFO DAGScheduler:59 - Registering RDD 96 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:43 INFO DAGScheduler:59 - Registering RDD 100 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:43 INFO DAGScheduler:59 - Got job 8 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:43 INFO DAGScheduler:59 - Final stage: ResultStage 78(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 63, ShuffleMapStage 67, ShuffleMapStage 71, ShuffleMapStage 75, ShuffleMapStage 69, ShuffleMapStage 73, ShuffleMapStage 65, ShuffleMapStage 62, ShuffleMapStage 77) -2016-06-13 19:38:43 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 77) -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 76 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[96] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(7160) called with curMem=229821, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 7.0 KB, free 730.4 MB) -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(3182) called with curMem=236981, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.4 MB) -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:49848 (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 76 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[96] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 2 tasks -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 42, localhost, PROCESS_LOCAL, 2319 bytes) -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 43, localhost, PROCESS_LOCAL, 2319 bytes) -2016-06-13 19:38:43 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 42) -2016-06-13 19:38:43 INFO BlockManager:59 - Found block rdd_91_0 locally -2016-06-13 19:38:43 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 42). 2254 bytes result sent to driver -2016-06-13 19:38:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 42) in 42 ms on localhost (1/2) -2016-06-13 19:38:43 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 43) -2016-06-13 19:38:43 INFO BlockManager:59 - Found block rdd_91_1 locally -2016-06-13 19:38:43 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 43). 2254 bytes result sent to driver -2016-06-13 19:38:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 43) in 132 ms on localhost (2/2) -2016-06-13 19:38:43 INFO DAGScheduler:59 - ShuffleMapStage 76 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.130 s -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-06-13 19:38:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:43 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:43 INFO DAGScheduler:59 - waiting: Set(ResultStage 78, ShuffleMapStage 77) -2016-06-13 19:38:43 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:43 INFO DAGScheduler:59 - Missing parents for ResultStage 78: List(ShuffleMapStage 77) -2016-06-13 19:38:43 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 77: List() -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 77 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[100] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=240163, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(2283) called with curMem=244059, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:49848 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 77 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[100] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 2 tasks -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 44, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 45, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:43 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 44) -2016-06-13 19:38:43 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 45) -2016-06-13 19:38:43 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:38:43 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 3 ms -2016-06-13 19:38:43 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 44). 2470 bytes result sent to driver -2016-06-13 19:38:43 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 45). 2470 bytes result sent to driver -2016-06-13 19:38:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 44) in 38 ms on localhost (1/2) -2016-06-13 19:38:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 45) in 39 ms on localhost (2/2) -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-06-13 19:38:43 INFO DAGScheduler:59 - ShuffleMapStage 77 (mapPartitions at GraphImpl.scala:235) finished in 0.041 s -2016-06-13 19:38:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:43 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:43 INFO DAGScheduler:59 - waiting: Set(ResultStage 78) -2016-06-13 19:38:43 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:43 INFO DAGScheduler:59 - Missing parents for ResultStage 78: List() -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting ResultStage 78 (MapPartitionsRDD[108] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(8576) called with curMem=246342, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 8.4 KB, free 730.3 MB) -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(3582) called with curMem=254918, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:49848 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 78 (MapPartitionsRDD[108] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 2 tasks -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 46, localhost, PROCESS_LOCAL, 2371 bytes) -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 47, localhost, PROCESS_LOCAL, 2371 bytes) -2016-06-13 19:38:43 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 47) -2016-06-13 19:38:43 INFO CacheManager:59 - Partition rdd_104_1 not found, computing it -2016-06-13 19:38:43 INFO BlockManager:59 - Found block rdd_91_1 locally -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:43 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 46) -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=258500, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block rdd_104_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Added rdd_104_1 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 47). 2971 bytes result sent to driver -2016-06-13 19:38:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 47) in 15 ms on localhost (1/2) -2016-06-13 19:38:43 INFO CacheManager:59 - Partition rdd_104_0 not found, computing it -2016-06-13 19:38:43 INFO BlockManager:59 - Found block rdd_91_0 locally -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=260948, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block rdd_104_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Added rdd_104_0 in memory on localhost:49848 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 46). 2971 bytes result sent to driver -2016-06-13 19:38:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 46) in 27 ms on localhost (2/2) -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-06-13 19:38:43 INFO DAGScheduler:59 - ResultStage 78 (fold at LDAOptimizer.scala:205) finished in 0.028 s -2016-06-13 19:38:43 INFO DAGScheduler:59 - Job 8 finished: fold at LDAOptimizer.scala:205, took 0.287014 s -2016-06-13 19:38:43 INFO MapPartitionsRDD:59 - Removing RDD 78 from persistence list -2016-06-13 19:38:43 INFO ZippedPartitionsRDD2:59 - Removing RDD 85 from persistence list -2016-06-13 19:38:43 INFO BlockManager:59 - Removing RDD 78 -2016-06-13 19:38:43 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:38:43 INFO DAGScheduler:59 - Registering RDD 109 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:43 INFO DAGScheduler:59 - Registering RDD 113 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:43 INFO DAGScheduler:59 - Got job 9 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:43 INFO DAGScheduler:59 - Final stage: ResultStage 97(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 84, ShuffleMapStage 96, ShuffleMapStage 88, ShuffleMapStage 82, ShuffleMapStage 86, ShuffleMapStage 90, ShuffleMapStage 79, ShuffleMapStage 94, ShuffleMapStage 80, ShuffleMapStage 92) -2016-06-13 19:38:43 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 96) -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 95 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[109] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(7584) called with curMem=258500, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 7.4 KB, free 730.3 MB) -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(3283) called with curMem=266084, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.3 MB) -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:49848 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 95 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[109] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Adding task set 95.0 with 2 tasks -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 95.0 (TID 48, localhost, PROCESS_LOCAL, 2360 bytes) -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 95.0 (TID 49, localhost, PROCESS_LOCAL, 2360 bytes) -2016-06-13 19:38:43 INFO Executor:59 - Running task 0.0 in stage 95.0 (TID 48) -2016-06-13 19:38:43 INFO BlockManager:59 - Found block rdd_104_0 locally -2016-06-13 19:38:43 INFO Executor:59 - Running task 1.0 in stage 95.0 (TID 49) -2016-06-13 19:38:43 INFO BlockManager:59 - Removing RDD 85 -2016-06-13 19:38:43 INFO Executor:59 - Finished task 0.0 in stage 95.0 (TID 48). 2254 bytes result sent to driver -2016-06-13 19:38:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 95.0 (TID 48) in 30 ms on localhost (1/2) -2016-06-13 19:38:43 INFO BlockManager:59 - Found block rdd_104_1 locally -2016-06-13 19:38:43 INFO Executor:59 - Finished task 1.0 in stage 95.0 (TID 49). 2254 bytes result sent to driver -2016-06-13 19:38:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 95.0 (TID 49) in 89 ms on localhost (2/2) -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 95.0, whose tasks have all completed, from pool -2016-06-13 19:38:43 INFO DAGScheduler:59 - ShuffleMapStage 95 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.088 s -2016-06-13 19:38:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:43 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:43 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 96, ResultStage 97) -2016-06-13 19:38:43 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:43 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 96: List() -2016-06-13 19:38:43 INFO DAGScheduler:59 - Missing parents for ResultStage 97: List(ShuffleMapStage 96) -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 96 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[113] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=269367, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=273263, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:49848 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 96 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[113] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Adding task set 96.0 with 2 tasks -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 96.0 (TID 50, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 96.0 (TID 51, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:43 INFO Executor:59 - Running task 1.0 in stage 96.0 (TID 51) -2016-06-13 19:38:43 INFO Executor:59 - Running task 0.0 in stage 96.0 (TID 50) -2016-06-13 19:38:43 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:43 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-06-13 19:38:43 INFO Executor:59 - Finished task 1.0 in stage 96.0 (TID 51). 2470 bytes result sent to driver -2016-06-13 19:38:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 96.0 (TID 51) in 36 ms on localhost (1/2) -2016-06-13 19:38:43 INFO Executor:59 - Finished task 0.0 in stage 96.0 (TID 50). 2470 bytes result sent to driver -2016-06-13 19:38:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 96.0 (TID 50) in 50 ms on localhost (2/2) -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 96.0, whose tasks have all completed, from pool -2016-06-13 19:38:43 INFO DAGScheduler:59 - ShuffleMapStage 96 (mapPartitions at GraphImpl.scala:235) finished in 0.051 s -2016-06-13 19:38:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:43 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:43 INFO DAGScheduler:59 - waiting: Set(ResultStage 97) -2016-06-13 19:38:43 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:43 INFO DAGScheduler:59 - Missing parents for ResultStage 97: List() -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting ResultStage 97 (MapPartitionsRDD[121] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(9000) called with curMem=275549, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 8.8 KB, free 730.3 MB) -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(3686) called with curMem=284549, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 3.6 KB, free 730.3 MB) -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:49848 (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 97 (MapPartitionsRDD[121] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Adding task set 97.0 with 2 tasks -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 97.0 (TID 52, localhost, PROCESS_LOCAL, 2412 bytes) -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 97.0 (TID 53, localhost, PROCESS_LOCAL, 2412 bytes) -2016-06-13 19:38:43 INFO Executor:59 - Running task 1.0 in stage 97.0 (TID 53) -2016-06-13 19:38:43 INFO Executor:59 - Running task 0.0 in stage 97.0 (TID 52) -2016-06-13 19:38:43 INFO CacheManager:59 - Partition rdd_117_1 not found, computing it -2016-06-13 19:38:43 INFO BlockManager:59 - Found block rdd_104_1 locally -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:43 INFO CacheManager:59 - Partition rdd_117_0 not found, computing it -2016-06-13 19:38:43 INFO BlockManager:59 - Found block rdd_104_0 locally -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=288235, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block rdd_117_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Added rdd_117_1 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO Executor:59 - Finished task 1.0 in stage 97.0 (TID 53). 2971 bytes result sent to driver -2016-06-13 19:38:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 97.0 (TID 53) in 19 ms on localhost (1/2) -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=290683, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block rdd_117_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Added rdd_117_0 in memory on localhost:49848 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO Executor:59 - Finished task 0.0 in stage 97.0 (TID 52). 2971 bytes result sent to driver -2016-06-13 19:38:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 97.0 (TID 52) in 33 ms on localhost (2/2) -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 97.0, whose tasks have all completed, from pool -2016-06-13 19:38:43 INFO DAGScheduler:59 - ResultStage 97 (fold at LDAOptimizer.scala:205) finished in 0.035 s -2016-06-13 19:38:43 INFO DAGScheduler:59 - Job 9 finished: fold at LDAOptimizer.scala:205, took 0.268770 s -2016-06-13 19:38:43 INFO MapPartitionsRDD:59 - Removing RDD 91 from persistence list -2016-06-13 19:38:43 INFO BlockManager:59 - Removing RDD 91 -2016-06-13 19:38:43 INFO ZippedPartitionsRDD2:59 - Removing RDD 98 from persistence list -2016-06-13 19:38:43 INFO BlockManager:59 - Removing RDD 98 -2016-06-13 19:38:43 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:38:43 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:38:43 INFO DAGScheduler:59 - Registering RDD 122 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:43 INFO DAGScheduler:59 - Registering RDD 126 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:43 INFO DAGScheduler:59 - Got job 10 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:43 INFO DAGScheduler:59 - Final stage: ResultStage 118(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:43 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 117, ShuffleMapStage 99, ShuffleMapStage 103, ShuffleMapStage 115, ShuffleMapStage 107, ShuffleMapStage 101, ShuffleMapStage 111, ShuffleMapStage 105, ShuffleMapStage 109, ShuffleMapStage 98, ShuffleMapStage 113) -2016-06-13 19:38:43 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 117) -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 116 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[122] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(8016) called with curMem=288235, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 7.8 KB, free 730.3 MB) -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(3371) called with curMem=296251, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.3 MB) -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:49848 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 116 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[122] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Adding task set 116.0 with 2 tasks -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 116.0 (TID 54, localhost, PROCESS_LOCAL, 2401 bytes) -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 116.0 (TID 55, localhost, PROCESS_LOCAL, 2401 bytes) -2016-06-13 19:38:43 INFO Executor:59 - Running task 1.0 in stage 116.0 (TID 55) -2016-06-13 19:38:43 INFO Executor:59 - Running task 0.0 in stage 116.0 (TID 54) -2016-06-13 19:38:43 INFO BlockManager:59 - Found block rdd_117_1 locally -2016-06-13 19:38:43 INFO BlockManager:59 - Found block rdd_117_0 locally -2016-06-13 19:38:43 INFO Executor:59 - Finished task 1.0 in stage 116.0 (TID 55). 2254 bytes result sent to driver -2016-06-13 19:38:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 116.0 (TID 55) in 19 ms on localhost (1/2) -2016-06-13 19:38:43 INFO Executor:59 - Finished task 0.0 in stage 116.0 (TID 54). 2254 bytes result sent to driver -2016-06-13 19:38:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 116.0 (TID 54) in 41 ms on localhost (2/2) -2016-06-13 19:38:43 INFO DAGScheduler:59 - ShuffleMapStage 116 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.041 s -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 116.0, whose tasks have all completed, from pool -2016-06-13 19:38:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:43 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:43 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 117, ResultStage 118) -2016-06-13 19:38:43 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:43 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 117: List() -2016-06-13 19:38:43 INFO DAGScheduler:59 - Missing parents for ResultStage 118: List(ShuffleMapStage 117) -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting ShuffleMapStage 117 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[126] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=299622, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=303518, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:49848 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 117 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[126] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Adding task set 117.0 with 2 tasks -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 117.0 (TID 56, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 117.0 (TID 57, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:43 INFO Executor:59 - Running task 1.0 in stage 117.0 (TID 57) -2016-06-13 19:38:43 INFO Executor:59 - Running task 0.0 in stage 117.0 (TID 56) -2016-06-13 19:38:43 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:38:43 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:43 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:43 INFO Executor:59 - Finished task 0.0 in stage 117.0 (TID 56). 2470 bytes result sent to driver -2016-06-13 19:38:43 INFO Executor:59 - Finished task 1.0 in stage 117.0 (TID 57). 2470 bytes result sent to driver -2016-06-13 19:38:43 INFO TaskSetManager:59 - Finished task 0.0 in stage 117.0 (TID 56) in 28 ms on localhost (1/2) -2016-06-13 19:38:43 INFO TaskSetManager:59 - Finished task 1.0 in stage 117.0 (TID 57) in 29 ms on localhost (2/2) -2016-06-13 19:38:43 INFO DAGScheduler:59 - ShuffleMapStage 117 (mapPartitions at GraphImpl.scala:235) finished in 0.029 s -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 117.0, whose tasks have all completed, from pool -2016-06-13 19:38:43 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:43 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:43 INFO DAGScheduler:59 - waiting: Set(ResultStage 118) -2016-06-13 19:38:43 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:43 INFO DAGScheduler:59 - Missing parents for ResultStage 118: List() -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting ResultStage 118 (MapPartitionsRDD[134] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(9432) called with curMem=305804, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 9.2 KB, free 730.3 MB) -2016-06-13 19:38:43 INFO MemoryStore:59 - ensureFreeSpace(3778) called with curMem=315236, maxMem=766075207 -2016-06-13 19:38:43 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.7 KB, free 730.3 MB) -2016-06-13 19:38:43 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:49848 (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:38:43 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:43 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 118 (MapPartitionsRDD[134] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:43 INFO TaskSchedulerImpl:59 - Adding task set 118.0 with 2 tasks -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 0.0 in stage 118.0 (TID 58, localhost, PROCESS_LOCAL, 2453 bytes) -2016-06-13 19:38:43 INFO TaskSetManager:59 - Starting task 1.0 in stage 118.0 (TID 59, localhost, PROCESS_LOCAL, 2453 bytes) -2016-06-13 19:38:43 INFO Executor:59 - Running task 0.0 in stage 118.0 (TID 58) -2016-06-13 19:38:43 INFO Executor:59 - Running task 1.0 in stage 118.0 (TID 59) -2016-06-13 19:38:44 INFO CacheManager:59 - Partition rdd_130_1 not found, computing it -2016-06-13 19:38:44 INFO CacheManager:59 - Partition rdd_130_0 not found, computing it -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_117_1 locally -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_117_0 locally -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=319014, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block rdd_130_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added rdd_130_0 in memory on localhost:49848 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO Executor:59 - Finished task 0.0 in stage 118.0 (TID 58). 2971 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 118.0 (TID 58) in 17 ms on localhost (1/2) -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=321558, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block rdd_130_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added rdd_130_1 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO Executor:59 - Finished task 1.0 in stage 118.0 (TID 59). 2971 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 118.0 (TID 59) in 27 ms on localhost (2/2) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 118.0, whose tasks have all completed, from pool -2016-06-13 19:38:44 INFO DAGScheduler:59 - ResultStage 118 (fold at LDAOptimizer.scala:205) finished in 0.029 s -2016-06-13 19:38:44 INFO DAGScheduler:59 - Job 10 finished: fold at LDAOptimizer.scala:205, took 0.161953 s -2016-06-13 19:38:44 INFO MapPartitionsRDD:59 - Removing RDD 104 from persistence list -2016-06-13 19:38:44 INFO ZippedPartitionsRDD2:59 - Removing RDD 111 from persistence list -2016-06-13 19:38:44 INFO BlockManager:59 - Removing RDD 104 -2016-06-13 19:38:44 INFO BlockManager:59 - Removing RDD 111 -2016-06-13 19:38:44 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:38:44 INFO DAGScheduler:59 - Registering RDD 135 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Registering RDD 139 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Got job 11 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:44 INFO DAGScheduler:59 - Final stage: ResultStage 141(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 132, ShuffleMapStage 136, ShuffleMapStage 140, ShuffleMapStage 122, ShuffleMapStage 119, ShuffleMapStage 126, ShuffleMapStage 138, ShuffleMapStage 120, ShuffleMapStage 130, ShuffleMapStage 124, ShuffleMapStage 134, ShuffleMapStage 128) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 140) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 139 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[135] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(8440) called with curMem=319014, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 8.2 KB, free 730.3 MB) -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(3476) called with curMem=327454, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.3 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:49848 (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 139 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[135] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Adding task set 139.0 with 2 tasks -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 139.0 (TID 60, localhost, PROCESS_LOCAL, 2442 bytes) -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 139.0 (TID 61, localhost, PROCESS_LOCAL, 2442 bytes) -2016-06-13 19:38:44 INFO Executor:59 - Running task 0.0 in stage 139.0 (TID 60) -2016-06-13 19:38:44 INFO Executor:59 - Running task 1.0 in stage 139.0 (TID 61) -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_130_1 locally -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_130_0 locally -2016-06-13 19:38:44 INFO Executor:59 - Finished task 1.0 in stage 139.0 (TID 61). 2254 bytes result sent to driver -2016-06-13 19:38:44 INFO Executor:59 - Finished task 0.0 in stage 139.0 (TID 60). 2254 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 139.0 (TID 61) in 24 ms on localhost (1/2) -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 139.0 (TID 60) in 25 ms on localhost (2/2) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 139.0, whose tasks have all completed, from pool -2016-06-13 19:38:44 INFO DAGScheduler:59 - ShuffleMapStage 139 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.026 s -2016-06-13 19:38:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:44 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:44 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 140, ResultStage 141) -2016-06-13 19:38:44 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:44 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 140: List() -2016-06-13 19:38:44 INFO DAGScheduler:59 - Missing parents for ResultStage 141: List(ShuffleMapStage 140) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 140 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[139] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=330930, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(2281) called with curMem=334826, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:49848 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 140 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[139] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Adding task set 140.0 with 2 tasks -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 140.0 (TID 62, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 140.0 (TID 63, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:44 INFO Executor:59 - Running task 1.0 in stage 140.0 (TID 63) -2016-06-13 19:38:44 INFO Executor:59 - Running task 0.0 in stage 140.0 (TID 62) -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:44 INFO Executor:59 - Finished task 1.0 in stage 140.0 (TID 63). 2470 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 140.0 (TID 63) in 20 ms on localhost (1/2) -2016-06-13 19:38:44 INFO Executor:59 - Finished task 0.0 in stage 140.0 (TID 62). 2470 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 140.0 (TID 62) in 97 ms on localhost (2/2) -2016-06-13 19:38:44 INFO DAGScheduler:59 - ShuffleMapStage 140 (mapPartitions at GraphImpl.scala:235) finished in 0.098 s -2016-06-13 19:38:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:44 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:44 INFO DAGScheduler:59 - waiting: Set(ResultStage 141) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 140.0, whose tasks have all completed, from pool -2016-06-13 19:38:44 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:44 INFO DAGScheduler:59 - Missing parents for ResultStage 141: List() -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting ResultStage 141 (MapPartitionsRDD[147] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(9856) called with curMem=337107, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 9.6 KB, free 730.3 MB) -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(3886) called with curMem=346963, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:49848 (size: 3.8 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 141 (MapPartitionsRDD[147] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Adding task set 141.0 with 2 tasks -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 141.0 (TID 64, localhost, PROCESS_LOCAL, 2494 bytes) -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 141.0 (TID 65, localhost, PROCESS_LOCAL, 2494 bytes) -2016-06-13 19:38:44 INFO Executor:59 - Running task 0.0 in stage 141.0 (TID 64) -2016-06-13 19:38:44 INFO Executor:59 - Running task 1.0 in stage 141.0 (TID 65) -2016-06-13 19:38:44 INFO CacheManager:59 - Partition rdd_143_1 not found, computing it -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_130_1 locally -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:44 INFO CacheManager:59 - Partition rdd_143_0 not found, computing it -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_130_0 locally -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=350849, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block rdd_143_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added rdd_143_0 in memory on localhost:49848 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO Executor:59 - Finished task 0.0 in stage 141.0 (TID 64). 2971 bytes result sent to driver -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=353393, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block rdd_143_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added rdd_143_1 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO Executor:59 - Finished task 1.0 in stage 141.0 (TID 65). 2971 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 141.0 (TID 64) in 17 ms on localhost (1/2) -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 141.0 (TID 65) in 18 ms on localhost (2/2) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 141.0, whose tasks have all completed, from pool -2016-06-13 19:38:44 INFO DAGScheduler:59 - ResultStage 141 (fold at LDAOptimizer.scala:205) finished in 0.019 s -2016-06-13 19:38:44 INFO DAGScheduler:59 - Job 11 finished: fold at LDAOptimizer.scala:205, took 0.211296 s -2016-06-13 19:38:44 INFO MapPartitionsRDD:59 - Removing RDD 117 from persistence list -2016-06-13 19:38:44 INFO ZippedPartitionsRDD2:59 - Removing RDD 124 from persistence list -2016-06-13 19:38:44 INFO BlockManager:59 - Removing RDD 117 -2016-06-13 19:38:44 INFO BlockManager:59 - Removing RDD 124 -2016-06-13 19:38:44 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:38:44 INFO DAGScheduler:59 - Registering RDD 148 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Registering RDD 152 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Got job 12 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:44 INFO DAGScheduler:59 - Final stage: ResultStage 166(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 161, ShuffleMapStage 143, ShuffleMapStage 153, ShuffleMapStage 165, ShuffleMapStage 147, ShuffleMapStage 157, ShuffleMapStage 151, ShuffleMapStage 155, ShuffleMapStage 159, ShuffleMapStage 163, ShuffleMapStage 145, ShuffleMapStage 142, ShuffleMapStage 149) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 165) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 164 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[148] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(8872) called with curMem=350849, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 8.7 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(3568) called with curMem=359721, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:49848 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 164 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[148] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Adding task set 164.0 with 2 tasks -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 164.0 (TID 66, localhost, PROCESS_LOCAL, 2483 bytes) -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 164.0 (TID 67, localhost, PROCESS_LOCAL, 2483 bytes) -2016-06-13 19:38:44 INFO Executor:59 - Running task 1.0 in stage 164.0 (TID 67) -2016-06-13 19:38:44 INFO Executor:59 - Running task 0.0 in stage 164.0 (TID 66) -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_143_0 locally -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_143_1 locally -2016-06-13 19:38:44 INFO Executor:59 - Finished task 1.0 in stage 164.0 (TID 67). 2254 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 164.0 (TID 67) in 22 ms on localhost (1/2) -2016-06-13 19:38:44 INFO Executor:59 - Finished task 0.0 in stage 164.0 (TID 66). 2254 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 164.0 (TID 66) in 42 ms on localhost (2/2) -2016-06-13 19:38:44 INFO DAGScheduler:59 - ShuffleMapStage 164 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.044 s -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 164.0, whose tasks have all completed, from pool -2016-06-13 19:38:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:44 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:44 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 165, ResultStage 166) -2016-06-13 19:38:44 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:44 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 165: List() -2016-06-13 19:38:44 INFO DAGScheduler:59 - Missing parents for ResultStage 166: List(ShuffleMapStage 165) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 165 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[152] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=363289, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(2289) called with curMem=367185, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:49848 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 165 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[152] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Adding task set 165.0 with 2 tasks -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 165.0 (TID 68, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 165.0 (TID 69, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:44 INFO Executor:59 - Running task 0.0 in stage 165.0 (TID 68) -2016-06-13 19:38:44 INFO Executor:59 - Running task 1.0 in stage 165.0 (TID 69) -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:44 INFO Executor:59 - Finished task 1.0 in stage 165.0 (TID 69). 2470 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 165.0 (TID 69) in 44 ms on localhost (1/2) -2016-06-13 19:38:44 INFO Executor:59 - Finished task 0.0 in stage 165.0 (TID 68). 2470 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 165.0 (TID 68) in 59 ms on localhost (2/2) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 165.0, whose tasks have all completed, from pool -2016-06-13 19:38:44 INFO DAGScheduler:59 - ShuffleMapStage 165 (mapPartitions at GraphImpl.scala:235) finished in 0.061 s -2016-06-13 19:38:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:44 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:44 INFO DAGScheduler:59 - waiting: Set(ResultStage 166) -2016-06-13 19:38:44 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:44 INFO DAGScheduler:59 - Missing parents for ResultStage 166: List() -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting ResultStage 166 (MapPartitionsRDD[160] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(10288) called with curMem=369474, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 10.0 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(4004) called with curMem=379762, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 3.9 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:49848 (size: 3.9 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 166 (MapPartitionsRDD[160] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Adding task set 166.0 with 2 tasks -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 166.0 (TID 70, localhost, PROCESS_LOCAL, 2535 bytes) -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 166.0 (TID 71, localhost, PROCESS_LOCAL, 2535 bytes) -2016-06-13 19:38:44 INFO Executor:59 - Running task 1.0 in stage 166.0 (TID 71) -2016-06-13 19:38:44 INFO Executor:59 - Running task 0.0 in stage 166.0 (TID 70) -2016-06-13 19:38:44 INFO CacheManager:59 - Partition rdd_156_1 not found, computing it -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_143_1 locally -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:44 INFO CacheManager:59 - Partition rdd_156_0 not found, computing it -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_143_0 locally -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=383766, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block rdd_156_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=386214, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block rdd_156_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added rdd_156_1 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added rdd_156_0 in memory on localhost:49848 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO Executor:59 - Finished task 1.0 in stage 166.0 (TID 71). 2971 bytes result sent to driver -2016-06-13 19:38:44 INFO Executor:59 - Finished task 0.0 in stage 166.0 (TID 70). 2971 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 166.0 (TID 71) in 13 ms on localhost (1/2) -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 166.0 (TID 70) in 15 ms on localhost (2/2) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 166.0, whose tasks have all completed, from pool -2016-06-13 19:38:44 INFO DAGScheduler:59 - ResultStage 166 (fold at LDAOptimizer.scala:205) finished in 0.016 s -2016-06-13 19:38:44 INFO DAGScheduler:59 - Job 12 finished: fold at LDAOptimizer.scala:205, took 0.197515 s -2016-06-13 19:38:44 INFO MapPartitionsRDD:59 - Removing RDD 130 from persistence list -2016-06-13 19:38:44 INFO ZippedPartitionsRDD2:59 - Removing RDD 137 from persistence list -2016-06-13 19:38:44 INFO BlockManager:59 - Removing RDD 130 -2016-06-13 19:38:44 INFO BlockManager:59 - Removing RDD 137 -2016-06-13 19:38:44 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:38:44 INFO DAGScheduler:59 - Registering RDD 161 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Registering RDD 165 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Got job 13 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:44 INFO DAGScheduler:59 - Final stage: ResultStage 193(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 168, ShuffleMapStage 190, ShuffleMapStage 184, ShuffleMapStage 176, ShuffleMapStage 170, ShuffleMapStage 192, ShuffleMapStage 178, ShuffleMapStage 186, ShuffleMapStage 180, ShuffleMapStage 172, ShuffleMapStage 188, ShuffleMapStage 174, ShuffleMapStage 167, ShuffleMapStage 182) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 192) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 191 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[161] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(9296) called with curMem=383766, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 9.1 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(3674) called with curMem=393062, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 3.6 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:49848 (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 191 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[161] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Adding task set 191.0 with 2 tasks -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 191.0 (TID 72, localhost, PROCESS_LOCAL, 2524 bytes) -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 191.0 (TID 73, localhost, PROCESS_LOCAL, 2524 bytes) -2016-06-13 19:38:44 INFO Executor:59 - Running task 0.0 in stage 191.0 (TID 72) -2016-06-13 19:38:44 INFO Executor:59 - Running task 1.0 in stage 191.0 (TID 73) -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_156_0 locally -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_156_1 locally -2016-06-13 19:38:44 INFO Executor:59 - Finished task 0.0 in stage 191.0 (TID 72). 2254 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 191.0 (TID 72) in 24 ms on localhost (1/2) -2016-06-13 19:38:44 INFO Executor:59 - Finished task 1.0 in stage 191.0 (TID 73). 2254 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 191.0 (TID 73) in 26 ms on localhost (2/2) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 191.0, whose tasks have all completed, from pool -2016-06-13 19:38:44 INFO DAGScheduler:59 - ShuffleMapStage 191 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.027 s -2016-06-13 19:38:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:44 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:44 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 192, ResultStage 193) -2016-06-13 19:38:44 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:44 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 192: List() -2016-06-13 19:38:44 INFO DAGScheduler:59 - Missing parents for ResultStage 193: List(ShuffleMapStage 192) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 192 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[165] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=396736, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=400632, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:49848 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 192 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[165] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Adding task set 192.0 with 2 tasks -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 192.0 (TID 74, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 192.0 (TID 75, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:44 INFO Executor:59 - Running task 1.0 in stage 192.0 (TID 75) -2016-06-13 19:38:44 INFO Executor:59 - Running task 0.0 in stage 192.0 (TID 74) -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:44 INFO Executor:59 - Finished task 1.0 in stage 192.0 (TID 75). 2470 bytes result sent to driver -2016-06-13 19:38:44 INFO Executor:59 - Finished task 0.0 in stage 192.0 (TID 74). 2470 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 192.0 (TID 75) in 27 ms on localhost (1/2) -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 192.0 (TID 74) in 28 ms on localhost (2/2) -2016-06-13 19:38:44 INFO DAGScheduler:59 - ShuffleMapStage 192 (mapPartitions at GraphImpl.scala:235) finished in 0.029 s -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 192.0, whose tasks have all completed, from pool -2016-06-13 19:38:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:44 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:44 INFO DAGScheduler:59 - waiting: Set(ResultStage 193) -2016-06-13 19:38:44 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:44 INFO DAGScheduler:59 - Missing parents for ResultStage 193: List() -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting ResultStage 193 (MapPartitionsRDD[173] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(10712) called with curMem=402918, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 10.5 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=413630, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:49848 (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 193 (MapPartitionsRDD[173] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Adding task set 193.0 with 2 tasks -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 193.0 (TID 76, localhost, PROCESS_LOCAL, 2576 bytes) -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 193.0 (TID 77, localhost, PROCESS_LOCAL, 2576 bytes) -2016-06-13 19:38:44 INFO Executor:59 - Running task 0.0 in stage 193.0 (TID 76) -2016-06-13 19:38:44 INFO Executor:59 - Running task 1.0 in stage 193.0 (TID 77) -2016-06-13 19:38:44 INFO CacheManager:59 - Partition rdd_169_0 not found, computing it -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_156_0 locally -2016-06-13 19:38:44 INFO CacheManager:59 - Partition rdd_169_1 not found, computing it -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_156_1 locally -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=417726, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block rdd_169_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=420270, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block rdd_169_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added rdd_169_0 in memory on localhost:49848 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added rdd_169_1 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO Executor:59 - Finished task 0.0 in stage 193.0 (TID 76). 2971 bytes result sent to driver -2016-06-13 19:38:44 INFO Executor:59 - Finished task 1.0 in stage 193.0 (TID 77). 2971 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 193.0 (TID 76) in 15 ms on localhost (1/2) -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 193.0 (TID 77) in 14 ms on localhost (2/2) -2016-06-13 19:38:44 INFO DAGScheduler:59 - ResultStage 193 (fold at LDAOptimizer.scala:205) finished in 0.015 s -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 193.0, whose tasks have all completed, from pool -2016-06-13 19:38:44 INFO DAGScheduler:59 - Job 13 finished: fold at LDAOptimizer.scala:205, took 0.137019 s -2016-06-13 19:38:44 INFO MapPartitionsRDD:59 - Removing RDD 143 from persistence list -2016-06-13 19:38:44 INFO ZippedPartitionsRDD2:59 - Removing RDD 150 from persistence list -2016-06-13 19:38:44 INFO BlockManager:59 - Removing RDD 143 -2016-06-13 19:38:44 INFO BlockManager:59 - Removing RDD 150 -2016-06-13 19:38:44 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:38:44 INFO DAGScheduler:59 - Registering RDD 174 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Registering RDD 178 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Got job 14 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:44 INFO DAGScheduler:59 - Final stage: ResultStage 222(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 197, ShuffleMapStage 219, ShuffleMapStage 205, ShuffleMapStage 221, ShuffleMapStage 213, ShuffleMapStage 207, ShuffleMapStage 199, ShuffleMapStage 215, ShuffleMapStage 201, ShuffleMapStage 194, ShuffleMapStage 209, ShuffleMapStage 203, ShuffleMapStage 195, ShuffleMapStage 217, ShuffleMapStage 211) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 221) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 220 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[174] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(9728) called with curMem=417726, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 9.5 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(3782) called with curMem=427454, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.7 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:49848 (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 220 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[174] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Adding task set 220.0 with 2 tasks -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 220.0 (TID 78, localhost, PROCESS_LOCAL, 2565 bytes) -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 220.0 (TID 79, localhost, PROCESS_LOCAL, 2565 bytes) -2016-06-13 19:38:44 INFO Executor:59 - Running task 1.0 in stage 220.0 (TID 79) -2016-06-13 19:38:44 INFO Executor:59 - Running task 0.0 in stage 220.0 (TID 78) -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_169_0 locally -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_169_1 locally -2016-06-13 19:38:44 INFO Executor:59 - Finished task 1.0 in stage 220.0 (TID 79). 2254 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 220.0 (TID 79) in 51 ms on localhost (1/2) -2016-06-13 19:38:44 INFO Executor:59 - Finished task 0.0 in stage 220.0 (TID 78). 2254 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 220.0 (TID 78) in 66 ms on localhost (2/2) -2016-06-13 19:38:44 INFO DAGScheduler:59 - ShuffleMapStage 220 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.067 s -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 220.0, whose tasks have all completed, from pool -2016-06-13 19:38:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:44 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:44 INFO DAGScheduler:59 - waiting: Set(ResultStage 222, ShuffleMapStage 221) -2016-06-13 19:38:44 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:44 INFO DAGScheduler:59 - Missing parents for ResultStage 222: List(ShuffleMapStage 221) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 221: List() -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 221 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[178] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=431236, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(2287) called with curMem=435132, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:49848 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 221 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[178] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Adding task set 221.0 with 2 tasks -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 221.0 (TID 80, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 221.0 (TID 81, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:44 INFO Executor:59 - Running task 0.0 in stage 221.0 (TID 80) -2016-06-13 19:38:44 INFO Executor:59 - Running task 1.0 in stage 221.0 (TID 81) -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:44 INFO Executor:59 - Finished task 0.0 in stage 221.0 (TID 80). 2470 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 221.0 (TID 80) in 29 ms on localhost (1/2) -2016-06-13 19:38:44 INFO Executor:59 - Finished task 1.0 in stage 221.0 (TID 81). 2470 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 221.0 (TID 81) in 31 ms on localhost (2/2) -2016-06-13 19:38:44 INFO DAGScheduler:59 - ShuffleMapStage 221 (mapPartitions at GraphImpl.scala:235) finished in 0.032 s -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 221.0, whose tasks have all completed, from pool -2016-06-13 19:38:44 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:44 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:44 INFO DAGScheduler:59 - waiting: Set(ResultStage 222) -2016-06-13 19:38:44 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:44 INFO DAGScheduler:59 - Missing parents for ResultStage 222: List() -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting ResultStage 222 (MapPartitionsRDD[186] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(11144) called with curMem=437419, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 10.9 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(4182) called with curMem=448563, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 4.1 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:49848 (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 222 (MapPartitionsRDD[186] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Adding task set 222.0 with 2 tasks -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 222.0 (TID 82, localhost, PROCESS_LOCAL, 2617 bytes) -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 222.0 (TID 83, localhost, PROCESS_LOCAL, 2617 bytes) -2016-06-13 19:38:44 INFO Executor:59 - Running task 0.0 in stage 222.0 (TID 82) -2016-06-13 19:38:44 INFO Executor:59 - Running task 1.0 in stage 222.0 (TID 83) -2016-06-13 19:38:44 INFO CacheManager:59 - Partition rdd_182_1 not found, computing it -2016-06-13 19:38:44 INFO CacheManager:59 - Partition rdd_182_0 not found, computing it -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_169_1 locally -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_169_0 locally -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:44 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=452745, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block rdd_182_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=455193, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block rdd_182_0 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added rdd_182_1 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added rdd_182_0 in memory on localhost:49848 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO Executor:59 - Finished task 1.0 in stage 222.0 (TID 83). 2971 bytes result sent to driver -2016-06-13 19:38:44 INFO Executor:59 - Finished task 0.0 in stage 222.0 (TID 82). 2971 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 222.0 (TID 83) in 11 ms on localhost (1/2) -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 0.0 in stage 222.0 (TID 82) in 15 ms on localhost (2/2) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Removed TaskSet 222.0, whose tasks have all completed, from pool -2016-06-13 19:38:44 INFO DAGScheduler:59 - ResultStage 222 (fold at LDAOptimizer.scala:205) finished in 0.016 s -2016-06-13 19:38:44 INFO DAGScheduler:59 - Job 14 finished: fold at LDAOptimizer.scala:205, took 0.186013 s -2016-06-13 19:38:44 INFO MapPartitionsRDD:59 - Removing RDD 156 from persistence list -2016-06-13 19:38:44 INFO ZippedPartitionsRDD2:59 - Removing RDD 163 from persistence list -2016-06-13 19:38:44 INFO BlockManager:59 - Removing RDD 156 -2016-06-13 19:38:44 INFO BlockManager:59 - Removing RDD 163 -2016-06-13 19:38:44 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:38:44 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:38:44 INFO DAGScheduler:59 - Registering RDD 187 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Registering RDD 191 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Got job 15 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:44 INFO DAGScheduler:59 - Final stage: ResultStage 253(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 248, ShuffleMapStage 234, ShuffleMapStage 242, ShuffleMapStage 228, ShuffleMapStage 250, ShuffleMapStage 244, ShuffleMapStage 236, ShuffleMapStage 230, ShuffleMapStage 252, ShuffleMapStage 223, ShuffleMapStage 238, ShuffleMapStage 224, ShuffleMapStage 246, ShuffleMapStage 240, ShuffleMapStage 232, ShuffleMapStage 226) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 252) -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting ShuffleMapStage 251 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[187] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(10152) called with curMem=452745, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 9.9 KB, free 730.1 MB) -2016-06-13 19:38:44 INFO MemoryStore:59 - ensureFreeSpace(3864) called with curMem=462897, maxMem=766075207 -2016-06-13 19:38:44 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 3.8 KB, free 730.1 MB) -2016-06-13 19:38:44 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:49848 (size: 3.8 KB, free: 730.5 MB) -2016-06-13 19:38:44 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:44 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 251 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[187] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:44 INFO TaskSchedulerImpl:59 - Adding task set 251.0 with 2 tasks -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 0.0 in stage 251.0 (TID 84, localhost, PROCESS_LOCAL, 2606 bytes) -2016-06-13 19:38:44 INFO TaskSetManager:59 - Starting task 1.0 in stage 251.0 (TID 85, localhost, PROCESS_LOCAL, 2606 bytes) -2016-06-13 19:38:44 INFO Executor:59 - Running task 1.0 in stage 251.0 (TID 85) -2016-06-13 19:38:44 INFO Executor:59 - Running task 0.0 in stage 251.0 (TID 84) -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_182_0 locally -2016-06-13 19:38:44 INFO BlockManager:59 - Found block rdd_182_1 locally -2016-06-13 19:38:44 INFO Executor:59 - Finished task 1.0 in stage 251.0 (TID 85). 2254 bytes result sent to driver -2016-06-13 19:38:44 INFO TaskSetManager:59 - Finished task 1.0 in stage 251.0 (TID 85) in 22 ms on localhost (1/2) -2016-06-13 19:38:45 INFO Executor:59 - Finished task 0.0 in stage 251.0 (TID 84). 2254 bytes result sent to driver -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 251.0 (TID 84) in 56 ms on localhost (2/2) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 251.0, whose tasks have all completed, from pool -2016-06-13 19:38:45 INFO DAGScheduler:59 - ShuffleMapStage 251 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.058 s -2016-06-13 19:38:45 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:45 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:45 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 252, ResultStage 253) -2016-06-13 19:38:45 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:45 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 252: List() -2016-06-13 19:38:45 INFO DAGScheduler:59 - Missing parents for ResultStage 253: List(ShuffleMapStage 252) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting ShuffleMapStage 252 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[191] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=466761, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 3.8 KB, free 730.1 MB) -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(2284) called with curMem=470657, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:49848 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 252 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[191] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Adding task set 252.0 with 2 tasks -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 252.0 (TID 86, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 252.0 (TID 87, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:45 INFO Executor:59 - Running task 0.0 in stage 252.0 (TID 86) -2016-06-13 19:38:45 INFO Executor:59 - Running task 1.0 in stage 252.0 (TID 87) -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:45 INFO Executor:59 - Finished task 0.0 in stage 252.0 (TID 86). 2470 bytes result sent to driver -2016-06-13 19:38:45 INFO Executor:59 - Finished task 1.0 in stage 252.0 (TID 87). 2470 bytes result sent to driver -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 252.0 (TID 86) in 37 ms on localhost (1/2) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 252.0 (TID 87) in 36 ms on localhost (2/2) -2016-06-13 19:38:45 INFO DAGScheduler:59 - ShuffleMapStage 252 (mapPartitions at GraphImpl.scala:235) finished in 0.037 s -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 252.0, whose tasks have all completed, from pool -2016-06-13 19:38:45 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:45 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:45 INFO DAGScheduler:59 - waiting: Set(ResultStage 253) -2016-06-13 19:38:45 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:45 INFO DAGScheduler:59 - Missing parents for ResultStage 253: List() -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting ResultStage 253 (MapPartitionsRDD[199] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(11568) called with curMem=472941, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 11.3 KB, free 730.1 MB) -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(4283) called with curMem=484509, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.1 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:49848 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 253 (MapPartitionsRDD[199] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Adding task set 253.0 with 2 tasks -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 253.0 (TID 88, localhost, PROCESS_LOCAL, 2658 bytes) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 253.0 (TID 89, localhost, PROCESS_LOCAL, 2658 bytes) -2016-06-13 19:38:45 INFO Executor:59 - Running task 0.0 in stage 253.0 (TID 88) -2016-06-13 19:38:45 INFO Executor:59 - Running task 1.0 in stage 253.0 (TID 89) -2016-06-13 19:38:45 INFO CacheManager:59 - Partition rdd_195_0 not found, computing it -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_182_0 locally -2016-06-13 19:38:45 INFO CacheManager:59 - Partition rdd_195_1 not found, computing it -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_182_1 locally -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=488792, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block rdd_195_0 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=491336, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block rdd_195_1 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added rdd_195_0 in memory on localhost:49848 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO Executor:59 - Finished task 0.0 in stage 253.0 (TID 88). 2971 bytes result sent to driver -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added rdd_195_1 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO Executor:59 - Finished task 1.0 in stage 253.0 (TID 89). 2971 bytes result sent to driver -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 253.0 (TID 88) in 15 ms on localhost (1/2) -2016-06-13 19:38:45 INFO DAGScheduler:59 - ResultStage 253 (fold at LDAOptimizer.scala:205) finished in 0.016 s -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 253.0 (TID 89) in 15 ms on localhost (2/2) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 253.0, whose tasks have all completed, from pool -2016-06-13 19:38:45 INFO DAGScheduler:59 - Job 15 finished: fold at LDAOptimizer.scala:205, took 0.183010 s -2016-06-13 19:38:45 INFO MapPartitionsRDD:59 - Removing RDD 169 from persistence list -2016-06-13 19:38:45 INFO ZippedPartitionsRDD2:59 - Removing RDD 176 from persistence list -2016-06-13 19:38:45 INFO BlockManager:59 - Removing RDD 169 -2016-06-13 19:38:45 INFO BlockManager:59 - Removing RDD 176 -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:49848 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-06-13 19:38:45 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:49848 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:49848 in memory (size: 3.8 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:49848 in memory (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:49848 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:49848 in memory (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:49848 in memory (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:49848 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:49848 in memory (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:38:45 INFO DAGScheduler:59 - Registering RDD 200 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Registering RDD 204 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Got job 16 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:45 INFO DAGScheduler:59 - Final stage: ResultStage 286(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 269, ShuffleMapStage 263, ShuffleMapStage 255, ShuffleMapStage 285, ShuffleMapStage 277, ShuffleMapStage 271, ShuffleMapStage 257, ShuffleMapStage 279, ShuffleMapStage 265, ShuffleMapStage 281, ShuffleMapStage 273, ShuffleMapStage 267, ShuffleMapStage 259, ShuffleMapStage 275, ShuffleMapStage 261, ShuffleMapStage 283, ShuffleMapStage 254) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 285) -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting ShuffleMapStage 284 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[200] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(10584) called with curMem=383766, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 10.3 KB, free 730.2 MB) -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(3951) called with curMem=394350, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 3.9 KB, free 730.2 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:49848 (size: 3.9 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 284 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[200] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Adding task set 284.0 with 2 tasks -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:49848 in memory (size: 3.9 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 284.0 (TID 90, localhost, PROCESS_LOCAL, 2647 bytes) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 284.0 (TID 91, localhost, PROCESS_LOCAL, 2647 bytes) -2016-06-13 19:38:45 INFO Executor:59 - Running task 1.0 in stage 284.0 (TID 91) -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_195_1 locally -2016-06-13 19:38:45 INFO Executor:59 - Running task 0.0 in stage 284.0 (TID 90) -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_195_0 locally -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:49848 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:49848 in memory (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-06-13 19:38:45 INFO Executor:59 - Finished task 1.0 in stage 284.0 (TID 91). 2254 bytes result sent to driver -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:49848 in memory (size: 3.8 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 284.0 (TID 91) in 39 ms on localhost (1/2) -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:49848 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:49848 in memory (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:49848 in memory (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:49848 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:49848 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-06-13 19:38:45 INFO Executor:59 - Finished task 0.0 in stage 284.0 (TID 90). 2254 bytes result sent to driver -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 284.0 (TID 90) in 64 ms on localhost (2/2) -2016-06-13 19:38:45 INFO DAGScheduler:59 - ShuffleMapStage 284 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.066 s -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 284.0, whose tasks have all completed, from pool -2016-06-13 19:38:45 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:45 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:45 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 285, ResultStage 286) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:49848 in memory (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-06-13 19:38:45 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 285: List() -2016-06-13 19:38:45 INFO DAGScheduler:59 - Missing parents for ResultStage 286: List(ShuffleMapStage 285) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting ShuffleMapStage 285 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[204] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:49848 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=283902, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=276931, maxMem=766075207 -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:49848 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:49848 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 285 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[204] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Adding task set 285.0 with 2 tasks -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 285.0 (TID 92, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 285.0 (TID 93, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:45 INFO Executor:59 - Running task 1.0 in stage 285.0 (TID 93) -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:49848 in memory (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO Executor:59 - Running task 0.0 in stage 285.0 (TID 92) -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:49848 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO Executor:59 - Finished task 1.0 in stage 285.0 (TID 93). 2470 bytes result sent to driver -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 285.0 (TID 93) in 33 ms on localhost (1/2) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:49848 in memory (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:49848 in memory (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 19:38:45 INFO Executor:59 - Finished task 0.0 in stage 285.0 (TID 92). 2470 bytes result sent to driver -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:49848 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 285.0 (TID 92) in 50 ms on localhost (2/2) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 285.0, whose tasks have all completed, from pool -2016-06-13 19:38:45 INFO DAGScheduler:59 - ShuffleMapStage 285 (mapPartitions at GraphImpl.scala:235) finished in 0.051 s -2016-06-13 19:38:45 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:45 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:45 INFO DAGScheduler:59 - waiting: Set(ResultStage 286) -2016-06-13 19:38:45 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 19:38:45 INFO DAGScheduler:59 - Missing parents for ResultStage 286: List() -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting ResultStage 286 (MapPartitionsRDD[212] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:49848 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(12000) called with curMem=222921, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 11.7 KB, free 730.4 MB) -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(4400) called with curMem=234921, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:49848 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 286 (MapPartitionsRDD[212] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Adding task set 286.0 with 2 tasks -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 286.0 (TID 94, localhost, PROCESS_LOCAL, 2699 bytes) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 286.0 (TID 95, localhost, PROCESS_LOCAL, 2699 bytes) -2016-06-13 19:38:45 INFO Executor:59 - Running task 0.0 in stage 286.0 (TID 94) -2016-06-13 19:38:45 INFO Executor:59 - Running task 1.0 in stage 286.0 (TID 95) -2016-06-13 19:38:45 INFO CacheManager:59 - Partition rdd_208_0 not found, computing it -2016-06-13 19:38:45 INFO CacheManager:59 - Partition rdd_208_1 not found, computing it -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_195_0 locally -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_195_1 locally -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=239321, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block rdd_208_1 stored as values in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=241769, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block rdd_208_0 stored as values in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added rdd_208_1 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added rdd_208_0 in memory on localhost:49848 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO Executor:59 - Finished task 1.0 in stage 286.0 (TID 95). 2971 bytes result sent to driver -2016-06-13 19:38:45 INFO Executor:59 - Finished task 0.0 in stage 286.0 (TID 94). 2971 bytes result sent to driver -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 286.0 (TID 95) in 13 ms on localhost (1/2) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 286.0 (TID 94) in 14 ms on localhost (2/2) -2016-06-13 19:38:45 INFO DAGScheduler:59 - ResultStage 286 (fold at LDAOptimizer.scala:205) finished in 0.015 s -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 286.0, whose tasks have all completed, from pool -2016-06-13 19:38:45 INFO DAGScheduler:59 - Job 16 finished: fold at LDAOptimizer.scala:205, took 0.223283 s -2016-06-13 19:38:45 INFO MapPartitionsRDD:59 - Removing RDD 182 from persistence list -2016-06-13 19:38:45 INFO BlockManager:59 - Removing RDD 182 -2016-06-13 19:38:45 INFO ZippedPartitionsRDD2:59 - Removing RDD 189 from persistence list -2016-06-13 19:38:45 INFO BlockManager:59 - Removing RDD 189 -2016-06-13 19:38:45 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:38:45 INFO DAGScheduler:59 - Registering RDD 213 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Registering RDD 217 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Got job 17 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:45 INFO DAGScheduler:59 - Final stage: ResultStage 321(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 320, ShuffleMapStage 306, ShuffleMapStage 300, ShuffleMapStage 292, ShuffleMapStage 314, ShuffleMapStage 308, ShuffleMapStage 294, ShuffleMapStage 316, ShuffleMapStage 287, ShuffleMapStage 302, ShuffleMapStage 288, ShuffleMapStage 318, ShuffleMapStage 310, ShuffleMapStage 304, ShuffleMapStage 296, ShuffleMapStage 290, ShuffleMapStage 312, ShuffleMapStage 298) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 320) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting ShuffleMapStage 319 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[213] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=239321, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 10.8 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(4062) called with curMem=250329, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:49848 (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 319 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[213] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Adding task set 319.0 with 2 tasks -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 319.0 (TID 96, localhost, PROCESS_LOCAL, 2688 bytes) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 319.0 (TID 97, localhost, PROCESS_LOCAL, 2688 bytes) -2016-06-13 19:38:45 INFO Executor:59 - Running task 1.0 in stage 319.0 (TID 97) -2016-06-13 19:38:45 INFO Executor:59 - Running task 0.0 in stage 319.0 (TID 96) -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_208_0 locally -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_208_1 locally -2016-06-13 19:38:45 INFO Executor:59 - Finished task 1.0 in stage 319.0 (TID 97). 2254 bytes result sent to driver -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 319.0 (TID 97) in 24 ms on localhost (1/2) -2016-06-13 19:38:45 INFO Executor:59 - Finished task 0.0 in stage 319.0 (TID 96). 2254 bytes result sent to driver -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 319.0 (TID 96) in 27 ms on localhost (2/2) -2016-06-13 19:38:45 INFO DAGScheduler:59 - ShuffleMapStage 319 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.027 s -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 319.0, whose tasks have all completed, from pool -2016-06-13 19:38:45 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:45 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:45 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 320, ResultStage 321) -2016-06-13 19:38:45 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:45 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 320: List() -2016-06-13 19:38:45 INFO DAGScheduler:59 - Missing parents for ResultStage 321: List(ShuffleMapStage 320) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting ShuffleMapStage 320 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[217] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=254391, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=258287, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:49848 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 320 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[217] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Adding task set 320.0 with 2 tasks -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 320.0 (TID 98, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 320.0 (TID 99, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:45 INFO Executor:59 - Running task 0.0 in stage 320.0 (TID 98) -2016-06-13 19:38:45 INFO Executor:59 - Running task 1.0 in stage 320.0 (TID 99) -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:45 INFO Executor:59 - Finished task 0.0 in stage 320.0 (TID 98). 2470 bytes result sent to driver -2016-06-13 19:38:45 INFO Executor:59 - Finished task 1.0 in stage 320.0 (TID 99). 2470 bytes result sent to driver -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 320.0 (TID 98) in 25 ms on localhost (1/2) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 320.0 (TID 99) in 25 ms on localhost (2/2) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 320.0, whose tasks have all completed, from pool -2016-06-13 19:38:45 INFO DAGScheduler:59 - ShuffleMapStage 320 (mapPartitions at GraphImpl.scala:235) finished in 0.026 s -2016-06-13 19:38:45 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:45 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:45 INFO DAGScheduler:59 - waiting: Set(ResultStage 321) -2016-06-13 19:38:45 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:45 INFO DAGScheduler:59 - Missing parents for ResultStage 321: List() -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting ResultStage 321 (MapPartitionsRDD[225] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(12424) called with curMem=260573, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 12.1 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(4491) called with curMem=272997, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:49848 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 321 (MapPartitionsRDD[225] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Adding task set 321.0 with 2 tasks -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 321.0 (TID 100, localhost, PROCESS_LOCAL, 2740 bytes) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 321.0 (TID 101, localhost, PROCESS_LOCAL, 2740 bytes) -2016-06-13 19:38:45 INFO Executor:59 - Running task 0.0 in stage 321.0 (TID 100) -2016-06-13 19:38:45 INFO Executor:59 - Running task 1.0 in stage 321.0 (TID 101) -2016-06-13 19:38:45 INFO CacheManager:59 - Partition rdd_221_0 not found, computing it -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_208_0 locally -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:45 INFO CacheManager:59 - Partition rdd_221_1 not found, computing it -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_208_1 locally -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=277488, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block rdd_221_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added rdd_221_0 in memory on localhost:49848 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=280032, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block rdd_221_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO Executor:59 - Finished task 0.0 in stage 321.0 (TID 100). 2971 bytes result sent to driver -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added rdd_221_1 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO Executor:59 - Finished task 1.0 in stage 321.0 (TID 101). 2971 bytes result sent to driver -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 321.0 (TID 100) in 13 ms on localhost (1/2) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 321.0 (TID 101) in 13 ms on localhost (2/2) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 321.0, whose tasks have all completed, from pool -2016-06-13 19:38:45 INFO DAGScheduler:59 - ResultStage 321 (fold at LDAOptimizer.scala:205) finished in 0.014 s -2016-06-13 19:38:45 INFO DAGScheduler:59 - Job 17 finished: fold at LDAOptimizer.scala:205, took 0.135199 s -2016-06-13 19:38:45 INFO MapPartitionsRDD:59 - Removing RDD 195 from persistence list -2016-06-13 19:38:45 INFO BlockManager:59 - Removing RDD 195 -2016-06-13 19:38:45 INFO ZippedPartitionsRDD2:59 - Removing RDD 202 from persistence list -2016-06-13 19:38:45 INFO BlockManager:59 - Removing RDD 202 -2016-06-13 19:38:45 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:38:45 INFO DAGScheduler:59 - Registering RDD 226 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Registering RDD 230 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Got job 18 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:45 INFO DAGScheduler:59 - Final stage: ResultStage 358(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 335, ShuffleMapStage 357, ShuffleMapStage 343, ShuffleMapStage 322, ShuffleMapStage 329, ShuffleMapStage 323, ShuffleMapStage 351, ShuffleMapStage 345, ShuffleMapStage 337, ShuffleMapStage 331, ShuffleMapStage 353, ShuffleMapStage 339, ShuffleMapStage 325, ShuffleMapStage 355, ShuffleMapStage 347, ShuffleMapStage 341, ShuffleMapStage 333, ShuffleMapStage 327, ShuffleMapStage 349) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 357) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting ShuffleMapStage 356 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[226] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(11440) called with curMem=277488, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 11.2 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(4162) called with curMem=288928, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 4.1 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:49848 (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 356 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[226] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Adding task set 356.0 with 2 tasks -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 356.0 (TID 102, localhost, PROCESS_LOCAL, 2729 bytes) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 356.0 (TID 103, localhost, PROCESS_LOCAL, 2729 bytes) -2016-06-13 19:38:45 INFO Executor:59 - Running task 1.0 in stage 356.0 (TID 103) -2016-06-13 19:38:45 INFO Executor:59 - Running task 0.0 in stage 356.0 (TID 102) -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_221_0 locally -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_221_1 locally -2016-06-13 19:38:45 INFO Executor:59 - Finished task 0.0 in stage 356.0 (TID 102). 2254 bytes result sent to driver -2016-06-13 19:38:45 INFO Executor:59 - Finished task 1.0 in stage 356.0 (TID 103). 2254 bytes result sent to driver -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 356.0 (TID 102) in 29 ms on localhost (1/2) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 356.0 (TID 103) in 29 ms on localhost (2/2) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 356.0, whose tasks have all completed, from pool -2016-06-13 19:38:45 INFO DAGScheduler:59 - ShuffleMapStage 356 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.031 s -2016-06-13 19:38:45 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:45 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:45 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 357, ResultStage 358) -2016-06-13 19:38:45 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:45 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 357: List() -2016-06-13 19:38:45 INFO DAGScheduler:59 - Missing parents for ResultStage 358: List(ShuffleMapStage 357) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting ShuffleMapStage 357 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[230] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=293090, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=296986, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:49848 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 357 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[230] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Adding task set 357.0 with 2 tasks -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 357.0 (TID 104, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 357.0 (TID 105, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:45 INFO Executor:59 - Running task 0.0 in stage 357.0 (TID 104) -2016-06-13 19:38:45 INFO Executor:59 - Running task 1.0 in stage 357.0 (TID 105) -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:45 INFO Executor:59 - Finished task 1.0 in stage 357.0 (TID 105). 2470 bytes result sent to driver -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 357.0 (TID 105) in 27 ms on localhost (1/2) -2016-06-13 19:38:45 INFO Executor:59 - Finished task 0.0 in stage 357.0 (TID 104). 2470 bytes result sent to driver -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 357.0 (TID 104) in 31 ms on localhost (2/2) -2016-06-13 19:38:45 INFO DAGScheduler:59 - ShuffleMapStage 357 (mapPartitions at GraphImpl.scala:235) finished in 0.031 s -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 357.0, whose tasks have all completed, from pool -2016-06-13 19:38:45 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:45 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:45 INFO DAGScheduler:59 - waiting: Set(ResultStage 358) -2016-06-13 19:38:45 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:45 INFO DAGScheduler:59 - Missing parents for ResultStage 358: List() -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting ResultStage 358 (MapPartitionsRDD[238] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(12856) called with curMem=299272, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 12.6 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(4586) called with curMem=312128, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:49848 (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 358 (MapPartitionsRDD[238] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Adding task set 358.0 with 2 tasks -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 358.0 (TID 106, localhost, PROCESS_LOCAL, 2781 bytes) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 358.0 (TID 107, localhost, PROCESS_LOCAL, 2781 bytes) -2016-06-13 19:38:45 INFO Executor:59 - Running task 1.0 in stage 358.0 (TID 107) -2016-06-13 19:38:45 INFO Executor:59 - Running task 0.0 in stage 358.0 (TID 106) -2016-06-13 19:38:45 INFO CacheManager:59 - Partition rdd_234_0 not found, computing it -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_221_0 locally -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:45 INFO CacheManager:59 - Partition rdd_234_1 not found, computing it -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=316714, maxMem=766075207 -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_221_1 locally -2016-06-13 19:38:45 INFO MemoryStore:59 - Block rdd_234_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added rdd_234_0 in memory on localhost:49848 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO Executor:59 - Finished task 0.0 in stage 358.0 (TID 106). 2971 bytes result sent to driver -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 358.0 (TID 106) in 17 ms on localhost (1/2) -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=319258, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block rdd_234_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added rdd_234_1 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO Executor:59 - Finished task 1.0 in stage 358.0 (TID 107). 2971 bytes result sent to driver -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 358.0 (TID 107) in 27 ms on localhost (2/2) -2016-06-13 19:38:45 INFO DAGScheduler:59 - ResultStage 358 (fold at LDAOptimizer.scala:205) finished in 0.028 s -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 358.0, whose tasks have all completed, from pool -2016-06-13 19:38:45 INFO DAGScheduler:59 - Job 18 finished: fold at LDAOptimizer.scala:205, took 0.152474 s -2016-06-13 19:38:45 INFO MapPartitionsRDD:59 - Removing RDD 208 from persistence list -2016-06-13 19:38:45 INFO BlockManager:59 - Removing RDD 208 -2016-06-13 19:38:45 INFO ZippedPartitionsRDD2:59 - Removing RDD 215 from persistence list -2016-06-13 19:38:45 INFO BlockManager:59 - Removing RDD 215 -2016-06-13 19:38:45 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:38:45 INFO DAGScheduler:59 - Registering RDD 239 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Registering RDD 243 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Got job 19 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:45 INFO DAGScheduler:59 - Final stage: ResultStage 397(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 386, ShuffleMapStage 372, ShuffleMapStage 394, ShuffleMapStage 380, ShuffleMapStage 359, ShuffleMapStage 366, ShuffleMapStage 396, ShuffleMapStage 360, ShuffleMapStage 388, ShuffleMapStage 382, ShuffleMapStage 374, ShuffleMapStage 368, ShuffleMapStage 390, ShuffleMapStage 376, ShuffleMapStage 362, ShuffleMapStage 392, ShuffleMapStage 384, ShuffleMapStage 378, ShuffleMapStage 370, ShuffleMapStage 364) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 396) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting ShuffleMapStage 395 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[239] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(11864) called with curMem=316714, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 11.6 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(4268) called with curMem=328578, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:49848 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 395 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[239] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Adding task set 395.0 with 2 tasks -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 395.0 (TID 108, localhost, PROCESS_LOCAL, 2770 bytes) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 395.0 (TID 109, localhost, PROCESS_LOCAL, 2770 bytes) -2016-06-13 19:38:45 INFO Executor:59 - Running task 1.0 in stage 395.0 (TID 109) -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_234_1 locally -2016-06-13 19:38:45 INFO Executor:59 - Running task 0.0 in stage 395.0 (TID 108) -2016-06-13 19:38:45 INFO Executor:59 - Finished task 1.0 in stage 395.0 (TID 109). 2254 bytes result sent to driver -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 395.0 (TID 109) in 30 ms on localhost (1/2) -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_234_0 locally -2016-06-13 19:38:45 INFO Executor:59 - Finished task 0.0 in stage 395.0 (TID 108). 2254 bytes result sent to driver -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 395.0 (TID 108) in 55 ms on localhost (2/2) -2016-06-13 19:38:45 INFO DAGScheduler:59 - ShuffleMapStage 395 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.056 s -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 395.0, whose tasks have all completed, from pool -2016-06-13 19:38:45 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:45 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:45 INFO DAGScheduler:59 - waiting: Set(ResultStage 397, ShuffleMapStage 396) -2016-06-13 19:38:45 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:45 INFO DAGScheduler:59 - Missing parents for ResultStage 397: List(ShuffleMapStage 396) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 396: List() -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting ShuffleMapStage 396 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[243] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=332846, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(2288) called with curMem=336742, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:49848 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 396 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[243] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Adding task set 396.0 with 2 tasks -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 396.0 (TID 110, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 396.0 (TID 111, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:45 INFO Executor:59 - Running task 0.0 in stage 396.0 (TID 110) -2016-06-13 19:38:45 INFO Executor:59 - Running task 1.0 in stage 396.0 (TID 111) -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:45 INFO Executor:59 - Finished task 0.0 in stage 396.0 (TID 110). 2470 bytes result sent to driver -2016-06-13 19:38:45 INFO Executor:59 - Finished task 1.0 in stage 396.0 (TID 111). 2470 bytes result sent to driver -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 396.0 (TID 110) in 23 ms on localhost (1/2) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 396.0 (TID 111) in 24 ms on localhost (2/2) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 396.0, whose tasks have all completed, from pool -2016-06-13 19:38:45 INFO DAGScheduler:59 - ShuffleMapStage 396 (mapPartitions at GraphImpl.scala:235) finished in 0.024 s -2016-06-13 19:38:45 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:45 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:45 INFO DAGScheduler:59 - waiting: Set(ResultStage 397) -2016-06-13 19:38:45 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:45 INFO DAGScheduler:59 - Missing parents for ResultStage 397: List() -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting ResultStage 397 (MapPartitionsRDD[251] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(13280) called with curMem=339030, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 13.0 KB, free 730.3 MB) -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(4699) called with curMem=352310, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 4.6 KB, free 730.2 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:49848 (size: 4.6 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 397 (MapPartitionsRDD[251] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Adding task set 397.0 with 2 tasks -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 397.0 (TID 112, localhost, PROCESS_LOCAL, 2822 bytes) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 397.0 (TID 113, localhost, PROCESS_LOCAL, 2822 bytes) -2016-06-13 19:38:45 INFO Executor:59 - Running task 0.0 in stage 397.0 (TID 112) -2016-06-13 19:38:45 INFO Executor:59 - Running task 1.0 in stage 397.0 (TID 113) -2016-06-13 19:38:45 INFO CacheManager:59 - Partition rdd_247_0 not found, computing it -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_234_0 locally -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:45 INFO CacheManager:59 - Partition rdd_247_1 not found, computing it -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_234_1 locally -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=357009, maxMem=766075207 -2016-06-13 19:38:45 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:45 INFO MemoryStore:59 - Block rdd_247_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added rdd_247_0 in memory on localhost:49848 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO Executor:59 - Finished task 0.0 in stage 397.0 (TID 112). 2971 bytes result sent to driver -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 0.0 in stage 397.0 (TID 112) in 14 ms on localhost (1/2) -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=359553, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block rdd_247_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added rdd_247_1 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO Executor:59 - Finished task 1.0 in stage 397.0 (TID 113). 2971 bytes result sent to driver -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 397.0 (TID 113) in 19 ms on localhost (2/2) -2016-06-13 19:38:45 INFO DAGScheduler:59 - ResultStage 397 (fold at LDAOptimizer.scala:205) finished in 0.020 s -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Removed TaskSet 397.0, whose tasks have all completed, from pool -2016-06-13 19:38:45 INFO DAGScheduler:59 - Job 19 finished: fold at LDAOptimizer.scala:205, took 0.161434 s -2016-06-13 19:38:45 INFO MapPartitionsRDD:59 - Removing RDD 221 from persistence list -2016-06-13 19:38:45 INFO ZippedPartitionsRDD2:59 - Removing RDD 228 from persistence list -2016-06-13 19:38:45 INFO BlockManager:59 - Removing RDD 221 -2016-06-13 19:38:45 INFO BlockManager:59 - Removing RDD 228 -2016-06-13 19:38:45 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:38:45 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 155 bytes -2016-06-13 19:38:45 INFO DAGScheduler:59 - Registering RDD 252 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Registering RDD 256 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Got job 20 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:45 INFO DAGScheduler:59 - Final stage: ResultStage 438(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 407, ShuffleMapStage 437, ShuffleMapStage 401, ShuffleMapStage 423, ShuffleMapStage 409, ShuffleMapStage 431, ShuffleMapStage 417, ShuffleMapStage 403, ShuffleMapStage 433, ShuffleMapStage 425, ShuffleMapStage 419, ShuffleMapStage 411, ShuffleMapStage 405, ShuffleMapStage 427, ShuffleMapStage 398, ShuffleMapStage 413, ShuffleMapStage 435, ShuffleMapStage 399, ShuffleMapStage 429, ShuffleMapStage 421, ShuffleMapStage 415) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 437) -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting ShuffleMapStage 436 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[252] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(12296) called with curMem=357009, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 12.0 KB, free 730.2 MB) -2016-06-13 19:38:45 INFO MemoryStore:59 - ensureFreeSpace(4356) called with curMem=369305, maxMem=766075207 -2016-06-13 19:38:45 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.2 MB) -2016-06-13 19:38:45 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:49848 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:38:45 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 436 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[252] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:45 INFO TaskSchedulerImpl:59 - Adding task set 436.0 with 2 tasks -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 436.0 (TID 114, localhost, PROCESS_LOCAL, 2811 bytes) -2016-06-13 19:38:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 436.0 (TID 115, localhost, PROCESS_LOCAL, 2811 bytes) -2016-06-13 19:38:45 INFO Executor:59 - Running task 1.0 in stage 436.0 (TID 115) -2016-06-13 19:38:45 INFO Executor:59 - Running task 0.0 in stage 436.0 (TID 114) -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_247_1 locally -2016-06-13 19:38:45 INFO BlockManager:59 - Found block rdd_247_0 locally -2016-06-13 19:38:45 INFO Executor:59 - Finished task 1.0 in stage 436.0 (TID 115). 2254 bytes result sent to driver -2016-06-13 19:38:45 INFO TaskSetManager:59 - Finished task 1.0 in stage 436.0 (TID 115) in 32 ms on localhost (1/2) -2016-06-13 19:38:46 INFO Executor:59 - Finished task 0.0 in stage 436.0 (TID 114). 2254 bytes result sent to driver -2016-06-13 19:38:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 436.0 (TID 114) in 53 ms on localhost (2/2) -2016-06-13 19:38:46 INFO DAGScheduler:59 - ShuffleMapStage 436 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.054 s -2016-06-13 19:38:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 436.0, whose tasks have all completed, from pool -2016-06-13 19:38:46 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:46 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:46 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 437, ResultStage 438) -2016-06-13 19:38:46 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:46 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 437: List() -2016-06-13 19:38:46 INFO DAGScheduler:59 - Missing parents for ResultStage 438: List(ShuffleMapStage 437) -2016-06-13 19:38:46 INFO DAGScheduler:59 - Submitting ShuffleMapStage 437 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[256] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=373661, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(2288) called with curMem=377557, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:49848 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 437 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[256] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:46 INFO TaskSchedulerImpl:59 - Adding task set 437.0 with 2 tasks -2016-06-13 19:38:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 437.0 (TID 116, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 437.0 (TID 117, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:46 INFO Executor:59 - Running task 1.0 in stage 437.0 (TID 117) -2016-06-13 19:38:46 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:46 INFO Executor:59 - Running task 0.0 in stage 437.0 (TID 116) -2016-06-13 19:38:46 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:46 INFO Executor:59 - Finished task 0.0 in stage 437.0 (TID 116). 2470 bytes result sent to driver -2016-06-13 19:38:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 437.0 (TID 116) in 30 ms on localhost (1/2) -2016-06-13 19:38:46 INFO Executor:59 - Finished task 1.0 in stage 437.0 (TID 117). 2470 bytes result sent to driver -2016-06-13 19:38:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 437.0 (TID 117) in 33 ms on localhost (2/2) -2016-06-13 19:38:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 437.0, whose tasks have all completed, from pool -2016-06-13 19:38:46 INFO DAGScheduler:59 - ShuffleMapStage 437 (mapPartitions at GraphImpl.scala:235) finished in 0.034 s -2016-06-13 19:38:46 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:46 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:46 INFO DAGScheduler:59 - waiting: Set(ResultStage 438) -2016-06-13 19:38:46 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:46 INFO DAGScheduler:59 - Missing parents for ResultStage 438: List() -2016-06-13 19:38:46 INFO DAGScheduler:59 - Submitting ResultStage 438 (MapPartitionsRDD[264] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(13712) called with curMem=379845, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 13.4 KB, free 730.2 MB) -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(4770) called with curMem=393557, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 4.7 KB, free 730.2 MB) -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:49848 (size: 4.7 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO SparkContext:59 - Created broadcast 61 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 438 (MapPartitionsRDD[264] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:46 INFO TaskSchedulerImpl:59 - Adding task set 438.0 with 2 tasks -2016-06-13 19:38:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 438.0 (TID 118, localhost, PROCESS_LOCAL, 2863 bytes) -2016-06-13 19:38:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 438.0 (TID 119, localhost, PROCESS_LOCAL, 2863 bytes) -2016-06-13 19:38:46 INFO Executor:59 - Running task 0.0 in stage 438.0 (TID 118) -2016-06-13 19:38:46 INFO Executor:59 - Running task 1.0 in stage 438.0 (TID 119) -2016-06-13 19:38:46 INFO CacheManager:59 - Partition rdd_260_0 not found, computing it -2016-06-13 19:38:46 INFO CacheManager:59 - Partition rdd_260_1 not found, computing it -2016-06-13 19:38:46 INFO BlockManager:59 - Found block rdd_247_0 locally -2016-06-13 19:38:46 INFO BlockManager:59 - Found block rdd_247_1 locally -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=398327, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block rdd_260_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=400871, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block rdd_260_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Added rdd_260_0 in memory on localhost:49848 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Added rdd_260_1 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO Executor:59 - Finished task 0.0 in stage 438.0 (TID 118). 2971 bytes result sent to driver -2016-06-13 19:38:46 INFO Executor:59 - Finished task 1.0 in stage 438.0 (TID 119). 2971 bytes result sent to driver -2016-06-13 19:38:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 438.0 (TID 118) in 11 ms on localhost (1/2) -2016-06-13 19:38:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 438.0 (TID 119) in 10 ms on localhost (2/2) -2016-06-13 19:38:46 INFO DAGScheduler:59 - ResultStage 438 (fold at LDAOptimizer.scala:205) finished in 0.011 s -2016-06-13 19:38:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 438.0, whose tasks have all completed, from pool -2016-06-13 19:38:46 INFO DAGScheduler:59 - Job 20 finished: fold at LDAOptimizer.scala:205, took 0.154331 s -2016-06-13 19:38:46 INFO MapPartitionsRDD:59 - Removing RDD 234 from persistence list -2016-06-13 19:38:46 INFO BlockManager:59 - Removing RDD 234 -2016-06-13 19:38:46 INFO ZippedPartitionsRDD2:59 - Removing RDD 241 from persistence list -2016-06-13 19:38:46 INFO BlockManager:59 - Removing RDD 241 -2016-06-13 19:38:46 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:38:46 INFO DAGScheduler:59 - Registering RDD 265 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:46 INFO DAGScheduler:59 - Registering RDD 269 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:46 INFO DAGScheduler:59 - Got job 21 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:46 INFO DAGScheduler:59 - Final stage: ResultStage 481(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:46 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 466, ShuffleMapStage 458, ShuffleMapStage 452, ShuffleMapStage 444, ShuffleMapStage 474, ShuffleMapStage 460, ShuffleMapStage 446, ShuffleMapStage 468, ShuffleMapStage 439, ShuffleMapStage 454, ShuffleMapStage 476, ShuffleMapStage 440, ShuffleMapStage 470, ShuffleMapStage 462, ShuffleMapStage 456, ShuffleMapStage 448, ShuffleMapStage 478, ShuffleMapStage 442, ShuffleMapStage 464, ShuffleMapStage 450, ShuffleMapStage 480, ShuffleMapStage 472) -2016-06-13 19:38:46 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 480) -2016-06-13 19:38:46 INFO DAGScheduler:59 - Submitting ShuffleMapStage 479 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[265] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(12720) called with curMem=398327, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 12.4 KB, free 730.2 MB) -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(4470) called with curMem=411047, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.2 MB) -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:49848 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 479 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[265] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:46 INFO TaskSchedulerImpl:59 - Adding task set 479.0 with 2 tasks -2016-06-13 19:38:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 479.0 (TID 120, localhost, PROCESS_LOCAL, 2852 bytes) -2016-06-13 19:38:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 479.0 (TID 121, localhost, PROCESS_LOCAL, 2852 bytes) -2016-06-13 19:38:46 INFO Executor:59 - Running task 0.0 in stage 479.0 (TID 120) -2016-06-13 19:38:46 INFO BlockManager:59 - Found block rdd_260_0 locally -2016-06-13 19:38:46 INFO Executor:59 - Running task 1.0 in stage 479.0 (TID 121) -2016-06-13 19:38:46 INFO BlockManager:59 - Found block rdd_260_1 locally -2016-06-13 19:38:46 INFO Executor:59 - Finished task 0.0 in stage 479.0 (TID 120). 2254 bytes result sent to driver -2016-06-13 19:38:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 479.0 (TID 120) in 32 ms on localhost (1/2) -2016-06-13 19:38:46 INFO Executor:59 - Finished task 1.0 in stage 479.0 (TID 121). 2254 bytes result sent to driver -2016-06-13 19:38:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 479.0 (TID 121) in 65 ms on localhost (2/2) -2016-06-13 19:38:46 INFO DAGScheduler:59 - ShuffleMapStage 479 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.066 s -2016-06-13 19:38:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 479.0, whose tasks have all completed, from pool -2016-06-13 19:38:46 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:46 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:46 INFO DAGScheduler:59 - waiting: Set(ResultStage 481, ShuffleMapStage 480) -2016-06-13 19:38:46 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:46 INFO DAGScheduler:59 - Missing parents for ResultStage 481: List(ShuffleMapStage 480) -2016-06-13 19:38:46 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 480: List() -2016-06-13 19:38:46 INFO DAGScheduler:59 - Submitting ShuffleMapStage 480 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[269] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=415517, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(2285) called with curMem=419413, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:49848 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 480 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[269] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:46 INFO TaskSchedulerImpl:59 - Adding task set 480.0 with 2 tasks -2016-06-13 19:38:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 480.0 (TID 122, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 480.0 (TID 123, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:46 INFO Executor:59 - Running task 0.0 in stage 480.0 (TID 122) -2016-06-13 19:38:46 INFO Executor:59 - Running task 1.0 in stage 480.0 (TID 123) -2016-06-13 19:38:46 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:38:46 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:46 INFO Executor:59 - Finished task 0.0 in stage 480.0 (TID 122). 2470 bytes result sent to driver -2016-06-13 19:38:46 INFO Executor:59 - Finished task 1.0 in stage 480.0 (TID 123). 2470 bytes result sent to driver -2016-06-13 19:38:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 480.0 (TID 122) in 23 ms on localhost (1/2) -2016-06-13 19:38:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 480.0 (TID 123) in 22 ms on localhost (2/2) -2016-06-13 19:38:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 480.0, whose tasks have all completed, from pool -2016-06-13 19:38:46 INFO DAGScheduler:59 - ShuffleMapStage 480 (mapPartitions at GraphImpl.scala:235) finished in 0.024 s -2016-06-13 19:38:46 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:46 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:46 INFO DAGScheduler:59 - waiting: Set(ResultStage 481) -2016-06-13 19:38:46 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:46 INFO DAGScheduler:59 - Missing parents for ResultStage 481: List() -2016-06-13 19:38:46 INFO DAGScheduler:59 - Submitting ResultStage 481 (MapPartitionsRDD[277] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(14136) called with curMem=421698, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 13.8 KB, free 730.2 MB) -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(4890) called with curMem=435834, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 4.8 KB, free 730.2 MB) -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:49848 (size: 4.8 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 481 (MapPartitionsRDD[277] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:46 INFO TaskSchedulerImpl:59 - Adding task set 481.0 with 2 tasks -2016-06-13 19:38:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 481.0 (TID 124, localhost, PROCESS_LOCAL, 2904 bytes) -2016-06-13 19:38:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 481.0 (TID 125, localhost, PROCESS_LOCAL, 2904 bytes) -2016-06-13 19:38:46 INFO Executor:59 - Running task 0.0 in stage 481.0 (TID 124) -2016-06-13 19:38:46 INFO Executor:59 - Running task 1.0 in stage 481.0 (TID 125) -2016-06-13 19:38:46 INFO CacheManager:59 - Partition rdd_273_0 not found, computing it -2016-06-13 19:38:46 INFO BlockManager:59 - Found block rdd_260_0 locally -2016-06-13 19:38:46 INFO CacheManager:59 - Partition rdd_273_1 not found, computing it -2016-06-13 19:38:46 INFO BlockManager:59 - Found block rdd_260_1 locally -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=440724, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block rdd_273_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=443268, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block rdd_273_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Added rdd_273_0 in memory on localhost:49848 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Added rdd_273_1 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO Executor:59 - Finished task 0.0 in stage 481.0 (TID 124). 2971 bytes result sent to driver -2016-06-13 19:38:46 INFO Executor:59 - Finished task 1.0 in stage 481.0 (TID 125). 2971 bytes result sent to driver -2016-06-13 19:38:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 481.0 (TID 124) in 12 ms on localhost (1/2) -2016-06-13 19:38:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 481.0 (TID 125) in 13 ms on localhost (2/2) -2016-06-13 19:38:46 INFO DAGScheduler:59 - ResultStage 481 (fold at LDAOptimizer.scala:205) finished in 0.015 s -2016-06-13 19:38:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 481.0, whose tasks have all completed, from pool -2016-06-13 19:38:46 INFO DAGScheduler:59 - Job 21 finished: fold at LDAOptimizer.scala:205, took 0.164356 s -2016-06-13 19:38:46 INFO MapPartitionsRDD:59 - Removing RDD 247 from persistence list -2016-06-13 19:38:46 INFO ZippedPartitionsRDD2:59 - Removing RDD 254 from persistence list -2016-06-13 19:38:46 INFO BlockManager:59 - Removing RDD 247 -2016-06-13 19:38:46 INFO BlockManager:59 - Removing RDD 254 -2016-06-13 19:38:46 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 41 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 40 is 155 bytes -2016-06-13 19:38:46 INFO DAGScheduler:59 - Registering RDD 278 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:46 INFO DAGScheduler:59 - Registering RDD 282 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:46 INFO DAGScheduler:59 - Got job 22 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:38:46 INFO DAGScheduler:59 - Final stage: ResultStage 526(fold at LDAOptimizer.scala:205) -2016-06-13 19:38:46 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 487, ShuffleMapStage 509, ShuffleMapStage 503, ShuffleMapStage 495, ShuffleMapStage 525, ShuffleMapStage 489, ShuffleMapStage 517, ShuffleMapStage 511, ShuffleMapStage 482, ShuffleMapStage 497, ShuffleMapStage 519, ShuffleMapStage 483, ShuffleMapStage 505, ShuffleMapStage 491, ShuffleMapStage 521, ShuffleMapStage 513, ShuffleMapStage 507, ShuffleMapStage 499, ShuffleMapStage 493, ShuffleMapStage 485, ShuffleMapStage 515, ShuffleMapStage 501, ShuffleMapStage 523) -2016-06-13 19:38:46 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 525) -2016-06-13 19:38:46 INFO DAGScheduler:59 - Submitting ShuffleMapStage 524 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[278] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(13152) called with curMem=440724, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 12.8 KB, free 730.2 MB) -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(4564) called with curMem=453876, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.1 MB) -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:49848 (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 524 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[278] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:38:46 INFO TaskSchedulerImpl:59 - Adding task set 524.0 with 2 tasks -2016-06-13 19:38:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 524.0 (TID 126, localhost, PROCESS_LOCAL, 2893 bytes) -2016-06-13 19:38:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 524.0 (TID 127, localhost, PROCESS_LOCAL, 2893 bytes) -2016-06-13 19:38:46 INFO Executor:59 - Running task 0.0 in stage 524.0 (TID 126) -2016-06-13 19:38:46 INFO Executor:59 - Running task 1.0 in stage 524.0 (TID 127) -2016-06-13 19:38:46 INFO BlockManager:59 - Found block rdd_273_1 locally -2016-06-13 19:38:46 INFO BlockManager:59 - Found block rdd_273_0 locally -2016-06-13 19:38:46 INFO Executor:59 - Finished task 0.0 in stage 524.0 (TID 126). 2254 bytes result sent to driver -2016-06-13 19:38:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 524.0 (TID 126) in 24 ms on localhost (1/2) -2016-06-13 19:38:46 INFO Executor:59 - Finished task 1.0 in stage 524.0 (TID 127). 2254 bytes result sent to driver -2016-06-13 19:38:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 524.0 (TID 127) in 43 ms on localhost (2/2) -2016-06-13 19:38:46 INFO DAGScheduler:59 - ShuffleMapStage 524 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.044 s -2016-06-13 19:38:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 524.0, whose tasks have all completed, from pool -2016-06-13 19:38:46 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:46 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:46 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 525, ResultStage 526) -2016-06-13 19:38:46 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:46 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 525: List() -2016-06-13 19:38:46 INFO DAGScheduler:59 - Missing parents for ResultStage 526: List(ShuffleMapStage 525) -2016-06-13 19:38:46 INFO DAGScheduler:59 - Submitting ShuffleMapStage 525 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[282] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=458440, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 3.8 KB, free 730.1 MB) -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(2289) called with curMem=462336, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:49848 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 525 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[282] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:38:46 INFO TaskSchedulerImpl:59 - Adding task set 525.0 with 2 tasks -2016-06-13 19:38:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 525.0 (TID 128, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 525.0 (TID 129, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:38:46 INFO Executor:59 - Running task 0.0 in stage 525.0 (TID 128) -2016-06-13 19:38:46 INFO Executor:59 - Running task 1.0 in stage 525.0 (TID 129) -2016-06-13 19:38:46 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:38:46 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:38:46 INFO Executor:59 - Finished task 1.0 in stage 525.0 (TID 129). 2470 bytes result sent to driver -2016-06-13 19:38:46 INFO Executor:59 - Finished task 0.0 in stage 525.0 (TID 128). 2470 bytes result sent to driver -2016-06-13 19:38:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 525.0 (TID 129) in 22 ms on localhost (1/2) -2016-06-13 19:38:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 525.0 (TID 128) in 23 ms on localhost (2/2) -2016-06-13 19:38:46 INFO DAGScheduler:59 - ShuffleMapStage 525 (mapPartitions at GraphImpl.scala:235) finished in 0.023 s -2016-06-13 19:38:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 525.0, whose tasks have all completed, from pool -2016-06-13 19:38:46 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:38:46 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:38:46 INFO DAGScheduler:59 - waiting: Set(ResultStage 526) -2016-06-13 19:38:46 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:38:46 INFO DAGScheduler:59 - Missing parents for ResultStage 526: List() -2016-06-13 19:38:46 INFO DAGScheduler:59 - Submitting ResultStage 526 (MapPartitionsRDD[290] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(14568) called with curMem=464625, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 14.2 KB, free 730.1 MB) -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(4990) called with curMem=479193, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 4.9 KB, free 730.1 MB) -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:49848 (size: 4.9 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO SparkContext:59 - Created broadcast 67 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 526 (MapPartitionsRDD[290] at values at LDAOptimizer.scala:205) -2016-06-13 19:38:46 INFO TaskSchedulerImpl:59 - Adding task set 526.0 with 2 tasks -2016-06-13 19:38:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 526.0 (TID 130, localhost, PROCESS_LOCAL, 2945 bytes) -2016-06-13 19:38:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 526.0 (TID 131, localhost, PROCESS_LOCAL, 2945 bytes) -2016-06-13 19:38:46 INFO Executor:59 - Running task 0.0 in stage 526.0 (TID 130) -2016-06-13 19:38:46 INFO Executor:59 - Running task 1.0 in stage 526.0 (TID 131) -2016-06-13 19:38:46 INFO CacheManager:59 - Partition rdd_286_0 not found, computing it -2016-06-13 19:38:46 INFO CacheManager:59 - Partition rdd_286_1 not found, computing it -2016-06-13 19:38:46 INFO BlockManager:59 - Found block rdd_273_0 locally -2016-06-13 19:38:46 INFO BlockManager:59 - Found block rdd_273_1 locally -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=484183, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block rdd_286_0 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=486727, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block rdd_286_1 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Added rdd_286_0 in memory on localhost:49848 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Added rdd_286_1 in memory on localhost:49848 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO Executor:59 - Finished task 0.0 in stage 526.0 (TID 130). 2971 bytes result sent to driver -2016-06-13 19:38:46 INFO Executor:59 - Finished task 1.0 in stage 526.0 (TID 131). 2971 bytes result sent to driver -2016-06-13 19:38:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 526.0 (TID 130) in 12 ms on localhost (1/2) -2016-06-13 19:38:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 526.0 (TID 131) in 11 ms on localhost (2/2) -2016-06-13 19:38:46 INFO DAGScheduler:59 - ResultStage 526 (fold at LDAOptimizer.scala:205) finished in 0.012 s -2016-06-13 19:38:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 526.0, whose tasks have all completed, from pool -2016-06-13 19:38:46 INFO DAGScheduler:59 - Job 22 finished: fold at LDAOptimizer.scala:205, took 0.148724 s -2016-06-13 19:38:46 INFO SparkContext:59 - Starting job: collect at LDAModel.scala:528 -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 41 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 40 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 43 is 155 bytes -2016-06-13 19:38:46 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 42 is 155 bytes -2016-06-13 19:38:46 INFO DAGScheduler:59 - Got job 23 (collect at LDAModel.scala:528) with 2 output partitions -2016-06-13 19:38:46 INFO DAGScheduler:59 - Final stage: ResultStage 571(collect at LDAModel.scala:528) -2016-06-13 19:38:46 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 538, ShuffleMapStage 560, ShuffleMapStage 546, ShuffleMapStage 540, ShuffleMapStage 568, ShuffleMapStage 532, ShuffleMapStage 562, ShuffleMapStage 554, ShuffleMapStage 548, ShuffleMapStage 570, ShuffleMapStage 534, ShuffleMapStage 556, ShuffleMapStage 527, ShuffleMapStage 542, ShuffleMapStage 564, ShuffleMapStage 528, ShuffleMapStage 558, ShuffleMapStage 550, ShuffleMapStage 544, ShuffleMapStage 536, ShuffleMapStage 566, ShuffleMapStage 530, ShuffleMapStage 552) -2016-06-13 19:38:46 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:38:46 INFO DAGScheduler:59 - Submitting ResultStage 571 (MapPartitionsRDD[293] at map at LDAModel.scala:526), which has no missing parents -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(14256) called with curMem=489175, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 13.9 KB, free 730.1 MB) -2016-06-13 19:38:46 INFO MemoryStore:59 - ensureFreeSpace(4871) called with curMem=503431, maxMem=766075207 -2016-06-13 19:38:46 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 4.8 KB, free 730.1 MB) -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:49848 (size: 4.8 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:38:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 571 (MapPartitionsRDD[293] at map at LDAModel.scala:526) -2016-06-13 19:38:46 INFO TaskSchedulerImpl:59 - Adding task set 571.0 with 2 tasks -2016-06-13 19:38:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 571.0 (TID 132, localhost, PROCESS_LOCAL, 2945 bytes) -2016-06-13 19:38:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 571.0 (TID 133, localhost, PROCESS_LOCAL, 2945 bytes) -2016-06-13 19:38:46 INFO Executor:59 - Running task 1.0 in stage 571.0 (TID 133) -2016-06-13 19:38:46 INFO Executor:59 - Running task 0.0 in stage 571.0 (TID 132) -2016-06-13 19:38:46 INFO BlockManager:59 - Found block rdd_286_1 locally -2016-06-13 19:38:46 INFO Executor:59 - Finished task 1.0 in stage 571.0 (TID 133). 2612 bytes result sent to driver -2016-06-13 19:38:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 571.0 (TID 133) in 16 ms on localhost (1/2) -2016-06-13 19:38:46 INFO BlockManager:59 - Found block rdd_286_0 locally -2016-06-13 19:38:46 INFO Executor:59 - Finished task 0.0 in stage 571.0 (TID 132). 2680 bytes result sent to driver -2016-06-13 19:38:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 571.0 (TID 132) in 31 ms on localhost (2/2) -2016-06-13 19:38:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 571.0, whose tasks have all completed, from pool -2016-06-13 19:38:46 INFO DAGScheduler:59 - ResultStage 571 (collect at LDAModel.scala:528) finished in 0.033 s -2016-06-13 19:38:46 INFO DAGScheduler:59 - Job 23 finished: collect at LDAModel.scala:528, took 0.138404 s -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:49848 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:49848 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:49848 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:49848 in memory (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:49848 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:49848 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:49848 in memory (size: 3.9 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:49848 in memory (size: 4.8 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:49848 in memory (size: 4.9 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:49848 in memory (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:49848 in memory (size: 4.8 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:49848 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:49848 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:49848 in memory (size: 4.7 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:49848 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:49848 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:49848 in memory (size: 4.6 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:49848 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:49848 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 19:38:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:49848 in memory (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:49848 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-06-13 19:38:46 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:49848 in memory (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:38:46 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-06-13 19:38:46 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 19:38:46 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 19:38:47 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 19:38:47 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 19:38:47 INFO BlockManager:59 - BlockManager stopped -2016-06-13 19:38:47 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 19:38:47 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 19:38:47 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 19:38:47 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 19:38:47 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 19:38:47 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 19:38:47 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-7b4dee71-2128-43dd-8d5f-4ebe4fb25ea1 -2016-06-13 19:40:52 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 19:40:52 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 19:40:53 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 19:40:53 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 19:40:53 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 19:40:54 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 19:40:54 INFO Remoting:74 - Starting remoting -2016-06-13 19:40:54 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:49870] -2016-06-13 19:40:54 INFO Utils:59 - Successfully started service 'sparkDriver' on port 49870. -2016-06-13 19:40:54 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 19:40:54 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 19:40:55 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-60cbb1ed-4d37-4532-8dfb-9fd786fca269 -2016-06-13 19:40:55 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 19:40:55 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-102f2053-859d-41bb-839b-9aaecbba369d\httpd-0e0cce58-946a-465f-a770-17e9f0c614f0 -2016-06-13 19:40:55 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 19:40:55 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:40:55 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:49871 -2016-06-13 19:40:55 INFO Utils:59 - Successfully started service 'HTTP file server' on port 49871. -2016-06-13 19:40:55 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 19:40:55 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:40:55 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 19:40:55 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 19:40:55 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 19:40:55 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 19:40:55 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 19:40:56 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 49890. -2016-06-13 19:40:56 INFO NettyBlockTransferService:59 - Server created on 49890 -2016-06-13 19:40:56 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 19:40:56 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:49890 with 730.6 MB RAM, BlockManagerId(driver, localhost, 49890) -2016-06-13 19:40:56 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 19:40:57 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 19:40:57 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 19:40:57 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 19:40:57 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 19:40:57 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:49890 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 19:40:57 INFO SparkContext:59 - Created broadcast 0 from textFile at LDAFromWebSuite.scala:22 -2016-06-13 19:40:58 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 19:40:58 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 19:40:58 INFO SparkContext:59 - Starting job: zipWithIndex at LDAFromWebSuite.scala:25 -2016-06-13 19:40:58 INFO DAGScheduler:59 - Got job 0 (zipWithIndex at LDAFromWebSuite.scala:25) with 1 output partitions -2016-06-13 19:40:58 INFO DAGScheduler:59 - Final stage: ResultStage 0(zipWithIndex at LDAFromWebSuite.scala:25) -2016-06-13 19:40:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:40:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:40:58 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at LDAFromWebSuite.scala:23), which has no missing parents -2016-06-13 19:40:58 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=171580, maxMem=766075207 -2016-06-13 19:40:58 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:40:58 INFO MemoryStore:59 - ensureFreeSpace(1939) called with curMem=174820, maxMem=766075207 -2016-06-13 19:40:58 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1939.0 B, free 730.4 MB) -2016-06-13 19:40:58 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:49890 (size: 1939.0 B, free: 730.6 MB) -2016-06-13 19:40:58 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:40:58 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at LDAFromWebSuite.scala:23) -2016-06-13 19:40:58 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 19:40:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2197 bytes) -2016-06-13 19:40:58 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 19:40:58 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:0+132 -2016-06-13 19:40:58 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 19:40:58 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 19:40:58 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 19:40:58 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 19:40:58 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 19:40:59 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 19:40:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 181 ms on localhost (1/1) -2016-06-13 19:40:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 19:40:59 INFO DAGScheduler:59 - ResultStage 0 (zipWithIndex at LDAFromWebSuite.scala:25) finished in 0.217 s -2016-06-13 19:40:59 INFO DAGScheduler:59 - Job 0 finished: zipWithIndex at LDAFromWebSuite.scala:25, took 0.388451 s -2016-06-13 19:40:59 INFO SparkContext:59 - Starting job: take at LDAOptimizer.scala:142 -2016-06-13 19:40:59 INFO DAGScheduler:59 - Got job 1 (take at LDAOptimizer.scala:142) with 1 output partitions -2016-06-13 19:40:59 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at LDAOptimizer.scala:142) -2016-06-13 19:40:59 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:40:59 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:40:59 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at map at LDAFromWebSuite.scala:25), which has no missing parents -2016-06-13 19:40:59 INFO MemoryStore:59 - ensureFreeSpace(3696) called with curMem=176759, maxMem=766075207 -2016-06-13 19:40:59 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-06-13 19:40:59 INFO MemoryStore:59 - ensureFreeSpace(2071) called with curMem=180455, maxMem=766075207 -2016-06-13 19:40:59 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:40:59 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:49890 (size: 2.0 KB, free: 730.6 MB) -2016-06-13 19:40:59 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:40:59 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at map at LDAFromWebSuite.scala:25) -2016-06-13 19:40:59 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 19:40:59 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2307 bytes) -2016-06-13 19:40:59 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 19:40:59 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-06-13 19:40:59 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:0+132 -2016-06-13 19:40:59 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=182526, maxMem=766075207 -2016-06-13 19:40:59 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 1224.0 B, free 730.4 MB) -2016-06-13 19:40:59 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:49890 (size: 1224.0 B, free: 730.6 MB) -2016-06-13 19:40:59 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2949 bytes result sent to driver -2016-06-13 19:40:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 147 ms on localhost (1/1) -2016-06-13 19:40:59 INFO DAGScheduler:59 - ResultStage 1 (take at LDAOptimizer.scala:142) finished in 0.148 s -2016-06-13 19:40:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 19:40:59 INFO DAGScheduler:59 - Job 1 finished: take at LDAOptimizer.scala:142, took 0.175803 s -2016-06-13 19:41:00 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:00 INFO DAGScheduler:59 - Registering RDD 19 (map at GraphImpl.scala:104) -2016-06-13 19:41:00 INFO DAGScheduler:59 - Registering RDD 23 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:41:00 INFO DAGScheduler:59 - Registering RDD 11 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:41:00 INFO DAGScheduler:59 - Registering RDD 6 (mapPartitionsWithIndex at LDAOptimizer.scala:128) -2016-06-13 19:41:00 INFO DAGScheduler:59 - Got job 2 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:00 INFO DAGScheduler:59 - Final stage: ResultStage 6(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:00 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 5, ShuffleMapStage 3, ShuffleMapStage 4) -2016-06-13 19:41:00 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 5, ShuffleMapStage 3, ShuffleMapStage 4) -2016-06-13 19:41:00 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[19] at map at GraphImpl.scala:104), which has no missing parents -2016-06-13 19:41:00 INFO MemoryStore:59 - ensureFreeSpace(5752) called with curMem=183750, maxMem=766075207 -2016-06-13 19:41:00 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.6 KB, free 730.4 MB) -2016-06-13 19:41:00 INFO MemoryStore:59 - ensureFreeSpace(3051) called with curMem=189502, maxMem=766075207 -2016-06-13 19:41:00 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 19:41:00 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:49890 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 19:41:00 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[19] at map at GraphImpl.scala:104) -2016-06-13 19:41:00 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 19:41:00 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[11] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:41:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:41:00 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=192553, maxMem=766075207 -2016-06-13 19:41:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:41:00 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.4 MB) -2016-06-13 19:41:00 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 3) -2016-06-13 19:41:00 INFO MemoryStore:59 - ensureFreeSpace(2760) called with curMem=197617, maxMem=766075207 -2016-06-13 19:41:00 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:41:00 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:49890 (size: 2.7 KB, free: 730.6 MB) -2016-06-13 19:41:00 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[11] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:41:00 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 19:41:00 INFO DAGScheduler:59 - Submitting ShuffleMapStage 5 (MapPartitionsRDD[6] at mapPartitionsWithIndex at LDAOptimizer.scala:128), which has no missing parents -2016-06-13 19:41:00 INFO MemoryStore:59 - ensureFreeSpace(4880) called with curMem=200377, maxMem=766075207 -2016-06-13 19:41:00 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-06-13 19:41:00 INFO MemoryStore:59 - ensureFreeSpace(2640) called with curMem=205257, maxMem=766075207 -2016-06-13 19:41:00 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-06-13 19:41:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 4, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:41:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 5, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:41:00 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 4) -2016-06-13 19:41:00 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:49890 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:41:00 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 5 (MapPartitionsRDD[6] at mapPartitionsWithIndex at LDAOptimizer.scala:128) -2016-06-13 19:41:00 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 19:41:00 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:41:00 INFO CacheManager:59 - Partition rdd_17_1 not found, computing it -2016-06-13 19:41:00 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-06-13 19:41:00 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:132+132 -2016-06-13 19:41:00 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 5) -2016-06-13 19:41:00 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-06-13 19:41:00 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=207897, maxMem=766075207 -2016-06-13 19:41:00 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 880.0 B, free 730.4 MB) -2016-06-13 19:41:00 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:49890 (size: 880.0 B, free: 730.6 MB) -2016-06-13 19:41:00 INFO MemoryStore:59 - ensureFreeSpace(3296) called with curMem=208777, maxMem=766075207 -2016-06-13 19:41:00 INFO MemoryStore:59 - Block rdd_17_1 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:41:00 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 19:41:00 INFO BlockManagerInfo:59 - Added rdd_17_1 in memory on localhost:49890 (size: 3.2 KB, free: 730.6 MB) -2016-06-13 19:41:00 INFO CacheManager:59 - Partition rdd_17_0 not found, computing it -2016-06-13 19:41:00 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:41:00 INFO MemoryStore:59 - ensureFreeSpace(3520) called with curMem=212073, maxMem=766075207 -2016-06-13 19:41:00 INFO MemoryStore:59 - Block rdd_17_0 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-06-13 19:41:00 INFO BlockManagerInfo:59 - Added rdd_17_0 in memory on localhost:49890 (size: 3.4 KB, free: 730.6 MB) -2016-06-13 19:41:00 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 5). 2254 bytes result sent to driver -2016-06-13 19:41:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 6, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:41:00 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 4). 2254 bytes result sent to driver -2016-06-13 19:41:00 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 6) -2016-06-13 19:41:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 7, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:41:00 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 7) -2016-06-13 19:41:00 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:41:00 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 19:41:00 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 19:41:00 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 19:41:00 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 2834 bytes result sent to driver -2016-06-13 19:41:00 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 6). 2254 bytes result sent to driver -2016-06-13 19:41:00 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 3). 2889 bytes result sent to driver -2016-06-13 19:41:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 5) in 462 ms on localhost (1/2) -2016-06-13 19:41:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 4) in 480 ms on localhost (2/2) -2016-06-13 19:41:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 19:41:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 547 ms on localhost (1/2) -2016-06-13 19:41:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 6) in 185 ms on localhost (1/2) -2016-06-13 19:41:00 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at VertexRDD.scala:358) finished in 0.501 s -2016-06-13 19:41:00 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:00 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 5, ShuffleMapStage 2) -2016-06-13 19:41:00 INFO DAGScheduler:59 - waiting: Set(ResultStage 6, ShuffleMapStage 3) -2016-06-13 19:41:00 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 3) in 548 ms on localhost (2/2) -2016-06-13 19:41:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 19:41:00 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 7). 2254 bytes result sent to driver -2016-06-13 19:41:00 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List(ShuffleMapStage 5, ShuffleMapStage 3) -2016-06-13 19:41:00 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List(ShuffleMapStage 2) -2016-06-13 19:41:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 7) in 192 ms on localhost (2/2) -2016-06-13 19:41:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 19:41:00 INFO DAGScheduler:59 - ShuffleMapStage 2 (map at GraphImpl.scala:104) finished in 0.578 s -2016-06-13 19:41:00 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:00 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 5) -2016-06-13 19:41:00 INFO DAGScheduler:59 - waiting: Set(ResultStage 6, ShuffleMapStage 3) -2016-06-13 19:41:00 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:00 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List(ShuffleMapStage 5, ShuffleMapStage 3) -2016-06-13 19:41:00 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List() -2016-06-13 19:41:00 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[23] at mapPartitions at VertexRDD.scala:358), which is now runnable -2016-06-13 19:41:00 INFO MemoryStore:59 - ensureFreeSpace(2784) called with curMem=215593, maxMem=766075207 -2016-06-13 19:41:00 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:41:00 INFO MemoryStore:59 - ensureFreeSpace(1698) called with curMem=218377, maxMem=766075207 -2016-06-13 19:41:00 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1698.0 B, free 730.4 MB) -2016-06-13 19:41:00 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:49890 (size: 1698.0 B, free: 730.6 MB) -2016-06-13 19:41:00 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 3 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[23] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:41:00 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 19:41:00 INFO DAGScheduler:59 - ShuffleMapStage 5 (mapPartitionsWithIndex at LDAOptimizer.scala:128) finished in 0.496 s -2016-06-13 19:41:00 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:00 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 3) -2016-06-13 19:41:00 INFO DAGScheduler:59 - waiting: Set(ResultStage 6) -2016-06-13 19:41:00 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 1890 bytes) -2016-06-13 19:41:00 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List(ShuffleMapStage 3) -2016-06-13 19:41:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 1890 bytes) -2016-06-13 19:41:00 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 9) -2016-06-13 19:41:00 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 8) -2016-06-13 19:41:00 INFO CacheManager:59 - Partition rdd_21_1 not found, computing it -2016-06-13 19:41:00 INFO CacheManager:59 - Partition rdd_21_0 not found, computing it -2016-06-13 19:41:00 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:00 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:00 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-06-13 19:41:00 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-06-13 19:41:00 INFO MemoryStore:59 - ensureFreeSpace(3416) called with curMem=220075, maxMem=766075207 -2016-06-13 19:41:00 INFO MemoryStore:59 - Block rdd_21_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 19:41:00 INFO MemoryStore:59 - ensureFreeSpace(3408) called with curMem=223491, maxMem=766075207 -2016-06-13 19:41:00 INFO MemoryStore:59 - Block rdd_21_0 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 19:41:00 INFO BlockManagerInfo:59 - Added rdd_21_1 in memory on localhost:49890 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:41:00 INFO BlockManagerInfo:59 - Added rdd_21_0 in memory on localhost:49890 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:41:00 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 9). 1955 bytes result sent to driver -2016-06-13 19:41:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 9) in 112 ms on localhost (1/2) -2016-06-13 19:41:00 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 8). 1955 bytes result sent to driver -2016-06-13 19:41:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 8) in 154 ms on localhost (2/2) -2016-06-13 19:41:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 19:41:00 INFO DAGScheduler:59 - ShuffleMapStage 3 (mapPartitions at VertexRDD.scala:358) finished in 0.155 s -2016-06-13 19:41:00 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:00 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:00 INFO DAGScheduler:59 - waiting: Set(ResultStage 6) -2016-06-13 19:41:00 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:00 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List() -2016-06-13 19:41:00 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[30] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:00 INFO MemoryStore:59 - ensureFreeSpace(5584) called with curMem=226899, maxMem=766075207 -2016-06-13 19:41:00 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.4 MB) -2016-06-13 19:41:00 INFO MemoryStore:59 - ensureFreeSpace(2802) called with curMem=232483, maxMem=766075207 -2016-06-13 19:41:00 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:41:00 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:49890 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:41:00 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[30] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:00 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-06-13 19:41:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 10, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:41:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:41:00 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 11) -2016-06-13 19:41:00 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 10) -2016-06-13 19:41:00 INFO CacheManager:59 - Partition rdd_26_1 not found, computing it -2016-06-13 19:41:00 INFO CacheManager:59 - Partition rdd_26_0 not found, computing it -2016-06-13 19:41:00 INFO CacheManager:59 - Partition rdd_14_1 not found, computing it -2016-06-13 19:41:00 INFO CacheManager:59 - Partition rdd_14_0 not found, computing it -2016-06-13 19:41:00 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:00 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:00 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:00 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:00 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:00 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:00 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:00 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:00 INFO MemoryStore:59 - ensureFreeSpace(2496) called with curMem=235285, maxMem=766075207 -2016-06-13 19:41:00 INFO MemoryStore:59 - Block rdd_14_0 stored as values in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 19:41:00 INFO MemoryStore:59 - ensureFreeSpace(2400) called with curMem=237781, maxMem=766075207 -2016-06-13 19:41:00 INFO BlockManagerInfo:59 - Added rdd_14_0 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:00 INFO MemoryStore:59 - Block rdd_14_1 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 19:41:00 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:00 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:00 INFO BlockManagerInfo:59 - Added rdd_14_1 in memory on localhost:49890 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:41:00 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:00 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:00 INFO MemoryStore:59 - ensureFreeSpace(2496) called with curMem=240181, maxMem=766075207 -2016-06-13 19:41:00 INFO MemoryStore:59 - Block rdd_26_0 stored as values in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 19:41:00 INFO MemoryStore:59 - ensureFreeSpace(2400) called with curMem=242677, maxMem=766075207 -2016-06-13 19:41:00 INFO BlockManagerInfo:59 - Added rdd_26_0 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:00 INFO MemoryStore:59 - Block rdd_26_1 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 19:41:00 INFO BlockManagerInfo:59 - Added rdd_26_1 in memory on localhost:49890 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:41:00 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 11). 1931 bytes result sent to driver -2016-06-13 19:41:00 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 10). 1931 bytes result sent to driver -2016-06-13 19:41:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 11) in 72 ms on localhost (1/2) -2016-06-13 19:41:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 10) in 77 ms on localhost (2/2) -2016-06-13 19:41:00 INFO DAGScheduler:59 - ResultStage 6 (fold at LDAOptimizer.scala:205) finished in 0.078 s -2016-06-13 19:41:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 19:41:00 INFO DAGScheduler:59 - Job 2 finished: fold at LDAOptimizer.scala:205, took 0.934841 s -2016-06-13 19:41:01 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:01 INFO DAGScheduler:59 - Registering RDD 31 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Registering RDD 35 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Got job 3 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:01 INFO DAGScheduler:59 - Final stage: ResultStage 13(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12, ShuffleMapStage 10, ShuffleMapStage 7, ShuffleMapStage 8) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Submitting ShuffleMapStage 11 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[31] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(4360) called with curMem=245077, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(2388) called with curMem=249437, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.3 MB) -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:49890 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 11 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[31] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:01 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 19:41:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 12, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:49890 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 13, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:41:01 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 13) -2016-06-13 19:41:01 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 12) -2016-06-13 19:41:01 INFO BlockManager:59 - Found block rdd_26_1 locally -2016-06-13 19:41:01 INFO BlockManager:59 - Found block rdd_26_0 locally -2016-06-13 19:41:01 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 13). 2254 bytes result sent to driver -2016-06-13 19:41:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 13) in 43 ms on localhost (1/2) -2016-06-13 19:41:01 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:49890 in memory (size: 1698.0 B, free: 730.5 MB) -2016-06-13 19:41:01 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 19:41:01 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 12). 2254 bytes result sent to driver -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:49890 in memory (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 19:41:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 12) in 88 ms on localhost (2/2) -2016-06-13 19:41:01 INFO DAGScheduler:59 - ShuffleMapStage 11 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.089 s -2016-06-13 19:41:01 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:01 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 19:41:01 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 12, ResultStage 13) -2016-06-13 19:41:01 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:49890 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 19:41:01 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 12: List() -2016-06-13 19:41:01 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List(ShuffleMapStage 12) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[35] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:49890 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=214810, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 19:41:01 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=218706, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:49890 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:49890 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[35] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:01 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:49890 in memory (size: 1939.0 B, free: 730.5 MB) -2016-06-13 19:41:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 14, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:01 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 19:41:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 15, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:01 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 15) -2016-06-13 19:41:01 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 14) -2016-06-13 19:41:01 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:01 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:01 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 15). 2470 bytes result sent to driver -2016-06-13 19:41:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 15) in 43 ms on localhost (1/2) -2016-06-13 19:41:01 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 14). 2470 bytes result sent to driver -2016-06-13 19:41:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 14) in 55 ms on localhost (2/2) -2016-06-13 19:41:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 19:41:01 INFO DAGScheduler:59 - ShuffleMapStage 12 (mapPartitions at GraphImpl.scala:235) finished in 0.059 s -2016-06-13 19:41:01 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:01 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:01 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-06-13 19:41:01 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:01 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-06-13 19:41:01 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[43] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(6432) called with curMem=210046, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 6.3 KB, free 730.4 MB) -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(3040) called with curMem=216478, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:49890 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[43] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:01 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 19:41:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 16, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:41:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:41:01 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 16) -2016-06-13 19:41:01 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 17) -2016-06-13 19:41:01 INFO CacheManager:59 - Partition rdd_39_0 not found, computing it -2016-06-13 19:41:01 INFO CacheManager:59 - Partition rdd_39_1 not found, computing it -2016-06-13 19:41:01 INFO BlockManager:59 - Found block rdd_26_0 locally -2016-06-13 19:41:01 INFO BlockManager:59 - Found block rdd_26_1 locally -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=219518, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block rdd_39_1 stored as values in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=221966, maxMem=766075207 -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Added rdd_39_1 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO MemoryStore:59 - Block rdd_39_0 stored as values in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Added rdd_39_0 in memory on localhost:49890 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 17). 2971 bytes result sent to driver -2016-06-13 19:41:01 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 16). 2971 bytes result sent to driver -2016-06-13 19:41:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 17) in 25 ms on localhost (1/2) -2016-06-13 19:41:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 16) in 28 ms on localhost (2/2) -2016-06-13 19:41:01 INFO DAGScheduler:59 - ResultStage 13 (fold at LDAOptimizer.scala:205) finished in 0.028 s -2016-06-13 19:41:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 19:41:01 INFO DAGScheduler:59 - Job 3 finished: fold at LDAOptimizer.scala:205, took 0.299077 s -2016-06-13 19:41:01 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:41:01 INFO DAGScheduler:59 - Registering RDD 44 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Registering RDD 48 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Got job 4 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:01 INFO DAGScheduler:59 - Final stage: ResultStage 22(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15, ShuffleMapStage 19, ShuffleMapStage 17, ShuffleMapStage 21, ShuffleMapStage 14) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 21) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Submitting ShuffleMapStage 20 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[44] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=224510, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.3 KB, free 730.4 MB) -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(2727) called with curMem=229958, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:49890 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 20 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[44] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:01 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-06-13 19:41:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 18, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:41:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 19, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:41:01 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 19) -2016-06-13 19:41:01 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 18) -2016-06-13 19:41:01 INFO BlockManager:59 - Found block rdd_39_0 locally -2016-06-13 19:41:01 INFO BlockManager:59 - Found block rdd_39_1 locally -2016-06-13 19:41:01 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 18). 2254 bytes result sent to driver -2016-06-13 19:41:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 18) in 35 ms on localhost (1/2) -2016-06-13 19:41:01 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 19). 2254 bytes result sent to driver -2016-06-13 19:41:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 19) in 50 ms on localhost (2/2) -2016-06-13 19:41:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 19:41:01 INFO DAGScheduler:59 - ShuffleMapStage 20 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.052 s -2016-06-13 19:41:01 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:01 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:01 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 21, ResultStage 22) -2016-06-13 19:41:01 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:01 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 21: List() -2016-06-13 19:41:01 INFO DAGScheduler:59 - Missing parents for ResultStage 22: List(ShuffleMapStage 21) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Submitting ShuffleMapStage 21 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[48] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=232685, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=236581, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:49890 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 21 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[48] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:01 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 19:41:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 20, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 21, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:01 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 20) -2016-06-13 19:41:01 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 21) -2016-06-13 19:41:01 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:41:01 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:01 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 20). 2470 bytes result sent to driver -2016-06-13 19:41:01 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 21). 2470 bytes result sent to driver -2016-06-13 19:41:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 21) in 38 ms on localhost (1/2) -2016-06-13 19:41:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 20) in 40 ms on localhost (2/2) -2016-06-13 19:41:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 19:41:01 INFO DAGScheduler:59 - ShuffleMapStage 21 (mapPartitions at GraphImpl.scala:235) finished in 0.040 s -2016-06-13 19:41:01 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:01 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:01 INFO DAGScheduler:59 - waiting: Set(ResultStage 22) -2016-06-13 19:41:01 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:01 INFO DAGScheduler:59 - Missing parents for ResultStage 22: List() -2016-06-13 19:41:01 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[56] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(6864) called with curMem=238867, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 6.7 KB, free 730.4 MB) -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(3157) called with curMem=245731, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.3 MB) -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:49890 (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[56] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:01 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-06-13 19:41:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 22, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:41:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 23, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:41:01 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 23) -2016-06-13 19:41:01 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 22) -2016-06-13 19:41:01 INFO CacheManager:59 - Partition rdd_52_0 not found, computing it -2016-06-13 19:41:01 INFO BlockManager:59 - Found block rdd_39_0 locally -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:01 INFO CacheManager:59 - Partition rdd_52_1 not found, computing it -2016-06-13 19:41:01 INFO BlockManager:59 - Found block rdd_39_1 locally -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=248888, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block rdd_52_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Added rdd_52_0 in memory on localhost:49890 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=251432, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block rdd_52_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:41:01 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 22). 2971 bytes result sent to driver -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Added rdd_52_1 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 23). 2971 bytes result sent to driver -2016-06-13 19:41:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 22) in 24 ms on localhost (1/2) -2016-06-13 19:41:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 23) in 24 ms on localhost (2/2) -2016-06-13 19:41:01 INFO DAGScheduler:59 - ResultStage 22 (fold at LDAOptimizer.scala:205) finished in 0.026 s -2016-06-13 19:41:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 19:41:01 INFO DAGScheduler:59 - Job 4 finished: fold at LDAOptimizer.scala:205, took 0.190687 s -2016-06-13 19:41:01 INFO ZippedPartitionsRDD2:59 - Removing RDD 26 from persistence list -2016-06-13 19:41:01 INFO ZippedPartitionsRDD2:59 - Removing RDD 33 from persistence list -2016-06-13 19:41:01 INFO BlockManager:59 - Removing RDD 26 -2016-06-13 19:41:01 INFO BlockManager:59 - Removing RDD 33 -2016-06-13 19:41:01 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:41:01 INFO DAGScheduler:59 - Registering RDD 57 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Registering RDD 61 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Got job 5 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:01 INFO DAGScheduler:59 - Final stage: ResultStage 33(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30, ShuffleMapStage 32, ShuffleMapStage 24, ShuffleMapStage 28, ShuffleMapStage 26, ShuffleMapStage 23) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 32) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Submitting ShuffleMapStage 31 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[57] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(5872) called with curMem=248984, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 5.7 KB, free 730.3 MB) -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(2848) called with curMem=254856, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:49890 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 31 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[57] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:01 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 19:41:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 24, localhost, PROCESS_LOCAL, 2196 bytes) -2016-06-13 19:41:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 25, localhost, PROCESS_LOCAL, 2196 bytes) -2016-06-13 19:41:01 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 24) -2016-06-13 19:41:01 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 25) -2016-06-13 19:41:01 INFO BlockManager:59 - Found block rdd_52_0 locally -2016-06-13 19:41:01 INFO BlockManager:59 - Found block rdd_52_1 locally -2016-06-13 19:41:01 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 24). 2254 bytes result sent to driver -2016-06-13 19:41:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 24) in 63 ms on localhost (1/2) -2016-06-13 19:41:01 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 25). 2254 bytes result sent to driver -2016-06-13 19:41:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 25) in 71 ms on localhost (2/2) -2016-06-13 19:41:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 19:41:01 INFO DAGScheduler:59 - ShuffleMapStage 31 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.075 s -2016-06-13 19:41:01 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:01 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:01 INFO DAGScheduler:59 - waiting: Set(ResultStage 33, ShuffleMapStage 32) -2016-06-13 19:41:01 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:01 INFO DAGScheduler:59 - Missing parents for ResultStage 33: List(ShuffleMapStage 32) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 32: List() -2016-06-13 19:41:01 INFO DAGScheduler:59 - Submitting ShuffleMapStage 32 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[61] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=257704, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=261600, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:49890 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 32 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[61] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:01 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-06-13 19:41:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 26, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 27, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:01 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 27) -2016-06-13 19:41:01 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 26) -2016-06-13 19:41:01 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:41:01 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:01 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 27). 2470 bytes result sent to driver -2016-06-13 19:41:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 27) in 39 ms on localhost (1/2) -2016-06-13 19:41:01 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 26). 2470 bytes result sent to driver -2016-06-13 19:41:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 26) in 46 ms on localhost (2/2) -2016-06-13 19:41:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 19:41:01 INFO DAGScheduler:59 - ShuffleMapStage 32 (mapPartitions at GraphImpl.scala:235) finished in 0.047 s -2016-06-13 19:41:01 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:01 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:01 INFO DAGScheduler:59 - waiting: Set(ResultStage 33) -2016-06-13 19:41:01 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:01 INFO DAGScheduler:59 - Missing parents for ResultStage 33: List() -2016-06-13 19:41:01 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[69] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(7288) called with curMem=263886, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 7.1 KB, free 730.3 MB) -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(3298) called with curMem=271174, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.3 MB) -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:49890 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[69] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:01 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-06-13 19:41:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 28, localhost, PROCESS_LOCAL, 2248 bytes) -2016-06-13 19:41:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 29, localhost, PROCESS_LOCAL, 2248 bytes) -2016-06-13 19:41:01 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 29) -2016-06-13 19:41:01 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 28) -2016-06-13 19:41:01 INFO CacheManager:59 - Partition rdd_65_1 not found, computing it -2016-06-13 19:41:01 INFO CacheManager:59 - Partition rdd_65_0 not found, computing it -2016-06-13 19:41:01 INFO BlockManager:59 - Found block rdd_52_1 locally -2016-06-13 19:41:01 INFO BlockManager:59 - Found block rdd_52_0 locally -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=274472, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block rdd_65_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=277016, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block rdd_65_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Added rdd_65_0 in memory on localhost:49890 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Added rdd_65_1 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 28). 2971 bytes result sent to driver -2016-06-13 19:41:01 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 29). 2971 bytes result sent to driver -2016-06-13 19:41:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 28) in 35 ms on localhost (1/2) -2016-06-13 19:41:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 29) in 36 ms on localhost (2/2) -2016-06-13 19:41:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 19:41:01 INFO DAGScheduler:59 - ResultStage 33 (fold at LDAOptimizer.scala:205) finished in 0.040 s -2016-06-13 19:41:01 INFO DAGScheduler:59 - Job 5 finished: fold at LDAOptimizer.scala:205, took 0.244852 s -2016-06-13 19:41:01 INFO MapPartitionsRDD:59 - Removing RDD 39 from persistence list -2016-06-13 19:41:01 INFO ZippedPartitionsRDD2:59 - Removing RDD 46 from persistence list -2016-06-13 19:41:01 INFO BlockManager:59 - Removing RDD 46 -2016-06-13 19:41:01 INFO BlockManager:59 - Removing RDD 39 -2016-06-13 19:41:01 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:41:01 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:41:01 INFO DAGScheduler:59 - Registering RDD 70 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Registering RDD 74 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Got job 6 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:01 INFO DAGScheduler:59 - Final stage: ResultStage 46(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 45, ShuffleMapStage 37, ShuffleMapStage 34, ShuffleMapStage 35, ShuffleMapStage 39, ShuffleMapStage 43, ShuffleMapStage 41) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 45) -2016-06-13 19:41:01 INFO DAGScheduler:59 - Submitting ShuffleMapStage 44 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[70] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(6304) called with curMem=274472, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 6.2 KB, free 730.3 MB) -2016-06-13 19:41:01 INFO MemoryStore:59 - ensureFreeSpace(2991) called with curMem=280776, maxMem=766075207 -2016-06-13 19:41:01 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-06-13 19:41:01 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:49890 (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:41:01 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 44 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[70] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:01 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-06-13 19:41:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 30, localhost, PROCESS_LOCAL, 2237 bytes) -2016-06-13 19:41:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 31, localhost, PROCESS_LOCAL, 2237 bytes) -2016-06-13 19:41:01 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 31) -2016-06-13 19:41:01 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 30) -2016-06-13 19:41:01 INFO BlockManager:59 - Found block rdd_65_1 locally -2016-06-13 19:41:01 INFO BlockManager:59 - Found block rdd_65_0 locally -2016-06-13 19:41:02 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 31). 2254 bytes result sent to driver -2016-06-13 19:41:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 31) in 43 ms on localhost (1/2) -2016-06-13 19:41:02 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 30). 2254 bytes result sent to driver -2016-06-13 19:41:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 30) in 55 ms on localhost (2/2) -2016-06-13 19:41:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-06-13 19:41:02 INFO DAGScheduler:59 - ShuffleMapStage 44 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.059 s -2016-06-13 19:41:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:02 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:02 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 45, ResultStage 46) -2016-06-13 19:41:02 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:02 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 45: List() -2016-06-13 19:41:02 INFO DAGScheduler:59 - Missing parents for ResultStage 46: List(ShuffleMapStage 45) -2016-06-13 19:41:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 45 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[74] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=283767, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(2284) called with curMem=287663, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:49890 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 45 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[74] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:02 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-06-13 19:41:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 32, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 33, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:02 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 32) -2016-06-13 19:41:02 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 33) -2016-06-13 19:41:02 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:02 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:02 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 33). 2470 bytes result sent to driver -2016-06-13 19:41:02 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 32). 2470 bytes result sent to driver -2016-06-13 19:41:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 33) in 60 ms on localhost (1/2) -2016-06-13 19:41:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 32) in 63 ms on localhost (2/2) -2016-06-13 19:41:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-06-13 19:41:02 INFO DAGScheduler:59 - ShuffleMapStage 45 (mapPartitions at GraphImpl.scala:235) finished in 0.066 s -2016-06-13 19:41:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:02 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 46) -2016-06-13 19:41:02 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:02 INFO DAGScheduler:59 - Missing parents for ResultStage 46: List() -2016-06-13 19:41:02 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[82] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(7720) called with curMem=289947, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 7.5 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=297667, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:49890 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[82] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:02 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-06-13 19:41:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 34, localhost, PROCESS_LOCAL, 2289 bytes) -2016-06-13 19:41:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 35, localhost, PROCESS_LOCAL, 2289 bytes) -2016-06-13 19:41:02 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 35) -2016-06-13 19:41:02 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 34) -2016-06-13 19:41:02 INFO CacheManager:59 - Partition rdd_78_1 not found, computing it -2016-06-13 19:41:02 INFO BlockManager:59 - Found block rdd_65_1 locally -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=301067, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block rdd_78_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO CacheManager:59 - Partition rdd_78_0 not found, computing it -2016-06-13 19:41:02 INFO BlockManager:59 - Found block rdd_65_0 locally -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Added rdd_78_1 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:02 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 35). 2971 bytes result sent to driver -2016-06-13 19:41:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 35) in 23 ms on localhost (1/2) -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=303515, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block rdd_78_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Added rdd_78_0 in memory on localhost:49890 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 34). 2971 bytes result sent to driver -2016-06-13 19:41:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 34) in 34 ms on localhost (2/2) -2016-06-13 19:41:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-06-13 19:41:02 INFO DAGScheduler:59 - ResultStage 46 (fold at LDAOptimizer.scala:205) finished in 0.036 s -2016-06-13 19:41:02 INFO DAGScheduler:59 - Job 6 finished: fold at LDAOptimizer.scala:205, took 0.261928 s -2016-06-13 19:41:02 INFO MapPartitionsRDD:59 - Removing RDD 52 from persistence list -2016-06-13 19:41:02 INFO ZippedPartitionsRDD2:59 - Removing RDD 59 from persistence list -2016-06-13 19:41:02 INFO BlockManager:59 - Removing RDD 52 -2016-06-13 19:41:02 INFO BlockManager:59 - Removing RDD 59 -2016-06-13 19:41:02 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:41:02 INFO DAGScheduler:59 - Registering RDD 83 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:02 INFO DAGScheduler:59 - Registering RDD 87 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:02 INFO DAGScheduler:59 - Got job 7 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:02 INFO DAGScheduler:59 - Final stage: ResultStage 61(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 48, ShuffleMapStage 60, ShuffleMapStage 52, ShuffleMapStage 56, ShuffleMapStage 50, ShuffleMapStage 54, ShuffleMapStage 58, ShuffleMapStage 47) -2016-06-13 19:41:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 60) -2016-06-13 19:41:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 59 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[83] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(6728) called with curMem=301067, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 6.6 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(3078) called with curMem=307795, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:49890 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 59 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[83] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:02 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 2 tasks -2016-06-13 19:41:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 36, localhost, PROCESS_LOCAL, 2278 bytes) -2016-06-13 19:41:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 37, localhost, PROCESS_LOCAL, 2278 bytes) -2016-06-13 19:41:02 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 36) -2016-06-13 19:41:02 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 37) -2016-06-13 19:41:02 INFO BlockManager:59 - Found block rdd_78_0 locally -2016-06-13 19:41:02 INFO BlockManager:59 - Found block rdd_78_1 locally -2016-06-13 19:41:02 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 36). 2254 bytes result sent to driver -2016-06-13 19:41:02 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 37). 2254 bytes result sent to driver -2016-06-13 19:41:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 36) in 58 ms on localhost (1/2) -2016-06-13 19:41:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 37) in 58 ms on localhost (2/2) -2016-06-13 19:41:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-06-13 19:41:02 INFO DAGScheduler:59 - ShuffleMapStage 59 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.065 s -2016-06-13 19:41:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:02 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:02 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 60, ResultStage 61) -2016-06-13 19:41:02 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:02 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 60: List() -2016-06-13 19:41:02 INFO DAGScheduler:59 - Missing parents for ResultStage 61: List(ShuffleMapStage 60) -2016-06-13 19:41:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 60 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[87] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=310873, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(2280) called with curMem=314769, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:49890 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 60 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[87] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:02 INFO TaskSchedulerImpl:59 - Adding task set 60.0 with 2 tasks -2016-06-13 19:41:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 60.0 (TID 38, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:49890 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 60.0 (TID 39, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:02 INFO Executor:59 - Running task 0.0 in stage 60.0 (TID 38) -2016-06-13 19:41:02 INFO Executor:59 - Running task 1.0 in stage 60.0 (TID 39) -2016-06-13 19:41:02 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 19:41:02 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:02 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:02 INFO Executor:59 - Finished task 1.0 in stage 60.0 (TID 39). 2470 bytes result sent to driver -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:49890 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 60.0 (TID 39) in 48 ms on localhost (1/2) -2016-06-13 19:41:02 INFO Executor:59 - Finished task 0.0 in stage 60.0 (TID 38). 2470 bytes result sent to driver -2016-06-13 19:41:02 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 19:41:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 60.0 (TID 38) in 56 ms on localhost (2/2) -2016-06-13 19:41:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 60.0, whose tasks have all completed, from pool -2016-06-13 19:41:02 INFO DAGScheduler:59 - ShuffleMapStage 60 (mapPartitions at GraphImpl.scala:235) finished in 0.058 s -2016-06-13 19:41:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:02 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 61) -2016-06-13 19:41:02 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:49890 in memory (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO DAGScheduler:59 - Missing parents for ResultStage 61: List() -2016-06-13 19:41:02 INFO DAGScheduler:59 - Submitting ResultStage 61 (MapPartitionsRDD[95] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(8144) called with curMem=290454, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 8.0 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=298598, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:49890 (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 61 (MapPartitionsRDD[95] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:02 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 2 tasks -2016-06-13 19:41:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 40, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:41:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 41, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:41:02 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 40) -2016-06-13 19:41:02 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 19:41:02 INFO CacheManager:59 - Partition rdd_91_0 not found, computing it -2016-06-13 19:41:02 INFO BlockManager:59 - Found block rdd_78_0 locally -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:02 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 41) -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=298791, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block rdd_91_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Added rdd_91_0 in memory on localhost:49890 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:49890 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO CacheManager:59 - Partition rdd_91_1 not found, computing it -2016-06-13 19:41:02 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 40). 2971 bytes result sent to driver -2016-06-13 19:41:02 INFO BlockManager:59 - Found block rdd_78_1 locally -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 40) in 21 ms on localhost (1/2) -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=294047, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block rdd_91_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Added rdd_91_1 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 19:41:02 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 41). 2971 bytes result sent to driver -2016-06-13 19:41:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 41) in 39 ms on localhost (2/2) -2016-06-13 19:41:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-06-13 19:41:02 INFO DAGScheduler:59 - ResultStage 61 (fold at LDAOptimizer.scala:205) finished in 0.044 s -2016-06-13 19:41:02 INFO DAGScheduler:59 - Job 7 finished: fold at LDAOptimizer.scala:205, took 0.343950 s -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:49890 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:49890 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO MapPartitionsRDD:59 - Removing RDD 65 from persistence list -2016-06-13 19:41:02 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 19:41:02 INFO ZippedPartitionsRDD2:59 - Removing RDD 72 from persistence list -2016-06-13 19:41:02 INFO BlockManager:59 - Removing RDD 72 -2016-06-13 19:41:02 INFO BlockManager:59 - Removing RDD 65 -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:49890 in memory (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:41:02 INFO DAGScheduler:59 - Registering RDD 96 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:02 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 19:41:02 INFO DAGScheduler:59 - Registering RDD 100 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:02 INFO DAGScheduler:59 - Got job 8 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:02 INFO DAGScheduler:59 - Final stage: ResultStage 78(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 63, ShuffleMapStage 67, ShuffleMapStage 71, ShuffleMapStage 75, ShuffleMapStage 69, ShuffleMapStage 73, ShuffleMapStage 65, ShuffleMapStage 62, ShuffleMapStage 77) -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:49890 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 77) -2016-06-13 19:41:02 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 19:41:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 76 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[96] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(7160) called with curMem=260398, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 7.0 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(3182) called with curMem=267558, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:49890 (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 76 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[96] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:02 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 2 tasks -2016-06-13 19:41:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 42, localhost, PROCESS_LOCAL, 2319 bytes) -2016-06-13 19:41:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 43, localhost, PROCESS_LOCAL, 2319 bytes) -2016-06-13 19:41:02 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 42) -2016-06-13 19:41:02 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 43) -2016-06-13 19:41:02 INFO BlockManager:59 - Found block rdd_91_0 locally -2016-06-13 19:41:02 INFO BlockManager:59 - Found block rdd_91_1 locally -2016-06-13 19:41:02 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 43). 2254 bytes result sent to driver -2016-06-13 19:41:02 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 42). 2254 bytes result sent to driver -2016-06-13 19:41:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 43) in 41 ms on localhost (1/2) -2016-06-13 19:41:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 42) in 44 ms on localhost (2/2) -2016-06-13 19:41:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-06-13 19:41:02 INFO DAGScheduler:59 - ShuffleMapStage 76 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.003 s -2016-06-13 19:41:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:02 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 78, ShuffleMapStage 77) -2016-06-13 19:41:02 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:49890 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO DAGScheduler:59 - Missing parents for ResultStage 78: List(ShuffleMapStage 77) -2016-06-13 19:41:02 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 77: List() -2016-06-13 19:41:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 77 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[100] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=262565, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(2283) called with curMem=266461, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:49890 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 77 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[100] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:02 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 2 tasks -2016-06-13 19:41:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 44, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 45, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:02 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 45) -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:49890 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 44) -2016-06-13 19:41:02 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:02 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:02 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:49890 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 45). 2470 bytes result sent to driver -2016-06-13 19:41:02 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 19:41:02 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 44). 2470 bytes result sent to driver -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:49890 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 45) in 83 ms on localhost (1/2) -2016-06-13 19:41:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 44) in 86 ms on localhost (2/2) -2016-06-13 19:41:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-06-13 19:41:02 INFO DAGScheduler:59 - ShuffleMapStage 77 (mapPartitions at GraphImpl.scala:235) finished in 0.086 s -2016-06-13 19:41:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:02 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 78) -2016-06-13 19:41:02 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:02 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 19:41:02 INFO DAGScheduler:59 - Missing parents for ResultStage 78: List() -2016-06-13 19:41:02 INFO DAGScheduler:59 - Submitting ResultStage 78 (MapPartitionsRDD[108] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(8576) called with curMem=246342, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 8.4 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(3582) called with curMem=254918, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:49890 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 78 (MapPartitionsRDD[108] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:02 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 2 tasks -2016-06-13 19:41:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 46, localhost, PROCESS_LOCAL, 2371 bytes) -2016-06-13 19:41:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 47, localhost, PROCESS_LOCAL, 2371 bytes) -2016-06-13 19:41:02 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 47) -2016-06-13 19:41:02 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 46) -2016-06-13 19:41:02 INFO CacheManager:59 - Partition rdd_104_0 not found, computing it -2016-06-13 19:41:02 INFO BlockManager:59 - Found block rdd_91_0 locally -2016-06-13 19:41:02 INFO CacheManager:59 - Partition rdd_104_1 not found, computing it -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:02 INFO BlockManager:59 - Found block rdd_91_1 locally -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=258500, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block rdd_104_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=261044, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block rdd_104_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Added rdd_104_0 in memory on localhost:49890 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Added rdd_104_1 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 46). 2971 bytes result sent to driver -2016-06-13 19:41:02 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 47). 2971 bytes result sent to driver -2016-06-13 19:41:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 46) in 14 ms on localhost (1/2) -2016-06-13 19:41:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 47) in 14 ms on localhost (2/2) -2016-06-13 19:41:02 INFO DAGScheduler:59 - ResultStage 78 (fold at LDAOptimizer.scala:205) finished in 0.015 s -2016-06-13 19:41:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-06-13 19:41:02 INFO DAGScheduler:59 - Job 8 finished: fold at LDAOptimizer.scala:205, took 0.243202 s -2016-06-13 19:41:02 INFO MapPartitionsRDD:59 - Removing RDD 78 from persistence list -2016-06-13 19:41:02 INFO ZippedPartitionsRDD2:59 - Removing RDD 85 from persistence list -2016-06-13 19:41:02 INFO BlockManager:59 - Removing RDD 78 -2016-06-13 19:41:02 INFO BlockManager:59 - Removing RDD 85 -2016-06-13 19:41:02 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:41:02 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:41:02 INFO DAGScheduler:59 - Registering RDD 109 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:02 INFO DAGScheduler:59 - Registering RDD 113 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:02 INFO DAGScheduler:59 - Got job 9 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:02 INFO DAGScheduler:59 - Final stage: ResultStage 97(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 84, ShuffleMapStage 96, ShuffleMapStage 88, ShuffleMapStage 82, ShuffleMapStage 86, ShuffleMapStage 90, ShuffleMapStage 79, ShuffleMapStage 94, ShuffleMapStage 80, ShuffleMapStage 92) -2016-06-13 19:41:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 96) -2016-06-13 19:41:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 95 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[109] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(7584) called with curMem=258500, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 7.4 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO MemoryStore:59 - ensureFreeSpace(3283) called with curMem=266084, maxMem=766075207 -2016-06-13 19:41:02 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.3 MB) -2016-06-13 19:41:02 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:49890 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:41:02 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 95 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[109] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:02 INFO TaskSchedulerImpl:59 - Adding task set 95.0 with 2 tasks -2016-06-13 19:41:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 95.0 (TID 48, localhost, PROCESS_LOCAL, 2360 bytes) -2016-06-13 19:41:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 95.0 (TID 49, localhost, PROCESS_LOCAL, 2360 bytes) -2016-06-13 19:41:02 INFO Executor:59 - Running task 0.0 in stage 95.0 (TID 48) -2016-06-13 19:41:02 INFO Executor:59 - Running task 1.0 in stage 95.0 (TID 49) -2016-06-13 19:41:02 INFO BlockManager:59 - Found block rdd_104_0 locally -2016-06-13 19:41:02 INFO BlockManager:59 - Found block rdd_104_1 locally -2016-06-13 19:41:02 INFO Executor:59 - Finished task 1.0 in stage 95.0 (TID 49). 2254 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 95.0 (TID 49) in 30 ms on localhost (1/2) -2016-06-13 19:41:03 INFO Executor:59 - Finished task 0.0 in stage 95.0 (TID 48). 2254 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 95.0 (TID 48) in 36 ms on localhost (2/2) -2016-06-13 19:41:03 INFO DAGScheduler:59 - ShuffleMapStage 95 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.044 s -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 95.0, whose tasks have all completed, from pool -2016-06-13 19:41:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 96, ResultStage 97) -2016-06-13 19:41:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 96: List() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents for ResultStage 97: List(ShuffleMapStage 96) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 96 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[113] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=269367, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=273263, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:49890 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 96 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[113] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Adding task set 96.0 with 2 tasks -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 96.0 (TID 50, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 96.0 (TID 51, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:03 INFO Executor:59 - Running task 0.0 in stage 96.0 (TID 50) -2016-06-13 19:41:03 INFO Executor:59 - Running task 1.0 in stage 96.0 (TID 51) -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:03 INFO Executor:59 - Finished task 0.0 in stage 96.0 (TID 50). 2470 bytes result sent to driver -2016-06-13 19:41:03 INFO Executor:59 - Finished task 1.0 in stage 96.0 (TID 51). 2470 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 96.0 (TID 50) in 27 ms on localhost (1/2) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 96.0 (TID 51) in 26 ms on localhost (2/2) -2016-06-13 19:41:03 INFO DAGScheduler:59 - ShuffleMapStage 96 (mapPartitions at GraphImpl.scala:235) finished in 0.027 s -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 96.0, whose tasks have all completed, from pool -2016-06-13 19:41:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - waiting: Set(ResultStage 97) -2016-06-13 19:41:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents for ResultStage 97: List() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting ResultStage 97 (MapPartitionsRDD[121] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(9000) called with curMem=275549, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 8.8 KB, free 730.3 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(3686) called with curMem=284549, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 3.6 KB, free 730.3 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:49890 (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 97 (MapPartitionsRDD[121] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Adding task set 97.0 with 2 tasks -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 97.0 (TID 52, localhost, PROCESS_LOCAL, 2412 bytes) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 97.0 (TID 53, localhost, PROCESS_LOCAL, 2412 bytes) -2016-06-13 19:41:03 INFO Executor:59 - Running task 0.0 in stage 97.0 (TID 52) -2016-06-13 19:41:03 INFO Executor:59 - Running task 1.0 in stage 97.0 (TID 53) -2016-06-13 19:41:03 INFO CacheManager:59 - Partition rdd_117_1 not found, computing it -2016-06-13 19:41:03 INFO CacheManager:59 - Partition rdd_117_0 not found, computing it -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_104_1 locally -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_104_0 locally -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=288235, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block rdd_117_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=290683, maxMem=766075207 -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added rdd_117_1 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - Block rdd_117_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:41:03 INFO Executor:59 - Finished task 1.0 in stage 97.0 (TID 53). 2971 bytes result sent to driver -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added rdd_117_0 in memory on localhost:49890 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO Executor:59 - Finished task 0.0 in stage 97.0 (TID 52). 2971 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 97.0 (TID 53) in 16 ms on localhost (1/2) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 97.0 (TID 52) in 17 ms on localhost (2/2) -2016-06-13 19:41:03 INFO DAGScheduler:59 - ResultStage 97 (fold at LDAOptimizer.scala:205) finished in 0.018 s -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 97.0, whose tasks have all completed, from pool -2016-06-13 19:41:03 INFO DAGScheduler:59 - Job 9 finished: fold at LDAOptimizer.scala:205, took 0.150779 s -2016-06-13 19:41:03 INFO MapPartitionsRDD:59 - Removing RDD 91 from persistence list -2016-06-13 19:41:03 INFO BlockManager:59 - Removing RDD 91 -2016-06-13 19:41:03 INFO ZippedPartitionsRDD2:59 - Removing RDD 98 from persistence list -2016-06-13 19:41:03 INFO BlockManager:59 - Removing RDD 98 -2016-06-13 19:41:03 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:41:03 INFO DAGScheduler:59 - Registering RDD 122 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Registering RDD 126 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Got job 10 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:03 INFO DAGScheduler:59 - Final stage: ResultStage 118(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 117, ShuffleMapStage 99, ShuffleMapStage 103, ShuffleMapStage 115, ShuffleMapStage 107, ShuffleMapStage 101, ShuffleMapStage 111, ShuffleMapStage 105, ShuffleMapStage 109, ShuffleMapStage 98, ShuffleMapStage 113) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 117) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 116 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[122] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(8016) called with curMem=288235, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 7.8 KB, free 730.3 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(3371) called with curMem=296251, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.3 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:49890 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 116 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[122] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Adding task set 116.0 with 2 tasks -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 116.0 (TID 54, localhost, PROCESS_LOCAL, 2401 bytes) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 116.0 (TID 55, localhost, PROCESS_LOCAL, 2401 bytes) -2016-06-13 19:41:03 INFO Executor:59 - Running task 1.0 in stage 116.0 (TID 55) -2016-06-13 19:41:03 INFO Executor:59 - Running task 0.0 in stage 116.0 (TID 54) -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_117_1 locally -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_117_0 locally -2016-06-13 19:41:03 INFO Executor:59 - Finished task 1.0 in stage 116.0 (TID 55). 2254 bytes result sent to driver -2016-06-13 19:41:03 INFO Executor:59 - Finished task 0.0 in stage 116.0 (TID 54). 2254 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 116.0 (TID 55) in 26 ms on localhost (1/2) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 116.0 (TID 54) in 28 ms on localhost (2/2) -2016-06-13 19:41:03 INFO DAGScheduler:59 - ShuffleMapStage 116 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.031 s -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 116.0, whose tasks have all completed, from pool -2016-06-13 19:41:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 117, ResultStage 118) -2016-06-13 19:41:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 117: List() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents for ResultStage 118: List(ShuffleMapStage 117) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 117 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[126] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=299622, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=303518, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:49890 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 117 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[126] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Adding task set 117.0 with 2 tasks -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 117.0 (TID 56, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 117.0 (TID 57, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:03 INFO Executor:59 - Running task 1.0 in stage 117.0 (TID 57) -2016-06-13 19:41:03 INFO Executor:59 - Running task 0.0 in stage 117.0 (TID 56) -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:03 INFO Executor:59 - Finished task 0.0 in stage 117.0 (TID 56). 2470 bytes result sent to driver -2016-06-13 19:41:03 INFO Executor:59 - Finished task 1.0 in stage 117.0 (TID 57). 2470 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 117.0 (TID 57) in 25 ms on localhost (1/2) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 117.0 (TID 56) in 26 ms on localhost (2/2) -2016-06-13 19:41:03 INFO DAGScheduler:59 - ShuffleMapStage 117 (mapPartitions at GraphImpl.scala:235) finished in 0.027 s -2016-06-13 19:41:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - waiting: Set(ResultStage 118) -2016-06-13 19:41:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 117.0, whose tasks have all completed, from pool -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents for ResultStage 118: List() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting ResultStage 118 (MapPartitionsRDD[134] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(9432) called with curMem=305804, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 9.2 KB, free 730.3 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(3778) called with curMem=315236, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.7 KB, free 730.3 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:49890 (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 118 (MapPartitionsRDD[134] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Adding task set 118.0 with 2 tasks -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 118.0 (TID 58, localhost, PROCESS_LOCAL, 2453 bytes) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 118.0 (TID 59, localhost, PROCESS_LOCAL, 2453 bytes) -2016-06-13 19:41:03 INFO Executor:59 - Running task 0.0 in stage 118.0 (TID 58) -2016-06-13 19:41:03 INFO Executor:59 - Running task 1.0 in stage 118.0 (TID 59) -2016-06-13 19:41:03 INFO CacheManager:59 - Partition rdd_130_0 not found, computing it -2016-06-13 19:41:03 INFO CacheManager:59 - Partition rdd_130_1 not found, computing it -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_117_0 locally -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_117_1 locally -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=319014, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block rdd_130_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=321558, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block rdd_130_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added rdd_130_0 in memory on localhost:49890 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added rdd_130_1 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO Executor:59 - Finished task 0.0 in stage 118.0 (TID 58). 2971 bytes result sent to driver -2016-06-13 19:41:03 INFO Executor:59 - Finished task 1.0 in stage 118.0 (TID 59). 2971 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 118.0 (TID 58) in 14 ms on localhost (1/2) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 118.0 (TID 59) in 14 ms on localhost (2/2) -2016-06-13 19:41:03 INFO DAGScheduler:59 - ResultStage 118 (fold at LDAOptimizer.scala:205) finished in 0.015 s -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 118.0, whose tasks have all completed, from pool -2016-06-13 19:41:03 INFO DAGScheduler:59 - Job 10 finished: fold at LDAOptimizer.scala:205, took 0.135598 s -2016-06-13 19:41:03 INFO MapPartitionsRDD:59 - Removing RDD 104 from persistence list -2016-06-13 19:41:03 INFO BlockManager:59 - Removing RDD 104 -2016-06-13 19:41:03 INFO ZippedPartitionsRDD2:59 - Removing RDD 111 from persistence list -2016-06-13 19:41:03 INFO BlockManager:59 - Removing RDD 111 -2016-06-13 19:41:03 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:41:03 INFO DAGScheduler:59 - Registering RDD 135 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Registering RDD 139 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Got job 11 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:03 INFO DAGScheduler:59 - Final stage: ResultStage 141(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 132, ShuffleMapStage 136, ShuffleMapStage 140, ShuffleMapStage 122, ShuffleMapStage 119, ShuffleMapStage 126, ShuffleMapStage 138, ShuffleMapStage 120, ShuffleMapStage 130, ShuffleMapStage 124, ShuffleMapStage 134, ShuffleMapStage 128) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 140) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 139 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[135] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(8440) called with curMem=319014, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 8.2 KB, free 730.3 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(3476) called with curMem=327454, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.3 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:49890 (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 139 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[135] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Adding task set 139.0 with 2 tasks -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 139.0 (TID 60, localhost, PROCESS_LOCAL, 2442 bytes) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 139.0 (TID 61, localhost, PROCESS_LOCAL, 2442 bytes) -2016-06-13 19:41:03 INFO Executor:59 - Running task 0.0 in stage 139.0 (TID 60) -2016-06-13 19:41:03 INFO Executor:59 - Running task 1.0 in stage 139.0 (TID 61) -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_130_0 locally -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_130_1 locally -2016-06-13 19:41:03 INFO Executor:59 - Finished task 0.0 in stage 139.0 (TID 60). 2254 bytes result sent to driver -2016-06-13 19:41:03 INFO Executor:59 - Finished task 1.0 in stage 139.0 (TID 61). 2254 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 139.0 (TID 60) in 26 ms on localhost (1/2) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 139.0 (TID 61) in 27 ms on localhost (2/2) -2016-06-13 19:41:03 INFO DAGScheduler:59 - ShuffleMapStage 139 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.028 s -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 139.0, whose tasks have all completed, from pool -2016-06-13 19:41:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 140, ResultStage 141) -2016-06-13 19:41:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 140: List() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents for ResultStage 141: List(ShuffleMapStage 140) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 140 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[139] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=330930, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(2281) called with curMem=334826, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:49890 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 140 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[139] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Adding task set 140.0 with 2 tasks -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 140.0 (TID 62, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 140.0 (TID 63, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:03 INFO Executor:59 - Running task 0.0 in stage 140.0 (TID 62) -2016-06-13 19:41:03 INFO Executor:59 - Running task 1.0 in stage 140.0 (TID 63) -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:03 INFO Executor:59 - Finished task 0.0 in stage 140.0 (TID 62). 2470 bytes result sent to driver -2016-06-13 19:41:03 INFO Executor:59 - Finished task 1.0 in stage 140.0 (TID 63). 2470 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 140.0 (TID 62) in 24 ms on localhost (1/2) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 140.0 (TID 63) in 23 ms on localhost (2/2) -2016-06-13 19:41:03 INFO DAGScheduler:59 - ShuffleMapStage 140 (mapPartitions at GraphImpl.scala:235) finished in 0.024 s -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 140.0, whose tasks have all completed, from pool -2016-06-13 19:41:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - waiting: Set(ResultStage 141) -2016-06-13 19:41:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents for ResultStage 141: List() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting ResultStage 141 (MapPartitionsRDD[147] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(9856) called with curMem=337107, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 9.6 KB, free 730.3 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(3886) called with curMem=346963, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:49890 (size: 3.8 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 141 (MapPartitionsRDD[147] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Adding task set 141.0 with 2 tasks -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 141.0 (TID 64, localhost, PROCESS_LOCAL, 2494 bytes) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 141.0 (TID 65, localhost, PROCESS_LOCAL, 2494 bytes) -2016-06-13 19:41:03 INFO Executor:59 - Running task 0.0 in stage 141.0 (TID 64) -2016-06-13 19:41:03 INFO Executor:59 - Running task 1.0 in stage 141.0 (TID 65) -2016-06-13 19:41:03 INFO CacheManager:59 - Partition rdd_143_0 not found, computing it -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_130_0 locally -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:03 INFO CacheManager:59 - Partition rdd_143_1 not found, computing it -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_130_1 locally -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=350849, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block rdd_143_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=353393, maxMem=766075207 -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added rdd_143_0 in memory on localhost:49890 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - Block rdd_143_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO Executor:59 - Finished task 0.0 in stage 141.0 (TID 64). 2971 bytes result sent to driver -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added rdd_143_1 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO Executor:59 - Finished task 1.0 in stage 141.0 (TID 65). 2971 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 141.0 (TID 64) in 17 ms on localhost (1/2) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 141.0 (TID 65) in 17 ms on localhost (2/2) -2016-06-13 19:41:03 INFO DAGScheduler:59 - ResultStage 141 (fold at LDAOptimizer.scala:205) finished in 0.019 s -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 141.0, whose tasks have all completed, from pool -2016-06-13 19:41:03 INFO DAGScheduler:59 - Job 11 finished: fold at LDAOptimizer.scala:205, took 0.146494 s -2016-06-13 19:41:03 INFO MapPartitionsRDD:59 - Removing RDD 117 from persistence list -2016-06-13 19:41:03 INFO ZippedPartitionsRDD2:59 - Removing RDD 124 from persistence list -2016-06-13 19:41:03 INFO BlockManager:59 - Removing RDD 117 -2016-06-13 19:41:03 INFO BlockManager:59 - Removing RDD 124 -2016-06-13 19:41:03 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:41:03 INFO DAGScheduler:59 - Registering RDD 148 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Registering RDD 152 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Got job 12 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:03 INFO DAGScheduler:59 - Final stage: ResultStage 166(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 161, ShuffleMapStage 143, ShuffleMapStage 153, ShuffleMapStage 165, ShuffleMapStage 147, ShuffleMapStage 157, ShuffleMapStage 151, ShuffleMapStage 155, ShuffleMapStage 159, ShuffleMapStage 163, ShuffleMapStage 145, ShuffleMapStage 142, ShuffleMapStage 149) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 165) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 164 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[148] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(8872) called with curMem=350849, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 8.7 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(3568) called with curMem=359721, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:49890 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 164 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[148] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Adding task set 164.0 with 2 tasks -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 164.0 (TID 66, localhost, PROCESS_LOCAL, 2483 bytes) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 164.0 (TID 67, localhost, PROCESS_LOCAL, 2483 bytes) -2016-06-13 19:41:03 INFO Executor:59 - Running task 1.0 in stage 164.0 (TID 67) -2016-06-13 19:41:03 INFO Executor:59 - Running task 0.0 in stage 164.0 (TID 66) -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_143_1 locally -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_143_0 locally -2016-06-13 19:41:03 INFO Executor:59 - Finished task 1.0 in stage 164.0 (TID 67). 2254 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 164.0 (TID 67) in 25 ms on localhost (1/2) -2016-06-13 19:41:03 INFO Executor:59 - Finished task 0.0 in stage 164.0 (TID 66). 2254 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 164.0 (TID 66) in 32 ms on localhost (2/2) -2016-06-13 19:41:03 INFO DAGScheduler:59 - ShuffleMapStage 164 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.033 s -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 164.0, whose tasks have all completed, from pool -2016-06-13 19:41:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 165, ResultStage 166) -2016-06-13 19:41:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 165: List() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents for ResultStage 166: List(ShuffleMapStage 165) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 165 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[152] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=363289, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=367185, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:49890 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 165 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[152] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Adding task set 165.0 with 2 tasks -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 165.0 (TID 68, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 165.0 (TID 69, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:03 INFO Executor:59 - Running task 0.0 in stage 165.0 (TID 68) -2016-06-13 19:41:03 INFO Executor:59 - Running task 1.0 in stage 165.0 (TID 69) -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:03 INFO Executor:59 - Finished task 0.0 in stage 165.0 (TID 68). 2470 bytes result sent to driver -2016-06-13 19:41:03 INFO Executor:59 - Finished task 1.0 in stage 165.0 (TID 69). 2470 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 165.0 (TID 68) in 27 ms on localhost (1/2) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 165.0 (TID 69) in 26 ms on localhost (2/2) -2016-06-13 19:41:03 INFO DAGScheduler:59 - ShuffleMapStage 165 (mapPartitions at GraphImpl.scala:235) finished in 0.027 s -2016-06-13 19:41:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 165.0, whose tasks have all completed, from pool -2016-06-13 19:41:03 INFO DAGScheduler:59 - waiting: Set(ResultStage 166) -2016-06-13 19:41:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents for ResultStage 166: List() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting ResultStage 166 (MapPartitionsRDD[160] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(10288) called with curMem=369471, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 10.0 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(4004) called with curMem=379759, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 3.9 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:49890 (size: 3.9 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 166 (MapPartitionsRDD[160] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Adding task set 166.0 with 2 tasks -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 166.0 (TID 70, localhost, PROCESS_LOCAL, 2535 bytes) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 166.0 (TID 71, localhost, PROCESS_LOCAL, 2535 bytes) -2016-06-13 19:41:03 INFO Executor:59 - Running task 0.0 in stage 166.0 (TID 70) -2016-06-13 19:41:03 INFO Executor:59 - Running task 1.0 in stage 166.0 (TID 71) -2016-06-13 19:41:03 INFO CacheManager:59 - Partition rdd_156_0 not found, computing it -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_143_0 locally -2016-06-13 19:41:03 INFO CacheManager:59 - Partition rdd_156_1 not found, computing it -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_143_1 locally -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=383763, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block rdd_156_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=386307, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block rdd_156_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added rdd_156_0 in memory on localhost:49890 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added rdd_156_1 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO Executor:59 - Finished task 0.0 in stage 166.0 (TID 70). 2971 bytes result sent to driver -2016-06-13 19:41:03 INFO Executor:59 - Finished task 1.0 in stage 166.0 (TID 71). 2971 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 166.0 (TID 70) in 13 ms on localhost (1/2) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 166.0 (TID 71) in 13 ms on localhost (2/2) -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 166.0, whose tasks have all completed, from pool -2016-06-13 19:41:03 INFO DAGScheduler:59 - ResultStage 166 (fold at LDAOptimizer.scala:205) finished in 0.014 s -2016-06-13 19:41:03 INFO DAGScheduler:59 - Job 12 finished: fold at LDAOptimizer.scala:205, took 0.134600 s -2016-06-13 19:41:03 INFO MapPartitionsRDD:59 - Removing RDD 130 from persistence list -2016-06-13 19:41:03 INFO BlockManager:59 - Removing RDD 130 -2016-06-13 19:41:03 INFO ZippedPartitionsRDD2:59 - Removing RDD 137 from persistence list -2016-06-13 19:41:03 INFO BlockManager:59 - Removing RDD 137 -2016-06-13 19:41:03 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:41:03 INFO DAGScheduler:59 - Registering RDD 161 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Registering RDD 165 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Got job 13 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:03 INFO DAGScheduler:59 - Final stage: ResultStage 193(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 168, ShuffleMapStage 190, ShuffleMapStage 184, ShuffleMapStage 176, ShuffleMapStage 170, ShuffleMapStage 192, ShuffleMapStage 178, ShuffleMapStage 186, ShuffleMapStage 180, ShuffleMapStage 172, ShuffleMapStage 188, ShuffleMapStage 174, ShuffleMapStage 167, ShuffleMapStage 182) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 192) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 191 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[161] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(9296) called with curMem=383763, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 9.1 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(3674) called with curMem=393059, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 3.6 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:49890 (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 191 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[161] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Adding task set 191.0 with 2 tasks -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 191.0 (TID 72, localhost, PROCESS_LOCAL, 2524 bytes) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 191.0 (TID 73, localhost, PROCESS_LOCAL, 2524 bytes) -2016-06-13 19:41:03 INFO Executor:59 - Running task 1.0 in stage 191.0 (TID 73) -2016-06-13 19:41:03 INFO Executor:59 - Running task 0.0 in stage 191.0 (TID 72) -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_156_1 locally -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_156_0 locally -2016-06-13 19:41:03 INFO Executor:59 - Finished task 1.0 in stage 191.0 (TID 73). 2254 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 191.0 (TID 73) in 23 ms on localhost (1/2) -2016-06-13 19:41:03 INFO Executor:59 - Finished task 0.0 in stage 191.0 (TID 72). 2254 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 191.0 (TID 72) in 28 ms on localhost (2/2) -2016-06-13 19:41:03 INFO DAGScheduler:59 - ShuffleMapStage 191 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.029 s -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 191.0, whose tasks have all completed, from pool -2016-06-13 19:41:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 192, ResultStage 193) -2016-06-13 19:41:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 192: List() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents for ResultStage 193: List(ShuffleMapStage 192) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 192 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[165] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=396733, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=400629, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:49890 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 192 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[165] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Adding task set 192.0 with 2 tasks -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 192.0 (TID 74, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 192.0 (TID 75, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:03 INFO Executor:59 - Running task 0.0 in stage 192.0 (TID 74) -2016-06-13 19:41:03 INFO Executor:59 - Running task 1.0 in stage 192.0 (TID 75) -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:03 INFO Executor:59 - Finished task 0.0 in stage 192.0 (TID 74). 2470 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 192.0 (TID 74) in 44 ms on localhost (1/2) -2016-06-13 19:41:03 INFO Executor:59 - Finished task 1.0 in stage 192.0 (TID 75). 2470 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 192.0 (TID 75) in 48 ms on localhost (2/2) -2016-06-13 19:41:03 INFO DAGScheduler:59 - ShuffleMapStage 192 (mapPartitions at GraphImpl.scala:235) finished in 0.049 s -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 192.0, whose tasks have all completed, from pool -2016-06-13 19:41:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - waiting: Set(ResultStage 193) -2016-06-13 19:41:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents for ResultStage 193: List() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting ResultStage 193 (MapPartitionsRDD[173] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(10712) called with curMem=402915, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 10.5 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=413627, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:49890 (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 193 (MapPartitionsRDD[173] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Adding task set 193.0 with 2 tasks -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 193.0 (TID 76, localhost, PROCESS_LOCAL, 2576 bytes) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 193.0 (TID 77, localhost, PROCESS_LOCAL, 2576 bytes) -2016-06-13 19:41:03 INFO Executor:59 - Running task 0.0 in stage 193.0 (TID 76) -2016-06-13 19:41:03 INFO Executor:59 - Running task 1.0 in stage 193.0 (TID 77) -2016-06-13 19:41:03 INFO CacheManager:59 - Partition rdd_169_1 not found, computing it -2016-06-13 19:41:03 INFO CacheManager:59 - Partition rdd_169_0 not found, computing it -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_156_1 locally -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_156_0 locally -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=417723, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block rdd_169_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=420171, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block rdd_169_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added rdd_169_1 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added rdd_169_0 in memory on localhost:49890 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO Executor:59 - Finished task 1.0 in stage 193.0 (TID 77). 2971 bytes result sent to driver -2016-06-13 19:41:03 INFO Executor:59 - Finished task 0.0 in stage 193.0 (TID 76). 2971 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 193.0 (TID 77) in 12 ms on localhost (1/2) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 193.0 (TID 76) in 14 ms on localhost (2/2) -2016-06-13 19:41:03 INFO DAGScheduler:59 - ResultStage 193 (fold at LDAOptimizer.scala:205) finished in 0.014 s -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 193.0, whose tasks have all completed, from pool -2016-06-13 19:41:03 INFO DAGScheduler:59 - Job 13 finished: fold at LDAOptimizer.scala:205, took 0.167948 s -2016-06-13 19:41:03 INFO MapPartitionsRDD:59 - Removing RDD 143 from persistence list -2016-06-13 19:41:03 INFO ZippedPartitionsRDD2:59 - Removing RDD 150 from persistence list -2016-06-13 19:41:03 INFO BlockManager:59 - Removing RDD 143 -2016-06-13 19:41:03 INFO BlockManager:59 - Removing RDD 150 -2016-06-13 19:41:03 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:41:03 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:41:03 INFO DAGScheduler:59 - Registering RDD 174 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Registering RDD 178 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Got job 14 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:03 INFO DAGScheduler:59 - Final stage: ResultStage 222(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 197, ShuffleMapStage 219, ShuffleMapStage 205, ShuffleMapStage 221, ShuffleMapStage 213, ShuffleMapStage 207, ShuffleMapStage 199, ShuffleMapStage 215, ShuffleMapStage 201, ShuffleMapStage 194, ShuffleMapStage 209, ShuffleMapStage 203, ShuffleMapStage 195, ShuffleMapStage 217, ShuffleMapStage 211) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 221) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 220 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[174] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(9728) called with curMem=417723, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 9.5 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(3782) called with curMem=427451, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.7 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:49890 (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 220 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[174] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Adding task set 220.0 with 2 tasks -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 220.0 (TID 78, localhost, PROCESS_LOCAL, 2565 bytes) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 220.0 (TID 79, localhost, PROCESS_LOCAL, 2565 bytes) -2016-06-13 19:41:03 INFO Executor:59 - Running task 0.0 in stage 220.0 (TID 78) -2016-06-13 19:41:03 INFO Executor:59 - Running task 1.0 in stage 220.0 (TID 79) -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_169_1 locally -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_169_0 locally -2016-06-13 19:41:03 INFO Executor:59 - Finished task 1.0 in stage 220.0 (TID 79). 2254 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 220.0 (TID 79) in 24 ms on localhost (1/2) -2016-06-13 19:41:03 INFO Executor:59 - Finished task 0.0 in stage 220.0 (TID 78). 2254 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 220.0 (TID 78) in 27 ms on localhost (2/2) -2016-06-13 19:41:03 INFO DAGScheduler:59 - ShuffleMapStage 220 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.027 s -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 220.0, whose tasks have all completed, from pool -2016-06-13 19:41:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - waiting: Set(ResultStage 222, ShuffleMapStage 221) -2016-06-13 19:41:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents for ResultStage 222: List(ShuffleMapStage 221) -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 221: List() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 221 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[178] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=431233, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(2290) called with curMem=435129, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:49890 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 221 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[178] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Adding task set 221.0 with 2 tasks -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 221.0 (TID 80, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 221.0 (TID 81, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:03 INFO Executor:59 - Running task 1.0 in stage 221.0 (TID 81) -2016-06-13 19:41:03 INFO Executor:59 - Running task 0.0 in stage 221.0 (TID 80) -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:03 INFO Executor:59 - Finished task 1.0 in stage 221.0 (TID 81). 2470 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 221.0 (TID 81) in 51 ms on localhost (1/2) -2016-06-13 19:41:03 INFO Executor:59 - Finished task 0.0 in stage 221.0 (TID 80). 2470 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 221.0 (TID 80) in 56 ms on localhost (2/2) -2016-06-13 19:41:03 INFO DAGScheduler:59 - ShuffleMapStage 221 (mapPartitions at GraphImpl.scala:235) finished in 0.057 s -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 221.0, whose tasks have all completed, from pool -2016-06-13 19:41:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - waiting: Set(ResultStage 222) -2016-06-13 19:41:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Missing parents for ResultStage 222: List() -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting ResultStage 222 (MapPartitionsRDD[186] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(11144) called with curMem=437419, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 10.9 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(4182) called with curMem=448563, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 4.1 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:49890 (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 222 (MapPartitionsRDD[186] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Adding task set 222.0 with 2 tasks -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 222.0 (TID 82, localhost, PROCESS_LOCAL, 2617 bytes) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 222.0 (TID 83, localhost, PROCESS_LOCAL, 2617 bytes) -2016-06-13 19:41:03 INFO Executor:59 - Running task 0.0 in stage 222.0 (TID 82) -2016-06-13 19:41:03 INFO Executor:59 - Running task 1.0 in stage 222.0 (TID 83) -2016-06-13 19:41:03 INFO CacheManager:59 - Partition rdd_182_0 not found, computing it -2016-06-13 19:41:03 INFO CacheManager:59 - Partition rdd_182_1 not found, computing it -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_169_0 locally -2016-06-13 19:41:03 INFO BlockManager:59 - Found block rdd_169_1 locally -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=452745, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block rdd_182_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:41:03 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=455193, maxMem=766075207 -2016-06-13 19:41:03 INFO MemoryStore:59 - Block rdd_182_0 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added rdd_182_1 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO BlockManagerInfo:59 - Added rdd_182_0 in memory on localhost:49890 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:41:03 INFO Executor:59 - Finished task 1.0 in stage 222.0 (TID 83). 2971 bytes result sent to driver -2016-06-13 19:41:03 INFO Executor:59 - Finished task 0.0 in stage 222.0 (TID 82). 2971 bytes result sent to driver -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 222.0 (TID 83) in 11 ms on localhost (1/2) -2016-06-13 19:41:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 222.0 (TID 82) in 13 ms on localhost (2/2) -2016-06-13 19:41:03 INFO DAGScheduler:59 - ResultStage 222 (fold at LDAOptimizer.scala:205) finished in 0.013 s -2016-06-13 19:41:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 222.0, whose tasks have all completed, from pool -2016-06-13 19:41:03 INFO DAGScheduler:59 - Job 14 finished: fold at LDAOptimizer.scala:205, took 0.158358 s -2016-06-13 19:41:04 INFO MapPartitionsRDD:59 - Removing RDD 156 from persistence list -2016-06-13 19:41:04 INFO ZippedPartitionsRDD2:59 - Removing RDD 163 from persistence list -2016-06-13 19:41:04 INFO BlockManager:59 - Removing RDD 156 -2016-06-13 19:41:04 INFO BlockManager:59 - Removing RDD 163 -2016-06-13 19:41:04 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:41:04 INFO DAGScheduler:59 - Registering RDD 187 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Registering RDD 191 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Got job 15 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:04 INFO DAGScheduler:59 - Final stage: ResultStage 253(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 248, ShuffleMapStage 234, ShuffleMapStage 242, ShuffleMapStage 228, ShuffleMapStage 250, ShuffleMapStage 244, ShuffleMapStage 236, ShuffleMapStage 230, ShuffleMapStage 252, ShuffleMapStage 223, ShuffleMapStage 238, ShuffleMapStage 224, ShuffleMapStage 246, ShuffleMapStage 240, ShuffleMapStage 232, ShuffleMapStage 226) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 252) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 251 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[187] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(10152) called with curMem=452745, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 9.9 KB, free 730.1 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(3864) called with curMem=462897, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 3.8 KB, free 730.1 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:49890 (size: 3.8 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 251 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[187] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Adding task set 251.0 with 2 tasks -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 251.0 (TID 84, localhost, PROCESS_LOCAL, 2606 bytes) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 251.0 (TID 85, localhost, PROCESS_LOCAL, 2606 bytes) -2016-06-13 19:41:04 INFO Executor:59 - Running task 1.0 in stage 251.0 (TID 85) -2016-06-13 19:41:04 INFO Executor:59 - Running task 0.0 in stage 251.0 (TID 84) -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_182_0 locally -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_182_1 locally -2016-06-13 19:41:04 INFO Executor:59 - Finished task 1.0 in stage 251.0 (TID 85). 2254 bytes result sent to driver -2016-06-13 19:41:04 INFO Executor:59 - Finished task 0.0 in stage 251.0 (TID 84). 2254 bytes result sent to driver -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 251.0 (TID 85) in 24 ms on localhost (1/2) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 251.0 (TID 84) in 26 ms on localhost (2/2) -2016-06-13 19:41:04 INFO DAGScheduler:59 - ShuffleMapStage 251 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.027 s -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 251.0, whose tasks have all completed, from pool -2016-06-13 19:41:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:04 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 252, ResultStage 253) -2016-06-13 19:41:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:04 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 252: List() -2016-06-13 19:41:04 INFO DAGScheduler:59 - Missing parents for ResultStage 253: List(ShuffleMapStage 252) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 252 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[191] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=466761, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 3.8 KB, free 730.1 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(2284) called with curMem=470657, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:49890 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 252 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[191] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Adding task set 252.0 with 2 tasks -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 252.0 (TID 86, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 252.0 (TID 87, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:04 INFO Executor:59 - Running task 0.0 in stage 252.0 (TID 86) -2016-06-13 19:41:04 INFO Executor:59 - Running task 1.0 in stage 252.0 (TID 87) -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:04 INFO Executor:59 - Finished task 0.0 in stage 252.0 (TID 86). 2470 bytes result sent to driver -2016-06-13 19:41:04 INFO Executor:59 - Finished task 1.0 in stage 252.0 (TID 87). 2470 bytes result sent to driver -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 252.0 (TID 86) in 33 ms on localhost (1/2) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 252.0 (TID 87) in 34 ms on localhost (2/2) -2016-06-13 19:41:04 INFO DAGScheduler:59 - ShuffleMapStage 252 (mapPartitions at GraphImpl.scala:235) finished in 0.034 s -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 252.0, whose tasks have all completed, from pool -2016-06-13 19:41:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 253) -2016-06-13 19:41:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:04 INFO DAGScheduler:59 - Missing parents for ResultStage 253: List() -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting ResultStage 253 (MapPartitionsRDD[199] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(11568) called with curMem=472941, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 11.3 KB, free 730.1 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(4283) called with curMem=484509, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.1 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:49890 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 253 (MapPartitionsRDD[199] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Adding task set 253.0 with 2 tasks -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 253.0 (TID 88, localhost, PROCESS_LOCAL, 2658 bytes) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 253.0 (TID 89, localhost, PROCESS_LOCAL, 2658 bytes) -2016-06-13 19:41:04 INFO Executor:59 - Running task 0.0 in stage 253.0 (TID 88) -2016-06-13 19:41:04 INFO Executor:59 - Running task 1.0 in stage 253.0 (TID 89) -2016-06-13 19:41:04 INFO CacheManager:59 - Partition rdd_195_1 not found, computing it -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_182_1 locally -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:04 INFO CacheManager:59 - Partition rdd_195_0 not found, computing it -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_182_0 locally -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=488792, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block rdd_195_1 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=491240, maxMem=766075207 -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added rdd_195_1 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - Block rdd_195_0 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-06-13 19:41:04 INFO Executor:59 - Finished task 1.0 in stage 253.0 (TID 89). 2971 bytes result sent to driver -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added rdd_195_0 in memory on localhost:49890 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO Executor:59 - Finished task 0.0 in stage 253.0 (TID 88). 2971 bytes result sent to driver -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 253.0 (TID 89) in 20 ms on localhost (1/2) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 253.0 (TID 88) in 22 ms on localhost (2/2) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 253.0, whose tasks have all completed, from pool -2016-06-13 19:41:04 INFO DAGScheduler:59 - ResultStage 253 (fold at LDAOptimizer.scala:205) finished in 0.022 s -2016-06-13 19:41:04 INFO DAGScheduler:59 - Job 15 finished: fold at LDAOptimizer.scala:205, took 0.150006 s -2016-06-13 19:41:04 INFO MapPartitionsRDD:59 - Removing RDD 169 from persistence list -2016-06-13 19:41:04 INFO BlockManager:59 - Removing RDD 169 -2016-06-13 19:41:04 INFO ZippedPartitionsRDD2:59 - Removing RDD 176 from persistence list -2016-06-13 19:41:04 INFO BlockManager:59 - Removing RDD 176 -2016-06-13 19:41:04 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:49890 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:49890 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:41:04 INFO DAGScheduler:59 - Registering RDD 200 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:49890 in memory (size: 3.8 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Registering RDD 204 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Got job 16 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:04 INFO DAGScheduler:59 - Final stage: ResultStage 286(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 269, ShuffleMapStage 263, ShuffleMapStage 255, ShuffleMapStage 285, ShuffleMapStage 277, ShuffleMapStage 271, ShuffleMapStage 257, ShuffleMapStage 279, ShuffleMapStage 265, ShuffleMapStage 281, ShuffleMapStage 273, ShuffleMapStage 267, ShuffleMapStage 259, ShuffleMapStage 275, ShuffleMapStage 261, ShuffleMapStage 283, ShuffleMapStage 254) -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-06-13 19:41:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 285) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 284 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[200] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:49890 in memory (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(10584) called with curMem=437419, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 10.3 KB, free 730.2 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(3951) called with curMem=448003, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 3.9 KB, free 730.2 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:49890 (size: 3.9 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 284 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[200] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Adding task set 284.0 with 2 tasks -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 284.0 (TID 90, localhost, PROCESS_LOCAL, 2647 bytes) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 284.0 (TID 91, localhost, PROCESS_LOCAL, 2647 bytes) -2016-06-13 19:41:04 INFO Executor:59 - Running task 0.0 in stage 284.0 (TID 90) -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_195_0 locally -2016-06-13 19:41:04 INFO Executor:59 - Running task 1.0 in stage 284.0 (TID 91) -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_195_1 locally -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-06-13 19:41:04 INFO Executor:59 - Finished task 0.0 in stage 284.0 (TID 90). 2254 bytes result sent to driver -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:49890 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 284.0 (TID 90) in 44 ms on localhost (1/2) -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-06-13 19:41:04 INFO Executor:59 - Finished task 1.0 in stage 284.0 (TID 91). 2254 bytes result sent to driver -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 284.0 (TID 91) in 47 ms on localhost (2/2) -2016-06-13 19:41:04 INFO DAGScheduler:59 - ShuffleMapStage 284 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.040 s -2016-06-13 19:41:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:04 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 285, ResultStage 286) -2016-06-13 19:41:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 284.0, whose tasks have all completed, from pool -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:49890 in memory (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 285: List() -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-06-13 19:41:04 INFO DAGScheduler:59 - Missing parents for ResultStage 286: List(ShuffleMapStage 285) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 285 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[204] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:49890 in memory (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=417450, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=419060, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:49890 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:49890 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-06-13 19:41:04 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 285 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[204] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Adding task set 285.0 with 2 tasks -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 285.0 (TID 92, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:49890 in memory (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 285.0 (TID 93, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:04 INFO Executor:59 - Running task 0.0 in stage 285.0 (TID 92) -2016-06-13 19:41:04 INFO Executor:59 - Running task 1.0 in stage 285.0 (TID 93) -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:49890 in memory (size: 3.9 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:49890 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-06-13 19:41:04 INFO Executor:59 - Finished task 1.0 in stage 285.0 (TID 93). 2470 bytes result sent to driver -2016-06-13 19:41:04 INFO Executor:59 - Finished task 0.0 in stage 285.0 (TID 92). 2470 bytes result sent to driver -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 285.0 (TID 93) in 31 ms on localhost (1/2) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 285.0 (TID 92) in 34 ms on localhost (2/2) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 285.0, whose tasks have all completed, from pool -2016-06-13 19:41:04 INFO DAGScheduler:59 - ShuffleMapStage 285 (mapPartitions at GraphImpl.scala:235) finished in 0.035 s -2016-06-13 19:41:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 286) -2016-06-13 19:41:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:49890 in memory (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-06-13 19:41:04 INFO DAGScheduler:59 - Missing parents for ResultStage 286: List() -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting ResultStage 286 (MapPartitionsRDD[212] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:49890 in memory (size: 3.8 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(12000) called with curMem=357824, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 11.7 KB, free 730.2 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:49890 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(4400) called with curMem=367543, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.2 MB) -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:49890 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 286 (MapPartitionsRDD[212] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Adding task set 286.0 with 2 tasks -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 286.0 (TID 94, localhost, PROCESS_LOCAL, 2699 bytes) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 286.0 (TID 95, localhost, PROCESS_LOCAL, 2699 bytes) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:49890 in memory (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO Executor:59 - Running task 1.0 in stage 286.0 (TID 95) -2016-06-13 19:41:04 INFO Executor:59 - Running task 0.0 in stage 286.0 (TID 94) -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-06-13 19:41:04 INFO CacheManager:59 - Partition rdd_208_1 not found, computing it -2016-06-13 19:41:04 INFO CacheManager:59 - Partition rdd_208_0 not found, computing it -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_195_0 locally -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_195_1 locally -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=342921, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block rdd_208_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=345465, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block rdd_208_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added rdd_208_0 in memory on localhost:49890 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:49890 in memory (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO Executor:59 - Finished task 0.0 in stage 286.0 (TID 94). 2971 bytes result sent to driver -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added rdd_208_1 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO Executor:59 - Finished task 1.0 in stage 286.0 (TID 95). 2971 bytes result sent to driver -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 286.0 (TID 94) in 16 ms on localhost (1/2) -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 286.0 (TID 95) in 17 ms on localhost (2/2) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 286.0, whose tasks have all completed, from pool -2016-06-13 19:41:04 INFO DAGScheduler:59 - ResultStage 286 (fold at LDAOptimizer.scala:205) finished in 0.019 s -2016-06-13 19:41:04 INFO DAGScheduler:59 - Job 16 finished: fold at LDAOptimizer.scala:205, took 0.200852 s -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:49890 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:49890 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:49890 in memory (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-06-13 19:41:04 INFO MapPartitionsRDD:59 - Removing RDD 182 from persistence list -2016-06-13 19:41:04 INFO ZippedPartitionsRDD2:59 - Removing RDD 189 from persistence list -2016-06-13 19:41:04 INFO BlockManager:59 - Removing RDD 182 -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:49890 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-06-13 19:41:04 INFO BlockManager:59 - Removing RDD 189 -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:49890 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:49890 in memory (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-06-13 19:41:04 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:49890 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:49890 in memory (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:49890 in memory (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:49890 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:49890 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:41:04 INFO DAGScheduler:59 - Registering RDD 213 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Registering RDD 217 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Got job 17 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:04 INFO DAGScheduler:59 - Final stage: ResultStage 321(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 320, ShuffleMapStage 306, ShuffleMapStage 300, ShuffleMapStage 292, ShuffleMapStage 314, ShuffleMapStage 308, ShuffleMapStage 294, ShuffleMapStage 316, ShuffleMapStage 287, ShuffleMapStage 302, ShuffleMapStage 288, ShuffleMapStage 318, ShuffleMapStage 310, ShuffleMapStage 304, ShuffleMapStage 296, ShuffleMapStage 290, ShuffleMapStage 312, ShuffleMapStage 298) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 320) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 319 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[213] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=239321, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 10.8 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(4062) called with curMem=250329, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:49890 (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 319 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[213] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Adding task set 319.0 with 2 tasks -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 319.0 (TID 96, localhost, PROCESS_LOCAL, 2688 bytes) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 319.0 (TID 97, localhost, PROCESS_LOCAL, 2688 bytes) -2016-06-13 19:41:04 INFO Executor:59 - Running task 1.0 in stage 319.0 (TID 97) -2016-06-13 19:41:04 INFO Executor:59 - Running task 0.0 in stage 319.0 (TID 96) -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_208_1 locally -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_208_0 locally -2016-06-13 19:41:04 INFO Executor:59 - Finished task 1.0 in stage 319.0 (TID 97). 2254 bytes result sent to driver -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 319.0 (TID 97) in 22 ms on localhost (1/2) -2016-06-13 19:41:04 INFO Executor:59 - Finished task 0.0 in stage 319.0 (TID 96). 2254 bytes result sent to driver -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 319.0 (TID 96) in 31 ms on localhost (2/2) -2016-06-13 19:41:04 INFO DAGScheduler:59 - ShuffleMapStage 319 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.031 s -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 319.0, whose tasks have all completed, from pool -2016-06-13 19:41:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:04 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 320, ResultStage 321) -2016-06-13 19:41:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:04 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 320: List() -2016-06-13 19:41:04 INFO DAGScheduler:59 - Missing parents for ResultStage 321: List(ShuffleMapStage 320) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 320 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[217] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=254391, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=258287, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:49890 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 320 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[217] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Adding task set 320.0 with 2 tasks -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 320.0 (TID 98, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 320.0 (TID 99, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:04 INFO Executor:59 - Running task 0.0 in stage 320.0 (TID 98) -2016-06-13 19:41:04 INFO Executor:59 - Running task 1.0 in stage 320.0 (TID 99) -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:04 INFO Executor:59 - Finished task 0.0 in stage 320.0 (TID 98). 2470 bytes result sent to driver -2016-06-13 19:41:04 INFO Executor:59 - Finished task 1.0 in stage 320.0 (TID 99). 2470 bytes result sent to driver -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 320.0 (TID 98) in 25 ms on localhost (1/2) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 320.0 (TID 99) in 25 ms on localhost (2/2) -2016-06-13 19:41:04 INFO DAGScheduler:59 - ShuffleMapStage 320 (mapPartitions at GraphImpl.scala:235) finished in 0.025 s -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 320.0, whose tasks have all completed, from pool -2016-06-13 19:41:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 321) -2016-06-13 19:41:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:04 INFO DAGScheduler:59 - Missing parents for ResultStage 321: List() -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting ResultStage 321 (MapPartitionsRDD[225] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(12424) called with curMem=260573, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 12.1 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(4491) called with curMem=272997, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:49890 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 321 (MapPartitionsRDD[225] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Adding task set 321.0 with 2 tasks -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 321.0 (TID 100, localhost, PROCESS_LOCAL, 2740 bytes) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 321.0 (TID 101, localhost, PROCESS_LOCAL, 2740 bytes) -2016-06-13 19:41:04 INFO Executor:59 - Running task 1.0 in stage 321.0 (TID 101) -2016-06-13 19:41:04 INFO Executor:59 - Running task 0.0 in stage 321.0 (TID 100) -2016-06-13 19:41:04 INFO CacheManager:59 - Partition rdd_221_1 not found, computing it -2016-06-13 19:41:04 INFO CacheManager:59 - Partition rdd_221_0 not found, computing it -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_208_0 locally -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_208_1 locally -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=277488, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block rdd_221_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=280032, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block rdd_221_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added rdd_221_0 in memory on localhost:49890 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added rdd_221_1 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO Executor:59 - Finished task 0.0 in stage 321.0 (TID 100). 2971 bytes result sent to driver -2016-06-13 19:41:04 INFO Executor:59 - Finished task 1.0 in stage 321.0 (TID 101). 2971 bytes result sent to driver -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 321.0 (TID 100) in 13 ms on localhost (1/2) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 321.0 (TID 101) in 13 ms on localhost (2/2) -2016-06-13 19:41:04 INFO DAGScheduler:59 - ResultStage 321 (fold at LDAOptimizer.scala:205) finished in 0.014 s -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 321.0, whose tasks have all completed, from pool -2016-06-13 19:41:04 INFO DAGScheduler:59 - Job 17 finished: fold at LDAOptimizer.scala:205, took 0.141546 s -2016-06-13 19:41:04 INFO MapPartitionsRDD:59 - Removing RDD 195 from persistence list -2016-06-13 19:41:04 INFO ZippedPartitionsRDD2:59 - Removing RDD 202 from persistence list -2016-06-13 19:41:04 INFO BlockManager:59 - Removing RDD 195 -2016-06-13 19:41:04 INFO BlockManager:59 - Removing RDD 202 -2016-06-13 19:41:04 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:41:04 INFO DAGScheduler:59 - Registering RDD 226 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Registering RDD 230 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Got job 18 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:04 INFO DAGScheduler:59 - Final stage: ResultStage 358(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 335, ShuffleMapStage 357, ShuffleMapStage 343, ShuffleMapStage 322, ShuffleMapStage 329, ShuffleMapStage 323, ShuffleMapStage 351, ShuffleMapStage 345, ShuffleMapStage 337, ShuffleMapStage 331, ShuffleMapStage 353, ShuffleMapStage 339, ShuffleMapStage 325, ShuffleMapStage 355, ShuffleMapStage 347, ShuffleMapStage 341, ShuffleMapStage 333, ShuffleMapStage 327, ShuffleMapStage 349) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 357) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 356 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[226] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(11440) called with curMem=277488, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 11.2 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(4162) called with curMem=288928, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 4.1 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:49890 (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 356 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[226] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Adding task set 356.0 with 2 tasks -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 356.0 (TID 102, localhost, PROCESS_LOCAL, 2729 bytes) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 356.0 (TID 103, localhost, PROCESS_LOCAL, 2729 bytes) -2016-06-13 19:41:04 INFO Executor:59 - Running task 0.0 in stage 356.0 (TID 102) -2016-06-13 19:41:04 INFO Executor:59 - Running task 1.0 in stage 356.0 (TID 103) -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_221_0 locally -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_221_1 locally -2016-06-13 19:41:04 INFO Executor:59 - Finished task 1.0 in stage 356.0 (TID 103). 2254 bytes result sent to driver -2016-06-13 19:41:04 INFO Executor:59 - Finished task 0.0 in stage 356.0 (TID 102). 2254 bytes result sent to driver -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 356.0 (TID 103) in 26 ms on localhost (1/2) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 356.0 (TID 102) in 28 ms on localhost (2/2) -2016-06-13 19:41:04 INFO DAGScheduler:59 - ShuffleMapStage 356 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.029 s -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 356.0, whose tasks have all completed, from pool -2016-06-13 19:41:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:04 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 357, ResultStage 358) -2016-06-13 19:41:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:04 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 357: List() -2016-06-13 19:41:04 INFO DAGScheduler:59 - Missing parents for ResultStage 358: List(ShuffleMapStage 357) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 357 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[230] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=293090, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=296986, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:49890 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 357 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[230] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Adding task set 357.0 with 2 tasks -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 357.0 (TID 104, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 357.0 (TID 105, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:04 INFO Executor:59 - Running task 0.0 in stage 357.0 (TID 104) -2016-06-13 19:41:04 INFO Executor:59 - Running task 1.0 in stage 357.0 (TID 105) -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:04 INFO Executor:59 - Finished task 1.0 in stage 357.0 (TID 105). 2470 bytes result sent to driver -2016-06-13 19:41:04 INFO Executor:59 - Finished task 0.0 in stage 357.0 (TID 104). 2470 bytes result sent to driver -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 357.0 (TID 105) in 23 ms on localhost (1/2) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 357.0 (TID 104) in 25 ms on localhost (2/2) -2016-06-13 19:41:04 INFO DAGScheduler:59 - ShuffleMapStage 357 (mapPartitions at GraphImpl.scala:235) finished in 0.025 s -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 357.0, whose tasks have all completed, from pool -2016-06-13 19:41:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 358) -2016-06-13 19:41:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:04 INFO DAGScheduler:59 - Missing parents for ResultStage 358: List() -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting ResultStage 358 (MapPartitionsRDD[238] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(12856) called with curMem=299272, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 12.6 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(4586) called with curMem=312128, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:49890 (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 358 (MapPartitionsRDD[238] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Adding task set 358.0 with 2 tasks -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 358.0 (TID 106, localhost, PROCESS_LOCAL, 2781 bytes) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 358.0 (TID 107, localhost, PROCESS_LOCAL, 2781 bytes) -2016-06-13 19:41:04 INFO Executor:59 - Running task 1.0 in stage 358.0 (TID 107) -2016-06-13 19:41:04 INFO Executor:59 - Running task 0.0 in stage 358.0 (TID 106) -2016-06-13 19:41:04 INFO CacheManager:59 - Partition rdd_234_0 not found, computing it -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_221_0 locally -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=316714, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block rdd_234_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added rdd_234_0 in memory on localhost:49890 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO Executor:59 - Finished task 0.0 in stage 358.0 (TID 106). 2971 bytes result sent to driver -2016-06-13 19:41:04 INFO CacheManager:59 - Partition rdd_234_1 not found, computing it -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_221_1 locally -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 358.0 (TID 106) in 15 ms on localhost (1/2) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=319258, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block rdd_234_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added rdd_234_1 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO Executor:59 - Finished task 1.0 in stage 358.0 (TID 107). 2971 bytes result sent to driver -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 358.0 (TID 107) in 23 ms on localhost (2/2) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 358.0, whose tasks have all completed, from pool -2016-06-13 19:41:04 INFO DAGScheduler:59 - ResultStage 358 (fold at LDAOptimizer.scala:205) finished in 0.024 s -2016-06-13 19:41:04 INFO DAGScheduler:59 - Job 18 finished: fold at LDAOptimizer.scala:205, took 0.146816 s -2016-06-13 19:41:04 INFO MapPartitionsRDD:59 - Removing RDD 208 from persistence list -2016-06-13 19:41:04 INFO ZippedPartitionsRDD2:59 - Removing RDD 215 from persistence list -2016-06-13 19:41:04 INFO BlockManager:59 - Removing RDD 208 -2016-06-13 19:41:04 INFO BlockManager:59 - Removing RDD 215 -2016-06-13 19:41:04 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:41:04 INFO DAGScheduler:59 - Registering RDD 239 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Registering RDD 243 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Got job 19 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:04 INFO DAGScheduler:59 - Final stage: ResultStage 397(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 386, ShuffleMapStage 372, ShuffleMapStage 394, ShuffleMapStage 380, ShuffleMapStage 359, ShuffleMapStage 366, ShuffleMapStage 396, ShuffleMapStage 360, ShuffleMapStage 388, ShuffleMapStage 382, ShuffleMapStage 374, ShuffleMapStage 368, ShuffleMapStage 390, ShuffleMapStage 376, ShuffleMapStage 362, ShuffleMapStage 392, ShuffleMapStage 384, ShuffleMapStage 378, ShuffleMapStage 370, ShuffleMapStage 364) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 396) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 395 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[239] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(11864) called with curMem=316714, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 11.6 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(4268) called with curMem=328578, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:49890 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 395 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[239] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Adding task set 395.0 with 2 tasks -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 395.0 (TID 108, localhost, PROCESS_LOCAL, 2770 bytes) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 395.0 (TID 109, localhost, PROCESS_LOCAL, 2770 bytes) -2016-06-13 19:41:04 INFO Executor:59 - Running task 1.0 in stage 395.0 (TID 109) -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_234_1 locally -2016-06-13 19:41:04 INFO Executor:59 - Running task 0.0 in stage 395.0 (TID 108) -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_234_0 locally -2016-06-13 19:41:04 INFO Executor:59 - Finished task 0.0 in stage 395.0 (TID 108). 2254 bytes result sent to driver -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 395.0 (TID 108) in 51 ms on localhost (1/2) -2016-06-13 19:41:04 INFO Executor:59 - Finished task 1.0 in stage 395.0 (TID 109). 2254 bytes result sent to driver -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 395.0 (TID 109) in 63 ms on localhost (2/2) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 395.0, whose tasks have all completed, from pool -2016-06-13 19:41:04 INFO DAGScheduler:59 - ShuffleMapStage 395 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.066 s -2016-06-13 19:41:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 397, ShuffleMapStage 396) -2016-06-13 19:41:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:04 INFO DAGScheduler:59 - Missing parents for ResultStage 397: List(ShuffleMapStage 396) -2016-06-13 19:41:04 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 396: List() -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 396 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[243] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=332846, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(2288) called with curMem=336742, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:49890 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 396 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[243] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Adding task set 396.0 with 2 tasks -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 396.0 (TID 110, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 396.0 (TID 111, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:04 INFO Executor:59 - Running task 1.0 in stage 396.0 (TID 111) -2016-06-13 19:41:04 INFO Executor:59 - Running task 0.0 in stage 396.0 (TID 110) -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:04 INFO Executor:59 - Finished task 1.0 in stage 396.0 (TID 111). 2470 bytes result sent to driver -2016-06-13 19:41:04 INFO Executor:59 - Finished task 0.0 in stage 396.0 (TID 110). 2470 bytes result sent to driver -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 396.0 (TID 111) in 26 ms on localhost (1/2) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 396.0 (TID 110) in 27 ms on localhost (2/2) -2016-06-13 19:41:04 INFO DAGScheduler:59 - ShuffleMapStage 396 (mapPartitions at GraphImpl.scala:235) finished in 0.028 s -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 396.0, whose tasks have all completed, from pool -2016-06-13 19:41:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 397) -2016-06-13 19:41:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:04 INFO DAGScheduler:59 - Missing parents for ResultStage 397: List() -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting ResultStage 397 (MapPartitionsRDD[251] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(13280) called with curMem=339030, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 13.0 KB, free 730.3 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(4699) called with curMem=352310, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 4.6 KB, free 730.2 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:49890 (size: 4.6 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 397 (MapPartitionsRDD[251] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Adding task set 397.0 with 2 tasks -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 397.0 (TID 112, localhost, PROCESS_LOCAL, 2822 bytes) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 397.0 (TID 113, localhost, PROCESS_LOCAL, 2822 bytes) -2016-06-13 19:41:04 INFO Executor:59 - Running task 0.0 in stage 397.0 (TID 112) -2016-06-13 19:41:04 INFO Executor:59 - Running task 1.0 in stage 397.0 (TID 113) -2016-06-13 19:41:04 INFO CacheManager:59 - Partition rdd_247_0 not found, computing it -2016-06-13 19:41:04 INFO CacheManager:59 - Partition rdd_247_1 not found, computing it -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_234_0 locally -2016-06-13 19:41:04 INFO BlockManager:59 - Found block rdd_234_1 locally -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=357009, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block rdd_247_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:41:04 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=359457, maxMem=766075207 -2016-06-13 19:41:04 INFO MemoryStore:59 - Block rdd_247_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added rdd_247_1 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO BlockManagerInfo:59 - Added rdd_247_0 in memory on localhost:49890 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:41:04 INFO Executor:59 - Finished task 1.0 in stage 397.0 (TID 113). 2971 bytes result sent to driver -2016-06-13 19:41:04 INFO Executor:59 - Finished task 0.0 in stage 397.0 (TID 112). 2971 bytes result sent to driver -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 397.0 (TID 113) in 12 ms on localhost (1/2) -2016-06-13 19:41:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 397.0 (TID 112) in 14 ms on localhost (2/2) -2016-06-13 19:41:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 397.0, whose tasks have all completed, from pool -2016-06-13 19:41:04 INFO DAGScheduler:59 - ResultStage 397 (fold at LDAOptimizer.scala:205) finished in 0.014 s -2016-06-13 19:41:04 INFO DAGScheduler:59 - Job 19 finished: fold at LDAOptimizer.scala:205, took 0.182974 s -2016-06-13 19:41:04 INFO MapPartitionsRDD:59 - Removing RDD 221 from persistence list -2016-06-13 19:41:04 INFO ZippedPartitionsRDD2:59 - Removing RDD 228 from persistence list -2016-06-13 19:41:04 INFO BlockManager:59 - Removing RDD 221 -2016-06-13 19:41:04 INFO BlockManager:59 - Removing RDD 228 -2016-06-13 19:41:04 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:41:04 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 155 bytes -2016-06-13 19:41:05 INFO DAGScheduler:59 - Registering RDD 252 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Registering RDD 256 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Got job 20 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:05 INFO DAGScheduler:59 - Final stage: ResultStage 438(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 407, ShuffleMapStage 437, ShuffleMapStage 401, ShuffleMapStage 423, ShuffleMapStage 409, ShuffleMapStage 431, ShuffleMapStage 417, ShuffleMapStage 403, ShuffleMapStage 433, ShuffleMapStage 425, ShuffleMapStage 419, ShuffleMapStage 411, ShuffleMapStage 405, ShuffleMapStage 427, ShuffleMapStage 398, ShuffleMapStage 413, ShuffleMapStage 435, ShuffleMapStage 399, ShuffleMapStage 429, ShuffleMapStage 421, ShuffleMapStage 415) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 437) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 436 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[252] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(12296) called with curMem=357009, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 12.0 KB, free 730.2 MB) -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(4356) called with curMem=369305, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.2 MB) -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:49890 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 436 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[252] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Adding task set 436.0 with 2 tasks -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 436.0 (TID 114, localhost, PROCESS_LOCAL, 2811 bytes) -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 436.0 (TID 115, localhost, PROCESS_LOCAL, 2811 bytes) -2016-06-13 19:41:05 INFO Executor:59 - Running task 1.0 in stage 436.0 (TID 115) -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_247_1 locally -2016-06-13 19:41:05 INFO Executor:59 - Running task 0.0 in stage 436.0 (TID 114) -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_247_0 locally -2016-06-13 19:41:05 INFO Executor:59 - Finished task 1.0 in stage 436.0 (TID 115). 2254 bytes result sent to driver -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 436.0 (TID 115) in 24 ms on localhost (1/2) -2016-06-13 19:41:05 INFO Executor:59 - Finished task 0.0 in stage 436.0 (TID 114). 2254 bytes result sent to driver -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 436.0 (TID 114) in 47 ms on localhost (2/2) -2016-06-13 19:41:05 INFO DAGScheduler:59 - ShuffleMapStage 436 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.049 s -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 436.0, whose tasks have all completed, from pool -2016-06-13 19:41:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:05 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 437, ResultStage 438) -2016-06-13 19:41:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 437: List() -2016-06-13 19:41:05 INFO DAGScheduler:59 - Missing parents for ResultStage 438: List(ShuffleMapStage 437) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 437 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[256] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=373661, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(2288) called with curMem=377557, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:49890 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 437 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[256] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Adding task set 437.0 with 2 tasks -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 437.0 (TID 116, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 437.0 (TID 117, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:05 INFO Executor:59 - Running task 0.0 in stage 437.0 (TID 116) -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:41:05 INFO Executor:59 - Running task 1.0 in stage 437.0 (TID 117) -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:05 INFO Executor:59 - Finished task 0.0 in stage 437.0 (TID 116). 2470 bytes result sent to driver -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 437.0 (TID 116) in 22 ms on localhost (1/2) -2016-06-13 19:41:05 INFO Executor:59 - Finished task 1.0 in stage 437.0 (TID 117). 2470 bytes result sent to driver -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 437.0 (TID 117) in 24 ms on localhost (2/2) -2016-06-13 19:41:05 INFO DAGScheduler:59 - ShuffleMapStage 437 (mapPartitions at GraphImpl.scala:235) finished in 0.025 s -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 437.0, whose tasks have all completed, from pool -2016-06-13 19:41:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 438) -2016-06-13 19:41:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:05 INFO DAGScheduler:59 - Missing parents for ResultStage 438: List() -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting ResultStage 438 (MapPartitionsRDD[264] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(13712) called with curMem=379845, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 13.4 KB, free 730.2 MB) -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(4770) called with curMem=393557, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 4.7 KB, free 730.2 MB) -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:49890 (size: 4.7 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO SparkContext:59 - Created broadcast 61 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 438 (MapPartitionsRDD[264] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Adding task set 438.0 with 2 tasks -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 438.0 (TID 118, localhost, PROCESS_LOCAL, 2863 bytes) -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 438.0 (TID 119, localhost, PROCESS_LOCAL, 2863 bytes) -2016-06-13 19:41:05 INFO Executor:59 - Running task 1.0 in stage 438.0 (TID 119) -2016-06-13 19:41:05 INFO Executor:59 - Running task 0.0 in stage 438.0 (TID 118) -2016-06-13 19:41:05 INFO CacheManager:59 - Partition rdd_260_1 not found, computing it -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_247_1 locally -2016-06-13 19:41:05 INFO CacheManager:59 - Partition rdd_260_0 not found, computing it -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_247_0 locally -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=398327, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block rdd_260_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=400775, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block rdd_260_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Added rdd_260_1 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO Executor:59 - Finished task 1.0 in stage 438.0 (TID 119). 2971 bytes result sent to driver -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Added rdd_260_0 in memory on localhost:49890 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 438.0 (TID 119) in 12 ms on localhost (1/2) -2016-06-13 19:41:05 INFO Executor:59 - Finished task 0.0 in stage 438.0 (TID 118). 2971 bytes result sent to driver -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 438.0 (TID 118) in 14 ms on localhost (2/2) -2016-06-13 19:41:05 INFO DAGScheduler:59 - ResultStage 438 (fold at LDAOptimizer.scala:205) finished in 0.015 s -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 438.0, whose tasks have all completed, from pool -2016-06-13 19:41:05 INFO DAGScheduler:59 - Job 20 finished: fold at LDAOptimizer.scala:205, took 0.152249 s -2016-06-13 19:41:05 INFO MapPartitionsRDD:59 - Removing RDD 234 from persistence list -2016-06-13 19:41:05 INFO ZippedPartitionsRDD2:59 - Removing RDD 241 from persistence list -2016-06-13 19:41:05 INFO BlockManager:59 - Removing RDD 234 -2016-06-13 19:41:05 INFO BlockManager:59 - Removing RDD 241 -2016-06-13 19:41:05 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:41:05 INFO DAGScheduler:59 - Registering RDD 265 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Registering RDD 269 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Got job 21 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:05 INFO DAGScheduler:59 - Final stage: ResultStage 481(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 466, ShuffleMapStage 458, ShuffleMapStage 452, ShuffleMapStage 444, ShuffleMapStage 474, ShuffleMapStage 460, ShuffleMapStage 446, ShuffleMapStage 468, ShuffleMapStage 439, ShuffleMapStage 454, ShuffleMapStage 476, ShuffleMapStage 440, ShuffleMapStage 470, ShuffleMapStage 462, ShuffleMapStage 456, ShuffleMapStage 448, ShuffleMapStage 478, ShuffleMapStage 442, ShuffleMapStage 464, ShuffleMapStage 450, ShuffleMapStage 480, ShuffleMapStage 472) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 480) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 479 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[265] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(12720) called with curMem=398327, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 12.4 KB, free 730.2 MB) -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(4470) called with curMem=411047, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.2 MB) -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:49890 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 479 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[265] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Adding task set 479.0 with 2 tasks -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 479.0 (TID 120, localhost, PROCESS_LOCAL, 2852 bytes) -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 479.0 (TID 121, localhost, PROCESS_LOCAL, 2852 bytes) -2016-06-13 19:41:05 INFO Executor:59 - Running task 0.0 in stage 479.0 (TID 120) -2016-06-13 19:41:05 INFO Executor:59 - Running task 1.0 in stage 479.0 (TID 121) -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_260_0 locally -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_260_1 locally -2016-06-13 19:41:05 INFO Executor:59 - Finished task 0.0 in stage 479.0 (TID 120). 2254 bytes result sent to driver -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 479.0 (TID 120) in 23 ms on localhost (1/2) -2016-06-13 19:41:05 INFO Executor:59 - Finished task 1.0 in stage 479.0 (TID 121). 2254 bytes result sent to driver -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 479.0 (TID 121) in 35 ms on localhost (2/2) -2016-06-13 19:41:05 INFO DAGScheduler:59 - ShuffleMapStage 479 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.036 s -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 479.0, whose tasks have all completed, from pool -2016-06-13 19:41:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 481, ShuffleMapStage 480) -2016-06-13 19:41:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:05 INFO DAGScheduler:59 - Missing parents for ResultStage 481: List(ShuffleMapStage 480) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 480: List() -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 480 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[269] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=415517, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(2285) called with curMem=419413, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:49890 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 480 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[269] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Adding task set 480.0 with 2 tasks -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 480.0 (TID 122, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 480.0 (TID 123, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:05 INFO Executor:59 - Running task 0.0 in stage 480.0 (TID 122) -2016-06-13 19:41:05 INFO Executor:59 - Running task 1.0 in stage 480.0 (TID 123) -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:05 INFO Executor:59 - Finished task 0.0 in stage 480.0 (TID 122). 2470 bytes result sent to driver -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 480.0 (TID 122) in 25 ms on localhost (1/2) -2016-06-13 19:41:05 INFO Executor:59 - Finished task 1.0 in stage 480.0 (TID 123). 2470 bytes result sent to driver -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 480.0 (TID 123) in 47 ms on localhost (2/2) -2016-06-13 19:41:05 INFO DAGScheduler:59 - ShuffleMapStage 480 (mapPartitions at GraphImpl.scala:235) finished in 0.048 s -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 480.0, whose tasks have all completed, from pool -2016-06-13 19:41:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 481) -2016-06-13 19:41:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:05 INFO DAGScheduler:59 - Missing parents for ResultStage 481: List() -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting ResultStage 481 (MapPartitionsRDD[277] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(14136) called with curMem=421698, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 13.8 KB, free 730.2 MB) -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(4890) called with curMem=435834, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 4.8 KB, free 730.2 MB) -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:49890 (size: 4.8 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 481 (MapPartitionsRDD[277] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Adding task set 481.0 with 2 tasks -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 481.0 (TID 124, localhost, PROCESS_LOCAL, 2904 bytes) -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 481.0 (TID 125, localhost, PROCESS_LOCAL, 2904 bytes) -2016-06-13 19:41:05 INFO Executor:59 - Running task 1.0 in stage 481.0 (TID 125) -2016-06-13 19:41:05 INFO Executor:59 - Running task 0.0 in stage 481.0 (TID 124) -2016-06-13 19:41:05 INFO CacheManager:59 - Partition rdd_273_1 not found, computing it -2016-06-13 19:41:05 INFO CacheManager:59 - Partition rdd_273_0 not found, computing it -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_260_1 locally -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_260_0 locally -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=440724, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block rdd_273_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=443268, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block rdd_273_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Added rdd_273_0 in memory on localhost:49890 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Added rdd_273_1 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO Executor:59 - Finished task 0.0 in stage 481.0 (TID 124). 2971 bytes result sent to driver -2016-06-13 19:41:05 INFO Executor:59 - Finished task 1.0 in stage 481.0 (TID 125). 2971 bytes result sent to driver -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 481.0 (TID 124) in 12 ms on localhost (1/2) -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 481.0 (TID 125) in 12 ms on localhost (2/2) -2016-06-13 19:41:05 INFO DAGScheduler:59 - ResultStage 481 (fold at LDAOptimizer.scala:205) finished in 0.013 s -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 481.0, whose tasks have all completed, from pool -2016-06-13 19:41:05 INFO DAGScheduler:59 - Job 21 finished: fold at LDAOptimizer.scala:205, took 0.160523 s -2016-06-13 19:41:05 INFO MapPartitionsRDD:59 - Removing RDD 247 from persistence list -2016-06-13 19:41:05 INFO ZippedPartitionsRDD2:59 - Removing RDD 254 from persistence list -2016-06-13 19:41:05 INFO BlockManager:59 - Removing RDD 247 -2016-06-13 19:41:05 INFO BlockManager:59 - Removing RDD 254 -2016-06-13 19:41:05 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 41 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 40 is 155 bytes -2016-06-13 19:41:05 INFO DAGScheduler:59 - Registering RDD 278 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Registering RDD 282 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Got job 22 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:41:05 INFO DAGScheduler:59 - Final stage: ResultStage 526(fold at LDAOptimizer.scala:205) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 487, ShuffleMapStage 509, ShuffleMapStage 503, ShuffleMapStage 495, ShuffleMapStage 525, ShuffleMapStage 489, ShuffleMapStage 517, ShuffleMapStage 511, ShuffleMapStage 482, ShuffleMapStage 497, ShuffleMapStage 519, ShuffleMapStage 483, ShuffleMapStage 505, ShuffleMapStage 491, ShuffleMapStage 521, ShuffleMapStage 513, ShuffleMapStage 507, ShuffleMapStage 499, ShuffleMapStage 493, ShuffleMapStage 485, ShuffleMapStage 515, ShuffleMapStage 501, ShuffleMapStage 523) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 525) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 524 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[278] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(13152) called with curMem=440724, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 12.8 KB, free 730.2 MB) -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(4564) called with curMem=453876, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.1 MB) -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:49890 (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 524 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[278] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Adding task set 524.0 with 2 tasks -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 524.0 (TID 126, localhost, PROCESS_LOCAL, 2893 bytes) -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 524.0 (TID 127, localhost, PROCESS_LOCAL, 2893 bytes) -2016-06-13 19:41:05 INFO Executor:59 - Running task 1.0 in stage 524.0 (TID 127) -2016-06-13 19:41:05 INFO Executor:59 - Running task 0.0 in stage 524.0 (TID 126) -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_273_1 locally -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_273_0 locally -2016-06-13 19:41:05 INFO Executor:59 - Finished task 1.0 in stage 524.0 (TID 127). 2254 bytes result sent to driver -2016-06-13 19:41:05 INFO Executor:59 - Finished task 0.0 in stage 524.0 (TID 126). 2254 bytes result sent to driver -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 524.0 (TID 127) in 39 ms on localhost (1/2) -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 524.0 (TID 126) in 40 ms on localhost (2/2) -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 524.0, whose tasks have all completed, from pool -2016-06-13 19:41:05 INFO DAGScheduler:59 - ShuffleMapStage 524 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.041 s -2016-06-13 19:41:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:05 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 525, ResultStage 526) -2016-06-13 19:41:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 525: List() -2016-06-13 19:41:05 INFO DAGScheduler:59 - Missing parents for ResultStage 526: List(ShuffleMapStage 525) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 525 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[282] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=458440, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 3.8 KB, free 730.1 MB) -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(2289) called with curMem=462336, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:49890 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 525 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[282] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Adding task set 525.0 with 2 tasks -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 525.0 (TID 128, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 525.0 (TID 129, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:41:05 INFO Executor:59 - Running task 1.0 in stage 525.0 (TID 129) -2016-06-13 19:41:05 INFO Executor:59 - Running task 0.0 in stage 525.0 (TID 128) -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:05 INFO Executor:59 - Finished task 1.0 in stage 525.0 (TID 129). 2470 bytes result sent to driver -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 525.0 (TID 129) in 34 ms on localhost (1/2) -2016-06-13 19:41:05 INFO Executor:59 - Finished task 0.0 in stage 525.0 (TID 128). 2470 bytes result sent to driver -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 525.0 (TID 128) in 39 ms on localhost (2/2) -2016-06-13 19:41:05 INFO DAGScheduler:59 - ShuffleMapStage 525 (mapPartitions at GraphImpl.scala:235) finished in 0.039 s -2016-06-13 19:41:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:41:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:41:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 526) -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 525.0, whose tasks have all completed, from pool -2016-06-13 19:41:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:41:05 INFO DAGScheduler:59 - Missing parents for ResultStage 526: List() -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting ResultStage 526 (MapPartitionsRDD[290] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(14568) called with curMem=464625, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 14.2 KB, free 730.1 MB) -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(4990) called with curMem=479193, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 4.9 KB, free 730.1 MB) -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:49890 (size: 4.9 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO SparkContext:59 - Created broadcast 67 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 526 (MapPartitionsRDD[290] at values at LDAOptimizer.scala:205) -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Adding task set 526.0 with 2 tasks -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 526.0 (TID 130, localhost, PROCESS_LOCAL, 2945 bytes) -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 526.0 (TID 131, localhost, PROCESS_LOCAL, 2945 bytes) -2016-06-13 19:41:05 INFO Executor:59 - Running task 0.0 in stage 526.0 (TID 130) -2016-06-13 19:41:05 INFO Executor:59 - Running task 1.0 in stage 526.0 (TID 131) -2016-06-13 19:41:05 INFO CacheManager:59 - Partition rdd_286_0 not found, computing it -2016-06-13 19:41:05 INFO CacheManager:59 - Partition rdd_286_1 not found, computing it -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_273_0 locally -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_273_1 locally -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=484183, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block rdd_286_0 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=486727, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block rdd_286_1 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Added rdd_286_0 in memory on localhost:49890 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Added rdd_286_1 in memory on localhost:49890 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO Executor:59 - Finished task 0.0 in stage 526.0 (TID 130). 2971 bytes result sent to driver -2016-06-13 19:41:05 INFO Executor:59 - Finished task 1.0 in stage 526.0 (TID 131). 2971 bytes result sent to driver -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 526.0 (TID 130) in 9 ms on localhost (1/2) -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 526.0 (TID 131) in 10 ms on localhost (2/2) -2016-06-13 19:41:05 INFO DAGScheduler:59 - ResultStage 526 (fold at LDAOptimizer.scala:205) finished in 0.010 s -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 526.0, whose tasks have all completed, from pool -2016-06-13 19:41:05 INFO DAGScheduler:59 - Job 22 finished: fold at LDAOptimizer.scala:205, took 0.156340 s -2016-06-13 19:41:05 INFO SparkContext:59 - Starting job: collect at LDAModel.scala:528 -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 41 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 40 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 43 is 155 bytes -2016-06-13 19:41:05 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 42 is 155 bytes -2016-06-13 19:41:05 INFO DAGScheduler:59 - Got job 23 (collect at LDAModel.scala:528) with 2 output partitions -2016-06-13 19:41:05 INFO DAGScheduler:59 - Final stage: ResultStage 571(collect at LDAModel.scala:528) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 538, ShuffleMapStage 560, ShuffleMapStage 546, ShuffleMapStage 540, ShuffleMapStage 568, ShuffleMapStage 532, ShuffleMapStage 562, ShuffleMapStage 554, ShuffleMapStage 548, ShuffleMapStage 570, ShuffleMapStage 534, ShuffleMapStage 556, ShuffleMapStage 527, ShuffleMapStage 542, ShuffleMapStage 564, ShuffleMapStage 528, ShuffleMapStage 558, ShuffleMapStage 550, ShuffleMapStage 544, ShuffleMapStage 536, ShuffleMapStage 566, ShuffleMapStage 530, ShuffleMapStage 552) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting ResultStage 571 (MapPartitionsRDD[293] at map at LDAModel.scala:526), which has no missing parents -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(14256) called with curMem=489175, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 13.9 KB, free 730.1 MB) -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(4871) called with curMem=503431, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 4.8 KB, free 730.1 MB) -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:49890 (size: 4.8 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 571 (MapPartitionsRDD[293] at map at LDAModel.scala:526) -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Adding task set 571.0 with 2 tasks -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 571.0 (TID 132, localhost, PROCESS_LOCAL, 2945 bytes) -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 571.0 (TID 133, localhost, PROCESS_LOCAL, 2945 bytes) -2016-06-13 19:41:05 INFO Executor:59 - Running task 0.0 in stage 571.0 (TID 132) -2016-06-13 19:41:05 INFO Executor:59 - Running task 1.0 in stage 571.0 (TID 133) -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_286_0 locally -2016-06-13 19:41:05 INFO Executor:59 - Finished task 0.0 in stage 571.0 (TID 132). 2680 bytes result sent to driver -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_286_1 locally -2016-06-13 19:41:05 INFO Executor:59 - Finished task 1.0 in stage 571.0 (TID 133). 2612 bytes result sent to driver -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 571.0 (TID 132) in 17 ms on localhost (1/2) -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 571.0 (TID 133) in 16 ms on localhost (2/2) -2016-06-13 19:41:05 INFO DAGScheduler:59 - ResultStage 571 (collect at LDAModel.scala:528) finished in 0.018 s -2016-06-13 19:41:05 INFO DAGScheduler:59 - Job 23 finished: collect at LDAModel.scala:528, took 0.096473 s -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 571.0, whose tasks have all completed, from pool -2016-06-13 19:41:05 INFO SparkContext:59 - Starting job: foreach at LDAFromWebSuite.scala:41 -2016-06-13 19:41:05 INFO DAGScheduler:59 - Got job 24 (foreach at LDAFromWebSuite.scala:41) with 2 output partitions -2016-06-13 19:41:05 INFO DAGScheduler:59 - Final stage: ResultStage 572(foreach at LDAFromWebSuite.scala:41) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:41:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting ResultStage 572 (MapPartitionsRDD[2] at map at LDAFromWebSuite.scala:23), which has no missing parents -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=508302, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(1967) called with curMem=511678, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 1967.0 B, free 730.1 MB) -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:49890 (size: 1967.0 B, free: 730.5 MB) -2016-06-13 19:41:05 INFO SparkContext:59 - Created broadcast 69 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 572 (MapPartitionsRDD[2] at map at LDAFromWebSuite.scala:23) -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:49890 in memory (size: 4.8 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Adding task set 572.0 with 2 tasks -2016-06-13 19:41:05 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 572.0 (TID 134, localhost, PROCESS_LOCAL, 2197 bytes) -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 572.0 (TID 135, localhost, PROCESS_LOCAL, 2197 bytes) -2016-06-13 19:41:05 INFO Executor:59 - Running task 1.0 in stage 572.0 (TID 135) -2016-06-13 19:41:05 INFO Executor:59 - Running task 0.0 in stage 572.0 (TID 134) -2016-06-13 19:41:05 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:132+132 -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:49890 in memory (size: 4.9 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:0+132 -2016-06-13 19:41:05 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:49890 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO Executor:59 - Finished task 1.0 in stage 572.0 (TID 135). 2044 bytes result sent to driver -2016-06-13 19:41:05 INFO Executor:59 - Finished task 0.0 in stage 572.0 (TID 134). 2044 bytes result sent to driver -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 572.0 (TID 135) in 13 ms on localhost (1/2) -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 572.0 (TID 134) in 13 ms on localhost (2/2) -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 572.0, whose tasks have all completed, from pool -2016-06-13 19:41:05 INFO DAGScheduler:59 - ResultStage 572 (foreach at LDAFromWebSuite.scala:41) finished in 0.015 s -2016-06-13 19:41:05 INFO DAGScheduler:59 - Job 24 finished: foreach at LDAFromWebSuite.scala:41, took 0.043274 s -2016-06-13 19:41:05 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:49890 in memory (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-06-13 19:41:05 INFO SparkContext:59 - Starting job: foreach at LDAFromWebSuite.scala:43 -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:49890 in memory (size: 4.8 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Got job 25 (foreach at LDAFromWebSuite.scala:43) with 2 output partitions -2016-06-13 19:41:05 INFO DAGScheduler:59 - Final stage: ResultStage 573(foreach at LDAFromWebSuite.scala:43) -2016-06-13 19:41:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:41:05 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-06-13 19:41:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting ResultStage 573 (MapPartitionsRDD[4] at map at LDAFromWebSuite.scala:25), which has no missing parents -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(3672) called with curMem=429748, maxMem=766075207 -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 3.6 KB, free 730.2 MB) -2016-06-13 19:41:05 INFO MemoryStore:59 - ensureFreeSpace(2055) called with curMem=433420, maxMem=766075207 -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:49890 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.2 MB) -2016-06-13 19:41:05 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:49890 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:41:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 573 (MapPartitionsRDD[4] at map at LDAFromWebSuite.scala:25) -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Adding task set 573.0 with 2 tasks -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 573.0 (TID 136, localhost, PROCESS_LOCAL, 2307 bytes) -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:49890 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 573.0 (TID 137, localhost, PROCESS_LOCAL, 2307 bytes) -2016-06-13 19:41:05 INFO Executor:59 - Running task 0.0 in stage 573.0 (TID 136) -2016-06-13 19:41:05 INFO Executor:59 - Running task 1.0 in stage 573.0 (TID 137) -2016-06-13 19:41:05 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 19:41:05 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:41:05 INFO Executor:59 - Finished task 1.0 in stage 573.0 (TID 137). 2044 bytes result sent to driver -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:49890 in memory (size: 4.7 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO Executor:59 - Finished task 0.0 in stage 573.0 (TID 136). 2044 bytes result sent to driver -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 573.0 (TID 137) in 4 ms on localhost (1/2) -2016-06-13 19:41:05 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-06-13 19:41:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 573.0 (TID 136) in 6 ms on localhost (2/2) -2016-06-13 19:41:05 INFO DAGScheduler:59 - ResultStage 573 (foreach at LDAFromWebSuite.scala:43) finished in 0.006 s -2016-06-13 19:41:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 573.0, whose tasks have all completed, from pool -2016-06-13 19:41:05 INFO DAGScheduler:59 - Job 25 finished: foreach at LDAFromWebSuite.scala:43, took 0.015089 s -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:49890 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:05 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-06-13 19:41:05 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:49890 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:41:06 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-06-13 19:41:06 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:49890 in memory (size: 4.6 KB, free: 730.5 MB) -2016-06-13 19:41:06 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-06-13 19:41:06 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:49890 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:06 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-06-13 19:41:06 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:49890 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:41:06 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-06-13 19:41:06 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:49890 in memory (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 19:41:06 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 19:41:06 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:49890 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 19:41:06 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 19:41:06 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:49890 in memory (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 19:41:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 19:41:06 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-06-13 19:41:06 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:49890 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:41:06 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-06-13 19:41:06 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:49890 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:06 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-06-13 19:41:06 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:49890 in memory (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:41:06 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-06-13 19:41:06 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:49890 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:41:06 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-06-13 19:41:06 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:49890 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:41:06 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-06-13 19:41:06 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:49890 in memory (size: 3.9 KB, free: 730.5 MB) -2016-06-13 19:41:06 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-06-13 19:41:06 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 19:41:06 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 19:41:06 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 19:41:06 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 19:41:06 INFO BlockManager:59 - BlockManager stopped -2016-06-13 19:41:06 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 19:41:06 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 19:41:06 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 19:41:06 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 19:41:06 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 19:41:06 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 19:41:06 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-102f2053-859d-41bb-839b-9aaecbba369d -2016-06-13 19:44:09 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 19:44:10 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 19:44:10 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 19:44:10 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 19:44:10 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 19:44:11 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 19:44:11 INFO Remoting:74 - Starting remoting -2016-06-13 19:44:11 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:49929] -2016-06-13 19:44:11 INFO Utils:59 - Successfully started service 'sparkDriver' on port 49929. -2016-06-13 19:44:11 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 19:44:11 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 19:44:11 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-94789d9a-b2ab-4b03-8885-b532c04cc928 -2016-06-13 19:44:11 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 19:44:11 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-1b3e1343-0e73-4f3c-91c5-0233776e90dd\httpd-6fe224cd-caea-48fa-b40f-2355977544ae -2016-06-13 19:44:11 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 19:44:11 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:44:11 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:49930 -2016-06-13 19:44:11 INFO Utils:59 - Successfully started service 'HTTP file server' on port 49930. -2016-06-13 19:44:11 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 19:44:11 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:44:11 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 19:44:11 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 19:44:11 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 19:44:12 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 19:44:12 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 19:44:12 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 49949. -2016-06-13 19:44:12 INFO NettyBlockTransferService:59 - Server created on 49949 -2016-06-13 19:44:12 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 19:44:12 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:49949 with 730.6 MB RAM, BlockManagerId(driver, localhost, 49949) -2016-06-13 19:44:12 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 19:44:13 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 19:44:13 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 19:44:13 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 19:44:13 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 19:44:13 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:49949 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 19:44:13 INFO SparkContext:59 - Created broadcast 0 from textFile at LDAFromWebSuite.scala:22 -2016-06-13 19:44:14 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 19:44:14 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 19:44:14 INFO SparkContext:59 - Starting job: zipWithIndex at LDAFromWebSuite.scala:25 -2016-06-13 19:44:14 INFO DAGScheduler:59 - Got job 0 (zipWithIndex at LDAFromWebSuite.scala:25) with 1 output partitions -2016-06-13 19:44:14 INFO DAGScheduler:59 - Final stage: ResultStage 0(zipWithIndex at LDAFromWebSuite.scala:25) -2016-06-13 19:44:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:44:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:44:14 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at LDAFromWebSuite.scala:23), which has no missing parents -2016-06-13 19:44:14 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=171580, maxMem=766075207 -2016-06-13 19:44:14 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:44:14 INFO MemoryStore:59 - ensureFreeSpace(1939) called with curMem=174820, maxMem=766075207 -2016-06-13 19:44:14 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1939.0 B, free 730.4 MB) -2016-06-13 19:44:14 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:49949 (size: 1939.0 B, free: 730.6 MB) -2016-06-13 19:44:14 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:14 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at LDAFromWebSuite.scala:23) -2016-06-13 19:44:14 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 19:44:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2197 bytes) -2016-06-13 19:44:14 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 19:44:14 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:0+132 -2016-06-13 19:44:14 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 19:44:14 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 19:44:14 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 19:44:14 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 19:44:14 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 19:44:14 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 19:44:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 102 ms on localhost (1/1) -2016-06-13 19:44:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 19:44:14 INFO DAGScheduler:59 - ResultStage 0 (zipWithIndex at LDAFromWebSuite.scala:25) finished in 0.119 s -2016-06-13 19:44:14 INFO DAGScheduler:59 - Job 0 finished: zipWithIndex at LDAFromWebSuite.scala:25, took 0.285232 s -2016-06-13 19:44:15 INFO SparkContext:59 - Starting job: take at LDAOptimizer.scala:142 -2016-06-13 19:44:15 INFO DAGScheduler:59 - Got job 1 (take at LDAOptimizer.scala:142) with 1 output partitions -2016-06-13 19:44:15 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at LDAOptimizer.scala:142) -2016-06-13 19:44:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:44:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:44:15 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at map at LDAFromWebSuite.scala:25), which has no missing parents -2016-06-13 19:44:15 INFO MemoryStore:59 - ensureFreeSpace(3696) called with curMem=176759, maxMem=766075207 -2016-06-13 19:44:15 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-06-13 19:44:15 INFO MemoryStore:59 - ensureFreeSpace(2071) called with curMem=180455, maxMem=766075207 -2016-06-13 19:44:15 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:44:15 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:49949 (size: 2.0 KB, free: 730.6 MB) -2016-06-13 19:44:15 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:15 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at map at LDAFromWebSuite.scala:25) -2016-06-13 19:44:15 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 19:44:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2307 bytes) -2016-06-13 19:44:15 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 19:44:15 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-06-13 19:44:15 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:0+132 -2016-06-13 19:44:15 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=182526, maxMem=766075207 -2016-06-13 19:44:15 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 1224.0 B, free 730.4 MB) -2016-06-13 19:44:15 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:49949 (size: 1224.0 B, free: 730.6 MB) -2016-06-13 19:44:15 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2949 bytes result sent to driver -2016-06-13 19:44:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 46 ms on localhost (1/1) -2016-06-13 19:44:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 19:44:15 INFO DAGScheduler:59 - ResultStage 1 (take at LDAOptimizer.scala:142) finished in 0.046 s -2016-06-13 19:44:15 INFO DAGScheduler:59 - Job 1 finished: take at LDAOptimizer.scala:142, took 0.071911 s -2016-06-13 19:44:15 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:15 INFO DAGScheduler:59 - Registering RDD 19 (map at GraphImpl.scala:104) -2016-06-13 19:44:15 INFO DAGScheduler:59 - Registering RDD 23 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:44:15 INFO DAGScheduler:59 - Registering RDD 11 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:44:15 INFO DAGScheduler:59 - Registering RDD 6 (mapPartitionsWithIndex at LDAOptimizer.scala:128) -2016-06-13 19:44:15 INFO DAGScheduler:59 - Got job 2 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:15 INFO DAGScheduler:59 - Final stage: ResultStage 6(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:15 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 5, ShuffleMapStage 3, ShuffleMapStage 4) -2016-06-13 19:44:15 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 5, ShuffleMapStage 3, ShuffleMapStage 4) -2016-06-13 19:44:15 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[19] at map at GraphImpl.scala:104), which has no missing parents -2016-06-13 19:44:15 INFO MemoryStore:59 - ensureFreeSpace(5752) called with curMem=183750, maxMem=766075207 -2016-06-13 19:44:15 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.6 KB, free 730.4 MB) -2016-06-13 19:44:15 INFO MemoryStore:59 - ensureFreeSpace(3051) called with curMem=189502, maxMem=766075207 -2016-06-13 19:44:15 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 19:44:15 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:49949 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 19:44:15 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[19] at map at GraphImpl.scala:104) -2016-06-13 19:44:15 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 19:44:15 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[11] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:44:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:44:15 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=192553, maxMem=766075207 -2016-06-13 19:44:15 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.4 MB) -2016-06-13 19:44:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:44:15 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 3) -2016-06-13 19:44:15 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-06-13 19:44:15 INFO MemoryStore:59 - ensureFreeSpace(2760) called with curMem=197617, maxMem=766075207 -2016-06-13 19:44:15 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:44:15 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:49949 (size: 2.7 KB, free: 730.6 MB) -2016-06-13 19:44:15 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[11] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:44:15 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 19:44:15 INFO DAGScheduler:59 - Submitting ShuffleMapStage 5 (MapPartitionsRDD[6] at mapPartitionsWithIndex at LDAOptimizer.scala:128), which has no missing parents -2016-06-13 19:44:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 4, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:44:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 5, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:44:15 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 5) -2016-06-13 19:44:15 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 4) -2016-06-13 19:44:15 INFO MemoryStore:59 - ensureFreeSpace(4880) called with curMem=200377, maxMem=766075207 -2016-06-13 19:44:15 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-06-13 19:44:15 INFO MemoryStore:59 - ensureFreeSpace(2638) called with curMem=205257, maxMem=766075207 -2016-06-13 19:44:15 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-06-13 19:44:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:44:15 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:49949 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:44:15 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 5 (MapPartitionsRDD[6] at mapPartitionsWithIndex at LDAOptimizer.scala:128) -2016-06-13 19:44:15 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 19:44:15 INFO CacheManager:59 - Partition rdd_17_1 not found, computing it -2016-06-13 19:44:15 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-06-13 19:44:15 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:132+132 -2016-06-13 19:44:15 INFO CacheManager:59 - Partition rdd_17_0 not found, computing it -2016-06-13 19:44:15 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=207895, maxMem=766075207 -2016-06-13 19:44:15 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 880.0 B, free 730.4 MB) -2016-06-13 19:44:15 INFO CacheManager:59 - Another thread is loading rdd_4_1, waiting for it to finish... -2016-06-13 19:44:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:44:15 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:49949 (size: 880.0 B, free: 730.6 MB) -2016-06-13 19:44:15 INFO CacheManager:59 - Finished waiting for rdd_4_1 -2016-06-13 19:44:15 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 19:44:15 INFO MemoryStore:59 - ensureFreeSpace(3520) called with curMem=208775, maxMem=766075207 -2016-06-13 19:44:15 INFO MemoryStore:59 - Block rdd_17_0 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-06-13 19:44:15 INFO MemoryStore:59 - ensureFreeSpace(3296) called with curMem=212295, maxMem=766075207 -2016-06-13 19:44:15 INFO MemoryStore:59 - Block rdd_17_1 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:44:15 INFO BlockManagerInfo:59 - Added rdd_17_0 in memory on localhost:49949 (size: 3.4 KB, free: 730.6 MB) -2016-06-13 19:44:15 INFO BlockManagerInfo:59 - Added rdd_17_1 in memory on localhost:49949 (size: 3.2 KB, free: 730.6 MB) -2016-06-13 19:44:15 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 5). 2834 bytes result sent to driver -2016-06-13 19:44:15 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 4). 2254 bytes result sent to driver -2016-06-13 19:44:15 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 3). 1739 bytes result sent to driver -2016-06-13 19:44:15 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 2834 bytes result sent to driver -2016-06-13 19:44:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 6, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:44:15 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 6) -2016-06-13 19:44:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 7, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:44:15 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 7) -2016-06-13 19:44:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 3) in 186 ms on localhost (1/2) -2016-06-13 19:44:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 4) in 165 ms on localhost (1/2) -2016-06-13 19:44:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 5) in 164 ms on localhost (2/2) -2016-06-13 19:44:15 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at VertexRDD.scala:358) finished in 0.167 s -2016-06-13 19:44:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 19:44:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 196 ms on localhost (2/2) -2016-06-13 19:44:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 19:44:15 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:15 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 5, ShuffleMapStage 2) -2016-06-13 19:44:15 INFO DAGScheduler:59 - waiting: Set(ResultStage 6, ShuffleMapStage 3) -2016-06-13 19:44:15 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:15 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List(ShuffleMapStage 5, ShuffleMapStage 3) -2016-06-13 19:44:15 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:44:15 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 19:44:15 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List(ShuffleMapStage 2) -2016-06-13 19:44:15 INFO DAGScheduler:59 - ShuffleMapStage 2 (map at GraphImpl.scala:104) finished in 0.208 s -2016-06-13 19:44:15 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:15 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 5) -2016-06-13 19:44:15 INFO DAGScheduler:59 - waiting: Set(ResultStage 6, ShuffleMapStage 3) -2016-06-13 19:44:15 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:15 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List(ShuffleMapStage 5, ShuffleMapStage 3) -2016-06-13 19:44:15 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List() -2016-06-13 19:44:15 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[23] at mapPartitions at VertexRDD.scala:358), which is now runnable -2016-06-13 19:44:15 INFO MemoryStore:59 - ensureFreeSpace(2784) called with curMem=215591, maxMem=766075207 -2016-06-13 19:44:15 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:44:15 INFO MemoryStore:59 - ensureFreeSpace(1698) called with curMem=218375, maxMem=766075207 -2016-06-13 19:44:15 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1698.0 B, free 730.4 MB) -2016-06-13 19:44:15 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:49949 (size: 1698.0 B, free: 730.6 MB) -2016-06-13 19:44:15 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 3 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[23] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:44:15 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 19:44:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 1890 bytes) -2016-06-13 19:44:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 1890 bytes) -2016-06-13 19:44:15 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 19:44:15 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 9) -2016-06-13 19:44:15 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 19:44:15 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 8) -2016-06-13 19:44:15 INFO CacheManager:59 - Partition rdd_21_1 not found, computing it -2016-06-13 19:44:15 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 20 ms -2016-06-13 19:44:15 INFO MemoryStore:59 - ensureFreeSpace(3416) called with curMem=220073, maxMem=766075207 -2016-06-13 19:44:15 INFO MemoryStore:59 - Block rdd_21_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 19:44:15 INFO BlockManagerInfo:59 - Added rdd_21_1 in memory on localhost:49949 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:44:15 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 7). 2254 bytes result sent to driver -2016-06-13 19:44:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 7) in 142 ms on localhost (1/2) -2016-06-13 19:44:15 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 9). 1955 bytes result sent to driver -2016-06-13 19:44:15 INFO CacheManager:59 - Partition rdd_21_0 not found, computing it -2016-06-13 19:44:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 9) in 102 ms on localhost (1/2) -2016-06-13 19:44:15 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-06-13 19:44:15 INFO MemoryStore:59 - ensureFreeSpace(3408) called with curMem=223489, maxMem=766075207 -2016-06-13 19:44:15 INFO MemoryStore:59 - Block rdd_21_0 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 19:44:15 INFO BlockManagerInfo:59 - Added rdd_21_0 in memory on localhost:49949 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:44:15 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 6). 2254 bytes result sent to driver -2016-06-13 19:44:15 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 8). 1955 bytes result sent to driver -2016-06-13 19:44:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 6) in 218 ms on localhost (2/2) -2016-06-13 19:44:15 INFO DAGScheduler:59 - ShuffleMapStage 5 (mapPartitionsWithIndex at LDAOptimizer.scala:128) finished in 0.341 s -2016-06-13 19:44:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 19:44:15 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:15 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 3) -2016-06-13 19:44:15 INFO DAGScheduler:59 - waiting: Set(ResultStage 6) -2016-06-13 19:44:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 8) in 159 ms on localhost (2/2) -2016-06-13 19:44:15 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 19:44:15 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List(ShuffleMapStage 3) -2016-06-13 19:44:15 INFO DAGScheduler:59 - ShuffleMapStage 3 (mapPartitions at VertexRDD.scala:358) finished in 0.166 s -2016-06-13 19:44:15 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:15 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:15 INFO DAGScheduler:59 - waiting: Set(ResultStage 6) -2016-06-13 19:44:15 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:15 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List() -2016-06-13 19:44:15 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[30] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:15 INFO MemoryStore:59 - ensureFreeSpace(5584) called with curMem=226897, maxMem=766075207 -2016-06-13 19:44:15 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.4 MB) -2016-06-13 19:44:15 INFO MemoryStore:59 - ensureFreeSpace(2802) called with curMem=232481, maxMem=766075207 -2016-06-13 19:44:15 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:44:15 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:49949 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:44:15 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[30] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:15 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-06-13 19:44:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 10, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:44:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:44:15 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 11) -2016-06-13 19:44:15 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 10) -2016-06-13 19:44:16 INFO CacheManager:59 - Partition rdd_26_0 not found, computing it -2016-06-13 19:44:16 INFO CacheManager:59 - Partition rdd_26_1 not found, computing it -2016-06-13 19:44:16 INFO CacheManager:59 - Partition rdd_14_1 not found, computing it -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:16 INFO CacheManager:59 - Partition rdd_14_0 not found, computing it -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(2496) called with curMem=235283, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block rdd_14_0 stored as values in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(2400) called with curMem=237779, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block rdd_14_1 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Added rdd_14_0 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Added rdd_14_1 in memory on localhost:49949 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(2400) called with curMem=240179, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block rdd_26_1 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(2496) called with curMem=242579, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block rdd_26_0 stored as values in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Added rdd_26_1 in memory on localhost:49949 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Added rdd_26_0 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 10). 1931 bytes result sent to driver -2016-06-13 19:44:16 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 11). 1931 bytes result sent to driver -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 10) in 76 ms on localhost (1/2) -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 11) in 73 ms on localhost (2/2) -2016-06-13 19:44:16 INFO DAGScheduler:59 - ResultStage 6 (fold at LDAOptimizer.scala:205) finished in 0.077 s -2016-06-13 19:44:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 19:44:16 INFO DAGScheduler:59 - Job 2 finished: fold at LDAOptimizer.scala:205, took 0.561443 s -2016-06-13 19:44:16 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:16 INFO DAGScheduler:59 - Registering RDD 31 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Registering RDD 35 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Got job 3 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:16 INFO DAGScheduler:59 - Final stage: ResultStage 13(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12, ShuffleMapStage 10, ShuffleMapStage 7, ShuffleMapStage 8) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 11 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[31] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(4360) called with curMem=245075, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(2388) called with curMem=249435, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.3 MB) -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:49949 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 11 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[31] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:16 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 19:44:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 12, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:44:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 13, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:44:16 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 13) -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:49949 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 12) -2016-06-13 19:44:16 INFO BlockManager:59 - Found block rdd_26_1 locally -2016-06-13 19:44:16 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 19:44:16 INFO BlockManager:59 - Found block rdd_26_0 locally -2016-06-13 19:44:16 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 12). 2254 bytes result sent to driver -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:49949 in memory (size: 1698.0 B, free: 730.5 MB) -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 12) in 64 ms on localhost (1/2) -2016-06-13 19:44:16 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 19:44:16 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 13). 2254 bytes result sent to driver -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:49949 in memory (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 13) in 74 ms on localhost (2/2) -2016-06-13 19:44:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 19:44:16 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 19:44:16 INFO DAGScheduler:59 - ShuffleMapStage 11 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.077 s -2016-06-13 19:44:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:16 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:16 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 12, ResultStage 13) -2016-06-13 19:44:16 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:16 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 12: List() -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:49949 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List(ShuffleMapStage 12) -2016-06-13 19:44:16 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 19:44:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[35] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:49949 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=214810, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 19:44:16 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=218706, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:49949 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:49949 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[35] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:16 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 19:44:16 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 19:44:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 14, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:49949 in memory (size: 1939.0 B, free: 730.5 MB) -2016-06-13 19:44:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 15, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:16 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 14) -2016-06-13 19:44:16 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 19:44:16 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 15) -2016-06-13 19:44:16 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:16 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:16 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 15). 2470 bytes result sent to driver -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 15) in 49 ms on localhost (1/2) -2016-06-13 19:44:16 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 14). 2470 bytes result sent to driver -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 14) in 61 ms on localhost (2/2) -2016-06-13 19:44:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 19:44:16 INFO DAGScheduler:59 - ShuffleMapStage 12 (mapPartitions at GraphImpl.scala:235) finished in 0.063 s -2016-06-13 19:44:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:16 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:16 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-06-13 19:44:16 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:16 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-06-13 19:44:16 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[43] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(6432) called with curMem=210046, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 6.3 KB, free 730.4 MB) -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(3040) called with curMem=216478, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:49949 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[43] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:16 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 19:44:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 16, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:44:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:44:16 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 16) -2016-06-13 19:44:16 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 17) -2016-06-13 19:44:16 INFO CacheManager:59 - Partition rdd_39_1 not found, computing it -2016-06-13 19:44:16 INFO BlockManager:59 - Found block rdd_26_1 locally -2016-06-13 19:44:16 INFO CacheManager:59 - Partition rdd_39_0 not found, computing it -2016-06-13 19:44:16 INFO BlockManager:59 - Found block rdd_26_0 locally -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=219518, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block rdd_39_1 stored as values in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=221966, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block rdd_39_0 stored as values in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Added rdd_39_1 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Added rdd_39_0 in memory on localhost:49949 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 17). 2971 bytes result sent to driver -2016-06-13 19:44:16 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 16). 2971 bytes result sent to driver -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 17) in 24 ms on localhost (1/2) -2016-06-13 19:44:16 INFO DAGScheduler:59 - ResultStage 13 (fold at LDAOptimizer.scala:205) finished in 0.027 s -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 16) in 26 ms on localhost (2/2) -2016-06-13 19:44:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 19:44:16 INFO DAGScheduler:59 - Job 3 finished: fold at LDAOptimizer.scala:205, took 0.299523 s -2016-06-13 19:44:16 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:44:16 INFO DAGScheduler:59 - Registering RDD 44 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Registering RDD 48 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Got job 4 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:16 INFO DAGScheduler:59 - Final stage: ResultStage 22(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15, ShuffleMapStage 19, ShuffleMapStage 17, ShuffleMapStage 21, ShuffleMapStage 14) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 21) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 20 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[44] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=224510, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.3 KB, free 730.4 MB) -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(2727) called with curMem=229958, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:49949 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 20 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[44] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:16 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-06-13 19:44:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 18, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:44:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 19, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:44:16 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 18) -2016-06-13 19:44:16 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 19) -2016-06-13 19:44:16 INFO BlockManager:59 - Found block rdd_39_0 locally -2016-06-13 19:44:16 INFO BlockManager:59 - Found block rdd_39_1 locally -2016-06-13 19:44:16 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 19). 2254 bytes result sent to driver -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 19) in 30 ms on localhost (1/2) -2016-06-13 19:44:16 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 18). 2254 bytes result sent to driver -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 18) in 34 ms on localhost (2/2) -2016-06-13 19:44:16 INFO DAGScheduler:59 - ShuffleMapStage 20 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.035 s -2016-06-13 19:44:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 19:44:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:16 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:16 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 21, ResultStage 22) -2016-06-13 19:44:16 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:16 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 21: List() -2016-06-13 19:44:16 INFO DAGScheduler:59 - Missing parents for ResultStage 22: List(ShuffleMapStage 21) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 21 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[48] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=232685, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=236581, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:49949 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 21 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[48] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:16 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 19:44:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 20, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 21, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:16 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 20) -2016-06-13 19:44:16 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 21) -2016-06-13 19:44:16 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:44:16 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:16 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 21). 2470 bytes result sent to driver -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 21) in 41 ms on localhost (1/2) -2016-06-13 19:44:16 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 20). 2470 bytes result sent to driver -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 20) in 47 ms on localhost (2/2) -2016-06-13 19:44:16 INFO DAGScheduler:59 - ShuffleMapStage 21 (mapPartitions at GraphImpl.scala:235) finished in 0.048 s -2016-06-13 19:44:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 19:44:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:16 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:16 INFO DAGScheduler:59 - waiting: Set(ResultStage 22) -2016-06-13 19:44:16 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:16 INFO DAGScheduler:59 - Missing parents for ResultStage 22: List() -2016-06-13 19:44:16 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[56] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(6864) called with curMem=238867, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 6.7 KB, free 730.4 MB) -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(3157) called with curMem=245731, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.3 MB) -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:49949 (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[56] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:16 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-06-13 19:44:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 22, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:44:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 23, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:44:16 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 23) -2016-06-13 19:44:16 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 22) -2016-06-13 19:44:16 INFO CacheManager:59 - Partition rdd_52_1 not found, computing it -2016-06-13 19:44:16 INFO BlockManager:59 - Found block rdd_39_1 locally -2016-06-13 19:44:16 INFO CacheManager:59 - Partition rdd_52_0 not found, computing it -2016-06-13 19:44:16 INFO BlockManager:59 - Found block rdd_39_0 locally -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=248888, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block rdd_52_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=251336, maxMem=766075207 -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Added rdd_52_1 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO MemoryStore:59 - Block rdd_52_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Added rdd_52_0 in memory on localhost:49949 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 23). 2971 bytes result sent to driver -2016-06-13 19:44:16 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 22). 2971 bytes result sent to driver -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 23) in 24 ms on localhost (1/2) -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 22) in 27 ms on localhost (2/2) -2016-06-13 19:44:16 INFO DAGScheduler:59 - ResultStage 22 (fold at LDAOptimizer.scala:205) finished in 0.028 s -2016-06-13 19:44:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 19:44:16 INFO DAGScheduler:59 - Job 4 finished: fold at LDAOptimizer.scala:205, took 0.172617 s -2016-06-13 19:44:16 INFO ZippedPartitionsRDD2:59 - Removing RDD 26 from persistence list -2016-06-13 19:44:16 INFO ZippedPartitionsRDD2:59 - Removing RDD 33 from persistence list -2016-06-13 19:44:16 INFO BlockManager:59 - Removing RDD 26 -2016-06-13 19:44:16 INFO BlockManager:59 - Removing RDD 33 -2016-06-13 19:44:16 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:44:16 INFO DAGScheduler:59 - Registering RDD 57 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Registering RDD 61 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Got job 5 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:16 INFO DAGScheduler:59 - Final stage: ResultStage 33(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30, ShuffleMapStage 32, ShuffleMapStage 24, ShuffleMapStage 28, ShuffleMapStage 26, ShuffleMapStage 23) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 32) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 31 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[57] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(5872) called with curMem=248984, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 5.7 KB, free 730.3 MB) -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(2848) called with curMem=254856, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:49949 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 31 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[57] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:16 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 19:44:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 24, localhost, PROCESS_LOCAL, 2196 bytes) -2016-06-13 19:44:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 25, localhost, PROCESS_LOCAL, 2196 bytes) -2016-06-13 19:44:16 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 24) -2016-06-13 19:44:16 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 25) -2016-06-13 19:44:16 INFO BlockManager:59 - Found block rdd_52_1 locally -2016-06-13 19:44:16 INFO BlockManager:59 - Found block rdd_52_0 locally -2016-06-13 19:44:16 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 25). 2254 bytes result sent to driver -2016-06-13 19:44:16 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 24). 2254 bytes result sent to driver -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 25) in 33 ms on localhost (1/2) -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 24) in 38 ms on localhost (2/2) -2016-06-13 19:44:16 INFO DAGScheduler:59 - ShuffleMapStage 31 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.038 s -2016-06-13 19:44:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:16 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:16 INFO DAGScheduler:59 - waiting: Set(ResultStage 33, ShuffleMapStage 32) -2016-06-13 19:44:16 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 19:44:16 INFO DAGScheduler:59 - Missing parents for ResultStage 33: List(ShuffleMapStage 32) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 32: List() -2016-06-13 19:44:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 32 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[61] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=257704, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(2289) called with curMem=261600, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:49949 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 32 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[61] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:16 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-06-13 19:44:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 26, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 27, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:16 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 27) -2016-06-13 19:44:16 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 26) -2016-06-13 19:44:16 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:44:16 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:16 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 27). 2470 bytes result sent to driver -2016-06-13 19:44:16 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 26). 2470 bytes result sent to driver -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 27) in 30 ms on localhost (1/2) -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 26) in 32 ms on localhost (2/2) -2016-06-13 19:44:16 INFO DAGScheduler:59 - ShuffleMapStage 32 (mapPartitions at GraphImpl.scala:235) finished in 0.032 s -2016-06-13 19:44:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 19:44:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:16 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:16 INFO DAGScheduler:59 - waiting: Set(ResultStage 33) -2016-06-13 19:44:16 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:16 INFO DAGScheduler:59 - Missing parents for ResultStage 33: List() -2016-06-13 19:44:16 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[69] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(7288) called with curMem=263889, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 7.1 KB, free 730.3 MB) -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(3298) called with curMem=271177, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.3 MB) -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:49949 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[69] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:16 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-06-13 19:44:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 28, localhost, PROCESS_LOCAL, 2248 bytes) -2016-06-13 19:44:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 29, localhost, PROCESS_LOCAL, 2248 bytes) -2016-06-13 19:44:16 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 28) -2016-06-13 19:44:16 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 29) -2016-06-13 19:44:16 INFO CacheManager:59 - Partition rdd_65_0 not found, computing it -2016-06-13 19:44:16 INFO BlockManager:59 - Found block rdd_52_0 locally -2016-06-13 19:44:16 INFO CacheManager:59 - Partition rdd_65_1 not found, computing it -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:16 INFO BlockManager:59 - Found block rdd_52_1 locally -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=274475, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block rdd_65_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=277019, maxMem=766075207 -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Added rdd_65_0 in memory on localhost:49949 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO MemoryStore:59 - Block rdd_65_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Added rdd_65_1 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 28). 2971 bytes result sent to driver -2016-06-13 19:44:16 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 29). 2971 bytes result sent to driver -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 29) in 26 ms on localhost (1/2) -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 28) in 27 ms on localhost (2/2) -2016-06-13 19:44:16 INFO DAGScheduler:59 - ResultStage 33 (fold at LDAOptimizer.scala:205) finished in 0.028 s -2016-06-13 19:44:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 19:44:16 INFO DAGScheduler:59 - Job 5 finished: fold at LDAOptimizer.scala:205, took 0.158831 s -2016-06-13 19:44:16 INFO MapPartitionsRDD:59 - Removing RDD 39 from persistence list -2016-06-13 19:44:16 INFO ZippedPartitionsRDD2:59 - Removing RDD 46 from persistence list -2016-06-13 19:44:16 INFO BlockManager:59 - Removing RDD 39 -2016-06-13 19:44:16 INFO BlockManager:59 - Removing RDD 46 -2016-06-13 19:44:16 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:44:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:44:16 INFO DAGScheduler:59 - Registering RDD 70 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Registering RDD 74 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Got job 6 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:16 INFO DAGScheduler:59 - Final stage: ResultStage 46(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 45, ShuffleMapStage 37, ShuffleMapStage 34, ShuffleMapStage 35, ShuffleMapStage 39, ShuffleMapStage 43, ShuffleMapStage 41) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 45) -2016-06-13 19:44:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 44 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[70] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(6304) called with curMem=274475, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 6.2 KB, free 730.3 MB) -2016-06-13 19:44:16 INFO MemoryStore:59 - ensureFreeSpace(2991) called with curMem=280779, maxMem=766075207 -2016-06-13 19:44:16 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-06-13 19:44:16 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:49949 (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:44:16 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 44 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[70] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:16 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-06-13 19:44:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 30, localhost, PROCESS_LOCAL, 2237 bytes) -2016-06-13 19:44:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 31, localhost, PROCESS_LOCAL, 2237 bytes) -2016-06-13 19:44:16 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 30) -2016-06-13 19:44:16 INFO BlockManager:59 - Found block rdd_65_0 locally -2016-06-13 19:44:16 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 30). 2254 bytes result sent to driver -2016-06-13 19:44:16 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 31) -2016-06-13 19:44:16 INFO BlockManager:59 - Found block rdd_65_1 locally -2016-06-13 19:44:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 30) in 46 ms on localhost (1/2) -2016-06-13 19:44:17 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 31). 2254 bytes result sent to driver -2016-06-13 19:44:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 31) in 63 ms on localhost (2/2) -2016-06-13 19:44:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-06-13 19:44:17 INFO DAGScheduler:59 - ShuffleMapStage 44 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.004 s -2016-06-13 19:44:17 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:17 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:17 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 45, ResultStage 46) -2016-06-13 19:44:17 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:17 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 45: List() -2016-06-13 19:44:17 INFO DAGScheduler:59 - Missing parents for ResultStage 46: List(ShuffleMapStage 45) -2016-06-13 19:44:17 INFO DAGScheduler:59 - Submitting ShuffleMapStage 45 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[74] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=283770, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(2284) called with curMem=287666, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:49949 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 45 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[74] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:17 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-06-13 19:44:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 32, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 33, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:17 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 32) -2016-06-13 19:44:17 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 33) -2016-06-13 19:44:17 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:44:17 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:17 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 33). 2470 bytes result sent to driver -2016-06-13 19:44:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 33) in 49 ms on localhost (1/2) -2016-06-13 19:44:17 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 32). 2470 bytes result sent to driver -2016-06-13 19:44:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 32) in 59 ms on localhost (2/2) -2016-06-13 19:44:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-06-13 19:44:17 INFO DAGScheduler:59 - ShuffleMapStage 45 (mapPartitions at GraphImpl.scala:235) finished in 0.062 s -2016-06-13 19:44:17 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:17 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:17 INFO DAGScheduler:59 - waiting: Set(ResultStage 46) -2016-06-13 19:44:17 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:17 INFO DAGScheduler:59 - Missing parents for ResultStage 46: List() -2016-06-13 19:44:17 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[82] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(7720) called with curMem=289950, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 7.5 KB, free 730.3 MB) -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=297670, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.3 MB) -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:49949 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[82] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:17 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-06-13 19:44:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 34, localhost, PROCESS_LOCAL, 2289 bytes) -2016-06-13 19:44:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 35, localhost, PROCESS_LOCAL, 2289 bytes) -2016-06-13 19:44:17 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 34) -2016-06-13 19:44:17 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 35) -2016-06-13 19:44:17 INFO CacheManager:59 - Partition rdd_78_0 not found, computing it -2016-06-13 19:44:17 INFO BlockManager:59 - Found block rdd_65_0 locally -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:17 INFO CacheManager:59 - Partition rdd_78_1 not found, computing it -2016-06-13 19:44:17 INFO BlockManager:59 - Found block rdd_65_1 locally -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=301070, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block rdd_78_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=303614, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block rdd_78_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Added rdd_78_0 in memory on localhost:49949 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Added rdd_78_1 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 34). 2971 bytes result sent to driver -2016-06-13 19:44:17 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 35). 2971 bytes result sent to driver -2016-06-13 19:44:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 34) in 37 ms on localhost (1/2) -2016-06-13 19:44:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 35) in 36 ms on localhost (2/2) -2016-06-13 19:44:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-06-13 19:44:17 INFO DAGScheduler:59 - ResultStage 46 (fold at LDAOptimizer.scala:205) finished in 0.039 s -2016-06-13 19:44:17 INFO DAGScheduler:59 - Job 6 finished: fold at LDAOptimizer.scala:205, took 0.253777 s -2016-06-13 19:44:17 INFO MapPartitionsRDD:59 - Removing RDD 52 from persistence list -2016-06-13 19:44:17 INFO ZippedPartitionsRDD2:59 - Removing RDD 59 from persistence list -2016-06-13 19:44:17 INFO BlockManager:59 - Removing RDD 52 -2016-06-13 19:44:17 INFO BlockManager:59 - Removing RDD 59 -2016-06-13 19:44:17 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:44:17 INFO DAGScheduler:59 - Registering RDD 83 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:17 INFO DAGScheduler:59 - Registering RDD 87 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:17 INFO DAGScheduler:59 - Got job 7 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:17 INFO DAGScheduler:59 - Final stage: ResultStage 61(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 48, ShuffleMapStage 60, ShuffleMapStage 52, ShuffleMapStage 56, ShuffleMapStage 50, ShuffleMapStage 54, ShuffleMapStage 58, ShuffleMapStage 47) -2016-06-13 19:44:17 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 60) -2016-06-13 19:44:17 INFO DAGScheduler:59 - Submitting ShuffleMapStage 59 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[83] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(6728) called with curMem=301070, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 6.6 KB, free 730.3 MB) -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(3078) called with curMem=307798, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.3 MB) -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:49949 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 59 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[83] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:17 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 2 tasks -2016-06-13 19:44:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 36, localhost, PROCESS_LOCAL, 2278 bytes) -2016-06-13 19:44:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 37, localhost, PROCESS_LOCAL, 2278 bytes) -2016-06-13 19:44:17 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 36) -2016-06-13 19:44:17 INFO BlockManager:59 - Found block rdd_78_0 locally -2016-06-13 19:44:17 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 37) -2016-06-13 19:44:17 INFO BlockManager:59 - Found block rdd_78_1 locally -2016-06-13 19:44:17 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 36). 2254 bytes result sent to driver -2016-06-13 19:44:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 36) in 37 ms on localhost (1/2) -2016-06-13 19:44:17 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 37). 2254 bytes result sent to driver -2016-06-13 19:44:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 37) in 48 ms on localhost (2/2) -2016-06-13 19:44:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-06-13 19:44:17 INFO DAGScheduler:59 - ShuffleMapStage 59 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.054 s -2016-06-13 19:44:17 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:17 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:17 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 60, ResultStage 61) -2016-06-13 19:44:17 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:17 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 60: List() -2016-06-13 19:44:17 INFO DAGScheduler:59 - Missing parents for ResultStage 61: List(ShuffleMapStage 60) -2016-06-13 19:44:17 INFO DAGScheduler:59 - Submitting ShuffleMapStage 60 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[87] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=310876, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(2280) called with curMem=314772, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:49949 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 60 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[87] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:49949 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO TaskSchedulerImpl:59 - Adding task set 60.0 with 2 tasks -2016-06-13 19:44:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 60.0 (TID 38, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 60.0 (TID 39, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:17 INFO Executor:59 - Running task 1.0 in stage 60.0 (TID 39) -2016-06-13 19:44:17 INFO Executor:59 - Running task 0.0 in stage 60.0 (TID 38) -2016-06-13 19:44:17 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:17 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:49949 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:17 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:49949 in memory (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO Executor:59 - Finished task 1.0 in stage 60.0 (TID 39). 2470 bytes result sent to driver -2016-06-13 19:44:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 60.0 (TID 39) in 69 ms on localhost (1/2) -2016-06-13 19:44:17 INFO Executor:59 - Finished task 0.0 in stage 60.0 (TID 38). 2470 bytes result sent to driver -2016-06-13 19:44:17 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 19:44:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 60.0 (TID 38) in 85 ms on localhost (2/2) -2016-06-13 19:44:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 60.0, whose tasks have all completed, from pool -2016-06-13 19:44:17 INFO DAGScheduler:59 - ShuffleMapStage 60 (mapPartitions at GraphImpl.scala:235) finished in 0.086 s -2016-06-13 19:44:17 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:17 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:17 INFO DAGScheduler:59 - waiting: Set(ResultStage 61) -2016-06-13 19:44:17 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:49949 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO DAGScheduler:59 - Missing parents for ResultStage 61: List() -2016-06-13 19:44:17 INFO DAGScheduler:59 - Submitting ResultStage 61 (MapPartitionsRDD[95] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(8144) called with curMem=279871, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 8.0 KB, free 730.3 MB) -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=288015, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.3 MB) -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:49949 (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 61 (MapPartitionsRDD[95] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:17 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 2 tasks -2016-06-13 19:44:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 40, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:44:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 41, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:44:17 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 41) -2016-06-13 19:44:17 INFO CacheManager:59 - Partition rdd_91_1 not found, computing it -2016-06-13 19:44:17 INFO BlockManager:59 - Found block rdd_78_1 locally -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:17 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=289217, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block rdd_91_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Added rdd_91_1 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 40) -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:49949 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 41). 2971 bytes result sent to driver -2016-06-13 19:44:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 41) in 29 ms on localhost (1/2) -2016-06-13 19:44:17 INFO CacheManager:59 - Partition rdd_91_0 not found, computing it -2016-06-13 19:44:17 INFO BlockManager:59 - Found block rdd_78_0 locally -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=287769, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block rdd_91_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Added rdd_91_0 in memory on localhost:49949 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 40). 2971 bytes result sent to driver -2016-06-13 19:44:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 40) in 48 ms on localhost (2/2) -2016-06-13 19:44:17 INFO DAGScheduler:59 - ResultStage 61 (fold at LDAOptimizer.scala:205) finished in 0.054 s -2016-06-13 19:44:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-06-13 19:44:17 INFO DAGScheduler:59 - Job 7 finished: fold at LDAOptimizer.scala:205, took 0.322188 s -2016-06-13 19:44:17 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:49949 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:49949 in memory (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:49949 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO MapPartitionsRDD:59 - Removing RDD 65 from persistence list -2016-06-13 19:44:17 INFO ZippedPartitionsRDD2:59 - Removing RDD 72 from persistence list -2016-06-13 19:44:17 INFO BlockManager:59 - Removing RDD 65 -2016-06-13 19:44:17 INFO BlockManager:59 - Removing RDD 72 -2016-06-13 19:44:17 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:49949 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:17 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:49949 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:17 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:49949 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:44:17 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:49949 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:44:17 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:44:17 INFO DAGScheduler:59 - Registering RDD 96 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:17 INFO DAGScheduler:59 - Registering RDD 100 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:17 INFO DAGScheduler:59 - Got job 8 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:17 INFO DAGScheduler:59 - Final stage: ResultStage 78(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 63, ShuffleMapStage 67, ShuffleMapStage 71, ShuffleMapStage 75, ShuffleMapStage 69, ShuffleMapStage 73, ShuffleMapStage 65, ShuffleMapStage 62, ShuffleMapStage 77) -2016-06-13 19:44:17 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 77) -2016-06-13 19:44:17 INFO DAGScheduler:59 - Submitting ShuffleMapStage 76 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[96] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(7160) called with curMem=229821, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 7.0 KB, free 730.4 MB) -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(3182) called with curMem=236981, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.4 MB) -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:49949 (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 76 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[96] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:17 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 2 tasks -2016-06-13 19:44:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 42, localhost, PROCESS_LOCAL, 2319 bytes) -2016-06-13 19:44:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 43, localhost, PROCESS_LOCAL, 2319 bytes) -2016-06-13 19:44:17 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 42) -2016-06-13 19:44:17 INFO BlockManager:59 - Found block rdd_91_0 locally -2016-06-13 19:44:17 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 43) -2016-06-13 19:44:17 INFO BlockManager:59 - Found block rdd_91_1 locally -2016-06-13 19:44:17 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 42). 2254 bytes result sent to driver -2016-06-13 19:44:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 42) in 70 ms on localhost (1/2) -2016-06-13 19:44:17 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 43). 2254 bytes result sent to driver -2016-06-13 19:44:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 43) in 81 ms on localhost (2/2) -2016-06-13 19:44:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-06-13 19:44:17 INFO DAGScheduler:59 - ShuffleMapStage 76 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.084 s -2016-06-13 19:44:17 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:17 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:17 INFO DAGScheduler:59 - waiting: Set(ResultStage 78, ShuffleMapStage 77) -2016-06-13 19:44:17 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:17 INFO DAGScheduler:59 - Missing parents for ResultStage 78: List(ShuffleMapStage 77) -2016-06-13 19:44:17 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 77: List() -2016-06-13 19:44:17 INFO DAGScheduler:59 - Submitting ShuffleMapStage 77 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[100] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=240163, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(2283) called with curMem=244059, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:49949 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 77 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[100] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:17 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 2 tasks -2016-06-13 19:44:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 44, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 45, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:17 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 44) -2016-06-13 19:44:17 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 45) -2016-06-13 19:44:17 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:17 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:17 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 44). 2470 bytes result sent to driver -2016-06-13 19:44:17 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 45). 2470 bytes result sent to driver -2016-06-13 19:44:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 44) in 49 ms on localhost (1/2) -2016-06-13 19:44:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 45) in 50 ms on localhost (2/2) -2016-06-13 19:44:17 INFO DAGScheduler:59 - ShuffleMapStage 77 (mapPartitions at GraphImpl.scala:235) finished in 0.052 s -2016-06-13 19:44:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-06-13 19:44:17 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:17 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:17 INFO DAGScheduler:59 - waiting: Set(ResultStage 78) -2016-06-13 19:44:17 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:17 INFO DAGScheduler:59 - Missing parents for ResultStage 78: List() -2016-06-13 19:44:17 INFO DAGScheduler:59 - Submitting ResultStage 78 (MapPartitionsRDD[108] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(8576) called with curMem=246342, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 8.4 KB, free 730.3 MB) -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(3582) called with curMem=254918, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:49949 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 78 (MapPartitionsRDD[108] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:17 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 2 tasks -2016-06-13 19:44:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 46, localhost, PROCESS_LOCAL, 2371 bytes) -2016-06-13 19:44:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 47, localhost, PROCESS_LOCAL, 2371 bytes) -2016-06-13 19:44:17 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 46) -2016-06-13 19:44:17 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 47) -2016-06-13 19:44:17 INFO CacheManager:59 - Partition rdd_104_0 not found, computing it -2016-06-13 19:44:17 INFO BlockManager:59 - Found block rdd_91_0 locally -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:17 INFO CacheManager:59 - Partition rdd_104_1 not found, computing it -2016-06-13 19:44:17 INFO BlockManager:59 - Found block rdd_91_1 locally -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=258500, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block rdd_104_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Added rdd_104_0 in memory on localhost:49949 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=261044, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block rdd_104_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Added rdd_104_1 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 46). 2971 bytes result sent to driver -2016-06-13 19:44:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 46) in 32 ms on localhost (1/2) -2016-06-13 19:44:17 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 47). 2971 bytes result sent to driver -2016-06-13 19:44:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 47) in 36 ms on localhost (2/2) -2016-06-13 19:44:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-06-13 19:44:17 INFO DAGScheduler:59 - ResultStage 78 (fold at LDAOptimizer.scala:205) finished in 0.038 s -2016-06-13 19:44:17 INFO DAGScheduler:59 - Job 8 finished: fold at LDAOptimizer.scala:205, took 0.269256 s -2016-06-13 19:44:17 INFO MapPartitionsRDD:59 - Removing RDD 78 from persistence list -2016-06-13 19:44:17 INFO ZippedPartitionsRDD2:59 - Removing RDD 85 from persistence list -2016-06-13 19:44:17 INFO BlockManager:59 - Removing RDD 78 -2016-06-13 19:44:17 INFO BlockManager:59 - Removing RDD 85 -2016-06-13 19:44:17 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:44:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:44:17 INFO DAGScheduler:59 - Registering RDD 109 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:17 INFO DAGScheduler:59 - Registering RDD 113 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:17 INFO DAGScheduler:59 - Got job 9 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:17 INFO DAGScheduler:59 - Final stage: ResultStage 97(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 84, ShuffleMapStage 96, ShuffleMapStage 88, ShuffleMapStage 82, ShuffleMapStage 86, ShuffleMapStage 90, ShuffleMapStage 79, ShuffleMapStage 94, ShuffleMapStage 80, ShuffleMapStage 92) -2016-06-13 19:44:17 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 96) -2016-06-13 19:44:17 INFO DAGScheduler:59 - Submitting ShuffleMapStage 95 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[109] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(7584) called with curMem=258500, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 7.4 KB, free 730.3 MB) -2016-06-13 19:44:17 INFO MemoryStore:59 - ensureFreeSpace(3283) called with curMem=266084, maxMem=766075207 -2016-06-13 19:44:17 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.3 MB) -2016-06-13 19:44:17 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:49949 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:44:17 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 95 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[109] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:17 INFO TaskSchedulerImpl:59 - Adding task set 95.0 with 2 tasks -2016-06-13 19:44:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 95.0 (TID 48, localhost, PROCESS_LOCAL, 2360 bytes) -2016-06-13 19:44:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 95.0 (TID 49, localhost, PROCESS_LOCAL, 2360 bytes) -2016-06-13 19:44:17 INFO Executor:59 - Running task 0.0 in stage 95.0 (TID 48) -2016-06-13 19:44:17 INFO Executor:59 - Running task 1.0 in stage 95.0 (TID 49) -2016-06-13 19:44:17 INFO BlockManager:59 - Found block rdd_104_0 locally -2016-06-13 19:44:17 INFO BlockManager:59 - Found block rdd_104_1 locally -2016-06-13 19:44:18 INFO Executor:59 - Finished task 1.0 in stage 95.0 (TID 49). 2254 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 95.0 (TID 49) in 68 ms on localhost (1/2) -2016-06-13 19:44:18 INFO Executor:59 - Finished task 0.0 in stage 95.0 (TID 48). 2254 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 95.0 (TID 48) in 75 ms on localhost (2/2) -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 95.0, whose tasks have all completed, from pool -2016-06-13 19:44:18 INFO DAGScheduler:59 - ShuffleMapStage 95 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.078 s -2016-06-13 19:44:18 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:18 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:18 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 96, ResultStage 97) -2016-06-13 19:44:18 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:18 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 96: List() -2016-06-13 19:44:18 INFO DAGScheduler:59 - Missing parents for ResultStage 97: List(ShuffleMapStage 96) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting ShuffleMapStage 96 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[113] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=269367, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=273263, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:49949 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 96 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[113] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Adding task set 96.0 with 2 tasks -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 96.0 (TID 50, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 96.0 (TID 51, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:18 INFO Executor:59 - Running task 0.0 in stage 96.0 (TID 50) -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:18 INFO Executor:59 - Running task 1.0 in stage 96.0 (TID 51) -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:18 INFO Executor:59 - Finished task 0.0 in stage 96.0 (TID 50). 2470 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 96.0 (TID 50) in 61 ms on localhost (1/2) -2016-06-13 19:44:18 INFO Executor:59 - Finished task 1.0 in stage 96.0 (TID 51). 2470 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 96.0 (TID 51) in 97 ms on localhost (2/2) -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 96.0, whose tasks have all completed, from pool -2016-06-13 19:44:18 INFO DAGScheduler:59 - ShuffleMapStage 96 (mapPartitions at GraphImpl.scala:235) finished in 0.099 s -2016-06-13 19:44:18 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:18 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:18 INFO DAGScheduler:59 - waiting: Set(ResultStage 97) -2016-06-13 19:44:18 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:18 INFO DAGScheduler:59 - Missing parents for ResultStage 97: List() -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting ResultStage 97 (MapPartitionsRDD[121] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(9000) called with curMem=275549, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 8.8 KB, free 730.3 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(3686) called with curMem=284549, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 3.6 KB, free 730.3 MB) -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:49949 (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 97 (MapPartitionsRDD[121] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Adding task set 97.0 with 2 tasks -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 97.0 (TID 52, localhost, PROCESS_LOCAL, 2412 bytes) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 97.0 (TID 53, localhost, PROCESS_LOCAL, 2412 bytes) -2016-06-13 19:44:18 INFO Executor:59 - Running task 1.0 in stage 97.0 (TID 53) -2016-06-13 19:44:18 INFO Executor:59 - Running task 0.0 in stage 97.0 (TID 52) -2016-06-13 19:44:18 INFO CacheManager:59 - Partition rdd_117_1 not found, computing it -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_104_1 locally -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:18 INFO CacheManager:59 - Partition rdd_117_0 not found, computing it -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_104_0 locally -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=288235, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block rdd_117_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added rdd_117_1 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=290683, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block rdd_117_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:44:18 INFO Executor:59 - Finished task 1.0 in stage 97.0 (TID 53). 2971 bytes result sent to driver -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added rdd_117_0 in memory on localhost:49949 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO Executor:59 - Finished task 0.0 in stage 97.0 (TID 52). 2971 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 97.0 (TID 53) in 34 ms on localhost (1/2) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 97.0 (TID 52) in 37 ms on localhost (2/2) -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 97.0, whose tasks have all completed, from pool -2016-06-13 19:44:18 INFO DAGScheduler:59 - ResultStage 97 (fold at LDAOptimizer.scala:205) finished in 0.039 s -2016-06-13 19:44:18 INFO DAGScheduler:59 - Job 9 finished: fold at LDAOptimizer.scala:205, took 0.316710 s -2016-06-13 19:44:18 INFO MapPartitionsRDD:59 - Removing RDD 91 from persistence list -2016-06-13 19:44:18 INFO ZippedPartitionsRDD2:59 - Removing RDD 98 from persistence list -2016-06-13 19:44:18 INFO BlockManager:59 - Removing RDD 91 -2016-06-13 19:44:18 INFO BlockManager:59 - Removing RDD 98 -2016-06-13 19:44:18 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:44:18 INFO DAGScheduler:59 - Registering RDD 122 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Registering RDD 126 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Got job 10 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:18 INFO DAGScheduler:59 - Final stage: ResultStage 118(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 117, ShuffleMapStage 99, ShuffleMapStage 103, ShuffleMapStage 115, ShuffleMapStage 107, ShuffleMapStage 101, ShuffleMapStage 111, ShuffleMapStage 105, ShuffleMapStage 109, ShuffleMapStage 98, ShuffleMapStage 113) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 117) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting ShuffleMapStage 116 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[122] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(8016) called with curMem=288235, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 7.8 KB, free 730.3 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(3371) called with curMem=296251, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.3 MB) -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:49949 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 116 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[122] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Adding task set 116.0 with 2 tasks -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 116.0 (TID 54, localhost, PROCESS_LOCAL, 2401 bytes) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 116.0 (TID 55, localhost, PROCESS_LOCAL, 2401 bytes) -2016-06-13 19:44:18 INFO Executor:59 - Running task 0.0 in stage 116.0 (TID 54) -2016-06-13 19:44:18 INFO Executor:59 - Running task 1.0 in stage 116.0 (TID 55) -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_117_0 locally -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_117_1 locally -2016-06-13 19:44:18 INFO Executor:59 - Finished task 0.0 in stage 116.0 (TID 54). 2254 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 116.0 (TID 54) in 36 ms on localhost (1/2) -2016-06-13 19:44:18 INFO Executor:59 - Finished task 1.0 in stage 116.0 (TID 55). 2254 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 116.0 (TID 55) in 40 ms on localhost (2/2) -2016-06-13 19:44:18 INFO DAGScheduler:59 - ShuffleMapStage 116 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.042 s -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 116.0, whose tasks have all completed, from pool -2016-06-13 19:44:18 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:18 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:18 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 117, ResultStage 118) -2016-06-13 19:44:18 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:18 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 117: List() -2016-06-13 19:44:18 INFO DAGScheduler:59 - Missing parents for ResultStage 118: List(ShuffleMapStage 117) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting ShuffleMapStage 117 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[126] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=299622, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=303518, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:49949 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 117 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[126] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Adding task set 117.0 with 2 tasks -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 117.0 (TID 56, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 117.0 (TID 57, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:18 INFO Executor:59 - Running task 0.0 in stage 117.0 (TID 56) -2016-06-13 19:44:18 INFO Executor:59 - Running task 1.0 in stage 117.0 (TID 57) -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:18 INFO Executor:59 - Finished task 1.0 in stage 117.0 (TID 57). 2470 bytes result sent to driver -2016-06-13 19:44:18 INFO Executor:59 - Finished task 0.0 in stage 117.0 (TID 56). 2470 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 117.0 (TID 57) in 30 ms on localhost (1/2) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 117.0 (TID 56) in 31 ms on localhost (2/2) -2016-06-13 19:44:18 INFO DAGScheduler:59 - ShuffleMapStage 117 (mapPartitions at GraphImpl.scala:235) finished in 0.031 s -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 117.0, whose tasks have all completed, from pool -2016-06-13 19:44:18 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:18 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:18 INFO DAGScheduler:59 - waiting: Set(ResultStage 118) -2016-06-13 19:44:18 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:18 INFO DAGScheduler:59 - Missing parents for ResultStage 118: List() -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting ResultStage 118 (MapPartitionsRDD[134] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(9432) called with curMem=305804, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 9.2 KB, free 730.3 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(3778) called with curMem=315236, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.7 KB, free 730.3 MB) -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:49949 (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 118 (MapPartitionsRDD[134] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Adding task set 118.0 with 2 tasks -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 118.0 (TID 58, localhost, PROCESS_LOCAL, 2453 bytes) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 118.0 (TID 59, localhost, PROCESS_LOCAL, 2453 bytes) -2016-06-13 19:44:18 INFO Executor:59 - Running task 0.0 in stage 118.0 (TID 58) -2016-06-13 19:44:18 INFO Executor:59 - Running task 1.0 in stage 118.0 (TID 59) -2016-06-13 19:44:18 INFO CacheManager:59 - Partition rdd_130_1 not found, computing it -2016-06-13 19:44:18 INFO CacheManager:59 - Partition rdd_130_0 not found, computing it -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_117_1 locally -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_117_0 locally -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=319014, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block rdd_130_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=321462, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block rdd_130_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added rdd_130_1 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO Executor:59 - Finished task 1.0 in stage 118.0 (TID 59). 2971 bytes result sent to driver -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added rdd_130_0 in memory on localhost:49949 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 118.0 (TID 59) in 13 ms on localhost (1/2) -2016-06-13 19:44:18 INFO Executor:59 - Finished task 0.0 in stage 118.0 (TID 58). 2971 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 118.0 (TID 58) in 15 ms on localhost (2/2) -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 118.0, whose tasks have all completed, from pool -2016-06-13 19:44:18 INFO DAGScheduler:59 - ResultStage 118 (fold at LDAOptimizer.scala:205) finished in 0.015 s -2016-06-13 19:44:18 INFO DAGScheduler:59 - Job 10 finished: fold at LDAOptimizer.scala:205, took 0.174324 s -2016-06-13 19:44:18 INFO MapPartitionsRDD:59 - Removing RDD 104 from persistence list -2016-06-13 19:44:18 INFO ZippedPartitionsRDD2:59 - Removing RDD 111 from persistence list -2016-06-13 19:44:18 INFO BlockManager:59 - Removing RDD 104 -2016-06-13 19:44:18 INFO BlockManager:59 - Removing RDD 111 -2016-06-13 19:44:18 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:44:18 INFO DAGScheduler:59 - Registering RDD 135 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Registering RDD 139 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Got job 11 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:18 INFO DAGScheduler:59 - Final stage: ResultStage 141(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 132, ShuffleMapStage 136, ShuffleMapStage 140, ShuffleMapStage 122, ShuffleMapStage 119, ShuffleMapStage 126, ShuffleMapStage 138, ShuffleMapStage 120, ShuffleMapStage 130, ShuffleMapStage 124, ShuffleMapStage 134, ShuffleMapStage 128) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 140) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting ShuffleMapStage 139 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[135] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(8440) called with curMem=319014, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 8.2 KB, free 730.3 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(3476) called with curMem=327454, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.3 MB) -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:49949 (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 139 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[135] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Adding task set 139.0 with 2 tasks -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 139.0 (TID 60, localhost, PROCESS_LOCAL, 2442 bytes) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 139.0 (TID 61, localhost, PROCESS_LOCAL, 2442 bytes) -2016-06-13 19:44:18 INFO Executor:59 - Running task 1.0 in stage 139.0 (TID 61) -2016-06-13 19:44:18 INFO Executor:59 - Running task 0.0 in stage 139.0 (TID 60) -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_130_1 locally -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_130_0 locally -2016-06-13 19:44:18 INFO Executor:59 - Finished task 0.0 in stage 139.0 (TID 60). 2254 bytes result sent to driver -2016-06-13 19:44:18 INFO Executor:59 - Finished task 1.0 in stage 139.0 (TID 61). 2254 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 139.0 (TID 60) in 24 ms on localhost (1/2) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 139.0 (TID 61) in 24 ms on localhost (2/2) -2016-06-13 19:44:18 INFO DAGScheduler:59 - ShuffleMapStage 139 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.025 s -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 139.0, whose tasks have all completed, from pool -2016-06-13 19:44:18 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:18 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:18 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 140, ResultStage 141) -2016-06-13 19:44:18 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:18 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 140: List() -2016-06-13 19:44:18 INFO DAGScheduler:59 - Missing parents for ResultStage 141: List(ShuffleMapStage 140) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting ShuffleMapStage 140 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[139] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=330930, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(2281) called with curMem=334826, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:49949 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 140 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[139] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Adding task set 140.0 with 2 tasks -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 140.0 (TID 62, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 140.0 (TID 63, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:18 INFO Executor:59 - Running task 1.0 in stage 140.0 (TID 63) -2016-06-13 19:44:18 INFO Executor:59 - Running task 0.0 in stage 140.0 (TID 62) -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:18 INFO Executor:59 - Finished task 0.0 in stage 140.0 (TID 62). 2470 bytes result sent to driver -2016-06-13 19:44:18 INFO Executor:59 - Finished task 1.0 in stage 140.0 (TID 63). 2470 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 140.0 (TID 62) in 25 ms on localhost (1/2) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 140.0 (TID 63) in 25 ms on localhost (2/2) -2016-06-13 19:44:18 INFO DAGScheduler:59 - ShuffleMapStage 140 (mapPartitions at GraphImpl.scala:235) finished in 0.026 s -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 140.0, whose tasks have all completed, from pool -2016-06-13 19:44:18 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:18 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:18 INFO DAGScheduler:59 - waiting: Set(ResultStage 141) -2016-06-13 19:44:18 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:18 INFO DAGScheduler:59 - Missing parents for ResultStage 141: List() -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting ResultStage 141 (MapPartitionsRDD[147] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(9856) called with curMem=337107, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 9.6 KB, free 730.3 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(3886) called with curMem=346963, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:49949 (size: 3.8 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 141 (MapPartitionsRDD[147] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Adding task set 141.0 with 2 tasks -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 141.0 (TID 64, localhost, PROCESS_LOCAL, 2494 bytes) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 141.0 (TID 65, localhost, PROCESS_LOCAL, 2494 bytes) -2016-06-13 19:44:18 INFO Executor:59 - Running task 1.0 in stage 141.0 (TID 65) -2016-06-13 19:44:18 INFO Executor:59 - Running task 0.0 in stage 141.0 (TID 64) -2016-06-13 19:44:18 INFO CacheManager:59 - Partition rdd_143_1 not found, computing it -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_130_1 locally -2016-06-13 19:44:18 INFO CacheManager:59 - Partition rdd_143_0 not found, computing it -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_130_0 locally -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=350849, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block rdd_143_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=353393, maxMem=766075207 -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added rdd_143_0 in memory on localhost:49949 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - Block rdd_143_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:44:18 INFO Executor:59 - Finished task 0.0 in stage 141.0 (TID 64). 2971 bytes result sent to driver -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added rdd_143_1 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO Executor:59 - Finished task 1.0 in stage 141.0 (TID 65). 2971 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 141.0 (TID 64) in 14 ms on localhost (1/2) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 141.0 (TID 65) in 14 ms on localhost (2/2) -2016-06-13 19:44:18 INFO DAGScheduler:59 - ResultStage 141 (fold at LDAOptimizer.scala:205) finished in 0.016 s -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 141.0, whose tasks have all completed, from pool -2016-06-13 19:44:18 INFO DAGScheduler:59 - Job 11 finished: fold at LDAOptimizer.scala:205, took 0.127881 s -2016-06-13 19:44:18 INFO MapPartitionsRDD:59 - Removing RDD 117 from persistence list -2016-06-13 19:44:18 INFO BlockManager:59 - Removing RDD 117 -2016-06-13 19:44:18 INFO ZippedPartitionsRDD2:59 - Removing RDD 124 from persistence list -2016-06-13 19:44:18 INFO BlockManager:59 - Removing RDD 124 -2016-06-13 19:44:18 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:44:18 INFO DAGScheduler:59 - Registering RDD 148 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Registering RDD 152 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Got job 12 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:18 INFO DAGScheduler:59 - Final stage: ResultStage 166(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 161, ShuffleMapStage 143, ShuffleMapStage 153, ShuffleMapStage 165, ShuffleMapStage 147, ShuffleMapStage 157, ShuffleMapStage 151, ShuffleMapStage 155, ShuffleMapStage 159, ShuffleMapStage 163, ShuffleMapStage 145, ShuffleMapStage 142, ShuffleMapStage 149) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 165) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting ShuffleMapStage 164 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[148] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(8872) called with curMem=350849, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 8.7 KB, free 730.2 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(3568) called with curMem=359721, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.2 MB) -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:49949 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 164 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[148] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Adding task set 164.0 with 2 tasks -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 164.0 (TID 66, localhost, PROCESS_LOCAL, 2483 bytes) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 164.0 (TID 67, localhost, PROCESS_LOCAL, 2483 bytes) -2016-06-13 19:44:18 INFO Executor:59 - Running task 1.0 in stage 164.0 (TID 67) -2016-06-13 19:44:18 INFO Executor:59 - Running task 0.0 in stage 164.0 (TID 66) -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_143_1 locally -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_143_0 locally -2016-06-13 19:44:18 INFO Executor:59 - Finished task 0.0 in stage 164.0 (TID 66). 2254 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 164.0 (TID 66) in 23 ms on localhost (1/2) -2016-06-13 19:44:18 INFO Executor:59 - Finished task 1.0 in stage 164.0 (TID 67). 2254 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 164.0 (TID 67) in 26 ms on localhost (2/2) -2016-06-13 19:44:18 INFO DAGScheduler:59 - ShuffleMapStage 164 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.027 s -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 164.0, whose tasks have all completed, from pool -2016-06-13 19:44:18 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:18 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:18 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 165, ResultStage 166) -2016-06-13 19:44:18 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:18 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 165: List() -2016-06-13 19:44:18 INFO DAGScheduler:59 - Missing parents for ResultStage 166: List(ShuffleMapStage 165) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting ShuffleMapStage 165 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[152] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=363289, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=367185, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:49949 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 165 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[152] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Adding task set 165.0 with 2 tasks -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 165.0 (TID 68, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 165.0 (TID 69, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:18 INFO Executor:59 - Running task 1.0 in stage 165.0 (TID 69) -2016-06-13 19:44:18 INFO Executor:59 - Running task 0.0 in stage 165.0 (TID 68) -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:18 INFO Executor:59 - Finished task 1.0 in stage 165.0 (TID 69). 2470 bytes result sent to driver -2016-06-13 19:44:18 INFO Executor:59 - Finished task 0.0 in stage 165.0 (TID 68). 2470 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 165.0 (TID 69) in 23 ms on localhost (1/2) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 165.0 (TID 68) in 25 ms on localhost (2/2) -2016-06-13 19:44:18 INFO DAGScheduler:59 - ShuffleMapStage 165 (mapPartitions at GraphImpl.scala:235) finished in 0.025 s -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 165.0, whose tasks have all completed, from pool -2016-06-13 19:44:18 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:18 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:18 INFO DAGScheduler:59 - waiting: Set(ResultStage 166) -2016-06-13 19:44:18 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:18 INFO DAGScheduler:59 - Missing parents for ResultStage 166: List() -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting ResultStage 166 (MapPartitionsRDD[160] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(10288) called with curMem=369471, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 10.0 KB, free 730.2 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(4004) called with curMem=379759, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 3.9 KB, free 730.2 MB) -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:49949 (size: 3.9 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 166 (MapPartitionsRDD[160] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Adding task set 166.0 with 2 tasks -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 166.0 (TID 70, localhost, PROCESS_LOCAL, 2535 bytes) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 166.0 (TID 71, localhost, PROCESS_LOCAL, 2535 bytes) -2016-06-13 19:44:18 INFO Executor:59 - Running task 0.0 in stage 166.0 (TID 70) -2016-06-13 19:44:18 INFO Executor:59 - Running task 1.0 in stage 166.0 (TID 71) -2016-06-13 19:44:18 INFO CacheManager:59 - Partition rdd_156_1 not found, computing it -2016-06-13 19:44:18 INFO CacheManager:59 - Partition rdd_156_0 not found, computing it -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_143_1 locally -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_143_0 locally -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=383763, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block rdd_156_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=386307, maxMem=766075207 -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added rdd_156_0 in memory on localhost:49949 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - Block rdd_156_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added rdd_156_1 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO Executor:59 - Finished task 0.0 in stage 166.0 (TID 70). 2971 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 166.0 (TID 70) in 14 ms on localhost (1/2) -2016-06-13 19:44:18 INFO Executor:59 - Finished task 1.0 in stage 166.0 (TID 71). 2971 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 166.0 (TID 71) in 17 ms on localhost (2/2) -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 166.0, whose tasks have all completed, from pool -2016-06-13 19:44:18 INFO DAGScheduler:59 - ResultStage 166 (fold at LDAOptimizer.scala:205) finished in 0.018 s -2016-06-13 19:44:18 INFO DAGScheduler:59 - Job 12 finished: fold at LDAOptimizer.scala:205, took 0.136170 s -2016-06-13 19:44:18 INFO MapPartitionsRDD:59 - Removing RDD 130 from persistence list -2016-06-13 19:44:18 INFO BlockManager:59 - Removing RDD 130 -2016-06-13 19:44:18 INFO ZippedPartitionsRDD2:59 - Removing RDD 137 from persistence list -2016-06-13 19:44:18 INFO BlockManager:59 - Removing RDD 137 -2016-06-13 19:44:18 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:44:18 INFO DAGScheduler:59 - Registering RDD 161 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Registering RDD 165 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Got job 13 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:18 INFO DAGScheduler:59 - Final stage: ResultStage 193(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 168, ShuffleMapStage 190, ShuffleMapStage 184, ShuffleMapStage 176, ShuffleMapStage 170, ShuffleMapStage 192, ShuffleMapStage 178, ShuffleMapStage 186, ShuffleMapStage 180, ShuffleMapStage 172, ShuffleMapStage 188, ShuffleMapStage 174, ShuffleMapStage 167, ShuffleMapStage 182) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 192) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting ShuffleMapStage 191 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[161] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(9296) called with curMem=383763, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 9.1 KB, free 730.2 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(3674) called with curMem=393059, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 3.6 KB, free 730.2 MB) -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:49949 (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 191 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[161] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Adding task set 191.0 with 2 tasks -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 191.0 (TID 72, localhost, PROCESS_LOCAL, 2524 bytes) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 191.0 (TID 73, localhost, PROCESS_LOCAL, 2524 bytes) -2016-06-13 19:44:18 INFO Executor:59 - Running task 1.0 in stage 191.0 (TID 73) -2016-06-13 19:44:18 INFO Executor:59 - Running task 0.0 in stage 191.0 (TID 72) -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_156_1 locally -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_156_0 locally -2016-06-13 19:44:18 INFO Executor:59 - Finished task 1.0 in stage 191.0 (TID 73). 2254 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 191.0 (TID 73) in 29 ms on localhost (1/2) -2016-06-13 19:44:18 INFO Executor:59 - Finished task 0.0 in stage 191.0 (TID 72). 2254 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 191.0 (TID 72) in 33 ms on localhost (2/2) -2016-06-13 19:44:18 INFO DAGScheduler:59 - ShuffleMapStage 191 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.034 s -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 191.0, whose tasks have all completed, from pool -2016-06-13 19:44:18 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:18 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:18 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 192, ResultStage 193) -2016-06-13 19:44:18 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:18 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 192: List() -2016-06-13 19:44:18 INFO DAGScheduler:59 - Missing parents for ResultStage 193: List(ShuffleMapStage 192) -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting ShuffleMapStage 192 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[165] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=396733, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=400629, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:49949 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 192 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[165] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Adding task set 192.0 with 2 tasks -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 192.0 (TID 74, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 192.0 (TID 75, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:18 INFO Executor:59 - Running task 0.0 in stage 192.0 (TID 74) -2016-06-13 19:44:18 INFO Executor:59 - Running task 1.0 in stage 192.0 (TID 75) -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:18 INFO Executor:59 - Finished task 0.0 in stage 192.0 (TID 74). 2470 bytes result sent to driver -2016-06-13 19:44:18 INFO Executor:59 - Finished task 1.0 in stage 192.0 (TID 75). 2470 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 192.0 (TID 74) in 26 ms on localhost (1/2) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 192.0 (TID 75) in 26 ms on localhost (2/2) -2016-06-13 19:44:18 INFO DAGScheduler:59 - ShuffleMapStage 192 (mapPartitions at GraphImpl.scala:235) finished in 0.027 s -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 192.0, whose tasks have all completed, from pool -2016-06-13 19:44:18 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:18 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:18 INFO DAGScheduler:59 - waiting: Set(ResultStage 193) -2016-06-13 19:44:18 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:18 INFO DAGScheduler:59 - Missing parents for ResultStage 193: List() -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting ResultStage 193 (MapPartitionsRDD[173] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(10712) called with curMem=402915, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 10.5 KB, free 730.2 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=413627, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.2 MB) -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:49949 (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 193 (MapPartitionsRDD[173] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Adding task set 193.0 with 2 tasks -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 193.0 (TID 76, localhost, PROCESS_LOCAL, 2576 bytes) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 193.0 (TID 77, localhost, PROCESS_LOCAL, 2576 bytes) -2016-06-13 19:44:18 INFO Executor:59 - Running task 1.0 in stage 193.0 (TID 77) -2016-06-13 19:44:18 INFO Executor:59 - Running task 0.0 in stage 193.0 (TID 76) -2016-06-13 19:44:18 INFO CacheManager:59 - Partition rdd_169_1 not found, computing it -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_156_1 locally -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:18 INFO CacheManager:59 - Partition rdd_169_0 not found, computing it -2016-06-13 19:44:18 INFO BlockManager:59 - Found block rdd_156_0 locally -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:18 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=417723, maxMem=766075207 -2016-06-13 19:44:18 INFO MemoryStore:59 - Block rdd_169_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=420171, maxMem=766075207 -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added rdd_169_1 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO MemoryStore:59 - Block rdd_169_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:44:18 INFO BlockManagerInfo:59 - Added rdd_169_0 in memory on localhost:49949 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:44:18 INFO Executor:59 - Finished task 1.0 in stage 193.0 (TID 77). 2971 bytes result sent to driver -2016-06-13 19:44:18 INFO Executor:59 - Finished task 0.0 in stage 193.0 (TID 76). 2971 bytes result sent to driver -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 193.0 (TID 77) in 12 ms on localhost (1/2) -2016-06-13 19:44:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 193.0 (TID 76) in 14 ms on localhost (2/2) -2016-06-13 19:44:18 INFO DAGScheduler:59 - ResultStage 193 (fold at LDAOptimizer.scala:205) finished in 0.014 s -2016-06-13 19:44:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 193.0, whose tasks have all completed, from pool -2016-06-13 19:44:18 INFO DAGScheduler:59 - Job 13 finished: fold at LDAOptimizer.scala:205, took 0.131012 s -2016-06-13 19:44:18 INFO MapPartitionsRDD:59 - Removing RDD 143 from persistence list -2016-06-13 19:44:18 INFO ZippedPartitionsRDD2:59 - Removing RDD 150 from persistence list -2016-06-13 19:44:18 INFO BlockManager:59 - Removing RDD 143 -2016-06-13 19:44:18 INFO BlockManager:59 - Removing RDD 150 -2016-06-13 19:44:18 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:44:18 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:44:19 INFO DAGScheduler:59 - Registering RDD 174 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Registering RDD 178 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Got job 14 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:19 INFO DAGScheduler:59 - Final stage: ResultStage 222(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 197, ShuffleMapStage 219, ShuffleMapStage 205, ShuffleMapStage 221, ShuffleMapStage 213, ShuffleMapStage 207, ShuffleMapStage 199, ShuffleMapStage 215, ShuffleMapStage 201, ShuffleMapStage 194, ShuffleMapStage 209, ShuffleMapStage 203, ShuffleMapStage 195, ShuffleMapStage 217, ShuffleMapStage 211) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 221) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting ShuffleMapStage 220 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[174] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(9728) called with curMem=417723, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 9.5 KB, free 730.2 MB) -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(3782) called with curMem=427451, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.7 KB, free 730.2 MB) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:49949 (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 220 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[174] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Adding task set 220.0 with 2 tasks -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 220.0 (TID 78, localhost, PROCESS_LOCAL, 2565 bytes) -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 220.0 (TID 79, localhost, PROCESS_LOCAL, 2565 bytes) -2016-06-13 19:44:19 INFO Executor:59 - Running task 1.0 in stage 220.0 (TID 79) -2016-06-13 19:44:19 INFO Executor:59 - Running task 0.0 in stage 220.0 (TID 78) -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_169_0 locally -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_169_1 locally -2016-06-13 19:44:19 INFO Executor:59 - Finished task 1.0 in stage 220.0 (TID 79). 2254 bytes result sent to driver -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 220.0 (TID 79) in 23 ms on localhost (1/2) -2016-06-13 19:44:19 INFO Executor:59 - Finished task 0.0 in stage 220.0 (TID 78). 2254 bytes result sent to driver -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 220.0 (TID 78) in 27 ms on localhost (2/2) -2016-06-13 19:44:19 INFO DAGScheduler:59 - ShuffleMapStage 220 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.027 s -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 220.0, whose tasks have all completed, from pool -2016-06-13 19:44:19 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:19 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:19 INFO DAGScheduler:59 - waiting: Set(ResultStage 222, ShuffleMapStage 221) -2016-06-13 19:44:19 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:19 INFO DAGScheduler:59 - Missing parents for ResultStage 222: List(ShuffleMapStage 221) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 221: List() -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting ShuffleMapStage 221 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[178] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=431233, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(2287) called with curMem=435129, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:49949 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 221 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[178] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Adding task set 221.0 with 2 tasks -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 221.0 (TID 80, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 221.0 (TID 81, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:19 INFO Executor:59 - Running task 1.0 in stage 221.0 (TID 81) -2016-06-13 19:44:19 INFO Executor:59 - Running task 0.0 in stage 221.0 (TID 80) -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:19 INFO Executor:59 - Finished task 0.0 in stage 221.0 (TID 80). 2470 bytes result sent to driver -2016-06-13 19:44:19 INFO Executor:59 - Finished task 1.0 in stage 221.0 (TID 81). 2470 bytes result sent to driver -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 221.0 (TID 80) in 33 ms on localhost (1/2) -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 221.0 (TID 81) in 33 ms on localhost (2/2) -2016-06-13 19:44:19 INFO DAGScheduler:59 - ShuffleMapStage 221 (mapPartitions at GraphImpl.scala:235) finished in 0.035 s -2016-06-13 19:44:19 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:19 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:19 INFO DAGScheduler:59 - waiting: Set(ResultStage 222) -2016-06-13 19:44:19 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 221.0, whose tasks have all completed, from pool -2016-06-13 19:44:19 INFO DAGScheduler:59 - Missing parents for ResultStage 222: List() -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting ResultStage 222 (MapPartitionsRDD[186] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(11144) called with curMem=437416, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 10.9 KB, free 730.2 MB) -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(4182) called with curMem=448560, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 4.1 KB, free 730.2 MB) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:49949 (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 222 (MapPartitionsRDD[186] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Adding task set 222.0 with 2 tasks -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 222.0 (TID 82, localhost, PROCESS_LOCAL, 2617 bytes) -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 222.0 (TID 83, localhost, PROCESS_LOCAL, 2617 bytes) -2016-06-13 19:44:19 INFO Executor:59 - Running task 0.0 in stage 222.0 (TID 82) -2016-06-13 19:44:19 INFO Executor:59 - Running task 1.0 in stage 222.0 (TID 83) -2016-06-13 19:44:19 INFO CacheManager:59 - Partition rdd_182_1 not found, computing it -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_169_1 locally -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:19 INFO CacheManager:59 - Partition rdd_182_0 not found, computing it -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_169_0 locally -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=452742, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block rdd_182_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=455190, maxMem=766075207 -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added rdd_182_1 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO MemoryStore:59 - Block rdd_182_0 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-06-13 19:44:19 INFO Executor:59 - Finished task 1.0 in stage 222.0 (TID 83). 2971 bytes result sent to driver -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added rdd_182_0 in memory on localhost:49949 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO Executor:59 - Finished task 0.0 in stage 222.0 (TID 82). 2971 bytes result sent to driver -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 222.0 (TID 83) in 14 ms on localhost (1/2) -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 222.0 (TID 82) in 16 ms on localhost (2/2) -2016-06-13 19:44:19 INFO DAGScheduler:59 - ResultStage 222 (fold at LDAOptimizer.scala:205) finished in 0.017 s -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 222.0, whose tasks have all completed, from pool -2016-06-13 19:44:19 INFO DAGScheduler:59 - Job 14 finished: fold at LDAOptimizer.scala:205, took 0.148770 s -2016-06-13 19:44:19 INFO MapPartitionsRDD:59 - Removing RDD 156 from persistence list -2016-06-13 19:44:19 INFO ZippedPartitionsRDD2:59 - Removing RDD 163 from persistence list -2016-06-13 19:44:19 INFO BlockManager:59 - Removing RDD 156 -2016-06-13 19:44:19 INFO BlockManager:59 - Removing RDD 163 -2016-06-13 19:44:19 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:44:19 INFO DAGScheduler:59 - Registering RDD 187 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Registering RDD 191 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Got job 15 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:19 INFO DAGScheduler:59 - Final stage: ResultStage 253(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 248, ShuffleMapStage 234, ShuffleMapStage 242, ShuffleMapStage 228, ShuffleMapStage 250, ShuffleMapStage 244, ShuffleMapStage 236, ShuffleMapStage 230, ShuffleMapStage 252, ShuffleMapStage 223, ShuffleMapStage 238, ShuffleMapStage 224, ShuffleMapStage 246, ShuffleMapStage 240, ShuffleMapStage 232, ShuffleMapStage 226) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 252) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting ShuffleMapStage 251 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[187] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(10152) called with curMem=452742, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 9.9 KB, free 730.1 MB) -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(3864) called with curMem=462894, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 3.8 KB, free 730.1 MB) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:49949 (size: 3.8 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 251 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[187] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Adding task set 251.0 with 2 tasks -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 251.0 (TID 84, localhost, PROCESS_LOCAL, 2606 bytes) -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 251.0 (TID 85, localhost, PROCESS_LOCAL, 2606 bytes) -2016-06-13 19:44:19 INFO Executor:59 - Running task 1.0 in stage 251.0 (TID 85) -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_182_1 locally -2016-06-13 19:44:19 INFO Executor:59 - Running task 0.0 in stage 251.0 (TID 84) -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_182_0 locally -2016-06-13 19:44:19 INFO Executor:59 - Finished task 1.0 in stage 251.0 (TID 85). 2254 bytes result sent to driver -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 251.0 (TID 85) in 26 ms on localhost (1/2) -2016-06-13 19:44:19 INFO Executor:59 - Finished task 0.0 in stage 251.0 (TID 84). 2254 bytes result sent to driver -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 251.0 (TID 84) in 41 ms on localhost (2/2) -2016-06-13 19:44:19 INFO DAGScheduler:59 - ShuffleMapStage 251 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.042 s -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 251.0, whose tasks have all completed, from pool -2016-06-13 19:44:19 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:19 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:19 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 252, ResultStage 253) -2016-06-13 19:44:19 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:19 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 252: List() -2016-06-13 19:44:19 INFO DAGScheduler:59 - Missing parents for ResultStage 253: List(ShuffleMapStage 252) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting ShuffleMapStage 252 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[191] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=466758, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 3.8 KB, free 730.1 MB) -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(2284) called with curMem=470654, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:49949 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 252 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[191] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Adding task set 252.0 with 2 tasks -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 252.0 (TID 86, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 252.0 (TID 87, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:19 INFO Executor:59 - Running task 1.0 in stage 252.0 (TID 87) -2016-06-13 19:44:19 INFO Executor:59 - Running task 0.0 in stage 252.0 (TID 86) -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:19 INFO Executor:59 - Finished task 1.0 in stage 252.0 (TID 87). 2470 bytes result sent to driver -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 252.0 (TID 87) in 67 ms on localhost (1/2) -2016-06-13 19:44:19 INFO Executor:59 - Finished task 0.0 in stage 252.0 (TID 86). 2470 bytes result sent to driver -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 252.0 (TID 86) in 73 ms on localhost (2/2) -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 252.0, whose tasks have all completed, from pool -2016-06-13 19:44:19 INFO DAGScheduler:59 - ShuffleMapStage 252 (mapPartitions at GraphImpl.scala:235) finished in 0.077 s -2016-06-13 19:44:19 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:19 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:19 INFO DAGScheduler:59 - waiting: Set(ResultStage 253) -2016-06-13 19:44:19 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:19 INFO DAGScheduler:59 - Missing parents for ResultStage 253: List() -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting ResultStage 253 (MapPartitionsRDD[199] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(11568) called with curMem=472938, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 11.3 KB, free 730.1 MB) -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(4283) called with curMem=484506, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.1 MB) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:49949 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 253 (MapPartitionsRDD[199] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Adding task set 253.0 with 2 tasks -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 253.0 (TID 88, localhost, PROCESS_LOCAL, 2658 bytes) -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 253.0 (TID 89, localhost, PROCESS_LOCAL, 2658 bytes) -2016-06-13 19:44:19 INFO Executor:59 - Running task 1.0 in stage 253.0 (TID 89) -2016-06-13 19:44:19 INFO Executor:59 - Running task 0.0 in stage 253.0 (TID 88) -2016-06-13 19:44:19 INFO CacheManager:59 - Partition rdd_195_1 not found, computing it -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_182_1 locally -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:19 INFO CacheManager:59 - Partition rdd_195_0 not found, computing it -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_182_0 locally -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=488789, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block rdd_195_1 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added rdd_195_1 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=491237, maxMem=766075207 -2016-06-13 19:44:19 INFO Executor:59 - Finished task 1.0 in stage 253.0 (TID 89). 2971 bytes result sent to driver -2016-06-13 19:44:19 INFO MemoryStore:59 - Block rdd_195_0 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added rdd_195_0 in memory on localhost:49949 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 253.0 (TID 89) in 14 ms on localhost (1/2) -2016-06-13 19:44:19 INFO Executor:59 - Finished task 0.0 in stage 253.0 (TID 88). 2971 bytes result sent to driver -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 253.0 (TID 88) in 16 ms on localhost (2/2) -2016-06-13 19:44:19 INFO DAGScheduler:59 - ResultStage 253 (fold at LDAOptimizer.scala:205) finished in 0.017 s -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 253.0, whose tasks have all completed, from pool -2016-06-13 19:44:19 INFO DAGScheduler:59 - Job 15 finished: fold at LDAOptimizer.scala:205, took 0.198212 s -2016-06-13 19:44:19 INFO MapPartitionsRDD:59 - Removing RDD 169 from persistence list -2016-06-13 19:44:19 INFO ZippedPartitionsRDD2:59 - Removing RDD 176 from persistence list -2016-06-13 19:44:19 INFO BlockManager:59 - Removing RDD 169 -2016-06-13 19:44:19 INFO BlockManager:59 - Removing RDD 176 -2016-06-13 19:44:19 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:49949 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:49949 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:49949 in memory (size: 3.8 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:49949 in memory (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-06-13 19:44:19 INFO DAGScheduler:59 - Registering RDD 200 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Registering RDD 204 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Got job 16 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:19 INFO DAGScheduler:59 - Final stage: ResultStage 286(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 269, ShuffleMapStage 263, ShuffleMapStage 255, ShuffleMapStage 285, ShuffleMapStage 277, ShuffleMapStage 271, ShuffleMapStage 257, ShuffleMapStage 279, ShuffleMapStage 265, ShuffleMapStage 281, ShuffleMapStage 273, ShuffleMapStage 267, ShuffleMapStage 259, ShuffleMapStage 275, ShuffleMapStage 261, ShuffleMapStage 283, ShuffleMapStage 254) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:49949 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 285) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting ShuffleMapStage 284 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[200] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(10584) called with curMem=431233, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 10.3 KB, free 730.2 MB) -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(3951) called with curMem=441817, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 3.9 KB, free 730.2 MB) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:49949 (size: 3.9 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 284 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[200] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Adding task set 284.0 with 2 tasks -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 284.0 (TID 90, localhost, PROCESS_LOCAL, 2647 bytes) -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 284.0 (TID 91, localhost, PROCESS_LOCAL, 2647 bytes) -2016-06-13 19:44:19 INFO Executor:59 - Running task 1.0 in stage 284.0 (TID 91) -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_195_1 locally -2016-06-13 19:44:19 INFO Executor:59 - Running task 0.0 in stage 284.0 (TID 90) -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_195_0 locally -2016-06-13 19:44:19 INFO Executor:59 - Finished task 1.0 in stage 284.0 (TID 91). 2254 bytes result sent to driver -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:49949 in memory (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 284.0 (TID 91) in 30 ms on localhost (1/2) -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:49949 in memory (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:49949 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO Executor:59 - Finished task 0.0 in stage 284.0 (TID 90). 2254 bytes result sent to driver -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 284.0 (TID 90) in 52 ms on localhost (2/2) -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 284.0, whose tasks have all completed, from pool -2016-06-13 19:44:19 INFO DAGScheduler:59 - ShuffleMapStage 284 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.055 s -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-06-13 19:44:19 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:19 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:19 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 285, ResultStage 286) -2016-06-13 19:44:19 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:19 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 285: List() -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:49949 in memory (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Missing parents for ResultStage 286: List(ShuffleMapStage 285) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting ShuffleMapStage 285 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[204] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=398298, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=398190, maxMem=766075207 -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:49949 in memory (size: 3.9 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:49949 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-06-13 19:44:19 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 285 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[204] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Adding task set 285.0 with 2 tasks -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 285.0 (TID 92, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:49949 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 285.0 (TID 93, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:19 INFO Executor:59 - Running task 0.0 in stage 285.0 (TID 92) -2016-06-13 19:44:19 INFO Executor:59 - Running task 1.0 in stage 285.0 (TID 93) -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:49949 in memory (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:49949 in memory (size: 3.8 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:49949 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-06-13 19:44:19 INFO Executor:59 - Finished task 1.0 in stage 285.0 (TID 93). 2470 bytes result sent to driver -2016-06-13 19:44:19 INFO Executor:59 - Finished task 0.0 in stage 285.0 (TID 92). 2470 bytes result sent to driver -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 285.0 (TID 93) in 31 ms on localhost (1/2) -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 285.0 (TID 92) in 33 ms on localhost (2/2) -2016-06-13 19:44:19 INFO DAGScheduler:59 - ShuffleMapStage 285 (mapPartitions at GraphImpl.scala:235) finished in 0.034 s -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 285.0, whose tasks have all completed, from pool -2016-06-13 19:44:19 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:19 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:19 INFO DAGScheduler:59 - waiting: Set(ResultStage 286) -2016-06-13 19:44:19 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:49949 in memory (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-06-13 19:44:19 INFO DAGScheduler:59 - Missing parents for ResultStage 286: List() -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting ResultStage 286 (MapPartitionsRDD[212] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:49949 in memory (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(12000) called with curMem=326521, maxMem=766075207 -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 11.7 KB, free 730.3 MB) -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(4400) called with curMem=336235, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:49949 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:49949 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 286 (MapPartitionsRDD[212] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Adding task set 286.0 with 2 tasks -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 286.0 (TID 94, localhost, PROCESS_LOCAL, 2699 bytes) -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 286.0 (TID 95, localhost, PROCESS_LOCAL, 2699 bytes) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:49949 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO Executor:59 - Running task 1.0 in stage 286.0 (TID 95) -2016-06-13 19:44:19 INFO Executor:59 - Running task 0.0 in stage 286.0 (TID 94) -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-06-13 19:44:19 INFO CacheManager:59 - Partition rdd_208_1 not found, computing it -2016-06-13 19:44:19 INFO CacheManager:59 - Partition rdd_208_0 not found, computing it -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_195_0 locally -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_195_1 locally -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:49949 in memory (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=306484, maxMem=766075207 -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:49949 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO MemoryStore:59 - Block rdd_208_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=308932, maxMem=766075207 -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block rdd_208_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added rdd_208_1 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO Executor:59 - Finished task 1.0 in stage 286.0 (TID 95). 2971 bytes result sent to driver -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added rdd_208_0 in memory on localhost:49949 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO Executor:59 - Finished task 0.0 in stage 286.0 (TID 94). 2971 bytes result sent to driver -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 286.0 (TID 95) in 16 ms on localhost (1/2) -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 286.0 (TID 94) in 17 ms on localhost (2/2) -2016-06-13 19:44:19 INFO DAGScheduler:59 - ResultStage 286 (fold at LDAOptimizer.scala:205) finished in 0.017 s -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 286.0, whose tasks have all completed, from pool -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:49949 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-06-13 19:44:19 INFO DAGScheduler:59 - Job 16 finished: fold at LDAOptimizer.scala:205, took 0.217144 s -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:49949 in memory (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:49949 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:49949 in memory (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:49949 in memory (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO MapPartitionsRDD:59 - Removing RDD 182 from persistence list -2016-06-13 19:44:19 INFO ZippedPartitionsRDD2:59 - Removing RDD 189 from persistence list -2016-06-13 19:44:19 INFO BlockManager:59 - Removing RDD 182 -2016-06-13 19:44:19 INFO BlockManager:59 - Removing RDD 189 -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 19:44:19 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:49949 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:49949 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:44:19 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:44:19 INFO DAGScheduler:59 - Registering RDD 213 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Registering RDD 217 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Got job 17 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:19 INFO DAGScheduler:59 - Final stage: ResultStage 321(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 320, ShuffleMapStage 306, ShuffleMapStage 300, ShuffleMapStage 292, ShuffleMapStage 314, ShuffleMapStage 308, ShuffleMapStage 294, ShuffleMapStage 316, ShuffleMapStage 287, ShuffleMapStage 302, ShuffleMapStage 288, ShuffleMapStage 318, ShuffleMapStage 310, ShuffleMapStage 304, ShuffleMapStage 296, ShuffleMapStage 290, ShuffleMapStage 312, ShuffleMapStage 298) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 320) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting ShuffleMapStage 319 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[213] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=239321, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 10.8 KB, free 730.3 MB) -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(4062) called with curMem=250329, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:49949 (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 319 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[213] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Adding task set 319.0 with 2 tasks -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 319.0 (TID 96, localhost, PROCESS_LOCAL, 2688 bytes) -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 319.0 (TID 97, localhost, PROCESS_LOCAL, 2688 bytes) -2016-06-13 19:44:19 INFO Executor:59 - Running task 1.0 in stage 319.0 (TID 97) -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_208_1 locally -2016-06-13 19:44:19 INFO Executor:59 - Running task 0.0 in stage 319.0 (TID 96) -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_208_0 locally -2016-06-13 19:44:19 INFO Executor:59 - Finished task 1.0 in stage 319.0 (TID 97). 2254 bytes result sent to driver -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 319.0 (TID 97) in 24 ms on localhost (1/2) -2016-06-13 19:44:19 INFO Executor:59 - Finished task 0.0 in stage 319.0 (TID 96). 2254 bytes result sent to driver -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 319.0 (TID 96) in 29 ms on localhost (2/2) -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 319.0, whose tasks have all completed, from pool -2016-06-13 19:44:19 INFO DAGScheduler:59 - ShuffleMapStage 319 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.029 s -2016-06-13 19:44:19 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:19 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:19 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 320, ResultStage 321) -2016-06-13 19:44:19 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:19 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 320: List() -2016-06-13 19:44:19 INFO DAGScheduler:59 - Missing parents for ResultStage 321: List(ShuffleMapStage 320) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting ShuffleMapStage 320 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[217] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=254391, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=258287, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:49949 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 320 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[217] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Adding task set 320.0 with 2 tasks -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 320.0 (TID 98, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 320.0 (TID 99, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:19 INFO Executor:59 - Running task 0.0 in stage 320.0 (TID 98) -2016-06-13 19:44:19 INFO Executor:59 - Running task 1.0 in stage 320.0 (TID 99) -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:19 INFO Executor:59 - Finished task 0.0 in stage 320.0 (TID 98). 2470 bytes result sent to driver -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 320.0 (TID 98) in 35 ms on localhost (1/2) -2016-06-13 19:44:19 INFO Executor:59 - Finished task 1.0 in stage 320.0 (TID 99). 2470 bytes result sent to driver -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 320.0 (TID 99) in 39 ms on localhost (2/2) -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 320.0, whose tasks have all completed, from pool -2016-06-13 19:44:19 INFO DAGScheduler:59 - ShuffleMapStage 320 (mapPartitions at GraphImpl.scala:235) finished in 0.040 s -2016-06-13 19:44:19 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:19 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:19 INFO DAGScheduler:59 - waiting: Set(ResultStage 321) -2016-06-13 19:44:19 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:19 INFO DAGScheduler:59 - Missing parents for ResultStage 321: List() -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting ResultStage 321 (MapPartitionsRDD[225] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(12424) called with curMem=260573, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 12.1 KB, free 730.3 MB) -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(4491) called with curMem=272997, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:49949 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 321 (MapPartitionsRDD[225] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Adding task set 321.0 with 2 tasks -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 321.0 (TID 100, localhost, PROCESS_LOCAL, 2740 bytes) -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 321.0 (TID 101, localhost, PROCESS_LOCAL, 2740 bytes) -2016-06-13 19:44:19 INFO Executor:59 - Running task 1.0 in stage 321.0 (TID 101) -2016-06-13 19:44:19 INFO Executor:59 - Running task 0.0 in stage 321.0 (TID 100) -2016-06-13 19:44:19 INFO CacheManager:59 - Partition rdd_221_1 not found, computing it -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_208_1 locally -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=277488, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block rdd_221_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:44:19 INFO CacheManager:59 - Partition rdd_221_0 not found, computing it -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_208_0 locally -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:19 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added rdd_221_1 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO Executor:59 - Finished task 1.0 in stage 321.0 (TID 101). 2971 bytes result sent to driver -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=279936, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block rdd_221_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added rdd_221_0 in memory on localhost:49949 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO Executor:59 - Finished task 0.0 in stage 321.0 (TID 100). 2971 bytes result sent to driver -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 321.0 (TID 101) in 28 ms on localhost (1/2) -2016-06-13 19:44:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 321.0 (TID 100) in 31 ms on localhost (2/2) -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 321.0, whose tasks have all completed, from pool -2016-06-13 19:44:19 INFO DAGScheduler:59 - ResultStage 321 (fold at LDAOptimizer.scala:205) finished in 0.033 s -2016-06-13 19:44:19 INFO DAGScheduler:59 - Job 17 finished: fold at LDAOptimizer.scala:205, took 0.225510 s -2016-06-13 19:44:19 INFO MapPartitionsRDD:59 - Removing RDD 195 from persistence list -2016-06-13 19:44:19 INFO ZippedPartitionsRDD2:59 - Removing RDD 202 from persistence list -2016-06-13 19:44:19 INFO BlockManager:59 - Removing RDD 195 -2016-06-13 19:44:19 INFO BlockManager:59 - Removing RDD 202 -2016-06-13 19:44:19 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:44:19 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:44:19 INFO DAGScheduler:59 - Registering RDD 226 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Registering RDD 230 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Got job 18 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:19 INFO DAGScheduler:59 - Final stage: ResultStage 358(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 335, ShuffleMapStage 357, ShuffleMapStage 343, ShuffleMapStage 322, ShuffleMapStage 329, ShuffleMapStage 323, ShuffleMapStage 351, ShuffleMapStage 345, ShuffleMapStage 337, ShuffleMapStage 331, ShuffleMapStage 353, ShuffleMapStage 339, ShuffleMapStage 325, ShuffleMapStage 355, ShuffleMapStage 347, ShuffleMapStage 341, ShuffleMapStage 333, ShuffleMapStage 327, ShuffleMapStage 349) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 357) -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting ShuffleMapStage 356 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[226] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(11440) called with curMem=277488, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 11.2 KB, free 730.3 MB) -2016-06-13 19:44:19 INFO MemoryStore:59 - ensureFreeSpace(4162) called with curMem=288928, maxMem=766075207 -2016-06-13 19:44:19 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 4.1 KB, free 730.3 MB) -2016-06-13 19:44:19 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:49949 (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:44:19 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:19 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 356 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[226] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:19 INFO TaskSchedulerImpl:59 - Adding task set 356.0 with 2 tasks -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 356.0 (TID 102, localhost, PROCESS_LOCAL, 2729 bytes) -2016-06-13 19:44:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 356.0 (TID 103, localhost, PROCESS_LOCAL, 2729 bytes) -2016-06-13 19:44:19 INFO Executor:59 - Running task 1.0 in stage 356.0 (TID 103) -2016-06-13 19:44:19 INFO Executor:59 - Running task 0.0 in stage 356.0 (TID 102) -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_221_1 locally -2016-06-13 19:44:19 INFO BlockManager:59 - Found block rdd_221_0 locally -2016-06-13 19:44:19 INFO Executor:59 - Finished task 0.0 in stage 356.0 (TID 102). 2254 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 356.0 (TID 102) in 41 ms on localhost (1/2) -2016-06-13 19:44:20 INFO Executor:59 - Finished task 1.0 in stage 356.0 (TID 103). 2254 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 356.0 (TID 103) in 48 ms on localhost (2/2) -2016-06-13 19:44:20 INFO DAGScheduler:59 - ShuffleMapStage 356 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.050 s -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 356.0, whose tasks have all completed, from pool -2016-06-13 19:44:20 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:20 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:20 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 357, ResultStage 358) -2016-06-13 19:44:20 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 357: List() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Missing parents for ResultStage 358: List(ShuffleMapStage 357) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting ShuffleMapStage 357 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[230] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=293090, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=296986, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:49949 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 357 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[230] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Adding task set 357.0 with 2 tasks -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 357.0 (TID 104, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 357.0 (TID 105, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:20 INFO Executor:59 - Running task 0.0 in stage 357.0 (TID 104) -2016-06-13 19:44:20 INFO Executor:59 - Running task 1.0 in stage 357.0 (TID 105) -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:20 INFO Executor:59 - Finished task 0.0 in stage 357.0 (TID 104). 2470 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 357.0 (TID 104) in 32 ms on localhost (1/2) -2016-06-13 19:44:20 INFO Executor:59 - Finished task 1.0 in stage 357.0 (TID 105). 2470 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 357.0 (TID 105) in 37 ms on localhost (2/2) -2016-06-13 19:44:20 INFO DAGScheduler:59 - ShuffleMapStage 357 (mapPartitions at GraphImpl.scala:235) finished in 0.038 s -2016-06-13 19:44:20 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:20 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:20 INFO DAGScheduler:59 - waiting: Set(ResultStage 358) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 357.0, whose tasks have all completed, from pool -2016-06-13 19:44:20 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Missing parents for ResultStage 358: List() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting ResultStage 358 (MapPartitionsRDD[238] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(12856) called with curMem=299272, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 12.6 KB, free 730.3 MB) -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(4586) called with curMem=312128, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.3 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:49949 (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 358 (MapPartitionsRDD[238] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Adding task set 358.0 with 2 tasks -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 358.0 (TID 106, localhost, PROCESS_LOCAL, 2781 bytes) -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 358.0 (TID 107, localhost, PROCESS_LOCAL, 2781 bytes) -2016-06-13 19:44:20 INFO Executor:59 - Running task 0.0 in stage 358.0 (TID 106) -2016-06-13 19:44:20 INFO Executor:59 - Running task 1.0 in stage 358.0 (TID 107) -2016-06-13 19:44:20 INFO CacheManager:59 - Partition rdd_234_0 not found, computing it -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_221_0 locally -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=316714, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block rdd_234_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:44:20 INFO CacheManager:59 - Partition rdd_234_1 not found, computing it -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_221_1 locally -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added rdd_234_0 in memory on localhost:49949 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=319258, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block rdd_234_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added rdd_234_1 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO Executor:59 - Finished task 1.0 in stage 358.0 (TID 107). 2971 bytes result sent to driver -2016-06-13 19:44:20 INFO Executor:59 - Finished task 0.0 in stage 358.0 (TID 106). 2971 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 358.0 (TID 107) in 21 ms on localhost (1/2) -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 358.0 (TID 106) in 23 ms on localhost (2/2) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 358.0, whose tasks have all completed, from pool -2016-06-13 19:44:20 INFO DAGScheduler:59 - ResultStage 358 (fold at LDAOptimizer.scala:205) finished in 0.025 s -2016-06-13 19:44:20 INFO DAGScheduler:59 - Job 18 finished: fold at LDAOptimizer.scala:205, took 0.193153 s -2016-06-13 19:44:20 INFO MapPartitionsRDD:59 - Removing RDD 208 from persistence list -2016-06-13 19:44:20 INFO BlockManager:59 - Removing RDD 208 -2016-06-13 19:44:20 INFO ZippedPartitionsRDD2:59 - Removing RDD 215 from persistence list -2016-06-13 19:44:20 INFO BlockManager:59 - Removing RDD 215 -2016-06-13 19:44:20 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:44:20 INFO DAGScheduler:59 - Registering RDD 239 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Registering RDD 243 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Got job 19 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:20 INFO DAGScheduler:59 - Final stage: ResultStage 397(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 386, ShuffleMapStage 372, ShuffleMapStage 394, ShuffleMapStage 380, ShuffleMapStage 359, ShuffleMapStage 366, ShuffleMapStage 396, ShuffleMapStage 360, ShuffleMapStage 388, ShuffleMapStage 382, ShuffleMapStage 374, ShuffleMapStage 368, ShuffleMapStage 390, ShuffleMapStage 376, ShuffleMapStage 362, ShuffleMapStage 392, ShuffleMapStage 384, ShuffleMapStage 378, ShuffleMapStage 370, ShuffleMapStage 364) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 396) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting ShuffleMapStage 395 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[239] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(11864) called with curMem=316714, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 11.6 KB, free 730.3 MB) -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(4268) called with curMem=328578, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.3 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:49949 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 395 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[239] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Adding task set 395.0 with 2 tasks -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 395.0 (TID 108, localhost, PROCESS_LOCAL, 2770 bytes) -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 395.0 (TID 109, localhost, PROCESS_LOCAL, 2770 bytes) -2016-06-13 19:44:20 INFO Executor:59 - Running task 0.0 in stage 395.0 (TID 108) -2016-06-13 19:44:20 INFO Executor:59 - Running task 1.0 in stage 395.0 (TID 109) -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_234_0 locally -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_234_1 locally -2016-06-13 19:44:20 INFO Executor:59 - Finished task 1.0 in stage 395.0 (TID 109). 2254 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 395.0 (TID 109) in 71 ms on localhost (1/2) -2016-06-13 19:44:20 INFO Executor:59 - Finished task 0.0 in stage 395.0 (TID 108). 2254 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 395.0 (TID 108) in 82 ms on localhost (2/2) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 395.0, whose tasks have all completed, from pool -2016-06-13 19:44:20 INFO DAGScheduler:59 - ShuffleMapStage 395 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.083 s -2016-06-13 19:44:20 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:20 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:20 INFO DAGScheduler:59 - waiting: Set(ResultStage 397, ShuffleMapStage 396) -2016-06-13 19:44:20 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Missing parents for ResultStage 397: List(ShuffleMapStage 396) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 396: List() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting ShuffleMapStage 396 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[243] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=332846, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(2288) called with curMem=336742, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:49949 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 396 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[243] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Adding task set 396.0 with 2 tasks -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 396.0 (TID 110, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 396.0 (TID 111, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:20 INFO Executor:59 - Running task 1.0 in stage 396.0 (TID 111) -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:20 INFO Executor:59 - Running task 0.0 in stage 396.0 (TID 110) -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:20 INFO Executor:59 - Finished task 1.0 in stage 396.0 (TID 111). 2470 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 396.0 (TID 111) in 49 ms on localhost (1/2) -2016-06-13 19:44:20 INFO Executor:59 - Finished task 0.0 in stage 396.0 (TID 110). 2470 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 396.0 (TID 110) in 61 ms on localhost (2/2) -2016-06-13 19:44:20 INFO DAGScheduler:59 - ShuffleMapStage 396 (mapPartitions at GraphImpl.scala:235) finished in 0.062 s -2016-06-13 19:44:20 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:20 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:20 INFO DAGScheduler:59 - waiting: Set(ResultStage 397) -2016-06-13 19:44:20 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 396.0, whose tasks have all completed, from pool -2016-06-13 19:44:20 INFO DAGScheduler:59 - Missing parents for ResultStage 397: List() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting ResultStage 397 (MapPartitionsRDD[251] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(13280) called with curMem=339030, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 13.0 KB, free 730.3 MB) -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(4699) called with curMem=352310, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 4.6 KB, free 730.2 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:49949 (size: 4.6 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 397 (MapPartitionsRDD[251] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Adding task set 397.0 with 2 tasks -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 397.0 (TID 112, localhost, PROCESS_LOCAL, 2822 bytes) -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 397.0 (TID 113, localhost, PROCESS_LOCAL, 2822 bytes) -2016-06-13 19:44:20 INFO Executor:59 - Running task 0.0 in stage 397.0 (TID 112) -2016-06-13 19:44:20 INFO Executor:59 - Running task 1.0 in stage 397.0 (TID 113) -2016-06-13 19:44:20 INFO CacheManager:59 - Partition rdd_247_1 not found, computing it -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_234_1 locally -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:20 INFO CacheManager:59 - Partition rdd_247_0 not found, computing it -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=357009, maxMem=766075207 -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_234_0 locally -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:20 INFO MemoryStore:59 - Block rdd_247_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added rdd_247_1 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO Executor:59 - Finished task 1.0 in stage 397.0 (TID 113). 2971 bytes result sent to driver -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=359457, maxMem=766075207 -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 397.0 (TID 113) in 20 ms on localhost (1/2) -2016-06-13 19:44:20 INFO MemoryStore:59 - Block rdd_247_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added rdd_247_0 in memory on localhost:49949 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO Executor:59 - Finished task 0.0 in stage 397.0 (TID 112). 2971 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 397.0 (TID 112) in 23 ms on localhost (2/2) -2016-06-13 19:44:20 INFO DAGScheduler:59 - ResultStage 397 (fold at LDAOptimizer.scala:205) finished in 0.023 s -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 397.0, whose tasks have all completed, from pool -2016-06-13 19:44:20 INFO DAGScheduler:59 - Job 19 finished: fold at LDAOptimizer.scala:205, took 0.259750 s -2016-06-13 19:44:20 INFO MapPartitionsRDD:59 - Removing RDD 221 from persistence list -2016-06-13 19:44:20 INFO ZippedPartitionsRDD2:59 - Removing RDD 228 from persistence list -2016-06-13 19:44:20 INFO BlockManager:59 - Removing RDD 221 -2016-06-13 19:44:20 INFO BlockManager:59 - Removing RDD 228 -2016-06-13 19:44:20 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 155 bytes -2016-06-13 19:44:20 INFO DAGScheduler:59 - Registering RDD 252 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Registering RDD 256 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Got job 20 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:20 INFO DAGScheduler:59 - Final stage: ResultStage 438(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 407, ShuffleMapStage 437, ShuffleMapStage 401, ShuffleMapStage 423, ShuffleMapStage 409, ShuffleMapStage 431, ShuffleMapStage 417, ShuffleMapStage 403, ShuffleMapStage 433, ShuffleMapStage 425, ShuffleMapStage 419, ShuffleMapStage 411, ShuffleMapStage 405, ShuffleMapStage 427, ShuffleMapStage 398, ShuffleMapStage 413, ShuffleMapStage 435, ShuffleMapStage 399, ShuffleMapStage 429, ShuffleMapStage 421, ShuffleMapStage 415) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 437) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting ShuffleMapStage 436 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[252] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(12296) called with curMem=357009, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 12.0 KB, free 730.2 MB) -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(4356) called with curMem=369305, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.2 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:49949 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 436 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[252] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Adding task set 436.0 with 2 tasks -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 436.0 (TID 114, localhost, PROCESS_LOCAL, 2811 bytes) -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 436.0 (TID 115, localhost, PROCESS_LOCAL, 2811 bytes) -2016-06-13 19:44:20 INFO Executor:59 - Running task 1.0 in stage 436.0 (TID 115) -2016-06-13 19:44:20 INFO Executor:59 - Running task 0.0 in stage 436.0 (TID 114) -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_247_1 locally -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_247_0 locally -2016-06-13 19:44:20 INFO Executor:59 - Finished task 1.0 in stage 436.0 (TID 115). 2254 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 436.0 (TID 115) in 27 ms on localhost (1/2) -2016-06-13 19:44:20 INFO Executor:59 - Finished task 0.0 in stage 436.0 (TID 114). 2254 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 436.0 (TID 114) in 36 ms on localhost (2/2) -2016-06-13 19:44:20 INFO DAGScheduler:59 - ShuffleMapStage 436 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.036 s -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 436.0, whose tasks have all completed, from pool -2016-06-13 19:44:20 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:20 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:20 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 437, ResultStage 438) -2016-06-13 19:44:20 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 437: List() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Missing parents for ResultStage 438: List(ShuffleMapStage 437) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting ShuffleMapStage 437 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[256] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=373661, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(2288) called with curMem=377557, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:49949 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 437 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[256] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Adding task set 437.0 with 2 tasks -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 437.0 (TID 116, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 437.0 (TID 117, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:20 INFO Executor:59 - Running task 0.0 in stage 437.0 (TID 116) -2016-06-13 19:44:20 INFO Executor:59 - Running task 1.0 in stage 437.0 (TID 117) -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:20 INFO Executor:59 - Finished task 0.0 in stage 437.0 (TID 116). 2470 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 437.0 (TID 116) in 33 ms on localhost (1/2) -2016-06-13 19:44:20 INFO Executor:59 - Finished task 1.0 in stage 437.0 (TID 117). 2470 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 437.0 (TID 117) in 34 ms on localhost (2/2) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 437.0, whose tasks have all completed, from pool -2016-06-13 19:44:20 INFO DAGScheduler:59 - ShuffleMapStage 437 (mapPartitions at GraphImpl.scala:235) finished in 0.035 s -2016-06-13 19:44:20 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:20 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:20 INFO DAGScheduler:59 - waiting: Set(ResultStage 438) -2016-06-13 19:44:20 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Missing parents for ResultStage 438: List() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting ResultStage 438 (MapPartitionsRDD[264] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(13712) called with curMem=379845, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 13.4 KB, free 730.2 MB) -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(4770) called with curMem=393557, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 4.7 KB, free 730.2 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:49949 (size: 4.7 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO SparkContext:59 - Created broadcast 61 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 438 (MapPartitionsRDD[264] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Adding task set 438.0 with 2 tasks -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 438.0 (TID 118, localhost, PROCESS_LOCAL, 2863 bytes) -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 438.0 (TID 119, localhost, PROCESS_LOCAL, 2863 bytes) -2016-06-13 19:44:20 INFO Executor:59 - Running task 0.0 in stage 438.0 (TID 118) -2016-06-13 19:44:20 INFO Executor:59 - Running task 1.0 in stage 438.0 (TID 119) -2016-06-13 19:44:20 INFO CacheManager:59 - Partition rdd_260_0 not found, computing it -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_247_0 locally -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:20 INFO CacheManager:59 - Partition rdd_260_1 not found, computing it -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_247_1 locally -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=398327, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block rdd_260_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added rdd_260_0 in memory on localhost:49949 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=400871, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block rdd_260_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:44:20 INFO Executor:59 - Finished task 0.0 in stage 438.0 (TID 118). 2971 bytes result sent to driver -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added rdd_260_1 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO Executor:59 - Finished task 1.0 in stage 438.0 (TID 119). 2971 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 438.0 (TID 118) in 13 ms on localhost (1/2) -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 438.0 (TID 119) in 14 ms on localhost (2/2) -2016-06-13 19:44:20 INFO DAGScheduler:59 - ResultStage 438 (fold at LDAOptimizer.scala:205) finished in 0.014 s -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 438.0, whose tasks have all completed, from pool -2016-06-13 19:44:20 INFO DAGScheduler:59 - Job 20 finished: fold at LDAOptimizer.scala:205, took 0.152985 s -2016-06-13 19:44:20 INFO MapPartitionsRDD:59 - Removing RDD 234 from persistence list -2016-06-13 19:44:20 INFO ZippedPartitionsRDD2:59 - Removing RDD 241 from persistence list -2016-06-13 19:44:20 INFO BlockManager:59 - Removing RDD 234 -2016-06-13 19:44:20 INFO BlockManager:59 - Removing RDD 241 -2016-06-13 19:44:20 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:44:20 INFO DAGScheduler:59 - Registering RDD 265 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Registering RDD 269 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Got job 21 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:20 INFO DAGScheduler:59 - Final stage: ResultStage 481(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 466, ShuffleMapStage 458, ShuffleMapStage 452, ShuffleMapStage 444, ShuffleMapStage 474, ShuffleMapStage 460, ShuffleMapStage 446, ShuffleMapStage 468, ShuffleMapStage 439, ShuffleMapStage 454, ShuffleMapStage 476, ShuffleMapStage 440, ShuffleMapStage 470, ShuffleMapStage 462, ShuffleMapStage 456, ShuffleMapStage 448, ShuffleMapStage 478, ShuffleMapStage 442, ShuffleMapStage 464, ShuffleMapStage 450, ShuffleMapStage 480, ShuffleMapStage 472) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 480) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting ShuffleMapStage 479 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[265] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(12720) called with curMem=398327, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 12.4 KB, free 730.2 MB) -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(4470) called with curMem=411047, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.2 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:49949 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 479 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[265] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Adding task set 479.0 with 2 tasks -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 479.0 (TID 120, localhost, PROCESS_LOCAL, 2852 bytes) -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 479.0 (TID 121, localhost, PROCESS_LOCAL, 2852 bytes) -2016-06-13 19:44:20 INFO Executor:59 - Running task 0.0 in stage 479.0 (TID 120) -2016-06-13 19:44:20 INFO Executor:59 - Running task 1.0 in stage 479.0 (TID 121) -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_260_0 locally -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_260_1 locally -2016-06-13 19:44:20 INFO Executor:59 - Finished task 0.0 in stage 479.0 (TID 120). 2254 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 479.0 (TID 120) in 26 ms on localhost (1/2) -2016-06-13 19:44:20 INFO Executor:59 - Finished task 1.0 in stage 479.0 (TID 121). 2254 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 479.0 (TID 121) in 39 ms on localhost (2/2) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 479.0, whose tasks have all completed, from pool -2016-06-13 19:44:20 INFO DAGScheduler:59 - ShuffleMapStage 479 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.040 s -2016-06-13 19:44:20 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:20 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:20 INFO DAGScheduler:59 - waiting: Set(ResultStage 481, ShuffleMapStage 480) -2016-06-13 19:44:20 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Missing parents for ResultStage 481: List(ShuffleMapStage 480) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 480: List() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting ShuffleMapStage 480 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[269] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=415517, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(2285) called with curMem=419413, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:49949 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 480 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[269] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Adding task set 480.0 with 2 tasks -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 480.0 (TID 122, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 480.0 (TID 123, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:20 INFO Executor:59 - Running task 0.0 in stage 480.0 (TID 122) -2016-06-13 19:44:20 INFO Executor:59 - Running task 1.0 in stage 480.0 (TID 123) -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:20 INFO Executor:59 - Finished task 0.0 in stage 480.0 (TID 122). 2470 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 480.0 (TID 122) in 25 ms on localhost (1/2) -2016-06-13 19:44:20 INFO Executor:59 - Finished task 1.0 in stage 480.0 (TID 123). 2470 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 480.0 (TID 123) in 38 ms on localhost (2/2) -2016-06-13 19:44:20 INFO DAGScheduler:59 - ShuffleMapStage 480 (mapPartitions at GraphImpl.scala:235) finished in 0.038 s -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 480.0, whose tasks have all completed, from pool -2016-06-13 19:44:20 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:20 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:20 INFO DAGScheduler:59 - waiting: Set(ResultStage 481) -2016-06-13 19:44:20 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Missing parents for ResultStage 481: List() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting ResultStage 481 (MapPartitionsRDD[277] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(14136) called with curMem=421698, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 13.8 KB, free 730.2 MB) -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(4890) called with curMem=435834, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 4.8 KB, free 730.2 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:49949 (size: 4.8 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 481 (MapPartitionsRDD[277] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Adding task set 481.0 with 2 tasks -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 481.0 (TID 124, localhost, PROCESS_LOCAL, 2904 bytes) -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 481.0 (TID 125, localhost, PROCESS_LOCAL, 2904 bytes) -2016-06-13 19:44:20 INFO Executor:59 - Running task 0.0 in stage 481.0 (TID 124) -2016-06-13 19:44:20 INFO Executor:59 - Running task 1.0 in stage 481.0 (TID 125) -2016-06-13 19:44:20 INFO CacheManager:59 - Partition rdd_273_0 not found, computing it -2016-06-13 19:44:20 INFO CacheManager:59 - Partition rdd_273_1 not found, computing it -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_260_1 locally -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_260_0 locally -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=440724, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block rdd_273_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=443268, maxMem=766075207 -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added rdd_273_0 in memory on localhost:49949 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO MemoryStore:59 - Block rdd_273_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added rdd_273_1 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO Executor:59 - Finished task 0.0 in stage 481.0 (TID 124). 2971 bytes result sent to driver -2016-06-13 19:44:20 INFO Executor:59 - Finished task 1.0 in stage 481.0 (TID 125). 2971 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 481.0 (TID 124) in 12 ms on localhost (1/2) -2016-06-13 19:44:20 INFO DAGScheduler:59 - ResultStage 481 (fold at LDAOptimizer.scala:205) finished in 0.013 s -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 481.0 (TID 125) in 11 ms on localhost (2/2) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 481.0, whose tasks have all completed, from pool -2016-06-13 19:44:20 INFO DAGScheduler:59 - Job 21 finished: fold at LDAOptimizer.scala:205, took 0.162598 s -2016-06-13 19:44:20 INFO MapPartitionsRDD:59 - Removing RDD 247 from persistence list -2016-06-13 19:44:20 INFO ZippedPartitionsRDD2:59 - Removing RDD 254 from persistence list -2016-06-13 19:44:20 INFO BlockManager:59 - Removing RDD 247 -2016-06-13 19:44:20 INFO BlockManager:59 - Removing RDD 254 -2016-06-13 19:44:20 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 41 is 155 bytes -2016-06-13 19:44:20 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 40 is 155 bytes -2016-06-13 19:44:20 INFO DAGScheduler:59 - Registering RDD 278 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Registering RDD 282 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Got job 22 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:44:20 INFO DAGScheduler:59 - Final stage: ResultStage 526(fold at LDAOptimizer.scala:205) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 487, ShuffleMapStage 509, ShuffleMapStage 503, ShuffleMapStage 495, ShuffleMapStage 525, ShuffleMapStage 489, ShuffleMapStage 517, ShuffleMapStage 511, ShuffleMapStage 482, ShuffleMapStage 497, ShuffleMapStage 519, ShuffleMapStage 483, ShuffleMapStage 505, ShuffleMapStage 491, ShuffleMapStage 521, ShuffleMapStage 513, ShuffleMapStage 507, ShuffleMapStage 499, ShuffleMapStage 493, ShuffleMapStage 485, ShuffleMapStage 515, ShuffleMapStage 501, ShuffleMapStage 523) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 525) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting ShuffleMapStage 524 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[278] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(13152) called with curMem=440724, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 12.8 KB, free 730.2 MB) -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(4564) called with curMem=453876, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.1 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:49949 (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 524 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[278] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Adding task set 524.0 with 2 tasks -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 524.0 (TID 126, localhost, PROCESS_LOCAL, 2893 bytes) -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 524.0 (TID 127, localhost, PROCESS_LOCAL, 2893 bytes) -2016-06-13 19:44:20 INFO Executor:59 - Running task 0.0 in stage 524.0 (TID 126) -2016-06-13 19:44:20 INFO Executor:59 - Running task 1.0 in stage 524.0 (TID 127) -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_273_0 locally -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_273_1 locally -2016-06-13 19:44:20 INFO Executor:59 - Finished task 1.0 in stage 524.0 (TID 127). 2254 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 524.0 (TID 127) in 30 ms on localhost (1/2) -2016-06-13 19:44:20 INFO Executor:59 - Finished task 0.0 in stage 524.0 (TID 126). 2254 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 524.0 (TID 126) in 40 ms on localhost (2/2) -2016-06-13 19:44:20 INFO DAGScheduler:59 - ShuffleMapStage 524 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.040 s -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 524.0, whose tasks have all completed, from pool -2016-06-13 19:44:20 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:20 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:20 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 525, ResultStage 526) -2016-06-13 19:44:20 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 525: List() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Missing parents for ResultStage 526: List(ShuffleMapStage 525) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting ShuffleMapStage 525 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[282] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=458440, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 3.8 KB, free 730.1 MB) -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(2289) called with curMem=462336, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:49949 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 525 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[282] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Adding task set 525.0 with 2 tasks -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 525.0 (TID 128, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 525.0 (TID 129, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:44:20 INFO Executor:59 - Running task 0.0 in stage 525.0 (TID 128) -2016-06-13 19:44:20 INFO Executor:59 - Running task 1.0 in stage 525.0 (TID 129) -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:20 INFO Executor:59 - Finished task 1.0 in stage 525.0 (TID 129). 2470 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 525.0 (TID 129) in 26 ms on localhost (1/2) -2016-06-13 19:44:20 INFO Executor:59 - Finished task 0.0 in stage 525.0 (TID 128). 2470 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 525.0 (TID 128) in 28 ms on localhost (2/2) -2016-06-13 19:44:20 INFO DAGScheduler:59 - ShuffleMapStage 525 (mapPartitions at GraphImpl.scala:235) finished in 0.028 s -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 525.0, whose tasks have all completed, from pool -2016-06-13 19:44:20 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:44:20 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:44:20 INFO DAGScheduler:59 - waiting: Set(ResultStage 526) -2016-06-13 19:44:20 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Missing parents for ResultStage 526: List() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting ResultStage 526 (MapPartitionsRDD[290] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(14568) called with curMem=464625, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 14.2 KB, free 730.1 MB) -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(4990) called with curMem=479193, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 4.9 KB, free 730.1 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:49949 (size: 4.9 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO SparkContext:59 - Created broadcast 67 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 526 (MapPartitionsRDD[290] at values at LDAOptimizer.scala:205) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Adding task set 526.0 with 2 tasks -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 526.0 (TID 130, localhost, PROCESS_LOCAL, 2945 bytes) -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 526.0 (TID 131, localhost, PROCESS_LOCAL, 2945 bytes) -2016-06-13 19:44:20 INFO Executor:59 - Running task 1.0 in stage 526.0 (TID 131) -2016-06-13 19:44:20 INFO Executor:59 - Running task 0.0 in stage 526.0 (TID 130) -2016-06-13 19:44:20 INFO CacheManager:59 - Partition rdd_286_1 not found, computing it -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_273_1 locally -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:20 INFO CacheManager:59 - Partition rdd_286_0 not found, computing it -2016-06-13 19:44:20 INFO BlockManager:59 - Found block rdd_273_0 locally -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:44:20 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=484183, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block rdd_286_1 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=486631, maxMem=766075207 -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added rdd_286_1 in memory on localhost:49949 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO MemoryStore:59 - Block rdd_286_0 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added rdd_286_0 in memory on localhost:49949 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:44:20 INFO Executor:59 - Finished task 1.0 in stage 526.0 (TID 131). 2971 bytes result sent to driver -2016-06-13 19:44:20 INFO Executor:59 - Finished task 0.0 in stage 526.0 (TID 130). 2971 bytes result sent to driver -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 526.0 (TID 131) in 10 ms on localhost (1/2) -2016-06-13 19:44:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 526.0 (TID 130) in 11 ms on localhost (2/2) -2016-06-13 19:44:20 INFO DAGScheduler:59 - ResultStage 526 (fold at LDAOptimizer.scala:205) finished in 0.011 s -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 526.0, whose tasks have all completed, from pool -2016-06-13 19:44:20 INFO DAGScheduler:59 - Job 22 finished: fold at LDAOptimizer.scala:205, took 0.167986 s -2016-06-13 19:44:20 INFO SparkContext:59 - Starting job: foreach at LDAFromWebSuite.scala:32 -2016-06-13 19:44:20 INFO DAGScheduler:59 - Got job 23 (foreach at LDAFromWebSuite.scala:32) with 2 output partitions -2016-06-13 19:44:20 INFO DAGScheduler:59 - Final stage: ResultStage 527(foreach at LDAFromWebSuite.scala:32) -2016-06-13 19:44:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting ResultStage 527 (MapPartitionsRDD[2] at map at LDAFromWebSuite.scala:23), which has no missing parents -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=489175, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-06-13 19:44:20 INFO MemoryStore:59 - ensureFreeSpace(1967) called with curMem=492551, maxMem=766075207 -2016-06-13 19:44:20 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 1967.0 B, free 730.1 MB) -2016-06-13 19:44:20 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:49949 (size: 1967.0 B, free: 730.5 MB) -2016-06-13 19:44:20 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 527 (MapPartitionsRDD[2] at map at LDAFromWebSuite.scala:23) -2016-06-13 19:44:20 INFO TaskSchedulerImpl:59 - Adding task set 527.0 with 2 tasks -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 527.0 (TID 132, localhost, PROCESS_LOCAL, 2197 bytes) -2016-06-13 19:44:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 527.0 (TID 133, localhost, PROCESS_LOCAL, 2197 bytes) -2016-06-13 19:44:20 INFO Executor:59 - Running task 0.0 in stage 527.0 (TID 132) -2016-06-13 19:44:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:0+132 -2016-06-13 19:44:20 INFO Executor:59 - Running task 1.0 in stage 527.0 (TID 133) -2016-06-13 19:44:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:132+132 -2016-06-13 19:44:21 INFO Executor:59 - Finished task 1.0 in stage 527.0 (TID 133). 2044 bytes result sent to driver -2016-06-13 19:44:21 INFO Executor:59 - Finished task 0.0 in stage 527.0 (TID 132). 2044 bytes result sent to driver -2016-06-13 19:44:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 527.0 (TID 132) in 30 ms on localhost (1/2) -2016-06-13 19:44:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 527.0 (TID 133) in 30 ms on localhost (2/2) -2016-06-13 19:44:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 527.0, whose tasks have all completed, from pool -2016-06-13 19:44:21 INFO DAGScheduler:59 - ResultStage 527 (foreach at LDAFromWebSuite.scala:32) finished in 0.032 s -2016-06-13 19:44:21 INFO DAGScheduler:59 - Job 23 finished: foreach at LDAFromWebSuite.scala:32, took 0.045200 s -2016-06-13 19:44:21 INFO SparkContext:59 - Starting job: foreach at LDAFromWebSuite.scala:34 -2016-06-13 19:44:21 INFO DAGScheduler:59 - Got job 24 (foreach at LDAFromWebSuite.scala:34) with 2 output partitions -2016-06-13 19:44:21 INFO DAGScheduler:59 - Final stage: ResultStage 528(foreach at LDAFromWebSuite.scala:34) -2016-06-13 19:44:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:44:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:44:21 INFO DAGScheduler:59 - Submitting ResultStage 528 (MapPartitionsRDD[4] at map at LDAFromWebSuite.scala:25), which has no missing parents -2016-06-13 19:44:21 INFO MemoryStore:59 - ensureFreeSpace(3672) called with curMem=494518, maxMem=766075207 -2016-06-13 19:44:21 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 3.6 KB, free 730.1 MB) -2016-06-13 19:44:21 INFO MemoryStore:59 - ensureFreeSpace(2055) called with curMem=498190, maxMem=766075207 -2016-06-13 19:44:21 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.1 MB) -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:49949 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO SparkContext:59 - Created broadcast 69 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 528 (MapPartitionsRDD[4] at map at LDAFromWebSuite.scala:25) -2016-06-13 19:44:21 INFO TaskSchedulerImpl:59 - Adding task set 528.0 with 2 tasks -2016-06-13 19:44:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 528.0 (TID 134, localhost, PROCESS_LOCAL, 2307 bytes) -2016-06-13 19:44:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 528.0 (TID 135, localhost, PROCESS_LOCAL, 2307 bytes) -2016-06-13 19:44:21 INFO Executor:59 - Running task 0.0 in stage 528.0 (TID 134) -2016-06-13 19:44:21 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:44:21 INFO Executor:59 - Finished task 0.0 in stage 528.0 (TID 134). 2044 bytes result sent to driver -2016-06-13 19:44:21 INFO Executor:59 - Running task 1.0 in stage 528.0 (TID 135) -2016-06-13 19:44:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 528.0 (TID 134) in 16 ms on localhost (1/2) -2016-06-13 19:44:21 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 19:44:21 INFO Executor:59 - Finished task 1.0 in stage 528.0 (TID 135). 2044 bytes result sent to driver -2016-06-13 19:44:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 528.0 (TID 135) in 21 ms on localhost (2/2) -2016-06-13 19:44:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 528.0, whose tasks have all completed, from pool -2016-06-13 19:44:21 INFO DAGScheduler:59 - ResultStage 528 (foreach at LDAFromWebSuite.scala:34) finished in 0.022 s -2016-06-13 19:44:21 INFO DAGScheduler:59 - Job 24 finished: foreach at LDAFromWebSuite.scala:34, took 0.035761 s -2016-06-13 19:44:21 INFO SparkContext:59 - Starting job: collect at LDAModel.scala:528 -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 41 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 40 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 43 is 155 bytes -2016-06-13 19:44:21 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 42 is 155 bytes -2016-06-13 19:44:21 INFO DAGScheduler:59 - Got job 25 (collect at LDAModel.scala:528) with 2 output partitions -2016-06-13 19:44:21 INFO DAGScheduler:59 - Final stage: ResultStage 573(collect at LDAModel.scala:528) -2016-06-13 19:44:21 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 538, ShuffleMapStage 560, ShuffleMapStage 546, ShuffleMapStage 540, ShuffleMapStage 568, ShuffleMapStage 532, ShuffleMapStage 562, ShuffleMapStage 554, ShuffleMapStage 548, ShuffleMapStage 570, ShuffleMapStage 534, ShuffleMapStage 556, ShuffleMapStage 542, ShuffleMapStage 572, ShuffleMapStage 564, ShuffleMapStage 558, ShuffleMapStage 550, ShuffleMapStage 529, ShuffleMapStage 544, ShuffleMapStage 536, ShuffleMapStage 566, ShuffleMapStage 530, ShuffleMapStage 552) -2016-06-13 19:44:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:44:21 INFO DAGScheduler:59 - Submitting ResultStage 573 (MapPartitionsRDD[293] at map at LDAModel.scala:526), which has no missing parents -2016-06-13 19:44:21 INFO MemoryStore:59 - ensureFreeSpace(14256) called with curMem=500245, maxMem=766075207 -2016-06-13 19:44:21 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 13.9 KB, free 730.1 MB) -2016-06-13 19:44:21 INFO MemoryStore:59 - ensureFreeSpace(4871) called with curMem=514501, maxMem=766075207 -2016-06-13 19:44:21 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 4.8 KB, free 730.1 MB) -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:49949 (size: 4.8 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:44:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 573 (MapPartitionsRDD[293] at map at LDAModel.scala:526) -2016-06-13 19:44:21 INFO TaskSchedulerImpl:59 - Adding task set 573.0 with 2 tasks -2016-06-13 19:44:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 573.0 (TID 136, localhost, PROCESS_LOCAL, 2945 bytes) -2016-06-13 19:44:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 573.0 (TID 137, localhost, PROCESS_LOCAL, 2945 bytes) -2016-06-13 19:44:21 INFO Executor:59 - Running task 0.0 in stage 573.0 (TID 136) -2016-06-13 19:44:21 INFO BlockManager:59 - Found block rdd_286_0 locally -2016-06-13 19:44:21 INFO Executor:59 - Finished task 0.0 in stage 573.0 (TID 136). 2680 bytes result sent to driver -2016-06-13 19:44:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 573.0 (TID 136) in 22 ms on localhost (1/2) -2016-06-13 19:44:21 INFO Executor:59 - Running task 1.0 in stage 573.0 (TID 137) -2016-06-13 19:44:21 INFO BlockManager:59 - Found block rdd_286_1 locally -2016-06-13 19:44:21 INFO Executor:59 - Finished task 1.0 in stage 573.0 (TID 137). 2612 bytes result sent to driver -2016-06-13 19:44:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 573.0 (TID 137) in 34 ms on localhost (2/2) -2016-06-13 19:44:21 INFO DAGScheduler:59 - ResultStage 573 (collect at LDAModel.scala:528) finished in 0.035 s -2016-06-13 19:44:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 573.0, whose tasks have all completed, from pool -2016-06-13 19:44:21 INFO DAGScheduler:59 - Job 25 finished: collect at LDAModel.scala:528, took 0.109642 s -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:49949 in memory (size: 4.8 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 70 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:49949 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 69 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:49949 in memory (size: 1967.0 B, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:49949 in memory (size: 4.9 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:49949 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:49949 in memory (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:49949 in memory (size: 4.8 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:49949 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:49949 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:49949 in memory (size: 4.7 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:49949 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:49949 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:49949 in memory (size: 4.6 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:49949 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:49949 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:49949 in memory (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:49949 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:49949 in memory (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:49949 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:49949 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:49949 in memory (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:49949 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:49949 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-06-13 19:44:21 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:49949 in memory (size: 3.9 KB, free: 730.5 MB) -2016-06-13 19:44:21 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 19:44:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 19:44:21 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 19:44:21 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 19:44:21 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 19:44:22 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 19:44:22 INFO BlockManager:59 - BlockManager stopped -2016-06-13 19:44:22 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 19:44:22 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 19:44:22 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 19:44:22 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 19:44:22 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 19:44:22 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-1b3e1343-0e73-4f3c-91c5-0233776e90dd -2016-06-13 19:44:22 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 19:45:00 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 19:45:01 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 19:45:01 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 19:45:01 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 19:45:01 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 19:45:02 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 19:45:03 INFO Remoting:74 - Starting remoting -2016-06-13 19:45:03 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:49980] -2016-06-13 19:45:03 INFO Utils:59 - Successfully started service 'sparkDriver' on port 49980. -2016-06-13 19:45:03 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 19:45:03 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 19:45:03 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-7e2d1f9f-457d-4cd8-a2be-18f20b9435c3 -2016-06-13 19:45:03 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 19:45:03 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-a892ed0a-dc6e-4c69-8272-b6c352cd294c\httpd-cd719b74-b390-4056-87f7-8beef9f03e7b -2016-06-13 19:45:03 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 19:45:03 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:45:03 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:49981 -2016-06-13 19:45:03 INFO Utils:59 - Successfully started service 'HTTP file server' on port 49981. -2016-06-13 19:45:03 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 19:45:03 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:45:03 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 19:45:03 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 19:45:03 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 19:45:04 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 19:45:04 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 19:45:04 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 50000. -2016-06-13 19:45:04 INFO NettyBlockTransferService:59 - Server created on 50000 -2016-06-13 19:45:04 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 19:45:04 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:50000 with 730.6 MB RAM, BlockManagerId(driver, localhost, 50000) -2016-06-13 19:45:04 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 19:45:06 INFO SparkContext:59 - Starting job: take at LDAOptimizer.scala:142 -2016-06-13 19:45:06 INFO DAGScheduler:59 - Got job 0 (take at LDAOptimizer.scala:142) with 1 output partitions -2016-06-13 19:45:06 INFO DAGScheduler:59 - Final stage: ResultStage 0(take at LDAOptimizer.scala:142) -2016-06-13 19:45:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:06 INFO DAGScheduler:59 - Submitting ResultStage 0 (ParallelCollectionRDD[0] at parallelize at LDASuite.scala:75), which has no missing parents -2016-06-13 19:45:06 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=0, maxMem=766075207 -2016-06-13 19:45:06 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1224.0 B, free 730.6 MB) -2016-06-13 19:45:06 INFO MemoryStore:59 - ensureFreeSpace(804) called with curMem=1224, maxMem=766075207 -2016-06-13 19:45:06 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 804.0 B, free 730.6 MB) -2016-06-13 19:45:06 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:50000 (size: 804.0 B, free: 730.6 MB) -2016-06-13 19:45:06 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:06 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (ParallelCollectionRDD[0] at parallelize at LDASuite.scala:75) -2016-06-13 19:45:06 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 19:45:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:06 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 19:45:06 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1192 bytes result sent to driver -2016-06-13 19:45:07 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 110 ms on localhost (1/1) -2016-06-13 19:45:07 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 19:45:07 INFO DAGScheduler:59 - ResultStage 0 (take at LDAOptimizer.scala:142) finished in 0.129 s -2016-06-13 19:45:07 INFO DAGScheduler:59 - Job 0 finished: take at LDAOptimizer.scala:142, took 0.551677 s -2016-06-13 19:45:07 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:45:07 INFO DAGScheduler:59 - Registering RDD 15 (map at GraphImpl.scala:104) -2016-06-13 19:45:07 INFO DAGScheduler:59 - Registering RDD 19 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:45:07 INFO DAGScheduler:59 - Registering RDD 7 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:45:07 INFO DAGScheduler:59 - Registering RDD 2 (mapPartitionsWithIndex at LDAOptimizer.scala:128) -2016-06-13 19:45:07 INFO DAGScheduler:59 - Got job 1 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:45:07 INFO DAGScheduler:59 - Final stage: ResultStage 5(fold at LDAOptimizer.scala:205) -2016-06-13 19:45:07 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 2, ShuffleMapStage 3, ShuffleMapStage 4) -2016-06-13 19:45:07 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 2, ShuffleMapStage 3, ShuffleMapStage 4) -2016-06-13 19:45:08 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1 (MapPartitionsRDD[15] at map at GraphImpl.scala:104), which has no missing parents -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(3880) called with curMem=0, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.6 MB) -2016-06-13 19:45:08 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:50000 in memory (size: 804.0 B, free: 730.6 MB) -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(2161) called with curMem=3880, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.6 MB) -2016-06-13 19:45:08 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:50000 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 19:45:08 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 19:45:08 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[15] at map at GraphImpl.scala:104) -2016-06-13 19:45:08 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 19:45:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2413 bytes) -2016-06-13 19:45:08 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[7] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:45:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(3072) called with curMem=6041, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.0 KB, free 730.6 MB) -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(1832) called with curMem=9113, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1832.0 B, free 730.6 MB) -2016-06-13 19:45:08 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 19:45:08 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:50000 (size: 1832.0 B, free: 730.6 MB) -2016-06-13 19:45:08 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 3 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[7] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:45:08 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 19:45:08 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[2] at mapPartitionsWithIndex at LDAOptimizer.scala:128), which has no missing parents -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(2904) called with curMem=10945, maxMem=766075207 -2016-06-13 19:45:08 INFO CacheManager:59 - Partition rdd_13_0 not found, computing it -2016-06-13 19:45:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 3, localhost, PROCESS_LOCAL, 2413 bytes) -2016-06-13 19:45:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 4, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:08 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 3) -2016-06-13 19:45:08 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 19:45:08 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 4) -2016-06-13 19:45:08 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.8 KB, free 730.6 MB) -2016-06-13 19:45:08 INFO CacheManager:59 - Partition rdd_13_1 not found, computing it -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(2680) called with curMem=13849, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block rdd_13_0 stored as values in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(2792) called with curMem=16529, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block rdd_13_1 stored as values in memory (estimated size 2.7 KB, free 730.6 MB) -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(1728) called with curMem=19321, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1728.0 B, free 730.6 MB) -2016-06-13 19:45:08 INFO BlockManagerInfo:59 - Added rdd_13_0 in memory on localhost:50000 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:45:08 INFO BlockManagerInfo:59 - Added rdd_13_1 in memory on localhost:50000 (size: 2.7 KB, free: 730.6 MB) -2016-06-13 19:45:08 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:50000 (size: 1728.0 B, free: 730.6 MB) -2016-06-13 19:45:08 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[2] at mapPartitionsWithIndex at LDAOptimizer.scala:128) -2016-06-13 19:45:08 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 19:45:08 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 1739 bytes result sent to driver -2016-06-13 19:45:08 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 3). 1159 bytes result sent to driver -2016-06-13 19:45:08 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 4). 1159 bytes result sent to driver -2016-06-13 19:45:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 5, localhost, PROCESS_LOCAL, 2413 bytes) -2016-06-13 19:45:08 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 5) -2016-06-13 19:45:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 6, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:08 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 6) -2016-06-13 19:45:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 171 ms on localhost (1/2) -2016-06-13 19:45:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 3) in 141 ms on localhost (1/2) -2016-06-13 19:45:08 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 19:45:08 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 19:45:08 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 1739 bytes result sent to driver -2016-06-13 19:45:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 262 ms on localhost (2/2) -2016-06-13 19:45:08 INFO DAGScheduler:59 - ShuffleMapStage 1 (map at GraphImpl.scala:104) finished in 0.268 s -2016-06-13 19:45:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 19:45:08 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:08 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 3, ShuffleMapStage 4) -2016-06-13 19:45:08 INFO DAGScheduler:59 - waiting: Set(ResultStage 5, ShuffleMapStage 2) -2016-06-13 19:45:08 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:08 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List(ShuffleMapStage 2, ShuffleMapStage 3, ShuffleMapStage 4) -2016-06-13 19:45:08 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 2: List() -2016-06-13 19:45:08 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[19] at mapPartitions at VertexRDD.scala:358), which is now runnable -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(2784) called with curMem=21049, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 2.7 KB, free 730.6 MB) -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(1698) called with curMem=23833, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 1698.0 B, free 730.6 MB) -2016-06-13 19:45:08 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 5). 1159 bytes result sent to driver -2016-06-13 19:45:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 5) in 135 ms on localhost (1/2) -2016-06-13 19:45:08 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:50000 (size: 1698.0 B, free: 730.6 MB) -2016-06-13 19:45:08 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[19] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:45:08 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 19:45:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 7, localhost, PROCESS_LOCAL, 1890 bytes) -2016-06-13 19:45:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 8, localhost, PROCESS_LOCAL, 1890 bytes) -2016-06-13 19:45:08 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 7) -2016-06-13 19:45:08 INFO CacheManager:59 - Partition rdd_17_0 not found, computing it -2016-06-13 19:45:08 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 8) -2016-06-13 19:45:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-06-13 19:45:08 INFO CacheManager:59 - Partition rdd_17_1 not found, computing it -2016-06-13 19:45:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(2688) called with curMem=25531, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block rdd_17_0 stored as values in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 19:45:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 4) in 316 ms on localhost (2/2) -2016-06-13 19:45:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(2784) called with curMem=28219, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block rdd_17_1 stored as values in memory (estimated size 2.7 KB, free 730.6 MB) -2016-06-13 19:45:08 INFO DAGScheduler:59 - ShuffleMapStage 3 (mapPartitions at VertexRDD.scala:358) finished in 0.350 s -2016-06-13 19:45:08 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:08 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 2, ShuffleMapStage 4) -2016-06-13 19:45:08 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-06-13 19:45:08 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:08 INFO BlockManagerInfo:59 - Added rdd_17_0 in memory on localhost:50000 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:45:08 INFO BlockManagerInfo:59 - Added rdd_17_1 in memory on localhost:50000 (size: 2.7 KB, free: 730.6 MB) -2016-06-13 19:45:08 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 6). 1159 bytes result sent to driver -2016-06-13 19:45:08 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 8). 1955 bytes result sent to driver -2016-06-13 19:45:08 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 7). 1955 bytes result sent to driver -2016-06-13 19:45:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 6) in 245 ms on localhost (2/2) -2016-06-13 19:45:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 19:45:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 8) in 100 ms on localhost (1/2) -2016-06-13 19:45:08 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List(ShuffleMapStage 2, ShuffleMapStage 4) -2016-06-13 19:45:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 7) in 104 ms on localhost (2/2) -2016-06-13 19:45:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 19:45:08 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitionsWithIndex at LDAOptimizer.scala:128) finished in 0.314 s -2016-06-13 19:45:08 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:08 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 2) -2016-06-13 19:45:08 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-06-13 19:45:08 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:08 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List(ShuffleMapStage 2) -2016-06-13 19:45:08 INFO DAGScheduler:59 - ShuffleMapStage 2 (mapPartitions at VertexRDD.scala:358) finished in 0.116 s -2016-06-13 19:45:08 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:08 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:08 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-06-13 19:45:08 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:08 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-06-13 19:45:08 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[26] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(5584) called with curMem=31003, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.5 KB, free 730.6 MB) -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(2803) called with curMem=36587, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.5 MB) -2016-06-13 19:45:08 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:50000 (size: 2.7 KB, free: 730.6 MB) -2016-06-13 19:45:08 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[26] at values at LDAOptimizer.scala:205) -2016-06-13 19:45:08 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 19:45:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:45:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 10, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:45:08 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 10) -2016-06-13 19:45:08 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 9) -2016-06-13 19:45:08 INFO CacheManager:59 - Partition rdd_22_1 not found, computing it -2016-06-13 19:45:08 INFO CacheManager:59 - Partition rdd_10_1 not found, computing it -2016-06-13 19:45:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:08 INFO CacheManager:59 - Partition rdd_22_0 not found, computing it -2016-06-13 19:45:08 INFO CacheManager:59 - Partition rdd_10_0 not found, computing it -2016-06-13 19:45:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(1848) called with curMem=39390, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block rdd_10_1 stored as values in memory (estimated size 1848.0 B, free 730.5 MB) -2016-06-13 19:45:08 INFO BlockManagerInfo:59 - Added rdd_10_1 in memory on localhost:50000 (size: 1848.0 B, free: 730.6 MB) -2016-06-13 19:45:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(1856) called with curMem=41238, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block rdd_10_0 stored as values in memory (estimated size 1856.0 B, free 730.5 MB) -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(1848) called with curMem=43094, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block rdd_22_1 stored as values in memory (estimated size 1848.0 B, free 730.5 MB) -2016-06-13 19:45:08 INFO BlockManagerInfo:59 - Added rdd_10_0 in memory on localhost:50000 (size: 1856.0 B, free: 730.6 MB) -2016-06-13 19:45:08 INFO BlockManagerInfo:59 - Added rdd_22_1 in memory on localhost:50000 (size: 1848.0 B, free: 730.6 MB) -2016-06-13 19:45:08 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 10). 1931 bytes result sent to driver -2016-06-13 19:45:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 10) in 105 ms on localhost (1/2) -2016-06-13 19:45:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(1856) called with curMem=44942, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block rdd_22_0 stored as values in memory (estimated size 1856.0 B, free 730.5 MB) -2016-06-13 19:45:08 INFO BlockManagerInfo:59 - Added rdd_22_0 in memory on localhost:50000 (size: 1856.0 B, free: 730.6 MB) -2016-06-13 19:45:08 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 9). 1931 bytes result sent to driver -2016-06-13 19:45:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 9) in 124 ms on localhost (2/2) -2016-06-13 19:45:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 19:45:08 INFO DAGScheduler:59 - ResultStage 5 (fold at LDAOptimizer.scala:205) finished in 0.125 s -2016-06-13 19:45:08 INFO DAGScheduler:59 - Job 1 finished: fold at LDAOptimizer.scala:205, took 1.519161 s -2016-06-13 19:45:08 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:45:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 154 bytes -2016-06-13 19:45:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 154 bytes -2016-06-13 19:45:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:45:08 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:45:08 INFO DAGScheduler:59 - Registering RDD 27 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:08 INFO DAGScheduler:59 - Registering RDD 31 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:08 INFO DAGScheduler:59 - Got job 2 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:45:08 INFO DAGScheduler:59 - Final stage: ResultStage 12(fold at LDAOptimizer.scala:205) -2016-06-13 19:45:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 9, ShuffleMapStage 6, ShuffleMapStage 7, ShuffleMapStage 11) -2016-06-13 19:45:08 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 11) -2016-06-13 19:45:08 INFO DAGScheduler:59 - Submitting ShuffleMapStage 10 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[27] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(4360) called with curMem=46798, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 4.3 KB, free 730.5 MB) -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(2390) called with curMem=51158, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 19:45:08 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:50000 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 19:45:08 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 10 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[27] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:08 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 2 tasks -2016-06-13 19:45:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 11, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:45:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 12, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:45:08 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 11) -2016-06-13 19:45:08 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 12) -2016-06-13 19:45:08 INFO BlockManager:59 - Found block rdd_22_0 locally -2016-06-13 19:45:08 INFO BlockManager:59 - Found block rdd_22_1 locally -2016-06-13 19:45:08 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 11). 2254 bytes result sent to driver -2016-06-13 19:45:08 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 12). 2254 bytes result sent to driver -2016-06-13 19:45:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 12) in 80 ms on localhost (1/2) -2016-06-13 19:45:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 11) in 84 ms on localhost (2/2) -2016-06-13 19:45:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 19:45:08 INFO DAGScheduler:59 - ShuffleMapStage 10 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.086 s -2016-06-13 19:45:08 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:08 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:08 INFO DAGScheduler:59 - waiting: Set(ResultStage 12, ShuffleMapStage 11) -2016-06-13 19:45:08 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:08 INFO DAGScheduler:59 - Missing parents for ResultStage 12: List(ShuffleMapStage 11) -2016-06-13 19:45:08 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 11: List() -2016-06-13 19:45:08 INFO DAGScheduler:59 - Submitting ShuffleMapStage 11 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[31] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=53548, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(2278) called with curMem=57444, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 19:45:08 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:50000 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 19:45:08 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 11 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[31] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:08 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 19:45:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 13, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 14, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:08 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 14) -2016-06-13 19:45:08 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 13) -2016-06-13 19:45:08 INFO BlockManager:59 - Found block rdd_17_1 locally -2016-06-13 19:45:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:08 INFO BlockManager:59 - Found block rdd_17_0 locally -2016-06-13 19:45:08 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:08 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:08 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 14). 2470 bytes result sent to driver -2016-06-13 19:45:08 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 14) in 49 ms on localhost (1/2) -2016-06-13 19:45:08 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 13). 2470 bytes result sent to driver -2016-06-13 19:45:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 13) in 59 ms on localhost (2/2) -2016-06-13 19:45:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 19:45:08 INFO DAGScheduler:59 - ShuffleMapStage 11 (mapPartitions at GraphImpl.scala:235) finished in 0.060 s -2016-06-13 19:45:08 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:08 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:08 INFO DAGScheduler:59 - waiting: Set(ResultStage 12) -2016-06-13 19:45:08 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:08 INFO DAGScheduler:59 - Missing parents for ResultStage 12: List() -2016-06-13 19:45:08 INFO DAGScheduler:59 - Submitting ResultStage 12 (MapPartitionsRDD[39] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(6432) called with curMem=59722, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 6.3 KB, free 730.5 MB) -2016-06-13 19:45:08 INFO MemoryStore:59 - ensureFreeSpace(3043) called with curMem=66154, maxMem=766075207 -2016-06-13 19:45:08 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.5 MB) -2016-06-13 19:45:08 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:50000 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 19:45:08 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 12 (MapPartitionsRDD[39] at values at LDAOptimizer.scala:205) -2016-06-13 19:45:08 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 19:45:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 15, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:45:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 16, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:45:08 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 16) -2016-06-13 19:45:09 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 15) -2016-06-13 19:45:09 INFO CacheManager:59 - Partition rdd_35_1 not found, computing it -2016-06-13 19:45:09 INFO BlockManager:59 - Found block rdd_22_1 locally -2016-06-13 19:45:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(1896) called with curMem=69197, maxMem=766075207 -2016-06-13 19:45:09 INFO MemoryStore:59 - Block rdd_35_1 stored as values in memory (estimated size 1896.0 B, free 730.5 MB) -2016-06-13 19:45:09 INFO BlockManagerInfo:59 - Added rdd_35_1 in memory on localhost:50000 (size: 1896.0 B, free: 730.5 MB) -2016-06-13 19:45:09 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 16). 2971 bytes result sent to driver -2016-06-13 19:45:09 INFO CacheManager:59 - Partition rdd_35_0 not found, computing it -2016-06-13 19:45:09 INFO BlockManager:59 - Found block rdd_22_0 locally -2016-06-13 19:45:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 16) in 96 ms on localhost (1/2) -2016-06-13 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(1904) called with curMem=71093, maxMem=766075207 -2016-06-13 19:45:09 INFO MemoryStore:59 - Block rdd_35_0 stored as values in memory (estimated size 1904.0 B, free 730.5 MB) -2016-06-13 19:45:09 INFO BlockManagerInfo:59 - Added rdd_35_0 in memory on localhost:50000 (size: 1904.0 B, free: 730.5 MB) -2016-06-13 19:45:09 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 15). 2971 bytes result sent to driver -2016-06-13 19:45:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 15) in 135 ms on localhost (2/2) -2016-06-13 19:45:09 INFO DAGScheduler:59 - ResultStage 12 (fold at LDAOptimizer.scala:205) finished in 0.137 s -2016-06-13 19:45:09 INFO DAGScheduler:59 - Job 2 finished: fold at LDAOptimizer.scala:205, took 0.372247 s -2016-06-13 19:45:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 19:45:09 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:45:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 154 bytes -2016-06-13 19:45:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 154 bytes -2016-06-13 19:45:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:45:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:45:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:45:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:45:09 INFO DAGScheduler:59 - Registering RDD 40 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:09 INFO DAGScheduler:59 - Registering RDD 44 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:09 INFO DAGScheduler:59 - Got job 3 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:45:09 INFO DAGScheduler:59 - Final stage: ResultStage 21(fold at LDAOptimizer.scala:205) -2016-06-13 19:45:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 16, ShuffleMapStage 13, ShuffleMapStage 20, ShuffleMapStage 18, ShuffleMapStage 14) -2016-06-13 19:45:09 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 20) -2016-06-13 19:45:09 INFO DAGScheduler:59 - Submitting ShuffleMapStage 19 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[40] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=72997, maxMem=766075207 -2016-06-13 19:45:09 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.3 KB, free 730.5 MB) -2016-06-13 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(2729) called with curMem=78445, maxMem=766075207 -2016-06-13 19:45:09 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.5 MB) -2016-06-13 19:45:09 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:50000 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:45:09 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 19 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[40] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:09 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-06-13 19:45:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 17, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:45:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 18, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:45:09 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 17) -2016-06-13 19:45:09 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 18) -2016-06-13 19:45:09 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 19:45:09 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 19:45:09 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 18). 2254 bytes result sent to driver -2016-06-13 19:45:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 18) in 34 ms on localhost (1/2) -2016-06-13 19:45:09 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 17). 2254 bytes result sent to driver -2016-06-13 19:45:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 17) in 42 ms on localhost (2/2) -2016-06-13 19:45:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 19:45:09 INFO DAGScheduler:59 - ShuffleMapStage 19 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.043 s -2016-06-13 19:45:09 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:09 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:09 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 20, ResultStage 21) -2016-06-13 19:45:09 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:09 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 20: List() -2016-06-13 19:45:09 INFO DAGScheduler:59 - Missing parents for ResultStage 21: List(ShuffleMapStage 20) -2016-06-13 19:45:09 INFO DAGScheduler:59 - Submitting ShuffleMapStage 20 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[44] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=81174, maxMem=766075207 -2016-06-13 19:45:09 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(2275) called with curMem=85070, maxMem=766075207 -2016-06-13 19:45:09 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 19:45:09 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:50000 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:45:09 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 20 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[44] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:09 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-06-13 19:45:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 19, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 20, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:09 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 20) -2016-06-13 19:45:09 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 19) -2016-06-13 19:45:09 INFO BlockManager:59 - Found block rdd_17_1 locally -2016-06-13 19:45:09 INFO BlockManager:59 - Found block rdd_17_0 locally -2016-06-13 19:45:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:09 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 19). 2470 bytes result sent to driver -2016-06-13 19:45:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 19) in 41 ms on localhost (1/2) -2016-06-13 19:45:09 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 20). 2470 bytes result sent to driver -2016-06-13 19:45:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 20) in 45 ms on localhost (2/2) -2016-06-13 19:45:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 19:45:09 INFO DAGScheduler:59 - ShuffleMapStage 20 (mapPartitions at GraphImpl.scala:235) finished in 0.047 s -2016-06-13 19:45:09 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:09 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:09 INFO DAGScheduler:59 - waiting: Set(ResultStage 21) -2016-06-13 19:45:09 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:09 INFO DAGScheduler:59 - Missing parents for ResultStage 21: List() -2016-06-13 19:45:09 INFO DAGScheduler:59 - Submitting ResultStage 21 (MapPartitionsRDD[52] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(6864) called with curMem=87345, maxMem=766075207 -2016-06-13 19:45:09 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 6.7 KB, free 730.5 MB) -2016-06-13 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=94209, maxMem=766075207 -2016-06-13 19:45:09 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.5 MB) -2016-06-13 19:45:09 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:50000 (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:45:09 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (MapPartitionsRDD[52] at values at LDAOptimizer.scala:205) -2016-06-13 19:45:09 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 19:45:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 21, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:45:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 22, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:45:09 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 21) -2016-06-13 19:45:09 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 22) -2016-06-13 19:45:09 INFO CacheManager:59 - Partition rdd_48_0 not found, computing it -2016-06-13 19:45:09 INFO BlockManager:59 - Found block rdd_35_0 locally -2016-06-13 19:45:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(1904) called with curMem=97373, maxMem=766075207 -2016-06-13 19:45:09 INFO MemoryStore:59 - Block rdd_48_0 stored as values in memory (estimated size 1904.0 B, free 730.5 MB) -2016-06-13 19:45:09 INFO BlockManagerInfo:59 - Added rdd_48_0 in memory on localhost:50000 (size: 1904.0 B, free: 730.5 MB) -2016-06-13 19:45:09 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 21). 2971 bytes result sent to driver -2016-06-13 19:45:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 21) in 39 ms on localhost (1/2) -2016-06-13 19:45:09 INFO CacheManager:59 - Partition rdd_48_1 not found, computing it -2016-06-13 19:45:09 INFO BlockManager:59 - Found block rdd_35_1 locally -2016-06-13 19:45:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(1896) called with curMem=99277, maxMem=766075207 -2016-06-13 19:45:09 INFO MemoryStore:59 - Block rdd_48_1 stored as values in memory (estimated size 1896.0 B, free 730.5 MB) -2016-06-13 19:45:09 INFO BlockManagerInfo:59 - Added rdd_48_1 in memory on localhost:50000 (size: 1896.0 B, free: 730.5 MB) -2016-06-13 19:45:09 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 22). 2971 bytes result sent to driver -2016-06-13 19:45:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 22) in 57 ms on localhost (2/2) -2016-06-13 19:45:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 19:45:09 INFO DAGScheduler:59 - ResultStage 21 (fold at LDAOptimizer.scala:205) finished in 0.062 s -2016-06-13 19:45:09 INFO DAGScheduler:59 - Job 3 finished: fold at LDAOptimizer.scala:205, took 0.234920 s -2016-06-13 19:45:09 INFO ZippedPartitionsRDD2:59 - Removing RDD 22 from persistence list -2016-06-13 19:45:09 INFO BlockManager:59 - Removing RDD 22 -2016-06-13 19:45:09 INFO ZippedPartitionsRDD2:59 - Removing RDD 29 from persistence list -2016-06-13 19:45:09 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:45:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 154 bytes -2016-06-13 19:45:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 154 bytes -2016-06-13 19:45:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:45:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:45:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:45:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:45:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:45:09 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:45:09 INFO DAGScheduler:59 - Registering RDD 53 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:09 INFO DAGScheduler:59 - Registering RDD 57 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:09 INFO DAGScheduler:59 - Got job 4 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:45:09 INFO DAGScheduler:59 - Final stage: ResultStage 32(fold at LDAOptimizer.scala:205) -2016-06-13 19:45:09 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 27, ShuffleMapStage 31, ShuffleMapStage 25, ShuffleMapStage 22, ShuffleMapStage 29, ShuffleMapStage 23) -2016-06-13 19:45:09 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 31) -2016-06-13 19:45:09 INFO BlockManager:59 - Removing RDD 29 -2016-06-13 19:45:09 INFO DAGScheduler:59 - Submitting ShuffleMapStage 30 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[53] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(5872) called with curMem=97469, maxMem=766075207 -2016-06-13 19:45:09 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 5.7 KB, free 730.5 MB) -2016-06-13 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(2858) called with curMem=103341, maxMem=766075207 -2016-06-13 19:45:09 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.5 MB) -2016-06-13 19:45:09 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:50000 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:45:09 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 30 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[53] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:09 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 2 tasks -2016-06-13 19:45:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 23, localhost, PROCESS_LOCAL, 2196 bytes) -2016-06-13 19:45:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 24, localhost, PROCESS_LOCAL, 2196 bytes) -2016-06-13 19:45:09 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 24) -2016-06-13 19:45:09 INFO BlockManager:59 - Found block rdd_48_1 locally -2016-06-13 19:45:09 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 24). 2254 bytes result sent to driver -2016-06-13 19:45:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 24) in 36 ms on localhost (1/2) -2016-06-13 19:45:09 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 23) -2016-06-13 19:45:09 INFO BlockManager:59 - Found block rdd_48_0 locally -2016-06-13 19:45:09 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 23). 2254 bytes result sent to driver -2016-06-13 19:45:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 23) in 82 ms on localhost (2/2) -2016-06-13 19:45:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 19:45:09 INFO DAGScheduler:59 - ShuffleMapStage 30 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.084 s -2016-06-13 19:45:09 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:09 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:09 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 31, ResultStage 32) -2016-06-13 19:45:09 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:09 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 31: List() -2016-06-13 19:45:09 INFO DAGScheduler:59 - Missing parents for ResultStage 32: List(ShuffleMapStage 31) -2016-06-13 19:45:09 INFO DAGScheduler:59 - Submitting ShuffleMapStage 31 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[57] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=106199, maxMem=766075207 -2016-06-13 19:45:09 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(2278) called with curMem=110095, maxMem=766075207 -2016-06-13 19:45:09 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 19:45:09 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:50000 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:45:09 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 31 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[57] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:09 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 19:45:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 25, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 26, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:09 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 26) -2016-06-13 19:45:09 INFO BlockManager:59 - Found block rdd_17_1 locally -2016-06-13 19:45:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:09 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:50000 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 19:45:09 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 25) -2016-06-13 19:45:09 INFO BlockManager:59 - Found block rdd_17_0 locally -2016-06-13 19:45:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:09 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 26). 2470 bytes result sent to driver -2016-06-13 19:45:09 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 25). 2470 bytes result sent to driver -2016-06-13 19:45:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 26) in 71 ms on localhost (1/2) -2016-06-13 19:45:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 25) in 75 ms on localhost (2/2) -2016-06-13 19:45:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 19:45:09 INFO DAGScheduler:59 - ShuffleMapStage 31 (mapPartitions at GraphImpl.scala:235) finished in 0.081 s -2016-06-13 19:45:09 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:09 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:09 INFO DAGScheduler:59 - waiting: Set(ResultStage 32) -2016-06-13 19:45:09 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:09 INFO DAGScheduler:59 - Missing parents for ResultStage 32: List() -2016-06-13 19:45:09 INFO DAGScheduler:59 - Submitting ResultStage 32 (MapPartitionsRDD[65] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(7288) called with curMem=96304, maxMem=766075207 -2016-06-13 19:45:09 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 7.1 KB, free 730.5 MB) -2016-06-13 19:45:09 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:50000 in memory (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(3278) called with curMem=103592, maxMem=766075207 -2016-06-13 19:45:09 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 19:45:09 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:50000 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:45:09 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:09 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 19:45:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 32 (MapPartitionsRDD[65] at values at LDAOptimizer.scala:205) -2016-06-13 19:45:09 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-06-13 19:45:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 27, localhost, PROCESS_LOCAL, 2248 bytes) -2016-06-13 19:45:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 28, localhost, PROCESS_LOCAL, 2248 bytes) -2016-06-13 19:45:09 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 27) -2016-06-13 19:45:09 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 28) -2016-06-13 19:45:09 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:50000 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:45:09 INFO CacheManager:59 - Partition rdd_61_0 not found, computing it -2016-06-13 19:45:09 INFO BlockManager:59 - Found block rdd_48_0 locally -2016-06-13 19:45:09 INFO CacheManager:59 - Partition rdd_61_1 not found, computing it -2016-06-13 19:45:09 INFO BlockManager:59 - Found block rdd_48_1 locally -2016-06-13 19:45:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(1896) called with curMem=100699, maxMem=766075207 -2016-06-13 19:45:09 INFO MemoryStore:59 - Block rdd_61_1 stored as values in memory (estimated size 1896.0 B, free 730.5 MB) -2016-06-13 19:45:09 INFO MemoryStore:59 - ensureFreeSpace(1904) called with curMem=102595, maxMem=766075207 -2016-06-13 19:45:09 INFO MemoryStore:59 - Block rdd_61_0 stored as values in memory (estimated size 1904.0 B, free 730.5 MB) -2016-06-13 19:45:09 INFO BlockManagerInfo:59 - Added rdd_61_1 in memory on localhost:50000 (size: 1896.0 B, free: 730.5 MB) -2016-06-13 19:45:09 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 28). 2971 bytes result sent to driver -2016-06-13 19:45:09 INFO BlockManagerInfo:59 - Added rdd_61_0 in memory on localhost:50000 (size: 1904.0 B, free: 730.5 MB) -2016-06-13 19:45:09 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 27). 2971 bytes result sent to driver -2016-06-13 19:45:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 28) in 35 ms on localhost (1/2) -2016-06-13 19:45:09 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 19:45:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 27) in 40 ms on localhost (2/2) -2016-06-13 19:45:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 19:45:09 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:50000 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:45:09 INFO DAGScheduler:59 - ResultStage 32 (fold at LDAOptimizer.scala:205) finished in 0.046 s -2016-06-13 19:45:09 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 19:45:09 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:50000 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:45:09 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 19:45:09 INFO DAGScheduler:59 - Job 4 finished: fold at LDAOptimizer.scala:205, took 0.329609 s -2016-06-13 19:45:09 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:50000 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:45:10 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 19:45:10 INFO MapPartitionsRDD:59 - Removing RDD 35 from persistence list -2016-06-13 19:45:10 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:50000 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:45:10 INFO BlockManager:59 - Removing RDD 35 -2016-06-13 19:45:10 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 19:45:10 INFO ZippedPartitionsRDD2:59 - Removing RDD 42 from persistence list -2016-06-13 19:45:10 INFO BlockManager:59 - Removing RDD 42 -2016-06-13 19:45:10 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:50000 in memory (size: 2.7 KB, free: 730.6 MB) -2016-06-13 19:45:10 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:50000 in memory (size: 1832.0 B, free: 730.6 MB) -2016-06-13 19:45:10 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 154 bytes -2016-06-13 19:45:10 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:50000 in memory (size: 1698.0 B, free: 730.6 MB) -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 154 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:45:10 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:50000 in memory (size: 1728.0 B, free: 730.6 MB) -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:45:10 INFO DAGScheduler:59 - Registering RDD 66 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:10 INFO DAGScheduler:59 - Registering RDD 70 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:10 INFO DAGScheduler:59 - Got job 5 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:45:10 INFO DAGScheduler:59 - Final stage: ResultStage 45(fold at LDAOptimizer.scala:205) -2016-06-13 19:45:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 33, ShuffleMapStage 34, ShuffleMapStage 38, ShuffleMapStage 42, ShuffleMapStage 36, ShuffleMapStage 40, ShuffleMapStage 44) -2016-06-13 19:45:10 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 44) -2016-06-13 19:45:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 43 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[66] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:45:10 INFO MemoryStore:59 - ensureFreeSpace(6304) called with curMem=47718, maxMem=766075207 -2016-06-13 19:45:10 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 6.2 KB, free 730.5 MB) -2016-06-13 19:45:10 INFO MemoryStore:59 - ensureFreeSpace(2999) called with curMem=54022, maxMem=766075207 -2016-06-13 19:45:10 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.5 MB) -2016-06-13 19:45:10 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:50000 (size: 2.9 KB, free: 730.6 MB) -2016-06-13 19:45:10 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 43 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[66] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:10 INFO TaskSchedulerImpl:59 - Adding task set 43.0 with 2 tasks -2016-06-13 19:45:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 43.0 (TID 29, localhost, PROCESS_LOCAL, 2237 bytes) -2016-06-13 19:45:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 43.0 (TID 30, localhost, PROCESS_LOCAL, 2237 bytes) -2016-06-13 19:45:10 INFO Executor:59 - Running task 1.0 in stage 43.0 (TID 30) -2016-06-13 19:45:10 INFO BlockManager:59 - Found block rdd_61_1 locally -2016-06-13 19:45:10 INFO Executor:59 - Running task 0.0 in stage 43.0 (TID 29) -2016-06-13 19:45:10 INFO BlockManager:59 - Found block rdd_61_0 locally -2016-06-13 19:45:10 INFO Executor:59 - Finished task 1.0 in stage 43.0 (TID 30). 2254 bytes result sent to driver -2016-06-13 19:45:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 43.0 (TID 30) in 42 ms on localhost (1/2) -2016-06-13 19:45:10 INFO Executor:59 - Finished task 0.0 in stage 43.0 (TID 29). 2254 bytes result sent to driver -2016-06-13 19:45:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 43.0 (TID 29) in 63 ms on localhost (2/2) -2016-06-13 19:45:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 43.0, whose tasks have all completed, from pool -2016-06-13 19:45:10 INFO DAGScheduler:59 - ShuffleMapStage 43 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.064 s -2016-06-13 19:45:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:10 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:10 INFO DAGScheduler:59 - waiting: Set(ResultStage 45, ShuffleMapStage 44) -2016-06-13 19:45:10 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:10 INFO DAGScheduler:59 - Missing parents for ResultStage 45: List(ShuffleMapStage 44) -2016-06-13 19:45:10 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 44: List() -2016-06-13 19:45:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 44 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[70] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:45:10 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=57021, maxMem=766075207 -2016-06-13 19:45:10 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 19:45:10 INFO MemoryStore:59 - ensureFreeSpace(2280) called with curMem=60917, maxMem=766075207 -2016-06-13 19:45:10 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 19:45:10 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:50000 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 19:45:10 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 44 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[70] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:10 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-06-13 19:45:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 31, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 32, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:10 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 32) -2016-06-13 19:45:10 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 31) -2016-06-13 19:45:10 INFO BlockManager:59 - Found block rdd_17_0 locally -2016-06-13 19:45:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:10 INFO BlockManager:59 - Found block rdd_17_1 locally -2016-06-13 19:45:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:10 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 32). 2470 bytes result sent to driver -2016-06-13 19:45:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 32) in 80 ms on localhost (1/2) -2016-06-13 19:45:10 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 31). 2470 bytes result sent to driver -2016-06-13 19:45:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 31) in 156 ms on localhost (2/2) -2016-06-13 19:45:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-06-13 19:45:10 INFO DAGScheduler:59 - ShuffleMapStage 44 (mapPartitions at GraphImpl.scala:235) finished in 0.160 s -2016-06-13 19:45:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:10 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:10 INFO DAGScheduler:59 - waiting: Set(ResultStage 45) -2016-06-13 19:45:10 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:10 INFO DAGScheduler:59 - Missing parents for ResultStage 45: List() -2016-06-13 19:45:10 INFO DAGScheduler:59 - Submitting ResultStage 45 (MapPartitionsRDD[78] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:45:10 INFO MemoryStore:59 - ensureFreeSpace(7720) called with curMem=63197, maxMem=766075207 -2016-06-13 19:45:10 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 7.5 KB, free 730.5 MB) -2016-06-13 19:45:10 INFO MemoryStore:59 - ensureFreeSpace(3393) called with curMem=70917, maxMem=766075207 -2016-06-13 19:45:10 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.5 MB) -2016-06-13 19:45:10 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:50000 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:45:10 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 45 (MapPartitionsRDD[78] at values at LDAOptimizer.scala:205) -2016-06-13 19:45:10 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-06-13 19:45:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 33, localhost, PROCESS_LOCAL, 2289 bytes) -2016-06-13 19:45:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 34, localhost, PROCESS_LOCAL, 2289 bytes) -2016-06-13 19:45:10 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 34) -2016-06-13 19:45:10 INFO CacheManager:59 - Partition rdd_74_1 not found, computing it -2016-06-13 19:45:10 INFO BlockManager:59 - Found block rdd_61_1 locally -2016-06-13 19:45:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:10 INFO MemoryStore:59 - ensureFreeSpace(1896) called with curMem=74310, maxMem=766075207 -2016-06-13 19:45:10 INFO MemoryStore:59 - Block rdd_74_1 stored as values in memory (estimated size 1896.0 B, free 730.5 MB) -2016-06-13 19:45:10 INFO BlockManagerInfo:59 - Added rdd_74_1 in memory on localhost:50000 (size: 1896.0 B, free: 730.5 MB) -2016-06-13 19:45:10 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 34). 2971 bytes result sent to driver -2016-06-13 19:45:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 34) in 22 ms on localhost (1/2) -2016-06-13 19:45:10 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 33) -2016-06-13 19:45:10 INFO CacheManager:59 - Partition rdd_74_0 not found, computing it -2016-06-13 19:45:10 INFO BlockManager:59 - Found block rdd_61_0 locally -2016-06-13 19:45:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:10 INFO MemoryStore:59 - ensureFreeSpace(1904) called with curMem=76206, maxMem=766075207 -2016-06-13 19:45:10 INFO MemoryStore:59 - Block rdd_74_0 stored as values in memory (estimated size 1904.0 B, free 730.5 MB) -2016-06-13 19:45:10 INFO BlockManagerInfo:59 - Added rdd_74_0 in memory on localhost:50000 (size: 1904.0 B, free: 730.5 MB) -2016-06-13 19:45:10 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 33). 2971 bytes result sent to driver -2016-06-13 19:45:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 33) in 52 ms on localhost (2/2) -2016-06-13 19:45:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-06-13 19:45:10 INFO DAGScheduler:59 - ResultStage 45 (fold at LDAOptimizer.scala:205) finished in 0.053 s -2016-06-13 19:45:10 INFO DAGScheduler:59 - Job 5 finished: fold at LDAOptimizer.scala:205, took 0.399813 s -2016-06-13 19:45:10 INFO MapPartitionsRDD:59 - Removing RDD 48 from persistence list -2016-06-13 19:45:10 INFO ZippedPartitionsRDD2:59 - Removing RDD 55 from persistence list -2016-06-13 19:45:10 INFO BlockManager:59 - Removing RDD 55 -2016-06-13 19:45:10 INFO BlockManager:59 - Removing RDD 48 -2016-06-13 19:45:10 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 154 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 154 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:45:10 INFO DAGScheduler:59 - Registering RDD 79 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:10 INFO DAGScheduler:59 - Registering RDD 83 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:10 INFO DAGScheduler:59 - Got job 6 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:45:10 INFO DAGScheduler:59 - Final stage: ResultStage 60(fold at LDAOptimizer.scala:205) -2016-06-13 19:45:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 51, ShuffleMapStage 49, ShuffleMapStage 46, ShuffleMapStage 53, ShuffleMapStage 57, ShuffleMapStage 55, ShuffleMapStage 47, ShuffleMapStage 59) -2016-06-13 19:45:10 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 59) -2016-06-13 19:45:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 58 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[79] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:45:10 INFO MemoryStore:59 - ensureFreeSpace(6728) called with curMem=74310, maxMem=766075207 -2016-06-13 19:45:10 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 6.6 KB, free 730.5 MB) -2016-06-13 19:45:10 INFO MemoryStore:59 - ensureFreeSpace(3084) called with curMem=81038, maxMem=766075207 -2016-06-13 19:45:10 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.5 MB) -2016-06-13 19:45:10 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:50000 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:45:10 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 58 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[79] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:10 INFO TaskSchedulerImpl:59 - Adding task set 58.0 with 2 tasks -2016-06-13 19:45:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 58.0 (TID 35, localhost, PROCESS_LOCAL, 2278 bytes) -2016-06-13 19:45:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 58.0 (TID 36, localhost, PROCESS_LOCAL, 2278 bytes) -2016-06-13 19:45:10 INFO Executor:59 - Running task 1.0 in stage 58.0 (TID 36) -2016-06-13 19:45:10 INFO BlockManager:59 - Found block rdd_74_1 locally -2016-06-13 19:45:10 INFO Executor:59 - Finished task 1.0 in stage 58.0 (TID 36). 2254 bytes result sent to driver -2016-06-13 19:45:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 58.0 (TID 36) in 29 ms on localhost (1/2) -2016-06-13 19:45:10 INFO Executor:59 - Running task 0.0 in stage 58.0 (TID 35) -2016-06-13 19:45:10 INFO BlockManager:59 - Found block rdd_74_0 locally -2016-06-13 19:45:10 INFO Executor:59 - Finished task 0.0 in stage 58.0 (TID 35). 2254 bytes result sent to driver -2016-06-13 19:45:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 58.0 (TID 35) in 95 ms on localhost (2/2) -2016-06-13 19:45:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 58.0, whose tasks have all completed, from pool -2016-06-13 19:45:10 INFO DAGScheduler:59 - ShuffleMapStage 58 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.096 s -2016-06-13 19:45:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:10 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:10 INFO DAGScheduler:59 - waiting: Set(ResultStage 60, ShuffleMapStage 59) -2016-06-13 19:45:10 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:10 INFO DAGScheduler:59 - Missing parents for ResultStage 60: List(ShuffleMapStage 59) -2016-06-13 19:45:10 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 59: List() -2016-06-13 19:45:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 59 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[83] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:45:10 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=84122, maxMem=766075207 -2016-06-13 19:45:10 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 19:45:10 INFO MemoryStore:59 - ensureFreeSpace(2274) called with curMem=88018, maxMem=766075207 -2016-06-13 19:45:10 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 19:45:10 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:50000 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:45:10 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 59 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[83] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:10 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 2 tasks -2016-06-13 19:45:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 37, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 38, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:10 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 38) -2016-06-13 19:45:10 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 37) -2016-06-13 19:45:10 INFO BlockManager:59 - Found block rdd_17_1 locally -2016-06-13 19:45:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:10 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 38). 2470 bytes result sent to driver -2016-06-13 19:45:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 38) in 30 ms on localhost (1/2) -2016-06-13 19:45:10 INFO BlockManager:59 - Found block rdd_17_0 locally -2016-06-13 19:45:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:10 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 37). 2470 bytes result sent to driver -2016-06-13 19:45:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 37) in 126 ms on localhost (2/2) -2016-06-13 19:45:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-06-13 19:45:10 INFO DAGScheduler:59 - ShuffleMapStage 59 (mapPartitions at GraphImpl.scala:235) finished in 0.131 s -2016-06-13 19:45:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:10 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:10 INFO DAGScheduler:59 - waiting: Set(ResultStage 60) -2016-06-13 19:45:10 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:10 INFO DAGScheduler:59 - Missing parents for ResultStage 60: List() -2016-06-13 19:45:10 INFO DAGScheduler:59 - Submitting ResultStage 60 (MapPartitionsRDD[91] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:45:10 INFO MemoryStore:59 - ensureFreeSpace(8144) called with curMem=90292, maxMem=766075207 -2016-06-13 19:45:10 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 8.0 KB, free 730.5 MB) -2016-06-13 19:45:10 INFO MemoryStore:59 - ensureFreeSpace(3511) called with curMem=98436, maxMem=766075207 -2016-06-13 19:45:10 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.5 MB) -2016-06-13 19:45:10 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:50000 (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:45:10 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 60 (MapPartitionsRDD[91] at values at LDAOptimizer.scala:205) -2016-06-13 19:45:10 INFO TaskSchedulerImpl:59 - Adding task set 60.0 with 2 tasks -2016-06-13 19:45:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 60.0 (TID 39, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:45:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 60.0 (TID 40, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:45:10 INFO Executor:59 - Running task 1.0 in stage 60.0 (TID 40) -2016-06-13 19:45:10 INFO CacheManager:59 - Partition rdd_87_1 not found, computing it -2016-06-13 19:45:10 INFO BlockManager:59 - Found block rdd_74_1 locally -2016-06-13 19:45:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:10 INFO MemoryStore:59 - ensureFreeSpace(1896) called with curMem=101947, maxMem=766075207 -2016-06-13 19:45:10 INFO MemoryStore:59 - Block rdd_87_1 stored as values in memory (estimated size 1896.0 B, free 730.5 MB) -2016-06-13 19:45:10 INFO BlockManagerInfo:59 - Added rdd_87_1 in memory on localhost:50000 (size: 1896.0 B, free: 730.5 MB) -2016-06-13 19:45:10 INFO Executor:59 - Running task 0.0 in stage 60.0 (TID 39) -2016-06-13 19:45:10 INFO Executor:59 - Finished task 1.0 in stage 60.0 (TID 40). 2971 bytes result sent to driver -2016-06-13 19:45:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 60.0 (TID 40) in 21 ms on localhost (1/2) -2016-06-13 19:45:10 INFO CacheManager:59 - Partition rdd_87_0 not found, computing it -2016-06-13 19:45:10 INFO BlockManager:59 - Found block rdd_74_0 locally -2016-06-13 19:45:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:10 INFO MemoryStore:59 - ensureFreeSpace(1904) called with curMem=103843, maxMem=766075207 -2016-06-13 19:45:10 INFO MemoryStore:59 - Block rdd_87_0 stored as values in memory (estimated size 1904.0 B, free 730.5 MB) -2016-06-13 19:45:10 INFO BlockManagerInfo:59 - Added rdd_87_0 in memory on localhost:50000 (size: 1904.0 B, free: 730.5 MB) -2016-06-13 19:45:10 INFO Executor:59 - Finished task 0.0 in stage 60.0 (TID 39). 2971 bytes result sent to driver -2016-06-13 19:45:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 60.0 (TID 39) in 53 ms on localhost (2/2) -2016-06-13 19:45:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 60.0, whose tasks have all completed, from pool -2016-06-13 19:45:10 INFO DAGScheduler:59 - ResultStage 60 (fold at LDAOptimizer.scala:205) finished in 0.054 s -2016-06-13 19:45:10 INFO DAGScheduler:59 - Job 6 finished: fold at LDAOptimizer.scala:205, took 0.376338 s -2016-06-13 19:45:10 INFO SparkContext:59 - Starting job: collect at LDAModel.scala:528 -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 154 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 154 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:45:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:45:10 INFO DAGScheduler:59 - Got job 7 (collect at LDAModel.scala:528) with 2 output partitions -2016-06-13 19:45:10 INFO DAGScheduler:59 - Final stage: ResultStage 75(collect at LDAModel.scala:528) -2016-06-13 19:45:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 66, ShuffleMapStage 70, ShuffleMapStage 64, ShuffleMapStage 74, ShuffleMapStage 68, ShuffleMapStage 72, ShuffleMapStage 61, ShuffleMapStage 62) -2016-06-13 19:45:10 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:10 INFO DAGScheduler:59 - Submitting ResultStage 75 (MapPartitionsRDD[94] at map at LDAModel.scala:526), which has no missing parents -2016-06-13 19:45:10 INFO MemoryStore:59 - ensureFreeSpace(7840) called with curMem=105747, maxMem=766075207 -2016-06-13 19:45:10 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 19:45:10 INFO MemoryStore:59 - ensureFreeSpace(3389) called with curMem=113587, maxMem=766075207 -2016-06-13 19:45:10 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.5 MB) -2016-06-13 19:45:10 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:50000 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:45:10 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 75 (MapPartitionsRDD[94] at map at LDAModel.scala:526) -2016-06-13 19:45:10 INFO TaskSchedulerImpl:59 - Adding task set 75.0 with 2 tasks -2016-06-13 19:45:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 75.0 (TID 41, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:45:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 75.0 (TID 42, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:45:10 INFO Executor:59 - Running task 0.0 in stage 75.0 (TID 41) -2016-06-13 19:45:10 INFO Executor:59 - Running task 1.0 in stage 75.0 (TID 42) -2016-06-13 19:45:10 INFO BlockManager:59 - Found block rdd_87_0 locally -2016-06-13 19:45:10 INFO BlockManager:59 - Found block rdd_87_1 locally -2016-06-13 19:45:10 INFO Executor:59 - Finished task 0.0 in stage 75.0 (TID 41). 2476 bytes result sent to driver -2016-06-13 19:45:10 INFO Executor:59 - Finished task 1.0 in stage 75.0 (TID 42). 2408 bytes result sent to driver -2016-06-13 19:45:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 75.0 (TID 41) in 10 ms on localhost (1/2) -2016-06-13 19:45:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 75.0 (TID 42) in 10 ms on localhost (2/2) -2016-06-13 19:45:10 INFO DAGScheduler:59 - ResultStage 75 (collect at LDAModel.scala:528) finished in 0.011 s -2016-06-13 19:45:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 75.0, whose tasks have all completed, from pool -2016-06-13 19:45:10 INFO DAGScheduler:59 - Job 7 finished: collect at LDAModel.scala:528, took 0.061115 s -2016-06-13 19:45:11 INFO SparkContext:59 - Starting job: reduce at LDAModel.scala:563 -2016-06-13 19:45:11 INFO DAGScheduler:59 - Got job 8 (reduce at LDAModel.scala:563) with 2 output partitions -2016-06-13 19:45:11 INFO DAGScheduler:59 - Final stage: ResultStage 90(reduce at LDAModel.scala:563) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 81, ShuffleMapStage 85, ShuffleMapStage 89, ShuffleMapStage 83, ShuffleMapStage 87, ShuffleMapStage 79, ShuffleMapStage 76, ShuffleMapStage 77) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting ResultStage 90 (MapPartitionsRDD[97] at mapPartitions at LDAModel.scala:549), which has no missing parents -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(8008) called with curMem=116976, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 7.8 KB, free 730.5 MB) -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(3551) called with curMem=124984, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:11 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:50000 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:45:11 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 90 (MapPartitionsRDD[97] at mapPartitions at LDAModel.scala:549) -2016-06-13 19:45:11 INFO TaskSchedulerImpl:59 - Adding task set 90.0 with 2 tasks -2016-06-13 19:45:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 90.0 (TID 43, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:45:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 90.0 (TID 44, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:45:11 INFO Executor:59 - Running task 0.0 in stage 90.0 (TID 43) -2016-06-13 19:45:11 INFO BlockManager:59 - Found block rdd_87_0 locally -2016-06-13 19:45:11 INFO Executor:59 - Finished task 0.0 in stage 90.0 (TID 43). 3042 bytes result sent to driver -2016-06-13 19:45:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 90.0 (TID 43) in 21 ms on localhost (1/2) -2016-06-13 19:45:11 INFO Executor:59 - Running task 1.0 in stage 90.0 (TID 44) -2016-06-13 19:45:11 INFO BlockManager:59 - Found block rdd_87_1 locally -2016-06-13 19:45:11 INFO Executor:59 - Finished task 1.0 in stage 90.0 (TID 44). 2960 bytes result sent to driver -2016-06-13 19:45:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 90.0 (TID 44) in 60 ms on localhost (2/2) -2016-06-13 19:45:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 90.0, whose tasks have all completed, from pool -2016-06-13 19:45:11 INFO DAGScheduler:59 - ResultStage 90 (reduce at LDAModel.scala:563) finished in 0.072 s -2016-06-13 19:45:11 INFO DAGScheduler:59 - Job 8 finished: reduce at LDAModel.scala:563, took 0.112177 s -2016-06-13 19:45:11 INFO SparkContext:59 - Starting job: collect at LDASuite.scala:99 -2016-06-13 19:45:11 INFO DAGScheduler:59 - Got job 9 (collect at LDASuite.scala:99) with 2 output partitions -2016-06-13 19:45:11 INFO DAGScheduler:59 - Final stage: ResultStage 105(collect at LDASuite.scala:99) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 102, ShuffleMapStage 96, ShuffleMapStage 100, ShuffleMapStage 104, ShuffleMapStage 94, ShuffleMapStage 91, ShuffleMapStage 98, ShuffleMapStage 92) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting ResultStage 105 (MapPartitionsRDD[100] at map at LDAModel.scala:728), which has no missing parents -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(7872) called with curMem=128535, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 7.7 KB, free 730.5 MB) -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(3398) called with curMem=136407, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.5 MB) -2016-06-13 19:45:11 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:50000 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:45:11 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 105 (MapPartitionsRDD[100] at map at LDAModel.scala:728) -2016-06-13 19:45:11 INFO TaskSchedulerImpl:59 - Adding task set 105.0 with 2 tasks -2016-06-13 19:45:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 105.0 (TID 45, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:45:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 105.0 (TID 46, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:45:11 INFO Executor:59 - Running task 1.0 in stage 105.0 (TID 46) -2016-06-13 19:45:11 INFO Executor:59 - Running task 0.0 in stage 105.0 (TID 45) -2016-06-13 19:45:11 INFO BlockManager:59 - Found block rdd_87_1 locally -2016-06-13 19:45:11 INFO Executor:59 - Finished task 1.0 in stage 105.0 (TID 46). 2425 bytes result sent to driver -2016-06-13 19:45:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 105.0 (TID 46) in 8 ms on localhost (1/2) -2016-06-13 19:45:11 INFO BlockManager:59 - Found block rdd_87_0 locally -2016-06-13 19:45:11 INFO Executor:59 - Finished task 0.0 in stage 105.0 (TID 45). 2365 bytes result sent to driver -2016-06-13 19:45:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 105.0 (TID 45) in 13 ms on localhost (2/2) -2016-06-13 19:45:11 INFO DAGScheduler:59 - ResultStage 105 (collect at LDASuite.scala:99) finished in 0.013 s -2016-06-13 19:45:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 105.0, whose tasks have all completed, from pool -2016-06-13 19:45:11 INFO DAGScheduler:59 - Job 9 finished: collect at LDASuite.scala:99, took 0.044899 s -2016-06-13 19:45:11 INFO SparkContext:59 - Starting job: collect at LDASuite.scala:114 -2016-06-13 19:45:11 INFO DAGScheduler:59 - Registering RDD 107 (map at LDAModel.scala:728) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Registering RDD 104 (map at LDASuite.scala:113) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Got job 10 (collect at LDASuite.scala:114) with 2 output partitions -2016-06-13 19:45:11 INFO DAGScheduler:59 - Final stage: ResultStage 122(collect at LDASuite.scala:114) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 121, ShuffleMapStage 120) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 121, ShuffleMapStage 120) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting ShuffleMapStage 120 (MapPartitionsRDD[107] at map at LDAModel.scala:728), which has no missing parents -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(7632) called with curMem=139805, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 7.5 KB, free 730.4 MB) -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(3317) called with curMem=147437, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:45:11 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:50000 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:45:11 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 120 (MapPartitionsRDD[107] at map at LDAModel.scala:728) -2016-06-13 19:45:11 INFO TaskSchedulerImpl:59 - Adding task set 120.0 with 2 tasks -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting ShuffleMapStage 121 (MapPartitionsRDD[104] at map at LDASuite.scala:113), which has no missing parents -2016-06-13 19:45:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 120.0 (TID 47, localhost, PROCESS_LOCAL, 2319 bytes) -2016-06-13 19:45:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 120.0 (TID 48, localhost, PROCESS_LOCAL, 2319 bytes) -2016-06-13 19:45:11 INFO Executor:59 - Running task 0.0 in stage 120.0 (TID 47) -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(7864) called with curMem=150754, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 7.7 KB, free 730.4 MB) -2016-06-13 19:45:11 INFO Executor:59 - Running task 1.0 in stage 120.0 (TID 48) -2016-06-13 19:45:11 INFO BlockManager:59 - Found block rdd_87_0 locally -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(3451) called with curMem=158618, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.4 MB) -2016-06-13 19:45:11 INFO BlockManager:59 - Found block rdd_87_1 locally -2016-06-13 19:45:11 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:50000 (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:45:11 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 121 (MapPartitionsRDD[104] at map at LDASuite.scala:113) -2016-06-13 19:45:11 INFO TaskSchedulerImpl:59 - Adding task set 121.0 with 2 tasks -2016-06-13 19:45:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 121.0 (TID 49, localhost, PROCESS_LOCAL, 2319 bytes) -2016-06-13 19:45:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 121.0 (TID 50, localhost, PROCESS_LOCAL, 2319 bytes) -2016-06-13 19:45:11 INFO Executor:59 - Running task 1.0 in stage 121.0 (TID 50) -2016-06-13 19:45:11 INFO Executor:59 - Finished task 0.0 in stage 120.0 (TID 47). 2254 bytes result sent to driver -2016-06-13 19:45:11 INFO Executor:59 - Running task 0.0 in stage 121.0 (TID 49) -2016-06-13 19:45:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 120.0 (TID 47) in 32 ms on localhost (1/2) -2016-06-13 19:45:11 INFO BlockManager:59 - Found block rdd_87_0 locally -2016-06-13 19:45:11 INFO Executor:59 - Finished task 1.0 in stage 120.0 (TID 48). 2254 bytes result sent to driver -2016-06-13 19:45:11 INFO BlockManager:59 - Found block rdd_87_1 locally -2016-06-13 19:45:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 120.0 (TID 48) in 43 ms on localhost (2/2) -2016-06-13 19:45:11 INFO DAGScheduler:59 - ShuffleMapStage 120 (map at LDAModel.scala:728) finished in 0.044 s -2016-06-13 19:45:11 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:11 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 121) -2016-06-13 19:45:11 INFO DAGScheduler:59 - waiting: Set(ResultStage 122) -2016-06-13 19:45:11 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 120.0, whose tasks have all completed, from pool -2016-06-13 19:45:11 INFO DAGScheduler:59 - Missing parents for ResultStage 122: List(ShuffleMapStage 121) -2016-06-13 19:45:11 INFO Executor:59 - Finished task 1.0 in stage 121.0 (TID 50). 2254 bytes result sent to driver -2016-06-13 19:45:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 121.0 (TID 50) in 160 ms on localhost (1/2) -2016-06-13 19:45:11 INFO Executor:59 - Finished task 0.0 in stage 121.0 (TID 49). 2254 bytes result sent to driver -2016-06-13 19:45:11 INFO DAGScheduler:59 - ShuffleMapStage 121 (map at LDASuite.scala:113) finished in 0.166 s -2016-06-13 19:45:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 121.0 (TID 49) in 164 ms on localhost (2/2) -2016-06-13 19:45:11 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:11 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:11 INFO DAGScheduler:59 - waiting: Set(ResultStage 122) -2016-06-13 19:45:11 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:11 INFO DAGScheduler:59 - Missing parents for ResultStage 122: List() -2016-06-13 19:45:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 121.0, whose tasks have all completed, from pool -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting ResultStage 122 (MapPartitionsRDD[110] at join at LDASuite.scala:114), which is now runnable -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(2816) called with curMem=162069, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 2.8 KB, free 730.4 MB) -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(1563) called with curMem=164885, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 1563.0 B, free 730.4 MB) -2016-06-13 19:45:11 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:50000 (size: 1563.0 B, free: 730.5 MB) -2016-06-13 19:45:11 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 122 (MapPartitionsRDD[110] at join at LDASuite.scala:114) -2016-06-13 19:45:11 INFO TaskSchedulerImpl:59 - Adding task set 122.0 with 2 tasks -2016-06-13 19:45:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 122.0 (TID 51, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 19:45:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 122.0 (TID 52, localhost, PROCESS_LOCAL, 1974 bytes) -2016-06-13 19:45:11 INFO Executor:59 - Running task 0.0 in stage 122.0 (TID 51) -2016-06-13 19:45:11 INFO Executor:59 - Running task 1.0 in stage 122.0 (TID 52) -2016-06-13 19:45:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:11 INFO Executor:59 - Finished task 1.0 in stage 122.0 (TID 52). 1727 bytes result sent to driver -2016-06-13 19:45:11 INFO Executor:59 - Finished task 0.0 in stage 122.0 (TID 51). 1611 bytes result sent to driver -2016-06-13 19:45:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 122.0 (TID 52) in 27 ms on localhost (1/2) -2016-06-13 19:45:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 122.0 (TID 51) in 30 ms on localhost (2/2) -2016-06-13 19:45:11 INFO DAGScheduler:59 - ResultStage 122 (collect at LDASuite.scala:114) finished in 0.030 s -2016-06-13 19:45:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 122.0, whose tasks have all completed, from pool -2016-06-13 19:45:11 INFO DAGScheduler:59 - Job 10 finished: collect at LDASuite.scala:114, took 0.260088 s -2016-06-13 19:45:11 INFO SparkContext:59 - Starting job: fold at LDAModel.scala:687 -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 154 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 154 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:45:11 INFO DAGScheduler:59 - Registering RDD 111 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Registering RDD 115 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Got job 11 (fold at LDAModel.scala:687) with 2 output partitions -2016-06-13 19:45:11 INFO DAGScheduler:59 - Final stage: ResultStage 139(fold at LDAModel.scala:687) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 132, ShuffleMapStage 136, ShuffleMapStage 126, ShuffleMapStage 123, ShuffleMapStage 138, ShuffleMapStage 130, ShuffleMapStage 124, ShuffleMapStage 134, ShuffleMapStage 128) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 138) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting ShuffleMapStage 137 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[111] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(7160) called with curMem=166448, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 7.0 KB, free 730.4 MB) -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(3186) called with curMem=173608, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.4 MB) -2016-06-13 19:45:11 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:50000 (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:45:11 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 137 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[111] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:11 INFO TaskSchedulerImpl:59 - Adding task set 137.0 with 2 tasks -2016-06-13 19:45:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 137.0 (TID 53, localhost, PROCESS_LOCAL, 2319 bytes) -2016-06-13 19:45:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 137.0 (TID 54, localhost, PROCESS_LOCAL, 2319 bytes) -2016-06-13 19:45:11 INFO Executor:59 - Running task 1.0 in stage 137.0 (TID 54) -2016-06-13 19:45:11 INFO Executor:59 - Running task 0.0 in stage 137.0 (TID 53) -2016-06-13 19:45:11 INFO BlockManager:59 - Found block rdd_87_1 locally -2016-06-13 19:45:11 INFO BlockManager:59 - Found block rdd_87_0 locally -2016-06-13 19:45:11 INFO Executor:59 - Finished task 1.0 in stage 137.0 (TID 54). 2254 bytes result sent to driver -2016-06-13 19:45:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 137.0 (TID 54) in 30 ms on localhost (1/2) -2016-06-13 19:45:11 INFO Executor:59 - Finished task 0.0 in stage 137.0 (TID 53). 2254 bytes result sent to driver -2016-06-13 19:45:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 137.0 (TID 53) in 36 ms on localhost (2/2) -2016-06-13 19:45:11 INFO DAGScheduler:59 - ShuffleMapStage 137 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.034 s -2016-06-13 19:45:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 137.0, whose tasks have all completed, from pool -2016-06-13 19:45:11 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:11 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:11 INFO DAGScheduler:59 - waiting: Set(ResultStage 139, ShuffleMapStage 138) -2016-06-13 19:45:11 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:11 INFO DAGScheduler:59 - Missing parents for ResultStage 139: List(ShuffleMapStage 138) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 138: List() -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting ShuffleMapStage 138 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[115] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(3920) called with curMem=176794, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(2296) called with curMem=180714, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 19:45:11 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:50000 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:45:11 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 138 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[115] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:11 INFO TaskSchedulerImpl:59 - Adding task set 138.0 with 2 tasks -2016-06-13 19:45:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 138.0 (TID 55, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 138.0 (TID 56, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:11 INFO Executor:59 - Running task 0.0 in stage 138.0 (TID 55) -2016-06-13 19:45:11 INFO Executor:59 - Running task 1.0 in stage 138.0 (TID 56) -2016-06-13 19:45:11 INFO BlockManager:59 - Found block rdd_17_0 locally -2016-06-13 19:45:11 INFO BlockManager:59 - Found block rdd_17_1 locally -2016-06-13 19:45:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:11 INFO Executor:59 - Finished task 0.0 in stage 138.0 (TID 55). 2470 bytes result sent to driver -2016-06-13 19:45:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 138.0 (TID 55) in 59 ms on localhost (1/2) -2016-06-13 19:45:11 INFO Executor:59 - Finished task 1.0 in stage 138.0 (TID 56). 2470 bytes result sent to driver -2016-06-13 19:45:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 138.0 (TID 56) in 63 ms on localhost (2/2) -2016-06-13 19:45:11 INFO DAGScheduler:59 - ShuffleMapStage 138 (mapPartitions at GraphImpl.scala:235) finished in 0.065 s -2016-06-13 19:45:11 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:11 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:11 INFO DAGScheduler:59 - waiting: Set(ResultStage 139) -2016-06-13 19:45:11 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:11 INFO DAGScheduler:59 - Missing parents for ResultStage 139: List() -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting ResultStage 139 (MapPartitionsRDD[119] at map at LDAModel.scala:687), which is now runnable -2016-06-13 19:45:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 138.0, whose tasks have all completed, from pool -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(8256) called with curMem=183010, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 8.1 KB, free 730.4 MB) -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(3594) called with curMem=191266, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:11 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:50000 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:45:11 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 139 (MapPartitionsRDD[119] at map at LDAModel.scala:687) -2016-06-13 19:45:11 INFO TaskSchedulerImpl:59 - Adding task set 139.0 with 2 tasks -2016-06-13 19:45:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 139.0 (TID 57, localhost, PROCESS_LOCAL, 2371 bytes) -2016-06-13 19:45:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 139.0 (TID 58, localhost, PROCESS_LOCAL, 2371 bytes) -2016-06-13 19:45:11 INFO Executor:59 - Running task 1.0 in stage 139.0 (TID 58) -2016-06-13 19:45:11 INFO Executor:59 - Running task 0.0 in stage 139.0 (TID 57) -2016-06-13 19:45:11 INFO BlockManager:59 - Found block rdd_87_1 locally -2016-06-13 19:45:11 INFO BlockManager:59 - Found block rdd_87_0 locally -2016-06-13 19:45:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:11 INFO Executor:59 - Finished task 1.0 in stage 139.0 (TID 58). 2300 bytes result sent to driver -2016-06-13 19:45:11 INFO Executor:59 - Finished task 0.0 in stage 139.0 (TID 57). 2300 bytes result sent to driver -2016-06-13 19:45:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 139.0 (TID 58) in 11 ms on localhost (1/2) -2016-06-13 19:45:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 139.0 (TID 57) in 13 ms on localhost (2/2) -2016-06-13 19:45:11 INFO DAGScheduler:59 - ResultStage 139 (fold at LDAModel.scala:687) finished in 0.013 s -2016-06-13 19:45:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 139.0, whose tasks have all completed, from pool -2016-06-13 19:45:11 INFO DAGScheduler:59 - Job 11 finished: fold at LDAModel.scala:687, took 0.189137 s -2016-06-13 19:45:11 INFO SparkContext:59 - Starting job: aggregate at LDAModel.scala:717 -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 154 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 154 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:45:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:45:11 INFO DAGScheduler:59 - Got job 12 (aggregate at LDAModel.scala:717) with 2 output partitions -2016-06-13 19:45:11 INFO DAGScheduler:59 - Final stage: ResultStage 154(aggregate at LDAModel.scala:717) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 143, ShuffleMapStage 153, ShuffleMapStage 147, ShuffleMapStage 151, ShuffleMapStage 140, ShuffleMapStage 141, ShuffleMapStage 145, ShuffleMapStage 149) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting ResultStage 154 (VertexRDDImpl[88] at RDD at VertexRDD.scala:57), which has no missing parents -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(8048) called with curMem=194860, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 7.9 KB, free 730.4 MB) -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(3659) called with curMem=202908, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 3.6 KB, free 730.4 MB) -2016-06-13 19:45:11 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:50000 (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:45:11 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 154 (VertexRDDImpl[88] at RDD at VertexRDD.scala:57) -2016-06-13 19:45:11 INFO TaskSchedulerImpl:59 - Adding task set 154.0 with 2 tasks -2016-06-13 19:45:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 154.0 (TID 59, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:45:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 154.0 (TID 60, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:45:11 INFO Executor:59 - Running task 0.0 in stage 154.0 (TID 59) -2016-06-13 19:45:11 INFO Executor:59 - Running task 1.0 in stage 154.0 (TID 60) -2016-06-13 19:45:11 INFO BlockManager:59 - Found block rdd_87_0 locally -2016-06-13 19:45:11 INFO BlockManager:59 - Found block rdd_87_1 locally -2016-06-13 19:45:11 INFO Executor:59 - Finished task 1.0 in stage 154.0 (TID 60). 2084 bytes result sent to driver -2016-06-13 19:45:11 INFO Executor:59 - Finished task 0.0 in stage 154.0 (TID 59). 2084 bytes result sent to driver -2016-06-13 19:45:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 154.0 (TID 60) in 8 ms on localhost (1/2) -2016-06-13 19:45:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 154.0 (TID 59) in 9 ms on localhost (2/2) -2016-06-13 19:45:11 INFO DAGScheduler:59 - ResultStage 154 (aggregate at LDAModel.scala:717) finished in 0.010 s -2016-06-13 19:45:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 154.0, whose tasks have all completed, from pool -2016-06-13 19:45:11 INFO DAGScheduler:59 - Job 12 finished: aggregate at LDAModel.scala:717, took 0.039164 s -2016-06-13 19:45:11 INFO SparkContext:59 - Starting job: treeReduce at LDAModel.scala:598 -2016-06-13 19:45:11 INFO DAGScheduler:59 - Got job 13 (treeReduce at LDAModel.scala:598) with 2 output partitions -2016-06-13 19:45:11 INFO DAGScheduler:59 - Final stage: ResultStage 169(treeReduce at LDAModel.scala:598) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 168, ShuffleMapStage 166, ShuffleMapStage 158, ShuffleMapStage 155, ShuffleMapStage 162, ShuffleMapStage 156, ShuffleMapStage 160, ShuffleMapStage 164) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting ResultStage 169 (MapPartitionsRDD[125] at treeReduce at LDAModel.scala:598), which has no missing parents -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(9016) called with curMem=206567, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 8.8 KB, free 730.4 MB) -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(3785) called with curMem=215583, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.7 KB, free 730.4 MB) -2016-06-13 19:45:11 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:50000 (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:45:11 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 169 (MapPartitionsRDD[125] at treeReduce at LDAModel.scala:598) -2016-06-13 19:45:11 INFO TaskSchedulerImpl:59 - Adding task set 169.0 with 2 tasks -2016-06-13 19:45:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 169.0 (TID 61, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:45:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 169.0 (TID 62, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:45:11 INFO Executor:59 - Running task 0.0 in stage 169.0 (TID 61) -2016-06-13 19:45:11 INFO Executor:59 - Running task 1.0 in stage 169.0 (TID 62) -2016-06-13 19:45:11 INFO BlockManager:59 - Found block rdd_87_1 locally -2016-06-13 19:45:11 INFO BlockManager:59 - Found block rdd_87_0 locally -2016-06-13 19:45:11 INFO Executor:59 - Finished task 0.0 in stage 169.0 (TID 61). 2972 bytes result sent to driver -2016-06-13 19:45:11 INFO Executor:59 - Finished task 1.0 in stage 169.0 (TID 62). 3058 bytes result sent to driver -2016-06-13 19:45:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 169.0 (TID 61) in 12 ms on localhost (1/2) -2016-06-13 19:45:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 169.0 (TID 62) in 13 ms on localhost (2/2) -2016-06-13 19:45:11 INFO DAGScheduler:59 - ResultStage 169 (treeReduce at LDAModel.scala:598) finished in 0.014 s -2016-06-13 19:45:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 169.0, whose tasks have all completed, from pool -2016-06-13 19:45:11 INFO DAGScheduler:59 - Job 13 finished: treeReduce at LDAModel.scala:598, took 0.043789 s -2016-06-13 19:45:11 INFO SparkContext:59 - Starting job: treeReduce at LDAModel.scala:598 -2016-06-13 19:45:11 INFO DAGScheduler:59 - Got job 14 (treeReduce at LDAModel.scala:598) with 2 output partitions -2016-06-13 19:45:11 INFO DAGScheduler:59 - Final stage: ResultStage 184(treeReduce at LDAModel.scala:598) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 179, ShuffleMapStage 171, ShuffleMapStage 183, ShuffleMapStage 173, ShuffleMapStage 170, ShuffleMapStage 177, ShuffleMapStage 181, ShuffleMapStage 175) -2016-06-13 19:45:11 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:11 INFO DAGScheduler:59 - Submitting ResultStage 184 (MapPartitionsRDD[131] at treeReduce at LDAModel.scala:598), which has no missing parents -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(9016) called with curMem=219368, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 8.8 KB, free 730.4 MB) -2016-06-13 19:45:11 INFO MemoryStore:59 - ensureFreeSpace(3783) called with curMem=228384, maxMem=766075207 -2016-06-13 19:45:11 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 3.7 KB, free 730.4 MB) -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:50000 (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:45:12 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 184 (MapPartitionsRDD[131] at treeReduce at LDAModel.scala:598) -2016-06-13 19:45:12 INFO TaskSchedulerImpl:59 - Adding task set 184.0 with 2 tasks -2016-06-13 19:45:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 184.0 (TID 63, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:45:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 184.0 (TID 64, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:45:12 INFO Executor:59 - Running task 1.0 in stage 184.0 (TID 64) -2016-06-13 19:45:12 INFO Executor:59 - Running task 0.0 in stage 184.0 (TID 63) -2016-06-13 19:45:12 INFO BlockManager:59 - Found block rdd_87_1 locally -2016-06-13 19:45:12 INFO BlockManager:59 - Found block rdd_87_0 locally -2016-06-13 19:45:12 INFO Executor:59 - Finished task 1.0 in stage 184.0 (TID 64). 3058 bytes result sent to driver -2016-06-13 19:45:12 INFO Executor:59 - Finished task 0.0 in stage 184.0 (TID 63). 2972 bytes result sent to driver -2016-06-13 19:45:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 184.0 (TID 64) in 9 ms on localhost (1/2) -2016-06-13 19:45:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 184.0 (TID 63) in 12 ms on localhost (2/2) -2016-06-13 19:45:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 184.0, whose tasks have all completed, from pool -2016-06-13 19:45:12 INFO DAGScheduler:59 - ResultStage 184 (treeReduce at LDAModel.scala:598) finished in 0.013 s -2016-06-13 19:45:12 INFO DAGScheduler:59 - Job 14 finished: treeReduce at LDAModel.scala:598, took 0.047831 s -2016-06-13 19:45:12 INFO SparkContext:59 - Starting job: collect at LDASuite.scala:153 -2016-06-13 19:45:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:45:12 INFO DAGScheduler:59 - Registering RDD 132 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:12 INFO DAGScheduler:59 - Got job 15 (collect at LDASuite.scala:153) with 2 output partitions -2016-06-13 19:45:12 INFO DAGScheduler:59 - Final stage: ResultStage 201(collect at LDASuite.scala:153) -2016-06-13 19:45:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 186, ShuffleMapStage 198, ShuffleMapStage 190, ShuffleMapStage 194, ShuffleMapStage 188, ShuffleMapStage 192, ShuffleMapStage 196, ShuffleMapStage 185, ShuffleMapStage 200) -2016-06-13 19:45:12 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 200) -2016-06-13 19:45:12 INFO DAGScheduler:59 - Submitting ShuffleMapStage 200 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[132] at mapPartitions at GraphImpl.scala:235), which has no missing parents -2016-06-13 19:45:12 INFO MemoryStore:59 - ensureFreeSpace(3912) called with curMem=232167, maxMem=766075207 -2016-06-13 19:45:12 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 19:45:12 INFO MemoryStore:59 - ensureFreeSpace(2278) called with curMem=236079, maxMem=766075207 -2016-06-13 19:45:12 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 19:45:12 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:50000 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:45:12 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 200 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[132] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:12 INFO TaskSchedulerImpl:59 - Adding task set 200.0 with 2 tasks -2016-06-13 19:45:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 200.0 (TID 65, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 200.0 (TID 66, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:12 INFO Executor:59 - Running task 0.0 in stage 200.0 (TID 65) -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:50000 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:45:12 INFO Executor:59 - Running task 1.0 in stage 200.0 (TID 66) -2016-06-13 19:45:12 INFO BlockManager:59 - Found block rdd_17_0 locally -2016-06-13 19:45:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:12 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 19:45:12 INFO BlockManager:59 - Found block rdd_17_1 locally -2016-06-13 19:45:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:50000 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:50000 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:50000 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:50000 in memory (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:50000 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:45:12 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:50000 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:45:12 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:50000 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:45:12 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:50000 in memory (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:45:12 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:50000 in memory (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:45:12 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:50000 in memory (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:45:12 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:50000 in memory (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:45:12 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:50000 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:45:12 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:50000 in memory (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:45:12 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-06-13 19:45:12 INFO ContextCleaner:59 - Cleaned shuffle 16 -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:50000 in memory (size: 1563.0 B, free: 730.5 MB) -2016-06-13 19:45:12 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-06-13 19:45:12 INFO Executor:59 - Finished task 0.0 in stage 200.0 (TID 65). 2470 bytes result sent to driver -2016-06-13 19:45:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 200.0 (TID 65) in 152 ms on localhost (1/2) -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:50000 in memory (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:45:12 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:50000 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:45:12 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-06-13 19:45:12 INFO ContextCleaner:59 - Cleaned shuffle 15 -2016-06-13 19:45:12 INFO ContextCleaner:59 - Cleaned shuffle 14 -2016-06-13 19:45:12 INFO Executor:59 - Finished task 1.0 in stage 200.0 (TID 66). 2470 bytes result sent to driver -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:50000 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:45:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 200.0 (TID 66) in 171 ms on localhost (2/2) -2016-06-13 19:45:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 200.0, whose tasks have all completed, from pool -2016-06-13 19:45:12 INFO DAGScheduler:59 - ShuffleMapStage 200 (mapPartitions at GraphImpl.scala:235) finished in 0.173 s -2016-06-13 19:45:12 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:12 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:12 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 19:45:12 INFO DAGScheduler:59 - waiting: Set(ResultStage 201) -2016-06-13 19:45:12 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:12 INFO DAGScheduler:59 - Missing parents for ResultStage 201: List() -2016-06-13 19:45:12 INFO DAGScheduler:59 - Submitting ResultStage 201 (MapPartitionsRDD[138] at map at LDAModel.scala:641), which is now runnable -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:50000 in memory (size: 3.5 KB, free: 730.6 MB) -2016-06-13 19:45:12 INFO MemoryStore:59 - ensureFreeSpace(8296) called with curMem=63130, maxMem=766075207 -2016-06-13 19:45:12 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 8.1 KB, free 730.5 MB) -2016-06-13 19:45:12 INFO MemoryStore:59 - ensureFreeSpace(3530) called with curMem=63418, maxMem=766075207 -2016-06-13 19:45:12 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.5 MB) -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:50000 (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:45:12 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 19:45:12 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 201 (MapPartitionsRDD[138] at map at LDAModel.scala:641) -2016-06-13 19:45:12 INFO TaskSchedulerImpl:59 - Adding task set 201.0 with 2 tasks -2016-06-13 19:45:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 201.0 (TID 67, localhost, PROCESS_LOCAL, 2371 bytes) -2016-06-13 19:45:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 201.0 (TID 68, localhost, PROCESS_LOCAL, 2371 bytes) -2016-06-13 19:45:12 INFO Executor:59 - Running task 0.0 in stage 201.0 (TID 67) -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:50000 in memory (size: 3.3 KB, free: 730.6 MB) -2016-06-13 19:45:12 INFO Executor:59 - Running task 1.0 in stage 201.0 (TID 68) -2016-06-13 19:45:12 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 19:45:12 INFO BlockManager:59 - Found block rdd_87_0 locally -2016-06-13 19:45:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:12 INFO BlockManager:59 - Found block rdd_87_1 locally -2016-06-13 19:45:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:50000 in memory (size: 3.4 KB, free: 730.6 MB) -2016-06-13 19:45:12 INFO Executor:59 - Finished task 0.0 in stage 201.0 (TID 67). 2549 bytes result sent to driver -2016-06-13 19:45:12 INFO Executor:59 - Finished task 1.0 in stage 201.0 (TID 68). 2629 bytes result sent to driver -2016-06-13 19:45:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 201.0 (TID 67) in 13 ms on localhost (1/2) -2016-06-13 19:45:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 201.0 (TID 68) in 13 ms on localhost (2/2) -2016-06-13 19:45:12 INFO DAGScheduler:59 - ResultStage 201 (collect at LDASuite.scala:153) finished in 0.014 s -2016-06-13 19:45:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 201.0, whose tasks have all completed, from pool -2016-06-13 19:45:12 INFO DAGScheduler:59 - Job 15 finished: collect at LDASuite.scala:153, took 0.252830 s -2016-06-13 19:45:12 INFO SparkContext:59 - Starting job: count at LDAOptimizer.scala:387 -2016-06-13 19:45:12 INFO DAGScheduler:59 - Got job 16 (count at LDAOptimizer.scala:387) with 2 output partitions -2016-06-13 19:45:12 INFO DAGScheduler:59 - Final stage: ResultStage 202(count at LDAOptimizer.scala:387) -2016-06-13 19:45:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:12 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:12 INFO DAGScheduler:59 - Submitting ResultStage 202 (ParallelCollectionRDD[141] at parallelize at LDASuite.scala:206), which has no missing parents -2016-06-13 19:45:12 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=44064, maxMem=766075207 -2016-06-13 19:45:12 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 1032.0 B, free 730.5 MB) -2016-06-13 19:45:12 INFO MemoryStore:59 - ensureFreeSpace(757) called with curMem=45096, maxMem=766075207 -2016-06-13 19:45:12 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 757.0 B, free 730.5 MB) -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:50000 (size: 757.0 B, free: 730.6 MB) -2016-06-13 19:45:12 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 202 (ParallelCollectionRDD[141] at parallelize at LDASuite.scala:206) -2016-06-13 19:45:12 INFO TaskSchedulerImpl:59 - Adding task set 202.0 with 2 tasks -2016-06-13 19:45:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 202.0 (TID 69, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 202.0 (TID 70, localhost, PROCESS_LOCAL, 2509 bytes) -2016-06-13 19:45:12 INFO Executor:59 - Running task 0.0 in stage 202.0 (TID 69) -2016-06-13 19:45:12 INFO Executor:59 - Finished task 0.0 in stage 202.0 (TID 69). 953 bytes result sent to driver -2016-06-13 19:45:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 202.0 (TID 69) in 6 ms on localhost (1/2) -2016-06-13 19:45:12 INFO Executor:59 - Running task 1.0 in stage 202.0 (TID 70) -2016-06-13 19:45:12 INFO Executor:59 - Finished task 1.0 in stage 202.0 (TID 70). 953 bytes result sent to driver -2016-06-13 19:45:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 202.0 (TID 70) in 9 ms on localhost (2/2) -2016-06-13 19:45:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 202.0, whose tasks have all completed, from pool -2016-06-13 19:45:12 INFO DAGScheduler:59 - ResultStage 202 (count at LDAOptimizer.scala:387) finished in 0.012 s -2016-06-13 19:45:12 INFO DAGScheduler:59 - Job 16 finished: count at LDAOptimizer.scala:387, took 0.026926 s -2016-06-13 19:45:12 INFO SparkContext:59 - Starting job: first at LDAOptimizer.scala:388 -2016-06-13 19:45:12 INFO DAGScheduler:59 - Got job 17 (first at LDAOptimizer.scala:388) with 1 output partitions -2016-06-13 19:45:12 INFO DAGScheduler:59 - Final stage: ResultStage 203(first at LDAOptimizer.scala:388) -2016-06-13 19:45:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:12 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:12 INFO DAGScheduler:59 - Submitting ResultStage 203 (ParallelCollectionRDD[141] at parallelize at LDASuite.scala:206), which has no missing parents -2016-06-13 19:45:12 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=45853, maxMem=766075207 -2016-06-13 19:45:12 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 1224.0 B, free 730.5 MB) -2016-06-13 19:45:12 INFO MemoryStore:59 - ensureFreeSpace(806) called with curMem=47077, maxMem=766075207 -2016-06-13 19:45:12 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 806.0 B, free 730.5 MB) -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:50000 (size: 806.0 B, free: 730.6 MB) -2016-06-13 19:45:12 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:12 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 203 (ParallelCollectionRDD[141] at parallelize at LDASuite.scala:206) -2016-06-13 19:45:12 INFO TaskSchedulerImpl:59 - Adding task set 203.0 with 1 tasks -2016-06-13 19:45:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 203.0 (TID 71, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:12 INFO Executor:59 - Running task 0.0 in stage 203.0 (TID 71) -2016-06-13 19:45:12 INFO Executor:59 - Finished task 0.0 in stage 203.0 (TID 71). 1192 bytes result sent to driver -2016-06-13 19:45:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 203.0 (TID 71) in 5 ms on localhost (1/1) -2016-06-13 19:45:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 203.0, whose tasks have all completed, from pool -2016-06-13 19:45:12 INFO DAGScheduler:59 - ResultStage 203 (first at LDAOptimizer.scala:388) finished in 0.007 s -2016-06-13 19:45:12 INFO DAGScheduler:59 - Job 17 finished: first at LDAOptimizer.scala:388, took 0.020072 s -2016-06-13 19:45:12 INFO SparkContext:59 - Starting job: count at LDAOptimizer.scala:387 -2016-06-13 19:45:12 INFO DAGScheduler:59 - Got job 18 (count at LDAOptimizer.scala:387) with 2 output partitions -2016-06-13 19:45:12 INFO DAGScheduler:59 - Final stage: ResultStage 204(count at LDAOptimizer.scala:387) -2016-06-13 19:45:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:12 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:12 INFO DAGScheduler:59 - Submitting ResultStage 204 (ParallelCollectionRDD[142] at parallelize at LDASuite.scala:226), which has no missing parents -2016-06-13 19:45:12 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=47883, maxMem=766075207 -2016-06-13 19:45:12 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 1032.0 B, free 730.5 MB) -2016-06-13 19:45:12 INFO MemoryStore:59 - ensureFreeSpace(757) called with curMem=48915, maxMem=766075207 -2016-06-13 19:45:12 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 757.0 B, free 730.5 MB) -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:50000 (size: 757.0 B, free: 730.6 MB) -2016-06-13 19:45:12 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 204 (ParallelCollectionRDD[142] at parallelize at LDASuite.scala:226) -2016-06-13 19:45:12 INFO TaskSchedulerImpl:59 - Adding task set 204.0 with 2 tasks -2016-06-13 19:45:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 204.0 (TID 72, localhost, PROCESS_LOCAL, 2318 bytes) -2016-06-13 19:45:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 204.0 (TID 73, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 19:45:12 INFO Executor:59 - Running task 0.0 in stage 204.0 (TID 72) -2016-06-13 19:45:12 INFO Executor:59 - Running task 1.0 in stage 204.0 (TID 73) -2016-06-13 19:45:12 INFO Executor:59 - Finished task 1.0 in stage 204.0 (TID 73). 953 bytes result sent to driver -2016-06-13 19:45:12 INFO Executor:59 - Finished task 0.0 in stage 204.0 (TID 72). 953 bytes result sent to driver -2016-06-13 19:45:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 204.0 (TID 73) in 3 ms on localhost (1/2) -2016-06-13 19:45:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 204.0 (TID 72) in 6 ms on localhost (2/2) -2016-06-13 19:45:12 INFO DAGScheduler:59 - ResultStage 204 (count at LDAOptimizer.scala:387) finished in 0.007 s -2016-06-13 19:45:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 204.0, whose tasks have all completed, from pool -2016-06-13 19:45:12 INFO DAGScheduler:59 - Job 18 finished: count at LDAOptimizer.scala:387, took 0.017144 s -2016-06-13 19:45:12 INFO SparkContext:59 - Starting job: first at LDAOptimizer.scala:388 -2016-06-13 19:45:12 INFO DAGScheduler:59 - Got job 19 (first at LDAOptimizer.scala:388) with 1 output partitions -2016-06-13 19:45:12 INFO DAGScheduler:59 - Final stage: ResultStage 205(first at LDAOptimizer.scala:388) -2016-06-13 19:45:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:12 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:12 INFO DAGScheduler:59 - Submitting ResultStage 205 (ParallelCollectionRDD[142] at parallelize at LDASuite.scala:226), which has no missing parents -2016-06-13 19:45:12 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=49672, maxMem=766075207 -2016-06-13 19:45:12 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 1224.0 B, free 730.5 MB) -2016-06-13 19:45:12 INFO MemoryStore:59 - ensureFreeSpace(806) called with curMem=50896, maxMem=766075207 -2016-06-13 19:45:12 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 806.0 B, free 730.5 MB) -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:50000 (size: 806.0 B, free: 730.6 MB) -2016-06-13 19:45:12 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:12 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 205 (ParallelCollectionRDD[142] at parallelize at LDASuite.scala:226) -2016-06-13 19:45:12 INFO TaskSchedulerImpl:59 - Adding task set 205.0 with 1 tasks -2016-06-13 19:45:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 205.0 (TID 74, localhost, PROCESS_LOCAL, 2318 bytes) -2016-06-13 19:45:12 INFO Executor:59 - Running task 0.0 in stage 205.0 (TID 74) -2016-06-13 19:45:12 INFO Executor:59 - Finished task 0.0 in stage 205.0 (TID 74). 1238 bytes result sent to driver -2016-06-13 19:45:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 205.0 (TID 74) in 6 ms on localhost (1/1) -2016-06-13 19:45:12 INFO DAGScheduler:59 - ResultStage 205 (first at LDAOptimizer.scala:388) finished in 0.006 s -2016-06-13 19:45:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 205.0, whose tasks have all completed, from pool -2016-06-13 19:45:12 INFO DAGScheduler:59 - Job 19 finished: first at LDAOptimizer.scala:388, took 0.016924 s -2016-06-13 19:45:12 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=51702, maxMem=766075207 -2016-06-13 19:45:12 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:12 INFO MemoryStore:59 - ensureFreeSpace(255) called with curMem=51878, maxMem=766075207 -2016-06-13 19:45:12 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 255.0 B, free 730.5 MB) -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:50000 (size: 255.0 B, free: 730.6 MB) -2016-06-13 19:45:12 INFO SparkContext:59 - Created broadcast 39 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:12 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:12 INFO DAGScheduler:59 - Got job 20 (reduce at LDAOptimizer.scala:453) with 2 output partitions -2016-06-13 19:45:12 INFO DAGScheduler:59 - Final stage: ResultStage 206(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:12 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:12 INFO DAGScheduler:59 - Submitting ResultStage 206 (MapPartitionsRDD[144] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:12 INFO MemoryStore:59 - ensureFreeSpace(3016) called with curMem=52133, maxMem=766075207 -2016-06-13 19:45:12 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 2.9 KB, free 730.5 MB) -2016-06-13 19:45:12 INFO MemoryStore:59 - ensureFreeSpace(1788) called with curMem=55149, maxMem=766075207 -2016-06-13 19:45:12 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 1788.0 B, free 730.5 MB) -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:50000 (size: 1788.0 B, free: 730.6 MB) -2016-06-13 19:45:12 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 206 (MapPartitionsRDD[144] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:12 INFO TaskSchedulerImpl:59 - Adding task set 206.0 with 2 tasks -2016-06-13 19:45:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 206.0 (TID 75, localhost, PROCESS_LOCAL, 2318 bytes) -2016-06-13 19:45:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 206.0 (TID 76, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 19:45:12 INFO Executor:59 - Running task 1.0 in stage 206.0 (TID 76) -2016-06-13 19:45:12 INFO Executor:59 - Running task 0.0 in stage 206.0 (TID 75) -2016-06-13 19:45:12 INFO Executor:59 - Finished task 0.0 in stage 206.0 (TID 75). 1265 bytes result sent to driver -2016-06-13 19:45:12 INFO Executor:59 - Finished task 1.0 in stage 206.0 (TID 76). 1265 bytes result sent to driver -2016-06-13 19:45:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 206.0 (TID 76) in 178 ms on localhost (1/2) -2016-06-13 19:45:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 206.0 (TID 75) in 180 ms on localhost (2/2) -2016-06-13 19:45:12 INFO DAGScheduler:59 - ResultStage 206 (reduce at LDAOptimizer.scala:453) finished in 0.180 s -2016-06-13 19:45:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 206.0, whose tasks have all completed, from pool -2016-06-13 19:45:12 INFO DAGScheduler:59 - Job 20 finished: reduce at LDAOptimizer.scala:453, took 0.189036 s -2016-06-13 19:45:12 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:12 INFO DAGScheduler:59 - Got job 21 (reduce at LDAOptimizer.scala:456) with 2 output partitions -2016-06-13 19:45:12 INFO DAGScheduler:59 - Final stage: ResultStage 207(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:12 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:12 INFO DAGScheduler:59 - Submitting ResultStage 207 (MapPartitionsRDD[145] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:12 INFO MemoryStore:59 - ensureFreeSpace(3056) called with curMem=56937, maxMem=766075207 -2016-06-13 19:45:12 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 3.0 KB, free 730.5 MB) -2016-06-13 19:45:12 INFO MemoryStore:59 - ensureFreeSpace(1789) called with curMem=59993, maxMem=766075207 -2016-06-13 19:45:12 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 1789.0 B, free 730.5 MB) -2016-06-13 19:45:12 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:50000 (size: 1789.0 B, free: 730.5 MB) -2016-06-13 19:45:12 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 207 (MapPartitionsRDD[145] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:12 INFO TaskSchedulerImpl:59 - Adding task set 207.0 with 2 tasks -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 207.0 (TID 77, localhost, PROCESS_LOCAL, 2318 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 207.0 (TID 78, localhost, PROCESS_LOCAL, 2327 bytes) -2016-06-13 19:45:13 INFO Executor:59 - Running task 0.0 in stage 207.0 (TID 77) -2016-06-13 19:45:13 INFO Executor:59 - Running task 1.0 in stage 207.0 (TID 78) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 0.0 in stage 207.0 (TID 77). 1314 bytes result sent to driver -2016-06-13 19:45:13 INFO Executor:59 - Finished task 1.0 in stage 207.0 (TID 78). 1314 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 207.0 (TID 77) in 9 ms on localhost (1/2) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 207.0 (TID 78) in 9 ms on localhost (2/2) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 207.0, whose tasks have all completed, from pool -2016-06-13 19:45:13 INFO DAGScheduler:59 - ResultStage 207 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:13 INFO DAGScheduler:59 - Job 21 finished: reduce at LDAOptimizer.scala:456, took 0.021201 s -2016-06-13 19:45:13 INFO SparkContext:59 - Starting job: count at LDAOptimizer.scala:387 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Got job 22 (count at LDAOptimizer.scala:387) with 4 output partitions -2016-06-13 19:45:13 INFO DAGScheduler:59 - Final stage: ResultStage 208(count at LDAOptimizer.scala:387) -2016-06-13 19:45:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting ResultStage 208 (ParallelCollectionRDD[146] at parallelize at LDASuite.scala:254), which has no missing parents -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=61782, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 1032.0 B, free 730.5 MB) -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(757) called with curMem=62814, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 757.0 B, free 730.5 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:50000 (size: 757.0 B, free: 730.5 MB) -2016-06-13 19:45:13 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 208 (ParallelCollectionRDD[146] at parallelize at LDASuite.scala:254) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Adding task set 208.0 with 4 tasks -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 208.0 (TID 79, localhost, PROCESS_LOCAL, 2306 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 208.0 (TID 80, localhost, PROCESS_LOCAL, 2389 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 208.0 (TID 81, localhost, PROCESS_LOCAL, 2315 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 3.0 in stage 208.0 (TID 82, localhost, PROCESS_LOCAL, 2389 bytes) -2016-06-13 19:45:13 INFO Executor:59 - Running task 3.0 in stage 208.0 (TID 82) -2016-06-13 19:45:13 INFO Executor:59 - Running task 1.0 in stage 208.0 (TID 80) -2016-06-13 19:45:13 INFO Executor:59 - Running task 2.0 in stage 208.0 (TID 81) -2016-06-13 19:45:13 INFO Executor:59 - Running task 0.0 in stage 208.0 (TID 79) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 1.0 in stage 208.0 (TID 80). 953 bytes result sent to driver -2016-06-13 19:45:13 INFO Executor:59 - Finished task 0.0 in stage 208.0 (TID 79). 953 bytes result sent to driver -2016-06-13 19:45:13 INFO Executor:59 - Finished task 2.0 in stage 208.0 (TID 81). 953 bytes result sent to driver -2016-06-13 19:45:13 INFO Executor:59 - Finished task 3.0 in stage 208.0 (TID 82). 953 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 208.0 (TID 80) in 12 ms on localhost (1/4) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 208.0 (TID 79) in 13 ms on localhost (2/4) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 3.0 in stage 208.0 (TID 82) in 11 ms on localhost (3/4) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 208.0 (TID 81) in 14 ms on localhost (4/4) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 208.0, whose tasks have all completed, from pool -2016-06-13 19:45:13 INFO DAGScheduler:59 - ResultStage 208 (count at LDAOptimizer.scala:387) finished in 0.019 s -2016-06-13 19:45:13 INFO DAGScheduler:59 - Job 22 finished: count at LDAOptimizer.scala:387, took 0.054457 s -2016-06-13 19:45:13 INFO SparkContext:59 - Starting job: first at LDAOptimizer.scala:388 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Got job 23 (first at LDAOptimizer.scala:388) with 1 output partitions -2016-06-13 19:45:13 INFO DAGScheduler:59 - Final stage: ResultStage 209(first at LDAOptimizer.scala:388) -2016-06-13 19:45:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting ResultStage 209 (ParallelCollectionRDD[146] at parallelize at LDASuite.scala:254), which has no missing parents -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=63571, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 1224.0 B, free 730.5 MB) -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(806) called with curMem=64795, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 806.0 B, free 730.5 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:50000 (size: 806.0 B, free: 730.5 MB) -2016-06-13 19:45:13 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 209 (ParallelCollectionRDD[146] at parallelize at LDASuite.scala:254) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Adding task set 209.0 with 1 tasks -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 209.0 (TID 83, localhost, PROCESS_LOCAL, 2306 bytes) -2016-06-13 19:45:13 INFO Executor:59 - Running task 0.0 in stage 209.0 (TID 83) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 0.0 in stage 209.0 (TID 83). 1226 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 209.0 (TID 83) in 5 ms on localhost (1/1) -2016-06-13 19:45:13 INFO DAGScheduler:59 - ResultStage 209 (first at LDAOptimizer.scala:388) finished in 0.006 s -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 209.0, whose tasks have all completed, from pool -2016-06-13 19:45:13 INFO DAGScheduler:59 - Job 23 finished: first at LDAOptimizer.scala:388, took 0.016690 s -2016-06-13 19:45:13 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Got job 24 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:13 INFO DAGScheduler:59 - Final stage: ResultStage 210(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting ResultStage 210 (PartitionwiseSampledRDD[147] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=65601, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 18.4 KB, free 730.5 MB) -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(7655) called with curMem=84417, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.5 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:13 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 210 (PartitionwiseSampledRDD[147] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Adding task set 210.0 with 1 tasks -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 210.0 (TID 84, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:13 INFO Executor:59 - Running task 0.0 in stage 210.0 (TID 84) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 0.0 in stage 210.0 (TID 84). 911 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 210.0 (TID 84) in 16 ms on localhost (1/1) -2016-06-13 19:45:13 INFO DAGScheduler:59 - ResultStage 210 (isEmpty at LDAOptimizer.scala:418) finished in 0.016 s -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 210.0, whose tasks have all completed, from pool -2016-06-13 19:45:13 INFO DAGScheduler:59 - Job 24 finished: isEmpty at LDAOptimizer.scala:418, took 0.036152 s -2016-06-13 19:45:13 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Got job 25 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:13 INFO DAGScheduler:59 - Final stage: ResultStage 211(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting ResultStage 211 (PartitionwiseSampledRDD[147] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=92072, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 18.4 KB, free 730.5 MB) -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(7655) called with curMem=110888, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.5 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:13 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 211 (PartitionwiseSampledRDD[147] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Adding task set 211.0 with 3 tasks -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 211.0 (TID 85, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 211.0 (TID 86, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 211.0 (TID 87, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:13 INFO Executor:59 - Running task 0.0 in stage 211.0 (TID 85) -2016-06-13 19:45:13 INFO Executor:59 - Running task 2.0 in stage 211.0 (TID 87) -2016-06-13 19:45:13 INFO Executor:59 - Running task 1.0 in stage 211.0 (TID 86) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 1.0 in stage 211.0 (TID 86). 911 bytes result sent to driver -2016-06-13 19:45:13 INFO Executor:59 - Finished task 2.0 in stage 211.0 (TID 87). 1226 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 211.0 (TID 86) in 7 ms on localhost (1/3) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 0.0 in stage 211.0 (TID 85). 1226 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 211.0 (TID 87) in 12 ms on localhost (2/3) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 211.0 (TID 85) in 13 ms on localhost (3/3) -2016-06-13 19:45:13 INFO DAGScheduler:59 - ResultStage 211 (isEmpty at LDAOptimizer.scala:418) finished in 0.014 s -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 211.0, whose tasks have all completed, from pool -2016-06-13 19:45:13 INFO DAGScheduler:59 - Job 25 finished: isEmpty at LDAOptimizer.scala:418, took 0.024429 s -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=118543, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=118719, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:13 INFO SparkContext:59 - Created broadcast 46 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:13 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Got job 26 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:13 INFO DAGScheduler:59 - Final stage: ResultStage 212(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting ResultStage 212 (MapPartitionsRDD[149] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=118999, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 19.8 KB, free 730.5 MB) -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(8423) called with curMem=139255, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.4 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:13 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 212 (MapPartitionsRDD[149] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Adding task set 212.0 with 4 tasks -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 212.0 (TID 88, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 212.0 (TID 89, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 212.0 (TID 90, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 3.0 in stage 212.0 (TID 91, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:13 INFO Executor:59 - Running task 2.0 in stage 212.0 (TID 90) -2016-06-13 19:45:13 INFO Executor:59 - Running task 3.0 in stage 212.0 (TID 91) -2016-06-13 19:45:13 INFO Executor:59 - Running task 0.0 in stage 212.0 (TID 88) -2016-06-13 19:45:13 INFO Executor:59 - Running task 1.0 in stage 212.0 (TID 89) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 2.0 in stage 212.0 (TID 90). 1265 bytes result sent to driver -2016-06-13 19:45:13 INFO Executor:59 - Finished task 1.0 in stage 212.0 (TID 89). 1265 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 212.0 (TID 90) in 9 ms on localhost (1/4) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 212.0 (TID 89) in 12 ms on localhost (2/4) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 3.0 in stage 212.0 (TID 91). 1265 bytes result sent to driver -2016-06-13 19:45:13 INFO Executor:59 - Finished task 0.0 in stage 212.0 (TID 88). 1265 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 212.0 (TID 88) in 22 ms on localhost (3/4) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 3.0 in stage 212.0 (TID 91) in 24 ms on localhost (4/4) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 212.0, whose tasks have all completed, from pool -2016-06-13 19:45:13 INFO DAGScheduler:59 - ResultStage 212 (reduce at LDAOptimizer.scala:453) finished in 0.027 s -2016-06-13 19:45:13 INFO DAGScheduler:59 - Job 26 finished: reduce at LDAOptimizer.scala:453, took 0.038212 s -2016-06-13 19:45:13 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Got job 27 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:13 INFO DAGScheduler:59 - Final stage: ResultStage 213(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting ResultStage 213 (MapPartitionsRDD[150] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=147678, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 19.8 KB, free 730.4 MB) -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=167974, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.4 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:13 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 213 (MapPartitionsRDD[150] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Adding task set 213.0 with 4 tasks -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 213.0 (TID 92, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 213.0 (TID 93, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 213.0 (TID 94, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 3.0 in stage 213.0 (TID 95, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:13 INFO Executor:59 - Running task 1.0 in stage 213.0 (TID 93) -2016-06-13 19:45:13 INFO Executor:59 - Running task 2.0 in stage 213.0 (TID 94) -2016-06-13 19:45:13 INFO Executor:59 - Running task 0.0 in stage 213.0 (TID 92) -2016-06-13 19:45:13 INFO Executor:59 - Running task 3.0 in stage 213.0 (TID 95) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 0.0 in stage 213.0 (TID 92). 1002 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 213.0 (TID 92) in 9 ms on localhost (1/4) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 1.0 in stage 213.0 (TID 93). 1314 bytes result sent to driver -2016-06-13 19:45:13 INFO Executor:59 - Finished task 3.0 in stage 213.0 (TID 95). 1314 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 213.0 (TID 93) in 11 ms on localhost (2/4) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 3.0 in stage 213.0 (TID 95) in 10 ms on localhost (3/4) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 2.0 in stage 213.0 (TID 94). 1002 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 213.0 (TID 94) in 12 ms on localhost (4/4) -2016-06-13 19:45:13 INFO DAGScheduler:59 - ResultStage 213 (reduce at LDAOptimizer.scala:456) finished in 0.014 s -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 213.0, whose tasks have all completed, from pool -2016-06-13 19:45:13 INFO DAGScheduler:59 - Job 27 finished: reduce at LDAOptimizer.scala:456, took 0.027266 s -2016-06-13 19:45:13 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Got job 28 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:13 INFO DAGScheduler:59 - Final stage: ResultStage 214(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting ResultStage 214 (PartitionwiseSampledRDD[151] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=176407, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 18.4 KB, free 730.4 MB) -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=195223, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.4 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:13 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 214 (PartitionwiseSampledRDD[151] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Adding task set 214.0 with 1 tasks -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 214.0 (TID 96, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:13 INFO Executor:59 - Running task 0.0 in stage 214.0 (TID 96) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 0.0 in stage 214.0 (TID 96). 911 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 214.0 (TID 96) in 6 ms on localhost (1/1) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 214.0, whose tasks have all completed, from pool -2016-06-13 19:45:13 INFO DAGScheduler:59 - ResultStage 214 (isEmpty at LDAOptimizer.scala:418) finished in 0.008 s -2016-06-13 19:45:13 INFO DAGScheduler:59 - Job 28 finished: isEmpty at LDAOptimizer.scala:418, took 0.019025 s -2016-06-13 19:45:13 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Got job 29 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:13 INFO DAGScheduler:59 - Final stage: ResultStage 215(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting ResultStage 215 (PartitionwiseSampledRDD[151] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=202879, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 18.4 KB, free 730.4 MB) -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=221695, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.4 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:13 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 215 (PartitionwiseSampledRDD[151] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Adding task set 215.0 with 3 tasks -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 215.0 (TID 97, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 215.0 (TID 98, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 215.0 (TID 99, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:13 INFO Executor:59 - Running task 0.0 in stage 215.0 (TID 97) -2016-06-13 19:45:13 INFO Executor:59 - Running task 2.0 in stage 215.0 (TID 99) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 0.0 in stage 215.0 (TID 97). 1226 bytes result sent to driver -2016-06-13 19:45:13 INFO Executor:59 - Finished task 2.0 in stage 215.0 (TID 99). 1226 bytes result sent to driver -2016-06-13 19:45:13 INFO Executor:59 - Running task 1.0 in stage 215.0 (TID 98) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 1.0 in stage 215.0 (TID 98). 1226 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 215.0 (TID 97) in 15 ms on localhost (1/3) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 215.0 (TID 99) in 14 ms on localhost (2/3) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 215.0 (TID 98) in 15 ms on localhost (3/3) -2016-06-13 19:45:13 INFO DAGScheduler:59 - ResultStage 215 (isEmpty at LDAOptimizer.scala:418) finished in 0.017 s -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 215.0, whose tasks have all completed, from pool -2016-06-13 19:45:13 INFO DAGScheduler:59 - Job 29 finished: isEmpty at LDAOptimizer.scala:418, took 0.029281 s -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=229351, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=229527, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:13 INFO SparkContext:59 - Created broadcast 51 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:13 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Got job 30 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:13 INFO DAGScheduler:59 - Final stage: ResultStage 216(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting ResultStage 216 (MapPartitionsRDD[153] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=229807, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 19.8 KB, free 730.3 MB) -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(8422) called with curMem=250063, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.3 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:13 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 216 (MapPartitionsRDD[153] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Adding task set 216.0 with 4 tasks -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 216.0 (TID 100, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 216.0 (TID 101, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 216.0 (TID 102, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 3.0 in stage 216.0 (TID 103, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:13 INFO Executor:59 - Running task 1.0 in stage 216.0 (TID 101) -2016-06-13 19:45:13 INFO Executor:59 - Running task 3.0 in stage 216.0 (TID 103) -2016-06-13 19:45:13 INFO Executor:59 - Running task 0.0 in stage 216.0 (TID 100) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 0.0 in stage 216.0 (TID 100). 1265 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 216.0 (TID 100) in 9 ms on localhost (1/4) -2016-06-13 19:45:13 INFO Executor:59 - Running task 2.0 in stage 216.0 (TID 102) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 1.0 in stage 216.0 (TID 101). 1265 bytes result sent to driver -2016-06-13 19:45:13 INFO Executor:59 - Finished task 3.0 in stage 216.0 (TID 103). 1265 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 216.0 (TID 101) in 12 ms on localhost (2/4) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 2.0 in stage 216.0 (TID 102). 1265 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 3.0 in stage 216.0 (TID 103) in 14 ms on localhost (3/4) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 216.0 (TID 102) in 15 ms on localhost (4/4) -2016-06-13 19:45:13 INFO DAGScheduler:59 - ResultStage 216 (reduce at LDAOptimizer.scala:453) finished in 0.017 s -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 216.0, whose tasks have all completed, from pool -2016-06-13 19:45:13 INFO DAGScheduler:59 - Job 30 finished: reduce at LDAOptimizer.scala:453, took 0.030441 s -2016-06-13 19:45:13 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Got job 31 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:13 INFO DAGScheduler:59 - Final stage: ResultStage 217(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting ResultStage 217 (MapPartitionsRDD[154] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=258485, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 19.8 KB, free 730.3 MB) -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=278781, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.3 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:13 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 217 (MapPartitionsRDD[154] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Adding task set 217.0 with 4 tasks -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 217.0 (TID 104, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 217.0 (TID 105, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 217.0 (TID 106, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 3.0 in stage 217.0 (TID 107, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:13 INFO Executor:59 - Running task 0.0 in stage 217.0 (TID 104) -2016-06-13 19:45:13 INFO Executor:59 - Running task 1.0 in stage 217.0 (TID 105) -2016-06-13 19:45:13 INFO Executor:59 - Running task 3.0 in stage 217.0 (TID 107) -2016-06-13 19:45:13 INFO Executor:59 - Running task 2.0 in stage 217.0 (TID 106) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 0.0 in stage 217.0 (TID 104). 1002 bytes result sent to driver -2016-06-13 19:45:13 INFO Executor:59 - Finished task 3.0 in stage 217.0 (TID 107). 1358 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 217.0 (TID 104) in 11 ms on localhost (1/4) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 3.0 in stage 217.0 (TID 107) in 10 ms on localhost (2/4) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 2.0 in stage 217.0 (TID 106). 1402 bytes result sent to driver -2016-06-13 19:45:13 INFO Executor:59 - Finished task 1.0 in stage 217.0 (TID 105). 1314 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 217.0 (TID 105) in 25 ms on localhost (3/4) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 217.0 (TID 106) in 26 ms on localhost (4/4) -2016-06-13 19:45:13 INFO DAGScheduler:59 - ResultStage 217 (reduce at LDAOptimizer.scala:456) finished in 0.027 s -2016-06-13 19:45:13 INFO DAGScheduler:59 - Job 31 finished: reduce at LDAOptimizer.scala:456, took 0.039686 s -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 217.0, whose tasks have all completed, from pool -2016-06-13 19:45:13 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Got job 32 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:13 INFO DAGScheduler:59 - Final stage: ResultStage 218(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting ResultStage 218 (PartitionwiseSampledRDD[155] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=287213, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 18.4 KB, free 730.3 MB) -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(7659) called with curMem=306029, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.3 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:13 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 218 (PartitionwiseSampledRDD[155] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Adding task set 218.0 with 1 tasks -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 218.0 (TID 108, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:13 INFO Executor:59 - Running task 0.0 in stage 218.0 (TID 108) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 0.0 in stage 218.0 (TID 108). 911 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 218.0 (TID 108) in 22 ms on localhost (1/1) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 218.0, whose tasks have all completed, from pool -2016-06-13 19:45:13 INFO DAGScheduler:59 - ResultStage 218 (isEmpty at LDAOptimizer.scala:418) finished in 0.023 s -2016-06-13 19:45:13 INFO DAGScheduler:59 - Job 32 finished: isEmpty at LDAOptimizer.scala:418, took 0.051650 s -2016-06-13 19:45:13 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Got job 33 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:13 INFO DAGScheduler:59 - Final stage: ResultStage 219(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting ResultStage 219 (PartitionwiseSampledRDD[155] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=313688, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 18.4 KB, free 730.3 MB) -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(7659) called with curMem=332504, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.3 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:13 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 219 (PartitionwiseSampledRDD[155] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Adding task set 219.0 with 3 tasks -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 219.0 (TID 109, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 219.0 (TID 110, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 219.0 (TID 111, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:13 INFO Executor:59 - Running task 1.0 in stage 219.0 (TID 110) -2016-06-13 19:45:13 INFO Executor:59 - Running task 2.0 in stage 219.0 (TID 111) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 2.0 in stage 219.0 (TID 111). 1226 bytes result sent to driver -2016-06-13 19:45:13 INFO Executor:59 - Finished task 1.0 in stage 219.0 (TID 110). 911 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 219.0 (TID 110) in 7 ms on localhost (1/3) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 219.0 (TID 111) in 6 ms on localhost (2/3) -2016-06-13 19:45:13 INFO Executor:59 - Running task 0.0 in stage 219.0 (TID 109) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 0.0 in stage 219.0 (TID 109). 1226 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 219.0 (TID 109) in 73 ms on localhost (3/3) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 219.0, whose tasks have all completed, from pool -2016-06-13 19:45:13 INFO DAGScheduler:59 - ResultStage 219 (isEmpty at LDAOptimizer.scala:418) finished in 0.075 s -2016-06-13 19:45:13 INFO DAGScheduler:59 - Job 33 finished: isEmpty at LDAOptimizer.scala:418, took 0.088105 s -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=340163, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=340339, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:13 INFO SparkContext:59 - Created broadcast 56 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:13 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Got job 34 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:13 INFO DAGScheduler:59 - Final stage: ResultStage 220(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting ResultStage 220 (MapPartitionsRDD[157] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=340619, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(8430) called with curMem=360875, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:13 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 220 (MapPartitionsRDD[157] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Adding task set 220.0 with 4 tasks -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 220.0 (TID 112, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 220.0 (TID 113, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 220.0 (TID 114, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 3.0 in stage 220.0 (TID 115, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:13 INFO Executor:59 - Running task 0.0 in stage 220.0 (TID 112) -2016-06-13 19:45:13 INFO Executor:59 - Running task 3.0 in stage 220.0 (TID 115) -2016-06-13 19:45:13 INFO Executor:59 - Running task 2.0 in stage 220.0 (TID 114) -2016-06-13 19:45:13 INFO Executor:59 - Running task 1.0 in stage 220.0 (TID 113) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 0.0 in stage 220.0 (TID 112). 1265 bytes result sent to driver -2016-06-13 19:45:13 INFO Executor:59 - Finished task 1.0 in stage 220.0 (TID 113). 1265 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 220.0 (TID 112) in 12 ms on localhost (1/4) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 220.0 (TID 113) in 12 ms on localhost (2/4) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 2.0 in stage 220.0 (TID 114). 1265 bytes result sent to driver -2016-06-13 19:45:13 INFO Executor:59 - Finished task 3.0 in stage 220.0 (TID 115). 1265 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 220.0 (TID 114) in 16 ms on localhost (3/4) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 3.0 in stage 220.0 (TID 115) in 18 ms on localhost (4/4) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 220.0, whose tasks have all completed, from pool -2016-06-13 19:45:13 INFO DAGScheduler:59 - ResultStage 220 (reduce at LDAOptimizer.scala:453) finished in 0.023 s -2016-06-13 19:45:13 INFO DAGScheduler:59 - Job 34 finished: reduce at LDAOptimizer.scala:453, took 0.036615 s -2016-06-13 19:45:13 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Got job 35 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:13 INFO DAGScheduler:59 - Final stage: ResultStage 221(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting ResultStage 221 (MapPartitionsRDD[158] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=369305, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(8443) called with curMem=389601, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:13 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 221 (MapPartitionsRDD[158] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Adding task set 221.0 with 4 tasks -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 221.0 (TID 116, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 221.0 (TID 117, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 221.0 (TID 118, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 3.0 in stage 221.0 (TID 119, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:13 INFO Executor:59 - Running task 0.0 in stage 221.0 (TID 116) -2016-06-13 19:45:13 INFO Executor:59 - Running task 2.0 in stage 221.0 (TID 118) -2016-06-13 19:45:13 INFO Executor:59 - Running task 1.0 in stage 221.0 (TID 117) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 0.0 in stage 221.0 (TID 116). 1002 bytes result sent to driver -2016-06-13 19:45:13 INFO Executor:59 - Finished task 2.0 in stage 221.0 (TID 118). 1002 bytes result sent to driver -2016-06-13 19:45:13 INFO Executor:59 - Finished task 1.0 in stage 221.0 (TID 117). 1358 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 221.0 (TID 116) in 12 ms on localhost (1/4) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 221.0 (TID 118) in 12 ms on localhost (2/4) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 221.0 (TID 117) in 14 ms on localhost (3/4) -2016-06-13 19:45:13 INFO Executor:59 - Running task 3.0 in stage 221.0 (TID 119) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 3.0 in stage 221.0 (TID 119). 1402 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 3.0 in stage 221.0 (TID 119) in 24 ms on localhost (4/4) -2016-06-13 19:45:13 INFO DAGScheduler:59 - ResultStage 221 (reduce at LDAOptimizer.scala:456) finished in 0.027 s -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 221.0, whose tasks have all completed, from pool -2016-06-13 19:45:13 INFO DAGScheduler:59 - Job 35 finished: reduce at LDAOptimizer.scala:456, took 0.039153 s -2016-06-13 19:45:13 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Got job 36 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:13 INFO DAGScheduler:59 - Final stage: ResultStage 222(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting ResultStage 222 (PartitionwiseSampledRDD[159] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=398044, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 18.4 KB, free 730.2 MB) -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(7655) called with curMem=416860, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:13 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 222 (PartitionwiseSampledRDD[159] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Adding task set 222.0 with 1 tasks -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 222.0 (TID 120, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:13 INFO Executor:59 - Running task 0.0 in stage 222.0 (TID 120) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 0.0 in stage 222.0 (TID 120). 1226 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 222.0 (TID 120) in 40 ms on localhost (1/1) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 222.0, whose tasks have all completed, from pool -2016-06-13 19:45:13 INFO DAGScheduler:59 - ResultStage 222 (isEmpty at LDAOptimizer.scala:418) finished in 0.040 s -2016-06-13 19:45:13 INFO DAGScheduler:59 - Job 36 finished: isEmpty at LDAOptimizer.scala:418, took 0.051489 s -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=424515, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 176.0 B, free 730.2 MB) -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(279) called with curMem=424691, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 279.0 B, free 730.2 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:50000 (size: 279.0 B, free: 730.4 MB) -2016-06-13 19:45:13 INFO SparkContext:59 - Created broadcast 60 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:50000 in memory (size: 2.2 KB, free: 730.4 MB) -2016-06-13 19:45:13 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-06-13 19:45:13 INFO ContextCleaner:59 - Cleaned shuffle 18 -2016-06-13 19:45:13 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Got job 37 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:13 INFO DAGScheduler:59 - Final stage: ResultStage 223(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting ResultStage 223 (MapPartitionsRDD[161] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:13 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=392309, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:13 INFO MemoryStore:59 - ensureFreeSpace(8423) called with curMem=392269, maxMem=766075207 -2016-06-13 19:45:13 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:13 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:13 INFO SparkContext:59 - Created broadcast 61 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:13 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 223 (MapPartitionsRDD[161] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:13 INFO TaskSchedulerImpl:59 - Adding task set 223.0 with 4 tasks -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 223.0 (TID 121, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 223.0 (TID 122, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 223.0 (TID 123, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Starting task 3.0 in stage 223.0 (TID 124, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:13 INFO Executor:59 - Running task 2.0 in stage 223.0 (TID 123) -2016-06-13 19:45:13 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-06-13 19:45:13 INFO Executor:59 - Running task 3.0 in stage 223.0 (TID 124) -2016-06-13 19:45:13 INFO Executor:59 - Running task 1.0 in stage 223.0 (TID 122) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 2.0 in stage 223.0 (TID 123). 1265 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 223.0 (TID 123) in 7 ms on localhost (1/4) -2016-06-13 19:45:13 INFO Executor:59 - Finished task 3.0 in stage 223.0 (TID 124). 1265 bytes result sent to driver -2016-06-13 19:45:13 INFO Executor:59 - Finished task 1.0 in stage 223.0 (TID 122). 1265 bytes result sent to driver -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 3.0 in stage 223.0 (TID 124) in 10 ms on localhost (2/4) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 223.0 (TID 122) in 12 ms on localhost (3/4) -2016-06-13 19:45:13 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:13 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:13 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:13 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:13 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:14 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:14 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:14 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:14 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 223.0 (TID 121) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 223.0 (TID 121). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:14 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:14 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 223.0 (TID 121) in 80 ms on localhost (4/4) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 223.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:50000 in memory (size: 806.0 B, free: 730.5 MB) -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 223 (reduce at LDAOptimizer.scala:453) finished in 0.084 s -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 37 finished: reduce at LDAOptimizer.scala:453, took 0.098960 s -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 38 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 224(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 224 (MapPartitionsRDD[162] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=86515, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 19.8 KB, free 730.5 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=106811, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.5 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 224 (MapPartitionsRDD[162] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 224.0 with 4 tasks -2016-06-13 19:45:14 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 224.0 (TID 125, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 224.0 (TID 126, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 224.0 (TID 127, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 224.0 (TID 128, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 224.0 (TID 125) -2016-06-13 19:45:14 INFO Executor:59 - Running task 1.0 in stage 224.0 (TID 126) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 1.0 in stage 224.0 (TID 126). 1446 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Running task 2.0 in stage 224.0 (TID 127) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:50000 in memory (size: 757.0 B, free: 730.5 MB) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 224.0 (TID 126) in 16 ms on localhost (1/4) -2016-06-13 19:45:14 INFO Executor:59 - Running task 3.0 in stage 224.0 (TID 128) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 2.0 in stage 224.0 (TID 127). 1002 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 224.0 (TID 127) in 18 ms on localhost (2/4) -2016-06-13 19:45:14 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-06-13 19:45:14 INFO Executor:59 - Finished task 3.0 in stage 224.0 (TID 128). 1314 bytes result sent to driver -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:50000 in memory (size: 1789.0 B, free: 730.5 MB) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 224.0 (TID 125). 1314 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 224.0 (TID 128) in 27 ms on localhost (3/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 224.0 (TID 125) in 29 ms on localhost (4/4) -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 224 (reduce at LDAOptimizer.scala:456) finished in 0.031 s -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 224.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 38 finished: reduce at LDAOptimizer.scala:456, took 0.043009 s -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 39 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 225(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 225 (PartitionwiseSampledRDD[163] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=108610, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 18.4 KB, free 730.5 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(7655) called with curMem=127426, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.5 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:14 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 225 (PartitionwiseSampledRDD[163] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 225.0 with 1 tasks -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 225.0 (TID 129, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:50000 in memory (size: 1788.0 B, free: 730.5 MB) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 225.0 (TID 129) -2016-06-13 19:45:14 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 225.0 (TID 129). 1226 bytes result sent to driver -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:50000 in memory (size: 255.0 B, free: 730.5 MB) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 225.0 (TID 129) in 8 ms on localhost (1/1) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 225.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 225 (isEmpty at LDAOptimizer.scala:418) finished in 0.011 s -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 39 finished: isEmpty at LDAOptimizer.scala:418, took 0.067496 s -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=129846, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=130022, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 64 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 40 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 226(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 226 (MapPartitionsRDD[165] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:50000 in memory (size: 806.0 B, free: 730.5 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=128272, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 19.8 KB, free 730.4 MB) -2016-06-13 19:45:14 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(8416) called with curMem=148528, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.4 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:50000 in memory (size: 757.0 B, free: 730.5 MB) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 226 (MapPartitionsRDD[165] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 226.0 with 4 tasks -2016-06-13 19:45:14 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 226.0 (TID 130, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 226.0 (TID 131, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 226.0 (TID 132, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 226.0 (TID 133, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 1.0 in stage 226.0 (TID 131) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 226.0 (TID 130) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 1.0 in stage 226.0 (TID 131). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Running task 2.0 in stage 226.0 (TID 132) -2016-06-13 19:45:14 INFO Executor:59 - Running task 3.0 in stage 226.0 (TID 133) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 226.0 (TID 130). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 2.0 in stage 226.0 (TID 132). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 3.0 in stage 226.0 (TID 133). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:50000 in memory (size: 806.0 B, free: 730.5 MB) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 226.0 (TID 130) in 17 ms on localhost (1/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 226.0 (TID 133) in 16 ms on localhost (2/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 226.0 (TID 132) in 16 ms on localhost (3/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 226.0 (TID 131) in 18 ms on localhost (4/4) -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 226 (reduce at LDAOptimizer.scala:453) finished in 0.019 s -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 226.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 40 finished: reduce at LDAOptimizer.scala:453, took 0.030559 s -2016-06-13 19:45:14 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 41 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 227(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 227 (MapPartitionsRDD[166] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:50000 in memory (size: 757.0 B, free: 730.5 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=151336, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 19.8 KB, free 730.4 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(8429) called with curMem=171632, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.4 MB) -2016-06-13 19:45:14 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 227 (MapPartitionsRDD[166] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 227.0 with 4 tasks -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 227.0 (TID 134, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 227.0 (TID 135, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:50000 in memory (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 227.0 (TID 136, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 227.0 (TID 137, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 2.0 in stage 227.0 (TID 136) -2016-06-13 19:45:14 INFO Executor:59 - Running task 1.0 in stage 227.0 (TID 135) -2016-06-13 19:45:14 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 227.0 (TID 134) -2016-06-13 19:45:14 INFO Executor:59 - Running task 3.0 in stage 227.0 (TID 137) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 1.0 in stage 227.0 (TID 135). 1314 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 2.0 in stage 227.0 (TID 136). 1358 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 227.0 (TID 134). 1358 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 227.0 (TID 135) in 14 ms on localhost (1/4) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 3.0 in stage 227.0 (TID 137). 1358 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 227.0 (TID 136) in 14 ms on localhost (2/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 227.0 (TID 134) in 16 ms on localhost (3/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 227.0 (TID 137) in 21 ms on localhost (4/4) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 227.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 227 (reduce at LDAOptimizer.scala:456) finished in 0.026 s -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 41 finished: reduce at LDAOptimizer.scala:456, took 0.036553 s -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 42 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 228(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 228 (PartitionwiseSampledRDD[167] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=168235, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 18.4 KB, free 730.4 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=187051, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.4 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 67 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 228 (PartitionwiseSampledRDD[167] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 228.0 with 1 tasks -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 228.0 (TID 138, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 228.0 (TID 138) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 228.0 (TID 138). 1226 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 228.0 (TID 138) in 7 ms on localhost (1/1) -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 228 (isEmpty at LDAOptimizer.scala:418) finished in 0.007 s -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 228.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 42 finished: isEmpty at LDAOptimizer.scala:418, took 0.021206 s -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=194707, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=194883, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 68 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 43 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 229(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 229 (MapPartitionsRDD[169] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=195163, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 19.8 KB, free 730.4 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(8442) called with curMem=215419, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.4 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 69 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 229 (MapPartitionsRDD[169] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 229.0 with 4 tasks -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 229.0 (TID 139, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 229.0 (TID 140, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 229.0 (TID 141, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 229.0 (TID 142, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 229.0 (TID 139) -2016-06-13 19:45:14 INFO Executor:59 - Running task 3.0 in stage 229.0 (TID 142) -2016-06-13 19:45:14 INFO Executor:59 - Running task 2.0 in stage 229.0 (TID 141) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 3.0 in stage 229.0 (TID 142). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Running task 1.0 in stage 229.0 (TID 140) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 229.0 (TID 139). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 2.0 in stage 229.0 (TID 141). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 1.0 in stage 229.0 (TID 140). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 229.0 (TID 142) in 13 ms on localhost (1/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 229.0 (TID 139) in 15 ms on localhost (2/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 229.0 (TID 141) in 16 ms on localhost (3/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 229.0 (TID 140) in 17 ms on localhost (4/4) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 229.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 229 (reduce at LDAOptimizer.scala:453) finished in 0.018 s -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 43 finished: reduce at LDAOptimizer.scala:453, took 0.032913 s -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 44 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 230(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 230 (MapPartitionsRDD[170] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=223861, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 19.8 KB, free 730.4 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(8444) called with curMem=244157, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.3 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 230 (MapPartitionsRDD[170] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 230.0 with 4 tasks -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 230.0 (TID 143, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 230.0 (TID 144, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 230.0 (TID 145, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 230.0 (TID 146, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 1.0 in stage 230.0 (TID 144) -2016-06-13 19:45:14 INFO Executor:59 - Running task 3.0 in stage 230.0 (TID 146) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 230.0 (TID 143) -2016-06-13 19:45:14 INFO Executor:59 - Running task 2.0 in stage 230.0 (TID 145) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 3.0 in stage 230.0 (TID 146). 1314 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 1.0 in stage 230.0 (TID 144). 1314 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 230.0 (TID 143). 1358 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 230.0 (TID 146) in 10 ms on localhost (1/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 230.0 (TID 144) in 15 ms on localhost (2/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 230.0 (TID 143) in 16 ms on localhost (3/4) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 2.0 in stage 230.0 (TID 145). 1402 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 230.0 (TID 145) in 16 ms on localhost (4/4) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 230.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 230 (reduce at LDAOptimizer.scala:456) finished in 0.019 s -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 44 finished: reduce at LDAOptimizer.scala:456, took 0.042856 s -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 45 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 231(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 231 (PartitionwiseSampledRDD[171] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=252601, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_71 stored as values in memory (estimated size 18.4 KB, free 730.3 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=271417, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_71_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.3 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_71_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 71 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 231 (PartitionwiseSampledRDD[171] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 231.0 with 1 tasks -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 231.0 (TID 147, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 231.0 (TID 147) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 231.0 (TID 147). 1226 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 231.0 (TID 147) in 6 ms on localhost (1/1) -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 231 (isEmpty at LDAOptimizer.scala:418) finished in 0.010 s -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 45 finished: isEmpty at LDAOptimizer.scala:418, took 0.020676 s -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=279073, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_72 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=279249, maxMem=766075207 -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 231.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_72_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_72_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 72 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 46 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 232(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 232 (MapPartitionsRDD[173] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=279529, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_73 stored as values in memory (estimated size 19.8 KB, free 730.3 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(8414) called with curMem=299785, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_73_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.3 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_73_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 73 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 232 (MapPartitionsRDD[173] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 232.0 with 4 tasks -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 232.0 (TID 148, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 232.0 (TID 149, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 232.0 (TID 150, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 232.0 (TID 151, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 1.0 in stage 232.0 (TID 149) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 1.0 in stage 232.0 (TID 149). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 232.0 (TID 148) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 232.0 (TID 148). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Running task 3.0 in stage 232.0 (TID 151) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 232.0 (TID 149) in 25 ms on localhost (1/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 232.0 (TID 148) in 26 ms on localhost (2/4) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 3.0 in stage 232.0 (TID 151). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Running task 2.0 in stage 232.0 (TID 150) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 232.0 (TID 151) in 25 ms on localhost (3/4) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 2.0 in stage 232.0 (TID 150). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 232.0 (TID 150) in 64 ms on localhost (4/4) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 232.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 232 (reduce at LDAOptimizer.scala:453) finished in 0.067 s -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 46 finished: reduce at LDAOptimizer.scala:453, took 0.088590 s -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 47 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 233(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 233 (MapPartitionsRDD[174] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=308199, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_74 stored as values in memory (estimated size 19.8 KB, free 730.3 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(8427) called with curMem=328495, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_74_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.3 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_74_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 74 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 233 (MapPartitionsRDD[174] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 233.0 with 4 tasks -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 233.0 (TID 152, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 233.0 (TID 153, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 233.0 (TID 154, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 233.0 (TID 155, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 1.0 in stage 233.0 (TID 153) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 1.0 in stage 233.0 (TID 153). 1446 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Running task 2.0 in stage 233.0 (TID 154) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 2.0 in stage 233.0 (TID 154). 1358 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Running task 3.0 in stage 233.0 (TID 155) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 3.0 in stage 233.0 (TID 155). 1314 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 233.0 (TID 154) in 25 ms on localhost (1/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 233.0 (TID 153) in 25 ms on localhost (2/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 233.0 (TID 155) in 25 ms on localhost (3/4) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 233.0 (TID 152) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 233.0 (TID 152). 1314 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 233.0 (TID 152) in 38 ms on localhost (4/4) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 233.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 233 (reduce at LDAOptimizer.scala:456) finished in 0.039 s -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 47 finished: reduce at LDAOptimizer.scala:456, took 0.050062 s -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 48 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 234(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 234 (PartitionwiseSampledRDD[175] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=336922, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_75 stored as values in memory (estimated size 18.4 KB, free 730.2 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=355738, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_75_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_75_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 75 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 234 (PartitionwiseSampledRDD[175] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 234.0 with 1 tasks -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 234.0 (TID 156, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 234.0 (TID 156) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 234.0 (TID 156). 1226 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 234.0 (TID 156) in 7 ms on localhost (1/1) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 234.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 234 (isEmpty at LDAOptimizer.scala:418) finished in 0.008 s -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 48 finished: isEmpty at LDAOptimizer.scala:418, took 0.023339 s -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=363400, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_76 stored as values in memory (estimated size 176.0 B, free 730.2 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=363576, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_76_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.2 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_76_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 76 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 49 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 235(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 235 (MapPartitionsRDD[177] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=363856, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_77 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(8422) called with curMem=384112, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_77_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_77_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 77 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 235 (MapPartitionsRDD[177] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 235.0 with 4 tasks -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 235.0 (TID 157, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 235.0 (TID 158, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 235.0 (TID 159, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 235.0 (TID 160, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 2.0 in stage 235.0 (TID 159) -2016-06-13 19:45:14 INFO Executor:59 - Running task 1.0 in stage 235.0 (TID 158) -2016-06-13 19:45:14 INFO Executor:59 - Running task 3.0 in stage 235.0 (TID 160) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 3.0 in stage 235.0 (TID 160). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 2.0 in stage 235.0 (TID 159). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 1.0 in stage 235.0 (TID 158). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 235.0 (TID 160) in 12 ms on localhost (1/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 235.0 (TID 159) in 13 ms on localhost (2/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 235.0 (TID 158) in 14 ms on localhost (3/4) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 235.0 (TID 157) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 235.0 (TID 157). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 235.0 (TID 157) in 42 ms on localhost (4/4) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 235.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 235 (reduce at LDAOptimizer.scala:453) finished in 0.044 s -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 49 finished: reduce at LDAOptimizer.scala:453, took 0.058805 s -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 50 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 236(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 236 (MapPartitionsRDD[178] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=392534, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_78 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(8435) called with curMem=412830, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_78_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_78_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 78 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 236 (MapPartitionsRDD[178] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 236.0 with 4 tasks -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 236.0 (TID 161, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 236.0 (TID 162, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 236.0 (TID 163, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 236.0 (TID 164, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 2.0 in stage 236.0 (TID 163) -2016-06-13 19:45:14 INFO Executor:59 - Running task 1.0 in stage 236.0 (TID 162) -2016-06-13 19:45:14 INFO Executor:59 - Running task 3.0 in stage 236.0 (TID 164) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 236.0 (TID 161) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 3.0 in stage 236.0 (TID 164). 1358 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 236.0 (TID 164) in 8 ms on localhost (1/4) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 236.0 (TID 161). 1314 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 2.0 in stage 236.0 (TID 163). 1314 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 1.0 in stage 236.0 (TID 162). 1314 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 236.0 (TID 161) in 29 ms on localhost (2/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 236.0 (TID 163) in 27 ms on localhost (3/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 236.0 (TID 162) in 29 ms on localhost (4/4) -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 236 (reduce at LDAOptimizer.scala:456) finished in 0.030 s -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 50 finished: reduce at LDAOptimizer.scala:456, took 0.040161 s -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 236.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 51 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 237(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 237 (PartitionwiseSampledRDD[179] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=421265, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_79 stored as values in memory (estimated size 18.4 KB, free 730.2 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(7655) called with curMem=440081, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_79_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_79_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 79 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 237 (PartitionwiseSampledRDD[179] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 237.0 with 1 tasks -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 237.0 (TID 165, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 237.0 (TID 165) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 237.0 (TID 165). 1226 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 237.0 (TID 165) in 4 ms on localhost (1/1) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 237.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 237 (isEmpty at LDAOptimizer.scala:418) finished in 0.006 s -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 51 finished: isEmpty at LDAOptimizer.scala:418, took 0.018796 s -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=447736, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_80 stored as values in memory (estimated size 176.0 B, free 730.2 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=447912, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_80_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.2 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_80_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 80 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 52 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 238(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 238 (MapPartitionsRDD[181] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=448192, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_81 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(8414) called with curMem=468448, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_81_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_81_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 81 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 238 (MapPartitionsRDD[181] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 238.0 with 4 tasks -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 238.0 (TID 166, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 238.0 (TID 167, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 238.0 (TID 168, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 238.0 (TID 169, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 238.0 (TID 166) -2016-06-13 19:45:14 INFO Executor:59 - Running task 3.0 in stage 238.0 (TID 169) -2016-06-13 19:45:14 INFO Executor:59 - Running task 1.0 in stage 238.0 (TID 167) -2016-06-13 19:45:14 INFO Executor:59 - Running task 2.0 in stage 238.0 (TID 168) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 238.0 (TID 166). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 238.0 (TID 166) in 6 ms on localhost (1/4) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 2.0 in stage 238.0 (TID 168). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 1.0 in stage 238.0 (TID 167). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 238.0 (TID 168) in 8 ms on localhost (2/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 238.0 (TID 167) in 8 ms on localhost (3/4) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 3.0 in stage 238.0 (TID 169). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 238.0 (TID 169) in 10 ms on localhost (4/4) -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 238 (reduce at LDAOptimizer.scala:453) finished in 0.011 s -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 238.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 52 finished: reduce at LDAOptimizer.scala:453, took 0.017447 s -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 53 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 239(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 239 (MapPartitionsRDD[182] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=476862, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_82 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(8425) called with curMem=497158, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_82_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_82_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 82 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 239 (MapPartitionsRDD[182] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 239.0 with 4 tasks -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 239.0 (TID 170, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 239.0 (TID 171, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 239.0 (TID 172, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 239.0 (TID 173, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 2.0 in stage 239.0 (TID 172) -2016-06-13 19:45:14 INFO Executor:59 - Running task 3.0 in stage 239.0 (TID 173) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 239.0 (TID 170) -2016-06-13 19:45:14 INFO Executor:59 - Running task 1.0 in stage 239.0 (TID 171) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 239.0 (TID 170). 1314 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 1.0 in stage 239.0 (TID 171). 1358 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 2.0 in stage 239.0 (TID 172). 1358 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 239.0 (TID 170) in 8 ms on localhost (1/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 239.0 (TID 171) in 8 ms on localhost (2/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 239.0 (TID 172) in 8 ms on localhost (3/4) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 3.0 in stage 239.0 (TID 173). 1578 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 239.0 (TID 173) in 9 ms on localhost (4/4) -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 239 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 239.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 53 finished: reduce at LDAOptimizer.scala:456, took 0.019035 s -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 54 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 240(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 240 (PartitionwiseSampledRDD[183] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=505583, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_83 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=524399, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_83_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_83_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 83 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 240 (PartitionwiseSampledRDD[183] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 240.0 with 1 tasks -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 240.0 (TID 174, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 240.0 (TID 174) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 240.0 (TID 174). 911 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 240.0 (TID 174) in 3 ms on localhost (1/1) -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 240 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 240.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 54 finished: isEmpty at LDAOptimizer.scala:418, took 0.011311 s -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 55 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 241(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 241 (PartitionwiseSampledRDD[183] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=532055, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_84 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=550871, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_84_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_84_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 84 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 241 (PartitionwiseSampledRDD[183] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 241.0 with 3 tasks -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 241.0 (TID 175, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 241.0 (TID 176, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 241.0 (TID 177, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 1.0 in stage 241.0 (TID 176) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 241.0 (TID 175) -2016-06-13 19:45:14 INFO Executor:59 - Running task 2.0 in stage 241.0 (TID 177) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 1.0 in stage 241.0 (TID 176). 911 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 2.0 in stage 241.0 (TID 177). 911 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 241.0 (TID 175). 911 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 241.0 (TID 176) in 4 ms on localhost (1/3) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 241.0 (TID 177) in 4 ms on localhost (2/3) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 241.0 (TID 175) in 5 ms on localhost (3/3) -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 241 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 241.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 55 finished: isEmpty at LDAOptimizer.scala:418, took 0.014004 s -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 56 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 242(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 242 (PartitionwiseSampledRDD[184] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=558527, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_85 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=577343, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_85_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_85_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 85 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 242 (PartitionwiseSampledRDD[184] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 242.0 with 1 tasks -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 242.0 (TID 178, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 242.0 (TID 178) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 242.0 (TID 178). 1226 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 242.0 (TID 178) in 4 ms on localhost (1/1) -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 242 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 242.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 56 finished: isEmpty at LDAOptimizer.scala:418, took 0.010998 s -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=585005, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_86 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=585181, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_86_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_86_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 86 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 57 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 243(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 243 (MapPartitionsRDD[186] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=585461, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_87 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(8426) called with curMem=605717, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_87_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_87_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 87 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 243 (MapPartitionsRDD[186] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 243.0 with 4 tasks -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 243.0 (TID 179, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 243.0 (TID 180, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 243.0 (TID 181, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 243.0 (TID 182, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 1.0 in stage 243.0 (TID 180) -2016-06-13 19:45:14 INFO Executor:59 - Running task 3.0 in stage 243.0 (TID 182) -2016-06-13 19:45:14 INFO Executor:59 - Running task 2.0 in stage 243.0 (TID 181) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 243.0 (TID 179) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 2.0 in stage 243.0 (TID 181). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 243.0 (TID 179). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 1.0 in stage 243.0 (TID 180). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 3.0 in stage 243.0 (TID 182). 1265 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 243.0 (TID 181) in 7 ms on localhost (1/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 243.0 (TID 180) in 8 ms on localhost (2/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 243.0 (TID 179) in 8 ms on localhost (3/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 243.0 (TID 182) in 7 ms on localhost (4/4) -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 243 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 243.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 57 finished: reduce at LDAOptimizer.scala:453, took 0.016149 s -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 58 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 244(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 244 (MapPartitionsRDD[187] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=614143, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_88 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(8439) called with curMem=634439, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_88_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_88_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 88 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 244 (MapPartitionsRDD[187] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 244.0 with 4 tasks -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 244.0 (TID 183, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 244.0 (TID 184, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 244.0 (TID 185, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 3.0 in stage 244.0 (TID 186, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 1.0 in stage 244.0 (TID 184) -2016-06-13 19:45:14 INFO Executor:59 - Running task 3.0 in stage 244.0 (TID 186) -2016-06-13 19:45:14 INFO Executor:59 - Running task 2.0 in stage 244.0 (TID 185) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 244.0 (TID 183) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 2.0 in stage 244.0 (TID 185). 1002 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 244.0 (TID 185) in 3 ms on localhost (1/4) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 244.0 (TID 183). 1314 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 244.0 (TID 183) in 6 ms on localhost (2/4) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 1.0 in stage 244.0 (TID 184). 1402 bytes result sent to driver -2016-06-13 19:45:14 INFO Executor:59 - Finished task 3.0 in stage 244.0 (TID 186). 1402 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 244.0 (TID 184) in 9 ms on localhost (3/4) -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 3.0 in stage 244.0 (TID 186) in 8 ms on localhost (4/4) -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 244 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 244.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 58 finished: reduce at LDAOptimizer.scala:456, took 0.017070 s -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 59 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 245(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 245 (PartitionwiseSampledRDD[188] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=642878, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_89 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=661694, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_89_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_89_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 89 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 245 (PartitionwiseSampledRDD[188] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Adding task set 245.0 with 1 tasks -2016-06-13 19:45:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 245.0 (TID 187, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:14 INFO Executor:59 - Running task 0.0 in stage 245.0 (TID 187) -2016-06-13 19:45:14 INFO Executor:59 - Finished task 0.0 in stage 245.0 (TID 187). 1226 bytes result sent to driver -2016-06-13 19:45:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 245.0 (TID 187) in 3 ms on localhost (1/1) -2016-06-13 19:45:14 INFO DAGScheduler:59 - ResultStage 245 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 245.0, whose tasks have all completed, from pool -2016-06-13 19:45:14 INFO DAGScheduler:59 - Job 59 finished: isEmpty at LDAOptimizer.scala:418, took 0.010888 s -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=669351, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_90 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=669527, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_90_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:14 INFO BlockManagerInfo:59 - Added broadcast_90_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:14 INFO SparkContext:59 - Created broadcast 90 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:14 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:14 INFO DAGScheduler:59 - Got job 60 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:14 INFO DAGScheduler:59 - Final stage: ResultStage 246(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:14 INFO DAGScheduler:59 - Submitting ResultStage 246 (MapPartitionsRDD[190] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=669807, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_91 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:14 INFO MemoryStore:59 - ensureFreeSpace(8442) called with curMem=690063, maxMem=766075207 -2016-06-13 19:45:14 INFO MemoryStore:59 - Block broadcast_91_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_91_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 91 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 246 (MapPartitionsRDD[190] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 246.0 with 4 tasks -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 246.0 (TID 188, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 246.0 (TID 189, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 246.0 (TID 190, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 246.0 (TID 191, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 1.0 in stage 246.0 (TID 189) -2016-06-13 19:45:15 INFO Executor:59 - Running task 2.0 in stage 246.0 (TID 190) -2016-06-13 19:45:15 INFO Executor:59 - Running task 3.0 in stage 246.0 (TID 191) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 246.0 (TID 188) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 3.0 in stage 246.0 (TID 191). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 2.0 in stage 246.0 (TID 190). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 3.0 in stage 246.0 (TID 191) in 4 ms on localhost (1/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 246.0 (TID 190) in 5 ms on localhost (2/4) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 1.0 in stage 246.0 (TID 189). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 246.0 (TID 189) in 7 ms on localhost (3/4) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 246.0 (TID 188). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 246.0 (TID 188) in 9 ms on localhost (4/4) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 246.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 246 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 60 finished: reduce at LDAOptimizer.scala:453, took 0.018190 s -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 61 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 247(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 247 (MapPartitionsRDD[191] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=698505, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_92 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(8452) called with curMem=718801, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_92_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.9 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_92_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 92 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 247 (MapPartitionsRDD[191] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 247.0 with 4 tasks -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 247.0 (TID 192, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 247.0 (TID 193, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 247.0 (TID 194, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 247.0 (TID 195, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 1.0 in stage 247.0 (TID 193) -2016-06-13 19:45:15 INFO Executor:59 - Running task 2.0 in stage 247.0 (TID 194) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 247.0 (TID 192) -2016-06-13 19:45:15 INFO Executor:59 - Running task 3.0 in stage 247.0 (TID 195) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 2.0 in stage 247.0 (TID 194). 1002 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 247.0 (TID 192). 1314 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 1.0 in stage 247.0 (TID 193). 1490 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 3.0 in stage 247.0 (TID 195). 1002 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 247.0 (TID 194) in 7 ms on localhost (1/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 3.0 in stage 247.0 (TID 195) in 7 ms on localhost (2/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 247.0 (TID 192) in 9 ms on localhost (3/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 247.0 (TID 193) in 8 ms on localhost (4/4) -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 247 (reduce at LDAOptimizer.scala:456) finished in 0.009 s -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 247.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 61 finished: reduce at LDAOptimizer.scala:456, took 0.017041 s -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 62 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 248(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 248 (PartitionwiseSampledRDD[192] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=727253, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_93 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(7658) called with curMem=746069, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_93_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_93_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 93 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 248 (PartitionwiseSampledRDD[192] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 248.0 with 1 tasks -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 248.0 (TID 196, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 248.0 (TID 196) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 248.0 (TID 196). 911 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 248.0 (TID 196) in 3 ms on localhost (1/1) -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 248 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 248.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 62 finished: isEmpty at LDAOptimizer.scala:418, took 0.010388 s -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 63 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 249(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 249 (PartitionwiseSampledRDD[192] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=753727, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_94 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(7658) called with curMem=772543, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_94_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_94_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 94 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 249 (PartitionwiseSampledRDD[192] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 249.0 with 3 tasks -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 249.0 (TID 197, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 249.0 (TID 198, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 249.0 (TID 199, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 249.0 (TID 197) -2016-06-13 19:45:15 INFO Executor:59 - Running task 2.0 in stage 249.0 (TID 199) -2016-06-13 19:45:15 INFO Executor:59 - Running task 1.0 in stage 249.0 (TID 198) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 2.0 in stage 249.0 (TID 199). 1226 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 249.0 (TID 197). 1226 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 1.0 in stage 249.0 (TID 198). 1226 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 249.0 (TID 199) in 5 ms on localhost (1/3) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 249.0 (TID 198) in 7 ms on localhost (2/3) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 249.0 (TID 197) in 7 ms on localhost (3/3) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 249.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 249 (isEmpty at LDAOptimizer.scala:418) finished in 0.008 s -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 63 finished: isEmpty at LDAOptimizer.scala:418, took 0.019150 s -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=780201, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_95 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=780377, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_95_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.8 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_95_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 95 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 64 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 250(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 250 (MapPartitionsRDD[194] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=780657, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_96 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(8429) called with curMem=800913, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_96_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_96_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 96 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 250 (MapPartitionsRDD[194] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 250.0 with 4 tasks -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 250.0 (TID 200, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 250.0 (TID 201, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 250.0 (TID 202, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 250.0 (TID 203, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 2.0 in stage 250.0 (TID 202) -2016-06-13 19:45:15 INFO Executor:59 - Running task 3.0 in stage 250.0 (TID 203) -2016-06-13 19:45:15 INFO Executor:59 - Running task 1.0 in stage 250.0 (TID 201) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 250.0 (TID 200) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 250.0 (TID 200). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 250.0 (TID 200) in 5 ms on localhost (1/4) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 1.0 in stage 250.0 (TID 201). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 2.0 in stage 250.0 (TID 202). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 3.0 in stage 250.0 (TID 203). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 250.0 (TID 201) in 8 ms on localhost (2/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 250.0 (TID 202) in 9 ms on localhost (3/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 3.0 in stage 250.0 (TID 203) in 8 ms on localhost (4/4) -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 250 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 250.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 64 finished: reduce at LDAOptimizer.scala:453, took 0.018031 s -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 65 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 251(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 251 (MapPartitionsRDD[195] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=809342, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_97 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(8439) called with curMem=829638, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_97_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_97_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 97 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 251 (MapPartitionsRDD[195] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 251.0 with 4 tasks -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 251.0 (TID 204, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 251.0 (TID 205, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 251.0 (TID 206, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 251.0 (TID 207, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 2.0 in stage 251.0 (TID 206) -2016-06-13 19:45:15 INFO Executor:59 - Running task 1.0 in stage 251.0 (TID 205) -2016-06-13 19:45:15 INFO Executor:59 - Running task 3.0 in stage 251.0 (TID 207) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 251.0 (TID 204) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 251.0 (TID 204). 1002 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 251.0 (TID 204) in 6 ms on localhost (1/4) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 1.0 in stage 251.0 (TID 205). 1314 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 251.0 (TID 205) in 7 ms on localhost (2/4) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 3.0 in stage 251.0 (TID 207). 1446 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 3.0 in stage 251.0 (TID 207) in 8 ms on localhost (3/4) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 2.0 in stage 251.0 (TID 206). 1446 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 251.0 (TID 206) in 8 ms on localhost (4/4) -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 251 (reduce at LDAOptimizer.scala:456) finished in 0.009 s -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 251.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 65 finished: reduce at LDAOptimizer.scala:456, took 0.018354 s -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 66 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 252(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 252 (PartitionwiseSampledRDD[196] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=838077, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_98 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=856893, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_98_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_98_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 98 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 252 (PartitionwiseSampledRDD[196] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 252.0 with 1 tasks -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 252.0 (TID 208, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_97_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 252.0 (TID 208) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 252.0 (TID 208). 911 bytes result sent to driver -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 85 -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 252.0 (TID 208) in 7 ms on localhost (1/1) -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 252 (isEmpty at LDAOptimizer.scala:418) finished in 0.008 s -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 252.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 66 finished: isEmpty at LDAOptimizer.scala:418, took 0.033398 s -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_96_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 84 -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_95_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 67 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 253(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 253 (PartitionwiseSampledRDD[196] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=799015, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_99 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_94_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 83 -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=799015, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_99_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_93_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 82 -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_99_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 99 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 253 (PartitionwiseSampledRDD[196] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 253.0 with 3 tasks -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 253.0 (TID 209, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 253.0 (TID 210, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 253.0 (TID 211, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 1.0 in stage 253.0 (TID 210) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 253.0 (TID 209) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 1.0 in stage 253.0 (TID 210). 1226 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 253.0 (TID 209). 1226 bytes result sent to driver -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_92_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 253.0 (TID 210) in 6 ms on localhost (1/3) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 253.0 (TID 209) in 8 ms on localhost (2/3) -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 81 -2016-06-13 19:45:15 INFO Executor:59 - Running task 2.0 in stage 253.0 (TID 211) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 2.0 in stage 253.0 (TID 211). 1226 bytes result sent to driver -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_91_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 253.0 (TID 211) in 11 ms on localhost (3/3) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 253.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 253 (isEmpty at LDAOptimizer.scala:418) finished in 0.014 s -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 80 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 67 finished: isEmpty at LDAOptimizer.scala:418, took 0.032096 s -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=722575, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_100 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_90_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=722471, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_100_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_100_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 100 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 68 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 254(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 254 (MapPartitionsRDD[198] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=715094, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_101 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(8421) called with curMem=735350, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_101_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_89_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_101_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 101 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 254 (MapPartitionsRDD[198] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 254.0 with 4 tasks -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 254.0 (TID 212, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 254.0 (TID 213, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 254.0 (TID 214, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 254.0 (TID 215, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 2.0 in stage 254.0 (TID 214) -2016-06-13 19:45:15 INFO Executor:59 - Running task 3.0 in stage 254.0 (TID 215) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 2.0 in stage 254.0 (TID 214). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 3.0 in stage 254.0 (TID 215). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Running task 1.0 in stage 254.0 (TID 213) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 1.0 in stage 254.0 (TID 213). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 254.0 (TID 214) in 31 ms on localhost (1/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 3.0 in stage 254.0 (TID 215) in 30 ms on localhost (2/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 254.0 (TID 213) in 32 ms on localhost (3/4) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 254.0 (TID 212) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 254.0 (TID 212). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 254.0 (TID 212) in 43 ms on localhost (4/4) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 254.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 254 (reduce at LDAOptimizer.scala:453) finished in 0.047 s -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 79 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 68 finished: reduce at LDAOptimizer.scala:453, took 0.060047 s -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 69 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 255(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 255 (MapPartitionsRDD[199] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_88_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=716516, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_102 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 78 -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(8431) called with curMem=708090, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_102_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_87_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_102_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 77 -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 102 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 255 (MapPartitionsRDD[199] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 255.0 with 4 tasks -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 255.0 (TID 216, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 255.0 (TID 217, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 255.0 (TID 218, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 255.0 (TID 219, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 1.0 in stage 255.0 (TID 217) -2016-06-13 19:45:15 INFO Executor:59 - Running task 2.0 in stage 255.0 (TID 218) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_86_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:15 INFO Executor:59 - Running task 3.0 in stage 255.0 (TID 219) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 1.0 in stage 255.0 (TID 217). 1314 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 255.0 (TID 217) in 8 ms on localhost (1/4) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 3.0 in stage 255.0 (TID 219). 1314 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 2.0 in stage 255.0 (TID 218). 1358 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 3.0 in stage 255.0 (TID 219) in 8 ms on localhost (2/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 255.0 (TID 218) in 9 ms on localhost (3/4) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 255.0 (TID 216) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 255.0 (TID 216). 1002 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 255.0 (TID 216) in 14 ms on localhost (4/4) -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 255 (reduce at LDAOptimizer.scala:456) finished in 0.015 s -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 255.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 69 finished: reduce at LDAOptimizer.scala:456, took 0.029714 s -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_85_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 76 -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_84_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 70 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 256(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 75 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 256 (PartitionwiseSampledRDD[200] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=616387, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_103 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_83_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 74 -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=635203, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_103_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_103_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 103 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_82_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 256 (PartitionwiseSampledRDD[200] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 256.0 with 1 tasks -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 73 -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 256.0 (TID 220, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 256.0 (TID 220) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_81_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 72 -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 256.0 (TID 220). 1226 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 256.0 (TID 220) in 3 ms on localhost (1/1) -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 256 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 256.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 70 finished: isEmpty at LDAOptimizer.scala:418, took 0.012442 s -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_80_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=585013, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_104 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=585189, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_104_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_104_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 104 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_79_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 71 -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 71 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 257(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 257 (MapPartitionsRDD[202] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=530267, maxMem=766075207 -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_78_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_105 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 70 -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(8428) called with curMem=550523, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_105_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_105_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 105 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_77_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 257 (MapPartitionsRDD[202] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 257.0 with 4 tasks -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 69 -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 257.0 (TID 221, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 257.0 (TID 222, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 257.0 (TID 223, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_76_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 257.0 (TID 224, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 257.0 (TID 221) -2016-06-13 19:45:15 INFO Executor:59 - Running task 3.0 in stage 257.0 (TID 224) -2016-06-13 19:45:15 INFO Executor:59 - Running task 2.0 in stage 257.0 (TID 223) -2016-06-13 19:45:15 INFO Executor:59 - Running task 1.0 in stage 257.0 (TID 222) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 1.0 in stage 257.0 (TID 222). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_75_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 257.0 (TID 221). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 2.0 in stage 257.0 (TID 223). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 257.0 (TID 222) in 11 ms on localhost (1/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 257.0 (TID 221) in 13 ms on localhost (2/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 257.0 (TID 223) in 12 ms on localhost (3/4) -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-06-13 19:45:15 INFO Executor:59 - Finished task 3.0 in stage 257.0 (TID 224). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 3.0 in stage 257.0 (TID 224) in 13 ms on localhost (4/4) -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 257 (reduce at LDAOptimizer.scala:453) finished in 0.017 s -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 257.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 71 finished: reduce at LDAOptimizer.scala:453, took 0.028881 s -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_74_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_73_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 72 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 258(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 258 (MapPartitionsRDD[203] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_72_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=445490, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_106 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(8438) called with curMem=458130, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_106_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_71_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_106_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 106 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 258 (MapPartitionsRDD[203] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 258.0 with 4 tasks -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 258.0 (TID 225, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 258.0 (TID 226, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 258.0 (TID 227, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 258.0 (TID 228, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 2.0 in stage 258.0 (TID 227) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 258.0 (TID 225) -2016-06-13 19:45:15 INFO Executor:59 - Running task 1.0 in stage 258.0 (TID 226) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:15 INFO Executor:59 - Running task 3.0 in stage 258.0 (TID 228) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 2.0 in stage 258.0 (TID 227). 1002 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 1.0 in stage 258.0 (TID 226). 1002 bytes result sent to driver -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-06-13 19:45:15 INFO Executor:59 - Finished task 3.0 in stage 258.0 (TID 228). 1402 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 258.0 (TID 227) in 13 ms on localhost (1/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 258.0 (TID 226) in 14 ms on localhost (2/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 3.0 in stage 258.0 (TID 228) in 13 ms on localhost (3/4) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 258.0 (TID 225). 1314 bytes result sent to driver -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 258.0 (TID 225) in 19 ms on localhost (4/4) -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 258 (reduce at LDAOptimizer.scala:456) finished in 0.019 s -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 258.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 72 finished: reduce at LDAOptimizer.scala:456, took 0.036251 s -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 73 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 259(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 259 (PartitionwiseSampledRDD[204] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=354957, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_107 stored as values in memory (estimated size 18.4 KB, free 730.2 MB) -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=353477, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_107_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_107_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 107 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 259 (PartitionwiseSampledRDD[204] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 259.0 with 1 tasks -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 259.0 (TID 229, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 259.0 (TID 229) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 259.0 (TID 229). 1226 bytes result sent to driver -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 259.0 (TID 229) in 6 ms on localhost (1/1) -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 259 (isEmpty at LDAOptimizer.scala:418) finished in 0.007 s -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 73 finished: isEmpty at LDAOptimizer.scala:418, took 0.014174 s -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 259.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=324351, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_108 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=305711, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_108_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_108_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 108 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:15 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:50000 in memory (size: 279.0 B, free: 730.5 MB) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 74 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 260(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 260 (MapPartitionsRDD[206] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=248128, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_109 stored as values in memory (estimated size 19.8 KB, free 730.3 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(8426) called with curMem=268384, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_109_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.3 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_109_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 109 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 260 (MapPartitionsRDD[206] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 260.0 with 4 tasks -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 260.0 (TID 230, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 260.0 (TID 231, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 260.0 (TID 232, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 260.0 (TID 233, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 260.0 (TID 230) -2016-06-13 19:45:15 INFO Executor:59 - Running task 2.0 in stage 260.0 (TID 232) -2016-06-13 19:45:15 INFO Executor:59 - Running task 1.0 in stage 260.0 (TID 231) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 260.0 (TID 230). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 1.0 in stage 260.0 (TID 231). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Running task 3.0 in stage 260.0 (TID 233) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 3.0 in stage 260.0 (TID 233). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 260.0 (TID 230) in 13 ms on localhost (1/4) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 2.0 in stage 260.0 (TID 232). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 260.0 (TID 231) in 12 ms on localhost (2/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 3.0 in stage 260.0 (TID 233) in 12 ms on localhost (3/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 260.0 (TID 232) in 12 ms on localhost (4/4) -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 260 (reduce at LDAOptimizer.scala:453) finished in 0.014 s -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 260.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 74 finished: reduce at LDAOptimizer.scala:453, took 0.023958 s -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 75 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 261(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 261 (MapPartitionsRDD[207] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=276810, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_110 stored as values in memory (estimated size 19.8 KB, free 730.3 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(8436) called with curMem=297106, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_110_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.3 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_110_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 110 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 261 (MapPartitionsRDD[207] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 261.0 with 4 tasks -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 261.0 (TID 234, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 261.0 (TID 235, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 261.0 (TID 236, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 261.0 (TID 237, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 261.0 (TID 234) -2016-06-13 19:45:15 INFO Executor:59 - Running task 3.0 in stage 261.0 (TID 237) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 3.0 in stage 261.0 (TID 237). 1002 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 261.0 (TID 234). 1314 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 3.0 in stage 261.0 (TID 237) in 4 ms on localhost (1/4) -2016-06-13 19:45:15 INFO Executor:59 - Running task 2.0 in stage 261.0 (TID 236) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 261.0 (TID 234) in 7 ms on localhost (2/4) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 2.0 in stage 261.0 (TID 236). 1358 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 261.0 (TID 236) in 11 ms on localhost (3/4) -2016-06-13 19:45:15 INFO Executor:59 - Running task 1.0 in stage 261.0 (TID 235) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 1.0 in stage 261.0 (TID 235). 1446 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 261.0 (TID 235) in 56 ms on localhost (4/4) -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 261 (reduce at LDAOptimizer.scala:456) finished in 0.057 s -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 75 finished: reduce at LDAOptimizer.scala:456, took 0.066933 s -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 261.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 76 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 262(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 262 (PartitionwiseSampledRDD[208] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=305542, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_111 stored as values in memory (estimated size 18.4 KB, free 730.3 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(7659) called with curMem=324358, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_111_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.3 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_111_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 111 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 262 (PartitionwiseSampledRDD[208] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 262.0 with 1 tasks -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 262.0 (TID 238, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 262.0 (TID 238) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 262.0 (TID 238). 1226 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 262.0 (TID 238) in 40 ms on localhost (1/1) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 262.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 262 (isEmpty at LDAOptimizer.scala:418) finished in 0.042 s -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 76 finished: isEmpty at LDAOptimizer.scala:418, took 0.068557 s -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=332017, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_112 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=332193, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_112_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_112_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 112 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 77 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 263(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 263 (MapPartitionsRDD[210] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=332473, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_113 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(8423) called with curMem=352729, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_113_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_113_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 113 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 263 (MapPartitionsRDD[210] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 263.0 with 4 tasks -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 263.0 (TID 239, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 263.0 (TID 240, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 263.0 (TID 241, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 263.0 (TID 242, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 1.0 in stage 263.0 (TID 240) -2016-06-13 19:45:15 INFO Executor:59 - Running task 2.0 in stage 263.0 (TID 241) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 263.0 (TID 239) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 1.0 in stage 263.0 (TID 240). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 2.0 in stage 263.0 (TID 241). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Running task 3.0 in stage 263.0 (TID 242) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 263.0 (TID 240) in 11 ms on localhost (1/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 263.0 (TID 241) in 11 ms on localhost (2/4) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 263.0 (TID 239). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 3.0 in stage 263.0 (TID 242). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 263.0 (TID 239) in 17 ms on localhost (3/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 3.0 in stage 263.0 (TID 242) in 16 ms on localhost (4/4) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 263.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 263 (reduce at LDAOptimizer.scala:453) finished in 0.020 s -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 77 finished: reduce at LDAOptimizer.scala:453, took 0.029484 s -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 78 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 264(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 264 (MapPartitionsRDD[211] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=361152, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_114 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=381448, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_114_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_114_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 114 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 264 (MapPartitionsRDD[211] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 264.0 with 4 tasks -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 264.0 (TID 243, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 264.0 (TID 244, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 264.0 (TID 245, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 264.0 (TID 246, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 2.0 in stage 264.0 (TID 245) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 264.0 (TID 243) -2016-06-13 19:45:15 INFO Executor:59 - Running task 1.0 in stage 264.0 (TID 244) -2016-06-13 19:45:15 INFO Executor:59 - Running task 3.0 in stage 264.0 (TID 246) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 1.0 in stage 264.0 (TID 244). 1002 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 2.0 in stage 264.0 (TID 245). 1314 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 264.0 (TID 243). 1358 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 264.0 (TID 244) in 16 ms on localhost (1/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 264.0 (TID 245) in 15 ms on localhost (2/4) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 3.0 in stage 264.0 (TID 246). 1358 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 264.0 (TID 243) in 19 ms on localhost (3/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 3.0 in stage 264.0 (TID 246) in 18 ms on localhost (4/4) -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 264 (reduce at LDAOptimizer.scala:456) finished in 0.020 s -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 78 finished: reduce at LDAOptimizer.scala:456, took 0.045818 s -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 264.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 79 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 265(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 265 (PartitionwiseSampledRDD[212] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=389880, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_115 stored as values in memory (estimated size 18.4 KB, free 730.2 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(7658) called with curMem=408696, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_115_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_115_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 115 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 265 (PartitionwiseSampledRDD[212] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 265.0 with 1 tasks -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 265.0 (TID 247, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 265.0 (TID 247) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 265.0 (TID 247). 1226 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 265.0 (TID 247) in 6 ms on localhost (1/1) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 265.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 265 (isEmpty at LDAOptimizer.scala:418) finished in 0.007 s -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 79 finished: isEmpty at LDAOptimizer.scala:418, took 0.016794 s -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=416354, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_116 stored as values in memory (estimated size 176.0 B, free 730.2 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=416530, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_116_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.2 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_116_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 116 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 80 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 266(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 266 (MapPartitionsRDD[214] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=416810, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_117 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(8426) called with curMem=437066, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_117_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_117_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 117 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 266 (MapPartitionsRDD[214] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 266.0 with 4 tasks -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 266.0 (TID 248, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 266.0 (TID 249, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 266.0 (TID 250, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 266.0 (TID 251, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 266.0 (TID 248) -2016-06-13 19:45:15 INFO Executor:59 - Running task 1.0 in stage 266.0 (TID 249) -2016-06-13 19:45:15 INFO Executor:59 - Running task 3.0 in stage 266.0 (TID 251) -2016-06-13 19:45:15 INFO Executor:59 - Running task 2.0 in stage 266.0 (TID 250) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 2.0 in stage 266.0 (TID 250). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 3.0 in stage 266.0 (TID 251). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 266.0 (TID 248). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Finished task 1.0 in stage 266.0 (TID 249). 1265 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 266.0 (TID 248) in 15 ms on localhost (1/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 3.0 in stage 266.0 (TID 251) in 14 ms on localhost (2/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 266.0 (TID 249) in 15 ms on localhost (3/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 266.0 (TID 250) in 66 ms on localhost (4/4) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 266.0, whose tasks have all completed, from pool -2016-06-13 19:45:15 INFO DAGScheduler:59 - ResultStage 266 (reduce at LDAOptimizer.scala:453) finished in 0.069 s -2016-06-13 19:45:15 INFO DAGScheduler:59 - Job 80 finished: reduce at LDAOptimizer.scala:453, took 0.079320 s -2016-06-13 19:45:15 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Got job 81 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:15 INFO DAGScheduler:59 - Final stage: ResultStage 267(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting ResultStage 267 (MapPartitionsRDD[215] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=445492, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_118 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:15 INFO MemoryStore:59 - ensureFreeSpace(8436) called with curMem=465788, maxMem=766075207 -2016-06-13 19:45:15 INFO MemoryStore:59 - Block broadcast_118_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:15 INFO BlockManagerInfo:59 - Added broadcast_118_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:15 INFO SparkContext:59 - Created broadcast 118 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:15 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 267 (MapPartitionsRDD[215] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:15 INFO TaskSchedulerImpl:59 - Adding task set 267.0 with 4 tasks -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 267.0 (TID 252, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 267.0 (TID 253, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 2.0 in stage 267.0 (TID 254, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Starting task 3.0 in stage 267.0 (TID 255, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:15 INFO Executor:59 - Running task 0.0 in stage 267.0 (TID 252) -2016-06-13 19:45:15 INFO Executor:59 - Running task 2.0 in stage 267.0 (TID 254) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 2.0 in stage 267.0 (TID 254). 1314 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Running task 1.0 in stage 267.0 (TID 253) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 0.0 in stage 267.0 (TID 252). 1402 bytes result sent to driver -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 2.0 in stage 267.0 (TID 254) in 14 ms on localhost (1/4) -2016-06-13 19:45:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 267.0 (TID 252) in 17 ms on localhost (2/4) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 1.0 in stage 267.0 (TID 253). 1402 bytes result sent to driver -2016-06-13 19:45:15 INFO Executor:59 - Running task 3.0 in stage 267.0 (TID 255) -2016-06-13 19:45:15 INFO Executor:59 - Finished task 3.0 in stage 267.0 (TID 255). 1314 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 267.0 (TID 255) in 22 ms on localhost (3/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 267.0 (TID 253) in 25 ms on localhost (4/4) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 267.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 267 (reduce at LDAOptimizer.scala:456) finished in 0.028 s -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 81 finished: reduce at LDAOptimizer.scala:456, took 0.066346 s -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 82 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 268(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 268 (PartitionwiseSampledRDD[216] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=474224, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_119 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(7658) called with curMem=493040, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_119_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_119_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 119 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 268 (PartitionwiseSampledRDD[216] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 268.0 with 1 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 268.0 (TID 256, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 268.0 (TID 256) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 268.0 (TID 256). 911 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 268.0 (TID 256) in 4 ms on localhost (1/1) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 268.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 268 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 82 finished: isEmpty at LDAOptimizer.scala:418, took 0.015806 s -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 83 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 269(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 269 (PartitionwiseSampledRDD[216] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=500698, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_120 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(7658) called with curMem=519514, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_120_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_120_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 120 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 269 (PartitionwiseSampledRDD[216] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 269.0 with 3 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 269.0 (TID 257, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 269.0 (TID 258, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 269.0 (TID 259, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 269.0 (TID 257) -2016-06-13 19:45:16 INFO Executor:59 - Running task 2.0 in stage 269.0 (TID 259) -2016-06-13 19:45:16 INFO Executor:59 - Running task 1.0 in stage 269.0 (TID 258) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 269.0 (TID 257). 1226 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 2.0 in stage 269.0 (TID 259). 911 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 1.0 in stage 269.0 (TID 258). 1226 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 269.0 (TID 257) in 5 ms on localhost (1/3) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 269.0 (TID 259) in 4 ms on localhost (2/3) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 269.0 (TID 258) in 6 ms on localhost (3/3) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 269.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 269 (isEmpty at LDAOptimizer.scala:418) finished in 0.007 s -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 83 finished: isEmpty at LDAOptimizer.scala:418, took 0.021079 s -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=527172, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_121 stored as values in memory (estimated size 176.0 B, free 730.1 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=527348, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_121_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.1 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_121_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 121 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 84 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 270(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 270 (MapPartitionsRDD[218] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=527628, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_122 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(8435) called with curMem=547884, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_122_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_122_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 122 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 270 (MapPartitionsRDD[218] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 270.0 with 4 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 270.0 (TID 260, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 270.0 (TID 261, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 270.0 (TID 262, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 270.0 (TID 263, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 270.0 (TID 260) -2016-06-13 19:45:16 INFO Executor:59 - Running task 1.0 in stage 270.0 (TID 261) -2016-06-13 19:45:16 INFO Executor:59 - Running task 3.0 in stage 270.0 (TID 263) -2016-06-13 19:45:16 INFO Executor:59 - Running task 2.0 in stage 270.0 (TID 262) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 1.0 in stage 270.0 (TID 261). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 3.0 in stage 270.0 (TID 263). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 270.0 (TID 261) in 7 ms on localhost (1/4) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 270.0 (TID 260). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 270.0 (TID 263) in 6 ms on localhost (2/4) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 2.0 in stage 270.0 (TID 262). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 270.0 (TID 262) in 8 ms on localhost (3/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 270.0 (TID 260) in 10 ms on localhost (4/4) -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 270 (reduce at LDAOptimizer.scala:453) finished in 0.011 s -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 270.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 84 finished: reduce at LDAOptimizer.scala:453, took 0.020242 s -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 85 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 271(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 271 (MapPartitionsRDD[219] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=556319, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_123 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(8444) called with curMem=576615, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_123_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_123_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 123 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 271 (MapPartitionsRDD[219] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 271.0 with 4 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 271.0 (TID 264, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 271.0 (TID 265, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 271.0 (TID 266, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 271.0 (TID 267, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 3.0 in stage 271.0 (TID 267) -2016-06-13 19:45:16 INFO Executor:59 - Running task 1.0 in stage 271.0 (TID 265) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 271.0 (TID 264) -2016-06-13 19:45:16 INFO Executor:59 - Running task 2.0 in stage 271.0 (TID 266) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 3.0 in stage 271.0 (TID 267). 1002 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 2.0 in stage 271.0 (TID 266). 1314 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 271.0 (TID 267) in 7 ms on localhost (1/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 271.0 (TID 266) in 7 ms on localhost (2/4) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 1.0 in stage 271.0 (TID 265). 1358 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 271.0 (TID 264). 1002 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 271.0 (TID 265) in 11 ms on localhost (3/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 271.0 (TID 264) in 11 ms on localhost (4/4) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 271.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 271 (reduce at LDAOptimizer.scala:456) finished in 0.013 s -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 85 finished: reduce at LDAOptimizer.scala:456, took 0.026979 s -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 86 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 272(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 272 (PartitionwiseSampledRDD[220] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=585059, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_124 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=603875, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_124_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_124_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 124 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 272 (PartitionwiseSampledRDD[220] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 272.0 with 1 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 272.0 (TID 268, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 272.0 (TID 268) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 272.0 (TID 268). 1226 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 272.0 (TID 268) in 4 ms on localhost (1/1) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 272.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 272 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 86 finished: isEmpty at LDAOptimizer.scala:418, took 0.023173 s -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=611531, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_125 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=611707, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_125_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_125_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 125 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 87 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 273(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 273 (MapPartitionsRDD[222] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=611987, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_126 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(8424) called with curMem=632243, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_126_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_126_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 126 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 273 (MapPartitionsRDD[222] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 273.0 with 4 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 273.0 (TID 269, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 273.0 (TID 270, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 273.0 (TID 271, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 273.0 (TID 272, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 273.0 (TID 269) -2016-06-13 19:45:16 INFO Executor:59 - Running task 3.0 in stage 273.0 (TID 272) -2016-06-13 19:45:16 INFO Executor:59 - Running task 2.0 in stage 273.0 (TID 271) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 2.0 in stage 273.0 (TID 271). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 3.0 in stage 273.0 (TID 272). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 273.0 (TID 269). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Running task 1.0 in stage 273.0 (TID 270) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 1.0 in stage 273.0 (TID 270). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 273.0 (TID 272) in 14 ms on localhost (1/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 273.0 (TID 271) in 14 ms on localhost (2/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 273.0 (TID 269) in 17 ms on localhost (3/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 273.0 (TID 270) in 17 ms on localhost (4/4) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 273.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 273 (reduce at LDAOptimizer.scala:453) finished in 0.018 s -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 87 finished: reduce at LDAOptimizer.scala:453, took 0.029326 s -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 88 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 274(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 274 (MapPartitionsRDD[223] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=640667, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_127 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(8434) called with curMem=660963, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_127_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_127_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 127 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 274 (MapPartitionsRDD[223] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 274.0 with 4 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 274.0 (TID 273, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 274.0 (TID 274, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 274.0 (TID 275, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 274.0 (TID 276, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 274.0 (TID 273) -2016-06-13 19:45:16 INFO Executor:59 - Running task 2.0 in stage 274.0 (TID 275) -2016-06-13 19:45:16 INFO Executor:59 - Running task 3.0 in stage 274.0 (TID 276) -2016-06-13 19:45:16 INFO Executor:59 - Running task 1.0 in stage 274.0 (TID 274) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 2.0 in stage 274.0 (TID 275). 1002 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 3.0 in stage 274.0 (TID 276). 1314 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 274.0 (TID 273). 1358 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 274.0 (TID 275) in 7 ms on localhost (1/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 274.0 (TID 276) in 7 ms on localhost (2/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 274.0 (TID 273) in 10 ms on localhost (3/4) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 1.0 in stage 274.0 (TID 274). 1490 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 274.0 (TID 274) in 12 ms on localhost (4/4) -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 274 (reduce at LDAOptimizer.scala:456) finished in 0.021 s -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 88 finished: reduce at LDAOptimizer.scala:456, took 0.030868 s -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 274.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 89 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 275(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 275 (PartitionwiseSampledRDD[224] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=669397, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_128 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=688213, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_128_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_128_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 128 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 275 (PartitionwiseSampledRDD[224] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 275.0 with 1 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 275.0 (TID 277, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 275.0 (TID 277) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 275.0 (TID 277). 911 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 275.0 (TID 277) in 4 ms on localhost (1/1) -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 275 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 275.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 89 finished: isEmpty at LDAOptimizer.scala:418, took 0.038283 s -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 90 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 276(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 276 (PartitionwiseSampledRDD[224] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=695869, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_129 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=714685, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_129_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_129_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 129 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 276 (PartitionwiseSampledRDD[224] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 276.0 with 3 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 276.0 (TID 278, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 276.0 (TID 279, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 276.0 (TID 280, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 2.0 in stage 276.0 (TID 280) -2016-06-13 19:45:16 INFO Executor:59 - Running task 1.0 in stage 276.0 (TID 279) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 1.0 in stage 276.0 (TID 279). 1226 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 276.0 (TID 278) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 2.0 in stage 276.0 (TID 280). 1226 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 276.0 (TID 279) in 5 ms on localhost (1/3) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 276.0 (TID 280) in 5 ms on localhost (2/3) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 276.0 (TID 278). 1226 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 276.0 (TID 278) in 9 ms on localhost (3/3) -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 276 (isEmpty at LDAOptimizer.scala:418) finished in 0.009 s -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 90 finished: isEmpty at LDAOptimizer.scala:418, took 0.027435 s -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 276.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=722341, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_130 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=722517, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_130_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_130_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 130 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 91 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 277(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 277 (MapPartitionsRDD[226] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=722797, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_131 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(8425) called with curMem=743053, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_131_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_131_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 131 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 277 (MapPartitionsRDD[226] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 277.0 with 4 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 277.0 (TID 281, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 277.0 (TID 282, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 277.0 (TID 283, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 277.0 (TID 284, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 277.0 (TID 281) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 277.0 (TID 281). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Running task 2.0 in stage 277.0 (TID 283) -2016-06-13 19:45:16 INFO Executor:59 - Running task 3.0 in stage 277.0 (TID 284) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 2.0 in stage 277.0 (TID 283). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 3.0 in stage 277.0 (TID 284). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 277.0 (TID 281) in 13 ms on localhost (1/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 277.0 (TID 283) in 12 ms on localhost (2/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 277.0 (TID 284) in 13 ms on localhost (3/4) -2016-06-13 19:45:16 INFO Executor:59 - Running task 1.0 in stage 277.0 (TID 282) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 1.0 in stage 277.0 (TID 282). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 277.0 (TID 282) in 113 ms on localhost (4/4) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 277.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 277 (reduce at LDAOptimizer.scala:453) finished in 0.119 s -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 91 finished: reduce at LDAOptimizer.scala:453, took 0.129822 s -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 92 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 278(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 278 (MapPartitionsRDD[227] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=751478, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_132 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(8435) called with curMem=771774, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_132_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_132_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 132 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 278 (MapPartitionsRDD[227] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 278.0 with 4 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 278.0 (TID 285, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 278.0 (TID 286, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 278.0 (TID 287, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 278.0 (TID 288, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 2.0 in stage 278.0 (TID 287) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 278.0 (TID 285) -2016-06-13 19:45:16 INFO Executor:59 - Running task 3.0 in stage 278.0 (TID 288) -2016-06-13 19:45:16 INFO Executor:59 - Running task 1.0 in stage 278.0 (TID 286) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 278.0 (TID 285). 1002 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 2.0 in stage 278.0 (TID 287). 1314 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 3.0 in stage 278.0 (TID 288). 1314 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 278.0 (TID 285) in 10 ms on localhost (1/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 278.0 (TID 287) in 10 ms on localhost (2/4) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 1.0 in stage 278.0 (TID 286). 1358 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 278.0 (TID 288) in 14 ms on localhost (3/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 278.0 (TID 286) in 14 ms on localhost (4/4) -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 278 (reduce at LDAOptimizer.scala:456) finished in 0.018 s -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 92 finished: reduce at LDAOptimizer.scala:456, took 0.026628 s -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 278.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 93 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 279(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 279 (PartitionwiseSampledRDD[228] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=780209, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_133 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(7655) called with curMem=799025, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_133_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_133_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 133 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 279 (PartitionwiseSampledRDD[228] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 279.0 with 1 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 279.0 (TID 289, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 279.0 (TID 289) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 279.0 (TID 289). 1226 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 279.0 (TID 289) in 4 ms on localhost (1/1) -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 279 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 279.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 93 finished: isEmpty at LDAOptimizer.scala:418, took 0.012020 s -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=806680, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_134 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=806856, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_134_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.8 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_134_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 134 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 94 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 280(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 280 (MapPartitionsRDD[230] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=807136, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_135 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(8422) called with curMem=827392, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_135_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_135_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 135 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 280 (MapPartitionsRDD[230] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 280.0 with 4 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 280.0 (TID 290, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 280.0 (TID 291, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 280.0 (TID 292, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 280.0 (TID 293, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 1.0 in stage 280.0 (TID 291) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 280.0 (TID 290) -2016-06-13 19:45:16 INFO Executor:59 - Running task 3.0 in stage 280.0 (TID 293) -2016-06-13 19:45:16 INFO Executor:59 - Running task 2.0 in stage 280.0 (TID 292) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 2.0 in stage 280.0 (TID 292). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 280.0 (TID 290). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 280.0 (TID 292) in 6 ms on localhost (1/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 280.0 (TID 290) in 9 ms on localhost (2/4) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 3.0 in stage 280.0 (TID 293). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 280.0 (TID 293) in 8 ms on localhost (3/4) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 1.0 in stage 280.0 (TID 291). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 280.0 (TID 291) in 10 ms on localhost (4/4) -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 280 (reduce at LDAOptimizer.scala:453) finished in 0.012 s -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 280.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 94 finished: reduce at LDAOptimizer.scala:453, took 0.021602 s -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 95 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 281(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 281 (MapPartitionsRDD[231] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:16 INFO ContextCleaner:59 - Cleaned accumulator 110 -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=835814, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_136 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=848454, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_136_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_128_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_136_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 136 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 281 (MapPartitionsRDD[231] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 281.0 with 4 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 281.0 (TID 294, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 281.0 (TID 295, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 281.0 (TID 296, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 281.0 (TID 297, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 281.0 (TID 294) -2016-06-13 19:45:16 INFO Executor:59 - Running task 1.0 in stage 281.0 (TID 295) -2016-06-13 19:45:16 INFO Executor:59 - Running task 2.0 in stage 281.0 (TID 296) -2016-06-13 19:45:16 INFO Executor:59 - Running task 3.0 in stage 281.0 (TID 297) -2016-06-13 19:45:16 INFO ContextCleaner:59 - Cleaned accumulator 109 -2016-06-13 19:45:16 INFO Executor:59 - Finished task 2.0 in stage 281.0 (TID 296). 1002 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 281.0 (TID 296) in 5 ms on localhost (1/4) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 281.0 (TID 294). 1314 bytes result sent to driver -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_127_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 281.0 (TID 294) in 9 ms on localhost (2/4) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 3.0 in stage 281.0 (TID 297). 1314 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 1.0 in stage 281.0 (TID 295). 1446 bytes result sent to driver -2016-06-13 19:45:16 INFO ContextCleaner:59 - Cleaned accumulator 108 -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 281.0 (TID 297) in 9 ms on localhost (3/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 281.0 (TID 295) in 11 ms on localhost (4/4) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 281.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 281 (reduce at LDAOptimizer.scala:456) finished in 0.016 s -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 95 finished: reduce at LDAOptimizer.scala:456, took 0.034531 s -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_126_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 96 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 282(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 282 (PartitionwiseSampledRDD[232] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=780660, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_137 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(7655) called with curMem=799476, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_137_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_137_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 137 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 282 (PartitionwiseSampledRDD[232] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 282.0 with 1 tasks -2016-06-13 19:45:16 INFO ContextCleaner:59 - Cleaned accumulator 107 -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 282.0 (TID 298, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 282.0 (TID 298) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 282.0 (TID 298). 1226 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 282.0 (TID 298) in 4 ms on localhost (1/1) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 282.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 282 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 96 finished: isEmpty at LDAOptimizer.scala:418, took 0.017448 s -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_125_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=806675, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_138 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=806851, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_138_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.8 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_138_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 138 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 97 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 283(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 283 (MapPartitionsRDD[234] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_124_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=780659, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_139 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(8424) called with curMem=800915, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_139_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_139_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 139 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 283 (MapPartitionsRDD[234] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 283.0 with 4 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 283.0 (TID 299, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 283.0 (TID 300, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 283.0 (TID 301, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 283.0 (TID 302, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 283.0 (TID 299) -2016-06-13 19:45:16 INFO Executor:59 - Running task 3.0 in stage 283.0 (TID 302) -2016-06-13 19:45:16 INFO Executor:59 - Running task 1.0 in stage 283.0 (TID 300) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 3.0 in stage 283.0 (TID 302). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 283.0 (TID 299). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO ContextCleaner:59 - Cleaned accumulator 106 -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 283.0 (TID 299) in 9 ms on localhost (1/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 283.0 (TID 302) in 7 ms on localhost (2/4) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 1.0 in stage 283.0 (TID 300). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Running task 2.0 in stage 283.0 (TID 301) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 2.0 in stage 283.0 (TID 301). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_123_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 283.0 (TID 300) in 16 ms on localhost (3/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 283.0 (TID 301) in 15 ms on localhost (4/4) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 283.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 283 (reduce at LDAOptimizer.scala:453) finished in 0.019 s -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 97 finished: reduce at LDAOptimizer.scala:453, took 0.027593 s -2016-06-13 19:45:16 INFO ContextCleaner:59 - Cleaned accumulator 105 -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 98 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_122_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 284(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 284 (MapPartitionsRDD[235] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=751908, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_140 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(8434) called with curMem=772204, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_140_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_140_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 140 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 284 (MapPartitionsRDD[235] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:16 INFO ContextCleaner:59 - Cleaned accumulator 104 -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 284.0 with 4 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 284.0 (TID 303, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 284.0 (TID 304, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 284.0 (TID 305, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 284.0 (TID 306, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 284.0 (TID 303) -2016-06-13 19:45:16 INFO Executor:59 - Running task 3.0 in stage 284.0 (TID 306) -2016-06-13 19:45:16 INFO Executor:59 - Running task 2.0 in stage 284.0 (TID 305) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_121_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:16 INFO Executor:59 - Running task 1.0 in stage 284.0 (TID 304) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 2.0 in stage 284.0 (TID 305). 1314 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 284.0 (TID 303). 1358 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 284.0 (TID 305) in 7 ms on localhost (1/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 284.0 (TID 303) in 9 ms on localhost (2/4) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 3.0 in stage 284.0 (TID 306). 1358 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 1.0 in stage 284.0 (TID 304). 1314 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 284.0 (TID 306) in 10 ms on localhost (3/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 284.0 (TID 304) in 12 ms on localhost (4/4) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 284.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 284 (reduce at LDAOptimizer.scala:456) finished in 0.020 s -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 98 finished: reduce at LDAOptimizer.scala:456, took 0.038679 s -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_120_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 99 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 285(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO ContextCleaner:59 - Cleaned accumulator 103 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 285 (PartitionwiseSampledRDD[236] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=753708, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_141 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(7658) called with curMem=772524, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_141_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_141_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 141 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_119_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 285 (PartitionwiseSampledRDD[236] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 285.0 with 1 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 285.0 (TID 307, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 285.0 (TID 307) -2016-06-13 19:45:16 INFO ContextCleaner:59 - Cleaned accumulator 102 -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 285.0 (TID 307). 1226 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 285.0 (TID 307) in 4 ms on localhost (1/1) -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 285 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 285.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 99 finished: isEmpty at LDAOptimizer.scala:418, took 0.015096 s -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_118_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=745272, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_142 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=745448, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_142_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_142_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:16 INFO ContextCleaner:59 - Cleaned accumulator 101 -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 142 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 100 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 286(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 286 (MapPartitionsRDD[238] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_117_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=717006, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_143 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(8424) called with curMem=737262, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_143_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_143_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 143 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 286 (MapPartitionsRDD[238] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 286.0 with 4 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 286.0 (TID 308, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 286.0 (TID 309, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 286.0 (TID 310, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 286.0 (TID 311, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 1.0 in stage 286.0 (TID 309) -2016-06-13 19:45:16 INFO Executor:59 - Running task 3.0 in stage 286.0 (TID 311) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 286.0 (TID 308) -2016-06-13 19:45:16 INFO Executor:59 - Running task 2.0 in stage 286.0 (TID 310) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 1.0 in stage 286.0 (TID 309). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 3.0 in stage 286.0 (TID 311). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 286.0 (TID 309) in 6 ms on localhost (1/4) -2016-06-13 19:45:16 INFO ContextCleaner:59 - Cleaned accumulator 100 -2016-06-13 19:45:16 INFO Executor:59 - Finished task 2.0 in stage 286.0 (TID 310). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 286.0 (TID 308). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 286.0 (TID 311) in 10 ms on localhost (2/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 286.0 (TID 310) in 11 ms on localhost (3/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 286.0 (TID 308) in 13 ms on localhost (4/4) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 286.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 286 (reduce at LDAOptimizer.scala:453) finished in 0.013 s -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 100 finished: reduce at LDAOptimizer.scala:453, took 0.023294 s -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_116_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 101 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 287(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 287 (MapPartitionsRDD[239] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=724974, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_144 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(8434) called with curMem=745270, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_144_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_144_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 144 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 287 (MapPartitionsRDD[239] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 287.0 with 4 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 287.0 (TID 312, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 287.0 (TID 313, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_115_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 287.0 (TID 314, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 287.0 (TID 315, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 287.0 (TID 312) -2016-06-13 19:45:16 INFO Executor:59 - Running task 3.0 in stage 287.0 (TID 315) -2016-06-13 19:45:16 INFO Executor:59 - Running task 2.0 in stage 287.0 (TID 314) -2016-06-13 19:45:16 INFO ContextCleaner:59 - Cleaned accumulator 99 -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 287.0 (TID 312). 1314 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Running task 1.0 in stage 287.0 (TID 313) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 2.0 in stage 287.0 (TID 314). 1314 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 3.0 in stage 287.0 (TID 315). 1402 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 287.0 (TID 312) in 12 ms on localhost (1/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 287.0 (TID 314) in 10 ms on localhost (2/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 287.0 (TID 315) in 9 ms on localhost (3/4) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 1.0 in stage 287.0 (TID 313). 1358 bytes result sent to driver -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_114_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 287.0 (TID 313) in 15 ms on localhost (4/4) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 287.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 287 (reduce at LDAOptimizer.scala:456) finished in 0.016 s -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 101 finished: reduce at LDAOptimizer.scala:456, took 0.031893 s -2016-06-13 19:45:16 INFO ContextCleaner:59 - Cleaned accumulator 98 -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_113_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 102 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:16 INFO ContextCleaner:59 - Cleaned accumulator 97 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 288(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 288 (PartitionwiseSampledRDD[240] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_112_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=669367, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_145 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_111_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO ContextCleaner:59 - Cleaned accumulator 96 -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=653272, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_145_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_110_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_145_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 145 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 288 (PartitionwiseSampledRDD[240] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 288.0 with 1 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 288.0 (TID 316, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 288.0 (TID 316) -2016-06-13 19:45:16 INFO ContextCleaner:59 - Cleaned accumulator 95 -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 288.0 (TID 316). 1226 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 288.0 (TID 316) in 4 ms on localhost (1/1) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 288.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 288 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 102 finished: isEmpty at LDAOptimizer.scala:418, took 0.025780 s -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_109_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=611950, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_146 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=612126, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_146_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_146_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:16 INFO ContextCleaner:59 - Cleaned accumulator 94 -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 146 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 103 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 289(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 289 (MapPartitionsRDD[242] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=611950, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_147 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(8423) called with curMem=632206, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_147_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_108_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_147_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 147 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 289 (MapPartitionsRDD[242] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 289.0 with 4 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 289.0 (TID 317, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 289.0 (TID 318, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 289.0 (TID 319, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 289.0 (TID 320, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 289.0 (TID 317) -2016-06-13 19:45:16 INFO Executor:59 - Running task 1.0 in stage 289.0 (TID 318) -2016-06-13 19:45:16 INFO Executor:59 - Running task 3.0 in stage 289.0 (TID 320) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_107_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO Executor:59 - Running task 2.0 in stage 289.0 (TID 319) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 289.0 (TID 317). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 1.0 in stage 289.0 (TID 318). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 2.0 in stage 289.0 (TID 319). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 3.0 in stage 289.0 (TID 320). 1265 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 289.0 (TID 317) in 11 ms on localhost (1/4) -2016-06-13 19:45:16 INFO ContextCleaner:59 - Cleaned accumulator 93 -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 289.0 (TID 320) in 10 ms on localhost (2/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 289.0 (TID 319) in 16 ms on localhost (3/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 289.0 (TID 318) in 17 ms on localhost (4/4) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 289.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 289 (reduce at LDAOptimizer.scala:453) finished in 0.019 s -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 103 finished: reduce at LDAOptimizer.scala:453, took 0.027732 s -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_106_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 104 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 290(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 290 (MapPartitionsRDD[243] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=585422, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_148 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=605718, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_148_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_148_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 148 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 290 (MapPartitionsRDD[243] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 290.0 with 4 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 290.0 (TID 321, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 290.0 (TID 322, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 290.0 (TID 323, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 3.0 in stage 290.0 (TID 324, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 1.0 in stage 290.0 (TID 322) -2016-06-13 19:45:16 INFO Executor:59 - Running task 3.0 in stage 290.0 (TID 324) -2016-06-13 19:45:16 INFO Executor:59 - Running task 2.0 in stage 290.0 (TID 323) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 290.0 (TID 321) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 2.0 in stage 290.0 (TID 323). 1002 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 2.0 in stage 290.0 (TID 323) in 5 ms on localhost (1/4) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 1.0 in stage 290.0 (TID 322). 1358 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 290.0 (TID 322) in 8 ms on localhost (2/4) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 3.0 in stage 290.0 (TID 324). 1402 bytes result sent to driver -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 290.0 (TID 321). 1314 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 290.0 (TID 321) in 10 ms on localhost (3/4) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 3.0 in stage 290.0 (TID 324) in 8 ms on localhost (4/4) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 290.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 290 (reduce at LDAOptimizer.scala:456) finished in 0.011 s -2016-06-13 19:45:16 INFO ContextCleaner:59 - Cleaned accumulator 92 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 104 finished: reduce at LDAOptimizer.scala:456, took 0.024092 s -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 105 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 291(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 291 (PartitionwiseSampledRDD[244] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_105_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=605723, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_149 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=624539, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_149_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_149_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 149 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 291 (PartitionwiseSampledRDD[244] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 291.0 with 1 tasks -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 291.0 (TID 325, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 291.0 (TID 325) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 0.0 in stage 291.0 (TID 325). 911 bytes result sent to driver -2016-06-13 19:45:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 291.0 (TID 325) in 4 ms on localhost (1/1) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 291.0, whose tasks have all completed, from pool -2016-06-13 19:45:16 INFO DAGScheduler:59 - ResultStage 291 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:16 INFO DAGScheduler:59 - Job 105 finished: isEmpty at LDAOptimizer.scala:418, took 0.018476 s -2016-06-13 19:45:16 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Got job 106 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:16 INFO DAGScheduler:59 - Final stage: ResultStage 292(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting ResultStage 292 (PartitionwiseSampledRDD[244] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=611940, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_150 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:16 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=630756, maxMem=766075207 -2016-06-13 19:45:16 INFO MemoryStore:59 - Block broadcast_150_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Added broadcast_150_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:16 INFO SparkContext:59 - Created broadcast 150 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:16 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 292 (PartitionwiseSampledRDD[244] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:16 INFO TaskSchedulerImpl:59 - Adding task set 292.0 with 3 tasks -2016-06-13 19:45:16 INFO ContextCleaner:59 - Cleaned accumulator 91 -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 292.0 (TID 326, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 292.0 (TID 327, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:16 INFO TaskSetManager:59 - Starting task 2.0 in stage 292.0 (TID 328, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:16 INFO Executor:59 - Running task 0.0 in stage 292.0 (TID 326) -2016-06-13 19:45:16 INFO Executor:59 - Running task 1.0 in stage 292.0 (TID 327) -2016-06-13 19:45:16 INFO Executor:59 - Running task 2.0 in stage 292.0 (TID 328) -2016-06-13 19:45:16 INFO BlockManagerInfo:59 - Removed broadcast_104_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:16 INFO Executor:59 - Finished task 1.0 in stage 292.0 (TID 327). 911 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 292.0 (TID 326). 1226 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 292.0 (TID 328). 1226 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 292.0 (TID 327) in 9 ms on localhost (1/3) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 292.0 (TID 326) in 10 ms on localhost (2/3) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 292.0 (TID 328) in 10 ms on localhost (3/3) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 292.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 292 (isEmpty at LDAOptimizer.scala:418) finished in 0.011 s -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 106 finished: isEmpty at LDAOptimizer.scala:418, took 0.028090 s -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=638133, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_151 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=638133, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_151_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_151_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 151 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 107 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 293(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 293 (MapPartitionsRDD[246] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=638413, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_152 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(8425) called with curMem=658669, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_152_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_152_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 152 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 293 (MapPartitionsRDD[246] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 293.0 with 4 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 293.0 (TID 329, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 293.0 (TID 330, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 293.0 (TID 331, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 293.0 (TID 332, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 293.0 (TID 329) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 293.0 (TID 331) -2016-06-13 19:45:17 INFO Executor:59 - Running task 3.0 in stage 293.0 (TID 332) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 293.0 (TID 330) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 293.0 (TID 329). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 293.0 (TID 331). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 3.0 in stage 293.0 (TID 332). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 293.0 (TID 329) in 10 ms on localhost (1/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 293.0 (TID 331) in 10 ms on localhost (2/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 293.0 (TID 330). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 293.0 (TID 332) in 9 ms on localhost (3/4) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_103_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 293.0 (TID 330) in 13 ms on localhost (4/4) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 293.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 293 (reduce at LDAOptimizer.scala:453) finished in 0.015 s -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 107 finished: reduce at LDAOptimizer.scala:453, took 0.059119 s -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 90 -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_102_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 108 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 294(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 294 (MapPartitionsRDD[247] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=611894, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_153 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 89 -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(8435) called with curMem=632190, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_153_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_153_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 153 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 294 (MapPartitionsRDD[247] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 294.0 with 4 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 294.0 (TID 333, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 294.0 (TID 334, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 294.0 (TID 335, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 294.0 (TID 336, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 3.0 in stage 294.0 (TID 336) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_101_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 294.0 (TID 333) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 294.0 (TID 333). 1002 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 294.0 (TID 335) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 3.0 in stage 294.0 (TID 336). 1358 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 294.0 (TID 333) in 7 ms on localhost (1/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 294.0 (TID 335). 1002 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 294.0 (TID 334) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 294.0 (TID 334). 1490 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 294.0 (TID 335) in 29 ms on localhost (2/4) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 88 -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 294.0 (TID 334) in 30 ms on localhost (3/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 294.0 (TID 336) in 32 ms on localhost (4/4) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 294 (reduce at LDAOptimizer.scala:456) finished in 0.042 s -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 108 finished: reduce at LDAOptimizer.scala:456, took 0.054198 s -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 109 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 295(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 295 (PartitionwiseSampledRDD[248] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=611492, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_154 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=630308, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_154_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_100_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 294.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_154_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 154 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 295 (PartitionwiseSampledRDD[248] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 295.0 with 1 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 295.0 (TID 337, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_99_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 295.0 (TID 337) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 87 -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 295.0 (TID 337). 911 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 295.0 (TID 337) in 5 ms on localhost (1/1) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 295.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 295 (isEmpty at LDAOptimizer.scala:418) finished in 0.006 s -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 109 finished: isEmpty at LDAOptimizer.scala:418, took 0.021358 s -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_98_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 110 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 296(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 296 (PartitionwiseSampledRDD[248] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 86 -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=585020, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_155 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=575158, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_155_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_135_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 114 -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_155_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 155 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 296 (PartitionwiseSampledRDD[248] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 296.0 with 3 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 296.0 (TID 338, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 296.0 (TID 339, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 296.0 (TID 340, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 296.0 (TID 340) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_133_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 296.0 (TID 338) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 296.0 (TID 339) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 113 -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 296.0 (TID 340). 1226 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 296.0 (TID 339). 1226 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 296.0 (TID 338). 1226 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 296.0 (TID 340) in 5 ms on localhost (1/3) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 296.0 (TID 339) in 7 ms on localhost (2/3) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 296.0 (TID 338) in 9 ms on localhost (3/3) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 296.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_132_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 296 (isEmpty at LDAOptimizer.scala:418) finished in 0.010 s -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 110 finished: isEmpty at LDAOptimizer.scala:418, took 0.021036 s -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=527612, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_156 stored as values in memory (estimated size 176.0 B, free 730.1 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 112 -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=527788, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_156_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.1 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_156_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 156 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_131_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 111 -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_130_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 111 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 297(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_129_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 297 (MapPartitionsRDD[250] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=472459, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_157 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(8422) called with curMem=492715, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_157_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_157_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 157 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 297 (MapPartitionsRDD[250] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 297.0 with 4 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 297.0 (TID 341, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 297.0 (TID 342, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 297.0 (TID 343, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 297.0 (TID 344, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 297.0 (TID 341) -2016-06-13 19:45:17 INFO Executor:59 - Running task 3.0 in stage 297.0 (TID 344) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 297.0 (TID 342) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 297.0 (TID 343) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 297.0 (TID 341). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 297.0 (TID 342). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 297.0 (TID 343). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 297.0 (TID 341) in 10 ms on localhost (1/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 297.0 (TID 343) in 10 ms on localhost (2/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 297.0 (TID 342) in 10 ms on localhost (3/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 3.0 in stage 297.0 (TID 344). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 297.0 (TID 344) in 14 ms on localhost (4/4) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 297 (reduce at LDAOptimizer.scala:453) finished in 0.016 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 297.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 111 finished: reduce at LDAOptimizer.scala:453, took 0.026479 s -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 112 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 298(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 298 (MapPartitionsRDD[251] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=501137, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_158 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(8435) called with curMem=521433, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_158_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_158_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 158 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 298 (MapPartitionsRDD[251] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 298.0 with 4 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 298.0 (TID 345, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 298.0 (TID 346, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 298.0 (TID 347, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 298.0 (TID 348, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 298.0 (TID 345) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 298.0 (TID 347) -2016-06-13 19:45:17 INFO Executor:59 - Running task 3.0 in stage 298.0 (TID 348) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 298.0 (TID 346) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 298.0 (TID 345). 1002 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 298.0 (TID 345) in 5 ms on localhost (1/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 298.0 (TID 347). 1314 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 298.0 (TID 346). 1358 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 298.0 (TID 347) in 7 ms on localhost (2/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 298.0 (TID 346) in 7 ms on localhost (3/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 3.0 in stage 298.0 (TID 348). 1402 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 298.0 (TID 348) in 10 ms on localhost (4/4) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 298 (reduce at LDAOptimizer.scala:456) finished in 0.013 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 298.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 112 finished: reduce at LDAOptimizer.scala:456, took 0.019974 s -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 113 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 299(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 299 (PartitionwiseSampledRDD[252] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=529868, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_159 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(7655) called with curMem=548684, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_159_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_159_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 159 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 299 (PartitionwiseSampledRDD[252] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 299.0 with 1 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 299.0 (TID 349, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 299.0 (TID 349) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 299.0 (TID 349). 911 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 299.0 (TID 349) in 3 ms on localhost (1/1) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 299 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 299.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 113 finished: isEmpty at LDAOptimizer.scala:418, took 0.010633 s -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 114 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 300(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 300 (PartitionwiseSampledRDD[252] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=556339, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_160 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(7655) called with curMem=575155, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_160_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_160_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 160 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 300 (PartitionwiseSampledRDD[252] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 300.0 with 3 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 300.0 (TID 350, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 300.0 (TID 351, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 300.0 (TID 352, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 300.0 (TID 351) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 300.0 (TID 350) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 300.0 (TID 352) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 300.0 (TID 352). 1226 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 300.0 (TID 351). 911 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 300.0 (TID 352) in 3 ms on localhost (1/3) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 300.0 (TID 350). 1226 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 300.0 (TID 351) in 3 ms on localhost (2/3) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 300.0 (TID 350) in 5 ms on localhost (3/3) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 300 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 300.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 114 finished: isEmpty at LDAOptimizer.scala:418, took 0.010984 s -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=582810, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_161 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=582986, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_161_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_161_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 161 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 115 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 301(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 301 (MapPartitionsRDD[254] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=583266, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_162 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(8417) called with curMem=603522, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_162_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_162_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 162 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 301 (MapPartitionsRDD[254] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 301.0 with 4 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 301.0 (TID 353, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 301.0 (TID 354, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 301.0 (TID 355, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 301.0 (TID 356, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 301.0 (TID 354) -2016-06-13 19:45:17 INFO Executor:59 - Running task 3.0 in stage 301.0 (TID 356) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 301.0 (TID 353) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 301.0 (TID 355) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 301.0 (TID 353). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 301.0 (TID 355). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 301.0 (TID 353) in 6 ms on localhost (1/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 301.0 (TID 355) in 7 ms on localhost (2/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 3.0 in stage 301.0 (TID 356). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 301.0 (TID 354). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 301.0 (TID 356) in 9 ms on localhost (3/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 301.0 (TID 354) in 10 ms on localhost (4/4) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 301 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 301.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 115 finished: reduce at LDAOptimizer.scala:453, took 0.018003 s -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 116 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 302(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 302 (MapPartitionsRDD[255] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=611939, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_163 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(8430) called with curMem=632235, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_163_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_163_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 163 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 302 (MapPartitionsRDD[255] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 302.0 with 4 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 302.0 (TID 357, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 302.0 (TID 358, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 302.0 (TID 359, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 302.0 (TID 360, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 302.0 (TID 358) -2016-06-13 19:45:17 INFO Executor:59 - Running task 3.0 in stage 302.0 (TID 360) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 302.0 (TID 357) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 302.0 (TID 359) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 302.0 (TID 357). 1002 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 302.0 (TID 359). 1002 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 302.0 (TID 357) in 5 ms on localhost (1/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 302.0 (TID 359) in 5 ms on localhost (2/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 302.0 (TID 358). 1314 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 302.0 (TID 358) in 7 ms on localhost (3/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 3.0 in stage 302.0 (TID 360). 1358 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 302.0 (TID 360) in 13 ms on localhost (4/4) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 302.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 302 (reduce at LDAOptimizer.scala:456) finished in 0.015 s -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 116 finished: reduce at LDAOptimizer.scala:456, took 0.021437 s -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 117 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 303(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 303 (PartitionwiseSampledRDD[256] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=640665, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_164 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(7655) called with curMem=659481, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_164_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_164_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 164 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 303 (PartitionwiseSampledRDD[256] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 303.0 with 1 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 303.0 (TID 361, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 303.0 (TID 361) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 303.0 (TID 361). 911 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 303.0 (TID 361) in 5 ms on localhost (1/1) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 303.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 303 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 117 finished: isEmpty at LDAOptimizer.scala:418, took 0.013707 s -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 118 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 304(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 304 (PartitionwiseSampledRDD[256] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=667136, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_165 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(7655) called with curMem=685952, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_165_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_165_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 165 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 304 (PartitionwiseSampledRDD[256] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 304.0 with 3 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 304.0 (TID 362, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 304.0 (TID 363, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 304.0 (TID 364, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 304.0 (TID 362) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 304.0 (TID 364) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 304.0 (TID 363) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 304.0 (TID 364). 1226 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 304.0 (TID 363). 1226 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 304.0 (TID 362). 911 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 304.0 (TID 364) in 8 ms on localhost (1/3) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 304.0 (TID 362) in 10 ms on localhost (2/3) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 304.0 (TID 363) in 9 ms on localhost (3/3) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 304 (isEmpty at LDAOptimizer.scala:418) finished in 0.011 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 304.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 118 finished: isEmpty at LDAOptimizer.scala:418, took 0.019814 s -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=693607, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_166 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=693783, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_166_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_166_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 166 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 119 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 305(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 305 (MapPartitionsRDD[258] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=694063, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_167 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(8438) called with curMem=714319, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_167_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_167_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 167 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 305 (MapPartitionsRDD[258] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 305.0 with 4 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 305.0 (TID 365, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 305.0 (TID 366, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 305.0 (TID 367, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 305.0 (TID 368, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 305.0 (TID 365) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 305.0 (TID 367) -2016-06-13 19:45:17 INFO Executor:59 - Running task 3.0 in stage 305.0 (TID 368) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 305.0 (TID 366) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 305.0 (TID 365). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 305.0 (TID 366). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 305.0 (TID 365) in 7 ms on localhost (1/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 3.0 in stage 305.0 (TID 368). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 305.0 (TID 366) in 9 ms on localhost (2/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 305.0 (TID 368) in 8 ms on localhost (3/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 305.0 (TID 367). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 305.0 (TID 367) in 12 ms on localhost (4/4) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 305 (reduce at LDAOptimizer.scala:453) finished in 0.014 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 305.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 119 finished: reduce at LDAOptimizer.scala:453, took 0.022720 s -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 120 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 306(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 306 (MapPartitionsRDD[259] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=722757, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_168 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(8450) called with curMem=743053, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_168_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.9 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_168_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 168 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 306 (MapPartitionsRDD[259] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 306.0 with 4 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 306.0 (TID 369, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 306.0 (TID 370, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 306.0 (TID 371, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 306.0 (TID 372, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 306.0 (TID 369) -2016-06-13 19:45:17 INFO Executor:59 - Running task 3.0 in stage 306.0 (TID 372) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 306.0 (TID 369). 1002 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 306.0 (TID 371) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 306.0 (TID 370) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 3.0 in stage 306.0 (TID 372). 1314 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 306.0 (TID 371). 1314 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 306.0 (TID 369) in 11 ms on localhost (1/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 306.0 (TID 370). 1002 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 306.0 (TID 372) in 10 ms on localhost (2/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 306.0 (TID 371) in 11 ms on localhost (3/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 306.0 (TID 370) in 12 ms on localhost (4/4) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 306 (reduce at LDAOptimizer.scala:456) finished in 0.013 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 306.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 120 finished: reduce at LDAOptimizer.scala:456, took 0.027282 s -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 121 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 307(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 307 (PartitionwiseSampledRDD[260] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=751503, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_169 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=770319, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_169_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_169_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 169 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 307 (PartitionwiseSampledRDD[260] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 307.0 with 1 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 307.0 (TID 373, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 307.0 (TID 373) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 307.0 (TID 373). 911 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 307.0 (TID 373) in 4 ms on localhost (1/1) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 307.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 307 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 121 finished: isEmpty at LDAOptimizer.scala:418, took 0.013071 s -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 122 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 308(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 308 (PartitionwiseSampledRDD[260] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=777975, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_170 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=796791, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_170_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_170_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 170 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 308 (PartitionwiseSampledRDD[260] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 308.0 with 3 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 308.0 (TID 374, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 308.0 (TID 375, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 308.0 (TID 376, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 308.0 (TID 375) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 308.0 (TID 374) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 308.0 (TID 376) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 308.0 (TID 375). 911 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 308.0 (TID 376). 1226 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 308.0 (TID 374). 1226 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 308.0 (TID 375) in 3 ms on localhost (1/3) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 308.0 (TID 376) in 4 ms on localhost (2/3) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 308.0 (TID 374) in 5 ms on localhost (3/3) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 308 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 308.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 122 finished: isEmpty at LDAOptimizer.scala:418, took 0.011881 s -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=804447, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_171 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=804623, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_171_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.8 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_171_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 171 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 123 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 309(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 309 (MapPartitionsRDD[262] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=804903, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_172 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(8427) called with curMem=825159, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_172_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_172_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 172 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 309 (MapPartitionsRDD[262] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 309.0 with 4 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 309.0 (TID 377, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 309.0 (TID 378, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 309.0 (TID 379, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 309.0 (TID 380, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 309.0 (TID 377) -2016-06-13 19:45:17 INFO Executor:59 - Running task 3.0 in stage 309.0 (TID 380) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 309.0 (TID 379) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 309.0 (TID 378) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 309.0 (TID 377). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 309.0 (TID 377) in 5 ms on localhost (1/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 309.0 (TID 379). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 309.0 (TID 379) in 6 ms on localhost (2/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 3.0 in stage 309.0 (TID 380). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 309.0 (TID 380) in 5 ms on localhost (3/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 309.0 (TID 378). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 309.0 (TID 378) in 9 ms on localhost (4/4) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 309 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 309.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 123 finished: reduce at LDAOptimizer.scala:453, took 0.016867 s -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 124 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 310(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 310 (MapPartitionsRDD[263] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=833586, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_173 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(8438) called with curMem=853882, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_173_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_173_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 173 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 310 (MapPartitionsRDD[263] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 310.0 with 4 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 310.0 (TID 381, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 310.0 (TID 382, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 310.0 (TID 383, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 310.0 (TID 384, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 310.0 (TID 382) -2016-06-13 19:45:17 INFO Executor:59 - Running task 3.0 in stage 310.0 (TID 384) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 310.0 (TID 383) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 310.0 (TID 381) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 310.0 (TID 383). 1002 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 310.0 (TID 382). 1314 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 310.0 (TID 383) in 5 ms on localhost (1/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 3.0 in stage 310.0 (TID 384). 1314 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 310.0 (TID 382) in 7 ms on localhost (2/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 310.0 (TID 384) in 6 ms on localhost (3/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 310.0 (TID 381). 1002 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 310.0 (TID 381) in 8 ms on localhost (4/4) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 310 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 310.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 124 finished: reduce at LDAOptimizer.scala:456, took 0.017829 s -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 125 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 311(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 311 (PartitionwiseSampledRDD[264] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=862320, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_174 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=881136, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_174_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_174_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 174 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 311 (PartitionwiseSampledRDD[264] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 311.0 with 1 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 311.0 (TID 385, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 311.0 (TID 385) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 311.0 (TID 385). 1226 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 311.0 (TID 385) in 4 ms on localhost (1/1) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 311 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 311.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 125 finished: isEmpty at LDAOptimizer.scala:418, took 0.013909 s -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=888792, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_175 stored as values in memory (estimated size 176.0 B, free 729.7 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=888968, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_175_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.7 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_175_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 175 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 126 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 312(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 312 (MapPartitionsRDD[266] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=889248, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_176 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(8424) called with curMem=909504, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_176_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_176_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 176 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 312 (MapPartitionsRDD[266] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 312.0 with 4 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 312.0 (TID 386, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 312.0 (TID 387, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 312.0 (TID 388, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 312.0 (TID 389, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 312.0 (TID 387) -2016-06-13 19:45:17 INFO Executor:59 - Running task 3.0 in stage 312.0 (TID 389) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 312.0 (TID 388) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 312.0 (TID 386) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 312.0 (TID 388). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 312.0 (TID 388) in 5 ms on localhost (1/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 312.0 (TID 386). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 3.0 in stage 312.0 (TID 389). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 312.0 (TID 387). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 312.0 (TID 386) in 8 ms on localhost (2/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 312.0 (TID 389) in 7 ms on localhost (3/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 312.0 (TID 387) in 8 ms on localhost (4/4) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 312 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 312.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 126 finished: reduce at LDAOptimizer.scala:453, took 0.016088 s -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 127 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 313(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 313 (MapPartitionsRDD[267] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=917928, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_177 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(8435) called with curMem=938224, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_177_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_177_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 177 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 313 (MapPartitionsRDD[267] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 313.0 with 4 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 313.0 (TID 390, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 313.0 (TID 391, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 313.0 (TID 392, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 313.0 (TID 393, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 313.0 (TID 390) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 313.0 (TID 392) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 313.0 (TID 391) -2016-06-13 19:45:17 INFO Executor:59 - Running task 3.0 in stage 313.0 (TID 393) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 3.0 in stage 313.0 (TID 393). 1402 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 313.0 (TID 391). 1402 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 313.0 (TID 392). 1002 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 313.0 (TID 390). 1358 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 313.0 (TID 393) in 10 ms on localhost (1/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 313.0 (TID 391) in 11 ms on localhost (2/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 313.0 (TID 392) in 12 ms on localhost (3/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 313.0 (TID 390) in 13 ms on localhost (4/4) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 313 (reduce at LDAOptimizer.scala:456) finished in 0.014 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 313.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 127 finished: reduce at LDAOptimizer.scala:456, took 0.022234 s -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 128 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 314(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 314 (PartitionwiseSampledRDD[268] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=946659, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_178 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=965475, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_178_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_178_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 178 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 314 (PartitionwiseSampledRDD[268] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 314.0 with 1 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 314.0 (TID 394, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 314.0 (TID 394) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 314.0 (TID 394). 1226 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 314.0 (TID 394) in 3 ms on localhost (1/1) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 314 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 314.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 128 finished: isEmpty at LDAOptimizer.scala:418, took 0.009699 s -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=973131, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_179 stored as values in memory (estimated size 176.0 B, free 729.7 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=973307, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_179_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.7 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_179_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 179 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 129 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 315(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 315 (MapPartitionsRDD[270] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=973587, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_180 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(8426) called with curMem=993843, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_180_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_180_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 180 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 315 (MapPartitionsRDD[270] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 315.0 with 4 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 315.0 (TID 395, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 315.0 (TID 396, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 315.0 (TID 397, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 315.0 (TID 398, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 315.0 (TID 395) -2016-06-13 19:45:17 INFO Executor:59 - Running task 3.0 in stage 315.0 (TID 398) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 315.0 (TID 397) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 315.0 (TID 396) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 3.0 in stage 315.0 (TID 398). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 315.0 (TID 397). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 315.0 (TID 398) in 5 ms on localhost (1/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 315.0 (TID 397) in 6 ms on localhost (2/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 315.0 (TID 395). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 315.0 (TID 395) in 9 ms on localhost (3/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 315.0 (TID 396). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 315.0 (TID 396) in 9 ms on localhost (4/4) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 315 (reduce at LDAOptimizer.scala:453) finished in 0.011 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 315.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 129 finished: reduce at LDAOptimizer.scala:453, took 0.018623 s -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 130 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 316(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 316 (MapPartitionsRDD[271] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1002269, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_181 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(8437) called with curMem=1022565, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_181_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_181_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 181 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 316 (MapPartitionsRDD[271] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 316.0 with 4 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 316.0 (TID 399, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 316.0 (TID 400, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 316.0 (TID 401, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 316.0 (TID 402, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 316.0 (TID 399) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 316.0 (TID 400) -2016-06-13 19:45:17 INFO Executor:59 - Running task 3.0 in stage 316.0 (TID 402) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 316.0 (TID 401) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 3.0 in stage 316.0 (TID 402). 1002 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 316.0 (TID 400). 1402 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 316.0 (TID 401). 1002 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 316.0 (TID 402) in 9 ms on localhost (1/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 316.0 (TID 400) in 12 ms on localhost (2/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 316.0 (TID 399). 1314 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 316.0 (TID 401) in 12 ms on localhost (3/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 316.0 (TID 399) in 14 ms on localhost (4/4) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 316 (reduce at LDAOptimizer.scala:456) finished in 0.014 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 316.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 130 finished: reduce at LDAOptimizer.scala:456, took 0.024150 s -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 131 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 317(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 317 (PartitionwiseSampledRDD[272] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1031002, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_182 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=1049818, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_182_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_182_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 182 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 317 (PartitionwiseSampledRDD[272] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 317.0 with 1 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 317.0 (TID 403, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 317.0 (TID 403) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 317.0 (TID 403). 911 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 317.0 (TID 403) in 4 ms on localhost (1/1) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 317.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 317 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 131 finished: isEmpty at LDAOptimizer.scala:418, took 0.013732 s -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 132 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 318(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 318 (PartitionwiseSampledRDD[272] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1057475, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_183 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=1076291, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_183_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_183_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 183 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 318 (PartitionwiseSampledRDD[272] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 318.0 with 3 tasks -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_182_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 318.0 (TID 404, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 318.0 (TID 405, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 151 -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 318.0 (TID 406, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 318.0 (TID 405) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 318.0 (TID 404) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 318.0 (TID 406) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 318.0 (TID 406). 1226 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 318.0 (TID 404). 911 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 318.0 (TID 405). 911 bytes result sent to driver -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_181_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 318.0 (TID 406) in 4 ms on localhost (1/3) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 150 -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 318.0 (TID 404) in 5 ms on localhost (2/3) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 318.0 (TID 405) in 5 ms on localhost (3/3) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 318 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 318.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 132 finished: isEmpty at LDAOptimizer.scala:418, took 0.022982 s -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1000060, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_184 stored as values in memory (estimated size 176.0 B, free 729.6 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_180_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 149 -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1000060, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_184_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.6 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_179_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_184_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 184 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_178_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 148 -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_177_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 147 -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 133 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 319(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 319 (MapPartitionsRDD[274] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_176_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 146 -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=916177, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_185 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_175_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(8429) called with curMem=935977, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_185_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_185_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 185 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 319 (MapPartitionsRDD[274] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 319.0 with 4 tasks -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_174_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 145 -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 319.0 (TID 407, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 319.0 (TID 408, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 319.0 (TID 409, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 319.0 (TID 410, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 319.0 (TID 408) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 319.0 (TID 409) -2016-06-13 19:45:17 INFO Executor:59 - Running task 3.0 in stage 319.0 (TID 410) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 319.0 (TID 407) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_173_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 144 -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 319.0 (TID 407). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 3.0 in stage 319.0 (TID 410). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_172_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 319.0 (TID 407) in 9 ms on localhost (1/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 319.0 (TID 408). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 319.0 (TID 410) in 8 ms on localhost (2/4) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 143 -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 319.0 (TID 408) in 9 ms on localhost (3/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 319.0 (TID 409). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 319.0 (TID 409) in 10 ms on localhost (4/4) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 319 (reduce at LDAOptimizer.scala:453) finished in 0.011 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 319.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 133 finished: reduce at LDAOptimizer.scala:453, took 0.019094 s -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_171_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_170_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 142 -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 134 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 320(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 320 (MapPartitionsRDD[275] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_169_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=807117, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_186 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 141 -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(8440) called with curMem=818963, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_186_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_168_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_186_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 140 -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 186 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 320 (MapPartitionsRDD[275] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 320.0 with 4 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 320.0 (TID 411, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 320.0 (TID 412, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_167_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 320.0 (TID 413, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 320.0 (TID 414, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 320.0 (TID 413) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 139 -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 320.0 (TID 411) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 320.0 (TID 412) -2016-06-13 19:45:17 INFO Executor:59 - Running task 3.0 in stage 320.0 (TID 414) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 320.0 (TID 413). 1002 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 320.0 (TID 411). 1002 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 320.0 (TID 413) in 5 ms on localhost (1/4) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_166_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 320.0 (TID 411) in 6 ms on localhost (2/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 320.0 (TID 412). 1002 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 3.0 in stage 320.0 (TID 414). 1358 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 320.0 (TID 412) in 8 ms on localhost (3/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 320.0 (TID 414) in 7 ms on localhost (4/4) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 320.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 320 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 134 finished: reduce at LDAOptimizer.scala:456, took 0.019347 s -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_165_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 138 -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_164_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 137 -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_163_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 135 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 321(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 321 (PartitionwiseSampledRDD[276] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 136 -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=696289, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_187 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(7663) called with curMem=706688, maxMem=766075207 -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_162_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_187_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 135 -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_187_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 187 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 321 (PartitionwiseSampledRDD[276] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 321.0 with 1 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 321.0 (TID 415, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_161_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 321.0 (TID 415) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 321.0 (TID 415). 1226 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 321.0 (TID 415) in 4 ms on localhost (1/1) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 321 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 321.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 135 finished: isEmpty at LDAOptimizer.scala:418, took 0.013350 s -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_160_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 134 -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=648352, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_188 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=640873, maxMem=766075207 -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_159_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_188_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 133 -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_188_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 188 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_158_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 132 -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_157_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 131 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 136 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 322(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 322 (MapPartitionsRDD[278] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_156_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=583288, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_189 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(8431) called with curMem=577072, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_189_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_189_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 189 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_155_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 322 (MapPartitionsRDD[278] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 322.0 with 4 tasks -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 130 -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 322.0 (TID 416, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 322.0 (TID 417, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 322.0 (TID 418, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 322.0 (TID 419, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 322.0 (TID 418) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_154_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO Executor:59 - Running task 3.0 in stage 322.0 (TID 419) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 322.0 (TID 416) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 322.0 (TID 417) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 129 -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 322.0 (TID 418). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 322.0 (TID 418) in 4 ms on localhost (1/4) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_153_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 322.0 (TID 417). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 322.0 (TID 417) in 7 ms on localhost (2/4) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 128 -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 322.0 (TID 416). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 3.0 in stage 322.0 (TID 419). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 322.0 (TID 416) in 10 ms on localhost (3/4) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_152_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 322.0 (TID 419) in 10 ms on localhost (4/4) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 322 (reduce at LDAOptimizer.scala:453) finished in 0.011 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 322.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 127 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 136 finished: reduce at LDAOptimizer.scala:453, took 0.019438 s -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_151_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_150_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 137 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 323(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 126 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 323 (MapPartitionsRDD[279] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_149_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=467033, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_190 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 125 -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(8442) called with curMem=460080, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_190_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_148_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_190_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 124 -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 190 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 323 (MapPartitionsRDD[279] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 323.0 with 4 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 323.0 (TID 420, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 323.0 (TID 421, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 323.0 (TID 422, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_147_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 323.0 (TID 423, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 323.0 (TID 420) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 323.0 (TID 421) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 323.0 (TID 422) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 123 -2016-06-13 19:45:17 INFO Executor:59 - Running task 3.0 in stage 323.0 (TID 423) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_146_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 323.0 (TID 422). 1002 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 323.0 (TID 420). 1358 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 323.0 (TID 421). 1314 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 323.0 (TID 422) in 7 ms on localhost (1/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 323.0 (TID 421) in 9 ms on localhost (2/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 323.0 (TID 420) in 10 ms on localhost (3/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 3.0 in stage 323.0 (TID 423). 1358 bytes result sent to driver -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_145_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 323 (reduce at LDAOptimizer.scala:456) finished in 0.011 s -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 323.0 (TID 423) in 10 ms on localhost (4/4) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 323.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 137 finished: reduce at LDAOptimizer.scala:456, took 0.019135 s -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 122 -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_144_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 121 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 138 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 324(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 324 (PartitionwiseSampledRDD[280] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_143_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=335209, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_191 stored as values in memory (estimated size 18.4 KB, free 730.2 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 120 -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=354025, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_191_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_191_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 191 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_142_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 324 (PartitionwiseSampledRDD[280] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 324.0 with 1 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 324.0 (TID 424, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 324.0 (TID 424) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_141_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 324.0 (TID 424). 1226 bytes result sent to driver -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 119 -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 324.0 (TID 424) in 3 ms on localhost (1/1) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 324 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 324.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 138 finished: isEmpty at LDAOptimizer.scala:418, took 0.010972 s -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_140_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=306022, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_192 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 118 -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=297774, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_192_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_139_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_192_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 117 -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 192 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_138_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_137_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 116 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 139 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 325(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 325 (MapPartitionsRDD[282] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_136_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=222143, maxMem=766075207 -2016-06-13 19:45:17 INFO ContextCleaner:59 - Cleaned accumulator 115 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_193 stored as values in memory (estimated size 19.8 KB, free 730.4 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(8421) called with curMem=241943, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_193_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.3 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Removed broadcast_134_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_193_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 193 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 325 (MapPartitionsRDD[282] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 325.0 with 4 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 325.0 (TID 425, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 325.0 (TID 426, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 325.0 (TID 427, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 325.0 (TID 428, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 325.0 (TID 425) -2016-06-13 19:45:17 INFO Executor:59 - Running task 3.0 in stage 325.0 (TID 428) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 325.0 (TID 427) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 325.0 (TID 426) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 325.0 (TID 426). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 325.0 (TID 426) in 6 ms on localhost (1/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 3.0 in stage 325.0 (TID 428). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 325.0 (TID 428) in 7 ms on localhost (2/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 325.0 (TID 425). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 325.0 (TID 427). 1265 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 325.0 (TID 425) in 11 ms on localhost (3/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 325.0 (TID 427) in 10 ms on localhost (4/4) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 325.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 325 (reduce at LDAOptimizer.scala:453) finished in 0.012 s -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 139 finished: reduce at LDAOptimizer.scala:453, took 0.019960 s -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 140 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 326(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 326 (MapPartitionsRDD[283] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=250364, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_194 stored as values in memory (estimated size 19.8 KB, free 730.3 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=270660, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_194_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.3 MB) -2016-06-13 19:45:17 INFO BlockManagerInfo:59 - Added broadcast_194_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:17 INFO SparkContext:59 - Created broadcast 194 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 326 (MapPartitionsRDD[283] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Adding task set 326.0 with 4 tasks -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 326.0 (TID 429, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 326.0 (TID 430, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 2.0 in stage 326.0 (TID 431, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Starting task 3.0 in stage 326.0 (TID 432, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:17 INFO Executor:59 - Running task 0.0 in stage 326.0 (TID 429) -2016-06-13 19:45:17 INFO Executor:59 - Running task 2.0 in stage 326.0 (TID 431) -2016-06-13 19:45:17 INFO Executor:59 - Running task 3.0 in stage 326.0 (TID 432) -2016-06-13 19:45:17 INFO Executor:59 - Running task 1.0 in stage 326.0 (TID 430) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 0.0 in stage 326.0 (TID 429). 1358 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 3.0 in stage 326.0 (TID 432). 1402 bytes result sent to driver -2016-06-13 19:45:17 INFO Executor:59 - Finished task 1.0 in stage 326.0 (TID 430). 1314 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 3.0 in stage 326.0 (TID 432) in 6 ms on localhost (1/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 326.0 (TID 429) in 7 ms on localhost (2/4) -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 326.0 (TID 430) in 8 ms on localhost (3/4) -2016-06-13 19:45:17 INFO Executor:59 - Finished task 2.0 in stage 326.0 (TID 431). 1446 bytes result sent to driver -2016-06-13 19:45:17 INFO TaskSetManager:59 - Finished task 2.0 in stage 326.0 (TID 431) in 9 ms on localhost (4/4) -2016-06-13 19:45:17 INFO DAGScheduler:59 - ResultStage 326 (reduce at LDAOptimizer.scala:456) finished in 0.009 s -2016-06-13 19:45:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 326.0, whose tasks have all completed, from pool -2016-06-13 19:45:17 INFO DAGScheduler:59 - Job 140 finished: reduce at LDAOptimizer.scala:456, took 0.017661 s -2016-06-13 19:45:17 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:17 INFO DAGScheduler:59 - Got job 141 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:17 INFO DAGScheduler:59 - Final stage: ResultStage 327(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:17 INFO DAGScheduler:59 - Submitting ResultStage 327 (PartitionwiseSampledRDD[284] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=279092, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_195 stored as values in memory (estimated size 18.4 KB, free 730.3 MB) -2016-06-13 19:45:17 INFO MemoryStore:59 - ensureFreeSpace(7659) called with curMem=297908, maxMem=766075207 -2016-06-13 19:45:17 INFO MemoryStore:59 - Block broadcast_195_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.3 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_195_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 195 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 327 (PartitionwiseSampledRDD[284] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 327.0 with 1 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 327.0 (TID 433, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 327.0 (TID 433) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 327.0 (TID 433). 911 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 327.0 (TID 433) in 3 ms on localhost (1/1) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 327 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 327.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 141 finished: isEmpty at LDAOptimizer.scala:418, took 0.008450 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 142 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 328(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 328 (PartitionwiseSampledRDD[284] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=305567, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_196 stored as values in memory (estimated size 18.4 KB, free 730.3 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(7659) called with curMem=324383, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_196_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.3 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_196_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 196 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 328 (PartitionwiseSampledRDD[284] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 328.0 with 3 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 328.0 (TID 434, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 328.0 (TID 435, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 328.0 (TID 436, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 328.0 (TID 435) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 328.0 (TID 434) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 328.0 (TID 436) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 328.0 (TID 435). 911 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 328.0 (TID 436). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 328.0 (TID 434). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 328.0 (TID 435) in 3 ms on localhost (1/3) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 328.0 (TID 436) in 4 ms on localhost (2/3) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 328.0 (TID 434) in 4 ms on localhost (3/3) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 328 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 328.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 142 finished: isEmpty at LDAOptimizer.scala:418, took 0.011407 s -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=332042, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_197 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=332218, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_197_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_197_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 197 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 143 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 329(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 329 (MapPartitionsRDD[286] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=332498, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_198 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8438) called with curMem=352754, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_198_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_198_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 198 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 329 (MapPartitionsRDD[286] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 329.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 329.0 (TID 437, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 329.0 (TID 438, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 329.0 (TID 439, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 329.0 (TID 440, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 329.0 (TID 437) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 329.0 (TID 438) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 329.0 (TID 440) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 329.0 (TID 439) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 329.0 (TID 439). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 329.0 (TID 438). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 329.0 (TID 439) in 5 ms on localhost (1/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 329.0 (TID 437). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 329.0 (TID 438) in 5 ms on localhost (2/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 329.0 (TID 440). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 329.0 (TID 437) in 8 ms on localhost (3/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 329.0 (TID 440) in 7 ms on localhost (4/4) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 329 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 329.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 143 finished: reduce at LDAOptimizer.scala:453, took 0.014250 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 144 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 330(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 330 (MapPartitionsRDD[287] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=361192, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_199 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8451) called with curMem=381488, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_199_piece0 stored as bytes in memory (estimated size 8.3 KB, free 730.2 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_199_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.5 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 199 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 330 (MapPartitionsRDD[287] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 330.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 330.0 (TID 441, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 330.0 (TID 442, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 330.0 (TID 443, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 330.0 (TID 444, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 330.0 (TID 441) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 330.0 (TID 444) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 330.0 (TID 443) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 330.0 (TID 442) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 330.0 (TID 443). 1002 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 330.0 (TID 443) in 4 ms on localhost (1/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 330.0 (TID 441). 1002 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 330.0 (TID 444). 1314 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 330.0 (TID 442). 1314 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 330.0 (TID 441) in 5 ms on localhost (2/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 330.0 (TID 444) in 4 ms on localhost (3/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 330.0 (TID 442) in 6 ms on localhost (4/4) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 330 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 330.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 144 finished: reduce at LDAOptimizer.scala:456, took 0.012974 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 145 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 331(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 331 (PartitionwiseSampledRDD[288] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=389939, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_200 stored as values in memory (estimated size 18.4 KB, free 730.2 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(7671) called with curMem=408755, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_200_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_200_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 200 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 331 (PartitionwiseSampledRDD[288] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 331.0 with 1 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 331.0 (TID 445, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 331.0 (TID 445) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 331.0 (TID 445). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 331.0 (TID 445) in 3 ms on localhost (1/1) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 331 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 331.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 145 finished: isEmpty at LDAOptimizer.scala:418, took 0.011326 s -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=416426, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_201 stored as values in memory (estimated size 176.0 B, free 730.2 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=416602, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_201_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.2 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_201_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 201 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 146 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 332(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 332 (MapPartitionsRDD[290] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=416882, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_202 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8439) called with curMem=437138, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_202_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_202_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 202 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 332 (MapPartitionsRDD[290] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 332.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 332.0 (TID 446, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 332.0 (TID 447, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 332.0 (TID 448, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 332.0 (TID 449, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 332.0 (TID 446) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 332.0 (TID 449) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 332.0 (TID 448) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 332.0 (TID 447) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 332.0 (TID 448). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 332.0 (TID 447). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 332.0 (TID 446). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 332.0 (TID 448) in 6 ms on localhost (1/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 332.0 (TID 447) in 7 ms on localhost (2/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 332.0 (TID 446) in 7 ms on localhost (3/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 332.0 (TID 449). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 332.0 (TID 449) in 7 ms on localhost (4/4) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 332 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 332.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 146 finished: reduce at LDAOptimizer.scala:453, took 0.014591 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 147 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 333(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 333 (MapPartitionsRDD[291] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=445577, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_203 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8450) called with curMem=465873, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_203_piece0 stored as bytes in memory (estimated size 8.3 KB, free 730.1 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_203_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 203 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 333 (MapPartitionsRDD[291] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 333.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 333.0 (TID 450, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 333.0 (TID 451, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 333.0 (TID 452, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 333.0 (TID 453, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 333.0 (TID 450) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 333.0 (TID 452) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 333.0 (TID 451) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 333.0 (TID 453) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 333.0 (TID 452). 1314 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 333.0 (TID 451). 1402 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 333.0 (TID 453). 1446 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 333.0 (TID 452) in 8 ms on localhost (1/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 333.0 (TID 451) in 8 ms on localhost (2/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 333.0 (TID 453) in 8 ms on localhost (3/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 333.0 (TID 450). 1402 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 333.0 (TID 450) in 11 ms on localhost (4/4) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 333 (reduce at LDAOptimizer.scala:456) finished in 0.011 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 333.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 147 finished: reduce at LDAOptimizer.scala:456, took 0.018496 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 148 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 334(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 334 (PartitionwiseSampledRDD[292] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=474323, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_204 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(7655) called with curMem=493139, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_204_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_204_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 204 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 334 (PartitionwiseSampledRDD[292] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 334.0 with 1 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 334.0 (TID 454, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 334.0 (TID 454) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 334.0 (TID 454). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 334.0 (TID 454) in 4 ms on localhost (1/1) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 334.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 334 (isEmpty at LDAOptimizer.scala:418) finished in 0.006 s -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 148 finished: isEmpty at LDAOptimizer.scala:418, took 0.012246 s -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=500794, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_205 stored as values in memory (estimated size 176.0 B, free 730.1 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=500970, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_205_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.1 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_205_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 205 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 149 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 335(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 335 (MapPartitionsRDD[294] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=501250, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_206 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8421) called with curMem=521506, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_206_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_206_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 206 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 335 (MapPartitionsRDD[294] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 335.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 335.0 (TID 455, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 335.0 (TID 456, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 335.0 (TID 457, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 335.0 (TID 458, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 335.0 (TID 455) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 335.0 (TID 456) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 335.0 (TID 458) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 335.0 (TID 457) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 335.0 (TID 458). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 335.0 (TID 458) in 7 ms on localhost (1/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 335.0 (TID 455). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 335.0 (TID 457). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 335.0 (TID 455) in 10 ms on localhost (2/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 335.0 (TID 457) in 9 ms on localhost (3/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 335.0 (TID 456). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 335.0 (TID 456) in 11 ms on localhost (4/4) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 335 (reduce at LDAOptimizer.scala:453) finished in 0.012 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 335.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 149 finished: reduce at LDAOptimizer.scala:453, took 0.019854 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 150 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 336(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 336 (MapPartitionsRDD[295] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=529927, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_207 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=550223, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_207_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_207_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 207 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 336 (MapPartitionsRDD[295] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 336.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 336.0 (TID 459, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 336.0 (TID 460, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 336.0 (TID 461, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 336.0 (TID 462, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 336.0 (TID 459) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 336.0 (TID 461) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 336.0 (TID 462) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 336.0 (TID 460) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 336.0 (TID 459). 1358 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 336.0 (TID 460). 1358 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 336.0 (TID 461). 1314 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 336.0 (TID 462). 1358 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 336.0 (TID 459) in 6 ms on localhost (1/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 336.0 (TID 460) in 6 ms on localhost (2/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 336.0 (TID 461) in 7 ms on localhost (3/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 336.0 (TID 462) in 6 ms on localhost (4/4) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 336 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 336.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 150 finished: reduce at LDAOptimizer.scala:456, took 0.015082 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 151 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 337(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 337 (PartitionwiseSampledRDD[296] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=558655, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_208 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=577471, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_208_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_208_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 208 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 337 (PartitionwiseSampledRDD[296] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 337.0 with 1 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 337.0 (TID 463, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 337.0 (TID 463) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 337.0 (TID 463). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 337.0 (TID 463) in 3 ms on localhost (1/1) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 337 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 337.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 151 finished: isEmpty at LDAOptimizer.scala:418, took 0.009138 s -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=585128, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_209 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=585304, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_209_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_209_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 209 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 152 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 338(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 338 (MapPartitionsRDD[298] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=585584, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_210 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8436) called with curMem=605840, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_210_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_210_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 210 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 338 (MapPartitionsRDD[298] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 338.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 338.0 (TID 464, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 338.0 (TID 465, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 338.0 (TID 466, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 338.0 (TID 467, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 338.0 (TID 465) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 338.0 (TID 466) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 338.0 (TID 464) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 338.0 (TID 467) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 338.0 (TID 467). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 338.0 (TID 465). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 338.0 (TID 466). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 338.0 (TID 467) in 6 ms on localhost (1/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 338.0 (TID 464). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 338.0 (TID 465) in 8 ms on localhost (2/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 338.0 (TID 464) in 9 ms on localhost (3/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 338.0 (TID 466) in 8 ms on localhost (4/4) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 338.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 338 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 152 finished: reduce at LDAOptimizer.scala:453, took 0.015857 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 153 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 339(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 339 (MapPartitionsRDD[299] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=614276, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_211 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8449) called with curMem=634572, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_211_piece0 stored as bytes in memory (estimated size 8.3 KB, free 730.0 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_211_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 211 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 339 (MapPartitionsRDD[299] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 339.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 339.0 (TID 468, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 339.0 (TID 469, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 339.0 (TID 470, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 339.0 (TID 471, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 339.0 (TID 468) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 339.0 (TID 471) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 339.0 (TID 470) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 339.0 (TID 469) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 339.0 (TID 471). 1002 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 339.0 (TID 468). 1314 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 339.0 (TID 471) in 4 ms on localhost (1/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 339.0 (TID 470). 1314 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 339.0 (TID 468) in 6 ms on localhost (2/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 339.0 (TID 469). 1358 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 339.0 (TID 470) in 5 ms on localhost (3/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 339.0 (TID 469) in 6 ms on localhost (4/4) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 339 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 339.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 153 finished: reduce at LDAOptimizer.scala:456, took 0.013841 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 154 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 340(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 340 (PartitionwiseSampledRDD[300] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=643021, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_212 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=661837, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_212_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_212_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 212 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 340 (PartitionwiseSampledRDD[300] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 340.0 with 1 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 340.0 (TID 472, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 340.0 (TID 472) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 340.0 (TID 472). 911 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 340.0 (TID 472) in 2 ms on localhost (1/1) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 340 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 340.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 154 finished: isEmpty at LDAOptimizer.scala:418, took 0.011314 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 155 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 341(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 341 (PartitionwiseSampledRDD[300] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=669498, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_213 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=688314, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_213_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_213_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 213 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 341 (PartitionwiseSampledRDD[300] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 341.0 with 3 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 341.0 (TID 473, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 341.0 (TID 474, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 341.0 (TID 475, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 341.0 (TID 474) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 341.0 (TID 475) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 341.0 (TID 473) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 341.0 (TID 474). 911 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 341.0 (TID 473). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 341.0 (TID 474) in 3 ms on localhost (1/3) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 341.0 (TID 475). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 341.0 (TID 473) in 4 ms on localhost (2/3) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 341.0 (TID 475) in 3 ms on localhost (3/3) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 341 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 341.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 155 finished: isEmpty at LDAOptimizer.scala:418, took 0.011739 s -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=695975, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_214 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=696151, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_214_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_214_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 214 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 156 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 342(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 342 (MapPartitionsRDD[302] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=696431, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_215 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8431) called with curMem=716687, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_215_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_215_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 215 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 342 (MapPartitionsRDD[302] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 342.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 342.0 (TID 476, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 342.0 (TID 477, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 342.0 (TID 478, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 342.0 (TID 479, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 342.0 (TID 478) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 342.0 (TID 477) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 342.0 (TID 476) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 342.0 (TID 479) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 342.0 (TID 478). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 342.0 (TID 476). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 342.0 (TID 478) in 7 ms on localhost (1/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 342.0 (TID 476) in 8 ms on localhost (2/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 342.0 (TID 477). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 342.0 (TID 477) in 10 ms on localhost (3/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 342.0 (TID 479). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 342.0 (TID 479) in 12 ms on localhost (4/4) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 342.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 342 (reduce at LDAOptimizer.scala:453) finished in 0.013 s -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 156 finished: reduce at LDAOptimizer.scala:453, took 0.019822 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 157 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 343(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 343 (MapPartitionsRDD[303] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=725118, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_216 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8442) called with curMem=745414, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_216_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_216_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 216 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 343 (MapPartitionsRDD[303] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 343.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 343.0 (TID 480, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 343.0 (TID 481, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 343.0 (TID 482, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 343.0 (TID 483, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 343.0 (TID 482) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 343.0 (TID 483) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 343.0 (TID 480) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 343.0 (TID 482). 1002 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 343.0 (TID 481) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 343.0 (TID 480). 1002 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 343.0 (TID 480) in 6 ms on localhost (1/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 343.0 (TID 482) in 6 ms on localhost (2/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 343.0 (TID 481). 1402 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 343.0 (TID 483). 1446 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 343.0 (TID 481) in 10 ms on localhost (3/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 343.0 (TID 483) in 10 ms on localhost (4/4) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 343 (reduce at LDAOptimizer.scala:456) finished in 0.011 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 343.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 157 finished: reduce at LDAOptimizer.scala:456, took 0.017600 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 158 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 344(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 344 (PartitionwiseSampledRDD[304] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=753856, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_217 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=772672, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_217_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_217_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 217 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 344 (PartitionwiseSampledRDD[304] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 344.0 with 1 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 344.0 (TID 484, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 344.0 (TID 484) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 344.0 (TID 484). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 344.0 (TID 484) in 3 ms on localhost (1/1) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 344 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 344.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 158 finished: isEmpty at LDAOptimizer.scala:418, took 0.008660 s -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=780333, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_218 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=780509, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_218_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.8 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_218_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 218 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 159 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 345(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 345 (MapPartitionsRDD[306] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=780789, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_219 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=801045, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_219_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_219_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 219 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 345 (MapPartitionsRDD[306] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 345.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 345.0 (TID 485, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 345.0 (TID 486, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 345.0 (TID 487, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 345.0 (TID 488, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 345.0 (TID 485) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 345.0 (TID 488) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 345.0 (TID 487) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 345.0 (TID 486) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 345.0 (TID 486). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 345.0 (TID 486) in 4 ms on localhost (1/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 345.0 (TID 487). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 345.0 (TID 488). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 345.0 (TID 485). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 345.0 (TID 488) in 9 ms on localhost (2/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 345.0 (TID 487) in 10 ms on localhost (3/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 345.0 (TID 485) in 11 ms on localhost (4/4) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 345.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 345 (reduce at LDAOptimizer.scala:453) finished in 0.012 s -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 159 finished: reduce at LDAOptimizer.scala:453, took 0.018013 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 160 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 346(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 346 (MapPartitionsRDD[307] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=809477, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_220 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8443) called with curMem=829773, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_220_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_220_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 220 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 346 (MapPartitionsRDD[307] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 346.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 346.0 (TID 489, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 346.0 (TID 490, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 346.0 (TID 491, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 346.0 (TID 492, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 346.0 (TID 489) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 346.0 (TID 492) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 346.0 (TID 490) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 346.0 (TID 491) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 346.0 (TID 490). 1002 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 346.0 (TID 489). 1402 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 346.0 (TID 492). 1314 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 346.0 (TID 490) in 10 ms on localhost (1/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 346.0 (TID 491). 1358 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 346.0 (TID 489) in 12 ms on localhost (2/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 346.0 (TID 492) in 10 ms on localhost (3/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 346.0 (TID 491) in 12 ms on localhost (4/4) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 346 (reduce at LDAOptimizer.scala:456) finished in 0.013 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 346.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 160 finished: reduce at LDAOptimizer.scala:456, took 0.021067 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 161 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 347(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 347 (PartitionwiseSampledRDD[308] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=838216, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_221 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=857032, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_221_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_221_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 221 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 347 (PartitionwiseSampledRDD[308] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 347.0 with 1 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 347.0 (TID 493, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 347.0 (TID 493) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 347.0 (TID 493). 911 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 347.0 (TID 493) in 3 ms on localhost (1/1) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 347.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 347 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 161 finished: isEmpty at LDAOptimizer.scala:418, took 0.010792 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 162 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 348(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 348 (PartitionwiseSampledRDD[308] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=864688, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_222 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=883504, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_222_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_222_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 222 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 348 (PartitionwiseSampledRDD[308] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 348.0 with 3 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 348.0 (TID 494, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 348.0 (TID 495, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 348.0 (TID 496, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 348.0 (TID 494) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 348.0 (TID 495) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 348.0 (TID 496) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 348.0 (TID 494). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 348.0 (TID 495). 911 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 348.0 (TID 496). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 348.0 (TID 495) in 4 ms on localhost (1/3) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 348.0 (TID 494) in 5 ms on localhost (2/3) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 348.0 (TID 496) in 4 ms on localhost (3/3) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 348 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 348.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 162 finished: isEmpty at LDAOptimizer.scala:418, took 0.012560 s -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=891160, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_223 stored as values in memory (estimated size 176.0 B, free 729.7 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=891336, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_223_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.7 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_223_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 223 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 163 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 349(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 349 (MapPartitionsRDD[310] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=891616, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_224 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8429) called with curMem=911872, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_224_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_224_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 224 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 349 (MapPartitionsRDD[310] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 349.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 349.0 (TID 497, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 349.0 (TID 498, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 349.0 (TID 499, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 349.0 (TID 500, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 349.0 (TID 497) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 349.0 (TID 499) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 349.0 (TID 498) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 349.0 (TID 500) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 349.0 (TID 497). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 349.0 (TID 497) in 6 ms on localhost (1/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 349.0 (TID 499). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 349.0 (TID 498). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 349.0 (TID 499) in 5 ms on localhost (2/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 349.0 (TID 498) in 6 ms on localhost (3/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 349.0 (TID 500). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 349.0 (TID 500) in 6 ms on localhost (4/4) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 349 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 349.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 163 finished: reduce at LDAOptimizer.scala:453, took 0.014561 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 164 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 350(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 350 (MapPartitionsRDD[311] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=920301, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_225 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8440) called with curMem=940597, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_225_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_225_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 225 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 350 (MapPartitionsRDD[311] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 350.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 350.0 (TID 501, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 350.0 (TID 502, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 350.0 (TID 503, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 350.0 (TID 504, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 350.0 (TID 502) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 350.0 (TID 504) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 350.0 (TID 503) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 350.0 (TID 501) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 350.0 (TID 501). 1002 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 350.0 (TID 503). 1002 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 350.0 (TID 501) in 4 ms on localhost (1/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 350.0 (TID 502). 1314 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 350.0 (TID 504). 1358 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 350.0 (TID 503) in 5 ms on localhost (2/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 350.0 (TID 502) in 5 ms on localhost (3/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 350.0 (TID 504) in 5 ms on localhost (4/4) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 350 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 350.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 164 finished: reduce at LDAOptimizer.scala:456, took 0.012787 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 165 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 351(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 351 (PartitionwiseSampledRDD[312] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=949037, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_226 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=967853, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_226_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_226_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 226 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 351 (PartitionwiseSampledRDD[312] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 351.0 with 1 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 351.0 (TID 505, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 351.0 (TID 505) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 351.0 (TID 505). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 351.0 (TID 505) in 4 ms on localhost (1/1) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 351.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 351 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 165 finished: isEmpty at LDAOptimizer.scala:418, took 0.009813 s -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=975509, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_227 stored as values in memory (estimated size 176.0 B, free 729.7 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=975685, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_227_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.7 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_227_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 227 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 166 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 352(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 352 (MapPartitionsRDD[314] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=975965, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_228 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8422) called with curMem=996221, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_228_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_228_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 228 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 352 (MapPartitionsRDD[314] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 352.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 352.0 (TID 506, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 352.0 (TID 507, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 352.0 (TID 508, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 352.0 (TID 509, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 352.0 (TID 507) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 352.0 (TID 509) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 352.0 (TID 506) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 352.0 (TID 508) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 352.0 (TID 507). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 352.0 (TID 508). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 352.0 (TID 507) in 7 ms on localhost (1/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 352.0 (TID 509). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 352.0 (TID 508) in 6 ms on localhost (2/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 352.0 (TID 509) in 8 ms on localhost (3/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 352.0 (TID 506). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 352.0 (TID 506) in 12 ms on localhost (4/4) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 352.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 352 (reduce at LDAOptimizer.scala:453) finished in 0.012 s -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 166 finished: reduce at LDAOptimizer.scala:453, took 0.018874 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 167 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 353(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 353 (MapPartitionsRDD[315] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1004643, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_229 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=1024939, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_229_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_229_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 229 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 353 (MapPartitionsRDD[315] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 353.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 353.0 (TID 510, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 353.0 (TID 511, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 353.0 (TID 512, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 353.0 (TID 513, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 353.0 (TID 511) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 353.0 (TID 513) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 353.0 (TID 512) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 353.0 (TID 510) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 353.0 (TID 512). 1002 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 353.0 (TID 513). 1002 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 353.0 (TID 512) in 4 ms on localhost (1/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 353.0 (TID 513) in 4 ms on localhost (2/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 353.0 (TID 511). 1358 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 353.0 (TID 510). 1402 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 353.0 (TID 511) in 19 ms on localhost (3/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 353.0 (TID 510) in 21 ms on localhost (4/4) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 353 (reduce at LDAOptimizer.scala:456) finished in 0.021 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 353.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 167 finished: reduce at LDAOptimizer.scala:456, took 0.029363 s -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_228_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 186 -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_226_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 185 -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 168 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 354(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_225_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 354 (PartitionwiseSampledRDD[316] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 184 -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=949486, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_230 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_224_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=959873, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_230_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 183 -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_230_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 230 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 354 (PartitionwiseSampledRDD[316] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 354.0 with 1 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 354.0 (TID 514, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 354.0 (TID 514) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_223_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 354.0 (TID 514). 911 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 354.0 (TID 514) in 4 ms on localhost (1/1) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 354.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 354 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_222_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 168 finished: isEmpty at LDAOptimizer.scala:418, took 0.009944 s -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 182 -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_221_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 181 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 169 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 355(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 355 (PartitionwiseSampledRDD[316] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_220_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=865135, maxMem=766075207 -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 180 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_231 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=875519, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_231_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_219_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 179 -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_231_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 231 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 355 (PartitionwiseSampledRDD[316] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 355.0 with 3 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 355.0 (TID 515, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 355.0 (TID 516, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 355.0 (TID 517, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_218_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 355.0 (TID 517) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 355.0 (TID 516) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 355.0 (TID 515) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 355.0 (TID 516). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 355.0 (TID 515). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 355.0 (TID 517). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 355.0 (TID 516) in 4 ms on localhost (1/3) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_217_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 355.0 (TID 515) in 6 ms on localhost (2/3) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 355.0 (TID 517) in 6 ms on localhost (3/3) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 355 (isEmpty at LDAOptimizer.scala:418) finished in 0.007 s -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 169 finished: isEmpty at LDAOptimizer.scala:418, took 0.015145 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 355.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 178 -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=835987, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_232 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=807425, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_232_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.8 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_216_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 177 -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_232_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 232 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_215_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 176 -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_214_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 170 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 356(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 356 (MapPartitionsRDD[318] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=752085, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_233 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_213_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 175 -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8425) called with curMem=772341, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_233_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_233_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 233 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 356 (MapPartitionsRDD[318] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_212_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 356.0 with 4 tasks -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 174 -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 356.0 (TID 518, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 356.0 (TID 519, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 356.0 (TID 520, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 356.0 (TID 521, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 356.0 (TID 518) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 356.0 (TID 519) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_211_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 356.0 (TID 518). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 356.0 (TID 521) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 356.0 (TID 519). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 356.0 (TID 518) in 10 ms on localhost (1/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 356.0 (TID 519) in 9 ms on localhost (2/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 356.0 (TID 521). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 356.0 (TID 520) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 356.0 (TID 520). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 356.0 (TID 521) in 15 ms on localhost (3/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 356.0 (TID 520) in 17 ms on localhost (4/4) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 356.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 173 -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 356 (reduce at LDAOptimizer.scala:453) finished in 0.020 s -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 170 finished: reduce at LDAOptimizer.scala:453, took 0.028439 s -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_210_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 171 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 357(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 357 (MapPartitionsRDD[319] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=696852, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_234 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8436) called with curMem=717148, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_234_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 172 -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_234_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 234 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 357 (MapPartitionsRDD[319] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 357.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 357.0 (TID 522, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 357.0 (TID 523, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 357.0 (TID 524, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_209_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 357.0 (TID 525, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 357.0 (TID 523) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 357.0 (TID 525) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 357.0 (TID 524) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 357.0 (TID 524). 1314 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 357.0 (TID 523). 1358 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 357.0 (TID 524) in 6 ms on localhost (1/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 357.0 (TID 523) in 8 ms on localhost (2/4) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 357.0 (TID 522) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 357.0 (TID 525). 1314 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 357.0 (TID 522). 1002 bytes result sent to driver -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_208_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 357.0 (TID 525) in 14 ms on localhost (3/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 357.0 (TID 522) in 17 ms on localhost (4/4) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 357 (reduce at LDAOptimizer.scala:456) finished in 0.017 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 357.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 171 finished: reduce at LDAOptimizer.scala:456, took 0.029403 s -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 171 -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_207_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 170 -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_206_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 169 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 172 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 358(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 358 (PartitionwiseSampledRDD[320] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=640794, maxMem=766075207 -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_205_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_235 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=651955, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_235_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_204_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_235_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 168 -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 235 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 358 (PartitionwiseSampledRDD[320] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 358.0 with 1 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 358.0 (TID 526, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 358.0 (TID 526) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_203_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 167 -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 358.0 (TID 526). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 358.0 (TID 526) in 4 ms on localhost (1/1) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 358 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 358.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 172 finished: isEmpty at LDAOptimizer.scala:418, took 0.010577 s -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_202_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=583355, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_236 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 166 -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=583251, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_236_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_201_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_236_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 236 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_200_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 165 -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_199_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 164 -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_198_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 173 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 359(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 359 (MapPartitionsRDD[322] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 163 -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=499147, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_237 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_197_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8427) called with curMem=519227, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_237_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_237_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 237 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 359 (MapPartitionsRDD[322] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 359.0 with 4 tasks -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_196_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 359.0 (TID 527, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 162 -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 359.0 (TID 528, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 359.0 (TID 529, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 359.0 (TID 530, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 359.0 (TID 529) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 359.0 (TID 530) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 359.0 (TID 528) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 359.0 (TID 527) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 359.0 (TID 530). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 359.0 (TID 529). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 359.0 (TID 527). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 359.0 (TID 528). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 359.0 (TID 529) in 8 ms on localhost (1/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 359.0 (TID 530) in 8 ms on localhost (2/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 359.0 (TID 527) in 10 ms on localhost (3/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 359.0 (TID 528) in 10 ms on localhost (4/4) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 359.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 359 (reduce at LDAOptimizer.scala:453) finished in 0.011 s -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_195_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 173 finished: reduce at LDAOptimizer.scala:453, took 0.020760 s -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 161 -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_194_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 160 -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 174 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 360(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_193_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 360 (MapPartitionsRDD[323] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 159 -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=417019, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_238 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_192_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8438) called with curMem=437139, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_238_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_238_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 238 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 360 (MapPartitionsRDD[323] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 360.0 with 4 tasks -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_191_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 158 -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 360.0 (TID 531, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 360.0 (TID 532, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 360.0 (TID 533, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 360.0 (TID 534, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 360.0 (TID 534) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 360.0 (TID 531) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 360.0 (TID 532) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 360.0 (TID 533) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_190_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 360.0 (TID 534). 1314 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 360.0 (TID 532). 1358 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 360.0 (TID 534) in 6 ms on localhost (1/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 360.0 (TID 532) in 8 ms on localhost (2/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 360.0 (TID 533). 1314 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 360.0 (TID 531). 1402 bytes result sent to driver -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 157 -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 360.0 (TID 533) in 9 ms on localhost (3/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 360.0 (TID 531) in 10 ms on localhost (4/4) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 360 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 360.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 174 finished: reduce at LDAOptimizer.scala:456, took 0.018083 s -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_189_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 156 -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_188_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_187_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 155 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 175 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 361(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 361 (PartitionwiseSampledRDD[324] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=326304, maxMem=766075207 -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_186_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_239 stored as values in memory (estimated size 18.4 KB, free 730.3 MB) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 154 -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(7658) called with curMem=324824, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_239_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.3 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_239_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 239 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_185_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 361 (PartitionwiseSampledRDD[324] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 361.0 with 1 tasks -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 153 -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 361.0 (TID 535, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 361.0 (TID 535) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_184_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 361.0 (TID 535). 911 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 361.0 (TID 535) in 3 ms on localhost (1/1) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 361 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 361.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 175 finished: isEmpty at LDAOptimizer.scala:418, took 0.009091 s -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Removed broadcast_183_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:18 INFO ContextCleaner:59 - Cleaned accumulator 152 -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 176 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 362(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 362 (PartitionwiseSampledRDD[324] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=276868, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_240 stored as values in memory (estimated size 18.4 KB, free 730.3 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(7658) called with curMem=295684, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_240_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.3 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_240_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 240 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 362 (PartitionwiseSampledRDD[324] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 362.0 with 3 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 362.0 (TID 536, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 362.0 (TID 537, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 362.0 (TID 538, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 362.0 (TID 536) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 362.0 (TID 538) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 362.0 (TID 537) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 362.0 (TID 538). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 362.0 (TID 537). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 362.0 (TID 536). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 362.0 (TID 538) in 3 ms on localhost (1/3) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 362.0 (TID 537) in 3 ms on localhost (2/3) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 362.0 (TID 536) in 4 ms on localhost (3/3) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 362 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 362.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 176 finished: isEmpty at LDAOptimizer.scala:418, took 0.010004 s -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=303342, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_241 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=303518, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_241_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_241_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 241 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 177 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 363(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 363 (MapPartitionsRDD[326] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=303798, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_242 stored as values in memory (estimated size 19.8 KB, free 730.3 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8430) called with curMem=324054, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_242_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.3 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_242_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 242 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 363 (MapPartitionsRDD[326] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 363.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 363.0 (TID 539, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 363.0 (TID 540, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 363.0 (TID 541, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 363.0 (TID 542, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 363.0 (TID 540) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 363.0 (TID 542) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 363.0 (TID 541) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 363.0 (TID 539) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 363.0 (TID 539). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 363.0 (TID 542). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 363.0 (TID 539) in 5 ms on localhost (1/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 363.0 (TID 541). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 363.0 (TID 540). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 363.0 (TID 542) in 5 ms on localhost (2/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 363.0 (TID 541) in 7 ms on localhost (3/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 363.0 (TID 540) in 7 ms on localhost (4/4) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 363 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 363.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 177 finished: reduce at LDAOptimizer.scala:453, took 0.015414 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 178 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 364(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 364 (MapPartitionsRDD[327] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=332484, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_243 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8441) called with curMem=352780, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_243_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_243_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 243 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 364 (MapPartitionsRDD[327] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 364.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 364.0 (TID 543, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 364.0 (TID 544, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 364.0 (TID 545, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 364.0 (TID 546, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 364.0 (TID 543) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 364.0 (TID 545) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 364.0 (TID 546) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 364.0 (TID 544) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 364.0 (TID 543). 1002 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 364.0 (TID 545). 1314 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 364.0 (TID 543) in 5 ms on localhost (1/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 364.0 (TID 546). 1358 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 364.0 (TID 545) in 5 ms on localhost (2/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 364.0 (TID 544). 1402 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 364.0 (TID 546) in 6 ms on localhost (3/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 364.0 (TID 544) in 8 ms on localhost (4/4) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 364 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 178 finished: reduce at LDAOptimizer.scala:456, took 0.013545 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 364.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 179 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 365(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 365 (PartitionwiseSampledRDD[328] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=361221, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_244 stored as values in memory (estimated size 18.4 KB, free 730.2 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=380037, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_244_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_244_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 244 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 365 (PartitionwiseSampledRDD[328] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 365.0 with 1 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 365.0 (TID 547, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 365.0 (TID 547) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 365.0 (TID 547). 911 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 365.0 (TID 547) in 2 ms on localhost (1/1) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 365 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 365.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 179 finished: isEmpty at LDAOptimizer.scala:418, took 0.007749 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 180 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 366(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 366 (PartitionwiseSampledRDD[328] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=387694, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_245 stored as values in memory (estimated size 18.4 KB, free 730.2 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=406510, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_245_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_245_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 245 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 366 (PartitionwiseSampledRDD[328] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 366.0 with 3 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 366.0 (TID 548, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 366.0 (TID 549, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 366.0 (TID 550, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 366.0 (TID 550) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 366.0 (TID 549) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 366.0 (TID 548) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 366.0 (TID 549). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 366.0 (TID 548). 911 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 366.0 (TID 550). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 366.0 (TID 549) in 3 ms on localhost (1/3) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 366.0 (TID 548) in 4 ms on localhost (2/3) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 366.0 (TID 550) in 4 ms on localhost (3/3) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 366 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 366.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 180 finished: isEmpty at LDAOptimizer.scala:418, took 0.010147 s -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=414167, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_246 stored as values in memory (estimated size 176.0 B, free 730.2 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=414343, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_246_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.2 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_246_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 246 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 181 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 367(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 367 (MapPartitionsRDD[330] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=414623, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_247 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8426) called with curMem=434879, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_247_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_247_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 247 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 367 (MapPartitionsRDD[330] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 367.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 367.0 (TID 551, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 367.0 (TID 552, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 367.0 (TID 553, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 367.0 (TID 554, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 367.0 (TID 551) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 367.0 (TID 553) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 367.0 (TID 554) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 367.0 (TID 552) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 367.0 (TID 551). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 367.0 (TID 552). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 367.0 (TID 551) in 7 ms on localhost (1/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 367.0 (TID 552) in 6 ms on localhost (2/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 367.0 (TID 554). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 367.0 (TID 553). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 367.0 (TID 554) in 7 ms on localhost (3/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 367.0 (TID 553) in 8 ms on localhost (4/4) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 367 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 367.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 181 finished: reduce at LDAOptimizer.scala:453, took 0.014043 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 182 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 368(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 368 (MapPartitionsRDD[331] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=443305, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_248 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8437) called with curMem=463601, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_248_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_248_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 248 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 368 (MapPartitionsRDD[331] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 368.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 368.0 (TID 555, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 368.0 (TID 556, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 368.0 (TID 557, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 368.0 (TID 558, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 368.0 (TID 555) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 368.0 (TID 557) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 368.0 (TID 558) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 368.0 (TID 556) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 368.0 (TID 555). 1002 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 368.0 (TID 556). 1002 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 368.0 (TID 555) in 4 ms on localhost (1/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 368.0 (TID 556) in 4 ms on localhost (2/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 368.0 (TID 557). 1314 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 368.0 (TID 558). 1358 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 368.0 (TID 557) in 6 ms on localhost (3/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 368.0 (TID 558) in 5 ms on localhost (4/4) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 368 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 368.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 182 finished: reduce at LDAOptimizer.scala:456, took 0.012378 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 183 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 369(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 369 (PartitionwiseSampledRDD[332] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=472038, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_249 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(7658) called with curMem=490854, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_249_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_249_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 249 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 369 (PartitionwiseSampledRDD[332] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 369.0 with 1 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 369.0 (TID 559, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 369.0 (TID 559) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 369.0 (TID 559). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 369.0 (TID 559) in 2 ms on localhost (1/1) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 369 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 369.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 183 finished: isEmpty at LDAOptimizer.scala:418, took 0.009291 s -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=498512, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_250 stored as values in memory (estimated size 176.0 B, free 730.1 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=498688, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_250_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.1 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_250_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 250 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 184 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 370(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 370 (MapPartitionsRDD[334] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=498968, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_251 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8429) called with curMem=519224, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_251_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_251_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 251 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 370 (MapPartitionsRDD[334] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 370.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 370.0 (TID 560, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 370.0 (TID 561, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 370.0 (TID 562, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 370.0 (TID 563, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 370.0 (TID 561) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 370.0 (TID 563) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 370.0 (TID 560) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 370.0 (TID 562) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 370.0 (TID 562). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 370.0 (TID 560). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 370.0 (TID 563). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 370.0 (TID 562) in 4 ms on localhost (1/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 370.0 (TID 560) in 6 ms on localhost (2/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 370.0 (TID 561). 1265 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 370.0 (TID 563) in 5 ms on localhost (3/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 370.0 (TID 561) in 7 ms on localhost (4/4) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 370 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 370.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 184 finished: reduce at LDAOptimizer.scala:453, took 0.014786 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 185 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 371(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 371 (MapPartitionsRDD[335] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=527653, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_252 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8441) called with curMem=547949, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_252_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_252_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 252 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 371 (MapPartitionsRDD[335] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 371.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 371.0 (TID 564, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 371.0 (TID 565, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 2.0 in stage 371.0 (TID 566, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 3.0 in stage 371.0 (TID 567, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 371.0 (TID 564) -2016-06-13 19:45:18 INFO Executor:59 - Running task 1.0 in stage 371.0 (TID 565) -2016-06-13 19:45:18 INFO Executor:59 - Running task 2.0 in stage 371.0 (TID 566) -2016-06-13 19:45:18 INFO Executor:59 - Running task 3.0 in stage 371.0 (TID 567) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 2.0 in stage 371.0 (TID 566). 1002 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 3.0 in stage 371.0 (TID 567). 1314 bytes result sent to driver -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 371.0 (TID 564). 1358 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 2.0 in stage 371.0 (TID 566) in 5 ms on localhost (1/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 3.0 in stage 371.0 (TID 567) in 5 ms on localhost (2/4) -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 371.0 (TID 564) in 7 ms on localhost (3/4) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 1.0 in stage 371.0 (TID 565). 1402 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 371.0 (TID 565) in 7 ms on localhost (4/4) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 371 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 371.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 185 finished: reduce at LDAOptimizer.scala:456, took 0.014710 s -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 186 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 372(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 372 (PartitionwiseSampledRDD[336] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=556390, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_253 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=575206, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_253_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_253_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 253 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 372 (PartitionwiseSampledRDD[336] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 372.0 with 1 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 372.0 (TID 568, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:18 INFO Executor:59 - Running task 0.0 in stage 372.0 (TID 568) -2016-06-13 19:45:18 INFO Executor:59 - Finished task 0.0 in stage 372.0 (TID 568). 1226 bytes result sent to driver -2016-06-13 19:45:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 372.0 (TID 568) in 2 ms on localhost (1/1) -2016-06-13 19:45:18 INFO DAGScheduler:59 - ResultStage 372 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 372.0, whose tasks have all completed, from pool -2016-06-13 19:45:18 INFO DAGScheduler:59 - Job 186 finished: isEmpty at LDAOptimizer.scala:418, took 0.008007 s -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=582863, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_254 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=583039, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_254_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_254_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 254 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:18 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Got job 187 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:18 INFO DAGScheduler:59 - Final stage: ResultStage 373(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting ResultStage 373 (MapPartitionsRDD[338] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=583319, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_255 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:18 INFO MemoryStore:59 - ensureFreeSpace(8430) called with curMem=603575, maxMem=766075207 -2016-06-13 19:45:18 INFO MemoryStore:59 - Block broadcast_255_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:18 INFO BlockManagerInfo:59 - Added broadcast_255_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:18 INFO SparkContext:59 - Created broadcast 255 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:18 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 373 (MapPartitionsRDD[338] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:18 INFO TaskSchedulerImpl:59 - Adding task set 373.0 with 4 tasks -2016-06-13 19:45:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 373.0 (TID 569, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 373.0 (TID 570, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 373.0 (TID 571, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 373.0 (TID 572, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 373.0 (TID 569) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 373.0 (TID 572) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 373.0 (TID 570) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 373.0 (TID 571) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 373.0 (TID 571). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 373.0 (TID 569). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 373.0 (TID 570). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 373.0 (TID 571) in 5 ms on localhost (1/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 373.0 (TID 572). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 373.0 (TID 569) in 6 ms on localhost (2/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 373.0 (TID 570) in 6 ms on localhost (3/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 373.0 (TID 572) in 6 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 373 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 373.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 187 finished: reduce at LDAOptimizer.scala:453, took 0.012799 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 188 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 374(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 374 (MapPartitionsRDD[339] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=612005, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_256 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8443) called with curMem=632301, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_256_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_256_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 256 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 374 (MapPartitionsRDD[339] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 374.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 374.0 (TID 573, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 374.0 (TID 574, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 374.0 (TID 575, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 374.0 (TID 576, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 374.0 (TID 574) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 374.0 (TID 575) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 374.0 (TID 576) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 374.0 (TID 573) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 374.0 (TID 575). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 374.0 (TID 573). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 374.0 (TID 575) in 5 ms on localhost (1/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 374.0 (TID 576). 1358 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 374.0 (TID 573) in 6 ms on localhost (2/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 374.0 (TID 574). 1402 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 374.0 (TID 576) in 5 ms on localhost (3/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 374.0 (TID 574) in 7 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 374 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 374.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 188 finished: reduce at LDAOptimizer.scala:456, took 0.014606 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 189 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 375(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 375 (PartitionwiseSampledRDD[340] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=640744, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_257 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=659560, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_257_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_257_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 257 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 375 (PartitionwiseSampledRDD[340] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 375.0 with 1 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 375.0 (TID 577, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 375.0 (TID 577) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 375.0 (TID 577). 1226 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 375.0 (TID 577) in 2 ms on localhost (1/1) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 375 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 375.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 189 finished: isEmpty at LDAOptimizer.scala:418, took 0.007860 s -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=667217, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_258 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=667393, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_258_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_258_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 258 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 190 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 376(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 376 (MapPartitionsRDD[342] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=667673, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_259 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8429) called with curMem=687929, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_259_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_259_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 259 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 376 (MapPartitionsRDD[342] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 376.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 376.0 (TID 578, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 376.0 (TID 579, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 376.0 (TID 580, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 376.0 (TID 581, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 376.0 (TID 578) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 376.0 (TID 580) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 376.0 (TID 581) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 376.0 (TID 579) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 376.0 (TID 580). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 376.0 (TID 581). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 376.0 (TID 578). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 376.0 (TID 581) in 5 ms on localhost (1/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 376.0 (TID 579). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 376.0 (TID 578) in 7 ms on localhost (2/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 376.0 (TID 579) in 7 ms on localhost (3/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 376.0 (TID 580) in 7 ms on localhost (4/4) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 376.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 376 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 190 finished: reduce at LDAOptimizer.scala:453, took 0.014476 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 191 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 377(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 377 (MapPartitionsRDD[343] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=696358, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_260 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8440) called with curMem=716654, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_260_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_260_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 260 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 377 (MapPartitionsRDD[343] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 377.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 377.0 (TID 582, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 377.0 (TID 583, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 377.0 (TID 584, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 377.0 (TID 585, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 377.0 (TID 582) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 377.0 (TID 585) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 377.0 (TID 584) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 377.0 (TID 583) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 377.0 (TID 585). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 377.0 (TID 584). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 377.0 (TID 582). 1358 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 377.0 (TID 584) in 5 ms on localhost (1/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 377.0 (TID 582) in 6 ms on localhost (2/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 377.0 (TID 585) in 6 ms on localhost (3/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 377.0 (TID 583). 1358 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 377.0 (TID 583) in 6 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 377 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 377.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 191 finished: reduce at LDAOptimizer.scala:456, took 0.016324 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 192 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 378(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 378 (PartitionwiseSampledRDD[344] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=725094, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_261 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(7659) called with curMem=743910, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_261_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_261_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 261 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 378 (PartitionwiseSampledRDD[344] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 378.0 with 1 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 378.0 (TID 586, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 378.0 (TID 586) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 378.0 (TID 586). 1226 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 378.0 (TID 586) in 3 ms on localhost (1/1) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 378 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 378.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 192 finished: isEmpty at LDAOptimizer.scala:418, took 0.009076 s -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=751569, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_262 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=751745, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_262_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_262_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 262 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 193 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 379(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 379 (MapPartitionsRDD[346] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=752025, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_263 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8425) called with curMem=772281, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_263_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_263_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 263 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 379 (MapPartitionsRDD[346] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 379.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 379.0 (TID 587, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 379.0 (TID 588, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 379.0 (TID 589, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 379.0 (TID 590, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 379.0 (TID 589) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 379.0 (TID 590) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 379.0 (TID 588) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 379.0 (TID 588). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 379.0 (TID 590). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 379.0 (TID 587) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 379.0 (TID 588) in 6 ms on localhost (1/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 379.0 (TID 590) in 6 ms on localhost (2/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 379.0 (TID 589). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 379.0 (TID 589) in 9 ms on localhost (3/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 379.0 (TID 587). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 379.0 (TID 587) in 12 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 379 (reduce at LDAOptimizer.scala:453) finished in 0.012 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 379.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 193 finished: reduce at LDAOptimizer.scala:453, took 0.020124 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 194 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 380(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 380 (MapPartitionsRDD[347] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=780706, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_264 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8436) called with curMem=801002, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_264_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_264_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 264 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 380 (MapPartitionsRDD[347] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 380.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 380.0 (TID 591, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 380.0 (TID 592, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 380.0 (TID 593, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 380.0 (TID 594, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 380.0 (TID 593) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 380.0 (TID 591) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 380.0 (TID 594) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 380.0 (TID 593). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 380.0 (TID 591). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 380.0 (TID 593) in 6 ms on localhost (1/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 380.0 (TID 591) in 7 ms on localhost (2/4) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 380.0 (TID 592) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 380.0 (TID 594). 1534 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 380.0 (TID 594) in 10 ms on localhost (3/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 380.0 (TID 592). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 380.0 (TID 592) in 13 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 380 (reduce at LDAOptimizer.scala:456) finished in 0.014 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 380.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 194 finished: reduce at LDAOptimizer.scala:456, took 0.020831 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 195 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 381(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 381 (PartitionwiseSampledRDD[348] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=809438, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_265 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(7654) called with curMem=828254, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_265_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_265_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 265 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 381 (PartitionwiseSampledRDD[348] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 381.0 with 1 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 381.0 (TID 595, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 381.0 (TID 595) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 381.0 (TID 595). 1226 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 381.0 (TID 595) in 3 ms on localhost (1/1) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 381 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 381.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 195 finished: isEmpty at LDAOptimizer.scala:418, took 0.010426 s -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=835908, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_266 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=836084, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_266_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.8 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_266_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 266 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 196 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 382(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 382 (MapPartitionsRDD[350] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=836364, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_267 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8425) called with curMem=856620, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_267_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_267_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 267 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 382 (MapPartitionsRDD[350] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 382.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 382.0 (TID 596, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 382.0 (TID 597, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 382.0 (TID 598, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 382.0 (TID 599, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 382.0 (TID 598) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 382.0 (TID 599) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 382.0 (TID 597) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 382.0 (TID 596) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 382.0 (TID 596). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 382.0 (TID 598). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 382.0 (TID 596) in 6 ms on localhost (1/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 382.0 (TID 599). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 382.0 (TID 598) in 5 ms on localhost (2/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 382.0 (TID 599) in 6 ms on localhost (3/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 382.0 (TID 597). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 382.0 (TID 597) in 8 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 382 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 382.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 196 finished: reduce at LDAOptimizer.scala:453, took 0.013341 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 197 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 383(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 383 (MapPartitionsRDD[351] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=865045, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_268 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8438) called with curMem=885341, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_268_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_268_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 268 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 383 (MapPartitionsRDD[351] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 383.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 383.0 (TID 600, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 383.0 (TID 601, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 383.0 (TID 602, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 383.0 (TID 603, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 383.0 (TID 600) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 383.0 (TID 603) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 383.0 (TID 602) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 383.0 (TID 601) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 383.0 (TID 602). 1358 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 383.0 (TID 603). 1358 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 383.0 (TID 600). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 383.0 (TID 603) in 6 ms on localhost (1/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 383.0 (TID 601). 1446 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 383.0 (TID 600) in 7 ms on localhost (2/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 383.0 (TID 602) in 6 ms on localhost (3/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 383.0 (TID 601) in 7 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 383 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 383.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 197 finished: reduce at LDAOptimizer.scala:456, took 0.014412 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 198 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 384(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 384 (PartitionwiseSampledRDD[352] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=893779, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_269 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=912595, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_269_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_269_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 269 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 384 (PartitionwiseSampledRDD[352] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 384.0 with 1 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 384.0 (TID 604, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 384.0 (TID 604) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 384.0 (TID 604). 1226 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 384.0 (TID 604) in 3 ms on localhost (1/1) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 384 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 384.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 198 finished: isEmpty at LDAOptimizer.scala:418, took 0.011048 s -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=920252, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_270 stored as values in memory (estimated size 176.0 B, free 729.7 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=920428, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_270_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.7 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_270_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 270 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 199 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 385(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 385 (MapPartitionsRDD[354] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=920708, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_271 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8434) called with curMem=940964, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_271_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_271_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 271 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 385 (MapPartitionsRDD[354] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 385.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 385.0 (TID 605, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 385.0 (TID 606, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 385.0 (TID 607, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 385.0 (TID 608, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 385.0 (TID 605) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 385.0 (TID 607) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 385.0 (TID 608) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 385.0 (TID 606) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 385.0 (TID 606). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 385.0 (TID 606) in 4 ms on localhost (1/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 385.0 (TID 608). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 385.0 (TID 607). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 385.0 (TID 608) in 5 ms on localhost (2/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 385.0 (TID 605). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 385.0 (TID 607) in 6 ms on localhost (3/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 385.0 (TID 605) in 6 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 385 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 385.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 199 finished: reduce at LDAOptimizer.scala:453, took 0.012444 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 200 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 386(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 386 (MapPartitionsRDD[355] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=949398, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_272 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8447) called with curMem=969694, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_272_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_272_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 272 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 386 (MapPartitionsRDD[355] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 386.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 386.0 (TID 609, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 386.0 (TID 610, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 386.0 (TID 611, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 386.0 (TID 612, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 386.0 (TID 609) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 386.0 (TID 610) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 386.0 (TID 612) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 386.0 (TID 611) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 386.0 (TID 609). 1402 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 386.0 (TID 610). 1002 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 386.0 (TID 612). 1358 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 386.0 (TID 611). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 386.0 (TID 610) in 8 ms on localhost (1/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 386.0 (TID 609) in 8 ms on localhost (2/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 386.0 (TID 611) in 9 ms on localhost (3/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 386.0 (TID 612) in 8 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 386 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 386.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 200 finished: reduce at LDAOptimizer.scala:456, took 0.016459 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 201 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 387(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 387 (PartitionwiseSampledRDD[356] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=978141, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_273 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=996957, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_273_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_273_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 273 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 387 (PartitionwiseSampledRDD[356] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 387.0 with 1 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 387.0 (TID 613, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 387.0 (TID 613) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 387.0 (TID 613). 911 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 387.0 (TID 613) in 2 ms on localhost (1/1) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 387 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 387.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 201 finished: isEmpty at LDAOptimizer.scala:418, took 0.008744 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 202 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 388(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 388 (PartitionwiseSampledRDD[356] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1004613, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_274 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=1023429, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_274_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_274_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 274 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 388 (PartitionwiseSampledRDD[356] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 388.0 with 3 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 388.0 (TID 614, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 388.0 (TID 615, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 388.0 (TID 616, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 388.0 (TID 614) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 388.0 (TID 616) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 388.0 (TID 615) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 388.0 (TID 614). 1226 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 388.0 (TID 615). 911 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 388.0 (TID 616). 1226 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 388.0 (TID 614) in 3 ms on localhost (1/3) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 388.0 (TID 615) in 4 ms on localhost (2/3) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 388.0 (TID 616) in 4 ms on localhost (3/3) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 388 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 388.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 202 finished: isEmpty at LDAOptimizer.scala:418, took 0.010452 s -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1031085, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_275 stored as values in memory (estimated size 176.0 B, free 729.6 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1031261, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_275_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.6 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_275_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 275 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 203 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 389(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 389 (MapPartitionsRDD[358] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1031541, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_276 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=1051797, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_276_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_276_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 276 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 389 (MapPartitionsRDD[358] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 389.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 389.0 (TID 617, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 389.0 (TID 618, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 389.0 (TID 619, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 389.0 (TID 620, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 389.0 (TID 618) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 389.0 (TID 620) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 389.0 (TID 619) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 389.0 (TID 617) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 389.0 (TID 617). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 389.0 (TID 620). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 389.0 (TID 618). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 389.0 (TID 619). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 389.0 (TID 620) in 5 ms on localhost (1/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 389.0 (TID 619) in 6 ms on localhost (2/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 389.0 (TID 617) in 7 ms on localhost (3/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 389.0 (TID 618) in 6 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 389 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 389.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 203 finished: reduce at LDAOptimizer.scala:453, took 0.015136 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 204 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 390(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 390 (MapPartitionsRDD[359] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1060229, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_277 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8445) called with curMem=1080525, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_277_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_277_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 277 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 390 (MapPartitionsRDD[359] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 390.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 390.0 (TID 621, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 390.0 (TID 622, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 390.0 (TID 623, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 390.0 (TID 624, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 390.0 (TID 621) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 390.0 (TID 624) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 390.0 (TID 622) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 390.0 (TID 623) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 390.0 (TID 621). 1002 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 390.0 (TID 623). 1002 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 390.0 (TID 621) in 4 ms on localhost (1/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 390.0 (TID 623) in 4 ms on localhost (2/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 390.0 (TID 624). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 390.0 (TID 622). 1358 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 390.0 (TID 624) in 5 ms on localhost (3/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 390.0 (TID 622) in 7 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 390 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 390.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 204 finished: reduce at LDAOptimizer.scala:456, took 0.013471 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 205 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 391(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 391 (PartitionwiseSampledRDD[360] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1088970, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_278 stored as values in memory (estimated size 18.4 KB, free 729.5 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(7674) called with curMem=1107786, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_278_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.5 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_278_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 278 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 391 (PartitionwiseSampledRDD[360] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 391.0 with 1 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 391.0 (TID 625, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 391.0 (TID 625) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 391.0 (TID 625). 911 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 391.0 (TID 625) in 2 ms on localhost (1/1) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 391 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 391.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 205 finished: isEmpty at LDAOptimizer.scala:418, took 0.008086 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 206 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 392(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 392 (PartitionwiseSampledRDD[360] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1115460, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_279 stored as values in memory (estimated size 18.4 KB, free 729.5 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(7674) called with curMem=1134276, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_279_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.5 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_279_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 279 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 392 (PartitionwiseSampledRDD[360] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 392.0 with 3 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 392.0 (TID 626, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 392.0 (TID 627, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 392.0 (TID 628, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 392.0 (TID 627) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 392.0 (TID 628) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 392.0 (TID 626) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 392.0 (TID 628). 1226 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 392.0 (TID 626). 911 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 392.0 (TID 627). 911 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 392.0 (TID 628) in 3 ms on localhost (1/3) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 392.0 (TID 626) in 4 ms on localhost (2/3) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 392.0 (TID 627) in 5 ms on localhost (3/3) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 392 (isEmpty at LDAOptimizer.scala:418) finished in 0.006 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 392.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 206 finished: isEmpty at LDAOptimizer.scala:418, took 0.014705 s -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1141950, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_280 stored as values in memory (estimated size 176.0 B, free 729.5 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1142126, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_280_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.5 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_280_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 280 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 207 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 393(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 393 (MapPartitionsRDD[362] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1142406, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_281 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8445) called with curMem=1162662, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_281_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_281_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 281 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 393 (MapPartitionsRDD[362] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 393.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 393.0 (TID 629, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 393.0 (TID 630, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 393.0 (TID 631, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 393.0 (TID 632, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 393.0 (TID 630) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 393.0 (TID 632) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 393.0 (TID 629) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 393.0 (TID 631) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 393.0 (TID 631). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 393.0 (TID 630). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 393.0 (TID 629). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 393.0 (TID 632). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 393.0 (TID 631) in 4 ms on localhost (1/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 393.0 (TID 629) in 6 ms on localhost (2/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 393.0 (TID 630) in 5 ms on localhost (3/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 393.0 (TID 632) in 5 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 393 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 393.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 207 finished: reduce at LDAOptimizer.scala:453, took 0.013539 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 208 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 394(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 394 (MapPartitionsRDD[363] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1171107, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_282 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8458) called with curMem=1191403, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_282_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.4 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_282_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 282 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 394 (MapPartitionsRDD[363] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 394.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 394.0 (TID 633, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 394.0 (TID 634, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 394.0 (TID 635, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 394.0 (TID 636, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 394.0 (TID 635) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 394.0 (TID 636) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 394.0 (TID 634) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 394.0 (TID 635). 1002 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 394.0 (TID 634). 1002 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 394.0 (TID 633) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 394.0 (TID 636). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 394.0 (TID 635) in 6 ms on localhost (1/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 394.0 (TID 634) in 8 ms on localhost (2/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 394.0 (TID 636) in 8 ms on localhost (3/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 394.0 (TID 633). 1002 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 394.0 (TID 633) in 11 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 394 (reduce at LDAOptimizer.scala:456) finished in 0.013 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 394.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 208 finished: reduce at LDAOptimizer.scala:456, took 0.021308 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 209 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 395(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 395 (PartitionwiseSampledRDD[364] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1199861, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_283 stored as values in memory (estimated size 18.4 KB, free 729.4 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=1218677, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_283_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.4 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_283_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 283 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 395 (PartitionwiseSampledRDD[364] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 395.0 with 1 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 395.0 (TID 637, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 395.0 (TID 637) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 395.0 (TID 637). 1226 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 395.0 (TID 637) in 5 ms on localhost (1/1) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 395.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 395 (isEmpty at LDAOptimizer.scala:418) finished in 0.006 s -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 209 finished: isEmpty at LDAOptimizer.scala:418, took 0.013455 s -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1226333, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_284 stored as values in memory (estimated size 176.0 B, free 729.4 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1226509, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_284_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.4 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_284_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 284 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 210 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 396(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 396 (MapPartitionsRDD[366] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1226789, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_285 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8426) called with curMem=1247045, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_285_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_285_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 285 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 396 (MapPartitionsRDD[366] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 396.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 396.0 (TID 638, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 396.0 (TID 639, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 396.0 (TID 640, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 396.0 (TID 641, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 396.0 (TID 639) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 396.0 (TID 640) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 396.0 (TID 641) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 396.0 (TID 638) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 396.0 (TID 639). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 396.0 (TID 641). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 396.0 (TID 638). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 396.0 (TID 640). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 396.0 (TID 639) in 8 ms on localhost (1/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 396.0 (TID 638) in 9 ms on localhost (2/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 396.0 (TID 641) in 8 ms on localhost (3/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 396.0 (TID 640) in 10 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 396 (reduce at LDAOptimizer.scala:453) finished in 0.011 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 396.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 210 finished: reduce at LDAOptimizer.scala:453, took 0.021740 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 211 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 397(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 397 (MapPartitionsRDD[367] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1255471, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_286 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8437) called with curMem=1275767, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_286_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_286_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 286 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_232_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 397 (MapPartitionsRDD[367] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 397.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 397.0 (TID 642, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 397.0 (TID 643, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 397.0 (TID 644, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 397.0 (TID 645, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 397.0 (TID 643) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 397.0 (TID 645) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 397.0 (TID 644) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 397.0 (TID 642) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_285_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 230 -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 397.0 (TID 644). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_283_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 397.0 (TID 645). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 397.0 (TID 644) in 8 ms on localhost (1/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 397.0 (TID 642). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 397.0 (TID 643). 1358 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 397.0 (TID 645) in 9 ms on localhost (2/4) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 229 -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 397.0 (TID 642) in 10 ms on localhost (3/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 397.0 (TID 643) in 10 ms on localhost (4/4) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 397.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 397 (reduce at LDAOptimizer.scala:456) finished in 0.011 s -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 211 finished: reduce at LDAOptimizer.scala:456, took 0.029294 s -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_282_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 228 -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_281_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 212 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 398(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 227 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 398 (PartitionwiseSampledRDD[368] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1170683, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_287 stored as values in memory (estimated size 18.4 KB, free 729.5 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_280_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=1189499, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_287_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.4 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_287_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 287 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_279_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 398 (PartitionwiseSampledRDD[368] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 398.0 with 1 tasks -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 226 -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 398.0 (TID 646, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 398.0 (TID 646) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_278_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 398.0 (TID 646). 911 bytes result sent to driver -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 225 -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 398 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 398.0 (TID 646) in 2 ms on localhost (1/1) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 398.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 212 finished: isEmpty at LDAOptimizer.scala:418, took 0.010326 s -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_277_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 224 -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_276_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 213 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 399(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 399 (PartitionwiseSampledRDD[368] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 223 -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1086467, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_288 stored as values in memory (estimated size 18.4 KB, free 729.5 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_275_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=1105107, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_288_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.5 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_288_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 288 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 399 (PartitionwiseSampledRDD[368] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 399.0 with 3 tasks -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_274_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 399.0 (TID 647, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 399.0 (TID 648, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 222 -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 399.0 (TID 649, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 399.0 (TID 647) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 399.0 (TID 649) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 399.0 (TID 648) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 399.0 (TID 648). 911 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 399.0 (TID 649). 1226 bytes result sent to driver -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_273_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 399.0 (TID 647). 1226 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 399.0 (TID 648) in 4 ms on localhost (1/3) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 221 -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 399.0 (TID 649) in 4 ms on localhost (2/3) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 399.0 (TID 647) in 5 ms on localhost (3/3) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 399 (isEmpty at LDAOptimizer.scala:418) finished in 0.006 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 399.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 213 finished: isEmpty at LDAOptimizer.scala:418, took 0.012885 s -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1031077, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_289 stored as values in memory (estimated size 176.0 B, free 729.6 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_272_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 220 -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1022819, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_289_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.6 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_271_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_289_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 219 -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 289 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_270_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_269_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 218 -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 214 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 400(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 400 (MapPartitionsRDD[370] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_268_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 217 -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=947180, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_290 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8429) called with curMem=967436, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_290_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_290_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_267_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 290 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 400 (MapPartitionsRDD[370] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 400.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 400.0 (TID 650, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 216 -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 400.0 (TID 651, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 400.0 (TID 652, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 400.0 (TID 653, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 400.0 (TID 651) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 400.0 (TID 652) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 400.0 (TID 653) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 400.0 (TID 650) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 400.0 (TID 652). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_266_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 400.0 (TID 650). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 400.0 (TID 652) in 5 ms on localhost (1/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 400.0 (TID 650) in 9 ms on localhost (2/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 400.0 (TID 651). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 400.0 (TID 651) in 10 ms on localhost (3/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 400.0 (TID 653). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 400.0 (TID 653) in 11 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 400 (reduce at LDAOptimizer.scala:453) finished in 0.012 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 400.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_265_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 214 finished: reduce at LDAOptimizer.scala:453, took 0.023721 s -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 215 -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_264_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 214 -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 215 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 401(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 401 (MapPartitionsRDD[371] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_263_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 213 -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=862845, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_291 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_262_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8442) called with curMem=882861, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_291_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_291_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 291 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 401 (MapPartitionsRDD[371] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 401.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 401.0 (TID 654, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 401.0 (TID 655, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 401.0 (TID 656, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 401.0 (TID 657, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_261_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 401.0 (TID 655) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 401.0 (TID 654) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 401.0 (TID 656) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 401.0 (TID 657) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 212 -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 401.0 (TID 654). 1002 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 401.0 (TID 656). 1002 bytes result sent to driver -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_260_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 401.0 (TID 654) in 6 ms on localhost (1/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 401.0 (TID 656) in 5 ms on localhost (2/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 401.0 (TID 657). 1402 bytes result sent to driver -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 211 -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 401.0 (TID 655). 1358 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 401.0 (TID 657) in 8 ms on localhost (3/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 401.0 (TID 655) in 9 ms on localhost (4/4) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 401.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_259_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 401 (reduce at LDAOptimizer.scala:456) finished in 0.011 s -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 215 finished: reduce at LDAOptimizer.scala:456, took 0.017878 s -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 210 -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_258_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 216 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 402(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 402 (PartitionwiseSampledRDD[372] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=806775, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_292 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=825591, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_292_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_292_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 292 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 402 (PartitionwiseSampledRDD[372] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 402.0 with 1 tasks -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_257_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 402.0 (TID 658, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 402.0 (TID 658) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 209 -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 402.0 (TID 658). 1226 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 402.0 (TID 658) in 4 ms on localhost (1/1) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 402 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_256_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 402.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 216 finished: isEmpty at LDAOptimizer.scala:418, took 0.013674 s -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 208 -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=778040, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_293 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=749530, maxMem=766075207 -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_255_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_293_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 207 -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_293_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 293 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_254_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_253_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 206 -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_252_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 217 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 403(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 205 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 403 (MapPartitionsRDD[374] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=665459, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_294 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_251_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8426) called with curMem=685715, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_294_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 204 -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_294_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 294 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 403 (MapPartitionsRDD[374] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 403.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 403.0 (TID 659, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_250_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 403.0 (TID 660, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 403.0 (TID 661, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 403.0 (TID 662, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 403.0 (TID 659) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 403.0 (TID 661) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 403.0 (TID 662) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 403.0 (TID 660) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_249_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 203 -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 403.0 (TID 659). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 403.0 (TID 662). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 403.0 (TID 659) in 7 ms on localhost (1/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 403.0 (TID 660). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 403.0 (TID 662) in 7 ms on localhost (2/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 403.0 (TID 660) in 7 ms on localhost (3/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 403.0 (TID 661). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_248_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 403.0 (TID 661) in 8 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 403 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 403.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 202 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 217 finished: reduce at LDAOptimizer.scala:453, took 0.016401 s -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_247_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 201 -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_246_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 218 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 404(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 404 (MapPartitionsRDD[375] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=582867, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_295 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_245_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8437) called with curMem=603163, maxMem=766075207 -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 200 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_295_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_295_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 295 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 404 (MapPartitionsRDD[375] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 404.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 404.0 (TID 663, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_244_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 404.0 (TID 664, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 199 -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 404.0 (TID 665, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 404.0 (TID 666, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 404.0 (TID 666) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 404.0 (TID 664) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 404.0 (TID 665) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 404.0 (TID 663) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_243_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 404.0 (TID 665). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 404.0 (TID 663). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 404.0 (TID 665) in 5 ms on localhost (1/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 404.0 (TID 663) in 8 ms on localhost (2/4) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 198 -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 404.0 (TID 666). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 404.0 (TID 664). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 404.0 (TID 664) in 11 ms on localhost (3/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 404.0 (TID 666) in 10 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 404 (reduce at LDAOptimizer.scala:456) finished in 0.012 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 404.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 218 finished: reduce at LDAOptimizer.scala:456, took 0.018574 s -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_242_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 197 -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_241_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 219 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 405(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_240_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 405 (PartitionwiseSampledRDD[376] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 196 -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=500774, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_296 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_239_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=511932, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_296_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 195 -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_296_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 296 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 405 (PartitionwiseSampledRDD[376] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 405.0 with 1 tasks -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_238_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 405.0 (TID 667, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 405.0 (TID 667) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 194 -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 405.0 (TID 667). 911 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 405.0 (TID 667) in 4 ms on localhost (1/1) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 405 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 405.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_237_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 219 finished: isEmpty at LDAOptimizer.scala:418, took 0.010847 s -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 193 -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_236_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 220 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 406(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_235_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 406 (PartitionwiseSampledRDD[376] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 192 -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=396135, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_297 stored as values in memory (estimated size 18.4 KB, free 730.2 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_234_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=406515, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_297_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 191 -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_297_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 297 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 406 (PartitionwiseSampledRDD[376] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 406.0 with 3 tasks -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_233_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 406.0 (TID 668, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 190 -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 406.0 (TID 669, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 406.0 (TID 670, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 406.0 (TID 668) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 406.0 (TID 669) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 406.0 (TID 670) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_231_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 406.0 (TID 668). 1226 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 406.0 (TID 669). 911 bytes result sent to driver -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 189 -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 406.0 (TID 668) in 4 ms on localhost (1/3) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 406.0 (TID 670). 911 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 406.0 (TID 669) in 4 ms on localhost (2/3) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 406.0 (TID 670) in 4 ms on localhost (3/3) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 406 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 406.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 220 finished: isEmpty at LDAOptimizer.scala:418, took 0.011304 s -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_230_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=332549, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_298 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 188 -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=312429, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_298_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_298_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_229_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 298 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:19 INFO ContextCleaner:59 - Cleaned accumulator 187 -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Removed broadcast_227_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 221 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 407(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 407 (MapPartitionsRDD[378] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=303820, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_299 stored as values in memory (estimated size 19.8 KB, free 730.3 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=324076, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_299_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.3 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_299_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 299 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 407 (MapPartitionsRDD[378] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 407.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 407.0 (TID 671, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 407.0 (TID 672, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 407.0 (TID 673, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 407.0 (TID 674, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 407.0 (TID 673) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 407.0 (TID 671) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 407.0 (TID 674) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 407.0 (TID 672) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 407.0 (TID 671). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 407.0 (TID 671) in 5 ms on localhost (1/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 407.0 (TID 673). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 407.0 (TID 674). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 407.0 (TID 672). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 407.0 (TID 673) in 10 ms on localhost (2/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 407.0 (TID 674) in 11 ms on localhost (3/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 407.0 (TID 672) in 11 ms on localhost (4/4) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 407.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 407 (reduce at LDAOptimizer.scala:453) finished in 0.012 s -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 221 finished: reduce at LDAOptimizer.scala:453, took 0.019301 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 222 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 408(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 408 (MapPartitionsRDD[379] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=332509, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_300 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8446) called with curMem=352805, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_300_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_300_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 300 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 408 (MapPartitionsRDD[379] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 408.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 408.0 (TID 675, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 408.0 (TID 676, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 408.0 (TID 677, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 408.0 (TID 678, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 408.0 (TID 676) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 408.0 (TID 677) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 408.0 (TID 678) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 408.0 (TID 675) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 408.0 (TID 677). 1002 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 408.0 (TID 678). 1002 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 408.0 (TID 677) in 5 ms on localhost (1/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 408.0 (TID 676). 1402 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 408.0 (TID 678) in 4 ms on localhost (2/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 408.0 (TID 675). 1002 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 408.0 (TID 676) in 6 ms on localhost (3/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 408.0 (TID 675) in 6 ms on localhost (4/4) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 408.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 408 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 222 finished: reduce at LDAOptimizer.scala:456, took 0.015966 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 223 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 409(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 409 (PartitionwiseSampledRDD[380] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=361251, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_301 stored as values in memory (estimated size 18.4 KB, free 730.2 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(7658) called with curMem=380067, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_301_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_301_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 301 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 409 (PartitionwiseSampledRDD[380] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 409.0 with 1 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 409.0 (TID 679, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 409.0 (TID 679) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 409.0 (TID 679). 1226 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 409.0 (TID 679) in 4 ms on localhost (1/1) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 409 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 409.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 223 finished: isEmpty at LDAOptimizer.scala:418, took 0.012284 s -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=387725, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_302 stored as values in memory (estimated size 176.0 B, free 730.2 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=387901, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_302_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.2 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_302_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 302 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 224 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 410(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 410 (MapPartitionsRDD[382] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=388181, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_303 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8429) called with curMem=408437, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_303_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_303_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 303 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 410 (MapPartitionsRDD[382] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 410.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 410.0 (TID 680, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 410.0 (TID 681, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 410.0 (TID 682, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 410.0 (TID 683, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 410.0 (TID 680) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 410.0 (TID 683) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 410.0 (TID 682) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 410.0 (TID 681) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 410.0 (TID 680). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 410.0 (TID 681). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 410.0 (TID 683). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 410.0 (TID 682). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 410.0 (TID 683) in 4 ms on localhost (1/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 410.0 (TID 681) in 5 ms on localhost (2/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 410.0 (TID 680) in 6 ms on localhost (3/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 410.0 (TID 682) in 5 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 410 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 410.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 224 finished: reduce at LDAOptimizer.scala:453, took 0.012721 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 225 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 411(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 411 (MapPartitionsRDD[383] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=416866, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_304 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8441) called with curMem=437162, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_304_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_304_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 304 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 411 (MapPartitionsRDD[383] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 411.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 411.0 (TID 684, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 411.0 (TID 685, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 411.0 (TID 686, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 411.0 (TID 687, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 411.0 (TID 685) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 411.0 (TID 687) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 411.0 (TID 686) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 411.0 (TID 684) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 411.0 (TID 686). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 411.0 (TID 685). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 411.0 (TID 687). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 411.0 (TID 686) in 5 ms on localhost (1/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 411.0 (TID 685) in 6 ms on localhost (2/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 411.0 (TID 687) in 5 ms on localhost (3/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 411.0 (TID 684). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 411.0 (TID 684) in 7 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 411 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 411.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 225 finished: reduce at LDAOptimizer.scala:456, took 0.014467 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 226 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 412(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 412 (PartitionwiseSampledRDD[384] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=445603, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_305 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=464419, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_305_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_305_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 305 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 412 (PartitionwiseSampledRDD[384] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 412.0 with 1 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 412.0 (TID 688, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 412.0 (TID 688) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 412.0 (TID 688). 1226 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 412.0 (TID 688) in 3 ms on localhost (1/1) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 412 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 412.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 226 finished: isEmpty at LDAOptimizer.scala:418, took 0.007765 s -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=472076, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_306 stored as values in memory (estimated size 176.0 B, free 730.1 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=472252, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_306_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.1 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_306_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 306 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 227 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 413(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 413 (MapPartitionsRDD[386] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=472532, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_307 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8431) called with curMem=492788, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_307_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_307_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 307 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 413 (MapPartitionsRDD[386] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 413.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 413.0 (TID 689, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 413.0 (TID 690, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 413.0 (TID 691, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 413.0 (TID 692, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 413.0 (TID 690) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 413.0 (TID 692) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 413.0 (TID 689) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 413.0 (TID 691) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 413.0 (TID 690). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 413.0 (TID 691). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 413.0 (TID 692). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 413.0 (TID 689). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 413.0 (TID 690) in 5 ms on localhost (1/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 413.0 (TID 691) in 5 ms on localhost (2/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 413.0 (TID 692) in 6 ms on localhost (3/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 413.0 (TID 689) in 7 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 413 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 413.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 227 finished: reduce at LDAOptimizer.scala:453, took 0.012992 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 228 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 414(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 414 (MapPartitionsRDD[387] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=501219, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_308 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8443) called with curMem=521515, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_308_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_308_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 308 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 414 (MapPartitionsRDD[387] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 414.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 414.0 (TID 693, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 414.0 (TID 694, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 414.0 (TID 695, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 414.0 (TID 696, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 414.0 (TID 693) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 414.0 (TID 696) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 414.0 (TID 695) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 414.0 (TID 694) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 414.0 (TID 695). 1402 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 414.0 (TID 696). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 414.0 (TID 695) in 8 ms on localhost (1/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 414.0 (TID 696) in 8 ms on localhost (2/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 414.0 (TID 694). 1358 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 414.0 (TID 694) in 10 ms on localhost (3/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 414.0 (TID 693). 1402 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 414.0 (TID 693) in 13 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 414 (reduce at LDAOptimizer.scala:456) finished in 0.013 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 414.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 228 finished: reduce at LDAOptimizer.scala:456, took 0.020761 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 229 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 415(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 415 (PartitionwiseSampledRDD[388] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=529958, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_309 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(7671) called with curMem=548774, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_309_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_309_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 309 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 415 (PartitionwiseSampledRDD[388] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 415.0 with 1 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 415.0 (TID 697, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 415.0 (TID 697) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 415.0 (TID 697). 1226 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 415.0 (TID 697) in 2 ms on localhost (1/1) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 415 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 415.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 229 finished: isEmpty at LDAOptimizer.scala:418, took 0.008872 s -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=556445, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_310 stored as values in memory (estimated size 176.0 B, free 730.1 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=556621, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_310_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.1 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_310_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 310 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 230 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 416(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 416 (MapPartitionsRDD[390] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=556901, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_311 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8440) called with curMem=577157, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_311_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_311_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 311 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 416 (MapPartitionsRDD[390] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 416.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 416.0 (TID 698, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 416.0 (TID 699, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 416.0 (TID 700, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 416.0 (TID 701, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 416.0 (TID 699) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 416.0 (TID 701) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 416.0 (TID 698) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 416.0 (TID 700) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 416.0 (TID 699). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 416.0 (TID 701). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 416.0 (TID 699) in 8 ms on localhost (1/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 416.0 (TID 701) in 7 ms on localhost (2/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 416.0 (TID 700). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 416.0 (TID 700) in 8 ms on localhost (3/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 416.0 (TID 698). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 416.0 (TID 698) in 11 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 416 (reduce at LDAOptimizer.scala:453) finished in 0.011 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 416.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 230 finished: reduce at LDAOptimizer.scala:453, took 0.017045 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 231 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 417(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 417 (MapPartitionsRDD[391] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=585597, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_312 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8452) called with curMem=605893, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_312_piece0 stored as bytes in memory (estimated size 8.3 KB, free 730.0 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_312_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 312 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 417 (MapPartitionsRDD[391] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 417.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 417.0 (TID 702, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 417.0 (TID 703, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 417.0 (TID 704, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 417.0 (TID 705, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 417.0 (TID 703) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 417.0 (TID 705) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 417.0 (TID 704) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 417.0 (TID 702) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 417.0 (TID 704). 1002 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 417.0 (TID 702). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 417.0 (TID 705). 1358 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 417.0 (TID 703). 1402 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 417.0 (TID 702) in 6 ms on localhost (1/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 417.0 (TID 705) in 5 ms on localhost (2/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 417.0 (TID 704) in 6 ms on localhost (3/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 417.0 (TID 703) in 6 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 417 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 417.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 231 finished: reduce at LDAOptimizer.scala:456, took 0.012865 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 232 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 418(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 418 (PartitionwiseSampledRDD[392] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=614345, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_313 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(7658) called with curMem=633161, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_313_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_313_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 313 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 418 (PartitionwiseSampledRDD[392] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 418.0 with 1 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 418.0 (TID 706, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 418.0 (TID 706) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 418.0 (TID 706). 1226 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 418.0 (TID 706) in 2 ms on localhost (1/1) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 418 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 418.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 232 finished: isEmpty at LDAOptimizer.scala:418, took 0.007958 s -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=640819, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_314 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=640995, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_314_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_314_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 314 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 233 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 419(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 419 (MapPartitionsRDD[394] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=641275, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_315 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8429) called with curMem=661531, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_315_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_315_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 315 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 419 (MapPartitionsRDD[394] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 419.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 419.0 (TID 707, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 419.0 (TID 708, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 419.0 (TID 709, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 419.0 (TID 710, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 419.0 (TID 708) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 419.0 (TID 709) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 419.0 (TID 707) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 419.0 (TID 710) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 419.0 (TID 707). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 419.0 (TID 708). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 419.0 (TID 707) in 7 ms on localhost (1/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 419.0 (TID 708) in 8 ms on localhost (2/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 419.0 (TID 709). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 419.0 (TID 709) in 8 ms on localhost (3/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 419.0 (TID 710). 1265 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 419.0 (TID 710) in 9 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 419 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 419.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 233 finished: reduce at LDAOptimizer.scala:453, took 0.017880 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Got job 234 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:19 INFO DAGScheduler:59 - Final stage: ResultStage 420(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting ResultStage 420 (MapPartitionsRDD[395] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=669960, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_316 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:19 INFO MemoryStore:59 - ensureFreeSpace(8442) called with curMem=690256, maxMem=766075207 -2016-06-13 19:45:19 INFO MemoryStore:59 - Block broadcast_316_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:19 INFO BlockManagerInfo:59 - Added broadcast_316_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:19 INFO SparkContext:59 - Created broadcast 316 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:19 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 420 (MapPartitionsRDD[395] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Adding task set 420.0 with 4 tasks -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 420.0 (TID 711, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 1.0 in stage 420.0 (TID 712, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 2.0 in stage 420.0 (TID 713, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Starting task 3.0 in stage 420.0 (TID 714, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:19 INFO Executor:59 - Running task 1.0 in stage 420.0 (TID 712) -2016-06-13 19:45:19 INFO Executor:59 - Running task 3.0 in stage 420.0 (TID 714) -2016-06-13 19:45:19 INFO Executor:59 - Running task 0.0 in stage 420.0 (TID 711) -2016-06-13 19:45:19 INFO Executor:59 - Running task 2.0 in stage 420.0 (TID 713) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 2.0 in stage 420.0 (TID 713). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 3.0 in stage 420.0 (TID 714). 1358 bytes result sent to driver -2016-06-13 19:45:19 INFO Executor:59 - Finished task 0.0 in stage 420.0 (TID 711). 1314 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 2.0 in stage 420.0 (TID 713) in 4 ms on localhost (1/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 3.0 in stage 420.0 (TID 714) in 4 ms on localhost (2/4) -2016-06-13 19:45:19 INFO Executor:59 - Finished task 1.0 in stage 420.0 (TID 712). 1446 bytes result sent to driver -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 420.0 (TID 711) in 6 ms on localhost (3/4) -2016-06-13 19:45:19 INFO TaskSetManager:59 - Finished task 1.0 in stage 420.0 (TID 712) in 6 ms on localhost (4/4) -2016-06-13 19:45:19 INFO DAGScheduler:59 - ResultStage 420 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 420.0, whose tasks have all completed, from pool -2016-06-13 19:45:19 INFO DAGScheduler:59 - Job 234 finished: reduce at LDAOptimizer.scala:456, took 0.011980 s -2016-06-13 19:45:19 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 235 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 421(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 421 (PartitionwiseSampledRDD[396] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=698698, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_317 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=717514, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_317_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_317_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 317 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 421 (PartitionwiseSampledRDD[396] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 421.0 with 1 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 421.0 (TID 715, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 421.0 (TID 715) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 421.0 (TID 715). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 421.0 (TID 715) in 3 ms on localhost (1/1) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 421 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 421.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 235 finished: isEmpty at LDAOptimizer.scala:418, took 0.007948 s -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=725174, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_318 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=725350, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_318_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_318_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 318 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 236 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 422(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 422 (MapPartitionsRDD[398] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=725630, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_319 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8428) called with curMem=745886, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_319_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_319_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 319 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 422 (MapPartitionsRDD[398] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 422.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 422.0 (TID 716, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 422.0 (TID 717, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 422.0 (TID 718, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 422.0 (TID 719, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 422.0 (TID 717) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 422.0 (TID 719) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 422.0 (TID 716) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 422.0 (TID 718) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 422.0 (TID 718). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 422.0 (TID 716). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 422.0 (TID 717). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 422.0 (TID 718) in 5 ms on localhost (1/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 422.0 (TID 719). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 422.0 (TID 716) in 6 ms on localhost (2/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 422.0 (TID 717) in 6 ms on localhost (3/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 422.0 (TID 719) in 6 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 422 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 422.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 236 finished: reduce at LDAOptimizer.scala:453, took 0.013085 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 237 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 423(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 423 (MapPartitionsRDD[399] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=754314, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_320 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8440) called with curMem=774610, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_320_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_320_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 320 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 423 (MapPartitionsRDD[399] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 423.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 423.0 (TID 720, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 423.0 (TID 721, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 423.0 (TID 722, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 423.0 (TID 723, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 423.0 (TID 720) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 423.0 (TID 723) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 423.0 (TID 721) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 423.0 (TID 722) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 423.0 (TID 722). 1314 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 423.0 (TID 721). 1358 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 423.0 (TID 720). 1358 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 423.0 (TID 722) in 4 ms on localhost (1/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 423.0 (TID 723). 1402 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 423.0 (TID 721) in 5 ms on localhost (2/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 423.0 (TID 720) in 6 ms on localhost (3/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 423.0 (TID 723) in 6 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 423 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 423.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 237 finished: reduce at LDAOptimizer.scala:456, took 0.012519 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 238 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 424(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 424 (PartitionwiseSampledRDD[400] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=783050, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_321 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(7655) called with curMem=801866, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_321_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_321_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 321 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 424 (PartitionwiseSampledRDD[400] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 424.0 with 1 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 424.0 (TID 724, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 424.0 (TID 724) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 424.0 (TID 724). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 424.0 (TID 724) in 3 ms on localhost (1/1) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 424 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 424.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 238 finished: isEmpty at LDAOptimizer.scala:418, took 0.009051 s -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=809521, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_322 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=809697, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_322_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.8 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_322_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 322 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 239 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 425(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 425 (MapPartitionsRDD[402] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=809977, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_323 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8429) called with curMem=830233, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_323_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_323_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 323 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 425 (MapPartitionsRDD[402] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 425.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 425.0 (TID 725, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 425.0 (TID 726, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 425.0 (TID 727, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 425.0 (TID 728, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 425.0 (TID 726) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 425.0 (TID 728) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 425.0 (TID 725) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 425.0 (TID 727) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 425.0 (TID 726). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 425.0 (TID 728). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 425.0 (TID 727). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 425.0 (TID 726) in 4 ms on localhost (1/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 425.0 (TID 725). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 425.0 (TID 728) in 5 ms on localhost (2/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 425.0 (TID 727) in 5 ms on localhost (3/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 425.0 (TID 725) in 6 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 425 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 425.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 239 finished: reduce at LDAOptimizer.scala:453, took 0.011867 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 240 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 426(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 426 (MapPartitionsRDD[403] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=838662, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_324 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8435) called with curMem=858958, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_324_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_324_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 324 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 426 (MapPartitionsRDD[403] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 426.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 426.0 (TID 729, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 426.0 (TID 730, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 426.0 (TID 731, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 426.0 (TID 732, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 426.0 (TID 729) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 426.0 (TID 732) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 426.0 (TID 731) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 426.0 (TID 730) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 426.0 (TID 730). 1002 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 426.0 (TID 732). 1314 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 426.0 (TID 729). 1314 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 426.0 (TID 730) in 5 ms on localhost (1/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 426.0 (TID 729) in 5 ms on localhost (2/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 426.0 (TID 732) in 5 ms on localhost (3/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 426.0 (TID 731). 1358 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 426.0 (TID 731) in 9 ms on localhost (4/4) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 426.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 426 (reduce at LDAOptimizer.scala:456) finished in 0.009 s -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 240 finished: reduce at LDAOptimizer.scala:456, took 0.016370 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 241 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 427(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 427 (PartitionwiseSampledRDD[404] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=867393, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_325 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(7659) called with curMem=886209, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_325_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_325_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 325 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 427 (PartitionwiseSampledRDD[404] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 427.0 with 1 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 427.0 (TID 733, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 427.0 (TID 733) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 427.0 (TID 733). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 427.0 (TID 733) in 3 ms on localhost (1/1) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 427 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 427.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 241 finished: isEmpty at LDAOptimizer.scala:418, took 0.012539 s -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=893868, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_326 stored as values in memory (estimated size 176.0 B, free 729.7 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=894044, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_326_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.7 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_326_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 326 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 242 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 428(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 428 (MapPartitionsRDD[406] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=894324, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_327 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8435) called with curMem=914580, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_327_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_327_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 327 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 428 (MapPartitionsRDD[406] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 428.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 428.0 (TID 734, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 428.0 (TID 735, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 428.0 (TID 736, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 428.0 (TID 737, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 428.0 (TID 734) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 428.0 (TID 736) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 428.0 (TID 737) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 428.0 (TID 735) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 428.0 (TID 734). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 428.0 (TID 736). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 428.0 (TID 737). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 428.0 (TID 734) in 7 ms on localhost (1/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 428.0 (TID 736) in 7 ms on localhost (2/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 428.0 (TID 737) in 7 ms on localhost (3/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 428.0 (TID 735). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 428.0 (TID 735) in 8 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 428 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 428.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 242 finished: reduce at LDAOptimizer.scala:453, took 0.015832 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 243 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 429(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 429 (MapPartitionsRDD[407] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=923015, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_328 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8442) called with curMem=943311, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_328_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_328_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 328 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 429 (MapPartitionsRDD[407] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 429.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 429.0 (TID 738, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 429.0 (TID 739, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 429.0 (TID 740, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 429.0 (TID 741, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 429.0 (TID 739) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 429.0 (TID 741) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 429.0 (TID 738) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 429.0 (TID 740) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 429.0 (TID 739). 1578 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 429.0 (TID 738). 1358 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 429.0 (TID 740). 1358 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 429.0 (TID 739) in 18 ms on localhost (1/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 429.0 (TID 738) in 20 ms on localhost (2/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 429.0 (TID 740) in 18 ms on localhost (3/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 429.0 (TID 741). 1314 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 429.0 (TID 741) in 29 ms on localhost (4/4) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 429.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 429 (reduce at LDAOptimizer.scala:456) finished in 0.031 s -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 243 finished: reduce at LDAOptimizer.scala:456, took 0.038528 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 244 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 430(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 430 (PartitionwiseSampledRDD[408] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=951753, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_329 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=970569, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_329_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_329_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 329 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 430 (PartitionwiseSampledRDD[408] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 430.0 with 1 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 430.0 (TID 742, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 430.0 (TID 742) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 430.0 (TID 742). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 430.0 (TID 742) in 3 ms on localhost (1/1) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 430 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 430.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 244 finished: isEmpty at LDAOptimizer.scala:418, took 0.012119 s -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=978225, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_330 stored as values in memory (estimated size 176.0 B, free 729.7 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=978401, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_330_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.7 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_330_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 330 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 245 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 431(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 431 (MapPartitionsRDD[410] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=978681, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_331 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8422) called with curMem=998937, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_331_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_331_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 331 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 431 (MapPartitionsRDD[410] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 431.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 431.0 (TID 743, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 431.0 (TID 744, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 431.0 (TID 745, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 431.0 (TID 746, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 431.0 (TID 743) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 431.0 (TID 746) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 431.0 (TID 745) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 431.0 (TID 744) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 431.0 (TID 743). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 431.0 (TID 746). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 431.0 (TID 745). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 431.0 (TID 743) in 6 ms on localhost (1/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 431.0 (TID 744). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 431.0 (TID 745) in 6 ms on localhost (2/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 431.0 (TID 746) in 6 ms on localhost (3/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 431.0 (TID 744) in 7 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 431 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 431.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 245 finished: reduce at LDAOptimizer.scala:453, took 0.013602 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 246 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 432(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 432 (MapPartitionsRDD[411] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1007359, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_332 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=1027655, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_332_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_332_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 332 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 432 (MapPartitionsRDD[411] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 432.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 432.0 (TID 747, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 432.0 (TID 748, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 432.0 (TID 749, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 432.0 (TID 750, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 432.0 (TID 748) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 432.0 (TID 750) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 432.0 (TID 749) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 432.0 (TID 747) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 432.0 (TID 750). 1314 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 432.0 (TID 749). 1358 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 432.0 (TID 747). 1314 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 432.0 (TID 750) in 7 ms on localhost (1/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 432.0 (TID 749) in 9 ms on localhost (2/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 432.0 (TID 748). 1490 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 432.0 (TID 747) in 10 ms on localhost (3/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 432.0 (TID 748) in 11 ms on localhost (4/4) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 432.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 432 (reduce at LDAOptimizer.scala:456) finished in 0.016 s -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 246 finished: reduce at LDAOptimizer.scala:456, took 0.023155 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 247 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 433(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 433 (PartitionwiseSampledRDD[412] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1036088, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_333 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=1054904, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_333_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_333_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 333 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 433 (PartitionwiseSampledRDD[412] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 433.0 with 1 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 433.0 (TID 751, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 433.0 (TID 751) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 433.0 (TID 751). 911 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 433.0 (TID 751) in 2 ms on localhost (1/1) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 433 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 433.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 247 finished: isEmpty at LDAOptimizer.scala:418, took 0.010800 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 248 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 434(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 434 (PartitionwiseSampledRDD[412] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1062561, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_334 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=1081377, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_334_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.5 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_334_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 334 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 434 (PartitionwiseSampledRDD[412] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 434.0 with 3 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 434.0 (TID 752, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 434.0 (TID 753, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 434.0 (TID 754, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 434.0 (TID 752) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 434.0 (TID 754) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 434.0 (TID 753) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 434.0 (TID 754). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 434.0 (TID 753). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 434.0 (TID 752). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 434.0 (TID 754) in 3 ms on localhost (1/3) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 434.0 (TID 753) in 3 ms on localhost (2/3) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 434.0 (TID 752) in 5 ms on localhost (3/3) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 434 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 434.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 248 finished: isEmpty at LDAOptimizer.scala:418, took 0.010249 s -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1089034, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_335 stored as values in memory (estimated size 176.0 B, free 729.5 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1089210, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_335_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.5 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_335_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 335 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 249 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 435(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 435 (MapPartitionsRDD[414] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1089490, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_336 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8430) called with curMem=1109746, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_336_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_336_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 336 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 435 (MapPartitionsRDD[414] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 435.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 435.0 (TID 755, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 435.0 (TID 756, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 435.0 (TID 757, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 435.0 (TID 758, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 435.0 (TID 755) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 435.0 (TID 757) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 435.0 (TID 758) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 435.0 (TID 756) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 435.0 (TID 757). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 435.0 (TID 755). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 435.0 (TID 758). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 435.0 (TID 757) in 7 ms on localhost (1/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 435.0 (TID 755) in 7 ms on localhost (2/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 435.0 (TID 758) in 7 ms on localhost (3/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 435.0 (TID 756). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 435.0 (TID 756) in 9 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 435 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 435.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 249 finished: reduce at LDAOptimizer.scala:453, took 0.016577 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 250 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 436(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 436 (MapPartitionsRDD[415] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1118176, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_337 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8442) called with curMem=1138472, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_337_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_337_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 337 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 436 (MapPartitionsRDD[415] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 436.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 436.0 (TID 759, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 436.0 (TID 760, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 436.0 (TID 761, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 436.0 (TID 762, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 436.0 (TID 760) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 436.0 (TID 761) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 436.0 (TID 762) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 436.0 (TID 759) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 436.0 (TID 760). 1358 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 436.0 (TID 759). 1002 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 436.0 (TID 760) in 7 ms on localhost (1/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 436.0 (TID 759) in 8 ms on localhost (2/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 436.0 (TID 762). 1314 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 436.0 (TID 762) in 8 ms on localhost (3/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 436.0 (TID 761). 1314 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 436.0 (TID 761) in 8 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 436 (reduce at LDAOptimizer.scala:456) finished in 0.009 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 436.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 250 finished: reduce at LDAOptimizer.scala:456, took 0.014748 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 251 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 437(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 437 (PartitionwiseSampledRDD[416] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1146914, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_338 stored as values in memory (estimated size 18.4 KB, free 729.5 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=1165730, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_338_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.5 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_338_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 338 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 437 (PartitionwiseSampledRDD[416] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 437.0 with 1 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 437.0 (TID 763, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 437.0 (TID 763) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 437.0 (TID 763). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 437.0 (TID 763) in 2 ms on localhost (1/1) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 437 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 437.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 251 finished: isEmpty at LDAOptimizer.scala:418, took 0.007479 s -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1173387, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_339 stored as values in memory (estimated size 176.0 B, free 729.5 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1173563, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_339_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.5 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_339_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 339 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 252 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 438(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 438 (MapPartitionsRDD[418] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1173843, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_340 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=1194099, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_340_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_340_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 340 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 438 (MapPartitionsRDD[418] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 438.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 438.0 (TID 764, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 438.0 (TID 765, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 438.0 (TID 766, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 438.0 (TID 767, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 438.0 (TID 764) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 438.0 (TID 767) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 438.0 (TID 765) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 438.0 (TID 766) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 438.0 (TID 766). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 438.0 (TID 764). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 438.0 (TID 765). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 438.0 (TID 766) in 4 ms on localhost (1/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 438.0 (TID 765) in 6 ms on localhost (2/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 438.0 (TID 767). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 438.0 (TID 764) in 7 ms on localhost (3/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 438.0 (TID 767) in 6 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 438 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 438.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 252 finished: reduce at LDAOptimizer.scala:453, took 0.015156 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 253 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 439(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 439 (MapPartitionsRDD[419] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1202531, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_341 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8445) called with curMem=1222827, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_341_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_341_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 341 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 439 (MapPartitionsRDD[419] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 439.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 439.0 (TID 768, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 439.0 (TID 769, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 439.0 (TID 770, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 439.0 (TID 771, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 439.0 (TID 769) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 439.0 (TID 771) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 439.0 (TID 770) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 439.0 (TID 768) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 439.0 (TID 770). 1002 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 439.0 (TID 769). 1402 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 439.0 (TID 770) in 6 ms on localhost (1/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 439.0 (TID 771). 1490 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 439.0 (TID 769) in 7 ms on localhost (2/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 439.0 (TID 771) in 6 ms on localhost (3/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 439.0 (TID 768). 1314 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 439.0 (TID 768) in 9 ms on localhost (4/4) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 439.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 439 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 253 finished: reduce at LDAOptimizer.scala:456, took 0.017140 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 254 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 440(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 440 (PartitionwiseSampledRDD[420] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1231272, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_342 stored as values in memory (estimated size 18.4 KB, free 729.4 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=1250088, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_342_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.4 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_342_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 342 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 440 (PartitionwiseSampledRDD[420] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 440.0 with 1 tasks -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_341_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 440.0 (TID 772, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 440.0 (TID 772) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 273 -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 440.0 (TID 772). 911 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 440.0 (TID 772) in 4 ms on localhost (1/1) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 440.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 440 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_340_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 254 finished: isEmpty at LDAOptimizer.scala:418, took 0.024188 s -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 272 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_339_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 255 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 441(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 441 (PartitionwiseSampledRDD[420] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_338_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1173386, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_343 stored as values in memory (estimated size 18.4 KB, free 729.4 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 271 -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=1171906, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_343_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.5 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_343_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 343 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_337_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 441 (PartitionwiseSampledRDD[420] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 441.0 with 3 tasks -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 270 -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 441.0 (TID 773, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 441.0 (TID 774, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 441.0 (TID 775, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 441.0 (TID 773) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 441.0 (TID 775) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 441.0 (TID 774) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_336_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 269 -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 441.0 (TID 773). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 441.0 (TID 775). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 441.0 (TID 774). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 441.0 (TID 773) in 5 ms on localhost (1/3) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 441.0 (TID 774) in 4 ms on localhost (2/3) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 441.0 (TID 775) in 3 ms on localhost (3/3) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 441 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 441.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 255 finished: isEmpty at LDAOptimizer.scala:418, took 0.012382 s -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_335_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1141978, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_344 stored as values in memory (estimated size 176.0 B, free 729.5 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1115681, maxMem=766075207 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_334_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_344_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.5 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 268 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_344_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 344 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_333_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 267 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_332_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 266 -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_331_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 256 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 442(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 265 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 442 (MapPartitionsRDD[422] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1031801, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_345 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_330_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8427) called with curMem=1051881, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_345_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_329_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_345_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 345 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 264 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 442 (MapPartitionsRDD[422] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 442.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 442.0 (TID 776, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 442.0 (TID 777, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 442.0 (TID 778, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 442.0 (TID 779, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 442.0 (TID 776) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_328_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 442.0 (TID 778) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 442.0 (TID 776). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 442.0 (TID 779) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 442.0 (TID 778). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 442.0 (TID 777) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 442.0 (TID 776) in 9 ms on localhost (1/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 442.0 (TID 779). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 442.0 (TID 778) in 8 ms on localhost (2/4) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 263 -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 442.0 (TID 777). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 442.0 (TID 779) in 14 ms on localhost (3/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 442.0 (TID 777) in 14 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 442 (reduce at LDAOptimizer.scala:453) finished in 0.015 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 442.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 256 finished: reduce at LDAOptimizer.scala:453, took 0.022049 s -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_327_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 262 -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_326_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 257 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 443(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 443 (MapPartitionsRDD[423] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=968292, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_346 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_325_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 261 -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8441) called with curMem=969772, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_346_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_346_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 346 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_324_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 443 (MapPartitionsRDD[423] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 443.0 with 4 tasks -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 260 -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 443.0 (TID 780, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 443.0 (TID 781, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 443.0 (TID 782, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 443.0 (TID 783, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 443.0 (TID 780) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 443.0 (TID 783) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 443.0 (TID 781) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 443.0 (TID 782) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_323_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 443.0 (TID 780). 1002 bytes result sent to driver -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 259 -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 443.0 (TID 780) in 4 ms on localhost (1/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 443.0 (TID 783). 1314 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 443.0 (TID 781). 1402 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 443.0 (TID 783) in 5 ms on localhost (2/4) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_322_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 443.0 (TID 781) in 7 ms on localhost (3/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 443.0 (TID 782). 1402 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 443.0 (TID 782) in 8 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 443 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 443.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 257 finished: reduce at LDAOptimizer.scala:456, took 0.015792 s -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_321_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 258 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_320_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 257 -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 258 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 444(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 444 (PartitionwiseSampledRDD[424] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=836450, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_347 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_319_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 256 -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(7655) called with curMem=855266, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_347_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_347_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 347 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_318_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 444 (PartitionwiseSampledRDD[424] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 444.0 with 1 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 444.0 (TID 784, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 444.0 (TID 784) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_317_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 444.0 (TID 784). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 255 -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 444.0 (TID 784) in 4 ms on localhost (1/1) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 444 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 444.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 258 finished: isEmpty at LDAOptimizer.scala:418, took 0.012058 s -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=807251, maxMem=766075207 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_316_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_348 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 254 -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=798998, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_348_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.8 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_315_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_348_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 348 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 253 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_314_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 259 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_313_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 445(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 252 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 445 (MapPartitionsRDD[426] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=743640, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_349 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_312_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 251 -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8427) called with curMem=743600, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_349_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_349_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 349 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_311_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 445 (MapPartitionsRDD[426] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 445.0 with 4 tasks -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 250 -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 445.0 (TID 785, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 445.0 (TID 786, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 445.0 (TID 787, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 445.0 (TID 788, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 445.0 (TID 785) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 445.0 (TID 787) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 445.0 (TID 788) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 445.0 (TID 786) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_310_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 445.0 (TID 788). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_309_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 445.0 (TID 785). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 445.0 (TID 787). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 445.0 (TID 788) in 7 ms on localhost (1/4) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 249 -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 445.0 (TID 785) in 9 ms on localhost (2/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 445.0 (TID 787) in 9 ms on localhost (3/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 445.0 (TID 786). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 445.0 (TID 786) in 9 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 445 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 445.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 259 finished: reduce at LDAOptimizer.scala:453, took 0.018407 s -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_308_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 248 -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 260 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_307_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 446(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 247 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 446 (MapPartitionsRDD[427] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_306_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=630849, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_350 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_305_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8439) called with curMem=651145, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_350_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 246 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_350_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 350 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 446 (MapPartitionsRDD[427] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 446.0 with 4 tasks -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_304_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 446.0 (TID 789, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 446.0 (TID 790, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 446.0 (TID 791, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 446.0 (TID 792, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 446.0 (TID 789) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 446.0 (TID 791) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 446.0 (TID 790) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 446.0 (TID 792) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 245 -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 446.0 (TID 791). 1358 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 446.0 (TID 789). 1402 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 446.0 (TID 791) in 7 ms on localhost (1/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 446.0 (TID 789) in 9 ms on localhost (2/4) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_303_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 446.0 (TID 792). 1358 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 446.0 (TID 792) in 9 ms on localhost (3/4) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 244 -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 446.0 (TID 790). 1402 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 446.0 (TID 790) in 11 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 446 (reduce at LDAOptimizer.scala:456) finished in 0.013 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 446.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 260 finished: reduce at LDAOptimizer.scala:456, took 0.023584 s -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_302_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_301_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 243 -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_300_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 261 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 447(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 242 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 447 (PartitionwiseSampledRDD[428] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=498985, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_351 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_299_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=517801, maxMem=766075207 -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 241 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_351_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_351_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 351 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 447 (PartitionwiseSampledRDD[428] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 447.0 with 1 tasks -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_298_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 447.0 (TID 793, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 447.0 (TID 793) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_297_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 447.0 (TID 793). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 447.0 (TID 793) in 2 ms on localhost (1/1) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 447 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 447.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 240 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 261 finished: isEmpty at LDAOptimizer.scala:418, took 0.008780 s -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=490869, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_352 stored as values in memory (estimated size 176.0 B, free 730.1 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_296_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 239 -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=472229, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_352_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.1 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_352_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_295_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 352 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 238 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_294_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 237 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_293_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 262 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 448(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 448 (MapPartitionsRDD[430] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=406977, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_353 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_292_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 236 -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8436) called with curMem=408417, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_353_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_353_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 353 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_291_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 448 (MapPartitionsRDD[430] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 448.0 with 4 tasks -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 235 -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 448.0 (TID 794, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 448.0 (TID 795, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 448.0 (TID 796, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 448.0 (TID 797, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 448.0 (TID 794) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 448.0 (TID 797) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 448.0 (TID 796) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_290_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 448.0 (TID 795) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 234 -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 448.0 (TID 796). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 448.0 (TID 794). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 448.0 (TID 796) in 6 ms on localhost (1/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 448.0 (TID 797). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 448.0 (TID 794) in 7 ms on localhost (2/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 448.0 (TID 795). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 448.0 (TID 797) in 7 ms on localhost (3/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 448.0 (TID 795) in 7 ms on localhost (4/4) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_289_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 448 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 448.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 262 finished: reduce at LDAOptimizer.scala:453, took 0.015931 s -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_288_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 233 -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 263 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 449(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 449 (MapPartitionsRDD[431] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_287_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 232 -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=306028, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_354 stored as values in memory (estimated size 19.8 KB, free 730.3 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8449) called with curMem=317887, maxMem=766075207 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_286_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_354_piece0 stored as bytes in memory (estimated size 8.3 KB, free 730.3 MB) -2016-06-13 19:45:20 INFO ContextCleaner:59 - Cleaned accumulator 231 -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_354_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.5 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 354 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 449 (MapPartitionsRDD[431] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 449.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 449.0 (TID 798, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 449.0 (TID 799, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Removed broadcast_284_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 449.0 (TID 800, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 449.0 (TID 801, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 449.0 (TID 800) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 449.0 (TID 799) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 449.0 (TID 801) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 449.0 (TID 801). 1002 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 449.0 (TID 798) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 449.0 (TID 801) in 4 ms on localhost (1/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 449.0 (TID 799). 1002 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 449.0 (TID 800). 1002 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 449.0 (TID 799) in 8 ms on localhost (2/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 449.0 (TID 800) in 7 ms on localhost (3/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 449.0 (TID 798). 1402 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 449.0 (TID 798) in 10 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 449 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 449.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 263 finished: reduce at LDAOptimizer.scala:456, took 0.016385 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 264 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 450(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 450 (PartitionwiseSampledRDD[432] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=305584, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_355 stored as values in memory (estimated size 18.4 KB, free 730.3 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(7659) called with curMem=324400, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_355_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.3 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_355_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 355 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 450 (PartitionwiseSampledRDD[432] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 450.0 with 1 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 450.0 (TID 802, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 450.0 (TID 802) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 450.0 (TID 802). 911 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 450.0 (TID 802) in 3 ms on localhost (1/1) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 450 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 450.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 264 finished: isEmpty at LDAOptimizer.scala:418, took 0.008291 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 265 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 451(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 451 (PartitionwiseSampledRDD[432] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=332059, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_356 stored as values in memory (estimated size 18.4 KB, free 730.3 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(7659) called with curMem=350875, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_356_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_356_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 356 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 451 (PartitionwiseSampledRDD[432] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 451.0 with 3 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 451.0 (TID 803, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 451.0 (TID 804, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 451.0 (TID 805, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 451.0 (TID 804) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 451.0 (TID 804). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 451.0 (TID 803) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 451.0 (TID 805) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 451.0 (TID 804) in 6 ms on localhost (1/3) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 451.0 (TID 803). 911 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 451.0 (TID 805). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 451.0 (TID 803) in 8 ms on localhost (2/3) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 451.0 (TID 805) in 6 ms on localhost (3/3) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 451 (isEmpty at LDAOptimizer.scala:418) finished in 0.009 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 451.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 265 finished: isEmpty at LDAOptimizer.scala:418, took 0.015280 s -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=358534, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_357 stored as values in memory (estimated size 176.0 B, free 730.2 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=358710, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_357_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.2 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_357_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 357 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 266 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 452(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 452 (MapPartitionsRDD[434] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=358990, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_358 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8437) called with curMem=379246, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_358_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_358_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 358 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 452 (MapPartitionsRDD[434] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 452.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 452.0 (TID 806, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 452.0 (TID 807, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 452.0 (TID 808, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 452.0 (TID 809, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 452.0 (TID 807) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 452.0 (TID 809) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 452.0 (TID 806) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 452.0 (TID 808) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 452.0 (TID 806). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 452.0 (TID 807). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 452.0 (TID 806) in 4 ms on localhost (1/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 452.0 (TID 807) in 4 ms on localhost (2/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 452.0 (TID 809). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 452.0 (TID 808). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 452.0 (TID 809) in 5 ms on localhost (3/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 452.0 (TID 808) in 7 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 452 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 452.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 266 finished: reduce at LDAOptimizer.scala:453, took 0.012538 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 267 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 453(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 453 (MapPartitionsRDD[435] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=387683, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_359 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8450) called with curMem=407979, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_359_piece0 stored as bytes in memory (estimated size 8.3 KB, free 730.2 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_359_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.5 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 359 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 453 (MapPartitionsRDD[435] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 453.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 453.0 (TID 810, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 453.0 (TID 811, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 453.0 (TID 812, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 453.0 (TID 813, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 453.0 (TID 811) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 453.0 (TID 812) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 453.0 (TID 813) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 453.0 (TID 810) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 453.0 (TID 811). 1002 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 453.0 (TID 810). 1002 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 453.0 (TID 811) in 4 ms on localhost (1/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 453.0 (TID 812). 1358 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 453.0 (TID 813). 1358 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 453.0 (TID 810) in 7 ms on localhost (2/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 453.0 (TID 812) in 6 ms on localhost (3/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 453.0 (TID 813) in 6 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 453 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 453.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 267 finished: reduce at LDAOptimizer.scala:456, took 0.013270 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 268 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 454(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 454 (PartitionwiseSampledRDD[436] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=416429, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_360 stored as values in memory (estimated size 18.4 KB, free 730.2 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(7663) called with curMem=435245, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_360_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_360_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 360 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 454 (PartitionwiseSampledRDD[436] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 454.0 with 1 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 454.0 (TID 814, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 454.0 (TID 814) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 454.0 (TID 814). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 454.0 (TID 814) in 2 ms on localhost (1/1) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 454 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 454.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 268 finished: isEmpty at LDAOptimizer.scala:418, took 0.007932 s -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=442908, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_361 stored as values in memory (estimated size 176.0 B, free 730.2 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=443084, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_361_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.2 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_361_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 361 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 269 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 455(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 455 (MapPartitionsRDD[438] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=443364, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_362 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=463620, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_362_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_362_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 362 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 455 (MapPartitionsRDD[438] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 455.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 455.0 (TID 815, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 455.0 (TID 816, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 455.0 (TID 817, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 455.0 (TID 818, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 455.0 (TID 816) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 455.0 (TID 818) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 455.0 (TID 817) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 455.0 (TID 815) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 455.0 (TID 817). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 455.0 (TID 817) in 4 ms on localhost (1/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 455.0 (TID 815). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 455.0 (TID 815) in 6 ms on localhost (2/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 455.0 (TID 816). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 455.0 (TID 818). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 455.0 (TID 816) in 6 ms on localhost (3/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 455.0 (TID 818) in 7 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 455 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 455.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 269 finished: reduce at LDAOptimizer.scala:453, took 0.013570 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 270 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 456(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 456 (MapPartitionsRDD[439] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=472052, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_363 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8445) called with curMem=492348, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_363_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_363_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 363 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 456 (MapPartitionsRDD[439] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 456.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 456.0 (TID 819, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 456.0 (TID 820, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 456.0 (TID 821, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 456.0 (TID 822, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 456.0 (TID 820) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 456.0 (TID 821) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 456.0 (TID 822) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 456.0 (TID 819) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 456.0 (TID 821). 1002 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 456.0 (TID 819). 1314 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 456.0 (TID 821) in 6 ms on localhost (1/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 456.0 (TID 822). 1446 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 456.0 (TID 819) in 9 ms on localhost (2/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 456.0 (TID 822) in 9 ms on localhost (3/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 456.0 (TID 820). 1358 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 456.0 (TID 820) in 12 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 456 (reduce at LDAOptimizer.scala:456) finished in 0.012 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 456.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 270 finished: reduce at LDAOptimizer.scala:456, took 0.018997 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 271 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 457(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 457 (PartitionwiseSampledRDD[440] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=500793, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_364 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=519609, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_364_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_364_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 364 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 457 (PartitionwiseSampledRDD[440] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 457.0 with 1 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 457.0 (TID 823, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 457.0 (TID 823) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 457.0 (TID 823). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 457.0 (TID 823) in 3 ms on localhost (1/1) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 457 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 457.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 271 finished: isEmpty at LDAOptimizer.scala:418, took 0.008570 s -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=527265, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_365 stored as values in memory (estimated size 176.0 B, free 730.1 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=527441, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_365_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.1 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_365_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 365 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 272 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 458(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 458 (MapPartitionsRDD[442] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=527721, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_366 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8428) called with curMem=547977, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_366_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_366_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 366 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 458 (MapPartitionsRDD[442] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 458.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 458.0 (TID 824, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 458.0 (TID 825, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 458.0 (TID 826, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 458.0 (TID 827, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 458.0 (TID 824) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 458.0 (TID 827) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 458.0 (TID 825) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 458.0 (TID 826) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 458.0 (TID 824). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 458.0 (TID 824) in 7 ms on localhost (1/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 458.0 (TID 826). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 458.0 (TID 827). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 458.0 (TID 825). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 458.0 (TID 826) in 7 ms on localhost (2/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 458.0 (TID 825) in 8 ms on localhost (3/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 458.0 (TID 827) in 7 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 458 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 458.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 272 finished: reduce at LDAOptimizer.scala:453, took 0.014659 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 273 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 459(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 459 (MapPartitionsRDD[443] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=556405, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_367 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8441) called with curMem=576701, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_367_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_367_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 367 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 459 (MapPartitionsRDD[443] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 459.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 459.0 (TID 828, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 459.0 (TID 829, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 459.0 (TID 830, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 459.0 (TID 831, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 459.0 (TID 828) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 459.0 (TID 831) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 459.0 (TID 830) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 459.0 (TID 829) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 459.0 (TID 830). 1002 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 459.0 (TID 829). 1314 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 459.0 (TID 828). 1314 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 459.0 (TID 830) in 4 ms on localhost (1/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 459.0 (TID 831). 1314 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 459.0 (TID 829) in 5 ms on localhost (2/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 459.0 (TID 828) in 6 ms on localhost (3/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 459.0 (TID 831) in 5 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 459 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 459.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 273 finished: reduce at LDAOptimizer.scala:456, took 0.013279 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 274 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 460(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 460 (PartitionwiseSampledRDD[444] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=585142, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_368 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(7659) called with curMem=603958, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_368_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_368_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 368 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 460 (PartitionwiseSampledRDD[444] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 460.0 with 1 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 460.0 (TID 832, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 460.0 (TID 832) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 460.0 (TID 832). 911 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 460.0 (TID 832) in 2 ms on localhost (1/1) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 460 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 460.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 274 finished: isEmpty at LDAOptimizer.scala:418, took 0.009237 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 275 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 461(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 461 (PartitionwiseSampledRDD[444] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=611617, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_369 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(7659) called with curMem=630433, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_369_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_369_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 369 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 461 (PartitionwiseSampledRDD[444] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 461.0 with 3 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 461.0 (TID 833, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 461.0 (TID 834, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 461.0 (TID 835, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 461.0 (TID 833) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 461.0 (TID 834) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 461.0 (TID 835) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 461.0 (TID 833). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 461.0 (TID 834). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 461.0 (TID 835). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 461.0 (TID 833) in 4 ms on localhost (1/3) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 461.0 (TID 834) in 4 ms on localhost (2/3) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 461.0 (TID 835) in 4 ms on localhost (3/3) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 461 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 461.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 275 finished: isEmpty at LDAOptimizer.scala:418, took 0.009841 s -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=638092, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_370 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=638268, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_370_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_370_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 370 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 276 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 462(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 462 (MapPartitionsRDD[446] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=638548, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_371 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8431) called with curMem=658804, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_371_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_371_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 371 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 462 (MapPartitionsRDD[446] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 462.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 462.0 (TID 836, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 462.0 (TID 837, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 462.0 (TID 838, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 462.0 (TID 839, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 462.0 (TID 837) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 462.0 (TID 839) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 462.0 (TID 836) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 462.0 (TID 838) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 462.0 (TID 836). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 462.0 (TID 836) in 4 ms on localhost (1/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 462.0 (TID 838). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 462.0 (TID 837). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 462.0 (TID 838) in 5 ms on localhost (2/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 462.0 (TID 837) in 5 ms on localhost (3/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 462.0 (TID 839). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 462.0 (TID 839) in 6 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 462 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 462.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 276 finished: reduce at LDAOptimizer.scala:453, took 0.014780 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 277 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 463(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 463 (MapPartitionsRDD[447] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=667235, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_372 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8444) called with curMem=687531, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_372_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_372_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 372 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 463 (MapPartitionsRDD[447] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 463.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 463.0 (TID 840, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 463.0 (TID 841, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 463.0 (TID 842, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 463.0 (TID 843, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 463.0 (TID 840) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 463.0 (TID 843) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 463.0 (TID 842) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 463.0 (TID 841) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 463.0 (TID 840). 1002 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 463.0 (TID 840) in 5 ms on localhost (1/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 463.0 (TID 842). 1358 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 463.0 (TID 843). 1402 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 463.0 (TID 841). 1358 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 463.0 (TID 842) in 5 ms on localhost (2/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 463.0 (TID 843) in 5 ms on localhost (3/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 463 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 463.0 (TID 841) in 5 ms on localhost (4/4) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 463.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 277 finished: reduce at LDAOptimizer.scala:456, took 0.011474 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 278 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 464(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 464 (PartitionwiseSampledRDD[448] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=695975, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_373 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=714791, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_373_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_373_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 373 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 464 (PartitionwiseSampledRDD[448] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 464.0 with 1 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 464.0 (TID 844, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 464.0 (TID 844) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 464.0 (TID 844). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 464.0 (TID 844) in 2 ms on localhost (1/1) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 464 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 464.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 278 finished: isEmpty at LDAOptimizer.scala:418, took 0.009843 s -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=722447, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_374 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=722623, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_374_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_374_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 374 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 279 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 465(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 465 (MapPartitionsRDD[450] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=722903, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_375 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=743159, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_375_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_375_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 375 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 465 (MapPartitionsRDD[450] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 465.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 465.0 (TID 845, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 465.0 (TID 846, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 465.0 (TID 847, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 465.0 (TID 848, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 465.0 (TID 845) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 465.0 (TID 848) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 465.0 (TID 847) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 465.0 (TID 846) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 465.0 (TID 845). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 465.0 (TID 848). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 465.0 (TID 846). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 465.0 (TID 847). 1265 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 465.0 (TID 845) in 5 ms on localhost (1/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 465.0 (TID 847) in 6 ms on localhost (2/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 465.0 (TID 848) in 6 ms on localhost (3/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 465.0 (TID 846) in 7 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 465 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 465.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 279 finished: reduce at LDAOptimizer.scala:453, took 0.013636 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 280 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 466(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 466 (MapPartitionsRDD[451] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=751592, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_376 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(8446) called with curMem=771888, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_376_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_376_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 376 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 466 (MapPartitionsRDD[451] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 466.0 with 4 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 466.0 (TID 849, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 466.0 (TID 850, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 466.0 (TID 851, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 3.0 in stage 466.0 (TID 852, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 466.0 (TID 849) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 466.0 (TID 851) -2016-06-13 19:45:20 INFO Executor:59 - Running task 3.0 in stage 466.0 (TID 852) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 466.0 (TID 850) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 466.0 (TID 849). 1314 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 466.0 (TID 851). 1314 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 3.0 in stage 466.0 (TID 852). 1314 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 466.0 (TID 849) in 6 ms on localhost (1/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 466.0 (TID 851) in 6 ms on localhost (2/4) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 466.0 (TID 850). 1314 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 3.0 in stage 466.0 (TID 852) in 6 ms on localhost (3/4) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 466.0 (TID 850) in 6 ms on localhost (4/4) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 466 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 466.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 280 finished: reduce at LDAOptimizer.scala:456, took 0.012758 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 281 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 467(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 467 (PartitionwiseSampledRDD[452] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=780334, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_377 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(7659) called with curMem=799150, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_377_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_377_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 377 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 467 (PartitionwiseSampledRDD[452] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 467.0 with 1 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 467.0 (TID 853, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 467.0 (TID 853) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 467.0 (TID 853). 911 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 467.0 (TID 853) in 3 ms on localhost (1/1) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 467 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 467.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 281 finished: isEmpty at LDAOptimizer.scala:418, took 0.008677 s -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 282 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 468(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 468 (PartitionwiseSampledRDD[452] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=806809, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_378 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(7659) called with curMem=825625, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_378_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_378_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 378 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 468 (PartitionwiseSampledRDD[452] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Adding task set 468.0 with 3 tasks -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 468.0 (TID 854, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 468.0 (TID 855, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 468.0 (TID 856, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:20 INFO Executor:59 - Running task 2.0 in stage 468.0 (TID 856) -2016-06-13 19:45:20 INFO Executor:59 - Running task 0.0 in stage 468.0 (TID 854) -2016-06-13 19:45:20 INFO Executor:59 - Running task 1.0 in stage 468.0 (TID 855) -2016-06-13 19:45:20 INFO Executor:59 - Finished task 0.0 in stage 468.0 (TID 854). 1226 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 1.0 in stage 468.0 (TID 855). 911 bytes result sent to driver -2016-06-13 19:45:20 INFO Executor:59 - Finished task 2.0 in stage 468.0 (TID 856). 911 bytes result sent to driver -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 468.0 (TID 854) in 3 ms on localhost (1/3) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 468.0 (TID 855) in 3 ms on localhost (2/3) -2016-06-13 19:45:20 INFO TaskSetManager:59 - Finished task 2.0 in stage 468.0 (TID 856) in 4 ms on localhost (3/3) -2016-06-13 19:45:20 INFO DAGScheduler:59 - ResultStage 468 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 468.0, whose tasks have all completed, from pool -2016-06-13 19:45:20 INFO DAGScheduler:59 - Job 282 finished: isEmpty at LDAOptimizer.scala:418, took 0.012049 s -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=833284, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_379 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=833460, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_379_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.8 MB) -2016-06-13 19:45:20 INFO BlockManagerInfo:59 - Added broadcast_379_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:20 INFO SparkContext:59 - Created broadcast 379 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:20 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:20 INFO DAGScheduler:59 - Got job 283 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:20 INFO DAGScheduler:59 - Final stage: ResultStage 469(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:20 INFO DAGScheduler:59 - Submitting ResultStage 469 (MapPartitionsRDD[454] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:20 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=833740, maxMem=766075207 -2016-06-13 19:45:20 INFO MemoryStore:59 - Block broadcast_380 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8424) called with curMem=853996, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_380_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_380_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 380 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 469 (MapPartitionsRDD[454] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 469.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 469.0 (TID 857, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 469.0 (TID 858, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 469.0 (TID 859, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 469.0 (TID 860, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 469.0 (TID 860) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 469.0 (TID 858) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 469.0 (TID 857) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 469.0 (TID 857). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 469.0 (TID 858). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 469.0 (TID 860). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 469.0 (TID 859) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 469.0 (TID 859). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 469.0 (TID 857) in 16 ms on localhost (1/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 469.0 (TID 858) in 15 ms on localhost (2/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 469.0 (TID 860) in 15 ms on localhost (3/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 469.0 (TID 859) in 16 ms on localhost (4/4) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 469.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 469 (reduce at LDAOptimizer.scala:453) finished in 0.019 s -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 283 finished: reduce at LDAOptimizer.scala:453, took 0.024423 s -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 284 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 470(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 470 (MapPartitionsRDD[455] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=862420, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_381 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8436) called with curMem=882716, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_381_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_381_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 381 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 470 (MapPartitionsRDD[455] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 470.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 470.0 (TID 861, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 470.0 (TID 862, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 470.0 (TID 863, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 470.0 (TID 864, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 470.0 (TID 863) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 470.0 (TID 864) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 470.0 (TID 862) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 470.0 (TID 863). 1002 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 470.0 (TID 862). 1358 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 470.0 (TID 864). 1002 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 470.0 (TID 861) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 470.0 (TID 863) in 9 ms on localhost (1/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 470.0 (TID 862) in 10 ms on localhost (2/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 470.0 (TID 864) in 9 ms on localhost (3/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 470.0 (TID 861). 1002 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 470.0 (TID 861) in 13 ms on localhost (4/4) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 470.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 470 (reduce at LDAOptimizer.scala:456) finished in 0.013 s -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 284 finished: reduce at LDAOptimizer.scala:456, took 0.020843 s -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 285 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 471(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 471 (PartitionwiseSampledRDD[456] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=891152, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_382 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=909968, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_382_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_382_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 382 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 471 (PartitionwiseSampledRDD[456] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 471.0 with 1 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 471.0 (TID 865, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 471.0 (TID 865) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 471.0 (TID 865). 1226 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 471.0 (TID 865) in 3 ms on localhost (1/1) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 471 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 471.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 285 finished: isEmpty at LDAOptimizer.scala:418, took 0.012071 s -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=917629, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_383 stored as values in memory (estimated size 176.0 B, free 729.7 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=917805, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_383_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.7 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_383_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 383 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 286 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 472(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 472 (MapPartitionsRDD[458] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=918085, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_384 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8434) called with curMem=938341, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_384_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_384_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 384 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 472 (MapPartitionsRDD[458] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 472.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 472.0 (TID 866, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 472.0 (TID 867, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 472.0 (TID 868, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 472.0 (TID 869, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 472.0 (TID 867) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 472.0 (TID 869) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 472.0 (TID 868) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 472.0 (TID 866) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 472.0 (TID 868). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 472.0 (TID 869). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 472.0 (TID 867). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 472.0 (TID 868) in 5 ms on localhost (1/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 472.0 (TID 866). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 472.0 (TID 869) in 7 ms on localhost (2/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 472.0 (TID 867) in 8 ms on localhost (3/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 472.0 (TID 866) in 8 ms on localhost (4/4) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 472.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 472 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 286 finished: reduce at LDAOptimizer.scala:453, took 0.014550 s -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 287 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 473(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 473 (MapPartitionsRDD[459] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=946775, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_385 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8447) called with curMem=967071, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_385_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_385_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 385 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 473 (MapPartitionsRDD[459] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 473.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 473.0 (TID 870, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 473.0 (TID 871, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 473.0 (TID 872, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 473.0 (TID 873, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 473.0 (TID 870) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 473.0 (TID 872) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 473.0 (TID 871) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 473.0 (TID 873) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 473.0 (TID 872). 1002 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 473.0 (TID 870). 1314 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 473.0 (TID 872) in 5 ms on localhost (1/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 473.0 (TID 870) in 7 ms on localhost (2/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 473.0 (TID 871). 1358 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 473.0 (TID 873). 1314 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 473.0 (TID 871) in 9 ms on localhost (3/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 473.0 (TID 873) in 9 ms on localhost (4/4) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 473 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 473.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 287 finished: reduce at LDAOptimizer.scala:456, took 0.016299 s -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 288 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 474(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 474 (PartitionwiseSampledRDD[460] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=975518, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_386 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(7664) called with curMem=994334, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_386_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_386_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 386 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 474 (PartitionwiseSampledRDD[460] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 474.0 with 1 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 474.0 (TID 874, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 474.0 (TID 874) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 474.0 (TID 874). 1226 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 474.0 (TID 874) in 3 ms on localhost (1/1) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 474 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 474.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 288 finished: isEmpty at LDAOptimizer.scala:418, took 0.009061 s -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1001998, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_387 stored as values in memory (estimated size 176.0 B, free 729.6 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1002174, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_387_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.6 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_387_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 387 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 289 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 475(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 475 (MapPartitionsRDD[462] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1002454, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_388 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=1022710, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_388_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_388_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 388 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 475 (MapPartitionsRDD[462] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 475.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 475.0 (TID 875, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 475.0 (TID 876, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 475.0 (TID 877, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 475.0 (TID 878, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 475.0 (TID 876) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 475.0 (TID 877) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 475.0 (TID 875) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 475.0 (TID 878) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 475.0 (TID 878). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 475.0 (TID 878) in 4 ms on localhost (1/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 475.0 (TID 876). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 475.0 (TID 875). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 475.0 (TID 877). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 475.0 (TID 876) in 6 ms on localhost (2/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 475.0 (TID 877) in 6 ms on localhost (3/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 475.0 (TID 875) in 8 ms on localhost (4/4) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 475 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 475.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 289 finished: reduce at LDAOptimizer.scala:453, took 0.015908 s -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 290 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 476(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 476 (MapPartitionsRDD[463] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1031142, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_389 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8444) called with curMem=1051438, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_389_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_389_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 389 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 476 (MapPartitionsRDD[463] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 476.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 476.0 (TID 879, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 476.0 (TID 880, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 476.0 (TID 881, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 476.0 (TID 882, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 476.0 (TID 880) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 476.0 (TID 882) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 476.0 (TID 881) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 476.0 (TID 879) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 476.0 (TID 882). 1002 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 476.0 (TID 881). 1002 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 476.0 (TID 882) in 4 ms on localhost (1/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 476.0 (TID 879). 1314 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 476.0 (TID 881) in 5 ms on localhost (2/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 476.0 (TID 879) in 5 ms on localhost (3/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 476.0 (TID 880). 1358 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 476.0 (TID 880) in 7 ms on localhost (4/4) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 476 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 290 finished: reduce at LDAOptimizer.scala:456, took 0.014297 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 476.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 291 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 477(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 477 (PartitionwiseSampledRDD[464] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1059882, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_390 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=1078698, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_390_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_390_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 390 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 477 (PartitionwiseSampledRDD[464] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 477.0 with 1 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 477.0 (TID 883, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 477.0 (TID 883) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 477.0 (TID 883). 1226 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 477.0 (TID 883) in 4 ms on localhost (1/1) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 477.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 477 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 291 finished: isEmpty at LDAOptimizer.scala:418, took 0.018140 s -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1086355, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_391 stored as values in memory (estimated size 176.0 B, free 729.6 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1086531, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_391_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.5 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_391_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 391 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 292 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 478(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 478 (MapPartitionsRDD[466] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1086811, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_392 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8436) called with curMem=1107067, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_392_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_392_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 392 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 478 (MapPartitionsRDD[466] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 478.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 478.0 (TID 884, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 478.0 (TID 885, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 478.0 (TID 886, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 478.0 (TID 887, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 478.0 (TID 884) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 478.0 (TID 885) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 478.0 (TID 887) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 478.0 (TID 886) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 478.0 (TID 887). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 478.0 (TID 884). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 478.0 (TID 885). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 478.0 (TID 886). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 478.0 (TID 887) in 9 ms on localhost (1/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 478.0 (TID 885) in 10 ms on localhost (2/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 478.0 (TID 884) in 11 ms on localhost (3/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 478.0 (TID 886) in 10 ms on localhost (4/4) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 478 (reduce at LDAOptimizer.scala:453) finished in 0.011 s -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 292 finished: reduce at LDAOptimizer.scala:453, took 0.020189 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 478.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 293 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 479(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 479 (MapPartitionsRDD[467] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1115503, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_393 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8449) called with curMem=1135799, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_393_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.5 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_393_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 393 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 479 (MapPartitionsRDD[467] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 479.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 479.0 (TID 888, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 479.0 (TID 889, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 479.0 (TID 890, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 479.0 (TID 891, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 479.0 (TID 889) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 479.0 (TID 888) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 479.0 (TID 890) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 479.0 (TID 891) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 479.0 (TID 888). 1314 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 479.0 (TID 889). 1358 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 479.0 (TID 888) in 7 ms on localhost (1/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 479.0 (TID 889) in 7 ms on localhost (2/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 479.0 (TID 891). 1002 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 479.0 (TID 891) in 6 ms on localhost (3/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 479.0 (TID 890). 1358 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 479.0 (TID 890) in 8 ms on localhost (4/4) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 479 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 479.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 293 finished: reduce at LDAOptimizer.scala:456, took 0.015467 s -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 294 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 480(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 480 (PartitionwiseSampledRDD[468] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1144248, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_394 stored as values in memory (estimated size 18.4 KB, free 729.5 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(7658) called with curMem=1163064, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_394_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.5 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_394_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 394 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 480 (PartitionwiseSampledRDD[468] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 480.0 with 1 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 480.0 (TID 892, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 480.0 (TID 892) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 480.0 (TID 892). 911 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 480.0 (TID 892) in 2 ms on localhost (1/1) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 480 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 480.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 294 finished: isEmpty at LDAOptimizer.scala:418, took 0.007454 s -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 295 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 481(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 481 (PartitionwiseSampledRDD[468] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1170722, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_395 stored as values in memory (estimated size 18.4 KB, free 729.5 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(7658) called with curMem=1189538, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_395_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.4 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_395_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 395 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 481 (PartitionwiseSampledRDD[468] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 481.0 with 3 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 481.0 (TID 893, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 481.0 (TID 894, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 481.0 (TID 895, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 481.0 (TID 894) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 481.0 (TID 895) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 481.0 (TID 893) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 481.0 (TID 894). 1226 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 481.0 (TID 893). 1226 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 481.0 (TID 894) in 3 ms on localhost (1/3) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 481.0 (TID 895). 1226 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 481.0 (TID 893) in 3 ms on localhost (2/3) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 481.0 (TID 895) in 4 ms on localhost (3/3) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 481 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 481.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 295 finished: isEmpty at LDAOptimizer.scala:418, took 0.010035 s -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1197196, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_396 stored as values in memory (estimated size 176.0 B, free 729.4 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1197372, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_396_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.4 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_396_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 396 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 296 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 482(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 482 (MapPartitionsRDD[470] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1197652, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_397 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=1217908, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_397_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_397_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 397 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 482 (MapPartitionsRDD[470] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 482.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 482.0 (TID 896, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 482.0 (TID 897, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 482.0 (TID 898, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 482.0 (TID 899, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 482.0 (TID 897) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 482.0 (TID 899) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 482.0 (TID 896) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 482.0 (TID 898) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 482.0 (TID 896). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 482.0 (TID 899). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 482.0 (TID 898). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 482.0 (TID 897). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 482.0 (TID 896) in 6 ms on localhost (1/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 482.0 (TID 899) in 4 ms on localhost (2/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 482.0 (TID 898) in 5 ms on localhost (3/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 482.0 (TID 897) in 6 ms on localhost (4/4) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 482 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 482.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 296 finished: reduce at LDAOptimizer.scala:453, took 0.012370 s -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 297 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 483(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 483 (MapPartitionsRDD[471] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1226340, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_398 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8445) called with curMem=1246636, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_398_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_398_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 398 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 483 (MapPartitionsRDD[471] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 483.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 483.0 (TID 900, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 483.0 (TID 901, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 483.0 (TID 902, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 483.0 (TID 903, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 483.0 (TID 900) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 483.0 (TID 903) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 483.0 (TID 901) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 483.0 (TID 902) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 483.0 (TID 900). 1002 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 483.0 (TID 900) in 5 ms on localhost (1/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 483.0 (TID 902). 1314 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 483.0 (TID 903). 1358 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 483.0 (TID 902) in 6 ms on localhost (2/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 483.0 (TID 901). 1402 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 483.0 (TID 903) in 7 ms on localhost (3/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 483.0 (TID 901) in 9 ms on localhost (4/4) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 483 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 483.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 297 finished: reduce at LDAOptimizer.scala:456, took 0.016195 s -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 298 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 484(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 484 (PartitionwiseSampledRDD[472] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1255081, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_399 stored as values in memory (estimated size 18.4 KB, free 729.4 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=1273897, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_399_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.4 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_399_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 399 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 484 (PartitionwiseSampledRDD[472] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 484.0 with 1 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 484.0 (TID 904, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 484.0 (TID 904) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 484.0 (TID 904). 1226 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 484.0 (TID 904) in 2 ms on localhost (1/1) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 484 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 484.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 298 finished: isEmpty at LDAOptimizer.scala:418, took 0.008110 s -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1281554, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_400 stored as values in memory (estimated size 176.0 B, free 729.4 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1281730, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_400_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.4 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_400_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 400 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 299 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 485(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 485 (MapPartitionsRDD[474] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1282010, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_401 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8431) called with curMem=1302266, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_401_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.3 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_401_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 401 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 485 (MapPartitionsRDD[474] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 485.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 485.0 (TID 905, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 485.0 (TID 906, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 485.0 (TID 907, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 485.0 (TID 908, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 485.0 (TID 905) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 485.0 (TID 907) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 485.0 (TID 908) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 485.0 (TID 906) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 485.0 (TID 907). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 485.0 (TID 906). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 485.0 (TID 908). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 485.0 (TID 907) in 6 ms on localhost (1/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 485.0 (TID 906) in 6 ms on localhost (2/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 485.0 (TID 905). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 485.0 (TID 908) in 6 ms on localhost (3/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 485.0 (TID 905) in 7 ms on localhost (4/4) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 485 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 485.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 299 finished: reduce at LDAOptimizer.scala:453, took 0.016053 s -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 300 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 486(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 486 (MapPartitionsRDD[475] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1310697, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_402 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8445) called with curMem=1330993, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_402_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.3 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_402_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 402 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 486 (MapPartitionsRDD[475] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 486.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 486.0 (TID 909, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 486.0 (TID 910, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 486.0 (TID 911, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 486.0 (TID 912, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 486.0 (TID 909) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 486.0 (TID 912) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 486.0 (TID 910) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 486.0 (TID 911) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 486.0 (TID 910). 1314 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 486.0 (TID 912). 1314 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 486.0 (TID 909). 1358 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 486.0 (TID 910) in 6 ms on localhost (1/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 486.0 (TID 911). 1358 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 486.0 (TID 912) in 6 ms on localhost (2/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 486.0 (TID 909) in 8 ms on localhost (3/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 486.0 (TID 911) in 7 ms on localhost (4/4) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 486 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 486.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 300 finished: reduce at LDAOptimizer.scala:456, took 0.016166 s -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 301 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 487(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 487 (PartitionwiseSampledRDD[476] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1339438, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_403 stored as values in memory (estimated size 18.4 KB, free 729.3 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=1358254, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_403_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.3 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_403_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 403 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 487 (PartitionwiseSampledRDD[476] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 487.0 with 1 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 487.0 (TID 913, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 487.0 (TID 913) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 487.0 (TID 913). 1226 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 487.0 (TID 913) in 4 ms on localhost (1/1) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 487 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 487.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 301 finished: isEmpty at LDAOptimizer.scala:418, took 0.010004 s -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1365915, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_404 stored as values in memory (estimated size 176.0 B, free 729.3 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1366091, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_404_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.3 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_404_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 404 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 302 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 488(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 488 (MapPartitionsRDD[478] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1366371, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_405 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8429) called with curMem=1386627, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_405_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.3 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_405_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 405 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 488 (MapPartitionsRDD[478] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 488.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 488.0 (TID 914, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 488.0 (TID 915, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 488.0 (TID 916, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 488.0 (TID 917, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 488.0 (TID 915) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 488.0 (TID 917) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 488.0 (TID 916) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 488.0 (TID 914) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 488.0 (TID 917). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 488.0 (TID 916). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 488.0 (TID 917) in 7 ms on localhost (1/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 488.0 (TID 914). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 488.0 (TID 916) in 7 ms on localhost (2/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 488.0 (TID 914) in 9 ms on localhost (3/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 488.0 (TID 915). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 488.0 (TID 915) in 9 ms on localhost (4/4) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 488.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 488 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 302 finished: reduce at LDAOptimizer.scala:453, took 0.020408 s -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 303 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 489(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 489 (MapPartitionsRDD[479] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1395056, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_406 stored as values in memory (estimated size 19.8 KB, free 729.2 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8442) called with curMem=1415352, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_406_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.2 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_406_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 406 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 489 (MapPartitionsRDD[479] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 489.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 489.0 (TID 918, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 489.0 (TID 919, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 489.0 (TID 920, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 489.0 (TID 921, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 489.0 (TID 918) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 489.0 (TID 919) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 489.0 (TID 921) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 489.0 (TID 920) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 489.0 (TID 921). 1314 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 489.0 (TID 919). 1358 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 489.0 (TID 918). 1314 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 489.0 (TID 921) in 8 ms on localhost (1/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 489.0 (TID 920). 1314 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 489.0 (TID 918) in 10 ms on localhost (2/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 489.0 (TID 919) in 9 ms on localhost (3/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 489.0 (TID 920) in 9 ms on localhost (4/4) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 489 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 489.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 303 finished: reduce at LDAOptimizer.scala:456, took 0.018359 s -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 304 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 490(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 490 (PartitionwiseSampledRDD[480] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1423794, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_407 stored as values in memory (estimated size 18.4 KB, free 729.2 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 282 -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=1442610, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_407_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.2 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_407_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 407 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 490 (PartitionwiseSampledRDD[480] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 490.0 with 1 tasks -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_406_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 490.0 (TID 922, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 323 -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 490.0 (TID 922) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 490.0 (TID 922). 1226 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 490.0 (TID 922) in 4 ms on localhost (1/1) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 490.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 490 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 304 finished: isEmpty at LDAOptimizer.scala:418, took 0.029315 s -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_405_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1392844, maxMem=766075207 -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 322 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_408 stored as values in memory (estimated size 176.0 B, free 729.3 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1392564, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_408_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.3 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_404_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_408_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 408 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_403_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 321 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_402_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 305 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 320 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 491(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 491 (MapPartitionsRDD[482] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1308939, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_409 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_401_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 319 -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=1329195, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_409_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.3 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_409_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 409 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_400_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 491 (MapPartitionsRDD[482] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 491.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 491.0 (TID 923, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 491.0 (TID 924, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 491.0 (TID 925, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 491.0 (TID 926, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 491.0 (TID 925) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 491.0 (TID 924) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 491.0 (TID 923) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 491.0 (TID 926) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_399_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 318 -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 491.0 (TID 924). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 491.0 (TID 926). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 491.0 (TID 925). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 491.0 (TID 924) in 6 ms on localhost (1/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 491.0 (TID 926) in 5 ms on localhost (2/4) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_398_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 491.0 (TID 923). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 317 -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 491.0 (TID 923) in 8 ms on localhost (3/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 491.0 (TID 925) in 8 ms on localhost (4/4) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 491.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 491 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 305 finished: reduce at LDAOptimizer.scala:453, took 0.015597 s -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_397_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 316 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_396_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 306 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 492(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 492 (MapPartitionsRDD[483] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_395_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1226340, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_410 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 315 -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8446) called with curMem=1238978, maxMem=766075207 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_394_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_410_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 314 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_410_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 410 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 492 (MapPartitionsRDD[483] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 492.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 492.0 (TID 927, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 492.0 (TID 928, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 492.0 (TID 929, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 492.0 (TID 930, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_393_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 492.0 (TID 927) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 492.0 (TID 929) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 313 -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 492.0 (TID 928) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 492.0 (TID 930) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_392_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 492.0 (TID 928). 1314 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 492.0 (TID 929). 1314 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 492.0 (TID 927). 1402 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 492.0 (TID 928) in 7 ms on localhost (1/4) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 312 -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 492.0 (TID 930). 1358 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 492.0 (TID 929) in 7 ms on localhost (2/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 492.0 (TID 927) in 8 ms on localhost (3/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 492.0 (TID 930) in 7 ms on localhost (4/4) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 492 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 492.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 306 finished: reduce at LDAOptimizer.scala:456, took 0.014600 s -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_391_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_390_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 311 -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 307 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 493(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_389_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 493 (PartitionwiseSampledRDD[484] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 310 -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1095246, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_411 stored as values in memory (estimated size 18.4 KB, free 729.5 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_388_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(7658) called with curMem=1105630, maxMem=766075207 -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 309 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_411_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.5 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_411_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 411 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 493 (PartitionwiseSampledRDD[484] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 493.0 with 1 tasks -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_387_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 493.0 (TID 931, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 493.0 (TID 931) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_386_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 493.0 (TID 931). 1226 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 493.0 (TID 931) in 2 ms on localhost (1/1) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 493 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 493.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 308 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 307 finished: isEmpty at LDAOptimizer.scala:418, took 0.008742 s -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1077905, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_412 stored as values in memory (estimated size 176.0 B, free 729.6 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_385_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 307 -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1057785, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_412_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.6 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_412_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_384_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 412 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 306 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_383_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_382_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 305 -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 308 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 494(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 494 (MapPartitionsRDD[486] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_381_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 304 -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=973710, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_413 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8434) called with curMem=965286, maxMem=766075207 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_380_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_413_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 303 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_413_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 413 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 494 (MapPartitionsRDD[486] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 494.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 494.0 (TID 932, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 494.0 (TID 933, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_379_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 494.0 (TID 934, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 494.0 (TID 935, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 494.0 (TID 932) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 494.0 (TID 934) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 494.0 (TID 935) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 494.0 (TID 934). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 494.0 (TID 932). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 494.0 (TID 933) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_378_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 494.0 (TID 934) in 5 ms on localhost (1/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 494.0 (TID 932) in 6 ms on localhost (2/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 494.0 (TID 935). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 302 -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 494.0 (TID 933). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 494.0 (TID 935) in 9 ms on localhost (3/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 494.0 (TID 933) in 10 ms on localhost (4/4) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 494.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 494 (reduce at LDAOptimizer.scala:453) finished in 0.011 s -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 308 finished: reduce at LDAOptimizer.scala:453, took 0.018226 s -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_377_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 301 -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 309 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 495(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 495 (MapPartitionsRDD[487] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_376_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 300 -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=891572, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_414 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8446) called with curMem=883179, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_414_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_375_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 299 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_414_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 414 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 495 (MapPartitionsRDD[487] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 495.0 with 4 tasks -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_374_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 495.0 (TID 936, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 495.0 (TID 937, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 495.0 (TID 938, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 495.0 (TID 939, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 495.0 (TID 936) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 495.0 (TID 938) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 495.0 (TID 939) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_373_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 495.0 (TID 938). 1002 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 495.0 (TID 936). 1314 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 495.0 (TID 938) in 5 ms on localhost (1/4) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 298 -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 495.0 (TID 937) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 495.0 (TID 936) in 8 ms on localhost (2/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 495.0 (TID 939). 1314 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 495.0 (TID 939) in 7 ms on localhost (3/4) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_372_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 297 -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 495.0 (TID 937). 1402 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 495.0 (TID 937) in 11 ms on localhost (4/4) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 495.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 495 (reduce at LDAOptimizer.scala:456) finished in 0.013 s -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 309 finished: reduce at LDAOptimizer.scala:456, took 0.020605 s -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_371_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 296 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_370_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 310 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 496(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_369_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 496 (PartitionwiseSampledRDD[488] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 295 -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=780339, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_415 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(7658) called with curMem=799155, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_415_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_415_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 415 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 496 (PartitionwiseSampledRDD[488] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 496.0 with 1 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 496.0 (TID 940, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 496.0 (TID 940) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 496.0 (TID 940). 1226 bytes result sent to driver -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_368_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 294 -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 496 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 496.0 (TID 940) in 3 ms on localhost (1/1) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 496.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 310 finished: isEmpty at LDAOptimizer.scala:418, took 0.010692 s -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=771897, maxMem=766075207 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_367_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_416 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 293 -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=751777, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_416_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_416_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 416 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_366_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 292 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_365_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_364_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 311 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 291 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 497(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 497 (MapPartitionsRDD[490] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=688000, maxMem=766075207 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_363_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_417 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 290 -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8422) called with curMem=687960, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_417_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_417_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 417 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_362_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 497 (MapPartitionsRDD[490] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 497.0 with 4 tasks -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 289 -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 497.0 (TID 941, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 497.0 (TID 942, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 497.0 (TID 943, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 497.0 (TID 944, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 497.0 (TID 942) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 497.0 (TID 941) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 497.0 (TID 943) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 497.0 (TID 944) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 497.0 (TID 942). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_361_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 497.0 (TID 944). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 497.0 (TID 942) in 9 ms on localhost (1/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 497.0 (TID 943). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 497.0 (TID 944) in 11 ms on localhost (2/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 497.0 (TID 941). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 497.0 (TID 943) in 12 ms on localhost (3/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 497.0 (TID 941) in 14 ms on localhost (4/4) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 497 (reduce at LDAOptimizer.scala:453) finished in 0.014 s -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 311 finished: reduce at LDAOptimizer.scala:453, took 0.020247 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 497.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_360_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 288 -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 312 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 498(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_359_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 498 (MapPartitionsRDD[491] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 287 -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=603576, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_418 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_358_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=603616, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_418_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 286 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_418_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 418 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 498 (MapPartitionsRDD[491] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 498.0 with 4 tasks -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_357_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 498.0 (TID 945, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 498.0 (TID 946, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 498.0 (TID 947, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 498.0 (TID 948, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 498.0 (TID 945) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 498.0 (TID 947) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 498.0 (TID 946) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 498.0 (TID 948) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_356_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 498.0 (TID 947). 1002 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 498.0 (TID 945). 1402 bytes result sent to driver -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 285 -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 498.0 (TID 946). 1358 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 498.0 (TID 947) in 7 ms on localhost (1/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 498.0 (TID 945) in 8 ms on localhost (2/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 498.0 (TID 946) in 7 ms on localhost (3/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 498.0 (TID 948). 1358 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 498.0 (TID 948) in 8 ms on localhost (4/4) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 498.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 498 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 312 finished: reduce at LDAOptimizer.scala:456, took 0.016825 s -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_355_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 284 -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_354_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 313 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 499(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 499 (PartitionwiseSampledRDD[492] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 283 -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=509642, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_419 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=520022, maxMem=766075207 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_353_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_419_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_419_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 419 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 499 (PartitionwiseSampledRDD[492] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 499.0 with 1 tasks -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_352_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 499.0 (TID 949, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 499.0 (TID 949) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_351_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 499.0 (TID 949). 1226 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 499.0 (TID 949) in 3 ms on localhost (1/1) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 499 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 499.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 313 finished: isEmpty at LDAOptimizer.scala:418, took 0.009048 s -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 281 -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=480449, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_420 stored as values in memory (estimated size 176.0 B, free 730.1 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_350_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=472186, maxMem=766075207 -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 280 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_420_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.1 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_420_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 420 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_349_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 279 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_348_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_347_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 314 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 500(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 278 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 500 (MapPartitionsRDD[494] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=408415, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_421 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_346_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=408375, maxMem=766075207 -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 277 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_421_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_421_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 421 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_345_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 500 (MapPartitionsRDD[494] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 500.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 500.0 (TID 950, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 276 -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 500.0 (TID 951, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 500.0 (TID 952, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 500.0 (TID 953, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 500.0 (TID 950) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 500.0 (TID 953) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 500.0 (TID 952) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 500.0 (TID 951) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_344_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 500.0 (TID 952). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 500.0 (TID 953). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_343_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 500.0 (TID 950). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 500.0 (TID 952) in 9 ms on localhost (1/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 500.0 (TID 951). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 275 -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 500.0 (TID 950) in 10 ms on localhost (2/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 500.0 (TID 953) in 9 ms on localhost (3/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 500.0 (TID 951) in 11 ms on localhost (4/4) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 500 (reduce at LDAOptimizer.scala:453) finished in 0.012 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 500.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 314 finished: reduce at LDAOptimizer.scala:453, took 0.018462 s -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Removed broadcast_342_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:21 INFO ContextCleaner:59 - Cleaned accumulator 274 -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 315 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 501(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 501 (MapPartitionsRDD[495] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=334724, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_422 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8443) called with curMem=355020, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_422_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_422_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 422 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 501 (MapPartitionsRDD[495] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 501.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 501.0 (TID 954, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 501.0 (TID 955, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 501.0 (TID 956, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 501.0 (TID 957, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 501.0 (TID 954) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 501.0 (TID 957) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 501.0 (TID 956) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 501.0 (TID 955) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 501.0 (TID 956). 1314 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 501.0 (TID 955). 1534 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 501.0 (TID 957). 1402 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 501.0 (TID 954). 1358 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 501.0 (TID 955) in 9 ms on localhost (1/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 501.0 (TID 956) in 10 ms on localhost (2/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 501.0 (TID 957) in 10 ms on localhost (3/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 501.0 (TID 954) in 11 ms on localhost (4/4) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 501.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 501 (reduce at LDAOptimizer.scala:456) finished in 0.012 s -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 315 finished: reduce at LDAOptimizer.scala:456, took 0.018152 s -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 316 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 502(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 502 (PartitionwiseSampledRDD[496] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=363463, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_423 stored as values in memory (estimated size 18.4 KB, free 730.2 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=382279, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_423_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_423_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 423 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 502 (PartitionwiseSampledRDD[496] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 502.0 with 1 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 502.0 (TID 958, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 502.0 (TID 958) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 502.0 (TID 958). 1226 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 502.0 (TID 958) in 3 ms on localhost (1/1) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 502.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 502 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 316 finished: isEmpty at LDAOptimizer.scala:418, took 0.011286 s -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=389935, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_424 stored as values in memory (estimated size 176.0 B, free 730.2 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=390111, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_424_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.2 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_424_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 424 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 317 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 503(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 503 (MapPartitionsRDD[498] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=390391, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_425 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8418) called with curMem=410647, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_425_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_425_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 425 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 503 (MapPartitionsRDD[498] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 503.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 503.0 (TID 959, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 503.0 (TID 960, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 503.0 (TID 961, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 503.0 (TID 962, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 503.0 (TID 959) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 503.0 (TID 959). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 503.0 (TID 962) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 503.0 (TID 961) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 503.0 (TID 961). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 503.0 (TID 962). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 503.0 (TID 959) in 31 ms on localhost (1/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 503.0 (TID 961) in 27 ms on localhost (2/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 503.0 (TID 962) in 26 ms on localhost (3/4) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 503.0 (TID 960) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 503.0 (TID 960). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 503.0 (TID 960) in 35 ms on localhost (4/4) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 503.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 503 (reduce at LDAOptimizer.scala:453) finished in 0.040 s -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 317 finished: reduce at LDAOptimizer.scala:453, took 0.047420 s -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 318 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 504(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 504 (MapPartitionsRDD[499] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=419065, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_426 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8429) called with curMem=439361, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_426_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_426_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 426 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 504 (MapPartitionsRDD[499] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 504.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 504.0 (TID 963, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 504.0 (TID 964, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 504.0 (TID 965, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 504.0 (TID 966, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 504.0 (TID 965) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 504.0 (TID 965). 1358 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 504.0 (TID 965) in 7 ms on localhost (1/4) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 504.0 (TID 964) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 504.0 (TID 966) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 504.0 (TID 964). 1358 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 504.0 (TID 964) in 13 ms on localhost (2/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 504.0 (TID 966). 1402 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 504.0 (TID 963) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 504.0 (TID 966) in 18 ms on localhost (3/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 504.0 (TID 963). 1314 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 504.0 (TID 963) in 24 ms on localhost (4/4) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 504.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 504 (reduce at LDAOptimizer.scala:456) finished in 0.024 s -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 318 finished: reduce at LDAOptimizer.scala:456, took 0.033188 s -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 319 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 505(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 505 (PartitionwiseSampledRDD[500] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=447790, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_427 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=466606, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_427_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_427_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 427 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 505 (PartitionwiseSampledRDD[500] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 505.0 with 1 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 505.0 (TID 967, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 505.0 (TID 967) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 505.0 (TID 967). 911 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 505.0 (TID 967) in 4 ms on localhost (1/1) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 505 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 505.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 319 finished: isEmpty at LDAOptimizer.scala:418, took 0.016494 s -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 320 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 506(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 506 (PartitionwiseSampledRDD[500] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=474262, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_428 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=493078, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_428_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_428_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 428 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 506 (PartitionwiseSampledRDD[500] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 506.0 with 3 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 506.0 (TID 968, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 506.0 (TID 969, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 506.0 (TID 970, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 506.0 (TID 969) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 506.0 (TID 968) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 506.0 (TID 970) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 506.0 (TID 968). 1226 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 506.0 (TID 970). 911 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 506.0 (TID 969). 1226 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 506.0 (TID 970) in 3 ms on localhost (1/3) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 506.0 (TID 968) in 4 ms on localhost (2/3) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 506.0 (TID 969) in 4 ms on localhost (3/3) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 506 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 506.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 320 finished: isEmpty at LDAOptimizer.scala:418, took 0.009317 s -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=500734, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_429 stored as values in memory (estimated size 176.0 B, free 730.1 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=500910, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_429_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.1 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_429_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 429 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 321 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 507(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 507 (MapPartitionsRDD[502] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=501190, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_430 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8419) called with curMem=521446, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_430_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_430_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 430 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 507 (MapPartitionsRDD[502] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 507.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 507.0 (TID 971, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 507.0 (TID 972, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 507.0 (TID 973, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 507.0 (TID 974, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 507.0 (TID 971) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 507.0 (TID 973) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 507.0 (TID 971). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 507.0 (TID 972) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 507.0 (TID 974) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 507.0 (TID 971) in 6 ms on localhost (1/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 507.0 (TID 973). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 507.0 (TID 972). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 507.0 (TID 973) in 8 ms on localhost (2/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 507.0 (TID 974). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 507.0 (TID 972) in 9 ms on localhost (3/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 507.0 (TID 974) in 9 ms on localhost (4/4) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 507.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 507 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 321 finished: reduce at LDAOptimizer.scala:453, took 0.016369 s -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 322 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 508(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 508 (MapPartitionsRDD[503] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=529865, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_431 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8430) called with curMem=550161, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_431_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_431_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 431 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 508 (MapPartitionsRDD[503] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 508.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 508.0 (TID 975, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 508.0 (TID 976, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 508.0 (TID 977, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 508.0 (TID 978, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 508.0 (TID 975) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 508.0 (TID 978) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 508.0 (TID 976) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 508.0 (TID 977) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 508.0 (TID 975). 1002 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 508.0 (TID 978). 1002 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 508.0 (TID 976). 1314 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 508.0 (TID 975) in 5 ms on localhost (1/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 508.0 (TID 978) in 4 ms on localhost (2/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 508.0 (TID 976) in 6 ms on localhost (3/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 508.0 (TID 977). 1358 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 508.0 (TID 977) in 8 ms on localhost (4/4) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 508.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 508 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 322 finished: reduce at LDAOptimizer.scala:456, took 0.013644 s -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 323 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 509(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 509 (PartitionwiseSampledRDD[504] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=558591, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_432 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(7663) called with curMem=577407, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_432_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_432_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 432 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 509 (PartitionwiseSampledRDD[504] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 509.0 with 1 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 509.0 (TID 979, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 509.0 (TID 979) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 509.0 (TID 979). 1226 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 509.0 (TID 979) in 2 ms on localhost (1/1) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 509 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 509.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 323 finished: isEmpty at LDAOptimizer.scala:418, took 0.006994 s -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=585070, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_433 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=585246, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_433_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_433_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 433 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 324 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 510(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 510 (MapPartitionsRDD[506] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=585526, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_434 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8422) called with curMem=605782, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_434_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_434_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 434 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 510 (MapPartitionsRDD[506] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 510.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 510.0 (TID 980, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 510.0 (TID 981, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 510.0 (TID 982, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 510.0 (TID 983, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 510.0 (TID 981) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 510.0 (TID 983) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 510.0 (TID 982) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 510.0 (TID 980) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 510.0 (TID 981). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 510.0 (TID 981) in 5 ms on localhost (1/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 510.0 (TID 982). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 510.0 (TID 982) in 7 ms on localhost (2/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 510.0 (TID 980). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 510.0 (TID 980) in 10 ms on localhost (3/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 510.0 (TID 983). 1265 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 510.0 (TID 983) in 11 ms on localhost (4/4) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 510 (reduce at LDAOptimizer.scala:453) finished in 0.012 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 510.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 324 finished: reduce at LDAOptimizer.scala:453, took 0.016514 s -2016-06-13 19:45:21 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Got job 325 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:21 INFO DAGScheduler:59 - Final stage: ResultStage 511(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting ResultStage 511 (MapPartitionsRDD[507] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=614204, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_435 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:21 INFO MemoryStore:59 - ensureFreeSpace(8435) called with curMem=634500, maxMem=766075207 -2016-06-13 19:45:21 INFO MemoryStore:59 - Block broadcast_435_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:21 INFO BlockManagerInfo:59 - Added broadcast_435_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:21 INFO SparkContext:59 - Created broadcast 435 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:21 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 511 (MapPartitionsRDD[507] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Adding task set 511.0 with 4 tasks -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 511.0 (TID 984, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 511.0 (TID 985, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 2.0 in stage 511.0 (TID 986, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Starting task 3.0 in stage 511.0 (TID 987, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:21 INFO Executor:59 - Running task 0.0 in stage 511.0 (TID 984) -2016-06-13 19:45:21 INFO Executor:59 - Running task 1.0 in stage 511.0 (TID 985) -2016-06-13 19:45:21 INFO Executor:59 - Running task 3.0 in stage 511.0 (TID 987) -2016-06-13 19:45:21 INFO Executor:59 - Running task 2.0 in stage 511.0 (TID 986) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 0.0 in stage 511.0 (TID 984). 1314 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 2.0 in stage 511.0 (TID 986). 1314 bytes result sent to driver -2016-06-13 19:45:21 INFO Executor:59 - Finished task 1.0 in stage 511.0 (TID 985). 1314 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 511.0 (TID 984) in 6 ms on localhost (1/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 2.0 in stage 511.0 (TID 986) in 5 ms on localhost (2/4) -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 511.0 (TID 985) in 5 ms on localhost (3/4) -2016-06-13 19:45:21 INFO Executor:59 - Finished task 3.0 in stage 511.0 (TID 987). 1578 bytes result sent to driver -2016-06-13 19:45:21 INFO TaskSetManager:59 - Finished task 3.0 in stage 511.0 (TID 987) in 6 ms on localhost (4/4) -2016-06-13 19:45:21 INFO DAGScheduler:59 - ResultStage 511 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 511.0, whose tasks have all completed, from pool -2016-06-13 19:45:21 INFO DAGScheduler:59 - Job 325 finished: reduce at LDAOptimizer.scala:456, took 0.012722 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 326 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 512(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 512 (PartitionwiseSampledRDD[508] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=642935, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_436 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=661751, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_436_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_436_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 436 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 512 (PartitionwiseSampledRDD[508] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 512.0 with 1 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 512.0 (TID 988, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 512.0 (TID 988) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 512.0 (TID 988). 1226 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 512.0 (TID 988) in 3 ms on localhost (1/1) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 512 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 512.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 326 finished: isEmpty at LDAOptimizer.scala:418, took 0.008170 s -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=669407, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_437 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=669583, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_437_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_437_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 437 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 327 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 513(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 513 (MapPartitionsRDD[510] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=669863, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_438 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(8419) called with curMem=690119, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_438_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_438_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 438 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 513 (MapPartitionsRDD[510] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 513.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 513.0 (TID 989, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 513.0 (TID 990, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 513.0 (TID 991, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 513.0 (TID 992, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 513.0 (TID 989) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 513.0 (TID 991) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 513.0 (TID 992) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 513.0 (TID 990) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 513.0 (TID 992). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 513.0 (TID 991). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 513.0 (TID 989). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 513.0 (TID 990). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 513.0 (TID 992) in 4 ms on localhost (1/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 513.0 (TID 991) in 4 ms on localhost (2/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 513.0 (TID 989) in 6 ms on localhost (3/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 513.0 (TID 990) in 6 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 513 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 513.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 327 finished: reduce at LDAOptimizer.scala:453, took 0.011557 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 328 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 514(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 514 (MapPartitionsRDD[511] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=698538, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_439 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=718834, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_439_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_439_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 439 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 514 (MapPartitionsRDD[511] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 514.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 514.0 (TID 993, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 514.0 (TID 994, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 514.0 (TID 995, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 514.0 (TID 996, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 514.0 (TID 993) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 514.0 (TID 996) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 514.0 (TID 994) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 514.0 (TID 995) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 514.0 (TID 995). 1314 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 514.0 (TID 993). 1314 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 514.0 (TID 996). 1314 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 514.0 (TID 995) in 4 ms on localhost (1/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 514.0 (TID 994). 1402 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 514.0 (TID 993) in 6 ms on localhost (2/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 514.0 (TID 996) in 5 ms on localhost (3/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 514.0 (TID 994) in 5 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 514 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 514.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 328 finished: reduce at LDAOptimizer.scala:456, took 0.011455 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 329 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 515(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 515 (PartitionwiseSampledRDD[512] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=727266, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_440 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=746082, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_440_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_440_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 440 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 515 (PartitionwiseSampledRDD[512] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 515.0 with 1 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 515.0 (TID 997, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 515.0 (TID 997) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 515.0 (TID 997). 1226 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 515.0 (TID 997) in 3 ms on localhost (1/1) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 515 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 515.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 329 finished: isEmpty at LDAOptimizer.scala:418, took 0.007122 s -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=753743, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_441 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=753919, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_441_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_441_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 441 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 330 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 516(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 516 (MapPartitionsRDD[514] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=754199, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_442 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(8427) called with curMem=774455, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_442_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_442_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 442 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 516 (MapPartitionsRDD[514] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 516.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 516.0 (TID 998, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 516.0 (TID 999, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 516.0 (TID 1000, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 516.0 (TID 1001, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 516.0 (TID 999) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 516.0 (TID 1000) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 516.0 (TID 1001) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 516.0 (TID 998) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 516.0 (TID 1000). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 516.0 (TID 1000) in 4 ms on localhost (1/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 516.0 (TID 1001). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 516.0 (TID 999). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 516.0 (TID 1001) in 4 ms on localhost (2/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 516.0 (TID 998). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 516.0 (TID 999) in 5 ms on localhost (3/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 516.0 (TID 998) in 6 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 516 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 516.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 330 finished: reduce at LDAOptimizer.scala:453, took 0.012352 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 331 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 517(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 517 (MapPartitionsRDD[515] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=782882, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_443 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(8440) called with curMem=803178, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_443_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_443_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 443 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 517 (MapPartitionsRDD[515] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 517.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 517.0 (TID 1002, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 517.0 (TID 1003, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 517.0 (TID 1004, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 517.0 (TID 1005, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 517.0 (TID 1003) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 517.0 (TID 1002) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 517.0 (TID 1004) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 517.0 (TID 1004). 1002 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 517.0 (TID 1002). 1358 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 517.0 (TID 1003). 1358 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 517.0 (TID 1002) in 8 ms on localhost (1/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 517.0 (TID 1004) in 7 ms on localhost (2/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 517.0 (TID 1003) in 8 ms on localhost (3/4) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 517.0 (TID 1005) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 517.0 (TID 1005). 1358 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 517.0 (TID 1005) in 12 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 517 (reduce at LDAOptimizer.scala:456) finished in 0.013 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 517.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 331 finished: reduce at LDAOptimizer.scala:456, took 0.017816 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 332 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 518(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 518 (PartitionwiseSampledRDD[516] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=811618, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_444 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(7672) called with curMem=830434, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_444_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_444_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 444 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 518 (PartitionwiseSampledRDD[516] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 518.0 with 1 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 518.0 (TID 1006, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 518.0 (TID 1006) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 518.0 (TID 1006). 911 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 518.0 (TID 1006) in 3 ms on localhost (1/1) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 518.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 518 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 332 finished: isEmpty at LDAOptimizer.scala:418, took 0.009975 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 333 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 519(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 519 (PartitionwiseSampledRDD[516] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=838106, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_445 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(7672) called with curMem=856922, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_445_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_445_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 445 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 519 (PartitionwiseSampledRDD[516] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 519.0 with 3 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 519.0 (TID 1007, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 519.0 (TID 1008, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 519.0 (TID 1009, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 519.0 (TID 1007) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 519.0 (TID 1009) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 519.0 (TID 1008) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 519.0 (TID 1009). 1226 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 519.0 (TID 1008). 911 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 519.0 (TID 1007). 1226 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 519.0 (TID 1009) in 3 ms on localhost (1/3) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 519.0 (TID 1008) in 3 ms on localhost (2/3) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 519.0 (TID 1007) in 5 ms on localhost (3/3) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 519 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 519.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 333 finished: isEmpty at LDAOptimizer.scala:418, took 0.009370 s -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=864594, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_446 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=864770, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_446_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.8 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_446_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 446 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 334 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 520(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 520 (MapPartitionsRDD[518] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=865050, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_447 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(8434) called with curMem=885306, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_447_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_447_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 447 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 520 (MapPartitionsRDD[518] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 520.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 520.0 (TID 1010, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 520.0 (TID 1011, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 520.0 (TID 1012, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 520.0 (TID 1013, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 520.0 (TID 1012) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 520.0 (TID 1011) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 520.0 (TID 1010) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 520.0 (TID 1013) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 520.0 (TID 1012). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 520.0 (TID 1010). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 520.0 (TID 1012) in 4 ms on localhost (1/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 520.0 (TID 1010) in 6 ms on localhost (2/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 520.0 (TID 1013). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 520.0 (TID 1011). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 520.0 (TID 1013) in 6 ms on localhost (3/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 520.0 (TID 1011) in 6 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 520 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 520.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 334 finished: reduce at LDAOptimizer.scala:453, took 0.013973 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 335 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 521(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 521 (MapPartitionsRDD[519] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=893740, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_448 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(8446) called with curMem=914036, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_448_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_448_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 448 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 521 (MapPartitionsRDD[519] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 521.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 521.0 (TID 1014, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 521.0 (TID 1015, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 521.0 (TID 1016, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 521.0 (TID 1017, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 521.0 (TID 1015) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 521.0 (TID 1014) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 521.0 (TID 1017) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 521.0 (TID 1016) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 521.0 (TID 1017). 1314 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 521.0 (TID 1015). 1446 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 521.0 (TID 1016). 1002 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 521.0 (TID 1017) in 6 ms on localhost (1/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 521.0 (TID 1014). 1002 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 521.0 (TID 1015) in 7 ms on localhost (2/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 521.0 (TID 1014) in 8 ms on localhost (3/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 521.0 (TID 1016) in 7 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 521 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 521.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 335 finished: reduce at LDAOptimizer.scala:456, took 0.014971 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 336 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 522(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 522 (PartitionwiseSampledRDD[520] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=922482, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_449 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(7656) called with curMem=941298, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_449_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_449_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 449 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 522 (PartitionwiseSampledRDD[520] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 522.0 with 1 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 522.0 (TID 1018, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 522.0 (TID 1018) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 522.0 (TID 1018). 1226 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 522.0 (TID 1018) in 3 ms on localhost (1/1) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 522 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 336 finished: isEmpty at LDAOptimizer.scala:418, took 0.009902 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 522.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=948954, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_450 stored as values in memory (estimated size 176.0 B, free 729.7 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=949130, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_450_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.7 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_450_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 450 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 337 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 523(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 523 (MapPartitionsRDD[522] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=949410, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_451 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(8431) called with curMem=969666, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_451_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_451_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 451 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 523 (MapPartitionsRDD[522] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 523.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 523.0 (TID 1019, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 523.0 (TID 1020, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 523.0 (TID 1021, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 523.0 (TID 1022, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 523.0 (TID 1020) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 523.0 (TID 1022) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 523.0 (TID 1021) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 523.0 (TID 1019) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 523.0 (TID 1019). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 523.0 (TID 1020). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 523.0 (TID 1021). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 523.0 (TID 1019) in 6 ms on localhost (1/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 523.0 (TID 1020) in 5 ms on localhost (2/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 523.0 (TID 1022). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 523.0 (TID 1021) in 6 ms on localhost (3/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 523.0 (TID 1022) in 5 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 523 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 337 finished: reduce at LDAOptimizer.scala:453, took 0.012586 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 523.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 338 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 524(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 524 (MapPartitionsRDD[523] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=978097, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_452 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(8444) called with curMem=998393, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_452_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_452_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 452 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 524 (MapPartitionsRDD[523] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 524.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 524.0 (TID 1023, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 524.0 (TID 1024, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 524.0 (TID 1025, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 524.0 (TID 1026, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 524.0 (TID 1024) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 524.0 (TID 1025) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 524.0 (TID 1026) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 524.0 (TID 1023) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 524.0 (TID 1023). 1358 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 524.0 (TID 1025). 1402 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 524.0 (TID 1026). 1402 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 524.0 (TID 1023) in 6 ms on localhost (1/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 524.0 (TID 1026) in 6 ms on localhost (2/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 524.0 (TID 1024). 1358 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 524.0 (TID 1025) in 6 ms on localhost (3/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 524.0 (TID 1024) in 6 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 524 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 524.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 338 finished: reduce at LDAOptimizer.scala:456, took 0.013374 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 339 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 525(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 525 (PartitionwiseSampledRDD[524] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1006837, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_453 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=1025653, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_453_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_453_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 453 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 525 (PartitionwiseSampledRDD[524] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 525.0 with 1 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 525.0 (TID 1027, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 525.0 (TID 1027) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 525.0 (TID 1027). 911 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 525.0 (TID 1027) in 3 ms on localhost (1/1) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 525 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 525.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 339 finished: isEmpty at LDAOptimizer.scala:418, took 0.009464 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 340 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 526(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 526 (PartitionwiseSampledRDD[524] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1033315, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_454 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=1052131, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_454_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_454_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 454 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 526 (PartitionwiseSampledRDD[524] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 526.0 with 3 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 526.0 (TID 1028, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 526.0 (TID 1029, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 526.0 (TID 1030, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 526.0 (TID 1029) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 526.0 (TID 1030) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 526.0 (TID 1028) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 526.0 (TID 1029). 911 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 526.0 (TID 1030). 1226 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 526.0 (TID 1028). 1226 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 526.0 (TID 1029) in 2 ms on localhost (1/3) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 526.0 (TID 1030) in 2 ms on localhost (2/3) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 526.0 (TID 1028) in 4 ms on localhost (3/3) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 526 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 526.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 340 finished: isEmpty at LDAOptimizer.scala:418, took 0.008789 s -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1059793, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_455 stored as values in memory (estimated size 176.0 B, free 729.6 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1059969, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_455_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.6 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_455_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 455 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 341 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 527(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 527 (MapPartitionsRDD[526] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1060249, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_456 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(8425) called with curMem=1080505, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_456_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_456_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 456 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 527 (MapPartitionsRDD[526] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 527.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 527.0 (TID 1031, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 527.0 (TID 1032, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 527.0 (TID 1033, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 527.0 (TID 1034, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 527.0 (TID 1031) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 527.0 (TID 1034) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 527.0 (TID 1034). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 527.0 (TID 1031). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 527.0 (TID 1033) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 527.0 (TID 1032) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 527.0 (TID 1032). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 527.0 (TID 1033). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 527.0 (TID 1034) in 24 ms on localhost (1/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 527.0 (TID 1031) in 25 ms on localhost (2/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 527.0 (TID 1032) in 26 ms on localhost (3/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 527.0 (TID 1033) in 28 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 527 (reduce at LDAOptimizer.scala:453) finished in 0.029 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 527.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 341 finished: reduce at LDAOptimizer.scala:453, took 0.034863 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 342 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 528(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 528 (MapPartitionsRDD[527] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1088930, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_457 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(8437) called with curMem=1109226, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_457_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_457_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 457 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 528 (MapPartitionsRDD[527] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 528.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 528.0 (TID 1035, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 528.0 (TID 1036, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 528.0 (TID 1037, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 528.0 (TID 1038, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 528.0 (TID 1037) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 528.0 (TID 1036) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 528.0 (TID 1037). 1002 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 528.0 (TID 1038) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 528.0 (TID 1035) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 528.0 (TID 1035). 1002 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 528.0 (TID 1038). 1402 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 528.0 (TID 1037) in 12 ms on localhost (1/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 528.0 (TID 1035) in 14 ms on localhost (2/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 528.0 (TID 1038) in 12 ms on localhost (3/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 528.0 (TID 1036). 1358 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 528.0 (TID 1036) in 17 ms on localhost (4/4) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 528.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 528 (reduce at LDAOptimizer.scala:456) finished in 0.023 s -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 342 finished: reduce at LDAOptimizer.scala:456, took 0.030916 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 343 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 529(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 529 (PartitionwiseSampledRDD[528] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1117663, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_458 stored as values in memory (estimated size 18.4 KB, free 729.5 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=1136479, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_458_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_458_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 458 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 529 (PartitionwiseSampledRDD[528] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 529.0 with 1 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 529.0 (TID 1039, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 529.0 (TID 1039) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 529.0 (TID 1039). 1226 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 529.0 (TID 1039) in 3 ms on localhost (1/1) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 529 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 529.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 343 finished: isEmpty at LDAOptimizer.scala:418, took 0.009949 s -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1144136, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_459 stored as values in memory (estimated size 176.0 B, free 729.5 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1144312, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_459_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_459_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 459 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 344 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 530(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 530 (MapPartitionsRDD[530] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1144592, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_460 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(8430) called with curMem=1164848, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_460_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_460_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 460 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 530 (MapPartitionsRDD[530] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 530.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 530.0 (TID 1040, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 530.0 (TID 1041, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 530.0 (TID 1042, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 530.0 (TID 1043, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 530.0 (TID 1040) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 530.0 (TID 1041) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 530.0 (TID 1043) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 530.0 (TID 1042) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 530.0 (TID 1040). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 530.0 (TID 1043). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 530.0 (TID 1041). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 530.0 (TID 1040) in 7 ms on localhost (1/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 530.0 (TID 1042). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 530.0 (TID 1043) in 7 ms on localhost (2/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 530.0 (TID 1041) in 8 ms on localhost (3/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 530.0 (TID 1042) in 9 ms on localhost (4/4) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 530.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 530 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 344 finished: reduce at LDAOptimizer.scala:453, took 0.016305 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 345 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 531(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 531 (MapPartitionsRDD[531] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1173278, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_461 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(8442) called with curMem=1193574, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_461_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_461_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 461 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 531 (MapPartitionsRDD[531] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 531.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 531.0 (TID 1044, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 531.0 (TID 1045, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 531.0 (TID 1046, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 531.0 (TID 1047, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 531.0 (TID 1044) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 531.0 (TID 1046) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 531.0 (TID 1047) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 531.0 (TID 1045) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 531.0 (TID 1046). 1314 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 531.0 (TID 1045). 1314 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 531.0 (TID 1046) in 5 ms on localhost (1/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 531.0 (TID 1045) in 6 ms on localhost (2/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 531.0 (TID 1044). 1358 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 531.0 (TID 1047). 1358 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 531.0 (TID 1044) in 8 ms on localhost (3/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 531.0 (TID 1047) in 6 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 531 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 531.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 345 finished: reduce at LDAOptimizer.scala:456, took 0.015836 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 346 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 532(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 532 (PartitionwiseSampledRDD[532] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1202016, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_462 stored as values in memory (estimated size 18.4 KB, free 729.4 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=1220832, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_462_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.4 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_462_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 462 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 532 (PartitionwiseSampledRDD[532] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 532.0 with 1 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 532.0 (TID 1048, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 532.0 (TID 1048) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 532.0 (TID 1048). 1226 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 532.0 (TID 1048) in 2 ms on localhost (1/1) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 532 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 346 finished: isEmpty at LDAOptimizer.scala:418, took 0.007138 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 532.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1228494, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_463 stored as values in memory (estimated size 176.0 B, free 729.4 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1228670, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_463_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.4 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_463_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 463 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 347 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 533(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 533 (MapPartitionsRDD[534] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1228950, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_464 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(8428) called with curMem=1249206, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_464_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_464_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 464 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 533 (MapPartitionsRDD[534] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 533.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 533.0 (TID 1049, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 533.0 (TID 1050, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 533.0 (TID 1051, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 533.0 (TID 1052, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 533.0 (TID 1051) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 533.0 (TID 1050) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 533.0 (TID 1049) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 533.0 (TID 1052) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 533.0 (TID 1051). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 533.0 (TID 1051) in 4 ms on localhost (1/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 533.0 (TID 1052). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 533.0 (TID 1052) in 4 ms on localhost (2/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 533.0 (TID 1049). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 533.0 (TID 1049) in 8 ms on localhost (3/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 533.0 (TID 1050). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 533.0 (TID 1050) in 9 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 533 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 533.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 347 finished: reduce at LDAOptimizer.scala:453, took 0.015407 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 348 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 534(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 534 (MapPartitionsRDD[535] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1257634, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_465 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(8440) called with curMem=1277930, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_465_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_465_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 465 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 534 (MapPartitionsRDD[535] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 534.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 534.0 (TID 1053, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 534.0 (TID 1054, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 534.0 (TID 1055, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 534.0 (TID 1056, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 534.0 (TID 1053) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 534.0 (TID 1054) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 534.0 (TID 1056) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 534.0 (TID 1055) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 534.0 (TID 1055). 1002 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 534.0 (TID 1055) in 4 ms on localhost (1/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 534.0 (TID 1053). 1314 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 534.0 (TID 1056). 1314 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 534.0 (TID 1053) in 6 ms on localhost (2/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 534.0 (TID 1056) in 5 ms on localhost (3/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 534.0 (TID 1054). 1622 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 534.0 (TID 1054) in 6 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 534 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 534.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 348 finished: reduce at LDAOptimizer.scala:456, took 0.012682 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 349 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 535(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 535 (PartitionwiseSampledRDD[536] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1286370, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_466 stored as values in memory (estimated size 18.4 KB, free 729.3 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(7655) called with curMem=1305186, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_466_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.3 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_466_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 466 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 535 (PartitionwiseSampledRDD[536] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 535.0 with 1 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 535.0 (TID 1057, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 535.0 (TID 1057) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 535.0 (TID 1057). 1226 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 535.0 (TID 1057) in 3 ms on localhost (1/1) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 535 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 535.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 349 finished: isEmpty at LDAOptimizer.scala:418, took 0.007664 s -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1312841, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_467 stored as values in memory (estimated size 176.0 B, free 729.3 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1313017, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_467_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.3 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_467_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 467 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 350 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 536(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 536 (MapPartitionsRDD[538] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1313297, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_468 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(8417) called with curMem=1333553, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_468_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.3 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_468_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 468 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 536 (MapPartitionsRDD[538] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 536.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 536.0 (TID 1058, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 536.0 (TID 1059, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 536.0 (TID 1060, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 536.0 (TID 1061, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 536.0 (TID 1060) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 536.0 (TID 1059) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 536.0 (TID 1061) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 536.0 (TID 1058) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 536.0 (TID 1061). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 536.0 (TID 1060). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 536.0 (TID 1061) in 3 ms on localhost (1/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 536.0 (TID 1059). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 536.0 (TID 1060) in 5 ms on localhost (2/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 536.0 (TID 1058). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 536.0 (TID 1059) in 5 ms on localhost (3/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 536.0 (TID 1058) in 6 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 536 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 536.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 350 finished: reduce at LDAOptimizer.scala:453, took 0.013886 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 351 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 537(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 537 (MapPartitionsRDD[539] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1341970, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_469 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(8429) called with curMem=1362266, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_469_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.3 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_469_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 469 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 537 (MapPartitionsRDD[539] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 537.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 537.0 (TID 1062, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 537.0 (TID 1063, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 537.0 (TID 1064, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 537.0 (TID 1065, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 537.0 (TID 1062) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 537.0 (TID 1065) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 537.0 (TID 1063) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 537.0 (TID 1064) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 537.0 (TID 1063). 1402 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 537.0 (TID 1065). 1358 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 537.0 (TID 1064). 1358 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 537.0 (TID 1062). 1402 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 537.0 (TID 1063) in 6 ms on localhost (1/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 537.0 (TID 1064) in 6 ms on localhost (2/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 537.0 (TID 1065) in 6 ms on localhost (3/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 537.0 (TID 1062) in 6 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 537 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 537.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 351 finished: reduce at LDAOptimizer.scala:456, took 0.013051 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 352 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 538(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 538 (PartitionwiseSampledRDD[540] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1370695, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_470 stored as values in memory (estimated size 18.4 KB, free 729.3 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(7657) called with curMem=1389511, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_470_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.3 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_470_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 470 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 538 (PartitionwiseSampledRDD[540] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 538.0 with 1 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 538.0 (TID 1066, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 538.0 (TID 1066) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 538.0 (TID 1066). 1226 bytes result sent to driver -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 538 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 352 finished: isEmpty at LDAOptimizer.scala:418, took 0.010277 s -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 538.0 (TID 1066) in 4 ms on localhost (1/1) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 538.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1397168, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_471 stored as values in memory (estimated size 176.0 B, free 729.3 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1397344, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_471_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.3 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_471_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 329 -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 471 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_470_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 372 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_469_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 371 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 353 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 539(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 539 (MapPartitionsRDD[542] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1334009, maxMem=766075207 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_468_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_472 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 370 -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=1334009, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_472_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.3 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_472_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 472 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 539 (MapPartitionsRDD[542] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 539.0 with 4 tasks -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_467_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 539.0 (TID 1067, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 539.0 (TID 1068, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 539.0 (TID 1069, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 539.0 (TID 1070, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 539.0 (TID 1068) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 539.0 (TID 1067) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 539.0 (TID 1070) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 539.0 (TID 1069) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 539.0 (TID 1068). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 539.0 (TID 1070). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 539.0 (TID 1068) in 5 ms on localhost (1/4) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_466_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 539.0 (TID 1069). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 539.0 (TID 1070) in 5 ms on localhost (2/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 539.0 (TID 1069) in 6 ms on localhost (3/4) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 369 -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 539.0 (TID 1067). 1265 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 539.0 (TID 1067) in 9 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 539 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 539.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 353 finished: reduce at LDAOptimizer.scala:453, took 0.016428 s -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_465_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 368 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_464_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 367 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 354 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 540(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 540 (MapPartitionsRDD[543] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1257814, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_473 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_463_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(8444) called with curMem=1277934, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_473_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_473_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 473 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 540 (MapPartitionsRDD[543] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 540.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 540.0 (TID 1071, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_462_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 540.0 (TID 1072, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 366 -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 540.0 (TID 1073, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 540.0 (TID 1074, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 540.0 (TID 1071) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 540.0 (TID 1073) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 540.0 (TID 1072) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 540.0 (TID 1074) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 540.0 (TID 1072). 1002 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 540.0 (TID 1073). 1002 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 540.0 (TID 1071). 1358 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 540.0 (TID 1072) in 5 ms on localhost (1/4) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_461_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 540.0 (TID 1073) in 5 ms on localhost (2/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 540.0 (TID 1074). 1002 bytes result sent to driver -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 365 -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 540.0 (TID 1071) in 7 ms on localhost (3/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 540.0 (TID 1074) in 5 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 540 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 354 finished: reduce at LDAOptimizer.scala:456, took 0.014115 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 540.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_460_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 364 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_459_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_458_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 363 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_457_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 362 -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1126558, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_474 stored as values in memory (estimated size 176.0 B, free 729.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_456_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1118309, maxMem=766075207 -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 361 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_474_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_474_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 474 from broadcast at LDAModel.scala:305 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_455_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_454_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 360 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_453_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 359 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_452_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 358 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_451_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 357 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_450_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_449_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 356 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_448_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 355 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_447_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 354 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_446_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_445_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 353 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_444_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 352 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_443_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 351 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_442_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 350 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_441_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_440_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 349 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_439_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 348 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_438_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 347 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_437_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_436_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 346 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_435_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 345 -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: sum at LDAModel.scala:327 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 355 (sum at LDAModel.scala:327) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 541(sum at LDAModel.scala:327) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 541 (MapPartitionsRDD[547] at map at LDAModel.scala:310), which has no missing parents -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_434_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(3360) called with curMem=643866, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_475 stored as values in memory (estimated size 3.3 KB, free 730.0 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 344 -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(2028) called with curMem=646770, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_475_piece0 stored as bytes in memory (estimated size 2028.0 B, free 730.0 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_433_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_475_piece0 in memory on localhost:50000 (size: 2028.0 B, free: 730.4 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 475 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 541 (MapPartitionsRDD[547] at map at LDAModel.scala:310) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 541.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 541.0 (TID 1075, localhost, PROCESS_LOCAL, 2083 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 541.0 (TID 1076, localhost, PROCESS_LOCAL, 2083 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 541.0 (TID 1077, localhost, PROCESS_LOCAL, 2083 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 541.0 (TID 1078, localhost, PROCESS_LOCAL, 2294 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 541.0 (TID 1077) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 541.0 (TID 1078) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 541.0 (TID 1075) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_432_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 541.0 (TID 1076) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 541.0 (TID 1077). 955 bytes result sent to driver -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 343 -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 541.0 (TID 1076). 955 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 541.0 (TID 1075). 955 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 541.0 (TID 1077) in 4 ms on localhost (1/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 541.0 (TID 1076) in 4 ms on localhost (2/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 541.0 (TID 1075) in 6 ms on localhost (3/4) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_431_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 342 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_430_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 341 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_429_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_428_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 340 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_427_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 339 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_426_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 338 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_425_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 337 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_424_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_423_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 336 -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 541.0 (TID 1078). 955 bytes result sent to driver -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_422_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 541.0 (TID 1078) in 31 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 541 (sum at LDAModel.scala:327) finished in 0.032 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 541.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 335 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 355 finished: sum at LDAModel.scala:327, took 0.038779 s -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_421_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 334 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_420_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_419_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 333 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_418_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 332 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_417_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 331 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_416_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_415_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 330 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_414_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_413_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 328 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_412_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=181794, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_476 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_411_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 327 -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=174312, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_476_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_476_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_410_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 476 from broadcast at LDAModel.scala:305 -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 326 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_409_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 325 -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: sum at LDAModel.scala:327 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_408_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 356 (sum at LDAModel.scala:327) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 542(sum at LDAModel.scala:327) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 542 (MapPartitionsRDD[549] at map at LDAModel.scala:310), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(3360) called with curMem=109048, maxMem=766075207 -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Removed broadcast_407_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_477 stored as values in memory (estimated size 3.3 KB, free 730.5 MB) -2016-06-13 19:45:22 INFO ContextCleaner:59 - Cleaned accumulator 324 -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(2032) called with curMem=93592, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_477_piece0 stored as bytes in memory (estimated size 2032.0 B, free 730.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_477_piece0 in memory on localhost:50000 (size: 2032.0 B, free: 730.5 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 477 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 542 (MapPartitionsRDD[549] at map at LDAModel.scala:310) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 542.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 542.0 (TID 1079, localhost, PROCESS_LOCAL, 2083 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 542.0 (TID 1080, localhost, PROCESS_LOCAL, 2083 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 542.0 (TID 1081, localhost, PROCESS_LOCAL, 2083 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 542.0 (TID 1082, localhost, PROCESS_LOCAL, 2294 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 542.0 (TID 1079) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 542.0 (TID 1082) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 542.0 (TID 1081) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 542.0 (TID 1080) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 542.0 (TID 1079). 955 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 542.0 (TID 1080). 955 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 542.0 (TID 1081). 955 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 542.0 (TID 1079) in 3 ms on localhost (1/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 542.0 (TID 1081) in 3 ms on localhost (2/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 542.0 (TID 1080) in 3 ms on localhost (3/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 542.0 (TID 1082). 955 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 542.0 (TID 1082) in 4 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 542 (sum at LDAModel.scala:327) finished in 0.005 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 542.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 356 finished: sum at LDAModel.scala:327, took 0.010329 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: sum at LDAModel.scala:265 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 357 (sum at LDAModel.scala:265) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 543(sum at LDAModel.scala:265) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 543 (MapPartitionsRDD[551] at map at LDAModel.scala:264), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(2264) called with curMem=95624, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_478 stored as values in memory (estimated size 2.2 KB, free 730.5 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(1417) called with curMem=97888, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_478_piece0 stored as bytes in memory (estimated size 1417.0 B, free 730.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_478_piece0 in memory on localhost:50000 (size: 1417.0 B, free: 730.5 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 478 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 543 (MapPartitionsRDD[551] at map at LDAModel.scala:264) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 543.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 543.0 (TID 1083, localhost, PROCESS_LOCAL, 2306 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 543.0 (TID 1084, localhost, PROCESS_LOCAL, 2389 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 543.0 (TID 1085, localhost, PROCESS_LOCAL, 2315 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 543.0 (TID 1086, localhost, PROCESS_LOCAL, 2389 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 543.0 (TID 1084) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 543.0 (TID 1086) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 543.0 (TID 1085) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 543.0 (TID 1083) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 543.0 (TID 1086). 955 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 543.0 (TID 1083). 955 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 543.0 (TID 1084). 955 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 543.0 (TID 1083) in 3 ms on localhost (1/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 543.0 (TID 1085). 955 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 543.0 (TID 1086) in 4 ms on localhost (2/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 543.0 (TID 1084) in 4 ms on localhost (3/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 543.0 (TID 1085) in 5 ms on localhost (4/4) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 543.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 543 (sum at LDAModel.scala:265) finished in 0.006 s -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 357 finished: sum at LDAModel.scala:265, took 0.011042 s -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=99305, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_479 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=99481, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_479_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_479_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 479 from broadcast at LDAModel.scala:305 -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: sum at LDAModel.scala:327 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 358 (sum at LDAModel.scala:327) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 544(sum at LDAModel.scala:327) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 544 (MapPartitionsRDD[553] at map at LDAModel.scala:310), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(3360) called with curMem=99761, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_480 stored as values in memory (estimated size 3.3 KB, free 730.5 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(2033) called with curMem=103121, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_480_piece0 stored as bytes in memory (estimated size 2033.0 B, free 730.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_480_piece0 in memory on localhost:50000 (size: 2033.0 B, free: 730.5 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 480 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 544 (MapPartitionsRDD[553] at map at LDAModel.scala:310) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 544.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 544.0 (TID 1087, localhost, PROCESS_LOCAL, 2306 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 544.0 (TID 1088, localhost, PROCESS_LOCAL, 2389 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 544.0 (TID 1089, localhost, PROCESS_LOCAL, 2315 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 544.0 (TID 1090, localhost, PROCESS_LOCAL, 2389 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 544.0 (TID 1089) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 544.0 (TID 1088) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 544.0 (TID 1090) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 544.0 (TID 1087) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 544.0 (TID 1089). 955 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 544.0 (TID 1088). 955 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 544.0 (TID 1089) in 4 ms on localhost (1/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 544.0 (TID 1090). 955 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 544.0 (TID 1087). 955 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 544.0 (TID 1088) in 4 ms on localhost (2/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 544.0 (TID 1090) in 5 ms on localhost (3/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 544.0 (TID 1087) in 6 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 544 (sum at LDAModel.scala:327) finished in 0.006 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 544.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 358 finished: sum at LDAModel.scala:327, took 0.011841 s -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=105154, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_481 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=105330, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_481_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_481_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 481 from broadcast at LDAModel.scala:355 -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: sortByKey at LDASuite.scala:372 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 359 (sortByKey at LDASuite.scala:372) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 545(sortByKey at LDASuite.scala:372) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 545 (MapPartitionsRDD[558] at sortByKey at LDASuite.scala:372), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(3584) called with curMem=105610, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_482 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(2131) called with curMem=109194, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_482_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_482_piece0 in memory on localhost:50000 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 482 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 545 (MapPartitionsRDD[558] at sortByKey at LDASuite.scala:372) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 545.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 545.0 (TID 1091, localhost, PROCESS_LOCAL, 2306 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 545.0 (TID 1092, localhost, PROCESS_LOCAL, 2389 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 545.0 (TID 1093, localhost, PROCESS_LOCAL, 2315 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 545.0 (TID 1094, localhost, PROCESS_LOCAL, 2389 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 545.0 (TID 1093) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 545.0 (TID 1094) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 545.0 (TID 1092) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 545.0 (TID 1091) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 545.0 (TID 1091). 1104 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 545.0 (TID 1094). 1115 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 545.0 (TID 1092). 1115 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 545.0 (TID 1093). 1104 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 545.0 (TID 1091) in 4 ms on localhost (1/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 545.0 (TID 1092) in 5 ms on localhost (2/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 545.0 (TID 1094) in 5 ms on localhost (3/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 545.0 (TID 1093) in 5 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 545 (sortByKey at LDASuite.scala:372) finished in 0.006 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 545.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 359 finished: sortByKey at LDASuite.scala:372, took 0.010723 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: collect at LDASuite.scala:374 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Registering RDD 556 (map at LDASuite.scala:368) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 360 (collect at LDASuite.scala:374) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 547(collect at LDASuite.scala:374) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 546) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 546) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ShuffleMapStage 546 (MapPartitionsRDD[556] at map at LDASuite.scala:368), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(3872) called with curMem=111325, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_483 stored as values in memory (estimated size 3.8 KB, free 730.5 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(2324) called with curMem=115197, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_483_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_483_piece0 in memory on localhost:50000 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 483 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ShuffleMapStage 546 (MapPartitionsRDD[556] at map at LDASuite.scala:368) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 546.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 546.0 (TID 1095, localhost, PROCESS_LOCAL, 2295 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 546.0 (TID 1096, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 546.0 (TID 1097, localhost, PROCESS_LOCAL, 2304 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 546.0 (TID 1098, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 546.0 (TID 1096) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 546.0 (TID 1097) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 546.0 (TID 1095) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 546.0 (TID 1098) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 546.0 (TID 1095). 1161 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 546.0 (TID 1097). 1161 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 546.0 (TID 1095) in 52 ms on localhost (1/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 546.0 (TID 1097) in 51 ms on localhost (2/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 546.0 (TID 1098). 1161 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 546.0 (TID 1096). 1161 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 546.0 (TID 1098) in 66 ms on localhost (3/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 546.0 (TID 1096) in 67 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ShuffleMapStage 546 (map at LDASuite.scala:368) finished in 0.068 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 546.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:22 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:22 INFO DAGScheduler:59 - waiting: Set(ResultStage 547) -2016-06-13 19:45:22 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents for ResultStage 547: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 547 (MapPartitionsRDD[560] at values at LDASuite.scala:373), which is now runnable -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(3024) called with curMem=117521, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_484 stored as values in memory (estimated size 3.0 KB, free 730.5 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(1763) called with curMem=120545, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_484_piece0 stored as bytes in memory (estimated size 1763.0 B, free 730.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_484_piece0 in memory on localhost:50000 (size: 1763.0 B, free: 730.5 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 484 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 547 (MapPartitionsRDD[560] at values at LDASuite.scala:373) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 547.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 547.0 (TID 1099, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 547.0 (TID 1100, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 547.0 (TID 1101, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 547.0 (TID 1102, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 547.0 (TID 1099) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 547.0 (TID 1102) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 547.0 (TID 1100) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 547.0 (TID 1101) -2016-06-13 19:45:22 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 19:45:22 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 19:45:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:22 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 19:45:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:22 INFO ShuffleBlockFetcherIterator:59 - Getting 4 non-empty blocks out of 4 blocks -2016-06-13 19:45:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 547.0 (TID 1100). 1319 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 547.0 (TID 1099). 1319 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 547.0 (TID 1102). 1299 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 547.0 (TID 1101). 1299 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 547.0 (TID 1100) in 16 ms on localhost (1/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 547.0 (TID 1099) in 18 ms on localhost (2/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 547.0 (TID 1102) in 17 ms on localhost (3/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 547.0 (TID 1101) in 17 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 547 (collect at LDASuite.scala:374) finished in 0.018 s -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 360 finished: collect at LDASuite.scala:374, took 0.097488 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 547.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: count at LDAOptimizer.scala:387 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 361 (count at LDAOptimizer.scala:387) with 4 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 548(count at LDAOptimizer.scala:387) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 548 (ParallelCollectionRDD[561] at parallelize at LDASuite.scala:382), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=122308, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_485 stored as values in memory (estimated size 1032.0 B, free 730.5 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(757) called with curMem=123340, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_485_piece0 stored as bytes in memory (estimated size 757.0 B, free 730.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_485_piece0 in memory on localhost:50000 (size: 757.0 B, free: 730.5 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 485 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 548 (ParallelCollectionRDD[561] at parallelize at LDASuite.scala:382) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 548.0 with 4 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 548.0 (TID 1103, localhost, PROCESS_LOCAL, 2306 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 548.0 (TID 1104, localhost, PROCESS_LOCAL, 2389 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 548.0 (TID 1105, localhost, PROCESS_LOCAL, 2315 bytes) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 3.0 in stage 548.0 (TID 1106, localhost, PROCESS_LOCAL, 2389 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 1.0 in stage 548.0 (TID 1104) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 548.0 (TID 1103) -2016-06-13 19:45:22 INFO Executor:59 - Running task 2.0 in stage 548.0 (TID 1105) -2016-06-13 19:45:22 INFO Executor:59 - Running task 3.0 in stage 548.0 (TID 1106) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 2.0 in stage 548.0 (TID 1105). 953 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 1.0 in stage 548.0 (TID 1104). 953 bytes result sent to driver -2016-06-13 19:45:22 INFO Executor:59 - Finished task 3.0 in stage 548.0 (TID 1106). 953 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 548.0 (TID 1105) in 3 ms on localhost (1/4) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 548.0 (TID 1103). 953 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 548.0 (TID 1104) in 4 ms on localhost (2/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 3.0 in stage 548.0 (TID 1106) in 4 ms on localhost (3/4) -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 548.0 (TID 1103) in 6 ms on localhost (4/4) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 548 (count at LDAOptimizer.scala:387) finished in 0.006 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 548.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 361 finished: count at LDAOptimizer.scala:387, took 0.011921 s -2016-06-13 19:45:22 INFO SparkContext:59 - Starting job: first at LDAOptimizer.scala:388 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Got job 362 (first at LDAOptimizer.scala:388) with 1 output partitions -2016-06-13 19:45:22 INFO DAGScheduler:59 - Final stage: ResultStage 549(first at LDAOptimizer.scala:388) -2016-06-13 19:45:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting ResultStage 549 (ParallelCollectionRDD[561] at parallelize at LDASuite.scala:382), which has no missing parents -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=124097, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_486 stored as values in memory (estimated size 1224.0 B, free 730.5 MB) -2016-06-13 19:45:22 INFO MemoryStore:59 - ensureFreeSpace(807) called with curMem=125321, maxMem=766075207 -2016-06-13 19:45:22 INFO MemoryStore:59 - Block broadcast_486_piece0 stored as bytes in memory (estimated size 807.0 B, free 730.5 MB) -2016-06-13 19:45:22 INFO BlockManagerInfo:59 - Added broadcast_486_piece0 in memory on localhost:50000 (size: 807.0 B, free: 730.5 MB) -2016-06-13 19:45:22 INFO SparkContext:59 - Created broadcast 486 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 549 (ParallelCollectionRDD[561] at parallelize at LDASuite.scala:382) -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Adding task set 549.0 with 1 tasks -2016-06-13 19:45:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 549.0 (TID 1107, localhost, PROCESS_LOCAL, 2306 bytes) -2016-06-13 19:45:22 INFO Executor:59 - Running task 0.0 in stage 549.0 (TID 1107) -2016-06-13 19:45:22 INFO Executor:59 - Finished task 0.0 in stage 549.0 (TID 1107). 1226 bytes result sent to driver -2016-06-13 19:45:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 549.0 (TID 1107) in 2 ms on localhost (1/1) -2016-06-13 19:45:22 INFO DAGScheduler:59 - ResultStage 549 (first at LDAOptimizer.scala:388) finished in 0.002 s -2016-06-13 19:45:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 549.0, whose tasks have all completed, from pool -2016-06-13 19:45:22 INFO DAGScheduler:59 - Job 362 finished: first at LDAOptimizer.scala:388, took 0.006658 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 363 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 550(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 550 (PartitionwiseSampledRDD[562] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=126128, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_487 stored as values in memory (estimated size 18.4 KB, free 730.4 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7663) called with curMem=144944, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_487_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.4 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_487_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 487 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 550 (PartitionwiseSampledRDD[562] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 550.0 with 1 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 550.0 (TID 1108, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 550.0 (TID 1108) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 550.0 (TID 1108). 911 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 550.0 (TID 1108) in 2 ms on localhost (1/1) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 550.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 550 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 363 finished: isEmpty at LDAOptimizer.scala:418, took 0.007691 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 364 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 551(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 551 (PartitionwiseSampledRDD[562] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=152607, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_488 stored as values in memory (estimated size 18.4 KB, free 730.4 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7663) called with curMem=171423, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_488_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.4 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_488_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 488 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 551 (PartitionwiseSampledRDD[562] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 551.0 with 3 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 551.0 (TID 1109, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 551.0 (TID 1110, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 551.0 (TID 1111, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 551.0 (TID 1111) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 551.0 (TID 1110) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 551.0 (TID 1109) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 551.0 (TID 1109). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 551.0 (TID 1111). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 551.0 (TID 1110). 911 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 551.0 (TID 1109) in 4 ms on localhost (1/3) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 551.0 (TID 1111) in 3 ms on localhost (2/3) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 551.0 (TID 1110) in 3 ms on localhost (3/3) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 551.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 551 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 364 finished: isEmpty at LDAOptimizer.scala:418, took 0.009944 s -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=179086, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_489 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=179262, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_489_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_489_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 489 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 365 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 552(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 552 (MapPartitionsRDD[564] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=179542, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_490 stored as values in memory (estimated size 19.8 KB, free 730.4 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8436) called with curMem=199798, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_490_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.4 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_490_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 490 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 552 (MapPartitionsRDD[564] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 552.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 552.0 (TID 1112, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 552.0 (TID 1113, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 552.0 (TID 1114, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 552.0 (TID 1115, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 552.0 (TID 1113) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 552.0 (TID 1112) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 552.0 (TID 1114) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 552.0 (TID 1115) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 552.0 (TID 1112). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 552.0 (TID 1113). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 552.0 (TID 1115). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 552.0 (TID 1112) in 5 ms on localhost (1/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 552.0 (TID 1113) in 6 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 552.0 (TID 1115) in 5 ms on localhost (3/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 552.0 (TID 1114). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 552.0 (TID 1114) in 6 ms on localhost (4/4) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 552.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 552 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 365 finished: reduce at LDAOptimizer.scala:453, took 0.013640 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 366 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 553(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 553 (MapPartitionsRDD[565] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=208234, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_491 stored as values in memory (estimated size 19.8 KB, free 730.4 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8449) called with curMem=228530, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_491_piece0 stored as bytes in memory (estimated size 8.3 KB, free 730.4 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_491_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.5 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 491 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 553 (MapPartitionsRDD[565] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 553.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 553.0 (TID 1116, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 553.0 (TID 1117, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 553.0 (TID 1118, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 553.0 (TID 1119, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 553.0 (TID 1118) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 553.0 (TID 1119) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 553.0 (TID 1116) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 553.0 (TID 1117) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 553.0 (TID 1118). 1002 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 553.0 (TID 1116). 1002 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 553.0 (TID 1117). 1314 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 553.0 (TID 1119). 1314 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 553.0 (TID 1118) in 5 ms on localhost (1/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 553.0 (TID 1116) in 6 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 553.0 (TID 1117) in 5 ms on localhost (3/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 553.0 (TID 1119) in 5 ms on localhost (4/4) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 553 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 553.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 366 finished: reduce at LDAOptimizer.scala:456, took 0.011167 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 367 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 554(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 554 (PartitionwiseSampledRDD[566] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=236979, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_492 stored as values in memory (estimated size 18.4 KB, free 730.3 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=255795, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_492_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.3 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_492_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 492 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 554 (PartitionwiseSampledRDD[566] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 554.0 with 1 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 554.0 (TID 1120, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 554.0 (TID 1120) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 554.0 (TID 1120). 911 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 554.0 (TID 1120) in 2 ms on localhost (1/1) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 554 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 554.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 367 finished: isEmpty at LDAOptimizer.scala:418, took 0.007507 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 368 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 555(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 555 (PartitionwiseSampledRDD[566] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=263455, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_493 stored as values in memory (estimated size 18.4 KB, free 730.3 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=282271, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_493_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.3 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_493_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 493 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 555 (PartitionwiseSampledRDD[566] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 555.0 with 3 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 555.0 (TID 1121, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 555.0 (TID 1122, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 555.0 (TID 1123, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 555.0 (TID 1123) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 555.0 (TID 1121) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 555.0 (TID 1122) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 555.0 (TID 1123). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 555.0 (TID 1122). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 555.0 (TID 1121). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 555.0 (TID 1123) in 2 ms on localhost (1/3) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 555.0 (TID 1122) in 3 ms on localhost (2/3) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 555.0 (TID 1121) in 3 ms on localhost (3/3) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 555.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 555 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 368 finished: isEmpty at LDAOptimizer.scala:418, took 0.009039 s -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=289931, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_494 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=290107, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_494_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_494_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 494 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 369 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 556(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 556 (MapPartitionsRDD[568] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=290387, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_495 stored as values in memory (estimated size 19.8 KB, free 730.3 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8429) called with curMem=310643, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_495_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.3 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_495_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 495 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 556 (MapPartitionsRDD[568] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 556.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 556.0 (TID 1124, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 556.0 (TID 1125, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 556.0 (TID 1126, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 556.0 (TID 1127, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 556.0 (TID 1124) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 556.0 (TID 1126) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 556.0 (TID 1125) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 556.0 (TID 1127) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 556.0 (TID 1124). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 556.0 (TID 1124) in 4 ms on localhost (1/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 556.0 (TID 1125). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 556.0 (TID 1127). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 556.0 (TID 1125) in 5 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 556.0 (TID 1127) in 4 ms on localhost (3/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 556.0 (TID 1126). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 556.0 (TID 1126) in 5 ms on localhost (4/4) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 556.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 556 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 369 finished: reduce at LDAOptimizer.scala:453, took 0.012177 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 370 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 557(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 557 (MapPartitionsRDD[569] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=319072, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_496 stored as values in memory (estimated size 19.8 KB, free 730.3 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8442) called with curMem=339368, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_496_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.3 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_496_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 496 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 557 (MapPartitionsRDD[569] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 557.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 557.0 (TID 1128, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 557.0 (TID 1129, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 557.0 (TID 1130, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 557.0 (TID 1131, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 557.0 (TID 1128) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 557.0 (TID 1129) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 557.0 (TID 1131) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 557.0 (TID 1130) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 557.0 (TID 1128). 1002 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 557.0 (TID 1128) in 5 ms on localhost (1/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 557.0 (TID 1129). 1314 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 557.0 (TID 1130). 1402 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 557.0 (TID 1129) in 6 ms on localhost (2/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 557.0 (TID 1131). 1358 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 557.0 (TID 1130) in 7 ms on localhost (3/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 557.0 (TID 1131) in 6 ms on localhost (4/4) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 557 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 557.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 370 finished: reduce at LDAOptimizer.scala:456, took 0.013885 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 371 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 558(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 558 (PartitionwiseSampledRDD[570] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=347810, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_497 stored as values in memory (estimated size 18.4 KB, free 730.2 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=366626, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_497_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_497_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 497 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 558 (PartitionwiseSampledRDD[570] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 558.0 with 1 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 558.0 (TID 1132, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 558.0 (TID 1132) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 558.0 (TID 1132). 911 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 558.0 (TID 1132) in 3 ms on localhost (1/1) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 558 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 558.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 371 finished: isEmpty at LDAOptimizer.scala:418, took 0.009780 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 372 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 559(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 559 (PartitionwiseSampledRDD[570] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=374287, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_498 stored as values in memory (estimated size 18.4 KB, free 730.2 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=393103, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_498_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_498_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 498 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 559 (PartitionwiseSampledRDD[570] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 559.0 with 3 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 559.0 (TID 1133, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 559.0 (TID 1134, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 559.0 (TID 1135, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 559.0 (TID 1134) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 559.0 (TID 1133) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 559.0 (TID 1135) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 559.0 (TID 1134). 911 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 559.0 (TID 1133). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 559.0 (TID 1134) in 4 ms on localhost (1/3) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 559.0 (TID 1135). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 559.0 (TID 1133) in 5 ms on localhost (2/3) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 559.0 (TID 1135) in 5 ms on localhost (3/3) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 559 (isEmpty at LDAOptimizer.scala:418) finished in 0.013 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 559.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 372 finished: isEmpty at LDAOptimizer.scala:418, took 0.020126 s -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=400764, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_499 stored as values in memory (estimated size 176.0 B, free 730.2 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=400940, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_499_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.2 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_499_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 499 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 373 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 560(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 560 (MapPartitionsRDD[572] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=401220, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_500 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8437) called with curMem=421476, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_500_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_500_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 500 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 560 (MapPartitionsRDD[572] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 560.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 560.0 (TID 1136, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 560.0 (TID 1137, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 560.0 (TID 1138, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 560.0 (TID 1139, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 560.0 (TID 1136) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 560.0 (TID 1139) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 560.0 (TID 1137) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 560.0 (TID 1138) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 560.0 (TID 1138). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 560.0 (TID 1136). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 560.0 (TID 1137). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 560.0 (TID 1138) in 4 ms on localhost (1/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 560.0 (TID 1136) in 5 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 560.0 (TID 1137) in 6 ms on localhost (3/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 560.0 (TID 1139). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 560.0 (TID 1139) in 5 ms on localhost (4/4) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 560 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 560.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 373 finished: reduce at LDAOptimizer.scala:453, took 0.011725 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 374 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 561(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 561 (MapPartitionsRDD[573] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=429913, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_501 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8450) called with curMem=450209, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_501_piece0 stored as bytes in memory (estimated size 8.3 KB, free 730.1 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_501_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 501 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 561 (MapPartitionsRDD[573] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 561.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 561.0 (TID 1140, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 561.0 (TID 1141, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 561.0 (TID 1142, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 561.0 (TID 1143, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 561.0 (TID 1141) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 561.0 (TID 1143) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 561.0 (TID 1142) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 561.0 (TID 1140) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 561.0 (TID 1140). 1002 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 561.0 (TID 1140) in 4 ms on localhost (1/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 561.0 (TID 1141). 1358 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 561.0 (TID 1142). 1002 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 561.0 (TID 1141) in 5 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 561.0 (TID 1142) in 5 ms on localhost (3/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 561.0 (TID 1143). 1402 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 561.0 (TID 1143) in 5 ms on localhost (4/4) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 561 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 561.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 374 finished: reduce at LDAOptimizer.scala:456, took 0.012768 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 375 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 562(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 562 (PartitionwiseSampledRDD[574] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=458659, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_502 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=477475, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_502_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_502_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 502 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 562 (PartitionwiseSampledRDD[574] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 562.0 with 1 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 562.0 (TID 1144, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 562.0 (TID 1144) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 562.0 (TID 1144). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 562.0 (TID 1144) in 2 ms on localhost (1/1) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 562 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 562.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 375 finished: isEmpty at LDAOptimizer.scala:418, took 0.007276 s -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=485136, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_503 stored as values in memory (estimated size 176.0 B, free 730.1 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=485312, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_503_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.1 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_503_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 503 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 376 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 563(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 563 (MapPartitionsRDD[576] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=485592, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_504 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8427) called with curMem=505848, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_504_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_504_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 504 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 563 (MapPartitionsRDD[576] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 563.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 563.0 (TID 1145, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 563.0 (TID 1146, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 563.0 (TID 1147, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 563.0 (TID 1148, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 563.0 (TID 1145) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 563.0 (TID 1148) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 563.0 (TID 1147) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 563.0 (TID 1146) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 563.0 (TID 1147). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 563.0 (TID 1148). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 563.0 (TID 1145). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 563.0 (TID 1147) in 3 ms on localhost (1/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 563.0 (TID 1148) in 4 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 563.0 (TID 1145) in 5 ms on localhost (3/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 563.0 (TID 1146). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 563.0 (TID 1146) in 5 ms on localhost (4/4) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 563 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 563.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 376 finished: reduce at LDAOptimizer.scala:453, took 0.011442 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 377 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 564(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 564 (MapPartitionsRDD[577] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=514275, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_505 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8439) called with curMem=534571, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_505_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_505_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 505 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 564 (MapPartitionsRDD[577] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 564.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 564.0 (TID 1149, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 564.0 (TID 1150, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 564.0 (TID 1151, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 564.0 (TID 1152, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 564.0 (TID 1149) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 564.0 (TID 1152) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 564.0 (TID 1151) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 564.0 (TID 1150) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 564.0 (TID 1151). 1002 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 564.0 (TID 1149). 1314 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 564.0 (TID 1151) in 6 ms on localhost (1/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 564.0 (TID 1149) in 7 ms on localhost (2/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 564.0 (TID 1150). 1446 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 564.0 (TID 1150) in 9 ms on localhost (3/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 564.0 (TID 1152). 1314 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 564.0 (TID 1152) in 10 ms on localhost (4/4) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 564 (reduce at LDAOptimizer.scala:456) finished in 0.011 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 564.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 377 finished: reduce at LDAOptimizer.scala:456, took 0.016548 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 378 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 565(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 565 (PartitionwiseSampledRDD[578] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=543010, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_506 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=561826, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_506_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_506_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 506 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 565 (PartitionwiseSampledRDD[578] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 565.0 with 1 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 565.0 (TID 1153, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 565.0 (TID 1153) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 565.0 (TID 1153). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 565.0 (TID 1153) in 3 ms on localhost (1/1) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 565 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 565.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 378 finished: isEmpty at LDAOptimizer.scala:418, took 0.008586 s -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=569486, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_507 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=569662, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_507_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_507_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 507 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 379 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 566(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 566 (MapPartitionsRDD[580] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=569942, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_508 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8435) called with curMem=590198, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_508_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_508_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 508 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 566 (MapPartitionsRDD[580] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 566.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 566.0 (TID 1154, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 566.0 (TID 1155, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 566.0 (TID 1156, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 566.0 (TID 1157, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 566.0 (TID 1154) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 566.0 (TID 1157) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 566.0 (TID 1156) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 566.0 (TID 1155) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 566.0 (TID 1157). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 566.0 (TID 1154). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 566.0 (TID 1155). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 566.0 (TID 1156). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 566.0 (TID 1157) in 8 ms on localhost (1/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 566.0 (TID 1154) in 10 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 566.0 (TID 1155) in 10 ms on localhost (3/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 566.0 (TID 1156) in 10 ms on localhost (4/4) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 566 (reduce at LDAOptimizer.scala:453) finished in 0.011 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 566.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 379 finished: reduce at LDAOptimizer.scala:453, took 0.016972 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 380 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 567(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 567 (MapPartitionsRDD[581] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=598633, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_509 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8448) called with curMem=618929, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_509_piece0 stored as bytes in memory (estimated size 8.3 KB, free 730.0 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_509_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 509 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 567 (MapPartitionsRDD[581] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 567.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 567.0 (TID 1158, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 567.0 (TID 1159, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 567.0 (TID 1160, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 567.0 (TID 1161, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 567.0 (TID 1158) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 567.0 (TID 1161) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 567.0 (TID 1160) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 567.0 (TID 1159) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 567.0 (TID 1158). 1358 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 567.0 (TID 1161). 1358 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 567.0 (TID 1160). 1358 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 567.0 (TID 1159). 1314 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 567.0 (TID 1158) in 10 ms on localhost (1/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 567.0 (TID 1161) in 10 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 567.0 (TID 1160) in 11 ms on localhost (3/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 567.0 (TID 1159) in 12 ms on localhost (4/4) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 567.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 567 (reduce at LDAOptimizer.scala:456) finished in 0.013 s -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 380 finished: reduce at LDAOptimizer.scala:456, took 0.019492 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 381 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 568(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 568 (PartitionwiseSampledRDD[582] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=627377, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_510 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=646193, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_510_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_510_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 510 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 568 (PartitionwiseSampledRDD[582] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 568.0 with 1 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 568.0 (TID 1162, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 568.0 (TID 1162) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 568.0 (TID 1162). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 568.0 (TID 1162) in 3 ms on localhost (1/1) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 568 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 568.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 381 finished: isEmpty at LDAOptimizer.scala:418, took 0.009631 s -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=653853, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_511 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=654029, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_511_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_511_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 511 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 382 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 569(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 569 (MapPartitionsRDD[584] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=654309, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_512 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8426) called with curMem=674565, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_512_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_512_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 512 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 569 (MapPartitionsRDD[584] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 569.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 569.0 (TID 1163, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 569.0 (TID 1164, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 569.0 (TID 1165, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 569.0 (TID 1166, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 569.0 (TID 1164) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 569.0 (TID 1165) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 569.0 (TID 1166) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 569.0 (TID 1163) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 569.0 (TID 1163). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 569.0 (TID 1164). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 569.0 (TID 1163) in 7 ms on localhost (1/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 569.0 (TID 1164) in 8 ms on localhost (2/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 569.0 (TID 1166). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 569.0 (TID 1166) in 10 ms on localhost (3/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 569.0 (TID 1165). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 569.0 (TID 1165) in 12 ms on localhost (4/4) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 569.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 569 (reduce at LDAOptimizer.scala:453) finished in 0.013 s -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 382 finished: reduce at LDAOptimizer.scala:453, took 0.018493 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 383 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 570(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 570 (MapPartitionsRDD[585] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=682991, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_513 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8438) called with curMem=703287, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_513_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_513_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 513 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 570 (MapPartitionsRDD[585] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 570.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 570.0 (TID 1167, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 570.0 (TID 1168, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 570.0 (TID 1169, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 570.0 (TID 1170, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 570.0 (TID 1167) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 570.0 (TID 1170) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 570.0 (TID 1169) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 570.0 (TID 1168) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 570.0 (TID 1167). 1358 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 570.0 (TID 1170). 1314 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 570.0 (TID 1168). 1314 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 570.0 (TID 1169). 1402 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 570.0 (TID 1167) in 8 ms on localhost (1/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 570.0 (TID 1170) in 7 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 570.0 (TID 1168) in 9 ms on localhost (3/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 570.0 (TID 1169) in 9 ms on localhost (4/4) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 570.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 570 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 383 finished: reduce at LDAOptimizer.scala:456, took 0.015636 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 384 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 571(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 571 (PartitionwiseSampledRDD[586] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=711725, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_514 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=730541, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_514_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_514_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 514 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 571 (PartitionwiseSampledRDD[586] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 571.0 with 1 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 571.0 (TID 1171, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 571.0 (TID 1171) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 571.0 (TID 1171). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 571.0 (TID 1171) in 4 ms on localhost (1/1) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 571.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 571 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 384 finished: isEmpty at LDAOptimizer.scala:418, took 0.012019 s -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=738201, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_515 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=738377, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_515_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_515_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 515 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 385 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 572(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 572 (MapPartitionsRDD[588] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=738657, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_516 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8431) called with curMem=758913, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_516_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_516_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 516 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 572 (MapPartitionsRDD[588] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 572.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 572.0 (TID 1172, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 572.0 (TID 1173, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 572.0 (TID 1174, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 572.0 (TID 1175, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 572.0 (TID 1174) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 572.0 (TID 1173) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 572.0 (TID 1174). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 572.0 (TID 1175) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 572.0 (TID 1172) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 572.0 (TID 1175). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 572.0 (TID 1172). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 572.0 (TID 1174) in 15 ms on localhost (1/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 572.0 (TID 1175) in 15 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 572.0 (TID 1172) in 17 ms on localhost (3/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 572.0 (TID 1173). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 572.0 (TID 1173) in 18 ms on localhost (4/4) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 572.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 572 (reduce at LDAOptimizer.scala:453) finished in 0.019 s -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 385 finished: reduce at LDAOptimizer.scala:453, took 0.025208 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 386 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 573(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 573 (MapPartitionsRDD[589] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=767344, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_517 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8442) called with curMem=787640, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_517_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_517_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 517 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 573 (MapPartitionsRDD[589] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 573.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 573.0 (TID 1176, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 573.0 (TID 1177, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 573.0 (TID 1178, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 573.0 (TID 1179, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 573.0 (TID 1177) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 573.0 (TID 1179) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 573.0 (TID 1178) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 573.0 (TID 1176) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 573.0 (TID 1177). 1446 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 573.0 (TID 1177) in 8 ms on localhost (1/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 573.0 (TID 1178). 1358 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 573.0 (TID 1179). 1314 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 573.0 (TID 1176). 1314 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 573.0 (TID 1178) in 9 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 573.0 (TID 1179) in 9 ms on localhost (3/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 573.0 (TID 1176) in 11 ms on localhost (4/4) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 573 (reduce at LDAOptimizer.scala:456) finished in 0.011 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 573.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 386 finished: reduce at LDAOptimizer.scala:456, took 0.017063 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 387 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 574(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 574 (PartitionwiseSampledRDD[590] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=796082, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_518 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7666) called with curMem=814898, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_518_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_518_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 518 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 574 (PartitionwiseSampledRDD[590] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 574.0 with 1 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 574.0 (TID 1180, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 574.0 (TID 1180) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 574.0 (TID 1180). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 574.0 (TID 1180) in 2 ms on localhost (1/1) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 574 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 574.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 387 finished: isEmpty at LDAOptimizer.scala:418, took 0.009008 s -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=822564, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_519 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=822740, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_519_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.8 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_519_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 519 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 388 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 575(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 575 (MapPartitionsRDD[592] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=823020, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_520 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8439) called with curMem=843276, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_520_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_520_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 520 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 575 (MapPartitionsRDD[592] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 575.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 575.0 (TID 1181, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 575.0 (TID 1182, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 575.0 (TID 1183, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 575.0 (TID 1184, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 575.0 (TID 1184) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 575.0 (TID 1183) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 575.0 (TID 1182) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 575.0 (TID 1182). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 575.0 (TID 1181) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 575.0 (TID 1183). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 575.0 (TID 1182) in 7 ms on localhost (1/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 575.0 (TID 1183) in 7 ms on localhost (2/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 575.0 (TID 1181). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 575.0 (TID 1184). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 575.0 (TID 1181) in 9 ms on localhost (3/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 575.0 (TID 1184) in 8 ms on localhost (4/4) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 575 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 575.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 388 finished: reduce at LDAOptimizer.scala:453, took 0.015129 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 389 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 576(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 576 (MapPartitionsRDD[593] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=851715, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_521 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8452) called with curMem=872011, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_521_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.7 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_521_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 521 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 576 (MapPartitionsRDD[593] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 576.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 576.0 (TID 1185, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 576.0 (TID 1186, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 576.0 (TID 1187, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 576.0 (TID 1188, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 576.0 (TID 1188) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 576.0 (TID 1185) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 576.0 (TID 1187) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 576.0 (TID 1186) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 576.0 (TID 1185). 1314 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 576.0 (TID 1188). 1358 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 576.0 (TID 1186). 1314 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 576.0 (TID 1187). 1314 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 576.0 (TID 1185) in 5 ms on localhost (1/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 576.0 (TID 1188) in 4 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 576.0 (TID 1186) in 6 ms on localhost (3/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 576.0 (TID 1187) in 5 ms on localhost (4/4) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 576 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 576.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 389 finished: reduce at LDAOptimizer.scala:456, took 0.010959 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 390 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 577(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 577 (PartitionwiseSampledRDD[594] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=880463, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_522 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7666) called with curMem=899279, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_522_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_522_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 522 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 577 (PartitionwiseSampledRDD[594] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 577.0 with 1 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 577.0 (TID 1189, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 577.0 (TID 1189) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 577.0 (TID 1189). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 577.0 (TID 1189) in 2 ms on localhost (1/1) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 577 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 577.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 390 finished: isEmpty at LDAOptimizer.scala:418, took 0.010164 s -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=906945, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_523 stored as values in memory (estimated size 176.0 B, free 729.7 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=907121, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_523_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.7 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_523_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 523 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 391 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 578(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 578 (MapPartitionsRDD[596] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=907401, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_524 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8439) called with curMem=927657, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_524_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_524_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 524 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 578 (MapPartitionsRDD[596] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 578.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 578.0 (TID 1190, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 578.0 (TID 1191, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 578.0 (TID 1192, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 578.0 (TID 1193, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 578.0 (TID 1190) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 578.0 (TID 1191) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 578.0 (TID 1192) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 578.0 (TID 1193) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 578.0 (TID 1191). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 578.0 (TID 1192). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 578.0 (TID 1193). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 578.0 (TID 1191) in 22 ms on localhost (1/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 578.0 (TID 1190). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 578.0 (TID 1193) in 21 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 578.0 (TID 1192) in 23 ms on localhost (3/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 578.0 (TID 1190) in 24 ms on localhost (4/4) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 578.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 578 (reduce at LDAOptimizer.scala:453) finished in 0.025 s -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 391 finished: reduce at LDAOptimizer.scala:453, took 0.031453 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 392 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 579(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 579 (MapPartitionsRDD[597] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=936096, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_525 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8453) called with curMem=956392, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_525_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.7 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_525_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 525 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 579 (MapPartitionsRDD[597] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 579.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 579.0 (TID 1194, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 579.0 (TID 1195, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 579.0 (TID 1196, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 579.0 (TID 1197, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 579.0 (TID 1194) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 579.0 (TID 1196) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 579.0 (TID 1195) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 579.0 (TID 1197) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 579.0 (TID 1194). 1314 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 579.0 (TID 1196). 1358 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 579.0 (TID 1194) in 6 ms on localhost (1/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 579.0 (TID 1196) in 6 ms on localhost (2/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 579.0 (TID 1195). 1358 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 579.0 (TID 1195) in 9 ms on localhost (3/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 579.0 (TID 1197). 1578 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 579.0 (TID 1197) in 14 ms on localhost (4/4) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 579.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 579 (reduce at LDAOptimizer.scala:456) finished in 0.015 s -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 392 finished: reduce at LDAOptimizer.scala:456, took 0.021255 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 393 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 580(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 580 (PartitionwiseSampledRDD[598] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=964845, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_526 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=983661, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_526_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_526_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 526 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 580 (PartitionwiseSampledRDD[598] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 580.0 with 1 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 580.0 (TID 1198, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 580.0 (TID 1198) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 580.0 (TID 1198). 911 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 580.0 (TID 1198) in 3 ms on localhost (1/1) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 580 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 580.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 393 finished: isEmpty at LDAOptimizer.scala:418, took 0.008983 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 394 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 581(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 581 (PartitionwiseSampledRDD[598] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=991322, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_527 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=1010138, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_527_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_527_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 527 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 581 (PartitionwiseSampledRDD[598] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 581.0 with 3 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 581.0 (TID 1199, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 581.0 (TID 1200, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 581.0 (TID 1201, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 581.0 (TID 1200) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 581.0 (TID 1201) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 581.0 (TID 1199) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 581.0 (TID 1200). 911 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 581.0 (TID 1199). 911 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 581.0 (TID 1201). 911 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 581.0 (TID 1200) in 4 ms on localhost (1/3) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 581.0 (TID 1199) in 4 ms on localhost (2/3) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 581.0 (TID 1201) in 5 ms on localhost (3/3) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 581 (isEmpty at LDAOptimizer.scala:418) finished in 0.006 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 581.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 394 finished: isEmpty at LDAOptimizer.scala:418, took 0.010133 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 395 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 582(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 582 (PartitionwiseSampledRDD[599] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1017799, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_528 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=1036615, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_528_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_528_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 528 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 582 (PartitionwiseSampledRDD[599] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 582.0 with 1 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 582.0 (TID 1202, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 582.0 (TID 1202) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 582.0 (TID 1202). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 582.0 (TID 1202) in 3 ms on localhost (1/1) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 582 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 582.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 395 finished: isEmpty at LDAOptimizer.scala:418, took 0.007241 s -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1044276, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_529 stored as values in memory (estimated size 176.0 B, free 729.6 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1044452, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_529_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.6 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_529_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 529 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 396 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 583(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 583 (MapPartitionsRDD[601] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1044732, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_530 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8435) called with curMem=1064988, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_530_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_530_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 530 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 583 (MapPartitionsRDD[601] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 583.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 583.0 (TID 1203, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 583.0 (TID 1204, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 583.0 (TID 1205, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 583.0 (TID 1206, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 583.0 (TID 1205) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 583.0 (TID 1204) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 583.0 (TID 1206) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 583.0 (TID 1203) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 583.0 (TID 1205). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 583.0 (TID 1205) in 4 ms on localhost (1/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 583.0 (TID 1206). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 583.0 (TID 1203). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 583.0 (TID 1206) in 8 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 583.0 (TID 1203) in 9 ms on localhost (3/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 583.0 (TID 1204). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 583.0 (TID 1204) in 9 ms on localhost (4/4) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 583 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 583.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 396 finished: reduce at LDAOptimizer.scala:453, took 0.017053 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 397 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 584(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 584 (MapPartitionsRDD[602] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1073423, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_531 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8448) called with curMem=1093719, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_531_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.5 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_531_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 531 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 584 (MapPartitionsRDD[602] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 584.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 584.0 (TID 1207, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 584.0 (TID 1208, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 584.0 (TID 1209, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 584.0 (TID 1210, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 584.0 (TID 1208) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 584.0 (TID 1210) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 584.0 (TID 1209) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 584.0 (TID 1207) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 584.0 (TID 1209). 1002 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 584.0 (TID 1209) in 4 ms on localhost (1/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 584.0 (TID 1207). 1314 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 584.0 (TID 1208). 1402 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 584.0 (TID 1210). 1402 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 584.0 (TID 1207) in 5 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 584.0 (TID 1208) in 5 ms on localhost (3/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 584.0 (TID 1210) in 6 ms on localhost (4/4) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 584 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 584.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 397 finished: reduce at LDAOptimizer.scala:456, took 0.012183 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 398 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 585(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 585 (PartitionwiseSampledRDD[603] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1102167, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_532 stored as values in memory (estimated size 18.4 KB, free 729.5 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=1120983, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_532_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.5 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_532_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 532 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 585 (PartitionwiseSampledRDD[603] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 585.0 with 1 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 585.0 (TID 1211, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 585.0 (TID 1211) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 585.0 (TID 1211). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 585.0 (TID 1211) in 2 ms on localhost (1/1) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 585 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 585.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 398 finished: isEmpty at LDAOptimizer.scala:418, took 0.007472 s -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1128644, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_533 stored as values in memory (estimated size 176.0 B, free 729.5 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1128820, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_533_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.5 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_533_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 533 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 399 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 586(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 586 (MapPartitionsRDD[605] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1129100, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_534 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8434) called with curMem=1149356, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_534_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_534_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 534 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 586 (MapPartitionsRDD[605] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 586.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 586.0 (TID 1212, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 586.0 (TID 1213, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 586.0 (TID 1214, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 586.0 (TID 1215, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 586.0 (TID 1212) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 586.0 (TID 1214) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 586.0 (TID 1215) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 586.0 (TID 1213) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 586.0 (TID 1214). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 586.0 (TID 1214) in 3 ms on localhost (1/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 586.0 (TID 1215). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 586.0 (TID 1212). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 586.0 (TID 1215) in 4 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 586.0 (TID 1212) in 5 ms on localhost (3/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 586.0 (TID 1213). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 586.0 (TID 1213) in 5 ms on localhost (4/4) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 586 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 586.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 399 finished: reduce at LDAOptimizer.scala:453, took 0.011232 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 400 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 587(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 587 (MapPartitionsRDD[606] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1157790, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_535 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8447) called with curMem=1178086, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_535_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_535_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 535 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 587 (MapPartitionsRDD[606] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 587.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 587.0 (TID 1216, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 587.0 (TID 1217, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 587.0 (TID 1218, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 587.0 (TID 1219, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 587.0 (TID 1217) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 587.0 (TID 1219) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 587.0 (TID 1218) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 587.0 (TID 1216) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 587.0 (TID 1219). 1002 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 587.0 (TID 1218). 1002 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 587.0 (TID 1219) in 3 ms on localhost (1/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 587.0 (TID 1218) in 3 ms on localhost (2/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 587.0 (TID 1216). 1314 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 587.0 (TID 1216) in 5 ms on localhost (3/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 587.0 (TID 1217). 1490 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 587.0 (TID 1217) in 5 ms on localhost (4/4) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 587 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 587.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 400 finished: reduce at LDAOptimizer.scala:456, took 0.011050 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 401 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 588(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 588 (PartitionwiseSampledRDD[607] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1186533, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_536 stored as values in memory (estimated size 18.4 KB, free 729.4 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=1205349, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_536_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.4 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_536_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 536 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 588 (PartitionwiseSampledRDD[607] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 588.0 with 1 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 588.0 (TID 1220, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 588.0 (TID 1220) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 588.0 (TID 1220). 911 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 588.0 (TID 1220) in 3 ms on localhost (1/1) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 588 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 588.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 401 finished: isEmpty at LDAOptimizer.scala:418, took 0.007187 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 402 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 589(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 589 (PartitionwiseSampledRDD[607] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1213011, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_537 stored as values in memory (estimated size 18.4 KB, free 729.4 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=1231827, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_537_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.4 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_537_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 537 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 589 (PartitionwiseSampledRDD[607] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 589.0 with 3 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 589.0 (TID 1221, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 589.0 (TID 1222, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 589.0 (TID 1223, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 589.0 (TID 1222) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 589.0 (TID 1223) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 589.0 (TID 1221) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 589.0 (TID 1221). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 589.0 (TID 1223). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 589.0 (TID 1221) in 3 ms on localhost (1/3) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 589.0 (TID 1222). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 589.0 (TID 1222) in 4 ms on localhost (2/3) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 589.0 (TID 1223) in 3 ms on localhost (3/3) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 589 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 589.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 402 finished: isEmpty at LDAOptimizer.scala:418, took 0.010252 s -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1239489, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_538 stored as values in memory (estimated size 176.0 B, free 729.4 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1239665, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_538_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.4 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_538_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 538 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 403 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 590(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 590 (MapPartitionsRDD[609] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1239945, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_539 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8437) called with curMem=1260201, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_539_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_539_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 539 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 590 (MapPartitionsRDD[609] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 590.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 590.0 (TID 1224, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 590.0 (TID 1225, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 590.0 (TID 1226, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 590.0 (TID 1227, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 590.0 (TID 1224) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 590.0 (TID 1226) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 590.0 (TID 1225) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 590.0 (TID 1227) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 590.0 (TID 1224). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 590.0 (TID 1224) in 5 ms on localhost (1/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 590.0 (TID 1225). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 590.0 (TID 1226). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 590.0 (TID 1225) in 5 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 590.0 (TID 1226) in 6 ms on localhost (3/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 590.0 (TID 1227). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 590.0 (TID 1227) in 7 ms on localhost (4/4) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 590 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 590.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 403 finished: reduce at LDAOptimizer.scala:453, took 0.018306 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 404 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 591(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 591 (MapPartitionsRDD[610] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1268638, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_540 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8450) called with curMem=1288934, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_540_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.3 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_540_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 540 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 591 (MapPartitionsRDD[610] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 591.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 591.0 (TID 1228, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 591.0 (TID 1229, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 591.0 (TID 1230, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 591.0 (TID 1231, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 591.0 (TID 1230) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 591.0 (TID 1229) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 591.0 (TID 1228) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 591.0 (TID 1230). 1446 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 591.0 (TID 1231) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 591.0 (TID 1228). 1002 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 591.0 (TID 1229). 1314 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 591.0 (TID 1230) in 5 ms on localhost (1/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 591.0 (TID 1228) in 7 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 591.0 (TID 1229) in 6 ms on localhost (3/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 591.0 (TID 1231). 1446 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 591.0 (TID 1231) in 7 ms on localhost (4/4) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 591 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 404 finished: reduce at LDAOptimizer.scala:456, took 0.016588 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 591.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 405 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 592(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 592 (PartitionwiseSampledRDD[611] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1297384, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_541 stored as values in memory (estimated size 18.4 KB, free 729.3 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7664) called with curMem=1316200, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_541_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.3 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_541_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 541 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 592 (PartitionwiseSampledRDD[611] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 592.0 with 1 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 592.0 (TID 1232, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 592.0 (TID 1232) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 592.0 (TID 1232). 911 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 592.0 (TID 1232) in 2 ms on localhost (1/1) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 592 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 592.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 405 finished: isEmpty at LDAOptimizer.scala:418, took 0.007865 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 406 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 593(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 593 (PartitionwiseSampledRDD[611] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1323864, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_542 stored as values in memory (estimated size 18.4 KB, free 729.3 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7664) called with curMem=1342680, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_542_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.3 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_542_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 542 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 593 (PartitionwiseSampledRDD[611] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 593.0 with 3 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 593.0 (TID 1233, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 593.0 (TID 1234, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 593.0 (TID 1235, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 593.0 (TID 1234) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 593.0 (TID 1235) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 593.0 (TID 1233) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 593.0 (TID 1234). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 593.0 (TID 1234) in 3 ms on localhost (1/3) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 593.0 (TID 1233). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 593.0 (TID 1235). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 593.0 (TID 1233) in 4 ms on localhost (2/3) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 593.0 (TID 1235) in 4 ms on localhost (3/3) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 593 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 593.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 406 finished: isEmpty at LDAOptimizer.scala:418, took 0.013376 s -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1350344, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_543 stored as values in memory (estimated size 176.0 B, free 729.3 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1350520, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_543_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.3 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_543_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 543 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 407 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 594(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 594 (MapPartitionsRDD[613] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1350800, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_544 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8434) called with curMem=1371056, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_544_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.3 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_544_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 544 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 594 (MapPartitionsRDD[613] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 594.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 594.0 (TID 1236, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 594.0 (TID 1237, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 594.0 (TID 1238, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 594.0 (TID 1239, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 594.0 (TID 1236) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 594.0 (TID 1237) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 594.0 (TID 1239) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 594.0 (TID 1238) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 594.0 (TID 1236). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 594.0 (TID 1239). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 594.0 (TID 1237). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 594.0 (TID 1236) in 5 ms on localhost (1/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 594.0 (TID 1239) in 5 ms on localhost (2/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 594.0 (TID 1238). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 594.0 (TID 1237) in 6 ms on localhost (3/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 594.0 (TID 1238) in 7 ms on localhost (4/4) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 594 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 594.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 407 finished: reduce at LDAOptimizer.scala:453, took 0.015155 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 408 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 595(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 595 (MapPartitionsRDD[614] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1379490, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_545 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8445) called with curMem=1399786, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_545_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.2 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_545_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 545 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 595 (MapPartitionsRDD[614] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 595.0 with 4 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 595.0 (TID 1240, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 595.0 (TID 1241, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 595.0 (TID 1242, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 595.0 (TID 1243, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 595.0 (TID 1241) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 595.0 (TID 1240) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 595.0 (TID 1243) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 595.0 (TID 1242) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 595.0 (TID 1241). 1314 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 595.0 (TID 1240). 1002 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 595.0 (TID 1242). 1358 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 595.0 (TID 1243). 1314 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 595.0 (TID 1241) in 5 ms on localhost (1/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 595.0 (TID 1240) in 6 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 595.0 (TID 1242) in 5 ms on localhost (3/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 595.0 (TID 1243) in 5 ms on localhost (4/4) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 595 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 595.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 408 finished: reduce at LDAOptimizer.scala:456, took 0.014334 s -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 409 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 596(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 596 (PartitionwiseSampledRDD[615] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1408231, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_546 stored as values in memory (estimated size 18.4 KB, free 729.2 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=1427047, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_546_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.2 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_546_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 546 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 596 (PartitionwiseSampledRDD[615] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 596.0 with 1 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 596.0 (TID 1244, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_483_piece0 on localhost:50000 in memory (size: 2.3 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 596.0 (TID 1244) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 596.0 (TID 1244). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_545_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 596.0 (TID 1244) in 6 ms on localhost (1/1) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 596.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 596 (isEmpty at LDAOptimizer.scala:418) finished in 0.007 s -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 409 finished: isEmpty at LDAOptimizer.scala:418, took 0.027345 s -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1399770, maxMem=766075207 -2016-06-13 19:45:23 INFO ContextCleaner:59 - Cleaned accumulator 429 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_547 stored as values in memory (estimated size 176.0 B, free 729.3 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1391512, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_547_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.3 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_544_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO ContextCleaner:59 - Cleaned accumulator 428 -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_547_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 547 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_543_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_542_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:23 INFO ContextCleaner:59 - Cleaned accumulator 427 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 410 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 597(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 597 (MapPartitionsRDD[617] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_541_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1336936, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_548 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:23 INFO ContextCleaner:59 - Cleaned accumulator 426 -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8417) called with curMem=1338376, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_548_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.3 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_548_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 548 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 597 (MapPartitionsRDD[617] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 597.0 with 4 tasks -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_540_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 597.0 (TID 1245, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO ContextCleaner:59 - Cleaned accumulator 425 -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 597.0 (TID 1246, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 597.0 (TID 1247, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 597.0 (TID 1248, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 597.0 (TID 1245) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 597.0 (TID 1248) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 597.0 (TID 1247) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_539_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 597.0 (TID 1246) -2016-06-13 19:45:23 INFO ContextCleaner:59 - Cleaned accumulator 424 -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 597.0 (TID 1247). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 597.0 (TID 1246). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 597.0 (TID 1245). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 597.0 (TID 1247) in 5 ms on localhost (1/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 597.0 (TID 1246) in 6 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 597.0 (TID 1245) in 6 ms on localhost (3/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 597.0 (TID 1248). 1265 bytes result sent to driver -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_538_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 597.0 (TID 1248) in 6 ms on localhost (4/4) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 597 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 597.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 410 finished: reduce at LDAOptimizer.scala:453, took 0.015198 s -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_537_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO ContextCleaner:59 - Cleaned accumulator 423 -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 411 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 598(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_536_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 598 (MapPartitionsRDD[618] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:23 INFO ContextCleaner:59 - Cleaned accumulator 422 -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1207199, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_549 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_535_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO ContextCleaner:59 - Cleaned accumulator 421 -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(8429) called with curMem=1227495, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_549_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_549_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 549 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_534_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 598 (MapPartitionsRDD[618] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 598.0 with 4 tasks -2016-06-13 19:45:23 INFO ContextCleaner:59 - Cleaned accumulator 420 -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 598.0 (TID 1249, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 598.0 (TID 1250, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 598.0 (TID 1251, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 3.0 in stage 598.0 (TID 1252, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 3.0 in stage 598.0 (TID 1252) -2016-06-13 19:45:23 INFO Executor:59 - Running task 2.0 in stage 598.0 (TID 1251) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_533_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 598.0 (TID 1249) -2016-06-13 19:45:23 INFO Executor:59 - Running task 1.0 in stage 598.0 (TID 1250) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 2.0 in stage 598.0 (TID 1251). 1002 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 598.0 (TID 1249). 1314 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 598.0 (TID 1251) in 4 ms on localhost (1/4) -2016-06-13 19:45:23 INFO Executor:59 - Finished task 3.0 in stage 598.0 (TID 1252). 1402 bytes result sent to driver -2016-06-13 19:45:23 INFO Executor:59 - Finished task 1.0 in stage 598.0 (TID 1250). 1002 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 598.0 (TID 1249) in 6 ms on localhost (2/4) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 3.0 in stage 598.0 (TID 1252) in 5 ms on localhost (3/4) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_532_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 598.0 (TID 1250) in 7 ms on localhost (4/4) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 598.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 598 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 411 finished: reduce at LDAOptimizer.scala:456, took 0.013622 s -2016-06-13 19:45:23 INFO ContextCleaner:59 - Cleaned accumulator 419 -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_531_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO ContextCleaner:59 - Cleaned accumulator 418 -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 412 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 599(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_530_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 599 (PartitionwiseSampledRDD[619] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:23 INFO ContextCleaner:59 - Cleaned accumulator 417 -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1122866, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_550 stored as values in memory (estimated size 18.4 KB, free 729.5 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=1141226, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_550_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.5 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_529_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_550_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 550 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 599 (PartitionwiseSampledRDD[619] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Adding task set 599.0 with 1 tasks -2016-06-13 19:45:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 599.0 (TID 1253, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:23 INFO Executor:59 - Running task 0.0 in stage 599.0 (TID 1253) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_528_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:23 INFO ContextCleaner:59 - Cleaned accumulator 416 -2016-06-13 19:45:23 INFO Executor:59 - Finished task 0.0 in stage 599.0 (TID 1253). 1226 bytes result sent to driver -2016-06-13 19:45:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 599.0 (TID 1253) in 5 ms on localhost (1/1) -2016-06-13 19:45:23 INFO DAGScheduler:59 - ResultStage 599 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 599.0, whose tasks have all completed, from pool -2016-06-13 19:45:23 INFO DAGScheduler:59 - Job 412 finished: isEmpty at LDAOptimizer.scala:418, took 0.010757 s -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_527_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1114748, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_551 stored as values in memory (estimated size 176.0 B, free 729.5 MB) -2016-06-13 19:45:23 INFO ContextCleaner:59 - Cleaned accumulator 415 -2016-06-13 19:45:23 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1096108, maxMem=766075207 -2016-06-13 19:45:23 INFO MemoryStore:59 - Block broadcast_551_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.5 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Added broadcast_551_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_526_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:23 INFO SparkContext:59 - Created broadcast 551 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:23 INFO ContextCleaner:59 - Cleaned accumulator 414 -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_525_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:23 INFO ContextCleaner:59 - Cleaned accumulator 413 -2016-06-13 19:45:23 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:23 INFO BlockManagerInfo:59 - Removed broadcast_524_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Got job 413 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:23 INFO DAGScheduler:59 - Final stage: ResultStage 600(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:23 INFO DAGScheduler:59 - Submitting ResultStage 600 (MapPartitionsRDD[621] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:23 INFO ContextCleaner:59 - Cleaned accumulator 412 -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1012291, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_552 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_523_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8423) called with curMem=1032267, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_552_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_552_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 552 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 600 (MapPartitionsRDD[621] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 600.0 with 4 tasks -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_522_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 600.0 (TID 1254, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 411 -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 600.0 (TID 1255, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 600.0 (TID 1256, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 600.0 (TID 1257, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 600.0 (TID 1256) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 600.0 (TID 1254) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 600.0 (TID 1255) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 600.0 (TID 1257) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_521_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 410 -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 600.0 (TID 1257). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 600.0 (TID 1256). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 600.0 (TID 1257) in 5 ms on localhost (1/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 600.0 (TID 1256) in 6 ms on localhost (2/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 600.0 (TID 1255). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_520_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 409 -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 600.0 (TID 1255) in 9 ms on localhost (3/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 600.0 (TID 1254). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 600.0 (TID 1254) in 11 ms on localhost (4/4) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 600.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 600 (reduce at LDAOptimizer.scala:453) finished in 0.011 s -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 413 finished: reduce at LDAOptimizer.scala:453, took 0.017099 s -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_519_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_518_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 408 -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 414 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 601(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 601 (MapPartitionsRDD[622] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_517_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=901089, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_553 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 407 -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8436) called with curMem=892698, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_553_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_516_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 406 -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_553_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 553 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 601 (MapPartitionsRDD[622] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 601.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 601.0 (TID 1258, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 601.0 (TID 1259, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_515_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 601.0 (TID 1260, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 601.0 (TID 1261, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 601.0 (TID 1260) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 601.0 (TID 1258) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 601.0 (TID 1259) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 601.0 (TID 1261) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 601.0 (TID 1261). 1002 bytes result sent to driver -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_514_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 601.0 (TID 1260). 1358 bytes result sent to driver -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 405 -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 601.0 (TID 1261) in 4 ms on localhost (1/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 601.0 (TID 1260) in 5 ms on localhost (2/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 601.0 (TID 1258). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 601.0 (TID 1259). 1446 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 601.0 (TID 1258) in 7 ms on localhost (3/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 601.0 (TID 1259) in 6 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 601 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 601.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 414 finished: reduce at LDAOptimizer.scala:456, took 0.013319 s -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_513_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 404 -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_512_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 403 -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 415 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 602(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_511_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 602 (PartitionwiseSampledRDD[623] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=816330, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_554 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_510_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=827486, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_554_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 402 -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_554_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 554 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 602 (PartitionwiseSampledRDD[623] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 602.0 with 1 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 602.0 (TID 1262, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 602.0 (TID 1262) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_509_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 401 -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 602.0 (TID 1262). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 602.0 (TID 1262) in 3 ms on localhost (1/1) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 602 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 602.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 415 finished: isEmpty at LDAOptimizer.scala:418, took 0.008274 s -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_508_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=758897, maxMem=766075207 -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 400 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_555 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=758617, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_555_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_507_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_555_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 555 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_506_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 399 -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_505_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 398 -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 416 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 603(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 603 (MapPartitionsRDD[625] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_504_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 397 -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=675003, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_556 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_503_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8435) called with curMem=694979, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_556_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_556_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 556 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 603 (MapPartitionsRDD[625] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 603.0 with 4 tasks -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_502_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 603.0 (TID 1263, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 603.0 (TID 1264, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 603.0 (TID 1265, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 396 -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 603.0 (TID 1266, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 603.0 (TID 1263) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 603.0 (TID 1266) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 603.0 (TID 1265) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 603.0 (TID 1264) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 603.0 (TID 1264). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 603.0 (TID 1264) in 4 ms on localhost (1/4) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_501_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 603.0 (TID 1265). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 395 -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 603.0 (TID 1263). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 603.0 (TID 1265) in 5 ms on localhost (2/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 603.0 (TID 1263) in 7 ms on localhost (3/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 603.0 (TID 1266). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 603.0 (TID 1266) in 7 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 603 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 603.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 416 finished: reduce at LDAOptimizer.scala:453, took 0.015870 s -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_500_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 394 -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_499_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 417 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 604(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 604 (MapPartitionsRDD[626] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_498_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 393 -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=592389, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_557 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8448) called with curMem=586208, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_557_piece0 stored as bytes in memory (estimated size 8.3 KB, free 730.0 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_497_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 392 -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_557_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 557 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 604 (MapPartitionsRDD[626] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 604.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 604.0 (TID 1267, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_496_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 604.0 (TID 1268, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 391 -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 604.0 (TID 1269, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 604.0 (TID 1270, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 604.0 (TID 1267) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 604.0 (TID 1268) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 604.0 (TID 1269) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 604.0 (TID 1270) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 604.0 (TID 1268). 1002 bytes result sent to driver -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_495_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 604.0 (TID 1268) in 4 ms on localhost (1/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 604.0 (TID 1269). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 604.0 (TID 1267). 1358 bytes result sent to driver -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 390 -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 604.0 (TID 1269) in 7 ms on localhost (2/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 604.0 (TID 1267) in 8 ms on localhost (3/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 604.0 (TID 1270). 1358 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 604.0 (TID 1270) in 8 ms on localhost (4/4) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 604.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 604 (reduce at LDAOptimizer.scala:456) finished in 0.009 s -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 417 finished: reduce at LDAOptimizer.scala:456, took 0.015503 s -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_494_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_493_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 389 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 418 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 605(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 605 (PartitionwiseSampledRDD[627] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=502641, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_558 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_492_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7663) called with curMem=502641, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_558_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 388 -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_558_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 558 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 605 (PartitionwiseSampledRDD[627] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 605.0 with 1 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 605.0 (TID 1271, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_491_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 605.0 (TID 1271) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 387 -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 605.0 (TID 1271). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 605.0 (TID 1271) in 3 ms on localhost (1/1) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 605.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 605 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 418 finished: isEmpty at LDAOptimizer.scala:418, took 0.012116 s -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=452867, maxMem=766075207 -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_490_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_559 stored as values in memory (estimated size 176.0 B, free 730.2 MB) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 386 -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=452587, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_559_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.2 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_489_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_559_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 559 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_488_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 419 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 606(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 606 (MapPartitionsRDD[629] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 385 -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=426388, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_560 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8435) called with curMem=420165, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_560_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_487_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 384 -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_560_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 560 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 606 (MapPartitionsRDD[629] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 606.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 606.0 (TID 1272, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 606.0 (TID 1273, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 606.0 (TID 1274, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 606.0 (TID 1275, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 606.0 (TID 1273) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 606.0 (TID 1274) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 606.0 (TID 1275) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 606.0 (TID 1272) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_486_piece0 on localhost:50000 in memory (size: 807.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 606.0 (TID 1273). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 606.0 (TID 1275). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 383 -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 606.0 (TID 1274). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_485_piece0 on localhost:50000 in memory (size: 757.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 606.0 (TID 1273) in 7 ms on localhost (1/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 606.0 (TID 1275) in 8 ms on localhost (2/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 606.0 (TID 1274) in 8 ms on localhost (3/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 606.0 (TID 1272). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 606.0 (TID 1272) in 10 ms on localhost (4/4) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 606.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 606 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 382 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 419 finished: reduce at LDAOptimizer.scala:453, took 0.018841 s -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_484_piece0 on localhost:50000 in memory (size: 1763.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 381 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 420 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 607(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 380 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 607 (MapPartitionsRDD[630] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned shuffle 19 -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=419993, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_561 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8449) called with curMem=434574, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_561_piece0 stored as bytes in memory (estimated size 8.3 KB, free 730.2 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_482_piece0 on localhost:50000 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 379 -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_561_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 561 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 607 (MapPartitionsRDD[630] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 607.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 607.0 (TID 1276, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 607.0 (TID 1277, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_481_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 607.0 (TID 1278, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 607.0 (TID 1279, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 607.0 (TID 1277) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 607.0 (TID 1278) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 607.0 (TID 1279) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 607.0 (TID 1277). 1402 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 607.0 (TID 1279). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 607.0 (TID 1278). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 607.0 (TID 1276) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 607.0 (TID 1279) in 8 ms on localhost (1/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 607.0 (TID 1278) in 10 ms on localhost (2/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 607.0 (TID 1276). 1402 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 607.0 (TID 1277) in 12 ms on localhost (3/4) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_480_piece0 on localhost:50000 in memory (size: 2033.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 607.0 (TID 1276) in 14 ms on localhost (4/4) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 607.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 607 (reduce at LDAOptimizer.scala:456) finished in 0.016 s -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 420 finished: reduce at LDAOptimizer.scala:456, took 0.029970 s -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 378 -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_479_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 421 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 608(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_478_piece0 on localhost:50000 in memory (size: 1417.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 377 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 608 (PartitionwiseSampledRDD[631] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=427645, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_562 stored as values in memory (estimated size 18.4 KB, free 730.2 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_477_piece0 on localhost:50000 in memory (size: 2032.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=446461, maxMem=766075207 -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 376 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_562_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_562_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 562 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 608 (PartitionwiseSampledRDD[631] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 608.0 with 1 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 608.0 (TID 1280, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_476_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 608.0 (TID 1280) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 608.0 (TID 1280). 911 bytes result sent to driver -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_475_piece0 on localhost:50000 in memory (size: 2028.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 608.0 (TID 1280) in 2 ms on localhost (1/1) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 608.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 608 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 375 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 421 finished: isEmpty at LDAOptimizer.scala:418, took 0.008779 s -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_474_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 422 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 609(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 609 (PartitionwiseSampledRDD[631] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_473_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=419082, maxMem=766075207 -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 374 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_563 stored as values in memory (estimated size 18.4 KB, free 730.2 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=429466, maxMem=766075207 -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_472_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_563_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:24 INFO ContextCleaner:59 - Cleaned accumulator 373 -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_563_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 563 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 609 (PartitionwiseSampledRDD[631] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 609.0 with 3 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 609.0 (TID 1281, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Removed broadcast_471_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 609.0 (TID 1282, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 609.0 (TID 1283, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 609.0 (TID 1283) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 609.0 (TID 1281) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 609.0 (TID 1282) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 609.0 (TID 1283). 911 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 609.0 (TID 1281). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 609.0 (TID 1283) in 2 ms on localhost (1/3) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 609.0 (TID 1281) in 4 ms on localhost (2/3) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 609.0 (TID 1282). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 609.0 (TID 1282) in 4 ms on localhost (3/3) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 609 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 609.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 422 finished: isEmpty at LDAOptimizer.scala:418, took 0.011702 s -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=416415, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_564 stored as values in memory (estimated size 176.0 B, free 730.2 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=416591, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_564_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.2 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_564_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 564 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 423 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 610(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 610 (MapPartitionsRDD[633] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=416871, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_565 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=437127, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_565_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_565_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 565 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 610 (MapPartitionsRDD[633] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 610.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 610.0 (TID 1284, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 610.0 (TID 1285, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 610.0 (TID 1286, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 610.0 (TID 1287, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 610.0 (TID 1284) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 610.0 (TID 1286) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 610.0 (TID 1285) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 610.0 (TID 1287) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 610.0 (TID 1286). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 610.0 (TID 1285). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 610.0 (TID 1285) in 6 ms on localhost (1/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 610.0 (TID 1286) in 6 ms on localhost (2/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 610.0 (TID 1284). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 610.0 (TID 1287). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 610.0 (TID 1284) in 7 ms on localhost (3/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 610.0 (TID 1287) in 7 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 610 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 610.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 423 finished: reduce at LDAOptimizer.scala:453, took 0.014023 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 424 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 611(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 611 (MapPartitionsRDD[634] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=445559, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_566 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8445) called with curMem=465855, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_566_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_566_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 566 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 611 (MapPartitionsRDD[634] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 611.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 611.0 (TID 1288, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 611.0 (TID 1289, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 611.0 (TID 1290, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 611.0 (TID 1291, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 611.0 (TID 1288) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 611.0 (TID 1290) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 611.0 (TID 1291) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 611.0 (TID 1289) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 611.0 (TID 1290). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 611.0 (TID 1291). 1002 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 611.0 (TID 1289). 1358 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 611.0 (TID 1288). 1002 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 611.0 (TID 1290) in 7 ms on localhost (1/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 611.0 (TID 1288) in 8 ms on localhost (2/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 611.0 (TID 1291) in 7 ms on localhost (3/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 611.0 (TID 1289) in 8 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 611 (reduce at LDAOptimizer.scala:456) finished in 0.009 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 611.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 424 finished: reduce at LDAOptimizer.scala:456, took 0.014404 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 425 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 612(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 612 (PartitionwiseSampledRDD[635] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=474300, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_567 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=493116, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_567_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_567_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 567 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 612 (PartitionwiseSampledRDD[635] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 612.0 with 1 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 612.0 (TID 1292, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 612.0 (TID 1292) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 612.0 (TID 1292). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 612.0 (TID 1292) in 3 ms on localhost (1/1) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 612 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 612.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 425 finished: isEmpty at LDAOptimizer.scala:418, took 0.009540 s -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=500777, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_568 stored as values in memory (estimated size 176.0 B, free 730.1 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=500953, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_568_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.1 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_568_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 568 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 426 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 613(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 613 (MapPartitionsRDD[637] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=501233, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_569 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8435) called with curMem=521489, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_569_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_569_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 569 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 613 (MapPartitionsRDD[637] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 613.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 613.0 (TID 1293, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 613.0 (TID 1294, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 613.0 (TID 1295, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 613.0 (TID 1296, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 613.0 (TID 1294) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 613.0 (TID 1296) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 613.0 (TID 1293) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 613.0 (TID 1295) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 613.0 (TID 1295). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 613.0 (TID 1295) in 3 ms on localhost (1/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 613.0 (TID 1296). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 613.0 (TID 1293). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 613.0 (TID 1293) in 5 ms on localhost (2/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 613.0 (TID 1296) in 4 ms on localhost (3/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 613.0 (TID 1294). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 613.0 (TID 1294) in 6 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 613 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 613.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 426 finished: reduce at LDAOptimizer.scala:453, took 0.013278 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 427 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 614(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 614 (MapPartitionsRDD[638] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=529924, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_570 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8448) called with curMem=550220, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_570_piece0 stored as bytes in memory (estimated size 8.3 KB, free 730.1 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_570_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 570 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 614 (MapPartitionsRDD[638] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 614.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 614.0 (TID 1297, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 614.0 (TID 1298, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 614.0 (TID 1299, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 614.0 (TID 1300, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 614.0 (TID 1298) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 614.0 (TID 1300) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 614.0 (TID 1297) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 614.0 (TID 1299) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 614.0 (TID 1299). 1002 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 614.0 (TID 1297). 1358 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 614.0 (TID 1300). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 614.0 (TID 1298). 1490 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 614.0 (TID 1299) in 5 ms on localhost (1/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 614.0 (TID 1297) in 6 ms on localhost (2/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 614.0 (TID 1300) in 6 ms on localhost (3/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 614.0 (TID 1298) in 8 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 614 (reduce at LDAOptimizer.scala:456) finished in 0.009 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 614.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 427 finished: reduce at LDAOptimizer.scala:456, took 0.016351 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 428 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 615(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 615 (PartitionwiseSampledRDD[639] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=558668, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_571 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=577484, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_571_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_571_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 571 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 615 (PartitionwiseSampledRDD[639] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 615.0 with 1 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 615.0 (TID 1301, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 615.0 (TID 1301) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 615.0 (TID 1301). 911 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 615.0 (TID 1301) in 3 ms on localhost (1/1) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 615 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 615.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 428 finished: isEmpty at LDAOptimizer.scala:418, took 0.007867 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 429 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 616(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 616 (PartitionwiseSampledRDD[639] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=585146, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_572 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=603962, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_572_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_572_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 572 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 616 (PartitionwiseSampledRDD[639] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 616.0 with 3 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 616.0 (TID 1302, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 616.0 (TID 1303, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 616.0 (TID 1304, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 616.0 (TID 1303) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 616.0 (TID 1304) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 616.0 (TID 1302) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 616.0 (TID 1304). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 616.0 (TID 1303). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 616.0 (TID 1302). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 616.0 (TID 1304) in 3 ms on localhost (1/3) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 616.0 (TID 1303) in 3 ms on localhost (2/3) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 616.0 (TID 1302) in 4 ms on localhost (3/3) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 616 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 616.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 429 finished: isEmpty at LDAOptimizer.scala:418, took 0.008765 s -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=611624, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_573 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=611800, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_573_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_573_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 573 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 430 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 617(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 617 (MapPartitionsRDD[641] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=612080, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_574 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8437) called with curMem=632336, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_574_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_574_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 574 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 617 (MapPartitionsRDD[641] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 617.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 617.0 (TID 1305, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 617.0 (TID 1306, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 617.0 (TID 1307, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 617.0 (TID 1308, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 617.0 (TID 1305) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 617.0 (TID 1308) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 617.0 (TID 1307) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 617.0 (TID 1306) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 617.0 (TID 1305). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 617.0 (TID 1307). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 617.0 (TID 1306). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 617.0 (TID 1305) in 4 ms on localhost (1/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 617.0 (TID 1308). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 617.0 (TID 1307) in 4 ms on localhost (2/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 617.0 (TID 1306) in 4 ms on localhost (3/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 617.0 (TID 1308) in 4 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 617 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 617.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 430 finished: reduce at LDAOptimizer.scala:453, took 0.011691 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 431 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 618(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 618 (MapPartitionsRDD[642] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=640773, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_575 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8450) called with curMem=661069, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_575_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.9 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_575_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 575 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 618 (MapPartitionsRDD[642] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 618.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 618.0 (TID 1309, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 618.0 (TID 1310, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 618.0 (TID 1311, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 618.0 (TID 1312, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 618.0 (TID 1309) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 618.0 (TID 1312) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 618.0 (TID 1311) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 618.0 (TID 1310) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 618.0 (TID 1309). 1002 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 618.0 (TID 1312). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 618.0 (TID 1309) in 4 ms on localhost (1/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 618.0 (TID 1312) in 4 ms on localhost (2/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 618.0 (TID 1310). 1358 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 618.0 (TID 1311). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 618.0 (TID 1310) in 6 ms on localhost (3/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 618.0 (TID 1311) in 5 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 618 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 618.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 431 finished: reduce at LDAOptimizer.scala:456, took 0.011322 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 432 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 619(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 619 (PartitionwiseSampledRDD[643] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=669519, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_576 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=688335, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_576_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_576_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 576 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 619 (PartitionwiseSampledRDD[643] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 619.0 with 1 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 619.0 (TID 1313, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 619.0 (TID 1313) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 619.0 (TID 1313). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 619.0 (TID 1313) in 4 ms on localhost (1/1) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 619.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 619 (isEmpty at LDAOptimizer.scala:418) finished in 0.006 s -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 432 finished: isEmpty at LDAOptimizer.scala:418, took 0.011760 s -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=695997, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_577 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=696173, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_577_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_577_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 577 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 433 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 620(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 620 (MapPartitionsRDD[645] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=696453, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_578 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=716709, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_578_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_578_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 578 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 620 (MapPartitionsRDD[645] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 620.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 620.0 (TID 1314, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 620.0 (TID 1315, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 620.0 (TID 1316, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 620.0 (TID 1317, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 620.0 (TID 1314) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 620.0 (TID 1317) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 620.0 (TID 1316) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 620.0 (TID 1315) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 620.0 (TID 1316). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 620.0 (TID 1316) in 3 ms on localhost (1/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 620.0 (TID 1314). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 620.0 (TID 1314) in 5 ms on localhost (2/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 620.0 (TID 1315). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 620.0 (TID 1315) in 4 ms on localhost (3/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 620.0 (TID 1317). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 620.0 (TID 1317) in 5 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 620 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 620.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 433 finished: reduce at LDAOptimizer.scala:453, took 0.013158 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 434 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 621(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 621 (MapPartitionsRDD[646] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=725141, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_579 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8445) called with curMem=745437, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_579_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_579_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 579 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 621 (MapPartitionsRDD[646] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 621.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 621.0 (TID 1318, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 621.0 (TID 1319, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 621.0 (TID 1320, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 621.0 (TID 1321, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 621.0 (TID 1318) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 621.0 (TID 1320) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 621.0 (TID 1321) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 621.0 (TID 1319) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 621.0 (TID 1320). 1002 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 621.0 (TID 1320) in 3 ms on localhost (1/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 621.0 (TID 1321). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 621.0 (TID 1318). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 621.0 (TID 1321) in 5 ms on localhost (2/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 621.0 (TID 1318) in 6 ms on localhost (3/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 621.0 (TID 1319). 1446 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 621.0 (TID 1319) in 6 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 621 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 621.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 434 finished: reduce at LDAOptimizer.scala:456, took 0.012400 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 435 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 622(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 622 (PartitionwiseSampledRDD[647] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=753882, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_580 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=772698, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_580_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_580_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 580 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 622 (PartitionwiseSampledRDD[647] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 622.0 with 1 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 622.0 (TID 1322, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 622.0 (TID 1322) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 622.0 (TID 1322). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 622.0 (TID 1322) in 3 ms on localhost (1/1) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 622 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 622.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 435 finished: isEmpty at LDAOptimizer.scala:418, took 0.009164 s -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=780358, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_581 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=780534, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_581_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.8 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_581_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 581 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 436 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 623(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 623 (MapPartitionsRDD[649] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=780814, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_582 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=801070, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_582_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_582_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 582 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 623 (MapPartitionsRDD[649] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 623.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 623.0 (TID 1323, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 623.0 (TID 1324, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 623.0 (TID 1325, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 623.0 (TID 1326, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 623.0 (TID 1324) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 623.0 (TID 1325) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 623.0 (TID 1326) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 623.0 (TID 1323) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 623.0 (TID 1324). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 623.0 (TID 1325). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 623.0 (TID 1326). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 623.0 (TID 1323). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 623.0 (TID 1325) in 6 ms on localhost (1/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 623.0 (TID 1324) in 7 ms on localhost (2/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 623.0 (TID 1323) in 7 ms on localhost (3/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 623.0 (TID 1326) in 7 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 623 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 623.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 436 finished: reduce at LDAOptimizer.scala:453, took 0.015916 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 437 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 624(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 624 (MapPartitionsRDD[650] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=809503, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_583 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8446) called with curMem=829799, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_583_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_583_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 583 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 624 (MapPartitionsRDD[650] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 624.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 624.0 (TID 1327, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 624.0 (TID 1328, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 624.0 (TID 1329, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 624.0 (TID 1330, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 624.0 (TID 1327) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 624.0 (TID 1329) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 624.0 (TID 1328) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 624.0 (TID 1330) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 624.0 (TID 1328). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 624.0 (TID 1330). 1358 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 624.0 (TID 1329). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 624.0 (TID 1328) in 5 ms on localhost (1/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 624.0 (TID 1327). 1358 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 624.0 (TID 1329) in 5 ms on localhost (2/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 624.0 (TID 1330) in 4 ms on localhost (3/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 624.0 (TID 1327) in 6 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 624 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 624.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 437 finished: reduce at LDAOptimizer.scala:456, took 0.012536 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 438 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 625(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 625 (PartitionwiseSampledRDD[651] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=838245, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_584 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=857061, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_584_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_584_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 584 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 625 (PartitionwiseSampledRDD[651] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 625.0 with 1 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 625.0 (TID 1331, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 625.0 (TID 1331) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 625.0 (TID 1331). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 625.0 (TID 1331) in 3 ms on localhost (1/1) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 625 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 625.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 438 finished: isEmpty at LDAOptimizer.scala:418, took 0.007251 s -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=864722, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_585 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=864898, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_585_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.8 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_585_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 585 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 439 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 626(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 626 (MapPartitionsRDD[653] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=865178, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_586 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=885434, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_586_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_586_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 586 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 626 (MapPartitionsRDD[653] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 626.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 626.0 (TID 1332, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 626.0 (TID 1333, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 626.0 (TID 1334, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 626.0 (TID 1335, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 626.0 (TID 1332) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 626.0 (TID 1335) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 626.0 (TID 1333) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 626.0 (TID 1334) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 626.0 (TID 1332). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 626.0 (TID 1334). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 626.0 (TID 1335). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 626.0 (TID 1332) in 4 ms on localhost (1/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 626.0 (TID 1333). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 626.0 (TID 1334) in 5 ms on localhost (2/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 626.0 (TID 1335) in 4 ms on localhost (3/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 626.0 (TID 1333) in 6 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 626 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 626.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 439 finished: reduce at LDAOptimizer.scala:453, took 0.011547 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 440 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 627(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 627 (MapPartitionsRDD[654] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=893866, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_587 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8445) called with curMem=914162, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_587_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_587_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 587 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 627 (MapPartitionsRDD[654] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 627.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 627.0 (TID 1336, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 627.0 (TID 1337, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 627.0 (TID 1338, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 627.0 (TID 1339, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 627.0 (TID 1337) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 627.0 (TID 1339) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 627.0 (TID 1338) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 627.0 (TID 1336) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 627.0 (TID 1336). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 627.0 (TID 1339). 1402 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 627.0 (TID 1338). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 627.0 (TID 1337). 1358 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 627.0 (TID 1336) in 6 ms on localhost (1/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 627.0 (TID 1337) in 5 ms on localhost (2/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 627.0 (TID 1338) in 5 ms on localhost (3/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 627.0 (TID 1339) in 5 ms on localhost (4/4) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 627.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 627 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 440 finished: reduce at LDAOptimizer.scala:456, took 0.011513 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 441 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 628(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 628 (PartitionwiseSampledRDD[655] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=922607, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_588 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=941423, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_588_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_588_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 588 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 628 (PartitionwiseSampledRDD[655] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 628.0 with 1 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 628.0 (TID 1340, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 628.0 (TID 1340) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 628.0 (TID 1340). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 628.0 (TID 1340) in 3 ms on localhost (1/1) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 628 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 628.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 441 finished: isEmpty at LDAOptimizer.scala:418, took 0.010511 s -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=949084, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_589 stored as values in memory (estimated size 176.0 B, free 729.7 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=949260, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_589_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.7 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_589_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 589 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 442 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 629(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 629 (MapPartitionsRDD[657] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=949540, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_590 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8434) called with curMem=969796, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_590_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_590_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 590 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 629 (MapPartitionsRDD[657] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 629.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 629.0 (TID 1341, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 629.0 (TID 1342, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 629.0 (TID 1343, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 629.0 (TID 1344, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 629.0 (TID 1342) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 629.0 (TID 1344) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 629.0 (TID 1341) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 629.0 (TID 1343) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 629.0 (TID 1343). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 629.0 (TID 1342). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 629.0 (TID 1343) in 4 ms on localhost (1/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 629.0 (TID 1341). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 629.0 (TID 1344). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 629.0 (TID 1342) in 5 ms on localhost (2/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 629.0 (TID 1341) in 6 ms on localhost (3/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 629.0 (TID 1344) in 5 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 629 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 629.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 442 finished: reduce at LDAOptimizer.scala:453, took 0.012660 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 443 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 630(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 630 (MapPartitionsRDD[658] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=978230, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_591 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8447) called with curMem=998526, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_591_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_591_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 591 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 630 (MapPartitionsRDD[658] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 630.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 630.0 (TID 1345, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 630.0 (TID 1346, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 630.0 (TID 1347, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 630.0 (TID 1348, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 630.0 (TID 1345) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 630.0 (TID 1348) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 630.0 (TID 1347) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 630.0 (TID 1346) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 630.0 (TID 1347). 1002 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 630.0 (TID 1345). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 630.0 (TID 1348). 1402 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 630.0 (TID 1346). 1358 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 630.0 (TID 1347) in 4 ms on localhost (1/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 630.0 (TID 1345) in 5 ms on localhost (2/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 630.0 (TID 1348) in 5 ms on localhost (3/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 630.0 (TID 1346) in 5 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 630 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 630.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 443 finished: reduce at LDAOptimizer.scala:456, took 0.012347 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 444 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 631(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 631 (PartitionwiseSampledRDD[659] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1006973, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_592 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=1025789, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_592_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_592_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 592 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 631 (PartitionwiseSampledRDD[659] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 631.0 with 1 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 631.0 (TID 1349, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 631.0 (TID 1349) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 631.0 (TID 1349). 911 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 631.0 (TID 1349) in 2 ms on localhost (1/1) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 631 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 631.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 444 finished: isEmpty at LDAOptimizer.scala:418, took 0.008140 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 445 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 632(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 632 (PartitionwiseSampledRDD[659] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1033450, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_593 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=1052266, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_593_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_593_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 593 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 632 (PartitionwiseSampledRDD[659] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 632.0 with 3 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 632.0 (TID 1350, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 632.0 (TID 1351, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 632.0 (TID 1352, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 632.0 (TID 1351) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 632.0 (TID 1352) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 632.0 (TID 1351). 911 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 632.0 (TID 1351) in 4 ms on localhost (1/3) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 632.0 (TID 1350) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 632.0 (TID 1352). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 632.0 (TID 1352) in 6 ms on localhost (2/3) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 632.0 (TID 1350). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 632.0 (TID 1350) in 8 ms on localhost (3/3) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 632 (isEmpty at LDAOptimizer.scala:418) finished in 0.008 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 632.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 445 finished: isEmpty at LDAOptimizer.scala:418, took 0.012331 s -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1059927, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_594 stored as values in memory (estimated size 176.0 B, free 729.6 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1060103, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_594_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.6 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_594_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 594 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 446 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 633(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 633 (MapPartitionsRDD[661] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1060383, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_595 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8434) called with curMem=1080639, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_595_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_595_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 595 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 633 (MapPartitionsRDD[661] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 633.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 633.0 (TID 1353, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 633.0 (TID 1354, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 633.0 (TID 1355, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 633.0 (TID 1356, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 633.0 (TID 1354) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 633.0 (TID 1353) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 633.0 (TID 1355) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 633.0 (TID 1356) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 633.0 (TID 1353). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 633.0 (TID 1354). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 633.0 (TID 1353) in 6 ms on localhost (1/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 633.0 (TID 1354) in 7 ms on localhost (2/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 633.0 (TID 1355). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 633.0 (TID 1356). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 633.0 (TID 1355) in 7 ms on localhost (3/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 633.0 (TID 1356) in 7 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 633 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 633.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 446 finished: reduce at LDAOptimizer.scala:453, took 0.015436 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 447 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 634(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 634 (MapPartitionsRDD[662] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1089073, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_596 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8447) called with curMem=1109369, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_596_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_596_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 596 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 634 (MapPartitionsRDD[662] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 634.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 634.0 (TID 1357, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 634.0 (TID 1358, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 634.0 (TID 1359, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 634.0 (TID 1360, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 634.0 (TID 1359) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 634.0 (TID 1360) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 634.0 (TID 1357) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 634.0 (TID 1358) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 634.0 (TID 1359). 1002 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 634.0 (TID 1357). 1002 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 634.0 (TID 1359) in 3 ms on localhost (1/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 634.0 (TID 1357) in 4 ms on localhost (2/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 634.0 (TID 1360). 1358 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 634.0 (TID 1358). 1490 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 634.0 (TID 1360) in 4 ms on localhost (3/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 634.0 (TID 1358) in 5 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 634 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 634.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 447 finished: reduce at LDAOptimizer.scala:456, took 0.011470 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 448 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 635(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 635 (PartitionwiseSampledRDD[663] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1117816, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_597 stored as values in memory (estimated size 18.4 KB, free 729.5 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=1136632, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_597_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.5 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_597_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 597 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 635 (PartitionwiseSampledRDD[663] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 635.0 with 1 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 635.0 (TID 1361, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 635.0 (TID 1361) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 635.0 (TID 1361). 911 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 635.0 (TID 1361) in 2 ms on localhost (1/1) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 635 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 635.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 448 finished: isEmpty at LDAOptimizer.scala:418, took 0.010670 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 449 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 636(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 636 (PartitionwiseSampledRDD[663] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1144293, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_598 stored as values in memory (estimated size 18.4 KB, free 729.5 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=1163109, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_598_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.5 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_598_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 598 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 636 (PartitionwiseSampledRDD[663] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 636.0 with 3 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 636.0 (TID 1362, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 636.0 (TID 1363, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 636.0 (TID 1364, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 636.0 (TID 1362) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 636.0 (TID 1364) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 636.0 (TID 1363) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 636.0 (TID 1362). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 636.0 (TID 1363). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 636.0 (TID 1364). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 636.0 (TID 1362) in 4 ms on localhost (1/3) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 636.0 (TID 1363) in 3 ms on localhost (2/3) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 636.0 (TID 1364) in 3 ms on localhost (3/3) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 636 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 636.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 449 finished: isEmpty at LDAOptimizer.scala:418, took 0.009176 s -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1170770, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_599 stored as values in memory (estimated size 176.0 B, free 729.5 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1170946, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_599_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.5 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_599_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 599 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 450 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 637(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 637 (MapPartitionsRDD[665] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1171226, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_600 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=1191482, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_600_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_600_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 600 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 637 (MapPartitionsRDD[665] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 637.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 637.0 (TID 1365, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 637.0 (TID 1366, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 637.0 (TID 1367, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 637.0 (TID 1368, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 637.0 (TID 1366) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 637.0 (TID 1368) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 637.0 (TID 1367) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 637.0 (TID 1365) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 637.0 (TID 1365). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 637.0 (TID 1367). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 637.0 (TID 1368). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 637.0 (TID 1365) in 5 ms on localhost (1/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 637.0 (TID 1367) in 4 ms on localhost (2/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 637.0 (TID 1368) in 5 ms on localhost (3/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 637.0 (TID 1366). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 637.0 (TID 1366) in 7 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 637 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 637.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 450 finished: reduce at LDAOptimizer.scala:453, took 0.014367 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 451 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 638(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 638 (MapPartitionsRDD[666] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1199915, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_601 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8447) called with curMem=1220211, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_601_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_601_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 601 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 638 (MapPartitionsRDD[666] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 638.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 638.0 (TID 1369, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 638.0 (TID 1370, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 638.0 (TID 1371, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 638.0 (TID 1372, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 638.0 (TID 1370) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 638.0 (TID 1372) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 638.0 (TID 1369) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 638.0 (TID 1371) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 638.0 (TID 1369). 1002 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 638.0 (TID 1369) in 3 ms on localhost (1/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 638.0 (TID 1371). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 638.0 (TID 1371) in 4 ms on localhost (2/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 638.0 (TID 1370). 1358 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 638.0 (TID 1372). 1402 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 638.0 (TID 1370) in 5 ms on localhost (3/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 638.0 (TID 1372) in 5 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 638 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 638.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 451 finished: reduce at LDAOptimizer.scala:456, took 0.010529 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 452 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 639(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 639 (PartitionwiseSampledRDD[667] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1228658, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_602 stored as values in memory (estimated size 18.4 KB, free 729.4 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7663) called with curMem=1247474, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_602_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.4 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_602_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 602 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 639 (PartitionwiseSampledRDD[667] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 639.0 with 1 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 639.0 (TID 1373, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 639.0 (TID 1373) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 639.0 (TID 1373). 911 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 639.0 (TID 1373) in 3 ms on localhost (1/1) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 639 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 639.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 452 finished: isEmpty at LDAOptimizer.scala:418, took 0.007989 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 453 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 640(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 640 (PartitionwiseSampledRDD[667] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1255137, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_603 stored as values in memory (estimated size 18.4 KB, free 729.4 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7663) called with curMem=1273953, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_603_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.4 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_603_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 603 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 640 (PartitionwiseSampledRDD[667] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 640.0 with 3 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 640.0 (TID 1374, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 640.0 (TID 1375, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 640.0 (TID 1376, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 640.0 (TID 1375) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 640.0 (TID 1374) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 640.0 (TID 1376) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 640.0 (TID 1375). 911 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 640.0 (TID 1374). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 640.0 (TID 1376). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 640.0 (TID 1375) in 4 ms on localhost (1/3) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 640.0 (TID 1374) in 5 ms on localhost (2/3) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 640.0 (TID 1376) in 4 ms on localhost (3/3) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 640 (isEmpty at LDAOptimizer.scala:418) finished in 0.006 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 640.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 453 finished: isEmpty at LDAOptimizer.scala:418, took 0.011032 s -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1281616, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_604 stored as values in memory (estimated size 176.0 B, free 729.4 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1281792, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_604_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.4 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_604_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 604 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 454 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 641(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 641 (MapPartitionsRDD[669] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1282072, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_605 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8436) called with curMem=1302328, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_605_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.3 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_605_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 605 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 641 (MapPartitionsRDD[669] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 641.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 641.0 (TID 1377, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 641.0 (TID 1378, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 641.0 (TID 1379, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 641.0 (TID 1380, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 641.0 (TID 1379) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 641.0 (TID 1380) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 641.0 (TID 1377) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 641.0 (TID 1378) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 641.0 (TID 1377). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 641.0 (TID 1380). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 641.0 (TID 1379). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 641.0 (TID 1377) in 6 ms on localhost (1/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 641.0 (TID 1380) in 5 ms on localhost (2/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 641.0 (TID 1379) in 5 ms on localhost (3/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 641.0 (TID 1378). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 641.0 (TID 1378) in 6 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 641 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 641.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 454 finished: reduce at LDAOptimizer.scala:453, took 0.012612 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 455 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 642(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 642 (MapPartitionsRDD[670] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1310764, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_606 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8449) called with curMem=1331060, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_606_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.3 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_606_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 606 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 642 (MapPartitionsRDD[670] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 642.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 642.0 (TID 1381, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 642.0 (TID 1382, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 642.0 (TID 1383, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 642.0 (TID 1384, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 642.0 (TID 1382) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 642.0 (TID 1384) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 642.0 (TID 1383) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 642.0 (TID 1381) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 642.0 (TID 1381). 1002 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 642.0 (TID 1383). 1002 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 642.0 (TID 1382). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 642.0 (TID 1384). 1358 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 642.0 (TID 1381) in 4 ms on localhost (1/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 642.0 (TID 1383) in 5 ms on localhost (2/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 642.0 (TID 1382) in 5 ms on localhost (3/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 642 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 642.0 (TID 1384) in 4 ms on localhost (4/4) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 642.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 455 finished: reduce at LDAOptimizer.scala:456, took 0.010861 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 456 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 643(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 643 (PartitionwiseSampledRDD[671] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1339509, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_607 stored as values in memory (estimated size 18.4 KB, free 729.3 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7668) called with curMem=1358325, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_607_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.3 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_607_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 607 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 643 (PartitionwiseSampledRDD[671] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 643.0 with 1 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 643.0 (TID 1385, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 643.0 (TID 1385) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 643.0 (TID 1385). 911 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 643.0 (TID 1385) in 3 ms on localhost (1/1) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 643.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 643 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 456 finished: isEmpty at LDAOptimizer.scala:418, took 0.008109 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 457 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 644(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 644 (PartitionwiseSampledRDD[671] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1365993, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_608 stored as values in memory (estimated size 18.4 KB, free 729.3 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7668) called with curMem=1384809, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_608_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.3 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_608_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 608 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 644 (PartitionwiseSampledRDD[671] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 644.0 with 3 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 644.0 (TID 1386, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 644.0 (TID 1387, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 644.0 (TID 1388, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 644.0 (TID 1387) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 644.0 (TID 1386) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 644.0 (TID 1388) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 644.0 (TID 1387). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 644.0 (TID 1386). 911 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 644.0 (TID 1388). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 644.0 (TID 1387) in 2 ms on localhost (1/3) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 644.0 (TID 1386) in 3 ms on localhost (2/3) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 644.0 (TID 1388) in 3 ms on localhost (3/3) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 644 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 644.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 457 finished: isEmpty at LDAOptimizer.scala:418, took 0.008402 s -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1392477, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_609 stored as values in memory (estimated size 176.0 B, free 729.3 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1392653, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_609_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.3 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_609_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 609 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 458 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 645(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 645 (MapPartitionsRDD[673] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1392933, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_610 stored as values in memory (estimated size 19.8 KB, free 729.2 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8440) called with curMem=1413189, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_610_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.2 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_610_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 610 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 645 (MapPartitionsRDD[673] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 645.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 645.0 (TID 1389, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 645.0 (TID 1390, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 645.0 (TID 1391, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 645.0 (TID 1392, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 645.0 (TID 1390) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 645.0 (TID 1392) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 645.0 (TID 1391) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 645.0 (TID 1389) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 645.0 (TID 1389). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 645.0 (TID 1390). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 645.0 (TID 1389) in 3 ms on localhost (1/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 645.0 (TID 1391). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 645.0 (TID 1390) in 4 ms on localhost (2/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 645.0 (TID 1392). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 645.0 (TID 1391) in 4 ms on localhost (3/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 645.0 (TID 1392) in 5 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 645 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 645.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 458 finished: reduce at LDAOptimizer.scala:453, took 0.010132 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 459 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 646(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 646 (MapPartitionsRDD[674] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1421629, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_611 stored as values in memory (estimated size 19.8 KB, free 729.2 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8453) called with curMem=1441925, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_611_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.2 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_611_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 611 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 646 (MapPartitionsRDD[674] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 646.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 646.0 (TID 1393, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 646.0 (TID 1394, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 646.0 (TID 1395, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 646.0 (TID 1396, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 646.0 (TID 1395) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 646.0 (TID 1396) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 646.0 (TID 1393) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 646.0 (TID 1394) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 646.0 (TID 1394). 1002 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 646.0 (TID 1396). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 646.0 (TID 1393). 1002 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 646.0 (TID 1395). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 646.0 (TID 1396) in 4 ms on localhost (1/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 646.0 (TID 1394) in 4 ms on localhost (2/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 646.0 (TID 1395) in 4 ms on localhost (3/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 646.0 (TID 1393) in 5 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 646 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 646.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 459 finished: reduce at LDAOptimizer.scala:456, took 0.010341 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 460 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 647(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 647 (PartitionwiseSampledRDD[675] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1450378, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_612 stored as values in memory (estimated size 18.4 KB, free 729.2 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=1469194, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_612_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.2 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_612_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 612 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 647 (PartitionwiseSampledRDD[675] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 647.0 with 1 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 647.0 (TID 1397, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 647.0 (TID 1397) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 647.0 (TID 1397). 911 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 647.0 (TID 1397) in 3 ms on localhost (1/1) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 647 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 647.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 460 finished: isEmpty at LDAOptimizer.scala:418, took 0.007166 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 461 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 648(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 648 (PartitionwiseSampledRDD[675] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1476855, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_613 stored as values in memory (estimated size 18.4 KB, free 729.2 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=1495671, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_613_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.2 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_613_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 613 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 648 (PartitionwiseSampledRDD[675] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 648.0 with 3 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 648.0 (TID 1398, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 648.0 (TID 1399, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 648.0 (TID 1400, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 648.0 (TID 1398) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 648.0 (TID 1400) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 648.0 (TID 1399) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 648.0 (TID 1400). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 648.0 (TID 1399). 911 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 648.0 (TID 1398). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 648.0 (TID 1399) in 4 ms on localhost (1/3) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 648.0 (TID 1400) in 3 ms on localhost (2/3) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 648.0 (TID 1398) in 4 ms on localhost (3/3) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 648 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 648.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 461 finished: isEmpty at LDAOptimizer.scala:418, took 0.010323 s -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1503332, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_614 stored as values in memory (estimated size 176.0 B, free 729.2 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1503508, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_614_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.2 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_614_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 614 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 462 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 649(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 649 (MapPartitionsRDD[677] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1503788, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_615 stored as values in memory (estimated size 19.8 KB, free 729.1 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8434) called with curMem=1524044, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_615_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.1 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_615_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 615 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 649 (MapPartitionsRDD[677] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 649.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 649.0 (TID 1401, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 649.0 (TID 1402, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 649.0 (TID 1403, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 649.0 (TID 1404, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 649.0 (TID 1403) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 649.0 (TID 1401) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 649.0 (TID 1403). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 649.0 (TID 1401). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 649.0 (TID 1404) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 649.0 (TID 1402) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 649.0 (TID 1403) in 5 ms on localhost (1/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 649.0 (TID 1401) in 6 ms on localhost (2/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 649.0 (TID 1404). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 649.0 (TID 1402). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 649.0 (TID 1404) in 7 ms on localhost (3/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 649.0 (TID 1402) in 8 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 649 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 649.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 462 finished: reduce at LDAOptimizer.scala:453, took 0.013980 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 463 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 650(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 650 (MapPartitionsRDD[678] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1532478, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_616 stored as values in memory (estimated size 19.8 KB, free 729.1 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8447) called with curMem=1552774, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_616_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.1 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_616_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 616 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 650 (MapPartitionsRDD[678] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 650.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 650.0 (TID 1405, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 650.0 (TID 1406, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 650.0 (TID 1407, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 650.0 (TID 1408, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 650.0 (TID 1406) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 650.0 (TID 1407) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 650.0 (TID 1405) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 650.0 (TID 1408) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 650.0 (TID 1405). 1002 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 650.0 (TID 1406). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 650.0 (TID 1407). 1002 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 650.0 (TID 1408). 1314 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 650.0 (TID 1405) in 5 ms on localhost (1/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 650.0 (TID 1406) in 4 ms on localhost (2/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 650.0 (TID 1407) in 4 ms on localhost (3/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 650.0 (TID 1408) in 5 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 650 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 650.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 463 finished: reduce at LDAOptimizer.scala:456, took 0.011345 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 464 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 651(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 651 (PartitionwiseSampledRDD[679] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1561221, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_617 stored as values in memory (estimated size 18.4 KB, free 729.1 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=1580037, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_617_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.1 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_617_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 617 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 651 (PartitionwiseSampledRDD[679] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 651.0 with 1 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 651.0 (TID 1409, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 651.0 (TID 1409) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 651.0 (TID 1409). 1226 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 651.0 (TID 1409) in 3 ms on localhost (1/1) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 651 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 651.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 464 finished: isEmpty at LDAOptimizer.scala:418, took 0.009147 s -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1587697, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_618 stored as values in memory (estimated size 176.0 B, free 729.1 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1587873, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_618_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.1 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_618_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 618 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 465 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 652(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 652 (MapPartitionsRDD[681] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1588153, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_619 stored as values in memory (estimated size 19.8 KB, free 729.1 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=1608409, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_619_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.0 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_619_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 619 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 652 (MapPartitionsRDD[681] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 652.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 652.0 (TID 1410, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 652.0 (TID 1411, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 652.0 (TID 1412, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 652.0 (TID 1413, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 652.0 (TID 1412) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 652.0 (TID 1413) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 652.0 (TID 1411) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 652.0 (TID 1410) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 652.0 (TID 1412). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 652.0 (TID 1413). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 652.0 (TID 1410). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 652.0 (TID 1412) in 5 ms on localhost (1/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 652.0 (TID 1413) in 5 ms on localhost (2/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 652.0 (TID 1411). 1265 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 652.0 (TID 1410) in 6 ms on localhost (3/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 652.0 (TID 1411) in 7 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 652 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 652.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 465 finished: reduce at LDAOptimizer.scala:453, took 0.011530 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 466 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 653(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 653 (MapPartitionsRDD[682] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1616842, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_620 stored as values in memory (estimated size 19.8 KB, free 729.0 MB) -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(8446) called with curMem=1637138, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_620_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.0 MB) -2016-06-13 19:45:24 INFO BlockManagerInfo:59 - Added broadcast_620_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:24 INFO SparkContext:59 - Created broadcast 620 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 653 (MapPartitionsRDD[682] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Adding task set 653.0 with 4 tasks -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 653.0 (TID 1414, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 653.0 (TID 1415, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 653.0 (TID 1416, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Starting task 3.0 in stage 653.0 (TID 1417, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:24 INFO Executor:59 - Running task 0.0 in stage 653.0 (TID 1414) -2016-06-13 19:45:24 INFO Executor:59 - Running task 2.0 in stage 653.0 (TID 1416) -2016-06-13 19:45:24 INFO Executor:59 - Running task 1.0 in stage 653.0 (TID 1415) -2016-06-13 19:45:24 INFO Executor:59 - Running task 3.0 in stage 653.0 (TID 1417) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 2.0 in stage 653.0 (TID 1416). 1002 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 653.0 (TID 1416) in 4 ms on localhost (1/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 1.0 in stage 653.0 (TID 1415). 1402 bytes result sent to driver -2016-06-13 19:45:24 INFO Executor:59 - Finished task 0.0 in stage 653.0 (TID 1414). 1358 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 653.0 (TID 1415) in 6 ms on localhost (2/4) -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 653.0 (TID 1414) in 7 ms on localhost (3/4) -2016-06-13 19:45:24 INFO Executor:59 - Finished task 3.0 in stage 653.0 (TID 1417). 1402 bytes result sent to driver -2016-06-13 19:45:24 INFO TaskSetManager:59 - Finished task 3.0 in stage 653.0 (TID 1417) in 7 ms on localhost (4/4) -2016-06-13 19:45:24 INFO DAGScheduler:59 - ResultStage 653 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 653.0, whose tasks have all completed, from pool -2016-06-13 19:45:24 INFO DAGScheduler:59 - Job 466 finished: reduce at LDAOptimizer.scala:456, took 0.014365 s -2016-06-13 19:45:24 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:24 INFO DAGScheduler:59 - Got job 467 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:24 INFO DAGScheduler:59 - Final stage: ResultStage 654(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:24 INFO DAGScheduler:59 - Submitting ResultStage 654 (PartitionwiseSampledRDD[683] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:24 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1645584, maxMem=766075207 -2016-06-13 19:45:24 INFO MemoryStore:59 - Block broadcast_621 stored as values in memory (estimated size 18.4 KB, free 729.0 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=1664400, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_621_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.0 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_621_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 621 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 654 (PartitionwiseSampledRDD[683] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 654.0 with 1 tasks -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_620_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 654.0 (TID 1418, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 654.0 (TID 1418) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 487 -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 654.0 (TID 1418). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_619_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 654.0 (TID 1418) in 4 ms on localhost (1/1) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 654.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 654 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 467 finished: isEmpty at LDAOptimizer.scala:418, took 0.024744 s -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 486 -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1614349, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_622 stored as values in memory (estimated size 176.0 B, free 729.0 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_618_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1614349, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_622_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.0 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_622_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 622 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_617_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 485 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_616_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 484 -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 468 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 655(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_615_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 655 (MapPartitionsRDD[685] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 483 -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1530720, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_623 stored as values in memory (estimated size 19.8 KB, free 729.1 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_614_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8436) called with curMem=1550520, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_623_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.1 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_623_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 623 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_613_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 655 (MapPartitionsRDD[685] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 655.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 655.0 (TID 1419, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 482 -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 655.0 (TID 1420, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 655.0 (TID 1421, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 655.0 (TID 1422, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 655.0 (TID 1419) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 655.0 (TID 1421) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 655.0 (TID 1420) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 655.0 (TID 1422) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_612_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 481 -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 655.0 (TID 1419). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 655.0 (TID 1421). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 655.0 (TID 1419) in 5 ms on localhost (1/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 655.0 (TID 1420). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 655.0 (TID 1421) in 5 ms on localhost (2/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 655.0 (TID 1422). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_611_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 655.0 (TID 1420) in 7 ms on localhost (3/4) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 480 -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 655.0 (TID 1422) in 6 ms on localhost (4/4) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 655.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 655 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 468 finished: reduce at LDAOptimizer.scala:453, took 0.015869 s -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_610_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 479 -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_609_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 469 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 656(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 656 (MapPartitionsRDD[686] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1440433, maxMem=766075207 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_608_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_624 stored as values in memory (estimated size 19.8 KB, free 729.2 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 478 -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8449) called with curMem=1441913, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_624_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.2 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_624_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 624 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_607_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 656 (MapPartitionsRDD[686] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 656.0 with 4 tasks -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 477 -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 656.0 (TID 1423, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 656.0 (TID 1424, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 656.0 (TID 1425, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 656.0 (TID 1426, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 656.0 (TID 1424) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 656.0 (TID 1425) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 656.0 (TID 1423) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 656.0 (TID 1426) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_606_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 476 -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 656.0 (TID 1426). 1002 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 656.0 (TID 1423). 1314 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 656.0 (TID 1424). 1402 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 656.0 (TID 1426) in 4 ms on localhost (1/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 656.0 (TID 1425). 1002 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 656.0 (TID 1423) in 5 ms on localhost (2/4) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_605_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 656.0 (TID 1425) in 5 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 656.0 (TID 1424) in 6 ms on localhost (4/4) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 475 -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 656 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 656.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 469 finished: reduce at LDAOptimizer.scala:456, took 0.012305 s -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_604_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_603_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 474 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 470 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 657(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 657 (PartitionwiseSampledRDD[687] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1331843, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_625 stored as values in memory (estimated size 18.4 KB, free 729.3 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_602_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=1350659, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_625_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.3 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_625_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 625 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 473 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 657 (PartitionwiseSampledRDD[687] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 657.0 with 1 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 657.0 (TID 1427, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 657.0 (TID 1427) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_601_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 472 -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 657.0 (TID 1427). 911 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 657.0 (TID 1427) in 4 ms on localhost (1/1) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 657.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 657 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 470 finished: isEmpty at LDAOptimizer.scala:418, took 0.011973 s -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_600_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 471 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_599_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 471 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 658(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 658 (PartitionwiseSampledRDD[687] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_598_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 470 -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1255139, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_626 stored as values in memory (estimated size 18.4 KB, free 729.4 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=1266294, maxMem=766075207 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_597_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_626_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.4 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 469 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_626_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 626 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 658 (PartitionwiseSampledRDD[687] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 658.0 with 3 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 658.0 (TID 1428, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 658.0 (TID 1429, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_596_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 658.0 (TID 1430, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 658.0 (TID 1428) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 658.0 (TID 1430) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 658.0 (TID 1429) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 468 -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 658.0 (TID 1429). 911 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 658.0 (TID 1430). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 658.0 (TID 1429) in 3 ms on localhost (1/3) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 658.0 (TID 1428). 911 bytes result sent to driver -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_595_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 658.0 (TID 1428) in 5 ms on localhost (2/3) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 658.0 (TID 1430) in 4 ms on localhost (3/3) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 658.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 658 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 471 finished: isEmpty at LDAOptimizer.scala:418, took 0.009997 s -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 467 -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1197426, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_627 stored as values in memory (estimated size 176.0 B, free 729.4 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_594_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1197426, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_627_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.4 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_627_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 627 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_593_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 466 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_592_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 465 -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_591_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 472 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 659(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 464 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 659 (MapPartitionsRDD[689] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1107575, maxMem=766075207 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_590_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_628 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 463 -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8436) called with curMem=1107399, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_628_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_628_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 628 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_589_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 659 (MapPartitionsRDD[689] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 659.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 659.0 (TID 1431, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 659.0 (TID 1432, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 659.0 (TID 1433, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_588_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 659.0 (TID 1434, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 659.0 (TID 1431) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 659.0 (TID 1434) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 659.0 (TID 1433) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 462 -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 659.0 (TID 1433). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 659.0 (TID 1431). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_587_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 659.0 (TID 1433) in 5 ms on localhost (1/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 659.0 (TID 1434). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 659.0 (TID 1431) in 7 ms on localhost (2/4) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 461 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_586_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 460 -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 659.0 (TID 1432) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_585_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 659.0 (TID 1432). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 659.0 (TID 1434) in 13 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 659.0 (TID 1432) in 14 ms on localhost (4/4) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 659.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 659 (reduce at LDAOptimizer.scala:453) finished in 0.015 s -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 472 finished: reduce at LDAOptimizer.scala:453, took 0.022202 s -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_584_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 459 -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_583_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 473 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 660(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 660 (MapPartitionsRDD[690] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 458 -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=955718, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_629 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_582_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8448) called with curMem=967581, maxMem=766075207 -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 457 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_629_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.7 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_629_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 629 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 660 (MapPartitionsRDD[690] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 660.0 with 4 tasks -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_581_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 660.0 (TID 1435, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 660.0 (TID 1436, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 660.0 (TID 1437, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 660.0 (TID 1438, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 660.0 (TID 1436) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 660.0 (TID 1438) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 660.0 (TID 1435) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 660.0 (TID 1437) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_580_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 456 -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 660.0 (TID 1436). 1002 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 660.0 (TID 1435). 1002 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 660.0 (TID 1436) in 4 ms on localhost (1/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 660.0 (TID 1437). 1002 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 660.0 (TID 1438). 1358 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 660.0 (TID 1435) in 5 ms on localhost (2/4) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_579_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 660.0 (TID 1437) in 5 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 660.0 (TID 1438) in 5 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 660 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 660.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 473 finished: reduce at LDAOptimizer.scala:456, took 0.012010 s -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 455 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_578_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 454 -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 474 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 661(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_577_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 661 (PartitionwiseSampledRDD[691] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=864734, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_630 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_576_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 453 -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=883550, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_630_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_630_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 630 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_575_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 661 (PartitionwiseSampledRDD[691] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 661.0 with 1 tasks -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 452 -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 661.0 (TID 1439, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 661.0 (TID 1439) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_574_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 661.0 (TID 1439). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 661.0 (TID 1439) in 2 ms on localhost (1/1) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 661 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 451 -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 661.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 474 finished: isEmpty at LDAOptimizer.scala:418, took 0.008797 s -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=833491, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_631 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_573_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=833491, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_631_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.8 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_631_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 631 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_572_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 450 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_571_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 449 -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 475 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 662(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_570_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 662 (MapPartitionsRDD[693] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 448 -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=752071, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_632 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=763892, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_632_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_569_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_632_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 632 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 662 (MapPartitionsRDD[693] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 662.0 with 4 tasks -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 447 -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 662.0 (TID 1440, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 662.0 (TID 1441, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 662.0 (TID 1442, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 662.0 (TID 1443, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 662.0 (TID 1440) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 662.0 (TID 1442) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 662.0 (TID 1443) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 662.0 (TID 1441) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_568_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 662.0 (TID 1440). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 662.0 (TID 1440) in 6 ms on localhost (1/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 662.0 (TID 1442). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 662.0 (TID 1442) in 6 ms on localhost (2/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 662.0 (TID 1441). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 662.0 (TID 1443). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 662.0 (TID 1441) in 10 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 662.0 (TID 1443) in 9 ms on localhost (4/4) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 662.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_567_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 662 (reduce at LDAOptimizer.scala:453) finished in 0.013 s -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 475 finished: reduce at LDAOptimizer.scala:453, took 0.027879 s -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 446 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_566_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 476 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 663(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 445 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 663 (MapPartitionsRDD[694] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=676139, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_633 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_565_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8446) called with curMem=688003, maxMem=766075207 -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 444 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_633_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_633_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 633 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 663 (MapPartitionsRDD[694] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 663.0 with 4 tasks -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_564_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 663.0 (TID 1444, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 663.0 (TID 1445, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 663.0 (TID 1446, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 663.0 (TID 1447, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 663.0 (TID 1444) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 663.0 (TID 1446) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 663.0 (TID 1447) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 663.0 (TID 1445) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 663.0 (TID 1446). 1002 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 663.0 (TID 1446) in 4 ms on localhost (1/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 663.0 (TID 1447). 1358 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 663.0 (TID 1444). 1358 bytes result sent to driver -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_563_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 663.0 (TID 1445). 1314 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 663.0 (TID 1447) in 7 ms on localhost (2/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 663.0 (TID 1444) in 10 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 663.0 (TID 1445) in 11 ms on localhost (4/4) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 663.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 663 (reduce at LDAOptimizer.scala:456) finished in 0.012 s -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 476 finished: reduce at LDAOptimizer.scala:456, took 0.019304 s -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 443 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_562_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 477 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 664(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 442 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 664 (PartitionwiseSampledRDD[695] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=643039, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_634 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=653406, maxMem=766075207 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_561_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_634_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 441 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_634_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 634 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 664 (PartitionwiseSampledRDD[695] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 664.0 with 1 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 664.0 (TID 1448, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 664.0 (TID 1448) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_560_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 440 -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 664.0 (TID 1448). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 664.0 (TID 1448) in 3 ms on localhost (1/1) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 664 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 664.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 477 finished: isEmpty at LDAOptimizer.scala:418, took 0.011455 s -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_559_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=611623, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_635 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=604136, maxMem=766075207 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_558_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_635_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 439 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_635_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 635 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_557_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 438 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_556_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 437 -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 478 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 665(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 665 (MapPartitionsRDD[697] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_555_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=520047, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_636 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_554_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8431) called with curMem=521487, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_636_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 436 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_636_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 636 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 665 (MapPartitionsRDD[697] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 665.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 665.0 (TID 1449, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_553_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 665.0 (TID 1450, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 435 -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 665.0 (TID 1451, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 665.0 (TID 1452, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 665.0 (TID 1450) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 665.0 (TID 1452) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 665.0 (TID 1451) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 665.0 (TID 1449) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_552_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 434 -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 665.0 (TID 1451). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 665.0 (TID 1450). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 665.0 (TID 1449). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 665.0 (TID 1451) in 6 ms on localhost (1/4) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_551_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 665.0 (TID 1450) in 7 ms on localhost (2/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 665.0 (TID 1449) in 9 ms on localhost (3/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 665.0 (TID 1452). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 665.0 (TID 1452) in 8 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 665 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 665.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 478 finished: reduce at LDAOptimizer.scala:453, took 0.015358 s -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_550_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 433 -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 479 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_549_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 666(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 432 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 666 (MapPartitionsRDD[698] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=408433, maxMem=766075207 -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_548_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_637 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 431 -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8444) called with curMem=408473, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_637_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_637_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 637 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 666 (MapPartitionsRDD[698] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 666.0 with 4 tasks -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_547_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 666.0 (TID 1453, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 666.0 (TID 1454, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 666.0 (TID 1455, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 666.0 (TID 1456, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 666.0 (TID 1453) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 666.0 (TID 1455) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 666.0 (TID 1456) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 666.0 (TID 1454) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Removed broadcast_546_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:25 INFO ContextCleaner:59 - Cleaned accumulator 430 -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 666.0 (TID 1453). 1358 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 666.0 (TID 1454). 1314 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 666.0 (TID 1456). 1402 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 666.0 (TID 1454) in 5 ms on localhost (1/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 666.0 (TID 1453) in 7 ms on localhost (2/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 666.0 (TID 1456) in 6 ms on localhost (3/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 666.0 (TID 1455). 1446 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 666.0 (TID 1455) in 6 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 666 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 666.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 479 finished: reduce at LDAOptimizer.scala:456, took 0.015810 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 480 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 667(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 667 (PartitionwiseSampledRDD[699] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=389985, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_638 stored as values in memory (estimated size 18.4 KB, free 730.2 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=408801, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_638_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_638_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 638 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 667 (PartitionwiseSampledRDD[699] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 667.0 with 1 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 667.0 (TID 1457, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 667.0 (TID 1457) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 667.0 (TID 1457). 911 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 667.0 (TID 1457) in 3 ms on localhost (1/1) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 667.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 667 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 480 finished: isEmpty at LDAOptimizer.scala:418, took 0.008923 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 481 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 668(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 668 (PartitionwiseSampledRDD[699] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=416461, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_639 stored as values in memory (estimated size 18.4 KB, free 730.2 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=435277, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_639_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_639_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 639 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 668 (PartitionwiseSampledRDD[699] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 668.0 with 3 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 668.0 (TID 1458, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 668.0 (TID 1459, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 668.0 (TID 1460, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 668.0 (TID 1460) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 668.0 (TID 1459) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 668.0 (TID 1458) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 668.0 (TID 1460). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 668.0 (TID 1458). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 668.0 (TID 1459). 911 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 668.0 (TID 1460) in 3 ms on localhost (1/3) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 668.0 (TID 1459) in 3 ms on localhost (2/3) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 668.0 (TID 1458) in 3 ms on localhost (3/3) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 668 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 668.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 481 finished: isEmpty at LDAOptimizer.scala:418, took 0.008769 s -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=442937, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_640 stored as values in memory (estimated size 176.0 B, free 730.2 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=443113, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_640_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.2 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_640_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 640 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 482 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 669(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 669 (MapPartitionsRDD[701] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=443393, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_641 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8431) called with curMem=463649, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_641_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_641_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 641 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 669 (MapPartitionsRDD[701] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 669.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 669.0 (TID 1461, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 669.0 (TID 1462, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 669.0 (TID 1463, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 669.0 (TID 1464, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 669.0 (TID 1462) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 669.0 (TID 1464) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 669.0 (TID 1461) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 669.0 (TID 1463) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 669.0 (TID 1461). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 669.0 (TID 1463). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 669.0 (TID 1464). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 669.0 (TID 1461) in 4 ms on localhost (1/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 669.0 (TID 1463) in 4 ms on localhost (2/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 669.0 (TID 1462). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 669.0 (TID 1464) in 4 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 669.0 (TID 1462) in 5 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 669 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 669.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 482 finished: reduce at LDAOptimizer.scala:453, took 0.011367 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 483 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 670(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 670 (MapPartitionsRDD[702] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=472080, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_642 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8444) called with curMem=492376, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_642_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_642_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 642 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 670 (MapPartitionsRDD[702] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 670.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 670.0 (TID 1465, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 670.0 (TID 1466, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 670.0 (TID 1467, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 670.0 (TID 1468, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 670.0 (TID 1465) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 670.0 (TID 1467) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 670.0 (TID 1468) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 670.0 (TID 1466) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 670.0 (TID 1465). 1002 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 670.0 (TID 1467). 1002 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 670.0 (TID 1465) in 4 ms on localhost (1/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 670.0 (TID 1468). 1314 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 670.0 (TID 1467) in 4 ms on localhost (2/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 670.0 (TID 1468) in 5 ms on localhost (3/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 670.0 (TID 1466). 1314 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 670.0 (TID 1466) in 6 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 670 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 670.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 483 finished: reduce at LDAOptimizer.scala:456, took 0.011948 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 484 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 671(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 671 (PartitionwiseSampledRDD[703] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=500820, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_643 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=519636, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_643_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_643_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 643 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 671 (PartitionwiseSampledRDD[703] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 671.0 with 1 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 671.0 (TID 1469, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 671.0 (TID 1469) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 671.0 (TID 1469). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 671.0 (TID 1469) in 2 ms on localhost (1/1) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 671 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 671.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 484 finished: isEmpty at LDAOptimizer.scala:418, took 0.006603 s -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=527297, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_644 stored as values in memory (estimated size 176.0 B, free 730.1 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=527473, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_644_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.1 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_644_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 644 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 485 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 672(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 672 (MapPartitionsRDD[705] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=527753, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_645 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8434) called with curMem=548009, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_645_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_645_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 645 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 672 (MapPartitionsRDD[705] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 672.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 672.0 (TID 1470, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 672.0 (TID 1471, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 672.0 (TID 1472, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 672.0 (TID 1473, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 672.0 (TID 1470) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 672.0 (TID 1473) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 672.0 (TID 1471) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 672.0 (TID 1472) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 672.0 (TID 1472). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 672.0 (TID 1470). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 672.0 (TID 1471). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 672.0 (TID 1470) in 5 ms on localhost (1/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 672.0 (TID 1471) in 4 ms on localhost (2/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 672.0 (TID 1473). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 672.0 (TID 1472) in 5 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 672.0 (TID 1473) in 6 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 672 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 672.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 485 finished: reduce at LDAOptimizer.scala:453, took 0.011582 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 486 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 673(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 673 (MapPartitionsRDD[706] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=556443, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_646 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8446) called with curMem=576739, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_646_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_646_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 646 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 673 (MapPartitionsRDD[706] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 673.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 673.0 (TID 1474, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 673.0 (TID 1475, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 673.0 (TID 1476, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 673.0 (TID 1477, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 673.0 (TID 1476) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 673.0 (TID 1477) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 673.0 (TID 1474) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 673.0 (TID 1475) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 673.0 (TID 1476). 1314 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 673.0 (TID 1477). 1446 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 673.0 (TID 1474). 1402 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 673.0 (TID 1476) in 6 ms on localhost (1/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 673.0 (TID 1475). 1402 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 673.0 (TID 1477) in 5 ms on localhost (2/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 673.0 (TID 1474) in 7 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 673.0 (TID 1475) in 7 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 673 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 673.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 486 finished: reduce at LDAOptimizer.scala:456, took 0.012254 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 487 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 674(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 674 (PartitionwiseSampledRDD[707] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=585185, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_647 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=604001, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_647_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_647_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 647 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 674 (PartitionwiseSampledRDD[707] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 674.0 with 1 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 674.0 (TID 1478, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 674.0 (TID 1478) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 674.0 (TID 1478). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 674.0 (TID 1478) in 3 ms on localhost (1/1) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 674 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 674.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 487 finished: isEmpty at LDAOptimizer.scala:418, took 0.008601 s -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=611662, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_648 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=611838, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_648_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_648_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 648 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 488 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 675(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 675 (MapPartitionsRDD[709] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=612118, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_649 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8434) called with curMem=632374, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_649_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_649_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 649 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 675 (MapPartitionsRDD[709] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 675.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 675.0 (TID 1479, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 675.0 (TID 1480, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 675.0 (TID 1481, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 675.0 (TID 1482, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 675.0 (TID 1479) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 675.0 (TID 1480) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 675.0 (TID 1481) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 675.0 (TID 1482) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 675.0 (TID 1479). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 675.0 (TID 1482). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 675.0 (TID 1481). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 675.0 (TID 1479) in 6 ms on localhost (1/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 675.0 (TID 1482) in 5 ms on localhost (2/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 675.0 (TID 1480). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 675.0 (TID 1481) in 6 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 675.0 (TID 1480) in 8 ms on localhost (4/4) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 675.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 675 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 488 finished: reduce at LDAOptimizer.scala:453, took 0.018054 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 489 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 676(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 676 (MapPartitionsRDD[710] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=640808, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_650 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8447) called with curMem=661104, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_650_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_650_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 650 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 676 (MapPartitionsRDD[710] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 676.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 676.0 (TID 1483, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 676.0 (TID 1484, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 676.0 (TID 1485, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 676.0 (TID 1486, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 676.0 (TID 1484) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 676.0 (TID 1486) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 676.0 (TID 1485) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 676.0 (TID 1483) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 676.0 (TID 1485). 1314 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 676.0 (TID 1486). 1358 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 676.0 (TID 1485) in 5 ms on localhost (1/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 676.0 (TID 1486) in 5 ms on localhost (2/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 676.0 (TID 1484). 1358 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 676.0 (TID 1483). 1358 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 676.0 (TID 1484) in 7 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 676.0 (TID 1483) in 8 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 676 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 676.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 489 finished: reduce at LDAOptimizer.scala:456, took 0.013721 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 490 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 677(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 677 (PartitionwiseSampledRDD[711] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=669551, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_651 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7663) called with curMem=688367, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_651_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_651_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 651 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 677 (PartitionwiseSampledRDD[711] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 677.0 with 1 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 677.0 (TID 1487, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 677.0 (TID 1487) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 677.0 (TID 1487). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 677.0 (TID 1487) in 3 ms on localhost (1/1) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 677 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 677.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 490 finished: isEmpty at LDAOptimizer.scala:418, took 0.007401 s -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=696030, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_652 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=696206, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_652_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_652_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 652 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 491 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 678(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 678 (MapPartitionsRDD[713] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=696486, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_653 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8437) called with curMem=716742, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_653_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_653_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 653 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 678 (MapPartitionsRDD[713] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 678.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 678.0 (TID 1488, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 678.0 (TID 1489, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 678.0 (TID 1490, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 678.0 (TID 1491, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 678.0 (TID 1488) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 678.0 (TID 1491) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 678.0 (TID 1489) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 678.0 (TID 1490) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 678.0 (TID 1491). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 678.0 (TID 1489). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 678.0 (TID 1490). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 678.0 (TID 1488). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 678.0 (TID 1491) in 5 ms on localhost (1/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 678.0 (TID 1489) in 6 ms on localhost (2/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 678.0 (TID 1488) in 6 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 678.0 (TID 1490) in 7 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 678 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 678.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 491 finished: reduce at LDAOptimizer.scala:453, took 0.011556 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 492 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 679(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 679 (MapPartitionsRDD[714] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=725179, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_654 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8450) called with curMem=745475, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_654_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.9 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_654_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 654 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 679 (MapPartitionsRDD[714] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 679.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 679.0 (TID 1492, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 679.0 (TID 1493, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 679.0 (TID 1494, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 679.0 (TID 1495, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 679.0 (TID 1493) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 679.0 (TID 1495) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 679.0 (TID 1494) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 679.0 (TID 1492) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 679.0 (TID 1493). 1358 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 679.0 (TID 1492). 1314 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 679.0 (TID 1495). 1002 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 679.0 (TID 1494). 1314 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 679.0 (TID 1493) in 5 ms on localhost (1/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 679.0 (TID 1492) in 5 ms on localhost (2/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 679.0 (TID 1495) in 4 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 679.0 (TID 1494) in 5 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 679 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 679.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 492 finished: reduce at LDAOptimizer.scala:456, took 0.011379 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 493 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 680(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 680 (PartitionwiseSampledRDD[715] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=753925, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_655 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=772741, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_655_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_655_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 655 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 680 (PartitionwiseSampledRDD[715] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 680.0 with 1 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 680.0 (TID 1496, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 680.0 (TID 1496) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 680.0 (TID 1496). 911 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 680.0 (TID 1496) in 3 ms on localhost (1/1) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 680 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 680.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 493 finished: isEmpty at LDAOptimizer.scala:418, took 0.007588 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 494 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 681(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 681 (PartitionwiseSampledRDD[715] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=780402, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_656 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=799218, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_656_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_656_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 656 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 681 (PartitionwiseSampledRDD[715] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 681.0 with 3 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 681.0 (TID 1497, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 681.0 (TID 1498, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 681.0 (TID 1499, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 681.0 (TID 1497) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 681.0 (TID 1498) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 681.0 (TID 1499) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 681.0 (TID 1497). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 681.0 (TID 1498). 911 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 681.0 (TID 1499). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 681.0 (TID 1498) in 3 ms on localhost (1/3) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 681.0 (TID 1497) in 3 ms on localhost (2/3) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 681.0 (TID 1499) in 4 ms on localhost (3/3) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 681 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 681.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 494 finished: isEmpty at LDAOptimizer.scala:418, took 0.009131 s -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=806879, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_657 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=807055, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_657_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.8 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_657_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 657 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 495 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 682(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 682 (MapPartitionsRDD[717] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=807335, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_658 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8434) called with curMem=827591, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_658_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_658_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 658 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 682 (MapPartitionsRDD[717] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 682.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 682.0 (TID 1500, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 682.0 (TID 1501, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 682.0 (TID 1502, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 682.0 (TID 1503, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 682.0 (TID 1502) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 682.0 (TID 1503) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 682.0 (TID 1501) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 682.0 (TID 1500) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 682.0 (TID 1500). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 682.0 (TID 1502). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 682.0 (TID 1501). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 682.0 (TID 1502) in 3 ms on localhost (1/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 682.0 (TID 1500) in 4 ms on localhost (2/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 682.0 (TID 1503). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 682.0 (TID 1501) in 5 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 682.0 (TID 1503) in 4 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 682 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 682.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 495 finished: reduce at LDAOptimizer.scala:453, took 0.010270 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 496 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 683(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 683 (MapPartitionsRDD[718] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=836025, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_659 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8447) called with curMem=856321, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_659_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_659_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 659 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 683 (MapPartitionsRDD[718] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 683.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 683.0 (TID 1504, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 683.0 (TID 1505, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 683.0 (TID 1506, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 683.0 (TID 1507, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 683.0 (TID 1507) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 683.0 (TID 1506) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 683.0 (TID 1505) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 683.0 (TID 1504) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 683.0 (TID 1504). 1002 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 683.0 (TID 1506). 1002 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 683.0 (TID 1504) in 5 ms on localhost (1/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 683.0 (TID 1507). 1446 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 683.0 (TID 1506) in 4 ms on localhost (2/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 683.0 (TID 1507) in 5 ms on localhost (3/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 683.0 (TID 1505). 1402 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 683.0 (TID 1505) in 6 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 683 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 496 finished: reduce at LDAOptimizer.scala:456, took 0.012347 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 683.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 497 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 684(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 684 (PartitionwiseSampledRDD[719] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=864768, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_660 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=883584, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_660_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_660_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 660 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 684 (PartitionwiseSampledRDD[719] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 684.0 with 1 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 684.0 (TID 1508, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 684.0 (TID 1508) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 684.0 (TID 1508). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 684.0 (TID 1508) in 2 ms on localhost (1/1) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 684 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 684.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 497 finished: isEmpty at LDAOptimizer.scala:418, took 0.006642 s -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=891244, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_661 stored as values in memory (estimated size 176.0 B, free 729.7 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=891420, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_661_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.7 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_661_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 661 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 498 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 685(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 685 (MapPartitionsRDD[721] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=891700, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_662 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8431) called with curMem=911956, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_662_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_662_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 662 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 685 (MapPartitionsRDD[721] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 685.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 685.0 (TID 1509, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 685.0 (TID 1510, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 685.0 (TID 1511, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 685.0 (TID 1512, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 685.0 (TID 1510) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 685.0 (TID 1512) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 685.0 (TID 1511) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 685.0 (TID 1509) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 685.0 (TID 1510). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 685.0 (TID 1511). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 685.0 (TID 1510) in 5 ms on localhost (1/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 685.0 (TID 1511) in 4 ms on localhost (2/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 685.0 (TID 1509). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 685.0 (TID 1512). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 685.0 (TID 1509) in 7 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 685.0 (TID 1512) in 6 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 685 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 685.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 498 finished: reduce at LDAOptimizer.scala:453, took 0.012636 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 499 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 686(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 686 (MapPartitionsRDD[722] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=920387, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_663 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8444) called with curMem=940683, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_663_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_663_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 663 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 686 (MapPartitionsRDD[722] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 686.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 686.0 (TID 1513, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 686.0 (TID 1514, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 686.0 (TID 1515, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 686.0 (TID 1516, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 686.0 (TID 1513) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 686.0 (TID 1514) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 686.0 (TID 1516) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 686.0 (TID 1515) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 686.0 (TID 1514). 1002 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 686.0 (TID 1514) in 4 ms on localhost (1/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 686.0 (TID 1516). 1314 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 686.0 (TID 1516) in 4 ms on localhost (2/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 686.0 (TID 1515). 1358 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 686.0 (TID 1513). 1402 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 686.0 (TID 1515) in 5 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 686.0 (TID 1513) in 6 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 686 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 686.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 499 finished: reduce at LDAOptimizer.scala:456, took 0.012263 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 500 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 687(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 687 (PartitionwiseSampledRDD[723] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=949127, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_664 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7667) called with curMem=967943, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_664_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_664_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 664 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 687 (PartitionwiseSampledRDD[723] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 687.0 with 1 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 687.0 (TID 1517, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 687.0 (TID 1517) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 687.0 (TID 1517). 911 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 687.0 (TID 1517) in 2 ms on localhost (1/1) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 687 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 687.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 500 finished: isEmpty at LDAOptimizer.scala:418, took 0.007159 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 501 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 688(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 688 (PartitionwiseSampledRDD[723] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=975610, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_665 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7667) called with curMem=994426, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_665_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_665_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 665 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 688 (PartitionwiseSampledRDD[723] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 688.0 with 3 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 688.0 (TID 1518, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 688.0 (TID 1519, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 688.0 (TID 1520, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 688.0 (TID 1518) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 688.0 (TID 1520) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 688.0 (TID 1519) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 688.0 (TID 1518). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 688.0 (TID 1519). 911 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 688.0 (TID 1518) in 5 ms on localhost (1/3) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 688.0 (TID 1519) in 4 ms on localhost (2/3) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 688.0 (TID 1520). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 688.0 (TID 1520) in 5 ms on localhost (3/3) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 688 (isEmpty at LDAOptimizer.scala:418) finished in 0.006 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 688.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 501 finished: isEmpty at LDAOptimizer.scala:418, took 0.010399 s -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1002093, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_666 stored as values in memory (estimated size 176.0 B, free 729.6 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1002269, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_666_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.6 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_666_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 666 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 502 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 689(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 689 (MapPartitionsRDD[725] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1002549, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_667 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8440) called with curMem=1022805, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_667_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_667_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 667 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 689 (MapPartitionsRDD[725] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 689.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 689.0 (TID 1521, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 689.0 (TID 1522, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 689.0 (TID 1523, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 689.0 (TID 1524, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 689.0 (TID 1521) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 689.0 (TID 1524) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 689.0 (TID 1523) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 689.0 (TID 1522) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 689.0 (TID 1521). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 689.0 (TID 1521) in 4 ms on localhost (1/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 689.0 (TID 1522). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 689.0 (TID 1523). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 689.0 (TID 1522) in 7 ms on localhost (2/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 689.0 (TID 1523) in 7 ms on localhost (3/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 689.0 (TID 1524). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 689.0 (TID 1524) in 8 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 689 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 689.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 502 finished: reduce at LDAOptimizer.scala:453, took 0.014004 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 503 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 690(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 690 (MapPartitionsRDD[726] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1031245, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_668 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8453) called with curMem=1051541, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_668_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.6 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_668_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 668 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 690 (MapPartitionsRDD[726] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 690.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 690.0 (TID 1525, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 690.0 (TID 1526, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 690.0 (TID 1527, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 690.0 (TID 1528, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 690.0 (TID 1527) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 690.0 (TID 1528) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 690.0 (TID 1526) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 690.0 (TID 1525) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 690.0 (TID 1527). 1002 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 690.0 (TID 1525). 1002 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 690.0 (TID 1528). 1358 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 690.0 (TID 1526). 1314 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 690.0 (TID 1527) in 4 ms on localhost (1/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 690.0 (TID 1525) in 4 ms on localhost (2/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 690.0 (TID 1526) in 4 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 690.0 (TID 1528) in 4 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 690 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 690.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 503 finished: reduce at LDAOptimizer.scala:456, took 0.010723 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 504 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 691(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 691 (PartitionwiseSampledRDD[727] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1059994, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_669 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=1078810, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_669_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_669_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 669 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 691 (PartitionwiseSampledRDD[727] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 691.0 with 1 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 691.0 (TID 1529, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 691.0 (TID 1529) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 691.0 (TID 1529). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 691.0 (TID 1529) in 3 ms on localhost (1/1) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 691 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 691.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 504 finished: isEmpty at LDAOptimizer.scala:418, took 0.007007 s -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1086470, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_670 stored as values in memory (estimated size 176.0 B, free 729.5 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1086646, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_670_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.5 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_670_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 670 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 505 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 692(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 692 (MapPartitionsRDD[729] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1086926, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_671 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8431) called with curMem=1107182, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_671_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_671_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 671 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 692 (MapPartitionsRDD[729] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 692.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 692.0 (TID 1530, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 692.0 (TID 1531, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 692.0 (TID 1532, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 692.0 (TID 1533, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 692.0 (TID 1531) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 692.0 (TID 1533) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 692.0 (TID 1532) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 692.0 (TID 1530) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 692.0 (TID 1532). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 692.0 (TID 1533). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 692.0 (TID 1531). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 692.0 (TID 1532) in 3 ms on localhost (1/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 692.0 (TID 1533) in 4 ms on localhost (2/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 692.0 (TID 1531) in 4 ms on localhost (3/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 692.0 (TID 1530). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 692.0 (TID 1530) in 5 ms on localhost (4/4) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 692.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 692 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 505 finished: reduce at LDAOptimizer.scala:453, took 0.011669 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 506 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 693(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 693 (MapPartitionsRDD[730] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1115613, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_672 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8446) called with curMem=1135909, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_672_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_672_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 672 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 693 (MapPartitionsRDD[730] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 693.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 693.0 (TID 1534, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 693.0 (TID 1535, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 693.0 (TID 1536, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 693.0 (TID 1537, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 693.0 (TID 1536) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 693.0 (TID 1534) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 693.0 (TID 1535) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 693.0 (TID 1537) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 693.0 (TID 1536). 1002 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 693.0 (TID 1536) in 4 ms on localhost (1/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 693.0 (TID 1537). 1002 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 693.0 (TID 1534). 1402 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 693.0 (TID 1535). 1358 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 693.0 (TID 1537) in 6 ms on localhost (2/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 693.0 (TID 1534) in 7 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 693.0 (TID 1535) in 6 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 693 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 693.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 506 finished: reduce at LDAOptimizer.scala:456, took 0.012431 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 507 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 694(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 694 (PartitionwiseSampledRDD[731] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1144355, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_673 stored as values in memory (estimated size 18.4 KB, free 729.5 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=1163171, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_673_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.5 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_673_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 673 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 694 (PartitionwiseSampledRDD[731] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 694.0 with 1 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 694.0 (TID 1538, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 694.0 (TID 1538) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 694.0 (TID 1538). 911 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 694.0 (TID 1538) in 2 ms on localhost (1/1) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 694 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 694.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 507 finished: isEmpty at LDAOptimizer.scala:418, took 0.007382 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 508 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 695(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 695 (PartitionwiseSampledRDD[731] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1170831, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_674 stored as values in memory (estimated size 18.4 KB, free 729.5 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=1189647, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_674_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.4 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_674_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 674 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 695 (PartitionwiseSampledRDD[731] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 695.0 with 3 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 695.0 (TID 1539, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 695.0 (TID 1540, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 695.0 (TID 1541, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 695.0 (TID 1540) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 695.0 (TID 1541) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 695.0 (TID 1539) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 695.0 (TID 1540). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 695.0 (TID 1541). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 695.0 (TID 1539). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 695.0 (TID 1540) in 2 ms on localhost (1/3) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 695.0 (TID 1541) in 3 ms on localhost (2/3) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 695.0 (TID 1539) in 4 ms on localhost (3/3) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 695 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 695.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 508 finished: isEmpty at LDAOptimizer.scala:418, took 0.007991 s -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1197307, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_675 stored as values in memory (estimated size 176.0 B, free 729.4 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1197483, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_675_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.4 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_675_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 675 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 509 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 696(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 696 (MapPartitionsRDD[733] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1197763, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_676 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=1218019, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_676_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_676_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 676 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 696 (MapPartitionsRDD[733] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 696.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 696.0 (TID 1542, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 696.0 (TID 1543, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 696.0 (TID 1544, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 696.0 (TID 1545, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 696.0 (TID 1545) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 696.0 (TID 1543) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 696.0 (TID 1542) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 696.0 (TID 1544) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 696.0 (TID 1542). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 696.0 (TID 1544). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 696.0 (TID 1545). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 696.0 (TID 1543). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 696.0 (TID 1542) in 5 ms on localhost (1/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 696.0 (TID 1544) in 4 ms on localhost (2/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 696.0 (TID 1545) in 4 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 696.0 (TID 1543) in 4 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 696 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 696.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 509 finished: reduce at LDAOptimizer.scala:453, took 0.010212 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 510 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 697(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 697 (MapPartitionsRDD[734] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1226452, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_677 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8446) called with curMem=1246748, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_677_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_677_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 677 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 697 (MapPartitionsRDD[734] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 697.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 697.0 (TID 1546, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 697.0 (TID 1547, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 697.0 (TID 1548, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 697.0 (TID 1549, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 697.0 (TID 1547) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 697.0 (TID 1546) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 697.0 (TID 1546). 1002 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 697.0 (TID 1547). 1358 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 697.0 (TID 1549) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 697.0 (TID 1546) in 5 ms on localhost (1/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 697.0 (TID 1547) in 6 ms on localhost (2/4) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 697.0 (TID 1548) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 697.0 (TID 1549). 1314 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 697.0 (TID 1549) in 7 ms on localhost (3/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 697.0 (TID 1548). 1314 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 697.0 (TID 1548) in 9 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 697 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 697.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 510 finished: reduce at LDAOptimizer.scala:456, took 0.018212 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 511 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 698(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 698 (PartitionwiseSampledRDD[735] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1255194, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_678 stored as values in memory (estimated size 18.4 KB, free 729.4 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7663) called with curMem=1274010, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_678_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.4 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_678_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 678 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 698 (PartitionwiseSampledRDD[735] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 698.0 with 1 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 698.0 (TID 1550, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 698.0 (TID 1550) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 698.0 (TID 1550). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 698.0 (TID 1550) in 3 ms on localhost (1/1) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 698 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 698.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 511 finished: isEmpty at LDAOptimizer.scala:418, took 0.007863 s -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1281673, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_679 stored as values in memory (estimated size 176.0 B, free 729.4 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1281849, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_679_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.4 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_679_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 679 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 512 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 699(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 699 (MapPartitionsRDD[737] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1282129, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_680 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8436) called with curMem=1302385, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_680_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.3 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_680_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 680 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 699 (MapPartitionsRDD[737] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 699.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 699.0 (TID 1551, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 699.0 (TID 1552, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 699.0 (TID 1553, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 699.0 (TID 1554, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 699.0 (TID 1552) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 699.0 (TID 1554) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 699.0 (TID 1553) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 699.0 (TID 1551) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 699.0 (TID 1553). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 699.0 (TID 1554). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 699.0 (TID 1551). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 699.0 (TID 1552). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 699.0 (TID 1553) in 5 ms on localhost (1/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 699.0 (TID 1551) in 5 ms on localhost (2/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 699.0 (TID 1554) in 4 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 699.0 (TID 1552) in 5 ms on localhost (4/4) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 699.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 699 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 512 finished: reduce at LDAOptimizer.scala:453, took 0.011207 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 513 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 700(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 700 (MapPartitionsRDD[738] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1310821, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_681 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8449) called with curMem=1331117, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_681_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.3 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_681_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 681 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 700 (MapPartitionsRDD[738] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 700.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 700.0 (TID 1555, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 700.0 (TID 1556, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 700.0 (TID 1557, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 700.0 (TID 1558, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 700.0 (TID 1555) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 700.0 (TID 1557) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 700.0 (TID 1558) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 700.0 (TID 1556) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 700.0 (TID 1557). 1314 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 700.0 (TID 1558). 1314 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 700.0 (TID 1555). 1402 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 700.0 (TID 1557) in 4 ms on localhost (1/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 700.0 (TID 1558) in 5 ms on localhost (2/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 700.0 (TID 1556). 1358 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 700.0 (TID 1555) in 5 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 700.0 (TID 1556) in 5 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 700 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 700.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 513 finished: reduce at LDAOptimizer.scala:456, took 0.010512 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 514 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 701(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 701 (PartitionwiseSampledRDD[739] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1339566, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_682 stored as values in memory (estimated size 18.4 KB, free 729.3 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=1358382, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_682_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.3 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_682_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 682 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 701 (PartitionwiseSampledRDD[739] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 701.0 with 1 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 701.0 (TID 1559, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 701.0 (TID 1559) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 701.0 (TID 1559). 911 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 701.0 (TID 1559) in 3 ms on localhost (1/1) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 701 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 701.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 514 finished: isEmpty at LDAOptimizer.scala:418, took 0.008139 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 515 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 702(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 702 (PartitionwiseSampledRDD[739] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1366042, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_683 stored as values in memory (estimated size 18.4 KB, free 729.3 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=1384858, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_683_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.3 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_683_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 683 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 702 (PartitionwiseSampledRDD[739] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 702.0 with 3 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 702.0 (TID 1560, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 702.0 (TID 1561, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 702.0 (TID 1562, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 702.0 (TID 1561) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 702.0 (TID 1560) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 702.0 (TID 1562) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 702.0 (TID 1560). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 702.0 (TID 1562). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 702.0 (TID 1561). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 702.0 (TID 1560) in 3 ms on localhost (1/3) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 702.0 (TID 1562) in 3 ms on localhost (2/3) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 702.0 (TID 1561) in 3 ms on localhost (3/3) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 702 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 702.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 515 finished: isEmpty at LDAOptimizer.scala:418, took 0.008230 s -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1392518, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_684 stored as values in memory (estimated size 176.0 B, free 729.3 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1392694, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_684_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.3 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_684_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 684 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 516 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 703(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 703 (MapPartitionsRDD[741] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1392974, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_685 stored as values in memory (estimated size 19.8 KB, free 729.2 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8438) called with curMem=1413230, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_685_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.2 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_685_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 685 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 703 (MapPartitionsRDD[741] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 703.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 703.0 (TID 1563, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 703.0 (TID 1564, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 703.0 (TID 1565, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 703.0 (TID 1566, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 703.0 (TID 1563) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 703.0 (TID 1565) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 703.0 (TID 1566) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 703.0 (TID 1564) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 703.0 (TID 1563). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 703.0 (TID 1565). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 703.0 (TID 1566). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 703.0 (TID 1564). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 703.0 (TID 1563) in 5 ms on localhost (1/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 703.0 (TID 1565) in 4 ms on localhost (2/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 703.0 (TID 1566) in 4 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 703.0 (TID 1564) in 5 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 703 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 703.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 516 finished: reduce at LDAOptimizer.scala:453, took 0.010967 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 517 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 704(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 704 (MapPartitionsRDD[742] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1421668, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_686 stored as values in memory (estimated size 19.8 KB, free 729.2 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8451) called with curMem=1441964, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_686_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.2 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_686_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 686 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 704 (MapPartitionsRDD[742] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 704.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 704.0 (TID 1567, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 704.0 (TID 1568, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 704.0 (TID 1569, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 704.0 (TID 1570, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 704.0 (TID 1569) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 704.0 (TID 1570) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 704.0 (TID 1567) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 704.0 (TID 1568) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 704.0 (TID 1567). 1002 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 704.0 (TID 1569). 1314 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 704.0 (TID 1570). 1358 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 704.0 (TID 1568). 1402 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 704.0 (TID 1567) in 5 ms on localhost (1/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 704.0 (TID 1569) in 5 ms on localhost (2/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 704.0 (TID 1570) in 5 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 704.0 (TID 1568) in 7 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 704 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 704.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 517 finished: reduce at LDAOptimizer.scala:456, took 0.012547 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 518 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 705(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 705 (PartitionwiseSampledRDD[743] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1450415, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_687 stored as values in memory (estimated size 18.4 KB, free 729.2 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=1469231, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_687_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.2 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_687_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 687 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 705 (PartitionwiseSampledRDD[743] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 705.0 with 1 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 705.0 (TID 1571, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 705.0 (TID 1571) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 705.0 (TID 1571). 911 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 705.0 (TID 1571) in 4 ms on localhost (1/1) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 705 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 518 finished: isEmpty at LDAOptimizer.scala:418, took 0.011387 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 705.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 519 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 706(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 706 (PartitionwiseSampledRDD[743] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1476892, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_688 stored as values in memory (estimated size 18.4 KB, free 729.2 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=1495708, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_688_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.2 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_688_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 688 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 706 (PartitionwiseSampledRDD[743] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 706.0 with 3 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 706.0 (TID 1572, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 706.0 (TID 1573, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 706.0 (TID 1574, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 706.0 (TID 1572) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 706.0 (TID 1572). 911 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 706.0 (TID 1573) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 706.0 (TID 1572) in 7 ms on localhost (1/3) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 706.0 (TID 1573). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 706.0 (TID 1573) in 7 ms on localhost (2/3) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 706.0 (TID 1574) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 706.0 (TID 1574). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 706.0 (TID 1574) in 13 ms on localhost (3/3) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 706.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 706 (isEmpty at LDAOptimizer.scala:418) finished in 0.014 s -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 519 finished: isEmpty at LDAOptimizer.scala:418, took 0.022882 s -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1503369, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_689 stored as values in memory (estimated size 176.0 B, free 729.2 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1503545, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_689_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.2 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_689_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 689 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 520 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 707(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 707 (MapPartitionsRDD[745] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1503825, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_690 stored as values in memory (estimated size 19.8 KB, free 729.1 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8434) called with curMem=1524081, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_690_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.1 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_690_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 690 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 707 (MapPartitionsRDD[745] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 707.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 707.0 (TID 1575, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 707.0 (TID 1576, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 707.0 (TID 1577, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 707.0 (TID 1578, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 707.0 (TID 1575) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 707.0 (TID 1577) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 707.0 (TID 1578) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 707.0 (TID 1576) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 707.0 (TID 1575). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 707.0 (TID 1577). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 707.0 (TID 1576). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 707.0 (TID 1575) in 4 ms on localhost (1/4) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 707.0 (TID 1578). 1265 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 707.0 (TID 1577) in 3 ms on localhost (2/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 707.0 (TID 1576) in 5 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 707.0 (TID 1578) in 4 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 707 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 707.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 520 finished: reduce at LDAOptimizer.scala:453, took 0.009947 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 521 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 708(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 708 (MapPartitionsRDD[746] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1532515, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_691 stored as values in memory (estimated size 19.8 KB, free 729.1 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(8441) called with curMem=1552811, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_691_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.1 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_691_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 691 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 708 (MapPartitionsRDD[746] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 708.0 with 4 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 708.0 (TID 1579, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 708.0 (TID 1580, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 708.0 (TID 1581, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 3.0 in stage 708.0 (TID 1582, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 2.0 in stage 708.0 (TID 1581) -2016-06-13 19:45:25 INFO Executor:59 - Running task 3.0 in stage 708.0 (TID 1582) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 708.0 (TID 1579) -2016-06-13 19:45:25 INFO Executor:59 - Running task 1.0 in stage 708.0 (TID 1580) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 708.0 (TID 1579). 1002 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 1.0 in stage 708.0 (TID 1580). 1002 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 2.0 in stage 708.0 (TID 1581). 1314 bytes result sent to driver -2016-06-13 19:45:25 INFO Executor:59 - Finished task 3.0 in stage 708.0 (TID 1582). 1358 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 708.0 (TID 1579) in 4 ms on localhost (1/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 708.0 (TID 1580) in 4 ms on localhost (2/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 708.0 (TID 1581) in 4 ms on localhost (3/4) -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 3.0 in stage 708.0 (TID 1582) in 3 ms on localhost (4/4) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 708 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 708.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 521 finished: reduce at LDAOptimizer.scala:456, took 0.011748 s -2016-06-13 19:45:25 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Got job 522 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:25 INFO DAGScheduler:59 - Final stage: ResultStage 709(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting ResultStage 709 (PartitionwiseSampledRDD[747] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1561252, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_692 stored as values in memory (estimated size 18.4 KB, free 729.1 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=1580068, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_692_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.1 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_692_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 692 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:25 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 709 (PartitionwiseSampledRDD[747] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Adding task set 709.0 with 1 tasks -2016-06-13 19:45:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 709.0 (TID 1583, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:25 INFO Executor:59 - Running task 0.0 in stage 709.0 (TID 1583) -2016-06-13 19:45:25 INFO Executor:59 - Finished task 0.0 in stage 709.0 (TID 1583). 1226 bytes result sent to driver -2016-06-13 19:45:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 709.0 (TID 1583) in 2 ms on localhost (1/1) -2016-06-13 19:45:25 INFO DAGScheduler:59 - ResultStage 709 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 709.0, whose tasks have all completed, from pool -2016-06-13 19:45:25 INFO DAGScheduler:59 - Job 522 finished: isEmpty at LDAOptimizer.scala:418, took 0.008070 s -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1587730, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_693 stored as values in memory (estimated size 176.0 B, free 729.1 MB) -2016-06-13 19:45:25 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1587906, maxMem=766075207 -2016-06-13 19:45:25 INFO MemoryStore:59 - Block broadcast_693_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.1 MB) -2016-06-13 19:45:25 INFO BlockManagerInfo:59 - Added broadcast_693_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:25 INFO SparkContext:59 - Created broadcast 693 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 523 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 710(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 710 (MapPartitionsRDD[749] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1588186, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_694 stored as values in memory (estimated size 19.8 KB, free 729.1 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8438) called with curMem=1608442, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_694_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.0 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_694_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 694 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 710 (MapPartitionsRDD[749] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 710.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 710.0 (TID 1584, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 710.0 (TID 1585, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 710.0 (TID 1586, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 710.0 (TID 1587, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 710.0 (TID 1584) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 710.0 (TID 1586) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 710.0 (TID 1585) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 710.0 (TID 1587) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 710.0 (TID 1586). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 710.0 (TID 1586) in 3 ms on localhost (1/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 710.0 (TID 1585). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 710.0 (TID 1587). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 710.0 (TID 1585) in 5 ms on localhost (2/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 710.0 (TID 1587) in 5 ms on localhost (3/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 710.0 (TID 1584). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 710.0 (TID 1584) in 8 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 710 (reduce at LDAOptimizer.scala:453) finished in 0.012 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 710.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 523 finished: reduce at LDAOptimizer.scala:453, took 0.019376 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 524 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 711(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 711 (MapPartitionsRDD[750] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1616880, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_695 stored as values in memory (estimated size 19.8 KB, free 729.0 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8451) called with curMem=1637176, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_695_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.0 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_695_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 695 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 711 (MapPartitionsRDD[750] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 711.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 711.0 (TID 1588, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 711.0 (TID 1589, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 711.0 (TID 1590, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 711.0 (TID 1591, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 711.0 (TID 1590) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 711.0 (TID 1591) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 711.0 (TID 1589) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 711.0 (TID 1588) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 711.0 (TID 1591). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 711.0 (TID 1591) in 5 ms on localhost (1/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 711.0 (TID 1589). 1402 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 711.0 (TID 1589) in 6 ms on localhost (2/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 711.0 (TID 1590). 1002 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 711.0 (TID 1588). 1358 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 711.0 (TID 1590) in 6 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 711.0 (TID 1588) in 8 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 711 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 711.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 524 finished: reduce at LDAOptimizer.scala:456, took 0.012693 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 525 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 712(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 712 (PartitionwiseSampledRDD[751] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1645627, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_696 stored as values in memory (estimated size 18.4 KB, free 729.0 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=1664443, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_696_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.0 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_696_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 696 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 712 (PartitionwiseSampledRDD[751] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 712.0 with 1 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 712.0 (TID 1592, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 712.0 (TID 1592) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 712.0 (TID 1592). 1226 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 712.0 (TID 1592) in 2 ms on localhost (1/1) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 712 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 712.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 525 finished: isEmpty at LDAOptimizer.scala:418, took 0.007648 s -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1672105, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_697 stored as values in memory (estimated size 176.0 B, free 729.0 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1672281, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_697_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.0 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_697_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 697 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 526 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 713(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 713 (MapPartitionsRDD[753] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1672561, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_698 stored as values in memory (estimated size 19.8 KB, free 729.0 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=1692817, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_698_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.0 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_698_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 698 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 713 (MapPartitionsRDD[753] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 713.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 713.0 (TID 1593, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 713.0 (TID 1594, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 713.0 (TID 1595, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 713.0 (TID 1596, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 713.0 (TID 1593) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 713.0 (TID 1596) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 713.0 (TID 1594) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 713.0 (TID 1595) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 713.0 (TID 1593). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 713.0 (TID 1595). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 713.0 (TID 1593) in 5 ms on localhost (1/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 713.0 (TID 1594). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 713.0 (TID 1595) in 4 ms on localhost (2/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 713.0 (TID 1596). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 713.0 (TID 1594) in 6 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 713.0 (TID 1596) in 6 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 713 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 713.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 526 finished: reduce at LDAOptimizer.scala:453, took 0.013005 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 527 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 714(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 714 (MapPartitionsRDD[754] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1701250, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_699 stored as values in memory (estimated size 19.8 KB, free 728.9 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8446) called with curMem=1721546, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_699_piece0 stored as bytes in memory (estimated size 8.2 KB, free 728.9 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_699_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 699 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 714 (MapPartitionsRDD[754] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 714.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 714.0 (TID 1597, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 714.0 (TID 1598, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 714.0 (TID 1599, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 714.0 (TID 1600, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 714.0 (TID 1597) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 714.0 (TID 1599) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 714.0 (TID 1598) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 714.0 (TID 1600) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 714.0 (TID 1599). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 714.0 (TID 1600). 1358 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 714.0 (TID 1597). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 714.0 (TID 1598). 1402 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 714.0 (TID 1599) in 4 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 714.0 (TID 1600) in 4 ms on localhost (2/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 714.0 (TID 1597) in 5 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 714.0 (TID 1598) in 6 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 714 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 714.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 527 finished: reduce at LDAOptimizer.scala:456, took 0.011691 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 528 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 715(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 715 (PartitionwiseSampledRDD[755] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1729992, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_700 stored as values in memory (estimated size 18.4 KB, free 728.9 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7663) called with curMem=1748808, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_700_piece0 stored as bytes in memory (estimated size 7.5 KB, free 728.9 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_700_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 700 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 715 (PartitionwiseSampledRDD[755] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 715.0 with 1 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 715.0 (TID 1601, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 715.0 (TID 1601) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 715.0 (TID 1601). 1226 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 715.0 (TID 1601) in 2 ms on localhost (1/1) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 715 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 715.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 528 finished: isEmpty at LDAOptimizer.scala:418, took 0.007170 s -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1756471, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_701 stored as values in memory (estimated size 176.0 B, free 728.9 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1756647, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_701_piece0 stored as bytes in memory (estimated size 280.0 B, free 728.9 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_701_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 701 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_629_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_700_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 549 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 529 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 716(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 716 (MapPartitionsRDD[757] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_699_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1672962, maxMem=766075207 -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 548 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_702 stored as values in memory (estimated size 19.8 KB, free 729.0 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8436) called with curMem=1664529, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_702_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.0 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_698_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 547 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_702_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 702 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 716 (MapPartitionsRDD[757] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 716.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 716.0 (TID 1602, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 716.0 (TID 1603, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 716.0 (TID 1604, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 716.0 (TID 1605, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 716.0 (TID 1603) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 716.0 (TID 1604) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_697_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 716.0 (TID 1605) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 716.0 (TID 1602) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 716.0 (TID 1602). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 716.0 (TID 1605). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 716.0 (TID 1602) in 6 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 716.0 (TID 1605) in 5 ms on localhost (2/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 716.0 (TID 1604). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_696_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 716.0 (TID 1603). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 716.0 (TID 1603) in 8 ms on localhost (3/4) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 546 -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 716.0 (TID 1604) in 8 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 716 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 716.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 529 finished: reduce at LDAOptimizer.scala:453, took 0.014743 s -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_695_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 545 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 530 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_694_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 717(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 544 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 717 (MapPartitionsRDD[758] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1588310, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_703 stored as values in memory (estimated size 19.8 KB, free 729.1 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_693_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8448) called with curMem=1600768, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_703_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.1 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_692_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_703_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 703 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 717 (MapPartitionsRDD[758] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 717.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 717.0 (TID 1606, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 717.0 (TID 1607, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 543 -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 717.0 (TID 1608, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 717.0 (TID 1609, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 717.0 (TID 1608) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 717.0 (TID 1606) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 717.0 (TID 1609) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 717.0 (TID 1607) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 717.0 (TID 1608). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_691_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 542 -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 717.0 (TID 1608) in 7 ms on localhost (1/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 717.0 (TID 1607). 1358 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 717.0 (TID 1609). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 717.0 (TID 1606). 1358 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 717.0 (TID 1607) in 9 ms on localhost (2/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 717.0 (TID 1609) in 8 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 717.0 (TID 1606) in 11 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 717 (reduce at LDAOptimizer.scala:456) finished in 0.013 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 717.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_690_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 530 finished: reduce at LDAOptimizer.scala:456, took 0.019454 s -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 541 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_689_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 531 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 718(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 718 (PartitionwiseSampledRDD[759] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_688_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1506040, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_704 stored as values in memory (estimated size 18.4 KB, free 729.1 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 540 -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7675) called with curMem=1506040, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_704_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.1 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_704_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 704 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_687_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 718 (PartitionwiseSampledRDD[759] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 718.0 with 1 tasks -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 539 -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 718.0 (TID 1610, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 718.0 (TID 1610) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_686_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 538 -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 718.0 (TID 1610). 1226 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 718.0 (TID 1610) in 4 ms on localhost (1/1) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 718 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 718.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 531 finished: isEmpty at LDAOptimizer.scala:418, took 0.010727 s -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1468869, maxMem=766075207 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_685_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_705 stored as values in memory (estimated size 176.0 B, free 729.2 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 537 -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1448789, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_705_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.2 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_684_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_705_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 705 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_683_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 536 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_682_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 535 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 532 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 719(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 719 (MapPartitionsRDD[761] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_681_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 534 -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1366916, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_706 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_680_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8455) called with curMem=1358480, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_706_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.3 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 533 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_706_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 706 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 719 (MapPartitionsRDD[761] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 719.0 with 4 tasks -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_679_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 719.0 (TID 1611, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 719.0 (TID 1612, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 719.0 (TID 1613, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 719.0 (TID 1614, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 719.0 (TID 1613) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_678_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 719.0 (TID 1611) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 719.0 (TID 1612) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 719.0 (TID 1613). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 719.0 (TID 1613) in 4 ms on localhost (1/4) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 719.0 (TID 1614) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 719.0 (TID 1612). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 719.0 (TID 1611). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 719.0 (TID 1614). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 719.0 (TID 1611) in 12 ms on localhost (2/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 719.0 (TID 1612) in 11 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 719.0 (TID 1614) in 11 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 719 (reduce at LDAOptimizer.scala:453) finished in 0.013 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 719.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 532 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 532 finished: reduce at LDAOptimizer.scala:453, took 0.020280 s -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_677_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 531 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 533 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 720(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 720 (MapPartitionsRDD[762] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_676_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1282569, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_707 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 530 -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8468) called with curMem=1302585, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_707_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.3 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_707_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 707 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_675_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 720 (MapPartitionsRDD[762] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 720.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 720.0 (TID 1615, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 720.0 (TID 1616, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 720.0 (TID 1617, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 720.0 (TID 1618, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 720.0 (TID 1615) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 720.0 (TID 1617) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 720.0 (TID 1616) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 720.0 (TID 1618) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_674_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 529 -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 720.0 (TID 1617). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 720.0 (TID 1615). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 720.0 (TID 1616). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_673_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 720.0 (TID 1617) in 5 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 720.0 (TID 1616) in 5 ms on localhost (2/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 720.0 (TID 1615) in 5 ms on localhost (3/4) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 528 -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 720.0 (TID 1618). 1534 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 720.0 (TID 1618) in 6 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 720 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 720.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 533 finished: reduce at LDAOptimizer.scala:456, took 0.013179 s -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_672_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 527 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_671_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 526 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 534 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 721(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 721 (PartitionwiseSampledRDD[763] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_670_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1200040, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_708 stored as values in memory (estimated size 18.4 KB, free 729.4 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7666) called with curMem=1211196, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_708_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.4 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_669_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_708_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 525 -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 708 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 721 (PartitionwiseSampledRDD[763] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 721.0 with 1 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 721.0 (TID 1619, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 721.0 (TID 1619) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_668_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 524 -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 721.0 (TID 1619). 1226 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 721.0 (TID 1619) in 2 ms on localhost (1/1) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 721 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 721.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 534 finished: isEmpty at LDAOptimizer.scala:418, took 0.007715 s -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_667_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1142601, maxMem=766075207 -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 523 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_709 stored as values in memory (estimated size 176.0 B, free 729.5 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1142497, maxMem=766075207 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_666_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_709_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.5 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_709_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 709 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_665_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 522 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_664_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 521 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 535 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 722(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 722 (MapPartitionsRDD[765] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_663_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1060895, maxMem=766075207 -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 520 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_710 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8440) called with curMem=1072720, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_710_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_662_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_710_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 519 -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 710 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 722 (MapPartitionsRDD[765] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 722.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 722.0 (TID 1620, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 722.0 (TID 1621, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 722.0 (TID 1622, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_661_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 722.0 (TID 1623, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 722.0 (TID 1622) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 722.0 (TID 1623) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 722.0 (TID 1621) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 722.0 (TID 1620) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_660_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 722.0 (TID 1623). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 722.0 (TID 1621). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 722.0 (TID 1620). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 518 -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 722.0 (TID 1623) in 4 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 722.0 (TID 1621) in 6 ms on localhost (2/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 722.0 (TID 1622). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 722.0 (TID 1620) in 7 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 722.0 (TID 1622) in 7 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 722 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 722.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 535 finished: reduce at LDAOptimizer.scala:453, took 0.014879 s -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_659_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 517 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_658_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 516 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 536 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 723(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 723 (MapPartitionsRDD[766] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_657_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=976083, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_711 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8453) called with curMem=988718, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_711_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.6 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_656_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_711_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 515 -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 711 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 723 (MapPartitionsRDD[766] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 723.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 723.0 (TID 1624, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 723.0 (TID 1625, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 723.0 (TID 1626, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 723.0 (TID 1627, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_655_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 723.0 (TID 1624) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 723.0 (TID 1627) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 723.0 (TID 1626) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 514 -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 723.0 (TID 1625) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 723.0 (TID 1624). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_654_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 723.0 (TID 1625). 1446 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 723.0 (TID 1626). 1358 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 723.0 (TID 1627). 1358 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 723.0 (TID 1624) in 7 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 723.0 (TID 1626) in 6 ms on localhost (2/4) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 513 -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 723.0 (TID 1625) in 6 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 723.0 (TID 1627) in 6 ms on localhost (4/4) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 723.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 723 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 536 finished: reduce at LDAOptimizer.scala:456, took 0.013365 s -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_653_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 512 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_652_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 537 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 724(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 724 (PartitionwiseSampledRDD[767] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=867504, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_712 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_651_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7665) called with curMem=886320, maxMem=766075207 -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 511 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_712_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_712_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 712 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 724 (PartitionwiseSampledRDD[767] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 724.0 with 1 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 724.0 (TID 1628, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 724.0 (TID 1628) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 724.0 (TID 1628). 1226 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 724.0 (TID 1628) in 4 ms on localhost (1/1) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 724.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_650_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 724 (isEmpty at LDAOptimizer.scala:418) finished in 0.006 s -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 537 finished: isEmpty at LDAOptimizer.scala:418, took 0.012942 s -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 510 -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=856808, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_713 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_649_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 509 -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=836728, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_713_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.8 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_713_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 713 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_648_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_647_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 508 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_646_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 538 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 725(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 725 (MapPartitionsRDD[769] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 507 -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=761077, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_714 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8444) called with curMem=772899, maxMem=766075207 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_645_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_714_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 506 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_714_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 714 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 725 (MapPartitionsRDD[769] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 725.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 725.0 (TID 1629, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_644_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 725.0 (TID 1630, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 725.0 (TID 1631, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 725.0 (TID 1632, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 725.0 (TID 1632) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 725.0 (TID 1630) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 725.0 (TID 1629) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 725.0 (TID 1631) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_643_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 725.0 (TID 1630). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 505 -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 725.0 (TID 1630) in 3 ms on localhost (1/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 725.0 (TID 1632). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 725.0 (TID 1629). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 725.0 (TID 1631). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 725.0 (TID 1632) in 4 ms on localhost (2/4) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_642_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 725.0 (TID 1629) in 6 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 725.0 (TID 1631) in 6 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 725 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 725.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 538 finished: reduce at LDAOptimizer.scala:453, took 0.012585 s -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 504 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_641_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 503 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 539 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 726(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 726 (MapPartitionsRDD[770] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=696527, maxMem=766075207 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_640_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_715 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8457) called with curMem=690347, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_715_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.9 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_639_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 502 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_715_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 715 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 726 (MapPartitionsRDD[770] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 726.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 726.0 (TID 1633, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 726.0 (TID 1634, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 726.0 (TID 1635, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 726.0 (TID 1636, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_638_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 726.0 (TID 1636) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 726.0 (TID 1635) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 726.0 (TID 1633) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 726.0 (TID 1634) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 501 -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 726.0 (TID 1634). 1002 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 726.0 (TID 1636). 1358 bytes result sent to driver -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_637_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 726.0 (TID 1635). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 726.0 (TID 1634) in 4 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 726.0 (TID 1636) in 5 ms on localhost (2/4) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 500 -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 726.0 (TID 1635) in 5 ms on localhost (3/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 726.0 (TID 1633). 1402 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 726.0 (TID 1633) in 7 ms on localhost (4/4) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_636_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 726 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 726.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 539 finished: reduce at LDAOptimizer.scala:456, took 0.013043 s -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 499 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_635_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_634_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 498 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 540 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 727(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 727 (PartitionwiseSampledRDD[771] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_633_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=559227, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_716 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 497 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_632_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=549354, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_716_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_716_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 716 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 496 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 727 (PartitionwiseSampledRDD[771] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 727.0 with 1 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 727.0 (TID 1637, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 727.0 (TID 1637) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_631_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 727.0 (TID 1637). 911 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 727.0 (TID 1637) in 5 ms on localhost (1/1) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 727.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 727 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 540 finished: isEmpty at LDAOptimizer.scala:418, took 0.016771 s -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_630_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 495 -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 494 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_628_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 541 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 728(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 493 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 728 (PartitionwiseSampledRDD[771] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=501390, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_717 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=519926, maxMem=766075207 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_627_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_717_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_717_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 717 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 728 (PartitionwiseSampledRDD[771] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 728.0 with 3 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 728.0 (TID 1638, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 728.0 (TID 1639, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 728.0 (TID 1640, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 728.0 (TID 1639) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 728.0 (TID 1640) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 728.0 (TID 1639). 911 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 728.0 (TID 1640). 1226 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 728.0 (TID 1639) in 4 ms on localhost (1/3) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 728.0 (TID 1640) in 3 ms on localhost (2/3) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 728.0 (TID 1638) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 728.0 (TID 1638). 1226 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 728.0 (TID 1638) in 7 ms on localhost (3/3) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 728.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_626_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 728 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 541 finished: isEmpty at LDAOptimizer.scala:418, took 0.018235 s -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 492 -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=493272, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_718 stored as values in memory (estimated size 176.0 B, free 730.1 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_625_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=474632, maxMem=766075207 -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 491 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_718_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.1 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_718_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 718 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_624_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 490 -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_623_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 489 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 542 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 729(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 729 (MapPartitionsRDD[773] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_622_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=417195, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_719 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=429615, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_719_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Removed broadcast_621_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_719_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 719 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 729 (MapPartitionsRDD[773] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO ContextCleaner:59 - Cleaned accumulator 488 -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 729.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 729.0 (TID 1641, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 729.0 (TID 1642, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 729.0 (TID 1643, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 729.0 (TID 1644, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 729.0 (TID 1642) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 729.0 (TID 1644) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 729.0 (TID 1643) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 729.0 (TID 1641) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 729.0 (TID 1643). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 729.0 (TID 1641). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 729.0 (TID 1643) in 5 ms on localhost (1/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 729.0 (TID 1644). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 729.0 (TID 1642). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 729.0 (TID 1641) in 7 ms on localhost (2/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 729.0 (TID 1644) in 5 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 729.0 (TID 1642) in 7 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 729 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 729.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 542 finished: reduce at LDAOptimizer.scala:453, took 0.014614 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 543 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 730(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 730 (MapPartitionsRDD[774] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=419231, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_720 stored as values in memory (estimated size 19.8 KB, free 730.2 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8446) called with curMem=439527, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_720_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_720_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 720 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 730 (MapPartitionsRDD[774] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 730.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 730.0 (TID 1645, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 730.0 (TID 1646, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 730.0 (TID 1647, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 730.0 (TID 1648, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 730.0 (TID 1645) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 730.0 (TID 1648) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 730.0 (TID 1647) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 730.0 (TID 1646) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 730.0 (TID 1647). 1002 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 730.0 (TID 1645). 1002 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 730.0 (TID 1647) in 3 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 730.0 (TID 1645) in 4 ms on localhost (2/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 730.0 (TID 1646). 1358 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 730.0 (TID 1646) in 4 ms on localhost (3/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 730.0 (TID 1648). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 730.0 (TID 1648) in 4 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 730 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 730.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 543 finished: reduce at LDAOptimizer.scala:456, took 0.011606 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 544 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 731(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 731 (PartitionwiseSampledRDD[775] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=447973, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_721 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=466789, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_721_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_721_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 721 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 731 (PartitionwiseSampledRDD[775] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 731.0 with 1 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 731.0 (TID 1649, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 731.0 (TID 1649) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 731.0 (TID 1649). 911 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 731.0 (TID 1649) in 2 ms on localhost (1/1) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 731 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 731.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 544 finished: isEmpty at LDAOptimizer.scala:418, took 0.007043 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 545 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 732(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 732 (PartitionwiseSampledRDD[775] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=474449, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_722 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=493265, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_722_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_722_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 722 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 732 (PartitionwiseSampledRDD[775] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 732.0 with 3 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 732.0 (TID 1650, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 732.0 (TID 1651, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 732.0 (TID 1652, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 732.0 (TID 1652) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 732.0 (TID 1650) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 732.0 (TID 1651) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 732.0 (TID 1651). 911 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 732.0 (TID 1652). 1226 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 732.0 (TID 1650). 911 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 732.0 (TID 1651) in 2 ms on localhost (1/3) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 732.0 (TID 1652) in 2 ms on localhost (2/3) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 732.0 (TID 1650) in 3 ms on localhost (3/3) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 732.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 732 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 545 finished: isEmpty at LDAOptimizer.scala:418, took 0.007946 s -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=500925, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_723 stored as values in memory (estimated size 176.0 B, free 730.1 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=501101, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_723_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.1 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_723_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 723 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 546 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 733(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 733 (MapPartitionsRDD[777] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=501381, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_724 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=521637, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_724_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_724_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 724 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 733 (MapPartitionsRDD[777] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 733.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 733.0 (TID 1653, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 733.0 (TID 1654, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 733.0 (TID 1655, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 733.0 (TID 1656, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 733.0 (TID 1654) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 733.0 (TID 1656) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 733.0 (TID 1655) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 733.0 (TID 1653) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 733.0 (TID 1654). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 733.0 (TID 1653). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 733.0 (TID 1655). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 733.0 (TID 1654) in 3 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 733.0 (TID 1653) in 4 ms on localhost (2/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 733.0 (TID 1655) in 3 ms on localhost (3/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 733.0 (TID 1656). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 733.0 (TID 1656) in 4 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 733 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 733.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 546 finished: reduce at LDAOptimizer.scala:453, took 0.009155 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 547 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 734(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 734 (MapPartitionsRDD[778] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=530070, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_725 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8440) called with curMem=550366, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_725_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_725_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 725 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 734 (MapPartitionsRDD[778] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 734.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 734.0 (TID 1657, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 734.0 (TID 1658, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 734.0 (TID 1659, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 734.0 (TID 1660, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 734.0 (TID 1658) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 734.0 (TID 1660) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 734.0 (TID 1659) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 734.0 (TID 1657) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 734.0 (TID 1658). 1002 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 734.0 (TID 1657). 1002 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 734.0 (TID 1658) in 3 ms on localhost (1/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 734.0 (TID 1659). 1002 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 734.0 (TID 1657) in 4 ms on localhost (2/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 734.0 (TID 1660). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 734.0 (TID 1659) in 3 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 734.0 (TID 1660) in 4 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 734 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 734.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 547 finished: reduce at LDAOptimizer.scala:456, took 0.009745 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 548 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 735(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 735 (PartitionwiseSampledRDD[779] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=558806, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_726 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7653) called with curMem=577622, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_726_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_726_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 726 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 735 (PartitionwiseSampledRDD[779] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 735.0 with 1 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 735.0 (TID 1661, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 735.0 (TID 1661) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 735.0 (TID 1661). 1226 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 735.0 (TID 1661) in 3 ms on localhost (1/1) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 735 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 735.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 548 finished: isEmpty at LDAOptimizer.scala:418, took 0.009723 s -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=585275, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_727 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=585451, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_727_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_727_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 727 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 549 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 736(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 736 (MapPartitionsRDD[781] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=585731, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_728 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8436) called with curMem=605987, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_728_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_728_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 728 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 736 (MapPartitionsRDD[781] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 736.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 736.0 (TID 1662, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 736.0 (TID 1663, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 736.0 (TID 1664, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 736.0 (TID 1665, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 736.0 (TID 1664) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 736.0 (TID 1665) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 736.0 (TID 1663) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 736.0 (TID 1662) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 736.0 (TID 1663). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 736.0 (TID 1662). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 736.0 (TID 1664). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 736.0 (TID 1665). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 736.0 (TID 1663) in 4 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 736.0 (TID 1664) in 4 ms on localhost (2/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 736.0 (TID 1662) in 4 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 736.0 (TID 1665) in 4 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 736 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 736.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 549 finished: reduce at LDAOptimizer.scala:453, took 0.009662 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 550 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 737(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 737 (MapPartitionsRDD[782] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=614423, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_729 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8451) called with curMem=634719, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_729_piece0 stored as bytes in memory (estimated size 8.3 KB, free 730.0 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_729_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 729 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 737 (MapPartitionsRDD[782] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 737.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 737.0 (TID 1666, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 737.0 (TID 1667, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 737.0 (TID 1668, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 737.0 (TID 1669, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 737.0 (TID 1667) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 737.0 (TID 1669) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 737.0 (TID 1668) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 737.0 (TID 1666) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 737.0 (TID 1668). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 737.0 (TID 1669). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 737.0 (TID 1666). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 737.0 (TID 1668) in 3 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 737.0 (TID 1669) in 4 ms on localhost (2/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 737.0 (TID 1666) in 5 ms on localhost (3/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 737.0 (TID 1667). 1358 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 737.0 (TID 1667) in 6 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 737 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 737.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 550 finished: reduce at LDAOptimizer.scala:456, took 0.011193 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 551 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 738(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 738 (PartitionwiseSampledRDD[783] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=643170, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_730 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=661986, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_730_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_730_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 730 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 738 (PartitionwiseSampledRDD[783] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 738.0 with 1 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 738.0 (TID 1670, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 738.0 (TID 1670) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 738.0 (TID 1670). 911 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 738.0 (TID 1670) in 2 ms on localhost (1/1) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 738 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 738.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 551 finished: isEmpty at LDAOptimizer.scala:418, took 0.006782 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 552 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 739(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 739 (PartitionwiseSampledRDD[783] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=669648, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_731 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=688464, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_731_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_731_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 731 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 739 (PartitionwiseSampledRDD[783] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 739.0 with 3 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 739.0 (TID 1671, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 739.0 (TID 1672, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 739.0 (TID 1673, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 739.0 (TID 1671) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 739.0 (TID 1673) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 739.0 (TID 1672) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 739.0 (TID 1671). 1226 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 739.0 (TID 1672). 911 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 739.0 (TID 1673). 1226 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 739.0 (TID 1671) in 3 ms on localhost (1/3) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 739.0 (TID 1672) in 3 ms on localhost (2/3) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 739.0 (TID 1673) in 3 ms on localhost (3/3) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 739 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 739.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 552 finished: isEmpty at LDAOptimizer.scala:418, took 0.009121 s -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=696126, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_732 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=696302, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_732_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_732_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 732 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 553 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 740(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 740 (MapPartitionsRDD[785] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=696582, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_733 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8439) called with curMem=716838, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_733_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_733_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 733 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 740 (MapPartitionsRDD[785] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 740.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 740.0 (TID 1674, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 740.0 (TID 1675, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 740.0 (TID 1676, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 740.0 (TID 1677, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 740.0 (TID 1674) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 740.0 (TID 1677) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 740.0 (TID 1675) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 740.0 (TID 1676) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 740.0 (TID 1674). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 740.0 (TID 1676). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 740.0 (TID 1674) in 3 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 740.0 (TID 1676) in 3 ms on localhost (2/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 740.0 (TID 1677). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 740.0 (TID 1675). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 740.0 (TID 1677) in 4 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 740.0 (TID 1675) in 5 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 740 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 740.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 553 finished: reduce at LDAOptimizer.scala:453, took 0.009412 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 554 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 741(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 741 (MapPartitionsRDD[786] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=725277, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_734 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8450) called with curMem=745573, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_734_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.9 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_734_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 734 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 741 (MapPartitionsRDD[786] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 741.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 741.0 (TID 1678, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 741.0 (TID 1679, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 741.0 (TID 1680, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 741.0 (TID 1681, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 741.0 (TID 1679) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 741.0 (TID 1681) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 741.0 (TID 1678) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 741.0 (TID 1680) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 741.0 (TID 1678). 1002 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 741.0 (TID 1680). 1002 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 741.0 (TID 1678) in 3 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 741.0 (TID 1680) in 3 ms on localhost (2/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 741.0 (TID 1681). 1402 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 741.0 (TID 1679). 1358 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 741.0 (TID 1681) in 4 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 741.0 (TID 1679) in 5 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 741 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 741.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 554 finished: reduce at LDAOptimizer.scala:456, took 0.009821 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 555 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 742(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 742 (PartitionwiseSampledRDD[787] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=754023, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_735 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=772839, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_735_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_735_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 735 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 742 (PartitionwiseSampledRDD[787] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 742.0 with 1 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 742.0 (TID 1682, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 742.0 (TID 1682) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 742.0 (TID 1682). 1226 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 742.0 (TID 1682) in 3 ms on localhost (1/1) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 742.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 742 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 555 finished: isEmpty at LDAOptimizer.scala:418, took 0.011321 s -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=780499, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_736 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=780675, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_736_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.8 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_736_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 736 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 556 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 743(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 743 (MapPartitionsRDD[789] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=780955, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_737 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=801211, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_737_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_737_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 737 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 743 (MapPartitionsRDD[789] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 743.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 743.0 (TID 1683, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 743.0 (TID 1684, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 743.0 (TID 1685, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 743.0 (TID 1686, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 743.0 (TID 1683) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 743.0 (TID 1685) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 743.0 (TID 1684) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 743.0 (TID 1686) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 743.0 (TID 1683). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 743.0 (TID 1686). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 743.0 (TID 1684). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 743.0 (TID 1685). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 743.0 (TID 1683) in 4 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 743.0 (TID 1686) in 4 ms on localhost (2/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 743.0 (TID 1684) in 5 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 743.0 (TID 1685) in 4 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 743 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 743.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 556 finished: reduce at LDAOptimizer.scala:453, took 0.009925 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 557 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 744(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 744 (MapPartitionsRDD[790] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=809643, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_738 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8443) called with curMem=829939, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_738_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_738_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 738 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 744 (MapPartitionsRDD[790] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 744.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 744.0 (TID 1687, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 744.0 (TID 1688, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 744.0 (TID 1689, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 744.0 (TID 1690, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 744.0 (TID 1687) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 744.0 (TID 1690) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 744.0 (TID 1689) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 744.0 (TID 1688) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 744.0 (TID 1690). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 744.0 (TID 1688). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 744.0 (TID 1690) in 4 ms on localhost (1/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 744.0 (TID 1689). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 744.0 (TID 1688) in 4 ms on localhost (2/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 744.0 (TID 1687). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 744.0 (TID 1687) in 6 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 744.0 (TID 1689) in 6 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 744 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 744.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 557 finished: reduce at LDAOptimizer.scala:456, took 0.011707 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 558 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 745(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 745 (PartitionwiseSampledRDD[791] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=838382, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_739 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=857198, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_739_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_739_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 739 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 745 (PartitionwiseSampledRDD[791] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 745.0 with 1 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 745.0 (TID 1691, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 745.0 (TID 1691) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 745.0 (TID 1691). 911 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 745.0 (TID 1691) in 2 ms on localhost (1/1) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 745 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 745.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 558 finished: isEmpty at LDAOptimizer.scala:418, took 0.008706 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 559 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 746(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 746 (PartitionwiseSampledRDD[791] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=864858, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_740 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=883674, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_740_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_740_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 740 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 746 (PartitionwiseSampledRDD[791] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 746.0 with 3 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 746.0 (TID 1692, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 746.0 (TID 1693, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 746.0 (TID 1694, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 746.0 (TID 1693) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 746.0 (TID 1692) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 746.0 (TID 1694) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 746.0 (TID 1693). 911 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 746.0 (TID 1694). 911 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 746.0 (TID 1693) in 2 ms on localhost (1/3) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 746.0 (TID 1692). 1226 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 746.0 (TID 1694) in 3 ms on localhost (2/3) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 746.0 (TID 1692) in 3 ms on localhost (3/3) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 746 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 746.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 559 finished: isEmpty at LDAOptimizer.scala:418, took 0.010285 s -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=891334, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_741 stored as values in memory (estimated size 176.0 B, free 729.7 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=891510, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_741_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.7 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_741_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 741 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 560 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 747(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 747 (MapPartitionsRDD[793] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=891790, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_742 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=912046, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_742_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_742_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 742 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 747 (MapPartitionsRDD[793] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 747.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 747.0 (TID 1695, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 747.0 (TID 1696, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 747.0 (TID 1697, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 747.0 (TID 1698, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 747.0 (TID 1697) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 747.0 (TID 1698) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 747.0 (TID 1695) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 747.0 (TID 1696) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 747.0 (TID 1697). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 747.0 (TID 1695). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 747.0 (TID 1697) in 3 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 747.0 (TID 1695) in 5 ms on localhost (2/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 747.0 (TID 1698). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 747.0 (TID 1696). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 747.0 (TID 1698) in 5 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 747.0 (TID 1696) in 5 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 747 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 747.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 560 finished: reduce at LDAOptimizer.scala:453, took 0.010220 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 561 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 748(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 748 (MapPartitionsRDD[794] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=920479, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_743 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8446) called with curMem=940775, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_743_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_743_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 743 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 748 (MapPartitionsRDD[794] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 748.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 748.0 (TID 1699, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 748.0 (TID 1700, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 748.0 (TID 1701, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 748.0 (TID 1702, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 748.0 (TID 1700) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 748.0 (TID 1702) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 748.0 (TID 1702). 1002 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 748.0 (TID 1700). 1402 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 748.0 (TID 1699) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 748.0 (TID 1701) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 748.0 (TID 1702) in 6 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 748.0 (TID 1700) in 7 ms on localhost (2/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 748.0 (TID 1699). 1002 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 748.0 (TID 1699) in 8 ms on localhost (3/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 748.0 (TID 1701). 1002 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 748.0 (TID 1701) in 9 ms on localhost (4/4) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 748.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 748 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 561 finished: reduce at LDAOptimizer.scala:456, took 0.015409 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 562 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 749(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 749 (PartitionwiseSampledRDD[795] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=949221, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_744 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7666) called with curMem=968037, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_744_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_744_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 744 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 749 (PartitionwiseSampledRDD[795] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 749.0 with 1 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 749.0 (TID 1703, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 749.0 (TID 1703) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 749.0 (TID 1703). 1226 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 749.0 (TID 1703) in 2 ms on localhost (1/1) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 749 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 749.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 562 finished: isEmpty at LDAOptimizer.scala:418, took 0.006733 s -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=975703, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_745 stored as values in memory (estimated size 176.0 B, free 729.7 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=975879, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_745_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.7 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_745_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 745 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 563 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 750(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 750 (MapPartitionsRDD[797] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=976159, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_746 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8439) called with curMem=996415, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_746_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_746_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 746 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 750 (MapPartitionsRDD[797] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 750.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 750.0 (TID 1704, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 750.0 (TID 1705, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 750.0 (TID 1706, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 750.0 (TID 1707, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 750.0 (TID 1706) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 750.0 (TID 1705) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 750.0 (TID 1707) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 750.0 (TID 1704) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 750.0 (TID 1706). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 750.0 (TID 1705). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 750.0 (TID 1707). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 750.0 (TID 1704). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 750.0 (TID 1706) in 4 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 750.0 (TID 1705) in 4 ms on localhost (2/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 750.0 (TID 1707) in 4 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 750.0 (TID 1704) in 4 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 750 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 750.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 563 finished: reduce at LDAOptimizer.scala:453, took 0.010382 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 564 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 751(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 751 (MapPartitionsRDD[798] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1004854, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_747 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8452) called with curMem=1025150, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_747_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.6 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_747_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 747 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 751 (MapPartitionsRDD[798] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 751.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 751.0 (TID 1708, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 751.0 (TID 1709, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 751.0 (TID 1710, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 751.0 (TID 1711, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 751.0 (TID 1709) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 751.0 (TID 1710) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 751.0 (TID 1708) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 751.0 (TID 1709). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 751.0 (TID 1711) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 751.0 (TID 1711). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 751.0 (TID 1710). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 751.0 (TID 1709) in 15 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 751.0 (TID 1711) in 14 ms on localhost (2/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 751.0 (TID 1710) in 15 ms on localhost (3/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 751.0 (TID 1708). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 751.0 (TID 1708) in 23 ms on localhost (4/4) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 751.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 751 (reduce at LDAOptimizer.scala:456) finished in 0.024 s -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 564 finished: reduce at LDAOptimizer.scala:456, took 0.031372 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 565 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 752(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 752 (PartitionwiseSampledRDD[799] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1033602, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_748 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=1052418, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_748_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_748_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 748 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 752 (PartitionwiseSampledRDD[799] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 752.0 with 1 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 752.0 (TID 1712, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 752.0 (TID 1712) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 752.0 (TID 1712). 1226 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 752.0 (TID 1712) in 3 ms on localhost (1/1) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 752 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 752.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 565 finished: isEmpty at LDAOptimizer.scala:418, took 0.007118 s -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1060078, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_749 stored as values in memory (estimated size 176.0 B, free 729.6 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1060254, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_749_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.6 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_749_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 749 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 566 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 753(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 753 (MapPartitionsRDD[801] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1060534, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_750 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=1080790, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_750_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_750_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 750 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 753 (MapPartitionsRDD[801] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 753.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 753.0 (TID 1713, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 753.0 (TID 1714, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 753.0 (TID 1715, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 753.0 (TID 1716, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 753.0 (TID 1715) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 753.0 (TID 1714) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 753.0 (TID 1716) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 753.0 (TID 1713) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 753.0 (TID 1715). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 753.0 (TID 1716). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 753.0 (TID 1713). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 753.0 (TID 1715) in 4 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 753.0 (TID 1716) in 5 ms on localhost (2/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 753.0 (TID 1713) in 6 ms on localhost (3/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 753.0 (TID 1714). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 753.0 (TID 1714) in 7 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 753 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 753.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 566 finished: reduce at LDAOptimizer.scala:453, took 0.014023 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 567 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 754(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 754 (MapPartitionsRDD[802] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1089223, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_751 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8446) called with curMem=1109519, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_751_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_751_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 751 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 754 (MapPartitionsRDD[802] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 754.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 754.0 (TID 1717, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 754.0 (TID 1718, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 754.0 (TID 1719, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 754.0 (TID 1720, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 754.0 (TID 1719) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 754.0 (TID 1717) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 754.0 (TID 1718) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 754.0 (TID 1720) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 754.0 (TID 1720). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 754.0 (TID 1717). 1402 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 754.0 (TID 1718). 1358 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 754.0 (TID 1720) in 4 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 754.0 (TID 1717) in 5 ms on localhost (2/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 754.0 (TID 1719). 1402 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 754.0 (TID 1719) in 5 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 754.0 (TID 1718) in 6 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 754 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 754.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 567 finished: reduce at LDAOptimizer.scala:456, took 0.011213 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 568 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 755(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 755 (PartitionwiseSampledRDD[803] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1117965, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_752 stored as values in memory (estimated size 18.4 KB, free 729.5 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=1136781, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_752_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.5 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_752_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 752 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 755 (PartitionwiseSampledRDD[803] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 755.0 with 1 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 755.0 (TID 1721, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 755.0 (TID 1721) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 755.0 (TID 1721). 1226 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 755.0 (TID 1721) in 3 ms on localhost (1/1) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 755 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 755.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 568 finished: isEmpty at LDAOptimizer.scala:418, took 0.009980 s -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1144443, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_753 stored as values in memory (estimated size 176.0 B, free 729.5 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1144619, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_753_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.5 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_753_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 753 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 569 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 756(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 756 (MapPartitionsRDD[805] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1144899, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_754 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8434) called with curMem=1165155, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_754_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_754_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 754 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 756 (MapPartitionsRDD[805] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 756.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 756.0 (TID 1722, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 756.0 (TID 1723, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 756.0 (TID 1724, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 756.0 (TID 1725, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 756.0 (TID 1723) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 756.0 (TID 1725) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 756.0 (TID 1724) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 756.0 (TID 1722) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 756.0 (TID 1724). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 756.0 (TID 1724) in 4 ms on localhost (1/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 756.0 (TID 1725). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 756.0 (TID 1725) in 4 ms on localhost (2/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 756.0 (TID 1722). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 756.0 (TID 1722) in 6 ms on localhost (3/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 756.0 (TID 1723). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 756.0 (TID 1723) in 7 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 756 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 756.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 569 finished: reduce at LDAOptimizer.scala:453, took 0.012972 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 570 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 757(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 757 (MapPartitionsRDD[806] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1173589, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_755 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8447) called with curMem=1193885, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_755_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_755_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 755 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 757 (MapPartitionsRDD[806] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 757.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 757.0 (TID 1726, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 757.0 (TID 1727, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 757.0 (TID 1728, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 757.0 (TID 1729, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 757.0 (TID 1727) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 757.0 (TID 1729) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 757.0 (TID 1728) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 757.0 (TID 1726) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 757.0 (TID 1728). 1002 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 757.0 (TID 1729). 1358 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 757.0 (TID 1727). 1402 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 757.0 (TID 1728) in 4 ms on localhost (1/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 757.0 (TID 1726). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 757.0 (TID 1729) in 4 ms on localhost (2/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 757.0 (TID 1726) in 4 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 757.0 (TID 1727) in 4 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 757 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 757.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 570 finished: reduce at LDAOptimizer.scala:456, took 0.009714 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 571 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 758(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 758 (PartitionwiseSampledRDD[807] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1202332, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_756 stored as values in memory (estimated size 18.4 KB, free 729.4 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=1221148, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_756_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.4 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_756_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 756 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 758 (PartitionwiseSampledRDD[807] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 758.0 with 1 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 758.0 (TID 1730, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 758.0 (TID 1730) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 758.0 (TID 1730). 1226 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 758.0 (TID 1730) in 2 ms on localhost (1/1) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 758 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 571 finished: isEmpty at LDAOptimizer.scala:418, took 0.007421 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 758.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1228808, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_757 stored as values in memory (estimated size 176.0 B, free 729.4 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1228984, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_757_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.4 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_757_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 757 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 572 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 759(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 759 (MapPartitionsRDD[809] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1229264, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_758 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8434) called with curMem=1249520, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_758_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_758_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 758 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 759 (MapPartitionsRDD[809] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 759.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 759.0 (TID 1731, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 759.0 (TID 1732, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 759.0 (TID 1733, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 759.0 (TID 1734, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 759.0 (TID 1732) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 759.0 (TID 1733) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 759.0 (TID 1731) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 759.0 (TID 1734) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 759.0 (TID 1731). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 759.0 (TID 1732). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 759.0 (TID 1731) in 5 ms on localhost (1/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 759.0 (TID 1733). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 759.0 (TID 1732) in 5 ms on localhost (2/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 759.0 (TID 1734). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 759.0 (TID 1733) in 6 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 759.0 (TID 1734) in 5 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 759 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 759.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 572 finished: reduce at LDAOptimizer.scala:453, took 0.012237 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 573 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 760(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 760 (MapPartitionsRDD[810] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1257954, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_759 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8447) called with curMem=1278250, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_759_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_759_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 759 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 760 (MapPartitionsRDD[810] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 760.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 760.0 (TID 1735, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 760.0 (TID 1736, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 760.0 (TID 1737, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 760.0 (TID 1738, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 760.0 (TID 1735) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 760.0 (TID 1737) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 760.0 (TID 1738) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 760.0 (TID 1736) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 760.0 (TID 1737). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 760.0 (TID 1738). 1358 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 760.0 (TID 1735). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 760.0 (TID 1737) in 5 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 760.0 (TID 1735) in 6 ms on localhost (2/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 760.0 (TID 1738) in 4 ms on localhost (3/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 760.0 (TID 1736). 1446 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 760.0 (TID 1736) in 6 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 760 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 760.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 573 finished: reduce at LDAOptimizer.scala:456, took 0.014071 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 574 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 761(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 761 (PartitionwiseSampledRDD[811] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1286697, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_760 stored as values in memory (estimated size 18.4 KB, free 729.3 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7665) called with curMem=1305513, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_760_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.3 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_760_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 760 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 761 (PartitionwiseSampledRDD[811] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 761.0 with 1 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 761.0 (TID 1739, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 761.0 (TID 1739) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 761.0 (TID 1739). 1226 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 761.0 (TID 1739) in 2 ms on localhost (1/1) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 761 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 761.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 574 finished: isEmpty at LDAOptimizer.scala:418, took 0.008816 s -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1313178, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_761 stored as values in memory (estimated size 176.0 B, free 729.3 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1313354, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_761_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.3 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_761_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 761 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 575 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 762(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 762 (MapPartitionsRDD[813] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1313634, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_762 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8442) called with curMem=1333890, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_762_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.3 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_762_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 762 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 762 (MapPartitionsRDD[813] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 762.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 762.0 (TID 1740, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 762.0 (TID 1741, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 762.0 (TID 1742, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 762.0 (TID 1743, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 762.0 (TID 1741) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 762.0 (TID 1742) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 762.0 (TID 1743) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 762.0 (TID 1740) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 762.0 (TID 1741). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 762.0 (TID 1742). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 762.0 (TID 1741) in 5 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 762.0 (TID 1742) in 5 ms on localhost (2/4) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 762.0 (TID 1743). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 762.0 (TID 1740). 1265 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 762.0 (TID 1743) in 5 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 762.0 (TID 1740) in 6 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 762 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 762.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 575 finished: reduce at LDAOptimizer.scala:453, took 0.011201 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 576 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 763(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 763 (MapPartitionsRDD[814] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1342332, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_763 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(8455) called with curMem=1362628, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_763_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.3 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_763_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 763 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 763 (MapPartitionsRDD[814] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 763.0 with 4 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 763.0 (TID 1744, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 763.0 (TID 1745, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 763.0 (TID 1746, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 3.0 in stage 763.0 (TID 1747, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 1.0 in stage 763.0 (TID 1745) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 763.0 (TID 1744) -2016-06-13 19:45:26 INFO Executor:59 - Running task 3.0 in stage 763.0 (TID 1747) -2016-06-13 19:45:26 INFO Executor:59 - Running task 2.0 in stage 763.0 (TID 1746) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 3.0 in stage 763.0 (TID 1747). 1402 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 2.0 in stage 763.0 (TID 1746). 1314 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 1.0 in stage 763.0 (TID 1745). 1358 bytes result sent to driver -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 763.0 (TID 1744). 1358 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 3.0 in stage 763.0 (TID 1747) in 5 ms on localhost (1/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 763.0 (TID 1745) in 5 ms on localhost (2/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 763.0 (TID 1746) in 5 ms on localhost (3/4) -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 763.0 (TID 1744) in 6 ms on localhost (4/4) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 763 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 763.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 576 finished: reduce at LDAOptimizer.scala:456, took 0.011783 s -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Got job 577 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:26 INFO DAGScheduler:59 - Final stage: ResultStage 764(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting ResultStage 764 (PartitionwiseSampledRDD[815] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1371083, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_764 stored as values in memory (estimated size 18.4 KB, free 729.3 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(7664) called with curMem=1389899, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_764_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.3 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_764_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 764 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 764 (PartitionwiseSampledRDD[815] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Adding task set 764.0 with 1 tasks -2016-06-13 19:45:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 764.0 (TID 1748, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:26 INFO Executor:59 - Running task 0.0 in stage 764.0 (TID 1748) -2016-06-13 19:45:26 INFO Executor:59 - Finished task 0.0 in stage 764.0 (TID 1748). 1226 bytes result sent to driver -2016-06-13 19:45:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 764.0 (TID 1748) in 3 ms on localhost (1/1) -2016-06-13 19:45:26 INFO DAGScheduler:59 - ResultStage 764 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 764.0, whose tasks have all completed, from pool -2016-06-13 19:45:26 INFO DAGScheduler:59 - Job 577 finished: isEmpty at LDAOptimizer.scala:418, took 0.007928 s -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1397563, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_765 stored as values in memory (estimated size 176.0 B, free 729.3 MB) -2016-06-13 19:45:26 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1397739, maxMem=766075207 -2016-06-13 19:45:26 INFO MemoryStore:59 - Block broadcast_765_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.3 MB) -2016-06-13 19:45:26 INFO BlockManagerInfo:59 - Added broadcast_765_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:26 INFO SparkContext:59 - Created broadcast 765 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:26 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 578 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 765(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 765 (MapPartitionsRDD[817] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1398019, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_766 stored as values in memory (estimated size 19.8 KB, free 729.2 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8439) called with curMem=1418275, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_766_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.2 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_766_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 766 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 765 (MapPartitionsRDD[817] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 765.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 765.0 (TID 1749, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 765.0 (TID 1750, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 765.0 (TID 1751, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 765.0 (TID 1752, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 765.0 (TID 1751) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 765.0 (TID 1752) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 765.0 (TID 1749) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 765.0 (TID 1750) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 765.0 (TID 1750). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 765.0 (TID 1749). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 765.0 (TID 1750) in 4 ms on localhost (1/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 765.0 (TID 1751). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 765.0 (TID 1752). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 765.0 (TID 1749) in 5 ms on localhost (2/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 765.0 (TID 1751) in 5 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 765.0 (TID 1752) in 4 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 765 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 578 finished: reduce at LDAOptimizer.scala:453, took 0.010521 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 765.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 579 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 766(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 766 (MapPartitionsRDD[818] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1426714, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_767 stored as values in memory (estimated size 19.8 KB, free 729.2 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8452) called with curMem=1447010, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_767_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.2 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_767_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 767 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 766 (MapPartitionsRDD[818] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 766.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 766.0 (TID 1753, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 766.0 (TID 1754, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 766.0 (TID 1755, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 766.0 (TID 1756, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 766.0 (TID 1755) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 766.0 (TID 1756) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 766.0 (TID 1754) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 766.0 (TID 1753) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 766.0 (TID 1756). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 766.0 (TID 1754). 1002 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 766.0 (TID 1753). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 766.0 (TID 1756) in 4 ms on localhost (1/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 766.0 (TID 1754) in 4 ms on localhost (2/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 766.0 (TID 1755). 1358 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 766.0 (TID 1753) in 5 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 766.0 (TID 1755) in 4 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 766 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 766.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 579 finished: reduce at LDAOptimizer.scala:456, took 0.011506 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 580 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 767(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 767 (PartitionwiseSampledRDD[819] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1455462, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_768 stored as values in memory (estimated size 18.4 KB, free 729.2 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=1474278, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_768_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.2 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_768_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 768 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 767 (PartitionwiseSampledRDD[819] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 767.0 with 1 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 767.0 (TID 1757, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 767.0 (TID 1757) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 767.0 (TID 1757). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 767.0 (TID 1757) in 2 ms on localhost (1/1) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 767 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 767.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 580 finished: isEmpty at LDAOptimizer.scala:418, took 0.006488 s -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1481940, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_769 stored as values in memory (estimated size 176.0 B, free 729.2 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1482116, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_769_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.2 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_769_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 769 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 581 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 768(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 768 (MapPartitionsRDD[821] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1482396, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_770 stored as values in memory (estimated size 19.8 KB, free 729.2 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8436) called with curMem=1502652, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_770_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.1 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_770_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 770 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 768 (MapPartitionsRDD[821] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 768.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 768.0 (TID 1758, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 768.0 (TID 1759, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 768.0 (TID 1760, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 768.0 (TID 1761, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 768.0 (TID 1760) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 768.0 (TID 1761) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 768.0 (TID 1759) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 768.0 (TID 1758) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 768.0 (TID 1760). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 768.0 (TID 1758). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 768.0 (TID 1759). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 768.0 (TID 1761). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 768.0 (TID 1760) in 4 ms on localhost (1/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 768.0 (TID 1758) in 5 ms on localhost (2/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 768.0 (TID 1759) in 5 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 768.0 (TID 1761) in 4 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 768 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 768.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 581 finished: reduce at LDAOptimizer.scala:453, took 0.010466 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 582 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 769(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 769 (MapPartitionsRDD[822] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1511088, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_771 stored as values in memory (estimated size 19.8 KB, free 729.1 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8449) called with curMem=1531384, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_771_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.1 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_771_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 771 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 769 (MapPartitionsRDD[822] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 769.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 769.0 (TID 1762, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 769.0 (TID 1763, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 769.0 (TID 1764, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 769.0 (TID 1765, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 769.0 (TID 1762) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 769.0 (TID 1764) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 769.0 (TID 1765) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 769.0 (TID 1763) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 769.0 (TID 1764). 1358 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 769.0 (TID 1763). 1578 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 769.0 (TID 1764) in 5 ms on localhost (1/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 769.0 (TID 1763) in 6 ms on localhost (2/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 769.0 (TID 1762). 1358 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 769.0 (TID 1765). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 769.0 (TID 1762) in 8 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 769.0 (TID 1765) in 7 ms on localhost (4/4) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 769.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 769 (reduce at LDAOptimizer.scala:456) finished in 0.009 s -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 582 finished: reduce at LDAOptimizer.scala:456, took 0.015488 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 583 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 770(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 770 (PartitionwiseSampledRDD[823] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1539833, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_772 stored as values in memory (estimated size 18.4 KB, free 729.1 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7663) called with curMem=1558649, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_772_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.1 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_772_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 772 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 770 (PartitionwiseSampledRDD[823] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 770.0 with 1 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 770.0 (TID 1766, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 770.0 (TID 1766) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 770.0 (TID 1766). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 770.0 (TID 1766) in 2 ms on localhost (1/1) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 770 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 770.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 583 finished: isEmpty at LDAOptimizer.scala:418, took 0.007009 s -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1566312, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_773 stored as values in memory (estimated size 176.0 B, free 729.1 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1566488, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_773_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.1 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_773_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 773 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 584 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 771(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 771 (MapPartitionsRDD[825] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1566768, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_774 stored as values in memory (estimated size 19.8 KB, free 729.1 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8436) called with curMem=1587024, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_774_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.1 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_774_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 774 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 771 (MapPartitionsRDD[825] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 771.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 771.0 (TID 1767, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 771.0 (TID 1768, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 771.0 (TID 1769, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 771.0 (TID 1770, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 771.0 (TID 1768) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 771.0 (TID 1767) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 771.0 (TID 1769) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 771.0 (TID 1770) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 771.0 (TID 1767). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 771.0 (TID 1769). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 771.0 (TID 1770). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 771.0 (TID 1767) in 6 ms on localhost (1/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 771.0 (TID 1769) in 6 ms on localhost (2/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 771.0 (TID 1770) in 5 ms on localhost (3/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 771.0 (TID 1768). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 771.0 (TID 1768) in 7 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 771 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 771.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 584 finished: reduce at LDAOptimizer.scala:453, took 0.012442 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 585 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 772(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 772 (MapPartitionsRDD[826] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1595460, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_775 stored as values in memory (estimated size 19.8 KB, free 729.0 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8449) called with curMem=1615756, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_775_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.0 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_775_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 775 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 772 (MapPartitionsRDD[826] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 772.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 772.0 (TID 1771, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 772.0 (TID 1772, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 772.0 (TID 1773, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 772.0 (TID 1774, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 772.0 (TID 1771) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 772.0 (TID 1772) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 772.0 (TID 1773) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 772.0 (TID 1774) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 772.0 (TID 1773). 1358 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 772.0 (TID 1772). 1490 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 772.0 (TID 1771). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 772.0 (TID 1772) in 5 ms on localhost (1/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 772.0 (TID 1771) in 7 ms on localhost (2/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 772.0 (TID 1774). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 772.0 (TID 1773) in 8 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 772.0 (TID 1774) in 9 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 772 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 585 finished: reduce at LDAOptimizer.scala:456, took 0.017851 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 772.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 586 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 773(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 773 (PartitionwiseSampledRDD[827] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1624205, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_776 stored as values in memory (estimated size 18.4 KB, free 729.0 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7663) called with curMem=1643021, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_776_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.0 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_776_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 776 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 773 (PartitionwiseSampledRDD[827] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 773.0 with 1 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 773.0 (TID 1775, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 773.0 (TID 1775) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 773.0 (TID 1775). 911 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 773.0 (TID 1775) in 2 ms on localhost (1/1) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 773 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 773.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 586 finished: isEmpty at LDAOptimizer.scala:418, took 0.006599 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 587 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 774(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 774 (PartitionwiseSampledRDD[827] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1650684, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_777 stored as values in memory (estimated size 18.4 KB, free 729.0 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7663) called with curMem=1669500, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_777_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.0 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_777_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 777 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 774 (PartitionwiseSampledRDD[827] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 774.0 with 3 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 774.0 (TID 1776, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 774.0 (TID 1777, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 774.0 (TID 1778, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 774.0 (TID 1776) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 774.0 (TID 1778) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 774.0 (TID 1777) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 774.0 (TID 1778). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 774.0 (TID 1776). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 774.0 (TID 1777). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 774.0 (TID 1778) in 2 ms on localhost (1/3) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 774.0 (TID 1776) in 3 ms on localhost (2/3) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 774.0 (TID 1777) in 4 ms on localhost (3/3) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 774 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 774.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 587 finished: isEmpty at LDAOptimizer.scala:418, took 0.009250 s -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1677163, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_778 stored as values in memory (estimated size 176.0 B, free 729.0 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1677339, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_778_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.0 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_778_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 778 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 588 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 775(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 775 (MapPartitionsRDD[829] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1677619, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_779 stored as values in memory (estimated size 19.8 KB, free 729.0 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8437) called with curMem=1697875, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_779_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.0 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_779_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 779 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 775 (MapPartitionsRDD[829] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 775.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 775.0 (TID 1779, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 775.0 (TID 1780, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 775.0 (TID 1781, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 775.0 (TID 1782, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 775.0 (TID 1780) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 775.0 (TID 1782) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 775.0 (TID 1781) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 775.0 (TID 1779) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 775.0 (TID 1779). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 775.0 (TID 1782). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 775.0 (TID 1781). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 775.0 (TID 1779) in 3 ms on localhost (1/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 775.0 (TID 1782) in 4 ms on localhost (2/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 775.0 (TID 1780). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 775.0 (TID 1781) in 4 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 775.0 (TID 1780) in 4 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 775 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 775.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 588 finished: reduce at LDAOptimizer.scala:453, took 0.010474 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 589 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 776(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 776 (MapPartitionsRDD[830] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1706312, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_780 stored as values in memory (estimated size 19.8 KB, free 728.9 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8450) called with curMem=1726608, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_780_piece0 stored as bytes in memory (estimated size 8.3 KB, free 728.9 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_780_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 780 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 776 (MapPartitionsRDD[830] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 776.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 776.0 (TID 1783, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 776.0 (TID 1784, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 776.0 (TID 1785, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 776.0 (TID 1786, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 776.0 (TID 1783) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 776.0 (TID 1786) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 776.0 (TID 1784) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 776.0 (TID 1785) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 776.0 (TID 1783). 1002 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 776.0 (TID 1783) in 4 ms on localhost (1/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 776.0 (TID 1785). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 776.0 (TID 1784). 1358 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 776.0 (TID 1786). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 776.0 (TID 1785) in 4 ms on localhost (2/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 776.0 (TID 1784) in 4 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 776.0 (TID 1786) in 4 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 776 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 776.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 589 finished: reduce at LDAOptimizer.scala:456, took 0.010634 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 590 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 777(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 777 (PartitionwiseSampledRDD[831] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1735058, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_781 stored as values in memory (estimated size 18.4 KB, free 728.9 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=1753874, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_781_piece0 stored as bytes in memory (estimated size 7.5 KB, free 728.9 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_781_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 781 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 777 (PartitionwiseSampledRDD[831] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 777.0 with 1 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 777.0 (TID 1787, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_716_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 777.0 (TID 1787) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 777.0 (TID 1787). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 777.0 (TID 1787) in 3 ms on localhost (1/1) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 777 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_780_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 777.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 590 finished: isEmpty at LDAOptimizer.scala:418, took 0.020711 s -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 610 -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1697876, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_782 stored as values in memory (estimated size 176.0 B, free 729.0 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_779_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 609 -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1677796, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_782_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.0 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_782_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_778_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 782 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_777_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 608 -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_776_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 591 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 778(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 607 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 778 (MapPartitionsRDD[833] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1616213, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_783 stored as values in memory (estimated size 19.8 KB, free 729.0 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_775_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 606 -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8436) called with curMem=1616173, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_783_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.0 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_783_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 783 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_774_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 778 (MapPartitionsRDD[833] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 778.0 with 4 tasks -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 605 -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 778.0 (TID 1788, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 778.0 (TID 1789, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 778.0 (TID 1790, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 778.0 (TID 1791, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 778.0 (TID 1789) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 778.0 (TID 1790) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 778.0 (TID 1791) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 778.0 (TID 1788) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_773_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 778.0 (TID 1790). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 778.0 (TID 1789). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 778.0 (TID 1789) in 5 ms on localhost (1/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 778.0 (TID 1788). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 778.0 (TID 1790) in 4 ms on localhost (2/4) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_772_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 604 -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 778.0 (TID 1788) in 6 ms on localhost (3/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 778.0 (TID 1791). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 778.0 (TID 1791) in 6 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 778 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 778.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_771_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 591 finished: reduce at LDAOptimizer.scala:453, took 0.013672 s -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 603 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_770_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 602 -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 592 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 779(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 779 (MapPartitionsRDD[834] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_769_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1511089, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_784 stored as values in memory (estimated size 19.8 KB, free 729.1 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8449) called with curMem=1504907, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_784_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.1 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_768_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 601 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_784_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.1 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 784 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 779 (MapPartitionsRDD[834] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 779.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 779.0 (TID 1792, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 779.0 (TID 1793, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 779.0 (TID 1794, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_767_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 779.0 (TID 1795, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 779.0 (TID 1793) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 779.0 (TID 1792) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 779.0 (TID 1794) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 779.0 (TID 1795) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 600 -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 779.0 (TID 1792). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_766_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 779.0 (TID 1794). 1002 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 779.0 (TID 1793). 1402 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 779.0 (TID 1795). 1490 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 779.0 (TID 1792) in 7 ms on localhost (1/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 779.0 (TID 1794) in 6 ms on localhost (2/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 779.0 (TID 1793) in 6 ms on localhost (3/4) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 599 -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 779.0 (TID 1795) in 7 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 779 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 779.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 592 finished: reduce at LDAOptimizer.scala:456, took 0.014327 s -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_765_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_764_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 598 -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 593 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 780(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 780 (PartitionwiseSampledRDD[835] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_763_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1420522, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_785 stored as values in memory (estimated size 18.4 KB, free 729.2 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 597 -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7663) called with curMem=1410600, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_785_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.2 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_762_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_785_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 785 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 596 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 780 (PartitionwiseSampledRDD[835] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 780.0 with 1 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 780.0 (TID 1796, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 780.0 (TID 1796) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_761_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 780.0 (TID 1796). 911 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 780.0 (TID 1796) in 2 ms on localhost (1/1) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 780.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 780 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 593 finished: isEmpty at LDAOptimizer.scala:418, took 0.009564 s -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_760_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 595 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_759_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 594 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 781(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 594 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 781 (PartitionwiseSampledRDD[835] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1342327, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_786 stored as values in memory (estimated size 18.4 KB, free 729.3 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7663) called with curMem=1332453, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_786_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.3 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_758_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 593 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_786_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 786 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 781 (PartitionwiseSampledRDD[835] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 781.0 with 3 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 781.0 (TID 1797, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 781.0 (TID 1798, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 781.0 (TID 1799, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 781.0 (TID 1797) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 781.0 (TID 1798) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_757_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 781.0 (TID 1799) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 781.0 (TID 1797). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 781.0 (TID 1799). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 781.0 (TID 1798). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 781.0 (TID 1797) in 4 ms on localhost (1/3) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 781.0 (TID 1799) in 3 ms on localhost (2/3) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_756_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 781 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 781.0 (TID 1798) in 3 ms on localhost (3/3) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 781.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 592 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 594 finished: isEmpty at LDAOptimizer.scala:418, took 0.010790 s -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1284441, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_787 stored as values in memory (estimated size 176.0 B, free 729.4 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_755_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 591 -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1284617, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_787_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.4 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_787_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 787 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_754_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 590 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_753_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_752_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 595 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 589 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 782(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 782 (MapPartitionsRDD[837] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1200531, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_788 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_751_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 588 -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8441) called with curMem=1200531, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_788_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_788_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 788 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_750_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 782 (MapPartitionsRDD[837] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 782.0 with 4 tasks -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 587 -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 782.0 (TID 1800, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 782.0 (TID 1801, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 782.0 (TID 1802, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 782.0 (TID 1803, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 782.0 (TID 1800) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 782.0 (TID 1802) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 782.0 (TID 1803) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 782.0 (TID 1801) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_749_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 782.0 (TID 1800). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 782.0 (TID 1800) in 4 ms on localhost (1/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 782.0 (TID 1802). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 782.0 (TID 1803). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 782.0 (TID 1801). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 782.0 (TID 1803) in 4 ms on localhost (2/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 782.0 (TID 1802) in 5 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 782.0 (TID 1801) in 6 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 782 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 782.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_748_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 595 finished: reduce at LDAOptimizer.scala:453, took 0.013182 s -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 586 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_747_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 585 -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 596 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 783(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 783 (MapPartitionsRDD[838] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_746_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1116164, maxMem=766075207 -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 584 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_789 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8454) called with curMem=1136180, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_789_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.5 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_745_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_789_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 789 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 783 (MapPartitionsRDD[838] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 783.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 783.0 (TID 1804, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 783.0 (TID 1805, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 783.0 (TID 1806, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 783.0 (TID 1807, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 783.0 (TID 1805) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_744_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 783.0 (TID 1806) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 783.0 (TID 1807) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 783.0 (TID 1804) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 783.0 (TID 1807). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 783.0 (TID 1805). 1402 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 783.0 (TID 1807) in 5 ms on localhost (1/4) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 583 -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 783.0 (TID 1804). 1002 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 783.0 (TID 1805) in 5 ms on localhost (2/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 783.0 (TID 1806). 1402 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 783.0 (TID 1804) in 7 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 783.0 (TID 1806) in 7 ms on localhost (4/4) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 783.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 783 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 596 finished: reduce at LDAOptimizer.scala:456, took 0.015522 s -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_743_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 582 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_742_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 581 -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 597 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 784(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 784 (PartitionwiseSampledRDD[839] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_741_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1060089, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_790 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_740_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=1071245, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_790_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 580 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_790_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 790 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 784 (PartitionwiseSampledRDD[839] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 784.0 with 1 tasks -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_739_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 784.0 (TID 1808, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 784.0 (TID 1808) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 579 -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 784.0 (TID 1808). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 784.0 (TID 1808) in 3 ms on localhost (1/1) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 784.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 784 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 597 finished: isEmpty at LDAOptimizer.scala:418, took 0.009265 s -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_738_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 578 -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1004876, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_791 stored as values in memory (estimated size 176.0 B, free 729.6 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1005052, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_791_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.6 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_791_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_737_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 791 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 577 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_736_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 598 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 785(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_735_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 785 (MapPartitionsRDD[841] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 576 -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=929416, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_792 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8439) called with curMem=941222, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_792_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_734_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 575 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_792_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 792 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 785 (MapPartitionsRDD[841] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 785.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 785.0 (TID 1809, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 785.0 (TID 1810, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 785.0 (TID 1811, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 785.0 (TID 1812, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_733_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 785.0 (TID 1810) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 785.0 (TID 1809) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 785.0 (TID 1812) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 574 -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 785.0 (TID 1812). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_732_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 785.0 (TID 1810). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 785.0 (TID 1812) in 6 ms on localhost (1/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 785.0 (TID 1809). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 785.0 (TID 1810) in 8 ms on localhost (2/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 785.0 (TID 1809) in 8 ms on localhost (3/4) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 785.0 (TID 1811) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 785.0 (TID 1811). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_731_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 785.0 (TID 1811) in 12 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 785 (reduce at LDAOptimizer.scala:453) finished in 0.014 s -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 598 finished: reduce at LDAOptimizer.scala:453, took 0.019309 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 785.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 573 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_730_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 572 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 599 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 786(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 786 (MapPartitionsRDD[842] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=838807, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_793 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_729_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 571 -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8452) called with curMem=850667, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_793_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.8 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_728_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 570 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_793_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 793 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 786 (MapPartitionsRDD[842] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 786.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 786.0 (TID 1813, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 786.0 (TID 1814, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_727_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 786.0 (TID 1815, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 786.0 (TID 1816, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 786.0 (TID 1816) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 786.0 (TID 1813) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 786.0 (TID 1815) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 786.0 (TID 1814) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 786.0 (TID 1816). 1358 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 786.0 (TID 1816) in 4 ms on localhost (1/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 786.0 (TID 1814). 1402 bytes result sent to driver -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_726_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 786.0 (TID 1814) in 6 ms on localhost (2/4) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 569 -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 786.0 (TID 1815). 1358 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 786.0 (TID 1815) in 9 ms on localhost (3/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 786.0 (TID 1813). 1402 bytes result sent to driver -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_725_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 786 (reduce at LDAOptimizer.scala:456) finished in 0.012 s -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 786.0 (TID 1813) in 12 ms on localhost (4/4) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 786.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 568 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 599 finished: reduce at LDAOptimizer.scala:456, took 0.018845 s -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_724_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 567 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_723_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 600 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 787(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 787 (PartitionwiseSampledRDD[843] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=727581, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_794 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_722_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 566 -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=727581, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_794_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_794_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 794 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_721_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 787 (PartitionwiseSampledRDD[843] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 787.0 with 1 tasks -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 565 -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 787.0 (TID 1817, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 787.0 (TID 1817) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_720_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 564 -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 787.0 (TID 1817). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 787.0 (TID 1817) in 3 ms on localhost (1/1) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 787 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 787.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 600 finished: isEmpty at LDAOptimizer.scala:418, took 0.007671 s -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_719_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=690408, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_795 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 563 -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=670152, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_795_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_795_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_718_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 795 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_717_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 562 -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 561 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_715_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 601 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 788(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 560 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 788 (MapPartitionsRDD[845] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=606479, maxMem=766075207 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_714_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_796 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 559 -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8435) called with curMem=606479, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_796_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_796_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_713_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 796 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 788 (MapPartitionsRDD[845] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 788.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 788.0 (TID 1818, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 788.0 (TID 1819, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 788.0 (TID 1820, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 788.0 (TID 1821, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 788.0 (TID 1820) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 788.0 (TID 1821) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 788.0 (TID 1818) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_712_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 788.0 (TID 1819) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 788.0 (TID 1820). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 788.0 (TID 1821). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 788.0 (TID 1819). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 788.0 (TID 1820) in 5 ms on localhost (1/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 788.0 (TID 1821) in 4 ms on localhost (2/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 788.0 (TID 1819) in 6 ms on localhost (3/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 788.0 (TID 1818). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 788.0 (TID 1818) in 7 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 788 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 788.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 558 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 601 finished: reduce at LDAOptimizer.scala:453, took 0.014752 s -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_711_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 557 -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 602 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_710_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 789(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 789 (MapPartitionsRDD[846] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 556 -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=530532, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_797 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8448) called with curMem=550372, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_797_piece0 stored as bytes in memory (estimated size 8.3 KB, free 730.1 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_709_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_797_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 797 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 789 (MapPartitionsRDD[846] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 789.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 789.0 (TID 1822, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 789.0 (TID 1823, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_708_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 789.0 (TID 1824, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 555 -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 789.0 (TID 1825, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 789.0 (TID 1825) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 789.0 (TID 1823) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 789.0 (TID 1822) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 789.0 (TID 1824) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 789.0 (TID 1823). 1002 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 789.0 (TID 1825). 1002 bytes result sent to driver -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_707_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 789.0 (TID 1823) in 4 ms on localhost (1/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 789.0 (TID 1822). 1402 bytes result sent to driver -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 554 -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 789.0 (TID 1825) in 4 ms on localhost (2/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 789.0 (TID 1824). 1002 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 789.0 (TID 1824) in 4 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 789.0 (TID 1822) in 6 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 789 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 789.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 602 finished: reduce at LDAOptimizer.scala:456, took 0.012852 s -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_706_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 553 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_705_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 603 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 790(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_704_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 790 (PartitionwiseSampledRDD[847] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 552 -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=447916, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_798 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=437988, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_798_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_703_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.5 MB) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 551 -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_798_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 798 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 790 (PartitionwiseSampledRDD[847] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 790.0 with 1 tasks -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_702_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 790.0 (TID 1826, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 790.0 (TID 1826) -2016-06-13 19:45:27 INFO ContextCleaner:59 - Cleaned accumulator 550 -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 790.0 (TID 1826). 911 bytes result sent to driver -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Removed broadcast_701_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 790 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 790.0 (TID 1826) in 3 ms on localhost (1/1) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 790.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 603 finished: isEmpty at LDAOptimizer.scala:418, took 0.009352 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 604 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 791(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 791 (PartitionwiseSampledRDD[847] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=416500, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_799 stored as values in memory (estimated size 18.4 KB, free 730.2 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=435316, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_799_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_799_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 799 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 791 (PartitionwiseSampledRDD[847] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 791.0 with 3 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 791.0 (TID 1827, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 791.0 (TID 1828, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 791.0 (TID 1829, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 791.0 (TID 1829) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 791.0 (TID 1828) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 791.0 (TID 1827) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 791.0 (TID 1829). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 791.0 (TID 1827). 911 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 791.0 (TID 1828). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 791.0 (TID 1827) in 5 ms on localhost (1/3) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 791.0 (TID 1829) in 4 ms on localhost (2/3) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 791.0 (TID 1828) in 4 ms on localhost (3/3) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 791 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 791.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 604 finished: isEmpty at LDAOptimizer.scala:418, took 0.009991 s -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=442976, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_800 stored as values in memory (estimated size 176.0 B, free 730.2 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=443152, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_800_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.2 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_800_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 800 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 605 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 792(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 792 (MapPartitionsRDD[849] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=443432, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_801 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=463688, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_801_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_801_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 801 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 792 (MapPartitionsRDD[849] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 792.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 792.0 (TID 1830, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 792.0 (TID 1831, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 792.0 (TID 1832, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 792.0 (TID 1833, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 792.0 (TID 1831) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 792.0 (TID 1832) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 792.0 (TID 1833) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 792.0 (TID 1830) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 792.0 (TID 1831). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 792.0 (TID 1831) in 4 ms on localhost (1/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 792.0 (TID 1830). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 792.0 (TID 1832). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 792.0 (TID 1830) in 5 ms on localhost (2/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 792.0 (TID 1833). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 792.0 (TID 1832) in 5 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 792.0 (TID 1833) in 5 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 792 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 792.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 605 finished: reduce at LDAOptimizer.scala:453, took 0.011167 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 606 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 793(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 793 (MapPartitionsRDD[850] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=472120, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_802 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8445) called with curMem=492416, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_802_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_802_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 802 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 793 (MapPartitionsRDD[850] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 793.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 793.0 (TID 1834, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 793.0 (TID 1835, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 793.0 (TID 1836, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 793.0 (TID 1837, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 793.0 (TID 1835) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 793.0 (TID 1836) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 793.0 (TID 1837) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 793.0 (TID 1834) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 793.0 (TID 1835). 1002 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 793.0 (TID 1834). 1002 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 793.0 (TID 1835) in 4 ms on localhost (1/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 793.0 (TID 1837). 1358 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 793.0 (TID 1834) in 4 ms on localhost (2/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 793.0 (TID 1837) in 4 ms on localhost (3/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 793.0 (TID 1836). 1358 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 793.0 (TID 1836) in 7 ms on localhost (4/4) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 793.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 793 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 606 finished: reduce at LDAOptimizer.scala:456, took 0.012862 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 607 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 794(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 794 (PartitionwiseSampledRDD[851] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=500861, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_803 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7664) called with curMem=519677, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_803_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_803_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 803 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 794 (PartitionwiseSampledRDD[851] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 794.0 with 1 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 794.0 (TID 1838, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 794.0 (TID 1838) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 794.0 (TID 1838). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 794.0 (TID 1838) in 2 ms on localhost (1/1) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 794 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 794.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 607 finished: isEmpty at LDAOptimizer.scala:418, took 0.007272 s -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=527341, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_804 stored as values in memory (estimated size 176.0 B, free 730.1 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=527517, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_804_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.1 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_804_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 804 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 608 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 795(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 795 (MapPartitionsRDD[853] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=527797, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_805 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8438) called with curMem=548053, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_805_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_805_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 805 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 795 (MapPartitionsRDD[853] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 795.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 795.0 (TID 1839, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 795.0 (TID 1840, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 795.0 (TID 1841, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 795.0 (TID 1842, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 795.0 (TID 1839) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 795.0 (TID 1842) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 795.0 (TID 1840) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 795.0 (TID 1841) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 795.0 (TID 1841). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 795.0 (TID 1841) in 3 ms on localhost (1/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 795.0 (TID 1840). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 795.0 (TID 1839). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 795.0 (TID 1840) in 4 ms on localhost (2/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 795.0 (TID 1842). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 795.0 (TID 1839) in 5 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 795.0 (TID 1842) in 4 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 795 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 795.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 608 finished: reduce at LDAOptimizer.scala:453, took 0.010523 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 609 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 796(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 796 (MapPartitionsRDD[854] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=556491, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_806 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8450) called with curMem=576787, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_806_piece0 stored as bytes in memory (estimated size 8.3 KB, free 730.0 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_806_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 806 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 796 (MapPartitionsRDD[854] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 796.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 796.0 (TID 1843, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 796.0 (TID 1844, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 796.0 (TID 1845, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 796.0 (TID 1846, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 796.0 (TID 1844) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 796.0 (TID 1843) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 796.0 (TID 1845) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 796.0 (TID 1846) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 796.0 (TID 1843). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 796.0 (TID 1845). 1002 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 796.0 (TID 1844). 1358 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 796.0 (TID 1846). 1446 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 796.0 (TID 1843) in 3 ms on localhost (1/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 796.0 (TID 1845) in 4 ms on localhost (2/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 796.0 (TID 1844) in 4 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 796.0 (TID 1846) in 4 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 796 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 796.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 609 finished: reduce at LDAOptimizer.scala:456, took 0.009657 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 610 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 797(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 797 (PartitionwiseSampledRDD[855] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=585237, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_807 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=604053, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_807_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_807_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 807 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 797 (PartitionwiseSampledRDD[855] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 797.0 with 1 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 797.0 (TID 1847, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 797.0 (TID 1847) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 797.0 (TID 1847). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 797.0 (TID 1847) in 2 ms on localhost (1/1) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 797 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 797.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 610 finished: isEmpty at LDAOptimizer.scala:418, took 0.006844 s -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=611713, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_808 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=611889, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_808_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_808_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 808 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 611 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 798(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 798 (MapPartitionsRDD[857] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=612169, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_809 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8435) called with curMem=632425, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_809_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_809_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 809 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 798 (MapPartitionsRDD[857] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 798.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 798.0 (TID 1848, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 798.0 (TID 1849, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 798.0 (TID 1850, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 798.0 (TID 1851, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 798.0 (TID 1848) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 798.0 (TID 1849) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 798.0 (TID 1851) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 798.0 (TID 1850) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 798.0 (TID 1849). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 798.0 (TID 1851). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 798.0 (TID 1848). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 798.0 (TID 1849) in 5 ms on localhost (1/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 798.0 (TID 1850). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 798.0 (TID 1848) in 5 ms on localhost (2/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 798.0 (TID 1851) in 5 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 798.0 (TID 1850) in 6 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 798 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 798.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 611 finished: reduce at LDAOptimizer.scala:453, took 0.010898 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 612 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 799(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 799 (MapPartitionsRDD[858] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=640860, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_810 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8448) called with curMem=661156, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_810_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.9 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_810_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 810 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 799 (MapPartitionsRDD[858] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 799.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 799.0 (TID 1852, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 799.0 (TID 1853, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 799.0 (TID 1854, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 799.0 (TID 1855, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 799.0 (TID 1852) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 799.0 (TID 1855) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 799.0 (TID 1853) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 799.0 (TID 1854) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 799.0 (TID 1854). 1002 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 799.0 (TID 1852). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 799.0 (TID 1855). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 799.0 (TID 1854) in 3 ms on localhost (1/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 799.0 (TID 1853). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 799.0 (TID 1852) in 4 ms on localhost (2/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 799.0 (TID 1855) in 4 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 799.0 (TID 1853) in 4 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 799 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 799.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 612 finished: reduce at LDAOptimizer.scala:456, took 0.010322 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 613 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 800(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 800 (PartitionwiseSampledRDD[859] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=669604, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_811 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=688420, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_811_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_811_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 811 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 800 (PartitionwiseSampledRDD[859] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 800.0 with 1 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 800.0 (TID 1856, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 800.0 (TID 1856) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 800.0 (TID 1856). 911 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 800.0 (TID 1856) in 2 ms on localhost (1/1) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 800 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 800.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 613 finished: isEmpty at LDAOptimizer.scala:418, took 0.008237 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 614 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 801(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 801 (PartitionwiseSampledRDD[859] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=696080, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_812 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=714896, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_812_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_812_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 812 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 801 (PartitionwiseSampledRDD[859] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 801.0 with 3 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 801.0 (TID 1857, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 801.0 (TID 1858, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 801.0 (TID 1859, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 801.0 (TID 1857) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 801.0 (TID 1858) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 801.0 (TID 1859) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 801.0 (TID 1858). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 801.0 (TID 1857). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 801.0 (TID 1859). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 801.0 (TID 1858) in 3 ms on localhost (1/3) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 801.0 (TID 1857) in 3 ms on localhost (2/3) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 801.0 (TID 1859) in 3 ms on localhost (3/3) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 801 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 801.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 614 finished: isEmpty at LDAOptimizer.scala:418, took 0.009046 s -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=722556, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_813 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=722732, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_813_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_813_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 813 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 615 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 802(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 802 (MapPartitionsRDD[861] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=723012, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_814 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=743268, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_814_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_814_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 814 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 802 (MapPartitionsRDD[861] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 802.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 802.0 (TID 1860, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 802.0 (TID 1861, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 802.0 (TID 1862, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 802.0 (TID 1863, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 802.0 (TID 1860) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 802.0 (TID 1863) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 802.0 (TID 1862) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 802.0 (TID 1861) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 802.0 (TID 1860). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 802.0 (TID 1863). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 802.0 (TID 1861). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 802.0 (TID 1860) in 5 ms on localhost (1/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 802.0 (TID 1861) in 6 ms on localhost (2/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 802.0 (TID 1862). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 802.0 (TID 1863) in 6 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 802.0 (TID 1862) in 6 ms on localhost (4/4) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 802.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 802 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 615 finished: reduce at LDAOptimizer.scala:453, took 0.013880 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 616 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 803(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 803 (MapPartitionsRDD[862] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=751701, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_815 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8446) called with curMem=771997, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_815_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_815_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 815 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 803 (MapPartitionsRDD[862] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 803.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 803.0 (TID 1864, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 803.0 (TID 1865, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 803.0 (TID 1866, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 803.0 (TID 1867, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 803.0 (TID 1866) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 803.0 (TID 1864) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 803.0 (TID 1866). 1358 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 803.0 (TID 1864). 1002 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 803.0 (TID 1867) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 803.0 (TID 1865) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 803.0 (TID 1866) in 6 ms on localhost (1/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 803.0 (TID 1864) in 7 ms on localhost (2/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 803.0 (TID 1867). 1402 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 803.0 (TID 1865). 1358 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 803.0 (TID 1867) in 9 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 803.0 (TID 1865) in 11 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 803 (reduce at LDAOptimizer.scala:456) finished in 0.012 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 803.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 616 finished: reduce at LDAOptimizer.scala:456, took 0.019608 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 617 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 804(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 804 (PartitionwiseSampledRDD[863] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=780443, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_816 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=799259, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_816_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_816_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 816 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 804 (PartitionwiseSampledRDD[863] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 804.0 with 1 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 804.0 (TID 1868, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 804.0 (TID 1868) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 804.0 (TID 1868). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 804.0 (TID 1868) in 2 ms on localhost (1/1) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 804 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 804.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 617 finished: isEmpty at LDAOptimizer.scala:418, took 0.007291 s -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=806921, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_817 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=807097, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_817_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.8 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_817_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 817 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 618 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 805(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 805 (MapPartitionsRDD[865] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=807377, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_818 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8437) called with curMem=827633, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_818_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_818_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 818 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 805 (MapPartitionsRDD[865] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 805.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 805.0 (TID 1869, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 805.0 (TID 1870, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 805.0 (TID 1871, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 805.0 (TID 1872, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 805.0 (TID 1871) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 805.0 (TID 1872) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 805.0 (TID 1869) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 805.0 (TID 1870) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 805.0 (TID 1870). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 805.0 (TID 1869). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 805.0 (TID 1872). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 805.0 (TID 1870) in 4 ms on localhost (1/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 805.0 (TID 1871). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 805.0 (TID 1872) in 4 ms on localhost (2/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 805.0 (TID 1869) in 5 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 805.0 (TID 1871) in 5 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 805 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 805.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 618 finished: reduce at LDAOptimizer.scala:453, took 0.010368 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 619 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 806(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 806 (MapPartitionsRDD[866] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=836070, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_819 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8450) called with curMem=856366, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_819_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.8 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_819_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 819 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 806 (MapPartitionsRDD[866] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 806.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 806.0 (TID 1873, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 806.0 (TID 1874, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 806.0 (TID 1875, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 806.0 (TID 1876, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 806.0 (TID 1875) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 806.0 (TID 1876) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 806.0 (TID 1874) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 806.0 (TID 1873) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 806.0 (TID 1875). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 806.0 (TID 1874). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 806.0 (TID 1876). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 806.0 (TID 1875) in 4 ms on localhost (1/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 806.0 (TID 1873). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 806.0 (TID 1876) in 5 ms on localhost (2/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 806.0 (TID 1874) in 6 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 806.0 (TID 1873) in 6 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 806 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 806.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 619 finished: reduce at LDAOptimizer.scala:456, took 0.012542 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 620 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 807(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 807 (PartitionwiseSampledRDD[867] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=864816, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_820 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7667) called with curMem=883632, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_820_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_820_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 820 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 807 (PartitionwiseSampledRDD[867] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 807.0 with 1 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 807.0 (TID 1877, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 807.0 (TID 1877) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 807.0 (TID 1877). 911 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 807.0 (TID 1877) in 3 ms on localhost (1/1) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 807 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 807.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 620 finished: isEmpty at LDAOptimizer.scala:418, took 0.007026 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 621 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 808(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 808 (PartitionwiseSampledRDD[867] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=891299, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_821 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7667) called with curMem=910115, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_821_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_821_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 821 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 808 (PartitionwiseSampledRDD[867] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 808.0 with 3 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 808.0 (TID 1878, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 808.0 (TID 1879, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 808.0 (TID 1880, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 808.0 (TID 1880) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 808.0 (TID 1879) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 808.0 (TID 1878) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 808.0 (TID 1880). 911 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 808.0 (TID 1878). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 808.0 (TID 1880) in 3 ms on localhost (1/3) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 808.0 (TID 1879). 911 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 808.0 (TID 1878) in 4 ms on localhost (2/3) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 808.0 (TID 1879) in 4 ms on localhost (3/3) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 808 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 808.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 621 finished: isEmpty at LDAOptimizer.scala:418, took 0.008821 s -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=917782, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_822 stored as values in memory (estimated size 176.0 B, free 729.7 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=917958, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_822_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.7 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_822_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 822 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 622 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 809(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 809 (MapPartitionsRDD[869] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=918238, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_823 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8442) called with curMem=938494, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_823_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_823_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 823 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 809 (MapPartitionsRDD[869] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 809.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 809.0 (TID 1881, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 809.0 (TID 1882, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 809.0 (TID 1883, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 809.0 (TID 1884, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 809.0 (TID 1881) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 809.0 (TID 1883) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 809.0 (TID 1884) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 809.0 (TID 1882) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 809.0 (TID 1883). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 809.0 (TID 1884). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 809.0 (TID 1883) in 3 ms on localhost (1/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 809.0 (TID 1884) in 4 ms on localhost (2/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 809.0 (TID 1882). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 809.0 (TID 1881). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 809.0 (TID 1882) in 4 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 809.0 (TID 1881) in 6 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 809 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 809.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 622 finished: reduce at LDAOptimizer.scala:453, took 0.010294 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 623 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 810(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 810 (MapPartitionsRDD[870] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=946936, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_824 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8454) called with curMem=967232, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_824_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.7 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_824_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 824 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 810 (MapPartitionsRDD[870] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 810.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 810.0 (TID 1885, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 810.0 (TID 1886, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 810.0 (TID 1887, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 810.0 (TID 1888, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 810.0 (TID 1886) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 810.0 (TID 1888) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 810.0 (TID 1887) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 810.0 (TID 1885) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 810.0 (TID 1887). 1002 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 810.0 (TID 1885). 1002 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 810.0 (TID 1888). 1002 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 810.0 (TID 1887) in 3 ms on localhost (1/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 810.0 (TID 1886). 1358 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 810.0 (TID 1885) in 4 ms on localhost (2/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 810.0 (TID 1888) in 4 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 810.0 (TID 1886) in 5 ms on localhost (4/4) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 810.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 810 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 623 finished: reduce at LDAOptimizer.scala:456, took 0.010831 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 624 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 811(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 811 (PartitionwiseSampledRDD[871] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=975686, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_825 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7659) called with curMem=994502, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_825_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_825_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 825 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 811 (PartitionwiseSampledRDD[871] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 811.0 with 1 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 811.0 (TID 1889, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 811.0 (TID 1889) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 811.0 (TID 1889). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 811.0 (TID 1889) in 3 ms on localhost (1/1) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 811 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 811.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 624 finished: isEmpty at LDAOptimizer.scala:418, took 0.006901 s -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1002161, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_826 stored as values in memory (estimated size 176.0 B, free 729.6 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1002337, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_826_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.6 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_826_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 826 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 625 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 812(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 812 (MapPartitionsRDD[873] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1002617, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_827 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8434) called with curMem=1022873, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_827_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_827_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 827 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 812 (MapPartitionsRDD[873] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 812.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 812.0 (TID 1890, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 812.0 (TID 1891, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 812.0 (TID 1892, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 812.0 (TID 1893, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 812.0 (TID 1892) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 812.0 (TID 1893) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 812.0 (TID 1891) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 812.0 (TID 1890) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 812.0 (TID 1892). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 812.0 (TID 1891). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 812.0 (TID 1890). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 812.0 (TID 1893). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 812.0 (TID 1892) in 3 ms on localhost (1/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 812.0 (TID 1891) in 3 ms on localhost (2/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 812.0 (TID 1890) in 4 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 812.0 (TID 1893) in 4 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 812 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 812.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 625 finished: reduce at LDAOptimizer.scala:453, took 0.008968 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 626 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 813(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 813 (MapPartitionsRDD[874] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1031307, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_828 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8447) called with curMem=1051603, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_828_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_828_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 828 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 813 (MapPartitionsRDD[874] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 813.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 813.0 (TID 1894, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 813.0 (TID 1895, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 813.0 (TID 1896, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 813.0 (TID 1897, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 813.0 (TID 1895) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 813.0 (TID 1897) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 813.0 (TID 1896) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 813.0 (TID 1894) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 813.0 (TID 1896). 1002 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 813.0 (TID 1894). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 813.0 (TID 1897). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 813.0 (TID 1895). 1358 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 813.0 (TID 1896) in 8 ms on localhost (1/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 813.0 (TID 1897) in 9 ms on localhost (2/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 813.0 (TID 1894) in 10 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 813.0 (TID 1895) in 10 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 813 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 626 finished: reduce at LDAOptimizer.scala:456, took 0.017732 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 813.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 627 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 814(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 814 (PartitionwiseSampledRDD[875] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1060050, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_829 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=1078866, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_829_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_829_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 829 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 814 (PartitionwiseSampledRDD[875] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 814.0 with 1 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 814.0 (TID 1898, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 814.0 (TID 1898) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 814.0 (TID 1898). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 814.0 (TID 1898) in 2 ms on localhost (1/1) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 814.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 814 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 627 finished: isEmpty at LDAOptimizer.scala:418, took 0.007048 s -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1086528, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_830 stored as values in memory (estimated size 176.0 B, free 729.5 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1086704, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_830_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.5 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_830_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 830 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 628 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 815(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 815 (MapPartitionsRDD[877] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1086984, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_831 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8434) called with curMem=1107240, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_831_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_831_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 831 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 815 (MapPartitionsRDD[877] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 815.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 815.0 (TID 1899, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 815.0 (TID 1900, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 815.0 (TID 1901, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 815.0 (TID 1902, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 815.0 (TID 1901) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 815.0 (TID 1902) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 815.0 (TID 1899) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 815.0 (TID 1900) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 815.0 (TID 1902). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 815.0 (TID 1901). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 815.0 (TID 1899). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 815.0 (TID 1900). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 815.0 (TID 1902) in 3 ms on localhost (1/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 815.0 (TID 1901) in 4 ms on localhost (2/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 815.0 (TID 1899) in 4 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 815.0 (TID 1900) in 4 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 815 (reduce at LDAOptimizer.scala:453) finished in 0.004 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 815.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 628 finished: reduce at LDAOptimizer.scala:453, took 0.009115 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 629 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 816(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 816 (MapPartitionsRDD[878] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1115674, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_832 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8448) called with curMem=1135970, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_832_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.5 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_832_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 832 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 816 (MapPartitionsRDD[878] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 816.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 816.0 (TID 1903, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 816.0 (TID 1904, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 816.0 (TID 1905, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 816.0 (TID 1906, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 816.0 (TID 1905) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 816.0 (TID 1906) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 816.0 (TID 1903) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 816.0 (TID 1904) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 816.0 (TID 1905). 1002 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 816.0 (TID 1906). 1002 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 816.0 (TID 1905) in 3 ms on localhost (1/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 816.0 (TID 1906) in 3 ms on localhost (2/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 816.0 (TID 1904). 1358 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 816.0 (TID 1903). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 816.0 (TID 1904) in 3 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 816.0 (TID 1903) in 5 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 816 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 816.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 629 finished: reduce at LDAOptimizer.scala:456, took 0.009251 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 630 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 817(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 817 (PartitionwiseSampledRDD[879] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1144418, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_833 stored as values in memory (estimated size 18.4 KB, free 729.5 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=1163234, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_833_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.5 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_833_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 833 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 817 (PartitionwiseSampledRDD[879] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 817.0 with 1 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 817.0 (TID 1907, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 817.0 (TID 1907) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 817.0 (TID 1907). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 817.0 (TID 1907) in 2 ms on localhost (1/1) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 817 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 817.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 630 finished: isEmpty at LDAOptimizer.scala:418, took 0.006500 s -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1170895, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_834 stored as values in memory (estimated size 176.0 B, free 729.5 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1171071, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_834_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.5 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_834_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 834 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 631 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 818(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 818 (MapPartitionsRDD[881] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1171351, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_835 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8435) called with curMem=1191607, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_835_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_835_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 835 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 818 (MapPartitionsRDD[881] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 818.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 818.0 (TID 1908, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 818.0 (TID 1909, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 818.0 (TID 1910, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 818.0 (TID 1911, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 818.0 (TID 1908) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 818.0 (TID 1911) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 818.0 (TID 1909) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 818.0 (TID 1910) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 818.0 (TID 1911). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 818.0 (TID 1909). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 818.0 (TID 1911) in 3 ms on localhost (1/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 818.0 (TID 1910). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 818.0 (TID 1909) in 4 ms on localhost (2/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 818.0 (TID 1910) in 4 ms on localhost (3/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 818.0 (TID 1908). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 818.0 (TID 1908) in 5 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 818 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 818.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 631 finished: reduce at LDAOptimizer.scala:453, took 0.010954 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 632 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 819(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 819 (MapPartitionsRDD[882] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1200042, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_836 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8448) called with curMem=1220338, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_836_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.4 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_836_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 836 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 819 (MapPartitionsRDD[882] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 819.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 819.0 (TID 1912, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 819.0 (TID 1913, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 819.0 (TID 1914, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 819.0 (TID 1915, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 819.0 (TID 1914) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 819.0 (TID 1915) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 819.0 (TID 1912) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 819.0 (TID 1913) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 819.0 (TID 1912). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 819.0 (TID 1915). 1002 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 819.0 (TID 1913). 1358 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 819.0 (TID 1914). 1358 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 819.0 (TID 1912) in 4 ms on localhost (1/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 819.0 (TID 1915) in 3 ms on localhost (2/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 819.0 (TID 1913) in 4 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 819.0 (TID 1914) in 5 ms on localhost (4/4) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 819.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 819 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 632 finished: reduce at LDAOptimizer.scala:456, took 0.009890 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 633 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 820(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 820 (PartitionwiseSampledRDD[883] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1228786, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_837 stored as values in memory (estimated size 18.4 KB, free 729.4 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=1247602, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_837_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.4 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_837_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 837 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 820 (PartitionwiseSampledRDD[883] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 820.0 with 1 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 820.0 (TID 1916, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 820.0 (TID 1916) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 820.0 (TID 1916). 911 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 820.0 (TID 1916) in 3 ms on localhost (1/1) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 820 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 633 finished: isEmpty at LDAOptimizer.scala:418, took 0.008500 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 820.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 634 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 821(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 821 (PartitionwiseSampledRDD[883] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1255262, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_838 stored as values in memory (estimated size 18.4 KB, free 729.4 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=1274078, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_838_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.4 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_838_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 838 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 821 (PartitionwiseSampledRDD[883] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 821.0 with 3 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 821.0 (TID 1917, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 821.0 (TID 1918, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 821.0 (TID 1919, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 821.0 (TID 1917) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 821.0 (TID 1919) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 821.0 (TID 1918) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 821.0 (TID 1919). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 821.0 (TID 1917). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 821.0 (TID 1918). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 821.0 (TID 1919) in 3 ms on localhost (1/3) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 821.0 (TID 1917) in 4 ms on localhost (2/3) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 821.0 (TID 1918) in 3 ms on localhost (3/3) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 821 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 821.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 634 finished: isEmpty at LDAOptimizer.scala:418, took 0.009878 s -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1281738, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_839 stored as values in memory (estimated size 176.0 B, free 729.4 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1281914, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_839_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.4 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_839_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 839 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 635 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 822(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 822 (MapPartitionsRDD[885] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1282194, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_840 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8432) called with curMem=1302450, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_840_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.3 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_840_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 840 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 822 (MapPartitionsRDD[885] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 822.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 822.0 (TID 1920, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 822.0 (TID 1921, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 822.0 (TID 1922, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 822.0 (TID 1923, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 822.0 (TID 1920) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 822.0 (TID 1921) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 822.0 (TID 1922) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 822.0 (TID 1923) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 822.0 (TID 1920). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 822.0 (TID 1920) in 4 ms on localhost (1/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 822.0 (TID 1921). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 822.0 (TID 1922). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 822.0 (TID 1921) in 5 ms on localhost (2/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 822.0 (TID 1923). 1265 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 822.0 (TID 1922) in 6 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 822.0 (TID 1923) in 6 ms on localhost (4/4) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 822.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 822 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 635 finished: reduce at LDAOptimizer.scala:453, took 0.013630 s -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 636 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 823(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 823 (MapPartitionsRDD[886] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1310882, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_841 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(8445) called with curMem=1331178, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_841_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.3 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_841_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 841 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 823 (MapPartitionsRDD[886] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 823.0 with 4 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 823.0 (TID 1924, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 823.0 (TID 1925, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 823.0 (TID 1926, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 3.0 in stage 823.0 (TID 1927, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 2.0 in stage 823.0 (TID 1926) -2016-06-13 19:45:27 INFO Executor:59 - Running task 1.0 in stage 823.0 (TID 1925) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 823.0 (TID 1924) -2016-06-13 19:45:27 INFO Executor:59 - Running task 3.0 in stage 823.0 (TID 1927) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 823.0 (TID 1924). 1002 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 1.0 in stage 823.0 (TID 1925). 1402 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 823.0 (TID 1924) in 6 ms on localhost (1/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 823.0 (TID 1925) in 6 ms on localhost (2/4) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 3.0 in stage 823.0 (TID 1927). 1358 bytes result sent to driver -2016-06-13 19:45:27 INFO Executor:59 - Finished task 2.0 in stage 823.0 (TID 1926). 1314 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 3.0 in stage 823.0 (TID 1927) in 7 ms on localhost (3/4) -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 823.0 (TID 1926) in 8 ms on localhost (4/4) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 823 (reduce at LDAOptimizer.scala:456) finished in 0.009 s -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 636 finished: reduce at LDAOptimizer.scala:456, took 0.016470 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 823.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Got job 637 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:27 INFO DAGScheduler:59 - Final stage: ResultStage 824(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:27 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting ResultStage 824 (PartitionwiseSampledRDD[887] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1339623, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_842 stored as values in memory (estimated size 18.4 KB, free 729.3 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=1358439, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_842_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.3 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_842_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 842 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:27 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 824 (PartitionwiseSampledRDD[887] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Adding task set 824.0 with 1 tasks -2016-06-13 19:45:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 824.0 (TID 1928, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:27 INFO Executor:59 - Running task 0.0 in stage 824.0 (TID 1928) -2016-06-13 19:45:27 INFO Executor:59 - Finished task 0.0 in stage 824.0 (TID 1928). 1226 bytes result sent to driver -2016-06-13 19:45:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 824.0 (TID 1928) in 3 ms on localhost (1/1) -2016-06-13 19:45:27 INFO DAGScheduler:59 - ResultStage 824 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 824.0, whose tasks have all completed, from pool -2016-06-13 19:45:27 INFO DAGScheduler:59 - Job 637 finished: isEmpty at LDAOptimizer.scala:418, took 0.008423 s -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1366100, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_843 stored as values in memory (estimated size 176.0 B, free 729.3 MB) -2016-06-13 19:45:27 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1366276, maxMem=766075207 -2016-06-13 19:45:27 INFO MemoryStore:59 - Block broadcast_843_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.3 MB) -2016-06-13 19:45:27 INFO BlockManagerInfo:59 - Added broadcast_843_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:27 INFO SparkContext:59 - Created broadcast 843 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 638 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 825(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 825 (MapPartitionsRDD[889] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1366556, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_844 stored as values in memory (estimated size 19.8 KB, free 729.3 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8431) called with curMem=1386812, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_844_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.3 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_844_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 844 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 825 (MapPartitionsRDD[889] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 825.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 825.0 (TID 1929, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 825.0 (TID 1930, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 825.0 (TID 1931, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 825.0 (TID 1932, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 825.0 (TID 1929) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 825.0 (TID 1932) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 825.0 (TID 1931) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 825.0 (TID 1930) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 825.0 (TID 1931). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 825.0 (TID 1931) in 5 ms on localhost (1/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 825.0 (TID 1929). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 825.0 (TID 1932). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 825.0 (TID 1929) in 8 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 825.0 (TID 1932) in 8 ms on localhost (3/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 825.0 (TID 1930). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 825.0 (TID 1930) in 9 ms on localhost (4/4) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 825 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 825.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 638 finished: reduce at LDAOptimizer.scala:453, took 0.014141 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 639 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 826(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 826 (MapPartitionsRDD[890] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1395243, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_845 stored as values in memory (estimated size 19.8 KB, free 729.2 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8444) called with curMem=1415539, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_845_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.2 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_845_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 845 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 826 (MapPartitionsRDD[890] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 826.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 826.0 (TID 1933, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 826.0 (TID 1934, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 826.0 (TID 1935, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 826.0 (TID 1936, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 826.0 (TID 1934) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 826.0 (TID 1936) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 826.0 (TID 1933) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 826.0 (TID 1935) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 826.0 (TID 1934). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 826.0 (TID 1936). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 826.0 (TID 1935). 1358 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 826.0 (TID 1934) in 4 ms on localhost (1/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 826.0 (TID 1933). 1358 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 826.0 (TID 1936) in 3 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 826.0 (TID 1935) in 4 ms on localhost (3/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 826.0 (TID 1933) in 5 ms on localhost (4/4) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 826 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 826.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 639 finished: reduce at LDAOptimizer.scala:456, took 0.010727 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 640 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 827(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 827 (PartitionwiseSampledRDD[891] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1423983, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_846 stored as values in memory (estimated size 18.4 KB, free 729.2 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=1442799, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_846_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.2 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_846_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 846 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 827 (PartitionwiseSampledRDD[891] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 827.0 with 1 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 827.0 (TID 1937, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 827.0 (TID 1937) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 827.0 (TID 1937). 1226 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 827.0 (TID 1937) in 2 ms on localhost (1/1) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 827 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 827.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 640 finished: isEmpty at LDAOptimizer.scala:418, took 0.006810 s -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1450459, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_847 stored as values in memory (estimated size 176.0 B, free 729.2 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1450635, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_847_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.2 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_847_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 847 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 641 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 828(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 828 (MapPartitionsRDD[893] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1450915, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_848 stored as values in memory (estimated size 19.8 KB, free 729.2 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8437) called with curMem=1471171, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_848_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.2 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_848_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 848 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 828 (MapPartitionsRDD[893] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 828.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 828.0 (TID 1938, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 828.0 (TID 1939, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 828.0 (TID 1940, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 828.0 (TID 1941, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 828.0 (TID 1938) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 828.0 (TID 1940) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 828.0 (TID 1941) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 828.0 (TID 1940). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 828.0 (TID 1941). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 828.0 (TID 1938). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 828.0 (TID 1940) in 7 ms on localhost (1/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 828.0 (TID 1941) in 7 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 828.0 (TID 1938) in 8 ms on localhost (3/4) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 828.0 (TID 1939) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 828.0 (TID 1939). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 828.0 (TID 1939) in 20 ms on localhost (4/4) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 828.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 828 (reduce at LDAOptimizer.scala:453) finished in 0.021 s -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 641 finished: reduce at LDAOptimizer.scala:453, took 0.027941 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 642 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 829(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 829 (MapPartitionsRDD[894] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1479608, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_849 stored as values in memory (estimated size 19.8 KB, free 729.2 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8448) called with curMem=1499904, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_849_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.1 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_849_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 849 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 829 (MapPartitionsRDD[894] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 829.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 829.0 (TID 1942, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 829.0 (TID 1943, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 829.0 (TID 1944, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 829.0 (TID 1945, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 829.0 (TID 1942) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 829.0 (TID 1945) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 829.0 (TID 1944) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 829.0 (TID 1943) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 829.0 (TID 1942). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 829.0 (TID 1944). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 829.0 (TID 1945). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 829.0 (TID 1942) in 5 ms on localhost (1/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 829.0 (TID 1944) in 5 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 829.0 (TID 1945) in 4 ms on localhost (3/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 829.0 (TID 1943). 1358 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 829.0 (TID 1943) in 6 ms on localhost (4/4) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 829.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 829 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 642 finished: reduce at LDAOptimizer.scala:456, took 0.012277 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 643 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 830(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 830 (PartitionwiseSampledRDD[895] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1508352, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_850 stored as values in memory (estimated size 18.4 KB, free 729.1 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=1527168, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_850_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.1 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_850_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 850 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 830 (PartitionwiseSampledRDD[895] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 830.0 with 1 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 830.0 (TID 1946, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 830.0 (TID 1946) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 830.0 (TID 1946). 1226 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 830.0 (TID 1946) in 2 ms on localhost (1/1) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 830 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 830.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 643 finished: isEmpty at LDAOptimizer.scala:418, took 0.007566 s -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1534828, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_851 stored as values in memory (estimated size 176.0 B, free 729.1 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1535004, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_851_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.1 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_851_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 851 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 644 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 831(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 831 (MapPartitionsRDD[897] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1535284, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_852 stored as values in memory (estimated size 19.8 KB, free 729.1 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=1555540, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_852_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.1 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_852_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 852 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 831 (MapPartitionsRDD[897] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 831.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 831.0 (TID 1947, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 831.0 (TID 1948, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 831.0 (TID 1949, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 831.0 (TID 1950, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 831.0 (TID 1947) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 831.0 (TID 1949) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 831.0 (TID 1950) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 831.0 (TID 1948) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 831.0 (TID 1947). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 831.0 (TID 1950). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 831.0 (TID 1947) in 6 ms on localhost (1/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 831.0 (TID 1950) in 5 ms on localhost (2/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 831.0 (TID 1948). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 831.0 (TID 1948) in 8 ms on localhost (3/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 831.0 (TID 1949). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 831.0 (TID 1949) in 8 ms on localhost (4/4) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 831.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 831 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 644 finished: reduce at LDAOptimizer.scala:453, took 0.018161 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 645 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 832(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 832 (MapPartitionsRDD[898] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1563973, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_853 stored as values in memory (estimated size 19.8 KB, free 729.1 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8446) called with curMem=1584269, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_853_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.1 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_853_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 853 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 832 (MapPartitionsRDD[898] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 832.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 832.0 (TID 1951, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 832.0 (TID 1952, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 832.0 (TID 1953, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 832.0 (TID 1954, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 832.0 (TID 1952) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 832.0 (TID 1953) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 832.0 (TID 1954) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 832.0 (TID 1951) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 832.0 (TID 1953). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 832.0 (TID 1954). 1358 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 832.0 (TID 1953) in 4 ms on localhost (1/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 832.0 (TID 1951). 1402 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 832.0 (TID 1952). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 832.0 (TID 1954) in 5 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 832.0 (TID 1952) in 9 ms on localhost (3/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 832.0 (TID 1951) in 9 ms on localhost (4/4) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 832.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 832 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 645 finished: reduce at LDAOptimizer.scala:456, took 0.016386 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 646 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 833(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 833 (PartitionwiseSampledRDD[899] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1592715, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_854 stored as values in memory (estimated size 18.4 KB, free 729.0 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=1611531, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_854_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.0 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_854_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 854 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 833 (PartitionwiseSampledRDD[899] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 833.0 with 1 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 833.0 (TID 1955, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 833.0 (TID 1955) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 833.0 (TID 1955). 1226 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 833.0 (TID 1955) in 4 ms on localhost (1/1) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 833.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 833 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 646 finished: isEmpty at LDAOptimizer.scala:418, took 0.011780 s -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1619193, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_855 stored as values in memory (estimated size 176.0 B, free 729.0 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1619369, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_855_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.0 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_855_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 855 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 647 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 834(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 834 (MapPartitionsRDD[901] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1619649, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_856 stored as values in memory (estimated size 19.8 KB, free 729.0 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8435) called with curMem=1639905, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_856_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.0 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_856_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 856 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 834 (MapPartitionsRDD[901] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 834.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 834.0 (TID 1956, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 834.0 (TID 1957, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 834.0 (TID 1958, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 834.0 (TID 1959, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 834.0 (TID 1956) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 834.0 (TID 1958) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 834.0 (TID 1959) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 834.0 (TID 1957) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 834.0 (TID 1956). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 834.0 (TID 1957). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 834.0 (TID 1959). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 834.0 (TID 1956) in 8 ms on localhost (1/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 834.0 (TID 1959) in 7 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 834.0 (TID 1957) in 9 ms on localhost (3/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 834.0 (TID 1958). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 834.0 (TID 1958) in 9 ms on localhost (4/4) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 834.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 834 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 647 finished: reduce at LDAOptimizer.scala:453, took 0.017257 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 648 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 835(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 835 (MapPartitionsRDD[902] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1648340, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_857 stored as values in memory (estimated size 19.8 KB, free 729.0 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8448) called with curMem=1668636, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_857_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.0 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_857_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 857 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 835 (MapPartitionsRDD[902] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 835.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 835.0 (TID 1960, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 835.0 (TID 1961, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 835.0 (TID 1962, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 835.0 (TID 1963, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 835.0 (TID 1960) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 835.0 (TID 1960). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 835.0 (TID 1962) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 835.0 (TID 1961) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 835.0 (TID 1962). 1002 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 835.0 (TID 1963) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 835.0 (TID 1963). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 835.0 (TID 1961). 1402 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 835.0 (TID 1960) in 14 ms on localhost (1/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 835.0 (TID 1963) in 13 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 835.0 (TID 1962) in 15 ms on localhost (3/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 835.0 (TID 1961) in 15 ms on localhost (4/4) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 835.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 835 (reduce at LDAOptimizer.scala:456) finished in 0.016 s -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 648 finished: reduce at LDAOptimizer.scala:456, took 0.024221 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 649 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 836(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 836 (PartitionwiseSampledRDD[903] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1677084, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_858 stored as values in memory (estimated size 18.4 KB, free 729.0 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(7665) called with curMem=1695900, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_858_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.0 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_858_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 858 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 836 (PartitionwiseSampledRDD[903] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 836.0 with 1 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 836.0 (TID 1964, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 836.0 (TID 1964) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 836.0 (TID 1964). 1226 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 836.0 (TID 1964) in 3 ms on localhost (1/1) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 836.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 836 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 649 finished: isEmpty at LDAOptimizer.scala:418, took 0.007724 s -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1703565, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_859 stored as values in memory (estimated size 176.0 B, free 729.0 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1703741, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_859_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.0 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_859_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 859 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 650 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 837(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 837 (MapPartitionsRDD[905] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1704021, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_860 stored as values in memory (estimated size 19.8 KB, free 728.9 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8437) called with curMem=1724277, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_860_piece0 stored as bytes in memory (estimated size 8.2 KB, free 728.9 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 634 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_860_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 860 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 837 (MapPartitionsRDD[905] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 837.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 837.0 (TID 1965, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 837.0 (TID 1966, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 837.0 (TID 1967, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 837.0 (TID 1968, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 837.0 (TID 1966) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 837.0 (TID 1967) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 837.0 (TID 1965) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 837.0 (TID 1967). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 837.0 (TID 1968) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 837.0 (TID 1966). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_810_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 837.0 (TID 1967) in 6 ms on localhost (1/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 837.0 (TID 1966) in 6 ms on localhost (2/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 837.0 (TID 1965). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 837.0 (TID 1968). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 837.0 (TID 1965) in 8 ms on localhost (3/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 837.0 (TID 1968) in 7 ms on localhost (4/4) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 837.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 837 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 650 finished: reduce at LDAOptimizer.scala:453, took 0.033313 s -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 633 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_809_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 651 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 838(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 838 (MapPartitionsRDD[906] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 632 -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1675279, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_861 stored as values in memory (estimated size 19.8 KB, free 729.0 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_808_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8449) called with curMem=1695119, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_861_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.0 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_861_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 861 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 838 (MapPartitionsRDD[906] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 838.0 with 4 tasks -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_807_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 631 -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 838.0 (TID 1969, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 838.0 (TID 1970, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 838.0 (TID 1971, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 838.0 (TID 1972, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 838.0 (TID 1971) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 838.0 (TID 1970) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 838.0 (TID 1969) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_806_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 838.0 (TID 1971). 1002 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 838.0 (TID 1972) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 838.0 (TID 1970). 1358 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 838.0 (TID 1969). 1402 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 838.0 (TID 1972). 1358 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 838.0 (TID 1971) in 4 ms on localhost (1/4) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 630 -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 838.0 (TID 1970) in 7 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 838.0 (TID 1969) in 8 ms on localhost (3/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 838.0 (TID 1972) in 7 ms on localhost (4/4) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 838.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_805_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 629 -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 838 (reduce at LDAOptimizer.scala:456) finished in 0.015 s -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 651 finished: reduce at LDAOptimizer.scala:456, took 0.021352 s -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_804_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 652 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 839(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 839 (PartitionwiseSampledRDD[907] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1592716, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_862 stored as values in memory (estimated size 18.4 KB, free 729.0 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_803_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=1611532, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_862_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.0 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_862_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 628 -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 862 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 839 (PartitionwiseSampledRDD[907] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 839.0 with 1 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 839.0 (TID 1973, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 839.0 (TID 1973) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_802_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 627 -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 839.0 (TID 1973). 1226 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 839.0 (TID 1973) in 3 ms on localhost (1/1) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 839.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 839 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 652 finished: isEmpty at LDAOptimizer.scala:418, took 0.010668 s -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_801_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1561765, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_863 stored as values in memory (estimated size 176.0 B, free 729.1 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 626 -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1561661, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_863_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.1 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_800_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_863_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.1 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 863 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_799_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 625 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_798_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.1 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 624 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 653 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 840(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 840 (MapPartitionsRDD[909] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1480069, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_864 stored as values in memory (estimated size 19.8 KB, free 729.2 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_797_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 623 -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8439) called with curMem=1491890, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_864_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.2 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_796_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_864_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 864 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 840 (MapPartitionsRDD[909] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 840.0 with 4 tasks -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 622 -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 840.0 (TID 1974, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 840.0 (TID 1975, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 840.0 (TID 1976, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 840.0 (TID 1977, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 840.0 (TID 1974) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 840.0 (TID 1977) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_795_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 840.0 (TID 1975) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 840.0 (TID 1974). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 840.0 (TID 1974) in 6 ms on localhost (1/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 840.0 (TID 1977). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 840.0 (TID 1977) in 5 ms on localhost (2/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 840.0 (TID 1975). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 840.0 (TID 1976) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 840.0 (TID 1975) in 8 ms on localhost (3/4) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_794_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 621 -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 840.0 (TID 1976). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_793_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 840.0 (TID 1976) in 11 ms on localhost (4/4) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 840.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 840 (reduce at LDAOptimizer.scala:453) finished in 0.013 s -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 653 finished: reduce at LDAOptimizer.scala:453, took 0.020116 s -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 620 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_792_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 654 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 841(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 841 (MapPartitionsRDD[910] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 619 -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1395697, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_865 stored as values in memory (estimated size 19.8 KB, free 729.2 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8452) called with curMem=1415993, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_865_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.2 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_865_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 865 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 841 (MapPartitionsRDD[910] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_791_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 841.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 841.0 (TID 1978, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 841.0 (TID 1979, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 841.0 (TID 1980, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 841.0 (TID 1981, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 841.0 (TID 1980) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 841.0 (TID 1978) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 841.0 (TID 1979) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 841.0 (TID 1980). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 841.0 (TID 1979). 1534 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 841.0 (TID 1980) in 7 ms on localhost (1/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 841.0 (TID 1979) in 8 ms on localhost (2/4) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 841.0 (TID 1981) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 841.0 (TID 1978). 1358 bytes result sent to driver -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_790_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 841.0 (TID 1981). 1402 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 841.0 (TID 1978) in 15 ms on localhost (3/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 841.0 (TID 1981) in 13 ms on localhost (4/4) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 841.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 841 (reduce at LDAOptimizer.scala:456) finished in 0.017 s -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 654 finished: reduce at LDAOptimizer.scala:456, took 0.024960 s -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 618 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 655 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 842(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 842 (PartitionwiseSampledRDD[911] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_789_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 617 -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1368761, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_866 stored as values in memory (estimated size 18.4 KB, free 729.3 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(7665) called with curMem=1358880, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_866_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.3 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_788_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 616 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_866_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 866 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 842 (PartitionwiseSampledRDD[911] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 842.0 with 1 tasks -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_787_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 842.0 (TID 1982, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 842.0 (TID 1982) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 842.0 (TID 1982). 1226 bytes result sent to driver -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_786_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 842.0 (TID 1982) in 2 ms on localhost (1/1) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 842 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 842.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 655 finished: isEmpty at LDAOptimizer.scala:418, took 0.008220 s -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 615 -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1320794, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_867 stored as values in memory (estimated size 176.0 B, free 729.3 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_785_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 614 -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1313307, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_867_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.3 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_867_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 867 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_784_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 613 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_783_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 612 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_782_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 656 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 843(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 843 (MapPartitionsRDD[913] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_781_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 611 -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1229217, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_868 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8438) called with curMem=1222992, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_868_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_858_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 670 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_868_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 868 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 843 (MapPartitionsRDD[913] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 843.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 843.0 (TID 1983, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_857_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 843.0 (TID 1984, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 843.0 (TID 1985, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 843.0 (TID 1986, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 669 -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 843.0 (TID 1986) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 843.0 (TID 1985) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 843.0 (TID 1983) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 843.0 (TID 1985). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 843.0 (TID 1984) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 843.0 (TID 1983). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_856_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 843.0 (TID 1986). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 843.0 (TID 1985) in 6 ms on localhost (1/4) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 668 -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 843.0 (TID 1983) in 7 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 843.0 (TID 1986) in 7 ms on localhost (3/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 843.0 (TID 1984). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 843.0 (TID 1984) in 9 ms on localhost (4/4) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 843 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 843.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 656 finished: reduce at LDAOptimizer.scala:453, took 0.014751 s -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_855_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 657 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_854_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 844(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 667 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 844 (MapPartitionsRDD[914] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1138615, maxMem=766075207 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_853_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_869 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 666 -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8451) called with curMem=1138615, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_869_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.5 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_869_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 869 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 844 (MapPartitionsRDD[914] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 844.0 with 4 tasks -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_852_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 665 -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 844.0 (TID 1987, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 844.0 (TID 1988, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 844.0 (TID 1989, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 844.0 (TID 1990, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 844.0 (TID 1988) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 844.0 (TID 1990) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 844.0 (TID 1987) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 844.0 (TID 1989) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_851_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 844.0 (TID 1987). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 844.0 (TID 1988). 1358 bytes result sent to driver -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_850_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 844.0 (TID 1989). 1358 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 844.0 (TID 1987) in 6 ms on localhost (1/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 844.0 (TID 1990). 1402 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 844.0 (TID 1989) in 6 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 844.0 (TID 1988) in 7 ms on localhost (3/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 844.0 (TID 1990) in 7 ms on localhost (4/4) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 844 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 844.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 657 finished: reduce at LDAOptimizer.scala:456, took 0.014394 s -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 664 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_849_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 663 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 658 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 845(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 845 (PartitionwiseSampledRDD[915] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_848_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 662 -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1034008, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_870 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(7663) called with curMem=1052368, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_870_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_847_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_870_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 870 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 845 (PartitionwiseSampledRDD[915] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 845.0 with 1 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 845.0 (TID 1991, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 845.0 (TID 1991) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 845.0 (TID 1991). 911 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 845.0 (TID 1991) in 4 ms on localhost (1/1) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 845 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_846_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 845.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 658 finished: isEmpty at LDAOptimizer.scala:418, took 0.010659 s -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 661 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_845_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 660 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 659 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 846(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 846 (PartitionwiseSampledRDD[915] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=976128, maxMem=766075207 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_844_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_871 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 659 -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(7663) called with curMem=994488, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_871_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_843_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_871_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 871 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 846 (PartitionwiseSampledRDD[915] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 846.0 with 3 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 846.0 (TID 1992, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 846.0 (TID 1993, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 846.0 (TID 1994, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 846.0 (TID 1992) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 846.0 (TID 1993) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_842_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 846.0 (TID 1994) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 658 -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 846.0 (TID 1993). 1226 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 846.0 (TID 1992). 1226 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 846.0 (TID 1993) in 4 ms on localhost (1/3) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 846.0 (TID 1994). 911 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 846.0 (TID 1992) in 4 ms on localhost (2/3) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 846.0 (TID 1994) in 3 ms on localhost (3/3) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 846 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 846.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_841_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 659 finished: isEmpty at LDAOptimizer.scala:418, took 0.011388 s -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 657 -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=926677, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_872 stored as values in memory (estimated size 176.0 B, free 729.7 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_840_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=918421, maxMem=766075207 -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 656 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_872_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.7 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_872_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 872 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_839_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_838_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 655 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_837_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 654 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 660 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 847(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 847 (MapPartitionsRDD[917] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=836549, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_873 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_836_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 653 -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8441) called with curMem=836549, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_873_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_873_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 873 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_835_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 847 (MapPartitionsRDD[917] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 847.0 with 4 tasks -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 652 -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 847.0 (TID 1995, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 847.0 (TID 1996, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 847.0 (TID 1997, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 847.0 (TID 1998, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 847.0 (TID 1996) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 847.0 (TID 1995) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 847.0 (TID 1997) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_834_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 847.0 (TID 1996). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 847.0 (TID 1995). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 847.0 (TID 1998) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 847.0 (TID 1996) in 8 ms on localhost (1/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 847.0 (TID 1997). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 847.0 (TID 1995) in 8 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 847.0 (TID 1997) in 8 ms on localhost (3/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 847.0 (TID 1998). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_833_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 847.0 (TID 1998) in 8 ms on localhost (4/4) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 651 -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 847 (reduce at LDAOptimizer.scala:453) finished in 0.011 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 847.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 660 finished: reduce at LDAOptimizer.scala:453, took 0.015660 s -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_832_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 650 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 661 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 848(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 848 (MapPartitionsRDD[918] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_831_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=752188, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_874 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 649 -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8454) called with curMem=772308, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_874_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.8 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_874_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 874 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 848 (MapPartitionsRDD[918] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_830_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 848.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 848.0 (TID 1999, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 848.0 (TID 2000, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 848.0 (TID 2001, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 848.0 (TID 2002, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 848.0 (TID 1999) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 848.0 (TID 2002) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 848.0 (TID 2001) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 848.0 (TID 2000) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 848.0 (TID 1999). 1002 bytes result sent to driver -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_829_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 848.0 (TID 2001). 1358 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 848.0 (TID 1999) in 5 ms on localhost (1/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 848.0 (TID 2002). 1002 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 848.0 (TID 2000). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 848.0 (TID 2001) in 5 ms on localhost (2/4) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 648 -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 848.0 (TID 2002) in 6 ms on localhost (3/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 848.0 (TID 2000) in 8 ms on localhost (4/4) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 848 (reduce at LDAOptimizer.scala:456) finished in 0.009 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 848.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 661 finished: reduce at LDAOptimizer.scala:456, took 0.015145 s -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_828_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 647 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_827_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 646 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 662 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 849(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_826_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 849 (PartitionwiseSampledRDD[919] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=696115, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_875 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_825_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(7662) called with curMem=707272, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_875_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 645 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_875_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 875 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 849 (PartitionwiseSampledRDD[919] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 849.0 with 1 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 849.0 (TID 2003, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 849.0 (TID 2003) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_824_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 644 -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 849.0 (TID 2003). 1226 bytes result sent to driver -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_823_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 849.0 (TID 2003) in 4 ms on localhost (1/1) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 849 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 849.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 662 finished: isEmpty at LDAOptimizer.scala:418, took 0.008911 s -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 643 -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=638494, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_876 stored as values in memory (estimated size 176.0 B, free 730.0 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_822_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=638390, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_876_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.0 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_876_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_821_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 876 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 642 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_820_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 641 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 663 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 850(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_819_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 850 (MapPartitionsRDD[921] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 640 -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=536702, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_877 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_818_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8440) called with curMem=548521, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_877_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 639 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_877_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 877 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 850 (MapPartitionsRDD[921] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 850.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 850.0 (TID 2004, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_817_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 850.0 (TID 2005, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 850.0 (TID 2006, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 850.0 (TID 2007, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 850.0 (TID 2004) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 850.0 (TID 2007) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 850.0 (TID 2005) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_816_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 850.0 (TID 2006) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 850.0 (TID 2004). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 850.0 (TID 2005). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 638 -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 850.0 (TID 2006). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 850.0 (TID 2004) in 7 ms on localhost (1/4) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_815_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 850.0 (TID 2005) in 7 ms on localhost (2/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 850.0 (TID 2007). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 637 -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 850.0 (TID 2007) in 8 ms on localhost (3/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 850.0 (TID 2006) in 9 ms on localhost (4/4) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 850 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 850.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 663 finished: reduce at LDAOptimizer.scala:453, took 0.016524 s -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_814_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 636 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 664 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 851(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_813_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 851 (MapPartitionsRDD[922] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=472140, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_878 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_812_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8454) called with curMem=484776, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_878_piece0 stored as bytes in memory (estimated size 8.3 KB, free 730.1 MB) -2016-06-13 19:45:28 INFO ContextCleaner:59 - Cleaned accumulator 635 -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_878_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 878 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 851 (MapPartitionsRDD[922] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 851.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 851.0 (TID 2008, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 851.0 (TID 2009, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Removed broadcast_811_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 851.0 (TID 2010, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 851.0 (TID 2011, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 851.0 (TID 2008) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 851.0 (TID 2011) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 851.0 (TID 2009) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 851.0 (TID 2010) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 851.0 (TID 2011). 1578 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 851.0 (TID 2010). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 851.0 (TID 2009). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 851.0 (TID 2008). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 851.0 (TID 2011) in 5 ms on localhost (1/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 851.0 (TID 2010) in 5 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 851.0 (TID 2009) in 5 ms on localhost (3/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 851.0 (TID 2008) in 6 ms on localhost (4/4) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 851 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 851.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 664 finished: reduce at LDAOptimizer.scala:456, took 0.012933 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 665 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 852(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 852 (PartitionwiseSampledRDD[923] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=447938, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_879 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=466754, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_879_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_879_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 879 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 852 (PartitionwiseSampledRDD[923] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 852.0 with 1 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 852.0 (TID 2012, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 852.0 (TID 2012) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 852.0 (TID 2012). 1226 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 852.0 (TID 2012) in 2 ms on localhost (1/1) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 852 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 852.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 665 finished: isEmpty at LDAOptimizer.scala:418, took 0.006706 s -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=474415, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_880 stored as values in memory (estimated size 176.0 B, free 730.1 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=474591, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_880_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.1 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_880_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 880 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 666 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 853(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 853 (MapPartitionsRDD[925] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=474871, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_881 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8439) called with curMem=495127, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_881_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.1 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_881_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 881 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 853 (MapPartitionsRDD[925] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 853.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 853.0 (TID 2013, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 853.0 (TID 2014, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 853.0 (TID 2015, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 853.0 (TID 2016, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 853.0 (TID 2016) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 853.0 (TID 2014) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 853.0 (TID 2013) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 853.0 (TID 2015) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 853.0 (TID 2016). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 853.0 (TID 2016) in 5 ms on localhost (1/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 853.0 (TID 2015). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 853.0 (TID 2015) in 5 ms on localhost (2/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 853.0 (TID 2013). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 853.0 (TID 2014). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 853.0 (TID 2013) in 6 ms on localhost (3/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 853.0 (TID 2014) in 6 ms on localhost (4/4) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 853 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 853.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 666 finished: reduce at LDAOptimizer.scala:453, took 0.011467 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 667 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 854(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 854 (MapPartitionsRDD[926] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=503566, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_882 stored as values in memory (estimated size 19.8 KB, free 730.1 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8451) called with curMem=523862, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_882_piece0 stored as bytes in memory (estimated size 8.3 KB, free 730.1 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_882_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 882 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 854 (MapPartitionsRDD[926] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 854.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 854.0 (TID 2017, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 854.0 (TID 2018, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 854.0 (TID 2019, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 854.0 (TID 2020, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 854.0 (TID 2018) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 854.0 (TID 2020) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 854.0 (TID 2019) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 854.0 (TID 2017) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 854.0 (TID 2019). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 854.0 (TID 2018). 1402 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 854.0 (TID 2019) in 5 ms on localhost (1/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 854.0 (TID 2018) in 7 ms on localhost (2/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 854.0 (TID 2017). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 854.0 (TID 2020). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 854.0 (TID 2017) in 9 ms on localhost (3/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 854.0 (TID 2020) in 9 ms on localhost (4/4) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 854 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 854.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 667 finished: reduce at LDAOptimizer.scala:456, took 0.015024 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 668 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 855(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 855 (PartitionwiseSampledRDD[927] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=532313, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_883 stored as values in memory (estimated size 18.4 KB, free 730.1 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=551129, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_883_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.1 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_883_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 883 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 855 (PartitionwiseSampledRDD[927] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 855.0 with 1 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 855.0 (TID 2021, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 855.0 (TID 2021) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 855.0 (TID 2021). 1226 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 855.0 (TID 2021) in 2 ms on localhost (1/1) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 855 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 855.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 668 finished: isEmpty at LDAOptimizer.scala:418, took 0.006976 s -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=558789, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_884 stored as values in memory (estimated size 176.0 B, free 730.1 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=558965, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_884_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.1 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_884_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 884 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 669 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 856(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 856 (MapPartitionsRDD[929] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=559245, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_885 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8438) called with curMem=579501, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_885_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_885_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 885 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 856 (MapPartitionsRDD[929] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 856.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 856.0 (TID 2022, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 856.0 (TID 2023, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 856.0 (TID 2024, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 856.0 (TID 2025, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 856.0 (TID 2023) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 856.0 (TID 2025) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 856.0 (TID 2022) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 856.0 (TID 2024) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 856.0 (TID 2024). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 856.0 (TID 2025). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 856.0 (TID 2022). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 856.0 (TID 2024) in 4 ms on localhost (1/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 856.0 (TID 2025) in 5 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 856.0 (TID 2022) in 6 ms on localhost (3/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 856.0 (TID 2023). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 856.0 (TID 2023) in 5 ms on localhost (4/4) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 856 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 856.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 669 finished: reduce at LDAOptimizer.scala:453, took 0.011565 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 670 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 857(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 857 (MapPartitionsRDD[930] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=587939, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_886 stored as values in memory (estimated size 19.8 KB, free 730.0 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8445) called with curMem=608235, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_886_piece0 stored as bytes in memory (estimated size 8.2 KB, free 730.0 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_886_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 886 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 857 (MapPartitionsRDD[930] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 857.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 857.0 (TID 2026, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 857.0 (TID 2027, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 857.0 (TID 2028, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 857.0 (TID 2029, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 857.0 (TID 2027) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 857.0 (TID 2029) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 857.0 (TID 2028) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 857.0 (TID 2026) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 857.0 (TID 2028). 1002 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 857.0 (TID 2027). 1358 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 857.0 (TID 2028) in 4 ms on localhost (1/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 857.0 (TID 2029). 1358 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 857.0 (TID 2027) in 5 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 857.0 (TID 2029) in 5 ms on localhost (3/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 857.0 (TID 2026). 1358 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 857.0 (TID 2026) in 7 ms on localhost (4/4) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 857 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 857.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 670 finished: reduce at LDAOptimizer.scala:456, took 0.011084 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 671 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 858(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 858 (PartitionwiseSampledRDD[931] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=616680, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_887 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=635496, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_887_piece0 stored as bytes in memory (estimated size 7.5 KB, free 730.0 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_887_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 887 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 858 (PartitionwiseSampledRDD[931] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 858.0 with 1 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 858.0 (TID 2030, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 858.0 (TID 2030) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 858.0 (TID 2030). 911 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 858.0 (TID 2030) in 2 ms on localhost (1/1) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 858 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 858.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 671 finished: isEmpty at LDAOptimizer.scala:418, took 0.006413 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 672 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 859(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 859 (PartitionwiseSampledRDD[931] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=643156, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_888 stored as values in memory (estimated size 18.4 KB, free 730.0 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=661972, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_888_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_888_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 888 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 859 (PartitionwiseSampledRDD[931] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 859.0 with 3 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 859.0 (TID 2031, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 859.0 (TID 2032, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 859.0 (TID 2033, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 859.0 (TID 2031) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 859.0 (TID 2032) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 859.0 (TID 2033) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 859.0 (TID 2032). 911 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 859.0 (TID 2033). 1226 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 859.0 (TID 2032) in 4 ms on localhost (1/3) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 859.0 (TID 2031). 1226 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 859.0 (TID 2033) in 4 ms on localhost (2/3) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 859.0 (TID 2031) in 6 ms on localhost (3/3) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 859.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 859 (isEmpty at LDAOptimizer.scala:418) finished in 0.006 s -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 672 finished: isEmpty at LDAOptimizer.scala:418, took 0.011543 s -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=669632, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_889 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=669808, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_889_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_889_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 889 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 673 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 860(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 860 (MapPartitionsRDD[933] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=670088, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_890 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=690344, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_890_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_890_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 890 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 860 (MapPartitionsRDD[933] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 860.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 860.0 (TID 2034, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 860.0 (TID 2035, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 860.0 (TID 2036, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 860.0 (TID 2037, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 860.0 (TID 2036) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 860.0 (TID 2037) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 860.0 (TID 2035) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 860.0 (TID 2034) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 860.0 (TID 2036). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 860.0 (TID 2034). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 860.0 (TID 2036) in 3 ms on localhost (1/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 860.0 (TID 2034) in 5 ms on localhost (2/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 860.0 (TID 2035). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 860.0 (TID 2037). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 860.0 (TID 2035) in 5 ms on localhost (3/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 860.0 (TID 2037) in 6 ms on localhost (4/4) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 860.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 860 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 673 finished: reduce at LDAOptimizer.scala:453, took 0.011485 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 674 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 861(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 861 (MapPartitionsRDD[934] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=698777, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_891 stored as values in memory (estimated size 19.8 KB, free 729.9 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8446) called with curMem=719073, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_891_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.9 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_891_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 891 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 861 (MapPartitionsRDD[934] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 861.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 861.0 (TID 2038, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 861.0 (TID 2039, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 861.0 (TID 2040, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 861.0 (TID 2041, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 861.0 (TID 2040) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 861.0 (TID 2041) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 861.0 (TID 2038) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 861.0 (TID 2039) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 861.0 (TID 2041). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 861.0 (TID 2040). 1002 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 861.0 (TID 2038). 1002 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 861.0 (TID 2039). 1446 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 861.0 (TID 2041) in 3 ms on localhost (1/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 861.0 (TID 2038) in 4 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 861.0 (TID 2040) in 3 ms on localhost (3/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 861.0 (TID 2039) in 4 ms on localhost (4/4) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 861 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 861.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 674 finished: reduce at LDAOptimizer.scala:456, took 0.009356 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 675 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 862(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 862 (PartitionwiseSampledRDD[935] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=727519, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_892 stored as values in memory (estimated size 18.4 KB, free 729.9 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=746335, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_892_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.9 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_892_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 892 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 862 (PartitionwiseSampledRDD[935] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 862.0 with 1 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 862.0 (TID 2042, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 862.0 (TID 2042) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 862.0 (TID 2042). 1226 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 862.0 (TID 2042) in 2 ms on localhost (1/1) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 862 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 862.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 675 finished: isEmpty at LDAOptimizer.scala:418, took 0.007457 s -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=753995, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_893 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=754171, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_893_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_893_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 893 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 676 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 863(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 863 (MapPartitionsRDD[937] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=754451, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_894 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8435) called with curMem=774707, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_894_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.8 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_894_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 894 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 863 (MapPartitionsRDD[937] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 863.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 863.0 (TID 2043, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 863.0 (TID 2044, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 863.0 (TID 2045, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 863.0 (TID 2046, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 863.0 (TID 2043) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 863.0 (TID 2044) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 863.0 (TID 2045) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 863.0 (TID 2044). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 863.0 (TID 2045). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 863.0 (TID 2046) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 863.0 (TID 2043). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 863.0 (TID 2044) in 6 ms on localhost (1/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 863.0 (TID 2046). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 863.0 (TID 2045) in 7 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 863.0 (TID 2043) in 8 ms on localhost (3/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 863.0 (TID 2046) in 8 ms on localhost (4/4) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 863.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 863 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 676 finished: reduce at LDAOptimizer.scala:453, took 0.014717 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 677 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 864(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 864 (MapPartitionsRDD[938] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=783142, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_895 stored as values in memory (estimated size 19.8 KB, free 729.8 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8448) called with curMem=803438, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_895_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.8 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_895_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 895 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 864 (MapPartitionsRDD[938] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 864.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 864.0 (TID 2047, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 864.0 (TID 2048, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 864.0 (TID 2049, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 864.0 (TID 2050, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 864.0 (TID 2047) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 864.0 (TID 2050) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 864.0 (TID 2049) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 864.0 (TID 2048) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 864.0 (TID 2050). 1402 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 864.0 (TID 2047). 1358 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 864.0 (TID 2050) in 4 ms on localhost (1/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 864.0 (TID 2048). 1358 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 864.0 (TID 2049). 1402 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 864.0 (TID 2047) in 6 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 864.0 (TID 2048) in 6 ms on localhost (3/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 864.0 (TID 2049) in 5 ms on localhost (4/4) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 864 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 864.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 677 finished: reduce at LDAOptimizer.scala:456, took 0.012992 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 678 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 865(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 865 (PartitionwiseSampledRDD[939] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=811886, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_896 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=830702, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_896_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_896_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 896 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 865 (PartitionwiseSampledRDD[939] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 865.0 with 1 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 865.0 (TID 2051, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 865.0 (TID 2051) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 865.0 (TID 2051). 911 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 865.0 (TID 2051) in 3 ms on localhost (1/1) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 865 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 865.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 678 finished: isEmpty at LDAOptimizer.scala:418, took 0.008690 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 679 (isEmpty at LDAOptimizer.scala:418) with 3 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 866(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 866 (PartitionwiseSampledRDD[939] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=838363, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_897 stored as values in memory (estimated size 18.4 KB, free 729.8 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(7661) called with curMem=857179, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_897_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.8 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_897_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 897 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 3 missing tasks from ResultStage 866 (PartitionwiseSampledRDD[939] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 866.0 with 3 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 866.0 (TID 2052, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 866.0 (TID 2053, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 866.0 (TID 2054, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 866.0 (TID 2054) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 866.0 (TID 2052) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 866.0 (TID 2053) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 866.0 (TID 2054). 1226 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 866.0 (TID 2052). 1226 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 866.0 (TID 2053). 911 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 866.0 (TID 2054) in 2 ms on localhost (1/3) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 866.0 (TID 2052) in 3 ms on localhost (2/3) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 866.0 (TID 2053) in 3 ms on localhost (3/3) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 866 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 866.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 679 finished: isEmpty at LDAOptimizer.scala:418, took 0.008893 s -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=864840, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_898 stored as values in memory (estimated size 176.0 B, free 729.8 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=865016, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_898_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.8 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_898_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 898 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 680 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 867(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 867 (MapPartitionsRDD[941] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=865296, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_899 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8440) called with curMem=885552, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_899_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_899_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 899 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 867 (MapPartitionsRDD[941] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 867.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 867.0 (TID 2055, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 867.0 (TID 2056, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 867.0 (TID 2057, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 867.0 (TID 2058, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 867.0 (TID 2056) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 867.0 (TID 2057) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 867.0 (TID 2058) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 867.0 (TID 2055) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 867.0 (TID 2055). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 867.0 (TID 2057). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 867.0 (TID 2056). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 867.0 (TID 2055) in 4 ms on localhost (1/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 867.0 (TID 2057) in 3 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 867.0 (TID 2056) in 4 ms on localhost (3/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 867.0 (TID 2058). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 867.0 (TID 2058) in 4 ms on localhost (4/4) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 867 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 867.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 680 finished: reduce at LDAOptimizer.scala:453, took 0.011480 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 681 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 868(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 868 (MapPartitionsRDD[942] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=893992, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_900 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8453) called with curMem=914288, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_900_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.7 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_900_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 900 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 868 (MapPartitionsRDD[942] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 868.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 868.0 (TID 2059, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 868.0 (TID 2060, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 868.0 (TID 2061, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 868.0 (TID 2062, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 868.0 (TID 2059) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 868.0 (TID 2061) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 868.0 (TID 2062) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 868.0 (TID 2060) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 868.0 (TID 2059). 1002 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 868.0 (TID 2061). 1002 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 868.0 (TID 2062). 1402 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 868.0 (TID 2060). 1358 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 868.0 (TID 2059) in 6 ms on localhost (1/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 868.0 (TID 2061) in 5 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 868.0 (TID 2062) in 6 ms on localhost (3/4) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 868 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 868.0 (TID 2060) in 7 ms on localhost (4/4) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 868.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 681 finished: reduce at LDAOptimizer.scala:456, took 0.012357 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 682 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 869(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 869 (PartitionwiseSampledRDD[943] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=922741, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_901 stored as values in memory (estimated size 18.4 KB, free 729.7 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(7660) called with curMem=941557, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_901_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.7 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_901_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 901 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 869 (PartitionwiseSampledRDD[943] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 869.0 with 1 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 869.0 (TID 2063, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 869.0 (TID 2063) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 869.0 (TID 2063). 1226 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 869.0 (TID 2063) in 3 ms on localhost (1/1) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 869 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 869.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 682 finished: isEmpty at LDAOptimizer.scala:418, took 0.008488 s -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=949217, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_902 stored as values in memory (estimated size 176.0 B, free 729.7 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=949393, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_902_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.7 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_902_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 902 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 683 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 870(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 870 (MapPartitionsRDD[945] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=949673, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_903 stored as values in memory (estimated size 19.8 KB, free 729.7 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8433) called with curMem=969929, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_903_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.7 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_903_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 903 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 870 (MapPartitionsRDD[945] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 870.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 870.0 (TID 2064, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 870.0 (TID 2065, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 870.0 (TID 2066, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 870.0 (TID 2067, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 870.0 (TID 2065) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 870.0 (TID 2067) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 870.0 (TID 2064) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 870.0 (TID 2066) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 870.0 (TID 2065). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 870.0 (TID 2066). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 870.0 (TID 2067). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 870.0 (TID 2065) in 3 ms on localhost (1/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 870.0 (TID 2064). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 870.0 (TID 2066) in 4 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 870.0 (TID 2067) in 4 ms on localhost (3/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 870.0 (TID 2064) in 4 ms on localhost (4/4) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 870 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 870.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 683 finished: reduce at LDAOptimizer.scala:453, took 0.010001 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 684 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 871(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 871 (MapPartitionsRDD[946] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=978362, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_904 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8446) called with curMem=998658, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_904_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_904_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 904 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 871 (MapPartitionsRDD[946] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 871.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 871.0 (TID 2068, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 871.0 (TID 2069, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 871.0 (TID 2070, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 871.0 (TID 2071, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 871.0 (TID 2068) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 871.0 (TID 2071) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 871.0 (TID 2070) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 871.0 (TID 2069) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 871.0 (TID 2071). 1358 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 871.0 (TID 2068). 1358 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 871.0 (TID 2070). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 871.0 (TID 2069). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 871.0 (TID 2071) in 5 ms on localhost (1/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 871.0 (TID 2070) in 5 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 871.0 (TID 2068) in 7 ms on localhost (3/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 871.0 (TID 2069) in 7 ms on localhost (4/4) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 871 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 871.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 684 finished: reduce at LDAOptimizer.scala:456, took 0.013901 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 685 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 872(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 872 (PartitionwiseSampledRDD[947] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1007104, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_905 stored as values in memory (estimated size 18.4 KB, free 729.6 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(7664) called with curMem=1025920, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_905_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.6 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_905_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 905 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 872 (PartitionwiseSampledRDD[947] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 872.0 with 1 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 872.0 (TID 2072, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 872.0 (TID 2072) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 872.0 (TID 2072). 1226 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 872.0 (TID 2072) in 2 ms on localhost (1/1) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 872 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 872.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 685 finished: isEmpty at LDAOptimizer.scala:418, took 0.007768 s -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1033584, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_906 stored as values in memory (estimated size 176.0 B, free 729.6 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1033760, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_906_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.6 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_906_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 906 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 686 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 873(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 873 (MapPartitionsRDD[949] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1034040, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_907 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8437) called with curMem=1054296, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_907_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.6 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_907_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 907 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 873 (MapPartitionsRDD[949] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 873.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 873.0 (TID 2073, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 873.0 (TID 2074, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 873.0 (TID 2075, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 873.0 (TID 2076, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 873.0 (TID 2073) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 873.0 (TID 2076) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 873.0 (TID 2074) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 873.0 (TID 2075) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 873.0 (TID 2073). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 873.0 (TID 2076). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 873.0 (TID 2075). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 873.0 (TID 2073) in 6 ms on localhost (1/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 873.0 (TID 2076) in 6 ms on localhost (2/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 873.0 (TID 2075) in 6 ms on localhost (3/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 873.0 (TID 2074). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 873.0 (TID 2074) in 8 ms on localhost (4/4) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 873 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 873.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 686 finished: reduce at LDAOptimizer.scala:453, took 0.013581 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 687 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 874(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 874 (MapPartitionsRDD[950] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1062733, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_908 stored as values in memory (estimated size 19.8 KB, free 729.6 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8450) called with curMem=1083029, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_908_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.5 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_908_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 908 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 874 (MapPartitionsRDD[950] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 874.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 874.0 (TID 2077, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 874.0 (TID 2078, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 874.0 (TID 2079, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 874.0 (TID 2080, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 874.0 (TID 2078) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 874.0 (TID 2080) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 874.0 (TID 2079) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 874.0 (TID 2077) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 874.0 (TID 2079). 1002 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 874.0 (TID 2079) in 3 ms on localhost (1/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 874.0 (TID 2080). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 874.0 (TID 2078). 1622 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 874.0 (TID 2080) in 4 ms on localhost (2/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 874.0 (TID 2077). 1314 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 874.0 (TID 2078) in 6 ms on localhost (3/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 874.0 (TID 2077) in 6 ms on localhost (4/4) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 874 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 874.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 687 finished: reduce at LDAOptimizer.scala:456, took 0.011620 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 688 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 875(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 875 (PartitionwiseSampledRDD[951] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1091479, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_909 stored as values in memory (estimated size 18.4 KB, free 729.5 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(7663) called with curMem=1110295, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_909_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.5 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_909_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 909 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 875 (PartitionwiseSampledRDD[951] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 875.0 with 1 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 875.0 (TID 2081, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 875.0 (TID 2081) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 875.0 (TID 2081). 1226 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 875.0 (TID 2081) in 3 ms on localhost (1/1) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 875.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 875 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 688 finished: isEmpty at LDAOptimizer.scala:418, took 0.007695 s -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1117958, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_910 stored as values in memory (estimated size 176.0 B, free 729.5 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1118134, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_910_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.5 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_910_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 910 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 689 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 876(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 876 (MapPartitionsRDD[953] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1118414, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_911 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8438) called with curMem=1138670, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_911_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_911_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 911 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 876 (MapPartitionsRDD[953] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 876.0 with 4 tasks -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 876.0 (TID 2082, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 876.0 (TID 2083, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 876.0 (TID 2084, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Starting task 3.0 in stage 876.0 (TID 2085, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:28 INFO Executor:59 - Running task 0.0 in stage 876.0 (TID 2082) -2016-06-13 19:45:28 INFO Executor:59 - Running task 3.0 in stage 876.0 (TID 2085) -2016-06-13 19:45:28 INFO Executor:59 - Running task 2.0 in stage 876.0 (TID 2084) -2016-06-13 19:45:28 INFO Executor:59 - Running task 1.0 in stage 876.0 (TID 2083) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 0.0 in stage 876.0 (TID 2082). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 1.0 in stage 876.0 (TID 2083). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO Executor:59 - Finished task 2.0 in stage 876.0 (TID 2084). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 876.0 (TID 2082) in 4 ms on localhost (1/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 876.0 (TID 2083) in 5 ms on localhost (2/4) -2016-06-13 19:45:28 INFO Executor:59 - Finished task 3.0 in stage 876.0 (TID 2085). 1265 bytes result sent to driver -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 876.0 (TID 2084) in 6 ms on localhost (3/4) -2016-06-13 19:45:28 INFO TaskSetManager:59 - Finished task 3.0 in stage 876.0 (TID 2085) in 6 ms on localhost (4/4) -2016-06-13 19:45:28 INFO DAGScheduler:59 - ResultStage 876 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 876.0, whose tasks have all completed, from pool -2016-06-13 19:45:28 INFO DAGScheduler:59 - Job 689 finished: reduce at LDAOptimizer.scala:453, took 0.011355 s -2016-06-13 19:45:28 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Got job 690 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:28 INFO DAGScheduler:59 - Final stage: ResultStage 877(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting ResultStage 877 (MapPartitionsRDD[954] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1147108, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_912 stored as values in memory (estimated size 19.8 KB, free 729.5 MB) -2016-06-13 19:45:28 INFO MemoryStore:59 - ensureFreeSpace(8445) called with curMem=1167404, maxMem=766075207 -2016-06-13 19:45:28 INFO MemoryStore:59 - Block broadcast_912_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.5 MB) -2016-06-13 19:45:28 INFO BlockManagerInfo:59 - Added broadcast_912_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:28 INFO SparkContext:59 - Created broadcast 912 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:28 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 877 (MapPartitionsRDD[954] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:28 INFO TaskSchedulerImpl:59 - Adding task set 877.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 877.0 (TID 2086, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 877.0 (TID 2087, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 877.0 (TID 2088, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 877.0 (TID 2089, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 877.0 (TID 2087) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 877.0 (TID 2089) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 877.0 (TID 2088) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 877.0 (TID 2086) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 877.0 (TID 2086). 1402 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 877.0 (TID 2089). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 877.0 (TID 2087). 1402 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 877.0 (TID 2088). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 877.0 (TID 2086) in 4 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 877.0 (TID 2089) in 4 ms on localhost (2/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 877.0 (TID 2087) in 4 ms on localhost (3/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 877.0 (TID 2088) in 4 ms on localhost (4/4) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 877 (reduce at LDAOptimizer.scala:456) finished in 0.004 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 877.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 690 finished: reduce at LDAOptimizer.scala:456, took 0.009313 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 691 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 878(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 878 (PartitionwiseSampledRDD[955] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(18816) called with curMem=1175849, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_913 stored as values in memory (estimated size 18.4 KB, free 729.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(7663) called with curMem=1194665, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_913_piece0 stored as bytes in memory (estimated size 7.5 KB, free 729.4 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_913_piece0 in memory on localhost:50000 (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 913 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 878 (PartitionwiseSampledRDD[955] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 878.0 with 1 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 878.0 (TID 2090, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 878.0 (TID 2090) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 878.0 (TID 2090). 1226 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 878.0 (TID 2090) in 2 ms on localhost (1/1) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 878 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 878.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 691 finished: isEmpty at LDAOptimizer.scala:418, took 0.008507 s -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1202328, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_914 stored as values in memory (estimated size 176.0 B, free 729.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1202504, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_914_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.4 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_914_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 914 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 692 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 879(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 879 (MapPartitionsRDD[957] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(20256) called with curMem=1202784, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_915 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(8439) called with curMem=1223040, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_915_piece0 stored as bytes in memory (estimated size 8.2 KB, free 729.4 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_915_piece0 in memory on localhost:50000 (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 915 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 879 (MapPartitionsRDD[957] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 879.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 879.0 (TID 2091, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 879.0 (TID 2092, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 879.0 (TID 2093, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 879.0 (TID 2094, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 879.0 (TID 2091) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 879.0 (TID 2092) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 879.0 (TID 2094) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 879.0 (TID 2093) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 879.0 (TID 2094). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 879.0 (TID 2092). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 879.0 (TID 2091). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 879.0 (TID 2094) in 3 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 879.0 (TID 2092) in 4 ms on localhost (2/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 879.0 (TID 2091) in 4 ms on localhost (3/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 879.0 (TID 2093). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 879.0 (TID 2093) in 5 ms on localhost (4/4) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 879 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 879.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 692 finished: reduce at LDAOptimizer.scala:453, took 0.010824 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 693 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 880(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 880 (MapPartitionsRDD[958] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(20296) called with curMem=1231479, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_916 stored as values in memory (estimated size 19.8 KB, free 729.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(8452) called with curMem=1251775, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_916_piece0 stored as bytes in memory (estimated size 8.3 KB, free 729.4 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_916_piece0 in memory on localhost:50000 (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 916 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 880 (MapPartitionsRDD[958] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 880.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 880.0 (TID 2095, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 880.0 (TID 2096, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 880.0 (TID 2097, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 880.0 (TID 2098, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 880.0 (TID 2097) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 880.0 (TID 2096) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 880.0 (TID 2098) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 880.0 (TID 2095) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 880.0 (TID 2096). 1002 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 880.0 (TID 2098). 1002 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 880.0 (TID 2097). 1002 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 880.0 (TID 2096) in 4 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 880.0 (TID 2097) in 4 ms on localhost (2/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 880.0 (TID 2098) in 4 ms on localhost (3/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 880.0 (TID 2095). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 880.0 (TID 2095) in 6 ms on localhost (4/4) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 880 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 880.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 693 finished: reduce at LDAOptimizer.scala:456, took 0.011734 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: count at LDAOptimizer.scala:387 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 694 (count at LDAOptimizer.scala:387) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 881(count at LDAOptimizer.scala:387) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 881 (ParallelCollectionRDD[959] at parallelize at LDASuite.scala:424), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(1032) called with curMem=1260227, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_917 stored as values in memory (estimated size 1032.0 B, free 729.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(756) called with curMem=1261259, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_917_piece0 stored as bytes in memory (estimated size 756.0 B, free 729.4 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_917_piece0 in memory on localhost:50000 (size: 756.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 917 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 881 (ParallelCollectionRDD[959] at parallelize at LDASuite.scala:424) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 881.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 881.0 (TID 2099, localhost, PROCESS_LOCAL, 2306 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 881.0 (TID 2100, localhost, PROCESS_LOCAL, 2389 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 881.0 (TID 2101, localhost, PROCESS_LOCAL, 2315 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 881.0 (TID 2102, localhost, PROCESS_LOCAL, 2389 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 881.0 (TID 2100) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 881.0 (TID 2101) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 881.0 (TID 2099) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 881.0 (TID 2101). 953 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 881.0 (TID 2099). 953 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 881.0 (TID 2100). 953 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 881.0 (TID 2101) in 4 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 881.0 (TID 2100) in 5 ms on localhost (2/4) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 881.0 (TID 2102) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 881.0 (TID 2099) in 8 ms on localhost (3/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 881.0 (TID 2102). 953 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 881.0 (TID 2102) in 10 ms on localhost (4/4) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 881 (count at LDAOptimizer.scala:387) finished in 0.011 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 881.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 694 finished: count at LDAOptimizer.scala:387, took 0.017776 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: first at LDAOptimizer.scala:388 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 695 (first at LDAOptimizer.scala:388) with 1 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 882(first at LDAOptimizer.scala:388) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 882 (ParallelCollectionRDD[959] at parallelize at LDASuite.scala:424), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=1262015, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_918 stored as values in memory (estimated size 1224.0 B, free 729.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(807) called with curMem=1263239, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_918_piece0 stored as bytes in memory (estimated size 807.0 B, free 729.4 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_918_piece0 in memory on localhost:50000 (size: 807.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 918 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 882 (ParallelCollectionRDD[959] at parallelize at LDASuite.scala:424) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 882.0 with 1 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 882.0 (TID 2103, localhost, PROCESS_LOCAL, 2306 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 882.0 (TID 2103) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 882.0 (TID 2103). 1226 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 882.0 (TID 2103) in 2 ms on localhost (1/1) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 882 (first at LDAOptimizer.scala:388) finished in 0.002 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 882.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 695 finished: first at LDAOptimizer.scala:388, took 0.006579 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 696 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 883(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 883 (PartitionwiseSampledRDD[960] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=1264046, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_919 stored as values in memory (estimated size 2.1 KB, free 729.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(1319) called with curMem=1266198, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_919_piece0 stored as bytes in memory (estimated size 1319.0 B, free 729.4 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_919_piece0 in memory on localhost:50000 (size: 1319.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 919 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 883 (PartitionwiseSampledRDD[960] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 883.0 with 1 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 883.0 (TID 2104, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 883.0 (TID 2104) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 883.0 (TID 2104). 1226 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 883.0 (TID 2104) in 2 ms on localhost (1/1) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 883 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 883.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 696 finished: isEmpty at LDAOptimizer.scala:418, took 0.006280 s -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1267517, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_920 stored as values in memory (estimated size 176.0 B, free 729.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1267693, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_920_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.4 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_920_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 920 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 697 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 884(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 884 (MapPartitionsRDD[962] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=1267973, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_921 stored as values in memory (estimated size 3.5 KB, free 729.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2081) called with curMem=1271565, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_921_piece0 stored as bytes in memory (estimated size 2.0 KB, free 729.4 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_921_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 921 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 884 (MapPartitionsRDD[962] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 884.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 884.0 (TID 2105, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 884.0 (TID 2106, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 884.0 (TID 2107, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 884.0 (TID 2108, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 884.0 (TID 2105) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 884.0 (TID 2108) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 884.0 (TID 2106) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 884.0 (TID 2105). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 884.0 (TID 2106). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 884.0 (TID 2107) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 884.0 (TID 2108). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 884.0 (TID 2105) in 4 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 884.0 (TID 2106) in 5 ms on localhost (2/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 884.0 (TID 2107). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 884.0 (TID 2108) in 4 ms on localhost (3/4) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 884 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 884.0 (TID 2107) in 5 ms on localhost (4/4) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 884.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 697 finished: reduce at LDAOptimizer.scala:453, took 0.011485 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 698 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 885(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 885 (MapPartitionsRDD[963] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=1273646, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_922 stored as values in memory (estimated size 3.5 KB, free 729.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2081) called with curMem=1277278, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_922_piece0 stored as bytes in memory (estimated size 2.0 KB, free 729.4 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_922_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 922 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 885 (MapPartitionsRDD[963] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 885.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 885.0 (TID 2109, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 885.0 (TID 2110, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 885.0 (TID 2111, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 885.0 (TID 2112, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 885.0 (TID 2110) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 885.0 (TID 2109) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 885.0 (TID 2110). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 885.0 (TID 2111) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 885.0 (TID 2112) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 885.0 (TID 2109). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 885.0 (TID 2110) in 5 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 885.0 (TID 2109) in 6 ms on localhost (2/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 885.0 (TID 2112). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 885.0 (TID 2111). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 885.0 (TID 2112) in 7 ms on localhost (3/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 885.0 (TID 2111) in 7 ms on localhost (4/4) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 885.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 885 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 698 finished: reduce at LDAOptimizer.scala:456, took 0.014424 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 699 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 886(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 886 (PartitionwiseSampledRDD[964] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=1279359, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_923 stored as values in memory (estimated size 2.1 KB, free 729.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=1281511, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_923_piece0 stored as bytes in memory (estimated size 1317.0 B, free 729.4 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_923_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 923 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 886 (PartitionwiseSampledRDD[964] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 886.0 with 1 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 886.0 (TID 2113, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 886.0 (TID 2113) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 886.0 (TID 2113). 1226 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 886.0 (TID 2113) in 1 ms on localhost (1/1) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 886 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 886.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 699 finished: isEmpty at LDAOptimizer.scala:418, took 0.006652 s -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1282828, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_924 stored as values in memory (estimated size 176.0 B, free 729.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1283004, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_924_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.4 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_924_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 924 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 700 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 887(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 887 (MapPartitionsRDD[966] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=1283284, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_925 stored as values in memory (estimated size 3.5 KB, free 729.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=1286876, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_925_piece0 stored as bytes in memory (estimated size 2.0 KB, free 729.4 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_925_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 925 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 887 (MapPartitionsRDD[966] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 887.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 887.0 (TID 2114, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 887.0 (TID 2115, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 887.0 (TID 2116, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 887.0 (TID 2117, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 887.0 (TID 2116) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 887.0 (TID 2116). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 887.0 (TID 2114) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 887.0 (TID 2116) in 4 ms on localhost (1/4) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 887.0 (TID 2115) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 887.0 (TID 2117) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 887.0 (TID 2117). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 887.0 (TID 2115). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 887.0 (TID 2114). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 887.0 (TID 2117) in 10 ms on localhost (2/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 887.0 (TID 2115) in 11 ms on localhost (3/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 887.0 (TID 2114) in 11 ms on localhost (4/4) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 887.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 887 (reduce at LDAOptimizer.scala:453) finished in 0.012 s -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 700 finished: reduce at LDAOptimizer.scala:453, took 0.018349 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 701 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 888(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 888 (MapPartitionsRDD[967] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=1288961, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_926 stored as values in memory (estimated size 3.5 KB, free 729.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=1292593, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_926_piece0 stored as bytes in memory (estimated size 2.0 KB, free 729.4 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_926_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 926 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 888 (MapPartitionsRDD[967] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 888.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 888.0 (TID 2118, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 888.0 (TID 2119, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 888.0 (TID 2120, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 888.0 (TID 2121, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 888.0 (TID 2121) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 888.0 (TID 2118) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 888.0 (TID 2119) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 888.0 (TID 2120) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 888.0 (TID 2120). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 888.0 (TID 2118). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 888.0 (TID 2119). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 888.0 (TID 2121). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 888.0 (TID 2120) in 16 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 888.0 (TID 2118) in 17 ms on localhost (2/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 888.0 (TID 2119) in 17 ms on localhost (3/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 888.0 (TID 2121) in 17 ms on localhost (4/4) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 888.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 888 (reduce at LDAOptimizer.scala:456) finished in 0.019 s -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 701 finished: reduce at LDAOptimizer.scala:456, took 0.025290 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 702 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 889(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 889 (PartitionwiseSampledRDD[968] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=1294678, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_927 stored as values in memory (estimated size 2.1 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=1296830, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_927_piece0 stored as bytes in memory (estimated size 1317.0 B, free 729.3 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_927_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 927 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 889 (PartitionwiseSampledRDD[968] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 889.0 with 1 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 889.0 (TID 2122, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 889.0 (TID 2122) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 889.0 (TID 2122). 1226 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 889.0 (TID 2122) in 2 ms on localhost (1/1) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 889 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 889.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 702 finished: isEmpty at LDAOptimizer.scala:418, took 0.006393 s -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1298147, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_928 stored as values in memory (estimated size 176.0 B, free 729.3 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1298323, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_928_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.3 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_928_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 928 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 703 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 890(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 890 (MapPartitionsRDD[970] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=1298603, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_929 stored as values in memory (estimated size 3.5 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=1302195, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_929_piece0 stored as bytes in memory (estimated size 2.0 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_929_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 929 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 890 (MapPartitionsRDD[970] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 890.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 890.0 (TID 2123, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 890.0 (TID 2124, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 890.0 (TID 2125, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 890.0 (TID 2126, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 890.0 (TID 2124) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 890.0 (TID 2126) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 890.0 (TID 2123) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 890.0 (TID 2124). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 890.0 (TID 2125) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 890.0 (TID 2123). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 890.0 (TID 2124) in 5 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 890.0 (TID 2123) in 5 ms on localhost (2/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 890.0 (TID 2125). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 890.0 (TID 2125) in 7 ms on localhost (3/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 890.0 (TID 2126). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 890 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 890.0 (TID 2126) in 7 ms on localhost (4/4) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 890.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 703 finished: reduce at LDAOptimizer.scala:453, took 0.013902 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 704 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 891(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 891 (MapPartitionsRDD[971] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=1304280, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_930 stored as values in memory (estimated size 3.5 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=1307912, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_930_piece0 stored as bytes in memory (estimated size 2.0 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_930_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 930 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 891 (MapPartitionsRDD[971] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 891.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 891.0 (TID 2127, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 891.0 (TID 2128, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 891.0 (TID 2129, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 891.0 (TID 2130, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 891.0 (TID 2127) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 891.0 (TID 2127). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 891.0 (TID 2128) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 891.0 (TID 2128). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 891.0 (TID 2130) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 891.0 (TID 2130). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 891.0 (TID 2129) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 891.0 (TID 2127) in 8 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 891.0 (TID 2128) in 8 ms on localhost (2/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 891.0 (TID 2129). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 891.0 (TID 2130) in 8 ms on localhost (3/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 891.0 (TID 2129) in 10 ms on localhost (4/4) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 891.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 891 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 704 finished: reduce at LDAOptimizer.scala:456, took 0.016524 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 705 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 892(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 892 (PartitionwiseSampledRDD[972] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=1309997, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_931 stored as values in memory (estimated size 2.1 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=1312149, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_931_piece0 stored as bytes in memory (estimated size 1315.0 B, free 729.3 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_931_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 931 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 892 (PartitionwiseSampledRDD[972] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 892.0 with 1 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 892.0 (TID 2131, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 892.0 (TID 2131) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 892.0 (TID 2131). 1226 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 892.0 (TID 2131) in 2 ms on localhost (1/1) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 892 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 892.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 705 finished: isEmpty at LDAOptimizer.scala:418, took 0.006103 s -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1313464, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_932 stored as values in memory (estimated size 176.0 B, free 729.3 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1313640, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_932_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.3 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_932_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 932 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 706 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 893(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 893 (MapPartitionsRDD[974] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=1313920, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_933 stored as values in memory (estimated size 3.5 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=1317512, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_933_piece0 stored as bytes in memory (estimated size 2.0 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_933_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 933 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 893 (MapPartitionsRDD[974] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 893.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 893.0 (TID 2132, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 893.0 (TID 2133, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 893.0 (TID 2134, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 893.0 (TID 2135, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 893.0 (TID 2133) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 893.0 (TID 2135) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 893.0 (TID 2134) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 893.0 (TID 2132) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 893.0 (TID 2132). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 893.0 (TID 2133). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 893.0 (TID 2135). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 893.0 (TID 2134). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 893.0 (TID 2132) in 3 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 893.0 (TID 2133) in 4 ms on localhost (2/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 893.0 (TID 2135) in 4 ms on localhost (3/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 893.0 (TID 2134) in 4 ms on localhost (4/4) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 893 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 893.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 706 finished: reduce at LDAOptimizer.scala:453, took 0.010223 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 707 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 894(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 894 (MapPartitionsRDD[975] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=1319596, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_934 stored as values in memory (estimated size 3.5 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=1323228, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_934_piece0 stored as bytes in memory (estimated size 2.0 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_934_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 934 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 894 (MapPartitionsRDD[975] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 894.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 894.0 (TID 2136, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 894.0 (TID 2137, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 894.0 (TID 2138, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 894.0 (TID 2139, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 894.0 (TID 2139) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 894.0 (TID 2138) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 894.0 (TID 2137) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 894.0 (TID 2136) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 894.0 (TID 2136). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 894.0 (TID 2139). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 894.0 (TID 2138). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 894.0 (TID 2136) in 4 ms on localhost (1/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 894.0 (TID 2137). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 894.0 (TID 2139) in 4 ms on localhost (2/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 894.0 (TID 2138) in 5 ms on localhost (3/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 894.0 (TID 2137) in 5 ms on localhost (4/4) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 894 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 894.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 707 finished: reduce at LDAOptimizer.scala:456, took 0.010479 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 708 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 895(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 895 (PartitionwiseSampledRDD[976] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=1325312, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_935 stored as values in memory (estimated size 2.1 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=1327464, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_935_piece0 stored as bytes in memory (estimated size 1317.0 B, free 729.3 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_935_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 935 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 895 (PartitionwiseSampledRDD[976] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 895.0 with 1 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 895.0 (TID 2140, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 895.0 (TID 2140) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 895.0 (TID 2140). 1226 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 895.0 (TID 2140) in 2 ms on localhost (1/1) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 895 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 895.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 708 finished: isEmpty at LDAOptimizer.scala:418, took 0.007255 s -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1328781, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_936 stored as values in memory (estimated size 176.0 B, free 729.3 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1328957, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_936_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.3 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_936_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 936 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 709 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 896(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 896 (MapPartitionsRDD[978] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=1329237, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_937 stored as values in memory (estimated size 3.5 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=1332829, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_937_piece0 stored as bytes in memory (estimated size 2.0 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_937_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 937 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 896 (MapPartitionsRDD[978] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 896.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 896.0 (TID 2141, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 896.0 (TID 2142, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 896.0 (TID 2143, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 896.0 (TID 2144, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 896.0 (TID 2142) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 896.0 (TID 2144) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 896.0 (TID 2141) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 896.0 (TID 2143) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 896.0 (TID 2144). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 896.0 (TID 2142). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 896.0 (TID 2143). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 896.0 (TID 2141). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 896.0 (TID 2142) in 6 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 896.0 (TID 2144) in 6 ms on localhost (2/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 896.0 (TID 2141) in 7 ms on localhost (3/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 896.0 (TID 2143) in 7 ms on localhost (4/4) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 896.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 896 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 709 finished: reduce at LDAOptimizer.scala:453, took 0.012646 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 710 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 897(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 897 (MapPartitionsRDD[979] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=1334916, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_938 stored as values in memory (estimated size 3.5 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=1338548, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_938_piece0 stored as bytes in memory (estimated size 2.0 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_938_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 938 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 897 (MapPartitionsRDD[979] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 897.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 897.0 (TID 2145, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 897.0 (TID 2146, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 897.0 (TID 2147, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 897.0 (TID 2148, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 897.0 (TID 2145) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 897.0 (TID 2148) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 897.0 (TID 2147) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 897.0 (TID 2146) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 897.0 (TID 2146). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 897.0 (TID 2147). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 897.0 (TID 2145). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 897.0 (TID 2148). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 897.0 (TID 2146) in 3 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 897.0 (TID 2145) in 5 ms on localhost (2/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 897.0 (TID 2147) in 4 ms on localhost (3/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 897.0 (TID 2148) in 4 ms on localhost (4/4) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 897 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 897.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 710 finished: reduce at LDAOptimizer.scala:456, took 0.009755 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 711 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 898(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 898 (PartitionwiseSampledRDD[980] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=1340635, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_939 stored as values in memory (estimated size 2.1 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=1342787, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_939_piece0 stored as bytes in memory (estimated size 1317.0 B, free 729.3 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_939_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 939 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 898 (PartitionwiseSampledRDD[980] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 898.0 with 1 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 898.0 (TID 2149, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 898.0 (TID 2149) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 898.0 (TID 2149). 1226 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 898.0 (TID 2149) in 2 ms on localhost (1/1) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 898 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 898.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 711 finished: isEmpty at LDAOptimizer.scala:418, took 0.007232 s -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1344104, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_940 stored as values in memory (estimated size 176.0 B, free 729.3 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1344280, maxMem=766075207 -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 682 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_940_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.3 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_940_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_939_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 940 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 732 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_938_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 731 -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 712 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 899(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_937_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 899 (MapPartitionsRDD[982] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 730 -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=1329693, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_941 stored as values in memory (estimated size 3.5 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=1332829, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_941_piece0 stored as bytes in memory (estimated size 2.0 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_936_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_941_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 941 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 899 (MapPartitionsRDD[982] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 899.0 with 4 tasks -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_935_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 899.0 (TID 2150, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 899.0 (TID 2151, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 899.0 (TID 2152, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 729 -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 899.0 (TID 2153, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 899.0 (TID 2150) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 899.0 (TID 2151) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 899.0 (TID 2153) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 899.0 (TID 2152) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_934_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 728 -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 899.0 (TID 2151). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 899.0 (TID 2152). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 899.0 (TID 2153). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 899.0 (TID 2150). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 899.0 (TID 2152) in 5 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 899.0 (TID 2151) in 6 ms on localhost (2/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 899.0 (TID 2153) in 6 ms on localhost (3/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 899.0 (TID 2150) in 7 ms on localhost (4/4) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 899 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_933_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 899.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 712 finished: reduce at LDAOptimizer.scala:453, took 0.013412 s -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 727 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_932_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 713 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 900(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 900 (MapPartitionsRDD[983] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_931_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=1316130, maxMem=766075207 -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 726 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_942 stored as values in memory (estimated size 3.5 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=1314045, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_942_piece0 stored as bytes in memory (estimated size 2.0 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_930_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 725 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_942_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 942 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 900 (MapPartitionsRDD[983] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 900.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 900.0 (TID 2154, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 900.0 (TID 2155, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 900.0 (TID 2156, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 900.0 (TID 2157, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 900.0 (TID 2154) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_929_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 900.0 (TID 2157) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 900.0 (TID 2154). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 900.0 (TID 2157). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 900.0 (TID 2156) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 900.0 (TID 2154) in 5 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 900.0 (TID 2157) in 4 ms on localhost (2/4) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 900.0 (TID 2155) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 724 -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 900.0 (TID 2156). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 900.0 (TID 2156) in 7 ms on localhost (3/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 900.0 (TID 2155). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_928_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 900.0 (TID 2155) in 9 ms on localhost (4/4) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 900.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 900 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 713 finished: reduce at LDAOptimizer.scala:456, took 0.016449 s -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_927_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 723 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_926_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 714 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 901(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 901 (PartitionwiseSampledRDD[984] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 722 -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=1300811, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_943 stored as values in memory (estimated size 2.1 KB, free 729.3 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(1314) called with curMem=1299371, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_943_piece0 stored as bytes in memory (estimated size 1314.0 B, free 729.3 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_943_piece0 in memory on localhost:50000 (size: 1314.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_925_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 943 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 901 (PartitionwiseSampledRDD[984] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 901.0 with 1 tasks -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 721 -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 901.0 (TID 2158, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 901.0 (TID 2158) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_924_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 901.0 (TID 2158). 1226 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 901.0 (TID 2158) in 3 ms on localhost (1/1) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 901.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 901 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 714 finished: isEmpty at LDAOptimizer.scala:418, took 0.010966 s -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1294675, maxMem=766075207 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_923_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_944 stored as values in memory (estimated size 176.0 B, free 729.4 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 720 -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1289138, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_944_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.4 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_922_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 719 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_944_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 944 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_921_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 715 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 902(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 902 (MapPartitionsRDD[986] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=1283745, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_945 stored as values in memory (estimated size 3.5 KB, free 729.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2082) called with curMem=1287337, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_945_piece0 stored as bytes in memory (estimated size 2.0 KB, free 729.4 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 718 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_945_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 945 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 902 (MapPartitionsRDD[986] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 902.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 902.0 (TID 2159, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 902.0 (TID 2160, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 902.0 (TID 2161, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 902.0 (TID 2162, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 902.0 (TID 2159) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_920_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 902.0 (TID 2161) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 902.0 (TID 2159). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 902.0 (TID 2161). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 902.0 (TID 2162) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 902.0 (TID 2159) in 7 ms on localhost (1/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 902.0 (TID 2162). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 902.0 (TID 2161) in 7 ms on localhost (2/4) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 902.0 (TID 2160) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 902.0 (TID 2162) in 9 ms on localhost (3/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 902.0 (TID 2160). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 902.0 (TID 2160) in 11 ms on localhost (4/4) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 902.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 902 (reduce at LDAOptimizer.scala:453) finished in 0.012 s -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 715 finished: reduce at LDAOptimizer.scala:453, took 0.026205 s -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_919_piece0 on localhost:50000 in memory (size: 1319.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 717 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_918_piece0 on localhost:50000 in memory (size: 807.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 716 -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 716 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 903(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 903 (MapPartitionsRDD[987] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_917_piece0 on localhost:50000 in memory (size: 756.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=1281673, maxMem=766075207 -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 715 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_946 stored as values in memory (estimated size 3.5 KB, free 729.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2082) called with curMem=1256557, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_946_piece0 stored as bytes in memory (estimated size 2.0 KB, free 729.4 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_916_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 714 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_946_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 946 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 903 (MapPartitionsRDD[987] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 903.0 with 4 tasks -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_915_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 903.0 (TID 2163, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 903.0 (TID 2164, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 903.0 (TID 2165, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 713 -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 903.0 (TID 2166, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 903.0 (TID 2164) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 903.0 (TID 2163) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 903.0 (TID 2165) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 903.0 (TID 2166) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_914_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 903.0 (TID 2164). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 903.0 (TID 2165). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 903.0 (TID 2163). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 903.0 (TID 2164) in 5 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 903.0 (TID 2163) in 6 ms on localhost (2/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 903.0 (TID 2165) in 6 ms on localhost (3/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 903.0 (TID 2166). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 903.0 (TID 2166) in 7 ms on localhost (4/4) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 903 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 716 finished: reduce at LDAOptimizer.scala:456, took 0.015054 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 903.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_913_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 712 -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 717 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 904(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_912_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 904 (PartitionwiseSampledRDD[988] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 711 -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=1154012, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_947 stored as values in memory (estimated size 2.1 KB, free 729.5 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=1147726, maxMem=766075207 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_911_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.2 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_947_piece0 stored as bytes in memory (estimated size 1315.0 B, free 729.5 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 710 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_947_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 947 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 904 (PartitionwiseSampledRDD[988] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 904.0 with 1 tasks -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_910_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.2 MB) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 904.0 (TID 2167, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 904.0 (TID 2167) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 904.0 (TID 2167). 1226 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 904.0 (TID 2167) in 3 ms on localhost (1/1) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 904 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 904.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_909_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 717 finished: isEmpty at LDAOptimizer.scala:418, took 0.008633 s -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 709 -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=1122106, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_948 stored as values in memory (estimated size 176.0 B, free 729.5 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=1093536, maxMem=766075207 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_908_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_948_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.5 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 708 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_948_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 948 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_907_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 707 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_906_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 718 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 905(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 905 (MapPartitionsRDD[990] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_905_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 706 -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=1017891, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_949 stored as values in memory (estimated size 3.5 KB, free 729.6 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_904_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2083) called with curMem=1013037, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_949_piece0 stored as bytes in memory (estimated size 2.0 KB, free 729.6 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 705 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_949_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.3 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 949 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 905 (MapPartitionsRDD[990] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 905.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 905.0 (TID 2168, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 905.0 (TID 2169, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_903_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 905.0 (TID 2170, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 704 -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 905.0 (TID 2171, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 905.0 (TID 2170) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 905.0 (TID 2168) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 905.0 (TID 2169) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 905.0 (TID 2171) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_902_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 905.0 (TID 2170). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 905.0 (TID 2168). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 905.0 (TID 2170) in 5 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 905.0 (TID 2168) in 6 ms on localhost (2/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 905.0 (TID 2171). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_901_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 905.0 (TID 2171) in 9 ms on localhost (3/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 905.0 (TID 2169). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 905.0 (TID 2169) in 12 ms on localhost (4/4) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 905 (reduce at LDAOptimizer.scala:453) finished in 0.012 s -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 703 -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 905.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 718 finished: reduce at LDAOptimizer.scala:453, took 0.018480 s -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_900_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 702 -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 719 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 906(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 906 (MapPartitionsRDD[991] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_899_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 701 -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=902054, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_950 stored as values in memory (estimated size 3.5 KB, free 729.7 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2083) called with curMem=905230, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_950_piece0 stored as bytes in memory (estimated size 2.0 KB, free 729.7 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_898_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_950_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.3 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 950 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 906 (MapPartitionsRDD[991] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 906.0 with 4 tasks -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_897_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 906.0 (TID 2172, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 906.0 (TID 2173, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 906.0 (TID 2174, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 700 -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 906.0 (TID 2175, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 906.0 (TID 2172) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 906.0 (TID 2175) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 906.0 (TID 2174) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 906.0 (TID 2173) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_896_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 906.0 (TID 2174). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 906.0 (TID 2173). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 699 -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 906.0 (TID 2174) in 4 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 906.0 (TID 2173) in 4 ms on localhost (2/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 906.0 (TID 2172). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 906.0 (TID 2172) in 6 ms on localhost (3/4) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_895_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.3 MB) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 906.0 (TID 2175). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 698 -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 906.0 (TID 2175) in 7 ms on localhost (4/4) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 906.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 906 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 719 finished: reduce at LDAOptimizer.scala:456, took 0.013261 s -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_894_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.3 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 697 -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_893_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.3 MB) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 720 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 907(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 907 (PartitionwiseSampledRDD[992] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=796468, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_951 stored as values in memory (estimated size 2.1 KB, free 729.8 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=772144, maxMem=766075207 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_892_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.3 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_951_piece0 stored as bytes in memory (estimated size 1317.0 B, free 729.8 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 696 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_951_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.3 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 951 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 907 (PartitionwiseSampledRDD[992] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 907.0 with 1 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 907.0 (TID 2176, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 907.0 (TID 2176) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_891_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 695 -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 907.0 (TID 2176). 1226 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 907.0 (TID 2176) in 2 ms on localhost (1/1) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 907 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_890_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 907.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 720 finished: isEmpty at LDAOptimizer.scala:418, took 0.008268 s -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 694 -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=716030, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_952 stored as values in memory (estimated size 176.0 B, free 729.9 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_889_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=715750, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_952_piece0 stored as bytes in memory (estimated size 280.0 B, free 729.9 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_952_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 952 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_888_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 693 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_887_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 692 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_886_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 721 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 908(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 691 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 908 (MapPartitionsRDD[994] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=625899, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_953 stored as values in memory (estimated size 3.5 KB, free 730.0 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_885_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=609235, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_953_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.0 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_953_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.4 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 953 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 690 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 908 (MapPartitionsRDD[994] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 908.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 908.0 (TID 2177, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 908.0 (TID 2178, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 908.0 (TID 2179, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 908.0 (TID 2180, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_884_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 908.0 (TID 2179) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 908.0 (TID 2178) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 908.0 (TID 2177) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 908.0 (TID 2180) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 908.0 (TID 2179). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 908.0 (TID 2177). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_883_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 908.0 (TID 2178). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 908.0 (TID 2180). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 689 -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 908.0 (TID 2179) in 4 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 908.0 (TID 2177) in 6 ms on localhost (2/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 908.0 (TID 2178) in 6 ms on localhost (3/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 908.0 (TID 2180) in 6 ms on localhost (4/4) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 908.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 908 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 721 finished: reduce at LDAOptimizer.scala:453, took 0.013920 s -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_882_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 688 -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 722 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 909(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_881_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 909 (MapPartitionsRDD[995] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 687 -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=526666, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_954 stored as values in memory (estimated size 3.5 KB, free 730.1 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_880_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=530122, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_954_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.1 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_954_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.4 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 954 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 909 (MapPartitionsRDD[995] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 909.0 with 4 tasks -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_879_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 909.0 (TID 2181, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 686 -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 909.0 (TID 2182, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 909.0 (TID 2183, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 909.0 (TID 2184, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 909.0 (TID 2183) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 909.0 (TID 2181) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 909.0 (TID 2182) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 909.0 (TID 2184) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_878_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.4 MB) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 909.0 (TID 2183). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 909.0 (TID 2181). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 909.0 (TID 2183) in 6 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 909.0 (TID 2181) in 7 ms on localhost (2/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 909.0 (TID 2184). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 685 -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 909.0 (TID 2184) in 6 ms on localhost (3/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 909.0 (TID 2182). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 909.0 (TID 2182) in 8 ms on localhost (4/4) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 909 (reduce at LDAOptimizer.scala:456) finished in 0.009 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 909.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 722 finished: reduce at LDAOptimizer.scala:456, took 0.015785 s -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_877_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.4 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 684 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_876_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_875_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.4 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 683 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 723 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 910(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 910 (PartitionwiseSampledRDD[996] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=392600, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_955 stored as values in memory (estimated size 2.1 KB, free 730.2 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_874_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.5 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=394752, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_955_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.2 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_955_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 955 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 910 (PartitionwiseSampledRDD[996] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_873_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 910.0 with 1 tasks -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 681 -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 910.0 (TID 2185, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 910.0 (TID 2185) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 910.0 (TID 2185). 1226 bytes result sent to driver -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_872_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 910.0 (TID 2185) in 5 ms on localhost (1/1) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 910 (isEmpty at LDAOptimizer.scala:418) finished in 0.005 s -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 723 finished: isEmpty at LDAOptimizer.scala:418, took 0.012192 s -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=366914, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_956 stored as values in memory (estimated size 176.0 B, free 730.2 MB) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 910.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_871_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=359427, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_956_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 680 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_956_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 956 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_870_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 679 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_869_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.5 MB) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 678 -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 724 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 911(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_868_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 911 (MapPartitionsRDD[998] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 677 -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=256691, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_957 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_867_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2083) called with curMem=260107, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_957_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_957_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 957 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_866_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 911 (MapPartitionsRDD[998] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 911.0 with 4 tasks -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 676 -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 911.0 (TID 2186, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 911.0 (TID 2187, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 911.0 (TID 2188, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 911.0 (TID 2189, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 911.0 (TID 2188) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 911.0 (TID 2189) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 911.0 (TID 2187) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_865_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.5 MB) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 911.0 (TID 2186) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 675 -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 911.0 (TID 2188). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 911.0 (TID 2189). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 911.0 (TID 2187). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 911.0 (TID 2188) in 4 ms on localhost (1/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 911.0 (TID 2186). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 911.0 (TID 2189) in 4 ms on localhost (2/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 911.0 (TID 2187) in 5 ms on localhost (3/4) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_864_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 911.0 (TID 2186) in 5 ms on localhost (4/4) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 911 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 911.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 724 finished: reduce at LDAOptimizer.scala:453, took 0.011017 s -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 674 -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_863_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 725 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 912(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 912 (MapPartitionsRDD[999] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_862_piece0 on localhost:50000 in memory (size: 7.5 KB, free: 730.5 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=151332, maxMem=766075207 -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 673 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_958 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2083) called with curMem=146515, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_958_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_861_piece0 on localhost:50000 in memory (size: 8.3 KB, free: 730.5 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_958_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 958 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 672 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 912 (MapPartitionsRDD[999] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 912.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 912.0 (TID 2190, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 912.0 (TID 2191, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 912.0 (TID 2192, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_860_piece0 on localhost:50000 in memory (size: 8.2 KB, free: 730.5 MB) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 912.0 (TID 2193, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 912.0 (TID 2191) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 912.0 (TID 2193) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 912.0 (TID 2190) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 912.0 (TID 2192) -2016-06-13 19:45:29 INFO ContextCleaner:59 - Cleaned accumulator 671 -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 912.0 (TID 2191). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 912.0 (TID 2190). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 912.0 (TID 2193). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 912.0 (TID 2191) in 4 ms on localhost (1/4) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Removed broadcast_859_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 912.0 (TID 2190) in 4 ms on localhost (2/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 912.0 (TID 2193) in 4 ms on localhost (3/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 912.0 (TID 2192). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 912.0 (TID 2192) in 5 ms on localhost (4/4) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 912 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 912.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 725 finished: reduce at LDAOptimizer.scala:456, took 0.012495 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 726 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 913(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 913 (PartitionwiseSampledRDD[1000] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=99153, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_959 stored as values in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=101305, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_959_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.5 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_959_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 959 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 913 (PartitionwiseSampledRDD[1000] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 913.0 with 1 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 913.0 (TID 2194, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 913.0 (TID 2194) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 913.0 (TID 2194). 1226 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 913.0 (TID 2194) in 2 ms on localhost (1/1) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 913 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 913.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 726 finished: isEmpty at LDAOptimizer.scala:418, took 0.006380 s -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=102622, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_960 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=102798, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_960_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_960_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 960 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 727 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 914(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 914 (MapPartitionsRDD[1002] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=103078, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_961 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=106670, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_961_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_961_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 961 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 914 (MapPartitionsRDD[1002] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 914.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 914.0 (TID 2195, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 914.0 (TID 2196, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 914.0 (TID 2197, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 914.0 (TID 2198, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 914.0 (TID 2195) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 914.0 (TID 2197) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 914.0 (TID 2198) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 914.0 (TID 2196) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 914.0 (TID 2195). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 914.0 (TID 2198). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 914.0 (TID 2195) in 5 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 914.0 (TID 2198) in 4 ms on localhost (2/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 914.0 (TID 2197). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 914.0 (TID 2197) in 6 ms on localhost (3/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 914.0 (TID 2196). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 914.0 (TID 2196) in 6 ms on localhost (4/4) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 914 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 914.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 727 finished: reduce at LDAOptimizer.scala:453, took 0.012419 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 728 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 915(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 915 (MapPartitionsRDD[1003] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=108755, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_962 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=112387, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_962_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_962_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 962 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 915 (MapPartitionsRDD[1003] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 915.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 915.0 (TID 2199, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 915.0 (TID 2200, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 915.0 (TID 2201, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 915.0 (TID 2202, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 915.0 (TID 2200) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 915.0 (TID 2202) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 915.0 (TID 2201) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 915.0 (TID 2199) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 915.0 (TID 2201). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 915.0 (TID 2202). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 915.0 (TID 2201) in 3 ms on localhost (1/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 915.0 (TID 2200). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 915.0 (TID 2202) in 4 ms on localhost (2/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 915.0 (TID 2199). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 915.0 (TID 2200) in 4 ms on localhost (3/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 915.0 (TID 2199) in 5 ms on localhost (4/4) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 915 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 915.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 728 finished: reduce at LDAOptimizer.scala:456, took 0.012926 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 729 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 916(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 916 (PartitionwiseSampledRDD[1004] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=114472, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_963 stored as values in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=116624, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_963_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.5 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_963_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 963 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 916 (PartitionwiseSampledRDD[1004] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 916.0 with 1 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 916.0 (TID 2203, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 916.0 (TID 2203) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 916.0 (TID 2203). 1226 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 916.0 (TID 2203) in 2 ms on localhost (1/1) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 916 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 916.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 729 finished: isEmpty at LDAOptimizer.scala:418, took 0.006078 s -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=117941, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_964 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=118117, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_964_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_964_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 964 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 730 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 917(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 917 (MapPartitionsRDD[1006] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=118397, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_965 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=121989, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_965_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_965_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 965 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 917 (MapPartitionsRDD[1006] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 917.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 917.0 (TID 2204, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 917.0 (TID 2205, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 917.0 (TID 2206, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 917.0 (TID 2207, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 917.0 (TID 2204) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 917.0 (TID 2207) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 917.0 (TID 2205) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 917.0 (TID 2206) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 917.0 (TID 2207). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 917.0 (TID 2204). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 917.0 (TID 2207) in 3 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 917.0 (TID 2204) in 5 ms on localhost (2/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 917.0 (TID 2205). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 917.0 (TID 2205) in 4 ms on localhost (3/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 917.0 (TID 2206). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 917.0 (TID 2206) in 5 ms on localhost (4/4) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 917 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 917.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 730 finished: reduce at LDAOptimizer.scala:453, took 0.010098 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 731 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 918(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 918 (MapPartitionsRDD[1007] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=124074, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_966 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=127706, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_966_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_966_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 966 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 918 (MapPartitionsRDD[1007] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 918.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 918.0 (TID 2208, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 918.0 (TID 2209, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 918.0 (TID 2210, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 918.0 (TID 2211, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 918.0 (TID 2208) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 918.0 (TID 2209) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 918.0 (TID 2210) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 918.0 (TID 2211) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 918.0 (TID 2208). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 918.0 (TID 2209). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 918.0 (TID 2208) in 4 ms on localhost (1/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 918.0 (TID 2210). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 918.0 (TID 2209) in 5 ms on localhost (2/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 918.0 (TID 2211). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 918.0 (TID 2210) in 4 ms on localhost (3/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 918.0 (TID 2211) in 4 ms on localhost (4/4) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 918 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 918.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 731 finished: reduce at LDAOptimizer.scala:456, took 0.010115 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 732 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 919(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 919 (PartitionwiseSampledRDD[1008] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=129791, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_967 stored as values in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=131943, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_967_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.5 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_967_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 967 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 919 (PartitionwiseSampledRDD[1008] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 919.0 with 1 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 919.0 (TID 2212, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 919.0 (TID 2212) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 919.0 (TID 2212). 1226 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 919.0 (TID 2212) in 3 ms on localhost (1/1) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 919 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 919.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 732 finished: isEmpty at LDAOptimizer.scala:418, took 0.006389 s -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=133260, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_968 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=133436, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_968_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_968_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 968 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 733 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 920(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 920 (MapPartitionsRDD[1010] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=133716, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_969 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=137308, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_969_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_969_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 969 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 920 (MapPartitionsRDD[1010] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 920.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 920.0 (TID 2213, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 920.0 (TID 2214, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 920.0 (TID 2215, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 920.0 (TID 2216, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 920.0 (TID 2214) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 920.0 (TID 2216) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 920.0 (TID 2215) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 920.0 (TID 2213) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 920.0 (TID 2213). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 920.0 (TID 2214). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 920.0 (TID 2215). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 920.0 (TID 2213) in 4 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 920.0 (TID 2214) in 4 ms on localhost (2/4) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 920.0 (TID 2216). 1265 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 920.0 (TID 2215) in 4 ms on localhost (3/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 920.0 (TID 2216) in 4 ms on localhost (4/4) -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 920 (reduce at LDAOptimizer.scala:453) finished in 0.004 s -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 920.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 733 finished: reduce at LDAOptimizer.scala:453, took 0.010059 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 734 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 921(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 921 (MapPartitionsRDD[1011] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=139393, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_970 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=143025, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_970_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_970_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 970 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 921 (MapPartitionsRDD[1011] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 921.0 with 4 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 921.0 (TID 2217, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 921.0 (TID 2218, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 2.0 in stage 921.0 (TID 2219, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 3.0 in stage 921.0 (TID 2220, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 2.0 in stage 921.0 (TID 2219) -2016-06-13 19:45:29 INFO Executor:59 - Running task 3.0 in stage 921.0 (TID 2220) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 921.0 (TID 2217) -2016-06-13 19:45:29 INFO Executor:59 - Running task 1.0 in stage 921.0 (TID 2218) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 3.0 in stage 921.0 (TID 2220). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 2.0 in stage 921.0 (TID 2219). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 1.0 in stage 921.0 (TID 2218). 1358 bytes result sent to driver -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 921.0 (TID 2217). 1314 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 3.0 in stage 921.0 (TID 2220) in 4 ms on localhost (1/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 2.0 in stage 921.0 (TID 2219) in 4 ms on localhost (2/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 921.0 (TID 2218) in 6 ms on localhost (3/4) -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 921.0 (TID 2217) in 6 ms on localhost (4/4) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 921.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 921 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 734 finished: reduce at LDAOptimizer.scala:456, took 0.012881 s -2016-06-13 19:45:29 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Got job 735 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:29 INFO DAGScheduler:59 - Final stage: ResultStage 922(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting ResultStage 922 (PartitionwiseSampledRDD[1012] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=145110, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_971 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=147262, maxMem=766075207 -2016-06-13 19:45:29 INFO MemoryStore:59 - Block broadcast_971_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.4 MB) -2016-06-13 19:45:29 INFO BlockManagerInfo:59 - Added broadcast_971_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:29 INFO SparkContext:59 - Created broadcast 971 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 922 (PartitionwiseSampledRDD[1012] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Adding task set 922.0 with 1 tasks -2016-06-13 19:45:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 922.0 (TID 2221, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:29 INFO Executor:59 - Running task 0.0 in stage 922.0 (TID 2221) -2016-06-13 19:45:29 INFO Executor:59 - Finished task 0.0 in stage 922.0 (TID 2221). 1226 bytes result sent to driver -2016-06-13 19:45:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 922.0 (TID 2221) in 1 ms on localhost (1/1) -2016-06-13 19:45:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 922.0, whose tasks have all completed, from pool -2016-06-13 19:45:29 INFO DAGScheduler:59 - ResultStage 922 (isEmpty at LDAOptimizer.scala:418) finished in 0.001 s -2016-06-13 19:45:29 INFO DAGScheduler:59 - Job 735 finished: isEmpty at LDAOptimizer.scala:418, took 0.008016 s -2016-06-13 19:45:29 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=148577, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_972 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=148753, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_972_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_972_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 972 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 736 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 923(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 923 (MapPartitionsRDD[1014] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=149033, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_973 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=152625, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_973_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_973_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 973 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 923 (MapPartitionsRDD[1014] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 923.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 923.0 (TID 2222, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 923.0 (TID 2223, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 923.0 (TID 2224, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 923.0 (TID 2225, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 923.0 (TID 2223) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 923.0 (TID 2224) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 923.0 (TID 2225) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 923.0 (TID 2222) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 923.0 (TID 2223). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 923.0 (TID 2223) in 5 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 923.0 (TID 2222). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 923.0 (TID 2225). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 923.0 (TID 2222) in 7 ms on localhost (2/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 923.0 (TID 2224). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 923.0 (TID 2225) in 6 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 923.0 (TID 2224) in 7 ms on localhost (4/4) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 923.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 923 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 736 finished: reduce at LDAOptimizer.scala:453, took 0.012958 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 737 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 924(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 924 (MapPartitionsRDD[1015] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=154709, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_974 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=158341, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_974_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_974_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 974 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 924 (MapPartitionsRDD[1015] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 924.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 924.0 (TID 2226, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 924.0 (TID 2227, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 924.0 (TID 2228, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 924.0 (TID 2229, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 924.0 (TID 2226) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 924.0 (TID 2229) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 924.0 (TID 2228) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 924.0 (TID 2227) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 924.0 (TID 2226). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 924.0 (TID 2228). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 924.0 (TID 2226) in 4 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 924.0 (TID 2227). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 924.0 (TID 2229). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 924.0 (TID 2228) in 4 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 924.0 (TID 2227) in 4 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 924.0 (TID 2229) in 4 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 924 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 924.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 737 finished: reduce at LDAOptimizer.scala:456, took 0.009869 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 738 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 925(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 925 (PartitionwiseSampledRDD[1016] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=160425, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_975 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=162577, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_975_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_975_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 975 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 925 (PartitionwiseSampledRDD[1016] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 925.0 with 1 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 925.0 (TID 2230, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 925.0 (TID 2230) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 925.0 (TID 2230). 1226 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 925.0 (TID 2230) in 2 ms on localhost (1/1) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 925 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 925.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 738 finished: isEmpty at LDAOptimizer.scala:418, took 0.008483 s -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=163894, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_976 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=164070, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_976_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_976_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 976 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 739 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 926(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 926 (MapPartitionsRDD[1018] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=164350, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_977 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=167942, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_977_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_977_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 977 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 926 (MapPartitionsRDD[1018] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 926.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 926.0 (TID 2231, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 926.0 (TID 2232, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 926.0 (TID 2233, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 926.0 (TID 2234, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 926.0 (TID 2233) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 926.0 (TID 2234) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 926.0 (TID 2232) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 926.0 (TID 2231) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 926.0 (TID 2232). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 926.0 (TID 2234). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 926.0 (TID 2233). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 926.0 (TID 2231). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 926.0 (TID 2232) in 4 ms on localhost (1/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 926.0 (TID 2234) in 4 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 926.0 (TID 2233) in 4 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 926.0 (TID 2231) in 6 ms on localhost (4/4) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 926.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 926 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 739 finished: reduce at LDAOptimizer.scala:453, took 0.009945 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 740 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 927(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 927 (MapPartitionsRDD[1019] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=170029, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_978 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=173661, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_978_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_978_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 978 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 927 (MapPartitionsRDD[1019] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 927.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 927.0 (TID 2235, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 927.0 (TID 2236, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 927.0 (TID 2237, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 927.0 (TID 2238, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 927.0 (TID 2236) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 927.0 (TID 2238) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 927.0 (TID 2237) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 927.0 (TID 2235) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 927.0 (TID 2237). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 927.0 (TID 2236). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 927.0 (TID 2237) in 5 ms on localhost (1/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 927.0 (TID 2236) in 6 ms on localhost (2/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 927.0 (TID 2238). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 927.0 (TID 2238) in 6 ms on localhost (3/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 927.0 (TID 2235). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 927.0 (TID 2235) in 9 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 927 (reduce at LDAOptimizer.scala:456) finished in 0.009 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 927.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 740 finished: reduce at LDAOptimizer.scala:456, took 0.015799 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 741 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 928(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 928 (PartitionwiseSampledRDD[1020] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=175748, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_979 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(1318) called with curMem=177900, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_979_piece0 stored as bytes in memory (estimated size 1318.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_979_piece0 in memory on localhost:50000 (size: 1318.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 979 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 928 (PartitionwiseSampledRDD[1020] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 928.0 with 1 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 928.0 (TID 2239, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 928.0 (TID 2239) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 928.0 (TID 2239). 1226 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 928.0 (TID 2239) in 3 ms on localhost (1/1) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 928 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 928.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 741 finished: isEmpty at LDAOptimizer.scala:418, took 0.011319 s -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=179218, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_980 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=179394, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_980_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_980_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 980 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 742 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 929(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 929 (MapPartitionsRDD[1022] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=179674, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_981 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=183266, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_981_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_981_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 981 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 929 (MapPartitionsRDD[1022] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 929.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 929.0 (TID 2240, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 929.0 (TID 2241, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 929.0 (TID 2242, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 929.0 (TID 2243, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 929.0 (TID 2241) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 929.0 (TID 2242) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 929.0 (TID 2240) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 929.0 (TID 2243) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 929.0 (TID 2240). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 929.0 (TID 2241). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 929.0 (TID 2242). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 929.0 (TID 2240) in 6 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 929.0 (TID 2243). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 929.0 (TID 2241) in 5 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 929.0 (TID 2242) in 5 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 929.0 (TID 2243) in 5 ms on localhost (4/4) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 929.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 929 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 742 finished: reduce at LDAOptimizer.scala:453, took 0.013294 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 743 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 930(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 930 (MapPartitionsRDD[1023] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=185351, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_982 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=188983, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_982_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_982_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 982 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 930 (MapPartitionsRDD[1023] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 930.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 930.0 (TID 2244, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 930.0 (TID 2245, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 930.0 (TID 2246, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 930.0 (TID 2247, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 930.0 (TID 2244) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 930.0 (TID 2247) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 930.0 (TID 2246) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 930.0 (TID 2245) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 930.0 (TID 2244). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 930.0 (TID 2247). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 930.0 (TID 2245). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 930.0 (TID 2244) in 4 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 930.0 (TID 2246). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 930.0 (TID 2247) in 3 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 930.0 (TID 2245) in 4 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 930.0 (TID 2246) in 3 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 930 (reduce at LDAOptimizer.scala:456) finished in 0.004 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 930.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 743 finished: reduce at LDAOptimizer.scala:456, took 0.009745 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 744 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 931(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 931 (PartitionwiseSampledRDD[1024] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=191069, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_983 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=193221, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_983_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_983_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 983 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 931 (PartitionwiseSampledRDD[1024] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 931.0 with 1 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 931.0 (TID 2248, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 931.0 (TID 2248) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 931.0 (TID 2248). 1226 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 931.0 (TID 2248) in 2 ms on localhost (1/1) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 931 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 931.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 744 finished: isEmpty at LDAOptimizer.scala:418, took 0.007878 s -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=194536, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_984 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=194712, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_984_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_984_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 984 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 745 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 932(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 932 (MapPartitionsRDD[1026] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=194992, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_985 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=198584, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_985_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_985_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 985 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 932 (MapPartitionsRDD[1026] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 932.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 932.0 (TID 2249, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 932.0 (TID 2250, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 932.0 (TID 2251, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 932.0 (TID 2252, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 932.0 (TID 2250) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 932.0 (TID 2251) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 932.0 (TID 2252) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 932.0 (TID 2249) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 932.0 (TID 2251). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 932.0 (TID 2250). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 932.0 (TID 2249). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 932.0 (TID 2251) in 4 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 932.0 (TID 2252). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 932.0 (TID 2250) in 4 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 932.0 (TID 2249) in 5 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 932.0 (TID 2252) in 5 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 932 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 932.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 745 finished: reduce at LDAOptimizer.scala:453, took 0.009926 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 746 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 933(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 933 (MapPartitionsRDD[1027] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=200670, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_986 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=204302, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_986_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_986_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 986 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 933 (MapPartitionsRDD[1027] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 933.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 933.0 (TID 2253, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 933.0 (TID 2254, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 933.0 (TID 2255, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 933.0 (TID 2256, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 933.0 (TID 2255) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 933.0 (TID 2256) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 933.0 (TID 2254) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 933.0 (TID 2253) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 933.0 (TID 2256). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 933.0 (TID 2253). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 933.0 (TID 2253) in 7 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 933.0 (TID 2254). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 933.0 (TID 2256) in 8 ms on localhost (2/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 933.0 (TID 2255). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 933.0 (TID 2254) in 10 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 933.0 (TID 2255) in 10 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 933 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 933.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 746 finished: reduce at LDAOptimizer.scala:456, took 0.014759 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 747 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 934(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 934 (PartitionwiseSampledRDD[1028] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=206389, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_987 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(1316) called with curMem=208541, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_987_piece0 stored as bytes in memory (estimated size 1316.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_987_piece0 in memory on localhost:50000 (size: 1316.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 987 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 934 (PartitionwiseSampledRDD[1028] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 934.0 with 1 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 934.0 (TID 2257, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 934.0 (TID 2257) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 934.0 (TID 2257). 1226 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 934.0 (TID 2257) in 3 ms on localhost (1/1) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 934.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 934 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 747 finished: isEmpty at LDAOptimizer.scala:418, took 0.007184 s -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=209857, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_988 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=210033, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_988_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_988_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 988 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 748 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 935(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 935 (MapPartitionsRDD[1030] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=210313, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_989 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2078) called with curMem=213905, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_989_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_989_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 989 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 935 (MapPartitionsRDD[1030] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 935.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 935.0 (TID 2258, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 935.0 (TID 2259, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 935.0 (TID 2260, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 935.0 (TID 2261, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 935.0 (TID 2259) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 935.0 (TID 2261) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 935.0 (TID 2258) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 935.0 (TID 2260) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 935.0 (TID 2258). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 935.0 (TID 2259). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 935.0 (TID 2261). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 935.0 (TID 2260). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 935.0 (TID 2258) in 4 ms on localhost (1/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 935.0 (TID 2261) in 4 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 935.0 (TID 2260) in 4 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 935.0 (TID 2259) in 4 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 935 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 935.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 748 finished: reduce at LDAOptimizer.scala:453, took 0.009238 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 749 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 936(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 936 (MapPartitionsRDD[1031] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=215983, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_990 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2078) called with curMem=219615, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_990_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_990_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 990 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 936 (MapPartitionsRDD[1031] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 936.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 936.0 (TID 2262, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 936.0 (TID 2263, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 936.0 (TID 2264, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 936.0 (TID 2265, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 936.0 (TID 2262) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 936.0 (TID 2265) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 936.0 (TID 2263) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 936.0 (TID 2264) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 936.0 (TID 2265). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 936.0 (TID 2264). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 936.0 (TID 2263). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 936.0 (TID 2265) in 4 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 936.0 (TID 2262). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 936.0 (TID 2264) in 4 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 936.0 (TID 2263) in 4 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 936.0 (TID 2262) in 4 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 936 (reduce at LDAOptimizer.scala:456) finished in 0.004 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 936.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 749 finished: reduce at LDAOptimizer.scala:456, took 0.009867 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 750 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 937(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 937 (PartitionwiseSampledRDD[1032] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=221693, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_991 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=223845, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_991_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_991_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 991 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 937 (PartitionwiseSampledRDD[1032] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 937.0 with 1 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 937.0 (TID 2266, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 937.0 (TID 2266) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 937.0 (TID 2266). 1226 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 937.0 (TID 2266) in 2 ms on localhost (1/1) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 937 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 937.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 750 finished: isEmpty at LDAOptimizer.scala:418, took 0.006550 s -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=225160, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_992 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=225336, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_992_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_992_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 992 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 751 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 938(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 938 (MapPartitionsRDD[1034] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=225616, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_993 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=229208, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_993_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_993_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 993 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 938 (MapPartitionsRDD[1034] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 938.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 938.0 (TID 2267, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 938.0 (TID 2268, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 938.0 (TID 2269, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 938.0 (TID 2270, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 938.0 (TID 2269) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 938.0 (TID 2270) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 938.0 (TID 2268) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 938.0 (TID 2267) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 938.0 (TID 2267). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 938.0 (TID 2268). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 938.0 (TID 2269). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 938.0 (TID 2270). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 938.0 (TID 2268) in 3 ms on localhost (1/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 938.0 (TID 2267) in 4 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 938.0 (TID 2270) in 4 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 938.0 (TID 2269) in 4 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 938 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 938.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 751 finished: reduce at LDAOptimizer.scala:453, took 0.008801 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 752 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 939(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 939 (MapPartitionsRDD[1035] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=231292, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_994 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=234924, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_994_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_994_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 994 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 939 (MapPartitionsRDD[1035] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 939.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 939.0 (TID 2271, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 939.0 (TID 2272, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 939.0 (TID 2273, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 939.0 (TID 2274, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 939.0 (TID 2272) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 939.0 (TID 2274) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 939.0 (TID 2273) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 939.0 (TID 2271) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 939.0 (TID 2272). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 939.0 (TID 2271). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 939.0 (TID 2272) in 4 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 939.0 (TID 2274). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 939.0 (TID 2273). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 939.0 (TID 2271) in 4 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 939.0 (TID 2274) in 5 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 939.0 (TID 2273) in 5 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 939 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 939.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 752 finished: reduce at LDAOptimizer.scala:456, took 0.010109 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 753 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 940(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 940 (PartitionwiseSampledRDD[1036] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=237008, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_995 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=239160, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_995_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_995_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 995 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 940 (PartitionwiseSampledRDD[1036] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 940.0 with 1 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 940.0 (TID 2275, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 940.0 (TID 2275) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 940.0 (TID 2275). 1226 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 940.0 (TID 2275) in 2 ms on localhost (1/1) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 940.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 940 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 753 finished: isEmpty at LDAOptimizer.scala:418, took 0.007177 s -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=240475, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_996 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=240651, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_996_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_996_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 996 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 754 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 941(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 941 (MapPartitionsRDD[1038] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=240931, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_997 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2083) called with curMem=244523, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_997_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_997_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 997 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 941 (MapPartitionsRDD[1038] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 941.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 941.0 (TID 2276, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 941.0 (TID 2277, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 941.0 (TID 2278, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 941.0 (TID 2279, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 941.0 (TID 2278) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 941.0 (TID 2279) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 941.0 (TID 2277) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 941.0 (TID 2276) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 941.0 (TID 2276). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 941.0 (TID 2278). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 941.0 (TID 2277). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 941.0 (TID 2279). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 941.0 (TID 2276) in 5 ms on localhost (1/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 941.0 (TID 2278) in 6 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 941.0 (TID 2279) in 6 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 941.0 (TID 2277) in 8 ms on localhost (4/4) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 941.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 941 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 754 finished: reduce at LDAOptimizer.scala:453, took 0.012891 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 755 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 942(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 942 (MapPartitionsRDD[1039] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=246606, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_998 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2083) called with curMem=250238, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_998_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_998_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 998 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 942 (MapPartitionsRDD[1039] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 942.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 942.0 (TID 2280, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 942.0 (TID 2281, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 942.0 (TID 2282, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 942.0 (TID 2283, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 942.0 (TID 2281) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 942.0 (TID 2283) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 942.0 (TID 2280) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 942.0 (TID 2282) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 942.0 (TID 2280). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 942.0 (TID 2283). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 942.0 (TID 2282). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 942.0 (TID 2280) in 3 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 942.0 (TID 2281). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 942.0 (TID 2283) in 3 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 942.0 (TID 2282) in 4 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 942.0 (TID 2281) in 4 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 942 (reduce at LDAOptimizer.scala:456) finished in 0.004 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 942.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 755 finished: reduce at LDAOptimizer.scala:456, took 0.008811 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 756 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 943(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 943 (PartitionwiseSampledRDD[1040] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=252321, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_999 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=254473, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_999_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_999_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 999 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 943 (PartitionwiseSampledRDD[1040] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 943.0 with 1 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 943.0 (TID 2284, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 943.0 (TID 2284) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 943.0 (TID 2284). 1226 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 943.0 (TID 2284) in 2 ms on localhost (1/1) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 943 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 943.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 756 finished: isEmpty at LDAOptimizer.scala:418, took 0.007581 s -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=255788, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1000 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=255964, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1000_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1000_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1000 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 757 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 944(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 944 (MapPartitionsRDD[1042] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=256244, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1001 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=259836, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1001_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1001_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1001 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 944 (MapPartitionsRDD[1042] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 944.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 944.0 (TID 2285, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 944.0 (TID 2286, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 944.0 (TID 2287, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 944.0 (TID 2288, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 944.0 (TID 2285) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 944.0 (TID 2288) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 944.0 (TID 2286) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 944.0 (TID 2287) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 944.0 (TID 2286). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 944.0 (TID 2287). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 944.0 (TID 2285). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 944.0 (TID 2286) in 3 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 944.0 (TID 2288). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 944.0 (TID 2287) in 4 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 944.0 (TID 2288) in 4 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 944.0 (TID 2285) in 4 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 944 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 944.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 757 finished: reduce at LDAOptimizer.scala:453, took 0.009726 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 758 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 945(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 945 (MapPartitionsRDD[1043] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=261920, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1002 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=265552, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1002_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1002_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1002 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 945 (MapPartitionsRDD[1043] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 945.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 945.0 (TID 2289, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 945.0 (TID 2290, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 945.0 (TID 2291, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 945.0 (TID 2292, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 945.0 (TID 2289) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 945.0 (TID 2290) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 945.0 (TID 2291) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 945.0 (TID 2292) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 945.0 (TID 2289). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 945.0 (TID 2291). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 945.0 (TID 2289) in 5 ms on localhost (1/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 945.0 (TID 2291) in 4 ms on localhost (2/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 945.0 (TID 2290). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 945.0 (TID 2292). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 945.0 (TID 2290) in 7 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 945.0 (TID 2292) in 7 ms on localhost (4/4) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 945.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 945 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 758 finished: reduce at LDAOptimizer.scala:456, took 0.012699 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 759 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 946(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 946 (PartitionwiseSampledRDD[1044] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=267636, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1003 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=269788, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1003_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1003_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1003 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 946 (PartitionwiseSampledRDD[1044] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 946.0 with 1 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 946.0 (TID 2293, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 946.0 (TID 2293) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 946.0 (TID 2293). 1226 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 946.0 (TID 2293) in 2 ms on localhost (1/1) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 946 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 946.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 759 finished: isEmpty at LDAOptimizer.scala:418, took 0.007426 s -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=271103, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1004 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=271279, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1004_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1004_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1004 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 760 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 947(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 947 (MapPartitionsRDD[1046] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=271559, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1005 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=275151, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1005_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1005_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1005 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 947 (MapPartitionsRDD[1046] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 947.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 947.0 (TID 2294, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 947.0 (TID 2295, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 947.0 (TID 2296, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 947.0 (TID 2297, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 947.0 (TID 2295) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 947.0 (TID 2297) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 947.0 (TID 2294) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 947.0 (TID 2296) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 947.0 (TID 2296). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 947.0 (TID 2295). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 947.0 (TID 2294). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 947.0 (TID 2296) in 3 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 947.0 (TID 2297). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 947.0 (TID 2295) in 3 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 947.0 (TID 2294) in 4 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 947.0 (TID 2297) in 4 ms on localhost (4/4) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 947.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 947 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 760 finished: reduce at LDAOptimizer.scala:453, took 0.009781 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 761 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 948(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 948 (MapPartitionsRDD[1047] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=277235, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1006 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=280867, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1006_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1006_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1006 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 948 (MapPartitionsRDD[1047] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 948.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 948.0 (TID 2298, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 948.0 (TID 2299, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 948.0 (TID 2300, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 948.0 (TID 2301, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 948.0 (TID 2298) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 948.0 (TID 2299) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 948.0 (TID 2301) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 948.0 (TID 2300) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 948.0 (TID 2298). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 948.0 (TID 2299). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 948.0 (TID 2300). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 948.0 (TID 2298) in 5 ms on localhost (1/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 948.0 (TID 2299) in 4 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 948.0 (TID 2300) in 5 ms on localhost (3/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 948.0 (TID 2301). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 948.0 (TID 2301) in 5 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 948 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 948.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 761 finished: reduce at LDAOptimizer.scala:456, took 0.012089 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 762 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 949(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 949 (PartitionwiseSampledRDD[1048] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=282951, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1007 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=285103, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1007_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1007_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1007 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 949 (PartitionwiseSampledRDD[1048] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 949.0 with 1 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 949.0 (TID 2302, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 949.0 (TID 2302) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 949.0 (TID 2302). 1226 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 949.0 (TID 2302) in 3 ms on localhost (1/1) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 949 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 949.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 762 finished: isEmpty at LDAOptimizer.scala:418, took 0.007189 s -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=286420, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1008 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=286596, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1008_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1008_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1008 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 763 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 950(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 950 (MapPartitionsRDD[1050] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=286876, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1009 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=290468, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1009_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1009_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1009 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 950 (MapPartitionsRDD[1050] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 950.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 950.0 (TID 2303, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 950.0 (TID 2304, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 950.0 (TID 2305, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 950.0 (TID 2306, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 950.0 (TID 2305) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 950.0 (TID 2303) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 950.0 (TID 2304) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 950.0 (TID 2306) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 950.0 (TID 2305). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 950.0 (TID 2303). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 950.0 (TID 2306). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 950.0 (TID 2304). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 950.0 (TID 2305) in 5 ms on localhost (1/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 950.0 (TID 2303) in 5 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 950.0 (TID 2306) in 4 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 950.0 (TID 2304) in 6 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 950 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 950.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 763 finished: reduce at LDAOptimizer.scala:453, took 0.011010 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 764 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 951(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 951 (MapPartitionsRDD[1051] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=292554, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1010 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=296186, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1010_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1010_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1010 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 951 (MapPartitionsRDD[1051] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 951.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 951.0 (TID 2307, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 951.0 (TID 2308, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 951.0 (TID 2309, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 951.0 (TID 2310, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 951.0 (TID 2308) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 951.0 (TID 2309) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 951.0 (TID 2310) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 951.0 (TID 2307) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 951.0 (TID 2308). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 951.0 (TID 2308) in 5 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 951.0 (TID 2310). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 951.0 (TID 2309). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 951.0 (TID 2307). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 951.0 (TID 2310) in 5 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 951.0 (TID 2309) in 6 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 951.0 (TID 2307) in 8 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 951 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 951.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 764 finished: reduce at LDAOptimizer.scala:456, took 0.013457 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 765 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 952(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 952 (PartitionwiseSampledRDD[1052] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=298272, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1011 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=300424, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1011_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1011_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1011 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 952 (PartitionwiseSampledRDD[1052] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 952.0 with 1 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 952.0 (TID 2311, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 952.0 (TID 2311) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 952.0 (TID 2311). 1226 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 952.0 (TID 2311) in 2 ms on localhost (1/1) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 952.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 952 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 765 finished: isEmpty at LDAOptimizer.scala:418, took 0.006265 s -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=301739, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1012 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=301915, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1012_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1012_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1012 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 766 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 953(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 953 (MapPartitionsRDD[1054] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=302195, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1013 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=305787, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1013_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1013_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1013 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 953 (MapPartitionsRDD[1054] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 953.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 953.0 (TID 2312, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 953.0 (TID 2313, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 953.0 (TID 2314, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 953.0 (TID 2315, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 953.0 (TID 2313) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 953.0 (TID 2312) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 953.0 (TID 2313). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 953.0 (TID 2312). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 953.0 (TID 2313) in 5 ms on localhost (1/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 953.0 (TID 2312) in 6 ms on localhost (2/4) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 953.0 (TID 2314) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 953.0 (TID 2315) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 953.0 (TID 2314). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 953.0 (TID 2314) in 10 ms on localhost (3/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 953.0 (TID 2315). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 953.0 (TID 2315) in 13 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 953 (reduce at LDAOptimizer.scala:453) finished in 0.014 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 953.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 766 finished: reduce at LDAOptimizer.scala:453, took 0.021008 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 767 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 954(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 954 (MapPartitionsRDD[1055] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=307871, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1014 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=311503, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1014_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1014_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1014 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 954 (MapPartitionsRDD[1055] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 954.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 954.0 (TID 2316, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 954.0 (TID 2317, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 954.0 (TID 2318, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 954.0 (TID 2319, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 954.0 (TID 2317) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 954.0 (TID 2319) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 954.0 (TID 2318) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 954.0 (TID 2316) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 954.0 (TID 2317). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 954.0 (TID 2317) in 4 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 954.0 (TID 2318). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 954.0 (TID 2319). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 954.0 (TID 2316). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 954.0 (TID 2318) in 5 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 954.0 (TID 2316) in 5 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 954.0 (TID 2319) in 4 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 954 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 954.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 767 finished: reduce at LDAOptimizer.scala:456, took 0.011580 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 768 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 955(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 955 (PartitionwiseSampledRDD[1056] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=313587, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1015 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(1318) called with curMem=315739, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1015_piece0 stored as bytes in memory (estimated size 1318.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1015_piece0 in memory on localhost:50000 (size: 1318.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1015 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 955 (PartitionwiseSampledRDD[1056] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 955.0 with 1 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 955.0 (TID 2320, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 955.0 (TID 2320) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 955.0 (TID 2320). 1226 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 955.0 (TID 2320) in 3 ms on localhost (1/1) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 955 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 955.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 768 finished: isEmpty at LDAOptimizer.scala:418, took 0.009469 s -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=317057, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1016 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=317233, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1016_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1016_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1016 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 769 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 956(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 956 (MapPartitionsRDD[1058] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=317513, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1017 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=321105, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1017_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1017_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1017 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 956 (MapPartitionsRDD[1058] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 956.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 956.0 (TID 2321, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 956.0 (TID 2322, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 956.0 (TID 2323, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 956.0 (TID 2324, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 956.0 (TID 2321) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 956.0 (TID 2323) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 956.0 (TID 2324) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 956.0 (TID 2322) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 956.0 (TID 2323). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 956.0 (TID 2324). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 956.0 (TID 2321). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 956.0 (TID 2323) in 4 ms on localhost (1/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 956.0 (TID 2324) in 3 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 956.0 (TID 2321) in 5 ms on localhost (3/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 956.0 (TID 2322). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 956.0 (TID 2322) in 5 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 956 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 956.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 769 finished: reduce at LDAOptimizer.scala:453, took 0.012158 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 770 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 957(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 957 (MapPartitionsRDD[1059] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=323191, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1018 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=326823, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1018_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1018_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1018 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 957 (MapPartitionsRDD[1059] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 957.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 957.0 (TID 2325, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 957.0 (TID 2326, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 957.0 (TID 2327, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 957.0 (TID 2328, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 957.0 (TID 2325) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 957.0 (TID 2326) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 957.0 (TID 2327) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 957.0 (TID 2328) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 957.0 (TID 2327). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 957.0 (TID 2325). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 957.0 (TID 2326). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 957.0 (TID 2327) in 4 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 957.0 (TID 2328). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 957.0 (TID 2325) in 4 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 957.0 (TID 2326) in 4 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 957.0 (TID 2328) in 4 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 957 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 957.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 770 finished: reduce at LDAOptimizer.scala:456, took 0.010650 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 771 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 958(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 958 (PartitionwiseSampledRDD[1060] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=328909, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1019 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=331061, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1019_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1019_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1019 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 958 (PartitionwiseSampledRDD[1060] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 958.0 with 1 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 958.0 (TID 2329, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 958.0 (TID 2329) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 958.0 (TID 2329). 1226 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 958.0 (TID 2329) in 2 ms on localhost (1/1) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 958.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 958 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 771 finished: isEmpty at LDAOptimizer.scala:418, took 0.006874 s -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=332376, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1020 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=332552, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1020_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1020_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1020 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 772 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 959(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 959 (MapPartitionsRDD[1062] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=332832, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1021 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2080) called with curMem=336424, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1021_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1021_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.4 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1021 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 959 (MapPartitionsRDD[1062] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 959.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 959.0 (TID 2330, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 959.0 (TID 2331, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 959.0 (TID 2332, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 959.0 (TID 2333, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 959.0 (TID 2331) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 959.0 (TID 2333) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 959.0 (TID 2332) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 959.0 (TID 2330) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 959.0 (TID 2331). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 959.0 (TID 2332). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 959.0 (TID 2331) in 3 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 959.0 (TID 2333). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 959.0 (TID 2330). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 959.0 (TID 2332) in 4 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 959.0 (TID 2330) in 4 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 959.0 (TID 2333) in 4 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 959 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 959.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 772 finished: reduce at LDAOptimizer.scala:453, took 0.009379 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 773 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 960(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 960 (MapPartitionsRDD[1063] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=338504, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1022 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2080) called with curMem=342136, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1022_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1022_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.4 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1022 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 960 (MapPartitionsRDD[1063] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 960.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 960.0 (TID 2334, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 960.0 (TID 2335, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 960.0 (TID 2336, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 960.0 (TID 2337, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 960.0 (TID 2334) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 960.0 (TID 2337) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 960.0 (TID 2336) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 960.0 (TID 2335) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 960.0 (TID 2337). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 960.0 (TID 2337) in 3 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 960.0 (TID 2336). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 960.0 (TID 2334). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 960.0 (TID 2335). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 960.0 (TID 2336) in 6 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 960.0 (TID 2334) in 7 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 960.0 (TID 2335) in 6 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 960 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 960.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 773 finished: reduce at LDAOptimizer.scala:456, took 0.013324 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 774 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 961(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 961 (PartitionwiseSampledRDD[1064] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1022_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=338504, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1023 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 794 -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=337064, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1023_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1023_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.4 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1023 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1021_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.4 MB) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 961 (PartitionwiseSampledRDD[1064] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 961.0 with 1 tasks -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 793 -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 961.0 (TID 2338, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 961.0 (TID 2338) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 961.0 (TID 2338). 1226 bytes result sent to driver -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1020_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 961.0 (TID 2338) in 2 ms on localhost (1/1) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 961 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 961.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 774 finished: isEmpty at LDAOptimizer.scala:418, took 0.007584 s -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=332376, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1024 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1019_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 792 -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=330466, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1024_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1018_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1024_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 791 -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1024 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1017_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 790 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1016_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 775 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 962(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 962 (MapPartitionsRDD[1066] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=317510, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1025 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1015_piece0 on localhost:50000 in memory (size: 1318.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 789 -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=321102, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1025_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1025_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1014_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1025 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 962 (MapPartitionsRDD[1066] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 962.0 with 4 tasks -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 788 -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 962.0 (TID 2339, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 962.0 (TID 2340, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 962.0 (TID 2341, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 962.0 (TID 2342, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 962.0 (TID 2340) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 962.0 (TID 2342) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1013_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 962.0 (TID 2339) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 962.0 (TID 2341) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 962.0 (TID 2340). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 962.0 (TID 2339). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 962.0 (TID 2340) in 5 ms on localhost (1/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 962.0 (TID 2339) in 6 ms on localhost (2/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 962.0 (TID 2341). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 787 -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 962.0 (TID 2341) in 6 ms on localhost (3/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 962.0 (TID 2342). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 962 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 962.0 (TID 2342) in 6 ms on localhost (4/4) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 962.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 775 finished: reduce at LDAOptimizer.scala:453, took 0.012798 s -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1012_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1011_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 786 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 776 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 963(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 963 (MapPartitionsRDD[1067] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=305785, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1026 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1010_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 785 -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=305785, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1026_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1026_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1026 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1009_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 963 (MapPartitionsRDD[1067] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 963.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 963.0 (TID 2343, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 784 -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 963.0 (TID 2344, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 963.0 (TID 2345, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 963.0 (TID 2346, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 963.0 (TID 2343) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 963.0 (TID 2345) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 963.0 (TID 2346) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 963.0 (TID 2344) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1008_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 963.0 (TID 2343). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 963.0 (TID 2346). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 963.0 (TID 2345). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 963.0 (TID 2343) in 6 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 963.0 (TID 2344). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1007_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 963.0 (TID 2346) in 5 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 963.0 (TID 2345) in 6 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 963.0 (TID 2344) in 6 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 963 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 783 -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 963.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 776 finished: reduce at LDAOptimizer.scala:456, took 0.011869 s -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1006_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 782 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1005_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 781 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 777 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 964(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 964 (PartitionwiseSampledRDD[1068] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=286594, maxMem=766075207 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1004_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1027 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=288570, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1027_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1027_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1027 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1003_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 964 (PartitionwiseSampledRDD[1068] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 964.0 with 1 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 964.0 (TID 2347, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 780 -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 964.0 (TID 2347) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 964.0 (TID 2347). 1226 bytes result sent to driver -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1002_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 964.0 (TID 2347) in 3 ms on localhost (1/1) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 964.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 964 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 779 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 777 finished: isEmpty at LDAOptimizer.scala:418, took 0.008457 s -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=278618, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1028 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1001_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 778 -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=275202, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1028_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1028_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_1000_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1028 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_999_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 777 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_998_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 776 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 778 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 965(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 965 (MapPartitionsRDD[1070] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=260169, maxMem=766075207 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_997_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1029 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 775 -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2088) called with curMem=263305, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1029_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_996_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1029_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1029 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 965 (MapPartitionsRDD[1070] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 965.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 965.0 (TID 2348, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 965.0 (TID 2349, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 965.0 (TID 2350, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_995_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 965.0 (TID 2351, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 965.0 (TID 2349) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 965.0 (TID 2351) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 965.0 (TID 2350) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 774 -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 965.0 (TID 2348) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_994_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 965.0 (TID 2348). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 965.0 (TID 2349). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 965.0 (TID 2350). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 773 -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 965.0 (TID 2351). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 965.0 (TID 2348) in 5 ms on localhost (1/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 965.0 (TID 2349) in 6 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 965.0 (TID 2350) in 5 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 965.0 (TID 2351) in 5 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 965 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 965.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 778 finished: reduce at LDAOptimizer.scala:453, took 0.011344 s -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_993_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 772 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_992_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 779 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 966(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 966 (MapPartitionsRDD[1071] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_991_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=246611, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1030 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 771 -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2088) called with curMem=248165, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1030_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_990_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1030_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1030 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 966 (MapPartitionsRDD[1071] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 770 -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 966.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 966.0 (TID 2352, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 966.0 (TID 2353, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 966.0 (TID 2354, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 966.0 (TID 2355, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 966.0 (TID 2352) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 966.0 (TID 2353) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 966.0 (TID 2355) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_989_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 966.0 (TID 2354) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 769 -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 966.0 (TID 2352). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_988_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 966.0 (TID 2352) in 5 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 966.0 (TID 2355). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 966.0 (TID 2354). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 966.0 (TID 2353). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 966.0 (TID 2355) in 5 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 966.0 (TID 2354) in 5 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 966.0 (TID 2353) in 5 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 966 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 966.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 779 finished: reduce at LDAOptimizer.scala:456, took 0.012693 s -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_987_piece0 on localhost:50000 in memory (size: 1316.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 768 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_986_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 767 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 780 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 967(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 967 (PartitionwiseSampledRDD[1072] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_985_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=225630, maxMem=766075207 -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 766 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1031 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=227502, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1031_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_984_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1031_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1031 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 967 (PartitionwiseSampledRDD[1072] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 967.0 with 1 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 967.0 (TID 2356, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 967.0 (TID 2356) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_983_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 765 -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 967.0 (TID 2356). 1226 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 967.0 (TID 2356) in 2 ms on localhost (1/1) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 967 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 967.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 780 finished: isEmpty at LDAOptimizer.scala:418, took 0.007001 s -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_982_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 764 -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=219458, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1032 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=213957, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1032_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_981_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 763 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1032_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1032 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_980_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_979_piece0 on localhost:50000 in memory (size: 1318.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 762 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 781 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_978_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 968(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 968 (MapPartitionsRDD[1074] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 761 -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=204592, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1033 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=202505, maxMem=766075207 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_977_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1033_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 760 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1033_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1033 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 968 (MapPartitionsRDD[1074] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 968.0 with 4 tasks -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_976_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 968.0 (TID 2357, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 968.0 (TID 2358, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 968.0 (TID 2359, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 968.0 (TID 2360, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 968.0 (TID 2359) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 968.0 (TID 2360) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 968.0 (TID 2357) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 968.0 (TID 2358) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_975_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 968.0 (TID 2359). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 968.0 (TID 2358). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 968.0 (TID 2360). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 968.0 (TID 2359) in 5 ms on localhost (1/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 968.0 (TID 2358) in 5 ms on localhost (2/4) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 759 -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 968.0 (TID 2360) in 5 ms on localhost (3/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 968.0 (TID 2357). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 968.0 (TID 2357) in 7 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 968 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 781 finished: reduce at LDAOptimizer.scala:453, took 0.012664 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 968.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_974_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 758 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_973_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 782 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 969(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 757 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 969 (MapPartitionsRDD[1075] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=188817, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1034 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_972_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=190297, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1034_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1034_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1034 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_971_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 969 (MapPartitionsRDD[1075] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 969.0 with 4 tasks -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 756 -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 969.0 (TID 2361, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 969.0 (TID 2362, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 969.0 (TID 2363, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 969.0 (TID 2364, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 969.0 (TID 2364) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 969.0 (TID 2362) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 969.0 (TID 2363) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 969.0 (TID 2361) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_970_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 755 -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 969.0 (TID 2364). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 969.0 (TID 2362). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 969.0 (TID 2363). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_969_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 969.0 (TID 2361). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 969.0 (TID 2364) in 6 ms on localhost (1/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 969.0 (TID 2362) in 7 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 969.0 (TID 2363) in 7 ms on localhost (3/4) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 754 -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 969.0 (TID 2361) in 8 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 969 (reduce at LDAOptimizer.scala:456) finished in 0.009 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 969.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 782 finished: reduce at LDAOptimizer.scala:456, took 0.013718 s -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_968_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_967_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 753 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 783 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 970(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 970 (PartitionwiseSampledRDD[1076] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=173663, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1035 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_966_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=172183, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1035_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 752 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1035_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1035 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 970 (PartitionwiseSampledRDD[1076] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 970.0 with 1 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 970.0 (TID 2365, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 970.0 (TID 2365) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_965_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 970.0 (TID 2365). 1226 bytes result sent to driver -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 751 -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 970.0 (TID 2365) in 3 ms on localhost (1/1) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 970.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 970 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 783 finished: isEmpty at LDAOptimizer.scala:418, took 0.010331 s -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_964_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=167365, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1036 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=166224, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1036_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_963_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1036_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 750 -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1036 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_962_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 784 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 971(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 971 (MapPartitionsRDD[1078] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=158635, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1037 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=162227, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1037_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1037_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1037 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 971 (MapPartitionsRDD[1078] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 971.0 with 4 tasks -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 749 -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 971.0 (TID 2366, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 971.0 (TID 2367, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 971.0 (TID 2368, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 971.0 (TID 2369, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 971.0 (TID 2367) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 971.0 (TID 2369) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 971.0 (TID 2367). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 971.0 (TID 2369). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 971.0 (TID 2368) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 971.0 (TID 2368). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 971.0 (TID 2367) in 8 ms on localhost (1/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 971.0 (TID 2369) in 7 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 971.0 (TID 2368) in 8 ms on localhost (3/4) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 971.0 (TID 2366) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 971.0 (TID 2366). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_961_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 971.0 (TID 2366) in 17 ms on localhost (4/4) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 971.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 971 (reduce at LDAOptimizer.scala:453) finished in 0.018 s -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 784 finished: reduce at LDAOptimizer.scala:453, took 0.024996 s -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 748 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_960_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 785 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 972(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 972 (MapPartitionsRDD[1079] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_959_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=156862, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1038 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 747 -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=156259, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1038_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1038_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_958_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1038 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 972 (MapPartitionsRDD[1079] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 972.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 972.0 (TID 2370, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 746 -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 972.0 (TID 2371, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 972.0 (TID 2372, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 972.0 (TID 2373, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 972.0 (TID 2370) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 972.0 (TID 2373) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 972.0 (TID 2371) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 972.0 (TID 2372) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_957_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 745 -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 972.0 (TID 2371). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 972.0 (TID 2373). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 972.0 (TID 2371) in 6 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 972.0 (TID 2372). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 972.0 (TID 2373) in 7 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 972.0 (TID 2372) in 7 ms on localhost (3/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 972.0 (TID 2370). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_956_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 972 (reduce at LDAOptimizer.scala:456) finished in 0.011 s -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 972.0 (TID 2370) in 11 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 785 finished: reduce at LDAOptimizer.scala:456, took 0.017252 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 972.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_955_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 744 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 786 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 973(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_954_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 973 (PartitionwiseSampledRDD[1080] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=139397, maxMem=766075207 -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 743 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1039 stored as values in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=135872, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1039_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_953_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 742 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1039_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1039 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 973 (PartitionwiseSampledRDD[1080] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 973.0 with 1 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 973.0 (TID 2374, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 973.0 (TID 2374) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_952_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 973.0 (TID 2374). 1226 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 973.0 (TID 2374) in 3 ms on localhost (1/1) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 973 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 973.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 786 finished: isEmpty at LDAOptimizer.scala:418, took 0.010093 s -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_951_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=133262, maxMem=766075207 -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 741 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1040 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=131355, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1040_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_950_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1040_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 740 -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1040 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_949_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 739 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 787 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 974(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_948_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 974 (MapPartitionsRDD[1082] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=121872, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1041 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=121997, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1041_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_947_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1041_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1041 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 974 (MapPartitionsRDD[1082] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 738 -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 974.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 974.0 (TID 2375, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 974.0 (TID 2376, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 974.0 (TID 2377, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 974.0 (TID 2378, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 974.0 (TID 2376) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 974.0 (TID 2377) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 974.0 (TID 2378) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 974.0 (TID 2376). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_946_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 974.0 (TID 2378). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 974.0 (TID 2377). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 974.0 (TID 2375) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 974.0 (TID 2375). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 974.0 (TID 2376) in 10 ms on localhost (1/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 974.0 (TID 2378) in 9 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 974.0 (TID 2377) in 10 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 974.0 (TID 2375) in 11 ms on localhost (4/4) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 974.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 974 (reduce at LDAOptimizer.scala:453) finished in 0.012 s -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 787 finished: reduce at LDAOptimizer.scala:453, took 0.018394 s -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 737 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_945_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 736 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 788 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 975(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 975 (MapPartitionsRDD[1083] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=112237, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1042 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_944_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=115869, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1042_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1042_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1042 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_943_piece0 on localhost:50000 in memory (size: 1314.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 975 (MapPartitionsRDD[1083] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 975.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 975.0 (TID 2379, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 735 -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 975.0 (TID 2380, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 975.0 (TID 2381, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 975.0 (TID 2382, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 975.0 (TID 2382) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 975.0 (TID 2380) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 975.0 (TID 2382). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_942_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 975.0 (TID 2382) in 5 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 975.0 (TID 2380). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 975.0 (TID 2381) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 975.0 (TID 2381). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 975.0 (TID 2379) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 734 -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 975.0 (TID 2380) in 14 ms on localhost (2/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 975.0 (TID 2381) in 15 ms on localhost (3/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 975.0 (TID 2379). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 975 (reduce at LDAOptimizer.scala:456) finished in 0.017 s -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 975.0 (TID 2379) in 17 ms on localhost (4/4) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 975.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 788 finished: reduce at LDAOptimizer.scala:456, took 0.023919 s -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_941_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO ContextCleaner:59 - Cleaned accumulator 733 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Removed broadcast_940_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 789 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 976(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 976 (PartitionwiseSampledRDD[1084] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=102813, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1043 stored as values in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=104965, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1043_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1043_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1043 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 976 (PartitionwiseSampledRDD[1084] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 976.0 with 1 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 976.0 (TID 2383, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 976.0 (TID 2383) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 976.0 (TID 2383). 1226 bytes result sent to driver -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 976 (isEmpty at LDAOptimizer.scala:418) finished in 0.012 s -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 789 finished: isEmpty at LDAOptimizer.scala:418, took 0.017920 s -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=106104, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1044 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 976.0 (TID 2383) in 11 ms on localhost (1/1) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=106280, maxMem=766075207 -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 976.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1044_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1044_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1044 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 790 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 977(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 977 (MapPartitionsRDD[1086] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=106560, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1045 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=110152, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1045_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1045_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1045 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 977 (MapPartitionsRDD[1086] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 977.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 977.0 (TID 2384, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 977.0 (TID 2385, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 977.0 (TID 2386, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 977.0 (TID 2387, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 977.0 (TID 2385) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 977.0 (TID 2384) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 977.0 (TID 2386) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 977.0 (TID 2387) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 977.0 (TID 2386). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 977.0 (TID 2385). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 977.0 (TID 2386) in 5 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 977.0 (TID 2387). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 977.0 (TID 2385) in 6 ms on localhost (2/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 977.0 (TID 2384). 1265 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 977.0 (TID 2387) in 6 ms on localhost (3/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 977 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 977.0 (TID 2384) in 9 ms on localhost (4/4) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 977.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 790 finished: reduce at LDAOptimizer.scala:453, took 0.014446 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 791 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 978(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 978 (MapPartitionsRDD[1087] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=112239, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1046 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2088) called with curMem=115871, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1046_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1046_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1046 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 978 (MapPartitionsRDD[1087] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 978.0 with 4 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 978.0 (TID 2388, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 978.0 (TID 2389, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 2.0 in stage 978.0 (TID 2390, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 3.0 in stage 978.0 (TID 2391, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 978.0 (TID 2388) -2016-06-13 19:45:30 INFO Executor:59 - Running task 3.0 in stage 978.0 (TID 2391) -2016-06-13 19:45:30 INFO Executor:59 - Running task 2.0 in stage 978.0 (TID 2390) -2016-06-13 19:45:30 INFO Executor:59 - Running task 1.0 in stage 978.0 (TID 2389) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 1.0 in stage 978.0 (TID 2389). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO Executor:59 - Finished task 3.0 in stage 978.0 (TID 2391). 1358 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 978.0 (TID 2389) in 4 ms on localhost (1/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 978.0 (TID 2388). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 3.0 in stage 978.0 (TID 2391) in 4 ms on localhost (2/4) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 2.0 in stage 978.0 (TID 2390). 1314 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 978.0 (TID 2388) in 6 ms on localhost (3/4) -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 2.0 in stage 978.0 (TID 2390) in 5 ms on localhost (4/4) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 978 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 978.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 791 finished: reduce at LDAOptimizer.scala:456, took 0.012249 s -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 792 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 979(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 979 (PartitionwiseSampledRDD[1088] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=117959, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1047 stored as values in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=120111, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1047_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1047_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1047 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 979 (PartitionwiseSampledRDD[1088] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Adding task set 979.0 with 1 tasks -2016-06-13 19:45:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 979.0 (TID 2392, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:30 INFO Executor:59 - Running task 0.0 in stage 979.0 (TID 2392) -2016-06-13 19:45:30 INFO Executor:59 - Finished task 0.0 in stage 979.0 (TID 2392). 1226 bytes result sent to driver -2016-06-13 19:45:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 979.0 (TID 2392) in 2 ms on localhost (1/1) -2016-06-13 19:45:30 INFO DAGScheduler:59 - ResultStage 979 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 979.0, whose tasks have all completed, from pool -2016-06-13 19:45:30 INFO DAGScheduler:59 - Job 792 finished: isEmpty at LDAOptimizer.scala:418, took 0.006342 s -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=121428, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1048 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=121604, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1048_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:30 INFO BlockManagerInfo:59 - Added broadcast_1048_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:30 INFO SparkContext:59 - Created broadcast 1048 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:30 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:30 INFO DAGScheduler:59 - Got job 793 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:30 INFO DAGScheduler:59 - Final stage: ResultStage 980(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:30 INFO DAGScheduler:59 - Submitting ResultStage 980 (MapPartitionsRDD[1090] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:30 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=121884, maxMem=766075207 -2016-06-13 19:45:30 INFO MemoryStore:59 - Block broadcast_1049 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=125476, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1049_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1049_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1049 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 980 (MapPartitionsRDD[1090] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 980.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 980.0 (TID 2393, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 980.0 (TID 2394, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 980.0 (TID 2395, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 980.0 (TID 2396, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 980.0 (TID 2394) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 980.0 (TID 2396) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 980.0 (TID 2395) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 980.0 (TID 2393) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 980.0 (TID 2395). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 980.0 (TID 2396). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 980.0 (TID 2394). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 980.0 (TID 2393). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 980.0 (TID 2395) in 3 ms on localhost (1/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 980.0 (TID 2396) in 4 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 980.0 (TID 2394) in 5 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 980.0 (TID 2393) in 5 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 980 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 980.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 793 finished: reduce at LDAOptimizer.scala:453, took 0.010858 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 794 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 981(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 981 (MapPartitionsRDD[1091] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=127563, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1050 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2088) called with curMem=131195, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1050_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1050_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1050 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 981 (MapPartitionsRDD[1091] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 981.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 981.0 (TID 2397, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 981.0 (TID 2398, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 981.0 (TID 2399, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 981.0 (TID 2400, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 981.0 (TID 2399) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 981.0 (TID 2400) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 981.0 (TID 2397) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 981.0 (TID 2398) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 981.0 (TID 2397). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 981.0 (TID 2399). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 981.0 (TID 2397) in 4 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 981.0 (TID 2398). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 981.0 (TID 2399) in 4 ms on localhost (2/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 981.0 (TID 2400). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 981.0 (TID 2398) in 5 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 981.0 (TID 2400) in 4 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 981 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 981.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 794 finished: reduce at LDAOptimizer.scala:456, took 0.011137 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 795 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 982(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 982 (PartitionwiseSampledRDD[1092] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=133283, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1051 stored as values in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=135435, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1051_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.5 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1051_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1051 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 982 (PartitionwiseSampledRDD[1092] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 982.0 with 1 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 982.0 (TID 2401, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 982.0 (TID 2401) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 982.0 (TID 2401). 1226 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 982.0 (TID 2401) in 2 ms on localhost (1/1) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 982 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 982.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 795 finished: isEmpty at LDAOptimizer.scala:418, took 0.006774 s -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=136750, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1052 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=136926, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1052_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1052_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1052 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 796 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 983(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 983 (MapPartitionsRDD[1094] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=137206, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1053 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=140798, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1053_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1053_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1053 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 983 (MapPartitionsRDD[1094] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 983.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 983.0 (TID 2402, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 983.0 (TID 2403, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 983.0 (TID 2404, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 983.0 (TID 2405, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 983.0 (TID 2404) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 983.0 (TID 2405) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 983.0 (TID 2403) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 983.0 (TID 2402) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 983.0 (TID 2404). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 983.0 (TID 2405). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 983.0 (TID 2403). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 983.0 (TID 2404) in 3 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 983.0 (TID 2402). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 983.0 (TID 2405) in 3 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 983.0 (TID 2402) in 4 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 983.0 (TID 2403) in 5 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 983 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 983.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 796 finished: reduce at LDAOptimizer.scala:453, took 0.008762 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 797 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 984(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 984 (MapPartitionsRDD[1095] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=142884, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1054 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=146516, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1054_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1054_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1054 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 984 (MapPartitionsRDD[1095] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 984.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 984.0 (TID 2406, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 984.0 (TID 2407, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 984.0 (TID 2408, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 984.0 (TID 2409, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 984.0 (TID 2407) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 984.0 (TID 2409) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 984.0 (TID 2406) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 984.0 (TID 2408) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 984.0 (TID 2407). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 984.0 (TID 2406). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 984.0 (TID 2407) in 3 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 984.0 (TID 2408). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 984.0 (TID 2406) in 4 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 984.0 (TID 2408) in 4 ms on localhost (3/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 984.0 (TID 2409). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 984.0 (TID 2409) in 5 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 984 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 984.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 797 finished: reduce at LDAOptimizer.scala:456, took 0.009690 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 798 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 985(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 985 (PartitionwiseSampledRDD[1096] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=148602, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1055 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=150754, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1055_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1055_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1055 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 985 (PartitionwiseSampledRDD[1096] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 985.0 with 1 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 985.0 (TID 2410, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 985.0 (TID 2410) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 985.0 (TID 2410). 1226 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 985.0 (TID 2410) in 2 ms on localhost (1/1) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 985 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 985.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 798 finished: isEmpty at LDAOptimizer.scala:418, took 0.007781 s -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=152071, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1056 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=152247, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1056_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1056_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1056 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 799 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 986(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 986 (MapPartitionsRDD[1098] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=152527, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1057 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=156119, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1057_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1057_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1057 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 986 (MapPartitionsRDD[1098] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 986.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 986.0 (TID 2411, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 986.0 (TID 2412, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 986.0 (TID 2413, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 986.0 (TID 2414, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 986.0 (TID 2411) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 986.0 (TID 2412) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 986.0 (TID 2414) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 986.0 (TID 2413) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 986.0 (TID 2412). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 986.0 (TID 2413). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 986.0 (TID 2412) in 4 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 986.0 (TID 2414). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 986.0 (TID 2413) in 4 ms on localhost (2/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 986.0 (TID 2411). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 986.0 (TID 2414) in 4 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 986.0 (TID 2411) in 5 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 986 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 986.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 799 finished: reduce at LDAOptimizer.scala:453, took 0.009559 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 800 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 987(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 987 (MapPartitionsRDD[1099] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=158205, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1058 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=161837, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1058_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1058_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1058 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 987 (MapPartitionsRDD[1099] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 987.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 987.0 (TID 2415, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 987.0 (TID 2416, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 987.0 (TID 2417, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 987.0 (TID 2418, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 987.0 (TID 2415) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 987.0 (TID 2418) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 987.0 (TID 2416) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 987.0 (TID 2417) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 987.0 (TID 2417). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 987.0 (TID 2415). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 987.0 (TID 2416). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 987.0 (TID 2418). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 987.0 (TID 2417) in 4 ms on localhost (1/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 987.0 (TID 2415) in 4 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 987.0 (TID 2418) in 3 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 987.0 (TID 2416) in 5 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 987 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 987.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 800 finished: reduce at LDAOptimizer.scala:456, took 0.009365 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 801 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 988(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 988 (PartitionwiseSampledRDD[1100] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=163923, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1059 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=166075, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1059_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1059_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1059 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 988 (PartitionwiseSampledRDD[1100] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 988.0 with 1 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 988.0 (TID 2419, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 988.0 (TID 2419) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 988.0 (TID 2419). 1226 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 988.0 (TID 2419) in 2 ms on localhost (1/1) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 988 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 988.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 801 finished: isEmpty at LDAOptimizer.scala:418, took 0.007573 s -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=167392, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1060 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=167568, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1060_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1060_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1060 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 802 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 989(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 989 (MapPartitionsRDD[1102] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=167848, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1061 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2088) called with curMem=171440, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1061_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1061_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1061 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 989 (MapPartitionsRDD[1102] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 989.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 989.0 (TID 2420, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 989.0 (TID 2421, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 989.0 (TID 2422, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 989.0 (TID 2423, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 989.0 (TID 2422) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 989.0 (TID 2423) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 989.0 (TID 2421) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 989.0 (TID 2420) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 989.0 (TID 2420). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 989.0 (TID 2421). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 989.0 (TID 2422). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 989.0 (TID 2420) in 4 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 989.0 (TID 2423). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 989.0 (TID 2421) in 4 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 989.0 (TID 2422) in 5 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 989.0 (TID 2423) in 4 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 989 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 989.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 802 finished: reduce at LDAOptimizer.scala:453, took 0.009437 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 803 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 990(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 990 (MapPartitionsRDD[1103] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=173528, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1062 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2088) called with curMem=177160, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1062_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1062_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1062 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 990 (MapPartitionsRDD[1103] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 990.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 990.0 (TID 2424, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 990.0 (TID 2425, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 990.0 (TID 2426, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 990.0 (TID 2427, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 990.0 (TID 2425) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 990.0 (TID 2426) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 990.0 (TID 2427) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 990.0 (TID 2424) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 990.0 (TID 2427). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 990.0 (TID 2425). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 990.0 (TID 2426). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 990.0 (TID 2424). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 990.0 (TID 2427) in 3 ms on localhost (1/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 990.0 (TID 2425) in 4 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 990.0 (TID 2426) in 4 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 990.0 (TID 2424) in 5 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 990 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 990.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 803 finished: reduce at LDAOptimizer.scala:456, took 0.009938 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 804 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 991(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 991 (PartitionwiseSampledRDD[1104] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=179248, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1063 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=181400, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1063_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1063_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1063 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 991 (PartitionwiseSampledRDD[1104] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 991.0 with 1 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 991.0 (TID 2428, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 991.0 (TID 2428) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 991.0 (TID 2428). 1226 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 991.0 (TID 2428) in 2 ms on localhost (1/1) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 991 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 991.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 804 finished: isEmpty at LDAOptimizer.scala:418, took 0.007155 s -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=182715, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1064 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=182891, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1064_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1064_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1064 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 805 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 992(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 992 (MapPartitionsRDD[1106] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=183171, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1065 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=186763, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1065_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1065_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1065 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 992 (MapPartitionsRDD[1106] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 992.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 992.0 (TID 2429, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 992.0 (TID 2430, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 992.0 (TID 2431, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 992.0 (TID 2432, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 992.0 (TID 2429) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 992.0 (TID 2431) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 992.0 (TID 2432) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 992.0 (TID 2430) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 992.0 (TID 2430). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 992.0 (TID 2429). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 992.0 (TID 2431). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 992.0 (TID 2430) in 4 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 992.0 (TID 2432). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 992.0 (TID 2429) in 5 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 992.0 (TID 2431) in 5 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 992.0 (TID 2432) in 4 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 992 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 992.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 805 finished: reduce at LDAOptimizer.scala:453, took 0.011880 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 806 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 993(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 993 (MapPartitionsRDD[1107] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=188847, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1066 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=192479, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1066_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1066_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1066 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 993 (MapPartitionsRDD[1107] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 993.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 993.0 (TID 2433, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 993.0 (TID 2434, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 993.0 (TID 2435, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 993.0 (TID 2436, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 993.0 (TID 2433) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 993.0 (TID 2436) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 993.0 (TID 2435) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 993.0 (TID 2434) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 993.0 (TID 2435). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 993.0 (TID 2435) in 4 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 993.0 (TID 2434). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 993.0 (TID 2433). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 993.0 (TID 2436). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 993.0 (TID 2434) in 5 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 993.0 (TID 2433) in 6 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 993.0 (TID 2436) in 6 ms on localhost (4/4) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 993.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 993 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 806 finished: reduce at LDAOptimizer.scala:456, took 0.011444 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 807 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 994(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 994 (PartitionwiseSampledRDD[1108] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=194563, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1067 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=196715, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1067_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1067_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1067 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 994 (PartitionwiseSampledRDD[1108] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 994.0 with 1 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 994.0 (TID 2437, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 994.0 (TID 2437) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 994.0 (TID 2437). 1226 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 994.0 (TID 2437) in 2 ms on localhost (1/1) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 994 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 994.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 807 finished: isEmpty at LDAOptimizer.scala:418, took 0.006922 s -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=198032, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1068 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=198208, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1068_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1068_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1068 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 808 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 995(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 995 (MapPartitionsRDD[1110] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=198488, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1069 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2089) called with curMem=202080, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1069_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1069_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1069 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 995 (MapPartitionsRDD[1110] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 995.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 995.0 (TID 2438, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 995.0 (TID 2439, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 995.0 (TID 2440, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 995.0 (TID 2441, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 995.0 (TID 2439) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 995.0 (TID 2441) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 995.0 (TID 2440) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 995.0 (TID 2438) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 995.0 (TID 2439). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 995.0 (TID 2440). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 995.0 (TID 2439) in 4 ms on localhost (1/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 995.0 (TID 2440) in 3 ms on localhost (2/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 995.0 (TID 2438). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 995.0 (TID 2441). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 995.0 (TID 2438) in 5 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 995.0 (TID 2441) in 4 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 995 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 995.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 808 finished: reduce at LDAOptimizer.scala:453, took 0.010044 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 809 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 996(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 996 (MapPartitionsRDD[1111] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=204169, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1070 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=207801, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1070_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1070_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1070 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 996 (MapPartitionsRDD[1111] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 996.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 996.0 (TID 2442, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 996.0 (TID 2443, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 996.0 (TID 2444, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 996.0 (TID 2445, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 996.0 (TID 2442) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 996.0 (TID 2445) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 996.0 (TID 2443) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 996.0 (TID 2444) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 996.0 (TID 2443). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 996.0 (TID 2444). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 996.0 (TID 2445). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 996.0 (TID 2442). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 996.0 (TID 2443) in 3 ms on localhost (1/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 996.0 (TID 2442) in 4 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 996.0 (TID 2444) in 5 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 996.0 (TID 2445) in 5 ms on localhost (4/4) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 996.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 996 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 809 finished: reduce at LDAOptimizer.scala:456, took 0.010068 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 810 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 997(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 997 (PartitionwiseSampledRDD[1112] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=209887, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1071 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=212039, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1071_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1071_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1071 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 997 (PartitionwiseSampledRDD[1112] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 997.0 with 1 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 997.0 (TID 2446, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 997.0 (TID 2446) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 997.0 (TID 2446). 1226 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 997.0 (TID 2446) in 2 ms on localhost (1/1) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 997 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 997.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 810 finished: isEmpty at LDAOptimizer.scala:418, took 0.005857 s -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=213356, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1072 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=213532, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1072_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1072_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1072 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 811 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 998(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 998 (MapPartitionsRDD[1114] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=213812, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1073 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=217404, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1073_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1073_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1073 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 998 (MapPartitionsRDD[1114] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 998.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 998.0 (TID 2447, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 998.0 (TID 2448, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 998.0 (TID 2449, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 998.0 (TID 2450, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 998.0 (TID 2447) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 998.0 (TID 2450) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 998.0 (TID 2449) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 998.0 (TID 2448) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 998.0 (TID 2447). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 998.0 (TID 2450). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 998.0 (TID 2449). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 998.0 (TID 2447) in 4 ms on localhost (1/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 998.0 (TID 2450) in 3 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 998.0 (TID 2449) in 3 ms on localhost (3/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 998.0 (TID 2448). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 998.0 (TID 2448) in 5 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 998 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 998.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 811 finished: reduce at LDAOptimizer.scala:453, took 0.009649 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 812 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 999(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 999 (MapPartitionsRDD[1115] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=219490, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1074 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=223122, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1074_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1074_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1074 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 999 (MapPartitionsRDD[1115] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 999.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 999.0 (TID 2451, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 999.0 (TID 2452, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 999.0 (TID 2453, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 999.0 (TID 2454, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 999.0 (TID 2452) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 999.0 (TID 2451) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 999.0 (TID 2452). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 999.0 (TID 2451). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 999.0 (TID 2454) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 999.0 (TID 2454). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 999.0 (TID 2452) in 9 ms on localhost (1/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 999.0 (TID 2454) in 8 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 999.0 (TID 2451) in 11 ms on localhost (3/4) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 999.0 (TID 2453) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 999.0 (TID 2453). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 999.0 (TID 2453) in 15 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 999 (reduce at LDAOptimizer.scala:456) finished in 0.016 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 999.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 812 finished: reduce at LDAOptimizer.scala:456, took 0.022728 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 813 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1000(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1000 (PartitionwiseSampledRDD[1116] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=225208, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1075 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=227360, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1075_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1075_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1075 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1000 (PartitionwiseSampledRDD[1116] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1000.0 with 1 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1000.0 (TID 2455, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1000.0 (TID 2455) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1000.0 (TID 2455). 1226 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1000.0 (TID 2455) in 17 ms on localhost (1/1) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1000.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1000 (isEmpty at LDAOptimizer.scala:418) finished in 0.017 s -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 813 finished: isEmpty at LDAOptimizer.scala:418, took 0.024333 s -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=228677, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1076 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=228853, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1076_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1076_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1076 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 814 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1001(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1001 (MapPartitionsRDD[1118] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=229133, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1077 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=232725, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1077_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1077_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1077 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1001 (MapPartitionsRDD[1118] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1001.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1001.0 (TID 2456, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1001.0 (TID 2457, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1001.0 (TID 2458, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1001.0 (TID 2459, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1001.0 (TID 2459) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1001.0 (TID 2457) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1001.0 (TID 2458) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1001.0 (TID 2459). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1001.0 (TID 2459) in 3 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1001.0 (TID 2456) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1001.0 (TID 2458). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1001.0 (TID 2457). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1001.0 (TID 2458) in 5 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1001.0 (TID 2457) in 6 ms on localhost (3/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1001.0 (TID 2456). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1001.0 (TID 2456) in 8 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1001 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1001.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 814 finished: reduce at LDAOptimizer.scala:453, took 0.014127 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 815 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1002(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1002 (MapPartitionsRDD[1119] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=234812, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1078 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2088) called with curMem=238444, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1078_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1078_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1078 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1002 (MapPartitionsRDD[1119] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1002.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1002.0 (TID 2460, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1002.0 (TID 2461, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1002.0 (TID 2462, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1002.0 (TID 2463, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1002.0 (TID 2460) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1002.0 (TID 2463) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1002.0 (TID 2461) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1002.0 (TID 2462) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1002.0 (TID 2460). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1002.0 (TID 2463). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1002.0 (TID 2460) in 4 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1002.0 (TID 2462). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1002.0 (TID 2461). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1002.0 (TID 2463) in 3 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1002.0 (TID 2462) in 3 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1002.0 (TID 2461) in 4 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1002 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1002.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 815 finished: reduce at LDAOptimizer.scala:456, took 0.008759 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 816 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1003(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1003 (PartitionwiseSampledRDD[1120] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=240532, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1079 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=242684, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1079_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1079_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1079 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1003 (PartitionwiseSampledRDD[1120] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1003.0 with 1 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1003.0 (TID 2464, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1003.0 (TID 2464) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1003.0 (TID 2464). 1226 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1003.0 (TID 2464) in 2 ms on localhost (1/1) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1003 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1003.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 816 finished: isEmpty at LDAOptimizer.scala:418, took 0.006256 s -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=243999, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1080 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=244175, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1080_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1080_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1080 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 817 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1004(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1004 (MapPartitionsRDD[1122] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=244455, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1081 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2083) called with curMem=248047, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1081_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1081_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1081 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1004 (MapPartitionsRDD[1122] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1004.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1004.0 (TID 2465, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1004.0 (TID 2466, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1004.0 (TID 2467, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1004.0 (TID 2468, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1004.0 (TID 2466) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1004.0 (TID 2468) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1004.0 (TID 2465) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1004.0 (TID 2467) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1004.0 (TID 2468). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1004.0 (TID 2467). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1004.0 (TID 2465). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1004.0 (TID 2468) in 2 ms on localhost (1/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1004.0 (TID 2467) in 4 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1004.0 (TID 2465) in 4 ms on localhost (3/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1004.0 (TID 2466). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1004.0 (TID 2466) in 5 ms on localhost (4/4) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1004.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1004 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 817 finished: reduce at LDAOptimizer.scala:453, took 0.009265 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 818 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1005(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1005 (MapPartitionsRDD[1123] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=250130, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1082 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2083) called with curMem=253762, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1082_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1082_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1082 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1005 (MapPartitionsRDD[1123] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1005.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1005.0 (TID 2469, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1005.0 (TID 2470, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1005.0 (TID 2471, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1005.0 (TID 2472, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1005.0 (TID 2470) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1005.0 (TID 2472) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1005.0 (TID 2469) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1005.0 (TID 2471) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1005.0 (TID 2471). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1005.0 (TID 2472). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1005.0 (TID 2470). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1005.0 (TID 2471) in 3 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1005.0 (TID 2469). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1005.0 (TID 2472) in 4 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1005.0 (TID 2470) in 4 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1005.0 (TID 2469) in 4 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1005 (reduce at LDAOptimizer.scala:456) finished in 0.004 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1005.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 818 finished: reduce at LDAOptimizer.scala:456, took 0.008929 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 819 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1006(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1006 (PartitionwiseSampledRDD[1124] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=255845, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1083 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=257997, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1083_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1083_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1083 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1006 (PartitionwiseSampledRDD[1124] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1006.0 with 1 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1006.0 (TID 2473, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1006.0 (TID 2473) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1006.0 (TID 2473). 1226 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1006.0 (TID 2473) in 2 ms on localhost (1/1) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1006 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1006.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 819 finished: isEmpty at LDAOptimizer.scala:418, took 0.005855 s -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=259312, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1084 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=259488, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1084_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1084_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1084 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 820 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1007(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1007 (MapPartitionsRDD[1126] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=259768, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1085 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=263360, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1085_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1085_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1085 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1007 (MapPartitionsRDD[1126] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1007.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1007.0 (TID 2474, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1007.0 (TID 2475, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1007.0 (TID 2476, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1007.0 (TID 2477, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1007.0 (TID 2474) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1007.0 (TID 2476) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1007.0 (TID 2477) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1007.0 (TID 2475) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1007.0 (TID 2475). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1007.0 (TID 2474). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1007.0 (TID 2476). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1007.0 (TID 2475) in 4 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1007.0 (TID 2477). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1007.0 (TID 2474) in 4 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1007.0 (TID 2476) in 5 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1007.0 (TID 2477) in 4 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1007 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1007.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 820 finished: reduce at LDAOptimizer.scala:453, took 0.010491 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 821 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1008(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1008 (MapPartitionsRDD[1127] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=265447, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1086 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=269079, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1086_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1086_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1086 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1008 (MapPartitionsRDD[1127] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1008.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1008.0 (TID 2478, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1008.0 (TID 2479, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1008.0 (TID 2480, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1008.0 (TID 2481, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1008.0 (TID 2478) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1008.0 (TID 2480) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1008.0 (TID 2481) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1008.0 (TID 2479) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1008.0 (TID 2478). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1008.0 (TID 2479). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1008.0 (TID 2478) in 4 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1008.0 (TID 2480). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1008.0 (TID 2479) in 5 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1008.0 (TID 2480) in 5 ms on localhost (3/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1008.0 (TID 2481). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1008.0 (TID 2481) in 6 ms on localhost (4/4) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1008.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1008 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 821 finished: reduce at LDAOptimizer.scala:456, took 0.012137 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 822 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1009(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1009 (PartitionwiseSampledRDD[1128] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=271166, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1087 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=273318, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1087_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1087_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1087 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1009 (PartitionwiseSampledRDD[1128] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1009.0 with 1 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1009.0 (TID 2482, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1009.0 (TID 2482) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1009.0 (TID 2482). 1226 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1009.0 (TID 2482) in 2 ms on localhost (1/1) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1009 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1009.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 822 finished: isEmpty at LDAOptimizer.scala:418, took 0.006725 s -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=274635, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1088 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=274811, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1088_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1088_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1088 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 823 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1010(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1010 (MapPartitionsRDD[1130] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=275091, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1089 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=278683, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1089_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1089_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1089 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1010 (MapPartitionsRDD[1130] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1010.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1010.0 (TID 2483, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1010.0 (TID 2484, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1010.0 (TID 2485, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1010.0 (TID 2486, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1010.0 (TID 2484) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1010.0 (TID 2486) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1010.0 (TID 2485) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1010.0 (TID 2483) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1010.0 (TID 2485). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1010.0 (TID 2483). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1010.0 (TID 2484). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1010.0 (TID 2485) in 3 ms on localhost (1/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1010.0 (TID 2483) in 4 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1010.0 (TID 2484) in 4 ms on localhost (3/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1010.0 (TID 2486). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1010.0 (TID 2486) in 4 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1010 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1010.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 823 finished: reduce at LDAOptimizer.scala:453, took 0.009560 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 824 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1011(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1011 (MapPartitionsRDD[1131] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=280769, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1090 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=284401, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1090_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1090_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1090 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1011 (MapPartitionsRDD[1131] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1011.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1011.0 (TID 2487, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1011.0 (TID 2488, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1011.0 (TID 2489, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1011.0 (TID 2490, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1011.0 (TID 2488) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1011.0 (TID 2489) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1011.0 (TID 2490) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1011.0 (TID 2487) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1011.0 (TID 2488). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1011.0 (TID 2490). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1011.0 (TID 2487). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1011.0 (TID 2488) in 4 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1011.0 (TID 2489). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1011.0 (TID 2490) in 4 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1011.0 (TID 2487) in 5 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1011.0 (TID 2489) in 4 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1011 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1011.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 824 finished: reduce at LDAOptimizer.scala:456, took 0.012626 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 825 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1012(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1012 (PartitionwiseSampledRDD[1132] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=286487, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1091 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=288639, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1091_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1091_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1091 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1012 (PartitionwiseSampledRDD[1132] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1012.0 with 1 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1012.0 (TID 2491, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1012.0 (TID 2491) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1012.0 (TID 2491). 1226 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1012.0 (TID 2491) in 2 ms on localhost (1/1) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1012 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1012.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 825 finished: isEmpty at LDAOptimizer.scala:418, took 0.006101 s -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=289954, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1092 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=290130, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1092_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1092_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1092 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 826 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1013(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1013 (MapPartitionsRDD[1134] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=290410, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1093 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=294002, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1093_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1093_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1093 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1013 (MapPartitionsRDD[1134] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1013.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1013.0 (TID 2492, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1013.0 (TID 2493, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1013.0 (TID 2494, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1013.0 (TID 2495, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1013.0 (TID 2494) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1013.0 (TID 2495) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1013.0 (TID 2493) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1013.0 (TID 2492) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1013.0 (TID 2494). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1013.0 (TID 2494) in 4 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1013.0 (TID 2495). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1013.0 (TID 2492). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1013.0 (TID 2493). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1013.0 (TID 2493) in 8 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1013.0 (TID 2495) in 7 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1013.0 (TID 2492) in 8 ms on localhost (4/4) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1013.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1013 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 826 finished: reduce at LDAOptimizer.scala:453, took 0.012825 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 827 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1014(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1014 (MapPartitionsRDD[1135] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=296086, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1094 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=299718, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1094_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1094_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1094 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1014 (MapPartitionsRDD[1135] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1014.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1014.0 (TID 2496, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1014.0 (TID 2497, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1014.0 (TID 2498, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1014.0 (TID 2499, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1014.0 (TID 2496) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1014.0 (TID 2498) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1014.0 (TID 2499) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1014.0 (TID 2497) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1014.0 (TID 2496). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1014.0 (TID 2499). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1014.0 (TID 2497). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1014.0 (TID 2496) in 4 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1014.0 (TID 2498). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1014.0 (TID 2499) in 3 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1014.0 (TID 2497) in 5 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1014.0 (TID 2498) in 4 ms on localhost (4/4) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1014.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1014 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 827 finished: reduce at LDAOptimizer.scala:456, took 0.010302 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 828 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1015(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1015 (PartitionwiseSampledRDD[1136] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=301802, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1095 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=303954, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1095_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1095_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1095 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1015 (PartitionwiseSampledRDD[1136] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1015.0 with 1 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1015.0 (TID 2500, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1015.0 (TID 2500) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1015.0 (TID 2500). 1226 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1015.0 (TID 2500) in 2 ms on localhost (1/1) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1015 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1015.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 828 finished: isEmpty at LDAOptimizer.scala:418, took 0.006417 s -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=305269, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1096 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=305445, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1096_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1096_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1096 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 829 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1016(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1016 (MapPartitionsRDD[1138] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=305725, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1097 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=309317, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1097_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1097_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1097 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1016 (MapPartitionsRDD[1138] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1016.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1016.0 (TID 2501, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1016.0 (TID 2502, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1016.0 (TID 2503, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1016.0 (TID 2504, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1016.0 (TID 2503) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1016.0 (TID 2504) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1016.0 (TID 2501) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1016.0 (TID 2502) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1016.0 (TID 2503). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1016.0 (TID 2502). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1016.0 (TID 2503) in 3 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1016.0 (TID 2504). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1016.0 (TID 2501). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1016.0 (TID 2502) in 3 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1016.0 (TID 2504) in 4 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1016.0 (TID 2501) in 5 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1016 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1016.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 829 finished: reduce at LDAOptimizer.scala:453, took 0.008852 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 830 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1017(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1017 (MapPartitionsRDD[1139] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=311403, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1098 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2088) called with curMem=315035, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1098_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1098_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1098 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1017 (MapPartitionsRDD[1139] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1017.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1017.0 (TID 2505, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1017.0 (TID 2506, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1017.0 (TID 2507, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1017.0 (TID 2508, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1017.0 (TID 2507) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1017.0 (TID 2508) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1017.0 (TID 2506) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1017.0 (TID 2505) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1017.0 (TID 2506). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1017.0 (TID 2507). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1017.0 (TID 2505). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1017.0 (TID 2506) in 4 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1017.0 (TID 2508). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1017.0 (TID 2507) in 3 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1017.0 (TID 2505) in 4 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1017.0 (TID 2508) in 3 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1017 (reduce at LDAOptimizer.scala:456) finished in 0.004 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1017.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 830 finished: reduce at LDAOptimizer.scala:456, took 0.009120 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 831 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1018(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1018 (PartitionwiseSampledRDD[1140] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=317123, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1099 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=319275, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1099_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1099_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1099 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1018 (PartitionwiseSampledRDD[1140] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1018.0 with 1 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1018.0 (TID 2509, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1018.0 (TID 2509) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1018.0 (TID 2509). 1226 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1018.0 (TID 2509) in 2 ms on localhost (1/1) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1018 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1018.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 831 finished: isEmpty at LDAOptimizer.scala:418, took 0.006100 s -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=320592, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1100 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=320768, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1100_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1100_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1100 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 832 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1019(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1019 (MapPartitionsRDD[1142] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=321048, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1101 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=324640, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1101_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1101_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1101 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1019 (MapPartitionsRDD[1142] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1019.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1019.0 (TID 2510, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1019.0 (TID 2511, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1019.0 (TID 2512, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1019.0 (TID 2513, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1019.0 (TID 2511) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1019.0 (TID 2513) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1019.0 (TID 2512) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1019.0 (TID 2510) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1019.0 (TID 2511). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1019.0 (TID 2513). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1019.0 (TID 2510). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1019.0 (TID 2511) in 5 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1019.0 (TID 2512). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1019.0 (TID 2510) in 6 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1019.0 (TID 2513) in 5 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1019.0 (TID 2512) in 5 ms on localhost (4/4) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1019.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1019 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 832 finished: reduce at LDAOptimizer.scala:453, took 0.011323 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 833 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1020(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1020 (MapPartitionsRDD[1143] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=326726, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1102 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=330358, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1102_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1102_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1102 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1020 (MapPartitionsRDD[1143] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1020.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1020.0 (TID 2514, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1020.0 (TID 2515, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1020.0 (TID 2516, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1020.0 (TID 2517, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1020.0 (TID 2514) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1020.0 (TID 2517) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1020.0 (TID 2516) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1020.0 (TID 2515) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1020.0 (TID 2517). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1020.0 (TID 2515). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1020.0 (TID 2514). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1020.0 (TID 2516). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1020.0 (TID 2517) in 3 ms on localhost (1/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1020.0 (TID 2515) in 3 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1020.0 (TID 2514) in 4 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1020.0 (TID 2516) in 4 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1020 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1020.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 833 finished: reduce at LDAOptimizer.scala:456, took 0.009219 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 834 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1021(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1021 (PartitionwiseSampledRDD[1144] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=332444, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1103 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=334596, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1103_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1103_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1103 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1021 (PartitionwiseSampledRDD[1144] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1021.0 with 1 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1021.0 (TID 2518, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1021.0 (TID 2518) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1021.0 (TID 2518). 1226 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1021.0 (TID 2518) in 2 ms on localhost (1/1) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1021 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1021.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 834 finished: isEmpty at LDAOptimizer.scala:418, took 0.007257 s -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=335913, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1104 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=336089, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1104_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1104_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1104 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 835 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1022(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1022 (MapPartitionsRDD[1146] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=336369, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1105 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2089) called with curMem=339961, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1105_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1105_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.4 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1105 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1022 (MapPartitionsRDD[1146] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1022.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1022.0 (TID 2519, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1022.0 (TID 2520, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1022.0 (TID 2521, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1022.0 (TID 2522, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1022.0 (TID 2520) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1022.0 (TID 2522) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1022.0 (TID 2521) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1022.0 (TID 2519) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1022.0 (TID 2520). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1022.0 (TID 2522). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1022.0 (TID 2519). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1022.0 (TID 2521). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1022.0 (TID 2520) in 3 ms on localhost (1/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1022.0 (TID 2519) in 5 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1022.0 (TID 2522) in 4 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1022.0 (TID 2521) in 4 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1022 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1022.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 835 finished: reduce at LDAOptimizer.scala:453, took 0.009379 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 836 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1023(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1023 (MapPartitionsRDD[1147] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=342050, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1106 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2089) called with curMem=345682, maxMem=766075207 -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 805 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1106_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1106_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.4 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1106 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1105_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.4 MB) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1023 (MapPartitionsRDD[1147] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1023.0 with 4 tasks -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 856 -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1023.0 (TID 2523, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1023.0 (TID 2524, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1023.0 (TID 2525, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1023.0 (TID 2526, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1023.0 (TID 2524) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1023.0 (TID 2525) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1023.0 (TID 2523) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1103_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.4 MB) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1023.0 (TID 2526) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 855 -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1023.0 (TID 2523). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1023.0 (TID 2525). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1023.0 (TID 2523) in 5 ms on localhost (1/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1023.0 (TID 2525) in 4 ms on localhost (2/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1023.0 (TID 2524). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1023.0 (TID 2526). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1023.0 (TID 2524) in 6 ms on localhost (3/4) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1102_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1023 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1023.0 (TID 2526) in 6 ms on localhost (4/4) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1023.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 836 finished: reduce at LDAOptimizer.scala:456, took 0.022516 s -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 854 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1101_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 853 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 837 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1024(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1024 (PartitionwiseSampledRDD[1148] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=326769, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1107 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=328921, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1107_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1100_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1107_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1107 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1024 (PartitionwiseSampledRDD[1148] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1024.0 with 1 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1024.0 (TID 2527, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1024.0 (TID 2527) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1099_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1024.0 (TID 2527). 1226 bytes result sent to driver -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 852 -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1024 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1024.0 (TID 2527) in 2 ms on localhost (1/1) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1024.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 837 finished: isEmpty at LDAOptimizer.scala:418, took 0.009307 s -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=321049, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1108 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1098_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 851 -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=321225, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1108_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1108_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1097_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1108 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 850 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1096_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 838 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1025(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1025 (MapPartitionsRDD[1150] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1095_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=311904, maxMem=766075207 -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 849 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1109 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=313412, maxMem=766075207 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1094_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1109_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 848 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1109_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1109 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1025 (MapPartitionsRDD[1150] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1025.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1025.0 (TID 2528, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1025.0 (TID 2529, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1093_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1025.0 (TID 2530, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 847 -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1025.0 (TID 2531, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1025.0 (TID 2528) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1025.0 (TID 2531) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1025.0 (TID 2530) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1025.0 (TID 2529) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1092_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1025.0 (TID 2528). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1025.0 (TID 2528) in 4 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1025.0 (TID 2530). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1025.0 (TID 2529). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1025.0 (TID 2530) in 5 ms on localhost (2/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1025.0 (TID 2531). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1025.0 (TID 2529) in 6 ms on localhost (3/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1025 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1091_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1025.0 (TID 2531) in 5 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 838 finished: reduce at LDAOptimizer.scala:453, took 0.013732 s -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 846 -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1025.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1090_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 845 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 839 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1026(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1026 (MapPartitionsRDD[1151] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1089_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=290871, maxMem=766075207 -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 844 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1110 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1088_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=294223, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1110_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1110_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1110 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1026 (MapPartitionsRDD[1151] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1026.0 with 4 tasks -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1087_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1026.0 (TID 2532, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1026.0 (TID 2533, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 843 -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1026.0 (TID 2534, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1026.0 (TID 2535, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1026.0 (TID 2535) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1026.0 (TID 2533) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1026.0 (TID 2532) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1086_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1026.0 (TID 2534) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1026.0 (TID 2535). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1026.0 (TID 2532). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1026.0 (TID 2535) in 4 ms on localhost (1/4) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 842 -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1026.0 (TID 2532) in 5 ms on localhost (2/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1026.0 (TID 2534). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1026.0 (TID 2533). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1026.0 (TID 2534) in 6 ms on localhost (3/4) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1085_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1026.0 (TID 2533) in 8 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1026 (reduce at LDAOptimizer.scala:456) finished in 0.009 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1026.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 839 finished: reduce at LDAOptimizer.scala:456, took 0.015476 s -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 841 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1084_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1083_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 840 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1027(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 840 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1027 (PartitionwiseSampledRDD[1152] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=275260, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1111 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1082_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 839 -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=273780, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1111_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1111_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1111 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1027 (PartitionwiseSampledRDD[1152] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1027.0 with 1 tasks -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1081_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1027.0 (TID 2536, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 838 -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1027.0 (TID 2536) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1080_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1027.0 (TID 2536). 1226 bytes result sent to driver -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1027 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1027.0 (TID 2536) in 3 ms on localhost (1/1) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1027.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 840 finished: isEmpty at LDAOptimizer.scala:418, took 0.008160 s -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=265497, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1112 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1079_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 837 -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=263585, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1112_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1078_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1112_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1112 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 836 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1077_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 835 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1076_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 841 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1028(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1028 (MapPartitionsRDD[1154] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=250629, maxMem=766075207 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1075_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1113 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 834 -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=248503, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1113_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1074_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 833 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1113_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1113 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1028 (MapPartitionsRDD[1154] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1028.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1028.0 (TID 2537, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1028.0 (TID 2538, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1073_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1028.0 (TID 2539, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1028.0 (TID 2540, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1028.0 (TID 2537) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1028.0 (TID 2539) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1028.0 (TID 2538) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1028.0 (TID 2540) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 832 -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1028.0 (TID 2538). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1028.0 (TID 2539). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1028.0 (TID 2538) in 4 ms on localhost (1/4) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1072_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1028.0 (TID 2540). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1028.0 (TID 2537). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1028.0 (TID 2539) in 5 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1028.0 (TID 2540) in 6 ms on localhost (3/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1028 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1028.0 (TID 2537) in 7 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 841 finished: reduce at LDAOptimizer.scala:453, took 0.012219 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1028.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1071_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 831 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 842 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1029(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1070_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1029 (MapPartitionsRDD[1155] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 830 -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=233177, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1114 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1069_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=233217, maxMem=766075207 -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 829 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1114_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1114_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1114 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1068_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1029 (MapPartitionsRDD[1155] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1029.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1029.0 (TID 2541, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1029.0 (TID 2542, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1029.0 (TID 2543, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1029.0 (TID 2544, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1029.0 (TID 2544) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1029.0 (TID 2541) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1067_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1029.0 (TID 2543) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1029.0 (TID 2542) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 828 -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1029.0 (TID 2544). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1029.0 (TID 2542). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1029.0 (TID 2541). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1066_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1029.0 (TID 2544) in 5 ms on localhost (1/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1029.0 (TID 2542) in 7 ms on localhost (2/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1029.0 (TID 2543). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1029.0 (TID 2541) in 9 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1029.0 (TID 2543) in 9 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1029 (reduce at LDAOptimizer.scala:456) finished in 0.010 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1029.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 842 finished: reduce at LDAOptimizer.scala:456, took 0.014742 s -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 827 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1065_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 826 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1064_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 843 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1030(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1030 (PartitionwiseSampledRDD[1156] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=218213, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1115 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1063_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 825 -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=218213, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1115_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1115_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1115 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1030 (PartitionwiseSampledRDD[1156] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1030.0 with 1 tasks -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1062_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 824 -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1030.0 (TID 2545, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1030.0 (TID 2545) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1030.0 (TID 2545). 1226 bytes result sent to driver -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1061_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 823 -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1030 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1030.0 (TID 2545) in 3 ms on localhost (1/1) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1030.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 843 finished: isEmpty at LDAOptimizer.scala:418, took 0.008363 s -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=207850, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1116 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=208026, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1116_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1060_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1116_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1116 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1059_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 822 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 844 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1031(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1031 (MapPartitionsRDD[1158] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1058_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=198943, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1117 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 821 -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=202535, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1117_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1117_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1117 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1031 (MapPartitionsRDD[1158] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1031.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1031.0 (TID 2546, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1031.0 (TID 2547, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1031.0 (TID 2548, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1031.0 (TID 2549, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1031.0 (TID 2547) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1057_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1031.0 (TID 2546) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1031.0 (TID 2546). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1031.0 (TID 2547). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1031.0 (TID 2546) in 5 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1031.0 (TID 2548) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1031.0 (TID 2548). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1031.0 (TID 2549) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1031.0 (TID 2549). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 820 -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1031.0 (TID 2548) in 14 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1031.0 (TID 2549) in 14 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1031.0 (TID 2547) in 15 ms on localhost (4/4) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1031.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1031 (reduce at LDAOptimizer.scala:453) finished in 0.016 s -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 844 finished: reduce at LDAOptimizer.scala:453, took 0.023628 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 845 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1032(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1032 (MapPartitionsRDD[1159] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1056_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=198485, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1118 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=202117, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1118_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1118_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1118 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1032 (MapPartitionsRDD[1159] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1032.0 with 4 tasks -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1055_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1032.0 (TID 2550, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1032.0 (TID 2551, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1032.0 (TID 2552, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1032.0 (TID 2553, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1032.0 (TID 2550) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1032.0 (TID 2552) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1032.0 (TID 2552). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1032.0 (TID 2550). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1032.0 (TID 2551) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 819 -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1032.0 (TID 2553) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1032.0 (TID 2552) in 6 ms on localhost (1/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1032.0 (TID 2550) in 7 ms on localhost (2/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1032.0 (TID 2551). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1032.0 (TID 2553). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1032.0 (TID 2551) in 9 ms on localhost (3/4) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1054_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1032.0 (TID 2553) in 9 ms on localhost (4/4) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1032.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1032 (reduce at LDAOptimizer.scala:456) finished in 0.021 s -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 845 finished: reduce at LDAOptimizer.scala:456, took 0.029907 s -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 818 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1053_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 817 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1052_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 846 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1033(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1033 (PartitionwiseSampledRDD[1160] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=185413, maxMem=766075207 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1051_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1119 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 816 -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=183933, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1119_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1119_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1050_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1119 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1033 (PartitionwiseSampledRDD[1160] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1033.0 with 1 tasks -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 815 -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1033.0 (TID 2554, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1033.0 (TID 2554) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1049_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1033.0 (TID 2554). 1226 bytes result sent to driver -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 814 -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1033.0 (TID 2554) in 2 ms on localhost (1/1) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1033 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1033.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 846 finished: isEmpty at LDAOptimizer.scala:418, took 0.008982 s -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=177027, maxMem=766075207 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1048_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1120 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=173734, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1120_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1047_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 813 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1120_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1120 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1046_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 812 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1045_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 811 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 847 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1034(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1034 (MapPartitionsRDD[1162] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1044_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=162159, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1121 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=162284, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1121_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1043_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 810 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1121_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1121 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1034 (MapPartitionsRDD[1162] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1034.0 with 4 tasks -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1042_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1034.0 (TID 2555, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1034.0 (TID 2556, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 809 -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1034.0 (TID 2557, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1034.0 (TID 2558, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1034.0 (TID 2556) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1034.0 (TID 2558) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1034.0 (TID 2557) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1034.0 (TID 2555) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1041_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 808 -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1034.0 (TID 2558). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1034.0 (TID 2556). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1034.0 (TID 2555). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1034.0 (TID 2557). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1034.0 (TID 2558) in 6 ms on localhost (1/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1034.0 (TID 2556) in 7 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1034.0 (TID 2555) in 9 ms on localhost (3/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1034.0 (TID 2557) in 10 ms on localhost (4/4) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1040_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1034 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 847 finished: reduce at LDAOptimizer.scala:453, took 0.015854 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1034.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1039_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 848 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1035(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1035 (MapPartitionsRDD[1163] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=149055, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1122 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 807 -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=152687, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1122_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1122_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1122 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1035 (MapPartitionsRDD[1163] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1035.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1035.0 (TID 2559, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1035.0 (TID 2560, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1035.0 (TID 2561, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1035.0 (TID 2562, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1035.0 (TID 2561) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1035.0 (TID 2562) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1035.0 (TID 2560) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1038_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1035.0 (TID 2562). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1035.0 (TID 2561). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1035.0 (TID 2559) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1035.0 (TID 2562) in 8 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1035.0 (TID 2560). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1035.0 (TID 2561) in 8 ms on localhost (2/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1035.0 (TID 2559). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 806 -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1035.0 (TID 2560) in 10 ms on localhost (3/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1035 (reduce at LDAOptimizer.scala:456) finished in 0.013 s -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1035.0 (TID 2559) in 13 ms on localhost (4/4) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1035.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 848 finished: reduce at LDAOptimizer.scala:456, took 0.022234 s -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1037_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1036_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1035_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 849 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1036(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1036 (PartitionwiseSampledRDD[1164] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 804 -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=135824, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1123 stored as values in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1034_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=135891, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1123_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 803 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1123_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1123 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1036 (PartitionwiseSampledRDD[1164] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1036.0 with 1 tasks -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1033_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1036.0 (TID 2563, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1036.0 (TID 2563) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 802 -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1036.0 (TID 2563). 1226 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1036.0 (TID 2563) in 3 ms on localhost (1/1) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1036 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1036.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 849 finished: isEmpty at LDAOptimizer.scala:418, took 0.010230 s -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1032_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=131251, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1124 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=131251, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1124_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1124_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1124 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1031_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 801 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1030_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 800 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 850 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1037(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1029_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1037 (MapPartitionsRDD[1166] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 799 -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=116382, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1125 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1028_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=119798, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1125_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1125_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1125 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1037 (MapPartitionsRDD[1166] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1037.0 with 4 tasks -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1027_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 798 -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1037.0 (TID 2564, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1037.0 (TID 2565, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1037.0 (TID 2566, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1037.0 (TID 2567, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1037.0 (TID 2566) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1037.0 (TID 2567) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1037.0 (TID 2565) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1037.0 (TID 2564) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1026_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1037.0 (TID 2565). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1037.0 (TID 2564). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 797 -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1037.0 (TID 2565) in 4 ms on localhost (1/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1037.0 (TID 2564) in 5 ms on localhost (2/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1037.0 (TID 2566). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1037.0 (TID 2566) in 5 ms on localhost (3/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1037.0 (TID 2567). 1265 bytes result sent to driver -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1025_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1037 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1037.0 (TID 2567) in 6 ms on localhost (4/4) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1037.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 850 finished: reduce at LDAOptimizer.scala:453, took 0.011806 s -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 796 -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1024_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 851 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1038(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1038 (MapPartitionsRDD[1167] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Removed broadcast_1023_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO ContextCleaner:59 - Cleaned accumulator 795 -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=103102, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1126 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=106734, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1126_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1126_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1126 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1038 (MapPartitionsRDD[1167] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1038.0 with 4 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1038.0 (TID 2568, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 1038.0 (TID 2569, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 2.0 in stage 1038.0 (TID 2570, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 3.0 in stage 1038.0 (TID 2571, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 2.0 in stage 1038.0 (TID 2570) -2016-06-13 19:45:31 INFO Executor:59 - Running task 1.0 in stage 1038.0 (TID 2569) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1038.0 (TID 2568) -2016-06-13 19:45:31 INFO Executor:59 - Running task 3.0 in stage 1038.0 (TID 2571) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 2.0 in stage 1038.0 (TID 2570). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1038.0 (TID 2568). 1314 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 2.0 in stage 1038.0 (TID 2570) in 4 ms on localhost (1/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 1.0 in stage 1038.0 (TID 2569). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1038.0 (TID 2568) in 6 ms on localhost (2/4) -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 1038.0 (TID 2569) in 5 ms on localhost (3/4) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 3.0 in stage 1038.0 (TID 2571). 1358 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 3.0 in stage 1038.0 (TID 2571) in 5 ms on localhost (4/4) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1038 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1038.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 851 finished: reduce at LDAOptimizer.scala:456, took 0.012089 s -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 852 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1039(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1039 (PartitionwiseSampledRDD[1168] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=108820, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1127 stored as values in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(1319) called with curMem=110972, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1127_piece0 stored as bytes in memory (estimated size 1319.0 B, free 730.5 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1127_piece0 in memory on localhost:50000 (size: 1319.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1127 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1039 (PartitionwiseSampledRDD[1168] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Adding task set 1039.0 with 1 tasks -2016-06-13 19:45:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 1039.0 (TID 2572, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:31 INFO Executor:59 - Running task 0.0 in stage 1039.0 (TID 2572) -2016-06-13 19:45:31 INFO Executor:59 - Finished task 0.0 in stage 1039.0 (TID 2572). 1226 bytes result sent to driver -2016-06-13 19:45:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 1039.0 (TID 2572) in 2 ms on localhost (1/1) -2016-06-13 19:45:31 INFO DAGScheduler:59 - ResultStage 1039 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 1039.0, whose tasks have all completed, from pool -2016-06-13 19:45:31 INFO DAGScheduler:59 - Job 852 finished: isEmpty at LDAOptimizer.scala:418, took 0.006122 s -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=112291, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1128 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=112467, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1128_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1128_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1128 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:31 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:31 INFO DAGScheduler:59 - Got job 853 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:31 INFO DAGScheduler:59 - Final stage: ResultStage 1040(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:31 INFO DAGScheduler:59 - Submitting ResultStage 1040 (MapPartitionsRDD[1170] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=112747, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1129 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:31 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=116339, maxMem=766075207 -2016-06-13 19:45:31 INFO MemoryStore:59 - Block broadcast_1129_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:31 INFO BlockManagerInfo:59 - Added broadcast_1129_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:31 INFO SparkContext:59 - Created broadcast 1129 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1040 (MapPartitionsRDD[1170] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1040.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1040.0 (TID 2573, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1040.0 (TID 2574, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1040.0 (TID 2575, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1040.0 (TID 2576, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1040.0 (TID 2573) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1040.0 (TID 2576) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1040.0 (TID 2575) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1040.0 (TID 2574) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1040.0 (TID 2576). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1040.0 (TID 2575). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1040.0 (TID 2576) in 3 ms on localhost (1/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1040.0 (TID 2573). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1040.0 (TID 2575) in 3 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1040.0 (TID 2573) in 5 ms on localhost (3/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1040.0 (TID 2574). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1040.0 (TID 2574) in 5 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1040 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1040.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 853 finished: reduce at LDAOptimizer.scala:453, took 0.009397 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 854 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1041(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1041 (MapPartitionsRDD[1171] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=118426, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1130 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=122058, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1130_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1130_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1130 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1041 (MapPartitionsRDD[1171] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1041.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1041.0 (TID 2577, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1041.0 (TID 2578, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1041.0 (TID 2579, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1041.0 (TID 2580, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1041.0 (TID 2580) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1041.0 (TID 2579) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1041.0 (TID 2578) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1041.0 (TID 2577) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1041.0 (TID 2578). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1041.0 (TID 2579). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1041.0 (TID 2580). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1041.0 (TID 2577). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1041.0 (TID 2578) in 3 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1041.0 (TID 2579) in 4 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1041.0 (TID 2580) in 4 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1041.0 (TID 2577) in 5 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1041 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1041.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 854 finished: reduce at LDAOptimizer.scala:456, took 0.010437 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 855 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1042(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1042 (PartitionwiseSampledRDD[1172] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=124145, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1131 stored as values in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=126297, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1131_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1131_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1131 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1042 (PartitionwiseSampledRDD[1172] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1042.0 with 1 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1042.0 (TID 2581, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1042.0 (TID 2581) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1042.0 (TID 2581). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1042.0 (TID 2581) in 2 ms on localhost (1/1) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1042 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1042.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 855 finished: isEmpty at LDAOptimizer.scala:418, took 0.007783 s -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=127612, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1132 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=127788, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1132_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1132_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1132 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 856 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1043(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1043 (MapPartitionsRDD[1174] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=128068, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1133 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=131660, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1133_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1133_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1133 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1043 (MapPartitionsRDD[1174] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1043.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1043.0 (TID 2582, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1043.0 (TID 2583, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1043.0 (TID 2584, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1043.0 (TID 2585, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1043.0 (TID 2583) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1043.0 (TID 2584) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1043.0 (TID 2582) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1043.0 (TID 2585) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1043.0 (TID 2584). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1043.0 (TID 2583). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1043.0 (TID 2585). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1043.0 (TID 2582). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1043.0 (TID 2584) in 3 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1043.0 (TID 2583) in 3 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1043.0 (TID 2585) in 3 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1043.0 (TID 2582) in 5 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1043 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1043.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 856 finished: reduce at LDAOptimizer.scala:453, took 0.009014 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 857 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1044(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1044 (MapPartitionsRDD[1175] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=133744, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1134 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=137376, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1134_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1134_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1134 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1044 (MapPartitionsRDD[1175] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1044.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1044.0 (TID 2586, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1044.0 (TID 2587, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1044.0 (TID 2588, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1044.0 (TID 2589, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1044.0 (TID 2587) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1044.0 (TID 2589) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1044.0 (TID 2586) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1044.0 (TID 2588) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1044.0 (TID 2589). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1044.0 (TID 2587). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1044.0 (TID 2588). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1044.0 (TID 2589) in 3 ms on localhost (1/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1044.0 (TID 2586). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1044.0 (TID 2587) in 3 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1044.0 (TID 2588) in 3 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1044.0 (TID 2586) in 4 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1044 (reduce at LDAOptimizer.scala:456) finished in 0.004 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1044.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 857 finished: reduce at LDAOptimizer.scala:456, took 0.008397 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 858 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1045(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1045 (PartitionwiseSampledRDD[1176] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=139460, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1135 stored as values in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=141612, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1135_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1135_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1135 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1045 (PartitionwiseSampledRDD[1176] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1045.0 with 1 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1045.0 (TID 2590, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1045.0 (TID 2590) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1045.0 (TID 2590). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1045.0 (TID 2590) in 2 ms on localhost (1/1) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1045 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1045.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 858 finished: isEmpty at LDAOptimizer.scala:418, took 0.006340 s -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=142929, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1136 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=143105, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1136_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1136_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1136 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 859 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1046(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1046 (MapPartitionsRDD[1178] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=143385, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1137 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=146977, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1137_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1137_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1137 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1046 (MapPartitionsRDD[1178] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1046.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1046.0 (TID 2591, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1046.0 (TID 2592, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1046.0 (TID 2593, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1046.0 (TID 2594, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1046.0 (TID 2591) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1046.0 (TID 2594) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1046.0 (TID 2592) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1046.0 (TID 2593) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1046.0 (TID 2594). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1046.0 (TID 2591). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1046.0 (TID 2593). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1046.0 (TID 2594) in 3 ms on localhost (1/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1046.0 (TID 2592). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1046.0 (TID 2593) in 3 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1046.0 (TID 2591) in 3 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1046.0 (TID 2592) in 3 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1046 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1046.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 859 finished: reduce at LDAOptimizer.scala:453, took 0.008333 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 860 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1047(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1047 (MapPartitionsRDD[1179] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=149064, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1138 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=152696, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1138_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1138_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1138 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1047 (MapPartitionsRDD[1179] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1047.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1047.0 (TID 2595, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1047.0 (TID 2596, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1047.0 (TID 2597, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1047.0 (TID 2598, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1047.0 (TID 2596) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1047.0 (TID 2598) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1047.0 (TID 2597) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1047.0 (TID 2595) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1047.0 (TID 2597). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1047.0 (TID 2596). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1047.0 (TID 2595). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1047.0 (TID 2598). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1047.0 (TID 2596) in 3 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1047.0 (TID 2595) in 4 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1047.0 (TID 2597) in 3 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1047.0 (TID 2598) in 4 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1047 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1047.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 860 finished: reduce at LDAOptimizer.scala:456, took 0.008671 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 861 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1048(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1048 (PartitionwiseSampledRDD[1180] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=154783, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1139 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1316) called with curMem=156935, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1139_piece0 stored as bytes in memory (estimated size 1316.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1139_piece0 in memory on localhost:50000 (size: 1316.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1139 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1048 (PartitionwiseSampledRDD[1180] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1048.0 with 1 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1048.0 (TID 2599, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1048.0 (TID 2599) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1048.0 (TID 2599). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1048.0 (TID 2599) in 2 ms on localhost (1/1) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1048 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1048.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 861 finished: isEmpty at LDAOptimizer.scala:418, took 0.005984 s -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=158251, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1140 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=158427, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1140_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1140_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1140 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 862 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1049(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1049 (MapPartitionsRDD[1182] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=158707, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1141 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=162299, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1141_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1141_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1141 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1049 (MapPartitionsRDD[1182] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1049.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1049.0 (TID 2600, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1049.0 (TID 2601, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1049.0 (TID 2602, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1049.0 (TID 2603, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1049.0 (TID 2600) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1049.0 (TID 2602) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1049.0 (TID 2601) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1049.0 (TID 2603) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1049.0 (TID 2601). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1049.0 (TID 2602). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1049.0 (TID 2601) in 5 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1049.0 (TID 2602) in 5 ms on localhost (2/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1049.0 (TID 2603). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1049.0 (TID 2600). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1049.0 (TID 2603) in 6 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1049.0 (TID 2600) in 8 ms on localhost (4/4) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1049.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1049 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 862 finished: reduce at LDAOptimizer.scala:453, took 0.014609 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 863 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1050(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1050 (MapPartitionsRDD[1183] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=164385, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1142 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=168017, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1142_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1142_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1142 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1050 (MapPartitionsRDD[1183] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1050.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1050.0 (TID 2604, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1050.0 (TID 2605, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1050.0 (TID 2606, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1050.0 (TID 2607, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1050.0 (TID 2607) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1050.0 (TID 2606) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1050.0 (TID 2604) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1050.0 (TID 2607). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1050.0 (TID 2605) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1050.0 (TID 2606). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1050.0 (TID 2606) in 4 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1050.0 (TID 2607) in 5 ms on localhost (2/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1050.0 (TID 2605). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1050.0 (TID 2604). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1050.0 (TID 2605) in 7 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1050.0 (TID 2604) in 7 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1050 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1050.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 863 finished: reduce at LDAOptimizer.scala:456, took 0.014540 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 864 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1051(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1051 (PartitionwiseSampledRDD[1184] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=170103, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1143 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=172255, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1143_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1143_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1143 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1051 (PartitionwiseSampledRDD[1184] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1051.0 with 1 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1051.0 (TID 2608, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1051.0 (TID 2608) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1051.0 (TID 2608). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1051.0 (TID 2608) in 2 ms on localhost (1/1) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1051 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1051.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 864 finished: isEmpty at LDAOptimizer.scala:418, took 0.007432 s -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=173570, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1144 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=173746, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1144_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1144_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1144 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 865 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1052(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1052 (MapPartitionsRDD[1186] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=174026, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1145 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=177618, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1145_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1145_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1145 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1052 (MapPartitionsRDD[1186] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1052.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1052.0 (TID 2609, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1052.0 (TID 2610, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1052.0 (TID 2611, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1052.0 (TID 2612, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1052.0 (TID 2612) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1052.0 (TID 2609) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1052.0 (TID 2609). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1052.0 (TID 2610) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1052.0 (TID 2611) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1052.0 (TID 2610). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1052.0 (TID 2609) in 6 ms on localhost (1/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1052.0 (TID 2612). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1052.0 (TID 2611). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1052.0 (TID 2612) in 6 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1052.0 (TID 2610) in 8 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1052.0 (TID 2611) in 8 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1052 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1052.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 865 finished: reduce at LDAOptimizer.scala:453, took 0.012859 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 866 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1053(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1053 (MapPartitionsRDD[1187] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=179702, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1146 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=183334, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1146_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1146_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1146 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1053 (MapPartitionsRDD[1187] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1053.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1053.0 (TID 2613, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1053.0 (TID 2614, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1053.0 (TID 2615, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1053.0 (TID 2616, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1053.0 (TID 2614) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1053.0 (TID 2616) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1053.0 (TID 2615) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1053.0 (TID 2613) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1053.0 (TID 2615). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1053.0 (TID 2614). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1053.0 (TID 2616). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1053.0 (TID 2615) in 5 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1053.0 (TID 2614) in 5 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1053.0 (TID 2616) in 4 ms on localhost (3/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1053.0 (TID 2613). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1053.0 (TID 2613) in 6 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1053 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1053.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 866 finished: reduce at LDAOptimizer.scala:456, took 0.010184 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 867 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1054(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1054 (PartitionwiseSampledRDD[1188] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=185418, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1147 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=187570, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1147_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1147_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1147 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1054 (PartitionwiseSampledRDD[1188] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1054.0 with 1 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1054.0 (TID 2617, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1054.0 (TID 2617) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1054.0 (TID 2617). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1054.0 (TID 2617) in 1 ms on localhost (1/1) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1054 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1054.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 867 finished: isEmpty at LDAOptimizer.scala:418, took 0.005796 s -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=188885, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1148 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=189061, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1148_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1148_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1148 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 868 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1055(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1055 (MapPartitionsRDD[1190] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=189341, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1149 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=192933, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1149_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1149_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1149 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1055 (MapPartitionsRDD[1190] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1055.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1055.0 (TID 2618, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1055.0 (TID 2619, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1055.0 (TID 2620, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1055.0 (TID 2621, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1055.0 (TID 2618) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1055.0 (TID 2621) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1055.0 (TID 2620) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1055.0 (TID 2619) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1055.0 (TID 2621). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1055.0 (TID 2619). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1055.0 (TID 2620). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1055.0 (TID 2621) in 3 ms on localhost (1/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1055.0 (TID 2618). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1055.0 (TID 2619) in 4 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1055.0 (TID 2620) in 4 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1055.0 (TID 2618) in 4 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1055 (reduce at LDAOptimizer.scala:453) finished in 0.004 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1055.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 868 finished: reduce at LDAOptimizer.scala:453, took 0.009495 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 869 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1056(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1056 (MapPartitionsRDD[1191] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=195018, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1150 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=198650, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1150_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1150_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1150 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1056 (MapPartitionsRDD[1191] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1056.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1056.0 (TID 2622, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1056.0 (TID 2623, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1056.0 (TID 2624, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1056.0 (TID 2625, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1056.0 (TID 2622) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1056.0 (TID 2623) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1056.0 (TID 2624) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1056.0 (TID 2625) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1056.0 (TID 2622). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1056.0 (TID 2623). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1056.0 (TID 2622) in 4 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1056.0 (TID 2623) in 3 ms on localhost (2/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1056.0 (TID 2624). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1056.0 (TID 2625). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1056.0 (TID 2624) in 4 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1056.0 (TID 2625) in 4 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1056 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1056.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 869 finished: reduce at LDAOptimizer.scala:456, took 0.010826 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 870 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1057(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1057 (PartitionwiseSampledRDD[1192] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=200735, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1151 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=202887, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1151_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1151_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1151 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1057 (PartitionwiseSampledRDD[1192] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1057.0 with 1 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1057.0 (TID 2626, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1057.0 (TID 2626) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1057.0 (TID 2626). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1057.0 (TID 2626) in 2 ms on localhost (1/1) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1057 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1057.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 870 finished: isEmpty at LDAOptimizer.scala:418, took 0.006363 s -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=204204, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1152 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=204380, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1152_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1152_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1152 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 871 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1058(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1058 (MapPartitionsRDD[1194] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=204660, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1153 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=208252, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1153_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1153_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1153 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1058 (MapPartitionsRDD[1194] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1058.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1058.0 (TID 2627, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1058.0 (TID 2628, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1058.0 (TID 2629, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1058.0 (TID 2630, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1058.0 (TID 2629) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1058.0 (TID 2630) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1058.0 (TID 2628) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1058.0 (TID 2627) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1058.0 (TID 2627). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1058.0 (TID 2630). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1058.0 (TID 2629). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1058.0 (TID 2628). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1058.0 (TID 2627) in 3 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1058.0 (TID 2630) in 3 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1058.0 (TID 2629) in 4 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1058.0 (TID 2628) in 5 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1058 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1058.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 871 finished: reduce at LDAOptimizer.scala:453, took 0.009164 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 872 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1059(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1059 (MapPartitionsRDD[1195] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=210338, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1154 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=213970, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1154_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1154_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1154 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1059 (MapPartitionsRDD[1195] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1059.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1059.0 (TID 2631, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1059.0 (TID 2632, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1059.0 (TID 2633, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1059.0 (TID 2634, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1059.0 (TID 2633) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1059.0 (TID 2634) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1059.0 (TID 2632) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1059.0 (TID 2631) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1059.0 (TID 2634). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1059.0 (TID 2633). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1059.0 (TID 2632). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1059.0 (TID 2631). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1059.0 (TID 2634) in 5 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1059.0 (TID 2633) in 5 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1059.0 (TID 2631) in 7 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1059.0 (TID 2632) in 7 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1059 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1059.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 872 finished: reduce at LDAOptimizer.scala:456, took 0.012379 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 873 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1060(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1060 (PartitionwiseSampledRDD[1196] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=216056, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1155 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=218208, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1155_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1155_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1155 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1060 (PartitionwiseSampledRDD[1196] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1060.0 with 1 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1060.0 (TID 2635, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1060.0 (TID 2635) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1060.0 (TID 2635). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1060.0 (TID 2635) in 2 ms on localhost (1/1) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1060 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1060.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 873 finished: isEmpty at LDAOptimizer.scala:418, took 0.006026 s -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=219525, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1156 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=219701, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1156_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1156_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1156 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 874 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1061(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1061 (MapPartitionsRDD[1198] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=219981, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1157 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=223573, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1157_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1157_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1157 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1061 (MapPartitionsRDD[1198] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1061.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1061.0 (TID 2636, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1061.0 (TID 2637, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1061.0 (TID 2638, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1061.0 (TID 2639, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1061.0 (TID 2636) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1061.0 (TID 2639) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1061.0 (TID 2638) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1061.0 (TID 2637) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1061.0 (TID 2639). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1061.0 (TID 2636). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1061.0 (TID 2637). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1061.0 (TID 2639) in 3 ms on localhost (1/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1061.0 (TID 2638). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1061.0 (TID 2636) in 4 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1061.0 (TID 2637) in 4 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1061.0 (TID 2638) in 3 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1061 (reduce at LDAOptimizer.scala:453) finished in 0.004 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1061.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 874 finished: reduce at LDAOptimizer.scala:453, took 0.008824 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 875 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1062(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1062 (MapPartitionsRDD[1199] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=225659, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1158 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=229291, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1158_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1158_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1158 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1062 (MapPartitionsRDD[1199] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1062.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1062.0 (TID 2640, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1062.0 (TID 2641, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1062.0 (TID 2642, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1062.0 (TID 2643, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1062.0 (TID 2642) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1062.0 (TID 2640) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1062.0 (TID 2641) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1062.0 (TID 2642). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1062.0 (TID 2643) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1062.0 (TID 2640). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1062.0 (TID 2642) in 4 ms on localhost (1/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1062.0 (TID 2641). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1062.0 (TID 2643). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1062.0 (TID 2640) in 5 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1062.0 (TID 2641) in 6 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1062.0 (TID 2643) in 5 ms on localhost (4/4) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1062.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1062 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 875 finished: reduce at LDAOptimizer.scala:456, took 0.012366 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 876 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1063(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1063 (PartitionwiseSampledRDD[1200] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=231377, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1159 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=233529, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1159_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1159_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1159 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1063 (PartitionwiseSampledRDD[1200] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1063.0 with 1 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1063.0 (TID 2644, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1063.0 (TID 2644) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1063.0 (TID 2644). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1063.0 (TID 2644) in 2 ms on localhost (1/1) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1063 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1063.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 876 finished: isEmpty at LDAOptimizer.scala:418, took 0.007272 s -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=234844, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1160 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=235020, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1160_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1160_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1160 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 877 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1064(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1064 (MapPartitionsRDD[1202] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=235300, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1161 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=238892, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1161_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1161_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1161 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1064 (MapPartitionsRDD[1202] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1064.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1064.0 (TID 2645, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1064.0 (TID 2646, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1064.0 (TID 2647, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1064.0 (TID 2648, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1064.0 (TID 2645) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1064.0 (TID 2647) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1064.0 (TID 2648) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1064.0 (TID 2646) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1064.0 (TID 2647). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1064.0 (TID 2646). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1064.0 (TID 2648). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1064.0 (TID 2645). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1064.0 (TID 2647) in 4 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1064.0 (TID 2646) in 4 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1064.0 (TID 2645) in 5 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1064.0 (TID 2648) in 4 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1064 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1064.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 877 finished: reduce at LDAOptimizer.scala:453, took 0.009848 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 878 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1065(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1065 (MapPartitionsRDD[1203] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=240977, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1162 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=244609, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1162_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1162_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1162 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1065 (MapPartitionsRDD[1203] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1065.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1065.0 (TID 2649, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1065.0 (TID 2650, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1065.0 (TID 2651, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1065.0 (TID 2652, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1065.0 (TID 2649) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1065.0 (TID 2652) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1065.0 (TID 2651) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1065.0 (TID 2650) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1065.0 (TID 2652). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1065.0 (TID 2649). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1065.0 (TID 2650). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1065.0 (TID 2652) in 3 ms on localhost (1/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1065.0 (TID 2651). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1065.0 (TID 2649) in 4 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1065.0 (TID 2650) in 4 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1065.0 (TID 2651) in 4 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1065 (reduce at LDAOptimizer.scala:456) finished in 0.004 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1065.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 878 finished: reduce at LDAOptimizer.scala:456, took 0.009440 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 879 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1066(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1066 (PartitionwiseSampledRDD[1204] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=246694, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1163 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=248846, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1163_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1163_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1163 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1066 (PartitionwiseSampledRDD[1204] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1066.0 with 1 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1066.0 (TID 2653, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1066.0 (TID 2653) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1066.0 (TID 2653). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1066.0 (TID 2653) in 2 ms on localhost (1/1) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1066 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1066.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 879 finished: isEmpty at LDAOptimizer.scala:418, took 0.006750 s -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=250161, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1164 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=250337, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1164_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1164_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1164 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 880 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1067(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1067 (MapPartitionsRDD[1206] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=250617, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1165 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=254209, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1165_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1165_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1165 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1067 (MapPartitionsRDD[1206] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1067.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1067.0 (TID 2654, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1067.0 (TID 2655, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1067.0 (TID 2656, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1067.0 (TID 2657, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1067.0 (TID 2654) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1067.0 (TID 2655) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1067.0 (TID 2656) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1067.0 (TID 2657) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1067.0 (TID 2657). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1067.0 (TID 2656). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1067.0 (TID 2655). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1067.0 (TID 2654). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1067.0 (TID 2657) in 4 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1067.0 (TID 2655) in 4 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1067.0 (TID 2656) in 4 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1067.0 (TID 2654) in 5 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1067 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1067.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 880 finished: reduce at LDAOptimizer.scala:453, took 0.010006 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 881 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1068(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1068 (MapPartitionsRDD[1207] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=256294, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1166 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=259926, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1166_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1166_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1166 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1068 (MapPartitionsRDD[1207] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1068.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1068.0 (TID 2658, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1068.0 (TID 2659, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1068.0 (TID 2660, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1068.0 (TID 2661, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1068.0 (TID 2658) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1068.0 (TID 2661) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1068.0 (TID 2660) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1068.0 (TID 2659) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1068.0 (TID 2660). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1068.0 (TID 2660) in 2 ms on localhost (1/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1068.0 (TID 2661). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1068.0 (TID 2659). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1068.0 (TID 2658). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1068.0 (TID 2661) in 3 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1068.0 (TID 2659) in 4 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1068.0 (TID 2658) in 5 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1068 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1068.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 881 finished: reduce at LDAOptimizer.scala:456, took 0.008386 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 882 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1069(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1069 (PartitionwiseSampledRDD[1208] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=262011, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1167 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1318) called with curMem=264163, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1167_piece0 stored as bytes in memory (estimated size 1318.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1167_piece0 in memory on localhost:50000 (size: 1318.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1167 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1069 (PartitionwiseSampledRDD[1208] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1069.0 with 1 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1069.0 (TID 2662, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1069.0 (TID 2662) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1069.0 (TID 2662). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1069.0 (TID 2662) in 3 ms on localhost (1/1) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1069.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1069 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 882 finished: isEmpty at LDAOptimizer.scala:418, took 0.006974 s -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=265481, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1168 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=265657, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1168_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1168_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1168 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 883 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1070(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1070 (MapPartitionsRDD[1210] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=265937, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1169 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=269529, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1169_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1169_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1169 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1070 (MapPartitionsRDD[1210] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1070.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1070.0 (TID 2663, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1070.0 (TID 2664, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1070.0 (TID 2665, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1070.0 (TID 2666, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1070.0 (TID 2664) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1070.0 (TID 2666) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1070.0 (TID 2663) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1070.0 (TID 2665) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1070.0 (TID 2666). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1070.0 (TID 2664). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1070.0 (TID 2663). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1070.0 (TID 2665). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1070.0 (TID 2666) in 2 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1070.0 (TID 2663) in 4 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1070.0 (TID 2664) in 4 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1070.0 (TID 2665) in 4 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1070 (reduce at LDAOptimizer.scala:453) finished in 0.004 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1070.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 883 finished: reduce at LDAOptimizer.scala:453, took 0.008802 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 884 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1071(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1071 (MapPartitionsRDD[1211] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=271616, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1170 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=275248, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1170_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1170_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1170 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1071 (MapPartitionsRDD[1211] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1071.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1071.0 (TID 2667, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1071.0 (TID 2668, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1071.0 (TID 2669, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1071.0 (TID 2670, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1071.0 (TID 2668) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1071.0 (TID 2670) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1071.0 (TID 2669) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1071.0 (TID 2667) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1071.0 (TID 2667). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1071.0 (TID 2669). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1071.0 (TID 2670). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1071.0 (TID 2668). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1071.0 (TID 2667) in 6 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1071.0 (TID 2669) in 5 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1071.0 (TID 2670) in 6 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1071.0 (TID 2668) in 7 ms on localhost (4/4) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1071.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1071 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 884 finished: reduce at LDAOptimizer.scala:456, took 0.012753 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 885 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1072(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1072 (PartitionwiseSampledRDD[1212] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=277335, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1171 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=279487, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1171_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1171_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1171 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1072 (PartitionwiseSampledRDD[1212] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1072.0 with 1 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1072.0 (TID 2671, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1072.0 (TID 2671) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1072.0 (TID 2671). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1072.0 (TID 2671) in 1 ms on localhost (1/1) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1072 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1072.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 885 finished: isEmpty at LDAOptimizer.scala:418, took 0.005799 s -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=280802, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1172 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=280978, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1172_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1172_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1172 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 886 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1073(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1073 (MapPartitionsRDD[1214] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=281258, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1173 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=284850, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1173_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1173_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1173 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1073 (MapPartitionsRDD[1214] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1073.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1073.0 (TID 2672, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1073.0 (TID 2673, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1073.0 (TID 2674, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1073.0 (TID 2675, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1073.0 (TID 2673) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1073.0 (TID 2674) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1073.0 (TID 2672) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1073.0 (TID 2675) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1073.0 (TID 2673). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1073.0 (TID 2672). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1073.0 (TID 2675). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1073.0 (TID 2674). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1073.0 (TID 2673) in 3 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1073.0 (TID 2672) in 3 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1073.0 (TID 2675) in 4 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1073.0 (TID 2674) in 4 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1073 (reduce at LDAOptimizer.scala:453) finished in 0.004 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1073.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 886 finished: reduce at LDAOptimizer.scala:453, took 0.008970 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 887 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1074(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1074 (MapPartitionsRDD[1215] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=286937, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1174 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=290569, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1174_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1174_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1174 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1074 (MapPartitionsRDD[1215] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1074.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1074.0 (TID 2676, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1074.0 (TID 2677, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1074.0 (TID 2678, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1074.0 (TID 2679, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1074.0 (TID 2678) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1074.0 (TID 2679) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1074.0 (TID 2676) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1074.0 (TID 2677) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1074.0 (TID 2677). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1074.0 (TID 2679). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1074.0 (TID 2678). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1074.0 (TID 2676). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1074.0 (TID 2677) in 5 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1074.0 (TID 2679) in 4 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1074.0 (TID 2676) in 5 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1074.0 (TID 2678) in 4 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1074 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1074.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 887 finished: reduce at LDAOptimizer.scala:456, took 0.010159 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 888 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1075(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1075 (PartitionwiseSampledRDD[1216] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=292656, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1175 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=294808, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1175_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1175_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1175 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1075 (PartitionwiseSampledRDD[1216] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1075.0 with 1 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1075.0 (TID 2680, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1075.0 (TID 2680) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1075.0 (TID 2680). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1075.0 (TID 2680) in 2 ms on localhost (1/1) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1075 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1075.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 888 finished: isEmpty at LDAOptimizer.scala:418, took 0.005889 s -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=296123, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1176 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=296299, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1176_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1176_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1176 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 889 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1076(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1076 (MapPartitionsRDD[1218] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=296579, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1177 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=300171, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1177_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1177_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1177 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1076 (MapPartitionsRDD[1218] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1076.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1076.0 (TID 2681, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1076.0 (TID 2682, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1076.0 (TID 2683, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1076.0 (TID 2684, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1076.0 (TID 2682) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1076.0 (TID 2684) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1076.0 (TID 2681) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1076.0 (TID 2683) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1076.0 (TID 2684). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1076.0 (TID 2683). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1076.0 (TID 2684) in 2 ms on localhost (1/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1076.0 (TID 2682). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1076.0 (TID 2681). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1076.0 (TID 2683) in 4 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1076.0 (TID 2682) in 4 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1076.0 (TID 2681) in 4 ms on localhost (4/4) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1076.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1076 (reduce at LDAOptimizer.scala:453) finished in 0.004 s -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 889 finished: reduce at LDAOptimizer.scala:453, took 0.009267 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 890 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1077(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1077 (MapPartitionsRDD[1219] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=302255, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1178 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=305887, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1178_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1178_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1178 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1077 (MapPartitionsRDD[1219] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1077.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1077.0 (TID 2685, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1077.0 (TID 2686, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1077.0 (TID 2687, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1077.0 (TID 2688, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1077.0 (TID 2685) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1077.0 (TID 2688) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1077.0 (TID 2686) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1077.0 (TID 2685). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1077.0 (TID 2687) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1077.0 (TID 2687). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1077.0 (TID 2686). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1077.0 (TID 2688). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1077.0 (TID 2685) in 12 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1077.0 (TID 2686) in 12 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1077.0 (TID 2687) in 12 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1077.0 (TID 2688) in 12 ms on localhost (4/4) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1077.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1077 (reduce at LDAOptimizer.scala:456) finished in 0.013 s -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 890 finished: reduce at LDAOptimizer.scala:456, took 0.018582 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 891 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1078(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1078 (PartitionwiseSampledRDD[1220] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=307971, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1179 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=310123, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1179_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1179_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1179 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1078 (PartitionwiseSampledRDD[1220] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1078.0 with 1 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1078.0 (TID 2689, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1078.0 (TID 2689) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1078.0 (TID 2689). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1078.0 (TID 2689) in 3 ms on localhost (1/1) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1078.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1078 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 891 finished: isEmpty at LDAOptimizer.scala:418, took 0.009406 s -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=311440, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1180 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=311616, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1180_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1180_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1180 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 892 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1079(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1079 (MapPartitionsRDD[1222] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=311896, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1181 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=315488, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1181_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1181_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1181 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1079 (MapPartitionsRDD[1222] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1079.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1079.0 (TID 2690, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1079.0 (TID 2691, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1079.0 (TID 2692, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1079.0 (TID 2693, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1079.0 (TID 2690) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1079.0 (TID 2693) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1079.0 (TID 2691) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1079.0 (TID 2692) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1079.0 (TID 2691). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1079.0 (TID 2690). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1079.0 (TID 2692). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1079.0 (TID 2691) in 4 ms on localhost (1/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1079.0 (TID 2693). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1079.0 (TID 2690) in 4 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1079.0 (TID 2692) in 4 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1079.0 (TID 2693) in 3 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1079 (reduce at LDAOptimizer.scala:453) finished in 0.004 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1079.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 892 finished: reduce at LDAOptimizer.scala:453, took 0.010501 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 893 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1080(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1080 (MapPartitionsRDD[1223] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=317575, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1182 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=321207, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1182_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1182_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1182 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1080 (MapPartitionsRDD[1223] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1080.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1080.0 (TID 2694, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1080.0 (TID 2695, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1080.0 (TID 2696, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1080.0 (TID 2697, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1080.0 (TID 2694) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1080.0 (TID 2697) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1080.0 (TID 2695) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1080.0 (TID 2696) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1080.0 (TID 2695). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1080.0 (TID 2696). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1080.0 (TID 2697). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1080.0 (TID 2695) in 3 ms on localhost (1/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1080.0 (TID 2694). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1080.0 (TID 2697) in 3 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1080.0 (TID 2696) in 4 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1080.0 (TID 2694) in 4 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1080 (reduce at LDAOptimizer.scala:456) finished in 0.004 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1080.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 893 finished: reduce at LDAOptimizer.scala:456, took 0.008631 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 894 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1081(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1081 (PartitionwiseSampledRDD[1224] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=323294, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1183 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=325446, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1183_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1183_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1183 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1081 (PartitionwiseSampledRDD[1224] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1081.0 with 1 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1081.0 (TID 2698, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1081.0 (TID 2698) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1081.0 (TID 2698). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1081.0 (TID 2698) in 2 ms on localhost (1/1) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1081 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1081.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 894 finished: isEmpty at LDAOptimizer.scala:418, took 0.006083 s -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=326763, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1184 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=326939, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1184_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1184_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1184 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 895 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1082(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1082 (MapPartitionsRDD[1226] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=327219, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1185 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=330811, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1185_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1185_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1185 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1082 (MapPartitionsRDD[1226] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1082.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1082.0 (TID 2699, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1082.0 (TID 2700, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1082.0 (TID 2701, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1082.0 (TID 2702, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1082.0 (TID 2699) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1082.0 (TID 2700) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1082.0 (TID 2702) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1082.0 (TID 2701) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1082.0 (TID 2700). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1082.0 (TID 2702). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1082.0 (TID 2699). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1082.0 (TID 2700) in 5 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1082.0 (TID 2699) in 5 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1082.0 (TID 2702) in 4 ms on localhost (3/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1082.0 (TID 2701). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1082.0 (TID 2701) in 5 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1082 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1082.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 895 finished: reduce at LDAOptimizer.scala:453, took 0.010301 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 896 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1083(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1083 (MapPartitionsRDD[1227] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=332897, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1186 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=336529, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1186_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1186_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.4 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1186 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1083 (MapPartitionsRDD[1227] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1083.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1083.0 (TID 2703, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1083.0 (TID 2704, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1083.0 (TID 2705, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1083.0 (TID 2706, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1083.0 (TID 2705) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1083.0 (TID 2706) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1083.0 (TID 2704) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1083.0 (TID 2703) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1083.0 (TID 2705). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1083.0 (TID 2703). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1083.0 (TID 2705) in 3 ms on localhost (1/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1083.0 (TID 2706). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1083.0 (TID 2703) in 4 ms on localhost (2/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1083.0 (TID 2704). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1083.0 (TID 2706) in 3 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1083.0 (TID 2704) in 4 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1083 (reduce at LDAOptimizer.scala:456) finished in 0.004 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1083.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 896 finished: reduce at LDAOptimizer.scala:456, took 0.008732 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 897 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1084(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1084 (PartitionwiseSampledRDD[1228] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=338615, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1187 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=340767, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1187_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1187_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.4 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1187 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1084 (PartitionwiseSampledRDD[1228] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1084.0 with 1 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1084.0 (TID 2707, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1084.0 (TID 2707) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1084.0 (TID 2707). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1084.0 (TID 2707) in 2 ms on localhost (1/1) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1084 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1084.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 897 finished: isEmpty at LDAOptimizer.scala:418, took 0.008261 s -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=342084, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1188 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=342260, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1188_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1188_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1188 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 898 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1085(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1085 (MapPartitionsRDD[1230] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=342540, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1189 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=346132, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1189_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1189_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.4 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1189 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1085 (MapPartitionsRDD[1230] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1085.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1085.0 (TID 2708, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1187_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.4 MB) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1085.0 (TID 2709, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1085.0 (TID 2710, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 918 -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1085.0 (TID 2711, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1085.0 (TID 2708) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1085.0 (TID 2710) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1085.0 (TID 2711) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1085.0 (TID 2709) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1085.0 (TID 2711). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1186_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.4 MB) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1085.0 (TID 2711) in 3 ms on localhost (1/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1085.0 (TID 2708). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1085.0 (TID 2709). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1085.0 (TID 2710). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1085.0 (TID 2708) in 7 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1085.0 (TID 2710) in 6 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1085.0 (TID 2709) in 6 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1085 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1085.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 898 finished: reduce at LDAOptimizer.scala:453, took 0.023274 s -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 917 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1185_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 916 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 899 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1086(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1086 (MapPartitionsRDD[1231] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=332898, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1190 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1184_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=336530, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1190_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1190_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1183_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1190 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1086 (MapPartitionsRDD[1231] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1086.0 with 4 tasks -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 915 -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1086.0 (TID 2712, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1086.0 (TID 2713, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1086.0 (TID 2714, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1086.0 (TID 2715, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1086.0 (TID 2714) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1086.0 (TID 2715) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1086.0 (TID 2712) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1086.0 (TID 2713) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1182_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1086.0 (TID 2713). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1086.0 (TID 2714). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1086.0 (TID 2715). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1086.0 (TID 2712). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 914 -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1086.0 (TID 2713) in 6 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1086.0 (TID 2715) in 7 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1086.0 (TID 2714) in 7 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1086.0 (TID 2712) in 9 ms on localhost (4/4) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1086.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1086 (reduce at LDAOptimizer.scala:456) finished in 0.009 s -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1181_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 899 finished: reduce at LDAOptimizer.scala:456, took 0.015699 s -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 913 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1180_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 900 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1087(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1087 (PartitionwiseSampledRDD[1232] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=323294, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1191 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=323294, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1191_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1191_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1179_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1191 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1087 (PartitionwiseSampledRDD[1232] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1087.0 with 1 tasks -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 912 -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1087.0 (TID 2716, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1087.0 (TID 2716) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1087.0 (TID 2716). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1087.0 (TID 2716) in 2 ms on localhost (1/1) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1087.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1087 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1178_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 900 finished: isEmpty at LDAOptimizer.scala:418, took 0.009619 s -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 911 -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=313986, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1192 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1177_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 910 -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=312078, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1192_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1192_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1176_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1192 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1175_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 909 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 901 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1088(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1174_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1088 (MapPartitionsRDD[1234] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 908 -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=302716, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1193 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=300629, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1193_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1173_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1193_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 907 -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1193 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1088 (MapPartitionsRDD[1234] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1088.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1088.0 (TID 2717, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1088.0 (TID 2718, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1088.0 (TID 2719, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1088.0 (TID 2720, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1088.0 (TID 2720) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1088.0 (TID 2718) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1088.0 (TID 2719) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1172_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1088.0 (TID 2720). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1088.0 (TID 2718). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1088.0 (TID 2717) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1088.0 (TID 2719). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1088.0 (TID 2717). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1088.0 (TID 2720) in 9 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1088.0 (TID 2718) in 10 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1088.0 (TID 2719) in 9 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1088.0 (TID 2717) in 11 ms on localhost (4/4) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1088.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1088 (reduce at LDAOptimizer.scala:453) finished in 0.011 s -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 901 finished: reduce at LDAOptimizer.scala:453, took 0.017327 s -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1171_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 906 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1170_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 902 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 905 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1089(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1089 (MapPartitionsRDD[1235] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=287394, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1194 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=291026, maxMem=766075207 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1169_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1194_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1194_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1194 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1089 (MapPartitionsRDD[1235] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 904 -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1089.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1089.0 (TID 2721, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1089.0 (TID 2722, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1089.0 (TID 2723, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1089.0 (TID 2724, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1089.0 (TID 2723) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1089.0 (TID 2721) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1168_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1089.0 (TID 2724) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1089.0 (TID 2722) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1089.0 (TID 2721). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1089.0 (TID 2723). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1167_piece0 on localhost:50000 in memory (size: 1318.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1089.0 (TID 2724). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1089.0 (TID 2722). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1089.0 (TID 2721) in 7 ms on localhost (1/4) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 903 -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1089.0 (TID 2723) in 5 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1089.0 (TID 2722) in 6 ms on localhost (3/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1089 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1089.0 (TID 2724) in 5 ms on localhost (4/4) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1089.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 902 finished: reduce at LDAOptimizer.scala:456, took 0.015040 s -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1166_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 902 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1165_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 901 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 903 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1090(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1090 (PartitionwiseSampledRDD[1236] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=277512, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1195 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1164_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=277336, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1195_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1163_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1195_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 900 -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1195 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1090 (PartitionwiseSampledRDD[1236] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1090.0 with 1 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1090.0 (TID 2725, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1090.0 (TID 2725) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1162_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 899 -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1090.0 (TID 2725). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1090.0 (TID 2725) in 3 ms on localhost (1/1) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1090 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1090.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 903 finished: isEmpty at LDAOptimizer.scala:418, took 0.009339 s -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1161_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=265942, maxMem=766075207 -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 898 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1196 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=265662, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1196_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1160_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1196_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1196 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1159_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 897 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1158_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 896 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 904 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1091(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1091 (MapPartitionsRDD[1238] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1157_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 895 -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=251079, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1197 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=254215, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1197_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1156_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1197_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1197 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1091 (MapPartitionsRDD[1238] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1091.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1091.0 (TID 2726, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1091.0 (TID 2727, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1155_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1091.0 (TID 2728, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 894 -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1091.0 (TID 2729, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1091.0 (TID 2726) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1091.0 (TID 2728) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1091.0 (TID 2727) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1091.0 (TID 2726). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1154_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1091.0 (TID 2728). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1091.0 (TID 2726) in 4 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1091.0 (TID 2728) in 5 ms on localhost (2/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1091.0 (TID 2727). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1091.0 (TID 2727) in 6 ms on localhost (3/4) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1091.0 (TID 2729) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 893 -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1091.0 (TID 2729). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1091.0 (TID 2729) in 8 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1091 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1091.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1153_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 904 finished: reduce at LDAOptimizer.scala:453, took 0.015516 s -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 892 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1152_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 905 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1092(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1092 (MapPartitionsRDD[1239] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1151_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 891 -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=237509, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1198 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1150_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=239056, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1198_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 890 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1198_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1198 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1092 (MapPartitionsRDD[1239] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1092.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1092.0 (TID 2730, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1149_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1092.0 (TID 2731, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 889 -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1092.0 (TID 2732, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1092.0 (TID 2733, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1092.0 (TID 2731) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1092.0 (TID 2730) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1092.0 (TID 2733) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1092.0 (TID 2732) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1148_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1092.0 (TID 2733). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1092.0 (TID 2733) in 3 ms on localhost (1/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1092.0 (TID 2732). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1092.0 (TID 2730). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1092.0 (TID 2732) in 5 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1092.0 (TID 2730) in 7 ms on localhost (3/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1092.0 (TID 2731). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1092.0 (TID 2731) in 7 ms on localhost (4/4) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1092.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1092 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 905 finished: reduce at LDAOptimizer.scala:456, took 0.013323 s -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1147_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 888 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1146_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 887 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1145_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 906 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1093(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1093 (PartitionwiseSampledRDD[1240] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=216516, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1199 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 886 -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=218212, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1199_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1199_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1144_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1199 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1093 (PartitionwiseSampledRDD[1240] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1093.0 with 1 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1093.0 (TID 2734, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1093.0 (TID 2734) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1143_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1093.0 (TID 2734). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 885 -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1093.0 (TID 2734) in 2 ms on localhost (1/1) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1093 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1093.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 906 finished: isEmpty at LDAOptimizer.scala:418, took 0.007374 s -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1142_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=213976, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1200 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 884 -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=208434, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1200_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1141_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1200_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 883 -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1200 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1140_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1139_piece0 on localhost:50000 in memory (size: 1316.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 882 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 907 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1094(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1094 (MapPartitionsRDD[1242] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1138_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=199111, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1201 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 881 -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=199071, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1201_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1201_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1137_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1201 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1094 (MapPartitionsRDD[1242] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1094.0 with 4 tasks -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 880 -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1094.0 (TID 2735, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1094.0 (TID 2736, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1094.0 (TID 2737, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1094.0 (TID 2738, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1136_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1094.0 (TID 2738) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1094.0 (TID 2736) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1094.0 (TID 2735) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1094.0 (TID 2737) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1094.0 (TID 2735). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1135_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1094.0 (TID 2737). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1094.0 (TID 2738). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1094.0 (TID 2736). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 879 -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1094.0 (TID 2735) in 5 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1094.0 (TID 2738) in 4 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1094.0 (TID 2737) in 5 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1094.0 (TID 2736) in 6 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1094 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1094.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 907 finished: reduce at LDAOptimizer.scala:453, took 0.012282 s -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1134_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 878 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1133_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 908 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1095(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 877 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1095 (MapPartitionsRDD[1243] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=179705, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1202 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1132_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=183337, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1202_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1202_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1202 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1095 (MapPartitionsRDD[1243] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1095.0 with 4 tasks -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1131_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 876 -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1095.0 (TID 2739, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1095.0 (TID 2740, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1095.0 (TID 2741, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1095.0 (TID 2742, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1095.0 (TID 2740) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1095.0 (TID 2742) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1095.0 (TID 2739) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1130_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1095.0 (TID 2741) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 875 -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1095.0 (TID 2740). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1095.0 (TID 2742). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1095.0 (TID 2741). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1095.0 (TID 2740) in 4 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1095.0 (TID 2742) in 4 ms on localhost (2/4) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1129_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1095.0 (TID 2739). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 874 -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1095.0 (TID 2741) in 5 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1095.0 (TID 2739) in 7 ms on localhost (4/4) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1095.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1095 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 908 finished: reduce at LDAOptimizer.scala:456, took 0.011612 s -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1128_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1127_piece0 on localhost:50000 in memory (size: 1319.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 873 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 909 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1096(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1096 (PartitionwiseSampledRDD[1244] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1126_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 872 -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=160913, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1203 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1125_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=160979, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1203_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 871 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1203_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1203 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1096 (PartitionwiseSampledRDD[1244] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1096.0 with 1 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1096.0 (TID 2743, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1096.0 (TID 2743) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1124_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1096.0 (TID 2743). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1096 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1096.0 (TID 2743) in 3 ms on localhost (1/1) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 909 finished: isEmpty at LDAOptimizer.scala:418, took 0.008291 s -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1123_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1096.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 870 -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=154777, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1204 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=149235, maxMem=766075207 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1122_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1204_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 869 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1204_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1204 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1121_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 868 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1120_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 910 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1097(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1097 (MapPartitionsRDD[1246] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1119_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=139912, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1205 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 867 -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=137788, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1205_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1118_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 866 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1205_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1205 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1097 (MapPartitionsRDD[1246] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1097.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1097.0 (TID 2744, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1097.0 (TID 2745, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1097.0 (TID 2746, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1097.0 (TID 2747, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1097.0 (TID 2745) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1097.0 (TID 2747) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1117_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1097.0 (TID 2746) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1097.0 (TID 2744) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 865 -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1097.0 (TID 2747). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1097.0 (TID 2745). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1097.0 (TID 2746). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1097.0 (TID 2744). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1097.0 (TID 2747) in 4 ms on localhost (1/4) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1116_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1097.0 (TID 2745) in 5 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1097.0 (TID 2746) in 5 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1097.0 (TID 2744) in 6 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1097 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1097.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 910 finished: reduce at LDAOptimizer.scala:453, took 0.010991 s -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1115_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 864 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 911 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1098(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1114_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1098 (MapPartitionsRDD[1247] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 863 -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=122472, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1206 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1113_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=122512, maxMem=766075207 -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 862 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1206_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1206_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1206 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1098 (MapPartitionsRDD[1247] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1098.0 with 4 tasks -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1112_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1098.0 (TID 2748, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1098.0 (TID 2749, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1098.0 (TID 2750, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1098.0 (TID 2751, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1098.0 (TID 2749) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1098.0 (TID 2748) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1098.0 (TID 2750) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1098.0 (TID 2751) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1111_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1098.0 (TID 2748). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 861 -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1098.0 (TID 2748) in 5 ms on localhost (1/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1098.0 (TID 2751). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1098.0 (TID 2751) in 3 ms on localhost (2/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1098.0 (TID 2750). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1098.0 (TID 2749). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1098.0 (TID 2750) in 6 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1098.0 (TID 2749) in 6 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1098 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1110_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 911 finished: reduce at LDAOptimizer.scala:456, took 0.012219 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1098.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 860 -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1109_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 859 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 912 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1108_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1099(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1099 (PartitionwiseSampledRDD[1248] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=107505, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1207 stored as values in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1107_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 858 -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=107505, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1207_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1207_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1207 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1099 (PartitionwiseSampledRDD[1248] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1106_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1099.0 with 1 tasks -2016-06-13 19:45:32 INFO ContextCleaner:59 - Cleaned accumulator 857 -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1099.0 (TID 2752, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1099.0 (TID 2752) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Removed broadcast_1104_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1099.0 (TID 2752). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1099.0 (TID 2752) in 2 ms on localhost (1/1) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1099 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1099.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 912 finished: isEmpty at LDAOptimizer.scala:418, took 0.007409 s -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=102643, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1208 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=102819, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1208_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1208_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1208 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 913 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1100(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1100 (MapPartitionsRDD[1250] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=103099, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1209 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=106691, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1209_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1209_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1209 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1100 (MapPartitionsRDD[1250] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1100.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1100.0 (TID 2753, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1100.0 (TID 2754, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1100.0 (TID 2755, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1100.0 (TID 2756, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1100.0 (TID 2755) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1100.0 (TID 2756) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1100.0 (TID 2753) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1100.0 (TID 2754) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1100.0 (TID 2756). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1100.0 (TID 2753). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1100.0 (TID 2754). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1100.0 (TID 2755). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1100.0 (TID 2756) in 8 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1100.0 (TID 2753) in 9 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1100.0 (TID 2754) in 10 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1100.0 (TID 2755) in 9 ms on localhost (4/4) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1100.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1100 (reduce at LDAOptimizer.scala:453) finished in 0.010 s -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 913 finished: reduce at LDAOptimizer.scala:453, took 0.015563 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 914 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1101(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1101 (MapPartitionsRDD[1251] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=108775, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1210 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=112407, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1210_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1210_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1210 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1101 (MapPartitionsRDD[1251] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1101.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1101.0 (TID 2757, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1101.0 (TID 2758, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1101.0 (TID 2759, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1101.0 (TID 2760, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1101.0 (TID 2757) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1101.0 (TID 2760) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1101.0 (TID 2759) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1101.0 (TID 2758) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1101.0 (TID 2760). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1101.0 (TID 2759). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1101.0 (TID 2760) in 3 ms on localhost (1/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1101.0 (TID 2758). 1358 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1101.0 (TID 2757). 1314 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1101.0 (TID 2759) in 4 ms on localhost (2/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1101.0 (TID 2757) in 5 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1101.0 (TID 2758) in 6 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1101 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 914 finished: reduce at LDAOptimizer.scala:456, took 0.011860 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1101.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 915 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1102(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1102 (PartitionwiseSampledRDD[1252] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=114491, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1211 stored as values in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(1316) called with curMem=116643, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1211_piece0 stored as bytes in memory (estimated size 1316.0 B, free 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1211_piece0 in memory on localhost:50000 (size: 1316.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1211 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1102 (PartitionwiseSampledRDD[1252] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1102.0 with 1 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1102.0 (TID 2761, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1102.0 (TID 2761) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1102.0 (TID 2761). 1226 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1102.0 (TID 2761) in 2 ms on localhost (1/1) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1102.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1102 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 915 finished: isEmpty at LDAOptimizer.scala:418, took 0.007927 s -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=117959, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1212 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=118135, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1212_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1212_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1212 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 916 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1103(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1103 (MapPartitionsRDD[1254] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=118415, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1213 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=122007, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1213_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:32 INFO BlockManagerInfo:59 - Added broadcast_1213_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:32 INFO SparkContext:59 - Created broadcast 1213 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1103 (MapPartitionsRDD[1254] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Adding task set 1103.0 with 4 tasks -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1103.0 (TID 2762, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1103.0 (TID 2763, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 2.0 in stage 1103.0 (TID 2764, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Starting task 3.0 in stage 1103.0 (TID 2765, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:32 INFO Executor:59 - Running task 1.0 in stage 1103.0 (TID 2763) -2016-06-13 19:45:32 INFO Executor:59 - Running task 3.0 in stage 1103.0 (TID 2765) -2016-06-13 19:45:32 INFO Executor:59 - Running task 2.0 in stage 1103.0 (TID 2764) -2016-06-13 19:45:32 INFO Executor:59 - Running task 0.0 in stage 1103.0 (TID 2762) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 3.0 in stage 1103.0 (TID 2765). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 1.0 in stage 1103.0 (TID 2763). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 3.0 in stage 1103.0 (TID 2765) in 4 ms on localhost (1/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 1103.0 (TID 2763) in 4 ms on localhost (2/4) -2016-06-13 19:45:32 INFO Executor:59 - Finished task 2.0 in stage 1103.0 (TID 2764). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO Executor:59 - Finished task 0.0 in stage 1103.0 (TID 2762). 1265 bytes result sent to driver -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 2.0 in stage 1103.0 (TID 2764) in 5 ms on localhost (3/4) -2016-06-13 19:45:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1103.0 (TID 2762) in 6 ms on localhost (4/4) -2016-06-13 19:45:32 INFO DAGScheduler:59 - ResultStage 1103 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1103.0, whose tasks have all completed, from pool -2016-06-13 19:45:32 INFO DAGScheduler:59 - Job 916 finished: reduce at LDAOptimizer.scala:453, took 0.010937 s -2016-06-13 19:45:32 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:32 INFO DAGScheduler:59 - Got job 917 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:32 INFO DAGScheduler:59 - Final stage: ResultStage 1104(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:32 INFO DAGScheduler:59 - Submitting ResultStage 1104 (MapPartitionsRDD[1255] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:32 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=124091, maxMem=766075207 -2016-06-13 19:45:32 INFO MemoryStore:59 - Block broadcast_1214 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=127723, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1214_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1214_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1214 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1104 (MapPartitionsRDD[1255] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1104.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1104.0 (TID 2766, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1104.0 (TID 2767, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1104.0 (TID 2768, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1104.0 (TID 2769, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1104.0 (TID 2766) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1104.0 (TID 2769) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1104.0 (TID 2767) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1104.0 (TID 2768) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1104.0 (TID 2766). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1104.0 (TID 2769). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1104.0 (TID 2766) in 5 ms on localhost (1/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1104.0 (TID 2767). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1104.0 (TID 2769) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1104.0 (TID 2768). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1104.0 (TID 2767) in 6 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1104.0 (TID 2768) in 5 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1104 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 917 finished: reduce at LDAOptimizer.scala:456, took 0.011935 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1104.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 918 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1105(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1105 (PartitionwiseSampledRDD[1256] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=129807, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1215 stored as values in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=131959, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1215_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1215_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1215 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1105 (PartitionwiseSampledRDD[1256] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1105.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1105.0 (TID 2770, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1105.0 (TID 2770) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1105.0 (TID 2770). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1105.0 (TID 2770) in 3 ms on localhost (1/1) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1105 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1105.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 918 finished: isEmpty at LDAOptimizer.scala:418, took 0.007653 s -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=133276, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1216 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=133452, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1216_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1216_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1216 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 919 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1106(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1106 (MapPartitionsRDD[1258] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=133732, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1217 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=137324, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1217_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1217_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1217 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1106 (MapPartitionsRDD[1258] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1106.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1106.0 (TID 2771, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1106.0 (TID 2772, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1106.0 (TID 2773, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1106.0 (TID 2774, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1106.0 (TID 2771) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1106.0 (TID 2774) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1106.0 (TID 2772) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1106.0 (TID 2772). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1106.0 (TID 2774). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1106.0 (TID 2773) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1106.0 (TID 2772) in 5 ms on localhost (1/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1106.0 (TID 2771). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1106.0 (TID 2774) in 5 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1106.0 (TID 2771) in 6 ms on localhost (3/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1106.0 (TID 2773). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1106.0 (TID 2773) in 7 ms on localhost (4/4) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1106.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1106 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 919 finished: reduce at LDAOptimizer.scala:453, took 0.014171 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 920 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1107(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1107 (MapPartitionsRDD[1259] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=139410, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1218 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=143042, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1218_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1218_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1218 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1107 (MapPartitionsRDD[1259] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1107.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1107.0 (TID 2775, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1107.0 (TID 2776, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1107.0 (TID 2777, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1107.0 (TID 2778, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1107.0 (TID 2777) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1107.0 (TID 2775) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1107.0 (TID 2778) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1107.0 (TID 2776) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1107.0 (TID 2777). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1107.0 (TID 2778). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1107.0 (TID 2775). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1107.0 (TID 2777) in 4 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1107.0 (TID 2778) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1107.0 (TID 2776). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1107.0 (TID 2775) in 6 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1107.0 (TID 2776) in 5 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1107 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 920 finished: reduce at LDAOptimizer.scala:456, took 0.012427 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1107.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 921 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1108(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1108 (PartitionwiseSampledRDD[1260] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=145128, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1219 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=147280, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1219_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1219_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1219 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1108 (PartitionwiseSampledRDD[1260] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1108.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1108.0 (TID 2779, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1108.0 (TID 2779) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1108.0 (TID 2779). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1108.0 (TID 2779) in 2 ms on localhost (1/1) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1108 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1108.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 921 finished: isEmpty at LDAOptimizer.scala:418, took 0.007844 s -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=148595, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1220 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=148771, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1220_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1220_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1220 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 922 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1109(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1109 (MapPartitionsRDD[1262] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=149051, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1221 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=152643, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1221_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1221_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1221 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1109 (MapPartitionsRDD[1262] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1109.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1109.0 (TID 2780, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1109.0 (TID 2781, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1109.0 (TID 2782, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1109.0 (TID 2783, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1109.0 (TID 2780) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1109.0 (TID 2782) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1109.0 (TID 2781) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1109.0 (TID 2783) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1109.0 (TID 2781). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1109.0 (TID 2780). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1109.0 (TID 2783). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1109.0 (TID 2781) in 4 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1109.0 (TID 2780) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1109.0 (TID 2782). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1109.0 (TID 2782) in 5 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1109.0 (TID 2783) in 4 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1109 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1109.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 922 finished: reduce at LDAOptimizer.scala:453, took 0.009701 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 923 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1110(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1110 (MapPartitionsRDD[1263] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=154727, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1222 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=158359, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1222_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1222_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1222 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1110 (MapPartitionsRDD[1263] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1110.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1110.0 (TID 2784, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1110.0 (TID 2785, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1110.0 (TID 2786, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1110.0 (TID 2787, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1110.0 (TID 2784) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1110.0 (TID 2787) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1110.0 (TID 2785) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1110.0 (TID 2786) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1110.0 (TID 2784). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1110.0 (TID 2787). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1110.0 (TID 2786). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1110.0 (TID 2784) in 4 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1110.0 (TID 2787) in 3 ms on localhost (2/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1110.0 (TID 2785). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1110.0 (TID 2786) in 4 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1110.0 (TID 2785) in 5 ms on localhost (4/4) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1110.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1110 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 923 finished: reduce at LDAOptimizer.scala:456, took 0.009365 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 924 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1111(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1111 (PartitionwiseSampledRDD[1264] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=160444, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1223 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=162596, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1223_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1223_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1223 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1111 (PartitionwiseSampledRDD[1264] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1111.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1111.0 (TID 2788, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1111.0 (TID 2788) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1111.0 (TID 2788). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1111.0 (TID 2788) in 2 ms on localhost (1/1) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1111 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1111.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 924 finished: isEmpty at LDAOptimizer.scala:418, took 0.006852 s -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=163911, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1224 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=164087, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1224_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1224_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1224 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 925 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1112(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1112 (MapPartitionsRDD[1266] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=164367, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1225 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=167959, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1225_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1225_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1225 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1112 (MapPartitionsRDD[1266] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1112.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1112.0 (TID 2789, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1112.0 (TID 2790, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1112.0 (TID 2791, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1112.0 (TID 2792, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1112.0 (TID 2791) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1112.0 (TID 2792) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1112.0 (TID 2789) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1112.0 (TID 2790) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1112.0 (TID 2791). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1112.0 (TID 2792). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1112.0 (TID 2790). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1112.0 (TID 2791) in 3 ms on localhost (1/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1112.0 (TID 2789). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1112.0 (TID 2792) in 3 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1112.0 (TID 2790) in 3 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1112.0 (TID 2789) in 4 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1112 (reduce at LDAOptimizer.scala:453) finished in 0.004 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1112.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 925 finished: reduce at LDAOptimizer.scala:453, took 0.008525 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 926 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1113(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1113 (MapPartitionsRDD[1267] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=170043, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1226 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=173675, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1226_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1226_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1226 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1113 (MapPartitionsRDD[1267] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1113.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1113.0 (TID 2793, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1113.0 (TID 2794, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1113.0 (TID 2795, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1113.0 (TID 2796, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1113.0 (TID 2794) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1113.0 (TID 2795) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1113.0 (TID 2796) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1113.0 (TID 2793) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1113.0 (TID 2795). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1113.0 (TID 2796). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1113.0 (TID 2794). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1113.0 (TID 2793). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1113.0 (TID 2795) in 4 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1113.0 (TID 2796) in 5 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1113.0 (TID 2793) in 6 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1113.0 (TID 2794) in 5 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1113 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1113.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 926 finished: reduce at LDAOptimizer.scala:456, took 0.011166 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 927 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1114(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1114 (PartitionwiseSampledRDD[1268] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=175759, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1227 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=177911, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1227_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1227_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1227 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1114 (PartitionwiseSampledRDD[1268] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1114.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1114.0 (TID 2797, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1114.0 (TID 2797) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1114.0 (TID 2797). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1114.0 (TID 2797) in 2 ms on localhost (1/1) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1114 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1114.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 927 finished: isEmpty at LDAOptimizer.scala:418, took 0.006940 s -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=179228, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1228 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=179404, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1228_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1228_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1228 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 928 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1115(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1115 (MapPartitionsRDD[1270] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=179684, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1229 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=183276, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1229_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1229_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1229 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1115 (MapPartitionsRDD[1270] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1115.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1115.0 (TID 2798, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1115.0 (TID 2799, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1115.0 (TID 2800, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1115.0 (TID 2801, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1115.0 (TID 2798) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1115.0 (TID 2799) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1115.0 (TID 2801) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1115.0 (TID 2800) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1115.0 (TID 2799). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1115.0 (TID 2798). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1115.0 (TID 2799) in 5 ms on localhost (1/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1115.0 (TID 2801). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1115.0 (TID 2798) in 6 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1115.0 (TID 2801) in 5 ms on localhost (3/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1115.0 (TID 2800). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1115.0 (TID 2800) in 7 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1115 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1115.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 928 finished: reduce at LDAOptimizer.scala:453, took 0.016262 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 929 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1116(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1116 (MapPartitionsRDD[1271] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=185361, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1230 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=188993, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1230_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1230_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1230 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1116 (MapPartitionsRDD[1271] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1116.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1116.0 (TID 2802, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1116.0 (TID 2803, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1116.0 (TID 2804, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1116.0 (TID 2805, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1116.0 (TID 2804) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1116.0 (TID 2805) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1116.0 (TID 2803) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1116.0 (TID 2802) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1116.0 (TID 2802). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1116.0 (TID 2804). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1116.0 (TID 2805). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1116.0 (TID 2802) in 4 ms on localhost (1/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1116.0 (TID 2803). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1116.0 (TID 2804) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1116.0 (TID 2805) in 4 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1116.0 (TID 2803) in 5 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1116 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1116.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 929 finished: reduce at LDAOptimizer.scala:456, took 0.009872 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 930 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1117(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1117 (PartitionwiseSampledRDD[1272] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=191078, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1231 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=193230, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1231_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1231_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1231 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1117 (PartitionwiseSampledRDD[1272] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1117.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1117.0 (TID 2806, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1117.0 (TID 2806) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1117.0 (TID 2806). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1117.0 (TID 2806) in 2 ms on localhost (1/1) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1117 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1117.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 930 finished: isEmpty at LDAOptimizer.scala:418, took 0.005872 s -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=194545, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1232 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=194721, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1232_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1232_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1232 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 931 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1118(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1118 (MapPartitionsRDD[1274] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=195001, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1233 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=198593, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1233_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1233_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1233 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1118 (MapPartitionsRDD[1274] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1118.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1118.0 (TID 2807, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1118.0 (TID 2808, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1118.0 (TID 2809, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1118.0 (TID 2810, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1118.0 (TID 2807) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1118.0 (TID 2810) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1118.0 (TID 2809) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1118.0 (TID 2808) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1118.0 (TID 2809). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1118.0 (TID 2808). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1118.0 (TID 2807). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1118.0 (TID 2809) in 3 ms on localhost (1/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1118.0 (TID 2810). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1118.0 (TID 2808) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1118.0 (TID 2810) in 4 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1118.0 (TID 2807) in 4 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1118 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1118.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 931 finished: reduce at LDAOptimizer.scala:453, took 0.008915 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 932 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1119(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1119 (MapPartitionsRDD[1275] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=200677, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1234 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=204309, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1234_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1234_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1234 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1119 (MapPartitionsRDD[1275] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1119.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1119.0 (TID 2811, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1119.0 (TID 2812, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1119.0 (TID 2813, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1119.0 (TID 2814, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1119.0 (TID 2812) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1119.0 (TID 2814) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1119.0 (TID 2813) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1119.0 (TID 2811) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1119.0 (TID 2814). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1119.0 (TID 2812). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1119.0 (TID 2813). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1119.0 (TID 2811). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1119.0 (TID 2814) in 3 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1119.0 (TID 2812) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1119.0 (TID 2811) in 5 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1119.0 (TID 2813) in 4 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1119 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1119.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 932 finished: reduce at LDAOptimizer.scala:456, took 0.009442 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 933 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1120(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1120 (PartitionwiseSampledRDD[1276] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=206393, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1235 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=208545, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1235_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1235_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1235 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1120 (PartitionwiseSampledRDD[1276] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1120.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1120.0 (TID 2815, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1120.0 (TID 2815) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1120.0 (TID 2815). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1120.0 (TID 2815) in 2 ms on localhost (1/1) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1120 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1120.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 933 finished: isEmpty at LDAOptimizer.scala:418, took 0.006974 s -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=209860, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1236 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=210036, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1236_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1236_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1236 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 934 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1121(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1121 (MapPartitionsRDD[1278] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=210316, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1237 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=213908, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1237_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1237_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1237 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1121 (MapPartitionsRDD[1278] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1121.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1121.0 (TID 2816, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1121.0 (TID 2817, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1121.0 (TID 2818, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1121.0 (TID 2819, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1121.0 (TID 2817) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1121.0 (TID 2819) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1121.0 (TID 2818) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1121.0 (TID 2816) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1121.0 (TID 2818). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1121.0 (TID 2817). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1121.0 (TID 2816). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1121.0 (TID 2818) in 3 ms on localhost (1/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1121.0 (TID 2819). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1121.0 (TID 2817) in 3 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1121.0 (TID 2816) in 4 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1121.0 (TID 2819) in 4 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1121 (reduce at LDAOptimizer.scala:453) finished in 0.004 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1121.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 934 finished: reduce at LDAOptimizer.scala:453, took 0.008994 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 935 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1122(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1122 (MapPartitionsRDD[1279] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=215992, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1238 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=219624, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1238_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1238_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1238 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1122 (MapPartitionsRDD[1279] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1122.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1122.0 (TID 2820, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1122.0 (TID 2821, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1122.0 (TID 2822, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1122.0 (TID 2823, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1122.0 (TID 2820) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1122.0 (TID 2823) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1122.0 (TID 2822) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1122.0 (TID 2821) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1122.0 (TID 2820). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1122.0 (TID 2821). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1122.0 (TID 2822). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1122.0 (TID 2823). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1122.0 (TID 2820) in 3 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1122.0 (TID 2822) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1122.0 (TID 2821) in 4 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1122.0 (TID 2823) in 4 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1122 (reduce at LDAOptimizer.scala:456) finished in 0.004 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1122.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 935 finished: reduce at LDAOptimizer.scala:456, took 0.009891 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 936 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1123(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1123 (PartitionwiseSampledRDD[1280] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=221708, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1239 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=223860, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1239_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1239_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1239 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1123 (PartitionwiseSampledRDD[1280] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1123.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1123.0 (TID 2824, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1123.0 (TID 2824) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1123.0 (TID 2824). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1123.0 (TID 2824) in 2 ms on localhost (1/1) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1123 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1123.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 936 finished: isEmpty at LDAOptimizer.scala:418, took 0.005915 s -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=225175, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1240 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=225351, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1240_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1240_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1240 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 937 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1124(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1124 (MapPartitionsRDD[1282] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=225631, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1241 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=229223, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1241_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1241_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1241 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1124 (MapPartitionsRDD[1282] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1124.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1124.0 (TID 2825, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1124.0 (TID 2826, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1124.0 (TID 2827, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1124.0 (TID 2828, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1124.0 (TID 2825) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1124.0 (TID 2827) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1124.0 (TID 2826) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1124.0 (TID 2828) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1124.0 (TID 2827). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1124.0 (TID 2828). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1124.0 (TID 2825). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1124.0 (TID 2826). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1124.0 (TID 2827) in 3 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1124.0 (TID 2825) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1124.0 (TID 2828) in 4 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1124.0 (TID 2826) in 4 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1124 (reduce at LDAOptimizer.scala:453) finished in 0.004 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1124.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 937 finished: reduce at LDAOptimizer.scala:453, took 0.009306 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 938 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1125(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1125 (MapPartitionsRDD[1283] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=231309, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1242 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=234941, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1242_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1242_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1242 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1125 (MapPartitionsRDD[1283] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1125.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1125.0 (TID 2829, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1125.0 (TID 2830, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1125.0 (TID 2831, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1125.0 (TID 2832, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1125.0 (TID 2831) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1125.0 (TID 2832) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1125.0 (TID 2830) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1125.0 (TID 2829) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1125.0 (TID 2832). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1125.0 (TID 2830). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1125.0 (TID 2829). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1125.0 (TID 2831). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1125.0 (TID 2832) in 5 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1125.0 (TID 2830) in 5 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1125.0 (TID 2829) in 6 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1125.0 (TID 2831) in 5 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1125 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 938 finished: reduce at LDAOptimizer.scala:456, took 0.010643 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1125.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 939 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1126(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1126 (PartitionwiseSampledRDD[1284] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=237027, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1243 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1316) called with curMem=239179, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1243_piece0 stored as bytes in memory (estimated size 1316.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1243_piece0 in memory on localhost:50000 (size: 1316.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1243 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1126 (PartitionwiseSampledRDD[1284] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1126.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1126.0 (TID 2833, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1126.0 (TID 2833) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1126.0 (TID 2833). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1126.0 (TID 2833) in 2 ms on localhost (1/1) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1126 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1126.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 939 finished: isEmpty at LDAOptimizer.scala:418, took 0.006166 s -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=240495, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1244 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=240671, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1244_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1244_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1244 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 940 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1127(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1127 (MapPartitionsRDD[1286] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=240951, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1245 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2090) called with curMem=244543, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1245_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1245_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1245 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1127 (MapPartitionsRDD[1286] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1127.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1127.0 (TID 2834, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1127.0 (TID 2835, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1127.0 (TID 2836, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1127.0 (TID 2837, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1127.0 (TID 2834) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1127.0 (TID 2836) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1127.0 (TID 2835) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1127.0 (TID 2837) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1127.0 (TID 2835). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1127.0 (TID 2834). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1127.0 (TID 2837). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1127.0 (TID 2836). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1127.0 (TID 2835) in 5 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1127.0 (TID 2834) in 7 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1127.0 (TID 2836) in 8 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1127.0 (TID 2837) in 7 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1127 (reduce at LDAOptimizer.scala:453) finished in 0.008 s -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 940 finished: reduce at LDAOptimizer.scala:453, took 0.012029 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1127.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 941 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1128(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1128 (MapPartitionsRDD[1287] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=246633, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1246 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2090) called with curMem=250265, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1246_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1246_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1246 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1128 (MapPartitionsRDD[1287] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1128.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1128.0 (TID 2838, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1128.0 (TID 2839, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1128.0 (TID 2840, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1128.0 (TID 2841, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1128.0 (TID 2838) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1128.0 (TID 2841) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1128.0 (TID 2840) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1128.0 (TID 2839) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1128.0 (TID 2838). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1128.0 (TID 2841). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1128.0 (TID 2839). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1128.0 (TID 2841) in 3 ms on localhost (1/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1128.0 (TID 2840). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1128.0 (TID 2839) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1128.0 (TID 2838) in 5 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1128.0 (TID 2840) in 4 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1128 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1128.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 941 finished: reduce at LDAOptimizer.scala:456, took 0.009856 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 942 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1129(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1129 (PartitionwiseSampledRDD[1288] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=252355, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1247 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=254507, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1247_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1247_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1247 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1129 (PartitionwiseSampledRDD[1288] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1129.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1129.0 (TID 2842, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1129.0 (TID 2842) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1129.0 (TID 2842). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1129.0 (TID 2842) in 3 ms on localhost (1/1) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1129 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1129.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 942 finished: isEmpty at LDAOptimizer.scala:418, took 0.007056 s -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=255822, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1248 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=255998, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1248_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1248_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1248 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 943 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1130(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1130 (MapPartitionsRDD[1290] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=256278, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1249 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=259870, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1249_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1249_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1249 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1130 (MapPartitionsRDD[1290] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1130.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1130.0 (TID 2843, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1130.0 (TID 2844, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1130.0 (TID 2845, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1130.0 (TID 2846, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1130.0 (TID 2843) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1130.0 (TID 2846) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1130.0 (TID 2844) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1130.0 (TID 2845) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1130.0 (TID 2845). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1130.0 (TID 2846). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1130.0 (TID 2844). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1130.0 (TID 2843). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1130.0 (TID 2845) in 4 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1130.0 (TID 2846) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1130.0 (TID 2844) in 5 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1130.0 (TID 2843) in 5 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1130 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1130.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 943 finished: reduce at LDAOptimizer.scala:453, took 0.009500 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 944 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1131(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1131 (MapPartitionsRDD[1291] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=261957, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1250 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=265589, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1250_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1250_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1250 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1131 (MapPartitionsRDD[1291] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1131.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1131.0 (TID 2847, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1131.0 (TID 2848, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1131.0 (TID 2849, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1131.0 (TID 2850, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1131.0 (TID 2848) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1131.0 (TID 2850) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1131.0 (TID 2849) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1131.0 (TID 2847) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1131.0 (TID 2847). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1131.0 (TID 2848). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1131.0 (TID 2849). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1131.0 (TID 2847) in 4 ms on localhost (1/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1131.0 (TID 2850). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1131.0 (TID 2849) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1131.0 (TID 2848) in 4 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1131.0 (TID 2850) in 4 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1131 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1131.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 944 finished: reduce at LDAOptimizer.scala:456, took 0.009699 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 945 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1132(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1132 (PartitionwiseSampledRDD[1292] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=267676, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1251 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=269828, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1251_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1251_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1251 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1132 (PartitionwiseSampledRDD[1292] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1132.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1132.0 (TID 2851, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1132.0 (TID 2851) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1132.0 (TID 2851). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1132.0 (TID 2851) in 2 ms on localhost (1/1) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1132 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1132.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 945 finished: isEmpty at LDAOptimizer.scala:418, took 0.005971 s -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=271145, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1252 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=271321, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1252_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1252_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1252 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 946 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1133(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1133 (MapPartitionsRDD[1294] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=271601, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1253 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=275193, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1253_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1253_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1253 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1133 (MapPartitionsRDD[1294] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1133.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1133.0 (TID 2852, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1133.0 (TID 2853, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1133.0 (TID 2854, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1133.0 (TID 2855, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1133.0 (TID 2852) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1133.0 (TID 2853) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1133.0 (TID 2854) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1133.0 (TID 2855) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1133.0 (TID 2854). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1133.0 (TID 2852). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1133.0 (TID 2855). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1133.0 (TID 2854) in 4 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1133.0 (TID 2852) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1133.0 (TID 2855) in 4 ms on localhost (3/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1133.0 (TID 2853). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1133.0 (TID 2853) in 5 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1133 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1133.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 946 finished: reduce at LDAOptimizer.scala:453, took 0.010900 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 947 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1134(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1134 (MapPartitionsRDD[1295] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=277278, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1254 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=280910, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1254_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1254_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1254 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1134 (MapPartitionsRDD[1295] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1134.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1134.0 (TID 2856, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1134.0 (TID 2857, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1134.0 (TID 2858, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1134.0 (TID 2859, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1134.0 (TID 2857) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1134.0 (TID 2859) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1134.0 (TID 2856) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1134.0 (TID 2858) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1134.0 (TID 2857). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1134.0 (TID 2859). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1134.0 (TID 2857) in 4 ms on localhost (1/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1134.0 (TID 2856). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1134.0 (TID 2858). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1134.0 (TID 2859) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1134.0 (TID 2856) in 5 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1134.0 (TID 2858) in 4 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1134 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1134.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 947 finished: reduce at LDAOptimizer.scala:456, took 0.009889 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 948 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1135(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1135 (PartitionwiseSampledRDD[1296] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=282995, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1255 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=285147, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1255_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1255_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1255 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1135 (PartitionwiseSampledRDD[1296] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1135.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1135.0 (TID 2860, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1135.0 (TID 2860) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1135.0 (TID 2860). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1135.0 (TID 2860) in 3 ms on localhost (1/1) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1135.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1135 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 948 finished: isEmpty at LDAOptimizer.scala:418, took 0.009686 s -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=286464, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1256 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=286640, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1256_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1256_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1256 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 949 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1136(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1136 (MapPartitionsRDD[1298] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=286920, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1257 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=290512, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1257_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1257_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1257 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1136 (MapPartitionsRDD[1298] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1136.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1136.0 (TID 2861, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1136.0 (TID 2862, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1136.0 (TID 2863, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1136.0 (TID 2864, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1136.0 (TID 2861) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1136.0 (TID 2864) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1136.0 (TID 2861). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1136.0 (TID 2863) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1136.0 (TID 2864). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1136.0 (TID 2862) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1136.0 (TID 2862). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1136.0 (TID 2861) in 11 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1136.0 (TID 2862) in 11 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1136.0 (TID 2864) in 10 ms on localhost (3/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1136.0 (TID 2863). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1136.0 (TID 2863) in 11 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1136 (reduce at LDAOptimizer.scala:453) finished in 0.012 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1136.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 949 finished: reduce at LDAOptimizer.scala:453, took 0.019541 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 950 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1137(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1137 (MapPartitionsRDD[1299] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=292598, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1258 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=296230, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1258_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1258_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1258 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1137 (MapPartitionsRDD[1299] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1137.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1137.0 (TID 2865, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1137.0 (TID 2866, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1137.0 (TID 2867, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1137.0 (TID 2868, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1137.0 (TID 2865) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1137.0 (TID 2867) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1137.0 (TID 2866) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1137.0 (TID 2868) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1137.0 (TID 2865). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1137.0 (TID 2866). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1137.0 (TID 2867). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1137.0 (TID 2865) in 5 ms on localhost (1/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1137.0 (TID 2868). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1137.0 (TID 2867) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1137.0 (TID 2866) in 5 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1137.0 (TID 2868) in 5 ms on localhost (4/4) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1137.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1137 (reduce at LDAOptimizer.scala:456) finished in 0.006 s -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 950 finished: reduce at LDAOptimizer.scala:456, took 0.011175 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 951 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1138(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1138 (PartitionwiseSampledRDD[1300] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=298316, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1259 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=300468, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1259_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1259_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1259 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1138 (PartitionwiseSampledRDD[1300] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1138.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1138.0 (TID 2869, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1138.0 (TID 2869) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1138.0 (TID 2869). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1138.0 (TID 2869) in 3 ms on localhost (1/1) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1138.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1138 (isEmpty at LDAOptimizer.scala:418) finished in 0.004 s -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 951 finished: isEmpty at LDAOptimizer.scala:418, took 0.010821 s -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=301783, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1260 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=301959, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1260_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1260_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1260 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 952 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1139(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1139 (MapPartitionsRDD[1302] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=302239, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1261 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=305831, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1261_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1261_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1261 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1139 (MapPartitionsRDD[1302] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1139.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1139.0 (TID 2870, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1139.0 (TID 2871, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1139.0 (TID 2872, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1139.0 (TID 2873, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1139.0 (TID 2871) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1139.0 (TID 2873) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1139.0 (TID 2872) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1139.0 (TID 2870) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1139.0 (TID 2870). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1139.0 (TID 2872). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1139.0 (TID 2871). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1139.0 (TID 2873). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1139.0 (TID 2870) in 3 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1139.0 (TID 2872) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1139.0 (TID 2871) in 4 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1139.0 (TID 2873) in 3 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1139 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1139.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 952 finished: reduce at LDAOptimizer.scala:453, took 0.009478 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 953 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1140(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1140 (MapPartitionsRDD[1303] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=307915, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1262 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=311547, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1262_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1262_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1262 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1140 (MapPartitionsRDD[1303] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1140.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1140.0 (TID 2874, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1140.0 (TID 2875, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1140.0 (TID 2876, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1140.0 (TID 2877, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1140.0 (TID 2875) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1140.0 (TID 2877) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1140.0 (TID 2876) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1140.0 (TID 2874) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1140.0 (TID 2876). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1140.0 (TID 2877). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1140.0 (TID 2875). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1140.0 (TID 2876) in 3 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1140.0 (TID 2877) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1140.0 (TID 2875) in 4 ms on localhost (3/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1140.0 (TID 2874). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1140.0 (TID 2874) in 7 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1140 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1140.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 953 finished: reduce at LDAOptimizer.scala:456, took 0.010998 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 954 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1141(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1141 (PartitionwiseSampledRDD[1304] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=313631, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1263 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=315783, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1263_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1263_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1263 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1141 (PartitionwiseSampledRDD[1304] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1141.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1141.0 (TID 2878, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1141.0 (TID 2878) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1141.0 (TID 2878). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1141.0 (TID 2878) in 2 ms on localhost (1/1) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1141 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1141.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 954 finished: isEmpty at LDAOptimizer.scala:418, took 0.006397 s -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=317100, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1264 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=317276, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1264_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1264_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1264 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 955 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1142(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1142 (MapPartitionsRDD[1306] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=317556, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1265 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=321148, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1265_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1265_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1265 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1142 (MapPartitionsRDD[1306] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1142.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1142.0 (TID 2879, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1142.0 (TID 2880, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1142.0 (TID 2881, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1142.0 (TID 2882, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1142.0 (TID 2881) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1142.0 (TID 2882) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1142.0 (TID 2879) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1142.0 (TID 2880) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1142.0 (TID 2880). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1142.0 (TID 2881). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1142.0 (TID 2879). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1142.0 (TID 2880) in 4 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1142.0 (TID 2881) in 3 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1142.0 (TID 2879) in 4 ms on localhost (3/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1142.0 (TID 2882). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1142.0 (TID 2882) in 4 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1142 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1142.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 955 finished: reduce at LDAOptimizer.scala:453, took 0.009742 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 956 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1143(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1143 (MapPartitionsRDD[1307] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=323233, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1266 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=326865, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1266_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1266_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1266 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1143 (MapPartitionsRDD[1307] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1143.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1143.0 (TID 2883, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1143.0 (TID 2884, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1143.0 (TID 2885, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1143.0 (TID 2886, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1143.0 (TID 2885) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1143.0 (TID 2886) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1143.0 (TID 2883) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1143.0 (TID 2884) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1143.0 (TID 2884). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1143.0 (TID 2883). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1143.0 (TID 2886). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1143.0 (TID 2885). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1143.0 (TID 2884) in 3 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1143.0 (TID 2883) in 5 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1143.0 (TID 2886) in 4 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1143.0 (TID 2885) in 4 ms on localhost (4/4) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1143.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1143 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 956 finished: reduce at LDAOptimizer.scala:456, took 0.009924 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 957 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1144(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1144 (PartitionwiseSampledRDD[1308] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=328950, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1267 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=331102, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1267_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1267_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1267 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1144 (PartitionwiseSampledRDD[1308] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1144.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1144.0 (TID 2887, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1144.0 (TID 2887) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1144.0 (TID 2887). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1144.0 (TID 2887) in 2 ms on localhost (1/1) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1144 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1144.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 957 finished: isEmpty at LDAOptimizer.scala:418, took 0.006459 s -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=332417, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1268 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=332593, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1268_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1268_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1268 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 958 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1145(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1145 (MapPartitionsRDD[1310] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=332873, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1269 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=336465, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1269_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1269_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.4 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1269 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1145 (MapPartitionsRDD[1310] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1145.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1145.0 (TID 2888, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1145.0 (TID 2889, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1145.0 (TID 2890, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1145.0 (TID 2891, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1145.0 (TID 2889) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1145.0 (TID 2891) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1145.0 (TID 2890) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1145.0 (TID 2888) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1145.0 (TID 2891). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1145.0 (TID 2889). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1145.0 (TID 2888). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1145.0 (TID 2891) in 3 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1145.0 (TID 2889) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1145.0 (TID 2890). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1145.0 (TID 2888) in 5 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1145.0 (TID 2890) in 4 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1145 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1145.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 958 finished: reduce at LDAOptimizer.scala:453, took 0.010415 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 959 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1146(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1146 (MapPartitionsRDD[1311] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=338552, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1270 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=342184, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1270_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1270_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.4 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1270 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1146 (MapPartitionsRDD[1311] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1146.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1146.0 (TID 2892, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1146.0 (TID 2893, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1146.0 (TID 2894, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1146.0 (TID 2895, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1146.0 (TID 2894) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1146.0 (TID 2895) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1146.0 (TID 2892) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1146.0 (TID 2893) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1146.0 (TID 2893). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1146.0 (TID 2894). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1146.0 (TID 2895). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1146.0 (TID 2892). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1146.0 (TID 2893) in 4 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1146.0 (TID 2894) in 3 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1146.0 (TID 2895) in 3 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1146.0 (TID 2892) in 4 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1146 (reduce at LDAOptimizer.scala:456) finished in 0.004 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1146.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 959 finished: reduce at LDAOptimizer.scala:456, took 0.009378 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 960 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1147(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1147 (PartitionwiseSampledRDD[1312] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=344271, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1271 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=346423, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1271_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1271_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.4 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1271 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1147 (PartitionwiseSampledRDD[1312] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1147.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1147.0 (TID 2896, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1147.0 (TID 2896) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1147.0 (TID 2896). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1147.0 (TID 2896) in 2 ms on localhost (1/1) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1147 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1147.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 960 finished: isEmpty at LDAOptimizer.scala:418, took 0.007135 s -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=347740, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1272 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=347916, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1272_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1272_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.4 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1272 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1271_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.4 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 981 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1270_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.4 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 980 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 961 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1148(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1148 (MapPartitionsRDD[1314] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1269_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 979 -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=333329, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1273 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=336465, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1273_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1268_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1273_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.4 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1273 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1148 (MapPartitionsRDD[1314] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1148.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1148.0 (TID 2897, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1267_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1148.0 (TID 2898, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 978 -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1148.0 (TID 2899, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1148.0 (TID 2900, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1148.0 (TID 2900) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1148.0 (TID 2899) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1148.0 (TID 2898) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1266_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1148.0 (TID 2897) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1148.0 (TID 2900). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1148.0 (TID 2900) in 5 ms on localhost (1/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1148.0 (TID 2897). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 977 -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1148.0 (TID 2897) in 7 ms on localhost (2/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1148.0 (TID 2898). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1148.0 (TID 2898) in 7 ms on localhost (3/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1148.0 (TID 2899). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1265_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1148.0 (TID 2899) in 8 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1148 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1148.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 961 finished: reduce at LDAOptimizer.scala:453, took 0.014566 s -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 976 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1264_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 962 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1149(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1149 (MapPartitionsRDD[1315] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1263_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 975 -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=316132, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1274 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1262_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2085) called with curMem=317680, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1274_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 974 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1274_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1274 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1149 (MapPartitionsRDD[1315] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1261_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1149.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1149.0 (TID 2901, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 973 -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1149.0 (TID 2902, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1149.0 (TID 2903, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1149.0 (TID 2904, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1149.0 (TID 2902) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1149.0 (TID 2904) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1260_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1149.0 (TID 2901) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1149.0 (TID 2903) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1149.0 (TID 2902). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1149.0 (TID 2904). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1149.0 (TID 2903). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1149.0 (TID 2902) in 6 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1149.0 (TID 2903) in 5 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1149.0 (TID 2904) in 6 ms on localhost (3/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1149.0 (TID 2901). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1149.0 (TID 2901) in 7 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1149 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1149.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 962 finished: reduce at LDAOptimizer.scala:456, took 0.013269 s -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1259_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 972 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1258_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 971 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 963 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1150(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1150 (PartitionwiseSampledRDD[1316] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1257_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 970 -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=298770, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1275 stored as values in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1256_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=300642, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1275_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1275_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1275 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1150 (PartitionwiseSampledRDD[1316] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1150.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1150.0 (TID 2905, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1255_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1150.0 (TID 2905) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 969 -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1150.0 (TID 2905). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1150.0 (TID 2905) in 3 ms on localhost (1/1) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1150 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1150.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1254_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 963 finished: isEmpty at LDAOptimizer.scala:418, took 0.007170 s -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 968 -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=292595, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1276 stored as values in memory (estimated size 176.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1253_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=287094, maxMem=766075207 -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 967 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1276_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1276_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1252_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1276 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1251_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 966 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1250_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 964 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 965 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1151(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1151 (MapPartitionsRDD[1318] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=272051, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1277 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1249_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 964 -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2080) called with curMem=275363, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1277_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1277_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1248_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1277 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1151 (MapPartitionsRDD[1318] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1151.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1151.0 (TID 2906, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1151.0 (TID 2907, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1151.0 (TID 2908, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1151.0 (TID 2909, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1247_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1151.0 (TID 2909) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1151.0 (TID 2906) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1151.0 (TID 2907) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1151.0 (TID 2908) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 963 -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1151.0 (TID 2909). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1151.0 (TID 2908). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1151.0 (TID 2909) in 3 ms on localhost (1/4) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1246_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 962 -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1151.0 (TID 2906). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1151.0 (TID 2908) in 5 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1151.0 (TID 2906) in 7 ms on localhost (3/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1151.0 (TID 2907). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1151.0 (TID 2907) in 8 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1151 (reduce at LDAOptimizer.scala:453) finished in 0.009 s -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1245_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 964 finished: reduce at LDAOptimizer.scala:453, took 0.013201 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1151.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 961 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1244_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 965 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1152(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1243_piece0 on localhost:50000 in memory (size: 1316.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1152 (MapPartitionsRDD[1319] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 960 -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=258472, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1278 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2080) called with curMem=256386, maxMem=766075207 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1242_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1278_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 959 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1278_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1278 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1152 (MapPartitionsRDD[1319] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1152.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1152.0 (TID 2910, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1152.0 (TID 2911, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1241_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1152.0 (TID 2912, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 958 -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1152.0 (TID 2913, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1152.0 (TID 2912) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1152.0 (TID 2910) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1152.0 (TID 2911) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1240_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1152.0 (TID 2910). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1152.0 (TID 2910) in 5 ms on localhost (1/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1152.0 (TID 2911). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1152.0 (TID 2912). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1152.0 (TID 2911) in 5 ms on localhost (2/4) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1152.0 (TID 2913) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1152.0 (TID 2912) in 5 ms on localhost (3/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1152.0 (TID 2913). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1239_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1152.0 (TID 2913) in 6 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1152 (reduce at LDAOptimizer.scala:456) finished in 0.009 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1152.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 965 finished: reduce at LDAOptimizer.scala:456, took 0.013884 s -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 957 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1238_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 956 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1237_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 955 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 966 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1153(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1153 (PartitionwiseSampledRDD[1320] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=237017, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1279 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1236_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=239169, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1279_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1279_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1279 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1153 (PartitionwiseSampledRDD[1320] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1153.0 with 1 tasks -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1235_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 954 -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1153.0 (TID 2914, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1153.0 (TID 2914) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1153.0 (TID 2914). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1234_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 953 -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1153.0 (TID 2914) in 2 ms on localhost (1/1) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1153 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1153.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 966 finished: isEmpty at LDAOptimizer.scala:418, took 0.008370 s -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=229219, maxMem=766075207 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1233_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1280 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 952 -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=225803, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1280_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1280_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1232_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1280 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1231_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 951 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1230_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 967 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 950 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1154(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1154 (MapPartitionsRDD[1322] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=214358, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1281 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1229_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 949 -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=214358, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1281_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1281_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1228_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1281 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1154 (MapPartitionsRDD[1322] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1154.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1154.0 (TID 2915, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1154.0 (TID 2916, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1154.0 (TID 2917, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1154.0 (TID 2918, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1154.0 (TID 2916) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1154.0 (TID 2917) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1154.0 (TID 2915) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1154.0 (TID 2918) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1227_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 948 -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1154.0 (TID 2917). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1154.0 (TID 2916). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1154.0 (TID 2917) in 4 ms on localhost (1/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1154.0 (TID 2915). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1226_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1154.0 (TID 2916) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1154.0 (TID 2918). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1154.0 (TID 2915) in 6 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1154.0 (TID 2918) in 5 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1154 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1154.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 947 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 967 finished: reduce at LDAOptimizer.scala:453, took 0.012042 s -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1225_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 946 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 968 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1155(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1155 (MapPartitionsRDD[1323] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1224_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=200671, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1282 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=200836, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1282_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1223_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 945 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1282_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1282 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1155 (MapPartitionsRDD[1323] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1155.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1155.0 (TID 2919, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1155.0 (TID 2920, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1155.0 (TID 2921, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1222_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1155.0 (TID 2922, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 944 -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1155.0 (TID 2919) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1155.0 (TID 2920) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1155.0 (TID 2921) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1155.0 (TID 2922) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1155.0 (TID 2919). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1155.0 (TID 2919) in 5 ms on localhost (1/4) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1221_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1155.0 (TID 2921). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1155.0 (TID 2920). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1155.0 (TID 2922). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1155.0 (TID 2921) in 5 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1155.0 (TID 2920) in 6 ms on localhost (3/4) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 943 -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1155.0 (TID 2922) in 6 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1155 (reduce at LDAOptimizer.scala:456) finished in 0.007 s -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 968 finished: reduce at LDAOptimizer.scala:456, took 0.011805 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1155.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1220_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1219_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 942 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 969 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1156(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1156 (PartitionwiseSampledRDD[1324] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1218_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=185520, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1283 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 941 -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=181954, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1283_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1217_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1283_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1283 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1156 (PartitionwiseSampledRDD[1324] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 940 -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1156.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1156.0 (TID 2923, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1156.0 (TID 2923) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1216_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1156.0 (TID 2923). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1156.0 (TID 2923) in 2 ms on localhost (1/1) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1156.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1156 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 969 finished: isEmpty at LDAOptimizer.scala:418, took 0.007737 s -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=175752, maxMem=766075207 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1215_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1284 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 939 -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=170212, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1284_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1214_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 938 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1284_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1284 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1213_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 937 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1212_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 970 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1157(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1157 (MapPartitionsRDD[1326] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1211_piece0 on localhost:50000 in memory (size: 1316.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 936 -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=160892, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1285 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1210_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=162400, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1285_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 935 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1285_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1285 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1157 (MapPartitionsRDD[1326] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1157.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1157.0 (TID 2924, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1157.0 (TID 2925, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1209_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1157.0 (TID 2926, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 934 -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1157.0 (TID 2927, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1157.0 (TID 2925) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1157.0 (TID 2926) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1157.0 (TID 2924) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1157.0 (TID 2927) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1208_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1157.0 (TID 2926). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1157.0 (TID 2926) in 3 ms on localhost (1/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1157.0 (TID 2924). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1157.0 (TID 2927). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1157.0 (TID 2924) in 6 ms on localhost (2/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1157.0 (TID 2925). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1157.0 (TID 2927) in 6 ms on localhost (3/4) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1207_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1157.0 (TID 2925) in 6 ms on localhost (4/4) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1157.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1157 (reduce at LDAOptimizer.scala:453) finished in 0.007 s -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 970 finished: reduce at LDAOptimizer.scala:453, took 0.012371 s -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 933 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1206_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 932 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 971 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1158(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1158 (MapPartitionsRDD[1327] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1205_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=139861, maxMem=766075207 -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 931 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1286 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=143213, maxMem=766075207 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1204_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1286_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1286_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1286 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1158 (MapPartitionsRDD[1327] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1158.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1158.0 (TID 2928, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1158.0 (TID 2929, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1158.0 (TID 2930, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1158.0 (TID 2931, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1158.0 (TID 2928) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1158.0 (TID 2929) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1158.0 (TID 2930) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1158.0 (TID 2931) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1203_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1158.0 (TID 2929). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1158.0 (TID 2928). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 930 -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1158.0 (TID 2930). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1158.0 (TID 2929) in 6 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1158.0 (TID 2928) in 6 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1158.0 (TID 2930) in 6 ms on localhost (3/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1158.0 (TID 2931). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1158.0 (TID 2931) in 7 ms on localhost (4/4) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1158.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1158 (reduce at LDAOptimizer.scala:456) finished in 0.009 s -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 971 finished: reduce at LDAOptimizer.scala:456, took 0.015755 s -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1202_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 929 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1201_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 928 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 972 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1159(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1159 (PartitionwiseSampledRDD[1328] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=129804, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1287 stored as values in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1200_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=131956, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1287_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1287_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1287 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1159 (PartitionwiseSampledRDD[1328] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1159.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1159.0 (TID 2932, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1159.0 (TID 2932) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1199_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 927 -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1159.0 (TID 2932). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1159.0 (TID 2932) in 3 ms on localhost (1/1) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1159 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1159.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 972 finished: isEmpty at LDAOptimizer.scala:418, took 0.008957 s -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1198_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=124086, maxMem=766075207 -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 926 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1288 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=122178, maxMem=766075207 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1197_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1288_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 925 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1288_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1288 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1196_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1195_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 924 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 973 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1160(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1160 (MapPartitionsRDD[1330] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1194_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=109225, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1289 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 923 -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=107139, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1289_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1193_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1289_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1289 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 922 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1160 (MapPartitionsRDD[1330] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1160.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1160.0 (TID 2933, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1160.0 (TID 2934, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1160.0 (TID 2935, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1160.0 (TID 2936, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1192_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1160.0 (TID 2936) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1160.0 (TID 2933) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1160.0 (TID 2934) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1160.0 (TID 2935) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1160.0 (TID 2933). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1160.0 (TID 2936). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1160.0 (TID 2934). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1160.0 (TID 2933) in 4 ms on localhost (1/4) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1191_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1160.0 (TID 2936) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1160.0 (TID 2934) in 5 ms on localhost (3/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1160.0 (TID 2935). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 921 -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1160.0 (TID 2935) in 5 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1160 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1160.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 973 finished: reduce at LDAOptimizer.scala:453, took 0.011271 s -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1190_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 920 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1189_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 974 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1161(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1161 (MapPartitionsRDD[1331] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO ContextCleaner:59 - Cleaned accumulator 919 -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=93726, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1290 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=97078, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1290_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Removed broadcast_1188_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1290_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1290 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1161 (MapPartitionsRDD[1331] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1161.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1161.0 (TID 2937, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1161.0 (TID 2938, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1161.0 (TID 2939, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1161.0 (TID 2940, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1161.0 (TID 2938) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1161.0 (TID 2939) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1161.0 (TID 2940) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1161.0 (TID 2937) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1161.0 (TID 2938). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1161.0 (TID 2939). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1161.0 (TID 2937). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1161.0 (TID 2938) in 3 ms on localhost (1/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1161.0 (TID 2939) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1161.0 (TID 2940). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1161.0 (TID 2937) in 4 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1161.0 (TID 2940) in 3 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1161 (reduce at LDAOptimizer.scala:456) finished in 0.004 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1161.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 974 finished: reduce at LDAOptimizer.scala:456, took 0.009156 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 975 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1162(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1162 (PartitionwiseSampledRDD[1332] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=99164, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1291 stored as values in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=101316, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1291_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1291_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1291 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1162 (PartitionwiseSampledRDD[1332] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1162.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1162.0 (TID 2941, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1162.0 (TID 2941) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1162.0 (TID 2941). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1162.0 (TID 2941) in 3 ms on localhost (1/1) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1162 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1162.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 975 finished: isEmpty at LDAOptimizer.scala:418, took 0.006848 s -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=102631, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1292 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=102807, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1292_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1292_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1292 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 976 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1163(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1163 (MapPartitionsRDD[1334] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=103087, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1293 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=106679, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1293_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1293_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1293 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1163 (MapPartitionsRDD[1334] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1163.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1163.0 (TID 2942, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1163.0 (TID 2943, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1163.0 (TID 2944, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1163.0 (TID 2945, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1163.0 (TID 2943) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1163.0 (TID 2945) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1163.0 (TID 2944) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1163.0 (TID 2942) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1163.0 (TID 2943). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1163.0 (TID 2945). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1163.0 (TID 2942). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1163.0 (TID 2943) in 3 ms on localhost (1/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1163.0 (TID 2944). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1163.0 (TID 2945) in 3 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1163.0 (TID 2942) in 4 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1163.0 (TID 2944) in 4 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1163 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1163.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 976 finished: reduce at LDAOptimizer.scala:453, took 0.009945 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 977 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1164(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1164 (MapPartitionsRDD[1335] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=108763, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1294 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=112395, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1294_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1294_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1294 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1164 (MapPartitionsRDD[1335] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1164.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1164.0 (TID 2946, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1164.0 (TID 2947, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1164.0 (TID 2948, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1164.0 (TID 2949, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1164.0 (TID 2947) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1164.0 (TID 2948) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1164.0 (TID 2949) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1164.0 (TID 2946) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1164.0 (TID 2946). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1164.0 (TID 2946) in 5 ms on localhost (1/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1164.0 (TID 2949). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1164.0 (TID 2947). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1164.0 (TID 2948). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1164.0 (TID 2947) in 7 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1164.0 (TID 2949) in 6 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1164.0 (TID 2948) in 8 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1164 (reduce at LDAOptimizer.scala:456) finished in 0.008 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1164.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 977 finished: reduce at LDAOptimizer.scala:456, took 0.012927 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 978 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1165(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1165 (PartitionwiseSampledRDD[1336] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=114479, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1295 stored as values in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=116631, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1295_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1295_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1295 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1165 (PartitionwiseSampledRDD[1336] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1165.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1165.0 (TID 2950, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1165.0 (TID 2950) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1165.0 (TID 2950). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1165.0 (TID 2950) in 2 ms on localhost (1/1) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1165 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1165.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 978 finished: isEmpty at LDAOptimizer.scala:418, took 0.007194 s -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=117948, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1296 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=118124, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1296_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1296_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1296 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 979 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1166(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1166 (MapPartitionsRDD[1338] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=118404, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1297 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=121996, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1297_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1297_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1297 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1166 (MapPartitionsRDD[1338] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1166.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1166.0 (TID 2951, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1166.0 (TID 2952, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1166.0 (TID 2953, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1166.0 (TID 2954, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1166.0 (TID 2951) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1166.0 (TID 2954) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1166.0 (TID 2952) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1166.0 (TID 2953) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1166.0 (TID 2951). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1166.0 (TID 2953). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1166.0 (TID 2952). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1166.0 (TID 2951) in 4 ms on localhost (1/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1166.0 (TID 2954). 1265 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1166.0 (TID 2953) in 5 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1166.0 (TID 2952) in 5 ms on localhost (3/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1166.0 (TID 2954) in 5 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1166 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1166.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 979 finished: reduce at LDAOptimizer.scala:453, took 0.010084 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 980 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1167(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1167 (MapPartitionsRDD[1339] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=124082, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1298 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=127714, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1298_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1298_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1298 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1167 (MapPartitionsRDD[1339] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1167.0 with 4 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1167.0 (TID 2955, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1167.0 (TID 2956, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 2.0 in stage 1167.0 (TID 2957, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 3.0 in stage 1167.0 (TID 2958, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 3.0 in stage 1167.0 (TID 2958) -2016-06-13 19:45:33 INFO Executor:59 - Running task 2.0 in stage 1167.0 (TID 2957) -2016-06-13 19:45:33 INFO Executor:59 - Running task 1.0 in stage 1167.0 (TID 2956) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1167.0 (TID 2955) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 3.0 in stage 1167.0 (TID 2958). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 3.0 in stage 1167.0 (TID 2958) in 3 ms on localhost (1/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 2.0 in stage 1167.0 (TID 2957). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1167.0 (TID 2955). 1314 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 2.0 in stage 1167.0 (TID 2957) in 4 ms on localhost (2/4) -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1167.0 (TID 2955) in 4 ms on localhost (3/4) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 1.0 in stage 1167.0 (TID 2956). 1358 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1167.0 (TID 2956) in 5 ms on localhost (4/4) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1167 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1167.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 980 finished: reduce at LDAOptimizer.scala:456, took 0.009781 s -2016-06-13 19:45:33 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Got job 981 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:33 INFO DAGScheduler:59 - Final stage: ResultStage 1168(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting ResultStage 1168 (PartitionwiseSampledRDD[1340] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=129800, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1299 stored as values in memory (estimated size 2.1 KB, free 730.5 MB) -2016-06-13 19:45:33 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=131952, maxMem=766075207 -2016-06-13 19:45:33 INFO MemoryStore:59 - Block broadcast_1299_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.5 MB) -2016-06-13 19:45:33 INFO BlockManagerInfo:59 - Added broadcast_1299_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:33 INFO SparkContext:59 - Created broadcast 1299 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1168 (PartitionwiseSampledRDD[1340] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Adding task set 1168.0 with 1 tasks -2016-06-13 19:45:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1168.0 (TID 2959, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:33 INFO Executor:59 - Running task 0.0 in stage 1168.0 (TID 2959) -2016-06-13 19:45:33 INFO Executor:59 - Finished task 0.0 in stage 1168.0 (TID 2959). 1226 bytes result sent to driver -2016-06-13 19:45:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1168.0 (TID 2959) in 3 ms on localhost (1/1) -2016-06-13 19:45:33 INFO DAGScheduler:59 - ResultStage 1168 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1168.0, whose tasks have all completed, from pool -2016-06-13 19:45:33 INFO DAGScheduler:59 - Job 981 finished: isEmpty at LDAOptimizer.scala:418, took 0.008421 s -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=133267, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1300 stored as values in memory (estimated size 176.0 B, free 730.5 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=133443, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1300_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.5 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1300_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1300 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 982 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1169(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1169 (MapPartitionsRDD[1342] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=133723, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1301 stored as values in memory (estimated size 3.5 KB, free 730.5 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=137315, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1301_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.5 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1301_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1301 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1169 (MapPartitionsRDD[1342] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1169.0 with 4 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1169.0 (TID 2960, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1169.0 (TID 2961, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 1169.0 (TID 2962, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 1169.0 (TID 2963, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1169.0 (TID 2961) -2016-06-13 19:45:34 INFO Executor:59 - Running task 3.0 in stage 1169.0 (TID 2963) -2016-06-13 19:45:34 INFO Executor:59 - Running task 2.0 in stage 1169.0 (TID 2962) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1169.0 (TID 2960) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1169.0 (TID 2961). 1265 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1169.0 (TID 2960). 1265 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 3.0 in stage 1169.0 (TID 2963). 1265 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 2.0 in stage 1169.0 (TID 2962). 1265 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1169.0 (TID 2961) in 4 ms on localhost (1/4) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1169.0 (TID 2960) in 4 ms on localhost (2/4) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 1169.0 (TID 2963) in 4 ms on localhost (3/4) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 1169.0 (TID 2962) in 4 ms on localhost (4/4) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1169 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1169.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 982 finished: reduce at LDAOptimizer.scala:453, took 0.010508 s -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 983 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1170(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1170 (MapPartitionsRDD[1343] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=139399, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1302 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2084) called with curMem=143031, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1302_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1302_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1302 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1170 (MapPartitionsRDD[1343] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1170.0 with 4 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1170.0 (TID 2964, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1170.0 (TID 2965, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 1170.0 (TID 2966, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 1170.0 (TID 2967, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1170.0 (TID 2964) -2016-06-13 19:45:34 INFO Executor:59 - Running task 2.0 in stage 1170.0 (TID 2966) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1170.0 (TID 2965) -2016-06-13 19:45:34 INFO Executor:59 - Running task 3.0 in stage 1170.0 (TID 2967) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 2.0 in stage 1170.0 (TID 2966). 1314 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1170.0 (TID 2964). 1314 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 3.0 in stage 1170.0 (TID 2967). 1358 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 1170.0 (TID 2966) in 3 ms on localhost (1/4) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1170.0 (TID 2965). 1358 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1170.0 (TID 2964) in 4 ms on localhost (2/4) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 1170.0 (TID 2967) in 3 ms on localhost (3/4) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1170.0 (TID 2965) in 5 ms on localhost (4/4) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1170 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1170.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 983 finished: reduce at LDAOptimizer.scala:456, took 0.009323 s -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 984 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1171(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1171 (PartitionwiseSampledRDD[1344] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=145115, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1303 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=147267, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1303_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1303_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1303 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1171 (PartitionwiseSampledRDD[1344] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1171.0 with 1 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1171.0 (TID 2968, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1171.0 (TID 2968) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1171.0 (TID 2968). 1226 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1171.0 (TID 2968) in 2 ms on localhost (1/1) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1171 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1171.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 984 finished: isEmpty at LDAOptimizer.scala:418, took 0.006411 s -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=148584, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1304 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=148760, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1304_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1304_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1304 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 985 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1172(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1172 (MapPartitionsRDD[1346] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=149040, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1305 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=152632, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1305_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1305_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1305 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1172 (MapPartitionsRDD[1346] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1172.0 with 4 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1172.0 (TID 2969, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1172.0 (TID 2970, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 1172.0 (TID 2971, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 1172.0 (TID 2972, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1172.0 (TID 2969) -2016-06-13 19:45:34 INFO Executor:59 - Running task 3.0 in stage 1172.0 (TID 2972) -2016-06-13 19:45:34 INFO Executor:59 - Running task 2.0 in stage 1172.0 (TID 2971) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1172.0 (TID 2970) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1172.0 (TID 2969). 1265 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1172.0 (TID 2970). 1265 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1172.0 (TID 2969) in 4 ms on localhost (1/4) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 2.0 in stage 1172.0 (TID 2971). 1265 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 3.0 in stage 1172.0 (TID 2972). 1265 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1172.0 (TID 2970) in 5 ms on localhost (2/4) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 1172.0 (TID 2971) in 4 ms on localhost (3/4) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 1172.0 (TID 2972) in 5 ms on localhost (4/4) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1172 (reduce at LDAOptimizer.scala:453) finished in 0.006 s -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 985 finished: reduce at LDAOptimizer.scala:453, took 0.010115 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1172.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 986 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1173(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1173 (MapPartitionsRDD[1347] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=154718, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1306 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2087) called with curMem=158350, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1306_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1306_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1306 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1173 (MapPartitionsRDD[1347] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1173.0 with 4 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1173.0 (TID 2973, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1173.0 (TID 2974, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 1173.0 (TID 2975, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 1173.0 (TID 2976, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1173.0 (TID 2973) -2016-06-13 19:45:34 INFO Executor:59 - Running task 2.0 in stage 1173.0 (TID 2975) -2016-06-13 19:45:34 INFO Executor:59 - Running task 3.0 in stage 1173.0 (TID 2976) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1173.0 (TID 2974) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1173.0 (TID 2974). 1358 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1173.0 (TID 2974) in 4 ms on localhost (1/4) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 2.0 in stage 1173.0 (TID 2975). 1314 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 3.0 in stage 1173.0 (TID 2976). 1358 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1173.0 (TID 2973). 1314 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 1173.0 (TID 2976) in 5 ms on localhost (2/4) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 1173.0 (TID 2975) in 5 ms on localhost (3/4) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1173.0 (TID 2973) in 5 ms on localhost (4/4) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1173 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1173.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 986 finished: reduce at LDAOptimizer.scala:456, took 0.010001 s -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 987 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1174(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1174 (PartitionwiseSampledRDD[1348] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=160437, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1307 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=162589, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1307_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1307_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1307 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1174 (PartitionwiseSampledRDD[1348] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1174.0 with 1 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1174.0 (TID 2977, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1174.0 (TID 2977) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1174.0 (TID 2977). 1226 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1174.0 (TID 2977) in 2 ms on localhost (1/1) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1174 (isEmpty at LDAOptimizer.scala:418) finished in 0.002 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1174.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 987 finished: isEmpty at LDAOptimizer.scala:418, took 0.005773 s -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=163906, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1308 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=164082, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1308_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1308_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1308 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 988 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1175(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1175 (MapPartitionsRDD[1350] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=164362, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1309 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=167954, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1309_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1309_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1309 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1175 (MapPartitionsRDD[1350] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1175.0 with 4 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1175.0 (TID 2978, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1175.0 (TID 2979, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 1175.0 (TID 2980, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 1175.0 (TID 2981, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1175.0 (TID 2979) -2016-06-13 19:45:34 INFO Executor:59 - Running task 3.0 in stage 1175.0 (TID 2981) -2016-06-13 19:45:34 INFO Executor:59 - Running task 2.0 in stage 1175.0 (TID 2980) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1175.0 (TID 2978) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1175.0 (TID 2979). 1265 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 3.0 in stage 1175.0 (TID 2981). 1265 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1175.0 (TID 2978). 1265 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1175.0 (TID 2979) in 3 ms on localhost (1/4) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 1175.0 (TID 2981) in 4 ms on localhost (2/4) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 2.0 in stage 1175.0 (TID 2980). 1265 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1175.0 (TID 2978) in 4 ms on localhost (3/4) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 1175.0 (TID 2980) in 4 ms on localhost (4/4) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1175 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1175.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 988 finished: reduce at LDAOptimizer.scala:453, took 0.009217 s -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 989 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1176(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1176 (MapPartitionsRDD[1351] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=170040, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1310 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=173672, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1310_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1310_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1310 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1176 (MapPartitionsRDD[1351] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1176.0 with 4 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1176.0 (TID 2982, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1176.0 (TID 2983, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 1176.0 (TID 2984, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 1176.0 (TID 2985, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1176.0 (TID 2983) -2016-06-13 19:45:34 INFO Executor:59 - Running task 2.0 in stage 1176.0 (TID 2984) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1176.0 (TID 2982) -2016-06-13 19:45:34 INFO Executor:59 - Running task 3.0 in stage 1176.0 (TID 2985) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1176.0 (TID 2983). 1358 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 2.0 in stage 1176.0 (TID 2984). 1314 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1176.0 (TID 2982). 1314 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1176.0 (TID 2983) in 3 ms on localhost (1/4) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 3.0 in stage 1176.0 (TID 2985). 1358 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1176.0 (TID 2982) in 4 ms on localhost (2/4) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 1176.0 (TID 2984) in 3 ms on localhost (3/4) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 1176.0 (TID 2985) in 4 ms on localhost (4/4) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1176.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1176 (reduce at LDAOptimizer.scala:456) finished in 0.005 s -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 989 finished: reduce at LDAOptimizer.scala:456, took 0.009384 s -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 990 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1177(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1177 (PartitionwiseSampledRDD[1352] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=175758, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1311 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1317) called with curMem=177910, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1311_piece0 stored as bytes in memory (estimated size 1317.0 B, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1311_piece0 in memory on localhost:50000 (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1311 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1177 (PartitionwiseSampledRDD[1352] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1177.0 with 1 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1177.0 (TID 2986, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1177.0 (TID 2986) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1177.0 (TID 2986). 1226 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1177.0 (TID 2986) in 3 ms on localhost (1/1) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1177 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1177.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 990 finished: isEmpty at LDAOptimizer.scala:418, took 0.007224 s -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=179227, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1312 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=179403, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1312_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1312_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1312 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 991 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1178(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1178 (MapPartitionsRDD[1354] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=179683, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1313 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=183275, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1313_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1313_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1313 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1178 (MapPartitionsRDD[1354] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1178.0 with 4 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1178.0 (TID 2987, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1178.0 (TID 2988, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 1178.0 (TID 2989, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 1178.0 (TID 2990, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 2.0 in stage 1178.0 (TID 2989) -2016-06-13 19:45:34 INFO Executor:59 - Running task 3.0 in stage 1178.0 (TID 2990) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1178.0 (TID 2988) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1178.0 (TID 2987) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 2.0 in stage 1178.0 (TID 2989). 1265 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 3.0 in stage 1178.0 (TID 2990). 1265 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 1178.0 (TID 2989) in 3 ms on localhost (1/4) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1178.0 (TID 2987). 1265 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1178.0 (TID 2988). 1265 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 1178.0 (TID 2990) in 3 ms on localhost (2/4) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1178.0 (TID 2988) in 4 ms on localhost (3/4) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1178.0 (TID 2987) in 5 ms on localhost (4/4) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1178 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1178.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 991 finished: reduce at LDAOptimizer.scala:453, took 0.008491 s -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 992 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1179(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1179 (MapPartitionsRDD[1355] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=185361, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1314 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2086) called with curMem=188993, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1314_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1314_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1314 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1179 (MapPartitionsRDD[1355] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1179.0 with 4 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1179.0 (TID 2991, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1179.0 (TID 2992, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 1179.0 (TID 2993, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 1179.0 (TID 2994, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 2.0 in stage 1179.0 (TID 2993) -2016-06-13 19:45:34 INFO Executor:59 - Running task 3.0 in stage 1179.0 (TID 2994) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1179.0 (TID 2991) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1179.0 (TID 2992) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 3.0 in stage 1179.0 (TID 2994). 1358 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 2.0 in stage 1179.0 (TID 2993). 1314 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1179.0 (TID 2992). 1358 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 1179.0 (TID 2994) in 3 ms on localhost (1/4) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1179.0 (TID 2991). 1314 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 1179.0 (TID 2993) in 3 ms on localhost (2/4) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1179.0 (TID 2992) in 4 ms on localhost (3/4) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1179.0 (TID 2991) in 4 ms on localhost (4/4) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1179 (reduce at LDAOptimizer.scala:456) finished in 0.004 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1179.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 992 finished: reduce at LDAOptimizer.scala:456, took 0.008833 s -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: isEmpty at LDAOptimizer.scala:418 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 993 (isEmpty at LDAOptimizer.scala:418) with 1 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1180(isEmpty at LDAOptimizer.scala:418) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1180 (PartitionwiseSampledRDD[1356] at sample at LDAOptimizer.scala:416), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2152) called with curMem=191079, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1315 stored as values in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1315) called with curMem=193231, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1315_piece0 stored as bytes in memory (estimated size 1315.0 B, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1315_piece0 in memory on localhost:50000 (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1315 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1180 (PartitionwiseSampledRDD[1356] at sample at LDAOptimizer.scala:416) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1180.0 with 1 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1180.0 (TID 2995, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1180.0 (TID 2995) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1180.0 (TID 2995). 1226 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1180.0 (TID 2995) in 3 ms on localhost (1/1) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1180 (isEmpty at LDAOptimizer.scala:418) finished in 0.003 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1180.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 993 finished: isEmpty at LDAOptimizer.scala:418, took 0.008045 s -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(176) called with curMem=194546, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1316 stored as values in memory (estimated size 176.0 B, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(280) called with curMem=194722, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1316_piece0 stored as bytes in memory (estimated size 280.0 B, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1316_piece0 in memory on localhost:50000 (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1316 from broadcast at LDAOptimizer.scala:432 -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:453 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 994 (reduce at LDAOptimizer.scala:453) with 4 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1181(reduce at LDAOptimizer.scala:453) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1181 (MapPartitionsRDD[1358] at map at LDAOptimizer.scala:453), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3592) called with curMem=195002, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1317 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2082) called with curMem=198594, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1317_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1317_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1317 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1181 (MapPartitionsRDD[1358] at map at LDAOptimizer.scala:453) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1181.0 with 4 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1181.0 (TID 2996, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1181.0 (TID 2997, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 1181.0 (TID 2998, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 1181.0 (TID 2999, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 2.0 in stage 1181.0 (TID 2998) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1181.0 (TID 2996) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1181.0 (TID 2997) -2016-06-13 19:45:34 INFO Executor:59 - Running task 3.0 in stage 1181.0 (TID 2999) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1181.0 (TID 2996). 1265 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 2.0 in stage 1181.0 (TID 2998). 1265 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1181.0 (TID 2996) in 4 ms on localhost (1/4) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 1181.0 (TID 2998) in 4 ms on localhost (2/4) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1181.0 (TID 2997). 1265 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1181.0 (TID 2997) in 5 ms on localhost (3/4) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 3.0 in stage 1181.0 (TID 2999). 1265 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 1181.0 (TID 2999) in 5 ms on localhost (4/4) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1181 (reduce at LDAOptimizer.scala:453) finished in 0.005 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1181.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 994 finished: reduce at LDAOptimizer.scala:453, took 0.010746 s -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: reduce at LDAOptimizer.scala:456 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 995 (reduce at LDAOptimizer.scala:456) with 4 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1182(reduce at LDAOptimizer.scala:456) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1182 (MapPartitionsRDD[1359] at map at LDAOptimizer.scala:456), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3632) called with curMem=200676, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1318 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2082) called with curMem=204308, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1318_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1318_piece0 in memory on localhost:50000 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1318 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 4 missing tasks from ResultStage 1182 (MapPartitionsRDD[1359] at map at LDAOptimizer.scala:456) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1182.0 with 4 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1182.0 (TID 3000, localhost, PROCESS_LOCAL, 2415 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1182.0 (TID 3001, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 2.0 in stage 1182.0 (TID 3002, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 3.0 in stage 1182.0 (TID 3003, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1182.0 (TID 3001) -2016-06-13 19:45:34 INFO Executor:59 - Running task 2.0 in stage 1182.0 (TID 3002) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1182.0 (TID 3000) -2016-06-13 19:45:34 INFO Executor:59 - Running task 3.0 in stage 1182.0 (TID 3003) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1182.0 (TID 3001). 1358 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 2.0 in stage 1182.0 (TID 3002). 1314 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1182.0 (TID 3001) in 5 ms on localhost (1/4) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 2.0 in stage 1182.0 (TID 3002) in 4 ms on localhost (2/4) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1182.0 (TID 3000). 1314 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1182.0 (TID 3000) in 7 ms on localhost (3/4) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 3.0 in stage 1182.0 (TID 3003). 1358 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 3.0 in stage 1182.0 (TID 3003) in 8 ms on localhost (4/4) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1182 (reduce at LDAOptimizer.scala:456) finished in 0.009 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1182.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 995 finished: reduce at LDAOptimizer.scala:456, took 0.013410 s -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: take at LDAOptimizer.scala:142 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 996 (take at LDAOptimizer.scala:142) with 1 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1183(take at LDAOptimizer.scala:142) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1183 (ParallelCollectionRDD[1360] at parallelize at LDASuite.scala:473), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=206390, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1319 stored as values in memory (estimated size 1224.0 B, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(807) called with curMem=207614, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1319_piece0 stored as bytes in memory (estimated size 807.0 B, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1319_piece0 in memory on localhost:50000 (size: 807.0 B, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1319 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1183 (ParallelCollectionRDD[1360] at parallelize at LDASuite.scala:473) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1183.0 with 1 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1183.0 (TID 3004, localhost, PROCESS_LOCAL, 2424 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1183.0 (TID 3004) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1183.0 (TID 3004). 1192 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1183.0 (TID 3004) in 2 ms on localhost (1/1) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1183 (take at LDAOptimizer.scala:142) finished in 0.002 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1183.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 996 finished: take at LDAOptimizer.scala:142, took 0.006915 s -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Registering RDD 1375 (map at GraphImpl.scala:104) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Registering RDD 1379 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Registering RDD 1367 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Registering RDD 1362 (mapPartitionsWithIndex at LDAOptimizer.scala:128) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 997 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1188(fold at LDAOptimizer.scala:205) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1185, ShuffleMapStage 1186, ShuffleMapStage 1187) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1185, ShuffleMapStage 1186, ShuffleMapStage 1187) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1184 (MapPartitionsRDD[1375] at map at GraphImpl.scala:104), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3880) called with curMem=208421, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1320 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2166) called with curMem=212301, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1320_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1320_piece0 in memory on localhost:50000 (size: 2.1 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1320 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1184 (MapPartitionsRDD[1375] at map at GraphImpl.scala:104) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1184.0 with 2 tasks -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1186 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[1367] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1184.0 (TID 3005, localhost, PROCESS_LOCAL, 2413 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1184.0 (TID 3006, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1184.0 (TID 3005) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1184.0 (TID 3006) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3072) called with curMem=214467, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1321 stored as values in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1829) called with curMem=217539, maxMem=766075207 -2016-06-13 19:45:34 INFO CacheManager:59 - Partition rdd_1373_0 not found, computing it -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1321_piece0 stored as bytes in memory (estimated size 1829.0 B, free 730.4 MB) -2016-06-13 19:45:34 INFO CacheManager:59 - Partition rdd_1373_1 not found, computing it -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2680) called with curMem=219368, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block rdd_1373_0 stored as values in memory (estimated size 2.6 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1321_piece0 in memory on localhost:50000 (size: 1829.0 B, free: 730.5 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2792) called with curMem=222048, maxMem=766075207 -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added rdd_1373_0 in memory on localhost:50000 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - Block rdd_1373_1 stored as values in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1321 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added rdd_1373_1 in memory on localhost:50000 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1186 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[1367] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1186.0 with 2 tasks -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1187 (MapPartitionsRDD[1362] at mapPartitionsWithIndex at LDAOptimizer.scala:128), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2904) called with curMem=224840, maxMem=766075207 -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1186.0 (TID 3007, localhost, PROCESS_LOCAL, 2413 bytes) -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1322 stored as values in memory (estimated size 2.8 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1186.0 (TID 3008, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1186.0 (TID 3007) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1726) called with curMem=227744, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1322_piece0 stored as bytes in memory (estimated size 1726.0 B, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1322_piece0 in memory on localhost:50000 (size: 1726.0 B, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1322 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1187 (MapPartitionsRDD[1362] at mapPartitionsWithIndex at LDAOptimizer.scala:128) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1187.0 with 2 tasks -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1186.0 (TID 3008) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1184.0 (TID 3006). 1739 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1187.0 (TID 3009, localhost, PROCESS_LOCAL, 2413 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1187.0 (TID 3009) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1184.0 (TID 3006) in 23 ms on localhost (1/2) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1184.0 (TID 3005). 1739 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1187.0 (TID 3010, localhost, PROCESS_LOCAL, 2498 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1184.0 (TID 3005) in 35 ms on localhost (2/2) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1184.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1187.0 (TID 3010) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ShuffleMapStage 1184 (map at GraphImpl.scala:104) finished in 0.036 s -2016-06-13 19:45:34 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:34 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 1186, ShuffleMapStage 1187) -2016-06-13 19:45:34 INFO DAGScheduler:59 - waiting: Set(ResultStage 1188, ShuffleMapStage 1185) -2016-06-13 19:45:34 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1186.0 (TID 3008). 1159 bytes result sent to driver -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents for ResultStage 1188: List(ShuffleMapStage 1185, ShuffleMapStage 1186, ShuffleMapStage 1187) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 1185: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1185 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[1379] at mapPartitions at VertexRDD.scala:358), which is now runnable -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2784) called with curMem=229470, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1323 stored as values in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1186.0 (TID 3007). 1159 bytes result sent to driver -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1704) called with curMem=232254, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1323_piece0 stored as bytes in memory (estimated size 1704.0 B, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1323_piece0 in memory on localhost:50000 (size: 1704.0 B, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1323 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1185 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[1379] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1185.0 with 2 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1185.0 (TID 3011, localhost, PROCESS_LOCAL, 1890 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1185.0 (TID 3012, localhost, PROCESS_LOCAL, 1890 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1185.0 (TID 3012) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1186.0 (TID 3007) in 38 ms on localhost (1/2) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1187.0 (TID 3009). 1159 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1186.0 (TID 3008) in 39 ms on localhost (2/2) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1186.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - ShuffleMapStage 1186 (mapPartitions at VertexRDD.scala:358) finished in 0.040 s -2016-06-13 19:45:34 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:34 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 1185, ShuffleMapStage 1187) -2016-06-13 19:45:34 INFO DAGScheduler:59 - waiting: Set(ResultStage 1188) -2016-06-13 19:45:34 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1187.0 (TID 3009) in 23 ms on localhost (1/2) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1185.0 (TID 3011) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1187.0 (TID 3010). 1159 bytes result sent to driver -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents for ResultStage 1188: List(ShuffleMapStage 1185, ShuffleMapStage 1187) -2016-06-13 19:45:34 INFO CacheManager:59 - Partition rdd_1377_1 not found, computing it -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1187.0 (TID 3010) in 14 ms on localhost (2/2) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1187.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - ShuffleMapStage 1187 (mapPartitionsWithIndex at LDAOptimizer.scala:128) finished in 0.038 s -2016-06-13 19:45:34 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:34 INFO CacheManager:59 - Partition rdd_1377_0 not found, computing it -2016-06-13 19:45:34 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 1185) -2016-06-13 19:45:34 INFO DAGScheduler:59 - waiting: Set(ResultStage 1188) -2016-06-13 19:45:34 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents for ResultStage 1188: List(ShuffleMapStage 1185) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2784) called with curMem=233958, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block rdd_1377_1 stored as values in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added rdd_1377_1 in memory on localhost:50000 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2688) called with curMem=236742, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block rdd_1377_0 stored as values in memory (estimated size 2.6 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added rdd_1377_0 in memory on localhost:50000 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1185.0 (TID 3012). 1955 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1185.0 (TID 3011). 1955 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1185.0 (TID 3012) in 29 ms on localhost (1/2) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1185.0 (TID 3011) in 29 ms on localhost (2/2) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1185.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - ShuffleMapStage 1185 (mapPartitions at VertexRDD.scala:358) finished in 0.031 s -2016-06-13 19:45:34 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:34 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - waiting: Set(ResultStage 1188) -2016-06-13 19:45:34 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents for ResultStage 1188: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1188 (MapPartitionsRDD[1386] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(5584) called with curMem=239430, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1324 stored as values in memory (estimated size 5.5 KB, free 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2804) called with curMem=245014, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1324_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1324_piece0 in memory on localhost:50000 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1324 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1188 (MapPartitionsRDD[1386] at values at LDAOptimizer.scala:205) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1188.0 with 2 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1188.0 (TID 3013, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1188.0 (TID 3014, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1188.0 (TID 3013) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1188.0 (TID 3014) -2016-06-13 19:45:34 INFO CacheManager:59 - Partition rdd_1382_1 not found, computing it -2016-06-13 19:45:34 INFO CacheManager:59 - Partition rdd_1370_1 not found, computing it -2016-06-13 19:45:34 INFO CacheManager:59 - Partition rdd_1382_0 not found, computing it -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO CacheManager:59 - Partition rdd_1370_0 not found, computing it -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1848) called with curMem=247818, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block rdd_1370_1 stored as values in memory (estimated size 1848.0 B, free 730.3 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1856) called with curMem=249666, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block rdd_1370_0 stored as values in memory (estimated size 1856.0 B, free 730.3 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added rdd_1370_1 in memory on localhost:50000 (size: 1848.0 B, free: 730.5 MB) -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added rdd_1370_0 in memory on localhost:50000 (size: 1856.0 B, free: 730.5 MB) -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1848) called with curMem=251522, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block rdd_1382_1 stored as values in memory (estimated size 1848.0 B, free 730.3 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added rdd_1382_1 in memory on localhost:50000 (size: 1848.0 B, free: 730.5 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1856) called with curMem=253370, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block rdd_1382_0 stored as values in memory (estimated size 1856.0 B, free 730.3 MB) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1188.0 (TID 3014). 1931 bytes result sent to driver -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added rdd_1382_0 in memory on localhost:50000 (size: 1856.0 B, free: 730.5 MB) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1188.0 (TID 3014) in 22 ms on localhost (1/2) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1188.0 (TID 3013). 1931 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1188.0 (TID 3013) in 24 ms on localhost (2/2) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1188 (fold at LDAOptimizer.scala:205) finished in 0.024 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1188.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 997 finished: fold at LDAOptimizer.scala:205, took 0.115574 s -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 154 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 154 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:45:34 INFO DAGScheduler:59 - Registering RDD 1387 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Registering RDD 1391 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 998 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1195(fold at LDAOptimizer.scala:205) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1192, ShuffleMapStage 1189, ShuffleMapStage 1190, ShuffleMapStage 1194) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1194) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1193 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[1387] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(4360) called with curMem=255226, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1325 stored as values in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2390) called with curMem=259586, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1325_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1325_piece0 in memory on localhost:50000 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1325 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1193 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[1387] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1193.0 with 2 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1193.0 (TID 3015, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1193.0 (TID 3016, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1193.0 (TID 3016) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1193.0 (TID 3015) -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1382_0 locally -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1382_1 locally -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1193.0 (TID 3016). 2254 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1193.0 (TID 3015). 2254 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1193.0 (TID 3016) in 18 ms on localhost (1/2) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1193.0 (TID 3015) in 20 ms on localhost (2/2) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ShuffleMapStage 1193 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.020 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1193.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:34 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 1194, ResultStage 1195) -2016-06-13 19:45:34 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 1194: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents for ResultStage 1195: List(ShuffleMapStage 1194) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1194 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[1391] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=261976, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1326 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2287) called with curMem=265872, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1326_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1326_piece0 in memory on localhost:50000 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1326 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1194 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[1391] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1194.0 with 2 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1194.0 (TID 3017, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1194.0 (TID 3018, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1194.0 (TID 3017) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1194.0 (TID 3018) -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1377_1 locally -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1377_0 locally -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1194.0 (TID 3018). 2470 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1194.0 (TID 3017). 2470 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1194.0 (TID 3018) in 21 ms on localhost (1/2) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1194.0 (TID 3017) in 22 ms on localhost (2/2) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1194.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - ShuffleMapStage 1194 (mapPartitions at GraphImpl.scala:235) finished in 0.022 s -2016-06-13 19:45:34 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:34 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - waiting: Set(ResultStage 1195) -2016-06-13 19:45:34 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents for ResultStage 1195: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1195 (MapPartitionsRDD[1399] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(6432) called with curMem=268159, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1327 stored as values in memory (estimated size 6.3 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3026) called with curMem=274591, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1327_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1327_piece0 in memory on localhost:50000 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1327 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1195 (MapPartitionsRDD[1399] at values at LDAOptimizer.scala:205) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1195.0 with 2 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1195.0 (TID 3019, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1195.0 (TID 3020, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1195.0 (TID 3019) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1195.0 (TID 3020) -2016-06-13 19:45:34 INFO CacheManager:59 - Partition rdd_1395_0 not found, computing it -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1382_0 locally -2016-06-13 19:45:34 INFO CacheManager:59 - Partition rdd_1395_1 not found, computing it -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1382_1 locally -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1904) called with curMem=277617, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block rdd_1395_0 stored as values in memory (estimated size 1904.0 B, free 730.3 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1896) called with curMem=279521, maxMem=766075207 -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added rdd_1395_0 in memory on localhost:50000 (size: 1904.0 B, free: 730.5 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - Block rdd_1395_1 stored as values in memory (estimated size 1896.0 B, free 730.3 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added rdd_1395_1 in memory on localhost:50000 (size: 1896.0 B, free: 730.5 MB) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1195.0 (TID 3019). 2971 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1195.0 (TID 3020). 2971 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1195.0 (TID 3019) in 8 ms on localhost (1/2) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1195.0 (TID 3020) in 9 ms on localhost (2/2) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1195.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1195 (fold at LDAOptimizer.scala:205) finished in 0.009 s -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 998 finished: fold at LDAOptimizer.scala:205, took 0.071444 s -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 154 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 154 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:45:34 INFO DAGScheduler:59 - Registering RDD 1400 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Registering RDD 1404 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 999 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1204(fold at LDAOptimizer.scala:205) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1199, ShuffleMapStage 1203, ShuffleMapStage 1196, ShuffleMapStage 1197, ShuffleMapStage 1201) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1203) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1202 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[1400] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=281417, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1328 stored as values in memory (estimated size 5.3 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2722) called with curMem=286865, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1328_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1328_piece0 in memory on localhost:50000 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1328 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1202 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[1400] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1202.0 with 2 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1202.0 (TID 3021, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1202.0 (TID 3022, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1202.0 (TID 3021) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1202.0 (TID 3022) -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1395_0 locally -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1395_1 locally -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1202.0 (TID 3022). 2254 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1202.0 (TID 3022) in 22 ms on localhost (1/2) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1202.0 (TID 3021). 2254 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1202.0 (TID 3021) in 26 ms on localhost (2/2) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1202.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - ShuffleMapStage 1202 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.026 s -2016-06-13 19:45:34 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:34 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 1203, ResultStage 1204) -2016-06-13 19:45:34 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 1203: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents for ResultStage 1204: List(ShuffleMapStage 1203) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1203 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[1404] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=289587, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1329 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2287) called with curMem=293483, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1329_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1329_piece0 in memory on localhost:50000 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1329 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1203 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[1404] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1203.0 with 2 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1203.0 (TID 3023, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1203.0 (TID 3024, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1203.0 (TID 3023) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1203.0 (TID 3024) -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1377_0 locally -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1377_1 locally -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1203.0 (TID 3023). 2470 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1203.0 (TID 3024). 2470 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1203.0 (TID 3023) in 22 ms on localhost (1/2) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1203.0 (TID 3024) in 21 ms on localhost (2/2) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ShuffleMapStage 1203 (mapPartitions at GraphImpl.scala:235) finished in 0.023 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1203.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:34 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - waiting: Set(ResultStage 1204) -2016-06-13 19:45:34 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents for ResultStage 1204: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1204 (MapPartitionsRDD[1412] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(6864) called with curMem=295770, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1330 stored as values in memory (estimated size 6.7 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3157) called with curMem=302634, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1330_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1330_piece0 in memory on localhost:50000 (size: 3.1 KB, free: 730.4 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1330 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1204 (MapPartitionsRDD[1412] at values at LDAOptimizer.scala:205) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1204.0 with 2 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1204.0 (TID 3025, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1204.0 (TID 3026, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1204.0 (TID 3026) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1204.0 (TID 3025) -2016-06-13 19:45:34 INFO CacheManager:59 - Partition rdd_1408_0 not found, computing it -2016-06-13 19:45:34 INFO CacheManager:59 - Partition rdd_1408_1 not found, computing it -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1395_0 locally -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1395_1 locally -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1896) called with curMem=305791, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block rdd_1408_1 stored as values in memory (estimated size 1896.0 B, free 730.3 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1904) called with curMem=307687, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block rdd_1408_0 stored as values in memory (estimated size 1904.0 B, free 730.3 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added rdd_1408_1 in memory on localhost:50000 (size: 1896.0 B, free: 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added rdd_1408_0 in memory on localhost:50000 (size: 1904.0 B, free: 730.4 MB) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1204.0 (TID 3026). 2971 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1204.0 (TID 3025). 2971 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1204.0 (TID 3025) in 7 ms on localhost (1/2) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1204.0 (TID 3026) in 7 ms on localhost (2/2) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1204 (fold at LDAOptimizer.scala:205) finished in 0.007 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1204.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 999 finished: fold at LDAOptimizer.scala:205, took 0.077586 s -2016-06-13 19:45:34 INFO ZippedPartitionsRDD2:59 - Removing RDD 1382 from persistence list -2016-06-13 19:45:34 INFO BlockManager:59 - Removing RDD 1382 -2016-06-13 19:45:34 INFO ZippedPartitionsRDD2:59 - Removing RDD 1389 from persistence list -2016-06-13 19:45:34 INFO BlockManager:59 - Removing RDD 1389 -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 154 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 154 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:45:34 INFO DAGScheduler:59 - Registering RDD 1413 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Registering RDD 1417 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 1000 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1215(fold at LDAOptimizer.scala:205) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1206, ShuffleMapStage 1210, ShuffleMapStage 1214, ShuffleMapStage 1208, ShuffleMapStage 1212, ShuffleMapStage 1205) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1214) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1213 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[1413] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(5872) called with curMem=305887, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1331 stored as values in memory (estimated size 5.7 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2843) called with curMem=311759, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1331_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1331_piece0 in memory on localhost:50000 (size: 2.8 KB, free: 730.4 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1331 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1213 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[1413] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1213.0 with 2 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1213.0 (TID 3027, localhost, PROCESS_LOCAL, 2196 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1213.0 (TID 3028, localhost, PROCESS_LOCAL, 2196 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1213.0 (TID 3027) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1213.0 (TID 3028) -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1408_1 locally -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1408_0 locally -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1213.0 (TID 3028). 2254 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1213.0 (TID 3027). 2254 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1213.0 (TID 3028) in 17 ms on localhost (1/2) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1213.0 (TID 3027) in 17 ms on localhost (2/2) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1213.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - ShuffleMapStage 1213 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.017 s -2016-06-13 19:45:34 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:34 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 1214, ResultStage 1215) -2016-06-13 19:45:34 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 1214: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents for ResultStage 1215: List(ShuffleMapStage 1214) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1214 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[1417] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=314602, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1332 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2285) called with curMem=318498, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1332_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1332_piece0 in memory on localhost:50000 (size: 2.2 KB, free: 730.4 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1332 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1214 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[1417] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1214.0 with 2 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1214.0 (TID 3029, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1214.0 (TID 3030, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1214.0 (TID 3029) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1214.0 (TID 3030) -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1377_1 locally -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1377_0 locally -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1214.0 (TID 3029). 2470 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1214.0 (TID 3029) in 18 ms on localhost (1/2) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1214.0 (TID 3030). 2470 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1214.0 (TID 3030) in 20 ms on localhost (2/2) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1214.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - ShuffleMapStage 1214 (mapPartitions at GraphImpl.scala:235) finished in 0.021 s -2016-06-13 19:45:34 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:34 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - waiting: Set(ResultStage 1215) -2016-06-13 19:45:34 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents for ResultStage 1215: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1215 (MapPartitionsRDD[1425] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(7288) called with curMem=320783, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1333 stored as values in memory (estimated size 7.1 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3285) called with curMem=328071, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1333_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1333_piece0 in memory on localhost:50000 (size: 3.2 KB, free: 730.4 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1333 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1215 (MapPartitionsRDD[1425] at values at LDAOptimizer.scala:205) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1215.0 with 2 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1215.0 (TID 3031, localhost, PROCESS_LOCAL, 2248 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1215.0 (TID 3032, localhost, PROCESS_LOCAL, 2248 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1215.0 (TID 3032) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1215.0 (TID 3031) -2016-06-13 19:45:34 INFO CacheManager:59 - Partition rdd_1421_1 not found, computing it -2016-06-13 19:45:34 INFO CacheManager:59 - Partition rdd_1421_0 not found, computing it -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1408_1 locally -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1408_0 locally -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1896) called with curMem=331356, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block rdd_1421_1 stored as values in memory (estimated size 1896.0 B, free 730.3 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1904) called with curMem=333252, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block rdd_1421_0 stored as values in memory (estimated size 1904.0 B, free 730.3 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added rdd_1421_1 in memory on localhost:50000 (size: 1896.0 B, free: 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added rdd_1421_0 in memory on localhost:50000 (size: 1904.0 B, free: 730.4 MB) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1215.0 (TID 3032). 2971 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1215.0 (TID 3031). 2971 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1215.0 (TID 3032) in 6 ms on localhost (1/2) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1215.0 (TID 3031) in 7 ms on localhost (2/2) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1215 (fold at LDAOptimizer.scala:205) finished in 0.007 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1215.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 1000 finished: fold at LDAOptimizer.scala:205, took 0.067198 s -2016-06-13 19:45:34 INFO MapPartitionsRDD:59 - Removing RDD 1395 from persistence list -2016-06-13 19:45:34 INFO BlockManager:59 - Removing RDD 1395 -2016-06-13 19:45:34 INFO ZippedPartitionsRDD2:59 - Removing RDD 1402 from persistence list -2016-06-13 19:45:34 INFO BlockManager:59 - Removing RDD 1402 -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 154 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 154 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:45:34 INFO DAGScheduler:59 - Registering RDD 1426 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Registering RDD 1430 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 1001 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1228(fold at LDAOptimizer.scala:205) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1217, ShuffleMapStage 1221, ShuffleMapStage 1225, ShuffleMapStage 1223, ShuffleMapStage 1227, ShuffleMapStage 1219, ShuffleMapStage 1216) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1227) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1226 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[1426] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(6304) called with curMem=331356, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1334 stored as values in memory (estimated size 6.2 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2962) called with curMem=337660, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1334_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1334_piece0 in memory on localhost:50000 (size: 2.9 KB, free: 730.4 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1334 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1226 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[1426] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1226.0 with 2 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1226.0 (TID 3033, localhost, PROCESS_LOCAL, 2237 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1226.0 (TID 3034, localhost, PROCESS_LOCAL, 2237 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1226.0 (TID 3034) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1226.0 (TID 3033) -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1421_1 locally -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1421_0 locally -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1226.0 (TID 3033). 2254 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1226.0 (TID 3034). 2254 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1226.0 (TID 3033) in 17 ms on localhost (1/2) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1226.0 (TID 3034) in 17 ms on localhost (2/2) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1226.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - ShuffleMapStage 1226 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.018 s -2016-06-13 19:45:34 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:34 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - waiting: Set(ResultStage 1228, ShuffleMapStage 1227) -2016-06-13 19:45:34 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents for ResultStage 1228: List(ShuffleMapStage 1227) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 1227: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1227 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[1430] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=340622, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1335 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=344518, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1335_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1335_piece0 in memory on localhost:50000 (size: 2.2 KB, free: 730.4 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1335 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1227 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[1430] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1227.0 with 2 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1227.0 (TID 3035, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1227.0 (TID 3036, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1227.0 (TID 3036) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1227.0 (TID 3035) -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1377_1 locally -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1377_0 locally -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1227.0 (TID 3036). 2470 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1227.0 (TID 3035). 2470 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1227.0 (TID 3036) in 20 ms on localhost (1/2) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1227.0 (TID 3035) in 20 ms on localhost (2/2) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ShuffleMapStage 1227 (mapPartitions at GraphImpl.scala:235) finished in 0.021 s -2016-06-13 19:45:34 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:34 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - waiting: Set(ResultStage 1228) -2016-06-13 19:45:34 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1227.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents for ResultStage 1228: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1228 (MapPartitionsRDD[1438] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(7720) called with curMem=346804, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1336 stored as values in memory (estimated size 7.5 KB, free 730.2 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3388) called with curMem=354524, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1336_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.2 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1336_piece0 in memory on localhost:50000 (size: 3.3 KB, free: 730.4 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1336 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1228 (MapPartitionsRDD[1438] at values at LDAOptimizer.scala:205) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1228.0 with 2 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1228.0 (TID 3037, localhost, PROCESS_LOCAL, 2289 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1228.0 (TID 3038, localhost, PROCESS_LOCAL, 2289 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1228.0 (TID 3038) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1228.0 (TID 3037) -2016-06-13 19:45:34 INFO CacheManager:59 - Partition rdd_1434_0 not found, computing it -2016-06-13 19:45:34 INFO CacheManager:59 - Partition rdd_1434_1 not found, computing it -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1421_0 locally -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1421_1 locally -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1904) called with curMem=357912, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block rdd_1434_0 stored as values in memory (estimated size 1904.0 B, free 730.2 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1896) called with curMem=359816, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block rdd_1434_1 stored as values in memory (estimated size 1896.0 B, free 730.2 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added rdd_1434_0 in memory on localhost:50000 (size: 1904.0 B, free: 730.4 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added rdd_1434_1 in memory on localhost:50000 (size: 1896.0 B, free: 730.4 MB) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1228.0 (TID 3037). 2971 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1228.0 (TID 3038). 2971 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1228.0 (TID 3037) in 7 ms on localhost (1/2) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1228.0 (TID 3038) in 7 ms on localhost (2/2) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1228 (fold at LDAOptimizer.scala:205) finished in 0.007 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1228.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 1001 finished: fold at LDAOptimizer.scala:205, took 0.068177 s -2016-06-13 19:45:34 INFO MapPartitionsRDD:59 - Removing RDD 1408 from persistence list -2016-06-13 19:45:34 INFO BlockManager:59 - Removing RDD 1408 -2016-06-13 19:45:34 INFO ZippedPartitionsRDD2:59 - Removing RDD 1415 from persistence list -2016-06-13 19:45:34 INFO BlockManager:59 - Removing RDD 1415 -2016-06-13 19:45:34 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 154 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 154 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:45:34 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:45:34 INFO DAGScheduler:59 - Registering RDD 1439 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Registering RDD 1443 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Got job 1002 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:45:34 INFO DAGScheduler:59 - Final stage: ResultStage 1243(fold at LDAOptimizer.scala:205) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1238, ShuffleMapStage 1232, ShuffleMapStage 1242, ShuffleMapStage 1236, ShuffleMapStage 1240, ShuffleMapStage 1229, ShuffleMapStage 1230, ShuffleMapStage 1234) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1242) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1241 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[1439] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(6728) called with curMem=357912, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1337 stored as values in memory (estimated size 6.6 KB, free 730.2 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3066) called with curMem=364640, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1337_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.2 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1337_piece0 in memory on localhost:50000 (size: 3.0 KB, free: 730.4 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1337 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1241 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[1439] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1241.0 with 2 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1241.0 (TID 3039, localhost, PROCESS_LOCAL, 2278 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1241.0 (TID 3040, localhost, PROCESS_LOCAL, 2278 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1241.0 (TID 3040) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1241.0 (TID 3039) -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1434_1 locally -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1434_0 locally -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1241.0 (TID 3039). 2254 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1241.0 (TID 3040). 2254 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1241.0 (TID 3039) in 19 ms on localhost (1/2) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1241.0 (TID 3040) in 18 ms on localhost (2/2) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1241.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - ShuffleMapStage 1241 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.020 s -2016-06-13 19:45:34 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:34 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 1242, ResultStage 1243) -2016-06-13 19:45:34 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 1242: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents for ResultStage 1243: List(ShuffleMapStage 1242) -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1242 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[1443] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=367706, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1338 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=371602, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1338_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1338_piece0 in memory on localhost:50000 (size: 2.2 KB, free: 730.4 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1338 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 1242 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[1443] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1242.0 with 2 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1242.0 (TID 3041, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1242.0 (TID 3042, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1242.0 (TID 3041) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1242.0 (TID 3042) -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1377_0 locally -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1377_1 locally -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1242.0 (TID 3042). 2470 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1242.0 (TID 3042) in 19 ms on localhost (1/2) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1242.0 (TID 3041). 2470 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1242.0 (TID 3041) in 21 ms on localhost (2/2) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ShuffleMapStage 1242 (mapPartitions at GraphImpl.scala:235) finished in 0.021 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1242.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:45:34 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - waiting: Set(ResultStage 1243) -2016-06-13 19:45:34 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Missing parents for ResultStage 1243: List() -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting ResultStage 1243 (MapPartitionsRDD[1451] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(8144) called with curMem=373888, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1339 stored as values in memory (estimated size 8.0 KB, free 730.2 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(3498) called with curMem=382032, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block broadcast_1339_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added broadcast_1339_piece0 in memory on localhost:50000 (size: 3.4 KB, free: 730.4 MB) -2016-06-13 19:45:34 INFO SparkContext:59 - Created broadcast 1339 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1243 (MapPartitionsRDD[1451] at values at LDAOptimizer.scala:205) -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Adding task set 1243.0 with 2 tasks -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1243.0 (TID 3043, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1243.0 (TID 3044, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:45:34 INFO Executor:59 - Running task 0.0 in stage 1243.0 (TID 3043) -2016-06-13 19:45:34 INFO Executor:59 - Running task 1.0 in stage 1243.0 (TID 3044) -2016-06-13 19:45:34 INFO CacheManager:59 - Partition rdd_1447_0 not found, computing it -2016-06-13 19:45:34 INFO CacheManager:59 - Partition rdd_1447_1 not found, computing it -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1434_0 locally -2016-06-13 19:45:34 INFO BlockManager:59 - Found block rdd_1434_1 locally -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1904) called with curMem=385530, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block rdd_1447_0 stored as values in memory (estimated size 1904.0 B, free 730.2 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added rdd_1447_0 in memory on localhost:50000 (size: 1904.0 B, free: 730.4 MB) -2016-06-13 19:45:34 INFO MemoryStore:59 - ensureFreeSpace(1896) called with curMem=387434, maxMem=766075207 -2016-06-13 19:45:34 INFO MemoryStore:59 - Block rdd_1447_1 stored as values in memory (estimated size 1896.0 B, free 730.2 MB) -2016-06-13 19:45:34 INFO BlockManagerInfo:59 - Added rdd_1447_1 in memory on localhost:50000 (size: 1896.0 B, free: 730.4 MB) -2016-06-13 19:45:34 INFO Executor:59 - Finished task 0.0 in stage 1243.0 (TID 3043). 2971 bytes result sent to driver -2016-06-13 19:45:34 INFO Executor:59 - Finished task 1.0 in stage 1243.0 (TID 3044). 2971 bytes result sent to driver -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 1243.0 (TID 3043) in 8 ms on localhost (1/2) -2016-06-13 19:45:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 1243.0 (TID 3044) in 8 ms on localhost (2/2) -2016-06-13 19:45:34 INFO DAGScheduler:59 - ResultStage 1243 (fold at LDAOptimizer.scala:205) finished in 0.008 s -2016-06-13 19:45:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 1243.0, whose tasks have all completed, from pool -2016-06-13 19:45:34 INFO DAGScheduler:59 - Job 1002 finished: fold at LDAOptimizer.scala:205, took 0.076036 s -2016-06-13 19:45:35 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 19:45:35 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 19:45:35 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 19:45:35 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 19:45:35 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 19:45:35 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 19:45:36 INFO SparkContext:59 - Starting job: saveAsTextFile at LDAModel.scala:418 -2016-06-13 19:45:36 INFO DAGScheduler:59 - Got job 1003 (saveAsTextFile at LDAModel.scala:418) with 1 output partitions -2016-06-13 19:45:36 INFO DAGScheduler:59 - Final stage: ResultStage 1244(saveAsTextFile at LDAModel.scala:418) -2016-06-13 19:45:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:36 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:36 INFO DAGScheduler:59 - Submitting ResultStage 1244 (MapPartitionsRDD[1453] at saveAsTextFile at LDAModel.scala:418), which has no missing parents -2016-06-13 19:45:36 INFO MemoryStore:59 - ensureFreeSpace(126936) called with curMem=389330, maxMem=766075207 -2016-06-13 19:45:36 INFO MemoryStore:59 - Block broadcast_1340 stored as values in memory (estimated size 124.0 KB, free 730.1 MB) -2016-06-13 19:45:36 INFO MemoryStore:59 - ensureFreeSpace(42521) called with curMem=516266, maxMem=766075207 -2016-06-13 19:45:36 INFO MemoryStore:59 - Block broadcast_1340_piece0 stored as bytes in memory (estimated size 41.5 KB, free 730.1 MB) -2016-06-13 19:45:36 INFO BlockManagerInfo:59 - Added broadcast_1340_piece0 in memory on localhost:50000 (size: 41.5 KB, free: 730.4 MB) -2016-06-13 19:45:36 INFO SparkContext:59 - Created broadcast 1340 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:36 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1244 (MapPartitionsRDD[1453] at saveAsTextFile at LDAModel.scala:418) -2016-06-13 19:45:36 INFO TaskSchedulerImpl:59 - Adding task set 1244.0 with 1 tasks -2016-06-13 19:45:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 1244.0 (TID 3045, localhost, PROCESS_LOCAL, 2274 bytes) -2016-06-13 19:45:36 INFO Executor:59 - Running task 0.0 in stage 1244.0 (TID 3045) -2016-06-13 19:45:36 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201606131945_1244_m_000000_3045' to file:/C:/Users/xubo/AppData/Local/Temp/spark-0dba2b03-76f0-4fdb-a7a8-90853794bdfa/metadata/_temporary/0/task_201606131945_1244_m_000000 -2016-06-13 19:45:36 INFO SparkHadoopMapRedUtil:59 - attempt_201606131945_1244_m_000000_3045: Committed -2016-06-13 19:45:36 INFO Executor:59 - Finished task 0.0 in stage 1244.0 (TID 3045). 1864 bytes result sent to driver -2016-06-13 19:45:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 1244.0 (TID 3045) in 642 ms on localhost (1/1) -2016-06-13 19:45:36 INFO DAGScheduler:59 - ResultStage 1244 (saveAsTextFile at LDAModel.scala:418) finished in 0.642 s -2016-06-13 19:45:36 INFO DAGScheduler:59 - Job 1003 finished: saveAsTextFile at LDAModel.scala:418, took 0.703830 s -2016-06-13 19:45:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 1244.0, whose tasks have all completed, from pool -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1340_piece0 on localhost:50000 in memory (size: 41.5 KB, free: 730.4 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1038 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1339_piece0 on localhost:50000 in memory (size: 3.4 KB, free: 730.4 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1037 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1338_piece0 on localhost:50000 in memory (size: 2.2 KB, free: 730.4 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1036 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1337_piece0 on localhost:50000 in memory (size: 3.0 KB, free: 730.4 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1035 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1336_piece0 on localhost:50000 in memory (size: 3.3 KB, free: 730.4 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1034 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1335_piece0 on localhost:50000 in memory (size: 2.2 KB, free: 730.4 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1033 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1334_piece0 on localhost:50000 in memory (size: 2.9 KB, free: 730.4 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1032 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1333_piece0 on localhost:50000 in memory (size: 3.2 KB, free: 730.4 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1031 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1332_piece0 on localhost:50000 in memory (size: 2.2 KB, free: 730.4 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1030 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1331_piece0 on localhost:50000 in memory (size: 2.8 KB, free: 730.4 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1029 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1330_piece0 on localhost:50000 in memory (size: 3.1 KB, free: 730.4 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1028 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1329_piece0 on localhost:50000 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1027 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1328_piece0 on localhost:50000 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1026 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1327_piece0 on localhost:50000 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1025 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1326_piece0 on localhost:50000 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1024 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1325_piece0 on localhost:50000 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1023 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1324_piece0 on localhost:50000 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1022 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1323_piece0 on localhost:50000 in memory (size: 1704.0 B, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1021 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1322_piece0 on localhost:50000 in memory (size: 1726.0 B, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1020 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1321_piece0 on localhost:50000 in memory (size: 1829.0 B, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1019 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1320_piece0 on localhost:50000 in memory (size: 2.1 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1018 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1319_piece0 on localhost:50000 in memory (size: 807.0 B, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1017 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1318_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1016 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1317_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1015 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1316_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1315_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1014 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1314_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1013 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1313_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1012 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1312_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1311_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1011 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1310_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1010 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1309_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1009 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1308_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1307_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1008 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1306_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1007 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1305_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1006 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1304_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1303_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1005 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1302_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1004 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1301_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1003 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1300_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1299_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1002 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1298_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1001 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1297_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 1000 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1296_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1295_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 999 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1294_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 998 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1293_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 997 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1292_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1291_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 996 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1290_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 995 -2016-06-13 19:45:37 INFO BlockManagerInfo:59 - Removed broadcast_1289_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:37 INFO ContextCleaner:59 - Cleaned accumulator 994 -2016-06-13 19:45:38 INFO BlockManagerInfo:59 - Removed broadcast_1288_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:38 INFO BlockManagerInfo:59 - Removed broadcast_1287_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 993 -2016-06-13 19:45:38 INFO BlockManagerInfo:59 - Removed broadcast_1286_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 992 -2016-06-13 19:45:38 INFO BlockManagerInfo:59 - Removed broadcast_1285_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 991 -2016-06-13 19:45:38 INFO BlockManagerInfo:59 - Removed broadcast_1284_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:38 INFO BlockManagerInfo:59 - Removed broadcast_1283_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 990 -2016-06-13 19:45:38 INFO BlockManagerInfo:59 - Removed broadcast_1282_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 989 -2016-06-13 19:45:38 INFO BlockManagerInfo:59 - Removed broadcast_1281_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 988 -2016-06-13 19:45:38 INFO BlockManagerInfo:59 - Removed broadcast_1280_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:38 INFO BlockManagerInfo:59 - Removed broadcast_1279_piece0 on localhost:50000 in memory (size: 1317.0 B, free: 730.5 MB) -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 987 -2016-06-13 19:45:38 INFO BlockManagerInfo:59 - Removed broadcast_1278_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 986 -2016-06-13 19:45:38 INFO BlockManagerInfo:59 - Removed broadcast_1277_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 985 -2016-06-13 19:45:38 INFO BlockManagerInfo:59 - Removed broadcast_1276_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:38 INFO BlockManagerInfo:59 - Removed broadcast_1275_piece0 on localhost:50000 in memory (size: 1315.0 B, free: 730.5 MB) -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 984 -2016-06-13 19:45:38 INFO BlockManagerInfo:59 - Removed broadcast_1274_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 983 -2016-06-13 19:45:38 INFO BlockManagerInfo:59 - Removed broadcast_1273_piece0 on localhost:50000 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 982 -2016-06-13 19:45:38 INFO BlockManagerInfo:59 - Removed broadcast_1272_piece0 on localhost:50000 in memory (size: 280.0 B, free: 730.5 MB) -2016-06-13 19:45:38 INFO ParquetRelation:59 - Using default output committer for Parquet: org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned shuffle 17 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned shuffle 13 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned shuffle 12 -2016-06-13 19:45:38 INFO BlockManager:59 - Removing RDD 87 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned RDD 87 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 19:45:38 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned shuffle 11 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned shuffle 10 -2016-06-13 19:45:38 INFO BlockManager:59 - Removing RDD 74 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned RDD 74 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned shuffle 9 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned shuffle 8 -2016-06-13 19:45:38 INFO BlockManager:59 - Removing RDD 61 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned RDD 61 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned shuffle 7 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned shuffle 6 -2016-06-13 19:45:38 INFO BlockManager:59 - Removing RDD 48 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned RDD 48 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned shuffle 5 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned shuffle 4 -2016-06-13 19:45:38 INFO BlockManager:59 - Removing RDD 35 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned RDD 35 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-06-13 19:45:38 INFO BlockManager:59 - Removing RDD 22 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned RDD 22 -2016-06-13 19:45:38 INFO BlockManager:59 - Removing RDD 17 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned RDD 17 -2016-06-13 19:45:38 INFO BlockManager:59 - Removing RDD 13 -2016-06-13 19:45:38 INFO SparkContext:59 - Starting job: parquet at LDAModel.scala:424 -2016-06-13 19:45:38 INFO DAGScheduler:59 - Got job 1004 (parquet at LDAModel.scala:424) with 1 output partitions -2016-06-13 19:45:38 INFO DAGScheduler:59 - Final stage: ResultStage 1245(parquet at LDAModel.scala:424) -2016-06-13 19:45:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:45:38 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:45:38 INFO DAGScheduler:59 - Submitting ResultStage 1245 (MapPartitionsRDD[1455] at rddToDataFrameHolder at LDAModel.scala:424), which has no missing parents -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned RDD 13 -2016-06-13 19:45:38 INFO BlockManager:59 - Removing RDD 10 -2016-06-13 19:45:38 INFO ContextCleaner:59 - Cleaned RDD 10 -2016-06-13 19:45:38 INFO MemoryStore:59 - ensureFreeSpace(66896) called with curMem=26048, maxMem=766075207 -2016-06-13 19:45:38 INFO MemoryStore:59 - Block broadcast_1341 stored as values in memory (estimated size 65.3 KB, free 730.5 MB) -2016-06-13 19:45:38 INFO MemoryStore:59 - ensureFreeSpace(23361) called with curMem=92944, maxMem=766075207 -2016-06-13 19:45:38 INFO MemoryStore:59 - Block broadcast_1341_piece0 stored as bytes in memory (estimated size 22.8 KB, free 730.5 MB) -2016-06-13 19:45:38 INFO BlockManagerInfo:59 - Added broadcast_1341_piece0 in memory on localhost:50000 (size: 22.8 KB, free: 730.5 MB) -2016-06-13 19:45:38 INFO SparkContext:59 - Created broadcast 1341 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:45:38 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1245 (MapPartitionsRDD[1455] at rddToDataFrameHolder at LDAModel.scala:424) -2016-06-13 19:45:38 INFO TaskSchedulerImpl:59 - Adding task set 1245.0 with 1 tasks -2016-06-13 19:45:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 1245.0 (TID 3046, localhost, PROCESS_LOCAL, 2546 bytes) -2016-06-13 19:45:38 INFO Executor:59 - Running task 0.0 in stage 1245.0 (TID 3046) -2016-06-13 19:45:38 INFO DefaultWriterContainer:59 - Using user defined output committer class org.apache.parquet.hadoop.ParquetOutputCommitter -2016-06-13 19:45:38 INFO CodecConfig:151 - Compression: GZIP -2016-06-13 19:45:38 INFO ParquetOutputFormat:151 - Parquet block size to 134217728 -2016-06-13 19:45:38 INFO ParquetOutputFormat:151 - Parquet page size to 1048576 -2016-06-13 19:45:38 INFO ParquetOutputFormat:151 - Parquet dictionary page size to 1048576 -2016-06-13 19:45:38 INFO ParquetOutputFormat:151 - Dictionary is on -2016-06-13 19:45:38 INFO ParquetOutputFormat:151 - Validation is off -2016-06-13 19:45:38 INFO ParquetOutputFormat:151 - Writer version is: PARQUET_1_0 -2016-06-13 19:45:38 INFO CodecPool:151 - Got brand-new compressor [.gz] -2016-06-13 19:47:02 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 19:47:03 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 19:47:03 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 19:47:03 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 19:47:03 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 19:47:04 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 19:47:04 INFO Remoting:74 - Starting remoting -2016-06-13 19:47:04 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:50033] -2016-06-13 19:47:04 INFO Utils:59 - Successfully started service 'sparkDriver' on port 50033. -2016-06-13 19:47:04 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 19:47:04 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 19:47:04 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-8007fff7-8ce1-4be7-81f7-643c73ebbee7 -2016-06-13 19:47:04 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 19:47:04 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-b2990e26-46d0-4b19-9771-d7a6e8f266ee\httpd-62293421-4264-413f-814f-f09c5a4908f4 -2016-06-13 19:47:04 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 19:47:04 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:47:04 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:50034 -2016-06-13 19:47:04 INFO Utils:59 - Successfully started service 'HTTP file server' on port 50034. -2016-06-13 19:47:04 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 19:47:05 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:47:05 WARN AbstractLifeCycle:204 - FAILED SelectChannelConnector@0.0.0.0:4040: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.util.SparkLearningFunSuite.beforeAll(SparkLearningFunSuite.scala:23) - at org.scalatest.BeforeAndAfterAll$class.beforeAll(BeforeAndAfterAll.scala:187) - at org.apache.spark.util.SparkLearningFunSuite.beforeAll(SparkLearningFunSuite.scala:10) - at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:253) - at org.apache.spark.util.SparkLearningFunSuite.run(SparkLearningFunSuite.scala:10) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 19:47:05 WARN AbstractLifeCycle:204 - FAILED org.spark-project.jetty.server.Server@73a47985: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.util.SparkLearningFunSuite.beforeAll(SparkLearningFunSuite.scala:23) - at org.scalatest.BeforeAndAfterAll$class.beforeAll(BeforeAndAfterAll.scala:187) - at org.apache.spark.util.SparkLearningFunSuite.beforeAll(SparkLearningFunSuite.scala:10) - at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:253) - at org.apache.spark.util.SparkLearningFunSuite.run(SparkLearningFunSuite.scala:10) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 19:47:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 19:47:05 WARN Utils:71 - Service 'SparkUI' could not bind on port 4040. Attempting port 4041. -2016-06-13 19:47:05 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:47:05 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4041 -2016-06-13 19:47:05 INFO Utils:59 - Successfully started service 'SparkUI' on port 4041. -2016-06-13 19:47:05 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4041 -2016-06-13 19:47:05 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 19:47:05 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 19:47:05 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 50053. -2016-06-13 19:47:05 INFO NettyBlockTransferService:59 - Server created on 50053 -2016-06-13 19:47:05 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 19:47:05 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:50053 with 730.6 MB RAM, BlockManagerId(driver, localhost, 50053) -2016-06-13 19:47:05 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 19:47:06 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 19:47:06 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 19:47:06 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 19:47:06 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 19:47:06 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:50053 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 19:47:06 INFO SparkContext:59 - Created broadcast 0 from textFile at LDAFromWebSuite.scala:22 -2016-06-13 19:47:07 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 19:47:07 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 19:47:07 INFO SparkContext:59 - Starting job: zipWithIndex at LDAFromWebSuite.scala:25 -2016-06-13 19:47:07 INFO DAGScheduler:59 - Got job 0 (zipWithIndex at LDAFromWebSuite.scala:25) with 1 output partitions -2016-06-13 19:47:07 INFO DAGScheduler:59 - Final stage: ResultStage 0(zipWithIndex at LDAFromWebSuite.scala:25) -2016-06-13 19:47:07 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:47:07 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:47:07 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at LDAFromWebSuite.scala:23), which has no missing parents -2016-06-13 19:47:08 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=171580, maxMem=766075207 -2016-06-13 19:47:08 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:47:08 INFO MemoryStore:59 - ensureFreeSpace(1939) called with curMem=174820, maxMem=766075207 -2016-06-13 19:47:08 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1939.0 B, free 730.4 MB) -2016-06-13 19:47:08 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:50053 (size: 1939.0 B, free: 730.6 MB) -2016-06-13 19:47:08 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:08 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at LDAFromWebSuite.scala:23) -2016-06-13 19:47:08 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 19:47:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2197 bytes) -2016-06-13 19:47:08 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 19:47:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:0+132 -2016-06-13 19:47:08 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 19:47:08 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 19:47:08 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 19:47:08 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 19:47:08 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 19:47:08 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 19:47:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 100 ms on localhost (1/1) -2016-06-13 19:47:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 19:47:08 INFO DAGScheduler:59 - ResultStage 0 (zipWithIndex at LDAFromWebSuite.scala:25) finished in 0.114 s -2016-06-13 19:47:08 INFO DAGScheduler:59 - Job 0 finished: zipWithIndex at LDAFromWebSuite.scala:25, took 0.287485 s -2016-06-13 19:47:08 INFO SparkContext:59 - Starting job: take at LDAOptimizer.scala:142 -2016-06-13 19:47:08 INFO DAGScheduler:59 - Got job 1 (take at LDAOptimizer.scala:142) with 1 output partitions -2016-06-13 19:47:08 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at LDAOptimizer.scala:142) -2016-06-13 19:47:08 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:47:08 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:47:08 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at map at LDAFromWebSuite.scala:25), which has no missing parents -2016-06-13 19:47:08 INFO MemoryStore:59 - ensureFreeSpace(3696) called with curMem=176759, maxMem=766075207 -2016-06-13 19:47:08 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-06-13 19:47:08 INFO MemoryStore:59 - ensureFreeSpace(2071) called with curMem=180455, maxMem=766075207 -2016-06-13 19:47:08 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:47:08 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:50053 (size: 2.0 KB, free: 730.6 MB) -2016-06-13 19:47:08 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:08 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at map at LDAFromWebSuite.scala:25) -2016-06-13 19:47:08 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 19:47:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2307 bytes) -2016-06-13 19:47:08 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 19:47:08 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-06-13 19:47:08 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:0+132 -2016-06-13 19:47:08 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=182526, maxMem=766075207 -2016-06-13 19:47:08 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 1224.0 B, free 730.4 MB) -2016-06-13 19:47:08 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:50053 (size: 1224.0 B, free: 730.6 MB) -2016-06-13 19:47:08 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2949 bytes result sent to driver -2016-06-13 19:47:08 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 52 ms on localhost (1/1) -2016-06-13 19:47:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 19:47:08 INFO DAGScheduler:59 - ResultStage 1 (take at LDAOptimizer.scala:142) finished in 0.062 s -2016-06-13 19:47:08 INFO DAGScheduler:59 - Job 1 finished: take at LDAOptimizer.scala:142, took 0.100602 s -2016-06-13 19:47:08 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:08 INFO DAGScheduler:59 - Registering RDD 19 (map at GraphImpl.scala:104) -2016-06-13 19:47:08 INFO DAGScheduler:59 - Registering RDD 23 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:47:08 INFO DAGScheduler:59 - Registering RDD 11 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:47:08 INFO DAGScheduler:59 - Registering RDD 6 (mapPartitionsWithIndex at LDAOptimizer.scala:128) -2016-06-13 19:47:08 INFO DAGScheduler:59 - Got job 2 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:08 INFO DAGScheduler:59 - Final stage: ResultStage 6(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:08 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 5, ShuffleMapStage 3, ShuffleMapStage 4) -2016-06-13 19:47:08 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 5, ShuffleMapStage 3, ShuffleMapStage 4) -2016-06-13 19:47:08 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[19] at map at GraphImpl.scala:104), which has no missing parents -2016-06-13 19:47:08 INFO MemoryStore:59 - ensureFreeSpace(5752) called with curMem=183750, maxMem=766075207 -2016-06-13 19:47:08 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.6 KB, free 730.4 MB) -2016-06-13 19:47:08 INFO MemoryStore:59 - ensureFreeSpace(3051) called with curMem=189502, maxMem=766075207 -2016-06-13 19:47:08 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 19:47:08 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:50053 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 19:47:08 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[19] at map at GraphImpl.scala:104) -2016-06-13 19:47:08 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 19:47:08 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[11] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:47:08 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:47:08 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=192553, maxMem=766075207 -2016-06-13 19:47:08 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:47:08 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.4 MB) -2016-06-13 19:47:08 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-06-13 19:47:08 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 3) -2016-06-13 19:47:08 INFO MemoryStore:59 - ensureFreeSpace(2760) called with curMem=197617, maxMem=766075207 -2016-06-13 19:47:08 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:47:08 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:50053 (size: 2.7 KB, free: 730.6 MB) -2016-06-13 19:47:08 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:08 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[11] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:47:08 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 19:47:09 INFO DAGScheduler:59 - Submitting ShuffleMapStage 5 (MapPartitionsRDD[6] at mapPartitionsWithIndex at LDAOptimizer.scala:128), which has no missing parents -2016-06-13 19:47:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 4, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:47:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 5, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:47:09 INFO MemoryStore:59 - ensureFreeSpace(4880) called with curMem=200377, maxMem=766075207 -2016-06-13 19:47:09 INFO CacheManager:59 - Partition rdd_17_0 not found, computing it -2016-06-13 19:47:09 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:47:09 INFO CacheManager:59 - Partition rdd_17_1 not found, computing it -2016-06-13 19:47:09 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-06-13 19:47:09 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-06-13 19:47:09 INFO MemoryStore:59 - ensureFreeSpace(2640) called with curMem=205257, maxMem=766075207 -2016-06-13 19:47:09 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-06-13 19:47:09 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:50053 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:47:09 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 4) -2016-06-13 19:47:09 INFO MemoryStore:59 - ensureFreeSpace(3520) called with curMem=207897, maxMem=766075207 -2016-06-13 19:47:09 INFO MemoryStore:59 - Block rdd_17_0 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-06-13 19:47:09 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 5 (MapPartitionsRDD[6] at mapPartitionsWithIndex at LDAOptimizer.scala:128) -2016-06-13 19:47:09 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 19:47:09 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 5) -2016-06-13 19:47:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:132+132 -2016-06-13 19:47:09 INFO BlockManagerInfo:59 - Added rdd_17_0 in memory on localhost:50053 (size: 3.4 KB, free: 730.6 MB) -2016-06-13 19:47:09 INFO CacheManager:59 - Another thread is loading rdd_4_1, waiting for it to finish... -2016-06-13 19:47:09 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:47:09 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=211417, maxMem=766075207 -2016-06-13 19:47:09 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 880.0 B, free 730.4 MB) -2016-06-13 19:47:09 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 2834 bytes result sent to driver -2016-06-13 19:47:09 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 4). 2254 bytes result sent to driver -2016-06-13 19:47:09 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:50053 (size: 880.0 B, free: 730.6 MB) -2016-06-13 19:47:09 INFO CacheManager:59 - Finished waiting for rdd_4_1 -2016-06-13 19:47:09 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 19:47:09 INFO MemoryStore:59 - ensureFreeSpace(3296) called with curMem=212297, maxMem=766075207 -2016-06-13 19:47:09 INFO MemoryStore:59 - Block rdd_17_1 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:47:09 INFO BlockManagerInfo:59 - Added rdd_17_1 in memory on localhost:50053 (size: 3.2 KB, free: 730.6 MB) -2016-06-13 19:47:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 6, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:47:09 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 6) -2016-06-13 19:47:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 7, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:47:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 344 ms on localhost (1/2) -2016-06-13 19:47:09 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:47:09 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 5). 1159 bytes result sent to driver -2016-06-13 19:47:09 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 7) -2016-06-13 19:47:09 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 19:47:09 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 19:47:09 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 19:47:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 5) in 381 ms on localhost (1/2) -2016-06-13 19:47:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 4) in 402 ms on localhost (2/2) -2016-06-13 19:47:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 19:47:09 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at VertexRDD.scala:358) finished in 0.410 s -2016-06-13 19:47:09 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:09 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 5, ShuffleMapStage 2) -2016-06-13 19:47:09 INFO DAGScheduler:59 - waiting: Set(ResultStage 6, ShuffleMapStage 3) -2016-06-13 19:47:09 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:09 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 3). 2889 bytes result sent to driver -2016-06-13 19:47:09 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List(ShuffleMapStage 5, ShuffleMapStage 3) -2016-06-13 19:47:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 3) in 503 ms on localhost (2/2) -2016-06-13 19:47:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 19:47:09 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List(ShuffleMapStage 2) -2016-06-13 19:47:09 INFO DAGScheduler:59 - ShuffleMapStage 2 (map at GraphImpl.scala:104) finished in 0.514 s -2016-06-13 19:47:09 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:09 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 5) -2016-06-13 19:47:09 INFO DAGScheduler:59 - waiting: Set(ResultStage 6, ShuffleMapStage 3) -2016-06-13 19:47:09 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:09 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List(ShuffleMapStage 5, ShuffleMapStage 3) -2016-06-13 19:47:09 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List() -2016-06-13 19:47:09 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[23] at mapPartitions at VertexRDD.scala:358), which is now runnable -2016-06-13 19:47:09 INFO MemoryStore:59 - ensureFreeSpace(2784) called with curMem=215593, maxMem=766075207 -2016-06-13 19:47:09 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:47:09 INFO MemoryStore:59 - ensureFreeSpace(1698) called with curMem=218377, maxMem=766075207 -2016-06-13 19:47:09 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1698.0 B, free 730.4 MB) -2016-06-13 19:47:09 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:50053 (size: 1698.0 B, free: 730.6 MB) -2016-06-13 19:47:09 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 3 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[23] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:47:09 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 19:47:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 1890 bytes) -2016-06-13 19:47:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 1890 bytes) -2016-06-13 19:47:09 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 9) -2016-06-13 19:47:09 INFO CacheManager:59 - Partition rdd_21_1 not found, computing it -2016-06-13 19:47:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 31 ms -2016-06-13 19:47:09 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 7). 2254 bytes result sent to driver -2016-06-13 19:47:09 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 6). 2254 bytes result sent to driver -2016-06-13 19:47:09 INFO MemoryStore:59 - ensureFreeSpace(3416) called with curMem=220075, maxMem=766075207 -2016-06-13 19:47:09 INFO MemoryStore:59 - Block rdd_21_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 19:47:09 INFO BlockManagerInfo:59 - Added rdd_21_1 in memory on localhost:50053 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:47:09 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 8) -2016-06-13 19:47:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 6) in 323 ms on localhost (1/2) -2016-06-13 19:47:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 7) in 302 ms on localhost (2/2) -2016-06-13 19:47:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 19:47:09 INFO CacheManager:59 - Partition rdd_21_0 not found, computing it -2016-06-13 19:47:09 INFO DAGScheduler:59 - ShuffleMapStage 5 (mapPartitionsWithIndex at LDAOptimizer.scala:128) finished in 0.464 s -2016-06-13 19:47:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:09 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:09 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 3) -2016-06-13 19:47:09 INFO DAGScheduler:59 - waiting: Set(ResultStage 6) -2016-06-13 19:47:09 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-06-13 19:47:09 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List(ShuffleMapStage 3) -2016-06-13 19:47:09 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 9). 1955 bytes result sent to driver -2016-06-13 19:47:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 9) in 162 ms on localhost (1/2) -2016-06-13 19:47:09 INFO MemoryStore:59 - ensureFreeSpace(3408) called with curMem=223491, maxMem=766075207 -2016-06-13 19:47:09 INFO MemoryStore:59 - Block rdd_21_0 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 19:47:09 INFO BlockManagerInfo:59 - Added rdd_21_0 in memory on localhost:50053 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:47:09 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 8). 1955 bytes result sent to driver -2016-06-13 19:47:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 8) in 212 ms on localhost (2/2) -2016-06-13 19:47:09 INFO DAGScheduler:59 - ShuffleMapStage 3 (mapPartitions at VertexRDD.scala:358) finished in 0.212 s -2016-06-13 19:47:09 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:09 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:09 INFO DAGScheduler:59 - waiting: Set(ResultStage 6) -2016-06-13 19:47:09 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 19:47:09 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List() -2016-06-13 19:47:09 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[30] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:09 INFO MemoryStore:59 - ensureFreeSpace(5584) called with curMem=226899, maxMem=766075207 -2016-06-13 19:47:09 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.4 MB) -2016-06-13 19:47:09 INFO MemoryStore:59 - ensureFreeSpace(2802) called with curMem=232483, maxMem=766075207 -2016-06-13 19:47:09 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:47:09 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:50053 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:47:09 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[30] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:09 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-06-13 19:47:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 10, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:47:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:47:09 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 11) -2016-06-13 19:47:09 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 10) -2016-06-13 19:47:09 INFO CacheManager:59 - Partition rdd_26_1 not found, computing it -2016-06-13 19:47:09 INFO CacheManager:59 - Partition rdd_14_1 not found, computing it -2016-06-13 19:47:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:09 INFO CacheManager:59 - Partition rdd_26_0 not found, computing it -2016-06-13 19:47:09 INFO CacheManager:59 - Partition rdd_14_0 not found, computing it -2016-06-13 19:47:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:09 INFO MemoryStore:59 - ensureFreeSpace(2400) called with curMem=235285, maxMem=766075207 -2016-06-13 19:47:09 INFO MemoryStore:59 - Block rdd_14_1 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 19:47:09 INFO BlockManagerInfo:59 - Added rdd_14_1 in memory on localhost:50053 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:47:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:09 INFO MemoryStore:59 - ensureFreeSpace(2400) called with curMem=237685, maxMem=766075207 -2016-06-13 19:47:09 INFO MemoryStore:59 - Block rdd_26_1 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 19:47:09 INFO MemoryStore:59 - ensureFreeSpace(2496) called with curMem=240085, maxMem=766075207 -2016-06-13 19:47:09 INFO BlockManagerInfo:59 - Added rdd_26_1 in memory on localhost:50053 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:47:09 INFO MemoryStore:59 - Block rdd_14_0 stored as values in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 19:47:09 INFO BlockManagerInfo:59 - Added rdd_14_0 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:09 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:09 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:09 INFO MemoryStore:59 - ensureFreeSpace(2496) called with curMem=242581, maxMem=766075207 -2016-06-13 19:47:09 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 11). 1931 bytes result sent to driver -2016-06-13 19:47:09 INFO MemoryStore:59 - Block rdd_26_0 stored as values in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 19:47:09 INFO BlockManagerInfo:59 - Added rdd_26_0 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 11) in 85 ms on localhost (1/2) -2016-06-13 19:47:09 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 10). 1931 bytes result sent to driver -2016-06-13 19:47:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 10) in 137 ms on localhost (2/2) -2016-06-13 19:47:09 INFO DAGScheduler:59 - ResultStage 6 (fold at LDAOptimizer.scala:205) finished in 0.140 s -2016-06-13 19:47:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 19:47:09 INFO DAGScheduler:59 - Job 2 finished: fold at LDAOptimizer.scala:205, took 0.991257 s -2016-06-13 19:47:10 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:10 INFO DAGScheduler:59 - Registering RDD 31 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:10 INFO DAGScheduler:59 - Registering RDD 35 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:10 INFO DAGScheduler:59 - Got job 3 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:10 INFO DAGScheduler:59 - Final stage: ResultStage 13(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12, ShuffleMapStage 10, ShuffleMapStage 7, ShuffleMapStage 8) -2016-06-13 19:47:10 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-06-13 19:47:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 11 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[31] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:10 INFO MemoryStore:59 - ensureFreeSpace(4360) called with curMem=245077, maxMem=766075207 -2016-06-13 19:47:10 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:47:10 INFO MemoryStore:59 - ensureFreeSpace(2388) called with curMem=249437, maxMem=766075207 -2016-06-13 19:47:10 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.3 MB) -2016-06-13 19:47:10 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:50053 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:47:10 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 11 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[31] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:10 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 19:47:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 12, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:47:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 13, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:47:10 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 12) -2016-06-13 19:47:10 INFO BlockManager:59 - Found block rdd_26_0 locally -2016-06-13 19:47:10 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 12). 2254 bytes result sent to driver -2016-06-13 19:47:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 12) in 66 ms on localhost (1/2) -2016-06-13 19:47:10 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 13) -2016-06-13 19:47:10 INFO BlockManager:59 - Found block rdd_26_1 locally -2016-06-13 19:47:10 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:50053 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:47:10 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 13). 2254 bytes result sent to driver -2016-06-13 19:47:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 13) in 107 ms on localhost (2/2) -2016-06-13 19:47:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 19:47:10 INFO DAGScheduler:59 - ShuffleMapStage 11 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.110 s -2016-06-13 19:47:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:10 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:10 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 12, ResultStage 13) -2016-06-13 19:47:10 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:10 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 12: List() -2016-06-13 19:47:10 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List(ShuffleMapStage 12) -2016-06-13 19:47:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[35] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:10 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 19:47:10 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=243439, maxMem=766075207 -2016-06-13 19:47:10 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 19:47:10 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=247335, maxMem=766075207 -2016-06-13 19:47:10 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:47:10 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:50053 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:10 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:10 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:50053 in memory (size: 1698.0 B, free: 730.5 MB) -2016-06-13 19:47:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[35] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:10 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 19:47:10 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 19:47:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 14, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 15, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:10 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 14) -2016-06-13 19:47:10 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 15) -2016-06-13 19:47:10 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:50053 in memory (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:47:10 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:10 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 19:47:10 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:10 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 14). 2470 bytes result sent to driver -2016-06-13 19:47:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 14) in 79 ms on localhost (1/2) -2016-06-13 19:47:10 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 15). 2470 bytes result sent to driver -2016-06-13 19:47:10 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:50053 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:47:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 15) in 91 ms on localhost (2/2) -2016-06-13 19:47:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 19:47:10 INFO DAGScheduler:59 - ShuffleMapStage 12 (mapPartitions at GraphImpl.scala:235) finished in 0.095 s -2016-06-13 19:47:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:10 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:10 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-06-13 19:47:10 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:10 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 19:47:10 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:50053 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:47:10 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 19:47:10 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-06-13 19:47:10 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[43] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:10 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:50053 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:47:10 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 19:47:10 INFO MemoryStore:59 - ensureFreeSpace(6432) called with curMem=215225, maxMem=766075207 -2016-06-13 19:47:10 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 6.3 KB, free 730.4 MB) -2016-06-13 19:47:10 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:50053 in memory (size: 1939.0 B, free: 730.5 MB) -2016-06-13 19:47:10 INFO MemoryStore:59 - ensureFreeSpace(3040) called with curMem=216478, maxMem=766075207 -2016-06-13 19:47:10 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 19:47:10 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 19:47:10 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:50053 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:47:10 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[43] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:10 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 19:47:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 16, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:47:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:47:10 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 17) -2016-06-13 19:47:10 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 16) -2016-06-13 19:47:10 INFO CacheManager:59 - Partition rdd_39_0 not found, computing it -2016-06-13 19:47:10 INFO BlockManager:59 - Found block rdd_26_0 locally -2016-06-13 19:47:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:10 INFO CacheManager:59 - Partition rdd_39_1 not found, computing it -2016-06-13 19:47:10 INFO BlockManager:59 - Found block rdd_26_1 locally -2016-06-13 19:47:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:10 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=219518, maxMem=766075207 -2016-06-13 19:47:10 INFO MemoryStore:59 - Block rdd_39_0 stored as values in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 19:47:10 INFO BlockManagerInfo:59 - Added rdd_39_0 in memory on localhost:50053 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:10 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 16). 2971 bytes result sent to driver -2016-06-13 19:47:10 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=222062, maxMem=766075207 -2016-06-13 19:47:10 INFO MemoryStore:59 - Block rdd_39_1 stored as values in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 19:47:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 16) in 36 ms on localhost (1/2) -2016-06-13 19:47:10 INFO BlockManagerInfo:59 - Added rdd_39_1 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:10 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 17). 2971 bytes result sent to driver -2016-06-13 19:47:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 17) in 41 ms on localhost (2/2) -2016-06-13 19:47:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 19:47:10 INFO DAGScheduler:59 - ResultStage 13 (fold at LDAOptimizer.scala:205) finished in 0.044 s -2016-06-13 19:47:10 INFO DAGScheduler:59 - Job 3 finished: fold at LDAOptimizer.scala:205, took 0.468400 s -2016-06-13 19:47:10 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:10 INFO DAGScheduler:59 - Registering RDD 44 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:10 INFO DAGScheduler:59 - Registering RDD 48 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:10 INFO DAGScheduler:59 - Got job 4 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:10 INFO DAGScheduler:59 - Final stage: ResultStage 22(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15, ShuffleMapStage 19, ShuffleMapStage 17, ShuffleMapStage 21, ShuffleMapStage 14) -2016-06-13 19:47:10 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 21) -2016-06-13 19:47:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 20 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[44] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:10 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=224510, maxMem=766075207 -2016-06-13 19:47:10 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.3 KB, free 730.4 MB) -2016-06-13 19:47:10 INFO MemoryStore:59 - ensureFreeSpace(2727) called with curMem=229958, maxMem=766075207 -2016-06-13 19:47:10 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:47:10 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:50053 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:47:10 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 20 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[44] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:10 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-06-13 19:47:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 18, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:47:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 19, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:47:10 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 18) -2016-06-13 19:47:10 INFO BlockManager:59 - Found block rdd_39_0 locally -2016-06-13 19:47:10 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 19) -2016-06-13 19:47:10 INFO BlockManager:59 - Found block rdd_39_1 locally -2016-06-13 19:47:10 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 18). 2254 bytes result sent to driver -2016-06-13 19:47:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 18) in 48 ms on localhost (1/2) -2016-06-13 19:47:10 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 19). 2254 bytes result sent to driver -2016-06-13 19:47:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 19) in 68 ms on localhost (2/2) -2016-06-13 19:47:10 INFO DAGScheduler:59 - ShuffleMapStage 20 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.070 s -2016-06-13 19:47:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 19:47:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:10 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:10 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 21, ResultStage 22) -2016-06-13 19:47:10 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:10 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 21: List() -2016-06-13 19:47:10 INFO DAGScheduler:59 - Missing parents for ResultStage 22: List(ShuffleMapStage 21) -2016-06-13 19:47:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 21 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[48] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:10 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=232685, maxMem=766075207 -2016-06-13 19:47:10 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 19:47:10 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=236581, maxMem=766075207 -2016-06-13 19:47:10 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 19:47:10 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:50053 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:10 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 21 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[48] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:10 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 19:47:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 20, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 21, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:10 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 21) -2016-06-13 19:47:10 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 20) -2016-06-13 19:47:10 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:10 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:10 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 21). 2470 bytes result sent to driver -2016-06-13 19:47:10 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 20). 2470 bytes result sent to driver -2016-06-13 19:47:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 21) in 42 ms on localhost (1/2) -2016-06-13 19:47:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 20) in 45 ms on localhost (2/2) -2016-06-13 19:47:10 INFO DAGScheduler:59 - ShuffleMapStage 21 (mapPartitions at GraphImpl.scala:235) finished in 0.045 s -2016-06-13 19:47:10 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:10 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:10 INFO DAGScheduler:59 - waiting: Set(ResultStage 22) -2016-06-13 19:47:10 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 19:47:10 INFO DAGScheduler:59 - Missing parents for ResultStage 22: List() -2016-06-13 19:47:10 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[56] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:10 INFO MemoryStore:59 - ensureFreeSpace(6864) called with curMem=238867, maxMem=766075207 -2016-06-13 19:47:10 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 6.7 KB, free 730.4 MB) -2016-06-13 19:47:10 INFO MemoryStore:59 - ensureFreeSpace(3157) called with curMem=245731, maxMem=766075207 -2016-06-13 19:47:10 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.3 MB) -2016-06-13 19:47:10 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:50053 (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:47:10 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[56] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:10 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-06-13 19:47:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 22, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:47:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 23, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:47:10 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 23) -2016-06-13 19:47:10 INFO CacheManager:59 - Partition rdd_52_1 not found, computing it -2016-06-13 19:47:10 INFO BlockManager:59 - Found block rdd_39_1 locally -2016-06-13 19:47:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:10 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=248888, maxMem=766075207 -2016-06-13 19:47:10 INFO MemoryStore:59 - Block rdd_52_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:47:10 INFO BlockManagerInfo:59 - Added rdd_52_1 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:10 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 22) -2016-06-13 19:47:10 INFO CacheManager:59 - Partition rdd_52_0 not found, computing it -2016-06-13 19:47:10 INFO BlockManager:59 - Found block rdd_39_0 locally -2016-06-13 19:47:10 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:10 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:10 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=251336, maxMem=766075207 -2016-06-13 19:47:10 INFO MemoryStore:59 - Block rdd_52_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:47:10 INFO BlockManagerInfo:59 - Added rdd_52_0 in memory on localhost:50053 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:10 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 23). 2971 bytes result sent to driver -2016-06-13 19:47:10 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 22). 2971 bytes result sent to driver -2016-06-13 19:47:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 23) in 64 ms on localhost (1/2) -2016-06-13 19:47:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 22) in 66 ms on localhost (2/2) -2016-06-13 19:47:10 INFO DAGScheduler:59 - ResultStage 22 (fold at LDAOptimizer.scala:205) finished in 0.067 s -2016-06-13 19:47:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 19:47:10 INFO DAGScheduler:59 - Job 4 finished: fold at LDAOptimizer.scala:205, took 0.270479 s -2016-06-13 19:47:10 INFO ZippedPartitionsRDD2:59 - Removing RDD 26 from persistence list -2016-06-13 19:47:10 INFO ZippedPartitionsRDD2:59 - Removing RDD 33 from persistence list -2016-06-13 19:47:10 INFO BlockManager:59 - Removing RDD 33 -2016-06-13 19:47:10 INFO BlockManager:59 - Removing RDD 26 -2016-06-13 19:47:10 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:10 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:10 INFO DAGScheduler:59 - Registering RDD 57 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:10 INFO DAGScheduler:59 - Registering RDD 61 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:10 INFO DAGScheduler:59 - Got job 5 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:10 INFO DAGScheduler:59 - Final stage: ResultStage 33(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:10 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30, ShuffleMapStage 32, ShuffleMapStage 24, ShuffleMapStage 28, ShuffleMapStage 26, ShuffleMapStage 23) -2016-06-13 19:47:10 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 32) -2016-06-13 19:47:10 INFO DAGScheduler:59 - Submitting ShuffleMapStage 31 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[57] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:10 INFO MemoryStore:59 - ensureFreeSpace(5872) called with curMem=248984, maxMem=766075207 -2016-06-13 19:47:10 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 5.7 KB, free 730.3 MB) -2016-06-13 19:47:10 INFO MemoryStore:59 - ensureFreeSpace(2848) called with curMem=254856, maxMem=766075207 -2016-06-13 19:47:10 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-06-13 19:47:10 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:50053 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:47:10 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 31 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[57] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:10 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 19:47:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 24, localhost, PROCESS_LOCAL, 2196 bytes) -2016-06-13 19:47:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 25, localhost, PROCESS_LOCAL, 2196 bytes) -2016-06-13 19:47:10 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 25) -2016-06-13 19:47:10 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 24) -2016-06-13 19:47:10 INFO BlockManager:59 - Found block rdd_52_1 locally -2016-06-13 19:47:10 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 25). 2254 bytes result sent to driver -2016-06-13 19:47:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 25) in 33 ms on localhost (1/2) -2016-06-13 19:47:11 INFO BlockManager:59 - Found block rdd_52_0 locally -2016-06-13 19:47:11 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 24). 2254 bytes result sent to driver -2016-06-13 19:47:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 24) in 114 ms on localhost (2/2) -2016-06-13 19:47:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 19:47:11 INFO DAGScheduler:59 - ShuffleMapStage 31 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.146 s -2016-06-13 19:47:11 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:11 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:11 INFO DAGScheduler:59 - waiting: Set(ResultStage 33, ShuffleMapStage 32) -2016-06-13 19:47:11 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:11 INFO DAGScheduler:59 - Missing parents for ResultStage 33: List(ShuffleMapStage 32) -2016-06-13 19:47:11 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 32: List() -2016-06-13 19:47:11 INFO DAGScheduler:59 - Submitting ShuffleMapStage 32 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[61] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:11 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=257704, maxMem=766075207 -2016-06-13 19:47:11 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:11 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=261600, maxMem=766075207 -2016-06-13 19:47:11 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:47:11 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:50053 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:11 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 32 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[61] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:11 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-06-13 19:47:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 26, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 27, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:11 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 27) -2016-06-13 19:47:11 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 26) -2016-06-13 19:47:11 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:11 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:11 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 26). 2470 bytes result sent to driver -2016-06-13 19:47:11 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 27). 2470 bytes result sent to driver -2016-06-13 19:47:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 26) in 68 ms on localhost (1/2) -2016-06-13 19:47:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 27) in 67 ms on localhost (2/2) -2016-06-13 19:47:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 19:47:11 INFO DAGScheduler:59 - ShuffleMapStage 32 (mapPartitions at GraphImpl.scala:235) finished in 0.076 s -2016-06-13 19:47:11 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:11 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:11 INFO DAGScheduler:59 - waiting: Set(ResultStage 33) -2016-06-13 19:47:11 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:11 INFO DAGScheduler:59 - Missing parents for ResultStage 33: List() -2016-06-13 19:47:11 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[69] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:11 INFO MemoryStore:59 - ensureFreeSpace(7288) called with curMem=263886, maxMem=766075207 -2016-06-13 19:47:11 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 7.1 KB, free 730.3 MB) -2016-06-13 19:47:11 INFO MemoryStore:59 - ensureFreeSpace(3298) called with curMem=271174, maxMem=766075207 -2016-06-13 19:47:11 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.3 MB) -2016-06-13 19:47:11 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:50053 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:47:11 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[69] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:11 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-06-13 19:47:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 28, localhost, PROCESS_LOCAL, 2248 bytes) -2016-06-13 19:47:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 29, localhost, PROCESS_LOCAL, 2248 bytes) -2016-06-13 19:47:11 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 29) -2016-06-13 19:47:11 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 28) -2016-06-13 19:47:11 INFO CacheManager:59 - Partition rdd_65_0 not found, computing it -2016-06-13 19:47:11 INFO BlockManager:59 - Found block rdd_52_0 locally -2016-06-13 19:47:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:11 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=274472, maxMem=766075207 -2016-06-13 19:47:11 INFO MemoryStore:59 - Block rdd_65_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:47:11 INFO BlockManagerInfo:59 - Added rdd_65_0 in memory on localhost:50053 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:11 INFO CacheManager:59 - Partition rdd_65_1 not found, computing it -2016-06-13 19:47:11 INFO BlockManager:59 - Found block rdd_52_1 locally -2016-06-13 19:47:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:11 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=277016, maxMem=766075207 -2016-06-13 19:47:11 INFO MemoryStore:59 - Block rdd_65_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:47:11 INFO BlockManagerInfo:59 - Added rdd_65_1 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:11 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 29). 2971 bytes result sent to driver -2016-06-13 19:47:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 29) in 54 ms on localhost (1/2) -2016-06-13 19:47:11 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 28). 2971 bytes result sent to driver -2016-06-13 19:47:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 28) in 87 ms on localhost (2/2) -2016-06-13 19:47:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 19:47:11 INFO DAGScheduler:59 - ResultStage 33 (fold at LDAOptimizer.scala:205) finished in 0.179 s -2016-06-13 19:47:11 INFO DAGScheduler:59 - Job 5 finished: fold at LDAOptimizer.scala:205, took 0.541104 s -2016-06-13 19:47:11 INFO MapPartitionsRDD:59 - Removing RDD 39 from persistence list -2016-06-13 19:47:11 INFO BlockManager:59 - Removing RDD 39 -2016-06-13 19:47:11 INFO ZippedPartitionsRDD2:59 - Removing RDD 46 from persistence list -2016-06-13 19:47:11 INFO BlockManager:59 - Removing RDD 46 -2016-06-13 19:47:11 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:11 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:11 INFO DAGScheduler:59 - Registering RDD 70 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:11 INFO DAGScheduler:59 - Registering RDD 74 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:11 INFO DAGScheduler:59 - Got job 6 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:11 INFO DAGScheduler:59 - Final stage: ResultStage 46(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 45, ShuffleMapStage 37, ShuffleMapStage 34, ShuffleMapStage 35, ShuffleMapStage 39, ShuffleMapStage 43, ShuffleMapStage 41) -2016-06-13 19:47:11 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 45) -2016-06-13 19:47:11 INFO DAGScheduler:59 - Submitting ShuffleMapStage 44 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[70] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:11 INFO MemoryStore:59 - ensureFreeSpace(6304) called with curMem=274472, maxMem=766075207 -2016-06-13 19:47:11 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 6.2 KB, free 730.3 MB) -2016-06-13 19:47:11 INFO MemoryStore:59 - ensureFreeSpace(2991) called with curMem=280776, maxMem=766075207 -2016-06-13 19:47:11 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-06-13 19:47:11 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:50053 (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:47:11 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 44 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[70] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:11 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-06-13 19:47:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 30, localhost, PROCESS_LOCAL, 2237 bytes) -2016-06-13 19:47:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 31, localhost, PROCESS_LOCAL, 2237 bytes) -2016-06-13 19:47:11 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 31) -2016-06-13 19:47:11 INFO BlockManager:59 - Found block rdd_65_1 locally -2016-06-13 19:47:11 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 30) -2016-06-13 19:47:11 INFO BlockManager:59 - Found block rdd_65_0 locally -2016-06-13 19:47:11 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 31). 2254 bytes result sent to driver -2016-06-13 19:47:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 31) in 53 ms on localhost (1/2) -2016-06-13 19:47:11 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 30). 2254 bytes result sent to driver -2016-06-13 19:47:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 30) in 67 ms on localhost (2/2) -2016-06-13 19:47:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-06-13 19:47:11 INFO DAGScheduler:59 - ShuffleMapStage 44 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.070 s -2016-06-13 19:47:11 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:11 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:11 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 45, ResultStage 46) -2016-06-13 19:47:11 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:11 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 45: List() -2016-06-13 19:47:11 INFO DAGScheduler:59 - Missing parents for ResultStage 46: List(ShuffleMapStage 45) -2016-06-13 19:47:11 INFO DAGScheduler:59 - Submitting ShuffleMapStage 45 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[74] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:11 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=283767, maxMem=766075207 -2016-06-13 19:47:11 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:11 INFO MemoryStore:59 - ensureFreeSpace(2284) called with curMem=287663, maxMem=766075207 -2016-06-13 19:47:11 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:47:11 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:50053 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:11 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 45 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[74] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:11 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-06-13 19:47:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 32, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 33, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:11 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 33) -2016-06-13 19:47:11 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 32) -2016-06-13 19:47:11 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:11 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 32). 2470 bytes result sent to driver -2016-06-13 19:47:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 32) in 77 ms on localhost (1/2) -2016-06-13 19:47:11 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:11 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:11 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:11 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 33). 2470 bytes result sent to driver -2016-06-13 19:47:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 33) in 284 ms on localhost (2/2) -2016-06-13 19:47:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-06-13 19:47:11 INFO DAGScheduler:59 - ShuffleMapStage 45 (mapPartitions at GraphImpl.scala:235) finished in 0.304 s -2016-06-13 19:47:11 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:11 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:11 INFO DAGScheduler:59 - waiting: Set(ResultStage 46) -2016-06-13 19:47:11 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:12 INFO DAGScheduler:59 - Missing parents for ResultStage 46: List() -2016-06-13 19:47:12 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[82] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:12 INFO MemoryStore:59 - ensureFreeSpace(7720) called with curMem=289947, maxMem=766075207 -2016-06-13 19:47:12 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 7.5 KB, free 730.3 MB) -2016-06-13 19:47:12 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=297667, maxMem=766075207 -2016-06-13 19:47:12 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.3 MB) -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:50053 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[82] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:12 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-06-13 19:47:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 34, localhost, PROCESS_LOCAL, 2289 bytes) -2016-06-13 19:47:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 35, localhost, PROCESS_LOCAL, 2289 bytes) -2016-06-13 19:47:12 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 35) -2016-06-13 19:47:12 INFO CacheManager:59 - Partition rdd_78_1 not found, computing it -2016-06-13 19:47:12 INFO BlockManager:59 - Found block rdd_65_1 locally -2016-06-13 19:47:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:12 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 34) -2016-06-13 19:47:12 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=301067, maxMem=766075207 -2016-06-13 19:47:12 INFO MemoryStore:59 - Block rdd_78_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Added rdd_78_1 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 35). 2971 bytes result sent to driver -2016-06-13 19:47:12 INFO CacheManager:59 - Partition rdd_78_0 not found, computing it -2016-06-13 19:47:12 INFO BlockManager:59 - Found block rdd_65_0 locally -2016-06-13 19:47:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 35) in 21 ms on localhost (1/2) -2016-06-13 19:47:12 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=303515, maxMem=766075207 -2016-06-13 19:47:12 INFO MemoryStore:59 - Block rdd_78_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Added rdd_78_0 in memory on localhost:50053 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 34). 2971 bytes result sent to driver -2016-06-13 19:47:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 34) in 39 ms on localhost (2/2) -2016-06-13 19:47:12 INFO DAGScheduler:59 - ResultStage 46 (fold at LDAOptimizer.scala:205) finished in 0.109 s -2016-06-13 19:47:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-06-13 19:47:12 INFO DAGScheduler:59 - Job 6 finished: fold at LDAOptimizer.scala:205, took 0.635440 s -2016-06-13 19:47:12 INFO MapPartitionsRDD:59 - Removing RDD 52 from persistence list -2016-06-13 19:47:12 INFO ZippedPartitionsRDD2:59 - Removing RDD 59 from persistence list -2016-06-13 19:47:12 INFO BlockManager:59 - Removing RDD 52 -2016-06-13 19:47:12 INFO BlockManager:59 - Removing RDD 59 -2016-06-13 19:47:12 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:12 INFO DAGScheduler:59 - Registering RDD 83 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:12 INFO DAGScheduler:59 - Registering RDD 87 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:12 INFO DAGScheduler:59 - Got job 7 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:12 INFO DAGScheduler:59 - Final stage: ResultStage 61(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 48, ShuffleMapStage 60, ShuffleMapStage 52, ShuffleMapStage 56, ShuffleMapStage 50, ShuffleMapStage 54, ShuffleMapStage 58, ShuffleMapStage 47) -2016-06-13 19:47:12 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 60) -2016-06-13 19:47:12 INFO DAGScheduler:59 - Submitting ShuffleMapStage 59 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[83] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:12 INFO MemoryStore:59 - ensureFreeSpace(6728) called with curMem=301067, maxMem=766075207 -2016-06-13 19:47:12 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 6.6 KB, free 730.3 MB) -2016-06-13 19:47:12 INFO MemoryStore:59 - ensureFreeSpace(3078) called with curMem=307795, maxMem=766075207 -2016-06-13 19:47:12 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.3 MB) -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:50053 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 59 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[83] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:12 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 2 tasks -2016-06-13 19:47:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 36, localhost, PROCESS_LOCAL, 2278 bytes) -2016-06-13 19:47:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 37, localhost, PROCESS_LOCAL, 2278 bytes) -2016-06-13 19:47:12 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 36) -2016-06-13 19:47:12 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 37) -2016-06-13 19:47:12 INFO BlockManager:59 - Found block rdd_78_1 locally -2016-06-13 19:47:12 INFO BlockManager:59 - Found block rdd_78_0 locally -2016-06-13 19:47:12 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 37). 2254 bytes result sent to driver -2016-06-13 19:47:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 37) in 50 ms on localhost (1/2) -2016-06-13 19:47:12 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 36). 2254 bytes result sent to driver -2016-06-13 19:47:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 36) in 55 ms on localhost (2/2) -2016-06-13 19:47:12 INFO DAGScheduler:59 - ShuffleMapStage 59 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.056 s -2016-06-13 19:47:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-06-13 19:47:12 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:12 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:12 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 60, ResultStage 61) -2016-06-13 19:47:12 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:12 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 60: List() -2016-06-13 19:47:12 INFO DAGScheduler:59 - Missing parents for ResultStage 61: List(ShuffleMapStage 60) -2016-06-13 19:47:12 INFO DAGScheduler:59 - Submitting ShuffleMapStage 60 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[87] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:12 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=310873, maxMem=766075207 -2016-06-13 19:47:12 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:12 INFO MemoryStore:59 - ensureFreeSpace(2283) called with curMem=314769, maxMem=766075207 -2016-06-13 19:47:12 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:50053 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:50053 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 60 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[87] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:12 INFO TaskSchedulerImpl:59 - Adding task set 60.0 with 2 tasks -2016-06-13 19:47:12 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 19:47:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 60.0 (TID 38, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 60.0 (TID 39, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:12 INFO Executor:59 - Running task 0.0 in stage 60.0 (TID 38) -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:50053 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO Executor:59 - Running task 1.0 in stage 60.0 (TID 39) -2016-06-13 19:47:12 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:12 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:12 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:50053 in memory (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO Executor:59 - Finished task 1.0 in stage 60.0 (TID 39). 2470 bytes result sent to driver -2016-06-13 19:47:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 60.0 (TID 39) in 38 ms on localhost (1/2) -2016-06-13 19:47:12 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:50053 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:50053 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:50053 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:50053 in memory (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:50053 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:50053 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO Executor:59 - Finished task 0.0 in stage 60.0 (TID 38). 2470 bytes result sent to driver -2016-06-13 19:47:12 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:50053 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO DAGScheduler:59 - ShuffleMapStage 60 (mapPartitions at GraphImpl.scala:235) finished in 0.177 s -2016-06-13 19:47:12 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:12 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:12 INFO DAGScheduler:59 - waiting: Set(ResultStage 61) -2016-06-13 19:47:12 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 60.0 (TID 38) in 175 ms on localhost (2/2) -2016-06-13 19:47:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 60.0, whose tasks have all completed, from pool -2016-06-13 19:47:12 INFO DAGScheduler:59 - Missing parents for ResultStage 61: List() -2016-06-13 19:47:12 INFO DAGScheduler:59 - Submitting ResultStage 61 (MapPartitionsRDD[95] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:12 INFO MemoryStore:59 - ensureFreeSpace(8144) called with curMem=231119, maxMem=766075207 -2016-06-13 19:47:12 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 8.0 KB, free 730.4 MB) -2016-06-13 19:47:12 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=239263, maxMem=766075207 -2016-06-13 19:47:12 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.4 MB) -2016-06-13 19:47:12 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:50053 (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 61 (MapPartitionsRDD[95] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:12 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 2 tasks -2016-06-13 19:47:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 40, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:47:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 41, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:47:12 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 41) -2016-06-13 19:47:12 INFO CacheManager:59 - Partition rdd_91_1 not found, computing it -2016-06-13 19:47:12 INFO BlockManager:59 - Found block rdd_78_1 locally -2016-06-13 19:47:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:12 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=242754, maxMem=766075207 -2016-06-13 19:47:12 INFO MemoryStore:59 - Block rdd_91_1 stored as values in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Added rdd_91_1 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 41). 2971 bytes result sent to driver -2016-06-13 19:47:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 41) in 19 ms on localhost (1/2) -2016-06-13 19:47:12 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 40) -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:50053 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO CacheManager:59 - Partition rdd_91_0 not found, computing it -2016-06-13 19:47:12 INFO BlockManager:59 - Found block rdd_78_0 locally -2016-06-13 19:47:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:12 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 19:47:12 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=232272, maxMem=766075207 -2016-06-13 19:47:12 INFO MemoryStore:59 - Block rdd_91_0 stored as values in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:50053 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Added rdd_91_0 in memory on localhost:50053 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 19:47:12 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 40). 2971 bytes result sent to driver -2016-06-13 19:47:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 40) in 113 ms on localhost (2/2) -2016-06-13 19:47:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-06-13 19:47:12 INFO DAGScheduler:59 - ResultStage 61 (fold at LDAOptimizer.scala:205) finished in 0.169 s -2016-06-13 19:47:12 INFO DAGScheduler:59 - Job 7 finished: fold at LDAOptimizer.scala:205, took 0.526025 s -2016-06-13 19:47:12 INFO MapPartitionsRDD:59 - Removing RDD 65 from persistence list -2016-06-13 19:47:12 INFO ZippedPartitionsRDD2:59 - Removing RDD 72 from persistence list -2016-06-13 19:47:12 INFO BlockManager:59 - Removing RDD 72 -2016-06-13 19:47:12 INFO BlockManager:59 - Removing RDD 65 -2016-06-13 19:47:12 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:12 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:12 INFO DAGScheduler:59 - Registering RDD 96 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:12 INFO DAGScheduler:59 - Registering RDD 100 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:12 INFO DAGScheduler:59 - Got job 8 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:12 INFO DAGScheduler:59 - Final stage: ResultStage 78(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 63, ShuffleMapStage 67, ShuffleMapStage 71, ShuffleMapStage 75, ShuffleMapStage 69, ShuffleMapStage 73, ShuffleMapStage 65, ShuffleMapStage 62, ShuffleMapStage 77) -2016-06-13 19:47:12 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 77) -2016-06-13 19:47:12 INFO DAGScheduler:59 - Submitting ShuffleMapStage 76 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[96] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:12 INFO MemoryStore:59 - ensureFreeSpace(7160) called with curMem=229824, maxMem=766075207 -2016-06-13 19:47:12 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 7.0 KB, free 730.4 MB) -2016-06-13 19:47:12 INFO MemoryStore:59 - ensureFreeSpace(3182) called with curMem=236984, maxMem=766075207 -2016-06-13 19:47:12 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.4 MB) -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:50053 (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 76 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[96] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:12 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 2 tasks -2016-06-13 19:47:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 42, localhost, PROCESS_LOCAL, 2319 bytes) -2016-06-13 19:47:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 43, localhost, PROCESS_LOCAL, 2319 bytes) -2016-06-13 19:47:12 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 43) -2016-06-13 19:47:12 INFO BlockManager:59 - Found block rdd_91_1 locally -2016-06-13 19:47:12 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 42) -2016-06-13 19:47:12 INFO BlockManager:59 - Found block rdd_91_0 locally -2016-06-13 19:47:12 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 43). 2254 bytes result sent to driver -2016-06-13 19:47:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 43) in 37 ms on localhost (1/2) -2016-06-13 19:47:12 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 42). 2254 bytes result sent to driver -2016-06-13 19:47:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 42) in 51 ms on localhost (2/2) -2016-06-13 19:47:12 INFO DAGScheduler:59 - ShuffleMapStage 76 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.051 s -2016-06-13 19:47:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-06-13 19:47:12 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:12 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:12 INFO DAGScheduler:59 - waiting: Set(ResultStage 78, ShuffleMapStage 77) -2016-06-13 19:47:12 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:12 INFO DAGScheduler:59 - Missing parents for ResultStage 78: List(ShuffleMapStage 77) -2016-06-13 19:47:12 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 77: List() -2016-06-13 19:47:12 INFO DAGScheduler:59 - Submitting ShuffleMapStage 77 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[100] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:12 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=240166, maxMem=766075207 -2016-06-13 19:47:12 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 19:47:12 INFO MemoryStore:59 - ensureFreeSpace(2283) called with curMem=244062, maxMem=766075207 -2016-06-13 19:47:12 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:50053 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 77 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[100] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:12 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 2 tasks -2016-06-13 19:47:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 44, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 45, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:12 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 45) -2016-06-13 19:47:12 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 44) -2016-06-13 19:47:12 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:12 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:12 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 45). 2470 bytes result sent to driver -2016-06-13 19:47:12 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 44). 2470 bytes result sent to driver -2016-06-13 19:47:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 45) in 36 ms on localhost (1/2) -2016-06-13 19:47:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 44) in 37 ms on localhost (2/2) -2016-06-13 19:47:12 INFO DAGScheduler:59 - ShuffleMapStage 77 (mapPartitions at GraphImpl.scala:235) finished in 0.037 s -2016-06-13 19:47:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-06-13 19:47:12 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:12 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:12 INFO DAGScheduler:59 - waiting: Set(ResultStage 78) -2016-06-13 19:47:12 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:12 INFO DAGScheduler:59 - Missing parents for ResultStage 78: List() -2016-06-13 19:47:12 INFO DAGScheduler:59 - Submitting ResultStage 78 (MapPartitionsRDD[108] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:12 INFO MemoryStore:59 - ensureFreeSpace(8576) called with curMem=246345, maxMem=766075207 -2016-06-13 19:47:12 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 8.4 KB, free 730.3 MB) -2016-06-13 19:47:12 INFO MemoryStore:59 - ensureFreeSpace(3582) called with curMem=254921, maxMem=766075207 -2016-06-13 19:47:12 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:50053 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 78 (MapPartitionsRDD[108] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:12 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 2 tasks -2016-06-13 19:47:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 46, localhost, PROCESS_LOCAL, 2371 bytes) -2016-06-13 19:47:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 47, localhost, PROCESS_LOCAL, 2371 bytes) -2016-06-13 19:47:12 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 47) -2016-06-13 19:47:12 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 46) -2016-06-13 19:47:12 INFO CacheManager:59 - Partition rdd_104_1 not found, computing it -2016-06-13 19:47:12 INFO BlockManager:59 - Found block rdd_91_1 locally -2016-06-13 19:47:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:12 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=258503, maxMem=766075207 -2016-06-13 19:47:12 INFO CacheManager:59 - Partition rdd_104_0 not found, computing it -2016-06-13 19:47:12 INFO MemoryStore:59 - Block rdd_104_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:47:12 INFO BlockManager:59 - Found block rdd_91_0 locally -2016-06-13 19:47:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Added rdd_104_1 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:12 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 47). 2971 bytes result sent to driver -2016-06-13 19:47:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 47) in 19 ms on localhost (1/2) -2016-06-13 19:47:12 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=260951, maxMem=766075207 -2016-06-13 19:47:12 INFO MemoryStore:59 - Block rdd_104_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:47:12 INFO BlockManagerInfo:59 - Added rdd_104_0 in memory on localhost:50053 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:12 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 46). 2971 bytes result sent to driver -2016-06-13 19:47:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 46) in 29 ms on localhost (2/2) -2016-06-13 19:47:12 INFO DAGScheduler:59 - ResultStage 78 (fold at LDAOptimizer.scala:205) finished in 0.029 s -2016-06-13 19:47:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-06-13 19:47:13 INFO DAGScheduler:59 - Job 8 finished: fold at LDAOptimizer.scala:205, took 0.217614 s -2016-06-13 19:47:13 INFO MapPartitionsRDD:59 - Removing RDD 78 from persistence list -2016-06-13 19:47:13 INFO BlockManager:59 - Removing RDD 78 -2016-06-13 19:47:13 INFO ZippedPartitionsRDD2:59 - Removing RDD 85 from persistence list -2016-06-13 19:47:13 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:13 INFO DAGScheduler:59 - Registering RDD 109 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:13 INFO DAGScheduler:59 - Registering RDD 113 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:13 INFO DAGScheduler:59 - Got job 9 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:13 INFO DAGScheduler:59 - Final stage: ResultStage 97(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 84, ShuffleMapStage 96, ShuffleMapStage 88, ShuffleMapStage 82, ShuffleMapStage 86, ShuffleMapStage 90, ShuffleMapStage 79, ShuffleMapStage 94, ShuffleMapStage 80, ShuffleMapStage 92) -2016-06-13 19:47:13 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 96) -2016-06-13 19:47:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 95 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[109] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(7584) called with curMem=258503, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 7.4 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(3283) called with curMem=266087, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:50053 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:47:13 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 95 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[109] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:13 INFO TaskSchedulerImpl:59 - Adding task set 95.0 with 2 tasks -2016-06-13 19:47:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 95.0 (TID 48, localhost, PROCESS_LOCAL, 2360 bytes) -2016-06-13 19:47:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 95.0 (TID 49, localhost, PROCESS_LOCAL, 2360 bytes) -2016-06-13 19:47:13 INFO Executor:59 - Running task 1.0 in stage 95.0 (TID 49) -2016-06-13 19:47:13 INFO Executor:59 - Running task 0.0 in stage 95.0 (TID 48) -2016-06-13 19:47:13 INFO BlockManager:59 - Found block rdd_104_1 locally -2016-06-13 19:47:13 INFO BlockManager:59 - Found block rdd_104_0 locally -2016-06-13 19:47:13 INFO BlockManager:59 - Removing RDD 85 -2016-06-13 19:47:13 INFO Executor:59 - Finished task 0.0 in stage 95.0 (TID 48). 2254 bytes result sent to driver -2016-06-13 19:47:13 INFO Executor:59 - Finished task 1.0 in stage 95.0 (TID 49). 2254 bytes result sent to driver -2016-06-13 19:47:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 95.0 (TID 48) in 57 ms on localhost (1/2) -2016-06-13 19:47:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 95.0 (TID 49) in 57 ms on localhost (2/2) -2016-06-13 19:47:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 95.0, whose tasks have all completed, from pool -2016-06-13 19:47:13 INFO DAGScheduler:59 - ShuffleMapStage 95 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.062 s -2016-06-13 19:47:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:13 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:13 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 96, ResultStage 97) -2016-06-13 19:47:13 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:13 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 96: List() -2016-06-13 19:47:13 INFO DAGScheduler:59 - Missing parents for ResultStage 97: List(ShuffleMapStage 96) -2016-06-13 19:47:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 96 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[113] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=269370, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=273266, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:50053 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:13 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 96 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[113] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:13 INFO TaskSchedulerImpl:59 - Adding task set 96.0 with 2 tasks -2016-06-13 19:47:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 96.0 (TID 50, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 96.0 (TID 51, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:13 INFO Executor:59 - Running task 0.0 in stage 96.0 (TID 50) -2016-06-13 19:47:13 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:13 INFO Executor:59 - Running task 1.0 in stage 96.0 (TID 51) -2016-06-13 19:47:13 INFO Executor:59 - Finished task 0.0 in stage 96.0 (TID 50). 2470 bytes result sent to driver -2016-06-13 19:47:13 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 96.0 (TID 50) in 61 ms on localhost (1/2) -2016-06-13 19:47:13 INFO Executor:59 - Finished task 1.0 in stage 96.0 (TID 51). 2470 bytes result sent to driver -2016-06-13 19:47:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 96.0 (TID 51) in 72 ms on localhost (2/2) -2016-06-13 19:47:13 INFO DAGScheduler:59 - ShuffleMapStage 96 (mapPartitions at GraphImpl.scala:235) finished in 0.076 s -2016-06-13 19:47:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:13 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 96.0, whose tasks have all completed, from pool -2016-06-13 19:47:13 INFO DAGScheduler:59 - waiting: Set(ResultStage 97) -2016-06-13 19:47:13 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:13 INFO DAGScheduler:59 - Missing parents for ResultStage 97: List() -2016-06-13 19:47:13 INFO DAGScheduler:59 - Submitting ResultStage 97 (MapPartitionsRDD[121] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(9000) called with curMem=275552, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 8.8 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(3686) called with curMem=284552, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 3.6 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:50053 (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:47:13 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 97 (MapPartitionsRDD[121] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:13 INFO TaskSchedulerImpl:59 - Adding task set 97.0 with 2 tasks -2016-06-13 19:47:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 97.0 (TID 52, localhost, PROCESS_LOCAL, 2412 bytes) -2016-06-13 19:47:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 97.0 (TID 53, localhost, PROCESS_LOCAL, 2412 bytes) -2016-06-13 19:47:13 INFO Executor:59 - Running task 1.0 in stage 97.0 (TID 53) -2016-06-13 19:47:13 INFO Executor:59 - Running task 0.0 in stage 97.0 (TID 52) -2016-06-13 19:47:13 INFO CacheManager:59 - Partition rdd_117_0 not found, computing it -2016-06-13 19:47:13 INFO BlockManager:59 - Found block rdd_104_0 locally -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:13 INFO CacheManager:59 - Partition rdd_117_1 not found, computing it -2016-06-13 19:47:13 INFO BlockManager:59 - Found block rdd_104_1 locally -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=288238, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block rdd_117_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=290782, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block rdd_117_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO BlockManagerInfo:59 - Added rdd_117_0 in memory on localhost:50053 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:13 INFO BlockManagerInfo:59 - Added rdd_117_1 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:13 INFO Executor:59 - Finished task 0.0 in stage 97.0 (TID 52). 2971 bytes result sent to driver -2016-06-13 19:47:13 INFO Executor:59 - Finished task 1.0 in stage 97.0 (TID 53). 2971 bytes result sent to driver -2016-06-13 19:47:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 97.0 (TID 52) in 20 ms on localhost (1/2) -2016-06-13 19:47:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 97.0 (TID 53) in 20 ms on localhost (2/2) -2016-06-13 19:47:13 INFO DAGScheduler:59 - ResultStage 97 (fold at LDAOptimizer.scala:205) finished in 0.021 s -2016-06-13 19:47:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 97.0, whose tasks have all completed, from pool -2016-06-13 19:47:13 INFO DAGScheduler:59 - Job 9 finished: fold at LDAOptimizer.scala:205, took 0.383776 s -2016-06-13 19:47:13 INFO MapPartitionsRDD:59 - Removing RDD 91 from persistence list -2016-06-13 19:47:13 INFO ZippedPartitionsRDD2:59 - Removing RDD 98 from persistence list -2016-06-13 19:47:13 INFO BlockManager:59 - Removing RDD 91 -2016-06-13 19:47:13 INFO BlockManager:59 - Removing RDD 98 -2016-06-13 19:47:13 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:13 INFO DAGScheduler:59 - Registering RDD 122 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:13 INFO DAGScheduler:59 - Registering RDD 126 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:13 INFO DAGScheduler:59 - Got job 10 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:13 INFO DAGScheduler:59 - Final stage: ResultStage 118(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 117, ShuffleMapStage 99, ShuffleMapStage 103, ShuffleMapStage 115, ShuffleMapStage 107, ShuffleMapStage 101, ShuffleMapStage 111, ShuffleMapStage 105, ShuffleMapStage 109, ShuffleMapStage 98, ShuffleMapStage 113) -2016-06-13 19:47:13 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 117) -2016-06-13 19:47:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 116 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[122] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(8016) called with curMem=288238, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 7.8 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(3371) called with curMem=296254, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:50053 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:47:13 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 116 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[122] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:13 INFO TaskSchedulerImpl:59 - Adding task set 116.0 with 2 tasks -2016-06-13 19:47:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 116.0 (TID 54, localhost, PROCESS_LOCAL, 2401 bytes) -2016-06-13 19:47:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 116.0 (TID 55, localhost, PROCESS_LOCAL, 2401 bytes) -2016-06-13 19:47:13 INFO Executor:59 - Running task 1.0 in stage 116.0 (TID 55) -2016-06-13 19:47:13 INFO Executor:59 - Running task 0.0 in stage 116.0 (TID 54) -2016-06-13 19:47:13 INFO BlockManager:59 - Found block rdd_117_1 locally -2016-06-13 19:47:13 INFO BlockManager:59 - Found block rdd_117_0 locally -2016-06-13 19:47:13 INFO Executor:59 - Finished task 0.0 in stage 116.0 (TID 54). 2254 bytes result sent to driver -2016-06-13 19:47:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 116.0 (TID 54) in 29 ms on localhost (1/2) -2016-06-13 19:47:13 INFO Executor:59 - Finished task 1.0 in stage 116.0 (TID 55). 2254 bytes result sent to driver -2016-06-13 19:47:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 116.0 (TID 55) in 39 ms on localhost (2/2) -2016-06-13 19:47:13 INFO DAGScheduler:59 - ShuffleMapStage 116 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.042 s -2016-06-13 19:47:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:13 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:13 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 117, ResultStage 118) -2016-06-13 19:47:13 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:13 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 117: List() -2016-06-13 19:47:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 116.0, whose tasks have all completed, from pool -2016-06-13 19:47:13 INFO DAGScheduler:59 - Missing parents for ResultStage 118: List(ShuffleMapStage 117) -2016-06-13 19:47:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 117 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[126] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=299625, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=303521, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:50053 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:13 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 117 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[126] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:13 INFO TaskSchedulerImpl:59 - Adding task set 117.0 with 2 tasks -2016-06-13 19:47:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 117.0 (TID 56, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 117.0 (TID 57, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:13 INFO Executor:59 - Running task 0.0 in stage 117.0 (TID 56) -2016-06-13 19:47:13 INFO Executor:59 - Running task 1.0 in stage 117.0 (TID 57) -2016-06-13 19:47:13 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:13 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:13 INFO Executor:59 - Finished task 1.0 in stage 117.0 (TID 57). 2470 bytes result sent to driver -2016-06-13 19:47:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 117.0 (TID 57) in 35 ms on localhost (1/2) -2016-06-13 19:47:13 INFO Executor:59 - Finished task 0.0 in stage 117.0 (TID 56). 2470 bytes result sent to driver -2016-06-13 19:47:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 117.0 (TID 56) in 43 ms on localhost (2/2) -2016-06-13 19:47:13 INFO DAGScheduler:59 - ShuffleMapStage 117 (mapPartitions at GraphImpl.scala:235) finished in 0.045 s -2016-06-13 19:47:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:13 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:13 INFO DAGScheduler:59 - waiting: Set(ResultStage 118) -2016-06-13 19:47:13 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 117.0, whose tasks have all completed, from pool -2016-06-13 19:47:13 INFO DAGScheduler:59 - Missing parents for ResultStage 118: List() -2016-06-13 19:47:13 INFO DAGScheduler:59 - Submitting ResultStage 118 (MapPartitionsRDD[134] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(9432) called with curMem=305807, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 9.2 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(3778) called with curMem=315239, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.7 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:50053 (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:47:13 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 118 (MapPartitionsRDD[134] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:13 INFO TaskSchedulerImpl:59 - Adding task set 118.0 with 2 tasks -2016-06-13 19:47:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 118.0 (TID 58, localhost, PROCESS_LOCAL, 2453 bytes) -2016-06-13 19:47:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 118.0 (TID 59, localhost, PROCESS_LOCAL, 2453 bytes) -2016-06-13 19:47:13 INFO Executor:59 - Running task 1.0 in stage 118.0 (TID 59) -2016-06-13 19:47:13 INFO Executor:59 - Running task 0.0 in stage 118.0 (TID 58) -2016-06-13 19:47:13 INFO CacheManager:59 - Partition rdd_130_1 not found, computing it -2016-06-13 19:47:13 INFO BlockManager:59 - Found block rdd_117_1 locally -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:13 INFO CacheManager:59 - Partition rdd_130_0 not found, computing it -2016-06-13 19:47:13 INFO BlockManager:59 - Found block rdd_117_0 locally -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=319017, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block rdd_130_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO BlockManagerInfo:59 - Added rdd_130_1 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:13 INFO Executor:59 - Finished task 1.0 in stage 118.0 (TID 59). 2971 bytes result sent to driver -2016-06-13 19:47:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 118.0 (TID 59) in 17 ms on localhost (1/2) -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=321465, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block rdd_130_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO BlockManagerInfo:59 - Added rdd_130_0 in memory on localhost:50053 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:13 INFO Executor:59 - Finished task 0.0 in stage 118.0 (TID 58). 2971 bytes result sent to driver -2016-06-13 19:47:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 118.0 (TID 58) in 70 ms on localhost (2/2) -2016-06-13 19:47:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 118.0, whose tasks have all completed, from pool -2016-06-13 19:47:13 INFO DAGScheduler:59 - ResultStage 118 (fold at LDAOptimizer.scala:205) finished in 0.072 s -2016-06-13 19:47:13 INFO DAGScheduler:59 - Job 10 finished: fold at LDAOptimizer.scala:205, took 0.261816 s -2016-06-13 19:47:13 INFO MapPartitionsRDD:59 - Removing RDD 104 from persistence list -2016-06-13 19:47:13 INFO BlockManager:59 - Removing RDD 104 -2016-06-13 19:47:13 INFO ZippedPartitionsRDD2:59 - Removing RDD 111 from persistence list -2016-06-13 19:47:13 INFO BlockManager:59 - Removing RDD 111 -2016-06-13 19:47:13 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:13 INFO DAGScheduler:59 - Registering RDD 135 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:13 INFO DAGScheduler:59 - Registering RDD 139 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:13 INFO DAGScheduler:59 - Got job 11 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:13 INFO DAGScheduler:59 - Final stage: ResultStage 141(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 132, ShuffleMapStage 136, ShuffleMapStage 140, ShuffleMapStage 122, ShuffleMapStage 119, ShuffleMapStage 126, ShuffleMapStage 138, ShuffleMapStage 120, ShuffleMapStage 130, ShuffleMapStage 124, ShuffleMapStage 134, ShuffleMapStage 128) -2016-06-13 19:47:13 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 140) -2016-06-13 19:47:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 139 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[135] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(8440) called with curMem=319017, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 8.2 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(3476) called with curMem=327457, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:50053 (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:47:13 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 139 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[135] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:13 INFO TaskSchedulerImpl:59 - Adding task set 139.0 with 2 tasks -2016-06-13 19:47:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 139.0 (TID 60, localhost, PROCESS_LOCAL, 2442 bytes) -2016-06-13 19:47:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 139.0 (TID 61, localhost, PROCESS_LOCAL, 2442 bytes) -2016-06-13 19:47:13 INFO Executor:59 - Running task 0.0 in stage 139.0 (TID 60) -2016-06-13 19:47:13 INFO Executor:59 - Running task 1.0 in stage 139.0 (TID 61) -2016-06-13 19:47:13 INFO BlockManager:59 - Found block rdd_130_0 locally -2016-06-13 19:47:13 INFO BlockManager:59 - Found block rdd_130_1 locally -2016-06-13 19:47:13 INFO Executor:59 - Finished task 1.0 in stage 139.0 (TID 61). 2254 bytes result sent to driver -2016-06-13 19:47:13 INFO Executor:59 - Finished task 0.0 in stage 139.0 (TID 60). 2254 bytes result sent to driver -2016-06-13 19:47:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 139.0 (TID 61) in 23 ms on localhost (1/2) -2016-06-13 19:47:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 139.0 (TID 60) in 25 ms on localhost (2/2) -2016-06-13 19:47:13 INFO DAGScheduler:59 - ShuffleMapStage 139 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.026 s -2016-06-13 19:47:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 139.0, whose tasks have all completed, from pool -2016-06-13 19:47:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:13 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:13 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 140, ResultStage 141) -2016-06-13 19:47:13 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:13 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 140: List() -2016-06-13 19:47:13 INFO DAGScheduler:59 - Missing parents for ResultStage 141: List(ShuffleMapStage 140) -2016-06-13 19:47:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 140 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[139] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=330933, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(2281) called with curMem=334829, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:50053 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:13 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 140 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[139] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:13 INFO TaskSchedulerImpl:59 - Adding task set 140.0 with 2 tasks -2016-06-13 19:47:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 140.0 (TID 62, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 140.0 (TID 63, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:13 INFO Executor:59 - Running task 1.0 in stage 140.0 (TID 63) -2016-06-13 19:47:13 INFO Executor:59 - Running task 0.0 in stage 140.0 (TID 62) -2016-06-13 19:47:13 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:13 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:13 INFO Executor:59 - Finished task 1.0 in stage 140.0 (TID 63). 2470 bytes result sent to driver -2016-06-13 19:47:13 INFO Executor:59 - Finished task 0.0 in stage 140.0 (TID 62). 2470 bytes result sent to driver -2016-06-13 19:47:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 140.0 (TID 63) in 31 ms on localhost (1/2) -2016-06-13 19:47:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 140.0 (TID 62) in 33 ms on localhost (2/2) -2016-06-13 19:47:13 INFO DAGScheduler:59 - ShuffleMapStage 140 (mapPartitions at GraphImpl.scala:235) finished in 0.034 s -2016-06-13 19:47:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 140.0, whose tasks have all completed, from pool -2016-06-13 19:47:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:13 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:13 INFO DAGScheduler:59 - waiting: Set(ResultStage 141) -2016-06-13 19:47:13 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:13 INFO DAGScheduler:59 - Missing parents for ResultStage 141: List() -2016-06-13 19:47:13 INFO DAGScheduler:59 - Submitting ResultStage 141 (MapPartitionsRDD[147] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(9856) called with curMem=337110, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 9.6 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(3886) called with curMem=346966, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:13 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:50053 (size: 3.8 KB, free: 730.5 MB) -2016-06-13 19:47:13 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 141 (MapPartitionsRDD[147] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:13 INFO TaskSchedulerImpl:59 - Adding task set 141.0 with 2 tasks -2016-06-13 19:47:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 141.0 (TID 64, localhost, PROCESS_LOCAL, 2494 bytes) -2016-06-13 19:47:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 141.0 (TID 65, localhost, PROCESS_LOCAL, 2494 bytes) -2016-06-13 19:47:13 INFO Executor:59 - Running task 0.0 in stage 141.0 (TID 64) -2016-06-13 19:47:13 INFO Executor:59 - Running task 1.0 in stage 141.0 (TID 65) -2016-06-13 19:47:13 INFO CacheManager:59 - Partition rdd_143_1 not found, computing it -2016-06-13 19:47:13 INFO CacheManager:59 - Partition rdd_143_0 not found, computing it -2016-06-13 19:47:13 INFO BlockManager:59 - Found block rdd_130_1 locally -2016-06-13 19:47:13 INFO BlockManager:59 - Found block rdd_130_0 locally -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=350852, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block rdd_143_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:47:13 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=353300, maxMem=766075207 -2016-06-13 19:47:13 INFO MemoryStore:59 - Block rdd_143_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:47:13 INFO BlockManagerInfo:59 - Added rdd_143_1 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:13 INFO BlockManagerInfo:59 - Added rdd_143_0 in memory on localhost:50053 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:13 INFO Executor:59 - Finished task 1.0 in stage 141.0 (TID 65). 2971 bytes result sent to driver -2016-06-13 19:47:13 INFO Executor:59 - Finished task 0.0 in stage 141.0 (TID 64). 2971 bytes result sent to driver -2016-06-13 19:47:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 141.0 (TID 65) in 15 ms on localhost (1/2) -2016-06-13 19:47:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 141.0 (TID 64) in 16 ms on localhost (2/2) -2016-06-13 19:47:13 INFO DAGScheduler:59 - ResultStage 141 (fold at LDAOptimizer.scala:205) finished in 0.016 s -2016-06-13 19:47:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 141.0, whose tasks have all completed, from pool -2016-06-13 19:47:13 INFO DAGScheduler:59 - Job 11 finished: fold at LDAOptimizer.scala:205, took 0.159013 s -2016-06-13 19:47:13 INFO MapPartitionsRDD:59 - Removing RDD 117 from persistence list -2016-06-13 19:47:13 INFO ZippedPartitionsRDD2:59 - Removing RDD 124 from persistence list -2016-06-13 19:47:13 INFO BlockManager:59 - Removing RDD 117 -2016-06-13 19:47:13 INFO BlockManager:59 - Removing RDD 124 -2016-06-13 19:47:13 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:13 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:13 INFO DAGScheduler:59 - Registering RDD 148 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:13 INFO DAGScheduler:59 - Registering RDD 152 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Got job 12 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:14 INFO DAGScheduler:59 - Final stage: ResultStage 166(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 161, ShuffleMapStage 143, ShuffleMapStage 153, ShuffleMapStage 165, ShuffleMapStage 147, ShuffleMapStage 157, ShuffleMapStage 151, ShuffleMapStage 155, ShuffleMapStage 159, ShuffleMapStage 163, ShuffleMapStage 145, ShuffleMapStage 142, ShuffleMapStage 149) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 165) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 164 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[148] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(8872) called with curMem=350852, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 8.7 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(3568) called with curMem=359724, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:50053 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:47:14 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 164 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[148] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Adding task set 164.0 with 2 tasks -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 164.0 (TID 66, localhost, PROCESS_LOCAL, 2483 bytes) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 164.0 (TID 67, localhost, PROCESS_LOCAL, 2483 bytes) -2016-06-13 19:47:14 INFO Executor:59 - Running task 0.0 in stage 164.0 (TID 66) -2016-06-13 19:47:14 INFO Executor:59 - Running task 1.0 in stage 164.0 (TID 67) -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_143_0 locally -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_143_1 locally -2016-06-13 19:47:14 INFO Executor:59 - Finished task 1.0 in stage 164.0 (TID 67). 2254 bytes result sent to driver -2016-06-13 19:47:14 INFO Executor:59 - Finished task 0.0 in stage 164.0 (TID 66). 2254 bytes result sent to driver -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 164.0 (TID 67) in 25 ms on localhost (1/2) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 164.0 (TID 66) in 27 ms on localhost (2/2) -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 164.0, whose tasks have all completed, from pool -2016-06-13 19:47:14 INFO DAGScheduler:59 - ShuffleMapStage 164 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.029 s -2016-06-13 19:47:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:14 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:14 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 165, ResultStage 166) -2016-06-13 19:47:14 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:14 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 165: List() -2016-06-13 19:47:14 INFO DAGScheduler:59 - Missing parents for ResultStage 166: List(ShuffleMapStage 165) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 165 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[152] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=363292, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=367188, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:50053 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:14 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 165 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[152] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Adding task set 165.0 with 2 tasks -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 165.0 (TID 68, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 165.0 (TID 69, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:14 INFO Executor:59 - Running task 0.0 in stage 165.0 (TID 68) -2016-06-13 19:47:14 INFO Executor:59 - Running task 1.0 in stage 165.0 (TID 69) -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:14 INFO Executor:59 - Finished task 0.0 in stage 165.0 (TID 68). 2470 bytes result sent to driver -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 165.0 (TID 68) in 30 ms on localhost (1/2) -2016-06-13 19:47:14 INFO Executor:59 - Finished task 1.0 in stage 165.0 (TID 69). 2470 bytes result sent to driver -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 165.0 (TID 69) in 32 ms on localhost (2/2) -2016-06-13 19:47:14 INFO DAGScheduler:59 - ShuffleMapStage 165 (mapPartitions at GraphImpl.scala:235) finished in 0.033 s -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 165.0, whose tasks have all completed, from pool -2016-06-13 19:47:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:14 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:14 INFO DAGScheduler:59 - waiting: Set(ResultStage 166) -2016-06-13 19:47:14 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:14 INFO DAGScheduler:59 - Missing parents for ResultStage 166: List() -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting ResultStage 166 (MapPartitionsRDD[160] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(10288) called with curMem=369474, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 10.0 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(4004) called with curMem=379762, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 3.9 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:50053 (size: 3.9 KB, free: 730.5 MB) -2016-06-13 19:47:14 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 166 (MapPartitionsRDD[160] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Adding task set 166.0 with 2 tasks -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 166.0 (TID 70, localhost, PROCESS_LOCAL, 2535 bytes) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 166.0 (TID 71, localhost, PROCESS_LOCAL, 2535 bytes) -2016-06-13 19:47:14 INFO Executor:59 - Running task 0.0 in stage 166.0 (TID 70) -2016-06-13 19:47:14 INFO Executor:59 - Running task 1.0 in stage 166.0 (TID 71) -2016-06-13 19:47:14 INFO CacheManager:59 - Partition rdd_156_0 not found, computing it -2016-06-13 19:47:14 INFO CacheManager:59 - Partition rdd_156_1 not found, computing it -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_143_0 locally -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_143_1 locally -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=383766, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block rdd_156_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=386214, maxMem=766075207 -2016-06-13 19:47:14 INFO BlockManagerInfo:59 - Added rdd_156_1 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:14 INFO MemoryStore:59 - Block rdd_156_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO Executor:59 - Finished task 1.0 in stage 166.0 (TID 71). 2971 bytes result sent to driver -2016-06-13 19:47:14 INFO BlockManagerInfo:59 - Added rdd_156_0 in memory on localhost:50053 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:14 INFO Executor:59 - Finished task 0.0 in stage 166.0 (TID 70). 2971 bytes result sent to driver -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 166.0 (TID 71) in 21 ms on localhost (1/2) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 166.0 (TID 70) in 23 ms on localhost (2/2) -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 166.0, whose tasks have all completed, from pool -2016-06-13 19:47:14 INFO DAGScheduler:59 - ResultStage 166 (fold at LDAOptimizer.scala:205) finished in 0.024 s -2016-06-13 19:47:14 INFO DAGScheduler:59 - Job 12 finished: fold at LDAOptimizer.scala:205, took 0.168820 s -2016-06-13 19:47:14 INFO MapPartitionsRDD:59 - Removing RDD 130 from persistence list -2016-06-13 19:47:14 INFO ZippedPartitionsRDD2:59 - Removing RDD 137 from persistence list -2016-06-13 19:47:14 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:14 INFO BlockManager:59 - Removing RDD 137 -2016-06-13 19:47:14 INFO BlockManager:59 - Removing RDD 130 -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:14 INFO DAGScheduler:59 - Registering RDD 161 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Registering RDD 165 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Got job 13 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:14 INFO DAGScheduler:59 - Final stage: ResultStage 193(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 168, ShuffleMapStage 190, ShuffleMapStage 184, ShuffleMapStage 176, ShuffleMapStage 170, ShuffleMapStage 192, ShuffleMapStage 178, ShuffleMapStage 186, ShuffleMapStage 180, ShuffleMapStage 172, ShuffleMapStage 188, ShuffleMapStage 174, ShuffleMapStage 167, ShuffleMapStage 182) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 192) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 191 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[161] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(9296) called with curMem=383766, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 9.1 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(3674) called with curMem=393062, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 3.6 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:50053 (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:47:14 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 191 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[161] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Adding task set 191.0 with 2 tasks -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 191.0 (TID 72, localhost, PROCESS_LOCAL, 2524 bytes) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 191.0 (TID 73, localhost, PROCESS_LOCAL, 2524 bytes) -2016-06-13 19:47:14 INFO Executor:59 - Running task 0.0 in stage 191.0 (TID 72) -2016-06-13 19:47:14 INFO Executor:59 - Running task 1.0 in stage 191.0 (TID 73) -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_156_0 locally -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_156_1 locally -2016-06-13 19:47:14 INFO Executor:59 - Finished task 0.0 in stage 191.0 (TID 72). 2254 bytes result sent to driver -2016-06-13 19:47:14 INFO Executor:59 - Finished task 1.0 in stage 191.0 (TID 73). 2254 bytes result sent to driver -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 191.0 (TID 72) in 27 ms on localhost (1/2) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 191.0 (TID 73) in 27 ms on localhost (2/2) -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 191.0, whose tasks have all completed, from pool -2016-06-13 19:47:14 INFO DAGScheduler:59 - ShuffleMapStage 191 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.029 s -2016-06-13 19:47:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:14 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:14 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 192, ResultStage 193) -2016-06-13 19:47:14 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:14 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 192: List() -2016-06-13 19:47:14 INFO DAGScheduler:59 - Missing parents for ResultStage 193: List(ShuffleMapStage 192) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 192 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[165] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=396736, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=400632, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:50053 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:14 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 192 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[165] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Adding task set 192.0 with 2 tasks -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 192.0 (TID 74, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 192.0 (TID 75, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:14 INFO Executor:59 - Running task 0.0 in stage 192.0 (TID 74) -2016-06-13 19:47:14 INFO Executor:59 - Running task 1.0 in stage 192.0 (TID 75) -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:14 INFO Executor:59 - Finished task 1.0 in stage 192.0 (TID 75). 2470 bytes result sent to driver -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 192.0 (TID 75) in 28 ms on localhost (1/2) -2016-06-13 19:47:14 INFO Executor:59 - Finished task 0.0 in stage 192.0 (TID 74). 2470 bytes result sent to driver -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 192.0 (TID 74) in 33 ms on localhost (2/2) -2016-06-13 19:47:14 INFO DAGScheduler:59 - ShuffleMapStage 192 (mapPartitions at GraphImpl.scala:235) finished in 0.035 s -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 192.0, whose tasks have all completed, from pool -2016-06-13 19:47:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:14 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:14 INFO DAGScheduler:59 - waiting: Set(ResultStage 193) -2016-06-13 19:47:14 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:14 INFO DAGScheduler:59 - Missing parents for ResultStage 193: List() -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting ResultStage 193 (MapPartitionsRDD[173] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(10712) called with curMem=402918, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 10.5 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=413630, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:50053 (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:47:14 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 193 (MapPartitionsRDD[173] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Adding task set 193.0 with 2 tasks -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 193.0 (TID 76, localhost, PROCESS_LOCAL, 2576 bytes) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 193.0 (TID 77, localhost, PROCESS_LOCAL, 2576 bytes) -2016-06-13 19:47:14 INFO Executor:59 - Running task 0.0 in stage 193.0 (TID 76) -2016-06-13 19:47:14 INFO Executor:59 - Running task 1.0 in stage 193.0 (TID 77) -2016-06-13 19:47:14 INFO CacheManager:59 - Partition rdd_169_1 not found, computing it -2016-06-13 19:47:14 INFO CacheManager:59 - Partition rdd_169_0 not found, computing it -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_156_1 locally -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_156_0 locally -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=417726, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block rdd_169_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=420174, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block rdd_169_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO BlockManagerInfo:59 - Added rdd_169_1 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:14 INFO BlockManagerInfo:59 - Added rdd_169_0 in memory on localhost:50053 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:14 INFO Executor:59 - Finished task 1.0 in stage 193.0 (TID 77). 2971 bytes result sent to driver -2016-06-13 19:47:14 INFO Executor:59 - Finished task 0.0 in stage 193.0 (TID 76). 2971 bytes result sent to driver -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 193.0 (TID 77) in 13 ms on localhost (1/2) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 193.0 (TID 76) in 15 ms on localhost (2/2) -2016-06-13 19:47:14 INFO DAGScheduler:59 - ResultStage 193 (fold at LDAOptimizer.scala:205) finished in 0.015 s -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 193.0, whose tasks have all completed, from pool -2016-06-13 19:47:14 INFO DAGScheduler:59 - Job 13 finished: fold at LDAOptimizer.scala:205, took 0.326908 s -2016-06-13 19:47:14 INFO MapPartitionsRDD:59 - Removing RDD 143 from persistence list -2016-06-13 19:47:14 INFO ZippedPartitionsRDD2:59 - Removing RDD 150 from persistence list -2016-06-13 19:47:14 INFO BlockManager:59 - Removing RDD 143 -2016-06-13 19:47:14 INFO BlockManager:59 - Removing RDD 150 -2016-06-13 19:47:14 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:47:14 INFO DAGScheduler:59 - Registering RDD 174 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Registering RDD 178 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Got job 14 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:14 INFO DAGScheduler:59 - Final stage: ResultStage 222(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 197, ShuffleMapStage 219, ShuffleMapStage 205, ShuffleMapStage 221, ShuffleMapStage 213, ShuffleMapStage 207, ShuffleMapStage 199, ShuffleMapStage 215, ShuffleMapStage 201, ShuffleMapStage 194, ShuffleMapStage 209, ShuffleMapStage 203, ShuffleMapStage 195, ShuffleMapStage 217, ShuffleMapStage 211) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 221) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 220 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[174] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(9728) called with curMem=417726, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 9.5 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(3782) called with curMem=427454, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.7 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:50053 (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:47:14 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 220 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[174] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Adding task set 220.0 with 2 tasks -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 220.0 (TID 78, localhost, PROCESS_LOCAL, 2565 bytes) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 220.0 (TID 79, localhost, PROCESS_LOCAL, 2565 bytes) -2016-06-13 19:47:14 INFO Executor:59 - Running task 1.0 in stage 220.0 (TID 79) -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_169_1 locally -2016-06-13 19:47:14 INFO Executor:59 - Running task 0.0 in stage 220.0 (TID 78) -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_169_0 locally -2016-06-13 19:47:14 INFO Executor:59 - Finished task 1.0 in stage 220.0 (TID 79). 2254 bytes result sent to driver -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 220.0 (TID 79) in 25 ms on localhost (1/2) -2016-06-13 19:47:14 INFO Executor:59 - Finished task 0.0 in stage 220.0 (TID 78). 2254 bytes result sent to driver -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 220.0 (TID 78) in 77 ms on localhost (2/2) -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 220.0, whose tasks have all completed, from pool -2016-06-13 19:47:14 INFO DAGScheduler:59 - ShuffleMapStage 220 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.078 s -2016-06-13 19:47:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:14 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:14 INFO DAGScheduler:59 - waiting: Set(ResultStage 222, ShuffleMapStage 221) -2016-06-13 19:47:14 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:14 INFO DAGScheduler:59 - Missing parents for ResultStage 222: List(ShuffleMapStage 221) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 221: List() -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 221 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[178] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=431236, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(2287) called with curMem=435132, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:50053 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:14 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 221 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[178] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Adding task set 221.0 with 2 tasks -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 221.0 (TID 80, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 221.0 (TID 81, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:14 INFO Executor:59 - Running task 0.0 in stage 221.0 (TID 80) -2016-06-13 19:47:14 INFO Executor:59 - Running task 1.0 in stage 221.0 (TID 81) -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:14 INFO Executor:59 - Finished task 0.0 in stage 221.0 (TID 80). 2470 bytes result sent to driver -2016-06-13 19:47:14 INFO Executor:59 - Finished task 1.0 in stage 221.0 (TID 81). 2470 bytes result sent to driver -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 221.0 (TID 81) in 90 ms on localhost (1/2) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 221.0 (TID 80) in 91 ms on localhost (2/2) -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 221.0, whose tasks have all completed, from pool -2016-06-13 19:47:14 INFO DAGScheduler:59 - ShuffleMapStage 221 (mapPartitions at GraphImpl.scala:235) finished in 0.092 s -2016-06-13 19:47:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:14 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:14 INFO DAGScheduler:59 - waiting: Set(ResultStage 222) -2016-06-13 19:47:14 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:14 INFO DAGScheduler:59 - Missing parents for ResultStage 222: List() -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting ResultStage 222 (MapPartitionsRDD[186] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(11144) called with curMem=437419, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 10.9 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(4182) called with curMem=448563, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 4.1 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:50053 (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:47:14 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 222 (MapPartitionsRDD[186] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Adding task set 222.0 with 2 tasks -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 222.0 (TID 82, localhost, PROCESS_LOCAL, 2617 bytes) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 222.0 (TID 83, localhost, PROCESS_LOCAL, 2617 bytes) -2016-06-13 19:47:14 INFO Executor:59 - Running task 0.0 in stage 222.0 (TID 82) -2016-06-13 19:47:14 INFO Executor:59 - Running task 1.0 in stage 222.0 (TID 83) -2016-06-13 19:47:14 INFO CacheManager:59 - Partition rdd_182_0 not found, computing it -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_169_0 locally -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:14 INFO CacheManager:59 - Partition rdd_182_1 not found, computing it -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_169_1 locally -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=452745, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block rdd_182_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=455289, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block rdd_182_1 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-06-13 19:47:14 INFO BlockManagerInfo:59 - Added rdd_182_0 in memory on localhost:50053 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:14 INFO BlockManagerInfo:59 - Added rdd_182_1 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:14 INFO Executor:59 - Finished task 0.0 in stage 222.0 (TID 82). 2971 bytes result sent to driver -2016-06-13 19:47:14 INFO Executor:59 - Finished task 1.0 in stage 222.0 (TID 83). 2971 bytes result sent to driver -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 222.0 (TID 82) in 13 ms on localhost (1/2) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 222.0 (TID 83) in 12 ms on localhost (2/2) -2016-06-13 19:47:14 INFO DAGScheduler:59 - ResultStage 222 (fold at LDAOptimizer.scala:205) finished in 0.014 s -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 222.0, whose tasks have all completed, from pool -2016-06-13 19:47:14 INFO DAGScheduler:59 - Job 14 finished: fold at LDAOptimizer.scala:205, took 0.260420 s -2016-06-13 19:47:14 INFO MapPartitionsRDD:59 - Removing RDD 156 from persistence list -2016-06-13 19:47:14 INFO ZippedPartitionsRDD2:59 - Removing RDD 163 from persistence list -2016-06-13 19:47:14 INFO BlockManager:59 - Removing RDD 163 -2016-06-13 19:47:14 INFO BlockManager:59 - Removing RDD 156 -2016-06-13 19:47:14 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:47:14 INFO DAGScheduler:59 - Registering RDD 187 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Registering RDD 191 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Got job 15 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:14 INFO DAGScheduler:59 - Final stage: ResultStage 253(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 248, ShuffleMapStage 234, ShuffleMapStage 242, ShuffleMapStage 228, ShuffleMapStage 250, ShuffleMapStage 244, ShuffleMapStage 236, ShuffleMapStage 230, ShuffleMapStage 252, ShuffleMapStage 223, ShuffleMapStage 238, ShuffleMapStage 224, ShuffleMapStage 246, ShuffleMapStage 240, ShuffleMapStage 232, ShuffleMapStage 226) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 252) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 251 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[187] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(10152) called with curMem=452745, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 9.9 KB, free 730.1 MB) -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(3864) called with curMem=462897, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 3.8 KB, free 730.1 MB) -2016-06-13 19:47:14 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:50053 (size: 3.8 KB, free: 730.5 MB) -2016-06-13 19:47:14 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 251 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[187] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Adding task set 251.0 with 2 tasks -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 251.0 (TID 84, localhost, PROCESS_LOCAL, 2606 bytes) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 251.0 (TID 85, localhost, PROCESS_LOCAL, 2606 bytes) -2016-06-13 19:47:14 INFO Executor:59 - Running task 0.0 in stage 251.0 (TID 84) -2016-06-13 19:47:14 INFO Executor:59 - Running task 1.0 in stage 251.0 (TID 85) -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_182_0 locally -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_182_1 locally -2016-06-13 19:47:14 INFO Executor:59 - Finished task 0.0 in stage 251.0 (TID 84). 2254 bytes result sent to driver -2016-06-13 19:47:14 INFO Executor:59 - Finished task 1.0 in stage 251.0 (TID 85). 2254 bytes result sent to driver -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 251.0 (TID 85) in 24 ms on localhost (1/2) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 251.0 (TID 84) in 28 ms on localhost (2/2) -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 251.0, whose tasks have all completed, from pool -2016-06-13 19:47:14 INFO DAGScheduler:59 - ShuffleMapStage 251 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.027 s -2016-06-13 19:47:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:14 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:14 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 252, ResultStage 253) -2016-06-13 19:47:14 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:14 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 252: List() -2016-06-13 19:47:14 INFO DAGScheduler:59 - Missing parents for ResultStage 253: List(ShuffleMapStage 252) -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 252 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[191] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=466761, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 3.8 KB, free 730.1 MB) -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(2284) called with curMem=470657, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-06-13 19:47:14 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:50053 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:14 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 252 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[191] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Adding task set 252.0 with 2 tasks -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 252.0 (TID 86, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 252.0 (TID 87, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:14 INFO Executor:59 - Running task 0.0 in stage 252.0 (TID 86) -2016-06-13 19:47:14 INFO Executor:59 - Running task 1.0 in stage 252.0 (TID 87) -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:14 INFO Executor:59 - Finished task 0.0 in stage 252.0 (TID 86). 2470 bytes result sent to driver -2016-06-13 19:47:14 INFO Executor:59 - Finished task 1.0 in stage 252.0 (TID 87). 2470 bytes result sent to driver -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 252.0 (TID 86) in 25 ms on localhost (1/2) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 252.0 (TID 87) in 26 ms on localhost (2/2) -2016-06-13 19:47:14 INFO DAGScheduler:59 - ShuffleMapStage 252 (mapPartitions at GraphImpl.scala:235) finished in 0.026 s -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 252.0, whose tasks have all completed, from pool -2016-06-13 19:47:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:14 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:14 INFO DAGScheduler:59 - waiting: Set(ResultStage 253) -2016-06-13 19:47:14 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:14 INFO DAGScheduler:59 - Missing parents for ResultStage 253: List() -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting ResultStage 253 (MapPartitionsRDD[199] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(11568) called with curMem=472941, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 11.3 KB, free 730.1 MB) -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(4283) called with curMem=484509, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.1 MB) -2016-06-13 19:47:14 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:50053 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:47:14 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 253 (MapPartitionsRDD[199] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Adding task set 253.0 with 2 tasks -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 253.0 (TID 88, localhost, PROCESS_LOCAL, 2658 bytes) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 253.0 (TID 89, localhost, PROCESS_LOCAL, 2658 bytes) -2016-06-13 19:47:14 INFO Executor:59 - Running task 0.0 in stage 253.0 (TID 88) -2016-06-13 19:47:14 INFO Executor:59 - Running task 1.0 in stage 253.0 (TID 89) -2016-06-13 19:47:14 INFO CacheManager:59 - Partition rdd_195_1 not found, computing it -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_182_1 locally -2016-06-13 19:47:14 INFO CacheManager:59 - Partition rdd_195_0 not found, computing it -2016-06-13 19:47:14 INFO BlockManager:59 - Found block rdd_182_0 locally -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=488792, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block rdd_195_1 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-06-13 19:47:14 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=491240, maxMem=766075207 -2016-06-13 19:47:14 INFO MemoryStore:59 - Block rdd_195_0 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-06-13 19:47:14 INFO BlockManagerInfo:59 - Added rdd_195_1 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:14 INFO BlockManagerInfo:59 - Added rdd_195_0 in memory on localhost:50053 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:14 INFO Executor:59 - Finished task 1.0 in stage 253.0 (TID 89). 2971 bytes result sent to driver -2016-06-13 19:47:14 INFO Executor:59 - Finished task 0.0 in stage 253.0 (TID 88). 2971 bytes result sent to driver -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 253.0 (TID 89) in 11 ms on localhost (1/2) -2016-06-13 19:47:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 253.0 (TID 88) in 13 ms on localhost (2/2) -2016-06-13 19:47:14 INFO DAGScheduler:59 - ResultStage 253 (fold at LDAOptimizer.scala:205) finished in 0.013 s -2016-06-13 19:47:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 253.0, whose tasks have all completed, from pool -2016-06-13 19:47:14 INFO DAGScheduler:59 - Job 15 finished: fold at LDAOptimizer.scala:205, took 0.132914 s -2016-06-13 19:47:14 INFO MapPartitionsRDD:59 - Removing RDD 169 from persistence list -2016-06-13 19:47:14 INFO ZippedPartitionsRDD2:59 - Removing RDD 176 from persistence list -2016-06-13 19:47:14 INFO BlockManager:59 - Removing RDD 169 -2016-06-13 19:47:14 INFO BlockManager:59 - Removing RDD 176 -2016-06-13 19:47:14 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:14 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:50053 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:50053 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:50053 in memory (size: 3.8 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:50053 in memory (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:50053 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:50053 in memory (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Registering RDD 200 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Registering RDD 204 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Got job 16 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:15 INFO DAGScheduler:59 - Final stage: ResultStage 286(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 269, ShuffleMapStage 263, ShuffleMapStage 255, ShuffleMapStage 285, ShuffleMapStage 277, ShuffleMapStage 271, ShuffleMapStage 257, ShuffleMapStage 279, ShuffleMapStage 265, ShuffleMapStage 281, ShuffleMapStage 273, ShuffleMapStage 267, ShuffleMapStage 259, ShuffleMapStage 275, ShuffleMapStage 261, ShuffleMapStage 283, ShuffleMapStage 254) -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-06-13 19:47:15 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 285) -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:50053 in memory (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Submitting ShuffleMapStage 284 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[200] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(10584) called with curMem=402918, maxMem=766075207 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 10.3 KB, free 730.2 MB) -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(3951) called with curMem=413502, maxMem=766075207 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 3.9 KB, free 730.2 MB) -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:50053 (size: 3.9 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:50053 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 284 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[200] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:15 INFO TaskSchedulerImpl:59 - Adding task set 284.0 with 2 tasks -2016-06-13 19:47:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 284.0 (TID 90, localhost, PROCESS_LOCAL, 2647 bytes) -2016-06-13 19:47:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 284.0 (TID 91, localhost, PROCESS_LOCAL, 2647 bytes) -2016-06-13 19:47:15 INFO Executor:59 - Running task 0.0 in stage 284.0 (TID 90) -2016-06-13 19:47:15 INFO BlockManager:59 - Found block rdd_195_0 locally -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-06-13 19:47:15 INFO Executor:59 - Running task 1.0 in stage 284.0 (TID 91) -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:50053 in memory (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO BlockManager:59 - Found block rdd_195_1 locally -2016-06-13 19:47:15 INFO Executor:59 - Finished task 0.0 in stage 284.0 (TID 90). 2254 bytes result sent to driver -2016-06-13 19:47:15 INFO Executor:59 - Finished task 1.0 in stage 284.0 (TID 91). 2254 bytes result sent to driver -2016-06-13 19:47:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 284.0 (TID 90) in 50 ms on localhost (1/2) -2016-06-13 19:47:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 284.0 (TID 91) in 51 ms on localhost (2/2) -2016-06-13 19:47:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 284.0, whose tasks have all completed, from pool -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-06-13 19:47:15 INFO DAGScheduler:59 - ShuffleMapStage 284 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.061 s -2016-06-13 19:47:15 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:15 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:15 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 285, ResultStage 286) -2016-06-13 19:47:15 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:50053 in memory (size: 3.9 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 285: List() -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-06-13 19:47:15 INFO DAGScheduler:59 - Missing parents for ResultStage 286: List(ShuffleMapStage 285) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Submitting ShuffleMapStage 285 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[204] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:50053 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=377827, maxMem=766075207 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=369283, maxMem=766075207 -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:50053 in memory (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:50053 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 285 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[204] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:15 INFO TaskSchedulerImpl:59 - Adding task set 285.0 with 2 tasks -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:50053 in memory (size: 3.8 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 285.0 (TID 92, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 285.0 (TID 93, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-06-13 19:47:15 INFO Executor:59 - Running task 0.0 in stage 285.0 (TID 92) -2016-06-13 19:47:15 INFO Executor:59 - Running task 1.0 in stage 285.0 (TID 93) -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:50053 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:15 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:50053 in memory (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:50053 in memory (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:50053 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO Executor:59 - Finished task 0.0 in stage 285.0 (TID 92). 2470 bytes result sent to driver -2016-06-13 19:47:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 285.0 (TID 92) in 46 ms on localhost (1/2) -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-06-13 19:47:15 INFO Executor:59 - Finished task 1.0 in stage 285.0 (TID 93). 2470 bytes result sent to driver -2016-06-13 19:47:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 285.0 (TID 93) in 51 ms on localhost (2/2) -2016-06-13 19:47:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 285.0, whose tasks have all completed, from pool -2016-06-13 19:47:15 INFO DAGScheduler:59 - ShuffleMapStage 285 (mapPartitions at GraphImpl.scala:235) finished in 0.053 s -2016-06-13 19:47:15 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:15 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:15 INFO DAGScheduler:59 - waiting: Set(ResultStage 286) -2016-06-13 19:47:15 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:50053 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Missing parents for ResultStage 286: List() -2016-06-13 19:47:15 INFO DAGScheduler:59 - Submitting ResultStage 286 (MapPartitionsRDD[212] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(12000) called with curMem=308955, maxMem=766075207 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 11.7 KB, free 730.3 MB) -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(4400) called with curMem=320955, maxMem=766075207 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:50053 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:50053 in memory (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 286 (MapPartitionsRDD[212] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:15 INFO TaskSchedulerImpl:59 - Adding task set 286.0 with 2 tasks -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-06-13 19:47:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 286.0 (TID 94, localhost, PROCESS_LOCAL, 2699 bytes) -2016-06-13 19:47:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 286.0 (TID 95, localhost, PROCESS_LOCAL, 2699 bytes) -2016-06-13 19:47:15 INFO Executor:59 - Running task 0.0 in stage 286.0 (TID 94) -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:50053 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO CacheManager:59 - Partition rdd_208_0 not found, computing it -2016-06-13 19:47:15 INFO BlockManager:59 - Found block rdd_195_0 locally -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:15 INFO Executor:59 - Running task 1.0 in stage 286.0 (TID 95) -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:15 INFO CacheManager:59 - Partition rdd_208_1 not found, computing it -2016-06-13 19:47:15 INFO BlockManager:59 - Found block rdd_195_1 locally -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=306487, maxMem=766075207 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block rdd_208_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=309031, maxMem=766075207 -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Added rdd_208_0 in memory on localhost:50053 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block rdd_208_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:47:15 INFO Executor:59 - Finished task 0.0 in stage 286.0 (TID 94). 2971 bytes result sent to driver -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Added rdd_208_1 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 286.0 (TID 94) in 24 ms on localhost (1/2) -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:50053 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO Executor:59 - Finished task 1.0 in stage 286.0 (TID 95). 2971 bytes result sent to driver -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-06-13 19:47:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 286.0 (TID 95) in 26 ms on localhost (2/2) -2016-06-13 19:47:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 286.0, whose tasks have all completed, from pool -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:50053 in memory (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO DAGScheduler:59 - ResultStage 286 (fold at LDAOptimizer.scala:205) finished in 0.034 s -2016-06-13 19:47:15 INFO DAGScheduler:59 - Job 16 finished: fold at LDAOptimizer.scala:205, took 0.278836 s -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:50053 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:50053 in memory (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:50053 in memory (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 19:47:15 INFO MapPartitionsRDD:59 - Removing RDD 182 from persistence list -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:50053 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO BlockManager:59 - Removing RDD 182 -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 19:47:15 INFO ZippedPartitionsRDD2:59 - Removing RDD 189 from persistence list -2016-06-13 19:47:15 INFO BlockManager:59 - Removing RDD 189 -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:50053 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 19:47:15 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:47:15 INFO DAGScheduler:59 - Registering RDD 213 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Registering RDD 217 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Got job 17 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:15 INFO DAGScheduler:59 - Final stage: ResultStage 321(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 320, ShuffleMapStage 306, ShuffleMapStage 300, ShuffleMapStage 292, ShuffleMapStage 314, ShuffleMapStage 308, ShuffleMapStage 294, ShuffleMapStage 316, ShuffleMapStage 287, ShuffleMapStage 302, ShuffleMapStage 288, ShuffleMapStage 318, ShuffleMapStage 310, ShuffleMapStage 304, ShuffleMapStage 296, ShuffleMapStage 290, ShuffleMapStage 312, ShuffleMapStage 298) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 320) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Submitting ShuffleMapStage 319 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[213] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=239321, maxMem=766075207 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 10.8 KB, free 730.3 MB) -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(4062) called with curMem=250329, maxMem=766075207 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:50053 (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 319 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[213] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:15 INFO TaskSchedulerImpl:59 - Adding task set 319.0 with 2 tasks -2016-06-13 19:47:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 319.0 (TID 96, localhost, PROCESS_LOCAL, 2688 bytes) -2016-06-13 19:47:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 319.0 (TID 97, localhost, PROCESS_LOCAL, 2688 bytes) -2016-06-13 19:47:15 INFO Executor:59 - Running task 1.0 in stage 319.0 (TID 97) -2016-06-13 19:47:15 INFO Executor:59 - Running task 0.0 in stage 319.0 (TID 96) -2016-06-13 19:47:15 INFO BlockManager:59 - Found block rdd_208_1 locally -2016-06-13 19:47:15 INFO BlockManager:59 - Found block rdd_208_0 locally -2016-06-13 19:47:15 INFO Executor:59 - Finished task 1.0 in stage 319.0 (TID 97). 2254 bytes result sent to driver -2016-06-13 19:47:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 319.0 (TID 97) in 28 ms on localhost (1/2) -2016-06-13 19:47:15 INFO Executor:59 - Finished task 0.0 in stage 319.0 (TID 96). 2254 bytes result sent to driver -2016-06-13 19:47:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 319.0 (TID 96) in 32 ms on localhost (2/2) -2016-06-13 19:47:15 INFO DAGScheduler:59 - ShuffleMapStage 319 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.033 s -2016-06-13 19:47:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 319.0, whose tasks have all completed, from pool -2016-06-13 19:47:15 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:15 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:15 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 320, ResultStage 321) -2016-06-13 19:47:15 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:15 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 320: List() -2016-06-13 19:47:15 INFO DAGScheduler:59 - Missing parents for ResultStage 321: List(ShuffleMapStage 320) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Submitting ShuffleMapStage 320 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[217] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=254391, maxMem=766075207 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=258287, maxMem=766075207 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:50053 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 320 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[217] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:15 INFO TaskSchedulerImpl:59 - Adding task set 320.0 with 2 tasks -2016-06-13 19:47:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 320.0 (TID 98, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 320.0 (TID 99, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:15 INFO Executor:59 - Running task 0.0 in stage 320.0 (TID 98) -2016-06-13 19:47:15 INFO Executor:59 - Running task 1.0 in stage 320.0 (TID 99) -2016-06-13 19:47:15 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:15 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:15 INFO Executor:59 - Finished task 0.0 in stage 320.0 (TID 98). 2470 bytes result sent to driver -2016-06-13 19:47:15 INFO Executor:59 - Finished task 1.0 in stage 320.0 (TID 99). 2470 bytes result sent to driver -2016-06-13 19:47:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 320.0 (TID 98) in 31 ms on localhost (1/2) -2016-06-13 19:47:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 320.0 (TID 99) in 31 ms on localhost (2/2) -2016-06-13 19:47:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 320.0, whose tasks have all completed, from pool -2016-06-13 19:47:15 INFO DAGScheduler:59 - ShuffleMapStage 320 (mapPartitions at GraphImpl.scala:235) finished in 0.033 s -2016-06-13 19:47:15 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:15 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:15 INFO DAGScheduler:59 - waiting: Set(ResultStage 321) -2016-06-13 19:47:15 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:15 INFO DAGScheduler:59 - Missing parents for ResultStage 321: List() -2016-06-13 19:47:15 INFO DAGScheduler:59 - Submitting ResultStage 321 (MapPartitionsRDD[225] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(12424) called with curMem=260573, maxMem=766075207 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 12.1 KB, free 730.3 MB) -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(4491) called with curMem=272997, maxMem=766075207 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.3 MB) -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:50053 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 321 (MapPartitionsRDD[225] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:15 INFO TaskSchedulerImpl:59 - Adding task set 321.0 with 2 tasks -2016-06-13 19:47:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 321.0 (TID 100, localhost, PROCESS_LOCAL, 2740 bytes) -2016-06-13 19:47:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 321.0 (TID 101, localhost, PROCESS_LOCAL, 2740 bytes) -2016-06-13 19:47:15 INFO Executor:59 - Running task 1.0 in stage 321.0 (TID 101) -2016-06-13 19:47:15 INFO Executor:59 - Running task 0.0 in stage 321.0 (TID 100) -2016-06-13 19:47:15 INFO CacheManager:59 - Partition rdd_221_1 not found, computing it -2016-06-13 19:47:15 INFO CacheManager:59 - Partition rdd_221_0 not found, computing it -2016-06-13 19:47:15 INFO BlockManager:59 - Found block rdd_208_1 locally -2016-06-13 19:47:15 INFO BlockManager:59 - Found block rdd_208_0 locally -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=277488, maxMem=766075207 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block rdd_221_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Added rdd_221_1 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=279936, maxMem=766075207 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block rdd_221_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:47:15 INFO Executor:59 - Finished task 1.0 in stage 321.0 (TID 101). 2971 bytes result sent to driver -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Added rdd_221_0 in memory on localhost:50053 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 321.0 (TID 101) in 15 ms on localhost (1/2) -2016-06-13 19:47:15 INFO Executor:59 - Finished task 0.0 in stage 321.0 (TID 100). 2971 bytes result sent to driver -2016-06-13 19:47:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 321.0 (TID 100) in 19 ms on localhost (2/2) -2016-06-13 19:47:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 321.0, whose tasks have all completed, from pool -2016-06-13 19:47:15 INFO DAGScheduler:59 - ResultStage 321 (fold at LDAOptimizer.scala:205) finished in 0.021 s -2016-06-13 19:47:15 INFO DAGScheduler:59 - Job 17 finished: fold at LDAOptimizer.scala:205, took 0.230488 s -2016-06-13 19:47:15 INFO MapPartitionsRDD:59 - Removing RDD 195 from persistence list -2016-06-13 19:47:15 INFO BlockManager:59 - Removing RDD 195 -2016-06-13 19:47:15 INFO ZippedPartitionsRDD2:59 - Removing RDD 202 from persistence list -2016-06-13 19:47:15 INFO BlockManager:59 - Removing RDD 202 -2016-06-13 19:47:15 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:47:15 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:47:15 INFO DAGScheduler:59 - Registering RDD 226 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Registering RDD 230 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Got job 18 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:15 INFO DAGScheduler:59 - Final stage: ResultStage 358(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 335, ShuffleMapStage 357, ShuffleMapStage 343, ShuffleMapStage 322, ShuffleMapStage 329, ShuffleMapStage 323, ShuffleMapStage 351, ShuffleMapStage 345, ShuffleMapStage 337, ShuffleMapStage 331, ShuffleMapStage 353, ShuffleMapStage 339, ShuffleMapStage 325, ShuffleMapStage 355, ShuffleMapStage 347, ShuffleMapStage 341, ShuffleMapStage 333, ShuffleMapStage 327, ShuffleMapStage 349) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 357) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Submitting ShuffleMapStage 356 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[226] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(11440) called with curMem=277488, maxMem=766075207 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 11.2 KB, free 730.3 MB) -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(4162) called with curMem=288928, maxMem=766075207 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 4.1 KB, free 730.3 MB) -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:50053 (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 356 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[226] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:15 INFO TaskSchedulerImpl:59 - Adding task set 356.0 with 2 tasks -2016-06-13 19:47:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 356.0 (TID 102, localhost, PROCESS_LOCAL, 2729 bytes) -2016-06-13 19:47:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 356.0 (TID 103, localhost, PROCESS_LOCAL, 2729 bytes) -2016-06-13 19:47:15 INFO Executor:59 - Running task 1.0 in stage 356.0 (TID 103) -2016-06-13 19:47:15 INFO Executor:59 - Running task 0.0 in stage 356.0 (TID 102) -2016-06-13 19:47:15 INFO BlockManager:59 - Found block rdd_221_1 locally -2016-06-13 19:47:15 INFO BlockManager:59 - Found block rdd_221_0 locally -2016-06-13 19:47:15 INFO Executor:59 - Finished task 0.0 in stage 356.0 (TID 102). 2254 bytes result sent to driver -2016-06-13 19:47:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 356.0 (TID 102) in 49 ms on localhost (1/2) -2016-06-13 19:47:15 INFO Executor:59 - Finished task 1.0 in stage 356.0 (TID 103). 2254 bytes result sent to driver -2016-06-13 19:47:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 356.0 (TID 103) in 73 ms on localhost (2/2) -2016-06-13 19:47:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 356.0, whose tasks have all completed, from pool -2016-06-13 19:47:15 INFO DAGScheduler:59 - ShuffleMapStage 356 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.076 s -2016-06-13 19:47:15 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:15 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:15 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 357, ResultStage 358) -2016-06-13 19:47:15 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:15 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 357: List() -2016-06-13 19:47:15 INFO DAGScheduler:59 - Missing parents for ResultStage 358: List(ShuffleMapStage 357) -2016-06-13 19:47:15 INFO DAGScheduler:59 - Submitting ShuffleMapStage 357 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[230] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=293090, maxMem=766075207 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=296986, maxMem=766075207 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:50053 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 357 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[230] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:15 INFO TaskSchedulerImpl:59 - Adding task set 357.0 with 2 tasks -2016-06-13 19:47:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 357.0 (TID 104, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 357.0 (TID 105, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:15 INFO Executor:59 - Running task 1.0 in stage 357.0 (TID 105) -2016-06-13 19:47:15 INFO Executor:59 - Running task 0.0 in stage 357.0 (TID 104) -2016-06-13 19:47:15 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:15 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:15 INFO Executor:59 - Finished task 1.0 in stage 357.0 (TID 105). 2470 bytes result sent to driver -2016-06-13 19:47:15 INFO Executor:59 - Finished task 0.0 in stage 357.0 (TID 104). 2470 bytes result sent to driver -2016-06-13 19:47:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 357.0 (TID 104) in 27 ms on localhost (1/2) -2016-06-13 19:47:15 INFO DAGScheduler:59 - ShuffleMapStage 357 (mapPartitions at GraphImpl.scala:235) finished in 0.028 s -2016-06-13 19:47:15 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:15 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:15 INFO DAGScheduler:59 - waiting: Set(ResultStage 358) -2016-06-13 19:47:15 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 357.0 (TID 105) in 27 ms on localhost (2/2) -2016-06-13 19:47:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 357.0, whose tasks have all completed, from pool -2016-06-13 19:47:15 INFO DAGScheduler:59 - Missing parents for ResultStage 358: List() -2016-06-13 19:47:15 INFO DAGScheduler:59 - Submitting ResultStage 358 (MapPartitionsRDD[238] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(12856) called with curMem=299272, maxMem=766075207 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 12.6 KB, free 730.3 MB) -2016-06-13 19:47:15 INFO MemoryStore:59 - ensureFreeSpace(4586) called with curMem=312128, maxMem=766075207 -2016-06-13 19:47:15 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.3 MB) -2016-06-13 19:47:15 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:50053 (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:47:15 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 358 (MapPartitionsRDD[238] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:15 INFO TaskSchedulerImpl:59 - Adding task set 358.0 with 2 tasks -2016-06-13 19:47:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 358.0 (TID 106, localhost, PROCESS_LOCAL, 2781 bytes) -2016-06-13 19:47:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 358.0 (TID 107, localhost, PROCESS_LOCAL, 2781 bytes) -2016-06-13 19:47:15 INFO Executor:59 - Running task 0.0 in stage 358.0 (TID 106) -2016-06-13 19:47:15 INFO Executor:59 - Running task 1.0 in stage 358.0 (TID 107) -2016-06-13 19:47:15 INFO CacheManager:59 - Partition rdd_234_0 not found, computing it -2016-06-13 19:47:15 INFO BlockManager:59 - Found block rdd_221_0 locally -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:15 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=316714, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block rdd_234_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:47:16 INFO BlockManagerInfo:59 - Added rdd_234_0 in memory on localhost:50053 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:16 INFO Executor:59 - Finished task 0.0 in stage 358.0 (TID 106). 2971 bytes result sent to driver -2016-06-13 19:47:16 INFO CacheManager:59 - Partition rdd_234_1 not found, computing it -2016-06-13 19:47:16 INFO BlockManager:59 - Found block rdd_221_1 locally -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 358.0 (TID 106) in 48 ms on localhost (1/2) -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=319258, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block rdd_234_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:47:16 INFO BlockManagerInfo:59 - Added rdd_234_1 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:16 INFO Executor:59 - Finished task 1.0 in stage 358.0 (TID 107). 2971 bytes result sent to driver -2016-06-13 19:47:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 358.0 (TID 107) in 54 ms on localhost (2/2) -2016-06-13 19:47:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 358.0, whose tasks have all completed, from pool -2016-06-13 19:47:16 INFO DAGScheduler:59 - ResultStage 358 (fold at LDAOptimizer.scala:205) finished in 0.056 s -2016-06-13 19:47:16 INFO DAGScheduler:59 - Job 18 finished: fold at LDAOptimizer.scala:205, took 0.453532 s -2016-06-13 19:47:16 INFO MapPartitionsRDD:59 - Removing RDD 208 from persistence list -2016-06-13 19:47:16 INFO ZippedPartitionsRDD2:59 - Removing RDD 215 from persistence list -2016-06-13 19:47:16 INFO BlockManager:59 - Removing RDD 208 -2016-06-13 19:47:16 INFO BlockManager:59 - Removing RDD 215 -2016-06-13 19:47:16 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:47:16 INFO DAGScheduler:59 - Registering RDD 239 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:16 INFO DAGScheduler:59 - Registering RDD 243 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:16 INFO DAGScheduler:59 - Got job 19 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:16 INFO DAGScheduler:59 - Final stage: ResultStage 397(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:16 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 386, ShuffleMapStage 372, ShuffleMapStage 394, ShuffleMapStage 380, ShuffleMapStage 359, ShuffleMapStage 366, ShuffleMapStage 396, ShuffleMapStage 360, ShuffleMapStage 388, ShuffleMapStage 382, ShuffleMapStage 374, ShuffleMapStage 368, ShuffleMapStage 390, ShuffleMapStage 376, ShuffleMapStage 362, ShuffleMapStage 392, ShuffleMapStage 384, ShuffleMapStage 378, ShuffleMapStage 370, ShuffleMapStage 364) -2016-06-13 19:47:16 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 396) -2016-06-13 19:47:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 395 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[239] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(11864) called with curMem=316714, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 11.6 KB, free 730.3 MB) -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(4268) called with curMem=328578, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.3 MB) -2016-06-13 19:47:16 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:50053 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:47:16 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 395 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[239] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:16 INFO TaskSchedulerImpl:59 - Adding task set 395.0 with 2 tasks -2016-06-13 19:47:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 395.0 (TID 108, localhost, PROCESS_LOCAL, 2770 bytes) -2016-06-13 19:47:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 395.0 (TID 109, localhost, PROCESS_LOCAL, 2770 bytes) -2016-06-13 19:47:16 INFO Executor:59 - Running task 1.0 in stage 395.0 (TID 109) -2016-06-13 19:47:16 INFO BlockManager:59 - Found block rdd_234_1 locally -2016-06-13 19:47:16 INFO Executor:59 - Running task 0.0 in stage 395.0 (TID 108) -2016-06-13 19:47:16 INFO Executor:59 - Finished task 1.0 in stage 395.0 (TID 109). 2254 bytes result sent to driver -2016-06-13 19:47:16 INFO BlockManager:59 - Found block rdd_234_0 locally -2016-06-13 19:47:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 395.0 (TID 109) in 28 ms on localhost (1/2) -2016-06-13 19:47:16 INFO Executor:59 - Finished task 0.0 in stage 395.0 (TID 108). 2254 bytes result sent to driver -2016-06-13 19:47:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 395.0 (TID 108) in 44 ms on localhost (2/2) -2016-06-13 19:47:16 INFO DAGScheduler:59 - ShuffleMapStage 395 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.044 s -2016-06-13 19:47:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:16 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:16 INFO DAGScheduler:59 - waiting: Set(ResultStage 397, ShuffleMapStage 396) -2016-06-13 19:47:16 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 395.0, whose tasks have all completed, from pool -2016-06-13 19:47:16 INFO DAGScheduler:59 - Missing parents for ResultStage 397: List(ShuffleMapStage 396) -2016-06-13 19:47:16 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 396: List() -2016-06-13 19:47:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 396 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[243] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=332846, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(2288) called with curMem=336742, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:47:16 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:50053 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:16 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 396 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[243] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:16 INFO TaskSchedulerImpl:59 - Adding task set 396.0 with 2 tasks -2016-06-13 19:47:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 396.0 (TID 110, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 396.0 (TID 111, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:16 INFO Executor:59 - Running task 1.0 in stage 396.0 (TID 111) -2016-06-13 19:47:16 INFO Executor:59 - Running task 0.0 in stage 396.0 (TID 110) -2016-06-13 19:47:16 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:16 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:16 INFO Executor:59 - Finished task 0.0 in stage 396.0 (TID 110). 2470 bytes result sent to driver -2016-06-13 19:47:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 396.0 (TID 110) in 36 ms on localhost (1/2) -2016-06-13 19:47:16 INFO Executor:59 - Finished task 1.0 in stage 396.0 (TID 111). 2470 bytes result sent to driver -2016-06-13 19:47:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 396.0 (TID 111) in 41 ms on localhost (2/2) -2016-06-13 19:47:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 396.0, whose tasks have all completed, from pool -2016-06-13 19:47:16 INFO DAGScheduler:59 - ShuffleMapStage 396 (mapPartitions at GraphImpl.scala:235) finished in 0.044 s -2016-06-13 19:47:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:16 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:16 INFO DAGScheduler:59 - waiting: Set(ResultStage 397) -2016-06-13 19:47:16 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:16 INFO DAGScheduler:59 - Missing parents for ResultStage 397: List() -2016-06-13 19:47:16 INFO DAGScheduler:59 - Submitting ResultStage 397 (MapPartitionsRDD[251] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(13280) called with curMem=339030, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 13.0 KB, free 730.3 MB) -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(4699) called with curMem=352310, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 4.6 KB, free 730.2 MB) -2016-06-13 19:47:16 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:50053 (size: 4.6 KB, free: 730.5 MB) -2016-06-13 19:47:16 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 397 (MapPartitionsRDD[251] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:16 INFO TaskSchedulerImpl:59 - Adding task set 397.0 with 2 tasks -2016-06-13 19:47:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 397.0 (TID 112, localhost, PROCESS_LOCAL, 2822 bytes) -2016-06-13 19:47:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 397.0 (TID 113, localhost, PROCESS_LOCAL, 2822 bytes) -2016-06-13 19:47:16 INFO Executor:59 - Running task 1.0 in stage 397.0 (TID 113) -2016-06-13 19:47:16 INFO CacheManager:59 - Partition rdd_247_1 not found, computing it -2016-06-13 19:47:16 INFO BlockManager:59 - Found block rdd_234_1 locally -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:16 INFO Executor:59 - Running task 0.0 in stage 397.0 (TID 112) -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=357009, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block rdd_247_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:47:16 INFO BlockManagerInfo:59 - Added rdd_247_1 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:16 INFO CacheManager:59 - Partition rdd_247_0 not found, computing it -2016-06-13 19:47:16 INFO BlockManager:59 - Found block rdd_234_0 locally -2016-06-13 19:47:16 INFO Executor:59 - Finished task 1.0 in stage 397.0 (TID 113). 2971 bytes result sent to driver -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 397.0 (TID 113) in 16 ms on localhost (1/2) -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=359457, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block rdd_247_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:47:16 INFO BlockManagerInfo:59 - Added rdd_247_0 in memory on localhost:50053 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:16 INFO Executor:59 - Finished task 0.0 in stage 397.0 (TID 112). 2971 bytes result sent to driver -2016-06-13 19:47:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 397.0 (TID 112) in 23 ms on localhost (2/2) -2016-06-13 19:47:16 INFO DAGScheduler:59 - ResultStage 397 (fold at LDAOptimizer.scala:205) finished in 0.024 s -2016-06-13 19:47:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 397.0, whose tasks have all completed, from pool -2016-06-13 19:47:16 INFO DAGScheduler:59 - Job 19 finished: fold at LDAOptimizer.scala:205, took 0.205855 s -2016-06-13 19:47:16 INFO MapPartitionsRDD:59 - Removing RDD 221 from persistence list -2016-06-13 19:47:16 INFO ZippedPartitionsRDD2:59 - Removing RDD 228 from persistence list -2016-06-13 19:47:16 INFO BlockManager:59 - Removing RDD 221 -2016-06-13 19:47:16 INFO BlockManager:59 - Removing RDD 228 -2016-06-13 19:47:16 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 155 bytes -2016-06-13 19:47:16 INFO DAGScheduler:59 - Registering RDD 252 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:16 INFO DAGScheduler:59 - Registering RDD 256 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:16 INFO DAGScheduler:59 - Got job 20 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:16 INFO DAGScheduler:59 - Final stage: ResultStage 438(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:16 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 407, ShuffleMapStage 437, ShuffleMapStage 401, ShuffleMapStage 423, ShuffleMapStage 409, ShuffleMapStage 431, ShuffleMapStage 417, ShuffleMapStage 403, ShuffleMapStage 433, ShuffleMapStage 425, ShuffleMapStage 419, ShuffleMapStage 411, ShuffleMapStage 405, ShuffleMapStage 427, ShuffleMapStage 398, ShuffleMapStage 413, ShuffleMapStage 435, ShuffleMapStage 399, ShuffleMapStage 429, ShuffleMapStage 421, ShuffleMapStage 415) -2016-06-13 19:47:16 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 437) -2016-06-13 19:47:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 436 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[252] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(12296) called with curMem=357009, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 12.0 KB, free 730.2 MB) -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(4356) called with curMem=369305, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.2 MB) -2016-06-13 19:47:16 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:50053 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:47:16 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 436 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[252] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:16 INFO TaskSchedulerImpl:59 - Adding task set 436.0 with 2 tasks -2016-06-13 19:47:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 436.0 (TID 114, localhost, PROCESS_LOCAL, 2811 bytes) -2016-06-13 19:47:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 436.0 (TID 115, localhost, PROCESS_LOCAL, 2811 bytes) -2016-06-13 19:47:16 INFO Executor:59 - Running task 1.0 in stage 436.0 (TID 115) -2016-06-13 19:47:16 INFO Executor:59 - Running task 0.0 in stage 436.0 (TID 114) -2016-06-13 19:47:16 INFO BlockManager:59 - Found block rdd_247_1 locally -2016-06-13 19:47:16 INFO BlockManager:59 - Found block rdd_247_0 locally -2016-06-13 19:47:16 INFO Executor:59 - Finished task 0.0 in stage 436.0 (TID 114). 2254 bytes result sent to driver -2016-06-13 19:47:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 436.0 (TID 114) in 31 ms on localhost (1/2) -2016-06-13 19:47:16 INFO Executor:59 - Finished task 1.0 in stage 436.0 (TID 115). 2254 bytes result sent to driver -2016-06-13 19:47:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 436.0 (TID 115) in 37 ms on localhost (2/2) -2016-06-13 19:47:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 436.0, whose tasks have all completed, from pool -2016-06-13 19:47:16 INFO DAGScheduler:59 - ShuffleMapStage 436 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.039 s -2016-06-13 19:47:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:16 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:16 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 437, ResultStage 438) -2016-06-13 19:47:16 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:16 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 437: List() -2016-06-13 19:47:16 INFO DAGScheduler:59 - Missing parents for ResultStage 438: List(ShuffleMapStage 437) -2016-06-13 19:47:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 437 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[256] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=373661, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(2289) called with curMem=377557, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:47:16 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:50053 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:16 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 437 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[256] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:16 INFO TaskSchedulerImpl:59 - Adding task set 437.0 with 2 tasks -2016-06-13 19:47:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 437.0 (TID 116, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 437.0 (TID 117, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:16 INFO Executor:59 - Running task 0.0 in stage 437.0 (TID 116) -2016-06-13 19:47:16 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:16 INFO Executor:59 - Running task 1.0 in stage 437.0 (TID 117) -2016-06-13 19:47:16 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:16 INFO Executor:59 - Finished task 0.0 in stage 437.0 (TID 116). 2470 bytes result sent to driver -2016-06-13 19:47:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 437.0 (TID 116) in 66 ms on localhost (1/2) -2016-06-13 19:47:16 INFO Executor:59 - Finished task 1.0 in stage 437.0 (TID 117). 2470 bytes result sent to driver -2016-06-13 19:47:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 437.0 (TID 117) in 78 ms on localhost (2/2) -2016-06-13 19:47:16 INFO DAGScheduler:59 - ShuffleMapStage 437 (mapPartitions at GraphImpl.scala:235) finished in 0.079 s -2016-06-13 19:47:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:16 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:16 INFO DAGScheduler:59 - waiting: Set(ResultStage 438) -2016-06-13 19:47:16 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 437.0, whose tasks have all completed, from pool -2016-06-13 19:47:16 INFO DAGScheduler:59 - Missing parents for ResultStage 438: List() -2016-06-13 19:47:16 INFO DAGScheduler:59 - Submitting ResultStage 438 (MapPartitionsRDD[264] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(13712) called with curMem=379846, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 13.4 KB, free 730.2 MB) -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(4770) called with curMem=393558, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 4.7 KB, free 730.2 MB) -2016-06-13 19:47:16 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:50053 (size: 4.7 KB, free: 730.5 MB) -2016-06-13 19:47:16 INFO SparkContext:59 - Created broadcast 61 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 438 (MapPartitionsRDD[264] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:16 INFO TaskSchedulerImpl:59 - Adding task set 438.0 with 2 tasks -2016-06-13 19:47:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 438.0 (TID 118, localhost, PROCESS_LOCAL, 2863 bytes) -2016-06-13 19:47:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 438.0 (TID 119, localhost, PROCESS_LOCAL, 2863 bytes) -2016-06-13 19:47:16 INFO Executor:59 - Running task 1.0 in stage 438.0 (TID 119) -2016-06-13 19:47:16 INFO Executor:59 - Running task 0.0 in stage 438.0 (TID 118) -2016-06-13 19:47:16 INFO CacheManager:59 - Partition rdd_260_0 not found, computing it -2016-06-13 19:47:16 INFO CacheManager:59 - Partition rdd_260_1 not found, computing it -2016-06-13 19:47:16 INFO BlockManager:59 - Found block rdd_247_0 locally -2016-06-13 19:47:16 INFO BlockManager:59 - Found block rdd_247_1 locally -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=398328, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block rdd_260_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:47:16 INFO BlockManagerInfo:59 - Added rdd_260_0 in memory on localhost:50053 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=400872, maxMem=766075207 -2016-06-13 19:47:16 INFO Executor:59 - Finished task 0.0 in stage 438.0 (TID 118). 2971 bytes result sent to driver -2016-06-13 19:47:16 INFO MemoryStore:59 - Block rdd_260_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:47:16 INFO BlockManagerInfo:59 - Added rdd_260_1 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 438.0 (TID 118) in 18 ms on localhost (1/2) -2016-06-13 19:47:16 INFO Executor:59 - Finished task 1.0 in stage 438.0 (TID 119). 2971 bytes result sent to driver -2016-06-13 19:47:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 438.0 (TID 119) in 22 ms on localhost (2/2) -2016-06-13 19:47:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 438.0, whose tasks have all completed, from pool -2016-06-13 19:47:16 INFO DAGScheduler:59 - ResultStage 438 (fold at LDAOptimizer.scala:205) finished in 0.025 s -2016-06-13 19:47:16 INFO DAGScheduler:59 - Job 20 finished: fold at LDAOptimizer.scala:205, took 0.284937 s -2016-06-13 19:47:16 INFO MapPartitionsRDD:59 - Removing RDD 234 from persistence list -2016-06-13 19:47:16 INFO ZippedPartitionsRDD2:59 - Removing RDD 241 from persistence list -2016-06-13 19:47:16 INFO BlockManager:59 - Removing RDD 241 -2016-06-13 19:47:16 INFO BlockManager:59 - Removing RDD 234 -2016-06-13 19:47:16 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 155 bytes -2016-06-13 19:47:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:47:16 INFO DAGScheduler:59 - Registering RDD 265 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:16 INFO DAGScheduler:59 - Registering RDD 269 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:16 INFO DAGScheduler:59 - Got job 21 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:16 INFO DAGScheduler:59 - Final stage: ResultStage 481(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:16 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 466, ShuffleMapStage 458, ShuffleMapStage 452, ShuffleMapStage 444, ShuffleMapStage 474, ShuffleMapStage 460, ShuffleMapStage 446, ShuffleMapStage 468, ShuffleMapStage 439, ShuffleMapStage 454, ShuffleMapStage 476, ShuffleMapStage 440, ShuffleMapStage 470, ShuffleMapStage 462, ShuffleMapStage 456, ShuffleMapStage 448, ShuffleMapStage 478, ShuffleMapStage 442, ShuffleMapStage 464, ShuffleMapStage 450, ShuffleMapStage 480, ShuffleMapStage 472) -2016-06-13 19:47:16 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 480) -2016-06-13 19:47:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 479 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[265] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(12720) called with curMem=398328, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 12.4 KB, free 730.2 MB) -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(4470) called with curMem=411048, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.2 MB) -2016-06-13 19:47:16 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:50053 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:47:16 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 479 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[265] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:16 INFO TaskSchedulerImpl:59 - Adding task set 479.0 with 2 tasks -2016-06-13 19:47:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 479.0 (TID 120, localhost, PROCESS_LOCAL, 2852 bytes) -2016-06-13 19:47:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 479.0 (TID 121, localhost, PROCESS_LOCAL, 2852 bytes) -2016-06-13 19:47:16 INFO Executor:59 - Running task 1.0 in stage 479.0 (TID 121) -2016-06-13 19:47:16 INFO Executor:59 - Running task 0.0 in stage 479.0 (TID 120) -2016-06-13 19:47:16 INFO BlockManager:59 - Found block rdd_260_1 locally -2016-06-13 19:47:16 INFO BlockManager:59 - Found block rdd_260_0 locally -2016-06-13 19:47:16 INFO Executor:59 - Finished task 0.0 in stage 479.0 (TID 120). 2254 bytes result sent to driver -2016-06-13 19:47:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 479.0 (TID 120) in 52 ms on localhost (1/2) -2016-06-13 19:47:16 INFO Executor:59 - Finished task 1.0 in stage 479.0 (TID 121). 2254 bytes result sent to driver -2016-06-13 19:47:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 479.0 (TID 121) in 66 ms on localhost (2/2) -2016-06-13 19:47:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 479.0, whose tasks have all completed, from pool -2016-06-13 19:47:16 INFO DAGScheduler:59 - ShuffleMapStage 479 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.073 s -2016-06-13 19:47:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:16 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:16 INFO DAGScheduler:59 - waiting: Set(ResultStage 481, ShuffleMapStage 480) -2016-06-13 19:47:16 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:16 INFO DAGScheduler:59 - Missing parents for ResultStage 481: List(ShuffleMapStage 480) -2016-06-13 19:47:16 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 480: List() -2016-06-13 19:47:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 480 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[269] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=415518, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(2285) called with curMem=419414, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:47:16 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:50053 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:16 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 480 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[269] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:16 INFO TaskSchedulerImpl:59 - Adding task set 480.0 with 2 tasks -2016-06-13 19:47:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 480.0 (TID 122, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 480.0 (TID 123, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:16 INFO Executor:59 - Running task 1.0 in stage 480.0 (TID 123) -2016-06-13 19:47:16 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:16 INFO Executor:59 - Running task 0.0 in stage 480.0 (TID 122) -2016-06-13 19:47:16 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:16 INFO Executor:59 - Finished task 1.0 in stage 480.0 (TID 123). 2470 bytes result sent to driver -2016-06-13 19:47:16 INFO TaskSetManager:59 - Finished task 1.0 in stage 480.0 (TID 123) in 71 ms on localhost (1/2) -2016-06-13 19:47:16 INFO Executor:59 - Finished task 0.0 in stage 480.0 (TID 122). 2470 bytes result sent to driver -2016-06-13 19:47:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 480.0 (TID 122) in 74 ms on localhost (2/2) -2016-06-13 19:47:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 480.0, whose tasks have all completed, from pool -2016-06-13 19:47:16 INFO DAGScheduler:59 - ShuffleMapStage 480 (mapPartitions at GraphImpl.scala:235) finished in 0.090 s -2016-06-13 19:47:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:16 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:16 INFO DAGScheduler:59 - waiting: Set(ResultStage 481) -2016-06-13 19:47:16 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:16 INFO DAGScheduler:59 - Missing parents for ResultStage 481: List() -2016-06-13 19:47:16 INFO DAGScheduler:59 - Submitting ResultStage 481 (MapPartitionsRDD[277] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(14136) called with curMem=421699, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 13.8 KB, free 730.2 MB) -2016-06-13 19:47:16 INFO MemoryStore:59 - ensureFreeSpace(4890) called with curMem=435835, maxMem=766075207 -2016-06-13 19:47:16 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 4.8 KB, free 730.2 MB) -2016-06-13 19:47:16 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:50053 (size: 4.8 KB, free: 730.5 MB) -2016-06-13 19:47:16 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 481 (MapPartitionsRDD[277] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:17 INFO TaskSchedulerImpl:59 - Adding task set 481.0 with 2 tasks -2016-06-13 19:47:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 481.0 (TID 124, localhost, PROCESS_LOCAL, 2904 bytes) -2016-06-13 19:47:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 481.0 (TID 125, localhost, PROCESS_LOCAL, 2904 bytes) -2016-06-13 19:47:17 INFO Executor:59 - Running task 1.0 in stage 481.0 (TID 125) -2016-06-13 19:47:17 INFO Executor:59 - Running task 0.0 in stage 481.0 (TID 124) -2016-06-13 19:47:17 INFO CacheManager:59 - Partition rdd_273_0 not found, computing it -2016-06-13 19:47:17 INFO BlockManager:59 - Found block rdd_260_0 locally -2016-06-13 19:47:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:17 INFO CacheManager:59 - Partition rdd_273_1 not found, computing it -2016-06-13 19:47:17 INFO BlockManager:59 - Found block rdd_260_1 locally -2016-06-13 19:47:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:17 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=440725, maxMem=766075207 -2016-06-13 19:47:17 INFO MemoryStore:59 - Block rdd_273_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:47:17 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=443173, maxMem=766075207 -2016-06-13 19:47:17 INFO MemoryStore:59 - Block rdd_273_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Added rdd_273_1 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Added rdd_273_0 in memory on localhost:50053 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:17 INFO Executor:59 - Finished task 1.0 in stage 481.0 (TID 125). 2971 bytes result sent to driver -2016-06-13 19:47:17 INFO Executor:59 - Finished task 0.0 in stage 481.0 (TID 124). 2971 bytes result sent to driver -2016-06-13 19:47:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 481.0 (TID 125) in 17 ms on localhost (1/2) -2016-06-13 19:47:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 481.0 (TID 124) in 19 ms on localhost (2/2) -2016-06-13 19:47:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 481.0, whose tasks have all completed, from pool -2016-06-13 19:47:17 INFO DAGScheduler:59 - ResultStage 481 (fold at LDAOptimizer.scala:205) finished in 0.020 s -2016-06-13 19:47:17 INFO DAGScheduler:59 - Job 21 finished: fold at LDAOptimizer.scala:205, took 0.362352 s -2016-06-13 19:47:17 INFO MapPartitionsRDD:59 - Removing RDD 247 from persistence list -2016-06-13 19:47:17 INFO ZippedPartitionsRDD2:59 - Removing RDD 254 from persistence list -2016-06-13 19:47:17 INFO BlockManager:59 - Removing RDD 247 -2016-06-13 19:47:17 INFO BlockManager:59 - Removing RDD 254 -2016-06-13 19:47:17 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 41 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 40 is 155 bytes -2016-06-13 19:47:17 INFO DAGScheduler:59 - Registering RDD 278 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:17 INFO DAGScheduler:59 - Registering RDD 282 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:17 INFO DAGScheduler:59 - Got job 22 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:17 INFO DAGScheduler:59 - Final stage: ResultStage 526(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 487, ShuffleMapStage 509, ShuffleMapStage 503, ShuffleMapStage 495, ShuffleMapStage 525, ShuffleMapStage 489, ShuffleMapStage 517, ShuffleMapStage 511, ShuffleMapStage 482, ShuffleMapStage 497, ShuffleMapStage 519, ShuffleMapStage 483, ShuffleMapStage 505, ShuffleMapStage 491, ShuffleMapStage 521, ShuffleMapStage 513, ShuffleMapStage 507, ShuffleMapStage 499, ShuffleMapStage 493, ShuffleMapStage 485, ShuffleMapStage 515, ShuffleMapStage 501, ShuffleMapStage 523) -2016-06-13 19:47:17 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 525) -2016-06-13 19:47:17 INFO DAGScheduler:59 - Submitting ShuffleMapStage 524 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[278] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:17 INFO MemoryStore:59 - ensureFreeSpace(13152) called with curMem=440725, maxMem=766075207 -2016-06-13 19:47:17 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 12.8 KB, free 730.2 MB) -2016-06-13 19:47:17 INFO MemoryStore:59 - ensureFreeSpace(4564) called with curMem=453877, maxMem=766075207 -2016-06-13 19:47:17 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.1 MB) -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:50053 (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:47:17 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 524 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[278] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:17 INFO TaskSchedulerImpl:59 - Adding task set 524.0 with 2 tasks -2016-06-13 19:47:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 524.0 (TID 126, localhost, PROCESS_LOCAL, 2893 bytes) -2016-06-13 19:47:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 524.0 (TID 127, localhost, PROCESS_LOCAL, 2893 bytes) -2016-06-13 19:47:17 INFO Executor:59 - Running task 0.0 in stage 524.0 (TID 126) -2016-06-13 19:47:17 INFO BlockManager:59 - Found block rdd_273_0 locally -2016-06-13 19:47:17 INFO Executor:59 - Running task 1.0 in stage 524.0 (TID 127) -2016-06-13 19:47:17 INFO BlockManager:59 - Found block rdd_273_1 locally -2016-06-13 19:47:17 INFO Executor:59 - Finished task 0.0 in stage 524.0 (TID 126). 2254 bytes result sent to driver -2016-06-13 19:47:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 524.0 (TID 126) in 62 ms on localhost (1/2) -2016-06-13 19:47:17 INFO Executor:59 - Finished task 1.0 in stage 524.0 (TID 127). 2254 bytes result sent to driver -2016-06-13 19:47:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 524.0 (TID 127) in 69 ms on localhost (2/2) -2016-06-13 19:47:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 524.0, whose tasks have all completed, from pool -2016-06-13 19:47:17 INFO DAGScheduler:59 - ShuffleMapStage 524 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.070 s -2016-06-13 19:47:17 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:17 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:17 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 525, ResultStage 526) -2016-06-13 19:47:17 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:17 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 525: List() -2016-06-13 19:47:17 INFO DAGScheduler:59 - Missing parents for ResultStage 526: List(ShuffleMapStage 525) -2016-06-13 19:47:17 INFO DAGScheduler:59 - Submitting ShuffleMapStage 525 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[282] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:17 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=458441, maxMem=766075207 -2016-06-13 19:47:17 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 3.8 KB, free 730.1 MB) -2016-06-13 19:47:17 INFO MemoryStore:59 - ensureFreeSpace(2289) called with curMem=462337, maxMem=766075207 -2016-06-13 19:47:17 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:50053 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:17 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 525 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[282] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:17 INFO TaskSchedulerImpl:59 - Adding task set 525.0 with 2 tasks -2016-06-13 19:47:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 525.0 (TID 128, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 525.0 (TID 129, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:17 INFO Executor:59 - Running task 1.0 in stage 525.0 (TID 129) -2016-06-13 19:47:17 INFO Executor:59 - Running task 0.0 in stage 525.0 (TID 128) -2016-06-13 19:47:17 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:17 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:17 INFO Executor:59 - Finished task 0.0 in stage 525.0 (TID 128). 2470 bytes result sent to driver -2016-06-13 19:47:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 525.0 (TID 128) in 41 ms on localhost (1/2) -2016-06-13 19:47:17 INFO Executor:59 - Finished task 1.0 in stage 525.0 (TID 129). 2470 bytes result sent to driver -2016-06-13 19:47:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 525.0 (TID 129) in 66 ms on localhost (2/2) -2016-06-13 19:47:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 525.0, whose tasks have all completed, from pool -2016-06-13 19:47:17 INFO DAGScheduler:59 - ShuffleMapStage 525 (mapPartitions at GraphImpl.scala:235) finished in 0.067 s -2016-06-13 19:47:17 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:17 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:17 INFO DAGScheduler:59 - waiting: Set(ResultStage 526) -2016-06-13 19:47:17 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:17 INFO DAGScheduler:59 - Missing parents for ResultStage 526: List() -2016-06-13 19:47:17 INFO DAGScheduler:59 - Submitting ResultStage 526 (MapPartitionsRDD[290] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:17 INFO MemoryStore:59 - ensureFreeSpace(14568) called with curMem=464626, maxMem=766075207 -2016-06-13 19:47:17 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 14.2 KB, free 730.1 MB) -2016-06-13 19:47:17 INFO MemoryStore:59 - ensureFreeSpace(4990) called with curMem=479194, maxMem=766075207 -2016-06-13 19:47:17 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 4.9 KB, free 730.1 MB) -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:50053 (size: 4.9 KB, free: 730.5 MB) -2016-06-13 19:47:17 INFO SparkContext:59 - Created broadcast 67 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 526 (MapPartitionsRDD[290] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:17 INFO TaskSchedulerImpl:59 - Adding task set 526.0 with 2 tasks -2016-06-13 19:47:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 526.0 (TID 130, localhost, PROCESS_LOCAL, 2945 bytes) -2016-06-13 19:47:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 526.0 (TID 131, localhost, PROCESS_LOCAL, 2945 bytes) -2016-06-13 19:47:17 INFO Executor:59 - Running task 1.0 in stage 526.0 (TID 131) -2016-06-13 19:47:17 INFO Executor:59 - Running task 0.0 in stage 526.0 (TID 130) -2016-06-13 19:47:17 INFO CacheManager:59 - Partition rdd_286_1 not found, computing it -2016-06-13 19:47:17 INFO BlockManager:59 - Found block rdd_273_1 locally -2016-06-13 19:47:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:17 INFO CacheManager:59 - Partition rdd_286_0 not found, computing it -2016-06-13 19:47:17 INFO BlockManager:59 - Found block rdd_273_0 locally -2016-06-13 19:47:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:17 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=484184, maxMem=766075207 -2016-06-13 19:47:17 INFO MemoryStore:59 - Block rdd_286_0 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Added rdd_286_0 in memory on localhost:50053 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:17 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=486728, maxMem=766075207 -2016-06-13 19:47:17 INFO MemoryStore:59 - Block rdd_286_1 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Added rdd_286_1 in memory on localhost:50053 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:17 INFO Executor:59 - Finished task 1.0 in stage 526.0 (TID 131). 2971 bytes result sent to driver -2016-06-13 19:47:17 INFO Executor:59 - Finished task 0.0 in stage 526.0 (TID 130). 2971 bytes result sent to driver -2016-06-13 19:47:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 526.0 (TID 131) in 21 ms on localhost (1/2) -2016-06-13 19:47:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 526.0 (TID 130) in 25 ms on localhost (2/2) -2016-06-13 19:47:17 INFO DAGScheduler:59 - ResultStage 526 (fold at LDAOptimizer.scala:205) finished in 0.025 s -2016-06-13 19:47:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 526.0, whose tasks have all completed, from pool -2016-06-13 19:47:17 INFO DAGScheduler:59 - Job 22 finished: fold at LDAOptimizer.scala:205, took 0.262727 s -2016-06-13 19:47:17 INFO SparkContext:59 - Starting job: foreach at LDAFromWebSuite.scala:32 -2016-06-13 19:47:17 INFO DAGScheduler:59 - Got job 23 (foreach at LDAFromWebSuite.scala:32) with 2 output partitions -2016-06-13 19:47:17 INFO DAGScheduler:59 - Final stage: ResultStage 527(foreach at LDAFromWebSuite.scala:32) -2016-06-13 19:47:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:47:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:47:17 INFO DAGScheduler:59 - Submitting ResultStage 527 (MapPartitionsRDD[2] at map at LDAFromWebSuite.scala:23), which has no missing parents -2016-06-13 19:47:17 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=489176, maxMem=766075207 -2016-06-13 19:47:17 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 3.3 KB, free 730.1 MB) -2016-06-13 19:47:17 INFO MemoryStore:59 - ensureFreeSpace(1967) called with curMem=492552, maxMem=766075207 -2016-06-13 19:47:17 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 1967.0 B, free 730.1 MB) -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:50053 (size: 1967.0 B, free: 730.5 MB) -2016-06-13 19:47:17 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 527 (MapPartitionsRDD[2] at map at LDAFromWebSuite.scala:23) -2016-06-13 19:47:17 INFO TaskSchedulerImpl:59 - Adding task set 527.0 with 2 tasks -2016-06-13 19:47:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 527.0 (TID 132, localhost, PROCESS_LOCAL, 2197 bytes) -2016-06-13 19:47:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 527.0 (TID 133, localhost, PROCESS_LOCAL, 2197 bytes) -2016-06-13 19:47:17 INFO Executor:59 - Running task 1.0 in stage 527.0 (TID 133) -2016-06-13 19:47:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:132+132 -2016-06-13 19:47:17 INFO Executor:59 - Running task 0.0 in stage 527.0 (TID 132) -2016-06-13 19:47:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:0+132 -2016-06-13 19:47:17 INFO Executor:59 - Finished task 0.0 in stage 527.0 (TID 132). 2044 bytes result sent to driver -2016-06-13 19:47:17 INFO Executor:59 - Finished task 1.0 in stage 527.0 (TID 133). 2044 bytes result sent to driver -2016-06-13 19:47:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 527.0 (TID 132) in 25 ms on localhost (1/2) -2016-06-13 19:47:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 527.0 (TID 133) in 24 ms on localhost (2/2) -2016-06-13 19:47:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 527.0, whose tasks have all completed, from pool -2016-06-13 19:47:17 INFO DAGScheduler:59 - ResultStage 527 (foreach at LDAFromWebSuite.scala:32) finished in 0.028 s -2016-06-13 19:47:17 INFO DAGScheduler:59 - Job 23 finished: foreach at LDAFromWebSuite.scala:32, took 0.039763 s -2016-06-13 19:47:17 INFO SparkContext:59 - Starting job: foreach at LDAFromWebSuite.scala:34 -2016-06-13 19:47:17 INFO DAGScheduler:59 - Got job 24 (foreach at LDAFromWebSuite.scala:34) with 2 output partitions -2016-06-13 19:47:17 INFO DAGScheduler:59 - Final stage: ResultStage 528(foreach at LDAFromWebSuite.scala:34) -2016-06-13 19:47:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:47:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:47:17 INFO DAGScheduler:59 - Submitting ResultStage 528 (MapPartitionsRDD[4] at map at LDAFromWebSuite.scala:25), which has no missing parents -2016-06-13 19:47:17 INFO MemoryStore:59 - ensureFreeSpace(3672) called with curMem=494519, maxMem=766075207 -2016-06-13 19:47:17 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 3.6 KB, free 730.1 MB) -2016-06-13 19:47:17 INFO MemoryStore:59 - ensureFreeSpace(2055) called with curMem=498191, maxMem=766075207 -2016-06-13 19:47:17 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.1 MB) -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:50053 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:47:17 INFO SparkContext:59 - Created broadcast 69 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 528 (MapPartitionsRDD[4] at map at LDAFromWebSuite.scala:25) -2016-06-13 19:47:17 INFO TaskSchedulerImpl:59 - Adding task set 528.0 with 2 tasks -2016-06-13 19:47:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 528.0 (TID 134, localhost, PROCESS_LOCAL, 2307 bytes) -2016-06-13 19:47:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 528.0 (TID 135, localhost, PROCESS_LOCAL, 2307 bytes) -2016-06-13 19:47:17 INFO Executor:59 - Running task 1.0 in stage 528.0 (TID 135) -2016-06-13 19:47:17 INFO Executor:59 - Running task 0.0 in stage 528.0 (TID 134) -2016-06-13 19:47:17 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 19:47:17 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:47:17 INFO Executor:59 - Finished task 1.0 in stage 528.0 (TID 135). 2044 bytes result sent to driver -2016-06-13 19:47:17 INFO Executor:59 - Finished task 0.0 in stage 528.0 (TID 134). 2044 bytes result sent to driver -2016-06-13 19:47:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 528.0 (TID 135) in 5 ms on localhost (1/2) -2016-06-13 19:47:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 528.0 (TID 134) in 6 ms on localhost (2/2) -2016-06-13 19:47:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 528.0, whose tasks have all completed, from pool -2016-06-13 19:47:17 INFO DAGScheduler:59 - ResultStage 528 (foreach at LDAFromWebSuite.scala:34) finished in 0.007 s -2016-06-13 19:47:17 INFO DAGScheduler:59 - Job 24 finished: foreach at LDAFromWebSuite.scala:34, took 0.018621 s -2016-06-13 19:47:17 INFO SparkContext:59 - Starting job: collect at LDAModel.scala:528 -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 41 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 40 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 43 is 155 bytes -2016-06-13 19:47:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 42 is 155 bytes -2016-06-13 19:47:17 INFO DAGScheduler:59 - Got job 25 (collect at LDAModel.scala:528) with 2 output partitions -2016-06-13 19:47:17 INFO DAGScheduler:59 - Final stage: ResultStage 573(collect at LDAModel.scala:528) -2016-06-13 19:47:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 538, ShuffleMapStage 560, ShuffleMapStage 546, ShuffleMapStage 540, ShuffleMapStage 568, ShuffleMapStage 532, ShuffleMapStage 562, ShuffleMapStage 554, ShuffleMapStage 548, ShuffleMapStage 570, ShuffleMapStage 534, ShuffleMapStage 556, ShuffleMapStage 542, ShuffleMapStage 572, ShuffleMapStage 564, ShuffleMapStage 558, ShuffleMapStage 550, ShuffleMapStage 529, ShuffleMapStage 544, ShuffleMapStage 536, ShuffleMapStage 566, ShuffleMapStage 530, ShuffleMapStage 552) -2016-06-13 19:47:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:47:17 INFO DAGScheduler:59 - Submitting ResultStage 573 (MapPartitionsRDD[293] at map at LDAModel.scala:526), which has no missing parents -2016-06-13 19:47:17 INFO MemoryStore:59 - ensureFreeSpace(14256) called with curMem=500246, maxMem=766075207 -2016-06-13 19:47:17 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 13.9 KB, free 730.1 MB) -2016-06-13 19:47:17 INFO MemoryStore:59 - ensureFreeSpace(4871) called with curMem=514502, maxMem=766075207 -2016-06-13 19:47:17 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 4.8 KB, free 730.1 MB) -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:50053 (size: 4.8 KB, free: 730.5 MB) -2016-06-13 19:47:17 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 573 (MapPartitionsRDD[293] at map at LDAModel.scala:526) -2016-06-13 19:47:17 INFO TaskSchedulerImpl:59 - Adding task set 573.0 with 2 tasks -2016-06-13 19:47:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 573.0 (TID 136, localhost, PROCESS_LOCAL, 2945 bytes) -2016-06-13 19:47:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 573.0 (TID 137, localhost, PROCESS_LOCAL, 2945 bytes) -2016-06-13 19:47:17 INFO Executor:59 - Running task 0.0 in stage 573.0 (TID 136) -2016-06-13 19:47:17 INFO Executor:59 - Running task 1.0 in stage 573.0 (TID 137) -2016-06-13 19:47:17 INFO BlockManager:59 - Found block rdd_286_0 locally -2016-06-13 19:47:17 INFO Executor:59 - Finished task 0.0 in stage 573.0 (TID 136). 2680 bytes result sent to driver -2016-06-13 19:47:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 573.0 (TID 136) in 22 ms on localhost (1/2) -2016-06-13 19:47:17 INFO BlockManager:59 - Found block rdd_286_1 locally -2016-06-13 19:47:17 INFO Executor:59 - Finished task 1.0 in stage 573.0 (TID 137). 2612 bytes result sent to driver -2016-06-13 19:47:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 573.0 (TID 137) in 40 ms on localhost (2/2) -2016-06-13 19:47:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 573.0, whose tasks have all completed, from pool -2016-06-13 19:47:17 INFO DAGScheduler:59 - ResultStage 573 (collect at LDAModel.scala:528) finished in 0.075 s -2016-06-13 19:47:17 INFO DAGScheduler:59 - Job 25 finished: collect at LDAModel.scala:528, took 0.236968 s -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:50053 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:47:17 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:50053 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:17 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:50053 in memory (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:47:17 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:50053 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:47:17 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:50053 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:17 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:50053 in memory (size: 3.9 KB, free: 730.5 MB) -2016-06-13 19:47:17 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Removed broadcast_70_piece0 on localhost:50053 in memory (size: 4.8 KB, free: 730.5 MB) -2016-06-13 19:47:17 INFO ContextCleaner:59 - Cleaned accumulator 70 -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Removed broadcast_69_piece0 on localhost:50053 in memory (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:47:17 INFO ContextCleaner:59 - Cleaned accumulator 69 -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Removed broadcast_68_piece0 on localhost:50053 in memory (size: 1967.0 B, free: 730.5 MB) -2016-06-13 19:47:17 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Removed broadcast_67_piece0 on localhost:50053 in memory (size: 4.9 KB, free: 730.5 MB) -2016-06-13 19:47:17 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Removed broadcast_66_piece0 on localhost:50053 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:17 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-06-13 19:47:17 INFO BlockManagerInfo:59 - Removed broadcast_65_piece0 on localhost:50053 in memory (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:47:17 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-06-13 19:47:18 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:50053 in memory (size: 4.8 KB, free: 730.5 MB) -2016-06-13 19:47:18 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-06-13 19:47:18 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:50053 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:18 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-06-13 19:47:18 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:50053 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:47:18 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-06-13 19:47:18 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:50053 in memory (size: 4.7 KB, free: 730.5 MB) -2016-06-13 19:47:18 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-06-13 19:47:18 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:50053 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:18 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-06-13 19:47:18 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:50053 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:47:18 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-06-13 19:47:18 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:50053 in memory (size: 4.6 KB, free: 730.5 MB) -2016-06-13 19:47:18 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-06-13 19:47:18 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:50053 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:18 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-06-13 19:47:18 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:50053 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:47:18 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-06-13 19:47:18 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:50053 in memory (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:47:18 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-06-13 19:47:18 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:50053 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:18 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-06-13 19:47:18 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:50053 in memory (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:47:18 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 19:47:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 19:47:18 WARN QueuedThreadPool:145 - 3 threads could not be stopped -2016-06-13 19:47:18 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4041 -2016-06-13 19:47:18 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 19:47:18 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 19:47:19 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 19:47:19 INFO BlockManager:59 - BlockManager stopped -2016-06-13 19:47:20 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 19:47:20 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 19:47:20 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 19:47:20 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 19:47:20 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 19:47:20 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 19:47:20 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-b2990e26-46d0-4b19-9771-d7a6e8f266ee -2016-06-13 19:47:20 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 19:47:20 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 19:47:20 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 19:47:20 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 19:47:20 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 19:47:22 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 19:47:22 INFO Remoting:74 - Starting remoting -2016-06-13 19:47:22 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:50072] -2016-06-13 19:47:22 INFO Utils:59 - Successfully started service 'sparkDriver' on port 50072. -2016-06-13 19:47:22 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 19:47:22 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 19:47:22 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-5ec0160b-70ac-4e57-a00b-2051af11f7d5 -2016-06-13 19:47:22 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 19:47:22 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-4b7db2a4-66ff-443f-b499-1052d0811472\httpd-08b5abf0-cf46-4b12-ae3e-e5da9a5fd5d6 -2016-06-13 19:47:22 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 19:47:22 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:47:22 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:50073 -2016-06-13 19:47:22 INFO Utils:59 - Successfully started service 'HTTP file server' on port 50073. -2016-06-13 19:47:22 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 19:47:22 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 19:47:22 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 19:47:22 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 19:47:22 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 19:47:23 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 19:47:23 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 19:47:23 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 50092. -2016-06-13 19:47:23 INFO NettyBlockTransferService:59 - Server created on 50092 -2016-06-13 19:47:23 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 19:47:23 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:50092 with 730.6 MB RAM, BlockManagerId(driver, localhost, 50092) -2016-06-13 19:47:23 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 19:47:24 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 19:47:24 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 19:47:24 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 19:47:24 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 19:47:24 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:50092 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 19:47:24 INFO SparkContext:59 - Created broadcast 0 from textFile at LDAFromWebSuite.scala:22 -2016-06-13 19:47:25 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 19:47:26 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 19:47:26 INFO SparkContext:59 - Starting job: zipWithIndex at LDAFromWebSuite.scala:25 -2016-06-13 19:47:26 INFO DAGScheduler:59 - Got job 0 (zipWithIndex at LDAFromWebSuite.scala:25) with 1 output partitions -2016-06-13 19:47:26 INFO DAGScheduler:59 - Final stage: ResultStage 0(zipWithIndex at LDAFromWebSuite.scala:25) -2016-06-13 19:47:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:47:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:47:26 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at map at LDAFromWebSuite.scala:23), which has no missing parents -2016-06-13 19:47:26 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=171580, maxMem=766075207 -2016-06-13 19:47:26 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:47:26 INFO MemoryStore:59 - ensureFreeSpace(1939) called with curMem=174820, maxMem=766075207 -2016-06-13 19:47:26 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 1939.0 B, free 730.4 MB) -2016-06-13 19:47:26 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:50092 (size: 1939.0 B, free: 730.6 MB) -2016-06-13 19:47:26 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at map at LDAFromWebSuite.scala:23) -2016-06-13 19:47:26 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 19:47:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2197 bytes) -2016-06-13 19:47:26 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 19:47:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:0+132 -2016-06-13 19:47:26 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 19:47:26 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 19:47:26 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 19:47:26 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 19:47:26 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 19:47:26 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2082 bytes result sent to driver -2016-06-13 19:47:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 148 ms on localhost (1/1) -2016-06-13 19:47:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 19:47:26 INFO DAGScheduler:59 - ResultStage 0 (zipWithIndex at LDAFromWebSuite.scala:25) finished in 0.176 s -2016-06-13 19:47:26 INFO DAGScheduler:59 - Job 0 finished: zipWithIndex at LDAFromWebSuite.scala:25, took 0.347571 s -2016-06-13 19:47:26 INFO SparkContext:59 - Starting job: take at LDAOptimizer.scala:142 -2016-06-13 19:47:26 INFO DAGScheduler:59 - Got job 1 (take at LDAOptimizer.scala:142) with 1 output partitions -2016-06-13 19:47:26 INFO DAGScheduler:59 - Final stage: ResultStage 1(take at LDAOptimizer.scala:142) -2016-06-13 19:47:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:47:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:47:26 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at map at LDAFromWebSuite.scala:25), which has no missing parents -2016-06-13 19:47:26 INFO MemoryStore:59 - ensureFreeSpace(3696) called with curMem=176759, maxMem=766075207 -2016-06-13 19:47:26 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-06-13 19:47:26 INFO MemoryStore:59 - ensureFreeSpace(2071) called with curMem=180455, maxMem=766075207 -2016-06-13 19:47:26 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 19:47:26 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:50092 (size: 2.0 KB, free: 730.6 MB) -2016-06-13 19:47:26 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at map at LDAFromWebSuite.scala:25) -2016-06-13 19:47:26 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 19:47:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2307 bytes) -2016-06-13 19:47:26 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 19:47:26 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-06-13 19:47:26 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:0+132 -2016-06-13 19:47:26 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=182526, maxMem=766075207 -2016-06-13 19:47:26 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 1224.0 B, free 730.4 MB) -2016-06-13 19:47:26 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:50092 (size: 1224.0 B, free: 730.6 MB) -2016-06-13 19:47:26 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2949 bytes result sent to driver -2016-06-13 19:47:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 48 ms on localhost (1/1) -2016-06-13 19:47:26 INFO DAGScheduler:59 - ResultStage 1 (take at LDAOptimizer.scala:142) finished in 0.048 s -2016-06-13 19:47:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 19:47:26 INFO DAGScheduler:59 - Job 1 finished: take at LDAOptimizer.scala:142, took 0.077027 s -2016-06-13 19:47:27 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:27 INFO DAGScheduler:59 - Registering RDD 19 (map at GraphImpl.scala:104) -2016-06-13 19:47:27 INFO DAGScheduler:59 - Registering RDD 23 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:47:27 INFO DAGScheduler:59 - Registering RDD 11 (mapPartitions at VertexRDD.scala:358) -2016-06-13 19:47:27 INFO DAGScheduler:59 - Registering RDD 6 (mapPartitionsWithIndex at LDAOptimizer.scala:128) -2016-06-13 19:47:27 INFO DAGScheduler:59 - Got job 2 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:27 INFO DAGScheduler:59 - Final stage: ResultStage 6(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 5, ShuffleMapStage 3, ShuffleMapStage 4) -2016-06-13 19:47:27 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 5, ShuffleMapStage 3, ShuffleMapStage 4) -2016-06-13 19:47:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[19] at map at GraphImpl.scala:104), which has no missing parents -2016-06-13 19:47:27 INFO MemoryStore:59 - ensureFreeSpace(5752) called with curMem=183750, maxMem=766075207 -2016-06-13 19:47:27 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.6 KB, free 730.4 MB) -2016-06-13 19:47:27 INFO MemoryStore:59 - ensureFreeSpace(3051) called with curMem=189502, maxMem=766075207 -2016-06-13 19:47:27 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 19:47:27 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:50092 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 19:47:27 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[19] at map at GraphImpl.scala:104) -2016-06-13 19:47:27 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 19:47:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[11] at mapPartitions at VertexRDD.scala:358), which has no missing parents -2016-06-13 19:47:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:47:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:47:27 INFO MemoryStore:59 - ensureFreeSpace(5064) called with curMem=192553, maxMem=766075207 -2016-06-13 19:47:27 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.9 KB, free 730.4 MB) -2016-06-13 19:47:27 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 3) -2016-06-13 19:47:27 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-06-13 19:47:27 INFO MemoryStore:59 - ensureFreeSpace(2760) called with curMem=197617, maxMem=766075207 -2016-06-13 19:47:27 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:47:27 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:50092 (size: 2.7 KB, free: 730.6 MB) -2016-06-13 19:47:27 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[11] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:47:27 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 19:47:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 5 (MapPartitionsRDD[6] at mapPartitionsWithIndex at LDAOptimizer.scala:128), which has no missing parents -2016-06-13 19:47:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 4, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:47:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 5, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:47:27 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 5) -2016-06-13 19:47:27 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 4) -2016-06-13 19:47:27 INFO MemoryStore:59 - ensureFreeSpace(4880) called with curMem=200377, maxMem=766075207 -2016-06-13 19:47:27 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 4.8 KB, free 730.4 MB) -2016-06-13 19:47:27 INFO CacheManager:59 - Partition rdd_17_1 not found, computing it -2016-06-13 19:47:27 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-06-13 19:47:27 INFO MemoryStore:59 - ensureFreeSpace(2640) called with curMem=205257, maxMem=766075207 -2016-06-13 19:47:27 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-06-13 19:47:27 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:132+132 -2016-06-13 19:47:27 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:47:27 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:50092 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 19:47:27 INFO CacheManager:59 - Another thread is loading rdd_4_1, waiting for it to finish... -2016-06-13 19:47:27 INFO CacheManager:59 - Partition rdd_17_0 not found, computing it -2016-06-13 19:47:27 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 5 (MapPartitionsRDD[6] at mapPartitionsWithIndex at LDAOptimizer.scala:128) -2016-06-13 19:47:27 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 19:47:27 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:47:27 INFO MemoryStore:59 - ensureFreeSpace(880) called with curMem=207897, maxMem=766075207 -2016-06-13 19:47:27 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 880.0 B, free 730.4 MB) -2016-06-13 19:47:27 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:50092 (size: 880.0 B, free: 730.6 MB) -2016-06-13 19:47:27 INFO CacheManager:59 - Finished waiting for rdd_4_1 -2016-06-13 19:47:27 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 19:47:27 INFO MemoryStore:59 - ensureFreeSpace(3296) called with curMem=208777, maxMem=766075207 -2016-06-13 19:47:27 INFO MemoryStore:59 - Block rdd_17_1 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 19:47:27 INFO BlockManagerInfo:59 - Added rdd_17_1 in memory on localhost:50092 (size: 3.2 KB, free: 730.6 MB) -2016-06-13 19:47:27 INFO MemoryStore:59 - ensureFreeSpace(3520) called with curMem=212073, maxMem=766075207 -2016-06-13 19:47:27 INFO MemoryStore:59 - Block rdd_17_0 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-06-13 19:47:27 INFO BlockManagerInfo:59 - Added rdd_17_0 in memory on localhost:50092 (size: 3.4 KB, free: 730.6 MB) -2016-06-13 19:47:27 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 3). 2889 bytes result sent to driver -2016-06-13 19:47:27 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 2834 bytes result sent to driver -2016-06-13 19:47:27 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 5). 1159 bytes result sent to driver -2016-06-13 19:47:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 6, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:47:27 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 6) -2016-06-13 19:47:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 7, localhost, PROCESS_LOCAL, 2296 bytes) -2016-06-13 19:47:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 3) in 178 ms on localhost (1/2) -2016-06-13 19:47:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 5) in 135 ms on localhost (1/2) -2016-06-13 19:47:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 187 ms on localhost (2/2) -2016-06-13 19:47:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 19:47:27 INFO DAGScheduler:59 - ShuffleMapStage 2 (map at GraphImpl.scala:104) finished in 0.190 s -2016-06-13 19:47:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:27 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 5, ShuffleMapStage 4) -2016-06-13 19:47:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 6, ShuffleMapStage 3) -2016-06-13 19:47:27 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:27 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List(ShuffleMapStage 5, ShuffleMapStage 3, ShuffleMapStage 4) -2016-06-13 19:47:27 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:47:27 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List() -2016-06-13 19:47:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[23] at mapPartitions at VertexRDD.scala:358), which is now runnable -2016-06-13 19:47:27 INFO MemoryStore:59 - ensureFreeSpace(2784) called with curMem=215593, maxMem=766075207 -2016-06-13 19:47:27 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:47:27 INFO MemoryStore:59 - ensureFreeSpace(1698) called with curMem=218377, maxMem=766075207 -2016-06-13 19:47:27 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1698.0 B, free 730.4 MB) -2016-06-13 19:47:27 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:50092 (size: 1698.0 B, free: 730.6 MB) -2016-06-13 19:47:27 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 3 (VertexRDD.createRoutingTables - vid2pid (aggregation) MapPartitionsRDD[23] at mapPartitions at VertexRDD.scala:358) -2016-06-13 19:47:27 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 19:47:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 1890 bytes) -2016-06-13 19:47:27 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 8) -2016-06-13 19:47:27 INFO CacheManager:59 - Partition rdd_21_0 not found, computing it -2016-06-13 19:47:27 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 7) -2016-06-13 19:47:27 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 19:47:27 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 19:47:27 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 19:47:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 10 ms -2016-06-13 19:47:27 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 4). 2254 bytes result sent to driver -2016-06-13 19:47:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 9, localhost, PROCESS_LOCAL, 1890 bytes) -2016-06-13 19:47:27 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 9) -2016-06-13 19:47:27 INFO MemoryStore:59 - ensureFreeSpace(3408) called with curMem=220075, maxMem=766075207 -2016-06-13 19:47:27 INFO MemoryStore:59 - Block rdd_21_0 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 19:47:27 INFO BlockManagerInfo:59 - Added rdd_21_0 in memory on localhost:50092 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:47:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 4) in 254 ms on localhost (2/2) -2016-06-13 19:47:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 19:47:27 INFO DAGScheduler:59 - ShuffleMapStage 4 (mapPartitions at VertexRDD.scala:358) finished in 0.282 s -2016-06-13 19:47:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:27 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 5, ShuffleMapStage 3) -2016-06-13 19:47:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 6) -2016-06-13 19:47:27 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:27 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 8). 1955 bytes result sent to driver -2016-06-13 19:47:27 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List(ShuffleMapStage 5, ShuffleMapStage 3) -2016-06-13 19:47:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 8) in 106 ms on localhost (1/2) -2016-06-13 19:47:27 INFO CacheManager:59 - Partition rdd_21_1 not found, computing it -2016-06-13 19:47:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:27 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 7). 2254 bytes result sent to driver -2016-06-13 19:47:27 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 6). 2254 bytes result sent to driver -2016-06-13 19:47:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 6) in 192 ms on localhost (1/2) -2016-06-13 19:47:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 7) in 187 ms on localhost (2/2) -2016-06-13 19:47:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 19:47:27 INFO MemoryStore:59 - ensureFreeSpace(3416) called with curMem=223483, maxMem=766075207 -2016-06-13 19:47:27 INFO DAGScheduler:59 - ShuffleMapStage 5 (mapPartitionsWithIndex at LDAOptimizer.scala:128) finished in 0.284 s -2016-06-13 19:47:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:27 INFO MemoryStore:59 - Block rdd_21_1 stored as values in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 19:47:27 INFO DAGScheduler:59 - running: Set(ShuffleMapStage 3) -2016-06-13 19:47:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 6) -2016-06-13 19:47:27 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:27 INFO BlockManagerInfo:59 - Added rdd_21_1 in memory on localhost:50092 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:47:27 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 9). 1955 bytes result sent to driver -2016-06-13 19:47:27 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List(ShuffleMapStage 3) -2016-06-13 19:47:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 9) in 157 ms on localhost (2/2) -2016-06-13 19:47:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 19:47:27 INFO DAGScheduler:59 - ShuffleMapStage 3 (mapPartitions at VertexRDD.scala:358) finished in 0.196 s -2016-06-13 19:47:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:27 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 6) -2016-06-13 19:47:27 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:27 INFO DAGScheduler:59 - Missing parents for ResultStage 6: List() -2016-06-13 19:47:27 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[30] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:27 INFO MemoryStore:59 - ensureFreeSpace(5584) called with curMem=226899, maxMem=766075207 -2016-06-13 19:47:27 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 5.5 KB, free 730.4 MB) -2016-06-13 19:47:27 INFO MemoryStore:59 - ensureFreeSpace(2802) called with curMem=232483, maxMem=766075207 -2016-06-13 19:47:27 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 19:47:27 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:50092 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:47:27 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[30] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:27 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-06-13 19:47:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 10, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:47:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2125 bytes) -2016-06-13 19:47:27 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 11) -2016-06-13 19:47:27 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 10) -2016-06-13 19:47:27 INFO CacheManager:59 - Partition rdd_26_1 not found, computing it -2016-06-13 19:47:27 INFO CacheManager:59 - Partition rdd_14_1 not found, computing it -2016-06-13 19:47:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:27 INFO CacheManager:59 - Partition rdd_26_0 not found, computing it -2016-06-13 19:47:27 INFO CacheManager:59 - Partition rdd_14_0 not found, computing it -2016-06-13 19:47:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:27 INFO MemoryStore:59 - ensureFreeSpace(2400) called with curMem=235285, maxMem=766075207 -2016-06-13 19:47:27 INFO MemoryStore:59 - Block rdd_14_1 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 19:47:27 INFO BlockManagerInfo:59 - Added rdd_14_1 in memory on localhost:50092 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:47:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:27 INFO MemoryStore:59 - ensureFreeSpace(2400) called with curMem=237685, maxMem=766075207 -2016-06-13 19:47:27 INFO MemoryStore:59 - Block rdd_26_1 stored as values in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 19:47:27 INFO BlockManagerInfo:59 - Added rdd_26_1 in memory on localhost:50092 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:47:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:27 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 11). 1931 bytes result sent to driver -2016-06-13 19:47:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 11) in 99 ms on localhost (1/2) -2016-06-13 19:47:27 INFO MemoryStore:59 - ensureFreeSpace(2496) called with curMem=240085, maxMem=766075207 -2016-06-13 19:47:27 INFO MemoryStore:59 - Block rdd_14_0 stored as values in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 19:47:27 INFO BlockManagerInfo:59 - Added rdd_14_0 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:27 INFO MemoryStore:59 - ensureFreeSpace(2496) called with curMem=242581, maxMem=766075207 -2016-06-13 19:47:27 INFO MemoryStore:59 - Block rdd_26_0 stored as values in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 19:47:27 INFO BlockManagerInfo:59 - Added rdd_26_0 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:27 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 10). 1931 bytes result sent to driver -2016-06-13 19:47:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 10) in 133 ms on localhost (2/2) -2016-06-13 19:47:27 INFO DAGScheduler:59 - ResultStage 6 (fold at LDAOptimizer.scala:205) finished in 0.128 s -2016-06-13 19:47:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 19:47:27 INFO DAGScheduler:59 - Job 2 finished: fold at LDAOptimizer.scala:205, took 0.706851 s -2016-06-13 19:47:27 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:28 INFO DAGScheduler:59 - Registering RDD 31 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:28 INFO DAGScheduler:59 - Registering RDD 35 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:28 INFO DAGScheduler:59 - Got job 3 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:28 INFO DAGScheduler:59 - Final stage: ResultStage 13(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12, ShuffleMapStage 10, ShuffleMapStage 7, ShuffleMapStage 8) -2016-06-13 19:47:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-06-13 19:47:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 11 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[31] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(4360) called with curMem=245077, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(2388) called with curMem=249437, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:50092 (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 11 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[31] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:28 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 19:47:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 12, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:47:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 13, localhost, PROCESS_LOCAL, 2114 bytes) -2016-06-13 19:47:28 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 12) -2016-06-13 19:47:28 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 13) -2016-06-13 19:47:28 INFO BlockManager:59 - Found block rdd_26_0 locally -2016-06-13 19:47:28 INFO BlockManager:59 - Found block rdd_26_1 locally -2016-06-13 19:47:28 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 13). 2254 bytes result sent to driver -2016-06-13 19:47:28 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 12). 2254 bytes result sent to driver -2016-06-13 19:47:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 13) in 50 ms on localhost (1/2) -2016-06-13 19:47:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 12) in 53 ms on localhost (2/2) -2016-06-13 19:47:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 19:47:28 INFO DAGScheduler:59 - ShuffleMapStage 11 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.057 s -2016-06-13 19:47:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:28 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:28 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 12, ResultStage 13) -2016-06-13 19:47:28 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:28 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 12: List() -2016-06-13 19:47:28 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List(ShuffleMapStage 12) -2016-06-13 19:47:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[35] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=251825, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=255721, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:50092 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[35] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:28 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 19:47:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 14, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 15, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:28 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 15) -2016-06-13 19:47:28 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 14) -2016-06-13 19:47:28 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:28 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:28 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 15). 2470 bytes result sent to driver -2016-06-13 19:47:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 15) in 46 ms on localhost (1/2) -2016-06-13 19:47:28 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 14). 2470 bytes result sent to driver -2016-06-13 19:47:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 14) in 52 ms on localhost (2/2) -2016-06-13 19:47:28 INFO DAGScheduler:59 - ShuffleMapStage 12 (mapPartitions at GraphImpl.scala:235) finished in 0.053 s -2016-06-13 19:47:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 19:47:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:28 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-06-13 19:47:28 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:28 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-06-13 19:47:28 INFO DAGScheduler:59 - Submitting ResultStage 13 (MapPartitionsRDD[43] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(6432) called with curMem=258007, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 6.3 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(3040) called with curMem=264439, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:50092 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (MapPartitionsRDD[43] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:28 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 19:47:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 16, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:47:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 17, localhost, PROCESS_LOCAL, 2166 bytes) -2016-06-13 19:47:28 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 17) -2016-06-13 19:47:28 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 16) -2016-06-13 19:47:28 INFO CacheManager:59 - Partition rdd_39_1 not found, computing it -2016-06-13 19:47:28 INFO BlockManager:59 - Found block rdd_26_1 locally -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:28 INFO CacheManager:59 - Partition rdd_39_0 not found, computing it -2016-06-13 19:47:28 INFO BlockManager:59 - Found block rdd_26_0 locally -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=267479, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block rdd_39_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=269927, maxMem=766075207 -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Added rdd_39_1 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO MemoryStore:59 - Block rdd_39_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Added rdd_39_0 in memory on localhost:50092 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 17). 2971 bytes result sent to driver -2016-06-13 19:47:28 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 16). 2971 bytes result sent to driver -2016-06-13 19:47:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 17) in 25 ms on localhost (1/2) -2016-06-13 19:47:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 16) in 28 ms on localhost (2/2) -2016-06-13 19:47:28 INFO DAGScheduler:59 - ResultStage 13 (fold at LDAOptimizer.scala:205) finished in 0.029 s -2016-06-13 19:47:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 19:47:28 INFO DAGScheduler:59 - Job 3 finished: fold at LDAOptimizer.scala:205, took 0.238317 s -2016-06-13 19:47:28 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:28 INFO DAGScheduler:59 - Registering RDD 44 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:28 INFO DAGScheduler:59 - Registering RDD 48 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:28 INFO DAGScheduler:59 - Got job 4 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:28 INFO DAGScheduler:59 - Final stage: ResultStage 22(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 15, ShuffleMapStage 19, ShuffleMapStage 17, ShuffleMapStage 21, ShuffleMapStage 14) -2016-06-13 19:47:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 21) -2016-06-13 19:47:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 20 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[44] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(5448) called with curMem=272471, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 5.3 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(2727) called with curMem=277919, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:50092 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 20 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[44] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:28 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-06-13 19:47:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 18, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:47:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 19, localhost, PROCESS_LOCAL, 2155 bytes) -2016-06-13 19:47:28 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 18) -2016-06-13 19:47:28 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 19) -2016-06-13 19:47:28 INFO BlockManager:59 - Found block rdd_39_1 locally -2016-06-13 19:47:28 INFO BlockManager:59 - Found block rdd_39_0 locally -2016-06-13 19:47:28 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 19). 2254 bytes result sent to driver -2016-06-13 19:47:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 19) in 35 ms on localhost (1/2) -2016-06-13 19:47:28 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 18). 2254 bytes result sent to driver -2016-06-13 19:47:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 18) in 44 ms on localhost (2/2) -2016-06-13 19:47:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 19:47:28 INFO DAGScheduler:59 - ShuffleMapStage 20 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.044 s -2016-06-13 19:47:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:28 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:28 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 21, ResultStage 22) -2016-06-13 19:47:28 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:28 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 21: List() -2016-06-13 19:47:28 INFO DAGScheduler:59 - Missing parents for ResultStage 22: List(ShuffleMapStage 21) -2016-06-13 19:47:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 21 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[48] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=280646, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=284542, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:50092 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 21 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[48] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:28 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 19:47:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 20, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 21, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:28 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 20) -2016-06-13 19:47:28 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 21) -2016-06-13 19:47:28 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:28 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:28 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 21). 2470 bytes result sent to driver -2016-06-13 19:47:28 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 20). 2470 bytes result sent to driver -2016-06-13 19:47:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 21) in 48 ms on localhost (1/2) -2016-06-13 19:47:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 20) in 50 ms on localhost (2/2) -2016-06-13 19:47:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 19:47:28 INFO DAGScheduler:59 - ShuffleMapStage 21 (mapPartitions at GraphImpl.scala:235) finished in 0.051 s -2016-06-13 19:47:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:28 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 22) -2016-06-13 19:47:28 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:28 INFO DAGScheduler:59 - Missing parents for ResultStage 22: List() -2016-06-13 19:47:28 INFO DAGScheduler:59 - Submitting ResultStage 22 (MapPartitionsRDD[56] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(6864) called with curMem=286828, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 6.7 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(3157) called with curMem=293692, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:50092 (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 22 (MapPartitionsRDD[56] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:28 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 2 tasks -2016-06-13 19:47:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 22, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:47:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 23, localhost, PROCESS_LOCAL, 2207 bytes) -2016-06-13 19:47:28 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 22) -2016-06-13 19:47:28 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 23) -2016-06-13 19:47:28 INFO CacheManager:59 - Partition rdd_52_1 not found, computing it -2016-06-13 19:47:28 INFO CacheManager:59 - Partition rdd_52_0 not found, computing it -2016-06-13 19:47:28 INFO BlockManager:59 - Found block rdd_39_1 locally -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:28 INFO BlockManager:59 - Found block rdd_39_0 locally -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=296849, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block rdd_52_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=299297, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block rdd_52_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Added rdd_52_1 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Added rdd_52_0 in memory on localhost:50092 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 23). 2971 bytes result sent to driver -2016-06-13 19:47:28 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 22). 2971 bytes result sent to driver -2016-06-13 19:47:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 23) in 56 ms on localhost (1/2) -2016-06-13 19:47:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 22) in 59 ms on localhost (2/2) -2016-06-13 19:47:28 INFO DAGScheduler:59 - ResultStage 22 (fold at LDAOptimizer.scala:205) finished in 0.060 s -2016-06-13 19:47:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 19:47:28 INFO DAGScheduler:59 - Job 4 finished: fold at LDAOptimizer.scala:205, took 0.225719 s -2016-06-13 19:47:28 INFO ZippedPartitionsRDD2:59 - Removing RDD 26 from persistence list -2016-06-13 19:47:28 INFO ZippedPartitionsRDD2:59 - Removing RDD 33 from persistence list -2016-06-13 19:47:28 INFO BlockManager:59 - Removing RDD 26 -2016-06-13 19:47:28 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:28 INFO BlockManager:59 - Removing RDD 33 -2016-06-13 19:47:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:28 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:28 INFO DAGScheduler:59 - Registering RDD 57 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:28 INFO DAGScheduler:59 - Registering RDD 61 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:28 INFO DAGScheduler:59 - Got job 5 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:28 INFO DAGScheduler:59 - Final stage: ResultStage 33(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30, ShuffleMapStage 32, ShuffleMapStage 24, ShuffleMapStage 28, ShuffleMapStage 26, ShuffleMapStage 23) -2016-06-13 19:47:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 32) -2016-06-13 19:47:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 31 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[57] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(5872) called with curMem=296945, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 5.7 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(2848) called with curMem=302817, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2.8 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:50092 (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 31 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[57] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:28 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 19:47:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 24, localhost, PROCESS_LOCAL, 2196 bytes) -2016-06-13 19:47:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 25, localhost, PROCESS_LOCAL, 2196 bytes) -2016-06-13 19:47:28 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 25) -2016-06-13 19:47:28 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 24) -2016-06-13 19:47:28 INFO BlockManager:59 - Found block rdd_52_0 locally -2016-06-13 19:47:28 INFO BlockManager:59 - Found block rdd_52_1 locally -2016-06-13 19:47:28 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 24). 2254 bytes result sent to driver -2016-06-13 19:47:28 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 25). 2254 bytes result sent to driver -2016-06-13 19:47:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 24) in 145 ms on localhost (1/2) -2016-06-13 19:47:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 25) in 148 ms on localhost (2/2) -2016-06-13 19:47:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 19:47:28 INFO DAGScheduler:59 - ShuffleMapStage 31 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.165 s -2016-06-13 19:47:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:28 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 33, ShuffleMapStage 32) -2016-06-13 19:47:28 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:28 INFO DAGScheduler:59 - Missing parents for ResultStage 33: List(ShuffleMapStage 32) -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:50092 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 32: List() -2016-06-13 19:47:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 32 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[61] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=296862, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=300758, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:50092 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 32 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[61] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:28 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-06-13 19:47:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 26, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 27, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:28 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 27) -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:50092 in memory (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 26) -2016-06-13 19:47:28 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:28 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:28 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 19:47:28 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 26). 2470 bytes result sent to driver -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:50092 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 27). 2470 bytes result sent to driver -2016-06-13 19:47:28 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 19:47:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 27) in 46 ms on localhost (1/2) -2016-06-13 19:47:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 26) in 49 ms on localhost (2/2) -2016-06-13 19:47:28 INFO DAGScheduler:59 - ShuffleMapStage 32 (mapPartitions at GraphImpl.scala:235) finished in 0.050 s -2016-06-13 19:47:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 19:47:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:28 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 33) -2016-06-13 19:47:28 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:50092 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 19:47:28 INFO DAGScheduler:59 - Missing parents for ResultStage 33: List() -2016-06-13 19:47:28 INFO DAGScheduler:59 - Submitting ResultStage 33 (MapPartitionsRDD[69] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:50092 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(7288) called with curMem=263012, maxMem=766075207 -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:50092 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 7.1 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(3298) called with curMem=265940, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:50092 in memory (size: 2.3 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:50092 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (MapPartitionsRDD[69] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:28 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-06-13 19:47:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 28, localhost, PROCESS_LOCAL, 2248 bytes) -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:50092 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 29, localhost, PROCESS_LOCAL, 2248 bytes) -2016-06-13 19:47:28 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 19:47:28 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 29) -2016-06-13 19:47:28 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 28) -2016-06-13 19:47:28 INFO CacheManager:59 - Partition rdd_65_1 not found, computing it -2016-06-13 19:47:28 INFO BlockManager:59 - Found block rdd_52_1 locally -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:28 INFO CacheManager:59 - Partition rdd_65_0 not found, computing it -2016-06-13 19:47:28 INFO BlockManager:59 - Found block rdd_52_0 locally -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=258464, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block rdd_65_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=259214, maxMem=766075207 -2016-06-13 19:47:28 INFO MemoryStore:59 - Block rdd_65_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Added rdd_65_1 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:50092 in memory (size: 1698.0 B, free: 730.5 MB) -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Added rdd_65_0 in memory on localhost:50092 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 29). 2971 bytes result sent to driver -2016-06-13 19:47:28 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 28). 2971 bytes result sent to driver -2016-06-13 19:47:28 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:50092 in memory (size: 2.6 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 29) in 37 ms on localhost (1/2) -2016-06-13 19:47:28 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 19:47:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 28) in 41 ms on localhost (2/2) -2016-06-13 19:47:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 19:47:28 INFO DAGScheduler:59 - ResultStage 33 (fold at LDAOptimizer.scala:205) finished in 0.041 s -2016-06-13 19:47:28 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:50092 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 19:47:28 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 19:47:28 INFO DAGScheduler:59 - Job 5 finished: fold at LDAOptimizer.scala:205, took 0.404860 s -2016-06-13 19:47:29 INFO MapPartitionsRDD:59 - Removing RDD 39 from persistence list -2016-06-13 19:47:29 INFO BlockManager:59 - Removing RDD 39 -2016-06-13 19:47:29 INFO ZippedPartitionsRDD2:59 - Removing RDD 46 from persistence list -2016-06-13 19:47:29 INFO BlockManager:59 - Removing RDD 46 -2016-06-13 19:47:29 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:29 INFO DAGScheduler:59 - Registering RDD 70 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:29 INFO DAGScheduler:59 - Registering RDD 74 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:29 INFO DAGScheduler:59 - Got job 6 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:29 INFO DAGScheduler:59 - Final stage: ResultStage 46(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:29 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 45, ShuffleMapStage 37, ShuffleMapStage 34, ShuffleMapStage 35, ShuffleMapStage 39, ShuffleMapStage 43, ShuffleMapStage 41) -2016-06-13 19:47:29 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 45) -2016-06-13 19:47:29 INFO DAGScheduler:59 - Submitting ShuffleMapStage 44 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[70] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:29 INFO MemoryStore:59 - ensureFreeSpace(6304) called with curMem=238638, maxMem=766075207 -2016-06-13 19:47:29 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 6.2 KB, free 730.4 MB) -2016-06-13 19:47:29 INFO MemoryStore:59 - ensureFreeSpace(2991) called with curMem=244942, maxMem=766075207 -2016-06-13 19:47:29 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.3 MB) -2016-06-13 19:47:29 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:50092 (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:47:29 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 44 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[70] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:29 INFO TaskSchedulerImpl:59 - Adding task set 44.0 with 2 tasks -2016-06-13 19:47:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 44.0 (TID 30, localhost, PROCESS_LOCAL, 2237 bytes) -2016-06-13 19:47:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 44.0 (TID 31, localhost, PROCESS_LOCAL, 2237 bytes) -2016-06-13 19:47:29 INFO Executor:59 - Running task 1.0 in stage 44.0 (TID 31) -2016-06-13 19:47:29 INFO Executor:59 - Running task 0.0 in stage 44.0 (TID 30) -2016-06-13 19:47:29 INFO BlockManager:59 - Found block rdd_65_1 locally -2016-06-13 19:47:29 INFO BlockManager:59 - Found block rdd_65_0 locally -2016-06-13 19:47:29 INFO Executor:59 - Finished task 1.0 in stage 44.0 (TID 31). 2254 bytes result sent to driver -2016-06-13 19:47:29 INFO Executor:59 - Finished task 0.0 in stage 44.0 (TID 30). 2254 bytes result sent to driver -2016-06-13 19:47:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 44.0 (TID 31) in 37 ms on localhost (1/2) -2016-06-13 19:47:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 44.0 (TID 30) in 41 ms on localhost (2/2) -2016-06-13 19:47:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 44.0, whose tasks have all completed, from pool -2016-06-13 19:47:29 INFO DAGScheduler:59 - ShuffleMapStage 44 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.048 s -2016-06-13 19:47:29 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:29 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:29 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 45, ResultStage 46) -2016-06-13 19:47:29 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:29 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 45: List() -2016-06-13 19:47:29 INFO DAGScheduler:59 - Missing parents for ResultStage 46: List(ShuffleMapStage 45) -2016-06-13 19:47:29 INFO DAGScheduler:59 - Submitting ShuffleMapStage 45 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[74] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:29 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=247933, maxMem=766075207 -2016-06-13 19:47:29 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:29 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=251829, maxMem=766075207 -2016-06-13 19:47:29 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:47:29 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:50092 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:29 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 45 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[74] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:29 INFO TaskSchedulerImpl:59 - Adding task set 45.0 with 2 tasks -2016-06-13 19:47:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 45.0 (TID 32, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 45.0 (TID 33, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:29 INFO Executor:59 - Running task 1.0 in stage 45.0 (TID 33) -2016-06-13 19:47:29 INFO Executor:59 - Running task 0.0 in stage 45.0 (TID 32) -2016-06-13 19:47:29 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:29 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:29 INFO Executor:59 - Finished task 0.0 in stage 45.0 (TID 32). 2470 bytes result sent to driver -2016-06-13 19:47:29 INFO Executor:59 - Finished task 1.0 in stage 45.0 (TID 33). 2470 bytes result sent to driver -2016-06-13 19:47:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 45.0 (TID 32) in 115 ms on localhost (1/2) -2016-06-13 19:47:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 45.0 (TID 33) in 122 ms on localhost (2/2) -2016-06-13 19:47:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 45.0, whose tasks have all completed, from pool -2016-06-13 19:47:29 INFO DAGScheduler:59 - ShuffleMapStage 45 (mapPartitions at GraphImpl.scala:235) finished in 0.126 s -2016-06-13 19:47:29 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:29 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:29 INFO DAGScheduler:59 - waiting: Set(ResultStage 46) -2016-06-13 19:47:29 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:29 INFO DAGScheduler:59 - Missing parents for ResultStage 46: List() -2016-06-13 19:47:29 INFO DAGScheduler:59 - Submitting ResultStage 46 (MapPartitionsRDD[82] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:29 INFO MemoryStore:59 - ensureFreeSpace(7720) called with curMem=254115, maxMem=766075207 -2016-06-13 19:47:29 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 7.5 KB, free 730.3 MB) -2016-06-13 19:47:29 INFO MemoryStore:59 - ensureFreeSpace(3400) called with curMem=261835, maxMem=766075207 -2016-06-13 19:47:29 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.3 MB) -2016-06-13 19:47:29 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:50092 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:47:29 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 46 (MapPartitionsRDD[82] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:29 INFO TaskSchedulerImpl:59 - Adding task set 46.0 with 2 tasks -2016-06-13 19:47:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 46.0 (TID 34, localhost, PROCESS_LOCAL, 2289 bytes) -2016-06-13 19:47:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 46.0 (TID 35, localhost, PROCESS_LOCAL, 2289 bytes) -2016-06-13 19:47:29 INFO Executor:59 - Running task 1.0 in stage 46.0 (TID 35) -2016-06-13 19:47:29 INFO Executor:59 - Running task 0.0 in stage 46.0 (TID 34) -2016-06-13 19:47:29 INFO CacheManager:59 - Partition rdd_78_1 not found, computing it -2016-06-13 19:47:29 INFO BlockManager:59 - Found block rdd_65_1 locally -2016-06-13 19:47:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:29 INFO CacheManager:59 - Partition rdd_78_0 not found, computing it -2016-06-13 19:47:29 INFO BlockManager:59 - Found block rdd_65_0 locally -2016-06-13 19:47:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:29 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=265235, maxMem=766075207 -2016-06-13 19:47:29 INFO MemoryStore:59 - Block rdd_78_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:47:29 INFO BlockManagerInfo:59 - Added rdd_78_0 in memory on localhost:50092 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:29 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=267779, maxMem=766075207 -2016-06-13 19:47:29 INFO MemoryStore:59 - Block rdd_78_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:47:29 INFO Executor:59 - Finished task 0.0 in stage 46.0 (TID 34). 2971 bytes result sent to driver -2016-06-13 19:47:29 INFO BlockManagerInfo:59 - Added rdd_78_1 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:29 INFO Executor:59 - Finished task 1.0 in stage 46.0 (TID 35). 2971 bytes result sent to driver -2016-06-13 19:47:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 46.0 (TID 34) in 52 ms on localhost (1/2) -2016-06-13 19:47:29 INFO DAGScheduler:59 - ResultStage 46 (fold at LDAOptimizer.scala:205) finished in 0.085 s -2016-06-13 19:47:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 46.0 (TID 35) in 76 ms on localhost (2/2) -2016-06-13 19:47:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 46.0, whose tasks have all completed, from pool -2016-06-13 19:47:29 INFO DAGScheduler:59 - Job 6 finished: fold at LDAOptimizer.scala:205, took 0.472948 s -2016-06-13 19:47:29 INFO MapPartitionsRDD:59 - Removing RDD 52 from persistence list -2016-06-13 19:47:29 INFO ZippedPartitionsRDD2:59 - Removing RDD 59 from persistence list -2016-06-13 19:47:29 INFO BlockManager:59 - Removing RDD 52 -2016-06-13 19:47:29 INFO BlockManager:59 - Removing RDD 59 -2016-06-13 19:47:29 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:29 INFO DAGScheduler:59 - Registering RDD 83 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:29 INFO DAGScheduler:59 - Registering RDD 87 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:29 INFO DAGScheduler:59 - Got job 7 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:29 INFO DAGScheduler:59 - Final stage: ResultStage 61(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:29 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 48, ShuffleMapStage 60, ShuffleMapStage 52, ShuffleMapStage 56, ShuffleMapStage 50, ShuffleMapStage 54, ShuffleMapStage 58, ShuffleMapStage 47) -2016-06-13 19:47:29 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 60) -2016-06-13 19:47:29 INFO DAGScheduler:59 - Submitting ShuffleMapStage 59 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[83] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:29 INFO MemoryStore:59 - ensureFreeSpace(6728) called with curMem=265235, maxMem=766075207 -2016-06-13 19:47:29 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 6.6 KB, free 730.3 MB) -2016-06-13 19:47:29 INFO MemoryStore:59 - ensureFreeSpace(3078) called with curMem=271963, maxMem=766075207 -2016-06-13 19:47:29 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.3 MB) -2016-06-13 19:47:29 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:50092 (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:47:29 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 59 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[83] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:29 INFO TaskSchedulerImpl:59 - Adding task set 59.0 with 2 tasks -2016-06-13 19:47:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 59.0 (TID 36, localhost, PROCESS_LOCAL, 2278 bytes) -2016-06-13 19:47:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 59.0 (TID 37, localhost, PROCESS_LOCAL, 2278 bytes) -2016-06-13 19:47:29 INFO Executor:59 - Running task 0.0 in stage 59.0 (TID 36) -2016-06-13 19:47:29 INFO Executor:59 - Running task 1.0 in stage 59.0 (TID 37) -2016-06-13 19:47:29 INFO BlockManager:59 - Found block rdd_78_1 locally -2016-06-13 19:47:29 INFO BlockManager:59 - Found block rdd_78_0 locally -2016-06-13 19:47:29 INFO Executor:59 - Finished task 0.0 in stage 59.0 (TID 36). 2254 bytes result sent to driver -2016-06-13 19:47:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 59.0 (TID 36) in 41 ms on localhost (1/2) -2016-06-13 19:47:29 INFO Executor:59 - Finished task 1.0 in stage 59.0 (TID 37). 2254 bytes result sent to driver -2016-06-13 19:47:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 59.0 (TID 37) in 45 ms on localhost (2/2) -2016-06-13 19:47:29 INFO DAGScheduler:59 - ShuffleMapStage 59 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.047 s -2016-06-13 19:47:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 59.0, whose tasks have all completed, from pool -2016-06-13 19:47:29 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:29 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:29 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 60, ResultStage 61) -2016-06-13 19:47:29 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:29 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 60: List() -2016-06-13 19:47:29 INFO DAGScheduler:59 - Missing parents for ResultStage 61: List(ShuffleMapStage 60) -2016-06-13 19:47:29 INFO DAGScheduler:59 - Submitting ShuffleMapStage 60 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[87] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:29 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=275041, maxMem=766075207 -2016-06-13 19:47:29 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:29 INFO MemoryStore:59 - ensureFreeSpace(2280) called with curMem=278937, maxMem=766075207 -2016-06-13 19:47:29 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:47:29 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:50092 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:29 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 60 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[87] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:29 INFO TaskSchedulerImpl:59 - Adding task set 60.0 with 2 tasks -2016-06-13 19:47:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 60.0 (TID 38, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 60.0 (TID 39, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:29 INFO Executor:59 - Running task 1.0 in stage 60.0 (TID 39) -2016-06-13 19:47:29 INFO Executor:59 - Running task 0.0 in stage 60.0 (TID 38) -2016-06-13 19:47:29 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:29 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:29 INFO Executor:59 - Finished task 0.0 in stage 60.0 (TID 38). 2470 bytes result sent to driver -2016-06-13 19:47:29 INFO Executor:59 - Finished task 1.0 in stage 60.0 (TID 39). 2470 bytes result sent to driver -2016-06-13 19:47:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 60.0 (TID 38) in 45 ms on localhost (1/2) -2016-06-13 19:47:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 60.0 (TID 39) in 45 ms on localhost (2/2) -2016-06-13 19:47:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 60.0, whose tasks have all completed, from pool -2016-06-13 19:47:29 INFO DAGScheduler:59 - ShuffleMapStage 60 (mapPartitions at GraphImpl.scala:235) finished in 0.046 s -2016-06-13 19:47:29 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:29 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:29 INFO DAGScheduler:59 - waiting: Set(ResultStage 61) -2016-06-13 19:47:29 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:29 INFO DAGScheduler:59 - Missing parents for ResultStage 61: List() -2016-06-13 19:47:29 INFO DAGScheduler:59 - Submitting ResultStage 61 (MapPartitionsRDD[95] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:29 INFO MemoryStore:59 - ensureFreeSpace(8144) called with curMem=281217, maxMem=766075207 -2016-06-13 19:47:29 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 8.0 KB, free 730.3 MB) -2016-06-13 19:47:29 INFO MemoryStore:59 - ensureFreeSpace(3491) called with curMem=289361, maxMem=766075207 -2016-06-13 19:47:29 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.3 MB) -2016-06-13 19:47:29 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:50092 (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:47:29 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 61 (MapPartitionsRDD[95] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:29 INFO TaskSchedulerImpl:59 - Adding task set 61.0 with 2 tasks -2016-06-13 19:47:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 61.0 (TID 40, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:47:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 61.0 (TID 41, localhost, PROCESS_LOCAL, 2330 bytes) -2016-06-13 19:47:29 INFO Executor:59 - Running task 1.0 in stage 61.0 (TID 41) -2016-06-13 19:47:29 INFO Executor:59 - Running task 0.0 in stage 61.0 (TID 40) -2016-06-13 19:47:29 INFO CacheManager:59 - Partition rdd_91_1 not found, computing it -2016-06-13 19:47:29 INFO BlockManager:59 - Found block rdd_78_1 locally -2016-06-13 19:47:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:29 INFO CacheManager:59 - Partition rdd_91_0 not found, computing it -2016-06-13 19:47:29 INFO BlockManager:59 - Found block rdd_78_0 locally -2016-06-13 19:47:29 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:29 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:29 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=292852, maxMem=766075207 -2016-06-13 19:47:29 INFO MemoryStore:59 - Block rdd_91_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:47:29 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=295300, maxMem=766075207 -2016-06-13 19:47:29 INFO MemoryStore:59 - Block rdd_91_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:47:29 INFO BlockManagerInfo:59 - Added rdd_91_1 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:29 INFO BlockManagerInfo:59 - Added rdd_91_0 in memory on localhost:50092 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:29 INFO Executor:59 - Finished task 1.0 in stage 61.0 (TID 41). 2971 bytes result sent to driver -2016-06-13 19:47:29 INFO Executor:59 - Finished task 0.0 in stage 61.0 (TID 40). 2971 bytes result sent to driver -2016-06-13 19:47:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 61.0 (TID 41) in 35 ms on localhost (1/2) -2016-06-13 19:47:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 61.0 (TID 40) in 39 ms on localhost (2/2) -2016-06-13 19:47:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 61.0, whose tasks have all completed, from pool -2016-06-13 19:47:29 INFO DAGScheduler:59 - ResultStage 61 (fold at LDAOptimizer.scala:205) finished in 0.041 s -2016-06-13 19:47:29 INFO DAGScheduler:59 - Job 7 finished: fold at LDAOptimizer.scala:205, took 0.245546 s -2016-06-13 19:47:29 INFO MapPartitionsRDD:59 - Removing RDD 65 from persistence list -2016-06-13 19:47:29 INFO BlockManager:59 - Removing RDD 65 -2016-06-13 19:47:29 INFO ZippedPartitionsRDD2:59 - Removing RDD 72 from persistence list -2016-06-13 19:47:29 INFO BlockManager:59 - Removing RDD 72 -2016-06-13 19:47:29 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:29 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:29 INFO DAGScheduler:59 - Registering RDD 96 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:29 INFO DAGScheduler:59 - Registering RDD 100 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:29 INFO DAGScheduler:59 - Got job 8 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:29 INFO DAGScheduler:59 - Final stage: ResultStage 78(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:29 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 63, ShuffleMapStage 67, ShuffleMapStage 71, ShuffleMapStage 75, ShuffleMapStage 69, ShuffleMapStage 73, ShuffleMapStage 65, ShuffleMapStage 62, ShuffleMapStage 77) -2016-06-13 19:47:29 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 77) -2016-06-13 19:47:29 INFO DAGScheduler:59 - Submitting ShuffleMapStage 76 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[96] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:29 INFO MemoryStore:59 - ensureFreeSpace(7160) called with curMem=292852, maxMem=766075207 -2016-06-13 19:47:29 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 7.0 KB, free 730.3 MB) -2016-06-13 19:47:29 INFO MemoryStore:59 - ensureFreeSpace(3182) called with curMem=300012, maxMem=766075207 -2016-06-13 19:47:29 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.3 MB) -2016-06-13 19:47:29 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:50092 (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:47:29 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 76 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[96] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:29 INFO TaskSchedulerImpl:59 - Adding task set 76.0 with 2 tasks -2016-06-13 19:47:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 76.0 (TID 42, localhost, PROCESS_LOCAL, 2319 bytes) -2016-06-13 19:47:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 76.0 (TID 43, localhost, PROCESS_LOCAL, 2319 bytes) -2016-06-13 19:47:29 INFO Executor:59 - Running task 1.0 in stage 76.0 (TID 43) -2016-06-13 19:47:29 INFO BlockManager:59 - Found block rdd_91_1 locally -2016-06-13 19:47:29 INFO Executor:59 - Running task 0.0 in stage 76.0 (TID 42) -2016-06-13 19:47:29 INFO BlockManager:59 - Found block rdd_91_0 locally -2016-06-13 19:47:29 INFO Executor:59 - Finished task 1.0 in stage 76.0 (TID 43). 2254 bytes result sent to driver -2016-06-13 19:47:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 76.0 (TID 43) in 28 ms on localhost (1/2) -2016-06-13 19:47:29 INFO Executor:59 - Finished task 0.0 in stage 76.0 (TID 42). 2254 bytes result sent to driver -2016-06-13 19:47:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 76.0 (TID 42) in 51 ms on localhost (2/2) -2016-06-13 19:47:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 76.0, whose tasks have all completed, from pool -2016-06-13 19:47:29 INFO DAGScheduler:59 - ShuffleMapStage 76 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.049 s -2016-06-13 19:47:29 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:29 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:29 INFO DAGScheduler:59 - waiting: Set(ResultStage 78, ShuffleMapStage 77) -2016-06-13 19:47:29 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:29 INFO DAGScheduler:59 - Missing parents for ResultStage 78: List(ShuffleMapStage 77) -2016-06-13 19:47:29 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 77: List() -2016-06-13 19:47:29 INFO DAGScheduler:59 - Submitting ShuffleMapStage 77 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[100] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=303194, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(2283) called with curMem=307090, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:50092 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 77 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[100] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Adding task set 77.0 with 2 tasks -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 77.0 (TID 44, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 77.0 (TID 45, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:30 INFO Executor:59 - Running task 0.0 in stage 77.0 (TID 44) -2016-06-13 19:47:30 INFO Executor:59 - Running task 1.0 in stage 77.0 (TID 45) -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:30 INFO Executor:59 - Finished task 1.0 in stage 77.0 (TID 45). 2470 bytes result sent to driver -2016-06-13 19:47:30 INFO Executor:59 - Finished task 0.0 in stage 77.0 (TID 44). 2470 bytes result sent to driver -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 77.0 (TID 45) in 27 ms on localhost (1/2) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 77.0 (TID 44) in 29 ms on localhost (2/2) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 77.0, whose tasks have all completed, from pool -2016-06-13 19:47:30 INFO DAGScheduler:59 - ShuffleMapStage 77 (mapPartitions at GraphImpl.scala:235) finished in 0.031 s -2016-06-13 19:47:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:30 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - waiting: Set(ResultStage 78) -2016-06-13 19:47:30 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents for ResultStage 78: List() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting ResultStage 78 (MapPartitionsRDD[108] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(8576) called with curMem=309373, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 8.4 KB, free 730.3 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(3582) called with curMem=317949, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.3 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:50092 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 78 (MapPartitionsRDD[108] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Adding task set 78.0 with 2 tasks -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 78.0 (TID 46, localhost, PROCESS_LOCAL, 2371 bytes) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 78.0 (TID 47, localhost, PROCESS_LOCAL, 2371 bytes) -2016-06-13 19:47:30 INFO Executor:59 - Running task 0.0 in stage 78.0 (TID 46) -2016-06-13 19:47:30 INFO Executor:59 - Running task 1.0 in stage 78.0 (TID 47) -2016-06-13 19:47:30 INFO CacheManager:59 - Partition rdd_104_0 not found, computing it -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_91_0 locally -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:30 INFO CacheManager:59 - Partition rdd_104_1 not found, computing it -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_91_1 locally -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=321531, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block rdd_104_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=324075, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block rdd_104_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added rdd_104_0 in memory on localhost:50092 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added rdd_104_1 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO Executor:59 - Finished task 0.0 in stage 78.0 (TID 46). 2971 bytes result sent to driver -2016-06-13 19:47:30 INFO Executor:59 - Finished task 1.0 in stage 78.0 (TID 47). 2971 bytes result sent to driver -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 78.0 (TID 46) in 19 ms on localhost (1/2) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 78.0 (TID 47) in 20 ms on localhost (2/2) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 78.0, whose tasks have all completed, from pool -2016-06-13 19:47:30 INFO DAGScheduler:59 - ResultStage 78 (fold at LDAOptimizer.scala:205) finished in 0.021 s -2016-06-13 19:47:30 INFO DAGScheduler:59 - Job 8 finished: fold at LDAOptimizer.scala:205, took 0.206465 s -2016-06-13 19:47:30 INFO MapPartitionsRDD:59 - Removing RDD 78 from persistence list -2016-06-13 19:47:30 INFO ZippedPartitionsRDD2:59 - Removing RDD 85 from persistence list -2016-06-13 19:47:30 INFO BlockManager:59 - Removing RDD 78 -2016-06-13 19:47:30 INFO BlockManager:59 - Removing RDD 85 -2016-06-13 19:47:30 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:30 INFO DAGScheduler:59 - Registering RDD 109 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Registering RDD 113 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Got job 9 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:30 INFO DAGScheduler:59 - Final stage: ResultStage 97(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 84, ShuffleMapStage 96, ShuffleMapStage 88, ShuffleMapStage 82, ShuffleMapStage 86, ShuffleMapStage 90, ShuffleMapStage 79, ShuffleMapStage 94, ShuffleMapStage 80, ShuffleMapStage 92) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 96) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting ShuffleMapStage 95 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[109] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(7584) called with curMem=321531, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 7.4 KB, free 730.3 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(3283) called with curMem=329115, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 3.2 KB, free 730.3 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:50092 (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 95 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[109] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Adding task set 95.0 with 2 tasks -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 95.0 (TID 48, localhost, PROCESS_LOCAL, 2360 bytes) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 95.0 (TID 49, localhost, PROCESS_LOCAL, 2360 bytes) -2016-06-13 19:47:30 INFO Executor:59 - Running task 1.0 in stage 95.0 (TID 49) -2016-06-13 19:47:30 INFO Executor:59 - Running task 0.0 in stage 95.0 (TID 48) -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_104_0 locally -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_104_1 locally -2016-06-13 19:47:30 INFO Executor:59 - Finished task 0.0 in stage 95.0 (TID 48). 2254 bytes result sent to driver -2016-06-13 19:47:30 INFO Executor:59 - Finished task 1.0 in stage 95.0 (TID 49). 2254 bytes result sent to driver -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 95.0 (TID 48) in 23 ms on localhost (1/2) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 95.0 (TID 49) in 23 ms on localhost (2/2) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 95.0, whose tasks have all completed, from pool -2016-06-13 19:47:30 INFO DAGScheduler:59 - ShuffleMapStage 95 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.025 s -2016-06-13 19:47:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:30 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 96, ResultStage 97) -2016-06-13 19:47:30 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 96: List() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents for ResultStage 97: List(ShuffleMapStage 96) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting ShuffleMapStage 96 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[113] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=332398, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=336294, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:50092 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 96 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[113] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Adding task set 96.0 with 2 tasks -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 96.0 (TID 50, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 96.0 (TID 51, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:30 INFO Executor:59 - Running task 0.0 in stage 96.0 (TID 50) -2016-06-13 19:47:30 INFO Executor:59 - Running task 1.0 in stage 96.0 (TID 51) -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:30 INFO Executor:59 - Finished task 1.0 in stage 96.0 (TID 51). 2470 bytes result sent to driver -2016-06-13 19:47:30 INFO Executor:59 - Finished task 0.0 in stage 96.0 (TID 50). 2470 bytes result sent to driver -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 96.0 (TID 51) in 33 ms on localhost (1/2) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 96.0 (TID 50) in 35 ms on localhost (2/2) -2016-06-13 19:47:30 INFO DAGScheduler:59 - ShuffleMapStage 96 (mapPartitions at GraphImpl.scala:235) finished in 0.035 s -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 96.0, whose tasks have all completed, from pool -2016-06-13 19:47:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:30 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - waiting: Set(ResultStage 97) -2016-06-13 19:47:30 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents for ResultStage 97: List() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting ResultStage 97 (MapPartitionsRDD[121] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(9000) called with curMem=338580, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 8.8 KB, free 730.3 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(3686) called with curMem=347580, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 3.6 KB, free 730.3 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:50092 (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 97 (MapPartitionsRDD[121] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Adding task set 97.0 with 2 tasks -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 97.0 (TID 52, localhost, PROCESS_LOCAL, 2412 bytes) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 97.0 (TID 53, localhost, PROCESS_LOCAL, 2412 bytes) -2016-06-13 19:47:30 INFO Executor:59 - Running task 0.0 in stage 97.0 (TID 52) -2016-06-13 19:47:30 INFO Executor:59 - Running task 1.0 in stage 97.0 (TID 53) -2016-06-13 19:47:30 INFO CacheManager:59 - Partition rdd_117_1 not found, computing it -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_104_1 locally -2016-06-13 19:47:30 INFO CacheManager:59 - Partition rdd_117_0 not found, computing it -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_104_0 locally -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=351266, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block rdd_117_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added rdd_117_1 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO Executor:59 - Finished task 1.0 in stage 97.0 (TID 53). 2971 bytes result sent to driver -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=353714, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block rdd_117_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 97.0 (TID 53) in 17 ms on localhost (1/2) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added rdd_117_0 in memory on localhost:50092 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO Executor:59 - Finished task 0.0 in stage 97.0 (TID 52). 2971 bytes result sent to driver -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 97.0 (TID 52) in 19 ms on localhost (2/2) -2016-06-13 19:47:30 INFO DAGScheduler:59 - ResultStage 97 (fold at LDAOptimizer.scala:205) finished in 0.019 s -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 97.0, whose tasks have all completed, from pool -2016-06-13 19:47:30 INFO DAGScheduler:59 - Job 9 finished: fold at LDAOptimizer.scala:205, took 0.147467 s -2016-06-13 19:47:30 INFO MapPartitionsRDD:59 - Removing RDD 91 from persistence list -2016-06-13 19:47:30 INFO ZippedPartitionsRDD2:59 - Removing RDD 98 from persistence list -2016-06-13 19:47:30 INFO BlockManager:59 - Removing RDD 91 -2016-06-13 19:47:30 INFO BlockManager:59 - Removing RDD 98 -2016-06-13 19:47:30 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:30 INFO DAGScheduler:59 - Registering RDD 122 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Registering RDD 126 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Got job 10 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:30 INFO DAGScheduler:59 - Final stage: ResultStage 118(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 117, ShuffleMapStage 99, ShuffleMapStage 103, ShuffleMapStage 115, ShuffleMapStage 107, ShuffleMapStage 101, ShuffleMapStage 111, ShuffleMapStage 105, ShuffleMapStage 109, ShuffleMapStage 98, ShuffleMapStage 113) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 117) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting ShuffleMapStage 116 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[122] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(8016) called with curMem=351266, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 7.8 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(3371) called with curMem=359282, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:50092 (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 116 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[122] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Adding task set 116.0 with 2 tasks -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 116.0 (TID 54, localhost, PROCESS_LOCAL, 2401 bytes) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 116.0 (TID 55, localhost, PROCESS_LOCAL, 2401 bytes) -2016-06-13 19:47:30 INFO Executor:59 - Running task 1.0 in stage 116.0 (TID 55) -2016-06-13 19:47:30 INFO Executor:59 - Running task 0.0 in stage 116.0 (TID 54) -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_117_0 locally -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_117_1 locally -2016-06-13 19:47:30 INFO Executor:59 - Finished task 0.0 in stage 116.0 (TID 54). 2254 bytes result sent to driver -2016-06-13 19:47:30 INFO Executor:59 - Finished task 1.0 in stage 116.0 (TID 55). 2254 bytes result sent to driver -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 116.0 (TID 54) in 22 ms on localhost (1/2) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 116.0 (TID 55) in 22 ms on localhost (2/2) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 116.0, whose tasks have all completed, from pool -2016-06-13 19:47:30 INFO DAGScheduler:59 - ShuffleMapStage 116 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.024 s -2016-06-13 19:47:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:30 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 117, ResultStage 118) -2016-06-13 19:47:30 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 117: List() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents for ResultStage 118: List(ShuffleMapStage 117) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting ShuffleMapStage 117 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[126] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=362653, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=366549, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:50092 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 117 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[126] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Adding task set 117.0 with 2 tasks -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 117.0 (TID 56, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 117.0 (TID 57, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:30 INFO Executor:59 - Running task 0.0 in stage 117.0 (TID 56) -2016-06-13 19:47:30 INFO Executor:59 - Running task 1.0 in stage 117.0 (TID 57) -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:30 INFO Executor:59 - Finished task 0.0 in stage 117.0 (TID 56). 2470 bytes result sent to driver -2016-06-13 19:47:30 INFO Executor:59 - Finished task 1.0 in stage 117.0 (TID 57). 2470 bytes result sent to driver -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 117.0 (TID 56) in 27 ms on localhost (1/2) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 117.0 (TID 57) in 29 ms on localhost (2/2) -2016-06-13 19:47:30 INFO DAGScheduler:59 - ShuffleMapStage 117 (mapPartitions at GraphImpl.scala:235) finished in 0.029 s -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 117.0, whose tasks have all completed, from pool -2016-06-13 19:47:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:30 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - waiting: Set(ResultStage 118) -2016-06-13 19:47:30 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents for ResultStage 118: List() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting ResultStage 118 (MapPartitionsRDD[134] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(9432) called with curMem=368835, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 9.2 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(3778) called with curMem=378267, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 3.7 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:50092 (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 118 (MapPartitionsRDD[134] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Adding task set 118.0 with 2 tasks -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 118.0 (TID 58, localhost, PROCESS_LOCAL, 2453 bytes) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 118.0 (TID 59, localhost, PROCESS_LOCAL, 2453 bytes) -2016-06-13 19:47:30 INFO Executor:59 - Running task 1.0 in stage 118.0 (TID 59) -2016-06-13 19:47:30 INFO Executor:59 - Running task 0.0 in stage 118.0 (TID 58) -2016-06-13 19:47:30 INFO CacheManager:59 - Partition rdd_130_1 not found, computing it -2016-06-13 19:47:30 INFO CacheManager:59 - Partition rdd_130_0 not found, computing it -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_117_1 locally -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_117_0 locally -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=382045, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block rdd_130_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=384589, maxMem=766075207 -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added rdd_130_0 in memory on localhost:50092 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - Block rdd_130_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added rdd_130_1 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO Executor:59 - Finished task 0.0 in stage 118.0 (TID 58). 2971 bytes result sent to driver -2016-06-13 19:47:30 INFO Executor:59 - Finished task 1.0 in stage 118.0 (TID 59). 2971 bytes result sent to driver -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 118.0 (TID 58) in 17 ms on localhost (1/2) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 118.0 (TID 59) in 16 ms on localhost (2/2) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 118.0, whose tasks have all completed, from pool -2016-06-13 19:47:30 INFO DAGScheduler:59 - ResultStage 118 (fold at LDAOptimizer.scala:205) finished in 0.017 s -2016-06-13 19:47:30 INFO DAGScheduler:59 - Job 10 finished: fold at LDAOptimizer.scala:205, took 0.139392 s -2016-06-13 19:47:30 INFO MapPartitionsRDD:59 - Removing RDD 104 from persistence list -2016-06-13 19:47:30 INFO ZippedPartitionsRDD2:59 - Removing RDD 111 from persistence list -2016-06-13 19:47:30 INFO BlockManager:59 - Removing RDD 104 -2016-06-13 19:47:30 INFO BlockManager:59 - Removing RDD 111 -2016-06-13 19:47:30 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:30 INFO DAGScheduler:59 - Registering RDD 135 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Registering RDD 139 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Got job 11 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:30 INFO DAGScheduler:59 - Final stage: ResultStage 141(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 132, ShuffleMapStage 136, ShuffleMapStage 140, ShuffleMapStage 122, ShuffleMapStage 119, ShuffleMapStage 126, ShuffleMapStage 138, ShuffleMapStage 120, ShuffleMapStage 130, ShuffleMapStage 124, ShuffleMapStage 134, ShuffleMapStage 128) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 140) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting ShuffleMapStage 139 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[135] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(8440) called with curMem=382045, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 8.2 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(3476) called with curMem=390485, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 3.4 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:50092 (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 139 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[135] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Adding task set 139.0 with 2 tasks -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 139.0 (TID 60, localhost, PROCESS_LOCAL, 2442 bytes) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 139.0 (TID 61, localhost, PROCESS_LOCAL, 2442 bytes) -2016-06-13 19:47:30 INFO Executor:59 - Running task 0.0 in stage 139.0 (TID 60) -2016-06-13 19:47:30 INFO Executor:59 - Running task 1.0 in stage 139.0 (TID 61) -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_130_0 locally -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_130_1 locally -2016-06-13 19:47:30 INFO Executor:59 - Finished task 0.0 in stage 139.0 (TID 60). 2254 bytes result sent to driver -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 139.0 (TID 60) in 24 ms on localhost (1/2) -2016-06-13 19:47:30 INFO Executor:59 - Finished task 1.0 in stage 139.0 (TID 61). 2254 bytes result sent to driver -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 139.0 (TID 61) in 26 ms on localhost (2/2) -2016-06-13 19:47:30 INFO DAGScheduler:59 - ShuffleMapStage 139 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.028 s -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 139.0, whose tasks have all completed, from pool -2016-06-13 19:47:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:30 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 140, ResultStage 141) -2016-06-13 19:47:30 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 140: List() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents for ResultStage 141: List(ShuffleMapStage 140) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting ShuffleMapStage 140 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[139] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=393961, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(2281) called with curMem=397857, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:50092 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 140 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[139] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Adding task set 140.0 with 2 tasks -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 140.0 (TID 62, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 140.0 (TID 63, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:30 INFO Executor:59 - Running task 0.0 in stage 140.0 (TID 62) -2016-06-13 19:47:30 INFO Executor:59 - Running task 1.0 in stage 140.0 (TID 63) -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:30 INFO Executor:59 - Finished task 1.0 in stage 140.0 (TID 63). 2470 bytes result sent to driver -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 140.0 (TID 63) in 27 ms on localhost (1/2) -2016-06-13 19:47:30 INFO Executor:59 - Finished task 0.0 in stage 140.0 (TID 62). 2470 bytes result sent to driver -2016-06-13 19:47:30 INFO DAGScheduler:59 - ShuffleMapStage 140 (mapPartitions at GraphImpl.scala:235) finished in 0.032 s -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 140.0 (TID 62) in 31 ms on localhost (2/2) -2016-06-13 19:47:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:30 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 140.0, whose tasks have all completed, from pool -2016-06-13 19:47:30 INFO DAGScheduler:59 - waiting: Set(ResultStage 141) -2016-06-13 19:47:30 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents for ResultStage 141: List() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting ResultStage 141 (MapPartitionsRDD[147] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(9856) called with curMem=400138, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 9.6 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(3886) called with curMem=409994, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:50092 (size: 3.8 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 141 (MapPartitionsRDD[147] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Adding task set 141.0 with 2 tasks -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 141.0 (TID 64, localhost, PROCESS_LOCAL, 2494 bytes) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 141.0 (TID 65, localhost, PROCESS_LOCAL, 2494 bytes) -2016-06-13 19:47:30 INFO Executor:59 - Running task 1.0 in stage 141.0 (TID 65) -2016-06-13 19:47:30 INFO Executor:59 - Running task 0.0 in stage 141.0 (TID 64) -2016-06-13 19:47:30 INFO CacheManager:59 - Partition rdd_143_1 not found, computing it -2016-06-13 19:47:30 INFO CacheManager:59 - Partition rdd_143_0 not found, computing it -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_130_1 locally -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_130_0 locally -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=413880, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block rdd_143_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=416328, maxMem=766075207 -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added rdd_143_1 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - Block rdd_143_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO Executor:59 - Finished task 1.0 in stage 141.0 (TID 65). 2971 bytes result sent to driver -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added rdd_143_0 in memory on localhost:50092 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO Executor:59 - Finished task 0.0 in stage 141.0 (TID 64). 2971 bytes result sent to driver -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 141.0 (TID 65) in 11 ms on localhost (1/2) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 141.0 (TID 64) in 13 ms on localhost (2/2) -2016-06-13 19:47:30 INFO DAGScheduler:59 - ResultStage 141 (fold at LDAOptimizer.scala:205) finished in 0.014 s -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 141.0, whose tasks have all completed, from pool -2016-06-13 19:47:30 INFO DAGScheduler:59 - Job 11 finished: fold at LDAOptimizer.scala:205, took 0.148662 s -2016-06-13 19:47:30 INFO MapPartitionsRDD:59 - Removing RDD 117 from persistence list -2016-06-13 19:47:30 INFO ZippedPartitionsRDD2:59 - Removing RDD 124 from persistence list -2016-06-13 19:47:30 INFO BlockManager:59 - Removing RDD 117 -2016-06-13 19:47:30 INFO BlockManager:59 - Removing RDD 124 -2016-06-13 19:47:30 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:30 INFO DAGScheduler:59 - Registering RDD 148 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Registering RDD 152 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Got job 12 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:30 INFO DAGScheduler:59 - Final stage: ResultStage 166(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 161, ShuffleMapStage 143, ShuffleMapStage 153, ShuffleMapStage 165, ShuffleMapStage 147, ShuffleMapStage 157, ShuffleMapStage 151, ShuffleMapStage 155, ShuffleMapStage 159, ShuffleMapStage 163, ShuffleMapStage 145, ShuffleMapStage 142, ShuffleMapStage 149) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 165) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting ShuffleMapStage 164 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[148] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(8872) called with curMem=413880, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 8.7 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(3568) called with curMem=422752, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 3.5 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:50092 (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 164 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[148] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Adding task set 164.0 with 2 tasks -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 164.0 (TID 66, localhost, PROCESS_LOCAL, 2483 bytes) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 164.0 (TID 67, localhost, PROCESS_LOCAL, 2483 bytes) -2016-06-13 19:47:30 INFO Executor:59 - Running task 1.0 in stage 164.0 (TID 67) -2016-06-13 19:47:30 INFO Executor:59 - Running task 0.0 in stage 164.0 (TID 66) -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_143_1 locally -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_143_0 locally -2016-06-13 19:47:30 INFO Executor:59 - Finished task 1.0 in stage 164.0 (TID 67). 2254 bytes result sent to driver -2016-06-13 19:47:30 INFO Executor:59 - Finished task 0.0 in stage 164.0 (TID 66). 2254 bytes result sent to driver -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 164.0 (TID 67) in 24 ms on localhost (1/2) -2016-06-13 19:47:30 INFO DAGScheduler:59 - ShuffleMapStage 164 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.029 s -2016-06-13 19:47:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 164.0 (TID 66) in 27 ms on localhost (2/2) -2016-06-13 19:47:30 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 165, ResultStage 166) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 164.0, whose tasks have all completed, from pool -2016-06-13 19:47:30 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 165: List() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents for ResultStage 166: List(ShuffleMapStage 165) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting ShuffleMapStage 165 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[152] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=426320, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=430216, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:50092 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 165 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[152] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Adding task set 165.0 with 2 tasks -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 165.0 (TID 68, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 165.0 (TID 69, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:30 INFO Executor:59 - Running task 0.0 in stage 165.0 (TID 68) -2016-06-13 19:47:30 INFO Executor:59 - Running task 1.0 in stage 165.0 (TID 69) -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:30 INFO Executor:59 - Finished task 1.0 in stage 165.0 (TID 69). 2470 bytes result sent to driver -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 165.0 (TID 69) in 27 ms on localhost (1/2) -2016-06-13 19:47:30 INFO Executor:59 - Finished task 0.0 in stage 165.0 (TID 68). 2470 bytes result sent to driver -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 165.0 (TID 68) in 30 ms on localhost (2/2) -2016-06-13 19:47:30 INFO DAGScheduler:59 - ShuffleMapStage 165 (mapPartitions at GraphImpl.scala:235) finished in 0.030 s -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 165.0, whose tasks have all completed, from pool -2016-06-13 19:47:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:30 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - waiting: Set(ResultStage 166) -2016-06-13 19:47:30 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents for ResultStage 166: List() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting ResultStage 166 (MapPartitionsRDD[160] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(10288) called with curMem=432502, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 10.0 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(4004) called with curMem=442790, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 3.9 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:50092 (size: 3.9 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 166 (MapPartitionsRDD[160] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Adding task set 166.0 with 2 tasks -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 166.0 (TID 70, localhost, PROCESS_LOCAL, 2535 bytes) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 166.0 (TID 71, localhost, PROCESS_LOCAL, 2535 bytes) -2016-06-13 19:47:30 INFO Executor:59 - Running task 1.0 in stage 166.0 (TID 71) -2016-06-13 19:47:30 INFO Executor:59 - Running task 0.0 in stage 166.0 (TID 70) -2016-06-13 19:47:30 INFO CacheManager:59 - Partition rdd_156_1 not found, computing it -2016-06-13 19:47:30 INFO CacheManager:59 - Partition rdd_156_0 not found, computing it -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_143_1 locally -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_143_0 locally -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=446794, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block rdd_156_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=449338, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block rdd_156_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added rdd_156_0 in memory on localhost:50092 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO Executor:59 - Finished task 0.0 in stage 166.0 (TID 70). 2971 bytes result sent to driver -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added rdd_156_1 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO Executor:59 - Finished task 1.0 in stage 166.0 (TID 71). 2971 bytes result sent to driver -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 166.0 (TID 70) in 15 ms on localhost (1/2) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 166.0 (TID 71) in 15 ms on localhost (2/2) -2016-06-13 19:47:30 INFO DAGScheduler:59 - ResultStage 166 (fold at LDAOptimizer.scala:205) finished in 0.016 s -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 166.0, whose tasks have all completed, from pool -2016-06-13 19:47:30 INFO DAGScheduler:59 - Job 12 finished: fold at LDAOptimizer.scala:205, took 0.150125 s -2016-06-13 19:47:30 INFO MapPartitionsRDD:59 - Removing RDD 130 from persistence list -2016-06-13 19:47:30 INFO BlockManager:59 - Removing RDD 130 -2016-06-13 19:47:30 INFO ZippedPartitionsRDD2:59 - Removing RDD 137 from persistence list -2016-06-13 19:47:30 INFO BlockManager:59 - Removing RDD 137 -2016-06-13 19:47:30 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:30 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:30 INFO DAGScheduler:59 - Registering RDD 161 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Registering RDD 165 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Got job 13 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:30 INFO DAGScheduler:59 - Final stage: ResultStage 193(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 168, ShuffleMapStage 190, ShuffleMapStage 184, ShuffleMapStage 176, ShuffleMapStage 170, ShuffleMapStage 192, ShuffleMapStage 178, ShuffleMapStage 186, ShuffleMapStage 180, ShuffleMapStage 172, ShuffleMapStage 188, ShuffleMapStage 174, ShuffleMapStage 167, ShuffleMapStage 182) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 192) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting ShuffleMapStage 191 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[161] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(9296) called with curMem=446794, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 9.1 KB, free 730.2 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(3674) called with curMem=456090, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 3.6 KB, free 730.1 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:50092 (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 191 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[161] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Adding task set 191.0 with 2 tasks -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 191.0 (TID 72, localhost, PROCESS_LOCAL, 2524 bytes) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 191.0 (TID 73, localhost, PROCESS_LOCAL, 2524 bytes) -2016-06-13 19:47:30 INFO Executor:59 - Running task 0.0 in stage 191.0 (TID 72) -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_156_0 locally -2016-06-13 19:47:30 INFO Executor:59 - Running task 1.0 in stage 191.0 (TID 73) -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_156_1 locally -2016-06-13 19:47:30 INFO Executor:59 - Finished task 0.0 in stage 191.0 (TID 72). 2254 bytes result sent to driver -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 191.0 (TID 72) in 27 ms on localhost (1/2) -2016-06-13 19:47:30 INFO Executor:59 - Finished task 1.0 in stage 191.0 (TID 73). 2254 bytes result sent to driver -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 191.0 (TID 73) in 38 ms on localhost (2/2) -2016-06-13 19:47:30 INFO DAGScheduler:59 - ShuffleMapStage 191 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.040 s -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 191.0, whose tasks have all completed, from pool -2016-06-13 19:47:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:30 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 192, ResultStage 193) -2016-06-13 19:47:30 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 192: List() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents for ResultStage 193: List(ShuffleMapStage 192) -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting ShuffleMapStage 192 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[165] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=459764, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 3.8 KB, free 730.1 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=463660, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:50092 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 192 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[165] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Adding task set 192.0 with 2 tasks -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 192.0 (TID 74, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 192.0 (TID 75, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:30 INFO Executor:59 - Running task 0.0 in stage 192.0 (TID 74) -2016-06-13 19:47:30 INFO Executor:59 - Running task 1.0 in stage 192.0 (TID 75) -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:30 INFO Executor:59 - Finished task 1.0 in stage 192.0 (TID 75). 2470 bytes result sent to driver -2016-06-13 19:47:30 INFO Executor:59 - Finished task 0.0 in stage 192.0 (TID 74). 2470 bytes result sent to driver -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 192.0 (TID 75) in 26 ms on localhost (1/2) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 192.0 (TID 74) in 28 ms on localhost (2/2) -2016-06-13 19:47:30 INFO DAGScheduler:59 - ShuffleMapStage 192 (mapPartitions at GraphImpl.scala:235) finished in 0.028 s -2016-06-13 19:47:30 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 192.0, whose tasks have all completed, from pool -2016-06-13 19:47:30 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - waiting: Set(ResultStage 193) -2016-06-13 19:47:30 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Missing parents for ResultStage 193: List() -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting ResultStage 193 (MapPartitionsRDD[173] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(10712) called with curMem=465946, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_40 stored as values in memory (estimated size 10.5 KB, free 730.1 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(4096) called with curMem=476658, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block broadcast_40_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.1 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added broadcast_40_piece0 in memory on localhost:50092 (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO SparkContext:59 - Created broadcast 40 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 193 (MapPartitionsRDD[173] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Adding task set 193.0 with 2 tasks -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 193.0 (TID 76, localhost, PROCESS_LOCAL, 2576 bytes) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 193.0 (TID 77, localhost, PROCESS_LOCAL, 2576 bytes) -2016-06-13 19:47:30 INFO Executor:59 - Running task 1.0 in stage 193.0 (TID 77) -2016-06-13 19:47:30 INFO Executor:59 - Running task 0.0 in stage 193.0 (TID 76) -2016-06-13 19:47:30 INFO CacheManager:59 - Partition rdd_169_0 not found, computing it -2016-06-13 19:47:30 INFO CacheManager:59 - Partition rdd_169_1 not found, computing it -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_156_0 locally -2016-06-13 19:47:30 INFO BlockManager:59 - Found block rdd_156_1 locally -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:30 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=480754, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block rdd_169_1 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-06-13 19:47:30 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=483202, maxMem=766075207 -2016-06-13 19:47:30 INFO MemoryStore:59 - Block rdd_169_0 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added rdd_169_1 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO BlockManagerInfo:59 - Added rdd_169_0 in memory on localhost:50092 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:30 INFO Executor:59 - Finished task 1.0 in stage 193.0 (TID 77). 2971 bytes result sent to driver -2016-06-13 19:47:30 INFO Executor:59 - Finished task 0.0 in stage 193.0 (TID 76). 2971 bytes result sent to driver -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 1.0 in stage 193.0 (TID 77) in 13 ms on localhost (1/2) -2016-06-13 19:47:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 193.0 (TID 76) in 14 ms on localhost (2/2) -2016-06-13 19:47:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 193.0, whose tasks have all completed, from pool -2016-06-13 19:47:30 INFO DAGScheduler:59 - ResultStage 193 (fold at LDAOptimizer.scala:205) finished in 0.015 s -2016-06-13 19:47:30 INFO DAGScheduler:59 - Job 13 finished: fold at LDAOptimizer.scala:205, took 0.150818 s -2016-06-13 19:47:31 INFO MapPartitionsRDD:59 - Removing RDD 143 from persistence list -2016-06-13 19:47:31 INFO BlockManager:59 - Removing RDD 143 -2016-06-13 19:47:31 INFO ZippedPartitionsRDD2:59 - Removing RDD 150 from persistence list -2016-06-13 19:47:31 INFO BlockManager:59 - Removing RDD 150 -2016-06-13 19:47:31 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:47:31 INFO DAGScheduler:59 - Registering RDD 174 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Registering RDD 178 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Got job 14 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:31 INFO DAGScheduler:59 - Final stage: ResultStage 222(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 197, ShuffleMapStage 219, ShuffleMapStage 205, ShuffleMapStage 221, ShuffleMapStage 213, ShuffleMapStage 207, ShuffleMapStage 199, ShuffleMapStage 215, ShuffleMapStage 201, ShuffleMapStage 194, ShuffleMapStage 209, ShuffleMapStage 203, ShuffleMapStage 195, ShuffleMapStage 217, ShuffleMapStage 211) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 221) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting ShuffleMapStage 220 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[174] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(9728) called with curMem=480754, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_41 stored as values in memory (estimated size 9.5 KB, free 730.1 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(3782) called with curMem=490482, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_41_piece0 stored as bytes in memory (estimated size 3.7 KB, free 730.1 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added broadcast_41_piece0 in memory on localhost:50092 (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO SparkContext:59 - Created broadcast 41 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 220 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[174] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Adding task set 220.0 with 2 tasks -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:50092 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 220.0 (TID 78, localhost, PROCESS_LOCAL, 2565 bytes) -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 220.0 (TID 79, localhost, PROCESS_LOCAL, 2565 bytes) -2016-06-13 19:47:31 INFO Executor:59 - Running task 1.0 in stage 220.0 (TID 79) -2016-06-13 19:47:31 INFO Executor:59 - Running task 0.0 in stage 220.0 (TID 78) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:50092 in memory (size: 2.9 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_169_1 locally -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_169_0 locally -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:50092 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:50092 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 19:47:31 INFO Executor:59 - Finished task 1.0 in stage 220.0 (TID 79). 2254 bytes result sent to driver -2016-06-13 19:47:31 INFO Executor:59 - Finished task 0.0 in stage 220.0 (TID 78). 2254 bytes result sent to driver -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 220.0 (TID 79) in 24 ms on localhost (1/2) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:50092 in memory (size: 2.8 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 220.0 (TID 78) in 26 ms on localhost (2/2) -2016-06-13 19:47:31 INFO DAGScheduler:59 - ShuffleMapStage 220 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.027 s -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 220.0, whose tasks have all completed, from pool -2016-06-13 19:47:31 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:31 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:31 INFO DAGScheduler:59 - waiting: Set(ResultStage 222, ShuffleMapStage 221) -2016-06-13 19:47:31 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents for ResultStage 222: List(ShuffleMapStage 221) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 221: List() -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting ShuffleMapStage 221 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[178] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_40_piece0 on localhost:50092 in memory (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 40 -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=438491, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_42 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(2290) called with curMem=440101, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_42_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_39_piece0 on localhost:50092 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added broadcast_42_piece0 in memory on localhost:50092 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 39 -2016-06-13 19:47:31 INFO SparkContext:59 - Created broadcast 42 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 221 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[178] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Adding task set 221.0 with 2 tasks -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 221.0 (TID 80, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_38_piece0 on localhost:50092 in memory (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 221.0 (TID 81, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:31 INFO Executor:59 - Running task 1.0 in stage 221.0 (TID 81) -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 38 -2016-06-13 19:47:31 INFO Executor:59 - Running task 0.0 in stage 221.0 (TID 80) -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_37_piece0 on localhost:50092 in memory (size: 3.9 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 37 -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_36_piece0 on localhost:50092 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 36 -2016-06-13 19:47:31 INFO Executor:59 - Finished task 0.0 in stage 221.0 (TID 80). 2470 bytes result sent to driver -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_35_piece0 on localhost:50092 in memory (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 221.0 (TID 80) in 29 ms on localhost (1/2) -2016-06-13 19:47:31 INFO Executor:59 - Finished task 1.0 in stage 221.0 (TID 81). 2470 bytes result sent to driver -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 35 -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 221.0 (TID 81) in 30 ms on localhost (2/2) -2016-06-13 19:47:31 INFO DAGScheduler:59 - ShuffleMapStage 221 (mapPartitions at GraphImpl.scala:235) finished in 0.031 s -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 221.0, whose tasks have all completed, from pool -2016-06-13 19:47:31 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:31 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:31 INFO DAGScheduler:59 - waiting: Set(ResultStage 222) -2016-06-13 19:47:31 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_34_piece0 on localhost:50092 in memory (size: 3.8 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents for ResultStage 222: List() -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 34 -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting ResultStage 222 (MapPartitionsRDD[186] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_33_piece0 on localhost:50092 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(11144) called with curMem=372692, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_43 stored as values in memory (estimated size 10.9 KB, free 730.2 MB) -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 33 -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(4182) called with curMem=375396, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_43_piece0 stored as bytes in memory (estimated size 4.1 KB, free 730.2 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_32_piece0 on localhost:50092 in memory (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added broadcast_43_piece0 in memory on localhost:50092 (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO SparkContext:59 - Created broadcast 43 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 222 (MapPartitionsRDD[186] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Adding task set 222.0 with 2 tasks -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 222.0 (TID 82, localhost, PROCESS_LOCAL, 2617 bytes) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 222.0 (TID 83, localhost, PROCESS_LOCAL, 2617 bytes) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:50092 in memory (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO Executor:59 - Running task 0.0 in stage 222.0 (TID 82) -2016-06-13 19:47:31 INFO Executor:59 - Running task 1.0 in stage 222.0 (TID 83) -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:50092 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO CacheManager:59 - Partition rdd_182_0 not found, computing it -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_169_0 locally -2016-06-13 19:47:31 INFO CacheManager:59 - Partition rdd_182_1 not found, computing it -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_169_1 locally -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:50092 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=345323, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block rdd_182_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=347771, maxMem=766075207 -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added rdd_182_1 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - Block rdd_182_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO Executor:59 - Finished task 1.0 in stage 222.0 (TID 83). 2971 bytes result sent to driver -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added rdd_182_0 in memory on localhost:50092 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO Executor:59 - Finished task 0.0 in stage 222.0 (TID 82). 2971 bytes result sent to driver -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 222.0 (TID 83) in 15 ms on localhost (1/2) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:50092 in memory (size: 3.6 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 222.0 (TID 82) in 16 ms on localhost (2/2) -2016-06-13 19:47:31 INFO DAGScheduler:59 - ResultStage 222 (fold at LDAOptimizer.scala:205) finished in 0.016 s -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 222.0, whose tasks have all completed, from pool -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-06-13 19:47:31 INFO DAGScheduler:59 - Job 14 finished: fold at LDAOptimizer.scala:205, took 0.161220 s -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:50092 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:50092 in memory (size: 3.2 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:50092 in memory (size: 3.5 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:50092 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:50092 in memory (size: 3.1 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:50092 in memory (size: 3.4 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO MapPartitionsRDD:59 - Removing RDD 156 from persistence list -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 19:47:31 INFO BlockManager:59 - Removing RDD 156 -2016-06-13 19:47:31 INFO ZippedPartitionsRDD2:59 - Removing RDD 163 from persistence list -2016-06-13 19:47:31 INFO BlockManager:59 - Removing RDD 163 -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:50092 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:50092 in memory (size: 3.0 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 19:47:31 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:50092 in memory (size: 3.3 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:47:31 INFO DAGScheduler:59 - Registering RDD 187 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Registering RDD 191 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Got job 15 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:31 INFO DAGScheduler:59 - Final stage: ResultStage 253(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 248, ShuffleMapStage 234, ShuffleMapStage 242, ShuffleMapStage 228, ShuffleMapStage 250, ShuffleMapStage 244, ShuffleMapStage 236, ShuffleMapStage 230, ShuffleMapStage 252, ShuffleMapStage 223, ShuffleMapStage 238, ShuffleMapStage 224, ShuffleMapStage 246, ShuffleMapStage 240, ShuffleMapStage 232, ShuffleMapStage 226) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 252) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting ShuffleMapStage 251 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[187] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(10152) called with curMem=248172, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_44 stored as values in memory (estimated size 9.9 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(3864) called with curMem=258324, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_44_piece0 stored as bytes in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added broadcast_44_piece0 in memory on localhost:50092 (size: 3.8 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO SparkContext:59 - Created broadcast 44 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 251 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[187] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Adding task set 251.0 with 2 tasks -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 251.0 (TID 84, localhost, PROCESS_LOCAL, 2606 bytes) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 251.0 (TID 85, localhost, PROCESS_LOCAL, 2606 bytes) -2016-06-13 19:47:31 INFO Executor:59 - Running task 0.0 in stage 251.0 (TID 84) -2016-06-13 19:47:31 INFO Executor:59 - Running task 1.0 in stage 251.0 (TID 85) -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_182_1 locally -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_182_0 locally -2016-06-13 19:47:31 INFO Executor:59 - Finished task 0.0 in stage 251.0 (TID 84). 2254 bytes result sent to driver -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 251.0 (TID 84) in 23 ms on localhost (1/2) -2016-06-13 19:47:31 INFO Executor:59 - Finished task 1.0 in stage 251.0 (TID 85). 2254 bytes result sent to driver -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 251.0 (TID 85) in 24 ms on localhost (2/2) -2016-06-13 19:47:31 INFO DAGScheduler:59 - ShuffleMapStage 251 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.025 s -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 251.0, whose tasks have all completed, from pool -2016-06-13 19:47:31 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:31 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:31 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 252, ResultStage 253) -2016-06-13 19:47:31 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 252: List() -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents for ResultStage 253: List(ShuffleMapStage 252) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting ShuffleMapStage 252 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[191] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=262188, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_45 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(2284) called with curMem=266084, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_45_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added broadcast_45_piece0 in memory on localhost:50092 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO SparkContext:59 - Created broadcast 45 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 252 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[191] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Adding task set 252.0 with 2 tasks -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 252.0 (TID 86, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 252.0 (TID 87, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:31 INFO Executor:59 - Running task 1.0 in stage 252.0 (TID 87) -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:31 INFO Executor:59 - Running task 0.0 in stage 252.0 (TID 86) -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:31 INFO Executor:59 - Finished task 1.0 in stage 252.0 (TID 87). 2470 bytes result sent to driver -2016-06-13 19:47:31 INFO Executor:59 - Finished task 0.0 in stage 252.0 (TID 86). 2470 bytes result sent to driver -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 252.0 (TID 87) in 30 ms on localhost (1/2) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 252.0 (TID 86) in 33 ms on localhost (2/2) -2016-06-13 19:47:31 INFO DAGScheduler:59 - ShuffleMapStage 252 (mapPartitions at GraphImpl.scala:235) finished in 0.033 s -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 252.0, whose tasks have all completed, from pool -2016-06-13 19:47:31 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:31 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:31 INFO DAGScheduler:59 - waiting: Set(ResultStage 253) -2016-06-13 19:47:31 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents for ResultStage 253: List() -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting ResultStage 253 (MapPartitionsRDD[199] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(11568) called with curMem=268368, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_46 stored as values in memory (estimated size 11.3 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(4283) called with curMem=279936, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_46_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added broadcast_46_piece0 in memory on localhost:50092 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO SparkContext:59 - Created broadcast 46 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 253 (MapPartitionsRDD[199] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Adding task set 253.0 with 2 tasks -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 253.0 (TID 88, localhost, PROCESS_LOCAL, 2658 bytes) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 253.0 (TID 89, localhost, PROCESS_LOCAL, 2658 bytes) -2016-06-13 19:47:31 INFO Executor:59 - Running task 0.0 in stage 253.0 (TID 88) -2016-06-13 19:47:31 INFO Executor:59 - Running task 1.0 in stage 253.0 (TID 89) -2016-06-13 19:47:31 INFO CacheManager:59 - Partition rdd_195_0 not found, computing it -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_182_0 locally -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:31 INFO CacheManager:59 - Partition rdd_195_1 not found, computing it -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_182_1 locally -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=284219, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block rdd_195_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=286763, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block rdd_195_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added rdd_195_0 in memory on localhost:50092 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added rdd_195_1 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO Executor:59 - Finished task 0.0 in stage 253.0 (TID 88). 2971 bytes result sent to driver -2016-06-13 19:47:31 INFO Executor:59 - Finished task 1.0 in stage 253.0 (TID 89). 2971 bytes result sent to driver -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 253.0 (TID 88) in 15 ms on localhost (1/2) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 253.0 (TID 89) in 14 ms on localhost (2/2) -2016-06-13 19:47:31 INFO DAGScheduler:59 - ResultStage 253 (fold at LDAOptimizer.scala:205) finished in 0.016 s -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 253.0, whose tasks have all completed, from pool -2016-06-13 19:47:31 INFO DAGScheduler:59 - Job 15 finished: fold at LDAOptimizer.scala:205, took 0.137914 s -2016-06-13 19:47:31 INFO MapPartitionsRDD:59 - Removing RDD 169 from persistence list -2016-06-13 19:47:31 INFO BlockManager:59 - Removing RDD 169 -2016-06-13 19:47:31 INFO ZippedPartitionsRDD2:59 - Removing RDD 176 from persistence list -2016-06-13 19:47:31 INFO BlockManager:59 - Removing RDD 176 -2016-06-13 19:47:31 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:47:31 INFO DAGScheduler:59 - Registering RDD 200 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Registering RDD 204 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Got job 16 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:31 INFO DAGScheduler:59 - Final stage: ResultStage 286(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 269, ShuffleMapStage 263, ShuffleMapStage 255, ShuffleMapStage 285, ShuffleMapStage 277, ShuffleMapStage 271, ShuffleMapStage 257, ShuffleMapStage 279, ShuffleMapStage 265, ShuffleMapStage 281, ShuffleMapStage 273, ShuffleMapStage 267, ShuffleMapStage 259, ShuffleMapStage 275, ShuffleMapStage 261, ShuffleMapStage 283, ShuffleMapStage 254) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 285) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting ShuffleMapStage 284 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[200] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(10584) called with curMem=284219, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_47 stored as values in memory (estimated size 10.3 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(3951) called with curMem=294803, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_47_piece0 stored as bytes in memory (estimated size 3.9 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added broadcast_47_piece0 in memory on localhost:50092 (size: 3.9 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO SparkContext:59 - Created broadcast 47 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 284 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[200] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Adding task set 284.0 with 2 tasks -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 284.0 (TID 90, localhost, PROCESS_LOCAL, 2647 bytes) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 284.0 (TID 91, localhost, PROCESS_LOCAL, 2647 bytes) -2016-06-13 19:47:31 INFO Executor:59 - Running task 0.0 in stage 284.0 (TID 90) -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_195_0 locally -2016-06-13 19:47:31 INFO Executor:59 - Running task 1.0 in stage 284.0 (TID 91) -2016-06-13 19:47:31 INFO Executor:59 - Finished task 0.0 in stage 284.0 (TID 90). 2254 bytes result sent to driver -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 284.0 (TID 90) in 27 ms on localhost (1/2) -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_195_1 locally -2016-06-13 19:47:31 INFO Executor:59 - Finished task 1.0 in stage 284.0 (TID 91). 2254 bytes result sent to driver -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 284.0 (TID 91) in 56 ms on localhost (2/2) -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 284.0, whose tasks have all completed, from pool -2016-06-13 19:47:31 INFO DAGScheduler:59 - ShuffleMapStage 284 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.058 s -2016-06-13 19:47:31 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:31 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:31 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 285, ResultStage 286) -2016-06-13 19:47:31 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 285: List() -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents for ResultStage 286: List(ShuffleMapStage 285) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting ShuffleMapStage 285 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[204] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=298754, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_48 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=302650, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_48_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added broadcast_48_piece0 in memory on localhost:50092 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO SparkContext:59 - Created broadcast 48 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 285 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[204] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Adding task set 285.0 with 2 tasks -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 285.0 (TID 92, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 285.0 (TID 93, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:31 INFO Executor:59 - Running task 0.0 in stage 285.0 (TID 92) -2016-06-13 19:47:31 INFO Executor:59 - Running task 1.0 in stage 285.0 (TID 93) -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:31 INFO Executor:59 - Finished task 0.0 in stage 285.0 (TID 92). 2470 bytes result sent to driver -2016-06-13 19:47:31 INFO Executor:59 - Finished task 1.0 in stage 285.0 (TID 93). 2470 bytes result sent to driver -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 285.0 (TID 92) in 29 ms on localhost (1/2) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 285.0 (TID 93) in 30 ms on localhost (2/2) -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 285.0, whose tasks have all completed, from pool -2016-06-13 19:47:31 INFO DAGScheduler:59 - ShuffleMapStage 285 (mapPartitions at GraphImpl.scala:235) finished in 0.031 s -2016-06-13 19:47:31 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:31 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:31 INFO DAGScheduler:59 - waiting: Set(ResultStage 286) -2016-06-13 19:47:31 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents for ResultStage 286: List() -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting ResultStage 286 (MapPartitionsRDD[212] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(12000) called with curMem=304936, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_49 stored as values in memory (estimated size 11.7 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(4400) called with curMem=316936, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_49_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added broadcast_49_piece0 in memory on localhost:50092 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO SparkContext:59 - Created broadcast 49 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 286 (MapPartitionsRDD[212] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Adding task set 286.0 with 2 tasks -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 286.0 (TID 94, localhost, PROCESS_LOCAL, 2699 bytes) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 286.0 (TID 95, localhost, PROCESS_LOCAL, 2699 bytes) -2016-06-13 19:47:31 INFO Executor:59 - Running task 1.0 in stage 286.0 (TID 95) -2016-06-13 19:47:31 INFO Executor:59 - Running task 0.0 in stage 286.0 (TID 94) -2016-06-13 19:47:31 INFO CacheManager:59 - Partition rdd_208_1 not found, computing it -2016-06-13 19:47:31 INFO CacheManager:59 - Partition rdd_208_0 not found, computing it -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_195_1 locally -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_195_0 locally -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=321336, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block rdd_208_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added rdd_208_1 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=323784, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block rdd_208_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO Executor:59 - Finished task 1.0 in stage 286.0 (TID 95). 2971 bytes result sent to driver -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added rdd_208_0 in memory on localhost:50092 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO Executor:59 - Finished task 0.0 in stage 286.0 (TID 94). 2971 bytes result sent to driver -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 286.0 (TID 95) in 15 ms on localhost (1/2) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 286.0 (TID 94) in 16 ms on localhost (2/2) -2016-06-13 19:47:31 INFO DAGScheduler:59 - ResultStage 286 (fold at LDAOptimizer.scala:205) finished in 0.017 s -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 286.0, whose tasks have all completed, from pool -2016-06-13 19:47:31 INFO DAGScheduler:59 - Job 16 finished: fold at LDAOptimizer.scala:205, took 0.182311 s -2016-06-13 19:47:31 INFO MapPartitionsRDD:59 - Removing RDD 182 from persistence list -2016-06-13 19:47:31 INFO ZippedPartitionsRDD2:59 - Removing RDD 189 from persistence list -2016-06-13 19:47:31 INFO BlockManager:59 - Removing RDD 182 -2016-06-13 19:47:31 INFO BlockManager:59 - Removing RDD 189 -2016-06-13 19:47:31 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:47:31 INFO DAGScheduler:59 - Registering RDD 213 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Registering RDD 217 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Got job 17 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:31 INFO DAGScheduler:59 - Final stage: ResultStage 321(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 320, ShuffleMapStage 306, ShuffleMapStage 300, ShuffleMapStage 292, ShuffleMapStage 314, ShuffleMapStage 308, ShuffleMapStage 294, ShuffleMapStage 316, ShuffleMapStage 287, ShuffleMapStage 302, ShuffleMapStage 288, ShuffleMapStage 318, ShuffleMapStage 310, ShuffleMapStage 304, ShuffleMapStage 296, ShuffleMapStage 290, ShuffleMapStage 312, ShuffleMapStage 298) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 320) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting ShuffleMapStage 319 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[213] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(11008) called with curMem=321336, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_50 stored as values in memory (estimated size 10.8 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(4062) called with curMem=332344, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_50_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added broadcast_50_piece0 in memory on localhost:50092 (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO SparkContext:59 - Created broadcast 50 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 319 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[213] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Adding task set 319.0 with 2 tasks -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 319.0 (TID 96, localhost, PROCESS_LOCAL, 2688 bytes) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 319.0 (TID 97, localhost, PROCESS_LOCAL, 2688 bytes) -2016-06-13 19:47:31 INFO Executor:59 - Running task 0.0 in stage 319.0 (TID 96) -2016-06-13 19:47:31 INFO Executor:59 - Running task 1.0 in stage 319.0 (TID 97) -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_208_0 locally -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_208_1 locally -2016-06-13 19:47:31 INFO Executor:59 - Finished task 0.0 in stage 319.0 (TID 96). 2254 bytes result sent to driver -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 319.0 (TID 96) in 20 ms on localhost (1/2) -2016-06-13 19:47:31 INFO Executor:59 - Finished task 1.0 in stage 319.0 (TID 97). 2254 bytes result sent to driver -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 319.0 (TID 97) in 22 ms on localhost (2/2) -2016-06-13 19:47:31 INFO DAGScheduler:59 - ShuffleMapStage 319 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.023 s -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 319.0, whose tasks have all completed, from pool -2016-06-13 19:47:31 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:31 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:31 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 320, ResultStage 321) -2016-06-13 19:47:31 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 320: List() -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents for ResultStage 321: List(ShuffleMapStage 320) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting ShuffleMapStage 320 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[217] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=336406, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_51 stored as values in memory (estimated size 3.8 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=340302, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_51_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added broadcast_51_piece0 in memory on localhost:50092 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO SparkContext:59 - Created broadcast 51 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 320 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[217] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Adding task set 320.0 with 2 tasks -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 320.0 (TID 98, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 320.0 (TID 99, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:31 INFO Executor:59 - Running task 1.0 in stage 320.0 (TID 99) -2016-06-13 19:47:31 INFO Executor:59 - Running task 0.0 in stage 320.0 (TID 98) -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:31 INFO Executor:59 - Finished task 0.0 in stage 320.0 (TID 98). 2470 bytes result sent to driver -2016-06-13 19:47:31 INFO Executor:59 - Finished task 1.0 in stage 320.0 (TID 99). 2470 bytes result sent to driver -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 320.0 (TID 98) in 23 ms on localhost (1/2) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 320.0 (TID 99) in 23 ms on localhost (2/2) -2016-06-13 19:47:31 INFO DAGScheduler:59 - ShuffleMapStage 320 (mapPartitions at GraphImpl.scala:235) finished in 0.023 s -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 320.0, whose tasks have all completed, from pool -2016-06-13 19:47:31 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:31 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:31 INFO DAGScheduler:59 - waiting: Set(ResultStage 321) -2016-06-13 19:47:31 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents for ResultStage 321: List() -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting ResultStage 321 (MapPartitionsRDD[225] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(12424) called with curMem=342588, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_52 stored as values in memory (estimated size 12.1 KB, free 730.2 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(4491) called with curMem=355012, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_52_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.2 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added broadcast_52_piece0 in memory on localhost:50092 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO SparkContext:59 - Created broadcast 52 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 321 (MapPartitionsRDD[225] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Adding task set 321.0 with 2 tasks -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 321.0 (TID 100, localhost, PROCESS_LOCAL, 2740 bytes) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 321.0 (TID 101, localhost, PROCESS_LOCAL, 2740 bytes) -2016-06-13 19:47:31 INFO Executor:59 - Running task 0.0 in stage 321.0 (TID 100) -2016-06-13 19:47:31 INFO Executor:59 - Running task 1.0 in stage 321.0 (TID 101) -2016-06-13 19:47:31 INFO CacheManager:59 - Partition rdd_221_1 not found, computing it -2016-06-13 19:47:31 INFO CacheManager:59 - Partition rdd_221_0 not found, computing it -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_208_1 locally -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_208_0 locally -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=359503, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block rdd_221_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=362047, maxMem=766075207 -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added rdd_221_0 in memory on localhost:50092 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - Block rdd_221_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:47:31 INFO Executor:59 - Finished task 0.0 in stage 321.0 (TID 100). 2971 bytes result sent to driver -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added rdd_221_1 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO Executor:59 - Finished task 1.0 in stage 321.0 (TID 101). 2971 bytes result sent to driver -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 321.0 (TID 100) in 12 ms on localhost (1/2) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 321.0 (TID 101) in 12 ms on localhost (2/2) -2016-06-13 19:47:31 INFO DAGScheduler:59 - ResultStage 321 (fold at LDAOptimizer.scala:205) finished in 0.013 s -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 321.0, whose tasks have all completed, from pool -2016-06-13 19:47:31 INFO DAGScheduler:59 - Job 17 finished: fold at LDAOptimizer.scala:205, took 0.122340 s -2016-06-13 19:47:31 INFO MapPartitionsRDD:59 - Removing RDD 195 from persistence list -2016-06-13 19:47:31 INFO ZippedPartitionsRDD2:59 - Removing RDD 202 from persistence list -2016-06-13 19:47:31 INFO BlockManager:59 - Removing RDD 195 -2016-06-13 19:47:31 INFO BlockManager:59 - Removing RDD 202 -2016-06-13 19:47:31 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:47:31 INFO DAGScheduler:59 - Registering RDD 226 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Registering RDD 230 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Got job 18 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:31 INFO DAGScheduler:59 - Final stage: ResultStage 358(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 335, ShuffleMapStage 357, ShuffleMapStage 343, ShuffleMapStage 322, ShuffleMapStage 329, ShuffleMapStage 323, ShuffleMapStage 351, ShuffleMapStage 345, ShuffleMapStage 337, ShuffleMapStage 331, ShuffleMapStage 353, ShuffleMapStage 339, ShuffleMapStage 325, ShuffleMapStage 355, ShuffleMapStage 347, ShuffleMapStage 341, ShuffleMapStage 333, ShuffleMapStage 327, ShuffleMapStage 349) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 357) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting ShuffleMapStage 356 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[226] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(11440) called with curMem=359503, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_53 stored as values in memory (estimated size 11.2 KB, free 730.2 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(4162) called with curMem=370943, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_53_piece0 stored as bytes in memory (estimated size 4.1 KB, free 730.2 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added broadcast_53_piece0 in memory on localhost:50092 (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO SparkContext:59 - Created broadcast 53 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 356 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[226] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Adding task set 356.0 with 2 tasks -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 356.0 (TID 102, localhost, PROCESS_LOCAL, 2729 bytes) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 356.0 (TID 103, localhost, PROCESS_LOCAL, 2729 bytes) -2016-06-13 19:47:31 INFO Executor:59 - Running task 1.0 in stage 356.0 (TID 103) -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_221_1 locally -2016-06-13 19:47:31 INFO Executor:59 - Running task 0.0 in stage 356.0 (TID 102) -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_221_0 locally -2016-06-13 19:47:31 INFO Executor:59 - Finished task 1.0 in stage 356.0 (TID 103). 2254 bytes result sent to driver -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 356.0 (TID 103) in 40 ms on localhost (1/2) -2016-06-13 19:47:31 INFO Executor:59 - Finished task 0.0 in stage 356.0 (TID 102). 2254 bytes result sent to driver -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 356.0 (TID 102) in 53 ms on localhost (2/2) -2016-06-13 19:47:31 INFO DAGScheduler:59 - ShuffleMapStage 356 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.053 s -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 356.0, whose tasks have all completed, from pool -2016-06-13 19:47:31 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:31 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:31 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 357, ResultStage 358) -2016-06-13 19:47:31 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 357: List() -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents for ResultStage 358: List(ShuffleMapStage 357) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting ShuffleMapStage 357 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[230] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=375105, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_54 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(2286) called with curMem=379001, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_54_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added broadcast_54_piece0 in memory on localhost:50092 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO SparkContext:59 - Created broadcast 54 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 357 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[230] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Adding task set 357.0 with 2 tasks -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 357.0 (TID 104, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 357.0 (TID 105, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:31 INFO Executor:59 - Running task 0.0 in stage 357.0 (TID 104) -2016-06-13 19:47:31 INFO Executor:59 - Running task 1.0 in stage 357.0 (TID 105) -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:31 INFO Executor:59 - Finished task 0.0 in stage 357.0 (TID 104). 2470 bytes result sent to driver -2016-06-13 19:47:31 INFO Executor:59 - Finished task 1.0 in stage 357.0 (TID 105). 2470 bytes result sent to driver -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 357.0 (TID 104) in 22 ms on localhost (1/2) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 357.0 (TID 105) in 22 ms on localhost (2/2) -2016-06-13 19:47:31 INFO DAGScheduler:59 - ShuffleMapStage 357 (mapPartitions at GraphImpl.scala:235) finished in 0.023 s -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 357.0, whose tasks have all completed, from pool -2016-06-13 19:47:31 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:31 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:31 INFO DAGScheduler:59 - waiting: Set(ResultStage 358) -2016-06-13 19:47:31 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents for ResultStage 358: List() -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting ResultStage 358 (MapPartitionsRDD[238] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(12856) called with curMem=381287, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_55 stored as values in memory (estimated size 12.6 KB, free 730.2 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(4586) called with curMem=394143, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_55_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.2 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added broadcast_55_piece0 in memory on localhost:50092 (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO SparkContext:59 - Created broadcast 55 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 358 (MapPartitionsRDD[238] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Adding task set 358.0 with 2 tasks -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 358.0 (TID 106, localhost, PROCESS_LOCAL, 2781 bytes) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 358.0 (TID 107, localhost, PROCESS_LOCAL, 2781 bytes) -2016-06-13 19:47:31 INFO Executor:59 - Running task 0.0 in stage 358.0 (TID 106) -2016-06-13 19:47:31 INFO Executor:59 - Running task 1.0 in stage 358.0 (TID 107) -2016-06-13 19:47:31 INFO CacheManager:59 - Partition rdd_234_0 not found, computing it -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_221_0 locally -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=398729, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block rdd_234_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:47:31 INFO CacheManager:59 - Partition rdd_234_1 not found, computing it -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added rdd_234_0 in memory on localhost:50092 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_221_1 locally -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:31 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:31 INFO Executor:59 - Finished task 0.0 in stage 358.0 (TID 106). 2971 bytes result sent to driver -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 358.0 (TID 106) in 18 ms on localhost (1/2) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=401273, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block rdd_234_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added rdd_234_1 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO Executor:59 - Finished task 1.0 in stage 358.0 (TID 107). 2971 bytes result sent to driver -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 358.0 (TID 107) in 25 ms on localhost (2/2) -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 358.0, whose tasks have all completed, from pool -2016-06-13 19:47:31 INFO DAGScheduler:59 - ResultStage 358 (fold at LDAOptimizer.scala:205) finished in 0.025 s -2016-06-13 19:47:31 INFO DAGScheduler:59 - Job 18 finished: fold at LDAOptimizer.scala:205, took 0.163573 s -2016-06-13 19:47:31 INFO MapPartitionsRDD:59 - Removing RDD 208 from persistence list -2016-06-13 19:47:31 INFO BlockManager:59 - Removing RDD 208 -2016-06-13 19:47:31 INFO ZippedPartitionsRDD2:59 - Removing RDD 215 from persistence list -2016-06-13 19:47:31 INFO BlockManager:59 - Removing RDD 215 -2016-06-13 19:47:31 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:47:31 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:47:31 INFO DAGScheduler:59 - Registering RDD 239 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Registering RDD 243 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Got job 19 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:31 INFO DAGScheduler:59 - Final stage: ResultStage 397(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 386, ShuffleMapStage 372, ShuffleMapStage 394, ShuffleMapStage 380, ShuffleMapStage 359, ShuffleMapStage 366, ShuffleMapStage 396, ShuffleMapStage 360, ShuffleMapStage 388, ShuffleMapStage 382, ShuffleMapStage 374, ShuffleMapStage 368, ShuffleMapStage 390, ShuffleMapStage 376, ShuffleMapStage 362, ShuffleMapStage 392, ShuffleMapStage 384, ShuffleMapStage 378, ShuffleMapStage 370, ShuffleMapStage 364) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 396) -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting ShuffleMapStage 395 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[239] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(11864) called with curMem=398729, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_56 stored as values in memory (estimated size 11.6 KB, free 730.2 MB) -2016-06-13 19:47:31 INFO MemoryStore:59 - ensureFreeSpace(4268) called with curMem=410593, maxMem=766075207 -2016-06-13 19:47:31 INFO MemoryStore:59 - Block broadcast_56_piece0 stored as bytes in memory (estimated size 4.2 KB, free 730.2 MB) -2016-06-13 19:47:31 INFO BlockManagerInfo:59 - Added broadcast_56_piece0 in memory on localhost:50092 (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:47:31 INFO SparkContext:59 - Created broadcast 56 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 395 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[239] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Adding task set 395.0 with 2 tasks -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 0.0 in stage 395.0 (TID 108, localhost, PROCESS_LOCAL, 2770 bytes) -2016-06-13 19:47:31 INFO TaskSetManager:59 - Starting task 1.0 in stage 395.0 (TID 109, localhost, PROCESS_LOCAL, 2770 bytes) -2016-06-13 19:47:31 INFO Executor:59 - Running task 0.0 in stage 395.0 (TID 108) -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_234_0 locally -2016-06-13 19:47:31 INFO Executor:59 - Finished task 0.0 in stage 395.0 (TID 108). 2254 bytes result sent to driver -2016-06-13 19:47:31 INFO Executor:59 - Running task 1.0 in stage 395.0 (TID 109) -2016-06-13 19:47:31 INFO BlockManager:59 - Found block rdd_234_1 locally -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 0.0 in stage 395.0 (TID 108) in 30 ms on localhost (1/2) -2016-06-13 19:47:31 INFO Executor:59 - Finished task 1.0 in stage 395.0 (TID 109). 2254 bytes result sent to driver -2016-06-13 19:47:31 INFO TaskSetManager:59 - Finished task 1.0 in stage 395.0 (TID 109) in 46 ms on localhost (2/2) -2016-06-13 19:47:31 INFO DAGScheduler:59 - ShuffleMapStage 395 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.048 s -2016-06-13 19:47:31 INFO TaskSchedulerImpl:59 - Removed TaskSet 395.0, whose tasks have all completed, from pool -2016-06-13 19:47:31 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:31 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:31 INFO DAGScheduler:59 - waiting: Set(ResultStage 397, ShuffleMapStage 396) -2016-06-13 19:47:32 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:32 INFO DAGScheduler:59 - Missing parents for ResultStage 397: List(ShuffleMapStage 396) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 396: List() -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting ShuffleMapStage 396 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[243] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=414861, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_57 stored as values in memory (estimated size 3.8 KB, free 730.2 MB) -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(2288) called with curMem=418757, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_57_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added broadcast_57_piece0 in memory on localhost:50092 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO SparkContext:59 - Created broadcast 57 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 396 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[243] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Adding task set 396.0 with 2 tasks -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 396.0 (TID 110, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 396.0 (TID 111, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:32 INFO Executor:59 - Running task 0.0 in stage 396.0 (TID 110) -2016-06-13 19:47:32 INFO Executor:59 - Running task 1.0 in stage 396.0 (TID 111) -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:32 INFO Executor:59 - Finished task 1.0 in stage 396.0 (TID 111). 2470 bytes result sent to driver -2016-06-13 19:47:32 INFO Executor:59 - Finished task 0.0 in stage 396.0 (TID 110). 2470 bytes result sent to driver -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 396.0 (TID 111) in 22 ms on localhost (1/2) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 396.0 (TID 110) in 24 ms on localhost (2/2) -2016-06-13 19:47:32 INFO DAGScheduler:59 - ShuffleMapStage 396 (mapPartitions at GraphImpl.scala:235) finished in 0.024 s -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 396.0, whose tasks have all completed, from pool -2016-06-13 19:47:32 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:32 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:32 INFO DAGScheduler:59 - waiting: Set(ResultStage 397) -2016-06-13 19:47:32 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:32 INFO DAGScheduler:59 - Missing parents for ResultStage 397: List() -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting ResultStage 397 (MapPartitionsRDD[251] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(13280) called with curMem=421045, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_58 stored as values in memory (estimated size 13.0 KB, free 730.2 MB) -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(4699) called with curMem=434325, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_58_piece0 stored as bytes in memory (estimated size 4.6 KB, free 730.2 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added broadcast_58_piece0 in memory on localhost:50092 (size: 4.6 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO SparkContext:59 - Created broadcast 58 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 397 (MapPartitionsRDD[251] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Adding task set 397.0 with 2 tasks -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 397.0 (TID 112, localhost, PROCESS_LOCAL, 2822 bytes) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 397.0 (TID 113, localhost, PROCESS_LOCAL, 2822 bytes) -2016-06-13 19:47:32 INFO Executor:59 - Running task 1.0 in stage 397.0 (TID 113) -2016-06-13 19:47:32 INFO Executor:59 - Running task 0.0 in stage 397.0 (TID 112) -2016-06-13 19:47:32 INFO CacheManager:59 - Partition rdd_247_0 not found, computing it -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_234_0 locally -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=439024, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block rdd_247_0 stored as values in memory (estimated size 2.5 KB, free 730.2 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added rdd_247_0 in memory on localhost:50092 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO CacheManager:59 - Partition rdd_247_1 not found, computing it -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_234_1 locally -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:32 INFO Executor:59 - Finished task 0.0 in stage 397.0 (TID 112). 2971 bytes result sent to driver -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=441568, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block rdd_247_1 stored as values in memory (estimated size 2.4 KB, free 730.2 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added rdd_247_1 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 397.0 (TID 112) in 21 ms on localhost (1/2) -2016-06-13 19:47:32 INFO Executor:59 - Finished task 1.0 in stage 397.0 (TID 113). 2971 bytes result sent to driver -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 397.0 (TID 113) in 22 ms on localhost (2/2) -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 397.0, whose tasks have all completed, from pool -2016-06-13 19:47:32 INFO DAGScheduler:59 - ResultStage 397 (fold at LDAOptimizer.scala:205) finished in 0.023 s -2016-06-13 19:47:32 INFO DAGScheduler:59 - Job 19 finished: fold at LDAOptimizer.scala:205, took 0.174027 s -2016-06-13 19:47:32 INFO MapPartitionsRDD:59 - Removing RDD 221 from persistence list -2016-06-13 19:47:32 INFO BlockManager:59 - Removing RDD 221 -2016-06-13 19:47:32 INFO ZippedPartitionsRDD2:59 - Removing RDD 228 from persistence list -2016-06-13 19:47:32 INFO BlockManager:59 - Removing RDD 228 -2016-06-13 19:47:32 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 155 bytes -2016-06-13 19:47:32 INFO DAGScheduler:59 - Registering RDD 252 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Registering RDD 256 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Got job 20 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:32 INFO DAGScheduler:59 - Final stage: ResultStage 438(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 407, ShuffleMapStage 437, ShuffleMapStage 401, ShuffleMapStage 423, ShuffleMapStage 409, ShuffleMapStage 431, ShuffleMapStage 417, ShuffleMapStage 403, ShuffleMapStage 433, ShuffleMapStage 425, ShuffleMapStage 419, ShuffleMapStage 411, ShuffleMapStage 405, ShuffleMapStage 427, ShuffleMapStage 398, ShuffleMapStage 413, ShuffleMapStage 435, ShuffleMapStage 399, ShuffleMapStage 429, ShuffleMapStage 421, ShuffleMapStage 415) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 437) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting ShuffleMapStage 436 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[252] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(12296) called with curMem=439024, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_59 stored as values in memory (estimated size 12.0 KB, free 730.2 MB) -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(4356) called with curMem=451320, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_59_piece0 stored as bytes in memory (estimated size 4.3 KB, free 730.2 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added broadcast_59_piece0 in memory on localhost:50092 (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO SparkContext:59 - Created broadcast 59 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 436 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[252] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Adding task set 436.0 with 2 tasks -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 436.0 (TID 114, localhost, PROCESS_LOCAL, 2811 bytes) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 436.0 (TID 115, localhost, PROCESS_LOCAL, 2811 bytes) -2016-06-13 19:47:32 INFO Executor:59 - Running task 1.0 in stage 436.0 (TID 115) -2016-06-13 19:47:32 INFO Executor:59 - Running task 0.0 in stage 436.0 (TID 114) -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_247_0 locally -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_247_1 locally -2016-06-13 19:47:32 INFO Executor:59 - Finished task 0.0 in stage 436.0 (TID 114). 2254 bytes result sent to driver -2016-06-13 19:47:32 INFO Executor:59 - Finished task 1.0 in stage 436.0 (TID 115). 2254 bytes result sent to driver -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 436.0 (TID 114) in 24 ms on localhost (1/2) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 436.0 (TID 115) in 24 ms on localhost (2/2) -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 436.0, whose tasks have all completed, from pool -2016-06-13 19:47:32 INFO DAGScheduler:59 - ShuffleMapStage 436 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.027 s -2016-06-13 19:47:32 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:32 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:32 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 437, ResultStage 438) -2016-06-13 19:47:32 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:32 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 437: List() -2016-06-13 19:47:32 INFO DAGScheduler:59 - Missing parents for ResultStage 438: List(ShuffleMapStage 437) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting ShuffleMapStage 437 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[256] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=455676, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_60 stored as values in memory (estimated size 3.8 KB, free 730.1 MB) -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(2288) called with curMem=459572, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_60_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added broadcast_60_piece0 in memory on localhost:50092 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO SparkContext:59 - Created broadcast 60 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 437 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[256] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Adding task set 437.0 with 2 tasks -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 437.0 (TID 116, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 437.0 (TID 117, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:32 INFO Executor:59 - Running task 0.0 in stage 437.0 (TID 116) -2016-06-13 19:47:32 INFO Executor:59 - Running task 1.0 in stage 437.0 (TID 117) -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:32 INFO Executor:59 - Finished task 1.0 in stage 437.0 (TID 117). 2470 bytes result sent to driver -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 437.0 (TID 117) in 28 ms on localhost (1/2) -2016-06-13 19:47:32 INFO Executor:59 - Finished task 0.0 in stage 437.0 (TID 116). 2470 bytes result sent to driver -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 437.0 (TID 116) in 34 ms on localhost (2/2) -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 437.0, whose tasks have all completed, from pool -2016-06-13 19:47:32 INFO DAGScheduler:59 - ShuffleMapStage 437 (mapPartitions at GraphImpl.scala:235) finished in 0.035 s -2016-06-13 19:47:32 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:32 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:32 INFO DAGScheduler:59 - waiting: Set(ResultStage 438) -2016-06-13 19:47:32 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:32 INFO DAGScheduler:59 - Missing parents for ResultStage 438: List() -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting ResultStage 438 (MapPartitionsRDD[264] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(13712) called with curMem=461860, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_61 stored as values in memory (estimated size 13.4 KB, free 730.1 MB) -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(4770) called with curMem=475572, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_61_piece0 stored as bytes in memory (estimated size 4.7 KB, free 730.1 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added broadcast_61_piece0 in memory on localhost:50092 (size: 4.7 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO SparkContext:59 - Created broadcast 61 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 438 (MapPartitionsRDD[264] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Adding task set 438.0 with 2 tasks -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 438.0 (TID 118, localhost, PROCESS_LOCAL, 2863 bytes) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 438.0 (TID 119, localhost, PROCESS_LOCAL, 2863 bytes) -2016-06-13 19:47:32 INFO Executor:59 - Running task 1.0 in stage 438.0 (TID 119) -2016-06-13 19:47:32 INFO Executor:59 - Running task 0.0 in stage 438.0 (TID 118) -2016-06-13 19:47:32 INFO CacheManager:59 - Partition rdd_260_1 not found, computing it -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_247_1 locally -2016-06-13 19:47:32 INFO CacheManager:59 - Partition rdd_260_0 not found, computing it -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_247_0 locally -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=480342, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block rdd_260_0 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=482886, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block rdd_260_1 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added rdd_260_0 in memory on localhost:50092 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added rdd_260_1 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO Executor:59 - Finished task 0.0 in stage 438.0 (TID 118). 2971 bytes result sent to driver -2016-06-13 19:47:32 INFO Executor:59 - Finished task 1.0 in stage 438.0 (TID 119). 2971 bytes result sent to driver -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 438.0 (TID 118) in 13 ms on localhost (1/2) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 438.0 (TID 119) in 13 ms on localhost (2/2) -2016-06-13 19:47:32 INFO DAGScheduler:59 - ResultStage 438 (fold at LDAOptimizer.scala:205) finished in 0.014 s -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 438.0, whose tasks have all completed, from pool -2016-06-13 19:47:32 INFO DAGScheduler:59 - Job 20 finished: fold at LDAOptimizer.scala:205, took 0.171190 s -2016-06-13 19:47:32 INFO MapPartitionsRDD:59 - Removing RDD 234 from persistence list -2016-06-13 19:47:32 INFO BlockManager:59 - Removing RDD 234 -2016-06-13 19:47:32 INFO ZippedPartitionsRDD2:59 - Removing RDD 241 from persistence list -2016-06-13 19:47:32 INFO BlockManager:59 - Removing RDD 241 -2016-06-13 19:47:32 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:47:32 INFO DAGScheduler:59 - Registering RDD 265 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Registering RDD 269 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Got job 21 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:32 INFO DAGScheduler:59 - Final stage: ResultStage 481(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 466, ShuffleMapStage 458, ShuffleMapStage 452, ShuffleMapStage 444, ShuffleMapStage 474, ShuffleMapStage 460, ShuffleMapStage 446, ShuffleMapStage 468, ShuffleMapStage 439, ShuffleMapStage 454, ShuffleMapStage 476, ShuffleMapStage 440, ShuffleMapStage 470, ShuffleMapStage 462, ShuffleMapStage 456, ShuffleMapStage 448, ShuffleMapStage 478, ShuffleMapStage 442, ShuffleMapStage 464, ShuffleMapStage 450, ShuffleMapStage 480, ShuffleMapStage 472) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 480) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting ShuffleMapStage 479 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[265] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(12720) called with curMem=480342, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_62 stored as values in memory (estimated size 12.4 KB, free 730.1 MB) -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(4470) called with curMem=493062, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_62_piece0 stored as bytes in memory (estimated size 4.4 KB, free 730.1 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added broadcast_62_piece0 in memory on localhost:50092 (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO SparkContext:59 - Created broadcast 62 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 479 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[265] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Adding task set 479.0 with 2 tasks -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 479.0 (TID 120, localhost, PROCESS_LOCAL, 2852 bytes) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 479.0 (TID 121, localhost, PROCESS_LOCAL, 2852 bytes) -2016-06-13 19:47:32 INFO Executor:59 - Running task 1.0 in stage 479.0 (TID 121) -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_260_1 locally -2016-06-13 19:47:32 INFO Executor:59 - Running task 0.0 in stage 479.0 (TID 120) -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_260_0 locally -2016-06-13 19:47:32 INFO Executor:59 - Finished task 1.0 in stage 479.0 (TID 121). 2254 bytes result sent to driver -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 479.0 (TID 121) in 24 ms on localhost (1/2) -2016-06-13 19:47:32 INFO Executor:59 - Finished task 0.0 in stage 479.0 (TID 120). 2254 bytes result sent to driver -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 479.0 (TID 120) in 28 ms on localhost (2/2) -2016-06-13 19:47:32 INFO DAGScheduler:59 - ShuffleMapStage 479 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.028 s -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 479.0, whose tasks have all completed, from pool -2016-06-13 19:47:32 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:32 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:32 INFO DAGScheduler:59 - waiting: Set(ResultStage 481, ShuffleMapStage 480) -2016-06-13 19:47:32 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:32 INFO DAGScheduler:59 - Missing parents for ResultStage 481: List(ShuffleMapStage 480) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 480: List() -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting ShuffleMapStage 480 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[269] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=497532, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_63 stored as values in memory (estimated size 3.8 KB, free 730.1 MB) -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(2285) called with curMem=501428, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_63_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added broadcast_63_piece0 in memory on localhost:50092 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO SparkContext:59 - Created broadcast 63 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 480 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[269] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Adding task set 480.0 with 2 tasks -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 480.0 (TID 122, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 480.0 (TID 123, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:32 INFO Executor:59 - Running task 0.0 in stage 480.0 (TID 122) -2016-06-13 19:47:32 INFO Executor:59 - Running task 1.0 in stage 480.0 (TID 123) -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:32 INFO Executor:59 - Finished task 1.0 in stage 480.0 (TID 123). 2470 bytes result sent to driver -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 480.0 (TID 123) in 23 ms on localhost (1/2) -2016-06-13 19:47:32 INFO Executor:59 - Finished task 0.0 in stage 480.0 (TID 122). 2470 bytes result sent to driver -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 480.0 (TID 122) in 25 ms on localhost (2/2) -2016-06-13 19:47:32 INFO DAGScheduler:59 - ShuffleMapStage 480 (mapPartitions at GraphImpl.scala:235) finished in 0.026 s -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 480.0, whose tasks have all completed, from pool -2016-06-13 19:47:32 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:32 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:32 INFO DAGScheduler:59 - waiting: Set(ResultStage 481) -2016-06-13 19:47:32 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:32 INFO DAGScheduler:59 - Missing parents for ResultStage 481: List() -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting ResultStage 481 (MapPartitionsRDD[277] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(14136) called with curMem=503713, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_64 stored as values in memory (estimated size 13.8 KB, free 730.1 MB) -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(4890) called with curMem=517849, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_64_piece0 stored as bytes in memory (estimated size 4.8 KB, free 730.1 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added broadcast_64_piece0 in memory on localhost:50092 (size: 4.8 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO SparkContext:59 - Created broadcast 64 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 481 (MapPartitionsRDD[277] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Adding task set 481.0 with 2 tasks -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 481.0 (TID 124, localhost, PROCESS_LOCAL, 2904 bytes) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 481.0 (TID 125, localhost, PROCESS_LOCAL, 2904 bytes) -2016-06-13 19:47:32 INFO Executor:59 - Running task 0.0 in stage 481.0 (TID 124) -2016-06-13 19:47:32 INFO Executor:59 - Running task 1.0 in stage 481.0 (TID 125) -2016-06-13 19:47:32 INFO CacheManager:59 - Partition rdd_273_1 not found, computing it -2016-06-13 19:47:32 INFO CacheManager:59 - Partition rdd_273_0 not found, computing it -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_260_1 locally -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_260_0 locally -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=522739, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block rdd_273_1 stored as values in memory (estimated size 2.4 KB, free 730.1 MB) -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=525187, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block rdd_273_0 stored as values in memory (estimated size 2.5 KB, free 730.1 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added rdd_273_1 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added rdd_273_0 in memory on localhost:50092 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO Executor:59 - Finished task 1.0 in stage 481.0 (TID 125). 2971 bytes result sent to driver -2016-06-13 19:47:32 INFO Executor:59 - Finished task 0.0 in stage 481.0 (TID 124). 2971 bytes result sent to driver -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 481.0 (TID 125) in 13 ms on localhost (1/2) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 481.0 (TID 124) in 14 ms on localhost (2/2) -2016-06-13 19:47:32 INFO DAGScheduler:59 - ResultStage 481 (fold at LDAOptimizer.scala:205) finished in 0.014 s -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 481.0, whose tasks have all completed, from pool -2016-06-13 19:47:32 INFO DAGScheduler:59 - Job 21 finished: fold at LDAOptimizer.scala:205, took 0.126435 s -2016-06-13 19:47:32 INFO MapPartitionsRDD:59 - Removing RDD 247 from persistence list -2016-06-13 19:47:32 INFO BlockManager:59 - Removing RDD 247 -2016-06-13 19:47:32 INFO ZippedPartitionsRDD2:59 - Removing RDD 254 from persistence list -2016-06-13 19:47:32 INFO BlockManager:59 - Removing RDD 254 -2016-06-13 19:47:32 INFO SparkContext:59 - Starting job: fold at LDAOptimizer.scala:205 -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 41 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 40 is 155 bytes -2016-06-13 19:47:32 INFO DAGScheduler:59 - Registering RDD 278 (mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Registering RDD 282 (mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Got job 22 (fold at LDAOptimizer.scala:205) with 2 output partitions -2016-06-13 19:47:32 INFO DAGScheduler:59 - Final stage: ResultStage 526(fold at LDAOptimizer.scala:205) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 487, ShuffleMapStage 509, ShuffleMapStage 503, ShuffleMapStage 495, ShuffleMapStage 525, ShuffleMapStage 489, ShuffleMapStage 517, ShuffleMapStage 511, ShuffleMapStage 482, ShuffleMapStage 497, ShuffleMapStage 519, ShuffleMapStage 483, ShuffleMapStage 505, ShuffleMapStage 491, ShuffleMapStage 521, ShuffleMapStage 513, ShuffleMapStage 507, ShuffleMapStage 499, ShuffleMapStage 493, ShuffleMapStage 485, ShuffleMapStage 515, ShuffleMapStage 501, ShuffleMapStage 523) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 525) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting ShuffleMapStage 524 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[278] at mapPartitions at VertexRDDImpl.scala:247), which has no missing parents -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(13152) called with curMem=522739, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_65 stored as values in memory (estimated size 12.8 KB, free 730.1 MB) -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(4564) called with curMem=535891, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_65_piece0 stored as bytes in memory (estimated size 4.5 KB, free 730.1 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added broadcast_65_piece0 in memory on localhost:50092 (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO SparkContext:59 - Created broadcast 65 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 524 (ReplicatedVertexView.upgrade(true, true) - shippedVerts true true (broadcast) MapPartitionsRDD[278] at mapPartitions at VertexRDDImpl.scala:247) -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Adding task set 524.0 with 2 tasks -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 524.0 (TID 126, localhost, PROCESS_LOCAL, 2893 bytes) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 524.0 (TID 127, localhost, PROCESS_LOCAL, 2893 bytes) -2016-06-13 19:47:32 INFO Executor:59 - Running task 1.0 in stage 524.0 (TID 127) -2016-06-13 19:47:32 INFO Executor:59 - Running task 0.0 in stage 524.0 (TID 126) -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_273_1 locally -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_64_piece0 on localhost:50092 in memory (size: 4.8 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO Executor:59 - Finished task 1.0 in stage 524.0 (TID 127). 2254 bytes result sent to driver -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_273_0 locally -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_63_piece0 on localhost:50092 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 524.0 (TID 127) in 62 ms on localhost (1/2) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_62_piece0 on localhost:50092 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-06-13 19:47:32 INFO Executor:59 - Finished task 0.0 in stage 524.0 (TID 126). 2254 bytes result sent to driver -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_61_piece0 on localhost:50092 in memory (size: 4.7 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 524.0 (TID 126) in 74 ms on localhost (2/2) -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 524.0, whose tasks have all completed, from pool -2016-06-13 19:47:32 INFO DAGScheduler:59 - ShuffleMapStage 524 (mapPartitions at VertexRDDImpl.scala:247) finished in 0.075 s -2016-06-13 19:47:32 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:32 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:32 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 525, ResultStage 526) -2016-06-13 19:47:32 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-06-13 19:47:32 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 525: List() -2016-06-13 19:47:32 INFO DAGScheduler:59 - Missing parents for ResultStage 526: List(ShuffleMapStage 525) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting ShuffleMapStage 525 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[282] at mapPartitions at GraphImpl.scala:235), which is now runnable -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_60_piece0 on localhost:50092 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=473392, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_66 stored as values in memory (estimated size 3.8 KB, free 730.1 MB) -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_59_piece0 on localhost:50092 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(2289) called with curMem=460636, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_66_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.1 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added broadcast_66_piece0 in memory on localhost:50092 (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO SparkContext:59 - Created broadcast 66 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 525 (GraphImpl.aggregateMessages - preAgg MapPartitionsRDD[282] at mapPartitions at GraphImpl.scala:235) -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Adding task set 525.0 with 2 tasks -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 525.0 (TID 128, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 525.0 (TID 129, localhost, PROCESS_LOCAL, 2073 bytes) -2016-06-13 19:47:32 INFO Executor:59 - Running task 1.0 in stage 525.0 (TID 129) -2016-06-13 19:47:32 INFO Executor:59 - Running task 0.0 in stage 525.0 (TID 128) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_58_piece0 on localhost:50092 in memory (size: 4.6 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_21_1 locally -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_21_0 locally -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_57_piece0 on localhost:50092 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_56_piece0 on localhost:50092 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_55_piece0 on localhost:50092 in memory (size: 4.5 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_54_piece0 on localhost:50092 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO Executor:59 - Finished task 1.0 in stage 525.0 (TID 129). 2470 bytes result sent to driver -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 525.0 (TID 129) in 24 ms on localhost (1/2) -2016-06-13 19:47:32 INFO Executor:59 - Finished task 0.0 in stage 525.0 (TID 128). 2470 bytes result sent to driver -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 525.0 (TID 128) in 27 ms on localhost (2/2) -2016-06-13 19:47:32 INFO DAGScheduler:59 - ShuffleMapStage 525 (mapPartitions at GraphImpl.scala:235) finished in 0.029 s -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 525.0, whose tasks have all completed, from pool -2016-06-13 19:47:32 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 19:47:32 INFO DAGScheduler:59 - running: Set() -2016-06-13 19:47:32 INFO DAGScheduler:59 - waiting: Set(ResultStage 526) -2016-06-13 19:47:32 INFO DAGScheduler:59 - failed: Set() -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_53_piece0 on localhost:50092 in memory (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Missing parents for ResultStage 526: List() -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting ResultStage 526 (MapPartitionsRDD[290] at values at LDAOptimizer.scala:205), which is now runnable -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_52_piece0 on localhost:50092 in memory (size: 4.4 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(14568) called with curMem=366489, maxMem=766075207 -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_67 stored as values in memory (estimated size 14.2 KB, free 730.2 MB) -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(4990) called with curMem=374875, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_67_piece0 stored as bytes in memory (estimated size 4.9 KB, free 730.2 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_51_piece0 on localhost:50092 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added broadcast_67_piece0 in memory on localhost:50092 (size: 4.9 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO SparkContext:59 - Created broadcast 67 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 526 (MapPartitionsRDD[290] at values at LDAOptimizer.scala:205) -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Adding task set 526.0 with 2 tasks -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 526.0 (TID 130, localhost, PROCESS_LOCAL, 2945 bytes) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 526.0 (TID 131, localhost, PROCESS_LOCAL, 2945 bytes) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_50_piece0 on localhost:50092 in memory (size: 4.0 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO Executor:59 - Running task 1.0 in stage 526.0 (TID 131) -2016-06-13 19:47:32 INFO Executor:59 - Running task 0.0 in stage 526.0 (TID 130) -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-06-13 19:47:32 INFO CacheManager:59 - Partition rdd_286_1 not found, computing it -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_273_1 locally -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_49_piece0 on localhost:50092 in memory (size: 4.3 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-06-13 19:47:32 INFO CacheManager:59 - Partition rdd_286_0 not found, computing it -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_273_0 locally -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 19:47:32 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_48_piece0 on localhost:50092 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(2448) called with curMem=342213, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block rdd_286_1 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=334077, maxMem=766075207 -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added rdd_286_1 in memory on localhost:50092 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO MemoryStore:59 - Block rdd_286_0 stored as values in memory (estimated size 2.5 KB, free 730.3 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added rdd_286_0 in memory on localhost:50092 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO Executor:59 - Finished task 1.0 in stage 526.0 (TID 131). 2971 bytes result sent to driver -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_47_piece0 on localhost:50092 in memory (size: 3.9 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO Executor:59 - Finished task 0.0 in stage 526.0 (TID 130). 2971 bytes result sent to driver -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 526.0 (TID 131) in 14 ms on localhost (1/2) -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 526.0 (TID 130) in 15 ms on localhost (2/2) -2016-06-13 19:47:32 INFO DAGScheduler:59 - ResultStage 526 (fold at LDAOptimizer.scala:205) finished in 0.016 s -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 526.0, whose tasks have all completed, from pool -2016-06-13 19:47:32 INFO DAGScheduler:59 - Job 22 finished: fold at LDAOptimizer.scala:205, took 0.199259 s -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_46_piece0 on localhost:50092 in memory (size: 4.2 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_45_piece0 on localhost:50092 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_44_piece0 on localhost:50092 in memory (size: 3.8 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_43_piece0 on localhost:50092 in memory (size: 4.1 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-06-13 19:47:32 INFO SparkContext:59 - Starting job: foreach at LDAFromWebSuite.scala:32 -2016-06-13 19:47:32 INFO DAGScheduler:59 - Got job 23 (foreach at LDAFromWebSuite.scala:32) with 2 output partitions -2016-06-13 19:47:32 INFO DAGScheduler:59 - Final stage: ResultStage 527(foreach at LDAFromWebSuite.scala:32) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:47:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting ResultStage 527 (MapPartitionsRDD[2] at map at LDAFromWebSuite.scala:23), which has no missing parents -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(3376) called with curMem=275111, maxMem=766075207 -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_42_piece0 on localhost:50092 in memory (size: 2.2 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_68 stored as values in memory (estimated size 3.3 KB, free 730.3 MB) -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(1967) called with curMem=278487, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_68_piece0 stored as bytes in memory (estimated size 1967.0 B, free 730.3 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Removed broadcast_41_piece0 on localhost:50092 in memory (size: 3.7 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added broadcast_68_piece0 in memory on localhost:50092 (size: 1967.0 B, free: 730.5 MB) -2016-06-13 19:47:32 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-06-13 19:47:32 INFO SparkContext:59 - Created broadcast 68 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 527 (MapPartitionsRDD[2] at map at LDAFromWebSuite.scala:23) -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Adding task set 527.0 with 2 tasks -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 527.0 (TID 132, localhost, PROCESS_LOCAL, 2197 bytes) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 527.0 (TID 133, localhost, PROCESS_LOCAL, 2197 bytes) -2016-06-13 19:47:32 INFO Executor:59 - Running task 1.0 in stage 527.0 (TID 133) -2016-06-13 19:47:32 INFO Executor:59 - Running task 0.0 in stage 527.0 (TID 132) -2016-06-13 19:47:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:0+132 -2016-06-13 19:47:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/mllibFromSpark/sample_lda_data.txt:132+132 -2016-06-13 19:47:32 INFO Executor:59 - Finished task 0.0 in stage 527.0 (TID 132). 2044 bytes result sent to driver -2016-06-13 19:47:32 INFO Executor:59 - Finished task 1.0 in stage 527.0 (TID 133). 2044 bytes result sent to driver -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 527.0 (TID 132) in 13 ms on localhost (1/2) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 527.0 (TID 133) in 12 ms on localhost (2/2) -2016-06-13 19:47:32 INFO DAGScheduler:59 - ResultStage 527 (foreach at LDAFromWebSuite.scala:32) finished in 0.014 s -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 527.0, whose tasks have all completed, from pool -2016-06-13 19:47:32 INFO DAGScheduler:59 - Job 23 finished: foreach at LDAFromWebSuite.scala:32, took 0.025989 s -2016-06-13 19:47:32 INFO SparkContext:59 - Starting job: foreach at LDAFromWebSuite.scala:34 -2016-06-13 19:47:32 INFO DAGScheduler:59 - Got job 24 (foreach at LDAFromWebSuite.scala:34) with 2 output partitions -2016-06-13 19:47:32 INFO DAGScheduler:59 - Final stage: ResultStage 528(foreach at LDAFromWebSuite.scala:34) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 19:47:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting ResultStage 528 (MapPartitionsRDD[4] at map at LDAFromWebSuite.scala:25), which has no missing parents -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(3672) called with curMem=266944, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_69 stored as values in memory (estimated size 3.6 KB, free 730.3 MB) -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(2055) called with curMem=270616, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_69_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.3 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added broadcast_69_piece0 in memory on localhost:50092 (size: 2.0 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO SparkContext:59 - Created broadcast 69 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 528 (MapPartitionsRDD[4] at map at LDAFromWebSuite.scala:25) -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Adding task set 528.0 with 2 tasks -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 528.0 (TID 134, localhost, PROCESS_LOCAL, 2307 bytes) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 528.0 (TID 135, localhost, PROCESS_LOCAL, 2307 bytes) -2016-06-13 19:47:32 INFO Executor:59 - Running task 0.0 in stage 528.0 (TID 134) -2016-06-13 19:47:32 INFO Executor:59 - Running task 1.0 in stage 528.0 (TID 135) -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 19:47:32 INFO Executor:59 - Finished task 1.0 in stage 528.0 (TID 135). 2044 bytes result sent to driver -2016-06-13 19:47:32 INFO Executor:59 - Finished task 0.0 in stage 528.0 (TID 134). 2044 bytes result sent to driver -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 528.0 (TID 135) in 4 ms on localhost (1/2) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 528.0 (TID 134) in 4 ms on localhost (2/2) -2016-06-13 19:47:32 INFO DAGScheduler:59 - ResultStage 528 (foreach at LDAFromWebSuite.scala:34) finished in 0.005 s -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 528.0, whose tasks have all completed, from pool -2016-06-13 19:47:32 INFO DAGScheduler:59 - Job 24 finished: foreach at LDAFromWebSuite.scala:34, took 0.013265 s -2016-06-13 19:47:32 INFO SparkContext:59 - Starting job: collect at LDAModel.scala:528 -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 3 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 2 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 5 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 4 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 7 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 6 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 9 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 8 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 11 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 10 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 13 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 12 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 15 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 14 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 17 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 16 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 19 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 18 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 21 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 20 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 23 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 22 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 25 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 24 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 27 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 26 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 29 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 28 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 31 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 30 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 33 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 32 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 35 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 34 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 37 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 36 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 39 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 38 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 41 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 40 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 43 is 155 bytes -2016-06-13 19:47:32 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 42 is 155 bytes -2016-06-13 19:47:32 INFO DAGScheduler:59 - Got job 25 (collect at LDAModel.scala:528) with 2 output partitions -2016-06-13 19:47:32 INFO DAGScheduler:59 - Final stage: ResultStage 573(collect at LDAModel.scala:528) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 538, ShuffleMapStage 560, ShuffleMapStage 546, ShuffleMapStage 540, ShuffleMapStage 568, ShuffleMapStage 532, ShuffleMapStage 562, ShuffleMapStage 554, ShuffleMapStage 548, ShuffleMapStage 570, ShuffleMapStage 534, ShuffleMapStage 556, ShuffleMapStage 542, ShuffleMapStage 572, ShuffleMapStage 564, ShuffleMapStage 558, ShuffleMapStage 550, ShuffleMapStage 529, ShuffleMapStage 544, ShuffleMapStage 536, ShuffleMapStage 566, ShuffleMapStage 530, ShuffleMapStage 552) -2016-06-13 19:47:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting ResultStage 573 (MapPartitionsRDD[293] at map at LDAModel.scala:526), which has no missing parents -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(14256) called with curMem=272671, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_70 stored as values in memory (estimated size 13.9 KB, free 730.3 MB) -2016-06-13 19:47:32 INFO MemoryStore:59 - ensureFreeSpace(4871) called with curMem=286927, maxMem=766075207 -2016-06-13 19:47:32 INFO MemoryStore:59 - Block broadcast_70_piece0 stored as bytes in memory (estimated size 4.8 KB, free 730.3 MB) -2016-06-13 19:47:32 INFO BlockManagerInfo:59 - Added broadcast_70_piece0 in memory on localhost:50092 (size: 4.8 KB, free: 730.5 MB) -2016-06-13 19:47:32 INFO SparkContext:59 - Created broadcast 70 from broadcast at DAGScheduler.scala:861 -2016-06-13 19:47:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 573 (MapPartitionsRDD[293] at map at LDAModel.scala:526) -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Adding task set 573.0 with 2 tasks -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 573.0 (TID 136, localhost, PROCESS_LOCAL, 2945 bytes) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 573.0 (TID 137, localhost, PROCESS_LOCAL, 2945 bytes) -2016-06-13 19:47:32 INFO Executor:59 - Running task 0.0 in stage 573.0 (TID 136) -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_286_0 locally -2016-06-13 19:47:32 INFO Executor:59 - Finished task 0.0 in stage 573.0 (TID 136). 2680 bytes result sent to driver -2016-06-13 19:47:32 INFO Executor:59 - Running task 1.0 in stage 573.0 (TID 137) -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 573.0 (TID 136) in 12 ms on localhost (1/2) -2016-06-13 19:47:32 INFO BlockManager:59 - Found block rdd_286_1 locally -2016-06-13 19:47:32 INFO Executor:59 - Finished task 1.0 in stage 573.0 (TID 137). 2612 bytes result sent to driver -2016-06-13 19:47:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 573.0 (TID 137) in 22 ms on localhost (2/2) -2016-06-13 19:47:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 573.0, whose tasks have all completed, from pool -2016-06-13 19:47:32 INFO DAGScheduler:59 - ResultStage 573 (collect at LDAModel.scala:528) finished in 0.041 s -2016-06-13 19:47:32 INFO DAGScheduler:59 - Job 25 finished: collect at LDAModel.scala:528, took 0.104657 s -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 19:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 19:47:33 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 19:47:33 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 19:47:33 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 19:47:33 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 19:47:33 INFO BlockManager:59 - BlockManager stopped -2016-06-13 19:47:33 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 19:47:33 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 19:47:33 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 19:47:33 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 19:47:33 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 19:47:33 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 19:47:33 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-4b7db2a4-66ff-443f-b499-1052d0811472 -2016-06-13 20:00:31 INFO StreamingKMeansSuite:59 - Using manual clock -2016-06-13 20:00:31 INFO StreamingKMeansSuite:59 - - -===== TEST OUTPUT FOR o.a.s.mllib.clustering.kmeans.StreamingKMeansSuite: 'accuracy for single center and equivalence to grand average' ===== - -2016-06-13 20:00:31 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:00:31 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 20:00:32 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:00:32 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:00:32 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:00:32 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:00:32 INFO Remoting:74 - Starting remoting -2016-06-13 20:00:33 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:50485] -2016-06-13 20:00:33 INFO Utils:59 - Successfully started service 'sparkDriver' on port 50485. -2016-06-13 20:00:33 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:00:33 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:00:33 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-80fedb57-1add-4b52-bff0-db7fb28a9d34 -2016-06-13 20:00:33 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:00:33 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-f8eb2236-34b2-4105-aaff-6c04ac3f507c\httpd-6b1a7419-fa0b-43b0-a469-24a201b5fdb0 -2016-06-13 20:00:33 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:00:33 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:00:33 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:50486 -2016-06-13 20:00:33 INFO Utils:59 - Successfully started service 'HTTP file server' on port 50486. -2016-06-13 20:00:33 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:00:33 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:00:33 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 20:00:33 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 20:00:33 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 20:00:33 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:00:33 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:00:34 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 50497. -2016-06-13 20:00:34 INFO NettyBlockTransferService:59 - Server created on 50497 -2016-06-13 20:00:34 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:00:34 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:50497 with 730.6 MB RAM, BlockManagerId(driver, localhost, 50497) -2016-06-13 20:00:34 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:00:35 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 20:00:35 INFO StreamingKMeansSuite:59 - numBatches = 10, numExpectedOutput = 10 -2016-06-13 20:00:36 INFO ForEachDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:00:36 INFO TestInputStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:00:36 INFO TestInputStream:59 - Slide time = 1000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:00:36 INFO TestInputStream:59 - Checkpoint interval = null -2016-06-13 20:00:36 INFO TestInputStream:59 - Remember duration = 1000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Initialized and validated org.apache.spark.streaming.TestInputStream@5324396a -2016-06-13 20:00:36 INFO ForEachDStream:59 - Slide time = 1000 ms -2016-06-13 20:00:36 INFO ForEachDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:00:36 INFO ForEachDStream:59 - Checkpoint interval = null -2016-06-13 20:00:36 INFO ForEachDStream:59 - Remember duration = 1000 ms -2016-06-13 20:00:36 INFO ForEachDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.ForEachDStream@4c8bb3e3 -2016-06-13 20:00:36 INFO TestOutputStreamWithPartitions:59 - metadataCleanupDelay = -1 -2016-06-13 20:00:36 INFO MappedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:00:36 INFO ShuffledDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:00:36 INFO TransformedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:00:36 INFO MappedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:00:36 INFO TestInputStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:00:36 INFO TestInputStream:59 - Slide time = 1000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:00:36 INFO TestInputStream:59 - Checkpoint interval = null -2016-06-13 20:00:36 INFO TestInputStream:59 - Remember duration = 1000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Initialized and validated org.apache.spark.streaming.TestInputStream@5324396a -2016-06-13 20:00:36 INFO MappedDStream:59 - Slide time = 1000 ms -2016-06-13 20:00:36 INFO MappedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:00:36 INFO MappedDStream:59 - Checkpoint interval = null -2016-06-13 20:00:36 INFO MappedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:00:36 INFO MappedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@7d6331fc -2016-06-13 20:00:36 INFO TransformedDStream:59 - Slide time = 1000 ms -2016-06-13 20:00:36 INFO TransformedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:00:36 INFO TransformedDStream:59 - Checkpoint interval = null -2016-06-13 20:00:36 INFO TransformedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:00:36 INFO TransformedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.TransformedDStream@12826436 -2016-06-13 20:00:36 INFO ShuffledDStream:59 - Slide time = 1000 ms -2016-06-13 20:00:36 INFO ShuffledDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:00:36 INFO ShuffledDStream:59 - Checkpoint interval = null -2016-06-13 20:00:36 INFO ShuffledDStream:59 - Remember duration = 1000 ms -2016-06-13 20:00:36 INFO ShuffledDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.ShuffledDStream@35896b6 -2016-06-13 20:00:36 INFO MappedDStream:59 - Slide time = 1000 ms -2016-06-13 20:00:36 INFO MappedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:00:36 INFO MappedDStream:59 - Checkpoint interval = null -2016-06-13 20:00:36 INFO MappedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:00:36 INFO MappedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@1e5c35ed -2016-06-13 20:00:36 INFO TestOutputStreamWithPartitions:59 - Slide time = 1000 ms -2016-06-13 20:00:36 INFO TestOutputStreamWithPartitions:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:00:36 INFO TestOutputStreamWithPartitions:59 - Checkpoint interval = null -2016-06-13 20:00:36 INFO TestOutputStreamWithPartitions:59 - Remember duration = 1000 ms -2016-06-13 20:00:36 INFO TestOutputStreamWithPartitions:59 - Initialized and validated org.apache.spark.streaming.TestOutputStreamWithPartitions@27349480 -2016-06-13 20:00:36 INFO RecurringTimer:59 - Started timer for JobGenerator at time 1000 -2016-06-13 20:00:36 INFO JobGenerator:59 - Started JobGenerator at 1000 ms -2016-06-13 20:00:36 INFO JobScheduler:59 - Started JobScheduler -2016-06-13 20:00:36 INFO StreamingContext:59 - StreamingContext started -2016-06-13 20:00:36 INFO StreamingKMeansSuite:59 - Manual clock before advancing = 0 -2016-06-13 20:00:36 INFO StreamingKMeansSuite:59 - Manual clock after advancing = 10000 -2016-06-13 20:00:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:36 INFO TestInputStream:59 - Computing RDD for time 1000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Created RDD 0 with Vector([-0.35685422167583125,0.9995288307555166,-1.5877809386448758,1.210845548490831,0.56222524157234], [-0.45216537133396134,0.8040574757345051,-1.5589888701250443,1.115603017032467,0.344649973028188], [-0.3801675945145341,1.0697589561731022,-1.6486682116186815,1.2790268654063703,0.5116371486214886], [-0.545300880562902,1.1093057253978866,-1.7582382472264182,1.211400006292075,0.2932044815946008], [-0.3995483137357882,0.929119621864747,-1.7401380541085598,1.2990457426846032,0.299090686584923], [-0.5964001160322344,1.0378164371074625,-1.553411818459791,1.2193968564632125,0.4492703031123452], [-0.33949398265706626,1.1369769008642154,-1.7816495259375202,1.2857769527411067,0.5642853362323552], [-0.4522936770981281,1.090819158309221,-1.4785463285656288,1.2734882859357213,0.31864385887835545], [-0.4895260213585818,1.024567290792244,-1.7744155605537828,1.4515329700677304,0.39720325477885093], [-0.5479321442182269,0.8926951810306638,-1.617388979581139,1.3192738326981768,0.3243774353000105], [-0.5468296230191492,0.9034725811394896,-1.8269524875493084,1.161899613497355,0.22607191229030854], [-0.4344367225435325,0.9054178856043291,-1.6658399540877864,1.13578899284944,0.46511292492021983], [-0.5914600674966851,0.9727747336030947,-1.7411162798640765,1.3902196062020102,0.43158411906136107], [-0.5644425684165094,0.996354296088912,-1.817848647277835,1.3768030733201289,0.28609432985591315], [-0.44148386701129744,1.025737906243747,-1.7073695034902865,1.2539087121413848,0.349242056080172], [-0.5287113084613145,0.8475916475163896,-1.8166057479945195,1.238631411179877,0.36922519791770014], [-0.2540595910515851,0.9698131032346381,-1.5866552625154318,1.2179915976157372,0.3240398576394729], [-0.5026644816450073,1.109581665015373,-1.7151161774591086,1.3901513268104455,0.39042855132933163], [-0.4131172722481001,1.0096927045418442,-1.811286466550916,1.2719080737538961,0.28293757989913904], [-0.4515790075180954,1.0935474561719332,-1.7896107864561164,1.3203383417885988,0.2994362800834273], [-0.4171607573128071,1.0818260184642114,-1.8219543038308885,1.2496462030096327,0.4964658371791977], [-0.45586902589910655,1.0378742745036647,-1.7000458648640797,1.446592332373813,0.348566870310007], [-0.3532087778209343,0.8044437973592731,-1.7162629066191757,1.3339281879504057,0.17946188844185906], [-0.5154063008996834,0.9595656827532482,-1.6519653379002484,1.322654750805598,0.3008929692388781], [-0.3967887546418169,0.7908879805460386,-1.7242582851383725,1.2141584886700174,0.36827019559433166], [-0.48352710482779404,1.0255449792765485,-1.7171397454598174,1.284877805985566,0.437232715780399], [-0.40324856443707463,1.020485534999759,-1.6517518343613002,1.2864020019792224,0.3984832850799499], [-0.6517069948565133,0.969461018860354,-1.684015510299471,1.2027906385718596,0.2014860155598368], [-0.6148278408211463,0.9795135452912322,-1.8642733722424907,1.2720240024566645,0.39785426587083933], [-0.6409674819545198,1.1778584543507387,-1.8094969072333307,1.24145841048891,0.1624672183638662], [-0.6368691555002903,0.8170131462860201,-1.6827833832635621,1.3031179618920097,0.31628673987022604], [-0.43096648399184834,0.9655746876954024,-1.7845188100637381,1.4052721272088269,0.5845842279495], [-0.4442125841627056,1.0257872008493307,-1.7242589434969455,1.2745428690859977,0.39998857867859966], [-0.4527127515828855,0.9000469054331385,-1.7497092961121075,1.2451107418571954,0.4344380036579415], [-0.6030914309562295,0.9699928952430515,-1.712604726536663,1.38261404269885,0.48344898725768926], [-0.34976726689041593,0.9355870894961762,-1.5766155567170825,1.3590936489794139,0.35690680463154484], [-0.575628267406687,0.9538063357613722,-1.6445003616008003,1.2104804829930789,0.3911315521177748], [-0.5582822637309535,0.9339694428739754,-1.6511037013459962,1.4401460976648193,0.3607766390547849], [-0.5468811641579405,0.997556943752324,-1.778985900216746,1.2440321529105447,0.4204585578447196], [-0.48994207115736743,1.0755700976056979,-1.5322042869816734,1.2132674597398667,0.5961642366275318], [-0.5345824744371771,0.7659168558371742,-1.7009697565551642,1.2200491259133113,0.24064266183571642], [-0.4425411108434083,0.9636579302553172,-1.6769715787345627,1.2060504970822528,0.1832578105698996], [-0.2803908635114948,0.9814260387904405,-1.6744903280757537,1.438894231115956,0.5307857877828275], [-0.4646639723438268,0.9422153316765177,-1.8936280948227913,1.1755014401606392,0.3269897050872188], [-0.5794967530942349,1.008452319256215,-1.8095212163297067,1.3053459835247618,0.3862084672749757], [-0.36564413314560745,0.9550404460199052,-1.671332555981891,1.2387784773127055,0.29524181369582553], [-0.4947387784852314,0.8968969114403345,-1.5032027608518188,1.263293789437244,0.41872133210307433], [-0.5901163249097482,0.9195890740589755,-1.76975579729397,1.2233696400583831,0.3403182341587175], [-0.7175598778778954,0.8837946345395844,-1.8329343739105675,1.3562350855120264,0.30734388952604197], [-0.4376962364418643,1.0046436887819825,-1.5427982731479972,1.1077094728701697,0.30461693506090337]) -2016-06-13 20:00:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:36 INFO JobScheduler:59 - Added jobs for time 1000 ms -2016-06-13 20:00:36 INFO JobScheduler:59 - Starting job streaming job 1000 ms.0 from job set of time 1000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Computing RDD for time 2000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Created RDD 6 with Vector([-0.5385984930004446,0.9989931115946847,-1.745310682734983,1.2763556288677276,0.41555544930746574], [-0.5140757026211953,0.9616396809310985,-1.6092748223737894,1.3080875431688948,0.3958240822167327], [-0.3317121400990889,1.1311521924692884,-1.5940394948396088,1.235663715627094,0.5352186835894586], [-0.3566003302425871,0.8649446433306678,-1.6281158497086519,1.0841327390682376,0.33108928300382867], [-0.4898654567486354,1.0095506134582393,-1.7900184464939166,1.4376638486285889,0.5476994415325478], [-0.5067718408631526,0.8330942372560937,-1.5447521454199076,1.3956201717944832,0.40502500771937533], [-0.45899690569395396,1.0114199686087044,-1.6793704227956527,1.2264029484806054,0.3617800498770533], [-0.42389468031403765,0.9744019756374155,-1.4339024359998267,1.3506788012287876,0.5071391796420295], [-0.3445608603995446,0.8999444931201762,-1.8295489983610813,1.3813011802363127,0.4406110935124823], [-0.4720580057434039,0.9616388382715773,-1.7885815681274932,1.2231064430079939,0.49611588805959794], [-0.6877856894979741,0.9509033529307679,-1.7111111174594673,1.1650825853084499,0.36541463301831684], [-0.41906769746785616,0.9643844813018809,-1.7088669498454399,1.2743442947780539,0.37857583863641653], [-0.45604703784103723,0.9704467329255722,-1.6908561828878124,1.2661005251088149,0.34761517755721416], [-0.45216413072759976,1.125638260528155,-1.8672240017957615,1.388348280279974,0.41289821746105937], [-0.6041998663111035,0.9518745991902821,-1.7482553616491074,1.3027968384522153,0.33003148862622383], [-0.7645915689875407,0.9231306641016475,-1.5908706609397762,1.0416797405804958,0.3435391589642622], [-0.5511129470810242,0.9789516647349409,-1.5896352104287992,1.2985407772348172,0.3074739718251722], [-0.3060111421519473,0.9693226097887153,-1.6503499878201646,1.2050435546957032,0.20529101796457794], [-0.41814813350738833,0.9065345647328998,-1.7369684308945867,1.3095490239413528,0.3931707179477215], [-0.7452070526618801,1.010695258373889,-1.6799990639878284,1.351913921189404,0.48041731347418104], [-0.19474072480894772,0.8574730260551575,-1.7114945513022097,1.2484211088395596,0.35791459113273333], [-0.679333385903053,1.136043445571062,-1.6819398713814546,1.2914383599572574,0.4846586154592743], [-0.38963314954900546,1.0499014443531718,-1.7075189569343263,1.3966853721193921,0.4922463621111953], [-0.4771733790096556,0.8615507710844875,-1.7824679274745288,1.1550649309651535,0.4634518570903764], [-0.46851141840829336,0.9490804068188984,-1.6654371185103947,1.1748885058064462,0.362811152170017], [-0.2486779458724137,1.0146748639649252,-1.8129854069448728,1.3198830969564963,0.3848127258680788], [-0.37278156677721497,0.9026389048304428,-1.6338760097656873,1.2146194636725334,0.305407696426058], [-0.5109164214840503,1.1025203353037651,-1.5805383770959172,1.3659800632138837,0.2507490333376023], [-0.42658774618748607,0.9583040463281658,-1.535401873083165,1.2270967800953845,0.30931926334482973], [-0.5761089637758711,0.9608023900436299,-1.6773446541045054,1.3896056325873751,0.3003385242281313], [-0.333657352857868,0.9284640162518879,-1.6506283984425263,1.4233119402331722,0.4817086358456392], [-0.4131260123218083,0.9257004348959492,-1.6385111064595121,1.2553085674173774,0.35994561408421055], [-0.41619483777131644,0.9791552402269297,-1.6621579631275758,1.1639596596963502,0.3356350519347727], [-0.42150886121352676,0.9497809597217938,-1.4493400602428224,1.2037323844449976,0.28412005647965366], [-0.4036704221590921,0.8682473056100086,-1.4499044130725358,1.4367312949083602,0.29801296100779273], [-0.7347169576181178,0.8472443819718873,-1.6912964247755202,1.1098267823656842,0.5128488081575056], [-0.4599723079910277,1.0313773063743503,-1.6693647567489862,1.1183374503284873,0.3065899617042307], [-0.5155727886799809,0.9793287424722477,-1.81766244647388,1.264051537384318,0.38002468301378506], [-0.461809330593855,0.94300958630738,-1.7083227759840551,1.2779916838408791,0.5269758739704866], [-0.6518981010977536,0.8745407308399856,-1.5963574332371322,1.2679054180945675,0.4099968212180263], [-0.46480686752956457,0.9217771428753339,-1.5266876398223685,1.1382426745675795,0.4778996312070509], [-0.4852820635337406,0.8218636910073019,-1.7767449598018135,1.1371130292343117,0.33381426065377506], [-0.41472873571993296,1.1622114863170214,-1.464996886657917,1.3123782230466363,0.2979393535788235], [-0.38817680425075585,1.0268858467994189,-1.5893492622912915,1.2621731986065745,0.435168936860357], [-0.4138872105825818,1.0639679744840822,-1.6793705616475731,1.0262313606697269,0.26542788201223744], [-0.32689002022123786,0.9411685364764196,-1.6079547828400877,1.222649045231373,0.44202772328420886], [-0.4534191990015764,0.9019382096024549,-1.6042217752367725,1.313476911051435,0.3118812623960018], [-0.3642257288812764,1.0568954661740726,-1.6408419112357935,1.3879191143662102,0.43428969659022393], [-0.4436033491575346,0.951120701402773,-1.7396961236381308,1.0696513684387006,0.1983721283216616], [-0.4195531379041837,0.9339655248016878,-1.6369085623469266,1.2113780206799198,0.32646374286948043]) -2016-06-13 20:00:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:36 INFO JobScheduler:59 - Added jobs for time 2000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Computing RDD for time 3000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Created RDD 13 with Vector([-0.3676131534931217,0.7205292433695121,-1.6148690586461516,1.318613134832133,0.2894000707552024], [-0.63227225792052,1.0480160164806966,-1.7186123125513257,1.3123133303248091,0.4835019323786134], [-0.5130669794355894,1.0744409567693907,-1.5423474727562314,1.4075951590487494,0.34400766625670576], [-0.6214787708363003,0.9941757670924664,-1.7059517928152113,1.1668792946674045,0.5351190865534026], [-0.5610084057337338,0.7736203857280138,-1.8315479385898956,1.3209260735925115,0.28582121198324356], [-0.37376999266386424,0.8253313331048502,-1.7924436246324382,1.3068506619750972,0.2325763177837035], [-0.5087037004414274,0.9607866442106467,-1.812291073591455,1.284458726328898,0.5328516211621773], [-0.3727437082846784,1.028767182687076,-1.7448007272489838,1.0984753757565202,0.3541211398247694], [-0.3814571980697206,1.0819060589675644,-1.733976127510556,1.2515969788209256,0.3640947425626813], [-0.38693583578076807,0.9054649043220095,-1.601255604158471,1.1869423507487187,0.27285330937505237], [-0.453879517976784,0.8265450932182703,-1.696264660015251,1.1219045234195482,0.3528336963815211], [-0.6440396833150053,0.9147072341447406,-1.574634779931242,1.2807375427757628,0.3400101945021918], [-0.5574376128841423,0.9786892277605028,-1.6539643457077906,1.2987195892477945,0.22091568760798275], [-0.3219564568616685,0.7344099605460779,-1.659291949575684,1.3277750279213558,0.23553171253321456], [-0.42287205017678026,0.8870628278654091,-1.7860892614211916,1.3638413188637932,0.35992654268460605], [-0.5114214572125407,1.0739488843585667,-1.803218590871721,1.2580620741259871,0.42523338095005675], [-0.49124237559267386,0.9657816736108445,-1.6156494419042913,1.3063855681095915,0.54155197550267], [-0.3951919316276779,1.144136446459301,-1.6606072772152598,1.4114586343482727,0.3224073809014021], [-0.439470602003815,1.0070997021461388,-1.4975209021956797,1.2890203902600919,0.22540217793727096], [-0.6679727959214776,0.999190248905179,-1.635890493454145,1.2938379558735829,0.20736771134970797], [-0.4118317638697319,0.9069236714993869,-1.7766219570089812,1.2254241772422947,0.3818774093786991], [-0.5019285203332016,1.071358441302261,-1.8681705848296024,1.3543938347082431,0.3694102461708769], [-0.43086686887184694,0.9871654497649167,-1.592715663423245,1.1292659913487786,0.24090456644129066], [-0.6554222411786117,0.9775977489456485,-1.7739871948427586,1.3756970876324615,0.5345682469155936], [-0.606670169993634,1.0414671201540915,-1.5816535152099802,1.2849832898961777,0.3443771799618759], [-0.6330347409363863,0.7614040653882075,-1.6420712005053097,1.344034328413472,0.5019032888808933], [-0.5268140556262974,1.1016458967835785,-1.581552701703922,1.1842999784487596,0.3683353834757425], [-0.49900894372274485,1.0645644739185587,-1.5355753177851752,1.2694091849907172,0.33468156327241705], [-0.31358398021960576,0.8302665811856854,-1.5422704772273437,1.1506707176948,0.37999408831157194], [-0.557673046622542,1.0913399597085338,-1.5871539417007638,1.0478537277843192,0.31421916981662246], [-0.4100320104849945,1.0606699393377843,-1.764930551709285,1.3173125309085043,0.32112514544554777], [-0.2578093453260759,0.9604640936063775,-1.6498739501347253,1.251662741314794,0.5352359752706277], [-0.5830353039132012,0.9147683830431232,-1.747368848835465,1.1923888734167842,0.4792523557272257], [-0.3091820573873779,1.0228642342621612,-1.472857793608509,1.165376819972311,0.3758448993415674], [-0.371190450361484,0.855956409330075,-1.82995473329898,1.3390818705810552,0.37113293274981823], [-0.5539807557276546,0.9677998945369973,-1.6350804443189515,1.3419255190805948,0.44899955987573437], [-0.5376287808107136,0.8413279828562651,-1.6762865574706214,1.19949852014792,0.5047576712678563], [-0.6003249423212255,0.9906964773703084,-1.5370089708944512,1.3072479529851824,0.430528835393197], [-0.4175368422724912,0.8187481538877078,-1.7996629160543502,1.1879999792808238,0.473510867372402], [-0.4828257087231799,1.0612823363596418,-1.6425767546040255,1.2270450796716112,0.39748339044894354], [-0.6332333955565639,0.9519761064158869,-1.5479445620477186,1.308465915443944,0.40088093604627134], [-0.5228584674134156,0.8768992895766171,-1.8336272924164618,1.1499724885170817,0.48668761341650246], [-0.36341836902693225,0.922366126760111,-1.4327493627862522,1.255336229334989,0.29637579738564857], [-0.45752242402693893,0.881182369510346,-1.8075367810499625,1.1836209711736738,0.5240597883932867], [-0.45127422400043055,0.7875661584033756,-1.9374943002165643,1.3022768816466794,0.3865261883663946], [-0.3385546242930144,0.8796112795269294,-1.6285471480854194,1.177302672296305,0.3611750358009876], [-0.3866867752880084,0.9445769802243262,-1.63589592788523,1.3115437594462822,0.30025415854590604], [-0.35087047853321573,0.8048561114260069,-1.6540296199948705,1.1069576251024358,0.3739108460295009], [-0.6118021930146711,1.0836229312504084,-1.6965417516081935,1.3814661848764345,0.5544639396063888], [-0.32579850447352154,0.9627503718472757,-1.7118868486570202,1.3378056459968657,0.26239820187294716]) -2016-06-13 20:00:36 INFO JobScheduler:59 - Added jobs for time 3000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Computing RDD for time 4000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Created RDD 19 with Vector([-0.4302807096972894,0.8599904088505619,-1.74473044235192,1.4559107913903748,0.3868065070182458], [-0.4809066717988696,1.1355083469098752,-1.568604721573235,1.1620770786476713,0.5052691040344791], [-0.37948367495671453,0.9612314328384234,-1.7288918432476479,1.1900277125124938,0.584148529955661], [-0.4830118079148656,0.767521114064722,-1.626523514538083,1.2344387914004133,0.2949434953618521], [-0.45982168221474795,0.9610478654584301,-1.4181098052958898,1.1960502956938037,0.35754207882888805], [-0.5403734624385554,1.0961985914256847,-1.4824794478647174,1.1331480033700656,0.4391107899439444], [-0.4078934785304544,0.9024846949827434,-1.772449379651882,1.1891755675558509,0.26886721197216756], [-0.4850713563393695,0.9854430300369302,-1.6392713106609738,1.1640485301086894,0.4058728213551233], [-0.5687863647866291,0.8969696803234791,-1.694332984322472,1.263598085578097,0.4916946976192132], [-0.4852057062952629,0.9746523770852169,-1.7152410694648534,1.290070394136892,0.15734923086759972], [-0.6042639508376677,0.9587258437320635,-1.796821218335353,1.2415150801911263,0.5036349732580911], [-0.5634160393957598,1.000927422915143,-1.7677443118596388,1.1056602828501068,0.3662560181172587], [-0.42260218853055576,1.038402090735988,-1.623454196568306,1.258759507363921,0.35817719144396376], [-0.6935865876752814,0.9767485891098648,-1.6119749119230544,1.3241871653009805,0.45959708474651645], [-0.34982681977824615,1.0697929536346964,-1.6430495277049175,1.5160135223968076,0.3720168274191172], [-0.4965183720272103,0.8867675212850072,-1.6326283681810188,1.1014099972707194,0.3464777829375711], [-0.4429523196454975,0.9569586297887442,-1.6975710388611154,1.3240706523955248,0.32888337316923105], [-0.6110986163387908,0.8513837967033338,-1.863122720264613,1.1805944382455067,0.36642329779978594], [-0.5770973903898762,0.908469634420286,-1.5969552518736576,1.2639442528889475,0.30334073173665643], [-0.6677676499752639,0.9993894428543435,-1.7484246181717757,1.3024629995033956,0.5254188589994735], [-0.43541597877213595,0.9518339300904892,-1.586032231723223,1.2859374872876397,0.3728538289653893], [-0.3981355023125287,1.0914065768869097,-1.604054951142783,1.5597686669581905,0.3200687403051776], [-0.3526167305034049,0.8582438505591754,-1.795079875034891,1.291753863011317,0.5193327226327783], [-0.5322224604485944,0.9225091285363366,-1.5406535924270095,1.1740734779955952,0.3985082190639408], [-0.459298613922177,0.8499917798311437,-1.7381604837279123,1.3363117012403978,0.39372367283850573], [-0.5029856763543168,0.9563691025906603,-1.6323146191672946,1.1564017424707789,0.2992277161858537], [-0.4702645909815308,0.9857893178047549,-1.6146159696178013,1.2941994476820615,0.29510677438190996], [-0.5010373585673528,1.0381084928077213,-1.6592946006629066,1.2275293781382213,0.6215806870134133], [-0.34251104673790145,1.0920152105960284,-1.4702129754795525,1.3052389503227437,0.5142842632664187], [-0.6014879705809496,1.0573110467882678,-1.6994508095500127,1.2345579924787826,0.27260709575033387], [-0.374393054126274,0.8520958347461538,-1.6227534538993043,1.400050919842349,0.3749155714488258], [-0.5537306132792297,1.0237199970470363,-1.5996513021748293,1.3277912515047763,0.3298858295939718], [-0.25019038644507985,0.8933568126093641,-1.6180540579171452,1.2514548524704747,0.4485167299028402], [-0.4083941282128968,0.9821808764315939,-1.7710803507794117,1.22246902401698,0.39616717157783654], [-0.6617567254520433,1.135205204672421,-1.7404704566578575,1.2634534028343156,0.48631393245586374], [-0.5102944111175142,0.7749069961072899,-1.3746153795885292,1.1761695768038691,0.44519535098184526], [-0.5273802259607313,1.0293844883271042,-1.8327600208787889,1.3032607095759106,0.4810318980867238], [-0.43680250540361804,0.818212719795147,-1.6397291545525832,1.2164204417976607,0.36911234859465697], [-0.44511271834694277,1.0039279649012336,-1.736482991573775,1.1407721179472239,0.32026695790267784], [-0.461571293170463,0.9623234284494812,-1.608651111923442,1.3054108488252942,0.2808744029932673], [-0.44626644352032524,0.9385912062499339,-1.5784088711897624,1.2996894478753678,0.3367841237735643], [-0.3458002275610911,1.0002670117410046,-1.556743986487141,1.220789398824709,0.3047413342602086], [-0.7006173067436341,1.0226530560056757,-1.7453818996615391,1.2468257997531524,0.43730757861422237], [-0.4921877365188532,0.9365903329931373,-1.6954478906187997,1.252660426269991,0.411305000260049], [-0.5866727516355298,0.8540617355784135,-1.7495419546675461,1.1992666379768737,0.5624798329629086], [-0.4552502733367031,0.9291860448712651,-1.5908193525700438,1.166718738832733,0.3281256456654895], [-0.3426613548769225,0.7538960431270546,-1.585524926597853,1.478740164978157,0.251131880834575], [-0.4680826818575833,1.1112444944704614,-1.6587653968723466,1.2870287092936261,0.4050920708933554], [-0.5624024717932173,0.880862871462712,-1.6303493150007178,1.1721463768303386,0.5696913393311782], [-0.48266079841446297,0.8916088275950524,-1.5393517554424276,1.1897790454674049,0.4411535715241579]) -2016-06-13 20:00:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:36 INFO JobScheduler:59 - Added jobs for time 4000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Computing RDD for time 5000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Created RDD 25 with Vector([-0.3816421005537525,1.0617035428188992,-1.7399461177422264,1.332980235806715,0.2916494841747297], [-0.6583584440016957,0.9555891409969888,-1.5454326525102315,1.1741091058996926,0.305220337835565], [-0.2989177604038451,0.8701688291879833,-1.8804582484676076,1.2986162711936382,0.35958131004690164], [-0.4939772883087695,0.9969220817446861,-1.5980382534286326,1.2969252872376784,0.37976918116852443], [-0.35668565131454827,0.8078147339434413,-1.657193160993533,1.359095599717262,0.31192105888186633], [-0.5450865703006346,0.8750391278919141,-1.7453306059026725,1.3721329076982913,0.42237633518141116], [-0.4917580165015238,0.9512146417235625,-1.9326523267650781,1.4266827266379667,0.3225713722197544], [-0.4048718474607664,1.0240256182146898,-1.6774401089659943,1.2669846165410654,0.33732641244233585], [-0.47579982131353826,0.7947369494624074,-1.7655419119652316,1.082577012553155,0.35401987740372537], [-0.35538929020910864,0.965963799724578,-1.7529404087985827,1.2055043918858341,0.3342486683177578], [-0.7014541659835856,1.034642549879647,-1.5544540159626516,1.3033886639865033,0.4054677613509847], [-0.43610110990224626,0.967077951705683,-1.5262368892289788,1.2639835242537958,0.3181179758885862], [-0.3425524324739105,0.9894095680520077,-1.7777745234498326,1.3091660388427224,0.331841648562285], [-0.4878205400579338,0.8560103963724833,-1.7352326850283464,1.3664274889095067,0.3984111315135807], [-0.7121859062829006,0.9130040407010276,-1.7874802517046422,1.1874648639293004,0.45017886399842666], [-0.696643078139105,1.1467663562988957,-1.7296410056290408,1.296144064682525,0.5048605187772839], [-0.4011230886898419,0.7361859073107719,-1.5317733987729902,1.2701183330425934,0.46062351610078545], [-0.45715694326129586,0.9190057703920946,-1.6904193500197242,1.3271044644288563,0.4842403797714084], [-0.5095249256026796,1.0659631362797184,-1.4402626928522713,1.3379987286264212,0.2673039786797317], [-0.4167094825032959,1.1091402127599272,-1.705921687753808,1.1183411370664753,0.48305727426817385], [-0.2924464534867928,0.8213969946455459,-1.662853823551024,1.3355427142685694,0.3770025002372693], [-0.4444725119583886,1.1046045679759258,-1.456164663000235,1.227847061522124,0.5255642346814823], [-0.4048329834921757,0.9788094275209757,-1.539852245269681,1.180160780953293,0.6787253334538687], [-0.38022990049474503,1.0144858953551337,-1.8072904612189131,1.3419776986257266,0.4041658441938728], [-0.3763555278351052,0.8998086161617835,-1.705061140148122,1.3495005676173746,0.5191631674639813], [-0.5540229100290982,1.0669438671830174,-1.636684794385797,1.3459012222977915,0.4181134505939225], [-0.45042550789662783,1.113240852339999,-1.486260571622101,1.2719922983443672,0.4102349177516307], [-0.5546179681493353,0.9938406825700987,-1.7084301934096207,1.3880541591389144,0.29690506797892435], [-0.5006405047598609,1.090292575906863,-1.7104271793907637,1.3062089077771892,0.47556018075456363], [-0.36520205819569046,0.9201397468605166,-1.7602021590969437,1.3876082296931265,0.2308718010458195], [-0.4938168714972593,1.0964903470900649,-1.7260349751744855,1.2048249569339422,0.40946925790527117], [-0.5092411838541296,0.9882559808204627,-1.7212908932580275,1.1127850487389395,0.3940121165600333], [-0.5882412062939182,1.1156935762589453,-1.8131619026625396,1.4146422480126442,0.31361816553861704], [-0.46990162437316874,0.9908156946435108,-1.748827835513362,1.3957588501434537,0.5512480921903505], [-0.5405868391855275,0.929247220281523,-1.5894550246414005,1.271255300937136,0.46529614772048844], [-0.46376208782353645,0.8174067828364345,-1.711422288030576,1.150388298327176,0.36268778857829526], [-0.4617415170232849,1.1203248336099854,-1.9346820049945674,1.1466858815338403,0.318614186382022], [-0.5200553635156823,1.1206383922863123,-1.721393160577411,1.2564342218266722,0.39290287510750027], [-0.45474634000888675,0.9538310968737261,-1.5572305892194418,1.2293659662686987,0.3951581444726625], [-0.31024997329419923,0.9626097763501761,-1.5774105560033171,1.3323705378196256,0.28213579823419327], [-0.6425546702182562,1.0413861558054223,-1.6696029184789263,1.3297020370998587,0.4667426127538354], [-0.5088483797878551,1.2170009632967245,-1.7143214767417212,1.2915402658283741,0.27813229259982863], [-0.5138048643277845,1.0537687232096613,-1.5306968187748822,1.2020601237200264,0.27213168784492703], [-0.5375432544160766,1.0418784698921373,-1.501800504998784,1.217708599285121,0.6443267111319482], [-0.5784571285851817,0.9518620188834632,-1.656094482882889,1.2785669143560554,0.34103467230048085], [-0.39057405488844044,0.995936983645342,-1.6323671359433234,1.271155031008547,0.3558239098317178], [-0.3682240577265206,0.8711375369867465,-1.6625627384453794,1.1969170451325948,0.47233733398400285], [-0.446008720311628,0.8976699326654971,-1.5289286019943875,1.2244855320582673,0.2984778353929738], [-0.5102538395816835,0.9986885141123109,-1.6414035100650266,1.2420020420446427,0.36032207135032196], [-0.6350165259371148,1.0900186871655364,-1.7907186442371252,1.4304360117451347,0.35387005412887984]) -2016-06-13 20:00:36 INFO JobScheduler:59 - Added jobs for time 5000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Computing RDD for time 6000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Created RDD 31 with Vector([-0.5768797964115748,1.028505413551091,-1.7521414076692798,1.410931775189879,0.30527264662469095], [-0.40591177777424625,0.8661878776717759,-1.7310200236861213,1.3594055750354543,0.3299548132445285], [-0.6383561683498007,0.9259033203935216,-1.6881320011035645,1.3172994752602565,0.3214908371039436], [-0.624062828319168,1.1219523756569185,-1.5603263506923595,1.228964619537528,0.35487057209905903], [-0.4734064808910865,0.9649955190976767,-1.6068693471541338,1.3196307767213002,0.37116726129033706], [-0.6101841770121739,1.059654359501757,-1.8099667264246553,1.3759685981009824,0.5438261473052933], [-0.18850356552447423,1.1440264277756391,-1.724865192544724,1.1359044353122627,0.31757393897273056], [-0.41254200355821263,0.8607204799101216,-1.616620448535661,1.3427363220425623,0.4109728277211949], [-0.393734447592108,1.057690953958827,-1.6695747980566427,1.327034390345423,0.13819106599587025], [-0.32090427274943434,1.0651687359751123,-1.6980368198411557,1.4017902860954718,0.36022562482340503], [-0.6053061980873911,1.0041632375635317,-1.7820663797212062,1.2269162643461793,0.4840114520476853], [-0.27268808986968307,1.013170380627259,-1.6005981535949472,1.4382064950763866,0.3892224005152082], [-0.535422550377724,0.8375173015602825,-1.661788348017347,1.3227481832299182,0.311533159171815], [-0.4009368458319039,0.9897832871295871,-1.6696688300152116,1.4424037410980737,0.32734398317968916], [-0.6825087101818776,0.9056073621397804,-1.659854864696752,1.2759644672519115,0.43884955280528926], [-0.43179250129551383,0.7465728726494555,-1.5636729803345486,1.3457117728614754,0.5721971492726551], [-0.526304418227463,1.0501002903902692,-1.5968858098432406,1.2982785754583357,0.37494021304786934], [-0.4320225453412649,1.1715904807145918,-1.5178148547272063,1.348200261386646,0.40779395798035983], [-0.42221995667102147,0.87623418086733,-1.73837906100808,1.4183919259975142,0.33228674704284467], [-0.5629520941212462,0.944212875923361,-1.5546071736198348,1.4519582067498829,0.262200899044119], [-0.30845271114048184,0.9681639591372486,-1.688842111249905,1.3763468548667668,0.3173125681909059], [-0.5198567190391367,0.9335053916544206,-1.509472274649017,1.454415814887731,0.4589687217941236], [-0.5711775989363378,1.0215061238027188,-1.6224000734110398,1.350665861834858,0.36668462127595286], [-0.39949682119974983,1.0408092246134175,-1.6587225284400737,1.301865863792736,0.4563742176523199], [-0.4902566261487769,1.0992586451801099,-1.4473476422759635,1.2309123590548212,0.2544829822903507], [-0.5293628624912633,0.8432327849564988,-1.7669603672683885,1.2911210280012069,0.41983427336223667], [-0.5738614502843032,0.9784937768658387,-1.7194271904022407,1.3116656980237082,0.46218317857671776], [-0.5902501988138692,0.9423215775018176,-1.684040151093259,1.4235274237923208,0.2039286490216381], [-0.4777727934676478,1.027586212069216,-1.7495753442806758,1.2757769414973703,0.3810309831295745], [-0.6566621783684151,0.9641681686268665,-1.5286420868141977,1.508411206136107,0.2501550388463756], [-0.46296281092785885,0.9727194845614605,-1.6171422039154086,1.3434828829348795,0.07809723939214058], [-0.5738547760098169,1.1077074999997352,-1.739110700752265,1.3494171899976157,0.3222123114107347], [-0.4002887849620737,0.9619411680270862,-1.663509769541054,1.2908149710547212,0.4244227376717664], [-0.4665533215112465,0.8596395857982323,-1.7467795346659782,1.251224446006115,0.42794754034954735], [-0.6374258626266393,1.0514903728245393,-1.651303971310752,1.236884794719695,0.5249538337308562], [-0.48340361610998744,0.8198136168919055,-1.690727257833199,1.398396991016531,0.31841739933411595], [-0.5410633313507479,0.8973900953721683,-1.4724867408929736,1.388151513303225,0.47893645155994446], [-0.3039096405457719,0.9155005236200422,-1.7173365893442798,1.256097860517092,0.24819750391337136], [-0.7090962179115853,0.8944749271173439,-1.8839838110448957,1.1446445350131944,0.22840975233443309], [-0.5100953498526793,0.8107010460411471,-1.7303477866873604,1.1647356509362308,0.4418371093135496], [-0.2962087734742913,0.7242125014742148,-1.5984563979400566,1.0588259317312654,0.28436170748337264], [-0.5505122810716748,0.7806970076803257,-1.72088933624607,1.4230022424760869,0.32673184311928194], [-0.45127942352855155,0.9959060503010477,-1.5524186364009327,1.2366972856510523,0.3578155222046753], [-0.5102066277582917,1.0949182323987419,-1.7142222646495175,1.1664611293679565,0.3602965879997574], [-0.20421136622466618,0.9866371147144095,-1.7538283133815977,1.0465139384173783,0.41274915410898083], [-0.3678643518086112,1.1128928060624481,-1.7416958328862653,1.1712943420281356,0.522586652590277], [-0.4272255324098842,0.9505411417476373,-1.5398631578697235,1.1670050581250606,0.18826195051488878], [-0.6294443147944699,1.0182211296265207,-1.6114292740208946,1.5494854908967275,0.2354819914065145], [-0.46052379116984477,0.8965751438720174,-1.6887957613207942,1.2678155037585463,0.3277249457164221], [-0.4661555388594827,1.0176554636610373,-1.660202673672577,1.1543960943838145,0.41490582984943747]) -2016-06-13 20:00:36 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:36 INFO JobScheduler:59 - Added jobs for time 6000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Computing RDD for time 7000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Created RDD 38 with Vector([-0.5184134212251081,0.9455318561030377,-1.7249072888539423,1.2476376390078272,0.539371371249466], [-0.4839353019153916,0.992343730650464,-1.4037670310964987,1.3650053852483648,0.287928402259341], [-0.45217444521287575,1.0425364789095595,-1.7771813152553952,1.2241390773159404,0.3053067128704871], [-0.34943560877945984,1.0407179160044182,-1.5177836780575653,0.9824846085205396,0.2603543256712868], [-0.45829448575134074,0.9725249117979422,-1.690056167350807,1.035512584568622,0.3026754916713263], [-0.42458145588018753,0.7854084174090814,-1.7649589009106188,1.1975914903392633,0.6551885852829413], [-0.5104497237497339,1.0402201914534017,-1.6859518565969145,1.1391556996625576,0.18881595045601088], [-0.5588856532488075,0.8709355475677711,-1.4735465340859104,1.207362168238276,0.5886898848511445], [-0.5286914346418723,0.9318873502365327,-1.654993790470044,1.274056873049218,0.29412525529417294], [-0.5420811906362676,0.9995645356070102,-1.6040227637757893,1.3412654820039975,0.3950459929854693], [-0.5105371229433685,0.8870982484234274,-1.6366208840493932,1.344244897498087,0.2842199976348651], [-0.5758159328886613,0.9361950961962593,-1.5151431024376867,1.2963264217469896,0.4195802203163277], [-0.46650906956232724,1.0883479790237522,-1.6209503836486092,1.2506119015001946,0.5239087792157935], [-0.49800785133272085,0.9121648111767798,-1.5664537098618618,1.4303891258276538,0.23796160092517643], [-0.6200401622139905,0.8819930763699753,-1.7712553265237971,1.144151394858828,0.37228012783293096], [-0.43683215654432145,0.8386127968983885,-1.6402554451670102,1.129209493050289,0.3612473529185551], [-0.4079221564079838,0.9656864104202829,-1.7359335721136713,1.4421338682074127,0.3616284206434529], [-0.43442591279896603,0.955449363348641,-1.5334677832140913,1.1666726886559795,0.5058368495019173], [-0.6457828019918121,0.931131898174338,-1.8532037937152797,1.0375316080434018,0.3133563527606291], [-0.38369598973754027,1.040856056086344,-1.7594025439463896,1.4917751006832976,0.45495744983872133], [-0.30166559339288085,1.0342180102112146,-1.5650152370405155,1.2673629485315197,0.4158807517084573], [-0.28580825731967757,0.8865395334100423,-1.4737235251638032,1.245008984525024,0.36435255784505255], [-0.34071421524008416,0.9427461052254856,-1.6571555985762556,1.116205755573728,0.3440594901972376], [-0.5707704618700846,0.9955277694976673,-1.815098893846346,1.2976867834380266,0.3708933869585827], [-0.6095164341318,0.9322989548597498,-1.6576644540187038,1.3001089252401639,0.32781904617649943], [-0.5884784232045771,0.9963121018549773,-1.4726908596796255,1.1232169328419297,0.466258074909203], [-0.6351651621700529,1.0173421935079714,-1.7981429560407272,1.0336059974520695,0.28162584897435694], [-0.38203401021631267,0.9790771913538491,-1.7244683271662424,1.288084387752797,0.4312827591331814], [-0.37183909143628785,1.1926117173870312,-1.5166716166223042,1.1696263300990144,0.4515586032102953], [-0.47462520704983,1.0498070539706301,-1.6863851371590932,1.2965114835042095,0.4645837570114517], [-0.35874130630231327,1.0042819076442355,-1.6407835501913508,1.214144240421076,0.35534792382876684], [-0.41389636789198914,0.9508170625312302,-1.6498089027313338,1.4272197180381103,0.374553819362672], [-0.32132785261092967,0.8535769504185499,-1.672835438070241,0.9421132176631009,0.24740658936307364], [-0.48253792736985773,1.0429236941137325,-1.4939460450129358,1.2832409419178106,0.40323476275374437], [-0.391512958705594,0.8859909853510953,-1.607045573914962,1.1770226037327562,0.5198479967492], [-0.4833163026675927,0.9200058086737815,-1.8188555219632765,1.2111735788924018,0.39618714901430313], [-0.2913135273832381,0.9301485161935364,-1.6975841978124846,1.4065399033140502,0.28453881400648173], [-0.3073756680829076,0.8325242716417236,-1.6753209327630654,1.4075134801045086,0.30351866480787215], [-0.2775051790445552,0.9452680429408632,-1.6224110577930677,1.1778711923142833,0.5531745712889398], [-0.3047686297259775,0.9468967188067203,-1.6929112749188848,1.4636396431851675,0.3586449109599954], [-0.5956899619218391,0.813236818549011,-1.757649523640403,1.2604654777105053,0.3185144577112101], [-0.32794499152459505,0.8140339815697406,-1.8666906537843855,1.2572887839007447,0.23549894606903882], [-0.40686690942678344,0.8619998816707273,-1.6319579560604098,1.3664940728963186,0.422156696202466], [-0.4138601150243667,1.021828823586237,-1.54841612428864,1.2398622089963618,0.5070042357490803], [-0.331869975870002,0.8782404384527787,-1.6635875859844171,1.2435143760020457,0.32101906837863314], [-0.49700214293681766,0.9856184010338479,-1.8232422669575907,1.2421051455581202,0.47592591320323163], [-0.535909591266969,0.9807383109826389,-1.6974325972167972,1.334548321075211,0.23946350824403165], [-0.6444021824207316,1.1849780210499694,-1.5121795557147575,1.3604386920675193,0.31690181205044465], [-0.49746220541003966,1.0643703042656938,-1.8499798436118189,1.1357972819777942,0.5780379400362996], [-0.4383285608503117,1.0163805234573544,-1.9030246206122645,1.438368352860807,0.3293850463477472]) -2016-06-13 20:00:36 INFO JobScheduler:59 - Added jobs for time 7000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Computing RDD for time 8000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Created RDD 44 with Vector([-0.374705866875181,0.9840691262184086,-1.5891551104970965,1.4814595125988181,0.47065837960007695], [-0.288810885883263,0.8437842751113669,-1.5395954061634287,1.1874811761270259,0.021249409500126437], [-0.4606335450965845,1.1012472510905122,-1.8287426699010534,1.24172318938614,0.4710333368456244], [-0.42944940507321583,1.0104988720375858,-1.5751317017658801,1.2831547194950401,0.3512701703197814], [-0.5152127056418667,1.1709583832128079,-1.6554391510043867,1.1397001494774806,0.23397277605773995], [-0.4399479870409779,1.024335104965922,-1.7045108897162344,1.196833551305392,0.4117115230011916], [-0.5430543625505585,1.1370095574712402,-1.7176429060334013,1.1901279759925694,0.46070443041212095], [-0.3968714897698767,1.0926860330748331,-1.6334286394688688,1.228234808051336,0.4665607383628074], [-0.46215342653947683,1.0343329141616782,-1.6499943761334228,1.337530812749641,0.23114556003826187], [-0.5000235665427208,0.9151728338053156,-1.8055132514640995,1.3037686620331144,0.2902025184248946], [-0.309233049825866,0.9728959274674948,-1.6392691162173578,1.101247705752869,0.3496423465401415], [-0.5981691090699208,0.8739296001671861,-1.5687282698937923,1.2550952881121755,0.46308264317451964], [-0.425561195913103,1.0676958602044149,-1.7135165364767921,1.3524692002054561,0.23228520033456318], [-0.7043177723021486,1.0283311625585259,-1.7478644671187442,1.2858120358708933,0.3418710936510813], [-0.41727892194293686,0.8625867727705405,-1.7914100057674134,1.1786792695315231,0.4917512427338196], [-0.40174243931692544,0.7828098802704966,-1.5073762714652954,1.281118669072956,0.44887930601066905], [-0.35310979651234775,0.9745364825956087,-1.6392317059509176,1.4513644518189859,0.3110119088453469], [-0.6853829628094648,1.1253397147829511,-1.4920081564639467,1.231135544811062,0.3139314829419366], [-0.44152016739085975,1.1304790288137783,-1.7325115907360644,1.3251089521914843,0.479567851123668], [-0.5526275004039871,0.9674440446022308,-1.7971453051937036,1.1664342514118335,0.31260632993291754], [-0.46562594489923503,0.96412818249357,-1.7756506925990838,1.1820367323525278,0.37368762092821634], [-0.42414895043860573,0.9369437276073684,-1.6228073393039317,1.2110129935336054,0.33600782254298567], [-0.4507513844294476,1.008422902581697,-1.6947838717172434,1.1258385535052762,0.23649573041213182], [-0.5011767543824581,1.0367482963168666,-1.8619227324329928,1.187686906059264,0.41060169959227016], [-0.29061733919433663,0.9125980333805904,-1.773384852821433,1.2318813347419482,0.5360023941871213], [-0.47558633395179245,1.0112018987992073,-1.6443927870523196,1.1868477340320063,0.378412637239359], [-0.47217857076866115,0.9027887018339104,-1.7359968088612976,1.372505255254142,0.4024631611010435], [-0.5635782393384723,1.0418520168542464,-1.6371483392749435,1.3046794831870752,0.36435743882876215], [-0.3566584096699375,0.9045378099174617,-1.6027843810019622,1.308376882406936,0.450420525347206], [-0.5161548558091542,0.9362466126941537,-1.543853038663444,1.351176902637456,0.37019671664199505], [-0.37734135089435306,1.1354760893808167,-1.5928707071728756,1.4230797310043477,0.5395830646701685], [-0.4935941975274797,0.9431897551602951,-1.7363999282764386,1.2592777434544042,0.34477211805728875], [-0.5399690948357754,1.0773952946269298,-1.623286131549866,1.3571017009557331,0.3820149575212495], [-0.47470272833792704,1.0279092716939924,-1.5583033476707477,1.2928147173688327,0.12854158780432537], [-0.3932748299826361,0.9168968642381998,-1.901294092833649,1.259188794957511,0.4011688713714673], [-0.3557493574970649,1.0003577059585267,-1.5111904842067994,1.3123818859934535,0.3019665980708428], [-0.38849198387448786,1.0634537152100978,-1.8742015500852314,1.207616787308903,0.527158124827907], [-0.32516837051884395,1.0806577326663074,-1.611781542302308,1.182593379846847,0.5196789294606137], [-0.5212725898858087,0.998490506190397,-1.6184763965153202,1.3731447165706872,0.29855378648402087], [-0.40257752034798155,0.8314340557777532,-1.6850156321125218,1.3427297403898506,0.5156164947536159], [-0.4889473969173199,0.8999745071099362,-1.4468367122261747,1.3090812041423647,0.5108182965913203], [-0.48274282778056277,1.1537075465803446,-1.5811276311378704,1.287632315416058,0.50882779799236], [-0.23362208323850905,0.9548750423156142,-1.572161725609678,1.6027003014260315,0.3939518404103326], [-0.4292988792888861,0.9366381149644413,-1.8250064321036654,1.2358072562830058,0.18892247590763034], [-0.484748914270206,0.7653236279449509,-1.7486834183984747,1.2949268753185315,0.39683334762547673], [-0.4399736279490563,0.9634644976145054,-1.7480413166448512,1.1775955441636703,0.36332416278682794], [-0.48373350642316953,0.9259530051332495,-1.7977337219360736,1.2876089889322726,0.4497462132788797], [-0.38070426405120866,0.8838236645507052,-1.6753928124899833,1.2449514905082557,0.3276558558550576], [-0.5717833110401351,1.0273299096906752,-1.6190879043151574,1.1860455839092494,0.32636997750947594], [-0.5231330958628938,0.924256801489448,-1.5484246875687981,1.3381528852011608,0.3757180006852602]) -2016-06-13 20:00:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:36 INFO JobScheduler:59 - Added jobs for time 8000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Computing RDD for time 9000 ms -2016-06-13 20:00:36 INFO DAGScheduler:59 - Registering RDD 7 (apply at Transformer.scala:22) -2016-06-13 20:00:36 INFO TestInputStream:59 - Created RDD 50 with Vector([-0.4026578572929527,1.0001062834205203,-1.5617908885948877,1.462733521830009,0.2061287167736916], [-0.3965287353033582,1.0683726645043978,-1.4873730061169563,1.1460792990307642,0.27746130384846085], [-0.4938197206621628,0.7709438014433136,-1.6076436749339549,1.0982028796258234,0.24486596655062928], [-0.4777650011847253,0.9489155740228515,-1.476414143798642,1.3062514135704215,0.6114439757268249], [-0.4285977940802228,0.9382568588433707,-1.48976698594476,1.1698457774900977,0.297712575086562], [-0.5993896749484521,0.8629058534578911,-1.6510698754517374,1.2090117714723978,0.5458042851487852], [-0.5321474831529757,1.0849054231681536,-1.667568850178397,1.4161664775537062,0.4493689535824759], [-0.6723705689897437,0.7517542425048965,-1.663645669684176,1.2240252996527488,0.346848907683753], [-0.40115735788941104,0.8163474979541504,-1.7694648704001712,1.1073947331922183,0.3856322637520626], [-0.4787497898360035,0.855418622108937,-1.7422359123781868,1.2371448524727167,0.5156131237949001], [-0.5321048883779993,0.7970778523677844,-1.7616773188005117,1.3821414318824692,0.48832517040159346], [-0.5285360880098152,1.0597395124384508,-1.6417892576104653,1.3464020747697325,0.31849105791627297], [-0.5324435853239631,0.9841563147023031,-1.5943852209495275,1.2707723748060116,0.4139910360672806], [-0.37531742848811983,0.7874289125205598,-1.6479869931712725,1.3181636612914271,0.3718789043065717], [-0.4560433134330382,1.0759812484577316,-1.697865869705092,1.2725177127861855,0.06883328163775021], [-0.45513942313770217,0.9498671339799809,-1.6933930804659583,1.3025139463978586,0.5350768600248874], [-0.526256522617052,0.8727003678813281,-1.4356120595661468,1.29993914263913,0.43387328603333664], [-0.3923483046111808,0.994060772829843,-1.6451266300509706,1.2983503204985913,0.3703765349262901], [-0.4374913726020475,0.8955789511391181,-1.4728897306445088,1.1357862983779858,0.30881407016500095], [-0.41951697820602707,1.0007695283696842,-1.708286281016387,1.4190975260604537,0.28731659836668877], [-0.6750333217393494,1.0688779823591679,-1.6846746627252691,1.2962992487600882,0.5030936797752793], [-0.5956866906574221,0.8121299274245898,-1.692929629203368,1.2338548732523056,0.38371772482125543], [-0.4303591738987757,0.9459620785706632,-1.884551120494729,1.4937212054647857,0.415792749586519], [-0.45504157760203423,0.9027897906482127,-1.799761299703666,1.3040399621270629,0.5736987408004388], [-0.5593669081347996,1.0931782446321396,-1.627955485174887,1.1391759184235553,0.39801309200504786], [-0.3825955534231694,0.9655263256725013,-1.7068123920171723,1.1861538643513039,0.2764471843224494], [-0.3587313623839552,0.9371265382727668,-1.487908236071683,1.3184799953634567,0.4370396706398445], [-0.475038660091078,0.797460346861311,-1.6835937249660264,1.3120302511529962,0.17031892070731325], [-0.4663190378327658,0.991352295299833,-1.6444316855302834,1.51526349576866,0.3929422369724258], [-0.5040823137977553,1.0575372259159488,-1.594322659684267,1.2971698194040933,0.5092825217956699], [-0.44790417664079346,0.8518362899333143,-1.648211585312696,1.1839806869130056,0.40832085080824754], [-0.537755895075537,0.9806637869508009,-1.7656906916731008,1.2433660326767015,0.42887767735060545], [-0.34293660736335746,0.7935119912222357,-1.6359022010243611,1.3593857228798436,0.3967833959145234], [-0.6208695835786292,0.9640100782092327,-1.874229161549032,1.343981009171891,0.20123531286937832], [-0.6723717817990262,0.9002960054466735,-1.6910864401441592,1.1818255352705958,0.5433986930426408], [-0.4588708699931968,1.2092994554375371,-1.5324768096769932,1.4083576723657105,0.3708605878943178], [-0.560638238383703,0.9092570275383438,-1.473047966472027,1.3497511209299733,0.3467764467476322], [-0.4409803705877342,0.9370238683801353,-1.572432943897284,1.41498798111251,0.2748026192628795], [-0.38741643074715315,1.071718105146961,-1.624146896559199,1.1401522638836572,0.3872910322155703], [-0.5140862086906481,0.8211106116047733,-1.6027037015807561,1.1677747630844688,0.2659604974429464], [-0.6693385873907269,0.7554393139905898,-1.7108087289854104,1.09991932687034,0.41821242261960917], [-0.3282503845126481,0.9257033715775067,-1.5235939065144024,1.1889128563024831,0.3368716615939142], [-0.5366461042406446,0.9240666515119298,-1.8135909127248846,1.3575511869067935,0.37505000938261085], [-0.39294848413804057,1.0124585311856609,-1.575086388417685,1.2467079941490156,0.40416560148386643], [-0.48738816985096806,0.8787565657383682,-1.810281961839409,1.3779890105636297,0.46019825317610896], [-0.36592476270170865,0.9132457437551841,-1.7089939281056865,1.1718159447241387,0.37244792446782193], [-0.40751855185193425,0.9541054981051484,-1.4879650472104635,1.1817057016562598,0.44919665548504983], [-0.29317450049605076,0.964443479319753,-1.656028808081747,1.2586398699632722,0.2526740377633023], [-0.4615968319838038,1.0243528061825597,-1.5054257187951623,1.351376529833804,0.42971544189574107], [-0.47988177538312266,1.0112860256804128,-1.5404476412782737,1.233668528335028,0.2028012939752821]) -2016-06-13 20:00:36 INFO DAGScheduler:59 - Got job 0 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:36 INFO DAGScheduler:59 - Final stage: ResultStage 1(apply at Transformer.scala:22) -2016-06-13 20:00:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 20:00:36 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 20:00:36 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[7] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:36 INFO JobScheduler:59 - Added jobs for time 9000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Computing RDD for time 10000 ms -2016-06-13 20:00:36 INFO TestInputStream:59 - Created RDD 56 with Vector([-0.4889010904178065,1.0197902904388123,-1.6434083794608938,1.2273894895374848,0.29880056758004836], [-0.3690054860756016,0.9083136794567547,-1.5975355891184535,1.1937534562371308,0.3514272190683493], [-0.5737783702048262,1.2273536117223582,-1.4617006566984774,1.342244921629087,0.3377586288639175], [-0.48387316003666864,0.9673070026559238,-1.716725721249248,1.2214389230026372,0.37779842180031725], [-0.3777277713061831,0.866118273628028,-1.8487356572074127,1.1851350383431707,0.5215534466266145], [-0.6076926717533072,0.922880686625322,-1.7313060678714964,1.3632488420681186,0.3176699282425657], [-0.3802374789807806,0.8439055679755229,-1.6219613687183736,1.1809867291088374,0.4174630121979538], [-0.567466556581487,0.9292300369161763,-1.5542588287823018,1.3568593851021735,0.38458895772840795], [-0.6764387691659441,0.9816754055716941,-1.6652926928722982,1.3324217076474294,0.2844320904428872], [-0.584138124369403,0.9651833514289697,-1.7100579024456408,1.3025648636881102,0.3575320807936941], [-0.45548159931880944,1.0181220363355081,-1.6506653550843506,1.1304474598171783,0.4285968106743953], [-0.5370002084128622,0.927571736048293,-1.7228464440630433,1.2925613214036213,0.3169164091166503], [-0.47996613069291666,0.884366449915093,-1.7212355955272085,1.152597880658151,0.43962468554199663], [-0.35493034949860885,0.8984344817504613,-1.65218132155661,1.3068108975806394,0.354531357717713], [-0.43226118157508875,0.90725353631203,-1.7628881371748841,1.314870897760879,0.40214500076142384], [-0.4387259831220203,0.9621751931555681,-1.7268764217836796,1.1093705777029121,0.17867972251315242], [-0.44765902746822306,1.1186404105474657,-1.750942737318709,1.322673860044468,0.41609066577279374], [-0.40351581789921565,1.046340578817358,-1.7120920286690795,1.244893797129159,0.3833023298379642], [-0.3093624994668177,0.8356650811564297,-1.8642891852543533,1.217524168702485,0.2103617986137151], [-0.2990644638630094,0.9070655034040485,-1.6094469406493193,1.1592980925831848,0.43244854200109223], [-0.5596021456993305,0.9019708087164819,-1.5105356278212536,1.2598551179472526,0.3428536586628008], [-0.583393764986965,0.9748568871770861,-1.5302538126845806,1.301546791122464,0.21927558650218368], [-0.49772324825286274,0.9716299176892979,-1.6169165069063722,1.3531471880552017,0.25858373282508224], [-0.43106771684572104,1.141003137647425,-1.7578104686857339,1.1835903210552496,0.5254045668739125], [-0.5997306192424074,1.0040726704610177,-1.8227253165238155,1.2317004811683654,0.46876771655575034], [-0.38252412898165267,0.6654527000305275,-1.6468024292953505,1.246225339488523,0.3090133585357682], [-0.44821229943324725,1.0038256342470713,-1.6968551076177416,1.4529627079929077,0.3454589917432014], [-0.43535992691090786,1.0110816584854143,-1.7453606983094134,1.248210478232135,0.2891585839680707], [-0.3652348315036368,0.8958964249587966,-1.9606863598829045,1.3389257198065394,0.2618900608154475], [-0.49376528406043635,1.1421599774098976,-1.7157624036331927,1.391772429187865,0.2876879628208894], [-0.4515063147230089,0.9428403433366573,-1.76394465423704,1.203639940182186,0.5058415243772909], [-0.566229150629525,1.0128616873896932,-1.6880365432069786,1.2877554523210368,0.1629872361106314], [-0.47658344463796903,1.0862736976313303,-1.7934284966158631,1.2272771650263852,0.41680211738171946], [-0.3618808351944353,0.9417137306142976,-1.8600076455984111,1.2206352320840592,0.3461749973804146], [-0.6214757024643818,0.9952948941676851,-1.62097329834717,1.3958184351557885,0.46499512939060494], [-0.35996348236796494,0.9954644862470238,-1.7933270864489543,1.4005533772243737,0.5639356774206108], [-0.6006970016720174,0.9686450943470077,-1.7310975948083858,1.2398543489295077,0.4483279499267394], [-0.4592797597414643,0.9631529442377594,-1.6797024055855931,1.4289800788354718,0.28667160760155463], [-0.32700881076504584,0.8147754680205297,-1.763026614596352,1.3194735072019403,0.36816545181467747], [-0.4305345983795718,1.131285559002078,-1.7583813300937976,1.363545141597296,0.2601165464857354], [-0.448497878644767,1.0701920363594992,-1.6308801358271674,1.2674713459930484,0.49959128317905865], [-0.3686293381026533,1.025599917053419,-1.5433629968114364,1.2801178859900646,0.4174115399252367], [-0.574113693976958,0.9757976338077295,-1.6388778822719372,1.2759164798546365,0.5542142157229283], [-0.4357071400660946,0.8338147376622728,-1.8285888680646112,1.271233025944097,0.270178918164786], [-0.47158188300536635,1.0122183541160776,-1.6010520252577112,1.5447804629514494,0.3533722934153737], [-0.5307926544952695,0.9107879044274507,-1.4490342775396574,1.4031526476841614,0.3809780658985748], [-0.4953668915084064,0.9298227962550192,-1.5651812648587027,1.11568589286655,0.4774983034589586], [-0.7383552095915331,0.7626241115292196,-1.7008182872047266,1.2502877442685654,0.43135198632241045], [-0.539114438923603,1.0489102632469824,-1.6736818095705865,1.2768121659352345,0.2213641562753643], [-0.46002266280222004,1.0294295650038185,-1.6033579773810744,1.407522155806443,0.5223722922756932]) -2016-06-13 20:00:36 INFO JobScheduler:59 - Added jobs for time 10000 ms -2016-06-13 20:00:36 INFO JobGenerator:59 - Checkpointing graph for time 1000 ms -2016-06-13 20:00:36 INFO DStreamGraph:59 - Updating checkpoint data for time 1000 ms -2016-06-13 20:00:36 INFO DStreamGraph:59 - Updated checkpoint data for time 1000 ms -2016-06-13 20:00:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:36 INFO JobGenerator:59 - Checkpointing graph for time 2000 ms -2016-06-13 20:00:36 INFO DStreamGraph:59 - Updating checkpoint data for time 2000 ms -2016-06-13 20:00:36 INFO DStreamGraph:59 - Updated checkpoint data for time 2000 ms -2016-06-13 20:00:36 INFO CheckpointWriter:59 - Saving checkpoint for time 1000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-1000' -2016-06-13 20:00:36 INFO JobGenerator:59 - Checkpointing graph for time 3000 ms -2016-06-13 20:00:36 INFO DStreamGraph:59 - Updating checkpoint data for time 3000 ms -2016-06-13 20:00:36 INFO DStreamGraph:59 - Updated checkpoint data for time 3000 ms -2016-06-13 20:00:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:36 INFO JobGenerator:59 - Checkpointing graph for time 4000 ms -2016-06-13 20:00:36 INFO DStreamGraph:59 - Updating checkpoint data for time 4000 ms -2016-06-13 20:00:36 INFO DStreamGraph:59 - Updated checkpoint data for time 4000 ms -2016-06-13 20:00:36 INFO JobGenerator:59 - Checkpointing graph for time 5000 ms -2016-06-13 20:00:36 INFO DStreamGraph:59 - Updating checkpoint data for time 5000 ms -2016-06-13 20:00:36 INFO DStreamGraph:59 - Updated checkpoint data for time 5000 ms -2016-06-13 20:00:36 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=0, maxMem=766075207 -2016-06-13 20:00:36 INFO JobGenerator:59 - Checkpointing graph for time 6000 ms -2016-06-13 20:00:36 INFO DStreamGraph:59 - Updating checkpoint data for time 6000 ms -2016-06-13 20:00:36 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:00:36 INFO DStreamGraph:59 - Updated checkpoint data for time 6000 ms -2016-06-13 20:00:36 INFO MemoryStore:59 - ensureFreeSpace(2428) called with curMem=4744, maxMem=766075207 -2016-06-13 20:00:36 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:00:36 INFO JobGenerator:59 - Checkpointing graph for time 7000 ms -2016-06-13 20:00:36 INFO DStreamGraph:59 - Updating checkpoint data for time 7000 ms -2016-06-13 20:00:36 INFO DStreamGraph:59 - Updated checkpoint data for time 7000 ms -2016-06-13 20:00:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:36 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:50497 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:00:36 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[7] at apply at Transformer.scala:22) -2016-06-13 20:00:36 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 20:00:36 INFO JobGenerator:59 - Checkpointing graph for time 8000 ms -2016-06-13 20:00:36 INFO DStreamGraph:59 - Updating checkpoint data for time 8000 ms -2016-06-13 20:00:36 INFO DStreamGraph:59 - Updated checkpoint data for time 8000 ms -2016-06-13 20:00:36 INFO JobGenerator:59 - Checkpointing graph for time 9000 ms -2016-06-13 20:00:36 INFO DStreamGraph:59 - Updating checkpoint data for time 9000 ms -2016-06-13 20:00:36 INFO DStreamGraph:59 - Updated checkpoint data for time 9000 ms -2016-06-13 20:00:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:00:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:00:36 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 20:00:36 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 20:00:36 INFO JobGenerator:59 - Checkpointing graph for time 10000 ms -2016-06-13 20:00:36 INFO DStreamGraph:59 - Updating checkpoint data for time 10000 ms -2016-06-13 20:00:36 INFO DStreamGraph:59 - Updated checkpoint data for time 10000 ms -2016-06-13 20:00:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:36 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 20:00:36 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 20:00:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:37 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:37 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1159 bytes result sent to driver -2016-06-13 20:00:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 177 ms on localhost (1/2) -2016-06-13 20:00:37 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:37 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:37 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:37 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:37 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1159 bytes result sent to driver -2016-06-13 20:00:37 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 466 ms on localhost (2/2) -2016-06-13 20:00:37 INFO DAGScheduler:59 - ShuffleMapStage 0 (apply at Transformer.scala:22) finished in 0.486 s -2016-06-13 20:00:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 20:00:37 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 20:00:37 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:37 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 20:00:37 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[32] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:37 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=7172, maxMem=766075207 -2016-06-13 20:00:37 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:00:37 INFO MemoryStore:59 - ensureFreeSpace(2657) called with curMem=12436, maxMem=766075207 -2016-06-13 20:00:37 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 20:00:37 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:50497 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:00:37 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[32] at apply at Transformer.scala:22) -2016-06-13 20:00:37 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 20:00:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:37 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 20:00:37 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 20:00:37 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:00:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-06-13 20:00:37 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:37 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-06-13 20:00:37 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-06-13 20:00:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 45 ms on localhost (1/2) -2016-06-13 20:00:37 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1498 bytes result sent to driver -2016-06-13 20:00:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 83 ms on localhost (2/2) -2016-06-13 20:00:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 20:00:37 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:37 INFO DAGScheduler:59 - ResultStage 1 (apply at Transformer.scala:22) finished in 0.085 s -2016-06-13 20:00:37 INFO CheckpointWriter:59 - Checkpoint for time 1000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-1000', took 27276 bytes and 735 ms -2016-06-13 20:00:37 INFO CheckpointWriter:59 - Saving checkpoint for time 2000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-2000' -2016-06-13 20:00:37 INFO DAGScheduler:59 - Job 0 finished: apply at Transformer.scala:22, took 0.837937 s -2016-06-13 20:00:37 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 50.0 and centroid: [-0.4843392480139548,0.9734525763849463,-1.7046336323611109,1.2778093795456185,0.3691650550997036] -2016-06-13 20:00:37 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:37 INFO CheckpointWriter:59 - Checkpoint for time 2000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-2000', took 27278 bytes and 89 ms -2016-06-13 20:00:37 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:37 INFO CheckpointWriter:59 - Saving checkpoint for time 3000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-3000' -2016-06-13 20:00:37 INFO JobScheduler:59 - Finished job streaming job 1000 ms.0 from job set of time 1000 ms -2016-06-13 20:00:37 INFO JobScheduler:59 - Starting job streaming job 1000 ms.1 from job set of time 1000 ms -2016-06-13 20:00:37 INFO DAGScheduler:59 - Registering RDD 3 (apply at Transformer.scala:22) -2016-06-13 20:00:37 INFO DAGScheduler:59 - Got job 1 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:37 INFO DAGScheduler:59 - Final stage: ResultStage 3(apply at Transformer.scala:22) -2016-06-13 20:00:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 2) -2016-06-13 20:00:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 2) -2016-06-13 20:00:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (UnionRDD[3] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:37 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=15093, maxMem=766075207 -2016-06-13 20:00:37 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:00:37 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:37 INFO MemoryStore:59 - ensureFreeSpace(2000) called with curMem=18373, maxMem=766075207 -2016-06-13 20:00:37 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2000.0 B, free 730.6 MB) -2016-06-13 20:00:37 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:50497 (size: 2000.0 B, free: 730.6 MB) -2016-06-13 20:00:37 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:37 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 2 (UnionRDD[3] at apply at Transformer.scala:22) -2016-06-13 20:00:37 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-06-13 20:00:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:00:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:00:37 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 20:00:37 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 20:00:37 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1159 bytes result sent to driver -2016-06-13 20:00:37 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:00:37 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-06-13 20:00:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 31 ms on localhost (1/3) -2016-06-13 20:00:37 INFO CheckpointWriter:59 - Checkpoint for time 3000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-3000', took 27278 bytes and 133 ms -2016-06-13 20:00:37 INFO CheckpointWriter:59 - Saving checkpoint for time 4000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-4000' -2016-06-13 20:00:37 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1159 bytes result sent to driver -2016-06-13 20:00:37 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 63 ms on localhost (2/3) -2016-06-13 20:00:37 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 6). 1159 bytes result sent to driver -2016-06-13 20:00:37 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 6) in 48 ms on localhost (3/3) -2016-06-13 20:00:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 20:00:37 INFO DAGScheduler:59 - ShuffleMapStage 2 (apply at Transformer.scala:22) finished in 0.084 s -2016-06-13 20:00:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:37 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 3) -2016-06-13 20:00:37 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:37 INFO DAGScheduler:59 - Missing parents for ResultStage 3: List() -2016-06-13 20:00:37 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[62] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:37 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=20373, maxMem=766075207 -2016-06-13 20:00:37 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:00:37 INFO MemoryStore:59 - ensureFreeSpace(1799) called with curMem=23613, maxMem=766075207 -2016-06-13 20:00:37 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1799.0 B, free 730.6 MB) -2016-06-13 20:00:37 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:50497 (size: 1799.0 B, free: 730.6 MB) -2016-06-13 20:00:37 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[62] at apply at Transformer.scala:22) -2016-06-13 20:00:37 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 20:00:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:37 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 8) -2016-06-13 20:00:37 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:37 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 7) -2016-06-13 20:00:37 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:00:37 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:00:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:37 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 8). 1172 bytes result sent to driver -2016-06-13 20:00:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 8) in 25 ms on localhost (1/2) -2016-06-13 20:00:37 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 7). 1180 bytes result sent to driver -2016-06-13 20:00:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 7) in 30 ms on localhost (2/2) -2016-06-13 20:00:37 INFO DAGScheduler:59 - ResultStage 3 (apply at Transformer.scala:22) finished in 0.031 s -2016-06-13 20:00:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 20:00:37 INFO DAGScheduler:59 - Job 1 finished: apply at Transformer.scala:22, took 0.205703 s -2016-06-13 20:00:37 INFO JobScheduler:59 - Finished job streaming job 1000 ms.1 from job set of time 1000 ms -2016-06-13 20:00:37 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:00:37 INFO JobScheduler:59 - Total delay: 1465819236.770 s for time 1000 ms (execution: 1.270 s) -2016-06-13 20:00:37 INFO CheckpointWriter:59 - Checkpoint for time 4000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-4000', took 27278 bytes and 114 ms -2016-06-13 20:00:37 INFO CheckpointWriter:59 - Saving checkpoint for time 5000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-5000' -2016-06-13 20:00:37 INFO JobScheduler:59 - Starting job streaming job 2000 ms.0 from job set of time 2000 ms -2016-06-13 20:00:37 INFO JobGenerator:59 - Checkpointing graph for time 1000 ms -2016-06-13 20:00:37 INFO DStreamGraph:59 - Updating checkpoint data for time 1000 ms -2016-06-13 20:00:37 INFO DStreamGraph:59 - Updated checkpoint data for time 1000 ms -2016-06-13 20:00:37 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:37 INFO DAGScheduler:59 - Registering RDD 63 (apply at Transformer.scala:22) -2016-06-13 20:00:37 INFO DAGScheduler:59 - Got job 2 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:37 INFO DAGScheduler:59 - Final stage: ResultStage 5(apply at Transformer.scala:22) -2016-06-13 20:00:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-06-13 20:00:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-06-13 20:00:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[63] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:37 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=25412, maxMem=766075207 -2016-06-13 20:00:37 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:00:37 INFO MemoryStore:59 - ensureFreeSpace(2471) called with curMem=30156, maxMem=766075207 -2016-06-13 20:00:37 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:00:37 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:50497 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:00:37 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[63] at apply at Transformer.scala:22) -2016-06-13 20:00:37 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 20:00:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:00:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 10, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:00:37 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 9) -2016-06-13 20:00:37 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 10) -2016-06-13 20:00:37 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:00:37 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 9). 1159 bytes result sent to driver -2016-06-13 20:00:37 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 10). 1159 bytes result sent to driver -2016-06-13 20:00:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 9) in 31 ms on localhost (1/2) -2016-06-13 20:00:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 10) in 30 ms on localhost (2/2) -2016-06-13 20:00:37 INFO DAGScheduler:59 - ShuffleMapStage 4 (apply at Transformer.scala:22) finished in 0.032 s -2016-06-13 20:00:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:37 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-06-13 20:00:37 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 20:00:37 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-06-13 20:00:37 INFO DAGScheduler:59 - Submitting ResultStage 5 (ShuffledRDD[64] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:37 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=32627, maxMem=766075207 -2016-06-13 20:00:37 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:00:37 INFO MemoryStore:59 - ensureFreeSpace(2692) called with curMem=37891, maxMem=766075207 -2016-06-13 20:00:37 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:00:37 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:50497 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:00:37 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (ShuffledRDD[64] at apply at Transformer.scala:22) -2016-06-13 20:00:37 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 20:00:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 12, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:37 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 11) -2016-06-13 20:00:37 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 12) -2016-06-13 20:00:37 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:37 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:00:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:37 INFO CheckpointWriter:59 - Checkpoint for time 5000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-5000', took 27278 bytes and 91 ms -2016-06-13 20:00:37 INFO CheckpointWriter:59 - Saving checkpoint for time 6000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-6000' -2016-06-13 20:00:37 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 12). 1161 bytes result sent to driver -2016-06-13 20:00:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 12) in 13 ms on localhost (1/2) -2016-06-13 20:00:37 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 11). 1498 bytes result sent to driver -2016-06-13 20:00:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 11) in 17 ms on localhost (2/2) -2016-06-13 20:00:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 20:00:37 INFO DAGScheduler:59 - ResultStage 5 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:00:37 INFO DAGScheduler:59 - Job 2 finished: apply at Transformer.scala:22, took 0.080083 s -2016-06-13 20:00:37 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 100.0 and centroid: [-0.4738959687352186,0.969729236815007,-1.6823805644329948,1.2679893451777908,0.37589591355279117] -2016-06-13 20:00:37 INFO JobScheduler:59 - Finished job streaming job 2000 ms.0 from job set of time 2000 ms -2016-06-13 20:00:37 INFO JobScheduler:59 - Starting job streaming job 2000 ms.1 from job set of time 2000 ms -2016-06-13 20:00:37 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:37 INFO DAGScheduler:59 - Registering RDD 10 (apply at Transformer.scala:22) -2016-06-13 20:00:37 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:00:37 INFO DAGScheduler:59 - Got job 3 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:37 INFO DAGScheduler:59 - Final stage: ResultStage 7(apply at Transformer.scala:22) -2016-06-13 20:00:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-06-13 20:00:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-06-13 20:00:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (UnionRDD[10] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:37 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=40583, maxMem=766075207 -2016-06-13 20:00:37 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:37 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=43863, maxMem=766075207 -2016-06-13 20:00:37 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:00:37 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:50497 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:00:37 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:37 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 6 (UnionRDD[10] at apply at Transformer.scala:22) -2016-06-13 20:00:37 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 3 tasks -2016-06-13 20:00:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:00:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 14, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:00:37 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 14) -2016-06-13 20:00:37 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 13) -2016-06-13 20:00:37 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 14). 1159 bytes result sent to driver -2016-06-13 20:00:37 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 15, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:00:37 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 15) -2016-06-13 20:00:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 14) in 21 ms on localhost (1/3) -2016-06-13 20:00:37 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 13). 1159 bytes result sent to driver -2016-06-13 20:00:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 13) in 30 ms on localhost (2/3) -2016-06-13 20:00:37 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:00:37 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 15). 1159 bytes result sent to driver -2016-06-13 20:00:37 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 15) in 19 ms on localhost (3/3) -2016-06-13 20:00:37 INFO DAGScheduler:59 - ShuffleMapStage 6 (apply at Transformer.scala:22) finished in 0.040 s -2016-06-13 20:00:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 20:00:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:37 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-06-13 20:00:37 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:37 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-06-13 20:00:37 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[65] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:37 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=45872, maxMem=766075207 -2016-06-13 20:00:37 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:37 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=49112, maxMem=766075207 -2016-06-13 20:00:37 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.5 MB) -2016-06-13 20:00:37 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:50497 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:00:37 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[65] at apply at Transformer.scala:22) -2016-06-13 20:00:37 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-06-13 20:00:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:37 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 16) -2016-06-13 20:00:37 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 17) -2016-06-13 20:00:37 INFO CheckpointWriter:59 - Checkpoint for time 6000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-6000', took 27276 bytes and 103 ms -2016-06-13 20:00:37 INFO CheckpointWriter:59 - Saving checkpoint for time 7000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-7000' -2016-06-13 20:00:37 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:00:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:37 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 16). 1180 bytes result sent to driver -2016-06-13 20:00:37 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:00:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:37 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 17). 1172 bytes result sent to driver -2016-06-13 20:00:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 16) in 18 ms on localhost (1/2) -2016-06-13 20:00:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 17) in 18 ms on localhost (2/2) -2016-06-13 20:00:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 20:00:37 INFO DAGScheduler:59 - ResultStage 7 (apply at Transformer.scala:22) finished in 0.020 s -2016-06-13 20:00:37 INFO DAGScheduler:59 - Job 3 finished: apply at Transformer.scala:22, took 0.097631 s -2016-06-13 20:00:37 INFO JobScheduler:59 - Finished job streaming job 2000 ms.1 from job set of time 2000 ms -2016-06-13 20:00:37 INFO JobScheduler:59 - Total delay: 1465819235.991 s for time 2000 ms (execution: 0.205 s) -2016-06-13 20:00:37 INFO JobScheduler:59 - Starting job streaming job 3000 ms.0 from job set of time 3000 ms -2016-06-13 20:00:37 INFO ParallelCollectionRDD:59 - Removing RDD 0 from persistence list -2016-06-13 20:00:38 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Registering RDD 66 (apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Got job 4 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:38 INFO DAGScheduler:59 - Final stage: ResultStage 9(apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 8) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 8) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 8 (MapPartitionsRDD[66] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=50914, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(2475) called with curMem=55658, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:50497 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:00:38 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 8 (MapPartitionsRDD[66] at apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 18, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 19, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:00:38 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 18) -2016-06-13 20:00:38 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 19) -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 0 -2016-06-13 20:00:38 INFO MapPartitionsRDD:59 - Removing RDD 5 from persistence list -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 5 -2016-06-13 20:00:38 INFO ShuffledRDD:59 - Removing RDD 4 from persistence list -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 4 -2016-06-13 20:00:38 INFO UnionRDD:59 - Removing RDD 3 from persistence list -2016-06-13 20:00:38 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 18). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 3 -2016-06-13 20:00:38 INFO MapPartitionsRDD:59 - Removing RDD 1 from persistence list -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 18) in 39 ms on localhost (1/2) -2016-06-13 20:00:38 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 19). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 1 -2016-06-13 20:00:38 INFO JobGenerator:59 - Checkpointing graph for time 2000 ms -2016-06-13 20:00:38 INFO DStreamGraph:59 - Updating checkpoint data for time 2000 ms -2016-06-13 20:00:38 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:00:38 INFO DStreamGraph:59 - Updated checkpoint data for time 2000 ms -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 19) in 41 ms on localhost (2/2) -2016-06-13 20:00:38 INFO DAGScheduler:59 - ShuffleMapStage 8 (apply at Transformer.scala:22) finished in 0.043 s -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 20:00:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 9) -2016-06-13 20:00:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:38 INFO DAGScheduler:59 - Missing parents for ResultStage 9: List() -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting ResultStage 9 (ShuffledRDD[67] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=58133, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=63397, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:50497 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:00:38 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (ShuffledRDD[67] at apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 21, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:38 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 20) -2016-06-13 20:00:38 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 21) -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:38 INFO CheckpointWriter:59 - Checkpoint for time 7000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-7000', took 27276 bytes and 130 ms -2016-06-13 20:00:38 INFO CheckpointWriter:59 - Saving checkpoint for time 8000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-8000' -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:38 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 21). 1161 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 21) in 12 ms on localhost (1/2) -2016-06-13 20:00:38 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 20). 1498 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 20) in 15 ms on localhost (2/2) -2016-06-13 20:00:38 INFO DAGScheduler:59 - ResultStage 9 (apply at Transformer.scala:22) finished in 0.016 s -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 20:00:38 INFO DAGScheduler:59 - Job 4 finished: apply at Transformer.scala:22, took 0.105271 s -2016-06-13 20:00:38 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 150.0 and centroid: [-0.4741102089605591,0.9622750167428704,-1.680286076986711,1.2654376587211662,0.37713329463498424] -2016-06-13 20:00:38 INFO JobScheduler:59 - Finished job streaming job 3000 ms.0 from job set of time 3000 ms -2016-06-13 20:00:38 INFO JobScheduler:59 - Starting job streaming job 3000 ms.1 from job set of time 3000 ms -2016-06-13 20:00:38 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Registering RDD 16 (apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Got job 5 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:38 INFO DAGScheduler:59 - Final stage: ResultStage 11(apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 10) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 10) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 10 (UnionRDD[16] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=66093, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(2004) called with curMem=69373, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2004.0 B, free 730.5 MB) -2016-06-13 20:00:38 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:50497 (size: 2004.0 B, free: 730.6 MB) -2016-06-13 20:00:38 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 10 (UnionRDD[16] at apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 3 tasks -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 22, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 23, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:00:38 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 23) -2016-06-13 20:00:38 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 22) -2016-06-13 20:00:38 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 23). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 2.0 in stage 10.0 (TID 24, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 23) in 21 ms on localhost (1/3) -2016-06-13 20:00:38 INFO Executor:59 - Running task 2.0 in stage 10.0 (TID 24) -2016-06-13 20:00:38 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 22). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO Executor:59 - Finished task 2.0 in stage 10.0 (TID 24). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 22) in 40 ms on localhost (2/3) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 2.0 in stage 10.0 (TID 24) in 21 ms on localhost (3/3) -2016-06-13 20:00:38 INFO DAGScheduler:59 - ShuffleMapStage 10 (apply at Transformer.scala:22) finished in 0.041 s -2016-06-13 20:00:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 20:00:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 11) -2016-06-13 20:00:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:38 INFO DAGScheduler:59 - Missing parents for ResultStage 11: List() -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[68] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=71377, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=74617, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.5 MB) -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:50497 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:00:38 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[68] at apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 25, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 26, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:38 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 25) -2016-06-13 20:00:38 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 26) -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:38 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 26). 1172 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 26) in 8 ms on localhost (1/2) -2016-06-13 20:00:38 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 25). 1180 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 25) in 13 ms on localhost (2/2) -2016-06-13 20:00:38 INFO CheckpointWriter:59 - Checkpoint for time 8000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-8000', took 27278 bytes and 102 ms -2016-06-13 20:00:38 INFO DAGScheduler:59 - ResultStage 11 (apply at Transformer.scala:22) finished in 0.013 s -2016-06-13 20:00:38 INFO CheckpointWriter:59 - Saving checkpoint for time 9000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-9000' -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 20:00:38 INFO DAGScheduler:59 - Job 5 finished: apply at Transformer.scala:22, took 0.089502 s -2016-06-13 20:00:38 INFO JobScheduler:59 - Finished job streaming job 3000 ms.1 from job set of time 3000 ms -2016-06-13 20:00:38 INFO JobScheduler:59 - Total delay: 1465819235.210 s for time 3000 ms (execution: 0.218 s) -2016-06-13 20:00:38 INFO ParallelCollectionRDD:59 - Removing RDD 6 from persistence list -2016-06-13 20:00:38 INFO JobScheduler:59 - Starting job streaming job 4000 ms.0 from job set of time 4000 ms -2016-06-13 20:00:38 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 6 -2016-06-13 20:00:38 INFO ShuffledRDD:59 - Removing RDD 11 from persistence list -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 20:00:38 INFO UnionRDD:59 - Removing RDD 10 from persistence list -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 11 -2016-06-13 20:00:38 INFO MapPartitionsRDD:59 - Removing RDD 8 from persistence list -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 10 -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 8 -2016-06-13 20:00:38 INFO JobGenerator:59 - Checkpointing graph for time 3000 ms -2016-06-13 20:00:38 INFO DStreamGraph:59 - Updating checkpoint data for time 3000 ms -2016-06-13 20:00:38 INFO DStreamGraph:59 - Updated checkpoint data for time 3000 ms -2016-06-13 20:00:38 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Registering RDD 69 (apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Got job 6 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:38 INFO DAGScheduler:59 - Final stage: ResultStage 13(apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (MapPartitionsRDD[69] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=76419, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(2484) called with curMem=81163, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:50497 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:00:38 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (MapPartitionsRDD[69] at apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 27, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 28, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:00:38 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 27) -2016-06-13 20:00:38 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 28) -2016-06-13 20:00:38 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 27). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 27) in 24 ms on localhost (1/2) -2016-06-13 20:00:38 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 28). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 28) in 27 ms on localhost (2/2) -2016-06-13 20:00:38 INFO DAGScheduler:59 - ShuffleMapStage 12 (apply at Transformer.scala:22) finished in 0.029 s -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 20:00:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-06-13 20:00:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:38 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting ResultStage 13 (ShuffledRDD[70] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=83647, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(2698) called with curMem=88911, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:50497 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:00:38 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (ShuffledRDD[70] at apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 29, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 30, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:38 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:00:38 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 29) -2016-06-13 20:00:38 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 30) -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:38 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 30). 1161 bytes result sent to driver -2016-06-13 20:00:38 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 29). 1498 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 30) in 17 ms on localhost (1/2) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 29) in 19 ms on localhost (2/2) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 20:00:38 INFO DAGScheduler:59 - ResultStage 13 (apply at Transformer.scala:22) finished in 0.020 s -2016-06-13 20:00:38 INFO DAGScheduler:59 - Job 6 finished: apply at Transformer.scala:22, took 0.082479 s -2016-06-13 20:00:38 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 200.0 and centroid: [-0.4768735011530239,0.9606386018112957,-1.673148729841545,1.2634974127814163,0.38189605546947214] -2016-06-13 20:00:38 INFO JobScheduler:59 - Finished job streaming job 4000 ms.0 from job set of time 4000 ms -2016-06-13 20:00:38 INFO JobScheduler:59 - Starting job streaming job 4000 ms.1 from job set of time 4000 ms -2016-06-13 20:00:38 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Registering RDD 22 (apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Got job 7 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:38 INFO DAGScheduler:59 - Final stage: ResultStage 15(apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 14) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 14) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 14 (UnionRDD[22] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=91609, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=94889, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:00:38 INFO CheckpointWriter:59 - Checkpoint for time 9000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-9000', took 27278 bytes and 120 ms -2016-06-13 20:00:38 INFO CheckpointWriter:59 - Saving checkpoint for time 10000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-10000' -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:50497 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:00:38 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 14 (UnionRDD[22] at apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 3 tasks -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 31, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 32, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:00:38 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 32) -2016-06-13 20:00:38 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 31) -2016-06-13 20:00:38 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:00:38 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 32). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 33, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 32) in 23 ms on localhost (1/3) -2016-06-13 20:00:38 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 33) -2016-06-13 20:00:38 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 31). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 31) in 35 ms on localhost (2/3) -2016-06-13 20:00:38 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 33). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 33) in 16 ms on localhost (3/3) -2016-06-13 20:00:38 INFO DAGScheduler:59 - ShuffleMapStage 14 (apply at Transformer.scala:22) finished in 0.039 s -2016-06-13 20:00:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 15) -2016-06-13 20:00:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 20:00:38 INFO DAGScheduler:59 - Missing parents for ResultStage 15: List() -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[71] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=96898, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(1801) called with curMem=100138, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1801.0 B, free 730.5 MB) -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:50497 (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:00:38 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[71] at apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 34, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 35, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:38 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 34) -2016-06-13 20:00:38 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 35) -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-06-13 20:00:38 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 35). 1172 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 35) in 14 ms on localhost (1/2) -2016-06-13 20:00:38 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:00:38 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 34). 1180 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 34) in 17 ms on localhost (2/2) -2016-06-13 20:00:38 INFO DAGScheduler:59 - ResultStage 15 (apply at Transformer.scala:22) finished in 0.017 s -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 20:00:38 INFO DAGScheduler:59 - Job 7 finished: apply at Transformer.scala:22, took 0.081239 s -2016-06-13 20:00:38 INFO JobScheduler:59 - Finished job streaming job 4000 ms.1 from job set of time 4000 ms -2016-06-13 20:00:38 INFO JobScheduler:59 - Total delay: 1465819234.400 s for time 4000 ms (execution: 0.189 s) -2016-06-13 20:00:38 INFO JobScheduler:59 - Starting job streaming job 5000 ms.0 from job set of time 5000 ms -2016-06-13 20:00:38 INFO ParallelCollectionRDD:59 - Removing RDD 13 from persistence list -2016-06-13 20:00:38 INFO MapPartitionsRDD:59 - Removing RDD 18 from persistence list -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 13 -2016-06-13 20:00:38 INFO ShuffledRDD:59 - Removing RDD 17 from persistence list -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 18 -2016-06-13 20:00:38 INFO UnionRDD:59 - Removing RDD 16 from persistence list -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 17 -2016-06-13 20:00:38 INFO MapPartitionsRDD:59 - Removing RDD 14 from persistence list -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 16 -2016-06-13 20:00:38 INFO JobGenerator:59 - Checkpointing graph for time 4000 ms -2016-06-13 20:00:38 INFO DStreamGraph:59 - Updating checkpoint data for time 4000 ms -2016-06-13 20:00:38 INFO DStreamGraph:59 - Updated checkpoint data for time 4000 ms -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 14 -2016-06-13 20:00:38 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Registering RDD 72 (apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Got job 8 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:38 INFO DAGScheduler:59 - Final stage: ResultStage 17(apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 16) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 16) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 16 (MapPartitionsRDD[72] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=101939, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(2476) called with curMem=106683, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:50497 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 20:00:38 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 16 (MapPartitionsRDD[72] at apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 36, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 37, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:00:38 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 36) -2016-06-13 20:00:38 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 37) -2016-06-13 20:00:38 INFO CheckpointWriter:59 - Checkpoint for time 10000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-10000', took 27290 bytes and 113 ms -2016-06-13 20:00:38 INFO CheckpointWriter:59 - Saving checkpoint for time 1000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-1000' -2016-06-13 20:00:38 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:00:38 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 37). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 37) in 19 ms on localhost (1/2) -2016-06-13 20:00:38 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 36). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 36) in 40 ms on localhost (2/2) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 20:00:38 INFO DAGScheduler:59 - ShuffleMapStage 16 (apply at Transformer.scala:22) finished in 0.041 s -2016-06-13 20:00:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 17) -2016-06-13 20:00:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:38 INFO DAGScheduler:59 - Missing parents for ResultStage 17: List() -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting ResultStage 17 (ShuffledRDD[73] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=109159, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=114423, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:50497 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 20:00:38 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (ShuffledRDD[73] at apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-06-13 20:00:38 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 38, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 39, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:38 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 39) -2016-06-13 20:00:38 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 38) -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:38 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 39). 1161 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 39) in 13 ms on localhost (1/2) -2016-06-13 20:00:38 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 38). 1498 bytes result sent to driver -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:50497 in memory (size: 2.4 KB, free: 730.5 MB) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 38) in 18 ms on localhost (2/2) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 20:00:38 INFO DAGScheduler:59 - ResultStage 17 (apply at Transformer.scala:22) finished in 0.025 s -2016-06-13 20:00:38 INFO DAGScheduler:59 - Job 8 finished: apply at Transformer.scala:22, took 0.121946 s -2016-06-13 20:00:38 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 250.0 and centroid: [-0.4770613340912709,0.9657053185238176,-1.6732261982319274,1.26651642628913,0.38386658573056776] -2016-06-13 20:00:38 INFO JobScheduler:59 - Finished job streaming job 5000 ms.0 from job set of time 5000 ms -2016-06-13 20:00:38 INFO JobScheduler:59 - Starting job streaming job 5000 ms.1 from job set of time 5000 ms -2016-06-13 20:00:38 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 20:00:38 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Registering RDD 28 (apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Got job 9 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:38 INFO DAGScheduler:59 - Final stage: ResultStage 19(apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 18) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 18) -2016-06-13 20:00:38 INFO ContextCleaner:59 - Cleaned shuffle 6 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 18 (UnionRDD[28] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=104847, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(2010) called with curMem=108127, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2010.0 B, free 730.5 MB) -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:50497 in memory (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:00:38 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:50497 (size: 2010.0 B, free: 730.5 MB) -2016-06-13 20:00:38 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 20:00:38 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 18 (UnionRDD[28] at apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 3 tasks -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 40, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 41, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:00:38 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 41) -2016-06-13 20:00:38 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 40) -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:50497 in memory (size: 2004.0 B, free: 730.6 MB) -2016-06-13 20:00:38 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 20:00:38 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 41). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 42, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:00:38 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 42) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 41) in 29 ms on localhost (1/3) -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:50497 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:00:38 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 40). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 40) in 38 ms on localhost (2/3) -2016-06-13 20:00:38 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 42). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 42) in 14 ms on localhost (3/3) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 20:00:38 INFO DAGScheduler:59 - ShuffleMapStage 18 (apply at Transformer.scala:22) finished in 0.043 s -2016-06-13 20:00:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 19) -2016-06-13 20:00:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:38 INFO DAGScheduler:59 - Missing parents for ResultStage 19: List() -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[74] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:50497 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:00:38 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=89674, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:00:38 INFO ContextCleaner:59 - Cleaned shuffle 4 -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=87872, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:50497 in memory (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:00:38 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:50497 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:00:38 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[74] at apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:50497 in memory (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:00:38 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-1000.bk -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 43, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 44, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:38 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 43) -2016-06-13 20:00:38 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 44) -2016-06-13 20:00:38 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:50497 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:38 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 20:00:38 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 44). 1172 bytes result sent to driver -2016-06-13 20:00:38 INFO CheckpointWriter:59 - Checkpoint for time 1000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-1000', took 27471 bytes and 194 ms -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 44) in 10 ms on localhost (1/2) -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:50497 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:00:38 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 20:00:38 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 43). 1180 bytes result sent to driver -2016-06-13 20:00:38 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 43) in 16 ms on localhost (2/2) -2016-06-13 20:00:38 INFO DAGScheduler:59 - ResultStage 19 (apply at Transformer.scala:22) finished in 0.017 s -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 20:00:38 INFO DAGScheduler:59 - Job 9 finished: apply at Transformer.scala:22, took 0.093263 s -2016-06-13 20:00:38 INFO JobScheduler:59 - Finished job streaming job 5000 ms.1 from job set of time 5000 ms -2016-06-13 20:00:38 INFO JobScheduler:59 - Total delay: 1465819233.643 s for time 5000 ms (execution: 0.242 s) -2016-06-13 20:00:38 INFO JobScheduler:59 - Starting job streaming job 6000 ms.0 from job set of time 6000 ms -2016-06-13 20:00:38 INFO ParallelCollectionRDD:59 - Removing RDD 19 from persistence list -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:50497 in memory (size: 1799.0 B, free: 730.6 MB) -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 19 -2016-06-13 20:00:38 INFO MapPartitionsRDD:59 - Removing RDD 24 from persistence list -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 24 -2016-06-13 20:00:38 INFO CheckpointWriter:59 - Saving checkpoint for time 2000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-2000' -2016-06-13 20:00:38 INFO ShuffledRDD:59 - Removing RDD 23 from persistence list -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:50497 in memory (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 23 -2016-06-13 20:00:38 INFO UnionRDD:59 - Removing RDD 22 from persistence list -2016-06-13 20:00:38 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 20:00:38 INFO MapPartitionsRDD:59 - Removing RDD 20 from persistence list -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 22 -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:50497 in memory (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 20 -2016-06-13 20:00:38 INFO DStreamGraph:59 - Clearing checkpoint data for time 1000 ms -2016-06-13 20:00:38 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 20:00:38 INFO DStreamGraph:59 - Cleared checkpoint data for time 1000 ms -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:50497 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:00:38 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:00:38 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:00:38 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Registering RDD 75 (apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Got job 10 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:38 INFO DAGScheduler:59 - Final stage: ResultStage 21(apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 20) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 20) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 20 (MapPartitionsRDD[75] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=45884, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=50628, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:50497 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:00:38 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 20 (MapPartitionsRDD[75] at apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-06-13 20:00:38 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/receivedBlockMetadata older than 0: -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 45, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 46, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:00:38 INFO InputInfoTracker:59 - remove old batch metadata: -2016-06-13 20:00:38 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 46) -2016-06-13 20:00:38 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 45) -2016-06-13 20:00:38 INFO JobGenerator:59 - Checkpointing graph for time 5000 ms -2016-06-13 20:00:38 INFO DStreamGraph:59 - Updating checkpoint data for time 5000 ms -2016-06-13 20:00:38 INFO DStreamGraph:59 - Updated checkpoint data for time 5000 ms -2016-06-13 20:00:38 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/receivedBlockMetadata older than 0 -2016-06-13 20:00:38 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 45). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 45) in 23 ms on localhost (1/2) -2016-06-13 20:00:38 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 46). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 46) in 32 ms on localhost (2/2) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 20:00:38 INFO DAGScheduler:59 - ShuffleMapStage 20 (apply at Transformer.scala:22) finished in 0.034 s -2016-06-13 20:00:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:38 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:00:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 21) -2016-06-13 20:00:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:38 INFO DAGScheduler:59 - Missing parents for ResultStage 21: List() -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting ResultStage 21 (ShuffledRDD[76] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=53110, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=58374, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:50497 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:00:38 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (ShuffledRDD[76] at apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:38 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 48) -2016-06-13 20:00:38 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 47) -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:38 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 48). 1161 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 48) in 14 ms on localhost (1/2) -2016-06-13 20:00:38 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 47). 1498 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 47) in 16 ms on localhost (2/2) -2016-06-13 20:00:38 INFO DAGScheduler:59 - ResultStage 21 (apply at Transformer.scala:22) finished in 0.017 s -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 20:00:38 INFO DAGScheduler:59 - Job 10 finished: apply at Transformer.scala:22, took 0.078824 s -2016-06-13 20:00:38 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 300.0 and centroid: [-0.4778380287460108,0.9658292271340416,-1.6712646630450996,1.273358698745343,0.37938624993358266] -2016-06-13 20:00:38 INFO JobScheduler:59 - Finished job streaming job 6000 ms.0 from job set of time 6000 ms -2016-06-13 20:00:38 INFO JobScheduler:59 - Starting job streaming job 6000 ms.1 from job set of time 6000 ms -2016-06-13 20:00:38 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Registering RDD 35 (apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Got job 11 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:38 INFO DAGScheduler:59 - Final stage: ResultStage 23(apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 22 (UnionRDD[35] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=61070, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=64350, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.5 MB) -2016-06-13 20:00:38 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:50497 (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:00:38 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 22 (UnionRDD[35] at apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 3 tasks -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 49, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 50, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:00:38 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 49) -2016-06-13 20:00:38 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 50) -2016-06-13 20:00:38 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 49). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 51, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:00:38 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 51) -2016-06-13 20:00:38 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 50). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 50) in 33 ms on localhost (1/3) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 49) in 37 ms on localhost (2/3) -2016-06-13 20:00:38 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 51). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO DAGScheduler:59 - ShuffleMapStage 22 (apply at Transformer.scala:22) finished in 0.041 s -2016-06-13 20:00:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 23) -2016-06-13 20:00:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:38 INFO DAGScheduler:59 - Missing parents for ResultStage 23: List() -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[77] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=66357, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=69597, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 51) in 15 ms on localhost (3/3) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:50497 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:00:38 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[77] at apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 52, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:38 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 53) -2016-06-13 20:00:38 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 52) -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:38 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 53). 1172 bytes result sent to driver -2016-06-13 20:00:38 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 52). 1180 bytes result sent to driver -2016-06-13 20:00:38 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-1000 -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 52) in 18 ms on localhost (1/2) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 53) in 19 ms on localhost (2/2) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 20:00:38 INFO DAGScheduler:59 - ResultStage 23 (apply at Transformer.scala:22) finished in 0.012 s -2016-06-13 20:00:38 INFO DAGScheduler:59 - Job 11 finished: apply at Transformer.scala:22, took 0.094511 s -2016-06-13 20:00:38 INFO CheckpointWriter:59 - Checkpoint for time 2000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-2000', took 27484 bytes and 203 ms -2016-06-13 20:00:38 INFO CheckpointWriter:59 - Saving checkpoint for time 3000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-3000' -2016-06-13 20:00:38 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:38 INFO JobScheduler:59 - Finished job streaming job 6000 ms.1 from job set of time 6000 ms -2016-06-13 20:00:38 INFO JobScheduler:59 - Total delay: 1465819232.867 s for time 6000 ms (execution: 0.224 s) -2016-06-13 20:00:38 INFO JobScheduler:59 - Starting job streaming job 7000 ms.0 from job set of time 7000 ms -2016-06-13 20:00:38 INFO DAGScheduler:59 - Registering RDD 78 (apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Got job 12 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:38 INFO DAGScheduler:59 - Final stage: ResultStage 25(apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 24) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 24) -2016-06-13 20:00:38 INFO DStreamGraph:59 - Clearing checkpoint data for time 2000 ms -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 24 (MapPartitionsRDD[78] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:38 INFO DStreamGraph:59 - Cleared checkpoint data for time 2000 ms -2016-06-13 20:00:38 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:00:38 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/receivedBlockMetadata older than 1000: -2016-06-13 20:00:38 INFO InputInfoTracker:59 - remove old batch metadata: -2016-06-13 20:00:38 INFO ParallelCollectionRDD:59 - Removing RDD 25 from persistence list -2016-06-13 20:00:38 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:00:38 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/receivedBlockMetadata older than 1000 -2016-06-13 20:00:38 INFO MapPartitionsRDD:59 - Removing RDD 30 from persistence list -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 25 -2016-06-13 20:00:38 INFO ShuffledRDD:59 - Removing RDD 29 from persistence list -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 29 -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 30 -2016-06-13 20:00:38 INFO UnionRDD:59 - Removing RDD 28 from persistence list -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 28 -2016-06-13 20:00:38 INFO MapPartitionsRDD:59 - Removing RDD 26 from persistence list -2016-06-13 20:00:38 INFO JobGenerator:59 - Checkpointing graph for time 6000 ms -2016-06-13 20:00:38 INFO DStreamGraph:59 - Updating checkpoint data for time 6000 ms -2016-06-13 20:00:38 INFO DStreamGraph:59 - Updated checkpoint data for time 6000 ms -2016-06-13 20:00:38 INFO BlockManager:59 - Removing RDD 26 -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=71400, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(2480) called with curMem=76144, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:50497 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:00:38 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 24 (MapPartitionsRDD[78] at apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 54, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 55, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:00:38 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 54) -2016-06-13 20:00:38 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 55) -2016-06-13 20:00:38 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:00:38 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 55). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 54). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 55) in 24 ms on localhost (1/2) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 54) in 27 ms on localhost (2/2) -2016-06-13 20:00:38 INFO DAGScheduler:59 - ShuffleMapStage 24 (apply at Transformer.scala:22) finished in 0.027 s -2016-06-13 20:00:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 25) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-06-13 20:00:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:38 INFO DAGScheduler:59 - Missing parents for ResultStage 25: List() -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting ResultStage 25 (ShuffledRDD[79] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=78624, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=83888, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:50497 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:00:38 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (ShuffledRDD[79] at apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:38 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 56) -2016-06-13 20:00:38 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 57) -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:00:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:38 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 57). 1161 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 57) in 7 ms on localhost (1/2) -2016-06-13 20:00:38 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 56). 1498 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 56) in 10 ms on localhost (2/2) -2016-06-13 20:00:38 INFO DAGScheduler:59 - ResultStage 25 (apply at Transformer.scala:22) finished in 0.010 s -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-06-13 20:00:38 INFO DAGScheduler:59 - Job 12 finished: apply at Transformer.scala:22, took 0.091425 s -2016-06-13 20:00:38 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 350.0 and centroid: [-0.47440048489638564,0.9650523283036628,-1.6700055274657652,1.269947562552048,0.3797915177328754] -2016-06-13 20:00:38 INFO JobScheduler:59 - Finished job streaming job 7000 ms.0 from job set of time 7000 ms -2016-06-13 20:00:38 INFO JobScheduler:59 - Starting job streaming job 7000 ms.1 from job set of time 7000 ms -2016-06-13 20:00:38 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Registering RDD 41 (apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Got job 13 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:38 INFO DAGScheduler:59 - Final stage: ResultStage 27(apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (UnionRDD[41] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=86584, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:38 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=89864, maxMem=766075207 -2016-06-13 20:00:38 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:00:38 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:00:38 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:50497 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:00:38 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:38 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 26 (UnionRDD[41] at apply at Transformer.scala:22) -2016-06-13 20:00:38 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 3 tasks -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 58, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 59, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:00:38 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 59) -2016-06-13 20:00:38 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 58) -2016-06-13 20:00:38 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 59). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 58). 1159 bytes result sent to driver -2016-06-13 20:00:38 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 60, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:00:38 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 60) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 59) in 17 ms on localhost (1/3) -2016-06-13 20:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 58) in 18 ms on localhost (2/3) -2016-06-13 20:00:39 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 60). 1159 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 60) in 9 ms on localhost (3/3) -2016-06-13 20:00:39 INFO DAGScheduler:59 - ShuffleMapStage 26 (apply at Transformer.scala:22) finished in 0.025 s -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-06-13 20:00:39 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:39 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:39 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-06-13 20:00:39 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:39 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[80] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=91873, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(1801) called with curMem=95113, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 1801.0 B, free 730.5 MB) -2016-06-13 20:00:39 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:50497 (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:00:39 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[80] at apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 62, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:39 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 61) -2016-06-13 20:00:39 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 62) -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:39 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 62). 1172 bytes result sent to driver -2016-06-13 20:00:39 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-2000.bk -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 62) in 7 ms on localhost (1/2) -2016-06-13 20:00:39 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 61). 1180 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 61) in 11 ms on localhost (2/2) -2016-06-13 20:00:39 INFO DAGScheduler:59 - ResultStage 27 (apply at Transformer.scala:22) finished in 0.012 s -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-06-13 20:00:39 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Job 13 finished: apply at Transformer.scala:22, took 0.062225 s -2016-06-13 20:00:39 INFO JobScheduler:59 - Finished job streaming job 7000 ms.1 from job set of time 7000 ms -2016-06-13 20:00:39 INFO JobScheduler:59 - Total delay: 1465819232.030 s for time 7000 ms (execution: 0.163 s) -2016-06-13 20:00:39 INFO JobScheduler:59 - Starting job streaming job 8000 ms.0 from job set of time 8000 ms -2016-06-13 20:00:39 INFO ParallelCollectionRDD:59 - Removing RDD 31 from persistence list -2016-06-13 20:00:39 INFO CheckpointWriter:59 - Checkpoint for time 3000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-3000', took 27495 bytes and 183 ms -2016-06-13 20:00:39 INFO CheckpointWriter:59 - Saving checkpoint for time 4000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-4000' -2016-06-13 20:00:39 INFO BlockManager:59 - Removing RDD 31 -2016-06-13 20:00:39 INFO MapPartitionsRDD:59 - Removing RDD 37 from persistence list -2016-06-13 20:00:39 INFO BlockManager:59 - Removing RDD 37 -2016-06-13 20:00:39 INFO ShuffledRDD:59 - Removing RDD 36 from persistence list -2016-06-13 20:00:39 INFO UnionRDD:59 - Removing RDD 35 from persistence list -2016-06-13 20:00:39 INFO BlockManager:59 - Removing RDD 36 -2016-06-13 20:00:39 INFO MapPartitionsRDD:59 - Removing RDD 33 from persistence list -2016-06-13 20:00:39 INFO BlockManager:59 - Removing RDD 35 -2016-06-13 20:00:39 INFO DStreamGraph:59 - Clearing checkpoint data for time 3000 ms -2016-06-13 20:00:39 INFO BlockManager:59 - Removing RDD 33 -2016-06-13 20:00:39 INFO DStreamGraph:59 - Cleared checkpoint data for time 3000 ms -2016-06-13 20:00:39 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:00:39 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/receivedBlockMetadata older than 2000: -2016-06-13 20:00:39 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/receivedBlockMetadata older than 2000 -2016-06-13 20:00:39 INFO InputInfoTracker:59 - remove old batch metadata: 1000 ms -2016-06-13 20:00:39 INFO JobGenerator:59 - Checkpointing graph for time 7000 ms -2016-06-13 20:00:39 INFO DStreamGraph:59 - Updating checkpoint data for time 7000 ms -2016-06-13 20:00:39 INFO DStreamGraph:59 - Updated checkpoint data for time 7000 ms -2016-06-13 20:00:39 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Registering RDD 81 (apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Got job 14 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:39 INFO DAGScheduler:59 - Final stage: ResultStage 29(apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 28) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 28) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting ShuffleMapStage 28 (MapPartitionsRDD[81] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=96914, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=101658, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:00:39 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:50497 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:00:39 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 28 (MapPartitionsRDD[81] at apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 63, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 64, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:00:39 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 63) -2016-06-13 20:00:39 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 64) -2016-06-13 20:00:39 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 64). 1159 bytes result sent to driver -2016-06-13 20:00:39 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 64) in 23 ms on localhost (1/2) -2016-06-13 20:00:39 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 63). 1159 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 63) in 28 ms on localhost (2/2) -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-06-13 20:00:39 INFO DAGScheduler:59 - ShuffleMapStage 28 (apply at Transformer.scala:22) finished in 0.029 s -2016-06-13 20:00:39 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:39 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:39 INFO DAGScheduler:59 - waiting: Set(ResultStage 29) -2016-06-13 20:00:39 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:39 INFO DAGScheduler:59 - Missing parents for ResultStage 29: List() -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting ResultStage 29 (ShuffledRDD[82] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=104140, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=109404, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:00:39 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:50497 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 20:00:39 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (ShuffledRDD[82] at apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 65, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 66, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:39 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 66) -2016-06-13 20:00:39 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 65) -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:39 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 66). 1161 bytes result sent to driver -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 66) in 9 ms on localhost (1/2) -2016-06-13 20:00:39 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 65). 1498 bytes result sent to driver -2016-06-13 20:00:39 INFO DAGScheduler:59 - ResultStage 29 (apply at Transformer.scala:22) finished in 0.020 s -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 65) in 16 ms on localhost (2/2) -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 20:00:39 INFO DAGScheduler:59 - Job 14 finished: apply at Transformer.scala:22, took 0.073117 s -2016-06-13 20:00:39 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 400.0 and centroid: [-0.4716182064591067,0.967586334056103,-1.6699954778983372,1.270341503100185,0.37935009425710725] -2016-06-13 20:00:39 INFO JobScheduler:59 - Finished job streaming job 8000 ms.0 from job set of time 8000 ms -2016-06-13 20:00:39 INFO JobScheduler:59 - Starting job streaming job 8000 ms.1 from job set of time 8000 ms -2016-06-13 20:00:39 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Registering RDD 47 (apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Got job 15 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:39 INFO DAGScheduler:59 - Final stage: ResultStage 31(apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 30) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting ShuffleMapStage 30 (UnionRDD[47] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:39 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=112098, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(2006) called with curMem=115378, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2006.0 B, free 730.5 MB) -2016-06-13 20:00:39 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:50497 (size: 2006.0 B, free: 730.5 MB) -2016-06-13 20:00:39 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 30 (UnionRDD[47] at apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 3 tasks -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 67, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 68, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:00:39 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 67) -2016-06-13 20:00:39 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 68) -2016-06-13 20:00:39 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 67). 1159 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 69, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:00:39 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 69) -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 67) in 15 ms on localhost (1/3) -2016-06-13 20:00:39 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 69). 1159 bytes result sent to driver -2016-06-13 20:00:39 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 68). 1159 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 69) in 12 ms on localhost (2/3) -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 68) in 26 ms on localhost (3/3) -2016-06-13 20:00:39 INFO DAGScheduler:59 - ShuffleMapStage 30 (apply at Transformer.scala:22) finished in 0.027 s -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 20:00:39 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:39 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:39 INFO DAGScheduler:59 - waiting: Set(ResultStage 31) -2016-06-13 20:00:39 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:39 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List() -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[83] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=117384, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=120624, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.5 MB) -2016-06-13 20:00:39 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:50497 (size: 1802.0 B, free: 730.5 MB) -2016-06-13 20:00:39 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[83] at apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 70, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 71, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:39 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 71) -2016-06-13 20:00:39 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 70) -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:39 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 71). 1172 bytes result sent to driver -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:39 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 71) in 5 ms on localhost (1/2) -2016-06-13 20:00:39 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 70). 1180 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 70) in 12 ms on localhost (2/2) -2016-06-13 20:00:39 INFO DAGScheduler:59 - ResultStage 31 (apply at Transformer.scala:22) finished in 0.013 s -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 20:00:39 INFO DAGScheduler:59 - Job 15 finished: apply at Transformer.scala:22, took 0.062973 s -2016-06-13 20:00:39 INFO JobScheduler:59 - Finished job streaming job 8000 ms.1 from job set of time 8000 ms -2016-06-13 20:00:39 INFO JobScheduler:59 - Total delay: 1465819231.189 s for time 8000 ms (execution: 0.158 s) -2016-06-13 20:00:39 INFO JobScheduler:59 - Starting job streaming job 9000 ms.0 from job set of time 9000 ms -2016-06-13 20:00:39 INFO ParallelCollectionRDD:59 - Removing RDD 38 from persistence list -2016-06-13 20:00:39 INFO MapPartitionsRDD:59 - Removing RDD 43 from persistence list -2016-06-13 20:00:39 INFO BlockManager:59 - Removing RDD 38 -2016-06-13 20:00:39 INFO ShuffledRDD:59 - Removing RDD 42 from persistence list -2016-06-13 20:00:39 INFO BlockManager:59 - Removing RDD 43 -2016-06-13 20:00:39 INFO UnionRDD:59 - Removing RDD 41 from persistence list -2016-06-13 20:00:39 INFO BlockManager:59 - Removing RDD 42 -2016-06-13 20:00:39 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-2000 -2016-06-13 20:00:39 INFO MapPartitionsRDD:59 - Removing RDD 39 from persistence list -2016-06-13 20:00:39 INFO JobGenerator:59 - Checkpointing graph for time 8000 ms -2016-06-13 20:00:39 INFO DStreamGraph:59 - Updating checkpoint data for time 8000 ms -2016-06-13 20:00:39 INFO DStreamGraph:59 - Updated checkpoint data for time 8000 ms -2016-06-13 20:00:39 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Registering RDD 84 (apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO BlockManager:59 - Removing RDD 39 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Got job 16 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:39 INFO DAGScheduler:59 - Final stage: ResultStage 33(apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 32) -2016-06-13 20:00:39 INFO BlockManager:59 - Removing RDD 41 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting ShuffleMapStage 32 (MapPartitionsRDD[84] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:39 INFO CheckpointWriter:59 - Checkpoint for time 4000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-4000', took 27487 bytes and 180 ms -2016-06-13 20:00:39 INFO CheckpointWriter:59 - Saving checkpoint for time 5000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-5000' -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=122426, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:00:39 INFO DStreamGraph:59 - Clearing checkpoint data for time 4000 ms -2016-06-13 20:00:39 INFO DStreamGraph:59 - Cleared checkpoint data for time 4000 ms -2016-06-13 20:00:39 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:00:39 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/receivedBlockMetadata older than 3000: -2016-06-13 20:00:39 INFO InputInfoTracker:59 - remove old batch metadata: 2000 ms -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(2477) called with curMem=127170, maxMem=766075207 -2016-06-13 20:00:39 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/receivedBlockMetadata older than 3000 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:00:39 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:50497 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 20:00:39 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 32 (MapPartitionsRDD[84] at apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 72, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 73, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:00:39 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 72) -2016-06-13 20:00:39 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 73) -2016-06-13 20:00:39 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 72). 1159 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 72) in 15 ms on localhost (1/2) -2016-06-13 20:00:39 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:00:39 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 73). 1159 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 73) in 33 ms on localhost (2/2) -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 20:00:39 INFO DAGScheduler:59 - ShuffleMapStage 32 (apply at Transformer.scala:22) finished in 0.035 s -2016-06-13 20:00:39 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:39 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:39 INFO DAGScheduler:59 - waiting: Set(ResultStage 33) -2016-06-13 20:00:39 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:39 INFO DAGScheduler:59 - Missing parents for ResultStage 33: List() -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting ResultStage 33 (ShuffledRDD[85] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=129647, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=134911, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:00:39 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:50497 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 20:00:39 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (ShuffledRDD[85] at apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 74, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 75, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:39 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 74) -2016-06-13 20:00:39 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 75) -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:39 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 75). 1161 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 75) in 7 ms on localhost (1/2) -2016-06-13 20:00:39 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 74). 1498 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 74) in 14 ms on localhost (2/2) -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 20:00:39 INFO DAGScheduler:59 - ResultStage 33 (apply at Transformer.scala:22) finished in 0.016 s -2016-06-13 20:00:39 INFO DAGScheduler:59 - Job 16 finished: apply at Transformer.scala:22, took 0.075470 s -2016-06-13 20:00:39 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 450.0 and centroid: [-0.4722098164150225,0.9642763266691793,-1.6666072662538036,1.2709270669491914,0.37911973669212223] -2016-06-13 20:00:39 INFO JobScheduler:59 - Finished job streaming job 9000 ms.0 from job set of time 9000 ms -2016-06-13 20:00:39 INFO JobScheduler:59 - Starting job streaming job 9000 ms.1 from job set of time 9000 ms -2016-06-13 20:00:39 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:00:39 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Registering RDD 53 (apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Got job 17 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:39 INFO DAGScheduler:59 - Final stage: ResultStage 35(apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 34) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 34) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting ShuffleMapStage 34 (UnionRDD[53] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=137605, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=140885, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.4 MB) -2016-06-13 20:00:39 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:50497 (size: 2007.0 B, free: 730.5 MB) -2016-06-13 20:00:39 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 34 (UnionRDD[53] at apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 3 tasks -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 76, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 77, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:00:39 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 76) -2016-06-13 20:00:39 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 77) -2016-06-13 20:00:39 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 76). 1159 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 78, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:00:39 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 78) -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 76) in 16 ms on localhost (1/3) -2016-06-13 20:00:39 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 77). 1159 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 77) in 23 ms on localhost (2/3) -2016-06-13 20:00:39 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 78). 1159 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 78) in 13 ms on localhost (3/3) -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-06-13 20:00:39 INFO DAGScheduler:59 - ShuffleMapStage 34 (apply at Transformer.scala:22) finished in 0.028 s -2016-06-13 20:00:39 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:39 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:39 INFO DAGScheduler:59 - waiting: Set(ResultStage 35) -2016-06-13 20:00:39 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:39 INFO DAGScheduler:59 - Missing parents for ResultStage 35: List() -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[86] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=142892, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=146132, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.4 MB) -2016-06-13 20:00:39 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:50497 (size: 1803.0 B, free: 730.5 MB) -2016-06-13 20:00:39 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[86] at apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-06-13 20:00:39 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 79, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 80, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:39 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 79) -2016-06-13 20:00:39 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 80) -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:39 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 80). 1172 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 80) in 7 ms on localhost (1/2) -2016-06-13 20:00:39 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-3000.bk -2016-06-13 20:00:39 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 79). 1180 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 79) in 13 ms on localhost (2/2) -2016-06-13 20:00:39 INFO DAGScheduler:59 - ResultStage 35 (apply at Transformer.scala:22) finished in 0.013 s -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-06-13 20:00:39 INFO DAGScheduler:59 - Job 17 finished: apply at Transformer.scala:22, took 0.070120 s -2016-06-13 20:00:39 INFO JobScheduler:59 - Finished job streaming job 9000 ms.1 from job set of time 9000 ms -2016-06-13 20:00:39 INFO JobScheduler:59 - Total delay: 1465819230.363 s for time 9000 ms (execution: 0.173 s) -2016-06-13 20:00:39 INFO JobScheduler:59 - Starting job streaming job 10000 ms.0 from job set of time 10000 ms -2016-06-13 20:00:39 INFO CheckpointWriter:59 - Checkpoint for time 5000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-5000', took 27483 bytes and 152 ms -2016-06-13 20:00:39 INFO CheckpointWriter:59 - Saving checkpoint for time 6000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-6000' -2016-06-13 20:00:39 INFO ParallelCollectionRDD:59 - Removing RDD 44 from persistence list -2016-06-13 20:00:39 INFO MapPartitionsRDD:59 - Removing RDD 49 from persistence list -2016-06-13 20:00:39 INFO BlockManager:59 - Removing RDD 44 -2016-06-13 20:00:39 INFO ShuffledRDD:59 - Removing RDD 48 from persistence list -2016-06-13 20:00:39 INFO BlockManager:59 - Removing RDD 48 -2016-06-13 20:00:39 INFO BlockManager:59 - Removing RDD 49 -2016-06-13 20:00:39 INFO UnionRDD:59 - Removing RDD 47 from persistence list -2016-06-13 20:00:39 INFO BlockManager:59 - Removing RDD 47 -2016-06-13 20:00:39 INFO MapPartitionsRDD:59 - Removing RDD 45 from persistence list -2016-06-13 20:00:39 INFO DStreamGraph:59 - Clearing checkpoint data for time 5000 ms -2016-06-13 20:00:39 INFO DStreamGraph:59 - Cleared checkpoint data for time 5000 ms -2016-06-13 20:00:39 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:00:39 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/receivedBlockMetadata older than 4000: -2016-06-13 20:00:39 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/receivedBlockMetadata older than 4000 -2016-06-13 20:00:39 INFO BlockManager:59 - Removing RDD 45 -2016-06-13 20:00:39 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:39 INFO InputInfoTracker:59 - remove old batch metadata: 3000 ms -2016-06-13 20:00:39 INFO JobGenerator:59 - Checkpointing graph for time 9000 ms -2016-06-13 20:00:39 INFO DStreamGraph:59 - Updating checkpoint data for time 9000 ms -2016-06-13 20:00:39 INFO DStreamGraph:59 - Updated checkpoint data for time 9000 ms -2016-06-13 20:00:39 INFO DAGScheduler:59 - Registering RDD 87 (apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Got job 18 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:39 INFO DAGScheduler:59 - Final stage: ResultStage 37(apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 36) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 36) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting ShuffleMapStage 36 (MapPartitionsRDD[87] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=147935, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 4.6 KB, free 730.4 MB) -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=152679, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 20:00:39 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:50497 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 20:00:39 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 36 (MapPartitionsRDD[87] at apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 81, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 82, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:00:39 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 82) -2016-06-13 20:00:39 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 81) -2016-06-13 20:00:39 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 82). 1159 bytes result sent to driver -2016-06-13 20:00:39 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 81). 1159 bytes result sent to driver -2016-06-13 20:00:39 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 82) in 22 ms on localhost (1/2) -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 81) in 24 ms on localhost (2/2) -2016-06-13 20:00:39 INFO DAGScheduler:59 - ShuffleMapStage 36 (apply at Transformer.scala:22) finished in 0.025 s -2016-06-13 20:00:39 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:39 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:39 INFO DAGScheduler:59 - waiting: Set(ResultStage 37) -2016-06-13 20:00:39 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:39 INFO DAGScheduler:59 - Missing parents for ResultStage 37: List() -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting ResultStage 37 (ShuffledRDD[88] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=155161, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=160425, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-06-13 20:00:39 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:50497 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 20:00:39 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (ShuffledRDD[88] at apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 83, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 84, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:39 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 84) -2016-06-13 20:00:39 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 83) -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:39 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 84). 1161 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 84) in 8 ms on localhost (1/2) -2016-06-13 20:00:39 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 83). 1498 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 83) in 13 ms on localhost (2/2) -2016-06-13 20:00:39 INFO DAGScheduler:59 - ResultStage 37 (apply at Transformer.scala:22) finished in 0.013 s -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-06-13 20:00:39 INFO DAGScheduler:59 - Job 18 finished: apply at Transformer.scala:22, took 0.068839 s -2016-06-13 20:00:39 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 500.0 and centroid: [-0.4725511979691583,0.9644503899125422,-1.668776373542808,1.2721254429935838,0.37815209739836425] -2016-06-13 20:00:39 INFO JobScheduler:59 - Finished job streaming job 10000 ms.0 from job set of time 10000 ms -2016-06-13 20:00:39 INFO JobScheduler:59 - Starting job streaming job 10000 ms.1 from job set of time 10000 ms -2016-06-13 20:00:39 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Registering RDD 59 (apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Got job 19 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:39 INFO DAGScheduler:59 - Final stage: ResultStage 39(apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 38) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 38) -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting ShuffleMapStage 38 (UnionRDD[59] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=163121, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=166401, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.4 MB) -2016-06-13 20:00:39 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:50497 (size: 2007.0 B, free: 730.5 MB) -2016-06-13 20:00:39 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:00:39 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 38 (UnionRDD[59] at apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 3 tasks -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 85, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 86, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:00:39 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 85) -2016-06-13 20:00:39 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 86) -2016-06-13 20:00:39 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 85). 1159 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 87, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:00:39 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 87) -2016-06-13 20:00:39 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 86). 1159 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 85) in 13 ms on localhost (1/3) -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 86) in 14 ms on localhost (2/3) -2016-06-13 20:00:39 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 87). 1159 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 87) in 9 ms on localhost (3/3) -2016-06-13 20:00:39 INFO DAGScheduler:59 - ShuffleMapStage 38 (apply at Transformer.scala:22) finished in 0.020 s -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-06-13 20:00:39 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:39 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:39 INFO DAGScheduler:59 - waiting: Set(ResultStage 39) -2016-06-13 20:00:39 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:39 INFO DAGScheduler:59 - Missing parents for ResultStage 39: List() -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[89] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=168408, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:00:39 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=171648, maxMem=766075207 -2016-06-13 20:00:39 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.4 MB) -2016-06-13 20:00:39 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:50497 (size: 1803.0 B, free: 730.5 MB) -2016-06-13 20:00:39 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[89] at apply at Transformer.scala:22) -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:39 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 88) -2016-06-13 20:00:39 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 89) -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:00:39 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 89). 1172 bytes result sent to driver -2016-06-13 20:00:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 89) in 5 ms on localhost (1/2) -2016-06-13 20:00:39 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 88). 1180 bytes result sent to driver -2016-06-13 20:00:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 88) in 10 ms on localhost (2/2) -2016-06-13 20:00:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-06-13 20:00:39 INFO DAGScheduler:59 - ResultStage 39 (apply at Transformer.scala:22) finished in 0.011 s -2016-06-13 20:00:39 INFO DAGScheduler:59 - Job 19 finished: apply at Transformer.scala:22, took 0.060327 s -2016-06-13 20:00:39 INFO JobScheduler:59 - Finished job streaming job 10000 ms.1 from job set of time 10000 ms -2016-06-13 20:00:39 INFO JobScheduler:59 - Total delay: 1465819229.519 s for time 10000 ms (execution: 0.155 s) -2016-06-13 20:00:39 INFO ParallelCollectionRDD:59 - Removing RDD 50 from persistence list -2016-06-13 20:00:39 INFO StreamingKMeansSuite:59 - Output generated in 3160 milliseconds -2016-06-13 20:00:39 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:00:39 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:00:39 INFO BlockManager:59 - Removing RDD 50 -2016-06-13 20:00:39 INFO MapPartitionsRDD:59 - Removing RDD 55 from persistence list -2016-06-13 20:00:39 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:00:39 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:00:39 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:00:39 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:00:39 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:00:39 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:00:39 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:00:39 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:00:39 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-3000 -2016-06-13 20:00:39 INFO ShuffledRDD:59 - Removing RDD 54 from persistence list -2016-06-13 20:00:39 INFO BlockManager:59 - Removing RDD 55 -2016-06-13 20:00:39 INFO UnionRDD:59 - Removing RDD 53 from persistence list -2016-06-13 20:00:39 INFO MapPartitionsRDD:59 - Removing RDD 51 from persistence list -2016-06-13 20:00:39 INFO BlockManager:59 - Removing RDD 53 -2016-06-13 20:00:39 INFO JobGenerator:59 - Checkpointing graph for time 10000 ms -2016-06-13 20:00:39 INFO DStreamGraph:59 - Updating checkpoint data for time 10000 ms -2016-06-13 20:00:39 INFO DStreamGraph:59 - Updated checkpoint data for time 10000 ms -2016-06-13 20:00:39 INFO BlockManager:59 - Removing RDD 51 -2016-06-13 20:00:39 INFO CheckpointWriter:59 - Checkpoint for time 6000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-6000', took 27479 bytes and 171 ms -2016-06-13 20:00:39 INFO BlockManager:59 - Removing RDD 54 -2016-06-13 20:00:39 INFO CheckpointWriter:59 - Saving checkpoint for time 7000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-7000' -2016-06-13 20:00:39 INFO DStreamGraph:59 - Clearing checkpoint data for time 6000 ms -2016-06-13 20:00:39 INFO DStreamGraph:59 - Cleared checkpoint data for time 6000 ms -2016-06-13 20:00:39 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:00:39 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/receivedBlockMetadata older than 5000: -2016-06-13 20:00:39 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/receivedBlockMetadata older than 5000 -2016-06-13 20:00:39 INFO InputInfoTracker:59 - remove old batch metadata: 4000 ms -2016-06-13 20:00:39 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-4000.bk -2016-06-13 20:00:39 INFO CheckpointWriter:59 - Checkpoint for time 7000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-7000', took 27477 bytes and 96 ms -2016-06-13 20:00:39 INFO DStreamGraph:59 - Clearing checkpoint data for time 7000 ms -2016-06-13 20:00:39 INFO CheckpointWriter:59 - Saving checkpoint for time 8000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-8000' -2016-06-13 20:00:39 INFO DStreamGraph:59 - Cleared checkpoint data for time 7000 ms -2016-06-13 20:00:39 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:00:39 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/receivedBlockMetadata older than 6000: -2016-06-13 20:00:39 INFO InputInfoTracker:59 - remove old batch metadata: 5000 ms -2016-06-13 20:00:39 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/receivedBlockMetadata older than 6000 -2016-06-13 20:00:39 INFO JobGenerator:59 - Stopping JobGenerator immediately -2016-06-13 20:00:39 INFO RecurringTimer:59 - Stopped timer for JobGenerator after time 10000 -2016-06-13 20:00:49 INFO CheckpointWriter:59 - CheckpointWriter executor terminated ? false, waited for 10002 ms. -2016-06-13 20:00:49 INFO JobGenerator:59 - Stopped JobGenerator -2016-06-13 20:00:49 INFO JobScheduler:59 - Stopped JobScheduler -2016-06-13 20:00:49 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-4000 -2016-06-13 20:00:49 INFO CheckpointWriter:59 - Checkpoint for time 8000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-8000', took 27463 bytes and 10133 ms -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/batch,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/streaming,null} -2016-06-13 20:00:49 INFO StreamingContext:59 - StreamingContext stopped successfully -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/batch/json,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/json,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:00:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:00:49 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 20:00:49 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:00:50 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:00:50 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:00:50 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:00:50 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:00:50 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:00:50 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:00:50 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:00:50 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:00:51 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 20:00:51 INFO StreamingKMeansSuite:59 - - -===== FINISHED o.a.s.mllib.clustering.kmeans.StreamingKMeansSuite: 'accuracy for single center and equivalence to grand average' ===== - -2016-06-13 20:00:51 WARN StreamingContext:71 - StreamingContext has already been stopped -2016-06-13 20:00:51 INFO SparkContext:59 - SparkContext already stopped. -2016-06-13 20:00:51 INFO StreamingKMeansSuite:59 - Using manual clock -2016-06-13 20:00:51 INFO StreamingKMeansSuite:59 - - -===== TEST OUTPUT FOR o.a.s.mllib.clustering.kmeans.StreamingKMeansSuite: 'accuracy for two centers' ===== - -2016-06-13 20:00:51 INFO StreamingKMeans:59 - Setting decay factor to: 0.707107 -2016-06-13 20:00:51 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:00:51 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:00:51 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:00:51 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:00:51 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:00:51 INFO Remoting:74 - Starting remoting -2016-06-13 20:00:51 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@localhost:50513] -2016-06-13 20:00:51 INFO Utils:59 - Successfully started service 'sparkDriver' on port 50513. -2016-06-13 20:00:51 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:00:51 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:00:51 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-1727d38b-fc9e-40d4-8e3b-dd53da1f8df2 -2016-06-13 20:00:51 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:00:51 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-f8eb2236-34b2-4105-aaff-6c04ac3f507c\httpd-3a668c5f-a7ae-40af-9fac-dad6a778f047 -2016-06-13 20:00:51 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:00:51 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:00:51 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:50514 -2016-06-13 20:00:51 INFO Utils:59 - Successfully started service 'HTTP file server' on port 50514. -2016-06-13 20:00:51 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:00:51 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:00:51 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 20:00:51 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 20:00:51 INFO SparkUI:59 - Started SparkUI at http://localhost:4040 -2016-06-13 20:00:51 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:00:51 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:00:51 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 50525. -2016-06-13 20:00:51 INFO NettyBlockTransferService:59 - Server created on 50525 -2016-06-13 20:00:51 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:00:51 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:50525 with 730.6 MB RAM, BlockManagerId(driver, localhost, 50525) -2016-06-13 20:00:51 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:00:51 INFO StreamingKMeansSuite:59 - numBatches = 10, numExpectedOutput = 10 -2016-06-13 20:00:51 INFO WriteAheadLogManager :59 - Recovered 1 write ahead log files from file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/receivedBlockMetadata -2016-06-13 20:00:51 INFO ForEachDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:00:51 INFO TestInputStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:00:51 INFO TestInputStream:59 - Slide time = 1000 ms -2016-06-13 20:00:51 INFO TestInputStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:00:51 INFO TestInputStream:59 - Checkpoint interval = null -2016-06-13 20:00:51 INFO TestInputStream:59 - Remember duration = 1000 ms -2016-06-13 20:00:51 INFO TestInputStream:59 - Initialized and validated org.apache.spark.streaming.TestInputStream@10e56ccf -2016-06-13 20:00:51 INFO ForEachDStream:59 - Slide time = 1000 ms -2016-06-13 20:00:51 INFO ForEachDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:00:51 INFO ForEachDStream:59 - Checkpoint interval = null -2016-06-13 20:00:51 INFO ForEachDStream:59 - Remember duration = 1000 ms -2016-06-13 20:00:51 INFO ForEachDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.ForEachDStream@7856b513 -2016-06-13 20:00:51 INFO TestOutputStreamWithPartitions:59 - metadataCleanupDelay = -1 -2016-06-13 20:00:51 INFO MappedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:00:51 INFO ShuffledDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:00:51 INFO TransformedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:00:51 INFO MappedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:00:51 INFO TestInputStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:00:51 INFO TestInputStream:59 - Slide time = 1000 ms -2016-06-13 20:00:51 INFO TestInputStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:00:51 INFO TestInputStream:59 - Checkpoint interval = null -2016-06-13 20:00:51 INFO TestInputStream:59 - Remember duration = 1000 ms -2016-06-13 20:00:51 INFO TestInputStream:59 - Initialized and validated org.apache.spark.streaming.TestInputStream@10e56ccf -2016-06-13 20:00:51 INFO MappedDStream:59 - Slide time = 1000 ms -2016-06-13 20:00:51 INFO MappedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:00:51 INFO MappedDStream:59 - Checkpoint interval = null -2016-06-13 20:00:51 INFO MappedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:00:51 INFO MappedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@1628ff62 -2016-06-13 20:00:51 INFO TransformedDStream:59 - Slide time = 1000 ms -2016-06-13 20:00:51 INFO TransformedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:00:51 INFO TransformedDStream:59 - Checkpoint interval = null -2016-06-13 20:00:51 INFO TransformedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:00:51 INFO TransformedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.TransformedDStream@6388062b -2016-06-13 20:00:51 INFO ShuffledDStream:59 - Slide time = 1000 ms -2016-06-13 20:00:51 INFO ShuffledDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:00:51 INFO ShuffledDStream:59 - Checkpoint interval = null -2016-06-13 20:00:51 INFO ShuffledDStream:59 - Remember duration = 1000 ms -2016-06-13 20:00:51 INFO ShuffledDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.ShuffledDStream@50b04f7b -2016-06-13 20:00:51 INFO MappedDStream:59 - Slide time = 1000 ms -2016-06-13 20:00:51 INFO MappedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:00:51 INFO MappedDStream:59 - Checkpoint interval = null -2016-06-13 20:00:51 INFO MappedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:00:51 INFO MappedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@11ffcdfc -2016-06-13 20:00:51 INFO TestOutputStreamWithPartitions:59 - Slide time = 1000 ms -2016-06-13 20:00:51 INFO TestOutputStreamWithPartitions:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:00:51 INFO TestOutputStreamWithPartitions:59 - Checkpoint interval = null -2016-06-13 20:00:51 INFO TestOutputStreamWithPartitions:59 - Remember duration = 1000 ms -2016-06-13 20:00:51 INFO TestOutputStreamWithPartitions:59 - Initialized and validated org.apache.spark.streaming.TestOutputStreamWithPartitions@571766c8 -2016-06-13 20:00:51 INFO RecurringTimer:59 - Started timer for JobGenerator at time 1000 -2016-06-13 20:00:51 INFO JobGenerator:59 - Started JobGenerator at 1000 ms -2016-06-13 20:00:51 INFO JobScheduler:59 - Started JobScheduler -2016-06-13 20:00:51 INFO StreamingContext:59 - StreamingContext started -2016-06-13 20:00:51 INFO StreamingKMeansSuite:59 - Manual clock before advancing = 0 -2016-06-13 20:00:51 INFO StreamingKMeansSuite:59 - Manual clock after advancing = 10000 -2016-06-13 20:00:51 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:51 INFO TestInputStream:59 - Computing RDD for time 1000 ms -2016-06-13 20:00:51 INFO TestInputStream:59 - Created RDD 0 with Vector([-0.45216537133396134,0.8040574757345051,-1.5589888701250443,1.115603017032467,0.344649973028188], [1.1281554448921418,0.5085279594712502,0.7868917870018506,-0.6391790333639171,2.0226179083827693], [-0.545300880562902,1.1093057253978866,-1.7582382472264182,1.211400006292075,0.2932044815946008], [1.1087747256708875,0.3678886251628949,0.6954219445119724,-0.6191601560856841,1.8100714463462038], [-0.5964001160322344,1.0378164371074625,-1.553411818459791,1.2193968564632125,0.4492703031123452]) -2016-06-13 20:00:51 INFO JobScheduler:59 - Added jobs for time 1000 ms -2016-06-13 20:00:51 INFO TestInputStream:59 - Computing RDD for time 2000 ms -2016-06-13 20:00:51 INFO JobScheduler:59 - Starting job streaming job 1000 ms.0 from job set of time 1000 ms -2016-06-13 20:00:51 INFO TestInputStream:59 - Created RDD 6 with Vector([-0.33949398265706626,1.1369769008642154,-1.7816495259375202,1.2857769527411067,0.5642853362323552], [1.0560293623085477,0.5295881616073689,0.9570136700549032,-0.6447176128345662,1.829624618639636], [-0.4895260213585818,1.024567290792244,-1.7744155605537828,1.4515329700677304,0.39720325477885093], [0.960390895188449,0.33146418432881175,0.8181710190393932,-0.5989320660721107,1.8353581950612912], [-0.5468296230191492,0.9034725811394896,-1.8269524875493084,1.161899613497355,0.22607191229030854]) -2016-06-13 20:00:51 INFO JobScheduler:59 - Added jobs for time 2000 ms -2016-06-13 20:00:51 INFO TestInputStream:59 - Computing RDD for time 3000 ms -2016-06-13 20:00:51 INFO TestInputStream:59 - Created RDD 14 with Vector([-0.4344367225435325,0.9054178856043291,-1.6658399540877864,1.13578899284944,0.46511292492021983], [0.9168629719099907,0.41154373690124263,0.6944437187564555,-0.5279862925682771,1.9425648788226417], [-0.5644425684165094,0.996354296088912,-1.817848647277835,1.3768030733201289,0.28609432985591315], [1.0668391723953785,0.464506909541895,0.7281904951302456,-0.6642971866289025,1.8602228158414527], [-0.5287113084613145,0.8475916475163896,-1.8166057479945195,1.238631411179877,0.36922519791770014]) -2016-06-13 20:00:51 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:51 INFO DAGScheduler:59 - Registering RDD 7 (apply at Transformer.scala:22) -2016-06-13 20:00:51 INFO DAGScheduler:59 - Got job 0 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:51 INFO DAGScheduler:59 - Final stage: ResultStage 1(apply at Transformer.scala:22) -2016-06-13 20:00:51 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 20:00:51 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 20:00:51 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[7] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:51 INFO MemoryStore:59 - ensureFreeSpace(4808) called with curMem=0, maxMem=766075207 -2016-06-13 20:00:51 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 4.7 KB, free 730.6 MB) -2016-06-13 20:00:51 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:51 INFO MemoryStore:59 - ensureFreeSpace(2484) called with curMem=4808, maxMem=766075207 -2016-06-13 20:00:51 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:00:51 INFO JobScheduler:59 - Added jobs for time 3000 ms -2016-06-13 20:00:51 INFO TestInputStream:59 - Computing RDD for time 4000 ms -2016-06-13 20:00:51 INFO TestInputStream:59 - Created RDD 20 with Vector([-0.2540595910515851,0.9698131032346381,-1.5866552625154318,1.2179915976157372,0.3240398576394729], [1.0056585577616686,0.548350668313521,0.7204438211614234,-0.5280545719598417,1.9014093110906123], [-0.4131172722481001,1.0096927045418442,-1.811286466550916,1.2719080737538961,0.28293757989913904], [1.0567440318885806,0.5323164594700811,0.6459492121644157,-0.5978675569816886,1.810417039844708], [-0.4171607573128071,1.0818260184642114,-1.8219543038308885,1.2496462030096327,0.4964658371791977]) -2016-06-13 20:00:51 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:50525 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:00:51 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:51 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[7] at apply at Transformer.scala:22) -2016-06-13 20:00:51 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 20:00:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2287 bytes) -2016-06-13 20:00:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2343 bytes) -2016-06-13 20:00:51 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 20:00:51 INFO JobScheduler:59 - Added jobs for time 4000 ms -2016-06-13 20:00:51 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 20:00:51 INFO TestInputStream:59 - Computing RDD for time 5000 ms -2016-06-13 20:00:51 INFO TestInputStream:59 - Created RDD 26 with Vector([-0.45586902589910655,1.0378742745036647,-1.7000458648640797,1.446592332373813,0.348566870310007], [1.1551142615857415,0.24321280065742099,0.7192970920013563,-0.5842777108198818,1.6904426482031396], [-0.5154063008996834,0.9595656827532482,-1.6519653379002484,1.322654750805598,0.3008929692388781], [1.111534284764859,0.22965698384418656,0.7113017134821596,-0.7040474101002699,1.8792509553556123], [-0.48352710482779404,1.0255449792765485,-1.7171397454598174,1.284877805985566,0.437232715780399]) -2016-06-13 20:00:51 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1159 bytes result sent to driver -2016-06-13 20:00:51 INFO JobScheduler:59 - Added jobs for time 5000 ms -2016-06-13 20:00:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 23 ms on localhost (1/2) -2016-06-13 20:00:51 INFO TestInputStream:59 - Computing RDD for time 6000 ms -2016-06-13 20:00:51 INFO TestInputStream:59 - Created RDD 32 with Vector([-0.40324856443707463,1.020485534999759,-1.6517518343613002,1.2864020019792224,0.3984832850799499], [0.8566160445501626,0.4082300221585019,0.7515444883210611,-0.7154152601984277,1.7124667753211176], [-0.6148278408211463,0.9795135452912322,-1.8642733722424907,1.2720240024566645,0.39785426587083933], [0.8673555574521561,0.6166274576488866,0.6260630913872013,-0.6767474882813775,1.6734479781251468], [-0.6368691555002903,0.8170131462860201,-1.6827833832635621,1.3031179618920097,0.31628673987022604]) -2016-06-13 20:00:51 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1159 bytes result sent to driver -2016-06-13 20:00:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 30 ms on localhost (2/2) -2016-06-13 20:00:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 20:00:51 INFO DAGScheduler:59 - ShuffleMapStage 0 (apply at Transformer.scala:22) finished in 0.034 s -2016-06-13 20:00:51 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:51 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:51 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 20:00:51 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:51 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 20:00:51 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[11] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:51 INFO JobScheduler:59 - Added jobs for time 6000 ms -2016-06-13 20:00:51 INFO TestInputStream:59 - Computing RDD for time 7000 ms -2016-06-13 20:00:51 INFO MemoryStore:59 - ensureFreeSpace(5328) called with curMem=7292, maxMem=766075207 -2016-06-13 20:00:51 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 5.2 KB, free 730.6 MB) -2016-06-13 20:00:51 INFO TestInputStream:59 - Created RDD 38 with Vector([-0.43096648399184834,0.9655746876954024,-1.7845188100637381,1.4052721272088269,0.5845842279495], [1.0641104552439702,0.4645562041474787,0.7113010551235865,-0.6436630296842897,1.9109693384398803], [-0.4527127515828855,0.9000469054331385,-1.7497092961121075,1.2451107418571954,0.4344380036579415], [0.9052316084504463,0.4087618985411995,0.7229552720838689,-0.5355918560714372,1.99442974701897], [-0.34976726689041593,0.9355870894961762,-1.5766155567170825,1.3590936489794139,0.35690680463154484]) -2016-06-13 20:00:51 INFO MemoryStore:59 - ensureFreeSpace(2706) called with curMem=12620, maxMem=766075207 -2016-06-13 20:00:51 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 20:00:51 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:51 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:50525 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:00:51 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:51 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[11] at apply at Transformer.scala:22) -2016-06-13 20:00:51 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 20:00:51 INFO JobScheduler:59 - Added jobs for time 7000 ms -2016-06-13 20:00:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:51 INFO TestInputStream:59 - Computing RDD for time 8000 ms -2016-06-13 20:00:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:51 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 20:00:51 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 20:00:51 INFO TestInputStream:59 - Created RDD 44 with Vector([-0.575628267406687,0.9538063357613722,-1.6445003616008003,1.2104804829930789,0.3911315521177748], [0.9500407756757224,0.3727384461721234,0.7844562972745359,-0.4780598011054681,1.8717573988160656], [-0.5468811641579405,0.997556943752324,-1.778985900216746,1.2440321529105447,0.4204585578447196], [1.0183809682493083,0.5143391009038458,0.9033557116388586,-0.7049384390304207,2.1071449963888123], [-0.5345824744371771,0.7659168558371742,-1.7009697565551642,1.2200491259133113,0.24064266183571642]) -2016-06-13 20:00:51 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 2 blocks -2016-06-13 20:00:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:51 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:51 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:51 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1498 bytes result sent to driver -2016-06-13 20:00:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 10 ms on localhost (1/2) -2016-06-13 20:00:51 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1498 bytes result sent to driver -2016-06-13 20:00:51 INFO JobScheduler:59 - Added jobs for time 8000 ms -2016-06-13 20:00:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 12 ms on localhost (2/2) -2016-06-13 20:00:51 INFO TestInputStream:59 - Computing RDD for time 9000 ms -2016-06-13 20:00:51 INFO DAGScheduler:59 - ResultStage 1 (apply at Transformer.scala:22) finished in 0.013 s -2016-06-13 20:00:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 20:00:51 INFO DAGScheduler:59 - Job 0 finished: apply at Transformer.scala:22, took 0.076786 s -2016-06-13 20:00:51 INFO TestInputStream:59 - Created RDD 50 with Vector([-0.4425411108434083,0.9636579302553172,-1.6769715787345627,1.2060504970822528,0.1832578105698996], [1.227932175895181,0.4201950420885885,0.7610696705447784,-0.4793116676543314,2.0417665475441082], [-0.4646639723438268,0.9422153316765177,-1.8936280948227913,1.1755014401606392,0.3269897050872188], [0.9288262863124409,0.44722132255436287,0.6260387822908252,-0.6128599152455255,1.8971892270362565], [-0.36564413314560745,0.9550404460199052,-1.671332555981891,1.2387784773127055,0.29524181369582553]) -2016-06-13 20:00:51 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 5.535533905932738 and centroid: [-0.27011927170852723,0.45174965876706324,-0.7259926350673479,0.24751662879474287,0.07025905734166736] -2016-06-13 20:00:51 INFO StreamingKMeansModel:59 - Cluster 1 updated with weight 6.535533905932738 and centroid: [0.3271833978064514,0.1489346231221466,-0.011732038990971834,0.08635417043083451,0.6932704785201166] -2016-06-13 20:00:51 INFO JobScheduler:59 - Finished job streaming job 1000 ms.0 from job set of time 1000 ms -2016-06-13 20:00:51 INFO JobScheduler:59 - Starting job streaming job 1000 ms.1 from job set of time 1000 ms -2016-06-13 20:00:51 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:51 INFO DAGScheduler:59 - Registering RDD 3 (apply at Transformer.scala:22) -2016-06-13 20:00:51 INFO DAGScheduler:59 - Got job 1 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:51 INFO DAGScheduler:59 - Final stage: ResultStage 3(apply at Transformer.scala:22) -2016-06-13 20:00:51 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 2) -2016-06-13 20:00:51 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 2) -2016-06-13 20:00:51 INFO JobScheduler:59 - Added jobs for time 9000 ms -2016-06-13 20:00:51 INFO TestInputStream:59 - Computing RDD for time 10000 ms -2016-06-13 20:00:51 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (UnionRDD[3] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:51 INFO TestInputStream:59 - Created RDD 57 with Vector([-0.4947387784852314,0.8968969114403345,-1.5032027608518188,1.263293789437244,0.41872133210307433], [0.9182067144969276,0.35835807735712344,0.665804201326562,-0.6948362587119044,1.8512989939199982], [-0.7175598778778954,0.8837946345395844,-1.8329343739105675,1.3562350855120264,0.30734388952604197], [1.0706268029648116,0.44341269208013045,0.8927617254725349,-0.8104964259001176,1.8155976948221841], [-0.5385984930004446,0.9989931115946847,-1.745310682734983,1.2763556288677276,0.41555544930746574]) -2016-06-13 20:00:51 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=15326, maxMem=766075207 -2016-06-13 20:00:51 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:00:51 INFO MemoryStore:59 - ensureFreeSpace(2000) called with curMem=18606, maxMem=766075207 -2016-06-13 20:00:51 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2000.0 B, free 730.6 MB) -2016-06-13 20:00:51 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:50525 (size: 2000.0 B, free: 730.6 MB) -2016-06-13 20:00:51 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:51 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 2 (UnionRDD[3] at apply at Transformer.scala:22) -2016-06-13 20:00:51 INFO JobScheduler:59 - Added jobs for time 10000 ms -2016-06-13 20:00:51 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-06-13 20:00:51 INFO JobGenerator:59 - Checkpointing graph for time 1000 ms -2016-06-13 20:00:51 INFO DStreamGraph:59 - Updating checkpoint data for time 1000 ms -2016-06-13 20:00:51 INFO DStreamGraph:59 - Updated checkpoint data for time 1000 ms -2016-06-13 20:00:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2396 bytes) -2016-06-13 20:00:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2452 bytes) -2016-06-13 20:00:51 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 20:00:51 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 20:00:51 INFO JobGenerator:59 - Checkpointing graph for time 2000 ms -2016-06-13 20:00:51 INFO DStreamGraph:59 - Updating checkpoint data for time 2000 ms -2016-06-13 20:00:51 INFO DStreamGraph:59 - Updated checkpoint data for time 2000 ms -2016-06-13 20:00:51 INFO JobGenerator:59 - Checkpointing graph for time 3000 ms -2016-06-13 20:00:51 INFO DStreamGraph:59 - Updating checkpoint data for time 3000 ms -2016-06-13 20:00:51 INFO DStreamGraph:59 - Updated checkpoint data for time 3000 ms -2016-06-13 20:00:51 INFO CheckpointWriter:59 - Saving checkpoint for time 1000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-1000' -2016-06-13 20:00:52 INFO JobGenerator:59 - Checkpointing graph for time 4000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updating checkpoint data for time 4000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updated checkpoint data for time 4000 ms -2016-06-13 20:00:52 INFO JobGenerator:59 - Checkpointing graph for time 5000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updating checkpoint data for time 5000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updated checkpoint data for time 5000 ms -2016-06-13 20:00:52 INFO JobGenerator:59 - Checkpointing graph for time 6000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updating checkpoint data for time 6000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updated checkpoint data for time 6000 ms -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO JobGenerator:59 - Checkpointing graph for time 7000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updating checkpoint data for time 7000 ms -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updated checkpoint data for time 7000 ms -2016-06-13 20:00:52 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO JobGenerator:59 - Checkpointing graph for time 8000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updating checkpoint data for time 8000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updated checkpoint data for time 8000 ms -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 17 ms on localhost (1/3) -2016-06-13 20:00:52 INFO JobGenerator:59 - Checkpointing graph for time 9000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updating checkpoint data for time 9000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updated checkpoint data for time 9000 ms -2016-06-13 20:00:52 INFO JobGenerator:59 - Checkpointing graph for time 10000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updating checkpoint data for time 10000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updated checkpoint data for time 10000 ms -2016-06-13 20:00:52 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 23 ms on localhost (2/3) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 6). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 6) in 12 ms on localhost (3/3) -2016-06-13 20:00:52 INFO DAGScheduler:59 - ShuffleMapStage 2 (apply at Transformer.scala:22) finished in 0.026 s -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:52 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - waiting: Set(ResultStage 3) -2016-06-13 20:00:52 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents for ResultStage 3: List() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[51] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=20606, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(1800) called with curMem=23846, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1800.0 B, free 730.6 MB) -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:50525 (size: 1800.0 B, free: 730.6 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[51] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 7) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 8) -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 8). 1172 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 8) in 12 ms on localhost (1/2) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 7). 1180 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 7) in 16 ms on localhost (2/2) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - ResultStage 3 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:00:52 INFO DAGScheduler:59 - Job 1 finished: apply at Transformer.scala:22, took 0.079995 s -2016-06-13 20:00:52 INFO JobScheduler:59 - Finished job streaming job 1000 ms.1 from job set of time 1000 ms -2016-06-13 20:00:52 INFO JobScheduler:59 - Total delay: 1465819251.064 s for time 1000 ms (execution: 0.182 s) -2016-06-13 20:00:52 INFO JobScheduler:59 - Starting job streaming job 2000 ms.0 from job set of time 2000 ms -2016-06-13 20:00:52 INFO JobGenerator:59 - Checkpointing graph for time 1000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updating checkpoint data for time 1000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updated checkpoint data for time 1000 ms -2016-06-13 20:00:52 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:00:52 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Registering RDD 63 (apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Got job 2 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:52 INFO DAGScheduler:59 - Final stage: ResultStage 5(apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[63] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(4808) called with curMem=25646, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.7 KB, free 730.6 MB) -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(2518) called with curMem=30454, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.6 MB) -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:50525 (size: 2.5 KB, free: 730.6 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[63] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2287 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 10, localhost, PROCESS_LOCAL, 2343 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 9) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 10) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 9). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 10). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 9) in 19 ms on localhost (1/2) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 10) in 20 ms on localhost (2/2) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - ShuffleMapStage 4 (apply at Transformer.scala:22) finished in 0.021 s -2016-06-13 20:00:52 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:52 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-06-13 20:00:52 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ResultStage 5 (ShuffledRDD[64] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(5328) called with curMem=32972, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.2 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(2745) called with curMem=38300, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:50525 (size: 2.7 KB, free: 730.6 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (ShuffledRDD[64] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 12, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 11) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 12) -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 11). 1498 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 11) in 7 ms on localhost (1/2) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 12). 1498 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 12) in 9 ms on localhost (2/2) -2016-06-13 20:00:52 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-1000 -2016-06-13 20:00:52 INFO DAGScheduler:59 - ResultStage 5 (apply at Transformer.scala:22) finished in 0.009 s -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - Job 2 finished: apply at Transformer.scala:22, took 0.059190 s -2016-06-13 20:00:52 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 6.914213562373095 and centroid: [-0.3519061310249073,0.6990327056488164,-1.1895362673150238,0.704063077851566,0.21153084804714006] -2016-06-13 20:00:52 INFO StreamingKMeansModel:59 - Cluster 1 updated with weight 6.621320343559642 and centroid: [0.5328906270615534,0.23399063468329673,0.2599129915693954,-0.1275545284808553,1.0373773542547826] -2016-06-13 20:00:52 INFO JobScheduler:59 - Finished job streaming job 2000 ms.0 from job set of time 2000 ms -2016-06-13 20:00:52 INFO JobScheduler:59 - Starting job streaming job 2000 ms.1 from job set of time 2000 ms -2016-06-13 20:00:52 INFO CheckpointWriter:59 - Checkpoint for time 1000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-1000', took 7349 bytes and 144 ms -2016-06-13 20:00:52 INFO CheckpointWriter:59 - Saving checkpoint for time 2000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-2000' -2016-06-13 20:00:52 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Registering RDD 10 (apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Got job 3 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:52 INFO DAGScheduler:59 - Final stage: ResultStage 7(apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (UnionRDD[10] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=41045, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=44325, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:50525 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 6 (UnionRDD[10] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 3 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2396 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 14, localhost, PROCESS_LOCAL, 2452 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 13) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 14) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 14). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 13). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 15, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 15) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 14) in 13 ms on localhost (1/3) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 13) in 15 ms on localhost (2/3) -2016-06-13 20:00:52 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:00:52 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 15). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO DAGScheduler:59 - ShuffleMapStage 6 (apply at Transformer.scala:22) finished in 0.027 s -2016-06-13 20:00:52 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:52 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-06-13 20:00:52 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[65] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 15) in 14 ms on localhost (3/3) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=46334, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=49574, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.5 MB) -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:50525 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[65] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 17) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 16) -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 17). 1172 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 17) in 8 ms on localhost (1/2) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 16). 1180 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 16) in 13 ms on localhost (2/2) -2016-06-13 20:00:52 INFO DAGScheduler:59 - ResultStage 7 (apply at Transformer.scala:22) finished in 0.010 s -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - Job 3 finished: apply at Transformer.scala:22, took 0.069042 s -2016-06-13 20:00:52 INFO JobScheduler:59 - Finished job streaming job 2000 ms.1 from job set of time 2000 ms -2016-06-13 20:00:52 INFO ParallelCollectionRDD:59 - Removing RDD 0 from persistence list -2016-06-13 20:00:52 INFO JobScheduler:59 - Total delay: 1465819250.214 s for time 2000 ms (execution: 0.149 s) -2016-06-13 20:00:52 INFO JobScheduler:59 - Starting job streaming job 3000 ms.0 from job set of time 3000 ms -2016-06-13 20:00:52 INFO MapPartitionsRDD:59 - Removing RDD 5 from persistence list -2016-06-13 20:00:52 INFO ShuffledRDD:59 - Removing RDD 4 from persistence list -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 5 -2016-06-13 20:00:52 INFO UnionRDD:59 - Removing RDD 3 from persistence list -2016-06-13 20:00:52 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:52 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Registering RDD 66 (apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 3 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Got job 4 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 0 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Final stage: ResultStage 9(apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 8) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 8) -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 4 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ShuffleMapStage 8 (MapPartitionsRDD[66] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:52 INFO MapPartitionsRDD:59 - Removing RDD 1 from persistence list -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 1 -2016-06-13 20:00:52 INFO JobGenerator:59 - Checkpointing graph for time 2000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updating checkpoint data for time 2000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updated checkpoint data for time 2000 ms -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(4808) called with curMem=51376, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.7 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(2523) called with curMem=56184, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:50525 (size: 2.5 KB, free: 730.6 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 8 (MapPartitionsRDD[66] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 18, localhost, PROCESS_LOCAL, 2287 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 19, localhost, PROCESS_LOCAL, 2343 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 19) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 18) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 19). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 19) in 19 ms on localhost (1/2) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 18). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 18) in 26 ms on localhost (2/2) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - ShuffleMapStage 8 (apply at Transformer.scala:22) finished in 0.027 s -2016-06-13 20:00:52 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:52 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - waiting: Set(ResultStage 9) -2016-06-13 20:00:52 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents for ResultStage 9: List() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ResultStage 9 (ShuffledRDD[67] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(5328) called with curMem=58707, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.2 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-2000 -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(2746) called with curMem=64035, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:50525 (size: 2.7 KB, free: 730.6 MB) -2016-06-13 20:00:52 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (ShuffledRDD[67] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 21, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 20) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 21) -2016-06-13 20:00:52 INFO CheckpointWriter:59 - Checkpoint for time 2000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-2000', took 7351 bytes and 142 ms -2016-06-13 20:00:52 INFO CheckpointWriter:59 - Saving checkpoint for time 3000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-3000' -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 21). 1498 bytes result sent to driver -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 20). 1498 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 21) in 9 ms on localhost (1/2) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 20) in 11 ms on localhost (2/2) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - ResultStage 9 (apply at Transformer.scala:22) finished in 0.011 s -2016-06-13 20:00:52 INFO DAGScheduler:59 - Job 4 finished: apply at Transformer.scala:22, took 0.064446 s -2016-06-13 20:00:52 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 7.889087296526011 and centroid: [-0.4117194134250685,0.7817121953979549,-1.4090401823214687,0.9118227567838174,0.27311463971925837] -2016-06-13 20:00:52 INFO StreamingKMeansModel:59 - Cluster 1 updated with weight 6.681980515339463 and centroid: [0.6702632051579787,0.2950607584497102,0.3950238660586852,-0.26780851756149504,1.2959882529527924] -2016-06-13 20:00:52 INFO JobScheduler:59 - Finished job streaming job 3000 ms.0 from job set of time 3000 ms -2016-06-13 20:00:52 INFO JobScheduler:59 - Starting job streaming job 3000 ms.1 from job set of time 3000 ms -2016-06-13 20:00:52 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Registering RDD 17 (apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Got job 5 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:52 INFO DAGScheduler:59 - Final stage: ResultStage 11(apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 10) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 10) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ShuffleMapStage 10 (UnionRDD[17] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=66781, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(2006) called with curMem=70061, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2006.0 B, free 730.5 MB) -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:50525 (size: 2006.0 B, free: 730.6 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 10 (UnionRDD[17] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 3 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 22, localhost, PROCESS_LOCAL, 2396 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 23, localhost, PROCESS_LOCAL, 2452 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 23) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 22) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 23). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 10.0 (TID 24, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 22). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 23) in 17 ms on localhost (1/3) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 22) in 19 ms on localhost (2/3) -2016-06-13 20:00:52 INFO Executor:59 - Running task 2.0 in stage 10.0 (TID 24) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 2.0 in stage 10.0 (TID 24). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 10.0 (TID 24) in 24 ms on localhost (3/3) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - ShuffleMapStage 10 (apply at Transformer.scala:22) finished in 0.038 s -2016-06-13 20:00:52 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:52 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - waiting: Set(ResultStage 11) -2016-06-13 20:00:52 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents for ResultStage 11: List() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[68] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=72067, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=75307, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.5 MB) -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:50525 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[68] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 25, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 26, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 26) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 25) -2016-06-13 20:00:52 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 26). 1172 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 26) in 7 ms on localhost (1/2) -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 25). 1180 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 25) in 15 ms on localhost (2/2) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - ResultStage 11 (apply at Transformer.scala:22) finished in 0.015 s -2016-06-13 20:00:52 INFO DAGScheduler:59 - Job 5 finished: apply at Transformer.scala:22, took 0.094772 s -2016-06-13 20:00:52 INFO JobScheduler:59 - Finished job streaming job 3000 ms.1 from job set of time 3000 ms -2016-06-13 20:00:52 INFO ParallelCollectionRDD:59 - Removing RDD 6 from persistence list -2016-06-13 20:00:52 INFO JobScheduler:59 - Total delay: 1465819249.396 s for time 3000 ms (execution: 0.171 s) -2016-06-13 20:00:52 INFO JobScheduler:59 - Starting job streaming job 4000 ms.0 from job set of time 4000 ms -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 6 -2016-06-13 20:00:52 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 13 -2016-06-13 20:00:52 INFO ShuffledRDD:59 - Removing RDD 12 from persistence list -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 20:00:52 INFO UnionRDD:59 - Removing RDD 10 from persistence list -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 10 -2016-06-13 20:00:52 INFO MapPartitionsRDD:59 - Removing RDD 8 from persistence list -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 8 -2016-06-13 20:00:52 INFO JobGenerator:59 - Checkpointing graph for time 3000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updating checkpoint data for time 3000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updated checkpoint data for time 3000 ms -2016-06-13 20:00:52 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:52 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-3000 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Registering RDD 69 (apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Got job 6 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:52 INFO DAGScheduler:59 - Final stage: ResultStage 13(apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (MapPartitionsRDD[69] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(4808) called with curMem=77109, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.7 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(2526) called with curMem=81917, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO CheckpointWriter:59 - Checkpoint for time 3000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-3000', took 7351 bytes and 134 ms -2016-06-13 20:00:52 INFO CheckpointWriter:59 - Saving checkpoint for time 4000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-4000' -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:50525 (size: 2.5 KB, free: 730.6 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (MapPartitionsRDD[69] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 27, localhost, PROCESS_LOCAL, 2287 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 28, localhost, PROCESS_LOCAL, 2343 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 28) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 27) -2016-06-13 20:00:52 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 28). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 27). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 28) in 14 ms on localhost (1/2) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 27) in 16 ms on localhost (2/2) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - ShuffleMapStage 12 (apply at Transformer.scala:22) finished in 0.016 s -2016-06-13 20:00:52 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:52 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-06-13 20:00:52 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ResultStage 13 (ShuffledRDD[70] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(5328) called with curMem=84443, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.2 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(2748) called with curMem=89771, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:50525 (size: 2.7 KB, free: 730.6 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (ShuffledRDD[70] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 29, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 30, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 29) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 30) -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 29). 1498 bytes result sent to driver -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 30). 1498 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 29) in 17 ms on localhost (1/2) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 30) in 19 ms on localhost (2/2) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - ResultStage 13 (apply at Transformer.scala:22) finished in 0.021 s -2016-06-13 20:00:52 INFO DAGScheduler:59 - Job 6 finished: apply at Transformer.scala:22, took 0.066491 s -2016-06-13 20:00:52 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 8.578427124746188 and centroid: [-0.3941380296271412,0.8652001390072,-1.5247694962733203,1.0288695754402761,0.3062325238521775] -2016-06-13 20:00:52 INFO StreamingKMeansModel:59 - Cluster 1 updated with weight 6.724873734152915 and centroid: [0.7776074034093413,0.3680057132959103,0.48072737871817856,-0.35558787478877196,1.462511802583542] -2016-06-13 20:00:52 INFO JobScheduler:59 - Finished job streaming job 4000 ms.0 from job set of time 4000 ms -2016-06-13 20:00:52 INFO JobScheduler:59 - Starting job streaming job 4000 ms.1 from job set of time 4000 ms -2016-06-13 20:00:52 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Registering RDD 23 (apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Got job 7 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:52 INFO DAGScheduler:59 - Final stage: ResultStage 15(apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 14) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 14) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ShuffleMapStage 14 (UnionRDD[23] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=92519, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=95799, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:50525 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 14 (UnionRDD[23] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 3 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 31, localhost, PROCESS_LOCAL, 2396 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 32, localhost, PROCESS_LOCAL, 2452 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 31) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 32) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 31). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 32). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 33, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 33) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 32) in 15 ms on localhost (1/3) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 31) in 16 ms on localhost (2/3) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 33). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 33) in 13 ms on localhost (3/3) -2016-06-13 20:00:52 INFO DAGScheduler:59 - ShuffleMapStage 14 (apply at Transformer.scala:22) finished in 0.028 s -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:52 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - waiting: Set(ResultStage 15) -2016-06-13 20:00:52 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents for ResultStage 15: List() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[71] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=97808, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(1801) called with curMem=101048, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1801.0 B, free 730.5 MB) -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:50525 (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[71] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 34, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 35, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 34) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 35) -2016-06-13 20:00:52 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 35). 1172 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 35) in 6 ms on localhost (1/2) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 34). 1180 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 34) in 10 ms on localhost (2/2) -2016-06-13 20:00:52 INFO DAGScheduler:59 - ResultStage 15 (apply at Transformer.scala:22) finished in 0.010 s -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - Job 7 finished: apply at Transformer.scala:22, took 0.060833 s -2016-06-13 20:00:52 INFO JobScheduler:59 - Finished job streaming job 4000 ms.1 from job set of time 4000 ms -2016-06-13 20:00:52 INFO ParallelCollectionRDD:59 - Removing RDD 14 from persistence list -2016-06-13 20:00:52 INFO JobScheduler:59 - Total delay: 1465819248.545 s for time 4000 ms (execution: 0.149 s) -2016-06-13 20:00:52 INFO JobScheduler:59 - Starting job streaming job 5000 ms.0 from job set of time 5000 ms -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 14 -2016-06-13 20:00:52 INFO MapPartitionsRDD:59 - Removing RDD 19 from persistence list -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 19 -2016-06-13 20:00:52 INFO ShuffledRDD:59 - Removing RDD 18 from persistence list -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 18 -2016-06-13 20:00:52 INFO UnionRDD:59 - Removing RDD 17 from persistence list -2016-06-13 20:00:52 INFO MapPartitionsRDD:59 - Removing RDD 15 from persistence list -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 17 -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 15 -2016-06-13 20:00:52 INFO JobGenerator:59 - Checkpointing graph for time 4000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updating checkpoint data for time 4000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updated checkpoint data for time 4000 ms -2016-06-13 20:00:52 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Registering RDD 72 (apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Got job 8 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:52 INFO DAGScheduler:59 - Final stage: ResultStage 17(apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 16) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 16) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ShuffleMapStage 16 (MapPartitionsRDD[72] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(4808) called with curMem=102849, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 4.7 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-4000 -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(2524) called with curMem=107657, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:50525 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 16 (MapPartitionsRDD[72] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 36, localhost, PROCESS_LOCAL, 2287 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 37, localhost, PROCESS_LOCAL, 2343 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 36) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 37) -2016-06-13 20:00:52 INFO CheckpointWriter:59 - Checkpoint for time 4000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-4000', took 7351 bytes and 157 ms -2016-06-13 20:00:52 INFO CheckpointWriter:59 - Saving checkpoint for time 5000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-5000' -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 37). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 37) in 15 ms on localhost (1/2) -2016-06-13 20:00:52 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 36). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 36) in 19 ms on localhost (2/2) -2016-06-13 20:00:52 INFO DAGScheduler:59 - ShuffleMapStage 16 (apply at Transformer.scala:22) finished in 0.020 s -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:52 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - waiting: Set(ResultStage 17) -2016-06-13 20:00:52 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents for ResultStage 17: List() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ResultStage 17 (ShuffledRDD[73] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(5328) called with curMem=110181, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.2 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(2749) called with curMem=115509, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:50525 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (ShuffledRDD[73] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 38, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 39, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 38) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 39) -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 38). 1498 bytes result sent to driver -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 39). 1498 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 38) in 12 ms on localhost (1/2) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 39) in 12 ms on localhost (2/2) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - ResultStage 17 (apply at Transformer.scala:22) finished in 0.014 s -2016-06-13 20:00:52 INFO DAGScheduler:59 - Job 8 finished: apply at Transformer.scala:22, took 0.056508 s -2016-06-13 20:00:52 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 9.065863991822646 and centroid: [-0.424183521484382,0.9123423109940465,-1.5793525409616729,1.1355903649554113,0.3247630228674155] -2016-06-13 20:00:52 INFO StreamingKMeansModel:59 - Cluster 1 updated with weight 6.7552038200428255 and centroid: [0.8829238022160035,0.3290517973075479,0.5501766596792058,-0.4410256182999587,1.5579448961756137] -2016-06-13 20:00:52 INFO JobScheduler:59 - Finished job streaming job 5000 ms.0 from job set of time 5000 ms -2016-06-13 20:00:52 INFO JobScheduler:59 - Starting job streaming job 5000 ms.1 from job set of time 5000 ms -2016-06-13 20:00:52 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Registering RDD 29 (apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Got job 9 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:52 INFO DAGScheduler:59 - Final stage: ResultStage 19(apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 18) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 18) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ShuffleMapStage 18 (UnionRDD[29] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=118258, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(2010) called with curMem=121538, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2010.0 B, free 730.5 MB) -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:50525 (size: 2010.0 B, free: 730.5 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 18 (UnionRDD[29] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 3 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 40, localhost, PROCESS_LOCAL, 2396 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 41, localhost, PROCESS_LOCAL, 2452 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 41) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 40) -2016-06-13 20:00:52 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 41). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 42, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 42) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 40). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 42). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 41) in 49 ms on localhost (1/3) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 40) in 51 ms on localhost (2/3) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 42) in 37 ms on localhost (3/3) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - ShuffleMapStage 18 (apply at Transformer.scala:22) finished in 0.055 s -2016-06-13 20:00:52 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:52 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - waiting: Set(ResultStage 19) -2016-06-13 20:00:52 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents for ResultStage 19: List() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[74] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=123548, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=126788, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:50525 (size: 1803.0 B, free: 730.5 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[74] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 43, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 44, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 43) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 44) -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 44). 1172 bytes result sent to driver -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 43). 1180 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 44) in 21 ms on localhost (1/2) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 43) in 23 ms on localhost (2/2) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - ResultStage 19 (apply at Transformer.scala:22) finished in 0.025 s -2016-06-13 20:00:52 INFO DAGScheduler:59 - Job 9 finished: apply at Transformer.scala:22, took 0.111969 s -2016-06-13 20:00:52 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Registering RDD 75 (apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Got job 10 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:52 INFO DAGScheduler:59 - Final stage: ResultStage 21(apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 20) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 20) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ShuffleMapStage 20 (MapPartitionsRDD[75] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(4808) called with curMem=128591, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 4.7 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(2523) called with curMem=133399, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO CheckpointWriter:59 - Checkpoint for time 5000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-5000', took 7353 bytes and 186 ms -2016-06-13 20:00:52 INFO CheckpointWriter:59 - Saving checkpoint for time 6000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-6000' -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:50525 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 20 (MapPartitionsRDD[75] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 45, localhost, PROCESS_LOCAL, 2287 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 46, localhost, PROCESS_LOCAL, 2343 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 46) -2016-06-13 20:00:52 INFO JobScheduler:59 - Finished job streaming job 5000 ms.1 from job set of time 5000 ms -2016-06-13 20:00:52 INFO JobScheduler:59 - Total delay: 1465819247.777 s for time 5000 ms (execution: 0.231 s) -2016-06-13 20:00:52 INFO JobScheduler:59 - Starting job streaming job 6000 ms.0 from job set of time 6000 ms -2016-06-13 20:00:52 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:00:52 INFO ParallelCollectionRDD:59 - Removing RDD 20 from persistence list -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 45) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 46). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 46) in 23 ms on localhost (1/2) -2016-06-13 20:00:52 INFO MapPartitionsRDD:59 - Removing RDD 25 from persistence list -2016-06-13 20:00:52 INFO ShuffledRDD:59 - Removing RDD 24 from persistence list -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 25 -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 24 -2016-06-13 20:00:52 INFO UnionRDD:59 - Removing RDD 23 from persistence list -2016-06-13 20:00:52 INFO MapPartitionsRDD:59 - Removing RDD 21 from persistence list -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 23 -2016-06-13 20:00:52 INFO JobGenerator:59 - Checkpointing graph for time 5000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updating checkpoint data for time 5000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updated checkpoint data for time 5000 ms -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 45). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 21 -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 20 -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 45) in 35 ms on localhost (2/2) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - ShuffleMapStage 20 (apply at Transformer.scala:22) finished in 0.037 s -2016-06-13 20:00:52 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:52 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - waiting: Set(ResultStage 21) -2016-06-13 20:00:52 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents for ResultStage 21: List() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ResultStage 21 (ShuffledRDD[76] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(5328) called with curMem=135922, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.2 KB, free 730.5 MB) -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(2748) called with curMem=141250, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:50525 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (ShuffledRDD[76] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 48) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 47) -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 47). 1498 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 47) in 9 ms on localhost (1/2) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 48). 1498 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 48) in 9 ms on localhost (2/2) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - ResultStage 21 (apply at Transformer.scala:22) finished in 0.011 s -2016-06-13 20:00:52 INFO DAGScheduler:59 - Job 10 finished: apply at Transformer.scala:22, took 0.075790 s -2016-06-13 20:00:52 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 9.410533905932734 and centroid: [-0.4648183037536739,0.9208418599457215,-1.628313733980898,1.1839163022515895,0.3394630625890255] -2016-06-13 20:00:52 INFO StreamingKMeansModel:59 - Cluster 1 updated with weight 6.7766504294495515 and centroid: [0.8767443477988616,0.383172028092727,0.5910898310514484,-0.5163004929877655,1.5977912721848555] -2016-06-13 20:00:52 INFO JobScheduler:59 - Finished job streaming job 6000 ms.0 from job set of time 6000 ms -2016-06-13 20:00:52 INFO JobScheduler:59 - Starting job streaming job 6000 ms.1 from job set of time 6000 ms -2016-06-13 20:00:52 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:52 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Registering RDD 35 (apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Got job 11 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:52 INFO DAGScheduler:59 - Final stage: ResultStage 23(apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ShuffleMapStage 22 (UnionRDD[35] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=143998, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=147278, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.4 MB) -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:50525 (size: 2007.0 B, free: 730.5 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 22 (UnionRDD[35] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 3 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 49, localhost, PROCESS_LOCAL, 2396 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 50, localhost, PROCESS_LOCAL, 2452 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 49) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 50) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 49). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 51, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 49) in 13 ms on localhost (1/3) -2016-06-13 20:00:52 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 51) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 50). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 50) in 21 ms on localhost (2/3) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 51). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 51) in 13 ms on localhost (3/3) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - ShuffleMapStage 22 (apply at Transformer.scala:22) finished in 0.026 s -2016-06-13 20:00:52 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:52 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - waiting: Set(ResultStage 23) -2016-06-13 20:00:52 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents for ResultStage 23: List() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[77] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=149285, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:00:52 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=152525, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.4 MB) -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:50525 (size: 1803.0 B, free: 730.5 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[77] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 52, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 53) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 52) -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 53). 1172 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 53) in 5 ms on localhost (1/2) -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 52). 1180 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 52) in 7 ms on localhost (2/2) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - ResultStage 23 (apply at Transformer.scala:22) finished in 0.009 s -2016-06-13 20:00:52 INFO DAGScheduler:59 - Job 11 finished: apply at Transformer.scala:22, took 0.065340 s -2016-06-13 20:00:52 INFO JobScheduler:59 - Finished job streaming job 6000 ms.1 from job set of time 6000 ms -2016-06-13 20:00:52 INFO JobScheduler:59 - Total delay: 1465819246.899 s for time 6000 ms (execution: 0.121 s) -2016-06-13 20:00:52 INFO JobScheduler:59 - Starting job streaming job 7000 ms.0 from job set of time 7000 ms -2016-06-13 20:00:52 INFO ParallelCollectionRDD:59 - Removing RDD 26 from persistence list -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 26 -2016-06-13 20:00:52 INFO MapPartitionsRDD:59 - Removing RDD 31 from persistence list -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 31 -2016-06-13 20:00:52 INFO ShuffledRDD:59 - Removing RDD 30 from persistence list -2016-06-13 20:00:52 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 30 -2016-06-13 20:00:52 INFO UnionRDD:59 - Removing RDD 29 from persistence list -2016-06-13 20:00:52 INFO DAGScheduler:59 - Registering RDD 78 (apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Got job 12 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:52 INFO DAGScheduler:59 - Final stage: ResultStage 25(apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 24) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 24) -2016-06-13 20:00:52 INFO MapPartitionsRDD:59 - Removing RDD 27 from persistence list -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ShuffleMapStage 24 (MapPartitionsRDD[78] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:52 INFO CheckpointWriter:59 - Checkpoint for time 6000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-6000', took 7348 bytes and 160 ms -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(4808) called with curMem=154328, maxMem=766075207 -2016-06-13 20:00:52 INFO CheckpointWriter:59 - Saving checkpoint for time 7000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-7000' -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 27 -2016-06-13 20:00:52 INFO JobGenerator:59 - Checkpointing graph for time 6000 ms -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updating checkpoint data for time 6000 ms -2016-06-13 20:00:52 INFO BlockManager:59 - Removing RDD 29 -2016-06-13 20:00:52 INFO DStreamGraph:59 - Updated checkpoint data for time 6000 ms -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(2521) called with curMem=159136, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:50525 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 24 (MapPartitionsRDD[78] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 54, localhost, PROCESS_LOCAL, 2287 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 55, localhost, PROCESS_LOCAL, 2343 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 54) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 55) -2016-06-13 20:00:52 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 54). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 55). 1159 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 54) in 11 ms on localhost (1/2) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 55) in 12 ms on localhost (2/2) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - ShuffleMapStage 24 (apply at Transformer.scala:22) finished in 0.012 s -2016-06-13 20:00:52 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:52 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - waiting: Set(ResultStage 25) -2016-06-13 20:00:52 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents for ResultStage 25: List() -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ResultStage 25 (ShuffledRDD[79] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(5328) called with curMem=161657, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.2 KB, free 730.4 MB) -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(2748) called with curMem=166985, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:50525 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (ShuffledRDD[79] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 56) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 57) -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:52 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:52 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 56). 1498 bytes result sent to driver -2016-06-13 20:00:52 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 57). 1498 bytes result sent to driver -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 56) in 10 ms on localhost (1/2) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 57) in 10 ms on localhost (2/2) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-06-13 20:00:52 INFO DAGScheduler:59 - ResultStage 25 (apply at Transformer.scala:22) finished in 0.012 s -2016-06-13 20:00:52 INFO DAGScheduler:59 - Job 12 finished: apply at Transformer.scala:22, took 0.057910 s -2016-06-13 20:00:52 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 9.654252339470965 and centroid: [-0.4481408435905915,0.9248487059890342,-1.6517130043756276,1.231328322902766,0.37649750460481207] -2016-06-13 20:00:52 INFO StreamingKMeansModel:59 - Cluster 1 updated with weight 6.791815472394507 and centroid: [0.90852574247515,0.3989224628532152,0.6282046004572324,-0.5378932321274364,1.7023018472990064] -2016-06-13 20:00:52 INFO JobScheduler:59 - Finished job streaming job 7000 ms.0 from job set of time 7000 ms -2016-06-13 20:00:52 INFO JobScheduler:59 - Starting job streaming job 7000 ms.1 from job set of time 7000 ms -2016-06-13 20:00:52 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Registering RDD 41 (apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Got job 13 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:52 INFO DAGScheduler:59 - Final stage: ResultStage 27(apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (UnionRDD[41] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=169733, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:00:52 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=173013, maxMem=766075207 -2016-06-13 20:00:52 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.4 MB) -2016-06-13 20:00:52 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:00:52 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:50525 (size: 2009.0 B, free: 730.5 MB) -2016-06-13 20:00:52 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:52 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 26 (UnionRDD[41] at apply at Transformer.scala:22) -2016-06-13 20:00:52 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 3 tasks -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 58, localhost, PROCESS_LOCAL, 2396 bytes) -2016-06-13 20:00:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 59, localhost, PROCESS_LOCAL, 2452 bytes) -2016-06-13 20:00:52 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 59) -2016-06-13 20:00:52 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 58) -2016-06-13 20:00:53 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 59). 1159 bytes result sent to driver -2016-06-13 20:00:53 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 58). 1159 bytes result sent to driver -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 60, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 59) in 13 ms on localhost (1/3) -2016-06-13 20:00:53 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 60) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 58) in 14 ms on localhost (2/3) -2016-06-13 20:00:53 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 60). 1159 bytes result sent to driver -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 60) in 9 ms on localhost (3/3) -2016-06-13 20:00:53 INFO DAGScheduler:59 - ShuffleMapStage 26 (apply at Transformer.scala:22) finished in 0.022 s -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-06-13 20:00:53 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:53 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:53 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-06-13 20:00:53 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:53 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[80] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=175022, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(1801) called with curMem=178262, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 1801.0 B, free 730.4 MB) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:50525 (size: 1801.0 B, free: 730.5 MB) -2016-06-13 20:00:53 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[80] at apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 62, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:53 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 62) -2016-06-13 20:00:53 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 61) -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:53 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 62). 1172 bytes result sent to driver -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 62) in 7 ms on localhost (1/2) -2016-06-13 20:00:53 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 61). 1180 bytes result sent to driver -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 61) in 11 ms on localhost (2/2) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-06-13 20:00:53 INFO DAGScheduler:59 - ResultStage 27 (apply at Transformer.scala:22) finished in 0.011 s -2016-06-13 20:00:53 INFO DAGScheduler:59 - Job 13 finished: apply at Transformer.scala:22, took 0.056840 s -2016-06-13 20:00:53 INFO JobScheduler:59 - Finished job streaming job 7000 ms.1 from job set of time 7000 ms -2016-06-13 20:00:53 INFO JobScheduler:59 - Total delay: 1465819246.037 s for time 7000 ms (execution: 0.137 s) -2016-06-13 20:00:53 INFO JobScheduler:59 - Starting job streaming job 8000 ms.0 from job set of time 8000 ms -2016-06-13 20:00:53 INFO ParallelCollectionRDD:59 - Removing RDD 32 from persistence list -2016-06-13 20:00:53 INFO MapPartitionsRDD:59 - Removing RDD 37 from persistence list -2016-06-13 20:00:53 INFO BlockManager:59 - Removing RDD 37 -2016-06-13 20:00:53 INFO BlockManager:59 - Removing RDD 32 -2016-06-13 20:00:53 INFO ShuffledRDD:59 - Removing RDD 36 from persistence list -2016-06-13 20:00:53 INFO BlockManager:59 - Removing RDD 36 -2016-06-13 20:00:53 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:53 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:00:53 INFO UnionRDD:59 - Removing RDD 35 from persistence list -2016-06-13 20:00:53 INFO DAGScheduler:59 - Registering RDD 81 (apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO BlockManager:59 - Removing RDD 35 -2016-06-13 20:00:53 INFO DAGScheduler:59 - Got job 14 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:53 INFO DAGScheduler:59 - Final stage: ResultStage 29(apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 28) -2016-06-13 20:00:53 INFO MapPartitionsRDD:59 - Removing RDD 33 from persistence list -2016-06-13 20:00:53 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 28) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting ShuffleMapStage 28 (MapPartitionsRDD[81] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:53 INFO JobGenerator:59 - Checkpointing graph for time 7000 ms -2016-06-13 20:00:53 INFO DStreamGraph:59 - Updating checkpoint data for time 7000 ms -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(4808) called with curMem=180063, maxMem=766075207 -2016-06-13 20:00:53 INFO DStreamGraph:59 - Updated checkpoint data for time 7000 ms -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-06-13 20:00:53 INFO BlockManager:59 - Removing RDD 33 -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(2523) called with curMem=184871, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:50525 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 20:00:53 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 28 (MapPartitionsRDD[81] at apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 63, localhost, PROCESS_LOCAL, 2287 bytes) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 64, localhost, PROCESS_LOCAL, 2343 bytes) -2016-06-13 20:00:53 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 64) -2016-06-13 20:00:53 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 63) -2016-06-13 20:00:53 INFO CheckpointWriter:59 - Checkpoint for time 7000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-7000', took 7349 bytes and 147 ms -2016-06-13 20:00:53 INFO CheckpointWriter:59 - Saving checkpoint for time 8000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-8000' -2016-06-13 20:00:53 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 63). 1159 bytes result sent to driver -2016-06-13 20:00:53 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 64). 1159 bytes result sent to driver -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 63) in 17 ms on localhost (1/2) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 64) in 18 ms on localhost (2/2) -2016-06-13 20:00:53 INFO DAGScheduler:59 - ShuffleMapStage 28 (apply at Transformer.scala:22) finished in 0.018 s -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-06-13 20:00:53 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:53 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:53 INFO DAGScheduler:59 - waiting: Set(ResultStage 29) -2016-06-13 20:00:53 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:53 INFO DAGScheduler:59 - Missing parents for ResultStage 29: List() -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting ResultStage 29 (ShuffledRDD[82] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(5328) called with curMem=187394, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.2 KB, free 730.4 MB) -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(2746) called with curMem=192722, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:50525 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 20:00:53 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (ShuffledRDD[82] at apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 65, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 66, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:53 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 65) -2016-06-13 20:00:53 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 66) -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:53 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 66). 1498 bytes result sent to driver -2016-06-13 20:00:53 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 65). 1498 bytes result sent to driver -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 66) in 7 ms on localhost (1/2) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 65) in 7 ms on localhost (2/2) -2016-06-13 20:00:53 INFO DAGScheduler:59 - ResultStage 29 (apply at Transformer.scala:22) finished in 0.008 s -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 20:00:53 INFO DAGScheduler:59 - Job 14 finished: apply at Transformer.scala:22, took 0.047821 s -2016-06-13 20:00:53 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 9.826587296526009 and centroid: [-0.47995955804302076,0.919021048747697,-1.6689435036462834,1.2293517255037656,0.3686351878426602] -2016-06-13 20:00:53 INFO StreamingKMeansModel:59 - Cluster 1 updated with weight 6.802538777097871 and centroid: [0.9307777668934964,0.4120400097417326,0.6916225128183254,-0.5536537855610996,1.7867260188363094] -2016-06-13 20:00:53 INFO JobScheduler:59 - Finished job streaming job 8000 ms.0 from job set of time 8000 ms -2016-06-13 20:00:53 INFO JobScheduler:59 - Starting job streaming job 8000 ms.1 from job set of time 8000 ms -2016-06-13 20:00:53 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:00:53 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:53 INFO DAGScheduler:59 - Registering RDD 47 (apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Got job 15 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:53 INFO DAGScheduler:59 - Final stage: ResultStage 31(apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 30) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting ShuffleMapStage 30 (UnionRDD[47] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=195468, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(2006) called with curMem=198748, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2006.0 B, free 730.4 MB) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:50525 (size: 2006.0 B, free: 730.5 MB) -2016-06-13 20:00:53 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 30 (UnionRDD[47] at apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 3 tasks -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 67, localhost, PROCESS_LOCAL, 2396 bytes) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 68, localhost, PROCESS_LOCAL, 2452 bytes) -2016-06-13 20:00:53 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 67) -2016-06-13 20:00:53 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 68) -2016-06-13 20:00:53 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 67). 1159 bytes result sent to driver -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 69, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:00:53 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 69) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 67) in 9 ms on localhost (1/3) -2016-06-13 20:00:53 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 68). 1159 bytes result sent to driver -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 68) in 9 ms on localhost (2/3) -2016-06-13 20:00:53 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 69). 1159 bytes result sent to driver -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 69) in 7 ms on localhost (3/3) -2016-06-13 20:00:53 INFO DAGScheduler:59 - ShuffleMapStage 30 (apply at Transformer.scala:22) finished in 0.015 s -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 20:00:53 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:53 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:53 INFO DAGScheduler:59 - waiting: Set(ResultStage 31) -2016-06-13 20:00:53 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:53 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List() -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[83] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=200754, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=203994, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.4 MB) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:50525 (size: 1802.0 B, free: 730.5 MB) -2016-06-13 20:00:53 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[83] at apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 70, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:50525 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 71, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:53 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 70) -2016-06-13 20:00:53 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 71) -2016-06-13 20:00:53 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 70 -2016-06-13 20:00:53 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 71). 1172 bytes result sent to driver -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 71) in 5 ms on localhost (1/2) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:50525 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 69 -2016-06-13 20:00:53 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 70). 1180 bytes result sent to driver -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned shuffle 14 -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 70) in 10 ms on localhost (2/2) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 20:00:53 INFO DAGScheduler:59 - ResultStage 31 (apply at Transformer.scala:22) finished in 0.011 s -2016-06-13 20:00:53 INFO DAGScheduler:59 - Job 15 finished: apply at Transformer.scala:22, took 0.058227 s -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:50525 in memory (size: 1801.0 B, free: 730.5 MB) -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 68 -2016-06-13 20:00:53 INFO JobScheduler:59 - Finished job streaming job 8000 ms.1 from job set of time 8000 ms -2016-06-13 20:00:53 INFO JobScheduler:59 - Total delay: 1465819245.169 s for time 8000 ms (execution: 0.131 s) -2016-06-13 20:00:53 INFO ParallelCollectionRDD:59 - Removing RDD 38 from persistence list -2016-06-13 20:00:53 INFO JobScheduler:59 - Starting job streaming job 9000 ms.0 from job set of time 9000 ms -2016-06-13 20:00:53 INFO BlockManager:59 - Removing RDD 38 -2016-06-13 20:00:53 INFO MapPartitionsRDD:59 - Removing RDD 43 from persistence list -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:50525 in memory (size: 2009.0 B, free: 730.5 MB) -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 67 -2016-06-13 20:00:53 INFO BlockManager:59 - Removing RDD 43 -2016-06-13 20:00:53 INFO ShuffledRDD:59 - Removing RDD 42 from persistence list -2016-06-13 20:00:53 INFO BlockManager:59 - Removing RDD 42 -2016-06-13 20:00:53 INFO UnionRDD:59 - Removing RDD 41 from persistence list -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:50525 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 20:00:53 INFO BlockManager:59 - Removing RDD 41 -2016-06-13 20:00:53 INFO MapPartitionsRDD:59 - Removing RDD 39 from persistence list -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 66 -2016-06-13 20:00:53 INFO BlockManager:59 - Removing RDD 39 -2016-06-13 20:00:53 INFO JobGenerator:59 - Checkpointing graph for time 8000 ms -2016-06-13 20:00:53 INFO DStreamGraph:59 - Updating checkpoint data for time 8000 ms -2016-06-13 20:00:53 INFO DStreamGraph:59 - Updated checkpoint data for time 8000 ms -2016-06-13 20:00:53 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:50525 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 65 -2016-06-13 20:00:53 INFO DAGScheduler:59 - Registering RDD 84 (apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Got job 16 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:53 INFO DAGScheduler:59 - Final stage: ResultStage 33(apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 32) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting ShuffleMapStage 32 (MapPartitionsRDD[84] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(4808) called with curMem=164656, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(2526) called with curMem=169464, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.4 MB) -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned shuffle 12 -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:50525 (size: 2.5 KB, free: 730.5 MB) -2016-06-13 20:00:53 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 32 (MapPartitionsRDD[84] at apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:50525 in memory (size: 1803.0 B, free: 730.5 MB) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 72, localhost, PROCESS_LOCAL, 2287 bytes) -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 64 -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 73, localhost, PROCESS_LOCAL, 2343 bytes) -2016-06-13 20:00:53 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 72) -2016-06-13 20:00:53 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 73) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:50525 in memory (size: 2007.0 B, free: 730.5 MB) -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 63 -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned shuffle 11 -2016-06-13 20:00:53 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:00:53 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 72). 1159 bytes result sent to driver -2016-06-13 20:00:53 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 73). 1159 bytes result sent to driver -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:50525 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 72) in 17 ms on localhost (1/2) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 73) in 17 ms on localhost (2/2) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 20:00:53 INFO DAGScheduler:59 - ShuffleMapStage 32 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:00:53 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:53 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:53 INFO DAGScheduler:59 - waiting: Set(ResultStage 33) -2016-06-13 20:00:53 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:53 INFO DAGScheduler:59 - Missing parents for ResultStage 33: List() -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 62 -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting ResultStage 33 (ShuffledRDD[85] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(5328) called with curMem=146253, maxMem=766075207 -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:50525 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.2 KB, free 730.4 MB) -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 61 -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned shuffle 10 -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(2747) called with curMem=151581, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:50525 (size: 2.7 KB, free: 730.5 MB) -2016-06-13 20:00:53 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (ShuffledRDD[85] at apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:50525 in memory (size: 1803.0 B, free: 730.5 MB) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 60 -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 74, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 75, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:53 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 75) -2016-06-13 20:00:53 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 74) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:50525 in memory (size: 2010.0 B, free: 730.5 MB) -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 59 -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned shuffle 9 -2016-06-13 20:00:53 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 75). 1498 bytes result sent to driver -2016-06-13 20:00:53 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 74). 1498 bytes result sent to driver -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:50525 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 75) in 12 ms on localhost (1/2) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 74) in 14 ms on localhost (2/2) -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 58 -2016-06-13 20:00:53 INFO DAGScheduler:59 - ResultStage 33 (apply at Transformer.scala:22) finished in 0.015 s -2016-06-13 20:00:53 INFO DAGScheduler:59 - Job 16 finished: apply at Transformer.scala:22, took 0.062032 s -2016-06-13 20:00:53 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 9.948446513295124 and centroid: [-0.4631700565290908,0.9294599209444628,-1.69257550672196,1.2225441539029944,0.33843788680942255] -2016-06-13 20:00:53 INFO StreamingKMeansModel:59 - Cluster 1 updated with weight 6.8101212985703485 and centroid: [0.9741254424308765,0.4184035300462217,0.6921898194828577,-0.5514312130754201,1.840396683554789] -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:50525 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 57 -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned shuffle 8 -2016-06-13 20:00:53 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:53 INFO JobScheduler:59 - Finished job streaming job 9000 ms.0 from job set of time 9000 ms -2016-06-13 20:00:53 INFO JobScheduler:59 - Starting job streaming job 9000 ms.1 from job set of time 9000 ms -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:50525 in memory (size: 1801.0 B, free: 730.5 MB) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Registering RDD 54 (apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Got job 17 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:53 INFO DAGScheduler:59 - Final stage: ResultStage 35(apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 34) -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 56 -2016-06-13 20:00:53 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 34) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting ShuffleMapStage 34 (UnionRDD[54] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=121536, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:50525 in memory (size: 2009.0 B, free: 730.5 MB) -2016-06-13 20:00:53 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=124816, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.5 MB) -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 55 -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:50525 (size: 2007.0 B, free: 730.5 MB) -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned shuffle 7 -2016-06-13 20:00:53 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 34 (UnionRDD[54] at apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 3 tasks -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 76, localhost, PROCESS_LOCAL, 2396 bytes) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 77, localhost, PROCESS_LOCAL, 2452 bytes) -2016-06-13 20:00:53 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 77) -2016-06-13 20:00:53 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 76) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:50525 in memory (size: 2.7 KB, free: 730.5 MB) -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 54 -2016-06-13 20:00:53 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 77). 1159 bytes result sent to driver -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 78, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:50525 in memory (size: 2.5 KB, free: 730.5 MB) -2016-06-13 20:00:53 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 78) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 77) in 16 ms on localhost (1/3) -2016-06-13 20:00:53 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 76). 1159 bytes result sent to driver -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 76) in 24 ms on localhost (2/3) -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 53 -2016-06-13 20:00:53 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 78). 1159 bytes result sent to driver -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 78) in 15 ms on localhost (3/3) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned shuffle 6 -2016-06-13 20:00:53 INFO DAGScheduler:59 - ShuffleMapStage 34 (apply at Transformer.scala:22) finished in 0.033 s -2016-06-13 20:00:53 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:53 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:53 INFO DAGScheduler:59 - waiting: Set(ResultStage 35) -2016-06-13 20:00:53 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:53 INFO DAGScheduler:59 - Missing parents for ResultStage 35: List() -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[86] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:50525 in memory (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=103091, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 52 -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=103051, maxMem=766075207 -2016-06-13 20:00:53 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:50525 (size: 1803.0 B, free: 730.5 MB) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:50525 in memory (size: 2006.0 B, free: 730.6 MB) -2016-06-13 20:00:53 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 51 -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[86] at apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned shuffle 5 -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 79, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 80, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:53 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 79) -2016-06-13 20:00:53 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 80) -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:53 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 80). 1172 bytes result sent to driver -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:50525 in memory (size: 2.7 KB, free: 730.6 MB) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 80) in 9 ms on localhost (1/2) -2016-06-13 20:00:53 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 79). 1180 bytes result sent to driver -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 50 -2016-06-13 20:00:53 INFO DAGScheduler:59 - ResultStage 35 (apply at Transformer.scala:22) finished in 0.012 s -2016-06-13 20:00:53 INFO DAGScheduler:59 - Job 17 finished: apply at Transformer.scala:22, took 0.078296 s -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 79) in 12 ms on localhost (2/2) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:50525 in memory (size: 2.5 KB, free: 730.6 MB) -2016-06-13 20:00:53 INFO JobScheduler:59 - Finished job streaming job 9000 ms.1 from job set of time 9000 ms -2016-06-13 20:00:53 INFO JobScheduler:59 - Total delay: 1465819244.334 s for time 9000 ms (execution: 0.164 s) -2016-06-13 20:00:53 INFO ParallelCollectionRDD:59 - Removing RDD 44 from persistence list -2016-06-13 20:00:53 INFO JobScheduler:59 - Starting job streaming job 10000 ms.0 from job set of time 10000 ms -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 49 -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned shuffle 4 -2016-06-13 20:00:53 INFO MapPartitionsRDD:59 - Removing RDD 49 from persistence list -2016-06-13 20:00:53 INFO BlockManager:59 - Removing RDD 49 -2016-06-13 20:00:53 INFO ShuffledRDD:59 - Removing RDD 48 from persistence list -2016-06-13 20:00:53 INFO BlockManager:59 - Removing RDD 48 -2016-06-13 20:00:53 INFO UnionRDD:59 - Removing RDD 47 from persistence list -2016-06-13 20:00:53 INFO BlockManager:59 - Removing RDD 47 -2016-06-13 20:00:53 INFO MapPartitionsRDD:59 - Removing RDD 45 from persistence list -2016-06-13 20:00:53 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:53 INFO BlockManager:59 - Removing RDD 45 -2016-06-13 20:00:53 INFO BlockManager:59 - Removing RDD 44 -2016-06-13 20:00:53 INFO DAGScheduler:59 - Registering RDD 87 (apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Got job 18 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:50525 in memory (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:00:53 INFO CheckpointWriter:59 - Checkpoint for time 8000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-8000', took 7351 bytes and 288 ms -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 48 -2016-06-13 20:00:53 INFO CheckpointWriter:59 - Saving checkpoint for time 9000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-9000' -2016-06-13 20:00:53 INFO DAGScheduler:59 - Final stage: ResultStage 37(apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 36) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 36) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting ShuffleMapStage 36 (MapPartitionsRDD[87] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:53 INFO JobGenerator:59 - Checkpointing graph for time 9000 ms -2016-06-13 20:00:53 INFO DStreamGraph:59 - Updating checkpoint data for time 9000 ms -2016-06-13 20:00:53 INFO DStreamGraph:59 - Updated checkpoint data for time 9000 ms -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(4808) called with curMem=80392, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 4.7 KB, free 730.5 MB) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:50525 in memory (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 47 -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(2524) called with curMem=81920, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.5 KB, free 730.5 MB) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:50525 (size: 2.5 KB, free: 730.6 MB) -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-06-13 20:00:53 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 36 (MapPartitionsRDD[87] at apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-06-13 20:00:53 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 81, localhost, PROCESS_LOCAL, 2287 bytes) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:50525 in memory (size: 2.7 KB, free: 730.6 MB) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 82, localhost, PROCESS_LOCAL, 2343 bytes) -2016-06-13 20:00:53 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 81) -2016-06-13 20:00:53 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 82) -2016-06-13 20:00:53 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 81). 1159 bytes result sent to driver -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 46 -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 81) in 18 ms on localhost (1/2) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:50525 in memory (size: 2.5 KB, free: 730.6 MB) -2016-06-13 20:00:53 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 82). 1159 bytes result sent to driver -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 45 -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 82) in 24 ms on localhost (2/2) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-06-13 20:00:53 INFO DAGScheduler:59 - ShuffleMapStage 36 (apply at Transformer.scala:22) finished in 0.027 s -2016-06-13 20:00:53 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:53 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:53 INFO DAGScheduler:59 - waiting: Set(ResultStage 37) -2016-06-13 20:00:53 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:53 INFO DAGScheduler:59 - Missing parents for ResultStage 37: List() -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting ResultStage 37 (ShuffledRDD[88] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:50525 in memory (size: 1800.0 B, free: 730.6 MB) -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(5328) called with curMem=64005, maxMem=766075207 -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 44 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.2 KB, free 730.5 MB) -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(2749) called with curMem=66053, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.5 MB) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:50525 in memory (size: 2000.0 B, free: 730.6 MB) -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 43 -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:50525 (size: 2.7 KB, free: 730.6 MB) -2016-06-13 20:00:53 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (ShuffledRDD[88] at apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 83, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 84, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-06-13 20:00:53 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 84) -2016-06-13 20:00:53 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 83) -2016-06-13 20:00:53 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:50525 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:00:53 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 84). 1498 bytes result sent to driver -2016-06-13 20:00:53 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 83). 1498 bytes result sent to driver -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 42 -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 84) in 28 ms on localhost (1/2) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:50525 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 83) in 30 ms on localhost (2/2) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-06-13 20:00:53 INFO DAGScheduler:59 - ResultStage 37 (apply at Transformer.scala:22) finished in 0.032 s -2016-06-13 20:00:53 INFO DAGScheduler:59 - Job 18 finished: apply at Transformer.scala:22, took 0.098469 s -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned accumulator 41 -2016-06-13 20:00:53 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 10.034613991822646 and centroid: [-0.499184095541533,0.9285934097596907,-1.6929463528087672,1.2452906235025247,0.3510250187719718] -2016-06-13 20:00:53 INFO StreamingKMeansModel:59 - Cluster 1 updated with weight 6.81548295092203 and centroid: [0.9800799188622931,0.4132628392239448,0.7177472581981605,-0.6104835592347974,1.8383576969609605] -2016-06-13 20:00:53 INFO JobScheduler:59 - Finished job streaming job 10000 ms.0 from job set of time 10000 ms -2016-06-13 20:00:53 INFO JobScheduler:59 - Starting job streaming job 10000 ms.1 from job set of time 10000 ms -2016-06-13 20:00:53 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-06-13 20:00:53 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:00:53 INFO DAGScheduler:59 - Registering RDD 60 (apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Got job 19 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:00:53 INFO DAGScheduler:59 - Final stage: ResultStage 39(apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 38) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 38) -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting ShuffleMapStage 38 (UnionRDD[60] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=51476, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=54756, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.5 MB) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:50525 (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:00:53 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 38 (UnionRDD[60] at apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 3 tasks -2016-06-13 20:00:53 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 85, localhost, PROCESS_LOCAL, 2396 bytes) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 86, localhost, PROCESS_LOCAL, 2452 bytes) -2016-06-13 20:00:53 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 86) -2016-06-13 20:00:53 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 85) -2016-06-13 20:00:53 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 86). 1159 bytes result sent to driver -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 87, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:00:53 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 87) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 86) in 10 ms on localhost (1/3) -2016-06-13 20:00:53 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 85). 1159 bytes result sent to driver -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 85) in 14 ms on localhost (2/3) -2016-06-13 20:00:53 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 87). 1159 bytes result sent to driver -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 87) in 9 ms on localhost (3/3) -2016-06-13 20:00:53 INFO DAGScheduler:59 - ShuffleMapStage 38 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-06-13 20:00:53 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:00:53 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:00:53 INFO DAGScheduler:59 - waiting: Set(ResultStage 39) -2016-06-13 20:00:53 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:00:53 INFO DAGScheduler:59 - Missing parents for ResultStage 39: List() -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[89] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=56763, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:00:53 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=60003, maxMem=766075207 -2016-06-13 20:00:53 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:00:53 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:50525 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:00:53 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-5000.bk -2016-06-13 20:00:53 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:00:53 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[89] at apply at Transformer.scala:22) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:53 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:00:53 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 88) -2016-06-13 20:00:53 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 89) -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:00:53 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 89). 1172 bytes result sent to driver -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:00:53 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 89) in 6 ms on localhost (1/2) -2016-06-13 20:00:53 INFO CheckpointWriter:59 - Checkpoint for time 9000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-9000', took 7351 bytes and 159 ms -2016-06-13 20:00:53 INFO CheckpointWriter:59 - Saving checkpoint for time 10000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-10000' -2016-06-13 20:00:53 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 88). 1180 bytes result sent to driver -2016-06-13 20:00:53 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 88) in 11 ms on localhost (2/2) -2016-06-13 20:00:53 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-06-13 20:00:53 INFO DAGScheduler:59 - ResultStage 39 (apply at Transformer.scala:22) finished in 0.012 s -2016-06-13 20:00:53 INFO DAGScheduler:59 - Job 19 finished: apply at Transformer.scala:22, took 0.064257 s -2016-06-13 20:00:53 INFO JobScheduler:59 - Finished job streaming job 10000 ms.1 from job set of time 10000 ms -2016-06-13 20:00:53 INFO ParallelCollectionRDD:59 - Removing RDD 50 from persistence list -2016-06-13 20:00:53 INFO JobScheduler:59 - Total delay: 1465819243.520 s for time 10000 ms (execution: 0.185 s) -2016-06-13 20:00:53 INFO BlockManager:59 - Removing RDD 50 -2016-06-13 20:00:53 INFO MapPartitionsRDD:59 - Removing RDD 56 from persistence list -2016-06-13 20:00:53 INFO ShuffledRDD:59 - Removing RDD 55 from persistence list -2016-06-13 20:00:53 INFO BlockManager:59 - Removing RDD 56 -2016-06-13 20:00:53 INFO UnionRDD:59 - Removing RDD 54 from persistence list -2016-06-13 20:00:53 INFO MapPartitionsRDD:59 - Removing RDD 52 from persistence list -2016-06-13 20:00:53 INFO BlockManager:59 - Removing RDD 55 -2016-06-13 20:00:53 INFO BlockManager:59 - Removing RDD 54 -2016-06-13 20:00:53 INFO JobGenerator:59 - Checkpointing graph for time 10000 ms -2016-06-13 20:00:53 INFO DStreamGraph:59 - Updating checkpoint data for time 10000 ms -2016-06-13 20:00:53 INFO BlockManager:59 - Removing RDD 52 -2016-06-13 20:00:53 INFO DStreamGraph:59 - Updated checkpoint data for time 10000 ms -2016-06-13 20:00:53 INFO StreamingKMeansSuite:59 - Output generated in 1676 milliseconds -2016-06-13 20:00:53 INFO StreamingKMeansSuite:59 - [WrappedArray(5),WrappedArray()] -2016-06-13 20:00:53 INFO StreamingKMeansSuite:59 - [WrappedArray(5),WrappedArray()] -2016-06-13 20:00:53 INFO StreamingKMeansSuite:59 - [WrappedArray(5),WrappedArray()] -2016-06-13 20:00:53 INFO StreamingKMeansSuite:59 - [WrappedArray(5),WrappedArray()] -2016-06-13 20:00:53 INFO StreamingKMeansSuite:59 - [WrappedArray(5),WrappedArray()] -2016-06-13 20:00:53 INFO StreamingKMeansSuite:59 - [WrappedArray(5),WrappedArray()] -2016-06-13 20:00:53 INFO StreamingKMeansSuite:59 - [WrappedArray(5),WrappedArray()] -2016-06-13 20:00:53 INFO StreamingKMeansSuite:59 - [WrappedArray(5),WrappedArray()] -2016-06-13 20:00:53 INFO StreamingKMeansSuite:59 - [WrappedArray(5),WrappedArray()] -2016-06-13 20:00:53 INFO StreamingKMeansSuite:59 - [WrappedArray(5),WrappedArray()] -2016-06-13 20:00:53 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-5000 -2016-06-13 20:00:53 INFO CheckpointWriter:59 - Checkpoint for time 10000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-10000', took 7360 bytes and 110 ms -2016-06-13 20:00:53 INFO CheckpointWriter:59 - Saving checkpoint for time 1000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-1000' -2016-06-13 20:00:53 INFO JobGenerator:59 - Stopping JobGenerator immediately -2016-06-13 20:00:53 INFO RecurringTimer:59 - Stopped timer for JobGenerator after time 10000 -2016-06-13 20:01:03 INFO CheckpointWriter:59 - CheckpointWriter executor terminated ? false, waited for 10001 ms. -2016-06-13 20:01:03 INFO JobGenerator:59 - Stopped JobGenerator -2016-06-13 20:01:03 INFO JobScheduler:59 - Stopped JobScheduler -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/batch,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/streaming,null} -2016-06-13 20:01:03 INFO StreamingContext:59 - StreamingContext stopped successfully -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/batch/json,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/json,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:01:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:01:03 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-1000 -2016-06-13 20:01:03 INFO CheckpointWriter:59 - Checkpoint for time 1000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-1000', took 7500 bytes and 10088 ms -2016-06-13 20:01:03 INFO SparkUI:59 - Stopped Spark web UI at http://localhost:4040 -2016-06-13 20:01:03 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:01:03 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:01:04 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:01:04 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:01:04 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:01:04 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:01:04 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:01:04 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:01:04 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:01:04 INFO StreamingKMeansSuite:59 - - -===== FINISHED o.a.s.mllib.clustering.kmeans.StreamingKMeansSuite: 'accuracy for two centers' ===== - -2016-06-13 20:01:04 WARN StreamingContext:71 - StreamingContext has already been stopped -2016-06-13 20:01:04 INFO SparkContext:59 - SparkContext already stopped. -2016-06-13 20:01:04 INFO StreamingKMeansSuite:59 - Using manual clock -2016-06-13 20:01:04 INFO StreamingKMeansSuite:59 - - -===== TEST OUTPUT FOR o.a.s.mllib.clustering.kmeans.StreamingKMeansSuite: 'detecting dying clusters' ===== - -2016-06-13 20:01:04 INFO StreamingKMeans:59 - Setting decay factor to: 0.250000 -2016-06-13 20:01:04 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:01:04 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:01:04 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:01:04 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:01:04 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 20:01:04 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:01:04 INFO Remoting:74 - Starting remoting -2016-06-13 20:01:04 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@localhost:50539] -2016-06-13 20:01:04 INFO Utils:59 - Successfully started service 'sparkDriver' on port 50539. -2016-06-13 20:01:04 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:01:04 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:01:04 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-712efe41-c69a-4577-93ce-1a9c2fec47d5 -2016-06-13 20:01:04 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:01:04 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-f8eb2236-34b2-4105-aaff-6c04ac3f507c\httpd-1b519ba2-cac5-4631-8f63-37fe8f41ea92 -2016-06-13 20:01:04 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:01:04 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:01:04 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:50540 -2016-06-13 20:01:04 INFO Utils:59 - Successfully started service 'HTTP file server' on port 50540. -2016-06-13 20:01:04 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:01:04 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:01:04 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 20:01:04 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 20:01:04 INFO SparkUI:59 - Started SparkUI at http://localhost:4040 -2016-06-13 20:01:04 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:01:04 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:01:04 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 50551. -2016-06-13 20:01:04 INFO NettyBlockTransferService:59 - Server created on 50551 -2016-06-13 20:01:04 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:01:04 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:50551 with 730.6 MB RAM, BlockManagerId(driver, localhost, 50551) -2016-06-13 20:01:04 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:01:04 INFO StreamingKMeansSuite:59 - numBatches = 10, numExpectedOutput = 10 -2016-06-13 20:01:04 INFO WriteAheadLogManager :59 - Recovered 1 write ahead log files from file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/receivedBlockMetadata -2016-06-13 20:01:04 INFO ForEachDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:01:04 INFO TestInputStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:01:04 INFO TestInputStream:59 - Slide time = 1000 ms -2016-06-13 20:01:04 INFO TestInputStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:01:04 INFO TestInputStream:59 - Checkpoint interval = null -2016-06-13 20:01:04 INFO TestInputStream:59 - Remember duration = 1000 ms -2016-06-13 20:01:04 INFO TestInputStream:59 - Initialized and validated org.apache.spark.streaming.TestInputStream@4ef89a38 -2016-06-13 20:01:04 INFO ForEachDStream:59 - Slide time = 1000 ms -2016-06-13 20:01:04 INFO ForEachDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:01:04 INFO ForEachDStream:59 - Checkpoint interval = null -2016-06-13 20:01:04 INFO ForEachDStream:59 - Remember duration = 1000 ms -2016-06-13 20:01:04 INFO ForEachDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.ForEachDStream@7803410d -2016-06-13 20:01:04 INFO TestOutputStreamWithPartitions:59 - metadataCleanupDelay = -1 -2016-06-13 20:01:04 INFO MappedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:01:04 INFO ShuffledDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:01:04 INFO TransformedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:01:04 INFO MappedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:01:04 INFO TestInputStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:01:04 INFO TestInputStream:59 - Slide time = 1000 ms -2016-06-13 20:01:04 INFO TestInputStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:01:04 INFO TestInputStream:59 - Checkpoint interval = null -2016-06-13 20:01:04 INFO TestInputStream:59 - Remember duration = 1000 ms -2016-06-13 20:01:04 INFO TestInputStream:59 - Initialized and validated org.apache.spark.streaming.TestInputStream@4ef89a38 -2016-06-13 20:01:04 INFO MappedDStream:59 - Slide time = 1000 ms -2016-06-13 20:01:04 INFO MappedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:01:04 INFO MappedDStream:59 - Checkpoint interval = null -2016-06-13 20:01:04 INFO MappedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:01:04 INFO MappedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@783b50ff -2016-06-13 20:01:04 INFO TransformedDStream:59 - Slide time = 1000 ms -2016-06-13 20:01:04 INFO TransformedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:01:04 INFO TransformedDStream:59 - Checkpoint interval = null -2016-06-13 20:01:04 INFO TransformedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:01:04 INFO TransformedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.TransformedDStream@72865044 -2016-06-13 20:01:04 INFO ShuffledDStream:59 - Slide time = 1000 ms -2016-06-13 20:01:04 INFO ShuffledDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:01:04 INFO ShuffledDStream:59 - Checkpoint interval = null -2016-06-13 20:01:04 INFO ShuffledDStream:59 - Remember duration = 1000 ms -2016-06-13 20:01:04 INFO ShuffledDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.ShuffledDStream@fdad1a8 -2016-06-13 20:01:04 INFO MappedDStream:59 - Slide time = 1000 ms -2016-06-13 20:01:04 INFO MappedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:01:04 INFO MappedDStream:59 - Checkpoint interval = null -2016-06-13 20:01:04 INFO MappedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:01:04 INFO MappedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@762e335f -2016-06-13 20:01:04 INFO TestOutputStreamWithPartitions:59 - Slide time = 1000 ms -2016-06-13 20:01:04 INFO TestOutputStreamWithPartitions:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:01:04 INFO TestOutputStreamWithPartitions:59 - Checkpoint interval = null -2016-06-13 20:01:04 INFO TestOutputStreamWithPartitions:59 - Remember duration = 1000 ms -2016-06-13 20:01:04 INFO TestOutputStreamWithPartitions:59 - Initialized and validated org.apache.spark.streaming.TestOutputStreamWithPartitions@55742246 -2016-06-13 20:01:04 INFO RecurringTimer:59 - Started timer for JobGenerator at time 1000 -2016-06-13 20:01:04 INFO JobGenerator:59 - Started JobGenerator at 1000 ms -2016-06-13 20:01:04 INFO JobScheduler:59 - Started JobScheduler -2016-06-13 20:01:04 INFO StreamingContext:59 - StreamingContext started -2016-06-13 20:01:04 INFO StreamingKMeansSuite:59 - Manual clock before advancing = 0 -2016-06-13 20:01:04 INFO StreamingKMeansSuite:59 - Manual clock after advancing = 10000 -2016-06-13 20:01:04 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:01:04 INFO TestInputStream:59 - Computing RDD for time 1000 ms -2016-06-13 20:01:04 INFO TestInputStream:59 - Created RDD 0 with Vector([-0.46153320510590806], [0.9596835731142743], [-1.6648517622790264], [1.2751510348798725], [0.373751968723829]) -2016-06-13 20:01:04 INFO JobScheduler:59 - Added jobs for time 1000 ms -2016-06-13 20:01:04 INFO JobScheduler:59 - Starting job streaming job 1000 ms.0 from job set of time 1000 ms -2016-06-13 20:01:04 INFO TestInputStream:59 - Computing RDD for time 2000 ms -2016-06-13 20:01:04 INFO TestInputStream:59 - Created RDD 6 with Vector([1.0467898343007678], [0.39845257641242227], [0.7707082363415056], [-0.6430548638904149], [1.8847327284851096]) -2016-06-13 20:01:04 INFO JobScheduler:59 - Added jobs for time 2000 ms -2016-06-13 20:01:04 INFO TestInputStream:59 - Computing RDD for time 3000 ms -2016-06-13 20:01:04 INFO TestInputStream:59 - Created RDD 14 with Vector([0.09367833771946699], [-1.556260973797692], [1.0586289215398206], [-1.5954801784740555], [-0.2910199569564099]) -2016-06-13 20:01:04 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Registering RDD 7 (apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Got job 0 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:01:04 INFO DAGScheduler:59 - Final stage: ResultStage 1(apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[7] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(4680) called with curMem=0, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:01:04 INFO JobScheduler:59 - Added jobs for time 3000 ms -2016-06-13 20:01:04 INFO TestInputStream:59 - Computing RDD for time 4000 ms -2016-06-13 20:01:04 INFO TestInputStream:59 - Created RDD 20 with Vector([0.8136561059137399], [1.100753830588279], [0.16183550660344975], [0.038758305264977685], [1.3788517989765954]) -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(2437) called with curMem=4680, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:50551 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[7] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2223 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2247 bytes) -2016-06-13 20:01:04 INFO JobScheduler:59 - Added jobs for time 4000 ms -2016-06-13 20:01:04 INFO TestInputStream:59 - Computing RDD for time 5000 ms -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 20:01:04 INFO TestInputStream:59 - Created RDD 26 with Vector([-0.8376767545699404], [1.4962215228361235], [-0.9338648494739169], [-0.6375102858779765], [-0.8054748712922818]) -2016-06-13 20:01:04 INFO JobScheduler:59 - Added jobs for time 5000 ms -2016-06-13 20:01:04 INFO TestInputStream:59 - Computing RDD for time 6000 ms -2016-06-13 20:01:04 INFO TestInputStream:59 - Created RDD 32 with Vector([0.6198489137011985], [-0.30563951249527377], [-0.7528629182953331], [0.23894707804730778], [-0.7466128213890597]) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 16 ms on localhost (1/2) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO JobScheduler:59 - Added jobs for time 6000 ms -2016-06-13 20:01:04 INFO TestInputStream:59 - Computing RDD for time 7000 ms -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 23 ms on localhost (2/2) -2016-06-13 20:01:04 INFO DAGScheduler:59 - ShuffleMapStage 0 (apply at Transformer.scala:22) finished in 0.025 s -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:01:04 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:01:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:01:04 INFO TestInputStream:59 - Created RDD 38 with Vector([-1.3486691092632632], [0.7813286399318814], [1.1143994381923554], [-0.5575417841666008], [0.7551833438851623]) -2016-06-13 20:01:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 20:01:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[11] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(5200) called with curMem=7117, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(2667) called with curMem=12317, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 20:01:04 INFO JobScheduler:59 - Added jobs for time 7000 ms -2016-06-13 20:01:04 INFO TestInputStream:59 - Computing RDD for time 8000 ms -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:50551 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:01:04 INFO TestInputStream:59 - Created RDD 44 with Vector([1.2203922244884178], [1.7729332774994113], [-1.1679776365849384], [0.10625917861234317], [1.9053336750852623]) -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[11] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 5 ms on localhost (1/2) -2016-06-13 20:01:04 INFO JobScheduler:59 - Added jobs for time 8000 ms -2016-06-13 20:01:04 INFO TestInputStream:59 - Computing RDD for time 9000 ms -2016-06-13 20:01:04 INFO TestInputStream:59 - Created RDD 50 with Vector([0.09239528007779935], [1.3113558519494664], [1.8630543371339758], [-0.01662748944151289], [-0.5510810984547353]) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1466 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 8 ms on localhost (2/2) -2016-06-13 20:01:04 INFO DAGScheduler:59 - ResultStage 1 (apply at Transformer.scala:22) finished in 0.009 s -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - Job 0 finished: apply at Transformer.scala:22, took 0.050379 s -2016-06-13 20:01:04 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 5.0009765625 and centroid: [0.0964214895444316] -2016-06-13 20:01:04 INFO JobScheduler:59 - Finished job streaming job 1000 ms.0 from job set of time 1000 ms -2016-06-13 20:01:04 INFO JobScheduler:59 - Starting job streaming job 1000 ms.1 from job set of time 1000 ms -2016-06-13 20:01:04 INFO JobScheduler:59 - Added jobs for time 9000 ms -2016-06-13 20:01:04 INFO TestInputStream:59 - Computing RDD for time 10000 ms -2016-06-13 20:01:04 INFO TestInputStream:59 - Created RDD 57 with Vector([-0.27992816252673763], [0.6488371767796974], [-1.0956379827475635], [1.7638193518785783], [0.2345128605502197]) -2016-06-13 20:01:04 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Registering RDD 3 (apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Got job 1 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:01:04 INFO DAGScheduler:59 - Final stage: ResultStage 3(apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 2) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 2) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (UnionRDD[3] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=14984, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:01:04 INFO JobScheduler:59 - Added jobs for time 10000 ms -2016-06-13 20:01:04 INFO JobGenerator:59 - Checkpointing graph for time 1000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updating checkpoint data for time 1000 ms -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(2000) called with curMem=18264, maxMem=766075207 -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updated checkpoint data for time 1000 ms -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2000.0 B, free 730.6 MB) -2016-06-13 20:01:04 INFO JobGenerator:59 - Checkpointing graph for time 2000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updating checkpoint data for time 2000 ms -2016-06-13 20:01:04 INFO CheckpointWriter:59 - Saving checkpoint for time 1000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-1000' -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updated checkpoint data for time 2000 ms -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:50551 (size: 2000.0 B, free: 730.6 MB) -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO JobGenerator:59 - Checkpointing graph for time 3000 ms -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 2 (UnionRDD[3] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updating checkpoint data for time 3000 ms -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updated checkpoint data for time 3000 ms -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2332 bytes) -2016-06-13 20:01:04 INFO JobGenerator:59 - Checkpointing graph for time 4000 ms -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2356 bytes) -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updating checkpoint data for time 4000 ms -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updated checkpoint data for time 4000 ms -2016-06-13 20:01:04 INFO JobGenerator:59 - Checkpointing graph for time 5000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updating checkpoint data for time 5000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updated checkpoint data for time 5000 ms -2016-06-13 20:01:04 INFO JobGenerator:59 - Checkpointing graph for time 6000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updating checkpoint data for time 6000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updated checkpoint data for time 6000 ms -2016-06-13 20:01:04 INFO JobGenerator:59 - Checkpointing graph for time 7000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updating checkpoint data for time 7000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updated checkpoint data for time 7000 ms -2016-06-13 20:01:04 INFO JobGenerator:59 - Checkpointing graph for time 8000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updating checkpoint data for time 8000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updated checkpoint data for time 8000 ms -2016-06-13 20:01:04 INFO JobGenerator:59 - Checkpointing graph for time 9000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updating checkpoint data for time 9000 ms -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updated checkpoint data for time 9000 ms -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-06-13 20:01:04 INFO JobGenerator:59 - Checkpointing graph for time 10000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updating checkpoint data for time 10000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updated checkpoint data for time 10000 ms -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 17 ms on localhost (1/3) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 20 ms on localhost (2/3) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 6). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 6) in 11 ms on localhost (3/3) -2016-06-13 20:01:04 INFO DAGScheduler:59 - ShuffleMapStage 2 (apply at Transformer.scala:22) finished in 0.027 s -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:01:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 3) -2016-06-13 20:01:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents for ResultStage 3: List() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[54] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=20264, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(1798) called with curMem=23504, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1798.0 B, free 730.6 MB) -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:50551 (size: 1798.0 B, free: 730.6 MB) -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[54] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 7) -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 8) -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 8). 1172 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 8) in 7 ms on localhost (1/2) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 7). 1180 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 7) in 11 ms on localhost (2/2) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - ResultStage 3 (apply at Transformer.scala:22) finished in 0.012 s -2016-06-13 20:01:04 INFO DAGScheduler:59 - Job 1 finished: apply at Transformer.scala:22, took 0.060932 s -2016-06-13 20:01:04 INFO JobScheduler:59 - Finished job streaming job 1000 ms.1 from job set of time 1000 ms -2016-06-13 20:01:04 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:01:04 INFO JobScheduler:59 - Total delay: 1465819263.375 s for time 1000 ms (execution: 0.140 s) -2016-06-13 20:01:04 INFO JobScheduler:59 - Starting job streaming job 2000 ms.0 from job set of time 2000 ms -2016-06-13 20:01:04 INFO JobGenerator:59 - Checkpointing graph for time 1000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updating checkpoint data for time 1000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updated checkpoint data for time 1000 ms -2016-06-13 20:01:04 INFO DAGScheduler:59 - Registering RDD 63 (apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Got job 2 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:01:04 INFO DAGScheduler:59 - Final stage: ResultStage 5(apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[63] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(4680) called with curMem=25302, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(2446) called with curMem=29982, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:50551 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[63] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2223 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 10, localhost, PROCESS_LOCAL, 2247 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 10) -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 9) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 10). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 10) in 12 ms on localhost (1/2) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 9). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 9) in 18 ms on localhost (2/2) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - ShuffleMapStage 4 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:01:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:01:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-06-13 20:01:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ResultStage 5 (ShuffledRDD[64] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(5200) called with curMem=32428, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(2675) called with curMem=37628, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:50551 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (ShuffledRDD[64] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 12, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 11) -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 12) -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 12). 1161 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 12) in 4 ms on localhost (1/2) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 11). 1466 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 11) in 9 ms on localhost (2/2) -2016-06-13 20:01:04 INFO DAGScheduler:59 - ResultStage 5 (apply at Transformer.scala:22) finished in 0.009 s -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - Job 2 finished: apply at Transformer.scala:22, took 0.055748 s -2016-06-13 20:01:04 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 5.004883766174316 and centroid: [0.69094499956826] -2016-06-13 20:01:04 INFO JobScheduler:59 - Finished job streaming job 2000 ms.0 from job set of time 2000 ms -2016-06-13 20:01:04 INFO JobScheduler:59 - Starting job streaming job 2000 ms.1 from job set of time 2000 ms -2016-06-13 20:01:04 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Registering RDD 10 (apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Got job 3 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:01:04 INFO DAGScheduler:59 - Final stage: ResultStage 7(apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (UnionRDD[10] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:01:04 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-1000 -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=40303, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=43583, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:01:04 INFO CheckpointWriter:59 - Checkpoint for time 1000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-1000', took 5640 bytes and 147 ms -2016-06-13 20:01:04 INFO CheckpointWriter:59 - Saving checkpoint for time 2000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-2000' -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:50551 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 6 (UnionRDD[10] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 3 tasks -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 2332 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 14, localhost, PROCESS_LOCAL, 2356 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 14) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 14). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 15, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 15) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 14) in 11 ms on localhost (1/3) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 15). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 13) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 15) in 13 ms on localhost (2/3) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 13). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 13) in 31 ms on localhost (3/3) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - ShuffleMapStage 6 (apply at Transformer.scala:22) finished in 0.032 s -2016-06-13 20:01:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:01:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-06-13 20:01:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[65] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=45592, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=48832, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.5 MB) -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:50551 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[65] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 16) -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 17) -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 17). 1172 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 17) in 6 ms on localhost (1/2) -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 16). 1180 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 16) in 14 ms on localhost (2/2) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - ResultStage 7 (apply at Transformer.scala:22) finished in 0.015 s -2016-06-13 20:01:04 INFO DAGScheduler:59 - Job 3 finished: apply at Transformer.scala:22, took 0.082259 s -2016-06-13 20:01:04 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:01:04 INFO JobScheduler:59 - Finished job streaming job 2000 ms.1 from job set of time 2000 ms -2016-06-13 20:01:04 INFO JobScheduler:59 - Total delay: 1465819262.538 s for time 2000 ms (execution: 0.159 s) -2016-06-13 20:01:04 INFO JobScheduler:59 - Starting job streaming job 3000 ms.0 from job set of time 3000 ms -2016-06-13 20:01:04 INFO ParallelCollectionRDD:59 - Removing RDD 0 from persistence list -2016-06-13 20:01:04 INFO DAGScheduler:59 - Registering RDD 66 (apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Got job 4 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:01:04 INFO DAGScheduler:59 - Final stage: ResultStage 9(apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 8) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 8) -2016-06-13 20:01:04 INFO MapPartitionsRDD:59 - Removing RDD 5 from persistence list -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 8 (MapPartitionsRDD[66] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:01:04 INFO ShuffledRDD:59 - Removing RDD 4 from persistence list -2016-06-13 20:01:04 INFO BlockManager:59 - Removing RDD 5 -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(4680) called with curMem=50634, maxMem=766075207 -2016-06-13 20:01:04 INFO BlockManager:59 - Removing RDD 0 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO UnionRDD:59 - Removing RDD 3 from persistence list -2016-06-13 20:01:04 INFO BlockManager:59 - Removing RDD 4 -2016-06-13 20:01:04 INFO MapPartitionsRDD:59 - Removing RDD 1 from persistence list -2016-06-13 20:01:04 INFO BlockManager:59 - Removing RDD 3 -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(2455) called with curMem=55314, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO JobGenerator:59 - Checkpointing graph for time 2000 ms -2016-06-13 20:01:04 INFO BlockManager:59 - Removing RDD 1 -2016-06-13 20:01:04 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updating checkpoint data for time 2000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updated checkpoint data for time 2000 ms -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:50551 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 8 (MapPartitionsRDD[66] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 18, localhost, PROCESS_LOCAL, 2223 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 19, localhost, PROCESS_LOCAL, 2247 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 18) -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 19) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 19). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 18). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 19) in 9 ms on localhost (1/2) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 18) in 10 ms on localhost (2/2) -2016-06-13 20:01:04 INFO DAGScheduler:59 - ShuffleMapStage 8 (apply at Transformer.scala:22) finished in 0.011 s -2016-06-13 20:01:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:01:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 9) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents for ResultStage 9: List() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ResultStage 9 (ShuffledRDD[67] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(5200) called with curMem=57769, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(2674) called with curMem=62969, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:50551 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (ShuffledRDD[67] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 21, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 20) -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 21) -2016-06-13 20:01:04 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-2000 -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 21). 1161 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 21) in 5 ms on localhost (1/2) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 20). 1466 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 20) in 7 ms on localhost (2/2) -2016-06-13 20:01:04 INFO DAGScheduler:59 - ResultStage 9 (apply at Transformer.scala:22) finished in 0.009 s -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - Job 4 finished: apply at Transformer.scala:22, took 0.048782 s -2016-06-13 20:01:04 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 5.004887581802905 and centroid: [-0.4569686656055322] -2016-06-13 20:01:04 INFO StreamingKMeansModel:59 - Cluster 1 is dying. Split the largest cluster 0 into two. -2016-06-13 20:01:04 INFO JobScheduler:59 - Finished job streaming job 3000 ms.0 from job set of time 3000 ms -2016-06-13 20:01:04 INFO JobScheduler:59 - Starting job streaming job 3000 ms.1 from job set of time 3000 ms -2016-06-13 20:01:04 INFO CheckpointWriter:59 - Checkpoint for time 2000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-2000', took 5642 bytes and 133 ms -2016-06-13 20:01:04 INFO CheckpointWriter:59 - Saving checkpoint for time 3000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-3000' -2016-06-13 20:01:04 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Registering RDD 17 (apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Got job 5 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:01:04 INFO DAGScheduler:59 - Final stage: ResultStage 11(apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 10) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 10) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 10 (UnionRDD[17] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=65643, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(2006) called with curMem=68923, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2006.0 B, free 730.5 MB) -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:50551 (size: 2006.0 B, free: 730.6 MB) -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 10 (UnionRDD[17] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 3 tasks -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 22, localhost, PROCESS_LOCAL, 2332 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 23, localhost, PROCESS_LOCAL, 2356 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 23) -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 22) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 23). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 2.0 in stage 10.0 (TID 24, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 2.0 in stage 10.0 (TID 24) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 23) in 11 ms on localhost (1/3) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 22). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 22) in 13 ms on localhost (2/3) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 2.0 in stage 10.0 (TID 24). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 2.0 in stage 10.0 (TID 24) in 12 ms on localhost (3/3) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - ShuffleMapStage 10 (apply at Transformer.scala:22) finished in 0.023 s -2016-06-13 20:01:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:01:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 11) -2016-06-13 20:01:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents for ResultStage 11: List() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[68] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=70929, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=74169, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.5 MB) -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:50551 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[68] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 25, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 26, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 25) -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 26) -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 26). 1172 bytes result sent to driver -2016-06-13 20:01:04 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 25). 1180 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 26) in 9 ms on localhost (1/2) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 25) in 10 ms on localhost (2/2) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - ResultStage 11 (apply at Transformer.scala:22) finished in 0.011 s -2016-06-13 20:01:04 INFO DAGScheduler:59 - Job 5 finished: apply at Transformer.scala:22, took 0.060069 s -2016-06-13 20:01:04 INFO JobScheduler:59 - Finished job streaming job 3000 ms.1 from job set of time 3000 ms -2016-06-13 20:01:04 INFO JobScheduler:59 - Total delay: 1465819261.655 s for time 3000 ms (execution: 0.117 s) -2016-06-13 20:01:04 INFO JobScheduler:59 - Starting job streaming job 4000 ms.0 from job set of time 4000 ms -2016-06-13 20:01:04 INFO ParallelCollectionRDD:59 - Removing RDD 6 from persistence list -2016-06-13 20:01:04 INFO MapPartitionsRDD:59 - Removing RDD 13 from persistence list -2016-06-13 20:01:04 INFO BlockManager:59 - Removing RDD 6 -2016-06-13 20:01:04 INFO ShuffledRDD:59 - Removing RDD 12 from persistence list -2016-06-13 20:01:04 INFO BlockManager:59 - Removing RDD 13 -2016-06-13 20:01:04 INFO UnionRDD:59 - Removing RDD 10 from persistence list -2016-06-13 20:01:04 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 20:01:04 INFO BlockManager:59 - Removing RDD 10 -2016-06-13 20:01:04 INFO MapPartitionsRDD:59 - Removing RDD 8 from persistence list -2016-06-13 20:01:04 INFO JobGenerator:59 - Checkpointing graph for time 3000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updating checkpoint data for time 3000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updated checkpoint data for time 3000 ms -2016-06-13 20:01:04 INFO BlockManager:59 - Removing RDD 8 -2016-06-13 20:01:04 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Registering RDD 69 (apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Got job 6 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:01:04 INFO DAGScheduler:59 - Final stage: ResultStage 13(apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (MapPartitionsRDD[69] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(4680) called with curMem=75971, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(2445) called with curMem=80651, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:50551 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (MapPartitionsRDD[69] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 27, localhost, PROCESS_LOCAL, 2223 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 28, localhost, PROCESS_LOCAL, 2247 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 28) -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 27) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 27). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 27) in 15 ms on localhost (1/2) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 28). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 28) in 16 ms on localhost (2/2) -2016-06-13 20:01:04 INFO DAGScheduler:59 - ShuffleMapStage 12 (apply at Transformer.scala:22) finished in 0.017 s -2016-06-13 20:01:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:01:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ResultStage 13 (ShuffledRDD[70] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:01:04 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(5200) called with curMem=83096, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(2675) called with curMem=88296, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:50551 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (ShuffledRDD[70] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 29, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 30, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 30) -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 29) -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 30). 1161 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 30) in 4 ms on localhost (1/2) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 29). 1466 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 29) in 7 ms on localhost (2/2) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - ResultStage 13 (apply at Transformer.scala:22) finished in 0.007 s -2016-06-13 20:01:04 INFO DAGScheduler:59 - Job 6 finished: apply at Transformer.scala:22, took 0.046429 s -2016-06-13 20:01:04 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-3000 -2016-06-13 20:01:04 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 5.002443792765007 and centroid: [0.698206507723229] -2016-06-13 20:01:04 INFO JobScheduler:59 - Finished job streaming job 4000 ms.0 from job set of time 4000 ms -2016-06-13 20:01:04 INFO JobScheduler:59 - Starting job streaming job 4000 ms.1 from job set of time 4000 ms -2016-06-13 20:01:04 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Registering RDD 23 (apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Got job 7 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:01:04 INFO DAGScheduler:59 - Final stage: ResultStage 15(apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 14) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 14) -2016-06-13 20:01:04 INFO CheckpointWriter:59 - Checkpoint for time 3000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-3000', took 5642 bytes and 138 ms -2016-06-13 20:01:04 INFO CheckpointWriter:59 - Saving checkpoint for time 4000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-4000' -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 14 (UnionRDD[23] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=90971, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=94251, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:50551 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 14 (UnionRDD[23] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 3 tasks -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 31, localhost, PROCESS_LOCAL, 2332 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 32, localhost, PROCESS_LOCAL, 2356 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 31) -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 32) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 32). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 33, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 31). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 32) in 9 ms on localhost (1/3) -2016-06-13 20:01:04 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 33) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 31) in 10 ms on localhost (2/3) -2016-06-13 20:01:04 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:01:04 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 33). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 33) in 12 ms on localhost (3/3) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - ShuffleMapStage 14 (apply at Transformer.scala:22) finished in 0.022 s -2016-06-13 20:01:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:01:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 15) -2016-06-13 20:01:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents for ResultStage 15: List() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[71] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=96260, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(1801) called with curMem=99500, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1801.0 B, free 730.5 MB) -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:50551 (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[71] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 34, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 35, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 34) -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 35) -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 35). 1172 bytes result sent to driver -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 34). 1180 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 34) in 10 ms on localhost (1/2) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 35) in 11 ms on localhost (2/2) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - ResultStage 15 (apply at Transformer.scala:22) finished in 0.014 s -2016-06-13 20:01:04 INFO DAGScheduler:59 - Job 7 finished: apply at Transformer.scala:22, took 0.057039 s -2016-06-13 20:01:04 INFO JobScheduler:59 - Finished job streaming job 4000 ms.1 from job set of time 4000 ms -2016-06-13 20:01:04 INFO JobScheduler:59 - Total delay: 1465819260.790 s for time 4000 ms (execution: 0.134 s) -2016-06-13 20:01:04 INFO JobScheduler:59 - Starting job streaming job 5000 ms.0 from job set of time 5000 ms -2016-06-13 20:01:04 INFO ParallelCollectionRDD:59 - Removing RDD 14 from persistence list -2016-06-13 20:01:04 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:01:04 INFO BlockManager:59 - Removing RDD 14 -2016-06-13 20:01:04 INFO MapPartitionsRDD:59 - Removing RDD 19 from persistence list -2016-06-13 20:01:04 INFO DAGScheduler:59 - Registering RDD 72 (apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO BlockManager:59 - Removing RDD 19 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Got job 8 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:01:04 INFO DAGScheduler:59 - Final stage: ResultStage 17(apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 16) -2016-06-13 20:01:04 INFO ShuffledRDD:59 - Removing RDD 18 from persistence list -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 16) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 16 (MapPartitionsRDD[72] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(4680) called with curMem=101301, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(2454) called with curMem=105981, maxMem=766075207 -2016-06-13 20:01:04 INFO BlockManager:59 - Removing RDD 18 -2016-06-13 20:01:04 INFO UnionRDD:59 - Removing RDD 17 from persistence list -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO MapPartitionsRDD:59 - Removing RDD 15 from persistence list -2016-06-13 20:01:04 INFO BlockManager:59 - Removing RDD 17 -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:50551 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 20:01:04 INFO JobGenerator:59 - Checkpointing graph for time 4000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updating checkpoint data for time 4000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updated checkpoint data for time 4000 ms -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 16 (MapPartitionsRDD[72] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-06-13 20:01:04 INFO BlockManager:59 - Removing RDD 15 -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 36, localhost, PROCESS_LOCAL, 2223 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 37, localhost, PROCESS_LOCAL, 2247 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 36) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 36). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 37) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 36) in 31 ms on localhost (1/2) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 37). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 37) in 32 ms on localhost (2/2) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - ShuffleMapStage 16 (apply at Transformer.scala:22) finished in 0.034 s -2016-06-13 20:01:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:01:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 17) -2016-06-13 20:01:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents for ResultStage 17: List() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ResultStage 17 (ShuffledRDD[73] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(5200) called with curMem=108435, maxMem=766075207 -2016-06-13 20:01:04 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(2684) called with curMem=113635, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:50551 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 20:01:04 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-4000 -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (ShuffledRDD[73] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 38, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 39, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 39) -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 38) -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 2 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 38). 1466 bytes result sent to driver -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 39). 1466 bytes result sent to driver -2016-06-13 20:01:04 INFO CheckpointWriter:59 - Checkpoint for time 4000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-4000', took 5642 bytes and 136 ms -2016-06-13 20:01:04 INFO CheckpointWriter:59 - Saving checkpoint for time 5000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-5000' -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 39) in 8 ms on localhost (1/2) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 38) in 8 ms on localhost (2/2) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - ResultStage 17 (apply at Transformer.scala:22) finished in 0.008 s -2016-06-13 20:01:04 INFO DAGScheduler:59 - Job 8 finished: apply at Transformer.scala:22, took 0.068941 s -2016-06-13 20:01:04 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 1.004885199016372 and centroid: [1.4923420128476237] -2016-06-13 20:01:04 INFO StreamingKMeansModel:59 - Cluster 1 updated with weight 4.000002386516372 and centroid: [-0.8036314834744063] -2016-06-13 20:01:04 INFO JobScheduler:59 - Finished job streaming job 5000 ms.0 from job set of time 5000 ms -2016-06-13 20:01:04 INFO JobScheduler:59 - Starting job streaming job 5000 ms.1 from job set of time 5000 ms -2016-06-13 20:01:04 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Registering RDD 29 (apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Got job 9 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:01:04 INFO DAGScheduler:59 - Final stage: ResultStage 19(apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 18) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 18) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 18 (UnionRDD[29] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=116319, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(2010) called with curMem=119599, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2010.0 B, free 730.5 MB) -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:50551 (size: 2010.0 B, free: 730.5 MB) -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 18 (UnionRDD[29] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 3 tasks -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 40, localhost, PROCESS_LOCAL, 2332 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 41, localhost, PROCESS_LOCAL, 2356 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 40) -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 41) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 40). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 42, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 40) in 11 ms on localhost (1/3) -2016-06-13 20:01:04 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 42) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 41). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 42). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 41) in 19 ms on localhost (2/3) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 42) in 11 ms on localhost (3/3) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - ShuffleMapStage 18 (apply at Transformer.scala:22) finished in 0.021 s -2016-06-13 20:01:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:01:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 19) -2016-06-13 20:01:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents for ResultStage 19: List() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[74] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=121609, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=124849, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:50551 (size: 1803.0 B, free: 730.5 MB) -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[74] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 43, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 44, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 44) -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 43) -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 44). 1172 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 44) in 5 ms on localhost (1/2) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 43). 1180 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 43) in 10 ms on localhost (2/2) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - ResultStage 19 (apply at Transformer.scala:22) finished in 0.011 s -2016-06-13 20:01:04 INFO DAGScheduler:59 - Job 9 finished: apply at Transformer.scala:22, took 0.059534 s -2016-06-13 20:01:04 INFO JobScheduler:59 - Finished job streaming job 5000 ms.1 from job set of time 5000 ms -2016-06-13 20:01:04 INFO JobScheduler:59 - Total delay: 1465819259.933 s for time 5000 ms (execution: 0.142 s) -2016-06-13 20:01:04 INFO JobScheduler:59 - Starting job streaming job 6000 ms.0 from job set of time 6000 ms -2016-06-13 20:01:04 INFO ParallelCollectionRDD:59 - Removing RDD 20 from persistence list -2016-06-13 20:01:04 INFO BlockManager:59 - Removing RDD 20 -2016-06-13 20:01:04 INFO MapPartitionsRDD:59 - Removing RDD 25 from persistence list -2016-06-13 20:01:04 INFO ShuffledRDD:59 - Removing RDD 24 from persistence list -2016-06-13 20:01:04 INFO UnionRDD:59 - Removing RDD 23 from persistence list -2016-06-13 20:01:04 INFO MapPartitionsRDD:59 - Removing RDD 21 from persistence list -2016-06-13 20:01:04 INFO BlockManager:59 - Removing RDD 25 -2016-06-13 20:01:04 INFO BlockManager:59 - Removing RDD 24 -2016-06-13 20:01:04 INFO JobGenerator:59 - Checkpointing graph for time 5000 ms -2016-06-13 20:01:04 INFO BlockManager:59 - Removing RDD 21 -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updating checkpoint data for time 5000 ms -2016-06-13 20:01:04 INFO DStreamGraph:59 - Updated checkpoint data for time 5000 ms -2016-06-13 20:01:04 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Registering RDD 75 (apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Got job 10 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:01:04 INFO DAGScheduler:59 - Final stage: ResultStage 21(apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 20) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 20) -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 20 (MapPartitionsRDD[75] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(4680) called with curMem=126652, maxMem=766075207 -2016-06-13 20:01:04 INFO BlockManager:59 - Removing RDD 23 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(2456) called with curMem=131332, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:50551 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 20 (MapPartitionsRDD[75] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 45, localhost, PROCESS_LOCAL, 2223 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 46, localhost, PROCESS_LOCAL, 2247 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 45) -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 46) -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 45). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 46). 1159 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 45) in 22 ms on localhost (1/2) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 46) in 22 ms on localhost (2/2) -2016-06-13 20:01:04 INFO DAGScheduler:59 - ShuffleMapStage 20 (apply at Transformer.scala:22) finished in 0.023 s -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 20:01:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:01:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 21) -2016-06-13 20:01:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Missing parents for ResultStage 21: List() -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting ResultStage 21 (ShuffledRDD[76] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(5200) called with curMem=133788, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO MemoryStore:59 - ensureFreeSpace(2685) called with curMem=138988, maxMem=766075207 -2016-06-13 20:01:04 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:01:04 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:50551 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 20:01:04 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (ShuffledRDD[76] at apply at Transformer.scala:22) -2016-06-13 20:01:04 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:04 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 48) -2016-06-13 20:01:04 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 47) -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 2 blocks -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:01:04 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-5000 -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:04 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 47). 1466 bytes result sent to driver -2016-06-13 20:01:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 47) in 7 ms on localhost (1/2) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 48). 1466 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 48) in 7 ms on localhost (2/2) -2016-06-13 20:01:05 INFO DAGScheduler:59 - ResultStage 21 (apply at Transformer.scala:22) finished in 0.008 s -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 20:01:05 INFO DAGScheduler:59 - Job 10 finished: apply at Transformer.scala:22, took 0.050826 s -2016-06-13 20:01:05 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 1.0009813332021644 and centroid: [0.6207042807479672] -2016-06-13 20:01:05 INFO StreamingKMeansModel:59 - Cluster 1 updated with weight 4.003906252330583 and centroid: [-0.3919440822507151] -2016-06-13 20:01:05 INFO JobScheduler:59 - Finished job streaming job 6000 ms.0 from job set of time 6000 ms -2016-06-13 20:01:05 INFO JobScheduler:59 - Starting job streaming job 6000 ms.1 from job set of time 6000 ms -2016-06-13 20:01:05 INFO CheckpointWriter:59 - Checkpoint for time 5000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-5000', took 5645 bytes and 141 ms -2016-06-13 20:01:05 INFO CheckpointWriter:59 - Saving checkpoint for time 6000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-6000' -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:01:05 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Registering RDD 35 (apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Got job 11 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:01:05 INFO DAGScheduler:59 - Final stage: ResultStage 23(apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 22) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 22) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 22 (UnionRDD[35] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=141673, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=144953, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.4 MB) -2016-06-13 20:01:05 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:50551 (size: 2007.0 B, free: 730.5 MB) -2016-06-13 20:01:05 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 22 (UnionRDD[35] at apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 3 tasks -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 49, localhost, PROCESS_LOCAL, 2332 bytes) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 50, localhost, PROCESS_LOCAL, 2356 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 49) -2016-06-13 20:01:05 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 50) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 49). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 51, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 51) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 50). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 49) in 12 ms on localhost (1/3) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 50) in 12 ms on localhost (2/3) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 51). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 51) in 11 ms on localhost (3/3) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 20:01:05 INFO DAGScheduler:59 - ShuffleMapStage 22 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:01:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:01:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:01:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 23) -2016-06-13 20:01:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:01:05 INFO DAGScheduler:59 - Missing parents for ResultStage 23: List() -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[77] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=146960, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=150200, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.4 MB) -2016-06-13 20:01:05 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:50551 (size: 1803.0 B, free: 730.5 MB) -2016-06-13 20:01:05 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[77] at apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 52, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 52) -2016-06-13 20:01:05 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 53) -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:05 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 53). 1172 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 53) in 3 ms on localhost (1/2) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 52). 1180 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 52) in 6 ms on localhost (2/2) -2016-06-13 20:01:05 INFO DAGScheduler:59 - ResultStage 23 (apply at Transformer.scala:22) finished in 0.006 s -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 20:01:05 INFO DAGScheduler:59 - Job 11 finished: apply at Transformer.scala:22, took 0.047651 s -2016-06-13 20:01:05 INFO JobScheduler:59 - Finished job streaming job 6000 ms.1 from job set of time 6000 ms -2016-06-13 20:01:05 INFO JobScheduler:59 - Total delay: 1465819259.056 s for time 6000 ms (execution: 0.123 s) -2016-06-13 20:01:05 INFO ParallelCollectionRDD:59 - Removing RDD 26 from persistence list -2016-06-13 20:01:05 INFO JobScheduler:59 - Starting job streaming job 7000 ms.0 from job set of time 7000 ms -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 26 -2016-06-13 20:01:05 INFO MapPartitionsRDD:59 - Removing RDD 31 from persistence list -2016-06-13 20:01:05 INFO ShuffledRDD:59 - Removing RDD 30 from persistence list -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 31 -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 30 -2016-06-13 20:01:05 INFO UnionRDD:59 - Removing RDD 29 from persistence list -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 29 -2016-06-13 20:01:05 INFO MapPartitionsRDD:59 - Removing RDD 27 from persistence list -2016-06-13 20:01:05 INFO JobGenerator:59 - Checkpointing graph for time 6000 ms -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 27 -2016-06-13 20:01:05 INFO DStreamGraph:59 - Updating checkpoint data for time 6000 ms -2016-06-13 20:01:05 INFO DStreamGraph:59 - Updated checkpoint data for time 6000 ms -2016-06-13 20:01:05 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Registering RDD 78 (apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Got job 12 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:01:05 INFO DAGScheduler:59 - Final stage: ResultStage 25(apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 24) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 24) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 24 (MapPartitionsRDD[78] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(4680) called with curMem=152003, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 4.6 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(2454) called with curMem=156683, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:50551 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 20:01:05 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 24 (MapPartitionsRDD[78] at apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 54, localhost, PROCESS_LOCAL, 2223 bytes) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 55, localhost, PROCESS_LOCAL, 2247 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 54) -2016-06-13 20:01:05 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 55) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 55). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 54). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 55) in 13 ms on localhost (1/2) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 54) in 15 ms on localhost (2/2) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-06-13 20:01:05 INFO DAGScheduler:59 - ShuffleMapStage 24 (apply at Transformer.scala:22) finished in 0.015 s -2016-06-13 20:01:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:01:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:01:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 25) -2016-06-13 20:01:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:01:05 INFO DAGScheduler:59 - Missing parents for ResultStage 25: List() -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting ResultStage 25 (ShuffledRDD[79] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(5200) called with curMem=159137, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(2686) called with curMem=164337, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:50551 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 20:01:05 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (ShuffledRDD[79] at apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 57) -2016-06-13 20:01:05 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 56) -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:01:05 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 56). 1466 bytes result sent to driver -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 56) in 15 ms on localhost (1/2) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 57). 1466 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 57) in 17 ms on localhost (2/2) -2016-06-13 20:01:05 INFO DAGScheduler:59 - ResultStage 25 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-06-13 20:01:05 INFO DAGScheduler:59 - Job 12 finished: apply at Transformer.scala:22, took 0.059733 s -2016-06-13 20:01:05 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 3.0009775208332052 and centroid: [0.8835514944606816] -2016-06-13 20:01:05 INFO StreamingKMeansModel:59 - Cluster 1 updated with weight 2.0039100646995416 and centroid: [-0.9520104987526505] -2016-06-13 20:01:05 INFO JobScheduler:59 - Finished job streaming job 7000 ms.0 from job set of time 7000 ms -2016-06-13 20:01:05 INFO JobScheduler:59 - Starting job streaming job 7000 ms.1 from job set of time 7000 ms -2016-06-13 20:01:05 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Registering RDD 41 (apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Got job 13 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:01:05 INFO DAGScheduler:59 - Final stage: ResultStage 27(apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (UnionRDD[41] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=167023, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=170303, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.4 MB) -2016-06-13 20:01:05 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:50551 (size: 2009.0 B, free: 730.5 MB) -2016-06-13 20:01:05 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 26 (UnionRDD[41] at apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 3 tasks -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 58, localhost, PROCESS_LOCAL, 2332 bytes) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 59, localhost, PROCESS_LOCAL, 2356 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 58) -2016-06-13 20:01:05 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 59) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 58). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 60, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 59). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 60) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 59) in 11 ms on localhost (1/3) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 58) in 12 ms on localhost (2/3) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 60). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:01:05 INFO DAGScheduler:59 - ShuffleMapStage 26 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 60) in 9 ms on localhost (3/3) -2016-06-13 20:01:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:01:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-06-13 20:01:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-06-13 20:01:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:01:05 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[80] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=172312, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(1801) called with curMem=175552, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 1801.0 B, free 730.4 MB) -2016-06-13 20:01:05 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:50551 (size: 1801.0 B, free: 730.5 MB) -2016-06-13 20:01:05 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[80] at apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 62, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 62) -2016-06-13 20:01:05 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 61) -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:05 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 62). 1172 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 62) in 5 ms on localhost (1/2) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 61). 1180 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 61) in 10 ms on localhost (2/2) -2016-06-13 20:01:05 INFO DAGScheduler:59 - ResultStage 27 (apply at Transformer.scala:22) finished in 0.010 s -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-06-13 20:01:05 INFO DAGScheduler:59 - Job 13 finished: apply at Transformer.scala:22, took 0.055197 s -2016-06-13 20:01:05 INFO JobScheduler:59 - Finished job streaming job 7000 ms.1 from job set of time 7000 ms -2016-06-13 20:01:05 INFO JobScheduler:59 - Total delay: 1465819258.190 s for time 7000 ms (execution: 0.134 s) -2016-06-13 20:01:05 INFO ParallelCollectionRDD:59 - Removing RDD 32 from persistence list -2016-06-13 20:01:05 INFO JobScheduler:59 - Starting job streaming job 8000 ms.0 from job set of time 8000 ms -2016-06-13 20:01:05 INFO CheckpointWriter:59 - Checkpoint for time 6000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-6000', took 5639 bytes and 186 ms -2016-06-13 20:01:05 INFO CheckpointWriter:59 - Saving checkpoint for time 7000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-7000' -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 32 -2016-06-13 20:01:05 INFO MapPartitionsRDD:59 - Removing RDD 37 from persistence list -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 37 -2016-06-13 20:01:05 INFO ShuffledRDD:59 - Removing RDD 36 from persistence list -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 36 -2016-06-13 20:01:05 INFO UnionRDD:59 - Removing RDD 35 from persistence list -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 35 -2016-06-13 20:01:05 INFO MapPartitionsRDD:59 - Removing RDD 33 from persistence list -2016-06-13 20:01:05 INFO JobGenerator:59 - Checkpointing graph for time 7000 ms -2016-06-13 20:01:05 INFO DStreamGraph:59 - Updating checkpoint data for time 7000 ms -2016-06-13 20:01:05 INFO DStreamGraph:59 - Updated checkpoint data for time 7000 ms -2016-06-13 20:01:05 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 33 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Registering RDD 81 (apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Got job 14 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:01:05 INFO DAGScheduler:59 - Final stage: ResultStage 29(apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 28) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 28) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 28 (MapPartitionsRDD[81] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(4680) called with curMem=177353, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 4.6 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(2456) called with curMem=182033, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:50551 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 20:01:05 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 28 (MapPartitionsRDD[81] at apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 63, localhost, PROCESS_LOCAL, 2223 bytes) -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 64, localhost, PROCESS_LOCAL, 2247 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 63) -2016-06-13 20:01:05 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 64) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 64). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 63). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 64) in 17 ms on localhost (1/2) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 63) in 18 ms on localhost (2/2) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-06-13 20:01:05 INFO DAGScheduler:59 - ShuffleMapStage 28 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:01:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:01:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:01:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 29) -2016-06-13 20:01:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:01:05 INFO DAGScheduler:59 - Missing parents for ResultStage 29: List() -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting ResultStage 29 (ShuffledRDD[82] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(5200) called with curMem=184489, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(2684) called with curMem=189689, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:50551 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 20:01:05 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (ShuffledRDD[82] at apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 65, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 66, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 66) -2016-06-13 20:01:05 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 65) -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 2 blocks -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:01:05 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 66). 1466 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 66) in 7 ms on localhost (1/2) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 65). 1466 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 65) in 8 ms on localhost (2/2) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 20:01:05 INFO DAGScheduler:59 - ResultStage 29 (apply at Transformer.scala:22) finished in 0.010 s -2016-06-13 20:01:05 INFO DAGScheduler:59 - Job 14 finished: apply at Transformer.scala:22, took 0.051059 s -2016-06-13 20:01:05 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 4.002930642110189 and centroid: [1.2509604029166639] -2016-06-13 20:01:05 INFO StreamingKMeansModel:59 - Cluster 1 updated with weight 1.0019569434225581 and centroid: [-1.1675558265733965] -2016-06-13 20:01:05 INFO JobScheduler:59 - Finished job streaming job 8000 ms.0 from job set of time 8000 ms -2016-06-13 20:01:05 INFO JobScheduler:59 - Starting job streaming job 8000 ms.1 from job set of time 8000 ms -2016-06-13 20:01:05 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Registering RDD 47 (apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Got job 15 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:01:05 INFO DAGScheduler:59 - Final stage: ResultStage 31(apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 30) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 30 (UnionRDD[47] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=192373, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(2006) called with curMem=195653, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2006.0 B, free 730.4 MB) -2016-06-13 20:01:05 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:50551 (size: 2006.0 B, free: 730.5 MB) -2016-06-13 20:01:05 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 30 (UnionRDD[47] at apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 3 tasks -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 67, localhost, PROCESS_LOCAL, 2332 bytes) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 68, localhost, PROCESS_LOCAL, 2356 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 68) -2016-06-13 20:01:05 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 67) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 68). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 69, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 69) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 68) in 12 ms on localhost (1/3) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 67). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 67) in 18 ms on localhost (2/3) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 69). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 69) in 8 ms on localhost (3/3) -2016-06-13 20:01:05 INFO DAGScheduler:59 - ShuffleMapStage 30 (apply at Transformer.scala:22) finished in 0.021 s -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 20:01:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:01:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:01:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 31) -2016-06-13 20:01:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:01:05 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List() -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[83] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=197659, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=200899, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.4 MB) -2016-06-13 20:01:05 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:50551 (size: 1802.0 B, free: 730.5 MB) -2016-06-13 20:01:05 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[83] at apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 70, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 71, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 70) -2016-06-13 20:01:05 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 71) -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:01:05 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 71). 1172 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 71) in 4 ms on localhost (1/2) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 70). 1180 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 70) in 8 ms on localhost (2/2) -2016-06-13 20:01:05 INFO DAGScheduler:59 - ResultStage 31 (apply at Transformer.scala:22) finished in 0.008 s -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 20:01:05 INFO DAGScheduler:59 - Job 15 finished: apply at Transformer.scala:22, took 0.052679 s -2016-06-13 20:01:05 INFO JobScheduler:59 - Finished job streaming job 8000 ms.1 from job set of time 8000 ms -2016-06-13 20:01:05 INFO JobScheduler:59 - Total delay: 1465819257.314 s for time 8000 ms (execution: 0.123 s) -2016-06-13 20:01:05 INFO ParallelCollectionRDD:59 - Removing RDD 38 from persistence list -2016-06-13 20:01:05 INFO JobScheduler:59 - Starting job streaming job 9000 ms.0 from job set of time 9000 ms -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 38 -2016-06-13 20:01:05 INFO MapPartitionsRDD:59 - Removing RDD 43 from persistence list -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 43 -2016-06-13 20:01:05 INFO ShuffledRDD:59 - Removing RDD 42 from persistence list -2016-06-13 20:01:05 INFO UnionRDD:59 - Removing RDD 41 from persistence list -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 42 -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:01:05 INFO MapPartitionsRDD:59 - Removing RDD 39 from persistence list -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 41 -2016-06-13 20:01:05 INFO JobGenerator:59 - Checkpointing graph for time 8000 ms -2016-06-13 20:01:05 INFO DStreamGraph:59 - Updating checkpoint data for time 8000 ms -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 39 -2016-06-13 20:01:05 INFO DStreamGraph:59 - Updated checkpoint data for time 8000 ms -2016-06-13 20:01:05 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Registering RDD 84 (apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Got job 16 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:01:05 INFO DAGScheduler:59 - Final stage: ResultStage 33(apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 32) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 32 (MapPartitionsRDD[84] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(4680) called with curMem=202701, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 4.6 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(2457) called with curMem=207381, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO CheckpointWriter:59 - Checkpoint for time 7000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-7000', took 5640 bytes and 142 ms -2016-06-13 20:01:05 INFO CheckpointWriter:59 - Saving checkpoint for time 8000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-8000' -2016-06-13 20:01:05 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:50551 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 20:01:05 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 32 (MapPartitionsRDD[84] at apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 72, localhost, PROCESS_LOCAL, 2223 bytes) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 73, localhost, PROCESS_LOCAL, 2247 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 72) -2016-06-13 20:01:05 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 73) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 72). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 73). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 72) in 14 ms on localhost (1/2) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 73) in 14 ms on localhost (2/2) -2016-06-13 20:01:05 INFO DAGScheduler:59 - ShuffleMapStage 32 (apply at Transformer.scala:22) finished in 0.015 s -2016-06-13 20:01:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:01:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:01:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 33) -2016-06-13 20:01:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 20:01:05 INFO DAGScheduler:59 - Missing parents for ResultStage 33: List() -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting ResultStage 33 (ShuffledRDD[85] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(5200) called with curMem=209838, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(2685) called with curMem=215038, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:50551 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 20:01:05 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (ShuffledRDD[85] at apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 74, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 75, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 74) -2016-06-13 20:01:05 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 75) -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 2 blocks -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:05 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 75). 1466 bytes result sent to driver -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:01:05 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 74). 1466 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 75) in 8 ms on localhost (1/2) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 74) in 10 ms on localhost (2/2) -2016-06-13 20:01:05 INFO DAGScheduler:59 - ResultStage 33 (apply at Transformer.scala:22) finished in 0.010 s -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 20:01:05 INFO DAGScheduler:59 - Job 16 finished: apply at Transformer.scala:22, took 0.046875 s -2016-06-13 20:01:05 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 3.0039091119551857 and centroid: [1.0891460065841554] -2016-06-13 20:01:05 INFO StreamingKMeansModel:59 - Cluster 1 updated with weight 2.000978473577561 and centroid: [-0.28428642183532626] -2016-06-13 20:01:05 INFO JobScheduler:59 - Finished job streaming job 9000 ms.0 from job set of time 9000 ms -2016-06-13 20:01:05 INFO JobScheduler:59 - Starting job streaming job 9000 ms.1 from job set of time 9000 ms -2016-06-13 20:01:05 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Registering RDD 53 (apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Got job 17 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:01:05 INFO DAGScheduler:59 - Final stage: ResultStage 35(apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 34) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 34) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 34 (UnionRDD[53] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=217723, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=221003, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.4 MB) -2016-06-13 20:01:05 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:50551 (size: 2007.0 B, free: 730.5 MB) -2016-06-13 20:01:05 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 34 (UnionRDD[53] at apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 3 tasks -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 76, localhost, PROCESS_LOCAL, 2332 bytes) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 77, localhost, PROCESS_LOCAL, 2356 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 76) -2016-06-13 20:01:05 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 77) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 76). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 78, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 78) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 76) in 7 ms on localhost (1/3) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 78). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 78) in 7 ms on localhost (2/3) -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:01:05 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 77). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 77) in 53 ms on localhost (3/3) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-06-13 20:01:05 INFO DAGScheduler:59 - ShuffleMapStage 34 (apply at Transformer.scala:22) finished in 0.056 s -2016-06-13 20:01:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:01:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:01:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 35) -2016-06-13 20:01:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:01:05 INFO DAGScheduler:59 - Missing parents for ResultStage 35: List() -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[86] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=223010, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=226250, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.4 MB) -2016-06-13 20:01:05 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:50551 (size: 1803.0 B, free: 730.5 MB) -2016-06-13 20:01:05 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[86] at apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 79, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 80, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 80) -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:05 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 80). 1172 bytes result sent to driver -2016-06-13 20:01:05 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 79) -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 80) in 8 ms on localhost (1/2) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 79). 1180 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 79) in 11 ms on localhost (2/2) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-06-13 20:01:05 INFO DAGScheduler:59 - ResultStage 35 (apply at Transformer.scala:22) finished in 0.013 s -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Job 17 finished: apply at Transformer.scala:22, took 0.094076 s -2016-06-13 20:01:05 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Registering RDD 87 (apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Got job 18 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:01:05 INFO DAGScheduler:59 - Final stage: ResultStage 37(apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 36) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 36) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 36 (MapPartitionsRDD[87] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(4680) called with curMem=228053, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 4.6 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO JobScheduler:59 - Finished job streaming job 9000 ms.1 from job set of time 9000 ms -2016-06-13 20:01:05 INFO JobScheduler:59 - Total delay: 1465819256.489 s for time 9000 ms (execution: 0.174 s) -2016-06-13 20:01:05 INFO JobScheduler:59 - Starting job streaming job 10000 ms.0 from job set of time 10000 ms -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(2456) called with curMem=232733, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO ParallelCollectionRDD:59 - Removing RDD 44 from persistence list -2016-06-13 20:01:05 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:50551 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 20:01:05 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 36 (MapPartitionsRDD[87] at apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-06-13 20:01:05 INFO CheckpointWriter:59 - Checkpoint for time 8000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-8000', took 5642 bytes and 161 ms -2016-06-13 20:01:05 INFO CheckpointWriter:59 - Saving checkpoint for time 9000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-9000' -2016-06-13 20:01:05 INFO MapPartitionsRDD:59 - Removing RDD 49 from persistence list -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 44 -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 81, localhost, PROCESS_LOCAL, 2223 bytes) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 82, localhost, PROCESS_LOCAL, 2247 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 82) -2016-06-13 20:01:05 INFO ShuffledRDD:59 - Removing RDD 48 from persistence list -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 49 -2016-06-13 20:01:05 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 81) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 82). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 81). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO UnionRDD:59 - Removing RDD 47 from persistence list -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 48 -2016-06-13 20:01:05 INFO MapPartitionsRDD:59 - Removing RDD 45 from persistence list -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 47 -2016-06-13 20:01:05 INFO JobGenerator:59 - Checkpointing graph for time 9000 ms -2016-06-13 20:01:05 INFO DStreamGraph:59 - Updating checkpoint data for time 9000 ms -2016-06-13 20:01:05 INFO DStreamGraph:59 - Updated checkpoint data for time 9000 ms -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 45 -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 82) in 20 ms on localhost (1/2) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 81) in 20 ms on localhost (2/2) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-06-13 20:01:05 INFO DAGScheduler:59 - ShuffleMapStage 36 (apply at Transformer.scala:22) finished in 0.022 s -2016-06-13 20:01:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:01:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:01:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 37) -2016-06-13 20:01:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:01:05 INFO DAGScheduler:59 - Missing parents for ResultStage 37: List() -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting ResultStage 37 (ShuffledRDD[88] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(5200) called with curMem=235189, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(2685) called with curMem=240389, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:50551 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:01:05 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (ShuffledRDD[88] at apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 83, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 84, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 84) -2016-06-13 20:01:05 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 83) -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:01:05 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 83). 1466 bytes result sent to driver -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:01:05 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 84). 1466 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 83) in 13 ms on localhost (1/2) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 84) in 14 ms on localhost (2/2) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-06-13 20:01:05 INFO DAGScheduler:59 - ResultStage 37 (apply at Transformer.scala:22) finished in 0.017 s -2016-06-13 20:01:05 INFO DAGScheduler:59 - Job 18 finished: apply at Transformer.scala:22, took 0.082864 s -2016-06-13 20:01:05 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 2.0029335049921437 and centroid: [1.2061566386924258] -2016-06-13 20:01:05 INFO StreamingKMeansModel:59 - Cluster 1 updated with weight 3.001954080540603 and centroid: [-0.3802885629361001] -2016-06-13 20:01:05 INFO JobScheduler:59 - Finished job streaming job 10000 ms.0 from job set of time 10000 ms -2016-06-13 20:01:05 INFO JobScheduler:59 - Starting job streaming job 10000 ms.1 from job set of time 10000 ms -2016-06-13 20:01:05 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Registering RDD 60 (apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Got job 19 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:01:05 INFO DAGScheduler:59 - Final stage: ResultStage 39(apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 38) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 38) -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 38 (UnionRDD[60] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=243074, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=246354, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.3 MB) -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:01:05 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:50551 (size: 2007.0 B, free: 730.5 MB) -2016-06-13 20:01:05 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 38 (UnionRDD[60] at apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 3 tasks -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 85, localhost, PROCESS_LOCAL, 2332 bytes) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 86, localhost, PROCESS_LOCAL, 2356 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 85) -2016-06-13 20:01:05 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 86) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 85). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 87, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 85) in 8 ms on localhost (1/3) -2016-06-13 20:01:05 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 87) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 86). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 86) in 11 ms on localhost (2/3) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 87). 1159 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 87) in 9 ms on localhost (3/3) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-06-13 20:01:05 INFO DAGScheduler:59 - ShuffleMapStage 38 (apply at Transformer.scala:22) finished in 0.020 s -2016-06-13 20:01:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:01:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:01:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 39) -2016-06-13 20:01:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:01:05 INFO DAGScheduler:59 - Missing parents for ResultStage 39: List() -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[89] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=248361, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 3.2 KB, free 730.3 MB) -2016-06-13 20:01:05 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=251601, maxMem=766075207 -2016-06-13 20:01:05 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.3 MB) -2016-06-13 20:01:05 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:50551 (size: 1803.0 B, free: 730.5 MB) -2016-06-13 20:01:05 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:01:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[89] at apply at Transformer.scala:22) -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:01:05 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 89) -2016-06-13 20:01:05 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 88) -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:01:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:01:05 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 89). 1172 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 89) in 6 ms on localhost (1/2) -2016-06-13 20:01:05 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 88). 1180 bytes result sent to driver -2016-06-13 20:01:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 88) in 9 ms on localhost (2/2) -2016-06-13 20:01:05 INFO DAGScheduler:59 - ResultStage 39 (apply at Transformer.scala:22) finished in 0.009 s -2016-06-13 20:01:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-06-13 20:01:05 INFO DAGScheduler:59 - Job 19 finished: apply at Transformer.scala:22, took 0.054879 s -2016-06-13 20:01:05 INFO JobScheduler:59 - Finished job streaming job 10000 ms.1 from job set of time 10000 ms -2016-06-13 20:01:05 INFO JobScheduler:59 - Total delay: 1465819255.628 s for time 10000 ms (execution: 0.139 s) -2016-06-13 20:01:05 INFO ParallelCollectionRDD:59 - Removing RDD 50 from persistence list -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 50 -2016-06-13 20:01:05 INFO MapPartitionsRDD:59 - Removing RDD 56 from persistence list -2016-06-13 20:01:05 INFO ShuffledRDD:59 - Removing RDD 55 from persistence list -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 56 -2016-06-13 20:01:05 INFO UnionRDD:59 - Removing RDD 53 from persistence list -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 55 -2016-06-13 20:01:05 INFO MapPartitionsRDD:59 - Removing RDD 51 from persistence list -2016-06-13 20:01:05 INFO JobGenerator:59 - Checkpointing graph for time 10000 ms -2016-06-13 20:01:05 INFO DStreamGraph:59 - Updating checkpoint data for time 10000 ms -2016-06-13 20:01:05 INFO DStreamGraph:59 - Updated checkpoint data for time 10000 ms -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 51 -2016-06-13 20:01:05 INFO BlockManager:59 - Removing RDD 53 -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - Output generated in 1409 milliseconds -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - [WrappedArray(5),WrappedArray()] -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - [WrappedArray(5),WrappedArray()] -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - [WrappedArray(5),WrappedArray()] -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - [WrappedArray(5),WrappedArray()] -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - [WrappedArray(5),WrappedArray()] -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - [WrappedArray(5),WrappedArray()] -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - [WrappedArray(5),WrappedArray()] -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - [WrappedArray(5),WrappedArray()] -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - [WrappedArray(5),WrappedArray()] -2016-06-13 20:01:05 INFO StreamingKMeansSuite:59 - [WrappedArray(5),WrappedArray()] -2016-06-13 20:01:05 INFO CheckpointWriter:59 - Checkpoint for time 9000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-9000', took 5642 bytes and 182 ms -2016-06-13 20:01:05 INFO CheckpointWriter:59 - Saving checkpoint for time 10000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-10000' -2016-06-13 20:01:05 INFO JobGenerator:59 - Stopping JobGenerator immediately -2016-06-13 20:01:05 INFO RecurringTimer:59 - Stopped timer for JobGenerator after time 10000 -2016-06-13 20:01:15 INFO CheckpointWriter:59 - CheckpointWriter executor terminated ? false, waited for 10000 ms. -2016-06-13 20:01:15 INFO JobGenerator:59 - Stopped JobGenerator -2016-06-13 20:01:15 INFO JobScheduler:59 - Stopped JobScheduler -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/batch,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/streaming,null} -2016-06-13 20:01:15 INFO StreamingContext:59 - StreamingContext stopped successfully -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/batch/json,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/json,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:01:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:01:15 INFO CheckpointWriter:59 - Checkpoint for time 10000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-dd7130cd-b174-413a-864d-5d21e30132ff/checkpoint-10000', took 5634 bytes and 10133 ms -2016-06-13 20:01:15 INFO SparkUI:59 - Stopped Spark web UI at http://localhost:4040 -2016-06-13 20:01:15 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:01:15 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:01:16 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:01:16 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:01:16 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:01:16 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:01:16 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:01:16 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:01:16 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:01:16 INFO StreamingKMeansSuite:59 - - -===== FINISHED o.a.s.mllib.clustering.kmeans.StreamingKMeansSuite: 'detecting dying clusters' ===== - -2016-06-13 20:01:16 WARN StreamingContext:71 - StreamingContext has already been stopped -2016-06-13 20:01:16 INFO SparkContext:59 - SparkContext already stopped. -2016-06-13 20:01:16 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 20:01:16 INFO StreamingKMeansSuite:59 - Using manual clock -2016-06-13 20:01:16 INFO StreamingKMeansSuite:59 - - -===== TEST OUTPUT FOR o.a.s.mllib.clustering.kmeans.StreamingKMeansSuite: 'SPARK-7946 setDecayFactor' ===== - -2016-06-13 20:01:16 INFO StreamingKMeansSuite:59 - - -===== FINISHED o.a.s.mllib.clustering.kmeans.StreamingKMeansSuite: 'SPARK-7946 setDecayFactor' ===== - -2016-06-13 20:01:16 WARN StreamingContext:71 - StreamingContext has already been stopped -2016-06-13 20:01:16 INFO SparkContext:59 - SparkContext already stopped. -2016-06-13 20:01:16 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 20:01:16 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-dd7130cd-b174-413a-864d-5d21e30132ff -2016-06-13 20:01:16 ERROR ShutdownHookManager:96 - Exception while deleting Spark temp dir: C:\Users\xubo\AppData\Local\Temp\spark-dd7130cd-b174-413a-864d-5d21e30132ff -java.io.IOException: Failed to delete: C:\Users\xubo\AppData\Local\Temp\spark-dd7130cd-b174-413a-864d-5d21e30132ff - at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:884) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:63) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:60) - at scala.collection.mutable.HashSet.foreach(HashSet.scala:79) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1.apply$mcV$sp(ShutdownHookManager.scala:60) - at org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:264) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1699) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply$mcV$sp(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:234) - at scala.util.Try$.apply(Try.scala:161) - at org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:216) - at org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54) -2016-06-13 20:01:16 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-f8eb2236-34b2-4105-aaff-6c04ac3f507c -2016-06-13 20:06:42 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:06:43 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 20:06:43 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:06:43 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:06:43 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:06:44 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:06:44 INFO Remoting:74 - Starting remoting -2016-06-13 20:06:44 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:50573] -2016-06-13 20:06:44 INFO Utils:59 - Successfully started service 'sparkDriver' on port 50573. -2016-06-13 20:06:44 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:06:44 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:06:44 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-0a9dbb0c-7e9a-4231-9c58-c228a6ff3b1e -2016-06-13 20:06:44 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:06:44 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-29807d52-a4c6-47bb-b6f0-6c6868209c67\httpd-3f3647fb-ca7b-452e-a4c4-6df363170518 -2016-06-13 20:06:44 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:06:44 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:06:44 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:50574 -2016-06-13 20:06:44 INFO Utils:59 - Successfully started service 'HTTP file server' on port 50574. -2016-06-13 20:06:44 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:06:44 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:06:44 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 20:06:44 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 20:06:44 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 20:06:44 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:06:44 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:06:45 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 50593. -2016-06-13 20:06:45 INFO NettyBlockTransferService:59 - Server created on 50593 -2016-06-13 20:06:45 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:06:45 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:50593 with 730.6 MB RAM, BlockManagerId(driver, localhost, 50593) -2016-06-13 20:06:45 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:06:45 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:06:45 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:06:45 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:06:45 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:06:45 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:06:45 INFO Remoting:74 - Starting remoting -2016-06-13 20:06:45 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@localhost:50606] -2016-06-13 20:06:45 INFO Utils:59 - Successfully started service 'sparkDriver' on port 50606. -2016-06-13 20:06:45 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:06:45 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:06:45 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-f2fbb1ce-5eb5-46f1-a024-d49aa78cac1f -2016-06-13 20:06:45 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:06:45 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-29807d52-a4c6-47bb-b6f0-6c6868209c67\httpd-4fa4f08f-0273-4cff-bde6-f7b5fd2c26d3 -2016-06-13 20:06:45 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:06:45 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:06:45 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:50607 -2016-06-13 20:06:45 INFO Utils:59 - Successfully started service 'HTTP file server' on port 50607. -2016-06-13 20:06:45 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:06:45 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:06:45 WARN AbstractLifeCycle:204 - FAILED SelectChannelConnector@0.0.0.0:4040: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.streaming.StreamingContext$.createNewSparkContext(StreamingContext.scala:878) - at org.apache.spark.streaming.StreamingContext.(StreamingContext.scala:81) - at org.apache.spark.mllib.clustering.kmeans.StreamingKmeansFromWebSuite$$anonfun$1.apply$mcV$sp(StreamingKmeansFromWebSuite.scala:24) - at org.apache.spark.mllib.clustering.kmeans.StreamingKmeansFromWebSuite$$anonfun$1.apply(StreamingKmeansFromWebSuite.scala:20) - at org.apache.spark.mllib.clustering.kmeans.StreamingKmeansFromWebSuite$$anonfun$1.apply(StreamingKmeansFromWebSuite.scala:20) - at org.scalatest.Transformer$$anonfun$apply$1.apply$mcV$sp(Transformer.scala:22) - at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85) - at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104) - at org.scalatest.Transformer.apply(Transformer.scala:22) - at org.scalatest.Transformer.apply(Transformer.scala:20) - at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:166) - at org.scalatest.Suite$class.withFixture(Suite.scala:1122) - at org.scalatest.FunSuite.withFixture(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$class.invokeWithFixture$1(FunSuiteLike.scala:163) - at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175) - at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175) - at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306) - at org.scalatest.FunSuiteLike$class.runTest(FunSuiteLike.scala:175) - at org.scalatest.FunSuite.runTest(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208) - at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208) - at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:413) - at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401) - at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:396) - at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:483) - at org.scalatest.FunSuiteLike$class.runTests(FunSuiteLike.scala:208) - at org.scalatest.FunSuite.runTests(FunSuite.scala:1555) - at org.scalatest.Suite$class.run(Suite.scala:1424) - at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212) - at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212) - at org.scalatest.SuperEngine.runImpl(Engine.scala:545) - at org.scalatest.FunSuiteLike$class.run(FunSuiteLike.scala:212) - at org.apache.spark.util.SparkLearningFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkLearningFunSuite.scala:10) - at org.scalatest.BeforeAndAfterAll$class.liftedTree1$1(BeforeAndAfterAll.scala:257) - at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:256) - at org.apache.spark.util.SparkLearningFunSuite.run(SparkLearningFunSuite.scala:10) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 20:06:45 WARN AbstractLifeCycle:204 - FAILED org.spark-project.jetty.server.Server@7a36f4a6: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.streaming.StreamingContext$.createNewSparkContext(StreamingContext.scala:878) - at org.apache.spark.streaming.StreamingContext.(StreamingContext.scala:81) - at org.apache.spark.mllib.clustering.kmeans.StreamingKmeansFromWebSuite$$anonfun$1.apply$mcV$sp(StreamingKmeansFromWebSuite.scala:24) - at org.apache.spark.mllib.clustering.kmeans.StreamingKmeansFromWebSuite$$anonfun$1.apply(StreamingKmeansFromWebSuite.scala:20) - at org.apache.spark.mllib.clustering.kmeans.StreamingKmeansFromWebSuite$$anonfun$1.apply(StreamingKmeansFromWebSuite.scala:20) - at org.scalatest.Transformer$$anonfun$apply$1.apply$mcV$sp(Transformer.scala:22) - at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85) - at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104) - at org.scalatest.Transformer.apply(Transformer.scala:22) - at org.scalatest.Transformer.apply(Transformer.scala:20) - at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:166) - at org.scalatest.Suite$class.withFixture(Suite.scala:1122) - at org.scalatest.FunSuite.withFixture(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$class.invokeWithFixture$1(FunSuiteLike.scala:163) - at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175) - at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175) - at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306) - at org.scalatest.FunSuiteLike$class.runTest(FunSuiteLike.scala:175) - at org.scalatest.FunSuite.runTest(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208) - at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208) - at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:413) - at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401) - at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:396) - at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:483) - at org.scalatest.FunSuiteLike$class.runTests(FunSuiteLike.scala:208) - at org.scalatest.FunSuite.runTests(FunSuite.scala:1555) - at org.scalatest.Suite$class.run(Suite.scala:1424) - at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212) - at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212) - at org.scalatest.SuperEngine.runImpl(Engine.scala:545) - at org.scalatest.FunSuiteLike$class.run(FunSuiteLike.scala:212) - at org.apache.spark.util.SparkLearningFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkLearningFunSuite.scala:10) - at org.scalatest.BeforeAndAfterAll$class.liftedTree1$1(BeforeAndAfterAll.scala:257) - at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:256) - at org.apache.spark.util.SparkLearningFunSuite.run(SparkLearningFunSuite.scala:10) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:06:45 WARN Utils:71 - Service 'SparkUI' could not bind on port 4040. Attempting port 4041. -2016-06-13 20:06:45 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:06:45 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4041 -2016-06-13 20:06:45 INFO Utils:59 - Successfully started service 'SparkUI' on port 4041. -2016-06-13 20:06:45 INFO SparkUI:59 - Started SparkUI at http://localhost:4041 -2016-06-13 20:06:45 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:06:45 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:06:45 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 50626. -2016-06-13 20:06:45 INFO NettyBlockTransferService:59 - Server created on 50626 -2016-06-13 20:06:45 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:06:45 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:50626 with 730.6 MB RAM, BlockManagerId(driver, localhost, 50626) -2016-06-13 20:06:45 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:06:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:06:46 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 20:06:46 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:06:46 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:06:46 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:06:46 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:06:46 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:06:46 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:06:46 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:06:46 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:06:46 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:06:46 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:06:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:06:46 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 20:06:46 INFO SparkUI:59 - Stopped Spark web UI at http://localhost:4041 -2016-06-13 20:06:46 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:06:46 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:06:46 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:06:46 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:06:46 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:06:46 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:06:46 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:06:46 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:06:46 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:06:46 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 20:06:46 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-29807d52-a4c6-47bb-b6f0-6c6868209c67 -2016-06-13 20:07:14 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:07:15 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 20:07:15 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:07:15 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:07:15 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:07:16 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:07:16 INFO Remoting:74 - Starting remoting -2016-06-13 20:07:16 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:50644] -2016-06-13 20:07:16 INFO Utils:59 - Successfully started service 'sparkDriver' on port 50644. -2016-06-13 20:07:16 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:07:16 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:07:16 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-004c6cf5-5fe2-4462-8a9e-964d08e006a3 -2016-06-13 20:07:16 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:07:16 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-25555b7b-c451-4a88-91a4-bdcc4340d77e\httpd-fe06e4ae-9af5-4c64-a602-c8eb30989d44 -2016-06-13 20:07:16 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:07:16 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:07:16 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:50645 -2016-06-13 20:07:16 INFO Utils:59 - Successfully started service 'HTTP file server' on port 50645. -2016-06-13 20:07:16 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:07:16 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:07:16 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 20:07:16 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 20:07:16 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 20:07:16 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:07:16 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:07:17 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 50664. -2016-06-13 20:07:17 INFO NettyBlockTransferService:59 - Server created on 50664 -2016-06-13 20:07:17 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:07:17 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:50664 with 730.6 MB RAM, BlockManagerId(driver, localhost, 50664) -2016-06-13 20:07:17 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:07:17 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:07:17 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:07:17 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:07:17 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:07:17 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:07:17 INFO Remoting:74 - Starting remoting -2016-06-13 20:07:17 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@localhost:50677] -2016-06-13 20:07:17 INFO Utils:59 - Successfully started service 'sparkDriver' on port 50677. -2016-06-13 20:07:17 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:07:17 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:07:17 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-170176a7-c68e-4f1e-b2f7-d9681f3441c9 -2016-06-13 20:07:17 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:07:17 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-25555b7b-c451-4a88-91a4-bdcc4340d77e\httpd-929d9a0f-8c64-4154-a62a-a6cfa472118d -2016-06-13 20:07:17 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:07:17 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:07:17 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:50678 -2016-06-13 20:07:17 INFO Utils:59 - Successfully started service 'HTTP file server' on port 50678. -2016-06-13 20:07:17 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:07:17 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:07:17 WARN AbstractLifeCycle:204 - FAILED SelectChannelConnector@0.0.0.0:4040: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.streaming.StreamingContext$.createNewSparkContext(StreamingContext.scala:878) - at org.apache.spark.streaming.StreamingContext.(StreamingContext.scala:81) - at org.apache.spark.mllib.clustering.kmeans.StreamingKmeansFromWebSuite$$anonfun$1.apply$mcV$sp(StreamingKmeansFromWebSuite.scala:24) - at org.apache.spark.mllib.clustering.kmeans.StreamingKmeansFromWebSuite$$anonfun$1.apply(StreamingKmeansFromWebSuite.scala:20) - at org.apache.spark.mllib.clustering.kmeans.StreamingKmeansFromWebSuite$$anonfun$1.apply(StreamingKmeansFromWebSuite.scala:20) - at org.scalatest.Transformer$$anonfun$apply$1.apply$mcV$sp(Transformer.scala:22) - at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85) - at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104) - at org.scalatest.Transformer.apply(Transformer.scala:22) - at org.scalatest.Transformer.apply(Transformer.scala:20) - at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:166) - at org.scalatest.Suite$class.withFixture(Suite.scala:1122) - at org.scalatest.FunSuite.withFixture(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$class.invokeWithFixture$1(FunSuiteLike.scala:163) - at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175) - at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175) - at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306) - at org.scalatest.FunSuiteLike$class.runTest(FunSuiteLike.scala:175) - at org.scalatest.FunSuite.runTest(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208) - at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208) - at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:413) - at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401) - at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:396) - at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:483) - at org.scalatest.FunSuiteLike$class.runTests(FunSuiteLike.scala:208) - at org.scalatest.FunSuite.runTests(FunSuite.scala:1555) - at org.scalatest.Suite$class.run(Suite.scala:1424) - at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212) - at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212) - at org.scalatest.SuperEngine.runImpl(Engine.scala:545) - at org.scalatest.FunSuiteLike$class.run(FunSuiteLike.scala:212) - at org.apache.spark.util.SparkLearningFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkLearningFunSuite.scala:10) - at org.scalatest.BeforeAndAfterAll$class.liftedTree1$1(BeforeAndAfterAll.scala:257) - at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:256) - at org.apache.spark.util.SparkLearningFunSuite.run(SparkLearningFunSuite.scala:10) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 20:07:17 WARN AbstractLifeCycle:204 - FAILED org.spark-project.jetty.server.Server@661d3f57: java.net.BindException: Address already in use: bind -java.net.BindException: Address already in use: bind - at sun.nio.ch.Net.bind0(Native Method) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.Net.bind(Unknown Source) - at sun.nio.ch.ServerSocketChannelImpl.bind(Unknown Source) - at sun.nio.ch.ServerSocketAdaptor.bind(Unknown Source) - at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187) - at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316) - at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.spark-project.jetty.server.Server.doStart(Server.java:293) - at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64) - at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:236) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.ui.JettyUtils$$anonfun$3.apply(JettyUtils.scala:246) - at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) - at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) - at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) - at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:246) - at org.apache.spark.ui.WebUI.bind(WebUI.scala:136) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:474) - at scala.Option.foreach(Option.scala:236) - at org.apache.spark.SparkContext.(SparkContext.scala:474) - at org.apache.spark.streaming.StreamingContext$.createNewSparkContext(StreamingContext.scala:878) - at org.apache.spark.streaming.StreamingContext.(StreamingContext.scala:81) - at org.apache.spark.mllib.clustering.kmeans.StreamingKmeansFromWebSuite$$anonfun$1.apply$mcV$sp(StreamingKmeansFromWebSuite.scala:24) - at org.apache.spark.mllib.clustering.kmeans.StreamingKmeansFromWebSuite$$anonfun$1.apply(StreamingKmeansFromWebSuite.scala:20) - at org.apache.spark.mllib.clustering.kmeans.StreamingKmeansFromWebSuite$$anonfun$1.apply(StreamingKmeansFromWebSuite.scala:20) - at org.scalatest.Transformer$$anonfun$apply$1.apply$mcV$sp(Transformer.scala:22) - at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85) - at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104) - at org.scalatest.Transformer.apply(Transformer.scala:22) - at org.scalatest.Transformer.apply(Transformer.scala:20) - at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:166) - at org.scalatest.Suite$class.withFixture(Suite.scala:1122) - at org.scalatest.FunSuite.withFixture(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$class.invokeWithFixture$1(FunSuiteLike.scala:163) - at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175) - at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:175) - at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306) - at org.scalatest.FunSuiteLike$class.runTest(FunSuiteLike.scala:175) - at org.scalatest.FunSuite.runTest(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208) - at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:208) - at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:413) - at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401) - at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:396) - at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:483) - at org.scalatest.FunSuiteLike$class.runTests(FunSuiteLike.scala:208) - at org.scalatest.FunSuite.runTests(FunSuite.scala:1555) - at org.scalatest.Suite$class.run(Suite.scala:1424) - at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1555) - at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212) - at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:212) - at org.scalatest.SuperEngine.runImpl(Engine.scala:545) - at org.scalatest.FunSuiteLike$class.run(FunSuiteLike.scala:212) - at org.apache.spark.util.SparkLearningFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkLearningFunSuite.scala:10) - at org.scalatest.BeforeAndAfterAll$class.liftedTree1$1(BeforeAndAfterAll.scala:257) - at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:256) - at org.apache.spark.util.SparkLearningFunSuite.run(SparkLearningFunSuite.scala:10) - at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) - at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) - at scala.collection.immutable.List.foreach(List.scala:318) - at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) - at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) - at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) - at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) - at org.scalatest.tools.Runner$.run(Runner.scala:883) - at org.scalatest.tools.Runner.run(Runner.scala) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) - at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) - at java.lang.reflect.Method.invoke(Unknown Source) - at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:07:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:07:17 WARN Utils:71 - Service 'SparkUI' could not bind on port 4040. Attempting port 4041. -2016-06-13 20:07:17 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:07:17 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4041 -2016-06-13 20:07:17 INFO Utils:59 - Successfully started service 'SparkUI' on port 4041. -2016-06-13 20:07:17 INFO SparkUI:59 - Started SparkUI at http://localhost:4041 -2016-06-13 20:07:17 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:07:17 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:07:18 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 50697. -2016-06-13 20:07:18 INFO NettyBlockTransferService:59 - Server created on 50697 -2016-06-13 20:07:18 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:07:18 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:50697 with 730.6 MB RAM, BlockManagerId(driver, localhost, 50697) -2016-06-13 20:07:18 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:07:18 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 20:07:18 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:07:18 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:07:18 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:07:18 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:07:18 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:07:18 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:07:18 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:07:18 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:07:18 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:07:18 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:07:18 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 20:07:18 INFO SparkUI:59 - Stopped Spark web UI at http://localhost:4041 -2016-06-13 20:07:18 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:07:18 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:07:18 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:07:18 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:07:18 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:07:18 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:07:18 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 20:07:18 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:07:18 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-25555b7b-c451-4a88-91a4-bdcc4340d77e -2016-06-13 20:07:18 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:07:18 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:07:18 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 20:15:05 INFO StreamingKMeansSuite:59 - Using manual clock -2016-06-13 20:15:05 INFO StreamingKMeansSuite:59 - - -===== TEST OUTPUT FOR o.a.s.mllib.clustering.kmeans.StreamingKMeansSuite: 'accuracy for single center and equivalence to grand average' ===== - -2016-06-13 20:15:05 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:15:06 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 20:15:06 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:15:06 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:15:06 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:15:07 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:15:07 INFO Remoting:74 - Starting remoting -2016-06-13 20:15:07 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:50953] -2016-06-13 20:15:07 INFO Utils:59 - Successfully started service 'sparkDriver' on port 50953. -2016-06-13 20:15:07 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:15:07 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:15:07 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-ae1c11f9-c7df-457c-881c-dec51d3603c2 -2016-06-13 20:15:07 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:15:07 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-32ae3a90-7866-4ba0-adf5-d5f54d6e6673\httpd-4c6db032-478f-4f5b-b31d-a152fc4916b4 -2016-06-13 20:15:07 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:15:07 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:15:07 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:50954 -2016-06-13 20:15:07 INFO Utils:59 - Successfully started service 'HTTP file server' on port 50954. -2016-06-13 20:15:07 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:15:07 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:15:07 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 20:15:07 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 20:15:07 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 20:15:07 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:15:07 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:15:08 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 50965. -2016-06-13 20:15:08 INFO NettyBlockTransferService:59 - Server created on 50965 -2016-06-13 20:15:08 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:15:08 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:50965 with 730.6 MB RAM, BlockManagerId(driver, localhost, 50965) -2016-06-13 20:15:08 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:15:10 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 20:15:10 INFO StreamingKMeansSuite:59 - numBatches = 10, numExpectedOutput = 10 -2016-06-13 20:15:10 INFO ForEachDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:15:10 INFO TestInputStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:15:10 INFO TestInputStream:59 - Slide time = 1000 ms -2016-06-13 20:15:10 INFO TestInputStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:15:10 INFO TestInputStream:59 - Checkpoint interval = null -2016-06-13 20:15:10 INFO TestInputStream:59 - Remember duration = 1000 ms -2016-06-13 20:15:10 INFO TestInputStream:59 - Initialized and validated org.apache.spark.streaming.TestInputStream@48daef85 -2016-06-13 20:15:10 INFO ForEachDStream:59 - Slide time = 1000 ms -2016-06-13 20:15:10 INFO ForEachDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:15:10 INFO ForEachDStream:59 - Checkpoint interval = null -2016-06-13 20:15:10 INFO ForEachDStream:59 - Remember duration = 1000 ms -2016-06-13 20:15:10 INFO ForEachDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.ForEachDStream@d353cde -2016-06-13 20:15:10 INFO TestOutputStreamWithPartitions:59 - metadataCleanupDelay = -1 -2016-06-13 20:15:10 INFO MappedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:15:10 INFO ShuffledDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:15:10 INFO TransformedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:15:10 INFO MappedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:15:10 INFO TestInputStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:15:10 INFO TestInputStream:59 - Slide time = 1000 ms -2016-06-13 20:15:10 INFO TestInputStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:15:10 INFO TestInputStream:59 - Checkpoint interval = null -2016-06-13 20:15:10 INFO TestInputStream:59 - Remember duration = 1000 ms -2016-06-13 20:15:10 INFO TestInputStream:59 - Initialized and validated org.apache.spark.streaming.TestInputStream@48daef85 -2016-06-13 20:15:10 INFO MappedDStream:59 - Slide time = 1000 ms -2016-06-13 20:15:10 INFO MappedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:15:10 INFO MappedDStream:59 - Checkpoint interval = null -2016-06-13 20:15:10 INFO MappedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:15:10 INFO MappedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@579800b3 -2016-06-13 20:15:10 INFO TransformedDStream:59 - Slide time = 1000 ms -2016-06-13 20:15:10 INFO TransformedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:15:10 INFO TransformedDStream:59 - Checkpoint interval = null -2016-06-13 20:15:10 INFO TransformedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:15:10 INFO TransformedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.TransformedDStream@947d127 -2016-06-13 20:15:10 INFO ShuffledDStream:59 - Slide time = 1000 ms -2016-06-13 20:15:10 INFO ShuffledDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:15:10 INFO ShuffledDStream:59 - Checkpoint interval = null -2016-06-13 20:15:10 INFO ShuffledDStream:59 - Remember duration = 1000 ms -2016-06-13 20:15:10 INFO ShuffledDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.ShuffledDStream@d8778c6 -2016-06-13 20:15:10 INFO MappedDStream:59 - Slide time = 1000 ms -2016-06-13 20:15:10 INFO MappedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:15:10 INFO MappedDStream:59 - Checkpoint interval = null -2016-06-13 20:15:10 INFO MappedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:15:10 INFO MappedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@ff6d7b0 -2016-06-13 20:15:10 INFO TestOutputStreamWithPartitions:59 - Slide time = 1000 ms -2016-06-13 20:15:10 INFO TestOutputStreamWithPartitions:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:15:10 INFO TestOutputStreamWithPartitions:59 - Checkpoint interval = null -2016-06-13 20:15:10 INFO TestOutputStreamWithPartitions:59 - Remember duration = 1000 ms -2016-06-13 20:15:10 INFO TestOutputStreamWithPartitions:59 - Initialized and validated org.apache.spark.streaming.TestOutputStreamWithPartitions@5a801c3a -2016-06-13 20:15:10 INFO RecurringTimer:59 - Started timer for JobGenerator at time 1000 -2016-06-13 20:15:10 INFO JobGenerator:59 - Started JobGenerator at 1000 ms -2016-06-13 20:15:10 INFO JobScheduler:59 - Started JobScheduler -2016-06-13 20:15:10 INFO StreamingContext:59 - StreamingContext started -2016-06-13 20:15:10 INFO StreamingKMeansSuite:59 - Manual clock before advancing = 0 -2016-06-13 20:15:10 INFO StreamingKMeansSuite:59 - Manual clock after advancing = 10000 -2016-06-13 20:15:10 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:10 INFO TestInputStream:59 - Computing RDD for time 1000 ms -2016-06-13 20:15:10 INFO TestInputStream:59 - Created RDD 0 with Vector([-0.35685422167583125,0.9995288307555166,-1.5877809386448758,1.210845548490831,0.56222524157234], [-0.45216537133396134,0.8040574757345051,-1.5589888701250443,1.115603017032467,0.344649973028188], [-0.3801675945145341,1.0697589561731022,-1.6486682116186815,1.2790268654063703,0.5116371486214886], [-0.545300880562902,1.1093057253978866,-1.7582382472264182,1.211400006292075,0.2932044815946008], [-0.3995483137357882,0.929119621864747,-1.7401380541085598,1.2990457426846032,0.299090686584923], [-0.5964001160322344,1.0378164371074625,-1.553411818459791,1.2193968564632125,0.4492703031123452], [-0.33949398265706626,1.1369769008642154,-1.7816495259375202,1.2857769527411067,0.5642853362323552], [-0.4522936770981281,1.090819158309221,-1.4785463285656288,1.2734882859357213,0.31864385887835545], [-0.4895260213585818,1.024567290792244,-1.7744155605537828,1.4515329700677304,0.39720325477885093], [-0.5479321442182269,0.8926951810306638,-1.617388979581139,1.3192738326981768,0.3243774353000105], [-0.5468296230191492,0.9034725811394896,-1.8269524875493084,1.161899613497355,0.22607191229030854], [-0.4344367225435325,0.9054178856043291,-1.6658399540877864,1.13578899284944,0.46511292492021983], [-0.5914600674966851,0.9727747336030947,-1.7411162798640765,1.3902196062020102,0.43158411906136107], [-0.5644425684165094,0.996354296088912,-1.817848647277835,1.3768030733201289,0.28609432985591315], [-0.44148386701129744,1.025737906243747,-1.7073695034902865,1.2539087121413848,0.349242056080172], [-0.5287113084613145,0.8475916475163896,-1.8166057479945195,1.238631411179877,0.36922519791770014], [-0.2540595910515851,0.9698131032346381,-1.5866552625154318,1.2179915976157372,0.3240398576394729], [-0.5026644816450073,1.109581665015373,-1.7151161774591086,1.3901513268104455,0.39042855132933163], [-0.4131172722481001,1.0096927045418442,-1.811286466550916,1.2719080737538961,0.28293757989913904], [-0.4515790075180954,1.0935474561719332,-1.7896107864561164,1.3203383417885988,0.2994362800834273], [-0.4171607573128071,1.0818260184642114,-1.8219543038308885,1.2496462030096327,0.4964658371791977], [-0.45586902589910655,1.0378742745036647,-1.7000458648640797,1.446592332373813,0.348566870310007], [-0.3532087778209343,0.8044437973592731,-1.7162629066191757,1.3339281879504057,0.17946188844185906], [-0.5154063008996834,0.9595656827532482,-1.6519653379002484,1.322654750805598,0.3008929692388781], [-0.3967887546418169,0.7908879805460386,-1.7242582851383725,1.2141584886700174,0.36827019559433166], [-0.48352710482779404,1.0255449792765485,-1.7171397454598174,1.284877805985566,0.437232715780399], [-0.40324856443707463,1.020485534999759,-1.6517518343613002,1.2864020019792224,0.3984832850799499], [-0.6517069948565133,0.969461018860354,-1.684015510299471,1.2027906385718596,0.2014860155598368], [-0.6148278408211463,0.9795135452912322,-1.8642733722424907,1.2720240024566645,0.39785426587083933], [-0.6409674819545198,1.1778584543507387,-1.8094969072333307,1.24145841048891,0.1624672183638662], [-0.6368691555002903,0.8170131462860201,-1.6827833832635621,1.3031179618920097,0.31628673987022604], [-0.43096648399184834,0.9655746876954024,-1.7845188100637381,1.4052721272088269,0.5845842279495], [-0.4442125841627056,1.0257872008493307,-1.7242589434969455,1.2745428690859977,0.39998857867859966], [-0.4527127515828855,0.9000469054331385,-1.7497092961121075,1.2451107418571954,0.4344380036579415], [-0.6030914309562295,0.9699928952430515,-1.712604726536663,1.38261404269885,0.48344898725768926], [-0.34976726689041593,0.9355870894961762,-1.5766155567170825,1.3590936489794139,0.35690680463154484], [-0.575628267406687,0.9538063357613722,-1.6445003616008003,1.2104804829930789,0.3911315521177748], [-0.5582822637309535,0.9339694428739754,-1.6511037013459962,1.4401460976648193,0.3607766390547849], [-0.5468811641579405,0.997556943752324,-1.778985900216746,1.2440321529105447,0.4204585578447196], [-0.48994207115736743,1.0755700976056979,-1.5322042869816734,1.2132674597398667,0.5961642366275318], [-0.5345824744371771,0.7659168558371742,-1.7009697565551642,1.2200491259133113,0.24064266183571642], [-0.4425411108434083,0.9636579302553172,-1.6769715787345627,1.2060504970822528,0.1832578105698996], [-0.2803908635114948,0.9814260387904405,-1.6744903280757537,1.438894231115956,0.5307857877828275], [-0.4646639723438268,0.9422153316765177,-1.8936280948227913,1.1755014401606392,0.3269897050872188], [-0.5794967530942349,1.008452319256215,-1.8095212163297067,1.3053459835247618,0.3862084672749757], [-0.36564413314560745,0.9550404460199052,-1.671332555981891,1.2387784773127055,0.29524181369582553], [-0.4947387784852314,0.8968969114403345,-1.5032027608518188,1.263293789437244,0.41872133210307433], [-0.5901163249097482,0.9195890740589755,-1.76975579729397,1.2233696400583831,0.3403182341587175], [-0.7175598778778954,0.8837946345395844,-1.8329343739105675,1.3562350855120264,0.30734388952604197], [-0.4376962364418643,1.0046436887819825,-1.5427982731479972,1.1077094728701697,0.30461693506090337]) -2016-06-13 20:15:10 INFO JobScheduler:59 - Added jobs for time 1000 ms -2016-06-13 20:15:10 INFO TestInputStream:59 - Computing RDD for time 2000 ms -2016-06-13 20:15:10 INFO JobScheduler:59 - Starting job streaming job 1000 ms.0 from job set of time 1000 ms -2016-06-13 20:15:11 INFO TestInputStream:59 - Created RDD 6 with Vector([-0.5385984930004446,0.9989931115946847,-1.745310682734983,1.2763556288677276,0.41555544930746574], [-0.5140757026211953,0.9616396809310985,-1.6092748223737894,1.3080875431688948,0.3958240822167327], [-0.3317121400990889,1.1311521924692884,-1.5940394948396088,1.235663715627094,0.5352186835894586], [-0.3566003302425871,0.8649446433306678,-1.6281158497086519,1.0841327390682376,0.33108928300382867], [-0.4898654567486354,1.0095506134582393,-1.7900184464939166,1.4376638486285889,0.5476994415325478], [-0.5067718408631526,0.8330942372560937,-1.5447521454199076,1.3956201717944832,0.40502500771937533], [-0.45899690569395396,1.0114199686087044,-1.6793704227956527,1.2264029484806054,0.3617800498770533], [-0.42389468031403765,0.9744019756374155,-1.4339024359998267,1.3506788012287876,0.5071391796420295], [-0.3445608603995446,0.8999444931201762,-1.8295489983610813,1.3813011802363127,0.4406110935124823], [-0.4720580057434039,0.9616388382715773,-1.7885815681274932,1.2231064430079939,0.49611588805959794], [-0.6877856894979741,0.9509033529307679,-1.7111111174594673,1.1650825853084499,0.36541463301831684], [-0.41906769746785616,0.9643844813018809,-1.7088669498454399,1.2743442947780539,0.37857583863641653], [-0.45604703784103723,0.9704467329255722,-1.6908561828878124,1.2661005251088149,0.34761517755721416], [-0.45216413072759976,1.125638260528155,-1.8672240017957615,1.388348280279974,0.41289821746105937], [-0.6041998663111035,0.9518745991902821,-1.7482553616491074,1.3027968384522153,0.33003148862622383], [-0.7645915689875407,0.9231306641016475,-1.5908706609397762,1.0416797405804958,0.3435391589642622], [-0.5511129470810242,0.9789516647349409,-1.5896352104287992,1.2985407772348172,0.3074739718251722], [-0.3060111421519473,0.9693226097887153,-1.6503499878201646,1.2050435546957032,0.20529101796457794], [-0.41814813350738833,0.9065345647328998,-1.7369684308945867,1.3095490239413528,0.3931707179477215], [-0.7452070526618801,1.010695258373889,-1.6799990639878284,1.351913921189404,0.48041731347418104], [-0.19474072480894772,0.8574730260551575,-1.7114945513022097,1.2484211088395596,0.35791459113273333], [-0.679333385903053,1.136043445571062,-1.6819398713814546,1.2914383599572574,0.4846586154592743], [-0.38963314954900546,1.0499014443531718,-1.7075189569343263,1.3966853721193921,0.4922463621111953], [-0.4771733790096556,0.8615507710844875,-1.7824679274745288,1.1550649309651535,0.4634518570903764], [-0.46851141840829336,0.9490804068188984,-1.6654371185103947,1.1748885058064462,0.362811152170017], [-0.2486779458724137,1.0146748639649252,-1.8129854069448728,1.3198830969564963,0.3848127258680788], [-0.37278156677721497,0.9026389048304428,-1.6338760097656873,1.2146194636725334,0.305407696426058], [-0.5109164214840503,1.1025203353037651,-1.5805383770959172,1.3659800632138837,0.2507490333376023], [-0.42658774618748607,0.9583040463281658,-1.535401873083165,1.2270967800953845,0.30931926334482973], [-0.5761089637758711,0.9608023900436299,-1.6773446541045054,1.3896056325873751,0.3003385242281313], [-0.333657352857868,0.9284640162518879,-1.6506283984425263,1.4233119402331722,0.4817086358456392], [-0.4131260123218083,0.9257004348959492,-1.6385111064595121,1.2553085674173774,0.35994561408421055], [-0.41619483777131644,0.9791552402269297,-1.6621579631275758,1.1639596596963502,0.3356350519347727], [-0.42150886121352676,0.9497809597217938,-1.4493400602428224,1.2037323844449976,0.28412005647965366], [-0.4036704221590921,0.8682473056100086,-1.4499044130725358,1.4367312949083602,0.29801296100779273], [-0.7347169576181178,0.8472443819718873,-1.6912964247755202,1.1098267823656842,0.5128488081575056], [-0.4599723079910277,1.0313773063743503,-1.6693647567489862,1.1183374503284873,0.3065899617042307], [-0.5155727886799809,0.9793287424722477,-1.81766244647388,1.264051537384318,0.38002468301378506], [-0.461809330593855,0.94300958630738,-1.7083227759840551,1.2779916838408791,0.5269758739704866], [-0.6518981010977536,0.8745407308399856,-1.5963574332371322,1.2679054180945675,0.4099968212180263], [-0.46480686752956457,0.9217771428753339,-1.5266876398223685,1.1382426745675795,0.4778996312070509], [-0.4852820635337406,0.8218636910073019,-1.7767449598018135,1.1371130292343117,0.33381426065377506], [-0.41472873571993296,1.1622114863170214,-1.464996886657917,1.3123782230466363,0.2979393535788235], [-0.38817680425075585,1.0268858467994189,-1.5893492622912915,1.2621731986065745,0.435168936860357], [-0.4138872105825818,1.0639679744840822,-1.6793705616475731,1.0262313606697269,0.26542788201223744], [-0.32689002022123786,0.9411685364764196,-1.6079547828400877,1.222649045231373,0.44202772328420886], [-0.4534191990015764,0.9019382096024549,-1.6042217752367725,1.313476911051435,0.3118812623960018], [-0.3642257288812764,1.0568954661740726,-1.6408419112357935,1.3879191143662102,0.43428969659022393], [-0.4436033491575346,0.951120701402773,-1.7396961236381308,1.0696513684387006,0.1983721283216616], [-0.4195531379041837,0.9339655248016878,-1.6369085623469266,1.2113780206799198,0.32646374286948043]) -2016-06-13 20:15:11 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:11 INFO JobScheduler:59 - Added jobs for time 2000 ms -2016-06-13 20:15:11 INFO TestInputStream:59 - Computing RDD for time 3000 ms -2016-06-13 20:15:11 INFO TestInputStream:59 - Created RDD 13 with Vector([-0.3676131534931217,0.7205292433695121,-1.6148690586461516,1.318613134832133,0.2894000707552024], [-0.63227225792052,1.0480160164806966,-1.7186123125513257,1.3123133303248091,0.4835019323786134], [-0.5130669794355894,1.0744409567693907,-1.5423474727562314,1.4075951590487494,0.34400766625670576], [-0.6214787708363003,0.9941757670924664,-1.7059517928152113,1.1668792946674045,0.5351190865534026], [-0.5610084057337338,0.7736203857280138,-1.8315479385898956,1.3209260735925115,0.28582121198324356], [-0.37376999266386424,0.8253313331048502,-1.7924436246324382,1.3068506619750972,0.2325763177837035], [-0.5087037004414274,0.9607866442106467,-1.812291073591455,1.284458726328898,0.5328516211621773], [-0.3727437082846784,1.028767182687076,-1.7448007272489838,1.0984753757565202,0.3541211398247694], [-0.3814571980697206,1.0819060589675644,-1.733976127510556,1.2515969788209256,0.3640947425626813], [-0.38693583578076807,0.9054649043220095,-1.601255604158471,1.1869423507487187,0.27285330937505237], [-0.453879517976784,0.8265450932182703,-1.696264660015251,1.1219045234195482,0.3528336963815211], [-0.6440396833150053,0.9147072341447406,-1.574634779931242,1.2807375427757628,0.3400101945021918], [-0.5574376128841423,0.9786892277605028,-1.6539643457077906,1.2987195892477945,0.22091568760798275], [-0.3219564568616685,0.7344099605460779,-1.659291949575684,1.3277750279213558,0.23553171253321456], [-0.42287205017678026,0.8870628278654091,-1.7860892614211916,1.3638413188637932,0.35992654268460605], [-0.5114214572125407,1.0739488843585667,-1.803218590871721,1.2580620741259871,0.42523338095005675], [-0.49124237559267386,0.9657816736108445,-1.6156494419042913,1.3063855681095915,0.54155197550267], [-0.3951919316276779,1.144136446459301,-1.6606072772152598,1.4114586343482727,0.3224073809014021], [-0.439470602003815,1.0070997021461388,-1.4975209021956797,1.2890203902600919,0.22540217793727096], [-0.6679727959214776,0.999190248905179,-1.635890493454145,1.2938379558735829,0.20736771134970797], [-0.4118317638697319,0.9069236714993869,-1.7766219570089812,1.2254241772422947,0.3818774093786991], [-0.5019285203332016,1.071358441302261,-1.8681705848296024,1.3543938347082431,0.3694102461708769], [-0.43086686887184694,0.9871654497649167,-1.592715663423245,1.1292659913487786,0.24090456644129066], [-0.6554222411786117,0.9775977489456485,-1.7739871948427586,1.3756970876324615,0.5345682469155936], [-0.606670169993634,1.0414671201540915,-1.5816535152099802,1.2849832898961777,0.3443771799618759], [-0.6330347409363863,0.7614040653882075,-1.6420712005053097,1.344034328413472,0.5019032888808933], [-0.5268140556262974,1.1016458967835785,-1.581552701703922,1.1842999784487596,0.3683353834757425], [-0.49900894372274485,1.0645644739185587,-1.5355753177851752,1.2694091849907172,0.33468156327241705], [-0.31358398021960576,0.8302665811856854,-1.5422704772273437,1.1506707176948,0.37999408831157194], [-0.557673046622542,1.0913399597085338,-1.5871539417007638,1.0478537277843192,0.31421916981662246], [-0.4100320104849945,1.0606699393377843,-1.764930551709285,1.3173125309085043,0.32112514544554777], [-0.2578093453260759,0.9604640936063775,-1.6498739501347253,1.251662741314794,0.5352359752706277], [-0.5830353039132012,0.9147683830431232,-1.747368848835465,1.1923888734167842,0.4792523557272257], [-0.3091820573873779,1.0228642342621612,-1.472857793608509,1.165376819972311,0.3758448993415674], [-0.371190450361484,0.855956409330075,-1.82995473329898,1.3390818705810552,0.37113293274981823], [-0.5539807557276546,0.9677998945369973,-1.6350804443189515,1.3419255190805948,0.44899955987573437], [-0.5376287808107136,0.8413279828562651,-1.6762865574706214,1.19949852014792,0.5047576712678563], [-0.6003249423212255,0.9906964773703084,-1.5370089708944512,1.3072479529851824,0.430528835393197], [-0.4175368422724912,0.8187481538877078,-1.7996629160543502,1.1879999792808238,0.473510867372402], [-0.4828257087231799,1.0612823363596418,-1.6425767546040255,1.2270450796716112,0.39748339044894354], [-0.6332333955565639,0.9519761064158869,-1.5479445620477186,1.308465915443944,0.40088093604627134], [-0.5228584674134156,0.8768992895766171,-1.8336272924164618,1.1499724885170817,0.48668761341650246], [-0.36341836902693225,0.922366126760111,-1.4327493627862522,1.255336229334989,0.29637579738564857], [-0.45752242402693893,0.881182369510346,-1.8075367810499625,1.1836209711736738,0.5240597883932867], [-0.45127422400043055,0.7875661584033756,-1.9374943002165643,1.3022768816466794,0.3865261883663946], [-0.3385546242930144,0.8796112795269294,-1.6285471480854194,1.177302672296305,0.3611750358009876], [-0.3866867752880084,0.9445769802243262,-1.63589592788523,1.3115437594462822,0.30025415854590604], [-0.35087047853321573,0.8048561114260069,-1.6540296199948705,1.1069576251024358,0.3739108460295009], [-0.6118021930146711,1.0836229312504084,-1.6965417516081935,1.3814661848764345,0.5544639396063888], [-0.32579850447352154,0.9627503718472757,-1.7118868486570202,1.3378056459968657,0.26239820187294716]) -2016-06-13 20:15:11 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:11 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:15:11 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:11 INFO JobScheduler:59 - Added jobs for time 3000 ms -2016-06-13 20:15:11 INFO TestInputStream:59 - Computing RDD for time 4000 ms -2016-06-13 20:15:11 INFO TestInputStream:59 - Created RDD 20 with Vector([-0.4302807096972894,0.8599904088505619,-1.74473044235192,1.4559107913903748,0.3868065070182458], [-0.4809066717988696,1.1355083469098752,-1.568604721573235,1.1620770786476713,0.5052691040344791], [-0.37948367495671453,0.9612314328384234,-1.7288918432476479,1.1900277125124938,0.584148529955661], [-0.4830118079148656,0.767521114064722,-1.626523514538083,1.2344387914004133,0.2949434953618521], [-0.45982168221474795,0.9610478654584301,-1.4181098052958898,1.1960502956938037,0.35754207882888805], [-0.5403734624385554,1.0961985914256847,-1.4824794478647174,1.1331480033700656,0.4391107899439444], [-0.4078934785304544,0.9024846949827434,-1.772449379651882,1.1891755675558509,0.26886721197216756], [-0.4850713563393695,0.9854430300369302,-1.6392713106609738,1.1640485301086894,0.4058728213551233], [-0.5687863647866291,0.8969696803234791,-1.694332984322472,1.263598085578097,0.4916946976192132], [-0.4852057062952629,0.9746523770852169,-1.7152410694648534,1.290070394136892,0.15734923086759972], [-0.6042639508376677,0.9587258437320635,-1.796821218335353,1.2415150801911263,0.5036349732580911], [-0.5634160393957598,1.000927422915143,-1.7677443118596388,1.1056602828501068,0.3662560181172587], [-0.42260218853055576,1.038402090735988,-1.623454196568306,1.258759507363921,0.35817719144396376], [-0.6935865876752814,0.9767485891098648,-1.6119749119230544,1.3241871653009805,0.45959708474651645], [-0.34982681977824615,1.0697929536346964,-1.6430495277049175,1.5160135223968076,0.3720168274191172], [-0.4965183720272103,0.8867675212850072,-1.6326283681810188,1.1014099972707194,0.3464777829375711], [-0.4429523196454975,0.9569586297887442,-1.6975710388611154,1.3240706523955248,0.32888337316923105], [-0.6110986163387908,0.8513837967033338,-1.863122720264613,1.1805944382455067,0.36642329779978594], [-0.5770973903898762,0.908469634420286,-1.5969552518736576,1.2639442528889475,0.30334073173665643], [-0.6677676499752639,0.9993894428543435,-1.7484246181717757,1.3024629995033956,0.5254188589994735], [-0.43541597877213595,0.9518339300904892,-1.586032231723223,1.2859374872876397,0.3728538289653893], [-0.3981355023125287,1.0914065768869097,-1.604054951142783,1.5597686669581905,0.3200687403051776], [-0.3526167305034049,0.8582438505591754,-1.795079875034891,1.291753863011317,0.5193327226327783], [-0.5322224604485944,0.9225091285363366,-1.5406535924270095,1.1740734779955952,0.3985082190639408], [-0.459298613922177,0.8499917798311437,-1.7381604837279123,1.3363117012403978,0.39372367283850573], [-0.5029856763543168,0.9563691025906603,-1.6323146191672946,1.1564017424707789,0.2992277161858537], [-0.4702645909815308,0.9857893178047549,-1.6146159696178013,1.2941994476820615,0.29510677438190996], [-0.5010373585673528,1.0381084928077213,-1.6592946006629066,1.2275293781382213,0.6215806870134133], [-0.34251104673790145,1.0920152105960284,-1.4702129754795525,1.3052389503227437,0.5142842632664187], [-0.6014879705809496,1.0573110467882678,-1.6994508095500127,1.2345579924787826,0.27260709575033387], [-0.374393054126274,0.8520958347461538,-1.6227534538993043,1.400050919842349,0.3749155714488258], [-0.5537306132792297,1.0237199970470363,-1.5996513021748293,1.3277912515047763,0.3298858295939718], [-0.25019038644507985,0.8933568126093641,-1.6180540579171452,1.2514548524704747,0.4485167299028402], [-0.4083941282128968,0.9821808764315939,-1.7710803507794117,1.22246902401698,0.39616717157783654], [-0.6617567254520433,1.135205204672421,-1.7404704566578575,1.2634534028343156,0.48631393245586374], [-0.5102944111175142,0.7749069961072899,-1.3746153795885292,1.1761695768038691,0.44519535098184526], [-0.5273802259607313,1.0293844883271042,-1.8327600208787889,1.3032607095759106,0.4810318980867238], [-0.43680250540361804,0.818212719795147,-1.6397291545525832,1.2164204417976607,0.36911234859465697], [-0.44511271834694277,1.0039279649012336,-1.736482991573775,1.1407721179472239,0.32026695790267784], [-0.461571293170463,0.9623234284494812,-1.608651111923442,1.3054108488252942,0.2808744029932673], [-0.44626644352032524,0.9385912062499339,-1.5784088711897624,1.2996894478753678,0.3367841237735643], [-0.3458002275610911,1.0002670117410046,-1.556743986487141,1.220789398824709,0.3047413342602086], [-0.7006173067436341,1.0226530560056757,-1.7453818996615391,1.2468257997531524,0.43730757861422237], [-0.4921877365188532,0.9365903329931373,-1.6954478906187997,1.252660426269991,0.411305000260049], [-0.5866727516355298,0.8540617355784135,-1.7495419546675461,1.1992666379768737,0.5624798329629086], [-0.4552502733367031,0.9291860448712651,-1.5908193525700438,1.166718738832733,0.3281256456654895], [-0.3426613548769225,0.7538960431270546,-1.585524926597853,1.478740164978157,0.251131880834575], [-0.4680826818575833,1.1112444944704614,-1.6587653968723466,1.2870287092936261,0.4050920708933554], [-0.5624024717932173,0.880862871462712,-1.6303493150007178,1.1721463768303386,0.5696913393311782], [-0.48266079841446297,0.8916088275950524,-1.5393517554424276,1.1897790454674049,0.4411535715241579]) -2016-06-13 20:15:11 INFO DAGScheduler:59 - Registering RDD 10 (apply at Transformer.scala:22) -2016-06-13 20:15:11 INFO JobScheduler:59 - Added jobs for time 4000 ms -2016-06-13 20:15:11 INFO DAGScheduler:59 - Got job 0 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:15:11 INFO TestInputStream:59 - Computing RDD for time 5000 ms -2016-06-13 20:15:11 INFO DAGScheduler:59 - Final stage: ResultStage 1(apply at Transformer.scala:22) -2016-06-13 20:15:11 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 20:15:11 INFO TestInputStream:59 - Created RDD 26 with Vector([-0.3816421005537525,1.0617035428188992,-1.7399461177422264,1.332980235806715,0.2916494841747297], [-0.6583584440016957,0.9555891409969888,-1.5454326525102315,1.1741091058996926,0.305220337835565], [-0.2989177604038451,0.8701688291879833,-1.8804582484676076,1.2986162711936382,0.35958131004690164], [-0.4939772883087695,0.9969220817446861,-1.5980382534286326,1.2969252872376784,0.37976918116852443], [-0.35668565131454827,0.8078147339434413,-1.657193160993533,1.359095599717262,0.31192105888186633], [-0.5450865703006346,0.8750391278919141,-1.7453306059026725,1.3721329076982913,0.42237633518141116], [-0.4917580165015238,0.9512146417235625,-1.9326523267650781,1.4266827266379667,0.3225713722197544], [-0.4048718474607664,1.0240256182146898,-1.6774401089659943,1.2669846165410654,0.33732641244233585], [-0.47579982131353826,0.7947369494624074,-1.7655419119652316,1.082577012553155,0.35401987740372537], [-0.35538929020910864,0.965963799724578,-1.7529404087985827,1.2055043918858341,0.3342486683177578], [-0.7014541659835856,1.034642549879647,-1.5544540159626516,1.3033886639865033,0.4054677613509847], [-0.43610110990224626,0.967077951705683,-1.5262368892289788,1.2639835242537958,0.3181179758885862], [-0.3425524324739105,0.9894095680520077,-1.7777745234498326,1.3091660388427224,0.331841648562285], [-0.4878205400579338,0.8560103963724833,-1.7352326850283464,1.3664274889095067,0.3984111315135807], [-0.7121859062829006,0.9130040407010276,-1.7874802517046422,1.1874648639293004,0.45017886399842666], [-0.696643078139105,1.1467663562988957,-1.7296410056290408,1.296144064682525,0.5048605187772839], [-0.4011230886898419,0.7361859073107719,-1.5317733987729902,1.2701183330425934,0.46062351610078545], [-0.45715694326129586,0.9190057703920946,-1.6904193500197242,1.3271044644288563,0.4842403797714084], [-0.5095249256026796,1.0659631362797184,-1.4402626928522713,1.3379987286264212,0.2673039786797317], [-0.4167094825032959,1.1091402127599272,-1.705921687753808,1.1183411370664753,0.48305727426817385], [-0.2924464534867928,0.8213969946455459,-1.662853823551024,1.3355427142685694,0.3770025002372693], [-0.4444725119583886,1.1046045679759258,-1.456164663000235,1.227847061522124,0.5255642346814823], [-0.4048329834921757,0.9788094275209757,-1.539852245269681,1.180160780953293,0.6787253334538687], [-0.38022990049474503,1.0144858953551337,-1.8072904612189131,1.3419776986257266,0.4041658441938728], [-0.3763555278351052,0.8998086161617835,-1.705061140148122,1.3495005676173746,0.5191631674639813], [-0.5540229100290982,1.0669438671830174,-1.636684794385797,1.3459012222977915,0.4181134505939225], [-0.45042550789662783,1.113240852339999,-1.486260571622101,1.2719922983443672,0.4102349177516307], [-0.5546179681493353,0.9938406825700987,-1.7084301934096207,1.3880541591389144,0.29690506797892435], [-0.5006405047598609,1.090292575906863,-1.7104271793907637,1.3062089077771892,0.47556018075456363], [-0.36520205819569046,0.9201397468605166,-1.7602021590969437,1.3876082296931265,0.2308718010458195], [-0.4938168714972593,1.0964903470900649,-1.7260349751744855,1.2048249569339422,0.40946925790527117], [-0.5092411838541296,0.9882559808204627,-1.7212908932580275,1.1127850487389395,0.3940121165600333], [-0.5882412062939182,1.1156935762589453,-1.8131619026625396,1.4146422480126442,0.31361816553861704], [-0.46990162437316874,0.9908156946435108,-1.748827835513362,1.3957588501434537,0.5512480921903505], [-0.5405868391855275,0.929247220281523,-1.5894550246414005,1.271255300937136,0.46529614772048844], [-0.46376208782353645,0.8174067828364345,-1.711422288030576,1.150388298327176,0.36268778857829526], [-0.4617415170232849,1.1203248336099854,-1.9346820049945674,1.1466858815338403,0.318614186382022], [-0.5200553635156823,1.1206383922863123,-1.721393160577411,1.2564342218266722,0.39290287510750027], [-0.45474634000888675,0.9538310968737261,-1.5572305892194418,1.2293659662686987,0.3951581444726625], [-0.31024997329419923,0.9626097763501761,-1.5774105560033171,1.3323705378196256,0.28213579823419327], [-0.6425546702182562,1.0413861558054223,-1.6696029184789263,1.3297020370998587,0.4667426127538354], [-0.5088483797878551,1.2170009632967245,-1.7143214767417212,1.2915402658283741,0.27813229259982863], [-0.5138048643277845,1.0537687232096613,-1.5306968187748822,1.2020601237200264,0.27213168784492703], [-0.5375432544160766,1.0418784698921373,-1.501800504998784,1.217708599285121,0.6443267111319482], [-0.5784571285851817,0.9518620188834632,-1.656094482882889,1.2785669143560554,0.34103467230048085], [-0.39057405488844044,0.995936983645342,-1.6323671359433234,1.271155031008547,0.3558239098317178], [-0.3682240577265206,0.8711375369867465,-1.6625627384453794,1.1969170451325948,0.47233733398400285], [-0.446008720311628,0.8976699326654971,-1.5289286019943875,1.2244855320582673,0.2984778353929738], [-0.5102538395816835,0.9986885141123109,-1.6414035100650266,1.2420020420446427,0.36032207135032196], [-0.6350165259371148,1.0900186871655364,-1.7907186442371252,1.4304360117451347,0.35387005412887984]) -2016-06-13 20:15:11 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 20:15:11 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[10] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:15:11 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:11 INFO JobScheduler:59 - Added jobs for time 5000 ms -2016-06-13 20:15:11 INFO TestInputStream:59 - Computing RDD for time 6000 ms -2016-06-13 20:15:11 INFO TestInputStream:59 - Created RDD 32 with Vector([-0.5768797964115748,1.028505413551091,-1.7521414076692798,1.410931775189879,0.30527264662469095], [-0.40591177777424625,0.8661878776717759,-1.7310200236861213,1.3594055750354543,0.3299548132445285], [-0.6383561683498007,0.9259033203935216,-1.6881320011035645,1.3172994752602565,0.3214908371039436], [-0.624062828319168,1.1219523756569185,-1.5603263506923595,1.228964619537528,0.35487057209905903], [-0.4734064808910865,0.9649955190976767,-1.6068693471541338,1.3196307767213002,0.37116726129033706], [-0.6101841770121739,1.059654359501757,-1.8099667264246553,1.3759685981009824,0.5438261473052933], [-0.18850356552447423,1.1440264277756391,-1.724865192544724,1.1359044353122627,0.31757393897273056], [-0.41254200355821263,0.8607204799101216,-1.616620448535661,1.3427363220425623,0.4109728277211949], [-0.393734447592108,1.057690953958827,-1.6695747980566427,1.327034390345423,0.13819106599587025], [-0.32090427274943434,1.0651687359751123,-1.6980368198411557,1.4017902860954718,0.36022562482340503], [-0.6053061980873911,1.0041632375635317,-1.7820663797212062,1.2269162643461793,0.4840114520476853], [-0.27268808986968307,1.013170380627259,-1.6005981535949472,1.4382064950763866,0.3892224005152082], [-0.535422550377724,0.8375173015602825,-1.661788348017347,1.3227481832299182,0.311533159171815], [-0.4009368458319039,0.9897832871295871,-1.6696688300152116,1.4424037410980737,0.32734398317968916], [-0.6825087101818776,0.9056073621397804,-1.659854864696752,1.2759644672519115,0.43884955280528926], [-0.43179250129551383,0.7465728726494555,-1.5636729803345486,1.3457117728614754,0.5721971492726551], [-0.526304418227463,1.0501002903902692,-1.5968858098432406,1.2982785754583357,0.37494021304786934], [-0.4320225453412649,1.1715904807145918,-1.5178148547272063,1.348200261386646,0.40779395798035983], [-0.42221995667102147,0.87623418086733,-1.73837906100808,1.4183919259975142,0.33228674704284467], [-0.5629520941212462,0.944212875923361,-1.5546071736198348,1.4519582067498829,0.262200899044119], [-0.30845271114048184,0.9681639591372486,-1.688842111249905,1.3763468548667668,0.3173125681909059], [-0.5198567190391367,0.9335053916544206,-1.509472274649017,1.454415814887731,0.4589687217941236], [-0.5711775989363378,1.0215061238027188,-1.6224000734110398,1.350665861834858,0.36668462127595286], [-0.39949682119974983,1.0408092246134175,-1.6587225284400737,1.301865863792736,0.4563742176523199], [-0.4902566261487769,1.0992586451801099,-1.4473476422759635,1.2309123590548212,0.2544829822903507], [-0.5293628624912633,0.8432327849564988,-1.7669603672683885,1.2911210280012069,0.41983427336223667], [-0.5738614502843032,0.9784937768658387,-1.7194271904022407,1.3116656980237082,0.46218317857671776], [-0.5902501988138692,0.9423215775018176,-1.684040151093259,1.4235274237923208,0.2039286490216381], [-0.4777727934676478,1.027586212069216,-1.7495753442806758,1.2757769414973703,0.3810309831295745], [-0.6566621783684151,0.9641681686268665,-1.5286420868141977,1.508411206136107,0.2501550388463756], [-0.46296281092785885,0.9727194845614605,-1.6171422039154086,1.3434828829348795,0.07809723939214058], [-0.5738547760098169,1.1077074999997352,-1.739110700752265,1.3494171899976157,0.3222123114107347], [-0.4002887849620737,0.9619411680270862,-1.663509769541054,1.2908149710547212,0.4244227376717664], [-0.4665533215112465,0.8596395857982323,-1.7467795346659782,1.251224446006115,0.42794754034954735], [-0.6374258626266393,1.0514903728245393,-1.651303971310752,1.236884794719695,0.5249538337308562], [-0.48340361610998744,0.8198136168919055,-1.690727257833199,1.398396991016531,0.31841739933411595], [-0.5410633313507479,0.8973900953721683,-1.4724867408929736,1.388151513303225,0.47893645155994446], [-0.3039096405457719,0.9155005236200422,-1.7173365893442798,1.256097860517092,0.24819750391337136], [-0.7090962179115853,0.8944749271173439,-1.8839838110448957,1.1446445350131944,0.22840975233443309], [-0.5100953498526793,0.8107010460411471,-1.7303477866873604,1.1647356509362308,0.4418371093135496], [-0.2962087734742913,0.7242125014742148,-1.5984563979400566,1.0588259317312654,0.28436170748337264], [-0.5505122810716748,0.7806970076803257,-1.72088933624607,1.4230022424760869,0.32673184311928194], [-0.45127942352855155,0.9959060503010477,-1.5524186364009327,1.2366972856510523,0.3578155222046753], [-0.5102066277582917,1.0949182323987419,-1.7142222646495175,1.1664611293679565,0.3602965879997574], [-0.20421136622466618,0.9866371147144095,-1.7538283133815977,1.0465139384173783,0.41274915410898083], [-0.3678643518086112,1.1128928060624481,-1.7416958328862653,1.1712943420281356,0.522586652590277], [-0.4272255324098842,0.9505411417476373,-1.5398631578697235,1.1670050581250606,0.18826195051488878], [-0.6294443147944699,1.0182211296265207,-1.6114292740208946,1.5494854908967275,0.2354819914065145], [-0.46052379116984477,0.8965751438720174,-1.6887957613207942,1.2678155037585463,0.3277249457164221], [-0.4661555388594827,1.0176554636610373,-1.660202673672577,1.1543960943838145,0.41490582984943747]) -2016-06-13 20:15:11 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:11 INFO JobScheduler:59 - Added jobs for time 6000 ms -2016-06-13 20:15:11 INFO TestInputStream:59 - Computing RDD for time 7000 ms -2016-06-13 20:15:11 INFO TestInputStream:59 - Created RDD 38 with Vector([-0.5184134212251081,0.9455318561030377,-1.7249072888539423,1.2476376390078272,0.539371371249466], [-0.4839353019153916,0.992343730650464,-1.4037670310964987,1.3650053852483648,0.287928402259341], [-0.45217444521287575,1.0425364789095595,-1.7771813152553952,1.2241390773159404,0.3053067128704871], [-0.34943560877945984,1.0407179160044182,-1.5177836780575653,0.9824846085205396,0.2603543256712868], [-0.45829448575134074,0.9725249117979422,-1.690056167350807,1.035512584568622,0.3026754916713263], [-0.42458145588018753,0.7854084174090814,-1.7649589009106188,1.1975914903392633,0.6551885852829413], [-0.5104497237497339,1.0402201914534017,-1.6859518565969145,1.1391556996625576,0.18881595045601088], [-0.5588856532488075,0.8709355475677711,-1.4735465340859104,1.207362168238276,0.5886898848511445], [-0.5286914346418723,0.9318873502365327,-1.654993790470044,1.274056873049218,0.29412525529417294], [-0.5420811906362676,0.9995645356070102,-1.6040227637757893,1.3412654820039975,0.3950459929854693], [-0.5105371229433685,0.8870982484234274,-1.6366208840493932,1.344244897498087,0.2842199976348651], [-0.5758159328886613,0.9361950961962593,-1.5151431024376867,1.2963264217469896,0.4195802203163277], [-0.46650906956232724,1.0883479790237522,-1.6209503836486092,1.2506119015001946,0.5239087792157935], [-0.49800785133272085,0.9121648111767798,-1.5664537098618618,1.4303891258276538,0.23796160092517643], [-0.6200401622139905,0.8819930763699753,-1.7712553265237971,1.144151394858828,0.37228012783293096], [-0.43683215654432145,0.8386127968983885,-1.6402554451670102,1.129209493050289,0.3612473529185551], [-0.4079221564079838,0.9656864104202829,-1.7359335721136713,1.4421338682074127,0.3616284206434529], [-0.43442591279896603,0.955449363348641,-1.5334677832140913,1.1666726886559795,0.5058368495019173], [-0.6457828019918121,0.931131898174338,-1.8532037937152797,1.0375316080434018,0.3133563527606291], [-0.38369598973754027,1.040856056086344,-1.7594025439463896,1.4917751006832976,0.45495744983872133], [-0.30166559339288085,1.0342180102112146,-1.5650152370405155,1.2673629485315197,0.4158807517084573], [-0.28580825731967757,0.8865395334100423,-1.4737235251638032,1.245008984525024,0.36435255784505255], [-0.34071421524008416,0.9427461052254856,-1.6571555985762556,1.116205755573728,0.3440594901972376], [-0.5707704618700846,0.9955277694976673,-1.815098893846346,1.2976867834380266,0.3708933869585827], [-0.6095164341318,0.9322989548597498,-1.6576644540187038,1.3001089252401639,0.32781904617649943], [-0.5884784232045771,0.9963121018549773,-1.4726908596796255,1.1232169328419297,0.466258074909203], [-0.6351651621700529,1.0173421935079714,-1.7981429560407272,1.0336059974520695,0.28162584897435694], [-0.38203401021631267,0.9790771913538491,-1.7244683271662424,1.288084387752797,0.4312827591331814], [-0.37183909143628785,1.1926117173870312,-1.5166716166223042,1.1696263300990144,0.4515586032102953], [-0.47462520704983,1.0498070539706301,-1.6863851371590932,1.2965114835042095,0.4645837570114517], [-0.35874130630231327,1.0042819076442355,-1.6407835501913508,1.214144240421076,0.35534792382876684], [-0.41389636789198914,0.9508170625312302,-1.6498089027313338,1.4272197180381103,0.374553819362672], [-0.32132785261092967,0.8535769504185499,-1.672835438070241,0.9421132176631009,0.24740658936307364], [-0.48253792736985773,1.0429236941137325,-1.4939460450129358,1.2832409419178106,0.40323476275374437], [-0.391512958705594,0.8859909853510953,-1.607045573914962,1.1770226037327562,0.5198479967492], [-0.4833163026675927,0.9200058086737815,-1.8188555219632765,1.2111735788924018,0.39618714901430313], [-0.2913135273832381,0.9301485161935364,-1.6975841978124846,1.4065399033140502,0.28453881400648173], [-0.3073756680829076,0.8325242716417236,-1.6753209327630654,1.4075134801045086,0.30351866480787215], [-0.2775051790445552,0.9452680429408632,-1.6224110577930677,1.1778711923142833,0.5531745712889398], [-0.3047686297259775,0.9468967188067203,-1.6929112749188848,1.4636396431851675,0.3586449109599954], [-0.5956899619218391,0.813236818549011,-1.757649523640403,1.2604654777105053,0.3185144577112101], [-0.32794499152459505,0.8140339815697406,-1.8666906537843855,1.2572887839007447,0.23549894606903882], [-0.40686690942678344,0.8619998816707273,-1.6319579560604098,1.3664940728963186,0.422156696202466], [-0.4138601150243667,1.021828823586237,-1.54841612428864,1.2398622089963618,0.5070042357490803], [-0.331869975870002,0.8782404384527787,-1.6635875859844171,1.2435143760020457,0.32101906837863314], [-0.49700214293681766,0.9856184010338479,-1.8232422669575907,1.2421051455581202,0.47592591320323163], [-0.535909591266969,0.9807383109826389,-1.6974325972167972,1.334548321075211,0.23946350824403165], [-0.6444021824207316,1.1849780210499694,-1.5121795557147575,1.3604386920675193,0.31690181205044465], [-0.49746220541003966,1.0643703042656938,-1.8499798436118189,1.1357972819777942,0.5780379400362996], [-0.4383285608503117,1.0163805234573544,-1.9030246206122645,1.438368352860807,0.3293850463477472]) -2016-06-13 20:15:11 INFO JobScheduler:59 - Added jobs for time 7000 ms -2016-06-13 20:15:11 INFO TestInputStream:59 - Computing RDD for time 8000 ms -2016-06-13 20:15:11 INFO TestInputStream:59 - Created RDD 44 with Vector([-0.374705866875181,0.9840691262184086,-1.5891551104970965,1.4814595125988181,0.47065837960007695], [-0.288810885883263,0.8437842751113669,-1.5395954061634287,1.1874811761270259,0.021249409500126437], [-0.4606335450965845,1.1012472510905122,-1.8287426699010534,1.24172318938614,0.4710333368456244], [-0.42944940507321583,1.0104988720375858,-1.5751317017658801,1.2831547194950401,0.3512701703197814], [-0.5152127056418667,1.1709583832128079,-1.6554391510043867,1.1397001494774806,0.23397277605773995], [-0.4399479870409779,1.024335104965922,-1.7045108897162344,1.196833551305392,0.4117115230011916], [-0.5430543625505585,1.1370095574712402,-1.7176429060334013,1.1901279759925694,0.46070443041212095], [-0.3968714897698767,1.0926860330748331,-1.6334286394688688,1.228234808051336,0.4665607383628074], [-0.46215342653947683,1.0343329141616782,-1.6499943761334228,1.337530812749641,0.23114556003826187], [-0.5000235665427208,0.9151728338053156,-1.8055132514640995,1.3037686620331144,0.2902025184248946], [-0.309233049825866,0.9728959274674948,-1.6392691162173578,1.101247705752869,0.3496423465401415], [-0.5981691090699208,0.8739296001671861,-1.5687282698937923,1.2550952881121755,0.46308264317451964], [-0.425561195913103,1.0676958602044149,-1.7135165364767921,1.3524692002054561,0.23228520033456318], [-0.7043177723021486,1.0283311625585259,-1.7478644671187442,1.2858120358708933,0.3418710936510813], [-0.41727892194293686,0.8625867727705405,-1.7914100057674134,1.1786792695315231,0.4917512427338196], [-0.40174243931692544,0.7828098802704966,-1.5073762714652954,1.281118669072956,0.44887930601066905], [-0.35310979651234775,0.9745364825956087,-1.6392317059509176,1.4513644518189859,0.3110119088453469], [-0.6853829628094648,1.1253397147829511,-1.4920081564639467,1.231135544811062,0.3139314829419366], [-0.44152016739085975,1.1304790288137783,-1.7325115907360644,1.3251089521914843,0.479567851123668], [-0.5526275004039871,0.9674440446022308,-1.7971453051937036,1.1664342514118335,0.31260632993291754], [-0.46562594489923503,0.96412818249357,-1.7756506925990838,1.1820367323525278,0.37368762092821634], [-0.42414895043860573,0.9369437276073684,-1.6228073393039317,1.2110129935336054,0.33600782254298567], [-0.4507513844294476,1.008422902581697,-1.6947838717172434,1.1258385535052762,0.23649573041213182], [-0.5011767543824581,1.0367482963168666,-1.8619227324329928,1.187686906059264,0.41060169959227016], [-0.29061733919433663,0.9125980333805904,-1.773384852821433,1.2318813347419482,0.5360023941871213], [-0.47558633395179245,1.0112018987992073,-1.6443927870523196,1.1868477340320063,0.378412637239359], [-0.47217857076866115,0.9027887018339104,-1.7359968088612976,1.372505255254142,0.4024631611010435], [-0.5635782393384723,1.0418520168542464,-1.6371483392749435,1.3046794831870752,0.36435743882876215], [-0.3566584096699375,0.9045378099174617,-1.6027843810019622,1.308376882406936,0.450420525347206], [-0.5161548558091542,0.9362466126941537,-1.543853038663444,1.351176902637456,0.37019671664199505], [-0.37734135089435306,1.1354760893808167,-1.5928707071728756,1.4230797310043477,0.5395830646701685], [-0.4935941975274797,0.9431897551602951,-1.7363999282764386,1.2592777434544042,0.34477211805728875], [-0.5399690948357754,1.0773952946269298,-1.623286131549866,1.3571017009557331,0.3820149575212495], [-0.47470272833792704,1.0279092716939924,-1.5583033476707477,1.2928147173688327,0.12854158780432537], [-0.3932748299826361,0.9168968642381998,-1.901294092833649,1.259188794957511,0.4011688713714673], [-0.3557493574970649,1.0003577059585267,-1.5111904842067994,1.3123818859934535,0.3019665980708428], [-0.38849198387448786,1.0634537152100978,-1.8742015500852314,1.207616787308903,0.527158124827907], [-0.32516837051884395,1.0806577326663074,-1.611781542302308,1.182593379846847,0.5196789294606137], [-0.5212725898858087,0.998490506190397,-1.6184763965153202,1.3731447165706872,0.29855378648402087], [-0.40257752034798155,0.8314340557777532,-1.6850156321125218,1.3427297403898506,0.5156164947536159], [-0.4889473969173199,0.8999745071099362,-1.4468367122261747,1.3090812041423647,0.5108182965913203], [-0.48274282778056277,1.1537075465803446,-1.5811276311378704,1.287632315416058,0.50882779799236], [-0.23362208323850905,0.9548750423156142,-1.572161725609678,1.6027003014260315,0.3939518404103326], [-0.4292988792888861,0.9366381149644413,-1.8250064321036654,1.2358072562830058,0.18892247590763034], [-0.484748914270206,0.7653236279449509,-1.7486834183984747,1.2949268753185315,0.39683334762547673], [-0.4399736279490563,0.9634644976145054,-1.7480413166448512,1.1775955441636703,0.36332416278682794], [-0.48373350642316953,0.9259530051332495,-1.7977337219360736,1.2876089889322726,0.4497462132788797], [-0.38070426405120866,0.8838236645507052,-1.6753928124899833,1.2449514905082557,0.3276558558550576], [-0.5717833110401351,1.0273299096906752,-1.6190879043151574,1.1860455839092494,0.32636997750947594], [-0.5231330958628938,0.924256801489448,-1.5484246875687981,1.3381528852011608,0.3757180006852602]) -2016-06-13 20:15:11 INFO JobScheduler:59 - Added jobs for time 8000 ms -2016-06-13 20:15:11 INFO TestInputStream:59 - Computing RDD for time 9000 ms -2016-06-13 20:15:11 INFO TestInputStream:59 - Created RDD 50 with Vector([-0.4026578572929527,1.0001062834205203,-1.5617908885948877,1.462733521830009,0.2061287167736916], [-0.3965287353033582,1.0683726645043978,-1.4873730061169563,1.1460792990307642,0.27746130384846085], [-0.4938197206621628,0.7709438014433136,-1.6076436749339549,1.0982028796258234,0.24486596655062928], [-0.4777650011847253,0.9489155740228515,-1.476414143798642,1.3062514135704215,0.6114439757268249], [-0.4285977940802228,0.9382568588433707,-1.48976698594476,1.1698457774900977,0.297712575086562], [-0.5993896749484521,0.8629058534578911,-1.6510698754517374,1.2090117714723978,0.5458042851487852], [-0.5321474831529757,1.0849054231681536,-1.667568850178397,1.4161664775537062,0.4493689535824759], [-0.6723705689897437,0.7517542425048965,-1.663645669684176,1.2240252996527488,0.346848907683753], [-0.40115735788941104,0.8163474979541504,-1.7694648704001712,1.1073947331922183,0.3856322637520626], [-0.4787497898360035,0.855418622108937,-1.7422359123781868,1.2371448524727167,0.5156131237949001], [-0.5321048883779993,0.7970778523677844,-1.7616773188005117,1.3821414318824692,0.48832517040159346], [-0.5285360880098152,1.0597395124384508,-1.6417892576104653,1.3464020747697325,0.31849105791627297], [-0.5324435853239631,0.9841563147023031,-1.5943852209495275,1.2707723748060116,0.4139910360672806], [-0.37531742848811983,0.7874289125205598,-1.6479869931712725,1.3181636612914271,0.3718789043065717], [-0.4560433134330382,1.0759812484577316,-1.697865869705092,1.2725177127861855,0.06883328163775021], [-0.45513942313770217,0.9498671339799809,-1.6933930804659583,1.3025139463978586,0.5350768600248874], [-0.526256522617052,0.8727003678813281,-1.4356120595661468,1.29993914263913,0.43387328603333664], [-0.3923483046111808,0.994060772829843,-1.6451266300509706,1.2983503204985913,0.3703765349262901], [-0.4374913726020475,0.8955789511391181,-1.4728897306445088,1.1357862983779858,0.30881407016500095], [-0.41951697820602707,1.0007695283696842,-1.708286281016387,1.4190975260604537,0.28731659836668877], [-0.6750333217393494,1.0688779823591679,-1.6846746627252691,1.2962992487600882,0.5030936797752793], [-0.5956866906574221,0.8121299274245898,-1.692929629203368,1.2338548732523056,0.38371772482125543], [-0.4303591738987757,0.9459620785706632,-1.884551120494729,1.4937212054647857,0.415792749586519], [-0.45504157760203423,0.9027897906482127,-1.799761299703666,1.3040399621270629,0.5736987408004388], [-0.5593669081347996,1.0931782446321396,-1.627955485174887,1.1391759184235553,0.39801309200504786], [-0.3825955534231694,0.9655263256725013,-1.7068123920171723,1.1861538643513039,0.2764471843224494], [-0.3587313623839552,0.9371265382727668,-1.487908236071683,1.3184799953634567,0.4370396706398445], [-0.475038660091078,0.797460346861311,-1.6835937249660264,1.3120302511529962,0.17031892070731325], [-0.4663190378327658,0.991352295299833,-1.6444316855302834,1.51526349576866,0.3929422369724258], [-0.5040823137977553,1.0575372259159488,-1.594322659684267,1.2971698194040933,0.5092825217956699], [-0.44790417664079346,0.8518362899333143,-1.648211585312696,1.1839806869130056,0.40832085080824754], [-0.537755895075537,0.9806637869508009,-1.7656906916731008,1.2433660326767015,0.42887767735060545], [-0.34293660736335746,0.7935119912222357,-1.6359022010243611,1.3593857228798436,0.3967833959145234], [-0.6208695835786292,0.9640100782092327,-1.874229161549032,1.343981009171891,0.20123531286937832], [-0.6723717817990262,0.9002960054466735,-1.6910864401441592,1.1818255352705958,0.5433986930426408], [-0.4588708699931968,1.2092994554375371,-1.5324768096769932,1.4083576723657105,0.3708605878943178], [-0.560638238383703,0.9092570275383438,-1.473047966472027,1.3497511209299733,0.3467764467476322], [-0.4409803705877342,0.9370238683801353,-1.572432943897284,1.41498798111251,0.2748026192628795], [-0.38741643074715315,1.071718105146961,-1.624146896559199,1.1401522638836572,0.3872910322155703], [-0.5140862086906481,0.8211106116047733,-1.6027037015807561,1.1677747630844688,0.2659604974429464], [-0.6693385873907269,0.7554393139905898,-1.7108087289854104,1.09991932687034,0.41821242261960917], [-0.3282503845126481,0.9257033715775067,-1.5235939065144024,1.1889128563024831,0.3368716615939142], [-0.5366461042406446,0.9240666515119298,-1.8135909127248846,1.3575511869067935,0.37505000938261085], [-0.39294848413804057,1.0124585311856609,-1.575086388417685,1.2467079941490156,0.40416560148386643], [-0.48738816985096806,0.8787565657383682,-1.810281961839409,1.3779890105636297,0.46019825317610896], [-0.36592476270170865,0.9132457437551841,-1.7089939281056865,1.1718159447241387,0.37244792446782193], [-0.40751855185193425,0.9541054981051484,-1.4879650472104635,1.1817057016562598,0.44919665548504983], [-0.29317450049605076,0.964443479319753,-1.656028808081747,1.2586398699632722,0.2526740377633023], [-0.4615968319838038,1.0243528061825597,-1.5054257187951623,1.351376529833804,0.42971544189574107], [-0.47988177538312266,1.0112860256804128,-1.5404476412782737,1.233668528335028,0.2028012939752821]) -2016-06-13 20:15:11 INFO JobScheduler:59 - Added jobs for time 9000 ms -2016-06-13 20:15:11 INFO TestInputStream:59 - Computing RDD for time 10000 ms -2016-06-13 20:15:11 INFO TestInputStream:59 - Created RDD 56 with Vector([-0.4889010904178065,1.0197902904388123,-1.6434083794608938,1.2273894895374848,0.29880056758004836], [-0.3690054860756016,0.9083136794567547,-1.5975355891184535,1.1937534562371308,0.3514272190683493], [-0.5737783702048262,1.2273536117223582,-1.4617006566984774,1.342244921629087,0.3377586288639175], [-0.48387316003666864,0.9673070026559238,-1.716725721249248,1.2214389230026372,0.37779842180031725], [-0.3777277713061831,0.866118273628028,-1.8487356572074127,1.1851350383431707,0.5215534466266145], [-0.6076926717533072,0.922880686625322,-1.7313060678714964,1.3632488420681186,0.3176699282425657], [-0.3802374789807806,0.8439055679755229,-1.6219613687183736,1.1809867291088374,0.4174630121979538], [-0.567466556581487,0.9292300369161763,-1.5542588287823018,1.3568593851021735,0.38458895772840795], [-0.6764387691659441,0.9816754055716941,-1.6652926928722982,1.3324217076474294,0.2844320904428872], [-0.584138124369403,0.9651833514289697,-1.7100579024456408,1.3025648636881102,0.3575320807936941], [-0.45548159931880944,1.0181220363355081,-1.6506653550843506,1.1304474598171783,0.4285968106743953], [-0.5370002084128622,0.927571736048293,-1.7228464440630433,1.2925613214036213,0.3169164091166503], [-0.47996613069291666,0.884366449915093,-1.7212355955272085,1.152597880658151,0.43962468554199663], [-0.35493034949860885,0.8984344817504613,-1.65218132155661,1.3068108975806394,0.354531357717713], [-0.43226118157508875,0.90725353631203,-1.7628881371748841,1.314870897760879,0.40214500076142384], [-0.4387259831220203,0.9621751931555681,-1.7268764217836796,1.1093705777029121,0.17867972251315242], [-0.44765902746822306,1.1186404105474657,-1.750942737318709,1.322673860044468,0.41609066577279374], [-0.40351581789921565,1.046340578817358,-1.7120920286690795,1.244893797129159,0.3833023298379642], [-0.3093624994668177,0.8356650811564297,-1.8642891852543533,1.217524168702485,0.2103617986137151], [-0.2990644638630094,0.9070655034040485,-1.6094469406493193,1.1592980925831848,0.43244854200109223], [-0.5596021456993305,0.9019708087164819,-1.5105356278212536,1.2598551179472526,0.3428536586628008], [-0.583393764986965,0.9748568871770861,-1.5302538126845806,1.301546791122464,0.21927558650218368], [-0.49772324825286274,0.9716299176892979,-1.6169165069063722,1.3531471880552017,0.25858373282508224], [-0.43106771684572104,1.141003137647425,-1.7578104686857339,1.1835903210552496,0.5254045668739125], [-0.5997306192424074,1.0040726704610177,-1.8227253165238155,1.2317004811683654,0.46876771655575034], [-0.38252412898165267,0.6654527000305275,-1.6468024292953505,1.246225339488523,0.3090133585357682], [-0.44821229943324725,1.0038256342470713,-1.6968551076177416,1.4529627079929077,0.3454589917432014], [-0.43535992691090786,1.0110816584854143,-1.7453606983094134,1.248210478232135,0.2891585839680707], [-0.3652348315036368,0.8958964249587966,-1.9606863598829045,1.3389257198065394,0.2618900608154475], [-0.49376528406043635,1.1421599774098976,-1.7157624036331927,1.391772429187865,0.2876879628208894], [-0.4515063147230089,0.9428403433366573,-1.76394465423704,1.203639940182186,0.5058415243772909], [-0.566229150629525,1.0128616873896932,-1.6880365432069786,1.2877554523210368,0.1629872361106314], [-0.47658344463796903,1.0862736976313303,-1.7934284966158631,1.2272771650263852,0.41680211738171946], [-0.3618808351944353,0.9417137306142976,-1.8600076455984111,1.2206352320840592,0.3461749973804146], [-0.6214757024643818,0.9952948941676851,-1.62097329834717,1.3958184351557885,0.46499512939060494], [-0.35996348236796494,0.9954644862470238,-1.7933270864489543,1.4005533772243737,0.5639356774206108], [-0.6006970016720174,0.9686450943470077,-1.7310975948083858,1.2398543489295077,0.4483279499267394], [-0.4592797597414643,0.9631529442377594,-1.6797024055855931,1.4289800788354718,0.28667160760155463], [-0.32700881076504584,0.8147754680205297,-1.763026614596352,1.3194735072019403,0.36816545181467747], [-0.4305345983795718,1.131285559002078,-1.7583813300937976,1.363545141597296,0.2601165464857354], [-0.448497878644767,1.0701920363594992,-1.6308801358271674,1.2674713459930484,0.49959128317905865], [-0.3686293381026533,1.025599917053419,-1.5433629968114364,1.2801178859900646,0.4174115399252367], [-0.574113693976958,0.9757976338077295,-1.6388778822719372,1.2759164798546365,0.5542142157229283], [-0.4357071400660946,0.8338147376622728,-1.8285888680646112,1.271233025944097,0.270178918164786], [-0.47158188300536635,1.0122183541160776,-1.6010520252577112,1.5447804629514494,0.3533722934153737], [-0.5307926544952695,0.9107879044274507,-1.4490342775396574,1.4031526476841614,0.3809780658985748], [-0.4953668915084064,0.9298227962550192,-1.5651812648587027,1.11568589286655,0.4774983034589586], [-0.7383552095915331,0.7626241115292196,-1.7008182872047266,1.2502877442685654,0.43135198632241045], [-0.539114438923603,1.0489102632469824,-1.6736818095705865,1.2768121659352345,0.2213641562753643], [-0.46002266280222004,1.0294295650038185,-1.6033579773810744,1.407522155806443,0.5223722922756932]) -2016-06-13 20:15:11 INFO JobScheduler:59 - Added jobs for time 10000 ms -2016-06-13 20:15:11 INFO JobGenerator:59 - Checkpointing graph for time 1000 ms -2016-06-13 20:15:11 INFO DStreamGraph:59 - Updating checkpoint data for time 1000 ms -2016-06-13 20:15:11 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:11 INFO DStreamGraph:59 - Updated checkpoint data for time 1000 ms -2016-06-13 20:15:11 INFO JobGenerator:59 - Checkpointing graph for time 2000 ms -2016-06-13 20:15:11 INFO DStreamGraph:59 - Updating checkpoint data for time 2000 ms -2016-06-13 20:15:11 INFO DStreamGraph:59 - Updated checkpoint data for time 2000 ms -2016-06-13 20:15:11 INFO CheckpointWriter:59 - Saving checkpoint for time 1000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-1000' -2016-06-13 20:15:11 INFO JobGenerator:59 - Checkpointing graph for time 3000 ms -2016-06-13 20:15:11 INFO DStreamGraph:59 - Updating checkpoint data for time 3000 ms -2016-06-13 20:15:11 INFO DStreamGraph:59 - Updated checkpoint data for time 3000 ms -2016-06-13 20:15:11 INFO JobGenerator:59 - Checkpointing graph for time 4000 ms -2016-06-13 20:15:11 INFO DStreamGraph:59 - Updating checkpoint data for time 4000 ms -2016-06-13 20:15:11 INFO DStreamGraph:59 - Updated checkpoint data for time 4000 ms -2016-06-13 20:15:11 INFO JobGenerator:59 - Checkpointing graph for time 5000 ms -2016-06-13 20:15:11 INFO DStreamGraph:59 - Updating checkpoint data for time 5000 ms -2016-06-13 20:15:11 INFO DStreamGraph:59 - Updated checkpoint data for time 5000 ms -2016-06-13 20:15:11 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:11 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=0, maxMem=766075207 -2016-06-13 20:15:11 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:15:11 INFO JobGenerator:59 - Checkpointing graph for time 6000 ms -2016-06-13 20:15:11 INFO DStreamGraph:59 - Updating checkpoint data for time 6000 ms -2016-06-13 20:15:11 INFO DStreamGraph:59 - Updated checkpoint data for time 6000 ms -2016-06-13 20:15:11 INFO MemoryStore:59 - ensureFreeSpace(2428) called with curMem=4744, maxMem=766075207 -2016-06-13 20:15:11 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:15:11 INFO JobGenerator:59 - Checkpointing graph for time 7000 ms -2016-06-13 20:15:11 INFO DStreamGraph:59 - Updating checkpoint data for time 7000 ms -2016-06-13 20:15:11 INFO DStreamGraph:59 - Updated checkpoint data for time 7000 ms -2016-06-13 20:15:11 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:50965 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:15:11 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:11 INFO JobGenerator:59 - Checkpointing graph for time 8000 ms -2016-06-13 20:15:11 INFO DStreamGraph:59 - Updating checkpoint data for time 8000 ms -2016-06-13 20:15:11 INFO DStreamGraph:59 - Updated checkpoint data for time 8000 ms -2016-06-13 20:15:11 INFO JobGenerator:59 - Checkpointing graph for time 9000 ms -2016-06-13 20:15:11 INFO DStreamGraph:59 - Updating checkpoint data for time 9000 ms -2016-06-13 20:15:11 INFO DStreamGraph:59 - Updated checkpoint data for time 9000 ms -2016-06-13 20:15:11 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:11 INFO JobGenerator:59 - Checkpointing graph for time 10000 ms -2016-06-13 20:15:11 INFO DStreamGraph:59 - Updating checkpoint data for time 10000 ms -2016-06-13 20:15:11 INFO DStreamGraph:59 - Updated checkpoint data for time 10000 ms -2016-06-13 20:15:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[10] at apply at Transformer.scala:22) -2016-06-13 20:15:11 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 20:15:11 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:15:11 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:15:11 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 20:15:11 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 20:15:11 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:12 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:12 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 20:15:12 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 20:15:12 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:12 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1159 bytes result sent to driver -2016-06-13 20:15:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 231 ms on localhost (1/2) -2016-06-13 20:15:12 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:12 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:12 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:12 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:12 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1159 bytes result sent to driver -2016-06-13 20:15:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 398 ms on localhost (2/2) -2016-06-13 20:15:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 20:15:12 INFO DAGScheduler:59 - ShuffleMapStage 0 (apply at Transformer.scala:22) finished in 0.498 s -2016-06-13 20:15:12 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:15:12 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:15:12 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 20:15:12 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:15:12 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 20:15:12 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:12 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[17] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:15:12 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=7172, maxMem=766075207 -2016-06-13 20:15:12 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:15:12 INFO MemoryStore:59 - ensureFreeSpace(2657) called with curMem=12436, maxMem=766075207 -2016-06-13 20:15:12 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 20:15:12 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:50965 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:15:12 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[17] at apply at Transformer.scala:22) -2016-06-13 20:15:12 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 20:15:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:12 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:12 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 20:15:12 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:15:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-06-13 20:15:12 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-06-13 20:15:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 49 ms on localhost (1/2) -2016-06-13 20:15:12 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:12 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 20:15:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:15:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-06-13 20:15:12 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1498 bytes result sent to driver -2016-06-13 20:15:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 122 ms on localhost (2/2) -2016-06-13 20:15:12 INFO DAGScheduler:59 - ResultStage 1 (apply at Transformer.scala:22) finished in 0.123 s -2016-06-13 20:15:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 20:15:12 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:12 INFO DAGScheduler:59 - Job 0 finished: apply at Transformer.scala:22, took 1.293803 s -2016-06-13 20:15:12 INFO CheckpointWriter:59 - Checkpoint for time 1000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-1000', took 27280 bytes and 978 ms -2016-06-13 20:15:12 INFO CheckpointWriter:59 - Saving checkpoint for time 2000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-2000' -2016-06-13 20:15:12 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 50.0 and centroid: [-0.4843392480139548,0.9734525763849463,-1.7046336323611109,1.2778093795456185,0.3691650550997036] -2016-06-13 20:15:12 INFO JobScheduler:59 - Finished job streaming job 1000 ms.0 from job set of time 1000 ms -2016-06-13 20:15:12 INFO JobScheduler:59 - Starting job streaming job 1000 ms.1 from job set of time 1000 ms -2016-06-13 20:15:12 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:12 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:15:12 INFO DAGScheduler:59 - Registering RDD 3 (apply at Transformer.scala:22) -2016-06-13 20:15:12 INFO DAGScheduler:59 - Got job 1 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:15:12 INFO DAGScheduler:59 - Final stage: ResultStage 3(apply at Transformer.scala:22) -2016-06-13 20:15:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 2) -2016-06-13 20:15:12 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 2) -2016-06-13 20:15:12 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (UnionRDD[3] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:15:12 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=15093, maxMem=766075207 -2016-06-13 20:15:12 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:15:12 INFO MemoryStore:59 - ensureFreeSpace(2000) called with curMem=18373, maxMem=766075207 -2016-06-13 20:15:12 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2000.0 B, free 730.6 MB) -2016-06-13 20:15:12 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:50965 (size: 2000.0 B, free: 730.6 MB) -2016-06-13 20:15:12 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:12 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 2 (UnionRDD[3] at apply at Transformer.scala:22) -2016-06-13 20:15:12 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-06-13 20:15:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:15:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:15:12 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 20:15:12 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:12 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 20:15:12 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1159 bytes result sent to driver -2016-06-13 20:15:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:15:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 34 ms on localhost (1/3) -2016-06-13 20:15:12 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-06-13 20:15:12 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1159 bytes result sent to driver -2016-06-13 20:15:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 47 ms on localhost (2/3) -2016-06-13 20:15:12 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 6). 1159 bytes result sent to driver -2016-06-13 20:15:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 6) in 22 ms on localhost (3/3) -2016-06-13 20:15:12 INFO DAGScheduler:59 - ShuffleMapStage 2 (apply at Transformer.scala:22) finished in 0.060 s -2016-06-13 20:15:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 20:15:12 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:15:12 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:15:12 INFO DAGScheduler:59 - waiting: Set(ResultStage 3) -2016-06-13 20:15:12 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:15:12 INFO DAGScheduler:59 - Missing parents for ResultStage 3: List() -2016-06-13 20:15:12 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[62] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:15:12 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=20373, maxMem=766075207 -2016-06-13 20:15:12 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:15:12 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:12 INFO MemoryStore:59 - ensureFreeSpace(1799) called with curMem=23613, maxMem=766075207 -2016-06-13 20:15:12 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1799.0 B, free 730.6 MB) -2016-06-13 20:15:12 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:50965 (size: 1799.0 B, free: 730.6 MB) -2016-06-13 20:15:12 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[62] at apply at Transformer.scala:22) -2016-06-13 20:15:12 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 20:15:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:12 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 8) -2016-06-13 20:15:12 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:15:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:15:12 INFO CheckpointWriter:59 - Checkpoint for time 2000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-2000', took 27282 bytes and 200 ms -2016-06-13 20:15:12 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 7) -2016-06-13 20:15:12 INFO CheckpointWriter:59 - Saving checkpoint for time 3000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-3000' -2016-06-13 20:15:12 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 8). 1172 bytes result sent to driver -2016-06-13 20:15:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 8) in 27 ms on localhost (1/2) -2016-06-13 20:15:12 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:15:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:12 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:50965 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:15:12 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:15:12 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 7). 1180 bytes result sent to driver -2016-06-13 20:15:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 7) in 40 ms on localhost (2/2) -2016-06-13 20:15:12 INFO DAGScheduler:59 - ResultStage 3 (apply at Transformer.scala:22) finished in 0.041 s -2016-06-13 20:15:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 20:15:12 INFO DAGScheduler:59 - Job 1 finished: apply at Transformer.scala:22, took 0.176706 s -2016-06-13 20:15:12 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 20:15:12 INFO JobScheduler:59 - Finished job streaming job 1000 ms.1 from job set of time 1000 ms -2016-06-13 20:15:12 INFO JobScheduler:59 - Total delay: 1465820111.796 s for time 1000 ms (execution: 1.807 s) -2016-06-13 20:15:12 INFO JobScheduler:59 - Starting job streaming job 2000 ms.0 from job set of time 2000 ms -2016-06-13 20:15:12 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:50965 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:15:12 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 20:15:12 INFO JobGenerator:59 - Checkpointing graph for time 1000 ms -2016-06-13 20:15:12 INFO DStreamGraph:59 - Updating checkpoint data for time 1000 ms -2016-06-13 20:15:12 INFO DStreamGraph:59 - Updated checkpoint data for time 1000 ms -2016-06-13 20:15:12 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-06-13 20:15:12 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:15:12 INFO DAGScheduler:59 - Registering RDD 63 (apply at Transformer.scala:22) -2016-06-13 20:15:12 INFO DAGScheduler:59 - Got job 2 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:15:12 INFO DAGScheduler:59 - Final stage: ResultStage 5(apply at Transformer.scala:22) -2016-06-13 20:15:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-06-13 20:15:12 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-06-13 20:15:12 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[63] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:15:12 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=10319, maxMem=766075207 -2016-06-13 20:15:12 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:15:12 INFO MemoryStore:59 - ensureFreeSpace(2471) called with curMem=15063, maxMem=766075207 -2016-06-13 20:15:12 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:15:12 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:15:12 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:50965 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:15:12 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[63] at apply at Transformer.scala:22) -2016-06-13 20:15:12 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 20:15:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:15:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 10, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:15:12 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 10) -2016-06-13 20:15:12 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 9) -2016-06-13 20:15:12 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 9). 1159 bytes result sent to driver -2016-06-13 20:15:12 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 10). 1159 bytes result sent to driver -2016-06-13 20:15:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 9) in 25 ms on localhost (1/2) -2016-06-13 20:15:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 10) in 25 ms on localhost (2/2) -2016-06-13 20:15:12 INFO DAGScheduler:59 - ShuffleMapStage 4 (apply at Transformer.scala:22) finished in 0.026 s -2016-06-13 20:15:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 20:15:12 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:15:12 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:15:12 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-06-13 20:15:12 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:15:12 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-06-13 20:15:12 INFO DAGScheduler:59 - Submitting ResultStage 5 (ShuffledRDD[64] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:15:12 INFO CheckpointWriter:59 - Checkpoint for time 3000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-3000', took 27282 bytes and 108 ms -2016-06-13 20:15:12 INFO CheckpointWriter:59 - Saving checkpoint for time 4000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-4000' -2016-06-13 20:15:12 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=17534, maxMem=766075207 -2016-06-13 20:15:12 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:15:12 INFO MemoryStore:59 - ensureFreeSpace(2692) called with curMem=22798, maxMem=766075207 -2016-06-13 20:15:12 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 20:15:12 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:50965 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:15:12 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (ShuffledRDD[64] at apply at Transformer.scala:22) -2016-06-13 20:15:12 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 20:15:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 12, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:12 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 12) -2016-06-13 20:15:12 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 11) -2016-06-13 20:15:12 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:15:12 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:15:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:12 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:15:12 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 12). 1161 bytes result sent to driver -2016-06-13 20:15:12 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:15:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 12) in 13 ms on localhost (1/2) -2016-06-13 20:15:12 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 11). 1498 bytes result sent to driver -2016-06-13 20:15:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 11) in 23 ms on localhost (2/2) -2016-06-13 20:15:12 INFO DAGScheduler:59 - ResultStage 5 (apply at Transformer.scala:22) finished in 0.023 s -2016-06-13 20:15:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 20:15:12 INFO DAGScheduler:59 - Job 2 finished: apply at Transformer.scala:22, took 0.079401 s -2016-06-13 20:15:12 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 100.0 and centroid: [-0.4738959687352186,0.969729236815007,-1.6823805644329948,1.2679893451777908,0.37589591355279117] -2016-06-13 20:15:12 INFO JobScheduler:59 - Finished job streaming job 2000 ms.0 from job set of time 2000 ms -2016-06-13 20:15:12 INFO JobScheduler:59 - Starting job streaming job 2000 ms.1 from job set of time 2000 ms -2016-06-13 20:15:12 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:15:12 INFO DAGScheduler:59 - Registering RDD 9 (apply at Transformer.scala:22) -2016-06-13 20:15:12 INFO DAGScheduler:59 - Got job 3 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:15:12 INFO DAGScheduler:59 - Final stage: ResultStage 7(apply at Transformer.scala:22) -2016-06-13 20:15:12 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-06-13 20:15:12 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-06-13 20:15:12 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (UnionRDD[9] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:15:12 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=25490, maxMem=766075207 -2016-06-13 20:15:12 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:15:12 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=28770, maxMem=766075207 -2016-06-13 20:15:12 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.6 MB) -2016-06-13 20:15:12 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:50965 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:15:12 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:12 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 6 (UnionRDD[9] at apply at Transformer.scala:22) -2016-06-13 20:15:12 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 3 tasks -2016-06-13 20:15:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:15:12 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 14, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:15:12 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 13) -2016-06-13 20:15:12 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 14) -2016-06-13 20:15:12 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:15:12 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 14). 1159 bytes result sent to driver -2016-06-13 20:15:12 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 15, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:15:12 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 15) -2016-06-13 20:15:12 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 14) in 28 ms on localhost (1/3) -2016-06-13 20:15:12 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 13). 1159 bytes result sent to driver -2016-06-13 20:15:12 INFO CheckpointWriter:59 - Checkpoint for time 4000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-4000', took 27282 bytes and 97 ms -2016-06-13 20:15:12 INFO CheckpointWriter:59 - Saving checkpoint for time 5000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-5000' -2016-06-13 20:15:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 13) in 35 ms on localhost (2/3) -2016-06-13 20:15:12 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 15). 1159 bytes result sent to driver -2016-06-13 20:15:12 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 15) in 24 ms on localhost (3/3) -2016-06-13 20:15:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 20:15:12 INFO DAGScheduler:59 - ShuffleMapStage 6 (apply at Transformer.scala:22) finished in 0.051 s -2016-06-13 20:15:12 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:15:12 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:15:12 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-06-13 20:15:12 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:15:12 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-06-13 20:15:12 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[65] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:15:12 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=30779, maxMem=766075207 -2016-06-13 20:15:12 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:15:12 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:15:12 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=34019, maxMem=766075207 -2016-06-13 20:15:12 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.6 MB) -2016-06-13 20:15:12 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:50965 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:15:12 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:12 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[65] at apply at Transformer.scala:22) -2016-06-13 20:15:12 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:13 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 16) -2016-06-13 20:15:13 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 17) -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:13 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 17). 1172 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 17) in 13 ms on localhost (1/2) -2016-06-13 20:15:13 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 16). 1180 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 16) in 19 ms on localhost (2/2) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 20:15:13 INFO DAGScheduler:59 - ResultStage 7 (apply at Transformer.scala:22) finished in 0.020 s -2016-06-13 20:15:13 INFO DAGScheduler:59 - Job 3 finished: apply at Transformer.scala:22, took 0.109518 s -2016-06-13 20:15:13 INFO JobScheduler:59 - Finished job streaming job 2000 ms.1 from job set of time 2000 ms -2016-06-13 20:15:13 INFO JobScheduler:59 - Total delay: 1465820111.025 s for time 2000 ms (execution: 0.226 s) -2016-06-13 20:15:13 INFO JobScheduler:59 - Starting job streaming job 3000 ms.0 from job set of time 3000 ms -2016-06-13 20:15:13 INFO ParallelCollectionRDD:59 - Removing RDD 0 from persistence list -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 0 -2016-06-13 20:15:13 INFO MapPartitionsRDD:59 - Removing RDD 5 from persistence list -2016-06-13 20:15:13 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:15:13 INFO ShuffledRDD:59 - Removing RDD 4 from persistence list -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 5 -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 4 -2016-06-13 20:15:13 INFO UnionRDD:59 - Removing RDD 3 from persistence list -2016-06-13 20:15:13 INFO MapPartitionsRDD:59 - Removing RDD 1 from persistence list -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 3 -2016-06-13 20:15:13 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:15:13 INFO JobGenerator:59 - Checkpointing graph for time 2000 ms -2016-06-13 20:15:13 INFO DStreamGraph:59 - Updating checkpoint data for time 2000 ms -2016-06-13 20:15:13 INFO DStreamGraph:59 - Updated checkpoint data for time 2000 ms -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 1 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Registering RDD 66 (apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Got job 4 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:15:13 INFO DAGScheduler:59 - Final stage: ResultStage 9(apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 8) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 8) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 8 (MapPartitionsRDD[66] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=35821, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(2475) called with curMem=40565, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:50965 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:15:13 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 8 (MapPartitionsRDD[66] at apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 18, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 19, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:15:13 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 18) -2016-06-13 20:15:13 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 19) -2016-06-13 20:15:13 INFO CheckpointWriter:59 - Checkpoint for time 5000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-5000', took 27282 bytes and 121 ms -2016-06-13 20:15:13 INFO CheckpointWriter:59 - Saving checkpoint for time 6000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-6000' -2016-06-13 20:15:13 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:15:13 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 19). 1159 bytes result sent to driver -2016-06-13 20:15:13 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 18). 1159 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 19) in 21 ms on localhost (1/2) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 18) in 23 ms on localhost (2/2) -2016-06-13 20:15:13 INFO DAGScheduler:59 - ShuffleMapStage 8 (apply at Transformer.scala:22) finished in 0.023 s -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 20:15:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:15:13 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:15:13 INFO DAGScheduler:59 - waiting: Set(ResultStage 9) -2016-06-13 20:15:13 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:15:13 INFO DAGScheduler:59 - Missing parents for ResultStage 9: List() -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting ResultStage 9 (ShuffledRDD[67] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=43040, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=48304, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:50965 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:15:13 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (ShuffledRDD[67] at apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 21, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:13 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 20) -2016-06-13 20:15:13 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 21) -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:13 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 20). 1498 bytes result sent to driver -2016-06-13 20:15:13 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 21). 1161 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 20) in 19 ms on localhost (1/2) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 21) in 19 ms on localhost (2/2) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 20:15:13 INFO DAGScheduler:59 - ResultStage 9 (apply at Transformer.scala:22) finished in 0.020 s -2016-06-13 20:15:13 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Job 4 finished: apply at Transformer.scala:22, took 0.095139 s -2016-06-13 20:15:13 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 150.0 and centroid: [-0.4741102089605591,0.9622750167428704,-1.680286076986711,1.2654376587211662,0.37713329463498424] -2016-06-13 20:15:13 INFO JobScheduler:59 - Finished job streaming job 3000 ms.0 from job set of time 3000 ms -2016-06-13 20:15:13 INFO JobScheduler:59 - Starting job streaming job 3000 ms.1 from job set of time 3000 ms -2016-06-13 20:15:13 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Registering RDD 16 (apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Got job 5 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:15:13 INFO DAGScheduler:59 - Final stage: ResultStage 11(apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 10) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 10) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 10 (UnionRDD[16] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=51000, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(2004) called with curMem=54280, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2004.0 B, free 730.5 MB) -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:50965 (size: 2004.0 B, free: 730.6 MB) -2016-06-13 20:15:13 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 10 (UnionRDD[16] at apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 3 tasks -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 22, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 23, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:15:13 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 23) -2016-06-13 20:15:13 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 22) -2016-06-13 20:15:13 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 23). 1159 bytes result sent to driver -2016-06-13 20:15:13 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 22). 1159 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 10.0 (TID 24, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:15:13 INFO Executor:59 - Running task 2.0 in stage 10.0 (TID 24) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 23) in 23 ms on localhost (1/3) -2016-06-13 20:15:13 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 22) in 25 ms on localhost (2/3) -2016-06-13 20:15:13 INFO Executor:59 - Finished task 2.0 in stage 10.0 (TID 24). 1159 bytes result sent to driver -2016-06-13 20:15:13 INFO CheckpointWriter:59 - Checkpoint for time 6000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-6000', took 27280 bytes and 118 ms -2016-06-13 20:15:13 INFO CheckpointWriter:59 - Saving checkpoint for time 7000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-7000' -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 10.0 (TID 24) in 14 ms on localhost (3/3) -2016-06-13 20:15:13 INFO DAGScheduler:59 - ShuffleMapStage 10 (apply at Transformer.scala:22) finished in 0.038 s -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 20:15:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:15:13 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:15:13 INFO DAGScheduler:59 - waiting: Set(ResultStage 11) -2016-06-13 20:15:13 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:15:13 INFO DAGScheduler:59 - Missing parents for ResultStage 11: List() -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[68] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=56284, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=59524, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.5 MB) -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:50965 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:15:13 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[68] at apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 25, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 26, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:13 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 26) -2016-06-13 20:15:13 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 25) -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:13 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 26). 1172 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 26) in 9 ms on localhost (1/2) -2016-06-13 20:15:13 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 25). 1180 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 25) in 17 ms on localhost (2/2) -2016-06-13 20:15:13 INFO DAGScheduler:59 - ResultStage 11 (apply at Transformer.scala:22) finished in 0.018 s -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 20:15:13 INFO DAGScheduler:59 - Job 5 finished: apply at Transformer.scala:22, took 0.084827 s -2016-06-13 20:15:13 INFO JobScheduler:59 - Finished job streaming job 3000 ms.1 from job set of time 3000 ms -2016-06-13 20:15:13 INFO JobScheduler:59 - Total delay: 1465820110.241 s for time 3000 ms (execution: 0.216 s) -2016-06-13 20:15:13 INFO JobScheduler:59 - Starting job streaming job 4000 ms.0 from job set of time 4000 ms -2016-06-13 20:15:13 INFO ParallelCollectionRDD:59 - Removing RDD 6 from persistence list -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 6 -2016-06-13 20:15:13 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-06-13 20:15:13 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:15:13 INFO ShuffledRDD:59 - Removing RDD 11 from persistence list -2016-06-13 20:15:13 INFO UnionRDD:59 - Removing RDD 9 from persistence list -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 11 -2016-06-13 20:15:13 INFO MapPartitionsRDD:59 - Removing RDD 7 from persistence list -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 9 -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 20:15:13 INFO JobGenerator:59 - Checkpointing graph for time 3000 ms -2016-06-13 20:15:13 INFO DStreamGraph:59 - Updating checkpoint data for time 3000 ms -2016-06-13 20:15:13 INFO DStreamGraph:59 - Updated checkpoint data for time 3000 ms -2016-06-13 20:15:13 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 7 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Registering RDD 69 (apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Got job 6 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:15:13 INFO DAGScheduler:59 - Final stage: ResultStage 13(apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (MapPartitionsRDD[69] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=61326, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(2484) called with curMem=66070, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:50965 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:15:13 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (MapPartitionsRDD[69] at apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 27, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 28, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:15:13 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 27) -2016-06-13 20:15:13 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 28) -2016-06-13 20:15:13 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:15:13 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 27). 1159 bytes result sent to driver -2016-06-13 20:15:13 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 28). 1159 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 27) in 30 ms on localhost (1/2) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 28) in 28 ms on localhost (2/2) -2016-06-13 20:15:13 INFO DAGScheduler:59 - ShuffleMapStage 12 (apply at Transformer.scala:22) finished in 0.031 s -2016-06-13 20:15:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:15:13 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 20:15:13 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-06-13 20:15:13 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:15:13 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting ResultStage 13 (ShuffledRDD[70] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=68554, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:15:13 INFO CheckpointWriter:59 - Checkpoint for time 7000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-7000', took 27280 bytes and 113 ms -2016-06-13 20:15:13 INFO CheckpointWriter:59 - Saving checkpoint for time 8000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-8000' -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(2698) called with curMem=73818, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:50965 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:15:13 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (ShuffledRDD[70] at apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 29, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 30, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:13 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 30) -2016-06-13 20:15:13 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 29) -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:15:13 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 30). 1161 bytes result sent to driver -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 30) in 13 ms on localhost (1/2) -2016-06-13 20:15:13 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 29). 1498 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 29) in 19 ms on localhost (2/2) -2016-06-13 20:15:13 INFO DAGScheduler:59 - ResultStage 13 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 20:15:13 INFO DAGScheduler:59 - Job 6 finished: apply at Transformer.scala:22, took 0.087838 s -2016-06-13 20:15:13 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 200.0 and centroid: [-0.4768735011530239,0.9606386018112957,-1.673148729841545,1.2634974127814163,0.38189605546947214] -2016-06-13 20:15:13 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:15:13 INFO JobScheduler:59 - Finished job streaming job 4000 ms.0 from job set of time 4000 ms -2016-06-13 20:15:13 INFO JobScheduler:59 - Starting job streaming job 4000 ms.1 from job set of time 4000 ms -2016-06-13 20:15:13 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Registering RDD 23 (apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Got job 7 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:15:13 INFO DAGScheduler:59 - Final stage: ResultStage 15(apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 14) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 14) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 14 (UnionRDD[23] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=76516, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=79796, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:50965 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:15:13 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 14 (UnionRDD[23] at apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 3 tasks -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 31, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 32, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:15:13 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 32) -2016-06-13 20:15:13 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 31) -2016-06-13 20:15:13 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 31). 1159 bytes result sent to driver -2016-06-13 20:15:13 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 33, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:15:13 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 32). 1159 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 31) in 23 ms on localhost (1/3) -2016-06-13 20:15:13 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 33) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 32) in 23 ms on localhost (2/3) -2016-06-13 20:15:13 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 33). 1159 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 33) in 15 ms on localhost (3/3) -2016-06-13 20:15:13 INFO DAGScheduler:59 - ShuffleMapStage 14 (apply at Transformer.scala:22) finished in 0.037 s -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 20:15:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:15:13 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:15:13 INFO DAGScheduler:59 - waiting: Set(ResultStage 15) -2016-06-13 20:15:13 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:15:13 INFO DAGScheduler:59 - Missing parents for ResultStage 15: List() -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[71] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=81805, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(1801) called with curMem=85045, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1801.0 B, free 730.5 MB) -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:50965 (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:15:13 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[71] at apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 34, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 35, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:13 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 34) -2016-06-13 20:15:13 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 35) -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:13 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 35). 1172 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 35) in 9 ms on localhost (1/2) -2016-06-13 20:15:13 INFO CheckpointWriter:59 - Checkpoint for time 8000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-8000', took 27282 bytes and 121 ms -2016-06-13 20:15:13 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 34). 1180 bytes result sent to driver -2016-06-13 20:15:13 INFO CheckpointWriter:59 - Saving checkpoint for time 9000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-9000' -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 34) in 14 ms on localhost (2/2) -2016-06-13 20:15:13 INFO DAGScheduler:59 - ResultStage 15 (apply at Transformer.scala:22) finished in 0.014 s -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 20:15:13 INFO DAGScheduler:59 - Job 7 finished: apply at Transformer.scala:22, took 0.086032 s -2016-06-13 20:15:13 INFO JobScheduler:59 - Finished job streaming job 4000 ms.1 from job set of time 4000 ms -2016-06-13 20:15:13 INFO JobScheduler:59 - Total delay: 1465820109.443 s for time 4000 ms (execution: 0.202 s) -2016-06-13 20:15:13 INFO JobScheduler:59 - Starting job streaming job 5000 ms.0 from job set of time 5000 ms -2016-06-13 20:15:13 INFO ParallelCollectionRDD:59 - Removing RDD 13 from persistence list -2016-06-13 20:15:13 INFO MapPartitionsRDD:59 - Removing RDD 19 from persistence list -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 13 -2016-06-13 20:15:13 INFO ShuffledRDD:59 - Removing RDD 18 from persistence list -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 19 -2016-06-13 20:15:13 INFO UnionRDD:59 - Removing RDD 16 from persistence list -2016-06-13 20:15:13 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 18 -2016-06-13 20:15:13 INFO MapPartitionsRDD:59 - Removing RDD 14 from persistence list -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 16 -2016-06-13 20:15:13 INFO JobGenerator:59 - Checkpointing graph for time 4000 ms -2016-06-13 20:15:13 INFO DStreamGraph:59 - Updating checkpoint data for time 4000 ms -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 14 -2016-06-13 20:15:13 INFO DStreamGraph:59 - Updated checkpoint data for time 4000 ms -2016-06-13 20:15:13 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Registering RDD 72 (apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Got job 8 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:15:13 INFO DAGScheduler:59 - Final stage: ResultStage 17(apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 16) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 16) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 16 (MapPartitionsRDD[72] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=86846, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(2476) called with curMem=91590, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:50965 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:15:13 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 16 (MapPartitionsRDD[72] at apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 36, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 37, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:15:13 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 36) -2016-06-13 20:15:13 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 37) -2016-06-13 20:15:13 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 37). 1159 bytes result sent to driver -2016-06-13 20:15:13 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 37) in 22 ms on localhost (1/2) -2016-06-13 20:15:13 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 36). 1159 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 36) in 33 ms on localhost (2/2) -2016-06-13 20:15:13 INFO DAGScheduler:59 - ShuffleMapStage 16 (apply at Transformer.scala:22) finished in 0.034 s -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 20:15:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:15:13 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:15:13 INFO DAGScheduler:59 - waiting: Set(ResultStage 17) -2016-06-13 20:15:13 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:15:13 INFO DAGScheduler:59 - Missing parents for ResultStage 17: List() -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting ResultStage 17 (ShuffledRDD[73] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=94066, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=99330, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:50965 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:15:13 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (ShuffledRDD[73] at apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 38, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 39, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:13 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 38) -2016-06-13 20:15:13 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 39) -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:13 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 39). 1161 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 39) in 11 ms on localhost (1/2) -2016-06-13 20:15:13 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 38). 1498 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 38) in 18 ms on localhost (2/2) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 20:15:13 INFO DAGScheduler:59 - ResultStage 17 (apply at Transformer.scala:22) finished in 0.023 s -2016-06-13 20:15:13 INFO DAGScheduler:59 - Job 8 finished: apply at Transformer.scala:22, took 0.089873 s -2016-06-13 20:15:13 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 250.0 and centroid: [-0.4770613340912709,0.9657053185238176,-1.6732261982319274,1.26651642628913,0.38386658573056776] -2016-06-13 20:15:13 INFO JobScheduler:59 - Finished job streaming job 5000 ms.0 from job set of time 5000 ms -2016-06-13 20:15:13 INFO JobScheduler:59 - Starting job streaming job 5000 ms.1 from job set of time 5000 ms -2016-06-13 20:15:13 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:15:13 INFO CheckpointWriter:59 - Checkpoint for time 9000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-9000', took 27282 bytes and 118 ms -2016-06-13 20:15:13 INFO CheckpointWriter:59 - Saving checkpoint for time 10000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-10000' -2016-06-13 20:15:13 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Registering RDD 29 (apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Got job 9 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:15:13 INFO DAGScheduler:59 - Final stage: ResultStage 19(apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 18) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 18) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 18 (UnionRDD[29] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=102024, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(2010) called with curMem=105304, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2010.0 B, free 730.5 MB) -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:50965 (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:15:13 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:50965 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 18 (UnionRDD[29] at apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 3 tasks -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 40, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 41, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:15:13 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 41) -2016-06-13 20:15:13 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 40) -2016-06-13 20:15:13 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:50965 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 20:15:13 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 40). 1159 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 42, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 40) in 24 ms on localhost (1/3) -2016-06-13 20:15:13 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 42) -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned shuffle 8 -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:50965 in memory (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:15:13 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 42). 1159 bytes result sent to driver -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 42) in 21 ms on localhost (2/3) -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:50965 in memory (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 20:15:13 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 41). 1159 bytes result sent to driver -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:50965 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 41) in 57 ms on localhost (3/3) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 20:15:13 INFO DAGScheduler:59 - ShuffleMapStage 18 (apply at Transformer.scala:22) finished in 0.060 s -2016-06-13 20:15:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:15:13 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:15:13 INFO DAGScheduler:59 - waiting: Set(ResultStage 19) -2016-06-13 20:15:13 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:15:13 INFO DAGScheduler:59 - Missing parents for ResultStage 19: List() -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[74] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=73844, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:15:13 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=69856, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:50965 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:50965 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 20:15:13 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[74] at apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned shuffle 6 -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 43, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 44, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:13 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 44) -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:50965 in memory (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:15:13 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 43) -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:13 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 44). 1172 bytes result sent to driver -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:50965 in memory (size: 2004.0 B, free: 730.6 MB) -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned shuffle 5 -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 44) in 25 ms on localhost (1/2) -2016-06-13 20:15:13 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 43). 1180 bytes result sent to driver -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:50965 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 43) in 30 ms on localhost (2/2) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 20:15:13 INFO DAGScheduler:59 - ResultStage 19 (apply at Transformer.scala:22) finished in 0.032 s -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Job 9 finished: apply at Transformer.scala:22, took 0.141749 s -2016-06-13 20:15:13 INFO JobScheduler:59 - Finished job streaming job 5000 ms.1 from job set of time 5000 ms -2016-06-13 20:15:13 INFO JobScheduler:59 - Total delay: 1465820108.703 s for time 5000 ms (execution: 0.260 s) -2016-06-13 20:15:13 INFO JobScheduler:59 - Starting job streaming job 6000 ms.0 from job set of time 6000 ms -2016-06-13 20:15:13 INFO ParallelCollectionRDD:59 - Removing RDD 20 from persistence list -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:50965 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 20 -2016-06-13 20:15:13 INFO MapPartitionsRDD:59 - Removing RDD 25 from persistence list -2016-06-13 20:15:13 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 25 -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned shuffle 4 -2016-06-13 20:15:13 INFO ShuffledRDD:59 - Removing RDD 24 from persistence list -2016-06-13 20:15:13 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Registering RDD 75 (apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO UnionRDD:59 - Removing RDD 23 from persistence list -2016-06-13 20:15:13 INFO DAGScheduler:59 - Got job 10 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:15:13 INFO DAGScheduler:59 - Final stage: ResultStage 21(apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 20) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 20) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 20 (MapPartitionsRDD[75] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 23 -2016-06-13 20:15:13 INFO MapPartitionsRDD:59 - Removing RDD 21 from persistence list -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=41112, maxMem=766075207 -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:50965 in memory (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 21 -2016-06-13 20:15:13 INFO JobGenerator:59 - Checkpointing graph for time 5000 ms -2016-06-13 20:15:13 INFO DStreamGraph:59 - Updating checkpoint data for time 5000 ms -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:15:13 INFO DStreamGraph:59 - Updated checkpoint data for time 5000 ms -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 24 -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=45856, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:50965 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 20:15:13 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 20 (MapPartitionsRDD[75] at apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 45, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 46, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:15:13 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 46) -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:50965 in memory (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:50965 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:50965 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 20:15:13 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-06-13 20:15:13 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 45) -2016-06-13 20:15:13 INFO CheckpointWriter:59 - Checkpoint for time 10000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-10000', took 27294 bytes and 205 ms -2016-06-13 20:15:13 INFO CheckpointWriter:59 - Saving checkpoint for time 1000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-1000' -2016-06-13 20:15:13 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 46). 1159 bytes result sent to driver -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:50965 in memory (size: 1799.0 B, free: 730.6 MB) -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 46) in 57 ms on localhost (1/2) -2016-06-13 20:15:13 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 45). 1159 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 45) in 61 ms on localhost (2/2) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 20:15:13 INFO DAGScheduler:59 - ShuffleMapStage 20 (apply at Transformer.scala:22) finished in 0.062 s -2016-06-13 20:15:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:15:13 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:15:13 INFO DAGScheduler:59 - waiting: Set(ResultStage 21) -2016-06-13 20:15:13 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:50965 in memory (size: 2000.0 B, free: 730.6 MB) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Missing parents for ResultStage 21: List() -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting ResultStage 21 (ShuffledRDD[76] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 20:15:13 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=17559, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=22823, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 20:15:13 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:50965 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:15:13 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (ShuffledRDD[76] at apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:13 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 48) -2016-06-13 20:15:13 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 47) -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:15:13 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 48). 1161 bytes result sent to driver -2016-06-13 20:15:13 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 47). 1498 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 48) in 16 ms on localhost (1/2) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 47) in 18 ms on localhost (2/2) -2016-06-13 20:15:13 INFO DAGScheduler:59 - ResultStage 21 (apply at Transformer.scala:22) finished in 0.018 s -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 20:15:13 INFO DAGScheduler:59 - Job 10 finished: apply at Transformer.scala:22, took 0.113413 s -2016-06-13 20:15:13 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 300.0 and centroid: [-0.4778380287460108,0.9658292271340416,-1.6712646630450996,1.273358698745343,0.37938624993358266] -2016-06-13 20:15:13 INFO JobScheduler:59 - Finished job streaming job 6000 ms.0 from job set of time 6000 ms -2016-06-13 20:15:13 INFO JobScheduler:59 - Starting job streaming job 6000 ms.1 from job set of time 6000 ms -2016-06-13 20:15:13 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Registering RDD 35 (apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Got job 11 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:15:13 INFO DAGScheduler:59 - Final stage: ResultStage 23(apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 22) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 22) -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 22 (UnionRDD[35] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=25519, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=28799, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.6 MB) -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:50965 (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:15:13 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:13 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 22 (UnionRDD[35] at apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 3 tasks -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 49, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 50, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:15:13 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 49) -2016-06-13 20:15:13 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 50) -2016-06-13 20:15:13 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 50). 1159 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 51, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:15:13 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 49). 1159 bytes result sent to driver -2016-06-13 20:15:13 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 51) -2016-06-13 20:15:13 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-1000.bk -2016-06-13 20:15:13 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:15:13 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 51). 1159 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 50) in 63 ms on localhost (1/3) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 49) in 65 ms on localhost (2/3) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 51) in 32 ms on localhost (3/3) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 20:15:13 INFO DAGScheduler:59 - ShuffleMapStage 22 (apply at Transformer.scala:22) finished in 0.069 s -2016-06-13 20:15:13 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:15:13 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:15:13 INFO DAGScheduler:59 - waiting: Set(ResultStage 23) -2016-06-13 20:15:13 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:15:13 INFO DAGScheduler:59 - Missing parents for ResultStage 23: List() -2016-06-13 20:15:13 INFO CheckpointWriter:59 - Checkpoint for time 1000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-1000', took 27472 bytes and 169 ms -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[77] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:15:13 INFO CheckpointWriter:59 - Saving checkpoint for time 2000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-2000' -2016-06-13 20:15:13 INFO DStreamGraph:59 - Clearing checkpoint data for time 1000 ms -2016-06-13 20:15:13 INFO DStreamGraph:59 - Cleared checkpoint data for time 1000 ms -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=30806, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:15:13 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:15:13 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=34046, maxMem=766075207 -2016-06-13 20:15:13 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.6 MB) -2016-06-13 20:15:13 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:50965 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:15:13 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[77] at apply at Transformer.scala:22) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 52, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:13 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 52) -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:13 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/receivedBlockMetadata older than 0: -2016-06-13 20:15:13 INFO InputInfoTracker:59 - remove old batch metadata: -2016-06-13 20:15:13 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/receivedBlockMetadata older than 0 -2016-06-13 20:15:13 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 52). 1180 bytes result sent to driver -2016-06-13 20:15:13 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 53) -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:15:13 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:13 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 53). 1172 bytes result sent to driver -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 52) in 30 ms on localhost (1/2) -2016-06-13 20:15:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 53) in 30 ms on localhost (2/2) -2016-06-13 20:15:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 20:15:13 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:15:13 INFO DAGScheduler:59 - ResultStage 23 (apply at Transformer.scala:22) finished in 0.036 s -2016-06-13 20:15:13 INFO DAGScheduler:59 - Job 11 finished: apply at Transformer.scala:22, took 0.142046 s -2016-06-13 20:15:13 INFO JobScheduler:59 - Finished job streaming job 6000 ms.1 from job set of time 6000 ms -2016-06-13 20:15:13 INFO ParallelCollectionRDD:59 - Removing RDD 26 from persistence list -2016-06-13 20:15:13 INFO JobScheduler:59 - Total delay: 1465820107.987 s for time 6000 ms (execution: 0.284 s) -2016-06-13 20:15:13 INFO JobScheduler:59 - Starting job streaming job 7000 ms.0 from job set of time 7000 ms -2016-06-13 20:15:13 INFO MapPartitionsRDD:59 - Removing RDD 31 from persistence list -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 26 -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 31 -2016-06-13 20:15:13 INFO ShuffledRDD:59 - Removing RDD 30 from persistence list -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 30 -2016-06-13 20:15:13 INFO UnionRDD:59 - Removing RDD 29 from persistence list -2016-06-13 20:15:13 INFO MapPartitionsRDD:59 - Removing RDD 27 from persistence list -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 29 -2016-06-13 20:15:13 INFO BlockManager:59 - Removing RDD 27 -2016-06-13 20:15:13 INFO JobGenerator:59 - Checkpointing graph for time 6000 ms -2016-06-13 20:15:13 INFO DStreamGraph:59 - Updating checkpoint data for time 6000 ms -2016-06-13 20:15:13 INFO DStreamGraph:59 - Updated checkpoint data for time 6000 ms -2016-06-13 20:15:14 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Registering RDD 78 (apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Got job 12 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:15:14 INFO DAGScheduler:59 - Final stage: ResultStage 25(apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 24) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 24) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 24 (MapPartitionsRDD[78] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=35849, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(2480) called with curMem=40593, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:50965 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:15:14 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 24 (MapPartitionsRDD[78] at apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 54, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 55, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:15:14 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 55) -2016-06-13 20:15:14 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 54) -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:15:14 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 55). 1159 bytes result sent to driver -2016-06-13 20:15:14 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 54). 1159 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 55) in 53 ms on localhost (1/2) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 54) in 54 ms on localhost (2/2) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:15:14 INFO DAGScheduler:59 - ShuffleMapStage 24 (apply at Transformer.scala:22) finished in 0.055 s -2016-06-13 20:15:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:15:14 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:15:14 INFO DAGScheduler:59 - waiting: Set(ResultStage 25) -2016-06-13 20:15:14 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:15:14 INFO DAGScheduler:59 - Missing parents for ResultStage 25: List() -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting ResultStage 25 (ShuffledRDD[79] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=43073, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=48337, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:50965 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:15:14 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (ShuffledRDD[79] at apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:14 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 56) -2016-06-13 20:15:14 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 57) -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:14 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 57). 1161 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 57) in 9 ms on localhost (1/2) -2016-06-13 20:15:14 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 56). 1498 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 56) in 12 ms on localhost (2/2) -2016-06-13 20:15:14 INFO DAGScheduler:59 - ResultStage 25 (apply at Transformer.scala:22) finished in 0.012 s -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-06-13 20:15:14 INFO DAGScheduler:59 - Job 12 finished: apply at Transformer.scala:22, took 0.108436 s -2016-06-13 20:15:14 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 350.0 and centroid: [-0.47440048489638564,0.9650523283036628,-1.6700055274657652,1.269947562552048,0.3797915177328754] -2016-06-13 20:15:14 INFO JobScheduler:59 - Finished job streaming job 7000 ms.0 from job set of time 7000 ms -2016-06-13 20:15:14 INFO JobScheduler:59 - Starting job streaming job 7000 ms.1 from job set of time 7000 ms -2016-06-13 20:15:14 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Registering RDD 41 (apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Got job 13 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:15:14 INFO DAGScheduler:59 - Final stage: ResultStage 27(apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (UnionRDD[41] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=51033, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=54313, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:50965 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:15:14 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 26 (UnionRDD[41] at apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 3 tasks -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 58, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 59, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:15:14 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 59) -2016-06-13 20:15:14 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 58) -2016-06-13 20:15:14 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-1000 -2016-06-13 20:15:14 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 59). 1159 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 60, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:15:14 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 58). 1159 bytes result sent to driver -2016-06-13 20:15:14 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 60) -2016-06-13 20:15:14 INFO CheckpointWriter:59 - Checkpoint for time 2000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-2000', took 27485 bytes and 222 ms -2016-06-13 20:15:14 INFO DStreamGraph:59 - Clearing checkpoint data for time 2000 ms -2016-06-13 20:15:14 INFO DStreamGraph:59 - Cleared checkpoint data for time 2000 ms -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 59) in 20 ms on localhost (1/3) -2016-06-13 20:15:14 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:15:14 INFO CheckpointWriter:59 - Saving checkpoint for time 3000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-3000' -2016-06-13 20:15:14 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/receivedBlockMetadata older than 1000: -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 58) in 22 ms on localhost (2/3) -2016-06-13 20:15:14 INFO InputInfoTracker:59 - remove old batch metadata: -2016-06-13 20:15:14 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/receivedBlockMetadata older than 1000 -2016-06-13 20:15:14 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 60). 1159 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 60) in 12 ms on localhost (3/3) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-06-13 20:15:14 INFO DAGScheduler:59 - ShuffleMapStage 26 (apply at Transformer.scala:22) finished in 0.029 s -2016-06-13 20:15:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:15:14 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:15:14 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-06-13 20:15:14 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:15:14 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[80] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=56322, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(1801) called with curMem=59562, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 1801.0 B, free 730.5 MB) -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:50965 (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:15:14 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[80] at apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 62, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:14 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 61) -2016-06-13 20:15:14 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 62) -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:14 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 62). 1172 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 62) in 14 ms on localhost (1/2) -2016-06-13 20:15:14 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 61). 1180 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 61) in 30 ms on localhost (2/2) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-06-13 20:15:14 INFO DAGScheduler:59 - ResultStage 27 (apply at Transformer.scala:22) finished in 0.032 s -2016-06-13 20:15:14 INFO DAGScheduler:59 - Job 13 finished: apply at Transformer.scala:22, took 0.086198 s -2016-06-13 20:15:14 INFO JobScheduler:59 - Finished job streaming job 7000 ms.1 from job set of time 7000 ms -2016-06-13 20:15:14 INFO JobScheduler:59 - Total delay: 1465820107.210 s for time 7000 ms (execution: 0.222 s) -2016-06-13 20:15:14 INFO JobScheduler:59 - Starting job streaming job 8000 ms.0 from job set of time 8000 ms -2016-06-13 20:15:14 INFO ParallelCollectionRDD:59 - Removing RDD 32 from persistence list -2016-06-13 20:15:14 INFO MapPartitionsRDD:59 - Removing RDD 37 from persistence list -2016-06-13 20:15:14 INFO BlockManager:59 - Removing RDD 32 -2016-06-13 20:15:14 INFO BlockManager:59 - Removing RDD 37 -2016-06-13 20:15:14 INFO ShuffledRDD:59 - Removing RDD 36 from persistence list -2016-06-13 20:15:14 INFO UnionRDD:59 - Removing RDD 35 from persistence list -2016-06-13 20:15:14 INFO MapPartitionsRDD:59 - Removing RDD 33 from persistence list -2016-06-13 20:15:14 INFO BlockManager:59 - Removing RDD 33 -2016-06-13 20:15:14 INFO JobGenerator:59 - Checkpointing graph for time 7000 ms -2016-06-13 20:15:14 INFO DStreamGraph:59 - Updating checkpoint data for time 7000 ms -2016-06-13 20:15:14 INFO DStreamGraph:59 - Updated checkpoint data for time 7000 ms -2016-06-13 20:15:14 INFO BlockManager:59 - Removing RDD 35 -2016-06-13 20:15:14 INFO BlockManager:59 - Removing RDD 36 -2016-06-13 20:15:14 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Registering RDD 81 (apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Got job 14 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:15:14 INFO DAGScheduler:59 - Final stage: ResultStage 29(apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 28) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 28) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 28 (MapPartitionsRDD[81] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=61363, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=66107, maxMem=766075207 -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:50965 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:15:14 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 28 (MapPartitionsRDD[81] at apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 63, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 64, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:15:14 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 63) -2016-06-13 20:15:14 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 64) -2016-06-13 20:15:14 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 64). 1159 bytes result sent to driver -2016-06-13 20:15:14 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 63). 1159 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 64) in 33 ms on localhost (1/2) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 63) in 34 ms on localhost (2/2) -2016-06-13 20:15:14 INFO DAGScheduler:59 - ShuffleMapStage 28 (apply at Transformer.scala:22) finished in 0.036 s -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-06-13 20:15:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:15:14 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:15:14 INFO DAGScheduler:59 - waiting: Set(ResultStage 29) -2016-06-13 20:15:14 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:15:14 INFO DAGScheduler:59 - Missing parents for ResultStage 29: List() -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting ResultStage 29 (ShuffledRDD[82] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=68589, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=73853, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:50965 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:15:14 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (ShuffledRDD[82] at apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 65, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 66, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:14 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 65) -2016-06-13 20:15:14 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 66) -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:15:14 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 66). 1161 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 66) in 7 ms on localhost (1/2) -2016-06-13 20:15:14 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 65). 1498 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 65) in 11 ms on localhost (2/2) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 20:15:14 INFO DAGScheduler:59 - ResultStage 29 (apply at Transformer.scala:22) finished in 0.011 s -2016-06-13 20:15:14 INFO DAGScheduler:59 - Job 14 finished: apply at Transformer.scala:22, took 0.070824 s -2016-06-13 20:15:14 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 400.0 and centroid: [-0.4716182064591067,0.967586334056103,-1.6699954778983372,1.270341503100185,0.37935009425710725] -2016-06-13 20:15:14 INFO JobScheduler:59 - Finished job streaming job 8000 ms.0 from job set of time 8000 ms -2016-06-13 20:15:14 INFO JobScheduler:59 - Starting job streaming job 8000 ms.1 from job set of time 8000 ms -2016-06-13 20:15:14 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Registering RDD 47 (apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Got job 15 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:15:14 INFO DAGScheduler:59 - Final stage: ResultStage 31(apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 30) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 30 (UnionRDD[47] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=76547, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(2006) called with curMem=79827, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2006.0 B, free 730.5 MB) -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:50965 (size: 2006.0 B, free: 730.6 MB) -2016-06-13 20:15:14 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 30 (UnionRDD[47] at apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 3 tasks -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 67, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 68, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:15:14 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 67) -2016-06-13 20:15:14 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 68) -2016-06-13 20:15:14 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-2000.bk -2016-06-13 20:15:14 INFO CheckpointWriter:59 - Checkpoint for time 3000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-3000', took 27496 bytes and 177 ms -2016-06-13 20:15:14 INFO DStreamGraph:59 - Clearing checkpoint data for time 3000 ms -2016-06-13 20:15:14 INFO CheckpointWriter:59 - Saving checkpoint for time 4000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-4000' -2016-06-13 20:15:14 INFO DStreamGraph:59 - Cleared checkpoint data for time 3000 ms -2016-06-13 20:15:14 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 68). 1159 bytes result sent to driver -2016-06-13 20:15:14 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:15:14 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/receivedBlockMetadata older than 2000: -2016-06-13 20:15:14 INFO InputInfoTracker:59 - remove old batch metadata: 1000 ms -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 69, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:15:14 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/receivedBlockMetadata older than 2000 -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 68) in 20 ms on localhost (1/3) -2016-06-13 20:15:14 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 67). 1159 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 67) in 22 ms on localhost (2/3) -2016-06-13 20:15:14 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 69) -2016-06-13 20:15:14 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 69). 1159 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 69) in 14 ms on localhost (3/3) -2016-06-13 20:15:14 INFO DAGScheduler:59 - ShuffleMapStage 30 (apply at Transformer.scala:22) finished in 0.033 s -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 20:15:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:15:14 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:15:14 INFO DAGScheduler:59 - waiting: Set(ResultStage 31) -2016-06-13 20:15:14 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:15:14 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List() -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[83] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=81833, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=85073, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.5 MB) -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:50965 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:15:14 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[83] at apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 70, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 71, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:14 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 71) -2016-06-13 20:15:14 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 70) -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:15:14 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 71). 1172 bytes result sent to driver -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 71) in 8 ms on localhost (1/2) -2016-06-13 20:15:14 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 70). 1180 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 70) in 15 ms on localhost (2/2) -2016-06-13 20:15:14 INFO DAGScheduler:59 - ResultStage 31 (apply at Transformer.scala:22) finished in 0.016 s -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 20:15:14 INFO DAGScheduler:59 - Job 15 finished: apply at Transformer.scala:22, took 0.073969 s -2016-06-13 20:15:14 INFO JobScheduler:59 - Finished job streaming job 8000 ms.1 from job set of time 8000 ms -2016-06-13 20:15:14 INFO JobScheduler:59 - Total delay: 1465820106.381 s for time 8000 ms (execution: 0.170 s) -2016-06-13 20:15:14 INFO JobScheduler:59 - Starting job streaming job 9000 ms.0 from job set of time 9000 ms -2016-06-13 20:15:14 INFO ParallelCollectionRDD:59 - Removing RDD 38 from persistence list -2016-06-13 20:15:14 INFO MapPartitionsRDD:59 - Removing RDD 43 from persistence list -2016-06-13 20:15:14 INFO BlockManager:59 - Removing RDD 43 -2016-06-13 20:15:14 INFO ShuffledRDD:59 - Removing RDD 42 from persistence list -2016-06-13 20:15:14 INFO BlockManager:59 - Removing RDD 38 -2016-06-13 20:15:14 INFO BlockManager:59 - Removing RDD 42 -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:15:14 INFO UnionRDD:59 - Removing RDD 41 from persistence list -2016-06-13 20:15:14 INFO MapPartitionsRDD:59 - Removing RDD 39 from persistence list -2016-06-13 20:15:14 INFO BlockManager:59 - Removing RDD 41 -2016-06-13 20:15:14 INFO BlockManager:59 - Removing RDD 39 -2016-06-13 20:15:14 INFO JobGenerator:59 - Checkpointing graph for time 8000 ms -2016-06-13 20:15:14 INFO DStreamGraph:59 - Updating checkpoint data for time 8000 ms -2016-06-13 20:15:14 INFO DStreamGraph:59 - Updated checkpoint data for time 8000 ms -2016-06-13 20:15:14 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Registering RDD 84 (apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Got job 16 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:15:14 INFO DAGScheduler:59 - Final stage: ResultStage 33(apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 32) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 32 (MapPartitionsRDD[84] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=86875, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(2477) called with curMem=91619, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:50965 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:15:14 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 32 (MapPartitionsRDD[84] at apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 72, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 73, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:15:14 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 73) -2016-06-13 20:15:14 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 72) -2016-06-13 20:15:14 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 73). 1159 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 73) in 21 ms on localhost (1/2) -2016-06-13 20:15:14 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 72). 1159 bytes result sent to driver -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 72) in 29 ms on localhost (2/2) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 20:15:14 INFO DAGScheduler:59 - ShuffleMapStage 32 (apply at Transformer.scala:22) finished in 0.030 s -2016-06-13 20:15:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:15:14 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:15:14 INFO DAGScheduler:59 - waiting: Set(ResultStage 33) -2016-06-13 20:15:14 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:15:14 INFO DAGScheduler:59 - Missing parents for ResultStage 33: List() -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting ResultStage 33 (ShuffledRDD[85] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=94096, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=99360, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned shuffle 9 -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned shuffle 7 -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:50965 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:15:14 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (ShuffledRDD[85] at apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned shuffle 11 -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 74, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 75, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:14 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 74) -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:50965 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:14 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 75) -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:15:14 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 75). 1161 bytes result sent to driver -2016-06-13 20:15:14 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 74). 1498 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 75) in 22 ms on localhost (1/2) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 74) in 23 ms on localhost (2/2) -2016-06-13 20:15:14 INFO DAGScheduler:59 - ResultStage 33 (apply at Transformer.scala:22) finished in 0.023 s -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Job 16 finished: apply at Transformer.scala:22, took 0.087812 s -2016-06-13 20:15:14 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 450.0 and centroid: [-0.4722098164150225,0.9642763266691793,-1.6666072662538036,1.2709270669491914,0.37911973669212223] -2016-06-13 20:15:14 INFO JobScheduler:59 - Finished job streaming job 9000 ms.0 from job set of time 9000 ms -2016-06-13 20:15:14 INFO JobScheduler:59 - Starting job streaming job 9000 ms.1 from job set of time 9000 ms -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:50965 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned shuffle 10 -2016-06-13 20:15:14 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Registering RDD 53 (apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Got job 17 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:15:14 INFO DAGScheduler:59 - Final stage: ResultStage 35(apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 34) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 34) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 34 (UnionRDD[53] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:50965 in memory (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=81825, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:50965 in memory (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=83095, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.5 MB) -2016-06-13 20:15:14 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-2000 -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:50965 (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:15:14 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 34 (UnionRDD[53] at apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 3 tasks -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 76, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Removed broadcast_31_piece0 on localhost:50965 in memory (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 77, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:15:14 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 77) -2016-06-13 20:15:14 INFO CheckpointWriter:59 - Checkpoint for time 4000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-4000', took 27488 bytes and 183 ms -2016-06-13 20:15:14 INFO CheckpointWriter:59 - Saving checkpoint for time 5000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-5000' -2016-06-13 20:15:14 INFO DStreamGraph:59 - Clearing checkpoint data for time 4000 ms -2016-06-13 20:15:14 INFO DStreamGraph:59 - Cleared checkpoint data for time 4000 ms -2016-06-13 20:15:14 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:15:14 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/receivedBlockMetadata older than 3000: -2016-06-13 20:15:14 INFO InputInfoTracker:59 - remove old batch metadata: 2000 ms -2016-06-13 20:15:14 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/receivedBlockMetadata older than 3000 -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned accumulator 32 -2016-06-13 20:15:14 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 76) -2016-06-13 20:15:14 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 77). 1159 bytes result sent to driver -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Removed broadcast_30_piece0 on localhost:50965 in memory (size: 2006.0 B, free: 730.6 MB) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 78, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:15:14 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 78) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 77) in 21 ms on localhost (1/3) -2016-06-13 20:15:14 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 76). 1159 bytes result sent to driver -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 76) in 31 ms on localhost (2/3) -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned accumulator 31 -2016-06-13 20:15:14 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 78). 1159 bytes result sent to driver -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Removed broadcast_29_piece0 on localhost:50965 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned accumulator 30 -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 78) in 28 ms on localhost (3/3) -2016-06-13 20:15:14 INFO DAGScheduler:59 - ShuffleMapStage 34 (apply at Transformer.scala:22) finished in 0.050 s -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Removed broadcast_28_piece0 on localhost:50965 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:15:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:15:14 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:15:14 INFO DAGScheduler:59 - waiting: Set(ResultStage 35) -2016-06-13 20:15:14 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned accumulator 29 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Missing parents for ResultStage 35: List() -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[86] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned shuffle 14 -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=51269, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Removed broadcast_27_piece0 on localhost:50965 in memory (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=54509, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned accumulator 28 -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Removed broadcast_26_piece0 on localhost:50965 in memory (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned accumulator 27 -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned shuffle 13 -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:50965 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Removed broadcast_25_piece0 on localhost:50965 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:15:14 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[86] at apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 79, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned accumulator 26 -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 80, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:14 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 79) -2016-06-13 20:15:14 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 80) -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:14 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 79). 1180 bytes result sent to driver -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Removed broadcast_24_piece0 on localhost:50965 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:15:14 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 80). 1172 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 79) in 20 ms on localhost (1/2) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 80) in 21 ms on localhost (2/2) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned accumulator 25 -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned shuffle 12 -2016-06-13 20:15:14 INFO DAGScheduler:59 - ResultStage 35 (apply at Transformer.scala:22) finished in 0.028 s -2016-06-13 20:15:14 INFO DAGScheduler:59 - Job 17 finished: apply at Transformer.scala:22, took 0.138099 s -2016-06-13 20:15:14 INFO JobScheduler:59 - Finished job streaming job 9000 ms.1 from job set of time 9000 ms -2016-06-13 20:15:14 INFO JobScheduler:59 - Total delay: 1465820105.634 s for time 9000 ms (execution: 0.253 s) -2016-06-13 20:15:14 INFO JobScheduler:59 - Starting job streaming job 10000 ms.0 from job set of time 10000 ms -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:50965 in memory (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:15:14 INFO ParallelCollectionRDD:59 - Removing RDD 44 from persistence list -2016-06-13 20:15:14 INFO BlockManager:59 - Removing RDD 44 -2016-06-13 20:15:14 INFO MapPartitionsRDD:59 - Removing RDD 49 from persistence list -2016-06-13 20:15:14 INFO ShuffledRDD:59 - Removing RDD 48 from persistence list -2016-06-13 20:15:14 INFO BlockManager:59 - Removing RDD 49 -2016-06-13 20:15:14 INFO UnionRDD:59 - Removing RDD 47 from persistence list -2016-06-13 20:15:14 INFO BlockManager:59 - Removing RDD 48 -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 20:15:14 INFO BlockManager:59 - Removing RDD 47 -2016-06-13 20:15:14 INFO MapPartitionsRDD:59 - Removing RDD 45 from persistence list -2016-06-13 20:15:14 INFO JobGenerator:59 - Checkpointing graph for time 9000 ms -2016-06-13 20:15:14 INFO DStreamGraph:59 - Updating checkpoint data for time 9000 ms -2016-06-13 20:15:14 INFO DStreamGraph:59 - Updated checkpoint data for time 9000 ms -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:15:14 INFO BlockManager:59 - Removing RDD 45 -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:50965 in memory (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:15:14 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 20:15:14 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Registering RDD 87 (apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Got job 18 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:15:14 INFO DAGScheduler:59 - Final stage: ResultStage 37(apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 36) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 36) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 36 (MapPartitionsRDD[87] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=25509, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=30253, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:50965 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:15:14 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 36 (MapPartitionsRDD[87] at apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 81, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 82, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:15:14 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 82) -2016-06-13 20:15:14 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 81) -2016-06-13 20:15:14 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 81). 1159 bytes result sent to driver -2016-06-13 20:15:14 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 82). 1159 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 81) in 15 ms on localhost (1/2) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 82) in 16 ms on localhost (2/2) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-06-13 20:15:14 INFO DAGScheduler:59 - ShuffleMapStage 36 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:15:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:15:14 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:15:14 INFO DAGScheduler:59 - waiting: Set(ResultStage 37) -2016-06-13 20:15:14 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:15:14 INFO DAGScheduler:59 - Missing parents for ResultStage 37: List() -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting ResultStage 37 (ShuffledRDD[88] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=32735, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=37999, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:50965 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:15:14 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (ShuffledRDD[88] at apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 83, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 84, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:14 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-3000.bk -2016-06-13 20:15:14 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 83) -2016-06-13 20:15:14 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 84) -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:14 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 84). 1161 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 84) in 8 ms on localhost (1/2) -2016-06-13 20:15:14 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 83). 1498 bytes result sent to driver -2016-06-13 20:15:14 INFO CheckpointWriter:59 - Checkpoint for time 5000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-5000', took 27484 bytes and 198 ms -2016-06-13 20:15:14 INFO DStreamGraph:59 - Clearing checkpoint data for time 5000 ms -2016-06-13 20:15:14 INFO DStreamGraph:59 - Cleared checkpoint data for time 5000 ms -2016-06-13 20:15:14 INFO CheckpointWriter:59 - Saving checkpoint for time 6000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-6000' -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 83) in 12 ms on localhost (2/2) -2016-06-13 20:15:14 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:15:14 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/receivedBlockMetadata older than 4000: -2016-06-13 20:15:14 INFO DAGScheduler:59 - ResultStage 37 (apply at Transformer.scala:22) finished in 0.013 s -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-06-13 20:15:14 INFO InputInfoTracker:59 - remove old batch metadata: 3000 ms -2016-06-13 20:15:14 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/receivedBlockMetadata older than 4000 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Job 18 finished: apply at Transformer.scala:22, took 0.051119 s -2016-06-13 20:15:14 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 500.0 and centroid: [-0.4725511979691583,0.9644503899125422,-1.668776373542808,1.2721254429935838,0.37815209739836425] -2016-06-13 20:15:14 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Registering RDD 59 (apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO JobScheduler:59 - Finished job streaming job 10000 ms.0 from job set of time 10000 ms -2016-06-13 20:15:14 INFO JobScheduler:59 - Starting job streaming job 10000 ms.1 from job set of time 10000 ms -2016-06-13 20:15:14 INFO DAGScheduler:59 - Got job 19 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:15:14 INFO DAGScheduler:59 - Final stage: ResultStage 39(apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 38) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 38) -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting ShuffleMapStage 38 (UnionRDD[59] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=40695, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=43975, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.5 MB) -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:50965 (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:15:14 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 38 (UnionRDD[59] at apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 3 tasks -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 85, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 86, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:15:14 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 85) -2016-06-13 20:15:14 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 86) -2016-06-13 20:15:14 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 85). 1159 bytes result sent to driver -2016-06-13 20:15:14 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 86). 1159 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 87, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:15:14 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 87) -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 85) in 25 ms on localhost (1/3) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 86) in 25 ms on localhost (2/3) -2016-06-13 20:15:14 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 87). 1159 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 87) in 12 ms on localhost (3/3) -2016-06-13 20:15:14 INFO DAGScheduler:59 - ShuffleMapStage 38 (apply at Transformer.scala:22) finished in 0.029 s -2016-06-13 20:15:14 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:15:14 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:15:14 INFO DAGScheduler:59 - waiting: Set(ResultStage 39) -2016-06-13 20:15:14 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-06-13 20:15:14 INFO DAGScheduler:59 - Missing parents for ResultStage 39: List() -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[89] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=45982, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:15:14 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=49222, maxMem=766075207 -2016-06-13 20:15:14 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:15:14 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:50965 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:15:14 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:15:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[89] at apply at Transformer.scala:22) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:15:14 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 89) -2016-06-13 20:15:14 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 88) -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:15:14 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:15:14 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 89). 1172 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 89) in 6 ms on localhost (1/2) -2016-06-13 20:15:14 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 88). 1180 bytes result sent to driver -2016-06-13 20:15:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 88) in 13 ms on localhost (2/2) -2016-06-13 20:15:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-06-13 20:15:14 INFO DAGScheduler:59 - ResultStage 39 (apply at Transformer.scala:22) finished in 0.014 s -2016-06-13 20:15:14 INFO DAGScheduler:59 - Job 19 finished: apply at Transformer.scala:22, took 0.075923 s -2016-06-13 20:15:14 INFO JobScheduler:59 - Finished job streaming job 10000 ms.1 from job set of time 10000 ms -2016-06-13 20:15:14 INFO JobScheduler:59 - Total delay: 1465820104.804 s for time 10000 ms (execution: 0.169 s) -2016-06-13 20:15:14 INFO ParallelCollectionRDD:59 - Removing RDD 50 from persistence list -2016-06-13 20:15:14 INFO BlockManager:59 - Removing RDD 50 -2016-06-13 20:15:14 INFO MapPartitionsRDD:59 - Removing RDD 55 from persistence list -2016-06-13 20:15:14 INFO ShuffledRDD:59 - Removing RDD 54 from persistence list -2016-06-13 20:15:14 INFO BlockManager:59 - Removing RDD 55 -2016-06-13 20:15:14 INFO BlockManager:59 - Removing RDD 54 -2016-06-13 20:15:14 INFO UnionRDD:59 - Removing RDD 53 from persistence list -2016-06-13 20:15:14 INFO MapPartitionsRDD:59 - Removing RDD 51 from persistence list -2016-06-13 20:15:14 INFO BlockManager:59 - Removing RDD 53 -2016-06-13 20:15:14 INFO JobGenerator:59 - Checkpointing graph for time 10000 ms -2016-06-13 20:15:14 INFO BlockManager:59 - Removing RDD 51 -2016-06-13 20:15:14 INFO DStreamGraph:59 - Updating checkpoint data for time 10000 ms -2016-06-13 20:15:14 INFO DStreamGraph:59 - Updated checkpoint data for time 10000 ms -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - Output generated in 3879 milliseconds -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:15:14 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:15:14 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-3000 -2016-06-13 20:15:14 INFO CheckpointWriter:59 - Checkpoint for time 6000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-6000', took 27480 bytes and 154 ms -2016-06-13 20:15:14 INFO DStreamGraph:59 - Clearing checkpoint data for time 6000 ms -2016-06-13 20:15:14 INFO CheckpointWriter:59 - Saving checkpoint for time 7000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-7000' -2016-06-13 20:15:14 INFO DStreamGraph:59 - Cleared checkpoint data for time 6000 ms -2016-06-13 20:15:14 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:15:14 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/receivedBlockMetadata older than 5000: -2016-06-13 20:15:14 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/receivedBlockMetadata older than 5000 -2016-06-13 20:15:14 INFO InputInfoTracker:59 - remove old batch metadata: 4000 ms -2016-06-13 20:15:14 INFO JobGenerator:59 - Stopping JobGenerator immediately -2016-06-13 20:15:14 INFO RecurringTimer:59 - Stopped timer for JobGenerator after time 10000 -2016-06-13 20:15:14 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-4000.bk -2016-06-13 20:15:24 INFO CheckpointWriter:59 - CheckpointWriter executor terminated ? false, waited for 10001 ms. -2016-06-13 20:15:24 INFO JobGenerator:59 - Stopped JobGenerator -2016-06-13 20:15:24 INFO JobScheduler:59 - Stopped JobScheduler -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/batch,null} -2016-06-13 20:15:24 INFO CheckpointWriter:59 - Checkpoint for time 7000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3/checkpoint-7000', took 27478 bytes and 10090 ms -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/streaming,null} -2016-06-13 20:15:24 INFO StreamingContext:59 - StreamingContext stopped successfully -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/batch/json,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/json,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:15:24 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:15:25 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 20:15:25 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:15:25 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:15:25 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:15:25 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:15:25 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:15:25 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:15:25 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:15:25 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:15:25 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:15:25 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 20:15:25 INFO StreamingKMeansSuite:59 - - -===== FINISHED o.a.s.mllib.clustering.kmeans.StreamingKMeansSuite: 'accuracy for single center and equivalence to grand average' ===== - -2016-06-13 20:15:25 WARN StreamingContext:71 - StreamingContext has already been stopped -2016-06-13 20:15:25 INFO SparkContext:59 - SparkContext already stopped. -2016-06-13 20:15:25 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 20:15:25 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3 -2016-06-13 20:15:25 ERROR ShutdownHookManager:96 - Exception while deleting Spark temp dir: C:\Users\xubo\AppData\Local\Temp\spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3 -java.io.IOException: Failed to delete: C:\Users\xubo\AppData\Local\Temp\spark-e9359c1c-24c5-4e08-814d-60cebfc21cf3 - at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:884) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:63) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:60) - at scala.collection.mutable.HashSet.foreach(HashSet.scala:79) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1.apply$mcV$sp(ShutdownHookManager.scala:60) - at org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:264) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1699) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply$mcV$sp(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:234) - at scala.util.Try$.apply(Try.scala:161) - at org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:216) - at org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54) -2016-06-13 20:15:25 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-32ae3a90-7866-4ba0-adf5-d5f54d6e6673 -2016-06-13 20:16:31 INFO StreamingKMeansSuite:59 - Using manual clock -2016-06-13 20:16:31 INFO StreamingKMeansSuite:59 - - -===== TEST OUTPUT FOR o.a.s.mllib.clustering.kmeans.StreamingKMeansSuite: 'accuracy for single center and equivalence to grand average' ===== - -2016-06-13 20:16:31 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:16:31 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 20:16:31 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:16:31 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:16:31 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:16:32 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:16:32 INFO Remoting:74 - Starting remoting -2016-06-13 20:16:32 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:50986] -2016-06-13 20:16:32 INFO Utils:59 - Successfully started service 'sparkDriver' on port 50986. -2016-06-13 20:16:33 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:16:33 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:16:33 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-8c531898-749e-488d-bc80-659a08320a00 -2016-06-13 20:16:33 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:16:33 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-e8a3a6fc-8575-4f87-8e69-6599a450a52d\httpd-7347d1ca-3ccb-4a54-8cd0-337064c5309e -2016-06-13 20:16:33 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:16:33 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:16:33 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:50987 -2016-06-13 20:16:33 INFO Utils:59 - Successfully started service 'HTTP file server' on port 50987. -2016-06-13 20:16:33 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:16:33 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:16:33 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 20:16:33 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 20:16:33 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 20:16:33 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:16:33 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:16:33 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 50998. -2016-06-13 20:16:33 INFO NettyBlockTransferService:59 - Server created on 50998 -2016-06-13 20:16:33 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:16:33 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:50998 with 730.6 MB RAM, BlockManagerId(driver, localhost, 50998) -2016-06-13 20:16:33 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:16:35 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 20:16:35 INFO StreamingKMeansSuite:59 - numBatches = 10, numExpectedOutput = 10 -2016-06-13 20:16:35 INFO ForEachDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:16:35 INFO TestInputStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:16:35 INFO TestInputStream:59 - Slide time = 1000 ms -2016-06-13 20:16:35 INFO TestInputStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:16:35 INFO TestInputStream:59 - Checkpoint interval = null -2016-06-13 20:16:35 INFO TestInputStream:59 - Remember duration = 1000 ms -2016-06-13 20:16:35 INFO TestInputStream:59 - Initialized and validated org.apache.spark.streaming.TestInputStream@4fb2c078 -2016-06-13 20:16:35 INFO ForEachDStream:59 - Slide time = 1000 ms -2016-06-13 20:16:35 INFO ForEachDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:16:35 INFO ForEachDStream:59 - Checkpoint interval = null -2016-06-13 20:16:35 INFO ForEachDStream:59 - Remember duration = 1000 ms -2016-06-13 20:16:35 INFO ForEachDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.ForEachDStream@7f19cbbb -2016-06-13 20:16:35 INFO TestOutputStreamWithPartitions:59 - metadataCleanupDelay = -1 -2016-06-13 20:16:35 INFO MappedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:16:35 INFO ShuffledDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:16:35 INFO TransformedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:16:35 INFO MappedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:16:35 INFO TestInputStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:16:35 INFO TestInputStream:59 - Slide time = 1000 ms -2016-06-13 20:16:35 INFO TestInputStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:16:35 INFO TestInputStream:59 - Checkpoint interval = null -2016-06-13 20:16:35 INFO TestInputStream:59 - Remember duration = 1000 ms -2016-06-13 20:16:35 INFO TestInputStream:59 - Initialized and validated org.apache.spark.streaming.TestInputStream@4fb2c078 -2016-06-13 20:16:35 INFO MappedDStream:59 - Slide time = 1000 ms -2016-06-13 20:16:35 INFO MappedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:16:35 INFO MappedDStream:59 - Checkpoint interval = null -2016-06-13 20:16:35 INFO MappedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:16:35 INFO MappedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@46915616 -2016-06-13 20:16:35 INFO TransformedDStream:59 - Slide time = 1000 ms -2016-06-13 20:16:35 INFO TransformedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:16:35 INFO TransformedDStream:59 - Checkpoint interval = null -2016-06-13 20:16:35 INFO TransformedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:16:35 INFO TransformedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.TransformedDStream@f3af2c0 -2016-06-13 20:16:35 INFO ShuffledDStream:59 - Slide time = 1000 ms -2016-06-13 20:16:35 INFO ShuffledDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:16:35 INFO ShuffledDStream:59 - Checkpoint interval = null -2016-06-13 20:16:35 INFO ShuffledDStream:59 - Remember duration = 1000 ms -2016-06-13 20:16:35 INFO ShuffledDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.ShuffledDStream@3efd899a -2016-06-13 20:16:35 INFO MappedDStream:59 - Slide time = 1000 ms -2016-06-13 20:16:35 INFO MappedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:16:35 INFO MappedDStream:59 - Checkpoint interval = null -2016-06-13 20:16:35 INFO MappedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:16:35 INFO MappedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@4b939fbd -2016-06-13 20:16:35 INFO TestOutputStreamWithPartitions:59 - Slide time = 1000 ms -2016-06-13 20:16:35 INFO TestOutputStreamWithPartitions:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:16:35 INFO TestOutputStreamWithPartitions:59 - Checkpoint interval = null -2016-06-13 20:16:35 INFO TestOutputStreamWithPartitions:59 - Remember duration = 1000 ms -2016-06-13 20:16:35 INFO TestOutputStreamWithPartitions:59 - Initialized and validated org.apache.spark.streaming.TestOutputStreamWithPartitions@65770081 -2016-06-13 20:16:35 INFO RecurringTimer:59 - Started timer for JobGenerator at time 1000 -2016-06-13 20:16:35 INFO JobGenerator:59 - Started JobGenerator at 1000 ms -2016-06-13 20:16:35 INFO JobScheduler:59 - Started JobScheduler -2016-06-13 20:16:35 INFO StreamingContext:59 - StreamingContext started -2016-06-13 20:16:35 INFO StreamingKMeansSuite:59 - Manual clock before advancing = 0 -2016-06-13 20:16:35 INFO StreamingKMeansSuite:59 - Manual clock after advancing = 10000 -2016-06-13 20:16:35 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:16:35 INFO TestInputStream:59 - Computing RDD for time 1000 ms -2016-06-13 20:16:35 INFO TestInputStream:59 - Created RDD 0 with Vector([-0.35685422167583125,0.9995288307555166,-1.5877809386448758,1.210845548490831,0.56222524157234], [-0.45216537133396134,0.8040574757345051,-1.5589888701250443,1.115603017032467,0.344649973028188], [-0.3801675945145341,1.0697589561731022,-1.6486682116186815,1.2790268654063703,0.5116371486214886], [-0.545300880562902,1.1093057253978866,-1.7582382472264182,1.211400006292075,0.2932044815946008], [-0.3995483137357882,0.929119621864747,-1.7401380541085598,1.2990457426846032,0.299090686584923], [-0.5964001160322344,1.0378164371074625,-1.553411818459791,1.2193968564632125,0.4492703031123452], [-0.33949398265706626,1.1369769008642154,-1.7816495259375202,1.2857769527411067,0.5642853362323552], [-0.4522936770981281,1.090819158309221,-1.4785463285656288,1.2734882859357213,0.31864385887835545], [-0.4895260213585818,1.024567290792244,-1.7744155605537828,1.4515329700677304,0.39720325477885093], [-0.5479321442182269,0.8926951810306638,-1.617388979581139,1.3192738326981768,0.3243774353000105], [-0.5468296230191492,0.9034725811394896,-1.8269524875493084,1.161899613497355,0.22607191229030854], [-0.4344367225435325,0.9054178856043291,-1.6658399540877864,1.13578899284944,0.46511292492021983], [-0.5914600674966851,0.9727747336030947,-1.7411162798640765,1.3902196062020102,0.43158411906136107], [-0.5644425684165094,0.996354296088912,-1.817848647277835,1.3768030733201289,0.28609432985591315], [-0.44148386701129744,1.025737906243747,-1.7073695034902865,1.2539087121413848,0.349242056080172], [-0.5287113084613145,0.8475916475163896,-1.8166057479945195,1.238631411179877,0.36922519791770014], [-0.2540595910515851,0.9698131032346381,-1.5866552625154318,1.2179915976157372,0.3240398576394729], [-0.5026644816450073,1.109581665015373,-1.7151161774591086,1.3901513268104455,0.39042855132933163], [-0.4131172722481001,1.0096927045418442,-1.811286466550916,1.2719080737538961,0.28293757989913904], [-0.4515790075180954,1.0935474561719332,-1.7896107864561164,1.3203383417885988,0.2994362800834273], [-0.4171607573128071,1.0818260184642114,-1.8219543038308885,1.2496462030096327,0.4964658371791977], [-0.45586902589910655,1.0378742745036647,-1.7000458648640797,1.446592332373813,0.348566870310007], [-0.3532087778209343,0.8044437973592731,-1.7162629066191757,1.3339281879504057,0.17946188844185906], [-0.5154063008996834,0.9595656827532482,-1.6519653379002484,1.322654750805598,0.3008929692388781], [-0.3967887546418169,0.7908879805460386,-1.7242582851383725,1.2141584886700174,0.36827019559433166], [-0.48352710482779404,1.0255449792765485,-1.7171397454598174,1.284877805985566,0.437232715780399], [-0.40324856443707463,1.020485534999759,-1.6517518343613002,1.2864020019792224,0.3984832850799499], [-0.6517069948565133,0.969461018860354,-1.684015510299471,1.2027906385718596,0.2014860155598368], [-0.6148278408211463,0.9795135452912322,-1.8642733722424907,1.2720240024566645,0.39785426587083933], [-0.6409674819545198,1.1778584543507387,-1.8094969072333307,1.24145841048891,0.1624672183638662], [-0.6368691555002903,0.8170131462860201,-1.6827833832635621,1.3031179618920097,0.31628673987022604], [-0.43096648399184834,0.9655746876954024,-1.7845188100637381,1.4052721272088269,0.5845842279495], [-0.4442125841627056,1.0257872008493307,-1.7242589434969455,1.2745428690859977,0.39998857867859966], [-0.4527127515828855,0.9000469054331385,-1.7497092961121075,1.2451107418571954,0.4344380036579415], [-0.6030914309562295,0.9699928952430515,-1.712604726536663,1.38261404269885,0.48344898725768926], [-0.34976726689041593,0.9355870894961762,-1.5766155567170825,1.3590936489794139,0.35690680463154484], [-0.575628267406687,0.9538063357613722,-1.6445003616008003,1.2104804829930789,0.3911315521177748], [-0.5582822637309535,0.9339694428739754,-1.6511037013459962,1.4401460976648193,0.3607766390547849], [-0.5468811641579405,0.997556943752324,-1.778985900216746,1.2440321529105447,0.4204585578447196], [-0.48994207115736743,1.0755700976056979,-1.5322042869816734,1.2132674597398667,0.5961642366275318], [-0.5345824744371771,0.7659168558371742,-1.7009697565551642,1.2200491259133113,0.24064266183571642], [-0.4425411108434083,0.9636579302553172,-1.6769715787345627,1.2060504970822528,0.1832578105698996], [-0.2803908635114948,0.9814260387904405,-1.6744903280757537,1.438894231115956,0.5307857877828275], [-0.4646639723438268,0.9422153316765177,-1.8936280948227913,1.1755014401606392,0.3269897050872188], [-0.5794967530942349,1.008452319256215,-1.8095212163297067,1.3053459835247618,0.3862084672749757], [-0.36564413314560745,0.9550404460199052,-1.671332555981891,1.2387784773127055,0.29524181369582553], [-0.4947387784852314,0.8968969114403345,-1.5032027608518188,1.263293789437244,0.41872133210307433], [-0.5901163249097482,0.9195890740589755,-1.76975579729397,1.2233696400583831,0.3403182341587175], [-0.7175598778778954,0.8837946345395844,-1.8329343739105675,1.3562350855120264,0.30734388952604197], [-0.4376962364418643,1.0046436887819825,-1.5427982731479972,1.1077094728701697,0.30461693506090337]) -2016-06-13 20:16:35 INFO JobScheduler:59 - Added jobs for time 1000 ms -2016-06-13 20:16:35 INFO TestInputStream:59 - Computing RDD for time 2000 ms -2016-06-13 20:16:35 INFO JobScheduler:59 - Starting job streaming job 1000 ms.0 from job set of time 1000 ms -2016-06-13 20:16:35 INFO TestInputStream:59 - Created RDD 6 with Vector([-0.5385984930004446,0.9989931115946847,-1.745310682734983,1.2763556288677276,0.41555544930746574], [-0.5140757026211953,0.9616396809310985,-1.6092748223737894,1.3080875431688948,0.3958240822167327], [-0.3317121400990889,1.1311521924692884,-1.5940394948396088,1.235663715627094,0.5352186835894586], [-0.3566003302425871,0.8649446433306678,-1.6281158497086519,1.0841327390682376,0.33108928300382867], [-0.4898654567486354,1.0095506134582393,-1.7900184464939166,1.4376638486285889,0.5476994415325478], [-0.5067718408631526,0.8330942372560937,-1.5447521454199076,1.3956201717944832,0.40502500771937533], [-0.45899690569395396,1.0114199686087044,-1.6793704227956527,1.2264029484806054,0.3617800498770533], [-0.42389468031403765,0.9744019756374155,-1.4339024359998267,1.3506788012287876,0.5071391796420295], [-0.3445608603995446,0.8999444931201762,-1.8295489983610813,1.3813011802363127,0.4406110935124823], [-0.4720580057434039,0.9616388382715773,-1.7885815681274932,1.2231064430079939,0.49611588805959794], [-0.6877856894979741,0.9509033529307679,-1.7111111174594673,1.1650825853084499,0.36541463301831684], [-0.41906769746785616,0.9643844813018809,-1.7088669498454399,1.2743442947780539,0.37857583863641653], [-0.45604703784103723,0.9704467329255722,-1.6908561828878124,1.2661005251088149,0.34761517755721416], [-0.45216413072759976,1.125638260528155,-1.8672240017957615,1.388348280279974,0.41289821746105937], [-0.6041998663111035,0.9518745991902821,-1.7482553616491074,1.3027968384522153,0.33003148862622383], [-0.7645915689875407,0.9231306641016475,-1.5908706609397762,1.0416797405804958,0.3435391589642622], [-0.5511129470810242,0.9789516647349409,-1.5896352104287992,1.2985407772348172,0.3074739718251722], [-0.3060111421519473,0.9693226097887153,-1.6503499878201646,1.2050435546957032,0.20529101796457794], [-0.41814813350738833,0.9065345647328998,-1.7369684308945867,1.3095490239413528,0.3931707179477215], [-0.7452070526618801,1.010695258373889,-1.6799990639878284,1.351913921189404,0.48041731347418104], [-0.19474072480894772,0.8574730260551575,-1.7114945513022097,1.2484211088395596,0.35791459113273333], [-0.679333385903053,1.136043445571062,-1.6819398713814546,1.2914383599572574,0.4846586154592743], [-0.38963314954900546,1.0499014443531718,-1.7075189569343263,1.3966853721193921,0.4922463621111953], [-0.4771733790096556,0.8615507710844875,-1.7824679274745288,1.1550649309651535,0.4634518570903764], [-0.46851141840829336,0.9490804068188984,-1.6654371185103947,1.1748885058064462,0.362811152170017], [-0.2486779458724137,1.0146748639649252,-1.8129854069448728,1.3198830969564963,0.3848127258680788], [-0.37278156677721497,0.9026389048304428,-1.6338760097656873,1.2146194636725334,0.305407696426058], [-0.5109164214840503,1.1025203353037651,-1.5805383770959172,1.3659800632138837,0.2507490333376023], [-0.42658774618748607,0.9583040463281658,-1.535401873083165,1.2270967800953845,0.30931926334482973], [-0.5761089637758711,0.9608023900436299,-1.6773446541045054,1.3896056325873751,0.3003385242281313], [-0.333657352857868,0.9284640162518879,-1.6506283984425263,1.4233119402331722,0.4817086358456392], [-0.4131260123218083,0.9257004348959492,-1.6385111064595121,1.2553085674173774,0.35994561408421055], [-0.41619483777131644,0.9791552402269297,-1.6621579631275758,1.1639596596963502,0.3356350519347727], [-0.42150886121352676,0.9497809597217938,-1.4493400602428224,1.2037323844449976,0.28412005647965366], [-0.4036704221590921,0.8682473056100086,-1.4499044130725358,1.4367312949083602,0.29801296100779273], [-0.7347169576181178,0.8472443819718873,-1.6912964247755202,1.1098267823656842,0.5128488081575056], [-0.4599723079910277,1.0313773063743503,-1.6693647567489862,1.1183374503284873,0.3065899617042307], [-0.5155727886799809,0.9793287424722477,-1.81766244647388,1.264051537384318,0.38002468301378506], [-0.461809330593855,0.94300958630738,-1.7083227759840551,1.2779916838408791,0.5269758739704866], [-0.6518981010977536,0.8745407308399856,-1.5963574332371322,1.2679054180945675,0.4099968212180263], [-0.46480686752956457,0.9217771428753339,-1.5266876398223685,1.1382426745675795,0.4778996312070509], [-0.4852820635337406,0.8218636910073019,-1.7767449598018135,1.1371130292343117,0.33381426065377506], [-0.41472873571993296,1.1622114863170214,-1.464996886657917,1.3123782230466363,0.2979393535788235], [-0.38817680425075585,1.0268858467994189,-1.5893492622912915,1.2621731986065745,0.435168936860357], [-0.4138872105825818,1.0639679744840822,-1.6793705616475731,1.0262313606697269,0.26542788201223744], [-0.32689002022123786,0.9411685364764196,-1.6079547828400877,1.222649045231373,0.44202772328420886], [-0.4534191990015764,0.9019382096024549,-1.6042217752367725,1.313476911051435,0.3118812623960018], [-0.3642257288812764,1.0568954661740726,-1.6408419112357935,1.3879191143662102,0.43428969659022393], [-0.4436033491575346,0.951120701402773,-1.7396961236381308,1.0696513684387006,0.1983721283216616], [-0.4195531379041837,0.9339655248016878,-1.6369085623469266,1.2113780206799198,0.32646374286948043]) -2016-06-13 20:16:35 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:16:35 INFO JobScheduler:59 - Added jobs for time 2000 ms -2016-06-13 20:16:35 INFO TestInputStream:59 - Computing RDD for time 3000 ms -2016-06-13 20:16:35 INFO TestInputStream:59 - Created RDD 13 with Vector([-0.3676131534931217,0.7205292433695121,-1.6148690586461516,1.318613134832133,0.2894000707552024], [-0.63227225792052,1.0480160164806966,-1.7186123125513257,1.3123133303248091,0.4835019323786134], [-0.5130669794355894,1.0744409567693907,-1.5423474727562314,1.4075951590487494,0.34400766625670576], [-0.6214787708363003,0.9941757670924664,-1.7059517928152113,1.1668792946674045,0.5351190865534026], [-0.5610084057337338,0.7736203857280138,-1.8315479385898956,1.3209260735925115,0.28582121198324356], [-0.37376999266386424,0.8253313331048502,-1.7924436246324382,1.3068506619750972,0.2325763177837035], [-0.5087037004414274,0.9607866442106467,-1.812291073591455,1.284458726328898,0.5328516211621773], [-0.3727437082846784,1.028767182687076,-1.7448007272489838,1.0984753757565202,0.3541211398247694], [-0.3814571980697206,1.0819060589675644,-1.733976127510556,1.2515969788209256,0.3640947425626813], [-0.38693583578076807,0.9054649043220095,-1.601255604158471,1.1869423507487187,0.27285330937505237], [-0.453879517976784,0.8265450932182703,-1.696264660015251,1.1219045234195482,0.3528336963815211], [-0.6440396833150053,0.9147072341447406,-1.574634779931242,1.2807375427757628,0.3400101945021918], [-0.5574376128841423,0.9786892277605028,-1.6539643457077906,1.2987195892477945,0.22091568760798275], [-0.3219564568616685,0.7344099605460779,-1.659291949575684,1.3277750279213558,0.23553171253321456], [-0.42287205017678026,0.8870628278654091,-1.7860892614211916,1.3638413188637932,0.35992654268460605], [-0.5114214572125407,1.0739488843585667,-1.803218590871721,1.2580620741259871,0.42523338095005675], [-0.49124237559267386,0.9657816736108445,-1.6156494419042913,1.3063855681095915,0.54155197550267], [-0.3951919316276779,1.144136446459301,-1.6606072772152598,1.4114586343482727,0.3224073809014021], [-0.439470602003815,1.0070997021461388,-1.4975209021956797,1.2890203902600919,0.22540217793727096], [-0.6679727959214776,0.999190248905179,-1.635890493454145,1.2938379558735829,0.20736771134970797], [-0.4118317638697319,0.9069236714993869,-1.7766219570089812,1.2254241772422947,0.3818774093786991], [-0.5019285203332016,1.071358441302261,-1.8681705848296024,1.3543938347082431,0.3694102461708769], [-0.43086686887184694,0.9871654497649167,-1.592715663423245,1.1292659913487786,0.24090456644129066], [-0.6554222411786117,0.9775977489456485,-1.7739871948427586,1.3756970876324615,0.5345682469155936], [-0.606670169993634,1.0414671201540915,-1.5816535152099802,1.2849832898961777,0.3443771799618759], [-0.6330347409363863,0.7614040653882075,-1.6420712005053097,1.344034328413472,0.5019032888808933], [-0.5268140556262974,1.1016458967835785,-1.581552701703922,1.1842999784487596,0.3683353834757425], [-0.49900894372274485,1.0645644739185587,-1.5355753177851752,1.2694091849907172,0.33468156327241705], [-0.31358398021960576,0.8302665811856854,-1.5422704772273437,1.1506707176948,0.37999408831157194], [-0.557673046622542,1.0913399597085338,-1.5871539417007638,1.0478537277843192,0.31421916981662246], [-0.4100320104849945,1.0606699393377843,-1.764930551709285,1.3173125309085043,0.32112514544554777], [-0.2578093453260759,0.9604640936063775,-1.6498739501347253,1.251662741314794,0.5352359752706277], [-0.5830353039132012,0.9147683830431232,-1.747368848835465,1.1923888734167842,0.4792523557272257], [-0.3091820573873779,1.0228642342621612,-1.472857793608509,1.165376819972311,0.3758448993415674], [-0.371190450361484,0.855956409330075,-1.82995473329898,1.3390818705810552,0.37113293274981823], [-0.5539807557276546,0.9677998945369973,-1.6350804443189515,1.3419255190805948,0.44899955987573437], [-0.5376287808107136,0.8413279828562651,-1.6762865574706214,1.19949852014792,0.5047576712678563], [-0.6003249423212255,0.9906964773703084,-1.5370089708944512,1.3072479529851824,0.430528835393197], [-0.4175368422724912,0.8187481538877078,-1.7996629160543502,1.1879999792808238,0.473510867372402], [-0.4828257087231799,1.0612823363596418,-1.6425767546040255,1.2270450796716112,0.39748339044894354], [-0.6332333955565639,0.9519761064158869,-1.5479445620477186,1.308465915443944,0.40088093604627134], [-0.5228584674134156,0.8768992895766171,-1.8336272924164618,1.1499724885170817,0.48668761341650246], [-0.36341836902693225,0.922366126760111,-1.4327493627862522,1.255336229334989,0.29637579738564857], [-0.45752242402693893,0.881182369510346,-1.8075367810499625,1.1836209711736738,0.5240597883932867], [-0.45127422400043055,0.7875661584033756,-1.9374943002165643,1.3022768816466794,0.3865261883663946], [-0.3385546242930144,0.8796112795269294,-1.6285471480854194,1.177302672296305,0.3611750358009876], [-0.3866867752880084,0.9445769802243262,-1.63589592788523,1.3115437594462822,0.30025415854590604], [-0.35087047853321573,0.8048561114260069,-1.6540296199948705,1.1069576251024358,0.3739108460295009], [-0.6118021930146711,1.0836229312504084,-1.6965417516081935,1.3814661848764345,0.5544639396063888], [-0.32579850447352154,0.9627503718472757,-1.7118868486570202,1.3378056459968657,0.26239820187294716]) -2016-06-13 20:16:36 INFO JobScheduler:59 - Added jobs for time 3000 ms -2016-06-13 20:16:36 INFO TestInputStream:59 - Computing RDD for time 4000 ms -2016-06-13 20:16:36 INFO TestInputStream:59 - Created RDD 19 with Vector([-0.4302807096972894,0.8599904088505619,-1.74473044235192,1.4559107913903748,0.3868065070182458], [-0.4809066717988696,1.1355083469098752,-1.568604721573235,1.1620770786476713,0.5052691040344791], [-0.37948367495671453,0.9612314328384234,-1.7288918432476479,1.1900277125124938,0.584148529955661], [-0.4830118079148656,0.767521114064722,-1.626523514538083,1.2344387914004133,0.2949434953618521], [-0.45982168221474795,0.9610478654584301,-1.4181098052958898,1.1960502956938037,0.35754207882888805], [-0.5403734624385554,1.0961985914256847,-1.4824794478647174,1.1331480033700656,0.4391107899439444], [-0.4078934785304544,0.9024846949827434,-1.772449379651882,1.1891755675558509,0.26886721197216756], [-0.4850713563393695,0.9854430300369302,-1.6392713106609738,1.1640485301086894,0.4058728213551233], [-0.5687863647866291,0.8969696803234791,-1.694332984322472,1.263598085578097,0.4916946976192132], [-0.4852057062952629,0.9746523770852169,-1.7152410694648534,1.290070394136892,0.15734923086759972], [-0.6042639508376677,0.9587258437320635,-1.796821218335353,1.2415150801911263,0.5036349732580911], [-0.5634160393957598,1.000927422915143,-1.7677443118596388,1.1056602828501068,0.3662560181172587], [-0.42260218853055576,1.038402090735988,-1.623454196568306,1.258759507363921,0.35817719144396376], [-0.6935865876752814,0.9767485891098648,-1.6119749119230544,1.3241871653009805,0.45959708474651645], [-0.34982681977824615,1.0697929536346964,-1.6430495277049175,1.5160135223968076,0.3720168274191172], [-0.4965183720272103,0.8867675212850072,-1.6326283681810188,1.1014099972707194,0.3464777829375711], [-0.4429523196454975,0.9569586297887442,-1.6975710388611154,1.3240706523955248,0.32888337316923105], [-0.6110986163387908,0.8513837967033338,-1.863122720264613,1.1805944382455067,0.36642329779978594], [-0.5770973903898762,0.908469634420286,-1.5969552518736576,1.2639442528889475,0.30334073173665643], [-0.6677676499752639,0.9993894428543435,-1.7484246181717757,1.3024629995033956,0.5254188589994735], [-0.43541597877213595,0.9518339300904892,-1.586032231723223,1.2859374872876397,0.3728538289653893], [-0.3981355023125287,1.0914065768869097,-1.604054951142783,1.5597686669581905,0.3200687403051776], [-0.3526167305034049,0.8582438505591754,-1.795079875034891,1.291753863011317,0.5193327226327783], [-0.5322224604485944,0.9225091285363366,-1.5406535924270095,1.1740734779955952,0.3985082190639408], [-0.459298613922177,0.8499917798311437,-1.7381604837279123,1.3363117012403978,0.39372367283850573], [-0.5029856763543168,0.9563691025906603,-1.6323146191672946,1.1564017424707789,0.2992277161858537], [-0.4702645909815308,0.9857893178047549,-1.6146159696178013,1.2941994476820615,0.29510677438190996], [-0.5010373585673528,1.0381084928077213,-1.6592946006629066,1.2275293781382213,0.6215806870134133], [-0.34251104673790145,1.0920152105960284,-1.4702129754795525,1.3052389503227437,0.5142842632664187], [-0.6014879705809496,1.0573110467882678,-1.6994508095500127,1.2345579924787826,0.27260709575033387], [-0.374393054126274,0.8520958347461538,-1.6227534538993043,1.400050919842349,0.3749155714488258], [-0.5537306132792297,1.0237199970470363,-1.5996513021748293,1.3277912515047763,0.3298858295939718], [-0.25019038644507985,0.8933568126093641,-1.6180540579171452,1.2514548524704747,0.4485167299028402], [-0.4083941282128968,0.9821808764315939,-1.7710803507794117,1.22246902401698,0.39616717157783654], [-0.6617567254520433,1.135205204672421,-1.7404704566578575,1.2634534028343156,0.48631393245586374], [-0.5102944111175142,0.7749069961072899,-1.3746153795885292,1.1761695768038691,0.44519535098184526], [-0.5273802259607313,1.0293844883271042,-1.8327600208787889,1.3032607095759106,0.4810318980867238], [-0.43680250540361804,0.818212719795147,-1.6397291545525832,1.2164204417976607,0.36911234859465697], [-0.44511271834694277,1.0039279649012336,-1.736482991573775,1.1407721179472239,0.32026695790267784], [-0.461571293170463,0.9623234284494812,-1.608651111923442,1.3054108488252942,0.2808744029932673], [-0.44626644352032524,0.9385912062499339,-1.5784088711897624,1.2996894478753678,0.3367841237735643], [-0.3458002275610911,1.0002670117410046,-1.556743986487141,1.220789398824709,0.3047413342602086], [-0.7006173067436341,1.0226530560056757,-1.7453818996615391,1.2468257997531524,0.43730757861422237], [-0.4921877365188532,0.9365903329931373,-1.6954478906187997,1.252660426269991,0.411305000260049], [-0.5866727516355298,0.8540617355784135,-1.7495419546675461,1.1992666379768737,0.5624798329629086], [-0.4552502733367031,0.9291860448712651,-1.5908193525700438,1.166718738832733,0.3281256456654895], [-0.3426613548769225,0.7538960431270546,-1.585524926597853,1.478740164978157,0.251131880834575], [-0.4680826818575833,1.1112444944704614,-1.6587653968723466,1.2870287092936261,0.4050920708933554], [-0.5624024717932173,0.880862871462712,-1.6303493150007178,1.1721463768303386,0.5696913393311782], [-0.48266079841446297,0.8916088275950524,-1.5393517554424276,1.1897790454674049,0.4411535715241579]) -2016-06-13 20:16:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:16:36 INFO JobScheduler:59 - Added jobs for time 4000 ms -2016-06-13 20:16:36 INFO TestInputStream:59 - Computing RDD for time 5000 ms -2016-06-13 20:16:36 INFO TestInputStream:59 - Created RDD 26 with Vector([-0.3816421005537525,1.0617035428188992,-1.7399461177422264,1.332980235806715,0.2916494841747297], [-0.6583584440016957,0.9555891409969888,-1.5454326525102315,1.1741091058996926,0.305220337835565], [-0.2989177604038451,0.8701688291879833,-1.8804582484676076,1.2986162711936382,0.35958131004690164], [-0.4939772883087695,0.9969220817446861,-1.5980382534286326,1.2969252872376784,0.37976918116852443], [-0.35668565131454827,0.8078147339434413,-1.657193160993533,1.359095599717262,0.31192105888186633], [-0.5450865703006346,0.8750391278919141,-1.7453306059026725,1.3721329076982913,0.42237633518141116], [-0.4917580165015238,0.9512146417235625,-1.9326523267650781,1.4266827266379667,0.3225713722197544], [-0.4048718474607664,1.0240256182146898,-1.6774401089659943,1.2669846165410654,0.33732641244233585], [-0.47579982131353826,0.7947369494624074,-1.7655419119652316,1.082577012553155,0.35401987740372537], [-0.35538929020910864,0.965963799724578,-1.7529404087985827,1.2055043918858341,0.3342486683177578], [-0.7014541659835856,1.034642549879647,-1.5544540159626516,1.3033886639865033,0.4054677613509847], [-0.43610110990224626,0.967077951705683,-1.5262368892289788,1.2639835242537958,0.3181179758885862], [-0.3425524324739105,0.9894095680520077,-1.7777745234498326,1.3091660388427224,0.331841648562285], [-0.4878205400579338,0.8560103963724833,-1.7352326850283464,1.3664274889095067,0.3984111315135807], [-0.7121859062829006,0.9130040407010276,-1.7874802517046422,1.1874648639293004,0.45017886399842666], [-0.696643078139105,1.1467663562988957,-1.7296410056290408,1.296144064682525,0.5048605187772839], [-0.4011230886898419,0.7361859073107719,-1.5317733987729902,1.2701183330425934,0.46062351610078545], [-0.45715694326129586,0.9190057703920946,-1.6904193500197242,1.3271044644288563,0.4842403797714084], [-0.5095249256026796,1.0659631362797184,-1.4402626928522713,1.3379987286264212,0.2673039786797317], [-0.4167094825032959,1.1091402127599272,-1.705921687753808,1.1183411370664753,0.48305727426817385], [-0.2924464534867928,0.8213969946455459,-1.662853823551024,1.3355427142685694,0.3770025002372693], [-0.4444725119583886,1.1046045679759258,-1.456164663000235,1.227847061522124,0.5255642346814823], [-0.4048329834921757,0.9788094275209757,-1.539852245269681,1.180160780953293,0.6787253334538687], [-0.38022990049474503,1.0144858953551337,-1.8072904612189131,1.3419776986257266,0.4041658441938728], [-0.3763555278351052,0.8998086161617835,-1.705061140148122,1.3495005676173746,0.5191631674639813], [-0.5540229100290982,1.0669438671830174,-1.636684794385797,1.3459012222977915,0.4181134505939225], [-0.45042550789662783,1.113240852339999,-1.486260571622101,1.2719922983443672,0.4102349177516307], [-0.5546179681493353,0.9938406825700987,-1.7084301934096207,1.3880541591389144,0.29690506797892435], [-0.5006405047598609,1.090292575906863,-1.7104271793907637,1.3062089077771892,0.47556018075456363], [-0.36520205819569046,0.9201397468605166,-1.7602021590969437,1.3876082296931265,0.2308718010458195], [-0.4938168714972593,1.0964903470900649,-1.7260349751744855,1.2048249569339422,0.40946925790527117], [-0.5092411838541296,0.9882559808204627,-1.7212908932580275,1.1127850487389395,0.3940121165600333], [-0.5882412062939182,1.1156935762589453,-1.8131619026625396,1.4146422480126442,0.31361816553861704], [-0.46990162437316874,0.9908156946435108,-1.748827835513362,1.3957588501434537,0.5512480921903505], [-0.5405868391855275,0.929247220281523,-1.5894550246414005,1.271255300937136,0.46529614772048844], [-0.46376208782353645,0.8174067828364345,-1.711422288030576,1.150388298327176,0.36268778857829526], [-0.4617415170232849,1.1203248336099854,-1.9346820049945674,1.1466858815338403,0.318614186382022], [-0.5200553635156823,1.1206383922863123,-1.721393160577411,1.2564342218266722,0.39290287510750027], [-0.45474634000888675,0.9538310968737261,-1.5572305892194418,1.2293659662686987,0.3951581444726625], [-0.31024997329419923,0.9626097763501761,-1.5774105560033171,1.3323705378196256,0.28213579823419327], [-0.6425546702182562,1.0413861558054223,-1.6696029184789263,1.3297020370998587,0.4667426127538354], [-0.5088483797878551,1.2170009632967245,-1.7143214767417212,1.2915402658283741,0.27813229259982863], [-0.5138048643277845,1.0537687232096613,-1.5306968187748822,1.2020601237200264,0.27213168784492703], [-0.5375432544160766,1.0418784698921373,-1.501800504998784,1.217708599285121,0.6443267111319482], [-0.5784571285851817,0.9518620188834632,-1.656094482882889,1.2785669143560554,0.34103467230048085], [-0.39057405488844044,0.995936983645342,-1.6323671359433234,1.271155031008547,0.3558239098317178], [-0.3682240577265206,0.8711375369867465,-1.6625627384453794,1.1969170451325948,0.47233733398400285], [-0.446008720311628,0.8976699326654971,-1.5289286019943875,1.2244855320582673,0.2984778353929738], [-0.5102538395816835,0.9986885141123109,-1.6414035100650266,1.2420020420446427,0.36032207135032196], [-0.6350165259371148,1.0900186871655364,-1.7907186442371252,1.4304360117451347,0.35387005412887984]) -2016-06-13 20:16:36 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:16:36 INFO JobScheduler:59 - Added jobs for time 5000 ms -2016-06-13 20:16:36 INFO TestInputStream:59 - Computing RDD for time 6000 ms -2016-06-13 20:16:36 INFO TestInputStream:59 - Created RDD 32 with Vector([-0.5768797964115748,1.028505413551091,-1.7521414076692798,1.410931775189879,0.30527264662469095], [-0.40591177777424625,0.8661878776717759,-1.7310200236861213,1.3594055750354543,0.3299548132445285], [-0.6383561683498007,0.9259033203935216,-1.6881320011035645,1.3172994752602565,0.3214908371039436], [-0.624062828319168,1.1219523756569185,-1.5603263506923595,1.228964619537528,0.35487057209905903], [-0.4734064808910865,0.9649955190976767,-1.6068693471541338,1.3196307767213002,0.37116726129033706], [-0.6101841770121739,1.059654359501757,-1.8099667264246553,1.3759685981009824,0.5438261473052933], [-0.18850356552447423,1.1440264277756391,-1.724865192544724,1.1359044353122627,0.31757393897273056], [-0.41254200355821263,0.8607204799101216,-1.616620448535661,1.3427363220425623,0.4109728277211949], [-0.393734447592108,1.057690953958827,-1.6695747980566427,1.327034390345423,0.13819106599587025], [-0.32090427274943434,1.0651687359751123,-1.6980368198411557,1.4017902860954718,0.36022562482340503], [-0.6053061980873911,1.0041632375635317,-1.7820663797212062,1.2269162643461793,0.4840114520476853], [-0.27268808986968307,1.013170380627259,-1.6005981535949472,1.4382064950763866,0.3892224005152082], [-0.535422550377724,0.8375173015602825,-1.661788348017347,1.3227481832299182,0.311533159171815], [-0.4009368458319039,0.9897832871295871,-1.6696688300152116,1.4424037410980737,0.32734398317968916], [-0.6825087101818776,0.9056073621397804,-1.659854864696752,1.2759644672519115,0.43884955280528926], [-0.43179250129551383,0.7465728726494555,-1.5636729803345486,1.3457117728614754,0.5721971492726551], [-0.526304418227463,1.0501002903902692,-1.5968858098432406,1.2982785754583357,0.37494021304786934], [-0.4320225453412649,1.1715904807145918,-1.5178148547272063,1.348200261386646,0.40779395798035983], [-0.42221995667102147,0.87623418086733,-1.73837906100808,1.4183919259975142,0.33228674704284467], [-0.5629520941212462,0.944212875923361,-1.5546071736198348,1.4519582067498829,0.262200899044119], [-0.30845271114048184,0.9681639591372486,-1.688842111249905,1.3763468548667668,0.3173125681909059], [-0.5198567190391367,0.9335053916544206,-1.509472274649017,1.454415814887731,0.4589687217941236], [-0.5711775989363378,1.0215061238027188,-1.6224000734110398,1.350665861834858,0.36668462127595286], [-0.39949682119974983,1.0408092246134175,-1.6587225284400737,1.301865863792736,0.4563742176523199], [-0.4902566261487769,1.0992586451801099,-1.4473476422759635,1.2309123590548212,0.2544829822903507], [-0.5293628624912633,0.8432327849564988,-1.7669603672683885,1.2911210280012069,0.41983427336223667], [-0.5738614502843032,0.9784937768658387,-1.7194271904022407,1.3116656980237082,0.46218317857671776], [-0.5902501988138692,0.9423215775018176,-1.684040151093259,1.4235274237923208,0.2039286490216381], [-0.4777727934676478,1.027586212069216,-1.7495753442806758,1.2757769414973703,0.3810309831295745], [-0.6566621783684151,0.9641681686268665,-1.5286420868141977,1.508411206136107,0.2501550388463756], [-0.46296281092785885,0.9727194845614605,-1.6171422039154086,1.3434828829348795,0.07809723939214058], [-0.5738547760098169,1.1077074999997352,-1.739110700752265,1.3494171899976157,0.3222123114107347], [-0.4002887849620737,0.9619411680270862,-1.663509769541054,1.2908149710547212,0.4244227376717664], [-0.4665533215112465,0.8596395857982323,-1.7467795346659782,1.251224446006115,0.42794754034954735], [-0.6374258626266393,1.0514903728245393,-1.651303971310752,1.236884794719695,0.5249538337308562], [-0.48340361610998744,0.8198136168919055,-1.690727257833199,1.398396991016531,0.31841739933411595], [-0.5410633313507479,0.8973900953721683,-1.4724867408929736,1.388151513303225,0.47893645155994446], [-0.3039096405457719,0.9155005236200422,-1.7173365893442798,1.256097860517092,0.24819750391337136], [-0.7090962179115853,0.8944749271173439,-1.8839838110448957,1.1446445350131944,0.22840975233443309], [-0.5100953498526793,0.8107010460411471,-1.7303477866873604,1.1647356509362308,0.4418371093135496], [-0.2962087734742913,0.7242125014742148,-1.5984563979400566,1.0588259317312654,0.28436170748337264], [-0.5505122810716748,0.7806970076803257,-1.72088933624607,1.4230022424760869,0.32673184311928194], [-0.45127942352855155,0.9959060503010477,-1.5524186364009327,1.2366972856510523,0.3578155222046753], [-0.5102066277582917,1.0949182323987419,-1.7142222646495175,1.1664611293679565,0.3602965879997574], [-0.20421136622466618,0.9866371147144095,-1.7538283133815977,1.0465139384173783,0.41274915410898083], [-0.3678643518086112,1.1128928060624481,-1.7416958328862653,1.1712943420281356,0.522586652590277], [-0.4272255324098842,0.9505411417476373,-1.5398631578697235,1.1670050581250606,0.18826195051488878], [-0.6294443147944699,1.0182211296265207,-1.6114292740208946,1.5494854908967275,0.2354819914065145], [-0.46052379116984477,0.8965751438720174,-1.6887957613207942,1.2678155037585463,0.3277249457164221], [-0.4661555388594827,1.0176554636610373,-1.660202673672577,1.1543960943838145,0.41490582984943747]) -2016-06-13 20:16:36 INFO JobScheduler:59 - Added jobs for time 6000 ms -2016-06-13 20:16:36 INFO TestInputStream:59 - Computing RDD for time 7000 ms -2016-06-13 20:16:36 INFO DAGScheduler:59 - Registering RDD 7 (apply at Transformer.scala:22) -2016-06-13 20:16:36 INFO TestInputStream:59 - Created RDD 38 with Vector([-0.5184134212251081,0.9455318561030377,-1.7249072888539423,1.2476376390078272,0.539371371249466], [-0.4839353019153916,0.992343730650464,-1.4037670310964987,1.3650053852483648,0.287928402259341], [-0.45217444521287575,1.0425364789095595,-1.7771813152553952,1.2241390773159404,0.3053067128704871], [-0.34943560877945984,1.0407179160044182,-1.5177836780575653,0.9824846085205396,0.2603543256712868], [-0.45829448575134074,0.9725249117979422,-1.690056167350807,1.035512584568622,0.3026754916713263], [-0.42458145588018753,0.7854084174090814,-1.7649589009106188,1.1975914903392633,0.6551885852829413], [-0.5104497237497339,1.0402201914534017,-1.6859518565969145,1.1391556996625576,0.18881595045601088], [-0.5588856532488075,0.8709355475677711,-1.4735465340859104,1.207362168238276,0.5886898848511445], [-0.5286914346418723,0.9318873502365327,-1.654993790470044,1.274056873049218,0.29412525529417294], [-0.5420811906362676,0.9995645356070102,-1.6040227637757893,1.3412654820039975,0.3950459929854693], [-0.5105371229433685,0.8870982484234274,-1.6366208840493932,1.344244897498087,0.2842199976348651], [-0.5758159328886613,0.9361950961962593,-1.5151431024376867,1.2963264217469896,0.4195802203163277], [-0.46650906956232724,1.0883479790237522,-1.6209503836486092,1.2506119015001946,0.5239087792157935], [-0.49800785133272085,0.9121648111767798,-1.5664537098618618,1.4303891258276538,0.23796160092517643], [-0.6200401622139905,0.8819930763699753,-1.7712553265237971,1.144151394858828,0.37228012783293096], [-0.43683215654432145,0.8386127968983885,-1.6402554451670102,1.129209493050289,0.3612473529185551], [-0.4079221564079838,0.9656864104202829,-1.7359335721136713,1.4421338682074127,0.3616284206434529], [-0.43442591279896603,0.955449363348641,-1.5334677832140913,1.1666726886559795,0.5058368495019173], [-0.6457828019918121,0.931131898174338,-1.8532037937152797,1.0375316080434018,0.3133563527606291], [-0.38369598973754027,1.040856056086344,-1.7594025439463896,1.4917751006832976,0.45495744983872133], [-0.30166559339288085,1.0342180102112146,-1.5650152370405155,1.2673629485315197,0.4158807517084573], [-0.28580825731967757,0.8865395334100423,-1.4737235251638032,1.245008984525024,0.36435255784505255], [-0.34071421524008416,0.9427461052254856,-1.6571555985762556,1.116205755573728,0.3440594901972376], [-0.5707704618700846,0.9955277694976673,-1.815098893846346,1.2976867834380266,0.3708933869585827], [-0.6095164341318,0.9322989548597498,-1.6576644540187038,1.3001089252401639,0.32781904617649943], [-0.5884784232045771,0.9963121018549773,-1.4726908596796255,1.1232169328419297,0.466258074909203], [-0.6351651621700529,1.0173421935079714,-1.7981429560407272,1.0336059974520695,0.28162584897435694], [-0.38203401021631267,0.9790771913538491,-1.7244683271662424,1.288084387752797,0.4312827591331814], [-0.37183909143628785,1.1926117173870312,-1.5166716166223042,1.1696263300990144,0.4515586032102953], [-0.47462520704983,1.0498070539706301,-1.6863851371590932,1.2965114835042095,0.4645837570114517], [-0.35874130630231327,1.0042819076442355,-1.6407835501913508,1.214144240421076,0.35534792382876684], [-0.41389636789198914,0.9508170625312302,-1.6498089027313338,1.4272197180381103,0.374553819362672], [-0.32132785261092967,0.8535769504185499,-1.672835438070241,0.9421132176631009,0.24740658936307364], [-0.48253792736985773,1.0429236941137325,-1.4939460450129358,1.2832409419178106,0.40323476275374437], [-0.391512958705594,0.8859909853510953,-1.607045573914962,1.1770226037327562,0.5198479967492], [-0.4833163026675927,0.9200058086737815,-1.8188555219632765,1.2111735788924018,0.39618714901430313], [-0.2913135273832381,0.9301485161935364,-1.6975841978124846,1.4065399033140502,0.28453881400648173], [-0.3073756680829076,0.8325242716417236,-1.6753209327630654,1.4075134801045086,0.30351866480787215], [-0.2775051790445552,0.9452680429408632,-1.6224110577930677,1.1778711923142833,0.5531745712889398], [-0.3047686297259775,0.9468967188067203,-1.6929112749188848,1.4636396431851675,0.3586449109599954], [-0.5956899619218391,0.813236818549011,-1.757649523640403,1.2604654777105053,0.3185144577112101], [-0.32794499152459505,0.8140339815697406,-1.8666906537843855,1.2572887839007447,0.23549894606903882], [-0.40686690942678344,0.8619998816707273,-1.6319579560604098,1.3664940728963186,0.422156696202466], [-0.4138601150243667,1.021828823586237,-1.54841612428864,1.2398622089963618,0.5070042357490803], [-0.331869975870002,0.8782404384527787,-1.6635875859844171,1.2435143760020457,0.32101906837863314], [-0.49700214293681766,0.9856184010338479,-1.8232422669575907,1.2421051455581202,0.47592591320323163], [-0.535909591266969,0.9807383109826389,-1.6974325972167972,1.334548321075211,0.23946350824403165], [-0.6444021824207316,1.1849780210499694,-1.5121795557147575,1.3604386920675193,0.31690181205044465], [-0.49746220541003966,1.0643703042656938,-1.8499798436118189,1.1357972819777942,0.5780379400362996], [-0.4383285608503117,1.0163805234573544,-1.9030246206122645,1.438368352860807,0.3293850463477472]) -2016-06-13 20:16:36 INFO DAGScheduler:59 - Got job 0 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:16:36 INFO DAGScheduler:59 - Final stage: ResultStage 1(apply at Transformer.scala:22) -2016-06-13 20:16:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 20:16:36 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 20:16:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:16:36 INFO JobScheduler:59 - Added jobs for time 7000 ms -2016-06-13 20:16:36 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[7] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:16:36 INFO TestInputStream:59 - Computing RDD for time 8000 ms -2016-06-13 20:16:36 INFO TestInputStream:59 - Created RDD 44 with Vector([-0.374705866875181,0.9840691262184086,-1.5891551104970965,1.4814595125988181,0.47065837960007695], [-0.288810885883263,0.8437842751113669,-1.5395954061634287,1.1874811761270259,0.021249409500126437], [-0.4606335450965845,1.1012472510905122,-1.8287426699010534,1.24172318938614,0.4710333368456244], [-0.42944940507321583,1.0104988720375858,-1.5751317017658801,1.2831547194950401,0.3512701703197814], [-0.5152127056418667,1.1709583832128079,-1.6554391510043867,1.1397001494774806,0.23397277605773995], [-0.4399479870409779,1.024335104965922,-1.7045108897162344,1.196833551305392,0.4117115230011916], [-0.5430543625505585,1.1370095574712402,-1.7176429060334013,1.1901279759925694,0.46070443041212095], [-0.3968714897698767,1.0926860330748331,-1.6334286394688688,1.228234808051336,0.4665607383628074], [-0.46215342653947683,1.0343329141616782,-1.6499943761334228,1.337530812749641,0.23114556003826187], [-0.5000235665427208,0.9151728338053156,-1.8055132514640995,1.3037686620331144,0.2902025184248946], [-0.309233049825866,0.9728959274674948,-1.6392691162173578,1.101247705752869,0.3496423465401415], [-0.5981691090699208,0.8739296001671861,-1.5687282698937923,1.2550952881121755,0.46308264317451964], [-0.425561195913103,1.0676958602044149,-1.7135165364767921,1.3524692002054561,0.23228520033456318], [-0.7043177723021486,1.0283311625585259,-1.7478644671187442,1.2858120358708933,0.3418710936510813], [-0.41727892194293686,0.8625867727705405,-1.7914100057674134,1.1786792695315231,0.4917512427338196], [-0.40174243931692544,0.7828098802704966,-1.5073762714652954,1.281118669072956,0.44887930601066905], [-0.35310979651234775,0.9745364825956087,-1.6392317059509176,1.4513644518189859,0.3110119088453469], [-0.6853829628094648,1.1253397147829511,-1.4920081564639467,1.231135544811062,0.3139314829419366], [-0.44152016739085975,1.1304790288137783,-1.7325115907360644,1.3251089521914843,0.479567851123668], [-0.5526275004039871,0.9674440446022308,-1.7971453051937036,1.1664342514118335,0.31260632993291754], [-0.46562594489923503,0.96412818249357,-1.7756506925990838,1.1820367323525278,0.37368762092821634], [-0.42414895043860573,0.9369437276073684,-1.6228073393039317,1.2110129935336054,0.33600782254298567], [-0.4507513844294476,1.008422902581697,-1.6947838717172434,1.1258385535052762,0.23649573041213182], [-0.5011767543824581,1.0367482963168666,-1.8619227324329928,1.187686906059264,0.41060169959227016], [-0.29061733919433663,0.9125980333805904,-1.773384852821433,1.2318813347419482,0.5360023941871213], [-0.47558633395179245,1.0112018987992073,-1.6443927870523196,1.1868477340320063,0.378412637239359], [-0.47217857076866115,0.9027887018339104,-1.7359968088612976,1.372505255254142,0.4024631611010435], [-0.5635782393384723,1.0418520168542464,-1.6371483392749435,1.3046794831870752,0.36435743882876215], [-0.3566584096699375,0.9045378099174617,-1.6027843810019622,1.308376882406936,0.450420525347206], [-0.5161548558091542,0.9362466126941537,-1.543853038663444,1.351176902637456,0.37019671664199505], [-0.37734135089435306,1.1354760893808167,-1.5928707071728756,1.4230797310043477,0.5395830646701685], [-0.4935941975274797,0.9431897551602951,-1.7363999282764386,1.2592777434544042,0.34477211805728875], [-0.5399690948357754,1.0773952946269298,-1.623286131549866,1.3571017009557331,0.3820149575212495], [-0.47470272833792704,1.0279092716939924,-1.5583033476707477,1.2928147173688327,0.12854158780432537], [-0.3932748299826361,0.9168968642381998,-1.901294092833649,1.259188794957511,0.4011688713714673], [-0.3557493574970649,1.0003577059585267,-1.5111904842067994,1.3123818859934535,0.3019665980708428], [-0.38849198387448786,1.0634537152100978,-1.8742015500852314,1.207616787308903,0.527158124827907], [-0.32516837051884395,1.0806577326663074,-1.611781542302308,1.182593379846847,0.5196789294606137], [-0.5212725898858087,0.998490506190397,-1.6184763965153202,1.3731447165706872,0.29855378648402087], [-0.40257752034798155,0.8314340557777532,-1.6850156321125218,1.3427297403898506,0.5156164947536159], [-0.4889473969173199,0.8999745071099362,-1.4468367122261747,1.3090812041423647,0.5108182965913203], [-0.48274282778056277,1.1537075465803446,-1.5811276311378704,1.287632315416058,0.50882779799236], [-0.23362208323850905,0.9548750423156142,-1.572161725609678,1.6027003014260315,0.3939518404103326], [-0.4292988792888861,0.9366381149644413,-1.8250064321036654,1.2358072562830058,0.18892247590763034], [-0.484748914270206,0.7653236279449509,-1.7486834183984747,1.2949268753185315,0.39683334762547673], [-0.4399736279490563,0.9634644976145054,-1.7480413166448512,1.1775955441636703,0.36332416278682794], [-0.48373350642316953,0.9259530051332495,-1.7977337219360736,1.2876089889322726,0.4497462132788797], [-0.38070426405120866,0.8838236645507052,-1.6753928124899833,1.2449514905082557,0.3276558558550576], [-0.5717833110401351,1.0273299096906752,-1.6190879043151574,1.1860455839092494,0.32636997750947594], [-0.5231330958628938,0.924256801489448,-1.5484246875687981,1.3381528852011608,0.3757180006852602]) -2016-06-13 20:16:36 INFO JobScheduler:59 - Added jobs for time 8000 ms -2016-06-13 20:16:36 INFO TestInputStream:59 - Computing RDD for time 9000 ms -2016-06-13 20:16:36 INFO TestInputStream:59 - Created RDD 50 with Vector([-0.4026578572929527,1.0001062834205203,-1.5617908885948877,1.462733521830009,0.2061287167736916], [-0.3965287353033582,1.0683726645043978,-1.4873730061169563,1.1460792990307642,0.27746130384846085], [-0.4938197206621628,0.7709438014433136,-1.6076436749339549,1.0982028796258234,0.24486596655062928], [-0.4777650011847253,0.9489155740228515,-1.476414143798642,1.3062514135704215,0.6114439757268249], [-0.4285977940802228,0.9382568588433707,-1.48976698594476,1.1698457774900977,0.297712575086562], [-0.5993896749484521,0.8629058534578911,-1.6510698754517374,1.2090117714723978,0.5458042851487852], [-0.5321474831529757,1.0849054231681536,-1.667568850178397,1.4161664775537062,0.4493689535824759], [-0.6723705689897437,0.7517542425048965,-1.663645669684176,1.2240252996527488,0.346848907683753], [-0.40115735788941104,0.8163474979541504,-1.7694648704001712,1.1073947331922183,0.3856322637520626], [-0.4787497898360035,0.855418622108937,-1.7422359123781868,1.2371448524727167,0.5156131237949001], [-0.5321048883779993,0.7970778523677844,-1.7616773188005117,1.3821414318824692,0.48832517040159346], [-0.5285360880098152,1.0597395124384508,-1.6417892576104653,1.3464020747697325,0.31849105791627297], [-0.5324435853239631,0.9841563147023031,-1.5943852209495275,1.2707723748060116,0.4139910360672806], [-0.37531742848811983,0.7874289125205598,-1.6479869931712725,1.3181636612914271,0.3718789043065717], [-0.4560433134330382,1.0759812484577316,-1.697865869705092,1.2725177127861855,0.06883328163775021], [-0.45513942313770217,0.9498671339799809,-1.6933930804659583,1.3025139463978586,0.5350768600248874], [-0.526256522617052,0.8727003678813281,-1.4356120595661468,1.29993914263913,0.43387328603333664], [-0.3923483046111808,0.994060772829843,-1.6451266300509706,1.2983503204985913,0.3703765349262901], [-0.4374913726020475,0.8955789511391181,-1.4728897306445088,1.1357862983779858,0.30881407016500095], [-0.41951697820602707,1.0007695283696842,-1.708286281016387,1.4190975260604537,0.28731659836668877], [-0.6750333217393494,1.0688779823591679,-1.6846746627252691,1.2962992487600882,0.5030936797752793], [-0.5956866906574221,0.8121299274245898,-1.692929629203368,1.2338548732523056,0.38371772482125543], [-0.4303591738987757,0.9459620785706632,-1.884551120494729,1.4937212054647857,0.415792749586519], [-0.45504157760203423,0.9027897906482127,-1.799761299703666,1.3040399621270629,0.5736987408004388], [-0.5593669081347996,1.0931782446321396,-1.627955485174887,1.1391759184235553,0.39801309200504786], [-0.3825955534231694,0.9655263256725013,-1.7068123920171723,1.1861538643513039,0.2764471843224494], [-0.3587313623839552,0.9371265382727668,-1.487908236071683,1.3184799953634567,0.4370396706398445], [-0.475038660091078,0.797460346861311,-1.6835937249660264,1.3120302511529962,0.17031892070731325], [-0.4663190378327658,0.991352295299833,-1.6444316855302834,1.51526349576866,0.3929422369724258], [-0.5040823137977553,1.0575372259159488,-1.594322659684267,1.2971698194040933,0.5092825217956699], [-0.44790417664079346,0.8518362899333143,-1.648211585312696,1.1839806869130056,0.40832085080824754], [-0.537755895075537,0.9806637869508009,-1.7656906916731008,1.2433660326767015,0.42887767735060545], [-0.34293660736335746,0.7935119912222357,-1.6359022010243611,1.3593857228798436,0.3967833959145234], [-0.6208695835786292,0.9640100782092327,-1.874229161549032,1.343981009171891,0.20123531286937832], [-0.6723717817990262,0.9002960054466735,-1.6910864401441592,1.1818255352705958,0.5433986930426408], [-0.4588708699931968,1.2092994554375371,-1.5324768096769932,1.4083576723657105,0.3708605878943178], [-0.560638238383703,0.9092570275383438,-1.473047966472027,1.3497511209299733,0.3467764467476322], [-0.4409803705877342,0.9370238683801353,-1.572432943897284,1.41498798111251,0.2748026192628795], [-0.38741643074715315,1.071718105146961,-1.624146896559199,1.1401522638836572,0.3872910322155703], [-0.5140862086906481,0.8211106116047733,-1.6027037015807561,1.1677747630844688,0.2659604974429464], [-0.6693385873907269,0.7554393139905898,-1.7108087289854104,1.09991932687034,0.41821242261960917], [-0.3282503845126481,0.9257033715775067,-1.5235939065144024,1.1889128563024831,0.3368716615939142], [-0.5366461042406446,0.9240666515119298,-1.8135909127248846,1.3575511869067935,0.37505000938261085], [-0.39294848413804057,1.0124585311856609,-1.575086388417685,1.2467079941490156,0.40416560148386643], [-0.48738816985096806,0.8787565657383682,-1.810281961839409,1.3779890105636297,0.46019825317610896], [-0.36592476270170865,0.9132457437551841,-1.7089939281056865,1.1718159447241387,0.37244792446782193], [-0.40751855185193425,0.9541054981051484,-1.4879650472104635,1.1817057016562598,0.44919665548504983], [-0.29317450049605076,0.964443479319753,-1.656028808081747,1.2586398699632722,0.2526740377633023], [-0.4615968319838038,1.0243528061825597,-1.5054257187951623,1.351376529833804,0.42971544189574107], [-0.47988177538312266,1.0112860256804128,-1.5404476412782737,1.233668528335028,0.2028012939752821]) -2016-06-13 20:16:36 INFO JobScheduler:59 - Added jobs for time 9000 ms -2016-06-13 20:16:36 INFO TestInputStream:59 - Computing RDD for time 10000 ms -2016-06-13 20:16:36 INFO TestInputStream:59 - Created RDD 56 with Vector([-0.4889010904178065,1.0197902904388123,-1.6434083794608938,1.2273894895374848,0.29880056758004836], [-0.3690054860756016,0.9083136794567547,-1.5975355891184535,1.1937534562371308,0.3514272190683493], [-0.5737783702048262,1.2273536117223582,-1.4617006566984774,1.342244921629087,0.3377586288639175], [-0.48387316003666864,0.9673070026559238,-1.716725721249248,1.2214389230026372,0.37779842180031725], [-0.3777277713061831,0.866118273628028,-1.8487356572074127,1.1851350383431707,0.5215534466266145], [-0.6076926717533072,0.922880686625322,-1.7313060678714964,1.3632488420681186,0.3176699282425657], [-0.3802374789807806,0.8439055679755229,-1.6219613687183736,1.1809867291088374,0.4174630121979538], [-0.567466556581487,0.9292300369161763,-1.5542588287823018,1.3568593851021735,0.38458895772840795], [-0.6764387691659441,0.9816754055716941,-1.6652926928722982,1.3324217076474294,0.2844320904428872], [-0.584138124369403,0.9651833514289697,-1.7100579024456408,1.3025648636881102,0.3575320807936941], [-0.45548159931880944,1.0181220363355081,-1.6506653550843506,1.1304474598171783,0.4285968106743953], [-0.5370002084128622,0.927571736048293,-1.7228464440630433,1.2925613214036213,0.3169164091166503], [-0.47996613069291666,0.884366449915093,-1.7212355955272085,1.152597880658151,0.43962468554199663], [-0.35493034949860885,0.8984344817504613,-1.65218132155661,1.3068108975806394,0.354531357717713], [-0.43226118157508875,0.90725353631203,-1.7628881371748841,1.314870897760879,0.40214500076142384], [-0.4387259831220203,0.9621751931555681,-1.7268764217836796,1.1093705777029121,0.17867972251315242], [-0.44765902746822306,1.1186404105474657,-1.750942737318709,1.322673860044468,0.41609066577279374], [-0.40351581789921565,1.046340578817358,-1.7120920286690795,1.244893797129159,0.3833023298379642], [-0.3093624994668177,0.8356650811564297,-1.8642891852543533,1.217524168702485,0.2103617986137151], [-0.2990644638630094,0.9070655034040485,-1.6094469406493193,1.1592980925831848,0.43244854200109223], [-0.5596021456993305,0.9019708087164819,-1.5105356278212536,1.2598551179472526,0.3428536586628008], [-0.583393764986965,0.9748568871770861,-1.5302538126845806,1.301546791122464,0.21927558650218368], [-0.49772324825286274,0.9716299176892979,-1.6169165069063722,1.3531471880552017,0.25858373282508224], [-0.43106771684572104,1.141003137647425,-1.7578104686857339,1.1835903210552496,0.5254045668739125], [-0.5997306192424074,1.0040726704610177,-1.8227253165238155,1.2317004811683654,0.46876771655575034], [-0.38252412898165267,0.6654527000305275,-1.6468024292953505,1.246225339488523,0.3090133585357682], [-0.44821229943324725,1.0038256342470713,-1.6968551076177416,1.4529627079929077,0.3454589917432014], [-0.43535992691090786,1.0110816584854143,-1.7453606983094134,1.248210478232135,0.2891585839680707], [-0.3652348315036368,0.8958964249587966,-1.9606863598829045,1.3389257198065394,0.2618900608154475], [-0.49376528406043635,1.1421599774098976,-1.7157624036331927,1.391772429187865,0.2876879628208894], [-0.4515063147230089,0.9428403433366573,-1.76394465423704,1.203639940182186,0.5058415243772909], [-0.566229150629525,1.0128616873896932,-1.6880365432069786,1.2877554523210368,0.1629872361106314], [-0.47658344463796903,1.0862736976313303,-1.7934284966158631,1.2272771650263852,0.41680211738171946], [-0.3618808351944353,0.9417137306142976,-1.8600076455984111,1.2206352320840592,0.3461749973804146], [-0.6214757024643818,0.9952948941676851,-1.62097329834717,1.3958184351557885,0.46499512939060494], [-0.35996348236796494,0.9954644862470238,-1.7933270864489543,1.4005533772243737,0.5639356774206108], [-0.6006970016720174,0.9686450943470077,-1.7310975948083858,1.2398543489295077,0.4483279499267394], [-0.4592797597414643,0.9631529442377594,-1.6797024055855931,1.4289800788354718,0.28667160760155463], [-0.32700881076504584,0.8147754680205297,-1.763026614596352,1.3194735072019403,0.36816545181467747], [-0.4305345983795718,1.131285559002078,-1.7583813300937976,1.363545141597296,0.2601165464857354], [-0.448497878644767,1.0701920363594992,-1.6308801358271674,1.2674713459930484,0.49959128317905865], [-0.3686293381026533,1.025599917053419,-1.5433629968114364,1.2801178859900646,0.4174115399252367], [-0.574113693976958,0.9757976338077295,-1.6388778822719372,1.2759164798546365,0.5542142157229283], [-0.4357071400660946,0.8338147376622728,-1.8285888680646112,1.271233025944097,0.270178918164786], [-0.47158188300536635,1.0122183541160776,-1.6010520252577112,1.5447804629514494,0.3533722934153737], [-0.5307926544952695,0.9107879044274507,-1.4490342775396574,1.4031526476841614,0.3809780658985748], [-0.4953668915084064,0.9298227962550192,-1.5651812648587027,1.11568589286655,0.4774983034589586], [-0.7383552095915331,0.7626241115292196,-1.7008182872047266,1.2502877442685654,0.43135198632241045], [-0.539114438923603,1.0489102632469824,-1.6736818095705865,1.2768121659352345,0.2213641562753643], [-0.46002266280222004,1.0294295650038185,-1.6033579773810744,1.407522155806443,0.5223722922756932]) -2016-06-13 20:16:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:16:36 INFO JobScheduler:59 - Added jobs for time 10000 ms -2016-06-13 20:16:36 INFO JobGenerator:59 - Checkpointing graph for time 1000 ms -2016-06-13 20:16:36 INFO DStreamGraph:59 - Updating checkpoint data for time 1000 ms -2016-06-13 20:16:36 INFO DStreamGraph:59 - Updated checkpoint data for time 1000 ms -2016-06-13 20:16:36 INFO JobGenerator:59 - Checkpointing graph for time 2000 ms -2016-06-13 20:16:36 INFO DStreamGraph:59 - Updating checkpoint data for time 2000 ms -2016-06-13 20:16:36 INFO DStreamGraph:59 - Updated checkpoint data for time 2000 ms -2016-06-13 20:16:36 INFO CheckpointWriter:59 - Saving checkpoint for time 1000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-1000' -2016-06-13 20:16:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:16:36 INFO JobGenerator:59 - Checkpointing graph for time 3000 ms -2016-06-13 20:16:36 INFO DStreamGraph:59 - Updating checkpoint data for time 3000 ms -2016-06-13 20:16:36 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=0, maxMem=766075207 -2016-06-13 20:16:36 INFO DStreamGraph:59 - Updated checkpoint data for time 3000 ms -2016-06-13 20:16:36 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:16:36 INFO MemoryStore:59 - ensureFreeSpace(2428) called with curMem=4744, maxMem=766075207 -2016-06-13 20:16:36 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:16:36 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:50998 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:16:36 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:16:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[7] at apply at Transformer.scala:22) -2016-06-13 20:16:36 INFO JobGenerator:59 - Checkpointing graph for time 4000 ms -2016-06-13 20:16:36 INFO DStreamGraph:59 - Updating checkpoint data for time 4000 ms -2016-06-13 20:16:36 INFO DStreamGraph:59 - Updated checkpoint data for time 4000 ms -2016-06-13 20:16:36 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 20:16:36 INFO JobGenerator:59 - Checkpointing graph for time 5000 ms -2016-06-13 20:16:36 INFO DStreamGraph:59 - Updating checkpoint data for time 5000 ms -2016-06-13 20:16:36 INFO DStreamGraph:59 - Updated checkpoint data for time 5000 ms -2016-06-13 20:16:36 INFO JobGenerator:59 - Checkpointing graph for time 6000 ms -2016-06-13 20:16:36 INFO DStreamGraph:59 - Updating checkpoint data for time 6000 ms -2016-06-13 20:16:36 INFO DStreamGraph:59 - Updated checkpoint data for time 6000 ms -2016-06-13 20:16:36 INFO JobGenerator:59 - Checkpointing graph for time 7000 ms -2016-06-13 20:16:36 INFO DStreamGraph:59 - Updating checkpoint data for time 7000 ms -2016-06-13 20:16:36 INFO DStreamGraph:59 - Updated checkpoint data for time 7000 ms -2016-06-13 20:16:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:16:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:16:36 INFO JobGenerator:59 - Checkpointing graph for time 8000 ms -2016-06-13 20:16:36 INFO DStreamGraph:59 - Updating checkpoint data for time 8000 ms -2016-06-13 20:16:36 INFO DStreamGraph:59 - Updated checkpoint data for time 8000 ms -2016-06-13 20:16:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:16:36 INFO JobGenerator:59 - Checkpointing graph for time 9000 ms -2016-06-13 20:16:36 INFO DStreamGraph:59 - Updating checkpoint data for time 9000 ms -2016-06-13 20:16:36 INFO DStreamGraph:59 - Updated checkpoint data for time 9000 ms -2016-06-13 20:16:36 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 20:16:36 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 20:16:36 INFO JobGenerator:59 - Checkpointing graph for time 10000 ms -2016-06-13 20:16:36 INFO DStreamGraph:59 - Updating checkpoint data for time 10000 ms -2016-06-13 20:16:36 INFO DStreamGraph:59 - Updated checkpoint data for time 10000 ms -2016-06-13 20:16:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:16:36 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 20:16:36 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 20:16:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:16:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:16:36 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1159 bytes result sent to driver -2016-06-13 20:16:36 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1159 bytes result sent to driver -2016-06-13 20:16:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 292 ms on localhost (1/2) -2016-06-13 20:16:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 274 ms on localhost (2/2) -2016-06-13 20:16:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 20:16:36 INFO DAGScheduler:59 - ShuffleMapStage 0 (apply at Transformer.scala:22) finished in 0.331 s -2016-06-13 20:16:36 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:16:36 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:16:36 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 20:16:36 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:16:36 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 20:16:36 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[23] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:16:36 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=7172, maxMem=766075207 -2016-06-13 20:16:36 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:16:36 INFO MemoryStore:59 - ensureFreeSpace(2657) called with curMem=12436, maxMem=766075207 -2016-06-13 20:16:36 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 20:16:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:16:36 INFO CheckpointWriter:59 - Checkpoint for time 1000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-1000', took 27279 bytes and 439 ms -2016-06-13 20:16:36 INFO CheckpointWriter:59 - Saving checkpoint for time 2000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-2000' -2016-06-13 20:16:36 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:50998 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:16:36 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[23] at apply at Transformer.scala:22) -2016-06-13 20:16:36 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 20:16:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:16:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:36 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 20:16:36 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 20:16:36 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:16:36 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:16:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 12 ms -2016-06-13 20:16:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 12 ms -2016-06-13 20:16:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:16:36 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-06-13 20:16:36 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1498 bytes result sent to driver -2016-06-13 20:16:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 68 ms on localhost (1/2) -2016-06-13 20:16:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 82 ms on localhost (2/2) -2016-06-13 20:16:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 20:16:36 INFO DAGScheduler:59 - ResultStage 1 (apply at Transformer.scala:22) finished in 0.085 s -2016-06-13 20:16:36 INFO CheckpointWriter:59 - Checkpoint for time 2000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-2000', took 27281 bytes and 142 ms -2016-06-13 20:16:36 INFO CheckpointWriter:59 - Saving checkpoint for time 3000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-3000' -2016-06-13 20:16:36 INFO DAGScheduler:59 - Job 0 finished: apply at Transformer.scala:22, took 0.722024 s -2016-06-13 20:16:36 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 50.0 and centroid: [-0.4843392480139548,0.9734525763849463,-1.7046336323611109,1.2778093795456185,0.3691650550997036] -2016-06-13 20:16:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:16:36 INFO JobScheduler:59 - Finished job streaming job 1000 ms.0 from job set of time 1000 ms -2016-06-13 20:16:36 INFO JobScheduler:59 - Starting job streaming job 1000 ms.1 from job set of time 1000 ms -2016-06-13 20:16:36 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:16:36 INFO DAGScheduler:59 - Registering RDD 3 (apply at Transformer.scala:22) -2016-06-13 20:16:36 INFO DAGScheduler:59 - Got job 1 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:16:36 INFO DAGScheduler:59 - Final stage: ResultStage 3(apply at Transformer.scala:22) -2016-06-13 20:16:36 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 2) -2016-06-13 20:16:36 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 2) -2016-06-13 20:16:36 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (UnionRDD[3] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:16:36 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=15093, maxMem=766075207 -2016-06-13 20:16:36 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:16:36 INFO MemoryStore:59 - ensureFreeSpace(2000) called with curMem=18373, maxMem=766075207 -2016-06-13 20:16:36 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2000.0 B, free 730.6 MB) -2016-06-13 20:16:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:16:36 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:50998 (size: 2000.0 B, free: 730.6 MB) -2016-06-13 20:16:36 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:36 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 2 (UnionRDD[3] at apply at Transformer.scala:22) -2016-06-13 20:16:36 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-06-13 20:16:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:16:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:16:36 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 20:16:36 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 20:16:36 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1159 bytes result sent to driver -2016-06-13 20:16:36 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:16:36 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-06-13 20:16:36 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1159 bytes result sent to driver -2016-06-13 20:16:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:16:36 INFO CheckpointWriter:59 - Checkpoint for time 3000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-3000', took 27281 bytes and 155 ms -2016-06-13 20:16:36 INFO CheckpointWriter:59 - Saving checkpoint for time 4000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-4000' -2016-06-13 20:16:36 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 6). 1159 bytes result sent to driver -2016-06-13 20:16:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 67 ms on localhost (1/3) -2016-06-13 20:16:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 77 ms on localhost (2/3) -2016-06-13 20:16:36 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 6) in 45 ms on localhost (3/3) -2016-06-13 20:16:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 20:16:36 INFO DAGScheduler:59 - ShuffleMapStage 2 (apply at Transformer.scala:22) finished in 0.082 s -2016-06-13 20:16:36 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:16:36 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:16:36 INFO DAGScheduler:59 - waiting: Set(ResultStage 3) -2016-06-13 20:16:36 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:16:36 INFO DAGScheduler:59 - Missing parents for ResultStage 3: List() -2016-06-13 20:16:36 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[62] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:16:36 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=20373, maxMem=766075207 -2016-06-13 20:16:36 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:16:36 INFO MemoryStore:59 - ensureFreeSpace(1799) called with curMem=23613, maxMem=766075207 -2016-06-13 20:16:36 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1799.0 B, free 730.6 MB) -2016-06-13 20:16:36 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:50998 (size: 1799.0 B, free: 730.6 MB) -2016-06-13 20:16:36 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:16:36 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[62] at apply at Transformer.scala:22) -2016-06-13 20:16:36 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 20:16:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:36 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 7) -2016-06-13 20:16:36 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 8) -2016-06-13 20:16:36 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:16:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:16:36 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:16:36 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:16:36 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 8). 1172 bytes result sent to driver -2016-06-13 20:16:36 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 7). 1180 bytes result sent to driver -2016-06-13 20:16:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 8) in 16 ms on localhost (1/2) -2016-06-13 20:16:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 7) in 20 ms on localhost (2/2) -2016-06-13 20:16:36 INFO DAGScheduler:59 - ResultStage 3 (apply at Transformer.scala:22) finished in 0.021 s -2016-06-13 20:16:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 20:16:36 INFO DAGScheduler:59 - Job 1 finished: apply at Transformer.scala:22, took 0.183081 s -2016-06-13 20:16:36 INFO JobScheduler:59 - Finished job streaming job 1000 ms.1 from job set of time 1000 ms -2016-06-13 20:16:36 INFO JobScheduler:59 - Total delay: 1465820195.998 s for time 1000 ms (execution: 1.037 s) -2016-06-13 20:16:37 INFO JobScheduler:59 - Starting job streaming job 2000 ms.0 from job set of time 2000 ms -2016-06-13 20:16:37 INFO JobGenerator:59 - Checkpointing graph for time 1000 ms -2016-06-13 20:16:37 INFO DStreamGraph:59 - Updating checkpoint data for time 1000 ms -2016-06-13 20:16:37 INFO DStreamGraph:59 - Updated checkpoint data for time 1000 ms -2016-06-13 20:16:37 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Registering RDD 63 (apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Got job 2 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:16:37 INFO DAGScheduler:59 - Final stage: ResultStage 5(apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[63] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=25412, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(2471) called with curMem=30156, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:16:37 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:50998 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:16:37 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[63] at apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 10, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:16:37 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 10) -2016-06-13 20:16:37 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 9) -2016-06-13 20:16:37 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 9). 1159 bytes result sent to driver -2016-06-13 20:16:37 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 10). 1159 bytes result sent to driver -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 9) in 35 ms on localhost (1/2) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 10) in 36 ms on localhost (2/2) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 20:16:37 INFO DAGScheduler:59 - ShuffleMapStage 4 (apply at Transformer.scala:22) finished in 0.038 s -2016-06-13 20:16:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:16:37 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:16:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-06-13 20:16:37 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:16:37 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting ResultStage 5 (ShuffledRDD[64] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=32627, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(2692) called with curMem=37891, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:16:37 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:50998 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:16:37 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (ShuffledRDD[64] at apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 20:16:37 INFO CheckpointWriter:59 - Checkpoint for time 4000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-4000', took 27281 bytes and 195 ms -2016-06-13 20:16:37 INFO CheckpointWriter:59 - Saving checkpoint for time 5000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-5000' -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 12, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:37 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 12) -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:37 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 12). 1161 bytes result sent to driver -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 12) in 22 ms on localhost (1/2) -2016-06-13 20:16:37 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 11) -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:37 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:16:37 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 11). 1498 bytes result sent to driver -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 11) in 48 ms on localhost (2/2) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 20:16:37 INFO DAGScheduler:59 - ResultStage 5 (apply at Transformer.scala:22) finished in 0.052 s -2016-06-13 20:16:37 INFO DAGScheduler:59 - Job 2 finished: apply at Transformer.scala:22, took 0.135525 s -2016-06-13 20:16:37 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 100.0 and centroid: [-0.4738959687352186,0.969729236815007,-1.6823805644329948,1.2679893451777908,0.37589591355279117] -2016-06-13 20:16:37 INFO JobScheduler:59 - Finished job streaming job 2000 ms.0 from job set of time 2000 ms -2016-06-13 20:16:37 INFO JobScheduler:59 - Starting job streaming job 2000 ms.1 from job set of time 2000 ms -2016-06-13 20:16:37 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Registering RDD 10 (apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Got job 3 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:16:37 INFO DAGScheduler:59 - Final stage: ResultStage 7(apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (UnionRDD[10] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=40583, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=43863, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:50998 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:16:37 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 6 (UnionRDD[10] at apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 3 tasks -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 14, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:16:37 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 14) -2016-06-13 20:16:37 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 13) -2016-06-13 20:16:37 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:16:37 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 13). 1159 bytes result sent to driver -2016-06-13 20:16:37 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 14). 1159 bytes result sent to driver -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 15, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:16:37 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 15) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 14) in 35 ms on localhost (1/3) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 13) in 42 ms on localhost (2/3) -2016-06-13 20:16:37 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 15). 1159 bytes result sent to driver -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 15) in 23 ms on localhost (3/3) -2016-06-13 20:16:37 INFO DAGScheduler:59 - ShuffleMapStage 6 (apply at Transformer.scala:22) finished in 0.047 s -2016-06-13 20:16:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:16:37 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:16:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-06-13 20:16:37 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 20:16:37 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[65] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=45872, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=49112, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.5 MB) -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:50998 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:16:37 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[65] at apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:37 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:16:37 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 17) -2016-06-13 20:16:37 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 16) -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:37 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 17). 1172 bytes result sent to driver -2016-06-13 20:16:37 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 16). 1180 bytes result sent to driver -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 17) in 28 ms on localhost (1/2) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 16) in 31 ms on localhost (2/2) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 20:16:37 INFO DAGScheduler:59 - ResultStage 7 (apply at Transformer.scala:22) finished in 0.033 s -2016-06-13 20:16:37 INFO DAGScheduler:59 - Job 3 finished: apply at Transformer.scala:22, took 0.121759 s -2016-06-13 20:16:37 INFO CheckpointWriter:59 - Checkpoint for time 5000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-5000', took 27281 bytes and 193 ms -2016-06-13 20:16:37 INFO CheckpointWriter:59 - Saving checkpoint for time 6000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-6000' -2016-06-13 20:16:37 INFO JobScheduler:59 - Finished job streaming job 2000 ms.1 from job set of time 2000 ms -2016-06-13 20:16:37 INFO JobScheduler:59 - Total delay: 1465820195.298 s for time 2000 ms (execution: 0.297 s) -2016-06-13 20:16:37 INFO JobScheduler:59 - Starting job streaming job 3000 ms.0 from job set of time 3000 ms -2016-06-13 20:16:37 INFO ParallelCollectionRDD:59 - Removing RDD 0 from persistence list -2016-06-13 20:16:37 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:16:37 INFO MapPartitionsRDD:59 - Removing RDD 5 from persistence list -2016-06-13 20:16:37 INFO ShuffledRDD:59 - Removing RDD 4 from persistence list -2016-06-13 20:16:37 INFO DAGScheduler:59 - Registering RDD 66 (apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO BlockManager:59 - Removing RDD 0 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Got job 4 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:16:37 INFO DAGScheduler:59 - Final stage: ResultStage 9(apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 8) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 8) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 8 (MapPartitionsRDD[66] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:16:37 INFO BlockManager:59 - Removing RDD 4 -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=50914, maxMem=766075207 -2016-06-13 20:16:37 INFO UnionRDD:59 - Removing RDD 3 from persistence list -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:16:37 INFO MapPartitionsRDD:59 - Removing RDD 1 from persistence list -2016-06-13 20:16:37 INFO BlockManager:59 - Removing RDD 3 -2016-06-13 20:16:37 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:16:37 INFO JobGenerator:59 - Checkpointing graph for time 2000 ms -2016-06-13 20:16:37 INFO DStreamGraph:59 - Updating checkpoint data for time 2000 ms -2016-06-13 20:16:37 INFO DStreamGraph:59 - Updated checkpoint data for time 2000 ms -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(2475) called with curMem=55658, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:50998 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:16:37 INFO BlockManager:59 - Removing RDD 1 -2016-06-13 20:16:37 INFO BlockManager:59 - Removing RDD 5 -2016-06-13 20:16:37 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 8 (MapPartitionsRDD[66] at apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 18, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 19, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:16:37 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 18) -2016-06-13 20:16:37 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 19) -2016-06-13 20:16:37 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 18). 1159 bytes result sent to driver -2016-06-13 20:16:37 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 19). 1159 bytes result sent to driver -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 18) in 26 ms on localhost (1/2) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 19) in 25 ms on localhost (2/2) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 20:16:37 INFO DAGScheduler:59 - ShuffleMapStage 8 (apply at Transformer.scala:22) finished in 0.028 s -2016-06-13 20:16:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:16:37 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:16:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 9) -2016-06-13 20:16:37 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:16:37 INFO DAGScheduler:59 - Missing parents for ResultStage 9: List() -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting ResultStage 9 (ShuffledRDD[67] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=58133, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:16:37 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=63397, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:50998 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:16:37 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (ShuffledRDD[67] at apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 21, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:37 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 20) -2016-06-13 20:16:37 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 21) -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:50998 in memory (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:16:37 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:16:37 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 21). 1161 bytes result sent to driver -2016-06-13 20:16:37 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 21) in 18 ms on localhost (1/2) -2016-06-13 20:16:37 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 20). 1498 bytes result sent to driver -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:50998 in memory (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 20) in 23 ms on localhost (2/2) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 20:16:37 INFO DAGScheduler:59 - ResultStage 9 (apply at Transformer.scala:22) finished in 0.025 s -2016-06-13 20:16:37 INFO DAGScheduler:59 - Job 4 finished: apply at Transformer.scala:22, took 0.142610 s -2016-06-13 20:16:37 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 20:16:37 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 150.0 and centroid: [-0.4741102089605591,0.9622750167428704,-1.680286076986711,1.2654376587211662,0.37713329463498424] -2016-06-13 20:16:37 INFO JobScheduler:59 - Finished job streaming job 3000 ms.0 from job set of time 3000 ms -2016-06-13 20:16:37 INFO JobScheduler:59 - Starting job streaming job 3000 ms.1 from job set of time 3000 ms -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:50998 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:16:37 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 20:16:37 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:50998 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Registering RDD 16 (apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Got job 5 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:16:37 INFO DAGScheduler:59 - Final stage: ResultStage 11(apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 10) -2016-06-13 20:16:37 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 10) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 10 (UnionRDD[16] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=40591, maxMem=766075207 -2016-06-13 20:16:37 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:16:37 INFO CheckpointWriter:59 - Checkpoint for time 6000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-6000', took 27279 bytes and 173 ms -2016-06-13 20:16:37 INFO CheckpointWriter:59 - Saving checkpoint for time 7000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-7000' -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(2004) called with curMem=42072, maxMem=766075207 -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:50998 in memory (size: 1799.0 B, free: 730.6 MB) -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2004.0 B, free 730.5 MB) -2016-06-13 20:16:37 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:50998 (size: 2004.0 B, free: 730.6 MB) -2016-06-13 20:16:37 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 10 (UnionRDD[16] at apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 3 tasks -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:50998 in memory (size: 2000.0 B, free: 730.6 MB) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 22, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:16:37 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 23, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:16:37 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 23) -2016-06-13 20:16:37 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 22) -2016-06-13 20:16:37 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-06-13 20:16:37 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:16:37 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 22). 1159 bytes result sent to driver -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 2.0 in stage 10.0 (TID 24, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:16:37 INFO Executor:59 - Running task 2.0 in stage 10.0 (TID 24) -2016-06-13 20:16:37 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 23). 1159 bytes result sent to driver -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 22) in 36 ms on localhost (1/3) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 23) in 36 ms on localhost (2/3) -2016-06-13 20:16:37 INFO Executor:59 - Finished task 2.0 in stage 10.0 (TID 24). 1159 bytes result sent to driver -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:50998 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 2.0 in stage 10.0 (TID 24) in 22 ms on localhost (3/3) -2016-06-13 20:16:37 INFO DAGScheduler:59 - ShuffleMapStage 10 (apply at Transformer.scala:22) finished in 0.043 s -2016-06-13 20:16:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:16:37 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 20:16:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 11) -2016-06-13 20:16:37 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:16:37 INFO DAGScheduler:59 - Missing parents for ResultStage 11: List() -2016-06-13 20:16:37 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[68] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:50998 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=20463, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:16:37 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=23703, maxMem=766075207 -2016-06-13 20:16:37 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.6 MB) -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:50998 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:16:37 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[68] at apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 25, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 26, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:37 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:16:37 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 26) -2016-06-13 20:16:37 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 25) -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:37 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 26). 1172 bytes result sent to driver -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 26) in 11 ms on localhost (1/2) -2016-06-13 20:16:37 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 25). 1180 bytes result sent to driver -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 25) in 19 ms on localhost (2/2) -2016-06-13 20:16:37 INFO DAGScheduler:59 - ResultStage 11 (apply at Transformer.scala:22) finished in 0.020 s -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 20:16:37 INFO DAGScheduler:59 - Job 5 finished: apply at Transformer.scala:22, took 0.105914 s -2016-06-13 20:16:37 INFO JobScheduler:59 - Finished job streaming job 3000 ms.1 from job set of time 3000 ms -2016-06-13 20:16:37 INFO JobScheduler:59 - Total delay: 1465820194.567 s for time 3000 ms (execution: 0.268 s) -2016-06-13 20:16:37 INFO ParallelCollectionRDD:59 - Removing RDD 6 from persistence list -2016-06-13 20:16:37 INFO JobScheduler:59 - Starting job streaming job 4000 ms.0 from job set of time 4000 ms -2016-06-13 20:16:37 INFO BlockManager:59 - Removing RDD 6 -2016-06-13 20:16:37 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-06-13 20:16:37 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 20:16:37 INFO ShuffledRDD:59 - Removing RDD 11 from persistence list -2016-06-13 20:16:37 INFO UnionRDD:59 - Removing RDD 10 from persistence list -2016-06-13 20:16:37 INFO BlockManager:59 - Removing RDD 11 -2016-06-13 20:16:37 INFO MapPartitionsRDD:59 - Removing RDD 8 from persistence list -2016-06-13 20:16:37 INFO BlockManager:59 - Removing RDD 10 -2016-06-13 20:16:37 INFO JobGenerator:59 - Checkpointing graph for time 3000 ms -2016-06-13 20:16:37 INFO BlockManager:59 - Removing RDD 8 -2016-06-13 20:16:37 INFO DStreamGraph:59 - Updating checkpoint data for time 3000 ms -2016-06-13 20:16:37 INFO DStreamGraph:59 - Updated checkpoint data for time 3000 ms -2016-06-13 20:16:37 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Registering RDD 69 (apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Got job 6 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:16:37 INFO DAGScheduler:59 - Final stage: ResultStage 13(apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (MapPartitionsRDD[69] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=25505, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(2484) called with curMem=30249, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:16:37 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:50998 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:16:37 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (MapPartitionsRDD[69] at apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 27, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 28, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:16:37 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 27) -2016-06-13 20:16:37 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 28) -2016-06-13 20:16:37 INFO CheckpointWriter:59 - Checkpoint for time 7000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-7000', took 27279 bytes and 154 ms -2016-06-13 20:16:37 INFO CheckpointWriter:59 - Saving checkpoint for time 8000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-8000' -2016-06-13 20:16:37 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 28). 1159 bytes result sent to driver -2016-06-13 20:16:37 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 27). 1159 bytes result sent to driver -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 28) in 24 ms on localhost (1/2) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 27) in 27 ms on localhost (2/2) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 20:16:37 INFO DAGScheduler:59 - ShuffleMapStage 12 (apply at Transformer.scala:22) finished in 0.027 s -2016-06-13 20:16:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:16:37 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:16:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-06-13 20:16:37 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:16:37 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting ResultStage 13 (ShuffledRDD[70] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=32733, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(2698) called with curMem=37997, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:50998 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:16:37 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (ShuffledRDD[70] at apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 29, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 30, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:37 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 29) -2016-06-13 20:16:37 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:16:37 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 30) -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:37 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 30). 1161 bytes result sent to driver -2016-06-13 20:16:37 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 29). 1498 bytes result sent to driver -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 30) in 17 ms on localhost (1/2) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 29) in 20 ms on localhost (2/2) -2016-06-13 20:16:37 INFO DAGScheduler:59 - ResultStage 13 (apply at Transformer.scala:22) finished in 0.020 s -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 20:16:37 INFO DAGScheduler:59 - Job 6 finished: apply at Transformer.scala:22, took 0.087770 s -2016-06-13 20:16:37 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 200.0 and centroid: [-0.4768735011530239,0.9606386018112957,-1.673148729841545,1.2634974127814163,0.38189605546947214] -2016-06-13 20:16:37 INFO JobScheduler:59 - Finished job streaming job 4000 ms.0 from job set of time 4000 ms -2016-06-13 20:16:37 INFO JobScheduler:59 - Starting job streaming job 4000 ms.1 from job set of time 4000 ms -2016-06-13 20:16:37 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Registering RDD 22 (apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Got job 7 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:16:37 INFO DAGScheduler:59 - Final stage: ResultStage 15(apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 14) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 14) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 14 (UnionRDD[22] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=40695, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=43975, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:50998 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:16:37 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 14 (UnionRDD[22] at apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 3 tasks -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 31, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 32, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:16:37 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 31) -2016-06-13 20:16:37 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 32) -2016-06-13 20:16:37 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:16:37 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 32). 1159 bytes result sent to driver -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 33, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 32) in 22 ms on localhost (1/3) -2016-06-13 20:16:37 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 33) -2016-06-13 20:16:37 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 31). 1159 bytes result sent to driver -2016-06-13 20:16:37 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 33). 1159 bytes result sent to driver -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 31) in 40 ms on localhost (2/3) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 33) in 21 ms on localhost (3/3) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 20:16:37 INFO DAGScheduler:59 - ShuffleMapStage 14 (apply at Transformer.scala:22) finished in 0.041 s -2016-06-13 20:16:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:16:37 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:16:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 15) -2016-06-13 20:16:37 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:16:37 INFO DAGScheduler:59 - Missing parents for ResultStage 15: List() -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[71] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=45984, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(1801) called with curMem=49224, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1801.0 B, free 730.5 MB) -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:50998 (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:16:37 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[71] at apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 34, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 35, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:37 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 34) -2016-06-13 20:16:37 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 35) -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:37 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 35). 1172 bytes result sent to driver -2016-06-13 20:16:37 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 35) in 11 ms on localhost (1/2) -2016-06-13 20:16:37 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 34). 1180 bytes result sent to driver -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 34) in 15 ms on localhost (2/2) -2016-06-13 20:16:37 INFO DAGScheduler:59 - ResultStage 15 (apply at Transformer.scala:22) finished in 0.015 s -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 20:16:37 INFO DAGScheduler:59 - Job 7 finished: apply at Transformer.scala:22, took 0.085222 s -2016-06-13 20:16:37 INFO CheckpointWriter:59 - Checkpoint for time 8000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-8000', took 27281 bytes and 150 ms -2016-06-13 20:16:37 INFO CheckpointWriter:59 - Saving checkpoint for time 9000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-9000' -2016-06-13 20:16:37 INFO JobScheduler:59 - Finished job streaming job 4000 ms.1 from job set of time 4000 ms -2016-06-13 20:16:37 INFO JobScheduler:59 - Total delay: 1465820193.775 s for time 4000 ms (execution: 0.207 s) -2016-06-13 20:16:37 INFO JobScheduler:59 - Starting job streaming job 5000 ms.0 from job set of time 5000 ms -2016-06-13 20:16:37 INFO ParallelCollectionRDD:59 - Removing RDD 13 from persistence list -2016-06-13 20:16:37 INFO MapPartitionsRDD:59 - Removing RDD 18 from persistence list -2016-06-13 20:16:37 INFO BlockManager:59 - Removing RDD 13 -2016-06-13 20:16:37 INFO ShuffledRDD:59 - Removing RDD 17 from persistence list -2016-06-13 20:16:37 INFO BlockManager:59 - Removing RDD 18 -2016-06-13 20:16:37 INFO UnionRDD:59 - Removing RDD 16 from persistence list -2016-06-13 20:16:37 INFO BlockManager:59 - Removing RDD 17 -2016-06-13 20:16:37 INFO MapPartitionsRDD:59 - Removing RDD 14 from persistence list -2016-06-13 20:16:37 INFO BlockManager:59 - Removing RDD 16 -2016-06-13 20:16:37 INFO BlockManager:59 - Removing RDD 14 -2016-06-13 20:16:37 INFO JobGenerator:59 - Checkpointing graph for time 4000 ms -2016-06-13 20:16:37 INFO DStreamGraph:59 - Updating checkpoint data for time 4000 ms -2016-06-13 20:16:37 INFO DStreamGraph:59 - Updated checkpoint data for time 4000 ms -2016-06-13 20:16:37 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Registering RDD 72 (apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Got job 8 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:16:37 INFO DAGScheduler:59 - Final stage: ResultStage 17(apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 16) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 16) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 16 (MapPartitionsRDD[72] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=51025, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(2476) called with curMem=55769, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:50998 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:16:37 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 16 (MapPartitionsRDD[72] at apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 36, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 37, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:16:37 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 37) -2016-06-13 20:16:37 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 36) -2016-06-13 20:16:37 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:16:37 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 37). 1159 bytes result sent to driver -2016-06-13 20:16:37 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 36). 1159 bytes result sent to driver -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 37) in 27 ms on localhost (1/2) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 36) in 31 ms on localhost (2/2) -2016-06-13 20:16:37 INFO DAGScheduler:59 - ShuffleMapStage 16 (apply at Transformer.scala:22) finished in 0.032 s -2016-06-13 20:16:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:16:37 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:16:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 17) -2016-06-13 20:16:37 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:16:37 INFO DAGScheduler:59 - Missing parents for ResultStage 17: List() -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting ResultStage 17 (ShuffledRDD[73] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=58245, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=63509, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:50998 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:16:37 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (ShuffledRDD[73] at apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 38, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 39, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:37 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 38) -2016-06-13 20:16:37 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 39) -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:37 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 39). 1161 bytes result sent to driver -2016-06-13 20:16:37 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 38). 1498 bytes result sent to driver -2016-06-13 20:16:37 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 39) in 16 ms on localhost (1/2) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 38) in 19 ms on localhost (2/2) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 20:16:37 INFO DAGScheduler:59 - ResultStage 17 (apply at Transformer.scala:22) finished in 0.020 s -2016-06-13 20:16:37 INFO DAGScheduler:59 - Job 8 finished: apply at Transformer.scala:22, took 0.082899 s -2016-06-13 20:16:37 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 250.0 and centroid: [-0.4770613340912709,0.9657053185238176,-1.6732261982319274,1.26651642628913,0.38386658573056776] -2016-06-13 20:16:37 INFO JobScheduler:59 - Finished job streaming job 5000 ms.0 from job set of time 5000 ms -2016-06-13 20:16:37 INFO JobScheduler:59 - Starting job streaming job 5000 ms.1 from job set of time 5000 ms -2016-06-13 20:16:37 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Registering RDD 29 (apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Got job 9 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:16:37 INFO DAGScheduler:59 - Final stage: ResultStage 19(apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 18) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 18) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 18 (UnionRDD[29] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=66203, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(2010) called with curMem=69483, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2010.0 B, free 730.5 MB) -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:50998 (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:16:37 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 18 (UnionRDD[29] at apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 3 tasks -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 40, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 41, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:16:37 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 41) -2016-06-13 20:16:37 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 40) -2016-06-13 20:16:37 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 41). 1159 bytes result sent to driver -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 42, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 41) in 27 ms on localhost (1/3) -2016-06-13 20:16:37 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 40). 1159 bytes result sent to driver -2016-06-13 20:16:37 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:16:37 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 42) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 40) in 32 ms on localhost (2/3) -2016-06-13 20:16:37 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 42). 1159 bytes result sent to driver -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 42) in 14 ms on localhost (3/3) -2016-06-13 20:16:37 INFO DAGScheduler:59 - ShuffleMapStage 18 (apply at Transformer.scala:22) finished in 0.041 s -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 20:16:37 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:16:37 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:16:37 INFO DAGScheduler:59 - waiting: Set(ResultStage 19) -2016-06-13 20:16:37 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:16:37 INFO DAGScheduler:59 - Missing parents for ResultStage 19: List() -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[74] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=71493, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=74733, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:50998 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:16:37 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[74] at apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 43, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 44, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:37 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 44) -2016-06-13 20:16:37 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 43) -2016-06-13 20:16:37 INFO CheckpointWriter:59 - Checkpoint for time 9000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-9000', took 27281 bytes and 182 ms -2016-06-13 20:16:37 INFO CheckpointWriter:59 - Saving checkpoint for time 10000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-10000' -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:16:37 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 44). 1172 bytes result sent to driver -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:16:37 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 44) in 7 ms on localhost (1/2) -2016-06-13 20:16:37 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 43). 1180 bytes result sent to driver -2016-06-13 20:16:37 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 43) in 11 ms on localhost (2/2) -2016-06-13 20:16:37 INFO DAGScheduler:59 - ResultStage 19 (apply at Transformer.scala:22) finished in 0.012 s -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 20:16:37 INFO DAGScheduler:59 - Job 9 finished: apply at Transformer.scala:22, took 0.083024 s -2016-06-13 20:16:37 INFO JobScheduler:59 - Finished job streaming job 5000 ms.1 from job set of time 5000 ms -2016-06-13 20:16:37 INFO ParallelCollectionRDD:59 - Removing RDD 19 from persistence list -2016-06-13 20:16:37 INFO JobScheduler:59 - Total delay: 1465820192.967 s for time 5000 ms (execution: 0.191 s) -2016-06-13 20:16:37 INFO JobScheduler:59 - Starting job streaming job 6000 ms.0 from job set of time 6000 ms -2016-06-13 20:16:37 INFO MapPartitionsRDD:59 - Removing RDD 25 from persistence list -2016-06-13 20:16:37 INFO BlockManager:59 - Removing RDD 19 -2016-06-13 20:16:37 INFO BlockManager:59 - Removing RDD 25 -2016-06-13 20:16:37 INFO ShuffledRDD:59 - Removing RDD 24 from persistence list -2016-06-13 20:16:37 INFO BlockManager:59 - Removing RDD 24 -2016-06-13 20:16:37 INFO UnionRDD:59 - Removing RDD 22 from persistence list -2016-06-13 20:16:37 INFO MapPartitionsRDD:59 - Removing RDD 20 from persistence list -2016-06-13 20:16:37 INFO JobGenerator:59 - Checkpointing graph for time 5000 ms -2016-06-13 20:16:37 INFO DStreamGraph:59 - Updating checkpoint data for time 5000 ms -2016-06-13 20:16:37 INFO DStreamGraph:59 - Updated checkpoint data for time 5000 ms -2016-06-13 20:16:37 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:16:37 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:16:37 INFO BlockManager:59 - Removing RDD 20 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Registering RDD 75 (apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO BlockManager:59 - Removing RDD 22 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Got job 10 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:16:37 INFO DAGScheduler:59 - Final stage: ResultStage 21(apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 20) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 20) -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting ShuffleMapStage 20 (MapPartitionsRDD[75] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=76536, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:16:37 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=81280, maxMem=766075207 -2016-06-13 20:16:37 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:16:37 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:50998 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:16:37 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 20 (MapPartitionsRDD[75] at apply at Transformer.scala:22) -2016-06-13 20:16:37 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 45, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:16:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 46, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:16:37 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 45) -2016-06-13 20:16:37 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 46) -2016-06-13 20:16:38 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 45). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 46). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 45) in 24 ms on localhost (1/2) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 46) in 26 ms on localhost (2/2) -2016-06-13 20:16:38 INFO DAGScheduler:59 - ShuffleMapStage 20 (apply at Transformer.scala:22) finished in 0.028 s -2016-06-13 20:16:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 20:16:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:16:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 21) -2016-06-13 20:16:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:16:38 INFO DAGScheduler:59 - Missing parents for ResultStage 21: List() -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting ResultStage 21 (ShuffledRDD[76] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=83762, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=89026, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:50998 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:16:38 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (ShuffledRDD[76] at apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:38 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 48) -2016-06-13 20:16:38 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 47) -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:38 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 48). 1161 bytes result sent to driver -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:38 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 47). 1498 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 48) in 25 ms on localhost (1/2) -2016-06-13 20:16:38 INFO DAGScheduler:59 - ResultStage 21 (apply at Transformer.scala:22) finished in 0.028 s -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 47) in 26 ms on localhost (2/2) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 20:16:38 INFO DAGScheduler:59 - Job 10 finished: apply at Transformer.scala:22, took 0.085228 s -2016-06-13 20:16:38 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 300.0 and centroid: [-0.4778380287460108,0.9658292271340416,-1.6712646630450996,1.273358698745343,0.37938624993358266] -2016-06-13 20:16:38 INFO JobScheduler:59 - Finished job streaming job 6000 ms.0 from job set of time 6000 ms -2016-06-13 20:16:38 INFO JobScheduler:59 - Starting job streaming job 6000 ms.1 from job set of time 6000 ms -2016-06-13 20:16:38 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Registering RDD 35 (apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Got job 11 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:16:38 INFO DAGScheduler:59 - Final stage: ResultStage 23(apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 22) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 22) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 22 (UnionRDD[35] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=91722, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=95002, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.5 MB) -2016-06-13 20:16:38 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:50998 (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:16:38 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 22 (UnionRDD[35] at apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 3 tasks -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 49, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 50, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:16:38 INFO CheckpointWriter:59 - Checkpoint for time 10000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-10000', took 27293 bytes and 139 ms -2016-06-13 20:16:38 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 49) -2016-06-13 20:16:38 INFO CheckpointWriter:59 - Saving checkpoint for time 1000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-1000' -2016-06-13 20:16:38 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 50) -2016-06-13 20:16:38 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 49). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 51, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:16:38 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 51) -2016-06-13 20:16:38 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 51). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 50). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 49) in 45 ms on localhost (1/3) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 51) in 29 ms on localhost (2/3) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 50) in 47 ms on localhost (3/3) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 20:16:38 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:16:38 INFO DAGScheduler:59 - ShuffleMapStage 22 (apply at Transformer.scala:22) finished in 0.051 s -2016-06-13 20:16:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:16:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:16:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 23) -2016-06-13 20:16:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:16:38 INFO DAGScheduler:59 - Missing parents for ResultStage 23: List() -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[77] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=97009, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=100249, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:50998 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:16:38 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[77] at apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 52, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:38 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 53) -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:38 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 53). 1172 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 53) in 9 ms on localhost (1/2) -2016-06-13 20:16:38 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 52) -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:16:38 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:16:38 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 52). 1180 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 52) in 27 ms on localhost (2/2) -2016-06-13 20:16:38 INFO DAGScheduler:59 - ResultStage 23 (apply at Transformer.scala:22) finished in 0.028 s -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 20:16:38 INFO DAGScheduler:59 - Job 11 finished: apply at Transformer.scala:22, took 0.127027 s -2016-06-13 20:16:38 INFO JobScheduler:59 - Finished job streaming job 6000 ms.1 from job set of time 6000 ms -2016-06-13 20:16:38 INFO JobScheduler:59 - Total delay: 1465820192.204 s for time 6000 ms (execution: 0.236 s) -2016-06-13 20:16:38 INFO JobScheduler:59 - Starting job streaming job 7000 ms.0 from job set of time 7000 ms -2016-06-13 20:16:38 INFO ParallelCollectionRDD:59 - Removing RDD 26 from persistence list -2016-06-13 20:16:38 INFO MapPartitionsRDD:59 - Removing RDD 31 from persistence list -2016-06-13 20:16:38 INFO ShuffledRDD:59 - Removing RDD 30 from persistence list -2016-06-13 20:16:38 INFO UnionRDD:59 - Removing RDD 29 from persistence list -2016-06-13 20:16:38 INFO BlockManager:59 - Removing RDD 26 -2016-06-13 20:16:38 INFO BlockManager:59 - Removing RDD 31 -2016-06-13 20:16:38 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:16:38 INFO BlockManager:59 - Removing RDD 30 -2016-06-13 20:16:38 INFO MapPartitionsRDD:59 - Removing RDD 27 from persistence list -2016-06-13 20:16:38 INFO BlockManager:59 - Removing RDD 29 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Registering RDD 78 (apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO BlockManager:59 - Removing RDD 27 -2016-06-13 20:16:38 INFO JobGenerator:59 - Checkpointing graph for time 6000 ms -2016-06-13 20:16:38 INFO DAGScheduler:59 - Got job 12 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:16:38 INFO DStreamGraph:59 - Updating checkpoint data for time 6000 ms -2016-06-13 20:16:38 INFO DAGScheduler:59 - Final stage: ResultStage 25(apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 24) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 24) -2016-06-13 20:16:38 INFO DStreamGraph:59 - Updated checkpoint data for time 6000 ms -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 24 (MapPartitionsRDD[78] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=102052, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(2480) called with curMem=106796, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:50998 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 20:16:38 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:50998 in memory (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 24 (MapPartitionsRDD[78] at apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 54, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 55, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:16:38 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 55) -2016-06-13 20:16:38 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 54) -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:50998 in memory (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:50998 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:50998 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned shuffle 10 -2016-06-13 20:16:38 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 54). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 55). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 54) in 19 ms on localhost (1/2) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 55) in 18 ms on localhost (2/2) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-06-13 20:16:38 INFO DAGScheduler:59 - ShuffleMapStage 24 (apply at Transformer.scala:22) finished in 0.020 s -2016-06-13 20:16:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:16:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:16:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 25) -2016-06-13 20:16:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:16:38 INFO DAGScheduler:59 - Missing parents for ResultStage 25: List() -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:50998 in memory (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting ResultStage 25 (ShuffledRDD[79] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=78717, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=81971, maxMem=766075207 -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:50998 in memory (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:50998 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned shuffle 9 -2016-06-13 20:16:38 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (ShuffledRDD[79] at apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:50998 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:16:38 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 57) -2016-06-13 20:16:38 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 56) -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:50998 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:16:38 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 57). 1161 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 57) in 10 ms on localhost (1/2) -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 20:16:38 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:16:38 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 56). 1498 bytes result sent to driver -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned shuffle 8 -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 56) in 15 ms on localhost (2/2) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-06-13 20:16:38 INFO DAGScheduler:59 - ResultStage 25 (apply at Transformer.scala:22) finished in 0.015 s -2016-06-13 20:16:38 INFO DAGScheduler:59 - Job 12 finished: apply at Transformer.scala:22, took 0.076497 s -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:50998 in memory (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:16:38 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 350.0 and centroid: [-0.47440048489638564,0.9650523283036628,-1.6700055274657652,1.269947562552048,0.3797915177328754] -2016-06-13 20:16:38 INFO JobScheduler:59 - Finished job streaming job 7000 ms.0 from job set of time 7000 ms -2016-06-13 20:16:38 INFO JobScheduler:59 - Starting job streaming job 7000 ms.1 from job set of time 7000 ms -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:50998 in memory (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned shuffle 7 -2016-06-13 20:16:38 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:50998 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Registering RDD 41 (apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Got job 13 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:16:38 INFO DAGScheduler:59 - Final stage: ResultStage 27(apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (UnionRDD[41] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=43173, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=43969, maxMem=766075207 -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:50998 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned shuffle 6 -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:50998 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:16:38 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 26 (UnionRDD[41] at apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 3 tasks -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 58, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 59, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:16:38 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 59) -2016-06-13 20:16:38 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 58) -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:50998 in memory (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:50998 in memory (size: 2004.0 B, free: 730.6 MB) -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned shuffle 5 -2016-06-13 20:16:38 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 59). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:50998 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 60, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:16:38 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 60) -2016-06-13 20:16:38 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 20:16:38 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 60). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:50998 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 59) in 35 ms on localhost (1/3) -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 60) in 14 ms on localhost (2/3) -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned shuffle 4 -2016-06-13 20:16:38 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-06-13 20:16:38 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 58). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 58) in 54 ms on localhost (3/3) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-06-13 20:16:38 INFO DAGScheduler:59 - ShuffleMapStage 26 (apply at Transformer.scala:22) finished in 0.055 s -2016-06-13 20:16:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:16:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:16:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-06-13 20:16:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:16:38 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[80] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=20473, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(1801) called with curMem=23713, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 1801.0 B, free 730.6 MB) -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:50998 (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:16:38 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[80] at apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 62, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:38 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 61) -2016-06-13 20:16:38 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 62) -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:16:38 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 62). 1172 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 62) in 5 ms on localhost (1/2) -2016-06-13 20:16:38 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 61). 1180 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 61) in 10 ms on localhost (2/2) -2016-06-13 20:16:38 INFO DAGScheduler:59 - ResultStage 27 (apply at Transformer.scala:22) finished in 0.010 s -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-06-13 20:16:38 INFO DAGScheduler:59 - Job 13 finished: apply at Transformer.scala:22, took 0.091949 s -2016-06-13 20:16:38 INFO JobScheduler:59 - Finished job streaming job 7000 ms.1 from job set of time 7000 ms -2016-06-13 20:16:38 INFO ParallelCollectionRDD:59 - Removing RDD 32 from persistence list -2016-06-13 20:16:38 INFO JobScheduler:59 - Total delay: 1465820191.421 s for time 7000 ms (execution: 0.216 s) -2016-06-13 20:16:38 INFO JobScheduler:59 - Starting job streaming job 8000 ms.0 from job set of time 8000 ms -2016-06-13 20:16:38 INFO MapPartitionsRDD:59 - Removing RDD 37 from persistence list -2016-06-13 20:16:38 INFO BlockManager:59 - Removing RDD 32 -2016-06-13 20:16:38 INFO BlockManager:59 - Removing RDD 37 -2016-06-13 20:16:38 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:16:38 INFO ShuffledRDD:59 - Removing RDD 36 from persistence list -2016-06-13 20:16:38 INFO UnionRDD:59 - Removing RDD 35 from persistence list -2016-06-13 20:16:38 INFO BlockManager:59 - Removing RDD 35 -2016-06-13 20:16:38 INFO MapPartitionsRDD:59 - Removing RDD 33 from persistence list -2016-06-13 20:16:38 INFO BlockManager:59 - Removing RDD 33 -2016-06-13 20:16:38 INFO BlockManager:59 - Removing RDD 36 -2016-06-13 20:16:38 INFO JobGenerator:59 - Checkpointing graph for time 7000 ms -2016-06-13 20:16:38 INFO DStreamGraph:59 - Updating checkpoint data for time 7000 ms -2016-06-13 20:16:38 INFO DStreamGraph:59 - Updated checkpoint data for time 7000 ms -2016-06-13 20:16:38 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Registering RDD 81 (apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Got job 14 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:16:38 INFO DAGScheduler:59 - Final stage: ResultStage 29(apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 28) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 28) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 28 (MapPartitionsRDD[81] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=25514, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=30258, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:50998 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:16:38 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 28 (MapPartitionsRDD[81] at apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 63, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 64, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:16:38 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 63) -2016-06-13 20:16:38 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 64) -2016-06-13 20:16:38 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 64). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 64) in 14 ms on localhost (1/2) -2016-06-13 20:16:38 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:16:38 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:16:38 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:16:38 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 63). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 63) in 192 ms on localhost (2/2) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-06-13 20:16:38 INFO DAGScheduler:59 - ShuffleMapStage 28 (apply at Transformer.scala:22) finished in 0.194 s -2016-06-13 20:16:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:16:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:16:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 29) -2016-06-13 20:16:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:16:38 INFO DAGScheduler:59 - Missing parents for ResultStage 29: List() -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting ResultStage 29 (ShuffledRDD[82] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=32740, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=38004, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:50998 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:16:38 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (ShuffledRDD[82] at apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 65, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 66, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:38 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 66) -2016-06-13 20:16:38 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 65) -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:16:38 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 66). 1161 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 66) in 9 ms on localhost (1/2) -2016-06-13 20:16:38 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 65). 1498 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 65) in 18 ms on localhost (2/2) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 20:16:38 INFO DAGScheduler:59 - ResultStage 29 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:16:38 INFO DAGScheduler:59 - Job 14 finished: apply at Transformer.scala:22, took 0.235447 s -2016-06-13 20:16:38 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 400.0 and centroid: [-0.4716182064591067,0.967586334056103,-1.6699954778983372,1.270341503100185,0.37935009425710725] -2016-06-13 20:16:38 INFO JobScheduler:59 - Finished job streaming job 8000 ms.0 from job set of time 8000 ms -2016-06-13 20:16:38 INFO JobScheduler:59 - Starting job streaming job 8000 ms.1 from job set of time 8000 ms -2016-06-13 20:16:38 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Registering RDD 47 (apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Got job 15 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:16:38 INFO DAGScheduler:59 - Final stage: ResultStage 31(apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 30) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 30 (UnionRDD[47] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=40698, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(2006) called with curMem=43978, maxMem=766075207 -2016-06-13 20:16:38 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2006.0 B, free 730.5 MB) -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:50998 (size: 2006.0 B, free: 730.6 MB) -2016-06-13 20:16:38 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 30 (UnionRDD[47] at apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 3 tasks -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 67, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 68, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:16:38 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 67) -2016-06-13 20:16:38 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 68) -2016-06-13 20:16:38 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-1000.bk -2016-06-13 20:16:38 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 67). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 69, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:16:38 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 69) -2016-06-13 20:16:38 INFO CheckpointWriter:59 - Checkpoint for time 1000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-1000', took 27471 bytes and 613 ms -2016-06-13 20:16:38 INFO CheckpointWriter:59 - Saving checkpoint for time 2000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-2000' -2016-06-13 20:16:38 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 69). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 67) in 28 ms on localhost (1/3) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 69) in 17 ms on localhost (2/3) -2016-06-13 20:16:38 INFO DStreamGraph:59 - Clearing checkpoint data for time 1000 ms -2016-06-13 20:16:38 INFO DStreamGraph:59 - Cleared checkpoint data for time 1000 ms -2016-06-13 20:16:38 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 68). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 68) in 37 ms on localhost (3/3) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 20:16:38 INFO DAGScheduler:59 - ShuffleMapStage 30 (apply at Transformer.scala:22) finished in 0.039 s -2016-06-13 20:16:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:16:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:16:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 31) -2016-06-13 20:16:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:16:38 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List() -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[83] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=45984, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=49224, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.5 MB) -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:50998 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:16:38 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/receivedBlockMetadata older than 0: -2016-06-13 20:16:38 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[83] at apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 20:16:38 INFO InputInfoTracker:59 - remove old batch metadata: -2016-06-13 20:16:38 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/receivedBlockMetadata older than 0 -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 70, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 71, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:38 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 70) -2016-06-13 20:16:38 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 71) -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:38 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 71). 1172 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 71) in 5 ms on localhost (1/2) -2016-06-13 20:16:38 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 70). 1180 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 70) in 9 ms on localhost (2/2) -2016-06-13 20:16:38 INFO DAGScheduler:59 - ResultStage 31 (apply at Transformer.scala:22) finished in 0.009 s -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 20:16:38 INFO DAGScheduler:59 - Job 15 finished: apply at Transformer.scala:22, took 0.074289 s -2016-06-13 20:16:38 INFO JobScheduler:59 - Finished job streaming job 8000 ms.1 from job set of time 8000 ms -2016-06-13 20:16:38 INFO ParallelCollectionRDD:59 - Removing RDD 38 from persistence list -2016-06-13 20:16:38 INFO JobScheduler:59 - Total delay: 1465820190.753 s for time 8000 ms (execution: 0.332 s) -2016-06-13 20:16:38 INFO JobScheduler:59 - Starting job streaming job 9000 ms.0 from job set of time 9000 ms -2016-06-13 20:16:38 INFO BlockManager:59 - Removing RDD 38 -2016-06-13 20:16:38 INFO MapPartitionsRDD:59 - Removing RDD 43 from persistence list -2016-06-13 20:16:38 INFO BlockManager:59 - Removing RDD 43 -2016-06-13 20:16:38 INFO ShuffledRDD:59 - Removing RDD 42 from persistence list -2016-06-13 20:16:38 INFO UnionRDD:59 - Removing RDD 41 from persistence list -2016-06-13 20:16:38 INFO BlockManager:59 - Removing RDD 42 -2016-06-13 20:16:38 INFO MapPartitionsRDD:59 - Removing RDD 39 from persistence list -2016-06-13 20:16:38 INFO BlockManager:59 - Removing RDD 41 -2016-06-13 20:16:38 INFO BlockManager:59 - Removing RDD 39 -2016-06-13 20:16:38 INFO JobGenerator:59 - Checkpointing graph for time 8000 ms -2016-06-13 20:16:38 INFO DStreamGraph:59 - Updating checkpoint data for time 8000 ms -2016-06-13 20:16:38 INFO DStreamGraph:59 - Updated checkpoint data for time 8000 ms -2016-06-13 20:16:38 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Registering RDD 84 (apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Got job 16 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:16:38 INFO DAGScheduler:59 - Final stage: ResultStage 33(apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 32) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 32 (MapPartitionsRDD[84] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=51026, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(2477) called with curMem=55770, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:50998 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:16:38 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 32 (MapPartitionsRDD[84] at apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 72, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 73, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:16:38 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 73) -2016-06-13 20:16:38 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 72) -2016-06-13 20:16:38 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:16:38 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 72). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 72) in 23 ms on localhost (1/2) -2016-06-13 20:16:38 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 73). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 73) in 27 ms on localhost (2/2) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 20:16:38 INFO DAGScheduler:59 - ShuffleMapStage 32 (apply at Transformer.scala:22) finished in 0.029 s -2016-06-13 20:16:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:16:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:16:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 33) -2016-06-13 20:16:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:16:38 INFO DAGScheduler:59 - Missing parents for ResultStage 33: List() -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting ResultStage 33 (ShuffledRDD[85] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=58247, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=63511, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:50998 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:16:38 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (ShuffledRDD[85] at apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 74, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 75, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:38 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 75) -2016-06-13 20:16:38 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 74) -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:38 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 75). 1161 bytes result sent to driver -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:38 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 74). 1498 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 75) in 14 ms on localhost (1/2) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 74) in 17 ms on localhost (2/2) -2016-06-13 20:16:38 INFO DAGScheduler:59 - ResultStage 33 (apply at Transformer.scala:22) finished in 0.017 s -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 20:16:38 INFO DAGScheduler:59 - Job 16 finished: apply at Transformer.scala:22, took 0.069643 s -2016-06-13 20:16:38 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 450.0 and centroid: [-0.4722098164150225,0.9642763266691793,-1.6666072662538036,1.2709270669491914,0.37911973669212223] -2016-06-13 20:16:38 INFO JobScheduler:59 - Finished job streaming job 9000 ms.0 from job set of time 9000 ms -2016-06-13 20:16:38 INFO JobScheduler:59 - Starting job streaming job 9000 ms.1 from job set of time 9000 ms -2016-06-13 20:16:38 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:16:38 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Registering RDD 53 (apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Got job 17 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:16:38 INFO DAGScheduler:59 - Final stage: ResultStage 35(apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 34) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 34) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 34 (UnionRDD[53] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=66205, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=69485, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.5 MB) -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:50998 (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:16:38 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 34 (UnionRDD[53] at apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 3 tasks -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 76, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 77, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:16:38 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 77) -2016-06-13 20:16:38 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 76) -2016-06-13 20:16:38 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 77). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 76). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 78, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 77) in 19 ms on localhost (1/3) -2016-06-13 20:16:38 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 78) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 76) in 20 ms on localhost (2/3) -2016-06-13 20:16:38 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 78). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 78) in 12 ms on localhost (3/3) -2016-06-13 20:16:38 INFO DAGScheduler:59 - ShuffleMapStage 34 (apply at Transformer.scala:22) finished in 0.032 s -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-06-13 20:16:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:16:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:16:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 35) -2016-06-13 20:16:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:16:38 INFO DAGScheduler:59 - Missing parents for ResultStage 35: List() -2016-06-13 20:16:38 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[86] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=71492, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=74732, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:50998 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:16:38 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[86] at apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 79, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 80, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:38 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 79) -2016-06-13 20:16:38 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 80) -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:16:38 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 79). 1180 bytes result sent to driver -2016-06-13 20:16:38 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 80). 1172 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 79) in 12 ms on localhost (1/2) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 80) in 13 ms on localhost (2/2) -2016-06-13 20:16:38 INFO DAGScheduler:59 - ResultStage 35 (apply at Transformer.scala:22) finished in 0.014 s -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-06-13 20:16:38 INFO DAGScheduler:59 - Job 17 finished: apply at Transformer.scala:22, took 0.071044 s -2016-06-13 20:16:38 INFO JobScheduler:59 - Finished job streaming job 9000 ms.1 from job set of time 9000 ms -2016-06-13 20:16:38 INFO JobScheduler:59 - Total delay: 1465820189.916 s for time 9000 ms (execution: 0.163 s) -2016-06-13 20:16:38 INFO JobScheduler:59 - Starting job streaming job 10000 ms.0 from job set of time 10000 ms -2016-06-13 20:16:38 INFO ParallelCollectionRDD:59 - Removing RDD 44 from persistence list -2016-06-13 20:16:38 INFO BlockManager:59 - Removing RDD 44 -2016-06-13 20:16:38 INFO MapPartitionsRDD:59 - Removing RDD 49 from persistence list -2016-06-13 20:16:38 INFO BlockManager:59 - Removing RDD 49 -2016-06-13 20:16:38 INFO ShuffledRDD:59 - Removing RDD 48 from persistence list -2016-06-13 20:16:38 INFO BlockManager:59 - Removing RDD 48 -2016-06-13 20:16:38 INFO UnionRDD:59 - Removing RDD 47 from persistence list -2016-06-13 20:16:38 INFO MapPartitionsRDD:59 - Removing RDD 45 from persistence list -2016-06-13 20:16:38 INFO BlockManager:59 - Removing RDD 47 -2016-06-13 20:16:38 INFO JobGenerator:59 - Checkpointing graph for time 9000 ms -2016-06-13 20:16:38 INFO DStreamGraph:59 - Updating checkpoint data for time 9000 ms -2016-06-13 20:16:38 INFO DStreamGraph:59 - Updated checkpoint data for time 9000 ms -2016-06-13 20:16:38 INFO BlockManager:59 - Removing RDD 45 -2016-06-13 20:16:38 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Registering RDD 87 (apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Got job 18 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:16:38 INFO DAGScheduler:59 - Final stage: ResultStage 37(apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 36) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 36) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 36 (MapPartitionsRDD[87] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=76535, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=81279, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:50998 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:16:38 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 36 (MapPartitionsRDD[87] at apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-06-13 20:16:38 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 81, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 82, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:16:38 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 82) -2016-06-13 20:16:38 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 81) -2016-06-13 20:16:38 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-1000 -2016-06-13 20:16:38 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 82). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 81). 1159 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 82) in 12 ms on localhost (1/2) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 81) in 15 ms on localhost (2/2) -2016-06-13 20:16:38 INFO DAGScheduler:59 - ShuffleMapStage 36 (apply at Transformer.scala:22) finished in 0.015 s -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-06-13 20:16:38 INFO CheckpointWriter:59 - Checkpoint for time 2000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-2000', took 27484 bytes and 247 ms -2016-06-13 20:16:38 INFO DStreamGraph:59 - Clearing checkpoint data for time 2000 ms -2016-06-13 20:16:38 INFO CheckpointWriter:59 - Saving checkpoint for time 3000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-3000' -2016-06-13 20:16:38 INFO DStreamGraph:59 - Cleared checkpoint data for time 2000 ms -2016-06-13 20:16:38 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:16:38 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:16:38 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:16:38 INFO DAGScheduler:59 - waiting: Set(ResultStage 37) -2016-06-13 20:16:38 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:16:38 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/receivedBlockMetadata older than 1000: -2016-06-13 20:16:38 INFO DAGScheduler:59 - Missing parents for ResultStage 37: List() -2016-06-13 20:16:38 INFO InputInfoTracker:59 - remove old batch metadata: -2016-06-13 20:16:38 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/receivedBlockMetadata older than 1000 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting ResultStage 37 (ShuffledRDD[88] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=83761, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=89025, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:50998 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:16:38 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (ShuffledRDD[88] at apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 83, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 84, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:38 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 84) -2016-06-13 20:16:38 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 83) -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:16:38 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:16:38 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 84). 1161 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 84) in 6 ms on localhost (1/2) -2016-06-13 20:16:38 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 83). 1498 bytes result sent to driver -2016-06-13 20:16:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 83) in 8 ms on localhost (2/2) -2016-06-13 20:16:38 INFO DAGScheduler:59 - ResultStage 37 (apply at Transformer.scala:22) finished in 0.010 s -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-06-13 20:16:38 INFO DAGScheduler:59 - Job 18 finished: apply at Transformer.scala:22, took 0.047828 s -2016-06-13 20:16:38 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 500.0 and centroid: [-0.4725511979691583,0.9644503899125422,-1.668776373542808,1.2721254429935838,0.37815209739836425] -2016-06-13 20:16:38 INFO JobScheduler:59 - Finished job streaming job 10000 ms.0 from job set of time 10000 ms -2016-06-13 20:16:38 INFO JobScheduler:59 - Starting job streaming job 10000 ms.1 from job set of time 10000 ms -2016-06-13 20:16:38 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Registering RDD 59 (apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Got job 19 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:16:38 INFO DAGScheduler:59 - Final stage: ResultStage 39(apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 38) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 38) -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting ShuffleMapStage 38 (UnionRDD[59] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=91721, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:16:38 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=95001, maxMem=766075207 -2016-06-13 20:16:38 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.5 MB) -2016-06-13 20:16:38 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:50998 (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:16:38 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:38 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 38 (UnionRDD[59] at apply at Transformer.scala:22) -2016-06-13 20:16:38 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 3 tasks -2016-06-13 20:16:38 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 85, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:16:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 86, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:16:38 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 86) -2016-06-13 20:16:38 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 85) -2016-06-13 20:16:39 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 85). 1159 bytes result sent to driver -2016-06-13 20:16:39 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 86). 1159 bytes result sent to driver -2016-06-13 20:16:39 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 87, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:16:39 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 87) -2016-06-13 20:16:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 85) in 10 ms on localhost (1/3) -2016-06-13 20:16:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 86) in 10 ms on localhost (2/3) -2016-06-13 20:16:39 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 87). 1159 bytes result sent to driver -2016-06-13 20:16:39 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 87) in 10 ms on localhost (3/3) -2016-06-13 20:16:39 INFO DAGScheduler:59 - ShuffleMapStage 38 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:16:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-06-13 20:16:39 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:16:39 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:16:39 INFO DAGScheduler:59 - waiting: Set(ResultStage 39) -2016-06-13 20:16:39 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:16:39 INFO DAGScheduler:59 - Missing parents for ResultStage 39: List() -2016-06-13 20:16:39 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[89] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:16:39 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=97008, maxMem=766075207 -2016-06-13 20:16:39 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:16:39 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=100248, maxMem=766075207 -2016-06-13 20:16:39 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:16:39 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:50998 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:16:39 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:16:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[89] at apply at Transformer.scala:22) -2016-06-13 20:16:39 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-06-13 20:16:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:16:39 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 89) -2016-06-13 20:16:39 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 88) -2016-06-13 20:16:39 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:16:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:16:39 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:16:39 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:16:39 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 89). 1172 bytes result sent to driver -2016-06-13 20:16:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 89) in 6 ms on localhost (1/2) -2016-06-13 20:16:39 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 88). 1180 bytes result sent to driver -2016-06-13 20:16:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 88) in 10 ms on localhost (2/2) -2016-06-13 20:16:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-06-13 20:16:39 INFO DAGScheduler:59 - ResultStage 39 (apply at Transformer.scala:22) finished in 0.010 s -2016-06-13 20:16:39 INFO DAGScheduler:59 - Job 19 finished: apply at Transformer.scala:22, took 0.048475 s -2016-06-13 20:16:39 INFO JobScheduler:59 - Finished job streaming job 10000 ms.1 from job set of time 10000 ms -2016-06-13 20:16:39 INFO ParallelCollectionRDD:59 - Removing RDD 50 from persistence list -2016-06-13 20:16:39 INFO JobScheduler:59 - Total delay: 1465820189.034 s for time 10000 ms (execution: 0.117 s) -2016-06-13 20:16:39 INFO BlockManager:59 - Removing RDD 50 -2016-06-13 20:16:39 INFO MapPartitionsRDD:59 - Removing RDD 55 from persistence list -2016-06-13 20:16:39 INFO ShuffledRDD:59 - Removing RDD 54 from persistence list -2016-06-13 20:16:39 INFO BlockManager:59 - Removing RDD 55 -2016-06-13 20:16:39 INFO UnionRDD:59 - Removing RDD 53 from persistence list -2016-06-13 20:16:39 INFO BlockManager:59 - Removing RDD 54 -2016-06-13 20:16:39 INFO BlockManager:59 - Removing RDD 53 -2016-06-13 20:16:39 INFO MapPartitionsRDD:59 - Removing RDD 51 from persistence list -2016-06-13 20:16:39 INFO BlockManager:59 - Removing RDD 51 -2016-06-13 20:16:39 INFO JobGenerator:59 - Checkpointing graph for time 10000 ms -2016-06-13 20:16:39 INFO DStreamGraph:59 - Updating checkpoint data for time 10000 ms -2016-06-13 20:16:39 INFO DStreamGraph:59 - Updated checkpoint data for time 10000 ms -2016-06-13 20:16:39 INFO StreamingKMeansSuite:59 - Output generated in 3130 milliseconds -2016-06-13 20:16:39 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:16:39 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:16:39 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:16:39 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:16:39 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:16:39 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:16:39 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:16:39 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:16:39 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:16:39 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:16:39 INFO JobGenerator:59 - Stopping JobGenerator immediately -2016-06-13 20:16:39 INFO RecurringTimer:59 - Stopped timer for JobGenerator after time 10000 -2016-06-13 20:16:49 INFO CheckpointWriter:59 - CheckpointWriter executor terminated ? false, waited for 10001 ms. -2016-06-13 20:16:49 INFO JobGenerator:59 - Stopped JobGenerator -2016-06-13 20:16:49 INFO JobScheduler:59 - Stopped JobScheduler -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/batch,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/streaming,null} -2016-06-13 20:16:49 INFO StreamingContext:59 - StreamingContext stopped successfully -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/batch/json,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/json,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:16:49 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:16:49 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-2000.bk -2016-06-13 20:16:49 INFO CheckpointWriter:59 - Checkpoint for time 3000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6063dd22-d19e-4770-8931-c6a1c7de301f/checkpoint-3000', took 27495 bytes and 10287 ms -2016-06-13 20:16:49 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 20:16:49 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:16:49 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:16:49 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:16:49 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:16:49 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:16:49 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:16:49 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:16:49 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:16:49 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:16:49 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 20:16:49 INFO StreamingKMeansSuite:59 - - -===== FINISHED o.a.s.mllib.clustering.kmeans.StreamingKMeansSuite: 'accuracy for single center and equivalence to grand average' ===== - -2016-06-13 20:16:49 WARN StreamingContext:71 - StreamingContext has already been stopped -2016-06-13 20:16:49 INFO SparkContext:59 - SparkContext already stopped. -2016-06-13 20:16:49 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 20:16:49 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-6063dd22-d19e-4770-8931-c6a1c7de301f -2016-06-13 20:16:50 ERROR ShutdownHookManager:96 - Exception while deleting Spark temp dir: C:\Users\xubo\AppData\Local\Temp\spark-6063dd22-d19e-4770-8931-c6a1c7de301f -java.io.IOException: Failed to delete: C:\Users\xubo\AppData\Local\Temp\spark-6063dd22-d19e-4770-8931-c6a1c7de301f - at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:884) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:63) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:60) - at scala.collection.mutable.HashSet.foreach(HashSet.scala:79) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1.apply$mcV$sp(ShutdownHookManager.scala:60) - at org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:264) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1699) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply$mcV$sp(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:234) - at scala.util.Try$.apply(Try.scala:161) - at org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:216) - at org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54) -2016-06-13 20:16:50 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-e8a3a6fc-8575-4f87-8e69-6599a450a52d -2016-06-13 20:17:16 INFO StreamingKMeansSuite:59 - Using manual clock -2016-06-13 20:17:16 INFO StreamingKMeansSuite:59 - - -===== TEST OUTPUT FOR o.a.s.mllib.clustering.kmeans.StreamingKMeansSuite: 'accuracy for single center and equivalence to grand average' ===== - -2016-06-13 20:17:16 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:17:17 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 20:17:17 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:17:17 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:17:17 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:17:18 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:17:18 INFO Remoting:74 - Starting remoting -2016-06-13 20:17:18 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:51019] -2016-06-13 20:17:18 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51019. -2016-06-13 20:17:18 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:17:18 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:17:18 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-96f96a27-22d3-44de-bc74-7473f414a301 -2016-06-13 20:17:18 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:17:18 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-acf407e9-09a9-4bc3-8721-8b8587bb50f0\httpd-62695df6-d722-4f54-92c7-827a23fe11d8 -2016-06-13 20:17:18 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:17:18 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:17:18 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51020 -2016-06-13 20:17:18 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51020. -2016-06-13 20:17:18 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:17:19 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:17:19 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 20:17:19 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 20:17:19 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 20:17:19 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:17:19 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:17:19 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51031. -2016-06-13 20:17:19 INFO NettyBlockTransferService:59 - Server created on 51031 -2016-06-13 20:17:19 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:17:19 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51031 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51031) -2016-06-13 20:17:19 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:17:20 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 20:17:21 INFO StreamingKMeansSuite:59 - numBatches = 10, numExpectedOutput = 10 -2016-06-13 20:17:21 INFO ForEachDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:17:21 INFO TestInputStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:17:21 INFO TestInputStream:59 - Slide time = 1000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:17:21 INFO TestInputStream:59 - Checkpoint interval = null -2016-06-13 20:17:21 INFO TestInputStream:59 - Remember duration = 1000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Initialized and validated org.apache.spark.streaming.TestInputStream@43748549 -2016-06-13 20:17:21 INFO ForEachDStream:59 - Slide time = 1000 ms -2016-06-13 20:17:21 INFO ForEachDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:17:21 INFO ForEachDStream:59 - Checkpoint interval = null -2016-06-13 20:17:21 INFO ForEachDStream:59 - Remember duration = 1000 ms -2016-06-13 20:17:21 INFO ForEachDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.ForEachDStream@6112da24 -2016-06-13 20:17:21 INFO TestOutputStreamWithPartitions:59 - metadataCleanupDelay = -1 -2016-06-13 20:17:21 INFO MappedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:17:21 INFO ShuffledDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:17:21 INFO TransformedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:17:21 INFO MappedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:17:21 INFO TestInputStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:17:21 INFO TestInputStream:59 - Slide time = 1000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:17:21 INFO TestInputStream:59 - Checkpoint interval = null -2016-06-13 20:17:21 INFO TestInputStream:59 - Remember duration = 1000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Initialized and validated org.apache.spark.streaming.TestInputStream@43748549 -2016-06-13 20:17:21 INFO MappedDStream:59 - Slide time = 1000 ms -2016-06-13 20:17:21 INFO MappedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:17:21 INFO MappedDStream:59 - Checkpoint interval = null -2016-06-13 20:17:21 INFO MappedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:17:21 INFO MappedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@2dd8b3c7 -2016-06-13 20:17:21 INFO TransformedDStream:59 - Slide time = 1000 ms -2016-06-13 20:17:21 INFO TransformedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:17:21 INFO TransformedDStream:59 - Checkpoint interval = null -2016-06-13 20:17:21 INFO TransformedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:17:21 INFO TransformedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.TransformedDStream@58159e0f -2016-06-13 20:17:21 INFO ShuffledDStream:59 - Slide time = 1000 ms -2016-06-13 20:17:21 INFO ShuffledDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:17:21 INFO ShuffledDStream:59 - Checkpoint interval = null -2016-06-13 20:17:21 INFO ShuffledDStream:59 - Remember duration = 1000 ms -2016-06-13 20:17:21 INFO ShuffledDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.ShuffledDStream@88d013d -2016-06-13 20:17:21 INFO MappedDStream:59 - Slide time = 1000 ms -2016-06-13 20:17:21 INFO MappedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:17:21 INFO MappedDStream:59 - Checkpoint interval = null -2016-06-13 20:17:21 INFO MappedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:17:21 INFO MappedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@1f07d9e2 -2016-06-13 20:17:21 INFO TestOutputStreamWithPartitions:59 - Slide time = 1000 ms -2016-06-13 20:17:21 INFO TestOutputStreamWithPartitions:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:17:21 INFO TestOutputStreamWithPartitions:59 - Checkpoint interval = null -2016-06-13 20:17:21 INFO TestOutputStreamWithPartitions:59 - Remember duration = 1000 ms -2016-06-13 20:17:21 INFO TestOutputStreamWithPartitions:59 - Initialized and validated org.apache.spark.streaming.TestOutputStreamWithPartitions@7e308c04 -2016-06-13 20:17:21 INFO RecurringTimer:59 - Started timer for JobGenerator at time 1000 -2016-06-13 20:17:21 INFO JobGenerator:59 - Started JobGenerator at 1000 ms -2016-06-13 20:17:21 INFO JobScheduler:59 - Started JobScheduler -2016-06-13 20:17:21 INFO StreamingContext:59 - StreamingContext started -2016-06-13 20:17:21 INFO StreamingKMeansSuite:59 - Manual clock before advancing = 0 -2016-06-13 20:17:21 INFO StreamingKMeansSuite:59 - Manual clock after advancing = 10000 -2016-06-13 20:17:21 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:17:21 INFO TestInputStream:59 - Computing RDD for time 1000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Created RDD 0 with Vector([-0.35685422167583125,0.9995288307555166,-1.5877809386448758,1.210845548490831,0.56222524157234], [-0.45216537133396134,0.8040574757345051,-1.5589888701250443,1.115603017032467,0.344649973028188], [-0.3801675945145341,1.0697589561731022,-1.6486682116186815,1.2790268654063703,0.5116371486214886], [-0.545300880562902,1.1093057253978866,-1.7582382472264182,1.211400006292075,0.2932044815946008], [-0.3995483137357882,0.929119621864747,-1.7401380541085598,1.2990457426846032,0.299090686584923], [-0.5964001160322344,1.0378164371074625,-1.553411818459791,1.2193968564632125,0.4492703031123452], [-0.33949398265706626,1.1369769008642154,-1.7816495259375202,1.2857769527411067,0.5642853362323552], [-0.4522936770981281,1.090819158309221,-1.4785463285656288,1.2734882859357213,0.31864385887835545], [-0.4895260213585818,1.024567290792244,-1.7744155605537828,1.4515329700677304,0.39720325477885093], [-0.5479321442182269,0.8926951810306638,-1.617388979581139,1.3192738326981768,0.3243774353000105], [-0.5468296230191492,0.9034725811394896,-1.8269524875493084,1.161899613497355,0.22607191229030854], [-0.4344367225435325,0.9054178856043291,-1.6658399540877864,1.13578899284944,0.46511292492021983], [-0.5914600674966851,0.9727747336030947,-1.7411162798640765,1.3902196062020102,0.43158411906136107], [-0.5644425684165094,0.996354296088912,-1.817848647277835,1.3768030733201289,0.28609432985591315], [-0.44148386701129744,1.025737906243747,-1.7073695034902865,1.2539087121413848,0.349242056080172], [-0.5287113084613145,0.8475916475163896,-1.8166057479945195,1.238631411179877,0.36922519791770014], [-0.2540595910515851,0.9698131032346381,-1.5866552625154318,1.2179915976157372,0.3240398576394729], [-0.5026644816450073,1.109581665015373,-1.7151161774591086,1.3901513268104455,0.39042855132933163], [-0.4131172722481001,1.0096927045418442,-1.811286466550916,1.2719080737538961,0.28293757989913904], [-0.4515790075180954,1.0935474561719332,-1.7896107864561164,1.3203383417885988,0.2994362800834273], [-0.4171607573128071,1.0818260184642114,-1.8219543038308885,1.2496462030096327,0.4964658371791977], [-0.45586902589910655,1.0378742745036647,-1.7000458648640797,1.446592332373813,0.348566870310007], [-0.3532087778209343,0.8044437973592731,-1.7162629066191757,1.3339281879504057,0.17946188844185906], [-0.5154063008996834,0.9595656827532482,-1.6519653379002484,1.322654750805598,0.3008929692388781], [-0.3967887546418169,0.7908879805460386,-1.7242582851383725,1.2141584886700174,0.36827019559433166], [-0.48352710482779404,1.0255449792765485,-1.7171397454598174,1.284877805985566,0.437232715780399], [-0.40324856443707463,1.020485534999759,-1.6517518343613002,1.2864020019792224,0.3984832850799499], [-0.6517069948565133,0.969461018860354,-1.684015510299471,1.2027906385718596,0.2014860155598368], [-0.6148278408211463,0.9795135452912322,-1.8642733722424907,1.2720240024566645,0.39785426587083933], [-0.6409674819545198,1.1778584543507387,-1.8094969072333307,1.24145841048891,0.1624672183638662], [-0.6368691555002903,0.8170131462860201,-1.6827833832635621,1.3031179618920097,0.31628673987022604], [-0.43096648399184834,0.9655746876954024,-1.7845188100637381,1.4052721272088269,0.5845842279495], [-0.4442125841627056,1.0257872008493307,-1.7242589434969455,1.2745428690859977,0.39998857867859966], [-0.4527127515828855,0.9000469054331385,-1.7497092961121075,1.2451107418571954,0.4344380036579415], [-0.6030914309562295,0.9699928952430515,-1.712604726536663,1.38261404269885,0.48344898725768926], [-0.34976726689041593,0.9355870894961762,-1.5766155567170825,1.3590936489794139,0.35690680463154484], [-0.575628267406687,0.9538063357613722,-1.6445003616008003,1.2104804829930789,0.3911315521177748], [-0.5582822637309535,0.9339694428739754,-1.6511037013459962,1.4401460976648193,0.3607766390547849], [-0.5468811641579405,0.997556943752324,-1.778985900216746,1.2440321529105447,0.4204585578447196], [-0.48994207115736743,1.0755700976056979,-1.5322042869816734,1.2132674597398667,0.5961642366275318], [-0.5345824744371771,0.7659168558371742,-1.7009697565551642,1.2200491259133113,0.24064266183571642], [-0.4425411108434083,0.9636579302553172,-1.6769715787345627,1.2060504970822528,0.1832578105698996], [-0.2803908635114948,0.9814260387904405,-1.6744903280757537,1.438894231115956,0.5307857877828275], [-0.4646639723438268,0.9422153316765177,-1.8936280948227913,1.1755014401606392,0.3269897050872188], [-0.5794967530942349,1.008452319256215,-1.8095212163297067,1.3053459835247618,0.3862084672749757], [-0.36564413314560745,0.9550404460199052,-1.671332555981891,1.2387784773127055,0.29524181369582553], [-0.4947387784852314,0.8968969114403345,-1.5032027608518188,1.263293789437244,0.41872133210307433], [-0.5901163249097482,0.9195890740589755,-1.76975579729397,1.2233696400583831,0.3403182341587175], [-0.7175598778778954,0.8837946345395844,-1.8329343739105675,1.3562350855120264,0.30734388952604197], [-0.4376962364418643,1.0046436887819825,-1.5427982731479972,1.1077094728701697,0.30461693506090337]) -2016-06-13 20:17:21 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:17:21 INFO JobScheduler:59 - Added jobs for time 1000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Computing RDD for time 2000 ms -2016-06-13 20:17:21 INFO JobScheduler:59 - Starting job streaming job 1000 ms.0 from job set of time 1000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Created RDD 6 with Vector([-0.5385984930004446,0.9989931115946847,-1.745310682734983,1.2763556288677276,0.41555544930746574], [-0.5140757026211953,0.9616396809310985,-1.6092748223737894,1.3080875431688948,0.3958240822167327], [-0.3317121400990889,1.1311521924692884,-1.5940394948396088,1.235663715627094,0.5352186835894586], [-0.3566003302425871,0.8649446433306678,-1.6281158497086519,1.0841327390682376,0.33108928300382867], [-0.4898654567486354,1.0095506134582393,-1.7900184464939166,1.4376638486285889,0.5476994415325478], [-0.5067718408631526,0.8330942372560937,-1.5447521454199076,1.3956201717944832,0.40502500771937533], [-0.45899690569395396,1.0114199686087044,-1.6793704227956527,1.2264029484806054,0.3617800498770533], [-0.42389468031403765,0.9744019756374155,-1.4339024359998267,1.3506788012287876,0.5071391796420295], [-0.3445608603995446,0.8999444931201762,-1.8295489983610813,1.3813011802363127,0.4406110935124823], [-0.4720580057434039,0.9616388382715773,-1.7885815681274932,1.2231064430079939,0.49611588805959794], [-0.6877856894979741,0.9509033529307679,-1.7111111174594673,1.1650825853084499,0.36541463301831684], [-0.41906769746785616,0.9643844813018809,-1.7088669498454399,1.2743442947780539,0.37857583863641653], [-0.45604703784103723,0.9704467329255722,-1.6908561828878124,1.2661005251088149,0.34761517755721416], [-0.45216413072759976,1.125638260528155,-1.8672240017957615,1.388348280279974,0.41289821746105937], [-0.6041998663111035,0.9518745991902821,-1.7482553616491074,1.3027968384522153,0.33003148862622383], [-0.7645915689875407,0.9231306641016475,-1.5908706609397762,1.0416797405804958,0.3435391589642622], [-0.5511129470810242,0.9789516647349409,-1.5896352104287992,1.2985407772348172,0.3074739718251722], [-0.3060111421519473,0.9693226097887153,-1.6503499878201646,1.2050435546957032,0.20529101796457794], [-0.41814813350738833,0.9065345647328998,-1.7369684308945867,1.3095490239413528,0.3931707179477215], [-0.7452070526618801,1.010695258373889,-1.6799990639878284,1.351913921189404,0.48041731347418104], [-0.19474072480894772,0.8574730260551575,-1.7114945513022097,1.2484211088395596,0.35791459113273333], [-0.679333385903053,1.136043445571062,-1.6819398713814546,1.2914383599572574,0.4846586154592743], [-0.38963314954900546,1.0499014443531718,-1.7075189569343263,1.3966853721193921,0.4922463621111953], [-0.4771733790096556,0.8615507710844875,-1.7824679274745288,1.1550649309651535,0.4634518570903764], [-0.46851141840829336,0.9490804068188984,-1.6654371185103947,1.1748885058064462,0.362811152170017], [-0.2486779458724137,1.0146748639649252,-1.8129854069448728,1.3198830969564963,0.3848127258680788], [-0.37278156677721497,0.9026389048304428,-1.6338760097656873,1.2146194636725334,0.305407696426058], [-0.5109164214840503,1.1025203353037651,-1.5805383770959172,1.3659800632138837,0.2507490333376023], [-0.42658774618748607,0.9583040463281658,-1.535401873083165,1.2270967800953845,0.30931926334482973], [-0.5761089637758711,0.9608023900436299,-1.6773446541045054,1.3896056325873751,0.3003385242281313], [-0.333657352857868,0.9284640162518879,-1.6506283984425263,1.4233119402331722,0.4817086358456392], [-0.4131260123218083,0.9257004348959492,-1.6385111064595121,1.2553085674173774,0.35994561408421055], [-0.41619483777131644,0.9791552402269297,-1.6621579631275758,1.1639596596963502,0.3356350519347727], [-0.42150886121352676,0.9497809597217938,-1.4493400602428224,1.2037323844449976,0.28412005647965366], [-0.4036704221590921,0.8682473056100086,-1.4499044130725358,1.4367312949083602,0.29801296100779273], [-0.7347169576181178,0.8472443819718873,-1.6912964247755202,1.1098267823656842,0.5128488081575056], [-0.4599723079910277,1.0313773063743503,-1.6693647567489862,1.1183374503284873,0.3065899617042307], [-0.5155727886799809,0.9793287424722477,-1.81766244647388,1.264051537384318,0.38002468301378506], [-0.461809330593855,0.94300958630738,-1.7083227759840551,1.2779916838408791,0.5269758739704866], [-0.6518981010977536,0.8745407308399856,-1.5963574332371322,1.2679054180945675,0.4099968212180263], [-0.46480686752956457,0.9217771428753339,-1.5266876398223685,1.1382426745675795,0.4778996312070509], [-0.4852820635337406,0.8218636910073019,-1.7767449598018135,1.1371130292343117,0.33381426065377506], [-0.41472873571993296,1.1622114863170214,-1.464996886657917,1.3123782230466363,0.2979393535788235], [-0.38817680425075585,1.0268858467994189,-1.5893492622912915,1.2621731986065745,0.435168936860357], [-0.4138872105825818,1.0639679744840822,-1.6793705616475731,1.0262313606697269,0.26542788201223744], [-0.32689002022123786,0.9411685364764196,-1.6079547828400877,1.222649045231373,0.44202772328420886], [-0.4534191990015764,0.9019382096024549,-1.6042217752367725,1.313476911051435,0.3118812623960018], [-0.3642257288812764,1.0568954661740726,-1.6408419112357935,1.3879191143662102,0.43428969659022393], [-0.4436033491575346,0.951120701402773,-1.7396961236381308,1.0696513684387006,0.1983721283216616], [-0.4195531379041837,0.9339655248016878,-1.6369085623469266,1.2113780206799198,0.32646374286948043]) -2016-06-13 20:17:21 INFO JobScheduler:59 - Added jobs for time 2000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Computing RDD for time 3000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Created RDD 13 with Vector([-0.3676131534931217,0.7205292433695121,-1.6148690586461516,1.318613134832133,0.2894000707552024], [-0.63227225792052,1.0480160164806966,-1.7186123125513257,1.3123133303248091,0.4835019323786134], [-0.5130669794355894,1.0744409567693907,-1.5423474727562314,1.4075951590487494,0.34400766625670576], [-0.6214787708363003,0.9941757670924664,-1.7059517928152113,1.1668792946674045,0.5351190865534026], [-0.5610084057337338,0.7736203857280138,-1.8315479385898956,1.3209260735925115,0.28582121198324356], [-0.37376999266386424,0.8253313331048502,-1.7924436246324382,1.3068506619750972,0.2325763177837035], [-0.5087037004414274,0.9607866442106467,-1.812291073591455,1.284458726328898,0.5328516211621773], [-0.3727437082846784,1.028767182687076,-1.7448007272489838,1.0984753757565202,0.3541211398247694], [-0.3814571980697206,1.0819060589675644,-1.733976127510556,1.2515969788209256,0.3640947425626813], [-0.38693583578076807,0.9054649043220095,-1.601255604158471,1.1869423507487187,0.27285330937505237], [-0.453879517976784,0.8265450932182703,-1.696264660015251,1.1219045234195482,0.3528336963815211], [-0.6440396833150053,0.9147072341447406,-1.574634779931242,1.2807375427757628,0.3400101945021918], [-0.5574376128841423,0.9786892277605028,-1.6539643457077906,1.2987195892477945,0.22091568760798275], [-0.3219564568616685,0.7344099605460779,-1.659291949575684,1.3277750279213558,0.23553171253321456], [-0.42287205017678026,0.8870628278654091,-1.7860892614211916,1.3638413188637932,0.35992654268460605], [-0.5114214572125407,1.0739488843585667,-1.803218590871721,1.2580620741259871,0.42523338095005675], [-0.49124237559267386,0.9657816736108445,-1.6156494419042913,1.3063855681095915,0.54155197550267], [-0.3951919316276779,1.144136446459301,-1.6606072772152598,1.4114586343482727,0.3224073809014021], [-0.439470602003815,1.0070997021461388,-1.4975209021956797,1.2890203902600919,0.22540217793727096], [-0.6679727959214776,0.999190248905179,-1.635890493454145,1.2938379558735829,0.20736771134970797], [-0.4118317638697319,0.9069236714993869,-1.7766219570089812,1.2254241772422947,0.3818774093786991], [-0.5019285203332016,1.071358441302261,-1.8681705848296024,1.3543938347082431,0.3694102461708769], [-0.43086686887184694,0.9871654497649167,-1.592715663423245,1.1292659913487786,0.24090456644129066], [-0.6554222411786117,0.9775977489456485,-1.7739871948427586,1.3756970876324615,0.5345682469155936], [-0.606670169993634,1.0414671201540915,-1.5816535152099802,1.2849832898961777,0.3443771799618759], [-0.6330347409363863,0.7614040653882075,-1.6420712005053097,1.344034328413472,0.5019032888808933], [-0.5268140556262974,1.1016458967835785,-1.581552701703922,1.1842999784487596,0.3683353834757425], [-0.49900894372274485,1.0645644739185587,-1.5355753177851752,1.2694091849907172,0.33468156327241705], [-0.31358398021960576,0.8302665811856854,-1.5422704772273437,1.1506707176948,0.37999408831157194], [-0.557673046622542,1.0913399597085338,-1.5871539417007638,1.0478537277843192,0.31421916981662246], [-0.4100320104849945,1.0606699393377843,-1.764930551709285,1.3173125309085043,0.32112514544554777], [-0.2578093453260759,0.9604640936063775,-1.6498739501347253,1.251662741314794,0.5352359752706277], [-0.5830353039132012,0.9147683830431232,-1.747368848835465,1.1923888734167842,0.4792523557272257], [-0.3091820573873779,1.0228642342621612,-1.472857793608509,1.165376819972311,0.3758448993415674], [-0.371190450361484,0.855956409330075,-1.82995473329898,1.3390818705810552,0.37113293274981823], [-0.5539807557276546,0.9677998945369973,-1.6350804443189515,1.3419255190805948,0.44899955987573437], [-0.5376287808107136,0.8413279828562651,-1.6762865574706214,1.19949852014792,0.5047576712678563], [-0.6003249423212255,0.9906964773703084,-1.5370089708944512,1.3072479529851824,0.430528835393197], [-0.4175368422724912,0.8187481538877078,-1.7996629160543502,1.1879999792808238,0.473510867372402], [-0.4828257087231799,1.0612823363596418,-1.6425767546040255,1.2270450796716112,0.39748339044894354], [-0.6332333955565639,0.9519761064158869,-1.5479445620477186,1.308465915443944,0.40088093604627134], [-0.5228584674134156,0.8768992895766171,-1.8336272924164618,1.1499724885170817,0.48668761341650246], [-0.36341836902693225,0.922366126760111,-1.4327493627862522,1.255336229334989,0.29637579738564857], [-0.45752242402693893,0.881182369510346,-1.8075367810499625,1.1836209711736738,0.5240597883932867], [-0.45127422400043055,0.7875661584033756,-1.9374943002165643,1.3022768816466794,0.3865261883663946], [-0.3385546242930144,0.8796112795269294,-1.6285471480854194,1.177302672296305,0.3611750358009876], [-0.3866867752880084,0.9445769802243262,-1.63589592788523,1.3115437594462822,0.30025415854590604], [-0.35087047853321573,0.8048561114260069,-1.6540296199948705,1.1069576251024358,0.3739108460295009], [-0.6118021930146711,1.0836229312504084,-1.6965417516081935,1.3814661848764345,0.5544639396063888], [-0.32579850447352154,0.9627503718472757,-1.7118868486570202,1.3378056459968657,0.26239820187294716]) -2016-06-13 20:17:21 INFO JobScheduler:59 - Added jobs for time 3000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Computing RDD for time 4000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Created RDD 19 with Vector([-0.4302807096972894,0.8599904088505619,-1.74473044235192,1.4559107913903748,0.3868065070182458], [-0.4809066717988696,1.1355083469098752,-1.568604721573235,1.1620770786476713,0.5052691040344791], [-0.37948367495671453,0.9612314328384234,-1.7288918432476479,1.1900277125124938,0.584148529955661], [-0.4830118079148656,0.767521114064722,-1.626523514538083,1.2344387914004133,0.2949434953618521], [-0.45982168221474795,0.9610478654584301,-1.4181098052958898,1.1960502956938037,0.35754207882888805], [-0.5403734624385554,1.0961985914256847,-1.4824794478647174,1.1331480033700656,0.4391107899439444], [-0.4078934785304544,0.9024846949827434,-1.772449379651882,1.1891755675558509,0.26886721197216756], [-0.4850713563393695,0.9854430300369302,-1.6392713106609738,1.1640485301086894,0.4058728213551233], [-0.5687863647866291,0.8969696803234791,-1.694332984322472,1.263598085578097,0.4916946976192132], [-0.4852057062952629,0.9746523770852169,-1.7152410694648534,1.290070394136892,0.15734923086759972], [-0.6042639508376677,0.9587258437320635,-1.796821218335353,1.2415150801911263,0.5036349732580911], [-0.5634160393957598,1.000927422915143,-1.7677443118596388,1.1056602828501068,0.3662560181172587], [-0.42260218853055576,1.038402090735988,-1.623454196568306,1.258759507363921,0.35817719144396376], [-0.6935865876752814,0.9767485891098648,-1.6119749119230544,1.3241871653009805,0.45959708474651645], [-0.34982681977824615,1.0697929536346964,-1.6430495277049175,1.5160135223968076,0.3720168274191172], [-0.4965183720272103,0.8867675212850072,-1.6326283681810188,1.1014099972707194,0.3464777829375711], [-0.4429523196454975,0.9569586297887442,-1.6975710388611154,1.3240706523955248,0.32888337316923105], [-0.6110986163387908,0.8513837967033338,-1.863122720264613,1.1805944382455067,0.36642329779978594], [-0.5770973903898762,0.908469634420286,-1.5969552518736576,1.2639442528889475,0.30334073173665643], [-0.6677676499752639,0.9993894428543435,-1.7484246181717757,1.3024629995033956,0.5254188589994735], [-0.43541597877213595,0.9518339300904892,-1.586032231723223,1.2859374872876397,0.3728538289653893], [-0.3981355023125287,1.0914065768869097,-1.604054951142783,1.5597686669581905,0.3200687403051776], [-0.3526167305034049,0.8582438505591754,-1.795079875034891,1.291753863011317,0.5193327226327783], [-0.5322224604485944,0.9225091285363366,-1.5406535924270095,1.1740734779955952,0.3985082190639408], [-0.459298613922177,0.8499917798311437,-1.7381604837279123,1.3363117012403978,0.39372367283850573], [-0.5029856763543168,0.9563691025906603,-1.6323146191672946,1.1564017424707789,0.2992277161858537], [-0.4702645909815308,0.9857893178047549,-1.6146159696178013,1.2941994476820615,0.29510677438190996], [-0.5010373585673528,1.0381084928077213,-1.6592946006629066,1.2275293781382213,0.6215806870134133], [-0.34251104673790145,1.0920152105960284,-1.4702129754795525,1.3052389503227437,0.5142842632664187], [-0.6014879705809496,1.0573110467882678,-1.6994508095500127,1.2345579924787826,0.27260709575033387], [-0.374393054126274,0.8520958347461538,-1.6227534538993043,1.400050919842349,0.3749155714488258], [-0.5537306132792297,1.0237199970470363,-1.5996513021748293,1.3277912515047763,0.3298858295939718], [-0.25019038644507985,0.8933568126093641,-1.6180540579171452,1.2514548524704747,0.4485167299028402], [-0.4083941282128968,0.9821808764315939,-1.7710803507794117,1.22246902401698,0.39616717157783654], [-0.6617567254520433,1.135205204672421,-1.7404704566578575,1.2634534028343156,0.48631393245586374], [-0.5102944111175142,0.7749069961072899,-1.3746153795885292,1.1761695768038691,0.44519535098184526], [-0.5273802259607313,1.0293844883271042,-1.8327600208787889,1.3032607095759106,0.4810318980867238], [-0.43680250540361804,0.818212719795147,-1.6397291545525832,1.2164204417976607,0.36911234859465697], [-0.44511271834694277,1.0039279649012336,-1.736482991573775,1.1407721179472239,0.32026695790267784], [-0.461571293170463,0.9623234284494812,-1.608651111923442,1.3054108488252942,0.2808744029932673], [-0.44626644352032524,0.9385912062499339,-1.5784088711897624,1.2996894478753678,0.3367841237735643], [-0.3458002275610911,1.0002670117410046,-1.556743986487141,1.220789398824709,0.3047413342602086], [-0.7006173067436341,1.0226530560056757,-1.7453818996615391,1.2468257997531524,0.43730757861422237], [-0.4921877365188532,0.9365903329931373,-1.6954478906187997,1.252660426269991,0.411305000260049], [-0.5866727516355298,0.8540617355784135,-1.7495419546675461,1.1992666379768737,0.5624798329629086], [-0.4552502733367031,0.9291860448712651,-1.5908193525700438,1.166718738832733,0.3281256456654895], [-0.3426613548769225,0.7538960431270546,-1.585524926597853,1.478740164978157,0.251131880834575], [-0.4680826818575833,1.1112444944704614,-1.6587653968723466,1.2870287092936261,0.4050920708933554], [-0.5624024717932173,0.880862871462712,-1.6303493150007178,1.1721463768303386,0.5696913393311782], [-0.48266079841446297,0.8916088275950524,-1.5393517554424276,1.1897790454674049,0.4411535715241579]) -2016-06-13 20:17:21 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:17:21 INFO JobScheduler:59 - Added jobs for time 4000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Computing RDD for time 5000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Created RDD 26 with Vector([-0.3816421005537525,1.0617035428188992,-1.7399461177422264,1.332980235806715,0.2916494841747297], [-0.6583584440016957,0.9555891409969888,-1.5454326525102315,1.1741091058996926,0.305220337835565], [-0.2989177604038451,0.8701688291879833,-1.8804582484676076,1.2986162711936382,0.35958131004690164], [-0.4939772883087695,0.9969220817446861,-1.5980382534286326,1.2969252872376784,0.37976918116852443], [-0.35668565131454827,0.8078147339434413,-1.657193160993533,1.359095599717262,0.31192105888186633], [-0.5450865703006346,0.8750391278919141,-1.7453306059026725,1.3721329076982913,0.42237633518141116], [-0.4917580165015238,0.9512146417235625,-1.9326523267650781,1.4266827266379667,0.3225713722197544], [-0.4048718474607664,1.0240256182146898,-1.6774401089659943,1.2669846165410654,0.33732641244233585], [-0.47579982131353826,0.7947369494624074,-1.7655419119652316,1.082577012553155,0.35401987740372537], [-0.35538929020910864,0.965963799724578,-1.7529404087985827,1.2055043918858341,0.3342486683177578], [-0.7014541659835856,1.034642549879647,-1.5544540159626516,1.3033886639865033,0.4054677613509847], [-0.43610110990224626,0.967077951705683,-1.5262368892289788,1.2639835242537958,0.3181179758885862], [-0.3425524324739105,0.9894095680520077,-1.7777745234498326,1.3091660388427224,0.331841648562285], [-0.4878205400579338,0.8560103963724833,-1.7352326850283464,1.3664274889095067,0.3984111315135807], [-0.7121859062829006,0.9130040407010276,-1.7874802517046422,1.1874648639293004,0.45017886399842666], [-0.696643078139105,1.1467663562988957,-1.7296410056290408,1.296144064682525,0.5048605187772839], [-0.4011230886898419,0.7361859073107719,-1.5317733987729902,1.2701183330425934,0.46062351610078545], [-0.45715694326129586,0.9190057703920946,-1.6904193500197242,1.3271044644288563,0.4842403797714084], [-0.5095249256026796,1.0659631362797184,-1.4402626928522713,1.3379987286264212,0.2673039786797317], [-0.4167094825032959,1.1091402127599272,-1.705921687753808,1.1183411370664753,0.48305727426817385], [-0.2924464534867928,0.8213969946455459,-1.662853823551024,1.3355427142685694,0.3770025002372693], [-0.4444725119583886,1.1046045679759258,-1.456164663000235,1.227847061522124,0.5255642346814823], [-0.4048329834921757,0.9788094275209757,-1.539852245269681,1.180160780953293,0.6787253334538687], [-0.38022990049474503,1.0144858953551337,-1.8072904612189131,1.3419776986257266,0.4041658441938728], [-0.3763555278351052,0.8998086161617835,-1.705061140148122,1.3495005676173746,0.5191631674639813], [-0.5540229100290982,1.0669438671830174,-1.636684794385797,1.3459012222977915,0.4181134505939225], [-0.45042550789662783,1.113240852339999,-1.486260571622101,1.2719922983443672,0.4102349177516307], [-0.5546179681493353,0.9938406825700987,-1.7084301934096207,1.3880541591389144,0.29690506797892435], [-0.5006405047598609,1.090292575906863,-1.7104271793907637,1.3062089077771892,0.47556018075456363], [-0.36520205819569046,0.9201397468605166,-1.7602021590969437,1.3876082296931265,0.2308718010458195], [-0.4938168714972593,1.0964903470900649,-1.7260349751744855,1.2048249569339422,0.40946925790527117], [-0.5092411838541296,0.9882559808204627,-1.7212908932580275,1.1127850487389395,0.3940121165600333], [-0.5882412062939182,1.1156935762589453,-1.8131619026625396,1.4146422480126442,0.31361816553861704], [-0.46990162437316874,0.9908156946435108,-1.748827835513362,1.3957588501434537,0.5512480921903505], [-0.5405868391855275,0.929247220281523,-1.5894550246414005,1.271255300937136,0.46529614772048844], [-0.46376208782353645,0.8174067828364345,-1.711422288030576,1.150388298327176,0.36268778857829526], [-0.4617415170232849,1.1203248336099854,-1.9346820049945674,1.1466858815338403,0.318614186382022], [-0.5200553635156823,1.1206383922863123,-1.721393160577411,1.2564342218266722,0.39290287510750027], [-0.45474634000888675,0.9538310968737261,-1.5572305892194418,1.2293659662686987,0.3951581444726625], [-0.31024997329419923,0.9626097763501761,-1.5774105560033171,1.3323705378196256,0.28213579823419327], [-0.6425546702182562,1.0413861558054223,-1.6696029184789263,1.3297020370998587,0.4667426127538354], [-0.5088483797878551,1.2170009632967245,-1.7143214767417212,1.2915402658283741,0.27813229259982863], [-0.5138048643277845,1.0537687232096613,-1.5306968187748822,1.2020601237200264,0.27213168784492703], [-0.5375432544160766,1.0418784698921373,-1.501800504998784,1.217708599285121,0.6443267111319482], [-0.5784571285851817,0.9518620188834632,-1.656094482882889,1.2785669143560554,0.34103467230048085], [-0.39057405488844044,0.995936983645342,-1.6323671359433234,1.271155031008547,0.3558239098317178], [-0.3682240577265206,0.8711375369867465,-1.6625627384453794,1.1969170451325948,0.47233733398400285], [-0.446008720311628,0.8976699326654971,-1.5289286019943875,1.2244855320582673,0.2984778353929738], [-0.5102538395816835,0.9986885141123109,-1.6414035100650266,1.2420020420446427,0.36032207135032196], [-0.6350165259371148,1.0900186871655364,-1.7907186442371252,1.4304360117451347,0.35387005412887984]) -2016-06-13 20:17:21 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:17:21 INFO JobScheduler:59 - Added jobs for time 5000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Computing RDD for time 6000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Created RDD 32 with Vector([-0.5768797964115748,1.028505413551091,-1.7521414076692798,1.410931775189879,0.30527264662469095], [-0.40591177777424625,0.8661878776717759,-1.7310200236861213,1.3594055750354543,0.3299548132445285], [-0.6383561683498007,0.9259033203935216,-1.6881320011035645,1.3172994752602565,0.3214908371039436], [-0.624062828319168,1.1219523756569185,-1.5603263506923595,1.228964619537528,0.35487057209905903], [-0.4734064808910865,0.9649955190976767,-1.6068693471541338,1.3196307767213002,0.37116726129033706], [-0.6101841770121739,1.059654359501757,-1.8099667264246553,1.3759685981009824,0.5438261473052933], [-0.18850356552447423,1.1440264277756391,-1.724865192544724,1.1359044353122627,0.31757393897273056], [-0.41254200355821263,0.8607204799101216,-1.616620448535661,1.3427363220425623,0.4109728277211949], [-0.393734447592108,1.057690953958827,-1.6695747980566427,1.327034390345423,0.13819106599587025], [-0.32090427274943434,1.0651687359751123,-1.6980368198411557,1.4017902860954718,0.36022562482340503], [-0.6053061980873911,1.0041632375635317,-1.7820663797212062,1.2269162643461793,0.4840114520476853], [-0.27268808986968307,1.013170380627259,-1.6005981535949472,1.4382064950763866,0.3892224005152082], [-0.535422550377724,0.8375173015602825,-1.661788348017347,1.3227481832299182,0.311533159171815], [-0.4009368458319039,0.9897832871295871,-1.6696688300152116,1.4424037410980737,0.32734398317968916], [-0.6825087101818776,0.9056073621397804,-1.659854864696752,1.2759644672519115,0.43884955280528926], [-0.43179250129551383,0.7465728726494555,-1.5636729803345486,1.3457117728614754,0.5721971492726551], [-0.526304418227463,1.0501002903902692,-1.5968858098432406,1.2982785754583357,0.37494021304786934], [-0.4320225453412649,1.1715904807145918,-1.5178148547272063,1.348200261386646,0.40779395798035983], [-0.42221995667102147,0.87623418086733,-1.73837906100808,1.4183919259975142,0.33228674704284467], [-0.5629520941212462,0.944212875923361,-1.5546071736198348,1.4519582067498829,0.262200899044119], [-0.30845271114048184,0.9681639591372486,-1.688842111249905,1.3763468548667668,0.3173125681909059], [-0.5198567190391367,0.9335053916544206,-1.509472274649017,1.454415814887731,0.4589687217941236], [-0.5711775989363378,1.0215061238027188,-1.6224000734110398,1.350665861834858,0.36668462127595286], [-0.39949682119974983,1.0408092246134175,-1.6587225284400737,1.301865863792736,0.4563742176523199], [-0.4902566261487769,1.0992586451801099,-1.4473476422759635,1.2309123590548212,0.2544829822903507], [-0.5293628624912633,0.8432327849564988,-1.7669603672683885,1.2911210280012069,0.41983427336223667], [-0.5738614502843032,0.9784937768658387,-1.7194271904022407,1.3116656980237082,0.46218317857671776], [-0.5902501988138692,0.9423215775018176,-1.684040151093259,1.4235274237923208,0.2039286490216381], [-0.4777727934676478,1.027586212069216,-1.7495753442806758,1.2757769414973703,0.3810309831295745], [-0.6566621783684151,0.9641681686268665,-1.5286420868141977,1.508411206136107,0.2501550388463756], [-0.46296281092785885,0.9727194845614605,-1.6171422039154086,1.3434828829348795,0.07809723939214058], [-0.5738547760098169,1.1077074999997352,-1.739110700752265,1.3494171899976157,0.3222123114107347], [-0.4002887849620737,0.9619411680270862,-1.663509769541054,1.2908149710547212,0.4244227376717664], [-0.4665533215112465,0.8596395857982323,-1.7467795346659782,1.251224446006115,0.42794754034954735], [-0.6374258626266393,1.0514903728245393,-1.651303971310752,1.236884794719695,0.5249538337308562], [-0.48340361610998744,0.8198136168919055,-1.690727257833199,1.398396991016531,0.31841739933411595], [-0.5410633313507479,0.8973900953721683,-1.4724867408929736,1.388151513303225,0.47893645155994446], [-0.3039096405457719,0.9155005236200422,-1.7173365893442798,1.256097860517092,0.24819750391337136], [-0.7090962179115853,0.8944749271173439,-1.8839838110448957,1.1446445350131944,0.22840975233443309], [-0.5100953498526793,0.8107010460411471,-1.7303477866873604,1.1647356509362308,0.4418371093135496], [-0.2962087734742913,0.7242125014742148,-1.5984563979400566,1.0588259317312654,0.28436170748337264], [-0.5505122810716748,0.7806970076803257,-1.72088933624607,1.4230022424760869,0.32673184311928194], [-0.45127942352855155,0.9959060503010477,-1.5524186364009327,1.2366972856510523,0.3578155222046753], [-0.5102066277582917,1.0949182323987419,-1.7142222646495175,1.1664611293679565,0.3602965879997574], [-0.20421136622466618,0.9866371147144095,-1.7538283133815977,1.0465139384173783,0.41274915410898083], [-0.3678643518086112,1.1128928060624481,-1.7416958328862653,1.1712943420281356,0.522586652590277], [-0.4272255324098842,0.9505411417476373,-1.5398631578697235,1.1670050581250606,0.18826195051488878], [-0.6294443147944699,1.0182211296265207,-1.6114292740208946,1.5494854908967275,0.2354819914065145], [-0.46052379116984477,0.8965751438720174,-1.6887957613207942,1.2678155037585463,0.3277249457164221], [-0.4661555388594827,1.0176554636610373,-1.660202673672577,1.1543960943838145,0.41490582984943747]) -2016-06-13 20:17:21 INFO DAGScheduler:59 - Registering RDD 7 (apply at Transformer.scala:22) -2016-06-13 20:17:21 INFO JobScheduler:59 - Added jobs for time 6000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Computing RDD for time 7000 ms -2016-06-13 20:17:21 INFO DAGScheduler:59 - Got job 0 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:17:21 INFO DAGScheduler:59 - Final stage: ResultStage 1(apply at Transformer.scala:22) -2016-06-13 20:17:21 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 20:17:21 INFO TestInputStream:59 - Created RDD 38 with Vector([-0.5184134212251081,0.9455318561030377,-1.7249072888539423,1.2476376390078272,0.539371371249466], [-0.4839353019153916,0.992343730650464,-1.4037670310964987,1.3650053852483648,0.287928402259341], [-0.45217444521287575,1.0425364789095595,-1.7771813152553952,1.2241390773159404,0.3053067128704871], [-0.34943560877945984,1.0407179160044182,-1.5177836780575653,0.9824846085205396,0.2603543256712868], [-0.45829448575134074,0.9725249117979422,-1.690056167350807,1.035512584568622,0.3026754916713263], [-0.42458145588018753,0.7854084174090814,-1.7649589009106188,1.1975914903392633,0.6551885852829413], [-0.5104497237497339,1.0402201914534017,-1.6859518565969145,1.1391556996625576,0.18881595045601088], [-0.5588856532488075,0.8709355475677711,-1.4735465340859104,1.207362168238276,0.5886898848511445], [-0.5286914346418723,0.9318873502365327,-1.654993790470044,1.274056873049218,0.29412525529417294], [-0.5420811906362676,0.9995645356070102,-1.6040227637757893,1.3412654820039975,0.3950459929854693], [-0.5105371229433685,0.8870982484234274,-1.6366208840493932,1.344244897498087,0.2842199976348651], [-0.5758159328886613,0.9361950961962593,-1.5151431024376867,1.2963264217469896,0.4195802203163277], [-0.46650906956232724,1.0883479790237522,-1.6209503836486092,1.2506119015001946,0.5239087792157935], [-0.49800785133272085,0.9121648111767798,-1.5664537098618618,1.4303891258276538,0.23796160092517643], [-0.6200401622139905,0.8819930763699753,-1.7712553265237971,1.144151394858828,0.37228012783293096], [-0.43683215654432145,0.8386127968983885,-1.6402554451670102,1.129209493050289,0.3612473529185551], [-0.4079221564079838,0.9656864104202829,-1.7359335721136713,1.4421338682074127,0.3616284206434529], [-0.43442591279896603,0.955449363348641,-1.5334677832140913,1.1666726886559795,0.5058368495019173], [-0.6457828019918121,0.931131898174338,-1.8532037937152797,1.0375316080434018,0.3133563527606291], [-0.38369598973754027,1.040856056086344,-1.7594025439463896,1.4917751006832976,0.45495744983872133], [-0.30166559339288085,1.0342180102112146,-1.5650152370405155,1.2673629485315197,0.4158807517084573], [-0.28580825731967757,0.8865395334100423,-1.4737235251638032,1.245008984525024,0.36435255784505255], [-0.34071421524008416,0.9427461052254856,-1.6571555985762556,1.116205755573728,0.3440594901972376], [-0.5707704618700846,0.9955277694976673,-1.815098893846346,1.2976867834380266,0.3708933869585827], [-0.6095164341318,0.9322989548597498,-1.6576644540187038,1.3001089252401639,0.32781904617649943], [-0.5884784232045771,0.9963121018549773,-1.4726908596796255,1.1232169328419297,0.466258074909203], [-0.6351651621700529,1.0173421935079714,-1.7981429560407272,1.0336059974520695,0.28162584897435694], [-0.38203401021631267,0.9790771913538491,-1.7244683271662424,1.288084387752797,0.4312827591331814], [-0.37183909143628785,1.1926117173870312,-1.5166716166223042,1.1696263300990144,0.4515586032102953], [-0.47462520704983,1.0498070539706301,-1.6863851371590932,1.2965114835042095,0.4645837570114517], [-0.35874130630231327,1.0042819076442355,-1.6407835501913508,1.214144240421076,0.35534792382876684], [-0.41389636789198914,0.9508170625312302,-1.6498089027313338,1.4272197180381103,0.374553819362672], [-0.32132785261092967,0.8535769504185499,-1.672835438070241,0.9421132176631009,0.24740658936307364], [-0.48253792736985773,1.0429236941137325,-1.4939460450129358,1.2832409419178106,0.40323476275374437], [-0.391512958705594,0.8859909853510953,-1.607045573914962,1.1770226037327562,0.5198479967492], [-0.4833163026675927,0.9200058086737815,-1.8188555219632765,1.2111735788924018,0.39618714901430313], [-0.2913135273832381,0.9301485161935364,-1.6975841978124846,1.4065399033140502,0.28453881400648173], [-0.3073756680829076,0.8325242716417236,-1.6753209327630654,1.4075134801045086,0.30351866480787215], [-0.2775051790445552,0.9452680429408632,-1.6224110577930677,1.1778711923142833,0.5531745712889398], [-0.3047686297259775,0.9468967188067203,-1.6929112749188848,1.4636396431851675,0.3586449109599954], [-0.5956899619218391,0.813236818549011,-1.757649523640403,1.2604654777105053,0.3185144577112101], [-0.32794499152459505,0.8140339815697406,-1.8666906537843855,1.2572887839007447,0.23549894606903882], [-0.40686690942678344,0.8619998816707273,-1.6319579560604098,1.3664940728963186,0.422156696202466], [-0.4138601150243667,1.021828823586237,-1.54841612428864,1.2398622089963618,0.5070042357490803], [-0.331869975870002,0.8782404384527787,-1.6635875859844171,1.2435143760020457,0.32101906837863314], [-0.49700214293681766,0.9856184010338479,-1.8232422669575907,1.2421051455581202,0.47592591320323163], [-0.535909591266969,0.9807383109826389,-1.6974325972167972,1.334548321075211,0.23946350824403165], [-0.6444021824207316,1.1849780210499694,-1.5121795557147575,1.3604386920675193,0.31690181205044465], [-0.49746220541003966,1.0643703042656938,-1.8499798436118189,1.1357972819777942,0.5780379400362996], [-0.4383285608503117,1.0163805234573544,-1.9030246206122645,1.438368352860807,0.3293850463477472]) -2016-06-13 20:17:21 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 20:17:21 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:17:21 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[7] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:17:21 INFO JobScheduler:59 - Added jobs for time 7000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Computing RDD for time 8000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Created RDD 44 with Vector([-0.374705866875181,0.9840691262184086,-1.5891551104970965,1.4814595125988181,0.47065837960007695], [-0.288810885883263,0.8437842751113669,-1.5395954061634287,1.1874811761270259,0.021249409500126437], [-0.4606335450965845,1.1012472510905122,-1.8287426699010534,1.24172318938614,0.4710333368456244], [-0.42944940507321583,1.0104988720375858,-1.5751317017658801,1.2831547194950401,0.3512701703197814], [-0.5152127056418667,1.1709583832128079,-1.6554391510043867,1.1397001494774806,0.23397277605773995], [-0.4399479870409779,1.024335104965922,-1.7045108897162344,1.196833551305392,0.4117115230011916], [-0.5430543625505585,1.1370095574712402,-1.7176429060334013,1.1901279759925694,0.46070443041212095], [-0.3968714897698767,1.0926860330748331,-1.6334286394688688,1.228234808051336,0.4665607383628074], [-0.46215342653947683,1.0343329141616782,-1.6499943761334228,1.337530812749641,0.23114556003826187], [-0.5000235665427208,0.9151728338053156,-1.8055132514640995,1.3037686620331144,0.2902025184248946], [-0.309233049825866,0.9728959274674948,-1.6392691162173578,1.101247705752869,0.3496423465401415], [-0.5981691090699208,0.8739296001671861,-1.5687282698937923,1.2550952881121755,0.46308264317451964], [-0.425561195913103,1.0676958602044149,-1.7135165364767921,1.3524692002054561,0.23228520033456318], [-0.7043177723021486,1.0283311625585259,-1.7478644671187442,1.2858120358708933,0.3418710936510813], [-0.41727892194293686,0.8625867727705405,-1.7914100057674134,1.1786792695315231,0.4917512427338196], [-0.40174243931692544,0.7828098802704966,-1.5073762714652954,1.281118669072956,0.44887930601066905], [-0.35310979651234775,0.9745364825956087,-1.6392317059509176,1.4513644518189859,0.3110119088453469], [-0.6853829628094648,1.1253397147829511,-1.4920081564639467,1.231135544811062,0.3139314829419366], [-0.44152016739085975,1.1304790288137783,-1.7325115907360644,1.3251089521914843,0.479567851123668], [-0.5526275004039871,0.9674440446022308,-1.7971453051937036,1.1664342514118335,0.31260632993291754], [-0.46562594489923503,0.96412818249357,-1.7756506925990838,1.1820367323525278,0.37368762092821634], [-0.42414895043860573,0.9369437276073684,-1.6228073393039317,1.2110129935336054,0.33600782254298567], [-0.4507513844294476,1.008422902581697,-1.6947838717172434,1.1258385535052762,0.23649573041213182], [-0.5011767543824581,1.0367482963168666,-1.8619227324329928,1.187686906059264,0.41060169959227016], [-0.29061733919433663,0.9125980333805904,-1.773384852821433,1.2318813347419482,0.5360023941871213], [-0.47558633395179245,1.0112018987992073,-1.6443927870523196,1.1868477340320063,0.378412637239359], [-0.47217857076866115,0.9027887018339104,-1.7359968088612976,1.372505255254142,0.4024631611010435], [-0.5635782393384723,1.0418520168542464,-1.6371483392749435,1.3046794831870752,0.36435743882876215], [-0.3566584096699375,0.9045378099174617,-1.6027843810019622,1.308376882406936,0.450420525347206], [-0.5161548558091542,0.9362466126941537,-1.543853038663444,1.351176902637456,0.37019671664199505], [-0.37734135089435306,1.1354760893808167,-1.5928707071728756,1.4230797310043477,0.5395830646701685], [-0.4935941975274797,0.9431897551602951,-1.7363999282764386,1.2592777434544042,0.34477211805728875], [-0.5399690948357754,1.0773952946269298,-1.623286131549866,1.3571017009557331,0.3820149575212495], [-0.47470272833792704,1.0279092716939924,-1.5583033476707477,1.2928147173688327,0.12854158780432537], [-0.3932748299826361,0.9168968642381998,-1.901294092833649,1.259188794957511,0.4011688713714673], [-0.3557493574970649,1.0003577059585267,-1.5111904842067994,1.3123818859934535,0.3019665980708428], [-0.38849198387448786,1.0634537152100978,-1.8742015500852314,1.207616787308903,0.527158124827907], [-0.32516837051884395,1.0806577326663074,-1.611781542302308,1.182593379846847,0.5196789294606137], [-0.5212725898858087,0.998490506190397,-1.6184763965153202,1.3731447165706872,0.29855378648402087], [-0.40257752034798155,0.8314340557777532,-1.6850156321125218,1.3427297403898506,0.5156164947536159], [-0.4889473969173199,0.8999745071099362,-1.4468367122261747,1.3090812041423647,0.5108182965913203], [-0.48274282778056277,1.1537075465803446,-1.5811276311378704,1.287632315416058,0.50882779799236], [-0.23362208323850905,0.9548750423156142,-1.572161725609678,1.6027003014260315,0.3939518404103326], [-0.4292988792888861,0.9366381149644413,-1.8250064321036654,1.2358072562830058,0.18892247590763034], [-0.484748914270206,0.7653236279449509,-1.7486834183984747,1.2949268753185315,0.39683334762547673], [-0.4399736279490563,0.9634644976145054,-1.7480413166448512,1.1775955441636703,0.36332416278682794], [-0.48373350642316953,0.9259530051332495,-1.7977337219360736,1.2876089889322726,0.4497462132788797], [-0.38070426405120866,0.8838236645507052,-1.6753928124899833,1.2449514905082557,0.3276558558550576], [-0.5717833110401351,1.0273299096906752,-1.6190879043151574,1.1860455839092494,0.32636997750947594], [-0.5231330958628938,0.924256801489448,-1.5484246875687981,1.3381528852011608,0.3757180006852602]) -2016-06-13 20:17:21 INFO JobScheduler:59 - Added jobs for time 8000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Computing RDD for time 9000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Created RDD 50 with Vector([-0.4026578572929527,1.0001062834205203,-1.5617908885948877,1.462733521830009,0.2061287167736916], [-0.3965287353033582,1.0683726645043978,-1.4873730061169563,1.1460792990307642,0.27746130384846085], [-0.4938197206621628,0.7709438014433136,-1.6076436749339549,1.0982028796258234,0.24486596655062928], [-0.4777650011847253,0.9489155740228515,-1.476414143798642,1.3062514135704215,0.6114439757268249], [-0.4285977940802228,0.9382568588433707,-1.48976698594476,1.1698457774900977,0.297712575086562], [-0.5993896749484521,0.8629058534578911,-1.6510698754517374,1.2090117714723978,0.5458042851487852], [-0.5321474831529757,1.0849054231681536,-1.667568850178397,1.4161664775537062,0.4493689535824759], [-0.6723705689897437,0.7517542425048965,-1.663645669684176,1.2240252996527488,0.346848907683753], [-0.40115735788941104,0.8163474979541504,-1.7694648704001712,1.1073947331922183,0.3856322637520626], [-0.4787497898360035,0.855418622108937,-1.7422359123781868,1.2371448524727167,0.5156131237949001], [-0.5321048883779993,0.7970778523677844,-1.7616773188005117,1.3821414318824692,0.48832517040159346], [-0.5285360880098152,1.0597395124384508,-1.6417892576104653,1.3464020747697325,0.31849105791627297], [-0.5324435853239631,0.9841563147023031,-1.5943852209495275,1.2707723748060116,0.4139910360672806], [-0.37531742848811983,0.7874289125205598,-1.6479869931712725,1.3181636612914271,0.3718789043065717], [-0.4560433134330382,1.0759812484577316,-1.697865869705092,1.2725177127861855,0.06883328163775021], [-0.45513942313770217,0.9498671339799809,-1.6933930804659583,1.3025139463978586,0.5350768600248874], [-0.526256522617052,0.8727003678813281,-1.4356120595661468,1.29993914263913,0.43387328603333664], [-0.3923483046111808,0.994060772829843,-1.6451266300509706,1.2983503204985913,0.3703765349262901], [-0.4374913726020475,0.8955789511391181,-1.4728897306445088,1.1357862983779858,0.30881407016500095], [-0.41951697820602707,1.0007695283696842,-1.708286281016387,1.4190975260604537,0.28731659836668877], [-0.6750333217393494,1.0688779823591679,-1.6846746627252691,1.2962992487600882,0.5030936797752793], [-0.5956866906574221,0.8121299274245898,-1.692929629203368,1.2338548732523056,0.38371772482125543], [-0.4303591738987757,0.9459620785706632,-1.884551120494729,1.4937212054647857,0.415792749586519], [-0.45504157760203423,0.9027897906482127,-1.799761299703666,1.3040399621270629,0.5736987408004388], [-0.5593669081347996,1.0931782446321396,-1.627955485174887,1.1391759184235553,0.39801309200504786], [-0.3825955534231694,0.9655263256725013,-1.7068123920171723,1.1861538643513039,0.2764471843224494], [-0.3587313623839552,0.9371265382727668,-1.487908236071683,1.3184799953634567,0.4370396706398445], [-0.475038660091078,0.797460346861311,-1.6835937249660264,1.3120302511529962,0.17031892070731325], [-0.4663190378327658,0.991352295299833,-1.6444316855302834,1.51526349576866,0.3929422369724258], [-0.5040823137977553,1.0575372259159488,-1.594322659684267,1.2971698194040933,0.5092825217956699], [-0.44790417664079346,0.8518362899333143,-1.648211585312696,1.1839806869130056,0.40832085080824754], [-0.537755895075537,0.9806637869508009,-1.7656906916731008,1.2433660326767015,0.42887767735060545], [-0.34293660736335746,0.7935119912222357,-1.6359022010243611,1.3593857228798436,0.3967833959145234], [-0.6208695835786292,0.9640100782092327,-1.874229161549032,1.343981009171891,0.20123531286937832], [-0.6723717817990262,0.9002960054466735,-1.6910864401441592,1.1818255352705958,0.5433986930426408], [-0.4588708699931968,1.2092994554375371,-1.5324768096769932,1.4083576723657105,0.3708605878943178], [-0.560638238383703,0.9092570275383438,-1.473047966472027,1.3497511209299733,0.3467764467476322], [-0.4409803705877342,0.9370238683801353,-1.572432943897284,1.41498798111251,0.2748026192628795], [-0.38741643074715315,1.071718105146961,-1.624146896559199,1.1401522638836572,0.3872910322155703], [-0.5140862086906481,0.8211106116047733,-1.6027037015807561,1.1677747630844688,0.2659604974429464], [-0.6693385873907269,0.7554393139905898,-1.7108087289854104,1.09991932687034,0.41821242261960917], [-0.3282503845126481,0.9257033715775067,-1.5235939065144024,1.1889128563024831,0.3368716615939142], [-0.5366461042406446,0.9240666515119298,-1.8135909127248846,1.3575511869067935,0.37505000938261085], [-0.39294848413804057,1.0124585311856609,-1.575086388417685,1.2467079941490156,0.40416560148386643], [-0.48738816985096806,0.8787565657383682,-1.810281961839409,1.3779890105636297,0.46019825317610896], [-0.36592476270170865,0.9132457437551841,-1.7089939281056865,1.1718159447241387,0.37244792446782193], [-0.40751855185193425,0.9541054981051484,-1.4879650472104635,1.1817057016562598,0.44919665548504983], [-0.29317450049605076,0.964443479319753,-1.656028808081747,1.2586398699632722,0.2526740377633023], [-0.4615968319838038,1.0243528061825597,-1.5054257187951623,1.351376529833804,0.42971544189574107], [-0.47988177538312266,1.0112860256804128,-1.5404476412782737,1.233668528335028,0.2028012939752821]) -2016-06-13 20:17:21 INFO JobScheduler:59 - Added jobs for time 9000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Computing RDD for time 10000 ms -2016-06-13 20:17:21 INFO TestInputStream:59 - Created RDD 56 with Vector([-0.4889010904178065,1.0197902904388123,-1.6434083794608938,1.2273894895374848,0.29880056758004836], [-0.3690054860756016,0.9083136794567547,-1.5975355891184535,1.1937534562371308,0.3514272190683493], [-0.5737783702048262,1.2273536117223582,-1.4617006566984774,1.342244921629087,0.3377586288639175], [-0.48387316003666864,0.9673070026559238,-1.716725721249248,1.2214389230026372,0.37779842180031725], [-0.3777277713061831,0.866118273628028,-1.8487356572074127,1.1851350383431707,0.5215534466266145], [-0.6076926717533072,0.922880686625322,-1.7313060678714964,1.3632488420681186,0.3176699282425657], [-0.3802374789807806,0.8439055679755229,-1.6219613687183736,1.1809867291088374,0.4174630121979538], [-0.567466556581487,0.9292300369161763,-1.5542588287823018,1.3568593851021735,0.38458895772840795], [-0.6764387691659441,0.9816754055716941,-1.6652926928722982,1.3324217076474294,0.2844320904428872], [-0.584138124369403,0.9651833514289697,-1.7100579024456408,1.3025648636881102,0.3575320807936941], [-0.45548159931880944,1.0181220363355081,-1.6506653550843506,1.1304474598171783,0.4285968106743953], [-0.5370002084128622,0.927571736048293,-1.7228464440630433,1.2925613214036213,0.3169164091166503], [-0.47996613069291666,0.884366449915093,-1.7212355955272085,1.152597880658151,0.43962468554199663], [-0.35493034949860885,0.8984344817504613,-1.65218132155661,1.3068108975806394,0.354531357717713], [-0.43226118157508875,0.90725353631203,-1.7628881371748841,1.314870897760879,0.40214500076142384], [-0.4387259831220203,0.9621751931555681,-1.7268764217836796,1.1093705777029121,0.17867972251315242], [-0.44765902746822306,1.1186404105474657,-1.750942737318709,1.322673860044468,0.41609066577279374], [-0.40351581789921565,1.046340578817358,-1.7120920286690795,1.244893797129159,0.3833023298379642], [-0.3093624994668177,0.8356650811564297,-1.8642891852543533,1.217524168702485,0.2103617986137151], [-0.2990644638630094,0.9070655034040485,-1.6094469406493193,1.1592980925831848,0.43244854200109223], [-0.5596021456993305,0.9019708087164819,-1.5105356278212536,1.2598551179472526,0.3428536586628008], [-0.583393764986965,0.9748568871770861,-1.5302538126845806,1.301546791122464,0.21927558650218368], [-0.49772324825286274,0.9716299176892979,-1.6169165069063722,1.3531471880552017,0.25858373282508224], [-0.43106771684572104,1.141003137647425,-1.7578104686857339,1.1835903210552496,0.5254045668739125], [-0.5997306192424074,1.0040726704610177,-1.8227253165238155,1.2317004811683654,0.46876771655575034], [-0.38252412898165267,0.6654527000305275,-1.6468024292953505,1.246225339488523,0.3090133585357682], [-0.44821229943324725,1.0038256342470713,-1.6968551076177416,1.4529627079929077,0.3454589917432014], [-0.43535992691090786,1.0110816584854143,-1.7453606983094134,1.248210478232135,0.2891585839680707], [-0.3652348315036368,0.8958964249587966,-1.9606863598829045,1.3389257198065394,0.2618900608154475], [-0.49376528406043635,1.1421599774098976,-1.7157624036331927,1.391772429187865,0.2876879628208894], [-0.4515063147230089,0.9428403433366573,-1.76394465423704,1.203639940182186,0.5058415243772909], [-0.566229150629525,1.0128616873896932,-1.6880365432069786,1.2877554523210368,0.1629872361106314], [-0.47658344463796903,1.0862736976313303,-1.7934284966158631,1.2272771650263852,0.41680211738171946], [-0.3618808351944353,0.9417137306142976,-1.8600076455984111,1.2206352320840592,0.3461749973804146], [-0.6214757024643818,0.9952948941676851,-1.62097329834717,1.3958184351557885,0.46499512939060494], [-0.35996348236796494,0.9954644862470238,-1.7933270864489543,1.4005533772243737,0.5639356774206108], [-0.6006970016720174,0.9686450943470077,-1.7310975948083858,1.2398543489295077,0.4483279499267394], [-0.4592797597414643,0.9631529442377594,-1.6797024055855931,1.4289800788354718,0.28667160760155463], [-0.32700881076504584,0.8147754680205297,-1.763026614596352,1.3194735072019403,0.36816545181467747], [-0.4305345983795718,1.131285559002078,-1.7583813300937976,1.363545141597296,0.2601165464857354], [-0.448497878644767,1.0701920363594992,-1.6308801358271674,1.2674713459930484,0.49959128317905865], [-0.3686293381026533,1.025599917053419,-1.5433629968114364,1.2801178859900646,0.4174115399252367], [-0.574113693976958,0.9757976338077295,-1.6388778822719372,1.2759164798546365,0.5542142157229283], [-0.4357071400660946,0.8338147376622728,-1.8285888680646112,1.271233025944097,0.270178918164786], [-0.47158188300536635,1.0122183541160776,-1.6010520252577112,1.5447804629514494,0.3533722934153737], [-0.5307926544952695,0.9107879044274507,-1.4490342775396574,1.4031526476841614,0.3809780658985748], [-0.4953668915084064,0.9298227962550192,-1.5651812648587027,1.11568589286655,0.4774983034589586], [-0.7383552095915331,0.7626241115292196,-1.7008182872047266,1.2502877442685654,0.43135198632241045], [-0.539114438923603,1.0489102632469824,-1.6736818095705865,1.2768121659352345,0.2213641562753643], [-0.46002266280222004,1.0294295650038185,-1.6033579773810744,1.407522155806443,0.5223722922756932]) -2016-06-13 20:17:21 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:17:21 INFO JobScheduler:59 - Added jobs for time 10000 ms -2016-06-13 20:17:21 INFO JobGenerator:59 - Checkpointing graph for time 1000 ms -2016-06-13 20:17:21 INFO DStreamGraph:59 - Updating checkpoint data for time 1000 ms -2016-06-13 20:17:21 INFO DStreamGraph:59 - Updated checkpoint data for time 1000 ms -2016-06-13 20:17:21 INFO JobGenerator:59 - Checkpointing graph for time 2000 ms -2016-06-13 20:17:21 INFO DStreamGraph:59 - Updating checkpoint data for time 2000 ms -2016-06-13 20:17:21 INFO DStreamGraph:59 - Updated checkpoint data for time 2000 ms -2016-06-13 20:17:21 INFO CheckpointWriter:59 - Saving checkpoint for time 1000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-1000' -2016-06-13 20:17:21 INFO JobGenerator:59 - Checkpointing graph for time 3000 ms -2016-06-13 20:17:21 INFO DStreamGraph:59 - Updating checkpoint data for time 3000 ms -2016-06-13 20:17:21 INFO DStreamGraph:59 - Updated checkpoint data for time 3000 ms -2016-06-13 20:17:21 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=0, maxMem=766075207 -2016-06-13 20:17:22 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:17:22 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:17:22 INFO JobGenerator:59 - Checkpointing graph for time 4000 ms -2016-06-13 20:17:22 INFO DStreamGraph:59 - Updating checkpoint data for time 4000 ms -2016-06-13 20:17:22 INFO DStreamGraph:59 - Updated checkpoint data for time 4000 ms -2016-06-13 20:17:22 INFO MemoryStore:59 - ensureFreeSpace(2428) called with curMem=4744, maxMem=766075207 -2016-06-13 20:17:22 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:17:22 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51031 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:17:22 INFO JobGenerator:59 - Checkpointing graph for time 5000 ms -2016-06-13 20:17:22 INFO DStreamGraph:59 - Updating checkpoint data for time 5000 ms -2016-06-13 20:17:22 INFO DStreamGraph:59 - Updated checkpoint data for time 5000 ms -2016-06-13 20:17:22 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[7] at apply at Transformer.scala:22) -2016-06-13 20:17:22 INFO JobGenerator:59 - Checkpointing graph for time 6000 ms -2016-06-13 20:17:22 INFO DStreamGraph:59 - Updating checkpoint data for time 6000 ms -2016-06-13 20:17:22 INFO DStreamGraph:59 - Updated checkpoint data for time 6000 ms -2016-06-13 20:17:22 INFO JobGenerator:59 - Checkpointing graph for time 7000 ms -2016-06-13 20:17:22 INFO DStreamGraph:59 - Updating checkpoint data for time 7000 ms -2016-06-13 20:17:22 INFO DStreamGraph:59 - Updated checkpoint data for time 7000 ms -2016-06-13 20:17:22 INFO JobGenerator:59 - Checkpointing graph for time 8000 ms -2016-06-13 20:17:22 INFO DStreamGraph:59 - Updating checkpoint data for time 8000 ms -2016-06-13 20:17:22 INFO DStreamGraph:59 - Updated checkpoint data for time 8000 ms -2016-06-13 20:17:22 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 20:17:22 INFO JobGenerator:59 - Checkpointing graph for time 9000 ms -2016-06-13 20:17:22 INFO DStreamGraph:59 - Updating checkpoint data for time 9000 ms -2016-06-13 20:17:22 INFO DStreamGraph:59 - Updated checkpoint data for time 9000 ms -2016-06-13 20:17:22 INFO JobGenerator:59 - Checkpointing graph for time 10000 ms -2016-06-13 20:17:22 INFO DStreamGraph:59 - Updating checkpoint data for time 10000 ms -2016-06-13 20:17:22 INFO DStreamGraph:59 - Updated checkpoint data for time 10000 ms -2016-06-13 20:17:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:17:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:17:22 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 20:17:22 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 20:17:22 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:17:22 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:17:22 INFO CheckpointWriter:59 - Checkpoint for time 1000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-1000', took 27282 bytes and 297 ms -2016-06-13 20:17:22 INFO CheckpointWriter:59 - Saving checkpoint for time 2000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-2000' -2016-06-13 20:17:22 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 20:17:22 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 20:17:22 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:17:22 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1159 bytes result sent to driver -2016-06-13 20:17:22 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1159 bytes result sent to driver -2016-06-13 20:17:22 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:17:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 234 ms on localhost (1/2) -2016-06-13 20:17:22 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:17:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 297 ms on localhost (2/2) -2016-06-13 20:17:22 INFO DAGScheduler:59 - ShuffleMapStage 0 (apply at Transformer.scala:22) finished in 0.321 s -2016-06-13 20:17:22 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:17:22 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:17:22 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 20:17:22 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:17:22 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 20:17:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 20:17:22 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[22] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:17:22 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=7172, maxMem=766075207 -2016-06-13 20:17:22 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:17:22 INFO MemoryStore:59 - ensureFreeSpace(2657) called with curMem=12436, maxMem=766075207 -2016-06-13 20:17:22 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 20:17:22 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51031 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:17:22 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[22] at apply at Transformer.scala:22) -2016-06-13 20:17:22 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 20:17:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:22 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 20:17:22 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 20:17:22 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:17:22 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:17:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-06-13 20:17:22 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:17:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 17 ms -2016-06-13 20:17:22 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-06-13 20:17:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 66 ms on localhost (1/2) -2016-06-13 20:17:22 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1498 bytes result sent to driver -2016-06-13 20:17:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 79 ms on localhost (2/2) -2016-06-13 20:17:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 20:17:22 INFO CheckpointWriter:59 - Checkpoint for time 2000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-2000', took 27284 bytes and 266 ms -2016-06-13 20:17:22 INFO CheckpointWriter:59 - Saving checkpoint for time 3000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-3000' -2016-06-13 20:17:22 INFO DAGScheduler:59 - ResultStage 1 (apply at Transformer.scala:22) finished in 0.122 s -2016-06-13 20:17:22 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:17:22 INFO DAGScheduler:59 - Job 0 finished: apply at Transformer.scala:22, took 0.752218 s -2016-06-13 20:17:22 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 50.0 and centroid: [-0.4843392480139548,0.9734525763849463,-1.7046336323611109,1.2778093795456185,0.3691650550997036] -2016-06-13 20:17:22 INFO JobScheduler:59 - Finished job streaming job 1000 ms.0 from job set of time 1000 ms -2016-06-13 20:17:22 INFO JobScheduler:59 - Starting job streaming job 1000 ms.1 from job set of time 1000 ms -2016-06-13 20:17:22 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:17:22 INFO DAGScheduler:59 - Registering RDD 3 (apply at Transformer.scala:22) -2016-06-13 20:17:22 INFO DAGScheduler:59 - Got job 1 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:17:22 INFO DAGScheduler:59 - Final stage: ResultStage 3(apply at Transformer.scala:22) -2016-06-13 20:17:22 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 2) -2016-06-13 20:17:22 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 2) -2016-06-13 20:17:22 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (UnionRDD[3] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:17:22 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=15093, maxMem=766075207 -2016-06-13 20:17:22 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:17:22 INFO MemoryStore:59 - ensureFreeSpace(2000) called with curMem=18373, maxMem=766075207 -2016-06-13 20:17:22 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2000.0 B, free 730.6 MB) -2016-06-13 20:17:22 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:17:22 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51031 (size: 2000.0 B, free: 730.6 MB) -2016-06-13 20:17:22 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:22 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 2 (UnionRDD[3] at apply at Transformer.scala:22) -2016-06-13 20:17:22 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-06-13 20:17:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:17:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:17:22 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 20:17:22 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 20:17:22 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1159 bytes result sent to driver -2016-06-13 20:17:22 INFO CheckpointWriter:59 - Checkpoint for time 3000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-3000', took 27284 bytes and 165 ms -2016-06-13 20:17:22 INFO CheckpointWriter:59 - Saving checkpoint for time 4000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-4000' -2016-06-13 20:17:22 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:17:22 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1159 bytes result sent to driver -2016-06-13 20:17:22 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:17:22 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-06-13 20:17:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 51 ms on localhost (1/3) -2016-06-13 20:17:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 48 ms on localhost (2/3) -2016-06-13 20:17:22 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 6). 1159 bytes result sent to driver -2016-06-13 20:17:22 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 6) in 27 ms on localhost (3/3) -2016-06-13 20:17:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 20:17:22 INFO DAGScheduler:59 - ShuffleMapStage 2 (apply at Transformer.scala:22) finished in 0.080 s -2016-06-13 20:17:22 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:17:22 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:17:22 INFO DAGScheduler:59 - waiting: Set(ResultStage 3) -2016-06-13 20:17:22 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:17:22 INFO DAGScheduler:59 - Missing parents for ResultStage 3: List() -2016-06-13 20:17:22 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[62] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:17:22 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=20373, maxMem=766075207 -2016-06-13 20:17:22 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:17:22 INFO MemoryStore:59 - ensureFreeSpace(1800) called with curMem=23613, maxMem=766075207 -2016-06-13 20:17:22 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1800.0 B, free 730.6 MB) -2016-06-13 20:17:22 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:51031 (size: 1800.0 B, free: 730.6 MB) -2016-06-13 20:17:22 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[62] at apply at Transformer.scala:22) -2016-06-13 20:17:22 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 20:17:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:22 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 7) -2016-06-13 20:17:22 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 8) -2016-06-13 20:17:22 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:17:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:22 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:17:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:22 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 7). 1180 bytes result sent to driver -2016-06-13 20:17:22 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:17:22 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 8). 1172 bytes result sent to driver -2016-06-13 20:17:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 7) in 24 ms on localhost (1/2) -2016-06-13 20:17:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 8) in 28 ms on localhost (2/2) -2016-06-13 20:17:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 20:17:22 INFO DAGScheduler:59 - ResultStage 3 (apply at Transformer.scala:22) finished in 0.030 s -2016-06-13 20:17:22 INFO DAGScheduler:59 - Job 1 finished: apply at Transformer.scala:22, took 0.195653 s -2016-06-13 20:17:22 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:17:22 INFO CheckpointWriter:59 - Checkpoint for time 4000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-4000', took 27284 bytes and 156 ms -2016-06-13 20:17:22 INFO CheckpointWriter:59 - Saving checkpoint for time 5000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-5000' -2016-06-13 20:17:22 INFO JobScheduler:59 - Finished job streaming job 1000 ms.1 from job set of time 1000 ms -2016-06-13 20:17:22 INFO JobScheduler:59 - Total delay: 1465820241.876 s for time 1000 ms (execution: 1.102 s) -2016-06-13 20:17:22 INFO DAGScheduler:59 - Registering RDD 63 (apply at Transformer.scala:22) -2016-06-13 20:17:22 INFO DAGScheduler:59 - Got job 2 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:17:22 INFO DAGScheduler:59 - Final stage: ResultStage 5(apply at Transformer.scala:22) -2016-06-13 20:17:22 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-06-13 20:17:22 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-06-13 20:17:22 INFO JobScheduler:59 - Starting job streaming job 2000 ms.0 from job set of time 2000 ms -2016-06-13 20:17:22 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[63] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:17:22 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=25413, maxMem=766075207 -2016-06-13 20:17:22 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:17:22 INFO JobGenerator:59 - Checkpointing graph for time 1000 ms -2016-06-13 20:17:22 INFO DStreamGraph:59 - Updating checkpoint data for time 1000 ms -2016-06-13 20:17:22 INFO DStreamGraph:59 - Updated checkpoint data for time 1000 ms -2016-06-13 20:17:22 INFO MemoryStore:59 - ensureFreeSpace(2471) called with curMem=30157, maxMem=766075207 -2016-06-13 20:17:22 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:17:22 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:51031 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:17:22 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[63] at apply at Transformer.scala:22) -2016-06-13 20:17:22 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 20:17:22 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:17:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:17:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 10, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:17:22 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 10) -2016-06-13 20:17:22 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 9) -2016-06-13 20:17:22 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 9). 1159 bytes result sent to driver -2016-06-13 20:17:22 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 10). 1159 bytes result sent to driver -2016-06-13 20:17:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 9) in 38 ms on localhost (1/2) -2016-06-13 20:17:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 10) in 39 ms on localhost (2/2) -2016-06-13 20:17:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 20:17:22 INFO DAGScheduler:59 - ShuffleMapStage 4 (apply at Transformer.scala:22) finished in 0.041 s -2016-06-13 20:17:22 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:17:22 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:17:22 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-06-13 20:17:22 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:17:22 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-06-13 20:17:22 INFO DAGScheduler:59 - Submitting ResultStage 5 (ShuffledRDD[64] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:17:22 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=32628, maxMem=766075207 -2016-06-13 20:17:22 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:17:22 INFO MemoryStore:59 - ensureFreeSpace(2692) called with curMem=37892, maxMem=766075207 -2016-06-13 20:17:22 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:17:22 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:17:22 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:51031 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:17:22 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (ShuffledRDD[64] at apply at Transformer.scala:22) -2016-06-13 20:17:22 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 20:17:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 12, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:22 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 11) -2016-06-13 20:17:22 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 12) -2016-06-13 20:17:22 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:17:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:22 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:17:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:22 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 12). 1161 bytes result sent to driver -2016-06-13 20:17:22 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 11). 1498 bytes result sent to driver -2016-06-13 20:17:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 12) in 21 ms on localhost (1/2) -2016-06-13 20:17:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 11) in 30 ms on localhost (2/2) -2016-06-13 20:17:22 INFO DAGScheduler:59 - ResultStage 5 (apply at Transformer.scala:22) finished in 0.030 s -2016-06-13 20:17:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 20:17:23 INFO DAGScheduler:59 - Job 2 finished: apply at Transformer.scala:22, took 0.134896 s -2016-06-13 20:17:23 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 100.0 and centroid: [-0.4738959687352186,0.969729236815007,-1.6823805644329948,1.2679893451777908,0.37589591355279117] -2016-06-13 20:17:23 INFO JobScheduler:59 - Finished job streaming job 2000 ms.0 from job set of time 2000 ms -2016-06-13 20:17:23 INFO JobScheduler:59 - Starting job streaming job 2000 ms.1 from job set of time 2000 ms -2016-06-13 20:17:23 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:17:23 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Registering RDD 10 (apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Got job 3 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:17:23 INFO DAGScheduler:59 - Final stage: ResultStage 7(apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (UnionRDD[10] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=40584, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=43864, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:17:23 INFO CheckpointWriter:59 - Checkpoint for time 5000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-5000', took 27283 bytes and 156 ms -2016-06-13 20:17:23 INFO CheckpointWriter:59 - Saving checkpoint for time 6000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-6000' -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:51031 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:17:23 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 6 (UnionRDD[10] at apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 3 tasks -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 14, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:17:23 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 14) -2016-06-13 20:17:23 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 13) -2016-06-13 20:17:23 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 13). 1159 bytes result sent to driver -2016-06-13 20:17:23 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 14). 1159 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 15, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:17:23 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 15) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 13) in 37 ms on localhost (1/3) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 14) in 38 ms on localhost (2/3) -2016-06-13 20:17:23 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 15). 1159 bytes result sent to driver -2016-06-13 20:17:23 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 15) in 23 ms on localhost (3/3) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 20:17:23 INFO DAGScheduler:59 - ShuffleMapStage 6 (apply at Transformer.scala:22) finished in 0.048 s -2016-06-13 20:17:23 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:17:23 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:17:23 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-06-13 20:17:23 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:17:23 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[65] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=45873, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=49113, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:51031 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:17:23 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[65] at apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:23 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 16) -2016-06-13 20:17:23 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 17) -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:23 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 16). 1180 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 16) in 19 ms on localhost (1/2) -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:23 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 17). 1172 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 17) in 27 ms on localhost (2/2) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 20:17:23 INFO DAGScheduler:59 - ResultStage 7 (apply at Transformer.scala:22) finished in 0.030 s -2016-06-13 20:17:23 INFO DAGScheduler:59 - Job 3 finished: apply at Transformer.scala:22, took 0.131706 s -2016-06-13 20:17:23 INFO JobScheduler:59 - Finished job streaming job 2000 ms.1 from job set of time 2000 ms -2016-06-13 20:17:23 INFO JobScheduler:59 - Total delay: 1465820241.143 s for time 2000 ms (execution: 0.264 s) -2016-06-13 20:17:23 INFO JobScheduler:59 - Starting job streaming job 3000 ms.0 from job set of time 3000 ms -2016-06-13 20:17:23 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:17:23 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Registering RDD 66 (apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Got job 4 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:17:23 INFO DAGScheduler:59 - Final stage: ResultStage 9(apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 8) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 8) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting ShuffleMapStage 8 (MapPartitionsRDD[66] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=50916, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(2475) called with curMem=55660, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO ParallelCollectionRDD:59 - Removing RDD 0 from persistence list -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:51031 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:17:23 INFO CheckpointWriter:59 - Checkpoint for time 6000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-6000', took 27280 bytes and 159 ms -2016-06-13 20:17:23 INFO CheckpointWriter:59 - Saving checkpoint for time 7000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-7000' -2016-06-13 20:17:23 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 8 (MapPartitionsRDD[66] at apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 20:17:23 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 18, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 19, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:17:23 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 18) -2016-06-13 20:17:23 INFO MapPartitionsRDD:59 - Removing RDD 5 from persistence list -2016-06-13 20:17:23 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 19) -2016-06-13 20:17:23 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 18). 1159 bytes result sent to driver -2016-06-13 20:17:23 INFO BlockManager:59 - Removing RDD 5 -2016-06-13 20:17:23 INFO ShuffledRDD:59 - Removing RDD 4 from persistence list -2016-06-13 20:17:23 INFO UnionRDD:59 - Removing RDD 3 from persistence list -2016-06-13 20:17:23 INFO BlockManager:59 - Removing RDD 0 -2016-06-13 20:17:23 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 19). 1159 bytes result sent to driver -2016-06-13 20:17:23 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:17:23 INFO MapPartitionsRDD:59 - Removing RDD 1 from persistence list -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 18) in 52 ms on localhost (1/2) -2016-06-13 20:17:23 INFO JobGenerator:59 - Checkpointing graph for time 2000 ms -2016-06-13 20:17:23 INFO DStreamGraph:59 - Updating checkpoint data for time 2000 ms -2016-06-13 20:17:23 INFO BlockManager:59 - Removing RDD 4 -2016-06-13 20:17:23 INFO BlockManager:59 - Removing RDD 3 -2016-06-13 20:17:23 INFO DStreamGraph:59 - Updated checkpoint data for time 2000 ms -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 19) in 43 ms on localhost (2/2) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 20:17:23 INFO BlockManager:59 - Removing RDD 1 -2016-06-13 20:17:23 INFO DAGScheduler:59 - ShuffleMapStage 8 (apply at Transformer.scala:22) finished in 0.056 s -2016-06-13 20:17:23 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:17:23 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:17:23 INFO DAGScheduler:59 - waiting: Set(ResultStage 9) -2016-06-13 20:17:23 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:17:23 INFO DAGScheduler:59 - Missing parents for ResultStage 9: List() -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting ResultStage 9 (ShuffledRDD[67] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=58135, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=63399, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:51031 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:17:23 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (ShuffledRDD[67] at apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 21, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:23 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 20) -2016-06-13 20:17:23 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 21) -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:51031 in memory (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:17:23 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:51031 in memory (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:23 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 21). 1161 bytes result sent to driver -2016-06-13 20:17:23 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 20). 1498 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 21) in 19 ms on localhost (1/2) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 20) in 20 ms on localhost (2/2) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 20:17:23 INFO DAGScheduler:59 - ResultStage 9 (apply at Transformer.scala:22) finished in 0.021 s -2016-06-13 20:17:23 INFO DAGScheduler:59 - Job 4 finished: apply at Transformer.scala:22, took 0.167129 s -2016-06-13 20:17:23 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 150.0 and centroid: [-0.4741102089605591,0.9622750167428704,-1.680286076986711,1.2654376587211662,0.37713329463498424] -2016-06-13 20:17:23 INFO JobScheduler:59 - Finished job streaming job 3000 ms.0 from job set of time 3000 ms -2016-06-13 20:17:23 INFO JobScheduler:59 - Starting job streaming job 3000 ms.1 from job set of time 3000 ms -2016-06-13 20:17:23 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:51031 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:17:23 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:17:23 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Registering RDD 16 (apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Got job 5 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:17:23 INFO DAGScheduler:59 - Final stage: ResultStage 11(apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 10) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 10) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting ShuffleMapStage 10 (UnionRDD[16] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:51031 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:17:23 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=40592, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(2004) called with curMem=43872, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2004.0 B, free 730.5 MB) -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:51031 (size: 2004.0 B, free: 730.6 MB) -2016-06-13 20:17:23 INFO CheckpointWriter:59 - Checkpoint for time 7000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-7000', took 27282 bytes and 168 ms -2016-06-13 20:17:23 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:23 INFO CheckpointWriter:59 - Saving checkpoint for time 8000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-8000' -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 10 (UnionRDD[16] at apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 3 tasks -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 22, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 23, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:17:23 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 22) -2016-06-13 20:17:23 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-06-13 20:17:23 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 23) -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:51031 in memory (size: 1800.0 B, free: 730.6 MB) -2016-06-13 20:17:23 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 22). 1159 bytes result sent to driver -2016-06-13 20:17:23 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 23). 1159 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 10.0 (TID 24, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:17:23 INFO Executor:59 - Running task 2.0 in stage 10.0 (TID 24) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 23) in 24 ms on localhost (1/3) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 22) in 26 ms on localhost (2/3) -2016-06-13 20:17:23 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 20:17:23 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:51031 in memory (size: 2000.0 B, free: 730.6 MB) -2016-06-13 20:17:23 INFO Executor:59 - Finished task 2.0 in stage 10.0 (TID 24). 1159 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 10.0 (TID 24) in 21 ms on localhost (3/3) -2016-06-13 20:17:23 INFO DAGScheduler:59 - ShuffleMapStage 10 (apply at Transformer.scala:22) finished in 0.044 s -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 20:17:23 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:17:23 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:17:23 INFO DAGScheduler:59 - waiting: Set(ResultStage 11) -2016-06-13 20:17:23 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:17:23 INFO DAGScheduler:59 - Missing parents for ResultStage 11: List() -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[68] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=35556, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 20:17:23 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=36139, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.6 MB) -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:51031 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:17:23 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[68] at apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:51031 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 25, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 26, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:23 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 26) -2016-06-13 20:17:23 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 25) -2016-06-13 20:17:23 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:51031 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:17:23 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 26). 1172 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 26) in 18 ms on localhost (1/2) -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:17:23 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 20:17:23 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-06-13 20:17:23 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:17:23 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 25). 1180 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 25) in 31 ms on localhost (2/2) -2016-06-13 20:17:23 INFO DAGScheduler:59 - ResultStage 11 (apply at Transformer.scala:22) finished in 0.033 s -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 20:17:23 INFO DAGScheduler:59 - Job 5 finished: apply at Transformer.scala:22, took 0.115092 s -2016-06-13 20:17:23 INFO JobScheduler:59 - Finished job streaming job 3000 ms.1 from job set of time 3000 ms -2016-06-13 20:17:23 INFO JobScheduler:59 - Total delay: 1465820240.457 s for time 3000 ms (execution: 0.313 s) -2016-06-13 20:17:23 INFO JobScheduler:59 - Starting job streaming job 4000 ms.0 from job set of time 4000 ms -2016-06-13 20:17:23 INFO ParallelCollectionRDD:59 - Removing RDD 6 from persistence list -2016-06-13 20:17:23 INFO BlockManager:59 - Removing RDD 6 -2016-06-13 20:17:23 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-06-13 20:17:23 INFO ShuffledRDD:59 - Removing RDD 11 from persistence list -2016-06-13 20:17:23 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 20:17:23 INFO BlockManager:59 - Removing RDD 11 -2016-06-13 20:17:23 INFO UnionRDD:59 - Removing RDD 10 from persistence list -2016-06-13 20:17:23 INFO BlockManager:59 - Removing RDD 10 -2016-06-13 20:17:23 INFO MapPartitionsRDD:59 - Removing RDD 8 from persistence list -2016-06-13 20:17:23 INFO BlockManager:59 - Removing RDD 8 -2016-06-13 20:17:23 INFO JobGenerator:59 - Checkpointing graph for time 3000 ms -2016-06-13 20:17:23 INFO DStreamGraph:59 - Updating checkpoint data for time 3000 ms -2016-06-13 20:17:23 INFO DStreamGraph:59 - Updated checkpoint data for time 3000 ms -2016-06-13 20:17:23 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Registering RDD 69 (apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Got job 6 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:17:23 INFO DAGScheduler:59 - Final stage: ResultStage 13(apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (MapPartitionsRDD[69] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=25506, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(2484) called with curMem=30250, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:51031 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:17:23 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (MapPartitionsRDD[69] at apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 27, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 28, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:17:23 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 28) -2016-06-13 20:17:23 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 27) -2016-06-13 20:17:23 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:17:23 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 27). 1159 bytes result sent to driver -2016-06-13 20:17:23 INFO CheckpointWriter:59 - Checkpoint for time 8000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-8000', took 27284 bytes and 157 ms -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 27) in 21 ms on localhost (1/2) -2016-06-13 20:17:23 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 28). 1159 bytes result sent to driver -2016-06-13 20:17:23 INFO CheckpointWriter:59 - Saving checkpoint for time 9000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-9000' -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 28) in 22 ms on localhost (2/2) -2016-06-13 20:17:23 INFO DAGScheduler:59 - ShuffleMapStage 12 (apply at Transformer.scala:22) finished in 0.025 s -2016-06-13 20:17:23 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:17:23 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:17:23 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-06-13 20:17:23 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 20:17:23 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting ResultStage 13 (ShuffledRDD[70] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=32734, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(2698) called with curMem=37998, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:51031 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:17:23 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (ShuffledRDD[70] at apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 29, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 30, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:23 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 30) -2016-06-13 20:17:23 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 29) -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:23 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 30). 1161 bytes result sent to driver -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:17:23 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 29). 1498 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 30) in 17 ms on localhost (1/2) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 29) in 19 ms on localhost (2/2) -2016-06-13 20:17:23 INFO DAGScheduler:59 - ResultStage 13 (apply at Transformer.scala:22) finished in 0.020 s -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 20:17:23 INFO DAGScheduler:59 - Job 6 finished: apply at Transformer.scala:22, took 0.075574 s -2016-06-13 20:17:23 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 200.0 and centroid: [-0.4768735011530239,0.9606386018112957,-1.673148729841545,1.2634974127814163,0.38189605546947214] -2016-06-13 20:17:23 INFO JobScheduler:59 - Finished job streaming job 4000 ms.0 from job set of time 4000 ms -2016-06-13 20:17:23 INFO JobScheduler:59 - Starting job streaming job 4000 ms.1 from job set of time 4000 ms -2016-06-13 20:17:23 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Registering RDD 23 (apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Got job 7 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:17:23 INFO DAGScheduler:59 - Final stage: ResultStage 15(apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 14) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 14) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting ShuffleMapStage 14 (UnionRDD[23] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=40696, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=43976, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:51031 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:17:23 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 14 (UnionRDD[23] at apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 3 tasks -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 31, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 32, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:17:23 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 31) -2016-06-13 20:17:23 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 32) -2016-06-13 20:17:23 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 32). 1159 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 33, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 32) in 19 ms on localhost (1/3) -2016-06-13 20:17:23 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 33) -2016-06-13 20:17:23 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 31). 1159 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 31) in 30 ms on localhost (2/3) -2016-06-13 20:17:23 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 33). 1159 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 33) in 17 ms on localhost (3/3) -2016-06-13 20:17:23 INFO DAGScheduler:59 - ShuffleMapStage 14 (apply at Transformer.scala:22) finished in 0.037 s -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 20:17:23 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:17:23 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:17:23 INFO DAGScheduler:59 - waiting: Set(ResultStage 15) -2016-06-13 20:17:23 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:17:23 INFO DAGScheduler:59 - Missing parents for ResultStage 15: List() -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[71] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=45985, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=49225, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.5 MB) -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:51031 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:17:23 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[71] at apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 34, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 35, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:23 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 34) -2016-06-13 20:17:23 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 35) -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:23 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 35). 1172 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 35) in 11 ms on localhost (1/2) -2016-06-13 20:17:23 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 34). 1180 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 34) in 16 ms on localhost (2/2) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 20:17:23 INFO DAGScheduler:59 - ResultStage 15 (apply at Transformer.scala:22) finished in 0.016 s -2016-06-13 20:17:23 INFO DAGScheduler:59 - Job 7 finished: apply at Transformer.scala:22, took 0.083488 s -2016-06-13 20:17:23 INFO JobScheduler:59 - Finished job streaming job 4000 ms.1 from job set of time 4000 ms -2016-06-13 20:17:23 INFO JobScheduler:59 - Total delay: 1465820239.643 s for time 4000 ms (execution: 0.186 s) -2016-06-13 20:17:23 INFO JobScheduler:59 - Starting job streaming job 5000 ms.0 from job set of time 5000 ms -2016-06-13 20:17:23 INFO ParallelCollectionRDD:59 - Removing RDD 13 from persistence list -2016-06-13 20:17:23 INFO BlockManager:59 - Removing RDD 13 -2016-06-13 20:17:23 INFO MapPartitionsRDD:59 - Removing RDD 18 from persistence list -2016-06-13 20:17:23 INFO BlockManager:59 - Removing RDD 18 -2016-06-13 20:17:23 INFO ShuffledRDD:59 - Removing RDD 17 from persistence list -2016-06-13 20:17:23 INFO BlockManager:59 - Removing RDD 17 -2016-06-13 20:17:23 INFO UnionRDD:59 - Removing RDD 16 from persistence list -2016-06-13 20:17:23 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:17:23 INFO MapPartitionsRDD:59 - Removing RDD 14 from persistence list -2016-06-13 20:17:23 INFO DAGScheduler:59 - Registering RDD 72 (apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Got job 8 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:17:23 INFO DAGScheduler:59 - Final stage: ResultStage 17(apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 16) -2016-06-13 20:17:23 INFO BlockManager:59 - Removing RDD 16 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 16) -2016-06-13 20:17:23 INFO JobGenerator:59 - Checkpointing graph for time 4000 ms -2016-06-13 20:17:23 INFO DStreamGraph:59 - Updating checkpoint data for time 4000 ms -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting ShuffleMapStage 16 (MapPartitionsRDD[72] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:17:23 INFO BlockManager:59 - Removing RDD 14 -2016-06-13 20:17:23 INFO DStreamGraph:59 - Updated checkpoint data for time 4000 ms -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=51027, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(2476) called with curMem=55771, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:51031 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:17:23 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:17:23 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 16 (MapPartitionsRDD[72] at apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 36, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:17:23 INFO CheckpointWriter:59 - Checkpoint for time 9000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-9000', took 27284 bytes and 166 ms -2016-06-13 20:17:23 INFO CheckpointWriter:59 - Saving checkpoint for time 10000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-10000' -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 37, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:17:23 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 37) -2016-06-13 20:17:23 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 36) -2016-06-13 20:17:23 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 36). 1159 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 36) in 55 ms on localhost (1/2) -2016-06-13 20:17:23 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 37). 1159 bytes result sent to driver -2016-06-13 20:17:23 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 37) in 60 ms on localhost (2/2) -2016-06-13 20:17:23 INFO DAGScheduler:59 - ShuffleMapStage 16 (apply at Transformer.scala:22) finished in 0.062 s -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 20:17:23 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:17:23 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:17:23 INFO DAGScheduler:59 - waiting: Set(ResultStage 17) -2016-06-13 20:17:23 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:17:23 INFO DAGScheduler:59 - Missing parents for ResultStage 17: List() -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting ResultStage 17 (ShuffledRDD[73] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=58247, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=63511, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:51031 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:17:23 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (ShuffledRDD[73] at apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 38, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 39, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:23 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 39) -2016-06-13 20:17:23 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 38) -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:17:23 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 39). 1161 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 39) in 12 ms on localhost (1/2) -2016-06-13 20:17:23 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 38). 1498 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 38) in 19 ms on localhost (2/2) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 20:17:23 INFO DAGScheduler:59 - ResultStage 17 (apply at Transformer.scala:22) finished in 0.021 s -2016-06-13 20:17:23 INFO DAGScheduler:59 - Job 8 finished: apply at Transformer.scala:22, took 0.114517 s -2016-06-13 20:17:23 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 250.0 and centroid: [-0.4770613340912709,0.9657053185238176,-1.6732261982319274,1.26651642628913,0.38386658573056776] -2016-06-13 20:17:23 INFO JobScheduler:59 - Finished job streaming job 5000 ms.0 from job set of time 5000 ms -2016-06-13 20:17:23 INFO JobScheduler:59 - Starting job streaming job 5000 ms.1 from job set of time 5000 ms -2016-06-13 20:17:23 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Registering RDD 29 (apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Got job 9 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:17:23 INFO DAGScheduler:59 - Final stage: ResultStage 19(apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 18) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 18) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting ShuffleMapStage 18 (UnionRDD[29] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=66205, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(2010) called with curMem=69485, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2010.0 B, free 730.5 MB) -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:51031 (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:17:23 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 18 (UnionRDD[29] at apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 3 tasks -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 40, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 41, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:17:23 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 41) -2016-06-13 20:17:23 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 40) -2016-06-13 20:17:23 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 41). 1159 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 42, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 41) in 20 ms on localhost (1/3) -2016-06-13 20:17:23 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 42) -2016-06-13 20:17:23 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 40). 1159 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 40) in 34 ms on localhost (2/3) -2016-06-13 20:17:23 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 42). 1159 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 42) in 17 ms on localhost (3/3) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 20:17:23 INFO DAGScheduler:59 - ShuffleMapStage 18 (apply at Transformer.scala:22) finished in 0.038 s -2016-06-13 20:17:23 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:17:23 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:17:23 INFO DAGScheduler:59 - waiting: Set(ResultStage 19) -2016-06-13 20:17:23 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:17:23 INFO DAGScheduler:59 - Missing parents for ResultStage 19: List() -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[74] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=71495, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(1804) called with curMem=74735, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1804.0 B, free 730.5 MB) -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:51031 (size: 1804.0 B, free: 730.6 MB) -2016-06-13 20:17:23 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[74] at apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 43, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 44, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:23 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 43) -2016-06-13 20:17:23 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 44) -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:17:23 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 44). 1172 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 44) in 9 ms on localhost (1/2) -2016-06-13 20:17:23 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 43). 1180 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 43) in 14 ms on localhost (2/2) -2016-06-13 20:17:23 INFO DAGScheduler:59 - ResultStage 19 (apply at Transformer.scala:22) finished in 0.015 s -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 20:17:23 INFO DAGScheduler:59 - Job 9 finished: apply at Transformer.scala:22, took 0.084609 s -2016-06-13 20:17:23 INFO JobScheduler:59 - Finished job streaming job 5000 ms.1 from job set of time 5000 ms -2016-06-13 20:17:23 INFO JobScheduler:59 - Total delay: 1465820238.871 s for time 5000 ms (execution: 0.227 s) -2016-06-13 20:17:23 INFO JobScheduler:59 - Starting job streaming job 6000 ms.0 from job set of time 6000 ms -2016-06-13 20:17:23 INFO ParallelCollectionRDD:59 - Removing RDD 19 from persistence list -2016-06-13 20:17:23 INFO BlockManager:59 - Removing RDD 19 -2016-06-13 20:17:23 INFO MapPartitionsRDD:59 - Removing RDD 25 from persistence list -2016-06-13 20:17:23 INFO ShuffledRDD:59 - Removing RDD 24 from persistence list -2016-06-13 20:17:23 INFO BlockManager:59 - Removing RDD 25 -2016-06-13 20:17:23 INFO UnionRDD:59 - Removing RDD 23 from persistence list -2016-06-13 20:17:23 INFO BlockManager:59 - Removing RDD 24 -2016-06-13 20:17:23 INFO BlockManager:59 - Removing RDD 23 -2016-06-13 20:17:23 INFO MapPartitionsRDD:59 - Removing RDD 20 from persistence list -2016-06-13 20:17:23 INFO BlockManager:59 - Removing RDD 20 -2016-06-13 20:17:23 INFO JobGenerator:59 - Checkpointing graph for time 5000 ms -2016-06-13 20:17:23 INFO DStreamGraph:59 - Updating checkpoint data for time 5000 ms -2016-06-13 20:17:23 INFO DStreamGraph:59 - Updated checkpoint data for time 5000 ms -2016-06-13 20:17:23 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Registering RDD 75 (apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Got job 10 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:17:23 INFO DAGScheduler:59 - Final stage: ResultStage 21(apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 20) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 20) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting ShuffleMapStage 20 (MapPartitionsRDD[75] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:17:23 INFO CheckpointWriter:59 - Checkpoint for time 10000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-10000', took 27295 bytes and 211 ms -2016-06-13 20:17:23 INFO CheckpointWriter:59 - Saving checkpoint for time 1000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-1000' -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=76539, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=81283, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:51031 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:17:23 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:23 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 20 (MapPartitionsRDD[75] at apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 45, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 46, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:17:23 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 45) -2016-06-13 20:17:23 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 46) -2016-06-13 20:17:23 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 45). 1159 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 45) in 17 ms on localhost (1/2) -2016-06-13 20:17:23 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 46). 1159 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 46) in 22 ms on localhost (2/2) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 20:17:23 INFO DAGScheduler:59 - ShuffleMapStage 20 (apply at Transformer.scala:22) finished in 0.024 s -2016-06-13 20:17:23 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:17:23 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:17:23 INFO DAGScheduler:59 - waiting: Set(ResultStage 21) -2016-06-13 20:17:23 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:17:23 INFO DAGScheduler:59 - Missing parents for ResultStage 21: List() -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting ResultStage 21 (ShuffledRDD[76] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=83765, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=89029, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:51031 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:17:23 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (ShuffledRDD[76] at apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:23 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 47) -2016-06-13 20:17:23 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 48) -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:17:23 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:17:23 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 47). 1498 bytes result sent to driver -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:17:23 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:17:23 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 48). 1161 bytes result sent to driver -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 47) in 23 ms on localhost (1/2) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 48) in 23 ms on localhost (2/2) -2016-06-13 20:17:23 INFO DAGScheduler:59 - ResultStage 21 (apply at Transformer.scala:22) finished in 0.025 s -2016-06-13 20:17:23 INFO DAGScheduler:59 - Job 10 finished: apply at Transformer.scala:22, took 0.074990 s -2016-06-13 20:17:23 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 300.0 and centroid: [-0.4778380287460108,0.9658292271340416,-1.6712646630450996,1.273358698745343,0.37938624993358266] -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 20:17:23 INFO JobScheduler:59 - Finished job streaming job 6000 ms.0 from job set of time 6000 ms -2016-06-13 20:17:23 INFO JobScheduler:59 - Starting job streaming job 6000 ms.1 from job set of time 6000 ms -2016-06-13 20:17:23 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Registering RDD 35 (apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Got job 11 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:17:23 INFO DAGScheduler:59 - Final stage: ResultStage 23(apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 22) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 22) -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting ShuffleMapStage 22 (UnionRDD[35] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=91725, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:17:23 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=95005, maxMem=766075207 -2016-06-13 20:17:23 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.5 MB) -2016-06-13 20:17:23 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:51031 (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:17:23 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:23 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 22 (UnionRDD[35] at apply at Transformer.scala:22) -2016-06-13 20:17:23 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 3 tasks -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 49, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:17:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 50, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:17:23 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 49) -2016-06-13 20:17:23 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:17:24 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 49). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 51, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:17:24 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 51) -2016-06-13 20:17:24 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 50) -2016-06-13 20:17:24 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 51). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 49) in 37 ms on localhost (1/3) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 51) in 19 ms on localhost (2/3) -2016-06-13 20:17:24 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 50). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 50) in 62 ms on localhost (3/3) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 20:17:24 INFO DAGScheduler:59 - ShuffleMapStage 22 (apply at Transformer.scala:22) finished in 0.066 s -2016-06-13 20:17:24 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:17:24 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:17:24 INFO DAGScheduler:59 - waiting: Set(ResultStage 23) -2016-06-13 20:17:24 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:17:24 INFO DAGScheduler:59 - Missing parents for ResultStage 23: List() -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[77] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=97012, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(1804) called with curMem=100252, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 1804.0 B, free 730.5 MB) -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:51031 (size: 1804.0 B, free: 730.6 MB) -2016-06-13 20:17:24 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[77] at apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 52, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:24 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-1000.bk -2016-06-13 20:17:24 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 53) -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:24 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 53). 1172 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 53) in 12 ms on localhost (1/2) -2016-06-13 20:17:24 INFO CheckpointWriter:59 - Checkpoint for time 1000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-1000', took 27472 bytes and 195 ms -2016-06-13 20:17:24 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 52) -2016-06-13 20:17:24 INFO CheckpointWriter:59 - Saving checkpoint for time 2000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-2000' -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:24 INFO DStreamGraph:59 - Clearing checkpoint data for time 1000 ms -2016-06-13 20:17:24 INFO DStreamGraph:59 - Cleared checkpoint data for time 1000 ms -2016-06-13 20:17:24 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:17:24 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 52). 1180 bytes result sent to driver -2016-06-13 20:17:24 INFO DAGScheduler:59 - ResultStage 23 (apply at Transformer.scala:22) finished in 0.045 s -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 52) in 44 ms on localhost (2/2) -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Job 11 finished: apply at Transformer.scala:22, took 0.149700 s -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 20:17:24 INFO JobScheduler:59 - Finished job streaming job 6000 ms.1 from job set of time 6000 ms -2016-06-13 20:17:24 INFO JobScheduler:59 - Total delay: 1465820238.128 s for time 6000 ms (execution: 0.256 s) -2016-06-13 20:17:24 INFO JobScheduler:59 - Starting job streaming job 7000 ms.0 from job set of time 7000 ms -2016-06-13 20:17:24 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/receivedBlockMetadata older than 0: -2016-06-13 20:17:24 INFO InputInfoTracker:59 - remove old batch metadata: -2016-06-13 20:17:24 INFO ParallelCollectionRDD:59 - Removing RDD 26 from persistence list -2016-06-13 20:17:24 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:17:24 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/receivedBlockMetadata older than 0 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Registering RDD 78 (apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Got job 12 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:17:24 INFO DAGScheduler:59 - Final stage: ResultStage 25(apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 24) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 24) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting ShuffleMapStage 24 (MapPartitionsRDD[78] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=102056, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 26 -2016-06-13 20:17:24 INFO MapPartitionsRDD:59 - Removing RDD 31 from persistence list -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 31 -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:17:24 INFO ShuffledRDD:59 - Removing RDD 30 from persistence list -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(2480) called with curMem=106800, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 30 -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:51031 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 20:17:24 INFO UnionRDD:59 - Removing RDD 29 from persistence list -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:51031 in memory (size: 1804.0 B, free: 730.6 MB) -2016-06-13 20:17:24 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 24 (MapPartitionsRDD[78] at apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-06-13 20:17:24 INFO MapPartitionsRDD:59 - Removing RDD 27 from persistence list -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 29 -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 27 -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 54, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:17:24 INFO JobGenerator:59 - Checkpointing graph for time 6000 ms -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 55, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:17:24 INFO DStreamGraph:59 - Updating checkpoint data for time 6000 ms -2016-06-13 20:17:24 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 54) -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:51031 in memory (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:17:24 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 55) -2016-06-13 20:17:24 INFO DStreamGraph:59 - Updated checkpoint data for time 6000 ms -2016-06-13 20:17:24 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 54). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 20:17:24 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 55). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:51031 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 54) in 30 ms on localhost (1/2) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 55) in 29 ms on localhost (2/2) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-06-13 20:17:24 INFO DAGScheduler:59 - ShuffleMapStage 24 (apply at Transformer.scala:22) finished in 0.032 s -2016-06-13 20:17:24 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:17:24 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:17:24 INFO DAGScheduler:59 - waiting: Set(ResultStage 25) -2016-06-13 20:17:24 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:17:24 INFO DAGScheduler:59 - Missing parents for ResultStage 25: List() -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:51031 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting ResultStage 25 (ShuffledRDD[79] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned shuffle 10 -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=78719, maxMem=766075207 -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:51031 in memory (size: 1804.0 B, free: 730.6 MB) -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=83983, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:51031 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:17:24 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (ShuffledRDD[79] at apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:24 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 56) -2016-06-13 20:17:24 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 57) -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:51031 in memory (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:17:24 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 57). 1161 bytes result sent to driver -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 57) in 11 ms on localhost (1/2) -2016-06-13 20:17:24 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 56). 1498 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 56) in 14 ms on localhost (2/2) -2016-06-13 20:17:24 INFO DAGScheduler:59 - ResultStage 25 (apply at Transformer.scala:22) finished in 0.016 s -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-06-13 20:17:24 INFO DAGScheduler:59 - Job 12 finished: apply at Transformer.scala:22, took 0.117929 s -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:51031 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:17:24 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 350.0 and centroid: [-0.47440048489638564,0.9650523283036628,-1.6700055274657652,1.269947562552048,0.3797915177328754] -2016-06-13 20:17:24 INFO JobScheduler:59 - Finished job streaming job 7000 ms.0 from job set of time 7000 ms -2016-06-13 20:17:24 INFO JobScheduler:59 - Starting job streaming job 7000 ms.1 from job set of time 7000 ms -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:51031 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:17:24 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Registering RDD 41 (apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned shuffle 8 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Got job 13 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:17:24 INFO DAGScheduler:59 - Final stage: ResultStage 27(apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (UnionRDD[41] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:51031 in memory (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=61169, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=64449, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:51031 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:17:24 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 26 (UnionRDD[41] at apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 3 tasks -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 58, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 59, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 20:17:24 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 59) -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:17:24 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 58) -2016-06-13 20:17:24 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 59). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-1000 -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 60, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:51031 in memory (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:17:24 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 60) -2016-06-13 20:17:24 INFO CheckpointWriter:59 - Checkpoint for time 2000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-2000', took 27485 bytes and 227 ms -2016-06-13 20:17:24 INFO CheckpointWriter:59 - Saving checkpoint for time 3000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-3000' -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 20:17:24 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 58). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned shuffle 7 -2016-06-13 20:17:24 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 60). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:51031 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:17:24 INFO DStreamGraph:59 - Clearing checkpoint data for time 2000 ms -2016-06-13 20:17:24 INFO DStreamGraph:59 - Cleared checkpoint data for time 2000 ms -2016-06-13 20:17:24 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:17:24 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/receivedBlockMetadata older than 1000: -2016-06-13 20:17:24 INFO InputInfoTracker:59 - remove old batch metadata: -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 59) in 42 ms on localhost (1/3) -2016-06-13 20:17:24 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/receivedBlockMetadata older than 1000 -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 60) in 23 ms on localhost (2/3) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 58) in 46 ms on localhost (3/3) -2016-06-13 20:17:24 INFO DAGScheduler:59 - ShuffleMapStage 26 (apply at Transformer.scala:22) finished in 0.048 s -2016-06-13 20:17:24 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:17:24 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:17:24 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-06-13 20:17:24 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:17:24 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[80] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=53207, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=49219, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.5 MB) -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:51031 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:51031 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 20:17:24 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[80] at apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 62, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:24 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 61) -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned shuffle 6 -2016-06-13 20:17:24 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 62) -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:51031 in memory (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:24 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 62). 1172 bytes result sent to driver -2016-06-13 20:17:24 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 61). 1180 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 62) in 15 ms on localhost (1/2) -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 61) in 21 ms on localhost (2/2) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-06-13 20:17:24 INFO DAGScheduler:59 - ResultStage 27 (apply at Transformer.scala:22) finished in 0.023 s -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:51031 in memory (size: 2004.0 B, free: 730.6 MB) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Job 13 finished: apply at Transformer.scala:22, took 0.101934 s -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 20:17:24 INFO JobScheduler:59 - Finished job streaming job 7000 ms.1 from job set of time 7000 ms -2016-06-13 20:17:24 INFO JobScheduler:59 - Total delay: 1465820237.369 s for time 7000 ms (execution: 0.240 s) -2016-06-13 20:17:24 INFO JobScheduler:59 - Starting job streaming job 8000 ms.0 from job set of time 8000 ms -2016-06-13 20:17:24 INFO ParallelCollectionRDD:59 - Removing RDD 32 from persistence list -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned shuffle 5 -2016-06-13 20:17:24 INFO MapPartitionsRDD:59 - Removing RDD 37 from persistence list -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 32 -2016-06-13 20:17:24 INFO ShuffledRDD:59 - Removing RDD 36 from persistence list -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 37 -2016-06-13 20:17:24 INFO UnionRDD:59 - Removing RDD 35 from persistence list -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 36 -2016-06-13 20:17:24 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:51031 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 35 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Registering RDD 81 (apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Got job 14 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:17:24 INFO DAGScheduler:59 - Final stage: ResultStage 29(apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 28) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 28) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting ShuffleMapStage 28 (MapPartitionsRDD[81] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=32734, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=37478, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO MapPartitionsRDD:59 - Removing RDD 33 from persistence list -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:51031 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:17:24 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:24 INFO JobGenerator:59 - Checkpointing graph for time 7000 ms -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 28 (MapPartitionsRDD[81] at apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 33 -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 20:17:24 INFO DStreamGraph:59 - Updating checkpoint data for time 7000 ms -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-06-13 20:17:24 INFO DStreamGraph:59 - Updated checkpoint data for time 7000 ms -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 63, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:51031 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 64, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:17:24 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 64) -2016-06-13 20:17:24 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 63) -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned shuffle 4 -2016-06-13 20:17:24 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-06-13 20:17:24 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 64). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 64) in 24 ms on localhost (1/2) -2016-06-13 20:17:24 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 63). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 63) in 40 ms on localhost (2/2) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-06-13 20:17:24 INFO DAGScheduler:59 - ShuffleMapStage 28 (apply at Transformer.scala:22) finished in 0.042 s -2016-06-13 20:17:24 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:17:24 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:17:24 INFO DAGScheduler:59 - waiting: Set(ResultStage 29) -2016-06-13 20:17:24 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:17:24 INFO DAGScheduler:59 - Missing parents for ResultStage 29: List() -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting ResultStage 29 (ShuffledRDD[82] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=32741, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=38005, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:51031 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:17:24 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (ShuffledRDD[82] at apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 65, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 66, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:24 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 66) -2016-06-13 20:17:24 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 65) -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:24 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 66). 1161 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 66) in 9 ms on localhost (1/2) -2016-06-13 20:17:24 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 65). 1498 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 65) in 14 ms on localhost (2/2) -2016-06-13 20:17:24 INFO DAGScheduler:59 - ResultStage 29 (apply at Transformer.scala:22) finished in 0.014 s -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 20:17:24 INFO DAGScheduler:59 - Job 14 finished: apply at Transformer.scala:22, took 0.084670 s -2016-06-13 20:17:24 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 400.0 and centroid: [-0.4716182064591067,0.967586334056103,-1.6699954778983372,1.270341503100185,0.37935009425710725] -2016-06-13 20:17:24 INFO JobScheduler:59 - Finished job streaming job 8000 ms.0 from job set of time 8000 ms -2016-06-13 20:17:24 INFO JobScheduler:59 - Starting job streaming job 8000 ms.1 from job set of time 8000 ms -2016-06-13 20:17:24 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Registering RDD 47 (apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Got job 15 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:17:24 INFO DAGScheduler:59 - Final stage: ResultStage 31(apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 30) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting ShuffleMapStage 30 (UnionRDD[47] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=40699, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(2006) called with curMem=43979, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2006.0 B, free 730.5 MB) -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:51031 (size: 2006.0 B, free: 730.6 MB) -2016-06-13 20:17:24 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 30 (UnionRDD[47] at apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 3 tasks -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 67, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 68, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:17:24 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 68) -2016-06-13 20:17:24 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 67) -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:17:24 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-2000.bk -2016-06-13 20:17:24 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 67). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 69, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 67) in 28 ms on localhost (1/3) -2016-06-13 20:17:24 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 69) -2016-06-13 20:17:24 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 68). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 68) in 29 ms on localhost (2/3) -2016-06-13 20:17:24 INFO CheckpointWriter:59 - Checkpoint for time 3000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-3000', took 27496 bytes and 212 ms -2016-06-13 20:17:24 INFO DStreamGraph:59 - Clearing checkpoint data for time 3000 ms -2016-06-13 20:17:24 INFO DStreamGraph:59 - Cleared checkpoint data for time 3000 ms -2016-06-13 20:17:24 INFO CheckpointWriter:59 - Saving checkpoint for time 4000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-4000' -2016-06-13 20:17:24 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:17:24 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/receivedBlockMetadata older than 2000: -2016-06-13 20:17:24 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/receivedBlockMetadata older than 2000 -2016-06-13 20:17:24 INFO InputInfoTracker:59 - remove old batch metadata: 1000 ms -2016-06-13 20:17:24 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 69). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 69) in 10 ms on localhost (3/3) -2016-06-13 20:17:24 INFO DAGScheduler:59 - ShuffleMapStage 30 (apply at Transformer.scala:22) finished in 0.037 s -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 20:17:24 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:17:24 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:17:24 INFO DAGScheduler:59 - waiting: Set(ResultStage 31) -2016-06-13 20:17:24 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:17:24 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List() -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[83] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=45985, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=49225, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:51031 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:17:24 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[83] at apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 70, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 71, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:24 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 71) -2016-06-13 20:17:24 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 70) -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:17:24 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 71). 1172 bytes result sent to driver -2016-06-13 20:17:24 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 70). 1180 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 70) in 15 ms on localhost (1/2) -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 71) in 44 ms on localhost (2/2) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 20:17:24 INFO DAGScheduler:59 - ResultStage 31 (apply at Transformer.scala:22) finished in 0.055 s -2016-06-13 20:17:24 INFO DAGScheduler:59 - Job 15 finished: apply at Transformer.scala:22, took 0.119163 s -2016-06-13 20:17:24 INFO JobScheduler:59 - Finished job streaming job 8000 ms.1 from job set of time 8000 ms -2016-06-13 20:17:24 INFO JobScheduler:59 - Total delay: 1465820236.601 s for time 8000 ms (execution: 0.232 s) -2016-06-13 20:17:24 INFO JobScheduler:59 - Starting job streaming job 9000 ms.0 from job set of time 9000 ms -2016-06-13 20:17:24 INFO ParallelCollectionRDD:59 - Removing RDD 38 from persistence list -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 38 -2016-06-13 20:17:24 INFO MapPartitionsRDD:59 - Removing RDD 43 from persistence list -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 43 -2016-06-13 20:17:24 INFO ShuffledRDD:59 - Removing RDD 42 from persistence list -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 42 -2016-06-13 20:17:24 INFO UnionRDD:59 - Removing RDD 41 from persistence list -2016-06-13 20:17:24 INFO MapPartitionsRDD:59 - Removing RDD 39 from persistence list -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 41 -2016-06-13 20:17:24 INFO JobGenerator:59 - Checkpointing graph for time 8000 ms -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 39 -2016-06-13 20:17:24 INFO DStreamGraph:59 - Updating checkpoint data for time 8000 ms -2016-06-13 20:17:24 INFO DStreamGraph:59 - Updated checkpoint data for time 8000 ms -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:17:24 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Registering RDD 84 (apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Got job 16 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:17:24 INFO DAGScheduler:59 - Final stage: ResultStage 33(apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 32) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting ShuffleMapStage 32 (MapPartitionsRDD[84] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=51028, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(2477) called with curMem=55772, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:51031 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:17:24 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 32 (MapPartitionsRDD[84] at apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 72, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 73, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:17:24 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 73) -2016-06-13 20:17:24 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 72) -2016-06-13 20:17:24 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 73). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 72). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 73) in 15 ms on localhost (1/2) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 72) in 17 ms on localhost (2/2) -2016-06-13 20:17:24 INFO DAGScheduler:59 - ShuffleMapStage 32 (apply at Transformer.scala:22) finished in 0.018 s -2016-06-13 20:17:24 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:17:24 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:17:24 INFO DAGScheduler:59 - waiting: Set(ResultStage 33) -2016-06-13 20:17:24 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 20:17:24 INFO DAGScheduler:59 - Missing parents for ResultStage 33: List() -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting ResultStage 33 (ShuffledRDD[85] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=58249, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=63513, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:51031 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:17:24 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (ShuffledRDD[85] at apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 74, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 75, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:24 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 74) -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:24 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 75) -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:17:24 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 74). 1498 bytes result sent to driver -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 74) in 10 ms on localhost (1/2) -2016-06-13 20:17:24 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 75). 1161 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 75) in 12 ms on localhost (2/2) -2016-06-13 20:17:24 INFO DAGScheduler:59 - ResultStage 33 (apply at Transformer.scala:22) finished in 0.013 s -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 20:17:24 INFO DAGScheduler:59 - Job 16 finished: apply at Transformer.scala:22, took 0.053904 s -2016-06-13 20:17:24 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 450.0 and centroid: [-0.4722098164150225,0.9642763266691793,-1.6666072662538036,1.2709270669491914,0.37911973669212223] -2016-06-13 20:17:24 INFO JobScheduler:59 - Finished job streaming job 9000 ms.0 from job set of time 9000 ms -2016-06-13 20:17:24 INFO JobScheduler:59 - Starting job streaming job 9000 ms.1 from job set of time 9000 ms -2016-06-13 20:17:24 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Registering RDD 53 (apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Got job 17 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:17:24 INFO DAGScheduler:59 - Final stage: ResultStage 35(apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 34) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 34) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting ShuffleMapStage 34 (UnionRDD[53] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=66207, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=69487, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.5 MB) -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:51031 (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:17:24 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 34 (UnionRDD[53] at apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 3 tasks -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 76, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 77, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:17:24 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 77) -2016-06-13 20:17:24 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 76) -2016-06-13 20:17:24 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-2000 -2016-06-13 20:17:24 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 77). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 78, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 77) in 16 ms on localhost (1/3) -2016-06-13 20:17:24 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 78) -2016-06-13 20:17:24 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 76). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 76) in 21 ms on localhost (2/3) -2016-06-13 20:17:24 INFO CheckpointWriter:59 - Checkpoint for time 4000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-4000', took 27488 bytes and 184 ms -2016-06-13 20:17:24 INFO DStreamGraph:59 - Clearing checkpoint data for time 4000 ms -2016-06-13 20:17:24 INFO DStreamGraph:59 - Cleared checkpoint data for time 4000 ms -2016-06-13 20:17:24 INFO CheckpointWriter:59 - Saving checkpoint for time 5000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-5000' -2016-06-13 20:17:24 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:17:24 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/receivedBlockMetadata older than 3000: -2016-06-13 20:17:24 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/receivedBlockMetadata older than 3000 -2016-06-13 20:17:24 INFO InputInfoTracker:59 - remove old batch metadata: 2000 ms -2016-06-13 20:17:24 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 78). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 78) in 11 ms on localhost (3/3) -2016-06-13 20:17:24 INFO DAGScheduler:59 - ShuffleMapStage 34 (apply at Transformer.scala:22) finished in 0.027 s -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-06-13 20:17:24 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:17:24 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:17:24 INFO DAGScheduler:59 - waiting: Set(ResultStage 35) -2016-06-13 20:17:24 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:17:24 INFO DAGScheduler:59 - Missing parents for ResultStage 35: List() -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[86] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=71494, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(1804) called with curMem=74734, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 1804.0 B, free 730.5 MB) -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:51031 (size: 1804.0 B, free: 730.6 MB) -2016-06-13 20:17:24 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[86] at apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 79, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 80, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:24 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 80) -2016-06-13 20:17:24 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 79) -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:24 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 80). 1172 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 80) in 6 ms on localhost (1/2) -2016-06-13 20:17:24 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 79). 1180 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 79) in 11 ms on localhost (2/2) -2016-06-13 20:17:24 INFO DAGScheduler:59 - ResultStage 35 (apply at Transformer.scala:22) finished in 0.012 s -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-06-13 20:17:24 INFO DAGScheduler:59 - Job 17 finished: apply at Transformer.scala:22, took 0.059324 s -2016-06-13 20:17:24 INFO JobScheduler:59 - Finished job streaming job 9000 ms.1 from job set of time 9000 ms -2016-06-13 20:17:24 INFO JobScheduler:59 - Total delay: 1465820235.738 s for time 9000 ms (execution: 0.136 s) -2016-06-13 20:17:24 INFO JobScheduler:59 - Starting job streaming job 10000 ms.0 from job set of time 10000 ms -2016-06-13 20:17:24 INFO ParallelCollectionRDD:59 - Removing RDD 44 from persistence list -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 44 -2016-06-13 20:17:24 INFO MapPartitionsRDD:59 - Removing RDD 49 from persistence list -2016-06-13 20:17:24 INFO ShuffledRDD:59 - Removing RDD 48 from persistence list -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 48 -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 49 -2016-06-13 20:17:24 INFO UnionRDD:59 - Removing RDD 47 from persistence list -2016-06-13 20:17:24 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Registering RDD 87 (apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Got job 18 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:17:24 INFO DAGScheduler:59 - Final stage: ResultStage 37(apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 36) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 36) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting ShuffleMapStage 36 (MapPartitionsRDD[87] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=76538, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=81282, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO MapPartitionsRDD:59 - Removing RDD 45 from persistence list -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 47 -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:51031 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:17:24 INFO JobGenerator:59 - Checkpointing graph for time 9000 ms -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 45 -2016-06-13 20:17:24 INFO DStreamGraph:59 - Updating checkpoint data for time 9000 ms -2016-06-13 20:17:24 INFO DStreamGraph:59 - Updated checkpoint data for time 9000 ms -2016-06-13 20:17:24 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 36 (MapPartitionsRDD[87] at apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 81, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 82, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:17:24 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 81) -2016-06-13 20:17:24 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 82) -2016-06-13 20:17:24 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 81). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 82). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 82) in 21 ms on localhost (1/2) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 81) in 23 ms on localhost (2/2) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-06-13 20:17:24 INFO DAGScheduler:59 - ShuffleMapStage 36 (apply at Transformer.scala:22) finished in 0.024 s -2016-06-13 20:17:24 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:17:24 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:17:24 INFO DAGScheduler:59 - waiting: Set(ResultStage 37) -2016-06-13 20:17:24 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:17:24 INFO DAGScheduler:59 - Missing parents for ResultStage 37: List() -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting ResultStage 37 (ShuffledRDD[88] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=83764, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=89028, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:51031 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:17:24 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (ShuffledRDD[88] at apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 83, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 84, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:24 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 83) -2016-06-13 20:17:24 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 84) -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:24 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 84). 1161 bytes result sent to driver -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 84) in 9 ms on localhost (1/2) -2016-06-13 20:17:24 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 83). 1498 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 83) in 15 ms on localhost (2/2) -2016-06-13 20:17:24 INFO DAGScheduler:59 - ResultStage 37 (apply at Transformer.scala:22) finished in 0.015 s -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-06-13 20:17:24 INFO DAGScheduler:59 - Job 18 finished: apply at Transformer.scala:22, took 0.065658 s -2016-06-13 20:17:24 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 500.0 and centroid: [-0.4725511979691583,0.9644503899125422,-1.668776373542808,1.2721254429935838,0.37815209739836425] -2016-06-13 20:17:24 INFO JobScheduler:59 - Finished job streaming job 10000 ms.0 from job set of time 10000 ms -2016-06-13 20:17:24 INFO JobScheduler:59 - Starting job streaming job 10000 ms.1 from job set of time 10000 ms -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:17:24 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Registering RDD 59 (apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Got job 19 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:17:24 INFO DAGScheduler:59 - Final stage: ResultStage 39(apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 38) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 38) -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting ShuffleMapStage 38 (UnionRDD[59] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=91724, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=95004, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.5 MB) -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:51031 (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:17:24 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 38 (UnionRDD[59] at apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 3 tasks -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 85, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 86, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:17:24 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 85) -2016-06-13 20:17:24 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 86) -2016-06-13 20:17:24 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 85). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 87, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 85) in 13 ms on localhost (1/3) -2016-06-13 20:17:24 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 87) -2016-06-13 20:17:24 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 86). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-3000.bk -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 86) in 27 ms on localhost (2/3) -2016-06-13 20:17:24 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 87). 1159 bytes result sent to driver -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 87) in 22 ms on localhost (3/3) -2016-06-13 20:17:24 INFO DAGScheduler:59 - ShuffleMapStage 38 (apply at Transformer.scala:22) finished in 0.033 s -2016-06-13 20:17:24 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:17:24 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:17:24 INFO DAGScheduler:59 - waiting: Set(ResultStage 39) -2016-06-13 20:17:24 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-06-13 20:17:24 INFO DAGScheduler:59 - Missing parents for ResultStage 39: List() -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[89] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:17:24 INFO CheckpointWriter:59 - Checkpoint for time 5000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-5000', took 27485 bytes and 163 ms -2016-06-13 20:17:24 INFO CheckpointWriter:59 - Saving checkpoint for time 6000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-6000' -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=97011, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:17:24 INFO MemoryStore:59 - ensureFreeSpace(1804) called with curMem=100251, maxMem=766075207 -2016-06-13 20:17:24 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 1804.0 B, free 730.5 MB) -2016-06-13 20:17:24 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:51031 (size: 1804.0 B, free: 730.6 MB) -2016-06-13 20:17:24 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:17:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[89] at apply at Transformer.scala:22) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:17:24 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 89) -2016-06-13 20:17:24 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 88) -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:24 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 89). 1172 bytes result sent to driver -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:17:24 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:17:24 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 88). 1180 bytes result sent to driver -2016-06-13 20:17:24 INFO DStreamGraph:59 - Clearing checkpoint data for time 5000 ms -2016-06-13 20:17:24 INFO DStreamGraph:59 - Cleared checkpoint data for time 5000 ms -2016-06-13 20:17:24 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:17:24 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/receivedBlockMetadata older than 4000: -2016-06-13 20:17:24 INFO InputInfoTracker:59 - remove old batch metadata: 3000 ms -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 89) in 22 ms on localhost (1/2) -2016-06-13 20:17:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 88) in 24 ms on localhost (2/2) -2016-06-13 20:17:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-06-13 20:17:24 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/receivedBlockMetadata older than 4000 -2016-06-13 20:17:24 INFO DAGScheduler:59 - ResultStage 39 (apply at Transformer.scala:22) finished in 0.035 s -2016-06-13 20:17:24 INFO DAGScheduler:59 - Job 19 finished: apply at Transformer.scala:22, took 0.093442 s -2016-06-13 20:17:24 INFO JobScheduler:59 - Finished job streaming job 10000 ms.1 from job set of time 10000 ms -2016-06-13 20:17:24 INFO JobScheduler:59 - Total delay: 1465820234.921 s for time 10000 ms (execution: 0.183 s) -2016-06-13 20:17:24 INFO ParallelCollectionRDD:59 - Removing RDD 50 from persistence list -2016-06-13 20:17:24 INFO MapPartitionsRDD:59 - Removing RDD 55 from persistence list -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 50 -2016-06-13 20:17:24 INFO ShuffledRDD:59 - Removing RDD 54 from persistence list -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 55 -2016-06-13 20:17:24 INFO UnionRDD:59 - Removing RDD 53 from persistence list -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 54 -2016-06-13 20:17:24 INFO MapPartitionsRDD:59 - Removing RDD 51 from persistence list -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 53 -2016-06-13 20:17:24 INFO BlockManager:59 - Removing RDD 51 -2016-06-13 20:17:24 INFO JobGenerator:59 - Checkpointing graph for time 10000 ms -2016-06-13 20:17:24 INFO DStreamGraph:59 - Updating checkpoint data for time 10000 ms -2016-06-13 20:17:24 INFO DStreamGraph:59 - Updated checkpoint data for time 10000 ms -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - Output generated in 3225 milliseconds -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:17:24 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:17:25 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-3000 -2016-06-13 20:17:25 INFO CheckpointWriter:59 - Checkpoint for time 6000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-6000', took 27481 bytes and 158 ms -2016-06-13 20:17:25 INFO DStreamGraph:59 - Clearing checkpoint data for time 6000 ms -2016-06-13 20:17:25 INFO CheckpointWriter:59 - Saving checkpoint for time 7000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-7000' -2016-06-13 20:17:25 INFO DStreamGraph:59 - Cleared checkpoint data for time 6000 ms -2016-06-13 20:17:25 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:17:25 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/receivedBlockMetadata older than 5000: -2016-06-13 20:17:25 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/receivedBlockMetadata older than 5000 -2016-06-13 20:17:25 INFO InputInfoTracker:59 - remove old batch metadata: 4000 ms -2016-06-13 20:17:25 INFO JobGenerator:59 - Stopping JobGenerator immediately -2016-06-13 20:17:25 INFO RecurringTimer:59 - Stopped timer for JobGenerator after time 10000 -2016-06-13 20:17:35 INFO CheckpointWriter:59 - CheckpointWriter executor terminated ? false, waited for 10000 ms. -2016-06-13 20:17:35 INFO JobGenerator:59 - Stopped JobGenerator -2016-06-13 20:17:35 INFO JobScheduler:59 - Stopped JobScheduler -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/batch,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/streaming,null} -2016-06-13 20:17:35 INFO StreamingContext:59 - StreamingContext stopped successfully -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/batch/json,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/json,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:17:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:17:35 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-4000.bk -2016-06-13 20:17:35 INFO CheckpointWriter:59 - Checkpoint for time 7000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b/checkpoint-7000', took 27479 bytes and 10124 ms -2016-06-13 20:17:35 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 20:17:35 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:17:35 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:17:35 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:17:35 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:17:35 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:17:35 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:17:35 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:17:35 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:17:35 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:17:35 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 20:17:35 INFO StreamingKMeansSuite:59 - - -===== FINISHED o.a.s.mllib.clustering.kmeans.StreamingKMeansSuite: 'accuracy for single center and equivalence to grand average' ===== - -2016-06-13 20:17:35 WARN StreamingContext:71 - StreamingContext has already been stopped -2016-06-13 20:17:35 INFO SparkContext:59 - SparkContext already stopped. -2016-06-13 20:17:35 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 20:17:35 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b -2016-06-13 20:17:35 ERROR ShutdownHookManager:96 - Exception while deleting Spark temp dir: C:\Users\xubo\AppData\Local\Temp\spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b -java.io.IOException: Failed to delete: C:\Users\xubo\AppData\Local\Temp\spark-71757ea0-af02-4a1e-a2c7-c0e9964ed38b - at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:884) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:63) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:60) - at scala.collection.mutable.HashSet.foreach(HashSet.scala:79) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1.apply$mcV$sp(ShutdownHookManager.scala:60) - at org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:264) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1699) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply$mcV$sp(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:234) - at scala.util.Try$.apply(Try.scala:161) - at org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:216) - at org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54) -2016-06-13 20:17:35 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-acf407e9-09a9-4bc3-8721-8b8587bb50f0 -2016-06-13 20:19:19 INFO StreamingKMeansSuite:59 - Using manual clock -2016-06-13 20:19:19 INFO StreamingKMeansSuite:59 - - -===== TEST OUTPUT FOR o.a.s.mllib.clustering.kmeans.StreamingKMeansSuite: 'accuracy for single center and equivalence to grand average' ===== - -2016-06-13 20:19:19 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:19:19 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 20:19:20 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:19:20 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:19:20 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:19:20 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:19:20 INFO Remoting:74 - Starting remoting -2016-06-13 20:19:21 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:51054] -2016-06-13 20:19:21 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51054. -2016-06-13 20:19:21 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:19:21 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:19:21 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-40a7a4a8-8e28-4f7e-92e6-8ba227810ae2 -2016-06-13 20:19:21 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:19:21 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-02abcfdb-f5ef-49b2-8ac5-b2491dd814c6\httpd-0aee35ee-da20-4aa2-82eb-f8a20e87546c -2016-06-13 20:19:21 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:19:21 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:19:21 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51055 -2016-06-13 20:19:21 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51055. -2016-06-13 20:19:21 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:19:21 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:19:21 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 20:19:21 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 20:19:21 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 20:19:21 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:19:21 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:19:21 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51066. -2016-06-13 20:19:21 INFO NettyBlockTransferService:59 - Server created on 51066 -2016-06-13 20:19:21 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:19:21 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51066 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51066) -2016-06-13 20:19:21 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:19:23 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 20:19:23 INFO StreamingKMeansSuite:59 - numBatches = 10, numExpectedOutput = 10 -2016-06-13 20:19:23 INFO ForEachDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:19:23 INFO TestInputStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:19:23 INFO TestInputStream:59 - Slide time = 1000 ms -2016-06-13 20:19:23 INFO TestInputStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:19:23 INFO TestInputStream:59 - Checkpoint interval = null -2016-06-13 20:19:23 INFO TestInputStream:59 - Remember duration = 1000 ms -2016-06-13 20:19:23 INFO TestInputStream:59 - Initialized and validated org.apache.spark.streaming.TestInputStream@51793685 -2016-06-13 20:19:23 INFO ForEachDStream:59 - Slide time = 1000 ms -2016-06-13 20:19:23 INFO ForEachDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:19:23 INFO ForEachDStream:59 - Checkpoint interval = null -2016-06-13 20:19:23 INFO ForEachDStream:59 - Remember duration = 1000 ms -2016-06-13 20:19:23 INFO ForEachDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.ForEachDStream@2ca08829 -2016-06-13 20:19:23 INFO TestOutputStreamWithPartitions:59 - metadataCleanupDelay = -1 -2016-06-13 20:19:23 INFO MappedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:19:23 INFO ShuffledDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:19:23 INFO TransformedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:19:23 INFO MappedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:19:23 INFO TestInputStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:19:23 INFO TestInputStream:59 - Slide time = 1000 ms -2016-06-13 20:19:23 INFO TestInputStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:19:23 INFO TestInputStream:59 - Checkpoint interval = null -2016-06-13 20:19:23 INFO TestInputStream:59 - Remember duration = 1000 ms -2016-06-13 20:19:23 INFO TestInputStream:59 - Initialized and validated org.apache.spark.streaming.TestInputStream@51793685 -2016-06-13 20:19:23 INFO MappedDStream:59 - Slide time = 1000 ms -2016-06-13 20:19:23 INFO MappedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:19:23 INFO MappedDStream:59 - Checkpoint interval = null -2016-06-13 20:19:23 INFO MappedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:19:23 INFO MappedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@3b5f683e -2016-06-13 20:19:23 INFO TransformedDStream:59 - Slide time = 1000 ms -2016-06-13 20:19:23 INFO TransformedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:19:23 INFO TransformedDStream:59 - Checkpoint interval = null -2016-06-13 20:19:23 INFO TransformedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:19:23 INFO TransformedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.TransformedDStream@fccab1d -2016-06-13 20:19:23 INFO ShuffledDStream:59 - Slide time = 1000 ms -2016-06-13 20:19:23 INFO ShuffledDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:19:23 INFO ShuffledDStream:59 - Checkpoint interval = null -2016-06-13 20:19:23 INFO ShuffledDStream:59 - Remember duration = 1000 ms -2016-06-13 20:19:23 INFO ShuffledDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.ShuffledDStream@456c0ec6 -2016-06-13 20:19:23 INFO MappedDStream:59 - Slide time = 1000 ms -2016-06-13 20:19:23 INFO MappedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:19:23 INFO MappedDStream:59 - Checkpoint interval = null -2016-06-13 20:19:23 INFO MappedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:19:23 INFO MappedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@2c0bf31c -2016-06-13 20:19:23 INFO TestOutputStreamWithPartitions:59 - Slide time = 1000 ms -2016-06-13 20:19:23 INFO TestOutputStreamWithPartitions:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:19:23 INFO TestOutputStreamWithPartitions:59 - Checkpoint interval = null -2016-06-13 20:19:23 INFO TestOutputStreamWithPartitions:59 - Remember duration = 1000 ms -2016-06-13 20:19:23 INFO TestOutputStreamWithPartitions:59 - Initialized and validated org.apache.spark.streaming.TestOutputStreamWithPartitions@b6edc37 -2016-06-13 20:19:23 INFO RecurringTimer:59 - Started timer for JobGenerator at time 1000 -2016-06-13 20:19:23 INFO JobGenerator:59 - Started JobGenerator at 1000 ms -2016-06-13 20:19:23 INFO JobScheduler:59 - Started JobScheduler -2016-06-13 20:19:23 INFO StreamingContext:59 - StreamingContext started -2016-06-13 20:19:23 INFO StreamingKMeansSuite:59 - Manual clock before advancing = 0 -2016-06-13 20:19:23 INFO StreamingKMeansSuite:59 - Manual clock after advancing = 10000 -2016-06-13 20:19:23 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:23 INFO TestInputStream:59 - Computing RDD for time 1000 ms -2016-06-13 20:19:23 INFO TestInputStream:59 - Created RDD 0 with Vector([-0.35685422167583125,0.9995288307555166,-1.5877809386448758,1.210845548490831,0.56222524157234], [-0.45216537133396134,0.8040574757345051,-1.5589888701250443,1.115603017032467,0.344649973028188], [-0.3801675945145341,1.0697589561731022,-1.6486682116186815,1.2790268654063703,0.5116371486214886], [-0.545300880562902,1.1093057253978866,-1.7582382472264182,1.211400006292075,0.2932044815946008], [-0.3995483137357882,0.929119621864747,-1.7401380541085598,1.2990457426846032,0.299090686584923], [-0.5964001160322344,1.0378164371074625,-1.553411818459791,1.2193968564632125,0.4492703031123452], [-0.33949398265706626,1.1369769008642154,-1.7816495259375202,1.2857769527411067,0.5642853362323552], [-0.4522936770981281,1.090819158309221,-1.4785463285656288,1.2734882859357213,0.31864385887835545], [-0.4895260213585818,1.024567290792244,-1.7744155605537828,1.4515329700677304,0.39720325477885093], [-0.5479321442182269,0.8926951810306638,-1.617388979581139,1.3192738326981768,0.3243774353000105], [-0.5468296230191492,0.9034725811394896,-1.8269524875493084,1.161899613497355,0.22607191229030854], [-0.4344367225435325,0.9054178856043291,-1.6658399540877864,1.13578899284944,0.46511292492021983], [-0.5914600674966851,0.9727747336030947,-1.7411162798640765,1.3902196062020102,0.43158411906136107], [-0.5644425684165094,0.996354296088912,-1.817848647277835,1.3768030733201289,0.28609432985591315], [-0.44148386701129744,1.025737906243747,-1.7073695034902865,1.2539087121413848,0.349242056080172], [-0.5287113084613145,0.8475916475163896,-1.8166057479945195,1.238631411179877,0.36922519791770014], [-0.2540595910515851,0.9698131032346381,-1.5866552625154318,1.2179915976157372,0.3240398576394729], [-0.5026644816450073,1.109581665015373,-1.7151161774591086,1.3901513268104455,0.39042855132933163], [-0.4131172722481001,1.0096927045418442,-1.811286466550916,1.2719080737538961,0.28293757989913904], [-0.4515790075180954,1.0935474561719332,-1.7896107864561164,1.3203383417885988,0.2994362800834273], [-0.4171607573128071,1.0818260184642114,-1.8219543038308885,1.2496462030096327,0.4964658371791977], [-0.45586902589910655,1.0378742745036647,-1.7000458648640797,1.446592332373813,0.348566870310007], [-0.3532087778209343,0.8044437973592731,-1.7162629066191757,1.3339281879504057,0.17946188844185906], [-0.5154063008996834,0.9595656827532482,-1.6519653379002484,1.322654750805598,0.3008929692388781], [-0.3967887546418169,0.7908879805460386,-1.7242582851383725,1.2141584886700174,0.36827019559433166], [-0.48352710482779404,1.0255449792765485,-1.7171397454598174,1.284877805985566,0.437232715780399], [-0.40324856443707463,1.020485534999759,-1.6517518343613002,1.2864020019792224,0.3984832850799499], [-0.6517069948565133,0.969461018860354,-1.684015510299471,1.2027906385718596,0.2014860155598368], [-0.6148278408211463,0.9795135452912322,-1.8642733722424907,1.2720240024566645,0.39785426587083933], [-0.6409674819545198,1.1778584543507387,-1.8094969072333307,1.24145841048891,0.1624672183638662], [-0.6368691555002903,0.8170131462860201,-1.6827833832635621,1.3031179618920097,0.31628673987022604], [-0.43096648399184834,0.9655746876954024,-1.7845188100637381,1.4052721272088269,0.5845842279495], [-0.4442125841627056,1.0257872008493307,-1.7242589434969455,1.2745428690859977,0.39998857867859966], [-0.4527127515828855,0.9000469054331385,-1.7497092961121075,1.2451107418571954,0.4344380036579415], [-0.6030914309562295,0.9699928952430515,-1.712604726536663,1.38261404269885,0.48344898725768926], [-0.34976726689041593,0.9355870894961762,-1.5766155567170825,1.3590936489794139,0.35690680463154484], [-0.575628267406687,0.9538063357613722,-1.6445003616008003,1.2104804829930789,0.3911315521177748], [-0.5582822637309535,0.9339694428739754,-1.6511037013459962,1.4401460976648193,0.3607766390547849], [-0.5468811641579405,0.997556943752324,-1.778985900216746,1.2440321529105447,0.4204585578447196], [-0.48994207115736743,1.0755700976056979,-1.5322042869816734,1.2132674597398667,0.5961642366275318], [-0.5345824744371771,0.7659168558371742,-1.7009697565551642,1.2200491259133113,0.24064266183571642], [-0.4425411108434083,0.9636579302553172,-1.6769715787345627,1.2060504970822528,0.1832578105698996], [-0.2803908635114948,0.9814260387904405,-1.6744903280757537,1.438894231115956,0.5307857877828275], [-0.4646639723438268,0.9422153316765177,-1.8936280948227913,1.1755014401606392,0.3269897050872188], [-0.5794967530942349,1.008452319256215,-1.8095212163297067,1.3053459835247618,0.3862084672749757], [-0.36564413314560745,0.9550404460199052,-1.671332555981891,1.2387784773127055,0.29524181369582553], [-0.4947387784852314,0.8968969114403345,-1.5032027608518188,1.263293789437244,0.41872133210307433], [-0.5901163249097482,0.9195890740589755,-1.76975579729397,1.2233696400583831,0.3403182341587175], [-0.7175598778778954,0.8837946345395844,-1.8329343739105675,1.3562350855120264,0.30734388952604197], [-0.4376962364418643,1.0046436887819825,-1.5427982731479972,1.1077094728701697,0.30461693506090337]) -2016-06-13 20:19:23 INFO JobScheduler:59 - Added jobs for time 1000 ms -2016-06-13 20:19:23 INFO TestInputStream:59 - Computing RDD for time 2000 ms -2016-06-13 20:19:23 INFO JobScheduler:59 - Starting job streaming job 1000 ms.0 from job set of time 1000 ms -2016-06-13 20:19:23 INFO TestInputStream:59 - Created RDD 6 with Vector([-0.5385984930004446,0.9989931115946847,-1.745310682734983,1.2763556288677276,0.41555544930746574], [-0.5140757026211953,0.9616396809310985,-1.6092748223737894,1.3080875431688948,0.3958240822167327], [-0.3317121400990889,1.1311521924692884,-1.5940394948396088,1.235663715627094,0.5352186835894586], [-0.3566003302425871,0.8649446433306678,-1.6281158497086519,1.0841327390682376,0.33108928300382867], [-0.4898654567486354,1.0095506134582393,-1.7900184464939166,1.4376638486285889,0.5476994415325478], [-0.5067718408631526,0.8330942372560937,-1.5447521454199076,1.3956201717944832,0.40502500771937533], [-0.45899690569395396,1.0114199686087044,-1.6793704227956527,1.2264029484806054,0.3617800498770533], [-0.42389468031403765,0.9744019756374155,-1.4339024359998267,1.3506788012287876,0.5071391796420295], [-0.3445608603995446,0.8999444931201762,-1.8295489983610813,1.3813011802363127,0.4406110935124823], [-0.4720580057434039,0.9616388382715773,-1.7885815681274932,1.2231064430079939,0.49611588805959794], [-0.6877856894979741,0.9509033529307679,-1.7111111174594673,1.1650825853084499,0.36541463301831684], [-0.41906769746785616,0.9643844813018809,-1.7088669498454399,1.2743442947780539,0.37857583863641653], [-0.45604703784103723,0.9704467329255722,-1.6908561828878124,1.2661005251088149,0.34761517755721416], [-0.45216413072759976,1.125638260528155,-1.8672240017957615,1.388348280279974,0.41289821746105937], [-0.6041998663111035,0.9518745991902821,-1.7482553616491074,1.3027968384522153,0.33003148862622383], [-0.7645915689875407,0.9231306641016475,-1.5908706609397762,1.0416797405804958,0.3435391589642622], [-0.5511129470810242,0.9789516647349409,-1.5896352104287992,1.2985407772348172,0.3074739718251722], [-0.3060111421519473,0.9693226097887153,-1.6503499878201646,1.2050435546957032,0.20529101796457794], [-0.41814813350738833,0.9065345647328998,-1.7369684308945867,1.3095490239413528,0.3931707179477215], [-0.7452070526618801,1.010695258373889,-1.6799990639878284,1.351913921189404,0.48041731347418104], [-0.19474072480894772,0.8574730260551575,-1.7114945513022097,1.2484211088395596,0.35791459113273333], [-0.679333385903053,1.136043445571062,-1.6819398713814546,1.2914383599572574,0.4846586154592743], [-0.38963314954900546,1.0499014443531718,-1.7075189569343263,1.3966853721193921,0.4922463621111953], [-0.4771733790096556,0.8615507710844875,-1.7824679274745288,1.1550649309651535,0.4634518570903764], [-0.46851141840829336,0.9490804068188984,-1.6654371185103947,1.1748885058064462,0.362811152170017], [-0.2486779458724137,1.0146748639649252,-1.8129854069448728,1.3198830969564963,0.3848127258680788], [-0.37278156677721497,0.9026389048304428,-1.6338760097656873,1.2146194636725334,0.305407696426058], [-0.5109164214840503,1.1025203353037651,-1.5805383770959172,1.3659800632138837,0.2507490333376023], [-0.42658774618748607,0.9583040463281658,-1.535401873083165,1.2270967800953845,0.30931926334482973], [-0.5761089637758711,0.9608023900436299,-1.6773446541045054,1.3896056325873751,0.3003385242281313], [-0.333657352857868,0.9284640162518879,-1.6506283984425263,1.4233119402331722,0.4817086358456392], [-0.4131260123218083,0.9257004348959492,-1.6385111064595121,1.2553085674173774,0.35994561408421055], [-0.41619483777131644,0.9791552402269297,-1.6621579631275758,1.1639596596963502,0.3356350519347727], [-0.42150886121352676,0.9497809597217938,-1.4493400602428224,1.2037323844449976,0.28412005647965366], [-0.4036704221590921,0.8682473056100086,-1.4499044130725358,1.4367312949083602,0.29801296100779273], [-0.7347169576181178,0.8472443819718873,-1.6912964247755202,1.1098267823656842,0.5128488081575056], [-0.4599723079910277,1.0313773063743503,-1.6693647567489862,1.1183374503284873,0.3065899617042307], [-0.5155727886799809,0.9793287424722477,-1.81766244647388,1.264051537384318,0.38002468301378506], [-0.461809330593855,0.94300958630738,-1.7083227759840551,1.2779916838408791,0.5269758739704866], [-0.6518981010977536,0.8745407308399856,-1.5963574332371322,1.2679054180945675,0.4099968212180263], [-0.46480686752956457,0.9217771428753339,-1.5266876398223685,1.1382426745675795,0.4778996312070509], [-0.4852820635337406,0.8218636910073019,-1.7767449598018135,1.1371130292343117,0.33381426065377506], [-0.41472873571993296,1.1622114863170214,-1.464996886657917,1.3123782230466363,0.2979393535788235], [-0.38817680425075585,1.0268858467994189,-1.5893492622912915,1.2621731986065745,0.435168936860357], [-0.4138872105825818,1.0639679744840822,-1.6793705616475731,1.0262313606697269,0.26542788201223744], [-0.32689002022123786,0.9411685364764196,-1.6079547828400877,1.222649045231373,0.44202772328420886], [-0.4534191990015764,0.9019382096024549,-1.6042217752367725,1.313476911051435,0.3118812623960018], [-0.3642257288812764,1.0568954661740726,-1.6408419112357935,1.3879191143662102,0.43428969659022393], [-0.4436033491575346,0.951120701402773,-1.7396961236381308,1.0696513684387006,0.1983721283216616], [-0.4195531379041837,0.9339655248016878,-1.6369085623469266,1.2113780206799198,0.32646374286948043]) -2016-06-13 20:19:23 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:23 INFO JobScheduler:59 - Added jobs for time 2000 ms -2016-06-13 20:19:23 INFO TestInputStream:59 - Computing RDD for time 3000 ms -2016-06-13 20:19:23 INFO TestInputStream:59 - Created RDD 13 with Vector([-0.3676131534931217,0.7205292433695121,-1.6148690586461516,1.318613134832133,0.2894000707552024], [-0.63227225792052,1.0480160164806966,-1.7186123125513257,1.3123133303248091,0.4835019323786134], [-0.5130669794355894,1.0744409567693907,-1.5423474727562314,1.4075951590487494,0.34400766625670576], [-0.6214787708363003,0.9941757670924664,-1.7059517928152113,1.1668792946674045,0.5351190865534026], [-0.5610084057337338,0.7736203857280138,-1.8315479385898956,1.3209260735925115,0.28582121198324356], [-0.37376999266386424,0.8253313331048502,-1.7924436246324382,1.3068506619750972,0.2325763177837035], [-0.5087037004414274,0.9607866442106467,-1.812291073591455,1.284458726328898,0.5328516211621773], [-0.3727437082846784,1.028767182687076,-1.7448007272489838,1.0984753757565202,0.3541211398247694], [-0.3814571980697206,1.0819060589675644,-1.733976127510556,1.2515969788209256,0.3640947425626813], [-0.38693583578076807,0.9054649043220095,-1.601255604158471,1.1869423507487187,0.27285330937505237], [-0.453879517976784,0.8265450932182703,-1.696264660015251,1.1219045234195482,0.3528336963815211], [-0.6440396833150053,0.9147072341447406,-1.574634779931242,1.2807375427757628,0.3400101945021918], [-0.5574376128841423,0.9786892277605028,-1.6539643457077906,1.2987195892477945,0.22091568760798275], [-0.3219564568616685,0.7344099605460779,-1.659291949575684,1.3277750279213558,0.23553171253321456], [-0.42287205017678026,0.8870628278654091,-1.7860892614211916,1.3638413188637932,0.35992654268460605], [-0.5114214572125407,1.0739488843585667,-1.803218590871721,1.2580620741259871,0.42523338095005675], [-0.49124237559267386,0.9657816736108445,-1.6156494419042913,1.3063855681095915,0.54155197550267], [-0.3951919316276779,1.144136446459301,-1.6606072772152598,1.4114586343482727,0.3224073809014021], [-0.439470602003815,1.0070997021461388,-1.4975209021956797,1.2890203902600919,0.22540217793727096], [-0.6679727959214776,0.999190248905179,-1.635890493454145,1.2938379558735829,0.20736771134970797], [-0.4118317638697319,0.9069236714993869,-1.7766219570089812,1.2254241772422947,0.3818774093786991], [-0.5019285203332016,1.071358441302261,-1.8681705848296024,1.3543938347082431,0.3694102461708769], [-0.43086686887184694,0.9871654497649167,-1.592715663423245,1.1292659913487786,0.24090456644129066], [-0.6554222411786117,0.9775977489456485,-1.7739871948427586,1.3756970876324615,0.5345682469155936], [-0.606670169993634,1.0414671201540915,-1.5816535152099802,1.2849832898961777,0.3443771799618759], [-0.6330347409363863,0.7614040653882075,-1.6420712005053097,1.344034328413472,0.5019032888808933], [-0.5268140556262974,1.1016458967835785,-1.581552701703922,1.1842999784487596,0.3683353834757425], [-0.49900894372274485,1.0645644739185587,-1.5355753177851752,1.2694091849907172,0.33468156327241705], [-0.31358398021960576,0.8302665811856854,-1.5422704772273437,1.1506707176948,0.37999408831157194], [-0.557673046622542,1.0913399597085338,-1.5871539417007638,1.0478537277843192,0.31421916981662246], [-0.4100320104849945,1.0606699393377843,-1.764930551709285,1.3173125309085043,0.32112514544554777], [-0.2578093453260759,0.9604640936063775,-1.6498739501347253,1.251662741314794,0.5352359752706277], [-0.5830353039132012,0.9147683830431232,-1.747368848835465,1.1923888734167842,0.4792523557272257], [-0.3091820573873779,1.0228642342621612,-1.472857793608509,1.165376819972311,0.3758448993415674], [-0.371190450361484,0.855956409330075,-1.82995473329898,1.3390818705810552,0.37113293274981823], [-0.5539807557276546,0.9677998945369973,-1.6350804443189515,1.3419255190805948,0.44899955987573437], [-0.5376287808107136,0.8413279828562651,-1.6762865574706214,1.19949852014792,0.5047576712678563], [-0.6003249423212255,0.9906964773703084,-1.5370089708944512,1.3072479529851824,0.430528835393197], [-0.4175368422724912,0.8187481538877078,-1.7996629160543502,1.1879999792808238,0.473510867372402], [-0.4828257087231799,1.0612823363596418,-1.6425767546040255,1.2270450796716112,0.39748339044894354], [-0.6332333955565639,0.9519761064158869,-1.5479445620477186,1.308465915443944,0.40088093604627134], [-0.5228584674134156,0.8768992895766171,-1.8336272924164618,1.1499724885170817,0.48668761341650246], [-0.36341836902693225,0.922366126760111,-1.4327493627862522,1.255336229334989,0.29637579738564857], [-0.45752242402693893,0.881182369510346,-1.8075367810499625,1.1836209711736738,0.5240597883932867], [-0.45127422400043055,0.7875661584033756,-1.9374943002165643,1.3022768816466794,0.3865261883663946], [-0.3385546242930144,0.8796112795269294,-1.6285471480854194,1.177302672296305,0.3611750358009876], [-0.3866867752880084,0.9445769802243262,-1.63589592788523,1.3115437594462822,0.30025415854590604], [-0.35087047853321573,0.8048561114260069,-1.6540296199948705,1.1069576251024358,0.3739108460295009], [-0.6118021930146711,1.0836229312504084,-1.6965417516081935,1.3814661848764345,0.5544639396063888], [-0.32579850447352154,0.9627503718472757,-1.7118868486570202,1.3378056459968657,0.26239820187294716]) -2016-06-13 20:19:23 INFO JobScheduler:59 - Added jobs for time 3000 ms -2016-06-13 20:19:23 INFO TestInputStream:59 - Computing RDD for time 4000 ms -2016-06-13 20:19:23 INFO TestInputStream:59 - Created RDD 19 with Vector([-0.4302807096972894,0.8599904088505619,-1.74473044235192,1.4559107913903748,0.3868065070182458], [-0.4809066717988696,1.1355083469098752,-1.568604721573235,1.1620770786476713,0.5052691040344791], [-0.37948367495671453,0.9612314328384234,-1.7288918432476479,1.1900277125124938,0.584148529955661], [-0.4830118079148656,0.767521114064722,-1.626523514538083,1.2344387914004133,0.2949434953618521], [-0.45982168221474795,0.9610478654584301,-1.4181098052958898,1.1960502956938037,0.35754207882888805], [-0.5403734624385554,1.0961985914256847,-1.4824794478647174,1.1331480033700656,0.4391107899439444], [-0.4078934785304544,0.9024846949827434,-1.772449379651882,1.1891755675558509,0.26886721197216756], [-0.4850713563393695,0.9854430300369302,-1.6392713106609738,1.1640485301086894,0.4058728213551233], [-0.5687863647866291,0.8969696803234791,-1.694332984322472,1.263598085578097,0.4916946976192132], [-0.4852057062952629,0.9746523770852169,-1.7152410694648534,1.290070394136892,0.15734923086759972], [-0.6042639508376677,0.9587258437320635,-1.796821218335353,1.2415150801911263,0.5036349732580911], [-0.5634160393957598,1.000927422915143,-1.7677443118596388,1.1056602828501068,0.3662560181172587], [-0.42260218853055576,1.038402090735988,-1.623454196568306,1.258759507363921,0.35817719144396376], [-0.6935865876752814,0.9767485891098648,-1.6119749119230544,1.3241871653009805,0.45959708474651645], [-0.34982681977824615,1.0697929536346964,-1.6430495277049175,1.5160135223968076,0.3720168274191172], [-0.4965183720272103,0.8867675212850072,-1.6326283681810188,1.1014099972707194,0.3464777829375711], [-0.4429523196454975,0.9569586297887442,-1.6975710388611154,1.3240706523955248,0.32888337316923105], [-0.6110986163387908,0.8513837967033338,-1.863122720264613,1.1805944382455067,0.36642329779978594], [-0.5770973903898762,0.908469634420286,-1.5969552518736576,1.2639442528889475,0.30334073173665643], [-0.6677676499752639,0.9993894428543435,-1.7484246181717757,1.3024629995033956,0.5254188589994735], [-0.43541597877213595,0.9518339300904892,-1.586032231723223,1.2859374872876397,0.3728538289653893], [-0.3981355023125287,1.0914065768869097,-1.604054951142783,1.5597686669581905,0.3200687403051776], [-0.3526167305034049,0.8582438505591754,-1.795079875034891,1.291753863011317,0.5193327226327783], [-0.5322224604485944,0.9225091285363366,-1.5406535924270095,1.1740734779955952,0.3985082190639408], [-0.459298613922177,0.8499917798311437,-1.7381604837279123,1.3363117012403978,0.39372367283850573], [-0.5029856763543168,0.9563691025906603,-1.6323146191672946,1.1564017424707789,0.2992277161858537], [-0.4702645909815308,0.9857893178047549,-1.6146159696178013,1.2941994476820615,0.29510677438190996], [-0.5010373585673528,1.0381084928077213,-1.6592946006629066,1.2275293781382213,0.6215806870134133], [-0.34251104673790145,1.0920152105960284,-1.4702129754795525,1.3052389503227437,0.5142842632664187], [-0.6014879705809496,1.0573110467882678,-1.6994508095500127,1.2345579924787826,0.27260709575033387], [-0.374393054126274,0.8520958347461538,-1.6227534538993043,1.400050919842349,0.3749155714488258], [-0.5537306132792297,1.0237199970470363,-1.5996513021748293,1.3277912515047763,0.3298858295939718], [-0.25019038644507985,0.8933568126093641,-1.6180540579171452,1.2514548524704747,0.4485167299028402], [-0.4083941282128968,0.9821808764315939,-1.7710803507794117,1.22246902401698,0.39616717157783654], [-0.6617567254520433,1.135205204672421,-1.7404704566578575,1.2634534028343156,0.48631393245586374], [-0.5102944111175142,0.7749069961072899,-1.3746153795885292,1.1761695768038691,0.44519535098184526], [-0.5273802259607313,1.0293844883271042,-1.8327600208787889,1.3032607095759106,0.4810318980867238], [-0.43680250540361804,0.818212719795147,-1.6397291545525832,1.2164204417976607,0.36911234859465697], [-0.44511271834694277,1.0039279649012336,-1.736482991573775,1.1407721179472239,0.32026695790267784], [-0.461571293170463,0.9623234284494812,-1.608651111923442,1.3054108488252942,0.2808744029932673], [-0.44626644352032524,0.9385912062499339,-1.5784088711897624,1.2996894478753678,0.3367841237735643], [-0.3458002275610911,1.0002670117410046,-1.556743986487141,1.220789398824709,0.3047413342602086], [-0.7006173067436341,1.0226530560056757,-1.7453818996615391,1.2468257997531524,0.43730757861422237], [-0.4921877365188532,0.9365903329931373,-1.6954478906187997,1.252660426269991,0.411305000260049], [-0.5866727516355298,0.8540617355784135,-1.7495419546675461,1.1992666379768737,0.5624798329629086], [-0.4552502733367031,0.9291860448712651,-1.5908193525700438,1.166718738832733,0.3281256456654895], [-0.3426613548769225,0.7538960431270546,-1.585524926597853,1.478740164978157,0.251131880834575], [-0.4680826818575833,1.1112444944704614,-1.6587653968723466,1.2870287092936261,0.4050920708933554], [-0.5624024717932173,0.880862871462712,-1.6303493150007178,1.1721463768303386,0.5696913393311782], [-0.48266079841446297,0.8916088275950524,-1.5393517554424276,1.1897790454674049,0.4411535715241579]) -2016-06-13 20:19:24 INFO JobScheduler:59 - Added jobs for time 4000 ms -2016-06-13 20:19:24 INFO TestInputStream:59 - Computing RDD for time 5000 ms -2016-06-13 20:19:24 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:24 INFO TestInputStream:59 - Created RDD 26 with Vector([-0.3816421005537525,1.0617035428188992,-1.7399461177422264,1.332980235806715,0.2916494841747297], [-0.6583584440016957,0.9555891409969888,-1.5454326525102315,1.1741091058996926,0.305220337835565], [-0.2989177604038451,0.8701688291879833,-1.8804582484676076,1.2986162711936382,0.35958131004690164], [-0.4939772883087695,0.9969220817446861,-1.5980382534286326,1.2969252872376784,0.37976918116852443], [-0.35668565131454827,0.8078147339434413,-1.657193160993533,1.359095599717262,0.31192105888186633], [-0.5450865703006346,0.8750391278919141,-1.7453306059026725,1.3721329076982913,0.42237633518141116], [-0.4917580165015238,0.9512146417235625,-1.9326523267650781,1.4266827266379667,0.3225713722197544], [-0.4048718474607664,1.0240256182146898,-1.6774401089659943,1.2669846165410654,0.33732641244233585], [-0.47579982131353826,0.7947369494624074,-1.7655419119652316,1.082577012553155,0.35401987740372537], [-0.35538929020910864,0.965963799724578,-1.7529404087985827,1.2055043918858341,0.3342486683177578], [-0.7014541659835856,1.034642549879647,-1.5544540159626516,1.3033886639865033,0.4054677613509847], [-0.43610110990224626,0.967077951705683,-1.5262368892289788,1.2639835242537958,0.3181179758885862], [-0.3425524324739105,0.9894095680520077,-1.7777745234498326,1.3091660388427224,0.331841648562285], [-0.4878205400579338,0.8560103963724833,-1.7352326850283464,1.3664274889095067,0.3984111315135807], [-0.7121859062829006,0.9130040407010276,-1.7874802517046422,1.1874648639293004,0.45017886399842666], [-0.696643078139105,1.1467663562988957,-1.7296410056290408,1.296144064682525,0.5048605187772839], [-0.4011230886898419,0.7361859073107719,-1.5317733987729902,1.2701183330425934,0.46062351610078545], [-0.45715694326129586,0.9190057703920946,-1.6904193500197242,1.3271044644288563,0.4842403797714084], [-0.5095249256026796,1.0659631362797184,-1.4402626928522713,1.3379987286264212,0.2673039786797317], [-0.4167094825032959,1.1091402127599272,-1.705921687753808,1.1183411370664753,0.48305727426817385], [-0.2924464534867928,0.8213969946455459,-1.662853823551024,1.3355427142685694,0.3770025002372693], [-0.4444725119583886,1.1046045679759258,-1.456164663000235,1.227847061522124,0.5255642346814823], [-0.4048329834921757,0.9788094275209757,-1.539852245269681,1.180160780953293,0.6787253334538687], [-0.38022990049474503,1.0144858953551337,-1.8072904612189131,1.3419776986257266,0.4041658441938728], [-0.3763555278351052,0.8998086161617835,-1.705061140148122,1.3495005676173746,0.5191631674639813], [-0.5540229100290982,1.0669438671830174,-1.636684794385797,1.3459012222977915,0.4181134505939225], [-0.45042550789662783,1.113240852339999,-1.486260571622101,1.2719922983443672,0.4102349177516307], [-0.5546179681493353,0.9938406825700987,-1.7084301934096207,1.3880541591389144,0.29690506797892435], [-0.5006405047598609,1.090292575906863,-1.7104271793907637,1.3062089077771892,0.47556018075456363], [-0.36520205819569046,0.9201397468605166,-1.7602021590969437,1.3876082296931265,0.2308718010458195], [-0.4938168714972593,1.0964903470900649,-1.7260349751744855,1.2048249569339422,0.40946925790527117], [-0.5092411838541296,0.9882559808204627,-1.7212908932580275,1.1127850487389395,0.3940121165600333], [-0.5882412062939182,1.1156935762589453,-1.8131619026625396,1.4146422480126442,0.31361816553861704], [-0.46990162437316874,0.9908156946435108,-1.748827835513362,1.3957588501434537,0.5512480921903505], [-0.5405868391855275,0.929247220281523,-1.5894550246414005,1.271255300937136,0.46529614772048844], [-0.46376208782353645,0.8174067828364345,-1.711422288030576,1.150388298327176,0.36268778857829526], [-0.4617415170232849,1.1203248336099854,-1.9346820049945674,1.1466858815338403,0.318614186382022], [-0.5200553635156823,1.1206383922863123,-1.721393160577411,1.2564342218266722,0.39290287510750027], [-0.45474634000888675,0.9538310968737261,-1.5572305892194418,1.2293659662686987,0.3951581444726625], [-0.31024997329419923,0.9626097763501761,-1.5774105560033171,1.3323705378196256,0.28213579823419327], [-0.6425546702182562,1.0413861558054223,-1.6696029184789263,1.3297020370998587,0.4667426127538354], [-0.5088483797878551,1.2170009632967245,-1.7143214767417212,1.2915402658283741,0.27813229259982863], [-0.5138048643277845,1.0537687232096613,-1.5306968187748822,1.2020601237200264,0.27213168784492703], [-0.5375432544160766,1.0418784698921373,-1.501800504998784,1.217708599285121,0.6443267111319482], [-0.5784571285851817,0.9518620188834632,-1.656094482882889,1.2785669143560554,0.34103467230048085], [-0.39057405488844044,0.995936983645342,-1.6323671359433234,1.271155031008547,0.3558239098317178], [-0.3682240577265206,0.8711375369867465,-1.6625627384453794,1.1969170451325948,0.47233733398400285], [-0.446008720311628,0.8976699326654971,-1.5289286019943875,1.2244855320582673,0.2984778353929738], [-0.5102538395816835,0.9986885141123109,-1.6414035100650266,1.2420020420446427,0.36032207135032196], [-0.6350165259371148,1.0900186871655364,-1.7907186442371252,1.4304360117451347,0.35387005412887984]) -2016-06-13 20:19:24 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:19:24 INFO JobScheduler:59 - Added jobs for time 5000 ms -2016-06-13 20:19:24 INFO TestInputStream:59 - Computing RDD for time 6000 ms -2016-06-13 20:19:24 INFO TestInputStream:59 - Created RDD 32 with Vector([-0.5768797964115748,1.028505413551091,-1.7521414076692798,1.410931775189879,0.30527264662469095], [-0.40591177777424625,0.8661878776717759,-1.7310200236861213,1.3594055750354543,0.3299548132445285], [-0.6383561683498007,0.9259033203935216,-1.6881320011035645,1.3172994752602565,0.3214908371039436], [-0.624062828319168,1.1219523756569185,-1.5603263506923595,1.228964619537528,0.35487057209905903], [-0.4734064808910865,0.9649955190976767,-1.6068693471541338,1.3196307767213002,0.37116726129033706], [-0.6101841770121739,1.059654359501757,-1.8099667264246553,1.3759685981009824,0.5438261473052933], [-0.18850356552447423,1.1440264277756391,-1.724865192544724,1.1359044353122627,0.31757393897273056], [-0.41254200355821263,0.8607204799101216,-1.616620448535661,1.3427363220425623,0.4109728277211949], [-0.393734447592108,1.057690953958827,-1.6695747980566427,1.327034390345423,0.13819106599587025], [-0.32090427274943434,1.0651687359751123,-1.6980368198411557,1.4017902860954718,0.36022562482340503], [-0.6053061980873911,1.0041632375635317,-1.7820663797212062,1.2269162643461793,0.4840114520476853], [-0.27268808986968307,1.013170380627259,-1.6005981535949472,1.4382064950763866,0.3892224005152082], [-0.535422550377724,0.8375173015602825,-1.661788348017347,1.3227481832299182,0.311533159171815], [-0.4009368458319039,0.9897832871295871,-1.6696688300152116,1.4424037410980737,0.32734398317968916], [-0.6825087101818776,0.9056073621397804,-1.659854864696752,1.2759644672519115,0.43884955280528926], [-0.43179250129551383,0.7465728726494555,-1.5636729803345486,1.3457117728614754,0.5721971492726551], [-0.526304418227463,1.0501002903902692,-1.5968858098432406,1.2982785754583357,0.37494021304786934], [-0.4320225453412649,1.1715904807145918,-1.5178148547272063,1.348200261386646,0.40779395798035983], [-0.42221995667102147,0.87623418086733,-1.73837906100808,1.4183919259975142,0.33228674704284467], [-0.5629520941212462,0.944212875923361,-1.5546071736198348,1.4519582067498829,0.262200899044119], [-0.30845271114048184,0.9681639591372486,-1.688842111249905,1.3763468548667668,0.3173125681909059], [-0.5198567190391367,0.9335053916544206,-1.509472274649017,1.454415814887731,0.4589687217941236], [-0.5711775989363378,1.0215061238027188,-1.6224000734110398,1.350665861834858,0.36668462127595286], [-0.39949682119974983,1.0408092246134175,-1.6587225284400737,1.301865863792736,0.4563742176523199], [-0.4902566261487769,1.0992586451801099,-1.4473476422759635,1.2309123590548212,0.2544829822903507], [-0.5293628624912633,0.8432327849564988,-1.7669603672683885,1.2911210280012069,0.41983427336223667], [-0.5738614502843032,0.9784937768658387,-1.7194271904022407,1.3116656980237082,0.46218317857671776], [-0.5902501988138692,0.9423215775018176,-1.684040151093259,1.4235274237923208,0.2039286490216381], [-0.4777727934676478,1.027586212069216,-1.7495753442806758,1.2757769414973703,0.3810309831295745], [-0.6566621783684151,0.9641681686268665,-1.5286420868141977,1.508411206136107,0.2501550388463756], [-0.46296281092785885,0.9727194845614605,-1.6171422039154086,1.3434828829348795,0.07809723939214058], [-0.5738547760098169,1.1077074999997352,-1.739110700752265,1.3494171899976157,0.3222123114107347], [-0.4002887849620737,0.9619411680270862,-1.663509769541054,1.2908149710547212,0.4244227376717664], [-0.4665533215112465,0.8596395857982323,-1.7467795346659782,1.251224446006115,0.42794754034954735], [-0.6374258626266393,1.0514903728245393,-1.651303971310752,1.236884794719695,0.5249538337308562], [-0.48340361610998744,0.8198136168919055,-1.690727257833199,1.398396991016531,0.31841739933411595], [-0.5410633313507479,0.8973900953721683,-1.4724867408929736,1.388151513303225,0.47893645155994446], [-0.3039096405457719,0.9155005236200422,-1.7173365893442798,1.256097860517092,0.24819750391337136], [-0.7090962179115853,0.8944749271173439,-1.8839838110448957,1.1446445350131944,0.22840975233443309], [-0.5100953498526793,0.8107010460411471,-1.7303477866873604,1.1647356509362308,0.4418371093135496], [-0.2962087734742913,0.7242125014742148,-1.5984563979400566,1.0588259317312654,0.28436170748337264], [-0.5505122810716748,0.7806970076803257,-1.72088933624607,1.4230022424760869,0.32673184311928194], [-0.45127942352855155,0.9959060503010477,-1.5524186364009327,1.2366972856510523,0.3578155222046753], [-0.5102066277582917,1.0949182323987419,-1.7142222646495175,1.1664611293679565,0.3602965879997574], [-0.20421136622466618,0.9866371147144095,-1.7538283133815977,1.0465139384173783,0.41274915410898083], [-0.3678643518086112,1.1128928060624481,-1.7416958328862653,1.1712943420281356,0.522586652590277], [-0.4272255324098842,0.9505411417476373,-1.5398631578697235,1.1670050581250606,0.18826195051488878], [-0.6294443147944699,1.0182211296265207,-1.6114292740208946,1.5494854908967275,0.2354819914065145], [-0.46052379116984477,0.8965751438720174,-1.6887957613207942,1.2678155037585463,0.3277249457164221], [-0.4661555388594827,1.0176554636610373,-1.660202673672577,1.1543960943838145,0.41490582984943747]) -2016-06-13 20:19:24 INFO JobScheduler:59 - Added jobs for time 6000 ms -2016-06-13 20:19:24 INFO DAGScheduler:59 - Registering RDD 7 (apply at Transformer.scala:22) -2016-06-13 20:19:24 INFO TestInputStream:59 - Computing RDD for time 7000 ms -2016-06-13 20:19:24 INFO DAGScheduler:59 - Got job 0 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:19:24 INFO DAGScheduler:59 - Final stage: ResultStage 1(apply at Transformer.scala:22) -2016-06-13 20:19:24 INFO TestInputStream:59 - Created RDD 38 with Vector([-0.5184134212251081,0.9455318561030377,-1.7249072888539423,1.2476376390078272,0.539371371249466], [-0.4839353019153916,0.992343730650464,-1.4037670310964987,1.3650053852483648,0.287928402259341], [-0.45217444521287575,1.0425364789095595,-1.7771813152553952,1.2241390773159404,0.3053067128704871], [-0.34943560877945984,1.0407179160044182,-1.5177836780575653,0.9824846085205396,0.2603543256712868], [-0.45829448575134074,0.9725249117979422,-1.690056167350807,1.035512584568622,0.3026754916713263], [-0.42458145588018753,0.7854084174090814,-1.7649589009106188,1.1975914903392633,0.6551885852829413], [-0.5104497237497339,1.0402201914534017,-1.6859518565969145,1.1391556996625576,0.18881595045601088], [-0.5588856532488075,0.8709355475677711,-1.4735465340859104,1.207362168238276,0.5886898848511445], [-0.5286914346418723,0.9318873502365327,-1.654993790470044,1.274056873049218,0.29412525529417294], [-0.5420811906362676,0.9995645356070102,-1.6040227637757893,1.3412654820039975,0.3950459929854693], [-0.5105371229433685,0.8870982484234274,-1.6366208840493932,1.344244897498087,0.2842199976348651], [-0.5758159328886613,0.9361950961962593,-1.5151431024376867,1.2963264217469896,0.4195802203163277], [-0.46650906956232724,1.0883479790237522,-1.6209503836486092,1.2506119015001946,0.5239087792157935], [-0.49800785133272085,0.9121648111767798,-1.5664537098618618,1.4303891258276538,0.23796160092517643], [-0.6200401622139905,0.8819930763699753,-1.7712553265237971,1.144151394858828,0.37228012783293096], [-0.43683215654432145,0.8386127968983885,-1.6402554451670102,1.129209493050289,0.3612473529185551], [-0.4079221564079838,0.9656864104202829,-1.7359335721136713,1.4421338682074127,0.3616284206434529], [-0.43442591279896603,0.955449363348641,-1.5334677832140913,1.1666726886559795,0.5058368495019173], [-0.6457828019918121,0.931131898174338,-1.8532037937152797,1.0375316080434018,0.3133563527606291], [-0.38369598973754027,1.040856056086344,-1.7594025439463896,1.4917751006832976,0.45495744983872133], [-0.30166559339288085,1.0342180102112146,-1.5650152370405155,1.2673629485315197,0.4158807517084573], [-0.28580825731967757,0.8865395334100423,-1.4737235251638032,1.245008984525024,0.36435255784505255], [-0.34071421524008416,0.9427461052254856,-1.6571555985762556,1.116205755573728,0.3440594901972376], [-0.5707704618700846,0.9955277694976673,-1.815098893846346,1.2976867834380266,0.3708933869585827], [-0.6095164341318,0.9322989548597498,-1.6576644540187038,1.3001089252401639,0.32781904617649943], [-0.5884784232045771,0.9963121018549773,-1.4726908596796255,1.1232169328419297,0.466258074909203], [-0.6351651621700529,1.0173421935079714,-1.7981429560407272,1.0336059974520695,0.28162584897435694], [-0.38203401021631267,0.9790771913538491,-1.7244683271662424,1.288084387752797,0.4312827591331814], [-0.37183909143628785,1.1926117173870312,-1.5166716166223042,1.1696263300990144,0.4515586032102953], [-0.47462520704983,1.0498070539706301,-1.6863851371590932,1.2965114835042095,0.4645837570114517], [-0.35874130630231327,1.0042819076442355,-1.6407835501913508,1.214144240421076,0.35534792382876684], [-0.41389636789198914,0.9508170625312302,-1.6498089027313338,1.4272197180381103,0.374553819362672], [-0.32132785261092967,0.8535769504185499,-1.672835438070241,0.9421132176631009,0.24740658936307364], [-0.48253792736985773,1.0429236941137325,-1.4939460450129358,1.2832409419178106,0.40323476275374437], [-0.391512958705594,0.8859909853510953,-1.607045573914962,1.1770226037327562,0.5198479967492], [-0.4833163026675927,0.9200058086737815,-1.8188555219632765,1.2111735788924018,0.39618714901430313], [-0.2913135273832381,0.9301485161935364,-1.6975841978124846,1.4065399033140502,0.28453881400648173], [-0.3073756680829076,0.8325242716417236,-1.6753209327630654,1.4075134801045086,0.30351866480787215], [-0.2775051790445552,0.9452680429408632,-1.6224110577930677,1.1778711923142833,0.5531745712889398], [-0.3047686297259775,0.9468967188067203,-1.6929112749188848,1.4636396431851675,0.3586449109599954], [-0.5956899619218391,0.813236818549011,-1.757649523640403,1.2604654777105053,0.3185144577112101], [-0.32794499152459505,0.8140339815697406,-1.8666906537843855,1.2572887839007447,0.23549894606903882], [-0.40686690942678344,0.8619998816707273,-1.6319579560604098,1.3664940728963186,0.422156696202466], [-0.4138601150243667,1.021828823586237,-1.54841612428864,1.2398622089963618,0.5070042357490803], [-0.331869975870002,0.8782404384527787,-1.6635875859844171,1.2435143760020457,0.32101906837863314], [-0.49700214293681766,0.9856184010338479,-1.8232422669575907,1.2421051455581202,0.47592591320323163], [-0.535909591266969,0.9807383109826389,-1.6974325972167972,1.334548321075211,0.23946350824403165], [-0.6444021824207316,1.1849780210499694,-1.5121795557147575,1.3604386920675193,0.31690181205044465], [-0.49746220541003966,1.0643703042656938,-1.8499798436118189,1.1357972819777942,0.5780379400362996], [-0.4383285608503117,1.0163805234573544,-1.9030246206122645,1.438368352860807,0.3293850463477472]) -2016-06-13 20:19:24 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 20:19:24 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 20:19:24 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[7] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:19:24 INFO JobScheduler:59 - Added jobs for time 7000 ms -2016-06-13 20:19:24 INFO TestInputStream:59 - Computing RDD for time 8000 ms -2016-06-13 20:19:24 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:24 INFO TestInputStream:59 - Created RDD 44 with Vector([-0.374705866875181,0.9840691262184086,-1.5891551104970965,1.4814595125988181,0.47065837960007695], [-0.288810885883263,0.8437842751113669,-1.5395954061634287,1.1874811761270259,0.021249409500126437], [-0.4606335450965845,1.1012472510905122,-1.8287426699010534,1.24172318938614,0.4710333368456244], [-0.42944940507321583,1.0104988720375858,-1.5751317017658801,1.2831547194950401,0.3512701703197814], [-0.5152127056418667,1.1709583832128079,-1.6554391510043867,1.1397001494774806,0.23397277605773995], [-0.4399479870409779,1.024335104965922,-1.7045108897162344,1.196833551305392,0.4117115230011916], [-0.5430543625505585,1.1370095574712402,-1.7176429060334013,1.1901279759925694,0.46070443041212095], [-0.3968714897698767,1.0926860330748331,-1.6334286394688688,1.228234808051336,0.4665607383628074], [-0.46215342653947683,1.0343329141616782,-1.6499943761334228,1.337530812749641,0.23114556003826187], [-0.5000235665427208,0.9151728338053156,-1.8055132514640995,1.3037686620331144,0.2902025184248946], [-0.309233049825866,0.9728959274674948,-1.6392691162173578,1.101247705752869,0.3496423465401415], [-0.5981691090699208,0.8739296001671861,-1.5687282698937923,1.2550952881121755,0.46308264317451964], [-0.425561195913103,1.0676958602044149,-1.7135165364767921,1.3524692002054561,0.23228520033456318], [-0.7043177723021486,1.0283311625585259,-1.7478644671187442,1.2858120358708933,0.3418710936510813], [-0.41727892194293686,0.8625867727705405,-1.7914100057674134,1.1786792695315231,0.4917512427338196], [-0.40174243931692544,0.7828098802704966,-1.5073762714652954,1.281118669072956,0.44887930601066905], [-0.35310979651234775,0.9745364825956087,-1.6392317059509176,1.4513644518189859,0.3110119088453469], [-0.6853829628094648,1.1253397147829511,-1.4920081564639467,1.231135544811062,0.3139314829419366], [-0.44152016739085975,1.1304790288137783,-1.7325115907360644,1.3251089521914843,0.479567851123668], [-0.5526275004039871,0.9674440446022308,-1.7971453051937036,1.1664342514118335,0.31260632993291754], [-0.46562594489923503,0.96412818249357,-1.7756506925990838,1.1820367323525278,0.37368762092821634], [-0.42414895043860573,0.9369437276073684,-1.6228073393039317,1.2110129935336054,0.33600782254298567], [-0.4507513844294476,1.008422902581697,-1.6947838717172434,1.1258385535052762,0.23649573041213182], [-0.5011767543824581,1.0367482963168666,-1.8619227324329928,1.187686906059264,0.41060169959227016], [-0.29061733919433663,0.9125980333805904,-1.773384852821433,1.2318813347419482,0.5360023941871213], [-0.47558633395179245,1.0112018987992073,-1.6443927870523196,1.1868477340320063,0.378412637239359], [-0.47217857076866115,0.9027887018339104,-1.7359968088612976,1.372505255254142,0.4024631611010435], [-0.5635782393384723,1.0418520168542464,-1.6371483392749435,1.3046794831870752,0.36435743882876215], [-0.3566584096699375,0.9045378099174617,-1.6027843810019622,1.308376882406936,0.450420525347206], [-0.5161548558091542,0.9362466126941537,-1.543853038663444,1.351176902637456,0.37019671664199505], [-0.37734135089435306,1.1354760893808167,-1.5928707071728756,1.4230797310043477,0.5395830646701685], [-0.4935941975274797,0.9431897551602951,-1.7363999282764386,1.2592777434544042,0.34477211805728875], [-0.5399690948357754,1.0773952946269298,-1.623286131549866,1.3571017009557331,0.3820149575212495], [-0.47470272833792704,1.0279092716939924,-1.5583033476707477,1.2928147173688327,0.12854158780432537], [-0.3932748299826361,0.9168968642381998,-1.901294092833649,1.259188794957511,0.4011688713714673], [-0.3557493574970649,1.0003577059585267,-1.5111904842067994,1.3123818859934535,0.3019665980708428], [-0.38849198387448786,1.0634537152100978,-1.8742015500852314,1.207616787308903,0.527158124827907], [-0.32516837051884395,1.0806577326663074,-1.611781542302308,1.182593379846847,0.5196789294606137], [-0.5212725898858087,0.998490506190397,-1.6184763965153202,1.3731447165706872,0.29855378648402087], [-0.40257752034798155,0.8314340557777532,-1.6850156321125218,1.3427297403898506,0.5156164947536159], [-0.4889473969173199,0.8999745071099362,-1.4468367122261747,1.3090812041423647,0.5108182965913203], [-0.48274282778056277,1.1537075465803446,-1.5811276311378704,1.287632315416058,0.50882779799236], [-0.23362208323850905,0.9548750423156142,-1.572161725609678,1.6027003014260315,0.3939518404103326], [-0.4292988792888861,0.9366381149644413,-1.8250064321036654,1.2358072562830058,0.18892247590763034], [-0.484748914270206,0.7653236279449509,-1.7486834183984747,1.2949268753185315,0.39683334762547673], [-0.4399736279490563,0.9634644976145054,-1.7480413166448512,1.1775955441636703,0.36332416278682794], [-0.48373350642316953,0.9259530051332495,-1.7977337219360736,1.2876089889322726,0.4497462132788797], [-0.38070426405120866,0.8838236645507052,-1.6753928124899833,1.2449514905082557,0.3276558558550576], [-0.5717833110401351,1.0273299096906752,-1.6190879043151574,1.1860455839092494,0.32636997750947594], [-0.5231330958628938,0.924256801489448,-1.5484246875687981,1.3381528852011608,0.3757180006852602]) -2016-06-13 20:19:24 INFO JobScheduler:59 - Added jobs for time 8000 ms -2016-06-13 20:19:24 INFO TestInputStream:59 - Computing RDD for time 9000 ms -2016-06-13 20:19:24 INFO TestInputStream:59 - Created RDD 50 with Vector([-0.4026578572929527,1.0001062834205203,-1.5617908885948877,1.462733521830009,0.2061287167736916], [-0.3965287353033582,1.0683726645043978,-1.4873730061169563,1.1460792990307642,0.27746130384846085], [-0.4938197206621628,0.7709438014433136,-1.6076436749339549,1.0982028796258234,0.24486596655062928], [-0.4777650011847253,0.9489155740228515,-1.476414143798642,1.3062514135704215,0.6114439757268249], [-0.4285977940802228,0.9382568588433707,-1.48976698594476,1.1698457774900977,0.297712575086562], [-0.5993896749484521,0.8629058534578911,-1.6510698754517374,1.2090117714723978,0.5458042851487852], [-0.5321474831529757,1.0849054231681536,-1.667568850178397,1.4161664775537062,0.4493689535824759], [-0.6723705689897437,0.7517542425048965,-1.663645669684176,1.2240252996527488,0.346848907683753], [-0.40115735788941104,0.8163474979541504,-1.7694648704001712,1.1073947331922183,0.3856322637520626], [-0.4787497898360035,0.855418622108937,-1.7422359123781868,1.2371448524727167,0.5156131237949001], [-0.5321048883779993,0.7970778523677844,-1.7616773188005117,1.3821414318824692,0.48832517040159346], [-0.5285360880098152,1.0597395124384508,-1.6417892576104653,1.3464020747697325,0.31849105791627297], [-0.5324435853239631,0.9841563147023031,-1.5943852209495275,1.2707723748060116,0.4139910360672806], [-0.37531742848811983,0.7874289125205598,-1.6479869931712725,1.3181636612914271,0.3718789043065717], [-0.4560433134330382,1.0759812484577316,-1.697865869705092,1.2725177127861855,0.06883328163775021], [-0.45513942313770217,0.9498671339799809,-1.6933930804659583,1.3025139463978586,0.5350768600248874], [-0.526256522617052,0.8727003678813281,-1.4356120595661468,1.29993914263913,0.43387328603333664], [-0.3923483046111808,0.994060772829843,-1.6451266300509706,1.2983503204985913,0.3703765349262901], [-0.4374913726020475,0.8955789511391181,-1.4728897306445088,1.1357862983779858,0.30881407016500095], [-0.41951697820602707,1.0007695283696842,-1.708286281016387,1.4190975260604537,0.28731659836668877], [-0.6750333217393494,1.0688779823591679,-1.6846746627252691,1.2962992487600882,0.5030936797752793], [-0.5956866906574221,0.8121299274245898,-1.692929629203368,1.2338548732523056,0.38371772482125543], [-0.4303591738987757,0.9459620785706632,-1.884551120494729,1.4937212054647857,0.415792749586519], [-0.45504157760203423,0.9027897906482127,-1.799761299703666,1.3040399621270629,0.5736987408004388], [-0.5593669081347996,1.0931782446321396,-1.627955485174887,1.1391759184235553,0.39801309200504786], [-0.3825955534231694,0.9655263256725013,-1.7068123920171723,1.1861538643513039,0.2764471843224494], [-0.3587313623839552,0.9371265382727668,-1.487908236071683,1.3184799953634567,0.4370396706398445], [-0.475038660091078,0.797460346861311,-1.6835937249660264,1.3120302511529962,0.17031892070731325], [-0.4663190378327658,0.991352295299833,-1.6444316855302834,1.51526349576866,0.3929422369724258], [-0.5040823137977553,1.0575372259159488,-1.594322659684267,1.2971698194040933,0.5092825217956699], [-0.44790417664079346,0.8518362899333143,-1.648211585312696,1.1839806869130056,0.40832085080824754], [-0.537755895075537,0.9806637869508009,-1.7656906916731008,1.2433660326767015,0.42887767735060545], [-0.34293660736335746,0.7935119912222357,-1.6359022010243611,1.3593857228798436,0.3967833959145234], [-0.6208695835786292,0.9640100782092327,-1.874229161549032,1.343981009171891,0.20123531286937832], [-0.6723717817990262,0.9002960054466735,-1.6910864401441592,1.1818255352705958,0.5433986930426408], [-0.4588708699931968,1.2092994554375371,-1.5324768096769932,1.4083576723657105,0.3708605878943178], [-0.560638238383703,0.9092570275383438,-1.473047966472027,1.3497511209299733,0.3467764467476322], [-0.4409803705877342,0.9370238683801353,-1.572432943897284,1.41498798111251,0.2748026192628795], [-0.38741643074715315,1.071718105146961,-1.624146896559199,1.1401522638836572,0.3872910322155703], [-0.5140862086906481,0.8211106116047733,-1.6027037015807561,1.1677747630844688,0.2659604974429464], [-0.6693385873907269,0.7554393139905898,-1.7108087289854104,1.09991932687034,0.41821242261960917], [-0.3282503845126481,0.9257033715775067,-1.5235939065144024,1.1889128563024831,0.3368716615939142], [-0.5366461042406446,0.9240666515119298,-1.8135909127248846,1.3575511869067935,0.37505000938261085], [-0.39294848413804057,1.0124585311856609,-1.575086388417685,1.2467079941490156,0.40416560148386643], [-0.48738816985096806,0.8787565657383682,-1.810281961839409,1.3779890105636297,0.46019825317610896], [-0.36592476270170865,0.9132457437551841,-1.7089939281056865,1.1718159447241387,0.37244792446782193], [-0.40751855185193425,0.9541054981051484,-1.4879650472104635,1.1817057016562598,0.44919665548504983], [-0.29317450049605076,0.964443479319753,-1.656028808081747,1.2586398699632722,0.2526740377633023], [-0.4615968319838038,1.0243528061825597,-1.5054257187951623,1.351376529833804,0.42971544189574107], [-0.47988177538312266,1.0112860256804128,-1.5404476412782737,1.233668528335028,0.2028012939752821]) -2016-06-13 20:19:24 INFO JobScheduler:59 - Added jobs for time 9000 ms -2016-06-13 20:19:24 INFO TestInputStream:59 - Computing RDD for time 10000 ms -2016-06-13 20:19:24 INFO TestInputStream:59 - Created RDD 56 with Vector([-0.4889010904178065,1.0197902904388123,-1.6434083794608938,1.2273894895374848,0.29880056758004836], [-0.3690054860756016,0.9083136794567547,-1.5975355891184535,1.1937534562371308,0.3514272190683493], [-0.5737783702048262,1.2273536117223582,-1.4617006566984774,1.342244921629087,0.3377586288639175], [-0.48387316003666864,0.9673070026559238,-1.716725721249248,1.2214389230026372,0.37779842180031725], [-0.3777277713061831,0.866118273628028,-1.8487356572074127,1.1851350383431707,0.5215534466266145], [-0.6076926717533072,0.922880686625322,-1.7313060678714964,1.3632488420681186,0.3176699282425657], [-0.3802374789807806,0.8439055679755229,-1.6219613687183736,1.1809867291088374,0.4174630121979538], [-0.567466556581487,0.9292300369161763,-1.5542588287823018,1.3568593851021735,0.38458895772840795], [-0.6764387691659441,0.9816754055716941,-1.6652926928722982,1.3324217076474294,0.2844320904428872], [-0.584138124369403,0.9651833514289697,-1.7100579024456408,1.3025648636881102,0.3575320807936941], [-0.45548159931880944,1.0181220363355081,-1.6506653550843506,1.1304474598171783,0.4285968106743953], [-0.5370002084128622,0.927571736048293,-1.7228464440630433,1.2925613214036213,0.3169164091166503], [-0.47996613069291666,0.884366449915093,-1.7212355955272085,1.152597880658151,0.43962468554199663], [-0.35493034949860885,0.8984344817504613,-1.65218132155661,1.3068108975806394,0.354531357717713], [-0.43226118157508875,0.90725353631203,-1.7628881371748841,1.314870897760879,0.40214500076142384], [-0.4387259831220203,0.9621751931555681,-1.7268764217836796,1.1093705777029121,0.17867972251315242], [-0.44765902746822306,1.1186404105474657,-1.750942737318709,1.322673860044468,0.41609066577279374], [-0.40351581789921565,1.046340578817358,-1.7120920286690795,1.244893797129159,0.3833023298379642], [-0.3093624994668177,0.8356650811564297,-1.8642891852543533,1.217524168702485,0.2103617986137151], [-0.2990644638630094,0.9070655034040485,-1.6094469406493193,1.1592980925831848,0.43244854200109223], [-0.5596021456993305,0.9019708087164819,-1.5105356278212536,1.2598551179472526,0.3428536586628008], [-0.583393764986965,0.9748568871770861,-1.5302538126845806,1.301546791122464,0.21927558650218368], [-0.49772324825286274,0.9716299176892979,-1.6169165069063722,1.3531471880552017,0.25858373282508224], [-0.43106771684572104,1.141003137647425,-1.7578104686857339,1.1835903210552496,0.5254045668739125], [-0.5997306192424074,1.0040726704610177,-1.8227253165238155,1.2317004811683654,0.46876771655575034], [-0.38252412898165267,0.6654527000305275,-1.6468024292953505,1.246225339488523,0.3090133585357682], [-0.44821229943324725,1.0038256342470713,-1.6968551076177416,1.4529627079929077,0.3454589917432014], [-0.43535992691090786,1.0110816584854143,-1.7453606983094134,1.248210478232135,0.2891585839680707], [-0.3652348315036368,0.8958964249587966,-1.9606863598829045,1.3389257198065394,0.2618900608154475], [-0.49376528406043635,1.1421599774098976,-1.7157624036331927,1.391772429187865,0.2876879628208894], [-0.4515063147230089,0.9428403433366573,-1.76394465423704,1.203639940182186,0.5058415243772909], [-0.566229150629525,1.0128616873896932,-1.6880365432069786,1.2877554523210368,0.1629872361106314], [-0.47658344463796903,1.0862736976313303,-1.7934284966158631,1.2272771650263852,0.41680211738171946], [-0.3618808351944353,0.9417137306142976,-1.8600076455984111,1.2206352320840592,0.3461749973804146], [-0.6214757024643818,0.9952948941676851,-1.62097329834717,1.3958184351557885,0.46499512939060494], [-0.35996348236796494,0.9954644862470238,-1.7933270864489543,1.4005533772243737,0.5639356774206108], [-0.6006970016720174,0.9686450943470077,-1.7310975948083858,1.2398543489295077,0.4483279499267394], [-0.4592797597414643,0.9631529442377594,-1.6797024055855931,1.4289800788354718,0.28667160760155463], [-0.32700881076504584,0.8147754680205297,-1.763026614596352,1.3194735072019403,0.36816545181467747], [-0.4305345983795718,1.131285559002078,-1.7583813300937976,1.363545141597296,0.2601165464857354], [-0.448497878644767,1.0701920363594992,-1.6308801358271674,1.2674713459930484,0.49959128317905865], [-0.3686293381026533,1.025599917053419,-1.5433629968114364,1.2801178859900646,0.4174115399252367], [-0.574113693976958,0.9757976338077295,-1.6388778822719372,1.2759164798546365,0.5542142157229283], [-0.4357071400660946,0.8338147376622728,-1.8285888680646112,1.271233025944097,0.270178918164786], [-0.47158188300536635,1.0122183541160776,-1.6010520252577112,1.5447804629514494,0.3533722934153737], [-0.5307926544952695,0.9107879044274507,-1.4490342775396574,1.4031526476841614,0.3809780658985748], [-0.4953668915084064,0.9298227962550192,-1.5651812648587027,1.11568589286655,0.4774983034589586], [-0.7383552095915331,0.7626241115292196,-1.7008182872047266,1.2502877442685654,0.43135198632241045], [-0.539114438923603,1.0489102632469824,-1.6736818095705865,1.2768121659352345,0.2213641562753643], [-0.46002266280222004,1.0294295650038185,-1.6033579773810744,1.407522155806443,0.5223722922756932]) -2016-06-13 20:19:24 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:24 INFO JobScheduler:59 - Added jobs for time 10000 ms -2016-06-13 20:19:24 INFO JobGenerator:59 - Checkpointing graph for time 1000 ms -2016-06-13 20:19:24 INFO DStreamGraph:59 - Updating checkpoint data for time 1000 ms -2016-06-13 20:19:24 INFO DStreamGraph:59 - Updated checkpoint data for time 1000 ms -2016-06-13 20:19:24 INFO JobGenerator:59 - Checkpointing graph for time 2000 ms -2016-06-13 20:19:24 INFO DStreamGraph:59 - Updating checkpoint data for time 2000 ms -2016-06-13 20:19:24 INFO DStreamGraph:59 - Updated checkpoint data for time 2000 ms -2016-06-13 20:19:24 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=0, maxMem=766075207 -2016-06-13 20:19:24 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:19:24 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:24 INFO MemoryStore:59 - ensureFreeSpace(2428) called with curMem=4744, maxMem=766075207 -2016-06-13 20:19:24 INFO JobGenerator:59 - Checkpointing graph for time 3000 ms -2016-06-13 20:19:24 INFO DStreamGraph:59 - Updating checkpoint data for time 3000 ms -2016-06-13 20:19:24 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:19:24 INFO DStreamGraph:59 - Updated checkpoint data for time 3000 ms -2016-06-13 20:19:24 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51066 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:19:24 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[7] at apply at Transformer.scala:22) -2016-06-13 20:19:24 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 20:19:24 INFO JobGenerator:59 - Checkpointing graph for time 4000 ms -2016-06-13 20:19:24 INFO DStreamGraph:59 - Updating checkpoint data for time 4000 ms -2016-06-13 20:19:24 INFO DStreamGraph:59 - Updated checkpoint data for time 4000 ms -2016-06-13 20:19:24 INFO JobGenerator:59 - Checkpointing graph for time 5000 ms -2016-06-13 20:19:24 INFO DStreamGraph:59 - Updating checkpoint data for time 5000 ms -2016-06-13 20:19:24 INFO DStreamGraph:59 - Updated checkpoint data for time 5000 ms -2016-06-13 20:19:24 INFO JobGenerator:59 - Checkpointing graph for time 6000 ms -2016-06-13 20:19:24 INFO DStreamGraph:59 - Updating checkpoint data for time 6000 ms -2016-06-13 20:19:24 INFO DStreamGraph:59 - Updated checkpoint data for time 6000 ms -2016-06-13 20:19:24 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:19:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:19:24 INFO JobGenerator:59 - Checkpointing graph for time 7000 ms -2016-06-13 20:19:24 INFO DStreamGraph:59 - Updating checkpoint data for time 7000 ms -2016-06-13 20:19:24 INFO DStreamGraph:59 - Updated checkpoint data for time 7000 ms -2016-06-13 20:19:24 INFO JobGenerator:59 - Checkpointing graph for time 8000 ms -2016-06-13 20:19:24 INFO DStreamGraph:59 - Updating checkpoint data for time 8000 ms -2016-06-13 20:19:24 INFO DStreamGraph:59 - Updated checkpoint data for time 8000 ms -2016-06-13 20:19:24 INFO JobGenerator:59 - Checkpointing graph for time 9000 ms -2016-06-13 20:19:24 INFO DStreamGraph:59 - Updating checkpoint data for time 9000 ms -2016-06-13 20:19:24 INFO DStreamGraph:59 - Updated checkpoint data for time 9000 ms -2016-06-13 20:19:24 INFO JobGenerator:59 - Checkpointing graph for time 10000 ms -2016-06-13 20:19:24 INFO DStreamGraph:59 - Updating checkpoint data for time 10000 ms -2016-06-13 20:19:24 INFO DStreamGraph:59 - Updated checkpoint data for time 10000 ms -2016-06-13 20:19:24 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:24 INFO CheckpointWriter:59 - Saving checkpoint for time 1000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-1000' -2016-06-13 20:19:24 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:24 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 20:19:24 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 20:19:24 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:24 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 20:19:24 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 20:19:24 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1159 bytes result sent to driver -2016-06-13 20:19:24 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:24 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1159 bytes result sent to driver -2016-06-13 20:19:24 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 672 ms on localhost (1/2) -2016-06-13 20:19:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 733 ms on localhost (2/2) -2016-06-13 20:19:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 20:19:25 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:25 INFO DAGScheduler:59 - ShuffleMapStage 0 (apply at Transformer.scala:22) finished in 0.793 s -2016-06-13 20:19:25 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:19:25 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:19:25 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 20:19:25 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:19:25 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 20:19:25 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[24] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:19:25 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=7172, maxMem=766075207 -2016-06-13 20:19:25 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:19:25 INFO MemoryStore:59 - ensureFreeSpace(2657) called with curMem=12436, maxMem=766075207 -2016-06-13 20:19:25 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 20:19:25 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51066 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:19:25 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[24] at apply at Transformer.scala:22) -2016-06-13 20:19:25 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 20:19:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:25 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 20:19:25 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:25 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:19:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-06-13 20:19:25 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-06-13 20:19:25 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:25 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 20:19:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 171 ms on localhost (1/2) -2016-06-13 20:19:25 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:25 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:19:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 47 ms -2016-06-13 20:19:25 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:25 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1498 bytes result sent to driver -2016-06-13 20:19:25 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 454 ms on localhost (2/2) -2016-06-13 20:19:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 20:19:25 INFO DAGScheduler:59 - ResultStage 1 (apply at Transformer.scala:22) finished in 0.458 s -2016-06-13 20:19:25 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:25 INFO CheckpointWriter:59 - Checkpoint for time 1000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-1000', took 27277 bytes and 1092 ms -2016-06-13 20:19:25 INFO CheckpointWriter:59 - Saving checkpoint for time 2000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-2000' -2016-06-13 20:19:25 INFO DAGScheduler:59 - Job 0 finished: apply at Transformer.scala:22, took 1.511829 s -2016-06-13 20:19:25 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 50.0 and centroid: [-0.4843392480139548,0.9734525763849463,-1.7046336323611109,1.2778093795456185,0.3691650550997036] -2016-06-13 20:19:25 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:19:25 INFO JobScheduler:59 - Finished job streaming job 1000 ms.0 from job set of time 1000 ms -2016-06-13 20:19:25 INFO JobScheduler:59 - Starting job streaming job 1000 ms.1 from job set of time 1000 ms -2016-06-13 20:19:25 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:25 INFO DAGScheduler:59 - Registering RDD 3 (apply at Transformer.scala:22) -2016-06-13 20:19:25 INFO DAGScheduler:59 - Got job 1 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:19:25 INFO DAGScheduler:59 - Final stage: ResultStage 3(apply at Transformer.scala:22) -2016-06-13 20:19:25 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 2) -2016-06-13 20:19:25 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 2) -2016-06-13 20:19:25 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (UnionRDD[3] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:19:25 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=15093, maxMem=766075207 -2016-06-13 20:19:25 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:19:25 INFO MemoryStore:59 - ensureFreeSpace(2000) called with curMem=18373, maxMem=766075207 -2016-06-13 20:19:25 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2000.0 B, free 730.6 MB) -2016-06-13 20:19:25 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51066 (size: 2000.0 B, free: 730.6 MB) -2016-06-13 20:19:25 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:25 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 2 (UnionRDD[3] at apply at Transformer.scala:22) -2016-06-13 20:19:25 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-06-13 20:19:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:19:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:19:25 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 20:19:25 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 20:19:25 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1159 bytes result sent to driver -2016-06-13 20:19:25 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1159 bytes result sent to driver -2016-06-13 20:19:25 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:19:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 32 ms on localhost (1/3) -2016-06-13 20:19:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 38 ms on localhost (2/3) -2016-06-13 20:19:25 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-06-13 20:19:25 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 6). 1159 bytes result sent to driver -2016-06-13 20:19:25 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 6) in 53 ms on localhost (3/3) -2016-06-13 20:19:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 20:19:25 INFO DAGScheduler:59 - ShuffleMapStage 2 (apply at Transformer.scala:22) finished in 0.086 s -2016-06-13 20:19:25 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:19:25 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:19:25 INFO DAGScheduler:59 - waiting: Set(ResultStage 3) -2016-06-13 20:19:25 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:19:25 INFO DAGScheduler:59 - Missing parents for ResultStage 3: List() -2016-06-13 20:19:25 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[62] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:19:25 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=20373, maxMem=766075207 -2016-06-13 20:19:25 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:19:25 INFO MemoryStore:59 - ensureFreeSpace(1799) called with curMem=23613, maxMem=766075207 -2016-06-13 20:19:25 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1799.0 B, free 730.6 MB) -2016-06-13 20:19:25 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:51066 (size: 1799.0 B, free: 730.6 MB) -2016-06-13 20:19:25 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[62] at apply at Transformer.scala:22) -2016-06-13 20:19:25 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 20:19:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:25 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 7) -2016-06-13 20:19:25 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 8) -2016-06-13 20:19:25 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:25 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:19:25 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:19:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:25 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 3 ms -2016-06-13 20:19:25 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 8). 1172 bytes result sent to driver -2016-06-13 20:19:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 8) in 20 ms on localhost (1/2) -2016-06-13 20:19:25 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 7). 1180 bytes result sent to driver -2016-06-13 20:19:25 INFO CheckpointWriter:59 - Checkpoint for time 2000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-2000', took 27279 bytes and 256 ms -2016-06-13 20:19:25 INFO CheckpointWriter:59 - Saving checkpoint for time 3000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-3000' -2016-06-13 20:19:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 7) in 60 ms on localhost (2/2) -2016-06-13 20:19:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 20:19:25 INFO DAGScheduler:59 - ResultStage 3 (apply at Transformer.scala:22) finished in 0.066 s -2016-06-13 20:19:25 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:25 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:25 INFO DAGScheduler:59 - Job 1 finished: apply at Transformer.scala:22, took 0.324080 s -2016-06-13 20:19:25 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:19:25 INFO JobScheduler:59 - Finished job streaming job 1000 ms.1 from job set of time 1000 ms -2016-06-13 20:19:25 INFO JobScheduler:59 - Total delay: 1465820364.967 s for time 1000 ms (execution: 2.021 s) -2016-06-13 20:19:25 INFO JobScheduler:59 - Starting job streaming job 2000 ms.0 from job set of time 2000 ms -2016-06-13 20:19:25 INFO JobGenerator:59 - Checkpointing graph for time 1000 ms -2016-06-13 20:19:25 INFO DStreamGraph:59 - Updating checkpoint data for time 1000 ms -2016-06-13 20:19:25 INFO DStreamGraph:59 - Updated checkpoint data for time 1000 ms -2016-06-13 20:19:26 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:19:26 INFO DAGScheduler:59 - Registering RDD 63 (apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Got job 2 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:19:26 INFO DAGScheduler:59 - Final stage: ResultStage 5(apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[63] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=25412, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(2471) called with curMem=30156, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:19:26 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:51066 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:19:26 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[63] at apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 10, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:19:26 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 10) -2016-06-13 20:19:26 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 10). 1159 bytes result sent to driver -2016-06-13 20:19:26 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 9) -2016-06-13 20:19:26 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 10) in 46 ms on localhost (1/2) -2016-06-13 20:19:26 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 9). 1159 bytes result sent to driver -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 9) in 55 ms on localhost (2/2) -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 20:19:26 INFO DAGScheduler:59 - ShuffleMapStage 4 (apply at Transformer.scala:22) finished in 0.057 s -2016-06-13 20:19:26 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:19:26 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:19:26 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-06-13 20:19:26 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:19:26 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting ResultStage 5 (ShuffledRDD[64] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=32627, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(2692) called with curMem=37891, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:19:26 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:51066 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:19:26 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (ShuffledRDD[64] at apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 12, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:26 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 11) -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:19:26 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 11). 1498 bytes result sent to driver -2016-06-13 20:19:26 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 11) in 31 ms on localhost (1/2) -2016-06-13 20:19:26 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 12) -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:19:26 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 12). 1161 bytes result sent to driver -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 12) in 60 ms on localhost (2/2) -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 20:19:26 INFO DAGScheduler:59 - ResultStage 5 (apply at Transformer.scala:22) finished in 0.062 s -2016-06-13 20:19:26 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:19:26 INFO DAGScheduler:59 - Job 2 finished: apply at Transformer.scala:22, took 0.203368 s -2016-06-13 20:19:26 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 100.0 and centroid: [-0.4738959687352186,0.969729236815007,-1.6823805644329948,1.2679893451777908,0.37589591355279117] -2016-06-13 20:19:26 INFO JobScheduler:59 - Finished job streaming job 2000 ms.0 from job set of time 2000 ms -2016-06-13 20:19:26 INFO JobScheduler:59 - Starting job streaming job 2000 ms.1 from job set of time 2000 ms -2016-06-13 20:19:26 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:19:26 INFO DAGScheduler:59 - Registering RDD 10 (apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Got job 3 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:19:26 INFO DAGScheduler:59 - Final stage: ResultStage 7(apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (UnionRDD[10] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=40583, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=43863, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:19:26 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:51066 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:19:26 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 6 (UnionRDD[10] at apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 3 tasks -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 14, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:19:26 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 14) -2016-06-13 20:19:26 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 13) -2016-06-13 20:19:26 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:19:26 INFO CheckpointWriter:59 - Checkpoint for time 3000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-3000', took 27279 bytes and 481 ms -2016-06-13 20:19:26 INFO CheckpointWriter:59 - Saving checkpoint for time 4000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-4000' -2016-06-13 20:19:26 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 13). 1159 bytes result sent to driver -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 15, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:19:26 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 15) -2016-06-13 20:19:26 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 15). 1159 bytes result sent to driver -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 13) in 41 ms on localhost (1/3) -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 15) in 22 ms on localhost (2/3) -2016-06-13 20:19:26 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 14). 1159 bytes result sent to driver -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 14) in 51 ms on localhost (3/3) -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 20:19:26 INFO DAGScheduler:59 - ShuffleMapStage 6 (apply at Transformer.scala:22) finished in 0.055 s -2016-06-13 20:19:26 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:19:26 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:19:26 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-06-13 20:19:26 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:19:26 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[65] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=45872, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=49112, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.5 MB) -2016-06-13 20:19:26 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:51066 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:19:26 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[65] at apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:26 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 17) -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:19:26 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 17). 1172 bytes result sent to driver -2016-06-13 20:19:26 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 17) in 11 ms on localhost (1/2) -2016-06-13 20:19:26 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 16) -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:19:26 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 16). 1180 bytes result sent to driver -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 16) in 57 ms on localhost (2/2) -2016-06-13 20:19:26 INFO DAGScheduler:59 - ResultStage 7 (apply at Transformer.scala:22) finished in 0.058 s -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 20:19:26 INFO DAGScheduler:59 - Job 3 finished: apply at Transformer.scala:22, took 0.151737 s -2016-06-13 20:19:26 INFO JobScheduler:59 - Finished job streaming job 2000 ms.1 from job set of time 2000 ms -2016-06-13 20:19:26 INFO JobScheduler:59 - Total delay: 1465820364.386 s for time 2000 ms (execution: 0.416 s) -2016-06-13 20:19:26 INFO JobScheduler:59 - Starting job streaming job 3000 ms.0 from job set of time 3000 ms -2016-06-13 20:19:26 INFO ParallelCollectionRDD:59 - Removing RDD 0 from persistence list -2016-06-13 20:19:26 INFO MapPartitionsRDD:59 - Removing RDD 5 from persistence list -2016-06-13 20:19:26 INFO ShuffledRDD:59 - Removing RDD 4 from persistence list -2016-06-13 20:19:26 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:19:26 INFO UnionRDD:59 - Removing RDD 3 from persistence list -2016-06-13 20:19:26 INFO MapPartitionsRDD:59 - Removing RDD 1 from persistence list -2016-06-13 20:19:26 INFO JobGenerator:59 - Checkpointing graph for time 2000 ms -2016-06-13 20:19:26 INFO DStreamGraph:59 - Updating checkpoint data for time 2000 ms -2016-06-13 20:19:26 INFO DStreamGraph:59 - Updated checkpoint data for time 2000 ms -2016-06-13 20:19:26 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:19:26 INFO DAGScheduler:59 - Registering RDD 66 (apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Got job 4 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:19:26 INFO DAGScheduler:59 - Final stage: ResultStage 9(apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 8) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 8) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting ShuffleMapStage 8 (MapPartitionsRDD[66] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=50914, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(2475) called with curMem=55658, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:19:26 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:51066 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:19:26 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 8 (MapPartitionsRDD[66] at apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 18, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 19, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:19:26 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 19) -2016-06-13 20:19:26 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 18) -2016-06-13 20:19:26 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 19). 1159 bytes result sent to driver -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 19) in 39 ms on localhost (1/2) -2016-06-13 20:19:26 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:19:26 INFO BlockManager:59 - Removing RDD 3 -2016-06-13 20:19:26 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 18). 1159 bytes result sent to driver -2016-06-13 20:19:26 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 18) in 94 ms on localhost (2/2) -2016-06-13 20:19:26 INFO DAGScheduler:59 - ShuffleMapStage 8 (apply at Transformer.scala:22) finished in 0.095 s -2016-06-13 20:19:26 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:19:26 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:19:26 INFO DAGScheduler:59 - waiting: Set(ResultStage 9) -2016-06-13 20:19:26 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 20:19:26 INFO DAGScheduler:59 - Missing parents for ResultStage 9: List() -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting ResultStage 9 (ShuffledRDD[67] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:19:26 INFO BlockManager:59 - Removing RDD 0 -2016-06-13 20:19:26 INFO BlockManager:59 - Removing RDD 4 -2016-06-13 20:19:26 INFO BlockManager:59 - Removing RDD 5 -2016-06-13 20:19:26 INFO BlockManager:59 - Removing RDD 1 -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=58133, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=63397, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:19:26 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:51066 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:19:26 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:19:26 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (ShuffledRDD[67] at apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 21, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:26 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 21) -2016-06-13 20:19:26 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:51066 in memory (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:19:26 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 20) -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:26 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 21). 1161 bytes result sent to driver -2016-06-13 20:19:26 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 20:19:26 INFO CheckpointWriter:59 - Checkpoint for time 4000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-4000', took 27279 bytes and 341 ms -2016-06-13 20:19:26 INFO CheckpointWriter:59 - Saving checkpoint for time 5000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-5000' -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 21) in 21 ms on localhost (1/2) -2016-06-13 20:19:26 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:51066 in memory (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:26 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 20:19:26 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:51066 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:19:26 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 20). 1498 bytes result sent to driver -2016-06-13 20:19:26 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 20) in 42 ms on localhost (2/2) -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 20:19:26 INFO DAGScheduler:59 - ResultStage 9 (apply at Transformer.scala:22) finished in 0.044 s -2016-06-13 20:19:26 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:19:26 INFO DAGScheduler:59 - Job 4 finished: apply at Transformer.scala:22, took 0.231365 s -2016-06-13 20:19:26 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 150.0 and centroid: [-0.4741102089605591,0.9622750167428704,-1.680286076986711,1.2654376587211662,0.37713329463498424] -2016-06-13 20:19:26 INFO JobScheduler:59 - Finished job streaming job 3000 ms.0 from job set of time 3000 ms -2016-06-13 20:19:26 INFO JobScheduler:59 - Starting job streaming job 3000 ms.1 from job set of time 3000 ms -2016-06-13 20:19:26 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:51066 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:19:26 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 20:19:26 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:19:26 INFO DAGScheduler:59 - Registering RDD 16 (apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Got job 5 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:19:26 INFO DAGScheduler:59 - Final stage: ResultStage 11(apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 10) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 10) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting ShuffleMapStage 10 (UnionRDD[16] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:19:26 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=38792, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(2004) called with curMem=42072, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2004.0 B, free 730.5 MB) -2016-06-13 20:19:26 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:51066 in memory (size: 1799.0 B, free: 730.6 MB) -2016-06-13 20:19:26 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:51066 (size: 2004.0 B, free: 730.6 MB) -2016-06-13 20:19:26 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 20:19:26 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 10 (UnionRDD[16] at apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 3 tasks -2016-06-13 20:19:26 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:51066 in memory (size: 2000.0 B, free: 730.6 MB) -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 22, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 23, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:19:26 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 20:19:26 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 22) -2016-06-13 20:19:26 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 23) -2016-06-13 20:19:26 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-06-13 20:19:26 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:51066 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:19:26 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:19:26 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 22). 1159 bytes result sent to driver -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 10.0 (TID 24, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:19:26 INFO Executor:59 - Running task 2.0 in stage 10.0 (TID 24) -2016-06-13 20:19:26 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 23). 1159 bytes result sent to driver -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 22) in 22 ms on localhost (1/3) -2016-06-13 20:19:26 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 23) in 23 ms on localhost (2/3) -2016-06-13 20:19:26 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:51066 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:19:26 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 20:19:26 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-06-13 20:19:26 INFO Executor:59 - Finished task 2.0 in stage 10.0 (TID 24). 1159 bytes result sent to driver -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 10.0 (TID 24) in 19 ms on localhost (3/3) -2016-06-13 20:19:26 INFO DAGScheduler:59 - ShuffleMapStage 10 (apply at Transformer.scala:22) finished in 0.042 s -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 20:19:26 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:19:26 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:19:26 INFO DAGScheduler:59 - waiting: Set(ResultStage 11) -2016-06-13 20:19:26 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:19:26 INFO DAGScheduler:59 - Missing parents for ResultStage 11: List() -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[68] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=20463, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=23703, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.6 MB) -2016-06-13 20:19:26 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:51066 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:19:26 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[68] at apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 25, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 26, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:26 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 26) -2016-06-13 20:19:26 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 25) -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:19:26 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 26). 1172 bytes result sent to driver -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:19:26 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 26) in 15 ms on localhost (1/2) -2016-06-13 20:19:26 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 25). 1180 bytes result sent to driver -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 25) in 25 ms on localhost (2/2) -2016-06-13 20:19:26 INFO DAGScheduler:59 - ResultStage 11 (apply at Transformer.scala:22) finished in 0.026 s -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 20:19:26 INFO DAGScheduler:59 - Job 5 finished: apply at Transformer.scala:22, took 0.107062 s -2016-06-13 20:19:26 INFO JobScheduler:59 - Finished job streaming job 3000 ms.1 from job set of time 3000 ms -2016-06-13 20:19:26 INFO JobScheduler:59 - Total delay: 1465820363.758 s for time 3000 ms (execution: 0.371 s) -2016-06-13 20:19:26 INFO JobScheduler:59 - Starting job streaming job 4000 ms.0 from job set of time 4000 ms -2016-06-13 20:19:26 INFO ParallelCollectionRDD:59 - Removing RDD 6 from persistence list -2016-06-13 20:19:26 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-06-13 20:19:26 INFO BlockManager:59 - Removing RDD 6 -2016-06-13 20:19:26 INFO ShuffledRDD:59 - Removing RDD 11 from persistence list -2016-06-13 20:19:26 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 20:19:26 INFO BlockManager:59 - Removing RDD 11 -2016-06-13 20:19:26 INFO UnionRDD:59 - Removing RDD 10 from persistence list -2016-06-13 20:19:26 INFO MapPartitionsRDD:59 - Removing RDD 8 from persistence list -2016-06-13 20:19:26 INFO BlockManager:59 - Removing RDD 10 -2016-06-13 20:19:26 INFO JobGenerator:59 - Checkpointing graph for time 3000 ms -2016-06-13 20:19:26 INFO DStreamGraph:59 - Updating checkpoint data for time 3000 ms -2016-06-13 20:19:26 INFO BlockManager:59 - Removing RDD 8 -2016-06-13 20:19:26 INFO DStreamGraph:59 - Updated checkpoint data for time 3000 ms -2016-06-13 20:19:26 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:19:26 INFO CheckpointWriter:59 - Checkpoint for time 5000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-5000', took 27279 bytes and 163 ms -2016-06-13 20:19:26 INFO CheckpointWriter:59 - Saving checkpoint for time 6000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-6000' -2016-06-13 20:19:26 INFO DAGScheduler:59 - Registering RDD 69 (apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Got job 6 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:19:26 INFO DAGScheduler:59 - Final stage: ResultStage 13(apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (MapPartitionsRDD[69] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=25505, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(2484) called with curMem=30249, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:19:26 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:19:26 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:51066 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:19:26 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (MapPartitionsRDD[69] at apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 27, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 28, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:19:26 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 27) -2016-06-13 20:19:26 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 28) -2016-06-13 20:19:26 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 27). 1159 bytes result sent to driver -2016-06-13 20:19:26 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 28). 1159 bytes result sent to driver -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 27) in 29 ms on localhost (1/2) -2016-06-13 20:19:26 INFO DAGScheduler:59 - ShuffleMapStage 12 (apply at Transformer.scala:22) finished in 0.032 s -2016-06-13 20:19:26 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:19:26 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:19:26 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-06-13 20:19:26 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:19:26 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting ResultStage 13 (ShuffledRDD[70] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=32733, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 28) in 30 ms on localhost (2/2) -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(2698) called with curMem=37997, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:19:26 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:51066 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:19:26 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:19:26 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (ShuffledRDD[70] at apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 29, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 30, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:26 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 29) -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:19:26 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 29). 1498 bytes result sent to driver -2016-06-13 20:19:26 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 30) -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:26 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 30). 1161 bytes result sent to driver -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 30) in 41 ms on localhost (1/2) -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 29) in 42 ms on localhost (2/2) -2016-06-13 20:19:26 INFO DAGScheduler:59 - ResultStage 13 (apply at Transformer.scala:22) finished in 0.044 s -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 20:19:26 INFO DAGScheduler:59 - Job 6 finished: apply at Transformer.scala:22, took 0.118628 s -2016-06-13 20:19:26 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 200.0 and centroid: [-0.4768735011530239,0.9606386018112957,-1.673148729841545,1.2634974127814163,0.38189605546947214] -2016-06-13 20:19:26 INFO JobScheduler:59 - Finished job streaming job 4000 ms.0 from job set of time 4000 ms -2016-06-13 20:19:26 INFO JobScheduler:59 - Starting job streaming job 4000 ms.1 from job set of time 4000 ms -2016-06-13 20:19:26 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:19:26 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:19:26 INFO DAGScheduler:59 - Registering RDD 22 (apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Got job 7 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:19:26 INFO DAGScheduler:59 - Final stage: ResultStage 15(apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 14) -2016-06-13 20:19:26 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 14) -2016-06-13 20:19:26 INFO CheckpointWriter:59 - Checkpoint for time 6000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-6000', took 27277 bytes and 130 ms -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting ShuffleMapStage 14 (UnionRDD[22] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:19:26 INFO CheckpointWriter:59 - Saving checkpoint for time 7000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-7000' -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=40695, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=43975, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:19:26 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:51066 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:19:26 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 14 (UnionRDD[22] at apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 3 tasks -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 31, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 32, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:19:26 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 32) -2016-06-13 20:19:26 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 31) -2016-06-13 20:19:26 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 31). 1159 bytes result sent to driver -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 33, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:19:26 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 33) -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 31) in 30 ms on localhost (1/3) -2016-06-13 20:19:26 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:19:26 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 32). 1159 bytes result sent to driver -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 32) in 35 ms on localhost (2/3) -2016-06-13 20:19:26 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 33). 1159 bytes result sent to driver -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 33) in 21 ms on localhost (3/3) -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 20:19:26 INFO DAGScheduler:59 - ShuffleMapStage 14 (apply at Transformer.scala:22) finished in 0.046 s -2016-06-13 20:19:26 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:19:26 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:19:26 INFO DAGScheduler:59 - waiting: Set(ResultStage 15) -2016-06-13 20:19:26 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:19:26 INFO DAGScheduler:59 - Missing parents for ResultStage 15: List() -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[71] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=45984, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:19:26 INFO MemoryStore:59 - ensureFreeSpace(1801) called with curMem=49224, maxMem=766075207 -2016-06-13 20:19:26 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1801.0 B, free 730.5 MB) -2016-06-13 20:19:26 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:51066 (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:19:26 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[71] at apply at Transformer.scala:22) -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 34, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 35, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:26 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 35) -2016-06-13 20:19:26 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 34) -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:26 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 35). 1172 bytes result sent to driver -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:19:26 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 35) in 14 ms on localhost (1/2) -2016-06-13 20:19:26 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 34). 1180 bytes result sent to driver -2016-06-13 20:19:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 34) in 20 ms on localhost (2/2) -2016-06-13 20:19:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 20:19:26 INFO DAGScheduler:59 - ResultStage 15 (apply at Transformer.scala:22) finished in 0.020 s -2016-06-13 20:19:26 INFO DAGScheduler:59 - Job 7 finished: apply at Transformer.scala:22, took 0.095744 s -2016-06-13 20:19:27 INFO JobScheduler:59 - Finished job streaming job 4000 ms.1 from job set of time 4000 ms -2016-06-13 20:19:27 INFO ParallelCollectionRDD:59 - Removing RDD 13 from persistence list -2016-06-13 20:19:27 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:19:27 INFO JobScheduler:59 - Total delay: 1465820363.002 s for time 4000 ms (execution: 0.243 s) -2016-06-13 20:19:27 INFO JobScheduler:59 - Starting job streaming job 5000 ms.0 from job set of time 5000 ms -2016-06-13 20:19:27 INFO BlockManager:59 - Removing RDD 13 -2016-06-13 20:19:27 INFO MapPartitionsRDD:59 - Removing RDD 18 from persistence list -2016-06-13 20:19:27 INFO BlockManager:59 - Removing RDD 18 -2016-06-13 20:19:27 INFO ShuffledRDD:59 - Removing RDD 17 from persistence list -2016-06-13 20:19:27 INFO UnionRDD:59 - Removing RDD 16 from persistence list -2016-06-13 20:19:27 INFO MapPartitionsRDD:59 - Removing RDD 14 from persistence list -2016-06-13 20:19:27 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:19:27 INFO JobGenerator:59 - Checkpointing graph for time 4000 ms -2016-06-13 20:19:27 INFO DStreamGraph:59 - Updating checkpoint data for time 4000 ms -2016-06-13 20:19:27 INFO DStreamGraph:59 - Updated checkpoint data for time 4000 ms -2016-06-13 20:19:27 INFO DAGScheduler:59 - Registering RDD 72 (apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Got job 8 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:19:27 INFO DAGScheduler:59 - Final stage: ResultStage 17(apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 16) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 16) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 16 (MapPartitionsRDD[72] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=51025, maxMem=766075207 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:19:27 INFO CheckpointWriter:59 - Checkpoint for time 7000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-7000', took 27277 bytes and 124 ms -2016-06-13 20:19:27 INFO CheckpointWriter:59 - Saving checkpoint for time 8000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-8000' -2016-06-13 20:19:27 INFO BlockManager:59 - Removing RDD 17 -2016-06-13 20:19:27 INFO BlockManager:59 - Removing RDD 14 -2016-06-13 20:19:27 INFO BlockManager:59 - Removing RDD 16 -2016-06-13 20:19:27 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(2476) called with curMem=55769, maxMem=766075207 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:51066 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:19:27 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 16 (MapPartitionsRDD[72] at apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 36, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 37, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:19:27 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 37) -2016-06-13 20:19:27 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 36) -2016-06-13 20:19:27 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:19:27 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 36). 1159 bytes result sent to driver -2016-06-13 20:19:27 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 37). 1159 bytes result sent to driver -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 36) in 67 ms on localhost (1/2) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 37) in 67 ms on localhost (2/2) -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 20:19:27 INFO DAGScheduler:59 - ShuffleMapStage 16 (apply at Transformer.scala:22) finished in 0.066 s -2016-06-13 20:19:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:19:27 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:19:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 17) -2016-06-13 20:19:27 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:19:27 INFO DAGScheduler:59 - Missing parents for ResultStage 17: List() -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting ResultStage 17 (ShuffledRDD[73] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=58245, maxMem=766075207 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=63509, maxMem=766075207 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:51066 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:19:27 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (ShuffledRDD[73] at apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 38, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 39, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:27 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 39) -2016-06-13 20:19:27 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 38) -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:27 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 39). 1161 bytes result sent to driver -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 39) in 13 ms on localhost (1/2) -2016-06-13 20:19:27 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:19:27 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 38). 1498 bytes result sent to driver -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 38) in 38 ms on localhost (2/2) -2016-06-13 20:19:27 INFO DAGScheduler:59 - ResultStage 17 (apply at Transformer.scala:22) finished in 0.039 s -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 20:19:27 INFO DAGScheduler:59 - Job 8 finished: apply at Transformer.scala:22, took 0.215069 s -2016-06-13 20:19:27 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 250.0 and centroid: [-0.4770613340912709,0.9657053185238176,-1.6732261982319274,1.26651642628913,0.38386658573056776] -2016-06-13 20:19:27 INFO JobScheduler:59 - Finished job streaming job 5000 ms.0 from job set of time 5000 ms -2016-06-13 20:19:27 INFO JobScheduler:59 - Starting job streaming job 5000 ms.1 from job set of time 5000 ms -2016-06-13 20:19:27 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:19:27 INFO DAGScheduler:59 - Registering RDD 29 (apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Got job 9 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:19:27 INFO DAGScheduler:59 - Final stage: ResultStage 19(apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 18) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 18) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 18 (UnionRDD[29] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=66203, maxMem=766075207 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(2010) called with curMem=69483, maxMem=766075207 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2010.0 B, free 730.5 MB) -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:51066 (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:19:27 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 18 (UnionRDD[29] at apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 3 tasks -2016-06-13 20:19:27 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 40, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 41, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:19:27 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 40) -2016-06-13 20:19:27 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 41) -2016-06-13 20:19:27 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 40). 1159 bytes result sent to driver -2016-06-13 20:19:27 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 41). 1159 bytes result sent to driver -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 42, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 40) in 54 ms on localhost (1/3) -2016-06-13 20:19:27 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 41) in 49 ms on localhost (2/3) -2016-06-13 20:19:27 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 42) -2016-06-13 20:19:27 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 42). 1159 bytes result sent to driver -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 42) in 16 ms on localhost (3/3) -2016-06-13 20:19:27 INFO DAGScheduler:59 - ShuffleMapStage 18 (apply at Transformer.scala:22) finished in 0.069 s -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 20:19:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:19:27 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:19:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 19) -2016-06-13 20:19:27 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:19:27 INFO DAGScheduler:59 - Missing parents for ResultStage 19: List() -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[74] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=71493, maxMem=766075207 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=74733, maxMem=766075207 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:51066 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:19:27 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[74] at apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 43, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 44, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:27 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 44) -2016-06-13 20:19:27 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 43) -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:19:27 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 44). 1172 bytes result sent to driver -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:27 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 43). 1180 bytes result sent to driver -2016-06-13 20:19:27 INFO CheckpointWriter:59 - Checkpoint for time 8000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-8000', took 27279 bytes and 355 ms -2016-06-13 20:19:27 INFO CheckpointWriter:59 - Saving checkpoint for time 9000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-9000' -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 44) in 44 ms on localhost (1/2) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 43) in 46 ms on localhost (2/2) -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 20:19:27 INFO DAGScheduler:59 - ResultStage 19 (apply at Transformer.scala:22) finished in 0.048 s -2016-06-13 20:19:27 INFO DAGScheduler:59 - Job 9 finished: apply at Transformer.scala:22, took 0.150307 s -2016-06-13 20:19:27 INFO JobScheduler:59 - Finished job streaming job 5000 ms.1 from job set of time 5000 ms -2016-06-13 20:19:27 INFO JobScheduler:59 - Total delay: 1465820362.392 s for time 5000 ms (execution: 0.388 s) -2016-06-13 20:19:27 INFO JobScheduler:59 - Starting job streaming job 6000 ms.0 from job set of time 6000 ms -2016-06-13 20:19:27 INFO ParallelCollectionRDD:59 - Removing RDD 19 from persistence list -2016-06-13 20:19:27 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:19:27 INFO MapPartitionsRDD:59 - Removing RDD 25 from persistence list -2016-06-13 20:19:27 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:19:27 INFO BlockManager:59 - Removing RDD 19 -2016-06-13 20:19:27 INFO DAGScheduler:59 - Registering RDD 75 (apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Got job 10 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:19:27 INFO DAGScheduler:59 - Final stage: ResultStage 21(apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 20) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 20) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 20 (MapPartitionsRDD[75] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:19:27 INFO ShuffledRDD:59 - Removing RDD 23 from persistence list -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=76536, maxMem=766075207 -2016-06-13 20:19:27 INFO BlockManager:59 - Removing RDD 25 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:19:27 INFO UnionRDD:59 - Removing RDD 22 from persistence list -2016-06-13 20:19:27 INFO BlockManager:59 - Removing RDD 23 -2016-06-13 20:19:27 INFO MapPartitionsRDD:59 - Removing RDD 20 from persistence list -2016-06-13 20:19:27 INFO BlockManager:59 - Removing RDD 22 -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=81280, maxMem=766075207 -2016-06-13 20:19:27 INFO BlockManager:59 - Removing RDD 20 -2016-06-13 20:19:27 INFO JobGenerator:59 - Checkpointing graph for time 5000 ms -2016-06-13 20:19:27 INFO DStreamGraph:59 - Updating checkpoint data for time 5000 ms -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:19:27 INFO DStreamGraph:59 - Updated checkpoint data for time 5000 ms -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:51066 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:19:27 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 20 (MapPartitionsRDD[75] at apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 45, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 46, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:19:27 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 45) -2016-06-13 20:19:27 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 46) -2016-06-13 20:19:27 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 46). 1159 bytes result sent to driver -2016-06-13 20:19:27 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 45). 1159 bytes result sent to driver -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 45) in 24 ms on localhost (1/2) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 46) in 24 ms on localhost (2/2) -2016-06-13 20:19:27 INFO DAGScheduler:59 - ShuffleMapStage 20 (apply at Transformer.scala:22) finished in 0.025 s -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 20:19:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:19:27 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:19:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 21) -2016-06-13 20:19:27 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:19:27 INFO DAGScheduler:59 - Missing parents for ResultStage 21: List() -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting ResultStage 21 (ShuffledRDD[76] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:19:27 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=83762, maxMem=766075207 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=89026, maxMem=766075207 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:51066 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:19:27 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (ShuffledRDD[76] at apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:27 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 47) -2016-06-13 20:19:27 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 48) -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:19:27 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 48). 1161 bytes result sent to driver -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 48) in 12 ms on localhost (1/2) -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:27 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 47). 1498 bytes result sent to driver -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 47) in 28 ms on localhost (2/2) -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 20:19:27 INFO DAGScheduler:59 - ResultStage 21 (apply at Transformer.scala:22) finished in 0.029 s -2016-06-13 20:19:27 INFO DAGScheduler:59 - Job 10 finished: apply at Transformer.scala:22, took 0.087173 s -2016-06-13 20:19:27 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 300.0 and centroid: [-0.4778380287460108,0.9658292271340416,-1.6712646630450996,1.273358698745343,0.37938624993358266] -2016-06-13 20:19:27 INFO JobScheduler:59 - Finished job streaming job 6000 ms.0 from job set of time 6000 ms -2016-06-13 20:19:27 INFO JobScheduler:59 - Starting job streaming job 6000 ms.1 from job set of time 6000 ms -2016-06-13 20:19:27 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:19:27 INFO DAGScheduler:59 - Registering RDD 35 (apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Got job 11 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:19:27 INFO DAGScheduler:59 - Final stage: ResultStage 23(apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 22) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 22) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 22 (UnionRDD[35] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:19:27 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=91722, maxMem=766075207 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=95002, maxMem=766075207 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.5 MB) -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:51066 (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:19:27 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 22 (UnionRDD[35] at apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 3 tasks -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 49, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 50, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:19:27 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 49) -2016-06-13 20:19:27 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 50) -2016-06-13 20:19:27 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 49). 1159 bytes result sent to driver -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 51, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 49) in 34 ms on localhost (1/3) -2016-06-13 20:19:27 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 51) -2016-06-13 20:19:27 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 50). 1159 bytes result sent to driver -2016-06-13 20:19:27 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:19:27 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 51). 1159 bytes result sent to driver -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 50) in 53 ms on localhost (2/3) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 51) in 24 ms on localhost (3/3) -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 20:19:27 INFO CheckpointWriter:59 - Checkpoint for time 9000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-9000', took 27279 bytes and 202 ms -2016-06-13 20:19:27 INFO CheckpointWriter:59 - Saving checkpoint for time 10000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-10000' -2016-06-13 20:19:27 INFO DAGScheduler:59 - ShuffleMapStage 22 (apply at Transformer.scala:22) finished in 0.082 s -2016-06-13 20:19:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:19:27 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:19:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 23) -2016-06-13 20:19:27 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:19:27 INFO DAGScheduler:59 - Missing parents for ResultStage 23: List() -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[77] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=97009, maxMem=766075207 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=100249, maxMem=766075207 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:51066 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:19:27 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[77] at apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-06-13 20:19:27 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 52, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:27 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 52) -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:19:27 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 53) -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:19:27 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 52). 1180 bytes result sent to driver -2016-06-13 20:19:27 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 53). 1172 bytes result sent to driver -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 52) in 17 ms on localhost (1/2) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 53) in 17 ms on localhost (2/2) -2016-06-13 20:19:27 INFO DAGScheduler:59 - ResultStage 23 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 20:19:27 INFO DAGScheduler:59 - Job 11 finished: apply at Transformer.scala:22, took 0.137112 s -2016-06-13 20:19:27 INFO JobScheduler:59 - Finished job streaming job 6000 ms.1 from job set of time 6000 ms -2016-06-13 20:19:27 INFO JobScheduler:59 - Total delay: 1465820361.643 s for time 6000 ms (execution: 0.250 s) -2016-06-13 20:19:27 INFO JobScheduler:59 - Starting job streaming job 7000 ms.0 from job set of time 7000 ms -2016-06-13 20:19:27 INFO ParallelCollectionRDD:59 - Removing RDD 26 from persistence list -2016-06-13 20:19:27 INFO BlockManager:59 - Removing RDD 26 -2016-06-13 20:19:27 INFO MapPartitionsRDD:59 - Removing RDD 31 from persistence list -2016-06-13 20:19:27 INFO BlockManager:59 - Removing RDD 31 -2016-06-13 20:19:27 INFO ShuffledRDD:59 - Removing RDD 30 from persistence list -2016-06-13 20:19:27 INFO BlockManager:59 - Removing RDD 30 -2016-06-13 20:19:27 INFO UnionRDD:59 - Removing RDD 29 from persistence list -2016-06-13 20:19:27 INFO BlockManager:59 - Removing RDD 29 -2016-06-13 20:19:27 INFO MapPartitionsRDD:59 - Removing RDD 27 from persistence list -2016-06-13 20:19:27 INFO BlockManager:59 - Removing RDD 27 -2016-06-13 20:19:27 INFO JobGenerator:59 - Checkpointing graph for time 6000 ms -2016-06-13 20:19:27 INFO DStreamGraph:59 - Updating checkpoint data for time 6000 ms -2016-06-13 20:19:27 INFO DStreamGraph:59 - Updated checkpoint data for time 6000 ms -2016-06-13 20:19:27 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:19:27 INFO DAGScheduler:59 - Registering RDD 78 (apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Got job 12 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:19:27 INFO DAGScheduler:59 - Final stage: ResultStage 25(apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 24) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 24) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 24 (MapPartitionsRDD[78] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:19:27 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=102052, maxMem=766075207 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(2480) called with curMem=106796, maxMem=766075207 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:51066 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 20:19:27 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 24 (MapPartitionsRDD[78] at apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:51066 in memory (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 54, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 55, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:19:27 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 54) -2016-06-13 20:19:27 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 55) -2016-06-13 20:19:27 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 20:19:27 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 55). 1159 bytes result sent to driver -2016-06-13 20:19:27 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 54). 1159 bytes result sent to driver -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 55) in 22 ms on localhost (1/2) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 54) in 24 ms on localhost (2/2) -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-06-13 20:19:27 INFO DAGScheduler:59 - ShuffleMapStage 24 (apply at Transformer.scala:22) finished in 0.026 s -2016-06-13 20:19:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:19:27 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:19:27 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:19:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 25) -2016-06-13 20:19:27 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:19:27 INFO DAGScheduler:59 - Missing parents for ResultStage 25: List() -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting ResultStage 25 (ShuffledRDD[79] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:51066 in memory (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=98946, maxMem=766075207 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=104210, maxMem=766075207 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:19:27 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:51066 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:19:27 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (ShuffledRDD[79] at apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:27 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 56) -2016-06-13 20:19:27 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 57) -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:51066 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:27 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 57). 1161 bytes result sent to driver -2016-06-13 20:19:27 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 56). 1498 bytes result sent to driver -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 57) in 12 ms on localhost (1/2) -2016-06-13 20:19:27 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 56) in 17 ms on localhost (2/2) -2016-06-13 20:19:27 INFO DAGScheduler:59 - ResultStage 25 (apply at Transformer.scala:22) finished in 0.018 s -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-06-13 20:19:27 INFO DAGScheduler:59 - Job 12 finished: apply at Transformer.scala:22, took 0.096114 s -2016-06-13 20:19:27 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 350.0 and centroid: [-0.47440048489638564,0.9650523283036628,-1.6700055274657652,1.269947562552048,0.3797915177328754] -2016-06-13 20:19:27 INFO JobScheduler:59 - Finished job streaming job 7000 ms.0 from job set of time 7000 ms -2016-06-13 20:19:27 INFO JobScheduler:59 - Starting job streaming job 7000 ms.1 from job set of time 7000 ms -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:51066 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:19:27 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:19:27 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 20:19:27 INFO DAGScheduler:59 - Registering RDD 41 (apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Got job 13 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:19:27 INFO DAGScheduler:59 - Final stage: ResultStage 27(apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (UnionRDD[41] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:19:27 INFO ContextCleaner:59 - Cleaned shuffle 10 -2016-06-13 20:19:27 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:51066 in memory (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=86677, maxMem=766075207 -2016-06-13 20:19:27 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:51066 in memory (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=84667, maxMem=766075207 -2016-06-13 20:19:27 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:19:27 INFO ContextCleaner:59 - Cleaned shuffle 9 -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:51066 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:19:27 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 26 (UnionRDD[41] at apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 3 tasks -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:51066 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 58, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 59, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:19:27 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 58) -2016-06-13 20:19:27 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 59) -2016-06-13 20:19:27 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 20:19:27 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 59). 1159 bytes result sent to driver -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 60, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 59) in 21 ms on localhost (1/3) -2016-06-13 20:19:27 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 60) -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:51066 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:19:27 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 58). 1159 bytes result sent to driver -2016-06-13 20:19:27 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 58) in 28 ms on localhost (2/3) -2016-06-13 20:19:27 INFO ContextCleaner:59 - Cleaned shuffle 8 -2016-06-13 20:19:27 INFO CheckpointWriter:59 - Checkpoint for time 10000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-10000', took 27291 bytes and 238 ms -2016-06-13 20:19:27 INFO CheckpointWriter:59 - Saving checkpoint for time 1000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-1000' -2016-06-13 20:19:27 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 60). 1159 bytes result sent to driver -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 60) in 15 ms on localhost (3/3) -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-06-13 20:19:27 INFO DAGScheduler:59 - ShuffleMapStage 26 (apply at Transformer.scala:22) finished in 0.038 s -2016-06-13 20:19:27 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:19:27 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:19:27 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:19:27 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-06-13 20:19:27 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:19:27 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[80] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=71498, maxMem=766075207 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:19:27 INFO MemoryStore:59 - ensureFreeSpace(1801) called with curMem=72937, maxMem=766075207 -2016-06-13 20:19:27 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 1801.0 B, free 730.5 MB) -2016-06-13 20:19:27 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:51066 (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:19:27 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[80] at apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:51066 in memory (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:27 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 62, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:27 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 61) -2016-06-13 20:19:27 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 62) -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:19:27 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:27 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 62). 1172 bytes result sent to driver -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 62) in 12 ms on localhost (1/2) -2016-06-13 20:19:27 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 61). 1180 bytes result sent to driver -2016-06-13 20:19:27 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:51066 in memory (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:19:27 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:19:27 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 61) in 60 ms on localhost (2/2) -2016-06-13 20:19:27 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-06-13 20:19:27 INFO DAGScheduler:59 - ResultStage 27 (apply at Transformer.scala:22) finished in 0.076 s -2016-06-13 20:19:27 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 20:19:27 INFO DAGScheduler:59 - Job 13 finished: apply at Transformer.scala:22, took 0.193866 s -2016-06-13 20:19:27 INFO JobScheduler:59 - Finished job streaming job 7000 ms.1 from job set of time 7000 ms -2016-06-13 20:19:27 INFO ParallelCollectionRDD:59 - Removing RDD 32 from persistence list -2016-06-13 20:19:27 INFO JobScheduler:59 - Total delay: 1465820360.967 s for time 7000 ms (execution: 0.323 s) -2016-06-13 20:19:27 INFO JobScheduler:59 - Starting job streaming job 8000 ms.0 from job set of time 8000 ms -2016-06-13 20:19:27 INFO ContextCleaner:59 - Cleaned shuffle 7 -2016-06-13 20:19:27 INFO MapPartitionsRDD:59 - Removing RDD 37 from persistence list -2016-06-13 20:19:27 INFO ShuffledRDD:59 - Removing RDD 36 from persistence list -2016-06-13 20:19:27 INFO BlockManager:59 - Removing RDD 32 -2016-06-13 20:19:27 INFO BlockManager:59 - Removing RDD 37 -2016-06-13 20:19:27 INFO UnionRDD:59 - Removing RDD 35 from persistence list -2016-06-13 20:19:27 INFO BlockManager:59 - Removing RDD 36 -2016-06-13 20:19:27 INFO MapPartitionsRDD:59 - Removing RDD 33 from persistence list -2016-06-13 20:19:27 INFO BlockManager:59 - Removing RDD 35 -2016-06-13 20:19:27 INFO JobGenerator:59 - Checkpointing graph for time 7000 ms -2016-06-13 20:19:27 INFO DStreamGraph:59 - Updating checkpoint data for time 7000 ms -2016-06-13 20:19:27 INFO DStreamGraph:59 - Updated checkpoint data for time 7000 ms -2016-06-13 20:19:27 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:51066 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:19:27 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:19:27 INFO DAGScheduler:59 - Registering RDD 81 (apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Got job 14 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:19:27 INFO DAGScheduler:59 - Final stage: ResultStage 29(apply at Transformer.scala:22) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 28) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 28) -2016-06-13 20:19:27 INFO DAGScheduler:59 - Submitting ShuffleMapStage 28 (MapPartitionsRDD[81] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=58247, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=62991, maxMem=766075207 -2016-06-13 20:19:28 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 20:19:28 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:51066 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:19:28 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 20:19:28 INFO ContextCleaner:59 - Cleaned shuffle 6 -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:19:28 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:51066 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:19:28 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 28 (MapPartitionsRDD[81] at apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-06-13 20:19:28 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:51066 in memory (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 63, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 64, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:19:28 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 63) -2016-06-13 20:19:28 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 64) -2016-06-13 20:19:28 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 20:19:28 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 63). 1159 bytes result sent to driver -2016-06-13 20:19:28 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:51066 in memory (size: 2004.0 B, free: 730.6 MB) -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 63) in 27 ms on localhost (1/2) -2016-06-13 20:19:28 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:19:28 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 64). 1159 bytes result sent to driver -2016-06-13 20:19:28 INFO ContextCleaner:59 - Cleaned shuffle 5 -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 64) in 36 ms on localhost (2/2) -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-06-13 20:19:28 INFO DAGScheduler:59 - ShuffleMapStage 28 (apply at Transformer.scala:22) finished in 0.038 s -2016-06-13 20:19:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:19:28 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:19:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 29) -2016-06-13 20:19:28 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:19:28 INFO DAGScheduler:59 - Missing parents for ResultStage 29: List() -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting ResultStage 29 (ShuffledRDD[82] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=39959, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=45223, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:19:28 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:51066 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:19:28 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:51066 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:19:28 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (ShuffledRDD[82] at apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-06-13 20:19:28 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 65, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 66, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:28 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 66) -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:28 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 66). 1161 bytes result sent to driver -2016-06-13 20:19:28 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 65) -2016-06-13 20:19:28 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:51066 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 66) in 10 ms on localhost (1/2) -2016-06-13 20:19:28 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:28 INFO ContextCleaner:59 - Cleaned shuffle 4 -2016-06-13 20:19:28 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 65). 1498 bytes result sent to driver -2016-06-13 20:19:28 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 65) in 25 ms on localhost (2/2) -2016-06-13 20:19:28 INFO DAGScheduler:59 - ResultStage 29 (apply at Transformer.scala:22) finished in 0.026 s -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:19:28 INFO DAGScheduler:59 - Job 14 finished: apply at Transformer.scala:22, took 0.123912 s -2016-06-13 20:19:28 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 400.0 and centroid: [-0.4716182064591067,0.967586334056103,-1.6699954778983372,1.270341503100185,0.37935009425710725] -2016-06-13 20:19:28 INFO JobScheduler:59 - Finished job streaming job 8000 ms.0 from job set of time 8000 ms -2016-06-13 20:19:28 INFO JobScheduler:59 - Starting job streaming job 8000 ms.1 from job set of time 8000 ms -2016-06-13 20:19:28 INFO BlockManager:59 - Removing RDD 33 -2016-06-13 20:19:28 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:19:28 INFO DAGScheduler:59 - Registering RDD 47 (apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO DAGScheduler:59 - Got job 15 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:19:28 INFO DAGScheduler:59 - Final stage: ResultStage 31(apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30) -2016-06-13 20:19:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 30) -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 30 (UnionRDD[47] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=40698, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(2006) called with curMem=43978, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2006.0 B, free 730.5 MB) -2016-06-13 20:19:28 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:51066 (size: 2006.0 B, free: 730.6 MB) -2016-06-13 20:19:28 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 30 (UnionRDD[47] at apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 3 tasks -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 67, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 68, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:19:28 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 68) -2016-06-13 20:19:28 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 67) -2016-06-13 20:19:28 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 68). 1159 bytes result sent to driver -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 69, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:19:28 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 69) -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 68) in 17 ms on localhost (1/3) -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:19:28 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 69). 1159 bytes result sent to driver -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 69) in 10 ms on localhost (2/3) -2016-06-13 20:19:28 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 67). 1159 bytes result sent to driver -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 67) in 47 ms on localhost (3/3) -2016-06-13 20:19:28 INFO DAGScheduler:59 - ShuffleMapStage 30 (apply at Transformer.scala:22) finished in 0.047 s -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 20:19:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:19:28 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:19:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 31) -2016-06-13 20:19:28 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:19:28 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List() -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[83] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=45984, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=49224, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.5 MB) -2016-06-13 20:19:28 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:51066 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:19:28 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[83] at apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 70, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 71, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:28 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 70) -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:28 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 70). 1180 bytes result sent to driver -2016-06-13 20:19:28 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 71) -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:28 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 71). 1172 bytes result sent to driver -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 71) in 54 ms on localhost (1/2) -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 70) in 55 ms on localhost (2/2) -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 20:19:28 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-1000.bk -2016-06-13 20:19:28 INFO DAGScheduler:59 - ResultStage 31 (apply at Transformer.scala:22) finished in 0.064 s -2016-06-13 20:19:28 INFO DAGScheduler:59 - Job 15 finished: apply at Transformer.scala:22, took 0.146879 s -2016-06-13 20:19:28 INFO JobScheduler:59 - Finished job streaming job 8000 ms.1 from job set of time 8000 ms -2016-06-13 20:19:28 INFO ParallelCollectionRDD:59 - Removing RDD 38 from persistence list -2016-06-13 20:19:28 INFO JobScheduler:59 - Total delay: 1465820360.275 s for time 8000 ms (execution: 0.307 s) -2016-06-13 20:19:28 INFO JobScheduler:59 - Starting job streaming job 9000 ms.0 from job set of time 9000 ms -2016-06-13 20:19:28 INFO BlockManager:59 - Removing RDD 38 -2016-06-13 20:19:28 INFO MapPartitionsRDD:59 - Removing RDD 43 from persistence list -2016-06-13 20:19:28 INFO ShuffledRDD:59 - Removing RDD 42 from persistence list -2016-06-13 20:19:28 INFO BlockManager:59 - Removing RDD 43 -2016-06-13 20:19:28 INFO UnionRDD:59 - Removing RDD 41 from persistence list -2016-06-13 20:19:28 INFO BlockManager:59 - Removing RDD 42 -2016-06-13 20:19:28 INFO MapPartitionsRDD:59 - Removing RDD 39 from persistence list -2016-06-13 20:19:28 INFO BlockManager:59 - Removing RDD 41 -2016-06-13 20:19:28 INFO JobGenerator:59 - Checkpointing graph for time 8000 ms -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:19:28 INFO CheckpointWriter:59 - Checkpoint for time 1000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-1000', took 27469 bytes and 457 ms -2016-06-13 20:19:28 INFO BlockManager:59 - Removing RDD 39 -2016-06-13 20:19:28 INFO DStreamGraph:59 - Updating checkpoint data for time 8000 ms -2016-06-13 20:19:28 INFO DStreamGraph:59 - Updated checkpoint data for time 8000 ms -2016-06-13 20:19:28 INFO CheckpointWriter:59 - Saving checkpoint for time 2000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-2000' -2016-06-13 20:19:28 INFO DStreamGraph:59 - Clearing checkpoint data for time 1000 ms -2016-06-13 20:19:28 INFO DStreamGraph:59 - Cleared checkpoint data for time 1000 ms -2016-06-13 20:19:28 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:19:28 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:19:28 INFO DAGScheduler:59 - Registering RDD 84 (apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO DAGScheduler:59 - Got job 16 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:19:28 INFO DAGScheduler:59 - Final stage: ResultStage 33(apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-06-13 20:19:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 32) -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 32 (MapPartitionsRDD[84] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=51026, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(2477) called with curMem=55770, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:19:28 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:51066 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:19:28 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 32 (MapPartitionsRDD[84] at apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-06-13 20:19:28 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/receivedBlockMetadata older than 0: -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 72, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 73, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:19:28 INFO InputInfoTracker:59 - remove old batch metadata: -2016-06-13 20:19:28 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 72) -2016-06-13 20:19:28 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 73) -2016-06-13 20:19:28 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 72). 1159 bytes result sent to driver -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 72) in 23 ms on localhost (1/2) -2016-06-13 20:19:28 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 73). 1159 bytes result sent to driver -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 73) in 25 ms on localhost (2/2) -2016-06-13 20:19:28 INFO DAGScheduler:59 - ShuffleMapStage 32 (apply at Transformer.scala:22) finished in 0.027 s -2016-06-13 20:19:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:19:28 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:19:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 33) -2016-06-13 20:19:28 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 20:19:28 INFO DAGScheduler:59 - Missing parents for ResultStage 33: List() -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting ResultStage 33 (ShuffledRDD[85] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=58247, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=63511, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:19:28 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:51066 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:19:28 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (ShuffledRDD[85] at apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 74, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 75, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:28 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 75) -2016-06-13 20:19:28 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 74) -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:19:28 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 75). 1161 bytes result sent to driver -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 75) in 27 ms on localhost (1/2) -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:19:28 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 74). 1498 bytes result sent to driver -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:19:28 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/receivedBlockMetadata older than 0 -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 74) in 78 ms on localhost (2/2) -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 20:19:28 INFO DAGScheduler:59 - ResultStage 33 (apply at Transformer.scala:22) finished in 0.079 s -2016-06-13 20:19:28 INFO DAGScheduler:59 - Job 16 finished: apply at Transformer.scala:22, took 0.144030 s -2016-06-13 20:19:28 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 450.0 and centroid: [-0.4722098164150225,0.9642763266691793,-1.6666072662538036,1.2709270669491914,0.37911973669212223] -2016-06-13 20:19:28 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:19:28 INFO JobScheduler:59 - Finished job streaming job 9000 ms.0 from job set of time 9000 ms -2016-06-13 20:19:28 INFO JobScheduler:59 - Starting job streaming job 9000 ms.1 from job set of time 9000 ms -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:19:28 INFO DAGScheduler:59 - Registering RDD 53 (apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO DAGScheduler:59 - Got job 17 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:19:28 INFO DAGScheduler:59 - Final stage: ResultStage 35(apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 34) -2016-06-13 20:19:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 34) -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 34 (UnionRDD[53] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=66205, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=69485, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.5 MB) -2016-06-13 20:19:28 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:51066 (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:19:28 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 34 (UnionRDD[53] at apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 3 tasks -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 76, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 77, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:19:28 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 77) -2016-06-13 20:19:28 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 76) -2016-06-13 20:19:28 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 76). 1159 bytes result sent to driver -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 78, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 76) in 37 ms on localhost (1/3) -2016-06-13 20:19:28 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 78) -2016-06-13 20:19:28 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 78). 1159 bytes result sent to driver -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 78) in 13 ms on localhost (2/3) -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:19:28 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 77). 1159 bytes result sent to driver -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 77) in 197 ms on localhost (3/3) -2016-06-13 20:19:28 INFO DAGScheduler:59 - ShuffleMapStage 34 (apply at Transformer.scala:22) finished in 0.198 s -2016-06-13 20:19:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:19:28 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:19:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 35) -2016-06-13 20:19:28 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:19:28 INFO DAGScheduler:59 - Missing parents for ResultStage 35: List() -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[86] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=71492, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=74732, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:19:28 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:51066 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:19:28 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[86] at apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 79, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 80, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:28 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 79) -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:19:28 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 80) -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:28 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 80). 1172 bytes result sent to driver -2016-06-13 20:19:28 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 79). 1180 bytes result sent to driver -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 80) in 17 ms on localhost (1/2) -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 79) in 20 ms on localhost (2/2) -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-06-13 20:19:28 INFO DAGScheduler:59 - ResultStage 35 (apply at Transformer.scala:22) finished in 0.021 s -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:19:28 INFO DAGScheduler:59 - Job 17 finished: apply at Transformer.scala:22, took 0.265246 s -2016-06-13 20:19:28 INFO JobScheduler:59 - Finished job streaming job 9000 ms.1 from job set of time 9000 ms -2016-06-13 20:19:28 INFO JobScheduler:59 - Total delay: 1465820359.719 s for time 9000 ms (execution: 0.443 s) -2016-06-13 20:19:28 INFO JobScheduler:59 - Starting job streaming job 10000 ms.0 from job set of time 10000 ms -2016-06-13 20:19:28 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-1000 -2016-06-13 20:19:28 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:19:28 INFO ParallelCollectionRDD:59 - Removing RDD 44 from persistence list -2016-06-13 20:19:28 INFO DAGScheduler:59 - Registering RDD 87 (apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO DAGScheduler:59 - Got job 18 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:19:28 INFO DAGScheduler:59 - Final stage: ResultStage 37(apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 36) -2016-06-13 20:19:28 INFO MapPartitionsRDD:59 - Removing RDD 49 from persistence list -2016-06-13 20:19:28 INFO BlockManager:59 - Removing RDD 44 -2016-06-13 20:19:28 INFO CheckpointWriter:59 - Checkpoint for time 2000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-2000', took 27482 bytes and 447 ms -2016-06-13 20:19:28 INFO CheckpointWriter:59 - Saving checkpoint for time 3000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-3000' -2016-06-13 20:19:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 36) -2016-06-13 20:19:28 INFO BlockManager:59 - Removing RDD 49 -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 36 (MapPartitionsRDD[87] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=76535, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:19:28 INFO ShuffledRDD:59 - Removing RDD 48 from persistence list -2016-06-13 20:19:28 INFO BlockManager:59 - Removing RDD 48 -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=81279, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:19:28 INFO UnionRDD:59 - Removing RDD 47 from persistence list -2016-06-13 20:19:28 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:51066 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:19:28 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 36 (MapPartitionsRDD[87] at apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 81, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 82, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:19:28 INFO BlockManager:59 - Removing RDD 47 -2016-06-13 20:19:28 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 81) -2016-06-13 20:19:28 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 82) -2016-06-13 20:19:28 INFO MapPartitionsRDD:59 - Removing RDD 45 from persistence list -2016-06-13 20:19:28 INFO DStreamGraph:59 - Clearing checkpoint data for time 2000 ms -2016-06-13 20:19:28 INFO DStreamGraph:59 - Cleared checkpoint data for time 2000 ms -2016-06-13 20:19:28 INFO BlockManager:59 - Removing RDD 45 -2016-06-13 20:19:28 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:19:28 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/receivedBlockMetadata older than 1000: -2016-06-13 20:19:28 INFO InputInfoTracker:59 - remove old batch metadata: -2016-06-13 20:19:28 INFO JobGenerator:59 - Checkpointing graph for time 9000 ms -2016-06-13 20:19:28 INFO DStreamGraph:59 - Updating checkpoint data for time 9000 ms -2016-06-13 20:19:28 INFO DStreamGraph:59 - Updated checkpoint data for time 9000 ms -2016-06-13 20:19:28 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/receivedBlockMetadata older than 1000 -2016-06-13 20:19:28 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 81). 1159 bytes result sent to driver -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 81) in 17 ms on localhost (1/2) -2016-06-13 20:19:28 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 82). 1159 bytes result sent to driver -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 82) in 18 ms on localhost (2/2) -2016-06-13 20:19:28 INFO DAGScheduler:59 - ShuffleMapStage 36 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-06-13 20:19:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:19:28 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:19:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 37) -2016-06-13 20:19:28 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:19:28 INFO DAGScheduler:59 - Missing parents for ResultStage 37: List() -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting ResultStage 37 (ShuffledRDD[88] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=83761, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=89025, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:19:28 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:51066 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:19:28 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (ShuffledRDD[88] at apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 83, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 84, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:28 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 84) -2016-06-13 20:19:28 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 83) -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:28 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 84). 1161 bytes result sent to driver -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 84) in 9 ms on localhost (1/2) -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:19:28 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 83). 1498 bytes result sent to driver -2016-06-13 20:19:28 INFO DAGScheduler:59 - ResultStage 37 (apply at Transformer.scala:22) finished in 0.017 s -2016-06-13 20:19:28 INFO DAGScheduler:59 - Job 18 finished: apply at Transformer.scala:22, took 0.067074 s -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 83) in 17 ms on localhost (2/2) -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-06-13 20:19:28 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 500.0 and centroid: [-0.4725511979691583,0.9644503899125422,-1.668776373542808,1.2721254429935838,0.37815209739836425] -2016-06-13 20:19:28 INFO JobScheduler:59 - Finished job streaming job 10000 ms.0 from job set of time 10000 ms -2016-06-13 20:19:28 INFO JobScheduler:59 - Starting job streaming job 10000 ms.1 from job set of time 10000 ms -2016-06-13 20:19:28 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:19:28 INFO DAGScheduler:59 - Registering RDD 59 (apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO DAGScheduler:59 - Got job 19 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:19:28 INFO DAGScheduler:59 - Final stage: ResultStage 39(apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 38) -2016-06-13 20:19:28 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 38) -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting ShuffleMapStage 38 (UnionRDD[59] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=91721, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=95001, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.5 MB) -2016-06-13 20:19:28 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:51066 (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:19:28 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 38 (UnionRDD[59] at apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 3 tasks -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 85, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 86, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:19:28 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 86) -2016-06-13 20:19:28 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 85) -2016-06-13 20:19:28 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 86). 1159 bytes result sent to driver -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 87, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:19:28 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 85). 1159 bytes result sent to driver -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 86) in 14 ms on localhost (1/3) -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 85) in 17 ms on localhost (2/3) -2016-06-13 20:19:28 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 87) -2016-06-13 20:19:28 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 87). 1159 bytes result sent to driver -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 87) in 16 ms on localhost (3/3) -2016-06-13 20:19:28 INFO DAGScheduler:59 - ShuffleMapStage 38 (apply at Transformer.scala:22) finished in 0.030 s -2016-06-13 20:19:28 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:19:28 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:19:28 INFO DAGScheduler:59 - waiting: Set(ResultStage 39) -2016-06-13 20:19:28 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-06-13 20:19:28 INFO DAGScheduler:59 - Missing parents for ResultStage 39: List() -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[89] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=97008, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:19:28 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=100248, maxMem=766075207 -2016-06-13 20:19:28 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:19:28 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:51066 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:19:28 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:19:28 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[89] at apply at Transformer.scala:22) -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:28 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:19:28 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 89) -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:19:28 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 88) -2016-06-13 20:19:28 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 89). 1172 bytes result sent to driver -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 89) in 6 ms on localhost (1/2) -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:19:28 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:19:28 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 88). 1180 bytes result sent to driver -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:19:28 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 88) in 79 ms on localhost (2/2) -2016-06-13 20:19:28 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-06-13 20:19:28 INFO DAGScheduler:59 - ResultStage 39 (apply at Transformer.scala:22) finished in 0.083 s -2016-06-13 20:19:28 INFO DAGScheduler:59 - Job 19 finished: apply at Transformer.scala:22, took 0.146783 s -2016-06-13 20:19:28 INFO JobScheduler:59 - Finished job streaming job 10000 ms.1 from job set of time 10000 ms -2016-06-13 20:19:28 INFO JobScheduler:59 - Total delay: 1465820358.956 s for time 10000 ms (execution: 0.237 s) -2016-06-13 20:19:28 INFO ParallelCollectionRDD:59 - Removing RDD 50 from persistence list -2016-06-13 20:19:28 INFO BlockManager:59 - Removing RDD 50 -2016-06-13 20:19:28 INFO MapPartitionsRDD:59 - Removing RDD 55 from persistence list -2016-06-13 20:19:28 INFO ShuffledRDD:59 - Removing RDD 54 from persistence list -2016-06-13 20:19:28 INFO UnionRDD:59 - Removing RDD 53 from persistence list -2016-06-13 20:19:28 INFO MapPartitionsRDD:59 - Removing RDD 51 from persistence list -2016-06-13 20:19:28 INFO BlockManager:59 - Removing RDD 53 -2016-06-13 20:19:28 INFO BlockManager:59 - Removing RDD 54 -2016-06-13 20:19:28 INFO BlockManager:59 - Removing RDD 55 -2016-06-13 20:19:28 INFO JobGenerator:59 - Checkpointing graph for time 10000 ms -2016-06-13 20:19:28 INFO DStreamGraph:59 - Updating checkpoint data for time 10000 ms -2016-06-13 20:19:28 INFO BlockManager:59 - Removing RDD 51 -2016-06-13 20:19:28 INFO DStreamGraph:59 - Updated checkpoint data for time 10000 ms -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - Output generated in 5093 milliseconds -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:19:28 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:19:29 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-2000.bk -2016-06-13 20:19:29 INFO CheckpointWriter:59 - Checkpoint for time 3000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-3000', took 27493 bytes and 335 ms -2016-06-13 20:19:29 INFO DStreamGraph:59 - Clearing checkpoint data for time 3000 ms -2016-06-13 20:19:29 INFO CheckpointWriter:59 - Saving checkpoint for time 4000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-4000' -2016-06-13 20:19:29 INFO DStreamGraph:59 - Cleared checkpoint data for time 3000 ms -2016-06-13 20:19:29 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:19:29 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/receivedBlockMetadata older than 2000: -2016-06-13 20:19:29 INFO InputInfoTracker:59 - remove old batch metadata: 1000 ms -2016-06-13 20:19:29 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/receivedBlockMetadata older than 2000 -2016-06-13 20:19:29 INFO JobGenerator:59 - Stopping JobGenerator immediately -2016-06-13 20:19:29 INFO RecurringTimer:59 - Stopped timer for JobGenerator after time 10000 -2016-06-13 20:19:39 INFO CheckpointWriter:59 - CheckpointWriter executor terminated ? false, waited for 10000 ms. -2016-06-13 20:19:39 INFO JobGenerator:59 - Stopped JobGenerator -2016-06-13 20:19:39 INFO JobScheduler:59 - Stopped JobScheduler -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/batch,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/streaming,null} -2016-06-13 20:19:39 INFO StreamingContext:59 - StreamingContext stopped successfully -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/batch/json,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/json,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:19:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:19:39 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-2000 -2016-06-13 20:19:39 INFO CheckpointWriter:59 - Checkpoint for time 4000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f/checkpoint-4000', took 27485 bytes and 10116 ms -2016-06-13 20:19:39 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 20:19:39 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:19:39 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:19:39 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:19:39 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:19:39 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:19:39 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:19:39 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:19:39 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:19:39 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:19:39 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 20:19:39 INFO StreamingKMeansSuite:59 - - -===== FINISHED o.a.s.mllib.clustering.kmeans.StreamingKMeansSuite: 'accuracy for single center and equivalence to grand average' ===== - -2016-06-13 20:19:39 WARN StreamingContext:71 - StreamingContext has already been stopped -2016-06-13 20:19:39 INFO SparkContext:59 - SparkContext already stopped. -2016-06-13 20:19:39 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 20:19:39 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f -2016-06-13 20:19:39 ERROR ShutdownHookManager:96 - Exception while deleting Spark temp dir: C:\Users\xubo\AppData\Local\Temp\spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f -java.io.IOException: Failed to delete: C:\Users\xubo\AppData\Local\Temp\spark-9d8e9272-cd1b-4baf-abb0-660eb18a521f - at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:884) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:63) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:60) - at scala.collection.mutable.HashSet.foreach(HashSet.scala:79) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1.apply$mcV$sp(ShutdownHookManager.scala:60) - at org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:264) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1699) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply$mcV$sp(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:234) - at scala.util.Try$.apply(Try.scala:161) - at org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:216) - at org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54) -2016-06-13 20:19:39 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-02abcfdb-f5ef-49b2-8ac5-b2491dd814c6 -2016-06-13 20:20:41 INFO StreamingKMeansSuite:59 - Using manual clock -2016-06-13 20:20:41 INFO StreamingKMeansSuite:59 - - -===== TEST OUTPUT FOR o.a.s.mllib.clustering.kmeans.StreamingKMeansSuite: 'accuracy for single center and equivalence to grand average' ===== - -2016-06-13 20:20:41 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:20:42 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 20:20:42 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:20:42 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:20:42 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:20:43 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:20:43 INFO Remoting:74 - Starting remoting -2016-06-13 20:20:43 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:51087] -2016-06-13 20:20:43 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51087. -2016-06-13 20:20:43 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:20:43 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:20:43 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-9e7c8d3c-e758-4001-81f4-7c549b983b83 -2016-06-13 20:20:43 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:20:43 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-36841f45-cdae-4750-83a7-c033260e3590\httpd-c675825e-7bb6-47ce-af0d-97cf2eb03201 -2016-06-13 20:20:43 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:20:43 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:20:43 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51088 -2016-06-13 20:20:43 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51088. -2016-06-13 20:20:43 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:20:43 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:20:43 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 20:20:43 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 20:20:43 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 20:20:43 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:20:43 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:20:44 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51099. -2016-06-13 20:20:44 INFO NettyBlockTransferService:59 - Server created on 51099 -2016-06-13 20:20:44 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:20:44 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51099 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51099) -2016-06-13 20:20:44 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:20:45 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 20:20:45 INFO StreamingKMeansSuite:59 - numBatches = 10, numExpectedOutput = 10 -2016-06-13 20:20:46 INFO ForEachDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:20:46 INFO TestInputStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:20:46 INFO TestInputStream:59 - Slide time = 1000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:20:46 INFO TestInputStream:59 - Checkpoint interval = null -2016-06-13 20:20:46 INFO TestInputStream:59 - Remember duration = 1000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Initialized and validated org.apache.spark.streaming.TestInputStream@51b3807d -2016-06-13 20:20:46 INFO ForEachDStream:59 - Slide time = 1000 ms -2016-06-13 20:20:46 INFO ForEachDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:20:46 INFO ForEachDStream:59 - Checkpoint interval = null -2016-06-13 20:20:46 INFO ForEachDStream:59 - Remember duration = 1000 ms -2016-06-13 20:20:46 INFO ForEachDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.ForEachDStream@1b17a9d2 -2016-06-13 20:20:46 INFO TestOutputStreamWithPartitions:59 - metadataCleanupDelay = -1 -2016-06-13 20:20:46 INFO MappedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:20:46 INFO ShuffledDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:20:46 INFO TransformedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:20:46 INFO MappedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:20:46 INFO TestInputStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:20:46 INFO TestInputStream:59 - Slide time = 1000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:20:46 INFO TestInputStream:59 - Checkpoint interval = null -2016-06-13 20:20:46 INFO TestInputStream:59 - Remember duration = 1000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Initialized and validated org.apache.spark.streaming.TestInputStream@51b3807d -2016-06-13 20:20:46 INFO MappedDStream:59 - Slide time = 1000 ms -2016-06-13 20:20:46 INFO MappedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:20:46 INFO MappedDStream:59 - Checkpoint interval = null -2016-06-13 20:20:46 INFO MappedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:20:46 INFO MappedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@1ffd4878 -2016-06-13 20:20:46 INFO TransformedDStream:59 - Slide time = 1000 ms -2016-06-13 20:20:46 INFO TransformedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:20:46 INFO TransformedDStream:59 - Checkpoint interval = null -2016-06-13 20:20:46 INFO TransformedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:20:46 INFO TransformedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.TransformedDStream@3b9dd782 -2016-06-13 20:20:46 INFO ShuffledDStream:59 - Slide time = 1000 ms -2016-06-13 20:20:46 INFO ShuffledDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:20:46 INFO ShuffledDStream:59 - Checkpoint interval = null -2016-06-13 20:20:46 INFO ShuffledDStream:59 - Remember duration = 1000 ms -2016-06-13 20:20:46 INFO ShuffledDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.ShuffledDStream@1e6acf9b -2016-06-13 20:20:46 INFO MappedDStream:59 - Slide time = 1000 ms -2016-06-13 20:20:46 INFO MappedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:20:46 INFO MappedDStream:59 - Checkpoint interval = null -2016-06-13 20:20:46 INFO MappedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:20:46 INFO MappedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@74b4dce5 -2016-06-13 20:20:46 INFO TestOutputStreamWithPartitions:59 - Slide time = 1000 ms -2016-06-13 20:20:46 INFO TestOutputStreamWithPartitions:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:20:46 INFO TestOutputStreamWithPartitions:59 - Checkpoint interval = null -2016-06-13 20:20:46 INFO TestOutputStreamWithPartitions:59 - Remember duration = 1000 ms -2016-06-13 20:20:46 INFO TestOutputStreamWithPartitions:59 - Initialized and validated org.apache.spark.streaming.TestOutputStreamWithPartitions@1cbd6c5e -2016-06-13 20:20:46 INFO RecurringTimer:59 - Started timer for JobGenerator at time 1000 -2016-06-13 20:20:46 INFO JobGenerator:59 - Started JobGenerator at 1000 ms -2016-06-13 20:20:46 INFO JobScheduler:59 - Started JobScheduler -2016-06-13 20:20:46 INFO StreamingContext:59 - StreamingContext started -2016-06-13 20:20:46 INFO StreamingKMeansSuite:59 - Manual clock before advancing = 0 -2016-06-13 20:20:46 INFO StreamingKMeansSuite:59 - Manual clock after advancing = 10000 -2016-06-13 20:20:46 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:20:46 INFO TestInputStream:59 - Computing RDD for time 1000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Created RDD 0 with Vector([-0.35685422167583125,0.9995288307555166,-1.5877809386448758,1.210845548490831,0.56222524157234], [-0.45216537133396134,0.8040574757345051,-1.5589888701250443,1.115603017032467,0.344649973028188], [-0.3801675945145341,1.0697589561731022,-1.6486682116186815,1.2790268654063703,0.5116371486214886], [-0.545300880562902,1.1093057253978866,-1.7582382472264182,1.211400006292075,0.2932044815946008], [-0.3995483137357882,0.929119621864747,-1.7401380541085598,1.2990457426846032,0.299090686584923], [-0.5964001160322344,1.0378164371074625,-1.553411818459791,1.2193968564632125,0.4492703031123452], [-0.33949398265706626,1.1369769008642154,-1.7816495259375202,1.2857769527411067,0.5642853362323552], [-0.4522936770981281,1.090819158309221,-1.4785463285656288,1.2734882859357213,0.31864385887835545], [-0.4895260213585818,1.024567290792244,-1.7744155605537828,1.4515329700677304,0.39720325477885093], [-0.5479321442182269,0.8926951810306638,-1.617388979581139,1.3192738326981768,0.3243774353000105], [-0.5468296230191492,0.9034725811394896,-1.8269524875493084,1.161899613497355,0.22607191229030854], [-0.4344367225435325,0.9054178856043291,-1.6658399540877864,1.13578899284944,0.46511292492021983], [-0.5914600674966851,0.9727747336030947,-1.7411162798640765,1.3902196062020102,0.43158411906136107], [-0.5644425684165094,0.996354296088912,-1.817848647277835,1.3768030733201289,0.28609432985591315], [-0.44148386701129744,1.025737906243747,-1.7073695034902865,1.2539087121413848,0.349242056080172], [-0.5287113084613145,0.8475916475163896,-1.8166057479945195,1.238631411179877,0.36922519791770014], [-0.2540595910515851,0.9698131032346381,-1.5866552625154318,1.2179915976157372,0.3240398576394729], [-0.5026644816450073,1.109581665015373,-1.7151161774591086,1.3901513268104455,0.39042855132933163], [-0.4131172722481001,1.0096927045418442,-1.811286466550916,1.2719080737538961,0.28293757989913904], [-0.4515790075180954,1.0935474561719332,-1.7896107864561164,1.3203383417885988,0.2994362800834273], [-0.4171607573128071,1.0818260184642114,-1.8219543038308885,1.2496462030096327,0.4964658371791977], [-0.45586902589910655,1.0378742745036647,-1.7000458648640797,1.446592332373813,0.348566870310007], [-0.3532087778209343,0.8044437973592731,-1.7162629066191757,1.3339281879504057,0.17946188844185906], [-0.5154063008996834,0.9595656827532482,-1.6519653379002484,1.322654750805598,0.3008929692388781], [-0.3967887546418169,0.7908879805460386,-1.7242582851383725,1.2141584886700174,0.36827019559433166], [-0.48352710482779404,1.0255449792765485,-1.7171397454598174,1.284877805985566,0.437232715780399], [-0.40324856443707463,1.020485534999759,-1.6517518343613002,1.2864020019792224,0.3984832850799499], [-0.6517069948565133,0.969461018860354,-1.684015510299471,1.2027906385718596,0.2014860155598368], [-0.6148278408211463,0.9795135452912322,-1.8642733722424907,1.2720240024566645,0.39785426587083933], [-0.6409674819545198,1.1778584543507387,-1.8094969072333307,1.24145841048891,0.1624672183638662], [-0.6368691555002903,0.8170131462860201,-1.6827833832635621,1.3031179618920097,0.31628673987022604], [-0.43096648399184834,0.9655746876954024,-1.7845188100637381,1.4052721272088269,0.5845842279495], [-0.4442125841627056,1.0257872008493307,-1.7242589434969455,1.2745428690859977,0.39998857867859966], [-0.4527127515828855,0.9000469054331385,-1.7497092961121075,1.2451107418571954,0.4344380036579415], [-0.6030914309562295,0.9699928952430515,-1.712604726536663,1.38261404269885,0.48344898725768926], [-0.34976726689041593,0.9355870894961762,-1.5766155567170825,1.3590936489794139,0.35690680463154484], [-0.575628267406687,0.9538063357613722,-1.6445003616008003,1.2104804829930789,0.3911315521177748], [-0.5582822637309535,0.9339694428739754,-1.6511037013459962,1.4401460976648193,0.3607766390547849], [-0.5468811641579405,0.997556943752324,-1.778985900216746,1.2440321529105447,0.4204585578447196], [-0.48994207115736743,1.0755700976056979,-1.5322042869816734,1.2132674597398667,0.5961642366275318], [-0.5345824744371771,0.7659168558371742,-1.7009697565551642,1.2200491259133113,0.24064266183571642], [-0.4425411108434083,0.9636579302553172,-1.6769715787345627,1.2060504970822528,0.1832578105698996], [-0.2803908635114948,0.9814260387904405,-1.6744903280757537,1.438894231115956,0.5307857877828275], [-0.4646639723438268,0.9422153316765177,-1.8936280948227913,1.1755014401606392,0.3269897050872188], [-0.5794967530942349,1.008452319256215,-1.8095212163297067,1.3053459835247618,0.3862084672749757], [-0.36564413314560745,0.9550404460199052,-1.671332555981891,1.2387784773127055,0.29524181369582553], [-0.4947387784852314,0.8968969114403345,-1.5032027608518188,1.263293789437244,0.41872133210307433], [-0.5901163249097482,0.9195890740589755,-1.76975579729397,1.2233696400583831,0.3403182341587175], [-0.7175598778778954,0.8837946345395844,-1.8329343739105675,1.3562350855120264,0.30734388952604197], [-0.4376962364418643,1.0046436887819825,-1.5427982731479972,1.1077094728701697,0.30461693506090337]) -2016-06-13 20:20:46 INFO JobScheduler:59 - Added jobs for time 1000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Computing RDD for time 2000 ms -2016-06-13 20:20:46 INFO JobScheduler:59 - Starting job streaming job 1000 ms.0 from job set of time 1000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Created RDD 6 with Vector([-0.5385984930004446,0.9989931115946847,-1.745310682734983,1.2763556288677276,0.41555544930746574], [-0.5140757026211953,0.9616396809310985,-1.6092748223737894,1.3080875431688948,0.3958240822167327], [-0.3317121400990889,1.1311521924692884,-1.5940394948396088,1.235663715627094,0.5352186835894586], [-0.3566003302425871,0.8649446433306678,-1.6281158497086519,1.0841327390682376,0.33108928300382867], [-0.4898654567486354,1.0095506134582393,-1.7900184464939166,1.4376638486285889,0.5476994415325478], [-0.5067718408631526,0.8330942372560937,-1.5447521454199076,1.3956201717944832,0.40502500771937533], [-0.45899690569395396,1.0114199686087044,-1.6793704227956527,1.2264029484806054,0.3617800498770533], [-0.42389468031403765,0.9744019756374155,-1.4339024359998267,1.3506788012287876,0.5071391796420295], [-0.3445608603995446,0.8999444931201762,-1.8295489983610813,1.3813011802363127,0.4406110935124823], [-0.4720580057434039,0.9616388382715773,-1.7885815681274932,1.2231064430079939,0.49611588805959794], [-0.6877856894979741,0.9509033529307679,-1.7111111174594673,1.1650825853084499,0.36541463301831684], [-0.41906769746785616,0.9643844813018809,-1.7088669498454399,1.2743442947780539,0.37857583863641653], [-0.45604703784103723,0.9704467329255722,-1.6908561828878124,1.2661005251088149,0.34761517755721416], [-0.45216413072759976,1.125638260528155,-1.8672240017957615,1.388348280279974,0.41289821746105937], [-0.6041998663111035,0.9518745991902821,-1.7482553616491074,1.3027968384522153,0.33003148862622383], [-0.7645915689875407,0.9231306641016475,-1.5908706609397762,1.0416797405804958,0.3435391589642622], [-0.5511129470810242,0.9789516647349409,-1.5896352104287992,1.2985407772348172,0.3074739718251722], [-0.3060111421519473,0.9693226097887153,-1.6503499878201646,1.2050435546957032,0.20529101796457794], [-0.41814813350738833,0.9065345647328998,-1.7369684308945867,1.3095490239413528,0.3931707179477215], [-0.7452070526618801,1.010695258373889,-1.6799990639878284,1.351913921189404,0.48041731347418104], [-0.19474072480894772,0.8574730260551575,-1.7114945513022097,1.2484211088395596,0.35791459113273333], [-0.679333385903053,1.136043445571062,-1.6819398713814546,1.2914383599572574,0.4846586154592743], [-0.38963314954900546,1.0499014443531718,-1.7075189569343263,1.3966853721193921,0.4922463621111953], [-0.4771733790096556,0.8615507710844875,-1.7824679274745288,1.1550649309651535,0.4634518570903764], [-0.46851141840829336,0.9490804068188984,-1.6654371185103947,1.1748885058064462,0.362811152170017], [-0.2486779458724137,1.0146748639649252,-1.8129854069448728,1.3198830969564963,0.3848127258680788], [-0.37278156677721497,0.9026389048304428,-1.6338760097656873,1.2146194636725334,0.305407696426058], [-0.5109164214840503,1.1025203353037651,-1.5805383770959172,1.3659800632138837,0.2507490333376023], [-0.42658774618748607,0.9583040463281658,-1.535401873083165,1.2270967800953845,0.30931926334482973], [-0.5761089637758711,0.9608023900436299,-1.6773446541045054,1.3896056325873751,0.3003385242281313], [-0.333657352857868,0.9284640162518879,-1.6506283984425263,1.4233119402331722,0.4817086358456392], [-0.4131260123218083,0.9257004348959492,-1.6385111064595121,1.2553085674173774,0.35994561408421055], [-0.41619483777131644,0.9791552402269297,-1.6621579631275758,1.1639596596963502,0.3356350519347727], [-0.42150886121352676,0.9497809597217938,-1.4493400602428224,1.2037323844449976,0.28412005647965366], [-0.4036704221590921,0.8682473056100086,-1.4499044130725358,1.4367312949083602,0.29801296100779273], [-0.7347169576181178,0.8472443819718873,-1.6912964247755202,1.1098267823656842,0.5128488081575056], [-0.4599723079910277,1.0313773063743503,-1.6693647567489862,1.1183374503284873,0.3065899617042307], [-0.5155727886799809,0.9793287424722477,-1.81766244647388,1.264051537384318,0.38002468301378506], [-0.461809330593855,0.94300958630738,-1.7083227759840551,1.2779916838408791,0.5269758739704866], [-0.6518981010977536,0.8745407308399856,-1.5963574332371322,1.2679054180945675,0.4099968212180263], [-0.46480686752956457,0.9217771428753339,-1.5266876398223685,1.1382426745675795,0.4778996312070509], [-0.4852820635337406,0.8218636910073019,-1.7767449598018135,1.1371130292343117,0.33381426065377506], [-0.41472873571993296,1.1622114863170214,-1.464996886657917,1.3123782230466363,0.2979393535788235], [-0.38817680425075585,1.0268858467994189,-1.5893492622912915,1.2621731986065745,0.435168936860357], [-0.4138872105825818,1.0639679744840822,-1.6793705616475731,1.0262313606697269,0.26542788201223744], [-0.32689002022123786,0.9411685364764196,-1.6079547828400877,1.222649045231373,0.44202772328420886], [-0.4534191990015764,0.9019382096024549,-1.6042217752367725,1.313476911051435,0.3118812623960018], [-0.3642257288812764,1.0568954661740726,-1.6408419112357935,1.3879191143662102,0.43428969659022393], [-0.4436033491575346,0.951120701402773,-1.7396961236381308,1.0696513684387006,0.1983721283216616], [-0.4195531379041837,0.9339655248016878,-1.6369085623469266,1.2113780206799198,0.32646374286948043]) -2016-06-13 20:20:46 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:20:46 INFO JobScheduler:59 - Added jobs for time 2000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Computing RDD for time 3000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Created RDD 13 with Vector([-0.3676131534931217,0.7205292433695121,-1.6148690586461516,1.318613134832133,0.2894000707552024], [-0.63227225792052,1.0480160164806966,-1.7186123125513257,1.3123133303248091,0.4835019323786134], [-0.5130669794355894,1.0744409567693907,-1.5423474727562314,1.4075951590487494,0.34400766625670576], [-0.6214787708363003,0.9941757670924664,-1.7059517928152113,1.1668792946674045,0.5351190865534026], [-0.5610084057337338,0.7736203857280138,-1.8315479385898956,1.3209260735925115,0.28582121198324356], [-0.37376999266386424,0.8253313331048502,-1.7924436246324382,1.3068506619750972,0.2325763177837035], [-0.5087037004414274,0.9607866442106467,-1.812291073591455,1.284458726328898,0.5328516211621773], [-0.3727437082846784,1.028767182687076,-1.7448007272489838,1.0984753757565202,0.3541211398247694], [-0.3814571980697206,1.0819060589675644,-1.733976127510556,1.2515969788209256,0.3640947425626813], [-0.38693583578076807,0.9054649043220095,-1.601255604158471,1.1869423507487187,0.27285330937505237], [-0.453879517976784,0.8265450932182703,-1.696264660015251,1.1219045234195482,0.3528336963815211], [-0.6440396833150053,0.9147072341447406,-1.574634779931242,1.2807375427757628,0.3400101945021918], [-0.5574376128841423,0.9786892277605028,-1.6539643457077906,1.2987195892477945,0.22091568760798275], [-0.3219564568616685,0.7344099605460779,-1.659291949575684,1.3277750279213558,0.23553171253321456], [-0.42287205017678026,0.8870628278654091,-1.7860892614211916,1.3638413188637932,0.35992654268460605], [-0.5114214572125407,1.0739488843585667,-1.803218590871721,1.2580620741259871,0.42523338095005675], [-0.49124237559267386,0.9657816736108445,-1.6156494419042913,1.3063855681095915,0.54155197550267], [-0.3951919316276779,1.144136446459301,-1.6606072772152598,1.4114586343482727,0.3224073809014021], [-0.439470602003815,1.0070997021461388,-1.4975209021956797,1.2890203902600919,0.22540217793727096], [-0.6679727959214776,0.999190248905179,-1.635890493454145,1.2938379558735829,0.20736771134970797], [-0.4118317638697319,0.9069236714993869,-1.7766219570089812,1.2254241772422947,0.3818774093786991], [-0.5019285203332016,1.071358441302261,-1.8681705848296024,1.3543938347082431,0.3694102461708769], [-0.43086686887184694,0.9871654497649167,-1.592715663423245,1.1292659913487786,0.24090456644129066], [-0.6554222411786117,0.9775977489456485,-1.7739871948427586,1.3756970876324615,0.5345682469155936], [-0.606670169993634,1.0414671201540915,-1.5816535152099802,1.2849832898961777,0.3443771799618759], [-0.6330347409363863,0.7614040653882075,-1.6420712005053097,1.344034328413472,0.5019032888808933], [-0.5268140556262974,1.1016458967835785,-1.581552701703922,1.1842999784487596,0.3683353834757425], [-0.49900894372274485,1.0645644739185587,-1.5355753177851752,1.2694091849907172,0.33468156327241705], [-0.31358398021960576,0.8302665811856854,-1.5422704772273437,1.1506707176948,0.37999408831157194], [-0.557673046622542,1.0913399597085338,-1.5871539417007638,1.0478537277843192,0.31421916981662246], [-0.4100320104849945,1.0606699393377843,-1.764930551709285,1.3173125309085043,0.32112514544554777], [-0.2578093453260759,0.9604640936063775,-1.6498739501347253,1.251662741314794,0.5352359752706277], [-0.5830353039132012,0.9147683830431232,-1.747368848835465,1.1923888734167842,0.4792523557272257], [-0.3091820573873779,1.0228642342621612,-1.472857793608509,1.165376819972311,0.3758448993415674], [-0.371190450361484,0.855956409330075,-1.82995473329898,1.3390818705810552,0.37113293274981823], [-0.5539807557276546,0.9677998945369973,-1.6350804443189515,1.3419255190805948,0.44899955987573437], [-0.5376287808107136,0.8413279828562651,-1.6762865574706214,1.19949852014792,0.5047576712678563], [-0.6003249423212255,0.9906964773703084,-1.5370089708944512,1.3072479529851824,0.430528835393197], [-0.4175368422724912,0.8187481538877078,-1.7996629160543502,1.1879999792808238,0.473510867372402], [-0.4828257087231799,1.0612823363596418,-1.6425767546040255,1.2270450796716112,0.39748339044894354], [-0.6332333955565639,0.9519761064158869,-1.5479445620477186,1.308465915443944,0.40088093604627134], [-0.5228584674134156,0.8768992895766171,-1.8336272924164618,1.1499724885170817,0.48668761341650246], [-0.36341836902693225,0.922366126760111,-1.4327493627862522,1.255336229334989,0.29637579738564857], [-0.45752242402693893,0.881182369510346,-1.8075367810499625,1.1836209711736738,0.5240597883932867], [-0.45127422400043055,0.7875661584033756,-1.9374943002165643,1.3022768816466794,0.3865261883663946], [-0.3385546242930144,0.8796112795269294,-1.6285471480854194,1.177302672296305,0.3611750358009876], [-0.3866867752880084,0.9445769802243262,-1.63589592788523,1.3115437594462822,0.30025415854590604], [-0.35087047853321573,0.8048561114260069,-1.6540296199948705,1.1069576251024358,0.3739108460295009], [-0.6118021930146711,1.0836229312504084,-1.6965417516081935,1.3814661848764345,0.5544639396063888], [-0.32579850447352154,0.9627503718472757,-1.7118868486570202,1.3378056459968657,0.26239820187294716]) -2016-06-13 20:20:46 INFO JobScheduler:59 - Added jobs for time 3000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Computing RDD for time 4000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Created RDD 19 with Vector([-0.4302807096972894,0.8599904088505619,-1.74473044235192,1.4559107913903748,0.3868065070182458], [-0.4809066717988696,1.1355083469098752,-1.568604721573235,1.1620770786476713,0.5052691040344791], [-0.37948367495671453,0.9612314328384234,-1.7288918432476479,1.1900277125124938,0.584148529955661], [-0.4830118079148656,0.767521114064722,-1.626523514538083,1.2344387914004133,0.2949434953618521], [-0.45982168221474795,0.9610478654584301,-1.4181098052958898,1.1960502956938037,0.35754207882888805], [-0.5403734624385554,1.0961985914256847,-1.4824794478647174,1.1331480033700656,0.4391107899439444], [-0.4078934785304544,0.9024846949827434,-1.772449379651882,1.1891755675558509,0.26886721197216756], [-0.4850713563393695,0.9854430300369302,-1.6392713106609738,1.1640485301086894,0.4058728213551233], [-0.5687863647866291,0.8969696803234791,-1.694332984322472,1.263598085578097,0.4916946976192132], [-0.4852057062952629,0.9746523770852169,-1.7152410694648534,1.290070394136892,0.15734923086759972], [-0.6042639508376677,0.9587258437320635,-1.796821218335353,1.2415150801911263,0.5036349732580911], [-0.5634160393957598,1.000927422915143,-1.7677443118596388,1.1056602828501068,0.3662560181172587], [-0.42260218853055576,1.038402090735988,-1.623454196568306,1.258759507363921,0.35817719144396376], [-0.6935865876752814,0.9767485891098648,-1.6119749119230544,1.3241871653009805,0.45959708474651645], [-0.34982681977824615,1.0697929536346964,-1.6430495277049175,1.5160135223968076,0.3720168274191172], [-0.4965183720272103,0.8867675212850072,-1.6326283681810188,1.1014099972707194,0.3464777829375711], [-0.4429523196454975,0.9569586297887442,-1.6975710388611154,1.3240706523955248,0.32888337316923105], [-0.6110986163387908,0.8513837967033338,-1.863122720264613,1.1805944382455067,0.36642329779978594], [-0.5770973903898762,0.908469634420286,-1.5969552518736576,1.2639442528889475,0.30334073173665643], [-0.6677676499752639,0.9993894428543435,-1.7484246181717757,1.3024629995033956,0.5254188589994735], [-0.43541597877213595,0.9518339300904892,-1.586032231723223,1.2859374872876397,0.3728538289653893], [-0.3981355023125287,1.0914065768869097,-1.604054951142783,1.5597686669581905,0.3200687403051776], [-0.3526167305034049,0.8582438505591754,-1.795079875034891,1.291753863011317,0.5193327226327783], [-0.5322224604485944,0.9225091285363366,-1.5406535924270095,1.1740734779955952,0.3985082190639408], [-0.459298613922177,0.8499917798311437,-1.7381604837279123,1.3363117012403978,0.39372367283850573], [-0.5029856763543168,0.9563691025906603,-1.6323146191672946,1.1564017424707789,0.2992277161858537], [-0.4702645909815308,0.9857893178047549,-1.6146159696178013,1.2941994476820615,0.29510677438190996], [-0.5010373585673528,1.0381084928077213,-1.6592946006629066,1.2275293781382213,0.6215806870134133], [-0.34251104673790145,1.0920152105960284,-1.4702129754795525,1.3052389503227437,0.5142842632664187], [-0.6014879705809496,1.0573110467882678,-1.6994508095500127,1.2345579924787826,0.27260709575033387], [-0.374393054126274,0.8520958347461538,-1.6227534538993043,1.400050919842349,0.3749155714488258], [-0.5537306132792297,1.0237199970470363,-1.5996513021748293,1.3277912515047763,0.3298858295939718], [-0.25019038644507985,0.8933568126093641,-1.6180540579171452,1.2514548524704747,0.4485167299028402], [-0.4083941282128968,0.9821808764315939,-1.7710803507794117,1.22246902401698,0.39616717157783654], [-0.6617567254520433,1.135205204672421,-1.7404704566578575,1.2634534028343156,0.48631393245586374], [-0.5102944111175142,0.7749069961072899,-1.3746153795885292,1.1761695768038691,0.44519535098184526], [-0.5273802259607313,1.0293844883271042,-1.8327600208787889,1.3032607095759106,0.4810318980867238], [-0.43680250540361804,0.818212719795147,-1.6397291545525832,1.2164204417976607,0.36911234859465697], [-0.44511271834694277,1.0039279649012336,-1.736482991573775,1.1407721179472239,0.32026695790267784], [-0.461571293170463,0.9623234284494812,-1.608651111923442,1.3054108488252942,0.2808744029932673], [-0.44626644352032524,0.9385912062499339,-1.5784088711897624,1.2996894478753678,0.3367841237735643], [-0.3458002275610911,1.0002670117410046,-1.556743986487141,1.220789398824709,0.3047413342602086], [-0.7006173067436341,1.0226530560056757,-1.7453818996615391,1.2468257997531524,0.43730757861422237], [-0.4921877365188532,0.9365903329931373,-1.6954478906187997,1.252660426269991,0.411305000260049], [-0.5866727516355298,0.8540617355784135,-1.7495419546675461,1.1992666379768737,0.5624798329629086], [-0.4552502733367031,0.9291860448712651,-1.5908193525700438,1.166718738832733,0.3281256456654895], [-0.3426613548769225,0.7538960431270546,-1.585524926597853,1.478740164978157,0.251131880834575], [-0.4680826818575833,1.1112444944704614,-1.6587653968723466,1.2870287092936261,0.4050920708933554], [-0.5624024717932173,0.880862871462712,-1.6303493150007178,1.1721463768303386,0.5696913393311782], [-0.48266079841446297,0.8916088275950524,-1.5393517554424276,1.1897790454674049,0.4411535715241579]) -2016-06-13 20:20:46 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:20:46 INFO JobScheduler:59 - Added jobs for time 4000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Computing RDD for time 5000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Created RDD 25 with Vector([-0.3816421005537525,1.0617035428188992,-1.7399461177422264,1.332980235806715,0.2916494841747297], [-0.6583584440016957,0.9555891409969888,-1.5454326525102315,1.1741091058996926,0.305220337835565], [-0.2989177604038451,0.8701688291879833,-1.8804582484676076,1.2986162711936382,0.35958131004690164], [-0.4939772883087695,0.9969220817446861,-1.5980382534286326,1.2969252872376784,0.37976918116852443], [-0.35668565131454827,0.8078147339434413,-1.657193160993533,1.359095599717262,0.31192105888186633], [-0.5450865703006346,0.8750391278919141,-1.7453306059026725,1.3721329076982913,0.42237633518141116], [-0.4917580165015238,0.9512146417235625,-1.9326523267650781,1.4266827266379667,0.3225713722197544], [-0.4048718474607664,1.0240256182146898,-1.6774401089659943,1.2669846165410654,0.33732641244233585], [-0.47579982131353826,0.7947369494624074,-1.7655419119652316,1.082577012553155,0.35401987740372537], [-0.35538929020910864,0.965963799724578,-1.7529404087985827,1.2055043918858341,0.3342486683177578], [-0.7014541659835856,1.034642549879647,-1.5544540159626516,1.3033886639865033,0.4054677613509847], [-0.43610110990224626,0.967077951705683,-1.5262368892289788,1.2639835242537958,0.3181179758885862], [-0.3425524324739105,0.9894095680520077,-1.7777745234498326,1.3091660388427224,0.331841648562285], [-0.4878205400579338,0.8560103963724833,-1.7352326850283464,1.3664274889095067,0.3984111315135807], [-0.7121859062829006,0.9130040407010276,-1.7874802517046422,1.1874648639293004,0.45017886399842666], [-0.696643078139105,1.1467663562988957,-1.7296410056290408,1.296144064682525,0.5048605187772839], [-0.4011230886898419,0.7361859073107719,-1.5317733987729902,1.2701183330425934,0.46062351610078545], [-0.45715694326129586,0.9190057703920946,-1.6904193500197242,1.3271044644288563,0.4842403797714084], [-0.5095249256026796,1.0659631362797184,-1.4402626928522713,1.3379987286264212,0.2673039786797317], [-0.4167094825032959,1.1091402127599272,-1.705921687753808,1.1183411370664753,0.48305727426817385], [-0.2924464534867928,0.8213969946455459,-1.662853823551024,1.3355427142685694,0.3770025002372693], [-0.4444725119583886,1.1046045679759258,-1.456164663000235,1.227847061522124,0.5255642346814823], [-0.4048329834921757,0.9788094275209757,-1.539852245269681,1.180160780953293,0.6787253334538687], [-0.38022990049474503,1.0144858953551337,-1.8072904612189131,1.3419776986257266,0.4041658441938728], [-0.3763555278351052,0.8998086161617835,-1.705061140148122,1.3495005676173746,0.5191631674639813], [-0.5540229100290982,1.0669438671830174,-1.636684794385797,1.3459012222977915,0.4181134505939225], [-0.45042550789662783,1.113240852339999,-1.486260571622101,1.2719922983443672,0.4102349177516307], [-0.5546179681493353,0.9938406825700987,-1.7084301934096207,1.3880541591389144,0.29690506797892435], [-0.5006405047598609,1.090292575906863,-1.7104271793907637,1.3062089077771892,0.47556018075456363], [-0.36520205819569046,0.9201397468605166,-1.7602021590969437,1.3876082296931265,0.2308718010458195], [-0.4938168714972593,1.0964903470900649,-1.7260349751744855,1.2048249569339422,0.40946925790527117], [-0.5092411838541296,0.9882559808204627,-1.7212908932580275,1.1127850487389395,0.3940121165600333], [-0.5882412062939182,1.1156935762589453,-1.8131619026625396,1.4146422480126442,0.31361816553861704], [-0.46990162437316874,0.9908156946435108,-1.748827835513362,1.3957588501434537,0.5512480921903505], [-0.5405868391855275,0.929247220281523,-1.5894550246414005,1.271255300937136,0.46529614772048844], [-0.46376208782353645,0.8174067828364345,-1.711422288030576,1.150388298327176,0.36268778857829526], [-0.4617415170232849,1.1203248336099854,-1.9346820049945674,1.1466858815338403,0.318614186382022], [-0.5200553635156823,1.1206383922863123,-1.721393160577411,1.2564342218266722,0.39290287510750027], [-0.45474634000888675,0.9538310968737261,-1.5572305892194418,1.2293659662686987,0.3951581444726625], [-0.31024997329419923,0.9626097763501761,-1.5774105560033171,1.3323705378196256,0.28213579823419327], [-0.6425546702182562,1.0413861558054223,-1.6696029184789263,1.3297020370998587,0.4667426127538354], [-0.5088483797878551,1.2170009632967245,-1.7143214767417212,1.2915402658283741,0.27813229259982863], [-0.5138048643277845,1.0537687232096613,-1.5306968187748822,1.2020601237200264,0.27213168784492703], [-0.5375432544160766,1.0418784698921373,-1.501800504998784,1.217708599285121,0.6443267111319482], [-0.5784571285851817,0.9518620188834632,-1.656094482882889,1.2785669143560554,0.34103467230048085], [-0.39057405488844044,0.995936983645342,-1.6323671359433234,1.271155031008547,0.3558239098317178], [-0.3682240577265206,0.8711375369867465,-1.6625627384453794,1.1969170451325948,0.47233733398400285], [-0.446008720311628,0.8976699326654971,-1.5289286019943875,1.2244855320582673,0.2984778353929738], [-0.5102538395816835,0.9986885141123109,-1.6414035100650266,1.2420020420446427,0.36032207135032196], [-0.6350165259371148,1.0900186871655364,-1.7907186442371252,1.4304360117451347,0.35387005412887984]) -2016-06-13 20:20:46 INFO JobScheduler:59 - Added jobs for time 5000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Computing RDD for time 6000 ms -2016-06-13 20:20:46 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:20:46 INFO TestInputStream:59 - Created RDD 32 with Vector([-0.5768797964115748,1.028505413551091,-1.7521414076692798,1.410931775189879,0.30527264662469095], [-0.40591177777424625,0.8661878776717759,-1.7310200236861213,1.3594055750354543,0.3299548132445285], [-0.6383561683498007,0.9259033203935216,-1.6881320011035645,1.3172994752602565,0.3214908371039436], [-0.624062828319168,1.1219523756569185,-1.5603263506923595,1.228964619537528,0.35487057209905903], [-0.4734064808910865,0.9649955190976767,-1.6068693471541338,1.3196307767213002,0.37116726129033706], [-0.6101841770121739,1.059654359501757,-1.8099667264246553,1.3759685981009824,0.5438261473052933], [-0.18850356552447423,1.1440264277756391,-1.724865192544724,1.1359044353122627,0.31757393897273056], [-0.41254200355821263,0.8607204799101216,-1.616620448535661,1.3427363220425623,0.4109728277211949], [-0.393734447592108,1.057690953958827,-1.6695747980566427,1.327034390345423,0.13819106599587025], [-0.32090427274943434,1.0651687359751123,-1.6980368198411557,1.4017902860954718,0.36022562482340503], [-0.6053061980873911,1.0041632375635317,-1.7820663797212062,1.2269162643461793,0.4840114520476853], [-0.27268808986968307,1.013170380627259,-1.6005981535949472,1.4382064950763866,0.3892224005152082], [-0.535422550377724,0.8375173015602825,-1.661788348017347,1.3227481832299182,0.311533159171815], [-0.4009368458319039,0.9897832871295871,-1.6696688300152116,1.4424037410980737,0.32734398317968916], [-0.6825087101818776,0.9056073621397804,-1.659854864696752,1.2759644672519115,0.43884955280528926], [-0.43179250129551383,0.7465728726494555,-1.5636729803345486,1.3457117728614754,0.5721971492726551], [-0.526304418227463,1.0501002903902692,-1.5968858098432406,1.2982785754583357,0.37494021304786934], [-0.4320225453412649,1.1715904807145918,-1.5178148547272063,1.348200261386646,0.40779395798035983], [-0.42221995667102147,0.87623418086733,-1.73837906100808,1.4183919259975142,0.33228674704284467], [-0.5629520941212462,0.944212875923361,-1.5546071736198348,1.4519582067498829,0.262200899044119], [-0.30845271114048184,0.9681639591372486,-1.688842111249905,1.3763468548667668,0.3173125681909059], [-0.5198567190391367,0.9335053916544206,-1.509472274649017,1.454415814887731,0.4589687217941236], [-0.5711775989363378,1.0215061238027188,-1.6224000734110398,1.350665861834858,0.36668462127595286], [-0.39949682119974983,1.0408092246134175,-1.6587225284400737,1.301865863792736,0.4563742176523199], [-0.4902566261487769,1.0992586451801099,-1.4473476422759635,1.2309123590548212,0.2544829822903507], [-0.5293628624912633,0.8432327849564988,-1.7669603672683885,1.2911210280012069,0.41983427336223667], [-0.5738614502843032,0.9784937768658387,-1.7194271904022407,1.3116656980237082,0.46218317857671776], [-0.5902501988138692,0.9423215775018176,-1.684040151093259,1.4235274237923208,0.2039286490216381], [-0.4777727934676478,1.027586212069216,-1.7495753442806758,1.2757769414973703,0.3810309831295745], [-0.6566621783684151,0.9641681686268665,-1.5286420868141977,1.508411206136107,0.2501550388463756], [-0.46296281092785885,0.9727194845614605,-1.6171422039154086,1.3434828829348795,0.07809723939214058], [-0.5738547760098169,1.1077074999997352,-1.739110700752265,1.3494171899976157,0.3222123114107347], [-0.4002887849620737,0.9619411680270862,-1.663509769541054,1.2908149710547212,0.4244227376717664], [-0.4665533215112465,0.8596395857982323,-1.7467795346659782,1.251224446006115,0.42794754034954735], [-0.6374258626266393,1.0514903728245393,-1.651303971310752,1.236884794719695,0.5249538337308562], [-0.48340361610998744,0.8198136168919055,-1.690727257833199,1.398396991016531,0.31841739933411595], [-0.5410633313507479,0.8973900953721683,-1.4724867408929736,1.388151513303225,0.47893645155994446], [-0.3039096405457719,0.9155005236200422,-1.7173365893442798,1.256097860517092,0.24819750391337136], [-0.7090962179115853,0.8944749271173439,-1.8839838110448957,1.1446445350131944,0.22840975233443309], [-0.5100953498526793,0.8107010460411471,-1.7303477866873604,1.1647356509362308,0.4418371093135496], [-0.2962087734742913,0.7242125014742148,-1.5984563979400566,1.0588259317312654,0.28436170748337264], [-0.5505122810716748,0.7806970076803257,-1.72088933624607,1.4230022424760869,0.32673184311928194], [-0.45127942352855155,0.9959060503010477,-1.5524186364009327,1.2366972856510523,0.3578155222046753], [-0.5102066277582917,1.0949182323987419,-1.7142222646495175,1.1664611293679565,0.3602965879997574], [-0.20421136622466618,0.9866371147144095,-1.7538283133815977,1.0465139384173783,0.41274915410898083], [-0.3678643518086112,1.1128928060624481,-1.7416958328862653,1.1712943420281356,0.522586652590277], [-0.4272255324098842,0.9505411417476373,-1.5398631578697235,1.1670050581250606,0.18826195051488878], [-0.6294443147944699,1.0182211296265207,-1.6114292740208946,1.5494854908967275,0.2354819914065145], [-0.46052379116984477,0.8965751438720174,-1.6887957613207942,1.2678155037585463,0.3277249457164221], [-0.4661555388594827,1.0176554636610373,-1.660202673672577,1.1543960943838145,0.41490582984943747]) -2016-06-13 20:20:46 INFO JobScheduler:59 - Added jobs for time 6000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Computing RDD for time 7000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Created RDD 38 with Vector([-0.5184134212251081,0.9455318561030377,-1.7249072888539423,1.2476376390078272,0.539371371249466], [-0.4839353019153916,0.992343730650464,-1.4037670310964987,1.3650053852483648,0.287928402259341], [-0.45217444521287575,1.0425364789095595,-1.7771813152553952,1.2241390773159404,0.3053067128704871], [-0.34943560877945984,1.0407179160044182,-1.5177836780575653,0.9824846085205396,0.2603543256712868], [-0.45829448575134074,0.9725249117979422,-1.690056167350807,1.035512584568622,0.3026754916713263], [-0.42458145588018753,0.7854084174090814,-1.7649589009106188,1.1975914903392633,0.6551885852829413], [-0.5104497237497339,1.0402201914534017,-1.6859518565969145,1.1391556996625576,0.18881595045601088], [-0.5588856532488075,0.8709355475677711,-1.4735465340859104,1.207362168238276,0.5886898848511445], [-0.5286914346418723,0.9318873502365327,-1.654993790470044,1.274056873049218,0.29412525529417294], [-0.5420811906362676,0.9995645356070102,-1.6040227637757893,1.3412654820039975,0.3950459929854693], [-0.5105371229433685,0.8870982484234274,-1.6366208840493932,1.344244897498087,0.2842199976348651], [-0.5758159328886613,0.9361950961962593,-1.5151431024376867,1.2963264217469896,0.4195802203163277], [-0.46650906956232724,1.0883479790237522,-1.6209503836486092,1.2506119015001946,0.5239087792157935], [-0.49800785133272085,0.9121648111767798,-1.5664537098618618,1.4303891258276538,0.23796160092517643], [-0.6200401622139905,0.8819930763699753,-1.7712553265237971,1.144151394858828,0.37228012783293096], [-0.43683215654432145,0.8386127968983885,-1.6402554451670102,1.129209493050289,0.3612473529185551], [-0.4079221564079838,0.9656864104202829,-1.7359335721136713,1.4421338682074127,0.3616284206434529], [-0.43442591279896603,0.955449363348641,-1.5334677832140913,1.1666726886559795,0.5058368495019173], [-0.6457828019918121,0.931131898174338,-1.8532037937152797,1.0375316080434018,0.3133563527606291], [-0.38369598973754027,1.040856056086344,-1.7594025439463896,1.4917751006832976,0.45495744983872133], [-0.30166559339288085,1.0342180102112146,-1.5650152370405155,1.2673629485315197,0.4158807517084573], [-0.28580825731967757,0.8865395334100423,-1.4737235251638032,1.245008984525024,0.36435255784505255], [-0.34071421524008416,0.9427461052254856,-1.6571555985762556,1.116205755573728,0.3440594901972376], [-0.5707704618700846,0.9955277694976673,-1.815098893846346,1.2976867834380266,0.3708933869585827], [-0.6095164341318,0.9322989548597498,-1.6576644540187038,1.3001089252401639,0.32781904617649943], [-0.5884784232045771,0.9963121018549773,-1.4726908596796255,1.1232169328419297,0.466258074909203], [-0.6351651621700529,1.0173421935079714,-1.7981429560407272,1.0336059974520695,0.28162584897435694], [-0.38203401021631267,0.9790771913538491,-1.7244683271662424,1.288084387752797,0.4312827591331814], [-0.37183909143628785,1.1926117173870312,-1.5166716166223042,1.1696263300990144,0.4515586032102953], [-0.47462520704983,1.0498070539706301,-1.6863851371590932,1.2965114835042095,0.4645837570114517], [-0.35874130630231327,1.0042819076442355,-1.6407835501913508,1.214144240421076,0.35534792382876684], [-0.41389636789198914,0.9508170625312302,-1.6498089027313338,1.4272197180381103,0.374553819362672], [-0.32132785261092967,0.8535769504185499,-1.672835438070241,0.9421132176631009,0.24740658936307364], [-0.48253792736985773,1.0429236941137325,-1.4939460450129358,1.2832409419178106,0.40323476275374437], [-0.391512958705594,0.8859909853510953,-1.607045573914962,1.1770226037327562,0.5198479967492], [-0.4833163026675927,0.9200058086737815,-1.8188555219632765,1.2111735788924018,0.39618714901430313], [-0.2913135273832381,0.9301485161935364,-1.6975841978124846,1.4065399033140502,0.28453881400648173], [-0.3073756680829076,0.8325242716417236,-1.6753209327630654,1.4075134801045086,0.30351866480787215], [-0.2775051790445552,0.9452680429408632,-1.6224110577930677,1.1778711923142833,0.5531745712889398], [-0.3047686297259775,0.9468967188067203,-1.6929112749188848,1.4636396431851675,0.3586449109599954], [-0.5956899619218391,0.813236818549011,-1.757649523640403,1.2604654777105053,0.3185144577112101], [-0.32794499152459505,0.8140339815697406,-1.8666906537843855,1.2572887839007447,0.23549894606903882], [-0.40686690942678344,0.8619998816707273,-1.6319579560604098,1.3664940728963186,0.422156696202466], [-0.4138601150243667,1.021828823586237,-1.54841612428864,1.2398622089963618,0.5070042357490803], [-0.331869975870002,0.8782404384527787,-1.6635875859844171,1.2435143760020457,0.32101906837863314], [-0.49700214293681766,0.9856184010338479,-1.8232422669575907,1.2421051455581202,0.47592591320323163], [-0.535909591266969,0.9807383109826389,-1.6974325972167972,1.334548321075211,0.23946350824403165], [-0.6444021824207316,1.1849780210499694,-1.5121795557147575,1.3604386920675193,0.31690181205044465], [-0.49746220541003966,1.0643703042656938,-1.8499798436118189,1.1357972819777942,0.5780379400362996], [-0.4383285608503117,1.0163805234573544,-1.9030246206122645,1.438368352860807,0.3293850463477472]) -2016-06-13 20:20:46 INFO DAGScheduler:59 - Registering RDD 7 (apply at Transformer.scala:22) -2016-06-13 20:20:46 INFO DAGScheduler:59 - Got job 0 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:20:46 INFO DAGScheduler:59 - Final stage: ResultStage 1(apply at Transformer.scala:22) -2016-06-13 20:20:46 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 20:20:46 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 20:20:46 INFO JobScheduler:59 - Added jobs for time 7000 ms -2016-06-13 20:20:46 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:20:46 INFO TestInputStream:59 - Computing RDD for time 8000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Created RDD 44 with Vector([-0.374705866875181,0.9840691262184086,-1.5891551104970965,1.4814595125988181,0.47065837960007695], [-0.288810885883263,0.8437842751113669,-1.5395954061634287,1.1874811761270259,0.021249409500126437], [-0.4606335450965845,1.1012472510905122,-1.8287426699010534,1.24172318938614,0.4710333368456244], [-0.42944940507321583,1.0104988720375858,-1.5751317017658801,1.2831547194950401,0.3512701703197814], [-0.5152127056418667,1.1709583832128079,-1.6554391510043867,1.1397001494774806,0.23397277605773995], [-0.4399479870409779,1.024335104965922,-1.7045108897162344,1.196833551305392,0.4117115230011916], [-0.5430543625505585,1.1370095574712402,-1.7176429060334013,1.1901279759925694,0.46070443041212095], [-0.3968714897698767,1.0926860330748331,-1.6334286394688688,1.228234808051336,0.4665607383628074], [-0.46215342653947683,1.0343329141616782,-1.6499943761334228,1.337530812749641,0.23114556003826187], [-0.5000235665427208,0.9151728338053156,-1.8055132514640995,1.3037686620331144,0.2902025184248946], [-0.309233049825866,0.9728959274674948,-1.6392691162173578,1.101247705752869,0.3496423465401415], [-0.5981691090699208,0.8739296001671861,-1.5687282698937923,1.2550952881121755,0.46308264317451964], [-0.425561195913103,1.0676958602044149,-1.7135165364767921,1.3524692002054561,0.23228520033456318], [-0.7043177723021486,1.0283311625585259,-1.7478644671187442,1.2858120358708933,0.3418710936510813], [-0.41727892194293686,0.8625867727705405,-1.7914100057674134,1.1786792695315231,0.4917512427338196], [-0.40174243931692544,0.7828098802704966,-1.5073762714652954,1.281118669072956,0.44887930601066905], [-0.35310979651234775,0.9745364825956087,-1.6392317059509176,1.4513644518189859,0.3110119088453469], [-0.6853829628094648,1.1253397147829511,-1.4920081564639467,1.231135544811062,0.3139314829419366], [-0.44152016739085975,1.1304790288137783,-1.7325115907360644,1.3251089521914843,0.479567851123668], [-0.5526275004039871,0.9674440446022308,-1.7971453051937036,1.1664342514118335,0.31260632993291754], [-0.46562594489923503,0.96412818249357,-1.7756506925990838,1.1820367323525278,0.37368762092821634], [-0.42414895043860573,0.9369437276073684,-1.6228073393039317,1.2110129935336054,0.33600782254298567], [-0.4507513844294476,1.008422902581697,-1.6947838717172434,1.1258385535052762,0.23649573041213182], [-0.5011767543824581,1.0367482963168666,-1.8619227324329928,1.187686906059264,0.41060169959227016], [-0.29061733919433663,0.9125980333805904,-1.773384852821433,1.2318813347419482,0.5360023941871213], [-0.47558633395179245,1.0112018987992073,-1.6443927870523196,1.1868477340320063,0.378412637239359], [-0.47217857076866115,0.9027887018339104,-1.7359968088612976,1.372505255254142,0.4024631611010435], [-0.5635782393384723,1.0418520168542464,-1.6371483392749435,1.3046794831870752,0.36435743882876215], [-0.3566584096699375,0.9045378099174617,-1.6027843810019622,1.308376882406936,0.450420525347206], [-0.5161548558091542,0.9362466126941537,-1.543853038663444,1.351176902637456,0.37019671664199505], [-0.37734135089435306,1.1354760893808167,-1.5928707071728756,1.4230797310043477,0.5395830646701685], [-0.4935941975274797,0.9431897551602951,-1.7363999282764386,1.2592777434544042,0.34477211805728875], [-0.5399690948357754,1.0773952946269298,-1.623286131549866,1.3571017009557331,0.3820149575212495], [-0.47470272833792704,1.0279092716939924,-1.5583033476707477,1.2928147173688327,0.12854158780432537], [-0.3932748299826361,0.9168968642381998,-1.901294092833649,1.259188794957511,0.4011688713714673], [-0.3557493574970649,1.0003577059585267,-1.5111904842067994,1.3123818859934535,0.3019665980708428], [-0.38849198387448786,1.0634537152100978,-1.8742015500852314,1.207616787308903,0.527158124827907], [-0.32516837051884395,1.0806577326663074,-1.611781542302308,1.182593379846847,0.5196789294606137], [-0.5212725898858087,0.998490506190397,-1.6184763965153202,1.3731447165706872,0.29855378648402087], [-0.40257752034798155,0.8314340557777532,-1.6850156321125218,1.3427297403898506,0.5156164947536159], [-0.4889473969173199,0.8999745071099362,-1.4468367122261747,1.3090812041423647,0.5108182965913203], [-0.48274282778056277,1.1537075465803446,-1.5811276311378704,1.287632315416058,0.50882779799236], [-0.23362208323850905,0.9548750423156142,-1.572161725609678,1.6027003014260315,0.3939518404103326], [-0.4292988792888861,0.9366381149644413,-1.8250064321036654,1.2358072562830058,0.18892247590763034], [-0.484748914270206,0.7653236279449509,-1.7486834183984747,1.2949268753185315,0.39683334762547673], [-0.4399736279490563,0.9634644976145054,-1.7480413166448512,1.1775955441636703,0.36332416278682794], [-0.48373350642316953,0.9259530051332495,-1.7977337219360736,1.2876089889322726,0.4497462132788797], [-0.38070426405120866,0.8838236645507052,-1.6753928124899833,1.2449514905082557,0.3276558558550576], [-0.5717833110401351,1.0273299096906752,-1.6190879043151574,1.1860455839092494,0.32636997750947594], [-0.5231330958628938,0.924256801489448,-1.5484246875687981,1.3381528852011608,0.3757180006852602]) -2016-06-13 20:20:46 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[7] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:20:46 INFO JobScheduler:59 - Added jobs for time 8000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Computing RDD for time 9000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Created RDD 50 with Vector([-0.4026578572929527,1.0001062834205203,-1.5617908885948877,1.462733521830009,0.2061287167736916], [-0.3965287353033582,1.0683726645043978,-1.4873730061169563,1.1460792990307642,0.27746130384846085], [-0.4938197206621628,0.7709438014433136,-1.6076436749339549,1.0982028796258234,0.24486596655062928], [-0.4777650011847253,0.9489155740228515,-1.476414143798642,1.3062514135704215,0.6114439757268249], [-0.4285977940802228,0.9382568588433707,-1.48976698594476,1.1698457774900977,0.297712575086562], [-0.5993896749484521,0.8629058534578911,-1.6510698754517374,1.2090117714723978,0.5458042851487852], [-0.5321474831529757,1.0849054231681536,-1.667568850178397,1.4161664775537062,0.4493689535824759], [-0.6723705689897437,0.7517542425048965,-1.663645669684176,1.2240252996527488,0.346848907683753], [-0.40115735788941104,0.8163474979541504,-1.7694648704001712,1.1073947331922183,0.3856322637520626], [-0.4787497898360035,0.855418622108937,-1.7422359123781868,1.2371448524727167,0.5156131237949001], [-0.5321048883779993,0.7970778523677844,-1.7616773188005117,1.3821414318824692,0.48832517040159346], [-0.5285360880098152,1.0597395124384508,-1.6417892576104653,1.3464020747697325,0.31849105791627297], [-0.5324435853239631,0.9841563147023031,-1.5943852209495275,1.2707723748060116,0.4139910360672806], [-0.37531742848811983,0.7874289125205598,-1.6479869931712725,1.3181636612914271,0.3718789043065717], [-0.4560433134330382,1.0759812484577316,-1.697865869705092,1.2725177127861855,0.06883328163775021], [-0.45513942313770217,0.9498671339799809,-1.6933930804659583,1.3025139463978586,0.5350768600248874], [-0.526256522617052,0.8727003678813281,-1.4356120595661468,1.29993914263913,0.43387328603333664], [-0.3923483046111808,0.994060772829843,-1.6451266300509706,1.2983503204985913,0.3703765349262901], [-0.4374913726020475,0.8955789511391181,-1.4728897306445088,1.1357862983779858,0.30881407016500095], [-0.41951697820602707,1.0007695283696842,-1.708286281016387,1.4190975260604537,0.28731659836668877], [-0.6750333217393494,1.0688779823591679,-1.6846746627252691,1.2962992487600882,0.5030936797752793], [-0.5956866906574221,0.8121299274245898,-1.692929629203368,1.2338548732523056,0.38371772482125543], [-0.4303591738987757,0.9459620785706632,-1.884551120494729,1.4937212054647857,0.415792749586519], [-0.45504157760203423,0.9027897906482127,-1.799761299703666,1.3040399621270629,0.5736987408004388], [-0.5593669081347996,1.0931782446321396,-1.627955485174887,1.1391759184235553,0.39801309200504786], [-0.3825955534231694,0.9655263256725013,-1.7068123920171723,1.1861538643513039,0.2764471843224494], [-0.3587313623839552,0.9371265382727668,-1.487908236071683,1.3184799953634567,0.4370396706398445], [-0.475038660091078,0.797460346861311,-1.6835937249660264,1.3120302511529962,0.17031892070731325], [-0.4663190378327658,0.991352295299833,-1.6444316855302834,1.51526349576866,0.3929422369724258], [-0.5040823137977553,1.0575372259159488,-1.594322659684267,1.2971698194040933,0.5092825217956699], [-0.44790417664079346,0.8518362899333143,-1.648211585312696,1.1839806869130056,0.40832085080824754], [-0.537755895075537,0.9806637869508009,-1.7656906916731008,1.2433660326767015,0.42887767735060545], [-0.34293660736335746,0.7935119912222357,-1.6359022010243611,1.3593857228798436,0.3967833959145234], [-0.6208695835786292,0.9640100782092327,-1.874229161549032,1.343981009171891,0.20123531286937832], [-0.6723717817990262,0.9002960054466735,-1.6910864401441592,1.1818255352705958,0.5433986930426408], [-0.4588708699931968,1.2092994554375371,-1.5324768096769932,1.4083576723657105,0.3708605878943178], [-0.560638238383703,0.9092570275383438,-1.473047966472027,1.3497511209299733,0.3467764467476322], [-0.4409803705877342,0.9370238683801353,-1.572432943897284,1.41498798111251,0.2748026192628795], [-0.38741643074715315,1.071718105146961,-1.624146896559199,1.1401522638836572,0.3872910322155703], [-0.5140862086906481,0.8211106116047733,-1.6027037015807561,1.1677747630844688,0.2659604974429464], [-0.6693385873907269,0.7554393139905898,-1.7108087289854104,1.09991932687034,0.41821242261960917], [-0.3282503845126481,0.9257033715775067,-1.5235939065144024,1.1889128563024831,0.3368716615939142], [-0.5366461042406446,0.9240666515119298,-1.8135909127248846,1.3575511869067935,0.37505000938261085], [-0.39294848413804057,1.0124585311856609,-1.575086388417685,1.2467079941490156,0.40416560148386643], [-0.48738816985096806,0.8787565657383682,-1.810281961839409,1.3779890105636297,0.46019825317610896], [-0.36592476270170865,0.9132457437551841,-1.7089939281056865,1.1718159447241387,0.37244792446782193], [-0.40751855185193425,0.9541054981051484,-1.4879650472104635,1.1817057016562598,0.44919665548504983], [-0.29317450049605076,0.964443479319753,-1.656028808081747,1.2586398699632722,0.2526740377633023], [-0.4615968319838038,1.0243528061825597,-1.5054257187951623,1.351376529833804,0.42971544189574107], [-0.47988177538312266,1.0112860256804128,-1.5404476412782737,1.233668528335028,0.2028012939752821]) -2016-06-13 20:20:46 INFO JobScheduler:59 - Added jobs for time 9000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Computing RDD for time 10000 ms -2016-06-13 20:20:46 INFO TestInputStream:59 - Created RDD 56 with Vector([-0.4889010904178065,1.0197902904388123,-1.6434083794608938,1.2273894895374848,0.29880056758004836], [-0.3690054860756016,0.9083136794567547,-1.5975355891184535,1.1937534562371308,0.3514272190683493], [-0.5737783702048262,1.2273536117223582,-1.4617006566984774,1.342244921629087,0.3377586288639175], [-0.48387316003666864,0.9673070026559238,-1.716725721249248,1.2214389230026372,0.37779842180031725], [-0.3777277713061831,0.866118273628028,-1.8487356572074127,1.1851350383431707,0.5215534466266145], [-0.6076926717533072,0.922880686625322,-1.7313060678714964,1.3632488420681186,0.3176699282425657], [-0.3802374789807806,0.8439055679755229,-1.6219613687183736,1.1809867291088374,0.4174630121979538], [-0.567466556581487,0.9292300369161763,-1.5542588287823018,1.3568593851021735,0.38458895772840795], [-0.6764387691659441,0.9816754055716941,-1.6652926928722982,1.3324217076474294,0.2844320904428872], [-0.584138124369403,0.9651833514289697,-1.7100579024456408,1.3025648636881102,0.3575320807936941], [-0.45548159931880944,1.0181220363355081,-1.6506653550843506,1.1304474598171783,0.4285968106743953], [-0.5370002084128622,0.927571736048293,-1.7228464440630433,1.2925613214036213,0.3169164091166503], [-0.47996613069291666,0.884366449915093,-1.7212355955272085,1.152597880658151,0.43962468554199663], [-0.35493034949860885,0.8984344817504613,-1.65218132155661,1.3068108975806394,0.354531357717713], [-0.43226118157508875,0.90725353631203,-1.7628881371748841,1.314870897760879,0.40214500076142384], [-0.4387259831220203,0.9621751931555681,-1.7268764217836796,1.1093705777029121,0.17867972251315242], [-0.44765902746822306,1.1186404105474657,-1.750942737318709,1.322673860044468,0.41609066577279374], [-0.40351581789921565,1.046340578817358,-1.7120920286690795,1.244893797129159,0.3833023298379642], [-0.3093624994668177,0.8356650811564297,-1.8642891852543533,1.217524168702485,0.2103617986137151], [-0.2990644638630094,0.9070655034040485,-1.6094469406493193,1.1592980925831848,0.43244854200109223], [-0.5596021456993305,0.9019708087164819,-1.5105356278212536,1.2598551179472526,0.3428536586628008], [-0.583393764986965,0.9748568871770861,-1.5302538126845806,1.301546791122464,0.21927558650218368], [-0.49772324825286274,0.9716299176892979,-1.6169165069063722,1.3531471880552017,0.25858373282508224], [-0.43106771684572104,1.141003137647425,-1.7578104686857339,1.1835903210552496,0.5254045668739125], [-0.5997306192424074,1.0040726704610177,-1.8227253165238155,1.2317004811683654,0.46876771655575034], [-0.38252412898165267,0.6654527000305275,-1.6468024292953505,1.246225339488523,0.3090133585357682], [-0.44821229943324725,1.0038256342470713,-1.6968551076177416,1.4529627079929077,0.3454589917432014], [-0.43535992691090786,1.0110816584854143,-1.7453606983094134,1.248210478232135,0.2891585839680707], [-0.3652348315036368,0.8958964249587966,-1.9606863598829045,1.3389257198065394,0.2618900608154475], [-0.49376528406043635,1.1421599774098976,-1.7157624036331927,1.391772429187865,0.2876879628208894], [-0.4515063147230089,0.9428403433366573,-1.76394465423704,1.203639940182186,0.5058415243772909], [-0.566229150629525,1.0128616873896932,-1.6880365432069786,1.2877554523210368,0.1629872361106314], [-0.47658344463796903,1.0862736976313303,-1.7934284966158631,1.2272771650263852,0.41680211738171946], [-0.3618808351944353,0.9417137306142976,-1.8600076455984111,1.2206352320840592,0.3461749973804146], [-0.6214757024643818,0.9952948941676851,-1.62097329834717,1.3958184351557885,0.46499512939060494], [-0.35996348236796494,0.9954644862470238,-1.7933270864489543,1.4005533772243737,0.5639356774206108], [-0.6006970016720174,0.9686450943470077,-1.7310975948083858,1.2398543489295077,0.4483279499267394], [-0.4592797597414643,0.9631529442377594,-1.6797024055855931,1.4289800788354718,0.28667160760155463], [-0.32700881076504584,0.8147754680205297,-1.763026614596352,1.3194735072019403,0.36816545181467747], [-0.4305345983795718,1.131285559002078,-1.7583813300937976,1.363545141597296,0.2601165464857354], [-0.448497878644767,1.0701920363594992,-1.6308801358271674,1.2674713459930484,0.49959128317905865], [-0.3686293381026533,1.025599917053419,-1.5433629968114364,1.2801178859900646,0.4174115399252367], [-0.574113693976958,0.9757976338077295,-1.6388778822719372,1.2759164798546365,0.5542142157229283], [-0.4357071400660946,0.8338147376622728,-1.8285888680646112,1.271233025944097,0.270178918164786], [-0.47158188300536635,1.0122183541160776,-1.6010520252577112,1.5447804629514494,0.3533722934153737], [-0.5307926544952695,0.9107879044274507,-1.4490342775396574,1.4031526476841614,0.3809780658985748], [-0.4953668915084064,0.9298227962550192,-1.5651812648587027,1.11568589286655,0.4774983034589586], [-0.7383552095915331,0.7626241115292196,-1.7008182872047266,1.2502877442685654,0.43135198632241045], [-0.539114438923603,1.0489102632469824,-1.6736818095705865,1.2768121659352345,0.2213641562753643], [-0.46002266280222004,1.0294295650038185,-1.6033579773810744,1.407522155806443,0.5223722922756932]) -2016-06-13 20:20:46 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:20:46 INFO JobScheduler:59 - Added jobs for time 10000 ms -2016-06-13 20:20:46 INFO JobGenerator:59 - Checkpointing graph for time 1000 ms -2016-06-13 20:20:46 INFO DStreamGraph:59 - Updating checkpoint data for time 1000 ms -2016-06-13 20:20:46 INFO DStreamGraph:59 - Updated checkpoint data for time 1000 ms -2016-06-13 20:20:46 INFO JobGenerator:59 - Checkpointing graph for time 2000 ms -2016-06-13 20:20:46 INFO DStreamGraph:59 - Updating checkpoint data for time 2000 ms -2016-06-13 20:20:46 INFO DStreamGraph:59 - Updated checkpoint data for time 2000 ms -2016-06-13 20:20:46 INFO CheckpointWriter:59 - Saving checkpoint for time 1000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-1000' -2016-06-13 20:20:46 INFO JobGenerator:59 - Checkpointing graph for time 3000 ms -2016-06-13 20:20:46 INFO DStreamGraph:59 - Updating checkpoint data for time 3000 ms -2016-06-13 20:20:46 INFO DStreamGraph:59 - Updated checkpoint data for time 3000 ms -2016-06-13 20:20:46 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:20:46 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=0, maxMem=766075207 -2016-06-13 20:20:46 INFO JobGenerator:59 - Checkpointing graph for time 4000 ms -2016-06-13 20:20:46 INFO DStreamGraph:59 - Updating checkpoint data for time 4000 ms -2016-06-13 20:20:46 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:20:46 INFO DStreamGraph:59 - Updated checkpoint data for time 4000 ms -2016-06-13 20:20:46 INFO MemoryStore:59 - ensureFreeSpace(2428) called with curMem=4744, maxMem=766075207 -2016-06-13 20:20:46 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:20:46 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51099 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:20:46 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[7] at apply at Transformer.scala:22) -2016-06-13 20:20:46 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 20:20:46 INFO JobGenerator:59 - Checkpointing graph for time 5000 ms -2016-06-13 20:20:46 INFO DStreamGraph:59 - Updating checkpoint data for time 5000 ms -2016-06-13 20:20:46 INFO DStreamGraph:59 - Updated checkpoint data for time 5000 ms -2016-06-13 20:20:46 INFO JobGenerator:59 - Checkpointing graph for time 6000 ms -2016-06-13 20:20:46 INFO DStreamGraph:59 - Updating checkpoint data for time 6000 ms -2016-06-13 20:20:46 INFO DStreamGraph:59 - Updated checkpoint data for time 6000 ms -2016-06-13 20:20:46 INFO JobGenerator:59 - Checkpointing graph for time 7000 ms -2016-06-13 20:20:46 INFO DStreamGraph:59 - Updating checkpoint data for time 7000 ms -2016-06-13 20:20:46 INFO DStreamGraph:59 - Updated checkpoint data for time 7000 ms -2016-06-13 20:20:46 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:20:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:20:46 INFO JobGenerator:59 - Checkpointing graph for time 8000 ms -2016-06-13 20:20:46 INFO DStreamGraph:59 - Updating checkpoint data for time 8000 ms -2016-06-13 20:20:46 INFO DStreamGraph:59 - Updated checkpoint data for time 8000 ms -2016-06-13 20:20:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:20:46 INFO JobGenerator:59 - Checkpointing graph for time 9000 ms -2016-06-13 20:20:46 INFO DStreamGraph:59 - Updating checkpoint data for time 9000 ms -2016-06-13 20:20:46 INFO DStreamGraph:59 - Updated checkpoint data for time 9000 ms -2016-06-13 20:20:46 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:20:46 INFO JobGenerator:59 - Checkpointing graph for time 10000 ms -2016-06-13 20:20:46 INFO DStreamGraph:59 - Updating checkpoint data for time 10000 ms -2016-06-13 20:20:46 INFO DStreamGraph:59 - Updated checkpoint data for time 10000 ms -2016-06-13 20:20:46 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 20:20:46 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 20:20:46 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:20:46 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:20:46 INFO CheckpointWriter:59 - Checkpoint for time 1000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-1000', took 27278 bytes and 311 ms -2016-06-13 20:20:46 INFO CheckpointWriter:59 - Saving checkpoint for time 2000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-2000' -2016-06-13 20:20:46 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:20:46 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 20:20:46 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 20:20:46 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:20:46 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:20:46 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1159 bytes result sent to driver -2016-06-13 20:20:46 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1159 bytes result sent to driver -2016-06-13 20:20:46 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:20:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 372 ms on localhost (1/2) -2016-06-13 20:20:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 410 ms on localhost (2/2) -2016-06-13 20:20:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 20:20:46 INFO DAGScheduler:59 - ShuffleMapStage 0 (apply at Transformer.scala:22) finished in 0.434 s -2016-06-13 20:20:46 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:20:46 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:20:46 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 20:20:46 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:20:46 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 20:20:46 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[26] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:20:46 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=7172, maxMem=766075207 -2016-06-13 20:20:46 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:20:46 INFO MemoryStore:59 - ensureFreeSpace(2657) called with curMem=12436, maxMem=766075207 -2016-06-13 20:20:46 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 20:20:46 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51099 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:20:47 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:47 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[26] at apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:47 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 20:20:47 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 9 ms -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 15 ms -2016-06-13 20:20:47 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:20:47 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1498 bytes result sent to driver -2016-06-13 20:20:47 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-06-13 20:20:47 INFO CheckpointWriter:59 - Checkpoint for time 2000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-2000', took 27280 bytes and 325 ms -2016-06-13 20:20:47 INFO CheckpointWriter:59 - Saving checkpoint for time 3000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-3000' -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 82 ms on localhost (1/2) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 81 ms on localhost (2/2) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 20:20:47 INFO DAGScheduler:59 - ResultStage 1 (apply at Transformer.scala:22) finished in 0.092 s -2016-06-13 20:20:47 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:20:47 INFO DAGScheduler:59 - Job 0 finished: apply at Transformer.scala:22, took 0.788888 s -2016-06-13 20:20:47 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 50.0 and centroid: [-0.4843392480139548,0.9734525763849463,-1.7046336323611109,1.2778093795456185,0.3691650550997036] -2016-06-13 20:20:47 INFO JobScheduler:59 - Finished job streaming job 1000 ms.0 from job set of time 1000 ms -2016-06-13 20:20:47 INFO JobScheduler:59 - Starting job streaming job 1000 ms.1 from job set of time 1000 ms -2016-06-13 20:20:47 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:20:47 INFO DAGScheduler:59 - Registering RDD 3 (apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Got job 1 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:20:47 INFO DAGScheduler:59 - Final stage: ResultStage 3(apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 2) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 2) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (UnionRDD[3] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=15093, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(2000) called with curMem=18373, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2000.0 B, free 730.6 MB) -2016-06-13 20:20:47 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51099 (size: 2000.0 B, free: 730.6 MB) -2016-06-13 20:20:47 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:47 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 2 (UnionRDD[3] at apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:20:47 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 20:20:47 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 20:20:47 INFO CheckpointWriter:59 - Checkpoint for time 3000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-3000', took 27280 bytes and 142 ms -2016-06-13 20:20:47 INFO CheckpointWriter:59 - Saving checkpoint for time 4000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-4000' -2016-06-13 20:20:47 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1159 bytes result sent to driver -2016-06-13 20:20:47 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1159 bytes result sent to driver -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 50 ms on localhost (1/3) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 57 ms on localhost (2/3) -2016-06-13 20:20:47 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-06-13 20:20:47 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:20:47 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 6). 1159 bytes result sent to driver -2016-06-13 20:20:47 INFO DAGScheduler:59 - ShuffleMapStage 2 (apply at Transformer.scala:22) finished in 0.087 s -2016-06-13 20:20:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:20:47 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:20:47 INFO DAGScheduler:59 - waiting: Set(ResultStage 3) -2016-06-13 20:20:47 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:20:47 INFO DAGScheduler:59 - Missing parents for ResultStage 3: List() -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[62] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 6) in 36 ms on localhost (3/3) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=20373, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(1799) called with curMem=23613, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1799.0 B, free 730.6 MB) -2016-06-13 20:20:47 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:51099 (size: 1799.0 B, free: 730.6 MB) -2016-06-13 20:20:47 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[62] at apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:47 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 7) -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:20:47 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 7). 1180 bytes result sent to driver -2016-06-13 20:20:47 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 7) in 24 ms on localhost (1/2) -2016-06-13 20:20:47 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 8) -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:47 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 8). 1172 bytes result sent to driver -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 8) in 41 ms on localhost (2/2) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 20:20:47 INFO DAGScheduler:59 - ResultStage 3 (apply at Transformer.scala:22) finished in 0.025 s -2016-06-13 20:20:47 INFO DAGScheduler:59 - Job 1 finished: apply at Transformer.scala:22, took 0.178024 s -2016-06-13 20:20:47 INFO JobScheduler:59 - Finished job streaming job 1000 ms.1 from job set of time 1000 ms -2016-06-13 20:20:47 INFO JobScheduler:59 - Total delay: 1465820446.347 s for time 1000 ms (execution: 1.104 s) -2016-06-13 20:20:47 INFO JobScheduler:59 - Starting job streaming job 2000 ms.0 from job set of time 2000 ms -2016-06-13 20:20:47 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:20:47 INFO DAGScheduler:59 - Registering RDD 63 (apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Got job 2 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:20:47 INFO DAGScheduler:59 - Final stage: ResultStage 5(apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-06-13 20:20:47 INFO JobGenerator:59 - Checkpointing graph for time 1000 ms -2016-06-13 20:20:47 INFO DStreamGraph:59 - Updating checkpoint data for time 1000 ms -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[63] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:20:47 INFO DStreamGraph:59 - Updated checkpoint data for time 1000 ms -2016-06-13 20:20:47 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=25412, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(2471) called with curMem=30156, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:20:47 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:51099 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:20:47 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[63] at apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 10, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:20:47 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 9) -2016-06-13 20:20:47 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 10) -2016-06-13 20:20:47 INFO CheckpointWriter:59 - Checkpoint for time 4000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-4000', took 27280 bytes and 184 ms -2016-06-13 20:20:47 INFO CheckpointWriter:59 - Saving checkpoint for time 5000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-5000' -2016-06-13 20:20:47 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 9). 1159 bytes result sent to driver -2016-06-13 20:20:47 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 10). 1159 bytes result sent to driver -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 9) in 35 ms on localhost (1/2) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 10) in 35 ms on localhost (2/2) -2016-06-13 20:20:47 INFO DAGScheduler:59 - ShuffleMapStage 4 (apply at Transformer.scala:22) finished in 0.037 s -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 20:20:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:20:47 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:20:47 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-06-13 20:20:47 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:20:47 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting ResultStage 5 (ShuffledRDD[64] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=32627, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:20:47 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(2692) called with curMem=37891, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:20:47 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:51099 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:20:47 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (ShuffledRDD[64] at apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 12, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:47 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 12) -2016-06-13 20:20:47 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 11) -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:47 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 12). 1161 bytes result sent to driver -2016-06-13 20:20:47 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 11). 1498 bytes result sent to driver -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 12) in 16 ms on localhost (1/2) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 11) in 20 ms on localhost (2/2) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 20:20:47 INFO DAGScheduler:59 - ResultStage 5 (apply at Transformer.scala:22) finished in 0.021 s -2016-06-13 20:20:47 INFO DAGScheduler:59 - Job 2 finished: apply at Transformer.scala:22, took 0.103630 s -2016-06-13 20:20:47 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 100.0 and centroid: [-0.4738959687352186,0.969729236815007,-1.6823805644329948,1.2679893451777908,0.37589591355279117] -2016-06-13 20:20:47 INFO JobScheduler:59 - Finished job streaming job 2000 ms.0 from job set of time 2000 ms -2016-06-13 20:20:47 INFO JobScheduler:59 - Starting job streaming job 2000 ms.1 from job set of time 2000 ms -2016-06-13 20:20:47 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:20:47 INFO DAGScheduler:59 - Registering RDD 10 (apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Got job 3 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:20:47 INFO DAGScheduler:59 - Final stage: ResultStage 7(apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (UnionRDD[10] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=40583, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=43863, maxMem=766075207 -2016-06-13 20:20:47 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:20:47 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:51099 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:20:47 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 6 (UnionRDD[10] at apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 3 tasks -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 14, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:20:47 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 13) -2016-06-13 20:20:47 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 14) -2016-06-13 20:20:47 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 13). 1159 bytes result sent to driver -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 15, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:20:47 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 15) -2016-06-13 20:20:47 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 14). 1159 bytes result sent to driver -2016-06-13 20:20:47 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:20:47 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 15). 1159 bytes result sent to driver -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 13) in 61 ms on localhost (1/3) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 14) in 61 ms on localhost (2/3) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 15) in 23 ms on localhost (3/3) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 20:20:47 INFO DAGScheduler:59 - ShuffleMapStage 6 (apply at Transformer.scala:22) finished in 0.053 s -2016-06-13 20:20:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:20:47 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:20:47 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-06-13 20:20:47 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:20:47 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[65] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=45872, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=49112, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.5 MB) -2016-06-13 20:20:47 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:51099 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:20:47 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[65] at apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:47 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 16) -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:20:47 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 16). 1180 bytes result sent to driver -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 16) in 19 ms on localhost (1/2) -2016-06-13 20:20:47 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:20:47 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 17) -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:47 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 17). 1172 bytes result sent to driver -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 17) in 38 ms on localhost (2/2) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 20:20:47 INFO DAGScheduler:59 - ResultStage 7 (apply at Transformer.scala:22) finished in 0.040 s -2016-06-13 20:20:47 INFO DAGScheduler:59 - Job 3 finished: apply at Transformer.scala:22, took 0.146392 s -2016-06-13 20:20:47 INFO JobScheduler:59 - Finished job streaming job 2000 ms.1 from job set of time 2000 ms -2016-06-13 20:20:47 INFO JobScheduler:59 - Total delay: 1465820445.634 s for time 2000 ms (execution: 0.284 s) -2016-06-13 20:20:47 INFO JobScheduler:59 - Starting job streaming job 3000 ms.0 from job set of time 3000 ms -2016-06-13 20:20:47 INFO ParallelCollectionRDD:59 - Removing RDD 0 from persistence list -2016-06-13 20:20:47 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:20:47 INFO MapPartitionsRDD:59 - Removing RDD 5 from persistence list -2016-06-13 20:20:47 INFO DAGScheduler:59 - Registering RDD 66 (apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO CheckpointWriter:59 - Checkpoint for time 5000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-5000', took 27280 bytes and 248 ms -2016-06-13 20:20:47 INFO DAGScheduler:59 - Got job 4 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:20:47 INFO DAGScheduler:59 - Final stage: ResultStage 9(apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 8) -2016-06-13 20:20:47 INFO CheckpointWriter:59 - Saving checkpoint for time 6000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-6000' -2016-06-13 20:20:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 8) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 8 (MapPartitionsRDD[66] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=50914, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:20:47 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(2475) called with curMem=55658, maxMem=766075207 -2016-06-13 20:20:47 INFO ShuffledRDD:59 - Removing RDD 4 from persistence list -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:20:47 INFO BlockManager:59 - Removing RDD 0 -2016-06-13 20:20:47 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:51099 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:20:47 INFO UnionRDD:59 - Removing RDD 3 from persistence list -2016-06-13 20:20:47 INFO BlockManager:59 - Removing RDD 5 -2016-06-13 20:20:47 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 8 (MapPartitionsRDD[66] at apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 20:20:47 INFO MapPartitionsRDD:59 - Removing RDD 1 from persistence list -2016-06-13 20:20:47 INFO BlockManager:59 - Removing RDD 4 -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 18, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 19, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:20:47 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 18) -2016-06-13 20:20:47 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 19) -2016-06-13 20:20:47 INFO JobGenerator:59 - Checkpointing graph for time 2000 ms -2016-06-13 20:20:47 INFO DStreamGraph:59 - Updating checkpoint data for time 2000 ms -2016-06-13 20:20:47 INFO DStreamGraph:59 - Updated checkpoint data for time 2000 ms -2016-06-13 20:20:47 INFO BlockManager:59 - Removing RDD 1 -2016-06-13 20:20:47 INFO BlockManager:59 - Removing RDD 3 -2016-06-13 20:20:47 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 19). 1159 bytes result sent to driver -2016-06-13 20:20:47 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 18). 1159 bytes result sent to driver -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 18) in 29 ms on localhost (1/2) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 19) in 29 ms on localhost (2/2) -2016-06-13 20:20:47 INFO DAGScheduler:59 - ShuffleMapStage 8 (apply at Transformer.scala:22) finished in 0.031 s -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 20:20:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:20:47 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:20:47 INFO DAGScheduler:59 - waiting: Set(ResultStage 9) -2016-06-13 20:20:47 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:20:47 INFO DAGScheduler:59 - Missing parents for ResultStage 9: List() -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting ResultStage 9 (ShuffledRDD[67] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=58133, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=63397, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:20:47 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:20:47 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:51099 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:20:47 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (ShuffledRDD[67] at apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-06-13 20:20:47 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:51099 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:20:47 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:51099 in memory (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 21, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:47 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 20) -2016-06-13 20:20:47 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 21) -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:47 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 21). 1161 bytes result sent to driver -2016-06-13 20:20:47 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 21) in 14 ms on localhost (1/2) -2016-06-13 20:20:47 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 20). 1498 bytes result sent to driver -2016-06-13 20:20:47 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 20) in 40 ms on localhost (2/2) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 20:20:47 INFO DAGScheduler:59 - ResultStage 9 (apply at Transformer.scala:22) finished in 0.041 s -2016-06-13 20:20:47 INFO DAGScheduler:59 - Job 4 finished: apply at Transformer.scala:22, took 0.149136 s -2016-06-13 20:20:47 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 150.0 and centroid: [-0.4741102089605591,0.9622750167428704,-1.680286076986711,1.2654376587211662,0.37713329463498424] -2016-06-13 20:20:47 INFO JobScheduler:59 - Finished job streaming job 3000 ms.0 from job set of time 3000 ms -2016-06-13 20:20:47 INFO JobScheduler:59 - Starting job streaming job 3000 ms.1 from job set of time 3000 ms -2016-06-13 20:20:47 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:51099 in memory (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:20:47 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 20:20:47 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:20:47 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:51099 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Registering RDD 16 (apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Got job 5 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:20:47 INFO DAGScheduler:59 - Final stage: ResultStage 11(apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 10) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 10) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 10 (UnionRDD[16] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:20:47 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=40634, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:20:47 INFO CheckpointWriter:59 - Checkpoint for time 6000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-6000', took 27278 bytes and 166 ms -2016-06-13 20:20:47 INFO CheckpointWriter:59 - Saving checkpoint for time 7000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-7000' -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(2004) called with curMem=36699, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2004.0 B, free 730.5 MB) -2016-06-13 20:20:47 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:51099 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:20:47 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 20:20:47 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:51099 (size: 2004.0 B, free: 730.6 MB) -2016-06-13 20:20:47 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-06-13 20:20:47 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 10 (UnionRDD[16] at apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 3 tasks -2016-06-13 20:20:47 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:51099 in memory (size: 1799.0 B, free: 730.6 MB) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 22, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 23, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:20:47 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 22) -2016-06-13 20:20:47 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 23) -2016-06-13 20:20:47 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 20:20:47 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:20:47 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:51099 in memory (size: 2000.0 B, free: 730.6 MB) -2016-06-13 20:20:47 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 23). 1159 bytes result sent to driver -2016-06-13 20:20:47 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 22). 1159 bytes result sent to driver -2016-06-13 20:20:47 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 2.0 in stage 10.0 (TID 24, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:20:47 INFO Executor:59 - Running task 2.0 in stage 10.0 (TID 24) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 23) in 22 ms on localhost (1/3) -2016-06-13 20:20:47 INFO ContextCleaner:59 - Cleaned shuffle 1 -2016-06-13 20:20:47 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:51099 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 22) in 30 ms on localhost (2/3) -2016-06-13 20:20:47 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 20:20:47 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 20:20:47 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-06-13 20:20:47 INFO Executor:59 - Finished task 2.0 in stage 10.0 (TID 24). 1159 bytes result sent to driver -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 2.0 in stage 10.0 (TID 24) in 19 ms on localhost (3/3) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 20:20:47 INFO DAGScheduler:59 - ShuffleMapStage 10 (apply at Transformer.scala:22) finished in 0.044 s -2016-06-13 20:20:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:20:47 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:20:47 INFO DAGScheduler:59 - waiting: Set(ResultStage 11) -2016-06-13 20:20:47 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:20:47 INFO DAGScheduler:59 - Missing parents for ResultStage 11: List() -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[68] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=20463, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=23703, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.6 MB) -2016-06-13 20:20:47 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:51099 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:20:47 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[68] at apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 25, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 26, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:47 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 25) -2016-06-13 20:20:47 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 26) -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:47 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:20:47 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:47 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 26). 1172 bytes result sent to driver -2016-06-13 20:20:47 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 25). 1180 bytes result sent to driver -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 26) in 16 ms on localhost (1/2) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 25) in 18 ms on localhost (2/2) -2016-06-13 20:20:47 INFO DAGScheduler:59 - ResultStage 11 (apply at Transformer.scala:22) finished in 0.018 s -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 20:20:47 INFO DAGScheduler:59 - Job 5 finished: apply at Transformer.scala:22, took 0.103858 s -2016-06-13 20:20:47 INFO JobScheduler:59 - Finished job streaming job 3000 ms.1 from job set of time 3000 ms -2016-06-13 20:20:47 INFO JobScheduler:59 - Total delay: 1465820444.919 s for time 3000 ms (execution: 0.284 s) -2016-06-13 20:20:47 INFO JobScheduler:59 - Starting job streaming job 4000 ms.0 from job set of time 4000 ms -2016-06-13 20:20:47 INFO ParallelCollectionRDD:59 - Removing RDD 6 from persistence list -2016-06-13 20:20:47 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-06-13 20:20:47 INFO ShuffledRDD:59 - Removing RDD 11 from persistence list -2016-06-13 20:20:47 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 20:20:47 INFO UnionRDD:59 - Removing RDD 10 from persistence list -2016-06-13 20:20:47 INFO BlockManager:59 - Removing RDD 10 -2016-06-13 20:20:47 INFO MapPartitionsRDD:59 - Removing RDD 8 from persistence list -2016-06-13 20:20:47 INFO BlockManager:59 - Removing RDD 11 -2016-06-13 20:20:47 INFO JobGenerator:59 - Checkpointing graph for time 3000 ms -2016-06-13 20:20:47 INFO DStreamGraph:59 - Updating checkpoint data for time 3000 ms -2016-06-13 20:20:47 INFO BlockManager:59 - Removing RDD 8 -2016-06-13 20:20:47 INFO DStreamGraph:59 - Updated checkpoint data for time 3000 ms -2016-06-13 20:20:47 INFO BlockManager:59 - Removing RDD 6 -2016-06-13 20:20:47 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:20:47 INFO DAGScheduler:59 - Registering RDD 69 (apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Got job 6 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:20:47 INFO DAGScheduler:59 - Final stage: ResultStage 13(apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (MapPartitionsRDD[69] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=25505, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(2484) called with curMem=30249, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:20:47 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:51099 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:20:47 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (MapPartitionsRDD[69] at apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 27, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 28, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:20:47 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 27) -2016-06-13 20:20:47 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 28) -2016-06-13 20:20:47 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:20:47 INFO CheckpointWriter:59 - Checkpoint for time 7000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-7000', took 27278 bytes and 138 ms -2016-06-13 20:20:47 INFO CheckpointWriter:59 - Saving checkpoint for time 8000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-8000' -2016-06-13 20:20:47 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 28). 1159 bytes result sent to driver -2016-06-13 20:20:47 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 27). 1159 bytes result sent to driver -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 28) in 24 ms on localhost (1/2) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 27) in 26 ms on localhost (2/2) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 20:20:47 INFO DAGScheduler:59 - ShuffleMapStage 12 (apply at Transformer.scala:22) finished in 0.026 s -2016-06-13 20:20:47 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:20:47 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:20:47 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-06-13 20:20:47 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:20:47 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting ResultStage 13 (ShuffledRDD[70] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=32733, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:20:47 INFO MemoryStore:59 - ensureFreeSpace(2698) called with curMem=37997, maxMem=766075207 -2016-06-13 20:20:47 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:20:47 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:51099 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:20:47 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (ShuffledRDD[70] at apply at Transformer.scala:22) -2016-06-13 20:20:47 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 29, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 30, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 29) -2016-06-13 20:20:48 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 30) -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:48 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 30). 1161 bytes result sent to driver -2016-06-13 20:20:48 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 30) in 13 ms on localhost (1/2) -2016-06-13 20:20:48 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 29). 1498 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 29) in 23 ms on localhost (2/2) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 20:20:48 INFO DAGScheduler:59 - ResultStage 13 (apply at Transformer.scala:22) finished in 0.026 s -2016-06-13 20:20:48 INFO DAGScheduler:59 - Job 6 finished: apply at Transformer.scala:22, took 0.086573 s -2016-06-13 20:20:48 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 200.0 and centroid: [-0.4768735011530239,0.9606386018112957,-1.673148729841545,1.2634974127814163,0.38189605546947214] -2016-06-13 20:20:48 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:20:48 INFO JobScheduler:59 - Finished job streaming job 4000 ms.0 from job set of time 4000 ms -2016-06-13 20:20:48 INFO JobScheduler:59 - Starting job streaming job 4000 ms.1 from job set of time 4000 ms -2016-06-13 20:20:48 INFO DAGScheduler:59 - Registering RDD 22 (apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Got job 7 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:20:48 INFO DAGScheduler:59 - Final stage: ResultStage 15(apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 14) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 14) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting ShuffleMapStage 14 (UnionRDD[22] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=40695, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=43975, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:51099 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:20:48 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 14 (UnionRDD[22] at apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 3 tasks -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 31, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 32, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 32) -2016-06-13 20:20:48 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 31) -2016-06-13 20:20:48 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:20:48 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 31). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 32). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 33, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 33) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 31) in 24 ms on localhost (1/3) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 32) in 25 ms on localhost (2/3) -2016-06-13 20:20:48 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 33). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 33) in 15 ms on localhost (3/3) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 20:20:48 INFO DAGScheduler:59 - ShuffleMapStage 14 (apply at Transformer.scala:22) finished in 0.038 s -2016-06-13 20:20:48 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:20:48 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:20:48 INFO DAGScheduler:59 - waiting: Set(ResultStage 15) -2016-06-13 20:20:48 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:20:48 INFO DAGScheduler:59 - Missing parents for ResultStage 15: List() -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[71] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=45984, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(1801) called with curMem=49224, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1801.0 B, free 730.5 MB) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:51099 (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:20:48 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[71] at apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 34, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 35, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 34) -2016-06-13 20:20:48 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 35) -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:48 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 35). 1172 bytes result sent to driver -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 35) in 10 ms on localhost (1/2) -2016-06-13 20:20:48 INFO CheckpointWriter:59 - Checkpoint for time 8000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-8000', took 27280 bytes and 156 ms -2016-06-13 20:20:48 INFO CheckpointWriter:59 - Saving checkpoint for time 9000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-9000' -2016-06-13 20:20:48 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 34). 1180 bytes result sent to driver -2016-06-13 20:20:48 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 34) in 19 ms on localhost (2/2) -2016-06-13 20:20:48 INFO DAGScheduler:59 - ResultStage 15 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 20:20:48 INFO DAGScheduler:59 - Job 7 finished: apply at Transformer.scala:22, took 0.089376 s -2016-06-13 20:20:48 INFO JobScheduler:59 - Finished job streaming job 4000 ms.1 from job set of time 4000 ms -2016-06-13 20:20:48 INFO JobScheduler:59 - Total delay: 1465820444.125 s for time 4000 ms (execution: 0.206 s) -2016-06-13 20:20:48 INFO JobScheduler:59 - Starting job streaming job 5000 ms.0 from job set of time 5000 ms -2016-06-13 20:20:48 INFO ParallelCollectionRDD:59 - Removing RDD 13 from persistence list -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 13 -2016-06-13 20:20:48 INFO MapPartitionsRDD:59 - Removing RDD 18 from persistence list -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 18 -2016-06-13 20:20:48 INFO ShuffledRDD:59 - Removing RDD 17 from persistence list -2016-06-13 20:20:48 INFO UnionRDD:59 - Removing RDD 16 from persistence list -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 17 -2016-06-13 20:20:48 INFO MapPartitionsRDD:59 - Removing RDD 14 from persistence list -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 16 -2016-06-13 20:20:48 INFO JobGenerator:59 - Checkpointing graph for time 4000 ms -2016-06-13 20:20:48 INFO DStreamGraph:59 - Updating checkpoint data for time 4000 ms -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 14 -2016-06-13 20:20:48 INFO DStreamGraph:59 - Updated checkpoint data for time 4000 ms -2016-06-13 20:20:48 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Registering RDD 72 (apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Got job 8 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:20:48 INFO DAGScheduler:59 - Final stage: ResultStage 17(apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 16) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 16) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting ShuffleMapStage 16 (MapPartitionsRDD[72] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=51025, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(2476) called with curMem=55769, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:51099 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:20:48 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 16 (MapPartitionsRDD[72] at apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 36, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 37, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 36) -2016-06-13 20:20:48 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 37) -2016-06-13 20:20:48 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:20:48 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 37). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 37) in 23 ms on localhost (1/2) -2016-06-13 20:20:48 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 36). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 36) in 39 ms on localhost (2/2) -2016-06-13 20:20:48 INFO DAGScheduler:59 - ShuffleMapStage 16 (apply at Transformer.scala:22) finished in 0.036 s -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 20:20:48 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:20:48 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:20:48 INFO DAGScheduler:59 - waiting: Set(ResultStage 17) -2016-06-13 20:20:48 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:20:48 INFO DAGScheduler:59 - Missing parents for ResultStage 17: List() -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting ResultStage 17 (ShuffledRDD[73] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=58245, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=63509, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:51099 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:20:48 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (ShuffledRDD[73] at apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 38, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 39, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 38) -2016-06-13 20:20:48 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 39) -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:20:48 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 39). 1161 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 39) in 13 ms on localhost (1/2) -2016-06-13 20:20:48 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:20:48 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 38). 1498 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 38) in 26 ms on localhost (2/2) -2016-06-13 20:20:48 INFO DAGScheduler:59 - ResultStage 17 (apply at Transformer.scala:22) finished in 0.027 s -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 20:20:48 INFO DAGScheduler:59 - Job 8 finished: apply at Transformer.scala:22, took 0.096549 s -2016-06-13 20:20:48 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 250.0 and centroid: [-0.4770613340912709,0.9657053185238176,-1.6732261982319274,1.26651642628913,0.38386658573056776] -2016-06-13 20:20:48 INFO JobScheduler:59 - Finished job streaming job 5000 ms.0 from job set of time 5000 ms -2016-06-13 20:20:48 INFO JobScheduler:59 - Starting job streaming job 5000 ms.1 from job set of time 5000 ms -2016-06-13 20:20:48 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Registering RDD 29 (apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Got job 9 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:20:48 INFO DAGScheduler:59 - Final stage: ResultStage 19(apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 18) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 18) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting ShuffleMapStage 18 (UnionRDD[29] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=66203, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(2008) called with curMem=69483, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2008.0 B, free 730.5 MB) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:51099 (size: 2008.0 B, free: 730.6 MB) -2016-06-13 20:20:48 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 18 (UnionRDD[29] at apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 3 tasks -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 40, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 41, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 41) -2016-06-13 20:20:48 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 40) -2016-06-13 20:20:48 INFO CheckpointWriter:59 - Checkpoint for time 9000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-9000', took 27280 bytes and 151 ms -2016-06-13 20:20:48 INFO CheckpointWriter:59 - Saving checkpoint for time 10000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-10000' -2016-06-13 20:20:48 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:20:48 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 41). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 42, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 42) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 41) in 21 ms on localhost (1/3) -2016-06-13 20:20:48 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 40). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 42). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 40) in 30 ms on localhost (2/3) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 42) in 12 ms on localhost (3/3) -2016-06-13 20:20:48 INFO DAGScheduler:59 - ShuffleMapStage 18 (apply at Transformer.scala:22) finished in 0.031 s -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 20:20:48 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:20:48 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:20:48 INFO DAGScheduler:59 - waiting: Set(ResultStage 19) -2016-06-13 20:20:48 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:20:48 INFO DAGScheduler:59 - Missing parents for ResultStage 19: List() -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[74] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=71491, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=74731, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:51099 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:20:48 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[74] at apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 43, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 44, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 43) -2016-06-13 20:20:48 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 44) -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:48 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 44). 1172 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 44) in 10 ms on localhost (1/2) -2016-06-13 20:20:48 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 43). 1180 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 43) in 18 ms on localhost (2/2) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 20:20:48 INFO DAGScheduler:59 - ResultStage 19 (apply at Transformer.scala:22) finished in 0.018 s -2016-06-13 20:20:48 INFO DAGScheduler:59 - Job 9 finished: apply at Transformer.scala:22, took 0.079007 s -2016-06-13 20:20:48 INFO JobScheduler:59 - Finished job streaming job 5000 ms.1 from job set of time 5000 ms -2016-06-13 20:20:48 INFO ParallelCollectionRDD:59 - Removing RDD 19 from persistence list -2016-06-13 20:20:48 INFO JobScheduler:59 - Total delay: 1465820443.327 s for time 5000 ms (execution: 0.201 s) -2016-06-13 20:20:48 INFO JobScheduler:59 - Starting job streaming job 6000 ms.0 from job set of time 6000 ms -2016-06-13 20:20:48 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:20:48 INFO MapPartitionsRDD:59 - Removing RDD 24 from persistence list -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 19 -2016-06-13 20:20:48 INFO ShuffledRDD:59 - Removing RDD 23 from persistence list -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 24 -2016-06-13 20:20:48 INFO UnionRDD:59 - Removing RDD 22 from persistence list -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 23 -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 22 -2016-06-13 20:20:48 INFO MapPartitionsRDD:59 - Removing RDD 20 from persistence list -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 20 -2016-06-13 20:20:48 INFO JobGenerator:59 - Checkpointing graph for time 5000 ms -2016-06-13 20:20:48 INFO DStreamGraph:59 - Updating checkpoint data for time 5000 ms -2016-06-13 20:20:48 INFO DStreamGraph:59 - Updated checkpoint data for time 5000 ms -2016-06-13 20:20:48 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Registering RDD 75 (apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Got job 10 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:20:48 INFO DAGScheduler:59 - Final stage: ResultStage 21(apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 20) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 20) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting ShuffleMapStage 20 (MapPartitionsRDD[75] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=76534, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=81278, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:51099 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:20:48 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 20 (MapPartitionsRDD[75] at apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 45, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 46, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 46) -2016-06-13 20:20:48 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 45) -2016-06-13 20:20:48 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 45). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 45) in 24 ms on localhost (1/2) -2016-06-13 20:20:48 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 46). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 46) in 25 ms on localhost (2/2) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 20:20:48 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:20:48 INFO DAGScheduler:59 - ShuffleMapStage 20 (apply at Transformer.scala:22) finished in 0.027 s -2016-06-13 20:20:48 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:20:48 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:20:48 INFO DAGScheduler:59 - waiting: Set(ResultStage 21) -2016-06-13 20:20:48 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:20:48 INFO DAGScheduler:59 - Missing parents for ResultStage 21: List() -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting ResultStage 21 (ShuffledRDD[76] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=83760, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=89024, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:51099 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:20:48 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (ShuffledRDD[76] at apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 47) -2016-06-13 20:20:48 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 48) -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:20:48 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 48). 1161 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 48) in 12 ms on localhost (1/2) -2016-06-13 20:20:48 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 47). 1498 bytes result sent to driver -2016-06-13 20:20:48 INFO DAGScheduler:59 - ResultStage 21 (apply at Transformer.scala:22) finished in 0.015 s -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 47) in 15 ms on localhost (2/2) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 20:20:48 INFO DAGScheduler:59 - Job 10 finished: apply at Transformer.scala:22, took 0.069630 s -2016-06-13 20:20:48 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 300.0 and centroid: [-0.4778380287460108,0.9658292271340416,-1.6712646630450996,1.273358698745343,0.37938624993358266] -2016-06-13 20:20:48 INFO JobScheduler:59 - Finished job streaming job 6000 ms.0 from job set of time 6000 ms -2016-06-13 20:20:48 INFO JobScheduler:59 - Starting job streaming job 6000 ms.1 from job set of time 6000 ms -2016-06-13 20:20:48 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Registering RDD 35 (apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Got job 11 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:20:48 INFO DAGScheduler:59 - Final stage: ResultStage 23(apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting ShuffleMapStage 22 (UnionRDD[35] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=91720, maxMem=766075207 -2016-06-13 20:20:48 INFO CheckpointWriter:59 - Checkpoint for time 10000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-10000', took 27292 bytes and 157 ms -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO CheckpointWriter:59 - Saving checkpoint for time 1000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-1000' -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=95000, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.5 MB) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:51099 (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:20:48 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:20:48 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 22 (UnionRDD[35] at apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 3 tasks -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 49, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 50, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 49) -2016-06-13 20:20:48 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 50) -2016-06-13 20:20:48 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 49). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 50). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 51, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 50) in 34 ms on localhost (1/3) -2016-06-13 20:20:48 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 51) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 49) in 50 ms on localhost (2/3) -2016-06-13 20:20:48 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 51). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 51) in 33 ms on localhost (3/3) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 20:20:48 INFO DAGScheduler:59 - ShuffleMapStage 22 (apply at Transformer.scala:22) finished in 0.061 s -2016-06-13 20:20:48 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:20:48 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:20:48 INFO DAGScheduler:59 - waiting: Set(ResultStage 23) -2016-06-13 20:20:48 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:20:48 INFO DAGScheduler:59 - Missing parents for ResultStage 23: List() -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[77] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=97007, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=100247, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:51099 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:20:48 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[77] at apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 52, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 53) -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:48 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 53). 1172 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 53) in 9 ms on localhost (1/2) -2016-06-13 20:20:48 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 52) -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:48 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 52). 1180 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 52) in 25 ms on localhost (2/2) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 20:20:48 INFO DAGScheduler:59 - ResultStage 23 (apply at Transformer.scala:22) finished in 0.026 s -2016-06-13 20:20:48 INFO DAGScheduler:59 - Job 11 finished: apply at Transformer.scala:22, took 0.120231 s -2016-06-13 20:20:48 INFO JobScheduler:59 - Finished job streaming job 6000 ms.1 from job set of time 6000 ms -2016-06-13 20:20:48 INFO JobScheduler:59 - Total delay: 1465820442.543 s for time 6000 ms (execution: 0.215 s) -2016-06-13 20:20:48 INFO JobScheduler:59 - Starting job streaming job 7000 ms.0 from job set of time 7000 ms -2016-06-13 20:20:48 INFO ParallelCollectionRDD:59 - Removing RDD 25 from persistence list -2016-06-13 20:20:48 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:20:48 INFO MapPartitionsRDD:59 - Removing RDD 31 from persistence list -2016-06-13 20:20:48 INFO ShuffledRDD:59 - Removing RDD 30 from persistence list -2016-06-13 20:20:48 INFO UnionRDD:59 - Removing RDD 29 from persistence list -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 25 -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 29 -2016-06-13 20:20:48 INFO MapPartitionsRDD:59 - Removing RDD 27 from persistence list -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 31 -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 27 -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 30 -2016-06-13 20:20:48 INFO JobGenerator:59 - Checkpointing graph for time 6000 ms -2016-06-13 20:20:48 INFO DStreamGraph:59 - Updating checkpoint data for time 6000 ms -2016-06-13 20:20:48 INFO DStreamGraph:59 - Updated checkpoint data for time 6000 ms -2016-06-13 20:20:48 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Registering RDD 78 (apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Got job 12 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:20:48 INFO DAGScheduler:59 - Final stage: ResultStage 25(apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 24) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 24) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting ShuffleMapStage 24 (MapPartitionsRDD[78] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=102050, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(2480) called with curMem=106794, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:51099 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 20:20:48 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Removed broadcast_23_piece0 on localhost:51099 in memory (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 24 (MapPartitionsRDD[78] at apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-06-13 20:20:48 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned accumulator 24 -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 54, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 55, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 54) -2016-06-13 20:20:48 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 55) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Removed broadcast_22_piece0 on localhost:51099 in memory (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned accumulator 23 -2016-06-13 20:20:48 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 55). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 54). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Removed broadcast_21_piece0 on localhost:51099 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned accumulator 22 -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 55) in 20 ms on localhost (1/2) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 54) in 22 ms on localhost (2/2) -2016-06-13 20:20:48 INFO DAGScheduler:59 - ShuffleMapStage 24 (apply at Transformer.scala:22) finished in 0.023 s -2016-06-13 20:20:48 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:20:48 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:20:48 INFO DAGScheduler:59 - waiting: Set(ResultStage 25) -2016-06-13 20:20:48 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:20:48 INFO DAGScheduler:59 - Missing parents for ResultStage 25: List() -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting ResultStage 25 (ShuffledRDD[79] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Removed broadcast_20_piece0 on localhost:51099 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=83758, maxMem=766075207 -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned accumulator 21 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned shuffle 10 -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=85782, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:51099 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:20:48 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Removed broadcast_19_piece0 on localhost:51099 in memory (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (ShuffledRDD[79] at apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned accumulator 20 -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 57) -2016-06-13 20:20:48 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 56) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Removed broadcast_18_piece0 on localhost:51099 in memory (size: 2008.0 B, free: 730.6 MB) -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned accumulator 19 -2016-06-13 20:20:48 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 57). 1161 bytes result sent to driver -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned shuffle 9 -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 57) in 9 ms on localhost (1/2) -2016-06-13 20:20:48 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 56). 1498 bytes result sent to driver -2016-06-13 20:20:48 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Removed broadcast_17_piece0 on localhost:51099 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 56) in 15 ms on localhost (2/2) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-06-13 20:20:48 INFO DAGScheduler:59 - ResultStage 25 (apply at Transformer.scala:22) finished in 0.016 s -2016-06-13 20:20:48 INFO DAGScheduler:59 - Job 12 finished: apply at Transformer.scala:22, took 0.091229 s -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned accumulator 18 -2016-06-13 20:20:48 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 350.0 and centroid: [-0.47440048489638564,0.9650523283036628,-1.6700055274657652,1.269947562552048,0.3797915177328754] -2016-06-13 20:20:48 INFO JobScheduler:59 - Finished job streaming job 7000 ms.0 from job set of time 7000 ms -2016-06-13 20:20:48 INFO JobScheduler:59 - Starting job streaming job 7000 ms.1 from job set of time 7000 ms -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Removed broadcast_16_piece0 on localhost:51099 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:20:48 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned accumulator 17 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Registering RDD 41 (apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned shuffle 8 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Got job 13 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:20:48 INFO DAGScheduler:59 - Final stage: ResultStage 27(apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (UnionRDD[41] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=64408, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:51099 in memory (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=67688, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:51099 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:20:48 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 26 (UnionRDD[41] at apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 3 tasks -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 58, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:51099 in memory (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 59, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 58) -2016-06-13 20:20:48 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 59) -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned shuffle 7 -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:51099 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:20:48 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 58). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 60, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 58) in 20 ms on localhost (1/3) -2016-06-13 20:20:48 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 60) -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 20:20:48 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:20:48 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 59). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 59) in 28 ms on localhost (2/3) -2016-06-13 20:20:48 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 60). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:51099 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 60) in 16 ms on localhost (3/3) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-06-13 20:20:48 INFO DAGScheduler:59 - ShuffleMapStage 26 (apply at Transformer.scala:22) finished in 0.036 s -2016-06-13 20:20:48 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:20:48 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:20:48 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-06-13 20:20:48 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned shuffle 6 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[80] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=40936, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-1000.bk -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:51099 in memory (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(1801) called with curMem=38892, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 1801.0 B, free 730.5 MB) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:51099 (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:51099 in memory (size: 2004.0 B, free: 730.6 MB) -2016-06-13 20:20:48 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[80] at apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-06-13 20:20:48 INFO CheckpointWriter:59 - Checkpoint for time 1000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-1000', took 27470 bytes and 301 ms -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned shuffle 5 -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:48 INFO CheckpointWriter:59 - Saving checkpoint for time 2000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-2000' -2016-06-13 20:20:48 INFO DStreamGraph:59 - Clearing checkpoint data for time 1000 ms -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 62, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 62) -2016-06-13 20:20:48 INFO DStreamGraph:59 - Cleared checkpoint data for time 1000 ms -2016-06-13 20:20:48 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 61) -2016-06-13 20:20:48 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:20:48 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 62). 1172 bytes result sent to driver -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:51099 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 62) in 15 ms on localhost (1/2) -2016-06-13 20:20:48 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 61). 1180 bytes result sent to driver -2016-06-13 20:20:48 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/receivedBlockMetadata older than 0: -2016-06-13 20:20:48 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 61) in 29 ms on localhost (2/2) -2016-06-13 20:20:48 INFO DAGScheduler:59 - ResultStage 27 (apply at Transformer.scala:22) finished in 0.029 s -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:51099 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Job 13 finished: apply at Transformer.scala:22, took 0.099674 s -2016-06-13 20:20:48 INFO InputInfoTracker:59 - remove old batch metadata: -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 20:20:48 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/receivedBlockMetadata older than 0 -2016-06-13 20:20:48 INFO JobScheduler:59 - Finished job streaming job 7000 ms.1 from job set of time 7000 ms -2016-06-13 20:20:48 INFO JobScheduler:59 - Total delay: 1465820441.763 s for time 7000 ms (execution: 0.219 s) -2016-06-13 20:20:48 INFO ParallelCollectionRDD:59 - Removing RDD 32 from persistence list -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned shuffle 4 -2016-06-13 20:20:48 INFO JobScheduler:59 - Starting job streaming job 8000 ms.0 from job set of time 8000 ms -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 32 -2016-06-13 20:20:48 INFO ContextCleaner:59 - Cleaned shuffle 3 -2016-06-13 20:20:48 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Registering RDD 81 (apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Got job 14 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:20:48 INFO DAGScheduler:59 - Final stage: ResultStage 29(apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 28) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 28) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting ShuffleMapStage 28 (MapPartitionsRDD[81] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=25514, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=30258, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:20:48 INFO MapPartitionsRDD:59 - Removing RDD 37 from persistence list -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:51099 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 37 -2016-06-13 20:20:48 INFO ShuffledRDD:59 - Removing RDD 36 from persistence list -2016-06-13 20:20:48 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 28 (MapPartitionsRDD[81] at apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-06-13 20:20:48 INFO UnionRDD:59 - Removing RDD 35 from persistence list -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 36 -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 63, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:20:48 INFO MapPartitionsRDD:59 - Removing RDD 33 from persistence list -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 35 -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 64, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 64) -2016-06-13 20:20:48 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 63) -2016-06-13 20:20:48 INFO JobGenerator:59 - Checkpointing graph for time 7000 ms -2016-06-13 20:20:48 INFO DStreamGraph:59 - Updating checkpoint data for time 7000 ms -2016-06-13 20:20:48 INFO DStreamGraph:59 - Updated checkpoint data for time 7000 ms -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 33 -2016-06-13 20:20:48 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:20:48 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 64). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 63). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 64) in 20 ms on localhost (1/2) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 63) in 22 ms on localhost (2/2) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-06-13 20:20:48 INFO DAGScheduler:59 - ShuffleMapStage 28 (apply at Transformer.scala:22) finished in 0.023 s -2016-06-13 20:20:48 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:20:48 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:20:48 INFO DAGScheduler:59 - waiting: Set(ResultStage 29) -2016-06-13 20:20:48 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:20:48 INFO DAGScheduler:59 - Missing parents for ResultStage 29: List() -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting ResultStage 29 (ShuffledRDD[82] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=32740, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=38004, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:51099 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:20:48 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (ShuffledRDD[82] at apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 65, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 66, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 66) -2016-06-13 20:20:48 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 65) -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:20:48 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 66). 1161 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 66) in 7 ms on localhost (1/2) -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:20:48 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:20:48 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 65). 1498 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 65) in 16 ms on localhost (2/2) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 20:20:48 INFO DAGScheduler:59 - ResultStage 29 (apply at Transformer.scala:22) finished in 0.018 s -2016-06-13 20:20:48 INFO DAGScheduler:59 - Job 14 finished: apply at Transformer.scala:22, took 0.079705 s -2016-06-13 20:20:48 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 400.0 and centroid: [-0.4716182064591067,0.967586334056103,-1.6699954778983372,1.270341503100185,0.37935009425710725] -2016-06-13 20:20:48 INFO JobScheduler:59 - Finished job streaming job 8000 ms.0 from job set of time 8000 ms -2016-06-13 20:20:48 INFO JobScheduler:59 - Starting job streaming job 8000 ms.1 from job set of time 8000 ms -2016-06-13 20:20:48 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Registering RDD 47 (apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Got job 15 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:20:48 INFO DAGScheduler:59 - Final stage: ResultStage 31(apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 30) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting ShuffleMapStage 30 (UnionRDD[47] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=40698, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(2006) called with curMem=43978, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2006.0 B, free 730.5 MB) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:51099 (size: 2006.0 B, free: 730.6 MB) -2016-06-13 20:20:48 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 30 (UnionRDD[47] at apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 3 tasks -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 67, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 68, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 68) -2016-06-13 20:20:48 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 67) -2016-06-13 20:20:48 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 68). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 67). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 69, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 69) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 68) in 18 ms on localhost (1/3) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 67) in 20 ms on localhost (2/3) -2016-06-13 20:20:48 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:20:48 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 69). 1159 bytes result sent to driver -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 69) in 10 ms on localhost (3/3) -2016-06-13 20:20:48 INFO DAGScheduler:59 - ShuffleMapStage 30 (apply at Transformer.scala:22) finished in 0.028 s -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 20:20:48 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:20:48 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:20:48 INFO DAGScheduler:59 - waiting: Set(ResultStage 31) -2016-06-13 20:20:48 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:20:48 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List() -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[83] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=45984, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=49224, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.5 MB) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:51099 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:20:48 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[83] at apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 70, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 71, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:48 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 71) -2016-06-13 20:20:48 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 70) -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:20:48 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 71). 1172 bytes result sent to driver -2016-06-13 20:20:48 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:20:48 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 70). 1180 bytes result sent to driver -2016-06-13 20:20:48 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-1000 -2016-06-13 20:20:48 INFO CheckpointWriter:59 - Checkpoint for time 2000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-2000', took 27483 bytes and 226 ms -2016-06-13 20:20:48 INFO CheckpointWriter:59 - Saving checkpoint for time 3000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-3000' -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 70) in 32 ms on localhost (1/2) -2016-06-13 20:20:48 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:20:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 71) in 33 ms on localhost (2/2) -2016-06-13 20:20:48 INFO DAGScheduler:59 - ResultStage 31 (apply at Transformer.scala:22) finished in 0.034 s -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 20:20:48 INFO DStreamGraph:59 - Clearing checkpoint data for time 2000 ms -2016-06-13 20:20:48 INFO DStreamGraph:59 - Cleared checkpoint data for time 2000 ms -2016-06-13 20:20:48 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:20:48 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/receivedBlockMetadata older than 1000: -2016-06-13 20:20:48 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/receivedBlockMetadata older than 1000 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Job 15 finished: apply at Transformer.scala:22, took 0.091447 s -2016-06-13 20:20:48 INFO InputInfoTracker:59 - remove old batch metadata: -2016-06-13 20:20:48 INFO JobScheduler:59 - Finished job streaming job 8000 ms.1 from job set of time 8000 ms -2016-06-13 20:20:48 INFO JobScheduler:59 - Total delay: 1465820440.968 s for time 8000 ms (execution: 0.204 s) -2016-06-13 20:20:48 INFO JobScheduler:59 - Starting job streaming job 9000 ms.0 from job set of time 9000 ms -2016-06-13 20:20:48 INFO ParallelCollectionRDD:59 - Removing RDD 38 from persistence list -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 38 -2016-06-13 20:20:48 INFO MapPartitionsRDD:59 - Removing RDD 43 from persistence list -2016-06-13 20:20:48 INFO ShuffledRDD:59 - Removing RDD 42 from persistence list -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 43 -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 42 -2016-06-13 20:20:48 INFO UnionRDD:59 - Removing RDD 41 from persistence list -2016-06-13 20:20:48 INFO MapPartitionsRDD:59 - Removing RDD 39 from persistence list -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 41 -2016-06-13 20:20:48 INFO BlockManager:59 - Removing RDD 39 -2016-06-13 20:20:48 INFO JobGenerator:59 - Checkpointing graph for time 8000 ms -2016-06-13 20:20:48 INFO DStreamGraph:59 - Updating checkpoint data for time 8000 ms -2016-06-13 20:20:48 INFO DStreamGraph:59 - Updated checkpoint data for time 8000 ms -2016-06-13 20:20:48 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Registering RDD 84 (apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Got job 16 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:20:48 INFO DAGScheduler:59 - Final stage: ResultStage 33(apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 32) -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting ShuffleMapStage 32 (MapPartitionsRDD[84] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=51026, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO MemoryStore:59 - ensureFreeSpace(2477) called with curMem=55770, maxMem=766075207 -2016-06-13 20:20:48 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:20:48 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:51099 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:20:48 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 32 (MapPartitionsRDD[84] at apply at Transformer.scala:22) -2016-06-13 20:20:48 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-06-13 20:20:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 72, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:20:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 73, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:20:49 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 73) -2016-06-13 20:20:49 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 72) -2016-06-13 20:20:49 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 72). 1159 bytes result sent to driver -2016-06-13 20:20:49 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 73). 1159 bytes result sent to driver -2016-06-13 20:20:49 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:20:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 72) in 17 ms on localhost (1/2) -2016-06-13 20:20:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 73) in 17 ms on localhost (2/2) -2016-06-13 20:20:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 20:20:49 INFO DAGScheduler:59 - ShuffleMapStage 32 (apply at Transformer.scala:22) finished in 0.018 s -2016-06-13 20:20:49 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:20:49 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:20:49 INFO DAGScheduler:59 - waiting: Set(ResultStage 33) -2016-06-13 20:20:49 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:20:49 INFO DAGScheduler:59 - Missing parents for ResultStage 33: List() -2016-06-13 20:20:49 INFO DAGScheduler:59 - Submitting ResultStage 33 (ShuffledRDD[85] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:20:49 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=58247, maxMem=766075207 -2016-06-13 20:20:49 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:20:49 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=63511, maxMem=766075207 -2016-06-13 20:20:49 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:20:49 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:51099 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:20:49 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (ShuffledRDD[85] at apply at Transformer.scala:22) -2016-06-13 20:20:49 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-06-13 20:20:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 74, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 75, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:49 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 74) -2016-06-13 20:20:49 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 75) -2016-06-13 20:20:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:20:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:49 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:20:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-06-13 20:20:49 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 75). 1161 bytes result sent to driver -2016-06-13 20:20:49 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 74). 1498 bytes result sent to driver -2016-06-13 20:20:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 74) in 12 ms on localhost (1/2) -2016-06-13 20:20:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 75) in 12 ms on localhost (2/2) -2016-06-13 20:20:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 20:20:49 INFO DAGScheduler:59 - ResultStage 33 (apply at Transformer.scala:22) finished in 0.014 s -2016-06-13 20:20:49 INFO DAGScheduler:59 - Job 16 finished: apply at Transformer.scala:22, took 0.057783 s -2016-06-13 20:20:49 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 450.0 and centroid: [-0.4722098164150225,0.9642763266691793,-1.6666072662538036,1.2709270669491914,0.37911973669212223] -2016-06-13 20:20:49 INFO JobScheduler:59 - Finished job streaming job 9000 ms.0 from job set of time 9000 ms -2016-06-13 20:20:49 INFO JobScheduler:59 - Starting job streaming job 9000 ms.1 from job set of time 9000 ms -2016-06-13 20:20:49 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:20:49 INFO DAGScheduler:59 - Registering RDD 53 (apply at Transformer.scala:22) -2016-06-13 20:20:49 INFO DAGScheduler:59 - Got job 17 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:20:49 INFO DAGScheduler:59 - Final stage: ResultStage 35(apply at Transformer.scala:22) -2016-06-13 20:20:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 34) -2016-06-13 20:20:49 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 34) -2016-06-13 20:20:49 INFO DAGScheduler:59 - Submitting ShuffleMapStage 34 (UnionRDD[53] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:20:49 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=66205, maxMem=766075207 -2016-06-13 20:20:49 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:20:49 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=69485, maxMem=766075207 -2016-06-13 20:20:49 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.5 MB) -2016-06-13 20:20:49 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:51099 (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:20:49 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:49 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 34 (UnionRDD[53] at apply at Transformer.scala:22) -2016-06-13 20:20:49 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 3 tasks -2016-06-13 20:20:49 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:20:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 76, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:20:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 77, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:20:49 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 77) -2016-06-13 20:20:49 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 76) -2016-06-13 20:20:49 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 76). 1159 bytes result sent to driver -2016-06-13 20:20:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 78, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:20:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 76) in 14 ms on localhost (1/3) -2016-06-13 20:20:49 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 78) -2016-06-13 20:20:49 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 77). 1159 bytes result sent to driver -2016-06-13 20:20:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 77) in 19 ms on localhost (2/3) -2016-06-13 20:20:49 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 78). 1159 bytes result sent to driver -2016-06-13 20:20:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 78) in 12 ms on localhost (3/3) -2016-06-13 20:20:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-06-13 20:20:49 INFO DAGScheduler:59 - ShuffleMapStage 34 (apply at Transformer.scala:22) finished in 0.025 s -2016-06-13 20:20:49 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:20:49 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:20:49 INFO DAGScheduler:59 - waiting: Set(ResultStage 35) -2016-06-13 20:20:49 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:20:49 INFO DAGScheduler:59 - Missing parents for ResultStage 35: List() -2016-06-13 20:20:49 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[86] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:20:49 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=71492, maxMem=766075207 -2016-06-13 20:20:49 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:20:49 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=74732, maxMem=766075207 -2016-06-13 20:20:49 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:20:49 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:51099 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:20:49 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[86] at apply at Transformer.scala:22) -2016-06-13 20:20:49 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-06-13 20:20:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 79, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 80, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:49 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 80) -2016-06-13 20:20:49 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 79) -2016-06-13 20:20:49 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:20:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:49 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 80). 1172 bytes result sent to driver -2016-06-13 20:20:49 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:20:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 80) in 4 ms on localhost (1/2) -2016-06-13 20:20:49 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 79). 1180 bytes result sent to driver -2016-06-13 20:20:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 79) in 11 ms on localhost (2/2) -2016-06-13 20:20:49 INFO DAGScheduler:59 - ResultStage 35 (apply at Transformer.scala:22) finished in 0.012 s -2016-06-13 20:20:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-06-13 20:20:49 INFO DAGScheduler:59 - Job 17 finished: apply at Transformer.scala:22, took 0.062195 s -2016-06-13 20:20:49 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:20:49 INFO JobScheduler:59 - Finished job streaming job 9000 ms.1 from job set of time 9000 ms -2016-06-13 20:20:49 INFO ParallelCollectionRDD:59 - Removing RDD 44 from persistence list -2016-06-13 20:20:49 INFO JobScheduler:59 - Total delay: 1465820440.117 s for time 9000 ms (execution: 0.148 s) -2016-06-13 20:20:49 INFO JobScheduler:59 - Starting job streaming job 10000 ms.0 from job set of time 10000 ms -2016-06-13 20:20:49 INFO MapPartitionsRDD:59 - Removing RDD 49 from persistence list -2016-06-13 20:20:49 INFO BlockManager:59 - Removing RDD 44 -2016-06-13 20:20:49 INFO BlockManager:59 - Removing RDD 49 -2016-06-13 20:20:49 INFO ShuffledRDD:59 - Removing RDD 48 from persistence list -2016-06-13 20:20:49 INFO UnionRDD:59 - Removing RDD 47 from persistence list -2016-06-13 20:20:49 INFO BlockManager:59 - Removing RDD 48 -2016-06-13 20:20:49 INFO MapPartitionsRDD:59 - Removing RDD 45 from persistence list -2016-06-13 20:20:49 INFO BlockManager:59 - Removing RDD 47 -2016-06-13 20:20:49 INFO BlockManager:59 - Removing RDD 45 -2016-06-13 20:20:49 INFO JobGenerator:59 - Checkpointing graph for time 9000 ms -2016-06-13 20:20:49 INFO DStreamGraph:59 - Updating checkpoint data for time 9000 ms -2016-06-13 20:20:49 INFO DStreamGraph:59 - Updated checkpoint data for time 9000 ms -2016-06-13 20:20:49 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-2000.bk -2016-06-13 20:20:49 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:20:49 INFO DAGScheduler:59 - Registering RDD 87 (apply at Transformer.scala:22) -2016-06-13 20:20:49 INFO DAGScheduler:59 - Got job 18 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:20:49 INFO DAGScheduler:59 - Final stage: ResultStage 37(apply at Transformer.scala:22) -2016-06-13 20:20:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 36) -2016-06-13 20:20:49 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 36) -2016-06-13 20:20:49 INFO DAGScheduler:59 - Submitting ShuffleMapStage 36 (MapPartitionsRDD[87] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:20:49 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=76535, maxMem=766075207 -2016-06-13 20:20:49 INFO CheckpointWriter:59 - Checkpoint for time 3000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-3000', took 27494 bytes and 180 ms -2016-06-13 20:20:49 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:20:49 INFO DStreamGraph:59 - Clearing checkpoint data for time 3000 ms -2016-06-13 20:20:49 INFO CheckpointWriter:59 - Saving checkpoint for time 4000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-4000' -2016-06-13 20:20:49 INFO DStreamGraph:59 - Cleared checkpoint data for time 3000 ms -2016-06-13 20:20:49 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:20:49 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/receivedBlockMetadata older than 2000: -2016-06-13 20:20:49 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/receivedBlockMetadata older than 2000 -2016-06-13 20:20:49 INFO InputInfoTracker:59 - remove old batch metadata: 1000 ms -2016-06-13 20:20:49 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=81279, maxMem=766075207 -2016-06-13 20:20:49 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:20:49 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:51099 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:20:49 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 36 (MapPartitionsRDD[87] at apply at Transformer.scala:22) -2016-06-13 20:20:49 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-06-13 20:20:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 81, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:20:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 82, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:20:49 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 82) -2016-06-13 20:20:49 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 81) -2016-06-13 20:20:49 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 82). 1159 bytes result sent to driver -2016-06-13 20:20:49 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 81). 1159 bytes result sent to driver -2016-06-13 20:20:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 82) in 16 ms on localhost (1/2) -2016-06-13 20:20:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 81) in 18 ms on localhost (2/2) -2016-06-13 20:20:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-06-13 20:20:49 INFO DAGScheduler:59 - ShuffleMapStage 36 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:20:49 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:20:49 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:20:49 INFO DAGScheduler:59 - waiting: Set(ResultStage 37) -2016-06-13 20:20:49 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:20:49 INFO DAGScheduler:59 - Missing parents for ResultStage 37: List() -2016-06-13 20:20:49 INFO DAGScheduler:59 - Submitting ResultStage 37 (ShuffledRDD[88] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:20:49 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=83761, maxMem=766075207 -2016-06-13 20:20:49 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:20:49 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=89025, maxMem=766075207 -2016-06-13 20:20:49 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:20:49 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:20:49 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:51099 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:20:49 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (ShuffledRDD[88] at apply at Transformer.scala:22) -2016-06-13 20:20:49 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-06-13 20:20:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 83, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 84, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:49 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 83) -2016-06-13 20:20:49 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 84) -2016-06-13 20:20:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:20:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:49 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:20:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:20:49 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 84). 1161 bytes result sent to driver -2016-06-13 20:20:49 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 83). 1498 bytes result sent to driver -2016-06-13 20:20:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 84) in 15 ms on localhost (1/2) -2016-06-13 20:20:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 83) in 18 ms on localhost (2/2) -2016-06-13 20:20:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-06-13 20:20:49 INFO DAGScheduler:59 - ResultStage 37 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:20:49 INFO DAGScheduler:59 - Job 18 finished: apply at Transformer.scala:22, took 0.061739 s -2016-06-13 20:20:49 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 500.0 and centroid: [-0.4725511979691583,0.9644503899125422,-1.668776373542808,1.2721254429935838,0.37815209739836425] -2016-06-13 20:20:49 INFO JobScheduler:59 - Finished job streaming job 10000 ms.0 from job set of time 10000 ms -2016-06-13 20:20:49 INFO JobScheduler:59 - Starting job streaming job 10000 ms.1 from job set of time 10000 ms -2016-06-13 20:20:49 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:20:49 INFO DAGScheduler:59 - Registering RDD 59 (apply at Transformer.scala:22) -2016-06-13 20:20:49 INFO DAGScheduler:59 - Got job 19 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:20:49 INFO DAGScheduler:59 - Final stage: ResultStage 39(apply at Transformer.scala:22) -2016-06-13 20:20:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 38) -2016-06-13 20:20:49 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 38) -2016-06-13 20:20:49 INFO DAGScheduler:59 - Submitting ShuffleMapStage 38 (UnionRDD[59] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:20:49 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=91721, maxMem=766075207 -2016-06-13 20:20:49 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:20:49 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=95001, maxMem=766075207 -2016-06-13 20:20:49 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.5 MB) -2016-06-13 20:20:49 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:51099 (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:20:49 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:49 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 38 (UnionRDD[59] at apply at Transformer.scala:22) -2016-06-13 20:20:49 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 3 tasks -2016-06-13 20:20:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 85, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:20:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 86, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:20:49 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 86) -2016-06-13 20:20:49 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 85) -2016-06-13 20:20:49 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:20:49 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 85). 1159 bytes result sent to driver -2016-06-13 20:20:49 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 87, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:20:49 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 87) -2016-06-13 20:20:49 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 86). 1159 bytes result sent to driver -2016-06-13 20:20:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 85) in 16 ms on localhost (1/3) -2016-06-13 20:20:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 86) in 18 ms on localhost (2/3) -2016-06-13 20:20:49 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 87). 1159 bytes result sent to driver -2016-06-13 20:20:49 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 87) in 11 ms on localhost (3/3) -2016-06-13 20:20:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-06-13 20:20:49 INFO DAGScheduler:59 - ShuffleMapStage 38 (apply at Transformer.scala:22) finished in 0.027 s -2016-06-13 20:20:49 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:20:49 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:20:49 INFO DAGScheduler:59 - waiting: Set(ResultStage 39) -2016-06-13 20:20:49 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:20:49 INFO DAGScheduler:59 - Missing parents for ResultStage 39: List() -2016-06-13 20:20:49 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[89] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:20:49 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=97008, maxMem=766075207 -2016-06-13 20:20:49 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:20:49 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=100248, maxMem=766075207 -2016-06-13 20:20:49 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:20:49 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:51099 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:20:49 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:20:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[89] at apply at Transformer.scala:22) -2016-06-13 20:20:49 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-06-13 20:20:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:20:49 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 88) -2016-06-13 20:20:49 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:20:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:49 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 89) -2016-06-13 20:20:49 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:20:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:20:49 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 89). 1172 bytes result sent to driver -2016-06-13 20:20:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 89) in 15 ms on localhost (1/2) -2016-06-13 20:20:49 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:20:49 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 88). 1180 bytes result sent to driver -2016-06-13 20:20:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 88) in 25 ms on localhost (2/2) -2016-06-13 20:20:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-06-13 20:20:49 INFO DAGScheduler:59 - ResultStage 39 (apply at Transformer.scala:22) finished in 0.026 s -2016-06-13 20:20:49 INFO DAGScheduler:59 - Job 19 finished: apply at Transformer.scala:22, took 0.080065 s -2016-06-13 20:20:49 INFO JobScheduler:59 - Finished job streaming job 10000 ms.1 from job set of time 10000 ms -2016-06-13 20:20:49 INFO JobScheduler:59 - Total delay: 1465820439.284 s for time 10000 ms (execution: 0.166 s) -2016-06-13 20:20:49 INFO ParallelCollectionRDD:59 - Removing RDD 50 from persistence list -2016-06-13 20:20:49 INFO BlockManager:59 - Removing RDD 50 -2016-06-13 20:20:49 INFO MapPartitionsRDD:59 - Removing RDD 55 from persistence list -2016-06-13 20:20:49 INFO ShuffledRDD:59 - Removing RDD 54 from persistence list -2016-06-13 20:20:49 INFO BlockManager:59 - Removing RDD 55 -2016-06-13 20:20:49 INFO UnionRDD:59 - Removing RDD 53 from persistence list -2016-06-13 20:20:49 INFO BlockManager:59 - Removing RDD 54 -2016-06-13 20:20:49 INFO MapPartitionsRDD:59 - Removing RDD 51 from persistence list -2016-06-13 20:20:49 INFO BlockManager:59 - Removing RDD 53 -2016-06-13 20:20:49 INFO JobGenerator:59 - Checkpointing graph for time 10000 ms -2016-06-13 20:20:49 INFO DStreamGraph:59 - Updating checkpoint data for time 10000 ms -2016-06-13 20:20:49 INFO DStreamGraph:59 - Updated checkpoint data for time 10000 ms -2016-06-13 20:20:49 INFO BlockManager:59 - Removing RDD 51 -2016-06-13 20:20:49 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-2000 -2016-06-13 20:20:49 INFO StreamingKMeansSuite:59 - Output generated in 3131 milliseconds -2016-06-13 20:20:49 INFO CheckpointWriter:59 - Checkpoint for time 4000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-4000', took 27486 bytes and 192 ms -2016-06-13 20:20:49 INFO DStreamGraph:59 - Clearing checkpoint data for time 4000 ms -2016-06-13 20:20:49 INFO CheckpointWriter:59 - Saving checkpoint for time 5000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-5000' -2016-06-13 20:20:49 INFO DStreamGraph:59 - Cleared checkpoint data for time 4000 ms -2016-06-13 20:20:49 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:20:49 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:20:49 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/receivedBlockMetadata older than 3000: -2016-06-13 20:20:49 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:20:49 INFO InputInfoTracker:59 - remove old batch metadata: 2000 ms -2016-06-13 20:20:49 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/receivedBlockMetadata older than 3000 -2016-06-13 20:20:49 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:20:49 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:20:49 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:20:49 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:20:49 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:20:49 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:20:49 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:20:49 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:20:49 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-3000.bk -2016-06-13 20:20:49 INFO CheckpointWriter:59 - Checkpoint for time 5000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-5000', took 27482 bytes and 102 ms -2016-06-13 20:20:49 INFO DStreamGraph:59 - Clearing checkpoint data for time 5000 ms -2016-06-13 20:20:49 INFO CheckpointWriter:59 - Saving checkpoint for time 6000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-6000' -2016-06-13 20:20:49 INFO DStreamGraph:59 - Cleared checkpoint data for time 5000 ms -2016-06-13 20:20:49 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:20:49 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/receivedBlockMetadata older than 4000: -2016-06-13 20:20:49 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/receivedBlockMetadata older than 4000 -2016-06-13 20:20:49 INFO InputInfoTracker:59 - remove old batch metadata: 3000 ms -2016-06-13 20:20:49 INFO JobGenerator:59 - Stopping JobGenerator immediately -2016-06-13 20:20:49 INFO RecurringTimer:59 - Stopped timer for JobGenerator after time 10000 -2016-06-13 20:20:59 INFO CheckpointWriter:59 - CheckpointWriter executor terminated ? false, waited for 10002 ms. -2016-06-13 20:20:59 INFO JobGenerator:59 - Stopped JobGenerator -2016-06-13 20:20:59 INFO JobScheduler:59 - Stopped JobScheduler -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/batch,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/streaming,null} -2016-06-13 20:20:59 INFO StreamingContext:59 - StreamingContext stopped successfully -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/batch/json,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/json,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:20:59 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:20:59 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-3000 -2016-06-13 20:20:59 INFO CheckpointWriter:59 - Checkpoint for time 6000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956/checkpoint-6000', took 27478 bytes and 10098 ms -2016-06-13 20:20:59 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 20:20:59 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:20:59 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:20:59 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:20:59 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:20:59 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:20:59 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:20:59 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:20:59 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:20:59 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:20:59 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 20:21:00 INFO StreamingKMeansSuite:59 - - -===== FINISHED o.a.s.mllib.clustering.kmeans.StreamingKMeansSuite: 'accuracy for single center and equivalence to grand average' ===== - -2016-06-13 20:21:00 WARN StreamingContext:71 - StreamingContext has already been stopped -2016-06-13 20:21:00 INFO SparkContext:59 - SparkContext already stopped. -2016-06-13 20:21:00 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 20:21:00 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-36841f45-cdae-4750-83a7-c033260e3590 -2016-06-13 20:21:00 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956 -2016-06-13 20:21:00 ERROR ShutdownHookManager:96 - Exception while deleting Spark temp dir: C:\Users\xubo\AppData\Local\Temp\spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956 -java.io.IOException: Failed to delete: C:\Users\xubo\AppData\Local\Temp\spark-59809000-9dad-4e5f-aaf3-f4eaf23b6956 - at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:884) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:63) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:60) - at scala.collection.mutable.HashSet.foreach(HashSet.scala:79) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1.apply$mcV$sp(ShutdownHookManager.scala:60) - at org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:264) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1699) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply$mcV$sp(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:234) - at scala.util.Try$.apply(Try.scala:161) - at org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:216) - at org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54) -2016-06-13 20:22:55 INFO StreamingKMeansSuite:59 - Using manual clock -2016-06-13 20:22:55 INFO StreamingKMeansSuite:59 - - -===== TEST OUTPUT FOR o.a.s.mllib.clustering.kmeans.StreamingKMeansSuite: 'accuracy for single center and equivalence to grand average' ===== - -2016-06-13 20:22:55 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:22:56 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 20:22:56 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:22:56 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:22:56 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:22:56 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:22:57 INFO Remoting:74 - Starting remoting -2016-06-13 20:22:57 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:51122] -2016-06-13 20:22:57 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51122. -2016-06-13 20:22:57 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:22:57 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:22:57 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-fe866539-e3ce-4640-a9f1-9b63a9146472 -2016-06-13 20:22:57 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:22:57 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-a70c3031-3b3d-4786-9b33-ebf3ce97f394\httpd-ac7fec8d-6726-4cfd-9446-872df101745a -2016-06-13 20:22:57 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:22:57 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:22:57 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51123 -2016-06-13 20:22:57 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51123. -2016-06-13 20:22:57 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:22:57 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:22:57 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 20:22:57 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 20:22:57 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 20:22:57 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:22:57 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:22:58 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51134. -2016-06-13 20:22:58 INFO NettyBlockTransferService:59 - Server created on 51134 -2016-06-13 20:22:58 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:22:58 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51134 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51134) -2016-06-13 20:22:58 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:22:59 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 20:22:59 INFO StreamingKMeansSuite:59 - numBatches = 10, numExpectedOutput = 10 -2016-06-13 20:22:59 INFO ForEachDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:22:59 INFO TestInputStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:22:59 INFO TestInputStream:59 - Slide time = 1000 ms -2016-06-13 20:22:59 INFO TestInputStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:22:59 INFO TestInputStream:59 - Checkpoint interval = null -2016-06-13 20:22:59 INFO TestInputStream:59 - Remember duration = 1000 ms -2016-06-13 20:22:59 INFO TestInputStream:59 - Initialized and validated org.apache.spark.streaming.TestInputStream@6aa0f050 -2016-06-13 20:22:59 INFO ForEachDStream:59 - Slide time = 1000 ms -2016-06-13 20:22:59 INFO ForEachDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:22:59 INFO ForEachDStream:59 - Checkpoint interval = null -2016-06-13 20:22:59 INFO ForEachDStream:59 - Remember duration = 1000 ms -2016-06-13 20:22:59 INFO ForEachDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.ForEachDStream@2aefa457 -2016-06-13 20:22:59 INFO TestOutputStreamWithPartitions:59 - metadataCleanupDelay = -1 -2016-06-13 20:22:59 INFO MappedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:22:59 INFO ShuffledDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:22:59 INFO TransformedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:22:59 INFO MappedDStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:22:59 INFO TestInputStream:59 - metadataCleanupDelay = -1 -2016-06-13 20:22:59 INFO TestInputStream:59 - Slide time = 1000 ms -2016-06-13 20:22:59 INFO TestInputStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:22:59 INFO TestInputStream:59 - Checkpoint interval = null -2016-06-13 20:22:59 INFO TestInputStream:59 - Remember duration = 1000 ms -2016-06-13 20:22:59 INFO TestInputStream:59 - Initialized and validated org.apache.spark.streaming.TestInputStream@6aa0f050 -2016-06-13 20:22:59 INFO MappedDStream:59 - Slide time = 1000 ms -2016-06-13 20:22:59 INFO MappedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:22:59 INFO MappedDStream:59 - Checkpoint interval = null -2016-06-13 20:22:59 INFO MappedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:22:59 INFO MappedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@8ab8e20 -2016-06-13 20:22:59 INFO TransformedDStream:59 - Slide time = 1000 ms -2016-06-13 20:22:59 INFO TransformedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:22:59 INFO TransformedDStream:59 - Checkpoint interval = null -2016-06-13 20:22:59 INFO TransformedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:22:59 INFO TransformedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.TransformedDStream@695d8c27 -2016-06-13 20:22:59 INFO ShuffledDStream:59 - Slide time = 1000 ms -2016-06-13 20:22:59 INFO ShuffledDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:22:59 INFO ShuffledDStream:59 - Checkpoint interval = null -2016-06-13 20:22:59 INFO ShuffledDStream:59 - Remember duration = 1000 ms -2016-06-13 20:22:59 INFO ShuffledDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.ShuffledDStream@46316ef0 -2016-06-13 20:22:59 INFO MappedDStream:59 - Slide time = 1000 ms -2016-06-13 20:22:59 INFO MappedDStream:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:22:59 INFO MappedDStream:59 - Checkpoint interval = null -2016-06-13 20:22:59 INFO MappedDStream:59 - Remember duration = 1000 ms -2016-06-13 20:22:59 INFO MappedDStream:59 - Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@384e68c3 -2016-06-13 20:22:59 INFO TestOutputStreamWithPartitions:59 - Slide time = 1000 ms -2016-06-13 20:22:59 INFO TestOutputStreamWithPartitions:59 - Storage level = StorageLevel(false, false, false, false, 1) -2016-06-13 20:22:59 INFO TestOutputStreamWithPartitions:59 - Checkpoint interval = null -2016-06-13 20:22:59 INFO TestOutputStreamWithPartitions:59 - Remember duration = 1000 ms -2016-06-13 20:22:59 INFO TestOutputStreamWithPartitions:59 - Initialized and validated org.apache.spark.streaming.TestOutputStreamWithPartitions@1d5a09f0 -2016-06-13 20:22:59 INFO RecurringTimer:59 - Started timer for JobGenerator at time 1000 -2016-06-13 20:22:59 INFO JobGenerator:59 - Started JobGenerator at 1000 ms -2016-06-13 20:22:59 INFO JobScheduler:59 - Started JobScheduler -2016-06-13 20:23:00 INFO StreamingContext:59 - StreamingContext started -2016-06-13 20:23:00 INFO StreamingKMeansSuite:59 - Manual clock before advancing = 0 -2016-06-13 20:23:00 INFO StreamingKMeansSuite:59 - Manual clock after advancing = 10000 -2016-06-13 20:23:00 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:00 INFO TestInputStream:59 - Computing RDD for time 1000 ms -2016-06-13 20:23:00 INFO TestInputStream:59 - Created RDD 0 with Vector([-0.35685422167583125,0.9995288307555166,-1.5877809386448758,1.210845548490831,0.56222524157234], [-0.45216537133396134,0.8040574757345051,-1.5589888701250443,1.115603017032467,0.344649973028188], [-0.3801675945145341,1.0697589561731022,-1.6486682116186815,1.2790268654063703,0.5116371486214886], [-0.545300880562902,1.1093057253978866,-1.7582382472264182,1.211400006292075,0.2932044815946008], [-0.3995483137357882,0.929119621864747,-1.7401380541085598,1.2990457426846032,0.299090686584923], [-0.5964001160322344,1.0378164371074625,-1.553411818459791,1.2193968564632125,0.4492703031123452], [-0.33949398265706626,1.1369769008642154,-1.7816495259375202,1.2857769527411067,0.5642853362323552], [-0.4522936770981281,1.090819158309221,-1.4785463285656288,1.2734882859357213,0.31864385887835545], [-0.4895260213585818,1.024567290792244,-1.7744155605537828,1.4515329700677304,0.39720325477885093], [-0.5479321442182269,0.8926951810306638,-1.617388979581139,1.3192738326981768,0.3243774353000105], [-0.5468296230191492,0.9034725811394896,-1.8269524875493084,1.161899613497355,0.22607191229030854], [-0.4344367225435325,0.9054178856043291,-1.6658399540877864,1.13578899284944,0.46511292492021983], [-0.5914600674966851,0.9727747336030947,-1.7411162798640765,1.3902196062020102,0.43158411906136107], [-0.5644425684165094,0.996354296088912,-1.817848647277835,1.3768030733201289,0.28609432985591315], [-0.44148386701129744,1.025737906243747,-1.7073695034902865,1.2539087121413848,0.349242056080172], [-0.5287113084613145,0.8475916475163896,-1.8166057479945195,1.238631411179877,0.36922519791770014], [-0.2540595910515851,0.9698131032346381,-1.5866552625154318,1.2179915976157372,0.3240398576394729], [-0.5026644816450073,1.109581665015373,-1.7151161774591086,1.3901513268104455,0.39042855132933163], [-0.4131172722481001,1.0096927045418442,-1.811286466550916,1.2719080737538961,0.28293757989913904], [-0.4515790075180954,1.0935474561719332,-1.7896107864561164,1.3203383417885988,0.2994362800834273], [-0.4171607573128071,1.0818260184642114,-1.8219543038308885,1.2496462030096327,0.4964658371791977], [-0.45586902589910655,1.0378742745036647,-1.7000458648640797,1.446592332373813,0.348566870310007], [-0.3532087778209343,0.8044437973592731,-1.7162629066191757,1.3339281879504057,0.17946188844185906], [-0.5154063008996834,0.9595656827532482,-1.6519653379002484,1.322654750805598,0.3008929692388781], [-0.3967887546418169,0.7908879805460386,-1.7242582851383725,1.2141584886700174,0.36827019559433166], [-0.48352710482779404,1.0255449792765485,-1.7171397454598174,1.284877805985566,0.437232715780399], [-0.40324856443707463,1.020485534999759,-1.6517518343613002,1.2864020019792224,0.3984832850799499], [-0.6517069948565133,0.969461018860354,-1.684015510299471,1.2027906385718596,0.2014860155598368], [-0.6148278408211463,0.9795135452912322,-1.8642733722424907,1.2720240024566645,0.39785426587083933], [-0.6409674819545198,1.1778584543507387,-1.8094969072333307,1.24145841048891,0.1624672183638662], [-0.6368691555002903,0.8170131462860201,-1.6827833832635621,1.3031179618920097,0.31628673987022604], [-0.43096648399184834,0.9655746876954024,-1.7845188100637381,1.4052721272088269,0.5845842279495], [-0.4442125841627056,1.0257872008493307,-1.7242589434969455,1.2745428690859977,0.39998857867859966], [-0.4527127515828855,0.9000469054331385,-1.7497092961121075,1.2451107418571954,0.4344380036579415], [-0.6030914309562295,0.9699928952430515,-1.712604726536663,1.38261404269885,0.48344898725768926], [-0.34976726689041593,0.9355870894961762,-1.5766155567170825,1.3590936489794139,0.35690680463154484], [-0.575628267406687,0.9538063357613722,-1.6445003616008003,1.2104804829930789,0.3911315521177748], [-0.5582822637309535,0.9339694428739754,-1.6511037013459962,1.4401460976648193,0.3607766390547849], [-0.5468811641579405,0.997556943752324,-1.778985900216746,1.2440321529105447,0.4204585578447196], [-0.48994207115736743,1.0755700976056979,-1.5322042869816734,1.2132674597398667,0.5961642366275318], [-0.5345824744371771,0.7659168558371742,-1.7009697565551642,1.2200491259133113,0.24064266183571642], [-0.4425411108434083,0.9636579302553172,-1.6769715787345627,1.2060504970822528,0.1832578105698996], [-0.2803908635114948,0.9814260387904405,-1.6744903280757537,1.438894231115956,0.5307857877828275], [-0.4646639723438268,0.9422153316765177,-1.8936280948227913,1.1755014401606392,0.3269897050872188], [-0.5794967530942349,1.008452319256215,-1.8095212163297067,1.3053459835247618,0.3862084672749757], [-0.36564413314560745,0.9550404460199052,-1.671332555981891,1.2387784773127055,0.29524181369582553], [-0.4947387784852314,0.8968969114403345,-1.5032027608518188,1.263293789437244,0.41872133210307433], [-0.5901163249097482,0.9195890740589755,-1.76975579729397,1.2233696400583831,0.3403182341587175], [-0.7175598778778954,0.8837946345395844,-1.8329343739105675,1.3562350855120264,0.30734388952604197], [-0.4376962364418643,1.0046436887819825,-1.5427982731479972,1.1077094728701697,0.30461693506090337]) -2016-06-13 20:23:00 INFO JobScheduler:59 - Added jobs for time 1000 ms -2016-06-13 20:23:00 INFO TestInputStream:59 - Computing RDD for time 2000 ms -2016-06-13 20:23:00 INFO JobScheduler:59 - Starting job streaming job 1000 ms.0 from job set of time 1000 ms -2016-06-13 20:23:00 INFO TestInputStream:59 - Created RDD 6 with Vector([-0.5385984930004446,0.9989931115946847,-1.745310682734983,1.2763556288677276,0.41555544930746574], [-0.5140757026211953,0.9616396809310985,-1.6092748223737894,1.3080875431688948,0.3958240822167327], [-0.3317121400990889,1.1311521924692884,-1.5940394948396088,1.235663715627094,0.5352186835894586], [-0.3566003302425871,0.8649446433306678,-1.6281158497086519,1.0841327390682376,0.33108928300382867], [-0.4898654567486354,1.0095506134582393,-1.7900184464939166,1.4376638486285889,0.5476994415325478], [-0.5067718408631526,0.8330942372560937,-1.5447521454199076,1.3956201717944832,0.40502500771937533], [-0.45899690569395396,1.0114199686087044,-1.6793704227956527,1.2264029484806054,0.3617800498770533], [-0.42389468031403765,0.9744019756374155,-1.4339024359998267,1.3506788012287876,0.5071391796420295], [-0.3445608603995446,0.8999444931201762,-1.8295489983610813,1.3813011802363127,0.4406110935124823], [-0.4720580057434039,0.9616388382715773,-1.7885815681274932,1.2231064430079939,0.49611588805959794], [-0.6877856894979741,0.9509033529307679,-1.7111111174594673,1.1650825853084499,0.36541463301831684], [-0.41906769746785616,0.9643844813018809,-1.7088669498454399,1.2743442947780539,0.37857583863641653], [-0.45604703784103723,0.9704467329255722,-1.6908561828878124,1.2661005251088149,0.34761517755721416], [-0.45216413072759976,1.125638260528155,-1.8672240017957615,1.388348280279974,0.41289821746105937], [-0.6041998663111035,0.9518745991902821,-1.7482553616491074,1.3027968384522153,0.33003148862622383], [-0.7645915689875407,0.9231306641016475,-1.5908706609397762,1.0416797405804958,0.3435391589642622], [-0.5511129470810242,0.9789516647349409,-1.5896352104287992,1.2985407772348172,0.3074739718251722], [-0.3060111421519473,0.9693226097887153,-1.6503499878201646,1.2050435546957032,0.20529101796457794], [-0.41814813350738833,0.9065345647328998,-1.7369684308945867,1.3095490239413528,0.3931707179477215], [-0.7452070526618801,1.010695258373889,-1.6799990639878284,1.351913921189404,0.48041731347418104], [-0.19474072480894772,0.8574730260551575,-1.7114945513022097,1.2484211088395596,0.35791459113273333], [-0.679333385903053,1.136043445571062,-1.6819398713814546,1.2914383599572574,0.4846586154592743], [-0.38963314954900546,1.0499014443531718,-1.7075189569343263,1.3966853721193921,0.4922463621111953], [-0.4771733790096556,0.8615507710844875,-1.7824679274745288,1.1550649309651535,0.4634518570903764], [-0.46851141840829336,0.9490804068188984,-1.6654371185103947,1.1748885058064462,0.362811152170017], [-0.2486779458724137,1.0146748639649252,-1.8129854069448728,1.3198830969564963,0.3848127258680788], [-0.37278156677721497,0.9026389048304428,-1.6338760097656873,1.2146194636725334,0.305407696426058], [-0.5109164214840503,1.1025203353037651,-1.5805383770959172,1.3659800632138837,0.2507490333376023], [-0.42658774618748607,0.9583040463281658,-1.535401873083165,1.2270967800953845,0.30931926334482973], [-0.5761089637758711,0.9608023900436299,-1.6773446541045054,1.3896056325873751,0.3003385242281313], [-0.333657352857868,0.9284640162518879,-1.6506283984425263,1.4233119402331722,0.4817086358456392], [-0.4131260123218083,0.9257004348959492,-1.6385111064595121,1.2553085674173774,0.35994561408421055], [-0.41619483777131644,0.9791552402269297,-1.6621579631275758,1.1639596596963502,0.3356350519347727], [-0.42150886121352676,0.9497809597217938,-1.4493400602428224,1.2037323844449976,0.28412005647965366], [-0.4036704221590921,0.8682473056100086,-1.4499044130725358,1.4367312949083602,0.29801296100779273], [-0.7347169576181178,0.8472443819718873,-1.6912964247755202,1.1098267823656842,0.5128488081575056], [-0.4599723079910277,1.0313773063743503,-1.6693647567489862,1.1183374503284873,0.3065899617042307], [-0.5155727886799809,0.9793287424722477,-1.81766244647388,1.264051537384318,0.38002468301378506], [-0.461809330593855,0.94300958630738,-1.7083227759840551,1.2779916838408791,0.5269758739704866], [-0.6518981010977536,0.8745407308399856,-1.5963574332371322,1.2679054180945675,0.4099968212180263], [-0.46480686752956457,0.9217771428753339,-1.5266876398223685,1.1382426745675795,0.4778996312070509], [-0.4852820635337406,0.8218636910073019,-1.7767449598018135,1.1371130292343117,0.33381426065377506], [-0.41472873571993296,1.1622114863170214,-1.464996886657917,1.3123782230466363,0.2979393535788235], [-0.38817680425075585,1.0268858467994189,-1.5893492622912915,1.2621731986065745,0.435168936860357], [-0.4138872105825818,1.0639679744840822,-1.6793705616475731,1.0262313606697269,0.26542788201223744], [-0.32689002022123786,0.9411685364764196,-1.6079547828400877,1.222649045231373,0.44202772328420886], [-0.4534191990015764,0.9019382096024549,-1.6042217752367725,1.313476911051435,0.3118812623960018], [-0.3642257288812764,1.0568954661740726,-1.6408419112357935,1.3879191143662102,0.43428969659022393], [-0.4436033491575346,0.951120701402773,-1.7396961236381308,1.0696513684387006,0.1983721283216616], [-0.4195531379041837,0.9339655248016878,-1.6369085623469266,1.2113780206799198,0.32646374286948043]) -2016-06-13 20:23:00 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:00 INFO JobScheduler:59 - Added jobs for time 2000 ms -2016-06-13 20:23:00 INFO TestInputStream:59 - Computing RDD for time 3000 ms -2016-06-13 20:23:00 INFO TestInputStream:59 - Created RDD 13 with Vector([-0.3676131534931217,0.7205292433695121,-1.6148690586461516,1.318613134832133,0.2894000707552024], [-0.63227225792052,1.0480160164806966,-1.7186123125513257,1.3123133303248091,0.4835019323786134], [-0.5130669794355894,1.0744409567693907,-1.5423474727562314,1.4075951590487494,0.34400766625670576], [-0.6214787708363003,0.9941757670924664,-1.7059517928152113,1.1668792946674045,0.5351190865534026], [-0.5610084057337338,0.7736203857280138,-1.8315479385898956,1.3209260735925115,0.28582121198324356], [-0.37376999266386424,0.8253313331048502,-1.7924436246324382,1.3068506619750972,0.2325763177837035], [-0.5087037004414274,0.9607866442106467,-1.812291073591455,1.284458726328898,0.5328516211621773], [-0.3727437082846784,1.028767182687076,-1.7448007272489838,1.0984753757565202,0.3541211398247694], [-0.3814571980697206,1.0819060589675644,-1.733976127510556,1.2515969788209256,0.3640947425626813], [-0.38693583578076807,0.9054649043220095,-1.601255604158471,1.1869423507487187,0.27285330937505237], [-0.453879517976784,0.8265450932182703,-1.696264660015251,1.1219045234195482,0.3528336963815211], [-0.6440396833150053,0.9147072341447406,-1.574634779931242,1.2807375427757628,0.3400101945021918], [-0.5574376128841423,0.9786892277605028,-1.6539643457077906,1.2987195892477945,0.22091568760798275], [-0.3219564568616685,0.7344099605460779,-1.659291949575684,1.3277750279213558,0.23553171253321456], [-0.42287205017678026,0.8870628278654091,-1.7860892614211916,1.3638413188637932,0.35992654268460605], [-0.5114214572125407,1.0739488843585667,-1.803218590871721,1.2580620741259871,0.42523338095005675], [-0.49124237559267386,0.9657816736108445,-1.6156494419042913,1.3063855681095915,0.54155197550267], [-0.3951919316276779,1.144136446459301,-1.6606072772152598,1.4114586343482727,0.3224073809014021], [-0.439470602003815,1.0070997021461388,-1.4975209021956797,1.2890203902600919,0.22540217793727096], [-0.6679727959214776,0.999190248905179,-1.635890493454145,1.2938379558735829,0.20736771134970797], [-0.4118317638697319,0.9069236714993869,-1.7766219570089812,1.2254241772422947,0.3818774093786991], [-0.5019285203332016,1.071358441302261,-1.8681705848296024,1.3543938347082431,0.3694102461708769], [-0.43086686887184694,0.9871654497649167,-1.592715663423245,1.1292659913487786,0.24090456644129066], [-0.6554222411786117,0.9775977489456485,-1.7739871948427586,1.3756970876324615,0.5345682469155936], [-0.606670169993634,1.0414671201540915,-1.5816535152099802,1.2849832898961777,0.3443771799618759], [-0.6330347409363863,0.7614040653882075,-1.6420712005053097,1.344034328413472,0.5019032888808933], [-0.5268140556262974,1.1016458967835785,-1.581552701703922,1.1842999784487596,0.3683353834757425], [-0.49900894372274485,1.0645644739185587,-1.5355753177851752,1.2694091849907172,0.33468156327241705], [-0.31358398021960576,0.8302665811856854,-1.5422704772273437,1.1506707176948,0.37999408831157194], [-0.557673046622542,1.0913399597085338,-1.5871539417007638,1.0478537277843192,0.31421916981662246], [-0.4100320104849945,1.0606699393377843,-1.764930551709285,1.3173125309085043,0.32112514544554777], [-0.2578093453260759,0.9604640936063775,-1.6498739501347253,1.251662741314794,0.5352359752706277], [-0.5830353039132012,0.9147683830431232,-1.747368848835465,1.1923888734167842,0.4792523557272257], [-0.3091820573873779,1.0228642342621612,-1.472857793608509,1.165376819972311,0.3758448993415674], [-0.371190450361484,0.855956409330075,-1.82995473329898,1.3390818705810552,0.37113293274981823], [-0.5539807557276546,0.9677998945369973,-1.6350804443189515,1.3419255190805948,0.44899955987573437], [-0.5376287808107136,0.8413279828562651,-1.6762865574706214,1.19949852014792,0.5047576712678563], [-0.6003249423212255,0.9906964773703084,-1.5370089708944512,1.3072479529851824,0.430528835393197], [-0.4175368422724912,0.8187481538877078,-1.7996629160543502,1.1879999792808238,0.473510867372402], [-0.4828257087231799,1.0612823363596418,-1.6425767546040255,1.2270450796716112,0.39748339044894354], [-0.6332333955565639,0.9519761064158869,-1.5479445620477186,1.308465915443944,0.40088093604627134], [-0.5228584674134156,0.8768992895766171,-1.8336272924164618,1.1499724885170817,0.48668761341650246], [-0.36341836902693225,0.922366126760111,-1.4327493627862522,1.255336229334989,0.29637579738564857], [-0.45752242402693893,0.881182369510346,-1.8075367810499625,1.1836209711736738,0.5240597883932867], [-0.45127422400043055,0.7875661584033756,-1.9374943002165643,1.3022768816466794,0.3865261883663946], [-0.3385546242930144,0.8796112795269294,-1.6285471480854194,1.177302672296305,0.3611750358009876], [-0.3866867752880084,0.9445769802243262,-1.63589592788523,1.3115437594462822,0.30025415854590604], [-0.35087047853321573,0.8048561114260069,-1.6540296199948705,1.1069576251024358,0.3739108460295009], [-0.6118021930146711,1.0836229312504084,-1.6965417516081935,1.3814661848764345,0.5544639396063888], [-0.32579850447352154,0.9627503718472757,-1.7118868486570202,1.3378056459968657,0.26239820187294716]) -2016-06-13 20:23:00 INFO JobScheduler:59 - Added jobs for time 3000 ms -2016-06-13 20:23:00 INFO TestInputStream:59 - Computing RDD for time 4000 ms -2016-06-13 20:23:00 INFO TestInputStream:59 - Created RDD 20 with Vector([-0.4302807096972894,0.8599904088505619,-1.74473044235192,1.4559107913903748,0.3868065070182458], [-0.4809066717988696,1.1355083469098752,-1.568604721573235,1.1620770786476713,0.5052691040344791], [-0.37948367495671453,0.9612314328384234,-1.7288918432476479,1.1900277125124938,0.584148529955661], [-0.4830118079148656,0.767521114064722,-1.626523514538083,1.2344387914004133,0.2949434953618521], [-0.45982168221474795,0.9610478654584301,-1.4181098052958898,1.1960502956938037,0.35754207882888805], [-0.5403734624385554,1.0961985914256847,-1.4824794478647174,1.1331480033700656,0.4391107899439444], [-0.4078934785304544,0.9024846949827434,-1.772449379651882,1.1891755675558509,0.26886721197216756], [-0.4850713563393695,0.9854430300369302,-1.6392713106609738,1.1640485301086894,0.4058728213551233], [-0.5687863647866291,0.8969696803234791,-1.694332984322472,1.263598085578097,0.4916946976192132], [-0.4852057062952629,0.9746523770852169,-1.7152410694648534,1.290070394136892,0.15734923086759972], [-0.6042639508376677,0.9587258437320635,-1.796821218335353,1.2415150801911263,0.5036349732580911], [-0.5634160393957598,1.000927422915143,-1.7677443118596388,1.1056602828501068,0.3662560181172587], [-0.42260218853055576,1.038402090735988,-1.623454196568306,1.258759507363921,0.35817719144396376], [-0.6935865876752814,0.9767485891098648,-1.6119749119230544,1.3241871653009805,0.45959708474651645], [-0.34982681977824615,1.0697929536346964,-1.6430495277049175,1.5160135223968076,0.3720168274191172], [-0.4965183720272103,0.8867675212850072,-1.6326283681810188,1.1014099972707194,0.3464777829375711], [-0.4429523196454975,0.9569586297887442,-1.6975710388611154,1.3240706523955248,0.32888337316923105], [-0.6110986163387908,0.8513837967033338,-1.863122720264613,1.1805944382455067,0.36642329779978594], [-0.5770973903898762,0.908469634420286,-1.5969552518736576,1.2639442528889475,0.30334073173665643], [-0.6677676499752639,0.9993894428543435,-1.7484246181717757,1.3024629995033956,0.5254188589994735], [-0.43541597877213595,0.9518339300904892,-1.586032231723223,1.2859374872876397,0.3728538289653893], [-0.3981355023125287,1.0914065768869097,-1.604054951142783,1.5597686669581905,0.3200687403051776], [-0.3526167305034049,0.8582438505591754,-1.795079875034891,1.291753863011317,0.5193327226327783], [-0.5322224604485944,0.9225091285363366,-1.5406535924270095,1.1740734779955952,0.3985082190639408], [-0.459298613922177,0.8499917798311437,-1.7381604837279123,1.3363117012403978,0.39372367283850573], [-0.5029856763543168,0.9563691025906603,-1.6323146191672946,1.1564017424707789,0.2992277161858537], [-0.4702645909815308,0.9857893178047549,-1.6146159696178013,1.2941994476820615,0.29510677438190996], [-0.5010373585673528,1.0381084928077213,-1.6592946006629066,1.2275293781382213,0.6215806870134133], [-0.34251104673790145,1.0920152105960284,-1.4702129754795525,1.3052389503227437,0.5142842632664187], [-0.6014879705809496,1.0573110467882678,-1.6994508095500127,1.2345579924787826,0.27260709575033387], [-0.374393054126274,0.8520958347461538,-1.6227534538993043,1.400050919842349,0.3749155714488258], [-0.5537306132792297,1.0237199970470363,-1.5996513021748293,1.3277912515047763,0.3298858295939718], [-0.25019038644507985,0.8933568126093641,-1.6180540579171452,1.2514548524704747,0.4485167299028402], [-0.4083941282128968,0.9821808764315939,-1.7710803507794117,1.22246902401698,0.39616717157783654], [-0.6617567254520433,1.135205204672421,-1.7404704566578575,1.2634534028343156,0.48631393245586374], [-0.5102944111175142,0.7749069961072899,-1.3746153795885292,1.1761695768038691,0.44519535098184526], [-0.5273802259607313,1.0293844883271042,-1.8327600208787889,1.3032607095759106,0.4810318980867238], [-0.43680250540361804,0.818212719795147,-1.6397291545525832,1.2164204417976607,0.36911234859465697], [-0.44511271834694277,1.0039279649012336,-1.736482991573775,1.1407721179472239,0.32026695790267784], [-0.461571293170463,0.9623234284494812,-1.608651111923442,1.3054108488252942,0.2808744029932673], [-0.44626644352032524,0.9385912062499339,-1.5784088711897624,1.2996894478753678,0.3367841237735643], [-0.3458002275610911,1.0002670117410046,-1.556743986487141,1.220789398824709,0.3047413342602086], [-0.7006173067436341,1.0226530560056757,-1.7453818996615391,1.2468257997531524,0.43730757861422237], [-0.4921877365188532,0.9365903329931373,-1.6954478906187997,1.252660426269991,0.411305000260049], [-0.5866727516355298,0.8540617355784135,-1.7495419546675461,1.1992666379768737,0.5624798329629086], [-0.4552502733367031,0.9291860448712651,-1.5908193525700438,1.166718738832733,0.3281256456654895], [-0.3426613548769225,0.7538960431270546,-1.585524926597853,1.478740164978157,0.251131880834575], [-0.4680826818575833,1.1112444944704614,-1.6587653968723466,1.2870287092936261,0.4050920708933554], [-0.5624024717932173,0.880862871462712,-1.6303493150007178,1.1721463768303386,0.5696913393311782], [-0.48266079841446297,0.8916088275950524,-1.5393517554424276,1.1897790454674049,0.4411535715241579]) -2016-06-13 20:23:00 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:23:00 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:00 INFO JobScheduler:59 - Added jobs for time 4000 ms -2016-06-13 20:23:00 INFO TestInputStream:59 - Computing RDD for time 5000 ms -2016-06-13 20:23:00 INFO TestInputStream:59 - Created RDD 26 with Vector([-0.3816421005537525,1.0617035428188992,-1.7399461177422264,1.332980235806715,0.2916494841747297], [-0.6583584440016957,0.9555891409969888,-1.5454326525102315,1.1741091058996926,0.305220337835565], [-0.2989177604038451,0.8701688291879833,-1.8804582484676076,1.2986162711936382,0.35958131004690164], [-0.4939772883087695,0.9969220817446861,-1.5980382534286326,1.2969252872376784,0.37976918116852443], [-0.35668565131454827,0.8078147339434413,-1.657193160993533,1.359095599717262,0.31192105888186633], [-0.5450865703006346,0.8750391278919141,-1.7453306059026725,1.3721329076982913,0.42237633518141116], [-0.4917580165015238,0.9512146417235625,-1.9326523267650781,1.4266827266379667,0.3225713722197544], [-0.4048718474607664,1.0240256182146898,-1.6774401089659943,1.2669846165410654,0.33732641244233585], [-0.47579982131353826,0.7947369494624074,-1.7655419119652316,1.082577012553155,0.35401987740372537], [-0.35538929020910864,0.965963799724578,-1.7529404087985827,1.2055043918858341,0.3342486683177578], [-0.7014541659835856,1.034642549879647,-1.5544540159626516,1.3033886639865033,0.4054677613509847], [-0.43610110990224626,0.967077951705683,-1.5262368892289788,1.2639835242537958,0.3181179758885862], [-0.3425524324739105,0.9894095680520077,-1.7777745234498326,1.3091660388427224,0.331841648562285], [-0.4878205400579338,0.8560103963724833,-1.7352326850283464,1.3664274889095067,0.3984111315135807], [-0.7121859062829006,0.9130040407010276,-1.7874802517046422,1.1874648639293004,0.45017886399842666], [-0.696643078139105,1.1467663562988957,-1.7296410056290408,1.296144064682525,0.5048605187772839], [-0.4011230886898419,0.7361859073107719,-1.5317733987729902,1.2701183330425934,0.46062351610078545], [-0.45715694326129586,0.9190057703920946,-1.6904193500197242,1.3271044644288563,0.4842403797714084], [-0.5095249256026796,1.0659631362797184,-1.4402626928522713,1.3379987286264212,0.2673039786797317], [-0.4167094825032959,1.1091402127599272,-1.705921687753808,1.1183411370664753,0.48305727426817385], [-0.2924464534867928,0.8213969946455459,-1.662853823551024,1.3355427142685694,0.3770025002372693], [-0.4444725119583886,1.1046045679759258,-1.456164663000235,1.227847061522124,0.5255642346814823], [-0.4048329834921757,0.9788094275209757,-1.539852245269681,1.180160780953293,0.6787253334538687], [-0.38022990049474503,1.0144858953551337,-1.8072904612189131,1.3419776986257266,0.4041658441938728], [-0.3763555278351052,0.8998086161617835,-1.705061140148122,1.3495005676173746,0.5191631674639813], [-0.5540229100290982,1.0669438671830174,-1.636684794385797,1.3459012222977915,0.4181134505939225], [-0.45042550789662783,1.113240852339999,-1.486260571622101,1.2719922983443672,0.4102349177516307], [-0.5546179681493353,0.9938406825700987,-1.7084301934096207,1.3880541591389144,0.29690506797892435], [-0.5006405047598609,1.090292575906863,-1.7104271793907637,1.3062089077771892,0.47556018075456363], [-0.36520205819569046,0.9201397468605166,-1.7602021590969437,1.3876082296931265,0.2308718010458195], [-0.4938168714972593,1.0964903470900649,-1.7260349751744855,1.2048249569339422,0.40946925790527117], [-0.5092411838541296,0.9882559808204627,-1.7212908932580275,1.1127850487389395,0.3940121165600333], [-0.5882412062939182,1.1156935762589453,-1.8131619026625396,1.4146422480126442,0.31361816553861704], [-0.46990162437316874,0.9908156946435108,-1.748827835513362,1.3957588501434537,0.5512480921903505], [-0.5405868391855275,0.929247220281523,-1.5894550246414005,1.271255300937136,0.46529614772048844], [-0.46376208782353645,0.8174067828364345,-1.711422288030576,1.150388298327176,0.36268778857829526], [-0.4617415170232849,1.1203248336099854,-1.9346820049945674,1.1466858815338403,0.318614186382022], [-0.5200553635156823,1.1206383922863123,-1.721393160577411,1.2564342218266722,0.39290287510750027], [-0.45474634000888675,0.9538310968737261,-1.5572305892194418,1.2293659662686987,0.3951581444726625], [-0.31024997329419923,0.9626097763501761,-1.5774105560033171,1.3323705378196256,0.28213579823419327], [-0.6425546702182562,1.0413861558054223,-1.6696029184789263,1.3297020370998587,0.4667426127538354], [-0.5088483797878551,1.2170009632967245,-1.7143214767417212,1.2915402658283741,0.27813229259982863], [-0.5138048643277845,1.0537687232096613,-1.5306968187748822,1.2020601237200264,0.27213168784492703], [-0.5375432544160766,1.0418784698921373,-1.501800504998784,1.217708599285121,0.6443267111319482], [-0.5784571285851817,0.9518620188834632,-1.656094482882889,1.2785669143560554,0.34103467230048085], [-0.39057405488844044,0.995936983645342,-1.6323671359433234,1.271155031008547,0.3558239098317178], [-0.3682240577265206,0.8711375369867465,-1.6625627384453794,1.1969170451325948,0.47233733398400285], [-0.446008720311628,0.8976699326654971,-1.5289286019943875,1.2244855320582673,0.2984778353929738], [-0.5102538395816835,0.9986885141123109,-1.6414035100650266,1.2420020420446427,0.36032207135032196], [-0.6350165259371148,1.0900186871655364,-1.7907186442371252,1.4304360117451347,0.35387005412887984]) -2016-06-13 20:23:00 INFO DAGScheduler:59 - Registering RDD 7 (apply at Transformer.scala:22) -2016-06-13 20:23:00 INFO DAGScheduler:59 - Got job 0 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:23:00 INFO DAGScheduler:59 - Final stage: ResultStage 1(apply at Transformer.scala:22) -2016-06-13 20:23:00 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 20:23:00 INFO JobScheduler:59 - Added jobs for time 5000 ms -2016-06-13 20:23:00 INFO TestInputStream:59 - Computing RDD for time 6000 ms -2016-06-13 20:23:00 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 20:23:00 INFO TestInputStream:59 - Created RDD 32 with Vector([-0.5768797964115748,1.028505413551091,-1.7521414076692798,1.410931775189879,0.30527264662469095], [-0.40591177777424625,0.8661878776717759,-1.7310200236861213,1.3594055750354543,0.3299548132445285], [-0.6383561683498007,0.9259033203935216,-1.6881320011035645,1.3172994752602565,0.3214908371039436], [-0.624062828319168,1.1219523756569185,-1.5603263506923595,1.228964619537528,0.35487057209905903], [-0.4734064808910865,0.9649955190976767,-1.6068693471541338,1.3196307767213002,0.37116726129033706], [-0.6101841770121739,1.059654359501757,-1.8099667264246553,1.3759685981009824,0.5438261473052933], [-0.18850356552447423,1.1440264277756391,-1.724865192544724,1.1359044353122627,0.31757393897273056], [-0.41254200355821263,0.8607204799101216,-1.616620448535661,1.3427363220425623,0.4109728277211949], [-0.393734447592108,1.057690953958827,-1.6695747980566427,1.327034390345423,0.13819106599587025], [-0.32090427274943434,1.0651687359751123,-1.6980368198411557,1.4017902860954718,0.36022562482340503], [-0.6053061980873911,1.0041632375635317,-1.7820663797212062,1.2269162643461793,0.4840114520476853], [-0.27268808986968307,1.013170380627259,-1.6005981535949472,1.4382064950763866,0.3892224005152082], [-0.535422550377724,0.8375173015602825,-1.661788348017347,1.3227481832299182,0.311533159171815], [-0.4009368458319039,0.9897832871295871,-1.6696688300152116,1.4424037410980737,0.32734398317968916], [-0.6825087101818776,0.9056073621397804,-1.659854864696752,1.2759644672519115,0.43884955280528926], [-0.43179250129551383,0.7465728726494555,-1.5636729803345486,1.3457117728614754,0.5721971492726551], [-0.526304418227463,1.0501002903902692,-1.5968858098432406,1.2982785754583357,0.37494021304786934], [-0.4320225453412649,1.1715904807145918,-1.5178148547272063,1.348200261386646,0.40779395798035983], [-0.42221995667102147,0.87623418086733,-1.73837906100808,1.4183919259975142,0.33228674704284467], [-0.5629520941212462,0.944212875923361,-1.5546071736198348,1.4519582067498829,0.262200899044119], [-0.30845271114048184,0.9681639591372486,-1.688842111249905,1.3763468548667668,0.3173125681909059], [-0.5198567190391367,0.9335053916544206,-1.509472274649017,1.454415814887731,0.4589687217941236], [-0.5711775989363378,1.0215061238027188,-1.6224000734110398,1.350665861834858,0.36668462127595286], [-0.39949682119974983,1.0408092246134175,-1.6587225284400737,1.301865863792736,0.4563742176523199], [-0.4902566261487769,1.0992586451801099,-1.4473476422759635,1.2309123590548212,0.2544829822903507], [-0.5293628624912633,0.8432327849564988,-1.7669603672683885,1.2911210280012069,0.41983427336223667], [-0.5738614502843032,0.9784937768658387,-1.7194271904022407,1.3116656980237082,0.46218317857671776], [-0.5902501988138692,0.9423215775018176,-1.684040151093259,1.4235274237923208,0.2039286490216381], [-0.4777727934676478,1.027586212069216,-1.7495753442806758,1.2757769414973703,0.3810309831295745], [-0.6566621783684151,0.9641681686268665,-1.5286420868141977,1.508411206136107,0.2501550388463756], [-0.46296281092785885,0.9727194845614605,-1.6171422039154086,1.3434828829348795,0.07809723939214058], [-0.5738547760098169,1.1077074999997352,-1.739110700752265,1.3494171899976157,0.3222123114107347], [-0.4002887849620737,0.9619411680270862,-1.663509769541054,1.2908149710547212,0.4244227376717664], [-0.4665533215112465,0.8596395857982323,-1.7467795346659782,1.251224446006115,0.42794754034954735], [-0.6374258626266393,1.0514903728245393,-1.651303971310752,1.236884794719695,0.5249538337308562], [-0.48340361610998744,0.8198136168919055,-1.690727257833199,1.398396991016531,0.31841739933411595], [-0.5410633313507479,0.8973900953721683,-1.4724867408929736,1.388151513303225,0.47893645155994446], [-0.3039096405457719,0.9155005236200422,-1.7173365893442798,1.256097860517092,0.24819750391337136], [-0.7090962179115853,0.8944749271173439,-1.8839838110448957,1.1446445350131944,0.22840975233443309], [-0.5100953498526793,0.8107010460411471,-1.7303477866873604,1.1647356509362308,0.4418371093135496], [-0.2962087734742913,0.7242125014742148,-1.5984563979400566,1.0588259317312654,0.28436170748337264], [-0.5505122810716748,0.7806970076803257,-1.72088933624607,1.4230022424760869,0.32673184311928194], [-0.45127942352855155,0.9959060503010477,-1.5524186364009327,1.2366972856510523,0.3578155222046753], [-0.5102066277582917,1.0949182323987419,-1.7142222646495175,1.1664611293679565,0.3602965879997574], [-0.20421136622466618,0.9866371147144095,-1.7538283133815977,1.0465139384173783,0.41274915410898083], [-0.3678643518086112,1.1128928060624481,-1.7416958328862653,1.1712943420281356,0.522586652590277], [-0.4272255324098842,0.9505411417476373,-1.5398631578697235,1.1670050581250606,0.18826195051488878], [-0.6294443147944699,1.0182211296265207,-1.6114292740208946,1.5494854908967275,0.2354819914065145], [-0.46052379116984477,0.8965751438720174,-1.6887957613207942,1.2678155037585463,0.3277249457164221], [-0.4661555388594827,1.0176554636610373,-1.660202673672577,1.1543960943838145,0.41490582984943747]) -2016-06-13 20:23:00 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[7] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:23:00 INFO JobScheduler:59 - Added jobs for time 6000 ms -2016-06-13 20:23:00 INFO TestInputStream:59 - Computing RDD for time 7000 ms -2016-06-13 20:23:00 INFO TestInputStream:59 - Created RDD 38 with Vector([-0.5184134212251081,0.9455318561030377,-1.7249072888539423,1.2476376390078272,0.539371371249466], [-0.4839353019153916,0.992343730650464,-1.4037670310964987,1.3650053852483648,0.287928402259341], [-0.45217444521287575,1.0425364789095595,-1.7771813152553952,1.2241390773159404,0.3053067128704871], [-0.34943560877945984,1.0407179160044182,-1.5177836780575653,0.9824846085205396,0.2603543256712868], [-0.45829448575134074,0.9725249117979422,-1.690056167350807,1.035512584568622,0.3026754916713263], [-0.42458145588018753,0.7854084174090814,-1.7649589009106188,1.1975914903392633,0.6551885852829413], [-0.5104497237497339,1.0402201914534017,-1.6859518565969145,1.1391556996625576,0.18881595045601088], [-0.5588856532488075,0.8709355475677711,-1.4735465340859104,1.207362168238276,0.5886898848511445], [-0.5286914346418723,0.9318873502365327,-1.654993790470044,1.274056873049218,0.29412525529417294], [-0.5420811906362676,0.9995645356070102,-1.6040227637757893,1.3412654820039975,0.3950459929854693], [-0.5105371229433685,0.8870982484234274,-1.6366208840493932,1.344244897498087,0.2842199976348651], [-0.5758159328886613,0.9361950961962593,-1.5151431024376867,1.2963264217469896,0.4195802203163277], [-0.46650906956232724,1.0883479790237522,-1.6209503836486092,1.2506119015001946,0.5239087792157935], [-0.49800785133272085,0.9121648111767798,-1.5664537098618618,1.4303891258276538,0.23796160092517643], [-0.6200401622139905,0.8819930763699753,-1.7712553265237971,1.144151394858828,0.37228012783293096], [-0.43683215654432145,0.8386127968983885,-1.6402554451670102,1.129209493050289,0.3612473529185551], [-0.4079221564079838,0.9656864104202829,-1.7359335721136713,1.4421338682074127,0.3616284206434529], [-0.43442591279896603,0.955449363348641,-1.5334677832140913,1.1666726886559795,0.5058368495019173], [-0.6457828019918121,0.931131898174338,-1.8532037937152797,1.0375316080434018,0.3133563527606291], [-0.38369598973754027,1.040856056086344,-1.7594025439463896,1.4917751006832976,0.45495744983872133], [-0.30166559339288085,1.0342180102112146,-1.5650152370405155,1.2673629485315197,0.4158807517084573], [-0.28580825731967757,0.8865395334100423,-1.4737235251638032,1.245008984525024,0.36435255784505255], [-0.34071421524008416,0.9427461052254856,-1.6571555985762556,1.116205755573728,0.3440594901972376], [-0.5707704618700846,0.9955277694976673,-1.815098893846346,1.2976867834380266,0.3708933869585827], [-0.6095164341318,0.9322989548597498,-1.6576644540187038,1.3001089252401639,0.32781904617649943], [-0.5884784232045771,0.9963121018549773,-1.4726908596796255,1.1232169328419297,0.466258074909203], [-0.6351651621700529,1.0173421935079714,-1.7981429560407272,1.0336059974520695,0.28162584897435694], [-0.38203401021631267,0.9790771913538491,-1.7244683271662424,1.288084387752797,0.4312827591331814], [-0.37183909143628785,1.1926117173870312,-1.5166716166223042,1.1696263300990144,0.4515586032102953], [-0.47462520704983,1.0498070539706301,-1.6863851371590932,1.2965114835042095,0.4645837570114517], [-0.35874130630231327,1.0042819076442355,-1.6407835501913508,1.214144240421076,0.35534792382876684], [-0.41389636789198914,0.9508170625312302,-1.6498089027313338,1.4272197180381103,0.374553819362672], [-0.32132785261092967,0.8535769504185499,-1.672835438070241,0.9421132176631009,0.24740658936307364], [-0.48253792736985773,1.0429236941137325,-1.4939460450129358,1.2832409419178106,0.40323476275374437], [-0.391512958705594,0.8859909853510953,-1.607045573914962,1.1770226037327562,0.5198479967492], [-0.4833163026675927,0.9200058086737815,-1.8188555219632765,1.2111735788924018,0.39618714901430313], [-0.2913135273832381,0.9301485161935364,-1.6975841978124846,1.4065399033140502,0.28453881400648173], [-0.3073756680829076,0.8325242716417236,-1.6753209327630654,1.4075134801045086,0.30351866480787215], [-0.2775051790445552,0.9452680429408632,-1.6224110577930677,1.1778711923142833,0.5531745712889398], [-0.3047686297259775,0.9468967188067203,-1.6929112749188848,1.4636396431851675,0.3586449109599954], [-0.5956899619218391,0.813236818549011,-1.757649523640403,1.2604654777105053,0.3185144577112101], [-0.32794499152459505,0.8140339815697406,-1.8666906537843855,1.2572887839007447,0.23549894606903882], [-0.40686690942678344,0.8619998816707273,-1.6319579560604098,1.3664940728963186,0.422156696202466], [-0.4138601150243667,1.021828823586237,-1.54841612428864,1.2398622089963618,0.5070042357490803], [-0.331869975870002,0.8782404384527787,-1.6635875859844171,1.2435143760020457,0.32101906837863314], [-0.49700214293681766,0.9856184010338479,-1.8232422669575907,1.2421051455581202,0.47592591320323163], [-0.535909591266969,0.9807383109826389,-1.6974325972167972,1.334548321075211,0.23946350824403165], [-0.6444021824207316,1.1849780210499694,-1.5121795557147575,1.3604386920675193,0.31690181205044465], [-0.49746220541003966,1.0643703042656938,-1.8499798436118189,1.1357972819777942,0.5780379400362996], [-0.4383285608503117,1.0163805234573544,-1.9030246206122645,1.438368352860807,0.3293850463477472]) -2016-06-13 20:23:00 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:00 INFO JobScheduler:59 - Added jobs for time 7000 ms -2016-06-13 20:23:00 INFO TestInputStream:59 - Computing RDD for time 8000 ms -2016-06-13 20:23:00 INFO TestInputStream:59 - Created RDD 44 with Vector([-0.374705866875181,0.9840691262184086,-1.5891551104970965,1.4814595125988181,0.47065837960007695], [-0.288810885883263,0.8437842751113669,-1.5395954061634287,1.1874811761270259,0.021249409500126437], [-0.4606335450965845,1.1012472510905122,-1.8287426699010534,1.24172318938614,0.4710333368456244], [-0.42944940507321583,1.0104988720375858,-1.5751317017658801,1.2831547194950401,0.3512701703197814], [-0.5152127056418667,1.1709583832128079,-1.6554391510043867,1.1397001494774806,0.23397277605773995], [-0.4399479870409779,1.024335104965922,-1.7045108897162344,1.196833551305392,0.4117115230011916], [-0.5430543625505585,1.1370095574712402,-1.7176429060334013,1.1901279759925694,0.46070443041212095], [-0.3968714897698767,1.0926860330748331,-1.6334286394688688,1.228234808051336,0.4665607383628074], [-0.46215342653947683,1.0343329141616782,-1.6499943761334228,1.337530812749641,0.23114556003826187], [-0.5000235665427208,0.9151728338053156,-1.8055132514640995,1.3037686620331144,0.2902025184248946], [-0.309233049825866,0.9728959274674948,-1.6392691162173578,1.101247705752869,0.3496423465401415], [-0.5981691090699208,0.8739296001671861,-1.5687282698937923,1.2550952881121755,0.46308264317451964], [-0.425561195913103,1.0676958602044149,-1.7135165364767921,1.3524692002054561,0.23228520033456318], [-0.7043177723021486,1.0283311625585259,-1.7478644671187442,1.2858120358708933,0.3418710936510813], [-0.41727892194293686,0.8625867727705405,-1.7914100057674134,1.1786792695315231,0.4917512427338196], [-0.40174243931692544,0.7828098802704966,-1.5073762714652954,1.281118669072956,0.44887930601066905], [-0.35310979651234775,0.9745364825956087,-1.6392317059509176,1.4513644518189859,0.3110119088453469], [-0.6853829628094648,1.1253397147829511,-1.4920081564639467,1.231135544811062,0.3139314829419366], [-0.44152016739085975,1.1304790288137783,-1.7325115907360644,1.3251089521914843,0.479567851123668], [-0.5526275004039871,0.9674440446022308,-1.7971453051937036,1.1664342514118335,0.31260632993291754], [-0.46562594489923503,0.96412818249357,-1.7756506925990838,1.1820367323525278,0.37368762092821634], [-0.42414895043860573,0.9369437276073684,-1.6228073393039317,1.2110129935336054,0.33600782254298567], [-0.4507513844294476,1.008422902581697,-1.6947838717172434,1.1258385535052762,0.23649573041213182], [-0.5011767543824581,1.0367482963168666,-1.8619227324329928,1.187686906059264,0.41060169959227016], [-0.29061733919433663,0.9125980333805904,-1.773384852821433,1.2318813347419482,0.5360023941871213], [-0.47558633395179245,1.0112018987992073,-1.6443927870523196,1.1868477340320063,0.378412637239359], [-0.47217857076866115,0.9027887018339104,-1.7359968088612976,1.372505255254142,0.4024631611010435], [-0.5635782393384723,1.0418520168542464,-1.6371483392749435,1.3046794831870752,0.36435743882876215], [-0.3566584096699375,0.9045378099174617,-1.6027843810019622,1.308376882406936,0.450420525347206], [-0.5161548558091542,0.9362466126941537,-1.543853038663444,1.351176902637456,0.37019671664199505], [-0.37734135089435306,1.1354760893808167,-1.5928707071728756,1.4230797310043477,0.5395830646701685], [-0.4935941975274797,0.9431897551602951,-1.7363999282764386,1.2592777434544042,0.34477211805728875], [-0.5399690948357754,1.0773952946269298,-1.623286131549866,1.3571017009557331,0.3820149575212495], [-0.47470272833792704,1.0279092716939924,-1.5583033476707477,1.2928147173688327,0.12854158780432537], [-0.3932748299826361,0.9168968642381998,-1.901294092833649,1.259188794957511,0.4011688713714673], [-0.3557493574970649,1.0003577059585267,-1.5111904842067994,1.3123818859934535,0.3019665980708428], [-0.38849198387448786,1.0634537152100978,-1.8742015500852314,1.207616787308903,0.527158124827907], [-0.32516837051884395,1.0806577326663074,-1.611781542302308,1.182593379846847,0.5196789294606137], [-0.5212725898858087,0.998490506190397,-1.6184763965153202,1.3731447165706872,0.29855378648402087], [-0.40257752034798155,0.8314340557777532,-1.6850156321125218,1.3427297403898506,0.5156164947536159], [-0.4889473969173199,0.8999745071099362,-1.4468367122261747,1.3090812041423647,0.5108182965913203], [-0.48274282778056277,1.1537075465803446,-1.5811276311378704,1.287632315416058,0.50882779799236], [-0.23362208323850905,0.9548750423156142,-1.572161725609678,1.6027003014260315,0.3939518404103326], [-0.4292988792888861,0.9366381149644413,-1.8250064321036654,1.2358072562830058,0.18892247590763034], [-0.484748914270206,0.7653236279449509,-1.7486834183984747,1.2949268753185315,0.39683334762547673], [-0.4399736279490563,0.9634644976145054,-1.7480413166448512,1.1775955441636703,0.36332416278682794], [-0.48373350642316953,0.9259530051332495,-1.7977337219360736,1.2876089889322726,0.4497462132788797], [-0.38070426405120866,0.8838236645507052,-1.6753928124899833,1.2449514905082557,0.3276558558550576], [-0.5717833110401351,1.0273299096906752,-1.6190879043151574,1.1860455839092494,0.32636997750947594], [-0.5231330958628938,0.924256801489448,-1.5484246875687981,1.3381528852011608,0.3757180006852602]) -2016-06-13 20:23:00 INFO JobScheduler:59 - Added jobs for time 8000 ms -2016-06-13 20:23:00 INFO TestInputStream:59 - Computing RDD for time 9000 ms -2016-06-13 20:23:00 INFO TestInputStream:59 - Created RDD 50 with Vector([-0.4026578572929527,1.0001062834205203,-1.5617908885948877,1.462733521830009,0.2061287167736916], [-0.3965287353033582,1.0683726645043978,-1.4873730061169563,1.1460792990307642,0.27746130384846085], [-0.4938197206621628,0.7709438014433136,-1.6076436749339549,1.0982028796258234,0.24486596655062928], [-0.4777650011847253,0.9489155740228515,-1.476414143798642,1.3062514135704215,0.6114439757268249], [-0.4285977940802228,0.9382568588433707,-1.48976698594476,1.1698457774900977,0.297712575086562], [-0.5993896749484521,0.8629058534578911,-1.6510698754517374,1.2090117714723978,0.5458042851487852], [-0.5321474831529757,1.0849054231681536,-1.667568850178397,1.4161664775537062,0.4493689535824759], [-0.6723705689897437,0.7517542425048965,-1.663645669684176,1.2240252996527488,0.346848907683753], [-0.40115735788941104,0.8163474979541504,-1.7694648704001712,1.1073947331922183,0.3856322637520626], [-0.4787497898360035,0.855418622108937,-1.7422359123781868,1.2371448524727167,0.5156131237949001], [-0.5321048883779993,0.7970778523677844,-1.7616773188005117,1.3821414318824692,0.48832517040159346], [-0.5285360880098152,1.0597395124384508,-1.6417892576104653,1.3464020747697325,0.31849105791627297], [-0.5324435853239631,0.9841563147023031,-1.5943852209495275,1.2707723748060116,0.4139910360672806], [-0.37531742848811983,0.7874289125205598,-1.6479869931712725,1.3181636612914271,0.3718789043065717], [-0.4560433134330382,1.0759812484577316,-1.697865869705092,1.2725177127861855,0.06883328163775021], [-0.45513942313770217,0.9498671339799809,-1.6933930804659583,1.3025139463978586,0.5350768600248874], [-0.526256522617052,0.8727003678813281,-1.4356120595661468,1.29993914263913,0.43387328603333664], [-0.3923483046111808,0.994060772829843,-1.6451266300509706,1.2983503204985913,0.3703765349262901], [-0.4374913726020475,0.8955789511391181,-1.4728897306445088,1.1357862983779858,0.30881407016500095], [-0.41951697820602707,1.0007695283696842,-1.708286281016387,1.4190975260604537,0.28731659836668877], [-0.6750333217393494,1.0688779823591679,-1.6846746627252691,1.2962992487600882,0.5030936797752793], [-0.5956866906574221,0.8121299274245898,-1.692929629203368,1.2338548732523056,0.38371772482125543], [-0.4303591738987757,0.9459620785706632,-1.884551120494729,1.4937212054647857,0.415792749586519], [-0.45504157760203423,0.9027897906482127,-1.799761299703666,1.3040399621270629,0.5736987408004388], [-0.5593669081347996,1.0931782446321396,-1.627955485174887,1.1391759184235553,0.39801309200504786], [-0.3825955534231694,0.9655263256725013,-1.7068123920171723,1.1861538643513039,0.2764471843224494], [-0.3587313623839552,0.9371265382727668,-1.487908236071683,1.3184799953634567,0.4370396706398445], [-0.475038660091078,0.797460346861311,-1.6835937249660264,1.3120302511529962,0.17031892070731325], [-0.4663190378327658,0.991352295299833,-1.6444316855302834,1.51526349576866,0.3929422369724258], [-0.5040823137977553,1.0575372259159488,-1.594322659684267,1.2971698194040933,0.5092825217956699], [-0.44790417664079346,0.8518362899333143,-1.648211585312696,1.1839806869130056,0.40832085080824754], [-0.537755895075537,0.9806637869508009,-1.7656906916731008,1.2433660326767015,0.42887767735060545], [-0.34293660736335746,0.7935119912222357,-1.6359022010243611,1.3593857228798436,0.3967833959145234], [-0.6208695835786292,0.9640100782092327,-1.874229161549032,1.343981009171891,0.20123531286937832], [-0.6723717817990262,0.9002960054466735,-1.6910864401441592,1.1818255352705958,0.5433986930426408], [-0.4588708699931968,1.2092994554375371,-1.5324768096769932,1.4083576723657105,0.3708605878943178], [-0.560638238383703,0.9092570275383438,-1.473047966472027,1.3497511209299733,0.3467764467476322], [-0.4409803705877342,0.9370238683801353,-1.572432943897284,1.41498798111251,0.2748026192628795], [-0.38741643074715315,1.071718105146961,-1.624146896559199,1.1401522638836572,0.3872910322155703], [-0.5140862086906481,0.8211106116047733,-1.6027037015807561,1.1677747630844688,0.2659604974429464], [-0.6693385873907269,0.7554393139905898,-1.7108087289854104,1.09991932687034,0.41821242261960917], [-0.3282503845126481,0.9257033715775067,-1.5235939065144024,1.1889128563024831,0.3368716615939142], [-0.5366461042406446,0.9240666515119298,-1.8135909127248846,1.3575511869067935,0.37505000938261085], [-0.39294848413804057,1.0124585311856609,-1.575086388417685,1.2467079941490156,0.40416560148386643], [-0.48738816985096806,0.8787565657383682,-1.810281961839409,1.3779890105636297,0.46019825317610896], [-0.36592476270170865,0.9132457437551841,-1.7089939281056865,1.1718159447241387,0.37244792446782193], [-0.40751855185193425,0.9541054981051484,-1.4879650472104635,1.1817057016562598,0.44919665548504983], [-0.29317450049605076,0.964443479319753,-1.656028808081747,1.2586398699632722,0.2526740377633023], [-0.4615968319838038,1.0243528061825597,-1.5054257187951623,1.351376529833804,0.42971544189574107], [-0.47988177538312266,1.0112860256804128,-1.5404476412782737,1.233668528335028,0.2028012939752821]) -2016-06-13 20:23:00 INFO JobScheduler:59 - Added jobs for time 9000 ms -2016-06-13 20:23:00 INFO TestInputStream:59 - Computing RDD for time 10000 ms -2016-06-13 20:23:00 INFO TestInputStream:59 - Created RDD 56 with Vector([-0.4889010904178065,1.0197902904388123,-1.6434083794608938,1.2273894895374848,0.29880056758004836], [-0.3690054860756016,0.9083136794567547,-1.5975355891184535,1.1937534562371308,0.3514272190683493], [-0.5737783702048262,1.2273536117223582,-1.4617006566984774,1.342244921629087,0.3377586288639175], [-0.48387316003666864,0.9673070026559238,-1.716725721249248,1.2214389230026372,0.37779842180031725], [-0.3777277713061831,0.866118273628028,-1.8487356572074127,1.1851350383431707,0.5215534466266145], [-0.6076926717533072,0.922880686625322,-1.7313060678714964,1.3632488420681186,0.3176699282425657], [-0.3802374789807806,0.8439055679755229,-1.6219613687183736,1.1809867291088374,0.4174630121979538], [-0.567466556581487,0.9292300369161763,-1.5542588287823018,1.3568593851021735,0.38458895772840795], [-0.6764387691659441,0.9816754055716941,-1.6652926928722982,1.3324217076474294,0.2844320904428872], [-0.584138124369403,0.9651833514289697,-1.7100579024456408,1.3025648636881102,0.3575320807936941], [-0.45548159931880944,1.0181220363355081,-1.6506653550843506,1.1304474598171783,0.4285968106743953], [-0.5370002084128622,0.927571736048293,-1.7228464440630433,1.2925613214036213,0.3169164091166503], [-0.47996613069291666,0.884366449915093,-1.7212355955272085,1.152597880658151,0.43962468554199663], [-0.35493034949860885,0.8984344817504613,-1.65218132155661,1.3068108975806394,0.354531357717713], [-0.43226118157508875,0.90725353631203,-1.7628881371748841,1.314870897760879,0.40214500076142384], [-0.4387259831220203,0.9621751931555681,-1.7268764217836796,1.1093705777029121,0.17867972251315242], [-0.44765902746822306,1.1186404105474657,-1.750942737318709,1.322673860044468,0.41609066577279374], [-0.40351581789921565,1.046340578817358,-1.7120920286690795,1.244893797129159,0.3833023298379642], [-0.3093624994668177,0.8356650811564297,-1.8642891852543533,1.217524168702485,0.2103617986137151], [-0.2990644638630094,0.9070655034040485,-1.6094469406493193,1.1592980925831848,0.43244854200109223], [-0.5596021456993305,0.9019708087164819,-1.5105356278212536,1.2598551179472526,0.3428536586628008], [-0.583393764986965,0.9748568871770861,-1.5302538126845806,1.301546791122464,0.21927558650218368], [-0.49772324825286274,0.9716299176892979,-1.6169165069063722,1.3531471880552017,0.25858373282508224], [-0.43106771684572104,1.141003137647425,-1.7578104686857339,1.1835903210552496,0.5254045668739125], [-0.5997306192424074,1.0040726704610177,-1.8227253165238155,1.2317004811683654,0.46876771655575034], [-0.38252412898165267,0.6654527000305275,-1.6468024292953505,1.246225339488523,0.3090133585357682], [-0.44821229943324725,1.0038256342470713,-1.6968551076177416,1.4529627079929077,0.3454589917432014], [-0.43535992691090786,1.0110816584854143,-1.7453606983094134,1.248210478232135,0.2891585839680707], [-0.3652348315036368,0.8958964249587966,-1.9606863598829045,1.3389257198065394,0.2618900608154475], [-0.49376528406043635,1.1421599774098976,-1.7157624036331927,1.391772429187865,0.2876879628208894], [-0.4515063147230089,0.9428403433366573,-1.76394465423704,1.203639940182186,0.5058415243772909], [-0.566229150629525,1.0128616873896932,-1.6880365432069786,1.2877554523210368,0.1629872361106314], [-0.47658344463796903,1.0862736976313303,-1.7934284966158631,1.2272771650263852,0.41680211738171946], [-0.3618808351944353,0.9417137306142976,-1.8600076455984111,1.2206352320840592,0.3461749973804146], [-0.6214757024643818,0.9952948941676851,-1.62097329834717,1.3958184351557885,0.46499512939060494], [-0.35996348236796494,0.9954644862470238,-1.7933270864489543,1.4005533772243737,0.5639356774206108], [-0.6006970016720174,0.9686450943470077,-1.7310975948083858,1.2398543489295077,0.4483279499267394], [-0.4592797597414643,0.9631529442377594,-1.6797024055855931,1.4289800788354718,0.28667160760155463], [-0.32700881076504584,0.8147754680205297,-1.763026614596352,1.3194735072019403,0.36816545181467747], [-0.4305345983795718,1.131285559002078,-1.7583813300937976,1.363545141597296,0.2601165464857354], [-0.448497878644767,1.0701920363594992,-1.6308801358271674,1.2674713459930484,0.49959128317905865], [-0.3686293381026533,1.025599917053419,-1.5433629968114364,1.2801178859900646,0.4174115399252367], [-0.574113693976958,0.9757976338077295,-1.6388778822719372,1.2759164798546365,0.5542142157229283], [-0.4357071400660946,0.8338147376622728,-1.8285888680646112,1.271233025944097,0.270178918164786], [-0.47158188300536635,1.0122183541160776,-1.6010520252577112,1.5447804629514494,0.3533722934153737], [-0.5307926544952695,0.9107879044274507,-1.4490342775396574,1.4031526476841614,0.3809780658985748], [-0.4953668915084064,0.9298227962550192,-1.5651812648587027,1.11568589286655,0.4774983034589586], [-0.7383552095915331,0.7626241115292196,-1.7008182872047266,1.2502877442685654,0.43135198632241045], [-0.539114438923603,1.0489102632469824,-1.6736818095705865,1.2768121659352345,0.2213641562753643], [-0.46002266280222004,1.0294295650038185,-1.6033579773810744,1.407522155806443,0.5223722922756932]) -2016-06-13 20:23:00 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:00 INFO JobScheduler:59 - Added jobs for time 10000 ms -2016-06-13 20:23:00 INFO JobGenerator:59 - Checkpointing graph for time 1000 ms -2016-06-13 20:23:00 INFO DStreamGraph:59 - Updating checkpoint data for time 1000 ms -2016-06-13 20:23:00 INFO DStreamGraph:59 - Updated checkpoint data for time 1000 ms -2016-06-13 20:23:00 INFO JobGenerator:59 - Checkpointing graph for time 2000 ms -2016-06-13 20:23:00 INFO DStreamGraph:59 - Updating checkpoint data for time 2000 ms -2016-06-13 20:23:00 INFO DStreamGraph:59 - Updated checkpoint data for time 2000 ms -2016-06-13 20:23:00 INFO CheckpointWriter:59 - Saving checkpoint for time 1000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-1000' -2016-06-13 20:23:00 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=0, maxMem=766075207 -2016-06-13 20:23:00 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:23:00 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:00 INFO JobGenerator:59 - Checkpointing graph for time 3000 ms -2016-06-13 20:23:00 INFO DStreamGraph:59 - Updating checkpoint data for time 3000 ms -2016-06-13 20:23:00 INFO DStreamGraph:59 - Updated checkpoint data for time 3000 ms -2016-06-13 20:23:00 INFO MemoryStore:59 - ensureFreeSpace(2428) called with curMem=4744, maxMem=766075207 -2016-06-13 20:23:00 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:23:00 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51134 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:23:00 INFO JobGenerator:59 - Checkpointing graph for time 4000 ms -2016-06-13 20:23:00 INFO DStreamGraph:59 - Updating checkpoint data for time 4000 ms -2016-06-13 20:23:00 INFO DStreamGraph:59 - Updated checkpoint data for time 4000 ms -2016-06-13 20:23:00 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[7] at apply at Transformer.scala:22) -2016-06-13 20:23:00 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 20:23:00 INFO JobGenerator:59 - Checkpointing graph for time 5000 ms -2016-06-13 20:23:00 INFO DStreamGraph:59 - Updating checkpoint data for time 5000 ms -2016-06-13 20:23:00 INFO DStreamGraph:59 - Updated checkpoint data for time 5000 ms -2016-06-13 20:23:00 INFO JobGenerator:59 - Checkpointing graph for time 6000 ms -2016-06-13 20:23:00 INFO DStreamGraph:59 - Updating checkpoint data for time 6000 ms -2016-06-13 20:23:00 INFO DStreamGraph:59 - Updated checkpoint data for time 6000 ms -2016-06-13 20:23:00 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:23:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:23:00 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:00 INFO JobGenerator:59 - Checkpointing graph for time 7000 ms -2016-06-13 20:23:00 INFO DStreamGraph:59 - Updating checkpoint data for time 7000 ms -2016-06-13 20:23:00 INFO DStreamGraph:59 - Updated checkpoint data for time 7000 ms -2016-06-13 20:23:00 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 20:23:00 INFO JobGenerator:59 - Checkpointing graph for time 8000 ms -2016-06-13 20:23:00 INFO DStreamGraph:59 - Updating checkpoint data for time 8000 ms -2016-06-13 20:23:00 INFO DStreamGraph:59 - Updated checkpoint data for time 8000 ms -2016-06-13 20:23:00 INFO JobGenerator:59 - Checkpointing graph for time 9000 ms -2016-06-13 20:23:00 INFO DStreamGraph:59 - Updating checkpoint data for time 9000 ms -2016-06-13 20:23:00 INFO DStreamGraph:59 - Updated checkpoint data for time 9000 ms -2016-06-13 20:23:00 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 20:23:00 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:00 INFO JobGenerator:59 - Checkpointing graph for time 10000 ms -2016-06-13 20:23:00 INFO DStreamGraph:59 - Updating checkpoint data for time 10000 ms -2016-06-13 20:23:00 INFO DStreamGraph:59 - Updated checkpoint data for time 10000 ms -2016-06-13 20:23:00 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 20:23:00 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 20:23:00 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:00 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:00 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1159 bytes result sent to driver -2016-06-13 20:23:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 248 ms on localhost (1/2) -2016-06-13 20:23:00 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:00 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:00 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:00 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:00 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1159 bytes result sent to driver -2016-06-13 20:23:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 497 ms on localhost (2/2) -2016-06-13 20:23:00 INFO DAGScheduler:59 - ShuffleMapStage 0 (apply at Transformer.scala:22) finished in 0.551 s -2016-06-13 20:23:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 20:23:00 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:23:00 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:23:00 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 20:23:00 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:23:00 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 20:23:00 INFO DAGScheduler:59 - Submitting ResultStage 1 (ShuffledRDD[17] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:23:00 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=7172, maxMem=766075207 -2016-06-13 20:23:00 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:23:00 INFO MemoryStore:59 - ensureFreeSpace(2657) called with curMem=12436, maxMem=766075207 -2016-06-13 20:23:00 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 20:23:00 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51134 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:23:00 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:00 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ShuffledRDD[17] at apply at Transformer.scala:22) -2016-06-13 20:23:00 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 20:23:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:00 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 20:23:00 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 20:23:00 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:23:00 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-06-13 20:23:00 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:23:00 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 6 ms -2016-06-13 20:23:00 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1161 bytes result sent to driver -2016-06-13 20:23:00 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 54 ms on localhost (1/2) -2016-06-13 20:23:00 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1498 bytes result sent to driver -2016-06-13 20:23:00 INFO CheckpointWriter:59 - Checkpoint for time 1000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-1000', took 27284 bytes and 718 ms -2016-06-13 20:23:00 INFO CheckpointWriter:59 - Saving checkpoint for time 2000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-2000' -2016-06-13 20:23:00 INFO DAGScheduler:59 - ResultStage 1 (apply at Transformer.scala:22) finished in 0.072 s -2016-06-13 20:23:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 71 ms on localhost (2/2) -2016-06-13 20:23:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 20:23:01 INFO DAGScheduler:59 - Job 0 finished: apply at Transformer.scala:22, took 0.897872 s -2016-06-13 20:23:01 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 50.0 and centroid: [-0.4843392480139548,0.9734525763849463,-1.7046336323611109,1.2778093795456185,0.3691650550997036] -2016-06-13 20:23:01 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:01 INFO JobScheduler:59 - Finished job streaming job 1000 ms.0 from job set of time 1000 ms -2016-06-13 20:23:01 INFO JobScheduler:59 - Starting job streaming job 1000 ms.1 from job set of time 1000 ms -2016-06-13 20:23:01 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:23:01 INFO DAGScheduler:59 - Registering RDD 3 (apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Got job 1 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:23:01 INFO DAGScheduler:59 - Final stage: ResultStage 3(apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 2) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 2) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (UnionRDD[3] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=15093, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(2000) called with curMem=18373, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2000.0 B, free 730.6 MB) -2016-06-13 20:23:01 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51134 (size: 2000.0 B, free: 730.6 MB) -2016-06-13 20:23:01 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:01 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 2 (UnionRDD[3] at apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 3 tasks -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:23:01 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 20:23:01 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 20:23:01 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1159 bytes result sent to driver -2016-06-13 20:23:01 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1159 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 2.0 in stage 2.0 (TID 6, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:23:01 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:01 INFO Executor:59 - Running task 2.0 in stage 2.0 (TID 6) -2016-06-13 20:23:01 INFO CheckpointWriter:59 - Checkpoint for time 2000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-2000', took 27286 bytes and 199 ms -2016-06-13 20:23:01 INFO CheckpointWriter:59 - Saving checkpoint for time 3000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-3000' -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 63 ms on localhost (1/3) -2016-06-13 20:23:01 INFO Executor:59 - Finished task 2.0 in stage 2.0 (TID 6). 1159 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 75 ms on localhost (2/3) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 2.0 in stage 2.0 (TID 6) in 45 ms on localhost (3/3) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 20:23:01 INFO DAGScheduler:59 - ShuffleMapStage 2 (apply at Transformer.scala:22) finished in 0.094 s -2016-06-13 20:23:01 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:23:01 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:23:01 INFO DAGScheduler:59 - waiting: Set(ResultStage 3) -2016-06-13 20:23:01 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:23:01 INFO DAGScheduler:59 - Missing parents for ResultStage 3: List() -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[62] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=20373, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.2 KB, free 730.6 MB) -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(1799) called with curMem=23613, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1799.0 B, free 730.6 MB) -2016-06-13 20:23:01 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:51134 (size: 1799.0 B, free: 730.6 MB) -2016-06-13 20:23:01 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[62] at apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 8, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:01 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 7) -2016-06-13 20:23:01 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 8) -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 2 ms -2016-06-13 20:23:01 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 7). 1180 bytes result sent to driver -2016-06-13 20:23:01 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 8). 1172 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 7) in 22 ms on localhost (1/2) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 8) in 25 ms on localhost (2/2) -2016-06-13 20:23:01 INFO StreamingKMeansSuite:59 - output.size = 0, numExpectedOutput = 10 -2016-06-13 20:23:01 INFO DAGScheduler:59 - ResultStage 3 (apply at Transformer.scala:22) finished in 0.027 s -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 20:23:01 INFO DAGScheduler:59 - Job 1 finished: apply at Transformer.scala:22, took 0.232162 s -2016-06-13 20:23:01 INFO JobScheduler:59 - Finished job streaming job 1000 ms.1 from job set of time 1000 ms -2016-06-13 20:23:01 INFO JobScheduler:59 - Total delay: 1465820580.281 s for time 1000 ms (execution: 1.234 s) -2016-06-13 20:23:01 INFO JobScheduler:59 - Starting job streaming job 2000 ms.0 from job set of time 2000 ms -2016-06-13 20:23:01 INFO JobGenerator:59 - Checkpointing graph for time 1000 ms -2016-06-13 20:23:01 INFO DStreamGraph:59 - Updating checkpoint data for time 1000 ms -2016-06-13 20:23:01 INFO DStreamGraph:59 - Updated checkpoint data for time 1000 ms -2016-06-13 20:23:01 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:23:01 INFO DAGScheduler:59 - Registering RDD 63 (apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Got job 2 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:23:01 INFO DAGScheduler:59 - Final stage: ResultStage 5(apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 4) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting ShuffleMapStage 4 (MapPartitionsRDD[63] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=25412, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 4.6 KB, free 730.6 MB) -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(2471) called with curMem=30156, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.6 MB) -2016-06-13 20:23:01 INFO CheckpointWriter:59 - Checkpoint for time 3000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-3000', took 27286 bytes and 152 ms -2016-06-13 20:23:01 INFO CheckpointWriter:59 - Saving checkpoint for time 4000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-4000' -2016-06-13 20:23:01 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:51134 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:23:01 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 4 (MapPartitionsRDD[63] at apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 10, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:23:01 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 10) -2016-06-13 20:23:01 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 9) -2016-06-13 20:23:01 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 10). 1159 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 10) in 32 ms on localhost (1/2) -2016-06-13 20:23:01 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 9). 1159 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 9) in 44 ms on localhost (2/2) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 20:23:01 INFO DAGScheduler:59 - ShuffleMapStage 4 (apply at Transformer.scala:22) finished in 0.047 s -2016-06-13 20:23:01 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:23:01 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:23:01 INFO DAGScheduler:59 - waiting: Set(ResultStage 5) -2016-06-13 20:23:01 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:23:01 INFO DAGScheduler:59 - Missing parents for ResultStage 5: List() -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting ResultStage 5 (ShuffledRDD[64] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=32627, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 5.1 KB, free 730.6 MB) -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(2692) called with curMem=37891, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:23:01 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:51134 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:23:01 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (ShuffledRDD[64] at apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 11, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 12, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:01 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 11) -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:23:01 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:23:01 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 12) -2016-06-13 20:23:01 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 11). 1498 bytes result sent to driver -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:23:01 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 12). 1161 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 11) in 33 ms on localhost (1/2) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 12) in 37 ms on localhost (2/2) -2016-06-13 20:23:01 INFO DAGScheduler:59 - ResultStage 5 (apply at Transformer.scala:22) finished in 0.040 s -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 20:23:01 INFO DAGScheduler:59 - Job 2 finished: apply at Transformer.scala:22, took 0.135398 s -2016-06-13 20:23:01 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 100.0 and centroid: [-0.4738959687352186,0.969729236815007,-1.6823805644329948,1.2679893451777908,0.37589591355279117] -2016-06-13 20:23:01 INFO JobScheduler:59 - Finished job streaming job 2000 ms.0 from job set of time 2000 ms -2016-06-13 20:23:01 INFO JobScheduler:59 - Starting job streaming job 2000 ms.1 from job set of time 2000 ms -2016-06-13 20:23:01 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:23:01 INFO DAGScheduler:59 - Registering RDD 10 (apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Got job 3 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:23:01 INFO DAGScheduler:59 - Final stage: ResultStage 7(apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 6) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 6) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting ShuffleMapStage 6 (UnionRDD[10] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=40583, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:23:01 INFO CheckpointWriter:59 - Checkpoint for time 4000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-4000', took 27286 bytes and 129 ms -2016-06-13 20:23:01 INFO CheckpointWriter:59 - Saving checkpoint for time 5000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-5000' -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=43863, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:23:01 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:51134 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:23:01 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 6 (UnionRDD[10] at apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 3 tasks -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 13, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 14, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:23:01 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 13) -2016-06-13 20:23:01 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:23:01 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 14) -2016-06-13 20:23:01 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 13). 1159 bytes result sent to driver -2016-06-13 20:23:01 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 14). 1159 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 2.0 in stage 6.0 (TID 15, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 13) in 26 ms on localhost (1/3) -2016-06-13 20:23:01 INFO Executor:59 - Running task 2.0 in stage 6.0 (TID 15) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 14) in 26 ms on localhost (2/3) -2016-06-13 20:23:01 INFO Executor:59 - Finished task 2.0 in stage 6.0 (TID 15). 1159 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 2.0 in stage 6.0 (TID 15) in 24 ms on localhost (3/3) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 20:23:01 INFO DAGScheduler:59 - ShuffleMapStage 6 (apply at Transformer.scala:22) finished in 0.048 s -2016-06-13 20:23:01 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:23:01 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:23:01 INFO DAGScheduler:59 - waiting: Set(ResultStage 7) -2016-06-13 20:23:01 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:23:01 INFO DAGScheduler:59 - Missing parents for ResultStage 7: List() -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[65] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=45872, maxMem=766075207 -2016-06-13 20:23:01 INFO StreamingKMeansSuite:59 - output.size = 1, numExpectedOutput = 10 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=49112, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.5 MB) -2016-06-13 20:23:01 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:51134 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:23:01 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[65] at apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 16, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 17, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:01 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 17) -2016-06-13 20:23:01 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 16) -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:23:01 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 17). 1172 bytes result sent to driver -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 17) in 13 ms on localhost (1/2) -2016-06-13 20:23:01 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 16). 1180 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 16) in 22 ms on localhost (2/2) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 20:23:01 INFO DAGScheduler:59 - ResultStage 7 (apply at Transformer.scala:22) finished in 0.022 s -2016-06-13 20:23:01 INFO DAGScheduler:59 - Job 3 finished: apply at Transformer.scala:22, took 0.102220 s -2016-06-13 20:23:01 INFO JobScheduler:59 - Finished job streaming job 2000 ms.1 from job set of time 2000 ms -2016-06-13 20:23:01 INFO JobScheduler:59 - Total delay: 1465820579.561 s for time 2000 ms (execution: 0.277 s) -2016-06-13 20:23:01 INFO JobScheduler:59 - Starting job streaming job 3000 ms.0 from job set of time 3000 ms -2016-06-13 20:23:01 INFO ParallelCollectionRDD:59 - Removing RDD 0 from persistence list -2016-06-13 20:23:01 INFO MapPartitionsRDD:59 - Removing RDD 5 from persistence list -2016-06-13 20:23:01 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:23:01 INFO CheckpointWriter:59 - Checkpoint for time 5000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-5000', took 27286 bytes and 115 ms -2016-06-13 20:23:01 INFO CheckpointWriter:59 - Saving checkpoint for time 6000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-6000' -2016-06-13 20:23:01 INFO ShuffledRDD:59 - Removing RDD 4 from persistence list -2016-06-13 20:23:01 INFO UnionRDD:59 - Removing RDD 3 from persistence list -2016-06-13 20:23:01 INFO MapPartitionsRDD:59 - Removing RDD 1 from persistence list -2016-06-13 20:23:01 INFO BlockManager:59 - Removing RDD 1 -2016-06-13 20:23:01 INFO JobGenerator:59 - Checkpointing graph for time 2000 ms -2016-06-13 20:23:01 INFO DStreamGraph:59 - Updating checkpoint data for time 2000 ms -2016-06-13 20:23:01 INFO DStreamGraph:59 - Updated checkpoint data for time 2000 ms -2016-06-13 20:23:01 INFO DAGScheduler:59 - Registering RDD 66 (apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Got job 4 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:23:01 INFO DAGScheduler:59 - Final stage: ResultStage 9(apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 8) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 8) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting ShuffleMapStage 8 (MapPartitionsRDD[66] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=50914, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:23:01 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(2475) called with curMem=55658, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:23:01 INFO BlockManager:59 - Removing RDD 0 -2016-06-13 20:23:01 INFO BlockManager:59 - Removing RDD 5 -2016-06-13 20:23:01 INFO BlockManager:59 - Removing RDD 4 -2016-06-13 20:23:01 INFO BlockManager:59 - Removing RDD 3 -2016-06-13 20:23:01 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:51134 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:23:01 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 8 (MapPartitionsRDD[66] at apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 18, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 19, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:23:01 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 19) -2016-06-13 20:23:01 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 18) -2016-06-13 20:23:01 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 19). 1159 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 19) in 21 ms on localhost (1/2) -2016-06-13 20:23:01 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 18). 1159 bytes result sent to driver -2016-06-13 20:23:01 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 18) in 37 ms on localhost (2/2) -2016-06-13 20:23:01 INFO DAGScheduler:59 - ShuffleMapStage 8 (apply at Transformer.scala:22) finished in 0.038 s -2016-06-13 20:23:01 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:23:01 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:23:01 INFO DAGScheduler:59 - waiting: Set(ResultStage 9) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 20:23:01 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:23:01 INFO DAGScheduler:59 - Missing parents for ResultStage 9: List() -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting ResultStage 9 (ShuffledRDD[67] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=58133, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=63397, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:23:01 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:51134 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:23:01 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (ShuffledRDD[67] at apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 20, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 21, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:01 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 20) -2016-06-13 20:23:01 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 21) -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:23:01 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 21). 1161 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 21) in 11 ms on localhost (1/2) -2016-06-13 20:23:01 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 20). 1498 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 20) in 16 ms on localhost (2/2) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 20:23:01 INFO DAGScheduler:59 - ResultStage 9 (apply at Transformer.scala:22) finished in 0.017 s -2016-06-13 20:23:01 INFO DAGScheduler:59 - Job 4 finished: apply at Transformer.scala:22, took 0.120071 s -2016-06-13 20:23:01 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 150.0 and centroid: [-0.4741102089605591,0.9622750167428704,-1.680286076986711,1.2654376587211662,0.37713329463498424] -2016-06-13 20:23:01 INFO JobScheduler:59 - Finished job streaming job 3000 ms.0 from job set of time 3000 ms -2016-06-13 20:23:01 INFO CheckpointWriter:59 - Checkpoint for time 6000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-6000', took 27284 bytes and 120 ms -2016-06-13 20:23:01 INFO JobScheduler:59 - Starting job streaming job 3000 ms.1 from job set of time 3000 ms -2016-06-13 20:23:01 INFO CheckpointWriter:59 - Saving checkpoint for time 7000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-7000' -2016-06-13 20:23:01 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:23:01 INFO DAGScheduler:59 - Registering RDD 16 (apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Got job 5 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:23:01 INFO DAGScheduler:59 - Final stage: ResultStage 11(apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 10) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 10) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting ShuffleMapStage 10 (UnionRDD[16] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=66093, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:23:01 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(2004) called with curMem=69373, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 2004.0 B, free 730.5 MB) -2016-06-13 20:23:01 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:51134 (size: 2004.0 B, free: 730.6 MB) -2016-06-13 20:23:01 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 10 (UnionRDD[16] at apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Adding task set 10.0 with 3 tasks -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 10.0 (TID 22, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 10.0 (TID 23, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:23:01 INFO Executor:59 - Running task 1.0 in stage 10.0 (TID 23) -2016-06-13 20:23:01 INFO Executor:59 - Running task 0.0 in stage 10.0 (TID 22) -2016-06-13 20:23:01 INFO Executor:59 - Finished task 0.0 in stage 10.0 (TID 22). 1159 bytes result sent to driver -2016-06-13 20:23:01 INFO Executor:59 - Finished task 1.0 in stage 10.0 (TID 23). 1159 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 2.0 in stage 10.0 (TID 24, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:23:01 INFO Executor:59 - Running task 2.0 in stage 10.0 (TID 24) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 10.0 (TID 22) in 23 ms on localhost (1/3) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 10.0 (TID 23) in 24 ms on localhost (2/3) -2016-06-13 20:23:01 INFO Executor:59 - Finished task 2.0 in stage 10.0 (TID 24). 1159 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 2.0 in stage 10.0 (TID 24) in 16 ms on localhost (3/3) -2016-06-13 20:23:01 INFO DAGScheduler:59 - ShuffleMapStage 10 (apply at Transformer.scala:22) finished in 0.038 s -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 10.0, whose tasks have all completed, from pool -2016-06-13 20:23:01 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:23:01 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:23:01 INFO DAGScheduler:59 - waiting: Set(ResultStage 11) -2016-06-13 20:23:01 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:23:01 INFO DAGScheduler:59 - Missing parents for ResultStage 11: List() -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[68] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:23:01 INFO StreamingKMeansSuite:59 - output.size = 2, numExpectedOutput = 10 -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=71377, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=74617, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.5 MB) -2016-06-13 20:23:01 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:51134 (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:23:01 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 11 (MapPartitionsRDD[68] at apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 2 tasks -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 25, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 11.0 (TID 26, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:01 INFO Executor:59 - Running task 1.0 in stage 11.0 (TID 26) -2016-06-13 20:23:01 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 25) -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:23:01 INFO Executor:59 - Finished task 1.0 in stage 11.0 (TID 26). 1172 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 11.0 (TID 26) in 9 ms on localhost (1/2) -2016-06-13 20:23:01 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 25). 1180 bytes result sent to driver -2016-06-13 20:23:01 INFO DAGScheduler:59 - ResultStage 11 (apply at Transformer.scala:22) finished in 0.015 s -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 25) in 15 ms on localhost (2/2) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Job 5 finished: apply at Transformer.scala:22, took 0.086119 s -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-06-13 20:23:01 INFO JobScheduler:59 - Finished job streaming job 3000 ms.1 from job set of time 3000 ms -2016-06-13 20:23:01 INFO ParallelCollectionRDD:59 - Removing RDD 6 from persistence list -2016-06-13 20:23:01 INFO JobScheduler:59 - Total delay: 1465820578.797 s for time 3000 ms (execution: 0.235 s) -2016-06-13 20:23:01 INFO JobScheduler:59 - Starting job streaming job 4000 ms.0 from job set of time 4000 ms -2016-06-13 20:23:01 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-06-13 20:23:01 INFO BlockManager:59 - Removing RDD 6 -2016-06-13 20:23:01 INFO ShuffledRDD:59 - Removing RDD 11 from persistence list -2016-06-13 20:23:01 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 20:23:01 INFO UnionRDD:59 - Removing RDD 10 from persistence list -2016-06-13 20:23:01 INFO BlockManager:59 - Removing RDD 11 -2016-06-13 20:23:01 INFO MapPartitionsRDD:59 - Removing RDD 8 from persistence list -2016-06-13 20:23:01 INFO BlockManager:59 - Removing RDD 10 -2016-06-13 20:23:01 INFO JobGenerator:59 - Checkpointing graph for time 3000 ms -2016-06-13 20:23:01 INFO DStreamGraph:59 - Updating checkpoint data for time 3000 ms -2016-06-13 20:23:01 INFO BlockManager:59 - Removing RDD 8 -2016-06-13 20:23:01 INFO DStreamGraph:59 - Updated checkpoint data for time 3000 ms -2016-06-13 20:23:01 INFO CheckpointWriter:59 - Checkpoint for time 7000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-7000', took 27284 bytes and 108 ms -2016-06-13 20:23:01 INFO CheckpointWriter:59 - Saving checkpoint for time 8000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-8000' -2016-06-13 20:23:01 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:23:01 INFO DAGScheduler:59 - Registering RDD 69 (apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Got job 6 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:23:01 INFO DAGScheduler:59 - Final stage: ResultStage 13(apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 12) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 12) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting ShuffleMapStage 12 (MapPartitionsRDD[69] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=76419, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_12 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:23:01 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(2484) called with curMem=81163, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_12_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:23:01 INFO BlockManagerInfo:59 - Added broadcast_12_piece0 in memory on localhost:51134 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:23:01 INFO SparkContext:59 - Created broadcast 12 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 12 (MapPartitionsRDD[69] at apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Adding task set 12.0 with 2 tasks -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 12.0 (TID 27, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 12.0 (TID 28, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:23:01 INFO Executor:59 - Running task 1.0 in stage 12.0 (TID 28) -2016-06-13 20:23:01 INFO Executor:59 - Running task 0.0 in stage 12.0 (TID 27) -2016-06-13 20:23:01 INFO Executor:59 - Finished task 0.0 in stage 12.0 (TID 27). 1159 bytes result sent to driver -2016-06-13 20:23:01 INFO Executor:59 - Finished task 1.0 in stage 12.0 (TID 28). 1159 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 12.0 (TID 28) in 24 ms on localhost (1/2) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 12.0 (TID 27) in 28 ms on localhost (2/2) -2016-06-13 20:23:01 INFO DAGScheduler:59 - ShuffleMapStage 12 (apply at Transformer.scala:22) finished in 0.029 s -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 12.0, whose tasks have all completed, from pool -2016-06-13 20:23:01 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:23:01 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:23:01 INFO DAGScheduler:59 - waiting: Set(ResultStage 13) -2016-06-13 20:23:01 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:23:01 INFO DAGScheduler:59 - Missing parents for ResultStage 13: List() -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting ResultStage 13 (ShuffledRDD[70] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=83647, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_13 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:23:01 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(2698) called with curMem=88911, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_13_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:23:01 INFO BlockManagerInfo:59 - Added broadcast_13_piece0 in memory on localhost:51134 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:23:01 INFO SparkContext:59 - Created broadcast 13 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 13 (ShuffledRDD[70] at apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Adding task set 13.0 with 2 tasks -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 13.0 (TID 29, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 13.0 (TID 30, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:01 INFO Executor:59 - Running task 0.0 in stage 13.0 (TID 29) -2016-06-13 20:23:01 INFO Executor:59 - Running task 1.0 in stage 13.0 (TID 30) -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:23:01 INFO Executor:59 - Finished task 1.0 in stage 13.0 (TID 30). 1161 bytes result sent to driver -2016-06-13 20:23:01 INFO Executor:59 - Finished task 0.0 in stage 13.0 (TID 29). 1498 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 13.0 (TID 30) in 17 ms on localhost (1/2) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 13.0 (TID 29) in 19 ms on localhost (2/2) -2016-06-13 20:23:01 INFO DAGScheduler:59 - ResultStage 13 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 13.0, whose tasks have all completed, from pool -2016-06-13 20:23:01 INFO DAGScheduler:59 - Job 6 finished: apply at Transformer.scala:22, took 0.086579 s -2016-06-13 20:23:01 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 200.0 and centroid: [-0.4768735011530239,0.9606386018112957,-1.673148729841545,1.2634974127814163,0.38189605546947214] -2016-06-13 20:23:01 INFO JobScheduler:59 - Finished job streaming job 4000 ms.0 from job set of time 4000 ms -2016-06-13 20:23:01 INFO JobScheduler:59 - Starting job streaming job 4000 ms.1 from job set of time 4000 ms -2016-06-13 20:23:01 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:23:01 INFO DAGScheduler:59 - Registering RDD 23 (apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Got job 7 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:23:01 INFO DAGScheduler:59 - Final stage: ResultStage 15(apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 14) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 14) -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting ShuffleMapStage 14 (UnionRDD[23] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=91609, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_14 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=94889, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_14_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:23:01 INFO BlockManagerInfo:59 - Added broadcast_14_piece0 in memory on localhost:51134 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:23:01 INFO SparkContext:59 - Created broadcast 14 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 14 (UnionRDD[23] at apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Adding task set 14.0 with 3 tasks -2016-06-13 20:23:01 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 14.0 (TID 31, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 14.0 (TID 32, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:23:01 INFO Executor:59 - Running task 0.0 in stage 14.0 (TID 31) -2016-06-13 20:23:01 INFO Executor:59 - Running task 1.0 in stage 14.0 (TID 32) -2016-06-13 20:23:01 INFO CheckpointWriter:59 - Checkpoint for time 8000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-8000', took 27286 bytes and 120 ms -2016-06-13 20:23:01 INFO CheckpointWriter:59 - Saving checkpoint for time 9000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-9000' -2016-06-13 20:23:01 INFO Executor:59 - Finished task 1.0 in stage 14.0 (TID 32). 1159 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 2.0 in stage 14.0 (TID 33, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:23:01 INFO Executor:59 - Running task 2.0 in stage 14.0 (TID 33) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 14.0 (TID 32) in 20 ms on localhost (1/3) -2016-06-13 20:23:01 INFO Executor:59 - Finished task 0.0 in stage 14.0 (TID 31). 1159 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 14.0 (TID 31) in 30 ms on localhost (2/3) -2016-06-13 20:23:01 INFO Executor:59 - Finished task 2.0 in stage 14.0 (TID 33). 1159 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 2.0 in stage 14.0 (TID 33) in 11 ms on localhost (3/3) -2016-06-13 20:23:01 INFO DAGScheduler:59 - ShuffleMapStage 14 (apply at Transformer.scala:22) finished in 0.033 s -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 14.0, whose tasks have all completed, from pool -2016-06-13 20:23:01 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:23:01 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:23:01 INFO DAGScheduler:59 - waiting: Set(ResultStage 15) -2016-06-13 20:23:01 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:23:01 INFO DAGScheduler:59 - Missing parents for ResultStage 15: List() -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting ResultStage 15 (MapPartitionsRDD[71] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=96898, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_15 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:23:01 INFO MemoryStore:59 - ensureFreeSpace(1801) called with curMem=100138, maxMem=766075207 -2016-06-13 20:23:01 INFO MemoryStore:59 - Block broadcast_15_piece0 stored as bytes in memory (estimated size 1801.0 B, free 730.5 MB) -2016-06-13 20:23:01 INFO BlockManagerInfo:59 - Added broadcast_15_piece0 in memory on localhost:51134 (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:23:01 INFO SparkContext:59 - Created broadcast 15 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 15 (MapPartitionsRDD[71] at apply at Transformer.scala:22) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Adding task set 15.0 with 2 tasks -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 15.0 (TID 34, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 15.0 (TID 35, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:01 INFO Executor:59 - Running task 1.0 in stage 15.0 (TID 35) -2016-06-13 20:23:01 INFO Executor:59 - Running task 0.0 in stage 15.0 (TID 34) -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:23:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:23:01 INFO Executor:59 - Finished task 1.0 in stage 15.0 (TID 35). 1172 bytes result sent to driver -2016-06-13 20:23:01 INFO StreamingKMeansSuite:59 - output.size = 3, numExpectedOutput = 10 -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 1.0 in stage 15.0 (TID 35) in 11 ms on localhost (1/2) -2016-06-13 20:23:01 INFO Executor:59 - Finished task 0.0 in stage 15.0 (TID 34). 1180 bytes result sent to driver -2016-06-13 20:23:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 15.0 (TID 34) in 17 ms on localhost (2/2) -2016-06-13 20:23:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 15.0, whose tasks have all completed, from pool -2016-06-13 20:23:01 INFO DAGScheduler:59 - ResultStage 15 (apply at Transformer.scala:22) finished in 0.017 s -2016-06-13 20:23:01 INFO DAGScheduler:59 - Job 7 finished: apply at Transformer.scala:22, took 0.078825 s -2016-06-13 20:23:01 INFO JobScheduler:59 - Finished job streaming job 4000 ms.1 from job set of time 4000 ms -2016-06-13 20:23:01 INFO JobScheduler:59 - Total delay: 1465820577.989 s for time 4000 ms (execution: 0.191 s) -2016-06-13 20:23:01 INFO JobScheduler:59 - Starting job streaming job 5000 ms.0 from job set of time 5000 ms -2016-06-13 20:23:01 INFO ParallelCollectionRDD:59 - Removing RDD 13 from persistence list -2016-06-13 20:23:01 INFO BlockManager:59 - Removing RDD 13 -2016-06-13 20:23:01 INFO MapPartitionsRDD:59 - Removing RDD 19 from persistence list -2016-06-13 20:23:01 INFO BlockManager:59 - Removing RDD 19 -2016-06-13 20:23:01 INFO ShuffledRDD:59 - Removing RDD 18 from persistence list -2016-06-13 20:23:01 INFO UnionRDD:59 - Removing RDD 16 from persistence list -2016-06-13 20:23:01 INFO BlockManager:59 - Removing RDD 18 -2016-06-13 20:23:02 INFO MapPartitionsRDD:59 - Removing RDD 14 from persistence list -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 14 -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 16 -2016-06-13 20:23:02 INFO JobGenerator:59 - Checkpointing graph for time 4000 ms -2016-06-13 20:23:02 INFO DStreamGraph:59 - Updating checkpoint data for time 4000 ms -2016-06-13 20:23:02 INFO DStreamGraph:59 - Updated checkpoint data for time 4000 ms -2016-06-13 20:23:02 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Registering RDD 72 (apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Got job 8 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:23:02 INFO DAGScheduler:59 - Final stage: ResultStage 17(apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 16) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 16) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 16 (MapPartitionsRDD[72] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=101939, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_16 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(2476) called with curMem=106683, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_16_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Added broadcast_16_piece0 in memory on localhost:51134 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 20:23:02 INFO SparkContext:59 - Created broadcast 16 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 16 (MapPartitionsRDD[72] at apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Adding task set 16.0 with 2 tasks -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 16.0 (TID 36, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 16.0 (TID 37, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 0.0 in stage 16.0 (TID 36) -2016-06-13 20:23:02 INFO Executor:59 - Running task 1.0 in stage 16.0 (TID 37) -2016-06-13 20:23:02 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:23:02 INFO Executor:59 - Finished task 1.0 in stage 16.0 (TID 37). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 16.0 (TID 37) in 20 ms on localhost (1/2) -2016-06-13 20:23:02 INFO Executor:59 - Finished task 0.0 in stage 16.0 (TID 36). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 16.0 (TID 36) in 23 ms on localhost (2/2) -2016-06-13 20:23:02 INFO DAGScheduler:59 - ShuffleMapStage 16 (apply at Transformer.scala:22) finished in 0.024 s -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 16.0, whose tasks have all completed, from pool -2016-06-13 20:23:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:23:02 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:23:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 17) -2016-06-13 20:23:02 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:23:02 INFO CheckpointWriter:59 - Checkpoint for time 9000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-9000', took 27286 bytes and 120 ms -2016-06-13 20:23:02 INFO DAGScheduler:59 - Missing parents for ResultStage 17: List() -2016-06-13 20:23:02 INFO CheckpointWriter:59 - Saving checkpoint for time 10000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-10000' -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting ResultStage 17 (ShuffledRDD[73] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=109159, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_17 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=114423, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_17_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Added broadcast_17_piece0 in memory on localhost:51134 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 20:23:02 INFO SparkContext:59 - Created broadcast 17 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 17 (ShuffledRDD[73] at apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Adding task set 17.0 with 2 tasks -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 17.0 (TID 38, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 17.0 (TID 39, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 1.0 in stage 17.0 (TID 39) -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:51134 in memory (size: 2.6 KB, free: 730.5 MB) -2016-06-13 20:23:02 INFO Executor:59 - Running task 0.0 in stage 17.0 (TID 38) -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:23:02 INFO Executor:59 - Finished task 1.0 in stage 17.0 (TID 39). 1161 bytes result sent to driver -2016-06-13 20:23:02 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 17.0 (TID 39) in 12 ms on localhost (1/2) -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:23:02 INFO Executor:59 - Finished task 0.0 in stage 17.0 (TID 38). 1498 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 17.0 (TID 38) in 25 ms on localhost (2/2) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 17.0, whose tasks have all completed, from pool -2016-06-13 20:23:02 INFO DAGScheduler:59 - ResultStage 17 (apply at Transformer.scala:22) finished in 0.028 s -2016-06-13 20:23:02 INFO DAGScheduler:59 - Job 8 finished: apply at Transformer.scala:22, took 0.096613 s -2016-06-13 20:23:02 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 20:23:02 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 250.0 and centroid: [-0.4770613340912709,0.9657053185238176,-1.6732261982319274,1.26651642628913,0.38386658573056776] -2016-06-13 20:23:02 INFO JobScheduler:59 - Finished job streaming job 5000 ms.0 from job set of time 5000 ms -2016-06-13 20:23:02 INFO JobScheduler:59 - Starting job streaming job 5000 ms.1 from job set of time 5000 ms -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:51134 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:23:02 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 20:23:02 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Registering RDD 29 (apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Got job 9 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:23:02 INFO DAGScheduler:59 - Final stage: ResultStage 19(apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 18) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 18) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 18 (UnionRDD[29] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=101946, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_18 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(2010) called with curMem=105226, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_18_piece0 stored as bytes in memory (estimated size 2010.0 B, free 730.5 MB) -2016-06-13 20:23:02 INFO ContextCleaner:59 - Cleaned shuffle 2 -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Added broadcast_18_piece0 in memory on localhost:51134 (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:23:02 INFO SparkContext:59 - Created broadcast 18 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:51134 in memory (size: 1799.0 B, free: 730.6 MB) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 18 (UnionRDD[29] at apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Adding task set 18.0 with 3 tasks -2016-06-13 20:23:02 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 18.0 (TID 40, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 18.0 (TID 41, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 0.0 in stage 18.0 (TID 40) -2016-06-13 20:23:02 INFO Executor:59 - Running task 1.0 in stage 18.0 (TID 41) -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Removed broadcast_15_piece0 on localhost:51134 in memory (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:23:02 INFO ContextCleaner:59 - Cleaned accumulator 16 -2016-06-13 20:23:02 INFO Executor:59 - Finished task 0.0 in stage 18.0 (TID 40). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 2.0 in stage 18.0 (TID 42, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Finished task 1.0 in stage 18.0 (TID 41). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO Executor:59 - Running task 2.0 in stage 18.0 (TID 42) -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Removed broadcast_14_piece0 on localhost:51134 in memory (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:23:02 INFO Executor:59 - Finished task 2.0 in stage 18.0 (TID 42). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 18.0 (TID 41) in 42 ms on localhost (1/3) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 2.0 in stage 18.0 (TID 42) in 16 ms on localhost (2/3) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 18.0 (TID 40) in 45 ms on localhost (3/3) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 18.0, whose tasks have all completed, from pool -2016-06-13 20:23:02 INFO DAGScheduler:59 - ShuffleMapStage 18 (apply at Transformer.scala:22) finished in 0.053 s -2016-06-13 20:23:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:23:02 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:23:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 19) -2016-06-13 20:23:02 INFO ContextCleaner:59 - Cleaned accumulator 15 -2016-06-13 20:23:02 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:23:02 INFO DAGScheduler:59 - Missing parents for ResultStage 19: List() -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting ResultStage 19 (MapPartitionsRDD[74] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Removed broadcast_13_piece0 on localhost:51134 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=89169, maxMem=766075207 -2016-06-13 20:23:02 INFO StreamingKMeansSuite:59 - output.size = 4, numExpectedOutput = 10 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_19 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO ContextCleaner:59 - Cleaned accumulator 14 -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=82401, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_19_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Removed broadcast_12_piece0 on localhost:51134 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:23:02 INFO ContextCleaner:59 - Cleaned accumulator 13 -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Added broadcast_19_piece0 in memory on localhost:51134 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:23:02 INFO ContextCleaner:59 - Cleaned shuffle 6 -2016-06-13 20:23:02 INFO SparkContext:59 - Created broadcast 19 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 19 (MapPartitionsRDD[74] at apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Adding task set 19.0 with 2 tasks -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 19.0 (TID 43, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 19.0 (TID 44, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 1.0 in stage 19.0 (TID 44) -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Removed broadcast_11_piece0 on localhost:51134 in memory (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:23:02 INFO Executor:59 - Running task 0.0 in stage 19.0 (TID 43) -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:23:02 INFO Executor:59 - Finished task 1.0 in stage 19.0 (TID 44). 1172 bytes result sent to driver -2016-06-13 20:23:02 INFO ContextCleaner:59 - Cleaned accumulator 12 -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 19.0 (TID 44) in 10 ms on localhost (1/2) -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Removed broadcast_10_piece0 on localhost:51134 in memory (size: 2004.0 B, free: 730.6 MB) -2016-06-13 20:23:02 INFO ContextCleaner:59 - Cleaned accumulator 11 -2016-06-13 20:23:02 INFO Executor:59 - Finished task 0.0 in stage 19.0 (TID 43). 1180 bytes result sent to driver -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:51134 in memory (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:23:02 INFO ContextCleaner:59 - Cleaned accumulator 10 -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 19.0 (TID 43) in 20 ms on localhost (2/2) -2016-06-13 20:23:02 INFO DAGScheduler:59 - ResultStage 19 (apply at Transformer.scala:22) finished in 0.020 s -2016-06-13 20:23:02 INFO DAGScheduler:59 - Job 9 finished: apply at Transformer.scala:22, took 0.109696 s -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 19.0, whose tasks have all completed, from pool -2016-06-13 20:23:02 INFO JobScheduler:59 - Finished job streaming job 5000 ms.1 from job set of time 5000 ms -2016-06-13 20:23:02 INFO JobScheduler:59 - Total delay: 1465820577.228 s for time 5000 ms (execution: 0.239 s) -2016-06-13 20:23:02 INFO JobScheduler:59 - Starting job streaming job 6000 ms.0 from job set of time 6000 ms -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:51134 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:23:02 INFO ParallelCollectionRDD:59 - Removing RDD 20 from persistence list -2016-06-13 20:23:02 INFO MapPartitionsRDD:59 - Removing RDD 25 from persistence list -2016-06-13 20:23:02 INFO ContextCleaner:59 - Cleaned accumulator 9 -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 20 -2016-06-13 20:23:02 INFO ShuffledRDD:59 - Removing RDD 24 from persistence list -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 24 -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 25 -2016-06-13 20:23:02 INFO UnionRDD:59 - Removing RDD 23 from persistence list -2016-06-13 20:23:02 INFO ContextCleaner:59 - Cleaned shuffle 4 -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 23 -2016-06-13 20:23:02 INFO MapPartitionsRDD:59 - Removing RDD 21 from persistence list -2016-06-13 20:23:02 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:23:02 INFO JobGenerator:59 - Checkpointing graph for time 5000 ms -2016-06-13 20:23:02 INFO DStreamGraph:59 - Updating checkpoint data for time 5000 ms -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:51134 in memory (size: 1802.0 B, free: 730.6 MB) -2016-06-13 20:23:02 INFO DStreamGraph:59 - Updated checkpoint data for time 5000 ms -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 21 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Registering RDD 75 (apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Got job 10 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:23:02 INFO DAGScheduler:59 - Final stage: ResultStage 21(apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 20) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 20) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 20 (MapPartitionsRDD[75] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:51134 in memory (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=45884, maxMem=766075207 -2016-06-13 20:23:02 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 20:23:02 INFO CheckpointWriter:59 - Checkpoint for time 10000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-10000', took 27298 bytes and 205 ms -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_20 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO CheckpointWriter:59 - Saving checkpoint for time 1000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-1000' -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=50628, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_20_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Added broadcast_20_piece0 in memory on localhost:51134 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:23:02 INFO SparkContext:59 - Created broadcast 20 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 20 (MapPartitionsRDD[75] at apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Adding task set 20.0 with 2 tasks -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 20.0 (TID 45, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 20.0 (TID 46, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 0.0 in stage 20.0 (TID 45) -2016-06-13 20:23:02 INFO Executor:59 - Running task 1.0 in stage 20.0 (TID 46) -2016-06-13 20:23:02 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:23:02 INFO Executor:59 - Finished task 1.0 in stage 20.0 (TID 46). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO Executor:59 - Finished task 0.0 in stage 20.0 (TID 45). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 20.0 (TID 46) in 50 ms on localhost (1/2) -2016-06-13 20:23:02 INFO DAGScheduler:59 - ShuffleMapStage 20 (apply at Transformer.scala:22) finished in 0.052 s -2016-06-13 20:23:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:23:02 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:23:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 21) -2016-06-13 20:23:02 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:23:02 INFO DAGScheduler:59 - Missing parents for ResultStage 21: List() -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting ResultStage 21 (ShuffledRDD[76] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 20.0 (TID 45) in 52 ms on localhost (2/2) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 20.0, whose tasks have all completed, from pool -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=53110, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_21 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=58374, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_21_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Added broadcast_21_piece0 in memory on localhost:51134 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:23:02 INFO SparkContext:59 - Created broadcast 21 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 21 (ShuffledRDD[76] at apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Adding task set 21.0 with 2 tasks -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 21.0 (TID 47, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 21.0 (TID 48, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 0.0 in stage 21.0 (TID 47) -2016-06-13 20:23:02 INFO Executor:59 - Running task 1.0 in stage 21.0 (TID 48) -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:23:02 INFO Executor:59 - Finished task 1.0 in stage 21.0 (TID 48). 1161 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 21.0 (TID 48) in 10 ms on localhost (1/2) -2016-06-13 20:23:02 INFO Executor:59 - Finished task 0.0 in stage 21.0 (TID 47). 1498 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 21.0 (TID 47) in 19 ms on localhost (2/2) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 21.0, whose tasks have all completed, from pool -2016-06-13 20:23:02 INFO DAGScheduler:59 - ResultStage 21 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:23:02 INFO DAGScheduler:59 - Job 10 finished: apply at Transformer.scala:22, took 0.104113 s -2016-06-13 20:23:02 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 300.0 and centroid: [-0.4778380287460108,0.9658292271340416,-1.6712646630450996,1.273358698745343,0.37938624993358266] -2016-06-13 20:23:02 INFO JobScheduler:59 - Finished job streaming job 6000 ms.0 from job set of time 6000 ms -2016-06-13 20:23:02 INFO JobScheduler:59 - Starting job streaming job 6000 ms.1 from job set of time 6000 ms -2016-06-13 20:23:02 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:23:02 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Registering RDD 35 (apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Got job 11 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:23:02 INFO DAGScheduler:59 - Final stage: ResultStage 23(apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 22) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 22) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 22 (UnionRDD[35] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=61070, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_22 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=64350, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_22_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.5 MB) -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Added broadcast_22_piece0 in memory on localhost:51134 (size: 2007.0 B, free: 730.6 MB) -2016-06-13 20:23:02 INFO SparkContext:59 - Created broadcast 22 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 22 (UnionRDD[35] at apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Adding task set 22.0 with 3 tasks -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 22.0 (TID 49, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 22.0 (TID 50, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 0.0 in stage 22.0 (TID 49) -2016-06-13 20:23:02 INFO Executor:59 - Running task 1.0 in stage 22.0 (TID 50) -2016-06-13 20:23:02 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-1000.bk -2016-06-13 20:23:02 INFO Executor:59 - Finished task 1.0 in stage 22.0 (TID 50). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 2.0 in stage 22.0 (TID 51, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 2.0 in stage 22.0 (TID 51) -2016-06-13 20:23:02 INFO CheckpointWriter:59 - Checkpoint for time 1000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-1000', took 27478 bytes and 154 ms -2016-06-13 20:23:02 INFO CheckpointWriter:59 - Saving checkpoint for time 2000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-2000' -2016-06-13 20:23:02 INFO Executor:59 - Finished task 0.0 in stage 22.0 (TID 49). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO Executor:59 - Finished task 2.0 in stage 22.0 (TID 51). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO DStreamGraph:59 - Clearing checkpoint data for time 1000 ms -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 22.0 (TID 50) in 45 ms on localhost (1/3) -2016-06-13 20:23:02 INFO DStreamGraph:59 - Cleared checkpoint data for time 1000 ms -2016-06-13 20:23:02 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 22.0 (TID 49) in 69 ms on localhost (2/3) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 2.0 in stage 22.0 (TID 51) in 39 ms on localhost (3/3) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 22.0, whose tasks have all completed, from pool -2016-06-13 20:23:02 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/receivedBlockMetadata older than 0: -2016-06-13 20:23:02 INFO InputInfoTracker:59 - remove old batch metadata: -2016-06-13 20:23:02 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/receivedBlockMetadata older than 0 -2016-06-13 20:23:02 INFO DAGScheduler:59 - ShuffleMapStage 22 (apply at Transformer.scala:22) finished in 0.112 s -2016-06-13 20:23:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:23:02 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:23:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 23) -2016-06-13 20:23:02 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:23:02 INFO DAGScheduler:59 - Missing parents for ResultStage 23: List() -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting ResultStage 23 (MapPartitionsRDD[77] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=66357, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_23 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=69597, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_23_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.5 MB) -2016-06-13 20:23:02 INFO StreamingKMeansSuite:59 - output.size = 5, numExpectedOutput = 10 -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Added broadcast_23_piece0 in memory on localhost:51134 (size: 1803.0 B, free: 730.6 MB) -2016-06-13 20:23:02 INFO SparkContext:59 - Created broadcast 23 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 23 (MapPartitionsRDD[77] at apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Adding task set 23.0 with 2 tasks -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 23.0 (TID 52, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 23.0 (TID 53, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 0.0 in stage 23.0 (TID 52) -2016-06-13 20:23:02 INFO Executor:59 - Running task 1.0 in stage 23.0 (TID 53) -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:23:02 INFO Executor:59 - Finished task 0.0 in stage 23.0 (TID 52). 1180 bytes result sent to driver -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:23:02 INFO Executor:59 - Finished task 1.0 in stage 23.0 (TID 53). 1172 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 23.0 (TID 52) in 15 ms on localhost (1/2) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 23.0 (TID 53) in 14 ms on localhost (2/2) -2016-06-13 20:23:02 INFO DAGScheduler:59 - ResultStage 23 (apply at Transformer.scala:22) finished in 0.018 s -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 23.0, whose tasks have all completed, from pool -2016-06-13 20:23:02 INFO DAGScheduler:59 - Job 11 finished: apply at Transformer.scala:22, took 0.162748 s -2016-06-13 20:23:02 INFO JobScheduler:59 - Finished job streaming job 6000 ms.1 from job set of time 6000 ms -2016-06-13 20:23:02 INFO ParallelCollectionRDD:59 - Removing RDD 26 from persistence list -2016-06-13 20:23:02 INFO JobScheduler:59 - Total delay: 1465820576.524 s for time 6000 ms (execution: 0.295 s) -2016-06-13 20:23:02 INFO JobScheduler:59 - Starting job streaming job 7000 ms.0 from job set of time 7000 ms -2016-06-13 20:23:02 INFO MapPartitionsRDD:59 - Removing RDD 31 from persistence list -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 26 -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 31 -2016-06-13 20:23:02 INFO ShuffledRDD:59 - Removing RDD 30 from persistence list -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 30 -2016-06-13 20:23:02 INFO UnionRDD:59 - Removing RDD 29 from persistence list -2016-06-13 20:23:02 INFO MapPartitionsRDD:59 - Removing RDD 27 from persistence list -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 29 -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 27 -2016-06-13 20:23:02 INFO JobGenerator:59 - Checkpointing graph for time 6000 ms -2016-06-13 20:23:02 INFO DStreamGraph:59 - Updating checkpoint data for time 6000 ms -2016-06-13 20:23:02 INFO DStreamGraph:59 - Updated checkpoint data for time 6000 ms -2016-06-13 20:23:02 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Registering RDD 78 (apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Got job 12 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:23:02 INFO DAGScheduler:59 - Final stage: ResultStage 25(apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 24) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 24) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 24 (MapPartitionsRDD[78] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:23:02 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=71400, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_24 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(2480) called with curMem=76144, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_24_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Added broadcast_24_piece0 in memory on localhost:51134 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:23:02 INFO SparkContext:59 - Created broadcast 24 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 24 (MapPartitionsRDD[78] at apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Adding task set 24.0 with 2 tasks -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 24.0 (TID 54, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 24.0 (TID 55, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 1.0 in stage 24.0 (TID 55) -2016-06-13 20:23:02 INFO Executor:59 - Running task 0.0 in stage 24.0 (TID 54) -2016-06-13 20:23:02 INFO Executor:59 - Finished task 1.0 in stage 24.0 (TID 55). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO Executor:59 - Finished task 0.0 in stage 24.0 (TID 54). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 24.0 (TID 55) in 18 ms on localhost (1/2) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 24.0 (TID 54) in 20 ms on localhost (2/2) -2016-06-13 20:23:02 INFO DAGScheduler:59 - ShuffleMapStage 24 (apply at Transformer.scala:22) finished in 0.020 s -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 24.0, whose tasks have all completed, from pool -2016-06-13 20:23:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:23:02 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:23:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 25) -2016-06-13 20:23:02 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:23:02 INFO DAGScheduler:59 - Missing parents for ResultStage 25: List() -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting ResultStage 25 (ShuffledRDD[79] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=78624, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_25 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=83888, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_25_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Added broadcast_25_piece0 in memory on localhost:51134 (size: 2.6 KB, free: 730.6 MB) -2016-06-13 20:23:02 INFO SparkContext:59 - Created broadcast 25 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 25 (ShuffledRDD[79] at apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Adding task set 25.0 with 2 tasks -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 25.0 (TID 56, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 25.0 (TID 57, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 0.0 in stage 25.0 (TID 56) -2016-06-13 20:23:02 INFO Executor:59 - Running task 1.0 in stage 25.0 (TID 57) -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:23:02 INFO Executor:59 - Finished task 1.0 in stage 25.0 (TID 57). 1161 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 25.0 (TID 57) in 8 ms on localhost (1/2) -2016-06-13 20:23:02 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:23:02 INFO Executor:59 - Finished task 0.0 in stage 25.0 (TID 56). 1498 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 25.0 (TID 56) in 13 ms on localhost (2/2) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 25.0, whose tasks have all completed, from pool -2016-06-13 20:23:02 INFO DAGScheduler:59 - ResultStage 25 (apply at Transformer.scala:22) finished in 0.013 s -2016-06-13 20:23:02 INFO DAGScheduler:59 - Job 12 finished: apply at Transformer.scala:22, took 0.064168 s -2016-06-13 20:23:02 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 350.0 and centroid: [-0.47440048489638564,0.9650523283036628,-1.6700055274657652,1.269947562552048,0.3797915177328754] -2016-06-13 20:23:02 INFO JobScheduler:59 - Finished job streaming job 7000 ms.0 from job set of time 7000 ms -2016-06-13 20:23:02 INFO JobScheduler:59 - Starting job streaming job 7000 ms.1 from job set of time 7000 ms -2016-06-13 20:23:02 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:23:02 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-1000 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Registering RDD 41 (apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Got job 13 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:23:02 INFO DAGScheduler:59 - Final stage: ResultStage 27(apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 26) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 26) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 26 (UnionRDD[41] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=86584, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_26 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(2009) called with curMem=89864, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_26_piece0 stored as bytes in memory (estimated size 2009.0 B, free 730.5 MB) -2016-06-13 20:23:02 INFO CheckpointWriter:59 - Checkpoint for time 2000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-2000', took 27491 bytes and 211 ms -2016-06-13 20:23:02 INFO DStreamGraph:59 - Clearing checkpoint data for time 2000 ms -2016-06-13 20:23:02 INFO DStreamGraph:59 - Cleared checkpoint data for time 2000 ms -2016-06-13 20:23:02 INFO CheckpointWriter:59 - Saving checkpoint for time 3000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-3000' -2016-06-13 20:23:02 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Added broadcast_26_piece0 in memory on localhost:51134 (size: 2009.0 B, free: 730.6 MB) -2016-06-13 20:23:02 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/receivedBlockMetadata older than 1000: -2016-06-13 20:23:02 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/receivedBlockMetadata older than 1000 -2016-06-13 20:23:02 INFO InputInfoTracker:59 - remove old batch metadata: -2016-06-13 20:23:02 INFO SparkContext:59 - Created broadcast 26 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 26 (UnionRDD[41] at apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Adding task set 26.0 with 3 tasks -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 26.0 (TID 58, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 26.0 (TID 59, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 0.0 in stage 26.0 (TID 58) -2016-06-13 20:23:02 INFO Executor:59 - Running task 1.0 in stage 26.0 (TID 59) -2016-06-13 20:23:02 INFO Executor:59 - Finished task 1.0 in stage 26.0 (TID 59). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 2.0 in stage 26.0 (TID 60, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 26.0 (TID 59) in 19 ms on localhost (1/3) -2016-06-13 20:23:02 INFO Executor:59 - Running task 2.0 in stage 26.0 (TID 60) -2016-06-13 20:23:02 INFO StreamingKMeansSuite:59 - output.size = 6, numExpectedOutput = 10 -2016-06-13 20:23:02 INFO Executor:59 - Finished task 2.0 in stage 26.0 (TID 60). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO Executor:59 - Finished task 0.0 in stage 26.0 (TID 58). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 2.0 in stage 26.0 (TID 60) in 15 ms on localhost (2/3) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 26.0 (TID 58) in 35 ms on localhost (3/3) -2016-06-13 20:23:02 INFO DAGScheduler:59 - ShuffleMapStage 26 (apply at Transformer.scala:22) finished in 0.036 s -2016-06-13 20:23:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:23:02 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:23:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 27) -2016-06-13 20:23:02 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:23:02 INFO DAGScheduler:59 - Missing parents for ResultStage 27: List() -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 26.0, whose tasks have all completed, from pool -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting ResultStage 27 (MapPartitionsRDD[80] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=91873, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_27 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(1801) called with curMem=95113, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_27_piece0 stored as bytes in memory (estimated size 1801.0 B, free 730.5 MB) -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Added broadcast_27_piece0 in memory on localhost:51134 (size: 1801.0 B, free: 730.6 MB) -2016-06-13 20:23:02 INFO SparkContext:59 - Created broadcast 27 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 27 (MapPartitionsRDD[80] at apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Adding task set 27.0 with 2 tasks -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 27.0 (TID 61, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 27.0 (TID 62, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 1.0 in stage 27.0 (TID 62) -2016-06-13 20:23:02 INFO Executor:59 - Running task 0.0 in stage 27.0 (TID 61) -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:23:02 INFO Executor:59 - Finished task 1.0 in stage 27.0 (TID 62). 1172 bytes result sent to driver -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 27.0 (TID 62) in 9 ms on localhost (1/2) -2016-06-13 20:23:02 INFO Executor:59 - Finished task 0.0 in stage 27.0 (TID 61). 1180 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 27.0 (TID 61) in 17 ms on localhost (2/2) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 27.0, whose tasks have all completed, from pool -2016-06-13 20:23:02 INFO DAGScheduler:59 - ResultStage 27 (apply at Transformer.scala:22) finished in 0.017 s -2016-06-13 20:23:02 INFO DAGScheduler:59 - Job 13 finished: apply at Transformer.scala:22, took 0.078420 s -2016-06-13 20:23:02 INFO JobScheduler:59 - Finished job streaming job 7000 ms.1 from job set of time 7000 ms -2016-06-13 20:23:02 INFO JobScheduler:59 - Total delay: 1465820575.693 s for time 7000 ms (execution: 0.168 s) -2016-06-13 20:23:02 INFO JobScheduler:59 - Starting job streaming job 8000 ms.0 from job set of time 8000 ms -2016-06-13 20:23:02 INFO ParallelCollectionRDD:59 - Removing RDD 32 from persistence list -2016-06-13 20:23:02 INFO MapPartitionsRDD:59 - Removing RDD 37 from persistence list -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 32 -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 37 -2016-06-13 20:23:02 INFO ShuffledRDD:59 - Removing RDD 36 from persistence list -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 36 -2016-06-13 20:23:02 INFO UnionRDD:59 - Removing RDD 35 from persistence list -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 35 -2016-06-13 20:23:02 INFO MapPartitionsRDD:59 - Removing RDD 33 from persistence list -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 33 -2016-06-13 20:23:02 INFO JobGenerator:59 - Checkpointing graph for time 7000 ms -2016-06-13 20:23:02 INFO DStreamGraph:59 - Updating checkpoint data for time 7000 ms -2016-06-13 20:23:02 INFO DStreamGraph:59 - Updated checkpoint data for time 7000 ms -2016-06-13 20:23:02 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:23:02 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Registering RDD 81 (apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Got job 14 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:23:02 INFO DAGScheduler:59 - Final stage: ResultStage 29(apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 28) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 28) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 28 (MapPartitionsRDD[81] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=96914, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_28 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=101658, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_28_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Added broadcast_28_piece0 in memory on localhost:51134 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 20:23:02 INFO SparkContext:59 - Created broadcast 28 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 28 (MapPartitionsRDD[81] at apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Adding task set 28.0 with 2 tasks -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 28.0 (TID 63, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 28.0 (TID 64, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 0.0 in stage 28.0 (TID 63) -2016-06-13 20:23:02 INFO Executor:59 - Running task 1.0 in stage 28.0 (TID 64) -2016-06-13 20:23:02 INFO Executor:59 - Finished task 1.0 in stage 28.0 (TID 64). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 28.0 (TID 64) in 15 ms on localhost (1/2) -2016-06-13 20:23:02 INFO Executor:59 - Finished task 0.0 in stage 28.0 (TID 63). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 28.0 (TID 63) in 18 ms on localhost (2/2) -2016-06-13 20:23:02 INFO DAGScheduler:59 - ShuffleMapStage 28 (apply at Transformer.scala:22) finished in 0.018 s -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 28.0, whose tasks have all completed, from pool -2016-06-13 20:23:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:23:02 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:23:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 29) -2016-06-13 20:23:02 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:23:02 INFO DAGScheduler:59 - Missing parents for ResultStage 29: List() -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting ResultStage 29 (ShuffledRDD[82] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=104140, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_29 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=109404, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_29_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Added broadcast_29_piece0 in memory on localhost:51134 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 20:23:02 INFO SparkContext:59 - Created broadcast 29 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 29 (ShuffledRDD[82] at apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Adding task set 29.0 with 2 tasks -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 29.0 (TID 65, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 29.0 (TID 66, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 1.0 in stage 29.0 (TID 66) -2016-06-13 20:23:02 INFO Executor:59 - Running task 0.0 in stage 29.0 (TID 65) -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:23:02 INFO Executor:59 - Finished task 1.0 in stage 29.0 (TID 66). 1161 bytes result sent to driver -2016-06-13 20:23:02 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 29.0 (TID 66) in 8 ms on localhost (1/2) -2016-06-13 20:23:02 INFO Executor:59 - Finished task 0.0 in stage 29.0 (TID 65). 1498 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 29.0 (TID 65) in 14 ms on localhost (2/2) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 29.0, whose tasks have all completed, from pool -2016-06-13 20:23:02 INFO DAGScheduler:59 - ResultStage 29 (apply at Transformer.scala:22) finished in 0.015 s -2016-06-13 20:23:02 INFO DAGScheduler:59 - Job 14 finished: apply at Transformer.scala:22, took 0.055961 s -2016-06-13 20:23:02 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-2000.bk -2016-06-13 20:23:02 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 400.0 and centroid: [-0.4716182064591067,0.967586334056103,-1.6699954778983372,1.270341503100185,0.37935009425710725] -2016-06-13 20:23:02 INFO JobScheduler:59 - Finished job streaming job 8000 ms.0 from job set of time 8000 ms -2016-06-13 20:23:02 INFO JobScheduler:59 - Starting job streaming job 8000 ms.1 from job set of time 8000 ms -2016-06-13 20:23:02 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:23:02 INFO CheckpointWriter:59 - Checkpoint for time 3000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-3000', took 27502 bytes and 152 ms -2016-06-13 20:23:02 INFO DAGScheduler:59 - Registering RDD 47 (apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO DStreamGraph:59 - Clearing checkpoint data for time 3000 ms -2016-06-13 20:23:02 INFO CheckpointWriter:59 - Saving checkpoint for time 4000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-4000' -2016-06-13 20:23:02 INFO DStreamGraph:59 - Cleared checkpoint data for time 3000 ms -2016-06-13 20:23:02 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:23:02 INFO DAGScheduler:59 - Got job 15 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:23:02 INFO DAGScheduler:59 - Final stage: ResultStage 31(apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 30) -2016-06-13 20:23:02 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/receivedBlockMetadata older than 2000: -2016-06-13 20:23:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 30) -2016-06-13 20:23:02 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/receivedBlockMetadata older than 2000 -2016-06-13 20:23:02 INFO InputInfoTracker:59 - remove old batch metadata: 1000 ms -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 30 (UnionRDD[47] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=112098, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_30 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(2006) called with curMem=115378, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_30_piece0 stored as bytes in memory (estimated size 2006.0 B, free 730.5 MB) -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Added broadcast_30_piece0 in memory on localhost:51134 (size: 2006.0 B, free: 730.5 MB) -2016-06-13 20:23:02 INFO SparkContext:59 - Created broadcast 30 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 30 (UnionRDD[47] at apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Adding task set 30.0 with 3 tasks -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 30.0 (TID 67, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 30.0 (TID 68, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 0.0 in stage 30.0 (TID 67) -2016-06-13 20:23:02 INFO Executor:59 - Running task 1.0 in stage 30.0 (TID 68) -2016-06-13 20:23:02 INFO Executor:59 - Finished task 0.0 in stage 30.0 (TID 67). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO Executor:59 - Finished task 1.0 in stage 30.0 (TID 68). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 2.0 in stage 30.0 (TID 69, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 2.0 in stage 30.0 (TID 69) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 30.0 (TID 67) in 21 ms on localhost (1/3) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 30.0 (TID 68) in 20 ms on localhost (2/3) -2016-06-13 20:23:02 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:23:02 INFO Executor:59 - Finished task 2.0 in stage 30.0 (TID 69). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO DAGScheduler:59 - ShuffleMapStage 30 (apply at Transformer.scala:22) finished in 0.031 s -2016-06-13 20:23:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:23:02 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:23:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 31) -2016-06-13 20:23:02 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:23:02 INFO DAGScheduler:59 - Missing parents for ResultStage 31: List() -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting ResultStage 31 (MapPartitionsRDD[83] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=117384, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_31 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 2.0 in stage 30.0 (TID 69) in 11 ms on localhost (3/3) -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(1802) called with curMem=120624, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_31_piece0 stored as bytes in memory (estimated size 1802.0 B, free 730.5 MB) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 30.0, whose tasks have all completed, from pool -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Added broadcast_31_piece0 in memory on localhost:51134 (size: 1802.0 B, free: 730.5 MB) -2016-06-13 20:23:02 INFO SparkContext:59 - Created broadcast 31 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 31 (MapPartitionsRDD[83] at apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Adding task set 31.0 with 2 tasks -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 31.0 (TID 70, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 31.0 (TID 71, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 0.0 in stage 31.0 (TID 70) -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:23:02 INFO Executor:59 - Running task 1.0 in stage 31.0 (TID 71) -2016-06-13 20:23:02 INFO Executor:59 - Finished task 0.0 in stage 31.0 (TID 70). 1180 bytes result sent to driver -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:23:02 INFO Executor:59 - Finished task 1.0 in stage 31.0 (TID 71). 1172 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 31.0 (TID 71) in 21 ms on localhost (1/2) -2016-06-13 20:23:02 INFO StreamingKMeansSuite:59 - output.size = 7, numExpectedOutput = 10 -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 31.0 (TID 70) in 37 ms on localhost (2/2) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 31.0, whose tasks have all completed, from pool -2016-06-13 20:23:02 INFO DAGScheduler:59 - ResultStage 31 (apply at Transformer.scala:22) finished in 0.049 s -2016-06-13 20:23:02 INFO DAGScheduler:59 - Job 15 finished: apply at Transformer.scala:22, took 0.108652 s -2016-06-13 20:23:02 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:23:02 INFO JobScheduler:59 - Finished job streaming job 8000 ms.1 from job set of time 8000 ms -2016-06-13 20:23:02 INFO ParallelCollectionRDD:59 - Removing RDD 38 from persistence list -2016-06-13 20:23:02 INFO JobScheduler:59 - Total delay: 1465820574.903 s for time 8000 ms (execution: 0.209 s) -2016-06-13 20:23:02 INFO JobScheduler:59 - Starting job streaming job 9000 ms.0 from job set of time 9000 ms -2016-06-13 20:23:02 INFO MapPartitionsRDD:59 - Removing RDD 43 from persistence list -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 38 -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 43 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Registering RDD 84 (apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO ShuffledRDD:59 - Removing RDD 42 from persistence list -2016-06-13 20:23:02 INFO DAGScheduler:59 - Got job 16 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:23:02 INFO DAGScheduler:59 - Final stage: ResultStage 33(apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 32) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 32) -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 42 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 32 (MapPartitionsRDD[84] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:23:02 INFO UnionRDD:59 - Removing RDD 41 from persistence list -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 41 -2016-06-13 20:23:02 INFO MapPartitionsRDD:59 - Removing RDD 39 from persistence list -2016-06-13 20:23:02 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:23:02 INFO JobGenerator:59 - Checkpointing graph for time 8000 ms -2016-06-13 20:23:02 INFO BlockManager:59 - Removing RDD 39 -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=122426, maxMem=766075207 -2016-06-13 20:23:02 INFO DStreamGraph:59 - Updating checkpoint data for time 8000 ms -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_32 stored as values in memory (estimated size 4.6 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO DStreamGraph:59 - Updated checkpoint data for time 8000 ms -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(2477) called with curMem=127170, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_32_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Added broadcast_32_piece0 in memory on localhost:51134 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 20:23:02 INFO SparkContext:59 - Created broadcast 32 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 32 (MapPartitionsRDD[84] at apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Adding task set 32.0 with 2 tasks -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 32.0 (TID 72, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 32.0 (TID 73, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 1.0 in stage 32.0 (TID 73) -2016-06-13 20:23:02 INFO Executor:59 - Running task 0.0 in stage 32.0 (TID 72) -2016-06-13 20:23:02 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-2000 -2016-06-13 20:23:02 INFO Executor:59 - Finished task 1.0 in stage 32.0 (TID 73). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 32.0 (TID 73) in 15 ms on localhost (1/2) -2016-06-13 20:23:02 INFO Executor:59 - Finished task 0.0 in stage 32.0 (TID 72). 1159 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 32.0 (TID 72) in 18 ms on localhost (2/2) -2016-06-13 20:23:02 INFO DAGScheduler:59 - ShuffleMapStage 32 (apply at Transformer.scala:22) finished in 0.019 s -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 32.0, whose tasks have all completed, from pool -2016-06-13 20:23:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:23:02 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:23:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 33) -2016-06-13 20:23:02 INFO CheckpointWriter:59 - Checkpoint for time 4000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-4000', took 27494 bytes and 167 ms -2016-06-13 20:23:02 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:23:02 INFO DStreamGraph:59 - Clearing checkpoint data for time 4000 ms -2016-06-13 20:23:02 INFO CheckpointWriter:59 - Saving checkpoint for time 5000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-5000' -2016-06-13 20:23:02 INFO DAGScheduler:59 - Missing parents for ResultStage 33: List() -2016-06-13 20:23:02 INFO DStreamGraph:59 - Cleared checkpoint data for time 4000 ms -2016-06-13 20:23:02 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting ResultStage 33 (ShuffledRDD[85] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:23:02 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/receivedBlockMetadata older than 3000: -2016-06-13 20:23:02 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/receivedBlockMetadata older than 3000 -2016-06-13 20:23:02 INFO InputInfoTracker:59 - remove old batch metadata: 2000 ms -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=129647, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_33 stored as values in memory (estimated size 5.1 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(2694) called with curMem=134911, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Added broadcast_33_piece0 in memory on localhost:51134 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 20:23:02 INFO SparkContext:59 - Created broadcast 33 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 33 (ShuffledRDD[85] at apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Adding task set 33.0 with 2 tasks -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 33.0 (TID 74, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 33.0 (TID 75, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 0.0 in stage 33.0 (TID 74) -2016-06-13 20:23:02 INFO Executor:59 - Running task 1.0 in stage 33.0 (TID 75) -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:23:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:23:02 INFO Executor:59 - Finished task 1.0 in stage 33.0 (TID 75). 1161 bytes result sent to driver -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 33.0 (TID 75) in 9 ms on localhost (1/2) -2016-06-13 20:23:02 INFO Executor:59 - Finished task 0.0 in stage 33.0 (TID 74). 1498 bytes result sent to driver -2016-06-13 20:23:02 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:23:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 33.0 (TID 74) in 17 ms on localhost (2/2) -2016-06-13 20:23:02 INFO DAGScheduler:59 - ResultStage 33 (apply at Transformer.scala:22) finished in 0.018 s -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 33.0, whose tasks have all completed, from pool -2016-06-13 20:23:02 INFO DAGScheduler:59 - Job 16 finished: apply at Transformer.scala:22, took 0.067843 s -2016-06-13 20:23:02 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 450.0 and centroid: [-0.4722098164150225,0.9642763266691793,-1.6666072662538036,1.2709270669491914,0.37911973669212223] -2016-06-13 20:23:02 INFO JobScheduler:59 - Finished job streaming job 9000 ms.0 from job set of time 9000 ms -2016-06-13 20:23:02 INFO JobScheduler:59 - Starting job streaming job 9000 ms.1 from job set of time 9000 ms -2016-06-13 20:23:02 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Registering RDD 53 (apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Got job 17 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:23:02 INFO DAGScheduler:59 - Final stage: ResultStage 35(apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 34) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 34) -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 34 (UnionRDD[53] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=137605, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_34 stored as values in memory (estimated size 3.2 KB, free 730.5 MB) -2016-06-13 20:23:02 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=140885, maxMem=766075207 -2016-06-13 20:23:02 INFO MemoryStore:59 - Block broadcast_34_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.4 MB) -2016-06-13 20:23:02 INFO BlockManagerInfo:59 - Added broadcast_34_piece0 in memory on localhost:51134 (size: 2007.0 B, free: 730.5 MB) -2016-06-13 20:23:02 INFO SparkContext:59 - Created broadcast 34 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:02 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 34 (UnionRDD[53] at apply at Transformer.scala:22) -2016-06-13 20:23:02 INFO TaskSchedulerImpl:59 - Adding task set 34.0 with 3 tasks -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 34.0 (TID 76, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:23:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 34.0 (TID 77, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:23:02 INFO Executor:59 - Running task 0.0 in stage 34.0 (TID 76) -2016-06-13 20:23:02 INFO Executor:59 - Running task 1.0 in stage 34.0 (TID 77) -2016-06-13 20:23:03 INFO Executor:59 - Finished task 0.0 in stage 34.0 (TID 76). 1159 bytes result sent to driver -2016-06-13 20:23:03 INFO TaskSetManager:59 - Starting task 2.0 in stage 34.0 (TID 78, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:23:03 INFO Executor:59 - Running task 2.0 in stage 34.0 (TID 78) -2016-06-13 20:23:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 34.0 (TID 76) in 13 ms on localhost (1/3) -2016-06-13 20:23:03 INFO Executor:59 - Finished task 2.0 in stage 34.0 (TID 78). 1159 bytes result sent to driver -2016-06-13 20:23:03 INFO TaskSetManager:59 - Finished task 2.0 in stage 34.0 (TID 78) in 10 ms on localhost (2/3) -2016-06-13 20:23:03 INFO Executor:59 - Finished task 1.0 in stage 34.0 (TID 77). 1159 bytes result sent to driver -2016-06-13 20:23:03 INFO StreamingKMeansSuite:59 - output.size = 8, numExpectedOutput = 10 -2016-06-13 20:23:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 34.0 (TID 77) in 24 ms on localhost (3/3) -2016-06-13 20:23:03 INFO DAGScheduler:59 - ShuffleMapStage 34 (apply at Transformer.scala:22) finished in 0.025 s -2016-06-13 20:23:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 34.0, whose tasks have all completed, from pool -2016-06-13 20:23:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:23:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:23:03 INFO DAGScheduler:59 - waiting: Set(ResultStage 35) -2016-06-13 20:23:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:23:03 INFO DAGScheduler:59 - Missing parents for ResultStage 35: List() -2016-06-13 20:23:03 INFO DAGScheduler:59 - Submitting ResultStage 35 (MapPartitionsRDD[86] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:23:03 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=142892, maxMem=766075207 -2016-06-13 20:23:03 INFO MemoryStore:59 - Block broadcast_35 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:23:03 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=146132, maxMem=766075207 -2016-06-13 20:23:03 INFO MemoryStore:59 - Block broadcast_35_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.4 MB) -2016-06-13 20:23:03 INFO BlockManagerInfo:59 - Added broadcast_35_piece0 in memory on localhost:51134 (size: 1803.0 B, free: 730.5 MB) -2016-06-13 20:23:03 INFO SparkContext:59 - Created broadcast 35 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 35 (MapPartitionsRDD[86] at apply at Transformer.scala:22) -2016-06-13 20:23:03 INFO TaskSchedulerImpl:59 - Adding task set 35.0 with 2 tasks -2016-06-13 20:23:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 35.0 (TID 79, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 35.0 (TID 80, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:03 INFO Executor:59 - Running task 0.0 in stage 35.0 (TID 79) -2016-06-13 20:23:03 INFO Executor:59 - Running task 1.0 in stage 35.0 (TID 80) -2016-06-13 20:23:03 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:23:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:23:03 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:23:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:23:03 INFO Executor:59 - Finished task 1.0 in stage 35.0 (TID 80). 1172 bytes result sent to driver -2016-06-13 20:23:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 35.0 (TID 80) in 9 ms on localhost (1/2) -2016-06-13 20:23:03 INFO Executor:59 - Finished task 0.0 in stage 35.0 (TID 79). 1180 bytes result sent to driver -2016-06-13 20:23:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 35.0 (TID 79) in 14 ms on localhost (2/2) -2016-06-13 20:23:03 INFO DAGScheduler:59 - ResultStage 35 (apply at Transformer.scala:22) finished in 0.015 s -2016-06-13 20:23:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 35.0, whose tasks have all completed, from pool -2016-06-13 20:23:03 INFO DAGScheduler:59 - Job 17 finished: apply at Transformer.scala:22, took 0.069259 s -2016-06-13 20:23:03 INFO JobScheduler:59 - Finished job streaming job 9000 ms.1 from job set of time 9000 ms -2016-06-13 20:23:03 INFO JobScheduler:59 - Total delay: 1465820574.049 s for time 9000 ms (execution: 0.145 s) -2016-06-13 20:23:03 INFO ParallelCollectionRDD:59 - Removing RDD 44 from persistence list -2016-06-13 20:23:03 INFO JobScheduler:59 - Starting job streaming job 10000 ms.0 from job set of time 10000 ms -2016-06-13 20:23:03 INFO BlockManager:59 - Removing RDD 44 -2016-06-13 20:23:03 INFO MapPartitionsRDD:59 - Removing RDD 49 from persistence list -2016-06-13 20:23:03 INFO ShuffledRDD:59 - Removing RDD 48 from persistence list -2016-06-13 20:23:03 INFO BlockManager:59 - Removing RDD 49 -2016-06-13 20:23:03 INFO UnionRDD:59 - Removing RDD 47 from persistence list -2016-06-13 20:23:03 INFO BlockManager:59 - Removing RDD 48 -2016-06-13 20:23:03 INFO BlockManager:59 - Removing RDD 47 -2016-06-13 20:23:03 INFO MapPartitionsRDD:59 - Removing RDD 45 from persistence list -2016-06-13 20:23:03 INFO BlockManager:59 - Removing RDD 45 -2016-06-13 20:23:03 INFO JobGenerator:59 - Checkpointing graph for time 9000 ms -2016-06-13 20:23:03 INFO DStreamGraph:59 - Updating checkpoint data for time 9000 ms -2016-06-13 20:23:03 INFO DStreamGraph:59 - Updated checkpoint data for time 9000 ms -2016-06-13 20:23:03 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:23:03 INFO DAGScheduler:59 - Registering RDD 87 (apply at Transformer.scala:22) -2016-06-13 20:23:03 INFO DAGScheduler:59 - Got job 18 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:23:03 INFO DAGScheduler:59 - Final stage: ResultStage 37(apply at Transformer.scala:22) -2016-06-13 20:23:03 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 36) -2016-06-13 20:23:03 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 36) -2016-06-13 20:23:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 36 (MapPartitionsRDD[87] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:23:03 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:23:03 INFO MemoryStore:59 - ensureFreeSpace(4744) called with curMem=147935, maxMem=766075207 -2016-06-13 20:23:03 INFO MemoryStore:59 - Block broadcast_36 stored as values in memory (estimated size 4.6 KB, free 730.4 MB) -2016-06-13 20:23:03 INFO MemoryStore:59 - ensureFreeSpace(2482) called with curMem=152679, maxMem=766075207 -2016-06-13 20:23:03 INFO MemoryStore:59 - Block broadcast_36_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 20:23:03 INFO BlockManagerInfo:59 - Added broadcast_36_piece0 in memory on localhost:51134 (size: 2.4 KB, free: 730.5 MB) -2016-06-13 20:23:03 INFO SparkContext:59 - Created broadcast 36 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 36 (MapPartitionsRDD[87] at apply at Transformer.scala:22) -2016-06-13 20:23:03 INFO TaskSchedulerImpl:59 - Adding task set 36.0 with 2 tasks -2016-06-13 20:23:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 36.0 (TID 81, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:23:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 36.0 (TID 82, localhost, PROCESS_LOCAL, 3575 bytes) -2016-06-13 20:23:03 INFO Executor:59 - Running task 1.0 in stage 36.0 (TID 82) -2016-06-13 20:23:03 INFO Executor:59 - Running task 0.0 in stage 36.0 (TID 81) -2016-06-13 20:23:03 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-3000.bk -2016-06-13 20:23:03 INFO Executor:59 - Finished task 0.0 in stage 36.0 (TID 81). 1159 bytes result sent to driver -2016-06-13 20:23:03 INFO CheckpointWriter:59 - Checkpoint for time 5000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-5000', took 27490 bytes and 150 ms -2016-06-13 20:23:03 INFO CheckpointWriter:59 - Saving checkpoint for time 6000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-6000' -2016-06-13 20:23:03 INFO Executor:59 - Finished task 1.0 in stage 36.0 (TID 82). 1159 bytes result sent to driver -2016-06-13 20:23:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 36.0 (TID 81) in 15 ms on localhost (1/2) -2016-06-13 20:23:03 INFO DAGScheduler:59 - ShuffleMapStage 36 (apply at Transformer.scala:22) finished in 0.016 s -2016-06-13 20:23:03 INFO DStreamGraph:59 - Clearing checkpoint data for time 5000 ms -2016-06-13 20:23:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:23:03 INFO DStreamGraph:59 - Cleared checkpoint data for time 5000 ms -2016-06-13 20:23:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:23:03 INFO DAGScheduler:59 - waiting: Set(ResultStage 37) -2016-06-13 20:23:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:23:03 INFO DAGScheduler:59 - Missing parents for ResultStage 37: List() -2016-06-13 20:23:03 INFO DAGScheduler:59 - Submitting ResultStage 37 (ShuffledRDD[88] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:23:03 INFO MemoryStore:59 - ensureFreeSpace(5264) called with curMem=155161, maxMem=766075207 -2016-06-13 20:23:03 INFO MemoryStore:59 - Block broadcast_37 stored as values in memory (estimated size 5.1 KB, free 730.4 MB) -2016-06-13 20:23:03 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:23:03 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/receivedBlockMetadata older than 4000: -2016-06-13 20:23:03 INFO InputInfoTracker:59 - remove old batch metadata: 3000 ms -2016-06-13 20:23:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 36.0 (TID 82) in 14 ms on localhost (2/2) -2016-06-13 20:23:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 36.0, whose tasks have all completed, from pool -2016-06-13 20:23:03 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/receivedBlockMetadata older than 4000 -2016-06-13 20:23:03 INFO MemoryStore:59 - ensureFreeSpace(2696) called with curMem=160425, maxMem=766075207 -2016-06-13 20:23:03 INFO MemoryStore:59 - Block broadcast_37_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.4 MB) -2016-06-13 20:23:03 INFO BlockManagerInfo:59 - Added broadcast_37_piece0 in memory on localhost:51134 (size: 2.6 KB, free: 730.5 MB) -2016-06-13 20:23:03 INFO SparkContext:59 - Created broadcast 37 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 37 (ShuffledRDD[88] at apply at Transformer.scala:22) -2016-06-13 20:23:03 INFO TaskSchedulerImpl:59 - Adding task set 37.0 with 2 tasks -2016-06-13 20:23:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 37.0 (TID 83, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 37.0 (TID 84, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:03 INFO Executor:59 - Running task 0.0 in stage 37.0 (TID 83) -2016-06-13 20:23:03 INFO Executor:59 - Running task 1.0 in stage 37.0 (TID 84) -2016-06-13 20:23:03 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 2 blocks -2016-06-13 20:23:03 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 20:23:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:23:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 20:23:03 INFO Executor:59 - Finished task 1.0 in stage 37.0 (TID 84). 1161 bytes result sent to driver -2016-06-13 20:23:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 37.0 (TID 84) in 8 ms on localhost (1/2) -2016-06-13 20:23:03 INFO Executor:59 - Finished task 0.0 in stage 37.0 (TID 83). 1498 bytes result sent to driver -2016-06-13 20:23:03 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:23:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 37.0 (TID 83) in 14 ms on localhost (2/2) -2016-06-13 20:23:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 37.0, whose tasks have all completed, from pool -2016-06-13 20:23:03 INFO DAGScheduler:59 - ResultStage 37 (apply at Transformer.scala:22) finished in 0.017 s -2016-06-13 20:23:03 INFO DAGScheduler:59 - Job 18 finished: apply at Transformer.scala:22, took 0.061006 s -2016-06-13 20:23:03 INFO StreamingKMeansModel:59 - Cluster 0 updated with weight 500.0 and centroid: [-0.4725511979691583,0.9644503899125422,-1.668776373542808,1.2721254429935838,0.37815209739836425] -2016-06-13 20:23:03 INFO SparkContext:59 - Starting job: apply at Transformer.scala:22 -2016-06-13 20:23:03 INFO JobScheduler:59 - Finished job streaming job 10000 ms.0 from job set of time 10000 ms -2016-06-13 20:23:03 INFO JobScheduler:59 - Starting job streaming job 10000 ms.1 from job set of time 10000 ms -2016-06-13 20:23:03 INFO DAGScheduler:59 - Registering RDD 59 (apply at Transformer.scala:22) -2016-06-13 20:23:03 INFO DAGScheduler:59 - Got job 19 (apply at Transformer.scala:22) with 2 output partitions -2016-06-13 20:23:03 INFO DAGScheduler:59 - Final stage: ResultStage 39(apply at Transformer.scala:22) -2016-06-13 20:23:03 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 38) -2016-06-13 20:23:03 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 38) -2016-06-13 20:23:03 INFO DAGScheduler:59 - Submitting ShuffleMapStage 38 (UnionRDD[59] at apply at Transformer.scala:22), which has no missing parents -2016-06-13 20:23:03 INFO MemoryStore:59 - ensureFreeSpace(3280) called with curMem=163121, maxMem=766075207 -2016-06-13 20:23:03 INFO MemoryStore:59 - Block broadcast_38 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:23:03 INFO MemoryStore:59 - ensureFreeSpace(2007) called with curMem=166401, maxMem=766075207 -2016-06-13 20:23:03 INFO MemoryStore:59 - Block broadcast_38_piece0 stored as bytes in memory (estimated size 2007.0 B, free 730.4 MB) -2016-06-13 20:23:03 INFO BlockManagerInfo:59 - Added broadcast_38_piece0 in memory on localhost:51134 (size: 2007.0 B, free: 730.5 MB) -2016-06-13 20:23:03 INFO SparkContext:59 - Created broadcast 38 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:03 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 38 (UnionRDD[59] at apply at Transformer.scala:22) -2016-06-13 20:23:03 INFO TaskSchedulerImpl:59 - Adding task set 38.0 with 3 tasks -2016-06-13 20:23:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 38.0 (TID 85, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:23:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 38.0 (TID 86, localhost, PROCESS_LOCAL, 3684 bytes) -2016-06-13 20:23:03 INFO Executor:59 - Running task 0.0 in stage 38.0 (TID 85) -2016-06-13 20:23:03 INFO Executor:59 - Running task 1.0 in stage 38.0 (TID 86) -2016-06-13 20:23:03 INFO Executor:59 - Finished task 0.0 in stage 38.0 (TID 85). 1159 bytes result sent to driver -2016-06-13 20:23:03 INFO TaskSetManager:59 - Starting task 2.0 in stage 38.0 (TID 87, localhost, PROCESS_LOCAL, 2236 bytes) -2016-06-13 20:23:03 INFO Executor:59 - Running task 2.0 in stage 38.0 (TID 87) -2016-06-13 20:23:03 INFO Executor:59 - Finished task 1.0 in stage 38.0 (TID 86). 1159 bytes result sent to driver -2016-06-13 20:23:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 38.0 (TID 85) in 20 ms on localhost (1/3) -2016-06-13 20:23:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 38.0 (TID 86) in 23 ms on localhost (2/3) -2016-06-13 20:23:03 INFO Executor:59 - Finished task 2.0 in stage 38.0 (TID 87). 1159 bytes result sent to driver -2016-06-13 20:23:03 INFO StreamingKMeansSuite:59 - output.size = 9, numExpectedOutput = 10 -2016-06-13 20:23:03 INFO DAGScheduler:59 - ShuffleMapStage 38 (apply at Transformer.scala:22) finished in 0.038 s -2016-06-13 20:23:03 INFO TaskSetManager:59 - Finished task 2.0 in stage 38.0 (TID 87) in 21 ms on localhost (3/3) -2016-06-13 20:23:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 38.0, whose tasks have all completed, from pool -2016-06-13 20:23:03 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 20:23:03 INFO DAGScheduler:59 - running: Set() -2016-06-13 20:23:03 INFO DAGScheduler:59 - waiting: Set(ResultStage 39) -2016-06-13 20:23:03 INFO DAGScheduler:59 - failed: Set() -2016-06-13 20:23:03 INFO DAGScheduler:59 - Missing parents for ResultStage 39: List() -2016-06-13 20:23:03 INFO DAGScheduler:59 - Submitting ResultStage 39 (MapPartitionsRDD[89] at apply at Transformer.scala:22), which is now runnable -2016-06-13 20:23:03 INFO MemoryStore:59 - ensureFreeSpace(3240) called with curMem=168408, maxMem=766075207 -2016-06-13 20:23:03 INFO MemoryStore:59 - Block broadcast_39 stored as values in memory (estimated size 3.2 KB, free 730.4 MB) -2016-06-13 20:23:03 INFO MemoryStore:59 - ensureFreeSpace(1803) called with curMem=171648, maxMem=766075207 -2016-06-13 20:23:03 INFO MemoryStore:59 - Block broadcast_39_piece0 stored as bytes in memory (estimated size 1803.0 B, free 730.4 MB) -2016-06-13 20:23:03 INFO BlockManagerInfo:59 - Added broadcast_39_piece0 in memory on localhost:51134 (size: 1803.0 B, free: 730.5 MB) -2016-06-13 20:23:03 INFO SparkContext:59 - Created broadcast 39 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:23:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 39 (MapPartitionsRDD[89] at apply at Transformer.scala:22) -2016-06-13 20:23:03 INFO TaskSchedulerImpl:59 - Adding task set 39.0 with 2 tasks -2016-06-13 20:23:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 39.0 (TID 88, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 39.0 (TID 89, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 20:23:03 INFO Executor:59 - Running task 1.0 in stage 39.0 (TID 89) -2016-06-13 20:23:03 INFO Executor:59 - Running task 0.0 in stage 39.0 (TID 88) -2016-06-13 20:23:03 INFO ShuffleBlockFetcherIterator:59 - Getting 3 non-empty blocks out of 3 blocks -2016-06-13 20:23:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:23:03 INFO ShuffleBlockFetcherIterator:59 - Getting 0 non-empty blocks out of 3 blocks -2016-06-13 20:23:03 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 20:23:03 INFO Executor:59 - Finished task 1.0 in stage 39.0 (TID 89). 1172 bytes result sent to driver -2016-06-13 20:23:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 39.0 (TID 89) in 9 ms on localhost (1/2) -2016-06-13 20:23:03 INFO Executor:59 - Finished task 0.0 in stage 39.0 (TID 88). 1180 bytes result sent to driver -2016-06-13 20:23:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 39.0 (TID 88) in 14 ms on localhost (2/2) -2016-06-13 20:23:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 39.0, whose tasks have all completed, from pool -2016-06-13 20:23:03 INFO DAGScheduler:59 - ResultStage 39 (apply at Transformer.scala:22) finished in 0.014 s -2016-06-13 20:23:03 INFO DAGScheduler:59 - Job 19 finished: apply at Transformer.scala:22, took 0.084220 s -2016-06-13 20:23:03 INFO JobScheduler:59 - Finished job streaming job 10000 ms.1 from job set of time 10000 ms -2016-06-13 20:23:03 INFO JobScheduler:59 - Total delay: 1465820573.223 s for time 10000 ms (execution: 0.174 s) -2016-06-13 20:23:03 INFO ParallelCollectionRDD:59 - Removing RDD 50 from persistence list -2016-06-13 20:23:03 INFO BlockManager:59 - Removing RDD 50 -2016-06-13 20:23:03 INFO MapPartitionsRDD:59 - Removing RDD 55 from persistence list -2016-06-13 20:23:03 INFO BlockManager:59 - Removing RDD 55 -2016-06-13 20:23:03 INFO ShuffledRDD:59 - Removing RDD 54 from persistence list -2016-06-13 20:23:03 INFO BlockManager:59 - Removing RDD 54 -2016-06-13 20:23:03 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-3000 -2016-06-13 20:23:03 INFO UnionRDD:59 - Removing RDD 53 from persistence list -2016-06-13 20:23:03 INFO BlockManager:59 - Removing RDD 53 -2016-06-13 20:23:03 INFO MapPartitionsRDD:59 - Removing RDD 51 from persistence list -2016-06-13 20:23:03 INFO CheckpointWriter:59 - Checkpoint for time 6000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-6000', took 27486 bytes and 151 ms -2016-06-13 20:23:03 INFO CheckpointWriter:59 - Saving checkpoint for time 7000 ms to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-7000' -2016-06-13 20:23:03 INFO BlockManager:59 - Removing RDD 51 -2016-06-13 20:23:03 INFO DStreamGraph:59 - Clearing checkpoint data for time 6000 ms -2016-06-13 20:23:03 INFO DStreamGraph:59 - Cleared checkpoint data for time 6000 ms -2016-06-13 20:23:03 INFO ReceivedBlockTracker:59 - Deleting batches ArrayBuffer() -2016-06-13 20:23:03 INFO WriteAheadLogManager :59 - Attempting to clear 0 old log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/receivedBlockMetadata older than 5000: -2016-06-13 20:23:03 INFO InputInfoTracker:59 - remove old batch metadata: 4000 ms -2016-06-13 20:23:03 INFO JobGenerator:59 - Checkpointing graph for time 10000 ms -2016-06-13 20:23:03 INFO DStreamGraph:59 - Updating checkpoint data for time 10000 ms -2016-06-13 20:23:03 INFO DStreamGraph:59 - Updated checkpoint data for time 10000 ms -2016-06-13 20:23:03 INFO WriteAheadLogManager :59 - Cleared log files in file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/receivedBlockMetadata older than 5000 -2016-06-13 20:23:03 INFO StreamingKMeansSuite:59 - Output generated in 3254 milliseconds -2016-06-13 20:23:03 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:23:03 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:23:03 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:23:03 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:23:03 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:23:03 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:23:03 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:23:03 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:23:03 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:23:03 INFO StreamingKMeansSuite:59 - [WrappedArray(50),WrappedArray()] -2016-06-13 20:23:03 INFO JobGenerator:59 - Stopping JobGenerator immediately -2016-06-13 20:23:03 INFO RecurringTimer:59 - Stopped timer for JobGenerator after time 10000 -2016-06-13 20:23:03 INFO CheckpointWriter:59 - Deleting file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-4000.bk -2016-06-13 20:23:13 INFO CheckpointWriter:59 - CheckpointWriter executor terminated ? false, waited for 10002 ms. -2016-06-13 20:23:13 INFO JobGenerator:59 - Stopped JobGenerator -2016-06-13 20:23:13 INFO JobScheduler:59 - Stopped JobScheduler -2016-06-13 20:23:13 INFO CheckpointWriter:59 - Checkpoint for time 7000 ms saved to file 'file:/C:/Users/xubo/AppData/Local/Temp/spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52/checkpoint-7000', took 27484 bytes and 10147 ms -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/batch,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/streaming,null} -2016-06-13 20:23:13 INFO StreamingContext:59 - StreamingContext stopped successfully -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/batch/json,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/streaming/json,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:23:13 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:23:13 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 20:23:13 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:23:13 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:23:13 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:23:13 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:23:13 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:23:13 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:23:13 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:23:13 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:23:13 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:23:13 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 20:23:14 INFO StreamingKMeansSuite:59 - - -===== FINISHED o.a.s.mllib.clustering.kmeans.StreamingKMeansSuite: 'accuracy for single center and equivalence to grand average' ===== - -2016-06-13 20:23:14 WARN StreamingContext:71 - StreamingContext has already been stopped -2016-06-13 20:23:14 INFO SparkContext:59 - SparkContext already stopped. -2016-06-13 20:23:14 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 20:23:14 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52 -2016-06-13 20:23:14 ERROR ShutdownHookManager:96 - Exception while deleting Spark temp dir: C:\Users\xubo\AppData\Local\Temp\spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52 -java.io.IOException: Failed to delete: C:\Users\xubo\AppData\Local\Temp\spark-6e044e32-3ff5-4149-ae07-84b27f4a8f52 - at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:884) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:63) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(ShutdownHookManager.scala:60) - at scala.collection.mutable.HashSet.foreach(HashSet.scala:79) - at org.apache.spark.util.ShutdownHookManager$$anonfun$1.apply$mcV$sp(ShutdownHookManager.scala:60) - at org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:264) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1699) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply$mcV$sp(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:234) - at scala.util.Try$.apply(Try.scala:161) - at org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:234) - at org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:216) - at org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54) -2016-06-13 20:23:14 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-a70c3031-3b3d-4786-9b33-ebf3ce97f394 -2016-06-13 20:36:30 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:36:30 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 20:36:31 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:36:31 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:36:31 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:36:31 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:36:32 INFO Remoting:74 - Starting remoting -2016-06-13 20:36:32 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:51710] -2016-06-13 20:36:32 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51710. -2016-06-13 20:36:32 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:36:32 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:36:32 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-ab52c5bf-2265-495f-9296-d7bd031a1873 -2016-06-13 20:36:32 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:36:32 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-5c7b863d-c9f9-45cd-ae21-1f981b829d48\httpd-72ae1018-bf5b-46c8-a96a-b1259c291c09 -2016-06-13 20:36:32 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:36:32 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:36:32 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51711 -2016-06-13 20:36:32 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51711. -2016-06-13 20:36:32 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:36:32 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:36:32 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 20:36:32 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 20:36:32 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 20:36:32 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:36:32 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:36:33 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51730. -2016-06-13 20:36:33 INFO NettyBlockTransferService:59 - Server created on 51730 -2016-06-13 20:36:33 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:36:33 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51730 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51730) -2016-06-13 20:36:33 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:36:34 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 20:36:34 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 20:36:34 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 20:36:34 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 20:36:34 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51730 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 20:36:34 INFO SparkContext:59 - Created broadcast 0 from textFile at SVDSuite.scala:22 -2016-06-13 20:36:35 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 20:36:35 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 20:36:35 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 20:36:35 INFO DAGScheduler:59 - Got job 0 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 20:36:35 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at RowMatrix.scala:65) -2016-06-13 20:36:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:36:35 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:36:35 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at SVDSuite.scala:25), which has no missing parents -2016-06-13 20:36:35 INFO MemoryStore:59 - ensureFreeSpace(3552) called with curMem=171580, maxMem=766075207 -2016-06-13 20:36:35 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 20:36:35 INFO MemoryStore:59 - ensureFreeSpace(2010) called with curMem=175132, maxMem=766075207 -2016-06-13 20:36:35 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2010.0 B, free 730.4 MB) -2016-06-13 20:36:35 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51730 (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:36:35 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:36:35 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at SVDSuite.scala:25) -2016-06-13 20:36:35 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 20:36:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:36:35 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 20:36:35 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:36:35 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 20:36:35 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 20:36:35 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 20:36:35 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 20:36:35 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 20:36:35 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2183 bytes result sent to driver -2016-06-13 20:36:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 116 ms on localhost (1/1) -2016-06-13 20:36:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 20:36:35 INFO DAGScheduler:59 - ResultStage 0 (first at RowMatrix.scala:65) finished in 0.134 s -2016-06-13 20:36:35 INFO DAGScheduler:59 - Job 0 finished: first at RowMatrix.scala:65, took 0.229522 s -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:36:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:36:35 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 20:36:35 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:36:36 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:36:36 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:36:36 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:36:36 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:36:36 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:36:36 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:36:36 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:36:36 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:36:36 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 20:36:36 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-5c7b863d-c9f9-45cd-ae21-1f981b829d48 -2016-06-13 20:37:11 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:37:12 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 20:37:12 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:37:12 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:37:12 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:37:13 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:37:13 INFO Remoting:74 - Starting remoting -2016-06-13 20:37:13 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:51752] -2016-06-13 20:37:13 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51752. -2016-06-13 20:37:13 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:37:13 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:37:13 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-bedf5fb2-b566-4b02-a47a-0185c90f5505 -2016-06-13 20:37:13 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:37:13 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-b995a16e-ed67-493d-b7ad-f82586a79c29\httpd-62453185-3f4c-4614-ab10-7fdcf853a6d7 -2016-06-13 20:37:13 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:37:13 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:37:13 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51753 -2016-06-13 20:37:13 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51753. -2016-06-13 20:37:13 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:37:14 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:37:14 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 20:37:14 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 20:37:14 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 20:37:14 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:37:14 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:37:14 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51772. -2016-06-13 20:37:14 INFO NettyBlockTransferService:59 - Server created on 51772 -2016-06-13 20:37:14 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:37:14 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51772 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51772) -2016-06-13 20:37:14 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:37:15 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 20:37:15 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 20:37:15 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 20:37:15 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 20:37:15 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51772 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 20:37:15 INFO SparkContext:59 - Created broadcast 0 from textFile at SVDSuite.scala:22 -2016-06-13 20:37:16 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 20:37:16 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 20:37:16 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 20:37:16 INFO DAGScheduler:59 - Got job 0 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 20:37:16 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at RowMatrix.scala:65) -2016-06-13 20:37:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:37:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:37:16 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at SVDSuite.scala:25), which has no missing parents -2016-06-13 20:37:16 INFO MemoryStore:59 - ensureFreeSpace(3552) called with curMem=171580, maxMem=766075207 -2016-06-13 20:37:16 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 20:37:16 INFO MemoryStore:59 - ensureFreeSpace(2010) called with curMem=175132, maxMem=766075207 -2016-06-13 20:37:16 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2010.0 B, free 730.4 MB) -2016-06-13 20:37:16 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51772 (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:37:17 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:37:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at SVDSuite.scala:25) -2016-06-13 20:37:17 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 20:37:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:37:17 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 20:37:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:37:17 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 20:37:17 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 20:37:17 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 20:37:17 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 20:37:17 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 20:37:17 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2183 bytes result sent to driver -2016-06-13 20:37:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 100 ms on localhost (1/1) -2016-06-13 20:37:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 20:37:17 INFO DAGScheduler:59 - ResultStage 0 (first at RowMatrix.scala:65) finished in 0.115 s -2016-06-13 20:37:17 INFO DAGScheduler:59 - Job 0 finished: first at RowMatrix.scala:65, took 0.217445 s -2016-06-13 20:37:17 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-06-13 20:37:17 INFO DAGScheduler:59 - Got job 1 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-06-13 20:37:17 INFO DAGScheduler:59 - Final stage: ResultStage 1(treeAggregate at RowMatrix.scala:124) -2016-06-13 20:37:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:37:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:37:17 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-06-13 20:37:17 INFO MemoryStore:59 - ensureFreeSpace(4512) called with curMem=177142, maxMem=766075207 -2016-06-13 20:37:17 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 20:37:17 INFO MemoryStore:59 - ensureFreeSpace(2369) called with curMem=181654, maxMem=766075207 -2016-06-13 20:37:17 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 20:37:17 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51772 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 20:37:17 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:37:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:124) -2016-06-13 20:37:17 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 20:37:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:37:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:37:17 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 20:37:17 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 20:37:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:37:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 20:37:17 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 20:37:17 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 20:37:17 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2265 bytes result sent to driver -2016-06-13 20:37:17 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2265 bytes result sent to driver -2016-06-13 20:37:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 45 ms on localhost (1/2) -2016-06-13 20:37:17 INFO DAGScheduler:59 - ResultStage 1 (treeAggregate at RowMatrix.scala:124) finished in 0.050 s -2016-06-13 20:37:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 48 ms on localhost (2/2) -2016-06-13 20:37:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 20:37:17 INFO DAGScheduler:59 - Job 1 finished: treeAggregate at RowMatrix.scala:124, took 0.075392 s -2016-06-13 20:37:17 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:51772 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 20:37:17 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 20:37:17 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:51772 in memory (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:37:17 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 20:37:18 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 20:37:18 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 20:37:18 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=171580, maxMem=766075207 -2016-06-13 20:37:18 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 112.0 B, free 730.4 MB) -2016-06-13 20:37:18 INFO MemoryStore:59 - ensureFreeSpace(122) called with curMem=171692, maxMem=766075207 -2016-06-13 20:37:18 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 122.0 B, free 730.4 MB) -2016-06-13 20:37:18 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:51772 (size: 122.0 B, free: 730.6 MB) -2016-06-13 20:37:18 INFO SparkContext:59 - Created broadcast 3 from broadcast at RowMatrix.scala:425 -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:37:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:37:18 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 20:37:18 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:37:18 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:37:18 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:37:18 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:37:18 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:37:18 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:37:18 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:37:18 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 20:37:18 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-b995a16e-ed67-493d-b7ad-f82586a79c29 -2016-06-13 20:37:18 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:37:18 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:39:11 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:39:12 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 20:39:12 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:39:12 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:39:12 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:39:13 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:39:13 INFO Remoting:74 - Starting remoting -2016-06-13 20:39:13 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:51797] -2016-06-13 20:39:13 INFO Utils:59 - Successfully started service 'sparkDriver' on port 51797. -2016-06-13 20:39:13 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:39:13 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:39:13 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-0b6bdeb5-3f66-46a7-a78c-73106a6b50c3 -2016-06-13 20:39:13 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:39:13 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-9bc7bd6d-361b-4819-90f3-a7be0a5d5b76\httpd-0c1192f6-0aaf-44d5-b57b-09aecdd09d28 -2016-06-13 20:39:13 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:39:13 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:39:13 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:51798 -2016-06-13 20:39:13 INFO Utils:59 - Successfully started service 'HTTP file server' on port 51798. -2016-06-13 20:39:13 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:39:14 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:39:14 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 20:39:14 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 20:39:14 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 20:39:14 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:39:14 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:39:14 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 51817. -2016-06-13 20:39:14 INFO NettyBlockTransferService:59 - Server created on 51817 -2016-06-13 20:39:14 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:39:14 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:51817 with 730.6 MB RAM, BlockManagerId(driver, localhost, 51817) -2016-06-13 20:39:14 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:39:15 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 20:39:15 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 20:39:15 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 20:39:15 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 20:39:15 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:51817 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 20:39:15 INFO SparkContext:59 - Created broadcast 0 from textFile at SVDSuite.scala:22 -2016-06-13 20:39:16 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 20:39:16 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 20:39:16 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 20:39:16 INFO DAGScheduler:59 - Got job 0 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 20:39:16 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at RowMatrix.scala:65) -2016-06-13 20:39:16 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:39:16 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:39:16 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at SVDSuite.scala:25), which has no missing parents -2016-06-13 20:39:16 INFO MemoryStore:59 - ensureFreeSpace(3552) called with curMem=171580, maxMem=766075207 -2016-06-13 20:39:16 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 20:39:16 INFO MemoryStore:59 - ensureFreeSpace(2010) called with curMem=175132, maxMem=766075207 -2016-06-13 20:39:16 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2010.0 B, free 730.4 MB) -2016-06-13 20:39:16 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:51817 (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:39:16 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:39:16 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at SVDSuite.scala:25) -2016-06-13 20:39:16 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 20:39:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:39:17 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 20:39:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:39:17 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 20:39:17 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 20:39:17 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 20:39:17 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 20:39:17 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 20:39:17 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2183 bytes result sent to driver -2016-06-13 20:39:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 106 ms on localhost (1/1) -2016-06-13 20:39:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 20:39:17 INFO DAGScheduler:59 - ResultStage 0 (first at RowMatrix.scala:65) finished in 0.125 s -2016-06-13 20:39:17 INFO DAGScheduler:59 - Job 0 finished: first at RowMatrix.scala:65, took 0.224281 s -2016-06-13 20:39:17 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-06-13 20:39:17 INFO DAGScheduler:59 - Got job 1 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-06-13 20:39:17 INFO DAGScheduler:59 - Final stage: ResultStage 1(treeAggregate at RowMatrix.scala:124) -2016-06-13 20:39:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:39:17 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:39:17 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-06-13 20:39:17 INFO MemoryStore:59 - ensureFreeSpace(4512) called with curMem=177142, maxMem=766075207 -2016-06-13 20:39:17 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 20:39:17 INFO MemoryStore:59 - ensureFreeSpace(2369) called with curMem=181654, maxMem=766075207 -2016-06-13 20:39:17 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 20:39:17 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:51817 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 20:39:17 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:39:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:124) -2016-06-13 20:39:17 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 20:39:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:39:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:39:17 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 20:39:17 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 20:39:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:39:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 20:39:17 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 20:39:17 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 20:39:17 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2265 bytes result sent to driver -2016-06-13 20:39:17 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2265 bytes result sent to driver -2016-06-13 20:39:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 59 ms on localhost (1/2) -2016-06-13 20:39:17 INFO DAGScheduler:59 - ResultStage 1 (treeAggregate at RowMatrix.scala:124) finished in 0.059 s -2016-06-13 20:39:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 57 ms on localhost (2/2) -2016-06-13 20:39:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 20:39:17 INFO DAGScheduler:59 - Job 1 finished: treeAggregate at RowMatrix.scala:124, took 0.080096 s -2016-06-13 20:39:17 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 20:39:17 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 20:39:17 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=184023, maxMem=766075207 -2016-06-13 20:39:17 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 112.0 B, free 730.4 MB) -2016-06-13 20:39:17 INFO MemoryStore:59 - ensureFreeSpace(122) called with curMem=184135, maxMem=766075207 -2016-06-13 20:39:17 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 122.0 B, free 730.4 MB) -2016-06-13 20:39:17 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:51817 (size: 122.0 B, free: 730.6 MB) -2016-06-13 20:39:17 INFO SparkContext:59 - Created broadcast 3 from broadcast at RowMatrix.scala:425 -2016-06-13 20:39:18 INFO SparkContext:59 - Starting job: foreach at SVDSuite.scala:36 -2016-06-13 20:39:18 INFO DAGScheduler:59 - Got job 2 (foreach at SVDSuite.scala:36) with 2 output partitions -2016-06-13 20:39:18 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at SVDSuite.scala:36) -2016-06-13 20:39:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:39:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:39:18 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[3] at map at SVDSuite.scala:25), which has no missing parents -2016-06-13 20:39:18 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=184257, maxMem=766075207 -2016-06-13 20:39:18 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-06-13 20:39:18 INFO MemoryStore:59 - ensureFreeSpace(1994) called with curMem=187785, maxMem=766075207 -2016-06-13 20:39:18 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 1994.0 B, free 730.4 MB) -2016-06-13 20:39:18 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:51817 (size: 1994.0 B, free: 730.6 MB) -2016-06-13 20:39:18 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:39:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[3] at map at SVDSuite.scala:25) -2016-06-13 20:39:18 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 20:39:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:39:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:39:18 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 20:39:18 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-06-13 20:39:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 20:39:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:39:18 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2044 bytes result sent to driver -2016-06-13 20:39:18 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2044 bytes result sent to driver -2016-06-13 20:39:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 22 ms on localhost (1/2) -2016-06-13 20:39:18 INFO DAGScheduler:59 - ResultStage 2 (foreach at SVDSuite.scala:36) finished in 0.024 s -2016-06-13 20:39:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 23 ms on localhost (2/2) -2016-06-13 20:39:18 INFO DAGScheduler:59 - Job 2 finished: foreach at SVDSuite.scala:36, took 0.044425 s -2016-06-13 20:39:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 20:39:18 INFO SparkContext:59 - Starting job: foreach at SVDSuite.scala:38 -2016-06-13 20:39:18 INFO DAGScheduler:59 - Got job 3 (foreach at SVDSuite.scala:38) with 2 output partitions -2016-06-13 20:39:18 INFO DAGScheduler:59 - Final stage: ResultStage 3(foreach at SVDSuite.scala:38) -2016-06-13 20:39:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:39:18 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:39:18 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[5] at mapPartitions at RowMatrix.scala:426), which has no missing parents -2016-06-13 20:39:18 INFO MemoryStore:59 - ensureFreeSpace(3856) called with curMem=189779, maxMem=766075207 -2016-06-13 20:39:18 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 20:39:18 INFO MemoryStore:59 - ensureFreeSpace(2128) called with curMem=193635, maxMem=766075207 -2016-06-13 20:39:18 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 20:39:18 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:51817 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 20:39:18 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:39:18 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[5] at mapPartitions at RowMatrix.scala:426) -2016-06-13 20:39:18 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 20:39:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:39:18 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:39:18 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-06-13 20:39:18 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-06-13 20:39:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 20:39:18 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:39:18 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2044 bytes result sent to driver -2016-06-13 20:39:18 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2044 bytes result sent to driver -2016-06-13 20:39:18 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 28 ms on localhost (1/2) -2016-06-13 20:39:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 31 ms on localhost (2/2) -2016-06-13 20:39:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 20:39:18 INFO DAGScheduler:59 - ResultStage 3 (foreach at SVDSuite.scala:38) finished in 0.032 s -2016-06-13 20:39:18 INFO DAGScheduler:59 - Job 3 finished: foreach at SVDSuite.scala:38, took 0.051268 s -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:39:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:39:18 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 20:39:18 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:39:18 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:39:18 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:39:18 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:39:18 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:39:18 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:39:18 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:39:18 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:39:18 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:39:18 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 20:39:18 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-9bc7bd6d-361b-4819-90f3-a7be0a5d5b76 -2016-06-13 20:48:53 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:48:53 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 20:48:53 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:48:53 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:48:53 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:48:54 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:48:54 INFO Remoting:74 - Starting remoting -2016-06-13 20:48:55 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:52143] -2016-06-13 20:48:55 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52143. -2016-06-13 20:48:55 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:48:55 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:48:55 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-66f901c0-cdda-4a96-ba22-0631f0c30deb -2016-06-13 20:48:55 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:48:55 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-ead3d5e5-68d7-4ba3-9201-b456948ac047\httpd-8b30b19a-cee6-43d7-9fea-ccbd3603a7fb -2016-06-13 20:48:55 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:48:55 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:48:55 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52144 -2016-06-13 20:48:55 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52144. -2016-06-13 20:48:55 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:48:55 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:48:55 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 20:48:55 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 20:48:55 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 20:48:55 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:48:55 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:48:55 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52163. -2016-06-13 20:48:55 INFO NettyBlockTransferService:59 - Server created on 52163 -2016-06-13 20:48:55 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:48:55 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52163 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52163) -2016-06-13 20:48:55 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:48:56 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 20:48:56 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 20:48:56 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 20:48:56 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 20:48:56 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52163 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 20:48:56 INFO SparkContext:59 - Created broadcast 0 from textFile at PCASuite.scala:21 -2016-06-13 20:48:57 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 20:48:57 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 20:48:58 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 20:48:58 INFO DAGScheduler:59 - Got job 0 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 20:48:58 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at RowMatrix.scala:65) -2016-06-13 20:48:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:48:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:48:58 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at PCASuite.scala:24), which has no missing parents -2016-06-13 20:48:58 INFO MemoryStore:59 - ensureFreeSpace(3552) called with curMem=171580, maxMem=766075207 -2016-06-13 20:48:58 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 20:48:58 INFO MemoryStore:59 - ensureFreeSpace(2010) called with curMem=175132, maxMem=766075207 -2016-06-13 20:48:58 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2010.0 B, free 730.4 MB) -2016-06-13 20:48:58 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52163 (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:48:58 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:48:58 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at PCASuite.scala:24) -2016-06-13 20:48:58 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 20:48:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:48:58 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 20:48:58 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:48:58 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 20:48:58 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 20:48:58 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 20:48:58 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 20:48:58 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 20:48:58 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2183 bytes result sent to driver -2016-06-13 20:48:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 92 ms on localhost (1/1) -2016-06-13 20:48:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 20:48:58 INFO DAGScheduler:59 - ResultStage 0 (first at RowMatrix.scala:65) finished in 0.107 s -2016-06-13 20:48:58 INFO DAGScheduler:59 - Job 0 finished: first at RowMatrix.scala:65, took 0.184256 s -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:48:58 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:48:58 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 20:48:58 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:48:58 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:48:58 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:48:58 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:48:58 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:48:58 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:48:58 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:48:58 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:48:58 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:48:58 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 20:48:58 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-ead3d5e5-68d7-4ba3-9201-b456948ac047 -2016-06-13 20:49:05 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:49:06 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 20:49:06 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:49:06 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:49:06 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:49:07 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:49:07 INFO Remoting:74 - Starting remoting -2016-06-13 20:49:07 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:52183] -2016-06-13 20:49:07 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52183. -2016-06-13 20:49:07 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:49:07 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:49:07 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-efedd448-6eed-45e1-9aba-0bd831be4dce -2016-06-13 20:49:07 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:49:07 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-53c7cd2c-0552-453d-a740-5b14da008ccf\httpd-686dc31e-dcc7-4bb9-bded-54f8b2c1d4e7 -2016-06-13 20:49:07 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:49:07 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:49:07 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52184 -2016-06-13 20:49:07 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52184. -2016-06-13 20:49:07 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:49:07 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:49:07 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 20:49:07 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 20:49:07 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 20:49:07 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:49:07 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:49:08 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52203. -2016-06-13 20:49:08 INFO NettyBlockTransferService:59 - Server created on 52203 -2016-06-13 20:49:08 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:49:08 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52203 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52203) -2016-06-13 20:49:08 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:49:09 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 20:49:09 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 20:49:09 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 20:49:09 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 20:49:09 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52203 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 20:49:09 INFO SparkContext:59 - Created broadcast 0 from textFile at PCASuite.scala:21 -2016-06-13 20:49:10 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 20:49:10 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 20:49:10 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 20:49:10 INFO DAGScheduler:59 - Got job 0 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 20:49:10 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at RowMatrix.scala:65) -2016-06-13 20:49:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:49:10 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:49:10 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at PCASuite.scala:24), which has no missing parents -2016-06-13 20:49:10 INFO MemoryStore:59 - ensureFreeSpace(3552) called with curMem=171580, maxMem=766075207 -2016-06-13 20:49:10 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 20:49:10 INFO MemoryStore:59 - ensureFreeSpace(2010) called with curMem=175132, maxMem=766075207 -2016-06-13 20:49:10 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2010.0 B, free 730.4 MB) -2016-06-13 20:49:10 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52203 (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:49:10 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:49:10 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at PCASuite.scala:24) -2016-06-13 20:49:10 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 20:49:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:49:10 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 20:49:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:49:10 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 20:49:10 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 20:49:10 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 20:49:10 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 20:49:10 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 20:49:10 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2183 bytes result sent to driver -2016-06-13 20:49:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 96 ms on localhost (1/1) -2016-06-13 20:49:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 20:49:10 INFO DAGScheduler:59 - ResultStage 0 (first at RowMatrix.scala:65) finished in 0.111 s -2016-06-13 20:49:10 INFO DAGScheduler:59 - Job 0 finished: first at RowMatrix.scala:65, took 0.233574 s -2016-06-13 20:49:11 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-06-13 20:49:11 INFO DAGScheduler:59 - Got job 1 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-06-13 20:49:11 INFO DAGScheduler:59 - Final stage: ResultStage 1(treeAggregate at RowMatrix.scala:332) -2016-06-13 20:49:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:49:11 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:49:11 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-06-13 20:49:11 INFO MemoryStore:59 - ensureFreeSpace(4456) called with curMem=177142, maxMem=766075207 -2016-06-13 20:49:11 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 20:49:11 INFO MemoryStore:59 - ensureFreeSpace(2380) called with curMem=181598, maxMem=766075207 -2016-06-13 20:49:11 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 20:49:11 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52203 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 20:49:11 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:49:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:332) -2016-06-13 20:49:11 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 20:49:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:49:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:49:11 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 20:49:11 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 20:49:11 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:49:11 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 20:49:11 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2368 bytes result sent to driver -2016-06-13 20:49:11 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2368 bytes result sent to driver -2016-06-13 20:49:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 54 ms on localhost (1/2) -2016-06-13 20:49:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 58 ms on localhost (2/2) -2016-06-13 20:49:11 INFO DAGScheduler:59 - ResultStage 1 (treeAggregate at RowMatrix.scala:332) finished in 0.058 s -2016-06-13 20:49:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 20:49:11 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 20:49:11 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 20:49:11 INFO DAGScheduler:59 - Job 1 finished: treeAggregate at RowMatrix.scala:332, took 0.091072 s -2016-06-13 20:49:11 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-06-13 20:49:11 INFO DAGScheduler:59 - Got job 2 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-06-13 20:49:11 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeAggregate at RowMatrix.scala:124) -2016-06-13 20:49:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:49:11 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:49:11 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[5] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-06-13 20:49:11 INFO MemoryStore:59 - ensureFreeSpace(4512) called with curMem=183978, maxMem=766075207 -2016-06-13 20:49:11 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 20:49:11 INFO MemoryStore:59 - ensureFreeSpace(2368) called with curMem=188490, maxMem=766075207 -2016-06-13 20:49:11 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 20:49:11 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52203 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 20:49:11 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:49:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[5] at treeAggregate at RowMatrix.scala:124) -2016-06-13 20:49:11 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 20:49:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:49:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:49:11 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 20:49:11 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-06-13 20:49:11 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52203 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 20:49:11 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 20:49:11 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:49:11 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52203 in memory (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:49:11 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2265 bytes result sent to driver -2016-06-13 20:49:11 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2265 bytes result sent to driver -2016-06-13 20:49:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 27 ms on localhost (1/2) -2016-06-13 20:49:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 27 ms on localhost (2/2) -2016-06-13 20:49:11 INFO DAGScheduler:59 - ResultStage 2 (treeAggregate at RowMatrix.scala:124) finished in 0.029 s -2016-06-13 20:49:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 20:49:11 INFO DAGScheduler:59 - Job 2 finished: treeAggregate at RowMatrix.scala:124, took 0.094021 s -2016-06-13 20:49:11 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 20:49:11 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 20:49:11 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=178460, maxMem=766075207 -2016-06-13 20:49:11 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 112.0 B, free 730.4 MB) -2016-06-13 20:49:11 INFO MemoryStore:59 - ensureFreeSpace(101) called with curMem=178572, maxMem=766075207 -2016-06-13 20:49:11 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 101.0 B, free 730.4 MB) -2016-06-13 20:49:11 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52203 (size: 101.0 B, free: 730.6 MB) -2016-06-13 20:49:11 INFO SparkContext:59 - Created broadcast 4 from broadcast at RowMatrix.scala:425 -2016-06-13 20:49:11 INFO SparkContext:59 - Starting job: foreach at PCASuite.scala:35 -2016-06-13 20:49:11 INFO DAGScheduler:59 - Got job 3 (foreach at PCASuite.scala:35) with 2 output partitions -2016-06-13 20:49:11 INFO DAGScheduler:59 - Final stage: ResultStage 3(foreach at PCASuite.scala:35) -2016-06-13 20:49:11 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:49:11 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:49:11 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[6] at mapPartitions at RowMatrix.scala:426), which has no missing parents -2016-06-13 20:49:11 INFO MemoryStore:59 - ensureFreeSpace(3856) called with curMem=178673, maxMem=766075207 -2016-06-13 20:49:11 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 20:49:11 INFO MemoryStore:59 - ensureFreeSpace(2130) called with curMem=182529, maxMem=766075207 -2016-06-13 20:49:11 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 20:49:11 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52203 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 20:49:11 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:49:11 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[6] at mapPartitions at RowMatrix.scala:426) -2016-06-13 20:49:11 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 20:49:11 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:49:11 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:49:11 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-06-13 20:49:11 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-06-13 20:49:11 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:49:11 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 20:49:11 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2044 bytes result sent to driver -2016-06-13 20:49:11 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2044 bytes result sent to driver -2016-06-13 20:49:11 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 25 ms on localhost (1/2) -2016-06-13 20:49:11 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 28 ms on localhost (2/2) -2016-06-13 20:49:11 INFO DAGScheduler:59 - ResultStage 3 (foreach at PCASuite.scala:35) finished in 0.028 s -2016-06-13 20:49:11 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 20:49:11 INFO DAGScheduler:59 - Job 3 finished: foreach at PCASuite.scala:35, took 0.043273 s -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:49:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:49:11 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 20:49:11 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:49:11 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:49:11 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:49:11 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:49:11 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:49:11 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:49:11 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:49:11 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 20:49:11 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-53c7cd2c-0552-453d-a740-5b14da008ccf -2016-06-13 20:49:46 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:49:46 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 20:49:46 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:49:46 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:49:46 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:49:47 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:49:47 INFO Remoting:74 - Starting remoting -2016-06-13 20:49:47 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:52227] -2016-06-13 20:49:47 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52227. -2016-06-13 20:49:47 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:49:47 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:49:48 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-0441914f-c015-43c3-a7d1-009eb4734834 -2016-06-13 20:49:48 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:49:48 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-98d80751-bbb3-4450-a62d-047747cb2f8f\httpd-af8d2845-3d05-4fb2-8404-03d2785483fb -2016-06-13 20:49:48 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:49:48 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:49:48 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52228 -2016-06-13 20:49:48 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52228. -2016-06-13 20:49:48 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:49:48 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:49:48 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 20:49:48 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 20:49:48 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 20:49:48 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:49:48 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:49:48 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52247. -2016-06-13 20:49:48 INFO NettyBlockTransferService:59 - Server created on 52247 -2016-06-13 20:49:48 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:49:48 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52247 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52247) -2016-06-13 20:49:48 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:49:49 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 20:49:49 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 20:49:49 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 20:49:49 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 20:49:49 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52247 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 20:49:49 INFO SparkContext:59 - Created broadcast 0 from textFile at PCASuite.scala:21 -2016-06-13 20:49:50 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 20:49:50 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 20:49:51 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 20:49:51 INFO DAGScheduler:59 - Got job 0 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 20:49:51 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at RowMatrix.scala:65) -2016-06-13 20:49:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:49:51 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:49:51 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at PCASuite.scala:24), which has no missing parents -2016-06-13 20:49:51 INFO MemoryStore:59 - ensureFreeSpace(3552) called with curMem=171580, maxMem=766075207 -2016-06-13 20:49:51 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 20:49:51 INFO MemoryStore:59 - ensureFreeSpace(2010) called with curMem=175132, maxMem=766075207 -2016-06-13 20:49:51 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2010.0 B, free 730.4 MB) -2016-06-13 20:49:51 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52247 (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:49:51 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:49:51 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at PCASuite.scala:24) -2016-06-13 20:49:51 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 20:49:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:49:51 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 20:49:51 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:49:51 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 20:49:51 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 20:49:51 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 20:49:51 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 20:49:51 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 20:49:51 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2183 bytes result sent to driver -2016-06-13 20:49:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 118 ms on localhost (1/1) -2016-06-13 20:49:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 20:49:51 INFO DAGScheduler:59 - ResultStage 0 (first at RowMatrix.scala:65) finished in 0.133 s -2016-06-13 20:49:51 INFO DAGScheduler:59 - Job 0 finished: first at RowMatrix.scala:65, took 0.237730 s -2016-06-13 20:49:51 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-06-13 20:49:51 INFO DAGScheduler:59 - Got job 1 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-06-13 20:49:51 INFO DAGScheduler:59 - Final stage: ResultStage 1(treeAggregate at RowMatrix.scala:332) -2016-06-13 20:49:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:49:51 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:49:51 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-06-13 20:49:51 INFO MemoryStore:59 - ensureFreeSpace(4456) called with curMem=177142, maxMem=766075207 -2016-06-13 20:49:51 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 20:49:51 INFO MemoryStore:59 - ensureFreeSpace(2380) called with curMem=181598, maxMem=766075207 -2016-06-13 20:49:51 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 20:49:51 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52247 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 20:49:51 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:49:51 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:332) -2016-06-13 20:49:51 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 20:49:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:49:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:49:51 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 20:49:51 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 20:49:51 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:49:51 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 20:49:51 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2368 bytes result sent to driver -2016-06-13 20:49:51 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2368 bytes result sent to driver -2016-06-13 20:49:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 44 ms on localhost (1/2) -2016-06-13 20:49:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 43 ms on localhost (2/2) -2016-06-13 20:49:51 INFO DAGScheduler:59 - ResultStage 1 (treeAggregate at RowMatrix.scala:332) finished in 0.046 s -2016-06-13 20:49:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 20:49:51 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 20:49:51 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 20:49:51 INFO DAGScheduler:59 - Job 1 finished: treeAggregate at RowMatrix.scala:332, took 0.082976 s -2016-06-13 20:49:51 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-06-13 20:49:51 INFO DAGScheduler:59 - Got job 2 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-06-13 20:49:51 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeAggregate at RowMatrix.scala:124) -2016-06-13 20:49:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:49:51 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:49:51 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[5] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-06-13 20:49:51 INFO MemoryStore:59 - ensureFreeSpace(4512) called with curMem=183978, maxMem=766075207 -2016-06-13 20:49:51 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 20:49:51 INFO MemoryStore:59 - ensureFreeSpace(2368) called with curMem=188490, maxMem=766075207 -2016-06-13 20:49:51 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 20:49:51 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52247 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 20:49:51 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:49:51 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[5] at treeAggregate at RowMatrix.scala:124) -2016-06-13 20:49:51 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 20:49:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:49:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:49:51 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 20:49:51 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-06-13 20:49:51 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 20:49:51 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:49:51 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2265 bytes result sent to driver -2016-06-13 20:49:51 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2265 bytes result sent to driver -2016-06-13 20:49:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 66 ms on localhost (1/2) -2016-06-13 20:49:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 66 ms on localhost (2/2) -2016-06-13 20:49:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 20:49:51 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52247 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 20:49:51 INFO DAGScheduler:59 - ResultStage 2 (treeAggregate at RowMatrix.scala:124) finished in 0.074 s -2016-06-13 20:49:51 INFO DAGScheduler:59 - Job 2 finished: treeAggregate at RowMatrix.scala:124, took 0.104146 s -2016-06-13 20:49:51 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 20:49:51 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52247 in memory (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:49:51 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 20:49:52 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 20:49:52 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 20:49:52 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=178460, maxMem=766075207 -2016-06-13 20:49:52 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 112.0 B, free 730.4 MB) -2016-06-13 20:49:52 INFO MemoryStore:59 - ensureFreeSpace(101) called with curMem=178572, maxMem=766075207 -2016-06-13 20:49:52 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 101.0 B, free 730.4 MB) -2016-06-13 20:49:52 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52247 (size: 101.0 B, free: 730.6 MB) -2016-06-13 20:49:52 INFO SparkContext:59 - Created broadcast 4 from broadcast at RowMatrix.scala:425 -2016-06-13 20:49:52 INFO SparkContext:59 - Starting job: foreach at PCASuite.scala:34 -2016-06-13 20:49:52 INFO DAGScheduler:59 - Got job 3 (foreach at PCASuite.scala:34) with 2 output partitions -2016-06-13 20:49:52 INFO DAGScheduler:59 - Final stage: ResultStage 3(foreach at PCASuite.scala:34) -2016-06-13 20:49:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:49:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:49:52 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[3] at map at PCASuite.scala:24), which has no missing parents -2016-06-13 20:49:52 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=178673, maxMem=766075207 -2016-06-13 20:49:52 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-06-13 20:49:52 INFO MemoryStore:59 - ensureFreeSpace(1994) called with curMem=182201, maxMem=766075207 -2016-06-13 20:49:52 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 1994.0 B, free 730.4 MB) -2016-06-13 20:49:52 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52247 (size: 1994.0 B, free: 730.6 MB) -2016-06-13 20:49:52 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:49:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[3] at map at PCASuite.scala:24) -2016-06-13 20:49:52 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 20:49:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:49:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:49:52 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-06-13 20:49:52 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-06-13 20:49:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:49:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 20:49:52 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2044 bytes result sent to driver -2016-06-13 20:49:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 30 ms on localhost (1/2) -2016-06-13 20:49:52 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2044 bytes result sent to driver -2016-06-13 20:49:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 33 ms on localhost (2/2) -2016-06-13 20:49:52 INFO DAGScheduler:59 - ResultStage 3 (foreach at PCASuite.scala:34) finished in 0.035 s -2016-06-13 20:49:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 20:49:52 INFO DAGScheduler:59 - Job 3 finished: foreach at PCASuite.scala:34, took 0.058061 s -2016-06-13 20:49:52 INFO SparkContext:59 - Starting job: foreach at PCASuite.scala:38 -2016-06-13 20:49:52 INFO DAGScheduler:59 - Got job 4 (foreach at PCASuite.scala:38) with 2 output partitions -2016-06-13 20:49:52 INFO DAGScheduler:59 - Final stage: ResultStage 4(foreach at PCASuite.scala:38) -2016-06-13 20:49:52 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:49:52 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:49:52 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[6] at mapPartitions at RowMatrix.scala:426), which has no missing parents -2016-06-13 20:49:52 INFO MemoryStore:59 - ensureFreeSpace(3856) called with curMem=184195, maxMem=766075207 -2016-06-13 20:49:52 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 20:49:52 INFO MemoryStore:59 - ensureFreeSpace(2130) called with curMem=188051, maxMem=766075207 -2016-06-13 20:49:52 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 20:49:52 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52247 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 20:49:52 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:49:52 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[6] at mapPartitions at RowMatrix.scala:426) -2016-06-13 20:49:52 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 20:49:52 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:49:52 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:49:52 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-06-13 20:49:52 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-06-13 20:49:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 20:49:52 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:49:52 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2044 bytes result sent to driver -2016-06-13 20:49:52 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2044 bytes result sent to driver -2016-06-13 20:49:52 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 34 ms on localhost (1/2) -2016-06-13 20:49:52 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 33 ms on localhost (2/2) -2016-06-13 20:49:52 INFO DAGScheduler:59 - ResultStage 4 (foreach at PCASuite.scala:38) finished in 0.035 s -2016-06-13 20:49:52 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 20:49:52 INFO DAGScheduler:59 - Job 4 finished: foreach at PCASuite.scala:38, took 0.073248 s -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:49:52 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:49:52 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 20:49:52 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:49:52 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:49:52 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:49:52 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:49:52 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:49:52 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:49:52 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:49:52 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:49:52 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 20:49:52 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:49:52 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-98d80751-bbb3-4450-a62d-047747cb2f8f -2016-06-13 20:55:59 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:55:59 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 20:55:59 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:55:59 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:55:59 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:56:00 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:56:00 INFO Remoting:74 - Starting remoting -2016-06-13 20:56:01 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:52370] -2016-06-13 20:56:01 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52370. -2016-06-13 20:56:01 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:56:01 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:56:01 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-892dd841-bec7-4b18-a36f-56a128896c83 -2016-06-13 20:56:01 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:56:01 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-627f17b1-54a9-4d66-a5e1-3401b1e00d78\httpd-1efe4770-3fd7-4743-bbdf-ad15d3cef82f -2016-06-13 20:56:01 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:56:01 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:56:01 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52371 -2016-06-13 20:56:01 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52371. -2016-06-13 20:56:01 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:56:01 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:56:01 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 20:56:01 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 20:56:01 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 20:56:01 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:56:01 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:56:01 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52390. -2016-06-13 20:56:01 INFO NettyBlockTransferService:59 - Server created on 52390 -2016-06-13 20:56:01 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:56:01 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52390 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52390) -2016-06-13 20:56:01 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:56:02 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 20:56:02 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 20:56:02 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 20:56:02 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 20:56:02 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52390 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 20:56:02 INFO SparkContext:59 - Created broadcast 0 from textFile at PCASuite.scala:21 -2016-06-13 20:56:03 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 20:56:04 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 20:56:04 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 20:56:04 INFO DAGScheduler:59 - Got job 0 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 20:56:04 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at RowMatrix.scala:65) -2016-06-13 20:56:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:56:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:56:04 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at PCASuite.scala:24), which has no missing parents -2016-06-13 20:56:04 INFO MemoryStore:59 - ensureFreeSpace(3552) called with curMem=171580, maxMem=766075207 -2016-06-13 20:56:04 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 20:56:04 INFO MemoryStore:59 - ensureFreeSpace(2010) called with curMem=175132, maxMem=766075207 -2016-06-13 20:56:04 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2010.0 B, free 730.4 MB) -2016-06-13 20:56:04 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52390 (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:56:04 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:56:04 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at PCASuite.scala:24) -2016-06-13 20:56:04 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 20:56:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:56:04 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 20:56:04 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:56:04 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 20:56:04 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 20:56:04 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 20:56:04 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 20:56:04 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 20:56:04 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2183 bytes result sent to driver -2016-06-13 20:56:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 91 ms on localhost (1/1) -2016-06-13 20:56:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 20:56:04 INFO DAGScheduler:59 - ResultStage 0 (first at RowMatrix.scala:65) finished in 0.104 s -2016-06-13 20:56:04 INFO DAGScheduler:59 - Job 0 finished: first at RowMatrix.scala:65, took 0.191044 s -2016-06-13 20:56:04 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-06-13 20:56:04 INFO DAGScheduler:59 - Got job 1 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-06-13 20:56:04 INFO DAGScheduler:59 - Final stage: ResultStage 1(treeAggregate at RowMatrix.scala:332) -2016-06-13 20:56:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:56:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:56:04 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-06-13 20:56:04 INFO MemoryStore:59 - ensureFreeSpace(4456) called with curMem=177142, maxMem=766075207 -2016-06-13 20:56:04 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 20:56:04 INFO MemoryStore:59 - ensureFreeSpace(2380) called with curMem=181598, maxMem=766075207 -2016-06-13 20:56:04 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 20:56:04 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52390 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 20:56:04 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:56:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:332) -2016-06-13 20:56:04 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 20:56:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:56:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:56:04 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 20:56:04 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 20:56:04 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:56:04 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 20:56:04 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2368 bytes result sent to driver -2016-06-13 20:56:04 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2368 bytes result sent to driver -2016-06-13 20:56:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 35 ms on localhost (1/2) -2016-06-13 20:56:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 38 ms on localhost (2/2) -2016-06-13 20:56:04 INFO DAGScheduler:59 - ResultStage 1 (treeAggregate at RowMatrix.scala:332) finished in 0.039 s -2016-06-13 20:56:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 20:56:04 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 20:56:04 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 20:56:04 INFO DAGScheduler:59 - Job 1 finished: treeAggregate at RowMatrix.scala:332, took 0.067762 s -2016-06-13 20:56:04 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-06-13 20:56:04 INFO DAGScheduler:59 - Got job 2 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-06-13 20:56:04 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeAggregate at RowMatrix.scala:124) -2016-06-13 20:56:04 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:56:04 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:56:04 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[5] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-06-13 20:56:04 INFO MemoryStore:59 - ensureFreeSpace(4512) called with curMem=183978, maxMem=766075207 -2016-06-13 20:56:04 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 20:56:04 INFO MemoryStore:59 - ensureFreeSpace(2368) called with curMem=188490, maxMem=766075207 -2016-06-13 20:56:04 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 20:56:04 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52390 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 20:56:04 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:56:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[5] at treeAggregate at RowMatrix.scala:124) -2016-06-13 20:56:04 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 20:56:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:56:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:56:04 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 20:56:04 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-06-13 20:56:04 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:56:04 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 20:56:04 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2265 bytes result sent to driver -2016-06-13 20:56:04 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2265 bytes result sent to driver -2016-06-13 20:56:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 27 ms on localhost (1/2) -2016-06-13 20:56:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 28 ms on localhost (2/2) -2016-06-13 20:56:04 INFO DAGScheduler:59 - ResultStage 2 (treeAggregate at RowMatrix.scala:124) finished in 0.030 s -2016-06-13 20:56:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 20:56:04 INFO DAGScheduler:59 - Job 2 finished: treeAggregate at RowMatrix.scala:124, took 0.042658 s -2016-06-13 20:56:05 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 20:56:05 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 20:56:05 INFO MemoryStore:59 - ensureFreeSpace(88) called with curMem=190858, maxMem=766075207 -2016-06-13 20:56:05 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 88.0 B, free 730.4 MB) -2016-06-13 20:56:05 INFO MemoryStore:59 - ensureFreeSpace(85) called with curMem=190946, maxMem=766075207 -2016-06-13 20:56:05 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 85.0 B, free 730.4 MB) -2016-06-13 20:56:05 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52390 (size: 85.0 B, free: 730.6 MB) -2016-06-13 20:56:05 INFO SparkContext:59 - Created broadcast 4 from broadcast at RowMatrix.scala:425 -2016-06-13 20:56:05 INFO SparkContext:59 - Starting job: foreach at PCASuite.scala:34 -2016-06-13 20:56:05 INFO DAGScheduler:59 - Got job 3 (foreach at PCASuite.scala:34) with 2 output partitions -2016-06-13 20:56:05 INFO DAGScheduler:59 - Final stage: ResultStage 3(foreach at PCASuite.scala:34) -2016-06-13 20:56:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:56:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:56:05 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[3] at map at PCASuite.scala:24), which has no missing parents -2016-06-13 20:56:05 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=191031, maxMem=766075207 -2016-06-13 20:56:05 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-06-13 20:56:05 INFO MemoryStore:59 - ensureFreeSpace(1994) called with curMem=194559, maxMem=766075207 -2016-06-13 20:56:05 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 1994.0 B, free 730.4 MB) -2016-06-13 20:56:05 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52390 (size: 1994.0 B, free: 730.6 MB) -2016-06-13 20:56:05 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:56:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[3] at map at PCASuite.scala:24) -2016-06-13 20:56:05 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 20:56:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:56:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:56:05 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-06-13 20:56:05 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-06-13 20:56:05 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 20:56:05 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:56:05 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2044 bytes result sent to driver -2016-06-13 20:56:05 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2044 bytes result sent to driver -2016-06-13 20:56:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 19 ms on localhost (1/2) -2016-06-13 20:56:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 18 ms on localhost (2/2) -2016-06-13 20:56:05 INFO DAGScheduler:59 - ResultStage 3 (foreach at PCASuite.scala:34) finished in 0.020 s -2016-06-13 20:56:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 20:56:05 INFO DAGScheduler:59 - Job 3 finished: foreach at PCASuite.scala:34, took 0.034343 s -2016-06-13 20:56:05 INFO SparkContext:59 - Starting job: foreach at PCASuite.scala:38 -2016-06-13 20:56:05 INFO DAGScheduler:59 - Got job 4 (foreach at PCASuite.scala:38) with 2 output partitions -2016-06-13 20:56:05 INFO DAGScheduler:59 - Final stage: ResultStage 4(foreach at PCASuite.scala:38) -2016-06-13 20:56:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:56:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:56:05 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[6] at mapPartitions at RowMatrix.scala:426), which has no missing parents -2016-06-13 20:56:05 INFO MemoryStore:59 - ensureFreeSpace(3856) called with curMem=196553, maxMem=766075207 -2016-06-13 20:56:05 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 20:56:05 INFO MemoryStore:59 - ensureFreeSpace(2132) called with curMem=200409, maxMem=766075207 -2016-06-13 20:56:05 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 20:56:05 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52390 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 20:56:05 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:56:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[6] at mapPartitions at RowMatrix.scala:426) -2016-06-13 20:56:05 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 20:56:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:56:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:56:05 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-06-13 20:56:05 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-06-13 20:56:05 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:56:05 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 20:56:05 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2044 bytes result sent to driver -2016-06-13 20:56:05 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2044 bytes result sent to driver -2016-06-13 20:56:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 27 ms on localhost (1/2) -2016-06-13 20:56:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 26 ms on localhost (2/2) -2016-06-13 20:56:05 INFO DAGScheduler:59 - ResultStage 4 (foreach at PCASuite.scala:38) finished in 0.027 s -2016-06-13 20:56:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 20:56:05 INFO DAGScheduler:59 - Job 4 finished: foreach at PCASuite.scala:38, took 0.042146 s -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:56:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:56:05 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 20:56:05 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:56:05 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:56:05 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:56:05 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:56:05 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:56:05 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:56:05 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:56:05 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:56:05 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:56:05 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 20:56:05 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-627f17b1-54a9-4d66-a5e1-3401b1e00d78 -2016-06-13 20:57:03 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:57:04 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 20:57:04 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:57:04 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:57:04 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:57:05 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:57:05 INFO Remoting:74 - Starting remoting -2016-06-13 20:57:05 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:52412] -2016-06-13 20:57:05 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52412. -2016-06-13 20:57:05 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:57:05 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:57:05 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-ad98dbfd-0dbb-40fc-a95b-76129c4739e9 -2016-06-13 20:57:05 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:57:06 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-4fe4172c-b5e7-450d-bfdf-7b8a7920e8b4\httpd-4d8098d4-7731-4a1b-99d2-c0144d90e5a8 -2016-06-13 20:57:06 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:57:06 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:57:06 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52413 -2016-06-13 20:57:06 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52413. -2016-06-13 20:57:06 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:57:06 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:57:06 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 20:57:06 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 20:57:06 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 20:57:06 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:57:06 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:57:06 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52432. -2016-06-13 20:57:06 INFO NettyBlockTransferService:59 - Server created on 52432 -2016-06-13 20:57:06 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:57:06 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52432 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52432) -2016-06-13 20:57:06 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:57:07 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 20:57:07 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 20:57:07 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 20:57:07 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 20:57:07 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52432 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 20:57:07 INFO SparkContext:59 - Created broadcast 0 from textFile at PCASuite.scala:21 -2016-06-13 20:57:08 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 20:57:08 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 20:57:08 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 20:57:09 INFO DAGScheduler:59 - Got job 0 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 20:57:09 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at RowMatrix.scala:65) -2016-06-13 20:57:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:57:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:57:09 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at PCASuite.scala:24), which has no missing parents -2016-06-13 20:57:09 INFO MemoryStore:59 - ensureFreeSpace(3552) called with curMem=171580, maxMem=766075207 -2016-06-13 20:57:09 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.5 KB, free 730.4 MB) -2016-06-13 20:57:09 INFO MemoryStore:59 - ensureFreeSpace(2010) called with curMem=175132, maxMem=766075207 -2016-06-13 20:57:09 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2010.0 B, free 730.4 MB) -2016-06-13 20:57:09 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52432 (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:57:09 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:57:09 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at PCASuite.scala:24) -2016-06-13 20:57:09 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-06-13 20:57:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:57:09 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 20:57:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:57:09 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 20:57:09 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 20:57:09 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 20:57:09 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 20:57:09 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 20:57:09 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2183 bytes result sent to driver -2016-06-13 20:57:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 96 ms on localhost (1/1) -2016-06-13 20:57:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 20:57:09 INFO DAGScheduler:59 - ResultStage 0 (first at RowMatrix.scala:65) finished in 0.116 s -2016-06-13 20:57:09 INFO DAGScheduler:59 - Job 0 finished: first at RowMatrix.scala:65, took 0.232207 s -2016-06-13 20:57:09 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-06-13 20:57:09 INFO DAGScheduler:59 - Got job 1 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-06-13 20:57:09 INFO DAGScheduler:59 - Final stage: ResultStage 1(treeAggregate at RowMatrix.scala:332) -2016-06-13 20:57:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:57:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:57:09 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-06-13 20:57:09 INFO MemoryStore:59 - ensureFreeSpace(4456) called with curMem=177142, maxMem=766075207 -2016-06-13 20:57:09 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 20:57:09 INFO MemoryStore:59 - ensureFreeSpace(2383) called with curMem=181598, maxMem=766075207 -2016-06-13 20:57:09 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 20:57:09 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52432 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 20:57:09 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:57:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:332) -2016-06-13 20:57:09 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 20:57:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:57:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:57:09 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-06-13 20:57:09 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 2) -2016-06-13 20:57:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:57:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 20:57:09 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2368 bytes result sent to driver -2016-06-13 20:57:09 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 2). 2368 bytes result sent to driver -2016-06-13 20:57:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 34 ms on localhost (1/2) -2016-06-13 20:57:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 2) in 34 ms on localhost (2/2) -2016-06-13 20:57:09 INFO DAGScheduler:59 - ResultStage 1 (treeAggregate at RowMatrix.scala:332) finished in 0.036 s -2016-06-13 20:57:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 20:57:09 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 20:57:09 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 20:57:09 INFO DAGScheduler:59 - Job 1 finished: treeAggregate at RowMatrix.scala:332, took 0.069397 s -2016-06-13 20:57:09 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-06-13 20:57:09 INFO DAGScheduler:59 - Got job 2 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-06-13 20:57:09 INFO DAGScheduler:59 - Final stage: ResultStage 2(treeAggregate at RowMatrix.scala:124) -2016-06-13 20:57:09 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:57:09 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:57:09 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[5] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-06-13 20:57:09 INFO MemoryStore:59 - ensureFreeSpace(4512) called with curMem=183981, maxMem=766075207 -2016-06-13 20:57:09 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 20:57:09 INFO MemoryStore:59 - ensureFreeSpace(2371) called with curMem=188493, maxMem=766075207 -2016-06-13 20:57:09 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.3 KB, free 730.4 MB) -2016-06-13 20:57:09 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52432 (size: 2.3 KB, free: 730.6 MB) -2016-06-13 20:57:09 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:57:09 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[5] at treeAggregate at RowMatrix.scala:124) -2016-06-13 20:57:09 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 20:57:09 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:57:09 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:57:09 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 20:57:09 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 4) -2016-06-13 20:57:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 20:57:09 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:57:09 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 2265 bytes result sent to driver -2016-06-13 20:57:09 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 4). 2265 bytes result sent to driver -2016-06-13 20:57:09 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52432 in memory (size: 2.3 KB, free: 730.6 MB) -2016-06-13 20:57:09 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 45 ms on localhost (1/2) -2016-06-13 20:57:09 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 4) in 45 ms on localhost (2/2) -2016-06-13 20:57:09 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 20:57:09 INFO DAGScheduler:59 - ResultStage 2 (treeAggregate at RowMatrix.scala:124) finished in 0.047 s -2016-06-13 20:57:09 INFO DAGScheduler:59 - Job 2 finished: treeAggregate at RowMatrix.scala:124, took 0.061155 s -2016-06-13 20:57:09 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 20:57:09 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52432 in memory (size: 2010.0 B, free: 730.6 MB) -2016-06-13 20:57:09 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 20:57:09 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 20:57:09 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 20:57:10 INFO MemoryStore:59 - ensureFreeSpace(112) called with curMem=178463, maxMem=766075207 -2016-06-13 20:57:10 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 112.0 B, free 730.4 MB) -2016-06-13 20:57:10 INFO MemoryStore:59 - ensureFreeSpace(101) called with curMem=178575, maxMem=766075207 -2016-06-13 20:57:10 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 101.0 B, free 730.4 MB) -2016-06-13 20:57:10 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52432 (size: 101.0 B, free: 730.6 MB) -2016-06-13 20:57:10 INFO SparkContext:59 - Created broadcast 4 from broadcast at RowMatrix.scala:425 -2016-06-13 20:57:10 INFO SparkContext:59 - Starting job: foreach at PCASuite.scala:34 -2016-06-13 20:57:10 INFO DAGScheduler:59 - Got job 3 (foreach at PCASuite.scala:34) with 2 output partitions -2016-06-13 20:57:10 INFO DAGScheduler:59 - Final stage: ResultStage 3(foreach at PCASuite.scala:34) -2016-06-13 20:57:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:57:10 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:57:10 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[3] at map at PCASuite.scala:24), which has no missing parents -2016-06-13 20:57:10 INFO MemoryStore:59 - ensureFreeSpace(3528) called with curMem=178676, maxMem=766075207 -2016-06-13 20:57:10 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 3.4 KB, free 730.4 MB) -2016-06-13 20:57:10 INFO MemoryStore:59 - ensureFreeSpace(1994) called with curMem=182204, maxMem=766075207 -2016-06-13 20:57:10 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 1994.0 B, free 730.4 MB) -2016-06-13 20:57:10 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52432 (size: 1994.0 B, free: 730.6 MB) -2016-06-13 20:57:10 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:57:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[3] at map at PCASuite.scala:24) -2016-06-13 20:57:10 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 20:57:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:57:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:57:10 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 5) -2016-06-13 20:57:10 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 6) -2016-06-13 20:57:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:57:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 20:57:10 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 5). 2044 bytes result sent to driver -2016-06-13 20:57:10 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 6). 2044 bytes result sent to driver -2016-06-13 20:57:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 5) in 17 ms on localhost (1/2) -2016-06-13 20:57:10 INFO DAGScheduler:59 - ResultStage 3 (foreach at PCASuite.scala:34) finished in 0.018 s -2016-06-13 20:57:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 6) in 17 ms on localhost (2/2) -2016-06-13 20:57:10 INFO DAGScheduler:59 - Job 3 finished: foreach at PCASuite.scala:34, took 0.031060 s -2016-06-13 20:57:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 20:57:10 INFO SparkContext:59 - Starting job: foreach at PCASuite.scala:38 -2016-06-13 20:57:10 INFO DAGScheduler:59 - Got job 4 (foreach at PCASuite.scala:38) with 2 output partitions -2016-06-13 20:57:10 INFO DAGScheduler:59 - Final stage: ResultStage 4(foreach at PCASuite.scala:38) -2016-06-13 20:57:10 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:57:10 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:57:10 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[6] at mapPartitions at RowMatrix.scala:426), which has no missing parents -2016-06-13 20:57:10 INFO MemoryStore:59 - ensureFreeSpace(3856) called with curMem=184198, maxMem=766075207 -2016-06-13 20:57:10 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 20:57:10 INFO MemoryStore:59 - ensureFreeSpace(2130) called with curMem=188054, maxMem=766075207 -2016-06-13 20:57:10 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 20:57:10 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52432 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 20:57:10 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:57:10 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[6] at mapPartitions at RowMatrix.scala:426) -2016-06-13 20:57:10 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 20:57:10 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:57:10 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2184 bytes) -2016-06-13 20:57:10 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-06-13 20:57:10 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 8) -2016-06-13 20:57:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:13+13 -2016-06-13 20:57:10 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/MatrixRow33.txt:0+13 -2016-06-13 20:57:10 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2044 bytes result sent to driver -2016-06-13 20:57:10 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 8). 2044 bytes result sent to driver -2016-06-13 20:57:10 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 8) in 26 ms on localhost (1/2) -2016-06-13 20:57:10 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 27 ms on localhost (2/2) -2016-06-13 20:57:10 INFO DAGScheduler:59 - ResultStage 4 (foreach at PCASuite.scala:38) finished in 0.027 s -2016-06-13 20:57:10 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 20:57:10 INFO DAGScheduler:59 - Job 4 finished: foreach at PCASuite.scala:38, took 0.042024 s -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:57:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:57:10 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 20:57:10 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:57:10 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:57:10 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:57:10 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:57:10 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:57:10 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:57:10 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:57:10 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:57:10 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:57:10 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 20:57:10 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-4fe4172c-b5e7-450d-bfdf-7b8a7920e8b4 -2016-06-13 20:59:24 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 20:59:24 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 20:59:25 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 20:59:25 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 20:59:25 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 20:59:25 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 20:59:25 INFO Remoting:74 - Starting remoting -2016-06-13 20:59:26 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:52459] -2016-06-13 20:59:26 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52459. -2016-06-13 20:59:26 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 20:59:26 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 20:59:26 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-30736dad-07b8-4b5a-8236-992dbe883140 -2016-06-13 20:59:26 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 20:59:26 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-5f5af81d-bdb5-404a-a901-80796487d35a\httpd-755d8d40-4155-44a9-a666-cd41e1f0f04e -2016-06-13 20:59:26 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 20:59:26 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:59:26 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52460 -2016-06-13 20:59:26 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52460. -2016-06-13 20:59:26 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 20:59:26 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 20:59:26 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 20:59:26 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 20:59:26 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 20:59:26 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 20:59:26 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 20:59:26 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52479. -2016-06-13 20:59:26 INFO NettyBlockTransferService:59 - Server created on 52479 -2016-06-13 20:59:26 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 20:59:26 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52479 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52479) -2016-06-13 20:59:26 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 20:59:27 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 20:59:27 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 20:59:27 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 20:59:27 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 20:59:27 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52479 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 20:59:27 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-06-13 20:59:29 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 20:59:29 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 20:59:29 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-06-13 20:59:29 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-06-13 20:59:29 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-06-13 20:59:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:59:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:59:29 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-06-13 20:59:29 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-06-13 20:59:29 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 20:59:29 INFO MemoryStore:59 - ensureFreeSpace(2140) called with curMem=175476, maxMem=766075207 -2016-06-13 20:59:29 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 20:59:29 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52479 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 20:59:29 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:59:29 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-06-13 20:59:29 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 20:59:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 20:59:29 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 20:59:29 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 20:59:29 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 20:59:29 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-06-13 20:59:29 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-06-13 20:59:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/sample_libsvm_data.txt:52368+52368 -2016-06-13 20:59:29 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/sample_libsvm_data.txt:0+52368 -2016-06-13 20:59:29 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 20:59:29 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 20:59:29 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 20:59:29 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 20:59:29 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 20:59:29 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177616, maxMem=766075207 -2016-06-13 20:59:29 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-06-13 20:59:29 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:52479 (size: 83.1 KB, free: 730.5 MB) -2016-06-13 20:59:29 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262696, maxMem=766075207 -2016-06-13 20:59:29 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-06-13 20:59:29 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:52479 (size: 83.6 KB, free: 730.4 MB) -2016-06-13 20:59:29 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-06-13 20:59:29 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-06-13 20:59:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 453 ms on localhost (1/2) -2016-06-13 20:59:29 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 438 ms on localhost (2/2) -2016-06-13 20:59:29 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.472 s -2016-06-13 20:59:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 20:59:29 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.616910 s -2016-06-13 20:59:29 INFO SparkContext:59 - Starting job: first at PCA.scala:42 -2016-06-13 20:59:29 INFO DAGScheduler:59 - Got job 1 (first at PCA.scala:42) with 1 output partitions -2016-06-13 20:59:29 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at PCA.scala:42) -2016-06-13 20:59:29 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:59:29 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:59:29 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29), which has no missing parents -2016-06-13 20:59:29 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=348344, maxMem=766075207 -2016-06-13 20:59:29 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.1 KB, free 730.3 MB) -2016-06-13 20:59:29 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=352536, maxMem=766075207 -2016-06-13 20:59:29 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 20:59:29 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52479 (size: 2.2 KB, free: 730.4 MB) -2016-06-13 20:59:29 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:59:29 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29) -2016-06-13 20:59:29 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 20:59:29 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52479 in memory (size: 2.1 KB, free: 730.4 MB) -2016-06-13 20:59:29 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 20:59:29 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 20:59:29 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 20:59:29 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 20:59:29 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 4334 bytes result sent to driver -2016-06-13 20:59:29 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 20 ms on localhost (1/1) -2016-06-13 20:59:29 INFO DAGScheduler:59 - ResultStage 1 (first at PCA.scala:42) finished in 0.021 s -2016-06-13 20:59:29 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 20:59:29 INFO DAGScheduler:59 - Job 1 finished: first at PCA.scala:42, took 0.064415 s -2016-06-13 20:59:29 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 20:59:29 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 20:59:30 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-06-13 20:59:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:59:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:59:30 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29), which has no missing parents -2016-06-13 20:59:30 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=348768, maxMem=766075207 -2016-06-13 20:59:30 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-06-13 20:59:30 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=352960, maxMem=766075207 -2016-06-13 20:59:30 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 20:59:30 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52479 (size: 2.2 KB, free: 730.4 MB) -2016-06-13 20:59:30 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:59:30 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29) -2016-06-13 20:59:30 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-06-13 20:59:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 20:59:30 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 20:59:30 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 20:59:30 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 4334 bytes result sent to driver -2016-06-13 20:59:30 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 10 ms on localhost (1/1) -2016-06-13 20:59:30 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.011 s -2016-06-13 20:59:30 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 20:59:30 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.024072 s -2016-06-13 20:59:30 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-06-13 20:59:30 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-06-13 20:59:30 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-06-13 20:59:30 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:59:30 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:59:30 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-06-13 20:59:30 INFO MemoryStore:59 - ensureFreeSpace(10616) called with curMem=355228, maxMem=766075207 -2016-06-13 20:59:30 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 10.4 KB, free 730.2 MB) -2016-06-13 20:59:30 INFO MemoryStore:59 - ensureFreeSpace(2925) called with curMem=365844, maxMem=766075207 -2016-06-13 20:59:30 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.2 MB) -2016-06-13 20:59:30 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52479 (size: 2.9 KB, free: 730.4 MB) -2016-06-13 20:59:30 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:59:30 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-06-13 20:59:30 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 20:59:30 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 4, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 20:59:30 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 20:59:30 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 4) -2016-06-13 20:59:30 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 5) -2016-06-13 20:59:30 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 20:59:30 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 20:59:32 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 4). 7905 bytes result sent to driver -2016-06-13 20:59:32 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 5). 7905 bytes result sent to driver -2016-06-13 20:59:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 5) in 2045 ms on localhost (1/2) -2016-06-13 20:59:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 4) in 2048 ms on localhost (2/2) -2016-06-13 20:59:32 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 2.049 s -2016-06-13 20:59:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 20:59:32 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 20:59:32 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 20:59:32 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 2.085231 s -2016-06-13 20:59:32 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-06-13 20:59:32 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-06-13 20:59:32 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-06-13 20:59:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:59:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:59:32 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-06-13 20:59:32 INFO MemoryStore:59 - ensureFreeSpace(1923344) called with curMem=368769, maxMem=766075207 -2016-06-13 20:59:32 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 1878.3 KB, free 728.4 MB) -2016-06-13 20:59:32 INFO MemoryStore:59 - ensureFreeSpace(93166) called with curMem=2292113, maxMem=766075207 -2016-06-13 20:59:32 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 91.0 KB, free 728.3 MB) -2016-06-13 20:59:32 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52479 (size: 91.0 KB, free: 730.3 MB) -2016-06-13 20:59:32 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:59:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-06-13 20:59:32 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 20:59:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 6, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 20:59:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 20:59:32 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 7) -2016-06-13 20:59:32 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 6) -2016-06-13 20:59:32 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 20:59:32 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 20:59:32 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 6). 1929806 bytes result sent to driver -2016-06-13 20:59:32 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 7). 1929806 bytes result sent to driver -2016-06-13 20:59:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 6) in 353 ms on localhost (1/2) -2016-06-13 20:59:32 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52479 in memory (size: 2.9 KB, free: 730.3 MB) -2016-06-13 20:59:32 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 20:59:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 7) in 372 ms on localhost (2/2) -2016-06-13 20:59:32 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.373 s -2016-06-13 20:59:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 20:59:32 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52479 in memory (size: 2.2 KB, free: 730.3 MB) -2016-06-13 20:59:32 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 20:59:32 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52479 in memory (size: 2.2 KB, free: 730.3 MB) -2016-06-13 20:59:32 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 20:59:32 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.417516 s -2016-06-13 20:59:32 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 20:59:32 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 20:59:34 INFO SparkContext:59 - Starting job: foreach at PCALabelPointSuite.scala:35 -2016-06-13 20:59:34 INFO DAGScheduler:59 - Got job 5 (foreach at PCALabelPointSuite.scala:35) with 2 output partitions -2016-06-13 20:59:34 INFO DAGScheduler:59 - Final stage: ResultStage 5(foreach at PCALabelPointSuite.scala:35) -2016-06-13 20:59:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:59:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:59:34 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-06-13 20:59:34 INFO MemoryStore:59 - ensureFreeSpace(3960) called with curMem=2358818, maxMem=766075207 -2016-06-13 20:59:34 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.9 KB, free 728.3 MB) -2016-06-13 20:59:34 INFO MemoryStore:59 - ensureFreeSpace(2205) called with curMem=2362778, maxMem=766075207 -2016-06-13 20:59:34 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.2 KB, free 728.3 MB) -2016-06-13 20:59:34 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52479 (size: 2.2 KB, free: 730.3 MB) -2016-06-13 20:59:34 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:59:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-06-13 20:59:34 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 20:59:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 8, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 20:59:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 20:59:34 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 8) -2016-06-13 20:59:34 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 9) -2016-06-13 20:59:34 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 20:59:34 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 20:59:34 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 8). 2044 bytes result sent to driver -2016-06-13 20:59:34 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 9). 2044 bytes result sent to driver -2016-06-13 20:59:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 8) in 89 ms on localhost (1/2) -2016-06-13 20:59:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 9) in 91 ms on localhost (2/2) -2016-06-13 20:59:34 INFO DAGScheduler:59 - ResultStage 5 (foreach at PCALabelPointSuite.scala:35) finished in 0.092 s -2016-06-13 20:59:34 INFO DAGScheduler:59 - Job 5 finished: foreach at PCALabelPointSuite.scala:35, took 0.121298 s -2016-06-13 20:59:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 20:59:34 INFO SparkContext:59 - Starting job: foreach at PCALabelPointSuite.scala:39 -2016-06-13 20:59:34 INFO DAGScheduler:59 - Got job 6 (foreach at PCALabelPointSuite.scala:39) with 2 output partitions -2016-06-13 20:59:34 INFO DAGScheduler:59 - Final stage: ResultStage 6(foreach at PCALabelPointSuite.scala:39) -2016-06-13 20:59:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 20:59:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 20:59:34 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[10] at map at PCALabelPointSuite.scala:32), which has no missing parents -2016-06-13 20:59:34 INFO MemoryStore:59 - ensureFreeSpace(59792) called with curMem=2364983, maxMem=766075207 -2016-06-13 20:59:34 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 58.4 KB, free 728.3 MB) -2016-06-13 20:59:34 INFO MemoryStore:59 - ensureFreeSpace(39871) called with curMem=2424775, maxMem=766075207 -2016-06-13 20:59:34 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 38.9 KB, free 728.2 MB) -2016-06-13 20:59:34 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52479 (size: 38.9 KB, free: 730.3 MB) -2016-06-13 20:59:34 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 20:59:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[10] at map at PCALabelPointSuite.scala:32) -2016-06-13 20:59:34 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-06-13 20:59:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 10, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 20:59:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 20:59:34 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 10) -2016-06-13 20:59:34 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 11) -2016-06-13 20:59:34 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 20:59:34 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 20:59:34 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 10). 2044 bytes result sent to driver -2016-06-13 20:59:34 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 11). 2044 bytes result sent to driver -2016-06-13 20:59:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 10) in 29 ms on localhost (1/2) -2016-06-13 20:59:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 11) in 29 ms on localhost (2/2) -2016-06-13 20:59:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 20:59:34 INFO DAGScheduler:59 - ResultStage 6 (foreach at PCALabelPointSuite.scala:39) finished in 0.030 s -2016-06-13 20:59:34 INFO DAGScheduler:59 - Job 6 finished: foreach at PCALabelPointSuite.scala:39, took 0.047828 s -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 20:59:34 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 20:59:34 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 20:59:34 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 20:59:34 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 20:59:34 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 20:59:34 INFO BlockManager:59 - BlockManager stopped -2016-06-13 20:59:34 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 20:59:34 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 20:59:34 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 20:59:34 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 20:59:34 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 20:59:34 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 20:59:34 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-5f5af81d-bdb5-404a-a901-80796487d35a -2016-06-13 21:00:32 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 21:00:32 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 21:00:33 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 21:00:33 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 21:00:33 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 21:00:33 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 21:00:33 INFO Remoting:74 - Starting remoting -2016-06-13 21:00:34 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:52500] -2016-06-13 21:00:34 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52500. -2016-06-13 21:00:34 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 21:00:34 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 21:00:34 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-ddc0393c-774d-4a0b-a1c9-9cc20656c152 -2016-06-13 21:00:34 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 21:00:34 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-49050aff-16a6-4ed8-bc71-dc50c32a32c3\httpd-927d9645-eed1-4b88-b1e6-90084418cb1c -2016-06-13 21:00:34 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 21:00:34 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:00:34 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52501 -2016-06-13 21:00:34 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52501. -2016-06-13 21:00:34 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 21:00:34 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:00:34 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 21:00:34 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 21:00:34 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 21:00:34 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 21:00:34 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 21:00:35 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52520. -2016-06-13 21:00:35 INFO NettyBlockTransferService:59 - Server created on 52520 -2016-06-13 21:00:35 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 21:00:35 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52520 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52520) -2016-06-13 21:00:35 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 21:00:36 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 21:00:36 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 21:00:36 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 21:00:36 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 21:00:36 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52520 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 21:00:36 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-06-13 21:00:37 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 21:00:37 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 21:00:37 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-06-13 21:00:37 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-06-13 21:00:37 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-06-13 21:00:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:00:37 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:00:37 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-06-13 21:00:37 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-06-13 21:00:37 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 21:00:37 INFO MemoryStore:59 - ensureFreeSpace(2140) called with curMem=175476, maxMem=766075207 -2016-06-13 21:00:37 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 21:00:37 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52520 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 21:00:37 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:00:37 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-06-13 21:00:37 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 21:00:37 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:00:37 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:00:37 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 21:00:37 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 21:00:37 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-06-13 21:00:37 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-06-13 21:00:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/sample_libsvm_data.txt:52368+52368 -2016-06-13 21:00:37 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/sample_libsvm_data.txt:0+52368 -2016-06-13 21:00:37 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 21:00:37 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 21:00:37 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 21:00:37 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 21:00:37 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 21:00:38 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177616, maxMem=766075207 -2016-06-13 21:00:38 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-06-13 21:00:38 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:52520 (size: 83.6 KB, free: 730.5 MB) -2016-06-13 21:00:38 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263264, maxMem=766075207 -2016-06-13 21:00:38 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-06-13 21:00:38 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:52520 (size: 83.1 KB, free: 730.4 MB) -2016-06-13 21:00:38 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-06-13 21:00:38 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-06-13 21:00:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 362 ms on localhost (1/2) -2016-06-13 21:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 395 ms on localhost (2/2) -2016-06-13 21:00:38 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.431 s -2016-06-13 21:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 21:00:38 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.696408 s -2016-06-13 21:00:38 INFO SparkContext:59 - Starting job: first at PCA.scala:42 -2016-06-13 21:00:38 INFO DAGScheduler:59 - Got job 1 (first at PCA.scala:42) with 1 output partitions -2016-06-13 21:00:38 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at PCA.scala:42) -2016-06-13 21:00:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:00:38 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:00:38 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29), which has no missing parents -2016-06-13 21:00:38 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=348344, maxMem=766075207 -2016-06-13 21:00:38 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.1 KB, free 730.3 MB) -2016-06-13 21:00:38 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=352536, maxMem=766075207 -2016-06-13 21:00:38 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 21:00:38 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52520 (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:00:38 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:00:38 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29) -2016-06-13 21:00:38 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 21:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:00:38 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 21:00:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:00:38 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 4334 bytes result sent to driver -2016-06-13 21:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 17 ms on localhost (1/1) -2016-06-13 21:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 21:00:38 INFO DAGScheduler:59 - ResultStage 1 (first at PCA.scala:42) finished in 0.018 s -2016-06-13 21:00:38 INFO DAGScheduler:59 - Job 1 finished: first at PCA.scala:42, took 0.035861 s -2016-06-13 21:00:38 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 21:00:38 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 21:00:38 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-06-13 21:00:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:00:38 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:00:38 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29), which has no missing parents -2016-06-13 21:00:38 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=354804, maxMem=766075207 -2016-06-13 21:00:38 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-06-13 21:00:38 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=358996, maxMem=766075207 -2016-06-13 21:00:38 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 21:00:38 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52520 (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:00:38 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:00:38 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29) -2016-06-13 21:00:38 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-06-13 21:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:00:38 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 21:00:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:00:38 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 4334 bytes result sent to driver -2016-06-13 21:00:38 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.011 s -2016-06-13 21:00:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 11 ms on localhost (1/1) -2016-06-13 21:00:38 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.025138 s -2016-06-13 21:00:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 21:00:38 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-06-13 21:00:38 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-06-13 21:00:38 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-06-13 21:00:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:00:38 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:00:38 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-06-13 21:00:38 INFO MemoryStore:59 - ensureFreeSpace(10616) called with curMem=361264, maxMem=766075207 -2016-06-13 21:00:38 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 10.4 KB, free 730.2 MB) -2016-06-13 21:00:38 INFO MemoryStore:59 - ensureFreeSpace(2925) called with curMem=371880, maxMem=766075207 -2016-06-13 21:00:38 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.2 MB) -2016-06-13 21:00:38 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52520 (size: 2.9 KB, free: 730.4 MB) -2016-06-13 21:00:38 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:00:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-06-13 21:00:38 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 21:00:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 4, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:00:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:00:38 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 4) -2016-06-13 21:00:38 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 5) -2016-06-13 21:00:38 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:00:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:00:39 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52520 in memory (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:00:39 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 21:00:39 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52520 in memory (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:00:39 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 21:00:39 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52520 in memory (size: 2.1 KB, free: 730.4 MB) -2016-06-13 21:00:39 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 21:00:39 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 5). 7905 bytes result sent to driver -2016-06-13 21:00:39 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 4). 7905 bytes result sent to driver -2016-06-13 21:00:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 5) in 645 ms on localhost (1/2) -2016-06-13 21:00:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 4) in 647 ms on localhost (2/2) -2016-06-13 21:00:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 21:00:39 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.648 s -2016-06-13 21:00:39 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 21:00:39 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 21:00:39 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.679411 s -2016-06-13 21:00:39 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-06-13 21:00:39 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-06-13 21:00:39 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-06-13 21:00:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:00:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:00:39 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-06-13 21:00:39 INFO MemoryStore:59 - ensureFreeSpace(1923344) called with curMem=355849, maxMem=766075207 -2016-06-13 21:00:39 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 1878.3 KB, free 728.4 MB) -2016-06-13 21:00:39 INFO MemoryStore:59 - ensureFreeSpace(93166) called with curMem=2279193, maxMem=766075207 -2016-06-13 21:00:39 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 91.0 KB, free 728.3 MB) -2016-06-13 21:00:39 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52520 (size: 91.0 KB, free: 730.3 MB) -2016-06-13 21:00:39 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:00:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-06-13 21:00:39 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 21:00:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 6, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:00:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:00:39 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 7) -2016-06-13 21:00:39 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 6) -2016-06-13 21:00:39 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:00:39 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:00:39 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 6). 1929806 bytes result sent to driver -2016-06-13 21:00:39 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 7). 1929806 bytes result sent to driver -2016-06-13 21:00:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 6) in 60 ms on localhost (1/2) -2016-06-13 21:00:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 7) in 64 ms on localhost (2/2) -2016-06-13 21:00:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 21:00:39 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.067 s -2016-06-13 21:00:39 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.119369 s -2016-06-13 21:00:39 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52520 in memory (size: 91.0 KB, free: 730.4 MB) -2016-06-13 21:00:39 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 21:00:39 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 21:00:39 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 21:00:39 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52520 in memory (size: 2.9 KB, free: 730.4 MB) -2016-06-13 21:00:39 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 21:00:41 INFO SparkContext:59 - Starting job: foreach at PCALabelPointSuite.scala:35 -2016-06-13 21:00:41 INFO DAGScheduler:59 - Got job 5 (foreach at PCALabelPointSuite.scala:35) with 2 output partitions -2016-06-13 21:00:41 INFO DAGScheduler:59 - Final stage: ResultStage 5(foreach at PCALabelPointSuite.scala:35) -2016-06-13 21:00:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:00:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:00:41 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-06-13 21:00:41 INFO MemoryStore:59 - ensureFreeSpace(3960) called with curMem=342308, maxMem=766075207 -2016-06-13 21:00:41 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.9 KB, free 730.3 MB) -2016-06-13 21:00:41 INFO MemoryStore:59 - ensureFreeSpace(2205) called with curMem=346268, maxMem=766075207 -2016-06-13 21:00:41 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.3 MB) -2016-06-13 21:00:41 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52520 (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:00:41 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:00:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-06-13 21:00:41 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 21:00:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 8, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:00:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:00:41 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 9) -2016-06-13 21:00:41 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 8) -2016-06-13 21:00:41 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:00:41 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:00:41 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 8). 2044 bytes result sent to driver -2016-06-13 21:00:41 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 9). 2044 bytes result sent to driver -2016-06-13 21:00:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 8) in 86 ms on localhost (1/2) -2016-06-13 21:00:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 9) in 87 ms on localhost (2/2) -2016-06-13 21:00:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 21:00:41 INFO DAGScheduler:59 - ResultStage 5 (foreach at PCALabelPointSuite.scala:35) finished in 0.088 s -2016-06-13 21:00:41 INFO DAGScheduler:59 - Job 5 finished: foreach at PCALabelPointSuite.scala:35, took 0.103522 s -2016-06-13 21:00:41 INFO SparkContext:59 - Starting job: foreach at PCALabelPointSuite.scala:41 -2016-06-13 21:00:41 INFO DAGScheduler:59 - Got job 6 (foreach at PCALabelPointSuite.scala:41) with 2 output partitions -2016-06-13 21:00:41 INFO DAGScheduler:59 - Final stage: ResultStage 6(foreach at PCALabelPointSuite.scala:41) -2016-06-13 21:00:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:00:41 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:00:41 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[10] at map at PCALabelPointSuite.scala:32), which has no missing parents -2016-06-13 21:00:41 INFO MemoryStore:59 - ensureFreeSpace(59792) called with curMem=348473, maxMem=766075207 -2016-06-13 21:00:41 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 58.4 KB, free 730.2 MB) -2016-06-13 21:00:41 INFO MemoryStore:59 - ensureFreeSpace(39871) called with curMem=408265, maxMem=766075207 -2016-06-13 21:00:41 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 38.9 KB, free 730.2 MB) -2016-06-13 21:00:41 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52520 (size: 38.9 KB, free: 730.4 MB) -2016-06-13 21:00:41 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:00:41 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[10] at map at PCALabelPointSuite.scala:32) -2016-06-13 21:00:41 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-06-13 21:00:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 10, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:00:41 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:00:41 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 10) -2016-06-13 21:00:41 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 11) -2016-06-13 21:00:41 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:00:41 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:00:41 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 11). 2044 bytes result sent to driver -2016-06-13 21:00:41 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 10). 2044 bytes result sent to driver -2016-06-13 21:00:41 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 11) in 28 ms on localhost (1/2) -2016-06-13 21:00:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 10) in 31 ms on localhost (2/2) -2016-06-13 21:00:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 21:00:41 INFO DAGScheduler:59 - ResultStage 6 (foreach at PCALabelPointSuite.scala:41) finished in 0.034 s -2016-06-13 21:00:41 INFO DAGScheduler:59 - Job 6 finished: foreach at PCALabelPointSuite.scala:41, took 0.082566 s -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 21:00:41 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 21:00:41 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 21:00:41 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 21:00:41 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 21:00:41 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 21:00:41 INFO BlockManager:59 - BlockManager stopped -2016-06-13 21:00:41 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 21:00:41 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 21:00:41 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 21:00:41 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 21:00:41 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 21:00:41 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 21:00:41 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-49050aff-16a6-4ed8-bc71-dc50c32a32c3 -2016-06-13 21:03:17 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 21:03:18 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 21:03:18 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 21:03:18 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 21:03:18 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 21:03:19 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 21:03:19 INFO Remoting:74 - Starting remoting -2016-06-13 21:03:19 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:52547] -2016-06-13 21:03:19 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52547. -2016-06-13 21:03:19 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 21:03:19 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 21:03:19 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-0a2b470e-bdde-491a-a1ac-8501407f6e23 -2016-06-13 21:03:19 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 21:03:19 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-d5a99203-b810-495e-8206-2a159ba8a771\httpd-3f67acd5-a693-44b9-adb9-72f7844749a6 -2016-06-13 21:03:19 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 21:03:19 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:03:19 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52548 -2016-06-13 21:03:19 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52548. -2016-06-13 21:03:19 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 21:03:19 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:03:19 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 21:03:19 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 21:03:19 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 21:03:19 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 21:03:19 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 21:03:20 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52567. -2016-06-13 21:03:20 INFO NettyBlockTransferService:59 - Server created on 52567 -2016-06-13 21:03:20 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 21:03:20 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52567 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52567) -2016-06-13 21:03:20 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 21:03:21 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 21:03:21 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 21:03:21 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 21:03:21 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 21:03:21 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52567 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 21:03:21 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-06-13 21:03:22 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 21:03:22 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 21:03:22 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-06-13 21:03:22 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-06-13 21:03:22 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-06-13 21:03:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:03:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:03:22 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-06-13 21:03:22 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-06-13 21:03:22 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 21:03:22 INFO MemoryStore:59 - ensureFreeSpace(2140) called with curMem=175476, maxMem=766075207 -2016-06-13 21:03:22 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 21:03:22 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52567 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 21:03:22 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:03:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-06-13 21:03:22 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 21:03:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:03:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:03:22 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 21:03:22 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 21:03:22 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-06-13 21:03:22 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-06-13 21:03:22 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/sample_libsvm_data.txt:0+52368 -2016-06-13 21:03:22 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/sample_libsvm_data.txt:52368+52368 -2016-06-13 21:03:22 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 21:03:22 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 21:03:22 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 21:03:22 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 21:03:22 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 21:03:22 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177616, maxMem=766075207 -2016-06-13 21:03:22 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-06-13 21:03:22 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:52567 (size: 83.1 KB, free: 730.5 MB) -2016-06-13 21:03:23 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262696, maxMem=766075207 -2016-06-13 21:03:23 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-06-13 21:03:23 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:52567 (size: 83.6 KB, free: 730.4 MB) -2016-06-13 21:03:23 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-06-13 21:03:23 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-06-13 21:03:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 360 ms on localhost (1/2) -2016-06-13 21:03:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 389 ms on localhost (2/2) -2016-06-13 21:03:23 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.403 s -2016-06-13 21:03:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 21:03:23 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.559385 s -2016-06-13 21:03:23 INFO SparkContext:59 - Starting job: first at PCA.scala:42 -2016-06-13 21:03:23 INFO DAGScheduler:59 - Got job 1 (first at PCA.scala:42) with 1 output partitions -2016-06-13 21:03:23 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at PCA.scala:42) -2016-06-13 21:03:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:03:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:03:23 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29), which has no missing parents -2016-06-13 21:03:23 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=348344, maxMem=766075207 -2016-06-13 21:03:23 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.1 KB, free 730.3 MB) -2016-06-13 21:03:23 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=352536, maxMem=766075207 -2016-06-13 21:03:23 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 21:03:23 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52567 (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:03:23 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:03:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29) -2016-06-13 21:03:23 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 21:03:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:03:23 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 21:03:23 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:03:23 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 4334 bytes result sent to driver -2016-06-13 21:03:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 19 ms on localhost (1/1) -2016-06-13 21:03:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 21:03:23 INFO DAGScheduler:59 - ResultStage 1 (first at PCA.scala:42) finished in 0.019 s -2016-06-13 21:03:23 INFO DAGScheduler:59 - Job 1 finished: first at PCA.scala:42, took 0.038654 s -2016-06-13 21:03:23 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 21:03:23 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 21:03:23 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-06-13 21:03:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:03:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:03:23 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29), which has no missing parents -2016-06-13 21:03:23 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=354804, maxMem=766075207 -2016-06-13 21:03:23 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-06-13 21:03:23 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=358996, maxMem=766075207 -2016-06-13 21:03:23 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 21:03:23 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52567 (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:03:23 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:03:23 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29) -2016-06-13 21:03:23 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-06-13 21:03:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:03:23 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 21:03:23 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:03:23 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 4334 bytes result sent to driver -2016-06-13 21:03:23 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 12 ms on localhost (1/1) -2016-06-13 21:03:23 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.013 s -2016-06-13 21:03:23 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 21:03:23 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.025918 s -2016-06-13 21:03:23 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-06-13 21:03:23 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-06-13 21:03:23 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-06-13 21:03:23 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:03:23 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:03:23 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-06-13 21:03:23 INFO MemoryStore:59 - ensureFreeSpace(10616) called with curMem=361264, maxMem=766075207 -2016-06-13 21:03:23 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 10.4 KB, free 730.2 MB) -2016-06-13 21:03:23 INFO MemoryStore:59 - ensureFreeSpace(2925) called with curMem=371880, maxMem=766075207 -2016-06-13 21:03:23 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.2 MB) -2016-06-13 21:03:23 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52567 (size: 2.9 KB, free: 730.4 MB) -2016-06-13 21:03:23 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:03:23 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-06-13 21:03:23 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 21:03:23 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 4, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:03:23 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:03:23 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 5) -2016-06-13 21:03:23 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:03:23 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 4) -2016-06-13 21:03:23 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:03:23 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52567 in memory (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:03:23 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 21:03:23 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52567 in memory (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:03:23 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 21:03:23 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52567 in memory (size: 2.1 KB, free: 730.4 MB) -2016-06-13 21:03:23 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 21:03:24 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 4). 7905 bytes result sent to driver -2016-06-13 21:03:24 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 5). 7905 bytes result sent to driver -2016-06-13 21:03:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 4) in 596 ms on localhost (1/2) -2016-06-13 21:03:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 5) in 596 ms on localhost (2/2) -2016-06-13 21:03:24 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.598 s -2016-06-13 21:03:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 21:03:24 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 21:03:24 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 21:03:24 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.632452 s -2016-06-13 21:03:24 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-06-13 21:03:24 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-06-13 21:03:24 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-06-13 21:03:24 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:03:24 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:03:24 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-06-13 21:03:24 INFO MemoryStore:59 - ensureFreeSpace(1923344) called with curMem=355849, maxMem=766075207 -2016-06-13 21:03:24 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 1878.3 KB, free 728.4 MB) -2016-06-13 21:03:24 INFO MemoryStore:59 - ensureFreeSpace(93166) called with curMem=2279193, maxMem=766075207 -2016-06-13 21:03:24 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 91.0 KB, free 728.3 MB) -2016-06-13 21:03:24 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52567 (size: 91.0 KB, free: 730.3 MB) -2016-06-13 21:03:24 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:03:24 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-06-13 21:03:24 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 21:03:24 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 6, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:03:24 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:03:24 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 7) -2016-06-13 21:03:24 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 6) -2016-06-13 21:03:24 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:03:24 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:03:24 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 6). 1929806 bytes result sent to driver -2016-06-13 21:03:24 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 7). 1929806 bytes result sent to driver -2016-06-13 21:03:24 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 7) in 64 ms on localhost (1/2) -2016-06-13 21:03:24 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 6) in 66 ms on localhost (2/2) -2016-06-13 21:03:24 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.066 s -2016-06-13 21:03:24 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 21:03:24 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.110271 s -2016-06-13 21:03:24 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52567 in memory (size: 91.0 KB, free: 730.4 MB) -2016-06-13 21:03:24 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 21:03:24 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52567 in memory (size: 2.9 KB, free: 730.4 MB) -2016-06-13 21:03:24 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 21:03:24 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 21:03:24 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 21:03:25 INFO SparkContext:59 - Starting job: count at PCALabelPointSuite.scala:34 -2016-06-13 21:03:25 INFO DAGScheduler:59 - Got job 5 (count at PCALabelPointSuite.scala:34) with 2 output partitions -2016-06-13 21:03:25 INFO DAGScheduler:59 - Final stage: ResultStage 5(count at PCALabelPointSuite.scala:34) -2016-06-13 21:03:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:03:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:03:25 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-06-13 21:03:25 INFO MemoryStore:59 - ensureFreeSpace(3808) called with curMem=342308, maxMem=766075207 -2016-06-13 21:03:25 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.7 KB, free 730.3 MB) -2016-06-13 21:03:25 INFO MemoryStore:59 - ensureFreeSpace(2136) called with curMem=346116, maxMem=766075207 -2016-06-13 21:03:25 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 21:03:25 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52567 (size: 2.1 KB, free: 730.4 MB) -2016-06-13 21:03:25 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:03:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-06-13 21:03:25 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 21:03:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 8, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:03:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:03:26 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 9) -2016-06-13 21:03:26 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 8) -2016-06-13 21:03:26 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:03:26 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 9). 2082 bytes result sent to driver -2016-06-13 21:03:26 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:03:26 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 8). 2082 bytes result sent to driver -2016-06-13 21:03:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 9) in 12 ms on localhost (1/2) -2016-06-13 21:03:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 8) in 15 ms on localhost (2/2) -2016-06-13 21:03:26 INFO DAGScheduler:59 - ResultStage 5 (count at PCALabelPointSuite.scala:34) finished in 0.016 s -2016-06-13 21:03:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 21:03:26 INFO DAGScheduler:59 - Job 5 finished: count at PCALabelPointSuite.scala:34, took 0.042710 s -2016-06-13 21:03:26 INFO SparkContext:59 - Starting job: take at PCALabelPointSuite.scala:35 -2016-06-13 21:03:26 INFO DAGScheduler:59 - Got job 6 (take at PCALabelPointSuite.scala:35) with 1 output partitions -2016-06-13 21:03:26 INFO DAGScheduler:59 - Final stage: ResultStage 6(take at PCALabelPointSuite.scala:35) -2016-06-13 21:03:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:03:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:03:26 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-06-13 21:03:26 INFO MemoryStore:59 - ensureFreeSpace(3968) called with curMem=348252, maxMem=766075207 -2016-06-13 21:03:26 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.9 KB, free 730.3 MB) -2016-06-13 21:03:26 INFO MemoryStore:59 - ensureFreeSpace(2197) called with curMem=352220, maxMem=766075207 -2016-06-13 21:03:26 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.2 MB) -2016-06-13 21:03:26 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52567 (size: 2.1 KB, free: 730.4 MB) -2016-06-13 21:03:26 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:03:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 6 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-06-13 21:03:26 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 1 tasks -2016-06-13 21:03:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 10, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:03:26 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 10) -2016-06-13 21:03:26 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:03:26 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 10). 18702 bytes result sent to driver -2016-06-13 21:03:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 10) in 10 ms on localhost (1/1) -2016-06-13 21:03:26 INFO DAGScheduler:59 - ResultStage 6 (take at PCALabelPointSuite.scala:35) finished in 0.011 s -2016-06-13 21:03:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 21:03:26 INFO DAGScheduler:59 - Job 6 finished: take at PCALabelPointSuite.scala:35, took 0.022944 s -2016-06-13 21:03:26 INFO SparkContext:59 - Starting job: count at PCALabelPointSuite.scala:40 -2016-06-13 21:03:26 INFO DAGScheduler:59 - Got job 7 (count at PCALabelPointSuite.scala:40) with 2 output partitions -2016-06-13 21:03:26 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at PCALabelPointSuite.scala:40) -2016-06-13 21:03:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:03:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:03:26 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[10] at map at PCALabelPointSuite.scala:32), which has no missing parents -2016-06-13 21:03:26 INFO MemoryStore:59 - ensureFreeSpace(59640) called with curMem=354417, maxMem=766075207 -2016-06-13 21:03:26 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 58.2 KB, free 730.2 MB) -2016-06-13 21:03:26 INFO MemoryStore:59 - ensureFreeSpace(39776) called with curMem=414057, maxMem=766075207 -2016-06-13 21:03:26 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 38.8 KB, free 730.2 MB) -2016-06-13 21:03:26 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52567 (size: 38.8 KB, free: 730.4 MB) -2016-06-13 21:03:26 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:03:26 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[10] at map at PCALabelPointSuite.scala:32) -2016-06-13 21:03:26 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-06-13 21:03:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 11, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:03:26 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 12, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:03:26 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 11) -2016-06-13 21:03:26 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 12) -2016-06-13 21:03:26 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:03:26 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:03:26 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 12). 2082 bytes result sent to driver -2016-06-13 21:03:26 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 11). 2082 bytes result sent to driver -2016-06-13 21:03:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 12) in 93 ms on localhost (1/2) -2016-06-13 21:03:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 11) in 97 ms on localhost (2/2) -2016-06-13 21:03:26 INFO DAGScheduler:59 - ResultStage 7 (count at PCALabelPointSuite.scala:40) finished in 0.099 s -2016-06-13 21:03:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 21:03:26 INFO DAGScheduler:59 - Job 7 finished: count at PCALabelPointSuite.scala:40, took 0.117399 s -2016-06-13 21:03:26 INFO SparkContext:59 - Starting job: take at PCALabelPointSuite.scala:41 -2016-06-13 21:03:26 INFO DAGScheduler:59 - Got job 8 (take at PCALabelPointSuite.scala:41) with 1 output partitions -2016-06-13 21:03:26 INFO DAGScheduler:59 - Final stage: ResultStage 8(take at PCALabelPointSuite.scala:41) -2016-06-13 21:03:26 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:03:26 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:03:26 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[10] at map at PCALabelPointSuite.scala:32), which has no missing parents -2016-06-13 21:03:26 INFO MemoryStore:59 - ensureFreeSpace(59808) called with curMem=453833, maxMem=766075207 -2016-06-13 21:03:26 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 58.4 KB, free 730.1 MB) -2016-06-13 21:03:26 INFO MemoryStore:59 - ensureFreeSpace(39857) called with curMem=513641, maxMem=766075207 -2016-06-13 21:03:26 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 38.9 KB, free 730.1 MB) -2016-06-13 21:03:26 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52567 (size: 38.9 KB, free: 730.3 MB) -2016-06-13 21:03:26 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:03:26 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[10] at map at PCALabelPointSuite.scala:32) -2016-06-13 21:03:26 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-06-13 21:03:26 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 13, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:03:26 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 13) -2016-06-13 21:03:26 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:03:26 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 13). 3378 bytes result sent to driver -2016-06-13 21:03:26 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 13) in 14 ms on localhost (1/1) -2016-06-13 21:03:26 INFO DAGScheduler:59 - ResultStage 8 (take at PCALabelPointSuite.scala:41) finished in 0.015 s -2016-06-13 21:03:26 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 21:03:26 INFO DAGScheduler:59 - Job 8 finished: take at PCALabelPointSuite.scala:41, took 0.033475 s -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 21:03:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 21:03:26 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 21:03:26 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 21:03:26 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 21:03:26 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 21:03:26 INFO BlockManager:59 - BlockManager stopped -2016-06-13 21:03:26 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 21:03:26 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 21:03:26 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 21:03:26 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 21:03:26 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 21:03:26 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 21:03:26 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-d5a99203-b810-495e-8206-2a159ba8a771 -2016-06-13 21:03:26 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 21:04:26 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 21:04:27 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 21:04:27 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 21:04:27 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 21:04:27 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 21:04:28 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 21:04:28 INFO Remoting:74 - Starting remoting -2016-06-13 21:04:28 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:52589] -2016-06-13 21:04:28 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52589. -2016-06-13 21:04:28 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 21:04:28 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 21:04:28 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-3543a5ea-f860-435a-8e85-3f0b5f430dbe -2016-06-13 21:04:28 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 21:04:28 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-98ea8fc1-0627-4b62-a690-0d7a32fc0871\httpd-23786193-82a2-48fa-9d79-9c841b3596e7 -2016-06-13 21:04:29 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 21:04:29 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:04:29 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52590 -2016-06-13 21:04:29 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52590. -2016-06-13 21:04:29 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 21:04:29 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:04:29 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 21:04:29 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 21:04:29 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 21:04:29 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 21:04:29 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 21:04:29 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52609. -2016-06-13 21:04:29 INFO NettyBlockTransferService:59 - Server created on 52609 -2016-06-13 21:04:29 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 21:04:29 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52609 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52609) -2016-06-13 21:04:29 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 21:04:30 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 21:04:30 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 21:04:30 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 21:04:30 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 21:04:30 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52609 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 21:04:30 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-06-13 21:04:31 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 21:04:31 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 21:04:31 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-06-13 21:04:31 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-06-13 21:04:31 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-06-13 21:04:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:04:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:04:31 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-06-13 21:04:31 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-06-13 21:04:31 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 21:04:31 INFO MemoryStore:59 - ensureFreeSpace(2140) called with curMem=175476, maxMem=766075207 -2016-06-13 21:04:31 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 21:04:31 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52609 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 21:04:31 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:04:31 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-06-13 21:04:31 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 21:04:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:04:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:04:32 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 21:04:32 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 21:04:32 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-06-13 21:04:32 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-06-13 21:04:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/sample_libsvm_data.txt:52368+52368 -2016-06-13 21:04:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/sample_libsvm_data.txt:0+52368 -2016-06-13 21:04:32 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 21:04:32 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 21:04:32 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 21:04:32 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 21:04:32 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 21:04:32 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177616, maxMem=766075207 -2016-06-13 21:04:32 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-06-13 21:04:32 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263264, maxMem=766075207 -2016-06-13 21:04:32 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-06-13 21:04:32 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:52609 (size: 83.6 KB, free: 730.5 MB) -2016-06-13 21:04:32 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:52609 (size: 83.1 KB, free: 730.4 MB) -2016-06-13 21:04:32 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-06-13 21:04:32 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-06-13 21:04:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 370 ms on localhost (1/2) -2016-06-13 21:04:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 399 ms on localhost (2/2) -2016-06-13 21:04:32 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.413 s -2016-06-13 21:04:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 21:04:32 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.548057 s -2016-06-13 21:04:32 INFO SparkContext:59 - Starting job: first at PCA.scala:42 -2016-06-13 21:04:32 INFO DAGScheduler:59 - Got job 1 (first at PCA.scala:42) with 1 output partitions -2016-06-13 21:04:32 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at PCA.scala:42) -2016-06-13 21:04:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:04:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:04:32 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29), which has no missing parents -2016-06-13 21:04:32 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=348344, maxMem=766075207 -2016-06-13 21:04:32 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.1 KB, free 730.3 MB) -2016-06-13 21:04:32 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=352536, maxMem=766075207 -2016-06-13 21:04:32 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 21:04:32 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52609 (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:04:32 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:04:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29) -2016-06-13 21:04:32 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 21:04:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:04:32 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 21:04:32 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:04:32 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 4334 bytes result sent to driver -2016-06-13 21:04:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 17 ms on localhost (1/1) -2016-06-13 21:04:32 INFO DAGScheduler:59 - ResultStage 1 (first at PCA.scala:42) finished in 0.018 s -2016-06-13 21:04:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 21:04:32 INFO DAGScheduler:59 - Job 1 finished: first at PCA.scala:42, took 0.040871 s -2016-06-13 21:04:32 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 21:04:32 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 21:04:32 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-06-13 21:04:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:04:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:04:32 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29), which has no missing parents -2016-06-13 21:04:32 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=354804, maxMem=766075207 -2016-06-13 21:04:32 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-06-13 21:04:32 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=358996, maxMem=766075207 -2016-06-13 21:04:32 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 21:04:32 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52609 (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:04:32 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:04:32 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29) -2016-06-13 21:04:32 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-06-13 21:04:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:04:32 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 21:04:32 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:04:32 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 4334 bytes result sent to driver -2016-06-13 21:04:32 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.012 s -2016-06-13 21:04:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 12 ms on localhost (1/1) -2016-06-13 21:04:32 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.025646 s -2016-06-13 21:04:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 21:04:32 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-06-13 21:04:32 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-06-13 21:04:32 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-06-13 21:04:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:04:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:04:32 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-06-13 21:04:32 INFO MemoryStore:59 - ensureFreeSpace(10616) called with curMem=361264, maxMem=766075207 -2016-06-13 21:04:32 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 10.4 KB, free 730.2 MB) -2016-06-13 21:04:32 INFO MemoryStore:59 - ensureFreeSpace(2925) called with curMem=371880, maxMem=766075207 -2016-06-13 21:04:32 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.2 MB) -2016-06-13 21:04:32 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52609 (size: 2.9 KB, free: 730.4 MB) -2016-06-13 21:04:32 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:04:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-06-13 21:04:32 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 21:04:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 4, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:04:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:04:32 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 4) -2016-06-13 21:04:32 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:04:32 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 5) -2016-06-13 21:04:32 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:04:33 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52609 in memory (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:04:33 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 21:04:33 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52609 in memory (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:04:33 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 21:04:33 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52609 in memory (size: 2.1 KB, free: 730.4 MB) -2016-06-13 21:04:33 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 21:04:33 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 4). 7905 bytes result sent to driver -2016-06-13 21:04:33 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 5). 7905 bytes result sent to driver -2016-06-13 21:04:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 4) in 615 ms on localhost (1/2) -2016-06-13 21:04:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 5) in 614 ms on localhost (2/2) -2016-06-13 21:04:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 21:04:33 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.617 s -2016-06-13 21:04:33 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 21:04:33 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 21:04:33 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.659233 s -2016-06-13 21:04:33 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-06-13 21:04:33 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-06-13 21:04:33 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-06-13 21:04:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:04:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:04:33 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-06-13 21:04:33 INFO MemoryStore:59 - ensureFreeSpace(1923344) called with curMem=355849, maxMem=766075207 -2016-06-13 21:04:33 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 1878.3 KB, free 728.4 MB) -2016-06-13 21:04:33 INFO MemoryStore:59 - ensureFreeSpace(93166) called with curMem=2279193, maxMem=766075207 -2016-06-13 21:04:33 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 91.0 KB, free 728.3 MB) -2016-06-13 21:04:33 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52609 (size: 91.0 KB, free: 730.3 MB) -2016-06-13 21:04:33 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:04:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-06-13 21:04:33 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 21:04:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 6, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:04:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:04:33 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 6) -2016-06-13 21:04:33 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 7) -2016-06-13 21:04:33 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:04:33 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:04:33 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 7). 1929806 bytes result sent to driver -2016-06-13 21:04:33 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 6). 1929806 bytes result sent to driver -2016-06-13 21:04:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 7) in 65 ms on localhost (1/2) -2016-06-13 21:04:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 6) in 67 ms on localhost (2/2) -2016-06-13 21:04:33 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.067 s -2016-06-13 21:04:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 21:04:33 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.113420 s -2016-06-13 21:04:33 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52609 in memory (size: 91.0 KB, free: 730.4 MB) -2016-06-13 21:04:33 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 21:04:33 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52609 in memory (size: 2.9 KB, free: 730.4 MB) -2016-06-13 21:04:33 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 21:04:33 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 21:04:33 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 21:04:35 INFO SparkContext:59 - Starting job: count at PCALabelPointSuite.scala:34 -2016-06-13 21:04:35 INFO DAGScheduler:59 - Got job 5 (count at PCALabelPointSuite.scala:34) with 2 output partitions -2016-06-13 21:04:35 INFO DAGScheduler:59 - Final stage: ResultStage 5(count at PCALabelPointSuite.scala:34) -2016-06-13 21:04:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:04:35 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:04:35 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-06-13 21:04:35 INFO MemoryStore:59 - ensureFreeSpace(3808) called with curMem=342308, maxMem=766075207 -2016-06-13 21:04:35 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.7 KB, free 730.3 MB) -2016-06-13 21:04:35 INFO MemoryStore:59 - ensureFreeSpace(2136) called with curMem=346116, maxMem=766075207 -2016-06-13 21:04:35 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 21:04:35 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52609 (size: 2.1 KB, free: 730.4 MB) -2016-06-13 21:04:35 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:04:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-06-13 21:04:35 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 21:04:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 8, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:04:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:04:35 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 8) -2016-06-13 21:04:35 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 9) -2016-06-13 21:04:35 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:04:35 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:04:35 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 9). 2082 bytes result sent to driver -2016-06-13 21:04:35 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 8). 2082 bytes result sent to driver -2016-06-13 21:04:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 9) in 10 ms on localhost (1/2) -2016-06-13 21:04:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 8) in 11 ms on localhost (2/2) -2016-06-13 21:04:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 21:04:35 INFO DAGScheduler:59 - ResultStage 5 (count at PCALabelPointSuite.scala:34) finished in 0.012 s -2016-06-13 21:04:35 INFO DAGScheduler:59 - Job 5 finished: count at PCALabelPointSuite.scala:34, took 0.030598 s -2016-06-13 21:04:35 INFO SparkContext:59 - Starting job: take at PCALabelPointSuite.scala:35 -2016-06-13 21:04:35 INFO DAGScheduler:59 - Got job 6 (take at PCALabelPointSuite.scala:35) with 1 output partitions -2016-06-13 21:04:35 INFO DAGScheduler:59 - Final stage: ResultStage 6(take at PCALabelPointSuite.scala:35) -2016-06-13 21:04:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:04:35 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:04:35 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-06-13 21:04:35 INFO MemoryStore:59 - ensureFreeSpace(3968) called with curMem=348252, maxMem=766075207 -2016-06-13 21:04:35 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.9 KB, free 730.3 MB) -2016-06-13 21:04:35 INFO MemoryStore:59 - ensureFreeSpace(2197) called with curMem=352220, maxMem=766075207 -2016-06-13 21:04:35 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.2 MB) -2016-06-13 21:04:35 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52609 (size: 2.1 KB, free: 730.4 MB) -2016-06-13 21:04:35 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:04:35 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 6 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-06-13 21:04:35 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 1 tasks -2016-06-13 21:04:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 10, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:04:35 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 10) -2016-06-13 21:04:35 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:04:35 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 10). 18702 bytes result sent to driver -2016-06-13 21:04:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 10) in 12 ms on localhost (1/1) -2016-06-13 21:04:35 INFO DAGScheduler:59 - ResultStage 6 (take at PCALabelPointSuite.scala:35) finished in 0.012 s -2016-06-13 21:04:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 21:04:35 INFO DAGScheduler:59 - Job 6 finished: take at PCALabelPointSuite.scala:35, took 0.024784 s -2016-06-13 21:04:35 INFO SparkContext:59 - Starting job: count at PCALabelPointSuite.scala:41 -2016-06-13 21:04:35 INFO DAGScheduler:59 - Got job 7 (count at PCALabelPointSuite.scala:41) with 2 output partitions -2016-06-13 21:04:35 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at PCALabelPointSuite.scala:41) -2016-06-13 21:04:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:04:35 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:04:35 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[10] at map at PCALabelPointSuite.scala:32), which has no missing parents -2016-06-13 21:04:35 INFO MemoryStore:59 - ensureFreeSpace(59640) called with curMem=354417, maxMem=766075207 -2016-06-13 21:04:35 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 58.2 KB, free 730.2 MB) -2016-06-13 21:04:35 INFO MemoryStore:59 - ensureFreeSpace(39776) called with curMem=414057, maxMem=766075207 -2016-06-13 21:04:35 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 38.8 KB, free 730.2 MB) -2016-06-13 21:04:35 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52609 (size: 38.8 KB, free: 730.4 MB) -2016-06-13 21:04:35 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:04:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[10] at map at PCALabelPointSuite.scala:32) -2016-06-13 21:04:35 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-06-13 21:04:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 11, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:04:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 12, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:04:35 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 11) -2016-06-13 21:04:35 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 12) -2016-06-13 21:04:35 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:04:35 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:04:35 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 12). 2082 bytes result sent to driver -2016-06-13 21:04:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 12) in 23 ms on localhost (1/2) -2016-06-13 21:04:35 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 11). 2082 bytes result sent to driver -2016-06-13 21:04:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 11) in 28 ms on localhost (2/2) -2016-06-13 21:04:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 21:04:35 INFO DAGScheduler:59 - ResultStage 7 (count at PCALabelPointSuite.scala:41) finished in 0.028 s -2016-06-13 21:04:35 INFO DAGScheduler:59 - Job 7 finished: count at PCALabelPointSuite.scala:41, took 0.046558 s -2016-06-13 21:04:35 INFO SparkContext:59 - Starting job: take at PCALabelPointSuite.scala:42 -2016-06-13 21:04:35 INFO DAGScheduler:59 - Got job 8 (take at PCALabelPointSuite.scala:42) with 1 output partitions -2016-06-13 21:04:35 INFO DAGScheduler:59 - Final stage: ResultStage 8(take at PCALabelPointSuite.scala:42) -2016-06-13 21:04:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:04:35 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:04:35 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[10] at map at PCALabelPointSuite.scala:32), which has no missing parents -2016-06-13 21:04:35 INFO MemoryStore:59 - ensureFreeSpace(59808) called with curMem=453833, maxMem=766075207 -2016-06-13 21:04:35 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 58.4 KB, free 730.1 MB) -2016-06-13 21:04:35 INFO MemoryStore:59 - ensureFreeSpace(39857) called with curMem=513641, maxMem=766075207 -2016-06-13 21:04:35 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 38.9 KB, free 730.1 MB) -2016-06-13 21:04:35 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52609 (size: 38.9 KB, free: 730.3 MB) -2016-06-13 21:04:35 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:04:35 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[10] at map at PCALabelPointSuite.scala:32) -2016-06-13 21:04:35 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-06-13 21:04:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 13, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:04:35 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 13) -2016-06-13 21:04:35 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:04:35 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 13). 3378 bytes result sent to driver -2016-06-13 21:04:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 13) in 10 ms on localhost (1/1) -2016-06-13 21:04:35 INFO DAGScheduler:59 - ResultStage 8 (take at PCALabelPointSuite.scala:42) finished in 0.010 s -2016-06-13 21:04:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 21:04:35 INFO DAGScheduler:59 - Job 8 finished: take at PCALabelPointSuite.scala:42, took 0.025628 s -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 21:04:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 21:04:35 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 21:04:35 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 21:04:35 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 21:04:35 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 21:04:35 INFO BlockManager:59 - BlockManager stopped -2016-06-13 21:04:35 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 21:04:35 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 21:04:35 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 21:04:35 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 21:04:35 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 21:04:35 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 21:04:35 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-98ea8fc1-0627-4b62-a690-0d7a32fc0871 -2016-06-13 21:06:57 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 21:06:58 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 21:06:58 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 21:06:58 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 21:06:58 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 21:06:59 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 21:06:59 INFO Remoting:74 - Starting remoting -2016-06-13 21:06:59 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:52635] -2016-06-13 21:06:59 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52635. -2016-06-13 21:06:59 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 21:06:59 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 21:06:59 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-45c957c8-c537-4207-bee0-380901350cfb -2016-06-13 21:06:59 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 21:06:59 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-252dede4-744d-4310-96b5-84aff79de5b0\httpd-a3c4cb8d-2e26-4d1e-9a23-21e7744face5 -2016-06-13 21:06:59 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 21:06:59 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:06:59 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52636 -2016-06-13 21:06:59 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52636. -2016-06-13 21:06:59 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 21:06:59 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:06:59 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 21:06:59 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 21:06:59 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 21:06:59 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 21:06:59 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 21:07:00 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52655. -2016-06-13 21:07:00 INFO NettyBlockTransferService:59 - Server created on 52655 -2016-06-13 21:07:00 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 21:07:00 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52655 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52655) -2016-06-13 21:07:00 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 21:07:01 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 21:07:01 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 21:07:01 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 21:07:01 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 21:07:01 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52655 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 21:07:01 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-06-13 21:07:02 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 21:07:02 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 21:07:02 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-06-13 21:07:02 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-06-13 21:07:02 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-06-13 21:07:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:07:02 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:07:02 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-06-13 21:07:02 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-06-13 21:07:02 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 21:07:02 INFO MemoryStore:59 - ensureFreeSpace(2141) called with curMem=175476, maxMem=766075207 -2016-06-13 21:07:02 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 21:07:02 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52655 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 21:07:02 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:07:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-06-13 21:07:02 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 21:07:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:02 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 21:07:02 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 21:07:02 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-06-13 21:07:02 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-06-13 21:07:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/sample_libsvm_data.txt:0+52368 -2016-06-13 21:07:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/sample_libsvm_data.txt:52368+52368 -2016-06-13 21:07:02 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 21:07:02 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 21:07:02 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 21:07:02 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 21:07:02 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 21:07:02 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=177617, maxMem=766075207 -2016-06-13 21:07:02 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-06-13 21:07:02 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=262697, maxMem=766075207 -2016-06-13 21:07:02 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-06-13 21:07:02 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:52655 (size: 83.1 KB, free: 730.5 MB) -2016-06-13 21:07:02 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:52655 (size: 83.6 KB, free: 730.4 MB) -2016-06-13 21:07:02 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-06-13 21:07:02 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-06-13 21:07:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 364 ms on localhost (1/2) -2016-06-13 21:07:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 384 ms on localhost (2/2) -2016-06-13 21:07:02 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.399 s -2016-06-13 21:07:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 21:07:02 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.512726 s -2016-06-13 21:07:02 INFO SparkContext:59 - Starting job: first at PCA.scala:42 -2016-06-13 21:07:02 INFO DAGScheduler:59 - Got job 1 (first at PCA.scala:42) with 1 output partitions -2016-06-13 21:07:02 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at PCA.scala:42) -2016-06-13 21:07:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:07:02 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:07:02 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29), which has no missing parents -2016-06-13 21:07:02 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=348345, maxMem=766075207 -2016-06-13 21:07:02 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.1 KB, free 730.3 MB) -2016-06-13 21:07:02 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=352537, maxMem=766075207 -2016-06-13 21:07:02 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 21:07:02 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52655 (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:07:02 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:07:02 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29) -2016-06-13 21:07:02 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 21:07:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:02 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 21:07:02 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:07:02 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 4334 bytes result sent to driver -2016-06-13 21:07:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 18 ms on localhost (1/1) -2016-06-13 21:07:02 INFO DAGScheduler:59 - ResultStage 1 (first at PCA.scala:42) finished in 0.018 s -2016-06-13 21:07:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 21:07:02 INFO DAGScheduler:59 - Job 1 finished: first at PCA.scala:42, took 0.034990 s -2016-06-13 21:07:02 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 21:07:02 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 21:07:02 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-06-13 21:07:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:07:02 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:07:02 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29), which has no missing parents -2016-06-13 21:07:02 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=354805, maxMem=766075207 -2016-06-13 21:07:02 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-06-13 21:07:02 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=358997, maxMem=766075207 -2016-06-13 21:07:02 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 21:07:02 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52655 (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:07:02 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:07:02 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29) -2016-06-13 21:07:02 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-06-13 21:07:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:02 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 21:07:03 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:07:03 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 4334 bytes result sent to driver -2016-06-13 21:07:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 13 ms on localhost (1/1) -2016-06-13 21:07:03 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.014 s -2016-06-13 21:07:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 21:07:03 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.029804 s -2016-06-13 21:07:03 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-06-13 21:07:03 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-06-13 21:07:03 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-06-13 21:07:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:07:03 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:07:03 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-06-13 21:07:03 INFO MemoryStore:59 - ensureFreeSpace(10616) called with curMem=361265, maxMem=766075207 -2016-06-13 21:07:03 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 10.4 KB, free 730.2 MB) -2016-06-13 21:07:03 INFO MemoryStore:59 - ensureFreeSpace(2925) called with curMem=371881, maxMem=766075207 -2016-06-13 21:07:03 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.2 MB) -2016-06-13 21:07:03 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52655 (size: 2.9 KB, free: 730.4 MB) -2016-06-13 21:07:03 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:07:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-06-13 21:07:03 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 21:07:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 4, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:03 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 4) -2016-06-13 21:07:03 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 5) -2016-06-13 21:07:03 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:07:03 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:07:03 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52655 in memory (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:07:03 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 21:07:03 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52655 in memory (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:07:03 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 21:07:03 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52655 in memory (size: 2.1 KB, free: 730.4 MB) -2016-06-13 21:07:03 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 21:07:03 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 5). 7905 bytes result sent to driver -2016-06-13 21:07:03 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 4). 7905 bytes result sent to driver -2016-06-13 21:07:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 5) in 617 ms on localhost (1/2) -2016-06-13 21:07:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 4) in 620 ms on localhost (2/2) -2016-06-13 21:07:03 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.621 s -2016-06-13 21:07:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 21:07:03 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 21:07:03 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 21:07:03 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.652979 s -2016-06-13 21:07:03 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-06-13 21:07:03 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-06-13 21:07:03 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-06-13 21:07:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:07:03 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:07:03 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-06-13 21:07:03 INFO MemoryStore:59 - ensureFreeSpace(1923344) called with curMem=355849, maxMem=766075207 -2016-06-13 21:07:03 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 1878.3 KB, free 728.4 MB) -2016-06-13 21:07:03 INFO MemoryStore:59 - ensureFreeSpace(93166) called with curMem=2279193, maxMem=766075207 -2016-06-13 21:07:03 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 91.0 KB, free 728.3 MB) -2016-06-13 21:07:04 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52655 (size: 91.0 KB, free: 730.3 MB) -2016-06-13 21:07:04 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:07:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-06-13 21:07:04 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 21:07:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 6, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:04 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 7) -2016-06-13 21:07:04 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 6) -2016-06-13 21:07:04 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:07:04 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:07:04 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 6). 1929806 bytes result sent to driver -2016-06-13 21:07:04 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 7). 1929806 bytes result sent to driver -2016-06-13 21:07:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 6) in 60 ms on localhost (1/2) -2016-06-13 21:07:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 7) in 60 ms on localhost (2/2) -2016-06-13 21:07:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 21:07:04 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.061 s -2016-06-13 21:07:04 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.107974 s -2016-06-13 21:07:04 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52655 in memory (size: 91.0 KB, free: 730.4 MB) -2016-06-13 21:07:04 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 21:07:04 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52655 in memory (size: 2.9 KB, free: 730.4 MB) -2016-06-13 21:07:04 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 21:07:04 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 21:07:04 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 21:07:05 INFO SparkContext:59 - Starting job: count at PCALabelPointSuite.scala:34 -2016-06-13 21:07:05 INFO DAGScheduler:59 - Got job 5 (count at PCALabelPointSuite.scala:34) with 2 output partitions -2016-06-13 21:07:05 INFO DAGScheduler:59 - Final stage: ResultStage 5(count at PCALabelPointSuite.scala:34) -2016-06-13 21:07:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:07:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:07:05 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-06-13 21:07:05 INFO MemoryStore:59 - ensureFreeSpace(3808) called with curMem=342308, maxMem=766075207 -2016-06-13 21:07:05 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.7 KB, free 730.3 MB) -2016-06-13 21:07:05 INFO MemoryStore:59 - ensureFreeSpace(2137) called with curMem=346116, maxMem=766075207 -2016-06-13 21:07:05 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 21:07:05 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52655 (size: 2.1 KB, free: 730.4 MB) -2016-06-13 21:07:05 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:07:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-06-13 21:07:05 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 21:07:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 8, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:05 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 8) -2016-06-13 21:07:05 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 9) -2016-06-13 21:07:05 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:07:05 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 8). 2082 bytes result sent to driver -2016-06-13 21:07:05 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:07:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 8) in 10 ms on localhost (1/2) -2016-06-13 21:07:05 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 9). 2082 bytes result sent to driver -2016-06-13 21:07:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 9) in 11 ms on localhost (2/2) -2016-06-13 21:07:05 INFO DAGScheduler:59 - ResultStage 5 (count at PCALabelPointSuite.scala:34) finished in 0.012 s -2016-06-13 21:07:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 21:07:05 INFO DAGScheduler:59 - Job 5 finished: count at PCALabelPointSuite.scala:34, took 0.037486 s -2016-06-13 21:07:05 INFO SparkContext:59 - Starting job: foreach at PCALabelPointSuite.scala:35 -2016-06-13 21:07:05 INFO DAGScheduler:59 - Got job 6 (foreach at PCALabelPointSuite.scala:35) with 2 output partitions -2016-06-13 21:07:05 INFO DAGScheduler:59 - Final stage: ResultStage 6(foreach at PCALabelPointSuite.scala:35) -2016-06-13 21:07:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:07:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:07:05 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-06-13 21:07:05 INFO MemoryStore:59 - ensureFreeSpace(3960) called with curMem=348253, maxMem=766075207 -2016-06-13 21:07:05 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.9 KB, free 730.3 MB) -2016-06-13 21:07:05 INFO MemoryStore:59 - ensureFreeSpace(2206) called with curMem=352213, maxMem=766075207 -2016-06-13 21:07:05 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 21:07:05 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52655 (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:07:05 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:07:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-06-13 21:07:05 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-06-13 21:07:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 10, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 11, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:05 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 10) -2016-06-13 21:07:05 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 11) -2016-06-13 21:07:05 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:07:05 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:07:05 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 10). 2044 bytes result sent to driver -2016-06-13 21:07:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 10) in 62 ms on localhost (1/2) -2016-06-13 21:07:05 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 11). 2044 bytes result sent to driver -2016-06-13 21:07:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 11) in 66 ms on localhost (2/2) -2016-06-13 21:07:05 INFO DAGScheduler:59 - ResultStage 6 (foreach at PCALabelPointSuite.scala:35) finished in 0.067 s -2016-06-13 21:07:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 21:07:05 INFO DAGScheduler:59 - Job 6 finished: foreach at PCALabelPointSuite.scala:35, took 0.081719 s -2016-06-13 21:07:05 INFO SparkContext:59 - Starting job: count at PCALabelPointSuite.scala:41 -2016-06-13 21:07:05 INFO DAGScheduler:59 - Got job 7 (count at PCALabelPointSuite.scala:41) with 2 output partitions -2016-06-13 21:07:05 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at PCALabelPointSuite.scala:41) -2016-06-13 21:07:05 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:07:05 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:07:05 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[10] at map at PCALabelPointSuite.scala:32), which has no missing parents -2016-06-13 21:07:05 INFO MemoryStore:59 - ensureFreeSpace(59640) called with curMem=354419, maxMem=766075207 -2016-06-13 21:07:05 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 58.2 KB, free 730.2 MB) -2016-06-13 21:07:05 INFO MemoryStore:59 - ensureFreeSpace(39776) called with curMem=414059, maxMem=766075207 -2016-06-13 21:07:05 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 38.8 KB, free 730.2 MB) -2016-06-13 21:07:06 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52655 (size: 38.8 KB, free: 730.4 MB) -2016-06-13 21:07:06 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:07:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[10] at map at PCALabelPointSuite.scala:32) -2016-06-13 21:07:06 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-06-13 21:07:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 12, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 13, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:06 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 12) -2016-06-13 21:07:06 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 13) -2016-06-13 21:07:06 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:07:06 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:07:06 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 12). 2082 bytes result sent to driver -2016-06-13 21:07:06 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 13). 2082 bytes result sent to driver -2016-06-13 21:07:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 12) in 23 ms on localhost (1/2) -2016-06-13 21:07:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 13) in 24 ms on localhost (2/2) -2016-06-13 21:07:06 INFO DAGScheduler:59 - ResultStage 7 (count at PCALabelPointSuite.scala:41) finished in 0.025 s -2016-06-13 21:07:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 21:07:06 INFO DAGScheduler:59 - Job 7 finished: count at PCALabelPointSuite.scala:41, took 0.056156 s -2016-06-13 21:07:06 INFO SparkContext:59 - Starting job: foreach at PCALabelPointSuite.scala:42 -2016-06-13 21:07:06 INFO DAGScheduler:59 - Got job 8 (foreach at PCALabelPointSuite.scala:42) with 2 output partitions -2016-06-13 21:07:06 INFO DAGScheduler:59 - Final stage: ResultStage 8(foreach at PCALabelPointSuite.scala:42) -2016-06-13 21:07:06 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:07:06 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:07:06 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[10] at map at PCALabelPointSuite.scala:32), which has no missing parents -2016-06-13 21:07:06 INFO MemoryStore:59 - ensureFreeSpace(59792) called with curMem=453835, maxMem=766075207 -2016-06-13 21:07:06 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 58.4 KB, free 730.1 MB) -2016-06-13 21:07:06 INFO MemoryStore:59 - ensureFreeSpace(39871) called with curMem=513627, maxMem=766075207 -2016-06-13 21:07:06 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 38.9 KB, free 730.1 MB) -2016-06-13 21:07:06 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52655 (size: 38.9 KB, free: 730.3 MB) -2016-06-13 21:07:06 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:07:06 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[10] at map at PCALabelPointSuite.scala:32) -2016-06-13 21:07:06 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 21:07:06 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 14, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:06 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 15, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:06 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 14) -2016-06-13 21:07:06 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 15) -2016-06-13 21:07:06 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:07:06 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:07:06 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 14). 2044 bytes result sent to driver -2016-06-13 21:07:06 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 15). 2044 bytes result sent to driver -2016-06-13 21:07:06 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 14) in 20 ms on localhost (1/2) -2016-06-13 21:07:06 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 15) in 20 ms on localhost (2/2) -2016-06-13 21:07:06 INFO DAGScheduler:59 - ResultStage 8 (foreach at PCALabelPointSuite.scala:42) finished in 0.021 s -2016-06-13 21:07:06 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 21:07:06 INFO DAGScheduler:59 - Job 8 finished: foreach at PCALabelPointSuite.scala:42, took 0.036303 s -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 21:07:06 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 21:07:06 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 21:07:06 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 21:07:06 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 21:07:06 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 21:07:06 INFO BlockManager:59 - BlockManager stopped -2016-06-13 21:07:06 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 21:07:06 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 21:07:06 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 21:07:06 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 21:07:06 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 21:07:06 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 21:07:06 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-252dede4-744d-4310-96b5-84aff79de5b0 -2016-06-13 21:07:29 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 21:07:29 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 21:07:30 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 21:07:30 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 21:07:30 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 21:07:30 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 21:07:30 INFO Remoting:74 - Starting remoting -2016-06-13 21:07:31 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:52675] -2016-06-13 21:07:31 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52675. -2016-06-13 21:07:31 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 21:07:31 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 21:07:31 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-625eafee-505d-4380-b4f0-bb18c60a2077 -2016-06-13 21:07:31 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 21:07:31 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-691a5fc0-336d-4803-ae38-deb4921fc0c9\httpd-3ed64b8e-11a2-42dc-b4bf-33a2b578fe99 -2016-06-13 21:07:31 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 21:07:31 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:07:31 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52676 -2016-06-13 21:07:31 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52676. -2016-06-13 21:07:31 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 21:07:31 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:07:31 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 21:07:31 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 21:07:31 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 21:07:31 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 21:07:31 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 21:07:32 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52695. -2016-06-13 21:07:32 INFO NettyBlockTransferService:59 - Server created on 52695 -2016-06-13 21:07:32 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 21:07:32 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52695 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52695) -2016-06-13 21:07:32 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 21:07:32 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 21:07:32 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 21:07:32 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 21:07:32 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 21:07:32 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52695 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 21:07:32 INFO SparkContext:59 - Created broadcast 0 from textFile at MLUtils.scala:75 -2016-06-13 21:07:34 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 21:07:34 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 21:07:34 INFO SparkContext:59 - Starting job: reduce at MLUtils.scala:109 -2016-06-13 21:07:34 INFO DAGScheduler:59 - Got job 0 (reduce at MLUtils.scala:109) with 2 output partitions -2016-06-13 21:07:34 INFO DAGScheduler:59 - Final stage: ResultStage 0(reduce at MLUtils.scala:109) -2016-06-13 21:07:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:07:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:07:34 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107), which has no missing parents -2016-06-13 21:07:34 INFO MemoryStore:59 - ensureFreeSpace(3896) called with curMem=171580, maxMem=766075207 -2016-06-13 21:07:34 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.8 KB, free 730.4 MB) -2016-06-13 21:07:34 INFO MemoryStore:59 - ensureFreeSpace(2140) called with curMem=175476, maxMem=766075207 -2016-06-13 21:07:34 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 21:07:34 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52695 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 21:07:34 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:07:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[5] at map at MLUtils.scala:107) -2016-06-13 21:07:34 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 21:07:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:34 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 21:07:34 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 21:07:34 INFO CacheManager:59 - Partition rdd_4_1 not found, computing it -2016-06-13 21:07:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/sample_libsvm_data.txt:52368+52368 -2016-06-13 21:07:34 INFO CacheManager:59 - Partition rdd_4_0 not found, computing it -2016-06-13 21:07:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/basic/sample_libsvm_data.txt:0+52368 -2016-06-13 21:07:34 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 21:07:34 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 21:07:34 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 21:07:34 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 21:07:34 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 21:07:34 INFO MemoryStore:59 - ensureFreeSpace(85648) called with curMem=177616, maxMem=766075207 -2016-06-13 21:07:34 INFO MemoryStore:59 - Block rdd_4_0 stored as values in memory (estimated size 83.6 KB, free 730.3 MB) -2016-06-13 21:07:34 INFO BlockManagerInfo:59 - Added rdd_4_0 in memory on localhost:52695 (size: 83.6 KB, free: 730.5 MB) -2016-06-13 21:07:34 INFO MemoryStore:59 - ensureFreeSpace(85080) called with curMem=263264, maxMem=766075207 -2016-06-13 21:07:34 INFO MemoryStore:59 - Block rdd_4_1 stored as values in memory (estimated size 83.1 KB, free 730.3 MB) -2016-06-13 21:07:34 INFO BlockManagerInfo:59 - Added rdd_4_1 in memory on localhost:52695 (size: 83.1 KB, free: 730.4 MB) -2016-06-13 21:07:34 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2740 bytes result sent to driver -2016-06-13 21:07:34 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2740 bytes result sent to driver -2016-06-13 21:07:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 473 ms on localhost (1/2) -2016-06-13 21:07:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 433 ms on localhost (2/2) -2016-06-13 21:07:34 INFO DAGScheduler:59 - ResultStage 0 (reduce at MLUtils.scala:109) finished in 0.470 s -2016-06-13 21:07:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 21:07:34 INFO DAGScheduler:59 - Job 0 finished: reduce at MLUtils.scala:109, took 0.693044 s -2016-06-13 21:07:35 INFO SparkContext:59 - Starting job: first at PCA.scala:42 -2016-06-13 21:07:35 INFO DAGScheduler:59 - Got job 1 (first at PCA.scala:42) with 1 output partitions -2016-06-13 21:07:35 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at PCA.scala:42) -2016-06-13 21:07:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:07:35 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:07:35 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29), which has no missing parents -2016-06-13 21:07:35 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=348344, maxMem=766075207 -2016-06-13 21:07:35 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.1 KB, free 730.3 MB) -2016-06-13 21:07:35 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=352536, maxMem=766075207 -2016-06-13 21:07:35 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 21:07:35 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52695 (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:07:35 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:07:35 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29) -2016-06-13 21:07:35 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 21:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:35 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 21:07:35 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:07:35 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 4334 bytes result sent to driver -2016-06-13 21:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 24 ms on localhost (1/1) -2016-06-13 21:07:35 INFO DAGScheduler:59 - ResultStage 1 (first at PCA.scala:42) finished in 0.024 s -2016-06-13 21:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 21:07:35 INFO DAGScheduler:59 - Job 1 finished: first at PCA.scala:42, took 0.045521 s -2016-06-13 21:07:35 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 21:07:35 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 21:07:35 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-06-13 21:07:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:07:35 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:07:35 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29), which has no missing parents -2016-06-13 21:07:35 INFO MemoryStore:59 - ensureFreeSpace(4192) called with curMem=354804, maxMem=766075207 -2016-06-13 21:07:35 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.1 KB, free 730.2 MB) -2016-06-13 21:07:35 INFO MemoryStore:59 - ensureFreeSpace(2268) called with curMem=358996, maxMem=766075207 -2016-06-13 21:07:35 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.2 MB) -2016-06-13 21:07:35 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52695 (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:07:35 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:07:35 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[7] at map at PCALabelPointSuite.scala:29) -2016-06-13 21:07:35 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-06-13 21:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:35 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 21:07:35 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:07:35 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 4334 bytes result sent to driver -2016-06-13 21:07:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 13 ms on localhost (1/1) -2016-06-13 21:07:35 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.014 s -2016-06-13 21:07:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 21:07:35 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.030898 s -2016-06-13 21:07:35 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-06-13 21:07:35 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-06-13 21:07:35 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-06-13 21:07:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:07:35 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:07:35 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-06-13 21:07:35 INFO MemoryStore:59 - ensureFreeSpace(10616) called with curMem=361264, maxMem=766075207 -2016-06-13 21:07:35 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 10.4 KB, free 730.2 MB) -2016-06-13 21:07:35 INFO MemoryStore:59 - ensureFreeSpace(2925) called with curMem=371880, maxMem=766075207 -2016-06-13 21:07:35 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.9 KB, free 730.2 MB) -2016-06-13 21:07:35 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52695 (size: 2.9 KB, free: 730.4 MB) -2016-06-13 21:07:35 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:07:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[8] at treeAggregate at RowMatrix.scala:332) -2016-06-13 21:07:35 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 21:07:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 4, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:35 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 4) -2016-06-13 21:07:35 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 5) -2016-06-13 21:07:35 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:07:35 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:07:35 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52695 in memory (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:07:35 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 21:07:35 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52695 in memory (size: 2.2 KB, free: 730.4 MB) -2016-06-13 21:07:35 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 21:07:35 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52695 in memory (size: 2.1 KB, free: 730.4 MB) -2016-06-13 21:07:35 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 21:07:36 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 5). 7905 bytes result sent to driver -2016-06-13 21:07:36 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 4). 7905 bytes result sent to driver -2016-06-13 21:07:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 5) in 642 ms on localhost (1/2) -2016-06-13 21:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 4) in 645 ms on localhost (2/2) -2016-06-13 21:07:36 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.645 s -2016-06-13 21:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 21:07:36 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 21:07:36 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 21:07:36 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.676990 s -2016-06-13 21:07:36 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-06-13 21:07:36 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-06-13 21:07:36 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-06-13 21:07:36 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:07:36 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:07:36 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-06-13 21:07:36 INFO MemoryStore:59 - ensureFreeSpace(1923344) called with curMem=355849, maxMem=766075207 -2016-06-13 21:07:36 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 1878.3 KB, free 728.4 MB) -2016-06-13 21:07:36 INFO MemoryStore:59 - ensureFreeSpace(93166) called with curMem=2279193, maxMem=766075207 -2016-06-13 21:07:36 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 91.0 KB, free 728.3 MB) -2016-06-13 21:07:36 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52695 (size: 91.0 KB, free: 730.3 MB) -2016-06-13 21:07:36 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:07:36 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[9] at treeAggregate at RowMatrix.scala:124) -2016-06-13 21:07:36 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 21:07:36 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 6, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:36 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:36 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 6) -2016-06-13 21:07:36 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 7) -2016-06-13 21:07:36 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:07:36 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:07:36 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 6). 1929806 bytes result sent to driver -2016-06-13 21:07:36 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 7). 1929806 bytes result sent to driver -2016-06-13 21:07:36 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 6) in 64 ms on localhost (1/2) -2016-06-13 21:07:36 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 7) in 65 ms on localhost (2/2) -2016-06-13 21:07:36 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.066 s -2016-06-13 21:07:36 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 21:07:36 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.113285 s -2016-06-13 21:07:36 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52695 in memory (size: 91.0 KB, free: 730.4 MB) -2016-06-13 21:07:36 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 21:07:36 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 21:07:36 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52695 in memory (size: 2.9 KB, free: 730.4 MB) -2016-06-13 21:07:36 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 21:07:36 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 21:07:38 INFO SparkContext:59 - Starting job: count at PCALabelPointSuite.scala:34 -2016-06-13 21:07:38 INFO DAGScheduler:59 - Got job 5 (count at PCALabelPointSuite.scala:34) with 2 output partitions -2016-06-13 21:07:38 INFO DAGScheduler:59 - Final stage: ResultStage 5(count at PCALabelPointSuite.scala:34) -2016-06-13 21:07:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:07:38 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:07:38 INFO DAGScheduler:59 - Submitting ResultStage 5 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-06-13 21:07:38 INFO MemoryStore:59 - ensureFreeSpace(3808) called with curMem=342308, maxMem=766075207 -2016-06-13 21:07:38 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.7 KB, free 730.3 MB) -2016-06-13 21:07:38 INFO MemoryStore:59 - ensureFreeSpace(2136) called with curMem=346116, maxMem=766075207 -2016-06-13 21:07:38 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.3 MB) -2016-06-13 21:07:38 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52695 (size: 2.1 KB, free: 730.4 MB) -2016-06-13 21:07:38 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:07:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 5 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-06-13 21:07:38 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 2 tasks -2016-06-13 21:07:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 8, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 5.0 (TID 9, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:38 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 8) -2016-06-13 21:07:38 INFO Executor:59 - Running task 1.0 in stage 5.0 (TID 9) -2016-06-13 21:07:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:07:38 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 8). 2082 bytes result sent to driver -2016-06-13 21:07:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 8) in 12 ms on localhost (1/2) -2016-06-13 21:07:38 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:07:38 INFO Executor:59 - Finished task 1.0 in stage 5.0 (TID 9). 2082 bytes result sent to driver -2016-06-13 21:07:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 5.0 (TID 9) in 19 ms on localhost (2/2) -2016-06-13 21:07:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 21:07:38 INFO DAGScheduler:59 - ResultStage 5 (count at PCALabelPointSuite.scala:34) finished in 0.020 s -2016-06-13 21:07:38 INFO DAGScheduler:59 - Job 5 finished: count at PCALabelPointSuite.scala:34, took 0.038651 s -2016-06-13 21:07:38 INFO SparkContext:59 - Starting job: take at PCALabelPointSuite.scala:35 -2016-06-13 21:07:38 INFO DAGScheduler:59 - Got job 6 (take at PCALabelPointSuite.scala:35) with 1 output partitions -2016-06-13 21:07:38 INFO DAGScheduler:59 - Final stage: ResultStage 6(take at PCALabelPointSuite.scala:35) -2016-06-13 21:07:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:07:38 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:07:38 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[6] at map at MLUtils.scala:112), which has no missing parents -2016-06-13 21:07:38 INFO MemoryStore:59 - ensureFreeSpace(3968) called with curMem=348252, maxMem=766075207 -2016-06-13 21:07:38 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.9 KB, free 730.3 MB) -2016-06-13 21:07:38 INFO MemoryStore:59 - ensureFreeSpace(2197) called with curMem=352220, maxMem=766075207 -2016-06-13 21:07:38 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.2 MB) -2016-06-13 21:07:38 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52695 (size: 2.1 KB, free: 730.4 MB) -2016-06-13 21:07:38 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:07:38 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 6 (MapPartitionsRDD[6] at map at MLUtils.scala:112) -2016-06-13 21:07:38 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 1 tasks -2016-06-13 21:07:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 10, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:38 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 10) -2016-06-13 21:07:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:07:38 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 10). 37259 bytes result sent to driver -2016-06-13 21:07:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 10) in 11 ms on localhost (1/1) -2016-06-13 21:07:38 INFO DAGScheduler:59 - ResultStage 6 (take at PCALabelPointSuite.scala:35) finished in 0.011 s -2016-06-13 21:07:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 21:07:38 INFO DAGScheduler:59 - Job 6 finished: take at PCALabelPointSuite.scala:35, took 0.025708 s -2016-06-13 21:07:38 INFO SparkContext:59 - Starting job: count at PCALabelPointSuite.scala:41 -2016-06-13 21:07:38 INFO DAGScheduler:59 - Got job 7 (count at PCALabelPointSuite.scala:41) with 2 output partitions -2016-06-13 21:07:38 INFO DAGScheduler:59 - Final stage: ResultStage 7(count at PCALabelPointSuite.scala:41) -2016-06-13 21:07:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:07:38 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:07:38 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[10] at map at PCALabelPointSuite.scala:32), which has no missing parents -2016-06-13 21:07:38 INFO MemoryStore:59 - ensureFreeSpace(59640) called with curMem=354417, maxMem=766075207 -2016-06-13 21:07:38 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 58.2 KB, free 730.2 MB) -2016-06-13 21:07:38 INFO MemoryStore:59 - ensureFreeSpace(39776) called with curMem=414057, maxMem=766075207 -2016-06-13 21:07:38 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 38.8 KB, free 730.2 MB) -2016-06-13 21:07:38 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52695 (size: 38.8 KB, free: 730.4 MB) -2016-06-13 21:07:38 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:07:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[10] at map at PCALabelPointSuite.scala:32) -2016-06-13 21:07:38 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-06-13 21:07:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 11, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 12, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:38 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 12) -2016-06-13 21:07:38 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 11) -2016-06-13 21:07:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:07:38 INFO BlockManager:59 - Found block rdd_4_1 locally -2016-06-13 21:07:38 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 11). 2082 bytes result sent to driver -2016-06-13 21:07:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 11) in 21 ms on localhost (1/2) -2016-06-13 21:07:38 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 12). 2082 bytes result sent to driver -2016-06-13 21:07:38 INFO DAGScheduler:59 - ResultStage 7 (count at PCALabelPointSuite.scala:41) finished in 0.025 s -2016-06-13 21:07:38 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 12) in 24 ms on localhost (2/2) -2016-06-13 21:07:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 21:07:38 INFO DAGScheduler:59 - Job 7 finished: count at PCALabelPointSuite.scala:41, took 0.039602 s -2016-06-13 21:07:38 INFO SparkContext:59 - Starting job: take at PCALabelPointSuite.scala:42 -2016-06-13 21:07:38 INFO DAGScheduler:59 - Got job 8 (take at PCALabelPointSuite.scala:42) with 1 output partitions -2016-06-13 21:07:38 INFO DAGScheduler:59 - Final stage: ResultStage 8(take at PCALabelPointSuite.scala:42) -2016-06-13 21:07:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:07:38 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:07:38 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[10] at map at PCALabelPointSuite.scala:32), which has no missing parents -2016-06-13 21:07:38 INFO MemoryStore:59 - ensureFreeSpace(59808) called with curMem=453833, maxMem=766075207 -2016-06-13 21:07:38 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 58.4 KB, free 730.1 MB) -2016-06-13 21:07:38 INFO MemoryStore:59 - ensureFreeSpace(39857) called with curMem=513641, maxMem=766075207 -2016-06-13 21:07:38 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 38.9 KB, free 730.1 MB) -2016-06-13 21:07:38 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52695 (size: 38.9 KB, free: 730.3 MB) -2016-06-13 21:07:38 INFO SparkContext:59 - Created broadcast 9 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:07:38 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[10] at map at PCALabelPointSuite.scala:32) -2016-06-13 21:07:38 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-06-13 21:07:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 13, localhost, PROCESS_LOCAL, 2191 bytes) -2016-06-13 21:07:38 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 13) -2016-06-13 21:07:38 INFO BlockManager:59 - Found block rdd_4_0 locally -2016-06-13 21:07:38 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 13). 4483 bytes result sent to driver -2016-06-13 21:07:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 13) in 12 ms on localhost (1/1) -2016-06-13 21:07:38 INFO DAGScheduler:59 - ResultStage 8 (take at PCALabelPointSuite.scala:42) finished in 0.013 s -2016-06-13 21:07:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 21:07:38 INFO DAGScheduler:59 - Job 8 finished: take at PCALabelPointSuite.scala:42, took 0.031330 s -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 21:07:38 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 21:07:38 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 21:07:38 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 21:07:38 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 21:07:38 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 21:07:38 INFO BlockManager:59 - BlockManager stopped -2016-06-13 21:07:38 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 21:07:38 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 21:07:38 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 21:07:38 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 21:07:38 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 21:07:38 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 21:07:38 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-691a5fc0-336d-4803-ae38-deb4921fc0c9 -2016-06-13 21:08:58 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 21:08:59 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 21:08:59 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 21:08:59 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 21:08:59 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 21:09:00 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 21:09:00 INFO Remoting:74 - Starting remoting -2016-06-13 21:09:00 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:52730] -2016-06-13 21:09:00 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52730. -2016-06-13 21:09:00 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 21:09:00 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 21:09:00 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-397bdf09-c485-48c6-8d53-3c255a2bc397 -2016-06-13 21:09:00 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 21:09:00 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-3b8fba97-9506-4597-b15a-228f3ff9cd9c\httpd-6f28a24f-23e3-41c5-9d38-1122829e3147 -2016-06-13 21:09:00 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 21:09:00 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:09:00 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52731 -2016-06-13 21:09:00 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52731. -2016-06-13 21:09:00 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 21:09:00 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:09:00 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 21:09:00 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 21:09:00 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 21:09:00 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 21:09:00 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 21:09:01 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52742. -2016-06-13 21:09:01 INFO NettyBlockTransferService:59 - Server created on 52742 -2016-06-13 21:09:01 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 21:09:01 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52742 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52742) -2016-06-13 21:09:01 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 21:09:01 INFO PCAFromSparkCodeSuite:59 - - -===== TEST OUTPUT FOR o.a.s.mllib.DimensionalityReduction.PCAFromSparkCodeSuite: 'Correct computing use a PCA wrapper' ===== - -2016-06-13 21:09:01 INFO SparkContext:59 - Starting job: count at PCAFromSparkCodeSuite.scala:37 -2016-06-13 21:09:01 INFO DAGScheduler:59 - Got job 0 (count at PCAFromSparkCodeSuite.scala:37) with 2 output partitions -2016-06-13 21:09:01 INFO DAGScheduler:59 - Final stage: ResultStage 0(count at PCAFromSparkCodeSuite.scala:37) -2016-06-13 21:09:01 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:09:01 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:09:01 INFO DAGScheduler:59 - Submitting ResultStage 0 (ParallelCollectionRDD[0] at parallelize at PCAFromSparkCodeSuite.scala:34), which has no missing parents -2016-06-13 21:09:01 INFO MemoryStore:59 - ensureFreeSpace(1080) called with curMem=0, maxMem=766075207 -2016-06-13 21:09:01 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 1080.0 B, free 730.6 MB) -2016-06-13 21:09:02 INFO MemoryStore:59 - ensureFreeSpace(768) called with curMem=1080, maxMem=766075207 -2016-06-13 21:09:02 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 768.0 B, free 730.6 MB) -2016-06-13 21:09:02 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52742 (size: 768.0 B, free: 730.6 MB) -2016-06-13 21:09:02 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:09:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (ParallelCollectionRDD[0] at parallelize at PCAFromSparkCodeSuite.scala:34) -2016-06-13 21:09:02 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 21:09:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 21:09:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2298 bytes) -2016-06-13 21:09:02 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 21:09:02 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 21:09:02 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 953 bytes result sent to driver -2016-06-13 21:09:02 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 953 bytes result sent to driver -2016-06-13 21:09:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 64 ms on localhost (1/2) -2016-06-13 21:09:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 42 ms on localhost (2/2) -2016-06-13 21:09:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 21:09:02 INFO DAGScheduler:59 - ResultStage 0 (count at PCAFromSparkCodeSuite.scala:37) finished in 0.078 s -2016-06-13 21:09:02 INFO DAGScheduler:59 - Job 0 finished: count at PCAFromSparkCodeSuite.scala:37, took 0.378625 s -2016-06-13 21:09:02 INFO SparkContext:59 - Starting job: first at PCA.scala:42 -2016-06-13 21:09:02 INFO DAGScheduler:59 - Got job 1 (first at PCA.scala:42) with 1 output partitions -2016-06-13 21:09:02 INFO DAGScheduler:59 - Final stage: ResultStage 1(first at PCA.scala:42) -2016-06-13 21:09:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:09:02 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:09:02 INFO DAGScheduler:59 - Submitting ResultStage 1 (ParallelCollectionRDD[0] at parallelize at PCAFromSparkCodeSuite.scala:34), which has no missing parents -2016-06-13 21:09:02 INFO MemoryStore:59 - ensureFreeSpace(1272) called with curMem=1848, maxMem=766075207 -2016-06-13 21:09:02 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1272.0 B, free 730.6 MB) -2016-06-13 21:09:02 INFO MemoryStore:59 - ensureFreeSpace(837) called with curMem=3120, maxMem=766075207 -2016-06-13 21:09:02 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 837.0 B, free 730.6 MB) -2016-06-13 21:09:02 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52742 (size: 837.0 B, free: 730.6 MB) -2016-06-13 21:09:02 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:09:02 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (ParallelCollectionRDD[0] at parallelize at PCAFromSparkCodeSuite.scala:34) -2016-06-13 21:09:02 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-06-13 21:09:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 21:09:02 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 21:09:02 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1104 bytes result sent to driver -2016-06-13 21:09:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 9 ms on localhost (1/1) -2016-06-13 21:09:02 INFO DAGScheduler:59 - ResultStage 1 (first at PCA.scala:42) finished in 0.009 s -2016-06-13 21:09:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 21:09:02 INFO DAGScheduler:59 - Job 1 finished: first at PCA.scala:42, took 0.022004 s -2016-06-13 21:09:02 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 21:09:02 INFO DAGScheduler:59 - Got job 2 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 21:09:02 INFO DAGScheduler:59 - Final stage: ResultStage 2(first at RowMatrix.scala:65) -2016-06-13 21:09:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:09:02 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:09:02 INFO DAGScheduler:59 - Submitting ResultStage 2 (ParallelCollectionRDD[0] at parallelize at PCAFromSparkCodeSuite.scala:34), which has no missing parents -2016-06-13 21:09:02 INFO MemoryStore:59 - ensureFreeSpace(1272) called with curMem=3957, maxMem=766075207 -2016-06-13 21:09:02 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 1272.0 B, free 730.6 MB) -2016-06-13 21:09:02 INFO MemoryStore:59 - ensureFreeSpace(837) called with curMem=5229, maxMem=766075207 -2016-06-13 21:09:02 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 837.0 B, free 730.6 MB) -2016-06-13 21:09:02 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52742 (size: 837.0 B, free: 730.6 MB) -2016-06-13 21:09:02 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:09:02 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (ParallelCollectionRDD[0] at parallelize at PCAFromSparkCodeSuite.scala:34) -2016-06-13 21:09:02 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-06-13 21:09:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 3, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 21:09:02 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 3) -2016-06-13 21:09:02 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 3). 1104 bytes result sent to driver -2016-06-13 21:09:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 3) in 8 ms on localhost (1/1) -2016-06-13 21:09:02 INFO DAGScheduler:59 - ResultStage 2 (first at RowMatrix.scala:65) finished in 0.009 s -2016-06-13 21:09:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 21:09:02 INFO DAGScheduler:59 - Job 2 finished: first at RowMatrix.scala:65, took 0.023989 s -2016-06-13 21:09:02 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-06-13 21:09:02 INFO DAGScheduler:59 - Got job 3 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-06-13 21:09:02 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at RowMatrix.scala:332) -2016-06-13 21:09:02 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:09:02 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:09:02 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[1] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-06-13 21:09:02 INFO MemoryStore:59 - ensureFreeSpace(2664) called with curMem=6066, maxMem=766075207 -2016-06-13 21:09:02 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 21:09:02 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=8730, maxMem=766075207 -2016-06-13 21:09:02 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.6 MB) -2016-06-13 21:09:02 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52742 (size: 1567.0 B, free: 730.6 MB) -2016-06-13 21:09:02 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:09:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[1] at treeAggregate at RowMatrix.scala:332) -2016-06-13 21:09:02 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 21:09:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 4, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 21:09:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 5, localhost, PROCESS_LOCAL, 2298 bytes) -2016-06-13 21:09:02 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 5) -2016-06-13 21:09:02 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 4) -2016-06-13 21:09:02 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 5). 1255 bytes result sent to driver -2016-06-13 21:09:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 5) in 22 ms on localhost (1/2) -2016-06-13 21:09:02 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 4). 1255 bytes result sent to driver -2016-06-13 21:09:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 4) in 410 ms on localhost (2/2) -2016-06-13 21:09:02 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at RowMatrix.scala:332) finished in 0.410 s -2016-06-13 21:09:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 21:09:02 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 21:09:02 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 21:09:02 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at RowMatrix.scala:332, took 0.439617 s -2016-06-13 21:09:02 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-06-13 21:09:03 INFO DAGScheduler:59 - Got job 4 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-06-13 21:09:03 INFO DAGScheduler:59 - Final stage: ResultStage 4(treeAggregate at RowMatrix.scala:124) -2016-06-13 21:09:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:09:03 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:09:03 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[2] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-06-13 21:09:03 INFO MemoryStore:59 - ensureFreeSpace(2776) called with curMem=10297, maxMem=766075207 -2016-06-13 21:09:03 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 2.7 KB, free 730.6 MB) -2016-06-13 21:09:03 INFO MemoryStore:59 - ensureFreeSpace(1544) called with curMem=13073, maxMem=766075207 -2016-06-13 21:09:03 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 1544.0 B, free 730.6 MB) -2016-06-13 21:09:03 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52742 (size: 1544.0 B, free: 730.6 MB) -2016-06-13 21:09:03 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:09:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[2] at treeAggregate at RowMatrix.scala:124) -2016-06-13 21:09:03 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 21:09:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 6, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 21:09:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 2298 bytes) -2016-06-13 21:09:03 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 7) -2016-06-13 21:09:03 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 6) -2016-06-13 21:09:03 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 6). 1208 bytes result sent to driver -2016-06-13 21:09:03 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 7). 1208 bytes result sent to driver -2016-06-13 21:09:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 6) in 11 ms on localhost (1/2) -2016-06-13 21:09:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 7) in 11 ms on localhost (2/2) -2016-06-13 21:09:03 INFO DAGScheduler:59 - ResultStage 4 (treeAggregate at RowMatrix.scala:124) finished in 0.013 s -2016-06-13 21:09:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 21:09:03 INFO DAGScheduler:59 - Job 4 finished: treeAggregate at RowMatrix.scala:124, took 0.026404 s -2016-06-13 21:09:03 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK -2016-06-13 21:09:03 WARN LAPACK:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK -2016-06-13 21:09:03 INFO SparkContext:59 - Starting job: first at RowMatrix.scala:65 -2016-06-13 21:09:03 INFO DAGScheduler:59 - Got job 5 (first at RowMatrix.scala:65) with 1 output partitions -2016-06-13 21:09:03 INFO DAGScheduler:59 - Final stage: ResultStage 5(first at RowMatrix.scala:65) -2016-06-13 21:09:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:09:03 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:09:03 INFO DAGScheduler:59 - Submitting ResultStage 5 (ParallelCollectionRDD[0] at parallelize at PCAFromSparkCodeSuite.scala:34), which has no missing parents -2016-06-13 21:09:03 INFO MemoryStore:59 - ensureFreeSpace(1272) called with curMem=14617, maxMem=766075207 -2016-06-13 21:09:03 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 1272.0 B, free 730.6 MB) -2016-06-13 21:09:03 INFO MemoryStore:59 - ensureFreeSpace(837) called with curMem=15889, maxMem=766075207 -2016-06-13 21:09:03 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 837.0 B, free 730.6 MB) -2016-06-13 21:09:03 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52742 (size: 837.0 B, free: 730.6 MB) -2016-06-13 21:09:03 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:09:03 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 5 (ParallelCollectionRDD[0] at parallelize at PCAFromSparkCodeSuite.scala:34) -2016-06-13 21:09:03 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 1 tasks -2016-06-13 21:09:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 8, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 21:09:03 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 8) -2016-06-13 21:09:03 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 8). 1104 bytes result sent to driver -2016-06-13 21:09:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 8) in 6 ms on localhost (1/1) -2016-06-13 21:09:03 INFO DAGScheduler:59 - ResultStage 5 (first at RowMatrix.scala:65) finished in 0.007 s -2016-06-13 21:09:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-06-13 21:09:03 INFO DAGScheduler:59 - Job 5 finished: first at RowMatrix.scala:65, took 0.019227 s -2016-06-13 21:09:03 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:332 -2016-06-13 21:09:03 INFO DAGScheduler:59 - Got job 6 (treeAggregate at RowMatrix.scala:332) with 2 output partitions -2016-06-13 21:09:03 INFO DAGScheduler:59 - Final stage: ResultStage 6(treeAggregate at RowMatrix.scala:332) -2016-06-13 21:09:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:09:03 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:09:03 INFO DAGScheduler:59 - Submitting ResultStage 6 (MapPartitionsRDD[3] at treeAggregate at RowMatrix.scala:332), which has no missing parents -2016-06-13 21:09:03 INFO MemoryStore:59 - ensureFreeSpace(2664) called with curMem=16726, maxMem=766075207 -2016-06-13 21:09:03 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 2.6 KB, free 730.6 MB) -2016-06-13 21:09:03 INFO MemoryStore:59 - ensureFreeSpace(1567) called with curMem=19390, maxMem=766075207 -2016-06-13 21:09:03 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 1567.0 B, free 730.6 MB) -2016-06-13 21:09:03 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52742 (size: 1567.0 B, free: 730.6 MB) -2016-06-13 21:09:03 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:09:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 6 (MapPartitionsRDD[3] at treeAggregate at RowMatrix.scala:332) -2016-06-13 21:09:03 INFO TaskSchedulerImpl:59 - Adding task set 6.0 with 2 tasks -2016-06-13 21:09:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 6.0 (TID 9, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 21:09:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 6.0 (TID 10, localhost, PROCESS_LOCAL, 2298 bytes) -2016-06-13 21:09:03 INFO Executor:59 - Running task 0.0 in stage 6.0 (TID 9) -2016-06-13 21:09:03 INFO Executor:59 - Running task 1.0 in stage 6.0 (TID 10) -2016-06-13 21:09:03 INFO Executor:59 - Finished task 0.0 in stage 6.0 (TID 9). 1255 bytes result sent to driver -2016-06-13 21:09:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 6.0 (TID 9) in 17 ms on localhost (1/2) -2016-06-13 21:09:03 INFO Executor:59 - Finished task 1.0 in stage 6.0 (TID 10). 1255 bytes result sent to driver -2016-06-13 21:09:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 6.0 (TID 10) in 34 ms on localhost (2/2) -2016-06-13 21:09:03 INFO DAGScheduler:59 - ResultStage 6 (treeAggregate at RowMatrix.scala:332) finished in 0.035 s -2016-06-13 21:09:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 6.0, whose tasks have all completed, from pool -2016-06-13 21:09:03 INFO DAGScheduler:59 - Job 6 finished: treeAggregate at RowMatrix.scala:332, took 0.051321 s -2016-06-13 21:09:03 INFO SparkContext:59 - Starting job: treeAggregate at RowMatrix.scala:124 -2016-06-13 21:09:03 INFO DAGScheduler:59 - Got job 7 (treeAggregate at RowMatrix.scala:124) with 2 output partitions -2016-06-13 21:09:03 INFO DAGScheduler:59 - Final stage: ResultStage 7(treeAggregate at RowMatrix.scala:124) -2016-06-13 21:09:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:09:03 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:09:03 INFO DAGScheduler:59 - Submitting ResultStage 7 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:124), which has no missing parents -2016-06-13 21:09:03 INFO MemoryStore:59 - ensureFreeSpace(2776) called with curMem=20957, maxMem=766075207 -2016-06-13 21:09:03 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 2.7 KB, free 730.6 MB) -2016-06-13 21:09:03 INFO MemoryStore:59 - ensureFreeSpace(1549) called with curMem=23733, maxMem=766075207 -2016-06-13 21:09:03 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 1549.0 B, free 730.6 MB) -2016-06-13 21:09:03 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:52742 (size: 1549.0 B, free: 730.6 MB) -2016-06-13 21:09:03 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:09:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 7 (MapPartitionsRDD[4] at treeAggregate at RowMatrix.scala:124) -2016-06-13 21:09:03 INFO TaskSchedulerImpl:59 - Adding task set 7.0 with 2 tasks -2016-06-13 21:09:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 7.0 (TID 11, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 21:09:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 7.0 (TID 12, localhost, PROCESS_LOCAL, 2298 bytes) -2016-06-13 21:09:03 INFO Executor:59 - Running task 0.0 in stage 7.0 (TID 11) -2016-06-13 21:09:03 INFO Executor:59 - Running task 1.0 in stage 7.0 (TID 12) -2016-06-13 21:09:03 INFO Executor:59 - Finished task 0.0 in stage 7.0 (TID 11). 1208 bytes result sent to driver -2016-06-13 21:09:03 INFO Executor:59 - Finished task 1.0 in stage 7.0 (TID 12). 1208 bytes result sent to driver -2016-06-13 21:09:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 7.0 (TID 11) in 12 ms on localhost (1/2) -2016-06-13 21:09:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 7.0 (TID 12) in 12 ms on localhost (2/2) -2016-06-13 21:09:03 INFO DAGScheduler:59 - ResultStage 7 (treeAggregate at RowMatrix.scala:124) finished in 0.014 s -2016-06-13 21:09:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 7.0, whose tasks have all completed, from pool -2016-06-13 21:09:03 INFO DAGScheduler:59 - Job 7 finished: treeAggregate at RowMatrix.scala:124, took 0.047026 s -2016-06-13 21:09:03 INFO SparkContext:59 - Starting job: collect at PCAFromSparkCodeSuite.scala:43 -2016-06-13 21:09:03 INFO DAGScheduler:59 - Got job 8 (collect at PCAFromSparkCodeSuite.scala:43) with 2 output partitions -2016-06-13 21:09:03 INFO DAGScheduler:59 - Final stage: ResultStage 8(collect at PCAFromSparkCodeSuite.scala:43) -2016-06-13 21:09:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:09:03 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:09:03 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[5] at map at VectorTransformer.scala:52), which has no missing parents -2016-06-13 21:09:03 INFO MemoryStore:59 - ensureFreeSpace(2384) called with curMem=25282, maxMem=766075207 -2016-06-13 21:09:03 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 2.3 KB, free 730.6 MB) -2016-06-13 21:09:03 INFO MemoryStore:59 - ensureFreeSpace(1511) called with curMem=27666, maxMem=766075207 -2016-06-13 21:09:03 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 1511.0 B, free 730.6 MB) -2016-06-13 21:09:03 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:52742 (size: 1511.0 B, free: 730.6 MB) -2016-06-13 21:09:03 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:09:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 8 (MapPartitionsRDD[5] at map at VectorTransformer.scala:52) -2016-06-13 21:09:03 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 2 tasks -2016-06-13 21:09:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 13, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 21:09:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 8.0 (TID 14, localhost, PROCESS_LOCAL, 2298 bytes) -2016-06-13 21:09:03 INFO Executor:59 - Running task 1.0 in stage 8.0 (TID 14) -2016-06-13 21:09:03 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 13) -2016-06-13 21:09:03 INFO Executor:59 - Finished task 1.0 in stage 8.0 (TID 14). 1094 bytes result sent to driver -2016-06-13 21:09:03 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 13). 1054 bytes result sent to driver -2016-06-13 21:09:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 8.0 (TID 14) in 45 ms on localhost (1/2) -2016-06-13 21:09:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 13) in 51 ms on localhost (2/2) -2016-06-13 21:09:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-06-13 21:09:03 INFO DAGScheduler:59 - ResultStage 8 (collect at PCAFromSparkCodeSuite.scala:43) finished in 0.053 s -2016-06-13 21:09:03 INFO DAGScheduler:59 - Job 8 finished: collect at PCAFromSparkCodeSuite.scala:43, took 0.069079 s -2016-06-13 21:09:03 INFO MemoryStore:59 - ensureFreeSpace(160) called with curMem=29177, maxMem=766075207 -2016-06-13 21:09:03 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 160.0 B, free 730.6 MB) -2016-06-13 21:09:03 INFO MemoryStore:59 - ensureFreeSpace(171) called with curMem=26561, maxMem=766075207 -2016-06-13 21:09:03 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 171.0 B, free 730.6 MB) -2016-06-13 21:09:03 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:52742 (size: 171.0 B, free: 730.6 MB) -2016-06-13 21:09:03 INFO SparkContext:59 - Created broadcast 9 from broadcast at RowMatrix.scala:425 -2016-06-13 21:09:03 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:52742 in memory (size: 1549.0 B, free: 730.6 MB) -2016-06-13 21:09:03 INFO ContextCleaner:59 - Cleaned accumulator 8 -2016-06-13 21:09:03 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:52742 in memory (size: 1567.0 B, free: 730.6 MB) -2016-06-13 21:09:03 INFO SparkContext:59 - Starting job: collect at PCAFromSparkCodeSuite.scala:44 -2016-06-13 21:09:03 INFO ContextCleaner:59 - Cleaned accumulator 7 -2016-06-13 21:09:03 INFO DAGScheduler:59 - Got job 9 (collect at PCAFromSparkCodeSuite.scala:44) with 2 output partitions -2016-06-13 21:09:03 INFO DAGScheduler:59 - Final stage: ResultStage 9(collect at PCAFromSparkCodeSuite.scala:44) -2016-06-13 21:09:03 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:09:03 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:09:03 INFO DAGScheduler:59 - Submitting ResultStage 9 (MapPartitionsRDD[6] at mapPartitions at RowMatrix.scala:426), which has no missing parents -2016-06-13 21:09:03 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52742 in memory (size: 837.0 B, free: 730.6 MB) -2016-06-13 21:09:03 INFO MemoryStore:59 - ensureFreeSpace(2544) called with curMem=18843, maxMem=766075207 -2016-06-13 21:09:03 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 2.5 KB, free 730.6 MB) -2016-06-13 21:09:03 INFO ContextCleaner:59 - Cleaned accumulator 6 -2016-06-13 21:09:03 INFO MemoryStore:59 - ensureFreeSpace(1502) called with curMem=17067, maxMem=766075207 -2016-06-13 21:09:03 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 1502.0 B, free 730.6 MB) -2016-06-13 21:09:03 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52742 in memory (size: 1544.0 B, free: 730.6 MB) -2016-06-13 21:09:03 INFO ContextCleaner:59 - Cleaned accumulator 5 -2016-06-13 21:09:03 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:52742 (size: 1502.0 B, free: 730.6 MB) -2016-06-13 21:09:03 INFO SparkContext:59 - Created broadcast 10 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:09:03 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:52742 in memory (size: 1567.0 B, free: 730.6 MB) -2016-06-13 21:09:03 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 9 (MapPartitionsRDD[6] at mapPartitions at RowMatrix.scala:426) -2016-06-13 21:09:03 INFO TaskSchedulerImpl:59 - Adding task set 9.0 with 2 tasks -2016-06-13 21:09:03 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 21:09:03 INFO TaskSetManager:59 - Starting task 0.0 in stage 9.0 (TID 15, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 21:09:03 INFO TaskSetManager:59 - Starting task 1.0 in stage 9.0 (TID 16, localhost, PROCESS_LOCAL, 2298 bytes) -2016-06-13 21:09:03 INFO Executor:59 - Running task 0.0 in stage 9.0 (TID 15) -2016-06-13 21:09:03 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52742 in memory (size: 837.0 B, free: 730.6 MB) -2016-06-13 21:09:03 INFO Executor:59 - Running task 1.0 in stage 9.0 (TID 16) -2016-06-13 21:09:03 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 21:09:03 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52742 in memory (size: 837.0 B, free: 730.6 MB) -2016-06-13 21:09:03 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 21:09:03 INFO Executor:59 - Finished task 0.0 in stage 9.0 (TID 15). 1054 bytes result sent to driver -2016-06-13 21:09:03 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:52742 in memory (size: 768.0 B, free: 730.6 MB) -2016-06-13 21:09:03 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 21:09:03 INFO TaskSetManager:59 - Finished task 0.0 in stage 9.0 (TID 15) in 25 ms on localhost (1/2) -2016-06-13 21:09:03 INFO Executor:59 - Finished task 1.0 in stage 9.0 (TID 16). 1094 bytes result sent to driver -2016-06-13 21:09:03 INFO TaskSetManager:59 - Finished task 1.0 in stage 9.0 (TID 16) in 26 ms on localhost (2/2) -2016-06-13 21:09:03 INFO DAGScheduler:59 - ResultStage 9 (collect at PCAFromSparkCodeSuite.scala:44) finished in 0.029 s -2016-06-13 21:09:03 INFO TaskSchedulerImpl:59 - Removed TaskSet 9.0, whose tasks have all completed, from pool -2016-06-13 21:09:03 INFO DAGScheduler:59 - Job 9 finished: collect at PCAFromSparkCodeSuite.scala:44, took 0.048361 s -2016-06-13 21:09:03 INFO PCAFromSparkCodeSuite:59 - - -===== FINISHED o.a.s.mllib.DimensionalityReduction.PCAFromSparkCodeSuite: 'Correct computing use a PCA wrapper' ===== - -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 21:09:03 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 21:09:03 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 21:09:03 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 21:09:03 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 21:09:03 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 21:09:03 INFO BlockManager:59 - BlockManager stopped -2016-06-13 21:09:03 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 21:09:03 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 21:09:03 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 21:09:03 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 21:09:03 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 21:09:03 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 21:09:03 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-3b8fba97-9506-4597-b15a-228f3ff9cd9c -2016-06-13 21:15:35 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 21:15:35 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 21:15:35 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 21:15:35 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 21:15:35 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 21:15:36 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 21:15:36 INFO Remoting:74 - Starting remoting -2016-06-13 21:15:37 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:52794] -2016-06-13 21:15:37 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52794. -2016-06-13 21:15:37 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 21:15:37 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 21:15:37 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-ad239f6b-3df2-481f-98dc-d1a4fcb07d11 -2016-06-13 21:15:37 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 21:15:37 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-aaa7d0e8-77ea-4013-9858-74c132f9614b\httpd-de8e6051-9429-458e-bc20-684b2604d3a5 -2016-06-13 21:15:37 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 21:15:37 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:15:37 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52795 -2016-06-13 21:15:37 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52795. -2016-06-13 21:15:37 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 21:15:37 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:15:37 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 21:15:37 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 21:15:37 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 21:15:37 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 21:15:37 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 21:15:37 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52806. -2016-06-13 21:15:37 INFO NettyBlockTransferService:59 - Server created on 52806 -2016-06-13 21:15:37 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 21:15:37 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52806 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52806) -2016-06-13 21:15:37 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 21:15:38 INFO IDFSuite:59 - - -===== TEST OUTPUT FOR o.a.s.mllib.FeatureExtractionAndTransformation.IDFSuite: 'idf' ===== - -2016-06-13 21:15:38 INFO SparkContext:59 - Starting job: treeAggregate at IDF.scala:56 -2016-06-13 21:15:38 INFO DAGScheduler:59 - Got job 0 (treeAggregate at IDF.scala:56) with 2 output partitions -2016-06-13 21:15:38 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at IDF.scala:56) -2016-06-13 21:15:38 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:15:38 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:15:38 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[1] at treeAggregate at IDF.scala:56), which has no missing parents -2016-06-13 21:15:38 INFO MemoryStore:59 - ensureFreeSpace(2520) called with curMem=0, maxMem=766075207 -2016-06-13 21:15:38 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 2.5 KB, free 730.6 MB) -2016-06-13 21:15:38 INFO MemoryStore:59 - ensureFreeSpace(1489) called with curMem=2520, maxMem=766075207 -2016-06-13 21:15:38 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1489.0 B, free 730.6 MB) -2016-06-13 21:15:38 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52806 (size: 1489.0 B, free: 730.6 MB) -2016-06-13 21:15:38 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:15:38 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at treeAggregate at IDF.scala:56) -2016-06-13 21:15:38 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 21:15:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 21:15:38 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 21:15:38 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 21:15:38 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 21:15:39 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1263 bytes result sent to driver -2016-06-13 21:15:39 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1263 bytes result sent to driver -2016-06-13 21:15:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 418 ms on localhost (1/2) -2016-06-13 21:15:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 395 ms on localhost (2/2) -2016-06-13 21:15:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 21:15:39 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at IDF.scala:56) finished in 0.434 s -2016-06-13 21:15:39 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at IDF.scala:56, took 0.624841 s -2016-06-13 21:15:39 INFO MemoryStore:59 - ensureFreeSpace(88) called with curMem=4009, maxMem=766075207 -2016-06-13 21:15:39 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 88.0 B, free 730.6 MB) -2016-06-13 21:15:39 INFO MemoryStore:59 - ensureFreeSpace(155) called with curMem=4097, maxMem=766075207 -2016-06-13 21:15:39 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 155.0 B, free 730.6 MB) -2016-06-13 21:15:39 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52806 (size: 155.0 B, free: 730.6 MB) -2016-06-13 21:15:39 INFO SparkContext:59 - Created broadcast 1 from broadcast at IDF.scala:181 -2016-06-13 21:15:39 INFO SparkContext:59 - Starting job: collect at IDFSuite.scala:58 -2016-06-13 21:15:39 INFO DAGScheduler:59 - Got job 1 (collect at IDFSuite.scala:58) with 2 output partitions -2016-06-13 21:15:39 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at IDFSuite.scala:58) -2016-06-13 21:15:39 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:15:39 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:15:39 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[2] at mapPartitions at IDF.scala:182), which has no missing parents -2016-06-13 21:15:39 INFO MemoryStore:59 - ensureFreeSpace(2512) called with curMem=4252, maxMem=766075207 -2016-06-13 21:15:39 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 2.5 KB, free 730.6 MB) -2016-06-13 21:15:39 INFO MemoryStore:59 - ensureFreeSpace(1467) called with curMem=6764, maxMem=766075207 -2016-06-13 21:15:39 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1467.0 B, free 730.6 MB) -2016-06-13 21:15:39 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52806 (size: 1467.0 B, free: 730.6 MB) -2016-06-13 21:15:39 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:15:39 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[2] at mapPartitions at IDF.scala:182) -2016-06-13 21:15:39 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 21:15:39 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2276 bytes) -2016-06-13 21:15:39 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2378 bytes) -2016-06-13 21:15:39 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 21:15:39 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 21:15:39 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1104 bytes result sent to driver -2016-06-13 21:15:39 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1209 bytes result sent to driver -2016-06-13 21:15:39 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 14 ms on localhost (1/2) -2016-06-13 21:15:39 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 15 ms on localhost (2/2) -2016-06-13 21:15:39 INFO DAGScheduler:59 - ResultStage 1 (collect at IDFSuite.scala:58) finished in 0.016 s -2016-06-13 21:15:39 INFO DAGScheduler:59 - Job 1 finished: collect at IDFSuite.scala:58, took 0.034309 s -2016-06-13 21:15:39 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 21:15:39 INFO IDFSuite:59 - - -===== FINISHED o.a.s.mllib.FeatureExtractionAndTransformation.IDFSuite: 'idf' ===== - -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 21:15:39 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 21:15:39 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 21:15:39 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 21:15:39 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 21:15:39 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 21:15:39 INFO BlockManager:59 - BlockManager stopped -2016-06-13 21:15:39 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 21:15:39 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 21:15:39 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 21:15:39 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 21:15:39 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 21:15:39 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 21:15:39 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-aaa7d0e8-77ea-4013-9858-74c132f9614b -2016-06-13 21:20:44 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 21:20:45 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 21:20:45 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 21:20:45 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 21:20:45 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 21:20:46 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 21:20:46 INFO Remoting:74 - Starting remoting -2016-06-13 21:20:46 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:52857] -2016-06-13 21:20:46 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52857. -2016-06-13 21:20:46 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 21:20:46 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 21:20:46 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-2f1dee33-045a-40f0-8cb3-35ffe7ed0595 -2016-06-13 21:20:46 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 21:20:46 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-3e9513de-1046-4bce-bd50-c0f50e05b2f1\httpd-e78c4118-0be1-46b8-849a-62ab23e2b2d8 -2016-06-13 21:20:46 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 21:20:46 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:20:46 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52858 -2016-06-13 21:20:46 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52858. -2016-06-13 21:20:46 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 21:20:46 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:20:46 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 21:20:46 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 21:20:46 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 21:20:46 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 21:20:46 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 21:20:47 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52877. -2016-06-13 21:20:47 INFO NettyBlockTransferService:59 - Server created on 52877 -2016-06-13 21:20:47 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 21:20:47 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52877 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52877) -2016-06-13 21:20:47 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 21:20:47 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 21:20:47 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 21:20:47 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 21:20:47 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 21:20:47 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52877 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 21:20:47 INFO SparkContext:59 - Created broadcast 0 from textFile at TFIDFSuite.scala:24 -2016-06-13 21:20:49 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 21:20:49 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 21:20:49 INFO SparkContext:59 - Starting job: treeAggregate at IDF.scala:56 -2016-06-13 21:20:49 INFO DAGScheduler:59 - Got job 0 (treeAggregate at IDF.scala:56) with 2 output partitions -2016-06-13 21:20:49 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at IDF.scala:56) -2016-06-13 21:20:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:20:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:20:49 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at IDF.scala:56), which has no missing parents -2016-06-13 21:20:49 INFO MemoryStore:59 - ensureFreeSpace(4504) called with curMem=171580, maxMem=766075207 -2016-06-13 21:20:49 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 21:20:49 INFO MemoryStore:59 - ensureFreeSpace(2416) called with curMem=176084, maxMem=766075207 -2016-06-13 21:20:49 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 21:20:49 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52877 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 21:20:49 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:20:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at IDF.scala:56) -2016-06-13 21:20:49 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 21:20:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:20:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:20:49 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 21:20:49 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 21:20:49 INFO CacheManager:59 - Partition rdd_3_1 not found, computing it -2016-06-13 21:20:49 INFO CacheManager:59 - Partition rdd_3_0 not found, computing it -2016-06-13 21:20:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:37+37 -2016-06-13 21:20:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+37 -2016-06-13 21:20:49 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 21:20:49 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 21:20:49 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 21:20:49 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 21:20:49 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 21:20:49 INFO MemoryStore:59 - ensureFreeSpace(272) called with curMem=178500, maxMem=766075207 -2016-06-13 21:20:49 INFO MemoryStore:59 - Block rdd_3_0 stored as values in memory (estimated size 272.0 B, free 730.4 MB) -2016-06-13 21:20:49 INFO MemoryStore:59 - ensureFreeSpace(264) called with curMem=178772, maxMem=766075207 -2016-06-13 21:20:49 INFO MemoryStore:59 - Block rdd_3_1 stored as values in memory (estimated size 264.0 B, free 730.4 MB) -2016-06-13 21:20:49 INFO BlockManagerInfo:59 - Added rdd_3_0 in memory on localhost:52877 (size: 272.0 B, free: 730.6 MB) -2016-06-13 21:20:49 INFO BlockManagerInfo:59 - Added rdd_3_1 in memory on localhost:52877 (size: 264.0 B, free: 730.6 MB) -2016-06-13 21:20:50 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 8432508 bytes result sent to driver -2016-06-13 21:20:50 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 8432508 bytes result sent to driver -2016-06-13 21:20:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 821 ms on localhost (1/2) -2016-06-13 21:20:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 842 ms on localhost (2/2) -2016-06-13 21:20:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 21:20:50 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at IDF.scala:56) finished in 0.859 s -2016-06-13 21:20:50 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at IDF.scala:56, took 0.965516 s -2016-06-13 21:20:50 INFO MemoryStore:59 - ensureFreeSpace(8388664) called with curMem=179036, maxMem=766075207 -2016-06-13 21:20:50 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 8.0 MB, free 722.4 MB) -2016-06-13 21:20:50 INFO MemoryStore:59 - ensureFreeSpace(397521) called with curMem=8567700, maxMem=766075207 -2016-06-13 21:20:50 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 388.2 KB, free 722.0 MB) -2016-06-13 21:20:50 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52877 (size: 388.2 KB, free: 730.2 MB) -2016-06-13 21:20:50 INFO SparkContext:59 - Created broadcast 2 from broadcast at IDF.scala:181 -2016-06-13 21:20:50 INFO SparkContext:59 - Starting job: treeAggregate at IDF.scala:56 -2016-06-13 21:20:50 INFO DAGScheduler:59 - Got job 1 (treeAggregate at IDF.scala:56) with 2 output partitions -2016-06-13 21:20:50 INFO DAGScheduler:59 - Final stage: ResultStage 1(treeAggregate at IDF.scala:56) -2016-06-13 21:20:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:20:50 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:20:50 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[6] at treeAggregate at IDF.scala:56), which has no missing parents -2016-06-13 21:20:50 INFO MemoryStore:59 - ensureFreeSpace(4504) called with curMem=8965221, maxMem=766075207 -2016-06-13 21:20:50 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.4 KB, free 722.0 MB) -2016-06-13 21:20:50 INFO MemoryStore:59 - ensureFreeSpace(2418) called with curMem=8969725, maxMem=766075207 -2016-06-13 21:20:50 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.4 KB, free 722.0 MB) -2016-06-13 21:20:50 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52877 (size: 2.4 KB, free: 730.2 MB) -2016-06-13 21:20:50 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:20:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[6] at treeAggregate at IDF.scala:56) -2016-06-13 21:20:50 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 21:20:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:20:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:20:50 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 21:20:50 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 21:20:50 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 21:20:50 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 21:20:50 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 8431928 bytes result sent to driver -2016-06-13 21:20:50 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 8431928 bytes result sent to driver -2016-06-13 21:20:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 247 ms on localhost (1/2) -2016-06-13 21:20:50 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52877 in memory (size: 2.4 KB, free: 730.2 MB) -2016-06-13 21:20:50 INFO DAGScheduler:59 - ResultStage 1 (treeAggregate at IDF.scala:56) finished in 0.255 s -2016-06-13 21:20:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 255 ms on localhost (2/2) -2016-06-13 21:20:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 21:20:50 INFO DAGScheduler:59 - Job 1 finished: treeAggregate at IDF.scala:56, took 0.281880 s -2016-06-13 21:20:50 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 21:20:50 INFO MemoryStore:59 - ensureFreeSpace(8388664) called with curMem=8965223, maxMem=766075207 -2016-06-13 21:20:50 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 8.0 MB, free 714.0 MB) -2016-06-13 21:20:50 INFO MemoryStore:59 - ensureFreeSpace(395684) called with curMem=17353887, maxMem=766075207 -2016-06-13 21:20:50 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 386.4 KB, free 713.7 MB) -2016-06-13 21:20:50 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52877 (size: 386.4 KB, free: 729.8 MB) -2016-06-13 21:20:50 INFO SparkContext:59 - Created broadcast 4 from broadcast at IDF.scala:181 -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 21:20:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 21:20:50 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 21:20:50 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 21:20:50 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 21:20:50 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 21:20:50 INFO BlockManager:59 - BlockManager stopped -2016-06-13 21:20:50 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 21:20:50 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 21:20:50 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 21:20:51 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 21:20:51 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 21:20:51 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 21:20:51 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-3e9513de-1046-4bce-bd50-c0f50e05b2f1 -2016-06-13 21:21:45 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 21:21:45 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 21:21:46 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 21:21:46 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 21:21:46 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 21:21:46 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 21:21:46 INFO Remoting:74 - Starting remoting -2016-06-13 21:21:47 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:52897] -2016-06-13 21:21:47 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52897. -2016-06-13 21:21:47 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 21:21:47 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 21:21:47 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-1c0a595b-975c-4d61-94d8-e803afef1c04 -2016-06-13 21:21:47 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 21:21:47 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-9e165d3d-bacf-410e-861b-363e076ae004\httpd-49def15b-b2e0-492a-b8de-0f548b170003 -2016-06-13 21:21:47 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 21:21:47 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:21:47 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52898 -2016-06-13 21:21:47 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52898. -2016-06-13 21:21:47 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 21:21:47 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:21:47 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 21:21:47 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 21:21:47 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 21:21:47 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 21:21:47 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 21:21:48 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52918. -2016-06-13 21:21:48 INFO NettyBlockTransferService:59 - Server created on 52918 -2016-06-13 21:21:48 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 21:21:48 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52918 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52918) -2016-06-13 21:21:48 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 21:21:48 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 21:21:48 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 21:21:48 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 21:21:48 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 21:21:48 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52918 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 21:21:48 INFO SparkContext:59 - Created broadcast 0 from textFile at TFIDFSuite.scala:24 -2016-06-13 21:21:49 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 21:21:50 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 21:21:50 INFO SparkContext:59 - Starting job: treeAggregate at IDF.scala:56 -2016-06-13 21:21:50 INFO DAGScheduler:59 - Got job 0 (treeAggregate at IDF.scala:56) with 2 output partitions -2016-06-13 21:21:50 INFO DAGScheduler:59 - Final stage: ResultStage 0(treeAggregate at IDF.scala:56) -2016-06-13 21:21:50 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:21:50 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:21:50 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at IDF.scala:56), which has no missing parents -2016-06-13 21:21:50 INFO MemoryStore:59 - ensureFreeSpace(4504) called with curMem=171580, maxMem=766075207 -2016-06-13 21:21:50 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 21:21:50 INFO MemoryStore:59 - ensureFreeSpace(2416) called with curMem=176084, maxMem=766075207 -2016-06-13 21:21:50 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 21:21:50 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52918 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 21:21:50 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:21:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at treeAggregate at IDF.scala:56) -2016-06-13 21:21:50 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 21:21:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:21:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:21:50 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 21:21:50 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 21:21:50 INFO CacheManager:59 - Partition rdd_3_0 not found, computing it -2016-06-13 21:21:50 INFO CacheManager:59 - Partition rdd_3_1 not found, computing it -2016-06-13 21:21:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:37+37 -2016-06-13 21:21:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+37 -2016-06-13 21:21:50 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 21:21:50 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 21:21:50 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 21:21:50 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 21:21:50 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 21:21:50 INFO MemoryStore:59 - ensureFreeSpace(264) called with curMem=178500, maxMem=766075207 -2016-06-13 21:21:50 INFO MemoryStore:59 - Block rdd_3_1 stored as values in memory (estimated size 264.0 B, free 730.4 MB) -2016-06-13 21:21:50 INFO MemoryStore:59 - ensureFreeSpace(272) called with curMem=178764, maxMem=766075207 -2016-06-13 21:21:50 INFO MemoryStore:59 - Block rdd_3_0 stored as values in memory (estimated size 272.0 B, free 730.4 MB) -2016-06-13 21:21:50 INFO BlockManagerInfo:59 - Added rdd_3_1 in memory on localhost:52918 (size: 264.0 B, free: 730.6 MB) -2016-06-13 21:21:50 INFO BlockManagerInfo:59 - Added rdd_3_0 in memory on localhost:52918 (size: 272.0 B, free: 730.6 MB) -2016-06-13 21:21:50 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 8432508 bytes result sent to driver -2016-06-13 21:21:50 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 8432508 bytes result sent to driver -2016-06-13 21:21:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 703 ms on localhost (1/2) -2016-06-13 21:21:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 685 ms on localhost (2/2) -2016-06-13 21:21:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 21:21:51 INFO DAGScheduler:59 - ResultStage 0 (treeAggregate at IDF.scala:56) finished in 0.722 s -2016-06-13 21:21:51 INFO DAGScheduler:59 - Job 0 finished: treeAggregate at IDF.scala:56, took 0.833046 s -2016-06-13 21:21:51 INFO MemoryStore:59 - ensureFreeSpace(8388664) called with curMem=179036, maxMem=766075207 -2016-06-13 21:21:51 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 8.0 MB, free 722.4 MB) -2016-06-13 21:21:51 INFO MemoryStore:59 - ensureFreeSpace(397521) called with curMem=8567700, maxMem=766075207 -2016-06-13 21:21:51 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 388.2 KB, free 722.0 MB) -2016-06-13 21:21:51 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52918 (size: 388.2 KB, free: 730.2 MB) -2016-06-13 21:21:51 INFO SparkContext:59 - Created broadcast 2 from broadcast at IDF.scala:181 -2016-06-13 21:21:51 INFO SparkContext:59 - Starting job: treeAggregate at IDF.scala:56 -2016-06-13 21:21:51 INFO DAGScheduler:59 - Got job 1 (treeAggregate at IDF.scala:56) with 2 output partitions -2016-06-13 21:21:51 INFO DAGScheduler:59 - Final stage: ResultStage 1(treeAggregate at IDF.scala:56) -2016-06-13 21:21:51 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:21:51 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:21:51 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[6] at treeAggregate at IDF.scala:56), which has no missing parents -2016-06-13 21:21:51 INFO MemoryStore:59 - ensureFreeSpace(4504) called with curMem=8965221, maxMem=766075207 -2016-06-13 21:21:51 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 4.4 KB, free 722.0 MB) -2016-06-13 21:21:51 INFO MemoryStore:59 - ensureFreeSpace(2418) called with curMem=8969725, maxMem=766075207 -2016-06-13 21:21:51 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 2.4 KB, free 722.0 MB) -2016-06-13 21:21:51 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52918 (size: 2.4 KB, free: 730.2 MB) -2016-06-13 21:21:51 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:21:51 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[6] at treeAggregate at IDF.scala:56) -2016-06-13 21:21:51 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 21:21:51 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:21:51 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:21:51 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 21:21:51 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 21:21:51 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 21:21:51 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 21:21:51 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 8431928 bytes result sent to driver -2016-06-13 21:21:51 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 8431928 bytes result sent to driver -2016-06-13 21:21:51 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 89 ms on localhost (1/2) -2016-06-13 21:21:51 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 100 ms on localhost (2/2) -2016-06-13 21:21:51 INFO DAGScheduler:59 - ResultStage 1 (treeAggregate at IDF.scala:56) finished in 0.100 s -2016-06-13 21:21:51 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 21:21:51 INFO DAGScheduler:59 - Job 1 finished: treeAggregate at IDF.scala:56, took 0.163143 s -2016-06-13 21:21:51 INFO MemoryStore:59 - ensureFreeSpace(8388664) called with curMem=8972143, maxMem=766075207 -2016-06-13 21:21:51 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 8.0 MB, free 714.0 MB) -2016-06-13 21:21:51 INFO MemoryStore:59 - ensureFreeSpace(395684) called with curMem=17360807, maxMem=766075207 -2016-06-13 21:21:51 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 386.4 KB, free 713.7 MB) -2016-06-13 21:21:51 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52918 (size: 386.4 KB, free: 729.8 MB) -2016-06-13 21:21:51 INFO SparkContext:59 - Created broadcast 4 from broadcast at IDF.scala:181 -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 21:21:51 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 21:21:51 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 21:21:51 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 21:21:51 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 21:21:51 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 21:21:51 INFO BlockManager:59 - BlockManager stopped -2016-06-13 21:21:51 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 21:21:51 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 21:21:51 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 21:21:51 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 21:21:51 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 21:21:51 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 21:21:51 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-9e165d3d-bacf-410e-861b-363e076ae004 -2016-06-13 21:23:41 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 21:23:42 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 21:23:42 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 21:23:42 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 21:23:42 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 21:23:43 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 21:23:43 INFO Remoting:74 - Starting remoting -2016-06-13 21:23:43 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:52940] -2016-06-13 21:23:43 INFO Utils:59 - Successfully started service 'sparkDriver' on port 52940. -2016-06-13 21:23:43 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 21:23:43 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 21:23:43 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-6a84513e-8d56-4723-be60-53aafbceb26a -2016-06-13 21:23:43 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 21:23:43 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-f57b3acd-9331-47ea-9267-9a3d65607c19\httpd-580a662d-ed7e-493b-ab08-bc0fa232168c -2016-06-13 21:23:43 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 21:23:44 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:23:44 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:52941 -2016-06-13 21:23:44 INFO Utils:59 - Successfully started service 'HTTP file server' on port 52941. -2016-06-13 21:23:44 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 21:23:44 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:23:44 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 21:23:44 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 21:23:44 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 21:23:44 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 21:23:44 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 21:23:44 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 52960. -2016-06-13 21:23:44 INFO NettyBlockTransferService:59 - Server created on 52960 -2016-06-13 21:23:44 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 21:23:44 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:52960 with 730.6 MB RAM, BlockManagerId(driver, localhost, 52960) -2016-06-13 21:23:44 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 21:23:45 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 21:23:45 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 21:23:45 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 21:23:45 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 21:23:45 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:52960 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 21:23:45 INFO SparkContext:59 - Created broadcast 0 from textFile at TFIDFSuite.scala:24 -2016-06-13 21:23:46 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 21:23:46 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 21:23:46 INFO SparkContext:59 - Starting job: foreach at TFIDFSuite.scala:29 -2016-06-13 21:23:46 INFO DAGScheduler:59 - Got job 0 (foreach at TFIDFSuite.scala:29) with 2 output partitions -2016-06-13 21:23:46 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at TFIDFSuite.scala:29) -2016-06-13 21:23:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:23:46 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:23:46 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at HashingTF.scala:78), which has no missing parents -2016-06-13 21:23:46 INFO MemoryStore:59 - ensureFreeSpace(3704) called with curMem=171580, maxMem=766075207 -2016-06-13 21:23:46 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-06-13 21:23:46 INFO MemoryStore:59 - ensureFreeSpace(2095) called with curMem=175284, maxMem=766075207 -2016-06-13 21:23:46 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 21:23:46 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:52960 (size: 2.0 KB, free: 730.6 MB) -2016-06-13 21:23:46 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:23:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at HashingTF.scala:78) -2016-06-13 21:23:46 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 21:23:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:23:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:23:46 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 21:23:46 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 21:23:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:37+37 -2016-06-13 21:23:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+37 -2016-06-13 21:23:46 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 21:23:46 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 21:23:46 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 21:23:46 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 21:23:46 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 21:23:46 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-06-13 21:23:46 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-06-13 21:23:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 105 ms on localhost (1/2) -2016-06-13 21:23:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 92 ms on localhost (2/2) -2016-06-13 21:23:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 21:23:46 INFO DAGScheduler:59 - ResultStage 0 (foreach at TFIDFSuite.scala:29) finished in 0.118 s -2016-06-13 21:23:46 INFO DAGScheduler:59 - Job 0 finished: foreach at TFIDFSuite.scala:29, took 0.262114 s -2016-06-13 21:23:46 INFO SparkContext:59 - Starting job: treeAggregate at IDF.scala:56 -2016-06-13 21:23:46 INFO DAGScheduler:59 - Got job 1 (treeAggregate at IDF.scala:56) with 2 output partitions -2016-06-13 21:23:46 INFO DAGScheduler:59 - Final stage: ResultStage 1(treeAggregate at IDF.scala:56) -2016-06-13 21:23:46 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:23:46 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:23:46 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at IDF.scala:56), which has no missing parents -2016-06-13 21:23:47 INFO MemoryStore:59 - ensureFreeSpace(4504) called with curMem=177379, maxMem=766075207 -2016-06-13 21:23:47 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 21:23:47 INFO MemoryStore:59 - ensureFreeSpace(2416) called with curMem=181883, maxMem=766075207 -2016-06-13 21:23:47 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 21:23:47 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:52960 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 21:23:47 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:23:47 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at IDF.scala:56) -2016-06-13 21:23:47 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 21:23:47 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:23:47 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:23:47 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 21:23:47 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 21:23:47 INFO CacheManager:59 - Partition rdd_3_1 not found, computing it -2016-06-13 21:23:47 INFO CacheManager:59 - Partition rdd_3_0 not found, computing it -2016-06-13 21:23:47 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:37+37 -2016-06-13 21:23:47 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+37 -2016-06-13 21:23:47 INFO MemoryStore:59 - ensureFreeSpace(272) called with curMem=184299, maxMem=766075207 -2016-06-13 21:23:47 INFO MemoryStore:59 - Block rdd_3_0 stored as values in memory (estimated size 272.0 B, free 730.4 MB) -2016-06-13 21:23:47 INFO MemoryStore:59 - ensureFreeSpace(264) called with curMem=184571, maxMem=766075207 -2016-06-13 21:23:47 INFO MemoryStore:59 - Block rdd_3_1 stored as values in memory (estimated size 264.0 B, free 730.4 MB) -2016-06-13 21:23:47 INFO BlockManagerInfo:59 - Added rdd_3_0 in memory on localhost:52960 (size: 272.0 B, free: 730.6 MB) -2016-06-13 21:23:47 INFO BlockManagerInfo:59 - Added rdd_3_1 in memory on localhost:52960 (size: 264.0 B, free: 730.6 MB) -2016-06-13 21:23:47 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 8432508 bytes result sent to driver -2016-06-13 21:23:47 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 8432508 bytes result sent to driver -2016-06-13 21:23:47 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:52960 in memory (size: 2.0 KB, free: 730.6 MB) -2016-06-13 21:23:47 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 738 ms on localhost (1/2) -2016-06-13 21:23:47 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 21:23:47 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 740 ms on localhost (2/2) -2016-06-13 21:23:47 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 21:23:47 INFO DAGScheduler:59 - ResultStage 1 (treeAggregate at IDF.scala:56) finished in 0.743 s -2016-06-13 21:23:47 INFO DAGScheduler:59 - Job 1 finished: treeAggregate at IDF.scala:56, took 0.775752 s -2016-06-13 21:23:47 INFO MemoryStore:59 - ensureFreeSpace(8388664) called with curMem=179036, maxMem=766075207 -2016-06-13 21:23:47 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 8.0 MB, free 722.4 MB) -2016-06-13 21:23:47 INFO MemoryStore:59 - ensureFreeSpace(397521) called with curMem=8567700, maxMem=766075207 -2016-06-13 21:23:47 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 388.2 KB, free 722.0 MB) -2016-06-13 21:23:47 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:52960 (size: 388.2 KB, free: 730.2 MB) -2016-06-13 21:23:47 INFO SparkContext:59 - Created broadcast 3 from broadcast at IDF.scala:181 -2016-06-13 21:23:48 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:52960 in memory (size: 2.4 KB, free: 730.2 MB) -2016-06-13 21:23:48 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 21:23:49 INFO SparkContext:59 - Starting job: foreach at TFIDFSuite.scala:39 -2016-06-13 21:23:49 INFO DAGScheduler:59 - Got job 2 (foreach at TFIDFSuite.scala:39) with 2 output partitions -2016-06-13 21:23:49 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at TFIDFSuite.scala:39) -2016-06-13 21:23:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:23:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:23:49 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[5] at mapPartitions at IDF.scala:182), which has no missing parents -2016-06-13 21:23:49 INFO MemoryStore:59 - ensureFreeSpace(4016) called with curMem=8958301, maxMem=766075207 -2016-06-13 21:23:49 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.9 KB, free 722.0 MB) -2016-06-13 21:23:49 INFO MemoryStore:59 - ensureFreeSpace(2224) called with curMem=8962317, maxMem=766075207 -2016-06-13 21:23:49 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.2 KB, free 722.0 MB) -2016-06-13 21:23:49 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:52960 (size: 2.2 KB, free: 730.2 MB) -2016-06-13 21:23:49 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:23:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[5] at mapPartitions at IDF.scala:182) -2016-06-13 21:23:49 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 21:23:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:23:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:23:49 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 21:23:49 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 21:23:49 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 21:23:49 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 21:23:49 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2044 bytes result sent to driver -2016-06-13 21:23:49 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2044 bytes result sent to driver -2016-06-13 21:23:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 16 ms on localhost (1/2) -2016-06-13 21:23:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 14 ms on localhost (2/2) -2016-06-13 21:23:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 21:23:49 INFO DAGScheduler:59 - ResultStage 2 (foreach at TFIDFSuite.scala:39) finished in 0.017 s -2016-06-13 21:23:49 INFO DAGScheduler:59 - Job 2 finished: foreach at TFIDFSuite.scala:39, took 0.085636 s -2016-06-13 21:23:49 INFO SparkContext:59 - Starting job: treeAggregate at IDF.scala:56 -2016-06-13 21:23:49 INFO DAGScheduler:59 - Got job 3 (treeAggregate at IDF.scala:56) with 2 output partitions -2016-06-13 21:23:49 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at IDF.scala:56) -2016-06-13 21:23:49 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:23:49 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:23:49 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[6] at treeAggregate at IDF.scala:56), which has no missing parents -2016-06-13 21:23:49 INFO MemoryStore:59 - ensureFreeSpace(4504) called with curMem=8964541, maxMem=766075207 -2016-06-13 21:23:49 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 4.4 KB, free 722.0 MB) -2016-06-13 21:23:49 INFO MemoryStore:59 - ensureFreeSpace(2418) called with curMem=8969045, maxMem=766075207 -2016-06-13 21:23:49 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.4 KB, free 722.0 MB) -2016-06-13 21:23:49 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:52960 (size: 2.4 KB, free: 730.2 MB) -2016-06-13 21:23:49 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:23:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[6] at treeAggregate at IDF.scala:56) -2016-06-13 21:23:49 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 21:23:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:23:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:23:49 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-06-13 21:23:49 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-06-13 21:23:49 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 21:23:49 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 21:23:49 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 8431928 bytes result sent to driver -2016-06-13 21:23:49 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 8431928 bytes result sent to driver -2016-06-13 21:23:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 100 ms on localhost (1/2) -2016-06-13 21:23:49 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:52960 in memory (size: 2.2 KB, free: 730.2 MB) -2016-06-13 21:23:49 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 21:23:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 114 ms on localhost (2/2) -2016-06-13 21:23:49 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at IDF.scala:56) finished in 0.116 s -2016-06-13 21:23:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 21:23:49 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at IDF.scala:56, took 0.135463 s -2016-06-13 21:23:49 INFO MemoryStore:59 - ensureFreeSpace(8388664) called with curMem=8965223, maxMem=766075207 -2016-06-13 21:23:49 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 8.0 MB, free 714.0 MB) -2016-06-13 21:23:49 INFO MemoryStore:59 - ensureFreeSpace(395684) called with curMem=17353887, maxMem=766075207 -2016-06-13 21:23:49 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 386.4 KB, free 713.7 MB) -2016-06-13 21:23:49 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:52960 (size: 386.4 KB, free: 729.8 MB) -2016-06-13 21:23:49 INFO SparkContext:59 - Created broadcast 6 from broadcast at IDF.scala:181 -2016-06-13 21:23:49 INFO ContextCleaner:59 - Cleaned accumulator 4 -2016-06-13 21:23:49 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:52960 in memory (size: 2.4 KB, free: 729.8 MB) -2016-06-13 21:27:53 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 21:27:54 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 21:27:54 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 21:27:54 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 21:27:54 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 21:27:55 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 21:27:55 INFO Remoting:74 - Starting remoting -2016-06-13 21:27:55 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:53031] -2016-06-13 21:27:55 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53031. -2016-06-13 21:27:55 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 21:27:55 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 21:27:55 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-d9dee0df-ae66-4c9a-a598-ad7c61b2fc9c -2016-06-13 21:27:55 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 21:27:55 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-2fa11711-bfee-499d-9587-2641fc932231\httpd-476cf179-097a-42f2-9bdc-5a03574e7d01 -2016-06-13 21:27:55 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 21:27:55 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:27:55 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53032 -2016-06-13 21:27:55 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53032. -2016-06-13 21:27:55 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 21:27:55 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:27:55 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 21:27:55 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 21:27:55 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 21:27:55 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 21:27:55 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 21:27:56 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53051. -2016-06-13 21:27:56 INFO NettyBlockTransferService:59 - Server created on 53051 -2016-06-13 21:27:56 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 21:27:56 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53051 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53051) -2016-06-13 21:27:56 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 21:27:56 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 21:27:56 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 21:27:57 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 21:27:57 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 21:27:57 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53051 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 21:27:57 INFO SparkContext:59 - Created broadcast 0 from textFile at TFIDFSuite.scala:24 -2016-06-13 21:27:58 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 21:27:58 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 21:27:58 INFO SparkContext:59 - Starting job: foreach at TFIDFSuite.scala:29 -2016-06-13 21:27:58 INFO DAGScheduler:59 - Got job 0 (foreach at TFIDFSuite.scala:29) with 2 output partitions -2016-06-13 21:27:58 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at TFIDFSuite.scala:29) -2016-06-13 21:27:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:27:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:27:58 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at HashingTF.scala:78), which has no missing parents -2016-06-13 21:27:58 INFO MemoryStore:59 - ensureFreeSpace(3704) called with curMem=171580, maxMem=766075207 -2016-06-13 21:27:58 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-06-13 21:27:58 INFO MemoryStore:59 - ensureFreeSpace(2095) called with curMem=175284, maxMem=766075207 -2016-06-13 21:27:58 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 21:27:58 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53051 (size: 2.0 KB, free: 730.6 MB) -2016-06-13 21:27:58 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:27:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at HashingTF.scala:78) -2016-06-13 21:27:58 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 21:27:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:27:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:27:58 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 21:27:58 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 21:27:58 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:37+37 -2016-06-13 21:27:58 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+37 -2016-06-13 21:27:58 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 21:27:58 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 21:27:58 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 21:27:58 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 21:27:58 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 21:27:58 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-06-13 21:27:58 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-06-13 21:27:58 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 113 ms on localhost (1/2) -2016-06-13 21:27:58 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 97 ms on localhost (2/2) -2016-06-13 21:27:58 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 21:27:58 INFO DAGScheduler:59 - ResultStage 0 (foreach at TFIDFSuite.scala:29) finished in 0.130 s -2016-06-13 21:27:58 INFO DAGScheduler:59 - Job 0 finished: foreach at TFIDFSuite.scala:29, took 0.275600 s -2016-06-13 21:27:58 INFO SparkContext:59 - Starting job: treeAggregate at IDF.scala:56 -2016-06-13 21:27:58 INFO DAGScheduler:59 - Got job 1 (treeAggregate at IDF.scala:56) with 2 output partitions -2016-06-13 21:27:58 INFO DAGScheduler:59 - Final stage: ResultStage 1(treeAggregate at IDF.scala:56) -2016-06-13 21:27:58 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:27:58 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:27:58 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at IDF.scala:56), which has no missing parents -2016-06-13 21:27:58 INFO MemoryStore:59 - ensureFreeSpace(4504) called with curMem=177379, maxMem=766075207 -2016-06-13 21:27:58 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 21:27:58 INFO MemoryStore:59 - ensureFreeSpace(2416) called with curMem=181883, maxMem=766075207 -2016-06-13 21:27:58 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 21:27:58 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53051 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 21:27:58 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:27:58 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at IDF.scala:56) -2016-06-13 21:27:58 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 21:27:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:27:58 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:27:58 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 21:27:58 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 21:27:58 INFO CacheManager:59 - Partition rdd_3_0 not found, computing it -2016-06-13 21:27:58 INFO CacheManager:59 - Partition rdd_3_1 not found, computing it -2016-06-13 21:27:58 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+37 -2016-06-13 21:27:58 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:37+37 -2016-06-13 21:27:58 INFO MemoryStore:59 - ensureFreeSpace(264) called with curMem=184299, maxMem=766075207 -2016-06-13 21:27:58 INFO MemoryStore:59 - Block rdd_3_1 stored as values in memory (estimated size 264.0 B, free 730.4 MB) -2016-06-13 21:27:58 INFO MemoryStore:59 - ensureFreeSpace(272) called with curMem=184563, maxMem=766075207 -2016-06-13 21:27:58 INFO MemoryStore:59 - Block rdd_3_0 stored as values in memory (estimated size 272.0 B, free 730.4 MB) -2016-06-13 21:27:58 INFO BlockManagerInfo:59 - Added rdd_3_1 in memory on localhost:53051 (size: 264.0 B, free: 730.6 MB) -2016-06-13 21:27:58 INFO BlockManagerInfo:59 - Added rdd_3_0 in memory on localhost:53051 (size: 272.0 B, free: 730.6 MB) -2016-06-13 21:27:59 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 8432508 bytes result sent to driver -2016-06-13 21:27:59 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 8432508 bytes result sent to driver -2016-06-13 21:27:59 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53051 in memory (size: 2.0 KB, free: 730.6 MB) -2016-06-13 21:27:59 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 21:27:59 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 620 ms on localhost (1/2) -2016-06-13 21:27:59 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 620 ms on localhost (2/2) -2016-06-13 21:27:59 INFO DAGScheduler:59 - ResultStage 1 (treeAggregate at IDF.scala:56) finished in 0.621 s -2016-06-13 21:27:59 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 21:27:59 INFO DAGScheduler:59 - Job 1 finished: treeAggregate at IDF.scala:56, took 0.657921 s -2016-06-13 21:27:59 INFO MemoryStore:59 - ensureFreeSpace(8388664) called with curMem=179036, maxMem=766075207 -2016-06-13 21:27:59 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 8.0 MB, free 722.4 MB) -2016-06-13 21:27:59 INFO MemoryStore:59 - ensureFreeSpace(397521) called with curMem=8567700, maxMem=766075207 -2016-06-13 21:27:59 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 388.2 KB, free 722.0 MB) -2016-06-13 21:27:59 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53051 (size: 388.2 KB, free: 730.2 MB) -2016-06-13 21:27:59 INFO SparkContext:59 - Created broadcast 3 from broadcast at IDF.scala:181 -2016-06-13 21:28:00 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53051 in memory (size: 2.4 KB, free: 730.2 MB) -2016-06-13 21:28:00 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 21:28:00 INFO SparkContext:59 - Starting job: foreach at TFIDFSuite.scala:39 -2016-06-13 21:28:00 INFO DAGScheduler:59 - Got job 2 (foreach at TFIDFSuite.scala:39) with 2 output partitions -2016-06-13 21:28:00 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at TFIDFSuite.scala:39) -2016-06-13 21:28:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:28:00 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:28:00 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[5] at mapPartitions at IDF.scala:182), which has no missing parents -2016-06-13 21:28:00 INFO MemoryStore:59 - ensureFreeSpace(4016) called with curMem=8958301, maxMem=766075207 -2016-06-13 21:28:00 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.9 KB, free 722.0 MB) -2016-06-13 21:28:00 INFO MemoryStore:59 - ensureFreeSpace(2224) called with curMem=8962317, maxMem=766075207 -2016-06-13 21:28:00 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.2 KB, free 722.0 MB) -2016-06-13 21:28:00 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53051 (size: 2.2 KB, free: 730.2 MB) -2016-06-13 21:28:00 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:28:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[5] at mapPartitions at IDF.scala:182) -2016-06-13 21:28:00 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 21:28:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:28:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:28:00 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 21:28:00 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 21:28:00 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2044 bytes result sent to driver -2016-06-13 21:28:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 37 ms on localhost (1/2) -2016-06-13 21:28:00 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 21:28:00 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 21:28:00 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2044 bytes result sent to driver -2016-06-13 21:28:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 58 ms on localhost (2/2) -2016-06-13 21:28:00 INFO DAGScheduler:59 - ResultStage 2 (foreach at TFIDFSuite.scala:39) finished in 0.059 s -2016-06-13 21:28:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 21:28:00 INFO DAGScheduler:59 - Job 2 finished: foreach at TFIDFSuite.scala:39, took 0.132384 s -2016-06-13 21:28:00 INFO SparkContext:59 - Starting job: treeAggregate at IDF.scala:56 -2016-06-13 21:28:00 INFO DAGScheduler:59 - Got job 3 (treeAggregate at IDF.scala:56) with 2 output partitions -2016-06-13 21:28:00 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at IDF.scala:56) -2016-06-13 21:28:00 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:28:00 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:28:00 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[6] at treeAggregate at IDF.scala:56), which has no missing parents -2016-06-13 21:28:00 INFO MemoryStore:59 - ensureFreeSpace(4504) called with curMem=8964541, maxMem=766075207 -2016-06-13 21:28:00 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 4.4 KB, free 722.0 MB) -2016-06-13 21:28:00 INFO MemoryStore:59 - ensureFreeSpace(2418) called with curMem=8969045, maxMem=766075207 -2016-06-13 21:28:00 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.4 KB, free 722.0 MB) -2016-06-13 21:28:00 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53051 (size: 2.4 KB, free: 730.2 MB) -2016-06-13 21:28:00 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:28:00 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[6] at treeAggregate at IDF.scala:56) -2016-06-13 21:28:00 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 21:28:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:28:00 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:28:00 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-06-13 21:28:00 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-06-13 21:28:00 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 21:28:00 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 21:28:00 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 21:28:00 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 8431928 bytes result sent to driver -2016-06-13 21:28:00 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53051 in memory (size: 2.2 KB, free: 730.2 MB) -2016-06-13 21:28:00 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 221 ms on localhost (1/2) -2016-06-13 21:28:00 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 8431928 bytes result sent to driver -2016-06-13 21:28:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 239 ms on localhost (2/2) -2016-06-13 21:28:00 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at IDF.scala:56) finished in 0.239 s -2016-06-13 21:28:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 21:28:00 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at IDF.scala:56, took 0.271694 s -2016-06-13 21:28:00 INFO MemoryStore:59 - ensureFreeSpace(8388664) called with curMem=8965223, maxMem=766075207 -2016-06-13 21:28:00 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 8.0 MB, free 714.0 MB) -2016-06-13 21:28:00 INFO MemoryStore:59 - ensureFreeSpace(395684) called with curMem=17353887, maxMem=766075207 -2016-06-13 21:28:00 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 386.4 KB, free 713.7 MB) -2016-06-13 21:28:00 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53051 (size: 386.4 KB, free: 729.8 MB) -2016-06-13 21:28:00 INFO SparkContext:59 - Created broadcast 6 from broadcast at IDF.scala:181 -2016-06-13 21:28:48 INFO SparkContext:59 - Starting job: foreach at TFIDFSuite.scala:49 -2016-06-13 21:28:48 INFO DAGScheduler:59 - Got job 4 (foreach at TFIDFSuite.scala:49) with 2 output partitions -2016-06-13 21:28:48 INFO DAGScheduler:59 - Final stage: ResultStage 4(foreach at TFIDFSuite.scala:49) -2016-06-13 21:28:48 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:28:48 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:28:48 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[7] at mapPartitions at IDF.scala:182), which has no missing parents -2016-06-13 21:28:48 INFO MemoryStore:59 - ensureFreeSpace(4016) called with curMem=17749571, maxMem=766075207 -2016-06-13 21:28:48 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.9 KB, free 713.7 MB) -2016-06-13 21:28:48 INFO MemoryStore:59 - ensureFreeSpace(2224) called with curMem=17753587, maxMem=766075207 -2016-06-13 21:28:48 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 713.7 MB) -2016-06-13 21:28:48 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53051 (size: 2.2 KB, free: 729.8 MB) -2016-06-13 21:28:48 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:28:48 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[7] at mapPartitions at IDF.scala:182) -2016-06-13 21:28:48 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 21:28:48 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:28:48 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:28:48 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-06-13 21:28:48 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-06-13 21:28:48 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 21:28:48 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2044 bytes result sent to driver -2016-06-13 21:28:48 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 21:28:48 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 14 ms on localhost (1/2) -2016-06-13 21:28:48 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2044 bytes result sent to driver -2016-06-13 21:28:48 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 19 ms on localhost (2/2) -2016-06-13 21:28:48 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 21:28:48 INFO DAGScheduler:59 - ResultStage 4 (foreach at TFIDFSuite.scala:49) finished in 0.021 s -2016-06-13 21:28:48 INFO DAGScheduler:59 - Job 4 finished: foreach at TFIDFSuite.scala:49, took 0.062483 s -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 21:28:48 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 21:28:48 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 21:28:48 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 21:28:49 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 21:28:49 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 21:28:49 INFO BlockManager:59 - BlockManager stopped -2016-06-13 21:28:49 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 21:28:49 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 21:28:49 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 21:28:49 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 21:28:49 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 21:28:49 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 21:28:49 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-2fa11711-bfee-499d-9587-2641fc932231 -2016-06-13 21:40:29 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 21:40:29 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 21:40:29 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 21:40:29 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 21:40:29 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 21:40:30 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 21:40:30 INFO Remoting:74 - Starting remoting -2016-06-13 21:40:30 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:53237] -2016-06-13 21:40:30 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53237. -2016-06-13 21:40:30 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 21:40:30 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 21:40:30 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-5fc917ca-f3d5-4b65-8d32-7c66110439c1 -2016-06-13 21:40:30 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 21:40:31 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-af9e2597-e2a5-4394-9b20-eacb0ce396de\httpd-eb9fdd0e-7e61-419f-896d-b5ae336beccb -2016-06-13 21:40:31 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 21:40:31 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:40:31 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53238 -2016-06-13 21:40:31 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53238. -2016-06-13 21:40:31 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 21:40:31 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:40:31 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 21:40:31 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 21:40:31 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 21:40:31 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 21:40:31 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 21:40:31 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53257. -2016-06-13 21:40:31 INFO NettyBlockTransferService:59 - Server created on 53257 -2016-06-13 21:40:31 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 21:40:31 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53257 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53257) -2016-06-13 21:40:31 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 21:40:32 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 21:40:32 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 21:40:32 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 21:40:32 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 21:40:32 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53257 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 21:40:32 INFO SparkContext:59 - Created broadcast 0 from textFile at TFIDFSuite.scala:24 -2016-06-13 21:40:33 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 21:40:34 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 21:40:34 INFO SparkContext:59 - Starting job: foreach at TFIDFSuite.scala:29 -2016-06-13 21:40:34 INFO DAGScheduler:59 - Got job 0 (foreach at TFIDFSuite.scala:29) with 2 output partitions -2016-06-13 21:40:34 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at TFIDFSuite.scala:29) -2016-06-13 21:40:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:40:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:40:34 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at HashingTF.scala:78), which has no missing parents -2016-06-13 21:40:34 INFO MemoryStore:59 - ensureFreeSpace(3704) called with curMem=171580, maxMem=766075207 -2016-06-13 21:40:34 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-06-13 21:40:34 INFO MemoryStore:59 - ensureFreeSpace(2095) called with curMem=175284, maxMem=766075207 -2016-06-13 21:40:34 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 21:40:34 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53257 (size: 2.0 KB, free: 730.6 MB) -2016-06-13 21:40:34 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:40:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at HashingTF.scala:78) -2016-06-13 21:40:34 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 21:40:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:40:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:40:34 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 21:40:34 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 21:40:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+37 -2016-06-13 21:40:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:37+37 -2016-06-13 21:40:34 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 21:40:34 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 21:40:34 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 21:40:34 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 21:40:34 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 21:40:34 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-06-13 21:40:34 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-06-13 21:40:34 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 90 ms on localhost (1/2) -2016-06-13 21:40:34 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 109 ms on localhost (2/2) -2016-06-13 21:40:34 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 21:40:34 INFO DAGScheduler:59 - ResultStage 0 (foreach at TFIDFSuite.scala:29) finished in 0.124 s -2016-06-13 21:40:34 INFO DAGScheduler:59 - Job 0 finished: foreach at TFIDFSuite.scala:29, took 0.251158 s -2016-06-13 21:40:34 INFO SparkContext:59 - Starting job: treeAggregate at IDF.scala:56 -2016-06-13 21:40:34 INFO DAGScheduler:59 - Got job 1 (treeAggregate at IDF.scala:56) with 2 output partitions -2016-06-13 21:40:34 INFO DAGScheduler:59 - Final stage: ResultStage 1(treeAggregate at IDF.scala:56) -2016-06-13 21:40:34 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:40:34 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:40:34 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at IDF.scala:56), which has no missing parents -2016-06-13 21:40:34 INFO MemoryStore:59 - ensureFreeSpace(4504) called with curMem=177379, maxMem=766075207 -2016-06-13 21:40:34 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 21:40:34 INFO MemoryStore:59 - ensureFreeSpace(2416) called with curMem=181883, maxMem=766075207 -2016-06-13 21:40:34 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 21:40:34 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53257 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 21:40:34 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:40:34 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at IDF.scala:56) -2016-06-13 21:40:34 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 21:40:34 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:40:34 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:40:34 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 21:40:34 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 21:40:34 INFO CacheManager:59 - Partition rdd_3_0 not found, computing it -2016-06-13 21:40:34 INFO CacheManager:59 - Partition rdd_3_1 not found, computing it -2016-06-13 21:40:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+37 -2016-06-13 21:40:34 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:37+37 -2016-06-13 21:40:34 INFO MemoryStore:59 - ensureFreeSpace(264) called with curMem=184299, maxMem=766075207 -2016-06-13 21:40:34 INFO MemoryStore:59 - Block rdd_3_1 stored as values in memory (estimated size 264.0 B, free 730.4 MB) -2016-06-13 21:40:34 INFO MemoryStore:59 - ensureFreeSpace(272) called with curMem=184563, maxMem=766075207 -2016-06-13 21:40:34 INFO MemoryStore:59 - Block rdd_3_0 stored as values in memory (estimated size 272.0 B, free 730.4 MB) -2016-06-13 21:40:34 INFO BlockManagerInfo:59 - Added rdd_3_1 in memory on localhost:53257 (size: 264.0 B, free: 730.6 MB) -2016-06-13 21:40:34 INFO BlockManagerInfo:59 - Added rdd_3_0 in memory on localhost:53257 (size: 272.0 B, free: 730.6 MB) -2016-06-13 21:40:34 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53257 in memory (size: 2.0 KB, free: 730.6 MB) -2016-06-13 21:40:35 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 8432508 bytes result sent to driver -2016-06-13 21:40:35 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 21:40:35 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 8432508 bytes result sent to driver -2016-06-13 21:40:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 685 ms on localhost (1/2) -2016-06-13 21:40:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 684 ms on localhost (2/2) -2016-06-13 21:40:35 INFO DAGScheduler:59 - ResultStage 1 (treeAggregate at IDF.scala:56) finished in 0.686 s -2016-06-13 21:40:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 21:40:35 INFO DAGScheduler:59 - Job 1 finished: treeAggregate at IDF.scala:56, took 0.742306 s -2016-06-13 21:40:35 INFO MemoryStore:59 - ensureFreeSpace(8388664) called with curMem=179036, maxMem=766075207 -2016-06-13 21:40:35 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 8.0 MB, free 722.4 MB) -2016-06-13 21:40:35 INFO MemoryStore:59 - ensureFreeSpace(397521) called with curMem=8567700, maxMem=766075207 -2016-06-13 21:40:35 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 388.2 KB, free 722.0 MB) -2016-06-13 21:40:35 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53257 (size: 388.2 KB, free: 730.2 MB) -2016-06-13 21:40:35 INFO SparkContext:59 - Created broadcast 3 from broadcast at IDF.scala:181 -2016-06-13 21:40:35 INFO SparkContext:59 - Starting job: foreach at TFIDFSuite.scala:39 -2016-06-13 21:40:35 INFO DAGScheduler:59 - Got job 2 (foreach at TFIDFSuite.scala:39) with 2 output partitions -2016-06-13 21:40:35 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at TFIDFSuite.scala:39) -2016-06-13 21:40:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:40:35 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:40:35 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[5] at mapPartitions at IDF.scala:182), which has no missing parents -2016-06-13 21:40:35 INFO MemoryStore:59 - ensureFreeSpace(4016) called with curMem=8965221, maxMem=766075207 -2016-06-13 21:40:35 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.9 KB, free 722.0 MB) -2016-06-13 21:40:35 INFO MemoryStore:59 - ensureFreeSpace(2224) called with curMem=8969237, maxMem=766075207 -2016-06-13 21:40:35 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.2 KB, free 722.0 MB) -2016-06-13 21:40:35 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53257 (size: 2.2 KB, free: 730.2 MB) -2016-06-13 21:40:35 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:40:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[5] at mapPartitions at IDF.scala:182) -2016-06-13 21:40:35 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 21:40:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:40:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:40:35 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 21:40:35 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 21:40:35 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 21:40:35 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2044 bytes result sent to driver -2016-06-13 21:40:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 22 ms on localhost (1/2) -2016-06-13 21:40:35 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 21:40:35 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2044 bytes result sent to driver -2016-06-13 21:40:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 26 ms on localhost (2/2) -2016-06-13 21:40:35 INFO DAGScheduler:59 - ResultStage 2 (foreach at TFIDFSuite.scala:39) finished in 0.028 s -2016-06-13 21:40:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 21:40:35 INFO DAGScheduler:59 - Job 2 finished: foreach at TFIDFSuite.scala:39, took 0.052465 s -2016-06-13 21:40:35 INFO SparkContext:59 - Starting job: treeAggregate at IDF.scala:56 -2016-06-13 21:40:35 INFO DAGScheduler:59 - Got job 3 (treeAggregate at IDF.scala:56) with 2 output partitions -2016-06-13 21:40:35 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at IDF.scala:56) -2016-06-13 21:40:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:40:35 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:40:35 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[6] at treeAggregate at IDF.scala:56), which has no missing parents -2016-06-13 21:40:35 INFO MemoryStore:59 - ensureFreeSpace(4504) called with curMem=8971461, maxMem=766075207 -2016-06-13 21:40:35 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 4.4 KB, free 722.0 MB) -2016-06-13 21:40:35 INFO MemoryStore:59 - ensureFreeSpace(2418) called with curMem=8975965, maxMem=766075207 -2016-06-13 21:40:35 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.4 KB, free 722.0 MB) -2016-06-13 21:40:35 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53257 (size: 2.4 KB, free: 730.2 MB) -2016-06-13 21:40:35 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:40:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[6] at treeAggregate at IDF.scala:56) -2016-06-13 21:40:35 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 21:40:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:40:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:40:35 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-06-13 21:40:35 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-06-13 21:40:35 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 21:40:35 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 21:40:35 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 21:40:35 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 8431928 bytes result sent to driver -2016-06-13 21:40:35 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53257 in memory (size: 2.2 KB, free: 730.2 MB) -2016-06-13 21:40:35 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 8431928 bytes result sent to driver -2016-06-13 21:40:35 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53257 in memory (size: 2.4 KB, free: 730.2 MB) -2016-06-13 21:40:35 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 21:40:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 274 ms on localhost (1/2) -2016-06-13 21:40:35 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at IDF.scala:56) finished in 0.277 s -2016-06-13 21:40:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 273 ms on localhost (2/2) -2016-06-13 21:40:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 21:40:35 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at IDF.scala:56, took 0.337938 s -2016-06-13 21:40:35 INFO MemoryStore:59 - ensureFreeSpace(8388664) called with curMem=8965223, maxMem=766075207 -2016-06-13 21:40:35 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 8.0 MB, free 714.0 MB) -2016-06-13 21:40:35 INFO MemoryStore:59 - ensureFreeSpace(395684) called with curMem=17353887, maxMem=766075207 -2016-06-13 21:40:35 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 386.4 KB, free 713.7 MB) -2016-06-13 21:40:35 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53257 (size: 386.4 KB, free: 729.8 MB) -2016-06-13 21:40:35 INFO SparkContext:59 - Created broadcast 6 from broadcast at IDF.scala:181 -2016-06-13 21:40:35 INFO SparkContext:59 - Starting job: foreach at TFIDFSuite.scala:49 -2016-06-13 21:40:35 INFO DAGScheduler:59 - Got job 4 (foreach at TFIDFSuite.scala:49) with 2 output partitions -2016-06-13 21:40:35 INFO DAGScheduler:59 - Final stage: ResultStage 4(foreach at TFIDFSuite.scala:49) -2016-06-13 21:40:35 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:40:35 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:40:35 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[7] at mapPartitions at IDF.scala:182), which has no missing parents -2016-06-13 21:40:35 INFO MemoryStore:59 - ensureFreeSpace(4016) called with curMem=17749571, maxMem=766075207 -2016-06-13 21:40:35 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.9 KB, free 713.7 MB) -2016-06-13 21:40:35 INFO MemoryStore:59 - ensureFreeSpace(2224) called with curMem=17753587, maxMem=766075207 -2016-06-13 21:40:35 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 713.7 MB) -2016-06-13 21:40:35 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53257 (size: 2.2 KB, free: 729.8 MB) -2016-06-13 21:40:35 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:40:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[7] at mapPartitions at IDF.scala:182) -2016-06-13 21:40:35 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 21:40:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:40:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:40:35 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-06-13 21:40:35 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-06-13 21:40:35 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 21:40:35 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 21:40:35 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2044 bytes result sent to driver -2016-06-13 21:40:35 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2044 bytes result sent to driver -2016-06-13 21:40:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 35 ms on localhost (1/2) -2016-06-13 21:40:35 INFO DAGScheduler:59 - ResultStage 4 (foreach at TFIDFSuite.scala:49) finished in 0.038 s -2016-06-13 21:40:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 33 ms on localhost (2/2) -2016-06-13 21:40:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 21:40:35 INFO DAGScheduler:59 - Job 4 finished: foreach at TFIDFSuite.scala:49, took 0.151559 s -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 21:40:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 21:40:36 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 21:40:36 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 21:40:36 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 21:40:36 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 21:40:36 INFO BlockManager:59 - BlockManager stopped -2016-06-13 21:40:36 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 21:40:36 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 21:40:36 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 21:40:36 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 21:40:36 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 21:40:36 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 21:40:36 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-af9e2597-e2a5-4394-9b20-eacb0ce396de -2016-06-13 21:40:36 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remoting shut down. -2016-06-13 21:47:26 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 21:47:27 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 21:47:27 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 21:47:27 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 21:47:27 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 21:47:28 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 21:47:28 INFO Remoting:74 - Starting remoting -2016-06-13 21:47:28 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:53463] -2016-06-13 21:47:28 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53463. -2016-06-13 21:47:28 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 21:47:28 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 21:47:28 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-a0c7e94c-a769-4648-9923-bd3fbdce896b -2016-06-13 21:47:28 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 21:47:28 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-32a085d4-e3bd-425f-bcaa-f10267239077\httpd-ddf4c1fd-1e7f-4542-8ca7-a3e53c01b227 -2016-06-13 21:47:28 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 21:47:29 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:47:29 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53464 -2016-06-13 21:47:29 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53464. -2016-06-13 21:47:29 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 21:47:29 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:47:29 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 21:47:29 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 21:47:29 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 21:47:29 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 21:47:29 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 21:47:29 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53484. -2016-06-13 21:47:29 INFO NettyBlockTransferService:59 - Server created on 53484 -2016-06-13 21:47:29 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 21:47:29 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53484 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53484) -2016-06-13 21:47:29 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 21:47:30 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 21:47:30 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 21:47:30 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 21:47:30 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 21:47:30 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53484 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 21:47:30 INFO SparkContext:59 - Created broadcast 0 from textFile at TFIDFSuite.scala:24 -2016-06-13 21:47:31 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 21:47:31 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 21:47:31 INFO SparkContext:59 - Starting job: foreach at TFIDFSuite.scala:29 -2016-06-13 21:47:31 INFO DAGScheduler:59 - Got job 0 (foreach at TFIDFSuite.scala:29) with 2 output partitions -2016-06-13 21:47:31 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at TFIDFSuite.scala:29) -2016-06-13 21:47:31 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:47:31 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:47:31 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at HashingTF.scala:78), which has no missing parents -2016-06-13 21:47:32 INFO MemoryStore:59 - ensureFreeSpace(3704) called with curMem=171580, maxMem=766075207 -2016-06-13 21:47:32 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-06-13 21:47:32 INFO MemoryStore:59 - ensureFreeSpace(2095) called with curMem=175284, maxMem=766075207 -2016-06-13 21:47:32 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 21:47:32 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53484 (size: 2.0 KB, free: 730.6 MB) -2016-06-13 21:47:32 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:47:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at HashingTF.scala:78) -2016-06-13 21:47:32 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 21:47:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:47:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:47:32 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 21:47:32 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 21:47:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+37 -2016-06-13 21:47:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:37+37 -2016-06-13 21:47:32 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 21:47:32 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 21:47:32 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 21:47:32 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 21:47:32 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 21:47:32 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-06-13 21:47:32 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-06-13 21:47:32 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 93 ms on localhost (1/2) -2016-06-13 21:47:32 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 114 ms on localhost (2/2) -2016-06-13 21:47:32 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 21:47:32 INFO DAGScheduler:59 - ResultStage 0 (foreach at TFIDFSuite.scala:29) finished in 0.168 s -2016-06-13 21:47:32 INFO DAGScheduler:59 - Job 0 finished: foreach at TFIDFSuite.scala:29, took 0.337598 s -2016-06-13 21:47:32 INFO SparkContext:59 - Starting job: treeAggregate at IDF.scala:56 -2016-06-13 21:47:32 INFO DAGScheduler:59 - Got job 1 (treeAggregate at IDF.scala:56) with 2 output partitions -2016-06-13 21:47:32 INFO DAGScheduler:59 - Final stage: ResultStage 1(treeAggregate at IDF.scala:56) -2016-06-13 21:47:32 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:47:32 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:47:32 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at IDF.scala:56), which has no missing parents -2016-06-13 21:47:32 INFO MemoryStore:59 - ensureFreeSpace(4504) called with curMem=177379, maxMem=766075207 -2016-06-13 21:47:32 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 21:47:32 INFO MemoryStore:59 - ensureFreeSpace(2416) called with curMem=181883, maxMem=766075207 -2016-06-13 21:47:32 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 21:47:32 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53484 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 21:47:32 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:47:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at IDF.scala:56) -2016-06-13 21:47:32 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 21:47:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:47:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:47:32 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 21:47:32 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 21:47:32 INFO CacheManager:59 - Partition rdd_3_1 not found, computing it -2016-06-13 21:47:32 INFO CacheManager:59 - Partition rdd_3_0 not found, computing it -2016-06-13 21:47:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:37+37 -2016-06-13 21:47:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+37 -2016-06-13 21:47:32 INFO MemoryStore:59 - ensureFreeSpace(272) called with curMem=184299, maxMem=766075207 -2016-06-13 21:47:32 INFO MemoryStore:59 - Block rdd_3_0 stored as values in memory (estimated size 272.0 B, free 730.4 MB) -2016-06-13 21:47:32 INFO MemoryStore:59 - ensureFreeSpace(264) called with curMem=184571, maxMem=766075207 -2016-06-13 21:47:32 INFO MemoryStore:59 - Block rdd_3_1 stored as values in memory (estimated size 264.0 B, free 730.4 MB) -2016-06-13 21:47:32 INFO BlockManagerInfo:59 - Added rdd_3_0 in memory on localhost:53484 (size: 272.0 B, free: 730.6 MB) -2016-06-13 21:47:32 INFO BlockManagerInfo:59 - Added rdd_3_1 in memory on localhost:53484 (size: 264.0 B, free: 730.6 MB) -2016-06-13 21:47:32 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 8432508 bytes result sent to driver -2016-06-13 21:47:32 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 8432508 bytes result sent to driver -2016-06-13 21:47:32 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53484 in memory (size: 2.0 KB, free: 730.6 MB) -2016-06-13 21:47:32 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 21:47:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 667 ms on localhost (1/2) -2016-06-13 21:47:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 667 ms on localhost (2/2) -2016-06-13 21:47:33 INFO DAGScheduler:59 - ResultStage 1 (treeAggregate at IDF.scala:56) finished in 0.669 s -2016-06-13 21:47:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 21:47:33 INFO DAGScheduler:59 - Job 1 finished: treeAggregate at IDF.scala:56, took 0.698636 s -2016-06-13 21:47:33 INFO MemoryStore:59 - ensureFreeSpace(8388664) called with curMem=179036, maxMem=766075207 -2016-06-13 21:47:33 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 8.0 MB, free 722.4 MB) -2016-06-13 21:47:33 INFO MemoryStore:59 - ensureFreeSpace(397521) called with curMem=8567700, maxMem=766075207 -2016-06-13 21:47:33 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 388.2 KB, free 722.0 MB) -2016-06-13 21:47:33 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53484 (size: 388.2 KB, free: 730.2 MB) -2016-06-13 21:47:33 INFO SparkContext:59 - Created broadcast 3 from broadcast at IDF.scala:181 -2016-06-13 21:47:33 INFO SparkContext:59 - Starting job: foreach at TFIDFSuite.scala:39 -2016-06-13 21:47:33 INFO DAGScheduler:59 - Got job 2 (foreach at TFIDFSuite.scala:39) with 2 output partitions -2016-06-13 21:47:33 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at TFIDFSuite.scala:39) -2016-06-13 21:47:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:47:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:47:33 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[5] at mapPartitions at IDF.scala:182), which has no missing parents -2016-06-13 21:47:33 INFO MemoryStore:59 - ensureFreeSpace(4016) called with curMem=8965221, maxMem=766075207 -2016-06-13 21:47:33 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.9 KB, free 722.0 MB) -2016-06-13 21:47:33 INFO MemoryStore:59 - ensureFreeSpace(2224) called with curMem=8969237, maxMem=766075207 -2016-06-13 21:47:33 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.2 KB, free 722.0 MB) -2016-06-13 21:47:33 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53484 (size: 2.2 KB, free: 730.2 MB) -2016-06-13 21:47:33 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:47:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[5] at mapPartitions at IDF.scala:182) -2016-06-13 21:47:33 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 21:47:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:47:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:47:33 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 21:47:33 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 21:47:33 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 21:47:33 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2044 bytes result sent to driver -2016-06-13 21:47:33 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 21:47:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 23 ms on localhost (1/2) -2016-06-13 21:47:33 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2044 bytes result sent to driver -2016-06-13 21:47:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 24 ms on localhost (2/2) -2016-06-13 21:47:33 INFO DAGScheduler:59 - ResultStage 2 (foreach at TFIDFSuite.scala:39) finished in 0.028 s -2016-06-13 21:47:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 21:47:33 INFO DAGScheduler:59 - Job 2 finished: foreach at TFIDFSuite.scala:39, took 0.064042 s -2016-06-13 21:47:33 INFO SparkContext:59 - Starting job: treeAggregate at IDF.scala:56 -2016-06-13 21:47:33 INFO DAGScheduler:59 - Got job 3 (treeAggregate at IDF.scala:56) with 2 output partitions -2016-06-13 21:47:33 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at IDF.scala:56) -2016-06-13 21:47:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:47:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:47:33 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[6] at treeAggregate at IDF.scala:56), which has no missing parents -2016-06-13 21:47:33 INFO MemoryStore:59 - ensureFreeSpace(4504) called with curMem=8971461, maxMem=766075207 -2016-06-13 21:47:33 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 4.4 KB, free 722.0 MB) -2016-06-13 21:47:33 INFO MemoryStore:59 - ensureFreeSpace(2418) called with curMem=8975965, maxMem=766075207 -2016-06-13 21:47:33 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.4 KB, free 722.0 MB) -2016-06-13 21:47:33 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53484 (size: 2.4 KB, free: 730.2 MB) -2016-06-13 21:47:33 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:47:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[6] at treeAggregate at IDF.scala:56) -2016-06-13 21:47:33 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 21:47:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:47:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:47:33 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-06-13 21:47:33 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-06-13 21:47:33 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 21:47:33 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 21:47:33 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 8431928 bytes result sent to driver -2016-06-13 21:47:33 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53484 in memory (size: 2.4 KB, free: 730.2 MB) -2016-06-13 21:47:33 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 8431928 bytes result sent to driver -2016-06-13 21:47:33 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53484 in memory (size: 2.2 KB, free: 730.2 MB) -2016-06-13 21:47:33 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 21:47:33 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 21:47:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 214 ms on localhost (1/2) -2016-06-13 21:47:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 222 ms on localhost (2/2) -2016-06-13 21:47:33 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at IDF.scala:56) finished in 0.222 s -2016-06-13 21:47:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 21:47:33 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at IDF.scala:56, took 0.240602 s -2016-06-13 21:47:33 INFO MemoryStore:59 - ensureFreeSpace(8388664) called with curMem=8965223, maxMem=766075207 -2016-06-13 21:47:33 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 8.0 MB, free 714.0 MB) -2016-06-13 21:47:33 INFO MemoryStore:59 - ensureFreeSpace(395684) called with curMem=17353887, maxMem=766075207 -2016-06-13 21:47:33 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 386.4 KB, free 713.7 MB) -2016-06-13 21:47:33 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53484 (size: 386.4 KB, free: 729.8 MB) -2016-06-13 21:47:33 INFO SparkContext:59 - Created broadcast 6 from broadcast at IDF.scala:181 -2016-06-13 21:47:33 INFO SparkContext:59 - Starting job: foreach at TFIDFSuite.scala:49 -2016-06-13 21:47:33 INFO DAGScheduler:59 - Got job 4 (foreach at TFIDFSuite.scala:49) with 2 output partitions -2016-06-13 21:47:33 INFO DAGScheduler:59 - Final stage: ResultStage 4(foreach at TFIDFSuite.scala:49) -2016-06-13 21:47:33 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:47:33 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:47:33 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[7] at mapPartitions at IDF.scala:182), which has no missing parents -2016-06-13 21:47:33 INFO MemoryStore:59 - ensureFreeSpace(4016) called with curMem=17749571, maxMem=766075207 -2016-06-13 21:47:33 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.9 KB, free 713.7 MB) -2016-06-13 21:47:33 INFO MemoryStore:59 - ensureFreeSpace(2224) called with curMem=17753587, maxMem=766075207 -2016-06-13 21:47:33 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 713.7 MB) -2016-06-13 21:47:33 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53484 (size: 2.2 KB, free: 729.8 MB) -2016-06-13 21:47:33 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:47:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[7] at mapPartitions at IDF.scala:182) -2016-06-13 21:47:33 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 21:47:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:47:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:47:33 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-06-13 21:47:33 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-06-13 21:47:33 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 21:47:33 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2044 bytes result sent to driver -2016-06-13 21:47:33 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 21:47:33 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2044 bytes result sent to driver -2016-06-13 21:47:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 12 ms on localhost (1/2) -2016-06-13 21:47:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 16 ms on localhost (2/2) -2016-06-13 21:47:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 21:47:33 INFO DAGScheduler:59 - ResultStage 4 (foreach at TFIDFSuite.scala:49) finished in 0.018 s -2016-06-13 21:47:33 INFO DAGScheduler:59 - Job 4 finished: foreach at TFIDFSuite.scala:49, took 0.035258 s -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 21:47:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 21:47:33 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 21:47:33 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 21:47:33 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 21:47:33 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 21:47:33 INFO BlockManager:59 - BlockManager stopped -2016-06-13 21:47:33 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 21:47:33 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 21:47:33 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 21:47:33 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 21:47:33 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 21:47:33 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 21:47:33 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-32a085d4-e3bd-425f-bcaa-f10267239077 -2016-06-13 21:50:08 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 21:50:09 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 21:50:09 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 21:50:09 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 21:50:09 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 21:50:10 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 21:50:10 INFO Remoting:74 - Starting remoting -2016-06-13 21:50:10 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:53512] -2016-06-13 21:50:10 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53512. -2016-06-13 21:50:10 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 21:50:10 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 21:50:10 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-193322dd-a309-4256-b0d7-6fb6cbf3ed29 -2016-06-13 21:50:10 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 21:50:10 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-58f7222f-0a2b-43df-80e4-6c36979a4128\httpd-4889fed7-6031-41fc-8706-06ddd22b436a -2016-06-13 21:50:10 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 21:50:10 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:50:10 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53513 -2016-06-13 21:50:10 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53513. -2016-06-13 21:50:10 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 21:50:10 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:50:10 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 21:50:10 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 21:50:10 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 21:50:10 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 21:50:10 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 21:50:11 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53532. -2016-06-13 21:50:11 INFO NettyBlockTransferService:59 - Server created on 53532 -2016-06-13 21:50:11 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 21:50:11 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53532 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53532) -2016-06-13 21:50:11 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 21:50:12 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 21:50:12 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 21:50:12 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 21:50:12 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 21:50:12 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53532 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 21:50:12 INFO SparkContext:59 - Created broadcast 0 from textFile at TFIDFSuite.scala:24 -2016-06-13 21:50:13 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 21:50:13 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 21:50:13 INFO SparkContext:59 - Starting job: foreach at TFIDFSuite.scala:29 -2016-06-13 21:50:13 INFO DAGScheduler:59 - Got job 0 (foreach at TFIDFSuite.scala:29) with 2 output partitions -2016-06-13 21:50:13 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at TFIDFSuite.scala:29) -2016-06-13 21:50:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:50:13 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:50:13 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at HashingTF.scala:78), which has no missing parents -2016-06-13 21:50:13 INFO MemoryStore:59 - ensureFreeSpace(3704) called with curMem=171580, maxMem=766075207 -2016-06-13 21:50:13 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-06-13 21:50:13 INFO MemoryStore:59 - ensureFreeSpace(2095) called with curMem=175284, maxMem=766075207 -2016-06-13 21:50:13 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 21:50:13 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53532 (size: 2.0 KB, free: 730.6 MB) -2016-06-13 21:50:13 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:50:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at HashingTF.scala:78) -2016-06-13 21:50:13 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 21:50:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:50:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:50:13 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 21:50:13 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 21:50:13 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+40 -2016-06-13 21:50:13 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:40+40 -2016-06-13 21:50:13 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 21:50:13 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 21:50:13 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 21:50:13 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 21:50:13 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 21:50:13 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-06-13 21:50:13 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-06-13 21:50:13 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 108 ms on localhost (1/2) -2016-06-13 21:50:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 129 ms on localhost (2/2) -2016-06-13 21:50:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 21:50:13 INFO DAGScheduler:59 - ResultStage 0 (foreach at TFIDFSuite.scala:29) finished in 0.149 s -2016-06-13 21:50:13 INFO DAGScheduler:59 - Job 0 finished: foreach at TFIDFSuite.scala:29, took 0.345662 s -2016-06-13 21:50:13 INFO SparkContext:59 - Starting job: treeAggregate at IDF.scala:56 -2016-06-13 21:50:13 INFO DAGScheduler:59 - Got job 1 (treeAggregate at IDF.scala:56) with 2 output partitions -2016-06-13 21:50:13 INFO DAGScheduler:59 - Final stage: ResultStage 1(treeAggregate at IDF.scala:56) -2016-06-13 21:50:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:50:13 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:50:13 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at IDF.scala:56), which has no missing parents -2016-06-13 21:50:13 INFO MemoryStore:59 - ensureFreeSpace(4504) called with curMem=177379, maxMem=766075207 -2016-06-13 21:50:13 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 21:50:13 INFO MemoryStore:59 - ensureFreeSpace(2416) called with curMem=181883, maxMem=766075207 -2016-06-13 21:50:13 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 21:50:13 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53532 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 21:50:13 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:50:13 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at IDF.scala:56) -2016-06-13 21:50:13 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 21:50:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:50:13 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:50:13 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 21:50:13 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 21:50:13 INFO CacheManager:59 - Partition rdd_3_0 not found, computing it -2016-06-13 21:50:13 INFO CacheManager:59 - Partition rdd_3_1 not found, computing it -2016-06-13 21:50:13 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+40 -2016-06-13 21:50:13 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:40+40 -2016-06-13 21:50:13 INFO MemoryStore:59 - ensureFreeSpace(264) called with curMem=184299, maxMem=766075207 -2016-06-13 21:50:13 INFO MemoryStore:59 - Block rdd_3_1 stored as values in memory (estimated size 264.0 B, free 730.4 MB) -2016-06-13 21:50:13 INFO MemoryStore:59 - ensureFreeSpace(272) called with curMem=184563, maxMem=766075207 -2016-06-13 21:50:13 INFO MemoryStore:59 - Block rdd_3_0 stored as values in memory (estimated size 272.0 B, free 730.4 MB) -2016-06-13 21:50:13 INFO BlockManagerInfo:59 - Added rdd_3_1 in memory on localhost:53532 (size: 264.0 B, free: 730.6 MB) -2016-06-13 21:50:13 INFO BlockManagerInfo:59 - Added rdd_3_0 in memory on localhost:53532 (size: 272.0 B, free: 730.6 MB) -2016-06-13 21:50:14 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53532 in memory (size: 2.0 KB, free: 730.6 MB) -2016-06-13 21:50:14 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 21:50:14 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 8432508 bytes result sent to driver -2016-06-13 21:50:14 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 8432508 bytes result sent to driver -2016-06-13 21:50:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 666 ms on localhost (1/2) -2016-06-13 21:50:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 666 ms on localhost (2/2) -2016-06-13 21:50:14 INFO DAGScheduler:59 - ResultStage 1 (treeAggregate at IDF.scala:56) finished in 0.668 s -2016-06-13 21:50:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 21:50:14 INFO DAGScheduler:59 - Job 1 finished: treeAggregate at IDF.scala:56, took 0.709160 s -2016-06-13 21:50:14 INFO MemoryStore:59 - ensureFreeSpace(8388664) called with curMem=179036, maxMem=766075207 -2016-06-13 21:50:14 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 8.0 MB, free 722.4 MB) -2016-06-13 21:50:14 INFO MemoryStore:59 - ensureFreeSpace(397521) called with curMem=8567700, maxMem=766075207 -2016-06-13 21:50:14 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 388.2 KB, free 722.0 MB) -2016-06-13 21:50:14 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53532 (size: 388.2 KB, free: 730.2 MB) -2016-06-13 21:50:14 INFO SparkContext:59 - Created broadcast 3 from broadcast at IDF.scala:181 -2016-06-13 21:50:14 INFO SparkContext:59 - Starting job: foreach at TFIDFSuite.scala:39 -2016-06-13 21:50:14 INFO DAGScheduler:59 - Got job 2 (foreach at TFIDFSuite.scala:39) with 2 output partitions -2016-06-13 21:50:14 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at TFIDFSuite.scala:39) -2016-06-13 21:50:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:50:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:50:14 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[5] at mapPartitions at IDF.scala:182), which has no missing parents -2016-06-13 21:50:14 INFO MemoryStore:59 - ensureFreeSpace(4016) called with curMem=8965221, maxMem=766075207 -2016-06-13 21:50:14 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.9 KB, free 722.0 MB) -2016-06-13 21:50:14 INFO MemoryStore:59 - ensureFreeSpace(2224) called with curMem=8969237, maxMem=766075207 -2016-06-13 21:50:14 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.2 KB, free 722.0 MB) -2016-06-13 21:50:14 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53532 (size: 2.2 KB, free: 730.2 MB) -2016-06-13 21:50:14 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:50:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[5] at mapPartitions at IDF.scala:182) -2016-06-13 21:50:14 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 21:50:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:50:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:50:14 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 21:50:14 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 21:50:14 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 21:50:14 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2044 bytes result sent to driver -2016-06-13 21:50:14 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 21:50:14 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2044 bytes result sent to driver -2016-06-13 21:50:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 33 ms on localhost (1/2) -2016-06-13 21:50:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 32 ms on localhost (2/2) -2016-06-13 21:50:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 21:50:14 INFO DAGScheduler:59 - ResultStage 2 (foreach at TFIDFSuite.scala:39) finished in 0.037 s -2016-06-13 21:50:14 INFO DAGScheduler:59 - Job 2 finished: foreach at TFIDFSuite.scala:39, took 0.060361 s -2016-06-13 21:50:14 INFO SparkContext:59 - Starting job: treeAggregate at IDF.scala:56 -2016-06-13 21:50:14 INFO DAGScheduler:59 - Got job 3 (treeAggregate at IDF.scala:56) with 2 output partitions -2016-06-13 21:50:14 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at IDF.scala:56) -2016-06-13 21:50:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:50:14 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:50:14 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[6] at treeAggregate at IDF.scala:56), which has no missing parents -2016-06-13 21:50:14 INFO MemoryStore:59 - ensureFreeSpace(4504) called with curMem=8971461, maxMem=766075207 -2016-06-13 21:50:14 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 4.4 KB, free 722.0 MB) -2016-06-13 21:50:14 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 21:50:14 INFO MemoryStore:59 - ensureFreeSpace(2418) called with curMem=8975965, maxMem=766075207 -2016-06-13 21:50:14 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.4 KB, free 722.0 MB) -2016-06-13 21:50:14 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53532 (size: 2.4 KB, free: 730.2 MB) -2016-06-13 21:50:14 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:50:14 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53532 in memory (size: 2.2 KB, free: 730.2 MB) -2016-06-13 21:50:14 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[6] at treeAggregate at IDF.scala:56) -2016-06-13 21:50:14 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 21:50:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:50:14 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:50:14 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-06-13 21:50:14 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-06-13 21:50:14 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 21:50:14 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 21:50:14 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 8431928 bytes result sent to driver -2016-06-13 21:50:14 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 8431928 bytes result sent to driver -2016-06-13 21:50:14 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 212 ms on localhost (1/2) -2016-06-13 21:50:14 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53532 in memory (size: 2.4 KB, free: 730.2 MB) -2016-06-13 21:50:14 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 21:50:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 225 ms on localhost (2/2) -2016-06-13 21:50:14 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at IDF.scala:56) finished in 0.226 s -2016-06-13 21:50:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 21:50:14 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at IDF.scala:56, took 0.255865 s -2016-06-13 21:50:14 INFO MemoryStore:59 - ensureFreeSpace(8388664) called with curMem=8965223, maxMem=766075207 -2016-06-13 21:50:14 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 8.0 MB, free 714.0 MB) -2016-06-13 21:50:14 INFO MemoryStore:59 - ensureFreeSpace(395684) called with curMem=17353887, maxMem=766075207 -2016-06-13 21:50:14 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 386.4 KB, free 713.7 MB) -2016-06-13 21:50:14 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53532 (size: 386.4 KB, free: 729.8 MB) -2016-06-13 21:50:15 INFO SparkContext:59 - Created broadcast 6 from broadcast at IDF.scala:181 -2016-06-13 21:50:15 INFO SparkContext:59 - Starting job: foreach at TFIDFSuite.scala:49 -2016-06-13 21:50:15 INFO DAGScheduler:59 - Got job 4 (foreach at TFIDFSuite.scala:49) with 2 output partitions -2016-06-13 21:50:15 INFO DAGScheduler:59 - Final stage: ResultStage 4(foreach at TFIDFSuite.scala:49) -2016-06-13 21:50:15 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:50:15 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:50:15 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[7] at mapPartitions at IDF.scala:182), which has no missing parents -2016-06-13 21:50:15 INFO MemoryStore:59 - ensureFreeSpace(4016) called with curMem=17749571, maxMem=766075207 -2016-06-13 21:50:15 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.9 KB, free 713.7 MB) -2016-06-13 21:50:15 INFO MemoryStore:59 - ensureFreeSpace(2224) called with curMem=17753587, maxMem=766075207 -2016-06-13 21:50:15 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 713.7 MB) -2016-06-13 21:50:15 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53532 (size: 2.2 KB, free: 729.8 MB) -2016-06-13 21:50:15 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:50:15 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[7] at mapPartitions at IDF.scala:182) -2016-06-13 21:50:15 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 21:50:15 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:50:15 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:50:15 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-06-13 21:50:15 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-06-13 21:50:15 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 21:50:15 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 21:50:15 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2044 bytes result sent to driver -2016-06-13 21:50:15 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2044 bytes result sent to driver -2016-06-13 21:50:15 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 13 ms on localhost (1/2) -2016-06-13 21:50:15 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 14 ms on localhost (2/2) -2016-06-13 21:50:15 INFO DAGScheduler:59 - ResultStage 4 (foreach at TFIDFSuite.scala:49) finished in 0.016 s -2016-06-13 21:50:15 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 21:50:15 INFO DAGScheduler:59 - Job 4 finished: foreach at TFIDFSuite.scala:49, took 0.033890 s -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 21:50:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 21:50:15 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 21:50:15 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 21:50:15 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 21:50:15 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 21:50:15 INFO BlockManager:59 - BlockManager stopped -2016-06-13 21:50:15 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 21:50:15 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 21:50:15 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 21:50:15 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 21:50:15 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 21:50:15 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 21:50:15 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-58f7222f-0a2b-43df-80e4-6c36979a4128 -2016-06-13 21:51:15 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 21:51:16 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 21:51:16 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 21:51:16 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 21:51:16 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 21:51:17 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 21:51:17 INFO Remoting:74 - Starting remoting -2016-06-13 21:51:17 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:53553] -2016-06-13 21:51:17 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53553. -2016-06-13 21:51:17 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 21:51:17 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 21:51:17 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-53714f59-63ea-4e79-b134-17c2a5734f7e -2016-06-13 21:51:17 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 21:51:17 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-5649e0bc-1d33-4f6a-af58-4c650104a1a8\httpd-29a6d432-c643-4239-845f-430a44ec5b7c -2016-06-13 21:51:17 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 21:51:17 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:51:17 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53554 -2016-06-13 21:51:17 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53554. -2016-06-13 21:51:17 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 21:51:18 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:51:18 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 21:51:18 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 21:51:18 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 21:51:18 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 21:51:18 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 21:51:18 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53573. -2016-06-13 21:51:18 INFO NettyBlockTransferService:59 - Server created on 53573 -2016-06-13 21:51:18 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 21:51:18 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53573 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53573) -2016-06-13 21:51:18 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 21:51:19 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 21:51:19 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 21:51:19 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 21:51:19 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 21:51:19 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53573 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 21:51:19 INFO SparkContext:59 - Created broadcast 0 from textFile at TFIDFSuite.scala:24 -2016-06-13 21:51:20 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 21:51:20 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 21:51:20 INFO SparkContext:59 - Starting job: foreach at TFIDFSuite.scala:29 -2016-06-13 21:51:20 INFO DAGScheduler:59 - Got job 0 (foreach at TFIDFSuite.scala:29) with 2 output partitions -2016-06-13 21:51:20 INFO DAGScheduler:59 - Final stage: ResultStage 0(foreach at TFIDFSuite.scala:29) -2016-06-13 21:51:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:51:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:51:20 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[3] at map at HashingTF.scala:78), which has no missing parents -2016-06-13 21:51:20 INFO MemoryStore:59 - ensureFreeSpace(3704) called with curMem=171580, maxMem=766075207 -2016-06-13 21:51:20 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.6 KB, free 730.4 MB) -2016-06-13 21:51:20 INFO MemoryStore:59 - ensureFreeSpace(2095) called with curMem=175284, maxMem=766075207 -2016-06-13 21:51:20 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.0 KB, free 730.4 MB) -2016-06-13 21:51:20 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53573 (size: 2.0 KB, free: 730.6 MB) -2016-06-13 21:51:20 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:51:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at HashingTF.scala:78) -2016-06-13 21:51:20 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 21:51:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:51:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:51:20 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 21:51:20 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 21:51:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 21:51:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 21:51:20 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 21:51:20 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 21:51:20 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 21:51:20 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 21:51:20 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 21:51:21 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2044 bytes result sent to driver -2016-06-13 21:51:21 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2044 bytes result sent to driver -2016-06-13 21:51:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 112 ms on localhost (1/2) -2016-06-13 21:51:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 96 ms on localhost (2/2) -2016-06-13 21:51:21 INFO DAGScheduler:59 - ResultStage 0 (foreach at TFIDFSuite.scala:29) finished in 0.129 s -2016-06-13 21:51:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 21:51:21 INFO DAGScheduler:59 - Job 0 finished: foreach at TFIDFSuite.scala:29, took 0.296455 s -2016-06-13 21:51:21 INFO SparkContext:59 - Starting job: treeAggregate at IDF.scala:56 -2016-06-13 21:51:21 INFO DAGScheduler:59 - Got job 1 (treeAggregate at IDF.scala:56) with 2 output partitions -2016-06-13 21:51:21 INFO DAGScheduler:59 - Final stage: ResultStage 1(treeAggregate at IDF.scala:56) -2016-06-13 21:51:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:51:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:51:21 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at IDF.scala:56), which has no missing parents -2016-06-13 21:51:21 INFO MemoryStore:59 - ensureFreeSpace(4504) called with curMem=177379, maxMem=766075207 -2016-06-13 21:51:21 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.4 KB, free 730.4 MB) -2016-06-13 21:51:21 INFO MemoryStore:59 - ensureFreeSpace(2416) called with curMem=181883, maxMem=766075207 -2016-06-13 21:51:21 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 21:51:21 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53573 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 21:51:21 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:51:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at treeAggregate at IDF.scala:56) -2016-06-13 21:51:21 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 21:51:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:51:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:51:21 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 21:51:21 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 21:51:21 INFO CacheManager:59 - Partition rdd_3_1 not found, computing it -2016-06-13 21:51:21 INFO CacheManager:59 - Partition rdd_3_0 not found, computing it -2016-06-13 21:51:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 21:51:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 21:51:21 INFO MemoryStore:59 - ensureFreeSpace(184) called with curMem=184299, maxMem=766075207 -2016-06-13 21:51:21 INFO MemoryStore:59 - Block rdd_3_0 stored as values in memory (estimated size 184.0 B, free 730.4 MB) -2016-06-13 21:51:21 INFO MemoryStore:59 - ensureFreeSpace(344) called with curMem=184483, maxMem=766075207 -2016-06-13 21:51:21 INFO MemoryStore:59 - Block rdd_3_1 stored as values in memory (estimated size 344.0 B, free 730.4 MB) -2016-06-13 21:51:21 INFO BlockManagerInfo:59 - Added rdd_3_0 in memory on localhost:53573 (size: 184.0 B, free: 730.6 MB) -2016-06-13 21:51:21 INFO BlockManagerInfo:59 - Added rdd_3_1 in memory on localhost:53573 (size: 344.0 B, free: 730.6 MB) -2016-06-13 21:51:21 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:53573 in memory (size: 2.0 KB, free: 730.6 MB) -2016-06-13 21:51:21 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 21:51:21 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 8432508 bytes result sent to driver -2016-06-13 21:51:21 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 8432508 bytes result sent to driver -2016-06-13 21:51:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 664 ms on localhost (1/2) -2016-06-13 21:51:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 666 ms on localhost (2/2) -2016-06-13 21:51:21 INFO DAGScheduler:59 - ResultStage 1 (treeAggregate at IDF.scala:56) finished in 0.666 s -2016-06-13 21:51:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 21:51:21 INFO DAGScheduler:59 - Job 1 finished: treeAggregate at IDF.scala:56, took 0.702936 s -2016-06-13 21:51:21 INFO MemoryStore:59 - ensureFreeSpace(8388664) called with curMem=179028, maxMem=766075207 -2016-06-13 21:51:21 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 8.0 MB, free 722.4 MB) -2016-06-13 21:51:21 INFO MemoryStore:59 - ensureFreeSpace(397521) called with curMem=8567692, maxMem=766075207 -2016-06-13 21:51:21 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 388.2 KB, free 722.0 MB) -2016-06-13 21:51:21 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:53573 (size: 388.2 KB, free: 730.2 MB) -2016-06-13 21:51:21 INFO SparkContext:59 - Created broadcast 3 from broadcast at IDF.scala:181 -2016-06-13 21:51:21 INFO SparkContext:59 - Starting job: foreach at TFIDFSuite.scala:39 -2016-06-13 21:51:21 INFO DAGScheduler:59 - Got job 2 (foreach at TFIDFSuite.scala:39) with 2 output partitions -2016-06-13 21:51:21 INFO DAGScheduler:59 - Final stage: ResultStage 2(foreach at TFIDFSuite.scala:39) -2016-06-13 21:51:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:51:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:51:21 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[5] at mapPartitions at IDF.scala:182), which has no missing parents -2016-06-13 21:51:21 INFO MemoryStore:59 - ensureFreeSpace(4016) called with curMem=8965213, maxMem=766075207 -2016-06-13 21:51:21 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 3.9 KB, free 722.0 MB) -2016-06-13 21:51:21 INFO MemoryStore:59 - ensureFreeSpace(2224) called with curMem=8969229, maxMem=766075207 -2016-06-13 21:51:21 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.2 KB, free 722.0 MB) -2016-06-13 21:51:21 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:53573 (size: 2.2 KB, free: 730.2 MB) -2016-06-13 21:51:21 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:51:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[5] at mapPartitions at IDF.scala:182) -2016-06-13 21:51:21 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 21:51:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:51:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:51:21 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 21:51:21 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 21:51:21 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2044 bytes result sent to driver -2016-06-13 21:51:21 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 21:51:21 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 21:51:21 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2044 bytes result sent to driver -2016-06-13 21:51:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 27 ms on localhost (1/2) -2016-06-13 21:51:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 28 ms on localhost (2/2) -2016-06-13 21:51:21 INFO DAGScheduler:59 - ResultStage 2 (foreach at TFIDFSuite.scala:39) finished in 0.030 s -2016-06-13 21:51:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 21:51:21 INFO DAGScheduler:59 - Job 2 finished: foreach at TFIDFSuite.scala:39, took 0.049226 s -2016-06-13 21:51:21 INFO SparkContext:59 - Starting job: treeAggregate at IDF.scala:56 -2016-06-13 21:51:21 INFO DAGScheduler:59 - Got job 3 (treeAggregate at IDF.scala:56) with 2 output partitions -2016-06-13 21:51:21 INFO DAGScheduler:59 - Final stage: ResultStage 3(treeAggregate at IDF.scala:56) -2016-06-13 21:51:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:51:21 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:51:21 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[6] at treeAggregate at IDF.scala:56), which has no missing parents -2016-06-13 21:51:21 INFO MemoryStore:59 - ensureFreeSpace(4504) called with curMem=8971453, maxMem=766075207 -2016-06-13 21:51:21 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 4.4 KB, free 722.0 MB) -2016-06-13 21:51:21 INFO MemoryStore:59 - ensureFreeSpace(2418) called with curMem=8975957, maxMem=766075207 -2016-06-13 21:51:21 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 2.4 KB, free 722.0 MB) -2016-06-13 21:51:21 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:53573 (size: 2.4 KB, free: 730.2 MB) -2016-06-13 21:51:21 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:53573 in memory (size: 2.2 KB, free: 730.2 MB) -2016-06-13 21:51:21 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:51:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 3 (MapPartitionsRDD[6] at treeAggregate at IDF.scala:56) -2016-06-13 21:51:21 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 2 tasks -2016-06-13 21:51:21 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-06-13 21:51:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:51:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 3.0 (TID 7, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:51:21 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-06-13 21:51:21 INFO Executor:59 - Running task 1.0 in stage 3.0 (TID 7) -2016-06-13 21:51:21 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 21:51:21 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 21:51:22 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 8431928 bytes result sent to driver -2016-06-13 21:51:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 58 ms on localhost (1/2) -2016-06-13 21:51:22 INFO Executor:59 - Finished task 1.0 in stage 3.0 (TID 7). 8431928 bytes result sent to driver -2016-06-13 21:51:22 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:53573 in memory (size: 2.4 KB, free: 730.2 MB) -2016-06-13 21:51:22 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 21:51:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 3.0 (TID 7) in 203 ms on localhost (2/2) -2016-06-13 21:51:22 INFO DAGScheduler:59 - ResultStage 3 (treeAggregate at IDF.scala:56) finished in 0.205 s -2016-06-13 21:51:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 21:51:22 INFO DAGScheduler:59 - Job 3 finished: treeAggregate at IDF.scala:56, took 0.239907 s -2016-06-13 21:51:22 INFO MemoryStore:59 - ensureFreeSpace(8388664) called with curMem=8965215, maxMem=766075207 -2016-06-13 21:51:22 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 8.0 MB, free 714.0 MB) -2016-06-13 21:51:22 INFO MemoryStore:59 - ensureFreeSpace(395684) called with curMem=17353879, maxMem=766075207 -2016-06-13 21:51:22 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 386.4 KB, free 713.7 MB) -2016-06-13 21:51:22 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:53573 (size: 386.4 KB, free: 729.8 MB) -2016-06-13 21:51:22 INFO SparkContext:59 - Created broadcast 6 from broadcast at IDF.scala:181 -2016-06-13 21:51:22 INFO SparkContext:59 - Starting job: foreach at TFIDFSuite.scala:50 -2016-06-13 21:51:22 INFO DAGScheduler:59 - Got job 4 (foreach at TFIDFSuite.scala:50) with 2 output partitions -2016-06-13 21:51:22 INFO DAGScheduler:59 - Final stage: ResultStage 4(foreach at TFIDFSuite.scala:50) -2016-06-13 21:51:22 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:51:22 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:51:22 INFO DAGScheduler:59 - Submitting ResultStage 4 (MapPartitionsRDD[7] at mapPartitions at IDF.scala:182), which has no missing parents -2016-06-13 21:51:22 INFO MemoryStore:59 - ensureFreeSpace(4016) called with curMem=17749563, maxMem=766075207 -2016-06-13 21:51:22 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 3.9 KB, free 713.7 MB) -2016-06-13 21:51:22 INFO MemoryStore:59 - ensureFreeSpace(2224) called with curMem=17753579, maxMem=766075207 -2016-06-13 21:51:22 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 2.2 KB, free 713.7 MB) -2016-06-13 21:51:22 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:53573 (size: 2.2 KB, free: 729.8 MB) -2016-06-13 21:51:22 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:51:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 4 (MapPartitionsRDD[7] at mapPartitions at IDF.scala:182) -2016-06-13 21:51:22 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 2 tasks -2016-06-13 21:51:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 8, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:51:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 4.0 (TID 9, localhost, PROCESS_LOCAL, 2203 bytes) -2016-06-13 21:51:22 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 8) -2016-06-13 21:51:22 INFO Executor:59 - Running task 1.0 in stage 4.0 (TID 9) -2016-06-13 21:51:22 INFO BlockManager:59 - Found block rdd_3_0 locally -2016-06-13 21:51:22 INFO BlockManager:59 - Found block rdd_3_1 locally -2016-06-13 21:51:22 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 8). 2044 bytes result sent to driver -2016-06-13 21:51:22 INFO Executor:59 - Finished task 1.0 in stage 4.0 (TID 9). 2044 bytes result sent to driver -2016-06-13 21:51:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 8) in 16 ms on localhost (1/2) -2016-06-13 21:51:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 4.0 (TID 9) in 15 ms on localhost (2/2) -2016-06-13 21:51:22 INFO DAGScheduler:59 - ResultStage 4 (foreach at TFIDFSuite.scala:50) finished in 0.017 s -2016-06-13 21:51:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 21:51:22 INFO DAGScheduler:59 - Job 4 finished: foreach at TFIDFSuite.scala:50, took 0.035520 s -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 21:51:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 21:51:22 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 21:51:22 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 21:51:22 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 21:51:22 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 21:51:22 INFO BlockManager:59 - BlockManager stopped -2016-06-13 21:51:22 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 21:51:22 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 21:51:22 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 21:51:22 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 21:51:22 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 21:51:22 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 21:51:22 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-5649e0bc-1d33-4f6a-af58-4c650104a1a8 -2016-06-13 21:53:42 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 21:53:42 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 21:53:43 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 21:53:43 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 21:53:43 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 21:53:43 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 21:53:43 INFO Remoting:74 - Starting remoting -2016-06-13 21:53:44 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:53595] -2016-06-13 21:53:44 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53595. -2016-06-13 21:53:44 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 21:53:44 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 21:53:44 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-a92e0dff-2d68-4189-831b-110350fff2a7 -2016-06-13 21:53:44 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 21:53:44 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-935c6948-8160-462e-9bd6-05b098483e04\httpd-499cfefe-91a4-47f9-9f3c-5523f77eb8da -2016-06-13 21:53:44 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 21:53:44 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:53:44 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53596 -2016-06-13 21:53:44 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53596. -2016-06-13 21:53:44 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 21:53:44 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:53:44 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 21:53:44 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 21:53:44 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 21:53:44 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 21:53:44 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 21:53:45 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53607. -2016-06-13 21:53:45 INFO NettyBlockTransferService:59 - Server created on 53607 -2016-06-13 21:53:45 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 21:53:45 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53607 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53607) -2016-06-13 21:53:45 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 21:53:45 INFO HashingTFSuite:59 - - -===== TEST OUTPUT FOR o.a.s.mllib.FeatureExtractionAndTransformation.HashingTFSuite: 'hashing tf on a single doc' ===== - -2016-06-13 21:53:45 INFO HashingTFSuite:59 - - -===== FINISHED o.a.s.mllib.FeatureExtractionAndTransformation.HashingTFSuite: 'hashing tf on a single doc' ===== - -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 21:53:45 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 21:53:45 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 21:53:45 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 21:53:45 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 21:53:45 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 21:53:45 INFO BlockManager:59 - BlockManager stopped -2016-06-13 21:53:45 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 21:53:45 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 21:53:45 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 21:53:45 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 21:53:45 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 21:53:45 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 21:53:45 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-935c6948-8160-462e-9bd6-05b098483e04 -2016-06-13 21:54:16 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 21:54:17 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 21:54:17 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 21:54:17 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 21:54:17 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 21:54:18 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 21:54:18 INFO Remoting:74 - Starting remoting -2016-06-13 21:54:18 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:53625] -2016-06-13 21:54:18 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53625. -2016-06-13 21:54:18 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 21:54:18 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 21:54:18 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-5a8fc6d6-8bc7-4f8a-9ed6-2e6c0b309ed9 -2016-06-13 21:54:18 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 21:54:18 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-af48b74a-aa32-45cc-8ed0-63ede11a25b7\httpd-85879b08-501e-4a0e-be0a-56e0340ccf06 -2016-06-13 21:54:18 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 21:54:18 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:54:18 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53626 -2016-06-13 21:54:18 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53626. -2016-06-13 21:54:18 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 21:54:18 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:54:18 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 21:54:18 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 21:54:18 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 21:54:18 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 21:54:18 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 21:54:19 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53637. -2016-06-13 21:54:19 INFO NettyBlockTransferService:59 - Server created on 53637 -2016-06-13 21:54:19 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 21:54:19 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53637 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53637) -2016-06-13 21:54:19 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 21:54:19 INFO HashingTFSuite:59 - - -===== TEST OUTPUT FOR o.a.s.mllib.FeatureExtractionAndTransformation.HashingTFSuite: 'hashing tf on an RDD' ===== - -2016-06-13 21:54:19 INFO SparkContext:59 - Starting job: collect at HashingTFSuite.scala:50 -2016-06-13 21:54:19 INFO DAGScheduler:59 - Got job 0 (collect at HashingTFSuite.scala:50) with 2 output partitions -2016-06-13 21:54:19 INFO DAGScheduler:59 - Final stage: ResultStage 0(collect at HashingTFSuite.scala:50) -2016-06-13 21:54:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:54:19 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:54:19 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[1] at map at HashingTF.scala:78), which has no missing parents -2016-06-13 21:54:20 INFO MemoryStore:59 - ensureFreeSpace(2104) called with curMem=0, maxMem=766075207 -2016-06-13 21:54:20 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 2.1 KB, free 730.6 MB) -2016-06-13 21:54:20 INFO MemoryStore:59 - ensureFreeSpace(1265) called with curMem=2104, maxMem=766075207 -2016-06-13 21:54:20 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1265.0 B, free 730.6 MB) -2016-06-13 21:54:20 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53637 (size: 1265.0 B, free: 730.6 MB) -2016-06-13 21:54:20 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:54:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at map at HashingTF.scala:78) -2016-06-13 21:54:20 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 21:54:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2167 bytes) -2016-06-13 21:54:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2213 bytes) -2016-06-13 21:54:20 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 21:54:20 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 21:54:20 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1197 bytes result sent to driver -2016-06-13 21:54:20 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1131 bytes result sent to driver -2016-06-13 21:54:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 78 ms on localhost (1/2) -2016-06-13 21:54:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 109 ms on localhost (2/2) -2016-06-13 21:54:20 INFO DAGScheduler:59 - ResultStage 0 (collect at HashingTFSuite.scala:50) finished in 0.119 s -2016-06-13 21:54:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 21:54:20 INFO DAGScheduler:59 - Job 0 finished: collect at HashingTFSuite.scala:50, took 0.288862 s -2016-06-13 21:54:20 INFO SparkContext:59 - Starting job: foreach at HashingTFSuite.scala:53 -2016-06-13 21:54:20 INFO DAGScheduler:59 - Got job 1 (foreach at HashingTFSuite.scala:53) with 2 output partitions -2016-06-13 21:54:20 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at HashingTFSuite.scala:53) -2016-06-13 21:54:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:54:20 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:54:20 INFO DAGScheduler:59 - Submitting ResultStage 1 (ParallelCollectionRDD[0] at parallelize at HashingTFSuite.scala:49), which has no missing parents -2016-06-13 21:54:20 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=3369, maxMem=766075207 -2016-06-13 21:54:20 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1224.0 B, free 730.6 MB) -2016-06-13 21:54:20 INFO MemoryStore:59 - ensureFreeSpace(854) called with curMem=4593, maxMem=766075207 -2016-06-13 21:54:20 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 854.0 B, free 730.6 MB) -2016-06-13 21:54:20 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53637 (size: 854.0 B, free: 730.6 MB) -2016-06-13 21:54:20 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:54:20 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ParallelCollectionRDD[0] at parallelize at HashingTFSuite.scala:49) -2016-06-13 21:54:20 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 21:54:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2167 bytes) -2016-06-13 21:54:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2213 bytes) -2016-06-13 21:54:20 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 21:54:20 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 21:54:20 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 915 bytes result sent to driver -2016-06-13 21:54:20 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 915 bytes result sent to driver -2016-06-13 21:54:20 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 15 ms on localhost (1/2) -2016-06-13 21:54:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 21 ms on localhost (2/2) -2016-06-13 21:54:20 INFO DAGScheduler:59 - ResultStage 1 (foreach at HashingTFSuite.scala:53) finished in 0.021 s -2016-06-13 21:54:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 21:54:20 INFO DAGScheduler:59 - Job 1 finished: foreach at HashingTFSuite.scala:53, took 0.037919 s -2016-06-13 21:54:20 INFO HashingTFSuite:59 - - -===== FINISHED o.a.s.mllib.FeatureExtractionAndTransformation.HashingTFSuite: 'hashing tf on an RDD' ===== - -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 21:54:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 21:54:20 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 21:54:20 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 21:54:20 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 21:54:20 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 21:54:20 INFO BlockManager:59 - BlockManager stopped -2016-06-13 21:54:20 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 21:54:20 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 21:54:20 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 21:54:20 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 21:54:20 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 21:54:20 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 21:54:20 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-af48b74a-aa32-45cc-8ed0-63ede11a25b7 -2016-06-13 21:56:08 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 21:56:08 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 21:56:09 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 21:56:09 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 21:56:09 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 21:56:09 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 21:56:09 INFO Remoting:74 - Starting remoting -2016-06-13 21:56:10 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:53655] -2016-06-13 21:56:10 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53655. -2016-06-13 21:56:10 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 21:56:10 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 21:56:10 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-69d34c19-d86f-4f3a-a2d1-a71496bff2bf -2016-06-13 21:56:10 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 21:56:10 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-9cbacb50-f2ce-418c-b247-e0b78477b657\httpd-d1e428fe-30d0-49b6-83ea-3a7812883f7a -2016-06-13 21:56:10 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 21:56:10 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:56:10 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53656 -2016-06-13 21:56:10 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53656. -2016-06-13 21:56:10 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 21:56:10 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:56:10 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 21:56:10 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 21:56:10 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 21:56:10 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 21:56:10 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 21:56:11 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53667. -2016-06-13 21:56:11 INFO NettyBlockTransferService:59 - Server created on 53667 -2016-06-13 21:56:11 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 21:56:11 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53667 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53667) -2016-06-13 21:56:11 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 21:56:11 INFO HashingTFSuite:59 - - -===== TEST OUTPUT FOR o.a.s.mllib.FeatureExtractionAndTransformation.HashingTFSuite: 'hashing tf on a single doc' ===== - -2016-06-13 21:56:11 INFO HashingTFSuite:59 - - -===== FINISHED o.a.s.mllib.FeatureExtractionAndTransformation.HashingTFSuite: 'hashing tf on a single doc' ===== - -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 21:56:11 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 21:56:11 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 21:56:11 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 21:56:11 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 21:56:11 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 21:56:11 INFO BlockManager:59 - BlockManager stopped -2016-06-13 21:56:11 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 21:56:11 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 21:56:11 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 21:56:11 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 21:56:11 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 21:56:11 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 21:56:11 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-9cbacb50-f2ce-418c-b247-e0b78477b657 -2016-06-13 21:56:52 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 21:56:53 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 21:56:53 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 21:56:53 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 21:56:53 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 21:56:54 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 21:56:54 INFO Remoting:74 - Starting remoting -2016-06-13 21:56:54 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:53685] -2016-06-13 21:56:54 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53685. -2016-06-13 21:56:54 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 21:56:54 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 21:56:54 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-eefb3b6a-ffeb-4fc0-982d-89edb8768933 -2016-06-13 21:56:54 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 21:56:54 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-6f6979d2-a166-4db0-a284-2fdc2f1bfbbf\httpd-e3ae9f20-a982-4417-8aa7-2cdec0732470 -2016-06-13 21:56:54 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 21:56:54 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:56:54 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53686 -2016-06-13 21:56:54 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53686. -2016-06-13 21:56:54 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 21:56:54 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 21:56:54 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 21:56:54 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 21:56:54 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 21:56:55 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 21:56:55 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 21:56:55 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53697. -2016-06-13 21:56:55 INFO NettyBlockTransferService:59 - Server created on 53697 -2016-06-13 21:56:55 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 21:56:55 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53697 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53697) -2016-06-13 21:56:55 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 21:56:55 INFO HashingTFSuite:59 - - -===== TEST OUTPUT FOR o.a.s.mllib.FeatureExtractionAndTransformation.HashingTFSuite: 'hashing tf on an RDD' ===== - -2016-06-13 21:56:56 INFO SparkContext:59 - Starting job: collect at HashingTFSuite.scala:53 -2016-06-13 21:56:56 INFO DAGScheduler:59 - Got job 0 (collect at HashingTFSuite.scala:53) with 2 output partitions -2016-06-13 21:56:56 INFO DAGScheduler:59 - Final stage: ResultStage 0(collect at HashingTFSuite.scala:53) -2016-06-13 21:56:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:56:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:56:56 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[1] at map at HashingTF.scala:78), which has no missing parents -2016-06-13 21:56:56 INFO MemoryStore:59 - ensureFreeSpace(2104) called with curMem=0, maxMem=766075207 -2016-06-13 21:56:56 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 2.1 KB, free 730.6 MB) -2016-06-13 21:56:56 INFO MemoryStore:59 - ensureFreeSpace(1265) called with curMem=2104, maxMem=766075207 -2016-06-13 21:56:56 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1265.0 B, free 730.6 MB) -2016-06-13 21:56:56 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53697 (size: 1265.0 B, free: 730.6 MB) -2016-06-13 21:56:56 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:56:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at map at HashingTF.scala:78) -2016-06-13 21:56:56 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 21:56:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2167 bytes) -2016-06-13 21:56:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2213 bytes) -2016-06-13 21:56:56 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 21:56:56 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 21:56:56 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1197 bytes result sent to driver -2016-06-13 21:56:56 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1131 bytes result sent to driver -2016-06-13 21:56:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 47 ms on localhost (1/2) -2016-06-13 21:56:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 72 ms on localhost (2/2) -2016-06-13 21:56:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 21:56:56 INFO DAGScheduler:59 - ResultStage 0 (collect at HashingTFSuite.scala:53) finished in 0.083 s -2016-06-13 21:56:56 INFO DAGScheduler:59 - Job 0 finished: collect at HashingTFSuite.scala:53, took 0.251981 s -2016-06-13 21:56:56 INFO SparkContext:59 - Starting job: foreach at HashingTFSuite.scala:56 -2016-06-13 21:56:56 INFO DAGScheduler:59 - Got job 1 (foreach at HashingTFSuite.scala:56) with 2 output partitions -2016-06-13 21:56:56 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at HashingTFSuite.scala:56) -2016-06-13 21:56:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:56:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:56:56 INFO DAGScheduler:59 - Submitting ResultStage 1 (ParallelCollectionRDD[0] at parallelize at HashingTFSuite.scala:52), which has no missing parents -2016-06-13 21:56:56 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=3369, maxMem=766075207 -2016-06-13 21:56:56 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1224.0 B, free 730.6 MB) -2016-06-13 21:56:56 INFO MemoryStore:59 - ensureFreeSpace(854) called with curMem=4593, maxMem=766075207 -2016-06-13 21:56:56 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 854.0 B, free 730.6 MB) -2016-06-13 21:56:56 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53697 (size: 854.0 B, free: 730.6 MB) -2016-06-13 21:56:56 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:56:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ParallelCollectionRDD[0] at parallelize at HashingTFSuite.scala:52) -2016-06-13 21:56:56 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 21:56:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2167 bytes) -2016-06-13 21:56:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2213 bytes) -2016-06-13 21:56:56 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 21:56:56 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 21:56:56 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 915 bytes result sent to driver -2016-06-13 21:56:56 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 915 bytes result sent to driver -2016-06-13 21:56:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 11 ms on localhost (1/2) -2016-06-13 21:56:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 13 ms on localhost (2/2) -2016-06-13 21:56:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 21:56:56 INFO DAGScheduler:59 - ResultStage 1 (foreach at HashingTFSuite.scala:56) finished in 0.014 s -2016-06-13 21:56:56 INFO DAGScheduler:59 - Job 1 finished: foreach at HashingTFSuite.scala:56, took 0.033010 s -2016-06-13 21:56:56 INFO SparkContext:59 - Starting job: collect at HashingTFSuite.scala:58 -2016-06-13 21:56:56 INFO DAGScheduler:59 - Got job 2 (collect at HashingTFSuite.scala:58) with 2 output partitions -2016-06-13 21:56:56 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at HashingTFSuite.scala:58) -2016-06-13 21:56:56 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 21:56:56 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 21:56:56 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[2] at map at HashingTF.scala:78), which has no missing parents -2016-06-13 21:56:56 INFO MemoryStore:59 - ensureFreeSpace(2104) called with curMem=5447, maxMem=766075207 -2016-06-13 21:56:56 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 2.1 KB, free 730.6 MB) -2016-06-13 21:56:56 INFO MemoryStore:59 - ensureFreeSpace(1259) called with curMem=7551, maxMem=766075207 -2016-06-13 21:56:56 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1259.0 B, free 730.6 MB) -2016-06-13 21:56:56 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53697 (size: 1259.0 B, free: 730.6 MB) -2016-06-13 21:56:56 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 21:56:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[2] at map at HashingTF.scala:78) -2016-06-13 21:56:56 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 21:56:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2167 bytes) -2016-06-13 21:56:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2213 bytes) -2016-06-13 21:56:56 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 21:56:56 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 21:56:56 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1131 bytes result sent to driver -2016-06-13 21:56:56 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1197 bytes result sent to driver -2016-06-13 21:56:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 11 ms on localhost (1/2) -2016-06-13 21:56:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 10 ms on localhost (2/2) -2016-06-13 21:56:56 INFO DAGScheduler:59 - ResultStage 2 (collect at HashingTFSuite.scala:58) finished in 0.012 s -2016-06-13 21:56:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 21:56:56 INFO DAGScheduler:59 - Job 2 finished: collect at HashingTFSuite.scala:58, took 0.027666 s -2016-06-13 21:56:56 INFO HashingTFSuite:59 - - -===== FINISHED o.a.s.mllib.FeatureExtractionAndTransformation.HashingTFSuite: 'hashing tf on an RDD' ===== - -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 21:56:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 21:56:56 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 21:56:56 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 21:56:56 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 21:56:56 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 21:56:56 INFO BlockManager:59 - BlockManager stopped -2016-06-13 21:56:56 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 21:56:56 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 21:56:56 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 21:56:56 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 21:56:56 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 21:56:56 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 21:56:56 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-6f6979d2-a166-4db0-a284-2fdc2f1bfbbf -2016-06-13 22:02:22 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 22:02:22 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 22:02:23 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 22:02:23 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 22:02:23 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 22:02:23 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 22:02:23 INFO Remoting:74 - Starting remoting -2016-06-13 22:02:23 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:53727] -2016-06-13 22:02:24 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53727. -2016-06-13 22:02:24 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 22:02:24 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 22:02:24 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-3ec41ad1-38b4-4c54-b3c8-c25cf0031270 -2016-06-13 22:02:24 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 22:02:24 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-780926e1-e08a-482c-bb83-6e80011d06ea\httpd-f1e41f0f-0bbf-48e2-8c14-3da24aba2f91 -2016-06-13 22:02:24 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 22:02:24 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:02:24 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53728 -2016-06-13 22:02:24 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53728. -2016-06-13 22:02:24 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 22:02:24 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:02:24 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 22:02:24 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 22:02:24 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 22:02:24 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 22:02:24 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 22:02:24 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53739. -2016-06-13 22:02:24 INFO NettyBlockTransferService:59 - Server created on 53739 -2016-06-13 22:02:24 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:02:24 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53739 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53739) -2016-06-13 22:02:24 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:02:25 INFO HashingTFSuite:59 - - -===== TEST OUTPUT FOR o.a.s.mllib.FeatureExtractionAndTransformation.HashingTFSuite: 'hashing tf on an RDD' ===== - -2016-06-13 22:02:25 INFO SparkContext:59 - Starting job: collect at HashingTFSuite.scala:55 -2016-06-13 22:02:25 INFO DAGScheduler:59 - Got job 0 (collect at HashingTFSuite.scala:55) with 2 output partitions -2016-06-13 22:02:25 INFO DAGScheduler:59 - Final stage: ResultStage 0(collect at HashingTFSuite.scala:55) -2016-06-13 22:02:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 22:02:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 22:02:25 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[1] at map at HashingTF.scala:78), which has no missing parents -2016-06-13 22:02:25 INFO MemoryStore:59 - ensureFreeSpace(2104) called with curMem=0, maxMem=766075207 -2016-06-13 22:02:25 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 2.1 KB, free 730.6 MB) -2016-06-13 22:02:25 INFO MemoryStore:59 - ensureFreeSpace(1265) called with curMem=2104, maxMem=766075207 -2016-06-13 22:02:25 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 1265.0 B, free 730.6 MB) -2016-06-13 22:02:25 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53739 (size: 1265.0 B, free: 730.6 MB) -2016-06-13 22:02:25 INFO SparkContext:59 - Created broadcast 0 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:02:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at map at HashingTF.scala:78) -2016-06-13 22:02:25 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 22:02:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2168 bytes) -2016-06-13 22:02:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2213 bytes) -2016-06-13 22:02:25 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 22:02:25 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 22:02:25 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 1197 bytes result sent to driver -2016-06-13 22:02:25 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 1143 bytes result sent to driver -2016-06-13 22:02:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 95 ms on localhost (1/2) -2016-06-13 22:02:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 120 ms on localhost (2/2) -2016-06-13 22:02:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 22:02:25 INFO DAGScheduler:59 - ResultStage 0 (collect at HashingTFSuite.scala:55) finished in 0.130 s -2016-06-13 22:02:25 INFO DAGScheduler:59 - Job 0 finished: collect at HashingTFSuite.scala:55, took 0.327776 s -2016-06-13 22:02:25 INFO SparkContext:59 - Starting job: foreach at HashingTFSuite.scala:58 -2016-06-13 22:02:25 INFO DAGScheduler:59 - Got job 1 (foreach at HashingTFSuite.scala:58) with 2 output partitions -2016-06-13 22:02:25 INFO DAGScheduler:59 - Final stage: ResultStage 1(foreach at HashingTFSuite.scala:58) -2016-06-13 22:02:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 22:02:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 22:02:25 INFO DAGScheduler:59 - Submitting ResultStage 1 (ParallelCollectionRDD[0] at parallelize at HashingTFSuite.scala:54), which has no missing parents -2016-06-13 22:02:25 INFO MemoryStore:59 - ensureFreeSpace(1224) called with curMem=3369, maxMem=766075207 -2016-06-13 22:02:25 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 1224.0 B, free 730.6 MB) -2016-06-13 22:02:25 INFO MemoryStore:59 - ensureFreeSpace(854) called with curMem=4593, maxMem=766075207 -2016-06-13 22:02:25 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 854.0 B, free 730.6 MB) -2016-06-13 22:02:25 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:53739 (size: 854.0 B, free: 730.6 MB) -2016-06-13 22:02:25 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:02:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (ParallelCollectionRDD[0] at parallelize at HashingTFSuite.scala:54) -2016-06-13 22:02:25 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 22:02:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 2168 bytes) -2016-06-13 22:02:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 2213 bytes) -2016-06-13 22:02:25 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 22:02:25 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 22:02:25 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 915 bytes result sent to driver -2016-06-13 22:02:25 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 915 bytes result sent to driver -2016-06-13 22:02:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 15 ms on localhost (1/2) -2016-06-13 22:02:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 14 ms on localhost (2/2) -2016-06-13 22:02:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 22:02:25 INFO DAGScheduler:59 - ResultStage 1 (foreach at HashingTFSuite.scala:58) finished in 0.016 s -2016-06-13 22:02:25 INFO DAGScheduler:59 - Job 1 finished: foreach at HashingTFSuite.scala:58, took 0.033838 s -2016-06-13 22:02:25 INFO SparkContext:59 - Starting job: collect at HashingTFSuite.scala:60 -2016-06-13 22:02:25 INFO DAGScheduler:59 - Got job 2 (collect at HashingTFSuite.scala:60) with 2 output partitions -2016-06-13 22:02:25 INFO DAGScheduler:59 - Final stage: ResultStage 2(collect at HashingTFSuite.scala:60) -2016-06-13 22:02:25 INFO DAGScheduler:59 - Parents of final stage: List() -2016-06-13 22:02:25 INFO DAGScheduler:59 - Missing parents: List() -2016-06-13 22:02:25 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[2] at map at HashingTF.scala:78), which has no missing parents -2016-06-13 22:02:25 INFO MemoryStore:59 - ensureFreeSpace(2104) called with curMem=5447, maxMem=766075207 -2016-06-13 22:02:25 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 2.1 KB, free 730.6 MB) -2016-06-13 22:02:25 INFO MemoryStore:59 - ensureFreeSpace(1259) called with curMem=7551, maxMem=766075207 -2016-06-13 22:02:25 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1259.0 B, free 730.6 MB) -2016-06-13 22:02:25 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:53739 (size: 1259.0 B, free: 730.6 MB) -2016-06-13 22:02:25 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:02:25 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 2 (MapPartitionsRDD[2] at map at HashingTF.scala:78) -2016-06-13 22:02:25 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 22:02:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2168 bytes) -2016-06-13 22:02:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2213 bytes) -2016-06-13 22:02:25 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 22:02:25 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 22:02:25 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 1143 bytes result sent to driver -2016-06-13 22:02:25 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 1197 bytes result sent to driver -2016-06-13 22:02:25 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 15 ms on localhost (1/2) -2016-06-13 22:02:25 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 15 ms on localhost (2/2) -2016-06-13 22:02:25 INFO DAGScheduler:59 - ResultStage 2 (collect at HashingTFSuite.scala:60) finished in 0.017 s -2016-06-13 22:02:25 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 22:02:25 INFO DAGScheduler:59 - Job 2 finished: collect at HashingTFSuite.scala:60, took 0.031925 s -2016-06-13 22:02:25 INFO HashingTFSuite:59 - - -===== FINISHED o.a.s.mllib.FeatureExtractionAndTransformation.HashingTFSuite: 'hashing tf on an RDD' ===== - -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 22:02:26 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 22:02:26 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 22:02:26 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 22:02:26 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 22:02:26 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 22:02:26 INFO BlockManager:59 - BlockManager stopped -2016-06-13 22:02:26 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 22:02:26 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 22:02:26 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 22:02:26 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 22:02:26 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 22:02:26 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-780926e1-e08a-482c-bb83-6e80011d06ea -2016-06-13 22:02:26 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 22:12:42 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 22:12:43 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 22:12:43 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 22:12:43 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 22:12:43 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 22:12:44 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 22:12:44 INFO Remoting:74 - Starting remoting -2016-06-13 22:12:44 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:53770] -2016-06-13 22:12:44 INFO Utils:59 - Successfully started service 'sparkDriver' on port 53770. -2016-06-13 22:12:44 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 22:12:44 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 22:12:44 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-015f0e97-0073-4b19-ba96-d77aa520601e -2016-06-13 22:12:44 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 22:12:45 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-2e17a802-c49d-47bd-9674-a2b78fd6d700\httpd-135ef6f0-2b2c-4fbf-b5a4-b881c6c7f98c -2016-06-13 22:12:45 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 22:12:45 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:12:45 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:53771 -2016-06-13 22:12:45 INFO Utils:59 - Successfully started service 'HTTP file server' on port 53771. -2016-06-13 22:12:45 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 22:12:45 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:12:45 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 22:12:45 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 22:12:45 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 22:12:45 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 22:12:45 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 22:12:45 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 53790. -2016-06-13 22:12:45 INFO NettyBlockTransferService:59 - Server created on 53790 -2016-06-13 22:12:45 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:12:45 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:53790 with 730.6 MB RAM, BlockManagerId(driver, localhost, 53790) -2016-06-13 22:12:45 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:12:47 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 22:12:47 INFO MemoryStore:59 - ensureFreeSpace(231424) called with curMem=0, maxMem=766075207 -2016-06-13 22:12:47 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 226.0 KB, free 730.4 MB) -2016-06-13 22:12:47 INFO MemoryStore:59 - ensureFreeSpace(19947) called with curMem=231424, maxMem=766075207 -2016-06-13 22:12:47 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 19.5 KB, free 730.3 MB) -2016-06-13 22:12:47 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:53790 (size: 19.5 KB, free: 730.6 MB) -2016-06-13 22:12:47 INFO SparkContext:59 - Created broadcast 0 from wholeTextFiles at TFIDFSuite.scala:57 -2016-06-13 22:22:33 INFO FileInputFormat:281 - Total input paths to process : 11314 -2016-06-13 22:38:20 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 22:38:20 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 22:38:20 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 22:38:20 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 22:38:20 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 22:38:21 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 22:38:21 INFO Remoting:74 - Starting remoting -2016-06-13 22:38:21 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:54290] -2016-06-13 22:38:21 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54290. -2016-06-13 22:38:21 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 22:38:21 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 22:38:21 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-58547caf-2bcc-42c8-8a25-f514d5502b31 -2016-06-13 22:38:21 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 22:38:21 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-c765bf0d-048a-4179-b864-ce681c90a95e\httpd-6f8e1a5d-8dd7-4848-a919-ee880d586d3d -2016-06-13 22:38:21 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 22:38:22 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:38:22 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54291 -2016-06-13 22:38:22 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54291. -2016-06-13 22:38:22 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 22:38:22 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:38:22 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 22:38:22 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 22:38:22 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 22:38:22 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 22:38:22 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 22:38:22 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54310. -2016-06-13 22:38:22 INFO NettyBlockTransferService:59 - Server created on 54310 -2016-06-13 22:38:22 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:38:22 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54310 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54310) -2016-06-13 22:38:22 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:38:23 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 22:38:23 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 22:38:23 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 22:38:23 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 22:38:23 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54310 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:38:23 INFO SparkContext:59 - Created broadcast 0 from textFile at word2VecSuite.scala:23 -2016-06-13 22:38:24 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 22:38:25 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 22:38:25 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:164 -2016-06-13 22:38:25 INFO DAGScheduler:59 - Registering RDD 4 (map at Word2Vec.scala:155) -2016-06-13 22:38:25 INFO DAGScheduler:59 - Got job 0 (collect at Word2Vec.scala:164) with 3 output partitions -2016-06-13 22:38:25 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at Word2Vec.scala:164) -2016-06-13 22:38:25 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 22:38:25 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 22:38:25 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155), which has no missing parents -2016-06-13 22:38:25 INFO MemoryStore:59 - ensureFreeSpace(4376) called with curMem=171580, maxMem=766075207 -2016-06-13 22:38:25 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 22:38:25 INFO MemoryStore:59 - ensureFreeSpace(2411) called with curMem=175956, maxMem=766075207 -2016-06-13 22:38:25 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 22:38:25 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54310 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:38:25 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:38:25 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155) -2016-06-13 22:38:25 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-06-13 22:38:25 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:38:25 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:38:25 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:38:25 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 22:38:25 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 22:38:25 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-06-13 22:38:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/text8:33554432+33554432 -2016-06-13 22:38:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/text8:67108864+32891136 -2016-06-13 22:38:25 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/text8:0+33554432 -2016-06-13 22:38:25 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 22:38:25 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 22:38:25 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 22:38:25 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 22:38:25 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 22:38:25 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 2255 bytes result sent to driver -2016-06-13 22:38:26 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 401 ms on localhost (1/3) -2016-06-13 22:38:26 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2255 bytes result sent to driver -2016-06-13 22:38:26 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 567 ms on localhost (2/3) -2016-06-13 22:38:58 ERROR Executor:96 - Exception in task 0.0 in stage 0.0 (TID 0) -java.lang.OutOfMemoryError: Java heap space - at java.util.Arrays.copyOf(Unknown Source) - at java.util.Arrays.copyOf(Unknown Source) - at java.util.ArrayList.grow(Unknown Source) - at java.util.ArrayList.ensureCapacityInternal(Unknown Source) - at java.util.ArrayList.add(Unknown Source) - at java.lang.String.split(Unknown Source) - at java.lang.String.split(Unknown Source) - at org.apache.spark.mllib.FeatureExtractionAndTransformation.word2VecSuite$$anonfun$1$$anonfun$2.apply(word2VecSuite.scala:23) - at org.apache.spark.mllib.FeatureExtractionAndTransformation.word2VecSuite$$anonfun$1$$anonfun$2.apply(word2VecSuite.scala:23) - at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) - at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:371) - at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) - at org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:209) - at org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:73) - at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73) - at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-06-13 22:39:02 ERROR SparkUncaughtExceptionHandler:96 - Uncaught exception in thread Thread[Executor task launch worker-0,5,main] -java.lang.OutOfMemoryError: Java heap space - at java.util.Arrays.copyOf(Unknown Source) - at java.util.Arrays.copyOf(Unknown Source) - at java.util.ArrayList.grow(Unknown Source) - at java.util.ArrayList.ensureCapacityInternal(Unknown Source) - at java.util.ArrayList.add(Unknown Source) - at java.lang.String.split(Unknown Source) - at java.lang.String.split(Unknown Source) - at org.apache.spark.mllib.FeatureExtractionAndTransformation.word2VecSuite$$anonfun$1$$anonfun$2.apply(word2VecSuite.scala:23) - at org.apache.spark.mllib.FeatureExtractionAndTransformation.word2VecSuite$$anonfun$1$$anonfun$2.apply(word2VecSuite.scala:23) - at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) - at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:371) - at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) - at org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:209) - at org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:73) - at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73) - at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-06-13 22:39:04 WARN TaskSetManager:71 - Lost task 0.0 in stage 0.0 (TID 0, localhost): java.lang.OutOfMemoryError: Java heap space - at java.util.Arrays.copyOf(Unknown Source) - at java.util.Arrays.copyOf(Unknown Source) - at java.util.ArrayList.grow(Unknown Source) - at java.util.ArrayList.ensureCapacityInternal(Unknown Source) - at java.util.ArrayList.add(Unknown Source) - at java.lang.String.split(Unknown Source) - at java.lang.String.split(Unknown Source) - at org.apache.spark.mllib.FeatureExtractionAndTransformation.word2VecSuite$$anonfun$1$$anonfun$2.apply(word2VecSuite.scala:23) - at org.apache.spark.mllib.FeatureExtractionAndTransformation.word2VecSuite$$anonfun$1$$anonfun$2.apply(word2VecSuite.scala:23) - at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) - at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:371) - at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) - at org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:209) - at org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:73) - at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73) - at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) - -2016-06-13 22:39:06 ERROR TaskSetManager:75 - Task 0 in stage 0.0 failed 1 times; aborting job -2016-06-13 22:39:06 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-06-13 22:39:08 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 22:39:09 INFO TaskSchedulerImpl:59 - Cancelling stage 0 -2016-06-13 22:39:09 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at Word2Vec.scala:155) failed in 44.082 s -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 22:39:10 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 22:39:10 INFO DAGScheduler:59 - Job 0 failed: collect at Word2Vec.scala:164, took 45.274950 s -2016-06-13 22:39:11 WARN QueuedThreadPool:145 - 5 threads could not be stopped -2016-06-13 22:39:11 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 22:39:12 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 22:39:13 INFO SparkContext:59 - SparkContext already stopped. -2016-06-13 22:39:13 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 22:39:14 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 22:39:14 INFO BlockManager:59 - BlockManager stopped -2016-06-13 22:39:14 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 22:39:14 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 22:39:14 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 22:39:14 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 22:39:14 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-c765bf0d-048a-4179-b864-ce681c90a95e -2016-06-13 22:41:58 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 22:42:00 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 22:42:01 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 22:42:01 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 22:42:01 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 22:42:03 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 22:42:04 INFO Remoting:74 - Starting remoting -2016-06-13 22:42:04 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:54366] -2016-06-13 22:42:04 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54366. -2016-06-13 22:42:05 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 22:42:06 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 22:42:07 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-1d86bbbb-7f28-4e45-a56e-a968c6ff9545 -2016-06-13 22:42:08 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 22:42:08 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-44b5b2cf-0169-4474-85b3-6ae8a243bb9d\httpd-2830bcaf-a6a2-4f60-a346-cf4b52aa6fc2 -2016-06-13 22:42:09 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 22:42:12 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:42:12 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54370 -2016-06-13 22:42:12 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54370. -2016-06-13 22:42:12 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 22:42:12 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:42:12 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 22:42:12 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 22:42:12 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 22:42:13 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 22:42:13 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 22:42:14 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54389. -2016-06-13 22:42:14 INFO NettyBlockTransferService:59 - Server created on 54389 -2016-06-13 22:42:14 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:42:14 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54389 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54389) -2016-06-13 22:42:14 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:42:17 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 22:42:17 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 22:42:17 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 22:42:17 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 22:42:17 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54389 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:42:17 INFO SparkContext:59 - Created broadcast 0 from textFile at word2VecSuite.scala:22 -2016-06-13 22:42:19 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 22:42:19 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 22:42:19 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:164 -2016-06-13 22:42:19 INFO DAGScheduler:59 - Registering RDD 4 (map at Word2Vec.scala:155) -2016-06-13 22:42:19 INFO DAGScheduler:59 - Got job 0 (collect at Word2Vec.scala:164) with 3 output partitions -2016-06-13 22:42:19 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at Word2Vec.scala:164) -2016-06-13 22:42:19 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 22:42:19 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 22:42:19 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155), which has no missing parents -2016-06-13 22:42:20 INFO MemoryStore:59 - ensureFreeSpace(4376) called with curMem=171580, maxMem=766075207 -2016-06-13 22:42:20 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 22:42:20 INFO MemoryStore:59 - ensureFreeSpace(2411) called with curMem=175956, maxMem=766075207 -2016-06-13 22:42:20 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 22:42:20 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54389 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:42:20 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:42:20 INFO DAGScheduler:59 - Submitting 3 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155) -2016-06-13 22:42:20 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 3 tasks -2016-06-13 22:42:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:42:20 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:42:20 INFO TaskSetManager:59 - Starting task 2.0 in stage 0.0 (TID 2, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:42:20 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 22:42:20 INFO Executor:59 - Running task 2.0 in stage 0.0 (TID 2) -2016-06-13 22:42:20 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 22:42:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/text8:67108864+32891136 -2016-06-13 22:42:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/text8:0+33554432 -2016-06-13 22:42:20 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/text8:33554432+33554432 -2016-06-13 22:42:20 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 22:42:20 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 22:42:20 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 22:42:20 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 22:42:20 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 22:42:23 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2255 bytes result sent to driver -2016-06-13 22:42:23 INFO Executor:59 - Finished task 2.0 in stage 0.0 (TID 2). 2255 bytes result sent to driver -2016-06-13 22:42:23 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 3459 ms on localhost (1/3) -2016-06-13 22:42:23 INFO TaskSetManager:59 - Finished task 2.0 in stage 0.0 (TID 2) in 3466 ms on localhost (2/3) -2016-06-13 22:46:43 ERROR Executor:96 - Exception in task 0.0 in stage 0.0 (TID 0) -java.lang.OutOfMemoryError: Java heap space - at java.util.Arrays.copyOfRange(Unknown Source) - at java.lang.String.(Unknown Source) - at java.lang.String.substring(Unknown Source) - at java.lang.String.split(Unknown Source) - at java.lang.String.split(Unknown Source) - at org.apache.spark.mllib.FeatureExtractionAndTransformation.word2VecSuite$$anonfun$1$$anonfun$2.apply(word2VecSuite.scala:22) - at org.apache.spark.mllib.FeatureExtractionAndTransformation.word2VecSuite$$anonfun$1$$anonfun$2.apply(word2VecSuite.scala:22) - at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) - at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:371) - at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) - at org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:209) - at org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:73) - at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73) - at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-06-13 22:46:43 WARN HeartbeatReceiver:71 - Removing executor driver with no recent heartbeats: 239345 ms exceeds timeout 120000 ms -2016-06-13 22:46:43 ERROR TaskSchedulerImpl:75 - Lost executor driver on localhost: Executor heartbeat timed out after 239345 ms -2016-06-13 22:46:43 INFO TaskSetManager:59 - Re-queueing tasks for driver from TaskSet 0.0 -2016-06-13 22:46:43 ERROR SparkUncaughtExceptionHandler:96 - Uncaught exception in thread Thread[Executor task launch worker-0,5,main] -java.lang.OutOfMemoryError: Java heap space - at java.util.Arrays.copyOfRange(Unknown Source) - at java.lang.String.(Unknown Source) - at java.lang.String.substring(Unknown Source) - at java.lang.String.split(Unknown Source) - at java.lang.String.split(Unknown Source) - at org.apache.spark.mllib.FeatureExtractionAndTransformation.word2VecSuite$$anonfun$1$$anonfun$2.apply(word2VecSuite.scala:22) - at org.apache.spark.mllib.FeatureExtractionAndTransformation.word2VecSuite$$anonfun$1$$anonfun$2.apply(word2VecSuite.scala:22) - at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) - at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:371) - at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) - at org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:209) - at org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:73) - at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73) - at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-06-13 22:46:43 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-06-13 22:46:43 INFO DAGScheduler:59 - Resubmitted ShuffleMapTask(0, 2), so marking it as still running -2016-06-13 22:46:43 INFO DAGScheduler:59 - Resubmitted ShuffleMapTask(0, 1), so marking it as still running -2016-06-13 22:46:43 WARN TaskSetManager:71 - Lost task 0.0 in stage 0.0 (TID 0, localhost): ExecutorLostFailure (executor driver lost) -2016-06-13 22:46:43 ERROR TaskSetManager:75 - Task 0 in stage 0.0 failed 1 times; aborting job -2016-06-13 22:46:43 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 22:46:43 WARN SparkContext:71 - Killing executors is only supported in coarse-grained mode -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 22:46:43 INFO Executor:59 - Told to re-register on heartbeat -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 22:46:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 22:46:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 22:46:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 22:46:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 22:46:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 22:46:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 22:46:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 22:46:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 22:46:44 INFO BlockManager:59 - BlockManager re-registering with master -2016-06-13 22:46:44 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:46:44 INFO TaskSchedulerImpl:59 - Cancelling stage 0 -2016-06-13 22:46:44 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:46:44 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at Word2Vec.scala:155) failed in 263.948 s -2016-06-13 22:46:44 INFO BlockManager:59 - Reporting 4 blocks to the master. -2016-06-13 22:46:44 INFO DAGScheduler:59 - Job 0 failed: collect at Word2Vec.scala:164, took 264.396759 s -2016-06-13 22:46:44 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54389 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:46:44 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54389 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:46:44 INFO DAGScheduler:59 - Executor lost: driver (epoch 0) -2016-06-13 22:46:44 INFO Executor:59 - Told to re-register on heartbeat -2016-06-13 22:46:44 INFO BlockManager:59 - BlockManager re-registering with master -2016-06-13 22:46:44 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:46:44 INFO BlockManagerMasterEndpoint:59 - Trying to remove executor driver from BlockManagerMaster. -2016-06-13 22:46:44 INFO BlockManagerMasterEndpoint:59 - Removing block manager BlockManagerId(driver, localhost, 54389) -2016-06-13 22:46:44 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54389 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54389) -2016-06-13 22:46:44 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:46:44 INFO BlockManager:59 - Reporting 4 blocks to the master. -2016-06-13 22:46:44 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54389 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:46:44 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54389 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:46:44 INFO Executor:59 - Told to re-register on heartbeat -2016-06-13 22:46:44 INFO BlockManager:59 - BlockManager re-registering with master -2016-06-13 22:46:44 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:46:44 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:46:44 INFO BlockManager:59 - Reporting 4 blocks to the master. -2016-06-13 22:46:44 INFO BlockManagerMaster:59 - Removed driver successfully in removeExecutor -2016-06-13 22:46:44 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 22:46:44 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54389 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:46:44 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54389 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:46:44 INFO Executor:59 - Told to re-register on heartbeat -2016-06-13 22:46:44 INFO BlockManager:59 - BlockManager re-registering with master -2016-06-13 22:46:44 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:46:44 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:46:44 INFO BlockManager:59 - Reporting 4 blocks to the master. -2016-06-13 22:46:44 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54389 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:46:44 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54389 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:46:44 INFO Executor:59 - Told to re-register on heartbeat -2016-06-13 22:46:44 INFO BlockManager:59 - BlockManager re-registering with master -2016-06-13 22:46:44 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:46:44 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:46:44 INFO BlockManager:59 - Reporting 4 blocks to the master. -2016-06-13 22:46:44 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54389 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:46:44 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54389 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:46:44 INFO Executor:59 - Told to re-register on heartbeat -2016-06-13 22:46:44 INFO BlockManager:59 - BlockManager re-registering with master -2016-06-13 22:46:44 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:46:44 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:46:44 INFO BlockManager:59 - Reporting 4 blocks to the master. -2016-06-13 22:46:44 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54389 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:46:44 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54389 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:46:44 INFO Executor:59 - Told to re-register on heartbeat -2016-06-13 22:46:44 INFO BlockManager:59 - BlockManager re-registering with master -2016-06-13 22:46:44 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:46:44 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:46:44 INFO BlockManager:59 - Reporting 4 blocks to the master. -2016-06-13 22:46:44 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54389 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:46:44 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54389 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:46:44 INFO SparkContext:59 - SparkContext already stopped. -2016-06-13 22:46:44 INFO DAGScheduler:59 - Host added was in lost list earlier: localhost -2016-06-13 22:46:44 INFO Executor:59 - Told to re-register on heartbeat -2016-06-13 22:46:44 INFO BlockManager:59 - BlockManager re-registering with master -2016-06-13 22:46:44 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:46:44 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 22:46:44 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:46:44 INFO BlockManager:59 - Reporting 4 blocks to the master. -2016-06-13 22:46:44 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54389 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:46:44 ERROR LiveListenerBus:75 - SparkListenerBus has already stopped! Dropping event SparkListenerBlockUpdated(BlockUpdatedInfo(BlockManagerId(driver, localhost, 54389),broadcast_0_piece0,StorageLevel(false, true, false, false, 1),14276,0,0)) -2016-06-13 22:46:44 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54389 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:46:44 ERROR LiveListenerBus:75 - SparkListenerBus has already stopped! Dropping event SparkListenerBlockUpdated(BlockUpdatedInfo(BlockManagerId(driver, localhost, 54389),broadcast_1_piece0,StorageLevel(false, true, false, false, 1),2411,0,0)) -2016-06-13 22:46:44 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 22:46:44 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 22:46:44 INFO BlockManager:59 - BlockManager stopped -2016-06-13 22:46:44 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 22:46:44 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 22:46:44 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 22:46:44 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 22:46:44 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-44b5b2cf-0169-4474-85b3-6ae8a243bb9d -2016-06-13 22:47:30 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 22:47:30 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 22:47:30 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 22:47:30 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 22:47:30 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 22:47:31 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 22:47:31 INFO Remoting:74 - Starting remoting -2016-06-13 22:47:31 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:54540] -2016-06-13 22:47:31 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54540. -2016-06-13 22:47:31 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 22:47:32 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 22:47:32 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-5a85e238-6a69-41fb-95af-07164394c0ca -2016-06-13 22:47:32 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 22:47:32 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-5eac4cb7-652c-432a-b703-98ab9fbd2ed6\httpd-eba16521-ffd9-4e85-8275-cc5464c5989e -2016-06-13 22:47:32 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 22:47:32 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:47:32 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54541 -2016-06-13 22:47:32 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54541. -2016-06-13 22:47:32 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 22:47:32 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:47:32 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 22:47:32 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 22:47:32 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 22:47:32 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 22:47:32 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 22:47:32 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54560. -2016-06-13 22:47:32 INFO NettyBlockTransferService:59 - Server created on 54560 -2016-06-13 22:47:32 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:47:32 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54560 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54560) -2016-06-13 22:47:32 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:47:34 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 22:47:34 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 22:47:34 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 22:47:34 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 22:47:34 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54560 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:47:34 INFO SparkContext:59 - Created broadcast 0 from textFile at word2VecSuite.scala:26 -2016-06-13 22:47:35 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 22:47:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 22:47:35 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 22:47:35 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 22:47:35 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 22:47:35 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 22:47:35 INFO BlockManager:59 - BlockManager stopped -2016-06-13 22:47:35 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 22:47:35 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 22:47:35 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 22:47:35 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 22:47:35 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 22:47:35 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 22:47:35 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-5eac4cb7-652c-432a-b703-98ab9fbd2ed6 -2016-06-13 22:47:56 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 22:47:56 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 22:47:57 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 22:47:57 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 22:47:57 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 22:47:58 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 22:47:58 INFO Remoting:74 - Starting remoting -2016-06-13 22:47:58 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:54581] -2016-06-13 22:47:58 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54581. -2016-06-13 22:47:58 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 22:47:58 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 22:47:58 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-f6288456-7436-42ae-adec-badf7118f2bf -2016-06-13 22:47:58 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 22:47:58 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-8bbcb4a1-f16d-4175-9f2d-83664d44ea91\httpd-fbe3e9f9-3753-4f27-92a6-554893ec57eb -2016-06-13 22:47:58 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 22:47:58 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:47:58 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54582 -2016-06-13 22:47:58 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54582. -2016-06-13 22:47:58 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 22:47:58 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:47:58 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 22:47:58 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 22:47:58 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 22:47:58 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 22:47:58 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 22:47:59 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54601. -2016-06-13 22:47:59 INFO NettyBlockTransferService:59 - Server created on 54601 -2016-06-13 22:47:59 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:47:59 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54601 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54601) -2016-06-13 22:47:59 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:48:00 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 22:48:00 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 22:48:00 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 22:48:00 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 22:48:00 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54601 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:48:00 INFO SparkContext:59 - Created broadcast 0 from textFile at word2VecSuite.scala:26 -2016-06-13 22:48:01 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 22:48:01 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 22:48:01 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:164 -2016-06-13 22:48:01 INFO DAGScheduler:59 - Registering RDD 4 (map at Word2Vec.scala:155) -2016-06-13 22:48:01 INFO DAGScheduler:59 - Got job 0 (collect at Word2Vec.scala:164) with 2 output partitions -2016-06-13 22:48:01 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at Word2Vec.scala:164) -2016-06-13 22:48:01 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 22:48:01 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 22:48:01 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155), which has no missing parents -2016-06-13 22:48:01 INFO MemoryStore:59 - ensureFreeSpace(4376) called with curMem=171580, maxMem=766075207 -2016-06-13 22:48:01 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 22:48:01 INFO MemoryStore:59 - ensureFreeSpace(2414) called with curMem=175956, maxMem=766075207 -2016-06-13 22:48:01 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 22:48:01 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54601 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:48:01 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:48:01 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155) -2016-06-13 22:48:01 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 22:48:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:48:01 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:48:01 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 22:48:01 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 22:48:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:48:01 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:48:01 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 22:48:01 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 22:48:01 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 22:48:01 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 22:48:01 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 22:48:02 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver -2016-06-13 22:48:02 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver -2016-06-13 22:48:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 147 ms on localhost (1/2) -2016-06-13 22:48:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 168 ms on localhost (2/2) -2016-06-13 22:48:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 22:48:02 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at Word2Vec.scala:155) finished in 0.188 s -2016-06-13 22:48:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:48:02 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:48:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 22:48:02 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:48:02 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 22:48:02 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163), which is now runnable -2016-06-13 22:48:02 INFO MemoryStore:59 - ensureFreeSpace(4144) called with curMem=178370, maxMem=766075207 -2016-06-13 22:48:02 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:48:02 INFO MemoryStore:59 - ensureFreeSpace(2163) called with curMem=182514, maxMem=766075207 -2016-06-13 22:48:02 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 22:48:02 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54601 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 22:48:02 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:48:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163) -2016-06-13 22:48:02 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 22:48:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:48:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:48:02 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 22:48:02 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 22:48:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:48:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:48:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 16 ms -2016-06-13 22:48:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 16 ms -2016-06-13 22:48:02 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1189 bytes result sent to driver -2016-06-13 22:48:02 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1686 bytes result sent to driver -2016-06-13 22:48:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 101 ms on localhost (1/2) -2016-06-13 22:48:02 INFO DAGScheduler:59 - ResultStage 1 (collect at Word2Vec.scala:164) finished in 0.105 s -2016-06-13 22:48:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 105 ms on localhost (2/2) -2016-06-13 22:48:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 22:48:02 INFO DAGScheduler:59 - Job 0 finished: collect at Word2Vec.scala:164, took 0.453194 s -2016-06-13 22:48:02 INFO Word2Vec:59 - trainWordsCount = 6 -2016-06-13 22:48:02 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=184677, maxMem=766075207 -2016-06-13 22:48:02 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-06-13 22:48:02 INFO MemoryStore:59 - ensureFreeSpace(4052) called with curMem=188717, maxMem=766075207 -2016-06-13 22:48:02 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:48:02 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54601 (size: 4.0 KB, free: 730.6 MB) -2016-06-13 22:48:02 INFO SparkContext:59 - Created broadcast 3 from broadcast at Word2Vec.scala:284 -2016-06-13 22:48:02 INFO MemoryStore:59 - ensureFreeSpace(488) called with curMem=192769, maxMem=766075207 -2016-06-13 22:48:02 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 488.0 B, free 730.4 MB) -2016-06-13 22:48:02 INFO MemoryStore:59 - ensureFreeSpace(246) called with curMem=193257, maxMem=766075207 -2016-06-13 22:48:02 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 246.0 B, free 730.4 MB) -2016-06-13 22:48:02 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54601 (size: 246.0 B, free: 730.6 MB) -2016-06-13 22:48:02 INFO SparkContext:59 - Created broadcast 4 from broadcast at Word2Vec.scala:285 -2016-06-13 22:48:02 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=193503, maxMem=766075207 -2016-06-13 22:48:02 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 22:48:02 INFO MemoryStore:59 - ensureFreeSpace(172) called with curMem=193743, maxMem=766075207 -2016-06-13 22:48:02 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 172.0 B, free 730.4 MB) -2016-06-13 22:48:02 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:54601 (size: 172.0 B, free: 730.6 MB) -2016-06-13 22:48:02 INFO SparkContext:59 - Created broadcast 5 from broadcast at Word2Vec.scala:286 -2016-06-13 22:48:02 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:400 -2016-06-13 22:48:02 INFO DAGScheduler:59 - Registering RDD 9 (repartition at Word2Vec.scala:309) -2016-06-13 22:48:02 INFO DAGScheduler:59 - Registering RDD 13 (mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:48:02 INFO DAGScheduler:59 - Got job 1 (collect at Word2Vec.scala:400) with 1 output partitions -2016-06-13 22:48:02 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at Word2Vec.scala:400) -2016-06-13 22:48:02 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3) -2016-06-13 22:48:02 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 3) -2016-06-13 22:48:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309), which has no missing parents -2016-06-13 22:48:02 INFO MemoryStore:59 - ensureFreeSpace(6280) called with curMem=193915, maxMem=766075207 -2016-06-13 22:48:02 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 6.1 KB, free 730.4 MB) -2016-06-13 22:48:02 INFO MemoryStore:59 - ensureFreeSpace(3096) called with curMem=200195, maxMem=766075207 -2016-06-13 22:48:02 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 22:48:02 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:54601 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 22:48:02 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:48:02 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309) -2016-06-13 22:48:02 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 22:48:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:48:02 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:48:02 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 22:48:02 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 22:48:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:48:02 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:48:02 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2253 bytes result sent to driver -2016-06-13 22:48:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 87 ms on localhost (1/2) -2016-06-13 22:48:02 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2253 bytes result sent to driver -2016-06-13 22:48:02 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 104 ms on localhost (2/2) -2016-06-13 22:48:02 INFO DAGScheduler:59 - ShuffleMapStage 2 (repartition at Word2Vec.scala:309) finished in 0.106 s -2016-06-13 22:48:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:48:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 22:48:02 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:48:02 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 3, ResultStage 4) -2016-06-13 22:48:02 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:48:02 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List() -2016-06-13 22:48:02 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List(ShuffleMapStage 3) -2016-06-13 22:48:02 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323), which is now runnable -2016-06-13 22:48:02 INFO MemoryStore:59 - ensureFreeSpace(6576) called with curMem=203291, maxMem=766075207 -2016-06-13 22:48:02 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 6.4 KB, free 730.4 MB) -2016-06-13 22:48:02 INFO MemoryStore:59 - ensureFreeSpace(3353) called with curMem=209867, maxMem=766075207 -2016-06-13 22:48:02 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 22:48:02 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:54601 (size: 3.3 KB, free: 730.6 MB) -2016-06-13 22:48:02 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:48:02 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:48:02 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-06-13 22:48:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2159 bytes) -2016-06-13 22:48:02 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-06-13 22:48:02 INFO CacheManager:59 - Partition rdd_12_0 not found, computing it -2016-06-13 22:48:02 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:48:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 22:48:02 INFO MemoryStore:59 - ensureFreeSpace(80) called with curMem=213220, maxMem=766075207 -2016-06-13 22:48:02 INFO MemoryStore:59 - Block rdd_12_0 stored as values in memory (estimated size 80.0 B, free 730.4 MB) -2016-06-13 22:48:02 INFO BlockManagerInfo:59 - Added rdd_12_0 in memory on localhost:54601 (size: 80.0 B, free: 730.6 MB) -2016-06-13 22:48:02 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 22:48:02 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 22:48:02 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 1954 bytes result sent to driver -2016-06-13 22:48:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 95 ms on localhost (1/1) -2016-06-13 22:48:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 22:48:02 INFO DAGScheduler:59 - ShuffleMapStage 3 (mapPartitionsWithIndex at Word2Vec.scala:323) finished in 0.095 s -2016-06-13 22:48:02 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:48:02 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:48:02 INFO DAGScheduler:59 - waiting: Set(ResultStage 4) -2016-06-13 22:48:02 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:48:02 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List() -2016-06-13 22:48:02 INFO DAGScheduler:59 - Submitting ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397), which is now runnable -2016-06-13 22:48:02 INFO MemoryStore:59 - ensureFreeSpace(4296) called with curMem=213300, maxMem=766075207 -2016-06-13 22:48:02 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 22:48:02 INFO MemoryStore:59 - ensureFreeSpace(2206) called with curMem=217596, maxMem=766075207 -2016-06-13 22:48:02 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 22:48:02 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:54601 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 22:48:02 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:48:02 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397) -2016-06-13 22:48:02 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-06-13 22:48:02 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:48:02 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-06-13 22:48:02 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 22:48:02 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 22:48:02 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 1729 bytes result sent to driver -2016-06-13 22:48:02 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 22 ms on localhost (1/1) -2016-06-13 22:48:02 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 22:48:02 INFO DAGScheduler:59 - ResultStage 4 (collect at Word2Vec.scala:400) finished in 0.023 s -2016-06-13 22:48:02 INFO DAGScheduler:59 - Job 1 finished: collect at Word2Vec.scala:400, took 0.335980 s -2016-06-13 22:48:02 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-06-13 22:48:02 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 22:48:02 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 22:48:03 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 22:48:03 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 22:48:03 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 22:48:03 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 22:48:03 INFO BlockManager:59 - BlockManager stopped -2016-06-13 22:48:03 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 22:48:03 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 22:48:03 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 22:48:03 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 22:48:03 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 22:48:03 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 22:48:03 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-8bbcb4a1-f16d-4175-9f2d-83664d44ea91 -2016-06-13 22:48:28 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 22:48:28 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 22:48:28 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 22:48:28 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 22:48:28 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 22:48:29 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 22:48:29 INFO Remoting:74 - Starting remoting -2016-06-13 22:48:29 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:54621] -2016-06-13 22:48:29 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54621. -2016-06-13 22:48:29 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 22:48:29 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 22:48:29 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-137dc4e5-0bad-4ab0-a550-d7f189981bc3 -2016-06-13 22:48:29 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 22:48:29 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-b9291e9e-2d3e-4111-8f4f-122beacfc734\httpd-066c68bf-ceac-4ca6-add5-9378af608d14 -2016-06-13 22:48:29 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 22:48:29 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:48:29 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54622 -2016-06-13 22:48:29 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54622. -2016-06-13 22:48:29 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 22:48:30 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:48:30 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 22:48:30 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 22:48:30 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 22:48:30 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 22:48:30 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 22:48:30 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54641. -2016-06-13 22:48:30 INFO NettyBlockTransferService:59 - Server created on 54641 -2016-06-13 22:48:30 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:48:30 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54641 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54641) -2016-06-13 22:48:30 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:48:31 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 22:48:31 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 22:48:31 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 22:48:31 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 22:48:31 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54641 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:48:31 INFO SparkContext:59 - Created broadcast 0 from textFile at word2VecSuite.scala:26 -2016-06-13 22:48:32 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 22:48:32 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 22:48:32 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:164 -2016-06-13 22:48:32 INFO DAGScheduler:59 - Registering RDD 4 (map at Word2Vec.scala:155) -2016-06-13 22:48:32 INFO DAGScheduler:59 - Got job 0 (collect at Word2Vec.scala:164) with 2 output partitions -2016-06-13 22:48:32 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at Word2Vec.scala:164) -2016-06-13 22:48:32 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 22:48:32 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 22:48:32 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155), which has no missing parents -2016-06-13 22:48:32 INFO MemoryStore:59 - ensureFreeSpace(4376) called with curMem=171580, maxMem=766075207 -2016-06-13 22:48:32 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 22:48:32 INFO MemoryStore:59 - ensureFreeSpace(2415) called with curMem=175956, maxMem=766075207 -2016-06-13 22:48:32 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 22:48:32 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54641 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:48:32 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:48:32 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155) -2016-06-13 22:48:32 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 22:48:32 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:48:32 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:48:32 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 22:48:32 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 22:48:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:48:32 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:48:32 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 22:48:32 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 22:48:32 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 22:48:32 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 22:48:32 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 22:48:33 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver -2016-06-13 22:48:33 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver -2016-06-13 22:48:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 132 ms on localhost (1/2) -2016-06-13 22:48:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 117 ms on localhost (2/2) -2016-06-13 22:48:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 22:48:33 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at Word2Vec.scala:155) finished in 0.146 s -2016-06-13 22:48:33 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:48:33 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:48:33 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 22:48:33 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:48:33 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 22:48:33 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163), which is now runnable -2016-06-13 22:48:33 INFO MemoryStore:59 - ensureFreeSpace(4144) called with curMem=178371, maxMem=766075207 -2016-06-13 22:48:33 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:48:33 INFO MemoryStore:59 - ensureFreeSpace(2163) called with curMem=182515, maxMem=766075207 -2016-06-13 22:48:33 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 22:48:33 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54641 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 22:48:33 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:48:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163) -2016-06-13 22:48:33 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 22:48:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:48:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:48:33 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 22:48:33 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 22:48:33 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:48:33 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:48:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-06-13 22:48:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-06-13 22:48:33 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1189 bytes result sent to driver -2016-06-13 22:48:33 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1686 bytes result sent to driver -2016-06-13 22:48:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 40 ms on localhost (1/2) -2016-06-13 22:48:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 44 ms on localhost (2/2) -2016-06-13 22:48:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 22:48:33 INFO DAGScheduler:59 - ResultStage 1 (collect at Word2Vec.scala:164) finished in 0.045 s -2016-06-13 22:48:33 INFO DAGScheduler:59 - Job 0 finished: collect at Word2Vec.scala:164, took 0.310970 s -2016-06-13 22:48:33 INFO Word2Vec:59 - trainWordsCount = 6 -2016-06-13 22:48:33 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=184678, maxMem=766075207 -2016-06-13 22:48:33 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-06-13 22:48:33 INFO MemoryStore:59 - ensureFreeSpace(4052) called with curMem=188718, maxMem=766075207 -2016-06-13 22:48:33 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:48:33 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54641 (size: 4.0 KB, free: 730.6 MB) -2016-06-13 22:48:33 INFO SparkContext:59 - Created broadcast 3 from broadcast at Word2Vec.scala:284 -2016-06-13 22:48:33 INFO MemoryStore:59 - ensureFreeSpace(488) called with curMem=192770, maxMem=766075207 -2016-06-13 22:48:33 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 488.0 B, free 730.4 MB) -2016-06-13 22:48:33 INFO MemoryStore:59 - ensureFreeSpace(246) called with curMem=193258, maxMem=766075207 -2016-06-13 22:48:33 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 246.0 B, free 730.4 MB) -2016-06-13 22:48:33 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54641 (size: 246.0 B, free: 730.6 MB) -2016-06-13 22:48:33 INFO SparkContext:59 - Created broadcast 4 from broadcast at Word2Vec.scala:285 -2016-06-13 22:48:33 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=193504, maxMem=766075207 -2016-06-13 22:48:33 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 22:48:33 INFO MemoryStore:59 - ensureFreeSpace(172) called with curMem=193744, maxMem=766075207 -2016-06-13 22:48:33 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 172.0 B, free 730.4 MB) -2016-06-13 22:48:33 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:54641 (size: 172.0 B, free: 730.6 MB) -2016-06-13 22:48:33 INFO SparkContext:59 - Created broadcast 5 from broadcast at Word2Vec.scala:286 -2016-06-13 22:48:33 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:54641 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 22:48:33 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:54641 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:48:33 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-06-13 22:48:33 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:400 -2016-06-13 22:48:33 INFO DAGScheduler:59 - Registering RDD 9 (repartition at Word2Vec.scala:309) -2016-06-13 22:48:33 INFO DAGScheduler:59 - Registering RDD 13 (mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:48:33 INFO DAGScheduler:59 - Got job 1 (collect at Word2Vec.scala:400) with 1 output partitions -2016-06-13 22:48:33 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at Word2Vec.scala:400) -2016-06-13 22:48:33 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3) -2016-06-13 22:48:33 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 3) -2016-06-13 22:48:33 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309), which has no missing parents -2016-06-13 22:48:33 INFO MemoryStore:59 - ensureFreeSpace(6280) called with curMem=180818, maxMem=766075207 -2016-06-13 22:48:33 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 6.1 KB, free 730.4 MB) -2016-06-13 22:48:33 INFO MemoryStore:59 - ensureFreeSpace(3097) called with curMem=187098, maxMem=766075207 -2016-06-13 22:48:33 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 22:48:33 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:54641 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 22:48:33 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:48:33 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309) -2016-06-13 22:48:33 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 22:48:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:48:33 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:48:33 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 22:48:33 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 22:48:33 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:48:33 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:48:33 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2253 bytes result sent to driver -2016-06-13 22:48:33 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2253 bytes result sent to driver -2016-06-13 22:48:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 43 ms on localhost (1/2) -2016-06-13 22:48:33 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 43 ms on localhost (2/2) -2016-06-13 22:48:33 INFO DAGScheduler:59 - ShuffleMapStage 2 (repartition at Word2Vec.scala:309) finished in 0.045 s -2016-06-13 22:48:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 22:48:33 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:48:33 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:48:33 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 3, ResultStage 4) -2016-06-13 22:48:33 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:48:33 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List() -2016-06-13 22:48:33 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List(ShuffleMapStage 3) -2016-06-13 22:48:33 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323), which is now runnable -2016-06-13 22:48:33 INFO MemoryStore:59 - ensureFreeSpace(6576) called with curMem=190195, maxMem=766075207 -2016-06-13 22:48:33 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 6.4 KB, free 730.4 MB) -2016-06-13 22:48:33 INFO MemoryStore:59 - ensureFreeSpace(3350) called with curMem=196771, maxMem=766075207 -2016-06-13 22:48:33 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 22:48:33 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:54641 (size: 3.3 KB, free: 730.6 MB) -2016-06-13 22:48:33 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:48:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:48:33 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-06-13 22:48:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2159 bytes) -2016-06-13 22:48:33 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-06-13 22:48:33 INFO CacheManager:59 - Partition rdd_12_0 not found, computing it -2016-06-13 22:48:33 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:48:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 22:48:33 INFO MemoryStore:59 - ensureFreeSpace(80) called with curMem=200121, maxMem=766075207 -2016-06-13 22:48:33 INFO MemoryStore:59 - Block rdd_12_0 stored as values in memory (estimated size 80.0 B, free 730.4 MB) -2016-06-13 22:48:33 INFO BlockManagerInfo:59 - Added rdd_12_0 in memory on localhost:54641 (size: 80.0 B, free: 730.6 MB) -2016-06-13 22:48:33 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 22:48:33 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 22:48:33 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 1954 bytes result sent to driver -2016-06-13 22:48:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 51 ms on localhost (1/1) -2016-06-13 22:48:33 INFO DAGScheduler:59 - ShuffleMapStage 3 (mapPartitionsWithIndex at Word2Vec.scala:323) finished in 0.052 s -2016-06-13 22:48:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 22:48:33 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:48:33 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:48:33 INFO DAGScheduler:59 - waiting: Set(ResultStage 4) -2016-06-13 22:48:33 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:48:33 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List() -2016-06-13 22:48:33 INFO DAGScheduler:59 - Submitting ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397), which is now runnable -2016-06-13 22:48:33 INFO MemoryStore:59 - ensureFreeSpace(4296) called with curMem=200201, maxMem=766075207 -2016-06-13 22:48:33 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 22:48:33 INFO MemoryStore:59 - ensureFreeSpace(2206) called with curMem=204497, maxMem=766075207 -2016-06-13 22:48:33 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 22:48:33 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:54641 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 22:48:33 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:48:33 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397) -2016-06-13 22:48:33 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-06-13 22:48:33 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:48:33 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-06-13 22:48:33 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 22:48:33 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 22:48:33 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 1729 bytes result sent to driver -2016-06-13 22:48:33 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 15 ms on localhost (1/1) -2016-06-13 22:48:33 INFO DAGScheduler:59 - ResultStage 4 (collect at Word2Vec.scala:400) finished in 0.015 s -2016-06-13 22:48:33 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 22:48:33 INFO DAGScheduler:59 - Job 1 finished: collect at Word2Vec.scala:400, took 0.176711 s -2016-06-13 22:48:33 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-06-13 22:48:33 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 22:48:33 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 22:48:33 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 22:48:33 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 22:48:33 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 22:48:33 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 22:48:33 INFO BlockManager:59 - BlockManager stopped -2016-06-13 22:48:33 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 22:48:33 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 22:48:33 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 22:48:33 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 22:48:33 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 22:48:33 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 22:48:33 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-b9291e9e-2d3e-4111-8f4f-122beacfc734 -2016-06-13 22:48:44 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 22:48:45 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 22:48:45 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 22:48:45 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 22:48:45 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 22:48:46 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 22:48:46 INFO Remoting:74 - Starting remoting -2016-06-13 22:48:46 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:54662] -2016-06-13 22:48:46 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54662. -2016-06-13 22:48:46 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 22:48:46 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 22:48:46 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-bb25c659-c6e6-4ffc-afbd-d565db465c81 -2016-06-13 22:48:46 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 22:48:46 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-071ded2e-4f99-45ce-9b77-9f866a740219\httpd-900250f1-1949-41ae-88ea-29fde1426561 -2016-06-13 22:48:46 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 22:48:46 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:48:46 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54663 -2016-06-13 22:48:46 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54663. -2016-06-13 22:48:46 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 22:48:46 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:48:46 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 22:48:46 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 22:48:46 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 22:48:47 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 22:48:47 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 22:48:47 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54682. -2016-06-13 22:48:47 INFO NettyBlockTransferService:59 - Server created on 54682 -2016-06-13 22:48:47 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:48:47 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54682 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54682) -2016-06-13 22:48:47 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:48:48 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 22:48:48 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 22:48:48 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 22:48:48 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 22:48:48 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54682 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:48:48 INFO SparkContext:59 - Created broadcast 0 from textFile at word2VecSuite.scala:26 -2016-06-13 22:48:49 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 22:48:49 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 22:48:49 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:164 -2016-06-13 22:48:49 INFO DAGScheduler:59 - Registering RDD 4 (map at Word2Vec.scala:155) -2016-06-13 22:48:49 INFO DAGScheduler:59 - Got job 0 (collect at Word2Vec.scala:164) with 2 output partitions -2016-06-13 22:48:49 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at Word2Vec.scala:164) -2016-06-13 22:48:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 22:48:49 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 22:48:49 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155), which has no missing parents -2016-06-13 22:48:49 INFO MemoryStore:59 - ensureFreeSpace(4376) called with curMem=171580, maxMem=766075207 -2016-06-13 22:48:49 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 22:48:49 INFO MemoryStore:59 - ensureFreeSpace(2414) called with curMem=175956, maxMem=766075207 -2016-06-13 22:48:49 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 22:48:49 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54682 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:48:49 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:48:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155) -2016-06-13 22:48:49 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 22:48:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:48:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:48:49 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 22:48:49 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 22:48:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:48:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:48:49 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 22:48:49 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 22:48:49 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 22:48:49 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 22:48:49 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 22:48:49 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver -2016-06-13 22:48:49 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver -2016-06-13 22:48:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 135 ms on localhost (1/2) -2016-06-13 22:48:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 156 ms on localhost (2/2) -2016-06-13 22:48:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 22:48:49 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at Word2Vec.scala:155) finished in 0.180 s -2016-06-13 22:48:49 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:48:49 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:48:49 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 22:48:49 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:48:49 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 22:48:49 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163), which is now runnable -2016-06-13 22:48:49 INFO MemoryStore:59 - ensureFreeSpace(4144) called with curMem=178370, maxMem=766075207 -2016-06-13 22:48:49 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:48:49 INFO MemoryStore:59 - ensureFreeSpace(2163) called with curMem=182514, maxMem=766075207 -2016-06-13 22:48:49 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 22:48:49 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54682 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 22:48:49 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:48:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163) -2016-06-13 22:48:49 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 22:48:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:48:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:48:49 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 22:48:49 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 22:48:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:48:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:48:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-06-13 22:48:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-06-13 22:48:49 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1189 bytes result sent to driver -2016-06-13 22:48:49 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1686 bytes result sent to driver -2016-06-13 22:48:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 49 ms on localhost (1/2) -2016-06-13 22:48:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 53 ms on localhost (2/2) -2016-06-13 22:48:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 22:48:50 INFO DAGScheduler:59 - ResultStage 1 (collect at Word2Vec.scala:164) finished in 0.054 s -2016-06-13 22:48:50 INFO DAGScheduler:59 - Job 0 finished: collect at Word2Vec.scala:164, took 0.364456 s -2016-06-13 22:48:50 INFO Word2Vec:59 - trainWordsCount = 6 -2016-06-13 22:48:50 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=184677, maxMem=766075207 -2016-06-13 22:48:50 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-06-13 22:48:50 INFO MemoryStore:59 - ensureFreeSpace(4052) called with curMem=188717, maxMem=766075207 -2016-06-13 22:48:50 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:48:50 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54682 (size: 4.0 KB, free: 730.6 MB) -2016-06-13 22:48:50 INFO SparkContext:59 - Created broadcast 3 from broadcast at Word2Vec.scala:284 -2016-06-13 22:48:50 INFO MemoryStore:59 - ensureFreeSpace(488) called with curMem=192769, maxMem=766075207 -2016-06-13 22:48:50 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 488.0 B, free 730.4 MB) -2016-06-13 22:48:50 INFO MemoryStore:59 - ensureFreeSpace(246) called with curMem=193257, maxMem=766075207 -2016-06-13 22:48:50 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 246.0 B, free 730.4 MB) -2016-06-13 22:48:50 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54682 (size: 246.0 B, free: 730.6 MB) -2016-06-13 22:48:50 INFO SparkContext:59 - Created broadcast 4 from broadcast at Word2Vec.scala:285 -2016-06-13 22:48:50 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=193503, maxMem=766075207 -2016-06-13 22:48:50 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 22:48:50 INFO MemoryStore:59 - ensureFreeSpace(172) called with curMem=193743, maxMem=766075207 -2016-06-13 22:48:50 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 172.0 B, free 730.4 MB) -2016-06-13 22:48:50 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:54682 (size: 172.0 B, free: 730.6 MB) -2016-06-13 22:48:50 INFO SparkContext:59 - Created broadcast 5 from broadcast at Word2Vec.scala:286 -2016-06-13 22:48:50 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:400 -2016-06-13 22:48:50 INFO DAGScheduler:59 - Registering RDD 9 (repartition at Word2Vec.scala:309) -2016-06-13 22:48:50 INFO DAGScheduler:59 - Registering RDD 13 (mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:48:50 INFO DAGScheduler:59 - Got job 1 (collect at Word2Vec.scala:400) with 1 output partitions -2016-06-13 22:48:50 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at Word2Vec.scala:400) -2016-06-13 22:48:50 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3) -2016-06-13 22:48:50 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 3) -2016-06-13 22:48:50 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309), which has no missing parents -2016-06-13 22:48:50 INFO MemoryStore:59 - ensureFreeSpace(6280) called with curMem=193915, maxMem=766075207 -2016-06-13 22:48:50 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 6.1 KB, free 730.4 MB) -2016-06-13 22:48:50 INFO MemoryStore:59 - ensureFreeSpace(3096) called with curMem=200195, maxMem=766075207 -2016-06-13 22:48:50 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 22:48:50 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:54682 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 22:48:50 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:48:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309) -2016-06-13 22:48:50 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 22:48:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:48:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:48:50 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 22:48:50 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 22:48:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:48:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:48:50 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2253 bytes result sent to driver -2016-06-13 22:48:50 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2253 bytes result sent to driver -2016-06-13 22:48:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 43 ms on localhost (1/2) -2016-06-13 22:48:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 48 ms on localhost (2/2) -2016-06-13 22:48:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 22:48:50 INFO DAGScheduler:59 - ShuffleMapStage 2 (repartition at Word2Vec.scala:309) finished in 0.049 s -2016-06-13 22:48:50 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:48:50 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:48:50 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 3, ResultStage 4) -2016-06-13 22:48:50 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:48:50 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List() -2016-06-13 22:48:50 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List(ShuffleMapStage 3) -2016-06-13 22:48:50 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323), which is now runnable -2016-06-13 22:48:50 INFO MemoryStore:59 - ensureFreeSpace(6576) called with curMem=203291, maxMem=766075207 -2016-06-13 22:48:50 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 6.4 KB, free 730.4 MB) -2016-06-13 22:48:50 INFO MemoryStore:59 - ensureFreeSpace(3353) called with curMem=209867, maxMem=766075207 -2016-06-13 22:48:50 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 22:48:50 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:54682 (size: 3.3 KB, free: 730.6 MB) -2016-06-13 22:48:50 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:48:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:48:50 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-06-13 22:48:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2159 bytes) -2016-06-13 22:48:50 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-06-13 22:48:50 INFO CacheManager:59 - Partition rdd_12_0 not found, computing it -2016-06-13 22:48:50 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:48:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 22:48:50 INFO MemoryStore:59 - ensureFreeSpace(80) called with curMem=213220, maxMem=766075207 -2016-06-13 22:48:50 INFO MemoryStore:59 - Block rdd_12_0 stored as values in memory (estimated size 80.0 B, free 730.4 MB) -2016-06-13 22:48:50 INFO BlockManagerInfo:59 - Added rdd_12_0 in memory on localhost:54682 (size: 80.0 B, free: 730.6 MB) -2016-06-13 22:48:50 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 22:48:50 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 22:48:50 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 1954 bytes result sent to driver -2016-06-13 22:48:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 72 ms on localhost (1/1) -2016-06-13 22:48:50 INFO DAGScheduler:59 - ShuffleMapStage 3 (mapPartitionsWithIndex at Word2Vec.scala:323) finished in 0.073 s -2016-06-13 22:48:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 22:48:50 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:48:50 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:48:50 INFO DAGScheduler:59 - waiting: Set(ResultStage 4) -2016-06-13 22:48:50 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:48:50 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List() -2016-06-13 22:48:50 INFO DAGScheduler:59 - Submitting ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397), which is now runnable -2016-06-13 22:48:50 INFO MemoryStore:59 - ensureFreeSpace(4296) called with curMem=213300, maxMem=766075207 -2016-06-13 22:48:50 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 22:48:50 INFO MemoryStore:59 - ensureFreeSpace(2206) called with curMem=217596, maxMem=766075207 -2016-06-13 22:48:50 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 22:48:50 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:54682 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 22:48:50 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:48:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397) -2016-06-13 22:48:50 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-06-13 22:48:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:48:50 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-06-13 22:48:50 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 22:48:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 22:48:50 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 1729 bytes result sent to driver -2016-06-13 22:48:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 11 ms on localhost (1/1) -2016-06-13 22:48:50 INFO DAGScheduler:59 - ResultStage 4 (collect at Word2Vec.scala:400) finished in 0.013 s -2016-06-13 22:48:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 22:48:50 INFO DAGScheduler:59 - Job 1 finished: collect at Word2Vec.scala:400, took 0.203607 s -2016-06-13 22:48:50 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-06-13 22:48:50 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 22:48:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 22:48:50 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 22:48:50 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 22:48:50 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 22:48:50 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 22:48:50 INFO BlockManager:59 - BlockManager stopped -2016-06-13 22:48:50 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 22:48:50 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 22:48:50 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 22:48:50 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 22:48:50 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-071ded2e-4f99-45ce-9b77-9f866a740219 -2016-06-13 22:48:50 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 22:48:50 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 22:49:30 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 22:49:30 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 22:49:30 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 22:49:30 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 22:49:30 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 22:49:31 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 22:49:31 INFO Remoting:74 - Starting remoting -2016-06-13 22:49:31 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:54702] -2016-06-13 22:49:31 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54702. -2016-06-13 22:49:31 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 22:49:32 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 22:49:32 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-0e41bc59-a8b4-4052-8ce3-c6a379fb2b5b -2016-06-13 22:49:32 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 22:49:32 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-25c497c1-edc8-47e8-9477-5f380123622c\httpd-602f56fd-9972-4412-ad21-9b06fc5daebf -2016-06-13 22:49:32 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 22:49:32 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:49:32 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54703 -2016-06-13 22:49:32 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54703. -2016-06-13 22:49:32 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 22:49:32 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:49:32 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 22:49:32 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 22:49:32 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 22:49:32 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 22:49:32 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 22:49:32 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54722. -2016-06-13 22:49:32 INFO NettyBlockTransferService:59 - Server created on 54722 -2016-06-13 22:49:32 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:49:32 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54722 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54722) -2016-06-13 22:49:32 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:49:33 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 22:49:33 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 22:49:33 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 22:49:33 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 22:49:33 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54722 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:49:33 INFO SparkContext:59 - Created broadcast 0 from textFile at word2VecSuite.scala:26 -2016-06-13 22:49:34 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 22:49:35 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 22:49:35 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:164 -2016-06-13 22:49:35 INFO DAGScheduler:59 - Registering RDD 4 (map at Word2Vec.scala:155) -2016-06-13 22:49:35 INFO DAGScheduler:59 - Got job 0 (collect at Word2Vec.scala:164) with 2 output partitions -2016-06-13 22:49:35 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at Word2Vec.scala:164) -2016-06-13 22:49:35 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 22:49:35 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 22:49:35 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155), which has no missing parents -2016-06-13 22:49:35 INFO MemoryStore:59 - ensureFreeSpace(4376) called with curMem=171580, maxMem=766075207 -2016-06-13 22:49:35 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 22:49:35 INFO MemoryStore:59 - ensureFreeSpace(2414) called with curMem=175956, maxMem=766075207 -2016-06-13 22:49:35 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 22:49:35 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54722 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:49:35 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:49:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155) -2016-06-13 22:49:35 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 22:49:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:49:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:49:35 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 22:49:35 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 22:49:35 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:49:35 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:49:35 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 22:49:35 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 22:49:35 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 22:49:35 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 22:49:35 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 22:49:35 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver -2016-06-13 22:49:35 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver -2016-06-13 22:49:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 127 ms on localhost (1/2) -2016-06-13 22:49:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 112 ms on localhost (2/2) -2016-06-13 22:49:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 22:49:35 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at Word2Vec.scala:155) finished in 0.146 s -2016-06-13 22:49:35 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:49:35 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:49:35 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 22:49:35 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:49:35 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 22:49:35 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163), which is now runnable -2016-06-13 22:49:35 INFO MemoryStore:59 - ensureFreeSpace(4144) called with curMem=178370, maxMem=766075207 -2016-06-13 22:49:35 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:49:35 INFO MemoryStore:59 - ensureFreeSpace(2163) called with curMem=182514, maxMem=766075207 -2016-06-13 22:49:35 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 22:49:35 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54722 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 22:49:35 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:49:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163) -2016-06-13 22:49:35 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 22:49:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:49:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:49:35 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 22:49:35 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 22:49:35 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:49:35 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:49:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-06-13 22:49:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-06-13 22:49:35 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1686 bytes result sent to driver -2016-06-13 22:49:35 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1189 bytes result sent to driver -2016-06-13 22:49:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 42 ms on localhost (1/2) -2016-06-13 22:49:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 40 ms on localhost (2/2) -2016-06-13 22:49:35 INFO DAGScheduler:59 - ResultStage 1 (collect at Word2Vec.scala:164) finished in 0.044 s -2016-06-13 22:49:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 22:49:35 INFO DAGScheduler:59 - Job 0 finished: collect at Word2Vec.scala:164, took 0.291596 s -2016-06-13 22:49:35 INFO Word2Vec:59 - trainWordsCount = 6 -2016-06-13 22:49:35 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=184677, maxMem=766075207 -2016-06-13 22:49:35 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-06-13 22:49:35 INFO MemoryStore:59 - ensureFreeSpace(4052) called with curMem=188717, maxMem=766075207 -2016-06-13 22:49:35 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:49:35 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54722 (size: 4.0 KB, free: 730.6 MB) -2016-06-13 22:49:35 INFO SparkContext:59 - Created broadcast 3 from broadcast at Word2Vec.scala:284 -2016-06-13 22:49:35 INFO MemoryStore:59 - ensureFreeSpace(488) called with curMem=192769, maxMem=766075207 -2016-06-13 22:49:35 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 488.0 B, free 730.4 MB) -2016-06-13 22:49:35 INFO MemoryStore:59 - ensureFreeSpace(246) called with curMem=193257, maxMem=766075207 -2016-06-13 22:49:35 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 246.0 B, free 730.4 MB) -2016-06-13 22:49:35 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54722 (size: 246.0 B, free: 730.6 MB) -2016-06-13 22:49:35 INFO SparkContext:59 - Created broadcast 4 from broadcast at Word2Vec.scala:285 -2016-06-13 22:49:35 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=193503, maxMem=766075207 -2016-06-13 22:49:35 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 22:49:35 INFO MemoryStore:59 - ensureFreeSpace(172) called with curMem=193743, maxMem=766075207 -2016-06-13 22:49:35 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 172.0 B, free 730.4 MB) -2016-06-13 22:49:35 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:54722 (size: 172.0 B, free: 730.6 MB) -2016-06-13 22:49:35 INFO SparkContext:59 - Created broadcast 5 from broadcast at Word2Vec.scala:286 -2016-06-13 22:49:35 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:54722 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 22:49:35 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:400 -2016-06-13 22:49:35 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 22:49:35 INFO DAGScheduler:59 - Registering RDD 9 (repartition at Word2Vec.scala:309) -2016-06-13 22:49:35 INFO DAGScheduler:59 - Registering RDD 13 (mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:49:35 INFO DAGScheduler:59 - Got job 1 (collect at Word2Vec.scala:400) with 1 output partitions -2016-06-13 22:49:35 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at Word2Vec.scala:400) -2016-06-13 22:49:35 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3) -2016-06-13 22:49:35 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 3) -2016-06-13 22:49:35 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:54722 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:49:35 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 22:49:35 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309), which has no missing parents -2016-06-13 22:49:35 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-06-13 22:49:35 INFO MemoryStore:59 - ensureFreeSpace(6280) called with curMem=180818, maxMem=766075207 -2016-06-13 22:49:35 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 6.1 KB, free 730.4 MB) -2016-06-13 22:49:35 INFO MemoryStore:59 - ensureFreeSpace(3096) called with curMem=187098, maxMem=766075207 -2016-06-13 22:49:35 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 22:49:35 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:54722 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 22:49:35 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:49:35 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309) -2016-06-13 22:49:35 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 22:49:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:49:35 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:49:35 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 22:49:35 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 22:49:35 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:49:35 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:49:35 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2253 bytes result sent to driver -2016-06-13 22:49:35 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2253 bytes result sent to driver -2016-06-13 22:49:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 41 ms on localhost (1/2) -2016-06-13 22:49:35 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 41 ms on localhost (2/2) -2016-06-13 22:49:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 22:49:35 INFO DAGScheduler:59 - ShuffleMapStage 2 (repartition at Word2Vec.scala:309) finished in 0.043 s -2016-06-13 22:49:35 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:49:35 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:49:35 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 3, ResultStage 4) -2016-06-13 22:49:35 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:49:35 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List() -2016-06-13 22:49:35 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List(ShuffleMapStage 3) -2016-06-13 22:49:35 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323), which is now runnable -2016-06-13 22:49:35 INFO MemoryStore:59 - ensureFreeSpace(6576) called with curMem=190194, maxMem=766075207 -2016-06-13 22:49:35 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 6.4 KB, free 730.4 MB) -2016-06-13 22:49:35 INFO MemoryStore:59 - ensureFreeSpace(3347) called with curMem=196770, maxMem=766075207 -2016-06-13 22:49:35 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 22:49:35 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:54722 (size: 3.3 KB, free: 730.6 MB) -2016-06-13 22:49:35 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:49:35 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:49:35 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-06-13 22:49:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2159 bytes) -2016-06-13 22:49:35 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-06-13 22:49:35 INFO CacheManager:59 - Partition rdd_12_0 not found, computing it -2016-06-13 22:49:35 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:49:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 22:49:35 INFO MemoryStore:59 - ensureFreeSpace(80) called with curMem=200117, maxMem=766075207 -2016-06-13 22:49:35 INFO MemoryStore:59 - Block rdd_12_0 stored as values in memory (estimated size 80.0 B, free 730.4 MB) -2016-06-13 22:49:35 INFO BlockManagerInfo:59 - Added rdd_12_0 in memory on localhost:54722 (size: 80.0 B, free: 730.6 MB) -2016-06-13 22:49:35 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 22:49:35 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 22:49:35 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 1954 bytes result sent to driver -2016-06-13 22:49:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 55 ms on localhost (1/1) -2016-06-13 22:49:35 INFO DAGScheduler:59 - ShuffleMapStage 3 (mapPartitionsWithIndex at Word2Vec.scala:323) finished in 0.057 s -2016-06-13 22:49:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 22:49:35 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:49:35 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:49:35 INFO DAGScheduler:59 - waiting: Set(ResultStage 4) -2016-06-13 22:49:35 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:49:35 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List() -2016-06-13 22:49:35 INFO DAGScheduler:59 - Submitting ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397), which is now runnable -2016-06-13 22:49:35 INFO MemoryStore:59 - ensureFreeSpace(4296) called with curMem=200197, maxMem=766075207 -2016-06-13 22:49:35 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 22:49:35 INFO MemoryStore:59 - ensureFreeSpace(2206) called with curMem=204493, maxMem=766075207 -2016-06-13 22:49:35 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 22:49:35 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:54722 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 22:49:35 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:49:35 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397) -2016-06-13 22:49:35 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-06-13 22:49:35 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:49:35 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-06-13 22:49:35 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 22:49:35 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 22:49:35 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 1729 bytes result sent to driver -2016-06-13 22:49:35 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 14 ms on localhost (1/1) -2016-06-13 22:49:35 INFO DAGScheduler:59 - ResultStage 4 (collect at Word2Vec.scala:400) finished in 0.014 s -2016-06-13 22:49:35 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 22:49:35 INFO DAGScheduler:59 - Job 1 finished: collect at Word2Vec.scala:400, took 0.178357 s -2016-06-13 22:49:35 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-06-13 22:49:35 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 22:49:35 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 22:49:35 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 22:49:35 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 22:49:36 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 22:49:36 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 22:49:36 INFO BlockManager:59 - BlockManager stopped -2016-06-13 22:49:36 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 22:49:36 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 22:49:36 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 22:49:36 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 22:49:36 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 22:49:36 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 22:49:36 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-25c497c1-edc8-47e8-9477-5f380123622c -2016-06-13 22:49:44 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 22:49:45 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 22:49:45 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 22:49:45 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 22:49:45 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 22:49:46 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 22:49:46 INFO Remoting:74 - Starting remoting -2016-06-13 22:49:46 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:54742] -2016-06-13 22:49:46 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54742. -2016-06-13 22:49:46 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 22:49:46 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 22:49:46 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-620abd37-afd9-41d6-9c15-9701ac8ce47c -2016-06-13 22:49:46 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 22:49:46 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-b3d57059-05c6-48c9-b662-db9a07447f7b\httpd-0fad4585-cf25-4619-b2a6-d0fb84edc2d2 -2016-06-13 22:49:46 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 22:49:46 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:49:47 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54743 -2016-06-13 22:49:47 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54743. -2016-06-13 22:49:47 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 22:49:47 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:49:47 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 22:49:47 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 22:49:47 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 22:49:47 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 22:49:47 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 22:49:47 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54762. -2016-06-13 22:49:47 INFO NettyBlockTransferService:59 - Server created on 54762 -2016-06-13 22:49:47 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:49:47 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54762 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54762) -2016-06-13 22:49:47 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:49:48 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 22:49:48 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 22:49:48 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 22:49:48 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 22:49:48 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54762 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:49:48 INFO SparkContext:59 - Created broadcast 0 from textFile at word2VecSuite.scala:26 -2016-06-13 22:49:49 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 22:49:49 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 22:49:50 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:164 -2016-06-13 22:49:50 INFO DAGScheduler:59 - Registering RDD 4 (map at Word2Vec.scala:155) -2016-06-13 22:49:50 INFO DAGScheduler:59 - Got job 0 (collect at Word2Vec.scala:164) with 2 output partitions -2016-06-13 22:49:50 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at Word2Vec.scala:164) -2016-06-13 22:49:50 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 22:49:50 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 22:49:50 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155), which has no missing parents -2016-06-13 22:49:50 INFO MemoryStore:59 - ensureFreeSpace(4376) called with curMem=171580, maxMem=766075207 -2016-06-13 22:49:50 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 22:49:50 INFO MemoryStore:59 - ensureFreeSpace(2414) called with curMem=175956, maxMem=766075207 -2016-06-13 22:49:50 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 22:49:50 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54762 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:49:50 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:49:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155) -2016-06-13 22:49:50 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 22:49:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:49:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:49:50 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 22:49:50 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 22:49:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:49:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:49:50 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 22:49:50 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 22:49:50 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 22:49:50 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 22:49:50 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 22:49:50 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver -2016-06-13 22:49:50 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver -2016-06-13 22:49:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 112 ms on localhost (1/2) -2016-06-13 22:49:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 130 ms on localhost (2/2) -2016-06-13 22:49:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 22:49:50 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at Word2Vec.scala:155) finished in 0.148 s -2016-06-13 22:49:50 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:49:50 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:49:50 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 22:49:50 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:49:50 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 22:49:50 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163), which is now runnable -2016-06-13 22:49:50 INFO MemoryStore:59 - ensureFreeSpace(4144) called with curMem=178370, maxMem=766075207 -2016-06-13 22:49:50 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:49:50 INFO MemoryStore:59 - ensureFreeSpace(2163) called with curMem=182514, maxMem=766075207 -2016-06-13 22:49:50 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 22:49:50 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54762 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 22:49:50 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:49:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163) -2016-06-13 22:49:50 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 22:49:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:49:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:49:50 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 22:49:50 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 22:49:50 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:49:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 7 ms -2016-06-13 22:49:50 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:49:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 22:49:50 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1189 bytes result sent to driver -2016-06-13 22:49:50 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1686 bytes result sent to driver -2016-06-13 22:49:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 67 ms on localhost (1/2) -2016-06-13 22:49:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 72 ms on localhost (2/2) -2016-06-13 22:49:50 INFO DAGScheduler:59 - ResultStage 1 (collect at Word2Vec.scala:164) finished in 0.076 s -2016-06-13 22:49:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 22:49:50 INFO DAGScheduler:59 - Job 0 finished: collect at Word2Vec.scala:164, took 0.346246 s -2016-06-13 22:49:50 INFO Word2Vec:59 - trainWordsCount = 6 -2016-06-13 22:49:50 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=184677, maxMem=766075207 -2016-06-13 22:49:50 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-06-13 22:49:50 INFO MemoryStore:59 - ensureFreeSpace(4052) called with curMem=188717, maxMem=766075207 -2016-06-13 22:49:50 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:49:50 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54762 (size: 4.0 KB, free: 730.6 MB) -2016-06-13 22:49:50 INFO SparkContext:59 - Created broadcast 3 from broadcast at Word2Vec.scala:284 -2016-06-13 22:49:50 INFO MemoryStore:59 - ensureFreeSpace(488) called with curMem=192769, maxMem=766075207 -2016-06-13 22:49:50 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 488.0 B, free 730.4 MB) -2016-06-13 22:49:50 INFO MemoryStore:59 - ensureFreeSpace(246) called with curMem=193257, maxMem=766075207 -2016-06-13 22:49:50 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 246.0 B, free 730.4 MB) -2016-06-13 22:49:50 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54762 (size: 246.0 B, free: 730.6 MB) -2016-06-13 22:49:50 INFO SparkContext:59 - Created broadcast 4 from broadcast at Word2Vec.scala:285 -2016-06-13 22:49:50 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=193503, maxMem=766075207 -2016-06-13 22:49:50 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 22:49:50 INFO MemoryStore:59 - ensureFreeSpace(172) called with curMem=193743, maxMem=766075207 -2016-06-13 22:49:50 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 172.0 B, free 730.4 MB) -2016-06-13 22:49:50 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:54762 (size: 172.0 B, free: 730.6 MB) -2016-06-13 22:49:50 INFO SparkContext:59 - Created broadcast 5 from broadcast at Word2Vec.scala:286 -2016-06-13 22:49:50 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:400 -2016-06-13 22:49:50 INFO DAGScheduler:59 - Registering RDD 9 (repartition at Word2Vec.scala:309) -2016-06-13 22:49:50 INFO DAGScheduler:59 - Registering RDD 13 (mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:49:50 INFO DAGScheduler:59 - Got job 1 (collect at Word2Vec.scala:400) with 1 output partitions -2016-06-13 22:49:50 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at Word2Vec.scala:400) -2016-06-13 22:49:50 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3) -2016-06-13 22:49:50 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 3) -2016-06-13 22:49:50 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309), which has no missing parents -2016-06-13 22:49:50 INFO MemoryStore:59 - ensureFreeSpace(6280) called with curMem=193915, maxMem=766075207 -2016-06-13 22:49:50 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 6.1 KB, free 730.4 MB) -2016-06-13 22:49:50 INFO MemoryStore:59 - ensureFreeSpace(3096) called with curMem=200195, maxMem=766075207 -2016-06-13 22:49:50 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 22:49:50 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:54762 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 22:49:50 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:49:50 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309) -2016-06-13 22:49:50 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 22:49:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:49:50 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:49:50 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 22:49:50 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 22:49:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:49:50 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:49:50 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2253 bytes result sent to driver -2016-06-13 22:49:50 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2253 bytes result sent to driver -2016-06-13 22:49:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 43 ms on localhost (1/2) -2016-06-13 22:49:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 47 ms on localhost (2/2) -2016-06-13 22:49:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 22:49:50 INFO DAGScheduler:59 - ShuffleMapStage 2 (repartition at Word2Vec.scala:309) finished in 0.049 s -2016-06-13 22:49:50 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:49:50 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:49:50 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 3, ResultStage 4) -2016-06-13 22:49:50 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:49:50 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List() -2016-06-13 22:49:50 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List(ShuffleMapStage 3) -2016-06-13 22:49:50 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323), which is now runnable -2016-06-13 22:49:50 INFO MemoryStore:59 - ensureFreeSpace(6576) called with curMem=203291, maxMem=766075207 -2016-06-13 22:49:50 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 6.4 KB, free 730.4 MB) -2016-06-13 22:49:50 INFO MemoryStore:59 - ensureFreeSpace(3350) called with curMem=209867, maxMem=766075207 -2016-06-13 22:49:50 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 22:49:50 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:54762 (size: 3.3 KB, free: 730.6 MB) -2016-06-13 22:49:50 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:49:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:49:50 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-06-13 22:49:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2159 bytes) -2016-06-13 22:49:50 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-06-13 22:49:50 INFO CacheManager:59 - Partition rdd_12_0 not found, computing it -2016-06-13 22:49:50 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:49:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 22:49:50 INFO MemoryStore:59 - ensureFreeSpace(80) called with curMem=213217, maxMem=766075207 -2016-06-13 22:49:50 INFO MemoryStore:59 - Block rdd_12_0 stored as values in memory (estimated size 80.0 B, free 730.4 MB) -2016-06-13 22:49:50 INFO BlockManagerInfo:59 - Added rdd_12_0 in memory on localhost:54762 (size: 80.0 B, free: 730.6 MB) -2016-06-13 22:49:50 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 22:49:50 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 22:49:50 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 1954 bytes result sent to driver -2016-06-13 22:49:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 66 ms on localhost (1/1) -2016-06-13 22:49:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 22:49:50 INFO DAGScheduler:59 - ShuffleMapStage 3 (mapPartitionsWithIndex at Word2Vec.scala:323) finished in 0.068 s -2016-06-13 22:49:50 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:49:50 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:49:50 INFO DAGScheduler:59 - waiting: Set(ResultStage 4) -2016-06-13 22:49:50 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:49:50 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List() -2016-06-13 22:49:50 INFO DAGScheduler:59 - Submitting ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397), which is now runnable -2016-06-13 22:49:50 INFO MemoryStore:59 - ensureFreeSpace(4296) called with curMem=213297, maxMem=766075207 -2016-06-13 22:49:50 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 22:49:50 INFO MemoryStore:59 - ensureFreeSpace(2206) called with curMem=217593, maxMem=766075207 -2016-06-13 22:49:50 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 22:49:50 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:54762 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 22:49:50 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:49:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397) -2016-06-13 22:49:50 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-06-13 22:49:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:49:50 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-06-13 22:49:50 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 22:49:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 22:49:50 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 1729 bytes result sent to driver -2016-06-13 22:49:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 13 ms on localhost (1/1) -2016-06-13 22:49:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 22:49:50 INFO DAGScheduler:59 - ResultStage 4 (collect at Word2Vec.scala:400) finished in 0.013 s -2016-06-13 22:49:50 INFO DAGScheduler:59 - Job 1 finished: collect at Word2Vec.scala:400, took 0.247826 s -2016-06-13 22:49:50 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-06-13 22:49:50 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 22:49:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 22:49:51 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 22:49:51 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 22:49:51 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 22:49:51 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 22:49:51 INFO BlockManager:59 - BlockManager stopped -2016-06-13 22:49:51 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 22:49:51 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 22:49:51 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 22:49:51 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 22:49:51 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 22:49:51 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 22:49:51 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-b3d57059-05c6-48c9-b662-db9a07447f7b -2016-06-13 22:51:37 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 22:51:37 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 22:51:37 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 22:51:37 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 22:51:37 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 22:51:38 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 22:51:38 INFO Remoting:74 - Starting remoting -2016-06-13 22:51:38 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:54783] -2016-06-13 22:51:38 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54783. -2016-06-13 22:51:39 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 22:51:39 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 22:51:39 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-641913f5-de38-4fa5-a1b3-25a611837bad -2016-06-13 22:51:39 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 22:51:39 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-90a11c56-ca11-4bbd-b174-736baba5d9ca\httpd-5e338ec8-2dd7-4942-891a-d3078133fa08 -2016-06-13 22:51:39 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 22:51:39 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:51:39 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54784 -2016-06-13 22:51:39 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54784. -2016-06-13 22:51:39 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 22:51:39 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:51:39 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 22:51:39 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 22:51:39 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 22:51:39 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 22:51:39 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 22:51:39 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54803. -2016-06-13 22:51:39 INFO NettyBlockTransferService:59 - Server created on 54803 -2016-06-13 22:51:39 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:51:39 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54803 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54803) -2016-06-13 22:51:39 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:51:40 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 22:51:40 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 22:51:40 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 22:51:40 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 22:51:40 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54803 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:51:40 INFO SparkContext:59 - Created broadcast 0 from textFile at word2VecFromBookPKSuite.scala:23 -2016-06-13 22:51:41 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 22:51:41 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 22:51:42 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:164 -2016-06-13 22:51:42 INFO DAGScheduler:59 - Registering RDD 4 (map at Word2Vec.scala:155) -2016-06-13 22:51:42 INFO DAGScheduler:59 - Got job 0 (collect at Word2Vec.scala:164) with 2 output partitions -2016-06-13 22:51:42 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at Word2Vec.scala:164) -2016-06-13 22:51:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 22:51:42 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 22:51:42 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155), which has no missing parents -2016-06-13 22:51:42 INFO MemoryStore:59 - ensureFreeSpace(4384) called with curMem=171580, maxMem=766075207 -2016-06-13 22:51:42 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 22:51:42 INFO MemoryStore:59 - ensureFreeSpace(2424) called with curMem=175964, maxMem=766075207 -2016-06-13 22:51:42 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 22:51:42 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54803 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:51:42 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:51:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155) -2016-06-13 22:51:42 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 22:51:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:51:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:51:42 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 22:51:42 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 22:51:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:51:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:51:42 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 22:51:42 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 22:51:42 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 22:51:42 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 22:51:42 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 22:51:42 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver -2016-06-13 22:51:42 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver -2016-06-13 22:51:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 118 ms on localhost (1/2) -2016-06-13 22:51:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 135 ms on localhost (2/2) -2016-06-13 22:51:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 22:51:42 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at Word2Vec.scala:155) finished in 0.149 s -2016-06-13 22:51:42 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:51:42 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:51:42 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 22:51:42 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:51:42 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 22:51:42 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163), which is now runnable -2016-06-13 22:51:42 INFO MemoryStore:59 - ensureFreeSpace(4144) called with curMem=178388, maxMem=766075207 -2016-06-13 22:51:42 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:51:42 INFO MemoryStore:59 - ensureFreeSpace(2163) called with curMem=182532, maxMem=766075207 -2016-06-13 22:51:42 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 22:51:42 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54803 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 22:51:42 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:51:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163) -2016-06-13 22:51:42 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 22:51:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:51:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:51:42 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 22:51:42 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 22:51:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:51:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:51:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-06-13 22:51:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-06-13 22:51:42 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1686 bytes result sent to driver -2016-06-13 22:51:42 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1189 bytes result sent to driver -2016-06-13 22:51:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 41 ms on localhost (1/2) -2016-06-13 22:51:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 44 ms on localhost (2/2) -2016-06-13 22:51:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 22:51:42 INFO DAGScheduler:59 - ResultStage 1 (collect at Word2Vec.scala:164) finished in 0.045 s -2016-06-13 22:51:42 INFO DAGScheduler:59 - Job 0 finished: collect at Word2Vec.scala:164, took 0.306603 s -2016-06-13 22:51:42 INFO Word2Vec:59 - trainWordsCount = 6 -2016-06-13 22:51:42 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=184695, maxMem=766075207 -2016-06-13 22:51:42 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-06-13 22:51:42 INFO MemoryStore:59 - ensureFreeSpace(4052) called with curMem=188735, maxMem=766075207 -2016-06-13 22:51:42 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:51:42 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54803 (size: 4.0 KB, free: 730.6 MB) -2016-06-13 22:51:42 INFO SparkContext:59 - Created broadcast 3 from broadcast at Word2Vec.scala:284 -2016-06-13 22:51:42 INFO MemoryStore:59 - ensureFreeSpace(488) called with curMem=192787, maxMem=766075207 -2016-06-13 22:51:42 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 488.0 B, free 730.4 MB) -2016-06-13 22:51:42 INFO MemoryStore:59 - ensureFreeSpace(246) called with curMem=193275, maxMem=766075207 -2016-06-13 22:51:42 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 246.0 B, free 730.4 MB) -2016-06-13 22:51:42 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54803 (size: 246.0 B, free: 730.6 MB) -2016-06-13 22:51:42 INFO SparkContext:59 - Created broadcast 4 from broadcast at Word2Vec.scala:285 -2016-06-13 22:51:42 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=193521, maxMem=766075207 -2016-06-13 22:51:42 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 22:51:42 INFO MemoryStore:59 - ensureFreeSpace(172) called with curMem=193761, maxMem=766075207 -2016-06-13 22:51:42 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 172.0 B, free 730.4 MB) -2016-06-13 22:51:42 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:54803 (size: 172.0 B, free: 730.6 MB) -2016-06-13 22:51:42 INFO SparkContext:59 - Created broadcast 5 from broadcast at Word2Vec.scala:286 -2016-06-13 22:51:42 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:400 -2016-06-13 22:51:42 INFO DAGScheduler:59 - Registering RDD 9 (repartition at Word2Vec.scala:309) -2016-06-13 22:51:42 INFO DAGScheduler:59 - Registering RDD 13 (mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:51:42 INFO DAGScheduler:59 - Got job 1 (collect at Word2Vec.scala:400) with 1 output partitions -2016-06-13 22:51:42 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at Word2Vec.scala:400) -2016-06-13 22:51:42 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3) -2016-06-13 22:51:42 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 3) -2016-06-13 22:51:42 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309), which has no missing parents -2016-06-13 22:51:42 INFO MemoryStore:59 - ensureFreeSpace(6288) called with curMem=193933, maxMem=766075207 -2016-06-13 22:51:42 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 6.1 KB, free 730.4 MB) -2016-06-13 22:51:42 INFO MemoryStore:59 - ensureFreeSpace(3106) called with curMem=200221, maxMem=766075207 -2016-06-13 22:51:42 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 22:51:42 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:54803 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 22:51:42 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:51:42 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309) -2016-06-13 22:51:42 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 22:51:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:51:42 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:51:42 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 22:51:42 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 22:51:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:51:42 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:51:42 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2253 bytes result sent to driver -2016-06-13 22:51:42 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2253 bytes result sent to driver -2016-06-13 22:51:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 42 ms on localhost (1/2) -2016-06-13 22:51:42 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 41 ms on localhost (2/2) -2016-06-13 22:51:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 22:51:42 INFO DAGScheduler:59 - ShuffleMapStage 2 (repartition at Word2Vec.scala:309) finished in 0.044 s -2016-06-13 22:51:42 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:51:42 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:51:42 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 3, ResultStage 4) -2016-06-13 22:51:42 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:51:42 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List() -2016-06-13 22:51:42 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List(ShuffleMapStage 3) -2016-06-13 22:51:42 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323), which is now runnable -2016-06-13 22:51:42 INFO MemoryStore:59 - ensureFreeSpace(6576) called with curMem=203327, maxMem=766075207 -2016-06-13 22:51:42 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 6.4 KB, free 730.4 MB) -2016-06-13 22:51:42 INFO MemoryStore:59 - ensureFreeSpace(3350) called with curMem=209903, maxMem=766075207 -2016-06-13 22:51:42 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 22:51:42 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:54803 (size: 3.3 KB, free: 730.6 MB) -2016-06-13 22:51:42 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:51:42 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:51:42 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-06-13 22:51:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2159 bytes) -2016-06-13 22:51:42 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-06-13 22:51:42 INFO CacheManager:59 - Partition rdd_12_0 not found, computing it -2016-06-13 22:51:42 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:51:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 22:51:42 INFO MemoryStore:59 - ensureFreeSpace(80) called with curMem=213253, maxMem=766075207 -2016-06-13 22:51:42 INFO MemoryStore:59 - Block rdd_12_0 stored as values in memory (estimated size 80.0 B, free 730.4 MB) -2016-06-13 22:51:42 INFO BlockManagerInfo:59 - Added rdd_12_0 in memory on localhost:54803 (size: 80.0 B, free: 730.6 MB) -2016-06-13 22:51:42 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 22:51:42 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 22:51:42 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 1954 bytes result sent to driver -2016-06-13 22:51:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 54 ms on localhost (1/1) -2016-06-13 22:51:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 22:51:42 INFO DAGScheduler:59 - ShuffleMapStage 3 (mapPartitionsWithIndex at Word2Vec.scala:323) finished in 0.056 s -2016-06-13 22:51:42 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:51:42 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:51:42 INFO DAGScheduler:59 - waiting: Set(ResultStage 4) -2016-06-13 22:51:42 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:51:42 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List() -2016-06-13 22:51:42 INFO DAGScheduler:59 - Submitting ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397), which is now runnable -2016-06-13 22:51:42 INFO MemoryStore:59 - ensureFreeSpace(4296) called with curMem=213333, maxMem=766075207 -2016-06-13 22:51:42 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 22:51:42 INFO MemoryStore:59 - ensureFreeSpace(2206) called with curMem=217629, maxMem=766075207 -2016-06-13 22:51:42 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 22:51:42 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:54803 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 22:51:42 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:51:42 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397) -2016-06-13 22:51:42 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-06-13 22:51:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:51:42 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-06-13 22:51:42 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 22:51:42 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 22:51:42 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 1729 bytes result sent to driver -2016-06-13 22:51:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 14 ms on localhost (1/1) -2016-06-13 22:51:42 INFO DAGScheduler:59 - ResultStage 4 (collect at Word2Vec.scala:400) finished in 0.014 s -2016-06-13 22:51:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 22:51:42 INFO DAGScheduler:59 - Job 1 finished: collect at Word2Vec.scala:400, took 0.188164 s -2016-06-13 22:51:42 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-06-13 22:51:42 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 22:51:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 22:51:42 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 22:51:42 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 22:51:42 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 22:51:43 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 22:51:43 INFO BlockManager:59 - BlockManager stopped -2016-06-13 22:51:43 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 22:51:43 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 22:51:43 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 22:51:43 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 22:51:43 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 22:51:43 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 22:51:43 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-90a11c56-ca11-4bbd-b174-736baba5d9ca -2016-06-13 22:52:35 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 22:52:35 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 22:52:35 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 22:52:35 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 22:52:35 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 22:52:36 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 22:52:36 INFO Remoting:74 - Starting remoting -2016-06-13 22:52:37 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:54825] -2016-06-13 22:52:37 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54825. -2016-06-13 22:52:37 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 22:52:37 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 22:52:37 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-b56a967f-efa5-4304-9ae9-db841b7d20a4 -2016-06-13 22:52:37 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 22:52:37 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-5e5a67f7-7abf-4406-bc27-ab53c37d7745\httpd-12eb5062-3e2d-4533-b298-1159c8fd7fef -2016-06-13 22:52:37 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 22:52:37 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:52:37 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54826 -2016-06-13 22:52:37 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54826. -2016-06-13 22:52:37 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 22:52:37 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:52:37 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 22:52:37 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 22:52:37 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 22:52:37 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 22:52:37 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 22:52:37 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54845. -2016-06-13 22:52:37 INFO NettyBlockTransferService:59 - Server created on 54845 -2016-06-13 22:52:37 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:52:37 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54845 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54845) -2016-06-13 22:52:37 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:52:38 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 22:52:38 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 22:52:38 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 22:52:38 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 22:52:38 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54845 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:52:38 INFO SparkContext:59 - Created broadcast 0 from textFile at word2VecSuite.scala:54 -2016-06-13 22:52:39 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 22:52:40 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 22:52:40 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:164 -2016-06-13 22:52:40 INFO DAGScheduler:59 - Registering RDD 4 (map at Word2Vec.scala:155) -2016-06-13 22:52:40 INFO DAGScheduler:59 - Got job 0 (collect at Word2Vec.scala:164) with 2 output partitions -2016-06-13 22:52:40 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at Word2Vec.scala:164) -2016-06-13 22:52:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 22:52:40 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 22:52:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155), which has no missing parents -2016-06-13 22:52:40 INFO MemoryStore:59 - ensureFreeSpace(4376) called with curMem=171580, maxMem=766075207 -2016-06-13 22:52:40 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 22:52:40 INFO MemoryStore:59 - ensureFreeSpace(2414) called with curMem=175956, maxMem=766075207 -2016-06-13 22:52:40 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 22:52:40 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54845 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:52:40 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:52:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155) -2016-06-13 22:52:40 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 22:52:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:52:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:52:40 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 22:52:40 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 22:52:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:52:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:52:40 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 22:52:40 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 22:52:40 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 22:52:40 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 22:52:40 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 22:52:40 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver -2016-06-13 22:52:40 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver -2016-06-13 22:52:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 137 ms on localhost (1/2) -2016-06-13 22:52:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 160 ms on localhost (2/2) -2016-06-13 22:52:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 22:52:40 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at Word2Vec.scala:155) finished in 0.183 s -2016-06-13 22:52:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:52:40 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:52:40 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 22:52:40 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:52:40 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 22:52:40 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163), which is now runnable -2016-06-13 22:52:40 INFO MemoryStore:59 - ensureFreeSpace(4144) called with curMem=178370, maxMem=766075207 -2016-06-13 22:52:40 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:52:40 INFO MemoryStore:59 - ensureFreeSpace(2163) called with curMem=182514, maxMem=766075207 -2016-06-13 22:52:40 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 22:52:40 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54845 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 22:52:40 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:52:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163) -2016-06-13 22:52:40 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 22:52:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:52:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:52:40 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 22:52:40 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 22:52:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:52:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:52:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-06-13 22:52:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-06-13 22:52:40 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1686 bytes result sent to driver -2016-06-13 22:52:40 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1189 bytes result sent to driver -2016-06-13 22:52:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 47 ms on localhost (1/2) -2016-06-13 22:52:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 44 ms on localhost (2/2) -2016-06-13 22:52:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 22:52:40 INFO DAGScheduler:59 - ResultStage 1 (collect at Word2Vec.scala:164) finished in 0.049 s -2016-06-13 22:52:40 INFO DAGScheduler:59 - Job 0 finished: collect at Word2Vec.scala:164, took 0.374114 s -2016-06-13 22:52:40 INFO Word2Vec:59 - trainWordsCount = 6 -2016-06-13 22:52:40 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=184677, maxMem=766075207 -2016-06-13 22:52:40 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-06-13 22:52:40 INFO MemoryStore:59 - ensureFreeSpace(4052) called with curMem=188717, maxMem=766075207 -2016-06-13 22:52:40 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:52:40 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54845 (size: 4.0 KB, free: 730.6 MB) -2016-06-13 22:52:40 INFO SparkContext:59 - Created broadcast 3 from broadcast at Word2Vec.scala:284 -2016-06-13 22:52:40 INFO MemoryStore:59 - ensureFreeSpace(488) called with curMem=192769, maxMem=766075207 -2016-06-13 22:52:40 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 488.0 B, free 730.4 MB) -2016-06-13 22:52:40 INFO MemoryStore:59 - ensureFreeSpace(246) called with curMem=193257, maxMem=766075207 -2016-06-13 22:52:40 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 246.0 B, free 730.4 MB) -2016-06-13 22:52:40 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54845 (size: 246.0 B, free: 730.6 MB) -2016-06-13 22:52:40 INFO SparkContext:59 - Created broadcast 4 from broadcast at Word2Vec.scala:285 -2016-06-13 22:52:40 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=193503, maxMem=766075207 -2016-06-13 22:52:40 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 22:52:40 INFO MemoryStore:59 - ensureFreeSpace(172) called with curMem=193743, maxMem=766075207 -2016-06-13 22:52:40 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 172.0 B, free 730.4 MB) -2016-06-13 22:52:40 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:54845 (size: 172.0 B, free: 730.6 MB) -2016-06-13 22:52:40 INFO SparkContext:59 - Created broadcast 5 from broadcast at Word2Vec.scala:286 -2016-06-13 22:52:40 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:400 -2016-06-13 22:52:40 INFO DAGScheduler:59 - Registering RDD 9 (repartition at Word2Vec.scala:309) -2016-06-13 22:52:40 INFO DAGScheduler:59 - Registering RDD 13 (mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:52:40 INFO DAGScheduler:59 - Got job 1 (collect at Word2Vec.scala:400) with 1 output partitions -2016-06-13 22:52:40 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at Word2Vec.scala:400) -2016-06-13 22:52:40 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3) -2016-06-13 22:52:40 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 3) -2016-06-13 22:52:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309), which has no missing parents -2016-06-13 22:52:40 INFO MemoryStore:59 - ensureFreeSpace(6280) called with curMem=193915, maxMem=766075207 -2016-06-13 22:52:40 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 6.1 KB, free 730.4 MB) -2016-06-13 22:52:40 INFO MemoryStore:59 - ensureFreeSpace(3098) called with curMem=200195, maxMem=766075207 -2016-06-13 22:52:40 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 22:52:40 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:54845 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 22:52:40 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:52:40 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309) -2016-06-13 22:52:40 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 22:52:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:52:40 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:52:40 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 22:52:40 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 22:52:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:52:40 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:52:40 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2253 bytes result sent to driver -2016-06-13 22:52:40 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2253 bytes result sent to driver -2016-06-13 22:52:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 42 ms on localhost (1/2) -2016-06-13 22:52:40 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 41 ms on localhost (2/2) -2016-06-13 22:52:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 22:52:40 INFO DAGScheduler:59 - ShuffleMapStage 2 (repartition at Word2Vec.scala:309) finished in 0.045 s -2016-06-13 22:52:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:52:40 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:52:40 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 3, ResultStage 4) -2016-06-13 22:52:40 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:52:40 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List() -2016-06-13 22:52:40 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List(ShuffleMapStage 3) -2016-06-13 22:52:40 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323), which is now runnable -2016-06-13 22:52:40 INFO MemoryStore:59 - ensureFreeSpace(6576) called with curMem=203293, maxMem=766075207 -2016-06-13 22:52:40 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 6.4 KB, free 730.4 MB) -2016-06-13 22:52:40 INFO MemoryStore:59 - ensureFreeSpace(3350) called with curMem=209869, maxMem=766075207 -2016-06-13 22:52:40 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 22:52:40 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:54845 (size: 3.3 KB, free: 730.6 MB) -2016-06-13 22:52:40 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:52:40 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:52:40 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-06-13 22:52:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2159 bytes) -2016-06-13 22:52:40 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-06-13 22:52:40 INFO CacheManager:59 - Partition rdd_12_0 not found, computing it -2016-06-13 22:52:40 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:52:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 22:52:40 INFO MemoryStore:59 - ensureFreeSpace(80) called with curMem=213219, maxMem=766075207 -2016-06-13 22:52:40 INFO MemoryStore:59 - Block rdd_12_0 stored as values in memory (estimated size 80.0 B, free 730.4 MB) -2016-06-13 22:52:40 INFO BlockManagerInfo:59 - Added rdd_12_0 in memory on localhost:54845 (size: 80.0 B, free: 730.6 MB) -2016-06-13 22:52:40 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 22:52:40 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 22:52:40 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 1954 bytes result sent to driver -2016-06-13 22:52:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 56 ms on localhost (1/1) -2016-06-13 22:52:40 INFO DAGScheduler:59 - ShuffleMapStage 3 (mapPartitionsWithIndex at Word2Vec.scala:323) finished in 0.056 s -2016-06-13 22:52:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 22:52:40 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:52:40 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:52:40 INFO DAGScheduler:59 - waiting: Set(ResultStage 4) -2016-06-13 22:52:40 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:52:40 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List() -2016-06-13 22:52:40 INFO DAGScheduler:59 - Submitting ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397), which is now runnable -2016-06-13 22:52:40 INFO MemoryStore:59 - ensureFreeSpace(4296) called with curMem=213299, maxMem=766075207 -2016-06-13 22:52:40 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 22:52:40 INFO MemoryStore:59 - ensureFreeSpace(2206) called with curMem=217595, maxMem=766075207 -2016-06-13 22:52:40 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 22:52:40 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:54845 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 22:52:40 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:52:40 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397) -2016-06-13 22:52:40 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-06-13 22:52:40 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:52:40 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-06-13 22:52:40 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 22:52:40 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 22:52:40 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 1729 bytes result sent to driver -2016-06-13 22:52:40 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 14 ms on localhost (1/1) -2016-06-13 22:52:40 INFO DAGScheduler:59 - ResultStage 4 (collect at Word2Vec.scala:400) finished in 0.015 s -2016-06-13 22:52:40 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 22:52:40 INFO DAGScheduler:59 - Job 1 finished: collect at Word2Vec.scala:400, took 0.179739 s -2016-06-13 22:52:40 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-06-13 22:52:40 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 22:52:40 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 22:52:40 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 22:52:40 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 22:52:41 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 22:52:41 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 22:52:41 INFO BlockManager:59 - BlockManager stopped -2016-06-13 22:52:41 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 22:52:41 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 22:52:41 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 22:52:41 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 22:52:41 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 22:52:41 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 22:52:41 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-5e5a67f7-7abf-4406-bc27-ab53c37d7745 -2016-06-13 22:52:59 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 22:53:00 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 22:53:00 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 22:53:00 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 22:53:00 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 22:53:01 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 22:53:01 INFO Remoting:74 - Starting remoting -2016-06-13 22:53:01 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:54866] -2016-06-13 22:53:01 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54866. -2016-06-13 22:53:01 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 22:53:01 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 22:53:01 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-c1f1c60c-5595-4b3b-b735-8b6b5fc787d0 -2016-06-13 22:53:01 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 22:53:01 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-f1c16b1f-f339-41ea-a043-11338270bfb9\httpd-8c4c14fa-73e2-43eb-8bb5-3e0a883f8314 -2016-06-13 22:53:01 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 22:53:01 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:53:01 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54867 -2016-06-13 22:53:01 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54867. -2016-06-13 22:53:01 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 22:53:01 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:53:01 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 22:53:01 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 22:53:01 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 22:53:01 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 22:53:01 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 22:53:02 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54886. -2016-06-13 22:53:02 INFO NettyBlockTransferService:59 - Server created on 54886 -2016-06-13 22:53:02 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:53:02 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54886 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54886) -2016-06-13 22:53:02 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:53:03 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 22:53:03 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 22:53:03 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 22:53:03 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 22:53:03 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54886 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:53:03 INFO SparkContext:59 - Created broadcast 0 from textFile at word2VecSuite.scala:54 -2016-06-13 22:53:04 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 22:53:04 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 22:53:04 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:164 -2016-06-13 22:53:04 INFO DAGScheduler:59 - Registering RDD 4 (map at Word2Vec.scala:155) -2016-06-13 22:53:04 INFO DAGScheduler:59 - Got job 0 (collect at Word2Vec.scala:164) with 2 output partitions -2016-06-13 22:53:04 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at Word2Vec.scala:164) -2016-06-13 22:53:04 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 22:53:04 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 22:53:04 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155), which has no missing parents -2016-06-13 22:53:04 INFO MemoryStore:59 - ensureFreeSpace(4376) called with curMem=171580, maxMem=766075207 -2016-06-13 22:53:04 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 22:53:04 INFO MemoryStore:59 - ensureFreeSpace(2414) called with curMem=175956, maxMem=766075207 -2016-06-13 22:53:04 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 22:53:04 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54886 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:53:04 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:53:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155) -2016-06-13 22:53:04 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 22:53:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:53:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:53:04 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 22:53:04 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 22:53:04 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:53:04 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:53:04 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 22:53:04 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 22:53:04 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 22:53:04 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 22:53:04 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 22:53:04 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver -2016-06-13 22:53:04 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver -2016-06-13 22:53:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 129 ms on localhost (1/2) -2016-06-13 22:53:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 114 ms on localhost (2/2) -2016-06-13 22:53:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 22:53:04 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at Word2Vec.scala:155) finished in 0.145 s -2016-06-13 22:53:04 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:53:04 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:53:04 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 22:53:04 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:53:04 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 22:53:04 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163), which is now runnable -2016-06-13 22:53:04 INFO MemoryStore:59 - ensureFreeSpace(4144) called with curMem=178370, maxMem=766075207 -2016-06-13 22:53:04 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:53:04 INFO MemoryStore:59 - ensureFreeSpace(2163) called with curMem=182514, maxMem=766075207 -2016-06-13 22:53:04 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 22:53:04 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54886 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 22:53:04 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:53:04 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163) -2016-06-13 22:53:04 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 22:53:04 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:53:04 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:53:04 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 22:53:04 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 22:53:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:53:04 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:53:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 3 ms -2016-06-13 22:53:04 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 3 ms -2016-06-13 22:53:04 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1189 bytes result sent to driver -2016-06-13 22:53:04 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1686 bytes result sent to driver -2016-06-13 22:53:04 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 40 ms on localhost (1/2) -2016-06-13 22:53:04 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 42 ms on localhost (2/2) -2016-06-13 22:53:04 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 22:53:04 INFO DAGScheduler:59 - ResultStage 1 (collect at Word2Vec.scala:164) finished in 0.043 s -2016-06-13 22:53:04 INFO DAGScheduler:59 - Job 0 finished: collect at Word2Vec.scala:164, took 0.306893 s -2016-06-13 22:53:04 INFO Word2Vec:59 - trainWordsCount = 6 -2016-06-13 22:53:04 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=184677, maxMem=766075207 -2016-06-13 22:53:04 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-06-13 22:53:04 INFO MemoryStore:59 - ensureFreeSpace(4052) called with curMem=188717, maxMem=766075207 -2016-06-13 22:53:04 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:53:04 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54886 (size: 4.0 KB, free: 730.6 MB) -2016-06-13 22:53:04 INFO SparkContext:59 - Created broadcast 3 from broadcast at Word2Vec.scala:284 -2016-06-13 22:53:04 INFO MemoryStore:59 - ensureFreeSpace(488) called with curMem=192769, maxMem=766075207 -2016-06-13 22:53:04 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 488.0 B, free 730.4 MB) -2016-06-13 22:53:04 INFO MemoryStore:59 - ensureFreeSpace(246) called with curMem=193257, maxMem=766075207 -2016-06-13 22:53:04 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 246.0 B, free 730.4 MB) -2016-06-13 22:53:04 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54886 (size: 246.0 B, free: 730.6 MB) -2016-06-13 22:53:04 INFO SparkContext:59 - Created broadcast 4 from broadcast at Word2Vec.scala:285 -2016-06-13 22:53:04 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=193503, maxMem=766075207 -2016-06-13 22:53:04 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 22:53:04 INFO MemoryStore:59 - ensureFreeSpace(172) called with curMem=193743, maxMem=766075207 -2016-06-13 22:53:04 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 172.0 B, free 730.4 MB) -2016-06-13 22:53:04 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:54886 (size: 172.0 B, free: 730.6 MB) -2016-06-13 22:53:04 INFO SparkContext:59 - Created broadcast 5 from broadcast at Word2Vec.scala:286 -2016-06-13 22:53:05 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:54886 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 22:53:05 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 22:53:05 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:54886 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:53:05 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 22:53:05 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:400 -2016-06-13 22:53:05 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-06-13 22:53:05 INFO DAGScheduler:59 - Registering RDD 9 (repartition at Word2Vec.scala:309) -2016-06-13 22:53:05 INFO DAGScheduler:59 - Registering RDD 13 (mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:53:05 INFO DAGScheduler:59 - Got job 1 (collect at Word2Vec.scala:400) with 1 output partitions -2016-06-13 22:53:05 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at Word2Vec.scala:400) -2016-06-13 22:53:05 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3) -2016-06-13 22:53:05 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 3) -2016-06-13 22:53:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309), which has no missing parents -2016-06-13 22:53:05 INFO MemoryStore:59 - ensureFreeSpace(6280) called with curMem=180818, maxMem=766075207 -2016-06-13 22:53:05 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 6.1 KB, free 730.4 MB) -2016-06-13 22:53:05 INFO MemoryStore:59 - ensureFreeSpace(3098) called with curMem=187098, maxMem=766075207 -2016-06-13 22:53:05 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 22:53:05 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:54886 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 22:53:05 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:53:05 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309) -2016-06-13 22:53:05 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 22:53:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:53:05 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:53:05 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 22:53:05 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 22:53:05 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:53:05 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:53:05 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2253 bytes result sent to driver -2016-06-13 22:53:05 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2253 bytes result sent to driver -2016-06-13 22:53:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 45 ms on localhost (1/2) -2016-06-13 22:53:05 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 45 ms on localhost (2/2) -2016-06-13 22:53:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 22:53:05 INFO DAGScheduler:59 - ShuffleMapStage 2 (repartition at Word2Vec.scala:309) finished in 0.047 s -2016-06-13 22:53:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:53:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:53:05 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 3, ResultStage 4) -2016-06-13 22:53:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:53:05 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List() -2016-06-13 22:53:05 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List(ShuffleMapStage 3) -2016-06-13 22:53:05 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323), which is now runnable -2016-06-13 22:53:05 INFO MemoryStore:59 - ensureFreeSpace(6576) called with curMem=190196, maxMem=766075207 -2016-06-13 22:53:05 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 6.4 KB, free 730.4 MB) -2016-06-13 22:53:05 INFO MemoryStore:59 - ensureFreeSpace(3350) called with curMem=196772, maxMem=766075207 -2016-06-13 22:53:05 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 22:53:05 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:54886 (size: 3.3 KB, free: 730.6 MB) -2016-06-13 22:53:05 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:53:05 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:53:05 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-06-13 22:53:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2159 bytes) -2016-06-13 22:53:05 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-06-13 22:53:05 INFO CacheManager:59 - Partition rdd_12_0 not found, computing it -2016-06-13 22:53:05 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:53:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 22:53:05 INFO MemoryStore:59 - ensureFreeSpace(80) called with curMem=200122, maxMem=766075207 -2016-06-13 22:53:05 INFO MemoryStore:59 - Block rdd_12_0 stored as values in memory (estimated size 80.0 B, free 730.4 MB) -2016-06-13 22:53:05 INFO BlockManagerInfo:59 - Added rdd_12_0 in memory on localhost:54886 (size: 80.0 B, free: 730.6 MB) -2016-06-13 22:53:05 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 22:53:05 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 22:53:05 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 1954 bytes result sent to driver -2016-06-13 22:53:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 57 ms on localhost (1/1) -2016-06-13 22:53:05 INFO DAGScheduler:59 - ShuffleMapStage 3 (mapPartitionsWithIndex at Word2Vec.scala:323) finished in 0.058 s -2016-06-13 22:53:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 22:53:05 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:53:05 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:53:05 INFO DAGScheduler:59 - waiting: Set(ResultStage 4) -2016-06-13 22:53:05 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:53:05 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List() -2016-06-13 22:53:05 INFO DAGScheduler:59 - Submitting ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397), which is now runnable -2016-06-13 22:53:05 INFO MemoryStore:59 - ensureFreeSpace(4296) called with curMem=200202, maxMem=766075207 -2016-06-13 22:53:05 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 22:53:05 INFO MemoryStore:59 - ensureFreeSpace(2206) called with curMem=204498, maxMem=766075207 -2016-06-13 22:53:05 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 22:53:05 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:54886 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 22:53:05 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:53:05 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397) -2016-06-13 22:53:05 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-06-13 22:53:05 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:53:05 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-06-13 22:53:05 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 22:53:05 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 22:53:05 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 1729 bytes result sent to driver -2016-06-13 22:53:05 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 14 ms on localhost (1/1) -2016-06-13 22:53:05 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 22:53:05 INFO DAGScheduler:59 - ResultStage 4 (collect at Word2Vec.scala:400) finished in 0.015 s -2016-06-13 22:53:05 INFO DAGScheduler:59 - Job 1 finished: collect at Word2Vec.scala:400, took 0.198309 s -2016-06-13 22:53:05 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-06-13 22:53:05 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 22:53:05 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 22:53:05 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 22:53:05 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 22:53:05 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 22:53:05 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 22:53:05 INFO BlockManager:59 - BlockManager stopped -2016-06-13 22:53:05 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 22:53:05 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 22:53:05 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 22:53:05 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 22:53:05 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 22:53:05 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-f1c16b1f-f339-41ea-a043-11338270bfb9 -2016-06-13 22:53:05 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 22:53:16 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 22:53:17 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 22:53:17 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 22:53:17 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 22:53:17 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 22:53:18 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 22:53:18 INFO Remoting:74 - Starting remoting -2016-06-13 22:53:18 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:54906] -2016-06-13 22:53:18 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54906. -2016-06-13 22:53:18 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 22:53:18 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 22:53:18 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-3cda780d-cf7e-4241-bc60-eb8dc3a5bf11 -2016-06-13 22:53:18 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 22:53:18 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-b8bd4261-f023-4b5d-9258-b1f3d6d30898\httpd-74908ee0-a644-440c-97d0-f2bd6d843700 -2016-06-13 22:53:18 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 22:53:18 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:53:18 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54907 -2016-06-13 22:53:18 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54907. -2016-06-13 22:53:18 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 22:53:19 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:53:19 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 22:53:19 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 22:53:19 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 22:53:19 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 22:53:19 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 22:53:19 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54926. -2016-06-13 22:53:19 INFO NettyBlockTransferService:59 - Server created on 54926 -2016-06-13 22:53:19 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:53:19 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54926 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54926) -2016-06-13 22:53:19 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:53:20 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 22:53:20 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 22:53:20 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 22:53:20 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 22:53:20 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54926 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:53:20 INFO SparkContext:59 - Created broadcast 0 from textFile at word2VecSuite.scala:54 -2016-06-13 22:53:21 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 22:53:21 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 22:53:21 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:164 -2016-06-13 22:53:21 INFO DAGScheduler:59 - Registering RDD 4 (map at Word2Vec.scala:155) -2016-06-13 22:53:21 INFO DAGScheduler:59 - Got job 0 (collect at Word2Vec.scala:164) with 2 output partitions -2016-06-13 22:53:21 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at Word2Vec.scala:164) -2016-06-13 22:53:21 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 22:53:21 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 22:53:21 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155), which has no missing parents -2016-06-13 22:53:21 INFO MemoryStore:59 - ensureFreeSpace(4376) called with curMem=171580, maxMem=766075207 -2016-06-13 22:53:21 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 22:53:21 INFO MemoryStore:59 - ensureFreeSpace(2414) called with curMem=175956, maxMem=766075207 -2016-06-13 22:53:21 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 22:53:21 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54926 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:53:21 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:53:21 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155) -2016-06-13 22:53:21 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 22:53:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:53:21 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:53:21 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 22:53:21 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 22:53:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:53:21 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:53:21 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 22:53:21 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 22:53:21 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 22:53:21 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 22:53:21 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 22:53:21 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver -2016-06-13 22:53:21 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver -2016-06-13 22:53:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 133 ms on localhost (1/2) -2016-06-13 22:53:21 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 114 ms on localhost (2/2) -2016-06-13 22:53:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 22:53:21 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at Word2Vec.scala:155) finished in 0.153 s -2016-06-13 22:53:21 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:53:21 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:53:21 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 22:53:21 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:53:21 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 22:53:21 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163), which is now runnable -2016-06-13 22:53:21 INFO MemoryStore:59 - ensureFreeSpace(4144) called with curMem=178370, maxMem=766075207 -2016-06-13 22:53:21 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:53:21 INFO MemoryStore:59 - ensureFreeSpace(2163) called with curMem=182514, maxMem=766075207 -2016-06-13 22:53:21 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 22:53:21 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54926 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 22:53:22 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:53:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163) -2016-06-13 22:53:22 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 22:53:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:53:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:53:22 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 22:53:22 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 22:53:22 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:53:22 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:53:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-06-13 22:53:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-06-13 22:53:22 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1686 bytes result sent to driver -2016-06-13 22:53:22 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1189 bytes result sent to driver -2016-06-13 22:53:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 40 ms on localhost (1/2) -2016-06-13 22:53:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 44 ms on localhost (2/2) -2016-06-13 22:53:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 22:53:22 INFO DAGScheduler:59 - ResultStage 1 (collect at Word2Vec.scala:164) finished in 0.045 s -2016-06-13 22:53:22 INFO DAGScheduler:59 - Job 0 finished: collect at Word2Vec.scala:164, took 0.324327 s -2016-06-13 22:53:22 INFO Word2Vec:59 - trainWordsCount = 6 -2016-06-13 22:53:22 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=184677, maxMem=766075207 -2016-06-13 22:53:22 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-06-13 22:53:22 INFO MemoryStore:59 - ensureFreeSpace(4052) called with curMem=188717, maxMem=766075207 -2016-06-13 22:53:22 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:53:22 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54926 (size: 4.0 KB, free: 730.6 MB) -2016-06-13 22:53:22 INFO SparkContext:59 - Created broadcast 3 from broadcast at Word2Vec.scala:284 -2016-06-13 22:53:22 INFO MemoryStore:59 - ensureFreeSpace(488) called with curMem=192769, maxMem=766075207 -2016-06-13 22:53:22 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 488.0 B, free 730.4 MB) -2016-06-13 22:53:22 INFO MemoryStore:59 - ensureFreeSpace(246) called with curMem=193257, maxMem=766075207 -2016-06-13 22:53:22 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 246.0 B, free 730.4 MB) -2016-06-13 22:53:22 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54926 (size: 246.0 B, free: 730.6 MB) -2016-06-13 22:53:22 INFO SparkContext:59 - Created broadcast 4 from broadcast at Word2Vec.scala:285 -2016-06-13 22:53:22 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=193503, maxMem=766075207 -2016-06-13 22:53:22 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 22:53:22 INFO MemoryStore:59 - ensureFreeSpace(172) called with curMem=193743, maxMem=766075207 -2016-06-13 22:53:22 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 172.0 B, free 730.4 MB) -2016-06-13 22:53:22 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:54926 (size: 172.0 B, free: 730.6 MB) -2016-06-13 22:53:22 INFO SparkContext:59 - Created broadcast 5 from broadcast at Word2Vec.scala:286 -2016-06-13 22:53:22 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:400 -2016-06-13 22:53:22 INFO DAGScheduler:59 - Registering RDD 9 (repartition at Word2Vec.scala:309) -2016-06-13 22:53:22 INFO DAGScheduler:59 - Registering RDD 13 (mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:53:22 INFO DAGScheduler:59 - Got job 1 (collect at Word2Vec.scala:400) with 1 output partitions -2016-06-13 22:53:22 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at Word2Vec.scala:400) -2016-06-13 22:53:22 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3) -2016-06-13 22:53:22 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 3) -2016-06-13 22:53:22 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309), which has no missing parents -2016-06-13 22:53:22 INFO MemoryStore:59 - ensureFreeSpace(6280) called with curMem=193915, maxMem=766075207 -2016-06-13 22:53:22 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 6.1 KB, free 730.4 MB) -2016-06-13 22:53:22 INFO MemoryStore:59 - ensureFreeSpace(3098) called with curMem=200195, maxMem=766075207 -2016-06-13 22:53:22 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 22:53:22 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:54926 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 22:53:22 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:53:22 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309) -2016-06-13 22:53:22 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 22:53:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:53:22 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:53:22 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 22:53:22 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 22:53:22 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:53:22 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:53:22 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2253 bytes result sent to driver -2016-06-13 22:53:22 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2253 bytes result sent to driver -2016-06-13 22:53:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 45 ms on localhost (1/2) -2016-06-13 22:53:22 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 45 ms on localhost (2/2) -2016-06-13 22:53:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 22:53:22 INFO DAGScheduler:59 - ShuffleMapStage 2 (repartition at Word2Vec.scala:309) finished in 0.047 s -2016-06-13 22:53:22 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:53:22 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:53:22 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 3, ResultStage 4) -2016-06-13 22:53:22 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:53:22 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List() -2016-06-13 22:53:22 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List(ShuffleMapStage 3) -2016-06-13 22:53:22 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323), which is now runnable -2016-06-13 22:53:22 INFO MemoryStore:59 - ensureFreeSpace(6576) called with curMem=203293, maxMem=766075207 -2016-06-13 22:53:22 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 6.4 KB, free 730.4 MB) -2016-06-13 22:53:22 INFO MemoryStore:59 - ensureFreeSpace(3350) called with curMem=209869, maxMem=766075207 -2016-06-13 22:53:22 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 22:53:22 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:54926 (size: 3.3 KB, free: 730.6 MB) -2016-06-13 22:53:22 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:53:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:53:22 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-06-13 22:53:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2159 bytes) -2016-06-13 22:53:22 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-06-13 22:53:22 INFO CacheManager:59 - Partition rdd_12_0 not found, computing it -2016-06-13 22:53:22 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:53:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 22:53:22 INFO MemoryStore:59 - ensureFreeSpace(80) called with curMem=213219, maxMem=766075207 -2016-06-13 22:53:22 INFO MemoryStore:59 - Block rdd_12_0 stored as values in memory (estimated size 80.0 B, free 730.4 MB) -2016-06-13 22:53:22 INFO BlockManagerInfo:59 - Added rdd_12_0 in memory on localhost:54926 (size: 80.0 B, free: 730.6 MB) -2016-06-13 22:53:22 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 22:53:22 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 22:53:22 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 1954 bytes result sent to driver -2016-06-13 22:53:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 58 ms on localhost (1/1) -2016-06-13 22:53:22 INFO DAGScheduler:59 - ShuffleMapStage 3 (mapPartitionsWithIndex at Word2Vec.scala:323) finished in 0.059 s -2016-06-13 22:53:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 22:53:22 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:53:22 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:53:22 INFO DAGScheduler:59 - waiting: Set(ResultStage 4) -2016-06-13 22:53:22 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:53:22 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List() -2016-06-13 22:53:22 INFO DAGScheduler:59 - Submitting ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397), which is now runnable -2016-06-13 22:53:22 INFO MemoryStore:59 - ensureFreeSpace(4296) called with curMem=213299, maxMem=766075207 -2016-06-13 22:53:22 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 22:53:22 INFO MemoryStore:59 - ensureFreeSpace(2206) called with curMem=217595, maxMem=766075207 -2016-06-13 22:53:22 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 22:53:22 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:54926 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 22:53:22 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:53:22 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397) -2016-06-13 22:53:22 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-06-13 22:53:22 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:53:22 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-06-13 22:53:22 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 22:53:22 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 22:53:22 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 1729 bytes result sent to driver -2016-06-13 22:53:22 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 16 ms on localhost (1/1) -2016-06-13 22:53:22 INFO DAGScheduler:59 - ResultStage 4 (collect at Word2Vec.scala:400) finished in 0.016 s -2016-06-13 22:53:22 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 22:53:22 INFO DAGScheduler:59 - Job 1 finished: collect at Word2Vec.scala:400, took 0.201054 s -2016-06-13 22:53:22 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-06-13 22:53:22 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 22:53:22 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 22:53:22 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 22:53:22 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 22:53:22 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 22:53:22 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 22:53:22 INFO BlockManager:59 - BlockManager stopped -2016-06-13 22:53:22 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 22:53:22 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 22:53:22 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 22:53:22 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 22:53:22 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 22:53:22 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-b8bd4261-f023-4b5d-9258-b1f3d6d30898 -2016-06-13 22:53:22 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 22:53:51 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 22:53:51 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 22:53:51 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 22:53:51 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 22:53:51 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 22:53:52 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 22:53:52 INFO Remoting:74 - Starting remoting -2016-06-13 22:53:52 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:54947] -2016-06-13 22:53:52 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54947. -2016-06-13 22:53:52 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 22:53:52 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 22:53:52 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-5d8eb708-7d4e-4c20-ab5d-dd2754a46337 -2016-06-13 22:53:52 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 22:53:52 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-dc3ee1ee-9d07-4719-8703-c04177f2e890\httpd-b909e57b-8d69-4663-b4eb-d17547fb7f86 -2016-06-13 22:53:52 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 22:53:53 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:53:53 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54948 -2016-06-13 22:53:53 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54948. -2016-06-13 22:53:53 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 22:53:53 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:53:53 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 22:53:53 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 22:53:53 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 22:53:53 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 22:53:53 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 22:53:53 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 54967. -2016-06-13 22:53:53 INFO NettyBlockTransferService:59 - Server created on 54967 -2016-06-13 22:53:53 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:53:53 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:54967 with 730.6 MB RAM, BlockManagerId(driver, localhost, 54967) -2016-06-13 22:53:53 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:53:54 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 22:53:54 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 22:53:54 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 22:53:54 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 22:53:54 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:54967 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:53:54 INFO SparkContext:59 - Created broadcast 0 from textFile at word2VecSuite.scala:54 -2016-06-13 22:53:55 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 22:53:55 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 22:53:55 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:164 -2016-06-13 22:53:55 INFO DAGScheduler:59 - Registering RDD 4 (map at Word2Vec.scala:155) -2016-06-13 22:53:55 INFO DAGScheduler:59 - Got job 0 (collect at Word2Vec.scala:164) with 2 output partitions -2016-06-13 22:53:55 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at Word2Vec.scala:164) -2016-06-13 22:53:55 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 22:53:55 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 22:53:55 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155), which has no missing parents -2016-06-13 22:53:56 INFO MemoryStore:59 - ensureFreeSpace(4376) called with curMem=171580, maxMem=766075207 -2016-06-13 22:53:56 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 22:53:56 INFO MemoryStore:59 - ensureFreeSpace(2415) called with curMem=175956, maxMem=766075207 -2016-06-13 22:53:56 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 22:53:56 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:54967 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:53:56 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:53:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155) -2016-06-13 22:53:56 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 22:53:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:53:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:53:56 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 22:53:56 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 22:53:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:53:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:53:56 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 22:53:56 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 22:53:56 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 22:53:56 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 22:53:56 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 22:53:56 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver -2016-06-13 22:53:56 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver -2016-06-13 22:53:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 155 ms on localhost (1/2) -2016-06-13 22:53:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 141 ms on localhost (2/2) -2016-06-13 22:53:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 22:53:56 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at Word2Vec.scala:155) finished in 0.175 s -2016-06-13 22:53:56 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:53:56 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:53:56 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 22:53:56 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:53:56 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 22:53:56 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163), which is now runnable -2016-06-13 22:53:56 INFO MemoryStore:59 - ensureFreeSpace(4144) called with curMem=178371, maxMem=766075207 -2016-06-13 22:53:56 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:53:56 INFO MemoryStore:59 - ensureFreeSpace(2163) called with curMem=182515, maxMem=766075207 -2016-06-13 22:53:56 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 22:53:56 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:54967 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 22:53:56 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:53:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163) -2016-06-13 22:53:56 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 22:53:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:53:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:53:56 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 22:53:56 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 22:53:56 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:53:56 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:53:56 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-06-13 22:53:56 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-06-13 22:53:56 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1189 bytes result sent to driver -2016-06-13 22:53:56 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1686 bytes result sent to driver -2016-06-13 22:53:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 44 ms on localhost (1/2) -2016-06-13 22:53:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 46 ms on localhost (2/2) -2016-06-13 22:53:56 INFO DAGScheduler:59 - ResultStage 1 (collect at Word2Vec.scala:164) finished in 0.048 s -2016-06-13 22:53:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 22:53:56 INFO DAGScheduler:59 - Job 0 finished: collect at Word2Vec.scala:164, took 0.360064 s -2016-06-13 22:53:56 INFO Word2Vec:59 - trainWordsCount = 6 -2016-06-13 22:53:56 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=184678, maxMem=766075207 -2016-06-13 22:53:56 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-06-13 22:53:56 INFO MemoryStore:59 - ensureFreeSpace(4052) called with curMem=188718, maxMem=766075207 -2016-06-13 22:53:56 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:53:56 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:54967 (size: 4.0 KB, free: 730.6 MB) -2016-06-13 22:53:56 INFO SparkContext:59 - Created broadcast 3 from broadcast at Word2Vec.scala:284 -2016-06-13 22:53:56 INFO MemoryStore:59 - ensureFreeSpace(488) called with curMem=192770, maxMem=766075207 -2016-06-13 22:53:56 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 488.0 B, free 730.4 MB) -2016-06-13 22:53:56 INFO MemoryStore:59 - ensureFreeSpace(246) called with curMem=193258, maxMem=766075207 -2016-06-13 22:53:56 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 246.0 B, free 730.4 MB) -2016-06-13 22:53:56 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:54967 (size: 246.0 B, free: 730.6 MB) -2016-06-13 22:53:56 INFO SparkContext:59 - Created broadcast 4 from broadcast at Word2Vec.scala:285 -2016-06-13 22:53:56 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=193504, maxMem=766075207 -2016-06-13 22:53:56 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 22:53:56 INFO MemoryStore:59 - ensureFreeSpace(172) called with curMem=193744, maxMem=766075207 -2016-06-13 22:53:56 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 172.0 B, free 730.4 MB) -2016-06-13 22:53:56 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:54967 (size: 172.0 B, free: 730.6 MB) -2016-06-13 22:53:56 INFO SparkContext:59 - Created broadcast 5 from broadcast at Word2Vec.scala:286 -2016-06-13 22:53:56 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:400 -2016-06-13 22:53:56 INFO DAGScheduler:59 - Registering RDD 9 (repartition at Word2Vec.scala:309) -2016-06-13 22:53:56 INFO DAGScheduler:59 - Registering RDD 13 (mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:53:56 INFO DAGScheduler:59 - Got job 1 (collect at Word2Vec.scala:400) with 1 output partitions -2016-06-13 22:53:56 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at Word2Vec.scala:400) -2016-06-13 22:53:56 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3) -2016-06-13 22:53:56 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 3) -2016-06-13 22:53:56 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309), which has no missing parents -2016-06-13 22:53:56 INFO MemoryStore:59 - ensureFreeSpace(6280) called with curMem=193916, maxMem=766075207 -2016-06-13 22:53:56 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 6.1 KB, free 730.4 MB) -2016-06-13 22:53:56 INFO MemoryStore:59 - ensureFreeSpace(3098) called with curMem=200196, maxMem=766075207 -2016-06-13 22:53:56 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 22:53:56 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:54967 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 22:53:56 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:53:56 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309) -2016-06-13 22:53:56 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 22:53:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:53:56 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:53:56 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 22:53:56 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 22:53:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:53:56 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:53:56 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2253 bytes result sent to driver -2016-06-13 22:53:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 50 ms on localhost (1/2) -2016-06-13 22:53:56 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2253 bytes result sent to driver -2016-06-13 22:53:56 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 55 ms on localhost (2/2) -2016-06-13 22:53:56 INFO DAGScheduler:59 - ShuffleMapStage 2 (repartition at Word2Vec.scala:309) finished in 0.058 s -2016-06-13 22:53:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 22:53:56 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:53:56 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:53:56 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 3, ResultStage 4) -2016-06-13 22:53:56 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:53:56 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List() -2016-06-13 22:53:56 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List(ShuffleMapStage 3) -2016-06-13 22:53:56 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323), which is now runnable -2016-06-13 22:53:56 INFO MemoryStore:59 - ensureFreeSpace(6576) called with curMem=203294, maxMem=766075207 -2016-06-13 22:53:56 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 6.4 KB, free 730.4 MB) -2016-06-13 22:53:56 INFO MemoryStore:59 - ensureFreeSpace(3347) called with curMem=209870, maxMem=766075207 -2016-06-13 22:53:56 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 22:53:56 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:54967 (size: 3.3 KB, free: 730.6 MB) -2016-06-13 22:53:56 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:53:56 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:53:56 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-06-13 22:53:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2159 bytes) -2016-06-13 22:53:56 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-06-13 22:53:56 INFO CacheManager:59 - Partition rdd_12_0 not found, computing it -2016-06-13 22:53:56 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:53:56 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 22:53:56 INFO MemoryStore:59 - ensureFreeSpace(80) called with curMem=213217, maxMem=766075207 -2016-06-13 22:53:56 INFO MemoryStore:59 - Block rdd_12_0 stored as values in memory (estimated size 80.0 B, free 730.4 MB) -2016-06-13 22:53:56 INFO BlockManagerInfo:59 - Added rdd_12_0 in memory on localhost:54967 (size: 80.0 B, free: 730.6 MB) -2016-06-13 22:53:56 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 22:53:56 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 22:53:56 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 1954 bytes result sent to driver -2016-06-13 22:53:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 59 ms on localhost (1/1) -2016-06-13 22:53:56 INFO DAGScheduler:59 - ShuffleMapStage 3 (mapPartitionsWithIndex at Word2Vec.scala:323) finished in 0.061 s -2016-06-13 22:53:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 22:53:56 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:53:56 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:53:56 INFO DAGScheduler:59 - waiting: Set(ResultStage 4) -2016-06-13 22:53:56 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:53:56 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List() -2016-06-13 22:53:56 INFO DAGScheduler:59 - Submitting ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397), which is now runnable -2016-06-13 22:53:56 INFO MemoryStore:59 - ensureFreeSpace(4296) called with curMem=213297, maxMem=766075207 -2016-06-13 22:53:56 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 22:53:56 INFO MemoryStore:59 - ensureFreeSpace(2206) called with curMem=217593, maxMem=766075207 -2016-06-13 22:53:56 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 22:53:56 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:54967 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 22:53:56 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:53:56 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397) -2016-06-13 22:53:56 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-06-13 22:53:56 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:53:56 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-06-13 22:53:56 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 22:53:56 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 22:53:56 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 1729 bytes result sent to driver -2016-06-13 22:53:56 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 15 ms on localhost (1/1) -2016-06-13 22:53:56 INFO DAGScheduler:59 - ResultStage 4 (collect at Word2Vec.scala:400) finished in 0.015 s -2016-06-13 22:53:56 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 22:53:56 INFO DAGScheduler:59 - Job 1 finished: collect at Word2Vec.scala:400, took 0.222552 s -2016-06-13 22:53:56 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-06-13 22:53:56 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 22:53:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 22:53:56 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 22:53:56 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 22:53:56 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 22:53:57 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 22:53:57 INFO BlockManager:59 - BlockManager stopped -2016-06-13 22:53:57 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 22:53:57 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 22:53:57 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 22:53:57 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 22:53:57 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 22:53:57 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 22:53:57 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-dc3ee1ee-9d07-4719-8703-c04177f2e890 -2016-06-13 22:54:44 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 22:54:44 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 22:54:45 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 22:54:45 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 22:54:45 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 22:54:45 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 22:54:45 INFO Remoting:74 - Starting remoting -2016-06-13 22:54:46 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:54987] -2016-06-13 22:54:46 INFO Utils:59 - Successfully started service 'sparkDriver' on port 54987. -2016-06-13 22:54:46 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 22:54:46 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 22:54:46 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-62fb8f1c-344b-4f59-b5c2-e9741bb21bb8 -2016-06-13 22:54:46 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 22:54:46 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-7a427465-39cf-467e-b330-25f4dca44b70\httpd-ce4acf1e-9f28-4889-9e26-70c5d8b05195 -2016-06-13 22:54:46 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 22:54:46 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:54:46 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:54988 -2016-06-13 22:54:46 INFO Utils:59 - Successfully started service 'HTTP file server' on port 54988. -2016-06-13 22:54:46 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 22:54:46 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:54:46 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 22:54:46 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 22:54:46 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 22:54:46 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 22:54:46 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 22:54:46 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55007. -2016-06-13 22:54:46 INFO NettyBlockTransferService:59 - Server created on 55007 -2016-06-13 22:54:46 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:54:46 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55007 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55007) -2016-06-13 22:54:46 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:54:47 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 22:54:47 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 22:54:47 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 22:54:47 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 22:54:47 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55007 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:54:47 INFO SparkContext:59 - Created broadcast 0 from textFile at word2VecSuite.scala:54 -2016-06-13 22:54:48 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 22:54:49 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 22:54:49 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:164 -2016-06-13 22:54:49 INFO DAGScheduler:59 - Registering RDD 4 (map at Word2Vec.scala:155) -2016-06-13 22:54:49 INFO DAGScheduler:59 - Got job 0 (collect at Word2Vec.scala:164) with 2 output partitions -2016-06-13 22:54:49 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at Word2Vec.scala:164) -2016-06-13 22:54:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 22:54:49 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 22:54:49 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155), which has no missing parents -2016-06-13 22:54:49 INFO MemoryStore:59 - ensureFreeSpace(4376) called with curMem=171580, maxMem=766075207 -2016-06-13 22:54:49 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 22:54:49 INFO MemoryStore:59 - ensureFreeSpace(2414) called with curMem=175956, maxMem=766075207 -2016-06-13 22:54:49 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 22:54:49 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55007 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:54:49 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:54:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155) -2016-06-13 22:54:49 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 22:54:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:54:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:54:49 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 22:54:49 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 22:54:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:54:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:54:49 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 22:54:49 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 22:54:49 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 22:54:49 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 22:54:49 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 22:54:49 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver -2016-06-13 22:54:49 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver -2016-06-13 22:54:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 328 ms on localhost (1/2) -2016-06-13 22:54:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 360 ms on localhost (2/2) -2016-06-13 22:54:49 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at Word2Vec.scala:155) finished in 0.378 s -2016-06-13 22:54:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 22:54:49 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:54:49 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:54:49 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 22:54:49 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:54:49 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 22:54:49 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163), which is now runnable -2016-06-13 22:54:49 INFO MemoryStore:59 - ensureFreeSpace(4144) called with curMem=178370, maxMem=766075207 -2016-06-13 22:54:49 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:54:49 INFO MemoryStore:59 - ensureFreeSpace(2162) called with curMem=182514, maxMem=766075207 -2016-06-13 22:54:49 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 22:54:49 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55007 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 22:54:49 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:54:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163) -2016-06-13 22:54:49 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 22:54:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:54:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:54:49 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 22:54:49 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 22:54:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:54:49 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:54:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-06-13 22:54:49 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-06-13 22:54:49 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1189 bytes result sent to driver -2016-06-13 22:54:49 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1686 bytes result sent to driver -2016-06-13 22:54:49 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 39 ms on localhost (1/2) -2016-06-13 22:54:49 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 44 ms on localhost (2/2) -2016-06-13 22:54:49 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 22:54:49 INFO DAGScheduler:59 - ResultStage 1 (collect at Word2Vec.scala:164) finished in 0.045 s -2016-06-13 22:54:49 INFO DAGScheduler:59 - Job 0 finished: collect at Word2Vec.scala:164, took 0.553095 s -2016-06-13 22:54:49 INFO Word2Vec:59 - trainWordsCount = 6 -2016-06-13 22:54:49 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=184676, maxMem=766075207 -2016-06-13 22:54:49 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-06-13 22:54:49 INFO MemoryStore:59 - ensureFreeSpace(4052) called with curMem=188716, maxMem=766075207 -2016-06-13 22:54:49 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:54:49 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55007 (size: 4.0 KB, free: 730.6 MB) -2016-06-13 22:54:49 INFO SparkContext:59 - Created broadcast 3 from broadcast at Word2Vec.scala:284 -2016-06-13 22:54:49 INFO MemoryStore:59 - ensureFreeSpace(488) called with curMem=192768, maxMem=766075207 -2016-06-13 22:54:49 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 488.0 B, free 730.4 MB) -2016-06-13 22:54:49 INFO MemoryStore:59 - ensureFreeSpace(246) called with curMem=193256, maxMem=766075207 -2016-06-13 22:54:49 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 246.0 B, free 730.4 MB) -2016-06-13 22:54:49 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55007 (size: 246.0 B, free: 730.6 MB) -2016-06-13 22:54:49 INFO SparkContext:59 - Created broadcast 4 from broadcast at Word2Vec.scala:285 -2016-06-13 22:54:49 INFO MemoryStore:59 - ensureFreeSpace(240) called with curMem=193502, maxMem=766075207 -2016-06-13 22:54:49 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 240.0 B, free 730.4 MB) -2016-06-13 22:54:49 INFO MemoryStore:59 - ensureFreeSpace(172) called with curMem=193742, maxMem=766075207 -2016-06-13 22:54:49 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 172.0 B, free 730.4 MB) -2016-06-13 22:54:49 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55007 (size: 172.0 B, free: 730.6 MB) -2016-06-13 22:54:49 INFO SparkContext:59 - Created broadcast 5 from broadcast at Word2Vec.scala:286 -2016-06-13 22:54:49 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:400 -2016-06-13 22:54:49 INFO DAGScheduler:59 - Registering RDD 9 (repartition at Word2Vec.scala:309) -2016-06-13 22:54:49 INFO DAGScheduler:59 - Registering RDD 13 (mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:54:49 INFO DAGScheduler:59 - Got job 1 (collect at Word2Vec.scala:400) with 1 output partitions -2016-06-13 22:54:49 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at Word2Vec.scala:400) -2016-06-13 22:54:49 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3) -2016-06-13 22:54:49 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 3) -2016-06-13 22:54:49 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309), which has no missing parents -2016-06-13 22:54:49 INFO MemoryStore:59 - ensureFreeSpace(6280) called with curMem=193914, maxMem=766075207 -2016-06-13 22:54:49 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 6.1 KB, free 730.4 MB) -2016-06-13 22:54:49 INFO MemoryStore:59 - ensureFreeSpace(3100) called with curMem=200194, maxMem=766075207 -2016-06-13 22:54:49 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.0 KB, free 730.4 MB) -2016-06-13 22:54:49 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55007 (size: 3.0 KB, free: 730.6 MB) -2016-06-13 22:54:49 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:54:49 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309) -2016-06-13 22:54:49 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 22:54:49 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:54:49 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2192 bytes) -2016-06-13 22:54:49 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 22:54:49 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 22:54:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:52+52 -2016-06-13 22:54:49 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/a.txt:0+52 -2016-06-13 22:54:50 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2253 bytes result sent to driver -2016-06-13 22:54:50 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2253 bytes result sent to driver -2016-06-13 22:54:50 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 46 ms on localhost (1/2) -2016-06-13 22:54:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 49 ms on localhost (2/2) -2016-06-13 22:54:50 INFO DAGScheduler:59 - ShuffleMapStage 2 (repartition at Word2Vec.scala:309) finished in 0.050 s -2016-06-13 22:54:50 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:54:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 22:54:50 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:54:50 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 3, ResultStage 4) -2016-06-13 22:54:50 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:54:50 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List() -2016-06-13 22:54:50 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List(ShuffleMapStage 3) -2016-06-13 22:54:50 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323), which is now runnable -2016-06-13 22:54:50 INFO MemoryStore:59 - ensureFreeSpace(6576) called with curMem=203294, maxMem=766075207 -2016-06-13 22:54:50 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 6.4 KB, free 730.4 MB) -2016-06-13 22:54:50 INFO MemoryStore:59 - ensureFreeSpace(3351) called with curMem=209870, maxMem=766075207 -2016-06-13 22:54:50 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.4 MB) -2016-06-13 22:54:50 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55007 (size: 3.3 KB, free: 730.6 MB) -2016-06-13 22:54:50 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:54:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:54:50 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-06-13 22:54:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2159 bytes) -2016-06-13 22:54:50 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-06-13 22:54:50 INFO CacheManager:59 - Partition rdd_12_0 not found, computing it -2016-06-13 22:54:50 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:54:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 22:54:50 INFO MemoryStore:59 - ensureFreeSpace(80) called with curMem=213221, maxMem=766075207 -2016-06-13 22:54:50 INFO MemoryStore:59 - Block rdd_12_0 stored as values in memory (estimated size 80.0 B, free 730.4 MB) -2016-06-13 22:54:50 INFO BlockManagerInfo:59 - Added rdd_12_0 in memory on localhost:55007 (size: 80.0 B, free: 730.6 MB) -2016-06-13 22:54:50 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 22:54:50 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 22:54:50 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 1954 bytes result sent to driver -2016-06-13 22:54:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 54 ms on localhost (1/1) -2016-06-13 22:54:50 INFO DAGScheduler:59 - ShuffleMapStage 3 (mapPartitionsWithIndex at Word2Vec.scala:323) finished in 0.054 s -2016-06-13 22:54:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 22:54:50 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:54:50 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:54:50 INFO DAGScheduler:59 - waiting: Set(ResultStage 4) -2016-06-13 22:54:50 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:54:50 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List() -2016-06-13 22:54:50 INFO DAGScheduler:59 - Submitting ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397), which is now runnable -2016-06-13 22:54:50 INFO MemoryStore:59 - ensureFreeSpace(4296) called with curMem=213301, maxMem=766075207 -2016-06-13 22:54:50 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.2 KB, free 730.4 MB) -2016-06-13 22:54:50 INFO MemoryStore:59 - ensureFreeSpace(2208) called with curMem=217597, maxMem=766075207 -2016-06-13 22:54:50 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 22:54:50 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55007 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 22:54:50 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:54:50 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397) -2016-06-13 22:54:50 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-06-13 22:54:50 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:54:50 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-06-13 22:54:50 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 22:54:50 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 22:54:50 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 1729 bytes result sent to driver -2016-06-13 22:54:50 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 15 ms on localhost (1/1) -2016-06-13 22:54:50 INFO DAGScheduler:59 - ResultStage 4 (collect at Word2Vec.scala:400) finished in 0.016 s -2016-06-13 22:54:50 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 22:54:50 INFO DAGScheduler:59 - Job 1 finished: collect at Word2Vec.scala:400, took 0.184410 s -2016-06-13 22:54:50 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-06-13 22:54:50 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 22:54:50 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 22:54:50 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 22:54:50 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 22:54:50 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 22:54:50 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 22:54:50 INFO BlockManager:59 - BlockManager stopped -2016-06-13 22:54:50 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 22:54:50 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 22:54:50 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 22:54:50 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 22:54:50 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 22:54:50 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 22:54:50 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-7a427465-39cf-467e-b330-25f4dca44b70 -2016-06-13 22:56:11 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 22:56:12 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 22:56:12 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 22:56:12 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 22:56:12 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 22:56:13 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 22:56:13 INFO Remoting:74 - Starting remoting -2016-06-13 22:56:13 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:55028] -2016-06-13 22:56:13 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55028. -2016-06-13 22:56:13 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 22:56:13 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 22:56:13 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-c8c0f6be-d5ce-4c37-a845-a9f9f24df810 -2016-06-13 22:56:13 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 22:56:13 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-6ed88428-f0f2-4e7d-9551-9b777db76799\httpd-9f99fef0-bda0-40b1-8a45-2078e60f51f0 -2016-06-13 22:56:13 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 22:56:13 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:56:13 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55029 -2016-06-13 22:56:13 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55029. -2016-06-13 22:56:13 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 22:56:13 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:56:13 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 22:56:13 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 22:56:13 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 22:56:14 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 22:56:14 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 22:56:14 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55048. -2016-06-13 22:56:14 INFO NettyBlockTransferService:59 - Server created on 55048 -2016-06-13 22:56:14 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:56:14 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55048 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55048) -2016-06-13 22:56:14 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:56:15 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 22:56:15 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 22:56:15 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 22:56:15 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 22:56:15 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55048 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:56:15 INFO SparkContext:59 - Created broadcast 0 from textFile at word2VecSuite.scala:54 -2016-06-13 22:56:16 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 22:56:16 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 22:56:16 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:164 -2016-06-13 22:56:16 INFO DAGScheduler:59 - Registering RDD 4 (map at Word2Vec.scala:155) -2016-06-13 22:56:16 INFO DAGScheduler:59 - Got job 0 (collect at Word2Vec.scala:164) with 2 output partitions -2016-06-13 22:56:16 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at Word2Vec.scala:164) -2016-06-13 22:56:16 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 22:56:16 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 22:56:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155), which has no missing parents -2016-06-13 22:56:16 INFO MemoryStore:59 - ensureFreeSpace(4384) called with curMem=171580, maxMem=766075207 -2016-06-13 22:56:16 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 22:56:16 INFO MemoryStore:59 - ensureFreeSpace(2424) called with curMem=175964, maxMem=766075207 -2016-06-13 22:56:16 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 22:56:16 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55048 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:56:16 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:56:16 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155) -2016-06-13 22:56:16 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 22:56:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-06-13 22:56:16 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-06-13 22:56:16 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 22:56:16 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 22:56:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/aWord2vec.txt:0+58 -2016-06-13 22:56:16 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/aWord2vec.txt:58+58 -2016-06-13 22:56:16 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 22:56:16 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 22:56:16 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 22:56:16 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 22:56:16 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 22:56:17 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver -2016-06-13 22:56:17 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver -2016-06-13 22:56:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 129 ms on localhost (1/2) -2016-06-13 22:56:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 114 ms on localhost (2/2) -2016-06-13 22:56:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 22:56:17 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at Word2Vec.scala:155) finished in 0.143 s -2016-06-13 22:56:17 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:56:17 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:56:17 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 22:56:17 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:56:17 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 22:56:17 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163), which is now runnable -2016-06-13 22:56:17 INFO MemoryStore:59 - ensureFreeSpace(4144) called with curMem=178388, maxMem=766075207 -2016-06-13 22:56:17 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:56:17 INFO MemoryStore:59 - ensureFreeSpace(2163) called with curMem=182532, maxMem=766075207 -2016-06-13 22:56:17 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 22:56:17 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55048 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 22:56:17 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:56:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163) -2016-06-13 22:56:17 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 22:56:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:56:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:56:17 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 22:56:17 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 22:56:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:56:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:56:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-06-13 22:56:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 4 ms -2016-06-13 22:56:17 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1686 bytes result sent to driver -2016-06-13 22:56:17 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1686 bytes result sent to driver -2016-06-13 22:56:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 40 ms on localhost (1/2) -2016-06-13 22:56:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 43 ms on localhost (2/2) -2016-06-13 22:56:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 22:56:17 INFO DAGScheduler:59 - ResultStage 1 (collect at Word2Vec.scala:164) finished in 0.045 s -2016-06-13 22:56:17 INFO DAGScheduler:59 - Job 0 finished: collect at Word2Vec.scala:164, took 0.296443 s -2016-06-13 22:56:17 INFO Word2Vec:59 - trainWordsCount = 11 -2016-06-13 22:56:17 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=184695, maxMem=766075207 -2016-06-13 22:56:17 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-06-13 22:56:17 INFO MemoryStore:59 - ensureFreeSpace(4052) called with curMem=188735, maxMem=766075207 -2016-06-13 22:56:17 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:56:17 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55048 (size: 4.0 KB, free: 730.6 MB) -2016-06-13 22:56:17 INFO SparkContext:59 - Created broadcast 3 from broadcast at Word2Vec.scala:284 -2016-06-13 22:56:17 INFO MemoryStore:59 - ensureFreeSpace(928) called with curMem=192787, maxMem=766075207 -2016-06-13 22:56:17 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 928.0 B, free 730.4 MB) -2016-06-13 22:56:17 INFO MemoryStore:59 - ensureFreeSpace(299) called with curMem=193715, maxMem=766075207 -2016-06-13 22:56:17 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 299.0 B, free 730.4 MB) -2016-06-13 22:56:17 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55048 (size: 299.0 B, free: 730.6 MB) -2016-06-13 22:56:17 INFO SparkContext:59 - Created broadcast 4 from broadcast at Word2Vec.scala:285 -2016-06-13 22:56:17 INFO MemoryStore:59 - ensureFreeSpace(336) called with curMem=194014, maxMem=766075207 -2016-06-13 22:56:17 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 336.0 B, free 730.4 MB) -2016-06-13 22:56:17 INFO MemoryStore:59 - ensureFreeSpace(189) called with curMem=194350, maxMem=766075207 -2016-06-13 22:56:17 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 189.0 B, free 730.4 MB) -2016-06-13 22:56:17 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55048 (size: 189.0 B, free: 730.6 MB) -2016-06-13 22:56:17 INFO SparkContext:59 - Created broadcast 5 from broadcast at Word2Vec.scala:286 -2016-06-13 22:56:17 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:55048 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 22:56:17 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 22:56:17 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:55048 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:56:17 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 22:56:17 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-06-13 22:56:17 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:400 -2016-06-13 22:56:17 INFO DAGScheduler:59 - Registering RDD 9 (repartition at Word2Vec.scala:309) -2016-06-13 22:56:17 INFO DAGScheduler:59 - Registering RDD 13 (mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:56:17 INFO DAGScheduler:59 - Got job 1 (collect at Word2Vec.scala:400) with 1 output partitions -2016-06-13 22:56:17 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at Word2Vec.scala:400) -2016-06-13 22:56:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3) -2016-06-13 22:56:17 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 3) -2016-06-13 22:56:17 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309), which has no missing parents -2016-06-13 22:56:17 INFO MemoryStore:59 - ensureFreeSpace(6664) called with curMem=181424, maxMem=766075207 -2016-06-13 22:56:17 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 6.5 KB, free 730.4 MB) -2016-06-13 22:56:17 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=188088, maxMem=766075207 -2016-06-13 22:56:17 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.4 MB) -2016-06-13 22:56:17 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55048 (size: 3.1 KB, free: 730.6 MB) -2016-06-13 22:56:17 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:56:17 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309) -2016-06-13 22:56:17 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 22:56:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-06-13 22:56:17 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2200 bytes) -2016-06-13 22:56:17 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 22:56:17 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 22:56:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/aWord2vec.txt:0+58 -2016-06-13 22:56:17 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/aWord2vec.txt:58+58 -2016-06-13 22:56:17 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2253 bytes result sent to driver -2016-06-13 22:56:17 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2253 bytes result sent to driver -2016-06-13 22:56:17 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 37 ms on localhost (1/2) -2016-06-13 22:56:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 41 ms on localhost (2/2) -2016-06-13 22:56:17 INFO DAGScheduler:59 - ShuffleMapStage 2 (repartition at Word2Vec.scala:309) finished in 0.041 s -2016-06-13 22:56:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 22:56:17 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:56:17 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:56:17 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 3, ResultStage 4) -2016-06-13 22:56:17 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:56:17 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List() -2016-06-13 22:56:17 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List(ShuffleMapStage 3) -2016-06-13 22:56:17 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323), which is now runnable -2016-06-13 22:56:17 INFO MemoryStore:59 - ensureFreeSpace(7752) called with curMem=191252, maxMem=766075207 -2016-06-13 22:56:17 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 7.6 KB, free 730.4 MB) -2016-06-13 22:56:17 INFO MemoryStore:59 - ensureFreeSpace(3829) called with curMem=199004, maxMem=766075207 -2016-06-13 22:56:17 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.7 KB, free 730.4 MB) -2016-06-13 22:56:17 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55048 (size: 3.7 KB, free: 730.6 MB) -2016-06-13 22:56:17 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:56:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:56:17 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-06-13 22:56:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2159 bytes) -2016-06-13 22:56:17 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-06-13 22:56:17 INFO CacheManager:59 - Partition rdd_12_0 not found, computing it -2016-06-13 22:56:17 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:56:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 22:56:17 INFO MemoryStore:59 - ensureFreeSpace(104) called with curMem=202833, maxMem=766075207 -2016-06-13 22:56:17 INFO MemoryStore:59 - Block rdd_12_0 stored as values in memory (estimated size 104.0 B, free 730.4 MB) -2016-06-13 22:56:17 INFO BlockManagerInfo:59 - Added rdd_12_0 in memory on localhost:55048 (size: 104.0 B, free: 730.6 MB) -2016-06-13 22:56:17 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 22:56:17 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 22:56:17 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 1954 bytes result sent to driver -2016-06-13 22:56:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 58 ms on localhost (1/1) -2016-06-13 22:56:17 INFO DAGScheduler:59 - ShuffleMapStage 3 (mapPartitionsWithIndex at Word2Vec.scala:323) finished in 0.058 s -2016-06-13 22:56:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 22:56:17 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:56:17 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:56:17 INFO DAGScheduler:59 - waiting: Set(ResultStage 4) -2016-06-13 22:56:17 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:56:17 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List() -2016-06-13 22:56:17 INFO DAGScheduler:59 - Submitting ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397), which is now runnable -2016-06-13 22:56:17 INFO MemoryStore:59 - ensureFreeSpace(4672) called with curMem=202937, maxMem=766075207 -2016-06-13 22:56:17 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.6 KB, free 730.4 MB) -2016-06-13 22:56:17 INFO MemoryStore:59 - ensureFreeSpace(2262) called with curMem=207609, maxMem=766075207 -2016-06-13 22:56:17 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 22:56:17 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55048 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 22:56:17 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:56:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397) -2016-06-13 22:56:17 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-06-13 22:56:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:56:17 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-06-13 22:56:17 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 22:56:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-06-13 22:56:17 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2586 bytes result sent to driver -2016-06-13 22:56:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 15 ms on localhost (1/1) -2016-06-13 22:56:17 INFO DAGScheduler:59 - ResultStage 4 (collect at Word2Vec.scala:400) finished in 0.017 s -2016-06-13 22:56:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 22:56:17 INFO DAGScheduler:59 - Job 1 finished: collect at Word2Vec.scala:400, took 0.182537 s -2016-06-13 22:56:17 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-06-13 22:56:17 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 22:56:17 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 22:56:17 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 22:56:17 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 22:56:17 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 22:56:17 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 22:56:17 INFO BlockManager:59 - BlockManager stopped -2016-06-13 22:56:17 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 22:56:17 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 22:56:17 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 22:56:17 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 22:56:17 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 22:56:17 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 22:56:17 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-6ed88428-f0f2-4e7d-9551-9b777db76799 -2016-06-13 22:57:41 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-06-13 22:57:41 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-06-13 22:57:41 INFO SecurityManager:59 - Changing view acls to: xubo -2016-06-13 22:57:41 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-06-13 22:57:41 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-06-13 22:57:42 INFO Slf4jLogger:80 - Slf4jLogger started -2016-06-13 22:57:42 INFO Remoting:74 - Starting remoting -2016-06-13 22:57:42 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@202.38.84.85:55074] -2016-06-13 22:57:42 INFO Utils:59 - Successfully started service 'sparkDriver' on port 55074. -2016-06-13 22:57:42 INFO SparkEnv:59 - Registering MapOutputTracker -2016-06-13 22:57:42 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-06-13 22:57:42 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-bbac4a60-71cd-4671-882e-06745ba42653 -2016-06-13 22:57:42 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-06-13 22:57:42 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-7015a1e4-0e59-4cd4-9822-ed967aaa0256\httpd-2238c616-3a18-4575-a40a-a617fb204f86 -2016-06-13 22:57:42 INFO HttpServer:59 - Starting HTTP Server -2016-06-13 22:57:42 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:57:42 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:55075 -2016-06-13 22:57:42 INFO Utils:59 - Successfully started service 'HTTP file server' on port 55075. -2016-06-13 22:57:43 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-06-13 22:57:43 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-06-13 22:57:43 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-06-13 22:57:43 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-06-13 22:57:43 INFO SparkUI:59 - Started SparkUI at http://202.38.84.85:4040 -2016-06-13 22:57:43 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-06-13 22:57:43 INFO Executor:59 - Starting executor ID driver on host localhost -2016-06-13 22:57:43 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 55094. -2016-06-13 22:57:43 INFO NettyBlockTransferService:59 - Server created on 55094 -2016-06-13 22:57:43 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-06-13 22:57:43 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:55094 with 730.6 MB RAM, BlockManagerId(driver, localhost, 55094) -2016-06-13 22:57:43 INFO BlockManagerMaster:59 - Registered BlockManager -2016-06-13 22:57:44 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-06-13 22:57:44 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-06-13 22:57:44 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-06-13 22:57:44 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-06-13 22:57:44 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:55094 (size: 13.9 KB, free: 730.6 MB) -2016-06-13 22:57:44 INFO SparkContext:59 - Created broadcast 0 from textFile at word2VecSuite.scala:52 -2016-06-13 22:57:45 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:482:722f:5976:ce1f%39, but we couldn't find any external IP address! -2016-06-13 22:57:45 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-06-13 22:57:45 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:164 -2016-06-13 22:57:45 INFO DAGScheduler:59 - Registering RDD 4 (map at Word2Vec.scala:155) -2016-06-13 22:57:45 INFO DAGScheduler:59 - Got job 0 (collect at Word2Vec.scala:164) with 2 output partitions -2016-06-13 22:57:45 INFO DAGScheduler:59 - Final stage: ResultStage 1(collect at Word2Vec.scala:164) -2016-06-13 22:57:45 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 0) -2016-06-13 22:57:45 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 0) -2016-06-13 22:57:45 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155), which has no missing parents -2016-06-13 22:57:45 INFO MemoryStore:59 - ensureFreeSpace(4384) called with curMem=171580, maxMem=766075207 -2016-06-13 22:57:45 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.3 KB, free 730.4 MB) -2016-06-13 22:57:45 INFO MemoryStore:59 - ensureFreeSpace(2424) called with curMem=175964, maxMem=766075207 -2016-06-13 22:57:45 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.4 KB, free 730.4 MB) -2016-06-13 22:57:45 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:55094 (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:57:45 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:57:45 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[4] at map at Word2Vec.scala:155) -2016-06-13 22:57:45 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 2 tasks -2016-06-13 22:57:45 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2200 bytes) -2016-06-13 22:57:45 INFO TaskSetManager:59 - Starting task 1.0 in stage 0.0 (TID 1, localhost, PROCESS_LOCAL, 2200 bytes) -2016-06-13 22:57:45 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-06-13 22:57:45 INFO Executor:59 - Running task 1.0 in stage 0.0 (TID 1) -2016-06-13 22:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/aWord2vec.txt:0+58 -2016-06-13 22:57:45 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/aWord2vec.txt:58+58 -2016-06-13 22:57:45 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-06-13 22:57:45 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-06-13 22:57:45 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-06-13 22:57:45 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-06-13 22:57:45 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-06-13 22:57:46 INFO Executor:59 - Finished task 1.0 in stage 0.0 (TID 1). 2254 bytes result sent to driver -2016-06-13 22:57:46 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2254 bytes result sent to driver -2016-06-13 22:57:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 0.0 (TID 1) in 112 ms on localhost (1/2) -2016-06-13 22:57:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 130 ms on localhost (2/2) -2016-06-13 22:57:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-06-13 22:57:46 INFO DAGScheduler:59 - ShuffleMapStage 0 (map at Word2Vec.scala:155) finished in 0.145 s -2016-06-13 22:57:46 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:57:46 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:57:46 INFO DAGScheduler:59 - waiting: Set(ResultStage 1) -2016-06-13 22:57:46 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:57:46 INFO DAGScheduler:59 - Missing parents for ResultStage 1: List() -2016-06-13 22:57:46 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163), which is now runnable -2016-06-13 22:57:46 INFO MemoryStore:59 - ensureFreeSpace(4144) called with curMem=178388, maxMem=766075207 -2016-06-13 22:57:46 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:57:46 INFO MemoryStore:59 - ensureFreeSpace(2163) called with curMem=182532, maxMem=766075207 -2016-06-13 22:57:46 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.4 MB) -2016-06-13 22:57:46 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:55094 (size: 2.1 KB, free: 730.6 MB) -2016-06-13 22:57:46 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:57:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at filter at Word2Vec.scala:163) -2016-06-13 22:57:46 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 2 tasks -2016-06-13 22:57:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:57:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 1.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:57:46 INFO Executor:59 - Running task 1.0 in stage 1.0 (TID 3) -2016-06-13 22:57:46 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 2) -2016-06-13 22:57:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:57:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:57:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-06-13 22:57:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 5 ms -2016-06-13 22:57:46 INFO Executor:59 - Finished task 1.0 in stage 1.0 (TID 3). 1686 bytes result sent to driver -2016-06-13 22:57:46 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 2). 1686 bytes result sent to driver -2016-06-13 22:57:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 1.0 (TID 3) in 42 ms on localhost (1/2) -2016-06-13 22:57:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 2) in 47 ms on localhost (2/2) -2016-06-13 22:57:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-06-13 22:57:46 INFO DAGScheduler:59 - ResultStage 1 (collect at Word2Vec.scala:164) finished in 0.048 s -2016-06-13 22:57:46 INFO DAGScheduler:59 - Job 0 finished: collect at Word2Vec.scala:164, took 0.307325 s -2016-06-13 22:57:46 INFO Word2Vec:59 - trainWordsCount = 11 -2016-06-13 22:57:46 INFO MemoryStore:59 - ensureFreeSpace(4040) called with curMem=184695, maxMem=766075207 -2016-06-13 22:57:46 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 3.9 KB, free 730.4 MB) -2016-06-13 22:57:46 INFO MemoryStore:59 - ensureFreeSpace(4052) called with curMem=188735, maxMem=766075207 -2016-06-13 22:57:46 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 4.0 KB, free 730.4 MB) -2016-06-13 22:57:46 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:55094 (size: 4.0 KB, free: 730.6 MB) -2016-06-13 22:57:46 INFO SparkContext:59 - Created broadcast 3 from broadcast at Word2Vec.scala:284 -2016-06-13 22:57:46 INFO MemoryStore:59 - ensureFreeSpace(928) called with curMem=192787, maxMem=766075207 -2016-06-13 22:57:46 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 928.0 B, free 730.4 MB) -2016-06-13 22:57:46 INFO MemoryStore:59 - ensureFreeSpace(299) called with curMem=193715, maxMem=766075207 -2016-06-13 22:57:46 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 299.0 B, free 730.4 MB) -2016-06-13 22:57:46 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:55094 (size: 299.0 B, free: 730.6 MB) -2016-06-13 22:57:46 INFO SparkContext:59 - Created broadcast 4 from broadcast at Word2Vec.scala:285 -2016-06-13 22:57:46 INFO MemoryStore:59 - ensureFreeSpace(336) called with curMem=194014, maxMem=766075207 -2016-06-13 22:57:46 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 336.0 B, free 730.4 MB) -2016-06-13 22:57:46 INFO MemoryStore:59 - ensureFreeSpace(189) called with curMem=194350, maxMem=766075207 -2016-06-13 22:57:46 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 189.0 B, free 730.4 MB) -2016-06-13 22:57:46 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:55094 (size: 189.0 B, free: 730.6 MB) -2016-06-13 22:57:46 INFO SparkContext:59 - Created broadcast 5 from broadcast at Word2Vec.scala:286 -2016-06-13 22:57:46 INFO SparkContext:59 - Starting job: collect at Word2Vec.scala:400 -2016-06-13 22:57:46 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:55094 in memory (size: 2.1 KB, free: 730.6 MB) -2016-06-13 22:57:46 INFO DAGScheduler:59 - Registering RDD 9 (repartition at Word2Vec.scala:309) -2016-06-13 22:57:46 INFO DAGScheduler:59 - Registering RDD 13 (mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:57:46 INFO DAGScheduler:59 - Got job 1 (collect at Word2Vec.scala:400) with 1 output partitions -2016-06-13 22:57:46 INFO DAGScheduler:59 - Final stage: ResultStage 4(collect at Word2Vec.scala:400) -2016-06-13 22:57:46 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 3) -2016-06-13 22:57:46 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 3) -2016-06-13 22:57:46 INFO DAGScheduler:59 - Submitting ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309), which has no missing parents -2016-06-13 22:57:46 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-06-13 22:57:46 INFO MemoryStore:59 - ensureFreeSpace(6664) called with curMem=185808, maxMem=766075207 -2016-06-13 22:57:46 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 6.5 KB, free 730.4 MB) -2016-06-13 22:57:46 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:55094 in memory (size: 2.4 KB, free: 730.6 MB) -2016-06-13 22:57:46 INFO MemoryStore:59 - ensureFreeSpace(3164) called with curMem=188088, maxMem=766075207 -2016-06-13 22:57:46 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.1 KB, free 730.4 MB) -2016-06-13 22:57:46 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-06-13 22:57:46 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:55094 (size: 3.1 KB, free: 730.6 MB) -2016-06-13 22:57:46 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:57:46 INFO DAGScheduler:59 - Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[9] at repartition at Word2Vec.scala:309) -2016-06-13 22:57:46 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 2 tasks -2016-06-13 22:57:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 4, localhost, PROCESS_LOCAL, 2200 bytes) -2016-06-13 22:57:46 INFO ContextCleaner:59 - Cleaned shuffle 0 -2016-06-13 22:57:46 INFO TaskSetManager:59 - Starting task 1.0 in stage 2.0 (TID 5, localhost, PROCESS_LOCAL, 2200 bytes) -2016-06-13 22:57:46 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 4) -2016-06-13 22:57:46 INFO Executor:59 - Running task 1.0 in stage 2.0 (TID 5) -2016-06-13 22:57:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/aWord2vec.txt:0+58 -2016-06-13 22:57:46 INFO HadoopRDD:59 - Input split: file:/D:/all/idea/SparkLearning/file/data/mllib/input/FeatureExtractionAndTransformation/aWord2vec.txt:58+58 -2016-06-13 22:57:46 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 4). 2253 bytes result sent to driver -2016-06-13 22:57:46 INFO Executor:59 - Finished task 1.0 in stage 2.0 (TID 5). 2253 bytes result sent to driver -2016-06-13 22:57:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 4) in 52 ms on localhost (1/2) -2016-06-13 22:57:46 INFO TaskSetManager:59 - Finished task 1.0 in stage 2.0 (TID 5) in 51 ms on localhost (2/2) -2016-06-13 22:57:46 INFO DAGScheduler:59 - ShuffleMapStage 2 (repartition at Word2Vec.scala:309) finished in 0.054 s -2016-06-13 22:57:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-06-13 22:57:46 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:57:46 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:57:46 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 3, ResultStage 4) -2016-06-13 22:57:46 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:57:46 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 3: List() -2016-06-13 22:57:46 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List(ShuffleMapStage 3) -2016-06-13 22:57:46 INFO DAGScheduler:59 - Submitting ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323), which is now runnable -2016-06-13 22:57:46 INFO MemoryStore:59 - ensureFreeSpace(7752) called with curMem=191252, maxMem=766075207 -2016-06-13 22:57:46 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 7.6 KB, free 730.4 MB) -2016-06-13 22:57:46 INFO MemoryStore:59 - ensureFreeSpace(3829) called with curMem=199004, maxMem=766075207 -2016-06-13 22:57:46 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.7 KB, free 730.4 MB) -2016-06-13 22:57:46 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:55094 (size: 3.7 KB, free: 730.6 MB) -2016-06-13 22:57:46 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:57:46 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[13] at mapPartitionsWithIndex at Word2Vec.scala:323) -2016-06-13 22:57:46 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-06-13 22:57:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 6, localhost, PROCESS_LOCAL, 2159 bytes) -2016-06-13 22:57:46 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 6) -2016-06-13 22:57:46 INFO CacheManager:59 - Partition rdd_12_0 not found, computing it -2016-06-13 22:57:46 INFO ShuffleBlockFetcherIterator:59 - Getting 2 non-empty blocks out of 2 blocks -2016-06-13 22:57:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 22:57:46 INFO MemoryStore:59 - ensureFreeSpace(104) called with curMem=202833, maxMem=766075207 -2016-06-13 22:57:46 INFO MemoryStore:59 - Block rdd_12_0 stored as values in memory (estimated size 104.0 B, free 730.4 MB) -2016-06-13 22:57:46 INFO BlockManagerInfo:59 - Added rdd_12_0 in memory on localhost:55094 (size: 104.0 B, free: 730.6 MB) -2016-06-13 22:57:46 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS -2016-06-13 22:57:46 WARN BLAS:61 - Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS -2016-06-13 22:57:46 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 6). 1954 bytes result sent to driver -2016-06-13 22:57:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 6) in 56 ms on localhost (1/1) -2016-06-13 22:57:46 INFO DAGScheduler:59 - ShuffleMapStage 3 (mapPartitionsWithIndex at Word2Vec.scala:323) finished in 0.057 s -2016-06-13 22:57:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-06-13 22:57:46 INFO DAGScheduler:59 - looking for newly runnable stages -2016-06-13 22:57:46 INFO DAGScheduler:59 - running: Set() -2016-06-13 22:57:46 INFO DAGScheduler:59 - waiting: Set(ResultStage 4) -2016-06-13 22:57:46 INFO DAGScheduler:59 - failed: Set() -2016-06-13 22:57:46 INFO DAGScheduler:59 - Missing parents for ResultStage 4: List() -2016-06-13 22:57:46 INFO DAGScheduler:59 - Submitting ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397), which is now runnable -2016-06-13 22:57:46 INFO MemoryStore:59 - ensureFreeSpace(4672) called with curMem=202937, maxMem=766075207 -2016-06-13 22:57:46 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.6 KB, free 730.4 MB) -2016-06-13 22:57:46 INFO MemoryStore:59 - ensureFreeSpace(2262) called with curMem=207609, maxMem=766075207 -2016-06-13 22:57:46 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.2 KB, free 730.4 MB) -2016-06-13 22:57:46 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:55094 (size: 2.2 KB, free: 730.6 MB) -2016-06-13 22:57:46 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-06-13 22:57:46 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 4 (ShuffledRDD[14] at reduceByKey at Word2Vec.scala:397) -2016-06-13 22:57:46 INFO TaskSchedulerImpl:59 - Adding task set 4.0 with 1 tasks -2016-06-13 22:57:46 INFO TaskSetManager:59 - Starting task 0.0 in stage 4.0 (TID 7, localhost, PROCESS_LOCAL, 1901 bytes) -2016-06-13 22:57:46 INFO Executor:59 - Running task 0.0 in stage 4.0 (TID 7) -2016-06-13 22:57:46 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-06-13 22:57:46 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-06-13 22:57:46 INFO Executor:59 - Finished task 0.0 in stage 4.0 (TID 7). 2586 bytes result sent to driver -2016-06-13 22:57:46 INFO TaskSetManager:59 - Finished task 0.0 in stage 4.0 (TID 7) in 15 ms on localhost (1/1) -2016-06-13 22:57:46 INFO DAGScheduler:59 - ResultStage 4 (collect at Word2Vec.scala:400) finished in 0.017 s -2016-06-13 22:57:46 INFO TaskSchedulerImpl:59 - Removed TaskSet 4.0, whose tasks have all completed, from pool -2016-06-13 22:57:46 INFO DAGScheduler:59 - Job 1 finished: collect at Word2Vec.scala:400, took 0.201408 s -2016-06-13 22:57:46 INFO MapPartitionsRDD:59 - Removing RDD 12 from persistence list -2016-06-13 22:57:46 INFO BlockManager:59 - Removing RDD 12 -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-06-13 22:57:46 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-06-13 22:57:46 INFO SparkUI:59 - Stopped Spark web UI at http://202.38.84.85:4040 -2016-06-13 22:57:46 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-06-13 22:57:46 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-06-13 22:57:46 INFO MemoryStore:59 - MemoryStore cleared -2016-06-13 22:57:46 INFO BlockManager:59 - BlockManager stopped -2016-06-13 22:57:46 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-06-13 22:57:46 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-06-13 22:57:46 INFO SparkContext:59 - Successfully stopped SparkContext -2016-06-13 22:57:46 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-06-13 22:57:46 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-06-13 22:57:46 INFO ShutdownHookManager:59 - Shutdown hook called -2016-06-13 22:57:46 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-7015a1e4-0e59-4cd4-9822-ed967aaa0256 -2016-10-05 11:06:10 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-10-05 11:06:19 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-10-05 11:06:23 INFO SecurityManager:59 - Changing view acls to: xubo -2016-10-05 11:06:23 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-10-05 11:06:23 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-10-05 11:06:46 INFO Slf4jLogger:80 - Slf4jLogger started -2016-10-05 11:06:51 INFO Remoting:74 - Starting remoting -2016-10-05 11:06:55 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@1.0.0.152:2283] -2016-10-05 11:06:55 INFO Utils:59 - Successfully started service 'sparkDriver' on port 2283. -2016-10-05 11:06:56 INFO SparkEnv:59 - Registering MapOutputTracker -2016-10-05 11:06:57 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-10-05 11:06:57 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-e9d2d602-febc-4bf1-967a-6f25e36e287f -2016-10-05 11:06:58 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-10-05 11:06:58 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-141c73d5-869c-4073-bd46-c202b46168de\httpd-988ffff2-85c6-44fc-a009-6324196aba22 -2016-10-05 11:06:58 INFO HttpServer:59 - Starting HTTP Server -2016-10-05 11:07:00 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-10-05 11:07:00 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:2298 -2016-10-05 11:07:00 INFO Utils:59 - Successfully started service 'HTTP file server' on port 2298. -2016-10-05 11:07:01 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-10-05 11:07:05 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-10-05 11:07:05 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-10-05 11:07:05 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-10-05 11:07:05 INFO SparkUI:59 - Started SparkUI at http://1.0.0.152:4040 -2016-10-05 11:07:06 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-10-05 11:07:06 INFO Executor:59 - Starting executor ID driver on host localhost -2016-10-05 11:07:06 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 2331. -2016-10-05 11:07:06 INFO NettyBlockTransferService:59 - Server created on 2331 -2016-10-05 11:07:06 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-10-05 11:07:06 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:2331 with 730.6 MB RAM, BlockManagerId(driver, localhost, 2331) -2016-10-05 11:07:06 INFO BlockManagerMaster:59 - Registered BlockManager -2016-10-05 11:07:09 INFO ADAMContext:697 - Loading file/data/examples/input/small.adam as Parquet of AlignmentRecords. -2016-10-05 11:07:09 INFO ADAMContext:144 - Reading the ADAM file at file/data/examples/input/small.adam to create RDD -2016-10-05 11:07:09 INFO ADAMContext:154 - Using the specified projection schema -2016-10-05 11:07:11 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:100:f3%17, but we couldn't find any external IP address! -2016-10-05 11:07:12 INFO MemoryStore:59 - ensureFreeSpace(247720) called with curMem=0, maxMem=766075207 -2016-10-05 11:07:12 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 241.9 KB, free 730.4 MB) -2016-10-05 11:07:12 INFO MemoryStore:59 - ensureFreeSpace(20343) called with curMem=247720, maxMem=766075207 -2016-10-05 11:07:12 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 19.9 KB, free 730.3 MB) -2016-10-05 11:07:12 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:2331 (size: 19.9 KB, free: 730.6 MB) -2016-10-05 11:07:12 INFO SparkContext:59 - Created broadcast 0 from newAPIHadoopFile at ADAMContext.scala:158 -2016-10-05 11:07:13 INFO FileInputFormat:281 - Total input paths to process : 1 -2016-10-05 11:07:13 INFO SparkContext:59 - Starting job: take at kmerSaveAsFile.scala:27 -2016-10-05 11:07:13 INFO DAGScheduler:59 - Registering RDD 2 (flatMap at kmerSaveAsFile.scala:26) -2016-10-05 11:07:13 INFO DAGScheduler:59 - Registering RDD 4 (map at kmerSaveAsFile.scala:26) -2016-10-05 11:07:13 INFO DAGScheduler:59 - Got job 0 (take at kmerSaveAsFile.scala:27) with 1 output partitions -2016-10-05 11:07:13 INFO DAGScheduler:59 - Final stage: ResultStage 2(take at kmerSaveAsFile.scala:27) -2016-10-05 11:07:13 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1) -2016-10-05 11:07:13 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1) -2016-10-05 11:07:13 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[2] at flatMap at kmerSaveAsFile.scala:26), which has no missing parents -2016-10-05 11:07:14 INFO MemoryStore:59 - ensureFreeSpace(3584) called with curMem=268063, maxMem=766075207 -2016-10-05 11:07:14 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-10-05 11:07:14 INFO MemoryStore:59 - ensureFreeSpace(2127) called with curMem=271647, maxMem=766075207 -2016-10-05 11:07:14 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.3 MB) -2016-10-05 11:07:14 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:2331 (size: 2.1 KB, free: 730.6 MB) -2016-10-05 11:07:14 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-10-05 11:07:14 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[2] at flatMap at kmerSaveAsFile.scala:26) -2016-10-05 11:07:14 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-10-05 11:07:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2217 bytes) -2016-10-05 11:07:14 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-10-05 11:07:14 INFO NewHadoopRDD:59 - Input split: ParquetInputSplit{part: file:/E:/all/git/SparkLearning/file/data/examples/input/small.adam/part-r-00000.gz.parquet start: 0 end: 19037 length: 19037 hosts: []} -2016-10-05 11:07:15 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-10-05 11:07:16 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2253 bytes result sent to driver -2016-10-05 11:07:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 2222 ms on localhost (1/1) -2016-10-05 11:07:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-10-05 11:07:16 INFO DAGScheduler:59 - ShuffleMapStage 0 (flatMap at kmerSaveAsFile.scala:26) finished in 2.246 s -2016-10-05 11:07:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-10-05 11:07:16 INFO DAGScheduler:59 - running: Set() -2016-10-05 11:07:16 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 1, ResultStage 2) -2016-10-05 11:07:16 INFO DAGScheduler:59 - failed: Set() -2016-10-05 11:07:16 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 1: List() -2016-10-05 11:07:16 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List(ShuffleMapStage 1) -2016-10-05 11:07:16 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1 (MapPartitionsRDD[4] at map at kmerSaveAsFile.scala:26), which is now runnable -2016-10-05 11:07:16 INFO MemoryStore:59 - ensureFreeSpace(3208) called with curMem=273774, maxMem=766075207 -2016-10-05 11:07:16 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.1 KB, free 730.3 MB) -2016-10-05 11:07:16 INFO MemoryStore:59 - ensureFreeSpace(1868) called with curMem=276982, maxMem=766075207 -2016-10-05 11:07:16 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1868.0 B, free 730.3 MB) -2016-10-05 11:07:16 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:2331 (size: 1868.0 B, free: 730.6 MB) -2016-10-05 11:07:16 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-10-05 11:07:16 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[4] at map at kmerSaveAsFile.scala:26) -2016-10-05 11:07:16 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-10-05 11:07:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 1890 bytes) -2016-10-05 11:07:16 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-10-05 11:07:16 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-10-05 11:07:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 42 ms -2016-10-05 11:07:16 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:2331 in memory (size: 2.1 KB, free: 730.6 MB) -2016-10-05 11:07:16 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 1374 bytes result sent to driver -2016-10-05 11:07:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 314 ms on localhost (1/1) -2016-10-05 11:07:16 INFO DAGScheduler:59 - ShuffleMapStage 1 (map at kmerSaveAsFile.scala:26) finished in 0.315 s -2016-10-05 11:07:16 INFO DAGScheduler:59 - looking for newly runnable stages -2016-10-05 11:07:16 INFO DAGScheduler:59 - running: Set() -2016-10-05 11:07:16 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-10-05 11:07:16 INFO DAGScheduler:59 - failed: Set() -2016-10-05 11:07:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-10-05 11:07:16 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List() -2016-10-05 11:07:16 INFO DAGScheduler:59 - Submitting ResultStage 2 (ShuffledRDD[5] at sortByKey at kmerSaveAsFile.scala:26), which is now runnable -2016-10-05 11:07:16 INFO MemoryStore:59 - ensureFreeSpace(2632) called with curMem=273139, maxMem=766075207 -2016-10-05 11:07:16 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.6 KB, free 730.3 MB) -2016-10-05 11:07:16 INFO MemoryStore:59 - ensureFreeSpace(1559) called with curMem=275771, maxMem=766075207 -2016-10-05 11:07:16 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1559.0 B, free 730.3 MB) -2016-10-05 11:07:16 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:2331 (size: 1559.0 B, free: 730.6 MB) -2016-10-05 11:07:16 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-10-05 11:07:16 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (ShuffledRDD[5] at sortByKey at kmerSaveAsFile.scala:26) -2016-10-05 11:07:16 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-10-05 11:07:16 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-10-05 11:07:16 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-10-05 11:07:16 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-10-05 11:07:16 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-10-05 11:07:16 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 1727 bytes result sent to driver -2016-10-05 11:07:16 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 146 ms on localhost (1/1) -2016-10-05 11:07:16 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-10-05 11:07:16 INFO DAGScheduler:59 - ResultStage 2 (take at kmerSaveAsFile.scala:27) finished in 0.148 s -2016-10-05 11:07:16 INFO DAGScheduler:59 - Job 0 finished: take at kmerSaveAsFile.scala:27, took 3.183424 s -2016-10-05 11:07:16 INFO SparkContext:59 - Starting job: count at kmerSaveAsFile.scala:33 -2016-10-05 11:07:16 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 142 bytes -2016-10-05 11:07:17 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 142 bytes -2016-10-05 11:07:17 INFO DAGScheduler:59 - Got job 1 (count at kmerSaveAsFile.scala:33) with 1 output partitions -2016-10-05 11:07:17 INFO DAGScheduler:59 - Final stage: ResultStage 5(count at kmerSaveAsFile.scala:33) -2016-10-05 11:07:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-10-05 11:07:17 INFO DAGScheduler:59 - Missing parents: List() -2016-10-05 11:07:17 INFO DAGScheduler:59 - Submitting ResultStage 5 (ShuffledRDD[5] at sortByKey at kmerSaveAsFile.scala:26), which has no missing parents -2016-10-05 11:07:17 INFO MemoryStore:59 - ensureFreeSpace(2440) called with curMem=277330, maxMem=766075207 -2016-10-05 11:07:17 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-10-05 11:07:17 INFO MemoryStore:59 - ensureFreeSpace(1488) called with curMem=279770, maxMem=766075207 -2016-10-05 11:07:17 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 1488.0 B, free 730.3 MB) -2016-10-05 11:07:17 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:2331 (size: 1488.0 B, free: 730.6 MB) -2016-10-05 11:07:17 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-10-05 11:07:17 INFO BlockManagerInfo:59 - Removed broadcast_3_piece0 on localhost:2331 in memory (size: 1559.0 B, free: 730.6 MB) -2016-10-05 11:07:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 5 (ShuffledRDD[5] at sortByKey at kmerSaveAsFile.scala:26) -2016-10-05 11:07:17 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 1 tasks -2016-10-05 11:07:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-10-05 11:07:17 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 3) -2016-10-05 11:07:17 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-10-05 11:07:17 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-10-05 11:07:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-10-05 11:07:17 INFO BlockManagerInfo:59 - Removed broadcast_2_piece0 on localhost:2331 in memory (size: 1868.0 B, free: 730.6 MB) -2016-10-05 11:07:17 INFO ContextCleaner:59 - Cleaned accumulator 2 -2016-10-05 11:07:17 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 3). 1203 bytes result sent to driver -2016-10-05 11:07:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 3) in 64 ms on localhost (1/1) -2016-10-05 11:07:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-10-05 11:07:17 INFO DAGScheduler:59 - ResultStage 5 (count at kmerSaveAsFile.scala:33) finished in 0.065 s -2016-10-05 11:07:17 INFO DAGScheduler:59 - Job 1 finished: count at kmerSaveAsFile.scala:33, took 0.107329 s -2016-10-05 11:07:17 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-10-05 11:07:17 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-10-05 11:07:17 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-10-05 11:07:17 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-10-05 11:07:17 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-10-05 11:07:17 INFO SparkContext:59 - Starting job: saveAsTextFile at kmerSaveAsFile.scala:34 -2016-10-05 11:07:17 INFO DAGScheduler:59 - Got job 2 (saveAsTextFile at kmerSaveAsFile.scala:34) with 1 output partitions -2016-10-05 11:07:17 INFO DAGScheduler:59 - Final stage: ResultStage 8(saveAsTextFile at kmerSaveAsFile.scala:34) -2016-10-05 11:07:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 7) -2016-10-05 11:07:17 INFO DAGScheduler:59 - Missing parents: List() -2016-10-05 11:07:17 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[6] at saveAsTextFile at kmerSaveAsFile.scala:34), which has no missing parents -2016-10-05 11:07:17 INFO BlockManagerInfo:59 - Removed broadcast_4_piece0 on localhost:2331 in memory (size: 1488.0 B, free: 730.6 MB) -2016-10-05 11:07:17 INFO MemoryStore:59 - ensureFreeSpace(128072) called with curMem=268063, maxMem=766075207 -2016-10-05 11:07:17 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 125.1 KB, free 730.2 MB) -2016-10-05 11:07:17 INFO MemoryStore:59 - ensureFreeSpace(42951) called with curMem=396135, maxMem=766075207 -2016-10-05 11:07:17 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 41.9 KB, free 730.2 MB) -2016-10-05 11:07:17 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:2331 (size: 41.9 KB, free: 730.5 MB) -2016-10-05 11:07:17 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-10-05 11:07:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[6] at saveAsTextFile at kmerSaveAsFile.scala:34) -2016-10-05 11:07:17 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-10-05 11:07:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 4, localhost, PROCESS_LOCAL, 1901 bytes) -2016-10-05 11:07:17 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 4) -2016-10-05 11:07:17 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-10-05 11:07:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-10-05 11:07:17 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201610051107_0008_m_000000_4' to file:/E:/all/git/SparkLearning/file/data/examples/output/kmer/20161005110716973/smallkmers21.adam/_temporary/0/task_201610051107_0008_m_000000 -2016-10-05 11:07:17 INFO SparkHadoopMapRedUtil:59 - attempt_201610051107_0008_m_000000_4: Committed -2016-10-05 11:07:17 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 4). 2080 bytes result sent to driver -2016-10-05 11:07:17 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 4) in 288 ms on localhost (1/1) -2016-10-05 11:07:17 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-10-05 11:07:17 INFO DAGScheduler:59 - ResultStage 8 (saveAsTextFile at kmerSaveAsFile.scala:34) finished in 0.289 s -2016-10-05 11:07:17 INFO DAGScheduler:59 - Job 2 finished: saveAsTextFile at kmerSaveAsFile.scala:34, took 0.368730 s -2016-10-05 11:07:17 INFO SparkContext:59 - Starting job: sum at kmerSaveAsFile.scala:36 -2016-10-05 11:07:17 INFO DAGScheduler:59 - Got job 3 (sum at kmerSaveAsFile.scala:36) with 1 output partitions -2016-10-05 11:07:17 INFO DAGScheduler:59 - Final stage: ResultStage 11(sum at kmerSaveAsFile.scala:36) -2016-10-05 11:07:17 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 10) -2016-10-05 11:07:17 INFO DAGScheduler:59 - Missing parents: List() -2016-10-05 11:07:17 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[9] at numericRDDToDoubleRDDFunctions at kmerSaveAsFile.scala:36), which has no missing parents -2016-10-05 11:07:17 INFO MemoryStore:59 - ensureFreeSpace(3776) called with curMem=439086, maxMem=766075207 -2016-10-05 11:07:17 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.7 KB, free 730.2 MB) -2016-10-05 11:07:17 INFO MemoryStore:59 - ensureFreeSpace(2131) called with curMem=442862, maxMem=766075207 -2016-10-05 11:07:17 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.2 MB) -2016-10-05 11:07:17 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:2331 (size: 2.1 KB, free: 730.5 MB) -2016-10-05 11:07:17 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-10-05 11:07:17 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 11 (MapPartitionsRDD[9] at numericRDDToDoubleRDDFunctions at kmerSaveAsFile.scala:36) -2016-10-05 11:07:17 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 1 tasks -2016-10-05 11:07:17 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:2331 in memory (size: 41.9 KB, free: 730.6 MB) -2016-10-05 11:07:17 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 5, localhost, PROCESS_LOCAL, 1901 bytes) -2016-10-05 11:07:17 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 5) -2016-10-05 11:07:17 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-10-05 11:07:17 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-10-05 11:07:18 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 5). 1205 bytes result sent to driver -2016-10-05 11:07:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 5) in 77 ms on localhost (1/1) -2016-10-05 11:07:18 INFO DAGScheduler:59 - ResultStage 11 (sum at kmerSaveAsFile.scala:36) finished in 0.077 s -2016-10-05 11:07:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-10-05 11:07:18 INFO DAGScheduler:59 - Job 3 finished: sum at kmerSaveAsFile.scala:36, took 0.119344 s -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-10-05 11:07:18 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-10-05 11:07:18 INFO SparkUI:59 - Stopped Spark web UI at http://1.0.0.152:4040 -2016-10-05 11:07:18 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-10-05 11:07:18 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-10-05 11:07:18 INFO MemoryStore:59 - MemoryStore cleared -2016-10-05 11:07:18 INFO BlockManager:59 - BlockManager stopped -2016-10-05 11:07:18 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-10-05 11:07:18 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-10-05 11:07:18 INFO SparkContext:59 - Successfully stopped SparkContext -2016-10-05 11:07:18 INFO ShutdownHookManager:59 - Shutdown hook called -2016-10-05 11:07:18 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-141c73d5-869c-4073-bd46-c202b46168de -2016-10-05 11:10:38 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-10-05 11:10:39 INFO Slf4jLogger:80 - Slf4jLogger started -2016-10-05 11:10:39 INFO Remoting:74 - Starting remoting -2016-10-05 11:10:39 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@1.0.0.152:2404] -2016-10-05 11:10:40 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-10-05 11:10:40 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:2405 -2016-10-05 11:10:40 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-10-05 11:10:40 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-10-05 11:10:44 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-10-05 11:10:45 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-10-05 11:10:45 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-10-05 11:16:50 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-10-05 11:16:52 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-10-05 11:16:52 INFO SecurityManager:59 - Changing view acls to: xubo -2016-10-05 11:16:52 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-10-05 11:16:52 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-10-05 11:16:53 INFO Slf4jLogger:80 - Slf4jLogger started -2016-10-05 11:16:53 INFO Remoting:74 - Starting remoting -2016-10-05 11:16:53 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@1.0.0.152:3542] -2016-10-05 11:16:53 INFO Utils:59 - Successfully started service 'sparkDriver' on port 3542. -2016-10-05 11:16:53 INFO SparkEnv:59 - Registering MapOutputTracker -2016-10-05 11:16:53 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-10-05 11:16:53 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-a9b19323-34b4-4ebd-9d5a-d3d7f467228b -2016-10-05 11:16:53 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-10-05 11:16:53 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-4264170c-52a5-482b-ae36-22221aac7cb1\httpd-f8ccac19-e39e-4dbd-a373-4a277a240aba -2016-10-05 11:16:53 INFO HttpServer:59 - Starting HTTP Server -2016-10-05 11:16:53 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-10-05 11:16:53 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:3543 -2016-10-05 11:16:53 INFO Utils:59 - Successfully started service 'HTTP file server' on port 3543. -2016-10-05 11:16:53 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-10-05 11:16:53 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-10-05 11:16:53 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-10-05 11:16:53 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-10-05 11:16:53 INFO SparkUI:59 - Started SparkUI at http://1.0.0.152:4040 -2016-10-05 11:16:53 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-10-05 11:16:53 INFO Executor:59 - Starting executor ID driver on host localhost -2016-10-05 11:16:54 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 3550. -2016-10-05 11:16:54 INFO NettyBlockTransferService:59 - Server created on 3550 -2016-10-05 11:16:54 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-10-05 11:16:54 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:3550 with 730.6 MB RAM, BlockManagerId(driver, localhost, 3550) -2016-10-05 11:16:54 INFO BlockManagerMaster:59 - Registered BlockManager -2016-10-05 11:16:55 INFO ADAMContext:697 - Loading file/data/examples/input/small.adam as Parquet of AlignmentRecords. -2016-10-05 11:16:55 INFO ADAMContext:144 - Reading the ADAM file at file/data/examples/input/small.adam to create RDD -2016-10-05 11:16:55 INFO ADAMContext:154 - Using the specified projection schema -2016-10-05 11:16:56 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:100:f3%17, but we couldn't find any external IP address! -2016-10-05 11:16:57 INFO MemoryStore:59 - ensureFreeSpace(247720) called with curMem=0, maxMem=766075207 -2016-10-05 11:16:57 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 241.9 KB, free 730.4 MB) -2016-10-05 11:16:57 INFO MemoryStore:59 - ensureFreeSpace(20343) called with curMem=247720, maxMem=766075207 -2016-10-05 11:16:57 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 19.9 KB, free 730.3 MB) -2016-10-05 11:16:57 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:3550 (size: 19.9 KB, free: 730.6 MB) -2016-10-05 11:16:57 INFO SparkContext:59 - Created broadcast 0 from newAPIHadoopFile at ADAMContext.scala:158 -2016-10-05 11:16:58 INFO FileInputFormat:281 - Total input paths to process : 1 -2016-10-05 11:16:58 INFO SparkContext:59 - Starting job: take at kmerSaveAsFile.scala:27 -2016-10-05 11:16:58 INFO DAGScheduler:59 - Registering RDD 2 (flatMap at kmerSaveAsFile.scala:26) -2016-10-05 11:16:58 INFO DAGScheduler:59 - Registering RDD 4 (map at kmerSaveAsFile.scala:26) -2016-10-05 11:16:58 INFO DAGScheduler:59 - Got job 0 (take at kmerSaveAsFile.scala:27) with 1 output partitions -2016-10-05 11:16:58 INFO DAGScheduler:59 - Final stage: ResultStage 2(take at kmerSaveAsFile.scala:27) -2016-10-05 11:16:58 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 1) -2016-10-05 11:16:58 INFO DAGScheduler:59 - Missing parents: List(ShuffleMapStage 1) -2016-10-05 11:16:58 INFO DAGScheduler:59 - Submitting ShuffleMapStage 0 (MapPartitionsRDD[2] at flatMap at kmerSaveAsFile.scala:26), which has no missing parents -2016-10-05 11:16:58 INFO MemoryStore:59 - ensureFreeSpace(3584) called with curMem=268063, maxMem=766075207 -2016-10-05 11:16:58 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 3.5 KB, free 730.3 MB) -2016-10-05 11:16:58 INFO MemoryStore:59 - ensureFreeSpace(2127) called with curMem=271647, maxMem=766075207 -2016-10-05 11:16:58 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.3 MB) -2016-10-05 11:16:58 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:3550 (size: 2.1 KB, free: 730.6 MB) -2016-10-05 11:16:58 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-10-05 11:16:58 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[2] at flatMap at kmerSaveAsFile.scala:26) -2016-10-05 11:16:58 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-10-05 11:16:58 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2217 bytes) -2016-10-05 11:16:58 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-10-05 11:16:59 INFO NewHadoopRDD:59 - Input split: ParquetInputSplit{part: file:/E:/all/git/SparkLearning/file/data/examples/input/small.adam/part-r-00000.gz.parquet start: 0 end: 19037 length: 19037 hosts: []} -2016-10-05 11:16:59 INFO CodecPool:179 - Got brand-new decompressor [.gz] -2016-10-05 11:17:00 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2253 bytes result sent to driver -2016-10-05 11:17:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 1227 ms on localhost (1/1) -2016-10-05 11:17:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-10-05 11:17:00 INFO DAGScheduler:59 - ShuffleMapStage 0 (flatMap at kmerSaveAsFile.scala:26) finished in 1.244 s -2016-10-05 11:17:00 INFO DAGScheduler:59 - looking for newly runnable stages -2016-10-05 11:17:00 INFO DAGScheduler:59 - running: Set() -2016-10-05 11:17:00 INFO DAGScheduler:59 - waiting: Set(ShuffleMapStage 1, ResultStage 2) -2016-10-05 11:17:00 INFO DAGScheduler:59 - failed: Set() -2016-10-05 11:17:00 INFO DAGScheduler:59 - Missing parents for ShuffleMapStage 1: List() -2016-10-05 11:17:00 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List(ShuffleMapStage 1) -2016-10-05 11:17:00 INFO DAGScheduler:59 - Submitting ShuffleMapStage 1 (MapPartitionsRDD[4] at map at kmerSaveAsFile.scala:26), which is now runnable -2016-10-05 11:17:00 INFO MemoryStore:59 - ensureFreeSpace(3208) called with curMem=273774, maxMem=766075207 -2016-10-05 11:17:00 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 3.1 KB, free 730.3 MB) -2016-10-05 11:17:00 INFO MemoryStore:59 - ensureFreeSpace(1868) called with curMem=276982, maxMem=766075207 -2016-10-05 11:17:00 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 1868.0 B, free 730.3 MB) -2016-10-05 11:17:00 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:3550 (size: 1868.0 B, free: 730.6 MB) -2016-10-05 11:17:00 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-10-05 11:17:00 INFO DAGScheduler:59 - Submitting 1 missing tasks from ShuffleMapStage 1 (MapPartitionsRDD[4] at map at kmerSaveAsFile.scala:26) -2016-10-05 11:17:00 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-10-05 11:17:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 1890 bytes) -2016-10-05 11:17:00 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-10-05 11:17:00 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-10-05 11:17:00 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 17 ms -2016-10-05 11:17:00 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 1374 bytes result sent to driver -2016-10-05 11:17:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 221 ms on localhost (1/1) -2016-10-05 11:17:00 INFO DAGScheduler:59 - ShuffleMapStage 1 (map at kmerSaveAsFile.scala:26) finished in 0.221 s -2016-10-05 11:17:00 INFO DAGScheduler:59 - looking for newly runnable stages -2016-10-05 11:17:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-10-05 11:17:00 INFO DAGScheduler:59 - running: Set() -2016-10-05 11:17:00 INFO DAGScheduler:59 - waiting: Set(ResultStage 2) -2016-10-05 11:17:00 INFO DAGScheduler:59 - failed: Set() -2016-10-05 11:17:00 INFO DAGScheduler:59 - Missing parents for ResultStage 2: List() -2016-10-05 11:17:00 INFO DAGScheduler:59 - Submitting ResultStage 2 (ShuffledRDD[5] at sortByKey at kmerSaveAsFile.scala:26), which is now runnable -2016-10-05 11:17:00 INFO MemoryStore:59 - ensureFreeSpace(2632) called with curMem=278850, maxMem=766075207 -2016-10-05 11:17:00 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 2.6 KB, free 730.3 MB) -2016-10-05 11:17:00 INFO MemoryStore:59 - ensureFreeSpace(1559) called with curMem=281482, maxMem=766075207 -2016-10-05 11:17:00 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 1559.0 B, free 730.3 MB) -2016-10-05 11:17:00 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:3550 (size: 1559.0 B, free: 730.6 MB) -2016-10-05 11:17:00 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-10-05 11:17:00 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (ShuffledRDD[5] at sortByKey at kmerSaveAsFile.scala:26) -2016-10-05 11:17:00 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-10-05 11:17:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 1901 bytes) -2016-10-05 11:17:00 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-10-05 11:17:00 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-10-05 11:17:00 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-10-05 11:17:00 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 1727 bytes result sent to driver -2016-10-05 11:17:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 127 ms on localhost (1/1) -2016-10-05 11:17:00 INFO DAGScheduler:59 - ResultStage 2 (take at kmerSaveAsFile.scala:27) finished in 0.129 s -2016-10-05 11:17:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-10-05 11:17:00 INFO DAGScheduler:59 - Job 0 finished: take at kmerSaveAsFile.scala:27, took 2.075589 s -2016-10-05 11:17:00 INFO SparkContext:59 - Starting job: count at kmerSaveAsFile.scala:33 -2016-10-05 11:17:00 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 1 is 143 bytes -2016-10-05 11:17:00 INFO MapOutputTrackerMaster:59 - Size of output statuses for shuffle 0 is 143 bytes -2016-10-05 11:17:00 INFO DAGScheduler:59 - Got job 1 (count at kmerSaveAsFile.scala:33) with 1 output partitions -2016-10-05 11:17:00 INFO DAGScheduler:59 - Final stage: ResultStage 5(count at kmerSaveAsFile.scala:33) -2016-10-05 11:17:00 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 4) -2016-10-05 11:17:00 INFO DAGScheduler:59 - Missing parents: List() -2016-10-05 11:17:00 INFO DAGScheduler:59 - Submitting ResultStage 5 (ShuffledRDD[5] at sortByKey at kmerSaveAsFile.scala:26), which has no missing parents -2016-10-05 11:17:00 INFO MemoryStore:59 - ensureFreeSpace(2440) called with curMem=283041, maxMem=766075207 -2016-10-05 11:17:00 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 2.4 KB, free 730.3 MB) -2016-10-05 11:17:00 INFO MemoryStore:59 - ensureFreeSpace(1488) called with curMem=285481, maxMem=766075207 -2016-10-05 11:17:00 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 1488.0 B, free 730.3 MB) -2016-10-05 11:17:00 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:3550 (size: 1488.0 B, free: 730.6 MB) -2016-10-05 11:17:00 INFO SparkContext:59 - Created broadcast 4 from broadcast at DAGScheduler.scala:861 -2016-10-05 11:17:00 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 5 (ShuffledRDD[5] at sortByKey at kmerSaveAsFile.scala:26) -2016-10-05 11:17:00 INFO TaskSchedulerImpl:59 - Adding task set 5.0 with 1 tasks -2016-10-05 11:17:00 INFO TaskSetManager:59 - Starting task 0.0 in stage 5.0 (TID 3, localhost, PROCESS_LOCAL, 1901 bytes) -2016-10-05 11:17:00 INFO Executor:59 - Running task 0.0 in stage 5.0 (TID 3) -2016-10-05 11:17:00 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-10-05 11:17:00 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-10-05 11:17:00 INFO Executor:59 - Finished task 0.0 in stage 5.0 (TID 3). 1203 bytes result sent to driver -2016-10-05 11:17:00 INFO TaskSetManager:59 - Finished task 0.0 in stage 5.0 (TID 3) in 62 ms on localhost (1/1) -2016-10-05 11:17:00 INFO TaskSchedulerImpl:59 - Removed TaskSet 5.0, whose tasks have all completed, from pool -2016-10-05 11:17:00 INFO DAGScheduler:59 - ResultStage 5 (count at kmerSaveAsFile.scala:33) finished in 0.062 s -2016-10-05 11:17:00 INFO DAGScheduler:59 - Job 1 finished: count at kmerSaveAsFile.scala:33, took 0.087263 s -2016-10-05 11:17:00 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-10-05 11:17:00 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-10-05 11:17:00 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-10-05 11:17:00 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-10-05 11:17:00 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-10-05 11:17:01 INFO SparkContext:59 - Starting job: saveAsTextFile at kmerSaveAsFile.scala:34 -2016-10-05 11:17:01 INFO DAGScheduler:59 - Got job 2 (saveAsTextFile at kmerSaveAsFile.scala:34) with 1 output partitions -2016-10-05 11:17:01 INFO DAGScheduler:59 - Final stage: ResultStage 8(saveAsTextFile at kmerSaveAsFile.scala:34) -2016-10-05 11:17:01 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 7) -2016-10-05 11:17:01 INFO DAGScheduler:59 - Missing parents: List() -2016-10-05 11:17:01 INFO DAGScheduler:59 - Submitting ResultStage 8 (MapPartitionsRDD[6] at saveAsTextFile at kmerSaveAsFile.scala:34), which has no missing parents -2016-10-05 11:17:01 INFO MemoryStore:59 - ensureFreeSpace(128072) called with curMem=286969, maxMem=766075207 -2016-10-05 11:17:01 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 125.1 KB, free 730.2 MB) -2016-10-05 11:17:01 INFO MemoryStore:59 - ensureFreeSpace(42951) called with curMem=415041, maxMem=766075207 -2016-10-05 11:17:01 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 41.9 KB, free 730.1 MB) -2016-10-05 11:17:01 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:3550 (size: 41.9 KB, free: 730.5 MB) -2016-10-05 11:17:01 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-10-05 11:17:01 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 8 (MapPartitionsRDD[6] at saveAsTextFile at kmerSaveAsFile.scala:34) -2016-10-05 11:17:01 INFO TaskSchedulerImpl:59 - Adding task set 8.0 with 1 tasks -2016-10-05 11:17:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 8.0 (TID 4, localhost, PROCESS_LOCAL, 1901 bytes) -2016-10-05 11:17:01 INFO Executor:59 - Running task 0.0 in stage 8.0 (TID 4) -2016-10-05 11:17:01 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-10-05 11:17:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 0 ms -2016-10-05 11:17:01 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201610051117_0008_m_000000_4' to file:/E:/all/git/SparkLearning/file/data/examples/output/kmer/20161005111700609/smallkmers21.adam/_temporary/0/task_201610051117_0008_m_000000 -2016-10-05 11:17:01 INFO SparkHadoopMapRedUtil:59 - attempt_201610051117_0008_m_000000_4: Committed -2016-10-05 11:17:01 INFO Executor:59 - Finished task 0.0 in stage 8.0 (TID 4). 2080 bytes result sent to driver -2016-10-05 11:17:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 8.0 (TID 4) in 278 ms on localhost (1/1) -2016-10-05 11:17:01 INFO DAGScheduler:59 - ResultStage 8 (saveAsTextFile at kmerSaveAsFile.scala:34) finished in 0.278 s -2016-10-05 11:17:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 8.0, whose tasks have all completed, from pool -2016-10-05 11:17:01 INFO DAGScheduler:59 - Job 2 finished: saveAsTextFile at kmerSaveAsFile.scala:34, took 0.357997 s -2016-10-05 11:17:01 INFO SparkContext:59 - Starting job: sum at kmerSaveAsFile.scala:36 -2016-10-05 11:17:01 INFO DAGScheduler:59 - Got job 3 (sum at kmerSaveAsFile.scala:36) with 1 output partitions -2016-10-05 11:17:01 INFO DAGScheduler:59 - Final stage: ResultStage 11(sum at kmerSaveAsFile.scala:36) -2016-10-05 11:17:01 INFO DAGScheduler:59 - Parents of final stage: List(ShuffleMapStage 10) -2016-10-05 11:17:01 INFO DAGScheduler:59 - Missing parents: List() -2016-10-05 11:17:01 INFO DAGScheduler:59 - Submitting ResultStage 11 (MapPartitionsRDD[9] at numericRDDToDoubleRDDFunctions at kmerSaveAsFile.scala:36), which has no missing parents -2016-10-05 11:17:01 INFO MemoryStore:59 - ensureFreeSpace(3776) called with curMem=457992, maxMem=766075207 -2016-10-05 11:17:01 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 3.7 KB, free 730.1 MB) -2016-10-05 11:17:01 INFO MemoryStore:59 - ensureFreeSpace(2131) called with curMem=461768, maxMem=766075207 -2016-10-05 11:17:01 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 2.1 KB, free 730.1 MB) -2016-10-05 11:17:01 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:3550 (size: 2.1 KB, free: 730.5 MB) -2016-10-05 11:17:01 INFO SparkContext:59 - Created broadcast 6 from broadcast at DAGScheduler.scala:861 -2016-10-05 11:17:01 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 11 (MapPartitionsRDD[9] at numericRDDToDoubleRDDFunctions at kmerSaveAsFile.scala:36) -2016-10-05 11:17:01 INFO TaskSchedulerImpl:59 - Adding task set 11.0 with 1 tasks -2016-10-05 11:17:01 INFO TaskSetManager:59 - Starting task 0.0 in stage 11.0 (TID 5, localhost, PROCESS_LOCAL, 1901 bytes) -2016-10-05 11:17:01 INFO Executor:59 - Running task 0.0 in stage 11.0 (TID 5) -2016-10-05 11:17:01 INFO ShuffleBlockFetcherIterator:59 - Getting 1 non-empty blocks out of 1 blocks -2016-10-05 11:17:01 INFO ShuffleBlockFetcherIterator:59 - Started 0 remote fetches in 1 ms -2016-10-05 11:17:01 INFO Executor:59 - Finished task 0.0 in stage 11.0 (TID 5). 1205 bytes result sent to driver -2016-10-05 11:17:01 INFO TaskSetManager:59 - Finished task 0.0 in stage 11.0 (TID 5) in 74 ms on localhost (1/1) -2016-10-05 11:17:01 INFO DAGScheduler:59 - ResultStage 11 (sum at kmerSaveAsFile.scala:36) finished in 0.075 s -2016-10-05 11:17:01 INFO TaskSchedulerImpl:59 - Removed TaskSet 11.0, whose tasks have all completed, from pool -2016-10-05 11:17:01 INFO DAGScheduler:59 - Job 3 finished: sum at kmerSaveAsFile.scala:36, took 0.103529 s -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-10-05 11:17:01 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-10-05 11:17:01 INFO SparkUI:59 - Stopped Spark web UI at http://1.0.0.152:4040 -2016-10-05 11:17:01 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-10-05 11:17:01 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-10-05 11:17:01 INFO MemoryStore:59 - MemoryStore cleared -2016-10-05 11:17:01 INFO BlockManager:59 - BlockManager stopped -2016-10-05 11:17:01 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-10-05 11:17:01 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-10-05 11:17:01 INFO SparkContext:59 - Successfully stopped SparkContext -2016-10-05 11:17:01 INFO ShutdownHookManager:59 - Shutdown hook called -2016-10-05 11:17:01 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-4264170c-52a5-482b-ae36-22221aac7cb1 -2016-10-05 11:17:01 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-10-05 11:17:01 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-10-05 11:22:58 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-10-05 11:23:01 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-10-05 11:23:01 INFO SecurityManager:59 - Changing view acls to: xubo -2016-10-05 11:23:01 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-10-05 11:23:01 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-10-05 11:23:02 INFO Slf4jLogger:80 - Slf4jLogger started -2016-10-05 11:23:02 INFO Remoting:74 - Starting remoting -2016-10-05 11:23:02 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@1.0.0.152:3834] -2016-10-05 11:23:02 INFO Utils:59 - Successfully started service 'sparkDriver' on port 3834. -2016-10-05 11:23:02 INFO SparkEnv:59 - Registering MapOutputTracker -2016-10-05 11:23:02 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-10-05 11:23:02 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-41699818-4625-485b-a54c-fcb14ec09209 -2016-10-05 11:23:03 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-10-05 11:23:03 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-041c29e0-fffb-4b2c-96ed-de1afc7ce1a3\httpd-5b532394-c354-40b7-b45d-9b574bd6cc4c -2016-10-05 11:23:03 INFO HttpServer:59 - Starting HTTP Server -2016-10-05 11:23:03 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-10-05 11:23:03 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:3835 -2016-10-05 11:23:03 INFO Utils:59 - Successfully started service 'HTTP file server' on port 3835. -2016-10-05 11:23:03 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-10-05 11:23:03 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-10-05 11:23:03 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-10-05 11:23:03 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-10-05 11:23:03 INFO SparkUI:59 - Started SparkUI at http://1.0.0.152:4040 -2016-10-05 11:23:03 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-10-05 11:23:03 INFO Executor:59 - Starting executor ID driver on host localhost -2016-10-05 11:23:03 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 3843. -2016-10-05 11:23:03 INFO NettyBlockTransferService:59 - Server created on 3843 -2016-10-05 11:23:03 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-10-05 11:23:03 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:3843 with 730.6 MB RAM, BlockManagerId(driver, localhost, 3843) -2016-10-05 11:23:03 INFO BlockManagerMaster:59 - Registered BlockManager -2016-10-05 11:23:16 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:100:f3%17, but we couldn't find any external IP address! -2016-10-05 11:23:24 INFO AvroRelation:59 - Listing file:/E:/all/git/SparkLearning/file/data/avro/input/episodes.avro on driver -2016-10-05 11:23:33 INFO MemoryStore:59 - ensureFreeSpace(88512) called with curMem=0, maxMem=766075207 -2016-10-05 11:23:33 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 86.4 KB, free 730.5 MB) -2016-10-05 11:23:33 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=88512, maxMem=766075207 -2016-10-05 11:23:33 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.5 MB) -2016-10-05 11:23:33 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:3843 (size: 19.3 KB, free: 730.6 MB) -2016-10-05 11:23:33 INFO SparkContext:59 - Created broadcast 0 from show at AvroReadSpecifyName.scala:28 -2016-10-05 11:23:33 INFO MemoryStore:59 - ensureFreeSpace(230704) called with curMem=108300, maxMem=766075207 -2016-10-05 11:23:33 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 225.3 KB, free 730.3 MB) -2016-10-05 11:23:33 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=339004, maxMem=766075207 -2016-10-05 11:23:33 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.2 MB) -2016-10-05 11:23:33 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:3843 (size: 19.3 KB, free: 730.5 MB) -2016-10-05 11:23:33 INFO SparkContext:59 - Created broadcast 1 from hadoopFile at AvroRelation.scala:121 -2016-10-05 11:23:37 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-10-05 11:23:37 INFO SparkContext:59 - Starting job: show at AvroReadSpecifyName.scala:28 -2016-10-05 11:23:37 INFO DAGScheduler:59 - Got job 0 (show at AvroReadSpecifyName.scala:28) with 1 output partitions -2016-10-05 11:23:37 INFO DAGScheduler:59 - Final stage: ResultStage 0(show at AvroReadSpecifyName.scala:28) -2016-10-05 11:23:37 INFO DAGScheduler:59 - Parents of final stage: List() -2016-10-05 11:23:37 INFO DAGScheduler:59 - Missing parents: List() -2016-10-05 11:23:37 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[6] at show at AvroReadSpecifyName.scala:28), which has no missing parents -2016-10-05 11:23:38 INFO MemoryStore:59 - ensureFreeSpace(4832) called with curMem=358792, maxMem=766075207 -2016-10-05 11:23:38 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 4.7 KB, free 730.2 MB) -2016-10-05 11:23:38 INFO MemoryStore:59 - ensureFreeSpace(2698) called with curMem=363624, maxMem=766075207 -2016-10-05 11:23:38 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.6 KB, free 730.2 MB) -2016-10-05 11:23:38 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:3843 (size: 2.6 KB, free: 730.5 MB) -2016-10-05 11:23:38 INFO SparkContext:59 - Created broadcast 2 from broadcast at DAGScheduler.scala:861 -2016-10-05 11:23:38 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[6] at show at AvroReadSpecifyName.scala:28) -2016-10-05 11:23:38 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-10-05 11:23:38 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2283 bytes) -2016-10-05 11:23:38 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-10-05 11:23:38 INFO HadoopRDD:59 - Input split: file:/E:/all/git/SparkLearning/file/data/avro/input/episodes.avro:0+597 -2016-10-05 11:23:38 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-10-05 11:23:38 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-10-05 11:23:38 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-10-05 11:23:38 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-10-05 11:23:38 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-10-05 11:23:38 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 3033 bytes result sent to driver -2016-10-05 11:23:38 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 858 ms on localhost (1/1) -2016-10-05 11:23:38 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-10-05 11:23:38 INFO DAGScheduler:59 - ResultStage 0 (show at AvroReadSpecifyName.scala:28) finished in 0.872 s -2016-10-05 11:23:38 INFO DAGScheduler:59 - Job 0 finished: show at AvroReadSpecifyName.scala:28, took 1.088723 s -2016-10-05 11:23:40 INFO MemoryStore:59 - ensureFreeSpace(230728) called with curMem=366322, maxMem=766075207 -2016-10-05 11:23:40 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 225.3 KB, free 730.0 MB) -2016-10-05 11:23:40 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=597050, maxMem=766075207 -2016-10-05 11:23:40 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.0 MB) -2016-10-05 11:23:40 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:3843 (size: 19.3 KB, free: 730.5 MB) -2016-10-05 11:23:40 INFO SparkContext:59 - Created broadcast 3 from save at package.scala:37 -2016-10-05 11:23:40 INFO MemoryStore:59 - ensureFreeSpace(230744) called with curMem=616838, maxMem=766075207 -2016-10-05 11:23:40 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 225.3 KB, free 729.8 MB) -2016-10-05 11:23:40 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=847582, maxMem=766075207 -2016-10-05 11:23:40 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.8 MB) -2016-10-05 11:23:40 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:3843 (size: 19.3 KB, free: 730.5 MB) -2016-10-05 11:23:40 INFO SparkContext:59 - Created broadcast 4 from hadoopFile at AvroRelation.scala:121 -2016-10-05 11:23:40 INFO AvroRelation:59 - using snappy for Avro output -2016-10-05 11:23:40 INFO deprecation:1049 - mapred.output.compress is deprecated. Instead, use mapreduce.output.fileoutputformat.compress -2016-10-05 11:23:40 INFO DefaultWriterContainer:59 - Using output committer class org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter -2016-10-05 11:23:41 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-10-05 11:23:41 INFO SparkContext:59 - Starting job: save at package.scala:37 -2016-10-05 11:23:41 INFO DAGScheduler:59 - Got job 1 (save at package.scala:37) with 1 output partitions -2016-10-05 11:23:41 INFO DAGScheduler:59 - Final stage: ResultStage 1(save at package.scala:37) -2016-10-05 11:23:41 INFO DAGScheduler:59 - Parents of final stage: List() -2016-10-05 11:23:41 INFO DAGScheduler:59 - Missing parents: List() -2016-10-05 11:23:41 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[12] at save at package.scala:37), which has no missing parents -2016-10-05 11:23:41 INFO MemoryStore:59 - ensureFreeSpace(68656) called with curMem=867370, maxMem=766075207 -2016-10-05 11:23:41 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 67.0 KB, free 729.7 MB) -2016-10-05 11:23:41 INFO MemoryStore:59 - ensureFreeSpace(24429) called with curMem=936026, maxMem=766075207 -2016-10-05 11:23:41 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 23.9 KB, free 729.7 MB) -2016-10-05 11:23:41 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:3843 (size: 23.9 KB, free: 730.5 MB) -2016-10-05 11:23:41 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-10-05 11:23:41 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[12] at save at package.scala:37) -2016-10-05 11:23:41 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-10-05 11:23:41 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2283 bytes) -2016-10-05 11:23:41 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-10-05 11:23:41 INFO HadoopRDD:59 - Input split: file:/E:/all/git/SparkLearning/file/data/avro/input/episodes.avro:0+597 -2016-10-05 11:23:41 INFO DefaultWriterContainer:59 - Using output committer class org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter -2016-10-05 11:23:41 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201610051123_0001_m_000000_0' to file:/E:/all/git/SparkLearning/file/data/avro/output/episodes/AvroReadSpecifyName20161005112339288/_temporary/0/task_201610051123_0001_m_000000 -2016-10-05 11:23:41 INFO SparkHadoopMapRedUtil:59 - attempt_201610051123_0001_m_000000_0: Committed -2016-10-05 11:23:41 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2044 bytes result sent to driver -2016-10-05 11:23:41 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 540 ms on localhost (1/1) -2016-10-05 11:23:41 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-10-05 11:23:41 INFO DAGScheduler:59 - ResultStage 1 (save at package.scala:37) finished in 0.541 s -2016-10-05 11:23:41 INFO DAGScheduler:59 - Job 1 finished: save at package.scala:37, took 0.601808 s -2016-10-05 11:23:41 INFO DefaultWriterContainer:59 - Job job_201610051123_0000 committed. -2016-10-05 11:23:41 INFO AvroRelation:59 - Listing file:/E:/all/git/SparkLearning/file/data/avro/output/episodes/AvroReadSpecifyName20161005112339288 on driver -2016-10-05 11:23:42 INFO AvroRelation:59 - Listing file:/E:/all/git/SparkLearning/file/data/avro/output/episodes/AvroReadSpecifyName20161005112339288 on driver -2016-10-05 11:23:42 INFO AvroRelation:59 - Listing file:/E:/all/git/SparkLearning/file/data/avro/output/episodes/AvroReadSpecifyName20161005112339288 on driver -2016-10-05 11:23:42 INFO MemoryStore:59 - ensureFreeSpace(230728) called with curMem=960455, maxMem=766075207 -2016-10-05 11:23:42 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 225.3 KB, free 729.5 MB) -2016-10-05 11:23:42 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1191183, maxMem=766075207 -2016-10-05 11:23:42 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.4 MB) -2016-10-05 11:23:42 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:3843 (size: 19.3 KB, free: 730.5 MB) -2016-10-05 11:23:42 INFO SparkContext:59 - Created broadcast 6 from show at AvroReadSpecifyName.scala:40 -2016-10-05 11:23:42 INFO MemoryStore:59 - ensureFreeSpace(230744) called with curMem=1210971, maxMem=766075207 -2016-10-05 11:23:42 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 225.3 KB, free 729.2 MB) -2016-10-05 11:23:42 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1441715, maxMem=766075207 -2016-10-05 11:23:42 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.2 MB) -2016-10-05 11:23:42 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:3843 (size: 19.3 KB, free: 730.4 MB) -2016-10-05 11:23:42 INFO SparkContext:59 - Created broadcast 7 from hadoopFile at AvroRelation.scala:121 -2016-10-05 11:23:42 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-10-05 11:23:42 INFO SparkContext:59 - Starting job: show at AvroReadSpecifyName.scala:40 -2016-10-05 11:23:42 INFO DAGScheduler:59 - Got job 2 (show at AvroReadSpecifyName.scala:40) with 1 output partitions -2016-10-05 11:23:42 INFO DAGScheduler:59 - Final stage: ResultStage 2(show at AvroReadSpecifyName.scala:40) -2016-10-05 11:23:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-10-05 11:23:42 INFO DAGScheduler:59 - Missing parents: List() -2016-10-05 11:23:42 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[20] at show at AvroReadSpecifyName.scala:40), which has no missing parents -2016-10-05 11:23:42 INFO MemoryStore:59 - ensureFreeSpace(4920) called with curMem=1461503, maxMem=766075207 -2016-10-05 11:23:42 INFO MemoryStore:59 - Block broadcast_8 stored as values in memory (estimated size 4.8 KB, free 729.2 MB) -2016-10-05 11:23:42 INFO MemoryStore:59 - ensureFreeSpace(2796) called with curMem=1466423, maxMem=766075207 -2016-10-05 11:23:42 INFO MemoryStore:59 - Block broadcast_8_piece0 stored as bytes in memory (estimated size 2.7 KB, free 729.2 MB) -2016-10-05 11:23:42 INFO BlockManagerInfo:59 - Added broadcast_8_piece0 in memory on localhost:3843 (size: 2.7 KB, free: 730.4 MB) -2016-10-05 11:23:42 INFO SparkContext:59 - Created broadcast 8 from broadcast at DAGScheduler.scala:861 -2016-10-05 11:23:42 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[20] at show at AvroReadSpecifyName.scala:40) -2016-10-05 11:23:42 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-10-05 11:23:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2372 bytes) -2016-10-05 11:23:42 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-10-05 11:23:42 INFO HadoopRDD:59 - Input split: file:/E:/all/git/SparkLearning/file/data/avro/output/episodes/AvroReadSpecifyName20161005112339288/part-r-00000-11a6575b-ddb9-43e8-888b-052d1814ed20.avro:0+555 -2016-10-05 11:23:42 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 3033 bytes result sent to driver -2016-10-05 11:23:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 30 ms on localhost (1/1) -2016-10-05 11:23:42 INFO DAGScheduler:59 - ResultStage 2 (show at AvroReadSpecifyName.scala:40) finished in 0.030 s -2016-10-05 11:23:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-10-05 11:23:42 INFO DAGScheduler:59 - Job 2 finished: show at AvroReadSpecifyName.scala:40, took 0.069825 s -2016-10-05 11:23:42 INFO AvroRelation:59 - Listing file:/E:/all/git/SparkLearning/file/data/avro/output/episodes/AvroReadSpecifyName20161005112339288 on driver -2016-10-05 11:23:42 INFO MemoryStore:59 - ensureFreeSpace(230728) called with curMem=1469219, maxMem=766075207 -2016-10-05 11:23:42 INFO MemoryStore:59 - Block broadcast_9 stored as values in memory (estimated size 225.3 KB, free 729.0 MB) -2016-10-05 11:23:42 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1699947, maxMem=766075207 -2016-10-05 11:23:42 INFO MemoryStore:59 - Block broadcast_9_piece0 stored as bytes in memory (estimated size 19.3 KB, free 728.9 MB) -2016-10-05 11:23:42 INFO BlockManagerInfo:59 - Added broadcast_9_piece0 in memory on localhost:3843 (size: 19.3 KB, free: 730.4 MB) -2016-10-05 11:23:42 INFO SparkContext:59 - Created broadcast 9 from show at AvroReadSpecifyName.scala:43 -2016-10-05 11:23:42 INFO MemoryStore:59 - ensureFreeSpace(230744) called with curMem=1719735, maxMem=766075207 -2016-10-05 11:23:42 INFO MemoryStore:59 - Block broadcast_10 stored as values in memory (estimated size 225.3 KB, free 728.7 MB) -2016-10-05 11:23:42 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=1950479, maxMem=766075207 -2016-10-05 11:23:42 INFO MemoryStore:59 - Block broadcast_10_piece0 stored as bytes in memory (estimated size 19.3 KB, free 728.7 MB) -2016-10-05 11:23:42 INFO BlockManagerInfo:59 - Added broadcast_10_piece0 in memory on localhost:3843 (size: 19.3 KB, free: 730.4 MB) -2016-10-05 11:23:42 INFO SparkContext:59 - Created broadcast 10 from hadoopFile at AvroRelation.scala:121 -2016-10-05 11:23:42 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-10-05 11:23:42 INFO SparkContext:59 - Starting job: show at AvroReadSpecifyName.scala:43 -2016-10-05 11:23:42 INFO DAGScheduler:59 - Got job 3 (show at AvroReadSpecifyName.scala:43) with 1 output partitions -2016-10-05 11:23:42 INFO DAGScheduler:59 - Final stage: ResultStage 3(show at AvroReadSpecifyName.scala:43) -2016-10-05 11:23:42 INFO DAGScheduler:59 - Parents of final stage: List() -2016-10-05 11:23:42 INFO DAGScheduler:59 - Missing parents: List() -2016-10-05 11:23:42 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[27] at show at AvroReadSpecifyName.scala:43), which has no missing parents -2016-10-05 11:23:42 INFO MemoryStore:59 - ensureFreeSpace(4920) called with curMem=1970267, maxMem=766075207 -2016-10-05 11:23:42 INFO MemoryStore:59 - Block broadcast_11 stored as values in memory (estimated size 4.8 KB, free 728.7 MB) -2016-10-05 11:23:42 INFO MemoryStore:59 - ensureFreeSpace(2796) called with curMem=1975187, maxMem=766075207 -2016-10-05 11:23:42 INFO MemoryStore:59 - Block broadcast_11_piece0 stored as bytes in memory (estimated size 2.7 KB, free 728.7 MB) -2016-10-05 11:23:42 INFO BlockManagerInfo:59 - Added broadcast_11_piece0 in memory on localhost:3843 (size: 2.7 KB, free: 730.4 MB) -2016-10-05 11:23:42 INFO SparkContext:59 - Created broadcast 11 from broadcast at DAGScheduler.scala:861 -2016-10-05 11:23:42 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 3 (MapPartitionsRDD[27] at show at AvroReadSpecifyName.scala:43) -2016-10-05 11:23:42 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-10-05 11:23:42 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 3, localhost, PROCESS_LOCAL, 2372 bytes) -2016-10-05 11:23:42 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 3) -2016-10-05 11:23:42 INFO HadoopRDD:59 - Input split: file:/E:/all/git/SparkLearning/file/data/avro/output/episodes/AvroReadSpecifyName20161005112339288/part-r-00000-11a6575b-ddb9-43e8-888b-052d1814ed20.avro:0+555 -2016-10-05 11:23:42 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 3). 3033 bytes result sent to driver -2016-10-05 11:23:42 INFO DAGScheduler:59 - ResultStage 3 (show at AvroReadSpecifyName.scala:43) finished in 0.151 s -2016-10-05 11:23:42 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 3) in 151 ms on localhost (1/1) -2016-10-05 11:23:42 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-10-05 11:23:42 INFO DAGScheduler:59 - Job 3 finished: show at AvroReadSpecifyName.scala:43, took 0.204381 s -2016-10-05 11:23:42 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-10-05 11:23:42 INFO BlockManagerInfo:59 - Removed broadcast_9_piece0 on localhost:3843 in memory (size: 19.3 KB, free: 730.4 MB) -2016-10-05 11:23:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-10-05 11:23:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-10-05 11:23:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-10-05 11:23:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-10-05 11:23:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-10-05 11:23:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-10-05 11:23:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-10-05 11:23:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-10-05 11:23:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-10-05 11:23:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-10-05 11:23:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-10-05 11:23:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-10-05 11:23:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-10-05 11:23:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-10-05 11:23:42 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-10-05 11:23:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-10-05 11:23:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-10-05 11:23:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-10-05 11:23:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-10-05 11:23:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-10-05 11:23:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-10-05 11:23:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-10-05 11:23:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-10-05 11:23:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-10-05 11:23:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-10-05 11:23:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-10-05 11:23:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-10-05 11:23:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-10-05 11:23:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-10-05 11:23:43 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-10-05 11:23:43 INFO BlockManagerInfo:59 - Removed broadcast_8_piece0 on localhost:3843 in memory (size: 2.7 KB, free: 730.4 MB) -2016-10-05 11:23:43 INFO ContextCleaner:59 - Cleaned accumulator 3 -2016-10-05 11:23:43 INFO BlockManagerInfo:59 - Removed broadcast_7_piece0 on localhost:3843 in memory (size: 19.3 KB, free: 730.4 MB) -2016-10-05 11:23:43 INFO BlockManagerInfo:59 - Removed broadcast_6_piece0 on localhost:3843 in memory (size: 19.3 KB, free: 730.5 MB) -2016-10-05 11:23:43 INFO SparkUI:59 - Stopped Spark web UI at http://1.0.0.152:4040 -2016-10-05 11:23:43 INFO BlockManagerInfo:59 - Removed broadcast_5_piece0 on localhost:3843 in memory (size: 23.9 KB, free: 730.5 MB) -2016-10-05 11:23:43 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-10-05 11:23:43 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-10-05 11:23:43 INFO MemoryStore:59 - MemoryStore cleared -2016-10-05 11:23:43 INFO BlockManager:59 - BlockManager stopped -2016-10-05 11:23:43 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-10-05 11:23:43 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-10-05 11:23:43 INFO SparkContext:59 - Successfully stopped SparkContext -2016-10-05 11:23:43 INFO ShutdownHookManager:59 - Shutdown hook called -2016-10-05 11:23:43 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-041c29e0-fffb-4b2c-96ed-de1afc7ce1a3 -2016-10-05 11:23:43 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-10-05 11:23:43 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-10-05 11:32:48 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-10-05 11:32:50 INFO Slf4jLogger:80 - Slf4jLogger started -2016-10-05 11:32:50 INFO Remoting:74 - Starting remoting -2016-10-05 11:32:50 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@1.0.0.152:3891] -2016-10-05 11:32:51 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-10-05 11:32:51 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:3892 -2016-10-05 11:32:51 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-10-05 11:32:51 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-10-05 11:32:54 ERROR Executor:96 - Exception in task 1.0 in stage 10.0 (TID 31) -org.apache.spark.SparkException: The MatrixBlock at (0, 1) has dimensions different than rowsPerBlock: 3, and colsPerBlock: 2. Blocks on the right and bottom edges can have smaller dimensions. You may use the repartition method to fix this issue. - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:225) - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:222) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-10-05 11:32:54 ERROR Executor:96 - Exception in task 2.0 in stage 10.0 (TID 32) -org.apache.spark.SparkException: The MatrixBlock at (1, 1) has dimensions different than rowsPerBlock: 3, and colsPerBlock: 2. Blocks on the right and bottom edges can have smaller dimensions. You may use the repartition method to fix this issue. - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:225) - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:222) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-10-05 11:32:54 ERROR Executor:96 - Exception in task 0.0 in stage 10.0 (TID 30) -org.apache.spark.SparkException: The MatrixBlock at (0, 0) has dimensions different than rowsPerBlock: 3, and colsPerBlock: 2. Blocks on the right and bottom edges can have smaller dimensions. You may use the repartition method to fix this issue. - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:225) - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:222) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-10-05 11:32:55 ERROR TaskSetManager:75 - Task 2 in stage 10.0 failed 1 times; aborting job -2016-10-05 11:32:55 ERROR Executor:96 - Exception in task 0.0 in stage 14.0 (TID 42) -org.apache.spark.SparkException: The MatrixBlock at (0, 0) has dimensions different than rowsPerBlock: 2, and colsPerBlock: 3. Blocks on the right and bottom edges can have smaller dimensions. You may use the repartition method to fix this issue. - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:230) - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:222) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-10-05 11:32:55 ERROR TaskSetManager:75 - Task 0 in stage 14.0 failed 1 times; aborting job -2016-10-05 11:32:55 ERROR Executor:96 - Exception in task 1.0 in stage 14.0 (TID 43) -org.apache.spark.SparkException: The MatrixBlock at (1, 0) has dimensions different than rowsPerBlock: 2, and colsPerBlock: 3. Blocks on the right and bottom edges can have smaller dimensions. You may use the repartition method to fix this issue. - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:230) - at org.apache.spark.mllib.linalg.distributed.BlockMatrix$$anonfun$validate$5.apply(BlockMatrix.scala:222) - at scala.collection.Iterator$class.foreach(Iterator.scala:727) - at org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.rdd.RDD$$anonfun$foreach$1$$anonfun$apply$28.apply(RDD.scala:894) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) - at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) - at org.apache.spark.scheduler.Task.run(Task.scala:88) - at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) - at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) - at java.lang.Thread.run(Unknown Source) -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-10-05 11:32:56 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-10-05 11:32:56 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-10-05 11:32:56 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-10-05 15:35:06 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-10-05 15:35:07 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-10-05 15:35:08 INFO SecurityManager:59 - Changing view acls to: xubo -2016-10-05 15:35:08 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-10-05 15:35:08 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-10-05 15:35:08 INFO Slf4jLogger:80 - Slf4jLogger started -2016-10-05 15:35:08 INFO Remoting:74 - Starting remoting -2016-10-05 15:35:09 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@1.0.0.152:5242] -2016-10-05 15:35:09 INFO Utils:59 - Successfully started service 'sparkDriver' on port 5242. -2016-10-05 15:35:09 INFO SparkEnv:59 - Registering MapOutputTracker -2016-10-05 15:35:09 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-10-05 15:35:09 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-01bc7368-fc73-46c2-9f1a-14f397a27436 -2016-10-05 15:35:09 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-10-05 15:35:09 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-84c696d2-578a-4633-871e-66e60e730934\httpd-2d810480-6448-41c0-b380-97d22502f1ca -2016-10-05 15:35:09 INFO HttpServer:59 - Starting HTTP Server -2016-10-05 15:35:09 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-10-05 15:35:09 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:5243 -2016-10-05 15:35:09 INFO Utils:59 - Successfully started service 'HTTP file server' on port 5243. -2016-10-05 15:35:09 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-10-05 15:35:09 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-10-05 15:35:09 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-10-05 15:35:09 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-10-05 15:35:09 INFO SparkUI:59 - Started SparkUI at http://1.0.0.152:4040 -2016-10-05 15:35:09 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-10-05 15:35:09 INFO Executor:59 - Starting executor ID driver on host localhost -2016-10-05 15:35:10 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 5250. -2016-10-05 15:35:10 INFO NettyBlockTransferService:59 - Server created on 5250 -2016-10-05 15:35:10 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-10-05 15:35:10 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:5250 with 730.6 MB RAM, BlockManagerId(driver, localhost, 5250) -2016-10-05 15:35:10 INFO BlockManagerMaster:59 - Registered BlockManager -2016-10-05 15:35:10 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-10-05 15:35:10 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-10-05 15:35:10 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-10-05 15:35:10 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-10-05 15:35:10 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:5250 (size: 13.9 KB, free: 730.6 MB) -2016-10-05 15:35:10 INFO SparkContext:59 - Created broadcast 0 from textFile at TextFile.scala:30 -2016-10-05 15:35:12 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:100:f3%17, but we couldn't find any external IP address! -2016-10-05 15:35:12 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-10-05 15:35:12 INFO SparkContext:59 - Starting job: first at CsvRelation.scala:267 -2016-10-05 15:35:12 INFO DAGScheduler:59 - Got job 0 (first at CsvRelation.scala:267) with 1 output partitions -2016-10-05 15:35:12 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at CsvRelation.scala:267) -2016-10-05 15:35:12 INFO DAGScheduler:59 - Parents of final stage: List() -2016-10-05 15:35:12 INFO DAGScheduler:59 - Missing parents: List() -2016-10-05 15:35:12 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at filter at CsvRelation.scala:265), which has no missing parents -2016-10-05 15:35:12 INFO MemoryStore:59 - ensureFreeSpace(4776) called with curMem=171580, maxMem=766075207 -2016-10-05 15:35:12 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-10-05 15:35:12 INFO MemoryStore:59 - ensureFreeSpace(2790) called with curMem=176356, maxMem=766075207 -2016-10-05 15:35:12 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-10-05 15:35:12 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:5250 (size: 2.7 KB, free: 730.6 MB) -2016-10-05 15:35:12 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-10-05 15:35:12 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at filter at CsvRelation.scala:265) -2016-10-05 15:35:12 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-10-05 15:35:12 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2173 bytes) -2016-10-05 15:35:12 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-10-05 15:35:12 INFO HadoopRDD:59 - Input split: file:/E:/all/git/SparkLearning/file/data/sparkCSV/input/cars.csv:0+134 -2016-10-05 15:35:12 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-10-05 15:35:12 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-10-05 15:35:12 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-10-05 15:35:12 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-10-05 15:35:12 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-10-05 15:35:12 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2076 bytes result sent to driver -2016-10-05 15:35:12 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 203 ms on localhost (1/1) -2016-10-05 15:35:12 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-10-05 15:35:12 INFO DAGScheduler:59 - ResultStage 0 (first at CsvRelation.scala:267) finished in 0.218 s -2016-10-05 15:35:12 INFO DAGScheduler:59 - Job 0 finished: first at CsvRelation.scala:267, took 0.283366 s -2016-10-05 15:35:13 INFO MemoryStore:59 - ensureFreeSpace(88528) called with curMem=179146, maxMem=766075207 -2016-10-05 15:35:13 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 86.5 KB, free 730.3 MB) -2016-10-05 15:35:13 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=267674, maxMem=766075207 -2016-10-05 15:35:13 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.3 MB) -2016-10-05 15:35:13 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:5250 (size: 19.3 KB, free: 730.6 MB) -2016-10-05 15:35:13 INFO SparkContext:59 - Created broadcast 2 from textFile at TextFile.scala:30 -2016-10-05 15:35:13 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-10-05 15:35:13 INFO SparkContext:59 - Starting job: aggregate at InferSchema.scala:41 -2016-10-05 15:35:13 INFO DAGScheduler:59 - Got job 1 (aggregate at InferSchema.scala:41) with 1 output partitions -2016-10-05 15:35:13 INFO DAGScheduler:59 - Final stage: ResultStage 1(aggregate at InferSchema.scala:41) -2016-10-05 15:35:13 INFO DAGScheduler:59 - Parents of final stage: List() -2016-10-05 15:35:13 INFO DAGScheduler:59 - Missing parents: List() -2016-10-05 15:35:13 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at mapPartitions at CsvRelation.scala:91), which has no missing parents -2016-10-05 15:35:13 INFO MemoryStore:59 - ensureFreeSpace(5808) called with curMem=287462, maxMem=766075207 -2016-10-05 15:35:13 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.7 KB, free 730.3 MB) -2016-10-05 15:35:13 INFO MemoryStore:59 - ensureFreeSpace(3354) called with curMem=293270, maxMem=766075207 -2016-10-05 15:35:13 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.3 MB) -2016-10-05 15:35:13 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:5250 (size: 3.3 KB, free: 730.5 MB) -2016-10-05 15:35:13 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-10-05 15:35:13 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at mapPartitions at CsvRelation.scala:91) -2016-10-05 15:35:13 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-10-05 15:35:13 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2173 bytes) -2016-10-05 15:35:13 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-10-05 15:35:13 INFO HadoopRDD:59 - Input split: file:/E:/all/git/SparkLearning/file/data/sparkCSV/input/cars.csv:0+134 -2016-10-05 15:35:13 WARN CsvRelation$:303 - Ignoring empty line: -2016-10-05 15:35:13 WARN CsvRelation$:303 - Ignoring empty line: -2016-10-05 15:35:13 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2237 bytes result sent to driver -2016-10-05 15:35:13 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 77 ms on localhost (1/1) -2016-10-05 15:35:13 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-10-05 15:35:13 INFO DAGScheduler:59 - ResultStage 1 (aggregate at InferSchema.scala:41) finished in 0.077 s -2016-10-05 15:35:13 INFO DAGScheduler:59 - Job 1 finished: aggregate at InferSchema.scala:41, took 0.122240 s -2016-10-05 15:35:14 INFO MemoryStore:59 - ensureFreeSpace(230744) called with curMem=296624, maxMem=766075207 -2016-10-05 15:35:14 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-10-05 15:35:14 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=527368, maxMem=766075207 -2016-10-05 15:35:14 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-10-05 15:35:14 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:5250 (size: 19.3 KB, free: 730.5 MB) -2016-10-05 15:35:14 INFO SparkContext:59 - Created broadcast 4 from textFile at TextFile.scala:30 -2016-10-05 15:35:14 INFO deprecation:1049 - mapred.output.compress is deprecated. Instead, use mapreduce.output.fileoutputformat.compress -2016-10-05 15:35:14 INFO deprecation:1049 - mapred.output.compression.codec is deprecated. Instead, use mapreduce.output.fileoutputformat.compress.codec -2016-10-05 15:35:14 INFO deprecation:1049 - mapred.output.compression.type is deprecated. Instead, use mapreduce.output.fileoutputformat.compress.type -2016-10-05 15:35:14 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-10-05 15:35:14 INFO SparkContext:59 - Starting job: saveAsTextFile at package.scala:180 -2016-10-05 15:35:14 INFO DAGScheduler:59 - Got job 2 (saveAsTextFile at package.scala:180) with 1 output partitions -2016-10-05 15:35:14 INFO DAGScheduler:59 - Final stage: ResultStage 2(saveAsTextFile at package.scala:180) -2016-10-05 15:35:14 INFO DAGScheduler:59 - Parents of final stage: List() -2016-10-05 15:35:14 INFO DAGScheduler:59 - Missing parents: List() -2016-10-05 15:35:14 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[13] at saveAsTextFile at package.scala:180), which has no missing parents -2016-10-05 15:35:14 INFO MemoryStore:59 - ensureFreeSpace(133280) called with curMem=547156, maxMem=766075207 -2016-10-05 15:35:14 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 130.2 KB, free 729.9 MB) -2016-10-05 15:35:14 INFO MemoryStore:59 - ensureFreeSpace(46255) called with curMem=680436, maxMem=766075207 -2016-10-05 15:35:14 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 45.2 KB, free 729.9 MB) -2016-10-05 15:35:14 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:5250 (size: 45.2 KB, free: 730.5 MB) -2016-10-05 15:35:14 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-10-05 15:35:14 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[13] at saveAsTextFile at package.scala:180) -2016-10-05 15:35:14 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-10-05 15:35:14 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2173 bytes) -2016-10-05 15:35:14 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-10-05 15:35:14 INFO HadoopRDD:59 - Input split: file:/E:/all/git/SparkLearning/file/data/sparkCSV/input/cars.csv:0+134 -2016-10-05 15:35:14 WARN CsvRelation$:303 - Ignoring empty line: -2016-10-05 15:35:14 WARN CsvRelation$:303 - Ignoring empty line: -2016-10-05 15:35:14 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201610051535_0002_m_000000_2' to file:/E:/all/git/SparkLearning/file/data/sparkCSV/output/newcars.csv.gz/_temporary/0/task_201610051535_0002_m_000000 -2016-10-05 15:35:14 INFO SparkHadoopMapRedUtil:59 - attempt_201610051535_0002_m_000000_2: Committed -2016-10-05 15:35:14 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 2337 bytes result sent to driver -2016-10-05 15:35:14 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 341 ms on localhost (1/1) -2016-10-05 15:35:14 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-10-05 15:35:14 INFO DAGScheduler:59 - ResultStage 2 (saveAsTextFile at package.scala:180) finished in 0.341 s -2016-10-05 15:35:14 INFO DAGScheduler:59 - Job 2 finished: saveAsTextFile at package.scala:180, took 0.431198 s -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-10-05 15:35:15 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-10-05 15:35:15 INFO SparkUI:59 - Stopped Spark web UI at http://1.0.0.152:4040 -2016-10-05 15:35:15 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-10-05 15:35:15 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-10-05 15:35:15 INFO MemoryStore:59 - MemoryStore cleared -2016-10-05 15:35:15 INFO BlockManager:59 - BlockManager stopped -2016-10-05 15:35:15 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-10-05 15:35:15 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-10-05 15:35:15 INFO SparkContext:59 - Successfully stopped SparkContext -2016-10-05 15:35:15 INFO ShutdownHookManager:59 - Shutdown hook called -2016-10-05 15:35:15 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-84c696d2-578a-4633-871e-66e60e730934 -2016-10-05 15:35:15 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-10-05 15:36:13 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-10-05 15:36:15 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-10-05 15:36:15 INFO SecurityManager:59 - Changing view acls to: xubo -2016-10-05 15:36:15 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-10-05 15:36:15 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-10-05 15:36:16 INFO Slf4jLogger:80 - Slf4jLogger started -2016-10-05 15:36:16 INFO Remoting:74 - Starting remoting -2016-10-05 15:36:16 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@1.0.0.152:5270] -2016-10-05 15:36:16 INFO Utils:59 - Successfully started service 'sparkDriver' on port 5270. -2016-10-05 15:36:16 INFO SparkEnv:59 - Registering MapOutputTracker -2016-10-05 15:36:16 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-10-05 15:36:16 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-ebc77c33-aacc-4b74-8a25-4a63d3fd611e -2016-10-05 15:36:16 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-10-05 15:36:16 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-dca44e22-30b4-4d9d-893c-c5ecbd347288\httpd-4f3fd1b0-a57a-4948-b3f0-4d0706a2ceff -2016-10-05 15:36:16 INFO HttpServer:59 - Starting HTTP Server -2016-10-05 15:36:16 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-10-05 15:36:16 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:5271 -2016-10-05 15:36:16 INFO Utils:59 - Successfully started service 'HTTP file server' on port 5271. -2016-10-05 15:36:16 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-10-05 15:36:17 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-10-05 15:36:17 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-10-05 15:36:17 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-10-05 15:36:17 INFO SparkUI:59 - Started SparkUI at http://1.0.0.152:4040 -2016-10-05 15:36:17 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-10-05 15:36:17 INFO Executor:59 - Starting executor ID driver on host localhost -2016-10-05 15:36:17 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 5278. -2016-10-05 15:36:17 INFO NettyBlockTransferService:59 - Server created on 5278 -2016-10-05 15:36:17 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-10-05 15:36:17 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:5278 with 730.6 MB RAM, BlockManagerId(driver, localhost, 5278) -2016-10-05 15:36:17 INFO BlockManagerMaster:59 - Registered BlockManager -2016-10-05 15:36:18 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-10-05 15:36:18 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-10-05 15:36:18 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-10-05 15:36:18 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-10-05 15:36:18 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:5278 (size: 13.9 KB, free: 730.6 MB) -2016-10-05 15:36:18 INFO SparkContext:59 - Created broadcast 0 from textFile at TextFile.scala:30 -2016-10-05 15:36:19 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:100:f3%17, but we couldn't find any external IP address! -2016-10-05 15:36:19 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-10-05 15:36:19 INFO SparkContext:59 - Starting job: first at CsvRelation.scala:267 -2016-10-05 15:36:19 INFO DAGScheduler:59 - Got job 0 (first at CsvRelation.scala:267) with 1 output partitions -2016-10-05 15:36:19 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at CsvRelation.scala:267) -2016-10-05 15:36:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-10-05 15:36:19 INFO DAGScheduler:59 - Missing parents: List() -2016-10-05 15:36:19 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at filter at CsvRelation.scala:265), which has no missing parents -2016-10-05 15:36:19 INFO MemoryStore:59 - ensureFreeSpace(4776) called with curMem=171580, maxMem=766075207 -2016-10-05 15:36:19 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-10-05 15:36:19 INFO MemoryStore:59 - ensureFreeSpace(2790) called with curMem=176356, maxMem=766075207 -2016-10-05 15:36:19 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-10-05 15:36:19 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:5278 (size: 2.7 KB, free: 730.6 MB) -2016-10-05 15:36:19 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-10-05 15:36:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at filter at CsvRelation.scala:265) -2016-10-05 15:36:19 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-10-05 15:36:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2173 bytes) -2016-10-05 15:36:19 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-10-05 15:36:19 INFO HadoopRDD:59 - Input split: file:/E:/all/git/SparkLearning/file/data/sparkCSV/input/cars.csv:0+134 -2016-10-05 15:36:19 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-10-05 15:36:19 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-10-05 15:36:19 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-10-05 15:36:19 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-10-05 15:36:19 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-10-05 15:36:19 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2076 bytes result sent to driver -2016-10-05 15:36:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 109 ms on localhost (1/1) -2016-10-05 15:36:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-10-05 15:36:19 INFO DAGScheduler:59 - ResultStage 0 (first at CsvRelation.scala:267) finished in 0.132 s -2016-10-05 15:36:19 INFO DAGScheduler:59 - Job 0 finished: first at CsvRelation.scala:267, took 0.222134 s -2016-10-05 15:36:19 INFO BlockManagerInfo:59 - Removed broadcast_1_piece0 on localhost:5278 in memory (size: 2.7 KB, free: 730.6 MB) -2016-10-05 15:36:19 INFO ContextCleaner:59 - Cleaned accumulator 1 -2016-10-05 15:36:19 INFO BlockManagerInfo:59 - Removed broadcast_0_piece0 on localhost:5278 in memory (size: 13.9 KB, free: 730.6 MB) -2016-10-05 15:36:20 INFO MemoryStore:59 - ensureFreeSpace(88528) called with curMem=0, maxMem=766075207 -2016-10-05 15:36:20 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 86.5 KB, free 730.5 MB) -2016-10-05 15:36:20 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=88528, maxMem=766075207 -2016-10-05 15:36:20 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.5 MB) -2016-10-05 15:36:20 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:5278 (size: 19.3 KB, free: 730.6 MB) -2016-10-05 15:36:20 INFO SparkContext:59 - Created broadcast 2 from textFile at TextFile.scala:30 -2016-10-05 15:36:20 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-10-05 15:36:20 INFO SparkContext:59 - Starting job: aggregate at InferSchema.scala:41 -2016-10-05 15:36:20 INFO DAGScheduler:59 - Got job 1 (aggregate at InferSchema.scala:41) with 1 output partitions -2016-10-05 15:36:20 INFO DAGScheduler:59 - Final stage: ResultStage 1(aggregate at InferSchema.scala:41) -2016-10-05 15:36:20 INFO DAGScheduler:59 - Parents of final stage: List() -2016-10-05 15:36:20 INFO DAGScheduler:59 - Missing parents: List() -2016-10-05 15:36:20 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at mapPartitions at CsvRelation.scala:91), which has no missing parents -2016-10-05 15:36:20 INFO MemoryStore:59 - ensureFreeSpace(5808) called with curMem=108316, maxMem=766075207 -2016-10-05 15:36:20 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.7 KB, free 730.5 MB) -2016-10-05 15:36:20 INFO MemoryStore:59 - ensureFreeSpace(3354) called with curMem=114124, maxMem=766075207 -2016-10-05 15:36:20 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.5 MB) -2016-10-05 15:36:20 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:5278 (size: 3.3 KB, free: 730.6 MB) -2016-10-05 15:36:20 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-10-05 15:36:20 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at mapPartitions at CsvRelation.scala:91) -2016-10-05 15:36:20 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-10-05 15:36:20 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2173 bytes) -2016-10-05 15:36:20 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-10-05 15:36:21 INFO HadoopRDD:59 - Input split: file:/E:/all/git/SparkLearning/file/data/sparkCSV/input/cars.csv:0+134 -2016-10-05 15:36:21 WARN CsvRelation$:303 - Ignoring empty line: -2016-10-05 15:36:21 WARN CsvRelation$:303 - Ignoring empty line: -2016-10-05 15:36:21 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2237 bytes result sent to driver -2016-10-05 15:36:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 117 ms on localhost (1/1) -2016-10-05 15:36:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-10-05 15:36:21 INFO DAGScheduler:59 - ResultStage 1 (aggregate at InferSchema.scala:41) finished in 0.117 s -2016-10-05 15:36:21 INFO DAGScheduler:59 - Job 1 finished: aggregate at InferSchema.scala:41, took 0.462201 s -2016-10-05 15:36:21 INFO MemoryStore:59 - ensureFreeSpace(230744) called with curMem=117478, maxMem=766075207 -2016-10-05 15:36:21 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 225.3 KB, free 730.3 MB) -2016-10-05 15:36:21 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=348222, maxMem=766075207 -2016-10-05 15:36:21 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.2 MB) -2016-10-05 15:36:21 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:5278 (size: 19.3 KB, free: 730.5 MB) -2016-10-05 15:36:21 INFO SparkContext:59 - Created broadcast 4 from textFile at TextFile.scala:30 -2016-10-05 15:36:21 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-10-05 15:36:21 INFO SparkContext:59 - Starting job: show at readCsvBySparkSQLLoadWithCompressedOutput.scala:31 -2016-10-05 15:36:21 INFO DAGScheduler:59 - Got job 2 (show at readCsvBySparkSQLLoadWithCompressedOutput.scala:31) with 1 output partitions -2016-10-05 15:36:21 INFO DAGScheduler:59 - Final stage: ResultStage 2(show at readCsvBySparkSQLLoadWithCompressedOutput.scala:31) -2016-10-05 15:36:21 INFO DAGScheduler:59 - Parents of final stage: List() -2016-10-05 15:36:21 INFO DAGScheduler:59 - Missing parents: List() -2016-10-05 15:36:21 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[11] at show at readCsvBySparkSQLLoadWithCompressedOutput.scala:31), which has no missing parents -2016-10-05 15:36:21 INFO MemoryStore:59 - ensureFreeSpace(6896) called with curMem=368010, maxMem=766075207 -2016-10-05 15:36:21 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 6.7 KB, free 730.2 MB) -2016-10-05 15:36:21 INFO MemoryStore:59 - ensureFreeSpace(3884) called with curMem=374906, maxMem=766075207 -2016-10-05 15:36:21 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 3.8 KB, free 730.2 MB) -2016-10-05 15:36:21 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:5278 (size: 3.8 KB, free: 730.5 MB) -2016-10-05 15:36:21 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-10-05 15:36:21 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[11] at show at readCsvBySparkSQLLoadWithCompressedOutput.scala:31) -2016-10-05 15:36:21 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-10-05 15:36:21 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2173 bytes) -2016-10-05 15:36:21 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-10-05 15:36:21 INFO HadoopRDD:59 - Input split: file:/E:/all/git/SparkLearning/file/data/sparkCSV/input/cars.csv:0+134 -2016-10-05 15:36:21 WARN CsvRelation$:303 - Ignoring empty line: -2016-10-05 15:36:21 WARN CsvRelation$:303 - Ignoring empty line: -2016-10-05 15:36:21 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 2655 bytes result sent to driver -2016-10-05 15:36:21 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 44 ms on localhost (1/1) -2016-10-05 15:36:21 INFO DAGScheduler:59 - ResultStage 2 (show at readCsvBySparkSQLLoadWithCompressedOutput.scala:31) finished in 0.044 s -2016-10-05 15:36:21 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-10-05 15:36:21 INFO DAGScheduler:59 - Job 2 finished: show at readCsvBySparkSQLLoadWithCompressedOutput.scala:31, took 0.069561 s -2016-10-05 15:36:21 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-10-05 15:36:21 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-10-05 15:36:21 INFO SparkUI:59 - Stopped Spark web UI at http://1.0.0.152:4040 -2016-10-05 15:36:21 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-10-05 15:36:21 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-10-05 15:36:21 INFO MemoryStore:59 - MemoryStore cleared -2016-10-05 15:36:21 INFO BlockManager:59 - BlockManager stopped -2016-10-05 15:36:21 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-10-05 15:36:21 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-10-05 15:36:21 INFO SparkContext:59 - Successfully stopped SparkContext -2016-10-05 15:36:21 INFO ShutdownHookManager:59 - Shutdown hook called -2016-10-05 15:36:21 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-dca44e22-30b4-4d9d-893c-c5ecbd347288 -2016-10-05 15:36:21 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-10-05 15:36:21 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-10-05 15:40:48 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-10-05 15:40:51 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-10-05 15:40:51 INFO SecurityManager:59 - Changing view acls to: xubo -2016-10-05 15:40:51 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-10-05 15:40:51 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-10-05 15:40:51 INFO Slf4jLogger:80 - Slf4jLogger started -2016-10-05 15:40:52 INFO Remoting:74 - Starting remoting -2016-10-05 15:40:52 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@1.0.0.152:5553] -2016-10-05 15:40:52 INFO Utils:59 - Successfully started service 'sparkDriver' on port 5553. -2016-10-05 15:40:52 INFO SparkEnv:59 - Registering MapOutputTracker -2016-10-05 15:40:52 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-10-05 15:40:52 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-e2f28422-b8b4-4bde-9c16-9955d38f40e4 -2016-10-05 15:40:52 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-10-05 15:40:52 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-e6ef34a9-ed64-443b-b7d7-3987e19b441f\httpd-bbfac156-b8ff-4eda-a708-b8b215396aa1 -2016-10-05 15:40:52 INFO HttpServer:59 - Starting HTTP Server -2016-10-05 15:40:52 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-10-05 15:40:52 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:5554 -2016-10-05 15:40:52 INFO Utils:59 - Successfully started service 'HTTP file server' on port 5554. -2016-10-05 15:40:52 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-10-05 15:40:52 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-10-05 15:40:52 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-10-05 15:40:52 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-10-05 15:40:52 INFO SparkUI:59 - Started SparkUI at http://1.0.0.152:4040 -2016-10-05 15:40:52 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-10-05 15:40:52 INFO Executor:59 - Starting executor ID driver on host localhost -2016-10-05 15:40:52 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 5561. -2016-10-05 15:40:52 INFO NettyBlockTransferService:59 - Server created on 5561 -2016-10-05 15:40:52 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-10-05 15:40:52 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:5561 with 730.6 MB RAM, BlockManagerId(driver, localhost, 5561) -2016-10-05 15:40:52 INFO BlockManagerMaster:59 - Registered BlockManager -2016-10-05 15:40:53 INFO SparkContext:59 - Invoking stop() from shutdown hook -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-10-05 15:40:53 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-10-05 15:40:53 INFO SparkUI:59 - Stopped Spark web UI at http://1.0.0.152:4040 -2016-10-05 15:40:53 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-10-05 15:40:53 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-10-05 15:40:53 INFO MemoryStore:59 - MemoryStore cleared -2016-10-05 15:40:53 INFO BlockManager:59 - BlockManager stopped -2016-10-05 15:40:53 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-10-05 15:40:53 INFO SparkContext:59 - Successfully stopped SparkContext -2016-10-05 15:40:53 INFO ShutdownHookManager:59 - Shutdown hook called -2016-10-05 15:40:53 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-e6ef34a9-ed64-443b-b7d7-3987e19b441f -2016-10-05 15:40:53 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-10-05 15:40:53 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-10-05 15:40:53 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. -2016-10-05 15:43:12 INFO SparkContext:59 - Running Spark version 1.5.2 -2016-10-05 15:43:13 WARN NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable -2016-10-05 15:43:14 INFO SecurityManager:59 - Changing view acls to: xubo -2016-10-05 15:43:14 INFO SecurityManager:59 - Changing modify acls to: xubo -2016-10-05 15:43:14 INFO SecurityManager:59 - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(xubo); users with modify permissions: Set(xubo) -2016-10-05 15:43:14 INFO Slf4jLogger:80 - Slf4jLogger started -2016-10-05 15:43:14 INFO Remoting:74 - Starting remoting -2016-10-05 15:43:15 INFO Remoting:74 - Remoting started; listening on addresses :[akka.tcp://sparkDriver@1.0.0.152:5693] -2016-10-05 15:43:15 INFO Utils:59 - Successfully started service 'sparkDriver' on port 5693. -2016-10-05 15:43:15 INFO SparkEnv:59 - Registering MapOutputTracker -2016-10-05 15:43:15 INFO SparkEnv:59 - Registering BlockManagerMaster -2016-10-05 15:43:15 INFO DiskBlockManager:59 - Created local directory at C:\Users\xubo\AppData\Local\Temp\blockmgr-6c84fc6e-13f6-42c6-8462-4570102f4585 -2016-10-05 15:43:15 INFO MemoryStore:59 - MemoryStore started with capacity 730.6 MB -2016-10-05 15:43:15 INFO HttpFileServer:59 - HTTP File server directory is C:\Users\xubo\AppData\Local\Temp\spark-12e5f4cc-940f-4466-99fb-f199d51a0d42\httpd-67d075bf-4921-4e39-b381-d10a2082102f -2016-10-05 15:43:15 INFO HttpServer:59 - Starting HTTP Server -2016-10-05 15:43:15 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-10-05 15:43:15 INFO AbstractConnector:338 - Started SocketConnector@0.0.0.0:5694 -2016-10-05 15:43:15 INFO Utils:59 - Successfully started service 'HTTP file server' on port 5694. -2016-10-05 15:43:15 INFO SparkEnv:59 - Registering OutputCommitCoordinator -2016-10-05 15:43:15 INFO Server:272 - jetty-8.y.z-SNAPSHOT -2016-10-05 15:43:15 INFO AbstractConnector:338 - Started SelectChannelConnector@0.0.0.0:4040 -2016-10-05 15:43:15 INFO Utils:59 - Successfully started service 'SparkUI' on port 4040. -2016-10-05 15:43:15 INFO SparkUI:59 - Started SparkUI at http://1.0.0.152:4040 -2016-10-05 15:43:15 WARN MetricsSystem:71 - Using default name DAGScheduler for source because spark.app.id is not set. -2016-10-05 15:43:15 INFO Executor:59 - Starting executor ID driver on host localhost -2016-10-05 15:43:15 INFO Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 5701. -2016-10-05 15:43:15 INFO NettyBlockTransferService:59 - Server created on 5701 -2016-10-05 15:43:15 INFO BlockManagerMaster:59 - Trying to register BlockManager -2016-10-05 15:43:15 INFO BlockManagerMasterEndpoint:59 - Registering block manager localhost:5701 with 730.6 MB RAM, BlockManagerId(driver, localhost, 5701) -2016-10-05 15:43:15 INFO BlockManagerMaster:59 - Registered BlockManager -2016-10-05 15:43:16 INFO MemoryStore:59 - ensureFreeSpace(157304) called with curMem=0, maxMem=766075207 -2016-10-05 15:43:16 INFO MemoryStore:59 - Block broadcast_0 stored as values in memory (estimated size 153.6 KB, free 730.4 MB) -2016-10-05 15:43:16 INFO MemoryStore:59 - ensureFreeSpace(14276) called with curMem=157304, maxMem=766075207 -2016-10-05 15:43:16 INFO MemoryStore:59 - Block broadcast_0_piece0 stored as bytes in memory (estimated size 13.9 KB, free 730.4 MB) -2016-10-05 15:43:16 INFO BlockManagerInfo:59 - Added broadcast_0_piece0 in memory on localhost:5701 (size: 13.9 KB, free: 730.6 MB) -2016-10-05 15:43:16 INFO SparkContext:59 - Created broadcast 0 from textFile at TextFile.scala:30 -2016-10-05 15:43:17 WARN :139 - Your hostname, xubo-PC resolves to a loopback/non-reachable address: fe80:0:0:0:200:5efe:100:f3%17, but we couldn't find any external IP address! -2016-10-05 15:43:17 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-10-05 15:43:17 INFO SparkContext:59 - Starting job: first at CsvRelation.scala:267 -2016-10-05 15:43:17 INFO DAGScheduler:59 - Got job 0 (first at CsvRelation.scala:267) with 1 output partitions -2016-10-05 15:43:17 INFO DAGScheduler:59 - Final stage: ResultStage 0(first at CsvRelation.scala:267) -2016-10-05 15:43:17 INFO DAGScheduler:59 - Parents of final stage: List() -2016-10-05 15:43:17 INFO DAGScheduler:59 - Missing parents: List() -2016-10-05 15:43:17 INFO DAGScheduler:59 - Submitting ResultStage 0 (MapPartitionsRDD[2] at filter at CsvRelation.scala:265), which has no missing parents -2016-10-05 15:43:17 INFO MemoryStore:59 - ensureFreeSpace(4776) called with curMem=171580, maxMem=766075207 -2016-10-05 15:43:17 INFO MemoryStore:59 - Block broadcast_1 stored as values in memory (estimated size 4.7 KB, free 730.4 MB) -2016-10-05 15:43:18 INFO MemoryStore:59 - ensureFreeSpace(2790) called with curMem=176356, maxMem=766075207 -2016-10-05 15:43:18 INFO MemoryStore:59 - Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.7 KB, free 730.4 MB) -2016-10-05 15:43:18 INFO BlockManagerInfo:59 - Added broadcast_1_piece0 in memory on localhost:5701 (size: 2.7 KB, free: 730.6 MB) -2016-10-05 15:43:18 INFO SparkContext:59 - Created broadcast 1 from broadcast at DAGScheduler.scala:861 -2016-10-05 15:43:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[2] at filter at CsvRelation.scala:265) -2016-10-05 15:43:18 INFO TaskSchedulerImpl:59 - Adding task set 0.0 with 1 tasks -2016-10-05 15:43:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2173 bytes) -2016-10-05 15:43:18 INFO Executor:59 - Running task 0.0 in stage 0.0 (TID 0) -2016-10-05 15:43:18 INFO HadoopRDD:59 - Input split: file:/E:/all/git/SparkLearning/file/data/sparkCSV/input/cars.csv:0+134 -2016-10-05 15:43:18 INFO deprecation:1049 - mapred.tip.id is deprecated. Instead, use mapreduce.task.id -2016-10-05 15:43:18 INFO deprecation:1049 - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id -2016-10-05 15:43:18 INFO deprecation:1049 - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap -2016-10-05 15:43:18 INFO deprecation:1049 - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition -2016-10-05 15:43:18 INFO deprecation:1049 - mapred.job.id is deprecated. Instead, use mapreduce.job.id -2016-10-05 15:43:18 INFO Executor:59 - Finished task 0.0 in stage 0.0 (TID 0). 2076 bytes result sent to driver -2016-10-05 15:43:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 0.0 (TID 0) in 83 ms on localhost (1/1) -2016-10-05 15:43:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 0.0, whose tasks have all completed, from pool -2016-10-05 15:43:18 INFO DAGScheduler:59 - ResultStage 0 (first at CsvRelation.scala:267) finished in 0.095 s -2016-10-05 15:43:18 INFO DAGScheduler:59 - Job 0 finished: first at CsvRelation.scala:267, took 0.186249 s -2016-10-05 15:43:18 INFO MemoryStore:59 - ensureFreeSpace(88528) called with curMem=179146, maxMem=766075207 -2016-10-05 15:43:18 INFO MemoryStore:59 - Block broadcast_2 stored as values in memory (estimated size 86.5 KB, free 730.3 MB) -2016-10-05 15:43:18 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=267674, maxMem=766075207 -2016-10-05 15:43:18 INFO MemoryStore:59 - Block broadcast_2_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.3 MB) -2016-10-05 15:43:18 INFO BlockManagerInfo:59 - Added broadcast_2_piece0 in memory on localhost:5701 (size: 19.3 KB, free: 730.6 MB) -2016-10-05 15:43:18 INFO SparkContext:59 - Created broadcast 2 from textFile at TextFile.scala:30 -2016-10-05 15:43:18 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-10-05 15:43:18 INFO SparkContext:59 - Starting job: aggregate at InferSchema.scala:41 -2016-10-05 15:43:18 INFO DAGScheduler:59 - Got job 1 (aggregate at InferSchema.scala:41) with 1 output partitions -2016-10-05 15:43:18 INFO DAGScheduler:59 - Final stage: ResultStage 1(aggregate at InferSchema.scala:41) -2016-10-05 15:43:18 INFO DAGScheduler:59 - Parents of final stage: List() -2016-10-05 15:43:18 INFO DAGScheduler:59 - Missing parents: List() -2016-10-05 15:43:18 INFO DAGScheduler:59 - Submitting ResultStage 1 (MapPartitionsRDD[5] at mapPartitions at CsvRelation.scala:91), which has no missing parents -2016-10-05 15:43:18 INFO MemoryStore:59 - ensureFreeSpace(5808) called with curMem=287462, maxMem=766075207 -2016-10-05 15:43:18 INFO MemoryStore:59 - Block broadcast_3 stored as values in memory (estimated size 5.7 KB, free 730.3 MB) -2016-10-05 15:43:18 INFO MemoryStore:59 - ensureFreeSpace(3354) called with curMem=293270, maxMem=766075207 -2016-10-05 15:43:18 INFO MemoryStore:59 - Block broadcast_3_piece0 stored as bytes in memory (estimated size 3.3 KB, free 730.3 MB) -2016-10-05 15:43:18 INFO BlockManagerInfo:59 - Added broadcast_3_piece0 in memory on localhost:5701 (size: 3.3 KB, free: 730.5 MB) -2016-10-05 15:43:18 INFO SparkContext:59 - Created broadcast 3 from broadcast at DAGScheduler.scala:861 -2016-10-05 15:43:18 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[5] at mapPartitions at CsvRelation.scala:91) -2016-10-05 15:43:18 INFO TaskSchedulerImpl:59 - Adding task set 1.0 with 1 tasks -2016-10-05 15:43:18 INFO TaskSetManager:59 - Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2173 bytes) -2016-10-05 15:43:18 INFO Executor:59 - Running task 0.0 in stage 1.0 (TID 1) -2016-10-05 15:43:18 INFO HadoopRDD:59 - Input split: file:/E:/all/git/SparkLearning/file/data/sparkCSV/input/cars.csv:0+134 -2016-10-05 15:43:18 WARN CsvRelation$:303 - Ignoring empty line: -2016-10-05 15:43:18 WARN CsvRelation$:303 - Ignoring empty line: -2016-10-05 15:43:18 INFO Executor:59 - Finished task 0.0 in stage 1.0 (TID 1). 2237 bytes result sent to driver -2016-10-05 15:43:18 INFO TaskSetManager:59 - Finished task 0.0 in stage 1.0 (TID 1) in 47 ms on localhost (1/1) -2016-10-05 15:43:18 INFO TaskSchedulerImpl:59 - Removed TaskSet 1.0, whose tasks have all completed, from pool -2016-10-05 15:43:18 INFO DAGScheduler:59 - ResultStage 1 (aggregate at InferSchema.scala:41) finished in 0.047 s -2016-10-05 15:43:18 INFO DAGScheduler:59 - Job 1 finished: aggregate at InferSchema.scala:41, took 0.082704 s -2016-10-05 15:43:19 INFO MemoryStore:59 - ensureFreeSpace(230744) called with curMem=296624, maxMem=766075207 -2016-10-05 15:43:19 INFO MemoryStore:59 - Block broadcast_4 stored as values in memory (estimated size 225.3 KB, free 730.1 MB) -2016-10-05 15:43:19 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=527368, maxMem=766075207 -2016-10-05 15:43:19 INFO MemoryStore:59 - Block broadcast_4_piece0 stored as bytes in memory (estimated size 19.3 KB, free 730.1 MB) -2016-10-05 15:43:19 INFO BlockManagerInfo:59 - Added broadcast_4_piece0 in memory on localhost:5701 (size: 19.3 KB, free: 730.5 MB) -2016-10-05 15:43:19 INFO SparkContext:59 - Created broadcast 4 from textFile at TextFile.scala:30 -2016-10-05 15:43:19 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-10-05 15:43:19 INFO SparkContext:59 - Starting job: show at readCsvBySparkSQLLoadWithCompressedOutput.scala:31 -2016-10-05 15:43:19 INFO DAGScheduler:59 - Got job 2 (show at readCsvBySparkSQLLoadWithCompressedOutput.scala:31) with 1 output partitions -2016-10-05 15:43:19 INFO DAGScheduler:59 - Final stage: ResultStage 2(show at readCsvBySparkSQLLoadWithCompressedOutput.scala:31) -2016-10-05 15:43:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-10-05 15:43:19 INFO DAGScheduler:59 - Missing parents: List() -2016-10-05 15:43:19 INFO DAGScheduler:59 - Submitting ResultStage 2 (MapPartitionsRDD[11] at show at readCsvBySparkSQLLoadWithCompressedOutput.scala:31), which has no missing parents -2016-10-05 15:43:19 INFO MemoryStore:59 - ensureFreeSpace(6896) called with curMem=547156, maxMem=766075207 -2016-10-05 15:43:19 INFO MemoryStore:59 - Block broadcast_5 stored as values in memory (estimated size 6.7 KB, free 730.1 MB) -2016-10-05 15:43:19 INFO MemoryStore:59 - ensureFreeSpace(3884) called with curMem=554052, maxMem=766075207 -2016-10-05 15:43:19 INFO MemoryStore:59 - Block broadcast_5_piece0 stored as bytes in memory (estimated size 3.8 KB, free 730.1 MB) -2016-10-05 15:43:19 INFO BlockManagerInfo:59 - Added broadcast_5_piece0 in memory on localhost:5701 (size: 3.8 KB, free: 730.5 MB) -2016-10-05 15:43:19 INFO SparkContext:59 - Created broadcast 5 from broadcast at DAGScheduler.scala:861 -2016-10-05 15:43:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[11] at show at readCsvBySparkSQLLoadWithCompressedOutput.scala:31) -2016-10-05 15:43:19 INFO TaskSchedulerImpl:59 - Adding task set 2.0 with 1 tasks -2016-10-05 15:43:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 2.0 (TID 2, localhost, PROCESS_LOCAL, 2173 bytes) -2016-10-05 15:43:19 INFO Executor:59 - Running task 0.0 in stage 2.0 (TID 2) -2016-10-05 15:43:19 INFO HadoopRDD:59 - Input split: file:/E:/all/git/SparkLearning/file/data/sparkCSV/input/cars.csv:0+134 -2016-10-05 15:43:19 WARN CsvRelation$:303 - Ignoring empty line: -2016-10-05 15:43:19 WARN CsvRelation$:303 - Ignoring empty line: -2016-10-05 15:43:19 INFO Executor:59 - Finished task 0.0 in stage 2.0 (TID 2). 2655 bytes result sent to driver -2016-10-05 15:43:19 INFO TaskSetManager:59 - Finished task 0.0 in stage 2.0 (TID 2) in 43 ms on localhost (1/1) -2016-10-05 15:43:19 INFO DAGScheduler:59 - ResultStage 2 (show at readCsvBySparkSQLLoadWithCompressedOutput.scala:31) finished in 0.043 s -2016-10-05 15:43:19 INFO TaskSchedulerImpl:59 - Removed TaskSet 2.0, whose tasks have all completed, from pool -2016-10-05 15:43:19 INFO DAGScheduler:59 - Job 2 finished: show at readCsvBySparkSQLLoadWithCompressedOutput.scala:31, took 0.067759 s -2016-10-05 15:43:19 INFO MemoryStore:59 - ensureFreeSpace(230744) called with curMem=557936, maxMem=766075207 -2016-10-05 15:43:19 INFO MemoryStore:59 - Block broadcast_6 stored as values in memory (estimated size 225.3 KB, free 729.8 MB) -2016-10-05 15:43:19 INFO MemoryStore:59 - ensureFreeSpace(19788) called with curMem=788680, maxMem=766075207 -2016-10-05 15:43:19 INFO MemoryStore:59 - Block broadcast_6_piece0 stored as bytes in memory (estimated size 19.3 KB, free 729.8 MB) -2016-10-05 15:43:19 INFO BlockManagerInfo:59 - Added broadcast_6_piece0 in memory on localhost:5701 (size: 19.3 KB, free: 730.5 MB) -2016-10-05 15:43:19 INFO SparkContext:59 - Created broadcast 6 from textFile at TextFile.scala:30 -2016-10-05 15:43:19 INFO deprecation:1049 - mapred.output.compress is deprecated. Instead, use mapreduce.output.fileoutputformat.compress -2016-10-05 15:43:19 INFO deprecation:1049 - mapred.output.compression.codec is deprecated. Instead, use mapreduce.output.fileoutputformat.compress.codec -2016-10-05 15:43:19 INFO deprecation:1049 - mapred.output.compression.type is deprecated. Instead, use mapreduce.output.fileoutputformat.compress.type -2016-10-05 15:43:19 INFO FileInputFormat:247 - Total input paths to process : 1 -2016-10-05 15:43:19 INFO SparkContext:59 - Starting job: saveAsTextFile at package.scala:180 -2016-10-05 15:43:19 INFO DAGScheduler:59 - Got job 3 (saveAsTextFile at package.scala:180) with 1 output partitions -2016-10-05 15:43:19 INFO DAGScheduler:59 - Final stage: ResultStage 3(saveAsTextFile at package.scala:180) -2016-10-05 15:43:19 INFO DAGScheduler:59 - Parents of final stage: List() -2016-10-05 15:43:19 INFO DAGScheduler:59 - Missing parents: List() -2016-10-05 15:43:19 INFO DAGScheduler:59 - Submitting ResultStage 3 (MapPartitionsRDD[19] at saveAsTextFile at package.scala:180), which has no missing parents -2016-10-05 15:43:19 INFO MemoryStore:59 - ensureFreeSpace(133288) called with curMem=808468, maxMem=766075207 -2016-10-05 15:43:19 INFO MemoryStore:59 - Block broadcast_7 stored as values in memory (estimated size 130.2 KB, free 729.7 MB) -2016-10-05 15:43:19 INFO MemoryStore:59 - ensureFreeSpace(46254) called with curMem=941756, maxMem=766075207 -2016-10-05 15:43:19 INFO MemoryStore:59 - Block broadcast_7_piece0 stored as bytes in memory (estimated size 45.2 KB, free 729.6 MB) -2016-10-05 15:43:19 INFO BlockManagerInfo:59 - Added broadcast_7_piece0 in memory on localhost:5701 (size: 45.2 KB, free: 730.5 MB) -2016-10-05 15:43:19 INFO SparkContext:59 - Created broadcast 7 from broadcast at DAGScheduler.scala:861 -2016-10-05 15:43:19 INFO DAGScheduler:59 - Submitting 1 missing tasks from ResultStage 3 (MapPartitionsRDD[19] at saveAsTextFile at package.scala:180) -2016-10-05 15:43:19 INFO TaskSchedulerImpl:59 - Adding task set 3.0 with 1 tasks -2016-10-05 15:43:19 INFO TaskSetManager:59 - Starting task 0.0 in stage 3.0 (TID 3, localhost, PROCESS_LOCAL, 2173 bytes) -2016-10-05 15:43:19 INFO Executor:59 - Running task 0.0 in stage 3.0 (TID 3) -2016-10-05 15:43:19 INFO HadoopRDD:59 - Input split: file:/E:/all/git/SparkLearning/file/data/sparkCSV/input/cars.csv:0+134 -2016-10-05 15:43:20 WARN CsvRelation$:303 - Ignoring empty line: -2016-10-05 15:43:20 WARN CsvRelation$:303 - Ignoring empty line: -2016-10-05 15:43:20 INFO FileOutputCommitter:439 - Saved output of task 'attempt_201610051543_0003_m_000000_3' to file:/E:/all/git/SparkLearning/file/data/sparkCSV/output/newcars2.csv.gz/_temporary/0/task_201610051543_0003_m_000000 -2016-10-05 15:43:20 INFO SparkHadoopMapRedUtil:59 - attempt_201610051543_0003_m_000000_3: Committed -2016-10-05 15:43:20 INFO Executor:59 - Finished task 0.0 in stage 3.0 (TID 3). 2337 bytes result sent to driver -2016-10-05 15:43:20 INFO TaskSetManager:59 - Finished task 0.0 in stage 3.0 (TID 3) in 347 ms on localhost (1/1) -2016-10-05 15:43:20 INFO TaskSchedulerImpl:59 - Removed TaskSet 3.0, whose tasks have all completed, from pool -2016-10-05 15:43:20 INFO DAGScheduler:59 - ResultStage 3 (saveAsTextFile at package.scala:180) finished in 0.348 s -2016-10-05 15:43:20 INFO DAGScheduler:59 - Job 3 finished: saveAsTextFile at package.scala:180, took 0.446529 s -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static/sql,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL/json,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/SQL,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/metrics/json,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/api,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/static,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors/json,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/executors,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment/json,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/environment,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage/json,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/storage,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/pool,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/stage,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages/json,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/stages,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/job,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs/json,null} -2016-10-05 15:43:20 INFO ContextHandler:843 - stopped o.s.j.s.ServletContextHandler{/jobs,null} -2016-10-05 15:43:20 INFO SparkUI:59 - Stopped Spark web UI at http://1.0.0.152:4040 -2016-10-05 15:43:20 INFO DAGScheduler:59 - Stopping DAGScheduler -2016-10-05 15:43:20 INFO MapOutputTrackerMasterEndpoint:59 - MapOutputTrackerMasterEndpoint stopped! -2016-10-05 15:43:20 INFO MemoryStore:59 - MemoryStore cleared -2016-10-05 15:43:20 INFO BlockManager:59 - BlockManager stopped -2016-10-05 15:43:20 INFO BlockManagerMaster:59 - BlockManagerMaster stopped -2016-10-05 15:43:20 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:59 - OutputCommitCoordinator stopped! -2016-10-05 15:43:20 INFO SparkContext:59 - Successfully stopped SparkContext -2016-10-05 15:43:20 INFO ShutdownHookManager:59 - Shutdown hook called -2016-10-05 15:43:20 INFO ShutdownHookManager:59 - Deleting directory C:\Users\xubo\AppData\Local\Temp\spark-12e5f4cc-940f-4466-99fb-f199d51a0d42 -2016-10-05 15:43:20 INFO RemoteActorRefProvider$RemotingTerminator:74 - Shutting down remote daemon. -2016-10-05 15:43:20 INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports. diff --git "a/docs/Issues/~$\346\212\245\351\224\231\346\203\205\345\206\265\346\261\207\346\212\245.docx" "b/docs/Issues/~$\346\212\245\351\224\231\346\203\205\345\206\265\346\261\207\346\212\245.docx" deleted file mode 100644 index 88f505e..0000000 Binary files "a/docs/Issues/~$\346\212\245\351\224\231\346\203\205\345\206\265\346\261\207\346\212\245.docx" and /dev/null differ diff --git "a/docs/LinuxLearning/Ubuntu\345\255\246\344\271\240100\344\271\213\346\250\241\347\211\210.md" "b/docs/LinuxLearning/Ubuntu\345\255\246\344\271\240100\344\271\213\346\250\241\347\211\210.md" new file mode 100644 index 0000000..f22c04d --- /dev/null +++ "b/docs/LinuxLearning/Ubuntu\345\255\246\344\271\240100\344\271\213\346\250\241\347\211\210.md" @@ -0,0 +1,22 @@ + +更多代码请见：https://github.com/xubo245 + +Ubuntu学习 + +# 1.解释 # + + + + +# 2.代码： # + + + +# 3.结果： # + + + +参考 + + 【1】https://github.com/xubo245 + 【2]http://blog.csdn.net/xubo245/ diff --git "a/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2401\344\271\213\347\273\231Ubuntu\347\263\273\347\273\237\346\201\242\345\244\215\345\217\246\344\270\200\345\217\260\346\234\272\345\231\250\347\232\204\347\243\201\347\233\230.md" "b/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2401\344\271\213\347\273\231Ubuntu\347\263\273\347\273\237\346\201\242\345\244\215\345\217\246\344\270\200\345\217\260\346\234\272\345\231\250\347\232\204\347\243\201\347\233\230.md" new file mode 100644 index 0000000..e529a03 --- /dev/null +++ "b/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2401\344\271\213\347\273\231Ubuntu\347\263\273\347\273\237\346\201\242\345\244\215\345\217\246\344\270\200\345\217\260\346\234\272\345\231\250\347\232\204\347\243\201\347\233\230.md" @@ -0,0 +1,93 @@ + +更多代码请见：https://github.com/xubo245 + +Ubuntu学习 + +# 1.解释 # + +原来系统应为RAID机制启动不了，包 Non RAID disk错误，RAID之后启动不了，dhcp获取ip，但是tftp timeout，所以将磁盘放到好的主机可以恢复 + + +# 2.代码： # +使用parted指令： parted为分区指令，参考【3】 + + hadoop@Mcnode7:~$ sudo parted /dev/sdb + GNU Parted 2.3 + Using /dev/sdb + Welcome to GNU Parted! Type 'help' to view a list of commands. + (parted) rm 1 + Error: Can't have a partition outside the disk! + (parted) print + Error: Can't have a partition outside the disk! + (parted) mklabel gpt + (parted) print + Model: ATA Hitachi HDT72105 (scsi) + Disk /dev/sdb: 500GB + Sector size (logical/physical): 512B/512B + Partition Table: gpt + + Number Start End Size File system Name Flags + + (parted) mkpart primary 0% 100% + (parted) print + Model: ATA Hitachi HDT72105 (scsi) + Disk /dev/sdb: 500GB + Sector size (logical/physical): 512B/512B + Partition Table: gpt + + Number Start End Size File system Name Flags + 1 1049kB 500GB 500GB ext4 primary + + +# 3.结果： # + + +查看：发下原来Mcnode5的数据都还在，而且还可以访问 + + hadoop@Mcnode7:~$ sudo fdisk -lu + [sudo] password for hadoop: + + Disk /dev/sda: 500.1 GB, 500107862016 bytes + 255 heads, 63 sectors/track, 60801 cylinders, total 976773168 sectors + Units = sectors of 1 * 512 = 512 bytes + Sector size (logical/physical): 512 bytes / 512 bytes + I/O size (minimum/optimal): 512 bytes / 512 bytes + Disk identifier: 0x000db939 + + Device Boot Start End Blocks Id System + /dev/sda1 * 2048 951625727 475811840 83 Linux + /dev/sda2 951627774 976771071 12571649 5 Extended + /dev/sda5 951627776 976771071 12571648 82 Linux swap / Solaris + + WARNING: GPT (GUID Partition Table) detected on '/dev/sdb'! The util fdisk doesn't support GPT. Use GNU Parted. + + + Disk /dev/sdb: 500.1 GB, 500107862016 bytes + 255 heads, 63 sectors/track, 60801 cylinders, total 976773168 sectors + Units = sectors of 1 * 512 = 512 bytes + Sector size (logical/physical): 512 bytes / 512 bytes + I/O size (minimum/optimal): 512 bytes / 512 bytes + Disk identifier: 0x00000000 + + Device Boot Start End Blocks Id System + /dev/sdb1 1 976773167 488386583+ ee GPT + hadoop@Mcnode7:~$ cd /media/hadoop/7c63f77d-11b0-4769-bf60-5380d8ae7b9e + hadoop@Mcnode7:/media/hadoop/7c63f77d-11b0-4769-bf60-5380d8ae7b9e$ ls + bin boot cdrom dev etc home initrd.img lib lib64 lost+found media mnt opt proc root run sbin srv sys tmp usr var vmlinuz + hadoop@Mcnode7:/media/hadoop/7c63f77d-11b0-4769-bf60-5380d8ae7b9e$ cd usr/ + hadoop@Mcnode7:/media/hadoop/7c63f77d-11b0-4769-bf60-5380d8ae7b9e/usr$ ls + bin games include lib local sbin share src + hadoop@Mcnode7:/media/hadoop/7c63f77d-11b0-4769-bf60-5380d8ae7b9e/usr$ cd ../home/ + hadoop@Mcnode7:/media/hadoop/7c63f77d-11b0-4769-bf60-5380d8ae7b9e/home$ ls + hadoop uftp + hadoop@Mcnode7:/media/hadoop/7c63f77d-11b0-4769-bf60-5380d8ae7b9e/home$ cd hadoop/ + hadoop@Mcnode7:/media/hadoop/7c63f77d-11b0-4769-bf60-5380d8ae7b9e/home/hadoop$ ls + backup223 cloud disk.txt Downloads logs Music Public redis1492.sh rmq_bk_gc.log store test Videos zookeeper.out + Chng Desktop Documents examples.desktop middleware Pictures recommend redis149.sh site Templates test.sh xubo + + +参考 + + 【1】https://github.com/xubo245 + 【2]http://blog.csdn.net/xubo245/ + 【3】http://www.cnblogs.com/net2012/archive/2013/02/01/2888453.html diff --git "a/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2402\344\271\213Ubuntu\347\263\273\347\273\237\345\242\236\345\212\240\347\243\201\347\233\230.md" "b/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2402\344\271\213Ubuntu\347\263\273\347\273\237\345\242\236\345\212\240\347\243\201\347\233\230.md" new file mode 100644 index 0000000..5f21d4d --- /dev/null +++ "b/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2402\344\271\213Ubuntu\347\263\273\347\273\237\345\242\236\345\212\240\347\243\201\347\233\230.md" @@ -0,0 +1,193 @@ + +更多代码请见：https://github.com/xubo245 + +Ubuntu学习 + +# 1.解释 # + +现在单机有500G，但是不够用，正好有磁盘多，如何挂载新的磁盘，使其能有1T？ + +1）分区 +2）格式化 +2）挂载 + + +# 2.代码： # + + + +## （0）分区前： ## + + hadoop@Mcnode7:~$ sudo fdisk -lu + [sudo] password for hadoop: + + Disk /dev/sda: 500.1 GB, 500107862016 bytes + 255 heads, 63 sectors/track, 60801 cylinders, total 976773168 sectors + Units = sectors of 1 * 512 = 512 bytes + Sector size (logical/physical): 512 bytes / 512 bytes + I/O size (minimum/optimal): 512 bytes / 512 bytes + Disk identifier: 0x000db939 + + Device Boot Start End Blocks Id System + /dev/sda1 * 2048 951625727 475811840 83 Linux + /dev/sda2 951627774 976771071 12571649 5 Extended + /dev/sda5 951627776 976771071 12571648 82 Linux swap / Solaris + + Disk /dev/sdb: 500.1 GB, 500107862016 bytes + 255 heads, 63 sectors/track, 60801 cylinders, total 976773168 sectors + Units = sectors of 1 * 512 = 512 bytes + Sector size (logical/physical): 512 bytes / 512 bytes + I/O size (minimum/optimal): 512 bytes / 512 bytes + Disk identifier: 0x00004386 + + Device Boot Start End Blocks Id System + hadoop@Mcnode7:~$ sudo parted + +可以看到/dev/sdb没有被分区，所以得先分区 +## （1）分区 ## + +进入分区： + + hadoop@Mcnode7:~$ sudo fdisk /edv/sdb + +查看指令： + + Command (m for help): m + Command action + a toggle a bootable flag + b edit bsd disklabel + c toggle the dos compatibility flag + d delete a partition + l list known partition types + m print this menu + n add a new partition + o create a new empty DOS partition table + p print the partition table + q quit without saving changes + s create a new empty Sun disklabel + t change a partition's system id + u change display/entry units + v verify the partition table + w write table to disk and exit + x extra functionality (experts only) + +操作：n为创建分区，p为主分区，1为partition为1个，后面为起始和end地址，选择了默认 + + Command (m for help): n + Partition type: + p primary (0 primary, 0 extended, 4 free) + e extended + Select (default p): p + Partition number (1-4, default 1): 1 + First sector (2048-976773167, default 2048): + Using default value 2048 + Last sector, +sectors or +size{K,M,G} (2048-976773167, default 976773167): + Using default value 976773167 + +查看：p为print，查看 + + Command (m for help): p + + Disk /dev/sdb: 500.1 GB, 500107862016 bytes + 255 heads, 63 sectors/track, 60801 cylinders, total 976773168 sectors + Units = sectors of 1 * 512 = 512 bytes + Sector size (logical/physical): 512 bytes / 512 bytes + I/O size (minimum/optimal): 512 bytes / 512 bytes + Disk identifier: 0x00004386 + + Device Boot Start End Blocks Id System + /dev/sdb1 2048 976773167 488385560 83 Linux + +新建的分区在系统中生效。 + + partprobe + +## （2）格式化 ## +格式化新建分区： + + hadoop@Mcnode7:~$ sudo mkfs.ext4 /dev/sdb1 + mke2fs 1.42.9 (4-Feb-2014) + Filesystem label= + OS type: Linux + Block size=4096 (log=2) + Fragment size=4096 (log=2) + Stride=0 blocks, Stripe width=0 blocks + 30531584 inodes, 122096390 blocks + 6104819 blocks (5.00%) reserved for the super user + First data block=0 + Maximum filesystem blocks=4294967296 + 3727 block groups + 32768 blocks per group, 32768 fragments per group + 8192 inodes per group + Superblock backups stored on blocks: + 32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208, + 4096000, 7962624, 11239424, 20480000, 23887872, 71663616, 78675968, + 102400000 + + Allocating group tables: done + Writing inode tables: done + Creating journal (32768 blocks): done + Writing superblocks and filesystem accounting information: done + +## （3）挂载 ## +查看UUID： + + hadoop@Mcnode7:~$ sudo blkid -p /dev/sdb1 + /dev/sdb1: UUID="0da52ea9-623e-45fa-80b9-f5982c7f63ac" VERSION="1.0" TYPE="ext4" USAGE="filesystem" PART_ENTRY_SCHEME="dos" PART_ENTRY_TYPE="0x83" PART_ENTRY_NUMBER="1" PART_ENTRY_OFFSET="2048" PART_ENTRY_SIZE="976771120" PART_ENTRY_DISK="8:16" + +或者： + + hadoop@Mcnode4:~$ sudo blkid /dev/sdb1 + /dev/sdb1: UUID="0da52ea9-623e-45fa-80b9-f5982c7f63ac" TYPE="ext4" + +编辑fstab：在fstab中新增一行 + + UUID=0da52ea9-623e-45fa-80b9-f5982c7f63ac /home/hadoop/disk2 ext4 defaults 0 2 + +操作记录：需要新建/home/hadoop/disk2路径 + + hadoop@Mcnode4:~$ sudo vi /etc/fstab + [sudo] password for hadoop: + + # /etc/fstab: static file system information. + # + # Use 'blkid' to print the universally unique identifier for a + # device; this may be used with UUID= as a more robust way to name devices + # that works even if disks are added and removed. See fstab(5). + # + # + # / was on /dev/sda1 during installation + UUID=75b77b05-3660-4d44-b914-cf6233aa18f6 / ext4 errors=remount-ro 0 1 + # swap was on /dev/sda5 during installation + UUID=d04b6d14-feef-4b22-8744-15d2360cfd61 none swap sw 0 0 + + UUID=0da52ea9-623e-45fa-80b9-f5982c7f63ac /home/hadoop/disk2 ext4 defaults 0 2 + +使其生效 + + hadoop@Mcnode4:~/disk2$ sudo mount -a + +查看效果： + + hadoop@Mcnode4:~/disk2$ df -h + Filesystem Size Used Avail Use% Mounted on + /dev/sda1 453G 368G 63G 86% / + none 4.0K 0 4.0K 0% /sys/fs/cgroup + udev 3.0G 8.0K 3.0G 1% /dev + tmpfs 597M 1.3M 595M 1% /run + none 5.0M 0 5.0M 0% /run/lock + none 3.0G 80K 3.0G 1% /run/shm + none 100M 36K 100M 1% /run/user + /dev/sdb1 459G 70M 435G 1% /home/hadoop/disk2 + +挂载成功，重启后依然挂载了。 + + +# 3.结果： # + + 配置成功了。 + +参考 + + 【1】https://github.com/xubo245 + 【2]http://blog.csdn.net/xubo245/ diff --git "a/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2403\344\271\213\350\256\276\347\275\256\351\235\231\346\200\201ip\345\234\260\345\235\200.md" "b/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2403\344\271\213\350\256\276\347\275\256\351\235\231\346\200\201ip\345\234\260\345\235\200.md" new file mode 100644 index 0000000..1f2561c --- /dev/null +++ "b/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2403\344\271\213\350\256\276\347\275\256\351\235\231\346\200\201ip\345\234\260\345\235\200.md" @@ -0,0 +1,44 @@ + +更多代码请见：https://github.com/xubo245 + +Ubuntu学习 + +# 1.解释 # + +搭集群的时候，需要设置静态ip，防止ip变化带来的集群连接问题 + +# 2.代码： # + +（1）interfaces文件修改： + + hadoop@Master:~$ sudo vi /etc/network/interfaces + [sudo] password for hadoop: + + # interfaces(5) file used by ifup(8) and ifdown(8) + auto eth0 + iface eth0 inet static + address 192.168.1.149 + gateway 192.168.1.1 + netmask 255.255.255.0 + +（2）设置dns + + hadoop@Master:~$ vi /etc/resolvconf/resolv.conf.d/base + + nameserver 192.168.223.10 + +（3）生效 + + sudo ifdown eth0 + sudo ifup eth0 +或者sudo reboot重启 + +# 3.结果： # + + 静态ip设置成功 + + +参考 + + 【1】https://github.com/xubo245 + 【2]http://blog.csdn.net/xubo245/ diff --git "a/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2404\344\271\213\347\243\201\347\233\230\346\240\274\345\274\217\345\214\226\351\224\231\350\257\257.md" "b/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2404\344\271\213\347\243\201\347\233\230\346\240\274\345\274\217\345\214\226\351\224\231\350\257\257.md" new file mode 100644 index 0000000..f1a6c60 --- /dev/null +++ "b/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2404\344\271\213\347\243\201\347\233\230\346\240\274\345\274\217\345\214\226\351\224\231\350\257\257.md" @@ -0,0 +1,53 @@ + +更多代码请见：https://github.com/xubo245 + +Ubuntu学习 + +# 1.解释 # + +错误： + + error "Error synchronizing after initial wipe: Timed out waiting for object (udisks-error-quark, 0) + +解决：使用parted分区 + +# 2.代码： # + + + + hadoop@Mcnode7:~$ sudo parted /dev/sdb + GNU Parted 2.3 + Using /dev/sdb + Welcome to GNU Parted! Type 'help' to view a list of commands. + (parted) rm 1 + Error: Can't have a partition outside the disk! + (parted) print + Error: Can't have a partition outside the disk! + (parted) mklabel gpt + (parted) print + Model: ATA Hitachi HDT72105 (scsi) + Disk /dev/sdb: 500GB + Sector size (logical/physical): 512B/512B + Partition Table: gpt + + Number Start End Size File system Name Flags + + (parted) mkpart primary 0% 100% + (parted) print + Model: ATA Hitachi HDT72105 (scsi) + Disk /dev/sdb: 500GB + Sector size (logical/physical): 512B/512B + Partition Table: gpt + + Number Start End Size File system Name Flags + 1 1049kB 500GB 500GB ext4 primary + + +# 3.结果： # + + + +参考 + + 【1】https://github.com/xubo245 + 【2]http://blog.csdn.net/xubo245/ diff --git "a/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2405\344\271\213ubuntu\345\242\236\345\212\240\347\243\201\347\233\230\347\232\204\345\205\266\344\273\226\351\224\231\350\257\257.md" "b/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2405\344\271\213ubuntu\345\242\236\345\212\240\347\243\201\347\233\230\347\232\204\345\205\266\344\273\226\351\224\231\350\257\257.md" new file mode 100644 index 0000000..3870955 --- /dev/null +++ "b/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2405\344\271\213ubuntu\345\242\236\345\212\240\347\243\201\347\233\230\347\232\204\345\205\266\344\273\226\351\224\231\350\257\257.md" @@ -0,0 +1,26 @@ + +更多代码请见：https://github.com/xubo245 + +Ubuntu学习 + +# 1.解释 # + +## 问题1： ## +create RAID之后开机报错： + +PXE32:tftp open timeout 1962:no oprating system found + +=》重装了，应该有其他办法。按照“Ubuntu学习1之给Ubuntu系统恢复另一台机器的磁盘”可以恢复 + +## 问题2： ## +non RAID disk + +办法： +create RAID +但是会导致问题1 + + +参考 + + 【1】https://github.com/xubo245 + 【2]http://blog.csdn.net/xubo245/ diff --git "a/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2406\344\271\213\346\237\245\347\234\213ubuntu\344\270\213\346\211\200\346\234\211\346\211\247\350\241\214\346\214\207\344\273\244\345\216\206\345\217\262history.md" "b/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2406\344\271\213\346\237\245\347\234\213ubuntu\344\270\213\346\211\200\346\234\211\346\211\247\350\241\214\346\214\207\344\273\244\345\216\206\345\217\262history.md" new file mode 100644 index 0000000..dd78d6c --- /dev/null +++ "b/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2406\344\271\213\346\237\245\347\234\213ubuntu\344\270\213\346\211\200\346\234\211\346\211\247\350\241\214\346\214\207\344\273\244\345\216\206\345\217\262history.md" @@ -0,0 +1,32 @@ + +更多代码请见：https://github.com/xubo245 + +Ubuntu学习 + +# 1.解释 # + +由于ubuntu默认限制了显示行数，有时候需要查看所有的history，所以需要查看 + +自己的ubuntu设置了按ip登录和时间的历史记录，并且有时间，所以都是分开的，查看所有history也麻烦 + +方法： + + cat /home/username/.bash_history + +# 2.代码： # + +执行hadoop用户 + + hadoop@Mcnode6:~$ cat /home/hadoop/.bash_history + +或者sz下载下来，有1995行 + +# 3.结果： # + +查看所有的很方便 + + +参考 + + 【1】https://github.com/xubo245 + 【2]http://blog.csdn.net/xubo245/ diff --git "a/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2407\344\271\213\346\237\245\347\234\213\347\231\273\345\275\225\345\216\206\345\217\262.md" "b/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2407\344\271\213\346\237\245\347\234\213\347\231\273\345\275\225\345\216\206\345\217\262.md" new file mode 100644 index 0000000..c62d764 --- /dev/null +++ "b/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2407\344\271\213\346\237\245\347\234\213\347\231\273\345\275\225\345\216\206\345\217\262.md" @@ -0,0 +1,107 @@ + +更多代码请见：https://github.com/xubo245 + +Ubuntu学习 + +# 1.解释 # + +1.1 last 指令 + + last命令可用于显示特定用户登录系统的历史记录。如果没有指定任何参数，则显示所有用户的历史信息。在默认情况下，这些信息（所显示的信息）将来源于/var/log/wtmp文件。该命令的输出结果包含以下几列信息： + 用户名称 + tty设备号 + 历史登录时间日期 + 登出时间日期 + 总工作时间 + +使用方法 + + hadoop@Mcnode6:~$ last --help + last: invalid option -- '-' + Usage: last [-num | -n num] [-f file] [-t YYYYMMDDHHMMSS] [-R] [-adioxFw] [username..] [tty..] + +1.2.使用who命令查看（登录）用户名称及所启动的进程 +who命令用于列举出当前已登录系统的用户名称。其输出为：用户名、tty号、时间日期、主机地址。 + +1.3 使用w命令查看登录用户正在使用的进程信息 +w命令用于显示已经登录系统的用户的名称，以及他们正在做的事。该命令所使用的信息来源于/var/run/utmp文件。w命令输出的信息包括： + + +# 2.代码： # + +2.1 last 执行 + + hadoop@Mcnode6:~$ last + hadoop pts/30 :1 Sat Nov 5 20:26 still logged in + hadoop :1 :1 Sat Nov 5 20:26 still logged in + hadoop pts/14 host-e-202.ustcs Sat Nov 5 20:25 still logged in + guest-2R pts/5 :0 Sat Nov 5 20:23 still logged in + hadoop pts/4 mcnodex1 Thu Nov 3 19:17 still logged in + guest-2R :0 :0 Thu Nov 3 15:36 still logged in + hadoop pts/4 host-e-150.ustcs Thu Nov 3 15:10 - 15:56 (00:45) + hadoop pts/1 host-e-150.ustcs Wed Nov 2 21:54 - 21:54 (00:00) + hadoop pts/1 host-e-150.ustcs Wed Nov 2 21:45 - 21:45 (00:00) + hadoop pts/18 mcnodex1 Wed Nov 2 19:59 - 14:15 (18:16) + hadoop pts/17 host-e-150.ustcs Wed Nov 2 19:23 - 21:11 (01:48) + hadoop pts/16 host-e-150.ustcs Wed Nov 2 19:12 - 21:54 (02:41) + hadoop pts/10 host-e-150.ustcs Wed Nov 2 19:01 - 21:53 (02:52) + hadoop pts/8 host-e-150.ustcs Wed Nov 2 19:01 - 21:53 (02:52) + hadoop pts/1 host-e-150.ustcs Wed Nov 2 18:56 - 20:53 (01:56) + reboot system boot 3.16.0-30-generi Wed Nov 2 18:56 - 20:36 (3+01:40) + hadoop pts/5 202.38.84.12 Wed Nov 2 18:47 - down (00:07) + reboot system boot 3.16.0-30-generi Wed Nov 2 18:47 - 18:54 (00:07) + hadoop pts/1 host-e-150.ustcs Wed Nov 2 18:45 - down (00:00) + hadoop pts/7 host-e-150.ustcs Wed Nov 2 18:44 - down (00:01) + guest-nz :0 :0 Wed Nov 2 18:44 - 18:45 (00:01) + reboot system boot 3.16.0-30-generi Wed Nov 2 18:43 - 18:46 (00:02) + reboot system boot 3.16.0-30-generi Wed Nov 2 18:37 - 18:37 (00:00) + reboot system boot 3.16.0-30-generi Wed Nov 2 18:35 - 18:36 (00:01) + reboot system boot 3.16.0-30-generi Wed Nov 2 18:27 - 18:28 (00:00) + hadoop pts/8 host-e-150.ustcs Wed Nov 2 16:45 - down (01:41) + hadoop pts/17 host-e-150.ustcs Wed Nov 2 15:51 - 16:14 (00:23) + hadoop pts/16 host-e-150.ustcs Wed Nov 2 15:35 - 16:14 (00:38) + hadoop pts/10 host-e-150.ustcs Wed Nov 2 15:34 - 16:14 (00:40) + hadoop pts/8 host-e-150.ustcs Wed Nov 2 15:33 - 16:15 (00:42) + hadoop pts/8 host-e-150.ustcs Wed Nov 2 13:04 - 15:21 (02:16) + hadoop pts/5 mcnodex1 Tue Nov 1 23:42 - down (18:44) + hadoop pts/5 master Tue Nov 1 22:07 - 22:07 (00:00) + hadoop pts/8 host-e-150.ustcs Tue Nov 1 19:53 - 20:54 (01:00) + hadoop pts/5 host-e-150.ustcs Tue Nov 1 17:37 - 20:59 (03:22) + reboot system boot 3.16.0-30-generi Tue Nov 1 17:32 - 18:26 (1+00:53) + +2.2 who 查看当前有哪些用户 + + hadoop@Mcnode6:~$ who + hadoop pts/4 2016-11-03 19:17 (mcnodex1) + guest-2RZAIN :0 2016-11-03 15:36 (:0) + guest-2RZAIN pts/5 2016-11-05 20:23 (:0) + hadoop pts/14 2016-11-05 20:25 (host-e-202.ustcsz.edu.cn) + hadoop :1 2016-11-05 20:26 (:1) + hadoop pts/30 2016-11-05 20:26 (:1) + hadoop pts/26 2016-11-05 20:37 (:1) + + +2.3 w 查看用户正在使用哪些进程 + + hadoop@Mcnode6:~$ w + 20:40:53 up 3 days, 1:45, 7 users, load average: 2.06, 2.10, 2.13 + USER TTY FROM LOGIN@ IDLE JCPU PCPU WHAT + hadoop pts/4 mcnodex1 四19 2days 3:45 3:45 top + guest-2R :0 :0 四15 ?xdm? 2:08m 0.19s init --user + guest-2R pts/5 :0 20:23 15:33 0.13s 0.12s bash + hadoop pts/14 host-e-202.ustcs 20:25 5.00s 0.11s 0.00s w + hadoop :1 :1 20:26 ?xdm? 2:08m 0.15s init --user + hadoop pts/30 :1 20:26 3:17 0.28s 0.00s scp -rf tmp/ hadoop@master:~/Chng/ + hadoop pts/26 :1 20:37 5.00s 0.12s 0.04s vim read26.py + + + +# 3.结果： # + +更详细请见【3】 + +参考 + + 【1】https://github.com/xubo245 + 【2]http://blog.csdn.net/xubo245/ + 【3】http://blog.csdn.net/wudiyi815/article/details/8061459 diff --git "a/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2408\344\271\213scp\346\226\255\347\202\271\347\273\255\344\274\240.md" "b/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2408\344\271\213scp\346\226\255\347\202\271\347\273\255\344\274\240.md" new file mode 100644 index 0000000..5811e00 --- /dev/null +++ "b/docs/LinuxLearning/Ubuntu\345\255\246\344\271\2408\344\271\213scp\346\226\255\347\202\271\347\273\255\344\274\240.md" @@ -0,0 +1,38 @@ + +更多代码请见：https://github.com/xubo245 + +Ubuntu学习 + +1.上传文件 + + scp /tmp/test.tar root@192.168.1.190:/home/test.tar + +2.下载文件 + + scp root@192.168.1.190:/home/test.tar /tmp/test.tar + +3.上传整个目录 + + scp -r /tmp/testdir root@192.168.1.190:home + +4.下载整个目录 + + scp -r root@192.168.1.190:home/testdir /tmp + +5.断点续传 + + rsync -P --rsh=ssh /tmp/test.tar 192.168.1.190:/home/test.tar + +6.效果 + + xubo@xubo:~/xubo/tools/sap/compare$ rsync -P --rsh=ssh hadoop@Mcnode1:/home/hadoop/disk2/xubo/data/SparkSW/uniref/uniref100Line.fasta ref/uniref100Line.fasta + uniref100Line.fasta + 34,305,501,776 100% 13.86MB/s 0:39:19 (xfr#1, to-chk=0/1) + + + +参考 + + 【1】https://github.com/xubo245 + 【2] http://blog.csdn.net/xubo245/ + 【3】http://blog.csdn.net/sysprogram/article/details/52494828 diff --git a/docs/Spark/AlluxioLearning/README.md b/docs/Spark/AlluxioLearning/README.md new file mode 100644 index 0000000..6af285e --- /dev/null +++ b/docs/Spark/AlluxioLearning/README.md @@ -0,0 +1,10 @@ +## Spark生态之Tachyon学习篇 ## +Spark生态之Tachyon学习1---单机版搭建和运行（Alluxio）：http://blog.csdn.net/xubo245/article/details/51318566 +Spark生态之Tachyon学习2---Spark从tachyon中读取文件（Alluxio）：http://blog.csdn.net/xubo245/article/details/51318863 +Spark生态之Tachyon学习3---机器重启后数据存储位置的变化：http://blog.csdn.net/xubo245/article/details/51322437 +Spark生态之Tachyon学习4---下载源码通过maven install安装失败记录：http://blog.csdn.net/xubo245/article/details/51322911 +Spark生态之Tachyon学习5--tachyon的几个问题（待解决）：http://blog.csdn.net/xubo245/article/details/51323101 +Spark生态之Tachyon学习6---集群版搭建和运行（Alluxio）：http://blog.csdn.net/xubo245/article/details/51324273 +Spark生态之Tachyon学习7--下载源码通过maven安装成功：http://blog.csdn.net/xubo245/article/details/51325776 +Spark生态之Tachyon学习6---集群版搭建问题之集群无法全部启动：http://blog.csdn.net/xubo245/article/details/51325834 +Spark生态之Tachyon学习7---Tachyon的优点：http://blog.csdn.net/xubo245/article/details/51326644 \ No newline at end of file diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2401---\345\215\225\346\234\272\347\211\210\346\220\255\345\273\272\345\222\214\350\277\220\350\241\214.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2401---\345\215\225\346\234\272\347\211\210\346\220\255\345\273\272\345\222\214\350\277\220\350\241\214.md" new file mode 100644 index 0000000..a98d732 --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2401---\345\215\225\346\234\272\347\211\210\346\220\255\345\273\272\345\222\214\350\277\220\350\241\214.md" @@ -0,0 +1,151 @@ +环境 +ubuntu 14.04 +Spark-1.5.2 +Tachyon-0.7.1 + +1．由于最近用的Spark-1.5.2系统默认的tachyon为0.7.1，在Spark-core的pom.xml可以查看 +另外虽然现在Tachyon改名为Alluxio ，但是不影响这个版本的使用 + +2.单机版搭建： +下载地址： + + https://github.com/Alluxio/alluxio/releases/tag/v0.7.1 + +下载的是：tachyon-0.7.1-bin.tar.gz + +3.安装 +修改配置文件： + + cp conf/alluxio-env.sh.template conf/alluxio-env.sh +然后format和start： + + ./bin/alluxio format + ./bin/alluxio-start.sh local + +查看安装情况： + + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ ./tachyon format + Connecting to localhost as xubo... + Warning: Permanently added 'localhost' (ECDSA) to the list of known hosts. + Formatting Tachyon Worker @ xubo + Connection to localhost closed. + Formatting Tachyon Master @ localhost + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ jps + 12576 SparkSubmit + 13760 Jps + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ ls + tachyon tachyon-mount.sh tachyon-start.sh tachyon-stop.sh tachyon-workers.sh + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ ./tachyon-start.sh local + Killed 0 processes on xubo + Killed 0 processes on xubo + Connecting to localhost as xubo... + Killed 0 processes on xubo + Connection to localhost closed. + [sudo] password for xubo: + Formatting RamFS: /mnt/ramdisk (1gb) + Starting master @ localhost + Starting worker @ xubo + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ jps + 12576 SparkSubmit + 14088 Jps + 14058 TachyonWorker + 14030 TachyonMaster + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ + +可以看出来master和worker已经启动了 +另外也可以从浏览器看： + + http://localhost:19999/home +localhost为自己的ip + +说明安装成功 + +4. 使用 +（1）指令 + + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ ./tachyon tfs -help + Usage: java TFsShell + [cat ] + [count ] + [ls ] + [lsr ] + [mkdir ] + [rm ] + [rmr ] + [tail ] + [touch ] + [mv ] + [copyFromLocal ] + [copyToLocal ] + [fileinfo ] + [location ] + [report ] + [request ] + [pin ] + [unpin ] + [free ] + [getUsedBytes] + [getCapacityBytes] + [du ] + +（2）测试用例 + + xubo@xubo:~/cloud/tachyon-0.7.1$ ./bin/tachyon runTest Basic CACHE_THROUGH + /default_tests_files/BasicFile_CACHE_THROUGH has been removed + 2016-05-04 22:19:45,075 INFO (MasterClient.java:connect) - Tachyon client (version 0.7.1) is trying to connect with master @ localhost/127.0.0.1:19998 + 2016-05-04 22:19:45,104 INFO (MasterClient.java:connect) - User registered with the master @ localhost/127.0.0.1:19998; got UserId 4 + 2016-05-04 22:19:45,130 INFO (CommonUtils.java:printTimeTakenMs) - createFile with fileId 3 took 59 ms. + 2016-05-04 22:19:45,153 INFO (WorkerClient.java:connect) - Trying to get local worker host : xubo + 2016-05-04 22:19:45,166 INFO (WorkerClient.java:connect) - Connecting local worker @ xubo/xuboIP:29998 + 2016-05-04 22:19:45,230 INFO (BlockOutStream.java:get) - Writing with local stream. tachyonFile: /default_tests_files/BasicFile_CACHE_THROUGH, blockIndex: 0, opType: CACHE_THROUGH + 2016-05-04 22:19:45,289 INFO (CommonUtils.java:createBlockPath) - Folder /mnt/ramdisk/tachyonworker/4 was created! + 2016-05-04 22:19:45,294 INFO (LocalBlockOutStream.java:) - /mnt/ramdisk/tachyonworker/4/3221225472 was created! tachyonFile: /default_tests_files/BasicFile_CACHE_THROUGH, blockIndex: 0, blockId: 3221225472, blockCapacityByte: 536870912 + 2016-05-04 22:19:45,370 INFO (CommonUtils.java:printTimeTakenMs) - writeFile to file /default_tests_files/BasicFile_CACHE_THROUGH took 239 ms. + 2016-05-04 22:19:45,420 INFO (CommonUtils.java:printTimeTakenMs) - readFile file /default_tests_files/BasicFile_CACHE_THROUGH took 50 ms. + Passed the test! + +（3）从本地上传文件到remote： + + xubo@xubo:~/cloud/tachyon-0.7.1$ ./bin/tachyon tfs copyFromLocal pom.xml / + Copied pom.xml to / +可以在浏览器中查看 + +（4）显示文件内容： + + xubo@xubo:~/cloud/test/tachyon$ ../../tachyon-0.7.1/bin/tachyon tfs copyFromLocal 1.txt / + Copied 1.txt to / + xubo@xubo:~/cloud/test/tachyon$ ../../tachyon-0.7.1/bin/tachyon tfs cat /1.txt + hello tachyon + 1 + 2 + 3 + +（5）Spark调用tachyon文件：待完成，Spark出问题了，请见下一篇博文 + +（6）关闭tachyon + + hadoop@Master:~/cloud/testByXubo/spark/tachyon/tachyon-0.7.1$ ./bin/tachyon-stop.sh + Killed 1 processes on Master + Killed 1 processes on Master + Connecting to localhost as hadoop... + jjKilled 0 processes on Master + Connection to localhost closed. + + +附：将tachyon脚本cp到usr的bin下：（需要本地编译，直接start的不行） + + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ sudo cp tachyon /usr/ + bin/ games/ include/ lib/ lib64/ local/ sbin/ share/ src/ + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ sudo cp tachyon /usr/local/bin/ + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ ta + tabs tac tachyon tail tailf tali tangle tap2deb tap2rpm tapconvert tar tarcat targen taskset +方便使用 + + +参考 +【1】http://alluxio.org/documentation/v0.7.1/ +【2】https://github.com/xubo245/SparkLearning +【3】http://alluxio.org/documentation/v0.7.1/Running-Tachyon-Locally.html + + + diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24010--\351\233\206\347\276\244\351\227\256\346\263\225\345\205\250\351\203\250\345\220\257\345\212\250\351\227\256\351\242\230\350\247\243\345\206\263.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24010--\351\233\206\347\276\244\351\227\256\346\263\225\345\205\250\351\203\250\345\220\257\345\212\250\351\227\256\351\242\230\350\247\243\345\206\263.md" new file mode 100644 index 0000000..73e6e40 --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24010--\351\233\206\347\276\244\351\227\256\346\263\225\345\205\250\351\203\250\345\220\257\345\212\250\351\227\256\351\242\230\350\247\243\345\206\263.md" @@ -0,0 +1,195 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio（tachyon） 0.7.1，spark-1.5.2,hadoop-2.6.0 + +# 1.解释 # +## 1.1 问题描述 ## + +之前博文没有解决的问题：http://blog.csdn.net/xubo245/article/details/51325834 + +具体： + + hadoop@Master:~/cloud/alluxio-0.7.1$ ./bin/tachyon format + Connecting to Mcnode1 as hadoop... + Formatting Tachyon Worker @ Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Formatting Tachyon Worker @ Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode3 + /home/hadoop/cloud/alluxio-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode4 + /home/hadoop/cloud/alluxio-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode5 + /home/hadoop/cloud/alluxio-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Formatting Tachyon Worker @ Mcnode6 + Connection to mcnode6 closed. + Formatting Tachyon Master @ Master +## 1.2分析： ## + + 主要是没有bin/java,因为Mcnode3，4，5节点在/usr/bin和/usr/local/bin没有java，java和javac在其他位置，而alluxio调用的是/usr/bin路径下的指令 + +## 1.3 解决办法 ## + + sudo update-alternatives --install /usr/bin/java java /usr/lib/jvm/jdk1.7.0/bin/java 300 + sudo update-alternatives --install /usr/bin/javac javac /usr/lib/jvm/jdk1.7.0/bin/javac 300 + +## 1.4 解决效果： ## + 将第五个节点按照解决办法处理后，就没有报错了： + + hadoop@Master:~/cloud/alluxio-0.7.1$ ./bin/tachyon format + Connecting to Mcnode1 as hadoop... + Formatting Tachyon Worker @ Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Formatting Tachyon Worker @ Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode3 + /home/hadoop/cloud/alluxio-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode4 + /home/hadoop/cloud/alluxio-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + Formatting Tachyon Worker @ Mcnode5 + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Formatting Tachyon Worker @ Mcnode6 + Connection to mcnode6 closed. + Formatting Tachyon Master @ Master + + + +# 2.代码： # + +## 2.1 解决办法 ## + + sudo update-alternatives --install /usr/bin/java java /usr/lib/jvm/jdk1.7.0/bin/java 300 + sudo update-alternatives --install /usr/bin/javac javac /usr/lib/jvm/jdk1.7.0/bin/javac 300 + +## 2.2 运行结果 ## + + hadoop@Mcnode3:~/cloud/alluxio-0.7.1$ sudo update-alternatives --install /usr/bin/java java /usr/lib/jvm/jdk1.7.0/bin/java 300 + [sudo] password for hadoop: + update-alternatives: using /usr/lib/jvm/jdk1.7.0/bin/java to provide /usr/bin/java (java) in auto mode + hadoop@Mcnode3:~/cloud/alluxio-0.7.1$ sudo update-alternatives --install /usr/bin/javac javac /usr/lib/jvm/jdk1.7.0/bin/javac 300 + update-alternatives: using /usr/lib/jvm/jdk1.7.0/bin/javac to provide /usr/bin/javac (javac) in auto mode + hadoop@Mcnode3:~/cloud/alluxio-0.7.1$ + +# 3.结果： # + +都按照解决办法解决：截图在本周的周报中 + + hadoop@Master:~/cloud/alluxio-0.7.1$ ./bin/tachyon format + Connecting to Mcnode1 as hadoop... + Formatting Tachyon Worker @ Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Formatting Tachyon Worker @ Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + Formatting Tachyon Worker @ Mcnode3 + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + Formatting Tachyon Worker @ Mcnode4 + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + Formatting Tachyon Worker @ Mcnode5 + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Formatting Tachyon Worker @ Mcnode6 + Connection to mcnode6 closed. + Connecting to Mcnode7 as hadoop... + Formatting Tachyon Worker @ Mcnode7 + Connection to mcnode7 closed. + Formatting Tachyon Master @ Master + hadoop@Master:~/cloud/alluxio-0.7.1$ ./bin/tachyon-start.sh all SudoMount + Killed 1 processes on Master + Killed 0 processes on Master + Connecting to Mcnode1 as hadoop... + Killed 1 processes on Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Killed 1 processes on Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + Killed 0 processes on Mcnode3 + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + Killed 0 processes on Mcnode4 + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + Killed 1 processes on Mcnode5 + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Killed 1 processes on Mcnode6 + Connection to mcnode6 closed. + Connecting to Mcnode7 as hadoop... + Killed 0 processes on Mcnode7 + Connection to mcnode7 closed. + Starting master @ Master + Connecting to Mcnode1 as hadoop... + [sudo] password for hadoop: + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + [sudo] password for hadoop: + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + [sudo] password for hadoop: + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode3 + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + [sudo] password for hadoop: + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode4 + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + [sudo] password for hadoop: + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode5 + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + [sudo] password for hadoop: + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode6 + Connection to mcnode6 closed. + Connecting to Mcnode7 as hadoop... + [sudo] password for hadoop: + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode7 + Connection to mcnode7 closed. + hadoop@Master:~/cloud/alluxio-0.7.1$ + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning + 【5】http://blog.csdn.net/edwzhang/article/details/7759933 diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\240100--.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\240100--.md" new file mode 100644 index 0000000..fb87aa0 --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\240100--.md" @@ -0,0 +1,31 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +# 1.解释 # + +## 1.1 ## + + +## 1.2 ## + + +# 2.代码： # + +## 2.1 ## + + +## 2.2 ## + + +# 3.结果： # + + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24011--alluxio-1.3.0\351\233\206\347\276\244\351\205\215\347\275\256.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24011--alluxio-1.3.0\351\233\206\347\276\244\351\205\215\347\275\256.md" new file mode 100644 index 0000000..6487a4b --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24011--alluxio-1.3.0\351\233\206\347\276\244\351\205\215\347\275\256.md" @@ -0,0 +1,237 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +在spark上运行alluxio请参考下一个博客 + +# 1.解释 # + +## 1.1 下载## + + wget http://alluxio.org/downloads/files/1.3.0/alluxio-1.3.0-bin.tar.gz + tar xvfz alluxio-1.3.0-bin.tar.gz +## 1.2 配置## +复制 + + cp conf/alluxio-env.sh.template conf/alluxio-env.sh + +修改： + vi conf/alluxio-env.sh +修改内容： + + # The directory where Alluxio deployment is installed. (Default: the parent directory of libexec/). + # ALLUXIO_HOME + export ALLUXIO_HOME=/home/hadoop/cloud/alluxio-1.3.0 + + # The directory where log files are stored. (Default: ${ALLUXIO_HOME}/logs). + # ALLUXIO_LOGS_DIR + + # Hostname of the master. + # ALLUXIO_MASTER_HOSTNAME + export ALLUXIO_MASTER_HOSTNAME=Master + + # This is now deprecated. Support will be removed in v2.0 + # ALLUXIO_MASTER_ADDRESS + + # The directory where a worker stores in-memory data. (Default: /mnt/ramdisk). + # E.g. On linux, /mnt/ramdisk for ramdisk, /dev/shm for tmpFS; on MacOS, /Volumes/ramdisk for ramdisk + # ALLUXIO_RAM_FOLDER + export ALLUXIO_RAM_FOLDER=/home/hadoop/cloud/alluxio-1.3.0/ramdisk + + # Address of the under filesystem address. (Default: ${ALLUXIO_HOME}/underFSStorage) + # E.g. "/my/local/path" to use local fs, "hdfs://localhost:9000/alluxio" to use a local hdfs + # ALLUXIO_UNDERFS_ADDRESS + export ALLUXIO_UNDERFS_ADDRESS=hdfs://Master:9000 + + # How much memory to use per worker. (Default: 1GB) + # E.g. "1000MB", "2GB" + # ALLUXIO_WORKER_MEMORY_SIZE + export ALLUXIO_WORKER_MEMORY_SIZE=6GB + +保存，scp到集群中每个节点 + +## 1.3 启动## + + hadoop@Master:~/cloud/alluxio-1.3.0$ ./bin/alluxio format + Waiting for MASTER tasks to finish... + All MASTER tasks finished, please analyze the log at /home/hadoop/cloud/alluxio-1.3.0/logs/task.log. + Formatting Alluxio Master @ Master + hadoop@Master:~/cloud/alluxio-1.3.0$ ./bin/alluxio-start.sh all Mount + /home/hadoop/cloud/alluxio-1.3.0/bin + Killed 1 processes on Master + Killed 0 processes on Master + Waiting for WORKERS tasks to finish... + All WORKERS tasks finished, please analyze the log at /home/hadoop/cloud/alluxio-1.3.0/logs/task.log. + Starting master @ Master. Logging to /home/hadoop/cloud/alluxio-1.3.0/logs + Waiting for MASTER tasks to finish... + All MASTER tasks finished, please analyze the log at /home/hadoop/cloud/alluxio-1.3.0/logs/task.log. + +# 3.结果： # + +![](http://i.imgur.com/J5IjaLM.png) + +alluxio fs 等命令测试可以用 + + hadoop@Mcnode1:~/disk2/xubo/data/SparkSW/uniref$ time cat D6Line.fasta |grep -c GC + 618260 + + real 0m6.313s + user 0m6.169s + sys 0m1.056s + hadoop@Mcnode1:~/disk2/xubo/data/SparkSW/uniref$ time ~/cloud/alluxio-1.3.0/bin/alluxio fs cat /D6Line.fasta |grep -c GC + 618260 + + real 0m7.725s + user 0m9.663s + sys 0m1.247s + + +测试： + + hadoop@Master:~/cloud/alluxio-1.3.0$ ./bin/alluxio runTests + 2016-11-09 23:23:10,739 INFO type (MetricsSystem.java:startSinksFromConfig) - Starting sinks with config: {}. + 2016-11-09 23:23:10,742 INFO type (MetricsSystem.java:startSinks) - Sinks have already been started. + 2016-11-09 23:23:10,843 INFO type (AbstractClient.java:connect) - Alluxio client (version 1.3.0) is trying to connect with FileSystemMasterClient master @ Master/MasterIP:19998 + 2016-11-09 23:23:10,868 INFO type (AbstractClient.java:connect) - Client registered with FileSystemMasterClient master @ Master/MasterIP:19998 + runTest Basic CACHE_PROMOTE MUST_CACHE + 2016-11-09 23:23:11,088 INFO type (AbstractClient.java:connect) - Alluxio client (version 1.3.0) is trying to connect with BlockMasterClient master @ Master/MasterIP:19998 + 2016-11-09 23:23:11,090 INFO type (AbstractClient.java:connect) - Client registered with BlockMasterClient master @ Master/MasterIP:19998 + 2016-11-09 23:23:11,209 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@8408396 + 2016-11-09 23:23:11,315 INFO type (NettyChannelPool.java:createNewResource) - Created netty channel to with netty boostrap alluxio.client.block.BlockStoreContext$2@5087f4d0. + 2016-11-09 23:23:11,406 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@62b3a353 + 2016-11-09 23:23:11,461 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_CACHE_PROMOTE_MUST_CACHE took 472 ms. + 2016-11-09 23:23:11,567 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_CACHE_PROMOTE_MUST_CACHE took 106 ms. + Passed the test! + runTest BasicNonByteBuffer CACHE_PROMOTE MUST_CACHE + 2016-11-09 23:23:11,600 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@6dd8ade8 + 2016-11-09 23:23:11,611 INFO type (NettyChannelPool.java:createNewResource) - Created netty channel to with netty boostrap alluxio.client.block.BlockStoreContext$2@4aaf32dc. + 2016-11-09 23:23:11,682 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@18607df6 + 2016-11-09 23:23:11,699 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_CACHE_PROMOTE_MUST_CACHE took 114 ms. + 2016-11-09 23:23:11,755 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_CACHE_PROMOTE_MUST_CACHE took 56 ms. + Passed the test! + runTest Basic CACHE_PROMOTE CACHE_THROUGH + 2016-11-09 23:23:11,806 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@32c81878 + 2016-11-09 23:23:11,825 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@2928fd34 + 2016-11-09 23:23:11,990 INFO type (NettyChannelPool.java:createNewResource) - Created netty channel to with netty boostrap alluxio.client.block.BlockStoreContext$2@7ae5fdad. + 2016-11-09 23:23:12,397 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_CACHE_PROMOTE_CACHE_THROUGH took 637 ms. + 2016-11-09 23:23:12,408 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_CACHE_PROMOTE_CACHE_THROUGH took 10 ms. + Passed the test! + runTest BasicNonByteBuffer CACHE_PROMOTE CACHE_THROUGH + 2016-11-09 23:23:12,422 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@9d14a9 + 2016-11-09 23:23:12,435 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@7fe96694 + 2016-11-09 23:23:12,587 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@366872f6 + 2016-11-09 23:23:12,597 INFO type (NettyChannelPool.java:createNewResource) - Created netty channel to with netty boostrap alluxio.client.block.BlockStoreContext$2@414ad9fb. + 2016-11-09 23:23:13,006 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@65c83981 + 2016-11-09 23:23:13,022 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_CACHE_PROMOTE_CACHE_THROUGH took 442 ms. + 2016-11-09 23:23:13,057 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_CACHE_PROMOTE_CACHE_THROUGH took 35 ms. + Passed the test! + runTest Basic CACHE_PROMOTE THROUGH + 2016-11-09 23:23:13,070 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@a4f5b6d + 2016-11-09 23:23:13,082 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@6de4f076 + 2016-11-09 23:23:13,231 INFO type (NettyChannelPool.java:createNewResource) - Created netty channel to with netty boostrap alluxio.client.block.BlockStoreContext$2@34aa287f. + 2016-11-09 23:23:13,614 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_CACHE_PROMOTE_THROUGH took 556 ms. + 2016-11-09 23:23:13,660 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@5ee179dc + 2016-11-09 23:23:13,674 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@6f860b23 + 2016-11-09 23:23:13,907 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_CACHE_PROMOTE_THROUGH took 292 ms. + Passed the test! + runTest BasicNonByteBuffer CACHE_PROMOTE THROUGH + 2016-11-09 23:23:13,917 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@aa1e2a5 + 2016-11-09 23:23:13,928 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@7713c735 + 2016-11-09 23:23:14,423 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_CACHE_PROMOTE_THROUGH took 352 ms. + 2016-11-09 23:23:14,472 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@633e5715 + 2016-11-09 23:23:14,537 INFO type (NettyChannelPool.java:createNewResource) - Created netty channel to with netty boostrap alluxio.client.block.BlockStoreContext$2@b14de02. + 2016-11-09 23:23:14,610 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@65a714ff + 2016-11-09 23:23:14,624 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_CACHE_PROMOTE_THROUGH took 201 ms. + Passed the test! + runTest Basic CACHE_PROMOTE ASYNC_THROUGH + 2016-11-09 23:23:14,633 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@4f4fe346 + 2016-11-09 23:23:14,664 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@5a62681b + 2016-11-09 23:23:14,707 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_CACHE_PROMOTE_ASYNC_THROUGH took 82 ms. + 2016-11-09 23:23:14,744 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_CACHE_PROMOTE_ASYNC_THROUGH took 37 ms. + Passed the test! + runTest BasicNonByteBuffer CACHE_PROMOTE ASYNC_THROUGH + 2016-11-09 23:23:14,763 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_CACHE_PROMOTE_ASYNC_THROUGH took 14 ms. + 2016-11-09 23:23:14,773 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_CACHE_PROMOTE_ASYNC_THROUGH took 10 ms. + Passed the test! + runTest Basic CACHE MUST_CACHE + 2016-11-09 23:23:14,786 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_CACHE_MUST_CACHE took 12 ms. + 2016-11-09 23:23:14,794 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_CACHE_MUST_CACHE took 8 ms. + Passed the test! + runTest BasicNonByteBuffer CACHE MUST_CACHE + 2016-11-09 23:23:14,806 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@2a37801c + 2016-11-09 23:23:14,834 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@47089c2c + 2016-11-09 23:23:14,852 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_CACHE_MUST_CACHE took 52 ms. + 2016-11-09 23:23:14,880 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_CACHE_MUST_CACHE took 28 ms. + Passed the test! + runTest Basic CACHE CACHE_THROUGH + 2016-11-09 23:23:15,195 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_CACHE_CACHE_THROUGH took 314 ms. + 2016-11-09 23:23:15,220 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_CACHE_CACHE_THROUGH took 25 ms. + Passed the test! + runTest BasicNonByteBuffer CACHE CACHE_THROUGH + 2016-11-09 23:23:15,325 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_CACHE_CACHE_THROUGH took 73 ms. + 2016-11-09 23:23:15,336 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_CACHE_CACHE_THROUGH took 11 ms. + Passed the test! + runTest Basic CACHE THROUGH + 2016-11-09 23:23:15,412 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_CACHE_THROUGH took 75 ms. + 2016-11-09 23:23:15,534 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_CACHE_THROUGH took 122 ms. + Passed the test! + runTest BasicNonByteBuffer CACHE THROUGH + 2016-11-09 23:23:15,549 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@7ef30fce + 2016-11-09 23:23:15,561 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@3b4d67bb + 2016-11-09 23:23:15,715 INFO type (NettyChannelPool.java:createNewResource) - Created netty channel to with netty boostrap alluxio.client.block.BlockStoreContext$2@2bf9bcc6. + 2016-11-09 23:23:16,130 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_CACHE_THROUGH took 417 ms. + 2016-11-09 23:23:16,277 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_CACHE_THROUGH took 146 ms. + Passed the test! + runTest Basic CACHE ASYNC_THROUGH + 2016-11-09 23:23:16,285 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@43036682 + 2016-11-09 23:23:16,309 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@5051f26e + 2016-11-09 23:23:16,326 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_CACHE_ASYNC_THROUGH took 48 ms. + 2016-11-09 23:23:16,350 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_CACHE_ASYNC_THROUGH took 23 ms. + Passed the test! + runTest BasicNonByteBuffer CACHE ASYNC_THROUGH + 2016-11-09 23:23:16,362 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_CACHE_ASYNC_THROUGH took 9 ms. + 2016-11-09 23:23:16,370 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_CACHE_ASYNC_THROUGH took 7 ms. + Passed the test! + runTest Basic NO_CACHE MUST_CACHE + 2016-11-09 23:23:16,382 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_NO_CACHE_MUST_CACHE took 12 ms. + 2016-11-09 23:23:16,389 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_NO_CACHE_MUST_CACHE took 7 ms. + Passed the test! + runTest BasicNonByteBuffer NO_CACHE MUST_CACHE + 2016-11-09 23:23:16,401 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_NO_CACHE_MUST_CACHE took 9 ms. + 2016-11-09 23:23:16,412 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_NO_CACHE_MUST_CACHE took 11 ms. + Passed the test! + runTest Basic NO_CACHE CACHE_THROUGH + 2016-11-09 23:23:16,543 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_NO_CACHE_CACHE_THROUGH took 130 ms. + 2016-11-09 23:23:16,552 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_NO_CACHE_CACHE_THROUGH took 9 ms. + Passed the test! + runTest BasicNonByteBuffer NO_CACHE CACHE_THROUGH + 2016-11-09 23:23:16,651 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_NO_CACHE_CACHE_THROUGH took 66 ms. + 2016-11-09 23:23:16,675 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_NO_CACHE_CACHE_THROUGH took 23 ms. + Passed the test! + runTest Basic NO_CACHE THROUGH + 2016-11-09 23:23:16,745 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_NO_CACHE_THROUGH took 69 ms. + 2016-11-09 23:23:16,813 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_NO_CACHE_THROUGH took 68 ms. + Passed the test! + runTest BasicNonByteBuffer NO_CACHE THROUGH + 2016-11-09 23:23:16,887 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_NO_CACHE_THROUGH took 52 ms. + 2016-11-09 23:23:16,961 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_NO_CACHE_THROUGH took 74 ms. + Passed the test! + runTest Basic NO_CACHE ASYNC_THROUGH + 2016-11-09 23:23:16,974 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_NO_CACHE_ASYNC_THROUGH took 12 ms. + 2016-11-09 23:23:16,982 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_NO_CACHE_ASYNC_THROUGH took 8 ms. + Passed the test! + runTest BasicNonByteBuffer NO_CACHE ASYNC_THROUGH + 2016-11-09 23:23:17,005 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_NO_CACHE_ASYNC_THROUGH took 19 ms. + 2016-11-09 23:23:17,012 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_NO_CACHE_ASYNC_THROUGH took 7 ms. + Passed the test! + hadoop@Master:~/cloud/alluxio-1.3.0$ + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning + 【5】http://www.cnblogs.com/seaspring/p/5726695.html diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24012--spark\350\260\203\347\224\250lluxio-1.3.0\351\205\215\347\275\256.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24012--spark\350\260\203\347\224\250lluxio-1.3.0\351\205\215\347\275\256.md" new file mode 100644 index 0000000..b63efe1 --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24012--spark\350\260\203\347\224\250lluxio-1.3.0\351\205\215\347\275\256.md" @@ -0,0 +1,47 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +# 1.解释 # + +alluxio-0.7.1 启动的时候spark集群直接可以调用，但是alluxio-1.3.0需要自动配置 + +# 2.代码： # + +## 2.1 下载## + http://www.alluxio.org/download 下选择： + + alluxio spark client + 或者： + + http://downloads.alluxio.org/downloads/files/1.3.0/alluxio-1.3.0-spark-client-jar-with-dependencies.jar + +## 2.2 配置## + 请添加如下代码到spark/conf/spark-defaults.conf： + + spark.driver.extraClassPath /home/hadoop/cloud/alluxio-1.3.0/jar/alluxio-1.3.0-spark-client-jar-with-dependencies.jar + spark.executor.extraClassPath /home/hadoop/cloud/alluxio-1.3.0/jar/alluxio-1.3.0-spark-client-jar-with-dependencies.jar + + jar包需要scp到每个节点 +## 2.2 启动## + 重启spark集群 +# 3.结果： # + spark-shell可以操作alluxio集群 + + scala> val rdd1= sc.textFile("hdfs://Master:9000//xubo/project/SparkSW/input/D1Line.fasta") + rdd1: org.apache.spark.rdd.RDD[String] = MapPartitionsRDD[1] at textFile at :21 + + scala> rdd1.count + res0: Long = 78295 + + scala> rdd1.saveAsTextFile("alluxio://Master:19998/D1Line.fasta") + + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24013--Alluxio-1.3.0\345\220\257\345\212\250\347\226\221\351\227\256.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24013--Alluxio-1.3.0\345\220\257\345\212\250\347\226\221\351\227\256.md" new file mode 100644 index 0000000..af716f2 --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24013--Alluxio-1.3.0\345\220\257\345\212\250\347\226\221\351\227\256.md" @@ -0,0 +1,144 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +# 1.解释 # + +## 1.1 alluxio.env.sh 配置## + + # The directory where a worker stores in-memory data. (Default: /mnt/ramdisk). + # E.g. On linux, /mnt/ramdisk for ramdisk, /dev/shm for tmpFS; on MacOS, /Volumes/ramdisk for ramdisk + # ALLUXIO_RAM_FOLDER + export ALLUXIO_RAM_FOLDER=/home/hadoop/cloud/alluxio-1.3.0/ramdisk + + # Address of the under filesystem address. (Default: ${ALLUXIO_HOME}/underFSStorage) + # E.g. "/my/local/path" to use local fs, "hdfs://localhost:9000/alluxio" to use a local hdfs + # ALLUXIO_UNDERFS_ADDRESS + export ALLUXIO_UNDERFS_ADDRESS=hdfs://Master:9000 + + # How much memory to use per worker. (Default: 1GB) + # E.g. "1000MB", "2GB" + # ALLUXIO_WORKER_MEMORY_SIZE + export ALLUXIO_WORKER_MEMORY_SIZE=6GB + + +启动： + + hadoop@Master:~/cloud/alluxio-1.3.0/conf$ ../bin/alluxio-stop.sh all + Killed 0 processes on Master + Killed 0 processes on Master + Waiting for WORKERS tasks to finish... + All WORKERS tasks finished, please analyze the log at /home/hadoop/cloud/alluxio-1.3.0/logs/task.log. + hadoop@Master:~/cloud/alluxio-1.3.0/conf$ ../bin/alluxio-start.sh all Mount + /home/hadoop/cloud/alluxio-1.3.0/bin + Killed 0 processes on Master + Killed 0 processes on Master + Waiting for WORKERS tasks to finish... + All WORKERS tasks finished, please analyze the log at /home/hadoop/cloud/alluxio-1.3.0/logs/task.log. + Starting master @ Master. Logging to /home/hadoop/cloud/alluxio-1.3.0/logs + Waiting for MASTER tasks to finish... + All MASTER tasks finished, please analyze the log at /home/hadoop/cloud/alluxio-1.3.0/logs/task.log. + + +## 1.2 启动记录 ## + + 2016-11-11 16:35:48,582 + INFO MASTER Connecting to Mcnode1 as hadoop... + 2016-11-11 16:35:48,595 + INFO MASTER Connecting to Mcnode2 as hadoop... + 2016-11-11 16:35:48,604 + INFO MASTER Connecting to Mcnode3 as hadoop... + 2016-11-11 16:35:48,627 + INFO MASTER Connecting to Mcnode4 as hadoop... + 2016-11-11 16:35:48,632 + INFO MASTER Connecting to Mcnode5 as hadoop... + 2016-11-11 16:35:48,649 + INFO MASTER Connecting to Mcnode6 as hadoop... + 2016-11-11 16:35:48,669 + INFO MASTER Connecting to Mcnode7 as hadoop... + /home/hadoop/cloud/alluxio-1.3.0/bin + /home/hadoop/cloud/alluxio-1.3.0/bin + /home/hadoop/cloud/alluxio-1.3.0/bin + Formatting RamFS: /home/hadoop/cloud/alluxio-1.3.0/ramdisk (6gb) + mount: only root can do that + Starting worker @ Mcnode1. Logging to /home/hadoop/cloud/alluxio-1.3.0/logs + /home/hadoop/cloud/alluxio-1.3.0/bin + /home/hadoop/cloud/alluxio-1.3.0/bin + Formatting RamFS: /home/hadoop/cloud/alluxio-1.3.0/ramdisk (6gb) + mount: only root can do that + Starting worker @ Mcnode3. Logging to /home/hadoop/cloud/alluxio-1.3.0/logs + /home/hadoop/cloud/alluxio-1.3.0/bin + Formatting RamFS: /home/hadoop/cloud/alluxio-1.3.0/ramdisk (6gb) + mount: only root can do that + Starting worker @ Mcnode2. Logging to /home/hadoop/cloud/alluxio-1.3.0/logs + /home/hadoop/cloud/alluxio-1.3.0/bin + Formatting RamFS: /home/hadoop/cloud/alluxio-1.3.0/ramdisk (6gb) + mount: only root can do that + Formatting RamFS: /home/hadoop/cloud/alluxio-1.3.0/ramdisk (6gb) + Starting worker @ Mcnode4. Logging to /home/hadoop/cloud/alluxio-1.3.0/logs + mount: only root can do that + Starting worker @ Mcnode5. Logging to /home/hadoop/cloud/alluxio-1.3.0/logs + Formatting RamFS: /home/hadoop/cloud/alluxio-1.3.0/ramdisk (6gb) + Formatting RamFS: /home/hadoop/cloud/alluxio-1.3.0/ramdisk (6gb) + mount: only root can do that + mount: only root can do that + Starting worker @ Mcnode6. Logging to /home/hadoop/cloud/alluxio-1.3.0/logs + Starting worker @ Mcnode7. Logging to /home/hadoop/cloud/alluxio-1.3.0/logs + Connection to mcnode1 closed. + Connection to mcnode3 closed. + Connection to mcnode2 closed. + Connection to mcnode4 closed. + Connection to mcnode5 closed. + Connection to mcnode6 closed. + Connection to mcnode7 closed. + + +# 2.疑问： # +显示mount: only root can do that +有问题吗？ +不确定会不会有影响 + +但是加sudo启动不了 + + hadoop@Master:~/cloud/alluxio-1.3.0/conf$ ../bin/alluxio-start.sh all SudoMount + /home/hadoop/cloud/alluxio-1.3.0/bin + Killed 0 processes on Master + Killed 0 processes on Master + Waiting for WORKERS tasks to finish... + All WORKERS tasks finished, please analyze the log at /home/hadoop/cloud/alluxio-1.3.0/logs/task.log. + Starting master @ Master. Logging to /home/hadoop/cloud/alluxio-1.3.0/logs + Waiting for MASTER tasks to finish... + +task.log： + + 2016-11-11 16:33:40,423 + INFO MASTER Connecting to Mcnode1 as hadoop... + 2016-11-11 16:33:40,435 + INFO MASTER Connecting to Mcnode2 as hadoop... + 2016-11-11 16:33:40,444 + INFO MASTER Connecting to Mcnode3 as hadoop... + 2016-11-11 16:33:40,463 + INFO MASTER Connecting to Mcnode4 as hadoop... + 2016-11-11 16:33:40,480 + INFO MASTER Connecting to Mcnode5 as hadoop... + 2016-11-11 16:33:40,486 + INFO MASTER Connecting to Mcnode6 as hadoop... + 2016-11-11 16:33:40,526 + INFO MASTER Connecting to Mcnode7 as hadoop... + /home/hadoop/cloud/alluxio-1.3.0/bin + /home/hadoop/cloud/alluxio-1.3.0/bin + /home/hadoop/cloud/alluxio-1.3.0/bin + /home/hadoop/cloud/alluxio-1.3.0/bin + /home/hadoop/cloud/alluxio-1.3.0/bin + /home/hadoop/cloud/alluxio-1.3.0/bin + [sudo] password for hadoop: [sudo] password for hadoop: /home/hadoop/cloud/alluxio-1.3.0/bin + [sudo] password for hadoop: [sudo] password for hadoop: [sudo] password for hadoop: [sudo] password for hadoop: [sudo] password for hadoop: 2016-11-11 16:35:40,728 + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24014--alluxio\345\206\205\345\255\230\346\226\207\344\273\266\345\212\240\350\275\275\346\226\271\345\274\217\345\222\214\345\210\206\345\270\203\346\203\205\345\206\265\345\210\206\346\236\220.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24014--alluxio\345\206\205\345\255\230\346\226\207\344\273\266\345\212\240\350\275\275\346\226\271\345\274\217\345\222\214\345\210\206\345\270\203\346\203\205\345\206\265\345\210\206\346\236\220.md" new file mode 100644 index 0000000..d455687 --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24014--alluxio\345\206\205\345\255\230\346\226\207\344\273\266\345\212\240\350\275\275\346\226\271\345\274\217\345\222\214\345\210\206\345\270\203\346\203\205\345\206\265\345\210\206\346\236\220.md" @@ -0,0 +1,397 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +>>alluxio-0.7.1 通过copyFromLocal指令上传文件，发现D1Line.fasta-D6Line.fasta都在一个节点，Mcnode1; +但是alluxio-1.3.1,通过load一个大文件，发现数据会分布在不同节点；但是使用sparkContext的textfile，发现有两份数据,具体请见下面的分析 + +master没有部署work + +#0 以下分析是主要是基于spark-1.5.2和alluxio-1.3.0分析的，默认不支持数据本地化，所以会有跨节点传输 +# 1 load和free分析 ## + +## 1.1 load和free分析和观察结果## + （1）master load,各个节点都会有数据;work节点load，只有work节点有数据，前提是master节点没有work节点 + （2）load数据来源是hdfs，数据大小放到内存也是128M + （3）alluxio数据分布与hdfs不一致，D7Line.fasta:在HDFS没有存在Mcnode5，但是load后会有（master运行） + （4）D7Line.fasta大概2.01G,load很慢，超过一分钟，但是free是秒级的 + （5）从hdfs加载到alluxio，内存中查看到是一个节点一个节点串行操作，没有并行，而且是一个block一个block操作，节点内也没有并行，因为load是单线程 + （6）load运行在Master，通过ganglia发现Mcnode1的节点cache很多，接近饱和14G + （7）数据分布基本均匀（master运行） + （8）串行加载时应该是按照文件的block顺序加载，应为最小的文件应该是最后加载的--可以优化成并行 + （9）free后释放了部分内存， cache还是比较大，但不确定这部分数据是什么 + (10) 在Mcnode2 load，在Mcnode3可以free + (11)load 加载数据一份，没有copy + +## 1.2 代码 ## +## 1.2.1 load## + + hadoop@Master:~/xubo/project/alignment/SparkSW/SparkSW20161026/file$ ~/cloud/alluxio-1.3.0/bin/alluxio fs load /xubo/project/SparkSW/input/D7Line.fasta + /xubo/project/SparkSW/input/D7Line.fasta loaded + +D10Line.fasta很久：14.16 GB大概load超过半个小时 + + hadoop@Master:~/xubo/project/alignment/SparkSW/SparkSW20161026/file$ ~/cloud/alluxio-1.3.0/bin/alluxio fs load /xubo/project/SparkSW/input/D10Line.fasta + /xubo/project/SparkSW/input/D10Line.fasta loaded + +## 1.2.2 free## + + hadoop@Master:~/xubo/project/alignment/SparkSW/SparkSW20161026/file$ ~/cloud/alluxio-1.3.0/bin/alluxio fs free /xubo/project/SparkSW/input/D7Line.fasta + /xubo/project/SparkSW/input/D7Line.fasta was successfully freed from memory. + +## 1.3 结果 ## +## 1.3.1 D7Line.fasta的load和free ## +load D7Line.fasta: +![](http://i.imgur.com/QKqvqI4.png) + +free D7Line.fasta: +![](http://i.imgur.com/HINvWot.png) + +## 1.3.2 D10Line.fasta的load ## +load D10Line.fasta: +![](http://i.imgur.com/NWbdfpq.png) + +load时内存监控 +![](http://i.imgur.com/rEdRzZx.png) + +## 1.3.3 D9Line.fasta的load（没有删除D10） ## + +D9D10 load： +![](http://i.imgur.com/GDekoTn.png) + +两个文件加载后内存情况： +![](http://i.imgur.com/lb4IQcv.png) + +network: +![](http://i.imgur.com/2tORima.png) + +## 1.3.4 free D9 D10 ## +该数据是在运行copyFromLocal之后再free的 + +![](http://i.imgur.com/Dus0yUI.png) + +## 1.3.5 work load## +Mcnode2 ： +![](http://i.imgur.com/aTy6kvu.png) + +Mcnode3： + +![](http://i.imgur.com/Oo0jmyN.png) + + +# 2 copyFromLocal分析 +运行在Mcnode1 +## 2.1 copyFromLocal分析和观察结果## + （1）copyFromLocal是本地缓存，即从本地磁盘加载到本地的内存，没有跨Alluio节点传输缓存（work节点运行） + （2）由于（1），但文件大于单个节点，小雨集群容量时，使用copyFromLocal没法成功（work节点运行），而且会删除原来节点缓存的文件，应该算是bug** + （3）copyFromLocal速度很快，12G左右的数据在2分钟左右，明显快于load + （3）copyFromLocal可以跨节点存储（master节点运行） + +## 2.2 代码： ## + +## 2.2.1 copyFromLocal ## + + hadoop@Mcnode1:~/disk2/xubo/data/SparkSW/uniref$ ~/cloud/alluxio-1.3.0/bin/alluxio fs copyFromLocal uniref100.fasta / + Failed to cache: Not enough space left on worker Mcnode1/Mcnode1IP:29998 to store blockId 33638318104. Please consult http://www.alluxio.org/docs/1.3/en/Debugging-Guide.html for common solutions to address this problem. + + +## 2.3 结果： +## 2.3.1 copyFromLocal数据大于alluxio单节点内存 ## + + hadoop@Mcnode1:~/disk2/xubo/data/SparkSW/uniref$ ~/cloud/alluxio-1.3.0/bin/alluxio fs copyFromLocal uniref100.fasta / + Failed to cache: Not enough space left on worker Mcnode1/Mcnode1:29998 to store blockId 33621540888. Please consult http://www.alluxio.org/docs/1.3/en/Debugging-Guide.html for common solutions to address this problem. + +99%的内存使用： + +![](http://i.imgur.com/eLXwSqQ.png) + +第二次运行： + + hadoop@Mcnode1:~/disk2/xubo/data/SparkSW/uniref$ ~/cloud/alluxio-1.3.0/bin/alluxio fs copyFromLocal uniref100.fasta / + Failed to cache: Not enough space left on worker Mcnode1/Mcnode1:29998 to store blockId 33638318104. Please consult http://www.alluxio.org/docs/1.3/en/Debugging-Guide.html for common solutions to address this problem. + +![](http://i.imgur.com/DpWAot2.png) + +fail之后变成0 +![](http://i.imgur.com/U80Cudf.png) + +但是内存变化没那么大： +![](http://i.imgur.com/ezvkfOn.png) + + + +# 3 SparkContext.textfile and count分析 # +形式：sc.textFile("alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta") +## 3.1 SparkContext.textfile and count分析和观察结果## + + （1）通过SparkContext.textfile加载alluxio中的文件也会缓存数据到alluxio内存中 + （2）通过分析D5Line.fasta数据集可以发现，在Mcnode6有D5Line.fasta的完整数据集，其他数据分布在Mcnode1、Mcnode5，Mcnode6中，而D5Line.fasta在hdfs主要分布在Mcnode1、2、3、4中，所以数据分布也不是按照HDFS进行读取和存储，需要跨节点传输 + （3）通过分析D7Line.fasta数据集可以发现，2.01G的数据并没有出现在一个节点上，而且多个节点 + （4）根据上一条发现，有两份数据集会被加载 + （5）D5,D7没有存在Mcnode5、6，但是textFile后会有，所以也不是按照HDFS读取到内存 + （6）Mcnode1、2、3、4的内存cache明显大于Mcnode5、6、7，猜测是HDFS主要存储在前四个节点 + （7）Master的内存使用已满，而且use部分相对于其他七个work节点，明显要大，master的use接近10G + +## 3.2 代码： + + val rdd1= sc.textFile("alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta") + rdd1.count + +spark-shell： + + scala> val rdd1= sc.textFile("alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta") + rdd1: org.apache.spark.rdd.RDD[String] = MapPartitionsRDD[3] at textFile at :21 + + scala> rdd1.count + res1: Long = 1252720 + + scala> val rdd1= sc.textFile("alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta") + rdd1: org.apache.spark.rdd.RDD[String] = MapPartitionsRDD[5] at textFile at :21 + + scala> rdd1.count + res2: Long = 5019006 + +## 3.3 结果： + +## 3.3.1 D5文件 +总共大小405.63 MB +Mcnode4和5同步加载，Mcnode4数据在前：开始 +![](http://i.imgur.com/8YTEtMd.png) +Mcnode4和Mcnode5同步加载，Mcnode4数据在前：增加 +![](http://i.imgur.com/tfEyBmY.png) +Mcnode4和Mcnode6同步加载，Mcnode4数据在前：Mcnode5已经加载好，Mcnode6开始 +![](http://i.imgur.com/WtJ2exX.png) +Mcnode4和Mcnode1同步加载，Mcnode4数据在前：Mcnode5、Mcnode6已经加载好，Mcnode1开始 +![](http://i.imgur.com/qY7NaL3.png) +均加载好： +![](http://i.imgur.com/9v4cVMe.png) + +## 3.3.2 D7文件 +![](http://i.imgur.com/CbgbBQa.png) + +加载顺序： +![](http://i.imgur.com/5lAVczv.png) + +![](http://i.imgur.com/xb3o4Jv.png) + +![](http://i.imgur.com/IJxT2NG.png) + +![](http://i.imgur.com/d9Z1dHu.png) + +## 3.3.3 Master和Mcnode1-7等work的内存情况 + +![](http://i.imgur.com/u5PkSZ5.png) + +# 4. textFile hdfs and saveAsTextFile alluxio分析 +形式： + + val rdd1= sc.textFile("hdfs://Master:9000//xubo/project/SparkSW/input/D1Line.fasta") + rdd1.saveAsTextFile("alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta") + +## 4.1 textFile hdfs and saveAsTextFile alluxio分析和观察结果## + （1）在master使用spark submit执行，rdd读入的时候会进行partition，然后存储的时候在hdfs block所在节点，试了7次，而且较大概率会分布在不同的节点 + （2）spark-shell报错 + （3）写到alluxio中的时候是多线程并行的 + + +## 4.2 代码 + + val rdd1= sc.textFile("hdfs://Master:9000//xubo/project/SparkSW/input/D1Line.fasta") + rdd1.saveAsTextFile("alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta") +脚本 + + hadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ vi saveFile.sh + + for i in 1 2 3 4 5 6 7 8 9 + do + echo 'time:'$i + sh test.sh 6G hdfs://Master:9000/xubo/project/SparkSW/input/'D'$i'Line.fasta' 'alluxio://Master:19998/xubo/alluxio/uniref/D'$i'Line.fasta' save + done +## 4.3 结果 +## 4.3.1 存储的是partition +![](http://i.imgur.com/Bmdn9MG.png) + +![](http://i.imgur.com/pwRCZtv.png) + +##4.3.2 多线程并行 +![](http://i.imgur.com/Fa29iTl.png) +![](http://i.imgur.com/irMpk6v.png) +![](http://i.imgur.com/PLcMgHf.png) +![](http://i.imgur.com/EuapQfr.png) +![](http://i.imgur.com/OfNW8tW.png) +![](http://i.imgur.com/L0MlpVC.png) +最后存储的数据分布很均匀 + +## 4.3.3 记录 + + hadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ sh test.sh 6G hdfs://Master:9000/xubo/project/SparkSW/input/'D6Line.fasta' alluxio://Master:19998/xubo/alluxio/uniref/'D6Line.fasta' + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta countTime:39.499s partitons.length:8 + + +##记录 + +##第一次：alluxio-1.3.0 + + hadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ tail -f saveFile201611122308.txt + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta/SaveFile20161112230807255 countTime:6.142s partitons.length:2 + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta/SaveFile20161112230818840 countTime:5.413s partitons.length:2 + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta/SaveFile20161112230829721 countTime:10.416s partitons.length:2 + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta/SaveFile20161112230845420 countTime:7.973s partitons.length:2 + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta/SaveFile20161112230858790 countTime:16.227s partitons.length:4 + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta/SaveFile20161112230920089 countTime:16.411s partitons.length:8 + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta/SaveFile20161112230941702 countTime:33.634s partitons.length:16 + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta/SaveFile20161112231020474 countTime:32.927s partitons.length:34 + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta/SaveFile20161112231058702 countTime:136.582s partitons.length:63 + +##第二次：Alluxio-1.3.0 + + hadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ tail -f saveFile201611122328.txt + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta countTime:5.697s partitons.length:2 + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta countTime:7.714s partitons.length:2 + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta countTime:7.813s partitons.length:4 + 16/11/12 23:29:43 WARN QueuedThreadPool: 1 threads could not be stopped + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta countTime:18.261s partitons.length:8 + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta countTime:77.04s partitons.length:16 + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta countTime:77.75s partitons.length:34 + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta countTime:92.287s partitons.length:63 + +D9： + + hadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ time sh test.sh 6G hdfs://Master:9000/xubo/project/SparkSW/input/'D9Line.fasta' alluxio://Master:19998/xubo/alluxio/uniref/'D9Line.fasta' + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta countTime:156.214s partitons.length:63 + + real 2m41.526s + user 0m14.578s + sys 0m0.941s + +##第三次：Alluxio-0.7.1 + + hadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ ./saveFile-0.7.1.sh + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta countTime:11.446s partitons.length:2 + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta countTime:11.666s partitons.length:2 + 16/11/13 19:22:22 WARN QueuedThreadPool: 2 threads could not be stopped + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta countTime:19.249s partitons.length:2 + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta countTime:23.373s partitons.length:2 + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta countTime:42.418s partitons.length:4 + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta countTime:89.122s partitons.length:8 + +第七个文件D7卡住了20多分钟没完成，重新跑： + + hadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ ./saveFile-0.7.1.sh + time:7 + ^C^[[Ahadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ vi saveFile-0.7.1.sh + hadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ ./saveFile-0.7.1.sh + file:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta countTime:92.267s partitons.length:16 + file:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta countTime:110.32s partitons.length:34 + file:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta countTime:373.661s partitons.length:63 + +##第四次：Alluxio-0.7.1 + + hadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ tail -f saveFile-0.7.1time201611132245.txt + file:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta countTime:6.339s partitons.length:2 + file:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta countTime:11.167s partitons.length:2 + file:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta countTime:10.849s partitons.length:2 + 16/11/13 22:45:49 WARN QueuedThreadPool: 2 threads could not be stopped + file:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta countTime:15.198s partitons.length:2 + file:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta countTime:28.717s partitons.length:4 + file:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta countTime:40.355s partitons.length:8 + file:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta countTime:60.063s partitons.length:16 + file:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta countTime:147.046s partitons.length:34 + file:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta countTime:320.456s partitons.length:63 + +## 第五次：Alluxio-0.7.1 + + hadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ tail -f saveFile-0.7.1time201611140810.txt + file:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta Time:7.743s partitons.length:2 + file:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta Time:7.977s partitons.length:2 + file:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta Time:10.45s partitons.length:2 + 16/11/14 08:11:32 WARN QueuedThreadPool: 3 threads could not be stopped + file:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta Time:29.436s partitons.length:2 + file:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta Time:18.999s partitons.length:4 + file:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta Time:64.348s partitons.length:8 + file:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta Time:102.527s partitons.length:16 + file:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta Time:149.429s partitons.length:34 + file:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta Time:338.072s partitons.length:63 + file:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D10Line.fasta Time:619.74s partitons.length:114 + + +# 5 总结# +## 5.1说明 + + 没去看源码，之后需要去学习，是观察所得 +## 5.1 区别 + +## 5.1.1 alluxio中load和textFile的区别： + +- （1）load单个一个一个加载，textFile多个block并行加载 + + +- （2）load和textFile加载后数据分布不一致 +![1](http://i.imgur.com/Y8N5rlo.png) + +- （3）load加载后一份，textFile加载后两份数据 +- （4）load串行，textFile两个并行加载 + +## 5.1.2 多次使用textfile + + 多次使用textfile，最后数据会比原来文件大，而且每个节点数据会差不多，推测是每次textfile会重新分布数据，而不是从alluxio读取，没有截图，之前跑alluxio-0.7.1留下的印象 +free之后同一个文件将在各个节点的cache都会free + +## 5.2 问题 +当在Mcnode1运行时，数据都存在Mcnode1，但在Master运行时，数据存储在work各个节点 + +原因分析：master没有alluxio的work节点，不存储数据，只管理元数据，所以需要分配到各个节点存储 + +## textfile（alluxiofile）与 textfile（hdfs file）在saveAsTextFile（alluxio file）的区别 + textfile（alluxiofile）会缓存block，进行count + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24015--alluxio\346\200\247\350\203\275\345\210\206\346\236\220\345\222\214\345\212\240\351\200\237\346\226\271\345\274\217.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24015--alluxio\346\200\247\350\203\275\345\210\206\346\236\220\345\222\214\345\212\240\351\200\237\346\226\271\345\274\217.md" new file mode 100644 index 0000000..dbb420c --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24015--alluxio\346\200\247\350\203\275\345\210\206\346\236\220\345\222\214\345\212\240\351\200\237\346\226\271\345\274\217.md" @@ -0,0 +1,126 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 +#0 以下分析是主要是基于spark-1.5.2和alluxio-1.3.0分析的，默认不支持数据本地化，所以会有跨节点传输 +# 1.解释 # + +## 1.1 使用count进行简单数据分析## + count比较常见，分别对D1Line.fasta等数据集进行分析， + +## 1.2 ## + + +# 2.代码： # + +## 2.1 run.sh 脚本## + + for j in 1 2 3 4 5 6 + do + sh test.sh 10G hdfs://Master:9000/xubo/project/SparkSW/input/'D'$j'Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + sh test.sh 10G alluxio://Master:19998/xubo/project/SparkSW/input/'D'$j'Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + #sh test.sh 10G alluxio://Master:19998/xubo/project/SparkSW/input/'D'$j'Line.fasta' alluxio://Master:19998/xubo/project/alluxio/output/TimeCount + done + + +## 2.2 ## + + +# 3.结果： # + +数据越大加速越明显 + +##第一次 + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run.sh + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fastacount:78295countTime:3.791s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fastacount:78295countTime:4.726s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fastacount:156590countTime:4.001s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fastacount:156590countTime:5.244s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fastacount:313180countTime:4.329s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fastacount:313180countTime:26.376s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fastacount:626360countTime:7.918s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fastacount:626360countTime:6.199s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fastacount:1252720countTime:15.128s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fastacount:1252720countTime:9.661s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fastacount:2505440countTime:15.288s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fastacount:2505440countTime:8.724s + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ rm SparkSW.jar + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ rz + rz waiting to receive. +##第二次 + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run.sh + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fastacount:78295countTime:4.092s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fastacount:78295countTime:4.819s + Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000003fff80000, 11453595648, 0) failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (malloc) failed to allocate 11453595648 bytes for committing reserved memory. + # An error report file with more information is saved as: + # /home/hadoop/disk2/xubo/project/alluxio/timeCount/hs_err_pid21271.log + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fastacount:156590countTime:12.183s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fastacount:313180countTime:10.532s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fastacount:313180countTime:4.685s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fastacount:626360countTime:8.789s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fastacount:626360countTime:26.1s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fastacount:1252720countTime:5.16s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fastacount:1252720countTime:6.694s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fastacount:2505440countTime:19.338s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fastacount:2505440countTime:29.997s +##第三次 + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run.sh + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fastacount:78295countTime:4.335s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fastacount:78295countTime:5.461s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fastacount:156590countTime:4.352s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fastacount:156590countTime:6.373s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fastacount:313180countTime:10.727s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fastacount:313180countTime:6.621s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fastacount:626360countTime:17.001s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fastacount:626360countTime:8.243s + Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000003fff80000, 11453595648, 0) failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (malloc) failed to allocate 11453595648 bytes for committing reserved memory. + # An error report file with more information is saved as: + # /home/hadoop/disk2/xubo/project/alluxio/timeCount/hs_err_pid25150.log + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fastacount:1252720countTime:21.476s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fastacount:2505440countTime:9.553s + Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000003fff80000, 11453595648, 0) failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (malloc) failed to allocate 11453595648 bytes for committing reserved memory. + # An error report file with more information is saved as: + # /home/hadoop/disk2/xubo/project/alluxio/timeCount/hs_err_pid25626.log + +## 第四次 + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run.sh + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:5.055s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:4.044s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.961s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.892s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:5.071s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.461s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:9.366s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:15.696s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:17.386s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:6.127s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:41.361s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:7.352s + +##问题： + Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000003fff80000, 11453595648, 0) failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (malloc) failed to allocate 11453595648 bytes for committing reserved memory. + # An error report file with more information is saved as: + # /home/hadoop/disk2/xubo/project/alluxio/timeCount/hs_err_pid25626.log + +节点间scp传输速度为10M： + +![](http://i.imgur.com/MNpJevk.png) + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24016--\345\260\206alluxio\346\226\207\344\273\266\347\232\204block\345\244\247\345\260\217\344\277\256\346\224\271.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24016--\345\260\206alluxio\346\226\207\344\273\266\347\232\204block\345\244\247\345\260\217\344\277\256\346\224\271.md" new file mode 100644 index 0000000..c1e2796 --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24016--\345\260\206alluxio\346\226\207\344\273\266\347\232\204block\345\244\247\345\260\217\344\277\256\346\224\271.md" @@ -0,0 +1,67 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +# 1.alluxio-1.3.0 # + +## 1.0 默认大小## +alluxio-1.3.0的block默认大小为512M，对于我每个节点只有10几个G的小内存来说，而且还需要进行spark运算，用于alluxio的大小不能太大，所以512M粒度比较大。需要修改 + +## 1.1 操作## + 将配置文件中增加为： + alluxio.user.block.size.bytes.default=128MB + + +# 1.2.代码： # + +## 1.2.1 copy## + + hadoop@Master:~/cloud/alluxio-1.3.0/conf$ cp alluxio-site.properties.template alluxio-site.properties + +## 1.2.2 修改## + 在alluxio-site.properties最后增加一行，大小根据情况设置，我的修改为128M，与HDFS一致 + + # User properties + # alluxio.user.file.readtype.default=CACHE_PROMOTE + # alluxio.user.file.writetype.default=MUST_CACHE + alluxio.user.block.size.bytes.default=128MB + +scp到其他节点，重启生效 + +# 1.3.结果： # + +##19999查看生效： + ![](http://i.imgur.com/Q4Vm2fV.png) + +##是128M的倍数，不是512的倍数： +![](http://i.imgur.com/K1rE4TW.png) + +# 2. alluxio-0.7.1 + +##2.1 修改 + + hadoop@Master:~/cloud/tachyon-0.7.1-hadoop2.6-bin/conf$ cp ../common/src/main/resources/tachyon-default.properties tachyon-site.properties + hadoop@Master:~/cloud/tachyon-0.7.1-hadoop2.6-bin/conf$ ls + core-site.xml.template log4j.properties metrics.properties.template slaves tachyon-env.sh tachyon-env.sh.swift tachyon-env.sh.template tachyon-glusterfs-env.sh.template tachyon-site.properties workers + hadoop@Master:~/cloud/tachyon-0.7.1-hadoop2.6-bin/conf$ vi tachyon-site.properties + +修改内容： + + # User properties + tachyon.user.failed.space.request.limits=3 + tachyon.user.heartbeat.interval.ms=1000 + tachyon.user.file.writetype.default=CACHE_THROUGH + tachyon.user.default.block.size.byte=128MB +分发到各个节点，重启alluxio + +## 2.2 结果 + +![](http://i.imgur.com/c79ZMs3.png) + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24017--\345\244\232\346\254\241\350\277\220\350\241\214Space Usage\345\242\236\345\212\240.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24017--\345\244\232\346\254\241\350\277\220\350\241\214Space Usage\345\242\236\345\212\240.md" new file mode 100644 index 0000000..8cda76f --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24017--\345\244\232\346\254\241\350\277\220\350\241\214Space Usage\345\242\236\345\212\240.md" @@ -0,0 +1,165 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +#0 以下分析是主要是基于spark-1.5.2和alluxio-1.3.0分析的，默认不支持数据本地化，所以会有跨节点传输 + +# 1.解释 # + +## 1.1 问题## + 当将文件通过fs copyFromLocal传到alluxio时，开始8台大概是16G左右，但是多次运行 Space Usage会增加，即使每增加新文件。 + 不知道alluxio里面的实现机制是什么？ + +# 2.代码： # + +## 2.1 run.sh## + + #~/cloud/alluxio-1.3.0/bin/alluxio fs mkdir -p /xubo/project/SparkSW/input/ + for j in 1 2 3 4 5 6 7 8 9 + do + sh test.sh 10G hdfs://Master:9000/xubo/project/SparkSW/input/'D'$j'Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + sh test.sh 10G alluxio://Master:19998/xubo/alluxio/uniref/'D'$j'Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + #sh test.sh 10G alluxio://Master:19998/xubo/project/SparkSW/input/'D'$j'Line.fasta' alluxio://Master:19998/xubo/project/alluxio/output/TimeCount + done + ~ + +## 2.2 test.sh## + +test.sh文件是一个读取并进行count的脚本 + +# 3.结果： # +## 3.1 运行前 +copyFromLocal之后：总共16G左右 +![](http://i.imgur.com/Stqg6ow.png) +##第一次运行 +第一次运行count后，明显增加，总共32G左右 +![](http://i.imgur.com/fHNRu6h.png) + +运行记录 + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run.sh + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:9.706s + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:7.656s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:6.719s + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:15.477s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:5.013s + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:16.75s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:7.942s + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:17.642s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:26.711s + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:31.092s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:15.409s + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:73.364s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:35.338s + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:64.195s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:87.111s + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:76.951s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:132.447s + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:190.699s + +ganglia中明显可以看到network开销： + +![](http://i.imgur.com/Hfi0wLv.png) + +##第二次运行 +![](http://i.imgur.com/deHFlpJ.png) +运行记录 + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run.sh + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:9.706s + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:7.656s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:6.719s + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:15.477s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:5.013s + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:16.75s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:7.942s + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:17.642s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:26.711s + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:31.092s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:15.409s + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:73.364s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:35.338s + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:64.195s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:87.111s + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:76.951s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:132.447s + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:190.699s + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run.sh + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:5.355s + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:7.633s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.989s + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.384s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.478s + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.671s + Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000003fff80000, 11453595648, 0) failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (malloc) failed to allocate 11453595648 bytes for committing reserved memory. + # An error report file with more information is saved as: + # /home/hadoop/disk2/xubo/project/alluxio/timeCount/hs_err_pid13258.log + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:8.492s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:6.681s + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:11.47s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:8.213s + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:21.302s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:45.31s + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:30.39s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:63.646s + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:69.009s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:96.257s + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:129.645s + + +第三次： + +![](http://i.imgur.com/Y8LZKs9.png) + +![](http://i.imgur.com/BB8I8U4.png) + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run.sh + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:4.205s + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.907s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:6.637s + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.212s + Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000003fff80000, 11453595648, 0) failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (malloc) failed to allocate 11453595648 bytes for committing reserved memory. + # An error report file with more information is saved as: + # /home/hadoop/disk2/xubo/project/alluxio/timeCount/hs_err_pid17054.log + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:6.732s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:17.264s + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:9.099s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:26.775s + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:30.943s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:18.174s + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:17.044s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:45.236s + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:36.763s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:61.551s + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:73.987s + Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000003fff80000, 11453595648, 0) failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (malloc) failed to allocate 11453595648 bytes for committing reserved memory. + # An error report file with more information is saved as: + # /home/hadoop/disk2/xubo/project/alluxio/timeCount/hs_err_pid19730.log + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:139.039s + +## 第N次后： +###总共： +![](http://i.imgur.com/Sev5mEY.png) +###Mcnode1: +![](http://i.imgur.com/VFcl9Tr.png) +###Mcnode2: +![](http://i.imgur.com/Pycc7Ib.png) +###Mcnode3: +![](http://i.imgur.com/h5ADuci.png) + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24018--alluxio work 3000\345\207\272\347\216\260\351\227\256\351\242\230.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24018--alluxio work 3000\345\207\272\347\216\260\351\227\256\351\242\230.md" new file mode 100644 index 0000000..e57b9ce --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24018--alluxio work 3000\345\207\272\347\216\260\351\227\256\351\242\230.md" @@ -0,0 +1,74 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +# 1.解释 # + +待解决 + +## 1.1 问题## +访问http://mcnode6:30000/home时出现的 + + Problem accessing /home. Reason: + + Server Error + Caused by: + + org.apache.jasper.JasperException: PWC6345: There is an error in invoking javac. A full JDK (not just JRE) is required + at org.apache.jasper.compiler.DefaultErrorHandler.jspError(DefaultErrorHandler.java:92) + at org.apache.jasper.compiler.ErrorDispatcher.dispatch(ErrorDispatcher.java:378) + at org.apache.jasper.compiler.ErrorDispatcher.jspError(ErrorDispatcher.java:119) + at org.apache.jasper.compiler.Jsr199JavaCompiler.compile(Jsr199JavaCompiler.java:208) + at org.apache.jasper.compiler.Compiler.generateClass(Compiler.java:384) + at org.apache.jasper.compiler.Compiler.compile(Compiler.java:453) + at org.apache.jasper.JspCompilationContext.compile(JspCompilationContext.java:625) + at org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:375) + at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:473) + at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:377) + at org.eclipse.jetty.jsp.JettyJspServlet.service(JettyJspServlet.java:103) + at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) + at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:812) + at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:587) + at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) + at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:595) + at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:223) + at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127) + at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515) + at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) + at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061) + at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) + at org.eclipse.jetty.server.Dispatcher.forward(Dispatcher.java:191) + at org.eclipse.jetty.server.Dispatcher.forward(Dispatcher.java:72) + at alluxio.web.WebInterfaceWorkerGeneralServlet.doGet(WebInterfaceWorkerGeneralServlet.java:200) + at javax.servlet.http.HttpServlet.service(HttpServlet.java:687) + at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) + at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:812) + at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:587) + at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) + at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:577) + at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:223) + at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127) + at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515) + at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) + at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061) + at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) + at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) + at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) + at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) + at org.eclipse.jetty.server.Server.handle(Server.java:499) + at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:311) + at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257) + at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:544) + at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) + at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) + at java.lang.Thread.run(Thread.java:745) + Powered by Jetty:// + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24019--alluxio\346\200\247\350\203\275\346\217\220\345\215\207\345\210\206\346\236\220\344\270\216\345\256\236\351\252\2141.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24019--alluxio\346\200\247\350\203\275\346\217\220\345\215\207\345\210\206\346\236\220\344\270\216\345\256\236\351\252\2141.md" new file mode 100644 index 0000000..fcd286e --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24019--alluxio\346\200\247\350\203\275\346\217\220\345\215\207\345\210\206\346\236\220\344\270\216\345\256\236\351\252\2141.md" @@ -0,0 +1,498 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +#0 以下分析是主要是基于spark-1.5.2和alluxio-1.3.0分析的，默认不支持数据本地化，所以会有跨节点传输 + +# 1.解释 # + +## 1.1 分析## +alluxio会把数据缓存在work的本地，虽然使用spark从alluxio读取hdfs会使得文件分布与hdfs不一致，但还是可以达到加速的目的，特别是hdfs与spark不在一个集群，alluxio可以达到很好的缓存作用；baidu 30倍提升的背景和方法类似 + +## 1.2 实验## + 分别从hdfs读取文件和alluxio读取文件，进行count计算，文件大小从32M到7.8G + +# 2.代码： # +脚本 + + #~/cloud/alluxio-1.3.0/bin/alluxio fs mkdir -p /xubo/project/SparkSW/input/ + for j in 1 2 3 4 5 6 7 8 9 + do + for i in 1 2 3 4 5 6 7 8 9 10 + do + echo 'time:'$i + sh test.sh 14G hdfs://Master:9000/xubo/project/SparkSW/input/'D'$j'Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + done + done + + for j in 1 2 3 4 5 6 7 8 9 + do + for i in 1 2 3 4 5 6 7 8 9 10 + do + echo 'time:'$i + sh test.sh 6G alluxio://Master:19998/xubo/project/SparkSW/input/'D'$j'Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + done + + + done + + + +# 3.结果： # + +## 3.1 从hdfs读取文件进行count + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f run1_201611121727.txt + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:4.1s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:5.14s + time:3 + Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000003fff80000, 11453595648, 0) failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (malloc) failed to allocate 11453595648 bytes for committing reserved memory. + # An error report file with more information is saved as: + # /home/hadoop/disk2/xubo/project/alluxio/timeCount/hs_err_pid26806.log + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:5.023s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.984s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:4.037s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.847s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.966s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:5.015s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:5.425s + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.012s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:6.533s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:6.44s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:6.528s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.048s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.145s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.055s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.031s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:6.665s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.065s + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:5.41s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.779s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:9.414s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.267s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:9.446s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.53s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.306s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:9.39s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:9.307s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.463s + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:16.953s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:8.625s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:8.015s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:6.104s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:12.854s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:6.274s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:17.06s + time:8 + Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000003fff80000, 11453595648, 0) failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (malloc) failed to allocate 11453595648 bytes for committing reserved memory. + # An error report file with more information is saved as: + # /home/hadoop/disk2/xubo/project/alluxio/timeCount/hs_err_pid1475.log + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:16.995s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:7.915s + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:15.202s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:7.891s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:6.638s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:5.049s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:5.176s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:5.181s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.888s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:15.406s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:6.229s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:6.14s + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:26.699s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:15.33s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:26.675s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:15.361s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:26.745s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:16.951s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:17.046s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:26.805s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:27.031s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:15.89s + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:55.006s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:43.435s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.068s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:35.5s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.111s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.015s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:44.477s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:41.526s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.072s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.481s + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:84.314s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:27.691s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:44.684s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:96.937s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:61.365s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:38.072s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:43.018s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:85.482s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:27.828s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:52.804s + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:110.129s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:166.146s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:97.836s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:51.683s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:107.544s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:96.966s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:100.279s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:61.431s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:38.482s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:63.901s + +## 3.2 从alluxio读取文件进行count + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f run1_201611121727_2.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:7.512s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:5.323s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.979s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:5.322s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:4.447s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:4.042s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.931s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:4.977s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.951s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:4.086s + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:5.199s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:5.605s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.147s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.868s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:5.324s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:10.215s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.079s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.097s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.082s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.134s + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:16.241s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:16.245s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:8.147s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:5.934s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.395s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.12s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.392s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.268s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.156s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.242s + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:16.096s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:25.805s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:13.501s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:4.419s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:13.76s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:7.432s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:7.975s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:4.239s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:4.489s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:4.519s + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:28.792s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:12.711s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:28.76s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:18.752s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:10.026s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.53s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.789s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.793s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:5.761s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.6s + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:63.302s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:11.117s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:9.801s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:9.064s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:28.815s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:7.811s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:9.686s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.816s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:6.598s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.728s + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:93.021s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:39.388s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:50.501s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:45.915s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:44.043s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.48s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:32.945s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.906s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.884s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:29.972s + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:145.837s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:57.805s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:66.785s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:53.351s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:48.176s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:31.527s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:49.615s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:44.39s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.008s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:31.557s + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:228.964s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:200.829s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:167.487s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:87.298s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:68.357s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:71.835s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:61.462s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:44.198s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:40.567s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.631s +## 3.3 从alluxio读取文件进行count后alluxio文件情况 +master： +![](http://i.imgur.com/AYOiuvl.png) +Mcnode7 +![](http://i.imgur.com/VsHYPar.png) +Mcnode5 +![](http://i.imgur.com/1aZkVTz.png) +Mcnode4 +![](http://i.imgur.com/WTku1Yl.png) +Mcnode3: +![](http://i.imgur.com/7ub8mTf.png) +Mcnode2: +![](http://i.imgur.com/xqmLCjF.png) +Mcnode1: +![](http://i.imgur.com/Pm5e6zv.png) +network: +![](http://i.imgur.com/X1VL4Vq.png) + +## 3.5 D9提升进一步实验 +由于alluxio中D9的数据仍然没有100%，所以再运行了20遍： + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f run1D9time201611122031.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:31.358s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:36.08s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.398s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:31.578s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.958s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.9s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.807s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.638s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.583s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.812s + ^C + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f run1D9time201611122038.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.169s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.898s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.523s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.764s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.492s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.109s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.657s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.765s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.927s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:33.635s + +运行D9 30次后： + +![](http://i.imgur.com/Shv4tN6.png) +![](http://i.imgur.com/aStad0e.png) +![](http://i.imgur.com/4pSfQSK.png) +![](http://i.imgur.com/Fwuhrk9.png) +![](http://i.imgur.com/q5vqsVy.png) +![](http://i.imgur.com/Q3DOcno.png) + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2402---Spark\344\273\216tachyon\344\270\255\350\257\273\345\217\226\346\226\207\344\273\266.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2402---Spark\344\273\216tachyon\344\270\255\350\257\273\345\217\226\346\226\207\344\273\266.md" new file mode 100644 index 0000000..e84e3a4 --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2402---Spark\344\273\216tachyon\344\270\255\350\257\273\345\217\226\346\226\207\344\273\266.md" @@ -0,0 +1,35 @@ + + +1.启动spark-shell + +2.上传文件到tachyon： + + xubo@xubo:~/cloud/test/tachyon$ ../../tachyon-0.7.1/bin/tachyon tfs copyFromLocal 1.txt / + Copied 1.txt to / + xubo@xubo:~/cloud/test/tachyon$ ../../tachyon-0.7.1/bin/tachyon tfs cat /1.txt + hello tachyon + 1 + 2 + 3 + +3.读取： + + + scala> val text1= sc.textFile("tachyon://localhost:19998/1.txt") + text1: org.apache.spark.rdd.RDD[String] = MapPartitionsRDD[1] at textFile at :21 + + scala> text + text1 text + + scala> text1.foreach + foreach foreachPartition foreachWith + + scala> text1.foreach + def foreach(f: T => Unit): Unit + + scala> text1.foreach(println) + 1 + hello tachyon + 2 + 3 + diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24020--\347\226\221\351\227\256\357\274\232\344\270\272\344\273\200\344\271\210D7-D9\345\210\260\346\234\200\345\220\216\347\232\204count\346\227\266\351\227\264\351\203\275\346\230\25730s\345\267\246\345\217\263\357\274\237.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24020--\347\226\221\351\227\256\357\274\232\344\270\272\344\273\200\344\271\210D7-D9\345\210\260\346\234\200\345\220\216\347\232\204count\346\227\266\351\227\264\351\203\275\346\230\25730s\345\267\246\345\217\263\357\274\237.md" new file mode 100644 index 0000000..8c58306 --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24020--\347\226\221\351\227\256\357\274\232\344\270\272\344\273\200\344\271\210D7-D9\345\210\260\346\234\200\345\220\216\347\232\204count\346\227\266\351\227\264\351\203\275\346\230\25730s\345\267\246\345\217\263\357\274\237.md" @@ -0,0 +1,419 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 +#0 以下分析是主要是基于spark-1.5.2和alluxio-1.3.0分析的，默认不支持数据本地化，所以会有跨节点传输 +# 1.解释 # + +## 1.1 数据## +D7:2.01 GB +D8:4.16 GB +D9:7.79 GB + +## 1.2 说明 + 在D7-9上运行count，spark分布式执行，最后都是稳定在30s左右，不知道为什么？三个数据集大小不一样，但是最后收敛一样？ + + 猜想： + block设置的大小为128M，但是 + +# 2.运行记录 # + +## 2.1 运行记录 D7-D9 + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:93.021s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:39.388s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:50.501s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:45.915s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:44.043s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.48s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:32.945s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.906s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.884s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:29.972s + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:145.837s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:57.805s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:66.785s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:53.351s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:48.176s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:31.527s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:49.615s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:44.39s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.008s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:31.557s + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:228.964s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:200.829s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:167.487s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:87.298s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:68.357s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:71.835s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:61.462s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:44.198s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:40.567s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.631s + ^C + +## 3.2 多次运行D9 + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f run1D9time201611122031.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:31.358s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:36.08s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.398s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:31.578s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.958s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.9s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.807s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.638s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.583s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.812s + ^C + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f run1D9time201611122038.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.169s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.898s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.523s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.764s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.492s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.109s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.657s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.765s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.927s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:33.635s + ^[[B^[[B^C + +## 2.3 多次运行D8 + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f rund8time201611122051.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:127.607s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:44.799s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:85.756s + time:4 + Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000003fff80000, 11453595648, 0) failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (malloc) failed to allocate 11453595648 bytes for committing reserved memory. + # An error report file with more information is saved as: + # /home/hadoop/disk2/xubo/project/alluxio/timeCount/hs_err_pid12705.log + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:46.596s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:46.065s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:53.594s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.673s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.201s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:41.336s + ^C + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f rund8time201611122104.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:31.531s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:29.741s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:33.394s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:29.74s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:34.75s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:29.809s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.483s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.412s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:29.812s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.721s + ^C + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f rund8time201611122110.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.438s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.56s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.617s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.418s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.432s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.504s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.574s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.526s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.793s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.535s + ^[[A^C + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f rund8time201611122118.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.539s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.512s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.536s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.771s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.383s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.626s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.361s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.663s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.693s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.894s + ^C + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f rund7time201611122125.txt + +## 3.4 多次运行D7 + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:118.782s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:50.964s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:45.109s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:44.266s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:31.002s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.822s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.46s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.673s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.686s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.062s + time:11 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:29.93s + time:12 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:31.033s + time:13 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.61s + time:14 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.656s + time:15 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:31.083s + time:16 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.639s + time:17 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:29.984s + time:18 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.637s + time:19 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.219s + time:20 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.777s + time:21 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.849s + time:22 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.763s + time:23 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.83s + time:24 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.728s + time:25 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.615s + time:26 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.652s + time:27 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.678s + time:28 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.705s + time:29 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:31.004s + time:30 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.04s + +## 3.5 改变运行内存：10G excutor + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f rund7time201611122159mem10g.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.642s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:29.375s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.765s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.63s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.639s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.66s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.659s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.741s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:29.925s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.55s + time:11 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.787s + time:12 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.585s + time:13 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.626s + time:14 + ^C + +## 3.6 改变运行内存：10G excutor， 且释放掉alluxio其他文件 + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f rund7time201611122211mem10g.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.76s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.588s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.821s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.517s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.626s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.831s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.68s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.684s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.73s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.741s + time:11 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.819s + time:12 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.812s + time:13 + [Stage 0:=================================================> (14 + 2) / 16] + +## 3.7 使用saveAsTextFile： + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f run1d9time201611131738.txt + time:1 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:176.505s + time:2 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:160.029s + time:3 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:77.369s + time:4 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:63.931s + time:5 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:51.872s + time:6 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:40.886s + time:7 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:36.764s + time:8 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.678s + time:9 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:36.897s + time:10 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:52.303s + time:11 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.534s + time:12 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.507s + time:13 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.622s + time:14 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.395s + time:15 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.366s + time:16 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:32.348s + time:17 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:44.949s + time:18 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.473s + time:19 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.086s + time:20 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.495s + time:21 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.259s + time:22 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.362s + time:23 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.78s + time:24 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.974s + time:25 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.396s + time:26 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.641s + time:27 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:29.052s + time:28 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.5s + time:29 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.573s + time:30 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.224s + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24021--\347\226\221\351\227\256\357\274\232\344\270\272\344\273\200\344\271\210master\347\234\213\344\270\215\345\210\260D9,\350\200\214work\350\212\202\347\202\271\346\234\211D9\351\203\250\345\210\206\346\225\260\346\215\256\357\274\237.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24021--\347\226\221\351\227\256\357\274\232\344\270\272\344\273\200\344\271\210master\347\234\213\344\270\215\345\210\260D9,\350\200\214work\350\212\202\347\202\271\346\234\211D9\351\203\250\345\210\206\346\225\260\346\215\256\357\274\237.md" new file mode 100644 index 0000000..028a73b --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24021--\347\226\221\351\227\256\357\274\232\344\270\272\344\273\200\344\271\210master\347\234\213\344\270\215\345\210\260D9,\350\200\214work\350\212\202\347\202\271\346\234\211D9\351\203\250\345\210\206\346\225\260\346\215\256\357\274\237.md" @@ -0,0 +1,37 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +# 1.解释 # + +## 1.1 疑问## +为什么master看不到D9,而work节点有D9部分数据？ + +master也显示内存不止D7,因为D7大概2G + +可以free掉D9 + +# 2.截图 # + +## 2.1 master +![](http://i.imgur.com/7lgfEo4.png) + +![](http://i.imgur.com/fgUeUk8.png) + +## 2.2 works +![](http://i.imgur.com/YMkfaE5.png) + +![](http://i.imgur.com/QoKJ1Px.png) + +![](http://i.imgur.com/FACVCgx.png) + + + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24022--saveAsTextFile alluxio\345\220\216count \357\274\210\346\234\211bug\357\274\211.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24022--saveAsTextFile alluxio\345\220\216count \357\274\210\346\234\211bug\357\274\211.md" new file mode 100644 index 0000000..cd98b4f --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24022--saveAsTextFile alluxio\345\220\216count \357\274\210\346\234\211bug\357\274\211.md" @@ -0,0 +1,826 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 +#0 以下分析是主要是基于spark-1.5.2和alluxio-1.3.0分析的，默认不支持数据本地化，所以会有跨节点传输 +# 1.解释 # + +先加载到mem，然后count + +出现问题，D9是block丢失，主要是因为alluxio内存使用问题，每个work会将数据缓存，而不是分布式缓存，导致后面的不经常用的数据丢失 + +# 2.代码： # + + for j in 1 2 3 4 5 6 7 8 9 + do + for i in 1 2 3 4 5 6 7 8 9 10 + do + echo 'time:'$i + sh test.sh 6G alluxio://Master:19998/xubo/alluxio/uniref/'D'$j'Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + done + + + done + + +# 3.结果： # + +##alluxio-1.3.0 + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f run1time201611122335.txt \ + > ; + time:1 + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.455s + time:2 + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.432s + time:3 + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.486s + time:4 + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.605s + time:5 + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.874s + time:6 + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.66s + time:7 + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.488s + time:8 + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.391s + time:9 + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.064s + time:10 + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.018s + time:1 + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.9s + time:2 + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.706s + time:3 + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.779s + time:4 + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.809s + time:5 + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.063s + time:6 + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.188s + time:7 + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.882s + time:8 + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:7.166s + time:9 + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.786s + time:10 + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.84s + time:1 + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:5.316s + time:2 + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:5.533s + time:3 + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:5.299s + time:4 + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:5.381s + time:5 + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:10.477s + time:6 + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:5.333s + time:7 + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.183s + time:8 + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:10.117s + time:9 + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:5.327s + time:10 + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.247s + time:1 + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:24.306s + 16/11/12 23:43:00 WARN QueuedThreadPool: 1 threads could not be stopped + time:2 + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:15.898s + time:3 + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:5.999s + time:4 + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:6.053s + time:5 + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:14.931s + time:6 + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.422s + time:7 + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.535s + time:8 + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:6.46s + time:9 + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:6.261s + time:10 + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.549s + time:1 + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:16.498s + time:2 + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:17.061s + time:3 + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:6.414s + time:4 + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:7.03s + time:5 + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:6.505s + time:6 + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:6.117s + time:7 + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:6.952s + time:8 + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.545s + time:9 + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.553s + time:10 + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:17.072s + time:1 + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:18.648s + time:2 + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:18.926s + time:3 + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:17.453s + time:4 + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:18.81s + time:5 + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:16.726s + time:6 + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:7.699s + time:7 + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:8.189s + time:8 + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:16.938s + time:9 + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:6.611s + time:10 + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:6.54s + time:1 + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:28.424s + time:2 + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:28.496s + time:3 + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:34.933s + time:4 + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:28.524s + time:5 + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:28.146s + time:6 + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:29.543s + time:7 + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:41.073s + time:8 + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:28.614s + time:9 + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:28.967s + time:10 + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:32.312s + time:1 + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:90.415s + time:2 + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:58.984s + time:3 + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:31.455s + time:4 + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:40.781s + time:5 + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:34.593s + time:6 + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:49.076s + time:7 + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:29.161s + time:8 + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:36.917s + time:9 + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:31.651s + time:10 + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:28.131s + time:1 + +## alluxio-0.7.1 + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f run1-0.7.1time201611131949.txt + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.893s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.882s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.912s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.939s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.877s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.784s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.835s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.835s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.01s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.865s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.01s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.107s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.029s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:3.908s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:3.929s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:3.716s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.085s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.016s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.071s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:3.903s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.228s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.168s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.232s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.072s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.254s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.385s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.201s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.381s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.162s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.18s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.288s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.337s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.447s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.312s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.376s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.322s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.446s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.47s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.354s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.429s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:6.288s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.551s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.598s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.623s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:5.115s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.651s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.472s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.659s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.565s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.891s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.928s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.711s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.772s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.683s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.58s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.882s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.819s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:5.149s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.627s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.74s + + +## 报错 + + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:28.131s + time:1 + [Stage 0:> (1 + 16) / 63]16/11/13 00:06:31 ERROR TaskSetManager: Task 43 in stage 0.0 failed 4 times; aborting job + Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 43 in stage 0.0 failed 4 times, most recent failure: Lost task 43.3 in stage 0.0 (TID 22, Mcnode3IP): java.io.IOException: Block 5469372416 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) + + Driver stacktrace: + at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1283) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1271) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1270) + at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) + at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) + at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1270) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) + at scala.Option.foreach(Option.scala:236) + at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:697) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1496) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1458) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1447) + at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) + at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:567) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1824) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1837) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1850) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1921) + at org.apache.spark.rdd.RDD.count(RDD.scala:1125) + at alluxio.TimeCount$.compute(TimeCount.scala:32) + at alluxio.TimeCount$.main(TimeCount.scala:24) + at alluxio.TimeCount.main(TimeCount.scala) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:606) + at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:674) + at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180) + at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205) + at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:120) + at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) + Caused by: java.io.IOException: Block 5469372416 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) + time:2 + [Stage 0:=> (2 + 16) / 63]16/11/13 00:06:41 ERROR TaskSetManager: Task 24 in stage 0.0 failed 4 times; aborting job + Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 24 in stage 0.0 failed 4 times, most recent failure: Lost task 24.3 in stage 0.0 (TID 23, Mcnode1IP): java.io.IOException: Block 5721030656 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) + + Driver stacktrace: + at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1283) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1271) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1270) + at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) + at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) + at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1270) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) + at scala.Option.foreach(Option.scala:236) + at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:697) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1496) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1458) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1447) + at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) + at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:567) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1824) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1837) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1850) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1921) + at org.apache.spark.rdd.RDD.count(RDD.scala:1125) + at alluxio.TimeCount$.compute(TimeCount.scala:32) + at alluxio.TimeCount$.main(TimeCount.scala:24) + at alluxio.TimeCount.main(TimeCount.scala) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:606) + at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:674) + at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180) + at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205) + at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:120) + at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) + Caused by: java.io.IOException: Block 5721030656 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) + time:3 + [Stage 0:> (1 + 16) / 63]16/11/13 00:06:52 ERROR TaskSetManager: Task 43 in stage 0.0 failed 4 times; aborting job + Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 43 in stage 0.0 failed 4 times, most recent failure: Lost task 43.3 in stage 0.0 (TID 23, Mcnode3): java.io.IOException: Block 5469372416 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) + + Driver stacktrace: + at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1283) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1271) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1270) + at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) + at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) + at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1270) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) + at scala.Option.foreach(Option.scala:236) + at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:697) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1496) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1458) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1447) + at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) + at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:567) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1824) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1837) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1850) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1921) + at org.apache.spark.rdd.RDD.count(RDD.scala:1125) + at alluxio.TimeCount$.compute(TimeCount.scala:32) + at alluxio.TimeCount$.main(TimeCount.scala:24) + at alluxio.TimeCount.main(TimeCount.scala) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:606) + at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:674) + at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180) + at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205) + at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:120) + at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) + Caused by: java.io.IOException: Block 5469372416 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) + time:4 + [Stage 0:> (0 + 16) / 63]16/11/13 00:07:02 ERROR TaskSetManager: Task 59 in stage 0.0 failed 4 times; aborting job + Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 59 in stage 0.0 failed 4 times, most recent failure: Lost task 59.3 in stage 0.0 (TID 22, Mcnode4IP): java.io.IOException: Block 5486149632 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) + + Driver stacktrace: + at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1283) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1271) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1270) + at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) + at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) + at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1270) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) + at scala.Option.foreach(Option.scala:236) + at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:697) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1496) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1458) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1447) + at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) + at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:567) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1824) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1837) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1850) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1921) + at org.apache.spark.rdd.RDD.count(RDD.scala:1125) + at alluxio.TimeCount$.compute(TimeCount.scala:32) + at alluxio.TimeCount$.main(TimeCount.scala:24) + at alluxio.TimeCount.main(TimeCount.scala) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:606) + at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:674) + at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180) + at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205) + at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:120) + at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) + Caused by: java.io.IOException: Block 5486149632 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) + time:5 + [Stage 0:> (1 + 16) / 63]16/11/13 00:07:13 ERROR TaskSetManager: Task 59 in stage 0.0 failed 4 times; aborting job + Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 59 in stage 0.0 failed 4 times, most recent failure: Lost task 59.3 in stage 0.0 (TID 23, Mcnode3): java.io.IOException: Block 5486149632 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) + + Driver stacktrace: + at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1283) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1271) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1270) + at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) + at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) + at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1270) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) + at scala.Option.foreach(Option.scala:236) + at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:697) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1496) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1458) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1447) + at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) + at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:567) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1824) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1837) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1850) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1921) + at org.apache.spark.rdd.RDD.count(RDD.scala:1125) + at alluxio.TimeCount$.compute(TimeCount.scala:32) + at alluxio.TimeCount$.main(TimeCount.scala:24) + at alluxio.TimeCount.main(TimeCount.scala) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:606) + at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:674) + at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180) + at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205) + at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:120) + at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) + Caused by: java.io.IOException: Block 5486149632 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) + time:6 + [Stage 0:=> (2 + 16) / 63]16/11/13 00:07:24 ERROR TaskSetManager: Task 59 in stage 0.0 failed 4 times; aborting job + Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 59 in stage 0.0 failed 4 times, most recent failure: Lost task 59.3 in stage 0.0 (TID 23, Mcnode3): java.io.IOException: Block 5486149632 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24023--alluxio-0.7.1\350\247\243\345\206\263\346\225\260\346\215\256\346\234\254\345\234\260\345\214\226\347\232\204\351\227\256\351\242\230.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24023--alluxio-0.7.1\350\247\243\345\206\263\346\225\260\346\215\256\346\234\254\345\234\260\345\214\226\347\232\204\351\227\256\351\242\230.md" new file mode 100644 index 0000000..236586b --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24023--alluxio-0.7.1\350\247\243\345\206\263\346\225\260\346\215\256\346\234\254\345\234\260\345\214\226\347\232\204\351\227\256\351\242\230.md" @@ -0,0 +1,212 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +# 1.解释 # + +spark-1.5.2和alluxio-1.3.0默认不支持数据本地化，所以会有跨节点传输 + +# 2.解决办法： # + +## 2.1 方法1## +使用alluxio-0.7.1 + +## 2.2 方法2## + +设置spark本地化配置，参考【5】 +各个节点分别配置 +实验分析支持NODE_LOCAL + +# 3.结果： # + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f run1-0.7.1time201611132026.txt + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.812s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.87s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.046s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.942s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.818s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.773s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.837s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.756s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.864s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.811s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:3.964s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.792s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.013s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:3.87s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.066s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:3.976s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:3.893s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.242s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:3.969s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.183s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:3.987s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.035s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.128s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.137s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.253s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.139s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.059s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.051s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.074s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.197s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.313s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.36s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.372s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.329s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.379s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.299s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.382s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.33s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.26s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.306s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.565s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.419s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.39s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:5.67s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.632s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.513s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.474s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.492s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.574s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.586s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.643s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.67s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:5.148s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.722s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.639s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.561s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:5.089s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:5.238s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.677s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.723s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:6.634s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:6.763s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:5.376s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:5.515s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:5.274s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:5.447s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:5.512s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:6.46s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:5.394s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:5.373s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:6.527s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:6.64s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:6.574s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:6.482s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:6.718s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:6.509s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:6.583s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:6.643s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:6.602s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:6.573s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:10.066s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:10.084s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:10.93s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:10.347s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:9.976s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:10.817s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:10.358s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:9.881s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:10.23s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:10.119s + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning + 【5】http://www.alluxio.org/docs/master/cn/Running-Spark-on-Alluxio.html diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24024--\345\210\206\345\210\253\350\257\273\345\217\226HDFS\345\222\214Alluxio\347\232\204\346\225\260\346\215\256\350\277\233\350\241\214line count\346\257\224\350\276\203\345\210\206\346\236\220.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24024--\345\210\206\345\210\253\350\257\273\345\217\226HDFS\345\222\214Alluxio\347\232\204\346\225\260\346\215\256\350\277\233\350\241\214line count\346\257\224\350\276\203\345\210\206\346\236\220.md" new file mode 100644 index 0000000..fbb26d9 --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24024--\345\210\206\345\210\253\350\257\273\345\217\226HDFS\345\222\214Alluxio\347\232\204\346\225\260\346\215\256\350\277\233\350\241\214line count\346\257\224\350\276\203\345\210\206\346\236\220.md" @@ -0,0 +1,375 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +# 1.解释 # + + 测量alluxio和hdfs哪个速度快 + + +# 2.代码： # + +## 2.1 hdfs ## + for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 + do + echo 'time:'$i + sh test.sh 10G hdfs://Master:9000/xubo/project/SparkSW/input/'D9Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + done + +## 2.2 aluxio ## + + for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 + do + echo 'time:'$i + sh test.sh 10G alluxio://Master:19998/xubo/project/SparkSW/input/'D9Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + done +## 2.3 test.sh + + mem=$1 + input=$2 + + #/home/zgg/lib/spark-1.0.1-bin-hadoop2/bin/spark-submit \ + spark-submit \ + --class alluxio.TimeCount \ + --master spark://Master:7077 \ + --executor-memory $mem \ + SparkSW.jar $input $3 + + +# 3.结果： # + +##3.1 HDFS + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run1D9hdfs.sh + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:34.178s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:23.82s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.884s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.471s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:14.446s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:12.18s + 16/11/17 17:45:25 ERROR LiveListenerBus: SparkListenerBus has already stopped! Dropping event SparkListenerExecutorMetricsUpdate(6,WrappedArray()) + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:12.474s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:13.26s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.699s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.991s + time:11 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.977s + time:12 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.933s + time:13 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.322s + time:14 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.735s + time:15 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:12.489s + time:16 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.832s + time:17 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.883s + time:18 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.917s + time:19 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.032s + time:20 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.208s + time:21 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:12.089s + time:22 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.055s + time:23 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.839s + time:24 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:12.651s + time:25 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.999s + time:26 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.73s + time:27 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.621s + time:28 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.672s + time:29 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.65s + time:30 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.054s + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run1D9hdfs.sh + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.726s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.86s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.724s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.393s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.963s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.96s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.866s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.52s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.084s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.081s + time:11 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.56s + time:12 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.043s + time:13 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.598s + time:14 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.569s + time:15 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.594s + time:16 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.674s + time:17 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.713s + time:18 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.733s + time:19 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.691s + time:20 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.816s + time:21 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.64s + time:22 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.841s + time:23 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.537s + time:24 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.629s + time:25 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.702s + time:26 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.089s + time:27 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.99s + time:28 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.672s + time:29 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.597s + time:30 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.612s + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run1D9hdfs.sh + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:13.733s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:12.415s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:12.306s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:12.588s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:12.306s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:12.848s + [Stage 1:==========================================> (12 + 4) / 16]^CException in thread "main" org.apache.spark.SparkException: Job cancelled because SparkContext was shut down + +## 3.2 alluxio-1.3.0 + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run1D9.sh + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:202.784s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:124.612s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:66.245s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:63.52s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.778s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:75.229s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:50.952s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:41.106s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:39.525s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:39.792s + time:11 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:17.647s + time:12 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.726s + time:13 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:39.813s + time:14 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:40.184s + time:15 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.303s + time:16 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.322s + time:17 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:21.089s + time:18 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:20.824s + time:19 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:16.833s + time:20 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:31.777s + time:21 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.146s + time:22 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:16.895s + time:23 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.393s + time:24 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.626s + time:25 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.516s + time:26 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.707s + time:27 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:17.168s + time:28 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:39.63s + time:29 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.63s + time:30 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:20.168s + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run1D9.sh + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:17.111s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.414s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:17.674s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.614s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.574s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.633s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.124s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:8.262s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.193s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.095s + time:11 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:20.203s + time:12 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.528s + time:13 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:9.224s + time:14 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.559s + time:15 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.397s + time:16 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.286s + time:17 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:40.486s + time:18 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:16.913s + time:19 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:16.914s + time:20 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.536s + time:21 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.72s + time:22 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.176s + time:23 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:39.896s + time:24 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.339s + time:25 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.388s + time:26 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.289s + time:27 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.886s + time:28 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.935s + time:29 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.754s + time:30 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.431s + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run1D9.sh + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:16.877s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.576s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.556s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:31.889s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:8.711s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.514s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.962s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:20.938s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.465s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.108s + time:11 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.865s + time:12 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.47s + time:13 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:17.94s + time:14 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.688s + time:15 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:31.033s + time:16 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:17.846s + time:17 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.637s + time:18 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.046s + time:19 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.653s + time:20 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.897s + time:21 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.641s + time:22 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.49s + time:23 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.21s + time:24 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.12s + time:25 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.996s + time:26 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.852s + time:27 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.955s + time:28 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.634s + time:29 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.581s + time:30 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.489s + + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24025--spark\344\273\216HDFS\345\222\214Alluxio\350\257\273\345\217\226\346\227\266\351\227\264\346\257\224\350\276\203.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24025--spark\344\273\216HDFS\345\222\214Alluxio\350\257\273\345\217\226\346\227\266\351\227\264\346\257\224\350\276\203.md" new file mode 100644 index 0000000..3a8f476 --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24025--spark\344\273\216HDFS\345\222\214Alluxio\350\257\273\345\217\226\346\227\266\351\227\264\346\257\224\350\276\203.md" @@ -0,0 +1,258 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +# 1.解释 # + + 想要分析alluxio加速效果，发现alluxio会出现长尾效应，导致有些task特别耗时，相对于HDFS，并没有明显优势。 + + + +# 2.代码： # + + #~/cloud/alluxio-1.3.0/bin/alluxio fs mkdir -p /xubo/project/SparkSW/input/ + for i in 1 2 3 4 5 6 7 8 9 10 + do + echo 'time:'$i + for j in 1 2 3 4 5 6 7 8 9 10 + do + sh test.sh 10G hdfs://Master:9000/xubo/project/SparkSW/input/'D'$i'Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + #sh test.sh 10G alluxio://Master:19998/xubo/alluxio/uniref/'D'$j'Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + #sh test.sh 10G alluxio://Master:19998/xubo/project/SparkSW/input/'D'$j'Line.fasta' alluxio://Master:19998/xubo/project/alluxio/output/TimeCount + done + + for j in 1 2 3 4 5 6 7 8 9 10 + do + sh test.sh 10G alluxio://Master:19998/xubo/project/SparkSW/input/'D'$i'Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + done + done + + + +# 3.结果： # + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f runtime201611231839.txt + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.918s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.923s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.848s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.959s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.85s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.816s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:4.208s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.925s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.001s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:2.817s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.466s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:2.83s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:2.757s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:2.824s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:2.813s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:2.802s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:2.847s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:2.88s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.963s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:2.788s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.074s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:2.945s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.023s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.999s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.073s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.206s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.028s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.049s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.927s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.964s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.875s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:6.505s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.001s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.276s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.047s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.081s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.365s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.083s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.141s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.027s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.572s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.45s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.342s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.336s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.508s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.268s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.558s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.401s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.457s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.513s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:5.515s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.846s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.215s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.242s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.181s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.236s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:3.047s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.178s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.24s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.178s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:7.852s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:6.148s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:7.575s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:16.612s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:8.009s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:8.02s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:7.937s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:8.007s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:16.949s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:7.795s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:7.182s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:4.506s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:3.463s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:3.501s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:3.544s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:3.747s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:3.523s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:3.4s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:3.325s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:3.519s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:3.988s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.004s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:5.494s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.916s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:5.479s + 16/11/23 18:56:20 WARN QueuedThreadPool: 4 threads could not be stopped + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:3.891s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.73s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.851s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.9s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.966s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:7.924s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:6.396s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:3.487s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:5.417s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.272s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:3.465s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:3.437s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.067s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:6.194s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:5.352s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.883s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.119s + 16/11/23 18:59:47 WARN QueuedThreadPool: 4 threads could not be stopped + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.172s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:5.09s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:7.57s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.578s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:7.317s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.857s + 16/11/23 19:01:03 WARN QueuedThreadPool: 1 threads could not be stopped + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.854s + 16/11/23 19:01:16 WARN QueuedThreadPool: 1 threads could not be stopped + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:3.994s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:15.504s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:6.336s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:6.456s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.951s + 16/11/23 19:02:27 WARN QueuedThreadPool: 2 threads could not be stopped + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:5.525s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:15.943s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.665s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.814s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:15.965s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.915s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:9.814s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:9.196s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:9.434s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:9.172s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:20.193s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:10.454s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:9.551s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:9.321s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:10.313s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:9.707s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:52.331s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:6.583s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:16.998s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:16.55s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:6.496s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:6.781s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:7.477s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:17.51s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:6.494s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:17.511s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:10.427s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:9.993s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:11.242s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:10.868s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:7.354s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:7.015s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:7.344s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:7.182s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:7.262s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:8.282s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:42.406s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:16.769s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:18.197s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:7.881s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:8.913s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:8.174s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:20.971s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:8.434s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:8.316s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:16.767s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:32.624s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.688s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:17.345s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:20.98s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:13.478s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:27.69s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:15.567s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:14.237s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:16.413s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.537s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:99.776s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.482s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.184s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:17.949s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:17.29s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.279s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.892s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.895s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.351s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:20.001s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:38.227s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:25.727s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:23.691s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:19.047s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:19.611s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:14.391s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:16.036s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:16.249s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:15.324s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:13.936s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:151.861s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:46.322s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:31.237s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:31.318s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:19.097s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:19.043s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:30.663s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:43.809s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:20.463s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:31.099s + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24026--SparkSW\344\273\216HDFS\345\222\214Alluxio\350\257\273\345\217\226\350\256\241\347\256\227\346\227\266\351\227\264\346\257\224\350\276\203.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24026--SparkSW\344\273\216HDFS\345\222\214Alluxio\350\257\273\345\217\226\350\256\241\347\256\227\346\227\266\351\227\264\346\257\224\350\276\203.md" new file mode 100644 index 0000000..d2159c9 --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24026--SparkSW\344\273\216HDFS\345\222\214Alluxio\350\257\273\345\217\226\350\256\241\347\256\227\346\227\266\351\227\264\346\257\224\350\276\203.md" @@ -0,0 +1,1093 @@ + + 更多代码请见：https://github.com/xubo245/SparkLearning + + Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + + +# 结果： # + + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:5 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:6 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + 16/11/23 21:02:08 WARN QueuedThreadPool: 1 threads could not be stopped + 16/11/23 21:02:14 ERROR AkkaRpcEnv: Ignore error: Error notifying standalone scheduler's driver endpoint + org.apache.spark.SparkException: Error notifying standalone scheduler's driver endpoint + at org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend.removeExecutor(CoarseGrainedSchedulerBackend.scala:312) + at org.apache.spark.scheduler.cluster.SparkDeploySchedulerBackend.executorRemoved(SparkDeploySchedulerBackend.scala:142) + at org.apache.spark.deploy.client.AppClient$ClientEndpoint$$anonfun$receive$1.applyOrElse(AppClient.scala:184) + at org.apache.spark.rpc.akka.AkkaRpcEnv.org$apache$spark$rpc$akka$AkkaRpcEnv$$processMessage(AkkaRpcEnv.scala:177) + at org.apache.spark.rpc.akka.AkkaRpcEnv$$anonfun$actorRef$lzycompute$1$1$$anon$1$$anonfun$receiveWithLogging$1$$anonfun$applyOrElse$4.apply$mcV$sp(AkkaRpcEnv.scala:126) + at org.apache.spark.rpc.akka.AkkaRpcEnv.org$apache$spark$rpc$akka$AkkaRpcEnv$$safelyCall(AkkaRpcEnv.scala:197) + at org.apache.spark.rpc.akka.AkkaRpcEnv$$anonfun$actorRef$lzycompute$1$1$$anon$1$$anonfun$receiveWithLogging$1.applyOrElse(AkkaRpcEnv.scala:125) + at scala.runtime.AbstractPartialFunction$mcVL$sp.apply$mcVL$sp(AbstractPartialFunction.scala:33) + at scala.runtime.AbstractPartialFunction$mcVL$sp.apply(AbstractPartialFunction.scala:33) + at scala.runtime.AbstractPartialFunction$mcVL$sp.apply(AbstractPartialFunction.scala:25) + at org.apache.spark.util.ActorLogReceive$$anon$1.apply(ActorLogReceive.scala:59) + at org.apache.spark.util.ActorLogReceive$$anon$1.apply(ActorLogReceive.scala:42) + at scala.PartialFunction$class.applyOrElse(PartialFunction.scala:118) + at org.apache.spark.util.ActorLogReceive$$anon$1.applyOrElse(ActorLogReceive.scala:42) + at akka.actor.Actor$class.aroundReceive(Actor.scala:467) + at org.apache.spark.rpc.akka.AkkaRpcEnv$$anonfun$actorRef$lzycompute$1$1$$anon$1.aroundReceive(AkkaRpcEnv.scala:92) + at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) + at akka.actor.ActorCell.invoke(ActorCell.scala:487) + at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) + at akka.dispatch.Mailbox.run(Mailbox.scala:220) + at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) + at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) + at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) + at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) + at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) + Caused by: org.apache.spark.SparkException: Error sending message [message = RemoveExecutor(4,Unknown executor exit code (0))] + at org.apache.spark.rpc.RpcEndpointRef.askWithRetry(RpcEndpointRef.scala:118) + at org.apache.spark.rpc.RpcEndpointRef.askWithRetry(RpcEndpointRef.scala:77) + at org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend.removeExecutor(CoarseGrainedSchedulerBackend.scala:309) + ... 24 more + Caused by: org.apache.spark.rpc.RpcTimeoutException: Recipient[Actor[akka://sparkDriver/user/CoarseGrainedScheduler#-679832735]] had already been terminated.. This timeout is controlled by spark.rpc.askTimeout + at org.apache.spark.rpc.RpcTimeout.org$apache$spark$rpc$RpcTimeout$$createRpcTimeoutException(RpcEnv.scala:214) + at org.apache.spark.rpc.RpcTimeout$$anonfun$addMessageIfTimeout$1.applyOrElse(RpcEnv.scala:229) + at org.apache.spark.rpc.RpcTimeout$$anonfun$addMessageIfTimeout$1.applyOrElse(RpcEnv.scala:225) + at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:33) + at scala.util.Failure$$anonfun$recover$1.apply(Try.scala:185) + at scala.util.Try$.apply(Try.scala:161) + at scala.util.Failure.recover(Try.scala:185) + at scala.concurrent.Future$$anonfun$recover$1.apply(Future.scala:324) + at scala.concurrent.Future$$anonfun$recover$1.apply(Future.scala:324) + at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32) + at org.spark-project.guava.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:293) + at scala.concurrent.impl.ExecutionContextImpl$$anon$1.execute(ExecutionContextImpl.scala:133) + at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40) + at scala.concurrent.impl.Promise$DefaultPromise.scala$concurrent$impl$Promise$DefaultPromise$$dispatchOrAddCallback(Promise.scala:280) + at scala.concurrent.impl.Promise$DefaultPromise.onComplete(Promise.scala:270) + at scala.concurrent.Future$class.recover(Future.scala:324) + at scala.concurrent.impl.Promise$DefaultPromise.recover(Promise.scala:153) + at org.apache.spark.rpc.akka.AkkaRpcEndpointRef.ask(AkkaRpcEnv.scala:319) + at org.apache.spark.rpc.RpcEndpointRef.askWithRetry(RpcEndpointRef.scala:100) + ... 26 more + Caused by: akka.pattern.AskTimeoutException: Recipient[Actor[akka://sparkDriver/user/CoarseGrainedScheduler#-679832735]] had already been terminated. + at akka.pattern.AskableActorRef$.ask$extension(AskSupport.scala:132) + at org.apache.spark.rpc.akka.AkkaRpcEndpointRef.ask(AkkaRpcEnv.scala:307) + ... 27 more + time:7 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:8 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + 16/11/23 21:03:30 WARN QueuedThreadPool: 5 threads could not be stopped + time:9 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:10 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + 16/11/23 21:04:45 WARN QueuedThreadPool: 4 threads could not be stopped + HDFS: + time:1 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + 16/11/23 21:05:28 WARN QueuedThreadPool: 4 threads could not be stopped + time:2 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:3 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:4 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:5 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:6 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:7 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:8 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:9 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:10 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + alluxio + time:1 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:2 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:3 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:4 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_P04406) + time:5 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:6 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:7 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_P04406) + time:8 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:9 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:10 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + HDFS: + time:1 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:2 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_A9NHD0) + time:3 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:4 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_A9NHD0) + time:5 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:6 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:7 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_A9NHD0) + time:8 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:9 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:10 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + alluxio + time:1 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:2 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:3 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:4 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:5 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:6 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:7 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_P04406) + time:8 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:9 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:10 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + HDFS: + time:1 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:2 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:3 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:4 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_A9NHD0) + time:5 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:6 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_A9NHD0) + time:7 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:8 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_P04406) + time:9 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:10 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_A9NHD0) + alluxio + time:1 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_P04406) + time:2 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:3 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_P04406) + time:4 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:5 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:6 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_A9NHD0) + time:7 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:8 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_A9NHD0) + time:9 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:10 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + HDFS: + time:1 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:2 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_P04406) + time:3 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:4 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:5 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_P04406) + time:6 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:7 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:8 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:9 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_P04406) + time:10 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + alluxio + time:1 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:2 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:3 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:4 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_P04406) + time:5 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:6 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:7 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:8 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:9 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:10 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + HDFS: + time:1 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:2 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:3 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:4 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:5 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:6 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:7 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:8 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:9 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:10 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + alluxio + time:1 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:2 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:3 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:4 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:5 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:6 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:7 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:8 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:9 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:10 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + HDFS: + time:1 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (57,UniRef100_A0A163CSK3) + time:2 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:3 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:4 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:5 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:6 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:7 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (57,UniRef100_A0A163CSK3) + time:8 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:9 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:10 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (57,UniRef100_A0A163CSK3) + alluxio + time:1 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:2 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:3 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:4 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:5 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:6 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (57,UniRef100_A0A163CSK3) + time:7 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:8 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (57,UniRef100_A0A163CSK3) + time:9 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:10 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (57,UniRef100_A0A163CSK3) + HDFS: + time:1 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_M3XW90) + time:2 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + time:3 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_M3WVW7) + time:4 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_M3WVW7) + time:5 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + time:6 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + time:7 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_D2I7L4) + time:8 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_D2I7L4) + time:9 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_Q0QEU8) + time:10 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_M3WVW7) + alluxio + time:1 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + time:2 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + time:3 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_Q0QEU8) + time:4 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + time:5 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_Q0QEU8) + time:6 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_J9NZ39) + time:7 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_J9NZ39) + time:8 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_J9NZ39) + time:9 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_M3WVW7) + time:10 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_M3WVW7) + HDFS: + time:1 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_G3WQA5) + time:2 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_D2I7L4) + time:3 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_M3XW90) + time:4 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_W4LL34) + time:5 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_M3XW90) + time:6 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_W4LL34) + time:7 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_D2I7L4) + time:8 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + time:9 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_W4LL34) + time:10 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_D2I7L4) + alluxio + time:1 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_D2I7L4) + time:2 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_M3XW90) + time:3 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_D2I7L4) + time:4 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_G3WQA5) + time:5 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + time:6 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_D2I7L4) + time:7 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_G3WQA5) + time:8 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_G3WQA5) + time:9 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_W4LL34) + time:10 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_W4LL34) + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2403---\346\234\272\345\231\250\351\207\215\345\220\257\345\220\216\346\225\260\346\215\256\345\255\230\345\202\250\344\275\215\347\275\256\347\232\204\345\217\230\345\214\226.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2403---\346\234\272\345\231\250\351\207\215\345\220\257\345\220\216\346\225\260\346\215\256\345\255\230\345\202\250\344\275\215\347\275\256\347\232\204\345\217\230\345\214\226.md" new file mode 100644 index 0000000..a07d111 --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2403---\346\234\272\345\231\250\351\207\215\345\220\257\345\220\216\346\225\260\346\215\256\345\255\230\345\202\250\344\275\215\347\275\256\347\232\204\345\217\230\345\214\226.md" @@ -0,0 +1,31 @@ + +1.重启后数据没有在内存里了，由于安装时用的是bin，而且设置的是local，及其重启后需要重启tachyon： + + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ ./tachyon-start.sh local + Killed 0 processes on xubo + Killed 0 processes on xubo + Connecting to localhost as xubo... + Killed 0 processes on xubo + Connection to localhost closed. + [sudo] password for xubo: + Formatting RamFS: /mnt/ramdisk (1gb) + Starting master @ localhost + Starting worker @ xubo + +2.重启后的数据没在内存，而在local，根据tachyon.env.sh可以知道存储的位置在： + + xubo@xubo:~/cloud/tachyon-0.7.1/underFSStorage/tmp/tachyon/data$ ll -h + total 114M + drwxrwxrwt 2 xubo xubo 4.0K May 5 14:02 ./ + drwxrwxr-x 4 xubo xubo 4.0K May 4 22:03 ../ + -rwxrwxrwt 1 xubo xubo 80 May 4 22:19 3* + -rwxrwxrwt 1 xubo xubo 17K May 4 22:22 4* + -rwxrwxrwt 1 xubo xubo 114M May 4 22:25 5* + -rwxrwxrwt 1 xubo xubo 21 May 4 22:29 6* + xubo@xubo:~/cloud/tachyon-0.7.1/underFSStorage/tmp/tachyon/data$ cat 6 + hello tachyon + 1 + 2 + 3 + + diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2404---\344\270\213\350\275\275\346\272\220\347\240\201\351\200\232\350\277\207maven install\345\256\211\350\243\205\345\244\261\350\264\245\350\256\260\345\275\225.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2404---\344\270\213\350\275\275\346\272\220\347\240\201\351\200\232\350\277\207maven install\345\256\211\350\243\205\345\244\261\350\264\245\350\256\260\345\275\225.md" new file mode 100644 index 0000000..ccef732 --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2404---\344\270\213\350\275\275\346\272\220\347\240\201\351\200\232\350\277\207maven install\345\256\211\350\243\205\345\244\261\350\264\245\350\256\260\345\275\225.md" @@ -0,0 +1,439 @@ +版本：v0.7.1.tar.gz +记录 + +（2）制定版本 + + mvn clean package -Djava.version=1.7 -Dhadoop.version=2.6.0 -Dspark.version=1.5.2 -DskipTests + + +（1） + + xubo@xubo:~/cloud/tachyon-0.7.1$ mvn install + [INFO] Scanning for projects... + [INFO] ------------------------------------------------------------------------ + [INFO] Reactor Build Order: + [INFO] + [INFO] Tachyon Parent + [INFO] Tachyon Common + [INFO] Tachyon Under File System + [INFO] Tachyon Under File System - Local FS + [INFO] Tachyon Under File System - HDFS + [INFO] Tachyon Under File System - Gluster FS + [INFO] Tachyon Under File System - Swift + [INFO] Tachyon Under File System - S3 + [INFO] Tachyon Clients + [INFO] Tachyon Clients - Implementation + [INFO] Tachyon Clients - Distribution + [INFO] Tachyon Servers + [INFO] Mock Tachyon Cluster + [INFO] Tachyon Integration Tests + [INFO] Tachyon Shell + [INFO] Tachyon Examples + [INFO] Tachyon Assemblies + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Parent 0.7.1 + [INFO] ------------------------------------------------------------------------ + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-source-plugin/2.3/maven-source-plugin-2.3.pom + May 05, 2016 2:10:48 PM org.apache.maven.wagon.providers.http.httpclient.impl.execchain.RetryExec execute + INFO: I/O exception (java.net.SocketException) caught when processing request to {s}->https://repo.maven.apache.org:443: Connection reset + May 05, 2016 2:10:48 PM org.apache.maven.wagon.providers.http.httpclient.impl.execchain.RetryExec execute + INFO: Retrying request to {s}->https://repo.maven.apache.org:443 + May 05, 2016 2:10:51 PM org.apache.maven.wagon.providers.http.httpclient.impl.execchain.RetryExec execute + INFO: I/O exception (java.net.SocketException) caught when processing request to {s}->https://repo.maven.apache.org:443: Connection reset + May 05, 2016 2:10:51 PM org.apache.maven.wagon.providers.http.httpclient.impl.execchain.RetryExec execute + INFO: Retrying request to {s}->https://repo.maven.apache.org:443 + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-source-plugin/2.3/maven-source-plugin-2.3.pom (7 KB at 0.7 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-source-plugin/2.3/maven-source-plugin-2.3.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-source-plugin/2.3/maven-source-plugin-2.3.jar (31 KB at 18.8 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-javadoc-plugin/2.9/maven-javadoc-plugin-2.9.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-javadoc-plugin/2.9/maven-javadoc-plugin-2.9.pom (16 KB at 22.8 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-javadoc-plugin/2.9/maven-javadoc-plugin-2.9.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-javadoc-plugin/2.9/maven-javadoc-plugin-2.9.jar (354 KB at 52.2 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-checkstyle-plugin/2.13/maven-checkstyle-plugin-2.13.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-checkstyle-plugin/2.13/maven-checkstyle-plugin-2.13.pom (13 KB at 18.8 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-checkstyle-plugin/2.13/maven-checkstyle-plugin-2.13.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-checkstyle-plugin/2.13/maven-checkstyle-plugin-2.13.jar (112 KB at 34.6 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/com/mycila/license-maven-plugin/2.9/license-maven-plugin-2.9.pom + Downloaded: https://repo.maven.apache.org/maven2/com/mycila/license-maven-plugin/2.9/license-maven-plugin-2.9.pom (5 KB at 5.6 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/com/mycila/license-maven-plugin-parent/2.9/license-maven-plugin-parent-2.9.pom + Downloaded: https://repo.maven.apache.org/maven2/com/mycila/license-maven-plugin-parent/2.9/license-maven-plugin-parent-2.9.pom (4 KB at 5.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/com/mycila/mycila-pom/3/mycila-pom-3.pom + Downloaded: https://repo.maven.apache.org/maven2/com/mycila/mycila-pom/3/mycila-pom-3.pom (20 KB at 18.9 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/com/mycila/license-maven-plugin/2.9/license-maven-plugin-2.9.jar + Downloaded: https://repo.maven.apache.org/maven2/com/mycila/license-maven-plugin/2.9/license-maven-plugin-2.9.jar (79 KB at 20.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-install-plugin/2.4/maven-install-plugin-2.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-install-plugin/2.4/maven-install-plugin-2.4.pom (7 KB at 9.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-install-plugin/2.4/maven-install-plugin-2.4.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-install-plugin/2.4/maven-install-plugin-2.4.jar (27 KB at 26.8 KB/sec) + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-parent --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-parent --- + Downloading: https://repo.maven.apache.org/maven2/com/puppycrawl/tools/checkstyle/5.9/checkstyle-5.9.pom + Downloaded: https://repo.maven.apache.org/maven2/com/puppycrawl/tools/checkstyle/5.9/checkstyle-5.9.pom (26 KB at 24.2 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-beanutils/commons-beanutils-core/1.8.3/commons-beanutils-core-1.8.3.pom + Downloaded: https://repo.maven.apache.org/maven2/commons-beanutils/commons-beanutils-core/1.8.3/commons-beanutils-core-1.8.3.pom (2 KB at 2.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/com/google/guava/guava-jdk5/14.0.1/guava-jdk5-14.0.1.pom + Downloaded: https://repo.maven.apache.org/maven2/com/google/guava/guava-jdk5/14.0.1/guava-jdk5-14.0.1.pom (7 KB at 9.6 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/com/google/guava/guava-parent-jdk5/14.0.1/guava-parent-jdk5-14.0.1.pom + Downloaded: https://repo.maven.apache.org/maven2/com/google/guava/guava-parent-jdk5/14.0.1/guava-parent-jdk5-14.0.1.pom (3 KB at 4.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/reporting/maven-reporting-impl/2.3/maven-reporting-impl-2.3.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/reporting/maven-reporting-impl/2.3/maven-reporting-impl-2.3.pom (5 KB at 7.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/shared/maven-shared-utils/0.6/maven-shared-utils-0.6.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/shared/maven-shared-utils/0.6/maven-shared-utils-0.6.pom (5 KB at 7.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sink-api/1.2/doxia-sink-api-1.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sink-api/1.2/doxia-sink-api-1.2.pom (2 KB at 2.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia/1.2/doxia-1.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia/1.2/doxia-1.2.pom (19 KB at 27.9 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/maven-parent/19/maven-parent-19.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/maven-parent/19/maven-parent-19.pom (25 KB at 36.8 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-logging-api/1.2/doxia-logging-api-1.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-logging-api/1.2/doxia-logging-api-1.2.pom (2 KB at 2.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-core/1.2/doxia-core-1.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-core/1.2/doxia-core-1.2.pom (4 KB at 5.9 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpclient/4.0.2/httpclient-4.0.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpclient/4.0.2/httpclient-4.0.2.pom (8 KB at 11.2 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpcomponents-client/4.0.2/httpcomponents-client-4.0.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpcomponents-client/4.0.2/httpcomponents-client-4.0.2.pom (9 KB at 13.4 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/project/4.1/project-4.1.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/project/4.1/project-4.1.pom (16 KB at 24.4 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.pom (5 KB at 7.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpcomponents-core/4.0.1/httpcomponents-core-4.0.1.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpcomponents-core/4.0.1/httpcomponents-core-4.0.1.pom (10 KB at 14.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/project/4.0/project-4.0.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/project/4.0/project-4.0.pom (13 KB at 19.6 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-site-renderer/1.2/doxia-site-renderer-1.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-site-renderer/1.2/doxia-site-renderer-1.2.pom (7 KB at 9.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sitetools/1.2/doxia-sitetools-1.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sitetools/1.2/doxia-sitetools-1.2.pom (16 KB at 24.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-decoration-model/1.2/doxia-decoration-model-1.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-decoration-model/1.2/doxia-decoration-model-1.2.pom (3 KB at 4.6 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-xhtml/1.2/doxia-module-xhtml-1.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-xhtml/1.2/doxia-module-xhtml-1.2.pom (2 KB at 2.6 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-modules/1.2/doxia-modules-1.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-modules/1.2/doxia-modules-1.2.pom (3 KB at 3.8 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-fml/1.2/doxia-module-fml-1.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-fml/1.2/doxia-module-fml-1.2.pom (6 KB at 8.4 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-validator/commons-validator/1.3.1/commons-validator-1.3.1.pom + Downloaded: https://repo.maven.apache.org/maven2/commons-validator/commons-validator/1.3.1/commons-validator-1.3.1.pom (9 KB at 13.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-digester/commons-digester/1.6/commons-digester-1.6.pom + Downloaded: https://repo.maven.apache.org/maven2/commons-digester/commons-digester/1.6/commons-digester-1.6.pom (974 B at 1.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-beanutils/commons-beanutils/1.6/commons-beanutils-1.6.pom + Downloaded: https://repo.maven.apache.org/maven2/commons-beanutils/commons-beanutils/1.6/commons-beanutils-1.6.pom (3 KB at 3.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-logging/commons-logging/1.0/commons-logging-1.0.pom + Downloaded: https://repo.maven.apache.org/maven2/commons-logging/commons-logging/1.0/commons-logging-1.0.pom (163 B at 0.2 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-collections/commons-collections/2.0/commons-collections-2.0.pom + Downloaded: https://repo.maven.apache.org/maven2/commons-collections/commons-collections/2.0/commons-collections-2.0.pom (171 B at 0.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-collections/commons-collections/2.1/commons-collections-2.1.pom + Downloaded: https://repo.maven.apache.org/maven2/commons-collections/commons-collections/2.1/commons-collections-2.1.pom (4 KB at 5.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sink-api/1.4/doxia-sink-api-1.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sink-api/1.4/doxia-sink-api-1.4.pom (2 KB at 2.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia/1.4/doxia-1.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia/1.4/doxia-1.4.pom (18 KB at 26.8 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-logging-api/1.4/doxia-logging-api-1.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-logging-api/1.4/doxia-logging-api-1.4.pom (2 KB at 2.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-decoration-model/1.4/doxia-decoration-model-1.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-decoration-model/1.4/doxia-decoration-model-1.4.pom (3 KB at 4.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sitetools/1.4/doxia-sitetools-1.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sitetools/1.4/doxia-sitetools-1.4.pom (17 KB at 25.6 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-site-renderer/1.4/doxia-site-renderer-1.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-site-renderer/1.4/doxia-site-renderer-1.4.pom (6 KB at 8.7 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-core/1.4/doxia-core-1.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-core/1.4/doxia-core-1.4.pom (4 KB at 6.1 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-xhtml/1.4/doxia-module-xhtml-1.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-xhtml/1.4/doxia-module-xhtml-1.4.pom (2 KB at 1.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-modules/1.4/doxia-modules-1.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-modules/1.4/doxia-modules-1.4.pom (3 KB at 3.9 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-fml/1.4/doxia-module-fml-1.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-fml/1.4/doxia-module-fml-1.4.pom (5 KB at 7.2 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/velocity/velocity-tools/2.0/velocity-tools-2.0.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/velocity/velocity-tools/2.0/velocity-tools-2.0.pom (18 KB at 27.2 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/javax/servlet/servlet-api/2.3/servlet-api-2.3.pom + Downloaded: https://repo.maven.apache.org/maven2/javax/servlet/servlet-api/2.3/servlet-api-2.3.pom (156 B at 0.2 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-chain/commons-chain/1.1/commons-chain-1.1.pom + Downloaded: https://repo.maven.apache.org/maven2/commons-chain/commons-chain/1.1/commons-chain-1.1.pom (6 KB at 9.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/dom4j/dom4j/1.1/dom4j-1.1.pom + Downloaded: https://repo.maven.apache.org/maven2/dom4j/dom4j/1.1/dom4j-1.1.pom (142 B at 0.2 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/sslext/sslext/1.2-0/sslext-1.2-0.pom + Downloaded: https://repo.maven.apache.org/maven2/sslext/sslext/1.2-0/sslext-1.2-0.pom (653 B at 1.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/struts/struts-core/1.3.8/struts-core-1.3.8.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/struts/struts-core/1.3.8/struts-core-1.3.8.pom (5 KB at 6.4 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/struts/struts-parent/1.3.8/struts-parent-1.3.8.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/struts/struts-parent/1.3.8/struts-parent-1.3.8.pom (10 KB at 14.7 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/struts/struts-master/4/struts-master-4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/struts/struts-master/4/struts-master-4.pom (12 KB at 17.7 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/apache/2/apache-2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/apache/2/apache-2.pom (4 KB at 5.1 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/antlr/antlr/2.7.2/antlr-2.7.2.pom + Downloaded: https://repo.maven.apache.org/maven2/antlr/antlr/2.7.2/antlr-2.7.2.pom (145 B at 0.2 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/struts/struts-taglib/1.3.8/struts-taglib-1.3.8.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/struts/struts-taglib/1.3.8/struts-taglib-1.3.8.pom (4 KB at 4.6 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/struts/struts-tiles/1.3.8/struts-tiles-1.3.8.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/struts/struts-tiles/1.3.8/struts-tiles-1.3.8.pom (3 KB at 1.4 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/velocity/velocity/1.6.2/velocity-1.6.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/velocity/velocity/1.6.2/velocity-1.6.2.pom (11 KB at 10.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-integration-tools/1.6/doxia-integration-tools-1.6.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-integration-tools/1.6/doxia-integration-tools-1.6.pom (7 KB at 10.4 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-tools/3/doxia-tools-3.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-tools/3/doxia-tools-3.pom (10 KB at 14.8 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-logging-api/1.6/doxia-logging-api-1.6.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-logging-api/1.6/doxia-logging-api-1.6.pom (2 KB at 2.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia/1.6/doxia-1.6.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia/1.6/doxia-1.6.pom (19 KB at 19.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-decoration-model/1.6/doxia-decoration-model-1.6.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-decoration-model/1.6/doxia-decoration-model-1.6.pom (3 KB at 4.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sitetools/1.6/doxia-sitetools-1.6.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sitetools/1.6/doxia-sitetools-1.6.pom (18 KB at 17.9 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-resources/1.0-alpha-7/plexus-resources-1.0-alpha-7.pom + Downloaded: https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-resources/1.0-alpha-7/plexus-resources-1.0-alpha-7.pom (2 KB at 2.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/com/puppycrawl/tools/checkstyle/5.9/checkstyle-5.9.jar + Downloading: https://repo.maven.apache.org/maven2/commons-beanutils/commons-beanutils-core/1.8.3/commons-beanutils-core-1.8.3.jar + Downloading: https://repo.maven.apache.org/maven2/com/google/guava/guava-jdk5/14.0.1/guava-jdk5-14.0.1.jar + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/reporting/maven-reporting-impl/2.3/maven-reporting-impl-2.3.jar + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/shared/maven-shared-utils/0.6/maven-shared-utils-0.6.jar + Downloaded: https://repo.maven.apache.org/maven2/com/puppycrawl/tools/checkstyle/5.9/checkstyle-5.9.jar (705 KB at 47.9 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-core/1.2/doxia-core-1.2.jar + May 05, 2016 2:12:19 PM org.apache.maven.wagon.providers.http.httpclient.impl.execchain.RetryExec execute + INFO: I/O exception (java.net.SocketException) caught when processing request to {s}->https://repo.maven.apache.org:443: Connection reset + May 05, 2016 2:12:19 PM org.apache.maven.wagon.providers.http.httpclient.impl.execchain.RetryExec execute + INFO: Retrying request to {s}->https://repo.maven.apache.org:443 + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-core/1.2/doxia-core-1.2.jar (151 KB at 7.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpclient/4.0.2/httpclient-4.0.2.jar + May 05, 2016 2:12:23 PM org.apache.maven.wagon.providers.http.httpclient.impl.execchain.RetryExec execute + INFO: I/O exception (java.net.SocketException) caught when processing request to {s}->https://repo.maven.apache.org:443: Connection reset + May 05, 2016 2:12:23 PM org.apache.maven.wagon.providers.http.httpclient.impl.execchain.RetryExec execute + INFO: Retrying request to {s}->https://repo.maven.apache.org:443 + Downloaded: https://repo.maven.apache.org/maven2/commons-beanutils/commons-beanutils-core/1.8.3/commons-beanutils-core-1.8.3.jar (202 KB at 7.2 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpclient/4.0.2/httpclient-4.0.2.jar (287 KB at 10.1 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-validator/commons-validator/1.3.1/commons-validator-1.3.1.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/shared/maven-shared-utils/0.6/maven-shared-utils-0.6.jar (161 KB at 5.4 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-digester/commons-digester/1.6/commons-digester-1.6.jar + Downloaded: https://repo.maven.apache.org/maven2/com/google/guava/guava-jdk5/14.0.1/guava-jdk5-14.0.1.jar (1957 KB at 59.7 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sink-api/1.4/doxia-sink-api-1.4.jar + Downloaded: https://repo.maven.apache.org/maven2/commons-digester/commons-digester/1.6/commons-digester-1.6.jar (165 KB at 5.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-logging-api/1.4/doxia-logging-api-1.4.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.jar (169 KB at 5.1 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-decoration-model/1.4/doxia-decoration-model-1.4.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sink-api/1.4/doxia-sink-api-1.4.jar (11 KB at 0.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-site-renderer/1.4/doxia-site-renderer-1.4.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-logging-api/1.4/doxia-logging-api-1.4.jar (12 KB at 0.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-xhtml/1.4/doxia-module-xhtml-1.4.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-xhtml/1.4/doxia-module-xhtml-1.4.jar (16 KB at 0.4 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-fml/1.4/doxia-module-fml-1.4.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-site-renderer/1.4/doxia-site-renderer-1.4.jar (52 KB at 1.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/velocity/velocity-tools/2.0/velocity-tools-2.0.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-fml/1.4/doxia-module-fml-1.4.jar (37 KB at 1.1 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-chain/commons-chain/1.1/commons-chain-1.1.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-decoration-model/1.4/doxia-decoration-model-1.4.jar (60 KB at 1.7 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/dom4j/dom4j/1.1/dom4j-1.1.jar + Downloaded: https://repo.maven.apache.org/maven2/commons-chain/commons-chain/1.1/commons-chain-1.1.jar (88 KB at 2.4 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/sslext/sslext/1.2-0/sslext-1.2-0.jar + Downloaded: https://repo.maven.apache.org/maven2/sslext/sslext/1.2-0/sslext-1.2-0.jar (26 KB at 0.7 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/struts/struts-core/1.3.8/struts-core-1.3.8.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/velocity/velocity-tools/2.0/velocity-tools-2.0.jar (339 KB at 8.9 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/struts/struts-taglib/1.3.8/struts-taglib-1.3.8.jar + Downloaded: https://repo.maven.apache.org/maven2/commons-validator/commons-validator/1.3.1/commons-validator-1.3.1.jar (136 KB at 3.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/struts/struts-tiles/1.3.8/struts-tiles-1.3.8.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/struts/struts-core/1.3.8/struts-core-1.3.8.jar (322 KB at 8.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-integration-tools/1.6/doxia-integration-tools-1.6.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/struts/struts-taglib/1.3.8/struts-taglib-1.3.8.jar (246 KB at 6.1 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-container-default/1.0-alpha-9/plexus-container-default-1.0-alpha-9.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-integration-tools/1.6/doxia-integration-tools-1.6.jar (44 KB at 1.1 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-resources/1.0-alpha-7/plexus-resources-1.0-alpha-7.jar + Downloaded: https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-resources/1.0-alpha-7/plexus-resources-1.0-alpha-7.jar (23 KB at 0.5 KB/sec) + Downloaded: https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-container-default/1.0-alpha-9/plexus-container-default-1.0-alpha-9.jar (191 KB at 4.5 KB/sec) + Downloaded: https://repo.maven.apache.org/maven2/org/apache/struts/struts-tiles/1.3.8/struts-tiles-1.3.8.jar (117 KB at 2.5 KB/sec) + Downloaded: https://repo.maven.apache.org/maven2/dom4j/dom4j/1.1/dom4j-1.1.jar (447 KB at 8.5 KB/sec) + [INFO] ------------------------------------------------------------------------ + [INFO] Reactor Summary: + [INFO] + [INFO] Tachyon Parent ..................................... FAILURE [31:36 min] + [INFO] Tachyon Common ..................................... SKIPPED + [INFO] Tachyon Under File System .......................... SKIPPED + [INFO] Tachyon Under File System - Local FS ............... SKIPPED + [INFO] Tachyon Under File System - HDFS ................... SKIPPED + [INFO] Tachyon Under File System - Gluster FS ............. SKIPPED + [INFO] Tachyon Under File System - Swift .................. SKIPPED + [INFO] Tachyon Under File System - S3 ..................... SKIPPED + [INFO] Tachyon Clients .................................... SKIPPED + [INFO] Tachyon Clients - Implementation ................... SKIPPED + [INFO] Tachyon Clients - Distribution ..................... SKIPPED + [INFO] Tachyon Servers .................................... SKIPPED + [INFO] Mock Tachyon Cluster ............................... SKIPPED + [INFO] Tachyon Integration Tests .......................... SKIPPED + [INFO] Tachyon Shell ...................................... SKIPPED + [INFO] Tachyon Examples ................................... SKIPPED + [INFO] Tachyon Assemblies ................................. SKIPPED + [INFO] ------------------------------------------------------------------------ + [INFO] BUILD FAILURE + [INFO] ------------------------------------------------------------------------ + [INFO] Total time: 31:36 min + [INFO] Finished at: 2016-05-05T14:42:18+08:00 + [INFO] Final Memory: 16M/170M + [INFO] ------------------------------------------------------------------------ + [ERROR] Failed to execute goal org.apache.maven.plugins:maven-checkstyle-plugin:2.13:check (checkstyle) on project tachyon-parent: Execution checkstyle of goal org.apache.maven.plugins:maven-checkstyle-plugin:2.13:check failed: Plugin org.apache.maven.plugins:maven-checkstyle-plugin:2.13 or one of its dependencies could not be resolved: Could not transfer artifact org.apache.maven.reporting:maven-reporting-impl:jar:2.3 from/to central (https://repo.maven.apache.org/maven2): Read timed out -> [Help 1] + [ERROR] + [ERROR] To see the full stack trace of the errors, re-run Maven with the -e switch. + [ERROR] Re-run Maven using the -X switch to enable full debug logging. + [ERROR] + [ERROR] For more information about the errors and possible solutions, please read the following articles: + [ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/PluginResolutionException + xubo@xubo:~/cloud/tachyon-0.7.1$ mvn clean + [INFO] Scanning for projects... + [INFO] ------------------------------------------------------------------------ + [INFO] Reactor Build Order: + [INFO] + [INFO] Tachyon Parent + [INFO] Tachyon Common + [INFO] Tachyon Under File System + [INFO] Tachyon Under File System - Local FS + [INFO] Tachyon Under File System - HDFS + [INFO] Tachyon Under File System - Gluster FS + [INFO] Tachyon Under File System - Swift + [INFO] Tachyon Under File System - S3 + [INFO] Tachyon Clients + [INFO] Tachyon Clients - Implementation + [INFO] Tachyon Clients - Distribution + [INFO] Tachyon Servers + [INFO] Mock Tachyon Cluster + [INFO] Tachyon Integration Tests + [INFO] Tachyon Shell + [INFO] Tachyon Examples + [INFO] Tachyon Assemblies + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Parent 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-parent --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Common 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-common --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Under File System 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-underfs --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Under File System - Local FS 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-underfs-local --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Under File System - HDFS 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-underfs-hdfs --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Under File System - Gluster FS 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-underfs-glusterfs --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Under File System - Swift 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-underfs-swift --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Under File System - S3 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-underfs-s3 --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Clients 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-clients --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Clients - Implementation 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-client-unshaded --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Clients - Distribution 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-client --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Servers 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-servers --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Mock Tachyon Cluster 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-minicluster --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Integration Tests 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-integration-tests --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Shell 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-shell --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Examples 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-examples --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Assemblies 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-assemblies --- + [INFO] ------------------------------------------------------------------------ + [INFO] Reactor Summary: + [INFO] + [INFO] Tachyon Parent ..................................... SUCCESS [ 0.564 s] + [INFO] Tachyon Common ..................................... SUCCESS [ 0.004 s] + [INFO] Tachyon Under File System .......................... SUCCESS [ 0.003 s] + [INFO] Tachyon Under File System - Local FS ............... SUCCESS [ 0.003 s] + [INFO] Tachyon Under File System - HDFS ................... SUCCESS [ 0.003 s] + [INFO] Tachyon Under File System - Gluster FS ............. SUCCESS [ 0.004 s] + [INFO] Tachyon Under File System - Swift .................. SUCCESS [ 0.003 s] + [INFO] Tachyon Under File System - S3 ..................... SUCCESS [ 0.003 s] + [INFO] Tachyon Clients .................................... SUCCESS [ 0.003 s] + [INFO] Tachyon Clients - Implementation ................... SUCCESS [ 0.003 s] + [INFO] Tachyon Clients - Distribution ..................... SUCCESS [ 0.004 s] + [INFO] Tachyon Servers .................................... SUCCESS [ 0.059 s] + [INFO] Mock Tachyon Cluster ............................... SUCCESS [ 0.012 s] + [INFO] Tachyon Integration Tests .......................... SUCCESS [ 0.004 s] + [INFO] Tachyon Shell ...................................... SUCCESS [ 0.004 s] + [INFO] Tachyon Examples ................................... SUCCESS [ 0.013 s] + [INFO] Tachyon Assemblies ................................. SUCCESS [ 0.004 s] + [INFO] ------------------------------------------------------------------------ + [INFO] BUILD SUCCESS + [INFO] ------------------------------------------------------------------------ + [INFO] Total time: 1.234 s + [INFO] Finished at: 2016-05-05T14:45:57+08:00 + [INFO] Final Memory: 11M/104M + [INFO] ------------------------------------------------------------------------ + + +参考： +【1】http://blog.csdn.net/stark_summer/article/details/48321605 + diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2405--tachyon\347\232\204\345\207\240\344\270\252\351\227\256\351\242\230\357\274\210\345\276\205\350\247\243\345\206\263\357\274\211.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2405--tachyon\347\232\204\345\207\240\344\270\252\351\227\256\351\242\230\357\274\210\345\276\205\350\247\243\345\206\263\357\274\211.md" new file mode 100644 index 0000000..1c4ae18 --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2405--tachyon\347\232\204\345\207\240\344\270\252\351\227\256\351\242\230\357\274\210\345\276\205\350\247\243\345\206\263\357\274\211.md" @@ -0,0 +1,12 @@ + +1.tachyon是否会占用额外的空间？ +当tachyon将数据存储在内存中时，spark读取的时候是copy还是使用lineage？ + + +2.在运行spark的时候persist可以制定off-heap，而这种默认使用tachyon，这种模式与使用saveAsFile等指令存储到tachyon有什么区别？ + +3.当tachyon内存存储到一定值时tachyon将文件存储到hdfs或者disk的具体实现机制是什么样的？ + +4.tachyon分布式如何实现？ + +5.tachyon域redis性能差别？ \ No newline at end of file diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2406---\351\233\206\347\276\244\347\211\210\346\220\255\345\273\272\345\222\214\350\277\220\350\241\214\357\274\210Tachyon\357\274\211.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2406---\351\233\206\347\276\244\347\211\210\346\220\255\345\273\272\345\222\214\350\277\220\350\241\214\357\274\210Tachyon\357\274\211.md" new file mode 100644 index 0000000..99bef84 --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2406---\351\233\206\347\276\244\347\211\210\346\220\255\345\273\272\345\222\214\350\277\220\350\241\214\357\274\210Tachyon\357\274\211.md" @@ -0,0 +1,103 @@ +更多代码请见：https://github.com/xubo245/SparkLearning +环境： +hadoop-2.6.0 +spark-1.5.2 +scala-1.7 + + +1解释 +完成tachyon的集群配置与运行 +前提：安装好java、hdfs等 + +2.下载 + + https://github.com/Alluxio/alluxio/releases +具体的： + + https://github.com/Alluxio/alluxio/releases/download/v0.7.1/tachyon-0.7.1-hadoop2.6-bin.tar.gz + +如果下载tachyon-0.7.1-bin.tar.g会出现版本不兼容，会报错： + + Server IPC version 9 cannot communicate with client version 4 + +3.配置：【3】讲的比较详细 +在conf/tachyon-env.sh配置： + + export TACHYON_MASTER_ADDRESS=Master + #export TACHYON_UNDERFS_ADDRESS=$TACHYON_HOME/underFSStorage + export TACHYON_UNDERFS_ADDRESS=hdfs://Master:9000 + export TACHYON_WORKER_MEMORY_SIZE=256MB + +Master为集群master IP，或者自己在本地设置好映射 + +4.安装： + + $ cd tachyon + $ ./bin/tachyon format + $ ./bin/tachyon-start.sh # use the right parameters here. e.g. all Mount +一般start需要sudo才能mount： + + ./bin/tachyon-start.sh all SudoMount + +每个节点都要输入密码，好麻烦 + +5.查看： +在Master的19999端口可以查看，六个worker启动了三个， +解决办法：http://blog.csdn.net/xubo245/article/details/53056482 + +6.使用： +上传： + + hadoop@Master:~/cloud/tachyon-0.7.1-hadoop2.6-bin$ ./bin/tachyon tfs copyFromLocal test/2.txt / + Copied test/2.txt to / + +展示没问题 + + +7.记录 +ipc版本不同： + + hadoop@Master:~/cloud/tachyon-0.7.1$ ./bin/tachyon format + Connecting to Mcnode1 as hadoop... + Formatting Tachyon Worker @ Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Formatting Tachyon Worker @ Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode3 + /home/hadoop/cloud/tachyon-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode4 + /home/hadoop/cloud/tachyon-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode5 + /home/hadoop/cloud/tachyon-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Formatting Tachyon Worker @ Mcnode6 + Connection to mcnode6 closed. + Formatting Tachyon Master @ localhost + Exception in thread "main" java.lang.IllegalArgumentException: All eligible Under File Systems were unable to create an instance for the given path: hdfs://Master:9000/tmp/tachyon/data + java.lang.RuntimeException: org.apache.hadoop.ipc.RemoteException: Server IPC version 9 cannot communicate with client version 4 + + at tachyon.underfs.UnderFileSystemRegistry.create(UnderFileSystemRegistry.java:132) + at tachyon.underfs.UnderFileSystem.get(UnderFileSystem.java:99) + at tachyon.underfs.UnderFileSystem.get(UnderFileSystem.java:83) + at tachyon.Format.formatFolder(Format.java:38) + at tachyon.Format.main(Format.java:77) + + + +参考 +【1】http://alluxio.org/documentation/v0.7.1/ +【2】https://github.com/xubo245/SparkLearning +【3】 http://alluxio.org/documentation/v0.7.1/Running-Tachyon-on-a-Cluster.html diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2407--\344\270\213\350\275\275\346\272\220\347\240\201\351\200\232\350\277\207maven\345\256\211\350\243\205\346\210\220\345\212\237.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2407--\344\270\213\350\275\275\346\272\220\347\240\201\351\200\232\350\277\207maven\345\256\211\350\243\205\346\210\220\345\212\237.md" new file mode 100644 index 0000000..6dd6a45 --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2407--\344\270\213\350\275\275\346\272\220\347\240\201\351\200\232\350\277\207maven\345\256\211\350\243\205\346\210\220\345\212\237.md" @@ -0,0 +1,1108 @@ +更多代码请见：https://github.com/xubo245/SparkLearning + +1.环境 + + hadoop 2.6.0 + spark 1.5.2 + java 1.7 + +2.下载： + + https://github.com/Alluxio/alluxio/archive/v0.7.1.tar.gz + +3.编译： + + mvn clean package -Djava.version=1.7 -Dhadoop.version=2.6.0 -Dspark.version=1.5.2 -DskipTests + +4.安装成功： + + [INFO] ------------------------------------------------------------------------ + [INFO] Reactor Summary: + [INFO] + [INFO] Tachyon Parent ..................................... SUCCESS [03:15 min] + [INFO] Tachyon Common ..................................... SUCCESS [01:23 min] + [INFO] Tachyon Under File System .......................... SUCCESS [ 0.609 s] + [INFO] Tachyon Under File System - Local FS ............... SUCCESS [ 32.970 s] + [INFO] Tachyon Under File System - HDFS ................... SUCCESS [ 5.169 s] + [INFO] Tachyon Under File System - Gluster FS ............. SUCCESS [01:43 min] + [INFO] Tachyon Under File System - Swift .................. SUCCESS [01:04 min] + [INFO] Tachyon Under File System - S3 ..................... SUCCESS [ 20.526 s] + [INFO] Tachyon Clients .................................... SUCCESS [ 0.523 s] + [INFO] Tachyon Clients - Implementation ................... SUCCESS [ 9.195 s] + [INFO] Tachyon Clients - Distribution ..................... SUCCESS [ 30.238 s] + [INFO] Tachyon Servers .................................... SUCCESS [ 50.587 s] + [INFO] Mock Tachyon Cluster ............................... SUCCESS [ 5.288 s] + [INFO] Tachyon Integration Tests .......................... SUCCESS [01:11 min] + [INFO] Tachyon Shell ...................................... SUCCESS [ 3.214 s] + [INFO] Tachyon Examples ................................... SUCCESS [ 3.489 s] + [INFO] Tachyon Assemblies ................................. SUCCESS [ 11.906 s] + [INFO] ------------------------------------------------------------------------ + [INFO] BUILD SUCCESS + [INFO] ------------------------------------------------------------------------ + [INFO] Total time: 11:32 min + [INFO] Finished at: 2016-05-05T19:29:29+08:00 + [INFO] Final Memory: 107M/642M + [INFO] ------------------------------------------------------------------------ + + + +5.安装记录： + + Constructing Javadoc information... + Standard Doclet version 1.7.0_79 + Building tree for all the packages and classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/MetricsConfig.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/MetricsSystem.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/ConsoleSink.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/CsvSink.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/GraphiteSink.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/JmxSink.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/MetricsServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/Sink.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/source/JvmSource.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/source/Source.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/Format.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/ServerConstants.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/StorageDirId.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/StorageLevelAlias.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/UserInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/Users.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/ValidateConf.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/DataServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/DataServer.Factory.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/SpaceCounter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/TachyonWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/WorkerSource.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/DataServerHandler.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/FileTransferType.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/NettyDataServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/PipelineHandler.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockDataManager.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockHeartbeatReport.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockHeartbeatReporter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockLockManager.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockLockType.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockMasterSync.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockMetadataManager.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockMetadataManagerView.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockMetricsReporter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockServiceHandler.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockStoreEventListener.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockStoreEventListenerBase.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockStoreLocation.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockStoreMeta.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockWorkerUtils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/ClientRWLock.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/PinListSync.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/TieredBlockStore.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/BlockReader.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/BlockWriter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/LocalFileBlockReader.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/LocalFileBlockWriter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/BlockMeta.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/BlockMetaBase.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/StorageDir.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/StorageDirView.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/StorageTier.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/StorageTierView.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/TempBlockMeta.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/Allocator.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/AllocatorFactory.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/AllocatorType.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/GreedyAllocator.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/MaxFreeAllocator.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/EvictionPlan.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/Evictor.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/EvictorFactory.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/EvictorType.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/EvictorUtils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/GreedyEvictor.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/LRUEvictor.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/nio/NIODataServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/ByteBufferReader.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/ByteBufferWriter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/JavaByteBufferReader.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/JavaByteBufferWriter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/Utils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/BlockInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/Counters.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/Dependency.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/DependencyType.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/DependencyVariables.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/EditLog.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/EditLogProcessor.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/Image.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/ImageWriter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/Inode.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/InodeFile.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/InodeFolder.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/Journal.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/MasterInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/MasterInfo.MasterInfoHeartbeatExecutor.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/MasterInfo.RecomputationScheduler.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/MasterServiceHandler.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/MasterSource.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/MasterWorkerInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/RawTables.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/RecomputeCommand.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/TachyonMaster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/MasterUIWebServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/UiBlockInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/UiFileInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/UiFileInfo.LocalFileInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/UIWebServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/Utils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceBrowseLogsServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceBrowseServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceConfigurationServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceDependencyServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceDownloadLocalServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceDownloadServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceGeneralServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceGeneralServlet.StorageTierInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceHeaderServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceMemoryServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceWorkerBlockInfoServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceWorkerGeneralServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceWorkerGeneralServlet.UiStorageDir.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceWorkersServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceWorkersServlet.NodeInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WorkerUIWebServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/overview-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/source/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/source/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/source/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/nio/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/nio/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/nio/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/constant-values.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/serialized-form.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/class-use/MetricsSystem.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/class-use/MetricsConfig.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/class-use/GraphiteSink.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/class-use/MetricsServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/class-use/JmxSink.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/class-use/CsvSink.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/class-use/ConsoleSink.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/class-use/Sink.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/source/class-use/Source.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/source/class-use/JvmSource.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/class-use/UserInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/class-use/ServerConstants.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/class-use/StorageDirId.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/class-use/ValidateConf.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/class-use/Users.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/class-use/StorageLevelAlias.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/class-use/Format.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/class-use/SpaceCounter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/class-use/WorkerSource.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/class-use/TachyonWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/class-use/DataServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/class-use/DataServer.Factory.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/class-use/FileTransferType.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/class-use/PipelineHandler.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/class-use/NettyDataServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/class-use/DataServerHandler.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/TieredBlockStore.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockStoreEventListenerBase.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/ClientRWLock.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/PinListSync.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockMetadataManager.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockStoreLocation.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockHeartbeatReporter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockHeartbeatReport.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockServiceHandler.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockMetadataManagerView.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockWorkerUtils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockLockType.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockStoreEventListener.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockLockManager.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockStoreMeta.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockMasterSync.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockMetricsReporter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockDataManager.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/class-use/BlockWriter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/class-use/LocalFileBlockReader.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/class-use/LocalFileBlockWriter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/class-use/BlockReader.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/class-use/BlockMeta.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/class-use/StorageTier.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/class-use/StorageDirView.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/class-use/BlockMetaBase.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/class-use/StorageTierView.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/class-use/TempBlockMeta.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/class-use/StorageDir.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/class-use/AllocatorFactory.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/class-use/GreedyAllocator.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/class-use/MaxFreeAllocator.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/class-use/Allocator.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/class-use/AllocatorType.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/class-use/EvictorUtils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/class-use/EvictorFactory.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/class-use/GreedyEvictor.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/class-use/EvictionPlan.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/class-use/LRUEvictor.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/class-use/EvictorType.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/class-use/Evictor.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/nio/class-use/NIODataServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/class-use/ByteBufferReader.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/class-use/JavaByteBufferWriter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/class-use/Utils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/class-use/ByteBufferWriter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/class-use/JavaByteBufferReader.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/Counters.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/EditLogProcessor.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/Image.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/MasterWorkerInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/DependencyVariables.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/RawTables.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/MasterInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/MasterInfo.MasterInfoHeartbeatExecutor.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/MasterInfo.RecomputationScheduler.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/EditLog.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/Journal.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/InodeFolder.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/DependencyType.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/Inode.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/RecomputeCommand.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/TachyonMaster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/InodeFile.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/MasterServiceHandler.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/MasterSource.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/Dependency.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/ImageWriter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/BlockInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceGeneralServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceGeneralServlet.StorageTierInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceWorkerGeneralServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceWorkerGeneralServlet.UiStorageDir.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/MasterUIWebServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceDependencyServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WorkerUIWebServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/Utils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceWorkersServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceWorkersServlet.NodeInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/UiFileInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/UiFileInfo.LocalFileInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/UiBlockInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceDownloadLocalServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceWorkerBlockInfoServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceDownloadServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceHeaderServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceConfigurationServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/UIWebServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceBrowseServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceMemoryServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceBrowseLogsServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/source/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/nio/package-use.html... + Building index for all the packages and classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/overview-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/index-all.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/deprecated-list.html... + Building index for all classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/allclasses-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/allclasses-noframe.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/index.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/overview-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/help-doc.html... + 16 warnings + [WARNING] Javadoc Warnings + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/BlockDataManager.java:191: warning - @return tag cannot be used in method with void return type. + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/BlockDataManager.java:476: warning - @param argument "pinnedInodes," is not a parameter name. + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/BlockMetadataManagerView.java:63: warning - Tag @return cannot be used in constructor documentation. It can only be used in the following types of documentation: method. + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/BlockMetadataManagerView.java:63: warning - @param argument "pinnedInodes," is not a parameter name. + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/BlockMetadataManagerView.java:63: warning - @param argument "lockedBlocks," is not a parameter name. + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/PinListSync.java:65: warning - Tag @return cannot be used in constructor documentation. It can only be used in the following types of documentation: method. + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/TieredBlockStore.java:525: warning - @param argument "inodes," is not a parameter name. + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/meta/StorageDirView.java:47: warning - Tag @return cannot be used in constructor documentation. It can only be used in the following types of documentation: method. + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/meta/StorageTierView.java:45: warning - @param argument "view," is not a parameter name. + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/meta/StorageTierView.java:45: warning - @param argument "StorageTierView" is not a parameter name. + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/allocator/Allocator.java:42: warning - Tag @link: reference not found: BlockStore + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/allocator/Allocator.java:42: warning - Tag @link: reference not found: BlockStore + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/allocator/Allocator.java:42: warning - Tag @link: reference not found: BlockStore + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/evictor/Evictor.java:47: warning - Tag @link: reference not found: BlockStore + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/evictor/Evictor.java:47: warning - Tag @link: reference not found: BlockStore + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/evictor/Evictor.java:47: warning - Tag @link: reference not found: BlockStore + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/servers/target/tachyon-servers-0.7.1-javadoc.jar + [INFO] + [INFO] --- maven-jar-plugin:2.4:test-jar (default) @ tachyon-servers --- + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/servers/target/tachyon-servers-0.7.1-tests.jar + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Mock Tachyon Cluster 0.7.1 + [INFO] ------------------------------------------------------------------------ + Downloading: https://repo1.maven.org/maven2/org/apache/curator/curator-test/2.1.0-incubating/curator-test-2.1.0-incubating.pom + Downloaded: https://repo1.maven.org/maven2/org/apache/curator/curator-test/2.1.0-incubating/curator-test-2.1.0-incubating.pom (2 KB at 4.3 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/curator/curator-test/2.1.0-incubating/curator-test-2.1.0-incubating.jar + Downloaded: https://repo1.maven.org/maven2/org/apache/curator/curator-test/2.1.0-incubating/curator-test-2.1.0-incubating.jar (35 KB at 26.6 KB/sec) + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-minicluster --- + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-minicluster --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-minicluster --- + [INFO] Starting audit... + Audit done. + + [INFO] + [INFO] --- license-maven-plugin:2.9:check (default) @ tachyon-minicluster --- + [INFO] Checking licenses... + [INFO] + [INFO] --- maven-resources-plugin:2.6:resources (default-resources) @ tachyon-minicluster --- + [INFO] Using 'UTF-8' encoding to copy filtered resources. + [INFO] skip non existing resourceDirectory /home/hadoop/cloud/alluxio-0.7.1/minicluster/src/main/resources + [INFO] + [INFO] --- maven-compiler-plugin:3.2:compile (default-compile) @ tachyon-minicluster --- + [INFO] Changes detected - recompiling the module! + [INFO] Compiling 6 source files to /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/classes + [INFO] + [INFO] --- maven-resources-plugin:2.6:testResources (default-testResources) @ tachyon-minicluster --- + [INFO] Using 'UTF-8' encoding to copy filtered resources. + [INFO] skip non existing resourceDirectory /home/hadoop/cloud/alluxio-0.7.1/minicluster/src/test/resources + [INFO] + [INFO] --- maven-compiler-plugin:3.2:testCompile (default-testCompile) @ tachyon-minicluster --- + [INFO] No sources to compile + [INFO] + [INFO] --- maven-surefire-plugin:2.14:test (default-test) @ tachyon-minicluster --- + [INFO] Tests are skipped. + [INFO] + [INFO] --- maven-jar-plugin:2.4:jar (default-jar) @ tachyon-minicluster --- + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/tachyon-minicluster-0.7.1.jar + [INFO] + [INFO] >>> maven-source-plugin:2.3:jar (attach-sources) > generate-sources @ tachyon-minicluster >>> + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-minicluster --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-minicluster --- + [INFO] Starting audit... + Audit done. + + [INFO] + [INFO] --- license-maven-plugin:2.9:check (default) @ tachyon-minicluster --- + [INFO] Checking licenses... + [INFO] + [INFO] <<< maven-source-plugin:2.3:jar (attach-sources) < generate-sources @ tachyon-minicluster <<< + [INFO] + [INFO] --- maven-source-plugin:2.3:jar (attach-sources) @ tachyon-minicluster --- + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/tachyon-minicluster-0.7.1-sources.jar + [INFO] + [INFO] --- maven-javadoc-plugin:2.9:jar (attach-javadoc) @ tachyon-minicluster --- + [INFO] + Loading source files for package tachyon.underfs... + Loading source files for package tachyon.master... + Loading source files for package tachyon... + Constructing Javadoc information... + Standard Doclet version 1.7.0_79 + Building tree for all the packages and classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/underfs/UnderFileSystemCluster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/ClientPool.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/LocalTachyonCluster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/LocalTachyonClusterMultiMaster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/LocalTachyonMaster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/LocalFilesystemCluster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/overview-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/underfs/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/underfs/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/underfs/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/constant-values.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/underfs/class-use/UnderFileSystemCluster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/class-use/LocalTachyonMaster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/class-use/LocalTachyonClusterMultiMaster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/class-use/ClientPool.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/class-use/LocalTachyonCluster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/class-use/LocalFilesystemCluster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/underfs/package-use.html... + Building index for all the packages and classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/overview-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/index-all.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/deprecated-list.html... + Building index for all classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/allclasses-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/allclasses-noframe.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/index.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/overview-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/help-doc.html... + 1 warning + [WARNING] Javadoc Warnings + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/minicluster/src/main/java/tachyon/underfs/UnderFileSystemCluster.java:141: warning - @return tag has no arguments. + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/tachyon-minicluster-0.7.1-javadoc.jar + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Integration Tests 0.7.1 + [INFO] ------------------------------------------------------------------------ + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-minicluster/2.6.0/hadoop-minicluster-2.6.0.pom + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-minicluster/2.6.0/hadoop-minicluster-2.6.0.pom (4 KB at 6.6 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-tests/2.6.0/hadoop-yarn-server-tests-2.6.0.pom + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-tests/2.6.0/hadoop-yarn-server-tests-2.6.0.pom (7 KB at 14.3 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-resourcemanager/2.6.0/hadoop-yarn-server-resourcemanager-2.6.0.pom + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-resourcemanager/2.6.0/hadoop-yarn-server-resourcemanager-2.6.0.pom (13 KB at 10.8 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-applicationhistoryservice/2.6.0/hadoop-yarn-server-applicationhistoryservice-2.6.0.pom + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-applicationhistoryservice/2.6.0/hadoop-yarn-server-applicationhistoryservice-2.6.0.pom (7 KB at 14.6 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-web-proxy/2.6.0/hadoop-yarn-server-web-proxy-2.6.0.pom + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-web-proxy/2.6.0/hadoop-yarn-server-web-proxy-2.6.0.pom (5 KB at 10.5 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-mapreduce-client-hs/2.6.0/hadoop-mapreduce-client-hs-2.6.0.pom + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-mapreduce-client-hs/2.6.0/hadoop-mapreduce-client-hs-2.6.0.pom (4 KB at 3.2 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-minicluster/2.6.0/hadoop-minicluster-2.6.0.jar + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-hdfs/2.6.0/hadoop-hdfs-2.6.0-tests.jar + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-tests/2.6.0/hadoop-yarn-server-tests-2.6.0-tests.jar + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-nodemanager/2.6.0/hadoop-yarn-server-nodemanager-2.6.0.jar + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-resourcemanager/2.6.0/hadoop-yarn-server-resourcemanager-2.6.0.jar + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-minicluster/2.6.0/hadoop-minicluster-2.6.0.jar (3 KB at 4.6 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-applicationhistoryservice/2.6.0/hadoop-yarn-server-applicationhistoryservice-2.6.0.jar + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-tests/2.6.0/hadoop-yarn-server-tests-2.6.0-tests.jar (52 KB at 27.7 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-mapreduce-client-jobclient/2.6.0/hadoop-mapreduce-client-jobclient-2.6.0-tests.jar + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-applicationhistoryservice/2.6.0/hadoop-yarn-server-applicationhistoryservice-2.6.0.jar (162 KB at 33.5 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-web-proxy/2.6.0/hadoop-yarn-server-web-proxy-2.6.0.jar + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-web-proxy/2.6.0/hadoop-yarn-server-web-proxy-2.6.0.jar (28 KB at 4.9 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-mapreduce-client-hs/2.6.0/hadoop-mapreduce-client-hs-2.6.0.jar + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-nodemanager/2.6.0/hadoop-yarn-server-nodemanager-2.6.0.jar (624 KB at 76.2 KB/sec) + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-mapreduce-client-hs/2.6.0/hadoop-mapreduce-client-hs-2.6.0.jar (229 KB at 23.7 KB/sec) + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-resourcemanager/2.6.0/hadoop-yarn-server-resourcemanager-2.6.0.jar (1108 KB at 36.6 KB/sec) + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-mapreduce-client-jobclient/2.6.0/hadoop-mapreduce-client-jobclient-2.6.0-tests.jar (1460 KB at 35.6 KB/sec) + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-hdfs/2.6.0/hadoop-hdfs-2.6.0-tests.jar (2959 KB at 45.4 KB/sec) + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-integration-tests --- + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-integration-tests --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-integration-tests --- + [INFO] Starting audit... + Audit done. + + [INFO] + [INFO] --- license-maven-plugin:2.9:check (default) @ tachyon-integration-tests --- + [INFO] Checking licenses... + [INFO] + [INFO] --- maven-resources-plugin:2.6:resources (default-resources) @ tachyon-integration-tests --- + [INFO] Using 'UTF-8' encoding to copy filtered resources. + [INFO] skip non existing resourceDirectory /home/hadoop/cloud/alluxio-0.7.1/integration-tests/src/main/resources + [INFO] + [INFO] --- maven-compiler-plugin:3.2:compile (default-compile) @ tachyon-integration-tests --- + [INFO] No sources to compile + [INFO] + [INFO] --- maven-resources-plugin:2.6:testResources (default-testResources) @ tachyon-integration-tests --- + [INFO] Using 'UTF-8' encoding to copy filtered resources. + [INFO] Copying 1 resource + [INFO] + [INFO] --- maven-compiler-plugin:3.2:testCompile (default-testCompile) @ tachyon-integration-tests --- + [INFO] Changes detected - recompiling the module! + [INFO] Compiling 37 source files to /home/hadoop/cloud/alluxio-0.7.1/integration-tests/target/test-classes + [INFO] /home/hadoop/cloud/alluxio-0.7.1/integration-tests/src/test/java/tachyon/hadoop/TFSRenameIntegrationTest.java: Some input files use or override a deprecated API. + [INFO] /home/hadoop/cloud/alluxio-0.7.1/integration-tests/src/test/java/tachyon/hadoop/TFSRenameIntegrationTest.java: Recompile with -Xlint:deprecation for details. + [INFO] + [INFO] --- maven-surefire-plugin:2.14:test (default-test) @ tachyon-integration-tests --- + [INFO] Tests are skipped. + [INFO] + [INFO] --- maven-jar-plugin:2.4:jar (default-jar) @ tachyon-integration-tests --- + [WARNING] JAR will be empty - no content was marked for inclusion! + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/integration-tests/target/tachyon-integration-tests-0.7.1.jar + [INFO] + [INFO] >>> maven-source-plugin:2.3:jar (attach-sources) > generate-sources @ tachyon-integration-tests >>> + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-integration-tests --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-integration-tests --- + [INFO] Starting audit... + Audit done. + + [INFO] + [INFO] --- license-maven-plugin:2.9:check (default) @ tachyon-integration-tests --- + [INFO] Checking licenses... + [INFO] + [INFO] <<< maven-source-plugin:2.3:jar (attach-sources) < generate-sources @ tachyon-integration-tests <<< + [INFO] + [INFO] --- maven-source-plugin:2.3:jar (attach-sources) @ tachyon-integration-tests --- + [INFO] No sources in project. Archive not created. + [INFO] + [INFO] --- maven-javadoc-plugin:2.9:jar (attach-javadoc) @ tachyon-integration-tests --- + [INFO] + [INFO] --- maven-jar-plugin:2.4:test-jar (default) @ tachyon-integration-tests --- + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/integration-tests/target/tachyon-integration-tests-0.7.1-tests.jar + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Shell 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-shell --- + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-shell --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-shell --- + [INFO] Starting audit... + Audit done. + + [INFO] + [INFO] --- license-maven-plugin:2.9:check (default) @ tachyon-shell --- + [INFO] Checking licenses... + [INFO] + [INFO] --- maven-resources-plugin:2.6:resources (default-resources) @ tachyon-shell --- + [INFO] Using 'UTF-8' encoding to copy filtered resources. + [INFO] skip non existing resourceDirectory /home/hadoop/cloud/alluxio-0.7.1/shell/src/main/resources + [INFO] + [INFO] --- maven-compiler-plugin:3.2:compile (default-compile) @ tachyon-shell --- + [INFO] Changes detected - recompiling the module! + [INFO] Compiling 2 source files to /home/hadoop/cloud/alluxio-0.7.1/shell/target/classes + [INFO] + [INFO] --- maven-resources-plugin:2.6:testResources (default-testResources) @ tachyon-shell --- + [INFO] Using 'UTF-8' encoding to copy filtered resources. + [INFO] Copying 1 resource + [INFO] + [INFO] --- maven-compiler-plugin:3.2:testCompile (default-testCompile) @ tachyon-shell --- + [INFO] Changes detected - recompiling the module! + [INFO] Compiling 2 source files to /home/hadoop/cloud/alluxio-0.7.1/shell/target/test-classes + [INFO] + [INFO] --- maven-surefire-plugin:2.14:test (default-test) @ tachyon-shell --- + [INFO] Tests are skipped. + [INFO] + [INFO] --- maven-jar-plugin:2.4:jar (default-jar) @ tachyon-shell --- + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/shell/target/tachyon-shell-0.7.1.jar + [INFO] + [INFO] >>> maven-source-plugin:2.3:jar (attach-sources) > generate-sources @ tachyon-shell >>> + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-shell --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-shell --- + [INFO] Starting audit... + Audit done. + + [INFO] + [INFO] --- license-maven-plugin:2.9:check (default) @ tachyon-shell --- + [INFO] Checking licenses... + [INFO] + [INFO] <<< maven-source-plugin:2.3:jar (attach-sources) < generate-sources @ tachyon-shell <<< + [INFO] + [INFO] --- maven-source-plugin:2.3:jar (attach-sources) @ tachyon-shell --- + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/shell/target/tachyon-shell-0.7.1-sources.jar + [INFO] + [INFO] --- maven-javadoc-plugin:2.9:jar (attach-javadoc) @ tachyon-shell --- + [INFO] + Loading source files for package tachyon.shell... + Constructing Javadoc information... + Standard Doclet version 1.7.0_79 + Building tree for all the packages and classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/tachyon/shell/TFsShell.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/tachyon/shell/TFsShellUtils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/tachyon/shell/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/tachyon/shell/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/tachyon/shell/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/constant-values.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/tachyon/shell/class-use/TFsShell.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/tachyon/shell/class-use/TFsShellUtils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/tachyon/shell/package-use.html... + Building index for all the packages and classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/overview-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/index-all.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/deprecated-list.html... + Building index for all classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/allclasses-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/allclasses-noframe.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/index.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/help-doc.html... + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/shell/target/tachyon-shell-0.7.1-javadoc.jar + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Examples 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-examples --- + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-examples --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-examples --- + [INFO] Starting audit... + Audit done. + + [INFO] + [INFO] --- license-maven-plugin:2.9:check (default) @ tachyon-examples --- + [INFO] Checking licenses... + [INFO] + [INFO] --- maven-resources-plugin:2.6:resources (default-resources) @ tachyon-examples --- + [INFO] Using 'UTF-8' encoding to copy filtered resources. + [INFO] skip non existing resourceDirectory /home/hadoop/cloud/alluxio-0.7.1/examples/src/main/resources + [INFO] + [INFO] --- maven-compiler-plugin:3.2:compile (default-compile) @ tachyon-examples --- + [INFO] Changes detected - recompiling the module! + [INFO] Compiling 6 source files to /home/hadoop/cloud/alluxio-0.7.1/examples/target/classes + [INFO] + [INFO] --- maven-resources-plugin:2.6:testResources (default-testResources) @ tachyon-examples --- + [INFO] Using 'UTF-8' encoding to copy filtered resources. + [INFO] skip non existing resourceDirectory /home/hadoop/cloud/alluxio-0.7.1/examples/src/test/resources + [INFO] + [INFO] --- maven-compiler-plugin:3.2:testCompile (default-testCompile) @ tachyon-examples --- + [INFO] No sources to compile + [INFO] + [INFO] --- maven-surefire-plugin:2.14:test (default-test) @ tachyon-examples --- + [INFO] Tests are skipped. + [INFO] + [INFO] --- maven-jar-plugin:2.4:jar (default-jar) @ tachyon-examples --- + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/examples/target/tachyon-examples-0.7.1.jar + [INFO] + [INFO] >>> maven-source-plugin:2.3:jar (attach-sources) > generate-sources @ tachyon-examples >>> + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-examples --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-examples --- + [INFO] Starting audit... + Audit done. + + [INFO] + [INFO] --- license-maven-plugin:2.9:check (default) @ tachyon-examples --- + [INFO] Checking licenses... + [INFO] + [INFO] <<< maven-source-plugin:2.3:jar (attach-sources) < generate-sources @ tachyon-examples <<< + [INFO] + [INFO] --- maven-source-plugin:2.3:jar (attach-sources) @ tachyon-examples --- + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/examples/target/tachyon-examples-0.7.1-sources.jar + [INFO] + [INFO] --- maven-javadoc-plugin:2.9:jar (attach-javadoc) @ tachyon-examples --- + [INFO] + Loading source files for package tachyon.examples... + Constructing Javadoc information... + Standard Doclet version 1.7.0_79 + Building tree for all the packages and classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/BasicCheckpoint.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/BasicNonByteBufferOperations.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/BasicOperations.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/BasicRawTableOperations.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/Performance.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/Performance.GeneralWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/Performance.HdfsWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/Performance.TachyonReadWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/Performance.TachyonWriterWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/Performance.Worker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/Utils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/constant-values.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/BasicNonByteBufferOperations.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/Utils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/BasicOperations.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/BasicRawTableOperations.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/Performance.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/Performance.Worker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/Performance.GeneralWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/Performance.TachyonWriterWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/Performance.TachyonReadWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/Performance.HdfsWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/BasicCheckpoint.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/package-use.html... + Building index for all the packages and classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/overview-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/index-all.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/deprecated-list.html... + Building index for all classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/allclasses-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/allclasses-noframe.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/index.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/help-doc.html... + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/examples/target/tachyon-examples-0.7.1-javadoc.jar + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Assemblies 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-assemblies --- + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-assemblies --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-assemblies --- + [INFO] Starting audit... + Audit done. + + [INFO] + [INFO] --- license-maven-plugin:2.9:check (default) @ tachyon-assemblies --- + [INFO] Checking licenses... + [INFO] + [INFO] --- maven-resources-plugin:2.6:resources (default-resources) @ tachyon-assemblies --- + [INFO] Using 'UTF-8' encoding to copy filtered resources. + [INFO] skip non existing resourceDirectory /home/hadoop/cloud/alluxio-0.7.1/assembly/src/main/resources + [INFO] + [INFO] --- maven-compiler-plugin:3.2:compile (default-compile) @ tachyon-assemblies --- + [INFO] No sources to compile + [INFO] + [INFO] --- maven-resources-plugin:2.6:testResources (default-testResources) @ tachyon-assemblies --- + [INFO] Using 'UTF-8' encoding to copy filtered resources. + [INFO] skip non existing resourceDirectory /home/hadoop/cloud/alluxio-0.7.1/assembly/src/test/resources + [INFO] + [INFO] --- maven-compiler-plugin:3.2:testCompile (default-testCompile) @ tachyon-assemblies --- + [INFO] No sources to compile + [INFO] + [INFO] --- maven-surefire-plugin:2.14:test (default-test) @ tachyon-assemblies --- + [INFO] Tests are skipped. + [INFO] + [INFO] --- maven-jar-plugin:2.4:jar (default-jar) @ tachyon-assemblies --- + [WARNING] JAR will be empty - no content was marked for inclusion! + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/assembly/target/tachyon-assemblies-0.7.1.jar + [INFO] + [INFO] --- maven-shade-plugin:2.2:shade (uber-jar) @ tachyon-assemblies --- + [INFO] Including org.tachyonproject:tachyon-common:jar:0.7.1 in the shaded jar. + [INFO] Including com.google.guava:guava:jar:14.0.1 in the shaded jar. + [INFO] Including commons-lang:commons-lang:jar:2.4 in the shaded jar. + [INFO] Including commons-io:commons-io:jar:2.4 in the shaded jar. + [INFO] Including io.netty:netty-all:jar:4.0.28.Final in the shaded jar. + [INFO] Including org.apache.commons:commons-lang3:jar:3.0 in the shaded jar. + [INFO] Including org.apache.curator:curator-framework:jar:2.1.0-incubating in the shaded jar. + [INFO] Including org.apache.zookeeper:zookeeper:jar:3.4.5 in the shaded jar. + [INFO] Including log4j:log4j:jar:1.2.15 in the shaded jar. + [INFO] Including javax.mail:mail:jar:1.4 in the shaded jar. + [INFO] Including jline:jline:jar:0.9.94 in the shaded jar. + [INFO] Including org.jboss.netty:netty:jar:3.2.2.Final in the shaded jar. + [INFO] Including org.apache.curator:curator-client:jar:2.1.0-incubating in the shaded jar. + [INFO] Including org.apache.curator:curator-recipes:jar:2.1.0-incubating in the shaded jar. + [INFO] Including org.apache.thrift:libthrift:jar:0.9.1 in the shaded jar. + [INFO] Including org.apache.httpcomponents:httpclient:jar:4.2.5 in the shaded jar. + [INFO] Including org.apache.httpcomponents:httpcore:jar:4.2.4 in the shaded jar. + [INFO] Including org.slf4j:slf4j-api:jar:1.7.2 in the shaded jar. + [INFO] Including org.tachyonproject:tachyon-servers:jar:0.7.1 in the shaded jar. + [INFO] Including org.tachyonproject:tachyon-client-unshaded:jar:0.7.1 in the shaded jar. + [INFO] Including io.dropwizard.metrics:metrics-core:jar:3.1.0 in the shaded jar. + [INFO] Including io.dropwizard.metrics:metrics-json:jar:3.1.0 in the shaded jar. + [INFO] Including io.dropwizard.metrics:metrics-jvm:jar:3.1.0 in the shaded jar. + [INFO] Including io.dropwizard.metrics:metrics-graphite:jar:3.1.0 in the shaded jar. + [INFO] Including org.eclipse.jetty:jetty-jsp:jar:7.6.15.v20140411 in the shaded jar. + [INFO] Including org.eclipse.jetty.orbit:javax.servlet.jsp:jar:2.1.0.v201105211820 in the shaded jar. + [INFO] Including org.eclipse.jetty.orbit:org.apache.jasper.glassfish:jar:2.1.0.v201110031002 in the shaded jar. + [INFO] Including org.eclipse.jetty.orbit:javax.servlet.jsp.jstl:jar:1.2.0.v201105211821 in the shaded jar. + [INFO] Including org.eclipse.jetty.orbit:org.apache.taglibs.standard.glassfish:jar:1.2.0.v201112081803 in the shaded jar. + [INFO] Including org.eclipse.jetty.orbit:javax.el:jar:2.1.0.v201105211819 in the shaded jar. + [INFO] Including org.eclipse.jetty.orbit:com.sun.el:jar:1.0.0.v201105211818 in the shaded jar. + [INFO] Including org.eclipse.jetty.orbit:org.eclipse.jdt.core:jar:3.7.1 in the shaded jar. + [INFO] Including org.eclipse.jetty:jetty-webapp:jar:7.6.15.v20140411 in the shaded jar. + [INFO] Including org.eclipse.jetty:jetty-xml:jar:7.6.15.v20140411 in the shaded jar. + [INFO] Including org.eclipse.jetty:jetty-util:jar:7.6.15.v20140411 in the shaded jar. + [INFO] Including org.eclipse.jetty:jetty-server:jar:7.6.15.v20140411 in the shaded jar. + [INFO] Including org.eclipse.jetty.orbit:javax.servlet:jar:2.5.0.v201103041518 in the shaded jar. + [INFO] Including org.eclipse.jetty:jetty-continuation:jar:7.6.15.v20140411 in the shaded jar. + [INFO] Including org.eclipse.jetty:jetty-http:jar:7.6.15.v20140411 in the shaded jar. + [INFO] Including org.eclipse.jetty:jetty-io:jar:7.6.15.v20140411 in the shaded jar. + [INFO] Including org.eclipse.jetty:jetty-servlet:jar:7.6.15.v20140411 in the shaded jar. + [INFO] Including org.eclipse.jetty:jetty-security:jar:7.6.15.v20140411 in the shaded jar. + [INFO] Including commons-codec:commons-codec:jar:1.5 in the shaded jar. + [INFO] Including com.fasterxml.jackson.core:jackson-databind:jar:2.3.0 in the shaded jar. + [INFO] Including com.fasterxml.jackson.core:jackson-annotations:jar:2.3.0 in the shaded jar. + [INFO] Including com.fasterxml.jackson.core:jackson-core:jar:2.3.0 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-client:jar:2.6.0 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-common:jar:2.6.0 in the shaded jar. + [INFO] Including commons-cli:commons-cli:jar:1.2 in the shaded jar. + [INFO] Including org.apache.commons:commons-math3:jar:3.1.1 in the shaded jar. + [INFO] Including xmlenc:xmlenc:jar:0.52 in the shaded jar. + [INFO] Including commons-net:commons-net:jar:3.1 in the shaded jar. + [INFO] Including commons-collections:commons-collections:jar:3.2.1 in the shaded jar. + [INFO] Including commons-configuration:commons-configuration:jar:1.6 in the shaded jar. + [INFO] Including commons-digester:commons-digester:jar:1.8 in the shaded jar. + [INFO] Including commons-beanutils:commons-beanutils:jar:1.7.0 in the shaded jar. + [INFO] Including commons-beanutils:commons-beanutils-core:jar:1.8.0 in the shaded jar. + [INFO] Including org.apache.avro:avro:jar:1.7.4 in the shaded jar. + [INFO] Including com.thoughtworks.paranamer:paranamer:jar:2.3 in the shaded jar. + [INFO] Including org.xerial.snappy:snappy-java:jar:1.0.4.1 in the shaded jar. + [INFO] Including com.google.protobuf:protobuf-java:jar:2.5.0 in the shaded jar. + [INFO] Including com.google.code.gson:gson:jar:2.2.4 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-auth:jar:2.6.0 in the shaded jar. + [INFO] Including org.apache.directory.server:apacheds-kerberos-codec:jar:2.0.0-M15 in the shaded jar. + [INFO] Including org.apache.directory.server:apacheds-i18n:jar:2.0.0-M15 in the shaded jar. + [INFO] Including org.apache.directory.api:api-asn1-api:jar:1.0.0-M20 in the shaded jar. + [INFO] Including org.apache.directory.api:api-util:jar:1.0.0-M20 in the shaded jar. + [INFO] Including com.google.code.findbugs:jsr305:jar:1.3.9 in the shaded jar. + [INFO] Including org.htrace:htrace-core:jar:3.0.4 in the shaded jar. + [INFO] Including org.apache.commons:commons-compress:jar:1.4.1 in the shaded jar. + [INFO] Including org.tukaani:xz:jar:1.0 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-hdfs:jar:2.6.0 in the shaded jar. + [INFO] Including org.mortbay.jetty:jetty-util:jar:6.1.26 in the shaded jar. + [INFO] Including io.netty:netty:jar:3.6.2.Final in the shaded jar. + [INFO] Including xerces:xercesImpl:jar:2.9.1 in the shaded jar. + [INFO] Including xml-apis:xml-apis:jar:1.3.04 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-mapreduce-client-app:jar:2.6.0 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-mapreduce-client-common:jar:2.6.0 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-yarn-client:jar:2.6.0 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-mapreduce-client-shuffle:jar:2.6.0 in the shaded jar. + [INFO] Including org.fusesource.leveldbjni:leveldbjni-all:jar:1.8 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-yarn-api:jar:2.6.0 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-mapreduce-client-core:jar:2.6.0 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-yarn-common:jar:2.6.0 in the shaded jar. + [INFO] Including org.codehaus.jackson:jackson-jaxrs:jar:1.9.13 in the shaded jar. + [INFO] Including org.codehaus.jackson:jackson-xc:jar:1.9.13 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-mapreduce-client-jobclient:jar:2.6.0 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-annotations:jar:2.6.0 in the shaded jar. + [INFO] Including org.tachyonproject:tachyon-underfs-hdfs:jar:0.7.1 in the shaded jar. + [INFO] Including org.tachyonproject:tachyon-underfs-glusterfs:jar:0.7.1 in the shaded jar. + [INFO] Including org.gluster:glusterfs-hadoop:jar:2.3.13 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-yarn-server-nodemanager:jar:2.3.0 in the shaded jar. + [INFO] Including javax.xml.bind:jaxb-api:jar:2.2.2 in the shaded jar. + [INFO] Including javax.xml.stream:stax-api:jar:1.0-2 in the shaded jar. + [INFO] Including javax.activation:activation:jar:1.1 in the shaded jar. + [INFO] Including org.codehaus.jettison:jettison:jar:1.1 in the shaded jar. + [INFO] Including javax.servlet:servlet-api:jar:2.5 in the shaded jar. + [INFO] Including com.sun.jersey:jersey-core:jar:1.9 in the shaded jar. + [INFO] Including com.sun.jersey:jersey-client:jar:1.9 in the shaded jar. + [INFO] Including com.google.inject.extensions:guice-servlet:jar:3.0 in the shaded jar. + [INFO] Including com.google.inject:guice:jar:3.0 in the shaded jar. + [INFO] Including javax.inject:javax.inject:jar:1 in the shaded jar. + [INFO] Including aopalliance:aopalliance:jar:1.0 in the shaded jar. + [INFO] Including com.sun.jersey:jersey-json:jar:1.9 in the shaded jar. + [INFO] Including com.sun.xml.bind:jaxb-impl:jar:2.2.3-1 in the shaded jar. + [INFO] Including com.sun.jersey.contribs:jersey-guice:jar:1.9 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-yarn-server-common:jar:2.3.0 in the shaded jar. + [INFO] Including org.slf4j:slf4j-log4j12:jar:1.7.2 in the shaded jar. + [INFO] Including org.tachyonproject:tachyon-underfs-s3:jar:0.7.1 in the shaded jar. + [INFO] Including net.java.dev.jets3t:jets3t:jar:0.8.1 in the shaded jar. + [INFO] Including commons-logging:commons-logging:jar:1.1.1 in the shaded jar. + [INFO] Including com.jamesmurty.utils:java-xmlbuilder:jar:0.4 in the shaded jar. + [INFO] Including org.tachyonproject:tachyon-underfs-swift:jar:0.7.1 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-openstack:jar:2.6.0 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-common:test-jar:tests:2.6.0 in the shaded jar. + [INFO] Including org.mortbay.jetty:jetty:jar:6.1.26 in the shaded jar. + [INFO] Including com.sun.jersey:jersey-server:jar:1.9 in the shaded jar. + [INFO] Including asm:asm:jar:3.1 in the shaded jar. + [INFO] Including tomcat:jasper-compiler:jar:5.5.23 in the shaded jar. + [INFO] Including tomcat:jasper-runtime:jar:5.5.23 in the shaded jar. + [INFO] Including javax.servlet.jsp:jsp-api:jar:2.1 in the shaded jar. + [INFO] Including commons-el:commons-el:jar:1.0 in the shaded jar. + [INFO] Including com.jcraft:jsch:jar:0.1.42 in the shaded jar. + [INFO] Including org.codehaus.jackson:jackson-mapper-asl:jar:1.9.13 in the shaded jar. + [INFO] Including org.codehaus.jackson:jackson-core-asl:jar:1.9.13 in the shaded jar. + [INFO] Including org.mockito:mockito-all:jar:1.8.5 in the shaded jar. + [INFO] Including commons-httpclient:commons-httpclient:jar:3.1 in the shaded jar. + [INFO] Including org.tachyonproject:tachyon-underfs-local:jar:0.7.1 in the shaded jar. + [INFO] Including org.tachyonproject:tachyon-examples:jar:0.7.1 in the shaded jar. + [INFO] Including org.tachyonproject:tachyon-shell:jar:0.7.1 in the shaded jar. + [WARNING] javax.el-2.1.0.v201105211819.jar, jsp-api-2.1.jar define 26 overlappping classes: + [WARNING] - javax.el.ELContextListener + [WARNING] - javax.el.Expression + [WARNING] - javax.el.MapELResolver + [WARNING] - javax.el.VariableMapper + [WARNING] - javax.el.ELException + [WARNING] - javax.el.MethodNotFoundException + [WARNING] - javax.el.ArrayELResolver + [WARNING] - javax.el.ValueExpression + [WARNING] - javax.el.CompositeELResolver$CompositeIterator + [WARNING] - javax.el.ELContextEvent + [WARNING] - 16 more... + [WARNING] jasper-compiler-5.5.23.jar, org.apache.jasper.glassfish-2.1.0.v201110031002.jar, jasper-runtime-5.5.23.jar define 1 overlappping classes: + [WARNING] - org.apache.jasper.compiler.Localizer + [WARNING] commons-beanutils-core-1.8.0.jar, commons-beanutils-1.7.0.jar, commons-collections-3.2.1.jar define 10 overlappping classes: + [WARNING] - org.apache.commons.collections.FastHashMap$EntrySet + [WARNING] - org.apache.commons.collections.ArrayStack + [WARNING] - org.apache.commons.collections.FastHashMap$1 + [WARNING] - org.apache.commons.collections.FastHashMap$KeySet + [WARNING] - org.apache.commons.collections.FastHashMap$CollectionView + [WARNING] - org.apache.commons.collections.BufferUnderflowException + [WARNING] - org.apache.commons.collections.Buffer + [WARNING] - org.apache.commons.collections.FastHashMap$CollectionView$CollectionViewIterator + [WARNING] - org.apache.commons.collections.FastHashMap$Values + [WARNING] - org.apache.commons.collections.FastHashMap + [WARNING] javax.servlet.jsp-2.1.0.v201105211820.jar, jsp-api-2.1.jar define 58 overlappping classes: + [WARNING] - javax.servlet.jsp.tagext.SimpleTagSupport + [WARNING] - javax.servlet.jsp.tagext.SimpleTag + [WARNING] - javax.servlet.jsp.el.ExpressionEvaluator + [WARNING] - javax.servlet.jsp.el.ImplicitObjectELResolver$ImplicitObjects + [WARNING] - javax.servlet.jsp.JspPage + [WARNING] - javax.servlet.jsp.tagext.TagLibraryValidator + [WARNING] - javax.servlet.jsp.el.ImplicitObjectELResolver$ImplicitObjects$2 + [WARNING] - javax.servlet.jsp.el.ImplicitObjectELResolver$ImplicitObjects$4 + [WARNING] - javax.servlet.jsp.JspApplicationContext + [WARNING] - javax.servlet.jsp.el.ImplicitObjectELResolver$ImplicitObjects$9 + [WARNING] - 48 more... + [WARNING] hadoop-common-2.6.0-tests.jar, hadoop-common-2.6.0.jar define 37 overlappping classes: + [WARNING] - org.apache.hadoop.ipc.protobuf.TestProtos$EmptyResponseProto$1 + [WARNING] - org.apache.hadoop.ipc.protobuf.TestProtos$EmptyResponseProtoOrBuilder + [WARNING] - org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub + [WARNING] - org.apache.hadoop.ipc.protobuf.TestProtos$EchoResponseProto + [WARNING] - org.apache.hadoop.ipc.protobuf.TestProtos$EchoResponseProtoOrBuilder + [WARNING] - org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos$TestProtobufRpc2Proto$BlockingInterface + [WARNING] - org.apache.hadoop.io.serializer.avro.AvroRecord$1 + [WARNING] - org.apache.hadoop.ipc.protobuf.TestProtos$EchoRequestProtoOrBuilder + [WARNING] - org.apache.hadoop.ipc.protobuf.TestProtos$EchoResponseProto$Builder + [WARNING] - org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos$1 + [WARNING] - 27 more... + [WARNING] commons-beanutils-core-1.8.0.jar, commons-beanutils-1.7.0.jar define 82 overlappping classes: + [WARNING] - org.apache.commons.beanutils.Converter + [WARNING] - org.apache.commons.beanutils.WrapDynaBean + [WARNING] - org.apache.commons.beanutils.converters.IntegerConverter + [WARNING] - org.apache.commons.beanutils.locale.LocaleBeanUtilsBean + [WARNING] - org.apache.commons.beanutils.locale.converters.DoubleLocaleConverter + [WARNING] - org.apache.commons.beanutils.locale.converters.DecimalLocaleConverter + [WARNING] - org.apache.commons.beanutils.converters.ShortConverter + [WARNING] - org.apache.commons.beanutils.converters.StringArrayConverter + [WARNING] - org.apache.commons.beanutils.locale.LocaleConvertUtilsBean + [WARNING] - org.apache.commons.beanutils.LazyDynaClass + [WARNING] - 72 more... + [WARNING] hadoop-yarn-common-2.6.0.jar, hadoop-yarn-api-2.6.0.jar define 3 overlappping classes: + [WARNING] - org.apache.hadoop.yarn.factories.package-info + [WARNING] - org.apache.hadoop.yarn.util.package-info + [WARNING] - org.apache.hadoop.yarn.factory.providers.package-info + [WARNING] netty-3.6.2.Final.jar, netty-3.2.2.Final.jar define 497 overlappping classes: + [WARNING] - org.jboss.netty.logging.CommonsLoggerFactory + [WARNING] - org.jboss.netty.channel.local.LocalChannel + [WARNING] - org.jboss.netty.buffer.ChannelBufferIndexFinder$5 + [WARNING] - org.jboss.netty.util.internal.jzlib.CRC32 + [WARNING] - org.jboss.netty.util.ExternalResourceUtil + [WARNING] - org.jboss.netty.handler.codec.base64.Base64Decoder + [WARNING] - org.jboss.netty.channel.socket.nio.NioDatagramPipelineSink$1 + [WARNING] - org.jboss.netty.util.VirtualExecutorService + [WARNING] - org.jboss.netty.util.DefaultObjectSizeEstimator + [WARNING] - org.jboss.netty.channel.socket.http.HttpTunnelingClientSocketChannel$6$1 + [WARNING] - 487 more... + [WARNING] jasper-compiler-5.5.23.jar, org.apache.jasper.glassfish-2.1.0.v201110031002.jar define 143 overlappping classes: + [WARNING] - org.apache.jasper.compiler.TagLibraryInfoImpl + [WARNING] - org.apache.jasper.xmlparser.SymbolTable + [WARNING] - org.apache.jasper.compiler.Generator$FragmentHelperClass$Fragment + [WARNING] - org.apache.jasper.compiler.Generator$1TagHandlerPoolVisitor + [WARNING] - org.apache.jasper.compiler.ScriptingVariabler + [WARNING] - org.apache.jasper.compiler.SmapStratum$LineInfo + [WARNING] - org.apache.jasper.compiler.Node$AttributeGenerator + [WARNING] - org.apache.jasper.compiler.Node + [WARNING] - org.apache.jasper.compiler.Node$JspAttribute + [WARNING] - org.apache.jasper.compiler.tagplugin.TagPlugin + [WARNING] - 133 more... + [WARNING] org.apache.jasper.glassfish-2.1.0.v201110031002.jar, jasper-runtime-5.5.23.jar define 43 overlappping classes: + [WARNING] - org.apache.jasper.runtime.ProtectedFunctionMapper$2 + [WARNING] - org.apache.jasper.runtime.PageContextImpl$3 + [WARNING] - org.apache.jasper.runtime.JspFactoryImpl$PrivilegedGetPageContext + [WARNING] - org.apache.jasper.runtime.PageContextImpl$4 + [WARNING] - org.apache.jasper.runtime.JspFactoryImpl + [WARNING] - org.apache.jasper.runtime.ProtectedFunctionMapper$1 + [WARNING] - org.apache.jasper.JasperException + [WARNING] - org.apache.jasper.runtime.PerThreadTagHandlerPool$PerThreadData + [WARNING] - org.apache.jasper.util.FastDateFormat + [WARNING] - org.apache.jasper.runtime.PageContextImpl$8 + [WARNING] - 33 more... + [WARNING] servlet-api-2.5.jar, javax.servlet-2.5.0.v201103041518.jar define 42 overlappping classes: + [WARNING] - javax.servlet.ServletRequestWrapper + [WARNING] - javax.servlet.FilterChain + [WARNING] - javax.servlet.SingleThreadModel + [WARNING] - javax.servlet.http.HttpServletResponse + [WARNING] - javax.servlet.http.HttpUtils + [WARNING] - javax.servlet.ServletContextAttributeEvent + [WARNING] - javax.servlet.ServletContextAttributeListener + [WARNING] - javax.servlet.http.HttpServlet + [WARNING] - javax.servlet.http.HttpSessionAttributeListener + [WARNING] - javax.servlet.http.HttpServletRequest + [WARNING] - 32 more... + [WARNING] hadoop-yarn-common-2.6.0.jar, hadoop-yarn-client-2.6.0.jar define 2 overlappping classes: + [WARNING] - org.apache.hadoop.yarn.client.api.impl.package-info + [WARNING] - org.apache.hadoop.yarn.client.api.package-info + [WARNING] maven-shade-plugin has detected that some .class files + [WARNING] are present in two or more JARs. When this happens, only + [WARNING] one single version of the class is copied in the uberjar. + [WARNING] Usually this is not harmful and you can skeep these + [WARNING] warnings, otherwise try to manually exclude artifacts + [WARNING] based on mvn dependency:tree -Ddetail=true and the above + [WARNING] output + [WARNING] See http://docs.codehaus.org/display/MAVENUSER/Shade+Plugin + [INFO] Replacing /home/hadoop/cloud/alluxio-0.7.1/assembly/target/tachyon-assemblies-0.7.1-jar-with-dependencies.jar with /home/hadoop/cloud/alluxio-0.7.1/assembly/target/tachyon-assemblies-0.7.1-shaded.jar + [INFO] + [INFO] >>> maven-source-plugin:2.3:jar (attach-sources) > generate-sources @ tachyon-assemblies >>> + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-assemblies --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-assemblies --- + [INFO] Starting audit... + Audit done. + + [INFO] + [INFO] --- license-maven-plugin:2.9:check (default) @ tachyon-assemblies --- + [INFO] Checking licenses... + [INFO] + [INFO] <<< maven-source-plugin:2.3:jar (attach-sources) < generate-sources @ tachyon-assemblies <<< + [INFO] + [INFO] --- maven-source-plugin:2.3:jar (attach-sources) @ tachyon-assemblies --- + [INFO] No sources in project. Archive not created. + [INFO] + [INFO] --- maven-javadoc-plugin:2.9:jar (attach-javadoc) @ tachyon-assemblies --- + [INFO] ------------------------------------------------------------------------ + [INFO] Reactor Summary: + [INFO] + [INFO] Tachyon Parent ..................................... SUCCESS [03:15 min] + [INFO] Tachyon Common ..................................... SUCCESS [01:23 min] + [INFO] Tachyon Under File System .......................... SUCCESS [ 0.609 s] + [INFO] Tachyon Under File System - Local FS ............... SUCCESS [ 32.970 s] + [INFO] Tachyon Under File System - HDFS ................... SUCCESS [ 5.169 s] + [INFO] Tachyon Under File System - Gluster FS ............. SUCCESS [01:43 min] + [INFO] Tachyon Under File System - Swift .................. SUCCESS [01:04 min] + [INFO] Tachyon Under File System - S3 ..................... SUCCESS [ 20.526 s] + [INFO] Tachyon Clients .................................... SUCCESS [ 0.523 s] + [INFO] Tachyon Clients - Implementation ................... SUCCESS [ 9.195 s] + [INFO] Tachyon Clients - Distribution ..................... SUCCESS [ 30.238 s] + [INFO] Tachyon Servers .................................... SUCCESS [ 50.587 s] + [INFO] Mock Tachyon Cluster ............................... SUCCESS [ 5.288 s] + [INFO] Tachyon Integration Tests .......................... SUCCESS [01:11 min] + [INFO] Tachyon Shell ...................................... SUCCESS [ 3.214 s] + [INFO] Tachyon Examples ................................... SUCCESS [ 3.489 s] + [INFO] Tachyon Assemblies ................................. SUCCESS [ 11.906 s] + [INFO] ------------------------------------------------------------------------ + [INFO] BUILD SUCCESS + [INFO] ------------------------------------------------------------------------ + [INFO] Total time: 11:32 min + [INFO] Finished at: 2016-05-05T19:29:29+08:00 + [INFO] Final Memory: 107M/642M + [INFO] ------------------------------------------------------------------------ + + + +参考 +【1】http://alluxio.org/documentation/v0.7.1/ +【2】https://github.com/xubo245/SparkLearning diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2408---\351\233\206\347\276\244\347\211\210\346\220\255\345\273\272\351\227\256\351\242\230\344\271\213\351\233\206\347\276\244\346\227\240\346\263\225\345\205\250\351\203\250\345\220\257\345\212\250.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2408---\351\233\206\347\276\244\347\211\210\346\220\255\345\273\272\351\227\256\351\242\230\344\271\213\351\233\206\347\276\244\346\227\240\346\263\225\345\205\250\351\203\250\345\220\257\345\212\250.md" new file mode 100644 index 0000000..2ac3ea1 --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2408---\351\233\206\347\276\244\347\211\210\346\220\255\345\273\272\351\227\256\351\242\230\344\271\213\351\233\206\347\276\244\346\227\240\346\263\225\345\205\250\351\203\250\345\220\257\345\212\250.md" @@ -0,0 +1,276 @@ +六个worker只启动了3个： + + hadoop@Master:~/cloud/alluxio-0.7.1$ ./bin/tachyon format + Connecting to Mcnode1 as hadoop... + Formatting Tachyon Worker @ Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Formatting Tachyon Worker @ Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode3 + /home/hadoop/cloud/alluxio-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode4 + /home/hadoop/cloud/alluxio-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode5 + /home/hadoop/cloud/alluxio-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Formatting Tachyon Worker @ Mcnode6 + Connection to mcnode6 closed. + Formatting Tachyon Master @ Master + hadoop@Master:~/cloud/alluxio-0.7.1$ ./bin/tachyon-start.sh all Mount + Killed 0 processes on Master + Killed 0 processes on Master + Connecting to Mcnode1 as hadoop... + Killed 0 processes on Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Killed 0 processes on Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Killed 0 processes on Mcnode3 + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Killed 0 processes on Mcnode4 + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Killed 0 processes on Mcnode5 + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Killed 0 processes on Mcnode6 + Connection to mcnode6 closed. + Starting master @ Master + Connecting to Mcnode1 as hadoop... + Formatting RamFS: /mnt/ramdisk (256mb) + umount: only root can do that + ERROR: umount RamFS /mnt/ramdisk failed + Mount failed, not starting worker + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Formatting RamFS: /mnt/ramdisk (256mb) + umount: only root can do that + ERROR: umount RamFS /mnt/ramdisk failed + Mount failed, not starting worker + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + TACHYON_LOGS_DIR: /home/hadoop/cloud/alluxio-0.7.1/logs + Formatting RamFS: /mnt/ramdisk (256mb) + umount: only root can do that + ERROR: umount RamFS /mnt/ramdisk failed + Mount failed, not starting worker + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + TACHYON_LOGS_DIR: /home/hadoop/cloud/alluxio-0.7.1/logs + Formatting RamFS: /mnt/ramdisk (256mb) + umount: only root can do that + ERROR: umount RamFS /mnt/ramdisk failed + Mount failed, not starting worker + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + TACHYON_LOGS_DIR: /home/hadoop/cloud/alluxio-0.7.1/logs + Formatting RamFS: /mnt/ramdisk (256mb) + umount: only root can do that + ERROR: umount RamFS /mnt/ramdisk failed + Mount failed, not starting worker + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Formatting RamFS: /mnt/ramdisk (256mb) + umount: only root can do that + ERROR: umount RamFS /mnt/ramdisk failed + Mount failed, not starting worker + Connection to mcnode6 closed. + hadoop@Master:~/cloud/alluxio-0.7.1$ ./bin/tachyon-start.sh all SudoMount + Killed 1 processes on Master + Killed 0 processes on Master + Connecting to Mcnode1 as hadoop... + Killed 0 processes on Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Killed 0 processes on Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Killed 0 processes on Mcnode3 + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Killed 0 processes on Mcnode4 + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Killed 0 processes on Mcnode5 + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Killed 0 processes on Mcnode6 + Connection to mcnode6 closed. + Starting master @ Master + Connecting to Mcnode1 as hadoop... + [sudo] password for hadoop: + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + [sudo] password for hadoop: + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + [sudo] password for hadoop: + dirname: missing operand + Try 'dirname --help' for more information. + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode3 + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + [sudo] password for hadoop: + dirname: missing operand + Try 'dirname --help' for more information. + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode4 + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + [sudo] password for hadoop: + dirname: missing operand + Try 'dirname --help' for more information. + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode5 + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + [sudo] password for hadoop: + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode6 + Connection to mcnode6 closed. + hadoop@Master:~/cloud/alluxio-0.7.1$ + hadoop@Master:~/cloud/alluxio-0.7.1$ jps + 30101 SparkSubmit + 17541 ResourceManager + 2526 TachyonMaster + 17389 SecondaryNameNode + 27677 CoarseGrainedExecutorBackend + 2610 Jps + 17173 NameNode + 17935 Master + 18125 Worker + 1450 SparkSubmit + 27574 SparkSubmit + 3528 HistoryServer + + +解决办法： +方法1： +在每个节点各自启动，可以将所有节点启动，在master的19999可以看到： + + Last login: Thu May 5 20:26:04 2016 from mcnode4 + hadoop@Mcnode5:~$ cd cloud/alluxio-0.7.1/bin/ + hadoop@Mcnode5:~/cloud/alluxio-0.7.1/bin$ ./tachyon-start.sh local + Killed 0 processes on Mcnode5 + Killed 0 processes on Mcnode5 + Connecting to Mcnode1 as hadoop... + Killed 1 processes on Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Killed 1 processes on Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + Warning: Permanently added 'mcnode3,mcnode3IP' (ECDSA) to the list of known hosts. + dirname: missing operand + Try 'dirname --help' for more information. + Killed 0 processes on Mcnode3 + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Killed 1 processes on Mcnode4 + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + Warning: Permanently added 'mcnode5,mcnode5IP' (ECDSA) to the list of known hosts. + dirname: missing operand + Try 'dirname --help' for more information. + Killed 0 processes on Mcnode5 + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Killed 1 processes on Mcnode6 + Connection to mcnode6 closed. + [sudo] password for hadoop: + dirname: missing operand + Try 'dirname --help' for more information. + Formatting RamFS: /mnt/ramdisk (256mb) + Starting master @ Master + Starting worker @ Mcnode5 + + + +但是上传一个文件后又报错： + + hadoop@Mcnode3:~/cloud/tachyon-0.7.1-hadoop2.6-bin$ jps + Error occurred during initialization of VM + java.lang.Error: Properties init: Could not determine current working directory. + at java.lang.System.initProperties(Native Method) + at java.lang.System.initializeSystemClass(System.java:1119) + +关闭集群： + + hadoop@Master:~/cloud/tachyon-0.7.1-hadoop2.6-bin$ ./bin/tachyon-stop.sh + Killed 1 processes on Master + Killed 0 processes on Master + Connecting to Mcnode1 as hadoop... + Killed 1 processes on Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Killed 1 processes on Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Killed 0 processes on Mcnode3 + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Killed 0 processes on Mcnode4 + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Killed 1 processes on Mcnode5 + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Killed 1 processes on Mcnode6 + Connection to mcnode6 closed. + +方法2：可以启动全部节点 + + http://blog.csdn.net/xubo245/article/details/53056482 + diff --git "a/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2409--Auuxio(Tachyon)\347\232\204\344\274\230\347\202\271.md" "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2409--Auuxio(Tachyon)\347\232\204\344\274\230\347\202\271.md" new file mode 100644 index 0000000..29ff96f --- /dev/null +++ "b/docs/Spark/AlluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2409--Auuxio(Tachyon)\347\232\204\344\274\230\347\202\271.md" @@ -0,0 +1,15 @@ + +1.Spark的不同Job之间，两个不用的application需要从HDFS中加载两次同样的数据，而使用tachyon就不用了： + 当两个Spark作业需要共享数据时，必须通过写磁盘操作。比如：作业1要先把生成的数据写入HDFS，然后作业2再从HDFS把数据读出来。在此，磁盘的读写可能造成性能瓶颈。 + 另外不同的应用框架也可以利用tachyon在内存中共享数据，比如Spark和Hadoop + +2. 由于Spark会利用自身的JVM对数据进行缓存，当Spark程序崩溃时，JVM进程退出，所缓存数据也随之丢失，因此在工作重启时又需要从HDFS把数据再次读出。 + 如果利用Tachyon，相当于在disk和任务间加一层，就算JVM或者及其crash掉了，重启时可以在tachyon读取 + 不同进程间数据的共享 + +3. 当两个Spark作业需操作相同的数据时，每个作业的JVM都需要缓存一份数据，不但造成资源浪费，也极易引发频繁的垃圾收集，造成性能的降低。 + =》不同job缓存同一份数据会增加内存开销，而是用tachyon就可以节省资源 + + +参考： +【1】http://www.wtoutiao.com/p/hf6YDg.html diff --git a/docs/Spark/GraphXLearning/README.md b/docs/Spark/GraphXLearning/README.md new file mode 100644 index 0000000..dd06253 --- /dev/null +++ b/docs/Spark/GraphXLearning/README.md @@ -0,0 +1,19 @@ +## Spark组件之GraphX学习篇 ## +Spark组件之GraphX学习1--入门实例Property Graph：http://blog.csdn.net/xubo245/article/details/51306975 +Spark组件之GraphX学习2--triplets实践：http://blog.csdn.net/xubo245/article/details/51307037 +Spark组件之GraphX学习3--Structural Operators：subgraph：http://blog.csdn.net/xubo245/article/details/51307162 +Spark组件之GraphX学习4--Structural Operators：mask：http://blog.csdn.net/xubo245/article/details/51307237 +Spark组件之GraphX学习5--随机图生成和消息发送aggregateMessages以及mapreduce操作（含源码分析）：http://blog.csdn.net/xubo245/article/details/51307386 +Spark组件之GraphX学习6--随机图生成和出度入度等信息显示：http://blog.csdn.net/xubo245/article/details/51307641 +Spark组件之GraphX学习7--随机图生成和reduce最大或最小出度/入度/度：http://blog.csdn.net/xubo245/article/details/51307774 +Spark组件之GraphX学习8--随机图生成和TopK最大入度：http://blog.csdn.net/xubo245/article/details/51308278 +Spark组件之GraphX学习8--邻居集合：http://blog.csdn.net/xubo245/article/details/51308337 +Spark组件之GraphX学习9--使用pregel函数求单源最短路径：http://blog.csdn.net/xubo245/article/details/51314928 +Spark组件之GraphX学习10--PageRank学习和使用（From examples）：http://blog.csdn.net/xubo245/article/details/51315240 +Spark组件之GraphX学习11--PageRank例子（PageRankAboutBerkeleyWiki）：http://blog.csdn.net/xubo245/article/details/51316151 +Spark组件之GraphX学习12--GraphX常见操作汇总SimpleGraphX：http://blog.csdn.net/xubo245/article/details/51316317 +Spark组件之GraphX学习13--ConnectedComponents操作：http://blog.csdn.net/xubo245/article/details/51316654 +Spark组件之GraphX学习14--TriangleCount实例和分析：http://blog.csdn.net/xubo245/article/details/51317245 +Spark组件之GraphX学习15--we-Google.txt大图分析：http://blog.csdn.net/xubo245/article/details/51317594 +Spark组件之GraphX学习16--最短路径ShortestPaths：http://blog.csdn.net/xubo245/article/details/51317892 +Spark组件之GraphX学习20--待学习部分：http://blog.csdn.net/xubo245/article/details/51317710 \ No newline at end of file diff --git "a/docs/Spark ML\345\255\246\344\271\240/Spark\344\270\255ML\347\232\204\345\255\246\344\271\240100\344\271\213\346\250\241\347\211\210.md" "b/docs/Spark/MLLearning/Spark\344\270\255ML\347\232\204\345\255\246\344\271\240100\344\271\213\346\250\241\347\211\210.md" similarity index 100% rename from "docs/Spark ML\345\255\246\344\271\240/Spark\344\270\255ML\347\232\204\345\255\246\344\271\240100\344\271\213\346\250\241\347\211\210.md" rename to "docs/Spark/MLLearning/Spark\344\270\255ML\347\232\204\345\255\246\344\271\240100\344\271\213\346\250\241\347\211\210.md" diff --git "a/docs/Spark ML\345\255\246\344\271\240/Spark\344\270\255ML\347\232\204\345\255\246\344\271\2401\344\271\213Estimator, Transformer, and Param.md" "b/docs/Spark/MLLearning/Spark\344\270\255ML\347\232\204\345\255\246\344\271\2401\344\271\213Estimator, Transformer, and Param.md" similarity index 100% rename from "docs/Spark ML\345\255\246\344\271\240/Spark\344\270\255ML\347\232\204\345\255\246\344\271\2401\344\271\213Estimator, Transformer, and Param.md" rename to "docs/Spark/MLLearning/Spark\344\270\255ML\347\232\204\345\255\246\344\271\2401\344\271\213Estimator, Transformer, and Param.md" diff --git "a/docs/Spark ML\345\255\246\344\271\240/Spark\344\270\255ML\347\232\204\345\255\246\344\271\2402\344\271\213pipeline.md" "b/docs/Spark/MLLearning/Spark\344\270\255ML\347\232\204\345\255\246\344\271\2402\344\271\213pipeline.md" similarity index 100% rename from "docs/Spark ML\345\255\246\344\271\240/Spark\344\270\255ML\347\232\204\345\255\246\344\271\2402\344\271\213pipeline.md" rename to "docs/Spark/MLLearning/Spark\344\270\255ML\347\232\204\345\255\246\344\271\2402\344\271\213pipeline.md" diff --git "a/docs/Spark ML\345\255\246\344\271\240/Spark\344\270\255ML\347\232\204\345\255\246\344\271\2403\344\271\213model selection via cross-validation.md" "b/docs/Spark/MLLearning/Spark\344\270\255ML\347\232\204\345\255\246\344\271\2403\344\271\213model selection via cross-validation.md" similarity index 100% rename from "docs/Spark ML\345\255\246\344\271\240/Spark\344\270\255ML\347\232\204\345\255\246\344\271\2403\344\271\213model selection via cross-validation.md" rename to "docs/Spark/MLLearning/Spark\344\270\255ML\347\232\204\345\255\246\344\271\2403\344\271\213model selection via cross-validation.md" diff --git "a/docs/Spark ML\345\255\246\344\271\240/Spark\344\270\255ML\347\232\204\345\255\246\344\271\2404\344\271\213model selection via train validation split.md" "b/docs/Spark/MLLearning/Spark\344\270\255ML\347\232\204\345\255\246\344\271\2404\344\271\213model selection via train validation split.md" similarity index 100% rename from "docs/Spark ML\345\255\246\344\271\240/Spark\344\270\255ML\347\232\204\345\255\246\344\271\2404\344\271\213model selection via train validation split.md" rename to "docs/Spark/MLLearning/Spark\344\270\255ML\347\232\204\345\255\246\344\271\2404\344\271\213model selection via train validation split.md" diff --git "a/docs/Spark ML\345\255\246\344\271\240/\345\246\202\344\275\225\350\207\252\345\212\250\347\224\237\346\210\220\347\233\256\345\275\225\346\240\221\345\222\214\346\257\217\347\257\207\346\226\207\347\253\240\347\232\204\351\223\276\346\216\245.md" "b/docs/Spark/MLLearning/\345\246\202\344\275\225\350\207\252\345\212\250\347\224\237\346\210\220\347\233\256\345\275\225\346\240\221\345\222\214\346\257\217\347\257\207\346\226\207\347\253\240\347\232\204\351\223\276\346\216\245.md" similarity index 100% rename from "docs/Spark ML\345\255\246\344\271\240/\345\246\202\344\275\225\350\207\252\345\212\250\347\224\237\346\210\220\347\233\256\345\275\225\346\240\221\345\222\214\346\257\217\347\257\207\346\226\207\347\253\240\347\232\204\351\223\276\346\216\245.md" rename to "docs/Spark/MLLearning/\345\246\202\344\275\225\350\207\252\345\212\250\347\224\237\346\210\220\347\233\256\345\275\225\346\240\221\345\222\214\346\257\217\347\257\207\346\226\207\347\253\240\347\232\204\351\223\276\346\216\245.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/10PMML\346\250\241\345\236\213\350\276\223\345\207\272/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24074\344\271\213\351\242\204\350\250\200\346\250\241\345\236\213\346\240\207\350\256\260\350\257\255\350\250\200PMML.md" "b/docs/Spark/MLlibLearning/10PMML\346\250\241\345\236\213\350\276\223\345\207\272/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24074\344\271\213\351\242\204\350\250\200\346\250\241\345\236\213\346\240\207\350\256\260\350\257\255\350\250\200PMML.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/10PMML\346\250\241\345\236\213\350\276\223\345\207\272/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24074\344\271\213\351\242\204\350\250\200\346\250\241\345\236\213\346\240\207\350\256\260\350\257\255\350\250\200PMML.md" rename to "docs/Spark/MLlibLearning/10PMML\346\250\241\345\236\213\350\276\223\345\207\272/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24074\344\271\213\351\242\204\350\250\200\346\250\241\345\236\213\346\240\207\350\256\260\350\257\255\350\250\200PMML.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/11\344\274\230\345\214\226/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24075\344\271\213L-BFGS.md" "b/docs/Spark/MLlibLearning/11\344\274\230\345\214\226/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24075\344\271\213L-BFGS.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/11\344\274\230\345\214\226/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24075\344\271\213L-BFGS.md" rename to "docs/Spark/MLlibLearning/11\344\274\230\345\214\226/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24075\344\271\213L-BFGS.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24012\344\271\213\345\257\206\351\233\206\345\220\221\351\207\217\345\222\214\347\250\200\347\226\217\345\220\221\351\207\217\347\232\204\347\224\237\346\210\220.md" "b/docs/Spark/MLlibLearning/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24012\344\271\213\345\257\206\351\233\206\345\220\221\351\207\217\345\222\214\347\250\200\347\226\217\345\220\221\351\207\217\347\232\204\347\224\237\346\210\220.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24012\344\271\213\345\257\206\351\233\206\345\220\221\351\207\217\345\222\214\347\250\200\347\226\217\345\220\221\351\207\217\347\232\204\347\224\237\346\210\220.md" rename to "docs/Spark/MLlibLearning/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24012\344\271\213\345\257\206\351\233\206\345\220\221\351\207\217\345\222\214\347\250\200\347\226\217\345\220\221\351\207\217\347\232\204\347\224\237\346\210\220.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24013\344\271\213\347\273\231\345\220\221\351\207\217\346\211\223\346\240\207\347\255\276.md" "b/docs/Spark/MLlibLearning/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24013\344\271\213\347\273\231\345\220\221\351\207\217\346\211\223\346\240\207\347\255\276.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24013\344\271\213\347\273\231\345\220\221\351\207\217\346\211\223\346\240\207\347\255\276.md" rename to "docs/Spark/MLlibLearning/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24013\344\271\213\347\273\231\345\220\221\351\207\217\346\211\223\346\240\207\347\255\276.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24014\344\271\213\344\273\216\346\226\207\346\234\254\344\270\255\350\257\273\345\217\226\345\270\246\346\240\207\347\255\276\347\232\204\346\225\260\346\215\256\357\274\214\347\224\237\346\210\220\345\270\246label\347\232\204\345\220\221\351\207\217.md" "b/docs/Spark/MLlibLearning/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24014\344\271\213\344\273\216\346\226\207\346\234\254\344\270\255\350\257\273\345\217\226\345\270\246\346\240\207\347\255\276\347\232\204\346\225\260\346\215\256\357\274\214\347\224\237\346\210\220\345\270\246label\347\232\204\345\220\221\351\207\217.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24014\344\271\213\344\273\216\346\226\207\346\234\254\344\270\255\350\257\273\345\217\226\345\270\246\346\240\207\347\255\276\347\232\204\346\225\260\346\215\256\357\274\214\347\224\237\346\210\220\345\270\246label\347\232\204\345\220\221\351\207\217.md" rename to "docs/Spark/MLlibLearning/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24014\344\271\213\344\273\216\346\226\207\346\234\254\344\270\255\350\257\273\345\217\226\345\270\246\346\240\207\347\255\276\347\232\204\346\225\260\346\215\256\357\274\214\347\224\237\346\210\220\345\270\246label\347\232\204\345\220\221\351\207\217.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24015\344\271\213\345\210\233\345\273\272\345\210\206\345\270\203\345\274\217\347\237\251\351\230\265.md" "b/docs/Spark/MLlibLearning/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24015\344\271\213\345\210\233\345\273\272\345\210\206\345\270\203\345\274\217\347\237\251\351\230\265.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24015\344\271\213\345\210\233\345\273\272\345\210\206\345\270\203\345\274\217\347\237\251\351\230\265.md" rename to "docs/Spark/MLlibLearning/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24015\344\271\213\345\210\233\345\273\272\345\210\206\345\270\203\345\274\217\347\237\251\351\230\265.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24016\344\271\213\345\210\206\345\270\203\345\274\217\350\241\214\347\237\251\351\230\265\347\232\204\345\233\233\347\247\215\345\275\242\345\274\217.md" "b/docs/Spark/MLlibLearning/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24016\344\271\213\345\210\206\345\270\203\345\274\217\350\241\214\347\237\251\351\230\265\347\232\204\345\233\233\347\247\215\345\275\242\345\274\217.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24016\344\271\213\345\210\206\345\270\203\345\274\217\350\241\214\347\237\251\351\230\265\347\232\204\345\233\233\347\247\215\345\275\242\345\274\217.md" rename to "docs/Spark/MLlibLearning/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24016\344\271\213\345\210\206\345\270\203\345\274\217\350\241\214\347\237\251\351\230\265\347\232\204\345\233\233\347\247\215\345\275\242\345\274\217.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2403\344\271\213\347\224\250\346\210\267\347\233\270\344\274\274\345\272\246\350\256\241\347\256\227.md" "b/docs/Spark/MLlibLearning/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2403\344\271\213\347\224\250\346\210\267\347\233\270\344\274\274\345\272\246\350\256\241\347\256\227.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2403\344\271\213\347\224\250\346\210\267\347\233\270\344\274\274\345\272\246\350\256\241\347\256\227.md" rename to "docs/Spark/MLlibLearning/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2403\344\271\213\347\224\250\346\210\267\347\233\270\344\274\274\345\272\246\350\256\241\347\256\227.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24043\344\271\213BlockMatrix.md" "b/docs/Spark/MLlibLearning/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24043\344\271\213BlockMatrix.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24043\344\271\213BlockMatrix.md" rename to "docs/Spark/MLlibLearning/1\346\225\260\346\215\256\347\261\273\345\236\213/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24043\344\271\213BlockMatrix.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24017\344\271\213colStats_\344\273\245\345\210\227\344\270\272\345\237\272\347\241\200\350\256\241\347\256\227\347\273\237\350\256\241\351\207\217\347\232\204\345\237\272\346\234\254\346\225\260\346\215\256.md" "b/docs/Spark/MLlibLearning/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24017\344\271\213colStats_\344\273\245\345\210\227\344\270\272\345\237\272\347\241\200\350\256\241\347\256\227\347\273\237\350\256\241\351\207\217\347\232\204\345\237\272\346\234\254\346\225\260\346\215\256.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24017\344\271\213colStats_\344\273\245\345\210\227\344\270\272\345\237\272\347\241\200\350\256\241\347\256\227\347\273\237\350\256\241\351\207\217\347\232\204\345\237\272\346\234\254\346\225\260\346\215\256.md" rename to "docs/Spark/MLlibLearning/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24017\344\271\213colStats_\344\273\245\345\210\227\344\270\272\345\237\272\347\241\200\350\256\241\347\256\227\347\273\237\350\256\241\351\207\217\347\232\204\345\237\272\346\234\254\346\225\260\346\215\256.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24018\344\271\213corr_\344\270\244\347\273\204\346\225\260\346\215\256\347\233\270\345\205\263\345\205\263\347\263\273\350\256\241\347\256\227\357\274\210Pearson\343\200\201Spearman\357\274\211.md" "b/docs/Spark/MLlibLearning/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24018\344\271\213corr_\344\270\244\347\273\204\346\225\260\346\215\256\347\233\270\345\205\263\345\205\263\347\263\273\350\256\241\347\256\227\357\274\210Pearson\343\200\201Spearman\357\274\211.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24018\344\271\213corr_\344\270\244\347\273\204\346\225\260\346\215\256\347\233\270\345\205\263\345\205\263\347\263\273\350\256\241\347\256\227\357\274\210Pearson\343\200\201Spearman\357\274\211.md" rename to "docs/Spark/MLlibLearning/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24018\344\271\213corr_\344\270\244\347\273\204\346\225\260\346\215\256\347\233\270\345\205\263\345\205\263\347\263\273\350\256\241\347\256\227\357\274\210Pearson\343\200\201Spearman\357\274\211.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24019\344\271\213\345\210\206\345\261\202\346\212\275\346\240\267.md" "b/docs/Spark/MLlibLearning/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24019\344\271\213\345\210\206\345\261\202\346\212\275\346\240\267.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24019\344\271\213\345\210\206\345\261\202\346\212\275\346\240\267.md" rename to "docs/Spark/MLlibLearning/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24019\344\271\213\345\210\206\345\261\202\346\212\275\346\240\267.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24020\344\271\213\345\201\207\350\256\276\346\243\200\351\252\214-\345\215\241\346\226\271\346\243\200\351\252\214.md" "b/docs/Spark/MLlibLearning/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24020\344\271\213\345\201\207\350\256\276\346\243\200\351\252\214-\345\215\241\346\226\271\346\243\200\351\252\214.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24020\344\271\213\345\201\207\350\256\276\346\243\200\351\252\214-\345\215\241\346\226\271\346\243\200\351\252\214.md" rename to "docs/Spark/MLlibLearning/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24020\344\271\213\345\201\207\350\256\276\346\243\200\351\252\214-\345\215\241\346\226\271\346\243\200\351\252\214.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24021\344\271\213\351\232\217\346\234\272\346\225\260-RandomRDD\344\272\247\347\224\237.md" "b/docs/Spark/MLlibLearning/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24021\344\271\213\351\232\217\346\234\272\346\225\260-RandomRDD\344\272\247\347\224\237.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24021\344\271\213\351\232\217\346\234\272\346\225\260-RandomRDD\344\272\247\347\224\237.md" rename to "docs/Spark/MLlibLearning/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24021\344\271\213\351\232\217\346\234\272\346\225\260-RandomRDD\344\272\247\347\224\237.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24022\344\271\213\345\201\207\350\256\276\346\243\200\351\252\214-\345\215\241\346\226\271\346\243\200\351\252\214\346\246\202\345\277\265\347\220\206\350\247\243.md" "b/docs/Spark/MLlibLearning/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24022\344\271\213\345\201\207\350\256\276\346\243\200\351\252\214-\345\215\241\346\226\271\346\243\200\351\252\214\346\246\202\345\277\265\347\220\206\350\247\243.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24022\344\271\213\345\201\207\350\256\276\346\243\200\351\252\214-\345\215\241\346\226\271\346\243\200\351\252\214\346\246\202\345\277\265\347\220\206\350\247\243.md" rename to "docs/Spark/MLlibLearning/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24022\344\271\213\345\201\207\350\256\276\346\243\200\351\252\214-\345\215\241\346\226\271\346\243\200\351\252\214\346\246\202\345\277\265\347\220\206\350\247\243.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2403\344\271\213\347\224\250\346\210\267\347\233\270\344\274\274\345\272\246\350\256\241\347\256\227.md" "b/docs/Spark/MLlibLearning/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2403\344\271\213\347\224\250\346\210\267\347\233\270\344\274\274\345\272\246\350\256\241\347\256\227.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2403\344\271\213\347\224\250\346\210\267\347\233\270\344\274\274\345\272\246\350\256\241\347\256\227.md" rename to "docs/Spark/MLlibLearning/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2403\344\271\213\347\224\250\346\210\267\347\233\270\344\274\274\345\272\246\350\256\241\347\256\227.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24042\344\271\213rowMatrix\347\232\204QR\345\210\206\350\247\243.md" "b/docs/Spark/MLlibLearning/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24042\344\271\213rowMatrix\347\232\204QR\345\210\206\350\247\243.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24042\344\271\213rowMatrix\347\232\204QR\345\210\206\350\247\243.md" rename to "docs/Spark/MLlibLearning/2\345\237\272\346\234\254\347\273\237\350\256\241/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24042\344\271\213rowMatrix\347\232\204QR\345\210\206\350\247\243.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24023\344\271\213\351\232\217\346\234\272\346\242\257\345\272\246\344\270\213\351\231\215\357\274\210SGD\357\274\211.md" "b/docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24023\344\271\213\351\232\217\346\234\272\346\242\257\345\272\246\344\270\213\351\231\215\357\274\210SGD\357\274\211.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24023\344\271\213\351\232\217\346\234\272\346\242\257\345\272\246\344\270\213\351\231\215\357\274\210SGD\357\274\211.md" rename to "docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24023\344\271\213\351\232\217\346\234\272\346\242\257\345\272\246\344\270\213\351\231\215\357\274\210SGD\357\274\211.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24024\344\271\213\347\272\277\346\200\247\345\233\236\345\275\2221-\345\260\217\346\225\260\346\215\256\351\233\206.md" "b/docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24024\344\271\213\347\272\277\346\200\247\345\233\236\345\275\2221-\345\260\217\346\225\260\346\215\256\351\233\206.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24024\344\271\213\347\272\277\346\200\247\345\233\236\345\275\2221-\345\260\217\346\225\260\346\215\256\351\233\206.md" rename to "docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24024\344\271\213\347\272\277\346\200\247\345\233\236\345\275\2221-\345\260\217\346\225\260\346\215\256\351\233\206.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24025\344\271\213\347\272\277\346\200\247\345\233\236\345\275\2222-\350\276\203\345\244\247\346\225\260\346\215\256\351\233\206\357\274\210\345\244\232\345\205\203\357\274\211.md" "b/docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24025\344\271\213\347\272\277\346\200\247\345\233\236\345\275\2222-\350\276\203\345\244\247\346\225\260\346\215\256\351\233\206\357\274\210\345\244\232\345\205\203\357\274\211.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24025\344\271\213\347\272\277\346\200\247\345\233\236\345\275\2222-\350\276\203\345\244\247\346\225\260\346\215\256\351\233\206\357\274\210\345\244\232\345\205\203\357\274\211.md" rename to "docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24025\344\271\213\347\272\277\346\200\247\345\233\236\345\275\2222-\350\276\203\345\244\247\346\225\260\346\215\256\351\233\206\357\274\210\345\244\232\345\205\203\357\274\211.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24026\344\271\213\351\200\273\350\276\221\345\233\236\345\275\222-\347\256\200\345\215\225\346\225\260\346\215\256\351\233\206\357\274\214\345\270\246\351\242\204\346\265\213.md" "b/docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24026\344\271\213\351\200\273\350\276\221\345\233\236\345\275\222-\347\256\200\345\215\225\346\225\260\346\215\256\351\233\206\357\274\214\345\270\246\351\242\204\346\265\213.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24026\344\271\213\351\200\273\350\276\221\345\233\236\345\275\222-\347\256\200\345\215\225\346\225\260\346\215\256\351\233\206\357\274\214\345\270\246\351\242\204\346\265\213.md" rename to "docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24026\344\271\213\351\200\273\350\276\221\345\233\236\345\275\222-\347\256\200\345\215\225\346\225\260\346\215\256\351\233\206\357\274\214\345\270\246\351\242\204\346\265\213.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24027\344\271\213\351\200\273\350\276\221\345\233\236\345\275\222-\345\244\232\345\205\203\351\200\273\350\276\221\345\233\236\345\275\222\357\274\214\350\276\203\345\244\247\346\225\260\346\215\256\351\233\206\357\274\214\345\270\246\351\242\204\346\265\213\345\207\206\347\241\256\345\272\246\350\256\241\347\256\227.md" "b/docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24027\344\271\213\351\200\273\350\276\221\345\233\236\345\275\222-\345\244\232\345\205\203\351\200\273\350\276\221\345\233\236\345\275\222\357\274\214\350\276\203\345\244\247\346\225\260\346\215\256\351\233\206\357\274\214\345\270\246\351\242\204\346\265\213\345\207\206\347\241\256\345\272\246\350\256\241\347\256\227.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24027\344\271\213\351\200\273\350\276\221\345\233\236\345\275\222-\345\244\232\345\205\203\351\200\273\350\276\221\345\233\236\345\275\222\357\274\214\350\276\203\345\244\247\346\225\260\346\215\256\351\233\206\357\274\214\345\270\246\351\242\204\346\265\213\345\207\206\347\241\256\345\272\246\350\256\241\347\256\227.md" rename to "docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24027\344\271\213\351\200\273\350\276\221\345\233\236\345\275\222-\345\244\232\345\205\203\351\200\273\350\276\221\345\233\236\345\275\222\357\274\214\350\276\203\345\244\247\346\225\260\346\215\256\351\233\206\357\274\214\345\270\246\351\242\204\346\265\213\345\207\206\347\241\256\345\272\246\350\256\241\347\256\227.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24028\344\271\213\346\224\257\346\214\201\345\220\221\351\207\217\346\234\272SVM-\346\226\271\346\263\2251.md" "b/docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24028\344\271\213\346\224\257\346\214\201\345\220\221\351\207\217\346\234\272SVM-\346\226\271\346\263\2251.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24028\344\271\213\346\224\257\346\214\201\345\220\221\351\207\217\346\234\272SVM-\346\226\271\346\263\2251.md" rename to "docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24028\344\271\213\346\224\257\346\214\201\345\220\221\351\207\217\346\234\272SVM-\346\226\271\346\263\2251.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24029\344\271\213\346\224\257\346\214\201\345\220\221\351\207\217\346\234\272SVM-\346\226\271\346\263\2252.md" "b/docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24029\344\271\213\346\224\257\346\214\201\345\220\221\351\207\217\346\234\272SVM-\346\226\271\346\263\2252.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24029\344\271\213\346\224\257\346\214\201\345\220\221\351\207\217\346\234\272SVM-\346\226\271\346\263\2252.md" rename to "docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24029\344\271\213\346\224\257\346\214\201\345\220\221\351\207\217\346\234\272SVM-\346\226\271\346\263\2252.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24030\344\271\213\351\200\273\350\276\221\345\233\236\345\275\222LogisticRegressionWithLBFGS.md" "b/docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24030\344\271\213\351\200\273\350\276\221\345\233\236\345\275\222LogisticRegressionWithLBFGS.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24030\344\271\213\351\200\273\350\276\221\345\233\236\345\275\222LogisticRegressionWithLBFGS.md" rename to "docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24030\344\271\213\351\200\273\350\276\221\345\233\236\345\275\222LogisticRegressionWithLBFGS.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24031\344\271\213\346\234\264\347\264\240\350\264\235\345\217\266\346\226\257\345\210\206\347\261\273\345\231\250\357\274\210\345\244\232\351\241\271\345\274\217\346\234\264\347\264\240\350\264\235\345\217\266\346\226\257\357\274\211.md" "b/docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24031\344\271\213\346\234\264\347\264\240\350\264\235\345\217\266\346\226\257\345\210\206\347\261\273\345\231\250\357\274\210\345\244\232\351\241\271\345\274\217\346\234\264\347\264\240\350\264\235\345\217\266\346\226\257\357\274\211.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24031\344\271\213\346\234\264\347\264\240\350\264\235\345\217\266\346\226\257\345\210\206\347\261\273\345\231\250\357\274\210\345\244\232\351\241\271\345\274\217\346\234\264\347\264\240\350\264\235\345\217\266\346\226\257\357\274\211.md" rename to "docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24031\344\271\213\346\234\264\347\264\240\350\264\235\345\217\266\346\226\257\345\210\206\347\261\273\345\231\250\357\274\210\345\244\232\351\241\271\345\274\217\346\234\264\347\264\240\350\264\235\345\217\266\346\226\257\357\274\211.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24032\344\271\213\346\234\264\347\264\240\350\264\235\345\217\266\346\226\257\345\210\206\347\261\273\345\231\250\357\274\210\344\274\257\345\212\252\345\210\251\346\234\264\347\264\240\350\264\235\345\217\266\346\226\257\357\274\211_.md" "b/docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24032\344\271\213\346\234\264\347\264\240\350\264\235\345\217\266\346\226\257\345\210\206\347\261\273\345\231\250\357\274\210\344\274\257\345\212\252\345\210\251\346\234\264\347\264\240\350\264\235\345\217\266\346\226\257\357\274\211_.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24032\344\271\213\346\234\264\347\264\240\350\264\235\345\217\266\346\226\257\345\210\206\347\261\273\345\231\250\357\274\210\344\274\257\345\212\252\345\210\251\346\234\264\347\264\240\350\264\235\345\217\266\346\226\257\357\274\211_.md" rename to "docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24032\344\271\213\346\234\264\347\264\240\350\264\235\345\217\266\346\226\257\345\210\206\347\261\273\345\231\250\357\274\210\344\274\257\345\212\252\345\210\251\346\234\264\347\264\240\350\264\235\345\217\266\346\226\257\357\274\211_.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24033\344\271\213\345\206\263\347\255\226\346\240\221\357\274\210\344\275\277\347\224\250Gini\357\274\211.md" "b/docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24033\344\271\213\345\206\263\347\255\226\346\240\221\357\274\210\344\275\277\347\224\250Gini\357\274\211.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24033\344\271\213\345\206\263\347\255\226\346\240\221\357\274\210\344\275\277\347\224\250Gini\357\274\211.md" rename to "docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24033\344\271\213\345\206\263\347\255\226\346\240\221\357\274\210\344\275\277\347\224\250Gini\357\274\211.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24034\344\271\213\345\206\263\347\255\226\346\240\221\357\274\210\344\275\277\347\224\250entropy\357\274\211_.md" "b/docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24034\344\271\213\345\206\263\347\255\226\346\240\221\357\274\210\344\275\277\347\224\250entropy\357\274\211_.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24034\344\271\213\345\206\263\347\255\226\346\240\221\357\274\210\344\275\277\347\224\250entropy\357\274\211_.md" rename to "docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24034\344\271\213\345\206\263\347\255\226\346\240\221\357\274\210\344\275\277\347\224\250entropy\357\274\211_.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24035\344\271\213\351\232\217\346\234\272\346\243\256\346\236\227\357\274\210entropy\357\274\211\350\277\233\350\241\214\345\210\206\347\261\273.md" "b/docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24035\344\271\213\351\232\217\346\234\272\346\243\256\346\236\227\357\274\210entropy\357\274\211\350\277\233\350\241\214\345\210\206\347\261\273.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24035\344\271\213\351\232\217\346\234\272\346\243\256\346\236\227\357\274\210entropy\357\274\211\350\277\233\350\241\214\345\210\206\347\261\273.md" rename to "docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24035\344\271\213\351\232\217\346\234\272\346\243\256\346\236\227\357\274\210entropy\357\274\211\350\277\233\350\241\214\345\210\206\347\261\273.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24036\344\271\213\345\206\263\347\255\226\346\240\221\357\274\210\344\275\277\347\224\250variance\357\274\211\350\277\233\350\241\214\345\233\236\345\275\222.md" "b/docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24036\344\271\213\345\206\263\347\255\226\346\240\221\357\274\210\344\275\277\347\224\250variance\357\274\211\350\277\233\350\241\214\345\233\236\345\275\222.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24036\344\271\213\345\206\263\347\255\226\346\240\221\357\274\210\344\275\277\347\224\250variance\357\274\211\350\277\233\350\241\214\345\233\236\345\275\222.md" rename to "docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24036\344\271\213\345\206\263\347\255\226\346\240\221\357\274\210\344\275\277\347\224\250variance\357\274\211\350\277\233\350\241\214\345\233\236\345\275\222.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24037\344\271\213\351\232\217\346\234\272\346\243\256\346\236\227\357\274\210Gini\357\274\211\350\277\233\350\241\214\345\210\206\347\261\273.md" "b/docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24037\344\271\213\351\232\217\346\234\272\346\243\256\346\236\227\357\274\210Gini\357\274\211\350\277\233\350\241\214\345\210\206\347\261\273.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24037\344\271\213\351\232\217\346\234\272\346\243\256\346\236\227\357\274\210Gini\357\274\211\350\277\233\350\241\214\345\210\206\347\261\273.md" rename to "docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24037\344\271\213\351\232\217\346\234\272\346\243\256\346\236\227\357\274\210Gini\357\274\211\350\277\233\350\241\214\345\210\206\347\261\273.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24038\344\271\213\351\232\217\346\234\272\346\243\256\346\236\227\357\274\210\344\275\277\347\224\250variance\357\274\211\350\277\233\350\241\214\345\233\236\345\275\222.md" "b/docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24038\344\271\213\351\232\217\346\234\272\346\243\256\346\236\227\357\274\210\344\275\277\347\224\250variance\357\274\211\350\277\233\350\241\214\345\233\236\345\275\222.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24038\344\271\213\351\232\217\346\234\272\346\243\256\346\236\227\357\274\210\344\275\277\347\224\250variance\357\274\211\350\277\233\350\241\214\345\233\236\345\275\222.md" rename to "docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24038\344\271\213\351\232\217\346\234\272\346\243\256\346\236\227\357\274\210\344\275\277\347\224\250variance\357\274\211\350\277\233\350\241\214\345\233\236\345\275\222.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24039\344\271\213\346\242\257\345\272\246\346\217\220\345\215\207\346\240\221\357\274\210GBT\357\274\211\347\224\250\344\272\216\345\210\206\347\261\273_.md" "b/docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24039\344\271\213\346\242\257\345\272\246\346\217\220\345\215\207\346\240\221\357\274\210GBT\357\274\211\347\224\250\344\272\216\345\210\206\347\261\273_.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24039\344\271\213\346\242\257\345\272\246\346\217\220\345\215\207\346\240\221\357\274\210GBT\357\274\211\347\224\250\344\272\216\345\210\206\347\261\273_.md" rename to "docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24039\344\271\213\346\242\257\345\272\246\346\217\220\345\215\207\346\240\221\357\274\210GBT\357\274\211\347\224\250\344\272\216\345\210\206\347\261\273_.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24040\344\271\213\346\242\257\345\272\246\346\217\220\345\215\207\346\240\221\357\274\210GBT\357\274\211\347\224\250\344\272\216\345\233\236\345\275\222_.md" "b/docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24040\344\271\213\346\242\257\345\272\246\346\217\220\345\215\207\346\240\221\357\274\210GBT\357\274\211\347\224\250\344\272\216\345\233\236\345\275\222_.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24040\344\271\213\346\242\257\345\272\246\346\217\220\345\215\207\346\240\221\357\274\210GBT\357\274\211\347\224\250\344\272\216\345\233\236\345\275\222_.md" rename to "docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24040\344\271\213\346\242\257\345\272\246\346\217\220\345\215\207\346\240\221\357\274\210GBT\357\274\211\347\224\250\344\272\216\345\233\236\345\275\222_.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24041\344\271\213\344\277\235\345\272\217\345\233\236\345\275\222\357\274\210Isotonic regression\357\274\211.md" "b/docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24041\344\271\213\344\277\235\345\272\217\345\233\236\345\275\222\357\274\210Isotonic regression\357\274\211.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24041\344\271\213\344\277\235\345\272\217\345\233\236\345\275\222\357\274\210Isotonic regression\357\274\211.md" rename to "docs/Spark/MLlibLearning/3\345\210\206\347\261\273\345\222\214\345\233\236\345\275\222/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24041\344\271\213\344\277\235\345\272\217\345\233\236\345\275\222\357\274\210Isotonic regression\357\274\211.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24010\344\271\213\344\277\256\346\224\271MovieLens\346\235\245\345\257\271movieLen\344\270\255\347\232\204100k\346\225\260\346\215\256\350\277\233\350\241\214\351\242\204\346\265\213.md" "b/docs/Spark/MLlibLearning/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24010\344\271\213\344\277\256\346\224\271MovieLens\346\235\245\345\257\271movieLen\344\270\255\347\232\204100k\346\225\260\346\215\256\350\277\233\350\241\214\351\242\204\346\265\213.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24010\344\271\213\344\277\256\346\224\271MovieLens\346\235\245\345\257\271movieLen\344\270\255\347\232\204100k\346\225\260\346\215\256\350\277\233\350\241\214\351\242\204\346\265\213.md" rename to "docs/Spark/MLlibLearning/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24010\344\271\213\344\277\256\346\224\271MovieLens\346\235\245\345\257\271movieLen\344\270\255\347\232\204100k\346\225\260\346\215\256\350\277\233\350\241\214\351\242\204\346\265\213.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24011\344\271\213\344\275\277\347\224\250ALS\345\257\271movieLens\344\270\255\344\270\200\347\231\276\344\270\207\346\235\241\357\274\2101M\357\274\211\346\225\260\346\215\256\351\233\206\350\277\233\350\241\214\350\256\255\347\273\203\357\274\214\345\271\266\345\257\271\350\276\223\345\205\245\347\232\204\346\226\260\347\224\250\346\210\267\346\225\260\346\215\256\350\277\233\350\241\214\347\224\265\345\275\261\346\216\250\350\215\220.md" "b/docs/Spark/MLlibLearning/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24011\344\271\213\344\275\277\347\224\250ALS\345\257\271movieLens\344\270\255\344\270\200\347\231\276\344\270\207\346\235\241\357\274\2101M\357\274\211\346\225\260\346\215\256\351\233\206\350\277\233\350\241\214\350\256\255\347\273\203\357\274\214\345\271\266\345\257\271\350\276\223\345\205\245\347\232\204\346\226\260\347\224\250\346\210\267\346\225\260\346\215\256\350\277\233\350\241\214\347\224\265\345\275\261\346\216\250\350\215\220.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24011\344\271\213\344\275\277\347\224\250ALS\345\257\271movieLens\344\270\255\344\270\200\347\231\276\344\270\207\346\235\241\357\274\2101M\357\274\211\346\225\260\346\215\256\351\233\206\350\277\233\350\241\214\350\256\255\347\273\203\357\274\214\345\271\266\345\257\271\350\276\223\345\205\245\347\232\204\346\226\260\347\224\250\346\210\267\346\225\260\346\215\256\350\277\233\350\241\214\347\224\265\345\275\261\346\216\250\350\215\220.md" rename to "docs/Spark/MLlibLearning/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24011\344\271\213\344\275\277\347\224\250ALS\345\257\271movieLens\344\270\255\344\270\200\347\231\276\344\270\207\346\235\241\357\274\2101M\357\274\211\346\225\260\346\215\256\351\233\206\350\277\233\350\241\214\350\256\255\347\273\203\357\274\214\345\271\266\345\257\271\350\276\223\345\205\245\347\232\204\346\226\260\347\224\250\346\210\267\346\225\260\346\215\256\350\277\233\350\241\214\347\224\265\345\275\261\346\216\250\350\215\220.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2402\344\271\213MovieLensALS\345\255\246\344\271\240\357\274\210\351\233\206\347\276\244run-eaxmples\350\277\220\350\241\214\357\274\211.md" "b/docs/Spark/MLlibLearning/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2402\344\271\213MovieLensALS\345\255\246\344\271\240\357\274\210\351\233\206\347\276\244run-eaxmples\350\277\220\350\241\214\357\274\211.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2402\344\271\213MovieLensALS\345\255\246\344\271\240\357\274\210\351\233\206\347\276\244run-eaxmples\350\277\220\350\241\214\357\274\211.md" rename to "docs/Spark/MLlibLearning/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2402\344\271\213MovieLensALS\345\255\246\344\271\240\357\274\210\351\233\206\347\276\244run-eaxmples\350\277\220\350\241\214\357\274\211.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2404\344\271\213examples\344\270\255\347\232\204MovieLensALS\344\277\256\346\224\271\346\234\254\345\234\260\350\277\220\350\241\214.md" "b/docs/Spark/MLlibLearning/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2404\344\271\213examples\344\270\255\347\232\204MovieLensALS\344\277\256\346\224\271\346\234\254\345\234\260\350\277\220\350\241\214.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2404\344\271\213examples\344\270\255\347\232\204MovieLensALS\344\277\256\346\224\271\346\234\254\345\234\260\350\277\220\350\241\214.md" rename to "docs/Spark/MLlibLearning/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2404\344\271\213examples\344\270\255\347\232\204MovieLensALS\344\277\256\346\224\271\346\234\254\345\234\260\350\277\220\350\241\214.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2405\344\271\213ALS\346\265\213\350\257\225\357\274\210apache spark\357\274\211.md" "b/docs/Spark/MLlibLearning/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2405\344\271\213ALS\346\265\213\350\257\225\357\274\210apache spark\357\274\211.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2405\344\271\213ALS\346\265\213\350\257\225\357\274\210apache spark\357\274\211.md" rename to "docs/Spark/MLlibLearning/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2405\344\271\213ALS\346\265\213\350\257\225\357\274\210apache spark\357\274\211.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2406\344\271\213ALS\346\265\213\350\257\225\357\274\210apache spark \345\220\253\351\232\220\345\274\217\350\275\254\346\215\242\357\274\211.md" "b/docs/Spark/MLlibLearning/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2406\344\271\213ALS\346\265\213\350\257\225\357\274\210apache spark \345\220\253\351\232\220\345\274\217\350\275\254\346\215\242\357\274\211.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2406\344\271\213ALS\346\265\213\350\257\225\357\274\210apache spark \345\220\253\351\232\220\345\274\217\350\275\254\346\215\242\357\274\211.md" rename to "docs/Spark/MLlibLearning/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2406\344\271\213ALS\346\265\213\350\257\225\357\274\210apache spark \345\220\253\351\232\220\345\274\217\350\275\254\346\215\242\357\274\211.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2407\344\271\213ALS\351\232\220\345\274\217\350\275\254\346\215\242\350\256\255\347\273\203\347\232\204model\346\235\245\351\242\204\346\265\213\346\225\260\346\215\256.md" "b/docs/Spark/MLlibLearning/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2407\344\271\213ALS\351\232\220\345\274\217\350\275\254\346\215\242\350\256\255\347\273\203\347\232\204model\346\235\245\351\242\204\346\265\213\346\225\260\346\215\256.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2407\344\271\213ALS\351\232\220\345\274\217\350\275\254\346\215\242\350\256\255\347\273\203\347\232\204model\346\235\245\351\242\204\346\265\213\346\225\260\346\215\256.md" rename to "docs/Spark/MLlibLearning/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2407\344\271\213ALS\351\232\220\345\274\217\350\275\254\346\215\242\350\256\255\347\273\203\347\232\204model\346\235\245\351\242\204\346\265\213\346\225\260\346\215\256.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2408\344\271\213ALS\350\256\255\347\273\203\347\232\204model\346\235\245\351\242\204\346\265\213\346\225\260\346\215\256.md" "b/docs/Spark/MLlibLearning/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2408\344\271\213ALS\350\256\255\347\273\203\347\232\204model\346\235\245\351\242\204\346\265\213\346\225\260\346\215\256.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2408\344\271\213ALS\350\256\255\347\273\203\347\232\204model\346\235\245\351\242\204\346\265\213\346\225\260\346\215\256.md" rename to "docs/Spark/MLlibLearning/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2408\344\271\213ALS\350\256\255\347\273\203\347\232\204model\346\235\245\351\242\204\346\265\213\346\225\260\346\215\256.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2409\344\271\213ALS\350\256\255\347\273\203\347\232\204model\346\235\245\351\242\204\346\265\213\346\225\260\346\215\256\347\232\204\345\207\206\347\241\256\347\216\207\347\240\224\347\251\266.md" "b/docs/Spark/MLlibLearning/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2409\344\271\213ALS\350\256\255\347\273\203\347\232\204model\346\235\245\351\242\204\346\265\213\346\225\260\346\215\256\347\232\204\345\207\206\347\241\256\347\216\207\347\240\224\347\251\266.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2409\344\271\213ALS\350\256\255\347\273\203\347\232\204model\346\235\245\351\242\204\346\265\213\346\225\260\346\215\256\347\232\204\345\207\206\347\241\256\347\216\207\347\240\224\347\251\266.md" rename to "docs/Spark/MLlibLearning/4\345\215\217\345\220\214\350\277\207\346\273\244/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2409\344\271\213ALS\350\256\255\347\273\203\347\232\204model\346\235\245\351\242\204\346\265\213\346\225\260\346\215\256\347\232\204\345\207\206\347\241\256\347\216\207\347\240\224\347\251\266.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2401\344\271\213Kmeans\351\224\231\350\257\257\350\247\243\345\206\263.md" "b/docs/Spark/MLlibLearning/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2401\344\271\213Kmeans\351\224\231\350\257\257\350\247\243\345\206\263.md" similarity index 99% rename from "docs/Spark MLlib\345\255\246\344\271\240/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2401\344\271\213Kmeans\351\224\231\350\257\257\350\247\243\345\206\263.md" rename to "docs/Spark/MLlibLearning/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2401\344\271\213Kmeans\351\224\231\350\257\257\350\247\243\345\206\263.md" index 5fa21c4..d35fbd3 100644 --- "a/docs/Spark MLlib\345\255\246\344\271\240/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2401\344\271\213Kmeans\351\224\231\350\257\257\350\247\243\345\206\263.md" +++ "b/docs/Spark/MLlibLearning/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\2401\344\271\213Kmeans\351\224\231\350\257\257\350\247\243\345\206\263.md" @@ -12,7 +12,7 @@ hadoop@Master:~/cloud/testByXubo/sh_spark_xubo/mllib/kmeans/KMeansTest3ByIBM$ ./submitJob.sh [Stage 0:> (0 + 2) / 2]16/03/29 17:16:07 ERROR TaskSetManager: Task 1 in stage 0.0 failed 4 times; aborting job - Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 0.0 failed 4 times, most recent failure: Lost task 1.3 in stage 0.0 (TID 6, 219.219.220.223): java.lang.NumberFormatException: For input string: "6373" + Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 0.0 failed 4 times, most recent failure: Lost task 1.3 in stage 0.0 (TID 6, Mcnode4): java.lang.NumberFormatException: For input string: "6373" at sun.misc.FloatingDecimal.readJavaFormatString(FloatingDecimal.java:1250) at java.lang.Double.parseDouble(Double.java:540) at scala.collection.immutable.StringLike$class.toDouble(StringLike.scala:232) @@ -118,7 +118,7 @@ hadoop@Master:~/cloud/testByXubo/sh_spark_xubo/mllib/kmeans/KMeansTest3ByIBM$ ^C hadoop@Master:~/cloud/testByXubo/sh_spark_xubo/mllib/kmeans/KMeansTest3ByIBM$ ./submitJob.sh [Stage 0:> (0 + 2) / 2]16/03/29 17:17:15 ERROR TaskSetManager: Task 0 in stage 0.0 failed 4 times; aborting job - Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 0.0 failed 4 times, most recent failure: Lost task 0.3 in stage 0.0 (TID 6, 219.219.220.223): java.lang.NumberFormatException: For input string: "??Channel" + Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 0.0 failed 4 times, most recent failure: Lost task 0.3 in stage 0.0 (TID 6, Mcnode4): java.lang.NumberFormatException: For input string: "??Channel" at sun.misc.FloatingDecimal.readJavaFormatString(FloatingDecimal.java:1250) at java.lang.Double.parseDouble(Double.java:540) at scala.collection.immutable.StringLike$class.toDouble(StringLike.scala:232) @@ -229,7 +229,7 @@ hadoop@Master:~/cloud/testByXubo/sh_spark_xubo/mllib/kmeans/KMeansTest3ByIBM$ vi submitJob.sh hadoop@Master:~/cloud/testByXubo/sh_spark_xubo/mllib/kmeans/KMeansTest3ByIBM$ ./submitJob.sh [Stage 0:> (0 + 1) / 2]16/03/29 17:19:53 ERROR TaskSetManager: Task 0 in stage 0.0 failed 4 times; aborting job - Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 0.0 failed 4 times, most recent failure: Lost task 0.3 in stage 0.0 (TID 5, 219.219.220.248): java.lang.NumberFormatException: For input string: "??Channel" + Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 0.0 failed 4 times, most recent failure: Lost task 0.3 in stage 0.0 (TID 5, Mcnode2): java.lang.NumberFormatException: For input string: "??Channel" at sun.misc.FloatingDecimal.readJavaFormatString(FloatingDecimal.java:1250) at java.lang.Double.parseDouble(Double.java:540) at scala.collection.immutable.StringLike$class.toDouble(StringLike.scala:232) diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24044\344\271\213\351\253\230\346\226\257\346\267\267\345\220\210\350\201\232\347\261\273GaussianMixture.md" "b/docs/Spark/MLlibLearning/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24044\344\271\213\351\253\230\346\226\257\346\267\267\345\220\210\350\201\232\347\261\273GaussianMixture.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24044\344\271\213\351\253\230\346\226\257\346\267\267\345\220\210\350\201\232\347\261\273GaussianMixture.md" rename to "docs/Spark/MLlibLearning/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24044\344\271\213\351\253\230\346\226\257\346\267\267\345\220\210\350\201\232\347\261\273GaussianMixture.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24045\344\271\213\347\224\250\351\253\230\346\226\257\346\267\267\345\220\210\346\250\241\345\236\213\346\235\245\351\242\204\346\265\213.md" "b/docs/Spark/MLlibLearning/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24045\344\271\213\347\224\250\351\253\230\346\226\257\346\267\267\345\220\210\346\250\241\345\236\213\346\235\245\351\242\204\346\265\213.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24045\344\271\213\347\224\250\351\253\230\346\226\257\346\267\267\345\220\210\346\250\241\345\236\213\346\235\245\351\242\204\346\265\213.md" rename to "docs/Spark/MLlibLearning/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24045\344\271\213\347\224\250\351\253\230\346\226\257\346\267\267\345\220\210\346\250\241\345\236\213\346\235\245\351\242\204\346\265\213.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24046\344\271\213Power iteration clustering.md" "b/docs/Spark/MLlibLearning/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24046\344\271\213Power iteration clustering.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24046\344\271\213Power iteration clustering.md" rename to "docs/Spark/MLlibLearning/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24046\344\271\213Power iteration clustering.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24047\344\271\213\351\232\220\345\220\253\347\213\204\345\210\251\345\205\213\351\233\267\345\210\206\345\270\203(Latent Dirichlet allocation (LDA)\345\255\246\344\271\240.md" "b/docs/Spark/MLlibLearning/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24047\344\271\213\351\232\220\345\220\253\347\213\204\345\210\251\345\205\213\351\233\267\345\210\206\345\270\203(Latent Dirichlet allocation (LDA)\345\255\246\344\271\240.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24047\344\271\213\351\232\220\345\220\253\347\213\204\345\210\251\345\205\213\351\233\267\345\210\206\345\270\203(Latent Dirichlet allocation (LDA)\345\255\246\344\271\240.md" rename to "docs/Spark/MLlibLearning/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24047\344\271\213\351\232\220\345\220\253\347\213\204\345\210\251\345\205\213\351\233\267\345\210\206\345\270\203(Latent Dirichlet allocation (LDA)\345\255\246\344\271\240.md" diff --git "a/docs/Spark/MLlibLearning/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24047\344\271\213\351\232\220\345\220\253\347\213\204\345\210\251\345\205\213\351\233\267\345\210\206\345\270\203\357\274\210Latent Dirichlet allocation\357\274\214LDA\357\274\211\345\255\246\344\271\240.md" "b/docs/Spark/MLlibLearning/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24047\344\271\213\351\232\220\345\220\253\347\213\204\345\210\251\345\205\213\351\233\267\345\210\206\345\270\203\357\274\210Latent Dirichlet allocation\357\274\214LDA\357\274\211\345\255\246\344\271\240.md" new file mode 100644 index 0000000..911c8ff --- /dev/null +++ "b/docs/Spark/MLlibLearning/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24047\344\271\213\351\232\220\345\220\253\347\213\204\345\210\251\345\205\213\351\233\267\345\210\206\345\270\203\357\274\210Latent Dirichlet allocation\357\274\214LDA\357\274\211\345\255\246\344\271\240.md" @@ -0,0 +1,143 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark中组件Mllib的学习 + +1.解释 + +隐含狄利克雷分布(Latent Dirichlet allocation (LDA) + +隐含狄利克雷分布(LDA) 是一个主题模型，它能够推理出一个文本文档集合的主体。LDA可以认为是一个聚类算法，原因如下： + + 主题对应聚类中心，文档对应数据集中的样本（数据行） + 主题和文档都在一个特征空间中，其特征向量是词频向量。 + 跟使用传统的距离来评估聚类不一样的是，LDA使用评估方式是一个函数，该函数基于文档如何生成的统计模型。 + +LDA以词频向量表示的文档集合作为输入。然后在最大似然函数上使用期望最大（EM）算法 来学习聚类。完成文档拟合之后，LDA提供： + + Topics: 推断出的主题，每个主体是单词上的概率分布。 + Topic distributions for documents: 对训练集中的每个文档，LDA给了一个在主题上的概率分布。 + +LDA参数如下： + + k: 主题数量（或者说聚簇中心数量） + maxIterations: EM算法的最大迭代次数。 + docConcentration: 文档在主题上分布的先验参数。当前必须大于1，值越大，推断出的分布越平滑。 + topicConcentration: 主题在单词上的先验分布参数。当前必须大于1，值越大，推断出的分布越平滑。 + checkpointInterval: 检查点间隔。maxIterations很大的时候，检查点可以帮助减少shuffle文件大小并且可以帮助故障恢复。 + +参考【4】 + +2.代码： + + /** + * @author xubo + * ref:http://spark.apache.org/docs/1.5.2/mllib-guide.html + * more code:https://github.com/xubo245/SparkLearning + * more blog:http://blog.csdn.net/xubo245 + */ + package org.apache.spark.mllib.clustering.LDALearning + + import org.apache.spark.mllib.clustering.LDA + import org.apache.spark.util.SparkLearningFunSuite + + /** + * Created by xubo on 2016/6/13. + */ + class LDAFromWebSuite extends SparkLearningFunSuite { + test("testFunSuite") { + + + import org.apache.spark.mllib.linalg.Vectors + + // Load and parse the data + val data = sc.textFile("file/data/mllib/input/mllibFromSpark/sample_lda_data.txt") + val parsedData = data.map(s => Vectors.dense(s.trim.split(' ').map(_.toDouble))) + // Index documents with unique IDs + val corpus = parsedData.zipWithIndex.map(_.swap).cache() + + // Cluster the documents into three topics using LDA + val ldaModel = new LDA().setK(3).run(corpus) + + //input data + println("parsedData:") + parsedData.foreach(println) + println("corpus:") + corpus.foreach(println) + + // Output topics. Each is a distribution over words (matching word count vectors) + println("Learned topics (as distributions over vocab of " + ldaModel.vocabSize + " words):") + val topics = ldaModel.topicsMatrix + for (topic <- Range(0, 3)) { + print("Topic " + topic + ":") + for (word <- Range(0, ldaModel.vocabSize)) { + print(" " + topics(word, topic)); + } + println() + } + + // Save and load model. + // ldaModel.save(sc, "myLDAModel") + // val sameModel = DistributedLDAModel.load(sc, "myLDAModel") + + + } + } + +数据： + + 1 2 6 0 2 3 1 1 0 0 3 + 1 3 0 1 3 0 0 2 0 0 1 + 1 4 1 0 0 4 9 0 1 2 0 + 2 1 0 3 0 0 5 0 2 3 9 + 3 1 1 9 3 0 2 0 0 1 3 + 4 2 0 3 4 5 1 1 1 4 0 + 2 1 0 3 0 0 5 0 2 2 9 + 1 1 1 9 2 1 2 0 0 1 3 + 4 4 0 3 4 2 1 3 0 0 0 + 2 8 2 0 3 0 2 0 2 7 2 + 1 1 1 9 0 2 2 0 0 3 3 + 4 1 0 0 4 5 1 3 0 1 0 + + +3.结果： + + parsedData: + [1.0,1.0,1.0,9.0,2.0,1.0,2.0,0.0,0.0,1.0,3.0] + [1.0,2.0,6.0,0.0,2.0,3.0,1.0,1.0,0.0,0.0,3.0] + [4.0,4.0,0.0,3.0,4.0,2.0,1.0,3.0,0.0,0.0,0.0] + [1.0,3.0,0.0,1.0,3.0,0.0,0.0,2.0,0.0,0.0,1.0] + [2.0,8.0,2.0,0.0,3.0,0.0,2.0,0.0,2.0,7.0,2.0] + [1.0,4.0,1.0,0.0,0.0,4.0,9.0,0.0,1.0,2.0,0.0] + [1.0,1.0,1.0,9.0,0.0,2.0,2.0,0.0,0.0,3.0,3.0] + [2.0,1.0,0.0,3.0,0.0,0.0,5.0,0.0,2.0,3.0,9.0] + [4.0,1.0,0.0,0.0,4.0,5.0,1.0,3.0,0.0,1.0,0.0] + [3.0,1.0,1.0,9.0,3.0,0.0,2.0,0.0,0.0,1.0,3.0] + [4.0,2.0,0.0,3.0,4.0,5.0,1.0,1.0,1.0,4.0,0.0] + [2.0,1.0,0.0,3.0,0.0,0.0,5.0,0.0,2.0,2.0,9.0] + corpus: + (7,[1.0,1.0,1.0,9.0,2.0,1.0,2.0,0.0,0.0,1.0,3.0]) + (8,[4.0,4.0,0.0,3.0,4.0,2.0,1.0,3.0,0.0,0.0,0.0]) + (9,[2.0,8.0,2.0,0.0,3.0,0.0,2.0,0.0,2.0,7.0,2.0]) + (0,[1.0,2.0,6.0,0.0,2.0,3.0,1.0,1.0,0.0,0.0,3.0]) + (10,[1.0,1.0,1.0,9.0,0.0,2.0,2.0,0.0,0.0,3.0,3.0]) + (1,[1.0,3.0,0.0,1.0,3.0,0.0,0.0,2.0,0.0,0.0,1.0]) + (11,[4.0,1.0,0.0,0.0,4.0,5.0,1.0,3.0,0.0,1.0,0.0]) + (2,[1.0,4.0,1.0,0.0,0.0,4.0,9.0,0.0,1.0,2.0,0.0]) + (3,[2.0,1.0,0.0,3.0,0.0,0.0,5.0,0.0,2.0,3.0,9.0]) + (4,[3.0,1.0,1.0,9.0,3.0,0.0,2.0,0.0,0.0,1.0,3.0]) + (5,[4.0,2.0,0.0,3.0,4.0,5.0,1.0,1.0,1.0,4.0,0.0]) + (6,[2.0,1.0,0.0,3.0,0.0,0.0,5.0,0.0,2.0,2.0,9.0]) + Learned topics (as distributions over vocab of 11 words): + Topic 0: 10.452679685126427 10.181668875779492 2.558644879228586 3.851438041310386 9.929534544713832 11.940154625598604 14.086100675895626 4.8961707413781115 2.2995952755592106 7.381361487130488 8.981150231959049 + Topic 1: 10.279220142758316 5.956661018866242 4.910211518699095 30.538789151743963 5.928882165794898 5.447495432535608 6.549479250479619 3.011959583638183 1.0753194351327675 3.217481558556803 9.62611924184504 + Topic 2: 5.268100172115256 12.861670105354264 4.531143602072319 5.6097728069456565 9.141583289491269 4.612349941865787 10.364420073624755 2.0918696749837067 4.625085289308021 13.40115695431271 14.392730526195912 + + + +参考 + + 【1】http://spark.apache.org/docs/1.5.2/mllib-guide.html + 【2】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【3】https://github.com/xubo245/SparkLearning + 【4】http://www.fuqingchuan.com/2015/03/609.html#latent-dirichlet-allocation-lda \ No newline at end of file diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24048\344\271\213\346\265\201\345\274\217k\345\235\207\345\200\274\357\274\210Streaming kmeans\357\274\211.md" "b/docs/Spark/MLlibLearning/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24048\344\271\213\346\265\201\345\274\217k\345\235\207\345\200\274\357\274\210Streaming kmeans\357\274\211.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24048\344\271\213\346\265\201\345\274\217k\345\235\207\345\200\274\357\274\210Streaming kmeans\357\274\211.md" rename to "docs/Spark/MLlibLearning/5\350\201\232\347\261\273/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24048\344\271\213\346\265\201\345\274\217k\345\235\207\345\200\274\357\274\210Streaming kmeans\357\274\211.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/6\351\231\215\347\273\264/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24049\344\271\213\345\245\207\345\274\202\345\200\274\345\210\206\350\247\243SVD(Singular value decomposition).md" "b/docs/Spark/MLlibLearning/6\351\231\215\347\273\264/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24049\344\271\213\345\245\207\345\274\202\345\200\274\345\210\206\350\247\243SVD(Singular value decomposition).md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/6\351\231\215\347\273\264/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24049\344\271\213\345\245\207\345\274\202\345\200\274\345\210\206\350\247\243SVD(Singular value decomposition).md" rename to "docs/Spark/MLlibLearning/6\351\231\215\347\273\264/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24049\344\271\213\345\245\207\345\274\202\345\200\274\345\210\206\350\247\243SVD(Singular value decomposition).md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/6\351\231\215\347\273\264/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24050\344\271\213\344\270\273\346\210\220\344\273\275\345\210\206\346\236\220PCA.md" "b/docs/Spark/MLlibLearning/6\351\231\215\347\273\264/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24050\344\271\213\344\270\273\346\210\220\344\273\275\345\210\206\346\236\220PCA.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/6\351\231\215\347\273\264/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24050\344\271\213\344\270\273\346\210\220\344\273\275\345\210\206\346\236\220PCA.md" rename to "docs/Spark/MLlibLearning/6\351\231\215\347\273\264/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24050\344\271\213\344\270\273\346\210\220\344\273\275\345\210\206\346\236\220PCA.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/6\351\231\215\347\273\264/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24051\344\271\213\344\275\277\347\224\250PCA\344\273\216\346\225\260\346\215\256\351\233\206\344\270\255\345\276\227\345\210\260\344\270\273\345\220\221\351\207\217.md" "b/docs/Spark/MLlibLearning/6\351\231\215\347\273\264/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24051\344\271\213\344\275\277\347\224\250PCA\344\273\216\346\225\260\346\215\256\351\233\206\344\270\255\345\276\227\345\210\260\344\270\273\345\220\221\351\207\217.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/6\351\231\215\347\273\264/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24051\344\271\213\344\275\277\347\224\250PCA\344\273\216\346\225\260\346\215\256\351\233\206\344\270\255\345\276\227\345\210\260\344\270\273\345\220\221\351\207\217.md" rename to "docs/Spark/MLlibLearning/6\351\231\215\347\273\264/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24051\344\271\213\344\275\277\347\224\250PCA\344\273\216\346\225\260\346\215\256\351\233\206\344\270\255\345\276\227\345\210\260\344\270\273\345\220\221\351\207\217.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24052\344\271\213TF-IDF\345\255\246\344\271\240.md" "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24052\344\271\213TF-IDF\345\255\246\344\271\240.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24052\344\271\213TF-IDF\345\255\246\344\271\240.md" rename to "docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24052\344\271\213TF-IDF\345\255\246\344\271\240.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24053\344\271\213HashingTF\347\220\206\350\247\243\345\222\214\344\275\277\347\224\250.md" "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24053\344\271\213HashingTF\347\220\206\350\247\243\345\222\214\344\275\277\347\224\250.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24053\344\271\213HashingTF\347\220\206\350\247\243\345\222\214\344\275\277\347\224\250.md" rename to "docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24053\344\271\213HashingTF\347\220\206\350\247\243\345\222\214\344\275\277\347\224\250.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24053\344\271\213Word2Vec\347\256\200\345\215\225\345\256\236\344\276\213.md" "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24053\344\271\213Word2Vec\347\256\200\345\215\225\345\256\236\344\276\213.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24053\344\271\213Word2Vec\347\256\200\345\215\225\345\256\236\344\276\213.md" rename to "docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24053\344\271\213Word2Vec\347\256\200\345\215\225\345\256\236\344\276\213.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24054\344\271\213word2Vec\345\256\236\344\276\213\345\210\206\346\236\220\357\274\210text8\346\225\260\346\215\256\351\233\206\357\274\211.md" "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24054\344\271\213word2Vec\345\256\236\344\276\213\345\210\206\346\236\220\357\274\210text8\346\225\260\346\215\256\351\233\206\357\274\211.md" similarity index 99% rename from "docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24054\344\271\213word2Vec\345\256\236\344\276\213\345\210\206\346\236\220\357\274\210text8\346\225\260\346\215\256\351\233\206\357\274\211.md" rename to "docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24054\344\271\213word2Vec\345\256\236\344\276\213\345\210\206\346\236\220\357\274\210text8\346\225\260\346\215\256\351\233\206\357\274\211.md" index c1b11af..dc41259 100644 --- "a/docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24054\344\271\213word2Vec\345\256\236\344\276\213\345\210\206\346\236\220\357\274\210text8\346\225\260\346\215\256\351\233\206\357\274\211.md" +++ "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24054\344\271\213word2Vec\345\256\236\344\276\213\345\210\206\346\236\220\357\274\210text8\346\225\260\346\215\256\351\233\206\357\274\211.md" @@ -58,7 +58,7 @@ text8数据集下载：http://mattmahoney.net/dc/text8.zip，没有上传到gith #!/usr/bin/env bash spark-submit \ --class org.apache.spark.mllib.FeatureExtractionAndTransformation.Word2VecSparkWeb \ - --master spark://219.219.220.149:7077 \ + --master spark://Master:7077 \ --executor-memory 4096M \ --total-executor-cores 20 SparkLearning.jar /xubo/project/sparkLearning/text8 diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24055\344\271\213\344\275\277\347\224\250TfIdf\346\235\245\345\210\206\346\236\22020news\346\225\260\346\215\256\351\233\206.md" "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24055\344\271\213\344\275\277\347\224\250TfIdf\346\235\245\345\210\206\346\236\22020news\346\225\260\346\215\256\351\233\206.md" similarity index 99% rename from "docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24055\344\271\213\344\275\277\347\224\250TfIdf\346\235\245\345\210\206\346\236\22020news\346\225\260\346\215\256\351\233\206.md" rename to "docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24055\344\271\213\344\275\277\347\224\250TfIdf\346\235\245\345\210\206\346\236\22020news\346\225\260\346\215\256\351\233\206.md" index a377192..65c71aa 100644 --- "a/docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24055\344\271\213\344\275\277\347\224\250TfIdf\346\235\245\345\210\206\346\236\22020news\346\225\260\346\215\256\351\233\206.md" +++ "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24055\344\271\213\344\275\277\347\224\250TfIdf\346\235\245\345\210\206\346\236\22020news\346\225\260\346\215\256\351\233\206.md" @@ -571,7 +571,7 @@ Spark中组件Mllib的学习 #!/usr/bin/env bash spark-submit \ --class org.apache.spark.mllib.FeatureExtractionAndTransformation.TfIdf20News \ - --master spark://219.219.220.149:7077 \ + --master spark://Master:7077 \ --executor-memory 4096M \ --total-executor-cores 20 SparkLearning.jar /xubo/project/sparkLearning/20news-bydate/20news-bydate-train/* /xubo/project/sparkLearning/20news-bydate/20news-bydate-test/* diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24056\344\271\213\346\240\207\345\207\206\345\214\226\357\274\210StandardScaler\357\274\214\346\235\245\350\207\252SparkWeb\357\274\211.md" "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24056\344\271\213\346\240\207\345\207\206\345\214\226\357\274\210StandardScaler\357\274\214\346\235\245\350\207\252SparkWeb\357\274\211.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24056\344\271\213\346\240\207\345\207\206\345\214\226\357\274\210StandardScaler\357\274\214\346\235\245\350\207\252SparkWeb\357\274\211.md" rename to "docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24056\344\271\213\346\240\207\345\207\206\345\214\226\357\274\210StandardScaler\357\274\214\346\235\245\350\207\252SparkWeb\357\274\211.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24057\344\271\213\346\240\207\345\207\206\345\214\226\345\217\202\346\225\260\345\222\214\345\205\254\345\274\217\347\220\206\350\247\243\357\274\210StandardScaler\357\274\214\346\235\245\350\207\252SparkCode).md" "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24057\344\271\213\346\240\207\345\207\206\345\214\226\345\217\202\346\225\260\345\222\214\345\205\254\345\274\217\347\220\206\350\247\243\357\274\210StandardScaler\357\274\214\346\235\245\350\207\252SparkCode).md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24057\344\271\213\346\240\207\345\207\206\345\214\226\345\217\202\346\225\260\345\222\214\345\205\254\345\274\217\347\220\206\350\247\243\357\274\210StandardScaler\357\274\214\346\235\245\350\207\252SparkCode).md" rename to "docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24057\344\271\213\346\240\207\345\207\206\345\214\226\345\217\202\346\225\260\345\222\214\345\205\254\345\274\217\347\220\206\350\247\243\357\274\210StandardScaler\357\274\214\346\235\245\350\207\252SparkCode).md" diff --git "a/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24057\344\271\213\346\240\207\345\207\206\345\214\226\345\217\202\346\225\260\345\222\214\345\205\254\345\274\217\347\220\206\350\247\243\357\274\210StandardScaler\357\274\214\346\235\245\350\207\252SparkCode\357\274\211.md" "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24057\344\271\213\346\240\207\345\207\206\345\214\226\345\217\202\346\225\260\345\222\214\345\205\254\345\274\217\347\220\206\350\247\243\357\274\210StandardScaler\357\274\214\346\235\245\350\207\252SparkCode\357\274\211.md" new file mode 100644 index 0000000..80df1dd --- /dev/null +++ "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24057\344\271\213\346\240\207\345\207\206\345\214\226\345\217\202\346\225\260\345\222\214\345\205\254\345\274\217\347\220\206\350\247\243\357\274\210StandardScaler\357\274\214\346\235\245\350\207\252SparkCode\357\274\211.md" @@ -0,0 +1,278 @@ + + 更多代码请见：https://github.com/xubo245/SparkLearning + + Spark中组件Mllib的学习 + +1.解释 + + 由于前一篇的ref的表达不一样，所以： + 学习spark mllib的test代码，理解标准化的计算 + 具体请看最后的结果分析 + +结论为： + +（1）当参数为： + + equivalentModel1.withMean:true + equivalentModel1.withStd:true +公式就是(x-u)/sqrt(variance)，x为标准化前的值，u为该列的平均值，variance为该列的方差，开方即为标准差 + +（2）当参数为： + + equivalentModel2.withMean:false + equivalentModel2.withStd:true +公式就是(x)/sqrt(variance)，x为标准化前的值，variance为该列的方差，开方即为标准差 + + +（3）当参数为： + + equivalentModel2.withMean:true + equivalentModel2.withStd:true +公式就是(x-u)，x为标准化前的值，u为该列的平均值 + +2.代码： + + val constantData = Array( + Vectors.dense(2.0), + Vectors.dense(2.0), + Vectors.dense(2.0) + ) + + val sparseData = Array( + Vectors.sparse(3, Seq((0, -2.0), (1, 2.3))), + Vectors.sparse(3, Seq((1, -1.0), (2, -3.0))), + Vectors.sparse(3, Seq((1, -5.1))), + Vectors.sparse(3, Seq((0, 3.8), (2, 1.9))), + Vectors.sparse(3, Seq((0, 1.7), (1, -0.6))), + Vectors.sparse(3, Seq((1, 1.9))) + ) + + val denseData = Array( + Vectors.dense(-2.0, 2.3, 0), + Vectors.dense(0.0, -1.0, -3.0), + Vectors.dense(0.0, -5.1, 0.0), + Vectors.dense(3.8, 0.0, 1.9), + Vectors.dense(1.7, -0.6, 0.0), + Vectors.dense(0.0, 1.9, 0.0) + ) + + private def computeSummary(data: RDD[Vector]): MultivariateStatisticalSummary = { + data.treeAggregate(new MultivariateOnlineSummarizer)( + (aggregator, data) => aggregator.add(data), + (aggregator1, aggregator2) => aggregator1.merge(aggregator2)) + } + + test("Standardization with dense input when means and stds are provided") { + + val dataRDD = sc.parallelize(denseData, 3) + + val standardizer1 = new StandardScaler(withMean = true, withStd = true) + val standardizer2 = new StandardScaler() + val standardizer3 = new StandardScaler(withMean = true, withStd = false) + + val model1 = standardizer1.fit(dataRDD) + val model2 = standardizer2.fit(dataRDD) + val model3 = standardizer3.fit(dataRDD) + + val equivalentModel1 = new StandardScalerModel(model1.std, model1.mean) + val equivalentModel2 = new StandardScalerModel(model2.std, model2.mean, true, false) + val equivalentModel3 = new StandardScalerModel(model3.std, model3.mean, false, true) + + val data1 = denseData.map(equivalentModel1.transform) + val data2 = denseData.map(equivalentModel2.transform) + val data3 = denseData.map(equivalentModel3.transform) + + val data1RDD = equivalentModel1.transform(dataRDD) + val data2RDD = equivalentModel2.transform(dataRDD) + val data3RDD = equivalentModel3.transform(dataRDD) + + val summary = computeSummary(dataRDD) + val summary1 = computeSummary(data1RDD) + val summary2 = computeSummary(data2RDD) + val summary3 = computeSummary(data3RDD) + + println("denseData:") + denseData.take(10).foreach { each => + println(each.size + ":" + each) + // println(each.size) + } + println("dataRDD:") + dataRDD.take(10).foreach { each => + println(each.size + ":" + each) + // println(each.size) + } + println("equivalentModel1.mean:" + equivalentModel1.mean) + println("equivalentModel1.std:" + equivalentModel1.std) + println("equivalentModel1.withMean:" + equivalentModel1.withMean) + println("equivalentModel1.withStd:" + equivalentModel1.withStd) + println("data1RDD:") + data1RDD.take(10).foreach { each => + println(each.size + ":" + each) + // println(each.size) + } + println("equivalentModel2.mean:" + equivalentModel2.mean) + println("equivalentModel2.std:" + equivalentModel2.std) + println("equivalentModel2.withMean:" + equivalentModel2.withMean) + println("equivalentModel2.withStd:" + equivalentModel2.withStd) + println("data2RDD:") + data2RDD.take(10).foreach { each => + println(each.size + ":" + each) + // println(each.size) + } + println("equivalentModel3.mean:" + equivalentModel3.mean) + println("equivalentModel3.std:" + equivalentModel3.std) + println("equivalentModel3.withMean:" + equivalentModel3.withMean) + println("equivalentModel3.withStd:" + equivalentModel3.withStd) + println("data3RDD:") + data3RDD.take(10).foreach { each => + println(each.size + ":" + each) + // println(each.size) + } + + +3.结果： + + denseData: + 3:[-2.0,2.3,0.0] + 3:[0.0,-1.0,-3.0] + 3:[0.0,-5.1,0.0] + 3:[3.8,0.0,1.9] + 3:[1.7,-0.6,0.0] + 3:[0.0,1.9,0.0] + dataRDD: + 3:[-2.0,2.3,0.0] + 3:[0.0,-1.0,-3.0] + 3:[0.0,-5.1,0.0] + 3:[3.8,0.0,1.9] + 3:[1.7,-0.6,0.0] + 3:[0.0,1.9,0.0] + equivalentModel1.mean:[0.5833333333333333,-0.4166666666666665,-0.1833333333333334] + equivalentModel1.std:[1.9640943629741079,2.6543674701643454,1.5753306531222795] + equivalentModel1.withMean:true + equivalentModel1.withStd:true + data1RDD: + 3:[-1.3152796433983698,1.023470449062753,0.11637768424677686] + 3:[-0.2969986291544706,-0.21976359335703294,-1.7879844216095708] + 3:[-0.2969986291544706,-1.7643877066664642,0.11637768424677686] + 3:[1.6377352979089383,0.15697399525502342,1.3224736846224638] + 3:[0.5685402329528438,-0.06906855791221038,0.11637768424677686] + 3:[-0.2969986291544706,0.8727754136179304,0.11637768424677686] + equivalentModel2.mean:[0.5833333333333333,-0.4166666666666667,-0.18333333333333326] + equivalentModel2.std:[1.9640943629741079,2.6543674701643454,1.5753306531222793] + equivalentModel2.withMean:false + equivalentModel2.withStd:true + data2RDD: + 3:[-1.0182810142438994,0.8664964538077295,0.0] + 3:[0.0,-0.37673758861205636,-1.904362105856348] + 3:[0.0,-1.9213617019214873,0.0] + 3:[1.9347339270634087,0.0,1.206096000375687] + 3:[0.8655388621073145,-0.22604255316723382,0.0] + 3:[0.0,0.715801418362907,0.0] + equivalentModel3.mean:[0.5833333333333333,-0.4166666666666665,-0.1833333333333334] + equivalentModel3.std:[1.9640943629741079,2.6543674701643454,1.5753306531222795] + equivalentModel3.withMean:true + equivalentModel3.withStd:false + data3RDD: + 3:[-2.583333333333333,2.7166666666666663,0.1833333333333334] + 3:[-0.5833333333333333,-0.5833333333333335,-2.8166666666666664] + 3:[-0.5833333333333333,-4.683333333333334,0.1833333333333334] + 3:[3.216666666666667,0.4166666666666665,2.0833333333333335] + 3:[1.1166666666666667,-0.18333333333333346,0.1833333333333334] + 3:[-0.5833333333333333,2.3166666666666664,0.1833333333333334] + + +结果分析： +（1） + + dataRDD: + 3:[-2.0,2.3,0.0] + 3:[0.0,-1.0,-3.0] + 3:[0.0,-5.1,0.0] + 3:[3.8,0.0,1.9] + 3:[1.7,-0.6,0.0] + 3:[0.0,1.9,0.0] +第一列的平均值是0.5833333333333333，标准差是1.9640943629741079，训练时： + + equivalentModel1.withMean:true + equivalentModel1.withStd:true + +所以结果为：】 + + data1RDD: + 3:[-1.3152796433983698,1.023470449062753,0.11637768424677686] + 3:[-0.2969986291544706,-0.21976359335703294,-1.7879844216095708] + 3:[-0.2969986291544706,-1.7643877066664642,0.11637768424677686] + 3:[1.6377352979089383,0.15697399525502342,1.3224736846224638] + 3:[0.5685402329528438,-0.06906855791221038,0.11637768424677686] + 3:[-0.2969986291544706,0.8727754136179304,0.11637768424677686] + +其中： +-1.3152796433983698=（-2-0.5833333333333333）/1.9640943629741079 + + scala> (-2-0.583)/1.96 + res11: Double = -1.317857142857143 + + scala> (-0.583)/1.96 + res12: Double = -0.29744897959183675 + + +所以公式为：公式就是(x-u)/sqrt(variance)，x为标准化前的值，u为该列的平均值，variance为该列的方差，开方即为标准差 + + +（2） + + dataRDD: + 3:[-2.0,2.3,0.0] + 3:[0.0,-1.0,-3.0] + 3:[0.0,-5.1,0.0] + 3:[3.8,0.0,1.9] + 3:[1.7,-0.6,0.0] + 3:[0.0,1.9,0.0] + + equivalentModel2.mean:[0.5833333333333333,-0.4166666666666667,-0.18333333333333326] + equivalentModel2.std:[1.9640943629741079,2.6543674701643454,1.5753306531222793] + equivalentModel2.withMean:false + equivalentModel2.withStd:true + data2RDD: + 3:[-1.0182810142438994,0.8664964538077295,0.0] + 3:[0.0,-0.37673758861205636,-1.904362105856348] + 3:[0.0,-1.9213617019214873,0.0] + 3:[1.9347339270634087,0.0,1.206096000375687] + 3:[0.8655388621073145,-0.22604255316723382,0.0] + 3:[0.0,0.715801418362907,0.0] + +同理计算： + + scala> (-2)/1.964094 + res14: Double = -1.018281202427175 + +所以公式为： + + (x)/sqrt(variance) + +（3） + + + equivalentModel3.mean:[0.5833333333333333,-0.4166666666666665,-0.1833333333333334] + equivalentModel3.std:[1.9640943629741079,2.6543674701643454,1.5753306531222795] + equivalentModel3.withMean:true + equivalentModel3.withStd:false + data3RDD: + 3:[-2.583333333333333,2.7166666666666663,0.1833333333333334] + 3:[-0.5833333333333333,-0.5833333333333335,-2.8166666666666664] + 3:[-0.5833333333333333,-4.683333333333334,0.1833333333333334] + 3:[3.216666666666667,0.4166666666666665,2.0833333333333335] + 3:[1.1166666666666667,-0.18333333333333346,0.1833333333333334] + 3:[-0.5833333333333333,2.3166666666666664,0.1833333333333334] + +同理，所以公式为： + + (x-u) + +参考 + + 【1】http://spark.apache.org/docs/1.5.2/mllib-guide.html + 【2】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【3】https://github.com/xubo245/SparkLearning + 【4】book:Machine Learning with Spark ,Nick Pertreach + 【5】book:Spark MlLib机器学习实战 diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24058\344\271\213\345\275\222\344\270\200\345\214\226(Normalizer)Normalization using L1 distance.md" "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24058\344\271\213\345\275\222\344\270\200\345\214\226(Normalizer)Normalization using L1 distance.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24058\344\271\213\345\275\222\344\270\200\345\214\226(Normalizer)Normalization using L1 distance.md" rename to "docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24058\344\271\213\345\275\222\344\270\200\345\214\226(Normalizer)Normalization using L1 distance.md" diff --git "a/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24058\344\271\213\345\275\222\344\270\200\345\214\226\357\274\210Normalizer\357\274\211Normalization using L1 distance.md" "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24058\344\271\213\345\275\222\344\270\200\345\214\226\357\274\210Normalizer\357\274\211Normalization using L1 distance.md" new file mode 100644 index 0000000..6c36098 --- /dev/null +++ "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24058\344\271\213\345\275\222\344\270\200\345\214\226\357\274\210Normalizer\357\274\211Normalization using L1 distance.md" @@ -0,0 +1,95 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark中组件Mllib的学习 + +1.解释 + +  规则化器缩放单个样本让其拥有单位范数。这是文本分类和聚类常用的操作。例如，两个规则化的TFIDF向量的点乘就是两个向量的cosine相似度。 + +  Normalizer实现VectorTransformer，将一个向量规则化为转换的向量，或者将一个RDD规则化为另一个RDD。下面是一个规则化的例子。 + +Normalization using L1 distance ,来自spark mllib 的test + + + +2.代码： + + val data = Array( + Vectors.sparse(3, Seq((0, -2.0), (1, 2.3))), + Vectors.dense(0.0, 0.0, 0.0), + Vectors.dense(0.6, -1.1, -3.0), + Vectors.sparse(3, Seq((1, 0.91), (2, 3.2))), + Vectors.sparse(3, Seq((0, 5.7), (1, 0.72), (2, 2.7))), + Vectors.sparse(3, Seq()) + ) + + lazy val dataRDD = sc.parallelize(data, 3) + + test("Normalization using L1 distance") { + val l1Normalizer = new Normalizer(1) + + val data1 = data.map(l1Normalizer.transform) + val data1RDD = l1Normalizer.transform(dataRDD) + + println("dataRDD:") + dataRDD.foreach(println) + println("data1RDD:") + data1RDD.foreach(println) + assert((data, data1, data1RDD.collect()).zipped.forall { + case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true + case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true + case _ => false + }, "The vector type should be preserved after normalization.") + + assert((data1, data1RDD.collect()).zipped.forall((v1, v2) => v1 ~== v2 absTol 1E-5)) + + assert(brzNorm(data1(0).toBreeze, 1) ~== 1.0 absTol 1E-5) + assert(brzNorm(data1(2).toBreeze, 1) ~== 1.0 absTol 1E-5) + assert(brzNorm(data1(3).toBreeze, 1) ~== 1.0 absTol 1E-5) + assert(brzNorm(data1(4).toBreeze, 1) ~== 1.0 absTol 1E-5) + + assert(data1(0) ~== Vectors.sparse(3, Seq((0, -0.465116279), (1, 0.53488372))) absTol 1E-5) + assert(data1(1) ~== Vectors.dense(0.0, 0.0, 0.0) absTol 1E-5) + assert(data1(2) ~== Vectors.dense(0.12765957, -0.23404255, -0.63829787) absTol 1E-5) + assert(data1(3) ~== Vectors.sparse(3, Seq((1, 0.22141119), (2, 0.7785888))) absTol 1E-5) + assert(data1(4) ~== Vectors.dense(0.625, 0.07894737, 0.29605263) absTol 1E-5) + assert(data1(5) ~== Vectors.sparse(3, Seq()) absTol 1E-5) + } + +3.结果： + + dataRDD: + [0.6,-1.1,-3.0] + (3,[0,1],[-2.0,2.3]) + (3,[1,2],[0.91,3.2]) + [0.0,0.0,0.0] + (3,[0,1,2],[5.7,0.72,2.7]) + (3,[],[]) + data1RDD: + [0.1276595744680851,-0.23404255319148937,-0.6382978723404255] + (3,[1,2],[0.2214111922141119,0.778588807785888]) + (3,[0,1],[-0.46511627906976744,0.5348837209302325]) + [0.0,0.0,0.0] + (3,[0,1,2],[0.625,0.07894736842105261,0.29605263157894735]) + (3,[],[]) + +结果分析: + +-3/(0.6+1.1+3)=-0.6382978723404255 + + scala> 3.0/4.7 + res16: Double = 0.6382978723404255 + + scala> 5.7/9.12 + res17: Double = 0.6250000000000001 + +故L1来归一化是将每个值除以所有元素绝对值只和 + +参考 + + 【1】http://spark.apache.org/docs/1.5.2/mllib-guide.html + 【2】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【3】https://github.com/xubo245/SparkLearning + 【4】book:Machine Learning with Spark ,Nick Pertreach + 【5】book:Spark MlLib机器学习实战 diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24059\344\271\213\345\275\222\344\270\200\345\214\226(Normalizer)Normalization using L2 distance.md" "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24059\344\271\213\345\275\222\344\270\200\345\214\226(Normalizer)Normalization using L2 distance.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24059\344\271\213\345\275\222\344\270\200\345\214\226(Normalizer)Normalization using L2 distance.md" rename to "docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24059\344\271\213\345\275\222\344\270\200\345\214\226(Normalizer)Normalization using L2 distance.md" diff --git "a/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24059\344\271\213\345\275\222\344\270\200\345\214\226\357\274\210Normalizer\357\274\211Normalization using L2 distance.md" "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24059\344\271\213\345\275\222\344\270\200\345\214\226\357\274\210Normalizer\357\274\211Normalization using L2 distance.md" new file mode 100644 index 0000000..3781f73 --- /dev/null +++ "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24059\344\271\213\345\275\222\344\270\200\345\214\226\357\274\210Normalizer\357\274\211Normalization using L2 distance.md" @@ -0,0 +1,91 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark中组件Mllib的学习 + +1.解释 + +归一化 + +Normalization using L2 distance + +公式： + +分母为：其中本例中p为2 + +![](http://i.imgur.com/Ll5M0O8.png) + + +2.代码： + + test("Normalization using L2 distance") { + val l2Normalizer = new Normalizer() + + val data2 = data.map(l2Normalizer.transform) + val data2RDD = l2Normalizer.transform(dataRDD) + + println("dataRDD:") + dataRDD.foreach(println) + println("data2RDD:") + data2RDD.foreach(println) + + assert((data, data2, data2RDD.collect()).zipped.forall { + case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true + case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true + case _ => false + }, "The vector type should be preserved after normalization.") + + assert((data2, data2RDD.collect()).zipped.forall((v1, v2) => v1 ~== v2 absTol 1E-5)) + + assert(brzNorm(data2(0).toBreeze, 2) ~== 1.0 absTol 1E-5) + assert(brzNorm(data2(2).toBreeze, 2) ~== 1.0 absTol 1E-5) + assert(brzNorm(data2(3).toBreeze, 2) ~== 1.0 absTol 1E-5) + assert(brzNorm(data2(4).toBreeze, 2) ~== 1.0 absTol 1E-5) + + assert(data2(0) ~== Vectors.sparse(3, Seq((0, -0.65617871), (1, 0.75460552))) absTol 1E-5) + assert(data2(1) ~== Vectors.dense(0.0, 0.0, 0.0) absTol 1E-5) + assert(data2(2) ~== Vectors.dense(0.184549876, -0.3383414, -0.922749378) absTol 1E-5) + assert(data2(3) ~== Vectors.sparse(3, Seq((1, 0.27352993), (2, 0.96186349))) absTol 1E-5) + assert(data2(4) ~== Vectors.dense(0.897906166, 0.113419726, 0.42532397) absTol 1E-5) + assert(data2(5) ~== Vectors.sparse(3, Seq()) absTol 1E-5) + } + + +3.结果： + + dataRDD: + [0.6,-1.1,-3.0] + (3,[0,1],[-2.0,2.3]) + (3,[1,2],[0.91,3.2]) + [0.0,0.0,0.0] + (3,[0,1,2],[5.7,0.72,2.7]) + (3,[],[]) + data2RDD: + [0.18454987557625951,-0.3383414385564758,-0.9227493778812975] + (3,[1,2],[0.2735299305180406,0.9618634919315713]) + (3,[0,1,2],[0.8979061661970154,0.11341972625646508,0.4253239734617441]) + (3,[],[]) + (3,[0,1],[-0.6561787149247866,0.7546055221635046]) + [0.0,0.0,0.0] + +结果分析： + + + scala> 0.6/Math.sqrt(0.6*0.6+1.1*1.1+3.0*3) + res32: Double = 0.18454987557625951 + + scala> -1.1/Math.sqrt(0.6*0.6+1.1*1.1+3.0*3) + res33: Double = -0.3383414385564758 + + scala> -3.0/Math.sqrt(0.6*0.6+1.1*1.1+3.0*3) + res34: Double = -0.9227493778812975 + +参考 + + 【1】http://spark.apache.org/docs/1.5.2/mllib-guide.html + 【2】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【3】https://github.com/xubo245/SparkLearning + 【4】book:Machine Learning with Spark ,Nick Pertreach + 【5】book:Spark MlLib机器学习实战 + 【6】https://github.com/endymecy/spark-ml-source-analysis/blob/master/%E7%89%B9%E5%BE%81%E6%8A%BD%E5%8F%96%E5%92%8C%E8%BD%AC%E6%8D%A2/normalizer.md + diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24060\344\271\213\345\275\222\344\270\200\345\214\226(Normalizer)Normalization using L^Inf distance.md" "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24060\344\271\213\345\275\222\344\270\200\345\214\226(Normalizer)Normalization using L^Inf distance.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24060\344\271\213\345\275\222\344\270\200\345\214\226(Normalizer)Normalization using L^Inf distance.md" rename to "docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24060\344\271\213\345\275\222\344\270\200\345\214\226(Normalizer)Normalization using L^Inf distance.md" diff --git "a/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24060\344\271\213\345\275\222\344\270\200\345\214\226\357\274\210Normalizer\357\274\211Normalization using L^Inf distance.md" "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24060\344\271\213\345\275\222\344\270\200\345\214\226\357\274\210Normalizer\357\274\211Normalization using L^Inf distance.md" new file mode 100644 index 0000000..76ebd2c --- /dev/null +++ "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24060\344\271\213\345\275\222\344\270\200\345\214\226\357\274\210Normalizer\357\274\211Normalization using L^Inf distance.md" @@ -0,0 +1,88 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark中组件Mllib的学习 + +1.解释 + +归一化 + +Normalization using L^Inf distance + +公式：分母为元素中绝对值最大值 + + +2.代码： + + test("Normalization using L^Inf distance.") { + val lInfNormalizer = new Normalizer(Double.PositiveInfinity) + + val dataInf = data.map(lInfNormalizer.transform) + val dataInfRDD = lInfNormalizer.transform(dataRDD) + + println("dataRDD:") + dataRDD.foreach(println) + println("dataInf:") + dataInf.foreach(println) + println("dataInfRDD:") + dataInfRDD.foreach(println) + + assert((data, dataInf, dataInfRDD.collect()).zipped.forall { + case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true + case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true + case _ => false + }, "The vector type should be preserved after normalization.") + + assert((dataInf, dataInfRDD.collect()).zipped.forall((v1, v2) => v1 ~== v2 absTol 1E-5)) + + assert(dataInf(0).toArray.map(math.abs).max ~== 1.0 absTol 1E-5) + assert(dataInf(2).toArray.map(math.abs).max ~== 1.0 absTol 1E-5) + assert(dataInf(3).toArray.map(math.abs).max ~== 1.0 absTol 1E-5) + assert(dataInf(4).toArray.map(math.abs).max ~== 1.0 absTol 1E-5) + + assert(dataInf(0) ~== Vectors.sparse(3, Seq((0, -0.86956522), (1, 1.0))) absTol 1E-5) + assert(dataInf(1) ~== Vectors.dense(0.0, 0.0, 0.0) absTol 1E-5) + assert(dataInf(2) ~== Vectors.dense(0.2, -0.36666667, -1.0) absTol 1E-5) + assert(dataInf(3) ~== Vectors.sparse(3, Seq((1, 0.284375), (2, 1.0))) absTol 1E-5) + assert(dataInf(4) ~== Vectors.dense(1.0, 0.12631579, 0.473684211) absTol 1E-5) + assert(dataInf(5) ~== Vectors.sparse(3, Seq()) absTol 1E-5) + } + + +3.结果： + + dataRDD: + [0.6,-1.1,-3.0] + (3,[1,2],[0.91,3.2]) + (3,[0,1],[-2.0,2.3]) + [0.0,0.0,0.0] + (3,[0,1,2],[5.7,0.72,2.7]) + (3,[],[]) + dataInf: + (3,[0,1],[-0.8695652173913044,1.0]) + [0.0,0.0,0.0] + [0.19999999999999998,-0.3666666666666667,-1.0] + (3,[1,2],[0.284375,1.0]) + (3,[0,1,2],[1.0,0.12631578947368421,0.4736842105263158]) + (3,[],[]) + dataInfRDD: + [0.19999999999999998,-0.3666666666666667,-1.0] + (3,[1,2],[0.284375,1.0]) + (3,[0,1],[-0.8695652173913044,1.0]) + [0.0,0.0,0.0] + (3,[0,1,2],[1.0,0.12631578947368421,0.4736842105263158]) + (3,[],[]) + + +结果分析： + + + -3/3=-1 + +参考 + + 【1】http://spark.apache.org/docs/1.5.2/mllib-guide.html + 【2】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【3】https://github.com/xubo245/SparkLearning + 【4】book:Machine Learning with Spark ,Nick Pertreach + 【5】book:Spark MlLib机器学习实战 diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24061\344\271\213\345\275\222\344\270\200\345\214\226(Normalizer)SparkWeb\345\256\236\344\276\213\345\210\206\346\236\220.md" "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24061\344\271\213\345\275\222\344\270\200\345\214\226(Normalizer)SparkWeb\345\256\236\344\276\213\345\210\206\346\236\220.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24061\344\271\213\345\275\222\344\270\200\345\214\226(Normalizer)SparkWeb\345\256\236\344\276\213\345\210\206\346\236\220.md" rename to "docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24061\344\271\213\345\275\222\344\270\200\345\214\226(Normalizer)SparkWeb\345\256\236\344\276\213\345\210\206\346\236\220.md" diff --git "a/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24061\344\271\213\345\275\222\344\270\200\345\214\226\357\274\210Normalizer\357\274\211SparkWeb\345\256\236\344\276\213\345\210\206\346\236\220.md" "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24061\344\271\213\345\275\222\344\270\200\345\214\226\357\274\210Normalizer\357\274\211SparkWeb\345\256\236\344\276\213\345\210\206\346\236\220.md" new file mode 100644 index 0000000..7ae9883 --- /dev/null +++ "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24061\344\271\213\345\275\222\344\270\200\345\214\226\357\274\210Normalizer\357\274\211SparkWeb\345\256\236\344\276\213\345\210\206\346\236\220.md" @@ -0,0 +1,119 @@ + + 更多代码请见：https://github.com/xubo245/SparkLearning + + Spark中组件Mllib的学习 + +1.解释 + + 这里分四种情况。当p=1时，即计算一阶范数，它的值为所有元素绝对值之和。当p=2时，它的值为所有元素的平方和。当p == Double.PositiveInfinity时，返回所有元素绝对值的最大值。 如果以上三种情况都不满足，那么按照下面的公式计算。 + + 分母为：其中本例中p为2 + + ![](http://i.imgur.com/Ll5M0O8.png) + + +2.代码： + + /** + * @author xubo + * ref:http://spark.apache.org/docs/1.5.2/mllib-guide.html + * more code:https://github.com/xubo245/SparkLearning + * more blog:http://blog.csdn.net/xubo245 + */ + package org.apache.spark.mllib.FeatureExtractionAndTransformation + + import org.apache.spark.util.SparkLearningFunSuite + + /** + * Created by xubo on 2016/6/13. + */ + class NormalizerSuite extends SparkLearningFunSuite { + test("testFunSuite") { + import org.apache.spark.SparkContext._ + import org.apache.spark.mllib.feature.Normalizer + import org.apache.spark.mllib.linalg.Vectors + import org.apache.spark.mllib.util.MLUtils + + val data = MLUtils.loadLibSVMFile(sc, "file/data/mllib/input/mllibFromSpark/sample_libsvm_data.txt") + + // val data = MLUtils.loadLibSVMFile(sc, "data/mllib/sample_libsvm_data.txt") + val normalizer1 = new Normalizer() + val normalizer2 = new Normalizer(p = Double.PositiveInfinity) + + + // Each sample in data1 will be normalized using $L^2$ norm. + val data1 = data.map(x => (x.label, normalizer1.transform(x.features))) + + // Each sample in data2 will be normalized using $L^\infty$ norm. + val data2 = data.map(x => (x.label, normalizer2.transform(x.features))) + println("model:") + + println("data:") + data.take(10).foreach { each => + println(each.label + " " + each.features) + // println(each.features.size) + } + println("data1:") + data1.take(10).foreach { each => + println(each._1 + " " + each._2) + // println(each._2.size) + } + println("data2:") + data2.take(10).foreach { each => + println(each._1 + " " + each._2) + // println(each._2.size) + } + + + } + } + + + +3.结果： + + model: + data: + 0.0 (692,[127,128,129,130,131,154,155,156,157,158,159,181,182,183,184,185,186,187,188,189,207,208,209,210,211,212,213,214,215,216,217,235,236,237,238,239,240,241,242,243,244,245,262,263,264,265,266,267,268,269,270,271,272,273,289,290,291,292,293,294,295,296,297,300,301,302,316,317,318,319,320,321,328,329,330,343,344,345,346,347,348,349,356,357,358,371,372,373,374,384,385,386,399,400,401,412,413,414,426,427,428,429,440,441,442,454,455,456,457,466,467,468,469,470,482,483,484,493,494,495,496,497,510,511,512,520,521,522,523,538,539,540,547,548,549,550,566,567,568,569,570,571,572,573,574,575,576,577,578,594,595,596,597,598,599,600,601,602,603,604,622,623,624,625,626,627,628,629,630,651,652,653,654,655,656,657],[51.0,159.0,253.0,159.0,50.0,48.0,238.0,252.0,252.0,252.0,237.0,54.0,227.0,253.0,252.0,239.0,233.0,252.0,57.0,6.0,10.0,60.0,224.0,252.0,253.0,252.0,202.0,84.0,252.0,253.0,122.0,163.0,252.0,252.0,252.0,253.0,252.0,252.0,96.0,189.0,253.0,167.0,51.0,238.0,253.0,253.0,190.0,114.0,253.0,228.0,47.0,79.0,255.0,168.0,48.0,238.0,252.0,252.0,179.0,12.0,75.0,121.0,21.0,253.0,243.0,50.0,38.0,165.0,253.0,233.0,208.0,84.0,253.0,252.0,165.0,7.0,178.0,252.0,240.0,71.0,19.0,28.0,253.0,252.0,195.0,57.0,252.0,252.0,63.0,253.0,252.0,195.0,198.0,253.0,190.0,255.0,253.0,196.0,76.0,246.0,252.0,112.0,253.0,252.0,148.0,85.0,252.0,230.0,25.0,7.0,135.0,253.0,186.0,12.0,85.0,252.0,223.0,7.0,131.0,252.0,225.0,71.0,85.0,252.0,145.0,48.0,165.0,252.0,173.0,86.0,253.0,225.0,114.0,238.0,253.0,162.0,85.0,252.0,249.0,146.0,48.0,29.0,85.0,178.0,225.0,253.0,223.0,167.0,56.0,85.0,252.0,252.0,252.0,229.0,215.0,252.0,252.0,252.0,196.0,130.0,28.0,199.0,252.0,252.0,253.0,252.0,252.0,233.0,145.0,25.0,128.0,252.0,253.0,252.0,141.0,37.0]) + 1.0 (692,[158,159,160,161,185,186,187,188,189,213,214,215,216,217,240,241,242,243,244,245,267,268,269,270,271,295,296,297,298,322,323,324,325,326,349,350,351,352,353,377,378,379,380,381,404,405,406,407,408,431,432,433,434,435,459,460,461,462,463,486,487,488,489,490,514,515,516,517,518,542,543,544,545,569,570,571,572,573,596,597,598,599,600,601,624,625,626,627,652,653,654,655,680,681,682,683],[124.0,253.0,255.0,63.0,96.0,244.0,251.0,253.0,62.0,127.0,251.0,251.0,253.0,62.0,68.0,236.0,251.0,211.0,31.0,8.0,60.0,228.0,251.0,251.0,94.0,155.0,253.0,253.0,189.0,20.0,253.0,251.0,235.0,66.0,32.0,205.0,253.0,251.0,126.0,104.0,251.0,253.0,184.0,15.0,80.0,240.0,251.0,193.0,23.0,32.0,253.0,253.0,253.0,159.0,151.0,251.0,251.0,251.0,39.0,48.0,221.0,251.0,251.0,172.0,234.0,251.0,251.0,196.0,12.0,253.0,251.0,251.0,89.0,159.0,255.0,253.0,253.0,31.0,48.0,228.0,253.0,247.0,140.0,8.0,64.0,251.0,253.0,220.0,64.0,251.0,253.0,220.0,24.0,193.0,253.0,220.0]) + 1.0 (692,[124,125,126,127,151,152,153,154,155,179,180,181,182,183,208,209,210,211,235,236,237,238,239,263,264,265,266,267,268,292,293,294,295,296,321,322,323,324,349,350,351,352,377,378,379,380,405,406,407,408,433,434,435,436,461,462,463,464,489,490,491,492,493,517,518,519,520,521,545,546,547,548,549,574,575,576,577,578,602,603,604,605,606,630,631,632,633,634,658,659,660,661,662],[145.0,255.0,211.0,31.0,32.0,237.0,253.0,252.0,71.0,11.0,175.0,253.0,252.0,71.0,144.0,253.0,252.0,71.0,16.0,191.0,253.0,252.0,71.0,26.0,221.0,253.0,252.0,124.0,31.0,125.0,253.0,252.0,252.0,108.0,253.0,252.0,252.0,108.0,255.0,253.0,253.0,108.0,253.0,252.0,252.0,108.0,253.0,252.0,252.0,108.0,253.0,252.0,252.0,108.0,255.0,253.0,253.0,170.0,253.0,252.0,252.0,252.0,42.0,149.0,252.0,252.0,252.0,144.0,109.0,252.0,252.0,252.0,144.0,218.0,253.0,253.0,255.0,35.0,175.0,252.0,252.0,253.0,35.0,73.0,252.0,252.0,253.0,35.0,31.0,211.0,252.0,253.0,35.0]) + 1.0 (692,[152,153,154,180,181,182,183,208,209,210,211,236,237,238,239,264,265,266,267,292,293,294,295,320,321,322,323,349,350,351,377,378,379,405,406,407,433,434,435,461,462,463,489,490,491,492,517,518,519,520,546,547,548,574,575,576,602,603,604,630,631,632,658,659,660,686,687,688],[5.0,63.0,197.0,20.0,254.0,230.0,24.0,20.0,254.0,254.0,48.0,20.0,254.0,255.0,48.0,20.0,254.0,254.0,57.0,20.0,254.0,254.0,108.0,16.0,239.0,254.0,143.0,178.0,254.0,143.0,178.0,254.0,143.0,178.0,254.0,162.0,178.0,254.0,240.0,113.0,254.0,240.0,83.0,254.0,245.0,31.0,79.0,254.0,246.0,38.0,214.0,254.0,150.0,144.0,241.0,8.0,144.0,240.0,2.0,144.0,254.0,82.0,230.0,247.0,40.0,168.0,209.0,31.0]) + 1.0 (692,[151,152,153,154,179,180,181,182,208,209,210,236,237,238,264,265,266,267,292,293,294,295,320,321,322,323,348,349,350,351,376,377,378,379,404,405,406,407,432,433,434,435,460,461,462,463,488,489,490,491,516,517,518,519,544,545,546,547,572,573,574,575,600,601,602,603,629,630,631,657,658,659,685,686,687],[1.0,168.0,242.0,28.0,10.0,228.0,254.0,100.0,190.0,254.0,122.0,83.0,254.0,162.0,29.0,254.0,248.0,25.0,29.0,255.0,254.0,103.0,29.0,254.0,254.0,109.0,29.0,254.0,254.0,109.0,29.0,254.0,254.0,109.0,29.0,255.0,254.0,109.0,29.0,254.0,254.0,109.0,29.0,254.0,254.0,63.0,29.0,254.0,254.0,28.0,29.0,254.0,254.0,28.0,29.0,254.0,254.0,35.0,29.0,254.0,254.0,109.0,6.0,212.0,254.0,109.0,203.0,254.0,178.0,155.0,254.0,190.0,32.0,199.0,104.0]) + 0.0 (692,[129,130,131,132,156,157,158,159,160,161,162,183,184,185,186,187,188,189,190,208,209,210,211,212,213,214,215,216,217,218,235,236,237,238,239,240,241,242,243,244,245,246,262,263,264,265,266,267,268,269,270,271,272,273,274,289,290,291,292,293,294,295,296,297,298,299,300,301,302,316,317,318,319,320,322,323,324,325,327,328,329,330,343,344,345,346,347,348,350,351,352,353,355,356,357,358,371,372,373,374,378,379,384,385,386,398,399,400,412,413,414,425,426,427,428,439,440,441,442,453,454,455,456,467,468,469,470,481,482,483,484,494,495,496,497,498,509,510,511,512,521,522,523,524,525,537,538,539,540,547,548,549,550,551,552,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,594,595,596,597,598,599,600,601,602,603,604,605,623,624,625,626,627,628,629,630,631,632,653,654,655,656,657,658],[64.0,253.0,255.0,63.0,96.0,205.0,251.0,253.0,205.0,111.0,4.0,96.0,189.0,251.0,251.0,253.0,251.0,251.0,31.0,16.0,64.0,223.0,244.0,251.0,251.0,211.0,213.0,251.0,251.0,31.0,80.0,181.0,251.0,253.0,251.0,251.0,251.0,94.0,96.0,251.0,251.0,31.0,92.0,253.0,253.0,253.0,255.0,253.0,253.0,253.0,95.0,96.0,253.0,253.0,31.0,92.0,236.0,251.0,243.0,220.0,233.0,251.0,251.0,243.0,82.0,96.0,251.0,251.0,31.0,80.0,253.0,251.0,251.0,188.0,96.0,251.0,251.0,109.0,96.0,251.0,251.0,31.0,96.0,240.0,253.0,243.0,188.0,42.0,96.0,204.0,109.0,4.0,12.0,197.0,251.0,31.0,221.0,251.0,253.0,121.0,36.0,23.0,190.0,251.0,31.0,48.0,234.0,253.0,191.0,253.0,31.0,44.0,221.0,251.0,251.0,12.0,197.0,251.0,31.0,190.0,251.0,251.0,251.0,96.0,251.0,251.0,31.0,190.0,251.0,251.0,113.0,40.0,234.0,251.0,219.0,23.0,190.0,251.0,251.0,94.0,40.0,217.0,253.0,231.0,47.0,191.0,253.0,253.0,253.0,12.0,174.0,253.0,253.0,219.0,39.0,67.0,236.0,251.0,251.0,191.0,190.0,111.0,72.0,190.0,191.0,197.0,251.0,243.0,121.0,39.0,63.0,236.0,251.0,253.0,251.0,251.0,251.0,251.0,253.0,251.0,188.0,94.0,27.0,129.0,253.0,251.0,251.0,251.0,251.0,229.0,168.0,15.0,95.0,212.0,251.0,211.0,94.0,59.0]) + 1.0 (692,[158,159,160,185,186,187,188,189,212,213,214,215,216,217,240,241,242,243,244,267,268,269,270,271,272,295,296,297,298,299,323,324,325,326,350,351,352,353,354,377,378,379,380,381,404,405,406,407,408,432,433,434,435,436,459,460,461,462,463,486,487,488,489,490,513,514,515,516,517,541,542,543,544,545,569,570,571,572,573,597,598,599,600,624,625,626,627,628,652,653,654,655,681,682,683],[121.0,254.0,136.0,13.0,230.0,253.0,248.0,99.0,4.0,118.0,253.0,253.0,225.0,42.0,61.0,253.0,253.0,253.0,74.0,32.0,206.0,253.0,253.0,186.0,9.0,211.0,253.0,253.0,239.0,69.0,254.0,253.0,253.0,133.0,142.0,255.0,253.0,186.0,8.0,149.0,229.0,254.0,207.0,21.0,54.0,229.0,253.0,254.0,105.0,152.0,254.0,254.0,213.0,26.0,112.0,251.0,253.0,253.0,26.0,29.0,212.0,253.0,250.0,149.0,36.0,214.0,253.0,253.0,137.0,75.0,253.0,253.0,253.0,59.0,93.0,253.0,253.0,189.0,17.0,224.0,253.0,253.0,84.0,43.0,235.0,253.0,126.0,1.0,99.0,248.0,253.0,119.0,225.0,235.0,49.0]) + 1.0 (692,[99,100,101,127,128,129,130,154,155,156,157,158,182,183,184,185,186,209,210,211,212,213,237,238,239,240,241,264,265,266,267,268,269,291,292,293,294,295,296,297,314,315,316,317,318,319,320,321,322,323,324,325,342,343,344,345,346,347,348,349,350,351,352,353,371,372,373,374,378,379,380,381,406,407,408,409,435,436,437,463,464,465,491,492,493,514,515,516,517,518,519,520,521,522,523,524,525,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,594,595,596,597,598,599,600,622,623,624,625],[166.0,222.0,55.0,197.0,254.0,218.0,5.0,29.0,249.0,254.0,254.0,9.0,45.0,254.0,254.0,174.0,2.0,4.0,164.0,254.0,254.0,85.0,146.0,254.0,254.0,254.0,85.0,101.0,245.0,254.0,254.0,254.0,85.0,97.0,248.0,254.0,204.0,254.0,254.0,85.0,12.0,59.0,98.0,151.0,237.0,254.0,254.0,109.0,35.0,254.0,254.0,85.0,41.0,216.0,254.0,254.0,239.0,153.0,37.0,4.0,32.0,254.0,254.0,85.0,7.0,44.0,44.0,30.0,32.0,254.0,254.0,96.0,19.0,230.0,254.0,174.0,197.0,254.0,110.0,197.0,254.0,85.0,197.0,253.0,63.0,37.0,54.0,54.0,45.0,26.0,84.0,221.0,84.0,21.0,31.0,162.0,78.0,6.0,41.0,141.0,244.0,254.0,254.0,248.0,236.0,254.0,254.0,254.0,233.0,239.0,254.0,138.0,23.0,167.0,254.0,254.0,254.0,254.0,229.0,228.0,185.0,138.0,138.0,138.0,138.0,138.0,138.0,44.0,113.0,254.0,254.0,254.0,179.0,64.0,5.0,32.0,209.0,183.0,97.0]) + 0.0 (692,[154,155,156,157,158,159,182,183,184,185,186,187,188,189,208,209,210,211,212,213,214,215,216,217,236,237,238,239,240,241,242,243,244,245,264,265,266,267,268,269,270,271,272,273,290,291,292,293,294,295,298,299,300,301,318,319,320,321,322,326,327,328,329,346,347,348,349,350,353,354,355,356,357,374,375,376,377,378,381,382,383,384,385,402,403,404,405,406,409,410,411,412,413,429,430,431,432,437,438,439,440,456,457,458,459,460,464,465,466,467,468,484,485,486,487,488,491,492,493,494,495,512,513,514,515,516,519,520,521,522,523,540,541,542,543,544,546,547,548,549,550,551,568,569,570,571,572,573,574,575,576,577,596,597,598,599,600,601,602,603,604,605,624,625,626,627,628,629,630,631,632,633,653,654,655,656,657,658,682,683,684,685,686],[53.0,255.0,253.0,253.0,253.0,124.0,180.0,253.0,251.0,251.0,251.0,251.0,145.0,62.0,32.0,217.0,241.0,253.0,251.0,251.0,251.0,251.0,253.0,107.0,37.0,251.0,251.0,253.0,251.0,251.0,251.0,251.0,253.0,107.0,166.0,251.0,251.0,253.0,251.0,96.0,148.0,251.0,253.0,107.0,73.0,253.0,253.0,253.0,253.0,130.0,110.0,253.0,255.0,108.0,73.0,251.0,251.0,251.0,251.0,109.0,251.0,253.0,107.0,202.0,251.0,251.0,251.0,225.0,6.0,129.0,251.0,253.0,107.0,150.0,251.0,251.0,251.0,71.0,115.0,251.0,251.0,253.0,107.0,253.0,251.0,251.0,173.0,20.0,217.0,251.0,251.0,253.0,107.0,182.0,255.0,253.0,216.0,218.0,253.0,253.0,182.0,63.0,221.0,253.0,251.0,215.0,84.0,236.0,251.0,251.0,77.0,109.0,251.0,253.0,251.0,215.0,11.0,160.0,251.0,251.0,96.0,109.0,251.0,253.0,251.0,137.0,150.0,251.0,251.0,251.0,71.0,109.0,251.0,253.0,251.0,35.0,130.0,253.0,251.0,251.0,173.0,20.0,110.0,253.0,255.0,253.0,98.0,150.0,253.0,255.0,253.0,164.0,109.0,251.0,253.0,251.0,251.0,251.0,251.0,253.0,251.0,35.0,93.0,241.0,253.0,251.0,251.0,251.0,251.0,216.0,112.0,5.0,103.0,253.0,251.0,251.0,251.0,251.0,124.0,251.0,225.0,71.0,71.0]) + 0.0 (692,[127,128,129,130,131,155,156,157,158,159,181,182,183,184,185,186,187,209,210,211,212,213,214,215,216,237,238,239,240,241,242,243,244,245,263,264,265,266,267,268,269,270,271,272,273,291,292,293,294,295,296,297,298,299,300,301,302,317,318,319,320,321,322,323,324,325,326,327,328,329,330,344,345,346,347,348,349,353,354,355,356,357,358,372,373,374,375,376,377,381,382,383,384,385,386,399,400,401,402,403,404,409,410,411,412,413,414,427,428,429,430,431,437,438,439,440,441,455,456,457,458,459,460,465,466,467,468,483,484,485,486,487,488,491,492,493,494,495,496,511,512,513,514,515,519,520,521,522,523,539,540,541,542,543,544,545,546,547,548,549,550,567,568,569,570,571,572,573,574,575,576,577,578,595,596,597,598,599,600,601,602,603,604,605,623,624,625,626,627,628,629,630,631,652,653,654,655,656,657,658],[73.0,253.0,227.0,73.0,21.0,73.0,251.0,251.0,251.0,174.0,16.0,166.0,228.0,251.0,251.0,251.0,122.0,62.0,220.0,253.0,251.0,251.0,251.0,251.0,79.0,79.0,231.0,253.0,251.0,251.0,251.0,251.0,232.0,77.0,145.0,253.0,253.0,253.0,255.0,253.0,253.0,253.0,253.0,255.0,108.0,144.0,251.0,251.0,251.0,253.0,168.0,107.0,169.0,251.0,253.0,189.0,20.0,27.0,89.0,236.0,251.0,235.0,215.0,164.0,15.0,6.0,129.0,251.0,253.0,251.0,35.0,47.0,211.0,253.0,251.0,251.0,142.0,37.0,251.0,251.0,253.0,251.0,35.0,109.0,251.0,253.0,251.0,251.0,142.0,11.0,148.0,251.0,253.0,251.0,164.0,11.0,150.0,253.0,255.0,211.0,25.0,11.0,150.0,253.0,255.0,211.0,25.0,140.0,251.0,251.0,253.0,107.0,37.0,251.0,251.0,211.0,46.0,190.0,251.0,251.0,253.0,128.0,5.0,37.0,251.0,251.0,51.0,115.0,251.0,251.0,253.0,188.0,20.0,32.0,109.0,129.0,251.0,173.0,103.0,217.0,251.0,251.0,201.0,30.0,73.0,251.0,251.0,251.0,71.0,166.0,253.0,253.0,255.0,149.0,73.0,150.0,253.0,255.0,253.0,253.0,143.0,140.0,251.0,251.0,253.0,251.0,251.0,251.0,251.0,253.0,251.0,230.0,61.0,190.0,251.0,251.0,253.0,251.0,251.0,251.0,251.0,242.0,215.0,55.0,21.0,189.0,251.0,253.0,251.0,251.0,251.0,173.0,103.0,31.0,200.0,253.0,251.0,96.0,71.0,20.0]) + data1: + 0.0 (692,[127,128,129,130,131,154,155,156,157,158,159,181,182,183,184,185,186,187,188,189,207,208,209,210,211,212,213,214,215,216,217,235,236,237,238,239,240,241,242,243,244,245,262,263,264,265,266,267,268,269,270,271,272,273,289,290,291,292,293,294,295,296,297,300,301,302,316,317,318,319,320,321,328,329,330,343,344,345,346,347,348,349,356,357,358,371,372,373,374,384,385,386,399,400,401,412,413,414,426,427,428,429,440,441,442,454,455,456,457,466,467,468,469,470,482,483,484,493,494,495,496,497,510,511,512,520,521,522,523,538,539,540,547,548,549,550,566,567,568,569,570,571,572,573,574,575,576,577,578,594,595,596,597,598,599,600,601,602,603,604,622,623,624,625,626,627,628,629,630,651,652,653,654,655,656,657],[0.01962941333426868,0.061197582748014116,0.09737728575627404,0.061197582748014116,0.019244522876733998,0.01847474196166464,0.09160392889325383,0.09699239529873935,0.09699239529873935,0.09699239529873935,0.09121903843571916,0.020784084706872718,0.08737013386037236,0.09737728575627404,0.09699239529873935,0.09198881935078851,0.08967947660558044,0.09699239529873935,0.02193875607947676,0.00230934274520808,0.0038489045753468,0.0230934274520808,0.08621546248776832,0.09699239529873935,0.09737728575627404,0.09699239529873935,0.07774787242200536,0.032330798432913116,0.09699239529873935,0.09737728575627404,0.04695663581923096,0.06273714457815284,0.09699239529873935,0.09699239529873935,0.09699239529873935,0.09737728575627404,0.09699239529873935,0.09699239529873935,0.03694948392332928,0.07274429647405452,0.09737728575627404,0.06427670640829156,0.01962941333426868,0.09160392889325383,0.09737728575627404,0.09737728575627404,0.0731291869315892,0.04387751215895352,0.09737728575627404,0.08775502431790703,0.01808985150412996,0.030406346145239717,0.09814706667134339,0.06466159686582623,0.01847474196166464,0.09160392889325383,0.09699239529873935,0.09699239529873935,0.06889539189870772,0.00461868549041616,0.028866784315100997,0.046571745361696275,0.008082699608228279,0.09737728575627404,0.09352838118092724,0.019244522876733998,0.01462583738631784,0.06350692549322219,0.09737728575627404,0.08967947660558044,0.08005721516721344,0.032330798432913116,0.09737728575627404,0.09699239529873935,0.06350692549322219,0.00269423320274276,0.06851050144117303,0.09699239529873935,0.0923737098083232,0.027327222484962277,0.00731291869315892,0.01077693281097104,0.09737728575627404,0.09699239529873935,0.0750536392192626,0.02193875607947676,0.09699239529873935,0.09699239529873935,0.024248098824684837,0.09737728575627404,0.09699239529873935,0.0750536392192626,0.07620831059186664,0.09737728575627404,0.0731291869315892,0.09814706667134339,0.09737728575627404,0.07543852967679728,0.02925167477263568,0.09468305255353128,0.09699239529873935,0.04310773124388416,0.09737728575627404,0.09699239529873935,0.056963787715132636,0.0327156888904478,0.09699239529873935,0.0885248052329764,0.009622261438366999,0.00269423320274276,0.051960211767181796,0.09737728575627404,0.07158962510145048,0.00461868549041616,0.0327156888904478,0.09699239529873935,0.08583057203023363,0.00269423320274276,0.05042064993704308,0.09699239529873935,0.086600352945303,0.027327222484962277,0.0327156888904478,0.09699239529873935,0.055809116342528595,0.01847474196166464,0.06350692549322219,0.09699239529873935,0.06658604915349964,0.03310057934798248,0.09737728575627404,0.086600352945303,0.04387751215895352,0.09160392889325383,0.09737728575627404,0.06235225412061816,0.0327156888904478,0.09699239529873935,0.09583772392613532,0.05619400680006328,0.01847474196166464,0.011161823268505719,0.0327156888904478,0.06851050144117303,0.086600352945303,0.09737728575627404,0.08583057203023363,0.06427670640829156,0.02155386562194208,0.0327156888904478,0.09699239529873935,0.09699239529873935,0.09699239529873935,0.08813991477544171,0.0827514483699562,0.09699239529873935,0.09699239529873935,0.09699239529873935,0.07543852967679728,0.0500357594795084,0.01077693281097104,0.07659320104940132,0.09699239529873935,0.09699239529873935,0.09737728575627404,0.09699239529873935,0.09699239529873935,0.08967947660558044,0.055809116342528595,0.009622261438366999,0.04926597856443904,0.09699239529873935,0.09737728575627404,0.09699239529873935,0.05426955451238988,0.014240946928783159]) + 1.0 (692,[158,159,160,161,185,186,187,188,189,213,214,215,216,217,240,241,242,243,244,245,267,268,269,270,271,295,296,297,298,322,323,324,325,326,349,350,351,352,353,377,378,379,380,381,404,405,406,407,408,431,432,433,434,435,459,460,461,462,463,486,487,488,489,490,514,515,516,517,518,542,543,544,545,569,570,571,572,573,596,597,598,599,600,601,624,625,626,627,652,653,654,655,680,681,682,683],[0.06362930562282999,0.12982430905303216,0.13085058817598103,0.03232779237288943,0.049261397901545795,0.12520605299976223,0.12879802993008327,0.12982430905303216,0.031814652811414997,0.06516872430725329,0.12879802993008327,0.12879802993008327,0.12982430905303216,0.031814652811414997,0.034893490180261603,0.12110093650796674,0.12879802993008327,0.10827244747110587,0.015907326405707498,0.0041051164917954835,0.030788373688466122,0.11699582001617127,0.12879802993008327,0.12879802993008327,0.048235118778596923,0.07953663202853749,0.12982430905303216,0.12982430905303216,0.09698337711866829,0.010262791229488708,0.12982430905303216,0.12879802993008327,0.12058779694649231,0.03386721105731273,0.016420465967181934,0.10519361010225925,0.12982430905303216,0.12879802993008327,0.06465558474577886,0.05336651439334128,0.12879802993008327,0.12982430905303216,0.0944176793112961,0.0076970934221165304,0.04105116491795483,0.12315349475386449,0.12879802993008327,0.09903593536456604,0.011802209913912013,0.016420465967181934,0.12982430905303216,0.12982430905303216,0.12982430905303216,0.08158919027443523,0.07748407378263975,0.12879802993008327,0.12879802993008327,0.12879802993008327,0.02001244289750298,0.024630698950772897,0.11340384308585022,0.12879802993008327,0.12879802993008327,0.08826000457360289,0.12007465738501788,0.12879802993008327,0.12879802993008327,0.10057535404898933,0.006157674737693224,0.12982430905303216,0.12879802993008327,0.12879802993008327,0.04566942097122475,0.08158919027443523,0.13085058817598103,0.12982430905303216,0.12982430905303216,0.015907326405707498,0.024630698950772897,0.11699582001617127,0.12982430905303216,0.12674547168418554,0.07183953860642095,0.0041051164917954835,0.03284093193436387,0.12879802993008327,0.12982430905303216,0.11289070352437579,0.03284093193436387,0.12879802993008327,0.12982430905303216,0.11289070352437579,0.012315349475386449,0.09903593536456604,0.12982430905303216,0.11289070352437579]) + 1.0 (692,[124,125,126,127,151,152,153,154,155,179,180,181,182,183,208,209,210,211,235,236,237,238,239,263,264,265,266,267,268,292,293,294,295,296,321,322,323,324,349,350,351,352,377,378,379,380,405,406,407,408,433,434,435,436,461,462,463,464,489,490,491,492,493,517,518,519,520,521,545,546,547,548,549,574,575,576,577,578,602,603,604,605,606,630,631,632,633,634,658,659,660,661,662],[0.07245002671640455,0.12741211594953902,0.10542728025628524,0.015489316056610627,0.015988971413275488,0.11841831952957158,0.12641280523620932,0.12591314987954447,0.03547553032320499,0.005496208923313448,0.08743968741635032,0.12641280523620932,0.12591314987954447,0.03547553032320499,0.0719503713597397,0.12641280523620932,0.12591314987954447,0.03547553032320499,0.007994485706637744,0.09543417312298806,0.12641280523620932,0.12591314987954447,0.03547553032320499,0.012991039273286332,0.11042383382293383,0.12641280523620932,0.12591314987954447,0.06195726422644251,0.015489316056610627,0.06245691958310737,0.12641280523620932,0.12591314987954447,0.12591314987954447,0.053962778519804765,0.12641280523620932,0.12591314987954447,0.12591314987954447,0.053962778519804765,0.12741211594953902,0.12641280523620932,0.12641280523620932,0.053962778519804765,0.12641280523620932,0.12591314987954447,0.12591314987954447,0.053962778519804765,0.12641280523620932,0.12591314987954447,0.12591314987954447,0.053962778519804765,0.12641280523620932,0.12591314987954447,0.12591314987954447,0.053962778519804765,0.12741211594953902,0.12641280523620932,0.12641280523620932,0.08494141063302602,0.12641280523620932,0.12591314987954447,0.12591314987954447,0.12591314987954447,0.020985524979924074,0.07444864814306398,0.12591314987954447,0.12591314987954447,0.12591314987954447,0.0719503713597397,0.054462433876469624,0.12591314987954447,0.12591314987954447,0.12591314987954447,0.0719503713597397,0.10892486775293925,0.12641280523620932,0.12641280523620932,0.12741211594953902,0.017487937483270065,0.08743968741635032,0.12591314987954447,0.12591314987954447,0.12641280523620932,0.017487937483270065,0.03647484103653471,0.12591314987954447,0.12591314987954447,0.12641280523620932,0.017487937483270065,0.015489316056610627,0.10542728025628524,0.12591314987954447,0.12641280523620932,0.017487937483270065]) + 1.0 (692,[152,153,154,180,181,182,183,208,209,210,211,236,237,238,239,264,265,266,267,292,293,294,295,320,321,322,323,349,350,351,377,378,379,405,406,407,433,434,435,461,462,463,489,490,491,492,517,518,519,520,546,547,548,574,575,576,602,603,604,630,631,632,658,659,660,686,687,688],[0.0032818022898294242,0.04135070885185074,0.1293030102192793,0.013127209159317697,0.16671555632333473,0.1509629053321535,0.015752650991181238,0.013127209159317697,0.16671555632333473,0.16671555632333473,0.031505301982362476,0.013127209159317697,0.16671555632333473,0.16737191678130064,0.031505301982362476,0.013127209159317697,0.16671555632333473,0.16671555632333473,0.037412546104055436,0.013127209159317697,0.16671555632333473,0.16671555632333473,0.07088692946031556,0.010501767327454158,0.15687014945384647,0.16671555632333473,0.09385954548912154,0.1168321615179275,0.16671555632333473,0.09385954548912154,0.1168321615179275,0.16671555632333473,0.09385954548912154,0.1168321615179275,0.16671555632333473,0.10633039419047334,0.1168321615179275,0.16671555632333473,0.15752650991181236,0.07416873175014499,0.16671555632333473,0.15752650991181236,0.05447791801116844,0.16671555632333473,0.1608083122016418,0.02034717419694243,0.0518524761793049,0.16671555632333473,0.16146467265960768,0.024941697402703623,0.14046113800469937,0.16671555632333473,0.09845406869488273,0.09451590594708742,0.15818287036977824,0.005250883663727079,0.09451590594708742,0.15752650991181236,0.0013127209159317697,0.09451590594708742,0.16671555632333473,0.05382155755320256,0.1509629053321535,0.16212103311757356,0.026254418318635394,0.11026855693826865,0.13717933571486993,0.02034717419694243]) + 1.0 (692,[151,152,153,154,179,180,181,182,208,209,210,236,237,238,264,265,266,267,292,293,294,295,320,321,322,323,348,349,350,351,376,377,378,379,404,405,406,407,432,433,434,435,460,461,462,463,488,489,490,491,516,517,518,519,544,545,546,547,572,573,574,575,600,601,602,603,629,630,631,657,658,659,685,686,687],[6.314518162060208E-4,0.1060839051226115,0.15281133952185705,0.017680650853768583,0.006314518162060208,0.14397101409497276,0.1603887613163293,0.06314518162060208,0.11997584507914395,0.1603887613163293,0.07703712157713455,0.05241050074509973,0.1603887613163293,0.10229519422537538,0.018312102669974604,0.1603887613163293,0.15660005041909317,0.01578629540515052,0.018312102669974604,0.1610202131325353,0.1603887613163293,0.06503953706922015,0.018312102669974604,0.1603887613163293,0.1603887613163293,0.06882824796645627,0.018312102669974604,0.1603887613163293,0.1603887613163293,0.06882824796645627,0.018312102669974604,0.1603887613163293,0.1603887613163293,0.06882824796645627,0.018312102669974604,0.1610202131325353,0.1603887613163293,0.06882824796645627,0.018312102669974604,0.1603887613163293,0.1603887613163293,0.06882824796645627,0.018312102669974604,0.1603887613163293,0.1603887613163293,0.03978146442097931,0.018312102669974604,0.1603887613163293,0.1603887613163293,0.017680650853768583,0.018312102669974604,0.1603887613163293,0.1603887613163293,0.017680650853768583,0.018312102669974604,0.1603887613163293,0.1603887613163293,0.02210081356721073,0.018312102669974604,0.1603887613163293,0.1603887613163293,0.06882824796645627,0.003788710897236125,0.13386778503567642,0.1603887613163293,0.06882824796645627,0.12818471868982223,0.1603887613163293,0.11239842328467171,0.09787503151193323,0.1603887613163293,0.11997584507914395,0.020206458118592667,0.12565891142499816,0.06567098888542616]) + 0.0 (692,[129,130,131,132,156,157,158,159,160,161,162,183,184,185,186,187,188,189,190,208,209,210,211,212,213,214,215,216,217,218,235,236,237,238,239,240,241,242,243,244,245,246,262,263,264,265,266,267,268,269,270,271,272,273,274,289,290,291,292,293,294,295,296,297,298,299,300,301,302,316,317,318,319,320,322,323,324,325,327,328,329,330,343,344,345,346,347,348,350,351,352,353,355,356,357,358,371,372,373,374,378,379,384,385,386,398,399,400,412,413,414,425,426,427,428,439,440,441,442,453,454,455,456,467,468,469,470,481,482,483,484,494,495,496,497,498,509,510,511,512,521,522,523,524,525,537,538,539,540,547,548,549,550,551,552,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,594,595,596,597,598,599,600,601,602,603,604,605,623,624,625,626,627,628,629,630,631,632,653,654,655,656,657,658],[0.02289277849949953,0.09049801500583408,0.09121341433394345,0.022535078835444852,0.0343391677492493,0.07332843113120945,0.08978261567772473,0.09049801500583408,0.07332843113120945,0.039704662710069505,0.0014307986562187207,0.0343391677492493,0.06760523650633456,0.08978261567772473,0.08978261567772473,0.09049801500583408,0.08978261567772473,0.08978261567772473,0.011088689585695087,0.005723194624874883,0.02289277849949953,0.07976702508419369,0.08727871802934196,0.08978261567772473,0.08978261567772473,0.07547462911553753,0.07619002844364688,0.08978261567772473,0.08978261567772473,0.011088689585695087,0.028615973124374416,0.06474363919389711,0.08978261567772473,0.09049801500583408,0.08978261567772473,0.08978261567772473,0.08978261567772473,0.03362376842113994,0.0343391677492493,0.08978261567772473,0.08978261567772473,0.011088689585695087,0.032908369093030576,0.09049801500583408,0.09049801500583408,0.09049801500583408,0.09121341433394345,0.09049801500583408,0.09049801500583408,0.09049801500583408,0.033981468085194616,0.0343391677492493,0.09049801500583408,0.09049801500583408,0.011088689585695087,0.032908369093030576,0.08441712071690452,0.08978261567772473,0.08692101836528729,0.07869392609202965,0.08334402172474048,0.08978261567772473,0.08978261567772473,0.08692101836528729,0.029331372452483778,0.0343391677492493,0.08978261567772473,0.08978261567772473,0.011088689585695087,0.028615973124374416,0.09049801500583408,0.08978261567772473,0.08978261567772473,0.06724753684227988,0.0343391677492493,0.08978261567772473,0.08978261567772473,0.03898926338196014,0.0343391677492493,0.08978261567772473,0.08978261567772473,0.011088689585695087,0.0343391677492493,0.08584791937312325,0.09049801500583408,0.08692101836528729,0.06724753684227988,0.015023385890296568,0.0343391677492493,0.07297073146715476,0.03898926338196014,0.0014307986562187207,0.004292395968656162,0.070466833818772,0.08978261567772473,0.011088689585695087,0.07905162575608432,0.08978261567772473,0.09049801500583408,0.04328165935061631,0.012877187905968488,0.008227092273257644,0.06796293617038923,0.08978261567772473,0.011088689585695087,0.01716958387462465,0.08370172138879517,0.09049801500583408,0.06832063583444392,0.09049801500583408,0.011088689585695087,0.015738785218405927,0.07905162575608432,0.08978261567772473,0.08978261567772473,0.004292395968656162,0.070466833818772,0.08978261567772473,0.011088689585695087,0.06796293617038923,0.08978261567772473,0.08978261567772473,0.08978261567772473,0.0343391677492493,0.08978261567772473,0.08978261567772473,0.011088689585695087,0.06796293617038923,0.08978261567772473,0.08978261567772473,0.04042006203817886,0.014307986562187208,0.08370172138879517,0.08978261567772473,0.07833622642797497,0.008227092273257644,0.06796293617038923,0.08978261567772473,0.08978261567772473,0.03362376842113994,0.014307986562187208,0.0776208270998656,0.09049801500583408,0.08262862239663113,0.01681188421056997,0.06832063583444392,0.09049801500583408,0.09049801500583408,0.09049801500583408,0.004292395968656162,0.06223974154551436,0.09049801500583408,0.09049801500583408,0.07833622642797497,0.013950286898132528,0.023965877491663574,0.08441712071690452,0.08978261567772473,0.08978261567772473,0.06832063583444392,0.06796293617038923,0.039704662710069505,0.025754375811936975,0.06796293617038923,0.06832063583444392,0.070466833818772,0.08978261567772473,0.08692101836528729,0.04328165935061631,0.013950286898132528,0.022535078835444852,0.08441712071690452,0.08978261567772473,0.09049801500583408,0.08978261567772473,0.08978261567772473,0.08978261567772473,0.08978261567772473,0.09049801500583408,0.08978261567772473,0.06724753684227988,0.03362376842113994,0.009657890929476365,0.046143256663053744,0.09049801500583408,0.08978261567772473,0.08978261567772473,0.08978261567772473,0.08978261567772473,0.08191322306852176,0.06009354356118627,0.005365494960820203,0.033981468085194616,0.0758323287795922,0.08978261567772473,0.07547462911553753,0.03362376842113994,0.02110428017922613]) + 1.0 (692,[158,159,160,185,186,187,188,189,212,213,214,215,216,217,240,241,242,243,244,267,268,269,270,271,272,295,296,297,298,299,323,324,325,326,350,351,352,353,354,377,378,379,380,381,404,405,406,407,408,432,433,434,435,436,459,460,461,462,463,486,487,488,489,490,513,514,515,516,517,541,542,543,544,545,569,570,571,572,573,597,598,599,600,624,625,626,627,628,652,653,654,655,681,682,683],[0.06343727696200163,0.13316585411858195,0.07130140220522498,0.006815575210793564,0.12058325372942459,0.13264157910236704,0.1300202040212926,0.051903226605274064,0.002097100064859558,0.06186445191335696,0.13264157910236704,0.13264157910236704,0.11796187864835014,0.02201955068102536,0.031980775989108264,0.13264157910236704,0.13264157910236704,0.13264157910236704,0.038796351199901824,0.016776800518876464,0.10800065334026725,0.13264157910236704,0.13264157910236704,0.09751515301596946,0.004718475145934006,0.1106220284213417,0.13264157910236704,0.13264157910236704,0.1253017288753586,0.036174976118827376,0.13316585411858195,0.13264157910236704,0.13264157910236704,0.0697285771565803,0.0744470523025143,0.13369012913479683,0.13264157910236704,0.09751515301596946,0.004194200129719116,0.07811697741601854,0.1200589787132097,0.13316585411858195,0.10852492835648213,0.01100977534051268,0.028310850875604036,0.1200589787132097,0.13264157910236704,0.13316585411858195,0.0550488767025634,0.0796898024646632,0.13316585411858195,0.13316585411858195,0.11167057845377147,0.013631150421587128,0.058718801816067624,0.13159302906993728,0.13264157910236704,0.13264157910236704,0.013631150421587128,0.015203975470231796,0.11114630343755658,0.13264157910236704,0.13106875405372237,0.07811697741601854,0.018873900583736024,0.11219485346998635,0.13264157910236704,0.13264157910236704,0.07182567722143987,0.03932062621611671,0.13264157910236704,0.13264157910236704,0.13264157910236704,0.03093222595667848,0.04875757650798473,0.13264157910236704,0.13264157910236704,0.09908797806461411,0.008912675275653122,0.11743760363213525,0.13264157910236704,0.13264157910236704,0.04403910136205072,0.022543825697240248,0.12320462881049904,0.13264157910236704,0.06605865204307608,5.242750162148895E-4,0.051903226605274064,0.1300202040212926,0.13264157910236704,0.062388726929571856,0.11796187864835014,0.12320462881049904,0.025689475794529588]) + 1.0 (692,[99,100,101,127,128,129,130,154,155,156,157,158,182,183,184,185,186,209,210,211,212,213,237,238,239,240,241,264,265,266,267,268,269,291,292,293,294,295,296,297,314,315,316,317,318,319,320,321,322,323,324,325,342,343,344,345,346,347,348,349,350,351,352,353,371,372,373,374,378,379,380,381,406,407,408,409,435,436,437,463,464,465,491,492,493,514,515,516,517,518,519,520,521,522,523,524,525,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,594,595,596,597,598,599,600,622,623,624,625],[0.07706738558874043,0.10306602169096613,0.02553437474325737,0.09145948771675823,0.11792238517795223,0.10120897625509286,0.0023213067948415793,0.01346357941008116,0.11560107838311065,0.11792238517795223,0.11792238517795223,0.004178352230714843,0.020891761153574214,0.11792238517795223,0.11792238517795223,0.08078147646048696,9.285227179366317E-4,0.0018570454358732634,0.0761388628708038,0.11792238517795223,0.11792238517795223,0.039462215512306845,0.06778215840937411,0.11792238517795223,0.11792238517795223,0.11792238517795223,0.039462215512306845,0.0468903972557999,0.11374403294723738,0.11792238517795223,0.11792238517795223,0.11792238517795223,0.039462215512306845,0.04503335181992664,0.11513681702414233,0.11792238517795223,0.09470931722953643,0.11792238517795223,0.11792238517795223,0.039462215512306845,0.005571136307619791,0.027391420179130637,0.045497613178894954,0.07010346520421569,0.11002994207549086,0.11792238517795223,0.11792238517795223,0.05060448812754643,0.016249147563891054,0.11792238517795223,0.11792238517795223,0.039462215512306845,0.01903471571770095,0.10028045353715623,0.11792238517795223,0.11792238517795223,0.1109584647934275,0.07103198792215233,0.017177670281827685,0.0018570454358732634,0.014856363486986107,0.11792238517795223,0.11792238517795223,0.039462215512306845,0.003249829512778211,0.020427499794605897,0.020427499794605897,0.013927840769049476,0.014856363486986107,0.11792238517795223,0.11792238517795223,0.044569090460958326,0.008820965820398001,0.10678011256271265,0.11792238517795223,0.08078147646048696,0.09145948771675823,0.11792238517795223,0.05106874948651474,0.09145948771675823,0.11792238517795223,0.039462215512306845,0.09145948771675823,0.1174581238189839,0.0292484656150039,0.017177670281827685,0.025070113384289057,0.025070113384289057,0.020891761153574214,0.012070795333176213,0.03899795415333853,0.1026017603319978,0.03899795415333853,0.009749488538334633,0.014392102128017791,0.07521034015286716,0.03621238599952864,0.0027855681538098954,0.01903471571770095,0.06546085161453254,0.11327977158826907,0.11792238517795223,0.11792238517795223,0.11513681702414233,0.10956568071652255,0.11792238517795223,0.11792238517795223,0.11792238517795223,0.1081728966396176,0.1109584647934275,0.11792238517795223,0.06406806753762759,0.010678011256271264,0.07753164694770875,0.11792238517795223,0.11792238517795223,0.11792238517795223,0.11792238517795223,0.10631585120374433,0.10585158984477602,0.08588835140913843,0.06406806753762759,0.06406806753762759,0.06406806753762759,0.06406806753762759,0.06406806753762759,0.06406806753762759,0.020427499794605897,0.05246153356341969,0.11792238517795223,0.11792238517795223,0.11792238517795223,0.08310278325532854,0.029712726973972214,0.0023213067948415793,0.014856363486986107,0.09703062402437801,0.0849598286912018,0.04503335181992664]) + 0.0 (692,[154,155,156,157,158,159,182,183,184,185,186,187,188,189,208,209,210,211,212,213,214,215,216,217,236,237,238,239,240,241,242,243,244,245,264,265,266,267,268,269,270,271,272,273,290,291,292,293,294,295,298,299,300,301,318,319,320,321,322,326,327,328,329,346,347,348,349,350,353,354,355,356,357,374,375,376,377,378,381,382,383,384,385,402,403,404,405,406,409,410,411,412,413,429,430,431,432,437,438,439,440,456,457,458,459,460,464,465,466,467,468,484,485,486,487,488,491,492,493,494,495,512,513,514,515,516,519,520,521,522,523,540,541,542,543,544,546,547,548,549,550,551,568,569,570,571,572,573,574,575,576,577,596,597,598,599,600,601,602,603,604,605,624,625,626,627,628,629,630,631,632,633,653,654,655,656,657,658,682,683,684,685,686],[0.018398329789892837,0.08852026597023911,0.0878259893743941,0.0878259893743941,0.0878259893743941,0.04304514894239078,0.06248489362605114,0.0878259893743941,0.08713171277854909,0.08713171277854909,0.08713171277854909,0.08713171277854909,0.05033505319876342,0.02152257447119539,0.011108425533520202,0.07532901064918388,0.08366032979932403,0.0878259893743941,0.08713171277854909,0.08713171277854909,0.08713171277854909,0.08713171277854909,0.0878259893743941,0.037143797877708176,0.012844117023132734,0.08713171277854909,0.08713171277854909,0.0878259893743941,0.08713171277854909,0.08713171277854909,0.08713171277854909,0.08713171277854909,0.0878259893743941,0.037143797877708176,0.05762495745513605,0.08713171277854909,0.08713171277854909,0.0878259893743941,0.08713171277854909,0.03332527660056061,0.051376468092530936,0.08713171277854909,0.0878259893743941,0.037143797877708176,0.02534109574834296,0.0878259893743941,0.0878259893743941,0.0878259893743941,0.0878259893743941,0.04512797872992582,0.0381852127714757,0.0878259893743941,0.08852026597023911,0.037490936175630686,0.02534109574834296,0.08713171277854909,0.08713171277854909,0.08713171277854909,0.08713171277854909,0.03783807447355319,0.08713171277854909,0.0878259893743941,0.037143797877708176,0.07012193618034628,0.08713171277854909,0.08713171277854909,0.08713171277854909,0.07810611703256393,0.002082829787535038,0.04478084043200332,0.08713171277854909,0.0878259893743941,0.037143797877708176,0.05207074468837595,0.08713171277854909,0.08713171277854909,0.08713171277854909,0.02464681915249795,0.03992090426108823,0.08713171277854909,0.08713171277854909,0.0878259893743941,0.037143797877708176,0.0878259893743941,0.08713171277854909,0.08713171277854909,0.06005492554059359,0.006942765958450127,0.07532901064918388,0.08713171277854909,0.08713171277854909,0.0878259893743941,0.037143797877708176,0.06317917022189615,0.08852026597023911,0.0878259893743941,0.07498187235126137,0.07567614894710638,0.0878259893743941,0.0878259893743941,0.06317917022189615,0.021869712769117897,0.0767175638408739,0.0878259893743941,0.08713171277854909,0.07463473405333886,0.029159617025490532,0.08192463830971149,0.08713171277854909,0.08713171277854909,0.026729648940032987,0.03783807447355319,0.08713171277854909,0.0878259893743941,0.08713171277854909,0.07463473405333886,0.0038185212771475697,0.05554212766760101,0.08713171277854909,0.08713171277854909,0.03332527660056061,0.03783807447355319,0.08713171277854909,0.0878259893743941,0.08713171277854909,0.04755794681538337,0.05207074468837595,0.08713171277854909,0.08713171277854909,0.08713171277854909,0.02464681915249795,0.03783807447355319,0.08713171277854909,0.0878259893743941,0.08713171277854909,0.012149840427287721,0.04512797872992582,0.0878259893743941,0.08713171277854909,0.08713171277854909,0.06005492554059359,0.006942765958450127,0.0381852127714757,0.0878259893743941,0.08852026597023911,0.0878259893743941,0.03401955319640562,0.05207074468837595,0.0878259893743941,0.08852026597023911,0.0878259893743941,0.05693068085929104,0.03783807447355319,0.08713171277854909,0.0878259893743941,0.08713171277854909,0.08713171277854909,0.08713171277854909,0.08713171277854909,0.0878259893743941,0.08713171277854909,0.012149840427287721,0.032283861706793086,0.08366032979932403,0.0878259893743941,0.08713171277854909,0.08713171277854909,0.08713171277854909,0.08713171277854909,0.07498187235126137,0.038879489367320705,0.0017356914896125317,0.03575524468601815,0.0878259893743941,0.08713171277854909,0.08713171277854909,0.08713171277854909,0.08713171277854909,0.04304514894239078,0.08713171277854909,0.07810611703256393,0.02464681915249795,0.02464681915249795]) + 0.0 (692,[127,128,129,130,131,155,156,157,158,159,181,182,183,184,185,186,187,209,210,211,212,213,214,215,216,237,238,239,240,241,242,243,244,245,263,264,265,266,267,268,269,270,271,272,273,291,292,293,294,295,296,297,298,299,300,301,302,317,318,319,320,321,322,323,324,325,326,327,328,329,330,344,345,346,347,348,349,353,354,355,356,357,358,372,373,374,375,376,377,381,382,383,384,385,386,399,400,401,402,403,404,409,410,411,412,413,414,427,428,429,430,431,437,438,439,440,441,455,456,457,458,459,460,465,466,467,468,483,484,485,486,487,488,491,492,493,494,495,496,511,512,513,514,515,519,520,521,522,523,539,540,541,542,543,544,545,546,547,548,549,550,567,568,569,570,571,572,573,574,575,576,577,578,595,596,597,598,599,600,601,602,603,604,605,623,624,625,626,627,628,629,630,631,652,653,654,655,656,657,658],[0.025243535061857707,0.08748786809109589,0.0784970199868726,0.025243535061857707,0.007261838853411121,0.025243535061857707,0.08679626439077101,0.08679626439077101,0.08679626439077101,0.060169521928263574,0.005532829602598949,0.0574031071269641,0.07884282183703503,0.08679626439077101,0.08679626439077101,0.08679626439077101,0.04218782571981699,0.021439714710070928,0.07607640703573555,0.08748786809109589,0.08679626439077101,0.08679626439077101,0.08679626439077101,0.08679626439077101,0.027318346162832314,0.027318346162832314,0.07988022738752233,0.08748786809109589,0.08679626439077101,0.08679626439077101,0.08679626439077101,0.08679626439077101,0.08022602923768477,0.026626742462507444,0.05014126827355298,0.08748786809109589,0.08748786809109589,0.08748786809109589,0.08817947179142076,0.08748786809109589,0.08748786809109589,0.08748786809109589,0.08748786809109589,0.08817947179142076,0.03734659981754291,0.04979546642339054,0.08679626439077101,0.08679626439077101,0.08679626439077101,0.08748786809109589,0.05809471082728897,0.037000797967380476,0.0584405126774514,0.08679626439077101,0.08748786809109589,0.0653565496807001,0.006916037003248687,0.009336649954385727,0.030776364664456657,0.0816092366383345,0.08679626439077101,0.08126343478817206,0.07434739778492339,0.05671150342663923,0.005187027752436515,0.002074811100974606,0.04460843867095403,0.08679626439077101,0.08748786809109589,0.08679626439077101,0.012103064755685202,0.016252686957634415,0.07296419038427364,0.08748786809109589,0.08679626439077101,0.08679626439077101,0.049103862723065675,0.01279466845601007,0.08679626439077101,0.08679626439077101,0.08748786809109589,0.08679626439077101,0.012103064755685202,0.03769240166770534,0.08679626439077101,0.08748786809109589,0.08679626439077101,0.08679626439077101,0.049103862723065675,0.0038038203517867775,0.05117867382404028,0.08679626439077101,0.08748786809109589,0.08679626439077101,0.05671150342663923,0.0038038203517867775,0.05187027752436515,0.08748786809109589,0.08817947179142076,0.07296419038427364,0.00864504625406086,0.0038038203517867775,0.05187027752436515,0.08748786809109589,0.08817947179142076,0.07296419038427364,0.00864504625406086,0.04841225902274081,0.08679626439077101,0.08679626439077101,0.08748786809109589,0.037000797967380476,0.01279466845601007,0.08679626439077101,0.08679626439077101,0.07296419038427364,0.01590688510747198,0.06570235153086253,0.08679626439077101,0.08679626439077101,0.08748786809109589,0.044262636820791595,0.0017290092508121717,0.01279466845601007,0.08679626439077101,0.08679626439077101,0.01763589435828415,0.03976721276867995,0.08679626439077101,0.08679626439077101,0.08748786809109589,0.06501074783053766,0.006916037003248687,0.011065659205197899,0.03769240166770534,0.04460843867095403,0.08679626439077101,0.05982372007810114,0.035617590566730736,0.07503900148524825,0.08679626439077101,0.08679626439077101,0.0695061718826493,0.01037405550487303,0.025243535061857707,0.08679626439077101,0.08679626439077101,0.08679626439077101,0.024551931361532837,0.0574031071269641,0.08748786809109589,0.08748786809109589,0.08817947179142076,0.051524475674202715,0.025243535061857707,0.05187027752436515,0.08748786809109589,0.08817947179142076,0.08748786809109589,0.08748786809109589,0.04944966457322811,0.04841225902274081,0.08679626439077101,0.08679626439077101,0.08748786809109589,0.08679626439077101,0.08679626439077101,0.08679626439077101,0.08679626439077101,0.08748786809109589,0.08679626439077101,0.0795344255373599,0.021093912859908494,0.06570235153086253,0.08679626439077101,0.08679626439077101,0.08748786809109589,0.08679626439077101,0.08679626439077101,0.08679626439077101,0.08679626439077101,0.08368404773930911,0.07434739778492339,0.019019101758933888,0.007261838853411121,0.0653565496807001,0.08679626439077101,0.08748786809109589,0.08679626439077101,0.08679626439077101,0.08679626439077101,0.05982372007810114,0.035617590566730736,0.010719857355035464,0.06916037003248687,0.08748786809109589,0.08679626439077101,0.0331969776155937,0.024551931361532837,0.006916037003248687]) + data2: + 0.0 (692,[127,128,129,130,131,154,155,156,157,158,159,181,182,183,184,185,186,187,188,189,207,208,209,210,211,212,213,214,215,216,217,235,236,237,238,239,240,241,242,243,244,245,262,263,264,265,266,267,268,269,270,271,272,273,289,290,291,292,293,294,295,296,297,300,301,302,316,317,318,319,320,321,328,329,330,343,344,345,346,347,348,349,356,357,358,371,372,373,374,384,385,386,399,400,401,412,413,414,426,427,428,429,440,441,442,454,455,456,457,466,467,468,469,470,482,483,484,493,494,495,496,497,510,511,512,520,521,522,523,538,539,540,547,548,549,550,566,567,568,569,570,571,572,573,574,575,576,577,578,594,595,596,597,598,599,600,601,602,603,604,622,623,624,625,626,627,628,629,630,651,652,653,654,655,656,657],[0.2,0.6235294117647059,0.9921568627450981,0.6235294117647059,0.19607843137254902,0.18823529411764706,0.9333333333333333,0.9882352941176471,0.9882352941176471,0.9882352941176471,0.9294117647058824,0.21176470588235294,0.8901960784313725,0.9921568627450981,0.9882352941176471,0.9372549019607843,0.9137254901960784,0.9882352941176471,0.2235294117647059,0.023529411764705882,0.0392156862745098,0.23529411764705882,0.8784313725490196,0.9882352941176471,0.9921568627450981,0.9882352941176471,0.792156862745098,0.32941176470588235,0.9882352941176471,0.9921568627450981,0.47843137254901963,0.6392156862745098,0.9882352941176471,0.9882352941176471,0.9882352941176471,0.9921568627450981,0.9882352941176471,0.9882352941176471,0.3764705882352941,0.7411764705882353,0.9921568627450981,0.6549019607843137,0.2,0.9333333333333333,0.9921568627450981,0.9921568627450981,0.7450980392156863,0.4470588235294118,0.9921568627450981,0.8941176470588236,0.1843137254901961,0.30980392156862746,1.0,0.6588235294117647,0.18823529411764706,0.9333333333333333,0.9882352941176471,0.9882352941176471,0.7019607843137254,0.047058823529411764,0.29411764705882354,0.4745098039215686,0.08235294117647059,0.9921568627450981,0.9529411764705882,0.19607843137254902,0.14901960784313725,0.6470588235294118,0.9921568627450981,0.9137254901960784,0.8156862745098039,0.32941176470588235,0.9921568627450981,0.9882352941176471,0.6470588235294118,0.027450980392156862,0.6980392156862745,0.9882352941176471,0.9411764705882353,0.2784313725490196,0.07450980392156863,0.10980392156862745,0.9921568627450981,0.9882352941176471,0.7647058823529411,0.2235294117647059,0.9882352941176471,0.9882352941176471,0.24705882352941178,0.9921568627450981,0.9882352941176471,0.7647058823529411,0.7764705882352941,0.9921568627450981,0.7450980392156863,1.0,0.9921568627450981,0.7686274509803922,0.2980392156862745,0.9647058823529412,0.9882352941176471,0.4392156862745098,0.9921568627450981,0.9882352941176471,0.5803921568627451,0.3333333333333333,0.9882352941176471,0.9019607843137255,0.09803921568627451,0.027450980392156862,0.5294117647058824,0.9921568627450981,0.7294117647058823,0.047058823529411764,0.3333333333333333,0.9882352941176471,0.8745098039215686,0.027450980392156862,0.5137254901960784,0.9882352941176471,0.8823529411764706,0.2784313725490196,0.3333333333333333,0.9882352941176471,0.5686274509803921,0.18823529411764706,0.6470588235294118,0.9882352941176471,0.6784313725490196,0.33725490196078434,0.9921568627450981,0.8823529411764706,0.4470588235294118,0.9333333333333333,0.9921568627450981,0.6352941176470588,0.3333333333333333,0.9882352941176471,0.9764705882352941,0.5725490196078431,0.18823529411764706,0.11372549019607843,0.3333333333333333,0.6980392156862745,0.8823529411764706,0.9921568627450981,0.8745098039215686,0.6549019607843137,0.2196078431372549,0.3333333333333333,0.9882352941176471,0.9882352941176471,0.9882352941176471,0.8980392156862745,0.8431372549019608,0.9882352941176471,0.9882352941176471,0.9882352941176471,0.7686274509803922,0.5098039215686274,0.10980392156862745,0.7803921568627451,0.9882352941176471,0.9882352941176471,0.9921568627450981,0.9882352941176471,0.9882352941176471,0.9137254901960784,0.5686274509803921,0.09803921568627451,0.5019607843137255,0.9882352941176471,0.9921568627450981,0.9882352941176471,0.5529411764705883,0.1450980392156863]) + 1.0 (692,[158,159,160,161,185,186,187,188,189,213,214,215,216,217,240,241,242,243,244,245,267,268,269,270,271,295,296,297,298,322,323,324,325,326,349,350,351,352,353,377,378,379,380,381,404,405,406,407,408,431,432,433,434,435,459,460,461,462,463,486,487,488,489,490,514,515,516,517,518,542,543,544,545,569,570,571,572,573,596,597,598,599,600,601,624,625,626,627,652,653,654,655,680,681,682,683],[0.48627450980392156,0.9921568627450981,1.0,0.24705882352941178,0.3764705882352941,0.9568627450980393,0.984313725490196,0.9921568627450981,0.24313725490196078,0.4980392156862745,0.984313725490196,0.984313725490196,0.9921568627450981,0.24313725490196078,0.26666666666666666,0.9254901960784314,0.984313725490196,0.8274509803921568,0.12156862745098039,0.03137254901960784,0.23529411764705882,0.8941176470588236,0.984313725490196,0.984313725490196,0.3686274509803922,0.6078431372549019,0.9921568627450981,0.9921568627450981,0.7411764705882353,0.0784313725490196,0.9921568627450981,0.984313725490196,0.9215686274509803,0.25882352941176473,0.12549019607843137,0.803921568627451,0.9921568627450981,0.984313725490196,0.49411764705882355,0.40784313725490196,0.984313725490196,0.9921568627450981,0.7215686274509804,0.058823529411764705,0.3137254901960784,0.9411764705882353,0.984313725490196,0.7568627450980392,0.09019607843137255,0.12549019607843137,0.9921568627450981,0.9921568627450981,0.9921568627450981,0.6235294117647059,0.592156862745098,0.984313725490196,0.984313725490196,0.984313725490196,0.15294117647058825,0.18823529411764706,0.8666666666666667,0.984313725490196,0.984313725490196,0.6745098039215687,0.9176470588235294,0.984313725490196,0.984313725490196,0.7686274509803922,0.047058823529411764,0.9921568627450981,0.984313725490196,0.984313725490196,0.34901960784313724,0.6235294117647059,1.0,0.9921568627450981,0.9921568627450981,0.12156862745098039,0.18823529411764706,0.8941176470588236,0.9921568627450981,0.9686274509803922,0.5490196078431373,0.03137254901960784,0.25098039215686274,0.984313725490196,0.9921568627450981,0.8627450980392157,0.25098039215686274,0.984313725490196,0.9921568627450981,0.8627450980392157,0.09411764705882353,0.7568627450980392,0.9921568627450981,0.8627450980392157]) + 1.0 (692,[124,125,126,127,151,152,153,154,155,179,180,181,182,183,208,209,210,211,235,236,237,238,239,263,264,265,266,267,268,292,293,294,295,296,321,322,323,324,349,350,351,352,377,378,379,380,405,406,407,408,433,434,435,436,461,462,463,464,489,490,491,492,493,517,518,519,520,521,545,546,547,548,549,574,575,576,577,578,602,603,604,605,606,630,631,632,633,634,658,659,660,661,662],[0.5686274509803921,1.0,0.8274509803921568,0.12156862745098039,0.12549019607843137,0.9294117647058824,0.9921568627450981,0.9882352941176471,0.2784313725490196,0.043137254901960784,0.6862745098039216,0.9921568627450981,0.9882352941176471,0.2784313725490196,0.5647058823529412,0.9921568627450981,0.9882352941176471,0.2784313725490196,0.06274509803921569,0.7490196078431373,0.9921568627450981,0.9882352941176471,0.2784313725490196,0.10196078431372549,0.8666666666666667,0.9921568627450981,0.9882352941176471,0.48627450980392156,0.12156862745098039,0.49019607843137253,0.9921568627450981,0.9882352941176471,0.9882352941176471,0.4235294117647059,0.9921568627450981,0.9882352941176471,0.9882352941176471,0.4235294117647059,1.0,0.9921568627450981,0.9921568627450981,0.4235294117647059,0.9921568627450981,0.9882352941176471,0.9882352941176471,0.4235294117647059,0.9921568627450981,0.9882352941176471,0.9882352941176471,0.4235294117647059,0.9921568627450981,0.9882352941176471,0.9882352941176471,0.4235294117647059,1.0,0.9921568627450981,0.9921568627450981,0.6666666666666666,0.9921568627450981,0.9882352941176471,0.9882352941176471,0.9882352941176471,0.16470588235294117,0.5843137254901961,0.9882352941176471,0.9882352941176471,0.9882352941176471,0.5647058823529412,0.42745098039215684,0.9882352941176471,0.9882352941176471,0.9882352941176471,0.5647058823529412,0.8549019607843137,0.9921568627450981,0.9921568627450981,1.0,0.13725490196078433,0.6862745098039216,0.9882352941176471,0.9882352941176471,0.9921568627450981,0.13725490196078433,0.28627450980392155,0.9882352941176471,0.9882352941176471,0.9921568627450981,0.13725490196078433,0.12156862745098039,0.8274509803921568,0.9882352941176471,0.9921568627450981,0.13725490196078433]) + 1.0 (692,[152,153,154,180,181,182,183,208,209,210,211,236,237,238,239,264,265,266,267,292,293,294,295,320,321,322,323,349,350,351,377,378,379,405,406,407,433,434,435,461,462,463,489,490,491,492,517,518,519,520,546,547,548,574,575,576,602,603,604,630,631,632,658,659,660,686,687,688],[0.0196078431372549,0.24705882352941178,0.7725490196078432,0.0784313725490196,0.996078431372549,0.9019607843137255,0.09411764705882353,0.0784313725490196,0.996078431372549,0.996078431372549,0.18823529411764706,0.0784313725490196,0.996078431372549,1.0,0.18823529411764706,0.0784313725490196,0.996078431372549,0.996078431372549,0.2235294117647059,0.0784313725490196,0.996078431372549,0.996078431372549,0.4235294117647059,0.06274509803921569,0.9372549019607843,0.996078431372549,0.5607843137254902,0.6980392156862745,0.996078431372549,0.5607843137254902,0.6980392156862745,0.996078431372549,0.5607843137254902,0.6980392156862745,0.996078431372549,0.6352941176470588,0.6980392156862745,0.996078431372549,0.9411764705882353,0.44313725490196076,0.996078431372549,0.9411764705882353,0.3254901960784314,0.996078431372549,0.9607843137254902,0.12156862745098039,0.30980392156862746,0.996078431372549,0.9647058823529412,0.14901960784313725,0.8392156862745098,0.996078431372549,0.5882352941176471,0.5647058823529412,0.9450980392156862,0.03137254901960784,0.5647058823529412,0.9411764705882353,0.00784313725490196,0.5647058823529412,0.996078431372549,0.3215686274509804,0.9019607843137255,0.9686274509803922,0.1568627450980392,0.6588235294117647,0.8196078431372549,0.12156862745098039]) + 1.0 (692,[151,152,153,154,179,180,181,182,208,209,210,236,237,238,264,265,266,267,292,293,294,295,320,321,322,323,348,349,350,351,376,377,378,379,404,405,406,407,432,433,434,435,460,461,462,463,488,489,490,491,516,517,518,519,544,545,546,547,572,573,574,575,600,601,602,603,629,630,631,657,658,659,685,686,687],[0.00392156862745098,0.6588235294117647,0.9490196078431372,0.10980392156862745,0.0392156862745098,0.8941176470588236,0.996078431372549,0.39215686274509803,0.7450980392156863,0.996078431372549,0.47843137254901963,0.3254901960784314,0.996078431372549,0.6352941176470588,0.11372549019607843,0.996078431372549,0.9725490196078431,0.09803921568627451,0.11372549019607843,1.0,0.996078431372549,0.403921568627451,0.11372549019607843,0.996078431372549,0.996078431372549,0.42745098039215684,0.11372549019607843,0.996078431372549,0.996078431372549,0.42745098039215684,0.11372549019607843,0.996078431372549,0.996078431372549,0.42745098039215684,0.11372549019607843,1.0,0.996078431372549,0.42745098039215684,0.11372549019607843,0.996078431372549,0.996078431372549,0.42745098039215684,0.11372549019607843,0.996078431372549,0.996078431372549,0.24705882352941178,0.11372549019607843,0.996078431372549,0.996078431372549,0.10980392156862745,0.11372549019607843,0.996078431372549,0.996078431372549,0.10980392156862745,0.11372549019607843,0.996078431372549,0.996078431372549,0.13725490196078433,0.11372549019607843,0.996078431372549,0.996078431372549,0.42745098039215684,0.023529411764705882,0.8313725490196079,0.996078431372549,0.42745098039215684,0.796078431372549,0.996078431372549,0.6980392156862745,0.6078431372549019,0.996078431372549,0.7450980392156863,0.12549019607843137,0.7803921568627451,0.40784313725490196]) + 0.0 (692,[129,130,131,132,156,157,158,159,160,161,162,183,184,185,186,187,188,189,190,208,209,210,211,212,213,214,215,216,217,218,235,236,237,238,239,240,241,242,243,244,245,246,262,263,264,265,266,267,268,269,270,271,272,273,274,289,290,291,292,293,294,295,296,297,298,299,300,301,302,316,317,318,319,320,322,323,324,325,327,328,329,330,343,344,345,346,347,348,350,351,352,353,355,356,357,358,371,372,373,374,378,379,384,385,386,398,399,400,412,413,414,425,426,427,428,439,440,441,442,453,454,455,456,467,468,469,470,481,482,483,484,494,495,496,497,498,509,510,511,512,521,522,523,524,525,537,538,539,540,547,548,549,550,551,552,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,594,595,596,597,598,599,600,601,602,603,604,605,623,624,625,626,627,628,629,630,631,632,653,654,655,656,657,658],[0.25098039215686274,0.9921568627450981,1.0,0.24705882352941178,0.3764705882352941,0.803921568627451,0.984313725490196,0.9921568627450981,0.803921568627451,0.43529411764705883,0.01568627450980392,0.3764705882352941,0.7411764705882353,0.984313725490196,0.984313725490196,0.9921568627450981,0.984313725490196,0.984313725490196,0.12156862745098039,0.06274509803921569,0.25098039215686274,0.8745098039215686,0.9568627450980393,0.984313725490196,0.984313725490196,0.8274509803921568,0.8352941176470589,0.984313725490196,0.984313725490196,0.12156862745098039,0.3137254901960784,0.7098039215686275,0.984313725490196,0.9921568627450981,0.984313725490196,0.984313725490196,0.984313725490196,0.3686274509803922,0.3764705882352941,0.984313725490196,0.984313725490196,0.12156862745098039,0.3607843137254902,0.9921568627450981,0.9921568627450981,0.9921568627450981,1.0,0.9921568627450981,0.9921568627450981,0.9921568627450981,0.37254901960784315,0.3764705882352941,0.9921568627450981,0.9921568627450981,0.12156862745098039,0.3607843137254902,0.9254901960784314,0.984313725490196,0.9529411764705882,0.8627450980392157,0.9137254901960784,0.984313725490196,0.984313725490196,0.9529411764705882,0.3215686274509804,0.3764705882352941,0.984313725490196,0.984313725490196,0.12156862745098039,0.3137254901960784,0.9921568627450981,0.984313725490196,0.984313725490196,0.7372549019607844,0.3764705882352941,0.984313725490196,0.984313725490196,0.42745098039215684,0.3764705882352941,0.984313725490196,0.984313725490196,0.12156862745098039,0.3764705882352941,0.9411764705882353,0.9921568627450981,0.9529411764705882,0.7372549019607844,0.16470588235294117,0.3764705882352941,0.8,0.42745098039215684,0.01568627450980392,0.047058823529411764,0.7725490196078432,0.984313725490196,0.12156862745098039,0.8666666666666667,0.984313725490196,0.9921568627450981,0.4745098039215686,0.1411764705882353,0.09019607843137255,0.7450980392156863,0.984313725490196,0.12156862745098039,0.18823529411764706,0.9176470588235294,0.9921568627450981,0.7490196078431373,0.9921568627450981,0.12156862745098039,0.17254901960784313,0.8666666666666667,0.984313725490196,0.984313725490196,0.047058823529411764,0.7725490196078432,0.984313725490196,0.12156862745098039,0.7450980392156863,0.984313725490196,0.984313725490196,0.984313725490196,0.3764705882352941,0.984313725490196,0.984313725490196,0.12156862745098039,0.7450980392156863,0.984313725490196,0.984313725490196,0.44313725490196076,0.1568627450980392,0.9176470588235294,0.984313725490196,0.8588235294117647,0.09019607843137255,0.7450980392156863,0.984313725490196,0.984313725490196,0.3686274509803922,0.1568627450980392,0.8509803921568627,0.9921568627450981,0.9058823529411765,0.1843137254901961,0.7490196078431373,0.9921568627450981,0.9921568627450981,0.9921568627450981,0.047058823529411764,0.6823529411764706,0.9921568627450981,0.9921568627450981,0.8588235294117647,0.15294117647058825,0.2627450980392157,0.9254901960784314,0.984313725490196,0.984313725490196,0.7490196078431373,0.7450980392156863,0.43529411764705883,0.2823529411764706,0.7450980392156863,0.7490196078431373,0.7725490196078432,0.984313725490196,0.9529411764705882,0.4745098039215686,0.15294117647058825,0.24705882352941178,0.9254901960784314,0.984313725490196,0.9921568627450981,0.984313725490196,0.984313725490196,0.984313725490196,0.984313725490196,0.9921568627450981,0.984313725490196,0.7372549019607844,0.3686274509803922,0.10588235294117647,0.5058823529411764,0.9921568627450981,0.984313725490196,0.984313725490196,0.984313725490196,0.984313725490196,0.8980392156862745,0.6588235294117647,0.058823529411764705,0.37254901960784315,0.8313725490196079,0.984313725490196,0.8274509803921568,0.3686274509803922,0.23137254901960785]) + 1.0 (692,[158,159,160,185,186,187,188,189,212,213,214,215,216,217,240,241,242,243,244,267,268,269,270,271,272,295,296,297,298,299,323,324,325,326,350,351,352,353,354,377,378,379,380,381,404,405,406,407,408,432,433,434,435,436,459,460,461,462,463,486,487,488,489,490,513,514,515,516,517,541,542,543,544,545,569,570,571,572,573,597,598,599,600,624,625,626,627,628,652,653,654,655,681,682,683],[0.4745098039215686,0.996078431372549,0.5333333333333333,0.050980392156862744,0.9019607843137255,0.9921568627450981,0.9725490196078431,0.38823529411764707,0.01568627450980392,0.4627450980392157,0.9921568627450981,0.9921568627450981,0.8823529411764706,0.16470588235294117,0.23921568627450981,0.9921568627450981,0.9921568627450981,0.9921568627450981,0.2901960784313726,0.12549019607843137,0.807843137254902,0.9921568627450981,0.9921568627450981,0.7294117647058823,0.03529411764705882,0.8274509803921568,0.9921568627450981,0.9921568627450981,0.9372549019607843,0.27058823529411763,0.996078431372549,0.9921568627450981,0.9921568627450981,0.5215686274509804,0.5568627450980392,1.0,0.9921568627450981,0.7294117647058823,0.03137254901960784,0.5843137254901961,0.8980392156862745,0.996078431372549,0.8117647058823529,0.08235294117647059,0.21176470588235294,0.8980392156862745,0.9921568627450981,0.996078431372549,0.4117647058823529,0.596078431372549,0.996078431372549,0.996078431372549,0.8352941176470589,0.10196078431372549,0.4392156862745098,0.984313725490196,0.9921568627450981,0.9921568627450981,0.10196078431372549,0.11372549019607843,0.8313725490196079,0.9921568627450981,0.9803921568627451,0.5843137254901961,0.1411764705882353,0.8392156862745098,0.9921568627450981,0.9921568627450981,0.5372549019607843,0.29411764705882354,0.9921568627450981,0.9921568627450981,0.9921568627450981,0.23137254901960785,0.36470588235294116,0.9921568627450981,0.9921568627450981,0.7411764705882353,0.06666666666666667,0.8784313725490196,0.9921568627450981,0.9921568627450981,0.32941176470588235,0.16862745098039217,0.9215686274509803,0.9921568627450981,0.49411764705882355,0.00392156862745098,0.38823529411764707,0.9725490196078431,0.9921568627450981,0.4666666666666667,0.8823529411764706,0.9215686274509803,0.19215686274509805]) + 1.0 (692,[99,100,101,127,128,129,130,154,155,156,157,158,182,183,184,185,186,209,210,211,212,213,237,238,239,240,241,264,265,266,267,268,269,291,292,293,294,295,296,297,314,315,316,317,318,319,320,321,322,323,324,325,342,343,344,345,346,347,348,349,350,351,352,353,371,372,373,374,378,379,380,381,406,407,408,409,435,436,437,463,464,465,491,492,493,514,515,516,517,518,519,520,521,522,523,524,525,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,594,595,596,597,598,599,600,622,623,624,625],[0.6535433070866141,0.8740157480314961,0.21653543307086615,0.7755905511811023,1.0,0.8582677165354331,0.01968503937007874,0.1141732283464567,0.9803149606299213,1.0,1.0,0.03543307086614173,0.17716535433070865,1.0,1.0,0.6850393700787402,0.007874015748031496,0.015748031496062992,0.6456692913385826,1.0,1.0,0.3346456692913386,0.5748031496062992,1.0,1.0,1.0,0.3346456692913386,0.39763779527559057,0.9645669291338582,1.0,1.0,1.0,0.3346456692913386,0.38188976377952755,0.9763779527559056,1.0,0.8031496062992126,1.0,1.0,0.3346456692913386,0.047244094488188976,0.23228346456692914,0.3858267716535433,0.594488188976378,0.9330708661417323,1.0,1.0,0.42913385826771655,0.1377952755905512,1.0,1.0,0.3346456692913386,0.16141732283464566,0.8503937007874016,1.0,1.0,0.9409448818897638,0.6023622047244095,0.14566929133858267,0.015748031496062992,0.12598425196850394,1.0,1.0,0.3346456692913386,0.027559055118110236,0.1732283464566929,0.1732283464566929,0.11811023622047244,0.12598425196850394,1.0,1.0,0.3779527559055118,0.07480314960629922,0.905511811023622,1.0,0.6850393700787402,0.7755905511811023,1.0,0.4330708661417323,0.7755905511811023,1.0,0.3346456692913386,0.7755905511811023,0.9960629921259843,0.24803149606299213,0.14566929133858267,0.2125984251968504,0.2125984251968504,0.17716535433070865,0.10236220472440945,0.33070866141732286,0.8700787401574803,0.33070866141732286,0.08267716535433071,0.1220472440944882,0.6377952755905512,0.30708661417322836,0.023622047244094488,0.16141732283464566,0.5551181102362205,0.9606299212598425,1.0,1.0,0.9763779527559056,0.9291338582677166,1.0,1.0,1.0,0.9173228346456693,0.9409448818897638,1.0,0.5433070866141733,0.09055118110236221,0.65748031496063,1.0,1.0,1.0,1.0,0.9015748031496063,0.8976377952755905,0.7283464566929134,0.5433070866141733,0.5433070866141733,0.5433070866141733,0.5433070866141733,0.5433070866141733,0.5433070866141733,0.1732283464566929,0.4448818897637795,1.0,1.0,1.0,0.7047244094488189,0.25196850393700787,0.01968503937007874,0.12598425196850394,0.8228346456692913,0.7204724409448819,0.38188976377952755]) + 0.0 (692,[154,155,156,157,158,159,182,183,184,185,186,187,188,189,208,209,210,211,212,213,214,215,216,217,236,237,238,239,240,241,242,243,244,245,264,265,266,267,268,269,270,271,272,273,290,291,292,293,294,295,298,299,300,301,318,319,320,321,322,326,327,328,329,346,347,348,349,350,353,354,355,356,357,374,375,376,377,378,381,382,383,384,385,402,403,404,405,406,409,410,411,412,413,429,430,431,432,437,438,439,440,456,457,458,459,460,464,465,466,467,468,484,485,486,487,488,491,492,493,494,495,512,513,514,515,516,519,520,521,522,523,540,541,542,543,544,546,547,548,549,550,551,568,569,570,571,572,573,574,575,576,577,596,597,598,599,600,601,602,603,604,605,624,625,626,627,628,629,630,631,632,633,653,654,655,656,657,658,682,683,684,685,686],[0.20784313725490197,1.0,0.9921568627450981,0.9921568627450981,0.9921568627450981,0.48627450980392156,0.7058823529411765,0.9921568627450981,0.984313725490196,0.984313725490196,0.984313725490196,0.984313725490196,0.5686274509803921,0.24313725490196078,0.12549019607843137,0.8509803921568627,0.9450980392156862,0.9921568627450981,0.984313725490196,0.984313725490196,0.984313725490196,0.984313725490196,0.9921568627450981,0.4196078431372549,0.1450980392156863,0.984313725490196,0.984313725490196,0.9921568627450981,0.984313725490196,0.984313725490196,0.984313725490196,0.984313725490196,0.9921568627450981,0.4196078431372549,0.6509803921568628,0.984313725490196,0.984313725490196,0.9921568627450981,0.984313725490196,0.3764705882352941,0.5803921568627451,0.984313725490196,0.9921568627450981,0.4196078431372549,0.28627450980392155,0.9921568627450981,0.9921568627450981,0.9921568627450981,0.9921568627450981,0.5098039215686274,0.43137254901960786,0.9921568627450981,1.0,0.4235294117647059,0.28627450980392155,0.984313725490196,0.984313725490196,0.984313725490196,0.984313725490196,0.42745098039215684,0.984313725490196,0.9921568627450981,0.4196078431372549,0.792156862745098,0.984313725490196,0.984313725490196,0.984313725490196,0.8823529411764706,0.023529411764705882,0.5058823529411764,0.984313725490196,0.9921568627450981,0.4196078431372549,0.5882352941176471,0.984313725490196,0.984313725490196,0.984313725490196,0.2784313725490196,0.45098039215686275,0.984313725490196,0.984313725490196,0.9921568627450981,0.4196078431372549,0.9921568627450981,0.984313725490196,0.984313725490196,0.6784313725490196,0.0784313725490196,0.8509803921568627,0.984313725490196,0.984313725490196,0.9921568627450981,0.4196078431372549,0.7137254901960784,1.0,0.9921568627450981,0.8470588235294118,0.8549019607843137,0.9921568627450981,0.9921568627450981,0.7137254901960784,0.24705882352941178,0.8666666666666667,0.9921568627450981,0.984313725490196,0.8431372549019608,0.32941176470588235,0.9254901960784314,0.984313725490196,0.984313725490196,0.30196078431372547,0.42745098039215684,0.984313725490196,0.9921568627450981,0.984313725490196,0.8431372549019608,0.043137254901960784,0.6274509803921569,0.984313725490196,0.984313725490196,0.3764705882352941,0.42745098039215684,0.984313725490196,0.9921568627450981,0.984313725490196,0.5372549019607843,0.5882352941176471,0.984313725490196,0.984313725490196,0.984313725490196,0.2784313725490196,0.42745098039215684,0.984313725490196,0.9921568627450981,0.984313725490196,0.13725490196078433,0.5098039215686274,0.9921568627450981,0.984313725490196,0.984313725490196,0.6784313725490196,0.0784313725490196,0.43137254901960786,0.9921568627450981,1.0,0.9921568627450981,0.3843137254901961,0.5882352941176471,0.9921568627450981,1.0,0.9921568627450981,0.6431372549019608,0.42745098039215684,0.984313725490196,0.9921568627450981,0.984313725490196,0.984313725490196,0.984313725490196,0.984313725490196,0.9921568627450981,0.984313725490196,0.13725490196078433,0.36470588235294116,0.9450980392156862,0.9921568627450981,0.984313725490196,0.984313725490196,0.984313725490196,0.984313725490196,0.8470588235294118,0.4392156862745098,0.0196078431372549,0.403921568627451,0.9921568627450981,0.984313725490196,0.984313725490196,0.984313725490196,0.984313725490196,0.48627450980392156,0.984313725490196,0.8823529411764706,0.2784313725490196,0.2784313725490196]) + 0.0 (692,[127,128,129,130,131,155,156,157,158,159,181,182,183,184,185,186,187,209,210,211,212,213,214,215,216,237,238,239,240,241,242,243,244,245,263,264,265,266,267,268,269,270,271,272,273,291,292,293,294,295,296,297,298,299,300,301,302,317,318,319,320,321,322,323,324,325,326,327,328,329,330,344,345,346,347,348,349,353,354,355,356,357,358,372,373,374,375,376,377,381,382,383,384,385,386,399,400,401,402,403,404,409,410,411,412,413,414,427,428,429,430,431,437,438,439,440,441,455,456,457,458,459,460,465,466,467,468,483,484,485,486,487,488,491,492,493,494,495,496,511,512,513,514,515,519,520,521,522,523,539,540,541,542,543,544,545,546,547,548,549,550,567,568,569,570,571,572,573,574,575,576,577,578,595,596,597,598,599,600,601,602,603,604,605,623,624,625,626,627,628,629,630,631,652,653,654,655,656,657,658],[0.28627450980392155,0.9921568627450981,0.8901960784313725,0.28627450980392155,0.08235294117647059,0.28627450980392155,0.984313725490196,0.984313725490196,0.984313725490196,0.6823529411764706,0.06274509803921569,0.6509803921568628,0.8941176470588236,0.984313725490196,0.984313725490196,0.984313725490196,0.47843137254901963,0.24313725490196078,0.8627450980392157,0.9921568627450981,0.984313725490196,0.984313725490196,0.984313725490196,0.984313725490196,0.30980392156862746,0.30980392156862746,0.9058823529411765,0.9921568627450981,0.984313725490196,0.984313725490196,0.984313725490196,0.984313725490196,0.9098039215686274,0.30196078431372547,0.5686274509803921,0.9921568627450981,0.9921568627450981,0.9921568627450981,1.0,0.9921568627450981,0.9921568627450981,0.9921568627450981,0.9921568627450981,1.0,0.4235294117647059,0.5647058823529412,0.984313725490196,0.984313725490196,0.984313725490196,0.9921568627450981,0.6588235294117647,0.4196078431372549,0.6627450980392157,0.984313725490196,0.9921568627450981,0.7411764705882353,0.0784313725490196,0.10588235294117647,0.34901960784313724,0.9254901960784314,0.984313725490196,0.9215686274509803,0.8431372549019608,0.6431372549019608,0.058823529411764705,0.023529411764705882,0.5058823529411764,0.984313725490196,0.9921568627450981,0.984313725490196,0.13725490196078433,0.1843137254901961,0.8274509803921568,0.9921568627450981,0.984313725490196,0.984313725490196,0.5568627450980392,0.1450980392156863,0.984313725490196,0.984313725490196,0.9921568627450981,0.984313725490196,0.13725490196078433,0.42745098039215684,0.984313725490196,0.9921568627450981,0.984313725490196,0.984313725490196,0.5568627450980392,0.043137254901960784,0.5803921568627451,0.984313725490196,0.9921568627450981,0.984313725490196,0.6431372549019608,0.043137254901960784,0.5882352941176471,0.9921568627450981,1.0,0.8274509803921568,0.09803921568627451,0.043137254901960784,0.5882352941176471,0.9921568627450981,1.0,0.8274509803921568,0.09803921568627451,0.5490196078431373,0.984313725490196,0.984313725490196,0.9921568627450981,0.4196078431372549,0.1450980392156863,0.984313725490196,0.984313725490196,0.8274509803921568,0.1803921568627451,0.7450980392156863,0.984313725490196,0.984313725490196,0.9921568627450981,0.5019607843137255,0.0196078431372549,0.1450980392156863,0.984313725490196,0.984313725490196,0.2,0.45098039215686275,0.984313725490196,0.984313725490196,0.9921568627450981,0.7372549019607844,0.0784313725490196,0.12549019607843137,0.42745098039215684,0.5058823529411764,0.984313725490196,0.6784313725490196,0.403921568627451,0.8509803921568627,0.984313725490196,0.984313725490196,0.788235294117647,0.11764705882352941,0.28627450980392155,0.984313725490196,0.984313725490196,0.984313725490196,0.2784313725490196,0.6509803921568628,0.9921568627450981,0.9921568627450981,1.0,0.5843137254901961,0.28627450980392155,0.5882352941176471,0.9921568627450981,1.0,0.9921568627450981,0.9921568627450981,0.5607843137254902,0.5490196078431373,0.984313725490196,0.984313725490196,0.9921568627450981,0.984313725490196,0.984313725490196,0.984313725490196,0.984313725490196,0.9921568627450981,0.984313725490196,0.9019607843137255,0.23921568627450981,0.7450980392156863,0.984313725490196,0.984313725490196,0.9921568627450981,0.984313725490196,0.984313725490196,0.984313725490196,0.984313725490196,0.9490196078431372,0.8431372549019608,0.21568627450980393,0.08235294117647059,0.7411764705882353,0.984313725490196,0.9921568627450981,0.984313725490196,0.984313725490196,0.984313725490196,0.6784313725490196,0.403921568627451,0.12156862745098039,0.7843137254901961,0.9921568627450981,0.984313725490196,0.3764705882352941,0.2784313725490196,0.0784313725490196]) + + + +参考 + + 【1】http://spark.apache.org/docs/1.5.2/mllib-guide.html + 【2】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【3】https://github.com/xubo245/SparkLearning + 【4】book:Machine Learning with Spark ,Nick Pertreach + 【5】book:Spark MlLib机器学习实战 + 【6】https://github.com/endymecy/spark-ml-source-analysis/blob/master/%E7%89%B9%E5%BE%81%E6%8A%BD%E5%8F%96%E5%92%8C%E8%BD%AC%E6%8D%A2/normalizer.md diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24062\344\271\213\347\211\271\345\276\201\351\200\211\346\213\251\344\270\255\347\232\204\345\215\241\346\226\271\351\200\211\346\213\251\345\231\250.md" "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24062\344\271\213\347\211\271\345\276\201\351\200\211\346\213\251\344\270\255\347\232\204\345\215\241\346\226\271\351\200\211\346\213\251\345\231\250.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24062\344\271\213\347\211\271\345\276\201\351\200\211\346\213\251\344\270\255\347\232\204\345\215\241\346\226\271\351\200\211\346\213\251\345\231\250.md" rename to "docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24062\344\271\213\347\211\271\345\276\201\351\200\211\346\213\251\344\270\255\347\232\204\345\215\241\346\226\271\351\200\211\346\213\251\345\231\250.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24063\344\271\213\347\211\271\345\276\201\351\200\211\346\213\251\344\270\255\347\232\204\345\215\241\346\226\271\351\200\211\346\213\251\345\231\250\345\256\236\344\276\213\357\274\210libsvm\346\225\260\346\215\256\351\233\206\357\274\211.md" "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24063\344\271\213\347\211\271\345\276\201\351\200\211\346\213\251\344\270\255\347\232\204\345\215\241\346\226\271\351\200\211\346\213\251\345\231\250\345\256\236\344\276\213\357\274\210libsvm\346\225\260\346\215\256\351\233\206\357\274\211.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24063\344\271\213\347\211\271\345\276\201\351\200\211\346\213\251\344\270\255\347\232\204\345\215\241\346\226\271\351\200\211\346\213\251\345\231\250\345\256\236\344\276\213\357\274\210libsvm\346\225\260\346\215\256\351\233\206\357\274\211.md" rename to "docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24063\344\271\213\347\211\271\345\276\201\351\200\211\346\213\251\344\270\255\347\232\204\345\215\241\346\226\271\351\200\211\346\213\251\345\231\250\345\256\236\344\276\213\357\274\210libsvm\346\225\260\346\215\256\351\233\206\357\274\211.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24064\344\271\213\345\205\203\347\264\240\346\231\272\350\203\275\344\271\230\347\247\257ElementwiseProduct.md" "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24064\344\271\213\345\205\203\347\264\240\346\231\272\350\203\275\344\271\230\347\247\257ElementwiseProduct.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24064\344\271\213\345\205\203\347\264\240\346\231\272\350\203\275\344\271\230\347\247\257ElementwiseProduct.md" rename to "docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24064\344\271\213\345\205\203\347\264\240\346\231\272\350\203\275\344\271\230\347\247\257ElementwiseProduct.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24065\344\271\213\344\275\277\347\224\250PCA\350\277\233\350\241\214\347\211\271\345\276\201\350\275\254\346\215\242.md" "b/docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24065\344\271\213\344\275\277\347\224\250PCA\350\277\233\350\241\214\347\211\271\345\276\201\350\275\254\346\215\242.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24065\344\271\213\344\275\277\347\224\250PCA\350\277\233\350\241\214\347\211\271\345\276\201\350\275\254\346\215\242.md" rename to "docs/Spark/MLlibLearning/7\347\211\271\345\276\201\346\217\220\345\217\226\345\222\214\350\275\254\346\215\242/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24065\344\271\213\344\275\277\347\224\250PCA\350\277\233\350\241\214\347\211\271\345\276\201\350\275\254\346\215\242.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/8\351\242\221\347\271\201\351\241\271\346\214\226\346\216\230/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24066\344\271\213FP-growth.md" "b/docs/Spark/MLlibLearning/8\351\242\221\347\271\201\351\241\271\346\214\226\346\216\230/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24066\344\271\213FP-growth.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/8\351\242\221\347\271\201\351\241\271\346\214\226\346\216\230/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24066\344\271\213FP-growth.md" rename to "docs/Spark/MLlibLearning/8\351\242\221\347\271\201\351\241\271\346\214\226\346\216\230/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24066\344\271\213FP-growth.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/8\351\242\221\347\271\201\351\241\271\346\214\226\346\216\230/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24067\344\271\213\345\205\263\350\201\224\350\247\204\345\210\231AssociationRules.md" "b/docs/Spark/MLlibLearning/8\351\242\221\347\271\201\351\241\271\346\214\226\346\216\230/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24067\344\271\213\345\205\263\350\201\224\350\247\204\345\210\231AssociationRules.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/8\351\242\221\347\271\201\351\241\271\346\214\226\346\216\230/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24067\344\271\213\345\205\263\350\201\224\350\247\204\345\210\231AssociationRules.md" rename to "docs/Spark/MLlibLearning/8\351\242\221\347\271\201\351\241\271\346\214\226\346\216\230/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24067\344\271\213\345\205\263\350\201\224\350\247\204\345\210\231AssociationRules.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/8\351\242\221\347\271\201\351\241\271\346\214\226\346\216\230/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24068\344\271\213PrefixSpan.md" "b/docs/Spark/MLlibLearning/8\351\242\221\347\271\201\351\241\271\346\214\226\346\216\230/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24068\344\271\213PrefixSpan.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/8\351\242\221\347\271\201\351\241\271\346\214\226\346\216\230/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24068\344\271\213PrefixSpan.md" rename to "docs/Spark/MLlibLearning/8\351\242\221\347\271\201\351\241\271\346\214\226\346\216\230/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24068\344\271\213PrefixSpan.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/9\350\257\204\344\274\260\345\272\246\351\207\217/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24069\344\271\213\345\257\271\344\272\214\345\210\206\347\261\273\350\277\233\350\241\214\350\257\204\344\274\260Binary classification.md" "b/docs/Spark/MLlibLearning/9\350\257\204\344\274\260\345\272\246\351\207\217/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24069\344\271\213\345\257\271\344\272\214\345\210\206\347\261\273\350\277\233\350\241\214\350\257\204\344\274\260Binary classification.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/9\350\257\204\344\274\260\345\272\246\351\207\217/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24069\344\271\213\345\257\271\344\272\214\345\210\206\347\261\273\350\277\233\350\241\214\350\257\204\344\274\260Binary classification.md" rename to "docs/Spark/MLlibLearning/9\350\257\204\344\274\260\345\272\246\351\207\217/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24069\344\271\213\345\257\271\344\272\214\345\210\206\347\261\273\350\277\233\350\241\214\350\257\204\344\274\260Binary classification.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/9\350\257\204\344\274\260\345\272\246\351\207\217/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24070\344\271\213\345\257\271\345\244\232\347\261\273\345\210\206\347\261\273\347\273\223\346\236\234\350\277\233\350\241\214\350\257\204\344\274\260Multiclass classification.md" "b/docs/Spark/MLlibLearning/9\350\257\204\344\274\260\345\272\246\351\207\217/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24070\344\271\213\345\257\271\345\244\232\347\261\273\345\210\206\347\261\273\347\273\223\346\236\234\350\277\233\350\241\214\350\257\204\344\274\260Multiclass classification.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/9\350\257\204\344\274\260\345\272\246\351\207\217/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24070\344\271\213\345\257\271\345\244\232\347\261\273\345\210\206\347\261\273\347\273\223\346\236\234\350\277\233\350\241\214\350\257\204\344\274\260Multiclass classification.md" rename to "docs/Spark/MLlibLearning/9\350\257\204\344\274\260\345\272\246\351\207\217/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24070\344\271\213\345\257\271\345\244\232\347\261\273\345\210\206\347\261\273\347\273\223\346\236\234\350\277\233\350\241\214\350\257\204\344\274\260Multiclass classification.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/9\350\257\204\344\274\260\345\272\246\351\207\217/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24071\344\271\213\345\257\271\345\244\232\346\240\207\347\255\276\345\210\206\347\261\273\350\277\233\350\241\214\350\257\204\344\274\260.md" "b/docs/Spark/MLlibLearning/9\350\257\204\344\274\260\345\272\246\351\207\217/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24071\344\271\213\345\257\271\345\244\232\346\240\207\347\255\276\345\210\206\347\261\273\350\277\233\350\241\214\350\257\204\344\274\260.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/9\350\257\204\344\274\260\345\272\246\351\207\217/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24071\344\271\213\345\257\271\345\244\232\346\240\207\347\255\276\345\210\206\347\261\273\350\277\233\350\241\214\350\257\204\344\274\260.md" rename to "docs/Spark/MLlibLearning/9\350\257\204\344\274\260\345\272\246\351\207\217/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24071\344\271\213\345\257\271\345\244\232\346\240\207\347\255\276\345\210\206\347\261\273\350\277\233\350\241\214\350\257\204\344\274\260.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/9\350\257\204\344\274\260\345\272\246\351\207\217/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24072\344\271\213RankingSystem\350\277\233\350\241\214\350\257\204\344\274\260.md" "b/docs/Spark/MLlibLearning/9\350\257\204\344\274\260\345\272\246\351\207\217/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24072\344\271\213RankingSystem\350\277\233\350\241\214\350\257\204\344\274\260.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/9\350\257\204\344\274\260\345\272\246\351\207\217/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24072\344\271\213RankingSystem\350\277\233\350\241\214\350\257\204\344\274\260.md" rename to "docs/Spark/MLlibLearning/9\350\257\204\344\274\260\345\272\246\351\207\217/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24072\344\271\213RankingSystem\350\277\233\350\241\214\350\257\204\344\274\260.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/9\350\257\204\344\274\260\345\272\246\351\207\217/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24073\344\271\213\345\233\236\345\275\222\351\227\256\351\242\230\347\232\204\350\257\204\344\274\260.md" "b/docs/Spark/MLlibLearning/9\350\257\204\344\274\260\345\272\246\351\207\217/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24073\344\271\213\345\233\236\345\275\222\351\227\256\351\242\230\347\232\204\350\257\204\344\274\260.md" similarity index 100% rename from "docs/Spark MLlib\345\255\246\344\271\240/9\350\257\204\344\274\260\345\272\246\351\207\217/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24073\344\271\213\345\233\236\345\275\222\351\227\256\351\242\230\347\232\204\350\257\204\344\274\260.md" rename to "docs/Spark/MLlibLearning/9\350\257\204\344\274\260\345\272\246\351\207\217/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\24073\344\271\213\345\233\236\345\275\222\351\227\256\351\242\230\347\232\204\350\257\204\344\274\260.md" diff --git "a/docs/Spark MLlib\345\255\246\344\271\240/README.md" b/docs/Spark/MLlibLearning/README.md similarity index 97% rename from "docs/Spark MLlib\345\255\246\344\271\240/README.md" rename to docs/Spark/MLlibLearning/README.md index 19d89df..d51010e 100644 --- "a/docs/Spark MLlib\345\255\246\344\271\240/README.md" +++ b/docs/Spark/MLlibLearning/README.md @@ -5,11 +5,10 @@ hadoop2.6.0 scala-2.10.4 idea 15.0.4 -*Spark mllib学习目录： +Spark mllib学习目录： **1.数据类型** - Spark中组件Mllib的学习3之用户相似度计算 Spark中组件Mllib的学习12之密集向量和稀疏向量的生成 Spark中组件Mllib的学习13之给向量打标签 Spark中组件Mllib的学习14之从文本中读取带标签的数据，生成带label的向量 @@ -116,4 +115,4 @@ idea 15.0.4 **11优化** - Spark中组件Mllib的学习75之L-BFGS + Spark中组件Mllib的学习75之L-BFGS \ No newline at end of file diff --git "a/docs/Spark/MLlibLearning/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\240100\344\271\213\346\250\241\347\211\210.md" "b/docs/Spark/MLlibLearning/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\240100\344\271\213\346\250\241\347\211\210.md" new file mode 100644 index 0000000..6eb1443 --- /dev/null +++ "b/docs/Spark/MLlibLearning/Spark\344\270\255\347\273\204\344\273\266Mllib\347\232\204\345\255\246\344\271\240100\344\271\213\346\250\241\347\211\210.md" @@ -0,0 +1,25 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark中组件Mllib的学习 + +1.解释 + + + + +2.代码： + + + +3.结果： + + + +参考 + + 【1】http://spark.apache.org/docs/1.5.2/mllib-guide.html + 【2】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【3】https://github.com/xubo245/SparkLearning + 【4】book:Machine Learning with Spark ,Nick Pertreach + 【5】book:Spark MlLib机器学习实战 diff --git a/docs/Spark/SparkAvroLearning/README.md b/docs/Spark/SparkAvroLearning/README.md new file mode 100644 index 0000000..34dad0a --- /dev/null +++ b/docs/Spark/SparkAvroLearning/README.md @@ -0,0 +1,10 @@ +## Spark-Avro学习篇 ## +Spark-Avro学习1之使用SparkSQL读取AVRO文件：http://blog.csdn.net/xubo245/article/details/51295474 +Spark-Avro学习2之使用byDatabricksSparkAvroL读取AVRO文件:http://blog.csdn.net/xubo245/article/details/51295593 +Spark-Avro学习3之使用AvroCompression存储AVRO文件:http://blog.csdn.net/xubo245/article/details/51295604 +Spark-Avro学习4之使用AvroWritePartitioned存储AVRO文件时进行划分:http://blog.csdn.net/xubo245/article/details/51295627 +Spark-Avro学习5之使用AvroReadSpecifyName存储AVRO文件时指定name和namespace:http://blog.csdn.net/xubo245/article/details/51295642 +Spark-Avro学习6之Ubuntu下安装:http://blog.csdn.net/xubo245/article/details/51295674 +Spark-Avro学习7之Java Avro使用（生成code方式）:http://blog.csdn.net/xubo245/article/details/51295843 +Spark-Avro学习8之Java Avro使用（不生成code方式）:Spark-Avro学习8之Java Avro使用（不生成code方式） +Spark-Avro学习9之SCALA环境下Avro使用（不生成code方式）:http://blog.csdn.net/xubo245/article/details/51296717 \ No newline at end of file diff --git a/docs/Spark/SparkBaseLearning/README.md b/docs/Spark/SparkBaseLearning/README.md new file mode 100644 index 0000000..dbc6572 --- /dev/null +++ b/docs/Spark/SparkBaseLearning/README.md @@ -0,0 +1,12 @@ +## Spark基本学习篇： ## +spark学习1之examples运行：http://blog.csdn.net/xubo245/article/details/48548079 +spark学习2之OutOfMemoryError错误的解决办法：http://blog.csdn.net/xubo245/article/details/48548507 +spark学习3之examples中的SparkPi：http://blog.csdn.net/xubo245/article/details/50596227 +spark学习4之集群上直接用scalac编译.scala出现的MissingRequirementError问题（已解决）:http://blog.csdn.net/xubo245/article/details/50596822 +spark学习5之sbt问题:http://blog.csdn.net/xubo245/article/details/50603502 +spark学习6之scala版本不同的问题：http://blog.csdn.net/xubo245/article/ details/50609476 +spark学习7之IDEA下搭建SPark本地编译环境并上传到集群运行：http://blog.csdn.net/xubo245/article/details/50789983 +spark学习8之eclipse安装scala2.10和spark编译环境并上传到集群运行：http://blog.csdn.net/xubo245/article/details/50790463 +spark学习9之在window下进行源码编译打包：http://blog.csdn.net/xubo245/article/details/51386564 +spark学习10之将spark的AppName设置为自动获取当前类名：http://blog.csdn.net/xubo245/article/details/51428158 +spark学习11之在idea中将eclipse导入的java project改成maven project：http://blog.csdn.net/xubo245/article/details/51428502 \ No newline at end of file diff --git a/docs/Spark/SparkCodeLearning/README.md b/docs/Spark/SparkCodeLearning/README.md new file mode 100644 index 0000000..3a4c8d6 --- /dev/null +++ b/docs/Spark/SparkCodeLearning/README.md @@ -0,0 +1,5 @@ +## Spark代码篇： ## +Spark代码1之RDDparallelizeSaveAsFile：http://blog.csdn.net/xubo245/article/details/50791485 +Spark代码2之Transformation：union，distinct，join：http://blog.csdn.net/xubo245/article/details/50792201 +Spark代码3之Action：reduce，reduceByKey,sorted,lookup,take,saveAsTextFile：http://blog.csdn.net/xubo245/article/details/50800934 +Spark代码4之Spark 文件API及其对搜狗数据的操作：http://blog.csdn.net/xubo245/article/details/50801827 \ No newline at end of file diff --git a/docs/Spark/SparkCsvLearning/README.md b/docs/Spark/SparkCsvLearning/README.md new file mode 100644 index 0000000..d38690b --- /dev/null +++ b/docs/Spark/SparkCsvLearning/README.md @@ -0,0 +1,2 @@ +## Spark生态之spark-csv篇： ## +Spark生态之Spark-csv学习1之安装和简单的examples：http://blog.csdn.net/xubo245/article/details/51184946 \ No newline at end of file diff --git a/docs/Spark/SparkQuestion/README.md b/docs/Spark/SparkQuestion/README.md new file mode 100644 index 0000000..0e90905 --- /dev/null +++ b/docs/Spark/SparkQuestion/README.md @@ -0,0 +1,3 @@ +## Spark疑问篇 ## +Spark疑问1之如何查看sparkContext没有关闭的sc：http://blog.csdn.net/xubo245/article/details/51173463 +Spark疑问2之spark 丢了executor会恢复吗？：http://blog.csdn.net/xubo245/article/details/51173493 \ No newline at end of file diff --git a/docs/Spark/SparkRLearning/README.md b/docs/Spark/SparkRLearning/README.md new file mode 100644 index 0000000..4a752ce --- /dev/null +++ b/docs/Spark/SparkRLearning/README.md @@ -0,0 +1,6 @@ +## Spark组件之SparkR学习篇 ## +Spark组件之SparkR学习1--安装与测试：http://blog.csdn.net/xubo245/article/details/51195287 +Spark组件之SparkR学习2--使用spark-submit向集群提交R代码文件dataframe.R：http://blog.csdn.net/xubo245/article/details/51199216 + Spark组件之SparkR学习3--使用spark-submit向集群提交R代码文件data-manipulation.R：http://blog.csdn.net/xubo245/article/details/51199813 + Spark组件之SparkR学习4--Eclipse下R语言环境搭建：http://blog.csdn.net/xubo245/article/details/51199918 + Spark组件之SparkR学习5--R语言函数调用（跨文件调用）：http://blog.csdn.net/xubo245/article/details/51205276 \ No newline at end of file diff --git a/docs/Spark/SparkSQLLearning/README.md b/docs/Spark/SparkSQLLearning/README.md new file mode 100644 index 0000000..1c19b51 --- /dev/null +++ b/docs/Spark/SparkSQLLearning/README.md @@ -0,0 +1,4 @@ +## Spark组件之SparkSQL学习篇 ## +Spark组件之SparkSQL学习1之问题报错No TypeTag available for Person：http://blog.csdn.net/xubo245/article/details/51153243 +SparkSQL在代码库中还有不少，当时没写成博客 +更多请见代码：[链接](../../../src\main\scala\org\apache\spark\sql\learning) \ No newline at end of file diff --git a/docs/Spark/SparkStreamingLearning/README.md b/docs/Spark/SparkStreamingLearning/README.md new file mode 100644 index 0000000..a57ee2a --- /dev/null +++ b/docs/Spark/SparkStreamingLearning/README.md @@ -0,0 +1,5 @@ +## Spark组件之Spark Streaming学习篇 ## +Spark组件之Spark Streaming学习1--NetworkWordCount学习：http://blog.csdn.net/xubo245/article/details/51251970 +Spark组件之Spark Streaming学习2--StatefulNetworkWordCount 学习：http://blog.csdn.net/xubo245/article/details/51252142 +Spark组件之Spark Streaming学习3--结合SparkSQL的使用(wordCount)：http://blog.csdn.net/xubo245/article/details/51252229 +Spark组件之Spark Streaming学习4--HdfsWordCount 学习：http://blog.csdn.net/xubo245/article/details/51254412 \ No newline at end of file diff --git "a/docs/Spark\351\227\256\351\242\230/Spark\351\227\256\351\242\2301\344\271\213\350\257\273\345\205\245\345\217\202\350\200\203\345\272\217\345\210\227\347\232\204adam\346\240\274\345\274\217\346\212\245\351\224\231empty max.md" "b/docs/SparkProblem/Spark\351\227\256\351\242\2301\344\271\213\350\257\273\345\205\245\345\217\202\350\200\203\345\272\217\345\210\227\347\232\204adam\346\240\274\345\274\217\346\212\245\351\224\231empty max.md" similarity index 98% rename from "docs/Spark\351\227\256\351\242\230/Spark\351\227\256\351\242\2301\344\271\213\350\257\273\345\205\245\345\217\202\350\200\203\345\272\217\345\210\227\347\232\204adam\346\240\274\345\274\217\346\212\245\351\224\231empty max.md" rename to "docs/SparkProblem/Spark\351\227\256\351\242\2301\344\271\213\350\257\273\345\205\245\345\217\202\350\200\203\345\272\217\345\210\227\347\232\204adam\346\240\274\345\274\217\346\212\245\351\224\231empty max.md" index 7dec457..9efd591 100644 --- "a/docs/Spark\351\227\256\351\242\230/Spark\351\227\256\351\242\2301\344\271\213\350\257\273\345\205\245\345\217\202\350\200\203\345\272\217\345\210\227\347\232\204adam\346\240\274\345\274\217\346\212\245\351\224\231empty max.md" +++ "b/docs/SparkProblem/Spark\351\227\256\351\242\2301\344\271\213\350\257\273\345\205\245\345\217\202\350\200\203\345\272\217\345\210\227\347\232\204adam\346\240\274\345\274\217\346\212\245\351\224\231empty max.md" @@ -30,7 +30,7 @@ hadoop@Master:~/xubo/project/load/loadfastaFromHDFSAdamAndCount$ ./loadGRCH38chr14.sh start: [Stage 1:=============================> (2 + 2) / 4]16/06/08 14:05:52 ERROR TaskSetManager: Task 0 in stage 1.0 failed 4 times; aborting job - Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 1.0 failed 4 times, most recent failure: Lost task 0.3 in stage 1.0 (TID 12, 219.219.220.131): java.lang.UnsupportedOperationException: empty.max + Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 1.0 failed 4 times, most recent failure: Lost task 0.3 in stage 1.0 (TID 12, Mcnode3): java.lang.UnsupportedOperationException: empty.max at scala.collection.TraversableOnce$class.max(TraversableOnce.scala:216) at scala.collection.AbstractTraversable.max(Traversable.scala:105) at org.bdgenomics.adam.converters.FastaConverter$.findContigIndex(FastaConverter.scala:126) diff --git "a/docs/SparkProblem/Spark\351\227\256\351\242\2302\344\271\213window\344\270\213\350\275\275hdfs\346\226\207\344\273\266\347\232\204hosts\350\256\276\347\275\256.md" "b/docs/SparkProblem/Spark\351\227\256\351\242\2302\344\271\213window\344\270\213\350\275\275hdfs\346\226\207\344\273\266\347\232\204hosts\350\256\276\347\275\256.md" new file mode 100644 index 0000000..8ed755d --- /dev/null +++ "b/docs/SparkProblem/Spark\351\227\256\351\242\2302\344\271\213window\344\270\213\350\275\275hdfs\346\226\207\344\273\266\347\232\204hosts\350\256\276\347\275\256.md" @@ -0,0 +1,30 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio（tachyon） 0.7.1，spark-1.5.2,hadoop-2.6.0 + +# 1.问题描述 # + +window下打开hdfs，会出现问题：hdfs常配置了hosts，访问或者下载hdfs的文件时（通过浏览器），回出现跳转的是hostname，而不是ip，而window下没有配置hosts，所以访问不了，需要手动设置ip，麻烦 + +# 2.解决办法： # +设置window的hosts +修改C:\Windows\System32\drivers\etc下的hosts文件 + +在 + + # 127.0.0.1 localhost + # ::1 localhost +后加入自己的host关联 + + +# 3.运行记录： # + +可以访问集群并且下载文件： + + http://Master:50070/webhdfs/v1/xubo/project/SparkSW/output/time/20161106204334687SparkSW_queryFile_0P18691.file_dbFile_D1Line.fasta_splitNum_32_taskNum_1_file/part-00000?op=OPEN + +参考 + + 【1】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【2】https://github.com/xubo245/SparkLearning diff --git "a/docs/SparkProblem/Spark\351\227\256\351\242\2303\344\271\213SparkException\357\274\232Error notifying standalone scheduler's driver endpoint.md" "b/docs/SparkProblem/Spark\351\227\256\351\242\2303\344\271\213SparkException\357\274\232Error notifying standalone scheduler's driver endpoint.md" new file mode 100644 index 0000000..557a5c1 --- /dev/null +++ "b/docs/SparkProblem/Spark\351\227\256\351\242\2303\344\271\213SparkException\357\274\232Error notifying standalone scheduler's driver endpoint.md" @@ -0,0 +1,105 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio（tachyon） 0.7.1，spark-1.5.2,hadoop-2.6.0 + +# 1.问题描述 # +# 1.1 +运行alluxioHDFS.sh的时候出现错误： + + hadoop@Master:~/disk2/xubo/project/alignment/SparkSW/SparkSW20161114/alluxio-1.3.0$ ./alluxioHDFS.sh &> alluxioHDFStime201611232044.txt + +错误概要： + + SparkException：Error notifying standalone scheduler's driver endpoint + +#1.2 问题查看 +按照时间点查，没有看到什么具体的问题： + + http://Master:18080/history/app-20161123210139-3796/stages/stage/?id=1&attempt=0 + + http://Master:18080/history/app-20161123210214-3797/stages/stage/?id=0&attempt=0 +# 2.解决办法： # + +待解决 + + + +# 3.运行记录： # + + time:6 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + 16/11/23 21:02:08 WARN QueuedThreadPool: 1 threads could not be stopped + 16/11/23 21:02:14 ERROR AkkaRpcEnv: Ignore error: Error notifying standalone scheduler's driver endpoint + org.apache.spark.SparkException: Error notifying standalone scheduler's driver endpoint + at org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend.removeExecutor(CoarseGrainedSchedulerBackend.scala:312) + at org.apache.spark.scheduler.cluster.SparkDeploySchedulerBackend.executorRemoved(SparkDeploySchedulerBackend.scala:142) + at org.apache.spark.deploy.client.AppClient$ClientEndpoint$$anonfun$receive$1.applyOrElse(AppClient.scala:184) + at org.apache.spark.rpc.akka.AkkaRpcEnv.org$apache$spark$rpc$akka$AkkaRpcEnv$$processMessage(AkkaRpcEnv.scala:177) + at org.apache.spark.rpc.akka.AkkaRpcEnv$$anonfun$actorRef$lzycompute$1$1$$anon$1$$anonfun$receiveWithLogging$1$$anonfun$applyOrElse$4.apply$mcV$sp(AkkaRpcEnv.scala:126) + at org.apache.spark.rpc.akka.AkkaRpcEnv.org$apache$spark$rpc$akka$AkkaRpcEnv$$safelyCall(AkkaRpcEnv.scala:197) + at org.apache.spark.rpc.akka.AkkaRpcEnv$$anonfun$actorRef$lzycompute$1$1$$anon$1$$anonfun$receiveWithLogging$1.applyOrElse(AkkaRpcEnv.scala:125) + at scala.runtime.AbstractPartialFunction$mcVL$sp.apply$mcVL$sp(AbstractPartialFunction.scala:33) + at scala.runtime.AbstractPartialFunction$mcVL$sp.apply(AbstractPartialFunction.scala:33) + at scala.runtime.AbstractPartialFunction$mcVL$sp.apply(AbstractPartialFunction.scala:25) + at org.apache.spark.util.ActorLogReceive$$anon$1.apply(ActorLogReceive.scala:59) + at org.apache.spark.util.ActorLogReceive$$anon$1.apply(ActorLogReceive.scala:42) + at scala.PartialFunction$class.applyOrElse(PartialFunction.scala:118) + at org.apache.spark.util.ActorLogReceive$$anon$1.applyOrElse(ActorLogReceive.scala:42) + at akka.actor.Actor$class.aroundReceive(Actor.scala:467) + at org.apache.spark.rpc.akka.AkkaRpcEnv$$anonfun$actorRef$lzycompute$1$1$$anon$1.aroundReceive(AkkaRpcEnv.scala:92) + at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) + at akka.actor.ActorCell.invoke(ActorCell.scala:487) + at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) + at akka.dispatch.Mailbox.run(Mailbox.scala:220) + at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) + at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) + at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) + at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) + at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) + Caused by: org.apache.spark.SparkException: Error sending message [message = RemoveExecutor(4,Unknown executor exit code (0))] + at org.apache.spark.rpc.RpcEndpointRef.askWithRetry(RpcEndpointRef.scala:118) + at org.apache.spark.rpc.RpcEndpointRef.askWithRetry(RpcEndpointRef.scala:77) + at org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend.removeExecutor(CoarseGrainedSchedulerBackend.scala:309) + ... 24 more + Caused by: org.apache.spark.rpc.RpcTimeoutException: Recipient[Actor[akka://sparkDriver/user/CoarseGrainedScheduler#-679832735]] had already been terminated.. This timeout is controlled by spark.rpc.askTimeout + at org.apache.spark.rpc.RpcTimeout.org$apache$spark$rpc$RpcTimeout$$createRpcTimeoutException(RpcEnv.scala:214) + at org.apache.spark.rpc.RpcTimeout$$anonfun$addMessageIfTimeout$1.applyOrElse(RpcEnv.scala:229) + at org.apache.spark.rpc.RpcTimeout$$anonfun$addMessageIfTimeout$1.applyOrElse(RpcEnv.scala:225) + at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:33) + at scala.util.Failure$$anonfun$recover$1.apply(Try.scala:185) + at scala.util.Try$.apply(Try.scala:161) + at scala.util.Failure.recover(Try.scala:185) + at scala.concurrent.Future$$anonfun$recover$1.apply(Future.scala:324) + at scala.concurrent.Future$$anonfun$recover$1.apply(Future.scala:324) + at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32) + at org.spark-project.guava.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:293) + at scala.concurrent.impl.ExecutionContextImpl$$anon$1.execute(ExecutionContextImpl.scala:133) + at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40) + at scala.concurrent.impl.Promise$DefaultPromise.scala$concurrent$impl$Promise$DefaultPromise$$dispatchOrAddCallback(Promise.scala:280) + at scala.concurrent.impl.Promise$DefaultPromise.onComplete(Promise.scala:270) + at scala.concurrent.Future$class.recover(Future.scala:324) + at scala.concurrent.impl.Promise$DefaultPromise.recover(Promise.scala:153) + at org.apache.spark.rpc.akka.AkkaRpcEndpointRef.ask(AkkaRpcEnv.scala:319) + at org.apache.spark.rpc.RpcEndpointRef.askWithRetry(RpcEndpointRef.scala:100) + ... 26 more + Caused by: akka.pattern.AskTimeoutException: Recipient[Actor[akka://sparkDriver/user/CoarseGrainedScheduler#-679832735]] had already been terminated. + at akka.pattern.AskableActorRef$.ask$extension(AskSupport.scala:132) + at org.apache.spark.rpc.akka.AkkaRpcEndpointRef.ask(AkkaRpcEnv.scala:307) + ... 27 more + time:7 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + + +参考 + + 【1】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【2】https://github.com/xubo245/SparkLearning diff --git "a/docs/SparkProblem/Spark\351\227\256\351\242\2304\344\271\213Excutor lost.md" "b/docs/SparkProblem/Spark\351\227\256\351\242\2304\344\271\213Excutor lost.md" new file mode 100644 index 0000000..addd0ea --- /dev/null +++ "b/docs/SparkProblem/Spark\351\227\256\351\242\2304\344\271\213Excutor lost.md" @@ -0,0 +1,32 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio（tachyon） 0.7.1，spark-1.5.2,hadoop-2.6.0 + +# 1.问题描述 # + +## 1.1 ## + +Mcnode1 的一个excutor丢失： ExecutorLostFailure (executor 2 lost) + +## 1.2 ## + + http://Master:18080/history/app-20161121121144-2893 + +导致整个计算过程拖长，其他节点都在一分钟结束，但是excutor丢失的节点耗时3min +# 2.解决办法： # + +待解决 + +# 3.运行记录： # + +![](http://i.imgur.com/WyfGE8c.png) + +![](http://i.imgur.com/hkpowyL.png) + +![](http://i.imgur.com/AcfUsHW.png) + +参考 + + 【1】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【2】https://github.com/xubo245/SparkLearning diff --git "a/docs/SparkProblem/Spark\351\227\256\351\242\2305\344\271\213ERROR LiveListenerBus SparkListenerBus has already stopped.md" "b/docs/SparkProblem/Spark\351\227\256\351\242\2305\344\271\213ERROR LiveListenerBus SparkListenerBus has already stopped.md" new file mode 100644 index 0000000..848a8f6 --- /dev/null +++ "b/docs/SparkProblem/Spark\351\227\256\351\242\2305\344\271\213ERROR LiveListenerBus SparkListenerBus has already stopped.md" @@ -0,0 +1,34 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio（tachyon） 0.7.1，spark-1.5.2,hadoop-2.6.0 + +# 1.问题描述 # + +## 1.1 描述## + +将uniref按照序列长度[0,100).[100,)...进行划分，然后使用sparkSW从HDFS和Alluxio分别读取，并进行性能分析 + +运行过程中出现ERROR LiveListenerBus: SparkListenerBus has already stopped + +## 1.2 问题显示## + hadoop@Master:~/disk2/xubo/project/alignment/SparkSW/SparkSW20161114/alluxio-1.3.0$ tail -f alluxioHDFSSplittime201611241456.txt + + time:2 + (55,UniRef100_UPI000697D1C0) + (55,UniRef100_UPI00064D62DA) + (53,UniRef100_A0A194XVF9) + (53,UniRef100_Q1ATC3) + (52,UniRef100_C5J6S2) + 16/11/25 03:32:15 ERROR LiveListenerBus: SparkListenerBus has already stopped! Dropping event SparkListenerExecutorMetricsUpdate(3,WrappedArray()) + + +# 2.解决办法： # + +待解决 + + +参考 + + 【1】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【2】https://github.com/xubo245/SparkLearning diff --git "a/docs/SparkProblem/Spark\351\227\256\351\242\2306\344\271\213Spark\344\270\242\345\244\261excutor\344\271\213\345\220\216appport\345\215\240CPU90%.md" "b/docs/SparkProblem/Spark\351\227\256\351\242\2306\344\271\213Spark\344\270\242\345\244\261excutor\344\271\213\345\220\216appport\345\215\240CPU90%.md" new file mode 100644 index 0000000..8ef7562 --- /dev/null +++ "b/docs/SparkProblem/Spark\351\227\256\351\242\2306\344\271\213Spark\344\270\242\345\244\261excutor\344\271\213\345\220\216appport\345\215\240CPU90%.md" @@ -0,0 +1,43 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio（tachyon） 0.7.1，spark-1.5.2,hadoop-2.6.0 + +# 1.问题描述 # + +## 1.1 spark lost excutor ## + +hadoop@Master:~/disk2/xubo/project/alignment/SparkSW/SparkSW20161114/alluxio-1.3.0$ tail -f DSWSparkSWlengthBigtime201611291725.txt +SparkSW: +time:1 +[Stage 1:=================> (41 + 16) / 128]16/11/29 17:52:31 ERROR TaskSchedulerImpl: Lost executor 1 on Mcnode6: Executor heartbeat timed out after 147971 ms +(268435469,UniRef100_B0B7U3) +(268435464,UniRef100_B8M044) +(268435462,UniRef100_Q9LZW4) +(268435458,UniRef100_Q9QWV9) +(268435456,UniRef100_Q8DD41) + + +## 1.2 ## + +![](http://i.imgur.com/VIjBXWI.png) + +博客【3】中说是系统crash导致ubuntu发送error，占比高 + +# 2.解决办法： # + +## 2.1 ## + + +## 2.2 ## + + + +# 3.运行记录： # + + +参考 + + 【1】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【2】https://github.com/xubo245/SparkLearning + 【3】http://blog.csdn.net/chrisniu1984/article/details/12050951 diff --git "a/docs/SparkProblem/Spark\351\227\256\351\242\2307\344\271\213\345\246\202\344\275\225\350\256\251\350\212\202\347\202\271\346\211\247\350\241\214\346\214\207\345\256\232\347\232\204core\346\225\260.md" "b/docs/SparkProblem/Spark\351\227\256\351\242\2307\344\271\213\345\246\202\344\275\225\350\256\251\350\212\202\347\202\271\346\211\247\350\241\214\346\214\207\345\256\232\347\232\204core\346\225\260.md" new file mode 100644 index 0000000..fb29de8 --- /dev/null +++ "b/docs/SparkProblem/Spark\351\227\256\351\242\2307\344\271\213\345\246\202\344\275\225\350\256\251\350\212\202\347\202\271\346\211\247\350\241\214\346\214\207\345\256\232\347\232\204core\346\225\260.md" @@ -0,0 +1,58 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio（tachyon） 0.7.1，spark-1.5.2,hadoop-2.6.0 + +# 1.问题描述 # + +## 1.1 ## +使用1，2的代码会有1个节点执行连个core + + +## 1.2 ## 代码 + +hadoop@Master:~/disk2/xubo/project/alignment/SparkSW/SparkSW20161114/alluxio-1.3.0$ vi testDSWscalability.sh + + #!/bin/bash + + #sbt clean + #sbt package + + mem=$1 + subMatrixFile=$2 + queryFile=$3 + dbFile=$4 + splitNum=$5 + taskNum=$6 + topK=$7 + cores=$8 + + #/home/zgg/lib/spark-1.0.1-bin-hadoop2/bin/spark-submit \ + spark-submit \ + --class "org.dsw.core.DSW" \ + --conf spark.executor.cores=1 \ + --conf "spark.executor.extraJavaOptions=-Djava.library.path=/home/hadoop/disk2/xubo/lib" \ + --master spark://219.219.220.149:7077 \ + --total-executor-cores $cores \ + --executor-memory $mem \ + SparkSW.jar $subMatrixFile $queryFile $dbFile $splitNum $taskNum $topK + +# 2.解决办法： # + +## 2.1 ## + +不知道如何解决？ + +## 2.2 ## + +不加--conf spark.executor.cores=1 \，core小于节点数，即8时都是每个节点执行一个core。 + + +# 3.运行记录： # + +展示采取2.2 + +参考 + + 【1】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【2】https://github.com/xubo245/SparkLearning diff --git "a/docs/SparkProblem/Spark\351\227\256\351\242\2308\344\271\213worker lost.md" "b/docs/SparkProblem/Spark\351\227\256\351\242\2308\344\271\213worker lost.md" new file mode 100644 index 0000000..bef92f7 --- /dev/null +++ "b/docs/SparkProblem/Spark\351\227\256\351\242\2308\344\271\213worker lost.md" @@ -0,0 +1,86 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio（tachyon） 0.7.1，spark-1.5.2,hadoop-2.6.0 + +# 1.问题描述 # + +## 1.1 第一次## + +八个节点七个节点dead，worker都lost了，不知道为什么 + +没找到其他日志 + +【3】中也有类似的问题，猜测可能是history增加的原因 + + + hadoop@Master:~/disk2/xubo/project/alignment/SparkSW/SparkSW20161114/alluxio-1.3.0$ ./DSWSparkSWquery.sh &> DSWSparkSWquerytime201612121233.txt +记录 + + hadoop@Master:~/disk2/xubo/project/alignment/SparkSW/SparkSW20161114/alluxio-1.3.0$ tail -f DSWSparkSWquerytime201612121233.txt + SparkSW: + time: + [Stage 1:====================> (48 + 16) / 128]16/12/12 14:35:46 ERROR TaskSchedulerImpl: Lost executor 5 on Mcnode7: worker lost + [Stage 1:=====================> (48 + 8) / 128]16/12/12 14:37:10 ERROR TaskSchedulerImpl: Lost executor 7 on Mcnode1: worker lost + 16/12/12 14:37:10 ERROR TaskSchedulerImpl: Lost executor 6 on Mcnode5: worker lost + 16/12/12 14:37:10 ERROR TaskSchedulerImpl: Lost executor 4 on Mcnode4: worker lost + 16/12/12 14:37:10 ERROR TaskSchedulerImpl: Lost executor 2 on Mcnode3: worker lost + 16/12/12 14:37:10 ERROR TaskSchedulerImpl: Lost executor 1 on Mcnode6: worker lost + 16/12/12 14:37:10 ERROR TaskSchedulerImpl: Lost executor 0 on Mcnode2: worker lost + [Stage 1:====================> (48 + -40) / 128]Exception in thread "main" org.apache.spark.SparkException: Job cancelled because SparkContext was shut down +## 1.2 第二次## + + hadoop@Master:~/disk2/xubo/project/alignment/SparkSW/SparkSW20161114/alluxio-1.3.0$ tail -f DSWSparkSWquerytime201612121433.txt + SparkSW: + time: + [Stage 1:> (0 + 16) / 128]16/12/12 15:06:25 ERROR TaskSchedulerImpl: Lost executor 1 on Mcnode7: worker lost + [Stage 1:> (0 + 14) / 128]16/12/12 15:06:58 ERROR TaskSchedulerImpl: Lost executor 7 on Mcnode6: worker lost + 16/12/12 15:06:58 ERROR TaskSchedulerImpl: Lost executor 6 on Mcnode1: worker lost + 16/12/12 15:06:58 ERROR TaskSchedulerImpl: Lost executor 5 on Mcnode3: worker lost + 16/12/12 15:06:58 ERROR TaskSchedulerImpl: Lost executor 4 on Mcnode2: worker lost + 16/12/12 15:06:58 ERROR TaskSchedulerImpl: Lost executor 3 on Mcnode4: worker lost + 16/12/12 15:06:58 ERROR TaskSchedulerImpl: Lost executor 2 on Mcnode5: worker lost + [Stage 1:> (0 + 2) / 128] + +![](http://i.imgur.com/j1i2FxX.png) + +# 2.解决办法： # + +## 2.1 进度 +待解决 + +## 2.2 尝试1 +按照【3】尝试： + + hadoop@Master:~/disk2/xubo/project/alignment/SparkSW/SparkSW20161114/alluxio-1.3.0$ tail -f DSWSparkSWquerytime201612121533.txt + SparkSW: + time: + /home/hadoop/cloud/spark-1.5.2/conf/spark-env.sh: line 50: spark.worker.ui.retainedExecutors: command not found + /home/hadoop/cloud/spark-1.5.2/conf/spark-env.sh: line 51: spark.worker.ui.retainedDrivers: command not found + [Stage 1:======> (16 + 16) / 128] + +## 2.3 尝试2 +然后自己把spark下面的work的上个月的app移到disk2备份，其他节点也是。 + + +## 2.4 学习 + +源代码阅读： 只有一处有 "worker lost" + + for (exec <- worker.executors.values) { + logInfo("Telling app of lost executor: " + exec.id) + exec.application.driver.send(ExecutorUpdated( + exec.id, ExecutorState.LOST, Some("worker lost"), None)) + exec.application.removeExecutor(exec) + } + + +# 3.运行记录： # + +待解决 + +参考 + + 【1】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【2】https://github.com/xubo245/SparkLearning + 【3】http://blog.csdn.net/lsshlsw/article/details/49155087 diff --git "a/docs/SparkProblem/Spark\351\227\256\351\242\230\344\271\213.md" "b/docs/SparkProblem/Spark\351\227\256\351\242\230\344\271\213.md" new file mode 100644 index 0000000..76dbe0c --- /dev/null +++ "b/docs/SparkProblem/Spark\351\227\256\351\242\230\344\271\213.md" @@ -0,0 +1,29 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio（tachyon） 0.7.1，spark-1.5.2,hadoop-2.6.0 + +# 1.问题描述 # + +## 1.1 ## + + +## 1.2 ## + + +# 2.解决办法： # + +## 2.1 ## + + +## 2.2 ## + + + +# 3.运行记录： # + + +参考 + + 【1】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【2】https://github.com/xubo245/SparkLearning diff --git "a/docs/SparkSourceLearning/spark\346\272\220\347\240\201\347\226\221\351\227\2561\344\271\213\345\246\202\344\275\225\344\275\277\347\224\250reduce\345\256\236\347\216\260reduceByKey\347\232\204\345\212\237\350\203\275\357\274\237.md" "b/docs/SparkSourceLearning/spark\346\272\220\347\240\201\347\226\221\351\227\2561\344\271\213\345\246\202\344\275\225\344\275\277\347\224\250reduce\345\256\236\347\216\260reduceByKey\347\232\204\345\212\237\350\203\275\357\274\237.md" new file mode 100644 index 0000000..f5ac933 --- /dev/null +++ "b/docs/SparkSourceLearning/spark\346\272\220\347\240\201\347\226\221\351\227\2561\344\271\213\345\246\202\344\275\225\344\275\277\347\224\250reduce\345\256\236\347\216\260reduceByKey\347\232\204\345\212\237\350\203\275\357\274\237.md" @@ -0,0 +1,63 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +spark源码解读系列环境：spark-1.5.2、hadoop-2.6.0、scala-2.10.4 + +# 1.理解 # + +不知道怎么返回RDD[(String, Int) + + + +# 2.代码： # + + package util + + import org.apache.spark.rdd.RDD + import org.apache.spark.{SparkConf, SparkContext} + + /** + * Created by xubo on 2016/10/31. + */ + object ReduceTest { + def main(args: Array[String]) { + val conf = new SparkConf().setAppName("ReduceTest Application").setMaster("local[4]") + val sc = new SparkContext(conf) + val rdd = sc.parallelize(Array(("hello", 1), ("hello", 2), ("world", 5))) + val result=reduceByKeyTest(sc, rdd) + println(result) + sc.stop() + } + + /** + * + * @param sc + * @param rdd + * @return + */ + def reduceByKeyTest(sc: SparkContext, rdd: RDD[(String, Int)]):(String, Int) = { + val rdd2=rdd.reduce { (a, b) => + if (a._1 == b._1) { + (a._1, a._2 + b._2) + } else { + a + } + } + rdd2 + } + } + + + + +# 3.结果： # + + + +参考 + + 【1】http://spark.apache.org/ + 【2】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【3】https://github.com/xubo245/SparkLearning + 【4】book:《深入理解spark核心思想与源码分析》 + 【5】book:《spark核心源码分析和开发实战》 diff --git "a/docs/SparkSourceLearning/spark\346\272\220\347\240\201\350\247\243\350\257\273100\344\271\213\346\250\241\347\211\210.md" "b/docs/SparkSourceLearning/spark\346\272\220\347\240\201\350\247\243\350\257\273100\344\271\213\346\250\241\347\211\210.md" new file mode 100644 index 0000000..2584f9b --- /dev/null +++ "b/docs/SparkSourceLearning/spark\346\272\220\347\240\201\350\247\243\350\257\273100\344\271\213\346\250\241\347\211\210.md" @@ -0,0 +1,25 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +spark源码解读系列环境：spark-1.5.2、hadoop-2.6.0、scala-2.10.4 + +# 1.理解 # + + + + +# 2.代码： # + + + +# 3.结果： # + + + +参考 + + 【1】http://spark.apache.org/ + 【2】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【3】https://github.com/xubo245/SparkLearning + 【4】book:《深入理解spark核心思想与源码分析》 + 【5】book:《spark核心源码分析和开发实战》 diff --git "a/docs/SparkSourceLearning/spark\346\272\220\347\240\201\350\247\243\350\257\2731\344\271\213Partitioner.md" "b/docs/SparkSourceLearning/spark\346\272\220\347\240\201\350\247\243\350\257\2731\344\271\213Partitioner.md" new file mode 100644 index 0000000..48ba653 --- /dev/null +++ "b/docs/SparkSourceLearning/spark\346\272\220\347\240\201\350\247\243\350\257\2731\344\271\213Partitioner.md" @@ -0,0 +1,302 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +spark源码解读系列环境：spark-1.5.2、hadoop-2.6.0、scala-2.10.4 + +# 1.理解 # + +# Partitioner类 # +Partitioner类是用于处理key-value类型的RDD，根据key进行元素划分。 + +Partitioner是一个抽象类。只有两个方法：numPartitions和getPartition(key: Any)。 + + /** + * An object that defines how the elements in a key-value pair RDD are partitioned by key. + * Maps each key to a partition ID, from 0 to `numPartitions - 1`. + */ + abstract class Partitioner extends Serializable { + def numPartitions: Int + def getPartition(key: Any): Int + } +Partitioner同时有一个伴生对象，有defaultPartitioner方法，默认是：HashPartitioner + + object Partitioner { + def defaultPartitioner(rdd: RDD[_], others: RDD[_]*): Partitioner = { + val bySize = (Seq(rdd) ++ others).sortBy(_.partitions.size).reverse + for (r <- bySize if r.partitioner.isDefined && r.partitioner.get.numPartitions > 0) { + return r.partitioner.get + } + if (rdd.context.conf.contains("spark.default.parallelism")) { + new HashPartitioner(rdd.context.defaultParallelism) + } else { + new HashPartitioner(bySize.head.partitions.size) + } + } + } + + +# Spark的Partitioner类的子类 # + +在Spark的Partitioner类中，存在两类Partitioner函数：HashPartitioner和RangePartitioner，它们都是继承自Partitioner，主要提供了每个RDD有几个分区（numPartitions）以及对于给定的值返回一个分区ID（0~numPartitions-1），也就是决定这个值是属于那个分区的。 + +## （1）HashPartitioner ## +　HashPartitioner分区的原理很简单，对于给定的key，计算其hashCode，并除于分区的个数取余，如果余数小于0，则用余数+分区的个数，最后返回的值就是这个key所属的分区ID。 + +子类继承父抽象类，需要实现抽象类中的抽象方法。Partitioner主要有两个：numPartitions和getPartition。 + +HashPartitioner实现： + def numPartitions: Int = partitions + + def getPartition(key: Any): Int = key match { + case null => 0 + case _ => Utils.nonNegativeMod(key.hashCode, numPartitions) + } +numPartitions好理解，getPartition使用了Utils的方法： + + def nonNegativeMod(x: Int, mod: Int): Int = { + val rawMod = x % mod + rawMod + (if (rawMod < 0) mod else 0) + } +主要是由于key.hashCode有可能为赋值，需要判断和处理。 + +另外HashPatitioner重写了两个方法： + + override def equals(other: Any): Boolean = other match { + case h: HashPartitioner => + h.numPartitions == numPartitions + case _ => + false + } + + override def hashCode: Int = numPartitions +这个好理解。 + + +## （2）RangePartitioner ## +从HashPartitioner分区的实现原理我们可以看出，其结果可能导致每个分区中数据量的不均匀，极端情况下会导致某些分区拥有RDD的全部数据，这显然不是我们需要的。而RangePartitioner分区则尽量保证每个分区中数据量的均匀，而且分区与分区之间是有序的，也就是说一个分区中的元素肯定都是比另一个分区内的元素小或者大；但是分区内的元素是不能保证顺序的。简单的说就是将一定范围内的数映射到某一个分区内。 + +RangePartitioner分区器的主要作用就是将一定范围内的数映射到某一个分区内，所以它的实现中分界的算法尤为重要 + +rangeBounds的边界方法rangeBounds，主要是利用 RangePartitioner伴生对象的sketch方法进行抽样，抽样方法采取的是水塘抽样(Reservoir Sampling)，详细可以参考【8】、【9】。 +水塘抽样(Reservoir Sampling)的主要思想是： + + 在序列流中取一个数，如何确保随机性，即取出某个数据的概率为:1/(已读取数据个数) + + 　　假设已经读取n个数，现在保留的数是Ax，取到Ax的概率为(1/n)。 + + 　　对于第n+1个数An+1，以1/(n+1)的概率取An+1，否则仍然取Ax。依次类推，可以保证取到数据的随机性。 + + 　　数学归纳法证明如下： + + 　　　　当n=1时，显然，取A1。取A1的概率为1/1。 + + 假设当n=k时，取到的数据Ax。取Ax的概率为1/k。 + + 当n=k+1时，以1/(k+1)的概率取An+1，否则仍然取Ax。 + + 　　　　(1)如果取Ak+1，则概率为1/(k+1)； + + 　　　　(2)如果仍然取Ax，则概率为(1/k)*(k/(k+1))=1/(k+1) + +采用水塘抽样(Reservoir Sampling)主要是解决Spark 1.1中rangeBounds需要先知道rdd的size，然后再去抽样的的问题，因为这种需要两次遍历RDD，而水塘抽样(Reservoir Sampling)可以在不知道总size的情况下进行抽样，特别适用于数据在内存存不下的情况。 + + RangePartitioner伴生对象的sketch方法： + + /** + * Sketches the input RDD via reservoir sampling on each partition. + * + * @param rdd the input RDD to sketch + * @param sampleSizePerPartition max sample size per partition + * @return (total number of items, an array of (partitionId, number of items, sample)) + */ + def sketch[K : ClassTag]( + rdd: RDD[K], + sampleSizePerPartition: Int): (Long, Array[(Int, Int, Array[K])]) = { + val shift = rdd.id + // val classTagK = classTag[K] // to avoid serializing the entire partitioner object + val sketched = rdd.mapPartitionsWithIndex { (idx, iter) => + val seed = byteswap32(idx ^ (shift << 16)) + val (sample, n) = SamplingUtils.reservoirSampleAndCount( + iter, sampleSizePerPartition, seed) + Iterator((idx, n, sample)) + }.collect() + val numItems = sketched.map(_._2.toLong).sum + (numItems, sketched) + } + +　RangePartitioner.sketch的第一个参数是rdd.map(_._1)，也就是把父RDD的key传进来，因为分区只需要对Key进行操作即可。该函数返回值是val (numItems, sketched) ，其中numItems相当于记录rdd元素的总数；而sketched的类型是Array[(Int, Int, Array[K])]，记录的是分区的编号、该分区中总元素的个数以及从父RDD中每个分区采样的数据。 + +　sketch函数对父RDD中的每个分区进行采样，并记录下分区的ID和分区中数据总和。 + +　reservoirSampleAndCount函数就是典型的水塘抽样实现，唯一不同的是该算法还记录下i的值，这个就是该分区中元素的总和。 + +如果获取的数据分布不均匀，则边界方法rangeBounds会再次抽样，但是只对抽象数少于要求的partition进行sample，其他抽样好的不会。 + +最后获取到每个partition中每个样本和对应的weight（ 类似candidates += ((key, weight))），weight为partition中元素数量与抽样数量的比值，对于重新抽样的，则为1. + +最后再进行边界确定：determineBounds + + /** + * Determines the bounds for range partitioning from candidates with weights indicating how many + * items each represents. Usually this is 1 over the probability used to sample this candidate. + * + * @param candidates unordered candidates with weights + * @param partitions number of partitions + * @return selected bounds + */ + def determineBounds[K : Ordering : ClassTag]( + candidates: ArrayBuffer[(K, Float)], + partitions: Int): Array[K] = { + val ordering = implicitly[Ordering[K]] + val ordered = candidates.sortBy(_._1) + val numCandidates = ordered.size + val sumWeights = ordered.map(_._2.toDouble).sum + val step = sumWeights / partitions + var cumWeight = 0.0 + var target = step + val bounds = ArrayBuffer.empty[K] + var i = 0 + var j = 0 + var previousBound = Option.empty[K] + while ((i < numCandidates) && (j < partitions - 1)) { + val (key, weight) = ordered(i) + cumWeight += weight + if (cumWeight > target) { + // Skip duplicate values. + if (previousBound.isEmpty || ordering.gt(key, previousBound.get)) { + bounds += key + target += step + j += 1 + previousBound = Some(key) + } + } + i += 1 + } + bounds.toArray + } + } + +determineBounds主要对 candidates先按key进行排序，然后获取总抽样元素除以partition大小即为每个partition理论的大小，即代码中的step。 +然后再对排序好的ordered进行遍历，当所代表的权重大于step的整数倍时，返回此时的key，作为划分条件。然后依次类推，获得每个partition的边界key。 + + +**总结：RangePartitioner是采取抽样的策略，每个partition理论的是抽取20个元素，实际采用水塘抽样(Reservoir Sampling)时为了避免抽样少于期望，会乘以3.然后再用determineBounds对抽样数据进行排序，weight是每个key所代表的抽样数量，再按weight确定每个partition接近理论的边界，并进行返回，即为partitionid（getPartition返回值）。** + +getPartition查找某个元素应该所属的partitionid时，如果partition数量过大，会采取二分查找。 +源代码中为了避免抽样数量少于partition数量，也采取了相对应的措施。 + + +## （3）GridPartitioner ## + +除了上述两个子类，Partitioner的子类还有：mllib中的GridPartitioner（org.apache.spark.mllib.linalg.distributed.GridPartitioner），是一个网格Partitioner，采用了规则的网格划分坐标。 +numPartitions等于行和列的partitioners之积： + + override val numPartitions: Int = rowPartitions * colPartitions +根据（i,j）获取partition的id： + + private def getPartitionId(i: Int, j: Int): Int = { + require(0 <= i && i < rows, s"Row index $i out of range [0, $rows).") + require(0 <= j && j < cols, s"Column index $j out of range [0, $cols).") + i / rowsPerPart + j / colsPerPart * rowPartitions + } + +## PartitionIdPassthrough ## +还有：PartitionIdPassthrough（org.apache.spark.sql.execution.PartitionIdPassthrough），是一个虚拟的partitioner，这些partition的id已经提前计算好了，所以直接返回。 + + /** + * A dummy partitioner for use with records whose partition ids have been pre-computed (i.e. for + * use on RDDs of (Int, Row) pairs where the Int is a partition id in the expected range). + */ + private class PartitionIdPassthrough(override val numPartitions: Int) extends Partitioner { + override def getPartition(key: Any): Int = key.asInstanceOf[Int] + } + +## ReferencePartitioner ## +除了spark源码中，spark上层的系统也有partitioner类的子类，比如Adam系统中的：ReferencePartitioner（org.bdgenomics.adam.rdd.ReferencePartitioner），ReferencePartitioner主要是根据参考位置或者参考域进行划分，划分一句是referenceName。 + +## GenomicPositionPartitioner和GenomicRegionPartitioner ## +Adam中还有： +GenomicPositionPartitioner（org.bdgenomics.adam.rdd.GenomicPositionPartitioner）和GenomicRegionPartitioner（org.bdgenomics.adam.rdd.GenomicRegionPartitioner） +这两个partitioner主要是将ReferencePosition对象划分到分开的，物理位置相关的基因组位置。这样主要是方便基因数据可以根据空间分布进行划分，然后进行计算。 + + +# 2.代码： # + +自己写的Partitioner：主要对url的数据进行划分，按域名来，如果域名相同，则hashCode相同。 +比如"https://github.com/xubo245/SparkLearning"和"https://github.com/xubo245"网址不同但域名相同，但想划分到一个Partition中，于是可以改写getPartition的数量 + + package org.apache.spark.sourceCode.partitionerLearning + + import org.apache.spark.Partitioner + + /** + * Created by xubo on 2016/10/8. + */ + class URLHashPartitioner(numParts: Int) extends Partitioner { + override def numPartitions: Int = numParts + + override def getPartition(key: Any): Int = { + val domain = new java.net.URL(key.toString).getHost() + val code = (domain.hashCode % numPartitions) + if (code < 0) { + code + numPartitions + } else { + code + } + } + + override def equals(other: Any): Boolean = other match { + case url: URLHashPartitioner => + url.numPartitions == numPartitions + case _ => + false + } + + override def hashCode: Int = numPartitions + } + + object URLHashPartitioner { + def main(args: Array[String]) { + + } + } + +测试代码： + + package org.apache.spark.sourceCode.partitionerLearning + + import org.apache.spark.util.SparkLearningFunSuite + + /** + * Created by xubo on 2016/10/8. + */ + class PartitionerTest extends SparkLearningFunSuite { + test("URLHashPartitioner") { + var url1 = new URLHashPartitioner(20) + var partition1 = url1.getPartition("https://github.com/xubo245/SparkLearning") + var partition2 = url1.getPartition("https://github.com/xubo245") + println(partition1) + println(partition2) + } + } + +SparkLearningFunSuite主要是方便测试，可以在参考【3】中看到源码 + +3.结果： + + 14 + 14 + +参考中【7】描述的比较详细，背景也介绍的很好，建议多参考。 + +参考 + + 【1】http://spark.apache.org/ + 【2】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【3】https://github.com/xubo245/SparkLearning + 【4】book:《深入理解spark核心思想与源码分析》 + 【5】book:《spark核心源码分析和开发实战》 + 【6】https://www.iteblog.com/archives/1368 + 【7】https://www.iteblog.com/archives/1522 + 【8】https://www.iteblog.com/archives/1525 + 【9】http://www.cnblogs.com/xudong-bupt/p/4053652.html diff --git "a/docs/SparkSourceLearning/spark\346\272\220\347\240\201\350\247\243\350\257\2732\344\271\213\346\260\264\345\241\230\346\212\275\346\240\267\347\256\227\346\263\225\357\274\210Reservoir Sampling\357\274\211.md" "b/docs/SparkSourceLearning/spark\346\272\220\347\240\201\350\247\243\350\257\2732\344\271\213\346\260\264\345\241\230\346\212\275\346\240\267\347\256\227\346\263\225\357\274\210Reservoir Sampling\357\274\211.md" new file mode 100644 index 0000000..5b0b845 --- /dev/null +++ "b/docs/SparkSourceLearning/spark\346\272\220\347\240\201\350\247\243\350\257\2732\344\271\213\346\260\264\345\241\230\346\212\275\346\240\267\347\256\227\346\263\225\357\274\210Reservoir Sampling\357\274\211.md" @@ -0,0 +1,156 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +spark源码解读系列环境：spark-1.5.2、hadoop-2.6.0、scala-2.10.4 + +# 1.理解 # + + +　　问题定义可以简化如下：在不知道文件总行数的情况下，如何从文件中随机的抽取一行？ + +　　首先想到的是我们做过类似的题目吗?当然，在知道文件行数的情况下，我们可以很容易的用C运行库的rand函数随机的获得一个行数，从而随机的取出一行，但是，当前的情况是不知道行数，这样如何求呢？我们需要一个概念来帮助我们做出猜想，来使得对每一行取出的概率相等，也即随机。这个概念即蓄水池抽样（Reservoir Sampling）。 + +## 水塘抽样算法（Reservoir Sampling）思想： ## + +　　**在序列流中取一个数，如何确保随机性，即取出某个数据的概率为:1/(已读取数据个数)** + +　　假设已经读取n个数，现在保留的数是Ax，取到Ax的概率为(1/n)。 + +　　对于第n+1个数An+1，以1/(n+1)的概率取An+1，否则仍然取Ax。依次类推，可以保证取到数据的随机性。 + +　　数学归纳法证明如下： + +　　　　当n=1时，显然，取A1。取A1的概率为1/1。 + + 假设当n=k时，取到的数据Ax。取Ax的概率为1/k。 + + 当n=k+1时，以1/(k+1)的概率取An+1，否则仍然取Ax。 + +　　　　(1)如果取Ak+1，则概率为1/(k+1)； + +　　　　(2)如果仍然取Ax，则概率为(1/k)*(k/(k+1))=1/(k+1) + +　　所以，对于之后的第n+1个数An+1，以1/(n+1)的概率取An+1，否则仍然取Ax。依次类推，可以保证取到数据的随机性。 + + +**在序列流中取k个数，如何确保随机性，即取出某个数据的概率为:k/(已读取数据个数)** + +　　建立一个数组，将序列流里的前k个数，保存在数组中。(也就是所谓的"蓄水池") + +　　对于第n个数An，以k/n的概率取An并以1/k的概率随机替换“蓄水池”中的某个元素；否则“蓄水池”数组不变。依次类推，可以保证取到数据的随机性。 + +　　数学归纳法证明如下： + +　　　　当n=k是，显然“蓄水池”中任何一个数都满足，保留这个数的概率为k/k。 + +　　　　假设当n=m(m>k)时，“蓄水池”中任何一个数都满足，保留这个数的概率为k/m。 +　　　　当n=m+1时，以k/(m+1)的概率取An，并以1/k的概率，随机替换“蓄水池”中的某个元素，否则“蓄水池”数组不变。则数组中保留下来的数的概率为： + +![](http://i.imgur.com/E1UyYIG.jpg) + +　　所以，对于第n个数An，以k/n的概率取An并以1/k的概率随机替换“蓄水池”中的某个元素；否则“蓄水池”数组不变。依次类推，可以保证取到数据的随机性。 +## Spark中的水塘抽样算法（Reservoir Sampling） ## +　　spark的Partitioner子类RangePartitioner中有用到Reservoir Sampling抽样算法(org.apache.spark.RangePartitioner#sketch). + +spark的util中有reservoirSampleAndCount方法（org.apache.spark.util.random.SamplingUtils#reservoirSampleAndCount） + +源码为： + + /** + * Reservoir sampling implementation that also returns the input size. + * + * @param input input size + * @param k reservoir size + * @param seed random seed + * @return (samples, input size) + */ + def reservoirSampleAndCount[T: ClassTag]( + input: Iterator[T], + k: Int, + seed: Long = Random.nextLong()) + : (Array[T], Int) = { + val reservoir = new Array[T](k) + // Put the first k elements in the reservoir. + var i = 0 + while (i < k && input.hasNext) { + val item = input.next() + reservoir(i) = item + i += 1 + } + + // If we have consumed all the elements, return them. Otherwise do the replacement. + if (i < k) { + // If input size < k, trim the array to return only an array of input size. + val trimReservoir = new Array[T](i) + System.arraycopy(reservoir, 0, trimReservoir, 0, i) + (trimReservoir, i) + } else { + // If input size > k, continue the sampling process. + val rand = new XORShiftRandom(seed) + while (input.hasNext) { + val item = input.next() + val replacementIndex = rand.nextInt(i) + if (replacementIndex < k) { + reservoir(replacementIndex) = item + } + i += 1 + } + (reservoir, i) + } + } + +代码实现思路比较简单，新建一个k大小的数组reservoir，如果元数据中数据少于k，直接返回原数据数组和原数据个数。如果大于，则对接下来的元素进行比较，随机生成一个数i，如果这个数小于k，则替换数组reservoir中第i个数，直至没有元素，则返回reservoir的copy数组。 + + +# 2.代码： # +测试org.apache.spark.util.random.SamplingUtils$#reservoirSampleAndCount方法： + + package org.apache.spark.sourceCode.partitionerLearning + + import org.apache.spark.util.SparkLearningFunSuite + import org.apache.spark.util.random.SamplingUtils + + import scala.util.Random + + /** + * Created by xubo on 2016/10/9. + */ + class reservoirSampleAndCountSuite extends SparkLearningFunSuite { + test("reservoirSampleAndCount") { + val input = Seq.fill(100)(Random.nextInt()) + val (sample1, count1) = SamplingUtils.reservoirSampleAndCount(input.iterator, 150) + assert(count1 === 100) + assert(input === sample1.toSeq) + + // input size == k + val (sample2, count2) = SamplingUtils.reservoirSampleAndCount(input.iterator, 100) + assert(count2 === 100) + assert(input === sample2.toSeq) + + // input size > k + val (sample3, count3) = SamplingUtils.reservoirSampleAndCount(input.iterator, 10) + assert(count3 === 100) + assert(sample3.length === 10) + println(input) + sample3.foreach{each=>print(each+" ")} + } + + } + + +# 3.结果： # + + List(1287104639, 547232730, -595310393, -1264894486, 427750044, -776002403, 32230947, -1390386390, 484259687, 774711013, -1989325813, -957970416, 945685455, -1322730587, -1919655222, 1642426087, -489524599, -1070401860, -1454008456, -1882431453, -1843815884, -1987533758, -854529853, 879991257, -864077378, 478381860, 111307761, 1504756336, -1892792571, -1413976846, -848218587, -101494119, 1592476609, 247606007, 1269634528, 568928892, 488930464, -2145986422, 1643110602, 280675891, -878405966, 1799740067, 981424562, -1552824965, -1760162041, -288189264, -373755181, -2112636248, -2108911467, -1815555415, 302051417, 254178521, -1137490849, 426066017, -819810525, 1408383341, 1183678420, 234717727, 1470632905, 271163573, -22448780, 486064749, 378168799, -1444541974, 419089337, 1700972847, 1291787131, 644012641, -1618133452, 313585654, 658987252, 869334013, -811750155, -1561229418, 814819564, -197177628, 1051344432, 1746109173, 358985873, -265551244, 1362130460, -1635943643, 168813599, -669120136, -1084593890, -150445899, 387678120, 1994726806, 71986215, 1323527748, 700729367, 219285004, -1513691303, -97767338, 2099894386, -652208741, 704524016, 123647594, -1281589410, -1713105982) + -197177628 -22448780 478381860 -1137490849 219285004 168813599 1269634528 -1454008456 658987252 378168799 + + +参考 + + 【1】http://spark.apache.org/ + 【2】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【3】https://github.com/xubo245/SparkLearning + 【4】book:《深入理解spark核心思想与源码分析》 + 【5】book:《spark核心源码分析和开发实战》 + 【6】http://www.cnblogs.com/xudong-bupt/p/4053652.html + 【7】http://www.cnblogs.com/HappyAngel/archive/2011/02/07/1949762.html + 【8】https://www.iteblog.com/archives/1525 diff --git "a/docs/SparkSourceLearning/spark\346\272\220\347\240\201\350\247\243\350\257\2733\344\271\213RDD\344\270\255top\346\272\220\347\240\201\350\247\243\350\257\273.md" "b/docs/SparkSourceLearning/spark\346\272\220\347\240\201\350\247\243\350\257\2733\344\271\213RDD\344\270\255top\346\272\220\347\240\201\350\247\243\350\257\273.md" new file mode 100644 index 0000000..9d72652 --- /dev/null +++ "b/docs/SparkSourceLearning/spark\346\272\220\347\240\201\350\247\243\350\257\2733\344\271\213RDD\344\270\255top\346\272\220\347\240\201\350\247\243\350\257\273.md" @@ -0,0 +1,357 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +spark源码解读系列环境：spark-2.0.1 （20161103github下载版） + +# 1.理解 # + +输出读取中常用到topK算法，RDD也提供了top方法。特别是RDD过大时，要慎用RDD的collect方法，建议使用take和top方法。如果要有序，可以使用top方法。 + +## 1.1 定义 ## + + def top(num: Int)(implicit ord: Ordering[T]): Array[T] = withScope { + takeOrdered(num)(ord.reverse) + } + +num为要取的额个数，ord为隐式转换，可以取最高的topK，也可以放入逆序取最低的topK，top方法调用的是takeOrdered方法。 + +## 1.2 源码理解 ## + +###1.2.1 takeOrdered### +top调用的是takeOrdered，top调用的是takeOrdered的源码为： + + /** + * Returns the first k (smallest) elements from this RDD as defined by the specified + * implicit Ordering[T] and maintains the ordering. This does the opposite of [[top]]. + * For example: + * {{{ + * sc.parallelize(Seq(10, 4, 2, 12, 3)).takeOrdered(1) + * // returns Array(2) + * + * sc.parallelize(Seq(2, 3, 4, 5, 6)).takeOrdered(2) + * // returns Array(2, 3) + * }}} + * + * @note this method should only be used if the resulting array is expected to be small, as + * all the data is loaded into the driver's memory. + * + * @param num k, the number of elements to return + * @param ord the implicit ordering for T + * @return an array of top elements + */ + + def takeOrdered(num: Int)(implicit ord: Ordering[T]): Array[T] = withScope { + if (num == 0) { + Array.empty + } else { + val mapRDDs = mapPartitions { items => + // Priority keeps the largest elements, so let's reverse the ordering. + val queue = new BoundedPriorityQueue[T](num)(ord.reverse) + queue ++= util.collection.Utils.takeOrdered(items, num)(ord) + Iterator.single(queue) + } + if (mapRDDs.partitions.length == 0) { + Array.empty + } else { + mapRDDs.reduce { (queue1, queue2) => + queue1 ++= queue2 + queue1 + }.toArray.sorted(ord) + } + } + } + +理解： +1.2.1.1 takeOrdered会使用有界的优先队列BoundedPriorityQueue，存储返回的k个元素。 +1.2.1.2 mapPartitions是对每一个partition进行操作，对每个partition元素集合items,调用org.apache.spark.util.collection.takeOrdered取num个数，然后生成由若干个partition组成的mapRDDs，每个partition为大小为k的有界优先队列queue +1.2.1.3 然后进行reduce操作，reduce是将两个queue进行++操作，即将两个长度为k的queue1和queue2合并成一个长为1的queue。然后进行toArray和sort（ord）。++方法为BoundedPriorityQueue类中的方法，++会调用+=方法进行操作： + + override def ++=(xs: TraversableOnce[A]): this.type = { + xs.foreach { this += _ } + this + } + + override def +=(elem: A): this.type = { + if (size < maxSize) { + underlying.offer(elem) + } else { + maybeReplaceLowest(elem) + } + this + } + 具体可以查看BoundedPriorityQueue的方法 + + sorted(ord)方法调用java.util.Arrays.sort，后面1.2.3.1 会讲到 + +###1.2.2 org.apache.spark.util.collection.takeOrdered### +org.apache.spark.util.collection.takeOrdered的takeOrdered是调用的com.google.common.collect.{Ordering => GuavaOrdering} +的方法，并且重写了compare方法，主要是Ordering默认的是从小到大，而top默认是取最大的num个元素 + + val ordering = new GuavaOrdering[T] { + override def compare(l: T, r: T): Int = ord.compare(l, r) + } + +然后再调用 ordering的leastOf方法，中间有java和scala的iterator容器的相互转换： + + ordering.leastOf(input.asJava, num).iterator.asScala +###1.2.3 com.google.common.collect.Ordering的leastOf方法### + 包的引入方式：在maven的pom文件中加入 + + + com.google.guava + guava + 14.0.1 + provided + + +源码： + + /** + * Returns the {@code k} least elements from the given iterator according to + * this ordering, in order from least to greatest. If there are fewer than + * {@code k} elements present, all will be included. + * + *

The implementation does not necessarily use a stable sorting + * algorithm; when multiple elements are equivalent, it is undefined which + * will come first. + * + * @return an immutable {@code RandomAccess} list of the {@code k} least + * elements in ascending order + * @throws IllegalArgumentException if {@code k} is negative + * @since 14.0 + */ + public List leastOf(Iterator elements, int k) { + checkNotNull(elements); + checkArgument(k >= 0, "k (%s) must be nonnegative", k); + + if (k == 0 || !elements.hasNext()) { + return ImmutableList.of(); + } else if (k >= Integer.MAX_VALUE / 2) { + // k is really large; just do a straightforward sorted-copy-and-sublist + ArrayList list = Lists.newArrayList(elements); + Collections.sort(list, this); + if (list.size() > k) { + list.subList(k, list.size()).clear(); + } + list.trimToSize(); + return Collections.unmodifiableList(list); + } + + /* + * Our goal is an O(n) algorithm using only one pass and O(k) additional + * memory. + * + * We use the following algorithm: maintain a buffer of size 2*k. Every time + * the buffer gets full, find the median and partition around it, keeping + * only the lowest k elements. This requires n/k find-median-and-partition + * steps, each of which take O(k) time with a traditional quickselect. + * + * After sorting the output, the whole algorithm is O(n + k log k). It + * degrades gracefully for worst-case input (descending order), performs + * competitively or wins outright for randomly ordered input, and doesn't + * require the whole collection to fit into memory. + */ + int bufferCap = k * 2; + @SuppressWarnings("unchecked") // we'll only put E's in + E[] buffer = (E[]) new Object[bufferCap]; + E threshold = elements.next(); + buffer[0] = threshold; + int bufferSize = 1; + // threshold is the kth smallest element seen so far. Once bufferSize >= k, + // anything larger than threshold can be ignored immediately. + + while (bufferSize < k && elements.hasNext()) { + E e = elements.next(); + buffer[bufferSize++] = e; + threshold = max(threshold, e); + } + + while (elements.hasNext()) { + E e = elements.next(); + if (compare(e, threshold) >= 0) { + continue; + } + + buffer[bufferSize++] = e; + if (bufferSize == bufferCap) { + // We apply the quickselect algorithm to partition about the median, + // and then ignore the last k elements. + int left = 0; + int right = bufferCap - 1; + + int minThresholdPosition = 0; + // The leftmost position at which the greatest of the k lower elements + // -- the new value of threshold -- might be found. + + while (left < right) { + int pivotIndex = (left + right + 1) >>> 1; + int pivotNewIndex = partition(buffer, left, right, pivotIndex); + if (pivotNewIndex > k) { + right = pivotNewIndex - 1; + } else if (pivotNewIndex < k) { + left = Math.max(pivotNewIndex, left + 1); + minThresholdPosition = pivotNewIndex; + } else { + break; + } + } + bufferSize = k; + + threshold = buffer[minThresholdPosition]; + for (int i = minThresholdPosition + 1; i < bufferSize; i++) { + threshold = max(threshold, buffer[i]); + } + } + } + + Arrays.sort(buffer, 0, bufferSize, this); + + bufferSize = Math.min(bufferSize, k); + return Collections.unmodifiableList( + Arrays.asList(ObjectArrays.arraysCopyOf(buffer, bufferSize))); + // We can't use ImmutableList; we have to be null-friendly! + } + + private int partition( + E[] values, int left, int right, int pivotIndex) { + E pivotValue = values[pivotIndex]; + + values[pivotIndex] = values[right]; + values[right] = pivotValue; + + int storeIndex = left; + for (int i = left; i < right; i++) { + if (compare(values[i], pivotValue) < 0) { + ObjectArrays.swap(values, storeIndex, i); + storeIndex++; + } + } + ObjectArrays.swap(values, right, storeIndex); + return storeIndex; + } + + +#### 源码分析 #### +1.2.3.1 当k满足(k >= Integer.MAX_VALUE / 2)时，采用“straightforward sorted-copy-and-sublist”，直接排序-复制和取子串的方式操作 +其中排序算法直接调用 Collections.sort(list, this)，而其又调用 Arrays.sort(a, (Comparator)c); + +Arrays.sort源码： + + public static void sort(T[] a, Comparator c) { + if (LegacyMergeSort.userRequested) + legacyMergeSort(a, c); + else + TimSort.sort(a, c); + } +legacyMergeSort方法为传统的归并排序，当分到小于INSERTIONSORT_THRESHOLD（代码中设为7）时，采用插入排序，当大于INSERTIONSORT_THRESHOLD时采用归并排序，代码可见：java.util.Arrays#mergeSort(java.lang.Object[], java.lang.Object[], int, int, int)，不详细讲 + +Array的sort方法中还提供年了TimSort： + + TimSort.sort(a, c); + + 具体采用的是Tim Peters's list sort for Python + ( + TimSort). + +1.2.3.2 当k < Integer.MAX_VALUE / 2时，新建一个buffer，大小为2*k，当buffer元素小于k且有元素时，直接插入： + + while (bufferSize < k && elements.hasNext()) { + E e = elements.next(); + buffer[bufferSize++] = e; + threshold = max(threshold, e); + } +threshold取max，max不一定是最大值，里面调用了compare，compare方法重写了，所以需要根据实际情况分析，top方法默认的max是取最小值 + +当buffer中元素多于k时，则与threshold比较，如果campare结果符合才插入，当buffer元素达到2*k时，会调用 quickselect algorithm 即快速选择算法，取buffer符合要求的前k个，实际没有删除，而是移动元素，将符合的放在buffer中的前k个，后k个后面可能会被覆盖。 + +**1.2.3.2.1 quickselect algorithm** + quickselect algorithm 大致思路是去中间值作为划分界限，然后遍历buffer中元素，compare符合的放在k前面，不符合的放在k后面，里面会调用partition去操作，并且返回中间值移动后的位置storeIndex，然后将该位置storeIndex再与比较k比较，如果大于k，则在left到storeIndex间继续partition操作，如果storeIndex小于k，则在storeIndex到right间partition操作，否则正好符合要求 + +#### 1.2.3.3 返回 #### +最后在Arrays.sort方法，对buffer中的元素进行排序，最后取k个，copy返回 + + Arrays.sort(buffer, 0, bufferSize, this); + + bufferSize = Math.min(bufferSize, k); + return Collections.unmodifiableList( + Arrays.asList(ObjectArrays.arraysCopyOf(buffer, bufferSize))); + + +# 2.代码： # + +## （1）使用 ## +取最大的topK： + + val nums = Array(4,5,3,2,1,6,7,9,8,10) + val ints = sc.makeRDD(scala.util.Random.shuffle(nums), 2) + val topK = ints.top(5) + topK.foreach(println) + assert(topK.size === 5) + +输出： + + 10 + 9 + 8 + 7 + 6 + +取最小的topK： + + val nums = Array(4,5,3,2,1,6,7,9,8,10) + implicit val ord = implicitly[Ordering[Int]].reverse + val ints = sc.makeRDD(scala.util.Random.shuffle(nums), 2) + val topK = ints.top(5) + topK.foreach(println) + +输出： + + 1 + 2 + 3 + 4 + 5 + +每个细节可以debug具体去看 + +# 3.结果： # + +样例运行成功，top方法基本理解，有几个疑问： + +3.1 为什么reduce结果toArray后要sorted？reduce返回的是有界的BoundedPriorityQueue对象，而且有序，为什么不用reverse操作，复杂度更低？ +可能情况：保证结果稳定？ + +代码：org.apache.spark.rdd.RDD#takeOrdered + + mapRDDs.reduce { (queue1, queue2) => + queue1 ++= queue2 + queue1 + }.toArray.sorted(ord) + +spark-2.0.1修改RDD的takeOrdered的sorted之后会报错：结果无序 + + Array(9, 10, 7, 8, 6) did not equal Array(10, 9, 8, 7, 6) + ScalaTestFailureLocation: org.apache.spark.rdd.RDDSuite$$anonfun$52 at (RDDSuite.scala:656) + org.scalatest.exceptions.TestFailedException: Array(9, 10, 7, 8, 6) did not equal Array(10, 9, 8, 7, 6) + + + +3.2 快排中为什么用2*k的buffer？为什么不直接用有界的优先队列？这样操作也简单，时间也更低？ +可能情况：避免极端情况？值相同的有多个；k比较大时维护有界的成本较大？ + +代码：com.google.common.collect.Ordering#leastOf(java.util.Iterator, int) + + int bufferCap = k * 2; + @SuppressWarnings("unchecked") // we'll only put E's in + E[] buffer = (E[]) new Object[bufferCap]; + E threshold = elements.next(); + buffer[0] = threshold; + int bufferSize = 1; + +参考 + + 【1】http://spark.apache.org/ + 【2】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【3】https://github.com/xubo245/SparkLearning + 【4】book:《深入理解spark核心思想与源码分析》 + 【5】book:《spark核心源码分析和开发实战》 diff --git "a/docs/SparkSourceLearning/spark\346\272\220\347\240\201\350\247\243\350\257\2734\344\271\213SortByKey.md" "b/docs/SparkSourceLearning/spark\346\272\220\347\240\201\350\247\243\350\257\2734\344\271\213SortByKey.md" new file mode 100644 index 0000000..e2d5c9b --- /dev/null +++ "b/docs/SparkSourceLearning/spark\346\272\220\347\240\201\350\247\243\350\257\2734\344\271\213SortByKey.md" @@ -0,0 +1,178 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +spark源码解读系列环境：spark-2.0.1 （20161103github下载版） + +# 1.理解 # + +## 1.1 需求 ## +使用spark的时候会经常使用sortBykey，比如wordCount后需要排序，可以使用sortBy，也可以先map然后再sortByKey，soerBy也是调用SortByKey + +## 1.2 源码 ## +SortByKey：org.apache.spark.rdd.OrderedRDDFunctions#sortByKey + + /** + * Sort the RDD by key, so that each partition contains a sorted range of the elements. Calling + * `collect` or `save` on the resulting RDD will return or output an ordered list of records + * (in the `save` case, they will be written to multiple `part-X` files in the filesystem, in + * order of the keys). + */ + // TODO: this currently doesn't work on P other than Tuple2! + def sortByKey(ascending: Boolean = true, numPartitions: Int = self.partitions.length) + : RDD[(K, V)] = self.withScope + { + val part = new RangePartitioner(numPartitions, self, ascending) + new ShuffledRDD[K, V, V](self, part) + .setKeyOrdering(if (ascending) ordering else ordering.reverse) + } + +sortBy：org.apache.spark.rdd.RDD#sortBy + + /** + * Return this RDD sorted by the given key function. + */ + def sortBy[K]( + f: (T) => K, + ascending: Boolean = true, + numPartitions: Int = this.partitions.length) + (implicit ord: Ordering[K], ctag: ClassTag[K]): RDD[T] = withScope { + this.keyBy[K](f) + .sortByKey(ascending, numPartitions) + .values + } + +## 1.3 分析 ## +### 1.3.1 sortByKey之数据partitioner类RangePartitioner ### +sortByKey使用了RangePartitioner，这个在前面的博文“spark源码解读1之Partitioner”中已经有初步分析。RangePartitioner能很大程度上避免hash出现数据的数据分布不均匀的情况 + +RangePartitioner会在determineBounds对边界进行排序，用的是scala.collection.SeqLike#sorted ，调用的是java.util.Arrays#sort(T[], java.util.Comparator)： + + public static void sort(T[] a, Comparator c) { + if (LegacyMergeSort.userRequested) + legacyMergeSort(a, c); + else + TimSort.sort(a, c); + } + +默认使用的是TimSort，一种java 7加入的新的排序算法，之前使用的是传统的归并排序。 + +TimSort使用的归并排序和binary insertion sort的结合，并且根据具体的数据特征确定runLength，更加高效，但也有人形式化分析会有bug：https://www.zhihu.com/question/28352462 + +TimSort下次再具体分析。 + +### 1.3.2 ShuffleRDD### + new ShuffledRDD并且返回，即为排序好的soetByKey的结果 + +1.3.2.1 partitions_属性 + new ShuffledRDD的数据存储在partitions_属性中，这个继承自父类RDD，final方法partitions会给partitions_赋值，调用的是getPartitions方法，然后zipWithIndex + +partitions源码： + + final def partitions: Array[Partition] = { + checkpointRDD.map(_.partitions).getOrElse { + if (partitions_ == null) { + partitions_ = getPartitions + partitions_.zipWithIndex.foreach { case (partition, index) => + require(partition.index == index, + s"partitions($index).partition == ${partition.index}, but it should equal $index") + } + } + partitions_ + } + } + +1.3.2.2 getPartitions方法 + +getPartitions在ShuffledRDD重写了： + + override def getPartitions: Array[Partition] = { + Array.tabulate[Partition](part.numPartitions)(i => new ShuffledRDDPartition(i)) + } + +part.numPartitions实际为1.3.1中传入的RangePartitioner的属性： + + def numPartitions: Int = rangeBounds.length + 1 + +而rangeBounds则是用水塘抽样算法（Reservoir Sampling）建立的边界范围， + + 1 = 632826677 + 0 = -841013005 + rangeBounds = {int[2]@5390} + +getPartitions后是确定RDD的partition数量和index + +只有当进行取数操作时，比如top（k）然后显示，数据才会划分到partitions_的每个values下 + +debug数据copy： + + "WrappedArray$ofRef" size = 333 + values = {WrappedArray$ofRef@5956} "WrappedArray$ofRef" size = 333 + 0 = {Tuple2$mcII$sp@7560} "(-1813557161,-1212512531)" + 1 = {Tuple2$mcII$sp@7561} "(-1144323740,933490971)" + 2 = {Tuple2$mcII$sp@7562} "(-12508600,-329995331)" + 3 = {Tuple2$mcII$sp@7563} "(-1570574142,-743284380)" + 5 = {Tuple2$mcII$sp@7565} "(-532362478,1106605038)" + 4 = {Tuple2$mcII$sp@7564} "(249668146,-1487774671)" + 6 = {Tuple2$mcII$sp@7566} "(-146176592,666226908)" + +本地debug的代码是： + + test("large array") { + val rand = new scala.util.Random() + val pairArr = Array.fill(1000) { + (rand.nextInt(), rand.nextInt()) + } + val pairs = sc.parallelize(pairArr, 3) + val sorted = pairs.sortByKey() + sorted.count() + sorted.top(3).foreach(println) + assert(sorted.partitions.size === 3) + assert(sorted.collect() === pairArr.sortBy(_._1)) + } + +ShuffledRDD的partitions_对应的是三个ParallelCollectionPartition，这个是RDD的依赖关系得到的，ParallelCollectionPartition类重写了getPartitions方法，所以 + + override def getPartitions: Array[Partition] = { + val slices = ParallelCollectionRDD.slice(data, numSlices).toArray + slices.indices.map(i => new ParallelCollectionPartition(id, i, slices(i))).toArray + } + +里面partition 的排序方法没找到，不知道逻辑，需要后续去学习RDD和DAG、Stage等代码。 + +# 2.代码： # + +sortByKey使用：org.apache.spark.rdd.SortingSuite + + test("sortByKey") { + val pairs = sc.parallelize(Array((1, 0), (2, 0), (0, 0), (3, 0)), 2) + assert(pairs.sortByKey().collect() === Array((0, 0), (1, 0), (2, 0), (3, 0))) + } + +sortBy：org.apache.spark.rdd.RDDSuite + + test("sortBy") { + val data = sc.parallelize(Seq("5|50|A", "4|60|C", "6|40|B")) + + val col1 = Array("4|60|C", "5|50|A", "6|40|B") + val col2 = Array("6|40|B", "5|50|A", "4|60|C") + val col3 = Array("5|50|A", "6|40|B", "4|60|C") + + assert(data.sortBy(_.split("\\|")(0)).collect() === col1) + assert(data.sortBy(_.split("\\|")(1)).collect() === col2) + assert(data.sortBy(_.split("\\|")(2)).collect() === col3) + } + + +# 3.结果： # + +3.1 TimSort有待学习 +3.2 RangePartitioner只是确定numPartitions和getPartition(key: Any)，partition内部如何排序没有看到 + +参考 + + 【1】http://spark.apache.org/ + 【2】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【3】https://github.com/xubo245/SparkLearning + 【4】book:《深入理解spark核心思想与源码分析》 + 【5】book:《spark核心源码分析和开发实战》 + 【6】http://blog.csdn.net/u014393917/article/details/50602047 diff --git "a/docs/SparkSourceLearning/spark\346\272\220\347\240\201\350\247\243\350\257\2735\344\271\213Java TimSort\357\274\210\346\234\252\345\256\214\346\210\220\357\274\211.md" "b/docs/SparkSourceLearning/spark\346\272\220\347\240\201\350\247\243\350\257\2735\344\271\213Java TimSort\357\274\210\346\234\252\345\256\214\346\210\220\357\274\211.md" new file mode 100644 index 0000000..2584f9b --- /dev/null +++ "b/docs/SparkSourceLearning/spark\346\272\220\347\240\201\350\247\243\350\257\2735\344\271\213Java TimSort\357\274\210\346\234\252\345\256\214\346\210\220\357\274\211.md" @@ -0,0 +1,25 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +spark源码解读系列环境：spark-1.5.2、hadoop-2.6.0、scala-2.10.4 + +# 1.理解 # + + + + +# 2.代码： # + + + +# 3.结果： # + + + +参考 + + 【1】http://spark.apache.org/ + 【2】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【3】https://github.com/xubo245/SparkLearning + 【4】book:《深入理解spark核心思想与源码分析》 + 【5】book:《spark核心源码分析和开发实战》 diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2401---\345\215\225\346\234\272\347\211\210\346\220\255\345\273\272\345\222\214\350\277\220\350\241\214.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2401---\345\215\225\346\234\272\347\211\210\346\220\255\345\273\272\345\222\214\350\277\220\350\241\214.md" new file mode 100644 index 0000000..1530208 --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2401---\345\215\225\346\234\272\347\211\210\346\220\255\345\273\272\345\222\214\350\277\220\350\241\214.md" @@ -0,0 +1,151 @@ +环境 +ubuntu 14.04 +Spark-1.5.2 +Tachyon-0.7.1 + +1．由于最近用的Spark-1.5.2系统默认的tachyon为0.7.1，在Spark-core的pom.xml可以查看 +另外虽然现在Tachyon改名为Alluxio ，但是不影响这个版本的使用 + +2.单机版搭建： +下载地址： + + https://github.com/Alluxio/alluxio/releases/tag/v0.7.1 + +下载的是：tachyon-0.7.1-bin.tar.gz + +3.安装 +修改配置文件： + + cp conf/alluxio-env.sh.template conf/alluxio-env.sh +然后format和start： + + ./bin/alluxio format + ./bin/alluxio-start.sh local + +查看安装情况： + + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ ./tachyon format + Connecting to localhost as xubo... + Warning: Permanently added 'localhost' (ECDSA) to the list of known hosts. + Formatting Tachyon Worker @ xubo + Connection to localhost closed. + Formatting Tachyon Master @ localhost + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ jps + 12576 SparkSubmit + 13760 Jps + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ ls + tachyon tachyon-mount.sh tachyon-start.sh tachyon-stop.sh tachyon-workers.sh + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ ./tachyon-start.sh local + Killed 0 processes on xubo + Killed 0 processes on xubo + Connecting to localhost as xubo... + Killed 0 processes on xubo + Connection to localhost closed. + [sudo] password for xubo: + Formatting RamFS: /mnt/ramdisk (1gb) + Starting master @ localhost + Starting worker @ xubo + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ jps + 12576 SparkSubmit + 14088 Jps + 14058 TachyonWorker + 14030 TachyonMaster + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ + +可以看出来master和worker已经启动了 +另外也可以从浏览器看： + + http://localhost:19999/home +localhost为自己的ip + +说明安装成功 + +4. 使用 +（1）指令 + + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ ./tachyon tfs -help + Usage: java TFsShell + [cat ] + [count ] + [ls ] + [lsr ] + [mkdir ] + [rm ] + [rmr ] + [tail ] + [touch ] + [mv ] + [copyFromLocal ] + [copyToLocal ] + [fileinfo ] + [location ] + [report ] + [request ] + [pin ] + [unpin ] + [free ] + [getUsedBytes] + [getCapacityBytes] + [du ] + +（2）测试用例 + + xubo@xubo:~/cloud/tachyon-0.7.1$ ./bin/tachyon runTest Basic CACHE_THROUGH + /default_tests_files/BasicFile_CACHE_THROUGH has been removed + 2016-05-04 22:19:45,075 INFO (MasterClient.java:connect) - Tachyon client (version 0.7.1) is trying to connect with master @ localhost/127.0.0.1:19998 + 2016-05-04 22:19:45,104 INFO (MasterClient.java:connect) - User registered with the master @ localhost/127.0.0.1:19998; got UserId 4 + 2016-05-04 22:19:45,130 INFO (CommonUtils.java:printTimeTakenMs) - createFile with fileId 3 took 59 ms. + 2016-05-04 22:19:45,153 INFO (WorkerClient.java:connect) - Trying to get local worker host : xubo + 2016-05-04 22:19:45,166 INFO (WorkerClient.java:connect) - Connecting local worker @ xubo/219.219.220.222:29998 + 2016-05-04 22:19:45,230 INFO (BlockOutStream.java:get) - Writing with local stream. tachyonFile: /default_tests_files/BasicFile_CACHE_THROUGH, blockIndex: 0, opType: CACHE_THROUGH + 2016-05-04 22:19:45,289 INFO (CommonUtils.java:createBlockPath) - Folder /mnt/ramdisk/tachyonworker/4 was created! + 2016-05-04 22:19:45,294 INFO (LocalBlockOutStream.java:) - /mnt/ramdisk/tachyonworker/4/3221225472 was created! tachyonFile: /default_tests_files/BasicFile_CACHE_THROUGH, blockIndex: 0, blockId: 3221225472, blockCapacityByte: 536870912 + 2016-05-04 22:19:45,370 INFO (CommonUtils.java:printTimeTakenMs) - writeFile to file /default_tests_files/BasicFile_CACHE_THROUGH took 239 ms. + 2016-05-04 22:19:45,420 INFO (CommonUtils.java:printTimeTakenMs) - readFile file /default_tests_files/BasicFile_CACHE_THROUGH took 50 ms. + Passed the test! + +（3）从本地上传文件到remote： + + xubo@xubo:~/cloud/tachyon-0.7.1$ ./bin/tachyon tfs copyFromLocal pom.xml / + Copied pom.xml to / +可以在浏览器中查看 + +（4）显示文件内容： + + xubo@xubo:~/cloud/test/tachyon$ ../../tachyon-0.7.1/bin/tachyon tfs copyFromLocal 1.txt / + Copied 1.txt to / + xubo@xubo:~/cloud/test/tachyon$ ../../tachyon-0.7.1/bin/tachyon tfs cat /1.txt + hello tachyon + 1 + 2 + 3 + +（5）Spark调用tachyon文件：待完成，Spark出问题了，请见下一篇博文 + +（6）关闭tachyon + + hadoop@Master:~/cloud/testByXubo/spark/tachyon/tachyon-0.7.1$ ./bin/tachyon-stop.sh + Killed 1 processes on Master + Killed 1 processes on Master + Connecting to localhost as hadoop... + jjKilled 0 processes on Master + Connection to localhost closed. + + +附：将tachyon脚本cp到usr的bin下：（需要本地编译，直接start的不行） + + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ sudo cp tachyon /usr/ + bin/ games/ include/ lib/ lib64/ local/ sbin/ share/ src/ + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ sudo cp tachyon /usr/local/bin/ + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ ta + tabs tac tachyon tail tailf tali tangle tap2deb tap2rpm tapconvert tar tarcat targen taskset +方便使用 + + +参考 +【1】http://alluxio.org/documentation/v0.7.1/ +【2】https://github.com/xubo245/SparkLearning +【3】http://alluxio.org/documentation/v0.7.1/Running-Tachyon-Locally.html + + + diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24010--\351\233\206\347\276\244\351\227\256\346\263\225\345\205\250\351\203\250\345\220\257\345\212\250\351\227\256\351\242\230\350\247\243\345\206\263.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24010--\351\233\206\347\276\244\351\227\256\346\263\225\345\205\250\351\203\250\345\220\257\345\212\250\351\227\256\351\242\230\350\247\243\345\206\263.md" new file mode 100644 index 0000000..73e6e40 --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24010--\351\233\206\347\276\244\351\227\256\346\263\225\345\205\250\351\203\250\345\220\257\345\212\250\351\227\256\351\242\230\350\247\243\345\206\263.md" @@ -0,0 +1,195 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio（tachyon） 0.7.1，spark-1.5.2,hadoop-2.6.0 + +# 1.解释 # +## 1.1 问题描述 ## + +之前博文没有解决的问题：http://blog.csdn.net/xubo245/article/details/51325834 + +具体： + + hadoop@Master:~/cloud/alluxio-0.7.1$ ./bin/tachyon format + Connecting to Mcnode1 as hadoop... + Formatting Tachyon Worker @ Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Formatting Tachyon Worker @ Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode3 + /home/hadoop/cloud/alluxio-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode4 + /home/hadoop/cloud/alluxio-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode5 + /home/hadoop/cloud/alluxio-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Formatting Tachyon Worker @ Mcnode6 + Connection to mcnode6 closed. + Formatting Tachyon Master @ Master +## 1.2分析： ## + + 主要是没有bin/java,因为Mcnode3，4，5节点在/usr/bin和/usr/local/bin没有java，java和javac在其他位置，而alluxio调用的是/usr/bin路径下的指令 + +## 1.3 解决办法 ## + + sudo update-alternatives --install /usr/bin/java java /usr/lib/jvm/jdk1.7.0/bin/java 300 + sudo update-alternatives --install /usr/bin/javac javac /usr/lib/jvm/jdk1.7.0/bin/javac 300 + +## 1.4 解决效果： ## + 将第五个节点按照解决办法处理后，就没有报错了： + + hadoop@Master:~/cloud/alluxio-0.7.1$ ./bin/tachyon format + Connecting to Mcnode1 as hadoop... + Formatting Tachyon Worker @ Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Formatting Tachyon Worker @ Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode3 + /home/hadoop/cloud/alluxio-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode4 + /home/hadoop/cloud/alluxio-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + Formatting Tachyon Worker @ Mcnode5 + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Formatting Tachyon Worker @ Mcnode6 + Connection to mcnode6 closed. + Formatting Tachyon Master @ Master + + + +# 2.代码： # + +## 2.1 解决办法 ## + + sudo update-alternatives --install /usr/bin/java java /usr/lib/jvm/jdk1.7.0/bin/java 300 + sudo update-alternatives --install /usr/bin/javac javac /usr/lib/jvm/jdk1.7.0/bin/javac 300 + +## 2.2 运行结果 ## + + hadoop@Mcnode3:~/cloud/alluxio-0.7.1$ sudo update-alternatives --install /usr/bin/java java /usr/lib/jvm/jdk1.7.0/bin/java 300 + [sudo] password for hadoop: + update-alternatives: using /usr/lib/jvm/jdk1.7.0/bin/java to provide /usr/bin/java (java) in auto mode + hadoop@Mcnode3:~/cloud/alluxio-0.7.1$ sudo update-alternatives --install /usr/bin/javac javac /usr/lib/jvm/jdk1.7.0/bin/javac 300 + update-alternatives: using /usr/lib/jvm/jdk1.7.0/bin/javac to provide /usr/bin/javac (javac) in auto mode + hadoop@Mcnode3:~/cloud/alluxio-0.7.1$ + +# 3.结果： # + +都按照解决办法解决：截图在本周的周报中 + + hadoop@Master:~/cloud/alluxio-0.7.1$ ./bin/tachyon format + Connecting to Mcnode1 as hadoop... + Formatting Tachyon Worker @ Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Formatting Tachyon Worker @ Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + Formatting Tachyon Worker @ Mcnode3 + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + Formatting Tachyon Worker @ Mcnode4 + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + Formatting Tachyon Worker @ Mcnode5 + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Formatting Tachyon Worker @ Mcnode6 + Connection to mcnode6 closed. + Connecting to Mcnode7 as hadoop... + Formatting Tachyon Worker @ Mcnode7 + Connection to mcnode7 closed. + Formatting Tachyon Master @ Master + hadoop@Master:~/cloud/alluxio-0.7.1$ ./bin/tachyon-start.sh all SudoMount + Killed 1 processes on Master + Killed 0 processes on Master + Connecting to Mcnode1 as hadoop... + Killed 1 processes on Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Killed 1 processes on Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + Killed 0 processes on Mcnode3 + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + Killed 0 processes on Mcnode4 + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + Killed 1 processes on Mcnode5 + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Killed 1 processes on Mcnode6 + Connection to mcnode6 closed. + Connecting to Mcnode7 as hadoop... + Killed 0 processes on Mcnode7 + Connection to mcnode7 closed. + Starting master @ Master + Connecting to Mcnode1 as hadoop... + [sudo] password for hadoop: + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + [sudo] password for hadoop: + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + [sudo] password for hadoop: + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode3 + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + [sudo] password for hadoop: + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode4 + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + [sudo] password for hadoop: + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode5 + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + [sudo] password for hadoop: + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode6 + Connection to mcnode6 closed. + Connecting to Mcnode7 as hadoop... + [sudo] password for hadoop: + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode7 + Connection to mcnode7 closed. + hadoop@Master:~/cloud/alluxio-0.7.1$ + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning + 【5】http://blog.csdn.net/edwzhang/article/details/7759933 diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24011--alluxio-1.3.0\351\233\206\347\276\244\351\205\215\347\275\256.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24011--alluxio-1.3.0\351\233\206\347\276\244\351\205\215\347\275\256.md" new file mode 100644 index 0000000..a920d31 --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24011--alluxio-1.3.0\351\233\206\347\276\244\351\205\215\347\275\256.md" @@ -0,0 +1,237 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +在spark上运行alluxio请参考下一个博客 + +# 1.解释 # + +## 1.1 下载## + + wget http://alluxio.org/downloads/files/1.3.0/alluxio-1.3.0-bin.tar.gz + tar xvfz alluxio-1.3.0-bin.tar.gz +## 1.2 配置## +复制 + + cp conf/alluxio-env.sh.template conf/alluxio-env.sh + +修改： + vi conf/alluxio-env.sh +修改内容： + + # The directory where Alluxio deployment is installed. (Default: the parent directory of libexec/). + # ALLUXIO_HOME + export ALLUXIO_HOME=/home/hadoop/cloud/alluxio-1.3.0 + + # The directory where log files are stored. (Default: ${ALLUXIO_HOME}/logs). + # ALLUXIO_LOGS_DIR + + # Hostname of the master. + # ALLUXIO_MASTER_HOSTNAME + export ALLUXIO_MASTER_HOSTNAME=Master + + # This is now deprecated. Support will be removed in v2.0 + # ALLUXIO_MASTER_ADDRESS + + # The directory where a worker stores in-memory data. (Default: /mnt/ramdisk). + # E.g. On linux, /mnt/ramdisk for ramdisk, /dev/shm for tmpFS; on MacOS, /Volumes/ramdisk for ramdisk + # ALLUXIO_RAM_FOLDER + export ALLUXIO_RAM_FOLDER=/home/hadoop/cloud/alluxio-1.3.0/ramdisk + + # Address of the under filesystem address. (Default: ${ALLUXIO_HOME}/underFSStorage) + # E.g. "/my/local/path" to use local fs, "hdfs://localhost:9000/alluxio" to use a local hdfs + # ALLUXIO_UNDERFS_ADDRESS + export ALLUXIO_UNDERFS_ADDRESS=hdfs://Master:9000 + + # How much memory to use per worker. (Default: 1GB) + # E.g. "1000MB", "2GB" + # ALLUXIO_WORKER_MEMORY_SIZE + export ALLUXIO_WORKER_MEMORY_SIZE=6GB + +保存，scp到集群中每个节点 + +## 1.3 启动## + + hadoop@Master:~/cloud/alluxio-1.3.0$ ./bin/alluxio format + Waiting for MASTER tasks to finish... + All MASTER tasks finished, please analyze the log at /home/hadoop/cloud/alluxio-1.3.0/logs/task.log. + Formatting Alluxio Master @ Master + hadoop@Master:~/cloud/alluxio-1.3.0$ ./bin/alluxio-start.sh all Mount + /home/hadoop/cloud/alluxio-1.3.0/bin + Killed 1 processes on Master + Killed 0 processes on Master + Waiting for WORKERS tasks to finish... + All WORKERS tasks finished, please analyze the log at /home/hadoop/cloud/alluxio-1.3.0/logs/task.log. + Starting master @ Master. Logging to /home/hadoop/cloud/alluxio-1.3.0/logs + Waiting for MASTER tasks to finish... + All MASTER tasks finished, please analyze the log at /home/hadoop/cloud/alluxio-1.3.0/logs/task.log. + +# 3.结果： # + +![](http://i.imgur.com/J5IjaLM.png) + +alluxio fs 等命令测试可以用 + + hadoop@Mcnode1:~/disk2/xubo/data/SparkSW/uniref$ time cat D6Line.fasta |grep -c GC + 618260 + + real 0m6.313s + user 0m6.169s + sys 0m1.056s + hadoop@Mcnode1:~/disk2/xubo/data/SparkSW/uniref$ time ~/cloud/alluxio-1.3.0/bin/alluxio fs cat /D6Line.fasta |grep -c GC + 618260 + + real 0m7.725s + user 0m9.663s + sys 0m1.247s + + +测试： + + hadoop@Master:~/cloud/alluxio-1.3.0$ ./bin/alluxio runTests + 2016-11-09 23:23:10,739 INFO type (MetricsSystem.java:startSinksFromConfig) - Starting sinks with config: {}. + 2016-11-09 23:23:10,742 INFO type (MetricsSystem.java:startSinks) - Sinks have already been started. + 2016-11-09 23:23:10,843 INFO type (AbstractClient.java:connect) - Alluxio client (version 1.3.0) is trying to connect with FileSystemMasterClient master @ Master/219.219.220.149:19998 + 2016-11-09 23:23:10,868 INFO type (AbstractClient.java:connect) - Client registered with FileSystemMasterClient master @ Master/219.219.220.149:19998 + runTest Basic CACHE_PROMOTE MUST_CACHE + 2016-11-09 23:23:11,088 INFO type (AbstractClient.java:connect) - Alluxio client (version 1.3.0) is trying to connect with BlockMasterClient master @ Master/219.219.220.149:19998 + 2016-11-09 23:23:11,090 INFO type (AbstractClient.java:connect) - Client registered with BlockMasterClient master @ Master/219.219.220.149:19998 + 2016-11-09 23:23:11,209 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@8408396 + 2016-11-09 23:23:11,315 INFO type (NettyChannelPool.java:createNewResource) - Created netty channel to with netty boostrap alluxio.client.block.BlockStoreContext$2@5087f4d0. + 2016-11-09 23:23:11,406 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@62b3a353 + 2016-11-09 23:23:11,461 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_CACHE_PROMOTE_MUST_CACHE took 472 ms. + 2016-11-09 23:23:11,567 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_CACHE_PROMOTE_MUST_CACHE took 106 ms. + Passed the test! + runTest BasicNonByteBuffer CACHE_PROMOTE MUST_CACHE + 2016-11-09 23:23:11,600 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@6dd8ade8 + 2016-11-09 23:23:11,611 INFO type (NettyChannelPool.java:createNewResource) - Created netty channel to with netty boostrap alluxio.client.block.BlockStoreContext$2@4aaf32dc. + 2016-11-09 23:23:11,682 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@18607df6 + 2016-11-09 23:23:11,699 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_CACHE_PROMOTE_MUST_CACHE took 114 ms. + 2016-11-09 23:23:11,755 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_CACHE_PROMOTE_MUST_CACHE took 56 ms. + Passed the test! + runTest Basic CACHE_PROMOTE CACHE_THROUGH + 2016-11-09 23:23:11,806 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@32c81878 + 2016-11-09 23:23:11,825 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@2928fd34 + 2016-11-09 23:23:11,990 INFO type (NettyChannelPool.java:createNewResource) - Created netty channel to with netty boostrap alluxio.client.block.BlockStoreContext$2@7ae5fdad. + 2016-11-09 23:23:12,397 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_CACHE_PROMOTE_CACHE_THROUGH took 637 ms. + 2016-11-09 23:23:12,408 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_CACHE_PROMOTE_CACHE_THROUGH took 10 ms. + Passed the test! + runTest BasicNonByteBuffer CACHE_PROMOTE CACHE_THROUGH + 2016-11-09 23:23:12,422 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@9d14a9 + 2016-11-09 23:23:12,435 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@7fe96694 + 2016-11-09 23:23:12,587 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@366872f6 + 2016-11-09 23:23:12,597 INFO type (NettyChannelPool.java:createNewResource) - Created netty channel to with netty boostrap alluxio.client.block.BlockStoreContext$2@414ad9fb. + 2016-11-09 23:23:13,006 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@65c83981 + 2016-11-09 23:23:13,022 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_CACHE_PROMOTE_CACHE_THROUGH took 442 ms. + 2016-11-09 23:23:13,057 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_CACHE_PROMOTE_CACHE_THROUGH took 35 ms. + Passed the test! + runTest Basic CACHE_PROMOTE THROUGH + 2016-11-09 23:23:13,070 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@a4f5b6d + 2016-11-09 23:23:13,082 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@6de4f076 + 2016-11-09 23:23:13,231 INFO type (NettyChannelPool.java:createNewResource) - Created netty channel to with netty boostrap alluxio.client.block.BlockStoreContext$2@34aa287f. + 2016-11-09 23:23:13,614 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_CACHE_PROMOTE_THROUGH took 556 ms. + 2016-11-09 23:23:13,660 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@5ee179dc + 2016-11-09 23:23:13,674 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@6f860b23 + 2016-11-09 23:23:13,907 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_CACHE_PROMOTE_THROUGH took 292 ms. + Passed the test! + runTest BasicNonByteBuffer CACHE_PROMOTE THROUGH + 2016-11-09 23:23:13,917 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@aa1e2a5 + 2016-11-09 23:23:13,928 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@7713c735 + 2016-11-09 23:23:14,423 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_CACHE_PROMOTE_THROUGH took 352 ms. + 2016-11-09 23:23:14,472 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@633e5715 + 2016-11-09 23:23:14,537 INFO type (NettyChannelPool.java:createNewResource) - Created netty channel to with netty boostrap alluxio.client.block.BlockStoreContext$2@b14de02. + 2016-11-09 23:23:14,610 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@65a714ff + 2016-11-09 23:23:14,624 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_CACHE_PROMOTE_THROUGH took 201 ms. + Passed the test! + runTest Basic CACHE_PROMOTE ASYNC_THROUGH + 2016-11-09 23:23:14,633 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@4f4fe346 + 2016-11-09 23:23:14,664 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@5a62681b + 2016-11-09 23:23:14,707 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_CACHE_PROMOTE_ASYNC_THROUGH took 82 ms. + 2016-11-09 23:23:14,744 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_CACHE_PROMOTE_ASYNC_THROUGH took 37 ms. + Passed the test! + runTest BasicNonByteBuffer CACHE_PROMOTE ASYNC_THROUGH + 2016-11-09 23:23:14,763 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_CACHE_PROMOTE_ASYNC_THROUGH took 14 ms. + 2016-11-09 23:23:14,773 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_CACHE_PROMOTE_ASYNC_THROUGH took 10 ms. + Passed the test! + runTest Basic CACHE MUST_CACHE + 2016-11-09 23:23:14,786 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_CACHE_MUST_CACHE took 12 ms. + 2016-11-09 23:23:14,794 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_CACHE_MUST_CACHE took 8 ms. + Passed the test! + runTest BasicNonByteBuffer CACHE MUST_CACHE + 2016-11-09 23:23:14,806 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@2a37801c + 2016-11-09 23:23:14,834 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@47089c2c + 2016-11-09 23:23:14,852 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_CACHE_MUST_CACHE took 52 ms. + 2016-11-09 23:23:14,880 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_CACHE_MUST_CACHE took 28 ms. + Passed the test! + runTest Basic CACHE CACHE_THROUGH + 2016-11-09 23:23:15,195 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_CACHE_CACHE_THROUGH took 314 ms. + 2016-11-09 23:23:15,220 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_CACHE_CACHE_THROUGH took 25 ms. + Passed the test! + runTest BasicNonByteBuffer CACHE CACHE_THROUGH + 2016-11-09 23:23:15,325 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_CACHE_CACHE_THROUGH took 73 ms. + 2016-11-09 23:23:15,336 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_CACHE_CACHE_THROUGH took 11 ms. + Passed the test! + runTest Basic CACHE THROUGH + 2016-11-09 23:23:15,412 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_CACHE_THROUGH took 75 ms. + 2016-11-09 23:23:15,534 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_CACHE_THROUGH took 122 ms. + Passed the test! + runTest BasicNonByteBuffer CACHE THROUGH + 2016-11-09 23:23:15,549 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@7ef30fce + 2016-11-09 23:23:15,561 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.FileSystemWorkerClientService$Client@3b4d67bb + 2016-11-09 23:23:15,715 INFO type (NettyChannelPool.java:createNewResource) - Created netty channel to with netty boostrap alluxio.client.block.BlockStoreContext$2@2bf9bcc6. + 2016-11-09 23:23:16,130 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_CACHE_THROUGH took 417 ms. + 2016-11-09 23:23:16,277 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_CACHE_THROUGH took 146 ms. + Passed the test! + runTest Basic CACHE ASYNC_THROUGH + 2016-11-09 23:23:16,285 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@43036682 + 2016-11-09 23:23:16,309 INFO type (ThriftClientPool.java:createNewResource) - Created a new thrift client alluxio.thrift.BlockWorkerClientService$Client@5051f26e + 2016-11-09 23:23:16,326 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_CACHE_ASYNC_THROUGH took 48 ms. + 2016-11-09 23:23:16,350 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_CACHE_ASYNC_THROUGH took 23 ms. + Passed the test! + runTest BasicNonByteBuffer CACHE ASYNC_THROUGH + 2016-11-09 23:23:16,362 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_CACHE_ASYNC_THROUGH took 9 ms. + 2016-11-09 23:23:16,370 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_CACHE_ASYNC_THROUGH took 7 ms. + Passed the test! + runTest Basic NO_CACHE MUST_CACHE + 2016-11-09 23:23:16,382 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_NO_CACHE_MUST_CACHE took 12 ms. + 2016-11-09 23:23:16,389 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_NO_CACHE_MUST_CACHE took 7 ms. + Passed the test! + runTest BasicNonByteBuffer NO_CACHE MUST_CACHE + 2016-11-09 23:23:16,401 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_NO_CACHE_MUST_CACHE took 9 ms. + 2016-11-09 23:23:16,412 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_NO_CACHE_MUST_CACHE took 11 ms. + Passed the test! + runTest Basic NO_CACHE CACHE_THROUGH + 2016-11-09 23:23:16,543 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_NO_CACHE_CACHE_THROUGH took 130 ms. + 2016-11-09 23:23:16,552 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_NO_CACHE_CACHE_THROUGH took 9 ms. + Passed the test! + runTest BasicNonByteBuffer NO_CACHE CACHE_THROUGH + 2016-11-09 23:23:16,651 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_NO_CACHE_CACHE_THROUGH took 66 ms. + 2016-11-09 23:23:16,675 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_NO_CACHE_CACHE_THROUGH took 23 ms. + Passed the test! + runTest Basic NO_CACHE THROUGH + 2016-11-09 23:23:16,745 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_NO_CACHE_THROUGH took 69 ms. + 2016-11-09 23:23:16,813 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_NO_CACHE_THROUGH took 68 ms. + Passed the test! + runTest BasicNonByteBuffer NO_CACHE THROUGH + 2016-11-09 23:23:16,887 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_NO_CACHE_THROUGH took 52 ms. + 2016-11-09 23:23:16,961 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_NO_CACHE_THROUGH took 74 ms. + Passed the test! + runTest Basic NO_CACHE ASYNC_THROUGH + 2016-11-09 23:23:16,974 INFO type (BasicOperations.java:writeFile) - writeFile to file /default_tests_files/Basic_NO_CACHE_ASYNC_THROUGH took 12 ms. + 2016-11-09 23:23:16,982 INFO type (BasicOperations.java:readFile) - readFile file /default_tests_files/Basic_NO_CACHE_ASYNC_THROUGH took 8 ms. + Passed the test! + runTest BasicNonByteBuffer NO_CACHE ASYNC_THROUGH + 2016-11-09 23:23:17,005 INFO type (BasicNonByteBufferOperations.java:write) - writeFile to file /default_tests_files/BasicNonByteBuffer_NO_CACHE_ASYNC_THROUGH took 19 ms. + 2016-11-09 23:23:17,012 INFO type (BasicNonByteBufferOperations.java:read) - readFile file /default_tests_files/BasicNonByteBuffer_NO_CACHE_ASYNC_THROUGH took 7 ms. + Passed the test! + hadoop@Master:~/cloud/alluxio-1.3.0$ + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning + 【5】http://www.cnblogs.com/seaspring/p/5726695.html diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24012--spark\350\260\203\347\224\250lluxio-1.3.0\351\205\215\347\275\256.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24012--spark\350\260\203\347\224\250lluxio-1.3.0\351\205\215\347\275\256.md" new file mode 100644 index 0000000..3c3e96c --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24012--spark\350\260\203\347\224\250lluxio-1.3.0\351\205\215\347\275\256.md" @@ -0,0 +1,47 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +# 1.解释 # + +alluxio-0.7.1 启动的时候spark集群直接可以调用，但是alluxio-1.3.0需要自动配置 + +# 2.代码： # + +## 2.1 下载## + http://www.alluxio.org/download 下选择： + + alluxio spark client + 或者： + + http://downloads.alluxio.org/downloads/files/1.3.0/alluxio-1.3.0-spark-client-jar-with-dependencies.jar + +## 2.2 配置## + 请添加如下代码到spark/conf/spark-defaults.conf： + + spark.driver.extraClassPath /home/hadoop/cloud/alluxio-1.3.0/jar/alluxio-1.3.0-spark-client-jar-with-dependencies.jar + spark.executor.extraClassPath /home/hadoop/cloud/alluxio-1.3.0/jar/alluxio-1.3.0-spark-client-jar-with-dependencies.jar + + jar包需要scp到每个节点 +## 2.2 启动## + 重启spark集群 +# 3.结果： # + spark-shell可以操作alluxio集群 + + scala> val rdd1= sc.textFile("hdfs://219.219.220.149:9000//xubo/project/SparkSW/input/D1Line.fasta") + rdd1: org.apache.spark.rdd.RDD[String] = MapPartitionsRDD[1] at textFile at :21 + + scala> rdd1.count + res0: Long = 78295 + + scala> rdd1.saveAsTextFile("alluxio://219.219.220.149:19998/D1Line.fasta") + + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24013--Alluxio-1.3.0\345\220\257\345\212\250\347\226\221\351\227\256.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24013--Alluxio-1.3.0\345\220\257\345\212\250\347\226\221\351\227\256.md" new file mode 100644 index 0000000..af716f2 --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24013--Alluxio-1.3.0\345\220\257\345\212\250\347\226\221\351\227\256.md" @@ -0,0 +1,144 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +# 1.解释 # + +## 1.1 alluxio.env.sh 配置## + + # The directory where a worker stores in-memory data. (Default: /mnt/ramdisk). + # E.g. On linux, /mnt/ramdisk for ramdisk, /dev/shm for tmpFS; on MacOS, /Volumes/ramdisk for ramdisk + # ALLUXIO_RAM_FOLDER + export ALLUXIO_RAM_FOLDER=/home/hadoop/cloud/alluxio-1.3.0/ramdisk + + # Address of the under filesystem address. (Default: ${ALLUXIO_HOME}/underFSStorage) + # E.g. "/my/local/path" to use local fs, "hdfs://localhost:9000/alluxio" to use a local hdfs + # ALLUXIO_UNDERFS_ADDRESS + export ALLUXIO_UNDERFS_ADDRESS=hdfs://Master:9000 + + # How much memory to use per worker. (Default: 1GB) + # E.g. "1000MB", "2GB" + # ALLUXIO_WORKER_MEMORY_SIZE + export ALLUXIO_WORKER_MEMORY_SIZE=6GB + + +启动： + + hadoop@Master:~/cloud/alluxio-1.3.0/conf$ ../bin/alluxio-stop.sh all + Killed 0 processes on Master + Killed 0 processes on Master + Waiting for WORKERS tasks to finish... + All WORKERS tasks finished, please analyze the log at /home/hadoop/cloud/alluxio-1.3.0/logs/task.log. + hadoop@Master:~/cloud/alluxio-1.3.0/conf$ ../bin/alluxio-start.sh all Mount + /home/hadoop/cloud/alluxio-1.3.0/bin + Killed 0 processes on Master + Killed 0 processes on Master + Waiting for WORKERS tasks to finish... + All WORKERS tasks finished, please analyze the log at /home/hadoop/cloud/alluxio-1.3.0/logs/task.log. + Starting master @ Master. Logging to /home/hadoop/cloud/alluxio-1.3.0/logs + Waiting for MASTER tasks to finish... + All MASTER tasks finished, please analyze the log at /home/hadoop/cloud/alluxio-1.3.0/logs/task.log. + + +## 1.2 启动记录 ## + + 2016-11-11 16:35:48,582 + INFO MASTER Connecting to Mcnode1 as hadoop... + 2016-11-11 16:35:48,595 + INFO MASTER Connecting to Mcnode2 as hadoop... + 2016-11-11 16:35:48,604 + INFO MASTER Connecting to Mcnode3 as hadoop... + 2016-11-11 16:35:48,627 + INFO MASTER Connecting to Mcnode4 as hadoop... + 2016-11-11 16:35:48,632 + INFO MASTER Connecting to Mcnode5 as hadoop... + 2016-11-11 16:35:48,649 + INFO MASTER Connecting to Mcnode6 as hadoop... + 2016-11-11 16:35:48,669 + INFO MASTER Connecting to Mcnode7 as hadoop... + /home/hadoop/cloud/alluxio-1.3.0/bin + /home/hadoop/cloud/alluxio-1.3.0/bin + /home/hadoop/cloud/alluxio-1.3.0/bin + Formatting RamFS: /home/hadoop/cloud/alluxio-1.3.0/ramdisk (6gb) + mount: only root can do that + Starting worker @ Mcnode1. Logging to /home/hadoop/cloud/alluxio-1.3.0/logs + /home/hadoop/cloud/alluxio-1.3.0/bin + /home/hadoop/cloud/alluxio-1.3.0/bin + Formatting RamFS: /home/hadoop/cloud/alluxio-1.3.0/ramdisk (6gb) + mount: only root can do that + Starting worker @ Mcnode3. Logging to /home/hadoop/cloud/alluxio-1.3.0/logs + /home/hadoop/cloud/alluxio-1.3.0/bin + Formatting RamFS: /home/hadoop/cloud/alluxio-1.3.0/ramdisk (6gb) + mount: only root can do that + Starting worker @ Mcnode2. Logging to /home/hadoop/cloud/alluxio-1.3.0/logs + /home/hadoop/cloud/alluxio-1.3.0/bin + Formatting RamFS: /home/hadoop/cloud/alluxio-1.3.0/ramdisk (6gb) + mount: only root can do that + Formatting RamFS: /home/hadoop/cloud/alluxio-1.3.0/ramdisk (6gb) + Starting worker @ Mcnode4. Logging to /home/hadoop/cloud/alluxio-1.3.0/logs + mount: only root can do that + Starting worker @ Mcnode5. Logging to /home/hadoop/cloud/alluxio-1.3.0/logs + Formatting RamFS: /home/hadoop/cloud/alluxio-1.3.0/ramdisk (6gb) + Formatting RamFS: /home/hadoop/cloud/alluxio-1.3.0/ramdisk (6gb) + mount: only root can do that + mount: only root can do that + Starting worker @ Mcnode6. Logging to /home/hadoop/cloud/alluxio-1.3.0/logs + Starting worker @ Mcnode7. Logging to /home/hadoop/cloud/alluxio-1.3.0/logs + Connection to mcnode1 closed. + Connection to mcnode3 closed. + Connection to mcnode2 closed. + Connection to mcnode4 closed. + Connection to mcnode5 closed. + Connection to mcnode6 closed. + Connection to mcnode7 closed. + + +# 2.疑问： # +显示mount: only root can do that +有问题吗？ +不确定会不会有影响 + +但是加sudo启动不了 + + hadoop@Master:~/cloud/alluxio-1.3.0/conf$ ../bin/alluxio-start.sh all SudoMount + /home/hadoop/cloud/alluxio-1.3.0/bin + Killed 0 processes on Master + Killed 0 processes on Master + Waiting for WORKERS tasks to finish... + All WORKERS tasks finished, please analyze the log at /home/hadoop/cloud/alluxio-1.3.0/logs/task.log. + Starting master @ Master. Logging to /home/hadoop/cloud/alluxio-1.3.0/logs + Waiting for MASTER tasks to finish... + +task.log： + + 2016-11-11 16:33:40,423 + INFO MASTER Connecting to Mcnode1 as hadoop... + 2016-11-11 16:33:40,435 + INFO MASTER Connecting to Mcnode2 as hadoop... + 2016-11-11 16:33:40,444 + INFO MASTER Connecting to Mcnode3 as hadoop... + 2016-11-11 16:33:40,463 + INFO MASTER Connecting to Mcnode4 as hadoop... + 2016-11-11 16:33:40,480 + INFO MASTER Connecting to Mcnode5 as hadoop... + 2016-11-11 16:33:40,486 + INFO MASTER Connecting to Mcnode6 as hadoop... + 2016-11-11 16:33:40,526 + INFO MASTER Connecting to Mcnode7 as hadoop... + /home/hadoop/cloud/alluxio-1.3.0/bin + /home/hadoop/cloud/alluxio-1.3.0/bin + /home/hadoop/cloud/alluxio-1.3.0/bin + /home/hadoop/cloud/alluxio-1.3.0/bin + /home/hadoop/cloud/alluxio-1.3.0/bin + /home/hadoop/cloud/alluxio-1.3.0/bin + [sudo] password for hadoop: [sudo] password for hadoop: /home/hadoop/cloud/alluxio-1.3.0/bin + [sudo] password for hadoop: [sudo] password for hadoop: [sudo] password for hadoop: [sudo] password for hadoop: [sudo] password for hadoop: 2016-11-11 16:35:40,728 + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24014--alluxio\345\206\205\345\255\230\346\226\207\344\273\266\345\212\240\350\275\275\346\226\271\345\274\217\345\222\214\345\210\206\345\270\203\346\203\205\345\206\265\345\210\206\346\236\220.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24014--alluxio\345\206\205\345\255\230\346\226\207\344\273\266\345\212\240\350\275\275\346\226\271\345\274\217\345\222\214\345\210\206\345\270\203\346\203\205\345\206\265\345\210\206\346\236\220.md" new file mode 100644 index 0000000..6775aae --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24014--alluxio\345\206\205\345\255\230\346\226\207\344\273\266\345\212\240\350\275\275\346\226\271\345\274\217\345\222\214\345\210\206\345\270\203\346\203\205\345\206\265\345\210\206\346\236\220.md" @@ -0,0 +1,397 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +>>alluxio-0.7.1 通过copyFromLocal指令上传文件，发现D1Line.fasta-D6Line.fasta都在一个节点，Mcnode1; +但是alluxio-1.3.1,通过load一个大文件，发现数据会分布在不同节点；但是使用sparkContext的textfile，发现有两份数据,具体请见下面的分析 + +master没有部署work + +#0 以下分析是主要是基于spark-1.5.2和alluxio-1.3.0分析的，默认不支持数据本地化，所以会有跨节点传输 +# 1 load和free分析 ## + +## 1.1 load和free分析和观察结果## + （1）master load,各个节点都会有数据;work节点load，只有work节点有数据，前提是master节点没有work节点 + （2）load数据来源是hdfs，数据大小放到内存也是128M + （3）alluxio数据分布与hdfs不一致，D7Line.fasta:在HDFS没有存在Mcnode5，但是load后会有（master运行） + （4）D7Line.fasta大概2.01G,load很慢，超过一分钟，但是free是秒级的 + （5）从hdfs加载到alluxio，内存中查看到是一个节点一个节点串行操作，没有并行，而且是一个block一个block操作，节点内也没有并行，因为load是单线程 + （6）load运行在Master，通过ganglia发现Mcnode1的节点cache很多，接近饱和14G + （7）数据分布基本均匀（master运行） + （8）串行加载时应该是按照文件的block顺序加载，应为最小的文件应该是最后加载的--可以优化成并行 + （9）free后释放了部分内存， cache还是比较大，但不确定这部分数据是什么 + (10) 在Mcnode2 load，在Mcnode3可以free + (11)load 加载数据一份，没有copy + +## 1.2 代码 ## +## 1.2.1 load## + + hadoop@Master:~/xubo/project/alignment/SparkSW/SparkSW20161026/file$ ~/cloud/alluxio-1.3.0/bin/alluxio fs load /xubo/project/SparkSW/input/D7Line.fasta + /xubo/project/SparkSW/input/D7Line.fasta loaded + +D10Line.fasta很久：14.16 GB大概load超过半个小时 + + hadoop@Master:~/xubo/project/alignment/SparkSW/SparkSW20161026/file$ ~/cloud/alluxio-1.3.0/bin/alluxio fs load /xubo/project/SparkSW/input/D10Line.fasta + /xubo/project/SparkSW/input/D10Line.fasta loaded + +## 1.2.2 free## + + hadoop@Master:~/xubo/project/alignment/SparkSW/SparkSW20161026/file$ ~/cloud/alluxio-1.3.0/bin/alluxio fs free /xubo/project/SparkSW/input/D7Line.fasta + /xubo/project/SparkSW/input/D7Line.fasta was successfully freed from memory. + +## 1.3 结果 ## +## 1.3.1 D7Line.fasta的load和free ## +load D7Line.fasta: +![](http://i.imgur.com/QKqvqI4.png) + +free D7Line.fasta: +![](http://i.imgur.com/HINvWot.png) + +## 1.3.2 D10Line.fasta的load ## +load D10Line.fasta: +![](http://i.imgur.com/NWbdfpq.png) + +load时内存监控 +![](http://i.imgur.com/rEdRzZx.png) + +## 1.3.3 D9Line.fasta的load（没有删除D10） ## + +D9D10 load： +![](http://i.imgur.com/GDekoTn.png) + +两个文件加载后内存情况： +![](http://i.imgur.com/lb4IQcv.png) + +network: +![](http://i.imgur.com/2tORima.png) + +## 1.3.4 free D9 D10 ## +该数据是在运行copyFromLocal之后再free的 + +![](http://i.imgur.com/Dus0yUI.png) + +## 1.3.5 work load## +Mcnode2 ： +![](http://i.imgur.com/aTy6kvu.png) + +Mcnode3： + +![](http://i.imgur.com/Oo0jmyN.png) + + +# 2 copyFromLocal分析 +运行在Mcnode1 +## 2.1 copyFromLocal分析和观察结果## + （1）copyFromLocal是本地缓存，即从本地磁盘加载到本地的内存，没有跨Alluio节点传输缓存（work节点运行） + （2）由于（1），但文件大于单个节点，小雨集群容量时，使用copyFromLocal没法成功（work节点运行），而且会删除原来节点缓存的文件，应该算是bug** + （3）copyFromLocal速度很快，12G左右的数据在2分钟左右，明显快于load + （3）copyFromLocal可以跨节点存储（master节点运行） + +## 2.2 代码： ## + +## 2.2.1 copyFromLocal ## + + hadoop@Mcnode1:~/disk2/xubo/data/SparkSW/uniref$ ~/cloud/alluxio-1.3.0/bin/alluxio fs copyFromLocal uniref100.fasta / + Failed to cache: Not enough space left on worker Mcnode1/219.219.220.180:29998 to store blockId 33638318104. Please consult http://www.alluxio.org/docs/1.3/en/Debugging-Guide.html for common solutions to address this problem. + + +## 2.3 结果： +## 2.3.1 copyFromLocal数据大于alluxio单节点内存 ## + + hadoop@Mcnode1:~/disk2/xubo/data/SparkSW/uniref$ ~/cloud/alluxio-1.3.0/bin/alluxio fs copyFromLocal uniref100.fasta / + Failed to cache: Not enough space left on worker Mcnode1/219.219.220.180:29998 to store blockId 33621540888. Please consult http://www.alluxio.org/docs/1.3/en/Debugging-Guide.html for common solutions to address this problem. + +99%的内存使用： + +![](http://i.imgur.com/eLXwSqQ.png) + +第二次运行： + + hadoop@Mcnode1:~/disk2/xubo/data/SparkSW/uniref$ ~/cloud/alluxio-1.3.0/bin/alluxio fs copyFromLocal uniref100.fasta / + Failed to cache: Not enough space left on worker Mcnode1/219.219.220.180:29998 to store blockId 33638318104. Please consult http://www.alluxio.org/docs/1.3/en/Debugging-Guide.html for common solutions to address this problem. + +![](http://i.imgur.com/DpWAot2.png) + +fail之后变成0 +![](http://i.imgur.com/U80Cudf.png) + +但是内存变化没那么大： +![](http://i.imgur.com/ezvkfOn.png) + + + +# 3 SparkContext.textfile and count分析 # +形式：sc.textFile("alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta") +## 3.1 SparkContext.textfile and count分析和观察结果## + + （1）通过SparkContext.textfile加载alluxio中的文件也会缓存数据到alluxio内存中 + （2）通过分析D5Line.fasta数据集可以发现，在Mcnode6有D5Line.fasta的完整数据集，其他数据分布在Mcnode1、Mcnode5，Mcnode6中，而D5Line.fasta在hdfs主要分布在Mcnode1、2、3、4中，所以数据分布也不是按照HDFS进行读取和存储，需要跨节点传输 + （3）通过分析D7Line.fasta数据集可以发现，2.01G的数据并没有出现在一个节点上，而且多个节点 + （4）根据上一条发现，有两份数据集会被加载 + （5）D5,D7没有存在Mcnode5、6，但是textFile后会有，所以也不是按照HDFS读取到内存 + （6）Mcnode1、2、3、4的内存cache明显大于Mcnode5、6、7，猜测是HDFS主要存储在前四个节点 + （7）Master的内存使用已满，而且use部分相对于其他七个work节点，明显要大，master的use接近10G + +## 3.2 代码： + + val rdd1= sc.textFile("alluxio://219.219.220.149:19998/xubo/project/SparkSW/input/D5Line.fasta") + rdd1.count + +spark-shell： + + scala> val rdd1= sc.textFile("alluxio://219.219.220.149:19998/xubo/project/SparkSW/input/D5Line.fasta") + rdd1: org.apache.spark.rdd.RDD[String] = MapPartitionsRDD[3] at textFile at :21 + + scala> rdd1.count + res1: Long = 1252720 + + scala> val rdd1= sc.textFile("alluxio://219.219.220.149:19998/xubo/project/SparkSW/input/D7Line.fasta") + rdd1: org.apache.spark.rdd.RDD[String] = MapPartitionsRDD[5] at textFile at :21 + + scala> rdd1.count + res2: Long = 5019006 + +## 3.3 结果： + +## 3.3.1 D5文件 +总共大小405.63 MB +Mcnode4和5同步加载，Mcnode4数据在前：开始 +![](http://i.imgur.com/8YTEtMd.png) +Mcnode4和Mcnode5同步加载，Mcnode4数据在前：增加 +![](http://i.imgur.com/tfEyBmY.png) +Mcnode4和Mcnode6同步加载，Mcnode4数据在前：Mcnode5已经加载好，Mcnode6开始 +![](http://i.imgur.com/WtJ2exX.png) +Mcnode4和Mcnode1同步加载，Mcnode4数据在前：Mcnode5、Mcnode6已经加载好，Mcnode1开始 +![](http://i.imgur.com/qY7NaL3.png) +均加载好： +![](http://i.imgur.com/9v4cVMe.png) + +## 3.3.2 D7文件 +![](http://i.imgur.com/CbgbBQa.png) + +加载顺序： +![](http://i.imgur.com/5lAVczv.png) + +![](http://i.imgur.com/xb3o4Jv.png) + +![](http://i.imgur.com/IJxT2NG.png) + +![](http://i.imgur.com/d9Z1dHu.png) + +## 3.3.3 Master和Mcnode1-7等work的内存情况 + +![](http://i.imgur.com/u5PkSZ5.png) + +# 4. textFile hdfs and saveAsTextFile alluxio分析 +形式： + + val rdd1= sc.textFile("hdfs://Master:9000//xubo/project/SparkSW/input/D1Line.fasta") + rdd1.saveAsTextFile("alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta") + +## 4.1 textFile hdfs and saveAsTextFile alluxio分析和观察结果## + （1）在master使用spark submit执行，rdd读入的时候会进行partition，然后存储的时候在hdfs block所在节点，试了7次，而且较大概率会分布在不同的节点 + （2）spark-shell报错 + （3）写到alluxio中的时候是多线程并行的 + + +## 4.2 代码 + + val rdd1= sc.textFile("hdfs://Master:9000//xubo/project/SparkSW/input/D1Line.fasta") + rdd1.saveAsTextFile("alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta") +脚本 + + hadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ vi saveFile.sh + + for i in 1 2 3 4 5 6 7 8 9 + do + echo 'time:'$i + sh test.sh 6G hdfs://Master:9000/xubo/project/SparkSW/input/'D'$i'Line.fasta' 'alluxio://Master:19998/xubo/alluxio/uniref/D'$i'Line.fasta' save + done +## 4.3 结果 +## 4.3.1 存储的是partition +![](http://i.imgur.com/Bmdn9MG.png) + +![](http://i.imgur.com/pwRCZtv.png) + +##4.3.2 多线程并行 +![](http://i.imgur.com/Fa29iTl.png) +![](http://i.imgur.com/irMpk6v.png) +![](http://i.imgur.com/PLcMgHf.png) +![](http://i.imgur.com/EuapQfr.png) +![](http://i.imgur.com/OfNW8tW.png) +![](http://i.imgur.com/L0MlpVC.png) +最后存储的数据分布很均匀 + +## 4.3.3 记录 + + hadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ sh test.sh 6G hdfs://Master:9000/xubo/project/SparkSW/input/'D6Line.fasta' alluxio://Master:19998/xubo/alluxio/uniref/'D6Line.fasta' + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta countTime:39.499s partitons.length:8 + + +##记录 + +##第一次：alluxio-1.3.0 + + hadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ tail -f saveFile201611122308.txt + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta/SaveFile20161112230807255 countTime:6.142s partitons.length:2 + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta/SaveFile20161112230818840 countTime:5.413s partitons.length:2 + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta/SaveFile20161112230829721 countTime:10.416s partitons.length:2 + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta/SaveFile20161112230845420 countTime:7.973s partitons.length:2 + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta/SaveFile20161112230858790 countTime:16.227s partitons.length:4 + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta/SaveFile20161112230920089 countTime:16.411s partitons.length:8 + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta/SaveFile20161112230941702 countTime:33.634s partitons.length:16 + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta/SaveFile20161112231020474 countTime:32.927s partitons.length:34 + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta/SaveFile20161112231058702 countTime:136.582s partitons.length:63 + +##第二次：Alluxio-1.3.0 + + hadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ tail -f saveFile201611122328.txt + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta countTime:5.697s partitons.length:2 + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta countTime:7.714s partitons.length:2 + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta countTime:7.813s partitons.length:4 + 16/11/12 23:29:43 WARN QueuedThreadPool: 1 threads could not be stopped + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta countTime:18.261s partitons.length:8 + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta countTime:77.04s partitons.length:16 + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta countTime:77.75s partitons.length:34 + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta countTime:92.287s partitons.length:63 + +D9： + + hadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ time sh test.sh 6G hdfs://Master:9000/xubo/project/SparkSW/input/'D9Line.fasta' alluxio://Master:19998/xubo/alluxio/uniref/'D9Line.fasta' + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta out:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta countTime:156.214s partitons.length:63 + + real 2m41.526s + user 0m14.578s + sys 0m0.941s + +##第三次：Alluxio-0.7.1 + + hadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ ./saveFile-0.7.1.sh + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta countTime:11.446s partitons.length:2 + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta countTime:11.666s partitons.length:2 + 16/11/13 19:22:22 WARN QueuedThreadPool: 2 threads could not be stopped + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta countTime:19.249s partitons.length:2 + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta countTime:23.373s partitons.length:2 + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta countTime:42.418s partitons.length:4 + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta countTime:89.122s partitons.length:8 + +第七个文件D7卡住了20多分钟没完成，重新跑： + + hadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ ./saveFile-0.7.1.sh + time:7 + ^C^[[Ahadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ vi saveFile-0.7.1.sh + hadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ ./saveFile-0.7.1.sh + file:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta countTime:92.267s partitons.length:16 + file:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta countTime:110.32s partitons.length:34 + file:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta countTime:373.661s partitons.length:63 + +##第四次：Alluxio-0.7.1 + + hadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ tail -f saveFile-0.7.1time201611132245.txt + file:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta countTime:6.339s partitons.length:2 + file:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta countTime:11.167s partitons.length:2 + file:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta countTime:10.849s partitons.length:2 + 16/11/13 22:45:49 WARN QueuedThreadPool: 2 threads could not be stopped + file:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta countTime:15.198s partitons.length:2 + file:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta countTime:28.717s partitons.length:4 + file:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta countTime:40.355s partitons.length:8 + file:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta countTime:60.063s partitons.length:16 + file:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta countTime:147.046s partitons.length:34 + file:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta countTime:320.456s partitons.length:63 + +## 第五次：Alluxio-0.7.1 + + hadoop@Master:~/disk2/xubo/project/alluxio/saveFile$ tail -f saveFile-0.7.1time201611140810.txt + file:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta Time:7.743s partitons.length:2 + file:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta Time:7.977s partitons.length:2 + file:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta Time:10.45s partitons.length:2 + 16/11/14 08:11:32 WARN QueuedThreadPool: 3 threads could not be stopped + file:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta Time:29.436s partitons.length:2 + file:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta Time:18.999s partitons.length:4 + file:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta Time:64.348s partitons.length:8 + file:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta Time:102.527s partitons.length:16 + file:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta Time:149.429s partitons.length:34 + file:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta Time:338.072s partitons.length:63 + file:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta out:tachyon://Master:19998/xubo/alluxio/uniref/D10Line.fasta Time:619.74s partitons.length:114 + + +# 5 总结# +## 5.1说明 + + 没去看源码，之后需要去学习，是观察所得 +## 5.1 区别 + +## 5.1.1 alluxio中load和textFile的区别： + +- （1）load单个一个一个加载，textFile多个block并行加载 + + +- （2）load和textFile加载后数据分布不一致 +![1](http://i.imgur.com/Y8N5rlo.png) + +- （3）load加载后一份，textFile加载后两份数据 +- （4）load串行，textFile两个并行加载 + +## 5.1.2 多次使用textfile + + 多次使用textfile，最后数据会比原来文件大，而且每个节点数据会差不多，推测是每次textfile会重新分布数据，而不是从alluxio读取，没有截图，之前跑alluxio-0.7.1留下的印象 +free之后同一个文件将在各个节点的cache都会free + +## 5.2 问题 +当在Mcnode1运行时，数据都存在Mcnode1，但在Master运行时，数据存储在work各个节点 + +原因分析：master没有alluxio的work节点，不存储数据，只管理元数据，所以需要分配到各个节点存储 + +## textfile（alluxiofile）与 textfile（hdfs file）在saveAsTextFile（alluxio file）的区别 + textfile（alluxiofile）会缓存block，进行count + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24015--alluxio\346\200\247\350\203\275\345\210\206\346\236\220\345\222\214\345\212\240\351\200\237\346\226\271\345\274\217.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24015--alluxio\346\200\247\350\203\275\345\210\206\346\236\220\345\222\214\345\212\240\351\200\237\346\226\271\345\274\217.md" new file mode 100644 index 0000000..dbb420c --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24015--alluxio\346\200\247\350\203\275\345\210\206\346\236\220\345\222\214\345\212\240\351\200\237\346\226\271\345\274\217.md" @@ -0,0 +1,126 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 +#0 以下分析是主要是基于spark-1.5.2和alluxio-1.3.0分析的，默认不支持数据本地化，所以会有跨节点传输 +# 1.解释 # + +## 1.1 使用count进行简单数据分析## + count比较常见，分别对D1Line.fasta等数据集进行分析， + +## 1.2 ## + + +# 2.代码： # + +## 2.1 run.sh 脚本## + + for j in 1 2 3 4 5 6 + do + sh test.sh 10G hdfs://Master:9000/xubo/project/SparkSW/input/'D'$j'Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + sh test.sh 10G alluxio://Master:19998/xubo/project/SparkSW/input/'D'$j'Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + #sh test.sh 10G alluxio://Master:19998/xubo/project/SparkSW/input/'D'$j'Line.fasta' alluxio://Master:19998/xubo/project/alluxio/output/TimeCount + done + + +## 2.2 ## + + +# 3.结果： # + +数据越大加速越明显 + +##第一次 + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run.sh + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fastacount:78295countTime:3.791s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fastacount:78295countTime:4.726s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fastacount:156590countTime:4.001s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fastacount:156590countTime:5.244s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fastacount:313180countTime:4.329s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fastacount:313180countTime:26.376s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fastacount:626360countTime:7.918s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fastacount:626360countTime:6.199s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fastacount:1252720countTime:15.128s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fastacount:1252720countTime:9.661s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fastacount:2505440countTime:15.288s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fastacount:2505440countTime:8.724s + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ rm SparkSW.jar + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ rz + rz waiting to receive. +##第二次 + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run.sh + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fastacount:78295countTime:4.092s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fastacount:78295countTime:4.819s + Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000003fff80000, 11453595648, 0) failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (malloc) failed to allocate 11453595648 bytes for committing reserved memory. + # An error report file with more information is saved as: + # /home/hadoop/disk2/xubo/project/alluxio/timeCount/hs_err_pid21271.log + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fastacount:156590countTime:12.183s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fastacount:313180countTime:10.532s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fastacount:313180countTime:4.685s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fastacount:626360countTime:8.789s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fastacount:626360countTime:26.1s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fastacount:1252720countTime:5.16s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fastacount:1252720countTime:6.694s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fastacount:2505440countTime:19.338s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fastacount:2505440countTime:29.997s +##第三次 + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run.sh + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fastacount:78295countTime:4.335s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fastacount:78295countTime:5.461s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fastacount:156590countTime:4.352s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fastacount:156590countTime:6.373s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fastacount:313180countTime:10.727s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fastacount:313180countTime:6.621s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fastacount:626360countTime:17.001s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fastacount:626360countTime:8.243s + Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000003fff80000, 11453595648, 0) failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (malloc) failed to allocate 11453595648 bytes for committing reserved memory. + # An error report file with more information is saved as: + # /home/hadoop/disk2/xubo/project/alluxio/timeCount/hs_err_pid25150.log + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fastacount:1252720countTime:21.476s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fastacount:2505440countTime:9.553s + Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000003fff80000, 11453595648, 0) failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (malloc) failed to allocate 11453595648 bytes for committing reserved memory. + # An error report file with more information is saved as: + # /home/hadoop/disk2/xubo/project/alluxio/timeCount/hs_err_pid25626.log + +## 第四次 + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run.sh + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:5.055s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:4.044s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.961s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.892s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:5.071s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.461s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:9.366s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:15.696s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:17.386s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:6.127s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:41.361s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:7.352s + +##问题： + Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000003fff80000, 11453595648, 0) failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (malloc) failed to allocate 11453595648 bytes for committing reserved memory. + # An error report file with more information is saved as: + # /home/hadoop/disk2/xubo/project/alluxio/timeCount/hs_err_pid25626.log + +节点间scp传输速度为10M： + +![](http://i.imgur.com/MNpJevk.png) + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24016--\345\260\206alluxio\346\226\207\344\273\266\347\232\204block\345\244\247\345\260\217\344\277\256\346\224\271.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24016--\345\260\206alluxio\346\226\207\344\273\266\347\232\204block\345\244\247\345\260\217\344\277\256\346\224\271.md" new file mode 100644 index 0000000..c1e2796 --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24016--\345\260\206alluxio\346\226\207\344\273\266\347\232\204block\345\244\247\345\260\217\344\277\256\346\224\271.md" @@ -0,0 +1,67 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +# 1.alluxio-1.3.0 # + +## 1.0 默认大小## +alluxio-1.3.0的block默认大小为512M，对于我每个节点只有10几个G的小内存来说，而且还需要进行spark运算，用于alluxio的大小不能太大，所以512M粒度比较大。需要修改 + +## 1.1 操作## + 将配置文件中增加为： + alluxio.user.block.size.bytes.default=128MB + + +# 1.2.代码： # + +## 1.2.1 copy## + + hadoop@Master:~/cloud/alluxio-1.3.0/conf$ cp alluxio-site.properties.template alluxio-site.properties + +## 1.2.2 修改## + 在alluxio-site.properties最后增加一行，大小根据情况设置，我的修改为128M，与HDFS一致 + + # User properties + # alluxio.user.file.readtype.default=CACHE_PROMOTE + # alluxio.user.file.writetype.default=MUST_CACHE + alluxio.user.block.size.bytes.default=128MB + +scp到其他节点，重启生效 + +# 1.3.结果： # + +##19999查看生效： + ![](http://i.imgur.com/Q4Vm2fV.png) + +##是128M的倍数，不是512的倍数： +![](http://i.imgur.com/K1rE4TW.png) + +# 2. alluxio-0.7.1 + +##2.1 修改 + + hadoop@Master:~/cloud/tachyon-0.7.1-hadoop2.6-bin/conf$ cp ../common/src/main/resources/tachyon-default.properties tachyon-site.properties + hadoop@Master:~/cloud/tachyon-0.7.1-hadoop2.6-bin/conf$ ls + core-site.xml.template log4j.properties metrics.properties.template slaves tachyon-env.sh tachyon-env.sh.swift tachyon-env.sh.template tachyon-glusterfs-env.sh.template tachyon-site.properties workers + hadoop@Master:~/cloud/tachyon-0.7.1-hadoop2.6-bin/conf$ vi tachyon-site.properties + +修改内容： + + # User properties + tachyon.user.failed.space.request.limits=3 + tachyon.user.heartbeat.interval.ms=1000 + tachyon.user.file.writetype.default=CACHE_THROUGH + tachyon.user.default.block.size.byte=128MB +分发到各个节点，重启alluxio + +## 2.2 结果 + +![](http://i.imgur.com/c79ZMs3.png) + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24017--\345\244\232\346\254\241\350\277\220\350\241\214Space Usage\345\242\236\345\212\240.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24017--\345\244\232\346\254\241\350\277\220\350\241\214Space Usage\345\242\236\345\212\240.md" new file mode 100644 index 0000000..8cda76f --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24017--\345\244\232\346\254\241\350\277\220\350\241\214Space Usage\345\242\236\345\212\240.md" @@ -0,0 +1,165 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +#0 以下分析是主要是基于spark-1.5.2和alluxio-1.3.0分析的，默认不支持数据本地化，所以会有跨节点传输 + +# 1.解释 # + +## 1.1 问题## + 当将文件通过fs copyFromLocal传到alluxio时，开始8台大概是16G左右，但是多次运行 Space Usage会增加，即使每增加新文件。 + 不知道alluxio里面的实现机制是什么？ + +# 2.代码： # + +## 2.1 run.sh## + + #~/cloud/alluxio-1.3.0/bin/alluxio fs mkdir -p /xubo/project/SparkSW/input/ + for j in 1 2 3 4 5 6 7 8 9 + do + sh test.sh 10G hdfs://Master:9000/xubo/project/SparkSW/input/'D'$j'Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + sh test.sh 10G alluxio://Master:19998/xubo/alluxio/uniref/'D'$j'Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + #sh test.sh 10G alluxio://Master:19998/xubo/project/SparkSW/input/'D'$j'Line.fasta' alluxio://Master:19998/xubo/project/alluxio/output/TimeCount + done + ~ + +## 2.2 test.sh## + +test.sh文件是一个读取并进行count的脚本 + +# 3.结果： # +## 3.1 运行前 +copyFromLocal之后：总共16G左右 +![](http://i.imgur.com/Stqg6ow.png) +##第一次运行 +第一次运行count后，明显增加，总共32G左右 +![](http://i.imgur.com/fHNRu6h.png) + +运行记录 + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run.sh + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:9.706s + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:7.656s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:6.719s + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:15.477s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:5.013s + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:16.75s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:7.942s + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:17.642s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:26.711s + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:31.092s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:15.409s + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:73.364s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:35.338s + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:64.195s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:87.111s + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:76.951s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:132.447s + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:190.699s + +ganglia中明显可以看到network开销： + +![](http://i.imgur.com/Hfi0wLv.png) + +##第二次运行 +![](http://i.imgur.com/deHFlpJ.png) +运行记录 + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run.sh + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:9.706s + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:7.656s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:6.719s + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:15.477s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:5.013s + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:16.75s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:7.942s + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:17.642s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:26.711s + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:31.092s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:15.409s + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:73.364s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:35.338s + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:64.195s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:87.111s + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:76.951s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:132.447s + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:190.699s + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run.sh + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:5.355s + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:7.633s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.989s + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.384s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.478s + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.671s + Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000003fff80000, 11453595648, 0) failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (malloc) failed to allocate 11453595648 bytes for committing reserved memory. + # An error report file with more information is saved as: + # /home/hadoop/disk2/xubo/project/alluxio/timeCount/hs_err_pid13258.log + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:8.492s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:6.681s + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:11.47s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:8.213s + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:21.302s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:45.31s + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:30.39s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:63.646s + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:69.009s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:96.257s + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:129.645s + + +第三次： + +![](http://i.imgur.com/Y8LZKs9.png) + +![](http://i.imgur.com/BB8I8U4.png) + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run.sh + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:4.205s + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.907s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:6.637s + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.212s + Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000003fff80000, 11453595648, 0) failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (malloc) failed to allocate 11453595648 bytes for committing reserved memory. + # An error report file with more information is saved as: + # /home/hadoop/disk2/xubo/project/alluxio/timeCount/hs_err_pid17054.log + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:6.732s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:17.264s + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:9.099s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:26.775s + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:30.943s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:18.174s + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:17.044s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:45.236s + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:36.763s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:61.551s + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:73.987s + Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000003fff80000, 11453595648, 0) failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (malloc) failed to allocate 11453595648 bytes for committing reserved memory. + # An error report file with more information is saved as: + # /home/hadoop/disk2/xubo/project/alluxio/timeCount/hs_err_pid19730.log + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:139.039s + +## 第N次后： +###总共： +![](http://i.imgur.com/Sev5mEY.png) +###Mcnode1: +![](http://i.imgur.com/VFcl9Tr.png) +###Mcnode2: +![](http://i.imgur.com/Pycc7Ib.png) +###Mcnode3: +![](http://i.imgur.com/h5ADuci.png) + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24018--alluxio work 3000\345\207\272\347\216\260\351\227\256\351\242\230.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24018--alluxio work 3000\345\207\272\347\216\260\351\227\256\351\242\230.md" new file mode 100644 index 0000000..e57b9ce --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24018--alluxio work 3000\345\207\272\347\216\260\351\227\256\351\242\230.md" @@ -0,0 +1,74 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +# 1.解释 # + +待解决 + +## 1.1 问题## +访问http://mcnode6:30000/home时出现的 + + Problem accessing /home. Reason: + + Server Error + Caused by: + + org.apache.jasper.JasperException: PWC6345: There is an error in invoking javac. A full JDK (not just JRE) is required + at org.apache.jasper.compiler.DefaultErrorHandler.jspError(DefaultErrorHandler.java:92) + at org.apache.jasper.compiler.ErrorDispatcher.dispatch(ErrorDispatcher.java:378) + at org.apache.jasper.compiler.ErrorDispatcher.jspError(ErrorDispatcher.java:119) + at org.apache.jasper.compiler.Jsr199JavaCompiler.compile(Jsr199JavaCompiler.java:208) + at org.apache.jasper.compiler.Compiler.generateClass(Compiler.java:384) + at org.apache.jasper.compiler.Compiler.compile(Compiler.java:453) + at org.apache.jasper.JspCompilationContext.compile(JspCompilationContext.java:625) + at org.apache.jasper.servlet.JspServletWrapper.service(JspServletWrapper.java:375) + at org.apache.jasper.servlet.JspServlet.serviceJspFile(JspServlet.java:473) + at org.apache.jasper.servlet.JspServlet.service(JspServlet.java:377) + at org.eclipse.jetty.jsp.JettyJspServlet.service(JettyJspServlet.java:103) + at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) + at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:812) + at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:587) + at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) + at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:595) + at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:223) + at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127) + at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515) + at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) + at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061) + at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) + at org.eclipse.jetty.server.Dispatcher.forward(Dispatcher.java:191) + at org.eclipse.jetty.server.Dispatcher.forward(Dispatcher.java:72) + at alluxio.web.WebInterfaceWorkerGeneralServlet.doGet(WebInterfaceWorkerGeneralServlet.java:200) + at javax.servlet.http.HttpServlet.service(HttpServlet.java:687) + at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) + at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:812) + at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:587) + at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) + at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:577) + at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:223) + at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127) + at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515) + at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) + at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061) + at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) + at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) + at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) + at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) + at org.eclipse.jetty.server.Server.handle(Server.java:499) + at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:311) + at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257) + at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:544) + at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) + at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) + at java.lang.Thread.run(Thread.java:745) + Powered by Jetty:// + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24019--alluxio\346\200\247\350\203\275\346\217\220\345\215\207\345\210\206\346\236\220\344\270\216\345\256\236\351\252\2141.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24019--alluxio\346\200\247\350\203\275\346\217\220\345\215\207\345\210\206\346\236\220\344\270\216\345\256\236\351\252\2141.md" new file mode 100644 index 0000000..fcd286e --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24019--alluxio\346\200\247\350\203\275\346\217\220\345\215\207\345\210\206\346\236\220\344\270\216\345\256\236\351\252\2141.md" @@ -0,0 +1,498 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +#0 以下分析是主要是基于spark-1.5.2和alluxio-1.3.0分析的，默认不支持数据本地化，所以会有跨节点传输 + +# 1.解释 # + +## 1.1 分析## +alluxio会把数据缓存在work的本地，虽然使用spark从alluxio读取hdfs会使得文件分布与hdfs不一致，但还是可以达到加速的目的，特别是hdfs与spark不在一个集群，alluxio可以达到很好的缓存作用；baidu 30倍提升的背景和方法类似 + +## 1.2 实验## + 分别从hdfs读取文件和alluxio读取文件，进行count计算，文件大小从32M到7.8G + +# 2.代码： # +脚本 + + #~/cloud/alluxio-1.3.0/bin/alluxio fs mkdir -p /xubo/project/SparkSW/input/ + for j in 1 2 3 4 5 6 7 8 9 + do + for i in 1 2 3 4 5 6 7 8 9 10 + do + echo 'time:'$i + sh test.sh 14G hdfs://Master:9000/xubo/project/SparkSW/input/'D'$j'Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + done + done + + for j in 1 2 3 4 5 6 7 8 9 + do + for i in 1 2 3 4 5 6 7 8 9 10 + do + echo 'time:'$i + sh test.sh 6G alluxio://Master:19998/xubo/project/SparkSW/input/'D'$j'Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + done + + + done + + + +# 3.结果： # + +## 3.1 从hdfs读取文件进行count + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f run1_201611121727.txt + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:4.1s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:5.14s + time:3 + Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000003fff80000, 11453595648, 0) failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (malloc) failed to allocate 11453595648 bytes for committing reserved memory. + # An error report file with more information is saved as: + # /home/hadoop/disk2/xubo/project/alluxio/timeCount/hs_err_pid26806.log + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:5.023s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.984s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:4.037s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.847s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.966s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:5.015s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:5.425s + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.012s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:6.533s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:6.44s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:6.528s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.048s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.145s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.055s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.031s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:6.665s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.065s + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:5.41s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.779s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:9.414s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.267s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:9.446s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.53s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.306s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:9.39s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:9.307s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.463s + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:16.953s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:8.625s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:8.015s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:6.104s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:12.854s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:6.274s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:17.06s + time:8 + Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000003fff80000, 11453595648, 0) failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (malloc) failed to allocate 11453595648 bytes for committing reserved memory. + # An error report file with more information is saved as: + # /home/hadoop/disk2/xubo/project/alluxio/timeCount/hs_err_pid1475.log + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:16.995s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:7.915s + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:15.202s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:7.891s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:6.638s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:5.049s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:5.176s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:5.181s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.888s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:15.406s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:6.229s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:6.14s + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:26.699s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:15.33s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:26.675s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:15.361s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:26.745s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:16.951s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:17.046s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:26.805s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:27.031s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:15.89s + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:55.006s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:43.435s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.068s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:35.5s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.111s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.015s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:44.477s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:41.526s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.072s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.481s + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:84.314s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:27.691s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:44.684s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:96.937s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:61.365s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:38.072s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:43.018s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:85.482s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:27.828s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:52.804s + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:110.129s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:166.146s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:97.836s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:51.683s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:107.544s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:96.966s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:100.279s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:61.431s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:38.482s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:63.901s + +## 3.2 从alluxio读取文件进行count + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f run1_201611121727_2.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:7.512s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:5.323s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.979s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:5.322s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:4.447s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:4.042s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.931s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:4.977s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.951s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:4.086s + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:5.199s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:5.605s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.147s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.868s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:5.324s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:10.215s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.079s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.097s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.082s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.134s + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:16.241s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:16.245s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:8.147s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:5.934s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.395s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.12s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.392s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.268s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.156s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.242s + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:16.096s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:25.805s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:13.501s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:4.419s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:13.76s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:7.432s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:7.975s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:4.239s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:4.489s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:4.519s + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:28.792s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:12.711s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:28.76s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:18.752s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:10.026s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.53s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.789s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.793s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:5.761s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.6s + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:63.302s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:11.117s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:9.801s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:9.064s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:28.815s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:7.811s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:9.686s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.816s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:6.598s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.728s + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:93.021s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:39.388s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:50.501s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:45.915s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:44.043s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.48s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:32.945s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.906s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.884s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:29.972s + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:145.837s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:57.805s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:66.785s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:53.351s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:48.176s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:31.527s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:49.615s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:44.39s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.008s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:31.557s + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:228.964s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:200.829s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:167.487s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:87.298s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:68.357s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:71.835s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:61.462s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:44.198s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:40.567s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.631s +## 3.3 从alluxio读取文件进行count后alluxio文件情况 +master： +![](http://i.imgur.com/AYOiuvl.png) +Mcnode7 +![](http://i.imgur.com/VsHYPar.png) +Mcnode5 +![](http://i.imgur.com/1aZkVTz.png) +Mcnode4 +![](http://i.imgur.com/WTku1Yl.png) +Mcnode3: +![](http://i.imgur.com/7ub8mTf.png) +Mcnode2: +![](http://i.imgur.com/xqmLCjF.png) +Mcnode1: +![](http://i.imgur.com/Pm5e6zv.png) +network: +![](http://i.imgur.com/X1VL4Vq.png) + +## 3.5 D9提升进一步实验 +由于alluxio中D9的数据仍然没有100%，所以再运行了20遍： + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f run1D9time201611122031.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:31.358s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:36.08s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.398s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:31.578s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.958s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.9s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.807s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.638s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.583s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.812s + ^C + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f run1D9time201611122038.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.169s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.898s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.523s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.764s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.492s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.109s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.657s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.765s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.927s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:33.635s + +运行D9 30次后： + +![](http://i.imgur.com/Shv4tN6.png) +![](http://i.imgur.com/aStad0e.png) +![](http://i.imgur.com/4pSfQSK.png) +![](http://i.imgur.com/Fwuhrk9.png) +![](http://i.imgur.com/q5vqsVy.png) +![](http://i.imgur.com/Q3DOcno.png) + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2402---Spark\344\273\216tachyon\344\270\255\350\257\273\345\217\226\346\226\207\344\273\266.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2402---Spark\344\273\216tachyon\344\270\255\350\257\273\345\217\226\346\226\207\344\273\266.md" new file mode 100644 index 0000000..e84e3a4 --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2402---Spark\344\273\216tachyon\344\270\255\350\257\273\345\217\226\346\226\207\344\273\266.md" @@ -0,0 +1,35 @@ + + +1.启动spark-shell + +2.上传文件到tachyon： + + xubo@xubo:~/cloud/test/tachyon$ ../../tachyon-0.7.1/bin/tachyon tfs copyFromLocal 1.txt / + Copied 1.txt to / + xubo@xubo:~/cloud/test/tachyon$ ../../tachyon-0.7.1/bin/tachyon tfs cat /1.txt + hello tachyon + 1 + 2 + 3 + +3.读取： + + + scala> val text1= sc.textFile("tachyon://localhost:19998/1.txt") + text1: org.apache.spark.rdd.RDD[String] = MapPartitionsRDD[1] at textFile at :21 + + scala> text + text1 text + + scala> text1.foreach + foreach foreachPartition foreachWith + + scala> text1.foreach + def foreach(f: T => Unit): Unit + + scala> text1.foreach(println) + 1 + hello tachyon + 2 + 3 + diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24020--\347\226\221\351\227\256\357\274\232\344\270\272\344\273\200\344\271\210D7-D9\345\210\260\346\234\200\345\220\216\347\232\204count\346\227\266\351\227\264\351\203\275\346\230\25730s\345\267\246\345\217\263\357\274\237.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24020--\347\226\221\351\227\256\357\274\232\344\270\272\344\273\200\344\271\210D7-D9\345\210\260\346\234\200\345\220\216\347\232\204count\346\227\266\351\227\264\351\203\275\346\230\25730s\345\267\246\345\217\263\357\274\237.md" new file mode 100644 index 0000000..8c58306 --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24020--\347\226\221\351\227\256\357\274\232\344\270\272\344\273\200\344\271\210D7-D9\345\210\260\346\234\200\345\220\216\347\232\204count\346\227\266\351\227\264\351\203\275\346\230\25730s\345\267\246\345\217\263\357\274\237.md" @@ -0,0 +1,419 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 +#0 以下分析是主要是基于spark-1.5.2和alluxio-1.3.0分析的，默认不支持数据本地化，所以会有跨节点传输 +# 1.解释 # + +## 1.1 数据## +D7:2.01 GB +D8:4.16 GB +D9:7.79 GB + +## 1.2 说明 + 在D7-9上运行count，spark分布式执行，最后都是稳定在30s左右，不知道为什么？三个数据集大小不一样，但是最后收敛一样？ + + 猜想： + block设置的大小为128M，但是 + +# 2.运行记录 # + +## 2.1 运行记录 D7-D9 + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:93.021s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:39.388s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:50.501s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:45.915s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:44.043s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.48s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:32.945s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.906s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.884s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:29.972s + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:145.837s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:57.805s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:66.785s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:53.351s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:48.176s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:31.527s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:49.615s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:44.39s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.008s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:31.557s + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:228.964s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:200.829s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:167.487s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:87.298s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:68.357s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:71.835s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:61.462s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:44.198s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:40.567s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.631s + ^C + +## 3.2 多次运行D9 + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f run1D9time201611122031.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:31.358s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:36.08s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.398s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:31.578s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.958s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.9s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.807s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.638s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.583s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.812s + ^C + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f run1D9time201611122038.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.169s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.898s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.523s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.764s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.492s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.109s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.657s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.765s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.927s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:33.635s + ^[[B^[[B^C + +## 2.3 多次运行D8 + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f rund8time201611122051.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:127.607s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:44.799s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:85.756s + time:4 + Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x00000003fff80000, 11453595648, 0) failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (malloc) failed to allocate 11453595648 bytes for committing reserved memory. + # An error report file with more information is saved as: + # /home/hadoop/disk2/xubo/project/alluxio/timeCount/hs_err_pid12705.log + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:46.596s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:46.065s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:53.594s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.673s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.201s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:41.336s + ^C + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f rund8time201611122104.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:31.531s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:29.741s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:33.394s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:29.74s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:34.75s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:29.809s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.483s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.412s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:29.812s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.721s + ^C + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f rund8time201611122110.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.438s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.56s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.617s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.418s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.432s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.504s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.574s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.526s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.793s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.535s + ^[[A^C + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f rund8time201611122118.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.539s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.512s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.536s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.771s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.383s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.626s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.361s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.663s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.693s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:30.894s + ^C + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f rund7time201611122125.txt + +## 3.4 多次运行D7 + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:118.782s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:50.964s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:45.109s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:44.266s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:31.002s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.822s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.46s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.673s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.686s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.062s + time:11 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:29.93s + time:12 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:31.033s + time:13 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.61s + time:14 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.656s + time:15 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:31.083s + time:16 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.639s + time:17 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:29.984s + time:18 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.637s + time:19 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.219s + time:20 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.777s + time:21 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.849s + time:22 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.763s + time:23 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.83s + time:24 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.728s + time:25 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.615s + time:26 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.652s + time:27 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.678s + time:28 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.705s + time:29 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:31.004s + time:30 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.04s + +## 3.5 改变运行内存：10G excutor + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f rund7time201611122159mem10g.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.642s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:29.375s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.765s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.63s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.639s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.66s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.659s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.741s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:29.925s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.55s + time:11 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.787s + time:12 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.585s + time:13 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.626s + time:14 + ^C + +## 3.6 改变运行内存：10G excutor， 且释放掉alluxio其他文件 + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f rund7time201611122211mem10g.txt + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.76s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.588s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.821s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.517s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.626s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.831s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.68s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.684s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.73s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.741s + time:11 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.819s + time:12 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:30.812s + time:13 + [Stage 0:=================================================> (14 + 2) / 16] + +## 3.7 使用saveAsTextFile： + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f run1d9time201611131738.txt + time:1 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:176.505s + time:2 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:160.029s + time:3 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:77.369s + time:4 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:63.931s + time:5 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:51.872s + time:6 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:40.886s + time:7 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:36.764s + time:8 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.678s + time:9 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:36.897s + time:10 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:52.303s + time:11 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.534s + time:12 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.507s + time:13 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.622s + time:14 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.395s + time:15 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.366s + time:16 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:32.348s + time:17 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:44.949s + time:18 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.473s + time:19 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.086s + time:20 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.495s + time:21 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.259s + time:22 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.362s + time:23 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.78s + time:24 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.974s + time:25 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.396s + time:26 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.641s + time:27 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:29.052s + time:28 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.5s + time:29 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.573s + time:30 + input:alluxio://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:28.224s + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24021--\347\226\221\351\227\256\357\274\232\344\270\272\344\273\200\344\271\210master\347\234\213\344\270\215\345\210\260D9,\350\200\214work\350\212\202\347\202\271\346\234\211D9\351\203\250\345\210\206\346\225\260\346\215\256\357\274\237.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24021--\347\226\221\351\227\256\357\274\232\344\270\272\344\273\200\344\271\210master\347\234\213\344\270\215\345\210\260D9,\350\200\214work\350\212\202\347\202\271\346\234\211D9\351\203\250\345\210\206\346\225\260\346\215\256\357\274\237.md" new file mode 100644 index 0000000..028a73b --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24021--\347\226\221\351\227\256\357\274\232\344\270\272\344\273\200\344\271\210master\347\234\213\344\270\215\345\210\260D9,\350\200\214work\350\212\202\347\202\271\346\234\211D9\351\203\250\345\210\206\346\225\260\346\215\256\357\274\237.md" @@ -0,0 +1,37 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +# 1.解释 # + +## 1.1 疑问## +为什么master看不到D9,而work节点有D9部分数据？ + +master也显示内存不止D7,因为D7大概2G + +可以free掉D9 + +# 2.截图 # + +## 2.1 master +![](http://i.imgur.com/7lgfEo4.png) + +![](http://i.imgur.com/fgUeUk8.png) + +## 2.2 works +![](http://i.imgur.com/YMkfaE5.png) + +![](http://i.imgur.com/QoKJ1Px.png) + +![](http://i.imgur.com/FACVCgx.png) + + + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24022--saveAsTextFile alluxio\345\220\216count \357\274\210\346\234\211bug\357\274\211.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24022--saveAsTextFile alluxio\345\220\216count \357\274\210\346\234\211bug\357\274\211.md" new file mode 100644 index 0000000..77a1ec2 --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24022--saveAsTextFile alluxio\345\220\216count \357\274\210\346\234\211bug\357\274\211.md" @@ -0,0 +1,826 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 +#0 以下分析是主要是基于spark-1.5.2和alluxio-1.3.0分析的，默认不支持数据本地化，所以会有跨节点传输 +# 1.解释 # + +先加载到mem，然后count + +出现问题，D9是block丢失，主要是因为alluxio内存使用问题，每个work会将数据缓存，而不是分布式缓存，导致后面的不经常用的数据丢失 + +# 2.代码： # + + for j in 1 2 3 4 5 6 7 8 9 + do + for i in 1 2 3 4 5 6 7 8 9 10 + do + echo 'time:'$i + sh test.sh 6G alluxio://Master:19998/xubo/alluxio/uniref/'D'$j'Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + done + + + done + + +# 3.结果： # + +##alluxio-1.3.0 + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f run1time201611122335.txt \ + > ; + time:1 + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.455s + time:2 + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.432s + time:3 + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.486s + time:4 + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.605s + time:5 + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.874s + time:6 + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.66s + time:7 + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.488s + time:8 + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.391s + time:9 + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.064s + time:10 + input:alluxio://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.018s + time:1 + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.9s + time:2 + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.706s + time:3 + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.779s + time:4 + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.809s + time:5 + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.063s + time:6 + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.188s + time:7 + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.882s + time:8 + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:7.166s + time:9 + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.786s + time:10 + input:alluxio://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.84s + time:1 + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:5.316s + time:2 + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:5.533s + time:3 + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:5.299s + time:4 + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:5.381s + time:5 + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:10.477s + time:6 + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:5.333s + time:7 + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.183s + time:8 + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:10.117s + time:9 + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:5.327s + time:10 + input:alluxio://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.247s + time:1 + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:24.306s + 16/11/12 23:43:00 WARN QueuedThreadPool: 1 threads could not be stopped + time:2 + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:15.898s + time:3 + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:5.999s + time:4 + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:6.053s + time:5 + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:14.931s + time:6 + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.422s + time:7 + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.535s + time:8 + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:6.46s + time:9 + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:6.261s + time:10 + input:alluxio://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.549s + time:1 + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:16.498s + time:2 + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:17.061s + time:3 + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:6.414s + time:4 + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:7.03s + time:5 + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:6.505s + time:6 + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:6.117s + time:7 + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:6.952s + time:8 + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.545s + time:9 + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.553s + time:10 + input:alluxio://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:17.072s + time:1 + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:18.648s + time:2 + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:18.926s + time:3 + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:17.453s + time:4 + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:18.81s + time:5 + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:16.726s + time:6 + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:7.699s + time:7 + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:8.189s + time:8 + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:16.938s + time:9 + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:6.611s + time:10 + input:alluxio://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:6.54s + time:1 + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:28.424s + time:2 + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:28.496s + time:3 + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:34.933s + time:4 + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:28.524s + time:5 + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:28.146s + time:6 + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:29.543s + time:7 + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:41.073s + time:8 + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:28.614s + time:9 + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:28.967s + time:10 + input:alluxio://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:32.312s + time:1 + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:90.415s + time:2 + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:58.984s + time:3 + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:31.455s + time:4 + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:40.781s + time:5 + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:34.593s + time:6 + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:49.076s + time:7 + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:29.161s + time:8 + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:36.917s + time:9 + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:31.651s + time:10 + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:28.131s + time:1 + +## alluxio-0.7.1 + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f run1-0.7.1time201611131949.txt + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.893s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.882s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.912s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.939s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.877s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.784s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.835s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.835s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.01s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.865s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.01s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.107s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.029s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:3.908s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:3.929s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:3.716s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.085s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.016s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.071s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:3.903s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.228s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.168s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.232s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.072s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.254s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.385s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.201s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.381s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.162s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.18s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.288s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.337s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.447s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.312s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.376s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.322s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.446s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.47s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.354s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.429s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:6.288s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.551s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.598s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.623s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:5.115s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.651s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.472s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.659s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.565s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.891s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.928s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.711s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.772s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.683s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.58s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.882s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.819s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:5.149s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.627s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.74s + + +## 报错 + + input:alluxio://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:28.131s + time:1 + [Stage 0:> (1 + 16) / 63]16/11/13 00:06:31 ERROR TaskSetManager: Task 43 in stage 0.0 failed 4 times; aborting job + Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 43 in stage 0.0 failed 4 times, most recent failure: Lost task 43.3 in stage 0.0 (TID 22, 219.219.220.131): java.io.IOException: Block 5469372416 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) + + Driver stacktrace: + at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1283) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1271) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1270) + at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) + at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) + at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1270) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) + at scala.Option.foreach(Option.scala:236) + at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:697) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1496) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1458) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1447) + at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) + at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:567) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1824) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1837) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1850) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1921) + at org.apache.spark.rdd.RDD.count(RDD.scala:1125) + at alluxio.TimeCount$.compute(TimeCount.scala:32) + at alluxio.TimeCount$.main(TimeCount.scala:24) + at alluxio.TimeCount.main(TimeCount.scala) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:606) + at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:674) + at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180) + at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205) + at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:120) + at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) + Caused by: java.io.IOException: Block 5469372416 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) + time:2 + [Stage 0:=> (2 + 16) / 63]16/11/13 00:06:41 ERROR TaskSetManager: Task 24 in stage 0.0 failed 4 times; aborting job + Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 24 in stage 0.0 failed 4 times, most recent failure: Lost task 24.3 in stage 0.0 (TID 23, 219.219.220.180): java.io.IOException: Block 5721030656 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) + + Driver stacktrace: + at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1283) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1271) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1270) + at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) + at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) + at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1270) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) + at scala.Option.foreach(Option.scala:236) + at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:697) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1496) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1458) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1447) + at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) + at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:567) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1824) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1837) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1850) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1921) + at org.apache.spark.rdd.RDD.count(RDD.scala:1125) + at alluxio.TimeCount$.compute(TimeCount.scala:32) + at alluxio.TimeCount$.main(TimeCount.scala:24) + at alluxio.TimeCount.main(TimeCount.scala) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:606) + at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:674) + at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180) + at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205) + at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:120) + at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) + Caused by: java.io.IOException: Block 5721030656 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) + time:3 + [Stage 0:> (1 + 16) / 63]16/11/13 00:06:52 ERROR TaskSetManager: Task 43 in stage 0.0 failed 4 times; aborting job + Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 43 in stage 0.0 failed 4 times, most recent failure: Lost task 43.3 in stage 0.0 (TID 23, 219.219.220.131): java.io.IOException: Block 5469372416 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) + + Driver stacktrace: + at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1283) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1271) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1270) + at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) + at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) + at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1270) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) + at scala.Option.foreach(Option.scala:236) + at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:697) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1496) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1458) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1447) + at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) + at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:567) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1824) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1837) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1850) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1921) + at org.apache.spark.rdd.RDD.count(RDD.scala:1125) + at alluxio.TimeCount$.compute(TimeCount.scala:32) + at alluxio.TimeCount$.main(TimeCount.scala:24) + at alluxio.TimeCount.main(TimeCount.scala) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:606) + at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:674) + at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180) + at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205) + at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:120) + at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) + Caused by: java.io.IOException: Block 5469372416 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) + time:4 + [Stage 0:> (0 + 16) / 63]16/11/13 00:07:02 ERROR TaskSetManager: Task 59 in stage 0.0 failed 4 times; aborting job + Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 59 in stage 0.0 failed 4 times, most recent failure: Lost task 59.3 in stage 0.0 (TID 22, 219.219.220.233): java.io.IOException: Block 5486149632 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) + + Driver stacktrace: + at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1283) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1271) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1270) + at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) + at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) + at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1270) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) + at scala.Option.foreach(Option.scala:236) + at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:697) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1496) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1458) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1447) + at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) + at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:567) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1824) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1837) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1850) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1921) + at org.apache.spark.rdd.RDD.count(RDD.scala:1125) + at alluxio.TimeCount$.compute(TimeCount.scala:32) + at alluxio.TimeCount$.main(TimeCount.scala:24) + at alluxio.TimeCount.main(TimeCount.scala) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:606) + at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:674) + at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180) + at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205) + at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:120) + at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) + Caused by: java.io.IOException: Block 5486149632 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) + time:5 + [Stage 0:> (1 + 16) / 63]16/11/13 00:07:13 ERROR TaskSetManager: Task 59 in stage 0.0 failed 4 times; aborting job + Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 59 in stage 0.0 failed 4 times, most recent failure: Lost task 59.3 in stage 0.0 (TID 23, 219.219.220.131): java.io.IOException: Block 5486149632 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) + + Driver stacktrace: + at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1283) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1271) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1270) + at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) + at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) + at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1270) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) + at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697) + at scala.Option.foreach(Option.scala:236) + at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:697) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1496) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1458) + at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1447) + at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) + at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:567) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1824) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1837) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1850) + at org.apache.spark.SparkContext.runJob(SparkContext.scala:1921) + at org.apache.spark.rdd.RDD.count(RDD.scala:1125) + at alluxio.TimeCount$.compute(TimeCount.scala:32) + at alluxio.TimeCount$.main(TimeCount.scala:24) + at alluxio.TimeCount.main(TimeCount.scala) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:606) + at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:674) + at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180) + at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205) + at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:120) + at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) + Caused by: java.io.IOException: Block 5486149632 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + at java.lang.Thread.run(Thread.java:745) + time:6 + [Stage 0:=> (2 + 16) / 63]16/11/13 00:07:24 ERROR TaskSetManager: Task 59 in stage 0.0 failed 4 times; aborting job + Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 59 in stage 0.0 failed 4 times, most recent failure: Lost task 59.3 in stage 0.0 (TID 23, 219.219.220.131): java.io.IOException: Block 5486149632 is not available in Alluxio + at alluxio.client.block.AlluxioBlockStore.getInStream(AlluxioBlockStore.java:134) + at alluxio.client.file.FileInStream.updateBlockInStream(FileInStream.java:519) + at alluxio.client.file.FileInStream.updateStreams(FileInStream.java:426) + at alluxio.client.file.FileInStream.close(FileInStream.java:145) + at alluxio.hadoop.HdfsFileInputStream.read(HdfsFileInputStream.java:194) + at java.io.DataInputStream.read(DataInputStream.java:100) + at org.apache.hadoop.util.LineReader.fillBuffer(LineReader.java:180) + at org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216) + at org.apache.hadoop.util.LineReader.readLine(LineReader.java:174) + at org.apache.hadoop.mapred.LineRecordReader.skipUtfByteOrderMark(LineRecordReader.java:206) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:244) + at org.apache.hadoop.mapred.LineRecordReader.next(LineRecordReader.java:47) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:248) + at org.apache.spark.rdd.HadoopRDD$$anon$1.getNext(HadoopRDD.scala:216) + at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) + at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39) + at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327) + at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1553) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1125) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850) + at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) + at org.apache.spark.scheduler.Task.run(Task.scala:88) + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24023--alluxio-0.7.1\350\247\243\345\206\263\346\225\260\346\215\256\346\234\254\345\234\260\345\214\226\347\232\204\351\227\256\351\242\230.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24023--alluxio-0.7.1\350\247\243\345\206\263\346\225\260\346\215\256\346\234\254\345\234\260\345\214\226\347\232\204\351\227\256\351\242\230.md" new file mode 100644 index 0000000..236586b --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24023--alluxio-0.7.1\350\247\243\345\206\263\346\225\260\346\215\256\346\234\254\345\234\260\345\214\226\347\232\204\351\227\256\351\242\230.md" @@ -0,0 +1,212 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +# 1.解释 # + +spark-1.5.2和alluxio-1.3.0默认不支持数据本地化，所以会有跨节点传输 + +# 2.解决办法： # + +## 2.1 方法1## +使用alluxio-0.7.1 + +## 2.2 方法2## + +设置spark本地化配置，参考【5】 +各个节点分别配置 +实验分析支持NODE_LOCAL + +# 3.结果： # + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f run1-0.7.1time201611132026.txt + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.812s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.87s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:4.046s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.942s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.818s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.773s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.837s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.756s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.864s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D1Line.fasta count:78295 countTime:3.811s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:3.964s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.792s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.013s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:3.87s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.066s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:3.976s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:3.893s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.242s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:3.969s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D2Line.fasta count:156590 countTime:4.183s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:3.987s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.035s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.128s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.137s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.253s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.139s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.059s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.051s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.074s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D3Line.fasta count:313180 countTime:4.197s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.313s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.36s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.372s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.329s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.379s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.299s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.382s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.33s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.26s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D4Line.fasta count:626360 countTime:4.306s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.565s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.419s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.39s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:5.67s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.632s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.513s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.474s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.492s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.574s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D5Line.fasta count:1252720 countTime:4.586s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.643s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.67s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:5.148s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.722s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.639s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.561s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:5.089s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:5.238s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.677s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D6Line.fasta count:2505440 countTime:4.723s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:6.634s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:6.763s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:5.376s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:5.515s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:5.274s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:5.447s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:5.512s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:6.46s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:5.394s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D7Line.fasta count:5019006 countTime:5.373s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:6.527s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:6.64s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:6.574s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:6.482s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:6.718s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:6.509s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:6.583s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:6.643s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:6.602s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D8Line.fasta count:10038012 countTime:6.573s + time:1 + input:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:10.066s + time:2 + input:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:10.084s + time:3 + input:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:10.93s + time:4 + input:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:10.347s + time:5 + input:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:9.976s + time:6 + input:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:10.817s + time:7 + input:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:10.358s + time:8 + input:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:9.881s + time:9 + input:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:10.23s + time:10 + input:tachyon://Master:19998/xubo/alluxio/uniref/D9Line.fasta count:20076024 countTime:10.119s + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning + 【5】http://www.alluxio.org/docs/master/cn/Running-Spark-on-Alluxio.html diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24024--\345\210\206\345\210\253\350\257\273\345\217\226HDFS\345\222\214Alluxio\347\232\204\346\225\260\346\215\256\350\277\233\350\241\214line count\346\257\224\350\276\203\345\210\206\346\236\220.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24024--\345\210\206\345\210\253\350\257\273\345\217\226HDFS\345\222\214Alluxio\347\232\204\346\225\260\346\215\256\350\277\233\350\241\214line count\346\257\224\350\276\203\345\210\206\346\236\220.md" new file mode 100644 index 0000000..20a02ad --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24024--\345\210\206\345\210\253\350\257\273\345\217\226HDFS\345\222\214Alluxio\347\232\204\346\225\260\346\215\256\350\277\233\350\241\214line count\346\257\224\350\276\203\345\210\206\346\236\220.md" @@ -0,0 +1,375 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +# 1.解释 # + + 测量alluxio和hdfs哪个速度快 + + +# 2.代码： # + +## 2.1 hdfs ## + for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 + do + echo 'time:'$i + sh test.sh 10G hdfs://Master:9000/xubo/project/SparkSW/input/'D9Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + done + +## 2.2 aluxio ## + + for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 + do + echo 'time:'$i + sh test.sh 10G alluxio://Master:19998/xubo/project/SparkSW/input/'D9Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + done +## 2.3 test.sh + + mem=$1 + input=$2 + + #/home/zgg/lib/spark-1.0.1-bin-hadoop2/bin/spark-submit \ + spark-submit \ + --class alluxio.TimeCount \ + --master spark://219.219.220.149:7077 \ + --executor-memory $mem \ + SparkSW.jar $input $3 + + +# 3.结果： # + +##3.1 HDFS + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run1D9hdfs.sh + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:34.178s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:23.82s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.884s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.471s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:14.446s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:12.18s + 16/11/17 17:45:25 ERROR LiveListenerBus: SparkListenerBus has already stopped! Dropping event SparkListenerExecutorMetricsUpdate(6,WrappedArray()) + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:12.474s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:13.26s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.699s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.991s + time:11 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.977s + time:12 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.933s + time:13 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.322s + time:14 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.735s + time:15 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:12.489s + time:16 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.832s + time:17 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.883s + time:18 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.917s + time:19 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.032s + time:20 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.208s + time:21 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:12.089s + time:22 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.055s + time:23 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.839s + time:24 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:12.651s + time:25 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.999s + time:26 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.73s + time:27 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.621s + time:28 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.672s + time:29 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.65s + time:30 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.054s + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run1D9hdfs.sh + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.726s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.86s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.724s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.393s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.963s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.96s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.866s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.52s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.084s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.081s + time:11 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.56s + time:12 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.043s + time:13 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.598s + time:14 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.569s + time:15 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.594s + time:16 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.674s + time:17 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.713s + time:18 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.733s + time:19 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.691s + time:20 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.816s + time:21 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.64s + time:22 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.841s + time:23 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.537s + time:24 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.629s + time:25 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.702s + time:26 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:11.089s + time:27 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.99s + time:28 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.672s + time:29 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.597s + time:30 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.612s + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run1D9hdfs.sh + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:13.733s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:12.415s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:12.306s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:12.588s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:12.306s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:12.848s + [Stage 1:==========================================> (12 + 4) / 16]^CException in thread "main" org.apache.spark.SparkException: Job cancelled because SparkContext was shut down + +## 3.2 alluxio-1.3.0 + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run1D9.sh + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:202.784s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:124.612s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:66.245s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:63.52s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.778s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:75.229s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:50.952s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:41.106s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:39.525s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:39.792s + time:11 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:17.647s + time:12 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.726s + time:13 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:39.813s + time:14 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:40.184s + time:15 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.303s + time:16 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.322s + time:17 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:21.089s + time:18 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:20.824s + time:19 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:16.833s + time:20 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:31.777s + time:21 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.146s + time:22 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:16.895s + time:23 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.393s + time:24 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.626s + time:25 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.516s + time:26 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.707s + time:27 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:17.168s + time:28 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:39.63s + time:29 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.63s + time:30 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:20.168s + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run1D9.sh + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:17.111s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.414s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:17.674s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.614s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.574s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.633s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.124s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:8.262s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.193s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.095s + time:11 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:20.203s + time:12 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.528s + time:13 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:9.224s + time:14 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.559s + time:15 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.397s + time:16 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.286s + time:17 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:40.486s + time:18 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:16.913s + time:19 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:16.914s + time:20 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.536s + time:21 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.72s + time:22 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.176s + time:23 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:39.896s + time:24 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.339s + time:25 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.388s + time:26 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.289s + time:27 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.886s + time:28 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.935s + time:29 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.754s + time:30 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.431s + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ ./run1D9.sh + time:1 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:16.877s + time:2 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.576s + time:3 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.556s + time:4 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:31.889s + time:5 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:8.711s + time:6 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.514s + time:7 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.962s + time:8 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:20.938s + time:9 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.465s + time:10 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.108s + time:11 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.865s + time:12 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.47s + time:13 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:17.94s + time:14 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.688s + time:15 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:31.033s + time:16 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:17.846s + time:17 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.637s + time:18 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.046s + time:19 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.653s + time:20 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.897s + time:21 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.641s + time:22 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.49s + time:23 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.21s + time:24 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.12s + time:25 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.996s + time:26 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.852s + time:27 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.955s + time:28 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.634s + time:29 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.581s + time:30 + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:7.489s + + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24025--spark\344\273\216HDFS\345\222\214Alluxio\350\257\273\345\217\226\346\227\266\351\227\264\346\257\224\350\276\203.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24025--spark\344\273\216HDFS\345\222\214Alluxio\350\257\273\345\217\226\346\227\266\351\227\264\346\257\224\350\276\203.md" new file mode 100644 index 0000000..3a8f476 --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24025--spark\344\273\216HDFS\345\222\214Alluxio\350\257\273\345\217\226\346\227\266\351\227\264\346\257\224\350\276\203.md" @@ -0,0 +1,258 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + +# 1.解释 # + + 想要分析alluxio加速效果，发现alluxio会出现长尾效应，导致有些task特别耗时，相对于HDFS，并没有明显优势。 + + + +# 2.代码： # + + #~/cloud/alluxio-1.3.0/bin/alluxio fs mkdir -p /xubo/project/SparkSW/input/ + for i in 1 2 3 4 5 6 7 8 9 10 + do + echo 'time:'$i + for j in 1 2 3 4 5 6 7 8 9 10 + do + sh test.sh 10G hdfs://Master:9000/xubo/project/SparkSW/input/'D'$i'Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + #sh test.sh 10G alluxio://Master:19998/xubo/alluxio/uniref/'D'$j'Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + #sh test.sh 10G alluxio://Master:19998/xubo/project/SparkSW/input/'D'$j'Line.fasta' alluxio://Master:19998/xubo/project/alluxio/output/TimeCount + done + + for j in 1 2 3 4 5 6 7 8 9 10 + do + sh test.sh 10G alluxio://Master:19998/xubo/project/SparkSW/input/'D'$i'Line.fasta' hdfs://Master:9000/xubo/project/alluxio/output/TimeCount + done + done + + + +# 3.结果： # + + hadoop@Master:~/disk2/xubo/project/alluxio/timeCount$ tail -f runtime201611231839.txt + time:1 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.918s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.923s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.848s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.959s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.85s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.816s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:4.208s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.925s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.001s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:2.817s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.466s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:2.83s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:2.757s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:2.824s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:2.813s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:2.802s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:2.847s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:2.88s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:3.963s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D1Line.fasta count:78295 countTime:2.788s + time:2 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.074s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:2.945s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.023s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.999s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.073s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.206s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.028s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:4.049s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.927s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.964s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.875s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:6.505s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.001s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.276s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.047s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.081s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.365s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.083s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.141s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D2Line.fasta count:156590 countTime:3.027s + time:3 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.572s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.45s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.342s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.336s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.508s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.268s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.558s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.401s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.457s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.513s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:5.515s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.846s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.215s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.242s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.181s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.236s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:3.047s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.178s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.24s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D3Line.fasta count:313180 countTime:4.178s + time:4 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:7.852s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:6.148s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:7.575s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:16.612s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:8.009s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:8.02s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:7.937s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:8.007s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:16.949s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:7.795s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:7.182s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:4.506s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:3.463s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:3.501s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:3.544s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:3.747s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:3.523s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:3.4s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:3.325s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D4Line.fasta count:626360 countTime:3.519s + time:5 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:3.988s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.004s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:5.494s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.916s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:5.479s + 16/11/23 18:56:20 WARN QueuedThreadPool: 4 threads could not be stopped + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:3.891s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.73s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.851s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.9s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.966s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:7.924s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:6.396s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:3.487s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:5.417s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.272s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:3.465s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:3.437s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:4.067s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:6.194s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D5Line.fasta count:1252720 countTime:5.352s + time:6 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.883s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.119s + 16/11/23 18:59:47 WARN QueuedThreadPool: 4 threads could not be stopped + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.172s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:5.09s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:7.57s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.578s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:7.317s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.857s + 16/11/23 19:01:03 WARN QueuedThreadPool: 1 threads could not be stopped + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.854s + 16/11/23 19:01:16 WARN QueuedThreadPool: 1 threads could not be stopped + input:hdfs://Master:9000/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:3.994s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:15.504s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:6.336s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:6.456s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.951s + 16/11/23 19:02:27 WARN QueuedThreadPool: 2 threads could not be stopped + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:5.525s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:15.943s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.665s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.814s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:15.965s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D6Line.fasta count:2505440 countTime:4.915s + time:7 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:9.814s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:9.196s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:9.434s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:9.172s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:20.193s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:10.454s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:9.551s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:9.321s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:10.313s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:9.707s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:52.331s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:6.583s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:16.998s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:16.55s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:6.496s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:6.781s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:7.477s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:17.51s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:6.494s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D7Line.fasta count:5019006 countTime:17.511s + time:8 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:10.427s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:9.993s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:11.242s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:10.868s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:7.354s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:7.015s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:7.344s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:7.182s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:7.262s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:8.282s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:42.406s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:16.769s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:18.197s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:7.881s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:8.913s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:8.174s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:20.971s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:8.434s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:8.316s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D8Line.fasta count:10038012 countTime:16.767s + time:9 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:32.624s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:19.688s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:17.345s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:20.98s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:13.478s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:27.69s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:15.567s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:14.237s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:16.413s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:10.537s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:99.776s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.482s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:29.184s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:17.949s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:17.29s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.279s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:30.892s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:18.895s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:28.351s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D9Line.fasta count:20076024 countTime:20.001s + time:10 + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:38.227s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:25.727s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:23.691s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:19.047s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:19.611s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:14.391s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:16.036s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:16.249s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:15.324s + input:hdfs://Master:9000/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:13.936s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:151.861s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:46.322s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:31.237s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:31.318s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:19.097s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:19.043s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:30.663s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:43.809s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:20.463s + input:alluxio://Master:19998/xubo/project/SparkSW/input/D10Line.fasta count:40152048 countTime:31.099s + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24026--SparkSW\344\273\216HDFS\345\222\214Alluxio\350\257\273\345\217\226\350\256\241\347\256\227\346\227\266\351\227\264\346\257\224\350\276\203.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24026--SparkSW\344\273\216HDFS\345\222\214Alluxio\350\257\273\345\217\226\350\256\241\347\256\227\346\227\266\351\227\264\346\257\224\350\276\203.md" new file mode 100644 index 0000000..d2159c9 --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\24026--SparkSW\344\273\216HDFS\345\222\214Alluxio\350\257\273\345\217\226\350\256\241\347\256\227\346\227\266\351\227\264\346\257\224\350\276\203.md" @@ -0,0 +1,1093 @@ + + 更多代码请见：https://github.com/xubo245/SparkLearning + + Spark生态之Alluxio学习 版本：alluxio-1.3.0（tachyon），spark-1.5.2,hadoop-2.6.0 + + +# 结果： # + + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:5 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:6 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + 16/11/23 21:02:08 WARN QueuedThreadPool: 1 threads could not be stopped + 16/11/23 21:02:14 ERROR AkkaRpcEnv: Ignore error: Error notifying standalone scheduler's driver endpoint + org.apache.spark.SparkException: Error notifying standalone scheduler's driver endpoint + at org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend.removeExecutor(CoarseGrainedSchedulerBackend.scala:312) + at org.apache.spark.scheduler.cluster.SparkDeploySchedulerBackend.executorRemoved(SparkDeploySchedulerBackend.scala:142) + at org.apache.spark.deploy.client.AppClient$ClientEndpoint$$anonfun$receive$1.applyOrElse(AppClient.scala:184) + at org.apache.spark.rpc.akka.AkkaRpcEnv.org$apache$spark$rpc$akka$AkkaRpcEnv$$processMessage(AkkaRpcEnv.scala:177) + at org.apache.spark.rpc.akka.AkkaRpcEnv$$anonfun$actorRef$lzycompute$1$1$$anon$1$$anonfun$receiveWithLogging$1$$anonfun$applyOrElse$4.apply$mcV$sp(AkkaRpcEnv.scala:126) + at org.apache.spark.rpc.akka.AkkaRpcEnv.org$apache$spark$rpc$akka$AkkaRpcEnv$$safelyCall(AkkaRpcEnv.scala:197) + at org.apache.spark.rpc.akka.AkkaRpcEnv$$anonfun$actorRef$lzycompute$1$1$$anon$1$$anonfun$receiveWithLogging$1.applyOrElse(AkkaRpcEnv.scala:125) + at scala.runtime.AbstractPartialFunction$mcVL$sp.apply$mcVL$sp(AbstractPartialFunction.scala:33) + at scala.runtime.AbstractPartialFunction$mcVL$sp.apply(AbstractPartialFunction.scala:33) + at scala.runtime.AbstractPartialFunction$mcVL$sp.apply(AbstractPartialFunction.scala:25) + at org.apache.spark.util.ActorLogReceive$$anon$1.apply(ActorLogReceive.scala:59) + at org.apache.spark.util.ActorLogReceive$$anon$1.apply(ActorLogReceive.scala:42) + at scala.PartialFunction$class.applyOrElse(PartialFunction.scala:118) + at org.apache.spark.util.ActorLogReceive$$anon$1.applyOrElse(ActorLogReceive.scala:42) + at akka.actor.Actor$class.aroundReceive(Actor.scala:467) + at org.apache.spark.rpc.akka.AkkaRpcEnv$$anonfun$actorRef$lzycompute$1$1$$anon$1.aroundReceive(AkkaRpcEnv.scala:92) + at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) + at akka.actor.ActorCell.invoke(ActorCell.scala:487) + at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) + at akka.dispatch.Mailbox.run(Mailbox.scala:220) + at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397) + at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) + at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) + at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) + at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) + Caused by: org.apache.spark.SparkException: Error sending message [message = RemoveExecutor(4,Unknown executor exit code (0))] + at org.apache.spark.rpc.RpcEndpointRef.askWithRetry(RpcEndpointRef.scala:118) + at org.apache.spark.rpc.RpcEndpointRef.askWithRetry(RpcEndpointRef.scala:77) + at org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend.removeExecutor(CoarseGrainedSchedulerBackend.scala:309) + ... 24 more + Caused by: org.apache.spark.rpc.RpcTimeoutException: Recipient[Actor[akka://sparkDriver/user/CoarseGrainedScheduler#-679832735]] had already been terminated.. This timeout is controlled by spark.rpc.askTimeout + at org.apache.spark.rpc.RpcTimeout.org$apache$spark$rpc$RpcTimeout$$createRpcTimeoutException(RpcEnv.scala:214) + at org.apache.spark.rpc.RpcTimeout$$anonfun$addMessageIfTimeout$1.applyOrElse(RpcEnv.scala:229) + at org.apache.spark.rpc.RpcTimeout$$anonfun$addMessageIfTimeout$1.applyOrElse(RpcEnv.scala:225) + at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:33) + at scala.util.Failure$$anonfun$recover$1.apply(Try.scala:185) + at scala.util.Try$.apply(Try.scala:161) + at scala.util.Failure.recover(Try.scala:185) + at scala.concurrent.Future$$anonfun$recover$1.apply(Future.scala:324) + at scala.concurrent.Future$$anonfun$recover$1.apply(Future.scala:324) + at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32) + at org.spark-project.guava.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:293) + at scala.concurrent.impl.ExecutionContextImpl$$anon$1.execute(ExecutionContextImpl.scala:133) + at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40) + at scala.concurrent.impl.Promise$DefaultPromise.scala$concurrent$impl$Promise$DefaultPromise$$dispatchOrAddCallback(Promise.scala:280) + at scala.concurrent.impl.Promise$DefaultPromise.onComplete(Promise.scala:270) + at scala.concurrent.Future$class.recover(Future.scala:324) + at scala.concurrent.impl.Promise$DefaultPromise.recover(Promise.scala:153) + at org.apache.spark.rpc.akka.AkkaRpcEndpointRef.ask(AkkaRpcEnv.scala:319) + at org.apache.spark.rpc.RpcEndpointRef.askWithRetry(RpcEndpointRef.scala:100) + ... 26 more + Caused by: akka.pattern.AskTimeoutException: Recipient[Actor[akka://sparkDriver/user/CoarseGrainedScheduler#-679832735]] had already been terminated. + at akka.pattern.AskableActorRef$.ask$extension(AskSupport.scala:132) + at org.apache.spark.rpc.akka.AkkaRpcEndpointRef.ask(AkkaRpcEnv.scala:307) + ... 27 more + time:7 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:8 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + 16/11/23 21:03:30 WARN QueuedThreadPool: 5 threads could not be stopped + time:9 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:10 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + 16/11/23 21:04:45 WARN QueuedThreadPool: 4 threads could not be stopped + HDFS: + time:1 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + 16/11/23 21:05:28 WARN QueuedThreadPool: 4 threads could not be stopped + time:2 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:3 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:4 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:5 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:6 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:7 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:8 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:9 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:10 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + alluxio + time:1 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:2 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:3 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:4 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_P04406) + time:5 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:6 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:7 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_P04406) + time:8 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:9 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:10 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + HDFS: + time:1 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:2 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_A9NHD0) + time:3 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:4 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_A9NHD0) + time:5 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:6 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:7 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_A9NHD0) + time:8 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:9 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:10 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + alluxio + time:1 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:2 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:3 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:4 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:5 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:6 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:7 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_P04406) + time:8 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:9 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:10 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + HDFS: + time:1 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:2 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:3 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:4 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_A9NHD0) + time:5 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:6 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_A9NHD0) + time:7 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:8 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_P04406) + time:9 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:10 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_A9NHD0) + alluxio + time:1 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_P04406) + time:2 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:3 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_P04406) + time:4 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:5 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:6 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_A9NHD0) + time:7 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:8 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_A9NHD0) + time:9 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:10 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + HDFS: + time:1 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:2 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_P04406) + time:3 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:4 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:5 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_P04406) + time:6 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:7 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:8 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:9 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_P04406) + time:10 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + alluxio + time:1 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:2 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:3 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:4 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_P04406) + time:5 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:6 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:7 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:8 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + time:9 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_Q05025) + (55,UniRef100_E9DFH0) + (54,UniRef100_P04406) + time:10 + (67,UniRef100_P18691) + (58,UniRef100_P51640) + (55,UniRef100_E9DFH0) + (55,UniRef100_Q05025) + (54,UniRef100_A9NHD0) + HDFS: + time:1 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:2 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:3 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:4 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:5 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:6 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:7 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:8 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:9 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:10 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + alluxio + time:1 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:2 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:3 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:4 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:5 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:6 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:7 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:8 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:9 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + time:10 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (56,UniRef100_B0CXI5) + HDFS: + time:1 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (57,UniRef100_A0A163CSK3) + time:2 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:3 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:4 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:5 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:6 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:7 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (57,UniRef100_A0A163CSK3) + time:8 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:9 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:10 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (57,UniRef100_A0A163CSK3) + alluxio + time:1 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:2 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:3 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:4 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:5 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:6 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (57,UniRef100_A0A163CSK3) + time:7 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:8 + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + (57,UniRef100_A0A163CSK3) + time:9 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (57,UniRef100_L7QH53) + time:10 + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + (57,UniRef100_A0A163CSK3) + HDFS: + time:1 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_M3XW90) + time:2 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + time:3 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_M3WVW7) + time:4 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_M3WVW7) + time:5 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + time:6 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + time:7 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_D2I7L4) + time:8 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_D2I7L4) + time:9 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_Q0QEU8) + time:10 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_M3WVW7) + alluxio + time:1 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + time:2 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + time:3 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_Q0QEU8) + time:4 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + time:5 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_Q0QEU8) + time:6 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_J9NZ39) + time:7 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_J9NZ39) + time:8 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_J9NZ39) + time:9 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_M3WVW7) + time:10 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_M3WVW7) + HDFS: + time:1 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_G3WQA5) + time:2 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_D2I7L4) + time:3 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_M3XW90) + time:4 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_W4LL34) + time:5 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_M3XW90) + time:6 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_W4LL34) + time:7 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_D2I7L4) + time:8 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_P51640) + time:9 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_W4LL34) + time:10 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_D2I7L4) + alluxio + time:1 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_D2I7L4) + time:2 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_M3XW90) + time:3 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_D2I7L4) + time:4 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_G3WQA5) + time:5 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_A0A0E1RXE0) + (61,UniRef100_C5P0L7) + (58,UniRef100_P51640) + time:6 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_D2I7L4) + time:7 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_G3WQA5) + time:8 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_G3WQA5) + time:9 + (67,UniRef100_G3WWG6) + (67,UniRef100_P18691) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_W4LL34) + time:10 + (67,UniRef100_P18691) + (67,UniRef100_G3WWG6) + (61,UniRef100_C5P0L7) + (61,UniRef100_A0A0E1RXE0) + (58,UniRef100_W4LL34) + + +参考 + + 【1】https://github.com/Alluxio/alluxio + 【2】http://www.alluxio.org/ + 【3】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【4】https://github.com/xubo245/SparkLearning diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2403---\346\234\272\345\231\250\351\207\215\345\220\257\345\220\216\346\225\260\346\215\256\345\255\230\345\202\250\344\275\215\347\275\256\347\232\204\345\217\230\345\214\226.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2403---\346\234\272\345\231\250\351\207\215\345\220\257\345\220\216\346\225\260\346\215\256\345\255\230\345\202\250\344\275\215\347\275\256\347\232\204\345\217\230\345\214\226.md" new file mode 100644 index 0000000..a07d111 --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2403---\346\234\272\345\231\250\351\207\215\345\220\257\345\220\216\346\225\260\346\215\256\345\255\230\345\202\250\344\275\215\347\275\256\347\232\204\345\217\230\345\214\226.md" @@ -0,0 +1,31 @@ + +1.重启后数据没有在内存里了，由于安装时用的是bin，而且设置的是local，及其重启后需要重启tachyon： + + xubo@xubo:~/cloud/tachyon-0.7.1/bin$ ./tachyon-start.sh local + Killed 0 processes on xubo + Killed 0 processes on xubo + Connecting to localhost as xubo... + Killed 0 processes on xubo + Connection to localhost closed. + [sudo] password for xubo: + Formatting RamFS: /mnt/ramdisk (1gb) + Starting master @ localhost + Starting worker @ xubo + +2.重启后的数据没在内存，而在local，根据tachyon.env.sh可以知道存储的位置在： + + xubo@xubo:~/cloud/tachyon-0.7.1/underFSStorage/tmp/tachyon/data$ ll -h + total 114M + drwxrwxrwt 2 xubo xubo 4.0K May 5 14:02 ./ + drwxrwxr-x 4 xubo xubo 4.0K May 4 22:03 ../ + -rwxrwxrwt 1 xubo xubo 80 May 4 22:19 3* + -rwxrwxrwt 1 xubo xubo 17K May 4 22:22 4* + -rwxrwxrwt 1 xubo xubo 114M May 4 22:25 5* + -rwxrwxrwt 1 xubo xubo 21 May 4 22:29 6* + xubo@xubo:~/cloud/tachyon-0.7.1/underFSStorage/tmp/tachyon/data$ cat 6 + hello tachyon + 1 + 2 + 3 + + diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2404---\344\270\213\350\275\275\346\272\220\347\240\201\351\200\232\350\277\207maven install\345\256\211\350\243\205\345\244\261\350\264\245\350\256\260\345\275\225.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2404---\344\270\213\350\275\275\346\272\220\347\240\201\351\200\232\350\277\207maven install\345\256\211\350\243\205\345\244\261\350\264\245\350\256\260\345\275\225.md" new file mode 100644 index 0000000..ccef732 --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2404---\344\270\213\350\275\275\346\272\220\347\240\201\351\200\232\350\277\207maven install\345\256\211\350\243\205\345\244\261\350\264\245\350\256\260\345\275\225.md" @@ -0,0 +1,439 @@ +版本：v0.7.1.tar.gz +记录 + +（2）制定版本 + + mvn clean package -Djava.version=1.7 -Dhadoop.version=2.6.0 -Dspark.version=1.5.2 -DskipTests + + +（1） + + xubo@xubo:~/cloud/tachyon-0.7.1$ mvn install + [INFO] Scanning for projects... + [INFO] ------------------------------------------------------------------------ + [INFO] Reactor Build Order: + [INFO] + [INFO] Tachyon Parent + [INFO] Tachyon Common + [INFO] Tachyon Under File System + [INFO] Tachyon Under File System - Local FS + [INFO] Tachyon Under File System - HDFS + [INFO] Tachyon Under File System - Gluster FS + [INFO] Tachyon Under File System - Swift + [INFO] Tachyon Under File System - S3 + [INFO] Tachyon Clients + [INFO] Tachyon Clients - Implementation + [INFO] Tachyon Clients - Distribution + [INFO] Tachyon Servers + [INFO] Mock Tachyon Cluster + [INFO] Tachyon Integration Tests + [INFO] Tachyon Shell + [INFO] Tachyon Examples + [INFO] Tachyon Assemblies + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Parent 0.7.1 + [INFO] ------------------------------------------------------------------------ + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-source-plugin/2.3/maven-source-plugin-2.3.pom + May 05, 2016 2:10:48 PM org.apache.maven.wagon.providers.http.httpclient.impl.execchain.RetryExec execute + INFO: I/O exception (java.net.SocketException) caught when processing request to {s}->https://repo.maven.apache.org:443: Connection reset + May 05, 2016 2:10:48 PM org.apache.maven.wagon.providers.http.httpclient.impl.execchain.RetryExec execute + INFO: Retrying request to {s}->https://repo.maven.apache.org:443 + May 05, 2016 2:10:51 PM org.apache.maven.wagon.providers.http.httpclient.impl.execchain.RetryExec execute + INFO: I/O exception (java.net.SocketException) caught when processing request to {s}->https://repo.maven.apache.org:443: Connection reset + May 05, 2016 2:10:51 PM org.apache.maven.wagon.providers.http.httpclient.impl.execchain.RetryExec execute + INFO: Retrying request to {s}->https://repo.maven.apache.org:443 + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-source-plugin/2.3/maven-source-plugin-2.3.pom (7 KB at 0.7 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-source-plugin/2.3/maven-source-plugin-2.3.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-source-plugin/2.3/maven-source-plugin-2.3.jar (31 KB at 18.8 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-javadoc-plugin/2.9/maven-javadoc-plugin-2.9.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-javadoc-plugin/2.9/maven-javadoc-plugin-2.9.pom (16 KB at 22.8 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-javadoc-plugin/2.9/maven-javadoc-plugin-2.9.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-javadoc-plugin/2.9/maven-javadoc-plugin-2.9.jar (354 KB at 52.2 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-checkstyle-plugin/2.13/maven-checkstyle-plugin-2.13.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-checkstyle-plugin/2.13/maven-checkstyle-plugin-2.13.pom (13 KB at 18.8 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-checkstyle-plugin/2.13/maven-checkstyle-plugin-2.13.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-checkstyle-plugin/2.13/maven-checkstyle-plugin-2.13.jar (112 KB at 34.6 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/com/mycila/license-maven-plugin/2.9/license-maven-plugin-2.9.pom + Downloaded: https://repo.maven.apache.org/maven2/com/mycila/license-maven-plugin/2.9/license-maven-plugin-2.9.pom (5 KB at 5.6 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/com/mycila/license-maven-plugin-parent/2.9/license-maven-plugin-parent-2.9.pom + Downloaded: https://repo.maven.apache.org/maven2/com/mycila/license-maven-plugin-parent/2.9/license-maven-plugin-parent-2.9.pom (4 KB at 5.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/com/mycila/mycila-pom/3/mycila-pom-3.pom + Downloaded: https://repo.maven.apache.org/maven2/com/mycila/mycila-pom/3/mycila-pom-3.pom (20 KB at 18.9 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/com/mycila/license-maven-plugin/2.9/license-maven-plugin-2.9.jar + Downloaded: https://repo.maven.apache.org/maven2/com/mycila/license-maven-plugin/2.9/license-maven-plugin-2.9.jar (79 KB at 20.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-install-plugin/2.4/maven-install-plugin-2.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-install-plugin/2.4/maven-install-plugin-2.4.pom (7 KB at 9.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-install-plugin/2.4/maven-install-plugin-2.4.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/plugins/maven-install-plugin/2.4/maven-install-plugin-2.4.jar (27 KB at 26.8 KB/sec) + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-parent --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-parent --- + Downloading: https://repo.maven.apache.org/maven2/com/puppycrawl/tools/checkstyle/5.9/checkstyle-5.9.pom + Downloaded: https://repo.maven.apache.org/maven2/com/puppycrawl/tools/checkstyle/5.9/checkstyle-5.9.pom (26 KB at 24.2 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-beanutils/commons-beanutils-core/1.8.3/commons-beanutils-core-1.8.3.pom + Downloaded: https://repo.maven.apache.org/maven2/commons-beanutils/commons-beanutils-core/1.8.3/commons-beanutils-core-1.8.3.pom (2 KB at 2.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/com/google/guava/guava-jdk5/14.0.1/guava-jdk5-14.0.1.pom + Downloaded: https://repo.maven.apache.org/maven2/com/google/guava/guava-jdk5/14.0.1/guava-jdk5-14.0.1.pom (7 KB at 9.6 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/com/google/guava/guava-parent-jdk5/14.0.1/guava-parent-jdk5-14.0.1.pom + Downloaded: https://repo.maven.apache.org/maven2/com/google/guava/guava-parent-jdk5/14.0.1/guava-parent-jdk5-14.0.1.pom (3 KB at 4.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/reporting/maven-reporting-impl/2.3/maven-reporting-impl-2.3.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/reporting/maven-reporting-impl/2.3/maven-reporting-impl-2.3.pom (5 KB at 7.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/shared/maven-shared-utils/0.6/maven-shared-utils-0.6.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/shared/maven-shared-utils/0.6/maven-shared-utils-0.6.pom (5 KB at 7.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sink-api/1.2/doxia-sink-api-1.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sink-api/1.2/doxia-sink-api-1.2.pom (2 KB at 2.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia/1.2/doxia-1.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia/1.2/doxia-1.2.pom (19 KB at 27.9 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/maven-parent/19/maven-parent-19.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/maven-parent/19/maven-parent-19.pom (25 KB at 36.8 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-logging-api/1.2/doxia-logging-api-1.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-logging-api/1.2/doxia-logging-api-1.2.pom (2 KB at 2.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-core/1.2/doxia-core-1.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-core/1.2/doxia-core-1.2.pom (4 KB at 5.9 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpclient/4.0.2/httpclient-4.0.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpclient/4.0.2/httpclient-4.0.2.pom (8 KB at 11.2 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpcomponents-client/4.0.2/httpcomponents-client-4.0.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpcomponents-client/4.0.2/httpcomponents-client-4.0.2.pom (9 KB at 13.4 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/project/4.1/project-4.1.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/project/4.1/project-4.1.pom (16 KB at 24.4 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.pom (5 KB at 7.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpcomponents-core/4.0.1/httpcomponents-core-4.0.1.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpcomponents-core/4.0.1/httpcomponents-core-4.0.1.pom (10 KB at 14.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/project/4.0/project-4.0.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/project/4.0/project-4.0.pom (13 KB at 19.6 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-site-renderer/1.2/doxia-site-renderer-1.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-site-renderer/1.2/doxia-site-renderer-1.2.pom (7 KB at 9.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sitetools/1.2/doxia-sitetools-1.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sitetools/1.2/doxia-sitetools-1.2.pom (16 KB at 24.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-decoration-model/1.2/doxia-decoration-model-1.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-decoration-model/1.2/doxia-decoration-model-1.2.pom (3 KB at 4.6 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-xhtml/1.2/doxia-module-xhtml-1.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-xhtml/1.2/doxia-module-xhtml-1.2.pom (2 KB at 2.6 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-modules/1.2/doxia-modules-1.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-modules/1.2/doxia-modules-1.2.pom (3 KB at 3.8 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-fml/1.2/doxia-module-fml-1.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-fml/1.2/doxia-module-fml-1.2.pom (6 KB at 8.4 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-validator/commons-validator/1.3.1/commons-validator-1.3.1.pom + Downloaded: https://repo.maven.apache.org/maven2/commons-validator/commons-validator/1.3.1/commons-validator-1.3.1.pom (9 KB at 13.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-digester/commons-digester/1.6/commons-digester-1.6.pom + Downloaded: https://repo.maven.apache.org/maven2/commons-digester/commons-digester/1.6/commons-digester-1.6.pom (974 B at 1.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-beanutils/commons-beanutils/1.6/commons-beanutils-1.6.pom + Downloaded: https://repo.maven.apache.org/maven2/commons-beanutils/commons-beanutils/1.6/commons-beanutils-1.6.pom (3 KB at 3.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-logging/commons-logging/1.0/commons-logging-1.0.pom + Downloaded: https://repo.maven.apache.org/maven2/commons-logging/commons-logging/1.0/commons-logging-1.0.pom (163 B at 0.2 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-collections/commons-collections/2.0/commons-collections-2.0.pom + Downloaded: https://repo.maven.apache.org/maven2/commons-collections/commons-collections/2.0/commons-collections-2.0.pom (171 B at 0.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-collections/commons-collections/2.1/commons-collections-2.1.pom + Downloaded: https://repo.maven.apache.org/maven2/commons-collections/commons-collections/2.1/commons-collections-2.1.pom (4 KB at 5.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sink-api/1.4/doxia-sink-api-1.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sink-api/1.4/doxia-sink-api-1.4.pom (2 KB at 2.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia/1.4/doxia-1.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia/1.4/doxia-1.4.pom (18 KB at 26.8 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-logging-api/1.4/doxia-logging-api-1.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-logging-api/1.4/doxia-logging-api-1.4.pom (2 KB at 2.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-decoration-model/1.4/doxia-decoration-model-1.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-decoration-model/1.4/doxia-decoration-model-1.4.pom (3 KB at 4.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sitetools/1.4/doxia-sitetools-1.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sitetools/1.4/doxia-sitetools-1.4.pom (17 KB at 25.6 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-site-renderer/1.4/doxia-site-renderer-1.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-site-renderer/1.4/doxia-site-renderer-1.4.pom (6 KB at 8.7 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-core/1.4/doxia-core-1.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-core/1.4/doxia-core-1.4.pom (4 KB at 6.1 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-xhtml/1.4/doxia-module-xhtml-1.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-xhtml/1.4/doxia-module-xhtml-1.4.pom (2 KB at 1.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-modules/1.4/doxia-modules-1.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-modules/1.4/doxia-modules-1.4.pom (3 KB at 3.9 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-fml/1.4/doxia-module-fml-1.4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-fml/1.4/doxia-module-fml-1.4.pom (5 KB at 7.2 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/velocity/velocity-tools/2.0/velocity-tools-2.0.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/velocity/velocity-tools/2.0/velocity-tools-2.0.pom (18 KB at 27.2 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/javax/servlet/servlet-api/2.3/servlet-api-2.3.pom + Downloaded: https://repo.maven.apache.org/maven2/javax/servlet/servlet-api/2.3/servlet-api-2.3.pom (156 B at 0.2 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-chain/commons-chain/1.1/commons-chain-1.1.pom + Downloaded: https://repo.maven.apache.org/maven2/commons-chain/commons-chain/1.1/commons-chain-1.1.pom (6 KB at 9.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/dom4j/dom4j/1.1/dom4j-1.1.pom + Downloaded: https://repo.maven.apache.org/maven2/dom4j/dom4j/1.1/dom4j-1.1.pom (142 B at 0.2 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/sslext/sslext/1.2-0/sslext-1.2-0.pom + Downloaded: https://repo.maven.apache.org/maven2/sslext/sslext/1.2-0/sslext-1.2-0.pom (653 B at 1.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/struts/struts-core/1.3.8/struts-core-1.3.8.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/struts/struts-core/1.3.8/struts-core-1.3.8.pom (5 KB at 6.4 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/struts/struts-parent/1.3.8/struts-parent-1.3.8.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/struts/struts-parent/1.3.8/struts-parent-1.3.8.pom (10 KB at 14.7 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/struts/struts-master/4/struts-master-4.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/struts/struts-master/4/struts-master-4.pom (12 KB at 17.7 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/apache/2/apache-2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/apache/2/apache-2.pom (4 KB at 5.1 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/antlr/antlr/2.7.2/antlr-2.7.2.pom + Downloaded: https://repo.maven.apache.org/maven2/antlr/antlr/2.7.2/antlr-2.7.2.pom (145 B at 0.2 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/struts/struts-taglib/1.3.8/struts-taglib-1.3.8.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/struts/struts-taglib/1.3.8/struts-taglib-1.3.8.pom (4 KB at 4.6 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/struts/struts-tiles/1.3.8/struts-tiles-1.3.8.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/struts/struts-tiles/1.3.8/struts-tiles-1.3.8.pom (3 KB at 1.4 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/velocity/velocity/1.6.2/velocity-1.6.2.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/velocity/velocity/1.6.2/velocity-1.6.2.pom (11 KB at 10.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-integration-tools/1.6/doxia-integration-tools-1.6.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-integration-tools/1.6/doxia-integration-tools-1.6.pom (7 KB at 10.4 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-tools/3/doxia-tools-3.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-tools/3/doxia-tools-3.pom (10 KB at 14.8 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-logging-api/1.6/doxia-logging-api-1.6.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-logging-api/1.6/doxia-logging-api-1.6.pom (2 KB at 2.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia/1.6/doxia-1.6.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia/1.6/doxia-1.6.pom (19 KB at 19.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-decoration-model/1.6/doxia-decoration-model-1.6.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-decoration-model/1.6/doxia-decoration-model-1.6.pom (3 KB at 4.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sitetools/1.6/doxia-sitetools-1.6.pom + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sitetools/1.6/doxia-sitetools-1.6.pom (18 KB at 17.9 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-resources/1.0-alpha-7/plexus-resources-1.0-alpha-7.pom + Downloaded: https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-resources/1.0-alpha-7/plexus-resources-1.0-alpha-7.pom (2 KB at 2.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/com/puppycrawl/tools/checkstyle/5.9/checkstyle-5.9.jar + Downloading: https://repo.maven.apache.org/maven2/commons-beanutils/commons-beanutils-core/1.8.3/commons-beanutils-core-1.8.3.jar + Downloading: https://repo.maven.apache.org/maven2/com/google/guava/guava-jdk5/14.0.1/guava-jdk5-14.0.1.jar + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/reporting/maven-reporting-impl/2.3/maven-reporting-impl-2.3.jar + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/shared/maven-shared-utils/0.6/maven-shared-utils-0.6.jar + Downloaded: https://repo.maven.apache.org/maven2/com/puppycrawl/tools/checkstyle/5.9/checkstyle-5.9.jar (705 KB at 47.9 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-core/1.2/doxia-core-1.2.jar + May 05, 2016 2:12:19 PM org.apache.maven.wagon.providers.http.httpclient.impl.execchain.RetryExec execute + INFO: I/O exception (java.net.SocketException) caught when processing request to {s}->https://repo.maven.apache.org:443: Connection reset + May 05, 2016 2:12:19 PM org.apache.maven.wagon.providers.http.httpclient.impl.execchain.RetryExec execute + INFO: Retrying request to {s}->https://repo.maven.apache.org:443 + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-core/1.2/doxia-core-1.2.jar (151 KB at 7.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpclient/4.0.2/httpclient-4.0.2.jar + May 05, 2016 2:12:23 PM org.apache.maven.wagon.providers.http.httpclient.impl.execchain.RetryExec execute + INFO: I/O exception (java.net.SocketException) caught when processing request to {s}->https://repo.maven.apache.org:443: Connection reset + May 05, 2016 2:12:23 PM org.apache.maven.wagon.providers.http.httpclient.impl.execchain.RetryExec execute + INFO: Retrying request to {s}->https://repo.maven.apache.org:443 + Downloaded: https://repo.maven.apache.org/maven2/commons-beanutils/commons-beanutils-core/1.8.3/commons-beanutils-core-1.8.3.jar (202 KB at 7.2 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpclient/4.0.2/httpclient-4.0.2.jar (287 KB at 10.1 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-validator/commons-validator/1.3.1/commons-validator-1.3.1.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/shared/maven-shared-utils/0.6/maven-shared-utils-0.6.jar (161 KB at 5.4 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-digester/commons-digester/1.6/commons-digester-1.6.jar + Downloaded: https://repo.maven.apache.org/maven2/com/google/guava/guava-jdk5/14.0.1/guava-jdk5-14.0.1.jar (1957 KB at 59.7 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sink-api/1.4/doxia-sink-api-1.4.jar + Downloaded: https://repo.maven.apache.org/maven2/commons-digester/commons-digester/1.6/commons-digester-1.6.jar (165 KB at 5.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-logging-api/1.4/doxia-logging-api-1.4.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpcore/4.0.1/httpcore-4.0.1.jar (169 KB at 5.1 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-decoration-model/1.4/doxia-decoration-model-1.4.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-sink-api/1.4/doxia-sink-api-1.4.jar (11 KB at 0.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-site-renderer/1.4/doxia-site-renderer-1.4.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-logging-api/1.4/doxia-logging-api-1.4.jar (12 KB at 0.3 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-xhtml/1.4/doxia-module-xhtml-1.4.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-xhtml/1.4/doxia-module-xhtml-1.4.jar (16 KB at 0.4 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-fml/1.4/doxia-module-fml-1.4.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-site-renderer/1.4/doxia-site-renderer-1.4.jar (52 KB at 1.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/velocity/velocity-tools/2.0/velocity-tools-2.0.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-module-fml/1.4/doxia-module-fml-1.4.jar (37 KB at 1.1 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/commons-chain/commons-chain/1.1/commons-chain-1.1.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-decoration-model/1.4/doxia-decoration-model-1.4.jar (60 KB at 1.7 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/dom4j/dom4j/1.1/dom4j-1.1.jar + Downloaded: https://repo.maven.apache.org/maven2/commons-chain/commons-chain/1.1/commons-chain-1.1.jar (88 KB at 2.4 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/sslext/sslext/1.2-0/sslext-1.2-0.jar + Downloaded: https://repo.maven.apache.org/maven2/sslext/sslext/1.2-0/sslext-1.2-0.jar (26 KB at 0.7 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/struts/struts-core/1.3.8/struts-core-1.3.8.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/velocity/velocity-tools/2.0/velocity-tools-2.0.jar (339 KB at 8.9 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/struts/struts-taglib/1.3.8/struts-taglib-1.3.8.jar + Downloaded: https://repo.maven.apache.org/maven2/commons-validator/commons-validator/1.3.1/commons-validator-1.3.1.jar (136 KB at 3.5 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/struts/struts-tiles/1.3.8/struts-tiles-1.3.8.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/struts/struts-core/1.3.8/struts-core-1.3.8.jar (322 KB at 8.0 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-integration-tools/1.6/doxia-integration-tools-1.6.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/struts/struts-taglib/1.3.8/struts-taglib-1.3.8.jar (246 KB at 6.1 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-container-default/1.0-alpha-9/plexus-container-default-1.0-alpha-9.jar + Downloaded: https://repo.maven.apache.org/maven2/org/apache/maven/doxia/doxia-integration-tools/1.6/doxia-integration-tools-1.6.jar (44 KB at 1.1 KB/sec) + Downloading: https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-resources/1.0-alpha-7/plexus-resources-1.0-alpha-7.jar + Downloaded: https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-resources/1.0-alpha-7/plexus-resources-1.0-alpha-7.jar (23 KB at 0.5 KB/sec) + Downloaded: https://repo.maven.apache.org/maven2/org/codehaus/plexus/plexus-container-default/1.0-alpha-9/plexus-container-default-1.0-alpha-9.jar (191 KB at 4.5 KB/sec) + Downloaded: https://repo.maven.apache.org/maven2/org/apache/struts/struts-tiles/1.3.8/struts-tiles-1.3.8.jar (117 KB at 2.5 KB/sec) + Downloaded: https://repo.maven.apache.org/maven2/dom4j/dom4j/1.1/dom4j-1.1.jar (447 KB at 8.5 KB/sec) + [INFO] ------------------------------------------------------------------------ + [INFO] Reactor Summary: + [INFO] + [INFO] Tachyon Parent ..................................... FAILURE [31:36 min] + [INFO] Tachyon Common ..................................... SKIPPED + [INFO] Tachyon Under File System .......................... SKIPPED + [INFO] Tachyon Under File System - Local FS ............... SKIPPED + [INFO] Tachyon Under File System - HDFS ................... SKIPPED + [INFO] Tachyon Under File System - Gluster FS ............. SKIPPED + [INFO] Tachyon Under File System - Swift .................. SKIPPED + [INFO] Tachyon Under File System - S3 ..................... SKIPPED + [INFO] Tachyon Clients .................................... SKIPPED + [INFO] Tachyon Clients - Implementation ................... SKIPPED + [INFO] Tachyon Clients - Distribution ..................... SKIPPED + [INFO] Tachyon Servers .................................... SKIPPED + [INFO] Mock Tachyon Cluster ............................... SKIPPED + [INFO] Tachyon Integration Tests .......................... SKIPPED + [INFO] Tachyon Shell ...................................... SKIPPED + [INFO] Tachyon Examples ................................... SKIPPED + [INFO] Tachyon Assemblies ................................. SKIPPED + [INFO] ------------------------------------------------------------------------ + [INFO] BUILD FAILURE + [INFO] ------------------------------------------------------------------------ + [INFO] Total time: 31:36 min + [INFO] Finished at: 2016-05-05T14:42:18+08:00 + [INFO] Final Memory: 16M/170M + [INFO] ------------------------------------------------------------------------ + [ERROR] Failed to execute goal org.apache.maven.plugins:maven-checkstyle-plugin:2.13:check (checkstyle) on project tachyon-parent: Execution checkstyle of goal org.apache.maven.plugins:maven-checkstyle-plugin:2.13:check failed: Plugin org.apache.maven.plugins:maven-checkstyle-plugin:2.13 or one of its dependencies could not be resolved: Could not transfer artifact org.apache.maven.reporting:maven-reporting-impl:jar:2.3 from/to central (https://repo.maven.apache.org/maven2): Read timed out -> [Help 1] + [ERROR] + [ERROR] To see the full stack trace of the errors, re-run Maven with the -e switch. + [ERROR] Re-run Maven using the -X switch to enable full debug logging. + [ERROR] + [ERROR] For more information about the errors and possible solutions, please read the following articles: + [ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/PluginResolutionException + xubo@xubo:~/cloud/tachyon-0.7.1$ mvn clean + [INFO] Scanning for projects... + [INFO] ------------------------------------------------------------------------ + [INFO] Reactor Build Order: + [INFO] + [INFO] Tachyon Parent + [INFO] Tachyon Common + [INFO] Tachyon Under File System + [INFO] Tachyon Under File System - Local FS + [INFO] Tachyon Under File System - HDFS + [INFO] Tachyon Under File System - Gluster FS + [INFO] Tachyon Under File System - Swift + [INFO] Tachyon Under File System - S3 + [INFO] Tachyon Clients + [INFO] Tachyon Clients - Implementation + [INFO] Tachyon Clients - Distribution + [INFO] Tachyon Servers + [INFO] Mock Tachyon Cluster + [INFO] Tachyon Integration Tests + [INFO] Tachyon Shell + [INFO] Tachyon Examples + [INFO] Tachyon Assemblies + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Parent 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-parent --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Common 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-common --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Under File System 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-underfs --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Under File System - Local FS 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-underfs-local --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Under File System - HDFS 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-underfs-hdfs --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Under File System - Gluster FS 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-underfs-glusterfs --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Under File System - Swift 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-underfs-swift --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Under File System - S3 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-underfs-s3 --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Clients 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-clients --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Clients - Implementation 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-client-unshaded --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Clients - Distribution 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-client --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Servers 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-servers --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Mock Tachyon Cluster 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-minicluster --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Integration Tests 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-integration-tests --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Shell 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-shell --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Examples 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-examples --- + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Assemblies 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-assemblies --- + [INFO] ------------------------------------------------------------------------ + [INFO] Reactor Summary: + [INFO] + [INFO] Tachyon Parent ..................................... SUCCESS [ 0.564 s] + [INFO] Tachyon Common ..................................... SUCCESS [ 0.004 s] + [INFO] Tachyon Under File System .......................... SUCCESS [ 0.003 s] + [INFO] Tachyon Under File System - Local FS ............... SUCCESS [ 0.003 s] + [INFO] Tachyon Under File System - HDFS ................... SUCCESS [ 0.003 s] + [INFO] Tachyon Under File System - Gluster FS ............. SUCCESS [ 0.004 s] + [INFO] Tachyon Under File System - Swift .................. SUCCESS [ 0.003 s] + [INFO] Tachyon Under File System - S3 ..................... SUCCESS [ 0.003 s] + [INFO] Tachyon Clients .................................... SUCCESS [ 0.003 s] + [INFO] Tachyon Clients - Implementation ................... SUCCESS [ 0.003 s] + [INFO] Tachyon Clients - Distribution ..................... SUCCESS [ 0.004 s] + [INFO] Tachyon Servers .................................... SUCCESS [ 0.059 s] + [INFO] Mock Tachyon Cluster ............................... SUCCESS [ 0.012 s] + [INFO] Tachyon Integration Tests .......................... SUCCESS [ 0.004 s] + [INFO] Tachyon Shell ...................................... SUCCESS [ 0.004 s] + [INFO] Tachyon Examples ................................... SUCCESS [ 0.013 s] + [INFO] Tachyon Assemblies ................................. SUCCESS [ 0.004 s] + [INFO] ------------------------------------------------------------------------ + [INFO] BUILD SUCCESS + [INFO] ------------------------------------------------------------------------ + [INFO] Total time: 1.234 s + [INFO] Finished at: 2016-05-05T14:45:57+08:00 + [INFO] Final Memory: 11M/104M + [INFO] ------------------------------------------------------------------------ + + +参考： +【1】http://blog.csdn.net/stark_summer/article/details/48321605 + diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2405--tachyon\347\232\204\345\207\240\344\270\252\351\227\256\351\242\230\357\274\210\345\276\205\350\247\243\345\206\263\357\274\211.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2405--tachyon\347\232\204\345\207\240\344\270\252\351\227\256\351\242\230\357\274\210\345\276\205\350\247\243\345\206\263\357\274\211.md" new file mode 100644 index 0000000..1c4ae18 --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2405--tachyon\347\232\204\345\207\240\344\270\252\351\227\256\351\242\230\357\274\210\345\276\205\350\247\243\345\206\263\357\274\211.md" @@ -0,0 +1,12 @@ + +1.tachyon是否会占用额外的空间？ +当tachyon将数据存储在内存中时，spark读取的时候是copy还是使用lineage？ + + +2.在运行spark的时候persist可以制定off-heap，而这种默认使用tachyon，这种模式与使用saveAsFile等指令存储到tachyon有什么区别？ + +3.当tachyon内存存储到一定值时tachyon将文件存储到hdfs或者disk的具体实现机制是什么样的？ + +4.tachyon分布式如何实现？ + +5.tachyon域redis性能差别？ \ No newline at end of file diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2406---\351\233\206\347\276\244\347\211\210\346\220\255\345\273\272\345\222\214\350\277\220\350\241\214\357\274\210Tachyon\357\274\211.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2406---\351\233\206\347\276\244\347\211\210\346\220\255\345\273\272\345\222\214\350\277\220\350\241\214\357\274\210Tachyon\357\274\211.md" new file mode 100644 index 0000000..99bef84 --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2406---\351\233\206\347\276\244\347\211\210\346\220\255\345\273\272\345\222\214\350\277\220\350\241\214\357\274\210Tachyon\357\274\211.md" @@ -0,0 +1,103 @@ +更多代码请见：https://github.com/xubo245/SparkLearning +环境： +hadoop-2.6.0 +spark-1.5.2 +scala-1.7 + + +1解释 +完成tachyon的集群配置与运行 +前提：安装好java、hdfs等 + +2.下载 + + https://github.com/Alluxio/alluxio/releases +具体的： + + https://github.com/Alluxio/alluxio/releases/download/v0.7.1/tachyon-0.7.1-hadoop2.6-bin.tar.gz + +如果下载tachyon-0.7.1-bin.tar.g会出现版本不兼容，会报错： + + Server IPC version 9 cannot communicate with client version 4 + +3.配置：【3】讲的比较详细 +在conf/tachyon-env.sh配置： + + export TACHYON_MASTER_ADDRESS=Master + #export TACHYON_UNDERFS_ADDRESS=$TACHYON_HOME/underFSStorage + export TACHYON_UNDERFS_ADDRESS=hdfs://Master:9000 + export TACHYON_WORKER_MEMORY_SIZE=256MB + +Master为集群master IP，或者自己在本地设置好映射 + +4.安装： + + $ cd tachyon + $ ./bin/tachyon format + $ ./bin/tachyon-start.sh # use the right parameters here. e.g. all Mount +一般start需要sudo才能mount： + + ./bin/tachyon-start.sh all SudoMount + +每个节点都要输入密码，好麻烦 + +5.查看： +在Master的19999端口可以查看，六个worker启动了三个， +解决办法：http://blog.csdn.net/xubo245/article/details/53056482 + +6.使用： +上传： + + hadoop@Master:~/cloud/tachyon-0.7.1-hadoop2.6-bin$ ./bin/tachyon tfs copyFromLocal test/2.txt / + Copied test/2.txt to / + +展示没问题 + + +7.记录 +ipc版本不同： + + hadoop@Master:~/cloud/tachyon-0.7.1$ ./bin/tachyon format + Connecting to Mcnode1 as hadoop... + Formatting Tachyon Worker @ Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Formatting Tachyon Worker @ Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode3 + /home/hadoop/cloud/tachyon-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode4 + /home/hadoop/cloud/tachyon-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode5 + /home/hadoop/cloud/tachyon-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Formatting Tachyon Worker @ Mcnode6 + Connection to mcnode6 closed. + Formatting Tachyon Master @ localhost + Exception in thread "main" java.lang.IllegalArgumentException: All eligible Under File Systems were unable to create an instance for the given path: hdfs://Master:9000/tmp/tachyon/data + java.lang.RuntimeException: org.apache.hadoop.ipc.RemoteException: Server IPC version 9 cannot communicate with client version 4 + + at tachyon.underfs.UnderFileSystemRegistry.create(UnderFileSystemRegistry.java:132) + at tachyon.underfs.UnderFileSystem.get(UnderFileSystem.java:99) + at tachyon.underfs.UnderFileSystem.get(UnderFileSystem.java:83) + at tachyon.Format.formatFolder(Format.java:38) + at tachyon.Format.main(Format.java:77) + + + +参考 +【1】http://alluxio.org/documentation/v0.7.1/ +【2】https://github.com/xubo245/SparkLearning +【3】 http://alluxio.org/documentation/v0.7.1/Running-Tachyon-on-a-Cluster.html diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2407--\344\270\213\350\275\275\346\272\220\347\240\201\351\200\232\350\277\207maven\345\256\211\350\243\205\346\210\220\345\212\237.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2407--\344\270\213\350\275\275\346\272\220\347\240\201\351\200\232\350\277\207maven\345\256\211\350\243\205\346\210\220\345\212\237.md" new file mode 100644 index 0000000..6dd6a45 --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2407--\344\270\213\350\275\275\346\272\220\347\240\201\351\200\232\350\277\207maven\345\256\211\350\243\205\346\210\220\345\212\237.md" @@ -0,0 +1,1108 @@ +更多代码请见：https://github.com/xubo245/SparkLearning + +1.环境 + + hadoop 2.6.0 + spark 1.5.2 + java 1.7 + +2.下载： + + https://github.com/Alluxio/alluxio/archive/v0.7.1.tar.gz + +3.编译： + + mvn clean package -Djava.version=1.7 -Dhadoop.version=2.6.0 -Dspark.version=1.5.2 -DskipTests + +4.安装成功： + + [INFO] ------------------------------------------------------------------------ + [INFO] Reactor Summary: + [INFO] + [INFO] Tachyon Parent ..................................... SUCCESS [03:15 min] + [INFO] Tachyon Common ..................................... SUCCESS [01:23 min] + [INFO] Tachyon Under File System .......................... SUCCESS [ 0.609 s] + [INFO] Tachyon Under File System - Local FS ............... SUCCESS [ 32.970 s] + [INFO] Tachyon Under File System - HDFS ................... SUCCESS [ 5.169 s] + [INFO] Tachyon Under File System - Gluster FS ............. SUCCESS [01:43 min] + [INFO] Tachyon Under File System - Swift .................. SUCCESS [01:04 min] + [INFO] Tachyon Under File System - S3 ..................... SUCCESS [ 20.526 s] + [INFO] Tachyon Clients .................................... SUCCESS [ 0.523 s] + [INFO] Tachyon Clients - Implementation ................... SUCCESS [ 9.195 s] + [INFO] Tachyon Clients - Distribution ..................... SUCCESS [ 30.238 s] + [INFO] Tachyon Servers .................................... SUCCESS [ 50.587 s] + [INFO] Mock Tachyon Cluster ............................... SUCCESS [ 5.288 s] + [INFO] Tachyon Integration Tests .......................... SUCCESS [01:11 min] + [INFO] Tachyon Shell ...................................... SUCCESS [ 3.214 s] + [INFO] Tachyon Examples ................................... SUCCESS [ 3.489 s] + [INFO] Tachyon Assemblies ................................. SUCCESS [ 11.906 s] + [INFO] ------------------------------------------------------------------------ + [INFO] BUILD SUCCESS + [INFO] ------------------------------------------------------------------------ + [INFO] Total time: 11:32 min + [INFO] Finished at: 2016-05-05T19:29:29+08:00 + [INFO] Final Memory: 107M/642M + [INFO] ------------------------------------------------------------------------ + + + +5.安装记录： + + Constructing Javadoc information... + Standard Doclet version 1.7.0_79 + Building tree for all the packages and classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/MetricsConfig.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/MetricsSystem.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/ConsoleSink.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/CsvSink.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/GraphiteSink.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/JmxSink.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/MetricsServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/Sink.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/source/JvmSource.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/source/Source.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/Format.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/ServerConstants.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/StorageDirId.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/StorageLevelAlias.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/UserInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/Users.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/ValidateConf.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/DataServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/DataServer.Factory.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/SpaceCounter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/TachyonWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/WorkerSource.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/DataServerHandler.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/FileTransferType.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/NettyDataServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/PipelineHandler.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockDataManager.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockHeartbeatReport.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockHeartbeatReporter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockLockManager.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockLockType.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockMasterSync.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockMetadataManager.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockMetadataManagerView.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockMetricsReporter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockServiceHandler.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockStoreEventListener.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockStoreEventListenerBase.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockStoreLocation.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockStoreMeta.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/BlockWorkerUtils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/ClientRWLock.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/PinListSync.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/TieredBlockStore.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/BlockReader.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/BlockWriter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/LocalFileBlockReader.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/LocalFileBlockWriter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/BlockMeta.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/BlockMetaBase.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/StorageDir.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/StorageDirView.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/StorageTier.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/StorageTierView.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/TempBlockMeta.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/Allocator.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/AllocatorFactory.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/AllocatorType.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/GreedyAllocator.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/MaxFreeAllocator.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/EvictionPlan.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/Evictor.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/EvictorFactory.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/EvictorType.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/EvictorUtils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/GreedyEvictor.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/LRUEvictor.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/nio/NIODataServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/ByteBufferReader.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/ByteBufferWriter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/JavaByteBufferReader.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/JavaByteBufferWriter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/Utils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/BlockInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/Counters.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/Dependency.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/DependencyType.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/DependencyVariables.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/EditLog.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/EditLogProcessor.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/Image.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/ImageWriter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/Inode.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/InodeFile.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/InodeFolder.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/Journal.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/MasterInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/MasterInfo.MasterInfoHeartbeatExecutor.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/MasterInfo.RecomputationScheduler.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/MasterServiceHandler.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/MasterSource.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/MasterWorkerInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/RawTables.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/RecomputeCommand.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/TachyonMaster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/MasterUIWebServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/UiBlockInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/UiFileInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/UiFileInfo.LocalFileInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/UIWebServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/Utils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceBrowseLogsServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceBrowseServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceConfigurationServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceDependencyServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceDownloadLocalServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceDownloadServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceGeneralServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceGeneralServlet.StorageTierInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceHeaderServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceMemoryServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceWorkerBlockInfoServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceWorkerGeneralServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceWorkerGeneralServlet.UiStorageDir.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceWorkersServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WebInterfaceWorkersServlet.NodeInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/WorkerUIWebServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/overview-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/source/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/source/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/source/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/nio/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/nio/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/nio/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/constant-values.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/serialized-form.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/class-use/MetricsSystem.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/class-use/MetricsConfig.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/class-use/GraphiteSink.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/class-use/MetricsServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/class-use/JmxSink.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/class-use/CsvSink.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/class-use/ConsoleSink.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/class-use/Sink.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/source/class-use/Source.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/source/class-use/JvmSource.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/class-use/UserInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/class-use/ServerConstants.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/class-use/StorageDirId.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/class-use/ValidateConf.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/class-use/Users.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/class-use/StorageLevelAlias.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/class-use/Format.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/class-use/SpaceCounter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/class-use/WorkerSource.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/class-use/TachyonWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/class-use/DataServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/class-use/DataServer.Factory.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/class-use/FileTransferType.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/class-use/PipelineHandler.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/class-use/NettyDataServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/class-use/DataServerHandler.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/TieredBlockStore.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockStoreEventListenerBase.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/ClientRWLock.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/PinListSync.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockMetadataManager.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockStoreLocation.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockHeartbeatReporter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockHeartbeatReport.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockServiceHandler.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockMetadataManagerView.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockWorkerUtils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockLockType.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockStoreEventListener.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockLockManager.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockStoreMeta.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockMasterSync.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockMetricsReporter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/class-use/BlockDataManager.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/class-use/BlockWriter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/class-use/LocalFileBlockReader.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/class-use/LocalFileBlockWriter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/class-use/BlockReader.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/class-use/BlockMeta.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/class-use/StorageTier.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/class-use/StorageDirView.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/class-use/BlockMetaBase.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/class-use/StorageTierView.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/class-use/TempBlockMeta.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/class-use/StorageDir.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/class-use/AllocatorFactory.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/class-use/GreedyAllocator.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/class-use/MaxFreeAllocator.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/class-use/Allocator.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/class-use/AllocatorType.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/class-use/EvictorUtils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/class-use/EvictorFactory.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/class-use/GreedyEvictor.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/class-use/EvictionPlan.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/class-use/LRUEvictor.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/class-use/EvictorType.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/class-use/Evictor.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/nio/class-use/NIODataServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/class-use/ByteBufferReader.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/class-use/JavaByteBufferWriter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/class-use/Utils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/class-use/ByteBufferWriter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/class-use/JavaByteBufferReader.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/Counters.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/EditLogProcessor.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/Image.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/MasterWorkerInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/DependencyVariables.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/RawTables.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/MasterInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/MasterInfo.MasterInfoHeartbeatExecutor.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/MasterInfo.RecomputationScheduler.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/EditLog.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/Journal.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/InodeFolder.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/DependencyType.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/Inode.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/RecomputeCommand.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/TachyonMaster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/InodeFile.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/MasterServiceHandler.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/MasterSource.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/Dependency.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/ImageWriter.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/class-use/BlockInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceGeneralServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceGeneralServlet.StorageTierInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceWorkerGeneralServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceWorkerGeneralServlet.UiStorageDir.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/MasterUIWebServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceDependencyServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WorkerUIWebServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/Utils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceWorkersServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceWorkersServlet.NodeInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/UiFileInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/UiFileInfo.LocalFileInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/UiBlockInfo.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceDownloadLocalServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceWorkerBlockInfoServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceDownloadServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceHeaderServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceConfigurationServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/UIWebServer.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceBrowseServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceMemoryServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/class-use/WebInterfaceBrowseLogsServlet.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/io/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/master/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/sink/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/metrics/source/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/web/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/allocator/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/evictor/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/io/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/block/meta/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/netty/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/tachyon/worker/nio/package-use.html... + Building index for all the packages and classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/overview-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/index-all.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/deprecated-list.html... + Building index for all classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/allclasses-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/allclasses-noframe.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/index.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/overview-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/servers/target/apidocs/help-doc.html... + 16 warnings + [WARNING] Javadoc Warnings + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/BlockDataManager.java:191: warning - @return tag cannot be used in method with void return type. + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/BlockDataManager.java:476: warning - @param argument "pinnedInodes," is not a parameter name. + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/BlockMetadataManagerView.java:63: warning - Tag @return cannot be used in constructor documentation. It can only be used in the following types of documentation: method. + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/BlockMetadataManagerView.java:63: warning - @param argument "pinnedInodes," is not a parameter name. + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/BlockMetadataManagerView.java:63: warning - @param argument "lockedBlocks," is not a parameter name. + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/PinListSync.java:65: warning - Tag @return cannot be used in constructor documentation. It can only be used in the following types of documentation: method. + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/TieredBlockStore.java:525: warning - @param argument "inodes," is not a parameter name. + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/meta/StorageDirView.java:47: warning - Tag @return cannot be used in constructor documentation. It can only be used in the following types of documentation: method. + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/meta/StorageTierView.java:45: warning - @param argument "view," is not a parameter name. + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/meta/StorageTierView.java:45: warning - @param argument "StorageTierView" is not a parameter name. + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/allocator/Allocator.java:42: warning - Tag @link: reference not found: BlockStore + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/allocator/Allocator.java:42: warning - Tag @link: reference not found: BlockStore + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/allocator/Allocator.java:42: warning - Tag @link: reference not found: BlockStore + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/evictor/Evictor.java:47: warning - Tag @link: reference not found: BlockStore + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/evictor/Evictor.java:47: warning - Tag @link: reference not found: BlockStore + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/servers/src/main/java/tachyon/worker/block/evictor/Evictor.java:47: warning - Tag @link: reference not found: BlockStore + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/servers/target/tachyon-servers-0.7.1-javadoc.jar + [INFO] + [INFO] --- maven-jar-plugin:2.4:test-jar (default) @ tachyon-servers --- + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/servers/target/tachyon-servers-0.7.1-tests.jar + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Mock Tachyon Cluster 0.7.1 + [INFO] ------------------------------------------------------------------------ + Downloading: https://repo1.maven.org/maven2/org/apache/curator/curator-test/2.1.0-incubating/curator-test-2.1.0-incubating.pom + Downloaded: https://repo1.maven.org/maven2/org/apache/curator/curator-test/2.1.0-incubating/curator-test-2.1.0-incubating.pom (2 KB at 4.3 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/curator/curator-test/2.1.0-incubating/curator-test-2.1.0-incubating.jar + Downloaded: https://repo1.maven.org/maven2/org/apache/curator/curator-test/2.1.0-incubating/curator-test-2.1.0-incubating.jar (35 KB at 26.6 KB/sec) + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-minicluster --- + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-minicluster --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-minicluster --- + [INFO] Starting audit... + Audit done. + + [INFO] + [INFO] --- license-maven-plugin:2.9:check (default) @ tachyon-minicluster --- + [INFO] Checking licenses... + [INFO] + [INFO] --- maven-resources-plugin:2.6:resources (default-resources) @ tachyon-minicluster --- + [INFO] Using 'UTF-8' encoding to copy filtered resources. + [INFO] skip non existing resourceDirectory /home/hadoop/cloud/alluxio-0.7.1/minicluster/src/main/resources + [INFO] + [INFO] --- maven-compiler-plugin:3.2:compile (default-compile) @ tachyon-minicluster --- + [INFO] Changes detected - recompiling the module! + [INFO] Compiling 6 source files to /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/classes + [INFO] + [INFO] --- maven-resources-plugin:2.6:testResources (default-testResources) @ tachyon-minicluster --- + [INFO] Using 'UTF-8' encoding to copy filtered resources. + [INFO] skip non existing resourceDirectory /home/hadoop/cloud/alluxio-0.7.1/minicluster/src/test/resources + [INFO] + [INFO] --- maven-compiler-plugin:3.2:testCompile (default-testCompile) @ tachyon-minicluster --- + [INFO] No sources to compile + [INFO] + [INFO] --- maven-surefire-plugin:2.14:test (default-test) @ tachyon-minicluster --- + [INFO] Tests are skipped. + [INFO] + [INFO] --- maven-jar-plugin:2.4:jar (default-jar) @ tachyon-minicluster --- + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/tachyon-minicluster-0.7.1.jar + [INFO] + [INFO] >>> maven-source-plugin:2.3:jar (attach-sources) > generate-sources @ tachyon-minicluster >>> + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-minicluster --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-minicluster --- + [INFO] Starting audit... + Audit done. + + [INFO] + [INFO] --- license-maven-plugin:2.9:check (default) @ tachyon-minicluster --- + [INFO] Checking licenses... + [INFO] + [INFO] <<< maven-source-plugin:2.3:jar (attach-sources) < generate-sources @ tachyon-minicluster <<< + [INFO] + [INFO] --- maven-source-plugin:2.3:jar (attach-sources) @ tachyon-minicluster --- + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/tachyon-minicluster-0.7.1-sources.jar + [INFO] + [INFO] --- maven-javadoc-plugin:2.9:jar (attach-javadoc) @ tachyon-minicluster --- + [INFO] + Loading source files for package tachyon.underfs... + Loading source files for package tachyon.master... + Loading source files for package tachyon... + Constructing Javadoc information... + Standard Doclet version 1.7.0_79 + Building tree for all the packages and classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/underfs/UnderFileSystemCluster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/ClientPool.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/LocalTachyonCluster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/LocalTachyonClusterMultiMaster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/LocalTachyonMaster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/LocalFilesystemCluster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/overview-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/underfs/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/underfs/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/underfs/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/constant-values.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/underfs/class-use/UnderFileSystemCluster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/class-use/LocalTachyonMaster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/class-use/LocalTachyonClusterMultiMaster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/class-use/ClientPool.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/class-use/LocalTachyonCluster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/class-use/LocalFilesystemCluster.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/master/package-use.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/tachyon/underfs/package-use.html... + Building index for all the packages and classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/overview-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/index-all.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/deprecated-list.html... + Building index for all classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/allclasses-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/allclasses-noframe.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/index.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/overview-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/apidocs/help-doc.html... + 1 warning + [WARNING] Javadoc Warnings + [WARNING] /home/hadoop/cloud/alluxio-0.7.1/minicluster/src/main/java/tachyon/underfs/UnderFileSystemCluster.java:141: warning - @return tag has no arguments. + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/minicluster/target/tachyon-minicluster-0.7.1-javadoc.jar + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Integration Tests 0.7.1 + [INFO] ------------------------------------------------------------------------ + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-minicluster/2.6.0/hadoop-minicluster-2.6.0.pom + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-minicluster/2.6.0/hadoop-minicluster-2.6.0.pom (4 KB at 6.6 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-tests/2.6.0/hadoop-yarn-server-tests-2.6.0.pom + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-tests/2.6.0/hadoop-yarn-server-tests-2.6.0.pom (7 KB at 14.3 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-resourcemanager/2.6.0/hadoop-yarn-server-resourcemanager-2.6.0.pom + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-resourcemanager/2.6.0/hadoop-yarn-server-resourcemanager-2.6.0.pom (13 KB at 10.8 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-applicationhistoryservice/2.6.0/hadoop-yarn-server-applicationhistoryservice-2.6.0.pom + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-applicationhistoryservice/2.6.0/hadoop-yarn-server-applicationhistoryservice-2.6.0.pom (7 KB at 14.6 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-web-proxy/2.6.0/hadoop-yarn-server-web-proxy-2.6.0.pom + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-web-proxy/2.6.0/hadoop-yarn-server-web-proxy-2.6.0.pom (5 KB at 10.5 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-mapreduce-client-hs/2.6.0/hadoop-mapreduce-client-hs-2.6.0.pom + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-mapreduce-client-hs/2.6.0/hadoop-mapreduce-client-hs-2.6.0.pom (4 KB at 3.2 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-minicluster/2.6.0/hadoop-minicluster-2.6.0.jar + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-hdfs/2.6.0/hadoop-hdfs-2.6.0-tests.jar + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-tests/2.6.0/hadoop-yarn-server-tests-2.6.0-tests.jar + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-nodemanager/2.6.0/hadoop-yarn-server-nodemanager-2.6.0.jar + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-resourcemanager/2.6.0/hadoop-yarn-server-resourcemanager-2.6.0.jar + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-minicluster/2.6.0/hadoop-minicluster-2.6.0.jar (3 KB at 4.6 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-applicationhistoryservice/2.6.0/hadoop-yarn-server-applicationhistoryservice-2.6.0.jar + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-tests/2.6.0/hadoop-yarn-server-tests-2.6.0-tests.jar (52 KB at 27.7 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-mapreduce-client-jobclient/2.6.0/hadoop-mapreduce-client-jobclient-2.6.0-tests.jar + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-applicationhistoryservice/2.6.0/hadoop-yarn-server-applicationhistoryservice-2.6.0.jar (162 KB at 33.5 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-web-proxy/2.6.0/hadoop-yarn-server-web-proxy-2.6.0.jar + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-web-proxy/2.6.0/hadoop-yarn-server-web-proxy-2.6.0.jar (28 KB at 4.9 KB/sec) + Downloading: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-mapreduce-client-hs/2.6.0/hadoop-mapreduce-client-hs-2.6.0.jar + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-nodemanager/2.6.0/hadoop-yarn-server-nodemanager-2.6.0.jar (624 KB at 76.2 KB/sec) + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-mapreduce-client-hs/2.6.0/hadoop-mapreduce-client-hs-2.6.0.jar (229 KB at 23.7 KB/sec) + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-resourcemanager/2.6.0/hadoop-yarn-server-resourcemanager-2.6.0.jar (1108 KB at 36.6 KB/sec) + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-mapreduce-client-jobclient/2.6.0/hadoop-mapreduce-client-jobclient-2.6.0-tests.jar (1460 KB at 35.6 KB/sec) + Downloaded: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-hdfs/2.6.0/hadoop-hdfs-2.6.0-tests.jar (2959 KB at 45.4 KB/sec) + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-integration-tests --- + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-integration-tests --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-integration-tests --- + [INFO] Starting audit... + Audit done. + + [INFO] + [INFO] --- license-maven-plugin:2.9:check (default) @ tachyon-integration-tests --- + [INFO] Checking licenses... + [INFO] + [INFO] --- maven-resources-plugin:2.6:resources (default-resources) @ tachyon-integration-tests --- + [INFO] Using 'UTF-8' encoding to copy filtered resources. + [INFO] skip non existing resourceDirectory /home/hadoop/cloud/alluxio-0.7.1/integration-tests/src/main/resources + [INFO] + [INFO] --- maven-compiler-plugin:3.2:compile (default-compile) @ tachyon-integration-tests --- + [INFO] No sources to compile + [INFO] + [INFO] --- maven-resources-plugin:2.6:testResources (default-testResources) @ tachyon-integration-tests --- + [INFO] Using 'UTF-8' encoding to copy filtered resources. + [INFO] Copying 1 resource + [INFO] + [INFO] --- maven-compiler-plugin:3.2:testCompile (default-testCompile) @ tachyon-integration-tests --- + [INFO] Changes detected - recompiling the module! + [INFO] Compiling 37 source files to /home/hadoop/cloud/alluxio-0.7.1/integration-tests/target/test-classes + [INFO] /home/hadoop/cloud/alluxio-0.7.1/integration-tests/src/test/java/tachyon/hadoop/TFSRenameIntegrationTest.java: Some input files use or override a deprecated API. + [INFO] /home/hadoop/cloud/alluxio-0.7.1/integration-tests/src/test/java/tachyon/hadoop/TFSRenameIntegrationTest.java: Recompile with -Xlint:deprecation for details. + [INFO] + [INFO] --- maven-surefire-plugin:2.14:test (default-test) @ tachyon-integration-tests --- + [INFO] Tests are skipped. + [INFO] + [INFO] --- maven-jar-plugin:2.4:jar (default-jar) @ tachyon-integration-tests --- + [WARNING] JAR will be empty - no content was marked for inclusion! + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/integration-tests/target/tachyon-integration-tests-0.7.1.jar + [INFO] + [INFO] >>> maven-source-plugin:2.3:jar (attach-sources) > generate-sources @ tachyon-integration-tests >>> + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-integration-tests --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-integration-tests --- + [INFO] Starting audit... + Audit done. + + [INFO] + [INFO] --- license-maven-plugin:2.9:check (default) @ tachyon-integration-tests --- + [INFO] Checking licenses... + [INFO] + [INFO] <<< maven-source-plugin:2.3:jar (attach-sources) < generate-sources @ tachyon-integration-tests <<< + [INFO] + [INFO] --- maven-source-plugin:2.3:jar (attach-sources) @ tachyon-integration-tests --- + [INFO] No sources in project. Archive not created. + [INFO] + [INFO] --- maven-javadoc-plugin:2.9:jar (attach-javadoc) @ tachyon-integration-tests --- + [INFO] + [INFO] --- maven-jar-plugin:2.4:test-jar (default) @ tachyon-integration-tests --- + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/integration-tests/target/tachyon-integration-tests-0.7.1-tests.jar + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Shell 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-shell --- + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-shell --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-shell --- + [INFO] Starting audit... + Audit done. + + [INFO] + [INFO] --- license-maven-plugin:2.9:check (default) @ tachyon-shell --- + [INFO] Checking licenses... + [INFO] + [INFO] --- maven-resources-plugin:2.6:resources (default-resources) @ tachyon-shell --- + [INFO] Using 'UTF-8' encoding to copy filtered resources. + [INFO] skip non existing resourceDirectory /home/hadoop/cloud/alluxio-0.7.1/shell/src/main/resources + [INFO] + [INFO] --- maven-compiler-plugin:3.2:compile (default-compile) @ tachyon-shell --- + [INFO] Changes detected - recompiling the module! + [INFO] Compiling 2 source files to /home/hadoop/cloud/alluxio-0.7.1/shell/target/classes + [INFO] + [INFO] --- maven-resources-plugin:2.6:testResources (default-testResources) @ tachyon-shell --- + [INFO] Using 'UTF-8' encoding to copy filtered resources. + [INFO] Copying 1 resource + [INFO] + [INFO] --- maven-compiler-plugin:3.2:testCompile (default-testCompile) @ tachyon-shell --- + [INFO] Changes detected - recompiling the module! + [INFO] Compiling 2 source files to /home/hadoop/cloud/alluxio-0.7.1/shell/target/test-classes + [INFO] + [INFO] --- maven-surefire-plugin:2.14:test (default-test) @ tachyon-shell --- + [INFO] Tests are skipped. + [INFO] + [INFO] --- maven-jar-plugin:2.4:jar (default-jar) @ tachyon-shell --- + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/shell/target/tachyon-shell-0.7.1.jar + [INFO] + [INFO] >>> maven-source-plugin:2.3:jar (attach-sources) > generate-sources @ tachyon-shell >>> + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-shell --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-shell --- + [INFO] Starting audit... + Audit done. + + [INFO] + [INFO] --- license-maven-plugin:2.9:check (default) @ tachyon-shell --- + [INFO] Checking licenses... + [INFO] + [INFO] <<< maven-source-plugin:2.3:jar (attach-sources) < generate-sources @ tachyon-shell <<< + [INFO] + [INFO] --- maven-source-plugin:2.3:jar (attach-sources) @ tachyon-shell --- + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/shell/target/tachyon-shell-0.7.1-sources.jar + [INFO] + [INFO] --- maven-javadoc-plugin:2.9:jar (attach-javadoc) @ tachyon-shell --- + [INFO] + Loading source files for package tachyon.shell... + Constructing Javadoc information... + Standard Doclet version 1.7.0_79 + Building tree for all the packages and classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/tachyon/shell/TFsShell.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/tachyon/shell/TFsShellUtils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/tachyon/shell/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/tachyon/shell/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/tachyon/shell/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/constant-values.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/tachyon/shell/class-use/TFsShell.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/tachyon/shell/class-use/TFsShellUtils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/tachyon/shell/package-use.html... + Building index for all the packages and classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/overview-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/index-all.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/deprecated-list.html... + Building index for all classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/allclasses-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/allclasses-noframe.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/index.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/shell/target/apidocs/help-doc.html... + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/shell/target/tachyon-shell-0.7.1-javadoc.jar + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Examples 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-examples --- + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-examples --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-examples --- + [INFO] Starting audit... + Audit done. + + [INFO] + [INFO] --- license-maven-plugin:2.9:check (default) @ tachyon-examples --- + [INFO] Checking licenses... + [INFO] + [INFO] --- maven-resources-plugin:2.6:resources (default-resources) @ tachyon-examples --- + [INFO] Using 'UTF-8' encoding to copy filtered resources. + [INFO] skip non existing resourceDirectory /home/hadoop/cloud/alluxio-0.7.1/examples/src/main/resources + [INFO] + [INFO] --- maven-compiler-plugin:3.2:compile (default-compile) @ tachyon-examples --- + [INFO] Changes detected - recompiling the module! + [INFO] Compiling 6 source files to /home/hadoop/cloud/alluxio-0.7.1/examples/target/classes + [INFO] + [INFO] --- maven-resources-plugin:2.6:testResources (default-testResources) @ tachyon-examples --- + [INFO] Using 'UTF-8' encoding to copy filtered resources. + [INFO] skip non existing resourceDirectory /home/hadoop/cloud/alluxio-0.7.1/examples/src/test/resources + [INFO] + [INFO] --- maven-compiler-plugin:3.2:testCompile (default-testCompile) @ tachyon-examples --- + [INFO] No sources to compile + [INFO] + [INFO] --- maven-surefire-plugin:2.14:test (default-test) @ tachyon-examples --- + [INFO] Tests are skipped. + [INFO] + [INFO] --- maven-jar-plugin:2.4:jar (default-jar) @ tachyon-examples --- + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/examples/target/tachyon-examples-0.7.1.jar + [INFO] + [INFO] >>> maven-source-plugin:2.3:jar (attach-sources) > generate-sources @ tachyon-examples >>> + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-examples --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-examples --- + [INFO] Starting audit... + Audit done. + + [INFO] + [INFO] --- license-maven-plugin:2.9:check (default) @ tachyon-examples --- + [INFO] Checking licenses... + [INFO] + [INFO] <<< maven-source-plugin:2.3:jar (attach-sources) < generate-sources @ tachyon-examples <<< + [INFO] + [INFO] --- maven-source-plugin:2.3:jar (attach-sources) @ tachyon-examples --- + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/examples/target/tachyon-examples-0.7.1-sources.jar + [INFO] + [INFO] --- maven-javadoc-plugin:2.9:jar (attach-javadoc) @ tachyon-examples --- + [INFO] + Loading source files for package tachyon.examples... + Constructing Javadoc information... + Standard Doclet version 1.7.0_79 + Building tree for all the packages and classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/BasicCheckpoint.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/BasicNonByteBufferOperations.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/BasicOperations.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/BasicRawTableOperations.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/Performance.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/Performance.GeneralWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/Performance.HdfsWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/Performance.TachyonReadWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/Performance.TachyonWriterWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/Performance.Worker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/Utils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/package-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/package-summary.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/package-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/constant-values.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/BasicNonByteBufferOperations.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/Utils.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/BasicOperations.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/BasicRawTableOperations.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/Performance.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/Performance.Worker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/Performance.GeneralWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/Performance.TachyonWriterWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/Performance.TachyonReadWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/Performance.HdfsWorker.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/class-use/BasicCheckpoint.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/tachyon/examples/package-use.html... + Building index for all the packages and classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/overview-tree.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/index-all.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/deprecated-list.html... + Building index for all classes... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/allclasses-frame.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/allclasses-noframe.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/index.html... + Generating /home/hadoop/cloud/alluxio-0.7.1/examples/target/apidocs/help-doc.html... + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/examples/target/tachyon-examples-0.7.1-javadoc.jar + [INFO] + [INFO] ------------------------------------------------------------------------ + [INFO] Building Tachyon Assemblies 0.7.1 + [INFO] ------------------------------------------------------------------------ + [INFO] + [INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ tachyon-assemblies --- + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-assemblies --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-assemblies --- + [INFO] Starting audit... + Audit done. + + [INFO] + [INFO] --- license-maven-plugin:2.9:check (default) @ tachyon-assemblies --- + [INFO] Checking licenses... + [INFO] + [INFO] --- maven-resources-plugin:2.6:resources (default-resources) @ tachyon-assemblies --- + [INFO] Using 'UTF-8' encoding to copy filtered resources. + [INFO] skip non existing resourceDirectory /home/hadoop/cloud/alluxio-0.7.1/assembly/src/main/resources + [INFO] + [INFO] --- maven-compiler-plugin:3.2:compile (default-compile) @ tachyon-assemblies --- + [INFO] No sources to compile + [INFO] + [INFO] --- maven-resources-plugin:2.6:testResources (default-testResources) @ tachyon-assemblies --- + [INFO] Using 'UTF-8' encoding to copy filtered resources. + [INFO] skip non existing resourceDirectory /home/hadoop/cloud/alluxio-0.7.1/assembly/src/test/resources + [INFO] + [INFO] --- maven-compiler-plugin:3.2:testCompile (default-testCompile) @ tachyon-assemblies --- + [INFO] No sources to compile + [INFO] + [INFO] --- maven-surefire-plugin:2.14:test (default-test) @ tachyon-assemblies --- + [INFO] Tests are skipped. + [INFO] + [INFO] --- maven-jar-plugin:2.4:jar (default-jar) @ tachyon-assemblies --- + [WARNING] JAR will be empty - no content was marked for inclusion! + [INFO] Building jar: /home/hadoop/cloud/alluxio-0.7.1/assembly/target/tachyon-assemblies-0.7.1.jar + [INFO] + [INFO] --- maven-shade-plugin:2.2:shade (uber-jar) @ tachyon-assemblies --- + [INFO] Including org.tachyonproject:tachyon-common:jar:0.7.1 in the shaded jar. + [INFO] Including com.google.guava:guava:jar:14.0.1 in the shaded jar. + [INFO] Including commons-lang:commons-lang:jar:2.4 in the shaded jar. + [INFO] Including commons-io:commons-io:jar:2.4 in the shaded jar. + [INFO] Including io.netty:netty-all:jar:4.0.28.Final in the shaded jar. + [INFO] Including org.apache.commons:commons-lang3:jar:3.0 in the shaded jar. + [INFO] Including org.apache.curator:curator-framework:jar:2.1.0-incubating in the shaded jar. + [INFO] Including org.apache.zookeeper:zookeeper:jar:3.4.5 in the shaded jar. + [INFO] Including log4j:log4j:jar:1.2.15 in the shaded jar. + [INFO] Including javax.mail:mail:jar:1.4 in the shaded jar. + [INFO] Including jline:jline:jar:0.9.94 in the shaded jar. + [INFO] Including org.jboss.netty:netty:jar:3.2.2.Final in the shaded jar. + [INFO] Including org.apache.curator:curator-client:jar:2.1.0-incubating in the shaded jar. + [INFO] Including org.apache.curator:curator-recipes:jar:2.1.0-incubating in the shaded jar. + [INFO] Including org.apache.thrift:libthrift:jar:0.9.1 in the shaded jar. + [INFO] Including org.apache.httpcomponents:httpclient:jar:4.2.5 in the shaded jar. + [INFO] Including org.apache.httpcomponents:httpcore:jar:4.2.4 in the shaded jar. + [INFO] Including org.slf4j:slf4j-api:jar:1.7.2 in the shaded jar. + [INFO] Including org.tachyonproject:tachyon-servers:jar:0.7.1 in the shaded jar. + [INFO] Including org.tachyonproject:tachyon-client-unshaded:jar:0.7.1 in the shaded jar. + [INFO] Including io.dropwizard.metrics:metrics-core:jar:3.1.0 in the shaded jar. + [INFO] Including io.dropwizard.metrics:metrics-json:jar:3.1.0 in the shaded jar. + [INFO] Including io.dropwizard.metrics:metrics-jvm:jar:3.1.0 in the shaded jar. + [INFO] Including io.dropwizard.metrics:metrics-graphite:jar:3.1.0 in the shaded jar. + [INFO] Including org.eclipse.jetty:jetty-jsp:jar:7.6.15.v20140411 in the shaded jar. + [INFO] Including org.eclipse.jetty.orbit:javax.servlet.jsp:jar:2.1.0.v201105211820 in the shaded jar. + [INFO] Including org.eclipse.jetty.orbit:org.apache.jasper.glassfish:jar:2.1.0.v201110031002 in the shaded jar. + [INFO] Including org.eclipse.jetty.orbit:javax.servlet.jsp.jstl:jar:1.2.0.v201105211821 in the shaded jar. + [INFO] Including org.eclipse.jetty.orbit:org.apache.taglibs.standard.glassfish:jar:1.2.0.v201112081803 in the shaded jar. + [INFO] Including org.eclipse.jetty.orbit:javax.el:jar:2.1.0.v201105211819 in the shaded jar. + [INFO] Including org.eclipse.jetty.orbit:com.sun.el:jar:1.0.0.v201105211818 in the shaded jar. + [INFO] Including org.eclipse.jetty.orbit:org.eclipse.jdt.core:jar:3.7.1 in the shaded jar. + [INFO] Including org.eclipse.jetty:jetty-webapp:jar:7.6.15.v20140411 in the shaded jar. + [INFO] Including org.eclipse.jetty:jetty-xml:jar:7.6.15.v20140411 in the shaded jar. + [INFO] Including org.eclipse.jetty:jetty-util:jar:7.6.15.v20140411 in the shaded jar. + [INFO] Including org.eclipse.jetty:jetty-server:jar:7.6.15.v20140411 in the shaded jar. + [INFO] Including org.eclipse.jetty.orbit:javax.servlet:jar:2.5.0.v201103041518 in the shaded jar. + [INFO] Including org.eclipse.jetty:jetty-continuation:jar:7.6.15.v20140411 in the shaded jar. + [INFO] Including org.eclipse.jetty:jetty-http:jar:7.6.15.v20140411 in the shaded jar. + [INFO] Including org.eclipse.jetty:jetty-io:jar:7.6.15.v20140411 in the shaded jar. + [INFO] Including org.eclipse.jetty:jetty-servlet:jar:7.6.15.v20140411 in the shaded jar. + [INFO] Including org.eclipse.jetty:jetty-security:jar:7.6.15.v20140411 in the shaded jar. + [INFO] Including commons-codec:commons-codec:jar:1.5 in the shaded jar. + [INFO] Including com.fasterxml.jackson.core:jackson-databind:jar:2.3.0 in the shaded jar. + [INFO] Including com.fasterxml.jackson.core:jackson-annotations:jar:2.3.0 in the shaded jar. + [INFO] Including com.fasterxml.jackson.core:jackson-core:jar:2.3.0 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-client:jar:2.6.0 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-common:jar:2.6.0 in the shaded jar. + [INFO] Including commons-cli:commons-cli:jar:1.2 in the shaded jar. + [INFO] Including org.apache.commons:commons-math3:jar:3.1.1 in the shaded jar. + [INFO] Including xmlenc:xmlenc:jar:0.52 in the shaded jar. + [INFO] Including commons-net:commons-net:jar:3.1 in the shaded jar. + [INFO] Including commons-collections:commons-collections:jar:3.2.1 in the shaded jar. + [INFO] Including commons-configuration:commons-configuration:jar:1.6 in the shaded jar. + [INFO] Including commons-digester:commons-digester:jar:1.8 in the shaded jar. + [INFO] Including commons-beanutils:commons-beanutils:jar:1.7.0 in the shaded jar. + [INFO] Including commons-beanutils:commons-beanutils-core:jar:1.8.0 in the shaded jar. + [INFO] Including org.apache.avro:avro:jar:1.7.4 in the shaded jar. + [INFO] Including com.thoughtworks.paranamer:paranamer:jar:2.3 in the shaded jar. + [INFO] Including org.xerial.snappy:snappy-java:jar:1.0.4.1 in the shaded jar. + [INFO] Including com.google.protobuf:protobuf-java:jar:2.5.0 in the shaded jar. + [INFO] Including com.google.code.gson:gson:jar:2.2.4 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-auth:jar:2.6.0 in the shaded jar. + [INFO] Including org.apache.directory.server:apacheds-kerberos-codec:jar:2.0.0-M15 in the shaded jar. + [INFO] Including org.apache.directory.server:apacheds-i18n:jar:2.0.0-M15 in the shaded jar. + [INFO] Including org.apache.directory.api:api-asn1-api:jar:1.0.0-M20 in the shaded jar. + [INFO] Including org.apache.directory.api:api-util:jar:1.0.0-M20 in the shaded jar. + [INFO] Including com.google.code.findbugs:jsr305:jar:1.3.9 in the shaded jar. + [INFO] Including org.htrace:htrace-core:jar:3.0.4 in the shaded jar. + [INFO] Including org.apache.commons:commons-compress:jar:1.4.1 in the shaded jar. + [INFO] Including org.tukaani:xz:jar:1.0 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-hdfs:jar:2.6.0 in the shaded jar. + [INFO] Including org.mortbay.jetty:jetty-util:jar:6.1.26 in the shaded jar. + [INFO] Including io.netty:netty:jar:3.6.2.Final in the shaded jar. + [INFO] Including xerces:xercesImpl:jar:2.9.1 in the shaded jar. + [INFO] Including xml-apis:xml-apis:jar:1.3.04 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-mapreduce-client-app:jar:2.6.0 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-mapreduce-client-common:jar:2.6.0 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-yarn-client:jar:2.6.0 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-mapreduce-client-shuffle:jar:2.6.0 in the shaded jar. + [INFO] Including org.fusesource.leveldbjni:leveldbjni-all:jar:1.8 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-yarn-api:jar:2.6.0 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-mapreduce-client-core:jar:2.6.0 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-yarn-common:jar:2.6.0 in the shaded jar. + [INFO] Including org.codehaus.jackson:jackson-jaxrs:jar:1.9.13 in the shaded jar. + [INFO] Including org.codehaus.jackson:jackson-xc:jar:1.9.13 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-mapreduce-client-jobclient:jar:2.6.0 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-annotations:jar:2.6.0 in the shaded jar. + [INFO] Including org.tachyonproject:tachyon-underfs-hdfs:jar:0.7.1 in the shaded jar. + [INFO] Including org.tachyonproject:tachyon-underfs-glusterfs:jar:0.7.1 in the shaded jar. + [INFO] Including org.gluster:glusterfs-hadoop:jar:2.3.13 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-yarn-server-nodemanager:jar:2.3.0 in the shaded jar. + [INFO] Including javax.xml.bind:jaxb-api:jar:2.2.2 in the shaded jar. + [INFO] Including javax.xml.stream:stax-api:jar:1.0-2 in the shaded jar. + [INFO] Including javax.activation:activation:jar:1.1 in the shaded jar. + [INFO] Including org.codehaus.jettison:jettison:jar:1.1 in the shaded jar. + [INFO] Including javax.servlet:servlet-api:jar:2.5 in the shaded jar. + [INFO] Including com.sun.jersey:jersey-core:jar:1.9 in the shaded jar. + [INFO] Including com.sun.jersey:jersey-client:jar:1.9 in the shaded jar. + [INFO] Including com.google.inject.extensions:guice-servlet:jar:3.0 in the shaded jar. + [INFO] Including com.google.inject:guice:jar:3.0 in the shaded jar. + [INFO] Including javax.inject:javax.inject:jar:1 in the shaded jar. + [INFO] Including aopalliance:aopalliance:jar:1.0 in the shaded jar. + [INFO] Including com.sun.jersey:jersey-json:jar:1.9 in the shaded jar. + [INFO] Including com.sun.xml.bind:jaxb-impl:jar:2.2.3-1 in the shaded jar. + [INFO] Including com.sun.jersey.contribs:jersey-guice:jar:1.9 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-yarn-server-common:jar:2.3.0 in the shaded jar. + [INFO] Including org.slf4j:slf4j-log4j12:jar:1.7.2 in the shaded jar. + [INFO] Including org.tachyonproject:tachyon-underfs-s3:jar:0.7.1 in the shaded jar. + [INFO] Including net.java.dev.jets3t:jets3t:jar:0.8.1 in the shaded jar. + [INFO] Including commons-logging:commons-logging:jar:1.1.1 in the shaded jar. + [INFO] Including com.jamesmurty.utils:java-xmlbuilder:jar:0.4 in the shaded jar. + [INFO] Including org.tachyonproject:tachyon-underfs-swift:jar:0.7.1 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-openstack:jar:2.6.0 in the shaded jar. + [INFO] Including org.apache.hadoop:hadoop-common:test-jar:tests:2.6.0 in the shaded jar. + [INFO] Including org.mortbay.jetty:jetty:jar:6.1.26 in the shaded jar. + [INFO] Including com.sun.jersey:jersey-server:jar:1.9 in the shaded jar. + [INFO] Including asm:asm:jar:3.1 in the shaded jar. + [INFO] Including tomcat:jasper-compiler:jar:5.5.23 in the shaded jar. + [INFO] Including tomcat:jasper-runtime:jar:5.5.23 in the shaded jar. + [INFO] Including javax.servlet.jsp:jsp-api:jar:2.1 in the shaded jar. + [INFO] Including commons-el:commons-el:jar:1.0 in the shaded jar. + [INFO] Including com.jcraft:jsch:jar:0.1.42 in the shaded jar. + [INFO] Including org.codehaus.jackson:jackson-mapper-asl:jar:1.9.13 in the shaded jar. + [INFO] Including org.codehaus.jackson:jackson-core-asl:jar:1.9.13 in the shaded jar. + [INFO] Including org.mockito:mockito-all:jar:1.8.5 in the shaded jar. + [INFO] Including commons-httpclient:commons-httpclient:jar:3.1 in the shaded jar. + [INFO] Including org.tachyonproject:tachyon-underfs-local:jar:0.7.1 in the shaded jar. + [INFO] Including org.tachyonproject:tachyon-examples:jar:0.7.1 in the shaded jar. + [INFO] Including org.tachyonproject:tachyon-shell:jar:0.7.1 in the shaded jar. + [WARNING] javax.el-2.1.0.v201105211819.jar, jsp-api-2.1.jar define 26 overlappping classes: + [WARNING] - javax.el.ELContextListener + [WARNING] - javax.el.Expression + [WARNING] - javax.el.MapELResolver + [WARNING] - javax.el.VariableMapper + [WARNING] - javax.el.ELException + [WARNING] - javax.el.MethodNotFoundException + [WARNING] - javax.el.ArrayELResolver + [WARNING] - javax.el.ValueExpression + [WARNING] - javax.el.CompositeELResolver$CompositeIterator + [WARNING] - javax.el.ELContextEvent + [WARNING] - 16 more... + [WARNING] jasper-compiler-5.5.23.jar, org.apache.jasper.glassfish-2.1.0.v201110031002.jar, jasper-runtime-5.5.23.jar define 1 overlappping classes: + [WARNING] - org.apache.jasper.compiler.Localizer + [WARNING] commons-beanutils-core-1.8.0.jar, commons-beanutils-1.7.0.jar, commons-collections-3.2.1.jar define 10 overlappping classes: + [WARNING] - org.apache.commons.collections.FastHashMap$EntrySet + [WARNING] - org.apache.commons.collections.ArrayStack + [WARNING] - org.apache.commons.collections.FastHashMap$1 + [WARNING] - org.apache.commons.collections.FastHashMap$KeySet + [WARNING] - org.apache.commons.collections.FastHashMap$CollectionView + [WARNING] - org.apache.commons.collections.BufferUnderflowException + [WARNING] - org.apache.commons.collections.Buffer + [WARNING] - org.apache.commons.collections.FastHashMap$CollectionView$CollectionViewIterator + [WARNING] - org.apache.commons.collections.FastHashMap$Values + [WARNING] - org.apache.commons.collections.FastHashMap + [WARNING] javax.servlet.jsp-2.1.0.v201105211820.jar, jsp-api-2.1.jar define 58 overlappping classes: + [WARNING] - javax.servlet.jsp.tagext.SimpleTagSupport + [WARNING] - javax.servlet.jsp.tagext.SimpleTag + [WARNING] - javax.servlet.jsp.el.ExpressionEvaluator + [WARNING] - javax.servlet.jsp.el.ImplicitObjectELResolver$ImplicitObjects + [WARNING] - javax.servlet.jsp.JspPage + [WARNING] - javax.servlet.jsp.tagext.TagLibraryValidator + [WARNING] - javax.servlet.jsp.el.ImplicitObjectELResolver$ImplicitObjects$2 + [WARNING] - javax.servlet.jsp.el.ImplicitObjectELResolver$ImplicitObjects$4 + [WARNING] - javax.servlet.jsp.JspApplicationContext + [WARNING] - javax.servlet.jsp.el.ImplicitObjectELResolver$ImplicitObjects$9 + [WARNING] - 48 more... + [WARNING] hadoop-common-2.6.0-tests.jar, hadoop-common-2.6.0.jar define 37 overlappping classes: + [WARNING] - org.apache.hadoop.ipc.protobuf.TestProtos$EmptyResponseProto$1 + [WARNING] - org.apache.hadoop.ipc.protobuf.TestProtos$EmptyResponseProtoOrBuilder + [WARNING] - org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub + [WARNING] - org.apache.hadoop.ipc.protobuf.TestProtos$EchoResponseProto + [WARNING] - org.apache.hadoop.ipc.protobuf.TestProtos$EchoResponseProtoOrBuilder + [WARNING] - org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos$TestProtobufRpc2Proto$BlockingInterface + [WARNING] - org.apache.hadoop.io.serializer.avro.AvroRecord$1 + [WARNING] - org.apache.hadoop.ipc.protobuf.TestProtos$EchoRequestProtoOrBuilder + [WARNING] - org.apache.hadoop.ipc.protobuf.TestProtos$EchoResponseProto$Builder + [WARNING] - org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos$1 + [WARNING] - 27 more... + [WARNING] commons-beanutils-core-1.8.0.jar, commons-beanutils-1.7.0.jar define 82 overlappping classes: + [WARNING] - org.apache.commons.beanutils.Converter + [WARNING] - org.apache.commons.beanutils.WrapDynaBean + [WARNING] - org.apache.commons.beanutils.converters.IntegerConverter + [WARNING] - org.apache.commons.beanutils.locale.LocaleBeanUtilsBean + [WARNING] - org.apache.commons.beanutils.locale.converters.DoubleLocaleConverter + [WARNING] - org.apache.commons.beanutils.locale.converters.DecimalLocaleConverter + [WARNING] - org.apache.commons.beanutils.converters.ShortConverter + [WARNING] - org.apache.commons.beanutils.converters.StringArrayConverter + [WARNING] - org.apache.commons.beanutils.locale.LocaleConvertUtilsBean + [WARNING] - org.apache.commons.beanutils.LazyDynaClass + [WARNING] - 72 more... + [WARNING] hadoop-yarn-common-2.6.0.jar, hadoop-yarn-api-2.6.0.jar define 3 overlappping classes: + [WARNING] - org.apache.hadoop.yarn.factories.package-info + [WARNING] - org.apache.hadoop.yarn.util.package-info + [WARNING] - org.apache.hadoop.yarn.factory.providers.package-info + [WARNING] netty-3.6.2.Final.jar, netty-3.2.2.Final.jar define 497 overlappping classes: + [WARNING] - org.jboss.netty.logging.CommonsLoggerFactory + [WARNING] - org.jboss.netty.channel.local.LocalChannel + [WARNING] - org.jboss.netty.buffer.ChannelBufferIndexFinder$5 + [WARNING] - org.jboss.netty.util.internal.jzlib.CRC32 + [WARNING] - org.jboss.netty.util.ExternalResourceUtil + [WARNING] - org.jboss.netty.handler.codec.base64.Base64Decoder + [WARNING] - org.jboss.netty.channel.socket.nio.NioDatagramPipelineSink$1 + [WARNING] - org.jboss.netty.util.VirtualExecutorService + [WARNING] - org.jboss.netty.util.DefaultObjectSizeEstimator + [WARNING] - org.jboss.netty.channel.socket.http.HttpTunnelingClientSocketChannel$6$1 + [WARNING] - 487 more... + [WARNING] jasper-compiler-5.5.23.jar, org.apache.jasper.glassfish-2.1.0.v201110031002.jar define 143 overlappping classes: + [WARNING] - org.apache.jasper.compiler.TagLibraryInfoImpl + [WARNING] - org.apache.jasper.xmlparser.SymbolTable + [WARNING] - org.apache.jasper.compiler.Generator$FragmentHelperClass$Fragment + [WARNING] - org.apache.jasper.compiler.Generator$1TagHandlerPoolVisitor + [WARNING] - org.apache.jasper.compiler.ScriptingVariabler + [WARNING] - org.apache.jasper.compiler.SmapStratum$LineInfo + [WARNING] - org.apache.jasper.compiler.Node$AttributeGenerator + [WARNING] - org.apache.jasper.compiler.Node + [WARNING] - org.apache.jasper.compiler.Node$JspAttribute + [WARNING] - org.apache.jasper.compiler.tagplugin.TagPlugin + [WARNING] - 133 more... + [WARNING] org.apache.jasper.glassfish-2.1.0.v201110031002.jar, jasper-runtime-5.5.23.jar define 43 overlappping classes: + [WARNING] - org.apache.jasper.runtime.ProtectedFunctionMapper$2 + [WARNING] - org.apache.jasper.runtime.PageContextImpl$3 + [WARNING] - org.apache.jasper.runtime.JspFactoryImpl$PrivilegedGetPageContext + [WARNING] - org.apache.jasper.runtime.PageContextImpl$4 + [WARNING] - org.apache.jasper.runtime.JspFactoryImpl + [WARNING] - org.apache.jasper.runtime.ProtectedFunctionMapper$1 + [WARNING] - org.apache.jasper.JasperException + [WARNING] - org.apache.jasper.runtime.PerThreadTagHandlerPool$PerThreadData + [WARNING] - org.apache.jasper.util.FastDateFormat + [WARNING] - org.apache.jasper.runtime.PageContextImpl$8 + [WARNING] - 33 more... + [WARNING] servlet-api-2.5.jar, javax.servlet-2.5.0.v201103041518.jar define 42 overlappping classes: + [WARNING] - javax.servlet.ServletRequestWrapper + [WARNING] - javax.servlet.FilterChain + [WARNING] - javax.servlet.SingleThreadModel + [WARNING] - javax.servlet.http.HttpServletResponse + [WARNING] - javax.servlet.http.HttpUtils + [WARNING] - javax.servlet.ServletContextAttributeEvent + [WARNING] - javax.servlet.ServletContextAttributeListener + [WARNING] - javax.servlet.http.HttpServlet + [WARNING] - javax.servlet.http.HttpSessionAttributeListener + [WARNING] - javax.servlet.http.HttpServletRequest + [WARNING] - 32 more... + [WARNING] hadoop-yarn-common-2.6.0.jar, hadoop-yarn-client-2.6.0.jar define 2 overlappping classes: + [WARNING] - org.apache.hadoop.yarn.client.api.impl.package-info + [WARNING] - org.apache.hadoop.yarn.client.api.package-info + [WARNING] maven-shade-plugin has detected that some .class files + [WARNING] are present in two or more JARs. When this happens, only + [WARNING] one single version of the class is copied in the uberjar. + [WARNING] Usually this is not harmful and you can skeep these + [WARNING] warnings, otherwise try to manually exclude artifacts + [WARNING] based on mvn dependency:tree -Ddetail=true and the above + [WARNING] output + [WARNING] See http://docs.codehaus.org/display/MAVENUSER/Shade+Plugin + [INFO] Replacing /home/hadoop/cloud/alluxio-0.7.1/assembly/target/tachyon-assemblies-0.7.1-jar-with-dependencies.jar with /home/hadoop/cloud/alluxio-0.7.1/assembly/target/tachyon-assemblies-0.7.1-shaded.jar + [INFO] + [INFO] >>> maven-source-plugin:2.3:jar (attach-sources) > generate-sources @ tachyon-assemblies >>> + [INFO] + [INFO] --- maven-enforcer-plugin:1.0:enforce (enforce-maven) @ tachyon-assemblies --- + [INFO] + [INFO] --- maven-checkstyle-plugin:2.13:check (checkstyle) @ tachyon-assemblies --- + [INFO] Starting audit... + Audit done. + + [INFO] + [INFO] --- license-maven-plugin:2.9:check (default) @ tachyon-assemblies --- + [INFO] Checking licenses... + [INFO] + [INFO] <<< maven-source-plugin:2.3:jar (attach-sources) < generate-sources @ tachyon-assemblies <<< + [INFO] + [INFO] --- maven-source-plugin:2.3:jar (attach-sources) @ tachyon-assemblies --- + [INFO] No sources in project. Archive not created. + [INFO] + [INFO] --- maven-javadoc-plugin:2.9:jar (attach-javadoc) @ tachyon-assemblies --- + [INFO] ------------------------------------------------------------------------ + [INFO] Reactor Summary: + [INFO] + [INFO] Tachyon Parent ..................................... SUCCESS [03:15 min] + [INFO] Tachyon Common ..................................... SUCCESS [01:23 min] + [INFO] Tachyon Under File System .......................... SUCCESS [ 0.609 s] + [INFO] Tachyon Under File System - Local FS ............... SUCCESS [ 32.970 s] + [INFO] Tachyon Under File System - HDFS ................... SUCCESS [ 5.169 s] + [INFO] Tachyon Under File System - Gluster FS ............. SUCCESS [01:43 min] + [INFO] Tachyon Under File System - Swift .................. SUCCESS [01:04 min] + [INFO] Tachyon Under File System - S3 ..................... SUCCESS [ 20.526 s] + [INFO] Tachyon Clients .................................... SUCCESS [ 0.523 s] + [INFO] Tachyon Clients - Implementation ................... SUCCESS [ 9.195 s] + [INFO] Tachyon Clients - Distribution ..................... SUCCESS [ 30.238 s] + [INFO] Tachyon Servers .................................... SUCCESS [ 50.587 s] + [INFO] Mock Tachyon Cluster ............................... SUCCESS [ 5.288 s] + [INFO] Tachyon Integration Tests .......................... SUCCESS [01:11 min] + [INFO] Tachyon Shell ...................................... SUCCESS [ 3.214 s] + [INFO] Tachyon Examples ................................... SUCCESS [ 3.489 s] + [INFO] Tachyon Assemblies ................................. SUCCESS [ 11.906 s] + [INFO] ------------------------------------------------------------------------ + [INFO] BUILD SUCCESS + [INFO] ------------------------------------------------------------------------ + [INFO] Total time: 11:32 min + [INFO] Finished at: 2016-05-05T19:29:29+08:00 + [INFO] Final Memory: 107M/642M + [INFO] ------------------------------------------------------------------------ + + + +参考 +【1】http://alluxio.org/documentation/v0.7.1/ +【2】https://github.com/xubo245/SparkLearning diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2408---\351\233\206\347\276\244\347\211\210\346\220\255\345\273\272\351\227\256\351\242\230\344\271\213\351\233\206\347\276\244\346\227\240\346\263\225\345\205\250\351\203\250\345\220\257\345\212\250.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2408---\351\233\206\347\276\244\347\211\210\346\220\255\345\273\272\351\227\256\351\242\230\344\271\213\351\233\206\347\276\244\346\227\240\346\263\225\345\205\250\351\203\250\345\220\257\345\212\250.md" new file mode 100644 index 0000000..fc3e69d --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2408---\351\233\206\347\276\244\347\211\210\346\220\255\345\273\272\351\227\256\351\242\230\344\271\213\351\233\206\347\276\244\346\227\240\346\263\225\345\205\250\351\203\250\345\220\257\345\212\250.md" @@ -0,0 +1,276 @@ +六个worker只启动了3个： + + hadoop@Master:~/cloud/alluxio-0.7.1$ ./bin/tachyon format + Connecting to Mcnode1 as hadoop... + Formatting Tachyon Worker @ Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Formatting Tachyon Worker @ Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode3 + /home/hadoop/cloud/alluxio-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode4 + /home/hadoop/cloud/alluxio-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Formatting Tachyon Worker @ Mcnode5 + /home/hadoop/cloud/alluxio-0.7.1/bin/tachyon: line 264: /../bin/java: No such file or directory + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Formatting Tachyon Worker @ Mcnode6 + Connection to mcnode6 closed. + Formatting Tachyon Master @ Master + hadoop@Master:~/cloud/alluxio-0.7.1$ ./bin/tachyon-start.sh all Mount + Killed 0 processes on Master + Killed 0 processes on Master + Connecting to Mcnode1 as hadoop... + Killed 0 processes on Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Killed 0 processes on Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Killed 0 processes on Mcnode3 + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Killed 0 processes on Mcnode4 + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Killed 0 processes on Mcnode5 + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Killed 0 processes on Mcnode6 + Connection to mcnode6 closed. + Starting master @ Master + Connecting to Mcnode1 as hadoop... + Formatting RamFS: /mnt/ramdisk (256mb) + umount: only root can do that + ERROR: umount RamFS /mnt/ramdisk failed + Mount failed, not starting worker + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Formatting RamFS: /mnt/ramdisk (256mb) + umount: only root can do that + ERROR: umount RamFS /mnt/ramdisk failed + Mount failed, not starting worker + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + TACHYON_LOGS_DIR: /home/hadoop/cloud/alluxio-0.7.1/logs + Formatting RamFS: /mnt/ramdisk (256mb) + umount: only root can do that + ERROR: umount RamFS /mnt/ramdisk failed + Mount failed, not starting worker + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + TACHYON_LOGS_DIR: /home/hadoop/cloud/alluxio-0.7.1/logs + Formatting RamFS: /mnt/ramdisk (256mb) + umount: only root can do that + ERROR: umount RamFS /mnt/ramdisk failed + Mount failed, not starting worker + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + TACHYON_LOGS_DIR: /home/hadoop/cloud/alluxio-0.7.1/logs + Formatting RamFS: /mnt/ramdisk (256mb) + umount: only root can do that + ERROR: umount RamFS /mnt/ramdisk failed + Mount failed, not starting worker + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Formatting RamFS: /mnt/ramdisk (256mb) + umount: only root can do that + ERROR: umount RamFS /mnt/ramdisk failed + Mount failed, not starting worker + Connection to mcnode6 closed. + hadoop@Master:~/cloud/alluxio-0.7.1$ ./bin/tachyon-start.sh all SudoMount + Killed 1 processes on Master + Killed 0 processes on Master + Connecting to Mcnode1 as hadoop... + Killed 0 processes on Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Killed 0 processes on Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Killed 0 processes on Mcnode3 + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Killed 0 processes on Mcnode4 + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Killed 0 processes on Mcnode5 + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Killed 0 processes on Mcnode6 + Connection to mcnode6 closed. + Starting master @ Master + Connecting to Mcnode1 as hadoop... + [sudo] password for hadoop: + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + [sudo] password for hadoop: + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + [sudo] password for hadoop: + dirname: missing operand + Try 'dirname --help' for more information. + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode3 + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + [sudo] password for hadoop: + dirname: missing operand + Try 'dirname --help' for more information. + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode4 + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + [sudo] password for hadoop: + dirname: missing operand + Try 'dirname --help' for more information. + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode5 + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + [sudo] password for hadoop: + Formatting RamFS: /mnt/ramdisk (256mb) + Starting worker @ Mcnode6 + Connection to mcnode6 closed. + hadoop@Master:~/cloud/alluxio-0.7.1$ + hadoop@Master:~/cloud/alluxio-0.7.1$ jps + 30101 SparkSubmit + 17541 ResourceManager + 2526 TachyonMaster + 17389 SecondaryNameNode + 27677 CoarseGrainedExecutorBackend + 2610 Jps + 17173 NameNode + 17935 Master + 18125 Worker + 1450 SparkSubmit + 27574 SparkSubmit + 3528 HistoryServer + + +解决办法： +方法1： +在每个节点各自启动，可以将所有节点启动，在master的19999可以看到： + + Last login: Thu May 5 20:26:04 2016 from mcnode4 + hadoop@Mcnode5:~$ cd cloud/alluxio-0.7.1/bin/ + hadoop@Mcnode5:~/cloud/alluxio-0.7.1/bin$ ./tachyon-start.sh local + Killed 0 processes on Mcnode5 + Killed 0 processes on Mcnode5 + Connecting to Mcnode1 as hadoop... + Killed 1 processes on Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Killed 1 processes on Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + Warning: Permanently added 'mcnode3,219.219.220.131' (ECDSA) to the list of known hosts. + dirname: missing operand + Try 'dirname --help' for more information. + Killed 0 processes on Mcnode3 + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Killed 1 processes on Mcnode4 + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + Warning: Permanently added 'mcnode5,219.219.220.215' (ECDSA) to the list of known hosts. + dirname: missing operand + Try 'dirname --help' for more information. + Killed 0 processes on Mcnode5 + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Killed 1 processes on Mcnode6 + Connection to mcnode6 closed. + [sudo] password for hadoop: + dirname: missing operand + Try 'dirname --help' for more information. + Formatting RamFS: /mnt/ramdisk (256mb) + Starting master @ Master + Starting worker @ Mcnode5 + + + +但是上传一个文件后又报错： + + hadoop@Mcnode3:~/cloud/tachyon-0.7.1-hadoop2.6-bin$ jps + Error occurred during initialization of VM + java.lang.Error: Properties init: Could not determine current working directory. + at java.lang.System.initProperties(Native Method) + at java.lang.System.initializeSystemClass(System.java:1119) + +关闭集群： + + hadoop@Master:~/cloud/tachyon-0.7.1-hadoop2.6-bin$ ./bin/tachyon-stop.sh + Killed 1 processes on Master + Killed 0 processes on Master + Connecting to Mcnode1 as hadoop... + Killed 1 processes on Mcnode1 + Connection to mcnode1 closed. + Connecting to Mcnode2 as hadoop... + Killed 1 processes on Mcnode2 + Connection to mcnode2 closed. + Connecting to Mcnode3 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Killed 0 processes on Mcnode3 + Connection to mcnode3 closed. + Connecting to Mcnode4 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Killed 0 processes on Mcnode4 + Connection to mcnode4 closed. + Connecting to Mcnode5 as hadoop... + dirname: missing operand + Try 'dirname --help' for more information. + Killed 1 processes on Mcnode5 + Connection to mcnode5 closed. + Connecting to Mcnode6 as hadoop... + Killed 1 processes on Mcnode6 + Connection to mcnode6 closed. + +方法2：可以启动全部节点 + + http://blog.csdn.net/xubo245/article/details/53056482 + diff --git "a/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2409--Auuxio(Tachyon)\347\232\204\344\274\230\347\202\271.md" "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2409--Auuxio(Tachyon)\347\232\204\344\274\230\347\202\271.md" new file mode 100644 index 0000000..29ff96f --- /dev/null +++ "b/docs/alluxioLearning/Spark\347\224\237\346\200\201\344\271\213Alluxio\345\255\246\344\271\2409--Auuxio(Tachyon)\347\232\204\344\274\230\347\202\271.md" @@ -0,0 +1,15 @@ + +1.Spark的不同Job之间，两个不用的application需要从HDFS中加载两次同样的数据，而使用tachyon就不用了： + 当两个Spark作业需要共享数据时，必须通过写磁盘操作。比如：作业1要先把生成的数据写入HDFS，然后作业2再从HDFS把数据读出来。在此，磁盘的读写可能造成性能瓶颈。 + 另外不同的应用框架也可以利用tachyon在内存中共享数据，比如Spark和Hadoop + +2. 由于Spark会利用自身的JVM对数据进行缓存，当Spark程序崩溃时，JVM进程退出，所缓存数据也随之丢失，因此在工作重启时又需要从HDFS把数据再次读出。 + 如果利用Tachyon，相当于在disk和任务间加一层，就算JVM或者及其crash掉了，重启时可以在tachyon读取 + 不同进程间数据的共享 + +3. 当两个Spark作业需操作相同的数据时，每个作业的JVM都需要缓存一份数据，不但造成资源浪费，也极易引发频繁的垃圾收集，造成性能的降低。 + =》不同job缓存同一份数据会增加内存开销，而是用tachyon就可以节省资源 + + +参考： +【1】http://www.wtoutiao.com/p/hf6YDg.html diff --git "a/docs/gangliaLearning/ganglia\345\255\246\344\271\2401\344\271\213ganglia\351\233\206\347\276\244\347\211\210\346\220\255\345\273\272\357\274\210\346\224\257\346\214\201hadoop\345\222\214spark\357\274\211.md" "b/docs/gangliaLearning/ganglia\345\255\246\344\271\2401\344\271\213ganglia\351\233\206\347\276\244\347\211\210\346\220\255\345\273\272\357\274\210\346\224\257\346\214\201hadoop\345\222\214spark\357\274\211.md" new file mode 100644 index 0000000..3474c20 --- /dev/null +++ "b/docs/gangliaLearning/ganglia\345\255\246\344\271\2401\344\271\213ganglia\351\233\206\347\276\244\347\211\210\346\220\255\345\273\272\357\274\210\346\224\257\346\214\201hadoop\345\222\214spark\357\274\211.md" @@ -0,0 +1,264 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +spark源码解读系列环境：spark-1.5.2、hadoop-2.6.0、scala-2.10.4，ganglia-3.6.1 + +系统：ubuntu 14.04 + +# 1.理解 # + +## 1.1 ganglia的概述 ## + +      Ganglia是UC Berkeley发起的一个开源集群监视项目，设计用于测量数以千计的节点。Ganglia的核心包含gmond、gmetad以及一个Web前端。主要是用来监控系统性能，如：cpu 、mem、硬盘利用率， I/O负载、网络流量情况等，通过曲线很容易见到每个节点的工作状态，对合理调整、分配系统资源，提高系统整体性能起到重要作用。 +      ganglia是一个集群监控工具，hadoop和spark已经支持 +      ganglia有论文，“The ganglia distributed monitoring system: +design, implementation, and experience”。 +## 1.2 Ganglia architecture： ## +![](http://i.imgur.com/sa6JsSV.png) + +## 1.3 集群配置 ## + +## 1.3.1 准备 ## +需要先配置apache2和php + +安装 Apache2： + + sudo apt-get install apache2 + +安装PHP模块： + + sudo apt-get install php5 +其他模块安装： + + sudo apt-get install libapache2-mod-php5 + sudo apt-get install libapache2-mod-auth-mysql +具体请参考【5】或者搜索，网上有很多 + +## 1.3.2 安装ganglia单节点 ## + +打开终端，运行以下命令： + + sudo apt-get install ganglia-monitor rrdtool gmetad ganglia-webfrontend + +在安装过程中，你应该会看到安装后与apache2服务器重启选择，直接 Yes，然后按Enter键。 + +需要复制 Ganglia webfrontend Apache 配置，使用下面的命令来正确的位置： + + sudo cp /etc/ganglia-webfrontend/apache.conf /etc/apache2/sites-enabled/ganglia.conf + +需要重启服务： + + sudo /etc/init.d/ganglia-monitor start + sudo /etc/init.d/gmetad start + sudo /etc/init.d/apache2 restart + +现在你可以使用以下命令访问gnglia webinterface： + + http://serverip/ganglia/ + +可以参考【6】 + + +## 1.3.3 安装ganglia集群 ## + +**主节点：Mcnode7** + +**客户端：Master Mcnode1 Mcnode2 Mcnode3 Mcnode4 Mcnode5 Mcnode6 Mcnode7** + +## 主节点(一个) ## + 现在需要使用以下命令来编辑 Ganglia 元守护程序的配置文件： + + sudo vi /etc/ganglia/gmetad.conf + +更改如下： + + data_source "Spark" 10 Mcnode7:8649 + +通过以上列出机器服务的数据源，IP：端口或服务器名称：端口。如果未指定端口号8649（默认gmond端口）。 + + case_sensitive_hostnames 0 +设置为1，则不会将hostname中大写变成小写 + + 需要重启服务： + + sudo /etc/init.d/ganglia-monitor restart + sudo /etc/init.d/gmetad start + sudo /etc/init.d/apache2 restart + + 现在你可以使用以下命令访问ganglia webinterface： + + http://192.168.1.50/ganglia/ + +## 客户端（8个） ## + +**（1） ganglia 客户端安装** + +你需要安装以下所有服务器要监视客户端包 + + sudo apt-get install ganglia-monitor +**（2）客户端配置：** + + 使用了单播模式，需要使用下面的命令编辑主节点的配置文件： + + sudo vi /etc/ganglia/gmond.conf + + 更改如下：（----注释为改动） + + /* If a cluster attribute is specified, then all gmond hosts are wrapped inside + * of a tag. If you do not specify a cluster tag, then all will + * NOT be wrapped inside of a tag. */ + cluster { + name = "Spark" ------集群名称 + owner = "unspecified" + latlong = "unspecified" + url = "unspecified" + } + + /* The host section describes attributes of the host, like the location */ + host { + location = "unspecified" + } + + /* Feel free to specify as many udp_send_channels as you like. Gmond + used to only support having a single channel */ + udp_send_channel { + #mcast_join = 239.2.11.71 ------注释掉组播 + host = Mcnode7 ------发送给安装gmetad的机器 + port = 8649 + ttl = 1 + } + + /* You can specify as many udp_recv_channels as you like as well. */ + udp_recv_channel { + #mcast_join = 239.2.11.71 ------注释掉组播 + port = 8649 + #bind = 239.2.11.71 ------注释掉 + } + + + 保存并关闭文件。 +**（3）客户端生效** +重启 ganglia monitor 服务 + + sudo /etc/init.d/ganglia-monitor restart + + 具体参考【6】【7】 + +效果： + +![](http://i.imgur.com/SEvew29.png) + + +## 1.3.4 配置hadoop metrics设置，连接ganglia集群 ## + + 修改Hadoop的配置文件hadoop/etc/hadoop/hadoop-metrics.properties（适用于Ganglia老版本） + + dfs.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 + dfs.period=10 + dfs.servers=192.168.1.50:8649 + + mapred.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 + mapred.period=10 + mapred.servers=192.168.1.50:8649 + + jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 + jvm.period=10 + jvm.servers=192.168.1.50:8649 + + rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 + rpc.period=10 + rpc.servers=192.168.1.50:8649 + + ugi.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 + ugi.period=10 + ugi.servers=192.168.1.50:8649 + +(2) 修改Hadoop的配置文件hadoop/etc/hadoop/hadoop-metrics2.properties（适用于Ganglia新版本） + + *.sink.file.class=org.apache.hadoop.metrics2.sink.FileSink + *.period=10 + *.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31 + *.sink.ganglia.period=10 + *.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both + *.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40 + + namenode.sink.ganglia.servers=192.168.1.50:8649 + datanode.sink.ganglia.servers=192.168.1.50:8649 + resourcemanager.sink.ganglia.servers=192.168.1.50:8649 + nodemanager.sink.ganglia.servers=192.168.1.50:8649 + + maptask.sink.ganglia.servers=192.168.1.50:8649 + reducetask.sink.ganglia.servers=192.168.1.50:8649 + 所有的servers都修改为安装为gmetad的机器IP（Spark-Master），保存完以后将配置文件分发到各个Slave节点的${HADOOP_HOME}/etc/hadoop目录下，重启Hadoop集群即可。 + +效果： + +![](http://i.imgur.com/z22xV9V.png) + +**dfs.datanode.BlocksRead：** + +![](http://i.imgur.com/gQVoRhh.png) +参考见【8】 +## 1.3.5 配置spark metrics设置，连接ganglia集群 ## + +## （1）配置环境 ## +由于license问题，spark的bin里面没有自带ganglia，需要自己弄 + +方法1： 编译的时候自己加上 +mvn -DskipTests -Pspark-ganglia-lgpl -Phadoop-2.6 -Dhadoop.version=2.6.0 clean package + +方法2：运行的时候用jar形式加入 +--jars lib/spark-ganglia-lgpl_2.10-x.x.x.jar ... + + +## （2）设置spark metrics ## +自行部署spark集群，编辑$SPARK_HOME/conf/metrics.properties文件（没有可以拷贝metrics.properties.template），添加以下内容： +复制代码 + + *.sink.ganglia.class=org.apache.spark.metrics.sink.GangliaSink + *.sink.ganglia.host=spmaster + *.sink.ganglia.port=8649 + *.sink.ganglia.period=10 + *.sink.ganglia.unit=seconds + *.sink.ganglia.ttl=1 + *.sink.ganglia.mode=unicast + *.sink.ganglia.name=unspecified + + master.source.jvm.class=org.apache.spark.metrics.source.JvmSource + worker.source.jvm.class=org.apache.spark.metrics.source.JvmSource + driver.source.jvm.class=org.apache.spark.metrics.source.JvmSource + executor.source.jvm.class=org.apache.spark.metrics.source.JvmSource + +更改后scp到其它spark节点，启动spark集群。 + +**(3) 启动服务进行监控** +启动ganglia的gmetad,gmond以及httpd服务： + + service gmond start + service gmetad start + service httpd start + +参考见【9】 + + + + +### 效果： ### + +![](http://i.imgur.com/Lz1CUam.png) + + + +参考 + + 【1】http://spark.apache.org/ + 【2】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【3】https://github.com/xubo245/SparkLearning + 【4】http://ganglia.info/ + 【5】http://www.cnblogs.com/ylks/p/4244532.html + 【6】http://www.linuxidc.com/Linux/2014-08/105838.htm + 【7】http://www.linuxidc.com/Linux/2014-08/105838p2.htm + 【8】http://blog.sina.com.cn/s/blog_8fd85b850102vv5o.html + 【9】http://www.cnblogs.com/czm1032851561/p/5869891.html + 【10】https://github.com/ganglia/ganglia-web/wiki#Installation + diff --git "a/docs/gangliaLearning/ganglia\345\255\246\344\271\2401\344\271\213\346\250\241\347\211\210.md" "b/docs/gangliaLearning/ganglia\345\255\246\344\271\2401\344\271\213\346\250\241\347\211\210.md" new file mode 100644 index 0000000..4747285 --- /dev/null +++ "b/docs/gangliaLearning/ganglia\345\255\246\344\271\2401\344\271\213\346\250\241\347\211\210.md" @@ -0,0 +1,25 @@ + +更多代码请见：https://github.com/xubo245/SparkLearning + +spark源码解读系列环境：spark-1.5.2、hadoop-2.6.0、scala-2.10.4，ganglia-3.6.1 + +# 1.理解 # + + + + +# 2.代码： # + + + +# 3.结果： # + + + +参考 + + 【1】http://spark.apache.org/ + 【2】http://spark.apache.org/docs/1.5.2/programming-guide.html + 【3】https://github.com/xubo245/SparkLearning + 【4】http://ganglia.info/ + diff --git "a/docs/spark Jar\344\270\213\350\275\275\345\234\260\345\235\200.txt" "b/docs/spark Jar\344\270\213\350\275\275\345\234\260\345\235\200.txt" new file mode 100644 index 0000000..6c4d38e --- /dev/null +++ "b/docs/spark Jar\344\270\213\350\275\275\345\234\260\345\235\200.txt" @@ -0,0 +1,3 @@ +jarأ +ѩ 22:33:51 +http://pan.baidu.com/s/1hrSxiDI diff --git a/src/main/scala/org/apache/spark/ml/MainConcepts/EstimatorTransformerAndParamFunSuite.scala b/src/main/scala/org/apache/spark/ml/MainConcepts/EstimatorTransformerAndParamFunSuite.scala new file mode 100644 index 0000000..174edb0 --- /dev/null +++ b/src/main/scala/org/apache/spark/ml/MainConcepts/EstimatorTransformerAndParamFunSuite.scala @@ -0,0 +1,80 @@ +/** + * @author xubo + * ref:http://spark.apache.org/docs/1.5.2/mllib-guide.html + * more code:https://github.com/xubo245/SparkLearning + * more blog:http://blog.csdn.net/xubo245 + */ +package org.apache.spark.ml.MainConcepts + +import org.apache.spark.util.SparkLearningFunSuite + +/** + * Created by xubo on 2016/6/13. + */ +class EstimatorTransformerAndParamFunSuite extends SparkLearningFunSuite { + test("testFunSuite") { + + import org.apache.spark.ml.classification.LogisticRegression + import org.apache.spark.ml.param.ParamMap + import org.apache.spark.mllib.linalg.{Vector, Vectors} + import org.apache.spark.sql.Row + + // Prepare training data from a list of (label, features) tuples. + val training = sqlContext.createDataFrame(Seq( + (1.0, Vectors.dense(0.0, 1.1, 0.1)), + (0.0, Vectors.dense(2.0, 1.0, -1.0)), + (0.0, Vectors.dense(2.0, 1.3, 1.0)), + (1.0, Vectors.dense(0.0, 1.2, -0.5)) + )).toDF("label", "features") + + // Create a LogisticRegression instance. This instance is an Estimator. + val lr = new LogisticRegression() + // Print out the parameters, documentation, and any default values. + println("LogisticRegression parameters:\n" + lr.explainParams() + "\n") + + // We may set parameters using setter methods. + lr.setMaxIter(10) + .setRegParam(0.01) + + // Learn a LogisticRegression model. This uses the parameters stored in lr. + val model1 = lr.fit(training) + // Since model1 is a Model (i.e., a Transformer produced by an Estimator), + // we can view the parameters it used during fit(). + // This prints the parameter (name: value) pairs, where names are unique IDs for this + // LogisticRegression instance. + println("Model 1 was fit using parameters: " + model1.parent.extractParamMap) + + // We may alternatively specify parameters using a ParamMap, + // which supports several methods for specifying parameters. + val paramMap = ParamMap(lr.maxIter -> 20) + .put(lr.maxIter, 30) // Specify 1 Param. This overwrites the original maxIter. + .put(lr.regParam -> 0.1, lr.threshold -> 0.55) // Specify multiple Params. + + // One can also combine ParamMaps. + val paramMap2 = ParamMap(lr.probabilityCol -> "myProbability") // Change output column name + val paramMapCombined = paramMap ++ paramMap2 + + // Now learn a new model using the paramMapCombined parameters. + // paramMapCombined overrides all parameters set earlier via lr.set* methods. + val model2 = lr.fit(training, paramMapCombined) + println("Model 2 was fit using parameters: " + model2.parent.extractParamMap) + + // Prepare test data. + val test = sqlContext.createDataFrame(Seq( + (1.0, Vectors.dense(-1.0, 1.5, 1.3)), + (0.0, Vectors.dense(3.0, 2.0, -0.1)), + (1.0, Vectors.dense(0.0, 2.2, -1.5)) + )).toDF("label", "features") + + // Make predictions on test data using the Transformer.transform() method. + // LogisticRegression.transform will only use the 'features' column. + // Note that model2.transform() outputs a 'myProbability' column instead of the usual + // 'probability' column since we renamed the lr.probabilityCol parameter previously. + model2.transform(test) + .select("features", "label", "myProbability", "prediction") + .collect() + .foreach { case Row(features: Vector, label: Double, prob: Vector, prediction: Double) => + println(s"($features, $label) -> prob=$prob, prediction=$prediction") + } + } +} diff --git a/src/main/scala/org/apache/spark/ml/MainConcepts/PipelineFunSuite.scala b/src/main/scala/org/apache/spark/ml/MainConcepts/PipelineFunSuite.scala new file mode 100644 index 0000000..4bafde3 --- /dev/null +++ b/src/main/scala/org/apache/spark/ml/MainConcepts/PipelineFunSuite.scala @@ -0,0 +1,65 @@ +/** + * @author xubo + * ref:http://spark.apache.org/docs/1.5.2/mllib-guide.html + * more code:https://github.com/xubo245/SparkLearning + * more blog:http://blog.csdn.net/xubo245 + */ +package org.apache.spark.ml.MainConcepts + +import org.apache.spark.util.SparkLearningFunSuite + +/** + * Created by xubo on 2016/6/13. + */ +class PipelineFunSuite extends SparkLearningFunSuite { + test("testFunSuite") { + + import org.apache.spark.ml.Pipeline + import org.apache.spark.ml.classification.LogisticRegression + import org.apache.spark.ml.feature.{HashingTF, Tokenizer} + import org.apache.spark.mllib.linalg.Vector + import org.apache.spark.sql.Row + + // Prepare training documents from a list of (id, text, label) tuples. + val training = sqlContext.createDataFrame(Seq( + (0L, "a b c d e spark", 1.0), + (1L, "b d", 0.0), + (2L, "spark f g h", 1.0), + (3L, "hadoop mapreduce", 0.0) + )).toDF("id", "text", "label") + + // Configure an ML pipeline, which consists of three stages: tokenizer, hashingTF, and lr. + val tokenizer = new Tokenizer() + .setInputCol("text") + .setOutputCol("words") + val hashingTF = new HashingTF() + .setNumFeatures(1000) + .setInputCol(tokenizer.getOutputCol) + .setOutputCol("features") + val lr = new LogisticRegression() + .setMaxIter(10) + .setRegParam(0.01) + val pipeline = new Pipeline() + .setStages(Array(tokenizer, hashingTF, lr)) + + // Fit the pipeline to training documents. + val model = pipeline.fit(training) + + // Prepare test documents, which are unlabeled (id, text) tuples. + val test = sqlContext.createDataFrame(Seq( + (4L, "spark i j k"), + (5L, "l m n"), + (6L, "mapreduce spark"), + (7L, "apache hadoop") + )).toDF("id", "text") + + // Make predictions on test documents. + model.transform(test) + .select("id", "text", "probability", "prediction") + .collect() + .foreach { case Row(id: Long, text: String, prob: Vector, prediction: Double) => + println(s"($id, $text) --> prob=$prob, prediction=$prediction") + } + + } +} diff --git a/src/main/scala/org/apache/spark/ml/MainConcepts/modelSelectionViaTrainValidationSplitFunSuite.scala b/src/main/scala/org/apache/spark/ml/MainConcepts/modelSelectionViaTrainValidationSplitFunSuite.scala new file mode 100644 index 0000000..5120d62 --- /dev/null +++ b/src/main/scala/org/apache/spark/ml/MainConcepts/modelSelectionViaTrainValidationSplitFunSuite.scala @@ -0,0 +1,63 @@ +/** + * @author xubo + * ref:http://spark.apache.org/docs/1.5.2/mllib-guide.html + * more code:https://github.com/xubo245/SparkLearning + * more blog:http://blog.csdn.net/xubo245 + */ +package org.apache.spark.ml.MainConcepts + +import org.apache.spark.util.SparkLearningFunSuite +import org.apache.spark.ml.evaluation.RegressionEvaluator +import org.apache.spark.ml.regression.LinearRegression +import org.apache.spark.ml.tuning.{ParamGridBuilder, TrainValidationSplit} +import org.apache.spark.mllib.util.MLUtils +import org.apache.spark.sql.SQLContext._ +import org.apache.spark.sql._ +import org.apache.spark.sql.functions._ +import org.apache.spark.sql.types._ +import org.apache.spark.sql.DataFrame + +/** + * Created by xubo on 2016/6/13. + */ +class modelSelectionViaTrainValidationSplitFunSuite extends SparkLearningFunSuite { + test("testFunSuite") { + println("RDD can not tranform to DR") + println("Error:(28, 22) value toDF is not a member of org.apache.spark.rdd.RDD[org.apache.spark.mllib.regression.LabeledPoint]\n val data=data0.toDF()\n ^") + // // Prepare training and test data. + // val data0 = MLUtils.loadLibSVMFile(sc, "file/data/mllib/input/mllibFromSpark/sample_libsvm_data.txt") + // // val data=data0.toDF() + // val Array(training, test) = data.randomSplit(Array(0.9, 0.1), seed = 12345) + // + // val lr = new LinearRegression() + // + // // We use a ParamGridBuilder to construct a grid of parameters to search over. + // // TrainValidationSplit will try all combinations of values and determine best model using + // // the evaluator. + // val paramGrid = new ParamGridBuilder() + // .addGrid(lr.regParam, Array(0.1, 0.01)) + // .addGrid(lr.fitIntercept) + // .addGrid(lr.elasticNetParam, Array(0.0, 0.5, 1.0)) + // .build() + // + // // In this case the estimator is simply the linear regression. + // // A TrainValidationSplit requires an Estimator, a set of Estimator ParamMaps, and an Evaluator. + // val trainValidationSplit = new TrainValidationSplit() + // .setEstimator(lr) + // .setEvaluator(new RegressionEvaluator) + // .setEstimatorParamMaps(paramGrid) + // // 80% of the data will be used for training and the remaining 20% for validation. + // .setTrainRatio(0.8) + // + // // Run train validation split, and choose the best set of parameters. + // val model = trainValidationSplit.fit(training) + // + // // Make predictions on test data. model is the model with combination of parameters + // // that performed best. + // model.transform(test) + // .select("features", "label", "prediction") + // .show() + + + } +} diff --git a/src/main/scala/org/apache/spark/ml/MainConcepts/modelSelectionViaVrossValidationFunSuite.scala b/src/main/scala/org/apache/spark/ml/MainConcepts/modelSelectionViaVrossValidationFunSuite.scala new file mode 100644 index 0000000..6bf4106 --- /dev/null +++ b/src/main/scala/org/apache/spark/ml/MainConcepts/modelSelectionViaVrossValidationFunSuite.scala @@ -0,0 +1,94 @@ +/** + * @author xubo + * ref:http://spark.apache.org/docs/1.5.2/mllib-guide.html + * more code:https://github.com/xubo245/SparkLearning + * more blog:http://blog.csdn.net/xubo245 + */ +package org.apache.spark.ml.MainConcepts + +import org.apache.spark.util.SparkLearningFunSuite + +/** + * Created by xubo on 2016/6/13. + */ +class modelSelectionViaVrossValidationFunSuite extends SparkLearningFunSuite { + test("testFunSuite") { + + + import org.apache.spark.ml.Pipeline + import org.apache.spark.ml.classification.LogisticRegression + import org.apache.spark.ml.evaluation.BinaryClassificationEvaluator + import org.apache.spark.ml.feature.{HashingTF, Tokenizer} + import org.apache.spark.ml.tuning.{ParamGridBuilder, CrossValidator} + import org.apache.spark.mllib.linalg.Vector + import org.apache.spark.sql.Row + + // Prepare training data from a list of (id, text, label) tuples. + val training = sqlContext.createDataFrame(Seq( + (0L, "a b c d e spark", 1.0), + (1L, "b d", 0.0), + (2L, "spark f g h", 1.0), + (3L, "hadoop mapreduce", 0.0), + (4L, "b spark who", 1.0), + (5L, "g d a y", 0.0), + (6L, "spark fly", 1.0), + (7L, "was mapreduce", 0.0), + (8L, "e spark program", 1.0), + (9L, "a e c l", 0.0), + (10L, "spark compile", 1.0), + (11L, "hadoop software", 0.0) + )).toDF("id", "text", "label") + + // Configure an ML pipeline, which consists of three stages: tokenizer, hashingTF, and lr. + val tokenizer = new Tokenizer() + .setInputCol("text") + .setOutputCol("words") + val hashingTF = new HashingTF() + .setInputCol(tokenizer.getOutputCol) + .setOutputCol("features") + val lr = new LogisticRegression() + .setMaxIter(10) + val pipeline = new Pipeline() + .setStages(Array(tokenizer, hashingTF, lr)) + + // We use a ParamGridBuilder to construct a grid of parameters to search over. + // With 3 values for hashingTF.numFeatures and 2 values for lr.regParam, + // this grid will have 3 x 2 = 6 parameter settings for CrossValidator to choose from. + val paramGrid = new ParamGridBuilder() + .addGrid(hashingTF.numFeatures, Array(10, 100, 1000)) + .addGrid(lr.regParam, Array(0.1, 0.01)) + .build() + + // We now treat the Pipeline as an Estimator, wrapping it in a CrossValidator instance. + // This will allow us to jointly choose parameters for all Pipeline stages. + // A CrossValidator requires an Estimator, a set of Estimator ParamMaps, and an Evaluator. + // Note that the evaluator here is a BinaryClassificationEvaluator and its default metric + // is areaUnderROC. + val cv = new CrossValidator() + .setEstimator(pipeline) + .setEvaluator(new BinaryClassificationEvaluator) + .setEstimatorParamMaps(paramGrid) + .setNumFolds(2) // Use 3+ in practice + + // Run cross-validation, and choose the best set of parameters. + val cvModel = cv.fit(training) + + // Prepare test documents, which are unlabeled (id, text) tuples. + val test = sqlContext.createDataFrame(Seq( + (4L, "spark i j k"), + (5L, "l m n"), + (6L, "mapreduce spark"), + (7L, "apache hadoop") + )).toDF("id", "text") + + // Make predictions on test documents. cvModel uses the best model found (lrModel). + cvModel.transform(test) + .select("id", "text", "probability", "prediction") + .collect() + .foreach { case Row(id: Long, text: String, prob: Vector, prediction: Double) => + println(s"($id, $text) --> prob=$prob, prediction=$prediction") + } + + + } +} diff --git a/src/main/scala/org/apache/spark/mllib/basic/breeze/BreezeLearningSuite.scala b/src/main/scala/org/apache/spark/mllib/basic/breeze/BreezeLearningSuite.scala deleted file mode 100644 index 17989bf..0000000 --- a/src/main/scala/org/apache/spark/mllib/basic/breeze/BreezeLearningSuite.scala +++ /dev/null @@ -1,48 +0,0 @@ -package org.apache.spark.mllib.basic.breeze - - -import java.util.Random - -import org.apache.log4j.{Level, Logger} -import org.apache.spark.mllib.linalg.DenseMatrix -import org.apache.spark.util.SparkLearningFunSuite - -/** - * ref:Spark MLlib 机器学习 算法、源码及实战详解 - * Created by xingyun.xb on 2016/7/24. - */ -class BreezeLearningSuite extends SparkLearningFunSuite { - test("mllib linalg Suite") { - Logger.getLogger("org.apache.spark").setLevel(Level.ERROR) - Logger.getLogger("org.eclipse.jetty.server").setLevel(Level.ERROR) - val dm1 = DenseMatrix.zeros(2, 3) - println(dm1) - - println() - val dm2 = DenseMatrix.ones(2, 3) - println(dm2) - - println() - println(DenseMatrix.eye(3)) - - println() - println(DenseMatrix.eye(6)) - - println() - println(DenseMatrix.eye(10)) - - - println() - // val rng = mock[Random] - - println(DenseMatrix.rand(2, 3, new Random(10))) - - // DenseVector - - } - -// test("Breeze suite"){ -// DenseMatrix -// } - -} diff --git a/src/main/scala/org/apache/spark/mllib/basic/breeze/MarticsRandTest.scala b/src/main/scala/org/apache/spark/mllib/basic/breeze/MarticsRandTest.scala deleted file mode 100644 index a1fd641..0000000 --- a/src/main/scala/org/apache/spark/mllib/basic/breeze/MarticsRandTest.scala +++ /dev/null @@ -1,30 +0,0 @@ -package org.apache.spark.mllib.basic.breeze - -import java.util.Random - -import org.apache.spark.mllib.linalg.{Matrices, Matrix} - -/** - * Created by xingyun.xb on 2016/8/7. - */ -object MarticsRandTest { - def main(args: Array[String]) { - val rng: Random = new Random(24) - val r: Matrix = Matrices.rand(3, 4, rng) - println(r) - for (i <- 1 to 10) { - println(i + ":" + rng.nextDouble()) - } - var rng1: Random = new Random(10) - for (i <- 1 to 10) { - println(i + ":" + rng1.nextDouble()) - } - rng1 = new Random() - for (i <- 1 to 10) { - println(i + ":" + rng1.nextDouble()) - } - - - } - -} diff --git a/src/main/scala/org/apache/spark/mllib/basic/breeze/breezeFromScalaNLP/BreezeFromScalaNLPSuite.scala b/src/main/scala/org/apache/spark/mllib/basic/breeze/breezeFromScalaNLP/BreezeFromScalaNLPSuite.scala deleted file mode 100644 index 1876182..0000000 --- a/src/main/scala/org/apache/spark/mllib/basic/breeze/breezeFromScalaNLP/BreezeFromScalaNLPSuite.scala +++ /dev/null @@ -1,209 +0,0 @@ -package org.apache.spark.mllib.basic.breeze.breezeFromScalaNLP - -import breeze.linalg.{diag, DenseMatrix => BDM, DenseVector => BDV} -import breeze.linalg._ -import breeze.numerics._ -import org.apache.log4j.{Level, Logger} -import org.apache.spark.util.SparkLearningFunSuite - -/** - * scalaNLP下面的project breeze的实用方法 - * Created by xingyun.xb on 2016/8/1. - */ -class BreezeFromScalaNLPSuite extends SparkLearningFunSuite { - - test(" create functions: scalaNLP breeze Suite") { - - println() - println(BDM.zeros[Double](2, 3)) - - println() - println(BDV.zeros[Double](2)) - - println() - println(BDV.ones[Double](2)) - - println() - println(BDV.fill[Double](5) { - 3.0 - }) - - println() - println(BDV.range(3, 10)) - - println() - println(BDV.range(3, 10, 2)) - - println() - println(BDM.eye[Double](5)) - - diag - println() - println(diag(BDV(1.0, 2, 3))) - - println() - println(BDM((1.0, 2.0), (3.0, 4.0))) - - println() - println(BDV(1, 2, 3, 10)) - - println() - println(BDV(1, 2, 3, 10).t) - - println() - println(BDV.tabulate(6) { i => 4 * i }) - - println() - println(BDM.tabulate(6, 8) { case (i, j) => i + j }) - - println() - println(new BDV(Array(1, 2, 3, 4))) - - println() - println(new BDM(2, 3, Array(1, 2, 3, 4, 5, 6))) - - println() - println(BDV.rand(10)) - - println() - println(BDM.rand(3, 10)) - - } - - test(" examples to get data: scalaNLP breeze Suite") { - val bdm = BDM.rand(3, 8) - println(bdm) - println() - println(bdm(::, 1)) - - println() - println(bdm(1, ::)) - - - println() - println(bdm(::, 0 to 1)) - - println() - println(bdm(1, 0 to 1)) - } - - test(" examples to set data: scalaNLP breeze Suite") { - var m = BDM.rand(4, 5) - println(m) - - println() - val n = m.reshape(10, 2) - println(m) - - println() - println(n) - - linePrintln(m.toDenseVector) - - //下三角 - linePrintln(lowerTriangular(m)) - - //下三角 - linePrintln(upperTriangular(m)) - - var x = BDM.rand(5, 5) - linePrintln(x) - linePrintln(diag(x)) - - x(::, 0) := 1.0 - linePrintln(x) - - var y = BDM.rand(5, 5) - linePrintln(y) - linePrintln(BDM.vertcat(x, y)) - linePrintln(BDM.horzcat(x, y)) - - var hc = BDM.horzcat(x, y) - - linePrintln(hc(0, ::)) - - - var a = BDV.rand(5) - var b = BDV.rand(4) - linePrintln(a) - linePrintln(b) - linePrintln(BDV.vertcat(a, b)) - } - - test(" examples to compute data: scalaNLP breeze Suite") { - // var a = BDV.rand(5) - // var b = BDV.rand(4) - var x = BDM.fill(4, 4)(2.0) - var y = BDM.fill(4, 4)(3.0) - linePrintln(x) - linePrintln(y) - linePrintln((x + y)) - linePrintln((x - y)) - linePrintln((x :/ y)) - linePrintln((x :* y)) - linePrintln((x :< y)) - linePrintln((x :== y)) - linePrintln((x :+= 10.0)) - linePrintln((x :*= 2.0)) - linePrintln((x :-= 3.0)) - linePrintln((x :/= 3.0)) - - var a = BDM.rand(4, 4) - var b = BDM.rand(4, 4) - linePrintln(a) - linePrintln(b) - val aa = max(a) - println(aa) - val aa1 = argmax(a) - println(aa1) - // linePrintln(aa) - // linePrintln(argmax(a)) - } - - test(" examples to compute data--sum: scalaNLP breeze Suite") { - // var x = BDM.fill(4, 4)(2.0) - // var y = BDM.fill(4, 4)(3.0) - var x = BDM.rand(4, 4) - var y = BDM.rand(4, 4) - linePrintln(x) - linePrintln(y) - val sumx = sum(x) - println(sumx) - - var sumx1 = sum(x(::, *)) - println(sumx1) - sumx1 = sum(x, Axis._0) - println(sumx1) - - var sumy1 = sum(y(::, *)) - println(sumy1) - sumy1 = sum(y, Axis._0) - println(sumy1) - - println(trace(x)) - println(trace(y)) - - println(accumulate(x(1, ::))) - - - } - - test(" examples to compute data--boolean function: scalaNLP breeze Suite") { - val a = BDV(true, false, true) - val b = BDV(false, true, false) - println(a :& b) - println(a :| b) - println(!a) - - val c = BDV(1.0, 0.0, -2.0) - println(any(c)) - println(all(c)) - } - - def linePrintln(a: Object) { - println() - println(a) - } - - -} diff --git a/src/main/scala/org/apache/spark/mllib/basic/breeze/breezeFromScalaNLP/breezeFromMllibSuite.scala b/src/main/scala/org/apache/spark/mllib/basic/breeze/breezeFromScalaNLP/breezeFromMllibSuite.scala deleted file mode 100644 index 016b6a7..0000000 --- a/src/main/scala/org/apache/spark/mllib/basic/breeze/breezeFromScalaNLP/breezeFromMllibSuite.scala +++ /dev/null @@ -1,19 +0,0 @@ -package org.apache.spark.mllib.basic.breeze.breezeFromScalaNLP - -import breeze.linalg.DenseMatrix -import org.apache.log4j.{Level, Logger} -import org.apache.spark.util.SparkLearningFunSuite - -/** - * ref:Spark MLlib 机器学习 算法、源码及实战详解 - * Created by xingyun.xb on 2016/7/24. - */ -class breezeFromMllibSuite extends SparkLearningFunSuite{ - test("Suite"){ - Logger.getLogger("org.apache.spark").setLevel(Level.ERROR) - Logger.getLogger("org.eclipse.jetty.server").setLevel(Level.ERROR) -// val dm1=DenseMatrix.zeros(2,3) -// println(dm1) - } - -} diff --git a/src/main/scala/org/apache/spark/mllib/basic/breeze/testLinalg/VectorsLearning.scala b/src/main/scala/org/apache/spark/mllib/basic/breeze/testLinalg/VectorsLearning.scala deleted file mode 100644 index d1f776c..0000000 --- a/src/main/scala/org/apache/spark/mllib/basic/breeze/testLinalg/VectorsLearning.scala +++ /dev/null @@ -1,26 +0,0 @@ -package org.apache.spark.mllib.basic.breeze.testLinalg - -import org.apache.spark.annotation.Since - -/** - * Created by xingyun.xb on 2016/7/30. - */ -object VectorsLearning { - -} - -//class DenseVector @Since("1.0.0") (){ -// -// /** -// * Generate a `DenseMatrix` consisting of zeros. -// * @param numRows number of rows of the matrix -// * @param numCols number of columns of the matrix -// * @return `DenseMatrix` with size `numRows` x `numCols` and values of zeros -// */ -// @Since("1.3.0") -// def zeros(numRows: Int, numCols: Int): DenseMatrix = { -// require(numRows.toLong * numCols <= Int.MaxValue, -// s"$numRows x $numCols dense matrix is too large to allocate") -// new DenseMatrix(numRows, numCols, new Array[Double](numRows * numCols)) -// } -//} diff --git a/src/main/scala/org/apache/spark/mllib/basic/breezeLearning/breezeFromScalaNLP/BreezeFromScalaNLPSuite.scala b/src/main/scala/org/apache/spark/mllib/basic/breezeLearning/breezeFromScalaNLP/BreezeFromScalaNLPSuite.scala index 2457d49..178ba65 100644 --- a/src/main/scala/org/apache/spark/mllib/basic/breezeLearning/breezeFromScalaNLP/BreezeFromScalaNLPSuite.scala +++ b/src/main/scala/org/apache/spark/mllib/basic/breezeLearning/breezeFromScalaNLP/BreezeFromScalaNLPSuite.scala @@ -12,6 +12,13 @@ import org.apache.spark.util.SparkLearningFunSuite */ class BreezeFromScalaNLPSuite extends SparkLearningFunSuite { + test("bv add"){ + val bd=new BDV(Array(1, 2, 3, 4)) + print(bd) + println(bd.+(bd)) + println(bd.+=(bd)) + + } test(" create functions: scalaNLP breeze Suite") { println() @@ -22,7 +29,6 @@ class BreezeFromScalaNLPSuite extends SparkLearningFunSuite { println() println(BDV.ones[Double](2)) - println() println(BDV.fill[Double](5) { 3.0 @@ -64,6 +70,7 @@ class BreezeFromScalaNLPSuite extends SparkLearningFunSuite { println() println(BDV.rand(10)) + BDV.rand(10) println() println(BDM.rand(3, 10)) diff --git a/src/main/scala/org/apache/spark/rdd/PartitionerSparkPackageTest.scala b/src/main/scala/org/apache/spark/rdd/PartitionerSparkPackageTest.scala new file mode 100644 index 0000000..ba49661 --- /dev/null +++ b/src/main/scala/org/apache/spark/rdd/PartitionerSparkPackageTest.scala @@ -0,0 +1,39 @@ +package org.apache.spark.rdd + +import org.apache.spark.util.SparkLearningFunSuite + +/** + * Created by xubo on 2016/10/8. + */ +class PartitionerSparkPackageTest extends SparkLearningFunSuite { + + test("rdd partitions_") { + var rdd = sc.parallelize(Array(11, 12, 13, 14, 15), 2) + println(rdd.partitions.size) + + /** + * 没有成功，不是subclass + */ + // val test1=rdd.getPartitions + // test1.foreach(println) + } + + test("rdd partition id") { + val RDD = sc.parallelize(Array(11, 12, 13, 14, 15), 2) + RDD.partitions.foreach { each => + println(each) + println("\t" + each.index) + println("\t" + each.hashCode()) + } + RDD.foreachPartition { each => + println("***" + each.size + "***") + each.foreach(println) + println(each.length) + + } + + //debug 可以在RDDzhong的partitions_看到partition数据，但是是private，无法调用 + // RDD.partitions_ + } + +} diff --git a/src/main/scala/org/apache/spark/rdd/learning/AccumulatorSuite.scala b/src/main/scala/org/apache/spark/rdd/learning/AccumulatorSuite.scala new file mode 100644 index 0000000..3952376 --- /dev/null +++ b/src/main/scala/org/apache/spark/rdd/learning/AccumulatorSuite.scala @@ -0,0 +1,17 @@ +package org.apache.spark.rdd.learning + +import org.apache.spark.util.SparkLearningFunSuite + +/** + * Created by xubo on 2017/3/20. + */ +class AccumulatorSuite extends SparkLearningFunSuite{ + + test(" Accumulator test"){ + val accum = sc.accumulator(0, "My Accumulator") + sc.parallelize(Array(1, 2, 3, 4)).foreach(x => accum += x) + println(accum.value) + assert(accum.value===10) + } + +} diff --git a/src/main/scala/org/apache/spark/rdd/learning/RDDTopSuite.scala b/src/main/scala/org/apache/spark/rdd/learning/RDDTopSuite.scala new file mode 100644 index 0000000..8e9d4f3 --- /dev/null +++ b/src/main/scala/org/apache/spark/rdd/learning/RDDTopSuite.scala @@ -0,0 +1,56 @@ +package org.apache.spark.rdd.learning + +import org.apache.log4j.{Level, Logger} +import org.apache.spark.rdd.learning.RDDTopSuite.NameOrdering +import org.apache.spark.util.SparkLearningFunSuite + +import scala.collection.mutable.ArrayBuffer + +/** + * ref:Spark MLlib 机器学习 算法、源码及实战详解 + * Created by xingyun.xb on 2016/7/24. + */ +class RDDTopSuite extends SparkLearningFunSuite { + test("top Suite") { + Logger.getLogger("org.apache.spark").setLevel(Level.ERROR) + Logger.getLogger("org.eclipse.jetty.server").setLevel(Level.ERROR) + + var arr = ArrayBuffer[Person]() + for (i <- 0 to 9) { + arr.append(new Person("name" + i, i)) + } + + val RDD = sc.parallelize(arr) + RDD.foreach(println) + + println("\n\nTop:") + implicit val ord = implicitly[Ordering[Int]] + RDD.top(2)(NameOrdering).foreach { + println + } + } + +} + +class Person extends Serializable { + var name = ""; + var age = 0; + + def this(name: String, age: Int) { + this() + this.name = name + this.age = age + } + + override def toString = s"Person($name, $age)" + +} + +object RDDTopSuite { + + object NameOrdering extends Ordering[Person] { + def compare(a: Person, b: Person): Int = + implicitly[Ordering[Int]].compare(a.age, b.age) + } + +} \ No newline at end of file diff --git a/src/main/scala/org/apache/spark/rdd/learning/partitionByArrayTest.scala b/src/main/scala/org/apache/spark/rdd/learning/partitionByArrayTest.scala new file mode 100644 index 0000000..733801c --- /dev/null +++ b/src/main/scala/org/apache/spark/rdd/learning/partitionByArrayTest.scala @@ -0,0 +1,44 @@ +package org.apache.spark.rdd.learning + +import org.apache.spark.{SparkConf, SparkContext} + +/** + * @author xubo + * 测试contains是否可以包含两个字符串，split可以用array。但是contains的默认是any，所以只能一个字符串 + * 下面是验证的代码，源码可以看SeqLike + */ +object partitionByArrayTest { + + def main(args: Array[String]): Unit = { + val conf = new SparkConf().setAppName("partitionByArrayTest").setMaster("local[4]") + val sc = new SparkContext(conf) + val text1 = sc.textFile("file/data/RDD/input/als/sample_movielens_ratings.txt") + println("text1:") + text1.take(10).foreach(println) + println("contains:") + val con = text1.map { each => if (each.contains("11::")) { + each + } else { + null + } + }.filter(_ != null) + + con.take(10).foreach(println) + println(con.count()) + println("con2:") + val con2 = text1.map { each => + // each.toString.contains + if (each.contains(Array("11::", "80::").toSeq)) { + // if (each.toString.contains(Array("11::", "80::").toSeq)) { + each + } else { + null + } + }.filter(_ != null) + con2.take(10).foreach(println) + println(con2.count()) + + sc.stop() + // rdd1.zip + } +} \ No newline at end of file diff --git a/src/main/scala/org/apache/spark/rdd/learning/rddJoin.scala b/src/main/scala/org/apache/spark/rdd/learning/rddJoin.scala index 23f9832..ed22a46 100644 --- a/src/main/scala/org/apache/spark/rdd/learning/rddJoin.scala +++ b/src/main/scala/org/apache/spark/rdd/learning/rddJoin.scala @@ -14,6 +14,7 @@ object rddJoin { val rdd3 = rdd1.join(rdd2) rdd3.foreach(println) rdd1.zipWithIndex.foreach(println) + var rdd4=rdd1.sortByKey(); // rdd1.zip } // def alignment()=withScope{} diff --git a/src/main/scala/org/apache/spark/sourceCode/partitionerLearning/PartitionerTest.scala b/src/main/scala/org/apache/spark/sourceCode/partitionerLearning/PartitionerTest.scala new file mode 100644 index 0000000..50aaea9 --- /dev/null +++ b/src/main/scala/org/apache/spark/sourceCode/partitionerLearning/PartitionerTest.scala @@ -0,0 +1,59 @@ +package org.apache.spark.sourceCode.partitionerLearning + +import org.apache.spark.util.SparkLearningFunSuite + +/** + * Created by xubo on 2016/10/8. + */ +class PartitionerTest extends SparkLearningFunSuite { + test("URLHashPartitioner") { + var url1 = new URLHashPartitioner(20) + var partition1 = url1.getPartition("https://github.com/xubo245/SparkLearning") + var partition2 = url1.getPartition("https://github.com/xubo245") + println(partition1) + println(partition2) + } + + test("rdd partition size1") { + var rdd = sc.parallelize(Array((11, 21), (12, 22), (13, 29), (14, 24)), 2) + println(rdd.partitions.size) + } + + test("rdd partition size 2") { + var rdd = sc.parallelize(Array((11, 21), (12, 22), (13, 29), (14, 24)), 3) + println(rdd.partitions.size) + } + + test("rdd id") { + var rdd = sc.parallelize(Array((11, 21), (12, 22), (13, 29), (14, 24)), 3) + println(rdd.id) + var rdd2 = sc.parallelize(Array((11, 21), (12, 22), (13, 29), (14, 24)), 5) + println(rdd2.id) + + println("rdd2.name:" + rdd2.name) + rdd2.partitioner.foreach { each => + println(each) + } + rdd2.setName("hello") + println("rdd2.name:" + rdd2.name) + } + + test("rdd partition id") { + val RDD = sc.parallelize(Array(11, 12, 13, 14, 15), 2) + RDD.partitions.foreach { each => + println(each) + println("\t" + each.index) + println("\t" + each.hashCode()) + } + RDD.foreachPartition { each => + println("***" + each.size + "***") + each.foreach(println) + println(each.length) + + } + + //debug 可以在RDDzhong的partitions_看到partition数据，但是是private，无法调用 + // RDD.partitions_ + } + +} diff --git a/src/main/scala/org/apache/spark/sourceCode/partitionerLearning/URLHashPartitioner.scala b/src/main/scala/org/apache/spark/sourceCode/partitionerLearning/URLHashPartitioner.scala new file mode 100644 index 0000000..99b59af --- /dev/null +++ b/src/main/scala/org/apache/spark/sourceCode/partitionerLearning/URLHashPartitioner.scala @@ -0,0 +1,35 @@ +package org.apache.spark.sourceCode.partitionerLearning + +import org.apache.spark.Partitioner + +/** + * Created by xubo on 2016/10/8. + */ +class URLHashPartitioner(numParts: Int) extends Partitioner { + override def numPartitions: Int = numParts + + override def getPartition(key: Any): Int = { + val domain = new java.net.URL(key.toString).getHost() + val code = (domain.hashCode % numPartitions) + if (code < 0) { + code + numPartitions + } else { + code + } + } + + override def equals(other: Any): Boolean = other match { + case url: URLHashPartitioner => + url.numPartitions == numPartitions + case _ => + false + } + + override def hashCode: Int = numPartitions +} + +object URLHashPartitioner { + def main(args: Array[String]) { + + } +} \ No newline at end of file diff --git a/src/main/scala/org/apache/spark/sourceCode/partitionerLearning/reservoirSampleAndCountSuite.scala b/src/main/scala/org/apache/spark/sourceCode/partitionerLearning/reservoirSampleAndCountSuite.scala new file mode 100644 index 0000000..7ef855d --- /dev/null +++ b/src/main/scala/org/apache/spark/sourceCode/partitionerLearning/reservoirSampleAndCountSuite.scala @@ -0,0 +1,31 @@ +package org.apache.spark.sourceCode.partitionerLearning + +import org.apache.spark.util.SparkLearningFunSuite +import org.apache.spark.util.random.SamplingUtils + +import scala.util.Random + +/** + * Created by xubo on 2016/10/9. + */ +class reservoirSampleAndCountSuite extends SparkLearningFunSuite { + test("reservoirSampleAndCount") { + val input = Seq.fill(100)(Random.nextInt()) + val (sample1, count1) = SamplingUtils.reservoirSampleAndCount(input.iterator, 150) + assert(count1 === 100) + assert(input === sample1.toSeq) + + // input size == k + val (sample2, count2) = SamplingUtils.reservoirSampleAndCount(input.iterator, 100) + assert(count2 === 100) + assert(input === sample2.toSeq) + + // input size > k + val (sample3, count3) = SamplingUtils.reservoirSampleAndCount(input.iterator, 10) + assert(count3 === 100) + assert(sample3.length === 10) + println(input) + sample3.foreach{each=>print(each+" ")} + } + +} diff --git a/src/main/scala/org/apache/spark/sql/learning/DataFrame/DFReadOpt.scala b/src/main/scala/org/apache/spark/sql/learning/DataFrame/DFReadOpt.scala index c09e317..138e446 100644 --- a/src/main/scala/org/apache/spark/sql/learning/DataFrame/DFReadOpt.scala +++ b/src/main/scala/org/apache/spark/sql/learning/DataFrame/DFReadOpt.scala @@ -1,38 +1,38 @@ package org.apache.spark.sql.learning.DataFrame -import org.apache.spark.SparkConf -import org.apache.spark.SparkContext +import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.sql.SQLContext object DFReadOpt { + case class Person(name: String, age: Int) + def main(args: Array[String]) { val conf = new SparkConf().setAppName("DataFrame").setMaster("local") val sc = new SparkContext(conf) val sqlContext = new SQLContext(sc) - import sqlContext.implicits._ /** - * produce dataFrame from txt and save as parquet file - */ + * produce dataFrame from txt and save as parquet file + */ // val people = sc.textFile("file/data/examples/src/main/resources/people.txt").map(_.split(",")).map(p => Person(p(0), p(1).trim.toInt)).toDF() // people.registerTempTable("people") // people.write.save("file/data/sql/input/people") /** - * read file from parquet - */ + * read file from parquet + */ val people = sqlContext.read.parquet("file/data/sql/input/people") people.show people.printSchema //To select a column from the data frame, use apply method in Scala and col in Java. val ageCol = people("age") - val groupByAge=people.groupBy("age").count.show - + val groupByAge = people.groupBy("age").count.show + people.registerTempTable("people") - val ageColumn=sqlContext.sql("select age from people") + val ageColumn = sqlContext.sql("select age from people") ageColumn.show println(ageCol); @@ -40,6 +40,21 @@ object DFReadOpt { people.select("age").show people("age") + 10 + println("***************") + val age = people.select("age") + age.show() + val ageArray = age.collect() + val str = new StringBuilder + + ageArray.foreach { each => + println(each) + str.append(each).append(",") + } + val outStr = str.toString() + val result=outStr.substring(0, outStr.length - 1).replace("[","").replace("]","") + println(result) + println("***************") + println("DataFrame end"); sc.stop } diff --git a/src/main/scala/org/apache/spark/sql/learning/DataFrame/dataFrameLearning.scala b/src/main/scala/org/apache/spark/sql/learning/DataFrame/dataFrameLearning.scala new file mode 100644 index 0000000..5594b03 --- /dev/null +++ b/src/main/scala/org/apache/spark/sql/learning/DataFrame/dataFrameLearning.scala @@ -0,0 +1,12 @@ +package org.apache.spark.sql.learning.DataFrame + +import org.apache.spark.util.SparkLearningFunSuite + +/** + * Created by xubo on 2016/11/14. + */ +class dataFrameLearning extends SparkLearningFunSuite { + test("merge dataframe") { + + } +} diff --git a/src/main/scala/org/apache/spark/sql/learning/SparkSQLExamplesFromParquet.scala b/src/main/scala/org/apache/spark/sql/learning/SparkSQLExamplesFromParquet.scala index b019aa8..f3d5f8f 100644 --- a/src/main/scala/org/apache/spark/sql/learning/SparkSQLExamplesFromParquet.scala +++ b/src/main/scala/org/apache/spark/sql/learning/SparkSQLExamplesFromParquet.scala @@ -6,10 +6,7 @@ */ package org.apache.spark.sql.learning -import org.apache.spark.SparkConf -import org.apache.spark.SparkContext -import org.apache.spark.sql.SQLContext -import org.apache.spark.rdd.RDD +import org.apache.spark.{SparkConf, SparkContext} object SparkSQLExamplesFromParquet { def main(args: Array[String]) { @@ -18,8 +15,6 @@ object SparkSQLExamplesFromParquet { val sc = new SparkContext(conf) // val sqlContext = new SQLContext(sc) val sqlContext = new org.apache.spark.sql.SQLContext(sc) - import sqlContext._ - import sqlContext.implicits._ val df = sqlContext.read.load("file/data/examples/src/main/resources/users.parquet") df.show